heap-inl.h revision a55cf41c9d1da7ee8b2f63974dedfb484042dd03
1/* 2 * Copyright (C) 2013 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17#ifndef ART_RUNTIME_GC_HEAP_INL_H_ 18#define ART_RUNTIME_GC_HEAP_INL_H_ 19 20#include "heap.h" 21 22#include "debugger.h" 23#include "gc/space/bump_pointer_space-inl.h" 24#include "gc/space/dlmalloc_space-inl.h" 25#include "gc/space/large_object_space.h" 26#include "gc/space/rosalloc_space-inl.h" 27#include "object_utils.h" 28#include "runtime.h" 29#include "thread.h" 30#include "thread-inl.h" 31#include "verify_object-inl.h" 32 33namespace art { 34namespace gc { 35 36template <bool kInstrumented, bool kCheckLargeObject, typename PreFenceVisitor> 37inline mirror::Object* Heap::AllocObjectWithAllocator(Thread* self, mirror::Class* klass, 38 size_t byte_count, AllocatorType allocator, 39 const PreFenceVisitor& pre_fence_visitor) { 40 DebugCheckPreconditionsForAllocObject(klass, byte_count); 41 // Since allocation can cause a GC which will need to SuspendAll, make sure all allocations are 42 // done in the runnable state where suspension is expected. 43 DCHECK_EQ(self->GetState(), kRunnable); 44 self->AssertThreadSuspensionIsAllowable(); 45 // Need to check that we arent the large object allocator since the large object allocation code 46 // path this function. If we didn't check we would have an infinite loop. 47 if (kCheckLargeObject && UNLIKELY(ShouldAllocLargeObject(klass, byte_count))) { 48 return AllocLargeObject<kInstrumented, PreFenceVisitor>(self, klass, byte_count, 49 pre_fence_visitor); 50 } 51 mirror::Object* obj; 52 AllocationTimer alloc_timer(this, &obj); 53 size_t bytes_allocated, usable_size; 54 obj = TryToAllocate<kInstrumented, false>(self, allocator, byte_count, &bytes_allocated, 55 &usable_size); 56 if (UNLIKELY(obj == nullptr)) { 57 bool is_current_allocator = allocator == GetCurrentAllocator(); 58 obj = AllocateInternalWithGc(self, allocator, byte_count, &bytes_allocated, &usable_size, 59 &klass); 60 if (obj == nullptr) { 61 bool after_is_current_allocator = allocator == GetCurrentAllocator(); 62 if (is_current_allocator && !after_is_current_allocator) { 63 // If the allocator changed, we need to restart the allocation. 64 return AllocObject<kInstrumented>(self, klass, byte_count); 65 } 66 return nullptr; 67 } 68 } 69 DCHECK_GT(bytes_allocated, 0u); 70 DCHECK_GT(usable_size, 0u); 71 obj->SetClass(klass); 72 if (kUseBrooksPointer) { 73 obj->SetBrooksPointer(obj); 74 obj->AssertSelfBrooksPointer(); 75 } 76 pre_fence_visitor(obj, usable_size); 77 if (kIsDebugBuild && Runtime::Current()->IsStarted()) { 78 CHECK_LE(obj->SizeOf(), usable_size); 79 } 80 const size_t new_num_bytes_allocated = 81 static_cast<size_t>(num_bytes_allocated_.FetchAndAdd(bytes_allocated)) + bytes_allocated; 82 // TODO: Deprecate. 83 if (kInstrumented) { 84 if (Runtime::Current()->HasStatsEnabled()) { 85 RuntimeStats* thread_stats = self->GetStats(); 86 ++thread_stats->allocated_objects; 87 thread_stats->allocated_bytes += bytes_allocated; 88 RuntimeStats* global_stats = Runtime::Current()->GetStats(); 89 ++global_stats->allocated_objects; 90 global_stats->allocated_bytes += bytes_allocated; 91 } 92 } else { 93 DCHECK(!Runtime::Current()->HasStatsEnabled()); 94 } 95 if (AllocatorHasAllocationStack(allocator)) { 96 PushOnAllocationStack(self, obj); 97 } 98 if (kInstrumented) { 99 if (Dbg::IsAllocTrackingEnabled()) { 100 Dbg::RecordAllocation(klass, bytes_allocated); 101 } 102 } else { 103 DCHECK(!Dbg::IsAllocTrackingEnabled()); 104 } 105 // concurrent_gc_ isn't known at compile time so we can optimize by not checking it for 106 // the BumpPointer or TLAB allocators. This is nice since it allows the entire if statement to be 107 // optimized out. And for the other allocators, AllocatorMayHaveConcurrentGC is a constant since 108 // the allocator_type should be constant propagated. 109 if (AllocatorMayHaveConcurrentGC(allocator) && concurrent_gc_) { 110 CheckConcurrentGC(self, new_num_bytes_allocated, obj); 111 } 112 VerifyObject(obj); 113 self->VerifyStack(); 114 return obj; 115} 116 117// The size of a thread-local allocation stack in the number of references. 118static constexpr size_t kThreadLocalAllocationStackSize = 128; 119 120inline void Heap::PushOnAllocationStack(Thread* self, mirror::Object* obj) { 121 if (kUseThreadLocalAllocationStack) { 122 bool success = self->PushOnThreadLocalAllocationStack(obj); 123 if (UNLIKELY(!success)) { 124 // Slow path. Allocate a new thread-local allocation stack. 125 mirror::Object** start_address; 126 mirror::Object** end_address; 127 while (!allocation_stack_->AtomicBumpBack(kThreadLocalAllocationStackSize, 128 &start_address, &end_address)) { 129 CollectGarbageInternal(collector::kGcTypeSticky, kGcCauseForAlloc, false); 130 } 131 self->SetThreadLocalAllocationStack(start_address, end_address); 132 // Retry on the new thread-local allocation stack. 133 success = self->PushOnThreadLocalAllocationStack(obj); 134 // Must succeed. 135 CHECK(success); 136 } 137 } else { 138 // This is safe to do since the GC will never free objects which are neither in the allocation 139 // stack or the live bitmap. 140 while (!allocation_stack_->AtomicPushBack(obj)) { 141 CollectGarbageInternal(collector::kGcTypeSticky, kGcCauseForAlloc, false); 142 } 143 } 144} 145 146template <bool kInstrumented, typename PreFenceVisitor> 147inline mirror::Object* Heap::AllocLargeObject(Thread* self, mirror::Class* klass, 148 size_t byte_count, 149 const PreFenceVisitor& pre_fence_visitor) { 150 return AllocObjectWithAllocator<kInstrumented, false, PreFenceVisitor>(self, klass, byte_count, 151 kAllocatorTypeLOS, 152 pre_fence_visitor); 153} 154 155template <const bool kInstrumented, const bool kGrow> 156inline mirror::Object* Heap::TryToAllocate(Thread* self, AllocatorType allocator_type, 157 size_t alloc_size, size_t* bytes_allocated, 158 size_t* usable_size) { 159 if (UNLIKELY(IsOutOfMemoryOnAllocation<kGrow>(allocator_type, alloc_size))) { 160 return nullptr; 161 } 162 mirror::Object* ret; 163 switch (allocator_type) { 164 case kAllocatorTypeBumpPointer: { 165 DCHECK(bump_pointer_space_ != nullptr); 166 alloc_size = RoundUp(alloc_size, space::BumpPointerSpace::kAlignment); 167 ret = bump_pointer_space_->AllocNonvirtual(alloc_size); 168 if (LIKELY(ret != nullptr)) { 169 *bytes_allocated = alloc_size; 170 *usable_size = alloc_size; 171 } 172 break; 173 } 174 case kAllocatorTypeRosAlloc: { 175 if (kInstrumented && UNLIKELY(running_on_valgrind_)) { 176 // If running on valgrind, we should be using the instrumented path. 177 ret = rosalloc_space_->Alloc(self, alloc_size, bytes_allocated, usable_size); 178 } else { 179 DCHECK(!running_on_valgrind_); 180 ret = rosalloc_space_->AllocNonvirtual(self, alloc_size, bytes_allocated, usable_size); 181 } 182 break; 183 } 184 case kAllocatorTypeDlMalloc: { 185 if (kInstrumented && UNLIKELY(running_on_valgrind_)) { 186 // If running on valgrind, we should be using the instrumented path. 187 ret = dlmalloc_space_->Alloc(self, alloc_size, bytes_allocated, usable_size); 188 } else { 189 DCHECK(!running_on_valgrind_); 190 ret = dlmalloc_space_->AllocNonvirtual(self, alloc_size, bytes_allocated, usable_size); 191 } 192 break; 193 } 194 case kAllocatorTypeNonMoving: { 195 ret = non_moving_space_->Alloc(self, alloc_size, bytes_allocated, usable_size); 196 break; 197 } 198 case kAllocatorTypeLOS: { 199 ret = large_object_space_->Alloc(self, alloc_size, bytes_allocated, usable_size); 200 // Note that the bump pointer spaces aren't necessarily next to 201 // the other continuous spaces like the non-moving alloc space or 202 // the zygote space. 203 DCHECK(ret == nullptr || large_object_space_->Contains(ret)); 204 break; 205 } 206 case kAllocatorTypeTLAB: { 207 alloc_size = RoundUp(alloc_size, space::BumpPointerSpace::kAlignment); 208 if (UNLIKELY(self->TlabSize() < alloc_size)) { 209 // Try allocating a new thread local buffer, if the allocaiton fails the space must be 210 // full so return nullptr. 211 if (!bump_pointer_space_->AllocNewTlab(self, alloc_size + kDefaultTLABSize)) { 212 return nullptr; 213 } 214 } 215 // The allocation can't fail. 216 ret = self->AllocTlab(alloc_size); 217 DCHECK(ret != nullptr); 218 *bytes_allocated = alloc_size; 219 break; 220 } 221 default: { 222 LOG(FATAL) << "Invalid allocator type"; 223 ret = nullptr; 224 } 225 } 226 return ret; 227} 228 229inline void Heap::DebugCheckPreconditionsForAllocObject(mirror::Class* c, size_t byte_count) { 230 DCHECK(c == NULL || (c->IsClassClass() && byte_count >= sizeof(mirror::Class)) || 231 (c->IsVariableSize() || c->GetObjectSize() == byte_count) || 232 strlen(ClassHelper(c).GetDescriptor()) == 0); 233 DCHECK_GE(byte_count, sizeof(mirror::Object)); 234} 235 236inline Heap::AllocationTimer::AllocationTimer(Heap* heap, mirror::Object** allocated_obj_ptr) 237 : heap_(heap), allocated_obj_ptr_(allocated_obj_ptr) { 238 if (kMeasureAllocationTime) { 239 allocation_start_time_ = NanoTime() / kTimeAdjust; 240 } 241} 242 243inline Heap::AllocationTimer::~AllocationTimer() { 244 if (kMeasureAllocationTime) { 245 mirror::Object* allocated_obj = *allocated_obj_ptr_; 246 // Only if the allocation succeeded, record the time. 247 if (allocated_obj != nullptr) { 248 uint64_t allocation_end_time = NanoTime() / kTimeAdjust; 249 heap_->total_allocation_time_.FetchAndAdd(allocation_end_time - allocation_start_time_); 250 } 251 } 252}; 253 254inline bool Heap::ShouldAllocLargeObject(mirror::Class* c, size_t byte_count) const { 255 // We need to have a zygote space or else our newly allocated large object can end up in the 256 // Zygote resulting in it being prematurely freed. 257 // We can only do this for primitive objects since large objects will not be within the card table 258 // range. This also means that we rely on SetClass not dirtying the object's card. 259 return byte_count >= kLargeObjectThreshold && have_zygote_space_ && c->IsPrimitiveArray(); 260} 261 262template <bool kGrow> 263inline bool Heap::IsOutOfMemoryOnAllocation(AllocatorType allocator_type, size_t alloc_size) { 264 size_t new_footprint = num_bytes_allocated_ + alloc_size; 265 if (UNLIKELY(new_footprint > max_allowed_footprint_)) { 266 if (UNLIKELY(new_footprint > growth_limit_)) { 267 return true; 268 } 269 if (!AllocatorMayHaveConcurrentGC(allocator_type) || !concurrent_gc_) { 270 if (!kGrow) { 271 return true; 272 } 273 // TODO: Grow for allocation is racy, fix it. 274 VLOG(heap) << "Growing heap from " << PrettySize(max_allowed_footprint_) << " to " 275 << PrettySize(new_footprint) << " for a " << PrettySize(alloc_size) << " allocation"; 276 max_allowed_footprint_ = new_footprint; 277 } 278 } 279 return false; 280} 281 282inline void Heap::CheckConcurrentGC(Thread* self, size_t new_num_bytes_allocated, 283 mirror::Object* obj) { 284 if (UNLIKELY(new_num_bytes_allocated >= concurrent_start_bytes_)) { 285 // The SirtRef is necessary since the calls in RequestConcurrentGC are a safepoint. 286 SirtRef<mirror::Object> ref(self, obj); 287 RequestConcurrentGC(self); 288 } 289} 290 291} // namespace gc 292} // namespace art 293 294#endif // ART_RUNTIME_GC_HEAP_INL_H_ 295