mark_sweep.cc revision e53225c7b8c98f8fc3855fc70f718e7f8abab307
1/* 2 * Copyright (C) 2011 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17#include "mark_sweep.h" 18 19#include <functional> 20#include <numeric> 21#include <climits> 22#include <vector> 23 24#include "base/bounded_fifo.h" 25#include "base/logging.h" 26#include "base/macros.h" 27#include "base/mutex-inl.h" 28#include "base/timing_logger.h" 29#include "gc/accounting/card_table-inl.h" 30#include "gc/accounting/heap_bitmap.h" 31#include "gc/accounting/space_bitmap-inl.h" 32#include "gc/heap.h" 33#include "gc/space/image_space.h" 34#include "gc/space/large_object_space.h" 35#include "gc/space/space-inl.h" 36#include "indirect_reference_table.h" 37#include "intern_table.h" 38#include "jni_internal.h" 39#include "monitor.h" 40#include "mark_sweep-inl.h" 41#include "mirror/art_field.h" 42#include "mirror/art_field-inl.h" 43#include "mirror/class-inl.h" 44#include "mirror/class_loader.h" 45#include "mirror/dex_cache.h" 46#include "mirror/object-inl.h" 47#include "mirror/object_array.h" 48#include "mirror/object_array-inl.h" 49#include "runtime.h" 50#include "thread-inl.h" 51#include "thread_list.h" 52#include "verifier/method_verifier.h" 53 54using ::art::mirror::ArtField; 55using ::art::mirror::Class; 56using ::art::mirror::Object; 57using ::art::mirror::ObjectArray; 58 59namespace art { 60namespace gc { 61namespace collector { 62 63// Performance options. 64constexpr bool kUseRecursiveMark = false; 65constexpr bool kUseMarkStackPrefetch = true; 66constexpr size_t kSweepArrayChunkFreeSize = 1024; 67 68// Parallelism options. 69constexpr bool kParallelCardScan = true; 70constexpr bool kParallelRecursiveMark = true; 71// Don't attempt to parallelize mark stack processing unless the mark stack is at least n 72// elements. This is temporary until we reduce the overhead caused by allocating tasks, etc.. Not 73// having this can add overhead in ProcessReferences since we may end up doing many calls of 74// ProcessMarkStack with very small mark stacks. 75constexpr size_t kMinimumParallelMarkStackSize = 128; 76constexpr bool kParallelProcessMarkStack = true; 77 78// Profiling and information flags. 79constexpr bool kCountClassesMarked = false; 80constexpr bool kProfileLargeObjects = false; 81constexpr bool kMeasureOverhead = false; 82constexpr bool kCountTasks = false; 83constexpr bool kCountJavaLangRefs = false; 84 85// Turn off kCheckLocks when profiling the GC since it slows the GC down by up to 40%. 86constexpr bool kCheckLocks = kDebugLocking; 87 88void MarkSweep::ImmuneSpace(space::ContinuousSpace* space) { 89 // Bind live to mark bitmap if necessary. 90 if (space->GetLiveBitmap() != space->GetMarkBitmap()) { 91 BindLiveToMarkBitmap(space); 92 } 93 94 // Add the space to the immune region. 95 if (immune_begin_ == NULL) { 96 DCHECK(immune_end_ == NULL); 97 SetImmuneRange(reinterpret_cast<Object*>(space->Begin()), 98 reinterpret_cast<Object*>(space->End())); 99 } else { 100 const space::ContinuousSpace* prev_space = nullptr; 101 // Find out if the previous space is immune. 102 for (space::ContinuousSpace* cur_space : GetHeap()->GetContinuousSpaces()) { 103 if (cur_space == space) { 104 break; 105 } 106 prev_space = cur_space; 107 } 108 // If previous space was immune, then extend the immune region. Relies on continuous spaces 109 // being sorted by Heap::AddContinuousSpace. 110 if (prev_space != NULL && 111 immune_begin_ <= reinterpret_cast<Object*>(prev_space->Begin()) && 112 immune_end_ >= reinterpret_cast<Object*>(prev_space->End())) { 113 immune_begin_ = std::min(reinterpret_cast<Object*>(space->Begin()), immune_begin_); 114 immune_end_ = std::max(reinterpret_cast<Object*>(space->End()), immune_end_); 115 } 116 } 117} 118 119void MarkSweep::BindBitmaps() { 120 timings_.StartSplit("BindBitmaps"); 121 WriterMutexLock mu(Thread::Current(), *Locks::heap_bitmap_lock_); 122 // Mark all of the spaces we never collect as immune. 123 for (const auto& space : GetHeap()->GetContinuousSpaces()) { 124 if (space->GetGcRetentionPolicy() == space::kGcRetentionPolicyNeverCollect) { 125 ImmuneSpace(space); 126 } 127 } 128 timings_.EndSplit(); 129} 130 131MarkSweep::MarkSweep(Heap* heap, bool is_concurrent, const std::string& name_prefix) 132 : GarbageCollector(heap, 133 name_prefix + (name_prefix.empty() ? "" : " ") + 134 (is_concurrent ? "concurrent mark sweep": "mark sweep")), 135 current_mark_bitmap_(NULL), 136 java_lang_Class_(NULL), 137 mark_stack_(NULL), 138 immune_begin_(NULL), 139 immune_end_(NULL), 140 soft_reference_list_(NULL), 141 weak_reference_list_(NULL), 142 finalizer_reference_list_(NULL), 143 phantom_reference_list_(NULL), 144 cleared_reference_list_(NULL), 145 gc_barrier_(new Barrier(0)), 146 large_object_lock_("mark sweep large object lock", kMarkSweepLargeObjectLock), 147 mark_stack_expand_lock_("mark sweep mark stack expand lock"), 148 is_concurrent_(is_concurrent), 149 clear_soft_references_(false) { 150} 151 152void MarkSweep::InitializePhase() { 153 timings_.Reset(); 154 base::TimingLogger::ScopedSplit split("InitializePhase", &timings_); 155 mark_stack_ = heap_->mark_stack_.get(); 156 DCHECK(mark_stack_ != nullptr); 157 SetImmuneRange(nullptr, nullptr); 158 soft_reference_list_ = nullptr; 159 weak_reference_list_ = nullptr; 160 finalizer_reference_list_ = nullptr; 161 phantom_reference_list_ = nullptr; 162 cleared_reference_list_ = nullptr; 163 freed_bytes_ = 0; 164 freed_large_object_bytes_ = 0; 165 freed_objects_ = 0; 166 freed_large_objects_ = 0; 167 class_count_ = 0; 168 array_count_ = 0; 169 other_count_ = 0; 170 large_object_test_ = 0; 171 large_object_mark_ = 0; 172 classes_marked_ = 0; 173 overhead_time_ = 0; 174 work_chunks_created_ = 0; 175 work_chunks_deleted_ = 0; 176 reference_count_ = 0; 177 java_lang_Class_ = Class::GetJavaLangClass(); 178 CHECK(java_lang_Class_ != nullptr); 179 180 FindDefaultMarkBitmap(); 181 182 // Do any pre GC verification. 183 timings_.NewSplit("PreGcVerification"); 184 heap_->PreGcVerification(this); 185} 186 187void MarkSweep::ProcessReferences(Thread* self) { 188 WriterMutexLock mu(self, *Locks::heap_bitmap_lock_); 189 ProcessReferences(&soft_reference_list_, clear_soft_references_, &weak_reference_list_, 190 &finalizer_reference_list_, &phantom_reference_list_); 191} 192 193bool MarkSweep::HandleDirtyObjectsPhase() { 194 base::TimingLogger::ScopedSplit split("HandleDirtyObjectsPhase", &timings_); 195 Thread* self = Thread::Current(); 196 Locks::mutator_lock_->AssertExclusiveHeld(self); 197 198 { 199 WriterMutexLock mu(self, *Locks::heap_bitmap_lock_); 200 201 // Re-mark root set. 202 ReMarkRoots(); 203 204 // Scan dirty objects, this is only required if we are not doing concurrent GC. 205 RecursiveMarkDirtyObjects(true, accounting::CardTable::kCardDirty); 206 } 207 208 ProcessReferences(self); 209 210 // Only need to do this if we have the card mark verification on, and only during concurrent GC. 211 if (GetHeap()->verify_missing_card_marks_) { 212 WriterMutexLock mu(self, *Locks::heap_bitmap_lock_); 213 // This second sweep makes sure that we don't have any objects in the live stack which point to 214 // freed objects. These cause problems since their references may be previously freed objects. 215 SweepArray(GetHeap()->allocation_stack_.get(), false); 216 } 217 return true; 218} 219 220bool MarkSweep::IsConcurrent() const { 221 return is_concurrent_; 222} 223 224void MarkSweep::MarkingPhase() { 225 base::TimingLogger::ScopedSplit split("MarkingPhase", &timings_); 226 Heap* heap = GetHeap(); 227 Thread* self = Thread::Current(); 228 229 BindBitmaps(); 230 FindDefaultMarkBitmap(); 231 232 // Process dirty cards and add dirty cards to mod union tables. 233 heap->ProcessCards(timings_); 234 235 // Need to do this before the checkpoint since we don't want any threads to add references to 236 // the live stack during the recursive mark. 237 timings_.NewSplit("SwapStacks"); 238 heap->SwapStacks(); 239 240 WriterMutexLock mu(self, *Locks::heap_bitmap_lock_); 241 if (Locks::mutator_lock_->IsExclusiveHeld(self)) { 242 // If we exclusively hold the mutator lock, all threads must be suspended. 243 MarkRoots(); 244 } else { 245 MarkThreadRoots(self); 246 MarkNonThreadRoots(); 247 } 248 MarkConcurrentRoots(); 249 250 heap->UpdateAndMarkModUnion(this, timings_, GetGcType()); 251 MarkReachableObjects(); 252} 253 254void MarkSweep::MarkThreadRoots(Thread* self) { 255 MarkRootsCheckpoint(self); 256} 257 258void MarkSweep::MarkReachableObjects() { 259 // Mark everything allocated since the last as GC live so that we can sweep concurrently, 260 // knowing that new allocations won't be marked as live. 261 timings_.StartSplit("MarkStackAsLive"); 262 accounting::ObjectStack* live_stack = heap_->GetLiveStack(); 263 heap_->MarkAllocStack(heap_->alloc_space_->GetLiveBitmap(), 264 heap_->large_object_space_->GetLiveObjects(), live_stack); 265 live_stack->Reset(); 266 timings_.EndSplit(); 267 // Recursively mark all the non-image bits set in the mark bitmap. 268 RecursiveMark(); 269} 270 271void MarkSweep::ReclaimPhase() { 272 base::TimingLogger::ScopedSplit split("ReclaimPhase", &timings_); 273 Thread* self = Thread::Current(); 274 275 if (!IsConcurrent()) { 276 base::TimingLogger::ScopedSplit split("ProcessReferences", &timings_); 277 ProcessReferences(self); 278 } else { 279 base::TimingLogger::ScopedSplit split("UnMarkAllocStack", &timings_); 280 accounting::ObjectStack* allocation_stack = GetHeap()->allocation_stack_.get(); 281 WriterMutexLock mu(self, *Locks::heap_bitmap_lock_); 282 // The allocation stack contains things allocated since the start of the GC. These may have been 283 // marked during this GC meaning they won't be eligible for reclaiming in the next sticky GC. 284 // Remove these objects from the mark bitmaps so that they will be eligible for sticky 285 // collection. 286 // There is a race here which is safely handled. Another thread such as the hprof could 287 // have flushed the alloc stack after we resumed the threads. This is safe however, since 288 // reseting the allocation stack zeros it out with madvise. This means that we will either 289 // read NULLs or attempt to unmark a newly allocated object which will not be marked in the 290 // first place. 291 mirror::Object** end = allocation_stack->End(); 292 for (mirror::Object** it = allocation_stack->Begin(); it != end; ++it) { 293 const Object* obj = *it; 294 if (obj != NULL) { 295 UnMarkObjectNonNull(obj); 296 } 297 } 298 } 299 300 // Before freeing anything, lets verify the heap. 301 if (kIsDebugBuild) { 302 ReaderMutexLock mu(self, *Locks::heap_bitmap_lock_); 303 VerifyImageRoots(); 304 } 305 timings_.StartSplit("PreSweepingGcVerification"); 306 heap_->PreSweepingGcVerification(this); 307 timings_.EndSplit(); 308 309 { 310 WriterMutexLock mu(self, *Locks::heap_bitmap_lock_); 311 312 // Reclaim unmarked objects. 313 Sweep(false); 314 315 // Swap the live and mark bitmaps for each space which we modified space. This is an 316 // optimization that enables us to not clear live bits inside of the sweep. Only swaps unbound 317 // bitmaps. 318 timings_.StartSplit("SwapBitmaps"); 319 SwapBitmaps(); 320 timings_.EndSplit(); 321 322 // Unbind the live and mark bitmaps. 323 UnBindBitmaps(); 324 } 325} 326 327void MarkSweep::SetImmuneRange(Object* begin, Object* end) { 328 immune_begin_ = begin; 329 immune_end_ = end; 330} 331 332void MarkSweep::FindDefaultMarkBitmap() { 333 base::TimingLogger::ScopedSplit split("FindDefaultMarkBitmap", &timings_); 334 for (const auto& space : GetHeap()->GetContinuousSpaces()) { 335 if (space->GetGcRetentionPolicy() == space::kGcRetentionPolicyAlwaysCollect) { 336 current_mark_bitmap_ = space->GetMarkBitmap(); 337 CHECK(current_mark_bitmap_ != NULL); 338 return; 339 } 340 } 341 GetHeap()->DumpSpaces(); 342 LOG(FATAL) << "Could not find a default mark bitmap"; 343} 344 345void MarkSweep::ExpandMarkStack() { 346 // Rare case, no need to have Thread::Current be a parameter. 347 MutexLock mu(Thread::Current(), mark_stack_expand_lock_); 348 if (UNLIKELY(mark_stack_->Size() < mark_stack_->Capacity())) { 349 // Someone else acquired the lock and expanded the mark stack before us. 350 return; 351 } 352 std::vector<Object*> temp(mark_stack_->Begin(), mark_stack_->End()); 353 mark_stack_->Resize(mark_stack_->Capacity() * 2); 354 for (const auto& obj : temp) { 355 mark_stack_->PushBack(obj); 356 } 357} 358 359inline void MarkSweep::MarkObjectNonNullParallel(const Object* obj) { 360 DCHECK(obj != NULL); 361 if (MarkObjectParallel(obj)) { 362 while (UNLIKELY(!mark_stack_->AtomicPushBack(const_cast<Object*>(obj)))) { 363 // Only reason a push can fail is that the mark stack is full. 364 ExpandMarkStack(); 365 } 366 } 367} 368 369inline void MarkSweep::UnMarkObjectNonNull(const Object* obj) { 370 DCHECK(!IsImmune(obj)); 371 // Try to take advantage of locality of references within a space, failing this find the space 372 // the hard way. 373 accounting::SpaceBitmap* object_bitmap = current_mark_bitmap_; 374 if (UNLIKELY(!object_bitmap->HasAddress(obj))) { 375 accounting::SpaceBitmap* new_bitmap = heap_->GetMarkBitmap()->GetContinuousSpaceBitmap(obj); 376 if (LIKELY(new_bitmap != NULL)) { 377 object_bitmap = new_bitmap; 378 } else { 379 MarkLargeObject(obj, false); 380 return; 381 } 382 } 383 384 DCHECK(object_bitmap->HasAddress(obj)); 385 object_bitmap->Clear(obj); 386} 387 388inline void MarkSweep::MarkObjectNonNull(const Object* obj) { 389 DCHECK(obj != NULL); 390 391 if (IsImmune(obj)) { 392 DCHECK(IsMarked(obj)); 393 return; 394 } 395 396 // Try to take advantage of locality of references within a space, failing this find the space 397 // the hard way. 398 accounting::SpaceBitmap* object_bitmap = current_mark_bitmap_; 399 if (UNLIKELY(!object_bitmap->HasAddress(obj))) { 400 accounting::SpaceBitmap* new_bitmap = heap_->GetMarkBitmap()->GetContinuousSpaceBitmap(obj); 401 if (LIKELY(new_bitmap != NULL)) { 402 object_bitmap = new_bitmap; 403 } else { 404 MarkLargeObject(obj, true); 405 return; 406 } 407 } 408 409 // This object was not previously marked. 410 if (!object_bitmap->Test(obj)) { 411 object_bitmap->Set(obj); 412 // Do we need to expand the mark stack? 413 if (UNLIKELY(mark_stack_->Size() >= mark_stack_->Capacity())) { 414 ExpandMarkStack(); 415 } 416 // The object must be pushed on to the mark stack. 417 mark_stack_->PushBack(const_cast<Object*>(obj)); 418 } 419} 420 421// Rare case, probably not worth inlining since it will increase instruction cache miss rate. 422bool MarkSweep::MarkLargeObject(const Object* obj, bool set) { 423 // TODO: support >1 discontinuous space. 424 space::LargeObjectSpace* large_object_space = GetHeap()->GetLargeObjectsSpace(); 425 accounting::SpaceSetMap* large_objects = large_object_space->GetMarkObjects(); 426 if (kProfileLargeObjects) { 427 ++large_object_test_; 428 } 429 if (UNLIKELY(!large_objects->Test(obj))) { 430 if (!large_object_space->Contains(obj)) { 431 LOG(ERROR) << "Tried to mark " << obj << " not contained by any spaces"; 432 LOG(ERROR) << "Attempting see if it's a bad root"; 433 VerifyRoots(); 434 LOG(FATAL) << "Can't mark bad root"; 435 } 436 if (kProfileLargeObjects) { 437 ++large_object_mark_; 438 } 439 if (set) { 440 large_objects->Set(obj); 441 } else { 442 large_objects->Clear(obj); 443 } 444 return true; 445 } 446 return false; 447} 448 449inline bool MarkSweep::MarkObjectParallel(const Object* obj) { 450 DCHECK(obj != NULL); 451 452 if (IsImmune(obj)) { 453 DCHECK(IsMarked(obj)); 454 return false; 455 } 456 457 // Try to take advantage of locality of references within a space, failing this find the space 458 // the hard way. 459 accounting::SpaceBitmap* object_bitmap = current_mark_bitmap_; 460 if (UNLIKELY(!object_bitmap->HasAddress(obj))) { 461 accounting::SpaceBitmap* new_bitmap = heap_->GetMarkBitmap()->GetContinuousSpaceBitmap(obj); 462 if (new_bitmap != NULL) { 463 object_bitmap = new_bitmap; 464 } else { 465 // TODO: Remove the Thread::Current here? 466 // TODO: Convert this to some kind of atomic marking? 467 MutexLock mu(Thread::Current(), large_object_lock_); 468 return MarkLargeObject(obj, true); 469 } 470 } 471 472 // Return true if the object was not previously marked. 473 return !object_bitmap->AtomicTestAndSet(obj); 474} 475 476// Used to mark objects when recursing. Recursion is done by moving 477// the finger across the bitmaps in address order and marking child 478// objects. Any newly-marked objects whose addresses are lower than 479// the finger won't be visited by the bitmap scan, so those objects 480// need to be added to the mark stack. 481inline void MarkSweep::MarkObject(const Object* obj) { 482 if (obj != NULL) { 483 MarkObjectNonNull(obj); 484 } 485} 486 487void MarkSweep::MarkRoot(const Object* obj) { 488 if (obj != NULL) { 489 MarkObjectNonNull(obj); 490 } 491} 492 493void MarkSweep::MarkRootParallelCallback(const Object* root, void* arg) { 494 DCHECK(root != NULL); 495 DCHECK(arg != NULL); 496 MarkSweep* mark_sweep = reinterpret_cast<MarkSweep*>(arg); 497 mark_sweep->MarkObjectNonNullParallel(root); 498} 499 500void MarkSweep::MarkObjectCallback(const Object* root, void* arg) { 501 DCHECK(root != NULL); 502 DCHECK(arg != NULL); 503 MarkSweep* mark_sweep = reinterpret_cast<MarkSweep*>(arg); 504 mark_sweep->MarkObjectNonNull(root); 505} 506 507void MarkSweep::ReMarkObjectVisitor(const Object* root, void* arg) { 508 DCHECK(root != NULL); 509 DCHECK(arg != NULL); 510 MarkSweep* mark_sweep = reinterpret_cast<MarkSweep*>(arg); 511 mark_sweep->MarkObjectNonNull(root); 512} 513 514void MarkSweep::VerifyRootCallback(const Object* root, void* arg, size_t vreg, 515 const StackVisitor* visitor) { 516 reinterpret_cast<MarkSweep*>(arg)->VerifyRoot(root, vreg, visitor); 517} 518 519void MarkSweep::VerifyRoot(const Object* root, size_t vreg, const StackVisitor* visitor) { 520 // See if the root is on any space bitmap. 521 if (GetHeap()->GetLiveBitmap()->GetContinuousSpaceBitmap(root) == NULL) { 522 space::LargeObjectSpace* large_object_space = GetHeap()->GetLargeObjectsSpace(); 523 if (!large_object_space->Contains(root)) { 524 LOG(ERROR) << "Found invalid root: " << root; 525 if (visitor != NULL) { 526 LOG(ERROR) << visitor->DescribeLocation() << " in VReg: " << vreg; 527 } 528 } 529 } 530} 531 532void MarkSweep::VerifyRoots() { 533 Runtime::Current()->GetThreadList()->VerifyRoots(VerifyRootCallback, this); 534} 535 536// Marks all objects in the root set. 537void MarkSweep::MarkRoots() { 538 timings_.StartSplit("MarkRoots"); 539 Runtime::Current()->VisitNonConcurrentRoots(MarkObjectCallback, this); 540 timings_.EndSplit(); 541} 542 543void MarkSweep::MarkNonThreadRoots() { 544 timings_.StartSplit("MarkNonThreadRoots"); 545 Runtime::Current()->VisitNonThreadRoots(MarkObjectCallback, this); 546 timings_.EndSplit(); 547} 548 549void MarkSweep::MarkConcurrentRoots() { 550 timings_.StartSplit("MarkConcurrentRoots"); 551 // Visit all runtime roots and clear dirty flags. 552 Runtime::Current()->VisitConcurrentRoots(MarkObjectCallback, this, false, true); 553 timings_.EndSplit(); 554} 555 556void MarkSweep::CheckObject(const Object* obj) { 557 DCHECK(obj != NULL); 558 VisitObjectReferences(obj, [this](const Object* obj, const Object* ref, MemberOffset offset, 559 bool is_static) NO_THREAD_SAFETY_ANALYSIS { 560 Locks::heap_bitmap_lock_->AssertSharedHeld(Thread::Current()); 561 CheckReference(obj, ref, offset, is_static); 562 }); 563} 564 565void MarkSweep::VerifyImageRootVisitor(Object* root, void* arg) { 566 DCHECK(root != NULL); 567 DCHECK(arg != NULL); 568 MarkSweep* mark_sweep = reinterpret_cast<MarkSweep*>(arg); 569 DCHECK(mark_sweep->heap_->GetMarkBitmap()->Test(root)); 570 mark_sweep->CheckObject(root); 571} 572 573void MarkSweep::BindLiveToMarkBitmap(space::ContinuousSpace* space) { 574 CHECK(space->IsDlMallocSpace()); 575 space::DlMallocSpace* alloc_space = space->AsDlMallocSpace(); 576 accounting::SpaceBitmap* live_bitmap = space->GetLiveBitmap(); 577 accounting::SpaceBitmap* mark_bitmap = alloc_space->mark_bitmap_.release(); 578 GetHeap()->GetMarkBitmap()->ReplaceBitmap(mark_bitmap, live_bitmap); 579 alloc_space->temp_bitmap_.reset(mark_bitmap); 580 alloc_space->mark_bitmap_.reset(live_bitmap); 581} 582 583class ScanObjectVisitor { 584 public: 585 explicit ScanObjectVisitor(MarkSweep* const mark_sweep) ALWAYS_INLINE 586 : mark_sweep_(mark_sweep) {} 587 588 // TODO: Fixme when anotatalysis works with visitors. 589 void operator()(const Object* obj) const ALWAYS_INLINE NO_THREAD_SAFETY_ANALYSIS { 590 if (kCheckLocks) { 591 Locks::mutator_lock_->AssertSharedHeld(Thread::Current()); 592 Locks::heap_bitmap_lock_->AssertExclusiveHeld(Thread::Current()); 593 } 594 mark_sweep_->ScanObject(obj); 595 } 596 597 private: 598 MarkSweep* const mark_sweep_; 599}; 600 601template <bool kUseFinger = false> 602class MarkStackTask : public Task { 603 public: 604 MarkStackTask(ThreadPool* thread_pool, MarkSweep* mark_sweep, size_t mark_stack_size, 605 const Object** mark_stack) 606 : mark_sweep_(mark_sweep), 607 thread_pool_(thread_pool), 608 mark_stack_pos_(mark_stack_size) { 609 // We may have to copy part of an existing mark stack when another mark stack overflows. 610 if (mark_stack_size != 0) { 611 DCHECK(mark_stack != NULL); 612 // TODO: Check performance? 613 std::copy(mark_stack, mark_stack + mark_stack_size, mark_stack_); 614 } 615 if (kCountTasks) { 616 ++mark_sweep_->work_chunks_created_; 617 } 618 } 619 620 static const size_t kMaxSize = 1 * KB; 621 622 protected: 623 class ScanObjectParallelVisitor { 624 public: 625 explicit ScanObjectParallelVisitor(MarkStackTask<kUseFinger>* chunk_task) ALWAYS_INLINE 626 : chunk_task_(chunk_task) {} 627 628 void operator()(const Object* obj) const { 629 MarkSweep* mark_sweep = chunk_task_->mark_sweep_; 630 mark_sweep->ScanObjectVisit(obj, 631 [mark_sweep, this](const Object* /* obj */, const Object* ref, 632 const MemberOffset& /* offset */, bool /* is_static */) ALWAYS_INLINE { 633 if (ref != nullptr && mark_sweep->MarkObjectParallel(ref)) { 634 if (kUseFinger) { 635 android_memory_barrier(); 636 if (reinterpret_cast<uintptr_t>(ref) >= 637 static_cast<uintptr_t>(mark_sweep->atomic_finger_)) { 638 return; 639 } 640 } 641 chunk_task_->MarkStackPush(ref); 642 } 643 }); 644 } 645 646 private: 647 MarkStackTask<kUseFinger>* const chunk_task_; 648 }; 649 650 virtual ~MarkStackTask() { 651 // Make sure that we have cleared our mark stack. 652 DCHECK_EQ(mark_stack_pos_, 0U); 653 if (kCountTasks) { 654 ++mark_sweep_->work_chunks_deleted_; 655 } 656 } 657 658 MarkSweep* const mark_sweep_; 659 ThreadPool* const thread_pool_; 660 // Thread local mark stack for this task. 661 const Object* mark_stack_[kMaxSize]; 662 // Mark stack position. 663 size_t mark_stack_pos_; 664 665 void MarkStackPush(const Object* obj) ALWAYS_INLINE { 666 if (UNLIKELY(mark_stack_pos_ == kMaxSize)) { 667 // Mark stack overflow, give 1/2 the stack to the thread pool as a new work task. 668 mark_stack_pos_ /= 2; 669 auto* task = new MarkStackTask(thread_pool_, mark_sweep_, kMaxSize - mark_stack_pos_, 670 mark_stack_ + mark_stack_pos_); 671 thread_pool_->AddTask(Thread::Current(), task); 672 } 673 DCHECK(obj != nullptr); 674 DCHECK(mark_stack_pos_ < kMaxSize); 675 mark_stack_[mark_stack_pos_++] = obj; 676 } 677 678 virtual void Finalize() { 679 delete this; 680 } 681 682 // Scans all of the objects 683 virtual void Run(Thread* self) { 684 ScanObjectParallelVisitor visitor(this); 685 // TODO: Tune this. 686 static const size_t kFifoSize = 4; 687 BoundedFifoPowerOfTwo<const Object*, kFifoSize> prefetch_fifo; 688 for (;;) { 689 const Object* obj = NULL; 690 if (kUseMarkStackPrefetch) { 691 while (mark_stack_pos_ != 0 && prefetch_fifo.size() < kFifoSize) { 692 const Object* obj = mark_stack_[--mark_stack_pos_]; 693 DCHECK(obj != NULL); 694 __builtin_prefetch(obj); 695 prefetch_fifo.push_back(obj); 696 } 697 if (UNLIKELY(prefetch_fifo.empty())) { 698 break; 699 } 700 obj = prefetch_fifo.front(); 701 prefetch_fifo.pop_front(); 702 } else { 703 if (UNLIKELY(mark_stack_pos_ == 0)) { 704 break; 705 } 706 obj = mark_stack_[--mark_stack_pos_]; 707 } 708 DCHECK(obj != NULL); 709 visitor(obj); 710 } 711 } 712}; 713 714class CardScanTask : public MarkStackTask<false> { 715 public: 716 CardScanTask(ThreadPool* thread_pool, MarkSweep* mark_sweep, accounting::SpaceBitmap* bitmap, 717 byte* begin, byte* end, byte minimum_age, size_t mark_stack_size, 718 const Object** mark_stack_obj) 719 : MarkStackTask<false>(thread_pool, mark_sweep, mark_stack_size, mark_stack_obj), 720 bitmap_(bitmap), 721 begin_(begin), 722 end_(end), 723 minimum_age_(minimum_age) { 724 } 725 726 protected: 727 accounting::SpaceBitmap* const bitmap_; 728 byte* const begin_; 729 byte* const end_; 730 const byte minimum_age_; 731 732 virtual void Finalize() { 733 delete this; 734 } 735 736 virtual void Run(Thread* self) NO_THREAD_SAFETY_ANALYSIS { 737 ScanObjectParallelVisitor visitor(this); 738 accounting::CardTable* card_table = mark_sweep_->GetHeap()->GetCardTable(); 739 card_table->Scan(bitmap_, begin_, end_, visitor, minimum_age_); 740 // Finish by emptying our local mark stack. 741 MarkStackTask::Run(self); 742 } 743}; 744 745void MarkSweep::ScanGrayObjects(bool paused, byte minimum_age) { 746 accounting::CardTable* card_table = GetHeap()->GetCardTable(); 747 ThreadPool* thread_pool = GetHeap()->GetThreadPool(); 748 const bool parallel = kParallelCardScan && thread_pool != nullptr; 749 if (parallel) { 750 Thread* self = Thread::Current(); 751 // Can't have a different split for each space since multiple spaces can have their cards being 752 // scanned at the same time. 753 timings_.StartSplit(paused ? "(Paused)ScanGrayObjects" : "ScanGrayObjects"); 754 // Try to take some of the mark stack since we can pass this off to the worker tasks. 755 const Object** mark_stack_begin = const_cast<const Object**>(mark_stack_->Begin()); 756 const Object** mark_stack_end = const_cast<const Object**>(mark_stack_->End()); 757 const size_t mark_stack_size = mark_stack_end - mark_stack_begin; 758 const size_t thread_count = thread_pool->GetThreadCount() + 1; 759 // Estimated number of work tasks we will create. 760 const size_t mark_stack_tasks = GetHeap()->GetContinuousSpaces().size() * thread_count; 761 DCHECK_NE(mark_stack_tasks, 0U); 762 const size_t mark_stack_delta = std::min(CardScanTask::kMaxSize / 2, 763 mark_stack_size / mark_stack_tasks + 1); 764 for (const auto& space : GetHeap()->GetContinuousSpaces()) { 765 byte* card_begin = space->Begin(); 766 byte* card_end = space->End(); 767 // Calculate how many bytes of heap we will scan, 768 const size_t address_range = card_end - card_begin; 769 // Calculate how much address range each task gets. 770 const size_t card_delta = RoundUp(address_range / thread_count + 1, 771 accounting::CardTable::kCardSize); 772 // Create the worker tasks for this space. 773 while (card_begin != card_end) { 774 // Add a range of cards. 775 size_t addr_remaining = card_end - card_begin; 776 size_t card_increment = std::min(card_delta, addr_remaining); 777 // Take from the back of the mark stack. 778 size_t mark_stack_remaining = mark_stack_end - mark_stack_begin; 779 size_t mark_stack_increment = std::min(mark_stack_delta, mark_stack_remaining); 780 mark_stack_end -= mark_stack_increment; 781 mark_stack_->PopBackCount(static_cast<int32_t>(mark_stack_increment)); 782 DCHECK_EQ(mark_stack_end, mark_stack_->End()); 783 // Add the new task to the thread pool. 784 auto* task = new CardScanTask(thread_pool, this, space->GetMarkBitmap(), card_begin, 785 card_begin + card_increment, minimum_age, 786 mark_stack_increment, mark_stack_end); 787 thread_pool->AddTask(self, task); 788 card_begin += card_increment; 789 } 790 } 791 thread_pool->StartWorkers(self); 792 thread_pool->Wait(self, paused, true); // Only do work in the main thread if we are paused. 793 thread_pool->StopWorkers(self); 794 timings_.EndSplit(); 795 } else { 796 for (const auto& space : GetHeap()->GetContinuousSpaces()) { 797 // Image spaces are handled properly since live == marked for them. 798 switch (space->GetGcRetentionPolicy()) { 799 case space::kGcRetentionPolicyNeverCollect: 800 timings_.StartSplit(paused ? "(Paused)ScanGrayImageSpaceObjects" : 801 "ScanGrayImageSpaceObjects"); 802 break; 803 case space::kGcRetentionPolicyFullCollect: 804 timings_.StartSplit(paused ? "(Paused)ScanGrayZygoteSpaceObjects" : 805 "ScanGrayZygoteSpaceObjects"); 806 break; 807 case space::kGcRetentionPolicyAlwaysCollect: 808 timings_.StartSplit(paused ? "(Paused)ScanGrayAllocSpaceObjects" : 809 "ScanGrayAllocSpaceObjects"); 810 break; 811 } 812 ScanObjectVisitor visitor(this); 813 card_table->Scan(space->GetMarkBitmap(), space->Begin(), space->End(), visitor, minimum_age); 814 timings_.EndSplit(); 815 } 816 } 817} 818 819void MarkSweep::VerifyImageRoots() { 820 // Verify roots ensures that all the references inside the image space point 821 // objects which are either in the image space or marked objects in the alloc 822 // space 823 timings_.StartSplit("VerifyImageRoots"); 824 for (const auto& space : GetHeap()->GetContinuousSpaces()) { 825 if (space->IsImageSpace()) { 826 space::ImageSpace* image_space = space->AsImageSpace(); 827 uintptr_t begin = reinterpret_cast<uintptr_t>(image_space->Begin()); 828 uintptr_t end = reinterpret_cast<uintptr_t>(image_space->End()); 829 accounting::SpaceBitmap* live_bitmap = image_space->GetLiveBitmap(); 830 DCHECK(live_bitmap != NULL); 831 live_bitmap->VisitMarkedRange(begin, end, [this](const Object* obj) { 832 if (kCheckLocks) { 833 Locks::heap_bitmap_lock_->AssertSharedHeld(Thread::Current()); 834 } 835 DCHECK(obj != NULL); 836 CheckObject(obj); 837 }); 838 } 839 } 840 timings_.EndSplit(); 841} 842 843class RecursiveMarkTask : public MarkStackTask<false> { 844 public: 845 RecursiveMarkTask(ThreadPool* thread_pool, MarkSweep* mark_sweep, 846 accounting::SpaceBitmap* bitmap, uintptr_t begin, uintptr_t end) 847 : MarkStackTask<false>(thread_pool, mark_sweep, 0, NULL), 848 bitmap_(bitmap), 849 begin_(begin), 850 end_(end) { 851 } 852 853 protected: 854 accounting::SpaceBitmap* const bitmap_; 855 const uintptr_t begin_; 856 const uintptr_t end_; 857 858 virtual void Finalize() { 859 delete this; 860 } 861 862 // Scans all of the objects 863 virtual void Run(Thread* self) NO_THREAD_SAFETY_ANALYSIS { 864 ScanObjectParallelVisitor visitor(this); 865 bitmap_->VisitMarkedRange(begin_, end_, visitor); 866 // Finish by emptying our local mark stack. 867 MarkStackTask::Run(self); 868 } 869}; 870 871// Populates the mark stack based on the set of marked objects and 872// recursively marks until the mark stack is emptied. 873void MarkSweep::RecursiveMark() { 874 base::TimingLogger::ScopedSplit split("RecursiveMark", &timings_); 875 // RecursiveMark will build the lists of known instances of the Reference classes. 876 // See DelayReferenceReferent for details. 877 CHECK(soft_reference_list_ == NULL); 878 CHECK(weak_reference_list_ == NULL); 879 CHECK(finalizer_reference_list_ == NULL); 880 CHECK(phantom_reference_list_ == NULL); 881 CHECK(cleared_reference_list_ == NULL); 882 883 if (kUseRecursiveMark) { 884 const bool partial = GetGcType() == kGcTypePartial; 885 ScanObjectVisitor scan_visitor(this); 886 auto* self = Thread::Current(); 887 ThreadPool* thread_pool = heap_->GetThreadPool(); 888 const bool parallel = kParallelRecursiveMark && thread_pool != NULL; 889 mark_stack_->Reset(); 890 for (const auto& space : GetHeap()->GetContinuousSpaces()) { 891 if ((space->GetGcRetentionPolicy() == space::kGcRetentionPolicyAlwaysCollect) || 892 (!partial && space->GetGcRetentionPolicy() == space::kGcRetentionPolicyFullCollect)) { 893 current_mark_bitmap_ = space->GetMarkBitmap(); 894 if (current_mark_bitmap_ == NULL) { 895 GetHeap()->DumpSpaces(); 896 LOG(FATAL) << "invalid bitmap"; 897 } 898 if (parallel) { 899 // We will use the mark stack the future. 900 // CHECK(mark_stack_->IsEmpty()); 901 // This function does not handle heap end increasing, so we must use the space end. 902 uintptr_t begin = reinterpret_cast<uintptr_t>(space->Begin()); 903 uintptr_t end = reinterpret_cast<uintptr_t>(space->End()); 904 atomic_finger_ = static_cast<int32_t>(0xFFFFFFFF); 905 906 // Create a few worker tasks. 907 size_t n = (thread_pool->GetThreadCount() + 1) * 2; 908 while (begin != end) { 909 uintptr_t start = begin; 910 uintptr_t delta = (end - begin) / n; 911 delta = RoundUp(delta, KB); 912 if (delta < 16 * KB) delta = end - begin; 913 begin += delta; 914 auto* task = new RecursiveMarkTask(thread_pool, this, current_mark_bitmap_, start, 915 begin); 916 thread_pool->AddTask(self, task); 917 } 918 thread_pool->StartWorkers(self); 919 thread_pool->Wait(self, false, true); 920 thread_pool->StopWorkers(self); 921 } else { 922 // This function does not handle heap end increasing, so we must use the space end. 923 uintptr_t begin = reinterpret_cast<uintptr_t>(space->Begin()); 924 uintptr_t end = reinterpret_cast<uintptr_t>(space->End()); 925 current_mark_bitmap_->VisitMarkedRange(begin, end, scan_visitor); 926 } 927 } 928 } 929 } 930 ProcessMarkStack(false); 931} 932 933bool MarkSweep::IsMarkedCallback(const Object* object, void* arg) { 934 return 935 reinterpret_cast<MarkSweep*>(arg)->IsMarked(object) || 936 !reinterpret_cast<MarkSweep*>(arg)->GetHeap()->GetLiveBitmap()->Test(object); 937} 938 939void MarkSweep::RecursiveMarkDirtyObjects(bool paused, byte minimum_age) { 940 ScanGrayObjects(paused, minimum_age); 941 ProcessMarkStack(paused); 942} 943 944void MarkSweep::ReMarkRoots() { 945 timings_.StartSplit("ReMarkRoots"); 946 Runtime::Current()->VisitRoots(ReMarkObjectVisitor, this, true, true); 947 timings_.EndSplit(); 948} 949 950void MarkSweep::SweepJniWeakGlobals(IsMarkedTester is_marked, void* arg) { 951 JavaVMExt* vm = Runtime::Current()->GetJavaVM(); 952 MutexLock mu(Thread::Current(), vm->weak_globals_lock); 953 for (const Object** entry : vm->weak_globals) { 954 if (!is_marked(*entry, arg)) { 955 *entry = kClearedJniWeakGlobal; 956 } 957 } 958} 959 960struct ArrayMarkedCheck { 961 accounting::ObjectStack* live_stack; 962 MarkSweep* mark_sweep; 963}; 964 965// Either marked or not live. 966bool MarkSweep::IsMarkedArrayCallback(const Object* object, void* arg) { 967 ArrayMarkedCheck* array_check = reinterpret_cast<ArrayMarkedCheck*>(arg); 968 if (array_check->mark_sweep->IsMarked(object)) { 969 return true; 970 } 971 accounting::ObjectStack* live_stack = array_check->live_stack; 972 return std::find(live_stack->Begin(), live_stack->End(), object) == live_stack->End(); 973} 974 975void MarkSweep::SweepSystemWeaksArray(accounting::ObjectStack* allocations) { 976 Runtime* runtime = Runtime::Current(); 977 // The callbacks check 978 // !is_marked where is_marked is the callback but we want 979 // !IsMarked && IsLive 980 // So compute !(!IsMarked && IsLive) which is equal to (IsMarked || !IsLive). 981 // Or for swapped (IsLive || !IsMarked). 982 983 timings_.StartSplit("SweepSystemWeaksArray"); 984 ArrayMarkedCheck visitor; 985 visitor.live_stack = allocations; 986 visitor.mark_sweep = this; 987 runtime->GetInternTable()->SweepInternTableWeaks(IsMarkedArrayCallback, &visitor); 988 runtime->GetMonitorList()->SweepMonitorList(IsMarkedArrayCallback, &visitor); 989 SweepJniWeakGlobals(IsMarkedArrayCallback, &visitor); 990 timings_.EndSplit(); 991} 992 993void MarkSweep::SweepSystemWeaks() { 994 Runtime* runtime = Runtime::Current(); 995 // The callbacks check 996 // !is_marked where is_marked is the callback but we want 997 // !IsMarked && IsLive 998 // So compute !(!IsMarked && IsLive) which is equal to (IsMarked || !IsLive). 999 // Or for swapped (IsLive || !IsMarked). 1000 timings_.StartSplit("SweepSystemWeaks"); 1001 runtime->GetInternTable()->SweepInternTableWeaks(IsMarkedCallback, this); 1002 runtime->GetMonitorList()->SweepMonitorList(IsMarkedCallback, this); 1003 SweepJniWeakGlobals(IsMarkedCallback, this); 1004 timings_.EndSplit(); 1005} 1006 1007bool MarkSweep::VerifyIsLiveCallback(const Object* obj, void* arg) { 1008 reinterpret_cast<MarkSweep*>(arg)->VerifyIsLive(obj); 1009 // We don't actually want to sweep the object, so lets return "marked" 1010 return true; 1011} 1012 1013void MarkSweep::VerifyIsLive(const Object* obj) { 1014 Heap* heap = GetHeap(); 1015 if (!heap->GetLiveBitmap()->Test(obj)) { 1016 space::LargeObjectSpace* large_object_space = GetHeap()->GetLargeObjectsSpace(); 1017 if (!large_object_space->GetLiveObjects()->Test(obj)) { 1018 if (std::find(heap->allocation_stack_->Begin(), heap->allocation_stack_->End(), obj) == 1019 heap->allocation_stack_->End()) { 1020 // Object not found! 1021 heap->DumpSpaces(); 1022 LOG(FATAL) << "Found dead object " << obj; 1023 } 1024 } 1025 } 1026} 1027 1028void MarkSweep::VerifySystemWeaks() { 1029 Runtime* runtime = Runtime::Current(); 1030 // Verify system weaks, uses a special IsMarked callback which always returns true. 1031 runtime->GetInternTable()->SweepInternTableWeaks(VerifyIsLiveCallback, this); 1032 runtime->GetMonitorList()->SweepMonitorList(VerifyIsLiveCallback, this); 1033 1034 JavaVMExt* vm = runtime->GetJavaVM(); 1035 MutexLock mu(Thread::Current(), vm->weak_globals_lock); 1036 for (const Object** entry : vm->weak_globals) { 1037 VerifyIsLive(*entry); 1038 } 1039} 1040 1041struct SweepCallbackContext { 1042 MarkSweep* mark_sweep; 1043 space::AllocSpace* space; 1044 Thread* self; 1045}; 1046 1047class CheckpointMarkThreadRoots : public Closure { 1048 public: 1049 explicit CheckpointMarkThreadRoots(MarkSweep* mark_sweep) : mark_sweep_(mark_sweep) {} 1050 1051 virtual void Run(Thread* thread) NO_THREAD_SAFETY_ANALYSIS { 1052 // Note: self is not necessarily equal to thread since thread may be suspended. 1053 Thread* self = Thread::Current(); 1054 CHECK(thread == self || thread->IsSuspended() || thread->GetState() == kWaitingPerformingGc) 1055 << thread->GetState() << " thread " << thread << " self " << self; 1056 thread->VisitRoots(MarkSweep::MarkRootParallelCallback, mark_sweep_); 1057 mark_sweep_->GetBarrier().Pass(self); 1058 } 1059 1060 private: 1061 MarkSweep* mark_sweep_; 1062}; 1063 1064void MarkSweep::MarkRootsCheckpoint(Thread* self) { 1065 CheckpointMarkThreadRoots check_point(this); 1066 timings_.StartSplit("MarkRootsCheckpoint"); 1067 ThreadList* thread_list = Runtime::Current()->GetThreadList(); 1068 // Request the check point is run on all threads returning a count of the threads that must 1069 // run through the barrier including self. 1070 size_t barrier_count = thread_list->RunCheckpoint(&check_point); 1071 // Release locks then wait for all mutator threads to pass the barrier. 1072 // TODO: optimize to not release locks when there are no threads to wait for. 1073 Locks::heap_bitmap_lock_->ExclusiveUnlock(self); 1074 Locks::mutator_lock_->SharedUnlock(self); 1075 ThreadState old_state = self->SetState(kWaitingForCheckPointsToRun); 1076 CHECK_EQ(old_state, kWaitingPerformingGc); 1077 gc_barrier_->Increment(self, barrier_count); 1078 self->SetState(kWaitingPerformingGc); 1079 Locks::mutator_lock_->SharedLock(self); 1080 Locks::heap_bitmap_lock_->ExclusiveLock(self); 1081 timings_.EndSplit(); 1082} 1083 1084void MarkSweep::SweepCallback(size_t num_ptrs, Object** ptrs, void* arg) { 1085 SweepCallbackContext* context = static_cast<SweepCallbackContext*>(arg); 1086 MarkSweep* mark_sweep = context->mark_sweep; 1087 Heap* heap = mark_sweep->GetHeap(); 1088 space::AllocSpace* space = context->space; 1089 Thread* self = context->self; 1090 Locks::heap_bitmap_lock_->AssertExclusiveHeld(self); 1091 // Use a bulk free, that merges consecutive objects before freeing or free per object? 1092 // Documentation suggests better free performance with merging, but this may be at the expensive 1093 // of allocation. 1094 size_t freed_objects = num_ptrs; 1095 // AllocSpace::FreeList clears the value in ptrs, so perform after clearing the live bit 1096 size_t freed_bytes = space->FreeList(self, num_ptrs, ptrs); 1097 heap->RecordFree(freed_objects, freed_bytes); 1098 mark_sweep->freed_objects_.fetch_add(freed_objects); 1099 mark_sweep->freed_bytes_.fetch_add(freed_bytes); 1100} 1101 1102void MarkSweep::ZygoteSweepCallback(size_t num_ptrs, Object** ptrs, void* arg) { 1103 SweepCallbackContext* context = static_cast<SweepCallbackContext*>(arg); 1104 Locks::heap_bitmap_lock_->AssertExclusiveHeld(context->self); 1105 Heap* heap = context->mark_sweep->GetHeap(); 1106 // We don't free any actual memory to avoid dirtying the shared zygote pages. 1107 for (size_t i = 0; i < num_ptrs; ++i) { 1108 Object* obj = static_cast<Object*>(ptrs[i]); 1109 heap->GetLiveBitmap()->Clear(obj); 1110 heap->GetCardTable()->MarkCard(obj); 1111 } 1112} 1113 1114void MarkSweep::SweepArray(accounting::ObjectStack* allocations, bool swap_bitmaps) { 1115 space::DlMallocSpace* space = heap_->GetAllocSpace(); 1116 1117 // If we don't swap bitmaps then newly allocated Weaks go into the live bitmap but not mark 1118 // bitmap, resulting in occasional frees of Weaks which are still in use. 1119 SweepSystemWeaksArray(allocations); 1120 1121 timings_.StartSplit("SweepArray"); 1122 // Newly allocated objects MUST be in the alloc space and those are the only objects which we are 1123 // going to free. 1124 accounting::SpaceBitmap* live_bitmap = space->GetLiveBitmap(); 1125 accounting::SpaceBitmap* mark_bitmap = space->GetMarkBitmap(); 1126 space::LargeObjectSpace* large_object_space = GetHeap()->GetLargeObjectsSpace(); 1127 accounting::SpaceSetMap* large_live_objects = large_object_space->GetLiveObjects(); 1128 accounting::SpaceSetMap* large_mark_objects = large_object_space->GetMarkObjects(); 1129 if (swap_bitmaps) { 1130 std::swap(live_bitmap, mark_bitmap); 1131 std::swap(large_live_objects, large_mark_objects); 1132 } 1133 1134 size_t freed_bytes = 0; 1135 size_t freed_large_object_bytes = 0; 1136 size_t freed_objects = 0; 1137 size_t freed_large_objects = 0; 1138 size_t count = allocations->Size(); 1139 Object** objects = const_cast<Object**>(allocations->Begin()); 1140 Object** out = objects; 1141 Object** objects_to_chunk_free = out; 1142 1143 // Empty the allocation stack. 1144 Thread* self = Thread::Current(); 1145 for (size_t i = 0; i < count; ++i) { 1146 Object* obj = objects[i]; 1147 // There should only be objects in the AllocSpace/LargeObjectSpace in the allocation stack. 1148 if (LIKELY(mark_bitmap->HasAddress(obj))) { 1149 if (!mark_bitmap->Test(obj)) { 1150 // Don't bother un-marking since we clear the mark bitmap anyways. 1151 *(out++) = obj; 1152 // Free objects in chunks. 1153 DCHECK_GE(out, objects_to_chunk_free); 1154 DCHECK_LE(static_cast<size_t>(out - objects_to_chunk_free), kSweepArrayChunkFreeSize); 1155 if (static_cast<size_t>(out - objects_to_chunk_free) == kSweepArrayChunkFreeSize) { 1156 timings_.StartSplit("FreeList"); 1157 size_t chunk_freed_objects = out - objects_to_chunk_free; 1158 freed_objects += chunk_freed_objects; 1159 freed_bytes += space->FreeList(self, chunk_freed_objects, objects_to_chunk_free); 1160 objects_to_chunk_free = out; 1161 timings_.EndSplit(); 1162 } 1163 } 1164 } else if (!large_mark_objects->Test(obj)) { 1165 ++freed_large_objects; 1166 freed_large_object_bytes += large_object_space->Free(self, obj); 1167 } 1168 } 1169 // Free the remaining objects in chunks. 1170 DCHECK_GE(out, objects_to_chunk_free); 1171 DCHECK_LE(static_cast<size_t>(out - objects_to_chunk_free), kSweepArrayChunkFreeSize); 1172 if (out - objects_to_chunk_free > 0) { 1173 timings_.StartSplit("FreeList"); 1174 size_t chunk_freed_objects = out - objects_to_chunk_free; 1175 freed_objects += chunk_freed_objects; 1176 freed_bytes += space->FreeList(self, chunk_freed_objects, objects_to_chunk_free); 1177 timings_.EndSplit(); 1178 } 1179 CHECK_EQ(count, allocations->Size()); 1180 timings_.EndSplit(); 1181 1182 timings_.StartSplit("RecordFree"); 1183 VLOG(heap) << "Freed " << freed_objects << "/" << count 1184 << " objects with size " << PrettySize(freed_bytes); 1185 heap_->RecordFree(freed_objects + freed_large_objects, freed_bytes + freed_large_object_bytes); 1186 freed_objects_.fetch_add(freed_objects); 1187 freed_large_objects_.fetch_add(freed_large_objects); 1188 freed_bytes_.fetch_add(freed_bytes); 1189 freed_large_object_bytes_.fetch_add(freed_large_object_bytes); 1190 timings_.EndSplit(); 1191 1192 timings_.StartSplit("ResetStack"); 1193 allocations->Reset(); 1194 timings_.EndSplit(); 1195} 1196 1197void MarkSweep::Sweep(bool swap_bitmaps) { 1198 DCHECK(mark_stack_->IsEmpty()); 1199 base::TimingLogger::ScopedSplit("Sweep", &timings_); 1200 1201 // If we don't swap bitmaps then newly allocated Weaks go into the live bitmap but not mark 1202 // bitmap, resulting in occasional frees of Weaks which are still in use. 1203 SweepSystemWeaks(); 1204 1205 const bool partial = (GetGcType() == kGcTypePartial); 1206 SweepCallbackContext scc; 1207 scc.mark_sweep = this; 1208 scc.self = Thread::Current(); 1209 for (const auto& space : GetHeap()->GetContinuousSpaces()) { 1210 // We always sweep always collect spaces. 1211 bool sweep_space = (space->GetGcRetentionPolicy() == space::kGcRetentionPolicyAlwaysCollect); 1212 if (!partial && !sweep_space) { 1213 // We sweep full collect spaces when the GC isn't a partial GC (ie its full). 1214 sweep_space = (space->GetGcRetentionPolicy() == space::kGcRetentionPolicyFullCollect); 1215 } 1216 if (sweep_space) { 1217 uintptr_t begin = reinterpret_cast<uintptr_t>(space->Begin()); 1218 uintptr_t end = reinterpret_cast<uintptr_t>(space->End()); 1219 scc.space = space->AsDlMallocSpace(); 1220 accounting::SpaceBitmap* live_bitmap = space->GetLiveBitmap(); 1221 accounting::SpaceBitmap* mark_bitmap = space->GetMarkBitmap(); 1222 if (swap_bitmaps) { 1223 std::swap(live_bitmap, mark_bitmap); 1224 } 1225 if (!space->IsZygoteSpace()) { 1226 base::TimingLogger::ScopedSplit split("SweepAllocSpace", &timings_); 1227 // Bitmaps are pre-swapped for optimization which enables sweeping with the heap unlocked. 1228 accounting::SpaceBitmap::SweepWalk(*live_bitmap, *mark_bitmap, begin, end, 1229 &SweepCallback, reinterpret_cast<void*>(&scc)); 1230 } else { 1231 base::TimingLogger::ScopedSplit split("SweepZygote", &timings_); 1232 // Zygote sweep takes care of dirtying cards and clearing live bits, does not free actual 1233 // memory. 1234 accounting::SpaceBitmap::SweepWalk(*live_bitmap, *mark_bitmap, begin, end, 1235 &ZygoteSweepCallback, reinterpret_cast<void*>(&scc)); 1236 } 1237 } 1238 } 1239 1240 SweepLargeObjects(swap_bitmaps); 1241} 1242 1243void MarkSweep::SweepLargeObjects(bool swap_bitmaps) { 1244 base::TimingLogger::ScopedSplit("SweepLargeObjects", &timings_); 1245 // Sweep large objects 1246 space::LargeObjectSpace* large_object_space = GetHeap()->GetLargeObjectsSpace(); 1247 accounting::SpaceSetMap* large_live_objects = large_object_space->GetLiveObjects(); 1248 accounting::SpaceSetMap* large_mark_objects = large_object_space->GetMarkObjects(); 1249 if (swap_bitmaps) { 1250 std::swap(large_live_objects, large_mark_objects); 1251 } 1252 // O(n*log(n)) but hopefully there are not too many large objects. 1253 size_t freed_objects = 0; 1254 size_t freed_bytes = 0; 1255 Thread* self = Thread::Current(); 1256 for (const Object* obj : large_live_objects->GetObjects()) { 1257 if (!large_mark_objects->Test(obj)) { 1258 freed_bytes += large_object_space->Free(self, const_cast<Object*>(obj)); 1259 ++freed_objects; 1260 } 1261 } 1262 freed_large_objects_.fetch_add(freed_objects); 1263 freed_large_object_bytes_.fetch_add(freed_bytes); 1264 GetHeap()->RecordFree(freed_objects, freed_bytes); 1265} 1266 1267void MarkSweep::CheckReference(const Object* obj, const Object* ref, MemberOffset offset, bool is_static) { 1268 for (const auto& space : GetHeap()->GetContinuousSpaces()) { 1269 if (space->IsDlMallocSpace() && space->Contains(ref)) { 1270 DCHECK(IsMarked(obj)); 1271 1272 bool is_marked = IsMarked(ref); 1273 if (!is_marked) { 1274 LOG(INFO) << *space; 1275 LOG(WARNING) << (is_static ? "Static ref'" : "Instance ref'") << PrettyTypeOf(ref) 1276 << "' (" << reinterpret_cast<const void*>(ref) << ") in '" << PrettyTypeOf(obj) 1277 << "' (" << reinterpret_cast<const void*>(obj) << ") at offset " 1278 << reinterpret_cast<void*>(offset.Int32Value()) << " wasn't marked"; 1279 1280 const Class* klass = is_static ? obj->AsClass() : obj->GetClass(); 1281 DCHECK(klass != NULL); 1282 const ObjectArray<ArtField>* fields = is_static ? klass->GetSFields() : klass->GetIFields(); 1283 DCHECK(fields != NULL); 1284 bool found = false; 1285 for (int32_t i = 0; i < fields->GetLength(); ++i) { 1286 const ArtField* cur = fields->Get(i); 1287 if (cur->GetOffset().Int32Value() == offset.Int32Value()) { 1288 LOG(WARNING) << "Field referencing the alloc space was " << PrettyField(cur); 1289 found = true; 1290 break; 1291 } 1292 } 1293 if (!found) { 1294 LOG(WARNING) << "Could not find field in object alloc space with offset " << offset.Int32Value(); 1295 } 1296 1297 bool obj_marked = heap_->GetCardTable()->IsDirty(obj); 1298 if (!obj_marked) { 1299 LOG(WARNING) << "Object '" << PrettyTypeOf(obj) << "' " 1300 << "(" << reinterpret_cast<const void*>(obj) << ") contains references to " 1301 << "the alloc space, but wasn't card marked"; 1302 } 1303 } 1304 } 1305 break; 1306 } 1307} 1308 1309// Process the "referent" field in a java.lang.ref.Reference. If the 1310// referent has not yet been marked, put it on the appropriate list in 1311// the heap for later processing. 1312void MarkSweep::DelayReferenceReferent(mirror::Class* klass, Object* obj) { 1313 DCHECK(klass != nullptr); 1314 DCHECK(klass->IsReferenceClass()); 1315 DCHECK(obj != NULL); 1316 Object* referent = heap_->GetReferenceReferent(obj); 1317 if (referent != NULL && !IsMarked(referent)) { 1318 if (kCountJavaLangRefs) { 1319 ++reference_count_; 1320 } 1321 Thread* self = Thread::Current(); 1322 // TODO: Remove these locks, and use atomic stacks for storing references? 1323 if (klass->IsSoftReferenceClass()) { 1324 MutexLock mu(self, *heap_->GetSoftRefQueueLock()); 1325 heap_->EnqueuePendingReference(obj, &soft_reference_list_); 1326 } else if (klass->IsWeakReferenceClass()) { 1327 MutexLock mu(self, *heap_->GetWeakRefQueueLock()); 1328 heap_->EnqueuePendingReference(obj, &weak_reference_list_); 1329 } else if (klass->IsFinalizerReferenceClass()) { 1330 MutexLock mu(self, *heap_->GetFinalizerRefQueueLock()); 1331 heap_->EnqueuePendingReference(obj, &finalizer_reference_list_); 1332 } else if (klass->IsPhantomReferenceClass()) { 1333 MutexLock mu(self, *heap_->GetPhantomRefQueueLock()); 1334 heap_->EnqueuePendingReference(obj, &phantom_reference_list_); 1335 } else { 1336 LOG(FATAL) << "Invalid reference type " << PrettyClass(klass) 1337 << " " << std::hex << klass->GetAccessFlags(); 1338 } 1339 } 1340} 1341 1342void MarkSweep::ScanRoot(const Object* obj) { 1343 ScanObject(obj); 1344} 1345 1346class MarkObjectVisitor { 1347 public: 1348 explicit MarkObjectVisitor(MarkSweep* const mark_sweep) ALWAYS_INLINE : mark_sweep_(mark_sweep) {} 1349 1350 // TODO: Fixme when anotatalysis works with visitors. 1351 void operator()(const Object* /* obj */, const Object* ref, const MemberOffset& /* offset */, 1352 bool /* is_static */) const ALWAYS_INLINE 1353 NO_THREAD_SAFETY_ANALYSIS { 1354 if (kCheckLocks) { 1355 Locks::mutator_lock_->AssertSharedHeld(Thread::Current()); 1356 Locks::heap_bitmap_lock_->AssertExclusiveHeld(Thread::Current()); 1357 } 1358 mark_sweep_->MarkObject(ref); 1359 } 1360 1361 private: 1362 MarkSweep* const mark_sweep_; 1363}; 1364 1365// Scans an object reference. Determines the type of the reference 1366// and dispatches to a specialized scanning routine. 1367void MarkSweep::ScanObject(const Object* obj) { 1368 MarkObjectVisitor visitor(this); 1369 ScanObjectVisit(obj, visitor); 1370} 1371 1372void MarkSweep::ProcessMarkStackParallel(bool paused) { 1373 Thread* self = Thread::Current(); 1374 ThreadPool* thread_pool = GetHeap()->GetThreadPool(); 1375 const size_t num_threads = thread_pool->GetThreadCount(); 1376 const size_t chunk_size = 1377 std::min(mark_stack_->Size() / num_threads + 1, 1378 static_cast<size_t>(MarkStackTask<false>::kMaxSize)); 1379 CHECK_GT(chunk_size, 0U); 1380 // Split the current mark stack up into work tasks. 1381 for (mirror::Object **it = mark_stack_->Begin(), **end = mark_stack_->End(); it < end; ) { 1382 const size_t delta = std::min(static_cast<size_t>(end - it), chunk_size); 1383 thread_pool->AddTask(self, new MarkStackTask<false>(thread_pool, this, delta, 1384 const_cast<const mirror::Object**>(it))); 1385 it += delta; 1386 } 1387 thread_pool->StartWorkers(self); 1388 // Don't do work in the main thread since it assumed at least one other thread will require CPU 1389 // time during the GC. 1390 thread_pool->Wait(self, paused, true); 1391 thread_pool->StopWorkers(self); 1392 mark_stack_->Reset(); 1393 CHECK_EQ(work_chunks_created_, work_chunks_deleted_) << " some of the work chunks were leaked"; 1394} 1395 1396// Scan anything that's on the mark stack. 1397void MarkSweep::ProcessMarkStack(bool paused) { 1398 timings_.StartSplit("ProcessMarkStack"); 1399 const bool parallel = kParallelProcessMarkStack && GetHeap()->GetThreadPool() && 1400 mark_stack_->Size() >= kMinimumParallelMarkStackSize; 1401 if (parallel) { 1402 ProcessMarkStackParallel(paused); 1403 } else { 1404 // TODO: Tune this. 1405 static const size_t kFifoSize = 4; 1406 BoundedFifoPowerOfTwo<const Object*, kFifoSize> prefetch_fifo; 1407 for (;;) { 1408 const Object* obj = NULL; 1409 if (kUseMarkStackPrefetch) { 1410 while (!mark_stack_->IsEmpty() && prefetch_fifo.size() < kFifoSize) { 1411 const Object* obj = mark_stack_->PopBack(); 1412 DCHECK(obj != NULL); 1413 __builtin_prefetch(obj); 1414 prefetch_fifo.push_back(obj); 1415 } 1416 if (prefetch_fifo.empty()) { 1417 break; 1418 } 1419 obj = prefetch_fifo.front(); 1420 prefetch_fifo.pop_front(); 1421 } else { 1422 if (mark_stack_->IsEmpty()) { 1423 break; 1424 } 1425 obj = mark_stack_->PopBack(); 1426 } 1427 DCHECK(obj != NULL); 1428 ScanObject(obj); 1429 } 1430 } 1431 timings_.EndSplit(); 1432} 1433 1434// Walks the reference list marking any references subject to the 1435// reference clearing policy. References with a black referent are 1436// removed from the list. References with white referents biased 1437// toward saving are blackened and also removed from the list. 1438void MarkSweep::PreserveSomeSoftReferences(Object** list) { 1439 DCHECK(list != NULL); 1440 Object* clear = NULL; 1441 size_t counter = 0; 1442 1443 DCHECK(mark_stack_->IsEmpty()); 1444 1445 timings_.StartSplit("PreserveSomeSoftReferences"); 1446 while (*list != NULL) { 1447 Object* ref = heap_->DequeuePendingReference(list); 1448 Object* referent = heap_->GetReferenceReferent(ref); 1449 if (referent == NULL) { 1450 // Referent was cleared by the user during marking. 1451 continue; 1452 } 1453 bool is_marked = IsMarked(referent); 1454 if (!is_marked && ((++counter) & 1)) { 1455 // Referent is white and biased toward saving, mark it. 1456 MarkObject(referent); 1457 is_marked = true; 1458 } 1459 if (!is_marked) { 1460 // Referent is white, queue it for clearing. 1461 heap_->EnqueuePendingReference(ref, &clear); 1462 } 1463 } 1464 *list = clear; 1465 timings_.EndSplit(); 1466 1467 // Restart the mark with the newly black references added to the root set. 1468 ProcessMarkStack(true); 1469} 1470 1471inline bool MarkSweep::IsMarked(const Object* object) const 1472 SHARED_LOCKS_REQUIRED(Locks::heap_bitmap_lock_) { 1473 if (IsImmune(object)) { 1474 return true; 1475 } 1476 DCHECK(current_mark_bitmap_ != NULL); 1477 if (current_mark_bitmap_->HasAddress(object)) { 1478 return current_mark_bitmap_->Test(object); 1479 } 1480 return heap_->GetMarkBitmap()->Test(object); 1481} 1482 1483 1484// Unlink the reference list clearing references objects with white 1485// referents. Cleared references registered to a reference queue are 1486// scheduled for appending by the heap worker thread. 1487void MarkSweep::ClearWhiteReferences(Object** list) { 1488 DCHECK(list != NULL); 1489 while (*list != NULL) { 1490 Object* ref = heap_->DequeuePendingReference(list); 1491 Object* referent = heap_->GetReferenceReferent(ref); 1492 if (referent != NULL && !IsMarked(referent)) { 1493 // Referent is white, clear it. 1494 heap_->ClearReferenceReferent(ref); 1495 if (heap_->IsEnqueuable(ref)) { 1496 heap_->EnqueueReference(ref, &cleared_reference_list_); 1497 } 1498 } 1499 } 1500 DCHECK(*list == NULL); 1501} 1502 1503// Enqueues finalizer references with white referents. White 1504// referents are blackened, moved to the zombie field, and the 1505// referent field is cleared. 1506void MarkSweep::EnqueueFinalizerReferences(Object** list) { 1507 DCHECK(list != NULL); 1508 timings_.StartSplit("EnqueueFinalizerReferences"); 1509 MemberOffset zombie_offset = heap_->GetFinalizerReferenceZombieOffset(); 1510 bool has_enqueued = false; 1511 while (*list != NULL) { 1512 Object* ref = heap_->DequeuePendingReference(list); 1513 Object* referent = heap_->GetReferenceReferent(ref); 1514 if (referent != NULL && !IsMarked(referent)) { 1515 MarkObject(referent); 1516 // If the referent is non-null the reference must queuable. 1517 DCHECK(heap_->IsEnqueuable(ref)); 1518 ref->SetFieldObject(zombie_offset, referent, false); 1519 heap_->ClearReferenceReferent(ref); 1520 heap_->EnqueueReference(ref, &cleared_reference_list_); 1521 has_enqueued = true; 1522 } 1523 } 1524 timings_.EndSplit(); 1525 if (has_enqueued) { 1526 ProcessMarkStack(true); 1527 } 1528 DCHECK(*list == NULL); 1529} 1530 1531// Process reference class instances and schedule finalizations. 1532void MarkSweep::ProcessReferences(Object** soft_references, bool clear_soft, 1533 Object** weak_references, 1534 Object** finalizer_references, 1535 Object** phantom_references) { 1536 DCHECK(soft_references != NULL); 1537 DCHECK(weak_references != NULL); 1538 DCHECK(finalizer_references != NULL); 1539 DCHECK(phantom_references != NULL); 1540 1541 // Unless we are in the zygote or required to clear soft references 1542 // with white references, preserve some white referents. 1543 if (!clear_soft && !Runtime::Current()->IsZygote()) { 1544 PreserveSomeSoftReferences(soft_references); 1545 } 1546 1547 timings_.StartSplit("ProcessReferences"); 1548 // Clear all remaining soft and weak references with white 1549 // referents. 1550 ClearWhiteReferences(soft_references); 1551 ClearWhiteReferences(weak_references); 1552 timings_.EndSplit(); 1553 1554 // Preserve all white objects with finalize methods and schedule 1555 // them for finalization. 1556 EnqueueFinalizerReferences(finalizer_references); 1557 1558 timings_.StartSplit("ProcessReferences"); 1559 // Clear all f-reachable soft and weak references with white 1560 // referents. 1561 ClearWhiteReferences(soft_references); 1562 ClearWhiteReferences(weak_references); 1563 1564 // Clear all phantom references with white referents. 1565 ClearWhiteReferences(phantom_references); 1566 1567 // At this point all reference lists should be empty. 1568 DCHECK(*soft_references == NULL); 1569 DCHECK(*weak_references == NULL); 1570 DCHECK(*finalizer_references == NULL); 1571 DCHECK(*phantom_references == NULL); 1572 timings_.EndSplit(); 1573} 1574 1575void MarkSweep::UnBindBitmaps() { 1576 base::TimingLogger::ScopedSplit split("UnBindBitmaps", &timings_); 1577 for (const auto& space : GetHeap()->GetContinuousSpaces()) { 1578 if (space->IsDlMallocSpace()) { 1579 space::DlMallocSpace* alloc_space = space->AsDlMallocSpace(); 1580 if (alloc_space->temp_bitmap_.get() != NULL) { 1581 // At this point, the temp_bitmap holds our old mark bitmap. 1582 accounting::SpaceBitmap* new_bitmap = alloc_space->temp_bitmap_.release(); 1583 GetHeap()->GetMarkBitmap()->ReplaceBitmap(alloc_space->mark_bitmap_.get(), new_bitmap); 1584 CHECK_EQ(alloc_space->mark_bitmap_.release(), alloc_space->live_bitmap_.get()); 1585 alloc_space->mark_bitmap_.reset(new_bitmap); 1586 DCHECK(alloc_space->temp_bitmap_.get() == NULL); 1587 } 1588 } 1589 } 1590} 1591 1592void MarkSweep::FinishPhase() { 1593 base::TimingLogger::ScopedSplit split("FinishPhase", &timings_); 1594 // Can't enqueue references if we hold the mutator lock. 1595 Object* cleared_references = GetClearedReferences(); 1596 Heap* heap = GetHeap(); 1597 timings_.NewSplit("EnqueueClearedReferences"); 1598 heap->EnqueueClearedReferences(&cleared_references); 1599 1600 timings_.NewSplit("PostGcVerification"); 1601 heap->PostGcVerification(this); 1602 1603 timings_.NewSplit("GrowForUtilization"); 1604 heap->GrowForUtilization(GetGcType(), GetDurationNs()); 1605 1606 timings_.NewSplit("RequestHeapTrim"); 1607 heap->RequestHeapTrim(); 1608 1609 // Update the cumulative statistics 1610 total_time_ns_ += GetDurationNs(); 1611 total_paused_time_ns_ += std::accumulate(GetPauseTimes().begin(), GetPauseTimes().end(), 0, 1612 std::plus<uint64_t>()); 1613 total_freed_objects_ += GetFreedObjects(); 1614 total_freed_bytes_ += GetFreedBytes(); 1615 1616 // Ensure that the mark stack is empty. 1617 CHECK(mark_stack_->IsEmpty()); 1618 1619 if (kCountScannedTypes) { 1620 VLOG(gc) << "MarkSweep scanned classes=" << class_count_ << " arrays=" << array_count_ 1621 << " other=" << other_count_; 1622 } 1623 1624 if (kCountTasks) { 1625 VLOG(gc) << "Total number of work chunks allocated: " << work_chunks_created_; 1626 } 1627 1628 if (kMeasureOverhead) { 1629 VLOG(gc) << "Overhead time " << PrettyDuration(overhead_time_); 1630 } 1631 1632 if (kProfileLargeObjects) { 1633 VLOG(gc) << "Large objects tested " << large_object_test_ << " marked " << large_object_mark_; 1634 } 1635 1636 if (kCountClassesMarked) { 1637 VLOG(gc) << "Classes marked " << classes_marked_; 1638 } 1639 1640 if (kCountJavaLangRefs) { 1641 VLOG(gc) << "References scanned " << reference_count_; 1642 } 1643 1644 // Update the cumulative loggers. 1645 cumulative_timings_.Start(); 1646 cumulative_timings_.AddLogger(timings_); 1647 cumulative_timings_.End(); 1648 1649 // Clear all of the spaces' mark bitmaps. 1650 for (const auto& space : GetHeap()->GetContinuousSpaces()) { 1651 if (space->GetGcRetentionPolicy() != space::kGcRetentionPolicyNeverCollect) { 1652 space->GetMarkBitmap()->Clear(); 1653 } 1654 } 1655 mark_stack_->Reset(); 1656 1657 // Reset the marked large objects. 1658 space::LargeObjectSpace* large_objects = GetHeap()->GetLargeObjectsSpace(); 1659 large_objects->GetMarkObjects()->Clear(); 1660} 1661 1662} // namespace collector 1663} // namespace gc 1664} // namespace art 1665