mark_sweep.cc revision 0941b0423537a6a5d7c1df6dd23e9864ea8f319c
1/* 2 * Copyright (C) 2011 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17#include "mark_sweep.h" 18 19#include <functional> 20#include <numeric> 21#include <climits> 22#include <vector> 23 24#include "base/bounded_fifo.h" 25#include "base/logging.h" 26#include "base/macros.h" 27#include "base/mutex-inl.h" 28#include "base/timing_logger.h" 29#include "gc/accounting/card_table-inl.h" 30#include "gc/accounting/heap_bitmap.h" 31#include "gc/accounting/mod_union_table.h" 32#include "gc/accounting/space_bitmap-inl.h" 33#include "gc/heap.h" 34#include "gc/space/image_space.h" 35#include "gc/space/large_object_space.h" 36#include "gc/space/space-inl.h" 37#include "indirect_reference_table.h" 38#include "intern_table.h" 39#include "jni_internal.h" 40#include "monitor.h" 41#include "mark_sweep-inl.h" 42#include "mirror/art_field.h" 43#include "mirror/art_field-inl.h" 44#include "mirror/class-inl.h" 45#include "mirror/class_loader.h" 46#include "mirror/dex_cache.h" 47#include "mirror/object-inl.h" 48#include "mirror/object_array.h" 49#include "mirror/object_array-inl.h" 50#include "runtime.h" 51#include "thread-inl.h" 52#include "thread_list.h" 53#include "verifier/method_verifier.h" 54 55using ::art::mirror::ArtField; 56using ::art::mirror::Class; 57using ::art::mirror::Object; 58using ::art::mirror::ObjectArray; 59 60namespace art { 61namespace gc { 62namespace collector { 63 64// Performance options. 65constexpr bool kUseRecursiveMark = false; 66constexpr bool kUseMarkStackPrefetch = true; 67constexpr size_t kSweepArrayChunkFreeSize = 1024; 68 69// Parallelism options. 70constexpr bool kParallelCardScan = true; 71constexpr bool kParallelRecursiveMark = true; 72// Don't attempt to parallelize mark stack processing unless the mark stack is at least n 73// elements. This is temporary until we reduce the overhead caused by allocating tasks, etc.. Not 74// having this can add overhead in ProcessReferences since we may end up doing many calls of 75// ProcessMarkStack with very small mark stacks. 76constexpr size_t kMinimumParallelMarkStackSize = 128; 77constexpr bool kParallelProcessMarkStack = true; 78 79// Profiling and information flags. 80constexpr bool kCountClassesMarked = false; 81constexpr bool kProfileLargeObjects = false; 82constexpr bool kMeasureOverhead = false; 83constexpr bool kCountTasks = false; 84constexpr bool kCountJavaLangRefs = false; 85 86// Turn off kCheckLocks when profiling the GC since it slows the GC down by up to 40%. 87constexpr bool kCheckLocks = kDebugLocking; 88 89void MarkSweep::ImmuneSpace(space::ContinuousSpace* space) { 90 // Bind live to mark bitmap if necessary. 91 if (space->GetLiveBitmap() != space->GetMarkBitmap()) { 92 BindLiveToMarkBitmap(space); 93 } 94 95 // Add the space to the immune region. 96 if (immune_begin_ == NULL) { 97 DCHECK(immune_end_ == NULL); 98 SetImmuneRange(reinterpret_cast<Object*>(space->Begin()), 99 reinterpret_cast<Object*>(space->End())); 100 } else { 101 const space::ContinuousSpace* prev_space = nullptr; 102 // Find out if the previous space is immune. 103 for (const space::ContinuousSpace* cur_space : GetHeap()->GetContinuousSpaces()) { 104 if (cur_space == space) { 105 break; 106 } 107 prev_space = cur_space; 108 } 109 // If previous space was immune, then extend the immune region. Relies on continuous spaces 110 // being sorted by Heap::AddContinuousSpace. 111 if (prev_space != NULL && IsImmuneSpace(prev_space)) { 112 immune_begin_ = std::min(reinterpret_cast<Object*>(space->Begin()), immune_begin_); 113 immune_end_ = std::max(reinterpret_cast<Object*>(space->End()), immune_end_); 114 } 115 } 116} 117 118bool MarkSweep::IsImmuneSpace(const space::ContinuousSpace* space) { 119 return 120 immune_begin_ <= reinterpret_cast<Object*>(space->Begin()) && 121 immune_end_ >= reinterpret_cast<Object*>(space->End()); 122} 123 124void MarkSweep::BindBitmaps() { 125 timings_.StartSplit("BindBitmaps"); 126 WriterMutexLock mu(Thread::Current(), *Locks::heap_bitmap_lock_); 127 // Mark all of the spaces we never collect as immune. 128 for (const auto& space : GetHeap()->GetContinuousSpaces()) { 129 if (space->GetGcRetentionPolicy() == space::kGcRetentionPolicyNeverCollect) { 130 ImmuneSpace(space); 131 } 132 } 133 timings_.EndSplit(); 134} 135 136MarkSweep::MarkSweep(Heap* heap, bool is_concurrent, const std::string& name_prefix) 137 : GarbageCollector(heap, 138 name_prefix + (name_prefix.empty() ? "" : " ") + 139 (is_concurrent ? "concurrent mark sweep": "mark sweep")), 140 current_mark_bitmap_(NULL), 141 java_lang_Class_(NULL), 142 mark_stack_(NULL), 143 immune_begin_(NULL), 144 immune_end_(NULL), 145 soft_reference_list_(NULL), 146 weak_reference_list_(NULL), 147 finalizer_reference_list_(NULL), 148 phantom_reference_list_(NULL), 149 cleared_reference_list_(NULL), 150 gc_barrier_(new Barrier(0)), 151 large_object_lock_("mark sweep large object lock", kMarkSweepLargeObjectLock), 152 mark_stack_lock_("mark sweep mark stack lock", kMarkSweepMarkStackLock), 153 is_concurrent_(is_concurrent), 154 clear_soft_references_(false) { 155} 156 157void MarkSweep::InitializePhase() { 158 timings_.Reset(); 159 base::TimingLogger::ScopedSplit split("InitializePhase", &timings_); 160 mark_stack_ = heap_->mark_stack_.get(); 161 DCHECK(mark_stack_ != nullptr); 162 SetImmuneRange(nullptr, nullptr); 163 soft_reference_list_ = nullptr; 164 weak_reference_list_ = nullptr; 165 finalizer_reference_list_ = nullptr; 166 phantom_reference_list_ = nullptr; 167 cleared_reference_list_ = nullptr; 168 freed_bytes_ = 0; 169 freed_large_object_bytes_ = 0; 170 freed_objects_ = 0; 171 freed_large_objects_ = 0; 172 class_count_ = 0; 173 array_count_ = 0; 174 other_count_ = 0; 175 large_object_test_ = 0; 176 large_object_mark_ = 0; 177 classes_marked_ = 0; 178 overhead_time_ = 0; 179 work_chunks_created_ = 0; 180 work_chunks_deleted_ = 0; 181 reference_count_ = 0; 182 java_lang_Class_ = Class::GetJavaLangClass(); 183 CHECK(java_lang_Class_ != nullptr); 184 185 FindDefaultMarkBitmap(); 186 187 // Do any pre GC verification. 188 timings_.NewSplit("PreGcVerification"); 189 heap_->PreGcVerification(this); 190} 191 192void MarkSweep::ProcessReferences(Thread* self) { 193 base::TimingLogger::ScopedSplit split("ProcessReferences", &timings_); 194 WriterMutexLock mu(self, *Locks::heap_bitmap_lock_); 195 ProcessReferences(&soft_reference_list_, clear_soft_references_, &weak_reference_list_, 196 &finalizer_reference_list_, &phantom_reference_list_); 197} 198 199bool MarkSweep::HandleDirtyObjectsPhase() { 200 base::TimingLogger::ScopedSplit split("HandleDirtyObjectsPhase", &timings_); 201 Thread* self = Thread::Current(); 202 Locks::mutator_lock_->AssertExclusiveHeld(self); 203 204 { 205 WriterMutexLock mu(self, *Locks::heap_bitmap_lock_); 206 207 // Re-mark root set. 208 ReMarkRoots(); 209 210 // Scan dirty objects, this is only required if we are not doing concurrent GC. 211 RecursiveMarkDirtyObjects(true, accounting::CardTable::kCardDirty); 212 } 213 214 ProcessReferences(self); 215 216 // Only need to do this if we have the card mark verification on, and only during concurrent GC. 217 if (GetHeap()->verify_missing_card_marks_ || GetHeap()->verify_pre_gc_heap_|| 218 GetHeap()->verify_post_gc_heap_) { 219 WriterMutexLock mu(self, *Locks::heap_bitmap_lock_); 220 // This second sweep makes sure that we don't have any objects in the live stack which point to 221 // freed objects. These cause problems since their references may be previously freed objects. 222 SweepArray(GetHeap()->allocation_stack_.get(), false); 223 } 224 225 timings_.StartSplit("PreSweepingGcVerification"); 226 heap_->PreSweepingGcVerification(this); 227 timings_.EndSplit(); 228 229 // Ensure that nobody inserted items in the live stack after we swapped the stacks. 230 ReaderMutexLock mu(self, *Locks::heap_bitmap_lock_); 231 CHECK_GE(live_stack_freeze_size_, GetHeap()->GetLiveStack()->Size()); 232 233 // Disallow new system weaks to prevent a race which occurs when someone adds a new system 234 // weak before we sweep them. Since this new system weak may not be marked, the GC may 235 // incorrectly sweep it. This also fixes a race where interning may attempt to return a strong 236 // reference to a string that is about to be swept. 237 Runtime::Current()->DisallowNewSystemWeaks(); 238 return true; 239} 240 241bool MarkSweep::IsConcurrent() const { 242 return is_concurrent_; 243} 244 245void MarkSweep::MarkingPhase() { 246 base::TimingLogger::ScopedSplit split("MarkingPhase", &timings_); 247 Thread* self = Thread::Current(); 248 249 BindBitmaps(); 250 FindDefaultMarkBitmap(); 251 252 // Process dirty cards and add dirty cards to mod union tables. 253 heap_->ProcessCards(timings_); 254 255 // Need to do this before the checkpoint since we don't want any threads to add references to 256 // the live stack during the recursive mark. 257 timings_.NewSplit("SwapStacks"); 258 heap_->SwapStacks(); 259 260 WriterMutexLock mu(self, *Locks::heap_bitmap_lock_); 261 if (Locks::mutator_lock_->IsExclusiveHeld(self)) { 262 // If we exclusively hold the mutator lock, all threads must be suspended. 263 MarkRoots(); 264 } else { 265 MarkThreadRoots(self); 266 // At this point the live stack should no longer have any mutators which push into it. 267 MarkNonThreadRoots(); 268 } 269 live_stack_freeze_size_ = heap_->GetLiveStack()->Size(); 270 MarkConcurrentRoots(); 271 UpdateAndMarkModUnion(); 272 MarkReachableObjects(); 273} 274 275void MarkSweep::UpdateAndMarkModUnion() { 276 for (const auto& space : heap_->GetContinuousSpaces()) { 277 if (IsImmuneSpace(space)) { 278 const char* name = space->IsZygoteSpace() ? "UpdateAndMarkZygoteModUnionTable" : 279 "UpdateAndMarkImageModUnionTable"; 280 base::TimingLogger::ScopedSplit split(name, &timings_); 281 accounting::ModUnionTable* mod_union_table = heap_->FindModUnionTableFromSpace(space); 282 CHECK(mod_union_table != nullptr); 283 mod_union_table->UpdateAndMarkReferences(MarkRootCallback, this); 284 } 285 } 286} 287 288void MarkSweep::MarkThreadRoots(Thread* self) { 289 MarkRootsCheckpoint(self); 290} 291 292void MarkSweep::MarkReachableObjects() { 293 // Mark everything allocated since the last as GC live so that we can sweep concurrently, 294 // knowing that new allocations won't be marked as live. 295 timings_.StartSplit("MarkStackAsLive"); 296 accounting::ObjectStack* live_stack = heap_->GetLiveStack(); 297 heap_->MarkAllocStack(heap_->alloc_space_->GetLiveBitmap(), 298 heap_->large_object_space_->GetLiveObjects(), live_stack); 299 live_stack->Reset(); 300 timings_.EndSplit(); 301 // Recursively mark all the non-image bits set in the mark bitmap. 302 RecursiveMark(); 303} 304 305void MarkSweep::ReclaimPhase() { 306 base::TimingLogger::ScopedSplit split("ReclaimPhase", &timings_); 307 Thread* self = Thread::Current(); 308 309 if (!IsConcurrent()) { 310 ProcessReferences(self); 311 } 312 313 { 314 WriterMutexLock mu(self, *Locks::heap_bitmap_lock_); 315 SweepSystemWeaks(); 316 } 317 318 if (IsConcurrent()) { 319 Runtime::Current()->AllowNewSystemWeaks(); 320 321 base::TimingLogger::ScopedSplit split("UnMarkAllocStack", &timings_); 322 WriterMutexLock mu(self, *Locks::heap_bitmap_lock_); 323 accounting::ObjectStack* allocation_stack = GetHeap()->allocation_stack_.get(); 324 // The allocation stack contains things allocated since the start of the GC. These may have been 325 // marked during this GC meaning they won't be eligible for reclaiming in the next sticky GC. 326 // Remove these objects from the mark bitmaps so that they will be eligible for sticky 327 // collection. 328 // There is a race here which is safely handled. Another thread such as the hprof could 329 // have flushed the alloc stack after we resumed the threads. This is safe however, since 330 // reseting the allocation stack zeros it out with madvise. This means that we will either 331 // read NULLs or attempt to unmark a newly allocated object which will not be marked in the 332 // first place. 333 mirror::Object** end = allocation_stack->End(); 334 for (mirror::Object** it = allocation_stack->Begin(); it != end; ++it) { 335 const Object* obj = *it; 336 if (obj != NULL) { 337 UnMarkObjectNonNull(obj); 338 } 339 } 340 } 341 342 // Before freeing anything, lets verify the heap. 343 if (kIsDebugBuild) { 344 ReaderMutexLock mu(self, *Locks::heap_bitmap_lock_); 345 VerifyImageRoots(); 346 } 347 348 { 349 WriterMutexLock mu(self, *Locks::heap_bitmap_lock_); 350 351 // Reclaim unmarked objects. 352 Sweep(false); 353 354 // Swap the live and mark bitmaps for each space which we modified space. This is an 355 // optimization that enables us to not clear live bits inside of the sweep. Only swaps unbound 356 // bitmaps. 357 timings_.StartSplit("SwapBitmaps"); 358 SwapBitmaps(); 359 timings_.EndSplit(); 360 361 // Unbind the live and mark bitmaps. 362 UnBindBitmaps(); 363 } 364} 365 366void MarkSweep::SetImmuneRange(Object* begin, Object* end) { 367 immune_begin_ = begin; 368 immune_end_ = end; 369} 370 371void MarkSweep::FindDefaultMarkBitmap() { 372 base::TimingLogger::ScopedSplit split("FindDefaultMarkBitmap", &timings_); 373 for (const auto& space : GetHeap()->GetContinuousSpaces()) { 374 if (space->GetGcRetentionPolicy() == space::kGcRetentionPolicyAlwaysCollect) { 375 current_mark_bitmap_ = space->GetMarkBitmap(); 376 CHECK(current_mark_bitmap_ != NULL); 377 return; 378 } 379 } 380 GetHeap()->DumpSpaces(); 381 LOG(FATAL) << "Could not find a default mark bitmap"; 382} 383 384void MarkSweep::ExpandMarkStack() { 385 ResizeMarkStack(mark_stack_->Capacity() * 2); 386} 387 388void MarkSweep::ResizeMarkStack(size_t new_size) { 389 // Rare case, no need to have Thread::Current be a parameter. 390 if (UNLIKELY(mark_stack_->Size() < mark_stack_->Capacity())) { 391 // Someone else acquired the lock and expanded the mark stack before us. 392 return; 393 } 394 std::vector<Object*> temp(mark_stack_->Begin(), mark_stack_->End()); 395 CHECK_LE(mark_stack_->Size(), new_size); 396 mark_stack_->Resize(new_size); 397 for (const auto& obj : temp) { 398 mark_stack_->PushBack(obj); 399 } 400} 401 402inline void MarkSweep::MarkObjectNonNullParallel(const Object* obj) { 403 DCHECK(obj != NULL); 404 if (MarkObjectParallel(obj)) { 405 MutexLock mu(Thread::Current(), mark_stack_lock_); 406 if (UNLIKELY(mark_stack_->Size() >= mark_stack_->Capacity())) { 407 ExpandMarkStack(); 408 } 409 // The object must be pushed on to the mark stack. 410 mark_stack_->PushBack(const_cast<Object*>(obj)); 411 } 412} 413 414inline void MarkSweep::UnMarkObjectNonNull(const Object* obj) { 415 DCHECK(!IsImmune(obj)); 416 // Try to take advantage of locality of references within a space, failing this find the space 417 // the hard way. 418 accounting::SpaceBitmap* object_bitmap = current_mark_bitmap_; 419 if (UNLIKELY(!object_bitmap->HasAddress(obj))) { 420 accounting::SpaceBitmap* new_bitmap = heap_->GetMarkBitmap()->GetContinuousSpaceBitmap(obj); 421 if (LIKELY(new_bitmap != NULL)) { 422 object_bitmap = new_bitmap; 423 } else { 424 MarkLargeObject(obj, false); 425 return; 426 } 427 } 428 429 DCHECK(object_bitmap->HasAddress(obj)); 430 object_bitmap->Clear(obj); 431} 432 433inline void MarkSweep::MarkObjectNonNull(const Object* obj) { 434 DCHECK(obj != NULL); 435 436 if (IsImmune(obj)) { 437 DCHECK(IsMarked(obj)); 438 return; 439 } 440 441 // Try to take advantage of locality of references within a space, failing this find the space 442 // the hard way. 443 accounting::SpaceBitmap* object_bitmap = current_mark_bitmap_; 444 if (UNLIKELY(!object_bitmap->HasAddress(obj))) { 445 accounting::SpaceBitmap* new_bitmap = heap_->GetMarkBitmap()->GetContinuousSpaceBitmap(obj); 446 if (LIKELY(new_bitmap != NULL)) { 447 object_bitmap = new_bitmap; 448 } else { 449 MarkLargeObject(obj, true); 450 return; 451 } 452 } 453 454 // This object was not previously marked. 455 if (!object_bitmap->Test(obj)) { 456 object_bitmap->Set(obj); 457 if (UNLIKELY(mark_stack_->Size() >= mark_stack_->Capacity())) { 458 // Lock is not needed but is here anyways to please annotalysis. 459 MutexLock mu(Thread::Current(), mark_stack_lock_); 460 ExpandMarkStack(); 461 } 462 // The object must be pushed on to the mark stack. 463 mark_stack_->PushBack(const_cast<Object*>(obj)); 464 } 465} 466 467// Rare case, probably not worth inlining since it will increase instruction cache miss rate. 468bool MarkSweep::MarkLargeObject(const Object* obj, bool set) { 469 // TODO: support >1 discontinuous space. 470 space::LargeObjectSpace* large_object_space = GetHeap()->GetLargeObjectsSpace(); 471 accounting::SpaceSetMap* large_objects = large_object_space->GetMarkObjects(); 472 if (kProfileLargeObjects) { 473 ++large_object_test_; 474 } 475 if (UNLIKELY(!large_objects->Test(obj))) { 476 if (!large_object_space->Contains(obj)) { 477 LOG(ERROR) << "Tried to mark " << obj << " not contained by any spaces"; 478 LOG(ERROR) << "Attempting see if it's a bad root"; 479 VerifyRoots(); 480 LOG(FATAL) << "Can't mark bad root"; 481 } 482 if (kProfileLargeObjects) { 483 ++large_object_mark_; 484 } 485 if (set) { 486 large_objects->Set(obj); 487 } else { 488 large_objects->Clear(obj); 489 } 490 return true; 491 } 492 return false; 493} 494 495inline bool MarkSweep::MarkObjectParallel(const Object* obj) { 496 DCHECK(obj != NULL); 497 498 if (IsImmune(obj)) { 499 DCHECK(IsMarked(obj)); 500 return false; 501 } 502 503 // Try to take advantage of locality of references within a space, failing this find the space 504 // the hard way. 505 accounting::SpaceBitmap* object_bitmap = current_mark_bitmap_; 506 if (UNLIKELY(!object_bitmap->HasAddress(obj))) { 507 accounting::SpaceBitmap* new_bitmap = heap_->GetMarkBitmap()->GetContinuousSpaceBitmap(obj); 508 if (new_bitmap != NULL) { 509 object_bitmap = new_bitmap; 510 } else { 511 // TODO: Remove the Thread::Current here? 512 // TODO: Convert this to some kind of atomic marking? 513 MutexLock mu(Thread::Current(), large_object_lock_); 514 return MarkLargeObject(obj, true); 515 } 516 } 517 518 // Return true if the object was not previously marked. 519 return !object_bitmap->AtomicTestAndSet(obj); 520} 521 522// Used to mark objects when recursing. Recursion is done by moving 523// the finger across the bitmaps in address order and marking child 524// objects. Any newly-marked objects whose addresses are lower than 525// the finger won't be visited by the bitmap scan, so those objects 526// need to be added to the mark stack. 527inline void MarkSweep::MarkObject(const Object* obj) { 528 if (obj != NULL) { 529 MarkObjectNonNull(obj); 530 } 531} 532 533void MarkSweep::MarkRoot(const Object* obj) { 534 if (obj != NULL) { 535 MarkObjectNonNull(obj); 536 } 537} 538 539Object* MarkSweep::MarkRootParallelCallback(Object* root, void* arg) { 540 DCHECK(root != NULL); 541 DCHECK(arg != NULL); 542 reinterpret_cast<MarkSweep*>(arg)->MarkObjectNonNullParallel(root); 543 return root; 544} 545 546Object* MarkSweep::MarkRootCallback(Object* root, void* arg) { 547 DCHECK(root != nullptr); 548 DCHECK(arg != nullptr); 549 reinterpret_cast<MarkSweep*>(arg)->MarkObjectNonNull(root); 550 return root; 551} 552 553void MarkSweep::VerifyRootCallback(const Object* root, void* arg, size_t vreg, 554 const StackVisitor* visitor) { 555 reinterpret_cast<MarkSweep*>(arg)->VerifyRoot(root, vreg, visitor); 556} 557 558void MarkSweep::VerifyRoot(const Object* root, size_t vreg, const StackVisitor* visitor) { 559 // See if the root is on any space bitmap. 560 if (GetHeap()->GetLiveBitmap()->GetContinuousSpaceBitmap(root) == NULL) { 561 space::LargeObjectSpace* large_object_space = GetHeap()->GetLargeObjectsSpace(); 562 if (!large_object_space->Contains(root)) { 563 LOG(ERROR) << "Found invalid root: " << root; 564 if (visitor != NULL) { 565 LOG(ERROR) << visitor->DescribeLocation() << " in VReg: " << vreg; 566 } 567 } 568 } 569} 570 571void MarkSweep::VerifyRoots() { 572 Runtime::Current()->GetThreadList()->VerifyRoots(VerifyRootCallback, this); 573} 574 575// Marks all objects in the root set. 576void MarkSweep::MarkRoots() { 577 timings_.StartSplit("MarkRoots"); 578 Runtime::Current()->VisitNonConcurrentRoots(MarkRootCallback, this); 579 timings_.EndSplit(); 580} 581 582void MarkSweep::MarkNonThreadRoots() { 583 timings_.StartSplit("MarkNonThreadRoots"); 584 Runtime::Current()->VisitNonThreadRoots(MarkRootCallback, this); 585 timings_.EndSplit(); 586} 587 588void MarkSweep::MarkConcurrentRoots() { 589 timings_.StartSplit("MarkConcurrentRoots"); 590 // Visit all runtime roots and clear dirty flags. 591 Runtime::Current()->VisitConcurrentRoots(MarkRootCallback, this, false, true); 592 timings_.EndSplit(); 593} 594 595void MarkSweep::CheckObject(const Object* obj) { 596 DCHECK(obj != NULL); 597 VisitObjectReferences(const_cast<Object*>(obj), [this](const Object* obj, const Object* ref, 598 MemberOffset offset, bool is_static) NO_THREAD_SAFETY_ANALYSIS { 599 Locks::heap_bitmap_lock_->AssertSharedHeld(Thread::Current()); 600 CheckReference(obj, ref, offset, is_static); 601 }, true); 602} 603 604void MarkSweep::VerifyImageRootVisitor(Object* root, void* arg) { 605 DCHECK(root != NULL); 606 DCHECK(arg != NULL); 607 MarkSweep* mark_sweep = reinterpret_cast<MarkSweep*>(arg); 608 DCHECK(mark_sweep->heap_->GetMarkBitmap()->Test(root)); 609 mark_sweep->CheckObject(root); 610} 611 612void MarkSweep::BindLiveToMarkBitmap(space::ContinuousSpace* space) { 613 CHECK(space->IsDlMallocSpace()); 614 space::DlMallocSpace* alloc_space = space->AsDlMallocSpace(); 615 accounting::SpaceBitmap* live_bitmap = space->GetLiveBitmap(); 616 accounting::SpaceBitmap* mark_bitmap = alloc_space->mark_bitmap_.release(); 617 GetHeap()->GetMarkBitmap()->ReplaceBitmap(mark_bitmap, live_bitmap); 618 alloc_space->temp_bitmap_.reset(mark_bitmap); 619 alloc_space->mark_bitmap_.reset(live_bitmap); 620} 621 622class ScanObjectVisitor { 623 public: 624 explicit ScanObjectVisitor(MarkSweep* const mark_sweep) ALWAYS_INLINE 625 : mark_sweep_(mark_sweep) {} 626 627 // TODO: Fixme when anotatalysis works with visitors. 628 void operator()(const Object* obj) const ALWAYS_INLINE NO_THREAD_SAFETY_ANALYSIS { 629 if (kCheckLocks) { 630 Locks::mutator_lock_->AssertSharedHeld(Thread::Current()); 631 Locks::heap_bitmap_lock_->AssertExclusiveHeld(Thread::Current()); 632 } 633 mark_sweep_->ScanObject(obj); 634 } 635 636 private: 637 MarkSweep* const mark_sweep_; 638}; 639 640template <bool kUseFinger = false> 641class MarkStackTask : public Task { 642 public: 643 MarkStackTask(ThreadPool* thread_pool, MarkSweep* mark_sweep, size_t mark_stack_size, 644 const Object** mark_stack) 645 : mark_sweep_(mark_sweep), 646 thread_pool_(thread_pool), 647 mark_stack_pos_(mark_stack_size) { 648 // We may have to copy part of an existing mark stack when another mark stack overflows. 649 if (mark_stack_size != 0) { 650 DCHECK(mark_stack != NULL); 651 // TODO: Check performance? 652 std::copy(mark_stack, mark_stack + mark_stack_size, mark_stack_); 653 } 654 if (kCountTasks) { 655 ++mark_sweep_->work_chunks_created_; 656 } 657 } 658 659 static const size_t kMaxSize = 1 * KB; 660 661 protected: 662 class ScanObjectParallelVisitor { 663 public: 664 explicit ScanObjectParallelVisitor(MarkStackTask<kUseFinger>* chunk_task) ALWAYS_INLINE 665 : chunk_task_(chunk_task) {} 666 667 void operator()(Object* obj) const { 668 MarkSweep* mark_sweep = chunk_task_->mark_sweep_; 669 mark_sweep->ScanObjectVisit(obj, 670 [mark_sweep, this](Object* /* obj */, Object* ref, const MemberOffset& /* offset */, 671 bool /* is_static */) ALWAYS_INLINE { 672 if (ref != nullptr && mark_sweep->MarkObjectParallel(ref)) { 673 if (kUseFinger) { 674 android_memory_barrier(); 675 if (reinterpret_cast<uintptr_t>(ref) >= 676 static_cast<uintptr_t>(mark_sweep->atomic_finger_)) { 677 return; 678 } 679 } 680 chunk_task_->MarkStackPush(ref); 681 } 682 }); 683 } 684 685 private: 686 MarkStackTask<kUseFinger>* const chunk_task_; 687 }; 688 689 virtual ~MarkStackTask() { 690 // Make sure that we have cleared our mark stack. 691 DCHECK_EQ(mark_stack_pos_, 0U); 692 if (kCountTasks) { 693 ++mark_sweep_->work_chunks_deleted_; 694 } 695 } 696 697 MarkSweep* const mark_sweep_; 698 ThreadPool* const thread_pool_; 699 // Thread local mark stack for this task. 700 const Object* mark_stack_[kMaxSize]; 701 // Mark stack position. 702 size_t mark_stack_pos_; 703 704 void MarkStackPush(const Object* obj) ALWAYS_INLINE { 705 if (UNLIKELY(mark_stack_pos_ == kMaxSize)) { 706 // Mark stack overflow, give 1/2 the stack to the thread pool as a new work task. 707 mark_stack_pos_ /= 2; 708 auto* task = new MarkStackTask(thread_pool_, mark_sweep_, kMaxSize - mark_stack_pos_, 709 mark_stack_ + mark_stack_pos_); 710 thread_pool_->AddTask(Thread::Current(), task); 711 } 712 DCHECK(obj != nullptr); 713 DCHECK(mark_stack_pos_ < kMaxSize); 714 mark_stack_[mark_stack_pos_++] = obj; 715 } 716 717 virtual void Finalize() { 718 delete this; 719 } 720 721 // Scans all of the objects 722 virtual void Run(Thread* self) { 723 ScanObjectParallelVisitor visitor(this); 724 // TODO: Tune this. 725 static const size_t kFifoSize = 4; 726 BoundedFifoPowerOfTwo<const Object*, kFifoSize> prefetch_fifo; 727 for (;;) { 728 const Object* obj = nullptr; 729 if (kUseMarkStackPrefetch) { 730 while (mark_stack_pos_ != 0 && prefetch_fifo.size() < kFifoSize) { 731 const Object* obj = mark_stack_[--mark_stack_pos_]; 732 DCHECK(obj != nullptr); 733 __builtin_prefetch(obj); 734 prefetch_fifo.push_back(obj); 735 } 736 if (UNLIKELY(prefetch_fifo.empty())) { 737 break; 738 } 739 obj = prefetch_fifo.front(); 740 prefetch_fifo.pop_front(); 741 } else { 742 if (UNLIKELY(mark_stack_pos_ == 0)) { 743 break; 744 } 745 obj = mark_stack_[--mark_stack_pos_]; 746 } 747 DCHECK(obj != nullptr); 748 visitor(const_cast<mirror::Object*>(obj)); 749 } 750 } 751}; 752 753class CardScanTask : public MarkStackTask<false> { 754 public: 755 CardScanTask(ThreadPool* thread_pool, MarkSweep* mark_sweep, accounting::SpaceBitmap* bitmap, 756 byte* begin, byte* end, byte minimum_age, size_t mark_stack_size, 757 const Object** mark_stack_obj) 758 : MarkStackTask<false>(thread_pool, mark_sweep, mark_stack_size, mark_stack_obj), 759 bitmap_(bitmap), 760 begin_(begin), 761 end_(end), 762 minimum_age_(minimum_age) { 763 } 764 765 protected: 766 accounting::SpaceBitmap* const bitmap_; 767 byte* const begin_; 768 byte* const end_; 769 const byte minimum_age_; 770 771 virtual void Finalize() { 772 delete this; 773 } 774 775 virtual void Run(Thread* self) NO_THREAD_SAFETY_ANALYSIS { 776 ScanObjectParallelVisitor visitor(this); 777 accounting::CardTable* card_table = mark_sweep_->GetHeap()->GetCardTable(); 778 size_t cards_scanned = card_table->Scan(bitmap_, begin_, end_, visitor, minimum_age_); 779 VLOG(heap) << "Parallel scanning cards " << reinterpret_cast<void*>(begin_) << " - " 780 << reinterpret_cast<void*>(end_) << " = " << cards_scanned; 781 // Finish by emptying our local mark stack. 782 MarkStackTask::Run(self); 783 } 784}; 785 786size_t MarkSweep::GetThreadCount(bool paused) const { 787 if (heap_->GetThreadPool() == nullptr || !heap_->CareAboutPauseTimes()) { 788 return 0; 789 } 790 if (paused) { 791 return heap_->GetParallelGCThreadCount() + 1; 792 } else { 793 return heap_->GetConcGCThreadCount() + 1; 794 } 795} 796 797void MarkSweep::ScanGrayObjects(bool paused, byte minimum_age) { 798 accounting::CardTable* card_table = GetHeap()->GetCardTable(); 799 ThreadPool* thread_pool = GetHeap()->GetThreadPool(); 800 size_t thread_count = GetThreadCount(paused); 801 // The parallel version with only one thread is faster for card scanning, TODO: fix. 802 if (kParallelCardScan && thread_count > 0) { 803 Thread* self = Thread::Current(); 804 // Can't have a different split for each space since multiple spaces can have their cards being 805 // scanned at the same time. 806 timings_.StartSplit(paused ? "(Paused)ScanGrayObjects" : "ScanGrayObjects"); 807 // Try to take some of the mark stack since we can pass this off to the worker tasks. 808 const Object** mark_stack_begin = const_cast<const Object**>(mark_stack_->Begin()); 809 const Object** mark_stack_end = const_cast<const Object**>(mark_stack_->End()); 810 const size_t mark_stack_size = mark_stack_end - mark_stack_begin; 811 // Estimated number of work tasks we will create. 812 const size_t mark_stack_tasks = GetHeap()->GetContinuousSpaces().size() * thread_count; 813 DCHECK_NE(mark_stack_tasks, 0U); 814 const size_t mark_stack_delta = std::min(CardScanTask::kMaxSize / 2, 815 mark_stack_size / mark_stack_tasks + 1); 816 for (const auto& space : GetHeap()->GetContinuousSpaces()) { 817 byte* card_begin = space->Begin(); 818 byte* card_end = space->End(); 819 // Align up the end address. For example, the image space's end 820 // may not be card-size-aligned. 821 card_end = AlignUp(card_end, accounting::CardTable::kCardSize); 822 DCHECK(IsAligned<accounting::CardTable::kCardSize>(card_begin)); 823 DCHECK(IsAligned<accounting::CardTable::kCardSize>(card_end)); 824 // Calculate how many bytes of heap we will scan, 825 const size_t address_range = card_end - card_begin; 826 // Calculate how much address range each task gets. 827 const size_t card_delta = RoundUp(address_range / thread_count + 1, 828 accounting::CardTable::kCardSize); 829 // Create the worker tasks for this space. 830 while (card_begin != card_end) { 831 // Add a range of cards. 832 size_t addr_remaining = card_end - card_begin; 833 size_t card_increment = std::min(card_delta, addr_remaining); 834 // Take from the back of the mark stack. 835 size_t mark_stack_remaining = mark_stack_end - mark_stack_begin; 836 size_t mark_stack_increment = std::min(mark_stack_delta, mark_stack_remaining); 837 mark_stack_end -= mark_stack_increment; 838 mark_stack_->PopBackCount(static_cast<int32_t>(mark_stack_increment)); 839 DCHECK_EQ(mark_stack_end, mark_stack_->End()); 840 // Add the new task to the thread pool. 841 auto* task = new CardScanTask(thread_pool, this, space->GetMarkBitmap(), card_begin, 842 card_begin + card_increment, minimum_age, 843 mark_stack_increment, mark_stack_end); 844 thread_pool->AddTask(self, task); 845 card_begin += card_increment; 846 } 847 } 848 849 // Note: the card scan below may dirty new cards (and scan them) 850 // as a side effect when a Reference object is encountered and 851 // queued during the marking. See b/11465268. 852 thread_pool->SetMaxActiveWorkers(thread_count - 1); 853 thread_pool->StartWorkers(self); 854 thread_pool->Wait(self, true, true); 855 thread_pool->StopWorkers(self); 856 timings_.EndSplit(); 857 } else { 858 for (const auto& space : GetHeap()->GetContinuousSpaces()) { 859 // Image spaces are handled properly since live == marked for them. 860 switch (space->GetGcRetentionPolicy()) { 861 case space::kGcRetentionPolicyNeverCollect: 862 timings_.StartSplit(paused ? "(Paused)ScanGrayImageSpaceObjects" : 863 "ScanGrayImageSpaceObjects"); 864 break; 865 case space::kGcRetentionPolicyFullCollect: 866 timings_.StartSplit(paused ? "(Paused)ScanGrayZygoteSpaceObjects" : 867 "ScanGrayZygoteSpaceObjects"); 868 break; 869 case space::kGcRetentionPolicyAlwaysCollect: 870 timings_.StartSplit(paused ? "(Paused)ScanGrayAllocSpaceObjects" : 871 "ScanGrayAllocSpaceObjects"); 872 break; 873 } 874 ScanObjectVisitor visitor(this); 875 card_table->Scan(space->GetMarkBitmap(), space->Begin(), space->End(), visitor, minimum_age); 876 timings_.EndSplit(); 877 } 878 } 879} 880 881void MarkSweep::VerifyImageRoots() { 882 // Verify roots ensures that all the references inside the image space point 883 // objects which are either in the image space or marked objects in the alloc 884 // space 885 timings_.StartSplit("VerifyImageRoots"); 886 for (const auto& space : GetHeap()->GetContinuousSpaces()) { 887 if (space->IsImageSpace()) { 888 space::ImageSpace* image_space = space->AsImageSpace(); 889 uintptr_t begin = reinterpret_cast<uintptr_t>(image_space->Begin()); 890 uintptr_t end = reinterpret_cast<uintptr_t>(image_space->End()); 891 accounting::SpaceBitmap* live_bitmap = image_space->GetLiveBitmap(); 892 DCHECK(live_bitmap != NULL); 893 live_bitmap->VisitMarkedRange(begin, end, [this](const Object* obj) { 894 if (kCheckLocks) { 895 Locks::heap_bitmap_lock_->AssertSharedHeld(Thread::Current()); 896 } 897 DCHECK(obj != NULL); 898 CheckObject(obj); 899 }); 900 } 901 } 902 timings_.EndSplit(); 903} 904 905class RecursiveMarkTask : public MarkStackTask<false> { 906 public: 907 RecursiveMarkTask(ThreadPool* thread_pool, MarkSweep* mark_sweep, 908 accounting::SpaceBitmap* bitmap, uintptr_t begin, uintptr_t end) 909 : MarkStackTask<false>(thread_pool, mark_sweep, 0, NULL), 910 bitmap_(bitmap), 911 begin_(begin), 912 end_(end) { 913 } 914 915 protected: 916 accounting::SpaceBitmap* const bitmap_; 917 const uintptr_t begin_; 918 const uintptr_t end_; 919 920 virtual void Finalize() { 921 delete this; 922 } 923 924 // Scans all of the objects 925 virtual void Run(Thread* self) NO_THREAD_SAFETY_ANALYSIS { 926 ScanObjectParallelVisitor visitor(this); 927 bitmap_->VisitMarkedRange(begin_, end_, visitor); 928 // Finish by emptying our local mark stack. 929 MarkStackTask::Run(self); 930 } 931}; 932 933// Populates the mark stack based on the set of marked objects and 934// recursively marks until the mark stack is emptied. 935void MarkSweep::RecursiveMark() { 936 base::TimingLogger::ScopedSplit split("RecursiveMark", &timings_); 937 // RecursiveMark will build the lists of known instances of the Reference classes. 938 // See DelayReferenceReferent for details. 939 CHECK(soft_reference_list_ == NULL); 940 CHECK(weak_reference_list_ == NULL); 941 CHECK(finalizer_reference_list_ == NULL); 942 CHECK(phantom_reference_list_ == NULL); 943 CHECK(cleared_reference_list_ == NULL); 944 945 if (kUseRecursiveMark) { 946 const bool partial = GetGcType() == kGcTypePartial; 947 ScanObjectVisitor scan_visitor(this); 948 auto* self = Thread::Current(); 949 ThreadPool* thread_pool = heap_->GetThreadPool(); 950 size_t thread_count = GetThreadCount(false); 951 const bool parallel = kParallelRecursiveMark && thread_count > 1; 952 mark_stack_->Reset(); 953 for (const auto& space : GetHeap()->GetContinuousSpaces()) { 954 if ((space->GetGcRetentionPolicy() == space::kGcRetentionPolicyAlwaysCollect) || 955 (!partial && space->GetGcRetentionPolicy() == space::kGcRetentionPolicyFullCollect)) { 956 current_mark_bitmap_ = space->GetMarkBitmap(); 957 if (current_mark_bitmap_ == NULL) { 958 GetHeap()->DumpSpaces(); 959 LOG(FATAL) << "invalid bitmap"; 960 } 961 if (parallel) { 962 // We will use the mark stack the future. 963 // CHECK(mark_stack_->IsEmpty()); 964 // This function does not handle heap end increasing, so we must use the space end. 965 uintptr_t begin = reinterpret_cast<uintptr_t>(space->Begin()); 966 uintptr_t end = reinterpret_cast<uintptr_t>(space->End()); 967 atomic_finger_ = static_cast<int32_t>(0xFFFFFFFF); 968 969 // Create a few worker tasks. 970 const size_t n = thread_count * 2; 971 while (begin != end) { 972 uintptr_t start = begin; 973 uintptr_t delta = (end - begin) / n; 974 delta = RoundUp(delta, KB); 975 if (delta < 16 * KB) delta = end - begin; 976 begin += delta; 977 auto* task = new RecursiveMarkTask(thread_pool, this, current_mark_bitmap_, start, 978 begin); 979 thread_pool->AddTask(self, task); 980 } 981 thread_pool->SetMaxActiveWorkers(thread_count - 1); 982 thread_pool->StartWorkers(self); 983 thread_pool->Wait(self, true, true); 984 thread_pool->StopWorkers(self); 985 } else { 986 // This function does not handle heap end increasing, so we must use the space end. 987 uintptr_t begin = reinterpret_cast<uintptr_t>(space->Begin()); 988 uintptr_t end = reinterpret_cast<uintptr_t>(space->End()); 989 current_mark_bitmap_->VisitMarkedRange(begin, end, scan_visitor); 990 } 991 } 992 } 993 } 994 ProcessMarkStack(false); 995} 996 997mirror::Object* MarkSweep::SystemWeakIsMarkedCallback(Object* object, void* arg) { 998 if (reinterpret_cast<MarkSweep*>(arg)->IsMarked(object)) { 999 return object; 1000 } 1001 return nullptr; 1002} 1003 1004void MarkSweep::RecursiveMarkDirtyObjects(bool paused, byte minimum_age) { 1005 ScanGrayObjects(paused, minimum_age); 1006 ProcessMarkStack(paused); 1007} 1008 1009void MarkSweep::ReMarkRoots() { 1010 timings_.StartSplit("ReMarkRoots"); 1011 Runtime::Current()->VisitRoots(MarkRootCallback, this, true, true); 1012 timings_.EndSplit(); 1013} 1014 1015void MarkSweep::SweepSystemWeaks() { 1016 Runtime* runtime = Runtime::Current(); 1017 timings_.StartSplit("SweepSystemWeaks"); 1018 runtime->SweepSystemWeaks(SystemWeakIsMarkedCallback, this); 1019 timings_.EndSplit(); 1020} 1021 1022mirror::Object* MarkSweep::VerifySystemWeakIsLiveCallback(Object* obj, void* arg) { 1023 reinterpret_cast<MarkSweep*>(arg)->VerifyIsLive(obj); 1024 // We don't actually want to sweep the object, so lets return "marked" 1025 return obj; 1026} 1027 1028void MarkSweep::VerifyIsLive(const Object* obj) { 1029 Heap* heap = GetHeap(); 1030 if (!heap->GetLiveBitmap()->Test(obj)) { 1031 space::LargeObjectSpace* large_object_space = GetHeap()->GetLargeObjectsSpace(); 1032 if (!large_object_space->GetLiveObjects()->Test(obj)) { 1033 if (std::find(heap->allocation_stack_->Begin(), heap->allocation_stack_->End(), obj) == 1034 heap->allocation_stack_->End()) { 1035 // Object not found! 1036 heap->DumpSpaces(); 1037 LOG(FATAL) << "Found dead object " << obj; 1038 } 1039 } 1040 } 1041} 1042 1043void MarkSweep::VerifySystemWeaks() { 1044 // Verify system weaks, uses a special object visitor which returns the input object. 1045 Runtime::Current()->SweepSystemWeaks(VerifySystemWeakIsLiveCallback, this); 1046} 1047 1048struct SweepCallbackContext { 1049 MarkSweep* mark_sweep; 1050 space::AllocSpace* space; 1051 Thread* self; 1052}; 1053 1054class CheckpointMarkThreadRoots : public Closure { 1055 public: 1056 explicit CheckpointMarkThreadRoots(MarkSweep* mark_sweep) : mark_sweep_(mark_sweep) {} 1057 1058 virtual void Run(Thread* thread) NO_THREAD_SAFETY_ANALYSIS { 1059 ATRACE_BEGIN("Marking thread roots"); 1060 // Note: self is not necessarily equal to thread since thread may be suspended. 1061 Thread* self = Thread::Current(); 1062 CHECK(thread == self || thread->IsSuspended() || thread->GetState() == kWaitingPerformingGc) 1063 << thread->GetState() << " thread " << thread << " self " << self; 1064 thread->VisitRoots(MarkSweep::MarkRootParallelCallback, mark_sweep_); 1065 ATRACE_END(); 1066 mark_sweep_->GetBarrier().Pass(self); 1067 } 1068 1069 private: 1070 MarkSweep* mark_sweep_; 1071}; 1072 1073void MarkSweep::MarkRootsCheckpoint(Thread* self) { 1074 CheckpointMarkThreadRoots check_point(this); 1075 timings_.StartSplit("MarkRootsCheckpoint"); 1076 ThreadList* thread_list = Runtime::Current()->GetThreadList(); 1077 // Request the check point is run on all threads returning a count of the threads that must 1078 // run through the barrier including self. 1079 size_t barrier_count = thread_list->RunCheckpoint(&check_point); 1080 // Release locks then wait for all mutator threads to pass the barrier. 1081 // TODO: optimize to not release locks when there are no threads to wait for. 1082 Locks::heap_bitmap_lock_->ExclusiveUnlock(self); 1083 Locks::mutator_lock_->SharedUnlock(self); 1084 ThreadState old_state = self->SetState(kWaitingForCheckPointsToRun); 1085 CHECK_EQ(old_state, kWaitingPerformingGc); 1086 gc_barrier_->Increment(self, barrier_count); 1087 self->SetState(kWaitingPerformingGc); 1088 Locks::mutator_lock_->SharedLock(self); 1089 Locks::heap_bitmap_lock_->ExclusiveLock(self); 1090 timings_.EndSplit(); 1091} 1092 1093void MarkSweep::SweepCallback(size_t num_ptrs, Object** ptrs, void* arg) { 1094 SweepCallbackContext* context = static_cast<SweepCallbackContext*>(arg); 1095 MarkSweep* mark_sweep = context->mark_sweep; 1096 Heap* heap = mark_sweep->GetHeap(); 1097 space::AllocSpace* space = context->space; 1098 Thread* self = context->self; 1099 Locks::heap_bitmap_lock_->AssertExclusiveHeld(self); 1100 // Use a bulk free, that merges consecutive objects before freeing or free per object? 1101 // Documentation suggests better free performance with merging, but this may be at the expensive 1102 // of allocation. 1103 size_t freed_objects = num_ptrs; 1104 // AllocSpace::FreeList clears the value in ptrs, so perform after clearing the live bit 1105 size_t freed_bytes = space->FreeList(self, num_ptrs, ptrs); 1106 heap->RecordFree(freed_objects, freed_bytes); 1107 mark_sweep->freed_objects_.fetch_add(freed_objects); 1108 mark_sweep->freed_bytes_.fetch_add(freed_bytes); 1109} 1110 1111void MarkSweep::ZygoteSweepCallback(size_t num_ptrs, Object** ptrs, void* arg) { 1112 SweepCallbackContext* context = static_cast<SweepCallbackContext*>(arg); 1113 Locks::heap_bitmap_lock_->AssertExclusiveHeld(context->self); 1114 Heap* heap = context->mark_sweep->GetHeap(); 1115 // We don't free any actual memory to avoid dirtying the shared zygote pages. 1116 for (size_t i = 0; i < num_ptrs; ++i) { 1117 Object* obj = static_cast<Object*>(ptrs[i]); 1118 heap->GetLiveBitmap()->Clear(obj); 1119 heap->GetCardTable()->MarkCard(obj); 1120 } 1121} 1122 1123void MarkSweep::SweepArray(accounting::ObjectStack* allocations, bool swap_bitmaps) { 1124 space::DlMallocSpace* space = heap_->GetAllocSpace(); 1125 timings_.StartSplit("SweepArray"); 1126 // Newly allocated objects MUST be in the alloc space and those are the only objects which we are 1127 // going to free. 1128 accounting::SpaceBitmap* live_bitmap = space->GetLiveBitmap(); 1129 accounting::SpaceBitmap* mark_bitmap = space->GetMarkBitmap(); 1130 space::LargeObjectSpace* large_object_space = GetHeap()->GetLargeObjectsSpace(); 1131 accounting::SpaceSetMap* large_live_objects = large_object_space->GetLiveObjects(); 1132 accounting::SpaceSetMap* large_mark_objects = large_object_space->GetMarkObjects(); 1133 if (swap_bitmaps) { 1134 std::swap(live_bitmap, mark_bitmap); 1135 std::swap(large_live_objects, large_mark_objects); 1136 } 1137 1138 size_t freed_bytes = 0; 1139 size_t freed_large_object_bytes = 0; 1140 size_t freed_objects = 0; 1141 size_t freed_large_objects = 0; 1142 size_t count = allocations->Size(); 1143 Object** objects = const_cast<Object**>(allocations->Begin()); 1144 Object** out = objects; 1145 Object** objects_to_chunk_free = out; 1146 1147 // Empty the allocation stack. 1148 Thread* self = Thread::Current(); 1149 for (size_t i = 0; i < count; ++i) { 1150 Object* obj = objects[i]; 1151 // There should only be objects in the AllocSpace/LargeObjectSpace in the allocation stack. 1152 if (LIKELY(mark_bitmap->HasAddress(obj))) { 1153 if (!mark_bitmap->Test(obj)) { 1154 // Don't bother un-marking since we clear the mark bitmap anyways. 1155 *(out++) = obj; 1156 // Free objects in chunks. 1157 DCHECK_GE(out, objects_to_chunk_free); 1158 DCHECK_LE(static_cast<size_t>(out - objects_to_chunk_free), kSweepArrayChunkFreeSize); 1159 if (static_cast<size_t>(out - objects_to_chunk_free) == kSweepArrayChunkFreeSize) { 1160 timings_.StartSplit("FreeList"); 1161 size_t chunk_freed_objects = out - objects_to_chunk_free; 1162 freed_objects += chunk_freed_objects; 1163 freed_bytes += space->FreeList(self, chunk_freed_objects, objects_to_chunk_free); 1164 objects_to_chunk_free = out; 1165 timings_.EndSplit(); 1166 } 1167 } 1168 } else if (!large_mark_objects->Test(obj)) { 1169 ++freed_large_objects; 1170 freed_large_object_bytes += large_object_space->Free(self, obj); 1171 } 1172 } 1173 // Free the remaining objects in chunks. 1174 DCHECK_GE(out, objects_to_chunk_free); 1175 DCHECK_LE(static_cast<size_t>(out - objects_to_chunk_free), kSweepArrayChunkFreeSize); 1176 if (out - objects_to_chunk_free > 0) { 1177 timings_.StartSplit("FreeList"); 1178 size_t chunk_freed_objects = out - objects_to_chunk_free; 1179 freed_objects += chunk_freed_objects; 1180 freed_bytes += space->FreeList(self, chunk_freed_objects, objects_to_chunk_free); 1181 timings_.EndSplit(); 1182 } 1183 CHECK_EQ(count, allocations->Size()); 1184 timings_.EndSplit(); 1185 1186 timings_.StartSplit("RecordFree"); 1187 VLOG(heap) << "Freed " << freed_objects << "/" << count 1188 << " objects with size " << PrettySize(freed_bytes); 1189 heap_->RecordFree(freed_objects + freed_large_objects, freed_bytes + freed_large_object_bytes); 1190 freed_objects_.fetch_add(freed_objects); 1191 freed_large_objects_.fetch_add(freed_large_objects); 1192 freed_bytes_.fetch_add(freed_bytes); 1193 freed_large_object_bytes_.fetch_add(freed_large_object_bytes); 1194 timings_.EndSplit(); 1195 1196 timings_.StartSplit("ResetStack"); 1197 allocations->Reset(); 1198 timings_.EndSplit(); 1199} 1200 1201void MarkSweep::Sweep(bool swap_bitmaps) { 1202 DCHECK(mark_stack_->IsEmpty()); 1203 base::TimingLogger::ScopedSplit("Sweep", &timings_); 1204 1205 const bool partial = (GetGcType() == kGcTypePartial); 1206 SweepCallbackContext scc; 1207 scc.mark_sweep = this; 1208 scc.self = Thread::Current(); 1209 for (const auto& space : GetHeap()->GetContinuousSpaces()) { 1210 // We always sweep always collect spaces. 1211 bool sweep_space = (space->GetGcRetentionPolicy() == space::kGcRetentionPolicyAlwaysCollect); 1212 if (!partial && !sweep_space) { 1213 // We sweep full collect spaces when the GC isn't a partial GC (ie its full). 1214 sweep_space = (space->GetGcRetentionPolicy() == space::kGcRetentionPolicyFullCollect); 1215 } 1216 if (sweep_space) { 1217 uintptr_t begin = reinterpret_cast<uintptr_t>(space->Begin()); 1218 uintptr_t end = reinterpret_cast<uintptr_t>(space->End()); 1219 scc.space = space->AsDlMallocSpace(); 1220 accounting::SpaceBitmap* live_bitmap = space->GetLiveBitmap(); 1221 accounting::SpaceBitmap* mark_bitmap = space->GetMarkBitmap(); 1222 if (swap_bitmaps) { 1223 std::swap(live_bitmap, mark_bitmap); 1224 } 1225 if (!space->IsZygoteSpace()) { 1226 base::TimingLogger::ScopedSplit split("SweepAllocSpace", &timings_); 1227 // Bitmaps are pre-swapped for optimization which enables sweeping with the heap unlocked. 1228 accounting::SpaceBitmap::SweepWalk(*live_bitmap, *mark_bitmap, begin, end, 1229 &SweepCallback, reinterpret_cast<void*>(&scc)); 1230 } else { 1231 base::TimingLogger::ScopedSplit split("SweepZygote", &timings_); 1232 // Zygote sweep takes care of dirtying cards and clearing live bits, does not free actual 1233 // memory. 1234 accounting::SpaceBitmap::SweepWalk(*live_bitmap, *mark_bitmap, begin, end, 1235 &ZygoteSweepCallback, reinterpret_cast<void*>(&scc)); 1236 } 1237 } 1238 } 1239 1240 SweepLargeObjects(swap_bitmaps); 1241} 1242 1243void MarkSweep::SweepLargeObjects(bool swap_bitmaps) { 1244 base::TimingLogger::ScopedSplit("SweepLargeObjects", &timings_); 1245 // Sweep large objects 1246 space::LargeObjectSpace* large_object_space = GetHeap()->GetLargeObjectsSpace(); 1247 accounting::SpaceSetMap* large_live_objects = large_object_space->GetLiveObjects(); 1248 accounting::SpaceSetMap* large_mark_objects = large_object_space->GetMarkObjects(); 1249 if (swap_bitmaps) { 1250 std::swap(large_live_objects, large_mark_objects); 1251 } 1252 // O(n*log(n)) but hopefully there are not too many large objects. 1253 size_t freed_objects = 0; 1254 size_t freed_bytes = 0; 1255 Thread* self = Thread::Current(); 1256 for (const Object* obj : large_live_objects->GetObjects()) { 1257 if (!large_mark_objects->Test(obj)) { 1258 freed_bytes += large_object_space->Free(self, const_cast<Object*>(obj)); 1259 ++freed_objects; 1260 } 1261 } 1262 freed_large_objects_.fetch_add(freed_objects); 1263 freed_large_object_bytes_.fetch_add(freed_bytes); 1264 GetHeap()->RecordFree(freed_objects, freed_bytes); 1265} 1266 1267void MarkSweep::CheckReference(const Object* obj, const Object* ref, MemberOffset offset, bool is_static) { 1268 for (const auto& space : GetHeap()->GetContinuousSpaces()) { 1269 if (space->IsDlMallocSpace() && space->Contains(ref)) { 1270 DCHECK(IsMarked(obj)); 1271 1272 bool is_marked = IsMarked(ref); 1273 if (!is_marked) { 1274 LOG(INFO) << *space; 1275 LOG(WARNING) << (is_static ? "Static ref'" : "Instance ref'") << PrettyTypeOf(ref) 1276 << "' (" << reinterpret_cast<const void*>(ref) << ") in '" << PrettyTypeOf(obj) 1277 << "' (" << reinterpret_cast<const void*>(obj) << ") at offset " 1278 << reinterpret_cast<void*>(offset.Int32Value()) << " wasn't marked"; 1279 1280 const Class* klass = is_static ? obj->AsClass() : obj->GetClass(); 1281 DCHECK(klass != NULL); 1282 const ObjectArray<ArtField>* fields = is_static ? klass->GetSFields() : klass->GetIFields(); 1283 DCHECK(fields != NULL); 1284 bool found = false; 1285 for (int32_t i = 0; i < fields->GetLength(); ++i) { 1286 const ArtField* cur = fields->Get(i); 1287 if (cur->GetOffset().Int32Value() == offset.Int32Value()) { 1288 LOG(WARNING) << "Field referencing the alloc space was " << PrettyField(cur); 1289 found = true; 1290 break; 1291 } 1292 } 1293 if (!found) { 1294 LOG(WARNING) << "Could not find field in object alloc space with offset " << offset.Int32Value(); 1295 } 1296 1297 bool obj_marked = heap_->GetCardTable()->IsDirty(obj); 1298 if (!obj_marked) { 1299 LOG(WARNING) << "Object '" << PrettyTypeOf(obj) << "' " 1300 << "(" << reinterpret_cast<const void*>(obj) << ") contains references to " 1301 << "the alloc space, but wasn't card marked"; 1302 } 1303 } 1304 } 1305 break; 1306 } 1307} 1308 1309// Process the "referent" field in a java.lang.ref.Reference. If the 1310// referent has not yet been marked, put it on the appropriate list in 1311// the heap for later processing. 1312void MarkSweep::DelayReferenceReferent(mirror::Class* klass, Object* obj) { 1313 DCHECK(klass != nullptr); 1314 DCHECK(klass->IsReferenceClass()); 1315 DCHECK(obj != NULL); 1316 Object* referent = heap_->GetReferenceReferent(obj); 1317 if (referent != NULL && !IsMarked(referent)) { 1318 if (kCountJavaLangRefs) { 1319 ++reference_count_; 1320 } 1321 Thread* self = Thread::Current(); 1322 // TODO: Remove these locks, and use atomic stacks for storing references? 1323 // We need to check that the references haven't already been enqueued since we can end up 1324 // scanning the same reference multiple times due to dirty cards. 1325 if (klass->IsSoftReferenceClass()) { 1326 MutexLock mu(self, *heap_->GetSoftRefQueueLock()); 1327 if (!heap_->IsEnqueued(obj)) { 1328 heap_->EnqueuePendingReference(obj, &soft_reference_list_); 1329 } 1330 } else if (klass->IsWeakReferenceClass()) { 1331 MutexLock mu(self, *heap_->GetWeakRefQueueLock()); 1332 if (!heap_->IsEnqueued(obj)) { 1333 heap_->EnqueuePendingReference(obj, &weak_reference_list_); 1334 } 1335 } else if (klass->IsFinalizerReferenceClass()) { 1336 MutexLock mu(self, *heap_->GetFinalizerRefQueueLock()); 1337 if (!heap_->IsEnqueued(obj)) { 1338 heap_->EnqueuePendingReference(obj, &finalizer_reference_list_); 1339 } 1340 } else if (klass->IsPhantomReferenceClass()) { 1341 MutexLock mu(self, *heap_->GetPhantomRefQueueLock()); 1342 if (!heap_->IsEnqueued(obj)) { 1343 heap_->EnqueuePendingReference(obj, &phantom_reference_list_); 1344 } 1345 } else { 1346 LOG(FATAL) << "Invalid reference type " << PrettyClass(klass) 1347 << " " << std::hex << klass->GetAccessFlags(); 1348 } 1349 } 1350} 1351 1352class MarkObjectVisitor { 1353 public: 1354 explicit MarkObjectVisitor(MarkSweep* const mark_sweep) ALWAYS_INLINE : mark_sweep_(mark_sweep) {} 1355 1356 // TODO: Fixme when anotatalysis works with visitors. 1357 void operator()(const Object* /* obj */, const Object* ref, const MemberOffset& /* offset */, 1358 bool /* is_static */) const ALWAYS_INLINE 1359 NO_THREAD_SAFETY_ANALYSIS { 1360 if (kCheckLocks) { 1361 Locks::mutator_lock_->AssertSharedHeld(Thread::Current()); 1362 Locks::heap_bitmap_lock_->AssertExclusiveHeld(Thread::Current()); 1363 } 1364 mark_sweep_->MarkObject(ref); 1365 } 1366 1367 private: 1368 MarkSweep* const mark_sweep_; 1369}; 1370 1371// Scans an object reference. Determines the type of the reference 1372// and dispatches to a specialized scanning routine. 1373void MarkSweep::ScanObject(const Object* obj) { 1374 MarkObjectVisitor visitor(this); 1375 ScanObjectVisit(const_cast<Object*>(obj), visitor); 1376} 1377 1378void MarkSweep::ProcessMarkStackParallel(size_t thread_count) { 1379 Thread* self = Thread::Current(); 1380 ThreadPool* thread_pool = GetHeap()->GetThreadPool(); 1381 const size_t chunk_size = std::min(mark_stack_->Size() / thread_count + 1, 1382 static_cast<size_t>(MarkStackTask<false>::kMaxSize)); 1383 CHECK_GT(chunk_size, 0U); 1384 // Split the current mark stack up into work tasks. 1385 for (mirror::Object **it = mark_stack_->Begin(), **end = mark_stack_->End(); it < end; ) { 1386 const size_t delta = std::min(static_cast<size_t>(end - it), chunk_size); 1387 thread_pool->AddTask(self, new MarkStackTask<false>(thread_pool, this, delta, 1388 const_cast<const mirror::Object**>(it))); 1389 it += delta; 1390 } 1391 thread_pool->SetMaxActiveWorkers(thread_count - 1); 1392 thread_pool->StartWorkers(self); 1393 thread_pool->Wait(self, true, true); 1394 thread_pool->StopWorkers(self); 1395 mark_stack_->Reset(); 1396 CHECK_EQ(work_chunks_created_, work_chunks_deleted_) << " some of the work chunks were leaked"; 1397} 1398 1399// Scan anything that's on the mark stack. 1400void MarkSweep::ProcessMarkStack(bool paused) { 1401 timings_.StartSplit("ProcessMarkStack"); 1402 size_t thread_count = GetThreadCount(paused); 1403 if (kParallelProcessMarkStack && thread_count > 1 && 1404 mark_stack_->Size() >= kMinimumParallelMarkStackSize) { 1405 ProcessMarkStackParallel(thread_count); 1406 } else { 1407 // TODO: Tune this. 1408 static const size_t kFifoSize = 4; 1409 BoundedFifoPowerOfTwo<const Object*, kFifoSize> prefetch_fifo; 1410 for (;;) { 1411 const Object* obj = NULL; 1412 if (kUseMarkStackPrefetch) { 1413 while (!mark_stack_->IsEmpty() && prefetch_fifo.size() < kFifoSize) { 1414 const Object* obj = mark_stack_->PopBack(); 1415 DCHECK(obj != NULL); 1416 __builtin_prefetch(obj); 1417 prefetch_fifo.push_back(obj); 1418 } 1419 if (prefetch_fifo.empty()) { 1420 break; 1421 } 1422 obj = prefetch_fifo.front(); 1423 prefetch_fifo.pop_front(); 1424 } else { 1425 if (mark_stack_->IsEmpty()) { 1426 break; 1427 } 1428 obj = mark_stack_->PopBack(); 1429 } 1430 DCHECK(obj != NULL); 1431 ScanObject(obj); 1432 } 1433 } 1434 timings_.EndSplit(); 1435} 1436 1437// Walks the reference list marking any references subject to the 1438// reference clearing policy. References with a black referent are 1439// removed from the list. References with white referents biased 1440// toward saving are blackened and also removed from the list. 1441void MarkSweep::PreserveSomeSoftReferences(Object** list) { 1442 DCHECK(list != NULL); 1443 Object* clear = NULL; 1444 size_t counter = 0; 1445 1446 DCHECK(mark_stack_->IsEmpty()); 1447 1448 timings_.StartSplit("PreserveSomeSoftReferences"); 1449 while (*list != NULL) { 1450 Object* ref = heap_->DequeuePendingReference(list); 1451 Object* referent = heap_->GetReferenceReferent(ref); 1452 if (referent == NULL) { 1453 // Referent was cleared by the user during marking. 1454 continue; 1455 } 1456 bool is_marked = IsMarked(referent); 1457 if (!is_marked && ((++counter) & 1)) { 1458 // Referent is white and biased toward saving, mark it. 1459 MarkObject(referent); 1460 is_marked = true; 1461 } 1462 if (!is_marked) { 1463 // Referent is white, queue it for clearing. 1464 heap_->EnqueuePendingReference(ref, &clear); 1465 } 1466 } 1467 *list = clear; 1468 timings_.EndSplit(); 1469 1470 // Restart the mark with the newly black references added to the root set. 1471 ProcessMarkStack(true); 1472} 1473 1474inline bool MarkSweep::IsMarked(const Object* object) const 1475 SHARED_LOCKS_REQUIRED(Locks::heap_bitmap_lock_) { 1476 if (IsImmune(object)) { 1477 return true; 1478 } 1479 DCHECK(current_mark_bitmap_ != NULL); 1480 if (current_mark_bitmap_->HasAddress(object)) { 1481 return current_mark_bitmap_->Test(object); 1482 } 1483 return heap_->GetMarkBitmap()->Test(object); 1484} 1485 1486// Unlink the reference list clearing references objects with white 1487// referents. Cleared references registered to a reference queue are 1488// scheduled for appending by the heap worker thread. 1489void MarkSweep::ClearWhiteReferences(Object** list) { 1490 DCHECK(list != NULL); 1491 while (*list != NULL) { 1492 Object* ref = heap_->DequeuePendingReference(list); 1493 Object* referent = heap_->GetReferenceReferent(ref); 1494 if (referent != NULL && !IsMarked(referent)) { 1495 // Referent is white, clear it. 1496 heap_->ClearReferenceReferent(ref); 1497 if (heap_->IsEnqueuable(ref)) { 1498 heap_->EnqueueReference(ref, &cleared_reference_list_); 1499 } 1500 } 1501 } 1502 DCHECK(*list == NULL); 1503} 1504 1505// Enqueues finalizer references with white referents. White 1506// referents are blackened, moved to the zombie field, and the 1507// referent field is cleared. 1508void MarkSweep::EnqueueFinalizerReferences(Object** list) { 1509 DCHECK(list != NULL); 1510 timings_.StartSplit("EnqueueFinalizerReferences"); 1511 MemberOffset zombie_offset = heap_->GetFinalizerReferenceZombieOffset(); 1512 bool has_enqueued = false; 1513 while (*list != NULL) { 1514 Object* ref = heap_->DequeuePendingReference(list); 1515 Object* referent = heap_->GetReferenceReferent(ref); 1516 if (referent != NULL && !IsMarked(referent)) { 1517 MarkObject(referent); 1518 // If the referent is non-null the reference must queuable. 1519 DCHECK(heap_->IsEnqueuable(ref)); 1520 ref->SetFieldObject(zombie_offset, referent, false); 1521 heap_->ClearReferenceReferent(ref); 1522 heap_->EnqueueReference(ref, &cleared_reference_list_); 1523 has_enqueued = true; 1524 } 1525 } 1526 timings_.EndSplit(); 1527 if (has_enqueued) { 1528 ProcessMarkStack(true); 1529 } 1530 DCHECK(*list == NULL); 1531} 1532 1533// Process reference class instances and schedule finalizations. 1534void MarkSweep::ProcessReferences(Object** soft_references, bool clear_soft, 1535 Object** weak_references, 1536 Object** finalizer_references, 1537 Object** phantom_references) { 1538 CHECK(soft_references != NULL); 1539 CHECK(weak_references != NULL); 1540 CHECK(finalizer_references != NULL); 1541 CHECK(phantom_references != NULL); 1542 CHECK(mark_stack_->IsEmpty()); 1543 1544 // Unless we are in the zygote or required to clear soft references 1545 // with white references, preserve some white referents. 1546 if (!clear_soft && !Runtime::Current()->IsZygote()) { 1547 PreserveSomeSoftReferences(soft_references); 1548 } 1549 1550 timings_.StartSplit("ProcessReferences"); 1551 // Clear all remaining soft and weak references with white 1552 // referents. 1553 ClearWhiteReferences(soft_references); 1554 ClearWhiteReferences(weak_references); 1555 timings_.EndSplit(); 1556 1557 // Preserve all white objects with finalize methods and schedule 1558 // them for finalization. 1559 EnqueueFinalizerReferences(finalizer_references); 1560 1561 timings_.StartSplit("ProcessReferences"); 1562 // Clear all f-reachable soft and weak references with white 1563 // referents. 1564 ClearWhiteReferences(soft_references); 1565 ClearWhiteReferences(weak_references); 1566 1567 // Clear all phantom references with white referents. 1568 ClearWhiteReferences(phantom_references); 1569 1570 // At this point all reference lists should be empty. 1571 DCHECK(*soft_references == NULL); 1572 DCHECK(*weak_references == NULL); 1573 DCHECK(*finalizer_references == NULL); 1574 DCHECK(*phantom_references == NULL); 1575 timings_.EndSplit(); 1576} 1577 1578void MarkSweep::UnBindBitmaps() { 1579 base::TimingLogger::ScopedSplit split("UnBindBitmaps", &timings_); 1580 for (const auto& space : GetHeap()->GetContinuousSpaces()) { 1581 if (space->IsDlMallocSpace()) { 1582 space::DlMallocSpace* alloc_space = space->AsDlMallocSpace(); 1583 if (alloc_space->temp_bitmap_.get() != NULL) { 1584 // At this point, the temp_bitmap holds our old mark bitmap. 1585 accounting::SpaceBitmap* new_bitmap = alloc_space->temp_bitmap_.release(); 1586 GetHeap()->GetMarkBitmap()->ReplaceBitmap(alloc_space->mark_bitmap_.get(), new_bitmap); 1587 CHECK_EQ(alloc_space->mark_bitmap_.release(), alloc_space->live_bitmap_.get()); 1588 alloc_space->mark_bitmap_.reset(new_bitmap); 1589 DCHECK(alloc_space->temp_bitmap_.get() == NULL); 1590 } 1591 } 1592 } 1593} 1594 1595void MarkSweep::FinishPhase() { 1596 base::TimingLogger::ScopedSplit split("FinishPhase", &timings_); 1597 // Can't enqueue references if we hold the mutator lock. 1598 Object* cleared_references = GetClearedReferences(); 1599 Heap* heap = GetHeap(); 1600 timings_.NewSplit("EnqueueClearedReferences"); 1601 heap->EnqueueClearedReferences(&cleared_references); 1602 1603 timings_.NewSplit("PostGcVerification"); 1604 heap->PostGcVerification(this); 1605 1606 timings_.NewSplit("GrowForUtilization"); 1607 heap->GrowForUtilization(GetGcType(), GetDurationNs()); 1608 1609 timings_.NewSplit("RequestHeapTrim"); 1610 heap->RequestHeapTrim(); 1611 1612 // Update the cumulative statistics 1613 total_time_ns_ += GetDurationNs(); 1614 total_paused_time_ns_ += std::accumulate(GetPauseTimes().begin(), GetPauseTimes().end(), 0, 1615 std::plus<uint64_t>()); 1616 total_freed_objects_ += GetFreedObjects() + GetFreedLargeObjects(); 1617 total_freed_bytes_ += GetFreedBytes() + GetFreedLargeObjectBytes(); 1618 1619 // Ensure that the mark stack is empty. 1620 CHECK(mark_stack_->IsEmpty()); 1621 1622 if (kCountScannedTypes) { 1623 VLOG(gc) << "MarkSweep scanned classes=" << class_count_ << " arrays=" << array_count_ 1624 << " other=" << other_count_; 1625 } 1626 1627 if (kCountTasks) { 1628 VLOG(gc) << "Total number of work chunks allocated: " << work_chunks_created_; 1629 } 1630 1631 if (kMeasureOverhead) { 1632 VLOG(gc) << "Overhead time " << PrettyDuration(overhead_time_); 1633 } 1634 1635 if (kProfileLargeObjects) { 1636 VLOG(gc) << "Large objects tested " << large_object_test_ << " marked " << large_object_mark_; 1637 } 1638 1639 if (kCountClassesMarked) { 1640 VLOG(gc) << "Classes marked " << classes_marked_; 1641 } 1642 1643 if (kCountJavaLangRefs) { 1644 VLOG(gc) << "References scanned " << reference_count_; 1645 } 1646 1647 // Update the cumulative loggers. 1648 cumulative_timings_.Start(); 1649 cumulative_timings_.AddLogger(timings_); 1650 cumulative_timings_.End(); 1651 1652 // Clear all of the spaces' mark bitmaps. 1653 for (const auto& space : GetHeap()->GetContinuousSpaces()) { 1654 if (space->GetGcRetentionPolicy() != space::kGcRetentionPolicyNeverCollect) { 1655 space->GetMarkBitmap()->Clear(); 1656 } 1657 } 1658 mark_stack_->Reset(); 1659 1660 // Reset the marked large objects. 1661 space::LargeObjectSpace* large_objects = GetHeap()->GetLargeObjectsSpace(); 1662 large_objects->GetMarkObjects()->Clear(); 1663} 1664 1665} // namespace collector 1666} // namespace gc 1667} // namespace art 1668