concurrent_copying.cc revision 951ec2c93c79c5539cbcc669566f0808d4460338
1/* 2 * Copyright (C) 2014 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17#include "concurrent_copying.h" 18 19#include "art_field-inl.h" 20#include "base/stl_util.h" 21#include "gc/accounting/heap_bitmap-inl.h" 22#include "gc/accounting/space_bitmap-inl.h" 23#include "gc/reference_processor.h" 24#include "gc/space/image_space.h" 25#include "gc/space/space.h" 26#include "intern_table.h" 27#include "mirror/class-inl.h" 28#include "mirror/object-inl.h" 29#include "scoped_thread_state_change.h" 30#include "thread-inl.h" 31#include "thread_list.h" 32#include "well_known_classes.h" 33 34namespace art { 35namespace gc { 36namespace collector { 37 38ConcurrentCopying::ConcurrentCopying(Heap* heap, const std::string& name_prefix) 39 : GarbageCollector(heap, 40 name_prefix + (name_prefix.empty() ? "" : " ") + 41 "concurrent copying + mark sweep"), 42 region_space_(nullptr), gc_barrier_(new Barrier(0)), 43 gc_mark_stack_(accounting::ObjectStack::Create("concurrent copying gc mark stack", 44 2 * MB, 2 * MB)), 45 mark_stack_lock_("concurrent copying mark stack lock", kMarkSweepMarkStackLock), 46 thread_running_gc_(nullptr), 47 is_marking_(false), is_active_(false), is_asserting_to_space_invariant_(false), 48 heap_mark_bitmap_(nullptr), live_stack_freeze_size_(0), mark_stack_mode_(kMarkStackModeOff), 49 weak_ref_access_enabled_(true), 50 skipped_blocks_lock_("concurrent copying bytes blocks lock", kMarkSweepMarkStackLock), 51 rb_table_(heap_->GetReadBarrierTable()), 52 force_evacuate_all_(false) { 53 static_assert(space::RegionSpace::kRegionSize == accounting::ReadBarrierTable::kRegionSize, 54 "The region space size and the read barrier table region size must match"); 55 cc_heap_bitmap_.reset(new accounting::HeapBitmap(heap)); 56 Thread* self = Thread::Current(); 57 { 58 ReaderMutexLock mu(self, *Locks::heap_bitmap_lock_); 59 // Cache this so that we won't have to lock heap_bitmap_lock_ in 60 // Mark() which could cause a nested lock on heap_bitmap_lock_ 61 // when GC causes a RB while doing GC or a lock order violation 62 // (class_linker_lock_ and heap_bitmap_lock_). 63 heap_mark_bitmap_ = heap->GetMarkBitmap(); 64 } 65 { 66 MutexLock mu(self, mark_stack_lock_); 67 for (size_t i = 0; i < kMarkStackPoolSize; ++i) { 68 accounting::AtomicStack<mirror::Object>* mark_stack = 69 accounting::AtomicStack<mirror::Object>::Create( 70 "thread local mark stack", kMarkStackSize, kMarkStackSize); 71 pooled_mark_stacks_.push_back(mark_stack); 72 } 73 } 74} 75 76void ConcurrentCopying::MarkHeapReference(mirror::HeapReference<mirror::Object>* from_ref) { 77 // Used for preserving soft references, should be OK to not have a CAS here since there should be 78 // no other threads which can trigger read barriers on the same referent during reference 79 // processing. 80 from_ref->Assign(Mark(from_ref->AsMirrorPtr())); 81 DCHECK(!from_ref->IsNull()); 82} 83 84ConcurrentCopying::~ConcurrentCopying() { 85 STLDeleteElements(&pooled_mark_stacks_); 86} 87 88void ConcurrentCopying::RunPhases() { 89 CHECK(kUseBakerReadBarrier || kUseTableLookupReadBarrier); 90 CHECK(!is_active_); 91 is_active_ = true; 92 Thread* self = Thread::Current(); 93 thread_running_gc_ = self; 94 Locks::mutator_lock_->AssertNotHeld(self); 95 { 96 ReaderMutexLock mu(self, *Locks::mutator_lock_); 97 InitializePhase(); 98 } 99 FlipThreadRoots(); 100 { 101 ReaderMutexLock mu(self, *Locks::mutator_lock_); 102 MarkingPhase(); 103 } 104 // Verify no from space refs. This causes a pause. 105 if (kEnableNoFromSpaceRefsVerification || kIsDebugBuild) { 106 TimingLogger::ScopedTiming split("(Paused)VerifyNoFromSpaceReferences", GetTimings()); 107 ScopedPause pause(this); 108 CheckEmptyMarkStack(); 109 if (kVerboseMode) { 110 LOG(INFO) << "Verifying no from-space refs"; 111 } 112 VerifyNoFromSpaceReferences(); 113 if (kVerboseMode) { 114 LOG(INFO) << "Done verifying no from-space refs"; 115 } 116 CheckEmptyMarkStack(); 117 } 118 { 119 ReaderMutexLock mu(self, *Locks::mutator_lock_); 120 ReclaimPhase(); 121 } 122 FinishPhase(); 123 CHECK(is_active_); 124 is_active_ = false; 125 thread_running_gc_ = nullptr; 126} 127 128void ConcurrentCopying::BindBitmaps() { 129 Thread* self = Thread::Current(); 130 WriterMutexLock mu(self, *Locks::heap_bitmap_lock_); 131 // Mark all of the spaces we never collect as immune. 132 for (const auto& space : heap_->GetContinuousSpaces()) { 133 if (space->GetGcRetentionPolicy() == space::kGcRetentionPolicyNeverCollect 134 || space->GetGcRetentionPolicy() == space::kGcRetentionPolicyFullCollect) { 135 CHECK(space->IsZygoteSpace() || space->IsImageSpace()); 136 CHECK(immune_region_.AddContinuousSpace(space)) << "Failed to add space " << *space; 137 const char* bitmap_name = space->IsImageSpace() ? "cc image space bitmap" : 138 "cc zygote space bitmap"; 139 // TODO: try avoiding using bitmaps for image/zygote to save space. 140 accounting::ContinuousSpaceBitmap* bitmap = 141 accounting::ContinuousSpaceBitmap::Create(bitmap_name, space->Begin(), space->Capacity()); 142 cc_heap_bitmap_->AddContinuousSpaceBitmap(bitmap); 143 cc_bitmaps_.push_back(bitmap); 144 } else if (space == region_space_) { 145 accounting::ContinuousSpaceBitmap* bitmap = 146 accounting::ContinuousSpaceBitmap::Create("cc region space bitmap", 147 space->Begin(), space->Capacity()); 148 cc_heap_bitmap_->AddContinuousSpaceBitmap(bitmap); 149 cc_bitmaps_.push_back(bitmap); 150 region_space_bitmap_ = bitmap; 151 } 152 } 153} 154 155void ConcurrentCopying::InitializePhase() { 156 TimingLogger::ScopedTiming split("InitializePhase", GetTimings()); 157 if (kVerboseMode) { 158 LOG(INFO) << "GC InitializePhase"; 159 LOG(INFO) << "Region-space : " << reinterpret_cast<void*>(region_space_->Begin()) << "-" 160 << reinterpret_cast<void*>(region_space_->Limit()); 161 } 162 CheckEmptyMarkStack(); 163 immune_region_.Reset(); 164 bytes_moved_.StoreRelaxed(0); 165 objects_moved_.StoreRelaxed(0); 166 if (GetCurrentIteration()->GetGcCause() == kGcCauseExplicit || 167 GetCurrentIteration()->GetGcCause() == kGcCauseForNativeAlloc || 168 GetCurrentIteration()->GetClearSoftReferences()) { 169 force_evacuate_all_ = true; 170 } else { 171 force_evacuate_all_ = false; 172 } 173 BindBitmaps(); 174 if (kVerboseMode) { 175 LOG(INFO) << "force_evacuate_all=" << force_evacuate_all_; 176 LOG(INFO) << "Immune region: " << immune_region_.Begin() << "-" << immune_region_.End(); 177 LOG(INFO) << "GC end of InitializePhase"; 178 } 179} 180 181// Used to switch the thread roots of a thread from from-space refs to to-space refs. 182class ThreadFlipVisitor : public Closure { 183 public: 184 ThreadFlipVisitor(ConcurrentCopying* concurrent_copying, bool use_tlab) 185 : concurrent_copying_(concurrent_copying), use_tlab_(use_tlab) { 186 } 187 188 virtual void Run(Thread* thread) OVERRIDE SHARED_REQUIRES(Locks::mutator_lock_) { 189 // Note: self is not necessarily equal to thread since thread may be suspended. 190 Thread* self = Thread::Current(); 191 CHECK(thread == self || thread->IsSuspended() || thread->GetState() == kWaitingPerformingGc) 192 << thread->GetState() << " thread " << thread << " self " << self; 193 thread->SetIsGcMarking(true); 194 if (use_tlab_ && thread->HasTlab()) { 195 if (ConcurrentCopying::kEnableFromSpaceAccountingCheck) { 196 // This must come before the revoke. 197 size_t thread_local_objects = thread->GetThreadLocalObjectsAllocated(); 198 concurrent_copying_->region_space_->RevokeThreadLocalBuffers(thread); 199 reinterpret_cast<Atomic<size_t>*>(&concurrent_copying_->from_space_num_objects_at_first_pause_)-> 200 FetchAndAddSequentiallyConsistent(thread_local_objects); 201 } else { 202 concurrent_copying_->region_space_->RevokeThreadLocalBuffers(thread); 203 } 204 } 205 if (kUseThreadLocalAllocationStack) { 206 thread->RevokeThreadLocalAllocationStack(); 207 } 208 ReaderMutexLock mu(self, *Locks::heap_bitmap_lock_); 209 thread->VisitRoots(concurrent_copying_); 210 concurrent_copying_->GetBarrier().Pass(self); 211 } 212 213 private: 214 ConcurrentCopying* const concurrent_copying_; 215 const bool use_tlab_; 216}; 217 218// Called back from Runtime::FlipThreadRoots() during a pause. 219class FlipCallback : public Closure { 220 public: 221 explicit FlipCallback(ConcurrentCopying* concurrent_copying) 222 : concurrent_copying_(concurrent_copying) { 223 } 224 225 virtual void Run(Thread* thread) OVERRIDE REQUIRES(Locks::mutator_lock_) { 226 ConcurrentCopying* cc = concurrent_copying_; 227 TimingLogger::ScopedTiming split("(Paused)FlipCallback", cc->GetTimings()); 228 // Note: self is not necessarily equal to thread since thread may be suspended. 229 Thread* self = Thread::Current(); 230 CHECK(thread == self); 231 Locks::mutator_lock_->AssertExclusiveHeld(self); 232 cc->region_space_->SetFromSpace(cc->rb_table_, cc->force_evacuate_all_); 233 cc->SwapStacks(); 234 if (ConcurrentCopying::kEnableFromSpaceAccountingCheck) { 235 cc->RecordLiveStackFreezeSize(self); 236 cc->from_space_num_objects_at_first_pause_ = cc->region_space_->GetObjectsAllocated(); 237 cc->from_space_num_bytes_at_first_pause_ = cc->region_space_->GetBytesAllocated(); 238 } 239 cc->is_marking_ = true; 240 cc->mark_stack_mode_.StoreRelaxed(ConcurrentCopying::kMarkStackModeThreadLocal); 241 if (UNLIKELY(Runtime::Current()->IsActiveTransaction())) { 242 CHECK(Runtime::Current()->IsAotCompiler()); 243 TimingLogger::ScopedTiming split2("(Paused)VisitTransactionRoots", cc->GetTimings()); 244 Runtime::Current()->VisitTransactionRoots(cc); 245 } 246 } 247 248 private: 249 ConcurrentCopying* const concurrent_copying_; 250}; 251 252// Switch threads that from from-space to to-space refs. Forward/mark the thread roots. 253void ConcurrentCopying::FlipThreadRoots() { 254 TimingLogger::ScopedTiming split("FlipThreadRoots", GetTimings()); 255 if (kVerboseMode) { 256 LOG(INFO) << "time=" << region_space_->Time(); 257 region_space_->DumpNonFreeRegions(LOG(INFO)); 258 } 259 Thread* self = Thread::Current(); 260 Locks::mutator_lock_->AssertNotHeld(self); 261 gc_barrier_->Init(self, 0); 262 ThreadFlipVisitor thread_flip_visitor(this, heap_->use_tlab_); 263 FlipCallback flip_callback(this); 264 heap_->ThreadFlipBegin(self); // Sync with JNI critical calls. 265 size_t barrier_count = Runtime::Current()->FlipThreadRoots( 266 &thread_flip_visitor, &flip_callback, this); 267 heap_->ThreadFlipEnd(self); 268 { 269 ScopedThreadStateChange tsc(self, kWaitingForCheckPointsToRun); 270 gc_barrier_->Increment(self, barrier_count); 271 } 272 is_asserting_to_space_invariant_ = true; 273 QuasiAtomic::ThreadFenceForConstructor(); 274 if (kVerboseMode) { 275 LOG(INFO) << "time=" << region_space_->Time(); 276 region_space_->DumpNonFreeRegions(LOG(INFO)); 277 LOG(INFO) << "GC end of FlipThreadRoots"; 278 } 279} 280 281void ConcurrentCopying::SwapStacks() { 282 heap_->SwapStacks(); 283} 284 285void ConcurrentCopying::RecordLiveStackFreezeSize(Thread* self) { 286 WriterMutexLock mu(self, *Locks::heap_bitmap_lock_); 287 live_stack_freeze_size_ = heap_->GetLiveStack()->Size(); 288} 289 290// Used to visit objects in the immune spaces. 291class ConcurrentCopyingImmuneSpaceObjVisitor { 292 public: 293 explicit ConcurrentCopyingImmuneSpaceObjVisitor(ConcurrentCopying* cc) 294 : collector_(cc) {} 295 296 void operator()(mirror::Object* obj) const SHARED_REQUIRES(Locks::mutator_lock_) 297 SHARED_REQUIRES(Locks::heap_bitmap_lock_) { 298 DCHECK(obj != nullptr); 299 DCHECK(collector_->immune_region_.ContainsObject(obj)); 300 accounting::ContinuousSpaceBitmap* cc_bitmap = 301 collector_->cc_heap_bitmap_->GetContinuousSpaceBitmap(obj); 302 DCHECK(cc_bitmap != nullptr) 303 << "An immune space object must have a bitmap"; 304 if (kIsDebugBuild) { 305 DCHECK(collector_->heap_->GetMarkBitmap()->Test(obj)) 306 << "Immune space object must be already marked"; 307 } 308 // This may or may not succeed, which is ok. 309 if (kUseBakerReadBarrier) { 310 obj->AtomicSetReadBarrierPointer(ReadBarrier::WhitePtr(), ReadBarrier::GrayPtr()); 311 } 312 if (cc_bitmap->AtomicTestAndSet(obj)) { 313 // Already marked. Do nothing. 314 } else { 315 // Newly marked. Set the gray bit and push it onto the mark stack. 316 CHECK(!kUseBakerReadBarrier || obj->GetReadBarrierPointer() == ReadBarrier::GrayPtr()); 317 collector_->PushOntoMarkStack(obj); 318 } 319 } 320 321 private: 322 ConcurrentCopying* const collector_; 323}; 324 325class EmptyCheckpoint : public Closure { 326 public: 327 explicit EmptyCheckpoint(ConcurrentCopying* concurrent_copying) 328 : concurrent_copying_(concurrent_copying) { 329 } 330 331 virtual void Run(Thread* thread) OVERRIDE NO_THREAD_SAFETY_ANALYSIS { 332 // Note: self is not necessarily equal to thread since thread may be suspended. 333 Thread* self = Thread::Current(); 334 CHECK(thread == self || thread->IsSuspended() || thread->GetState() == kWaitingPerformingGc) 335 << thread->GetState() << " thread " << thread << " self " << self; 336 // If thread is a running mutator, then act on behalf of the garbage collector. 337 // See the code in ThreadList::RunCheckpoint. 338 if (thread->GetState() == kRunnable) { 339 concurrent_copying_->GetBarrier().Pass(self); 340 } 341 } 342 343 private: 344 ConcurrentCopying* const concurrent_copying_; 345}; 346 347// Concurrently mark roots that are guarded by read barriers and process the mark stack. 348void ConcurrentCopying::MarkingPhase() { 349 TimingLogger::ScopedTiming split("MarkingPhase", GetTimings()); 350 if (kVerboseMode) { 351 LOG(INFO) << "GC MarkingPhase"; 352 } 353 CHECK(weak_ref_access_enabled_); 354 { 355 // Mark the image root. The WB-based collectors do not need to 356 // scan the image objects from roots by relying on the card table, 357 // but it's necessary for the RB to-space invariant to hold. 358 TimingLogger::ScopedTiming split1("VisitImageRoots", GetTimings()); 359 gc::space::ImageSpace* image = heap_->GetImageSpace(); 360 if (image != nullptr) { 361 mirror::ObjectArray<mirror::Object>* image_root = image->GetImageHeader().GetImageRoots(); 362 mirror::Object* marked_image_root = Mark(image_root); 363 CHECK_EQ(image_root, marked_image_root) << "An image object does not move"; 364 if (ReadBarrier::kEnableToSpaceInvariantChecks) { 365 AssertToSpaceInvariant(nullptr, MemberOffset(0), marked_image_root); 366 } 367 } 368 } 369 // TODO: Other garbage collectors uses Runtime::VisitConcurrentRoots(), refactor this part 370 // to also use the same function. 371 { 372 TimingLogger::ScopedTiming split2("VisitConstantRoots", GetTimings()); 373 Runtime::Current()->VisitConstantRoots(this); 374 } 375 { 376 TimingLogger::ScopedTiming split3("VisitInternTableRoots", GetTimings()); 377 Runtime::Current()->GetInternTable()->VisitRoots(this, kVisitRootFlagAllRoots); 378 } 379 { 380 TimingLogger::ScopedTiming split4("VisitClassLinkerRoots", GetTimings()); 381 Runtime::Current()->GetClassLinker()->VisitRoots(this, kVisitRootFlagAllRoots); 382 } 383 { 384 // TODO: don't visit the transaction roots if it's not active. 385 TimingLogger::ScopedTiming split5("VisitNonThreadRoots", GetTimings()); 386 Runtime::Current()->VisitNonThreadRoots(this); 387 } 388 Runtime::Current()->GetHeap()->VisitAllocationRecords(this); 389 390 // Immune spaces. 391 for (auto& space : heap_->GetContinuousSpaces()) { 392 if (immune_region_.ContainsSpace(space)) { 393 DCHECK(space->IsImageSpace() || space->IsZygoteSpace()); 394 accounting::ContinuousSpaceBitmap* live_bitmap = space->GetLiveBitmap(); 395 ConcurrentCopyingImmuneSpaceObjVisitor visitor(this); 396 live_bitmap->VisitMarkedRange(reinterpret_cast<uintptr_t>(space->Begin()), 397 reinterpret_cast<uintptr_t>(space->Limit()), 398 visitor); 399 } 400 } 401 402 Thread* self = Thread::Current(); 403 { 404 TimingLogger::ScopedTiming split6("ProcessMarkStack", GetTimings()); 405 // We transition through three mark stack modes (thread-local, shared, GC-exclusive). The 406 // primary reasons are the fact that we need to use a checkpoint to process thread-local mark 407 // stacks, but after we disable weak refs accesses, we can't use a checkpoint due to a deadlock 408 // issue because running threads potentially blocking at WaitHoldingLocks, and that once we 409 // reach the point where we process weak references, we can avoid using a lock when accessing 410 // the GC mark stack, which makes mark stack processing more efficient. 411 412 // Process the mark stack once in the thread local stack mode. This marks most of the live 413 // objects, aside from weak ref accesses with read barriers (Reference::GetReferent() and system 414 // weaks) that may happen concurrently while we processing the mark stack and newly mark/gray 415 // objects and push refs on the mark stack. 416 ProcessMarkStack(); 417 // Switch to the shared mark stack mode. That is, revoke and process thread-local mark stacks 418 // for the last time before transitioning to the shared mark stack mode, which would process new 419 // refs that may have been concurrently pushed onto the mark stack during the ProcessMarkStack() 420 // call above. At the same time, disable weak ref accesses using a per-thread flag. It's 421 // important to do these together in a single checkpoint so that we can ensure that mutators 422 // won't newly gray objects and push new refs onto the mark stack due to weak ref accesses and 423 // mutators safely transition to the shared mark stack mode (without leaving unprocessed refs on 424 // the thread-local mark stacks), without a race. This is why we use a thread-local weak ref 425 // access flag Thread::tls32_.weak_ref_access_enabled_ instead of the global ones. 426 SwitchToSharedMarkStackMode(); 427 CHECK(!self->GetWeakRefAccessEnabled()); 428 // Now that weak refs accesses are disabled, once we exhaust the shared mark stack again here 429 // (which may be non-empty if there were refs found on thread-local mark stacks during the above 430 // SwitchToSharedMarkStackMode() call), we won't have new refs to process, that is, mutators 431 // (via read barriers) have no way to produce any more refs to process. Marking converges once 432 // before we process weak refs below. 433 ProcessMarkStack(); 434 CheckEmptyMarkStack(); 435 // Switch to the GC exclusive mark stack mode so that we can process the mark stack without a 436 // lock from this point on. 437 SwitchToGcExclusiveMarkStackMode(); 438 CheckEmptyMarkStack(); 439 if (kVerboseMode) { 440 LOG(INFO) << "ProcessReferences"; 441 } 442 // Process weak references. This may produce new refs to process and have them processed via 443 // ProcessMarkStack (in the GC exclusive mark stack mode). 444 ProcessReferences(self); 445 CheckEmptyMarkStack(); 446 if (kVerboseMode) { 447 LOG(INFO) << "SweepSystemWeaks"; 448 } 449 SweepSystemWeaks(self); 450 if (kVerboseMode) { 451 LOG(INFO) << "SweepSystemWeaks done"; 452 } 453 // Process the mark stack here one last time because the above SweepSystemWeaks() call may have 454 // marked some objects (strings alive) as hash_set::Erase() can call the hash function for 455 // arbitrary elements in the weak intern table in InternTable::Table::SweepWeaks(). 456 ProcessMarkStack(); 457 CheckEmptyMarkStack(); 458 // Re-enable weak ref accesses. 459 ReenableWeakRefAccess(self); 460 // Free data for class loaders that we unloaded. 461 Runtime::Current()->GetClassLinker()->CleanupClassLoaders(); 462 // Marking is done. Disable marking. 463 DisableMarking(); 464 CheckEmptyMarkStack(); 465 } 466 467 CHECK(weak_ref_access_enabled_); 468 if (kVerboseMode) { 469 LOG(INFO) << "GC end of MarkingPhase"; 470 } 471} 472 473void ConcurrentCopying::ReenableWeakRefAccess(Thread* self) { 474 if (kVerboseMode) { 475 LOG(INFO) << "ReenableWeakRefAccess"; 476 } 477 weak_ref_access_enabled_.StoreRelaxed(true); // This is for new threads. 478 QuasiAtomic::ThreadFenceForConstructor(); 479 // Iterate all threads (don't need to or can't use a checkpoint) and re-enable weak ref access. 480 { 481 MutexLock mu(self, *Locks::thread_list_lock_); 482 std::list<Thread*> thread_list = Runtime::Current()->GetThreadList()->GetList(); 483 for (Thread* thread : thread_list) { 484 thread->SetWeakRefAccessEnabled(true); 485 } 486 } 487 // Unblock blocking threads. 488 GetHeap()->GetReferenceProcessor()->BroadcastForSlowPath(self); 489 Runtime::Current()->BroadcastForNewSystemWeaks(); 490} 491 492class DisableMarkingCheckpoint : public Closure { 493 public: 494 explicit DisableMarkingCheckpoint(ConcurrentCopying* concurrent_copying) 495 : concurrent_copying_(concurrent_copying) { 496 } 497 498 void Run(Thread* thread) OVERRIDE NO_THREAD_SAFETY_ANALYSIS { 499 // Note: self is not necessarily equal to thread since thread may be suspended. 500 Thread* self = Thread::Current(); 501 DCHECK(thread == self || thread->IsSuspended() || thread->GetState() == kWaitingPerformingGc) 502 << thread->GetState() << " thread " << thread << " self " << self; 503 // Disable the thread-local is_gc_marking flag. 504 // Note a thread that has just started right before this checkpoint may have already this flag 505 // set to false, which is ok. 506 thread->SetIsGcMarking(false); 507 // If thread is a running mutator, then act on behalf of the garbage collector. 508 // See the code in ThreadList::RunCheckpoint. 509 if (thread->GetState() == kRunnable) { 510 concurrent_copying_->GetBarrier().Pass(self); 511 } 512 } 513 514 private: 515 ConcurrentCopying* const concurrent_copying_; 516}; 517 518void ConcurrentCopying::IssueDisableMarkingCheckpoint() { 519 Thread* self = Thread::Current(); 520 DisableMarkingCheckpoint check_point(this); 521 ThreadList* thread_list = Runtime::Current()->GetThreadList(); 522 gc_barrier_->Init(self, 0); 523 size_t barrier_count = thread_list->RunCheckpoint(&check_point); 524 // If there are no threads to wait which implies that all the checkpoint functions are finished, 525 // then no need to release the mutator lock. 526 if (barrier_count == 0) { 527 return; 528 } 529 // Release locks then wait for all mutator threads to pass the barrier. 530 Locks::mutator_lock_->SharedUnlock(self); 531 { 532 ScopedThreadStateChange tsc(self, kWaitingForCheckPointsToRun); 533 gc_barrier_->Increment(self, barrier_count); 534 } 535 Locks::mutator_lock_->SharedLock(self); 536} 537 538void ConcurrentCopying::DisableMarking() { 539 // Change the global is_marking flag to false. Do a fence before doing a checkpoint to update the 540 // thread-local flags so that a new thread starting up will get the correct is_marking flag. 541 is_marking_ = false; 542 QuasiAtomic::ThreadFenceForConstructor(); 543 // Use a checkpoint to turn off the thread-local is_gc_marking flags and to ensure no threads are 544 // still in the middle of a read barrier which may have a from-space ref cached in a local 545 // variable. 546 IssueDisableMarkingCheckpoint(); 547 if (kUseTableLookupReadBarrier) { 548 heap_->rb_table_->ClearAll(); 549 DCHECK(heap_->rb_table_->IsAllCleared()); 550 } 551 is_mark_stack_push_disallowed_.StoreSequentiallyConsistent(1); 552 mark_stack_mode_.StoreSequentiallyConsistent(kMarkStackModeOff); 553} 554 555void ConcurrentCopying::IssueEmptyCheckpoint() { 556 Thread* self = Thread::Current(); 557 EmptyCheckpoint check_point(this); 558 ThreadList* thread_list = Runtime::Current()->GetThreadList(); 559 gc_barrier_->Init(self, 0); 560 size_t barrier_count = thread_list->RunCheckpoint(&check_point); 561 // If there are no threads to wait which implys that all the checkpoint functions are finished, 562 // then no need to release the mutator lock. 563 if (barrier_count == 0) { 564 return; 565 } 566 // Release locks then wait for all mutator threads to pass the barrier. 567 Locks::mutator_lock_->SharedUnlock(self); 568 { 569 ScopedThreadStateChange tsc(self, kWaitingForCheckPointsToRun); 570 gc_barrier_->Increment(self, barrier_count); 571 } 572 Locks::mutator_lock_->SharedLock(self); 573} 574 575void ConcurrentCopying::PushOntoMarkStack(mirror::Object* to_ref) { 576 CHECK_EQ(is_mark_stack_push_disallowed_.LoadRelaxed(), 0) 577 << " " << to_ref << " " << PrettyTypeOf(to_ref); 578 Thread* self = Thread::Current(); // TODO: pass self as an argument from call sites? 579 CHECK(thread_running_gc_ != nullptr); 580 MarkStackMode mark_stack_mode = mark_stack_mode_.LoadRelaxed(); 581 if (mark_stack_mode == kMarkStackModeThreadLocal) { 582 if (self == thread_running_gc_) { 583 // If GC-running thread, use the GC mark stack instead of a thread-local mark stack. 584 CHECK(self->GetThreadLocalMarkStack() == nullptr); 585 CHECK(!gc_mark_stack_->IsFull()); 586 gc_mark_stack_->PushBack(to_ref); 587 } else { 588 // Otherwise, use a thread-local mark stack. 589 accounting::AtomicStack<mirror::Object>* tl_mark_stack = self->GetThreadLocalMarkStack(); 590 if (UNLIKELY(tl_mark_stack == nullptr || tl_mark_stack->IsFull())) { 591 MutexLock mu(self, mark_stack_lock_); 592 // Get a new thread local mark stack. 593 accounting::AtomicStack<mirror::Object>* new_tl_mark_stack; 594 if (!pooled_mark_stacks_.empty()) { 595 // Use a pooled mark stack. 596 new_tl_mark_stack = pooled_mark_stacks_.back(); 597 pooled_mark_stacks_.pop_back(); 598 } else { 599 // None pooled. Create a new one. 600 new_tl_mark_stack = 601 accounting::AtomicStack<mirror::Object>::Create( 602 "thread local mark stack", 4 * KB, 4 * KB); 603 } 604 DCHECK(new_tl_mark_stack != nullptr); 605 DCHECK(new_tl_mark_stack->IsEmpty()); 606 new_tl_mark_stack->PushBack(to_ref); 607 self->SetThreadLocalMarkStack(new_tl_mark_stack); 608 if (tl_mark_stack != nullptr) { 609 // Store the old full stack into a vector. 610 revoked_mark_stacks_.push_back(tl_mark_stack); 611 } 612 } else { 613 tl_mark_stack->PushBack(to_ref); 614 } 615 } 616 } else if (mark_stack_mode == kMarkStackModeShared) { 617 // Access the shared GC mark stack with a lock. 618 MutexLock mu(self, mark_stack_lock_); 619 CHECK(!gc_mark_stack_->IsFull()); 620 gc_mark_stack_->PushBack(to_ref); 621 } else { 622 CHECK_EQ(static_cast<uint32_t>(mark_stack_mode), 623 static_cast<uint32_t>(kMarkStackModeGcExclusive)); 624 CHECK(self == thread_running_gc_) 625 << "Only GC-running thread should access the mark stack " 626 << "in the GC exclusive mark stack mode"; 627 // Access the GC mark stack without a lock. 628 CHECK(!gc_mark_stack_->IsFull()); 629 gc_mark_stack_->PushBack(to_ref); 630 } 631} 632 633accounting::ObjectStack* ConcurrentCopying::GetAllocationStack() { 634 return heap_->allocation_stack_.get(); 635} 636 637accounting::ObjectStack* ConcurrentCopying::GetLiveStack() { 638 return heap_->live_stack_.get(); 639} 640 641inline mirror::Object* ConcurrentCopying::GetFwdPtr(mirror::Object* from_ref) { 642 DCHECK(region_space_->IsInFromSpace(from_ref)); 643 LockWord lw = from_ref->GetLockWord(false); 644 if (lw.GetState() == LockWord::kForwardingAddress) { 645 mirror::Object* fwd_ptr = reinterpret_cast<mirror::Object*>(lw.ForwardingAddress()); 646 CHECK(fwd_ptr != nullptr); 647 return fwd_ptr; 648 } else { 649 return nullptr; 650 } 651} 652 653// The following visitors are that used to verify that there's no 654// references to the from-space left after marking. 655class ConcurrentCopyingVerifyNoFromSpaceRefsVisitor : public SingleRootVisitor { 656 public: 657 explicit ConcurrentCopyingVerifyNoFromSpaceRefsVisitor(ConcurrentCopying* collector) 658 : collector_(collector) {} 659 660 void operator()(mirror::Object* ref) const 661 SHARED_REQUIRES(Locks::mutator_lock_) ALWAYS_INLINE { 662 if (ref == nullptr) { 663 // OK. 664 return; 665 } 666 collector_->AssertToSpaceInvariant(nullptr, MemberOffset(0), ref); 667 if (kUseBakerReadBarrier) { 668 if (collector_->RegionSpace()->IsInToSpace(ref)) { 669 CHECK(ref->GetReadBarrierPointer() == nullptr) 670 << "To-space ref " << ref << " " << PrettyTypeOf(ref) 671 << " has non-white rb_ptr " << ref->GetReadBarrierPointer(); 672 } else { 673 CHECK(ref->GetReadBarrierPointer() == ReadBarrier::BlackPtr() || 674 (ref->GetReadBarrierPointer() == ReadBarrier::WhitePtr() && 675 collector_->IsOnAllocStack(ref))) 676 << "Non-moving/unevac from space ref " << ref << " " << PrettyTypeOf(ref) 677 << " has non-black rb_ptr " << ref->GetReadBarrierPointer() 678 << " but isn't on the alloc stack (and has white rb_ptr)." 679 << " Is it in the non-moving space=" 680 << (collector_->GetHeap()->GetNonMovingSpace()->HasAddress(ref)); 681 } 682 } 683 } 684 685 void VisitRoot(mirror::Object* root, const RootInfo& info ATTRIBUTE_UNUSED) 686 OVERRIDE SHARED_REQUIRES(Locks::mutator_lock_) { 687 DCHECK(root != nullptr); 688 operator()(root); 689 } 690 691 private: 692 ConcurrentCopying* const collector_; 693}; 694 695class ConcurrentCopyingVerifyNoFromSpaceRefsFieldVisitor { 696 public: 697 explicit ConcurrentCopyingVerifyNoFromSpaceRefsFieldVisitor(ConcurrentCopying* collector) 698 : collector_(collector) {} 699 700 void operator()(mirror::Object* obj, MemberOffset offset, bool is_static ATTRIBUTE_UNUSED) const 701 SHARED_REQUIRES(Locks::mutator_lock_) ALWAYS_INLINE { 702 mirror::Object* ref = 703 obj->GetFieldObject<mirror::Object, kDefaultVerifyFlags, kWithoutReadBarrier>(offset); 704 ConcurrentCopyingVerifyNoFromSpaceRefsVisitor visitor(collector_); 705 visitor(ref); 706 } 707 void operator()(mirror::Class* klass, mirror::Reference* ref) const 708 SHARED_REQUIRES(Locks::mutator_lock_) ALWAYS_INLINE { 709 CHECK(klass->IsTypeOfReferenceClass()); 710 this->operator()(ref, mirror::Reference::ReferentOffset(), false); 711 } 712 713 void VisitRootIfNonNull(mirror::CompressedReference<mirror::Object>* root) const 714 SHARED_REQUIRES(Locks::mutator_lock_) { 715 if (!root->IsNull()) { 716 VisitRoot(root); 717 } 718 } 719 720 void VisitRoot(mirror::CompressedReference<mirror::Object>* root) const 721 SHARED_REQUIRES(Locks::mutator_lock_) { 722 ConcurrentCopyingVerifyNoFromSpaceRefsVisitor visitor(collector_); 723 visitor(root->AsMirrorPtr()); 724 } 725 726 private: 727 ConcurrentCopying* const collector_; 728}; 729 730class ConcurrentCopyingVerifyNoFromSpaceRefsObjectVisitor { 731 public: 732 explicit ConcurrentCopyingVerifyNoFromSpaceRefsObjectVisitor(ConcurrentCopying* collector) 733 : collector_(collector) {} 734 void operator()(mirror::Object* obj) const 735 SHARED_REQUIRES(Locks::mutator_lock_) { 736 ObjectCallback(obj, collector_); 737 } 738 static void ObjectCallback(mirror::Object* obj, void *arg) 739 SHARED_REQUIRES(Locks::mutator_lock_) { 740 CHECK(obj != nullptr); 741 ConcurrentCopying* collector = reinterpret_cast<ConcurrentCopying*>(arg); 742 space::RegionSpace* region_space = collector->RegionSpace(); 743 CHECK(!region_space->IsInFromSpace(obj)) << "Scanning object " << obj << " in from space"; 744 ConcurrentCopyingVerifyNoFromSpaceRefsFieldVisitor visitor(collector); 745 obj->VisitReferences(visitor, visitor); 746 if (kUseBakerReadBarrier) { 747 if (collector->RegionSpace()->IsInToSpace(obj)) { 748 CHECK(obj->GetReadBarrierPointer() == nullptr) 749 << "obj=" << obj << " non-white rb_ptr " << obj->GetReadBarrierPointer(); 750 } else { 751 CHECK(obj->GetReadBarrierPointer() == ReadBarrier::BlackPtr() || 752 (obj->GetReadBarrierPointer() == ReadBarrier::WhitePtr() && 753 collector->IsOnAllocStack(obj))) 754 << "Non-moving space/unevac from space ref " << obj << " " << PrettyTypeOf(obj) 755 << " has non-black rb_ptr " << obj->GetReadBarrierPointer() 756 << " but isn't on the alloc stack (and has white rb_ptr). Is it in the non-moving space=" 757 << (collector->GetHeap()->GetNonMovingSpace()->HasAddress(obj)); 758 } 759 } 760 } 761 762 private: 763 ConcurrentCopying* const collector_; 764}; 765 766// Verify there's no from-space references left after the marking phase. 767void ConcurrentCopying::VerifyNoFromSpaceReferences() { 768 Thread* self = Thread::Current(); 769 DCHECK(Locks::mutator_lock_->IsExclusiveHeld(self)); 770 // Verify all threads have is_gc_marking to be false 771 { 772 MutexLock mu(self, *Locks::thread_list_lock_); 773 std::list<Thread*> thread_list = Runtime::Current()->GetThreadList()->GetList(); 774 for (Thread* thread : thread_list) { 775 CHECK(!thread->GetIsGcMarking()); 776 } 777 } 778 ConcurrentCopyingVerifyNoFromSpaceRefsObjectVisitor visitor(this); 779 // Roots. 780 { 781 ReaderMutexLock mu(self, *Locks::heap_bitmap_lock_); 782 ConcurrentCopyingVerifyNoFromSpaceRefsVisitor ref_visitor(this); 783 Runtime::Current()->VisitRoots(&ref_visitor); 784 } 785 // The to-space. 786 region_space_->WalkToSpace(ConcurrentCopyingVerifyNoFromSpaceRefsObjectVisitor::ObjectCallback, 787 this); 788 // Non-moving spaces. 789 { 790 WriterMutexLock mu(self, *Locks::heap_bitmap_lock_); 791 heap_->GetMarkBitmap()->Visit(visitor); 792 } 793 // The alloc stack. 794 { 795 ConcurrentCopyingVerifyNoFromSpaceRefsVisitor ref_visitor(this); 796 for (auto* it = heap_->allocation_stack_->Begin(), *end = heap_->allocation_stack_->End(); 797 it < end; ++it) { 798 mirror::Object* const obj = it->AsMirrorPtr(); 799 if (obj != nullptr && obj->GetClass() != nullptr) { 800 // TODO: need to call this only if obj is alive? 801 ref_visitor(obj); 802 visitor(obj); 803 } 804 } 805 } 806 // TODO: LOS. But only refs in LOS are classes. 807} 808 809// The following visitors are used to assert the to-space invariant. 810class ConcurrentCopyingAssertToSpaceInvariantRefsVisitor { 811 public: 812 explicit ConcurrentCopyingAssertToSpaceInvariantRefsVisitor(ConcurrentCopying* collector) 813 : collector_(collector) {} 814 815 void operator()(mirror::Object* ref) const 816 SHARED_REQUIRES(Locks::mutator_lock_) ALWAYS_INLINE { 817 if (ref == nullptr) { 818 // OK. 819 return; 820 } 821 collector_->AssertToSpaceInvariant(nullptr, MemberOffset(0), ref); 822 } 823 824 private: 825 ConcurrentCopying* const collector_; 826}; 827 828class ConcurrentCopyingAssertToSpaceInvariantFieldVisitor { 829 public: 830 explicit ConcurrentCopyingAssertToSpaceInvariantFieldVisitor(ConcurrentCopying* collector) 831 : collector_(collector) {} 832 833 void operator()(mirror::Object* obj, MemberOffset offset, bool is_static ATTRIBUTE_UNUSED) const 834 SHARED_REQUIRES(Locks::mutator_lock_) ALWAYS_INLINE { 835 mirror::Object* ref = 836 obj->GetFieldObject<mirror::Object, kDefaultVerifyFlags, kWithoutReadBarrier>(offset); 837 ConcurrentCopyingAssertToSpaceInvariantRefsVisitor visitor(collector_); 838 visitor(ref); 839 } 840 void operator()(mirror::Class* klass, mirror::Reference* ref ATTRIBUTE_UNUSED) const 841 SHARED_REQUIRES(Locks::mutator_lock_) ALWAYS_INLINE { 842 CHECK(klass->IsTypeOfReferenceClass()); 843 } 844 845 void VisitRootIfNonNull(mirror::CompressedReference<mirror::Object>* root) const 846 SHARED_REQUIRES(Locks::mutator_lock_) { 847 if (!root->IsNull()) { 848 VisitRoot(root); 849 } 850 } 851 852 void VisitRoot(mirror::CompressedReference<mirror::Object>* root) const 853 SHARED_REQUIRES(Locks::mutator_lock_) { 854 ConcurrentCopyingAssertToSpaceInvariantRefsVisitor visitor(collector_); 855 visitor(root->AsMirrorPtr()); 856 } 857 858 private: 859 ConcurrentCopying* const collector_; 860}; 861 862class ConcurrentCopyingAssertToSpaceInvariantObjectVisitor { 863 public: 864 explicit ConcurrentCopyingAssertToSpaceInvariantObjectVisitor(ConcurrentCopying* collector) 865 : collector_(collector) {} 866 void operator()(mirror::Object* obj) const 867 SHARED_REQUIRES(Locks::mutator_lock_) { 868 ObjectCallback(obj, collector_); 869 } 870 static void ObjectCallback(mirror::Object* obj, void *arg) 871 SHARED_REQUIRES(Locks::mutator_lock_) { 872 CHECK(obj != nullptr); 873 ConcurrentCopying* collector = reinterpret_cast<ConcurrentCopying*>(arg); 874 space::RegionSpace* region_space = collector->RegionSpace(); 875 CHECK(!region_space->IsInFromSpace(obj)) << "Scanning object " << obj << " in from space"; 876 collector->AssertToSpaceInvariant(nullptr, MemberOffset(0), obj); 877 ConcurrentCopyingAssertToSpaceInvariantFieldVisitor visitor(collector); 878 obj->VisitReferences(visitor, visitor); 879 } 880 881 private: 882 ConcurrentCopying* const collector_; 883}; 884 885class RevokeThreadLocalMarkStackCheckpoint : public Closure { 886 public: 887 RevokeThreadLocalMarkStackCheckpoint(ConcurrentCopying* concurrent_copying, 888 bool disable_weak_ref_access) 889 : concurrent_copying_(concurrent_copying), 890 disable_weak_ref_access_(disable_weak_ref_access) { 891 } 892 893 virtual void Run(Thread* thread) OVERRIDE NO_THREAD_SAFETY_ANALYSIS { 894 // Note: self is not necessarily equal to thread since thread may be suspended. 895 Thread* self = Thread::Current(); 896 CHECK(thread == self || thread->IsSuspended() || thread->GetState() == kWaitingPerformingGc) 897 << thread->GetState() << " thread " << thread << " self " << self; 898 // Revoke thread local mark stacks. 899 accounting::AtomicStack<mirror::Object>* tl_mark_stack = thread->GetThreadLocalMarkStack(); 900 if (tl_mark_stack != nullptr) { 901 MutexLock mu(self, concurrent_copying_->mark_stack_lock_); 902 concurrent_copying_->revoked_mark_stacks_.push_back(tl_mark_stack); 903 thread->SetThreadLocalMarkStack(nullptr); 904 } 905 // Disable weak ref access. 906 if (disable_weak_ref_access_) { 907 thread->SetWeakRefAccessEnabled(false); 908 } 909 // If thread is a running mutator, then act on behalf of the garbage collector. 910 // See the code in ThreadList::RunCheckpoint. 911 if (thread->GetState() == kRunnable) { 912 concurrent_copying_->GetBarrier().Pass(self); 913 } 914 } 915 916 private: 917 ConcurrentCopying* const concurrent_copying_; 918 const bool disable_weak_ref_access_; 919}; 920 921void ConcurrentCopying::RevokeThreadLocalMarkStacks(bool disable_weak_ref_access) { 922 Thread* self = Thread::Current(); 923 RevokeThreadLocalMarkStackCheckpoint check_point(this, disable_weak_ref_access); 924 ThreadList* thread_list = Runtime::Current()->GetThreadList(); 925 gc_barrier_->Init(self, 0); 926 size_t barrier_count = thread_list->RunCheckpoint(&check_point); 927 // If there are no threads to wait which implys that all the checkpoint functions are finished, 928 // then no need to release the mutator lock. 929 if (barrier_count == 0) { 930 return; 931 } 932 Locks::mutator_lock_->SharedUnlock(self); 933 { 934 ScopedThreadStateChange tsc(self, kWaitingForCheckPointsToRun); 935 gc_barrier_->Increment(self, barrier_count); 936 } 937 Locks::mutator_lock_->SharedLock(self); 938} 939 940void ConcurrentCopying::RevokeThreadLocalMarkStack(Thread* thread) { 941 Thread* self = Thread::Current(); 942 CHECK_EQ(self, thread); 943 accounting::AtomicStack<mirror::Object>* tl_mark_stack = thread->GetThreadLocalMarkStack(); 944 if (tl_mark_stack != nullptr) { 945 CHECK(is_marking_); 946 MutexLock mu(self, mark_stack_lock_); 947 revoked_mark_stacks_.push_back(tl_mark_stack); 948 thread->SetThreadLocalMarkStack(nullptr); 949 } 950} 951 952void ConcurrentCopying::ProcessMarkStack() { 953 if (kVerboseMode) { 954 LOG(INFO) << "ProcessMarkStack. "; 955 } 956 bool empty_prev = false; 957 while (true) { 958 bool empty = ProcessMarkStackOnce(); 959 if (empty_prev && empty) { 960 // Saw empty mark stack for a second time, done. 961 break; 962 } 963 empty_prev = empty; 964 } 965} 966 967bool ConcurrentCopying::ProcessMarkStackOnce() { 968 Thread* self = Thread::Current(); 969 CHECK(thread_running_gc_ != nullptr); 970 CHECK(self == thread_running_gc_); 971 CHECK(self->GetThreadLocalMarkStack() == nullptr); 972 size_t count = 0; 973 MarkStackMode mark_stack_mode = mark_stack_mode_.LoadRelaxed(); 974 if (mark_stack_mode == kMarkStackModeThreadLocal) { 975 // Process the thread-local mark stacks and the GC mark stack. 976 count += ProcessThreadLocalMarkStacks(false); 977 while (!gc_mark_stack_->IsEmpty()) { 978 mirror::Object* to_ref = gc_mark_stack_->PopBack(); 979 ProcessMarkStackRef(to_ref); 980 ++count; 981 } 982 gc_mark_stack_->Reset(); 983 } else if (mark_stack_mode == kMarkStackModeShared) { 984 // Process the shared GC mark stack with a lock. 985 { 986 MutexLock mu(self, mark_stack_lock_); 987 CHECK(revoked_mark_stacks_.empty()); 988 } 989 while (true) { 990 std::vector<mirror::Object*> refs; 991 { 992 // Copy refs with lock. Note the number of refs should be small. 993 MutexLock mu(self, mark_stack_lock_); 994 if (gc_mark_stack_->IsEmpty()) { 995 break; 996 } 997 for (StackReference<mirror::Object>* p = gc_mark_stack_->Begin(); 998 p != gc_mark_stack_->End(); ++p) { 999 refs.push_back(p->AsMirrorPtr()); 1000 } 1001 gc_mark_stack_->Reset(); 1002 } 1003 for (mirror::Object* ref : refs) { 1004 ProcessMarkStackRef(ref); 1005 ++count; 1006 } 1007 } 1008 } else { 1009 CHECK_EQ(static_cast<uint32_t>(mark_stack_mode), 1010 static_cast<uint32_t>(kMarkStackModeGcExclusive)); 1011 { 1012 MutexLock mu(self, mark_stack_lock_); 1013 CHECK(revoked_mark_stacks_.empty()); 1014 } 1015 // Process the GC mark stack in the exclusive mode. No need to take the lock. 1016 while (!gc_mark_stack_->IsEmpty()) { 1017 mirror::Object* to_ref = gc_mark_stack_->PopBack(); 1018 ProcessMarkStackRef(to_ref); 1019 ++count; 1020 } 1021 gc_mark_stack_->Reset(); 1022 } 1023 1024 // Return true if the stack was empty. 1025 return count == 0; 1026} 1027 1028size_t ConcurrentCopying::ProcessThreadLocalMarkStacks(bool disable_weak_ref_access) { 1029 // Run a checkpoint to collect all thread local mark stacks and iterate over them all. 1030 RevokeThreadLocalMarkStacks(disable_weak_ref_access); 1031 size_t count = 0; 1032 std::vector<accounting::AtomicStack<mirror::Object>*> mark_stacks; 1033 { 1034 MutexLock mu(Thread::Current(), mark_stack_lock_); 1035 // Make a copy of the mark stack vector. 1036 mark_stacks = revoked_mark_stacks_; 1037 revoked_mark_stacks_.clear(); 1038 } 1039 for (accounting::AtomicStack<mirror::Object>* mark_stack : mark_stacks) { 1040 for (StackReference<mirror::Object>* p = mark_stack->Begin(); p != mark_stack->End(); ++p) { 1041 mirror::Object* to_ref = p->AsMirrorPtr(); 1042 ProcessMarkStackRef(to_ref); 1043 ++count; 1044 } 1045 { 1046 MutexLock mu(Thread::Current(), mark_stack_lock_); 1047 if (pooled_mark_stacks_.size() >= kMarkStackPoolSize) { 1048 // The pool has enough. Delete it. 1049 delete mark_stack; 1050 } else { 1051 // Otherwise, put it into the pool for later reuse. 1052 mark_stack->Reset(); 1053 pooled_mark_stacks_.push_back(mark_stack); 1054 } 1055 } 1056 } 1057 return count; 1058} 1059 1060void ConcurrentCopying::ProcessMarkStackRef(mirror::Object* to_ref) { 1061 DCHECK(!region_space_->IsInFromSpace(to_ref)); 1062 if (kUseBakerReadBarrier) { 1063 DCHECK(to_ref->GetReadBarrierPointer() == ReadBarrier::GrayPtr()) 1064 << " " << to_ref << " " << to_ref->GetReadBarrierPointer() 1065 << " is_marked=" << IsMarked(to_ref); 1066 } 1067 // Scan ref fields. 1068 Scan(to_ref); 1069 // Mark the gray ref as white or black. 1070 if (kUseBakerReadBarrier) { 1071 DCHECK(to_ref->GetReadBarrierPointer() == ReadBarrier::GrayPtr()) 1072 << " " << to_ref << " " << to_ref->GetReadBarrierPointer() 1073 << " is_marked=" << IsMarked(to_ref); 1074 } 1075 if (to_ref->GetClass<kVerifyNone, kWithoutReadBarrier>()->IsTypeOfReferenceClass() && 1076 to_ref->AsReference()->GetReferent<kWithoutReadBarrier>() != nullptr && 1077 !IsInToSpace(to_ref->AsReference()->GetReferent<kWithoutReadBarrier>())) { 1078 // Leave this Reference gray in the queue so that GetReferent() will trigger a read barrier. We 1079 // will change it to black or white later in ReferenceQueue::DequeuePendingReference(). 1080 CHECK(to_ref->AsReference()->IsEnqueued()) << "Left unenqueued ref gray " << to_ref; 1081 } else { 1082 // We may occasionally leave a Reference black or white in the queue if its referent happens to 1083 // be concurrently marked after the Scan() call above has enqueued the Reference, in which case 1084 // the above IsInToSpace() evaluates to true and we change the color from gray to black or white 1085 // here in this else block. 1086#ifdef USE_BAKER_OR_BROOKS_READ_BARRIER 1087 if (kUseBakerReadBarrier) { 1088 if (region_space_->IsInToSpace(to_ref)) { 1089 // If to-space, change from gray to white. 1090 bool success = to_ref->AtomicSetReadBarrierPointer(ReadBarrier::GrayPtr(), 1091 ReadBarrier::WhitePtr()); 1092 CHECK(success) << "Must succeed as we won the race."; 1093 CHECK(to_ref->GetReadBarrierPointer() == ReadBarrier::WhitePtr()); 1094 } else { 1095 // If non-moving space/unevac from space, change from gray 1096 // to black. We can't change gray to white because it's not 1097 // safe to use CAS if two threads change values in opposite 1098 // directions (A->B and B->A). So, we change it to black to 1099 // indicate non-moving objects that have been marked 1100 // through. Note we'd need to change from black to white 1101 // later (concurrently). 1102 bool success = to_ref->AtomicSetReadBarrierPointer(ReadBarrier::GrayPtr(), 1103 ReadBarrier::BlackPtr()); 1104 CHECK(success) << "Must succeed as we won the race."; 1105 CHECK(to_ref->GetReadBarrierPointer() == ReadBarrier::BlackPtr()); 1106 } 1107 } 1108#else 1109 DCHECK(!kUseBakerReadBarrier); 1110#endif 1111 } 1112 if (ReadBarrier::kEnableToSpaceInvariantChecks || kIsDebugBuild) { 1113 ConcurrentCopyingAssertToSpaceInvariantObjectVisitor visitor(this); 1114 visitor(to_ref); 1115 } 1116} 1117 1118void ConcurrentCopying::SwitchToSharedMarkStackMode() { 1119 Thread* self = Thread::Current(); 1120 CHECK(thread_running_gc_ != nullptr); 1121 CHECK_EQ(self, thread_running_gc_); 1122 CHECK(self->GetThreadLocalMarkStack() == nullptr); 1123 MarkStackMode before_mark_stack_mode = mark_stack_mode_.LoadRelaxed(); 1124 CHECK_EQ(static_cast<uint32_t>(before_mark_stack_mode), 1125 static_cast<uint32_t>(kMarkStackModeThreadLocal)); 1126 mark_stack_mode_.StoreRelaxed(kMarkStackModeShared); 1127 CHECK(weak_ref_access_enabled_.LoadRelaxed()); 1128 weak_ref_access_enabled_.StoreRelaxed(false); 1129 QuasiAtomic::ThreadFenceForConstructor(); 1130 // Process the thread local mark stacks one last time after switching to the shared mark stack 1131 // mode and disable weak ref accesses. 1132 ProcessThreadLocalMarkStacks(true); 1133 if (kVerboseMode) { 1134 LOG(INFO) << "Switched to shared mark stack mode and disabled weak ref access"; 1135 } 1136} 1137 1138void ConcurrentCopying::SwitchToGcExclusiveMarkStackMode() { 1139 Thread* self = Thread::Current(); 1140 CHECK(thread_running_gc_ != nullptr); 1141 CHECK_EQ(self, thread_running_gc_); 1142 CHECK(self->GetThreadLocalMarkStack() == nullptr); 1143 MarkStackMode before_mark_stack_mode = mark_stack_mode_.LoadRelaxed(); 1144 CHECK_EQ(static_cast<uint32_t>(before_mark_stack_mode), 1145 static_cast<uint32_t>(kMarkStackModeShared)); 1146 mark_stack_mode_.StoreRelaxed(kMarkStackModeGcExclusive); 1147 QuasiAtomic::ThreadFenceForConstructor(); 1148 if (kVerboseMode) { 1149 LOG(INFO) << "Switched to GC exclusive mark stack mode"; 1150 } 1151} 1152 1153void ConcurrentCopying::CheckEmptyMarkStack() { 1154 Thread* self = Thread::Current(); 1155 CHECK(thread_running_gc_ != nullptr); 1156 CHECK_EQ(self, thread_running_gc_); 1157 CHECK(self->GetThreadLocalMarkStack() == nullptr); 1158 MarkStackMode mark_stack_mode = mark_stack_mode_.LoadRelaxed(); 1159 if (mark_stack_mode == kMarkStackModeThreadLocal) { 1160 // Thread-local mark stack mode. 1161 RevokeThreadLocalMarkStacks(false); 1162 MutexLock mu(Thread::Current(), mark_stack_lock_); 1163 if (!revoked_mark_stacks_.empty()) { 1164 for (accounting::AtomicStack<mirror::Object>* mark_stack : revoked_mark_stacks_) { 1165 while (!mark_stack->IsEmpty()) { 1166 mirror::Object* obj = mark_stack->PopBack(); 1167 if (kUseBakerReadBarrier) { 1168 mirror::Object* rb_ptr = obj->GetReadBarrierPointer(); 1169 LOG(INFO) << "On mark queue : " << obj << " " << PrettyTypeOf(obj) << " rb_ptr=" << rb_ptr 1170 << " is_marked=" << IsMarked(obj); 1171 } else { 1172 LOG(INFO) << "On mark queue : " << obj << " " << PrettyTypeOf(obj) 1173 << " is_marked=" << IsMarked(obj); 1174 } 1175 } 1176 } 1177 LOG(FATAL) << "mark stack is not empty"; 1178 } 1179 } else { 1180 // Shared, GC-exclusive, or off. 1181 MutexLock mu(Thread::Current(), mark_stack_lock_); 1182 CHECK(gc_mark_stack_->IsEmpty()); 1183 CHECK(revoked_mark_stacks_.empty()); 1184 } 1185} 1186 1187void ConcurrentCopying::SweepSystemWeaks(Thread* self) { 1188 TimingLogger::ScopedTiming split("SweepSystemWeaks", GetTimings()); 1189 ReaderMutexLock mu(self, *Locks::heap_bitmap_lock_); 1190 Runtime::Current()->SweepSystemWeaks(this); 1191} 1192 1193void ConcurrentCopying::Sweep(bool swap_bitmaps) { 1194 { 1195 TimingLogger::ScopedTiming t("MarkStackAsLive", GetTimings()); 1196 accounting::ObjectStack* live_stack = heap_->GetLiveStack(); 1197 if (kEnableFromSpaceAccountingCheck) { 1198 CHECK_GE(live_stack_freeze_size_, live_stack->Size()); 1199 } 1200 heap_->MarkAllocStackAsLive(live_stack); 1201 live_stack->Reset(); 1202 } 1203 CheckEmptyMarkStack(); 1204 TimingLogger::ScopedTiming split("Sweep", GetTimings()); 1205 for (const auto& space : GetHeap()->GetContinuousSpaces()) { 1206 if (space->IsContinuousMemMapAllocSpace()) { 1207 space::ContinuousMemMapAllocSpace* alloc_space = space->AsContinuousMemMapAllocSpace(); 1208 if (space == region_space_ || immune_region_.ContainsSpace(space)) { 1209 continue; 1210 } 1211 TimingLogger::ScopedTiming split2( 1212 alloc_space->IsZygoteSpace() ? "SweepZygoteSpace" : "SweepAllocSpace", GetTimings()); 1213 RecordFree(alloc_space->Sweep(swap_bitmaps)); 1214 } 1215 } 1216 SweepLargeObjects(swap_bitmaps); 1217} 1218 1219void ConcurrentCopying::SweepLargeObjects(bool swap_bitmaps) { 1220 TimingLogger::ScopedTiming split("SweepLargeObjects", GetTimings()); 1221 RecordFreeLOS(heap_->GetLargeObjectsSpace()->Sweep(swap_bitmaps)); 1222} 1223 1224class ConcurrentCopyingClearBlackPtrsVisitor { 1225 public: 1226 explicit ConcurrentCopyingClearBlackPtrsVisitor(ConcurrentCopying* cc) 1227 : collector_(cc) {} 1228#ifndef USE_BAKER_OR_BROOKS_READ_BARRIER 1229 NO_RETURN 1230#endif 1231 void operator()(mirror::Object* obj) const SHARED_REQUIRES(Locks::mutator_lock_) 1232 SHARED_REQUIRES(Locks::heap_bitmap_lock_) { 1233 DCHECK(obj != nullptr); 1234 DCHECK(collector_->heap_->GetMarkBitmap()->Test(obj)) << obj; 1235 DCHECK_EQ(obj->GetReadBarrierPointer(), ReadBarrier::BlackPtr()) << obj; 1236 obj->AtomicSetReadBarrierPointer(ReadBarrier::BlackPtr(), ReadBarrier::WhitePtr()); 1237 DCHECK_EQ(obj->GetReadBarrierPointer(), ReadBarrier::WhitePtr()) << obj; 1238 } 1239 1240 private: 1241 ConcurrentCopying* const collector_; 1242}; 1243 1244// Clear the black ptrs in non-moving objects back to white. 1245void ConcurrentCopying::ClearBlackPtrs() { 1246 CHECK(kUseBakerReadBarrier); 1247 TimingLogger::ScopedTiming split("ClearBlackPtrs", GetTimings()); 1248 ConcurrentCopyingClearBlackPtrsVisitor visitor(this); 1249 for (auto& space : heap_->GetContinuousSpaces()) { 1250 if (space == region_space_) { 1251 continue; 1252 } 1253 accounting::ContinuousSpaceBitmap* mark_bitmap = space->GetMarkBitmap(); 1254 if (kVerboseMode) { 1255 LOG(INFO) << "ClearBlackPtrs: " << *space << " bitmap: " << *mark_bitmap; 1256 } 1257 mark_bitmap->VisitMarkedRange(reinterpret_cast<uintptr_t>(space->Begin()), 1258 reinterpret_cast<uintptr_t>(space->Limit()), 1259 visitor); 1260 } 1261 space::LargeObjectSpace* large_object_space = heap_->GetLargeObjectsSpace(); 1262 large_object_space->GetMarkBitmap()->VisitMarkedRange( 1263 reinterpret_cast<uintptr_t>(large_object_space->Begin()), 1264 reinterpret_cast<uintptr_t>(large_object_space->End()), 1265 visitor); 1266 // Objects on the allocation stack? 1267 if (ReadBarrier::kEnableReadBarrierInvariantChecks || kIsDebugBuild) { 1268 size_t count = GetAllocationStack()->Size(); 1269 auto* it = GetAllocationStack()->Begin(); 1270 auto* end = GetAllocationStack()->End(); 1271 for (size_t i = 0; i < count; ++i, ++it) { 1272 CHECK_LT(it, end); 1273 mirror::Object* obj = it->AsMirrorPtr(); 1274 if (obj != nullptr) { 1275 // Must have been cleared above. 1276 CHECK_EQ(obj->GetReadBarrierPointer(), ReadBarrier::WhitePtr()) << obj; 1277 } 1278 } 1279 } 1280} 1281 1282void ConcurrentCopying::ReclaimPhase() { 1283 TimingLogger::ScopedTiming split("ReclaimPhase", GetTimings()); 1284 if (kVerboseMode) { 1285 LOG(INFO) << "GC ReclaimPhase"; 1286 } 1287 Thread* self = Thread::Current(); 1288 1289 { 1290 // Double-check that the mark stack is empty. 1291 // Note: need to set this after VerifyNoFromSpaceRef(). 1292 is_asserting_to_space_invariant_ = false; 1293 QuasiAtomic::ThreadFenceForConstructor(); 1294 if (kVerboseMode) { 1295 LOG(INFO) << "Issue an empty check point. "; 1296 } 1297 IssueEmptyCheckpoint(); 1298 // Disable the check. 1299 is_mark_stack_push_disallowed_.StoreSequentiallyConsistent(0); 1300 CheckEmptyMarkStack(); 1301 } 1302 1303 { 1304 // Record freed objects. 1305 TimingLogger::ScopedTiming split2("RecordFree", GetTimings()); 1306 // Don't include thread-locals that are in the to-space. 1307 uint64_t from_bytes = region_space_->GetBytesAllocatedInFromSpace(); 1308 uint64_t from_objects = region_space_->GetObjectsAllocatedInFromSpace(); 1309 uint64_t unevac_from_bytes = region_space_->GetBytesAllocatedInUnevacFromSpace(); 1310 uint64_t unevac_from_objects = region_space_->GetObjectsAllocatedInUnevacFromSpace(); 1311 uint64_t to_bytes = bytes_moved_.LoadSequentiallyConsistent(); 1312 uint64_t to_objects = objects_moved_.LoadSequentiallyConsistent(); 1313 if (kEnableFromSpaceAccountingCheck) { 1314 CHECK_EQ(from_space_num_objects_at_first_pause_, from_objects + unevac_from_objects); 1315 CHECK_EQ(from_space_num_bytes_at_first_pause_, from_bytes + unevac_from_bytes); 1316 } 1317 CHECK_LE(to_objects, from_objects); 1318 CHECK_LE(to_bytes, from_bytes); 1319 int64_t freed_bytes = from_bytes - to_bytes; 1320 int64_t freed_objects = from_objects - to_objects; 1321 if (kVerboseMode) { 1322 LOG(INFO) << "RecordFree:" 1323 << " from_bytes=" << from_bytes << " from_objects=" << from_objects 1324 << " unevac_from_bytes=" << unevac_from_bytes << " unevac_from_objects=" << unevac_from_objects 1325 << " to_bytes=" << to_bytes << " to_objects=" << to_objects 1326 << " freed_bytes=" << freed_bytes << " freed_objects=" << freed_objects 1327 << " from_space size=" << region_space_->FromSpaceSize() 1328 << " unevac_from_space size=" << region_space_->UnevacFromSpaceSize() 1329 << " to_space size=" << region_space_->ToSpaceSize(); 1330 LOG(INFO) << "(before) num_bytes_allocated=" << heap_->num_bytes_allocated_.LoadSequentiallyConsistent(); 1331 } 1332 RecordFree(ObjectBytePair(freed_objects, freed_bytes)); 1333 if (kVerboseMode) { 1334 LOG(INFO) << "(after) num_bytes_allocated=" << heap_->num_bytes_allocated_.LoadSequentiallyConsistent(); 1335 } 1336 } 1337 1338 { 1339 TimingLogger::ScopedTiming split3("ComputeUnevacFromSpaceLiveRatio", GetTimings()); 1340 ComputeUnevacFromSpaceLiveRatio(); 1341 } 1342 1343 { 1344 TimingLogger::ScopedTiming split4("ClearFromSpace", GetTimings()); 1345 region_space_->ClearFromSpace(); 1346 } 1347 1348 { 1349 WriterMutexLock mu(self, *Locks::heap_bitmap_lock_); 1350 if (kUseBakerReadBarrier) { 1351 ClearBlackPtrs(); 1352 } 1353 Sweep(false); 1354 SwapBitmaps(); 1355 heap_->UnBindBitmaps(); 1356 1357 // Remove bitmaps for the immune spaces. 1358 while (!cc_bitmaps_.empty()) { 1359 accounting::ContinuousSpaceBitmap* cc_bitmap = cc_bitmaps_.back(); 1360 cc_heap_bitmap_->RemoveContinuousSpaceBitmap(cc_bitmap); 1361 delete cc_bitmap; 1362 cc_bitmaps_.pop_back(); 1363 } 1364 region_space_bitmap_ = nullptr; 1365 } 1366 1367 CheckEmptyMarkStack(); 1368 1369 if (kVerboseMode) { 1370 LOG(INFO) << "GC end of ReclaimPhase"; 1371 } 1372} 1373 1374class ConcurrentCopyingComputeUnevacFromSpaceLiveRatioVisitor { 1375 public: 1376 explicit ConcurrentCopyingComputeUnevacFromSpaceLiveRatioVisitor(ConcurrentCopying* cc) 1377 : collector_(cc) {} 1378 void operator()(mirror::Object* ref) const SHARED_REQUIRES(Locks::mutator_lock_) 1379 SHARED_REQUIRES(Locks::heap_bitmap_lock_) { 1380 DCHECK(ref != nullptr); 1381 DCHECK(collector_->region_space_bitmap_->Test(ref)) << ref; 1382 DCHECK(collector_->region_space_->IsInUnevacFromSpace(ref)) << ref; 1383 if (kUseBakerReadBarrier) { 1384 DCHECK_EQ(ref->GetReadBarrierPointer(), ReadBarrier::BlackPtr()) << ref; 1385 // Clear the black ptr. 1386 ref->AtomicSetReadBarrierPointer(ReadBarrier::BlackPtr(), ReadBarrier::WhitePtr()); 1387 DCHECK_EQ(ref->GetReadBarrierPointer(), ReadBarrier::WhitePtr()) << ref; 1388 } 1389 size_t obj_size = ref->SizeOf(); 1390 size_t alloc_size = RoundUp(obj_size, space::RegionSpace::kAlignment); 1391 collector_->region_space_->AddLiveBytes(ref, alloc_size); 1392 } 1393 1394 private: 1395 ConcurrentCopying* const collector_; 1396}; 1397 1398// Compute how much live objects are left in regions. 1399void ConcurrentCopying::ComputeUnevacFromSpaceLiveRatio() { 1400 region_space_->AssertAllRegionLiveBytesZeroOrCleared(); 1401 ConcurrentCopyingComputeUnevacFromSpaceLiveRatioVisitor visitor(this); 1402 region_space_bitmap_->VisitMarkedRange(reinterpret_cast<uintptr_t>(region_space_->Begin()), 1403 reinterpret_cast<uintptr_t>(region_space_->Limit()), 1404 visitor); 1405} 1406 1407// Assert the to-space invariant. 1408void ConcurrentCopying::AssertToSpaceInvariant(mirror::Object* obj, MemberOffset offset, 1409 mirror::Object* ref) { 1410 CHECK(heap_->collector_type_ == kCollectorTypeCC) << static_cast<size_t>(heap_->collector_type_); 1411 if (is_asserting_to_space_invariant_) { 1412 if (region_space_->IsInToSpace(ref)) { 1413 // OK. 1414 return; 1415 } else if (region_space_->IsInUnevacFromSpace(ref)) { 1416 CHECK(region_space_bitmap_->Test(ref)) << ref; 1417 } else if (region_space_->IsInFromSpace(ref)) { 1418 // Not OK. Do extra logging. 1419 if (obj != nullptr) { 1420 LogFromSpaceRefHolder(obj, offset); 1421 } 1422 ref->GetLockWord(false).Dump(LOG(INTERNAL_FATAL)); 1423 CHECK(false) << "Found from-space ref " << ref << " " << PrettyTypeOf(ref); 1424 } else { 1425 AssertToSpaceInvariantInNonMovingSpace(obj, ref); 1426 } 1427 } 1428} 1429 1430class RootPrinter { 1431 public: 1432 RootPrinter() { } 1433 1434 template <class MirrorType> 1435 ALWAYS_INLINE void VisitRootIfNonNull(mirror::CompressedReference<MirrorType>* root) 1436 SHARED_REQUIRES(Locks::mutator_lock_) { 1437 if (!root->IsNull()) { 1438 VisitRoot(root); 1439 } 1440 } 1441 1442 template <class MirrorType> 1443 void VisitRoot(mirror::Object** root) 1444 SHARED_REQUIRES(Locks::mutator_lock_) { 1445 LOG(INTERNAL_FATAL) << "root=" << root << " ref=" << *root; 1446 } 1447 1448 template <class MirrorType> 1449 void VisitRoot(mirror::CompressedReference<MirrorType>* root) 1450 SHARED_REQUIRES(Locks::mutator_lock_) { 1451 LOG(INTERNAL_FATAL) << "root=" << root << " ref=" << root->AsMirrorPtr(); 1452 } 1453}; 1454 1455void ConcurrentCopying::AssertToSpaceInvariant(GcRootSource* gc_root_source, 1456 mirror::Object* ref) { 1457 CHECK(heap_->collector_type_ == kCollectorTypeCC) << static_cast<size_t>(heap_->collector_type_); 1458 if (is_asserting_to_space_invariant_) { 1459 if (region_space_->IsInToSpace(ref)) { 1460 // OK. 1461 return; 1462 } else if (region_space_->IsInUnevacFromSpace(ref)) { 1463 CHECK(region_space_bitmap_->Test(ref)) << ref; 1464 } else if (region_space_->IsInFromSpace(ref)) { 1465 // Not OK. Do extra logging. 1466 if (gc_root_source == nullptr) { 1467 // No info. 1468 } else if (gc_root_source->HasArtField()) { 1469 ArtField* field = gc_root_source->GetArtField(); 1470 LOG(INTERNAL_FATAL) << "gc root in field " << field << " " << PrettyField(field); 1471 RootPrinter root_printer; 1472 field->VisitRoots(root_printer); 1473 } else if (gc_root_source->HasArtMethod()) { 1474 ArtMethod* method = gc_root_source->GetArtMethod(); 1475 LOG(INTERNAL_FATAL) << "gc root in method " << method << " " << PrettyMethod(method); 1476 RootPrinter root_printer; 1477 method->VisitRoots(root_printer, sizeof(void*)); 1478 } 1479 ref->GetLockWord(false).Dump(LOG(INTERNAL_FATAL)); 1480 region_space_->DumpNonFreeRegions(LOG(INTERNAL_FATAL)); 1481 PrintFileToLog("/proc/self/maps", LogSeverity::INTERNAL_FATAL); 1482 MemMap::DumpMaps(LOG(INTERNAL_FATAL), true); 1483 CHECK(false) << "Found from-space ref " << ref << " " << PrettyTypeOf(ref); 1484 } else { 1485 AssertToSpaceInvariantInNonMovingSpace(nullptr, ref); 1486 } 1487 } 1488} 1489 1490void ConcurrentCopying::LogFromSpaceRefHolder(mirror::Object* obj, MemberOffset offset) { 1491 if (kUseBakerReadBarrier) { 1492 LOG(INFO) << "holder=" << obj << " " << PrettyTypeOf(obj) 1493 << " holder rb_ptr=" << obj->GetReadBarrierPointer(); 1494 } else { 1495 LOG(INFO) << "holder=" << obj << " " << PrettyTypeOf(obj); 1496 } 1497 if (region_space_->IsInFromSpace(obj)) { 1498 LOG(INFO) << "holder is in the from-space."; 1499 } else if (region_space_->IsInToSpace(obj)) { 1500 LOG(INFO) << "holder is in the to-space."; 1501 } else if (region_space_->IsInUnevacFromSpace(obj)) { 1502 LOG(INFO) << "holder is in the unevac from-space."; 1503 if (region_space_bitmap_->Test(obj)) { 1504 LOG(INFO) << "holder is marked in the region space bitmap."; 1505 } else { 1506 LOG(INFO) << "holder is not marked in the region space bitmap."; 1507 } 1508 } else { 1509 // In a non-moving space. 1510 if (immune_region_.ContainsObject(obj)) { 1511 LOG(INFO) << "holder is in the image or the zygote space."; 1512 accounting::ContinuousSpaceBitmap* cc_bitmap = 1513 cc_heap_bitmap_->GetContinuousSpaceBitmap(obj); 1514 CHECK(cc_bitmap != nullptr) 1515 << "An immune space object must have a bitmap."; 1516 if (cc_bitmap->Test(obj)) { 1517 LOG(INFO) << "holder is marked in the bit map."; 1518 } else { 1519 LOG(INFO) << "holder is NOT marked in the bit map."; 1520 } 1521 } else { 1522 LOG(INFO) << "holder is in a non-moving (or main) space."; 1523 accounting::ContinuousSpaceBitmap* mark_bitmap = 1524 heap_mark_bitmap_->GetContinuousSpaceBitmap(obj); 1525 accounting::LargeObjectBitmap* los_bitmap = 1526 heap_mark_bitmap_->GetLargeObjectBitmap(obj); 1527 CHECK(los_bitmap != nullptr) << "LOS bitmap covers the entire address range"; 1528 bool is_los = mark_bitmap == nullptr; 1529 if (!is_los && mark_bitmap->Test(obj)) { 1530 LOG(INFO) << "holder is marked in the mark bit map."; 1531 } else if (is_los && los_bitmap->Test(obj)) { 1532 LOG(INFO) << "holder is marked in the los bit map."; 1533 } else { 1534 // If ref is on the allocation stack, then it is considered 1535 // mark/alive (but not necessarily on the live stack.) 1536 if (IsOnAllocStack(obj)) { 1537 LOG(INFO) << "holder is on the alloc stack."; 1538 } else { 1539 LOG(INFO) << "holder is not marked or on the alloc stack."; 1540 } 1541 } 1542 } 1543 } 1544 LOG(INFO) << "offset=" << offset.SizeValue(); 1545} 1546 1547void ConcurrentCopying::AssertToSpaceInvariantInNonMovingSpace(mirror::Object* obj, 1548 mirror::Object* ref) { 1549 // In a non-moving spaces. Check that the ref is marked. 1550 if (immune_region_.ContainsObject(ref)) { 1551 accounting::ContinuousSpaceBitmap* cc_bitmap = 1552 cc_heap_bitmap_->GetContinuousSpaceBitmap(ref); 1553 CHECK(cc_bitmap != nullptr) 1554 << "An immune space ref must have a bitmap. " << ref; 1555 if (kUseBakerReadBarrier) { 1556 CHECK(cc_bitmap->Test(ref)) 1557 << "Unmarked immune space ref. obj=" << obj << " rb_ptr=" 1558 << obj->GetReadBarrierPointer() << " ref=" << ref; 1559 } else { 1560 CHECK(cc_bitmap->Test(ref)) 1561 << "Unmarked immune space ref. obj=" << obj << " ref=" << ref; 1562 } 1563 } else { 1564 accounting::ContinuousSpaceBitmap* mark_bitmap = 1565 heap_mark_bitmap_->GetContinuousSpaceBitmap(ref); 1566 accounting::LargeObjectBitmap* los_bitmap = 1567 heap_mark_bitmap_->GetLargeObjectBitmap(ref); 1568 CHECK(los_bitmap != nullptr) << "LOS bitmap covers the entire address range"; 1569 bool is_los = mark_bitmap == nullptr; 1570 if ((!is_los && mark_bitmap->Test(ref)) || 1571 (is_los && los_bitmap->Test(ref))) { 1572 // OK. 1573 } else { 1574 // If ref is on the allocation stack, then it may not be 1575 // marked live, but considered marked/alive (but not 1576 // necessarily on the live stack). 1577 CHECK(IsOnAllocStack(ref)) << "Unmarked ref that's not on the allocation stack. " 1578 << "obj=" << obj << " ref=" << ref; 1579 } 1580 } 1581} 1582 1583// Used to scan ref fields of an object. 1584class ConcurrentCopyingRefFieldsVisitor { 1585 public: 1586 explicit ConcurrentCopyingRefFieldsVisitor(ConcurrentCopying* collector) 1587 : collector_(collector) {} 1588 1589 void operator()(mirror::Object* obj, MemberOffset offset, bool /* is_static */) 1590 const ALWAYS_INLINE SHARED_REQUIRES(Locks::mutator_lock_) 1591 SHARED_REQUIRES(Locks::heap_bitmap_lock_) { 1592 collector_->Process(obj, offset); 1593 } 1594 1595 void operator()(mirror::Class* klass, mirror::Reference* ref) const 1596 SHARED_REQUIRES(Locks::mutator_lock_) ALWAYS_INLINE { 1597 CHECK(klass->IsTypeOfReferenceClass()); 1598 collector_->DelayReferenceReferent(klass, ref); 1599 } 1600 1601 void VisitRootIfNonNull(mirror::CompressedReference<mirror::Object>* root) const 1602 SHARED_REQUIRES(Locks::mutator_lock_) { 1603 if (!root->IsNull()) { 1604 VisitRoot(root); 1605 } 1606 } 1607 1608 void VisitRoot(mirror::CompressedReference<mirror::Object>* root) const 1609 SHARED_REQUIRES(Locks::mutator_lock_) { 1610 collector_->MarkRoot(root); 1611 } 1612 1613 private: 1614 ConcurrentCopying* const collector_; 1615}; 1616 1617// Scan ref fields of an object. 1618void ConcurrentCopying::Scan(mirror::Object* to_ref) { 1619 DCHECK(!region_space_->IsInFromSpace(to_ref)); 1620 ConcurrentCopyingRefFieldsVisitor visitor(this); 1621 to_ref->VisitReferences(visitor, visitor); 1622} 1623 1624// Process a field. 1625inline void ConcurrentCopying::Process(mirror::Object* obj, MemberOffset offset) { 1626 mirror::Object* ref = obj->GetFieldObject< 1627 mirror::Object, kVerifyNone, kWithoutReadBarrier, false>(offset); 1628 if (ref == nullptr || region_space_->IsInToSpace(ref)) { 1629 return; 1630 } 1631 mirror::Object* to_ref = Mark(ref); 1632 if (to_ref == ref) { 1633 return; 1634 } 1635 // This may fail if the mutator writes to the field at the same time. But it's ok. 1636 mirror::Object* expected_ref = ref; 1637 mirror::Object* new_ref = to_ref; 1638 do { 1639 if (expected_ref != 1640 obj->GetFieldObject<mirror::Object, kVerifyNone, kWithoutReadBarrier, false>(offset)) { 1641 // It was updated by the mutator. 1642 break; 1643 } 1644 } while (!obj->CasFieldWeakSequentiallyConsistentObjectWithoutWriteBarrier< 1645 false, false, kVerifyNone>(offset, expected_ref, new_ref)); 1646} 1647 1648// Process some roots. 1649void ConcurrentCopying::VisitRoots( 1650 mirror::Object*** roots, size_t count, const RootInfo& info ATTRIBUTE_UNUSED) { 1651 for (size_t i = 0; i < count; ++i) { 1652 mirror::Object** root = roots[i]; 1653 mirror::Object* ref = *root; 1654 if (ref == nullptr || region_space_->IsInToSpace(ref)) { 1655 continue; 1656 } 1657 mirror::Object* to_ref = Mark(ref); 1658 if (to_ref == ref) { 1659 continue; 1660 } 1661 Atomic<mirror::Object*>* addr = reinterpret_cast<Atomic<mirror::Object*>*>(root); 1662 mirror::Object* expected_ref = ref; 1663 mirror::Object* new_ref = to_ref; 1664 do { 1665 if (expected_ref != addr->LoadRelaxed()) { 1666 // It was updated by the mutator. 1667 break; 1668 } 1669 } while (!addr->CompareExchangeWeakSequentiallyConsistent(expected_ref, new_ref)); 1670 } 1671} 1672 1673void ConcurrentCopying::MarkRoot(mirror::CompressedReference<mirror::Object>* root) { 1674 DCHECK(!root->IsNull()); 1675 mirror::Object* const ref = root->AsMirrorPtr(); 1676 if (region_space_->IsInToSpace(ref)) { 1677 return; 1678 } 1679 mirror::Object* to_ref = Mark(ref); 1680 if (to_ref != ref) { 1681 auto* addr = reinterpret_cast<Atomic<mirror::CompressedReference<mirror::Object>>*>(root); 1682 auto expected_ref = mirror::CompressedReference<mirror::Object>::FromMirrorPtr(ref); 1683 auto new_ref = mirror::CompressedReference<mirror::Object>::FromMirrorPtr(to_ref); 1684 // If the cas fails, then it was updated by the mutator. 1685 do { 1686 if (ref != addr->LoadRelaxed().AsMirrorPtr()) { 1687 // It was updated by the mutator. 1688 break; 1689 } 1690 } while (!addr->CompareExchangeWeakSequentiallyConsistent(expected_ref, new_ref)); 1691 } 1692} 1693 1694void ConcurrentCopying::VisitRoots( 1695 mirror::CompressedReference<mirror::Object>** roots, size_t count, 1696 const RootInfo& info ATTRIBUTE_UNUSED) { 1697 for (size_t i = 0; i < count; ++i) { 1698 mirror::CompressedReference<mirror::Object>* const root = roots[i]; 1699 if (!root->IsNull()) { 1700 MarkRoot(root); 1701 } 1702 } 1703} 1704 1705// Fill the given memory block with a dummy object. Used to fill in a 1706// copy of objects that was lost in race. 1707void ConcurrentCopying::FillWithDummyObject(mirror::Object* dummy_obj, size_t byte_size) { 1708 CHECK_ALIGNED(byte_size, kObjectAlignment); 1709 memset(dummy_obj, 0, byte_size); 1710 mirror::Class* int_array_class = mirror::IntArray::GetArrayClass(); 1711 CHECK(int_array_class != nullptr); 1712 AssertToSpaceInvariant(nullptr, MemberOffset(0), int_array_class); 1713 size_t component_size = int_array_class->GetComponentSize(); 1714 CHECK_EQ(component_size, sizeof(int32_t)); 1715 size_t data_offset = mirror::Array::DataOffset(component_size).SizeValue(); 1716 if (data_offset > byte_size) { 1717 // An int array is too big. Use java.lang.Object. 1718 mirror::Class* java_lang_Object = WellKnownClasses::ToClass(WellKnownClasses::java_lang_Object); 1719 AssertToSpaceInvariant(nullptr, MemberOffset(0), java_lang_Object); 1720 CHECK_EQ(byte_size, java_lang_Object->GetObjectSize()); 1721 dummy_obj->SetClass(java_lang_Object); 1722 CHECK_EQ(byte_size, dummy_obj->SizeOf()); 1723 } else { 1724 // Use an int array. 1725 dummy_obj->SetClass(int_array_class); 1726 CHECK(dummy_obj->IsArrayInstance()); 1727 int32_t length = (byte_size - data_offset) / component_size; 1728 dummy_obj->AsArray()->SetLength(length); 1729 CHECK_EQ(dummy_obj->AsArray()->GetLength(), length) 1730 << "byte_size=" << byte_size << " length=" << length 1731 << " component_size=" << component_size << " data_offset=" << data_offset; 1732 CHECK_EQ(byte_size, dummy_obj->SizeOf()) 1733 << "byte_size=" << byte_size << " length=" << length 1734 << " component_size=" << component_size << " data_offset=" << data_offset; 1735 } 1736} 1737 1738// Reuse the memory blocks that were copy of objects that were lost in race. 1739mirror::Object* ConcurrentCopying::AllocateInSkippedBlock(size_t alloc_size) { 1740 // Try to reuse the blocks that were unused due to CAS failures. 1741 CHECK_ALIGNED(alloc_size, space::RegionSpace::kAlignment); 1742 Thread* self = Thread::Current(); 1743 size_t min_object_size = RoundUp(sizeof(mirror::Object), space::RegionSpace::kAlignment); 1744 MutexLock mu(self, skipped_blocks_lock_); 1745 auto it = skipped_blocks_map_.lower_bound(alloc_size); 1746 if (it == skipped_blocks_map_.end()) { 1747 // Not found. 1748 return nullptr; 1749 } 1750 { 1751 size_t byte_size = it->first; 1752 CHECK_GE(byte_size, alloc_size); 1753 if (byte_size > alloc_size && byte_size - alloc_size < min_object_size) { 1754 // If remainder would be too small for a dummy object, retry with a larger request size. 1755 it = skipped_blocks_map_.lower_bound(alloc_size + min_object_size); 1756 if (it == skipped_blocks_map_.end()) { 1757 // Not found. 1758 return nullptr; 1759 } 1760 CHECK_ALIGNED(it->first - alloc_size, space::RegionSpace::kAlignment); 1761 CHECK_GE(it->first - alloc_size, min_object_size) 1762 << "byte_size=" << byte_size << " it->first=" << it->first << " alloc_size=" << alloc_size; 1763 } 1764 } 1765 // Found a block. 1766 CHECK(it != skipped_blocks_map_.end()); 1767 size_t byte_size = it->first; 1768 uint8_t* addr = it->second; 1769 CHECK_GE(byte_size, alloc_size); 1770 CHECK(region_space_->IsInToSpace(reinterpret_cast<mirror::Object*>(addr))); 1771 CHECK_ALIGNED(byte_size, space::RegionSpace::kAlignment); 1772 if (kVerboseMode) { 1773 LOG(INFO) << "Reusing skipped bytes : " << reinterpret_cast<void*>(addr) << ", " << byte_size; 1774 } 1775 skipped_blocks_map_.erase(it); 1776 memset(addr, 0, byte_size); 1777 if (byte_size > alloc_size) { 1778 // Return the remainder to the map. 1779 CHECK_ALIGNED(byte_size - alloc_size, space::RegionSpace::kAlignment); 1780 CHECK_GE(byte_size - alloc_size, min_object_size); 1781 FillWithDummyObject(reinterpret_cast<mirror::Object*>(addr + alloc_size), 1782 byte_size - alloc_size); 1783 CHECK(region_space_->IsInToSpace(reinterpret_cast<mirror::Object*>(addr + alloc_size))); 1784 skipped_blocks_map_.insert(std::make_pair(byte_size - alloc_size, addr + alloc_size)); 1785 } 1786 return reinterpret_cast<mirror::Object*>(addr); 1787} 1788 1789mirror::Object* ConcurrentCopying::Copy(mirror::Object* from_ref) { 1790 DCHECK(region_space_->IsInFromSpace(from_ref)); 1791 // No read barrier to avoid nested RB that might violate the to-space 1792 // invariant. Note that from_ref is a from space ref so the SizeOf() 1793 // call will access the from-space meta objects, but it's ok and necessary. 1794 size_t obj_size = from_ref->SizeOf<kDefaultVerifyFlags, kWithoutReadBarrier>(); 1795 size_t region_space_alloc_size = RoundUp(obj_size, space::RegionSpace::kAlignment); 1796 size_t region_space_bytes_allocated = 0U; 1797 size_t non_moving_space_bytes_allocated = 0U; 1798 size_t bytes_allocated = 0U; 1799 size_t dummy; 1800 mirror::Object* to_ref = region_space_->AllocNonvirtual<true>( 1801 region_space_alloc_size, ®ion_space_bytes_allocated, nullptr, &dummy); 1802 bytes_allocated = region_space_bytes_allocated; 1803 if (to_ref != nullptr) { 1804 DCHECK_EQ(region_space_alloc_size, region_space_bytes_allocated); 1805 } 1806 bool fall_back_to_non_moving = false; 1807 if (UNLIKELY(to_ref == nullptr)) { 1808 // Failed to allocate in the region space. Try the skipped blocks. 1809 to_ref = AllocateInSkippedBlock(region_space_alloc_size); 1810 if (to_ref != nullptr) { 1811 // Succeeded to allocate in a skipped block. 1812 if (heap_->use_tlab_) { 1813 // This is necessary for the tlab case as it's not accounted in the space. 1814 region_space_->RecordAlloc(to_ref); 1815 } 1816 bytes_allocated = region_space_alloc_size; 1817 } else { 1818 // Fall back to the non-moving space. 1819 fall_back_to_non_moving = true; 1820 if (kVerboseMode) { 1821 LOG(INFO) << "Out of memory in the to-space. Fall back to non-moving. skipped_bytes=" 1822 << to_space_bytes_skipped_.LoadSequentiallyConsistent() 1823 << " skipped_objects=" << to_space_objects_skipped_.LoadSequentiallyConsistent(); 1824 } 1825 fall_back_to_non_moving = true; 1826 to_ref = heap_->non_moving_space_->Alloc(Thread::Current(), obj_size, 1827 &non_moving_space_bytes_allocated, nullptr, &dummy); 1828 CHECK(to_ref != nullptr) << "Fall-back non-moving space allocation failed"; 1829 bytes_allocated = non_moving_space_bytes_allocated; 1830 // Mark it in the mark bitmap. 1831 accounting::ContinuousSpaceBitmap* mark_bitmap = 1832 heap_mark_bitmap_->GetContinuousSpaceBitmap(to_ref); 1833 CHECK(mark_bitmap != nullptr); 1834 CHECK(!mark_bitmap->AtomicTestAndSet(to_ref)); 1835 } 1836 } 1837 DCHECK(to_ref != nullptr); 1838 1839 // Attempt to install the forward pointer. This is in a loop as the 1840 // lock word atomic write can fail. 1841 while (true) { 1842 // Copy the object. TODO: copy only the lockword in the second iteration and on? 1843 memcpy(to_ref, from_ref, obj_size); 1844 1845 LockWord old_lock_word = to_ref->GetLockWord(false); 1846 1847 if (old_lock_word.GetState() == LockWord::kForwardingAddress) { 1848 // Lost the race. Another thread (either GC or mutator) stored 1849 // the forwarding pointer first. Make the lost copy (to_ref) 1850 // look like a valid but dead (dummy) object and keep it for 1851 // future reuse. 1852 FillWithDummyObject(to_ref, bytes_allocated); 1853 if (!fall_back_to_non_moving) { 1854 DCHECK(region_space_->IsInToSpace(to_ref)); 1855 if (bytes_allocated > space::RegionSpace::kRegionSize) { 1856 // Free the large alloc. 1857 region_space_->FreeLarge(to_ref, bytes_allocated); 1858 } else { 1859 // Record the lost copy for later reuse. 1860 heap_->num_bytes_allocated_.FetchAndAddSequentiallyConsistent(bytes_allocated); 1861 to_space_bytes_skipped_.FetchAndAddSequentiallyConsistent(bytes_allocated); 1862 to_space_objects_skipped_.FetchAndAddSequentiallyConsistent(1); 1863 MutexLock mu(Thread::Current(), skipped_blocks_lock_); 1864 skipped_blocks_map_.insert(std::make_pair(bytes_allocated, 1865 reinterpret_cast<uint8_t*>(to_ref))); 1866 } 1867 } else { 1868 DCHECK(heap_->non_moving_space_->HasAddress(to_ref)); 1869 DCHECK_EQ(bytes_allocated, non_moving_space_bytes_allocated); 1870 // Free the non-moving-space chunk. 1871 accounting::ContinuousSpaceBitmap* mark_bitmap = 1872 heap_mark_bitmap_->GetContinuousSpaceBitmap(to_ref); 1873 CHECK(mark_bitmap != nullptr); 1874 CHECK(mark_bitmap->Clear(to_ref)); 1875 heap_->non_moving_space_->Free(Thread::Current(), to_ref); 1876 } 1877 1878 // Get the winner's forward ptr. 1879 mirror::Object* lost_fwd_ptr = to_ref; 1880 to_ref = reinterpret_cast<mirror::Object*>(old_lock_word.ForwardingAddress()); 1881 CHECK(to_ref != nullptr); 1882 CHECK_NE(to_ref, lost_fwd_ptr); 1883 CHECK(region_space_->IsInToSpace(to_ref) || heap_->non_moving_space_->HasAddress(to_ref)); 1884 CHECK_NE(to_ref->GetLockWord(false).GetState(), LockWord::kForwardingAddress); 1885 return to_ref; 1886 } 1887 1888 // Set the gray ptr. 1889 if (kUseBakerReadBarrier) { 1890 to_ref->SetReadBarrierPointer(ReadBarrier::GrayPtr()); 1891 } 1892 1893 LockWord new_lock_word = LockWord::FromForwardingAddress(reinterpret_cast<size_t>(to_ref)); 1894 1895 // Try to atomically write the fwd ptr. 1896 bool success = from_ref->CasLockWordWeakSequentiallyConsistent(old_lock_word, new_lock_word); 1897 if (LIKELY(success)) { 1898 // The CAS succeeded. 1899 objects_moved_.FetchAndAddSequentiallyConsistent(1); 1900 bytes_moved_.FetchAndAddSequentiallyConsistent(region_space_alloc_size); 1901 if (LIKELY(!fall_back_to_non_moving)) { 1902 DCHECK(region_space_->IsInToSpace(to_ref)); 1903 } else { 1904 DCHECK(heap_->non_moving_space_->HasAddress(to_ref)); 1905 DCHECK_EQ(bytes_allocated, non_moving_space_bytes_allocated); 1906 } 1907 if (kUseBakerReadBarrier) { 1908 DCHECK(to_ref->GetReadBarrierPointer() == ReadBarrier::GrayPtr()); 1909 } 1910 DCHECK(GetFwdPtr(from_ref) == to_ref); 1911 CHECK_NE(to_ref->GetLockWord(false).GetState(), LockWord::kForwardingAddress); 1912 PushOntoMarkStack(to_ref); 1913 return to_ref; 1914 } else { 1915 // The CAS failed. It may have lost the race or may have failed 1916 // due to monitor/hashcode ops. Either way, retry. 1917 } 1918 } 1919} 1920 1921mirror::Object* ConcurrentCopying::IsMarked(mirror::Object* from_ref) { 1922 DCHECK(from_ref != nullptr); 1923 space::RegionSpace::RegionType rtype = region_space_->GetRegionType(from_ref); 1924 if (rtype == space::RegionSpace::RegionType::kRegionTypeToSpace) { 1925 // It's already marked. 1926 return from_ref; 1927 } 1928 mirror::Object* to_ref; 1929 if (rtype == space::RegionSpace::RegionType::kRegionTypeFromSpace) { 1930 to_ref = GetFwdPtr(from_ref); 1931 DCHECK(to_ref == nullptr || region_space_->IsInToSpace(to_ref) || 1932 heap_->non_moving_space_->HasAddress(to_ref)) 1933 << "from_ref=" << from_ref << " to_ref=" << to_ref; 1934 } else if (rtype == space::RegionSpace::RegionType::kRegionTypeUnevacFromSpace) { 1935 if (region_space_bitmap_->Test(from_ref)) { 1936 to_ref = from_ref; 1937 } else { 1938 to_ref = nullptr; 1939 } 1940 } else { 1941 // from_ref is in a non-moving space. 1942 if (immune_region_.ContainsObject(from_ref)) { 1943 accounting::ContinuousSpaceBitmap* cc_bitmap = 1944 cc_heap_bitmap_->GetContinuousSpaceBitmap(from_ref); 1945 DCHECK(cc_bitmap != nullptr) 1946 << "An immune space object must have a bitmap"; 1947 if (kIsDebugBuild) { 1948 DCHECK(heap_mark_bitmap_->GetContinuousSpaceBitmap(from_ref)->Test(from_ref)) 1949 << "Immune space object must be already marked"; 1950 } 1951 if (cc_bitmap->Test(from_ref)) { 1952 // Already marked. 1953 to_ref = from_ref; 1954 } else { 1955 // Newly marked. 1956 to_ref = nullptr; 1957 } 1958 } else { 1959 // Non-immune non-moving space. Use the mark bitmap. 1960 accounting::ContinuousSpaceBitmap* mark_bitmap = 1961 heap_mark_bitmap_->GetContinuousSpaceBitmap(from_ref); 1962 accounting::LargeObjectBitmap* los_bitmap = 1963 heap_mark_bitmap_->GetLargeObjectBitmap(from_ref); 1964 CHECK(los_bitmap != nullptr) << "LOS bitmap covers the entire address range"; 1965 bool is_los = mark_bitmap == nullptr; 1966 if (!is_los && mark_bitmap->Test(from_ref)) { 1967 // Already marked. 1968 to_ref = from_ref; 1969 } else if (is_los && los_bitmap->Test(from_ref)) { 1970 // Already marked in LOS. 1971 to_ref = from_ref; 1972 } else { 1973 // Not marked. 1974 if (IsOnAllocStack(from_ref)) { 1975 // If on the allocation stack, it's considered marked. 1976 to_ref = from_ref; 1977 } else { 1978 // Not marked. 1979 to_ref = nullptr; 1980 } 1981 } 1982 } 1983 } 1984 return to_ref; 1985} 1986 1987bool ConcurrentCopying::IsOnAllocStack(mirror::Object* ref) { 1988 QuasiAtomic::ThreadFenceAcquire(); 1989 accounting::ObjectStack* alloc_stack = GetAllocationStack(); 1990 return alloc_stack->Contains(ref); 1991} 1992 1993mirror::Object* ConcurrentCopying::Mark(mirror::Object* from_ref) { 1994 if (from_ref == nullptr) { 1995 return nullptr; 1996 } 1997 DCHECK(from_ref != nullptr); 1998 DCHECK(heap_->collector_type_ == kCollectorTypeCC); 1999 if (kUseBakerReadBarrier && !is_active_) { 2000 // In the lock word forward address state, the read barrier bits 2001 // in the lock word are part of the stored forwarding address and 2002 // invalid. This is usually OK as the from-space copy of objects 2003 // aren't accessed by mutators due to the to-space 2004 // invariant. However, during the dex2oat image writing relocation 2005 // and the zygote compaction, objects can be in the forward 2006 // address state (to store the forward/relocation addresses) and 2007 // they can still be accessed and the invalid read barrier bits 2008 // are consulted. If they look like gray but aren't really, the 2009 // read barriers slow path can trigger when it shouldn't. To guard 2010 // against this, return here if the CC collector isn't running. 2011 return from_ref; 2012 } 2013 DCHECK(region_space_ != nullptr) << "Read barrier slow path taken when CC isn't running?"; 2014 space::RegionSpace::RegionType rtype = region_space_->GetRegionType(from_ref); 2015 if (rtype == space::RegionSpace::RegionType::kRegionTypeToSpace) { 2016 // It's already marked. 2017 return from_ref; 2018 } 2019 mirror::Object* to_ref; 2020 if (rtype == space::RegionSpace::RegionType::kRegionTypeFromSpace) { 2021 to_ref = GetFwdPtr(from_ref); 2022 if (kUseBakerReadBarrier) { 2023 DCHECK(to_ref != ReadBarrier::GrayPtr()) << "from_ref=" << from_ref << " to_ref=" << to_ref; 2024 } 2025 if (to_ref == nullptr) { 2026 // It isn't marked yet. Mark it by copying it to the to-space. 2027 to_ref = Copy(from_ref); 2028 } 2029 DCHECK(region_space_->IsInToSpace(to_ref) || heap_->non_moving_space_->HasAddress(to_ref)) 2030 << "from_ref=" << from_ref << " to_ref=" << to_ref; 2031 } else if (rtype == space::RegionSpace::RegionType::kRegionTypeUnevacFromSpace) { 2032 // This may or may not succeed, which is ok. 2033 if (kUseBakerReadBarrier) { 2034 from_ref->AtomicSetReadBarrierPointer(ReadBarrier::WhitePtr(), ReadBarrier::GrayPtr()); 2035 } 2036 if (region_space_bitmap_->AtomicTestAndSet(from_ref)) { 2037 // Already marked. 2038 to_ref = from_ref; 2039 } else { 2040 // Newly marked. 2041 to_ref = from_ref; 2042 if (kUseBakerReadBarrier) { 2043 DCHECK(to_ref->GetReadBarrierPointer() == ReadBarrier::GrayPtr()); 2044 } 2045 PushOntoMarkStack(to_ref); 2046 } 2047 } else { 2048 // from_ref is in a non-moving space. 2049 DCHECK(!region_space_->HasAddress(from_ref)) << from_ref; 2050 if (immune_region_.ContainsObject(from_ref)) { 2051 accounting::ContinuousSpaceBitmap* cc_bitmap = 2052 cc_heap_bitmap_->GetContinuousSpaceBitmap(from_ref); 2053 DCHECK(cc_bitmap != nullptr) 2054 << "An immune space object must have a bitmap"; 2055 if (kIsDebugBuild) { 2056 DCHECK(heap_mark_bitmap_->GetContinuousSpaceBitmap(from_ref)->Test(from_ref)) 2057 << "Immune space object must be already marked"; 2058 } 2059 // This may or may not succeed, which is ok. 2060 if (kUseBakerReadBarrier) { 2061 from_ref->AtomicSetReadBarrierPointer(ReadBarrier::WhitePtr(), ReadBarrier::GrayPtr()); 2062 } 2063 if (cc_bitmap->AtomicTestAndSet(from_ref)) { 2064 // Already marked. 2065 to_ref = from_ref; 2066 } else { 2067 // Newly marked. 2068 to_ref = from_ref; 2069 if (kUseBakerReadBarrier) { 2070 DCHECK(to_ref->GetReadBarrierPointer() == ReadBarrier::GrayPtr()); 2071 } 2072 PushOntoMarkStack(to_ref); 2073 } 2074 } else { 2075 // Use the mark bitmap. 2076 accounting::ContinuousSpaceBitmap* mark_bitmap = 2077 heap_mark_bitmap_->GetContinuousSpaceBitmap(from_ref); 2078 accounting::LargeObjectBitmap* los_bitmap = 2079 heap_mark_bitmap_->GetLargeObjectBitmap(from_ref); 2080 CHECK(los_bitmap != nullptr) << "LOS bitmap covers the entire address range"; 2081 bool is_los = mark_bitmap == nullptr; 2082 if (!is_los && mark_bitmap->Test(from_ref)) { 2083 // Already marked. 2084 to_ref = from_ref; 2085 if (kUseBakerReadBarrier) { 2086 DCHECK(to_ref->GetReadBarrierPointer() == ReadBarrier::GrayPtr() || 2087 to_ref->GetReadBarrierPointer() == ReadBarrier::BlackPtr()); 2088 } 2089 } else if (is_los && los_bitmap->Test(from_ref)) { 2090 // Already marked in LOS. 2091 to_ref = from_ref; 2092 if (kUseBakerReadBarrier) { 2093 DCHECK(to_ref->GetReadBarrierPointer() == ReadBarrier::GrayPtr() || 2094 to_ref->GetReadBarrierPointer() == ReadBarrier::BlackPtr()); 2095 } 2096 } else { 2097 // Not marked. 2098 if (IsOnAllocStack(from_ref)) { 2099 // If it's on the allocation stack, it's considered marked. Keep it white. 2100 to_ref = from_ref; 2101 // Objects on the allocation stack need not be marked. 2102 if (!is_los) { 2103 DCHECK(!mark_bitmap->Test(to_ref)); 2104 } else { 2105 DCHECK(!los_bitmap->Test(to_ref)); 2106 } 2107 if (kUseBakerReadBarrier) { 2108 DCHECK(to_ref->GetReadBarrierPointer() == ReadBarrier::WhitePtr()); 2109 } 2110 } else { 2111 // Not marked or on the allocation stack. Try to mark it. 2112 // This may or may not succeed, which is ok. 2113 if (kUseBakerReadBarrier) { 2114 from_ref->AtomicSetReadBarrierPointer(ReadBarrier::WhitePtr(), ReadBarrier::GrayPtr()); 2115 } 2116 if (!is_los && mark_bitmap->AtomicTestAndSet(from_ref)) { 2117 // Already marked. 2118 to_ref = from_ref; 2119 } else if (is_los && los_bitmap->AtomicTestAndSet(from_ref)) { 2120 // Already marked in LOS. 2121 to_ref = from_ref; 2122 } else { 2123 // Newly marked. 2124 to_ref = from_ref; 2125 if (kUseBakerReadBarrier) { 2126 DCHECK(to_ref->GetReadBarrierPointer() == ReadBarrier::GrayPtr()); 2127 } 2128 PushOntoMarkStack(to_ref); 2129 } 2130 } 2131 } 2132 } 2133 } 2134 return to_ref; 2135} 2136 2137void ConcurrentCopying::FinishPhase() { 2138 { 2139 MutexLock mu(Thread::Current(), mark_stack_lock_); 2140 CHECK_EQ(pooled_mark_stacks_.size(), kMarkStackPoolSize); 2141 } 2142 region_space_ = nullptr; 2143 { 2144 MutexLock mu(Thread::Current(), skipped_blocks_lock_); 2145 skipped_blocks_map_.clear(); 2146 } 2147 WriterMutexLock mu(Thread::Current(), *Locks::heap_bitmap_lock_); 2148 heap_->ClearMarkedObjects(); 2149} 2150 2151bool ConcurrentCopying::IsMarkedHeapReference(mirror::HeapReference<mirror::Object>* field) { 2152 mirror::Object* from_ref = field->AsMirrorPtr(); 2153 mirror::Object* to_ref = IsMarked(from_ref); 2154 if (to_ref == nullptr) { 2155 return false; 2156 } 2157 if (from_ref != to_ref) { 2158 QuasiAtomic::ThreadFenceRelease(); 2159 field->Assign(to_ref); 2160 QuasiAtomic::ThreadFenceSequentiallyConsistent(); 2161 } 2162 return true; 2163} 2164 2165mirror::Object* ConcurrentCopying::MarkObject(mirror::Object* from_ref) { 2166 return Mark(from_ref); 2167} 2168 2169void ConcurrentCopying::DelayReferenceReferent(mirror::Class* klass, mirror::Reference* reference) { 2170 heap_->GetReferenceProcessor()->DelayReferenceReferent(klass, reference, this); 2171} 2172 2173void ConcurrentCopying::ProcessReferences(Thread* self) { 2174 TimingLogger::ScopedTiming split("ProcessReferences", GetTimings()); 2175 // We don't really need to lock the heap bitmap lock as we use CAS to mark in bitmaps. 2176 WriterMutexLock mu(self, *Locks::heap_bitmap_lock_); 2177 GetHeap()->GetReferenceProcessor()->ProcessReferences( 2178 true /*concurrent*/, GetTimings(), GetCurrentIteration()->GetClearSoftReferences(), this); 2179} 2180 2181void ConcurrentCopying::RevokeAllThreadLocalBuffers() { 2182 TimingLogger::ScopedTiming t(__FUNCTION__, GetTimings()); 2183 region_space_->RevokeAllThreadLocalBuffers(); 2184} 2185 2186} // namespace collector 2187} // namespace gc 2188} // namespace art 2189