concurrent_copying.cc revision a9d82fe8bc6960b565245b920e99107a824ca515
1/* 2 * Copyright (C) 2014 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17#include "concurrent_copying.h" 18 19#include "art_field-inl.h" 20#include "base/stl_util.h" 21#include "debugger.h" 22#include "gc/accounting/heap_bitmap-inl.h" 23#include "gc/accounting/space_bitmap-inl.h" 24#include "gc/reference_processor.h" 25#include "gc/space/image_space.h" 26#include "gc/space/space-inl.h" 27#include "intern_table.h" 28#include "mirror/class-inl.h" 29#include "mirror/object-inl.h" 30#include "scoped_thread_state_change.h" 31#include "thread-inl.h" 32#include "thread_list.h" 33#include "well_known_classes.h" 34 35namespace art { 36namespace gc { 37namespace collector { 38 39static constexpr size_t kDefaultGcMarkStackSize = 2 * MB; 40 41ConcurrentCopying::ConcurrentCopying(Heap* heap, const std::string& name_prefix) 42 : GarbageCollector(heap, 43 name_prefix + (name_prefix.empty() ? "" : " ") + 44 "concurrent copying + mark sweep"), 45 region_space_(nullptr), gc_barrier_(new Barrier(0)), 46 gc_mark_stack_(accounting::ObjectStack::Create("concurrent copying gc mark stack", 47 kDefaultGcMarkStackSize, 48 kDefaultGcMarkStackSize)), 49 mark_stack_lock_("concurrent copying mark stack lock", kMarkSweepMarkStackLock), 50 thread_running_gc_(nullptr), 51 is_marking_(false), is_active_(false), is_asserting_to_space_invariant_(false), 52 heap_mark_bitmap_(nullptr), live_stack_freeze_size_(0), mark_stack_mode_(kMarkStackModeOff), 53 weak_ref_access_enabled_(true), 54 skipped_blocks_lock_("concurrent copying bytes blocks lock", kMarkSweepMarkStackLock), 55 rb_table_(heap_->GetReadBarrierTable()), 56 force_evacuate_all_(false) { 57 static_assert(space::RegionSpace::kRegionSize == accounting::ReadBarrierTable::kRegionSize, 58 "The region space size and the read barrier table region size must match"); 59 cc_heap_bitmap_.reset(new accounting::HeapBitmap(heap)); 60 Thread* self = Thread::Current(); 61 { 62 ReaderMutexLock mu(self, *Locks::heap_bitmap_lock_); 63 // Cache this so that we won't have to lock heap_bitmap_lock_ in 64 // Mark() which could cause a nested lock on heap_bitmap_lock_ 65 // when GC causes a RB while doing GC or a lock order violation 66 // (class_linker_lock_ and heap_bitmap_lock_). 67 heap_mark_bitmap_ = heap->GetMarkBitmap(); 68 } 69 { 70 MutexLock mu(self, mark_stack_lock_); 71 for (size_t i = 0; i < kMarkStackPoolSize; ++i) { 72 accounting::AtomicStack<mirror::Object>* mark_stack = 73 accounting::AtomicStack<mirror::Object>::Create( 74 "thread local mark stack", kMarkStackSize, kMarkStackSize); 75 pooled_mark_stacks_.push_back(mark_stack); 76 } 77 } 78} 79 80void ConcurrentCopying::MarkHeapReference(mirror::HeapReference<mirror::Object>* from_ref) { 81 // Used for preserving soft references, should be OK to not have a CAS here since there should be 82 // no other threads which can trigger read barriers on the same referent during reference 83 // processing. 84 from_ref->Assign(Mark(from_ref->AsMirrorPtr())); 85 DCHECK(!from_ref->IsNull()); 86} 87 88ConcurrentCopying::~ConcurrentCopying() { 89 STLDeleteElements(&pooled_mark_stacks_); 90} 91 92void ConcurrentCopying::RunPhases() { 93 CHECK(kUseBakerReadBarrier || kUseTableLookupReadBarrier); 94 CHECK(!is_active_); 95 is_active_ = true; 96 Thread* self = Thread::Current(); 97 thread_running_gc_ = self; 98 Locks::mutator_lock_->AssertNotHeld(self); 99 { 100 ReaderMutexLock mu(self, *Locks::mutator_lock_); 101 InitializePhase(); 102 } 103 FlipThreadRoots(); 104 { 105 ReaderMutexLock mu(self, *Locks::mutator_lock_); 106 MarkingPhase(); 107 } 108 // Verify no from space refs. This causes a pause. 109 if (kEnableNoFromSpaceRefsVerification || kIsDebugBuild) { 110 TimingLogger::ScopedTiming split("(Paused)VerifyNoFromSpaceReferences", GetTimings()); 111 ScopedPause pause(this); 112 CheckEmptyMarkStack(); 113 if (kVerboseMode) { 114 LOG(INFO) << "Verifying no from-space refs"; 115 } 116 VerifyNoFromSpaceReferences(); 117 if (kVerboseMode) { 118 LOG(INFO) << "Done verifying no from-space refs"; 119 } 120 CheckEmptyMarkStack(); 121 } 122 { 123 ReaderMutexLock mu(self, *Locks::mutator_lock_); 124 ReclaimPhase(); 125 } 126 FinishPhase(); 127 CHECK(is_active_); 128 is_active_ = false; 129 thread_running_gc_ = nullptr; 130} 131 132void ConcurrentCopying::BindBitmaps() { 133 Thread* self = Thread::Current(); 134 WriterMutexLock mu(self, *Locks::heap_bitmap_lock_); 135 // Mark all of the spaces we never collect as immune. 136 for (const auto& space : heap_->GetContinuousSpaces()) { 137 if (space->GetGcRetentionPolicy() == space::kGcRetentionPolicyNeverCollect || 138 space->GetGcRetentionPolicy() == space::kGcRetentionPolicyFullCollect) { 139 CHECK(space->IsZygoteSpace() || space->IsImageSpace()); 140 immune_spaces_.AddSpace(space); 141 const char* bitmap_name = space->IsImageSpace() ? "cc image space bitmap" : 142 "cc zygote space bitmap"; 143 // TODO: try avoiding using bitmaps for image/zygote to save space. 144 accounting::ContinuousSpaceBitmap* bitmap = 145 accounting::ContinuousSpaceBitmap::Create(bitmap_name, space->Begin(), space->Capacity()); 146 cc_heap_bitmap_->AddContinuousSpaceBitmap(bitmap); 147 cc_bitmaps_.push_back(bitmap); 148 } else if (space == region_space_) { 149 accounting::ContinuousSpaceBitmap* bitmap = 150 accounting::ContinuousSpaceBitmap::Create("cc region space bitmap", 151 space->Begin(), space->Capacity()); 152 cc_heap_bitmap_->AddContinuousSpaceBitmap(bitmap); 153 cc_bitmaps_.push_back(bitmap); 154 region_space_bitmap_ = bitmap; 155 } 156 } 157} 158 159void ConcurrentCopying::InitializePhase() { 160 TimingLogger::ScopedTiming split("InitializePhase", GetTimings()); 161 if (kVerboseMode) { 162 LOG(INFO) << "GC InitializePhase"; 163 LOG(INFO) << "Region-space : " << reinterpret_cast<void*>(region_space_->Begin()) << "-" 164 << reinterpret_cast<void*>(region_space_->Limit()); 165 } 166 CheckEmptyMarkStack(); 167 immune_spaces_.Reset(); 168 bytes_moved_.StoreRelaxed(0); 169 objects_moved_.StoreRelaxed(0); 170 if (GetCurrentIteration()->GetGcCause() == kGcCauseExplicit || 171 GetCurrentIteration()->GetGcCause() == kGcCauseForNativeAlloc || 172 GetCurrentIteration()->GetClearSoftReferences()) { 173 force_evacuate_all_ = true; 174 } else { 175 force_evacuate_all_ = false; 176 } 177 BindBitmaps(); 178 if (kVerboseMode) { 179 LOG(INFO) << "force_evacuate_all=" << force_evacuate_all_; 180 LOG(INFO) << "Largest immune region: " << immune_spaces_.GetLargestImmuneRegion().Begin() 181 << "-" << immune_spaces_.GetLargestImmuneRegion().End(); 182 for (space::ContinuousSpace* space : immune_spaces_.GetSpaces()) { 183 LOG(INFO) << "Immune space: " << *space; 184 } 185 LOG(INFO) << "GC end of InitializePhase"; 186 } 187} 188 189// Used to switch the thread roots of a thread from from-space refs to to-space refs. 190class ThreadFlipVisitor : public Closure { 191 public: 192 ThreadFlipVisitor(ConcurrentCopying* concurrent_copying, bool use_tlab) 193 : concurrent_copying_(concurrent_copying), use_tlab_(use_tlab) { 194 } 195 196 virtual void Run(Thread* thread) OVERRIDE SHARED_REQUIRES(Locks::mutator_lock_) { 197 // Note: self is not necessarily equal to thread since thread may be suspended. 198 Thread* self = Thread::Current(); 199 CHECK(thread == self || thread->IsSuspended() || thread->GetState() == kWaitingPerformingGc) 200 << thread->GetState() << " thread " << thread << " self " << self; 201 thread->SetIsGcMarking(true); 202 if (use_tlab_ && thread->HasTlab()) { 203 if (ConcurrentCopying::kEnableFromSpaceAccountingCheck) { 204 // This must come before the revoke. 205 size_t thread_local_objects = thread->GetThreadLocalObjectsAllocated(); 206 concurrent_copying_->region_space_->RevokeThreadLocalBuffers(thread); 207 reinterpret_cast<Atomic<size_t>*>(&concurrent_copying_->from_space_num_objects_at_first_pause_)-> 208 FetchAndAddSequentiallyConsistent(thread_local_objects); 209 } else { 210 concurrent_copying_->region_space_->RevokeThreadLocalBuffers(thread); 211 } 212 } 213 if (kUseThreadLocalAllocationStack) { 214 thread->RevokeThreadLocalAllocationStack(); 215 } 216 ReaderMutexLock mu(self, *Locks::heap_bitmap_lock_); 217 thread->VisitRoots(concurrent_copying_); 218 concurrent_copying_->GetBarrier().Pass(self); 219 } 220 221 private: 222 ConcurrentCopying* const concurrent_copying_; 223 const bool use_tlab_; 224}; 225 226// Called back from Runtime::FlipThreadRoots() during a pause. 227class FlipCallback : public Closure { 228 public: 229 explicit FlipCallback(ConcurrentCopying* concurrent_copying) 230 : concurrent_copying_(concurrent_copying) { 231 } 232 233 virtual void Run(Thread* thread) OVERRIDE REQUIRES(Locks::mutator_lock_) { 234 ConcurrentCopying* cc = concurrent_copying_; 235 TimingLogger::ScopedTiming split("(Paused)FlipCallback", cc->GetTimings()); 236 // Note: self is not necessarily equal to thread since thread may be suspended. 237 Thread* self = Thread::Current(); 238 CHECK(thread == self); 239 Locks::mutator_lock_->AssertExclusiveHeld(self); 240 cc->region_space_->SetFromSpace(cc->rb_table_, cc->force_evacuate_all_); 241 cc->SwapStacks(); 242 if (ConcurrentCopying::kEnableFromSpaceAccountingCheck) { 243 cc->RecordLiveStackFreezeSize(self); 244 cc->from_space_num_objects_at_first_pause_ = cc->region_space_->GetObjectsAllocated(); 245 cc->from_space_num_bytes_at_first_pause_ = cc->region_space_->GetBytesAllocated(); 246 } 247 cc->is_marking_ = true; 248 cc->mark_stack_mode_.StoreRelaxed(ConcurrentCopying::kMarkStackModeThreadLocal); 249 if (UNLIKELY(Runtime::Current()->IsActiveTransaction())) { 250 CHECK(Runtime::Current()->IsAotCompiler()); 251 TimingLogger::ScopedTiming split2("(Paused)VisitTransactionRoots", cc->GetTimings()); 252 Runtime::Current()->VisitTransactionRoots(cc); 253 } 254 } 255 256 private: 257 ConcurrentCopying* const concurrent_copying_; 258}; 259 260// Switch threads that from from-space to to-space refs. Forward/mark the thread roots. 261void ConcurrentCopying::FlipThreadRoots() { 262 TimingLogger::ScopedTiming split("FlipThreadRoots", GetTimings()); 263 if (kVerboseMode) { 264 LOG(INFO) << "time=" << region_space_->Time(); 265 region_space_->DumpNonFreeRegions(LOG(INFO)); 266 } 267 Thread* self = Thread::Current(); 268 Locks::mutator_lock_->AssertNotHeld(self); 269 gc_barrier_->Init(self, 0); 270 ThreadFlipVisitor thread_flip_visitor(this, heap_->use_tlab_); 271 FlipCallback flip_callback(this); 272 heap_->ThreadFlipBegin(self); // Sync with JNI critical calls. 273 size_t barrier_count = Runtime::Current()->FlipThreadRoots( 274 &thread_flip_visitor, &flip_callback, this); 275 heap_->ThreadFlipEnd(self); 276 { 277 ScopedThreadStateChange tsc(self, kWaitingForCheckPointsToRun); 278 gc_barrier_->Increment(self, barrier_count); 279 } 280 is_asserting_to_space_invariant_ = true; 281 QuasiAtomic::ThreadFenceForConstructor(); 282 if (kVerboseMode) { 283 LOG(INFO) << "time=" << region_space_->Time(); 284 region_space_->DumpNonFreeRegions(LOG(INFO)); 285 LOG(INFO) << "GC end of FlipThreadRoots"; 286 } 287} 288 289void ConcurrentCopying::SwapStacks() { 290 heap_->SwapStacks(); 291} 292 293void ConcurrentCopying::RecordLiveStackFreezeSize(Thread* self) { 294 WriterMutexLock mu(self, *Locks::heap_bitmap_lock_); 295 live_stack_freeze_size_ = heap_->GetLiveStack()->Size(); 296} 297 298// Used to visit objects in the immune spaces. 299class ConcurrentCopyingImmuneSpaceObjVisitor { 300 public: 301 explicit ConcurrentCopyingImmuneSpaceObjVisitor(ConcurrentCopying* cc) 302 : collector_(cc) {} 303 304 void operator()(mirror::Object* obj) const SHARED_REQUIRES(Locks::mutator_lock_) 305 SHARED_REQUIRES(Locks::heap_bitmap_lock_) { 306 DCHECK(obj != nullptr); 307 DCHECK(collector_->immune_spaces_.ContainsObject(obj)); 308 accounting::ContinuousSpaceBitmap* cc_bitmap = 309 collector_->cc_heap_bitmap_->GetContinuousSpaceBitmap(obj); 310 DCHECK(cc_bitmap != nullptr) 311 << "An immune space object must have a bitmap"; 312 if (kIsDebugBuild) { 313 DCHECK(collector_->heap_->GetMarkBitmap()->Test(obj)) 314 << "Immune space object must be already marked"; 315 } 316 // This may or may not succeed, which is ok. 317 if (kUseBakerReadBarrier) { 318 obj->AtomicSetReadBarrierPointer(ReadBarrier::WhitePtr(), ReadBarrier::GrayPtr()); 319 } 320 if (cc_bitmap->AtomicTestAndSet(obj)) { 321 // Already marked. Do nothing. 322 } else { 323 // Newly marked. Set the gray bit and push it onto the mark stack. 324 CHECK(!kUseBakerReadBarrier || obj->GetReadBarrierPointer() == ReadBarrier::GrayPtr()); 325 collector_->PushOntoMarkStack(obj); 326 } 327 } 328 329 private: 330 ConcurrentCopying* const collector_; 331}; 332 333class EmptyCheckpoint : public Closure { 334 public: 335 explicit EmptyCheckpoint(ConcurrentCopying* concurrent_copying) 336 : concurrent_copying_(concurrent_copying) { 337 } 338 339 virtual void Run(Thread* thread) OVERRIDE NO_THREAD_SAFETY_ANALYSIS { 340 // Note: self is not necessarily equal to thread since thread may be suspended. 341 Thread* self = Thread::Current(); 342 CHECK(thread == self || thread->IsSuspended() || thread->GetState() == kWaitingPerformingGc) 343 << thread->GetState() << " thread " << thread << " self " << self; 344 // If thread is a running mutator, then act on behalf of the garbage collector. 345 // See the code in ThreadList::RunCheckpoint. 346 concurrent_copying_->GetBarrier().Pass(self); 347 } 348 349 private: 350 ConcurrentCopying* const concurrent_copying_; 351}; 352 353// Concurrently mark roots that are guarded by read barriers and process the mark stack. 354void ConcurrentCopying::MarkingPhase() { 355 TimingLogger::ScopedTiming split("MarkingPhase", GetTimings()); 356 if (kVerboseMode) { 357 LOG(INFO) << "GC MarkingPhase"; 358 } 359 CHECK(weak_ref_access_enabled_); 360 { 361 // Mark the image root. The WB-based collectors do not need to 362 // scan the image objects from roots by relying on the card table, 363 // but it's necessary for the RB to-space invariant to hold. 364 TimingLogger::ScopedTiming split1("VisitImageRoots", GetTimings()); 365 for (space::ContinuousSpace* space : heap_->GetContinuousSpaces()) { 366 if (space->IsImageSpace()) { 367 gc::space::ImageSpace* image = space->AsImageSpace(); 368 if (image != nullptr) { 369 mirror::ObjectArray<mirror::Object>* image_root = image->GetImageHeader().GetImageRoots(); 370 mirror::Object* marked_image_root = Mark(image_root); 371 CHECK_EQ(image_root, marked_image_root) << "An image object does not move"; 372 if (ReadBarrier::kEnableToSpaceInvariantChecks) { 373 AssertToSpaceInvariant(nullptr, MemberOffset(0), marked_image_root); 374 } 375 } 376 } 377 } 378 } 379 { 380 TimingLogger::ScopedTiming split2("VisitConcurrentRoots", GetTimings()); 381 Runtime::Current()->VisitConcurrentRoots(this, kVisitRootFlagAllRoots); 382 } 383 { 384 // TODO: don't visit the transaction roots if it's not active. 385 TimingLogger::ScopedTiming split5("VisitNonThreadRoots", GetTimings()); 386 Runtime::Current()->VisitNonThreadRoots(this); 387 } 388 389 // Immune spaces. 390 for (auto& space : immune_spaces_.GetSpaces()) { 391 DCHECK(space->IsImageSpace() || space->IsZygoteSpace()); 392 accounting::ContinuousSpaceBitmap* live_bitmap = space->GetLiveBitmap(); 393 ConcurrentCopyingImmuneSpaceObjVisitor visitor(this); 394 live_bitmap->VisitMarkedRange(reinterpret_cast<uintptr_t>(space->Begin()), 395 reinterpret_cast<uintptr_t>(space->Limit()), 396 visitor); 397 } 398 399 Thread* self = Thread::Current(); 400 { 401 TimingLogger::ScopedTiming split7("ProcessMarkStack", GetTimings()); 402 // We transition through three mark stack modes (thread-local, shared, GC-exclusive). The 403 // primary reasons are the fact that we need to use a checkpoint to process thread-local mark 404 // stacks, but after we disable weak refs accesses, we can't use a checkpoint due to a deadlock 405 // issue because running threads potentially blocking at WaitHoldingLocks, and that once we 406 // reach the point where we process weak references, we can avoid using a lock when accessing 407 // the GC mark stack, which makes mark stack processing more efficient. 408 409 // Process the mark stack once in the thread local stack mode. This marks most of the live 410 // objects, aside from weak ref accesses with read barriers (Reference::GetReferent() and system 411 // weaks) that may happen concurrently while we processing the mark stack and newly mark/gray 412 // objects and push refs on the mark stack. 413 ProcessMarkStack(); 414 // Switch to the shared mark stack mode. That is, revoke and process thread-local mark stacks 415 // for the last time before transitioning to the shared mark stack mode, which would process new 416 // refs that may have been concurrently pushed onto the mark stack during the ProcessMarkStack() 417 // call above. At the same time, disable weak ref accesses using a per-thread flag. It's 418 // important to do these together in a single checkpoint so that we can ensure that mutators 419 // won't newly gray objects and push new refs onto the mark stack due to weak ref accesses and 420 // mutators safely transition to the shared mark stack mode (without leaving unprocessed refs on 421 // the thread-local mark stacks), without a race. This is why we use a thread-local weak ref 422 // access flag Thread::tls32_.weak_ref_access_enabled_ instead of the global ones. 423 SwitchToSharedMarkStackMode(); 424 CHECK(!self->GetWeakRefAccessEnabled()); 425 // Now that weak refs accesses are disabled, once we exhaust the shared mark stack again here 426 // (which may be non-empty if there were refs found on thread-local mark stacks during the above 427 // SwitchToSharedMarkStackMode() call), we won't have new refs to process, that is, mutators 428 // (via read barriers) have no way to produce any more refs to process. Marking converges once 429 // before we process weak refs below. 430 ProcessMarkStack(); 431 CheckEmptyMarkStack(); 432 // Switch to the GC exclusive mark stack mode so that we can process the mark stack without a 433 // lock from this point on. 434 SwitchToGcExclusiveMarkStackMode(); 435 CheckEmptyMarkStack(); 436 if (kVerboseMode) { 437 LOG(INFO) << "ProcessReferences"; 438 } 439 // Process weak references. This may produce new refs to process and have them processed via 440 // ProcessMarkStack (in the GC exclusive mark stack mode). 441 ProcessReferences(self); 442 CheckEmptyMarkStack(); 443 if (kVerboseMode) { 444 LOG(INFO) << "SweepSystemWeaks"; 445 } 446 SweepSystemWeaks(self); 447 if (kVerboseMode) { 448 LOG(INFO) << "SweepSystemWeaks done"; 449 } 450 // Process the mark stack here one last time because the above SweepSystemWeaks() call may have 451 // marked some objects (strings alive) as hash_set::Erase() can call the hash function for 452 // arbitrary elements in the weak intern table in InternTable::Table::SweepWeaks(). 453 ProcessMarkStack(); 454 CheckEmptyMarkStack(); 455 // Re-enable weak ref accesses. 456 ReenableWeakRefAccess(self); 457 // Free data for class loaders that we unloaded. 458 Runtime::Current()->GetClassLinker()->CleanupClassLoaders(); 459 // Marking is done. Disable marking. 460 DisableMarking(); 461 CheckEmptyMarkStack(); 462 } 463 464 CHECK(weak_ref_access_enabled_); 465 if (kVerboseMode) { 466 LOG(INFO) << "GC end of MarkingPhase"; 467 } 468} 469 470void ConcurrentCopying::ReenableWeakRefAccess(Thread* self) { 471 if (kVerboseMode) { 472 LOG(INFO) << "ReenableWeakRefAccess"; 473 } 474 weak_ref_access_enabled_.StoreRelaxed(true); // This is for new threads. 475 QuasiAtomic::ThreadFenceForConstructor(); 476 // Iterate all threads (don't need to or can't use a checkpoint) and re-enable weak ref access. 477 { 478 MutexLock mu(self, *Locks::thread_list_lock_); 479 std::list<Thread*> thread_list = Runtime::Current()->GetThreadList()->GetList(); 480 for (Thread* thread : thread_list) { 481 thread->SetWeakRefAccessEnabled(true); 482 } 483 } 484 // Unblock blocking threads. 485 GetHeap()->GetReferenceProcessor()->BroadcastForSlowPath(self); 486 Runtime::Current()->BroadcastForNewSystemWeaks(); 487} 488 489class DisableMarkingCheckpoint : public Closure { 490 public: 491 explicit DisableMarkingCheckpoint(ConcurrentCopying* concurrent_copying) 492 : concurrent_copying_(concurrent_copying) { 493 } 494 495 void Run(Thread* thread) OVERRIDE NO_THREAD_SAFETY_ANALYSIS { 496 // Note: self is not necessarily equal to thread since thread may be suspended. 497 Thread* self = Thread::Current(); 498 DCHECK(thread == self || thread->IsSuspended() || thread->GetState() == kWaitingPerformingGc) 499 << thread->GetState() << " thread " << thread << " self " << self; 500 // Disable the thread-local is_gc_marking flag. 501 // Note a thread that has just started right before this checkpoint may have already this flag 502 // set to false, which is ok. 503 thread->SetIsGcMarking(false); 504 // If thread is a running mutator, then act on behalf of the garbage collector. 505 // See the code in ThreadList::RunCheckpoint. 506 concurrent_copying_->GetBarrier().Pass(self); 507 } 508 509 private: 510 ConcurrentCopying* const concurrent_copying_; 511}; 512 513void ConcurrentCopying::IssueDisableMarkingCheckpoint() { 514 Thread* self = Thread::Current(); 515 DisableMarkingCheckpoint check_point(this); 516 ThreadList* thread_list = Runtime::Current()->GetThreadList(); 517 gc_barrier_->Init(self, 0); 518 size_t barrier_count = thread_list->RunCheckpoint(&check_point); 519 // If there are no threads to wait which implies that all the checkpoint functions are finished, 520 // then no need to release the mutator lock. 521 if (barrier_count == 0) { 522 return; 523 } 524 // Release locks then wait for all mutator threads to pass the barrier. 525 Locks::mutator_lock_->SharedUnlock(self); 526 { 527 ScopedThreadStateChange tsc(self, kWaitingForCheckPointsToRun); 528 gc_barrier_->Increment(self, barrier_count); 529 } 530 Locks::mutator_lock_->SharedLock(self); 531} 532 533void ConcurrentCopying::DisableMarking() { 534 // Change the global is_marking flag to false. Do a fence before doing a checkpoint to update the 535 // thread-local flags so that a new thread starting up will get the correct is_marking flag. 536 is_marking_ = false; 537 QuasiAtomic::ThreadFenceForConstructor(); 538 // Use a checkpoint to turn off the thread-local is_gc_marking flags and to ensure no threads are 539 // still in the middle of a read barrier which may have a from-space ref cached in a local 540 // variable. 541 IssueDisableMarkingCheckpoint(); 542 if (kUseTableLookupReadBarrier) { 543 heap_->rb_table_->ClearAll(); 544 DCHECK(heap_->rb_table_->IsAllCleared()); 545 } 546 is_mark_stack_push_disallowed_.StoreSequentiallyConsistent(1); 547 mark_stack_mode_.StoreSequentiallyConsistent(kMarkStackModeOff); 548} 549 550void ConcurrentCopying::IssueEmptyCheckpoint() { 551 Thread* self = Thread::Current(); 552 EmptyCheckpoint check_point(this); 553 ThreadList* thread_list = Runtime::Current()->GetThreadList(); 554 gc_barrier_->Init(self, 0); 555 size_t barrier_count = thread_list->RunCheckpoint(&check_point); 556 // If there are no threads to wait which implys that all the checkpoint functions are finished, 557 // then no need to release the mutator lock. 558 if (barrier_count == 0) { 559 return; 560 } 561 // Release locks then wait for all mutator threads to pass the barrier. 562 Locks::mutator_lock_->SharedUnlock(self); 563 { 564 ScopedThreadStateChange tsc(self, kWaitingForCheckPointsToRun); 565 gc_barrier_->Increment(self, barrier_count); 566 } 567 Locks::mutator_lock_->SharedLock(self); 568} 569 570void ConcurrentCopying::ExpandGcMarkStack() { 571 DCHECK(gc_mark_stack_->IsFull()); 572 const size_t new_size = gc_mark_stack_->Capacity() * 2; 573 std::vector<StackReference<mirror::Object>> temp(gc_mark_stack_->Begin(), 574 gc_mark_stack_->End()); 575 gc_mark_stack_->Resize(new_size); 576 for (auto& ref : temp) { 577 gc_mark_stack_->PushBack(ref.AsMirrorPtr()); 578 } 579 DCHECK(!gc_mark_stack_->IsFull()); 580} 581 582void ConcurrentCopying::PushOntoMarkStack(mirror::Object* to_ref) { 583 CHECK_EQ(is_mark_stack_push_disallowed_.LoadRelaxed(), 0) 584 << " " << to_ref << " " << PrettyTypeOf(to_ref); 585 Thread* self = Thread::Current(); // TODO: pass self as an argument from call sites? 586 CHECK(thread_running_gc_ != nullptr); 587 MarkStackMode mark_stack_mode = mark_stack_mode_.LoadRelaxed(); 588 if (LIKELY(mark_stack_mode == kMarkStackModeThreadLocal)) { 589 if (LIKELY(self == thread_running_gc_)) { 590 // If GC-running thread, use the GC mark stack instead of a thread-local mark stack. 591 CHECK(self->GetThreadLocalMarkStack() == nullptr); 592 if (UNLIKELY(gc_mark_stack_->IsFull())) { 593 ExpandGcMarkStack(); 594 } 595 gc_mark_stack_->PushBack(to_ref); 596 } else { 597 // Otherwise, use a thread-local mark stack. 598 accounting::AtomicStack<mirror::Object>* tl_mark_stack = self->GetThreadLocalMarkStack(); 599 if (UNLIKELY(tl_mark_stack == nullptr || tl_mark_stack->IsFull())) { 600 MutexLock mu(self, mark_stack_lock_); 601 // Get a new thread local mark stack. 602 accounting::AtomicStack<mirror::Object>* new_tl_mark_stack; 603 if (!pooled_mark_stacks_.empty()) { 604 // Use a pooled mark stack. 605 new_tl_mark_stack = pooled_mark_stacks_.back(); 606 pooled_mark_stacks_.pop_back(); 607 } else { 608 // None pooled. Create a new one. 609 new_tl_mark_stack = 610 accounting::AtomicStack<mirror::Object>::Create( 611 "thread local mark stack", 4 * KB, 4 * KB); 612 } 613 DCHECK(new_tl_mark_stack != nullptr); 614 DCHECK(new_tl_mark_stack->IsEmpty()); 615 new_tl_mark_stack->PushBack(to_ref); 616 self->SetThreadLocalMarkStack(new_tl_mark_stack); 617 if (tl_mark_stack != nullptr) { 618 // Store the old full stack into a vector. 619 revoked_mark_stacks_.push_back(tl_mark_stack); 620 } 621 } else { 622 tl_mark_stack->PushBack(to_ref); 623 } 624 } 625 } else if (mark_stack_mode == kMarkStackModeShared) { 626 // Access the shared GC mark stack with a lock. 627 MutexLock mu(self, mark_stack_lock_); 628 if (UNLIKELY(gc_mark_stack_->IsFull())) { 629 ExpandGcMarkStack(); 630 } 631 gc_mark_stack_->PushBack(to_ref); 632 } else { 633 CHECK_EQ(static_cast<uint32_t>(mark_stack_mode), 634 static_cast<uint32_t>(kMarkStackModeGcExclusive)) 635 << "ref=" << to_ref 636 << " self->gc_marking=" << self->GetIsGcMarking() 637 << " cc->is_marking=" << is_marking_; 638 CHECK(self == thread_running_gc_) 639 << "Only GC-running thread should access the mark stack " 640 << "in the GC exclusive mark stack mode"; 641 // Access the GC mark stack without a lock. 642 if (UNLIKELY(gc_mark_stack_->IsFull())) { 643 ExpandGcMarkStack(); 644 } 645 gc_mark_stack_->PushBack(to_ref); 646 } 647} 648 649accounting::ObjectStack* ConcurrentCopying::GetAllocationStack() { 650 return heap_->allocation_stack_.get(); 651} 652 653accounting::ObjectStack* ConcurrentCopying::GetLiveStack() { 654 return heap_->live_stack_.get(); 655} 656 657// The following visitors are that used to verify that there's no 658// references to the from-space left after marking. 659class ConcurrentCopyingVerifyNoFromSpaceRefsVisitor : public SingleRootVisitor { 660 public: 661 explicit ConcurrentCopyingVerifyNoFromSpaceRefsVisitor(ConcurrentCopying* collector) 662 : collector_(collector) {} 663 664 void operator()(mirror::Object* ref) const 665 SHARED_REQUIRES(Locks::mutator_lock_) ALWAYS_INLINE { 666 if (ref == nullptr) { 667 // OK. 668 return; 669 } 670 collector_->AssertToSpaceInvariant(nullptr, MemberOffset(0), ref); 671 if (kUseBakerReadBarrier) { 672 if (collector_->RegionSpace()->IsInToSpace(ref)) { 673 CHECK(ref->GetReadBarrierPointer() == nullptr) 674 << "To-space ref " << ref << " " << PrettyTypeOf(ref) 675 << " has non-white rb_ptr " << ref->GetReadBarrierPointer(); 676 } else { 677 CHECK(ref->GetReadBarrierPointer() == ReadBarrier::BlackPtr() || 678 (ref->GetReadBarrierPointer() == ReadBarrier::WhitePtr() && 679 collector_->IsOnAllocStack(ref))) 680 << "Non-moving/unevac from space ref " << ref << " " << PrettyTypeOf(ref) 681 << " has non-black rb_ptr " << ref->GetReadBarrierPointer() 682 << " but isn't on the alloc stack (and has white rb_ptr)." 683 << " Is it in the non-moving space=" 684 << (collector_->GetHeap()->GetNonMovingSpace()->HasAddress(ref)); 685 } 686 } 687 } 688 689 void VisitRoot(mirror::Object* root, const RootInfo& info ATTRIBUTE_UNUSED) 690 OVERRIDE SHARED_REQUIRES(Locks::mutator_lock_) { 691 DCHECK(root != nullptr); 692 operator()(root); 693 } 694 695 private: 696 ConcurrentCopying* const collector_; 697}; 698 699class ConcurrentCopyingVerifyNoFromSpaceRefsFieldVisitor { 700 public: 701 explicit ConcurrentCopyingVerifyNoFromSpaceRefsFieldVisitor(ConcurrentCopying* collector) 702 : collector_(collector) {} 703 704 void operator()(mirror::Object* obj, MemberOffset offset, bool is_static ATTRIBUTE_UNUSED) const 705 SHARED_REQUIRES(Locks::mutator_lock_) ALWAYS_INLINE { 706 mirror::Object* ref = 707 obj->GetFieldObject<mirror::Object, kDefaultVerifyFlags, kWithoutReadBarrier>(offset); 708 ConcurrentCopyingVerifyNoFromSpaceRefsVisitor visitor(collector_); 709 visitor(ref); 710 } 711 void operator()(mirror::Class* klass, mirror::Reference* ref) const 712 SHARED_REQUIRES(Locks::mutator_lock_) ALWAYS_INLINE { 713 CHECK(klass->IsTypeOfReferenceClass()); 714 this->operator()(ref, mirror::Reference::ReferentOffset(), false); 715 } 716 717 void VisitRootIfNonNull(mirror::CompressedReference<mirror::Object>* root) const 718 SHARED_REQUIRES(Locks::mutator_lock_) { 719 if (!root->IsNull()) { 720 VisitRoot(root); 721 } 722 } 723 724 void VisitRoot(mirror::CompressedReference<mirror::Object>* root) const 725 SHARED_REQUIRES(Locks::mutator_lock_) { 726 ConcurrentCopyingVerifyNoFromSpaceRefsVisitor visitor(collector_); 727 visitor(root->AsMirrorPtr()); 728 } 729 730 private: 731 ConcurrentCopying* const collector_; 732}; 733 734class ConcurrentCopyingVerifyNoFromSpaceRefsObjectVisitor { 735 public: 736 explicit ConcurrentCopyingVerifyNoFromSpaceRefsObjectVisitor(ConcurrentCopying* collector) 737 : collector_(collector) {} 738 void operator()(mirror::Object* obj) const 739 SHARED_REQUIRES(Locks::mutator_lock_) { 740 ObjectCallback(obj, collector_); 741 } 742 static void ObjectCallback(mirror::Object* obj, void *arg) 743 SHARED_REQUIRES(Locks::mutator_lock_) { 744 CHECK(obj != nullptr); 745 ConcurrentCopying* collector = reinterpret_cast<ConcurrentCopying*>(arg); 746 space::RegionSpace* region_space = collector->RegionSpace(); 747 CHECK(!region_space->IsInFromSpace(obj)) << "Scanning object " << obj << " in from space"; 748 ConcurrentCopyingVerifyNoFromSpaceRefsFieldVisitor visitor(collector); 749 obj->VisitReferences(visitor, visitor); 750 if (kUseBakerReadBarrier) { 751 if (collector->RegionSpace()->IsInToSpace(obj)) { 752 CHECK(obj->GetReadBarrierPointer() == nullptr) 753 << "obj=" << obj << " non-white rb_ptr " << obj->GetReadBarrierPointer(); 754 } else { 755 CHECK(obj->GetReadBarrierPointer() == ReadBarrier::BlackPtr() || 756 (obj->GetReadBarrierPointer() == ReadBarrier::WhitePtr() && 757 collector->IsOnAllocStack(obj))) 758 << "Non-moving space/unevac from space ref " << obj << " " << PrettyTypeOf(obj) 759 << " has non-black rb_ptr " << obj->GetReadBarrierPointer() 760 << " but isn't on the alloc stack (and has white rb_ptr). Is it in the non-moving space=" 761 << (collector->GetHeap()->GetNonMovingSpace()->HasAddress(obj)); 762 } 763 } 764 } 765 766 private: 767 ConcurrentCopying* const collector_; 768}; 769 770// Verify there's no from-space references left after the marking phase. 771void ConcurrentCopying::VerifyNoFromSpaceReferences() { 772 Thread* self = Thread::Current(); 773 DCHECK(Locks::mutator_lock_->IsExclusiveHeld(self)); 774 // Verify all threads have is_gc_marking to be false 775 { 776 MutexLock mu(self, *Locks::thread_list_lock_); 777 std::list<Thread*> thread_list = Runtime::Current()->GetThreadList()->GetList(); 778 for (Thread* thread : thread_list) { 779 CHECK(!thread->GetIsGcMarking()); 780 } 781 } 782 ConcurrentCopyingVerifyNoFromSpaceRefsObjectVisitor visitor(this); 783 // Roots. 784 { 785 ReaderMutexLock mu(self, *Locks::heap_bitmap_lock_); 786 ConcurrentCopyingVerifyNoFromSpaceRefsVisitor ref_visitor(this); 787 Runtime::Current()->VisitRoots(&ref_visitor); 788 } 789 // The to-space. 790 region_space_->WalkToSpace(ConcurrentCopyingVerifyNoFromSpaceRefsObjectVisitor::ObjectCallback, 791 this); 792 // Non-moving spaces. 793 { 794 WriterMutexLock mu(self, *Locks::heap_bitmap_lock_); 795 heap_->GetMarkBitmap()->Visit(visitor); 796 } 797 // The alloc stack. 798 { 799 ConcurrentCopyingVerifyNoFromSpaceRefsVisitor ref_visitor(this); 800 for (auto* it = heap_->allocation_stack_->Begin(), *end = heap_->allocation_stack_->End(); 801 it < end; ++it) { 802 mirror::Object* const obj = it->AsMirrorPtr(); 803 if (obj != nullptr && obj->GetClass() != nullptr) { 804 // TODO: need to call this only if obj is alive? 805 ref_visitor(obj); 806 visitor(obj); 807 } 808 } 809 } 810 // TODO: LOS. But only refs in LOS are classes. 811} 812 813// The following visitors are used to assert the to-space invariant. 814class ConcurrentCopyingAssertToSpaceInvariantRefsVisitor { 815 public: 816 explicit ConcurrentCopyingAssertToSpaceInvariantRefsVisitor(ConcurrentCopying* collector) 817 : collector_(collector) {} 818 819 void operator()(mirror::Object* ref) const 820 SHARED_REQUIRES(Locks::mutator_lock_) ALWAYS_INLINE { 821 if (ref == nullptr) { 822 // OK. 823 return; 824 } 825 collector_->AssertToSpaceInvariant(nullptr, MemberOffset(0), ref); 826 } 827 828 private: 829 ConcurrentCopying* const collector_; 830}; 831 832class ConcurrentCopyingAssertToSpaceInvariantFieldVisitor { 833 public: 834 explicit ConcurrentCopyingAssertToSpaceInvariantFieldVisitor(ConcurrentCopying* collector) 835 : collector_(collector) {} 836 837 void operator()(mirror::Object* obj, MemberOffset offset, bool is_static ATTRIBUTE_UNUSED) const 838 SHARED_REQUIRES(Locks::mutator_lock_) ALWAYS_INLINE { 839 mirror::Object* ref = 840 obj->GetFieldObject<mirror::Object, kDefaultVerifyFlags, kWithoutReadBarrier>(offset); 841 ConcurrentCopyingAssertToSpaceInvariantRefsVisitor visitor(collector_); 842 visitor(ref); 843 } 844 void operator()(mirror::Class* klass, mirror::Reference* ref ATTRIBUTE_UNUSED) const 845 SHARED_REQUIRES(Locks::mutator_lock_) ALWAYS_INLINE { 846 CHECK(klass->IsTypeOfReferenceClass()); 847 } 848 849 void VisitRootIfNonNull(mirror::CompressedReference<mirror::Object>* root) const 850 SHARED_REQUIRES(Locks::mutator_lock_) { 851 if (!root->IsNull()) { 852 VisitRoot(root); 853 } 854 } 855 856 void VisitRoot(mirror::CompressedReference<mirror::Object>* root) const 857 SHARED_REQUIRES(Locks::mutator_lock_) { 858 ConcurrentCopyingAssertToSpaceInvariantRefsVisitor visitor(collector_); 859 visitor(root->AsMirrorPtr()); 860 } 861 862 private: 863 ConcurrentCopying* const collector_; 864}; 865 866class ConcurrentCopyingAssertToSpaceInvariantObjectVisitor { 867 public: 868 explicit ConcurrentCopyingAssertToSpaceInvariantObjectVisitor(ConcurrentCopying* collector) 869 : collector_(collector) {} 870 void operator()(mirror::Object* obj) const 871 SHARED_REQUIRES(Locks::mutator_lock_) { 872 ObjectCallback(obj, collector_); 873 } 874 static void ObjectCallback(mirror::Object* obj, void *arg) 875 SHARED_REQUIRES(Locks::mutator_lock_) { 876 CHECK(obj != nullptr); 877 ConcurrentCopying* collector = reinterpret_cast<ConcurrentCopying*>(arg); 878 space::RegionSpace* region_space = collector->RegionSpace(); 879 CHECK(!region_space->IsInFromSpace(obj)) << "Scanning object " << obj << " in from space"; 880 collector->AssertToSpaceInvariant(nullptr, MemberOffset(0), obj); 881 ConcurrentCopyingAssertToSpaceInvariantFieldVisitor visitor(collector); 882 obj->VisitReferences(visitor, visitor); 883 } 884 885 private: 886 ConcurrentCopying* const collector_; 887}; 888 889class RevokeThreadLocalMarkStackCheckpoint : public Closure { 890 public: 891 RevokeThreadLocalMarkStackCheckpoint(ConcurrentCopying* concurrent_copying, 892 bool disable_weak_ref_access) 893 : concurrent_copying_(concurrent_copying), 894 disable_weak_ref_access_(disable_weak_ref_access) { 895 } 896 897 virtual void Run(Thread* thread) OVERRIDE NO_THREAD_SAFETY_ANALYSIS { 898 // Note: self is not necessarily equal to thread since thread may be suspended. 899 Thread* self = Thread::Current(); 900 CHECK(thread == self || thread->IsSuspended() || thread->GetState() == kWaitingPerformingGc) 901 << thread->GetState() << " thread " << thread << " self " << self; 902 // Revoke thread local mark stacks. 903 accounting::AtomicStack<mirror::Object>* tl_mark_stack = thread->GetThreadLocalMarkStack(); 904 if (tl_mark_stack != nullptr) { 905 MutexLock mu(self, concurrent_copying_->mark_stack_lock_); 906 concurrent_copying_->revoked_mark_stacks_.push_back(tl_mark_stack); 907 thread->SetThreadLocalMarkStack(nullptr); 908 } 909 // Disable weak ref access. 910 if (disable_weak_ref_access_) { 911 thread->SetWeakRefAccessEnabled(false); 912 } 913 // If thread is a running mutator, then act on behalf of the garbage collector. 914 // See the code in ThreadList::RunCheckpoint. 915 concurrent_copying_->GetBarrier().Pass(self); 916 } 917 918 private: 919 ConcurrentCopying* const concurrent_copying_; 920 const bool disable_weak_ref_access_; 921}; 922 923void ConcurrentCopying::RevokeThreadLocalMarkStacks(bool disable_weak_ref_access) { 924 Thread* self = Thread::Current(); 925 RevokeThreadLocalMarkStackCheckpoint check_point(this, disable_weak_ref_access); 926 ThreadList* thread_list = Runtime::Current()->GetThreadList(); 927 gc_barrier_->Init(self, 0); 928 size_t barrier_count = thread_list->RunCheckpoint(&check_point); 929 // If there are no threads to wait which implys that all the checkpoint functions are finished, 930 // then no need to release the mutator lock. 931 if (barrier_count == 0) { 932 return; 933 } 934 Locks::mutator_lock_->SharedUnlock(self); 935 { 936 ScopedThreadStateChange tsc(self, kWaitingForCheckPointsToRun); 937 gc_barrier_->Increment(self, barrier_count); 938 } 939 Locks::mutator_lock_->SharedLock(self); 940} 941 942void ConcurrentCopying::RevokeThreadLocalMarkStack(Thread* thread) { 943 Thread* self = Thread::Current(); 944 CHECK_EQ(self, thread); 945 accounting::AtomicStack<mirror::Object>* tl_mark_stack = thread->GetThreadLocalMarkStack(); 946 if (tl_mark_stack != nullptr) { 947 CHECK(is_marking_); 948 MutexLock mu(self, mark_stack_lock_); 949 revoked_mark_stacks_.push_back(tl_mark_stack); 950 thread->SetThreadLocalMarkStack(nullptr); 951 } 952} 953 954void ConcurrentCopying::ProcessMarkStack() { 955 if (kVerboseMode) { 956 LOG(INFO) << "ProcessMarkStack. "; 957 } 958 bool empty_prev = false; 959 while (true) { 960 bool empty = ProcessMarkStackOnce(); 961 if (empty_prev && empty) { 962 // Saw empty mark stack for a second time, done. 963 break; 964 } 965 empty_prev = empty; 966 } 967} 968 969bool ConcurrentCopying::ProcessMarkStackOnce() { 970 Thread* self = Thread::Current(); 971 CHECK(thread_running_gc_ != nullptr); 972 CHECK(self == thread_running_gc_); 973 CHECK(self->GetThreadLocalMarkStack() == nullptr); 974 size_t count = 0; 975 MarkStackMode mark_stack_mode = mark_stack_mode_.LoadRelaxed(); 976 if (mark_stack_mode == kMarkStackModeThreadLocal) { 977 // Process the thread-local mark stacks and the GC mark stack. 978 count += ProcessThreadLocalMarkStacks(false); 979 while (!gc_mark_stack_->IsEmpty()) { 980 mirror::Object* to_ref = gc_mark_stack_->PopBack(); 981 ProcessMarkStackRef(to_ref); 982 ++count; 983 } 984 gc_mark_stack_->Reset(); 985 } else if (mark_stack_mode == kMarkStackModeShared) { 986 // Process the shared GC mark stack with a lock. 987 { 988 MutexLock mu(self, mark_stack_lock_); 989 CHECK(revoked_mark_stacks_.empty()); 990 } 991 while (true) { 992 std::vector<mirror::Object*> refs; 993 { 994 // Copy refs with lock. Note the number of refs should be small. 995 MutexLock mu(self, mark_stack_lock_); 996 if (gc_mark_stack_->IsEmpty()) { 997 break; 998 } 999 for (StackReference<mirror::Object>* p = gc_mark_stack_->Begin(); 1000 p != gc_mark_stack_->End(); ++p) { 1001 refs.push_back(p->AsMirrorPtr()); 1002 } 1003 gc_mark_stack_->Reset(); 1004 } 1005 for (mirror::Object* ref : refs) { 1006 ProcessMarkStackRef(ref); 1007 ++count; 1008 } 1009 } 1010 } else { 1011 CHECK_EQ(static_cast<uint32_t>(mark_stack_mode), 1012 static_cast<uint32_t>(kMarkStackModeGcExclusive)); 1013 { 1014 MutexLock mu(self, mark_stack_lock_); 1015 CHECK(revoked_mark_stacks_.empty()); 1016 } 1017 // Process the GC mark stack in the exclusive mode. No need to take the lock. 1018 while (!gc_mark_stack_->IsEmpty()) { 1019 mirror::Object* to_ref = gc_mark_stack_->PopBack(); 1020 ProcessMarkStackRef(to_ref); 1021 ++count; 1022 } 1023 gc_mark_stack_->Reset(); 1024 } 1025 1026 // Return true if the stack was empty. 1027 return count == 0; 1028} 1029 1030size_t ConcurrentCopying::ProcessThreadLocalMarkStacks(bool disable_weak_ref_access) { 1031 // Run a checkpoint to collect all thread local mark stacks and iterate over them all. 1032 RevokeThreadLocalMarkStacks(disable_weak_ref_access); 1033 size_t count = 0; 1034 std::vector<accounting::AtomicStack<mirror::Object>*> mark_stacks; 1035 { 1036 MutexLock mu(Thread::Current(), mark_stack_lock_); 1037 // Make a copy of the mark stack vector. 1038 mark_stacks = revoked_mark_stacks_; 1039 revoked_mark_stacks_.clear(); 1040 } 1041 for (accounting::AtomicStack<mirror::Object>* mark_stack : mark_stacks) { 1042 for (StackReference<mirror::Object>* p = mark_stack->Begin(); p != mark_stack->End(); ++p) { 1043 mirror::Object* to_ref = p->AsMirrorPtr(); 1044 ProcessMarkStackRef(to_ref); 1045 ++count; 1046 } 1047 { 1048 MutexLock mu(Thread::Current(), mark_stack_lock_); 1049 if (pooled_mark_stacks_.size() >= kMarkStackPoolSize) { 1050 // The pool has enough. Delete it. 1051 delete mark_stack; 1052 } else { 1053 // Otherwise, put it into the pool for later reuse. 1054 mark_stack->Reset(); 1055 pooled_mark_stacks_.push_back(mark_stack); 1056 } 1057 } 1058 } 1059 return count; 1060} 1061 1062inline void ConcurrentCopying::ProcessMarkStackRef(mirror::Object* to_ref) { 1063 DCHECK(!region_space_->IsInFromSpace(to_ref)); 1064 if (kUseBakerReadBarrier) { 1065 DCHECK(to_ref->GetReadBarrierPointer() == ReadBarrier::GrayPtr()) 1066 << " " << to_ref << " " << to_ref->GetReadBarrierPointer() 1067 << " is_marked=" << IsMarked(to_ref); 1068 } 1069 // Scan ref fields. 1070 Scan(to_ref); 1071 // Mark the gray ref as white or black. 1072 if (kUseBakerReadBarrier) { 1073 DCHECK(to_ref->GetReadBarrierPointer() == ReadBarrier::GrayPtr()) 1074 << " " << to_ref << " " << to_ref->GetReadBarrierPointer() 1075 << " is_marked=" << IsMarked(to_ref); 1076 } 1077#ifdef USE_BAKER_OR_BROOKS_READ_BARRIER 1078 if (UNLIKELY((to_ref->GetClass<kVerifyNone, kWithoutReadBarrier>()->IsTypeOfReferenceClass() && 1079 to_ref->AsReference()->GetReferent<kWithoutReadBarrier>() != nullptr && 1080 !IsInToSpace(to_ref->AsReference()->GetReferent<kWithoutReadBarrier>())))) { 1081 // Leave this Reference gray in the queue so that GetReferent() will trigger a read barrier. We 1082 // will change it to black or white later in ReferenceQueue::DequeuePendingReference(). 1083 DCHECK(to_ref->AsReference()->IsEnqueued()) << "Left unenqueued ref gray " << to_ref; 1084 } else { 1085 // We may occasionally leave a Reference black or white in the queue if its referent happens to 1086 // be concurrently marked after the Scan() call above has enqueued the Reference, in which case 1087 // the above IsInToSpace() evaluates to true and we change the color from gray to black or white 1088 // here in this else block. 1089 if (kUseBakerReadBarrier) { 1090 if (region_space_->IsInToSpace(to_ref)) { 1091 // If to-space, change from gray to white. 1092 bool success = to_ref->AtomicSetReadBarrierPointer</*kCasRelease*/true>( 1093 ReadBarrier::GrayPtr(), 1094 ReadBarrier::WhitePtr()); 1095 DCHECK(success) << "Must succeed as we won the race."; 1096 DCHECK(to_ref->GetReadBarrierPointer() == ReadBarrier::WhitePtr()); 1097 } else { 1098 // If non-moving space/unevac from space, change from gray 1099 // to black. We can't change gray to white because it's not 1100 // safe to use CAS if two threads change values in opposite 1101 // directions (A->B and B->A). So, we change it to black to 1102 // indicate non-moving objects that have been marked 1103 // through. Note we'd need to change from black to white 1104 // later (concurrently). 1105 bool success = to_ref->AtomicSetReadBarrierPointer</*kCasRelease*/true>( 1106 ReadBarrier::GrayPtr(), 1107 ReadBarrier::BlackPtr()); 1108 DCHECK(success) << "Must succeed as we won the race."; 1109 DCHECK(to_ref->GetReadBarrierPointer() == ReadBarrier::BlackPtr()); 1110 } 1111 } 1112 } 1113#else 1114 DCHECK(!kUseBakerReadBarrier); 1115#endif 1116 if (ReadBarrier::kEnableToSpaceInvariantChecks || kIsDebugBuild) { 1117 ConcurrentCopyingAssertToSpaceInvariantObjectVisitor visitor(this); 1118 visitor(to_ref); 1119 } 1120} 1121 1122void ConcurrentCopying::SwitchToSharedMarkStackMode() { 1123 Thread* self = Thread::Current(); 1124 CHECK(thread_running_gc_ != nullptr); 1125 CHECK_EQ(self, thread_running_gc_); 1126 CHECK(self->GetThreadLocalMarkStack() == nullptr); 1127 MarkStackMode before_mark_stack_mode = mark_stack_mode_.LoadRelaxed(); 1128 CHECK_EQ(static_cast<uint32_t>(before_mark_stack_mode), 1129 static_cast<uint32_t>(kMarkStackModeThreadLocal)); 1130 mark_stack_mode_.StoreRelaxed(kMarkStackModeShared); 1131 CHECK(weak_ref_access_enabled_.LoadRelaxed()); 1132 weak_ref_access_enabled_.StoreRelaxed(false); 1133 QuasiAtomic::ThreadFenceForConstructor(); 1134 // Process the thread local mark stacks one last time after switching to the shared mark stack 1135 // mode and disable weak ref accesses. 1136 ProcessThreadLocalMarkStacks(true); 1137 if (kVerboseMode) { 1138 LOG(INFO) << "Switched to shared mark stack mode and disabled weak ref access"; 1139 } 1140} 1141 1142void ConcurrentCopying::SwitchToGcExclusiveMarkStackMode() { 1143 Thread* self = Thread::Current(); 1144 CHECK(thread_running_gc_ != nullptr); 1145 CHECK_EQ(self, thread_running_gc_); 1146 CHECK(self->GetThreadLocalMarkStack() == nullptr); 1147 MarkStackMode before_mark_stack_mode = mark_stack_mode_.LoadRelaxed(); 1148 CHECK_EQ(static_cast<uint32_t>(before_mark_stack_mode), 1149 static_cast<uint32_t>(kMarkStackModeShared)); 1150 mark_stack_mode_.StoreRelaxed(kMarkStackModeGcExclusive); 1151 QuasiAtomic::ThreadFenceForConstructor(); 1152 if (kVerboseMode) { 1153 LOG(INFO) << "Switched to GC exclusive mark stack mode"; 1154 } 1155} 1156 1157void ConcurrentCopying::CheckEmptyMarkStack() { 1158 Thread* self = Thread::Current(); 1159 CHECK(thread_running_gc_ != nullptr); 1160 CHECK_EQ(self, thread_running_gc_); 1161 CHECK(self->GetThreadLocalMarkStack() == nullptr); 1162 MarkStackMode mark_stack_mode = mark_stack_mode_.LoadRelaxed(); 1163 if (mark_stack_mode == kMarkStackModeThreadLocal) { 1164 // Thread-local mark stack mode. 1165 RevokeThreadLocalMarkStacks(false); 1166 MutexLock mu(Thread::Current(), mark_stack_lock_); 1167 if (!revoked_mark_stacks_.empty()) { 1168 for (accounting::AtomicStack<mirror::Object>* mark_stack : revoked_mark_stacks_) { 1169 while (!mark_stack->IsEmpty()) { 1170 mirror::Object* obj = mark_stack->PopBack(); 1171 if (kUseBakerReadBarrier) { 1172 mirror::Object* rb_ptr = obj->GetReadBarrierPointer(); 1173 LOG(INFO) << "On mark queue : " << obj << " " << PrettyTypeOf(obj) << " rb_ptr=" << rb_ptr 1174 << " is_marked=" << IsMarked(obj); 1175 } else { 1176 LOG(INFO) << "On mark queue : " << obj << " " << PrettyTypeOf(obj) 1177 << " is_marked=" << IsMarked(obj); 1178 } 1179 } 1180 } 1181 LOG(FATAL) << "mark stack is not empty"; 1182 } 1183 } else { 1184 // Shared, GC-exclusive, or off. 1185 MutexLock mu(Thread::Current(), mark_stack_lock_); 1186 CHECK(gc_mark_stack_->IsEmpty()); 1187 CHECK(revoked_mark_stacks_.empty()); 1188 } 1189} 1190 1191void ConcurrentCopying::SweepSystemWeaks(Thread* self) { 1192 TimingLogger::ScopedTiming split("SweepSystemWeaks", GetTimings()); 1193 ReaderMutexLock mu(self, *Locks::heap_bitmap_lock_); 1194 Runtime::Current()->SweepSystemWeaks(this); 1195} 1196 1197void ConcurrentCopying::Sweep(bool swap_bitmaps) { 1198 { 1199 TimingLogger::ScopedTiming t("MarkStackAsLive", GetTimings()); 1200 accounting::ObjectStack* live_stack = heap_->GetLiveStack(); 1201 if (kEnableFromSpaceAccountingCheck) { 1202 CHECK_GE(live_stack_freeze_size_, live_stack->Size()); 1203 } 1204 heap_->MarkAllocStackAsLive(live_stack); 1205 live_stack->Reset(); 1206 } 1207 CheckEmptyMarkStack(); 1208 TimingLogger::ScopedTiming split("Sweep", GetTimings()); 1209 for (const auto& space : GetHeap()->GetContinuousSpaces()) { 1210 if (space->IsContinuousMemMapAllocSpace()) { 1211 space::ContinuousMemMapAllocSpace* alloc_space = space->AsContinuousMemMapAllocSpace(); 1212 if (space == region_space_ || immune_spaces_.ContainsSpace(space)) { 1213 continue; 1214 } 1215 TimingLogger::ScopedTiming split2( 1216 alloc_space->IsZygoteSpace() ? "SweepZygoteSpace" : "SweepAllocSpace", GetTimings()); 1217 RecordFree(alloc_space->Sweep(swap_bitmaps)); 1218 } 1219 } 1220 SweepLargeObjects(swap_bitmaps); 1221} 1222 1223void ConcurrentCopying::SweepLargeObjects(bool swap_bitmaps) { 1224 TimingLogger::ScopedTiming split("SweepLargeObjects", GetTimings()); 1225 RecordFreeLOS(heap_->GetLargeObjectsSpace()->Sweep(swap_bitmaps)); 1226} 1227 1228class ConcurrentCopyingClearBlackPtrsVisitor { 1229 public: 1230 explicit ConcurrentCopyingClearBlackPtrsVisitor(ConcurrentCopying* cc) 1231 : collector_(cc) {} 1232 void operator()(mirror::Object* obj) const SHARED_REQUIRES(Locks::mutator_lock_) 1233 SHARED_REQUIRES(Locks::heap_bitmap_lock_) { 1234 DCHECK(obj != nullptr); 1235 DCHECK(collector_->heap_->GetMarkBitmap()->Test(obj)) << obj; 1236 DCHECK_EQ(obj->GetReadBarrierPointer(), ReadBarrier::BlackPtr()) << obj; 1237 obj->AtomicSetReadBarrierPointer(ReadBarrier::BlackPtr(), ReadBarrier::WhitePtr()); 1238 DCHECK_EQ(obj->GetReadBarrierPointer(), ReadBarrier::WhitePtr()) << obj; 1239 } 1240 1241 private: 1242 ConcurrentCopying* const collector_; 1243}; 1244 1245// Clear the black ptrs in non-moving objects back to white. 1246void ConcurrentCopying::ClearBlackPtrs() { 1247 CHECK(kUseBakerReadBarrier); 1248 TimingLogger::ScopedTiming split("ClearBlackPtrs", GetTimings()); 1249 ConcurrentCopyingClearBlackPtrsVisitor visitor(this); 1250 for (auto& space : heap_->GetContinuousSpaces()) { 1251 if (space == region_space_) { 1252 continue; 1253 } 1254 accounting::ContinuousSpaceBitmap* mark_bitmap = space->GetMarkBitmap(); 1255 if (kVerboseMode) { 1256 LOG(INFO) << "ClearBlackPtrs: " << *space << " bitmap: " << *mark_bitmap; 1257 } 1258 mark_bitmap->VisitMarkedRange(reinterpret_cast<uintptr_t>(space->Begin()), 1259 reinterpret_cast<uintptr_t>(space->Limit()), 1260 visitor); 1261 } 1262 space::LargeObjectSpace* large_object_space = heap_->GetLargeObjectsSpace(); 1263 large_object_space->GetMarkBitmap()->VisitMarkedRange( 1264 reinterpret_cast<uintptr_t>(large_object_space->Begin()), 1265 reinterpret_cast<uintptr_t>(large_object_space->End()), 1266 visitor); 1267 // Objects on the allocation stack? 1268 if (ReadBarrier::kEnableReadBarrierInvariantChecks || kIsDebugBuild) { 1269 size_t count = GetAllocationStack()->Size(); 1270 auto* it = GetAllocationStack()->Begin(); 1271 auto* end = GetAllocationStack()->End(); 1272 for (size_t i = 0; i < count; ++i, ++it) { 1273 CHECK_LT(it, end); 1274 mirror::Object* obj = it->AsMirrorPtr(); 1275 if (obj != nullptr) { 1276 // Must have been cleared above. 1277 CHECK_EQ(obj->GetReadBarrierPointer(), ReadBarrier::WhitePtr()) << obj; 1278 } 1279 } 1280 } 1281} 1282 1283void ConcurrentCopying::ReclaimPhase() { 1284 TimingLogger::ScopedTiming split("ReclaimPhase", GetTimings()); 1285 if (kVerboseMode) { 1286 LOG(INFO) << "GC ReclaimPhase"; 1287 } 1288 Thread* self = Thread::Current(); 1289 1290 { 1291 // Double-check that the mark stack is empty. 1292 // Note: need to set this after VerifyNoFromSpaceRef(). 1293 is_asserting_to_space_invariant_ = false; 1294 QuasiAtomic::ThreadFenceForConstructor(); 1295 if (kVerboseMode) { 1296 LOG(INFO) << "Issue an empty check point. "; 1297 } 1298 IssueEmptyCheckpoint(); 1299 // Disable the check. 1300 is_mark_stack_push_disallowed_.StoreSequentiallyConsistent(0); 1301 CheckEmptyMarkStack(); 1302 } 1303 1304 { 1305 // Record freed objects. 1306 TimingLogger::ScopedTiming split2("RecordFree", GetTimings()); 1307 // Don't include thread-locals that are in the to-space. 1308 uint64_t from_bytes = region_space_->GetBytesAllocatedInFromSpace(); 1309 uint64_t from_objects = region_space_->GetObjectsAllocatedInFromSpace(); 1310 uint64_t unevac_from_bytes = region_space_->GetBytesAllocatedInUnevacFromSpace(); 1311 uint64_t unevac_from_objects = region_space_->GetObjectsAllocatedInUnevacFromSpace(); 1312 uint64_t to_bytes = bytes_moved_.LoadSequentiallyConsistent(); 1313 uint64_t to_objects = objects_moved_.LoadSequentiallyConsistent(); 1314 if (kEnableFromSpaceAccountingCheck) { 1315 CHECK_EQ(from_space_num_objects_at_first_pause_, from_objects + unevac_from_objects); 1316 CHECK_EQ(from_space_num_bytes_at_first_pause_, from_bytes + unevac_from_bytes); 1317 } 1318 CHECK_LE(to_objects, from_objects); 1319 CHECK_LE(to_bytes, from_bytes); 1320 int64_t freed_bytes = from_bytes - to_bytes; 1321 int64_t freed_objects = from_objects - to_objects; 1322 if (kVerboseMode) { 1323 LOG(INFO) << "RecordFree:" 1324 << " from_bytes=" << from_bytes << " from_objects=" << from_objects 1325 << " unevac_from_bytes=" << unevac_from_bytes << " unevac_from_objects=" << unevac_from_objects 1326 << " to_bytes=" << to_bytes << " to_objects=" << to_objects 1327 << " freed_bytes=" << freed_bytes << " freed_objects=" << freed_objects 1328 << " from_space size=" << region_space_->FromSpaceSize() 1329 << " unevac_from_space size=" << region_space_->UnevacFromSpaceSize() 1330 << " to_space size=" << region_space_->ToSpaceSize(); 1331 LOG(INFO) << "(before) num_bytes_allocated=" << heap_->num_bytes_allocated_.LoadSequentiallyConsistent(); 1332 } 1333 RecordFree(ObjectBytePair(freed_objects, freed_bytes)); 1334 if (kVerboseMode) { 1335 LOG(INFO) << "(after) num_bytes_allocated=" << heap_->num_bytes_allocated_.LoadSequentiallyConsistent(); 1336 } 1337 } 1338 1339 { 1340 TimingLogger::ScopedTiming split3("ComputeUnevacFromSpaceLiveRatio", GetTimings()); 1341 ComputeUnevacFromSpaceLiveRatio(); 1342 } 1343 1344 { 1345 TimingLogger::ScopedTiming split4("ClearFromSpace", GetTimings()); 1346 region_space_->ClearFromSpace(); 1347 } 1348 1349 { 1350 WriterMutexLock mu(self, *Locks::heap_bitmap_lock_); 1351 if (kUseBakerReadBarrier) { 1352 ClearBlackPtrs(); 1353 } 1354 Sweep(false); 1355 SwapBitmaps(); 1356 heap_->UnBindBitmaps(); 1357 1358 // Remove bitmaps for the immune spaces. 1359 while (!cc_bitmaps_.empty()) { 1360 accounting::ContinuousSpaceBitmap* cc_bitmap = cc_bitmaps_.back(); 1361 cc_heap_bitmap_->RemoveContinuousSpaceBitmap(cc_bitmap); 1362 delete cc_bitmap; 1363 cc_bitmaps_.pop_back(); 1364 } 1365 region_space_bitmap_ = nullptr; 1366 } 1367 1368 CheckEmptyMarkStack(); 1369 1370 if (kVerboseMode) { 1371 LOG(INFO) << "GC end of ReclaimPhase"; 1372 } 1373} 1374 1375class ConcurrentCopyingComputeUnevacFromSpaceLiveRatioVisitor { 1376 public: 1377 explicit ConcurrentCopyingComputeUnevacFromSpaceLiveRatioVisitor(ConcurrentCopying* cc) 1378 : collector_(cc) {} 1379 void operator()(mirror::Object* ref) const SHARED_REQUIRES(Locks::mutator_lock_) 1380 SHARED_REQUIRES(Locks::heap_bitmap_lock_) { 1381 DCHECK(ref != nullptr); 1382 DCHECK(collector_->region_space_bitmap_->Test(ref)) << ref; 1383 DCHECK(collector_->region_space_->IsInUnevacFromSpace(ref)) << ref; 1384 if (kUseBakerReadBarrier) { 1385 DCHECK_EQ(ref->GetReadBarrierPointer(), ReadBarrier::BlackPtr()) << ref; 1386 // Clear the black ptr. 1387 ref->AtomicSetReadBarrierPointer(ReadBarrier::BlackPtr(), ReadBarrier::WhitePtr()); 1388 DCHECK_EQ(ref->GetReadBarrierPointer(), ReadBarrier::WhitePtr()) << ref; 1389 } 1390 size_t obj_size = ref->SizeOf(); 1391 size_t alloc_size = RoundUp(obj_size, space::RegionSpace::kAlignment); 1392 collector_->region_space_->AddLiveBytes(ref, alloc_size); 1393 } 1394 1395 private: 1396 ConcurrentCopying* const collector_; 1397}; 1398 1399// Compute how much live objects are left in regions. 1400void ConcurrentCopying::ComputeUnevacFromSpaceLiveRatio() { 1401 region_space_->AssertAllRegionLiveBytesZeroOrCleared(); 1402 ConcurrentCopyingComputeUnevacFromSpaceLiveRatioVisitor visitor(this); 1403 region_space_bitmap_->VisitMarkedRange(reinterpret_cast<uintptr_t>(region_space_->Begin()), 1404 reinterpret_cast<uintptr_t>(region_space_->Limit()), 1405 visitor); 1406} 1407 1408// Assert the to-space invariant. 1409void ConcurrentCopying::AssertToSpaceInvariant(mirror::Object* obj, MemberOffset offset, 1410 mirror::Object* ref) { 1411 CHECK(heap_->collector_type_ == kCollectorTypeCC) << static_cast<size_t>(heap_->collector_type_); 1412 if (is_asserting_to_space_invariant_) { 1413 if (region_space_->IsInToSpace(ref)) { 1414 // OK. 1415 return; 1416 } else if (region_space_->IsInUnevacFromSpace(ref)) { 1417 CHECK(region_space_bitmap_->Test(ref)) << ref; 1418 } else if (region_space_->IsInFromSpace(ref)) { 1419 // Not OK. Do extra logging. 1420 if (obj != nullptr) { 1421 LogFromSpaceRefHolder(obj, offset); 1422 } 1423 ref->GetLockWord(false).Dump(LOG(INTERNAL_FATAL)); 1424 CHECK(false) << "Found from-space ref " << ref << " " << PrettyTypeOf(ref); 1425 } else { 1426 AssertToSpaceInvariantInNonMovingSpace(obj, ref); 1427 } 1428 } 1429} 1430 1431class RootPrinter { 1432 public: 1433 RootPrinter() { } 1434 1435 template <class MirrorType> 1436 ALWAYS_INLINE void VisitRootIfNonNull(mirror::CompressedReference<MirrorType>* root) 1437 SHARED_REQUIRES(Locks::mutator_lock_) { 1438 if (!root->IsNull()) { 1439 VisitRoot(root); 1440 } 1441 } 1442 1443 template <class MirrorType> 1444 void VisitRoot(mirror::Object** root) 1445 SHARED_REQUIRES(Locks::mutator_lock_) { 1446 LOG(INTERNAL_FATAL) << "root=" << root << " ref=" << *root; 1447 } 1448 1449 template <class MirrorType> 1450 void VisitRoot(mirror::CompressedReference<MirrorType>* root) 1451 SHARED_REQUIRES(Locks::mutator_lock_) { 1452 LOG(INTERNAL_FATAL) << "root=" << root << " ref=" << root->AsMirrorPtr(); 1453 } 1454}; 1455 1456void ConcurrentCopying::AssertToSpaceInvariant(GcRootSource* gc_root_source, 1457 mirror::Object* ref) { 1458 CHECK(heap_->collector_type_ == kCollectorTypeCC) << static_cast<size_t>(heap_->collector_type_); 1459 if (is_asserting_to_space_invariant_) { 1460 if (region_space_->IsInToSpace(ref)) { 1461 // OK. 1462 return; 1463 } else if (region_space_->IsInUnevacFromSpace(ref)) { 1464 CHECK(region_space_bitmap_->Test(ref)) << ref; 1465 } else if (region_space_->IsInFromSpace(ref)) { 1466 // Not OK. Do extra logging. 1467 if (gc_root_source == nullptr) { 1468 // No info. 1469 } else if (gc_root_source->HasArtField()) { 1470 ArtField* field = gc_root_source->GetArtField(); 1471 LOG(INTERNAL_FATAL) << "gc root in field " << field << " " << PrettyField(field); 1472 RootPrinter root_printer; 1473 field->VisitRoots(root_printer); 1474 } else if (gc_root_source->HasArtMethod()) { 1475 ArtMethod* method = gc_root_source->GetArtMethod(); 1476 LOG(INTERNAL_FATAL) << "gc root in method " << method << " " << PrettyMethod(method); 1477 RootPrinter root_printer; 1478 method->VisitRoots(root_printer, sizeof(void*)); 1479 } 1480 ref->GetLockWord(false).Dump(LOG(INTERNAL_FATAL)); 1481 region_space_->DumpNonFreeRegions(LOG(INTERNAL_FATAL)); 1482 PrintFileToLog("/proc/self/maps", LogSeverity::INTERNAL_FATAL); 1483 MemMap::DumpMaps(LOG(INTERNAL_FATAL), true); 1484 CHECK(false) << "Found from-space ref " << ref << " " << PrettyTypeOf(ref); 1485 } else { 1486 AssertToSpaceInvariantInNonMovingSpace(nullptr, ref); 1487 } 1488 } 1489} 1490 1491void ConcurrentCopying::LogFromSpaceRefHolder(mirror::Object* obj, MemberOffset offset) { 1492 if (kUseBakerReadBarrier) { 1493 LOG(INFO) << "holder=" << obj << " " << PrettyTypeOf(obj) 1494 << " holder rb_ptr=" << obj->GetReadBarrierPointer(); 1495 } else { 1496 LOG(INFO) << "holder=" << obj << " " << PrettyTypeOf(obj); 1497 } 1498 if (region_space_->IsInFromSpace(obj)) { 1499 LOG(INFO) << "holder is in the from-space."; 1500 } else if (region_space_->IsInToSpace(obj)) { 1501 LOG(INFO) << "holder is in the to-space."; 1502 } else if (region_space_->IsInUnevacFromSpace(obj)) { 1503 LOG(INFO) << "holder is in the unevac from-space."; 1504 if (region_space_bitmap_->Test(obj)) { 1505 LOG(INFO) << "holder is marked in the region space bitmap."; 1506 } else { 1507 LOG(INFO) << "holder is not marked in the region space bitmap."; 1508 } 1509 } else { 1510 // In a non-moving space. 1511 if (immune_spaces_.ContainsObject(obj)) { 1512 LOG(INFO) << "holder is in an immune image or the zygote space."; 1513 accounting::ContinuousSpaceBitmap* cc_bitmap = 1514 cc_heap_bitmap_->GetContinuousSpaceBitmap(obj); 1515 CHECK(cc_bitmap != nullptr) 1516 << "An immune space object must have a bitmap."; 1517 if (cc_bitmap->Test(obj)) { 1518 LOG(INFO) << "holder is marked in the bit map."; 1519 } else { 1520 LOG(INFO) << "holder is NOT marked in the bit map."; 1521 } 1522 } else { 1523 LOG(INFO) << "holder is in a non-immune, non-moving (or main) space."; 1524 accounting::ContinuousSpaceBitmap* mark_bitmap = 1525 heap_mark_bitmap_->GetContinuousSpaceBitmap(obj); 1526 accounting::LargeObjectBitmap* los_bitmap = 1527 heap_mark_bitmap_->GetLargeObjectBitmap(obj); 1528 CHECK(los_bitmap != nullptr) << "LOS bitmap covers the entire address range"; 1529 bool is_los = mark_bitmap == nullptr; 1530 if (!is_los && mark_bitmap->Test(obj)) { 1531 LOG(INFO) << "holder is marked in the mark bit map."; 1532 } else if (is_los && los_bitmap->Test(obj)) { 1533 LOG(INFO) << "holder is marked in the los bit map."; 1534 } else { 1535 // If ref is on the allocation stack, then it is considered 1536 // mark/alive (but not necessarily on the live stack.) 1537 if (IsOnAllocStack(obj)) { 1538 LOG(INFO) << "holder is on the alloc stack."; 1539 } else { 1540 LOG(INFO) << "holder is not marked or on the alloc stack."; 1541 } 1542 } 1543 } 1544 } 1545 LOG(INFO) << "offset=" << offset.SizeValue(); 1546} 1547 1548void ConcurrentCopying::AssertToSpaceInvariantInNonMovingSpace(mirror::Object* obj, 1549 mirror::Object* ref) { 1550 // In a non-moving spaces. Check that the ref is marked. 1551 if (immune_spaces_.ContainsObject(ref)) { 1552 accounting::ContinuousSpaceBitmap* cc_bitmap = 1553 cc_heap_bitmap_->GetContinuousSpaceBitmap(ref); 1554 CHECK(cc_bitmap != nullptr) 1555 << "An immune space ref must have a bitmap. " << ref; 1556 if (kUseBakerReadBarrier) { 1557 CHECK(cc_bitmap->Test(ref)) 1558 << "Unmarked immune space ref. obj=" << obj << " rb_ptr=" 1559 << obj->GetReadBarrierPointer() << " ref=" << ref; 1560 } else { 1561 CHECK(cc_bitmap->Test(ref)) 1562 << "Unmarked immune space ref. obj=" << obj << " ref=" << ref; 1563 } 1564 } else { 1565 accounting::ContinuousSpaceBitmap* mark_bitmap = 1566 heap_mark_bitmap_->GetContinuousSpaceBitmap(ref); 1567 accounting::LargeObjectBitmap* los_bitmap = 1568 heap_mark_bitmap_->GetLargeObjectBitmap(ref); 1569 CHECK(los_bitmap != nullptr) << "LOS bitmap covers the entire address range"; 1570 bool is_los = mark_bitmap == nullptr; 1571 if ((!is_los && mark_bitmap->Test(ref)) || 1572 (is_los && los_bitmap->Test(ref))) { 1573 // OK. 1574 } else { 1575 // If ref is on the allocation stack, then it may not be 1576 // marked live, but considered marked/alive (but not 1577 // necessarily on the live stack). 1578 CHECK(IsOnAllocStack(ref)) << "Unmarked ref that's not on the allocation stack. " 1579 << "obj=" << obj << " ref=" << ref; 1580 } 1581 } 1582} 1583 1584// Used to scan ref fields of an object. 1585class ConcurrentCopyingRefFieldsVisitor { 1586 public: 1587 explicit ConcurrentCopyingRefFieldsVisitor(ConcurrentCopying* collector) 1588 : collector_(collector) {} 1589 1590 void operator()(mirror::Object* obj, MemberOffset offset, bool /* is_static */) 1591 const ALWAYS_INLINE SHARED_REQUIRES(Locks::mutator_lock_) 1592 SHARED_REQUIRES(Locks::heap_bitmap_lock_) { 1593 collector_->Process(obj, offset); 1594 } 1595 1596 void operator()(mirror::Class* klass, mirror::Reference* ref) const 1597 SHARED_REQUIRES(Locks::mutator_lock_) ALWAYS_INLINE { 1598 CHECK(klass->IsTypeOfReferenceClass()); 1599 collector_->DelayReferenceReferent(klass, ref); 1600 } 1601 1602 void VisitRootIfNonNull(mirror::CompressedReference<mirror::Object>* root) const 1603 ALWAYS_INLINE 1604 SHARED_REQUIRES(Locks::mutator_lock_) { 1605 if (!root->IsNull()) { 1606 VisitRoot(root); 1607 } 1608 } 1609 1610 void VisitRoot(mirror::CompressedReference<mirror::Object>* root) const 1611 ALWAYS_INLINE 1612 SHARED_REQUIRES(Locks::mutator_lock_) { 1613 collector_->MarkRoot(root); 1614 } 1615 1616 private: 1617 ConcurrentCopying* const collector_; 1618}; 1619 1620// Scan ref fields of an object. 1621inline void ConcurrentCopying::Scan(mirror::Object* to_ref) { 1622 DCHECK(!region_space_->IsInFromSpace(to_ref)); 1623 ConcurrentCopyingRefFieldsVisitor visitor(this); 1624 to_ref->VisitReferences(visitor, visitor); 1625} 1626 1627// Process a field. 1628inline void ConcurrentCopying::Process(mirror::Object* obj, MemberOffset offset) { 1629 mirror::Object* ref = obj->GetFieldObject< 1630 mirror::Object, kVerifyNone, kWithoutReadBarrier, false>(offset); 1631 mirror::Object* to_ref = Mark(ref); 1632 if (to_ref == ref) { 1633 return; 1634 } 1635 // This may fail if the mutator writes to the field at the same time. But it's ok. 1636 mirror::Object* expected_ref = ref; 1637 mirror::Object* new_ref = to_ref; 1638 do { 1639 if (expected_ref != 1640 obj->GetFieldObject<mirror::Object, kVerifyNone, kWithoutReadBarrier, false>(offset)) { 1641 // It was updated by the mutator. 1642 break; 1643 } 1644 } while (!obj->CasFieldWeakRelaxedObjectWithoutWriteBarrier< 1645 false, false, kVerifyNone>(offset, expected_ref, new_ref)); 1646} 1647 1648// Process some roots. 1649inline void ConcurrentCopying::VisitRoots( 1650 mirror::Object*** roots, size_t count, const RootInfo& info ATTRIBUTE_UNUSED) { 1651 for (size_t i = 0; i < count; ++i) { 1652 mirror::Object** root = roots[i]; 1653 mirror::Object* ref = *root; 1654 mirror::Object* to_ref = Mark(ref); 1655 if (to_ref == ref) { 1656 continue; 1657 } 1658 Atomic<mirror::Object*>* addr = reinterpret_cast<Atomic<mirror::Object*>*>(root); 1659 mirror::Object* expected_ref = ref; 1660 mirror::Object* new_ref = to_ref; 1661 do { 1662 if (expected_ref != addr->LoadRelaxed()) { 1663 // It was updated by the mutator. 1664 break; 1665 } 1666 } while (!addr->CompareExchangeWeakRelaxed(expected_ref, new_ref)); 1667 } 1668} 1669 1670inline void ConcurrentCopying::MarkRoot(mirror::CompressedReference<mirror::Object>* root) { 1671 DCHECK(!root->IsNull()); 1672 mirror::Object* const ref = root->AsMirrorPtr(); 1673 mirror::Object* to_ref = Mark(ref); 1674 if (to_ref != ref) { 1675 auto* addr = reinterpret_cast<Atomic<mirror::CompressedReference<mirror::Object>>*>(root); 1676 auto expected_ref = mirror::CompressedReference<mirror::Object>::FromMirrorPtr(ref); 1677 auto new_ref = mirror::CompressedReference<mirror::Object>::FromMirrorPtr(to_ref); 1678 // If the cas fails, then it was updated by the mutator. 1679 do { 1680 if (ref != addr->LoadRelaxed().AsMirrorPtr()) { 1681 // It was updated by the mutator. 1682 break; 1683 } 1684 } while (!addr->CompareExchangeWeakRelaxed(expected_ref, new_ref)); 1685 } 1686} 1687 1688inline void ConcurrentCopying::VisitRoots( 1689 mirror::CompressedReference<mirror::Object>** roots, size_t count, 1690 const RootInfo& info ATTRIBUTE_UNUSED) { 1691 for (size_t i = 0; i < count; ++i) { 1692 mirror::CompressedReference<mirror::Object>* const root = roots[i]; 1693 if (!root->IsNull()) { 1694 MarkRoot(root); 1695 } 1696 } 1697} 1698 1699// Fill the given memory block with a dummy object. Used to fill in a 1700// copy of objects that was lost in race. 1701void ConcurrentCopying::FillWithDummyObject(mirror::Object* dummy_obj, size_t byte_size) { 1702 CHECK_ALIGNED(byte_size, kObjectAlignment); 1703 memset(dummy_obj, 0, byte_size); 1704 mirror::Class* int_array_class = mirror::IntArray::GetArrayClass(); 1705 CHECK(int_array_class != nullptr); 1706 AssertToSpaceInvariant(nullptr, MemberOffset(0), int_array_class); 1707 size_t component_size = int_array_class->GetComponentSize(); 1708 CHECK_EQ(component_size, sizeof(int32_t)); 1709 size_t data_offset = mirror::Array::DataOffset(component_size).SizeValue(); 1710 if (data_offset > byte_size) { 1711 // An int array is too big. Use java.lang.Object. 1712 mirror::Class* java_lang_Object = WellKnownClasses::ToClass(WellKnownClasses::java_lang_Object); 1713 AssertToSpaceInvariant(nullptr, MemberOffset(0), java_lang_Object); 1714 CHECK_EQ(byte_size, java_lang_Object->GetObjectSize()); 1715 dummy_obj->SetClass(java_lang_Object); 1716 CHECK_EQ(byte_size, dummy_obj->SizeOf()); 1717 } else { 1718 // Use an int array. 1719 dummy_obj->SetClass(int_array_class); 1720 CHECK(dummy_obj->IsArrayInstance()); 1721 int32_t length = (byte_size - data_offset) / component_size; 1722 dummy_obj->AsArray()->SetLength(length); 1723 CHECK_EQ(dummy_obj->AsArray()->GetLength(), length) 1724 << "byte_size=" << byte_size << " length=" << length 1725 << " component_size=" << component_size << " data_offset=" << data_offset; 1726 CHECK_EQ(byte_size, dummy_obj->SizeOf()) 1727 << "byte_size=" << byte_size << " length=" << length 1728 << " component_size=" << component_size << " data_offset=" << data_offset; 1729 } 1730} 1731 1732// Reuse the memory blocks that were copy of objects that were lost in race. 1733mirror::Object* ConcurrentCopying::AllocateInSkippedBlock(size_t alloc_size) { 1734 // Try to reuse the blocks that were unused due to CAS failures. 1735 CHECK_ALIGNED(alloc_size, space::RegionSpace::kAlignment); 1736 Thread* self = Thread::Current(); 1737 size_t min_object_size = RoundUp(sizeof(mirror::Object), space::RegionSpace::kAlignment); 1738 MutexLock mu(self, skipped_blocks_lock_); 1739 auto it = skipped_blocks_map_.lower_bound(alloc_size); 1740 if (it == skipped_blocks_map_.end()) { 1741 // Not found. 1742 return nullptr; 1743 } 1744 { 1745 size_t byte_size = it->first; 1746 CHECK_GE(byte_size, alloc_size); 1747 if (byte_size > alloc_size && byte_size - alloc_size < min_object_size) { 1748 // If remainder would be too small for a dummy object, retry with a larger request size. 1749 it = skipped_blocks_map_.lower_bound(alloc_size + min_object_size); 1750 if (it == skipped_blocks_map_.end()) { 1751 // Not found. 1752 return nullptr; 1753 } 1754 CHECK_ALIGNED(it->first - alloc_size, space::RegionSpace::kAlignment); 1755 CHECK_GE(it->first - alloc_size, min_object_size) 1756 << "byte_size=" << byte_size << " it->first=" << it->first << " alloc_size=" << alloc_size; 1757 } 1758 } 1759 // Found a block. 1760 CHECK(it != skipped_blocks_map_.end()); 1761 size_t byte_size = it->first; 1762 uint8_t* addr = it->second; 1763 CHECK_GE(byte_size, alloc_size); 1764 CHECK(region_space_->IsInToSpace(reinterpret_cast<mirror::Object*>(addr))); 1765 CHECK_ALIGNED(byte_size, space::RegionSpace::kAlignment); 1766 if (kVerboseMode) { 1767 LOG(INFO) << "Reusing skipped bytes : " << reinterpret_cast<void*>(addr) << ", " << byte_size; 1768 } 1769 skipped_blocks_map_.erase(it); 1770 memset(addr, 0, byte_size); 1771 if (byte_size > alloc_size) { 1772 // Return the remainder to the map. 1773 CHECK_ALIGNED(byte_size - alloc_size, space::RegionSpace::kAlignment); 1774 CHECK_GE(byte_size - alloc_size, min_object_size); 1775 FillWithDummyObject(reinterpret_cast<mirror::Object*>(addr + alloc_size), 1776 byte_size - alloc_size); 1777 CHECK(region_space_->IsInToSpace(reinterpret_cast<mirror::Object*>(addr + alloc_size))); 1778 skipped_blocks_map_.insert(std::make_pair(byte_size - alloc_size, addr + alloc_size)); 1779 } 1780 return reinterpret_cast<mirror::Object*>(addr); 1781} 1782 1783mirror::Object* ConcurrentCopying::Copy(mirror::Object* from_ref) { 1784 DCHECK(region_space_->IsInFromSpace(from_ref)); 1785 // No read barrier to avoid nested RB that might violate the to-space 1786 // invariant. Note that from_ref is a from space ref so the SizeOf() 1787 // call will access the from-space meta objects, but it's ok and necessary. 1788 size_t obj_size = from_ref->SizeOf<kDefaultVerifyFlags, kWithoutReadBarrier>(); 1789 size_t region_space_alloc_size = RoundUp(obj_size, space::RegionSpace::kAlignment); 1790 size_t region_space_bytes_allocated = 0U; 1791 size_t non_moving_space_bytes_allocated = 0U; 1792 size_t bytes_allocated = 0U; 1793 size_t dummy; 1794 mirror::Object* to_ref = region_space_->AllocNonvirtual<true>( 1795 region_space_alloc_size, ®ion_space_bytes_allocated, nullptr, &dummy); 1796 bytes_allocated = region_space_bytes_allocated; 1797 if (to_ref != nullptr) { 1798 DCHECK_EQ(region_space_alloc_size, region_space_bytes_allocated); 1799 } 1800 bool fall_back_to_non_moving = false; 1801 if (UNLIKELY(to_ref == nullptr)) { 1802 // Failed to allocate in the region space. Try the skipped blocks. 1803 to_ref = AllocateInSkippedBlock(region_space_alloc_size); 1804 if (to_ref != nullptr) { 1805 // Succeeded to allocate in a skipped block. 1806 if (heap_->use_tlab_) { 1807 // This is necessary for the tlab case as it's not accounted in the space. 1808 region_space_->RecordAlloc(to_ref); 1809 } 1810 bytes_allocated = region_space_alloc_size; 1811 } else { 1812 // Fall back to the non-moving space. 1813 fall_back_to_non_moving = true; 1814 if (kVerboseMode) { 1815 LOG(INFO) << "Out of memory in the to-space. Fall back to non-moving. skipped_bytes=" 1816 << to_space_bytes_skipped_.LoadSequentiallyConsistent() 1817 << " skipped_objects=" << to_space_objects_skipped_.LoadSequentiallyConsistent(); 1818 } 1819 fall_back_to_non_moving = true; 1820 to_ref = heap_->non_moving_space_->Alloc(Thread::Current(), obj_size, 1821 &non_moving_space_bytes_allocated, nullptr, &dummy); 1822 CHECK(to_ref != nullptr) << "Fall-back non-moving space allocation failed"; 1823 bytes_allocated = non_moving_space_bytes_allocated; 1824 // Mark it in the mark bitmap. 1825 accounting::ContinuousSpaceBitmap* mark_bitmap = 1826 heap_mark_bitmap_->GetContinuousSpaceBitmap(to_ref); 1827 CHECK(mark_bitmap != nullptr); 1828 CHECK(!mark_bitmap->AtomicTestAndSet(to_ref)); 1829 } 1830 } 1831 DCHECK(to_ref != nullptr); 1832 1833 // Attempt to install the forward pointer. This is in a loop as the 1834 // lock word atomic write can fail. 1835 while (true) { 1836 // Copy the object. TODO: copy only the lockword in the second iteration and on? 1837 memcpy(to_ref, from_ref, obj_size); 1838 1839 LockWord old_lock_word = to_ref->GetLockWord(false); 1840 1841 if (old_lock_word.GetState() == LockWord::kForwardingAddress) { 1842 // Lost the race. Another thread (either GC or mutator) stored 1843 // the forwarding pointer first. Make the lost copy (to_ref) 1844 // look like a valid but dead (dummy) object and keep it for 1845 // future reuse. 1846 FillWithDummyObject(to_ref, bytes_allocated); 1847 if (!fall_back_to_non_moving) { 1848 DCHECK(region_space_->IsInToSpace(to_ref)); 1849 if (bytes_allocated > space::RegionSpace::kRegionSize) { 1850 // Free the large alloc. 1851 region_space_->FreeLarge(to_ref, bytes_allocated); 1852 } else { 1853 // Record the lost copy for later reuse. 1854 heap_->num_bytes_allocated_.FetchAndAddSequentiallyConsistent(bytes_allocated); 1855 to_space_bytes_skipped_.FetchAndAddSequentiallyConsistent(bytes_allocated); 1856 to_space_objects_skipped_.FetchAndAddSequentiallyConsistent(1); 1857 MutexLock mu(Thread::Current(), skipped_blocks_lock_); 1858 skipped_blocks_map_.insert(std::make_pair(bytes_allocated, 1859 reinterpret_cast<uint8_t*>(to_ref))); 1860 } 1861 } else { 1862 DCHECK(heap_->non_moving_space_->HasAddress(to_ref)); 1863 DCHECK_EQ(bytes_allocated, non_moving_space_bytes_allocated); 1864 // Free the non-moving-space chunk. 1865 accounting::ContinuousSpaceBitmap* mark_bitmap = 1866 heap_mark_bitmap_->GetContinuousSpaceBitmap(to_ref); 1867 CHECK(mark_bitmap != nullptr); 1868 CHECK(mark_bitmap->Clear(to_ref)); 1869 heap_->non_moving_space_->Free(Thread::Current(), to_ref); 1870 } 1871 1872 // Get the winner's forward ptr. 1873 mirror::Object* lost_fwd_ptr = to_ref; 1874 to_ref = reinterpret_cast<mirror::Object*>(old_lock_word.ForwardingAddress()); 1875 CHECK(to_ref != nullptr); 1876 CHECK_NE(to_ref, lost_fwd_ptr); 1877 CHECK(region_space_->IsInToSpace(to_ref) || heap_->non_moving_space_->HasAddress(to_ref)); 1878 CHECK_NE(to_ref->GetLockWord(false).GetState(), LockWord::kForwardingAddress); 1879 return to_ref; 1880 } 1881 1882 // Set the gray ptr. 1883 if (kUseBakerReadBarrier) { 1884 to_ref->SetReadBarrierPointer(ReadBarrier::GrayPtr()); 1885 } 1886 1887 LockWord new_lock_word = LockWord::FromForwardingAddress(reinterpret_cast<size_t>(to_ref)); 1888 1889 // Try to atomically write the fwd ptr. 1890 bool success = from_ref->CasLockWordWeakSequentiallyConsistent(old_lock_word, new_lock_word); 1891 if (LIKELY(success)) { 1892 // The CAS succeeded. 1893 objects_moved_.FetchAndAddSequentiallyConsistent(1); 1894 bytes_moved_.FetchAndAddSequentiallyConsistent(region_space_alloc_size); 1895 if (LIKELY(!fall_back_to_non_moving)) { 1896 DCHECK(region_space_->IsInToSpace(to_ref)); 1897 } else { 1898 DCHECK(heap_->non_moving_space_->HasAddress(to_ref)); 1899 DCHECK_EQ(bytes_allocated, non_moving_space_bytes_allocated); 1900 } 1901 if (kUseBakerReadBarrier) { 1902 DCHECK(to_ref->GetReadBarrierPointer() == ReadBarrier::GrayPtr()); 1903 } 1904 DCHECK(GetFwdPtr(from_ref) == to_ref); 1905 CHECK_NE(to_ref->GetLockWord(false).GetState(), LockWord::kForwardingAddress); 1906 PushOntoMarkStack(to_ref); 1907 return to_ref; 1908 } else { 1909 // The CAS failed. It may have lost the race or may have failed 1910 // due to monitor/hashcode ops. Either way, retry. 1911 } 1912 } 1913} 1914 1915mirror::Object* ConcurrentCopying::IsMarked(mirror::Object* from_ref) { 1916 DCHECK(from_ref != nullptr); 1917 space::RegionSpace::RegionType rtype = region_space_->GetRegionType(from_ref); 1918 if (rtype == space::RegionSpace::RegionType::kRegionTypeToSpace) { 1919 // It's already marked. 1920 return from_ref; 1921 } 1922 mirror::Object* to_ref; 1923 if (rtype == space::RegionSpace::RegionType::kRegionTypeFromSpace) { 1924 to_ref = GetFwdPtr(from_ref); 1925 DCHECK(to_ref == nullptr || region_space_->IsInToSpace(to_ref) || 1926 heap_->non_moving_space_->HasAddress(to_ref)) 1927 << "from_ref=" << from_ref << " to_ref=" << to_ref; 1928 } else if (rtype == space::RegionSpace::RegionType::kRegionTypeUnevacFromSpace) { 1929 if (region_space_bitmap_->Test(from_ref)) { 1930 to_ref = from_ref; 1931 } else { 1932 to_ref = nullptr; 1933 } 1934 } else { 1935 // from_ref is in a non-moving space. 1936 if (immune_spaces_.ContainsObject(from_ref)) { 1937 accounting::ContinuousSpaceBitmap* cc_bitmap = 1938 cc_heap_bitmap_->GetContinuousSpaceBitmap(from_ref); 1939 DCHECK(cc_bitmap != nullptr) 1940 << "An immune space object must have a bitmap"; 1941 if (kIsDebugBuild) { 1942 DCHECK(heap_mark_bitmap_->GetContinuousSpaceBitmap(from_ref)->Test(from_ref)) 1943 << "Immune space object must be already marked"; 1944 } 1945 if (cc_bitmap->Test(from_ref)) { 1946 // Already marked. 1947 to_ref = from_ref; 1948 } else { 1949 // Newly marked. 1950 to_ref = nullptr; 1951 } 1952 } else { 1953 // Non-immune non-moving space. Use the mark bitmap. 1954 accounting::ContinuousSpaceBitmap* mark_bitmap = 1955 heap_mark_bitmap_->GetContinuousSpaceBitmap(from_ref); 1956 accounting::LargeObjectBitmap* los_bitmap = 1957 heap_mark_bitmap_->GetLargeObjectBitmap(from_ref); 1958 CHECK(los_bitmap != nullptr) << "LOS bitmap covers the entire address range"; 1959 bool is_los = mark_bitmap == nullptr; 1960 if (!is_los && mark_bitmap->Test(from_ref)) { 1961 // Already marked. 1962 to_ref = from_ref; 1963 } else if (is_los && los_bitmap->Test(from_ref)) { 1964 // Already marked in LOS. 1965 to_ref = from_ref; 1966 } else { 1967 // Not marked. 1968 if (IsOnAllocStack(from_ref)) { 1969 // If on the allocation stack, it's considered marked. 1970 to_ref = from_ref; 1971 } else { 1972 // Not marked. 1973 to_ref = nullptr; 1974 } 1975 } 1976 } 1977 } 1978 return to_ref; 1979} 1980 1981bool ConcurrentCopying::IsOnAllocStack(mirror::Object* ref) { 1982 QuasiAtomic::ThreadFenceAcquire(); 1983 accounting::ObjectStack* alloc_stack = GetAllocationStack(); 1984 return alloc_stack->Contains(ref); 1985} 1986 1987mirror::Object* ConcurrentCopying::MarkNonMoving(mirror::Object* ref) { 1988 // ref is in a non-moving space (from_ref == to_ref). 1989 DCHECK(!region_space_->HasAddress(ref)) << ref; 1990 if (immune_spaces_.ContainsObject(ref)) { 1991 accounting::ContinuousSpaceBitmap* cc_bitmap = 1992 cc_heap_bitmap_->GetContinuousSpaceBitmap(ref); 1993 DCHECK(cc_bitmap != nullptr) 1994 << "An immune space object must have a bitmap"; 1995 if (kIsDebugBuild) { 1996 DCHECK(heap_mark_bitmap_->GetContinuousSpaceBitmap(ref)->Test(ref)) 1997 << "Immune space object must be already marked"; 1998 } 1999 // This may or may not succeed, which is ok. 2000 if (kUseBakerReadBarrier) { 2001 ref->AtomicSetReadBarrierPointer(ReadBarrier::WhitePtr(), ReadBarrier::GrayPtr()); 2002 } 2003 if (cc_bitmap->AtomicTestAndSet(ref)) { 2004 // Already marked. 2005 } else { 2006 // Newly marked. 2007 if (kUseBakerReadBarrier) { 2008 DCHECK_EQ(ref->GetReadBarrierPointer(), ReadBarrier::GrayPtr()); 2009 } 2010 PushOntoMarkStack(ref); 2011 } 2012 } else { 2013 // Use the mark bitmap. 2014 accounting::ContinuousSpaceBitmap* mark_bitmap = 2015 heap_mark_bitmap_->GetContinuousSpaceBitmap(ref); 2016 accounting::LargeObjectBitmap* los_bitmap = 2017 heap_mark_bitmap_->GetLargeObjectBitmap(ref); 2018 CHECK(los_bitmap != nullptr) << "LOS bitmap covers the entire address range"; 2019 bool is_los = mark_bitmap == nullptr; 2020 if (!is_los && mark_bitmap->Test(ref)) { 2021 // Already marked. 2022 if (kUseBakerReadBarrier) { 2023 DCHECK(ref->GetReadBarrierPointer() == ReadBarrier::GrayPtr() || 2024 ref->GetReadBarrierPointer() == ReadBarrier::BlackPtr()); 2025 } 2026 } else if (is_los && los_bitmap->Test(ref)) { 2027 // Already marked in LOS. 2028 if (kUseBakerReadBarrier) { 2029 DCHECK(ref->GetReadBarrierPointer() == ReadBarrier::GrayPtr() || 2030 ref->GetReadBarrierPointer() == ReadBarrier::BlackPtr()); 2031 } 2032 } else { 2033 // Not marked. 2034 if (IsOnAllocStack(ref)) { 2035 // If it's on the allocation stack, it's considered marked. Keep it white. 2036 // Objects on the allocation stack need not be marked. 2037 if (!is_los) { 2038 DCHECK(!mark_bitmap->Test(ref)); 2039 } else { 2040 DCHECK(!los_bitmap->Test(ref)); 2041 } 2042 if (kUseBakerReadBarrier) { 2043 DCHECK_EQ(ref->GetReadBarrierPointer(), ReadBarrier::WhitePtr()); 2044 } 2045 } else { 2046 // Not marked or on the allocation stack. Try to mark it. 2047 // This may or may not succeed, which is ok. 2048 if (kUseBakerReadBarrier) { 2049 ref->AtomicSetReadBarrierPointer(ReadBarrier::WhitePtr(), ReadBarrier::GrayPtr()); 2050 } 2051 if (!is_los && mark_bitmap->AtomicTestAndSet(ref)) { 2052 // Already marked. 2053 } else if (is_los && los_bitmap->AtomicTestAndSet(ref)) { 2054 // Already marked in LOS. 2055 } else { 2056 // Newly marked. 2057 if (kUseBakerReadBarrier) { 2058 DCHECK_EQ(ref->GetReadBarrierPointer(), ReadBarrier::GrayPtr()); 2059 } 2060 PushOntoMarkStack(ref); 2061 } 2062 } 2063 } 2064 } 2065 return ref; 2066} 2067 2068void ConcurrentCopying::FinishPhase() { 2069 Thread* const self = Thread::Current(); 2070 { 2071 MutexLock mu(self, mark_stack_lock_); 2072 CHECK_EQ(pooled_mark_stacks_.size(), kMarkStackPoolSize); 2073 } 2074 region_space_ = nullptr; 2075 { 2076 MutexLock mu(Thread::Current(), skipped_blocks_lock_); 2077 skipped_blocks_map_.clear(); 2078 } 2079 ReaderMutexLock mu(self, *Locks::mutator_lock_); 2080 WriterMutexLock mu2(self, *Locks::heap_bitmap_lock_); 2081 heap_->ClearMarkedObjects(); 2082} 2083 2084bool ConcurrentCopying::IsMarkedHeapReference(mirror::HeapReference<mirror::Object>* field) { 2085 mirror::Object* from_ref = field->AsMirrorPtr(); 2086 mirror::Object* to_ref = IsMarked(from_ref); 2087 if (to_ref == nullptr) { 2088 return false; 2089 } 2090 if (from_ref != to_ref) { 2091 QuasiAtomic::ThreadFenceRelease(); 2092 field->Assign(to_ref); 2093 QuasiAtomic::ThreadFenceSequentiallyConsistent(); 2094 } 2095 return true; 2096} 2097 2098mirror::Object* ConcurrentCopying::MarkObject(mirror::Object* from_ref) { 2099 return Mark(from_ref); 2100} 2101 2102void ConcurrentCopying::DelayReferenceReferent(mirror::Class* klass, mirror::Reference* reference) { 2103 heap_->GetReferenceProcessor()->DelayReferenceReferent(klass, reference, this); 2104} 2105 2106void ConcurrentCopying::ProcessReferences(Thread* self) { 2107 TimingLogger::ScopedTiming split("ProcessReferences", GetTimings()); 2108 // We don't really need to lock the heap bitmap lock as we use CAS to mark in bitmaps. 2109 WriterMutexLock mu(self, *Locks::heap_bitmap_lock_); 2110 GetHeap()->GetReferenceProcessor()->ProcessReferences( 2111 true /*concurrent*/, GetTimings(), GetCurrentIteration()->GetClearSoftReferences(), this); 2112} 2113 2114void ConcurrentCopying::RevokeAllThreadLocalBuffers() { 2115 TimingLogger::ScopedTiming t(__FUNCTION__, GetTimings()); 2116 region_space_->RevokeAllThreadLocalBuffers(); 2117} 2118 2119} // namespace collector 2120} // namespace gc 2121} // namespace art 2122