1d5307ec41c8344be0c32273ec4f574064036187dHiroshi Yamauchi/* 2d5307ec41c8344be0c32273ec4f574064036187dHiroshi Yamauchi * Copyright (C) 2014 The Android Open Source Project 3d5307ec41c8344be0c32273ec4f574064036187dHiroshi Yamauchi * 4d5307ec41c8344be0c32273ec4f574064036187dHiroshi Yamauchi * Licensed under the Apache License, Version 2.0 (the "License"); 5d5307ec41c8344be0c32273ec4f574064036187dHiroshi Yamauchi * you may not use this file except in compliance with the License. 6d5307ec41c8344be0c32273ec4f574064036187dHiroshi Yamauchi * You may obtain a copy of the License at 7d5307ec41c8344be0c32273ec4f574064036187dHiroshi Yamauchi * 8d5307ec41c8344be0c32273ec4f574064036187dHiroshi Yamauchi * http://www.apache.org/licenses/LICENSE-2.0 9d5307ec41c8344be0c32273ec4f574064036187dHiroshi Yamauchi * 10d5307ec41c8344be0c32273ec4f574064036187dHiroshi Yamauchi * Unless required by applicable law or agreed to in writing, software 11d5307ec41c8344be0c32273ec4f574064036187dHiroshi Yamauchi * distributed under the License is distributed on an "AS IS" BASIS, 12d5307ec41c8344be0c32273ec4f574064036187dHiroshi Yamauchi * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13d5307ec41c8344be0c32273ec4f574064036187dHiroshi Yamauchi * See the License for the specific language governing permissions and 14d5307ec41c8344be0c32273ec4f574064036187dHiroshi Yamauchi * limitations under the License. 15d5307ec41c8344be0c32273ec4f574064036187dHiroshi Yamauchi */ 16d5307ec41c8344be0c32273ec4f574064036187dHiroshi Yamauchi 17d5307ec41c8344be0c32273ec4f574064036187dHiroshi Yamauchi#include "concurrent_copying.h" 18d5307ec41c8344be0c32273ec4f574064036187dHiroshi Yamauchi 19c785344b87221f5e4e6473e5b762e4e61fe65dcfMathieu Chartier#include "art_field-inl.h" 200b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi#include "base/stl_util.h" 21a6b1ead81603513fd40b77fd72f06d8cb1f35276Mathieu Chartier#include "debugger.h" 222cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi#include "gc/accounting/heap_bitmap-inl.h" 232cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi#include "gc/accounting/space_bitmap-inl.h" 243cf225386e8129dcbe32b289279ecb87ec255318Mathieu Chartier#include "gc/reference_processor.h" 252cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi#include "gc/space/image_space.h" 26073b16c8429d302d5413e8ffc488b03b8f770780Mathieu Chartier#include "gc/space/space-inl.h" 274a26f17b055cadc949c3e9fdfa637fe5656339d9Mathieu Chartier#include "image-inl.h" 282cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi#include "intern_table.h" 29e401d146407d61eeb99f8d6176b2ac13c4df1e33Mathieu Chartier#include "mirror/class-inl.h" 302cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi#include "mirror/object-inl.h" 312cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi#include "scoped_thread_state_change.h" 322cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi#include "thread-inl.h" 332cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi#include "thread_list.h" 342cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi#include "well_known_classes.h" 352cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi 36d5307ec41c8344be0c32273ec4f574064036187dHiroshi Yamauchinamespace art { 37d5307ec41c8344be0c32273ec4f574064036187dHiroshi Yamauchinamespace gc { 38d5307ec41c8344be0c32273ec4f574064036187dHiroshi Yamauchinamespace collector { 39d5307ec41c8344be0c32273ec4f574064036187dHiroshi Yamauchi 4019eab409b3efab3889885b71db708fbe56594088Hiroshi Yamauchistatic constexpr size_t kDefaultGcMarkStackSize = 2 * MB; 4119eab409b3efab3889885b71db708fbe56594088Hiroshi Yamauchi 422cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi YamauchiConcurrentCopying::ConcurrentCopying(Heap* heap, const std::string& name_prefix) 432cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi : GarbageCollector(heap, 442cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi name_prefix + (name_prefix.empty() ? "" : " ") + 452cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi "concurrent copying + mark sweep"), 460b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi region_space_(nullptr), gc_barrier_(new Barrier(0)), 470b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi gc_mark_stack_(accounting::ObjectStack::Create("concurrent copying gc mark stack", 4819eab409b3efab3889885b71db708fbe56594088Hiroshi Yamauchi kDefaultGcMarkStackSize, 4919eab409b3efab3889885b71db708fbe56594088Hiroshi Yamauchi kDefaultGcMarkStackSize)), 500b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi mark_stack_lock_("concurrent copying mark stack lock", kMarkSweepMarkStackLock), 510b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi thread_running_gc_(nullptr), 522cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi is_marking_(false), is_active_(false), is_asserting_to_space_invariant_(false), 530b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi heap_mark_bitmap_(nullptr), live_stack_freeze_size_(0), mark_stack_mode_(kMarkStackModeOff), 540b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi weak_ref_access_enabled_(true), 552cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi skipped_blocks_lock_("concurrent copying bytes blocks lock", kMarkSweepMarkStackLock), 562cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi rb_table_(heap_->GetReadBarrierTable()), 572cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi force_evacuate_all_(false) { 582cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi static_assert(space::RegionSpace::kRegionSize == accounting::ReadBarrierTable::kRegionSize, 592cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi "The region space size and the read barrier table region size must match"); 602cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi cc_heap_bitmap_.reset(new accounting::HeapBitmap(heap)); 610b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi Thread* self = Thread::Current(); 622cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi { 632cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi ReaderMutexLock mu(self, *Locks::heap_bitmap_lock_); 642cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi // Cache this so that we won't have to lock heap_bitmap_lock_ in 652cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi // Mark() which could cause a nested lock on heap_bitmap_lock_ 662cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi // when GC causes a RB while doing GC or a lock order violation 672cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi // (class_linker_lock_ and heap_bitmap_lock_). 682cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi heap_mark_bitmap_ = heap->GetMarkBitmap(); 692cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi } 700b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi { 710b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi MutexLock mu(self, mark_stack_lock_); 720b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi for (size_t i = 0; i < kMarkStackPoolSize; ++i) { 730b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi accounting::AtomicStack<mirror::Object>* mark_stack = 740b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi accounting::AtomicStack<mirror::Object>::Create( 750b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi "thread local mark stack", kMarkStackSize, kMarkStackSize); 760b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi pooled_mark_stacks_.push_back(mark_stack); 770b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi } 780b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi } 792cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi} 802cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi 81b19ccb1d88f0bff7371c9b72f265148677c18e95Mathieu Chartiervoid ConcurrentCopying::MarkHeapReference(mirror::HeapReference<mirror::Object>* from_ref) { 82b19ccb1d88f0bff7371c9b72f265148677c18e95Mathieu Chartier // Used for preserving soft references, should be OK to not have a CAS here since there should be 83b19ccb1d88f0bff7371c9b72f265148677c18e95Mathieu Chartier // no other threads which can trigger read barriers on the same referent during reference 84b19ccb1d88f0bff7371c9b72f265148677c18e95Mathieu Chartier // processing. 85b19ccb1d88f0bff7371c9b72f265148677c18e95Mathieu Chartier from_ref->Assign(Mark(from_ref->AsMirrorPtr())); 868118781ebc9659f806716c451bdb3fe9b77ae32bMathieu Chartier DCHECK(!from_ref->IsNull()); 8797509954404d031594b2ecbda607314d169d512eMathieu Chartier} 8897509954404d031594b2ecbda607314d169d512eMathieu Chartier 892cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi YamauchiConcurrentCopying::~ConcurrentCopying() { 900b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi STLDeleteElements(&pooled_mark_stacks_); 912cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi} 922cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi 932cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchivoid ConcurrentCopying::RunPhases() { 942cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi CHECK(kUseBakerReadBarrier || kUseTableLookupReadBarrier); 952cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi CHECK(!is_active_); 962cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi is_active_ = true; 972cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi Thread* self = Thread::Current(); 980b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi thread_running_gc_ = self; 992cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi Locks::mutator_lock_->AssertNotHeld(self); 1002cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi { 1012cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi ReaderMutexLock mu(self, *Locks::mutator_lock_); 1022cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi InitializePhase(); 1032cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi } 1042cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi FlipThreadRoots(); 1052cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi { 1062cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi ReaderMutexLock mu(self, *Locks::mutator_lock_); 1072cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi MarkingPhase(); 1082cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi } 1092cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi // Verify no from space refs. This causes a pause. 1102cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi if (kEnableNoFromSpaceRefsVerification || kIsDebugBuild) { 1112cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi TimingLogger::ScopedTiming split("(Paused)VerifyNoFromSpaceReferences", GetTimings()); 1122cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi ScopedPause pause(this); 1130b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi CheckEmptyMarkStack(); 1142cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi if (kVerboseMode) { 1152cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi LOG(INFO) << "Verifying no from-space refs"; 1162cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi } 1172cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi VerifyNoFromSpaceReferences(); 118720e71af6c5f92fbcddd0cff5b94d02366b74f89Mathieu Chartier if (kVerboseMode) { 119720e71af6c5f92fbcddd0cff5b94d02366b74f89Mathieu Chartier LOG(INFO) << "Done verifying no from-space refs"; 120720e71af6c5f92fbcddd0cff5b94d02366b74f89Mathieu Chartier } 1210b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi CheckEmptyMarkStack(); 1222cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi } 1232cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi { 1242cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi ReaderMutexLock mu(self, *Locks::mutator_lock_); 1252cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi ReclaimPhase(); 1262cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi } 1272cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi FinishPhase(); 1282cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi CHECK(is_active_); 1292cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi is_active_ = false; 1300b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi thread_running_gc_ = nullptr; 1312cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi} 1322cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi 1332cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchivoid ConcurrentCopying::BindBitmaps() { 1342cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi Thread* self = Thread::Current(); 1352cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi WriterMutexLock mu(self, *Locks::heap_bitmap_lock_); 1362cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi // Mark all of the spaces we never collect as immune. 1372cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi for (const auto& space : heap_->GetContinuousSpaces()) { 138763a31ed7a2bfad22a9cb07f5301a71c0f97ca49Mathieu Chartier if (space->GetGcRetentionPolicy() == space::kGcRetentionPolicyNeverCollect || 139763a31ed7a2bfad22a9cb07f5301a71c0f97ca49Mathieu Chartier space->GetGcRetentionPolicy() == space::kGcRetentionPolicyFullCollect) { 1402cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi CHECK(space->IsZygoteSpace() || space->IsImageSpace()); 141763a31ed7a2bfad22a9cb07f5301a71c0f97ca49Mathieu Chartier immune_spaces_.AddSpace(space); 1422cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi const char* bitmap_name = space->IsImageSpace() ? "cc image space bitmap" : 1432cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi "cc zygote space bitmap"; 1442cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi // TODO: try avoiding using bitmaps for image/zygote to save space. 1452cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi accounting::ContinuousSpaceBitmap* bitmap = 1462cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi accounting::ContinuousSpaceBitmap::Create(bitmap_name, space->Begin(), space->Capacity()); 1472cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi cc_heap_bitmap_->AddContinuousSpaceBitmap(bitmap); 1482cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi cc_bitmaps_.push_back(bitmap); 1492cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi } else if (space == region_space_) { 1502cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi accounting::ContinuousSpaceBitmap* bitmap = 1512cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi accounting::ContinuousSpaceBitmap::Create("cc region space bitmap", 1522cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi space->Begin(), space->Capacity()); 1532cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi cc_heap_bitmap_->AddContinuousSpaceBitmap(bitmap); 1542cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi cc_bitmaps_.push_back(bitmap); 1552cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi region_space_bitmap_ = bitmap; 1562cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi } 1572cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi } 1582cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi} 1592cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi 1602cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchivoid ConcurrentCopying::InitializePhase() { 1612cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi TimingLogger::ScopedTiming split("InitializePhase", GetTimings()); 1622cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi if (kVerboseMode) { 1632cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi LOG(INFO) << "GC InitializePhase"; 1642cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi LOG(INFO) << "Region-space : " << reinterpret_cast<void*>(region_space_->Begin()) << "-" 1652cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi << reinterpret_cast<void*>(region_space_->Limit()); 1662cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi } 1670b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi CheckEmptyMarkStack(); 168763a31ed7a2bfad22a9cb07f5301a71c0f97ca49Mathieu Chartier immune_spaces_.Reset(); 1692cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi bytes_moved_.StoreRelaxed(0); 1702cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi objects_moved_.StoreRelaxed(0); 1712cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi if (GetCurrentIteration()->GetGcCause() == kGcCauseExplicit || 1722cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi GetCurrentIteration()->GetGcCause() == kGcCauseForNativeAlloc || 1732cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi GetCurrentIteration()->GetClearSoftReferences()) { 1742cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi force_evacuate_all_ = true; 1752cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi } else { 1762cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi force_evacuate_all_ = false; 1772cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi } 1782cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi BindBitmaps(); 1792cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi if (kVerboseMode) { 1802cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi LOG(INFO) << "force_evacuate_all=" << force_evacuate_all_; 181763a31ed7a2bfad22a9cb07f5301a71c0f97ca49Mathieu Chartier LOG(INFO) << "Largest immune region: " << immune_spaces_.GetLargestImmuneRegion().Begin() 182763a31ed7a2bfad22a9cb07f5301a71c0f97ca49Mathieu Chartier << "-" << immune_spaces_.GetLargestImmuneRegion().End(); 183763a31ed7a2bfad22a9cb07f5301a71c0f97ca49Mathieu Chartier for (space::ContinuousSpace* space : immune_spaces_.GetSpaces()) { 184763a31ed7a2bfad22a9cb07f5301a71c0f97ca49Mathieu Chartier LOG(INFO) << "Immune space: " << *space; 185763a31ed7a2bfad22a9cb07f5301a71c0f97ca49Mathieu Chartier } 1862cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi LOG(INFO) << "GC end of InitializePhase"; 1872cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi } 1882cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi} 1892cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi 1902cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi// Used to switch the thread roots of a thread from from-space refs to to-space refs. 1918016bdee7ca1a066221a5d2fe5e60890de950a5bMathieu Chartierclass ConcurrentCopying::ThreadFlipVisitor : public Closure { 1922cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi public: 1933887c468d731420e929e6ad3acf190d5431e94fcRoland Levillain ThreadFlipVisitor(ConcurrentCopying* concurrent_copying, bool use_tlab) 1942cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi : concurrent_copying_(concurrent_copying), use_tlab_(use_tlab) { 1952cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi } 1962cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi 19790443477f9a0061581c420775ce3b7eeae7468bcMathieu Chartier virtual void Run(Thread* thread) OVERRIDE SHARED_REQUIRES(Locks::mutator_lock_) { 1982cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi // Note: self is not necessarily equal to thread since thread may be suspended. 1992cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi Thread* self = Thread::Current(); 2002cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi CHECK(thread == self || thread->IsSuspended() || thread->GetState() == kWaitingPerformingGc) 2012cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi << thread->GetState() << " thread " << thread << " self " << self; 20200370827646cc21cb370c3e7e93f9c0cff4c30c2Hiroshi Yamauchi thread->SetIsGcMarking(true); 2032cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi if (use_tlab_ && thread->HasTlab()) { 2042cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi if (ConcurrentCopying::kEnableFromSpaceAccountingCheck) { 2052cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi // This must come before the revoke. 2062cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi size_t thread_local_objects = thread->GetThreadLocalObjectsAllocated(); 2072cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi concurrent_copying_->region_space_->RevokeThreadLocalBuffers(thread); 2082cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi reinterpret_cast<Atomic<size_t>*>(&concurrent_copying_->from_space_num_objects_at_first_pause_)-> 2092cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi FetchAndAddSequentiallyConsistent(thread_local_objects); 2102cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi } else { 2112cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi concurrent_copying_->region_space_->RevokeThreadLocalBuffers(thread); 2122cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi } 2132cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi } 2142cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi if (kUseThreadLocalAllocationStack) { 2152cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi thread->RevokeThreadLocalAllocationStack(); 2162cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi } 2172cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi ReaderMutexLock mu(self, *Locks::heap_bitmap_lock_); 218bb87e0f1a52de656bc77cb01cb887e51a0e5198bMathieu Chartier thread->VisitRoots(concurrent_copying_); 2192cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi concurrent_copying_->GetBarrier().Pass(self); 2202cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi } 2212cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi 2222cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi private: 2232cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi ConcurrentCopying* const concurrent_copying_; 2242cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi const bool use_tlab_; 2252cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi}; 2262cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi 2272cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi// Called back from Runtime::FlipThreadRoots() during a pause. 2288016bdee7ca1a066221a5d2fe5e60890de950a5bMathieu Chartierclass ConcurrentCopying::FlipCallback : public Closure { 2292cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi public: 2302cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi explicit FlipCallback(ConcurrentCopying* concurrent_copying) 2312cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi : concurrent_copying_(concurrent_copying) { 2322cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi } 2332cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi 23490443477f9a0061581c420775ce3b7eeae7468bcMathieu Chartier virtual void Run(Thread* thread) OVERRIDE REQUIRES(Locks::mutator_lock_) { 2352cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi ConcurrentCopying* cc = concurrent_copying_; 2362cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi TimingLogger::ScopedTiming split("(Paused)FlipCallback", cc->GetTimings()); 2372cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi // Note: self is not necessarily equal to thread since thread may be suspended. 2382cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi Thread* self = Thread::Current(); 2392cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi CHECK(thread == self); 2402cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi Locks::mutator_lock_->AssertExclusiveHeld(self); 2412cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi cc->region_space_->SetFromSpace(cc->rb_table_, cc->force_evacuate_all_); 242a4f6af9b1e6380b31674d7ac645b1732c846ac06Mathieu Chartier cc->SwapStacks(); 2432cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi if (ConcurrentCopying::kEnableFromSpaceAccountingCheck) { 2442cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi cc->RecordLiveStackFreezeSize(self); 2452cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi cc->from_space_num_objects_at_first_pause_ = cc->region_space_->GetObjectsAllocated(); 2462cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi cc->from_space_num_bytes_at_first_pause_ = cc->region_space_->GetBytesAllocated(); 2472cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi } 2482cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi cc->is_marking_ = true; 2490b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi cc->mark_stack_mode_.StoreRelaxed(ConcurrentCopying::kMarkStackModeThreadLocal); 2502cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi if (UNLIKELY(Runtime::Current()->IsActiveTransaction())) { 251184c9dc3bfc500134fdb2fbea0a7fab290c0abb0Mathieu Chartier CHECK(Runtime::Current()->IsAotCompiler()); 2522cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi TimingLogger::ScopedTiming split2("(Paused)VisitTransactionRoots", cc->GetTimings()); 253bb87e0f1a52de656bc77cb01cb887e51a0e5198bMathieu Chartier Runtime::Current()->VisitTransactionRoots(cc); 2542cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi } 2552cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi } 2562cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi 2572cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi private: 2582cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi ConcurrentCopying* const concurrent_copying_; 2592cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi}; 2602cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi 2612cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi// Switch threads that from from-space to to-space refs. Forward/mark the thread roots. 2622cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchivoid ConcurrentCopying::FlipThreadRoots() { 2632cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi TimingLogger::ScopedTiming split("FlipThreadRoots", GetTimings()); 2642cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi if (kVerboseMode) { 2652cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi LOG(INFO) << "time=" << region_space_->Time(); 2662cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi region_space_->DumpNonFreeRegions(LOG(INFO)); 2672cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi } 2682cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi Thread* self = Thread::Current(); 2692cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi Locks::mutator_lock_->AssertNotHeld(self); 2702cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi gc_barrier_->Init(self, 0); 2712cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi ThreadFlipVisitor thread_flip_visitor(this, heap_->use_tlab_); 2722cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi FlipCallback flip_callback(this); 27376f55b030d2517d434f227bee2363c3fb760e5c6Hiroshi Yamauchi heap_->ThreadFlipBegin(self); // Sync with JNI critical calls. 2742cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi size_t barrier_count = Runtime::Current()->FlipThreadRoots( 2752cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi &thread_flip_visitor, &flip_callback, this); 27676f55b030d2517d434f227bee2363c3fb760e5c6Hiroshi Yamauchi heap_->ThreadFlipEnd(self); 2772cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi { 2782cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi ScopedThreadStateChange tsc(self, kWaitingForCheckPointsToRun); 2792cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi gc_barrier_->Increment(self, barrier_count); 2802cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi } 2812cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi is_asserting_to_space_invariant_ = true; 2822cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi QuasiAtomic::ThreadFenceForConstructor(); 2832cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi if (kVerboseMode) { 2842cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi LOG(INFO) << "time=" << region_space_->Time(); 2852cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi region_space_->DumpNonFreeRegions(LOG(INFO)); 2862cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi LOG(INFO) << "GC end of FlipThreadRoots"; 2872cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi } 2882cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi} 2892cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi 290a4f6af9b1e6380b31674d7ac645b1732c846ac06Mathieu Chartiervoid ConcurrentCopying::SwapStacks() { 291a4f6af9b1e6380b31674d7ac645b1732c846ac06Mathieu Chartier heap_->SwapStacks(); 2922cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi} 2932cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi 2942cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchivoid ConcurrentCopying::RecordLiveStackFreezeSize(Thread* self) { 2952cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi WriterMutexLock mu(self, *Locks::heap_bitmap_lock_); 2962cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi live_stack_freeze_size_ = heap_->GetLiveStack()->Size(); 2972cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi} 2982cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi 2992cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi// Used to visit objects in the immune spaces. 3008016bdee7ca1a066221a5d2fe5e60890de950a5bMathieu Chartierclass ConcurrentCopying::ImmuneSpaceObjVisitor { 3012cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi public: 3028016bdee7ca1a066221a5d2fe5e60890de950a5bMathieu Chartier explicit ImmuneSpaceObjVisitor(ConcurrentCopying* cc) : collector_(cc) {} 3032cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi 30490443477f9a0061581c420775ce3b7eeae7468bcMathieu Chartier void operator()(mirror::Object* obj) const SHARED_REQUIRES(Locks::mutator_lock_) 30590443477f9a0061581c420775ce3b7eeae7468bcMathieu Chartier SHARED_REQUIRES(Locks::heap_bitmap_lock_) { 3062cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi DCHECK(obj != nullptr); 307763a31ed7a2bfad22a9cb07f5301a71c0f97ca49Mathieu Chartier DCHECK(collector_->immune_spaces_.ContainsObject(obj)); 3082cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi accounting::ContinuousSpaceBitmap* cc_bitmap = 3092cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi collector_->cc_heap_bitmap_->GetContinuousSpaceBitmap(obj); 3102cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi DCHECK(cc_bitmap != nullptr) 3112cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi << "An immune space object must have a bitmap"; 3122cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi if (kIsDebugBuild) { 3132cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi DCHECK(collector_->heap_->GetMarkBitmap()->Test(obj)) 3142cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi << "Immune space object must be already marked"; 3152cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi } 3162cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi // This may or may not succeed, which is ok. 3172cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi if (kUseBakerReadBarrier) { 3182cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi obj->AtomicSetReadBarrierPointer(ReadBarrier::WhitePtr(), ReadBarrier::GrayPtr()); 3192cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi } 3202cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi if (cc_bitmap->AtomicTestAndSet(obj)) { 3212cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi // Already marked. Do nothing. 3222cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi } else { 3232cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi // Newly marked. Set the gray bit and push it onto the mark stack. 3242cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi CHECK(!kUseBakerReadBarrier || obj->GetReadBarrierPointer() == ReadBarrier::GrayPtr()); 3250b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi collector_->PushOntoMarkStack(obj); 3262cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi } 3272cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi } 3282cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi 3292cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi private: 33097509954404d031594b2ecbda607314d169d512eMathieu Chartier ConcurrentCopying* const collector_; 3312cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi}; 3322cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi 3332cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchiclass EmptyCheckpoint : public Closure { 3342cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi public: 3352cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi explicit EmptyCheckpoint(ConcurrentCopying* concurrent_copying) 3362cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi : concurrent_copying_(concurrent_copying) { 3372cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi } 3382cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi 3392cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi virtual void Run(Thread* thread) OVERRIDE NO_THREAD_SAFETY_ANALYSIS { 3402cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi // Note: self is not necessarily equal to thread since thread may be suspended. 3412cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi Thread* self = Thread::Current(); 3422cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi CHECK(thread == self || thread->IsSuspended() || thread->GetState() == kWaitingPerformingGc) 3432cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi << thread->GetState() << " thread " << thread << " self " << self; 344dd9943d4466b052ef6c5ee5b32187adb48cbce74Lei Li // If thread is a running mutator, then act on behalf of the garbage collector. 345dd9943d4466b052ef6c5ee5b32187adb48cbce74Lei Li // See the code in ThreadList::RunCheckpoint. 34610d2508b105427ef1bcaf0c222873bae7acc66d3Mathieu Chartier concurrent_copying_->GetBarrier().Pass(self); 3472cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi } 3482cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi 3492cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi private: 3502cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi ConcurrentCopying* const concurrent_copying_; 3512cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi}; 3522cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi 3532cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi// Concurrently mark roots that are guarded by read barriers and process the mark stack. 3542cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchivoid ConcurrentCopying::MarkingPhase() { 3552cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi TimingLogger::ScopedTiming split("MarkingPhase", GetTimings()); 3562cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi if (kVerboseMode) { 3572cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi LOG(INFO) << "GC MarkingPhase"; 3582cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi } 3590b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi CHECK(weak_ref_access_enabled_); 3602cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi { 3612cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi // Mark the image root. The WB-based collectors do not need to 3622cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi // scan the image objects from roots by relying on the card table, 3632cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi // but it's necessary for the RB to-space invariant to hold. 3642cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi TimingLogger::ScopedTiming split1("VisitImageRoots", GetTimings()); 365073b16c8429d302d5413e8ffc488b03b8f770780Mathieu Chartier for (space::ContinuousSpace* space : heap_->GetContinuousSpaces()) { 366073b16c8429d302d5413e8ffc488b03b8f770780Mathieu Chartier if (space->IsImageSpace()) { 367073b16c8429d302d5413e8ffc488b03b8f770780Mathieu Chartier gc::space::ImageSpace* image = space->AsImageSpace(); 368073b16c8429d302d5413e8ffc488b03b8f770780Mathieu Chartier if (image != nullptr) { 369073b16c8429d302d5413e8ffc488b03b8f770780Mathieu Chartier mirror::ObjectArray<mirror::Object>* image_root = image->GetImageHeader().GetImageRoots(); 370073b16c8429d302d5413e8ffc488b03b8f770780Mathieu Chartier mirror::Object* marked_image_root = Mark(image_root); 371073b16c8429d302d5413e8ffc488b03b8f770780Mathieu Chartier CHECK_EQ(image_root, marked_image_root) << "An image object does not move"; 372073b16c8429d302d5413e8ffc488b03b8f770780Mathieu Chartier if (ReadBarrier::kEnableToSpaceInvariantChecks) { 373073b16c8429d302d5413e8ffc488b03b8f770780Mathieu Chartier AssertToSpaceInvariant(nullptr, MemberOffset(0), marked_image_root); 374073b16c8429d302d5413e8ffc488b03b8f770780Mathieu Chartier } 375073b16c8429d302d5413e8ffc488b03b8f770780Mathieu Chartier } 3762cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi } 3772cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi } 3782cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi } 3792cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi { 380723e6cee35671d2dd9aeb884dd11f6994307c01fHiroshi Yamauchi TimingLogger::ScopedTiming split2("VisitConcurrentRoots", GetTimings()); 381723e6cee35671d2dd9aeb884dd11f6994307c01fHiroshi Yamauchi Runtime::Current()->VisitConcurrentRoots(this, kVisitRootFlagAllRoots); 3822cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi } 3832cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi { 3842cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi // TODO: don't visit the transaction roots if it's not active. 3852cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi TimingLogger::ScopedTiming split5("VisitNonThreadRoots", GetTimings()); 386bb87e0f1a52de656bc77cb01cb887e51a0e5198bMathieu Chartier Runtime::Current()->VisitNonThreadRoots(this); 3872cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi } 3882cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi 3892cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi // Immune spaces. 390763a31ed7a2bfad22a9cb07f5301a71c0f97ca49Mathieu Chartier for (auto& space : immune_spaces_.GetSpaces()) { 391763a31ed7a2bfad22a9cb07f5301a71c0f97ca49Mathieu Chartier DCHECK(space->IsImageSpace() || space->IsZygoteSpace()); 392763a31ed7a2bfad22a9cb07f5301a71c0f97ca49Mathieu Chartier accounting::ContinuousSpaceBitmap* live_bitmap = space->GetLiveBitmap(); 3938016bdee7ca1a066221a5d2fe5e60890de950a5bMathieu Chartier ImmuneSpaceObjVisitor visitor(this); 394763a31ed7a2bfad22a9cb07f5301a71c0f97ca49Mathieu Chartier live_bitmap->VisitMarkedRange(reinterpret_cast<uintptr_t>(space->Begin()), 395763a31ed7a2bfad22a9cb07f5301a71c0f97ca49Mathieu Chartier reinterpret_cast<uintptr_t>(space->Limit()), 396763a31ed7a2bfad22a9cb07f5301a71c0f97ca49Mathieu Chartier visitor); 3972cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi } 3982cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi 3992cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi Thread* self = Thread::Current(); 4002cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi { 401a6b1ead81603513fd40b77fd72f06d8cb1f35276Mathieu Chartier TimingLogger::ScopedTiming split7("ProcessMarkStack", GetTimings()); 4020b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi // We transition through three mark stack modes (thread-local, shared, GC-exclusive). The 4030b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi // primary reasons are the fact that we need to use a checkpoint to process thread-local mark 4040b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi // stacks, but after we disable weak refs accesses, we can't use a checkpoint due to a deadlock 4050b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi // issue because running threads potentially blocking at WaitHoldingLocks, and that once we 4060b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi // reach the point where we process weak references, we can avoid using a lock when accessing 4070b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi // the GC mark stack, which makes mark stack processing more efficient. 4080b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi 4090b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi // Process the mark stack once in the thread local stack mode. This marks most of the live 4100b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi // objects, aside from weak ref accesses with read barriers (Reference::GetReferent() and system 4110b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi // weaks) that may happen concurrently while we processing the mark stack and newly mark/gray 4120b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi // objects and push refs on the mark stack. 4132cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi ProcessMarkStack(); 4140b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi // Switch to the shared mark stack mode. That is, revoke and process thread-local mark stacks 4150b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi // for the last time before transitioning to the shared mark stack mode, which would process new 4160b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi // refs that may have been concurrently pushed onto the mark stack during the ProcessMarkStack() 4170b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi // call above. At the same time, disable weak ref accesses using a per-thread flag. It's 4180b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi // important to do these together in a single checkpoint so that we can ensure that mutators 4190b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi // won't newly gray objects and push new refs onto the mark stack due to weak ref accesses and 4200b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi // mutators safely transition to the shared mark stack mode (without leaving unprocessed refs on 4210b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi // the thread-local mark stacks), without a race. This is why we use a thread-local weak ref 4220b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi // access flag Thread::tls32_.weak_ref_access_enabled_ instead of the global ones. 4230b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi SwitchToSharedMarkStackMode(); 4240b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi CHECK(!self->GetWeakRefAccessEnabled()); 4250b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi // Now that weak refs accesses are disabled, once we exhaust the shared mark stack again here 4260b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi // (which may be non-empty if there were refs found on thread-local mark stacks during the above 4270b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi // SwitchToSharedMarkStackMode() call), we won't have new refs to process, that is, mutators 4280b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi // (via read barriers) have no way to produce any more refs to process. Marking converges once 4290b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi // before we process weak refs below. 4300b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi ProcessMarkStack(); 4310b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi CheckEmptyMarkStack(); 4320b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi // Switch to the GC exclusive mark stack mode so that we can process the mark stack without a 4330b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi // lock from this point on. 4340b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi SwitchToGcExclusiveMarkStackMode(); 4350b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi CheckEmptyMarkStack(); 4362cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi if (kVerboseMode) { 4372cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi LOG(INFO) << "ProcessReferences"; 4382cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi } 4390b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi // Process weak references. This may produce new refs to process and have them processed via 44097509954404d031594b2ecbda607314d169d512eMathieu Chartier // ProcessMarkStack (in the GC exclusive mark stack mode). 4410b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi ProcessReferences(self); 4420b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi CheckEmptyMarkStack(); 4432cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi if (kVerboseMode) { 4442cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi LOG(INFO) << "SweepSystemWeaks"; 4452cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi } 4462cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi SweepSystemWeaks(self); 4472cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi if (kVerboseMode) { 4482cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi LOG(INFO) << "SweepSystemWeaks done"; 4492cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi } 4500b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi // Process the mark stack here one last time because the above SweepSystemWeaks() call may have 4510b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi // marked some objects (strings alive) as hash_set::Erase() can call the hash function for 4520b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi // arbitrary elements in the weak intern table in InternTable::Table::SweepWeaks(). 4532cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi ProcessMarkStack(); 4540b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi CheckEmptyMarkStack(); 4550b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi // Re-enable weak ref accesses. 4560b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi ReenableWeakRefAccess(self); 457951ec2c93c79c5539cbcc669566f0808d4460338Mathieu Chartier // Free data for class loaders that we unloaded. 458951ec2c93c79c5539cbcc669566f0808d4460338Mathieu Chartier Runtime::Current()->GetClassLinker()->CleanupClassLoaders(); 4590b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi // Marking is done. Disable marking. 46000370827646cc21cb370c3e7e93f9c0cff4c30c2Hiroshi Yamauchi DisableMarking(); 4610b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi CheckEmptyMarkStack(); 4622cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi } 4632cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi 4640b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi CHECK(weak_ref_access_enabled_); 4652cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi if (kVerboseMode) { 4662cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi LOG(INFO) << "GC end of MarkingPhase"; 4672cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi } 4682cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi} 4692cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi 4700b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchivoid ConcurrentCopying::ReenableWeakRefAccess(Thread* self) { 4710b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi if (kVerboseMode) { 4720b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi LOG(INFO) << "ReenableWeakRefAccess"; 4730b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi } 4740b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi weak_ref_access_enabled_.StoreRelaxed(true); // This is for new threads. 4750b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi QuasiAtomic::ThreadFenceForConstructor(); 4760b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi // Iterate all threads (don't need to or can't use a checkpoint) and re-enable weak ref access. 4770b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi { 4780b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi MutexLock mu(self, *Locks::thread_list_lock_); 4790b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi std::list<Thread*> thread_list = Runtime::Current()->GetThreadList()->GetList(); 4800b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi for (Thread* thread : thread_list) { 4810b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi thread->SetWeakRefAccessEnabled(true); 4820b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi } 4830b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi } 4840b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi // Unblock blocking threads. 4850b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi GetHeap()->GetReferenceProcessor()->BroadcastForSlowPath(self); 4860b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi Runtime::Current()->BroadcastForNewSystemWeaks(); 4870b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi} 4880b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi 4898016bdee7ca1a066221a5d2fe5e60890de950a5bMathieu Chartierclass ConcurrentCopying::DisableMarkingCheckpoint : public Closure { 49000370827646cc21cb370c3e7e93f9c0cff4c30c2Hiroshi Yamauchi public: 49100370827646cc21cb370c3e7e93f9c0cff4c30c2Hiroshi Yamauchi explicit DisableMarkingCheckpoint(ConcurrentCopying* concurrent_copying) 49200370827646cc21cb370c3e7e93f9c0cff4c30c2Hiroshi Yamauchi : concurrent_copying_(concurrent_copying) { 49300370827646cc21cb370c3e7e93f9c0cff4c30c2Hiroshi Yamauchi } 49400370827646cc21cb370c3e7e93f9c0cff4c30c2Hiroshi Yamauchi 49500370827646cc21cb370c3e7e93f9c0cff4c30c2Hiroshi Yamauchi void Run(Thread* thread) OVERRIDE NO_THREAD_SAFETY_ANALYSIS { 49600370827646cc21cb370c3e7e93f9c0cff4c30c2Hiroshi Yamauchi // Note: self is not necessarily equal to thread since thread may be suspended. 49700370827646cc21cb370c3e7e93f9c0cff4c30c2Hiroshi Yamauchi Thread* self = Thread::Current(); 49800370827646cc21cb370c3e7e93f9c0cff4c30c2Hiroshi Yamauchi DCHECK(thread == self || thread->IsSuspended() || thread->GetState() == kWaitingPerformingGc) 49900370827646cc21cb370c3e7e93f9c0cff4c30c2Hiroshi Yamauchi << thread->GetState() << " thread " << thread << " self " << self; 50000370827646cc21cb370c3e7e93f9c0cff4c30c2Hiroshi Yamauchi // Disable the thread-local is_gc_marking flag. 501fdbd13c7af91a042eda753e436eeebf0e1937250Hiroshi Yamauchi // Note a thread that has just started right before this checkpoint may have already this flag 502fdbd13c7af91a042eda753e436eeebf0e1937250Hiroshi Yamauchi // set to false, which is ok. 50300370827646cc21cb370c3e7e93f9c0cff4c30c2Hiroshi Yamauchi thread->SetIsGcMarking(false); 50400370827646cc21cb370c3e7e93f9c0cff4c30c2Hiroshi Yamauchi // If thread is a running mutator, then act on behalf of the garbage collector. 50500370827646cc21cb370c3e7e93f9c0cff4c30c2Hiroshi Yamauchi // See the code in ThreadList::RunCheckpoint. 50610d2508b105427ef1bcaf0c222873bae7acc66d3Mathieu Chartier concurrent_copying_->GetBarrier().Pass(self); 50700370827646cc21cb370c3e7e93f9c0cff4c30c2Hiroshi Yamauchi } 50800370827646cc21cb370c3e7e93f9c0cff4c30c2Hiroshi Yamauchi 50900370827646cc21cb370c3e7e93f9c0cff4c30c2Hiroshi Yamauchi private: 51000370827646cc21cb370c3e7e93f9c0cff4c30c2Hiroshi Yamauchi ConcurrentCopying* const concurrent_copying_; 51100370827646cc21cb370c3e7e93f9c0cff4c30c2Hiroshi Yamauchi}; 51200370827646cc21cb370c3e7e93f9c0cff4c30c2Hiroshi Yamauchi 51300370827646cc21cb370c3e7e93f9c0cff4c30c2Hiroshi Yamauchivoid ConcurrentCopying::IssueDisableMarkingCheckpoint() { 51400370827646cc21cb370c3e7e93f9c0cff4c30c2Hiroshi Yamauchi Thread* self = Thread::Current(); 51500370827646cc21cb370c3e7e93f9c0cff4c30c2Hiroshi Yamauchi DisableMarkingCheckpoint check_point(this); 51600370827646cc21cb370c3e7e93f9c0cff4c30c2Hiroshi Yamauchi ThreadList* thread_list = Runtime::Current()->GetThreadList(); 51700370827646cc21cb370c3e7e93f9c0cff4c30c2Hiroshi Yamauchi gc_barrier_->Init(self, 0); 51800370827646cc21cb370c3e7e93f9c0cff4c30c2Hiroshi Yamauchi size_t barrier_count = thread_list->RunCheckpoint(&check_point); 51900370827646cc21cb370c3e7e93f9c0cff4c30c2Hiroshi Yamauchi // If there are no threads to wait which implies that all the checkpoint functions are finished, 52000370827646cc21cb370c3e7e93f9c0cff4c30c2Hiroshi Yamauchi // then no need to release the mutator lock. 52100370827646cc21cb370c3e7e93f9c0cff4c30c2Hiroshi Yamauchi if (barrier_count == 0) { 52200370827646cc21cb370c3e7e93f9c0cff4c30c2Hiroshi Yamauchi return; 52300370827646cc21cb370c3e7e93f9c0cff4c30c2Hiroshi Yamauchi } 52400370827646cc21cb370c3e7e93f9c0cff4c30c2Hiroshi Yamauchi // Release locks then wait for all mutator threads to pass the barrier. 52500370827646cc21cb370c3e7e93f9c0cff4c30c2Hiroshi Yamauchi Locks::mutator_lock_->SharedUnlock(self); 52600370827646cc21cb370c3e7e93f9c0cff4c30c2Hiroshi Yamauchi { 52700370827646cc21cb370c3e7e93f9c0cff4c30c2Hiroshi Yamauchi ScopedThreadStateChange tsc(self, kWaitingForCheckPointsToRun); 52800370827646cc21cb370c3e7e93f9c0cff4c30c2Hiroshi Yamauchi gc_barrier_->Increment(self, barrier_count); 52900370827646cc21cb370c3e7e93f9c0cff4c30c2Hiroshi Yamauchi } 53000370827646cc21cb370c3e7e93f9c0cff4c30c2Hiroshi Yamauchi Locks::mutator_lock_->SharedLock(self); 53100370827646cc21cb370c3e7e93f9c0cff4c30c2Hiroshi Yamauchi} 53200370827646cc21cb370c3e7e93f9c0cff4c30c2Hiroshi Yamauchi 53300370827646cc21cb370c3e7e93f9c0cff4c30c2Hiroshi Yamauchivoid ConcurrentCopying::DisableMarking() { 53400370827646cc21cb370c3e7e93f9c0cff4c30c2Hiroshi Yamauchi // Change the global is_marking flag to false. Do a fence before doing a checkpoint to update the 53500370827646cc21cb370c3e7e93f9c0cff4c30c2Hiroshi Yamauchi // thread-local flags so that a new thread starting up will get the correct is_marking flag. 53600370827646cc21cb370c3e7e93f9c0cff4c30c2Hiroshi Yamauchi is_marking_ = false; 53700370827646cc21cb370c3e7e93f9c0cff4c30c2Hiroshi Yamauchi QuasiAtomic::ThreadFenceForConstructor(); 53800370827646cc21cb370c3e7e93f9c0cff4c30c2Hiroshi Yamauchi // Use a checkpoint to turn off the thread-local is_gc_marking flags and to ensure no threads are 53900370827646cc21cb370c3e7e93f9c0cff4c30c2Hiroshi Yamauchi // still in the middle of a read barrier which may have a from-space ref cached in a local 54000370827646cc21cb370c3e7e93f9c0cff4c30c2Hiroshi Yamauchi // variable. 54100370827646cc21cb370c3e7e93f9c0cff4c30c2Hiroshi Yamauchi IssueDisableMarkingCheckpoint(); 54200370827646cc21cb370c3e7e93f9c0cff4c30c2Hiroshi Yamauchi if (kUseTableLookupReadBarrier) { 54300370827646cc21cb370c3e7e93f9c0cff4c30c2Hiroshi Yamauchi heap_->rb_table_->ClearAll(); 54400370827646cc21cb370c3e7e93f9c0cff4c30c2Hiroshi Yamauchi DCHECK(heap_->rb_table_->IsAllCleared()); 54500370827646cc21cb370c3e7e93f9c0cff4c30c2Hiroshi Yamauchi } 54600370827646cc21cb370c3e7e93f9c0cff4c30c2Hiroshi Yamauchi is_mark_stack_push_disallowed_.StoreSequentiallyConsistent(1); 54700370827646cc21cb370c3e7e93f9c0cff4c30c2Hiroshi Yamauchi mark_stack_mode_.StoreSequentiallyConsistent(kMarkStackModeOff); 54800370827646cc21cb370c3e7e93f9c0cff4c30c2Hiroshi Yamauchi} 54900370827646cc21cb370c3e7e93f9c0cff4c30c2Hiroshi Yamauchi 5502cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchivoid ConcurrentCopying::IssueEmptyCheckpoint() { 5512cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi Thread* self = Thread::Current(); 5522cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi EmptyCheckpoint check_point(this); 5532cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi ThreadList* thread_list = Runtime::Current()->GetThreadList(); 5542cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi gc_barrier_->Init(self, 0); 5552cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi size_t barrier_count = thread_list->RunCheckpoint(&check_point); 556dd9943d4466b052ef6c5ee5b32187adb48cbce74Lei Li // If there are no threads to wait which implys that all the checkpoint functions are finished, 557dd9943d4466b052ef6c5ee5b32187adb48cbce74Lei Li // then no need to release the mutator lock. 558dd9943d4466b052ef6c5ee5b32187adb48cbce74Lei Li if (barrier_count == 0) { 559dd9943d4466b052ef6c5ee5b32187adb48cbce74Lei Li return; 560dd9943d4466b052ef6c5ee5b32187adb48cbce74Lei Li } 5612cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi // Release locks then wait for all mutator threads to pass the barrier. 5622cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi Locks::mutator_lock_->SharedUnlock(self); 5632cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi { 5642cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi ScopedThreadStateChange tsc(self, kWaitingForCheckPointsToRun); 5652cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi gc_barrier_->Increment(self, barrier_count); 5662cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi } 5672cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi Locks::mutator_lock_->SharedLock(self); 5682cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi} 5692cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi 57019eab409b3efab3889885b71db708fbe56594088Hiroshi Yamauchivoid ConcurrentCopying::ExpandGcMarkStack() { 57119eab409b3efab3889885b71db708fbe56594088Hiroshi Yamauchi DCHECK(gc_mark_stack_->IsFull()); 57219eab409b3efab3889885b71db708fbe56594088Hiroshi Yamauchi const size_t new_size = gc_mark_stack_->Capacity() * 2; 57319eab409b3efab3889885b71db708fbe56594088Hiroshi Yamauchi std::vector<StackReference<mirror::Object>> temp(gc_mark_stack_->Begin(), 57419eab409b3efab3889885b71db708fbe56594088Hiroshi Yamauchi gc_mark_stack_->End()); 57519eab409b3efab3889885b71db708fbe56594088Hiroshi Yamauchi gc_mark_stack_->Resize(new_size); 57619eab409b3efab3889885b71db708fbe56594088Hiroshi Yamauchi for (auto& ref : temp) { 57719eab409b3efab3889885b71db708fbe56594088Hiroshi Yamauchi gc_mark_stack_->PushBack(ref.AsMirrorPtr()); 57819eab409b3efab3889885b71db708fbe56594088Hiroshi Yamauchi } 57919eab409b3efab3889885b71db708fbe56594088Hiroshi Yamauchi DCHECK(!gc_mark_stack_->IsFull()); 58019eab409b3efab3889885b71db708fbe56594088Hiroshi Yamauchi} 58119eab409b3efab3889885b71db708fbe56594088Hiroshi Yamauchi 5822cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchivoid ConcurrentCopying::PushOntoMarkStack(mirror::Object* to_ref) { 5830b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi CHECK_EQ(is_mark_stack_push_disallowed_.LoadRelaxed(), 0) 5842cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi << " " << to_ref << " " << PrettyTypeOf(to_ref); 5850b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi Thread* self = Thread::Current(); // TODO: pass self as an argument from call sites? 5860b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi CHECK(thread_running_gc_ != nullptr); 5870b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi MarkStackMode mark_stack_mode = mark_stack_mode_.LoadRelaxed(); 588723e6cee35671d2dd9aeb884dd11f6994307c01fHiroshi Yamauchi if (LIKELY(mark_stack_mode == kMarkStackModeThreadLocal)) { 589723e6cee35671d2dd9aeb884dd11f6994307c01fHiroshi Yamauchi if (LIKELY(self == thread_running_gc_)) { 5900b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi // If GC-running thread, use the GC mark stack instead of a thread-local mark stack. 5910b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi CHECK(self->GetThreadLocalMarkStack() == nullptr); 59219eab409b3efab3889885b71db708fbe56594088Hiroshi Yamauchi if (UNLIKELY(gc_mark_stack_->IsFull())) { 59319eab409b3efab3889885b71db708fbe56594088Hiroshi Yamauchi ExpandGcMarkStack(); 59419eab409b3efab3889885b71db708fbe56594088Hiroshi Yamauchi } 5950b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi gc_mark_stack_->PushBack(to_ref); 5960b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi } else { 5970b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi // Otherwise, use a thread-local mark stack. 5980b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi accounting::AtomicStack<mirror::Object>* tl_mark_stack = self->GetThreadLocalMarkStack(); 5990b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi if (UNLIKELY(tl_mark_stack == nullptr || tl_mark_stack->IsFull())) { 6000b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi MutexLock mu(self, mark_stack_lock_); 6010b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi // Get a new thread local mark stack. 6020b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi accounting::AtomicStack<mirror::Object>* new_tl_mark_stack; 6030b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi if (!pooled_mark_stacks_.empty()) { 6040b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi // Use a pooled mark stack. 6050b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi new_tl_mark_stack = pooled_mark_stacks_.back(); 6060b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi pooled_mark_stacks_.pop_back(); 6070b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi } else { 6080b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi // None pooled. Create a new one. 6090b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi new_tl_mark_stack = 6100b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi accounting::AtomicStack<mirror::Object>::Create( 6110b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi "thread local mark stack", 4 * KB, 4 * KB); 6120b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi } 6130b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi DCHECK(new_tl_mark_stack != nullptr); 6140b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi DCHECK(new_tl_mark_stack->IsEmpty()); 6150b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi new_tl_mark_stack->PushBack(to_ref); 6160b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi self->SetThreadLocalMarkStack(new_tl_mark_stack); 6170b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi if (tl_mark_stack != nullptr) { 6180b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi // Store the old full stack into a vector. 6190b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi revoked_mark_stacks_.push_back(tl_mark_stack); 6200b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi } 6210b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi } else { 6220b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi tl_mark_stack->PushBack(to_ref); 6230b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi } 6240b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi } 6250b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi } else if (mark_stack_mode == kMarkStackModeShared) { 6260b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi // Access the shared GC mark stack with a lock. 6270b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi MutexLock mu(self, mark_stack_lock_); 62819eab409b3efab3889885b71db708fbe56594088Hiroshi Yamauchi if (UNLIKELY(gc_mark_stack_->IsFull())) { 62919eab409b3efab3889885b71db708fbe56594088Hiroshi Yamauchi ExpandGcMarkStack(); 63019eab409b3efab3889885b71db708fbe56594088Hiroshi Yamauchi } 6310b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi gc_mark_stack_->PushBack(to_ref); 6322cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi } else { 6330b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi CHECK_EQ(static_cast<uint32_t>(mark_stack_mode), 634fa75518907c2b997770ced9adb34df7c3c4665c4Hiroshi Yamauchi static_cast<uint32_t>(kMarkStackModeGcExclusive)) 635fa75518907c2b997770ced9adb34df7c3c4665c4Hiroshi Yamauchi << "ref=" << to_ref 636fa75518907c2b997770ced9adb34df7c3c4665c4Hiroshi Yamauchi << " self->gc_marking=" << self->GetIsGcMarking() 637fa75518907c2b997770ced9adb34df7c3c4665c4Hiroshi Yamauchi << " cc->is_marking=" << is_marking_; 6380b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi CHECK(self == thread_running_gc_) 6390b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi << "Only GC-running thread should access the mark stack " 6400b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi << "in the GC exclusive mark stack mode"; 6410b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi // Access the GC mark stack without a lock. 64219eab409b3efab3889885b71db708fbe56594088Hiroshi Yamauchi if (UNLIKELY(gc_mark_stack_->IsFull())) { 64319eab409b3efab3889885b71db708fbe56594088Hiroshi Yamauchi ExpandGcMarkStack(); 64419eab409b3efab3889885b71db708fbe56594088Hiroshi Yamauchi } 6450b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi gc_mark_stack_->PushBack(to_ref); 6462cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi } 6472cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi} 6482cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi 6492cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchiaccounting::ObjectStack* ConcurrentCopying::GetAllocationStack() { 6502cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi return heap_->allocation_stack_.get(); 6512cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi} 6522cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi 6532cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchiaccounting::ObjectStack* ConcurrentCopying::GetLiveStack() { 6542cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi return heap_->live_stack_.get(); 6552cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi} 6562cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi 6572cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi// The following visitors are that used to verify that there's no 6582cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi// references to the from-space left after marking. 6598016bdee7ca1a066221a5d2fe5e60890de950a5bMathieu Chartierclass ConcurrentCopying::VerifyNoFromSpaceRefsVisitor : public SingleRootVisitor { 6602cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi public: 6618016bdee7ca1a066221a5d2fe5e60890de950a5bMathieu Chartier explicit VerifyNoFromSpaceRefsVisitor(ConcurrentCopying* collector) 6622cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi : collector_(collector) {} 6632cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi 6642cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi void operator()(mirror::Object* ref) const 66590443477f9a0061581c420775ce3b7eeae7468bcMathieu Chartier SHARED_REQUIRES(Locks::mutator_lock_) ALWAYS_INLINE { 6662cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi if (ref == nullptr) { 6672cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi // OK. 6682cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi return; 6692cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi } 6702cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi collector_->AssertToSpaceInvariant(nullptr, MemberOffset(0), ref); 6712cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi if (kUseBakerReadBarrier) { 6722cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi if (collector_->RegionSpace()->IsInToSpace(ref)) { 6732cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi CHECK(ref->GetReadBarrierPointer() == nullptr) 6742cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi << "To-space ref " << ref << " " << PrettyTypeOf(ref) 6752cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi << " has non-white rb_ptr " << ref->GetReadBarrierPointer(); 6762cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi } else { 6772cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi CHECK(ref->GetReadBarrierPointer() == ReadBarrier::BlackPtr() || 6782cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi (ref->GetReadBarrierPointer() == ReadBarrier::WhitePtr() && 6792cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi collector_->IsOnAllocStack(ref))) 6802cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi << "Non-moving/unevac from space ref " << ref << " " << PrettyTypeOf(ref) 6812cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi << " has non-black rb_ptr " << ref->GetReadBarrierPointer() 6822cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi << " but isn't on the alloc stack (and has white rb_ptr)." 6832cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi << " Is it in the non-moving space=" 6842cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi << (collector_->GetHeap()->GetNonMovingSpace()->HasAddress(ref)); 6852cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi } 6862cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi } 6872cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi } 6882cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi 689bb87e0f1a52de656bc77cb01cb887e51a0e5198bMathieu Chartier void VisitRoot(mirror::Object* root, const RootInfo& info ATTRIBUTE_UNUSED) 69090443477f9a0061581c420775ce3b7eeae7468bcMathieu Chartier OVERRIDE SHARED_REQUIRES(Locks::mutator_lock_) { 6912cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi DCHECK(root != nullptr); 692bb87e0f1a52de656bc77cb01cb887e51a0e5198bMathieu Chartier operator()(root); 6932cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi } 6942cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi 6952cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi private: 696bb87e0f1a52de656bc77cb01cb887e51a0e5198bMathieu Chartier ConcurrentCopying* const collector_; 6972cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi}; 6982cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi 6998016bdee7ca1a066221a5d2fe5e60890de950a5bMathieu Chartierclass ConcurrentCopying::VerifyNoFromSpaceRefsFieldVisitor { 7002cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi public: 7018016bdee7ca1a066221a5d2fe5e60890de950a5bMathieu Chartier explicit VerifyNoFromSpaceRefsFieldVisitor(ConcurrentCopying* collector) 7022cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi : collector_(collector) {} 7032cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi 704da7c650022a974be10e2f00fa07d5109e3d8826fMathieu Chartier void operator()(mirror::Object* obj, MemberOffset offset, bool is_static ATTRIBUTE_UNUSED) const 70590443477f9a0061581c420775ce3b7eeae7468bcMathieu Chartier SHARED_REQUIRES(Locks::mutator_lock_) ALWAYS_INLINE { 7062cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi mirror::Object* ref = 7072cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi obj->GetFieldObject<mirror::Object, kDefaultVerifyFlags, kWithoutReadBarrier>(offset); 7088016bdee7ca1a066221a5d2fe5e60890de950a5bMathieu Chartier VerifyNoFromSpaceRefsVisitor visitor(collector_); 7092cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi visitor(ref); 7102cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi } 7112cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi void operator()(mirror::Class* klass, mirror::Reference* ref) const 71290443477f9a0061581c420775ce3b7eeae7468bcMathieu Chartier SHARED_REQUIRES(Locks::mutator_lock_) ALWAYS_INLINE { 7132cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi CHECK(klass->IsTypeOfReferenceClass()); 7142cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi this->operator()(ref, mirror::Reference::ReferentOffset(), false); 7152cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi } 7162cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi 717da7c650022a974be10e2f00fa07d5109e3d8826fMathieu Chartier void VisitRootIfNonNull(mirror::CompressedReference<mirror::Object>* root) const 718da7c650022a974be10e2f00fa07d5109e3d8826fMathieu Chartier SHARED_REQUIRES(Locks::mutator_lock_) { 719da7c650022a974be10e2f00fa07d5109e3d8826fMathieu Chartier if (!root->IsNull()) { 720da7c650022a974be10e2f00fa07d5109e3d8826fMathieu Chartier VisitRoot(root); 721da7c650022a974be10e2f00fa07d5109e3d8826fMathieu Chartier } 722da7c650022a974be10e2f00fa07d5109e3d8826fMathieu Chartier } 723da7c650022a974be10e2f00fa07d5109e3d8826fMathieu Chartier 724da7c650022a974be10e2f00fa07d5109e3d8826fMathieu Chartier void VisitRoot(mirror::CompressedReference<mirror::Object>* root) const 725da7c650022a974be10e2f00fa07d5109e3d8826fMathieu Chartier SHARED_REQUIRES(Locks::mutator_lock_) { 7268016bdee7ca1a066221a5d2fe5e60890de950a5bMathieu Chartier VerifyNoFromSpaceRefsVisitor visitor(collector_); 727da7c650022a974be10e2f00fa07d5109e3d8826fMathieu Chartier visitor(root->AsMirrorPtr()); 728da7c650022a974be10e2f00fa07d5109e3d8826fMathieu Chartier } 729da7c650022a974be10e2f00fa07d5109e3d8826fMathieu Chartier 7302cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi private: 73197509954404d031594b2ecbda607314d169d512eMathieu Chartier ConcurrentCopying* const collector_; 7322cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi}; 7332cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi 7348016bdee7ca1a066221a5d2fe5e60890de950a5bMathieu Chartierclass ConcurrentCopying::VerifyNoFromSpaceRefsObjectVisitor { 7352cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi public: 7368016bdee7ca1a066221a5d2fe5e60890de950a5bMathieu Chartier explicit VerifyNoFromSpaceRefsObjectVisitor(ConcurrentCopying* collector) 7372cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi : collector_(collector) {} 7382cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi void operator()(mirror::Object* obj) const 73990443477f9a0061581c420775ce3b7eeae7468bcMathieu Chartier SHARED_REQUIRES(Locks::mutator_lock_) { 7402cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi ObjectCallback(obj, collector_); 7412cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi } 7422cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi static void ObjectCallback(mirror::Object* obj, void *arg) 74390443477f9a0061581c420775ce3b7eeae7468bcMathieu Chartier SHARED_REQUIRES(Locks::mutator_lock_) { 7442cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi CHECK(obj != nullptr); 7452cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi ConcurrentCopying* collector = reinterpret_cast<ConcurrentCopying*>(arg); 7462cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi space::RegionSpace* region_space = collector->RegionSpace(); 7472cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi CHECK(!region_space->IsInFromSpace(obj)) << "Scanning object " << obj << " in from space"; 7488016bdee7ca1a066221a5d2fe5e60890de950a5bMathieu Chartier VerifyNoFromSpaceRefsFieldVisitor visitor(collector); 749059ef3ddb2088f926ac452889e0953fdcd646a5eMathieu Chartier obj->VisitReferences(visitor, visitor); 7502cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi if (kUseBakerReadBarrier) { 7512cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi if (collector->RegionSpace()->IsInToSpace(obj)) { 7522cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi CHECK(obj->GetReadBarrierPointer() == nullptr) 7532cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi << "obj=" << obj << " non-white rb_ptr " << obj->GetReadBarrierPointer(); 7542cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi } else { 7552cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi CHECK(obj->GetReadBarrierPointer() == ReadBarrier::BlackPtr() || 7562cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi (obj->GetReadBarrierPointer() == ReadBarrier::WhitePtr() && 7572cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi collector->IsOnAllocStack(obj))) 7582cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi << "Non-moving space/unevac from space ref " << obj << " " << PrettyTypeOf(obj) 7592cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi << " has non-black rb_ptr " << obj->GetReadBarrierPointer() 7602cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi << " but isn't on the alloc stack (and has white rb_ptr). Is it in the non-moving space=" 7612cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi << (collector->GetHeap()->GetNonMovingSpace()->HasAddress(obj)); 7622cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi } 7632cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi } 7642cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi } 7652cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi 7662cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi private: 7672cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi ConcurrentCopying* const collector_; 7682cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi}; 7692cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi 7702cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi// Verify there's no from-space references left after the marking phase. 7712cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchivoid ConcurrentCopying::VerifyNoFromSpaceReferences() { 7722cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi Thread* self = Thread::Current(); 7732cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi DCHECK(Locks::mutator_lock_->IsExclusiveHeld(self)); 77400370827646cc21cb370c3e7e93f9c0cff4c30c2Hiroshi Yamauchi // Verify all threads have is_gc_marking to be false 77500370827646cc21cb370c3e7e93f9c0cff4c30c2Hiroshi Yamauchi { 77600370827646cc21cb370c3e7e93f9c0cff4c30c2Hiroshi Yamauchi MutexLock mu(self, *Locks::thread_list_lock_); 77700370827646cc21cb370c3e7e93f9c0cff4c30c2Hiroshi Yamauchi std::list<Thread*> thread_list = Runtime::Current()->GetThreadList()->GetList(); 77800370827646cc21cb370c3e7e93f9c0cff4c30c2Hiroshi Yamauchi for (Thread* thread : thread_list) { 77900370827646cc21cb370c3e7e93f9c0cff4c30c2Hiroshi Yamauchi CHECK(!thread->GetIsGcMarking()); 78000370827646cc21cb370c3e7e93f9c0cff4c30c2Hiroshi Yamauchi } 78100370827646cc21cb370c3e7e93f9c0cff4c30c2Hiroshi Yamauchi } 7828016bdee7ca1a066221a5d2fe5e60890de950a5bMathieu Chartier VerifyNoFromSpaceRefsObjectVisitor visitor(this); 7832cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi // Roots. 7842cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi { 7852cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi ReaderMutexLock mu(self, *Locks::heap_bitmap_lock_); 7868016bdee7ca1a066221a5d2fe5e60890de950a5bMathieu Chartier VerifyNoFromSpaceRefsVisitor ref_visitor(this); 787bb87e0f1a52de656bc77cb01cb887e51a0e5198bMathieu Chartier Runtime::Current()->VisitRoots(&ref_visitor); 7882cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi } 7892cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi // The to-space. 7908016bdee7ca1a066221a5d2fe5e60890de950a5bMathieu Chartier region_space_->WalkToSpace(VerifyNoFromSpaceRefsObjectVisitor::ObjectCallback, this); 7912cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi // Non-moving spaces. 7922cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi { 7932cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi WriterMutexLock mu(self, *Locks::heap_bitmap_lock_); 7942cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi heap_->GetMarkBitmap()->Visit(visitor); 7952cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi } 7962cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi // The alloc stack. 7972cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi { 7988016bdee7ca1a066221a5d2fe5e60890de950a5bMathieu Chartier VerifyNoFromSpaceRefsVisitor ref_visitor(this); 799cb535da36915f9d10bec3880b46f1de1f7a69f22Mathieu Chartier for (auto* it = heap_->allocation_stack_->Begin(), *end = heap_->allocation_stack_->End(); 800cb535da36915f9d10bec3880b46f1de1f7a69f22Mathieu Chartier it < end; ++it) { 801cb535da36915f9d10bec3880b46f1de1f7a69f22Mathieu Chartier mirror::Object* const obj = it->AsMirrorPtr(); 8022cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi if (obj != nullptr && obj->GetClass() != nullptr) { 8032cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi // TODO: need to call this only if obj is alive? 8042cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi ref_visitor(obj); 8052cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi visitor(obj); 8062cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi } 8072cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi } 8082cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi } 8092cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi // TODO: LOS. But only refs in LOS are classes. 8102cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi} 8112cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi 8122cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi// The following visitors are used to assert the to-space invariant. 8138016bdee7ca1a066221a5d2fe5e60890de950a5bMathieu Chartierclass ConcurrentCopying::AssertToSpaceInvariantRefsVisitor { 8142cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi public: 8158016bdee7ca1a066221a5d2fe5e60890de950a5bMathieu Chartier explicit AssertToSpaceInvariantRefsVisitor(ConcurrentCopying* collector) 8162cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi : collector_(collector) {} 8172cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi 8182cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi void operator()(mirror::Object* ref) const 81990443477f9a0061581c420775ce3b7eeae7468bcMathieu Chartier SHARED_REQUIRES(Locks::mutator_lock_) ALWAYS_INLINE { 8202cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi if (ref == nullptr) { 8212cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi // OK. 8222cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi return; 8232cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi } 8242cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi collector_->AssertToSpaceInvariant(nullptr, MemberOffset(0), ref); 8252cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi } 8262cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi 8272cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi private: 82897509954404d031594b2ecbda607314d169d512eMathieu Chartier ConcurrentCopying* const collector_; 8292cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi}; 8302cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi 8318016bdee7ca1a066221a5d2fe5e60890de950a5bMathieu Chartierclass ConcurrentCopying::AssertToSpaceInvariantFieldVisitor { 8322cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi public: 8338016bdee7ca1a066221a5d2fe5e60890de950a5bMathieu Chartier explicit AssertToSpaceInvariantFieldVisitor(ConcurrentCopying* collector) 8342cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi : collector_(collector) {} 8352cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi 836da7c650022a974be10e2f00fa07d5109e3d8826fMathieu Chartier void operator()(mirror::Object* obj, MemberOffset offset, bool is_static ATTRIBUTE_UNUSED) const 83790443477f9a0061581c420775ce3b7eeae7468bcMathieu Chartier SHARED_REQUIRES(Locks::mutator_lock_) ALWAYS_INLINE { 8382cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi mirror::Object* ref = 8392cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi obj->GetFieldObject<mirror::Object, kDefaultVerifyFlags, kWithoutReadBarrier>(offset); 8408016bdee7ca1a066221a5d2fe5e60890de950a5bMathieu Chartier AssertToSpaceInvariantRefsVisitor visitor(collector_); 8412cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi visitor(ref); 8422cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi } 843da7c650022a974be10e2f00fa07d5109e3d8826fMathieu Chartier void operator()(mirror::Class* klass, mirror::Reference* ref ATTRIBUTE_UNUSED) const 84490443477f9a0061581c420775ce3b7eeae7468bcMathieu Chartier SHARED_REQUIRES(Locks::mutator_lock_) ALWAYS_INLINE { 8452cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi CHECK(klass->IsTypeOfReferenceClass()); 8462cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi } 8472cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi 848da7c650022a974be10e2f00fa07d5109e3d8826fMathieu Chartier void VisitRootIfNonNull(mirror::CompressedReference<mirror::Object>* root) const 849da7c650022a974be10e2f00fa07d5109e3d8826fMathieu Chartier SHARED_REQUIRES(Locks::mutator_lock_) { 850da7c650022a974be10e2f00fa07d5109e3d8826fMathieu Chartier if (!root->IsNull()) { 851da7c650022a974be10e2f00fa07d5109e3d8826fMathieu Chartier VisitRoot(root); 852da7c650022a974be10e2f00fa07d5109e3d8826fMathieu Chartier } 853da7c650022a974be10e2f00fa07d5109e3d8826fMathieu Chartier } 854da7c650022a974be10e2f00fa07d5109e3d8826fMathieu Chartier 855da7c650022a974be10e2f00fa07d5109e3d8826fMathieu Chartier void VisitRoot(mirror::CompressedReference<mirror::Object>* root) const 856da7c650022a974be10e2f00fa07d5109e3d8826fMathieu Chartier SHARED_REQUIRES(Locks::mutator_lock_) { 8578016bdee7ca1a066221a5d2fe5e60890de950a5bMathieu Chartier AssertToSpaceInvariantRefsVisitor visitor(collector_); 858da7c650022a974be10e2f00fa07d5109e3d8826fMathieu Chartier visitor(root->AsMirrorPtr()); 859da7c650022a974be10e2f00fa07d5109e3d8826fMathieu Chartier } 860da7c650022a974be10e2f00fa07d5109e3d8826fMathieu Chartier 8612cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi private: 86297509954404d031594b2ecbda607314d169d512eMathieu Chartier ConcurrentCopying* const collector_; 8632cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi}; 8642cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi 8658016bdee7ca1a066221a5d2fe5e60890de950a5bMathieu Chartierclass ConcurrentCopying::AssertToSpaceInvariantObjectVisitor { 8662cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi public: 8678016bdee7ca1a066221a5d2fe5e60890de950a5bMathieu Chartier explicit AssertToSpaceInvariantObjectVisitor(ConcurrentCopying* collector) 8682cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi : collector_(collector) {} 8692cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi void operator()(mirror::Object* obj) const 87090443477f9a0061581c420775ce3b7eeae7468bcMathieu Chartier SHARED_REQUIRES(Locks::mutator_lock_) { 8712cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi ObjectCallback(obj, collector_); 8722cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi } 8732cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi static void ObjectCallback(mirror::Object* obj, void *arg) 87490443477f9a0061581c420775ce3b7eeae7468bcMathieu Chartier SHARED_REQUIRES(Locks::mutator_lock_) { 8752cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi CHECK(obj != nullptr); 8762cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi ConcurrentCopying* collector = reinterpret_cast<ConcurrentCopying*>(arg); 8772cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi space::RegionSpace* region_space = collector->RegionSpace(); 8782cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi CHECK(!region_space->IsInFromSpace(obj)) << "Scanning object " << obj << " in from space"; 8792cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi collector->AssertToSpaceInvariant(nullptr, MemberOffset(0), obj); 8808016bdee7ca1a066221a5d2fe5e60890de950a5bMathieu Chartier AssertToSpaceInvariantFieldVisitor visitor(collector); 881059ef3ddb2088f926ac452889e0953fdcd646a5eMathieu Chartier obj->VisitReferences(visitor, visitor); 8822cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi } 8832cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi 8842cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi private: 88597509954404d031594b2ecbda607314d169d512eMathieu Chartier ConcurrentCopying* const collector_; 8862cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi}; 8872cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi 8888016bdee7ca1a066221a5d2fe5e60890de950a5bMathieu Chartierclass ConcurrentCopying::RevokeThreadLocalMarkStackCheckpoint : public Closure { 8890b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi public: 8903887c468d731420e929e6ad3acf190d5431e94fcRoland Levillain RevokeThreadLocalMarkStackCheckpoint(ConcurrentCopying* concurrent_copying, 8913887c468d731420e929e6ad3acf190d5431e94fcRoland Levillain bool disable_weak_ref_access) 8920b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi : concurrent_copying_(concurrent_copying), 8930b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi disable_weak_ref_access_(disable_weak_ref_access) { 8940b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi } 8950b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi 8960b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi virtual void Run(Thread* thread) OVERRIDE NO_THREAD_SAFETY_ANALYSIS { 8970b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi // Note: self is not necessarily equal to thread since thread may be suspended. 8980b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi Thread* self = Thread::Current(); 8990b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi CHECK(thread == self || thread->IsSuspended() || thread->GetState() == kWaitingPerformingGc) 9000b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi << thread->GetState() << " thread " << thread << " self " << self; 9010b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi // Revoke thread local mark stacks. 9020b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi accounting::AtomicStack<mirror::Object>* tl_mark_stack = thread->GetThreadLocalMarkStack(); 9030b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi if (tl_mark_stack != nullptr) { 9040b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi MutexLock mu(self, concurrent_copying_->mark_stack_lock_); 9050b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi concurrent_copying_->revoked_mark_stacks_.push_back(tl_mark_stack); 9060b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi thread->SetThreadLocalMarkStack(nullptr); 9070b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi } 9080b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi // Disable weak ref access. 9090b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi if (disable_weak_ref_access_) { 9100b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi thread->SetWeakRefAccessEnabled(false); 9110b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi } 9120b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi // If thread is a running mutator, then act on behalf of the garbage collector. 9130b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi // See the code in ThreadList::RunCheckpoint. 91410d2508b105427ef1bcaf0c222873bae7acc66d3Mathieu Chartier concurrent_copying_->GetBarrier().Pass(self); 9150b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi } 9160b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi 9170b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi private: 9180b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi ConcurrentCopying* const concurrent_copying_; 9190b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi const bool disable_weak_ref_access_; 9200b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi}; 9210b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi 9220b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchivoid ConcurrentCopying::RevokeThreadLocalMarkStacks(bool disable_weak_ref_access) { 9230b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi Thread* self = Thread::Current(); 9240b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi RevokeThreadLocalMarkStackCheckpoint check_point(this, disable_weak_ref_access); 9250b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi ThreadList* thread_list = Runtime::Current()->GetThreadList(); 9260b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi gc_barrier_->Init(self, 0); 9270b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi size_t barrier_count = thread_list->RunCheckpoint(&check_point); 9280b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi // If there are no threads to wait which implys that all the checkpoint functions are finished, 9290b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi // then no need to release the mutator lock. 9300b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi if (barrier_count == 0) { 9310b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi return; 9320b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi } 9330b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi Locks::mutator_lock_->SharedUnlock(self); 9340b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi { 9350b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi ScopedThreadStateChange tsc(self, kWaitingForCheckPointsToRun); 9360b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi gc_barrier_->Increment(self, barrier_count); 9370b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi } 9380b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi Locks::mutator_lock_->SharedLock(self); 9390b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi} 9400b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi 9410b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchivoid ConcurrentCopying::RevokeThreadLocalMarkStack(Thread* thread) { 9420b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi Thread* self = Thread::Current(); 9430b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi CHECK_EQ(self, thread); 9440b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi accounting::AtomicStack<mirror::Object>* tl_mark_stack = thread->GetThreadLocalMarkStack(); 9450b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi if (tl_mark_stack != nullptr) { 9460b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi CHECK(is_marking_); 9470b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi MutexLock mu(self, mark_stack_lock_); 9480b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi revoked_mark_stacks_.push_back(tl_mark_stack); 9490b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi thread->SetThreadLocalMarkStack(nullptr); 9500b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi } 9510b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi} 9520b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi 9530b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchivoid ConcurrentCopying::ProcessMarkStack() { 9542cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi if (kVerboseMode) { 9552cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi LOG(INFO) << "ProcessMarkStack. "; 9562cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi } 9570b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi bool empty_prev = false; 9580b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi while (true) { 9590b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi bool empty = ProcessMarkStackOnce(); 9600b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi if (empty_prev && empty) { 9610b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi // Saw empty mark stack for a second time, done. 9620b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi break; 9630b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi } 9640b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi empty_prev = empty; 9650b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi } 9660b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi} 9670b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi 9680b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchibool ConcurrentCopying::ProcessMarkStackOnce() { 9690b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi Thread* self = Thread::Current(); 9700b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi CHECK(thread_running_gc_ != nullptr); 9710b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi CHECK(self == thread_running_gc_); 9720b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi CHECK(self->GetThreadLocalMarkStack() == nullptr); 9732cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi size_t count = 0; 9740b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi MarkStackMode mark_stack_mode = mark_stack_mode_.LoadRelaxed(); 9750b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi if (mark_stack_mode == kMarkStackModeThreadLocal) { 9760b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi // Process the thread-local mark stacks and the GC mark stack. 9770b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi count += ProcessThreadLocalMarkStacks(false); 9780b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi while (!gc_mark_stack_->IsEmpty()) { 9790b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi mirror::Object* to_ref = gc_mark_stack_->PopBack(); 9800b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi ProcessMarkStackRef(to_ref); 9810b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi ++count; 9822cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi } 9830b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi gc_mark_stack_->Reset(); 9840b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi } else if (mark_stack_mode == kMarkStackModeShared) { 9850b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi // Process the shared GC mark stack with a lock. 9860b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi { 9870b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi MutexLock mu(self, mark_stack_lock_); 9880b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi CHECK(revoked_mark_stacks_.empty()); 9890b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi } 9900b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi while (true) { 9910b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi std::vector<mirror::Object*> refs; 9920b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi { 9930b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi // Copy refs with lock. Note the number of refs should be small. 9940b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi MutexLock mu(self, mark_stack_lock_); 9950b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi if (gc_mark_stack_->IsEmpty()) { 9960b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi break; 9972cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi } 9980b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi for (StackReference<mirror::Object>* p = gc_mark_stack_->Begin(); 9990b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi p != gc_mark_stack_->End(); ++p) { 10000b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi refs.push_back(p->AsMirrorPtr()); 10010b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi } 10020b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi gc_mark_stack_->Reset(); 10030b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi } 10040b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi for (mirror::Object* ref : refs) { 10050b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi ProcessMarkStackRef(ref); 10060b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi ++count; 10072cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi } 10082cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi } 10090b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi } else { 10100b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi CHECK_EQ(static_cast<uint32_t>(mark_stack_mode), 10110b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi static_cast<uint32_t>(kMarkStackModeGcExclusive)); 10120b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi { 10130b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi MutexLock mu(self, mark_stack_lock_); 10140b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi CHECK(revoked_mark_stacks_.empty()); 10150b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi } 10160b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi // Process the GC mark stack in the exclusive mode. No need to take the lock. 10170b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi while (!gc_mark_stack_->IsEmpty()) { 10180b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi mirror::Object* to_ref = gc_mark_stack_->PopBack(); 10190b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi ProcessMarkStackRef(to_ref); 10200b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi ++count; 10212cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi } 10220b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi gc_mark_stack_->Reset(); 10232cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi } 10240b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi 10252cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi // Return true if the stack was empty. 10262cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi return count == 0; 10272cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi} 10282cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi 10290b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchisize_t ConcurrentCopying::ProcessThreadLocalMarkStacks(bool disable_weak_ref_access) { 10300b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi // Run a checkpoint to collect all thread local mark stacks and iterate over them all. 10310b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi RevokeThreadLocalMarkStacks(disable_weak_ref_access); 10320b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi size_t count = 0; 10330b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi std::vector<accounting::AtomicStack<mirror::Object>*> mark_stacks; 10340b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi { 10350b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi MutexLock mu(Thread::Current(), mark_stack_lock_); 10360b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi // Make a copy of the mark stack vector. 10370b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi mark_stacks = revoked_mark_stacks_; 10380b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi revoked_mark_stacks_.clear(); 10390b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi } 10400b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi for (accounting::AtomicStack<mirror::Object>* mark_stack : mark_stacks) { 10410b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi for (StackReference<mirror::Object>* p = mark_stack->Begin(); p != mark_stack->End(); ++p) { 10420b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi mirror::Object* to_ref = p->AsMirrorPtr(); 10430b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi ProcessMarkStackRef(to_ref); 10440b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi ++count; 10450b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi } 10460b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi { 10470b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi MutexLock mu(Thread::Current(), mark_stack_lock_); 10480b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi if (pooled_mark_stacks_.size() >= kMarkStackPoolSize) { 10490b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi // The pool has enough. Delete it. 10500b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi delete mark_stack; 10510b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi } else { 10520b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi // Otherwise, put it into the pool for later reuse. 10530b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi mark_stack->Reset(); 10540b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi pooled_mark_stacks_.push_back(mark_stack); 10550b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi } 10560b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi } 10570b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi } 10580b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi return count; 10590b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi} 10600b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi 1061723e6cee35671d2dd9aeb884dd11f6994307c01fHiroshi Yamauchiinline void ConcurrentCopying::ProcessMarkStackRef(mirror::Object* to_ref) { 10620b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi DCHECK(!region_space_->IsInFromSpace(to_ref)); 10630b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi if (kUseBakerReadBarrier) { 10640b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi DCHECK(to_ref->GetReadBarrierPointer() == ReadBarrier::GrayPtr()) 10650b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi << " " << to_ref << " " << to_ref->GetReadBarrierPointer() 10660b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi << " is_marked=" << IsMarked(to_ref); 10670b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi } 10680b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi // Scan ref fields. 10690b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi Scan(to_ref); 10700b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi // Mark the gray ref as white or black. 10710b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi if (kUseBakerReadBarrier) { 10720b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi DCHECK(to_ref->GetReadBarrierPointer() == ReadBarrier::GrayPtr()) 10730b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi << " " << to_ref << " " << to_ref->GetReadBarrierPointer() 10740b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi << " is_marked=" << IsMarked(to_ref); 10750b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi } 1076723e6cee35671d2dd9aeb884dd11f6994307c01fHiroshi Yamauchi#ifdef USE_BAKER_OR_BROOKS_READ_BARRIER 1077723e6cee35671d2dd9aeb884dd11f6994307c01fHiroshi Yamauchi if (UNLIKELY((to_ref->GetClass<kVerifyNone, kWithoutReadBarrier>()->IsTypeOfReferenceClass() && 1078723e6cee35671d2dd9aeb884dd11f6994307c01fHiroshi Yamauchi to_ref->AsReference()->GetReferent<kWithoutReadBarrier>() != nullptr && 1079723e6cee35671d2dd9aeb884dd11f6994307c01fHiroshi Yamauchi !IsInToSpace(to_ref->AsReference()->GetReferent<kWithoutReadBarrier>())))) { 108070c08d3c913ce2150cd620ea87b919f8eb5bd953Hiroshi Yamauchi // Leave this Reference gray in the queue so that GetReferent() will trigger a read barrier. We 108170c08d3c913ce2150cd620ea87b919f8eb5bd953Hiroshi Yamauchi // will change it to black or white later in ReferenceQueue::DequeuePendingReference(). 1082e362740479483d4eecfe11cb9ac642401c411cc3Richard Uhler DCHECK(to_ref->AsReference()->GetPendingNext() != nullptr) << "Left unenqueued ref gray " << to_ref; 10830b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi } else { 108470c08d3c913ce2150cd620ea87b919f8eb5bd953Hiroshi Yamauchi // We may occasionally leave a Reference black or white in the queue if its referent happens to 108570c08d3c913ce2150cd620ea87b919f8eb5bd953Hiroshi Yamauchi // be concurrently marked after the Scan() call above has enqueued the Reference, in which case 108670c08d3c913ce2150cd620ea87b919f8eb5bd953Hiroshi Yamauchi // the above IsInToSpace() evaluates to true and we change the color from gray to black or white 108770c08d3c913ce2150cd620ea87b919f8eb5bd953Hiroshi Yamauchi // here in this else block. 10880b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi if (kUseBakerReadBarrier) { 10890b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi if (region_space_->IsInToSpace(to_ref)) { 10900b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi // If to-space, change from gray to white. 1091ed70b4a439acea537d55266bb9b1f309de8cbec9Hiroshi Yamauchi bool success = to_ref->AtomicSetReadBarrierPointer</*kCasRelease*/true>( 1092ed70b4a439acea537d55266bb9b1f309de8cbec9Hiroshi Yamauchi ReadBarrier::GrayPtr(), 1093ed70b4a439acea537d55266bb9b1f309de8cbec9Hiroshi Yamauchi ReadBarrier::WhitePtr()); 1094ed70b4a439acea537d55266bb9b1f309de8cbec9Hiroshi Yamauchi DCHECK(success) << "Must succeed as we won the race."; 1095723e6cee35671d2dd9aeb884dd11f6994307c01fHiroshi Yamauchi DCHECK(to_ref->GetReadBarrierPointer() == ReadBarrier::WhitePtr()); 10962cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi } else { 10970b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi // If non-moving space/unevac from space, change from gray 10980b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi // to black. We can't change gray to white because it's not 10990b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi // safe to use CAS if two threads change values in opposite 11000b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi // directions (A->B and B->A). So, we change it to black to 11010b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi // indicate non-moving objects that have been marked 11020b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi // through. Note we'd need to change from black to white 11030b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi // later (concurrently). 1104ed70b4a439acea537d55266bb9b1f309de8cbec9Hiroshi Yamauchi bool success = to_ref->AtomicSetReadBarrierPointer</*kCasRelease*/true>( 1105ed70b4a439acea537d55266bb9b1f309de8cbec9Hiroshi Yamauchi ReadBarrier::GrayPtr(), 1106ed70b4a439acea537d55266bb9b1f309de8cbec9Hiroshi Yamauchi ReadBarrier::BlackPtr()); 1107ed70b4a439acea537d55266bb9b1f309de8cbec9Hiroshi Yamauchi DCHECK(success) << "Must succeed as we won the race."; 1108723e6cee35671d2dd9aeb884dd11f6994307c01fHiroshi Yamauchi DCHECK(to_ref->GetReadBarrierPointer() == ReadBarrier::BlackPtr()); 11092cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi } 11102cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi } 1111723e6cee35671d2dd9aeb884dd11f6994307c01fHiroshi Yamauchi } 11120b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi#else 1113723e6cee35671d2dd9aeb884dd11f6994307c01fHiroshi Yamauchi DCHECK(!kUseBakerReadBarrier); 11140b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi#endif 11150b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi if (ReadBarrier::kEnableToSpaceInvariantChecks || kIsDebugBuild) { 11168016bdee7ca1a066221a5d2fe5e60890de950a5bMathieu Chartier AssertToSpaceInvariantObjectVisitor visitor(this); 11170b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi visitor(to_ref); 11180b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi } 11190b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi} 11200b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi 11210b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchivoid ConcurrentCopying::SwitchToSharedMarkStackMode() { 11220b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi Thread* self = Thread::Current(); 11230b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi CHECK(thread_running_gc_ != nullptr); 11240b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi CHECK_EQ(self, thread_running_gc_); 11250b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi CHECK(self->GetThreadLocalMarkStack() == nullptr); 11260b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi MarkStackMode before_mark_stack_mode = mark_stack_mode_.LoadRelaxed(); 11270b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi CHECK_EQ(static_cast<uint32_t>(before_mark_stack_mode), 11280b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi static_cast<uint32_t>(kMarkStackModeThreadLocal)); 11290b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi mark_stack_mode_.StoreRelaxed(kMarkStackModeShared); 11300b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi CHECK(weak_ref_access_enabled_.LoadRelaxed()); 11310b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi weak_ref_access_enabled_.StoreRelaxed(false); 11320b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi QuasiAtomic::ThreadFenceForConstructor(); 11330b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi // Process the thread local mark stacks one last time after switching to the shared mark stack 11340b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi // mode and disable weak ref accesses. 11350b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi ProcessThreadLocalMarkStacks(true); 11360b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi if (kVerboseMode) { 11370b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi LOG(INFO) << "Switched to shared mark stack mode and disabled weak ref access"; 11380b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi } 11390b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi} 11400b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi 11410b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchivoid ConcurrentCopying::SwitchToGcExclusiveMarkStackMode() { 11420b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi Thread* self = Thread::Current(); 11430b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi CHECK(thread_running_gc_ != nullptr); 11440b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi CHECK_EQ(self, thread_running_gc_); 11450b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi CHECK(self->GetThreadLocalMarkStack() == nullptr); 11460b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi MarkStackMode before_mark_stack_mode = mark_stack_mode_.LoadRelaxed(); 11470b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi CHECK_EQ(static_cast<uint32_t>(before_mark_stack_mode), 11480b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi static_cast<uint32_t>(kMarkStackModeShared)); 11490b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi mark_stack_mode_.StoreRelaxed(kMarkStackModeGcExclusive); 11500b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi QuasiAtomic::ThreadFenceForConstructor(); 11510b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi if (kVerboseMode) { 11520b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi LOG(INFO) << "Switched to GC exclusive mark stack mode"; 11530b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi } 11540b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi} 11550b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi 11560b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchivoid ConcurrentCopying::CheckEmptyMarkStack() { 11570b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi Thread* self = Thread::Current(); 11580b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi CHECK(thread_running_gc_ != nullptr); 11590b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi CHECK_EQ(self, thread_running_gc_); 11600b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi CHECK(self->GetThreadLocalMarkStack() == nullptr); 11610b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi MarkStackMode mark_stack_mode = mark_stack_mode_.LoadRelaxed(); 11620b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi if (mark_stack_mode == kMarkStackModeThreadLocal) { 11630b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi // Thread-local mark stack mode. 11640b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi RevokeThreadLocalMarkStacks(false); 11650b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi MutexLock mu(Thread::Current(), mark_stack_lock_); 11660b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi if (!revoked_mark_stacks_.empty()) { 11670b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi for (accounting::AtomicStack<mirror::Object>* mark_stack : revoked_mark_stacks_) { 11680b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi while (!mark_stack->IsEmpty()) { 11690b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi mirror::Object* obj = mark_stack->PopBack(); 11700b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi if (kUseBakerReadBarrier) { 11710b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi mirror::Object* rb_ptr = obj->GetReadBarrierPointer(); 11720b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi LOG(INFO) << "On mark queue : " << obj << " " << PrettyTypeOf(obj) << " rb_ptr=" << rb_ptr 11730b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi << " is_marked=" << IsMarked(obj); 11740b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi } else { 11750b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi LOG(INFO) << "On mark queue : " << obj << " " << PrettyTypeOf(obj) 11760b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi << " is_marked=" << IsMarked(obj); 11770b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi } 11780b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi } 11790b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi } 11800b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi LOG(FATAL) << "mark stack is not empty"; 11810b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi } 11820b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi } else { 11830b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi // Shared, GC-exclusive, or off. 11840b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi MutexLock mu(Thread::Current(), mark_stack_lock_); 11850b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi CHECK(gc_mark_stack_->IsEmpty()); 11860b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi CHECK(revoked_mark_stacks_.empty()); 11872cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi } 11882cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi} 11892cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi 11902cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchivoid ConcurrentCopying::SweepSystemWeaks(Thread* self) { 11912cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi TimingLogger::ScopedTiming split("SweepSystemWeaks", GetTimings()); 11922cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi ReaderMutexLock mu(self, *Locks::heap_bitmap_lock_); 119397509954404d031594b2ecbda607314d169d512eMathieu Chartier Runtime::Current()->SweepSystemWeaks(this); 11942cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi} 11952cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi 11962cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchivoid ConcurrentCopying::Sweep(bool swap_bitmaps) { 11972cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi { 11982cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi TimingLogger::ScopedTiming t("MarkStackAsLive", GetTimings()); 11992cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi accounting::ObjectStack* live_stack = heap_->GetLiveStack(); 12002cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi if (kEnableFromSpaceAccountingCheck) { 12012cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi CHECK_GE(live_stack_freeze_size_, live_stack->Size()); 12022cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi } 12032cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi heap_->MarkAllocStackAsLive(live_stack); 12042cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi live_stack->Reset(); 12052cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi } 12060b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi CheckEmptyMarkStack(); 12072cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi TimingLogger::ScopedTiming split("Sweep", GetTimings()); 12082cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi for (const auto& space : GetHeap()->GetContinuousSpaces()) { 12092cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi if (space->IsContinuousMemMapAllocSpace()) { 12102cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi space::ContinuousMemMapAllocSpace* alloc_space = space->AsContinuousMemMapAllocSpace(); 1211763a31ed7a2bfad22a9cb07f5301a71c0f97ca49Mathieu Chartier if (space == region_space_ || immune_spaces_.ContainsSpace(space)) { 12122cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi continue; 12132cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi } 12142cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi TimingLogger::ScopedTiming split2( 12152cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi alloc_space->IsZygoteSpace() ? "SweepZygoteSpace" : "SweepAllocSpace", GetTimings()); 12162cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi RecordFree(alloc_space->Sweep(swap_bitmaps)); 12172cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi } 12182cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi } 12192cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi SweepLargeObjects(swap_bitmaps); 12202cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi} 12212cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi 12222cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchivoid ConcurrentCopying::SweepLargeObjects(bool swap_bitmaps) { 12232cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi TimingLogger::ScopedTiming split("SweepLargeObjects", GetTimings()); 12242cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi RecordFreeLOS(heap_->GetLargeObjectsSpace()->Sweep(swap_bitmaps)); 12252cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi} 12262cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi 12278016bdee7ca1a066221a5d2fe5e60890de950a5bMathieu Chartierclass ConcurrentCopying::ClearBlackPtrsVisitor { 12282cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi public: 12298016bdee7ca1a066221a5d2fe5e60890de950a5bMathieu Chartier explicit ClearBlackPtrsVisitor(ConcurrentCopying* cc) : collector_(cc) {} 123090443477f9a0061581c420775ce3b7eeae7468bcMathieu Chartier void operator()(mirror::Object* obj) const SHARED_REQUIRES(Locks::mutator_lock_) 123190443477f9a0061581c420775ce3b7eeae7468bcMathieu Chartier SHARED_REQUIRES(Locks::heap_bitmap_lock_) { 12322cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi DCHECK(obj != nullptr); 1233d25f84250700c35f006d5a1d295231af174c3734Hiroshi Yamauchi DCHECK(collector_->heap_->GetMarkBitmap()->Test(obj)) << obj; 1234d25f84250700c35f006d5a1d295231af174c3734Hiroshi Yamauchi DCHECK_EQ(obj->GetReadBarrierPointer(), ReadBarrier::BlackPtr()) << obj; 123560f63f53c01cb38ca18a815603282e802a6cf918Hiroshi Yamauchi obj->AtomicSetReadBarrierPointer(ReadBarrier::BlackPtr(), ReadBarrier::WhitePtr()); 1236d25f84250700c35f006d5a1d295231af174c3734Hiroshi Yamauchi DCHECK_EQ(obj->GetReadBarrierPointer(), ReadBarrier::WhitePtr()) << obj; 12372cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi } 12382cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi 12392cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi private: 12402cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi ConcurrentCopying* const collector_; 12412cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi}; 12422cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi 12432cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi// Clear the black ptrs in non-moving objects back to white. 12442cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchivoid ConcurrentCopying::ClearBlackPtrs() { 12452cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi CHECK(kUseBakerReadBarrier); 12462cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi TimingLogger::ScopedTiming split("ClearBlackPtrs", GetTimings()); 12478016bdee7ca1a066221a5d2fe5e60890de950a5bMathieu Chartier ClearBlackPtrsVisitor visitor(this); 12482cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi for (auto& space : heap_->GetContinuousSpaces()) { 12492cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi if (space == region_space_) { 12502cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi continue; 12512cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi } 12522cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi accounting::ContinuousSpaceBitmap* mark_bitmap = space->GetMarkBitmap(); 12532cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi if (kVerboseMode) { 12542cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi LOG(INFO) << "ClearBlackPtrs: " << *space << " bitmap: " << *mark_bitmap; 12552cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi } 12562cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi mark_bitmap->VisitMarkedRange(reinterpret_cast<uintptr_t>(space->Begin()), 12572cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi reinterpret_cast<uintptr_t>(space->Limit()), 12582cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi visitor); 12592cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi } 12602cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi space::LargeObjectSpace* large_object_space = heap_->GetLargeObjectsSpace(); 12612cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi large_object_space->GetMarkBitmap()->VisitMarkedRange( 12622cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi reinterpret_cast<uintptr_t>(large_object_space->Begin()), 12632cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi reinterpret_cast<uintptr_t>(large_object_space->End()), 12642cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi visitor); 12652cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi // Objects on the allocation stack? 12662cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi if (ReadBarrier::kEnableReadBarrierInvariantChecks || kIsDebugBuild) { 12672cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi size_t count = GetAllocationStack()->Size(); 1268cb535da36915f9d10bec3880b46f1de1f7a69f22Mathieu Chartier auto* it = GetAllocationStack()->Begin(); 1269cb535da36915f9d10bec3880b46f1de1f7a69f22Mathieu Chartier auto* end = GetAllocationStack()->End(); 12702cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi for (size_t i = 0; i < count; ++i, ++it) { 1271cb535da36915f9d10bec3880b46f1de1f7a69f22Mathieu Chartier CHECK_LT(it, end); 1272cb535da36915f9d10bec3880b46f1de1f7a69f22Mathieu Chartier mirror::Object* obj = it->AsMirrorPtr(); 12732cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi if (obj != nullptr) { 12742cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi // Must have been cleared above. 1275cb535da36915f9d10bec3880b46f1de1f7a69f22Mathieu Chartier CHECK_EQ(obj->GetReadBarrierPointer(), ReadBarrier::WhitePtr()) << obj; 12762cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi } 12772cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi } 12782cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi } 12792cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi} 12802cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi 12812cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchivoid ConcurrentCopying::ReclaimPhase() { 12822cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi TimingLogger::ScopedTiming split("ReclaimPhase", GetTimings()); 12832cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi if (kVerboseMode) { 12842cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi LOG(INFO) << "GC ReclaimPhase"; 12852cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi } 12862cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi Thread* self = Thread::Current(); 12872cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi 12882cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi { 12892cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi // Double-check that the mark stack is empty. 12902cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi // Note: need to set this after VerifyNoFromSpaceRef(). 12912cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi is_asserting_to_space_invariant_ = false; 12922cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi QuasiAtomic::ThreadFenceForConstructor(); 12932cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi if (kVerboseMode) { 12942cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi LOG(INFO) << "Issue an empty check point. "; 12952cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi } 12962cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi IssueEmptyCheckpoint(); 12972cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi // Disable the check. 12980b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi is_mark_stack_push_disallowed_.StoreSequentiallyConsistent(0); 12990b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi CheckEmptyMarkStack(); 13002cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi } 13012cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi 13022cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi { 13032cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi // Record freed objects. 13042cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi TimingLogger::ScopedTiming split2("RecordFree", GetTimings()); 13052cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi // Don't include thread-locals that are in the to-space. 13062cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi uint64_t from_bytes = region_space_->GetBytesAllocatedInFromSpace(); 13072cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi uint64_t from_objects = region_space_->GetObjectsAllocatedInFromSpace(); 13082cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi uint64_t unevac_from_bytes = region_space_->GetBytesAllocatedInUnevacFromSpace(); 13092cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi uint64_t unevac_from_objects = region_space_->GetObjectsAllocatedInUnevacFromSpace(); 13102cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi uint64_t to_bytes = bytes_moved_.LoadSequentiallyConsistent(); 13112cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi uint64_t to_objects = objects_moved_.LoadSequentiallyConsistent(); 13122cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi if (kEnableFromSpaceAccountingCheck) { 13132cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi CHECK_EQ(from_space_num_objects_at_first_pause_, from_objects + unevac_from_objects); 13142cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi CHECK_EQ(from_space_num_bytes_at_first_pause_, from_bytes + unevac_from_bytes); 13152cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi } 13162cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi CHECK_LE(to_objects, from_objects); 13172cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi CHECK_LE(to_bytes, from_bytes); 13182cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi int64_t freed_bytes = from_bytes - to_bytes; 13192cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi int64_t freed_objects = from_objects - to_objects; 13202cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi if (kVerboseMode) { 13212cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi LOG(INFO) << "RecordFree:" 13222cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi << " from_bytes=" << from_bytes << " from_objects=" << from_objects 13232cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi << " unevac_from_bytes=" << unevac_from_bytes << " unevac_from_objects=" << unevac_from_objects 13242cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi << " to_bytes=" << to_bytes << " to_objects=" << to_objects 13252cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi << " freed_bytes=" << freed_bytes << " freed_objects=" << freed_objects 13262cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi << " from_space size=" << region_space_->FromSpaceSize() 13272cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi << " unevac_from_space size=" << region_space_->UnevacFromSpaceSize() 13282cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi << " to_space size=" << region_space_->ToSpaceSize(); 13292cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi LOG(INFO) << "(before) num_bytes_allocated=" << heap_->num_bytes_allocated_.LoadSequentiallyConsistent(); 13302cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi } 13312cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi RecordFree(ObjectBytePair(freed_objects, freed_bytes)); 13322cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi if (kVerboseMode) { 13332cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi LOG(INFO) << "(after) num_bytes_allocated=" << heap_->num_bytes_allocated_.LoadSequentiallyConsistent(); 13342cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi } 13352cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi } 13362cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi 13372cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi { 13382cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi TimingLogger::ScopedTiming split3("ComputeUnevacFromSpaceLiveRatio", GetTimings()); 13392cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi ComputeUnevacFromSpaceLiveRatio(); 13402cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi } 13412cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi 13422cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi { 13432cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi TimingLogger::ScopedTiming split4("ClearFromSpace", GetTimings()); 13442cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi region_space_->ClearFromSpace(); 13452cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi } 13462cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi 13472cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi { 13482cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi WriterMutexLock mu(self, *Locks::heap_bitmap_lock_); 13492cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi if (kUseBakerReadBarrier) { 13502cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi ClearBlackPtrs(); 13512cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi } 13522cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi Sweep(false); 13532cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi SwapBitmaps(); 13542cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi heap_->UnBindBitmaps(); 13552cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi 13562cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi // Remove bitmaps for the immune spaces. 13572cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi while (!cc_bitmaps_.empty()) { 13582cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi accounting::ContinuousSpaceBitmap* cc_bitmap = cc_bitmaps_.back(); 13592cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi cc_heap_bitmap_->RemoveContinuousSpaceBitmap(cc_bitmap); 13602cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi delete cc_bitmap; 13612cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi cc_bitmaps_.pop_back(); 13622cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi } 13632cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi region_space_bitmap_ = nullptr; 13642cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi } 13652cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi 13660b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi CheckEmptyMarkStack(); 13670b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi 13682cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi if (kVerboseMode) { 13692cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi LOG(INFO) << "GC end of ReclaimPhase"; 13702cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi } 13712cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi} 13722cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi 13738016bdee7ca1a066221a5d2fe5e60890de950a5bMathieu Chartierclass ConcurrentCopying::ComputeUnevacFromSpaceLiveRatioVisitor { 13742cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi public: 13758016bdee7ca1a066221a5d2fe5e60890de950a5bMathieu Chartier explicit ComputeUnevacFromSpaceLiveRatioVisitor(ConcurrentCopying* cc) 13762cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi : collector_(cc) {} 137790443477f9a0061581c420775ce3b7eeae7468bcMathieu Chartier void operator()(mirror::Object* ref) const SHARED_REQUIRES(Locks::mutator_lock_) 137890443477f9a0061581c420775ce3b7eeae7468bcMathieu Chartier SHARED_REQUIRES(Locks::heap_bitmap_lock_) { 13792cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi DCHECK(ref != nullptr); 1380d25f84250700c35f006d5a1d295231af174c3734Hiroshi Yamauchi DCHECK(collector_->region_space_bitmap_->Test(ref)) << ref; 1381d25f84250700c35f006d5a1d295231af174c3734Hiroshi Yamauchi DCHECK(collector_->region_space_->IsInUnevacFromSpace(ref)) << ref; 13822cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi if (kUseBakerReadBarrier) { 1383d25f84250700c35f006d5a1d295231af174c3734Hiroshi Yamauchi DCHECK_EQ(ref->GetReadBarrierPointer(), ReadBarrier::BlackPtr()) << ref; 13842cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi // Clear the black ptr. 138560f63f53c01cb38ca18a815603282e802a6cf918Hiroshi Yamauchi ref->AtomicSetReadBarrierPointer(ReadBarrier::BlackPtr(), ReadBarrier::WhitePtr()); 138660f63f53c01cb38ca18a815603282e802a6cf918Hiroshi Yamauchi DCHECK_EQ(ref->GetReadBarrierPointer(), ReadBarrier::WhitePtr()) << ref; 13872cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi } 13882cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi size_t obj_size = ref->SizeOf(); 13892cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi size_t alloc_size = RoundUp(obj_size, space::RegionSpace::kAlignment); 13902cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi collector_->region_space_->AddLiveBytes(ref, alloc_size); 13912cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi } 13922cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi 13932cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi private: 139497509954404d031594b2ecbda607314d169d512eMathieu Chartier ConcurrentCopying* const collector_; 13952cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi}; 13962cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi 13972cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi// Compute how much live objects are left in regions. 13982cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchivoid ConcurrentCopying::ComputeUnevacFromSpaceLiveRatio() { 13992cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi region_space_->AssertAllRegionLiveBytesZeroOrCleared(); 14008016bdee7ca1a066221a5d2fe5e60890de950a5bMathieu Chartier ComputeUnevacFromSpaceLiveRatioVisitor visitor(this); 14012cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi region_space_bitmap_->VisitMarkedRange(reinterpret_cast<uintptr_t>(region_space_->Begin()), 14022cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi reinterpret_cast<uintptr_t>(region_space_->Limit()), 14032cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi visitor); 14042cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi} 14052cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi 14062cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi// Assert the to-space invariant. 14072cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchivoid ConcurrentCopying::AssertToSpaceInvariant(mirror::Object* obj, MemberOffset offset, 14082cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi mirror::Object* ref) { 14092cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi CHECK(heap_->collector_type_ == kCollectorTypeCC) << static_cast<size_t>(heap_->collector_type_); 14102cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi if (is_asserting_to_space_invariant_) { 14112cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi if (region_space_->IsInToSpace(ref)) { 14122cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi // OK. 14132cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi return; 14142cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi } else if (region_space_->IsInUnevacFromSpace(ref)) { 14152cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi CHECK(region_space_bitmap_->Test(ref)) << ref; 14162cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi } else if (region_space_->IsInFromSpace(ref)) { 14172cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi // Not OK. Do extra logging. 14182cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi if (obj != nullptr) { 14193f64f25151780fdea3511be62b4fe50775f86541Hiroshi Yamauchi LogFromSpaceRefHolder(obj, offset); 14202cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi } 14213f64f25151780fdea3511be62b4fe50775f86541Hiroshi Yamauchi ref->GetLockWord(false).Dump(LOG(INTERNAL_FATAL)); 14222cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi CHECK(false) << "Found from-space ref " << ref << " " << PrettyTypeOf(ref); 14232cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi } else { 14243f64f25151780fdea3511be62b4fe50775f86541Hiroshi Yamauchi AssertToSpaceInvariantInNonMovingSpace(obj, ref); 14253f64f25151780fdea3511be62b4fe50775f86541Hiroshi Yamauchi } 14263f64f25151780fdea3511be62b4fe50775f86541Hiroshi Yamauchi } 14273f64f25151780fdea3511be62b4fe50775f86541Hiroshi Yamauchi} 14283f64f25151780fdea3511be62b4fe50775f86541Hiroshi Yamauchi 14293f64f25151780fdea3511be62b4fe50775f86541Hiroshi Yamauchiclass RootPrinter { 14303f64f25151780fdea3511be62b4fe50775f86541Hiroshi Yamauchi public: 14313f64f25151780fdea3511be62b4fe50775f86541Hiroshi Yamauchi RootPrinter() { } 14323f64f25151780fdea3511be62b4fe50775f86541Hiroshi Yamauchi 14333f64f25151780fdea3511be62b4fe50775f86541Hiroshi Yamauchi template <class MirrorType> 14343f64f25151780fdea3511be62b4fe50775f86541Hiroshi Yamauchi ALWAYS_INLINE void VisitRootIfNonNull(mirror::CompressedReference<MirrorType>* root) 143590443477f9a0061581c420775ce3b7eeae7468bcMathieu Chartier SHARED_REQUIRES(Locks::mutator_lock_) { 14363f64f25151780fdea3511be62b4fe50775f86541Hiroshi Yamauchi if (!root->IsNull()) { 14373f64f25151780fdea3511be62b4fe50775f86541Hiroshi Yamauchi VisitRoot(root); 14383f64f25151780fdea3511be62b4fe50775f86541Hiroshi Yamauchi } 14393f64f25151780fdea3511be62b4fe50775f86541Hiroshi Yamauchi } 14403f64f25151780fdea3511be62b4fe50775f86541Hiroshi Yamauchi 14413f64f25151780fdea3511be62b4fe50775f86541Hiroshi Yamauchi template <class MirrorType> 14423f64f25151780fdea3511be62b4fe50775f86541Hiroshi Yamauchi void VisitRoot(mirror::Object** root) 144390443477f9a0061581c420775ce3b7eeae7468bcMathieu Chartier SHARED_REQUIRES(Locks::mutator_lock_) { 14443f64f25151780fdea3511be62b4fe50775f86541Hiroshi Yamauchi LOG(INTERNAL_FATAL) << "root=" << root << " ref=" << *root; 14453f64f25151780fdea3511be62b4fe50775f86541Hiroshi Yamauchi } 14463f64f25151780fdea3511be62b4fe50775f86541Hiroshi Yamauchi 14473f64f25151780fdea3511be62b4fe50775f86541Hiroshi Yamauchi template <class MirrorType> 14483f64f25151780fdea3511be62b4fe50775f86541Hiroshi Yamauchi void VisitRoot(mirror::CompressedReference<MirrorType>* root) 144990443477f9a0061581c420775ce3b7eeae7468bcMathieu Chartier SHARED_REQUIRES(Locks::mutator_lock_) { 14503f64f25151780fdea3511be62b4fe50775f86541Hiroshi Yamauchi LOG(INTERNAL_FATAL) << "root=" << root << " ref=" << root->AsMirrorPtr(); 14513f64f25151780fdea3511be62b4fe50775f86541Hiroshi Yamauchi } 14523f64f25151780fdea3511be62b4fe50775f86541Hiroshi Yamauchi}; 14533f64f25151780fdea3511be62b4fe50775f86541Hiroshi Yamauchi 14543f64f25151780fdea3511be62b4fe50775f86541Hiroshi Yamauchivoid ConcurrentCopying::AssertToSpaceInvariant(GcRootSource* gc_root_source, 14553f64f25151780fdea3511be62b4fe50775f86541Hiroshi Yamauchi mirror::Object* ref) { 14563f64f25151780fdea3511be62b4fe50775f86541Hiroshi Yamauchi CHECK(heap_->collector_type_ == kCollectorTypeCC) << static_cast<size_t>(heap_->collector_type_); 14573f64f25151780fdea3511be62b4fe50775f86541Hiroshi Yamauchi if (is_asserting_to_space_invariant_) { 14583f64f25151780fdea3511be62b4fe50775f86541Hiroshi Yamauchi if (region_space_->IsInToSpace(ref)) { 14593f64f25151780fdea3511be62b4fe50775f86541Hiroshi Yamauchi // OK. 14603f64f25151780fdea3511be62b4fe50775f86541Hiroshi Yamauchi return; 14613f64f25151780fdea3511be62b4fe50775f86541Hiroshi Yamauchi } else if (region_space_->IsInUnevacFromSpace(ref)) { 14623f64f25151780fdea3511be62b4fe50775f86541Hiroshi Yamauchi CHECK(region_space_bitmap_->Test(ref)) << ref; 14633f64f25151780fdea3511be62b4fe50775f86541Hiroshi Yamauchi } else if (region_space_->IsInFromSpace(ref)) { 14643f64f25151780fdea3511be62b4fe50775f86541Hiroshi Yamauchi // Not OK. Do extra logging. 14653f64f25151780fdea3511be62b4fe50775f86541Hiroshi Yamauchi if (gc_root_source == nullptr) { 14663f64f25151780fdea3511be62b4fe50775f86541Hiroshi Yamauchi // No info. 14673f64f25151780fdea3511be62b4fe50775f86541Hiroshi Yamauchi } else if (gc_root_source->HasArtField()) { 14683f64f25151780fdea3511be62b4fe50775f86541Hiroshi Yamauchi ArtField* field = gc_root_source->GetArtField(); 14693f64f25151780fdea3511be62b4fe50775f86541Hiroshi Yamauchi LOG(INTERNAL_FATAL) << "gc root in field " << field << " " << PrettyField(field); 14703f64f25151780fdea3511be62b4fe50775f86541Hiroshi Yamauchi RootPrinter root_printer; 14713f64f25151780fdea3511be62b4fe50775f86541Hiroshi Yamauchi field->VisitRoots(root_printer); 14723f64f25151780fdea3511be62b4fe50775f86541Hiroshi Yamauchi } else if (gc_root_source->HasArtMethod()) { 14733f64f25151780fdea3511be62b4fe50775f86541Hiroshi Yamauchi ArtMethod* method = gc_root_source->GetArtMethod(); 14743f64f25151780fdea3511be62b4fe50775f86541Hiroshi Yamauchi LOG(INTERNAL_FATAL) << "gc root in method " << method << " " << PrettyMethod(method); 14753f64f25151780fdea3511be62b4fe50775f86541Hiroshi Yamauchi RootPrinter root_printer; 14761147b9bd68323c753ed1a0b6106b205fd640c820Mathieu Chartier method->VisitRoots(root_printer, sizeof(void*)); 14773f64f25151780fdea3511be62b4fe50775f86541Hiroshi Yamauchi } 14783f64f25151780fdea3511be62b4fe50775f86541Hiroshi Yamauchi ref->GetLockWord(false).Dump(LOG(INTERNAL_FATAL)); 14793f64f25151780fdea3511be62b4fe50775f86541Hiroshi Yamauchi region_space_->DumpNonFreeRegions(LOG(INTERNAL_FATAL)); 14803f64f25151780fdea3511be62b4fe50775f86541Hiroshi Yamauchi PrintFileToLog("/proc/self/maps", LogSeverity::INTERNAL_FATAL); 14813f64f25151780fdea3511be62b4fe50775f86541Hiroshi Yamauchi MemMap::DumpMaps(LOG(INTERNAL_FATAL), true); 14823f64f25151780fdea3511be62b4fe50775f86541Hiroshi Yamauchi CHECK(false) << "Found from-space ref " << ref << " " << PrettyTypeOf(ref); 14833f64f25151780fdea3511be62b4fe50775f86541Hiroshi Yamauchi } else { 14843f64f25151780fdea3511be62b4fe50775f86541Hiroshi Yamauchi AssertToSpaceInvariantInNonMovingSpace(nullptr, ref); 14853f64f25151780fdea3511be62b4fe50775f86541Hiroshi Yamauchi } 14863f64f25151780fdea3511be62b4fe50775f86541Hiroshi Yamauchi } 14873f64f25151780fdea3511be62b4fe50775f86541Hiroshi Yamauchi} 14883f64f25151780fdea3511be62b4fe50775f86541Hiroshi Yamauchi 14893f64f25151780fdea3511be62b4fe50775f86541Hiroshi Yamauchivoid ConcurrentCopying::LogFromSpaceRefHolder(mirror::Object* obj, MemberOffset offset) { 14903f64f25151780fdea3511be62b4fe50775f86541Hiroshi Yamauchi if (kUseBakerReadBarrier) { 14913f64f25151780fdea3511be62b4fe50775f86541Hiroshi Yamauchi LOG(INFO) << "holder=" << obj << " " << PrettyTypeOf(obj) 14923f64f25151780fdea3511be62b4fe50775f86541Hiroshi Yamauchi << " holder rb_ptr=" << obj->GetReadBarrierPointer(); 14933f64f25151780fdea3511be62b4fe50775f86541Hiroshi Yamauchi } else { 14943f64f25151780fdea3511be62b4fe50775f86541Hiroshi Yamauchi LOG(INFO) << "holder=" << obj << " " << PrettyTypeOf(obj); 14953f64f25151780fdea3511be62b4fe50775f86541Hiroshi Yamauchi } 14963f64f25151780fdea3511be62b4fe50775f86541Hiroshi Yamauchi if (region_space_->IsInFromSpace(obj)) { 14973f64f25151780fdea3511be62b4fe50775f86541Hiroshi Yamauchi LOG(INFO) << "holder is in the from-space."; 14983f64f25151780fdea3511be62b4fe50775f86541Hiroshi Yamauchi } else if (region_space_->IsInToSpace(obj)) { 14993f64f25151780fdea3511be62b4fe50775f86541Hiroshi Yamauchi LOG(INFO) << "holder is in the to-space."; 15003f64f25151780fdea3511be62b4fe50775f86541Hiroshi Yamauchi } else if (region_space_->IsInUnevacFromSpace(obj)) { 15013f64f25151780fdea3511be62b4fe50775f86541Hiroshi Yamauchi LOG(INFO) << "holder is in the unevac from-space."; 15023f64f25151780fdea3511be62b4fe50775f86541Hiroshi Yamauchi if (region_space_bitmap_->Test(obj)) { 15033f64f25151780fdea3511be62b4fe50775f86541Hiroshi Yamauchi LOG(INFO) << "holder is marked in the region space bitmap."; 15043f64f25151780fdea3511be62b4fe50775f86541Hiroshi Yamauchi } else { 15053f64f25151780fdea3511be62b4fe50775f86541Hiroshi Yamauchi LOG(INFO) << "holder is not marked in the region space bitmap."; 15063f64f25151780fdea3511be62b4fe50775f86541Hiroshi Yamauchi } 15073f64f25151780fdea3511be62b4fe50775f86541Hiroshi Yamauchi } else { 15083f64f25151780fdea3511be62b4fe50775f86541Hiroshi Yamauchi // In a non-moving space. 1509763a31ed7a2bfad22a9cb07f5301a71c0f97ca49Mathieu Chartier if (immune_spaces_.ContainsObject(obj)) { 1510763a31ed7a2bfad22a9cb07f5301a71c0f97ca49Mathieu Chartier LOG(INFO) << "holder is in an immune image or the zygote space."; 15113f64f25151780fdea3511be62b4fe50775f86541Hiroshi Yamauchi accounting::ContinuousSpaceBitmap* cc_bitmap = 15123f64f25151780fdea3511be62b4fe50775f86541Hiroshi Yamauchi cc_heap_bitmap_->GetContinuousSpaceBitmap(obj); 15133f64f25151780fdea3511be62b4fe50775f86541Hiroshi Yamauchi CHECK(cc_bitmap != nullptr) 15143f64f25151780fdea3511be62b4fe50775f86541Hiroshi Yamauchi << "An immune space object must have a bitmap."; 15153f64f25151780fdea3511be62b4fe50775f86541Hiroshi Yamauchi if (cc_bitmap->Test(obj)) { 15163f64f25151780fdea3511be62b4fe50775f86541Hiroshi Yamauchi LOG(INFO) << "holder is marked in the bit map."; 15172cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi } else { 15183f64f25151780fdea3511be62b4fe50775f86541Hiroshi Yamauchi LOG(INFO) << "holder is NOT marked in the bit map."; 15193f64f25151780fdea3511be62b4fe50775f86541Hiroshi Yamauchi } 15203f64f25151780fdea3511be62b4fe50775f86541Hiroshi Yamauchi } else { 1521763a31ed7a2bfad22a9cb07f5301a71c0f97ca49Mathieu Chartier LOG(INFO) << "holder is in a non-immune, non-moving (or main) space."; 15223f64f25151780fdea3511be62b4fe50775f86541Hiroshi Yamauchi accounting::ContinuousSpaceBitmap* mark_bitmap = 15233f64f25151780fdea3511be62b4fe50775f86541Hiroshi Yamauchi heap_mark_bitmap_->GetContinuousSpaceBitmap(obj); 15243f64f25151780fdea3511be62b4fe50775f86541Hiroshi Yamauchi accounting::LargeObjectBitmap* los_bitmap = 15253f64f25151780fdea3511be62b4fe50775f86541Hiroshi Yamauchi heap_mark_bitmap_->GetLargeObjectBitmap(obj); 15263f64f25151780fdea3511be62b4fe50775f86541Hiroshi Yamauchi CHECK(los_bitmap != nullptr) << "LOS bitmap covers the entire address range"; 15273f64f25151780fdea3511be62b4fe50775f86541Hiroshi Yamauchi bool is_los = mark_bitmap == nullptr; 15283f64f25151780fdea3511be62b4fe50775f86541Hiroshi Yamauchi if (!is_los && mark_bitmap->Test(obj)) { 15293f64f25151780fdea3511be62b4fe50775f86541Hiroshi Yamauchi LOG(INFO) << "holder is marked in the mark bit map."; 15303f64f25151780fdea3511be62b4fe50775f86541Hiroshi Yamauchi } else if (is_los && los_bitmap->Test(obj)) { 15313f64f25151780fdea3511be62b4fe50775f86541Hiroshi Yamauchi LOG(INFO) << "holder is marked in the los bit map."; 15323f64f25151780fdea3511be62b4fe50775f86541Hiroshi Yamauchi } else { 15333f64f25151780fdea3511be62b4fe50775f86541Hiroshi Yamauchi // If ref is on the allocation stack, then it is considered 15343f64f25151780fdea3511be62b4fe50775f86541Hiroshi Yamauchi // mark/alive (but not necessarily on the live stack.) 15353f64f25151780fdea3511be62b4fe50775f86541Hiroshi Yamauchi if (IsOnAllocStack(obj)) { 15363f64f25151780fdea3511be62b4fe50775f86541Hiroshi Yamauchi LOG(INFO) << "holder is on the alloc stack."; 15372cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi } else { 15383f64f25151780fdea3511be62b4fe50775f86541Hiroshi Yamauchi LOG(INFO) << "holder is not marked or on the alloc stack."; 15392cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi } 15402cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi } 15412cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi } 15422cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi } 15433f64f25151780fdea3511be62b4fe50775f86541Hiroshi Yamauchi LOG(INFO) << "offset=" << offset.SizeValue(); 15443f64f25151780fdea3511be62b4fe50775f86541Hiroshi Yamauchi} 15453f64f25151780fdea3511be62b4fe50775f86541Hiroshi Yamauchi 15463f64f25151780fdea3511be62b4fe50775f86541Hiroshi Yamauchivoid ConcurrentCopying::AssertToSpaceInvariantInNonMovingSpace(mirror::Object* obj, 15473f64f25151780fdea3511be62b4fe50775f86541Hiroshi Yamauchi mirror::Object* ref) { 15483f64f25151780fdea3511be62b4fe50775f86541Hiroshi Yamauchi // In a non-moving spaces. Check that the ref is marked. 1549763a31ed7a2bfad22a9cb07f5301a71c0f97ca49Mathieu Chartier if (immune_spaces_.ContainsObject(ref)) { 15503f64f25151780fdea3511be62b4fe50775f86541Hiroshi Yamauchi accounting::ContinuousSpaceBitmap* cc_bitmap = 15513f64f25151780fdea3511be62b4fe50775f86541Hiroshi Yamauchi cc_heap_bitmap_->GetContinuousSpaceBitmap(ref); 15523f64f25151780fdea3511be62b4fe50775f86541Hiroshi Yamauchi CHECK(cc_bitmap != nullptr) 15533f64f25151780fdea3511be62b4fe50775f86541Hiroshi Yamauchi << "An immune space ref must have a bitmap. " << ref; 15543f64f25151780fdea3511be62b4fe50775f86541Hiroshi Yamauchi if (kUseBakerReadBarrier) { 15553f64f25151780fdea3511be62b4fe50775f86541Hiroshi Yamauchi CHECK(cc_bitmap->Test(ref)) 15563f64f25151780fdea3511be62b4fe50775f86541Hiroshi Yamauchi << "Unmarked immune space ref. obj=" << obj << " rb_ptr=" 15573f64f25151780fdea3511be62b4fe50775f86541Hiroshi Yamauchi << obj->GetReadBarrierPointer() << " ref=" << ref; 15583f64f25151780fdea3511be62b4fe50775f86541Hiroshi Yamauchi } else { 15593f64f25151780fdea3511be62b4fe50775f86541Hiroshi Yamauchi CHECK(cc_bitmap->Test(ref)) 15603f64f25151780fdea3511be62b4fe50775f86541Hiroshi Yamauchi << "Unmarked immune space ref. obj=" << obj << " ref=" << ref; 15613f64f25151780fdea3511be62b4fe50775f86541Hiroshi Yamauchi } 15623f64f25151780fdea3511be62b4fe50775f86541Hiroshi Yamauchi } else { 15633f64f25151780fdea3511be62b4fe50775f86541Hiroshi Yamauchi accounting::ContinuousSpaceBitmap* mark_bitmap = 15643f64f25151780fdea3511be62b4fe50775f86541Hiroshi Yamauchi heap_mark_bitmap_->GetContinuousSpaceBitmap(ref); 15653f64f25151780fdea3511be62b4fe50775f86541Hiroshi Yamauchi accounting::LargeObjectBitmap* los_bitmap = 15663f64f25151780fdea3511be62b4fe50775f86541Hiroshi Yamauchi heap_mark_bitmap_->GetLargeObjectBitmap(ref); 15673f64f25151780fdea3511be62b4fe50775f86541Hiroshi Yamauchi CHECK(los_bitmap != nullptr) << "LOS bitmap covers the entire address range"; 15683f64f25151780fdea3511be62b4fe50775f86541Hiroshi Yamauchi bool is_los = mark_bitmap == nullptr; 15693f64f25151780fdea3511be62b4fe50775f86541Hiroshi Yamauchi if ((!is_los && mark_bitmap->Test(ref)) || 15703f64f25151780fdea3511be62b4fe50775f86541Hiroshi Yamauchi (is_los && los_bitmap->Test(ref))) { 15713f64f25151780fdea3511be62b4fe50775f86541Hiroshi Yamauchi // OK. 15723f64f25151780fdea3511be62b4fe50775f86541Hiroshi Yamauchi } else { 15733f64f25151780fdea3511be62b4fe50775f86541Hiroshi Yamauchi // If ref is on the allocation stack, then it may not be 15743f64f25151780fdea3511be62b4fe50775f86541Hiroshi Yamauchi // marked live, but considered marked/alive (but not 15753f64f25151780fdea3511be62b4fe50775f86541Hiroshi Yamauchi // necessarily on the live stack). 15763f64f25151780fdea3511be62b4fe50775f86541Hiroshi Yamauchi CHECK(IsOnAllocStack(ref)) << "Unmarked ref that's not on the allocation stack. " 15773f64f25151780fdea3511be62b4fe50775f86541Hiroshi Yamauchi << "obj=" << obj << " ref=" << ref; 15783f64f25151780fdea3511be62b4fe50775f86541Hiroshi Yamauchi } 15793f64f25151780fdea3511be62b4fe50775f86541Hiroshi Yamauchi } 15802cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi} 15812cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi 15822cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi// Used to scan ref fields of an object. 15838016bdee7ca1a066221a5d2fe5e60890de950a5bMathieu Chartierclass ConcurrentCopying::RefFieldsVisitor { 15842cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi public: 15858016bdee7ca1a066221a5d2fe5e60890de950a5bMathieu Chartier explicit RefFieldsVisitor(ConcurrentCopying* collector) 15862cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi : collector_(collector) {} 15872cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi 15882cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi void operator()(mirror::Object* obj, MemberOffset offset, bool /* is_static */) 158990443477f9a0061581c420775ce3b7eeae7468bcMathieu Chartier const ALWAYS_INLINE SHARED_REQUIRES(Locks::mutator_lock_) 159090443477f9a0061581c420775ce3b7eeae7468bcMathieu Chartier SHARED_REQUIRES(Locks::heap_bitmap_lock_) { 15912cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi collector_->Process(obj, offset); 15922cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi } 15932cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi 15942cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi void operator()(mirror::Class* klass, mirror::Reference* ref) const 159590443477f9a0061581c420775ce3b7eeae7468bcMathieu Chartier SHARED_REQUIRES(Locks::mutator_lock_) ALWAYS_INLINE { 15962cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi CHECK(klass->IsTypeOfReferenceClass()); 15972cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi collector_->DelayReferenceReferent(klass, ref); 15982cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi } 15992cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi 1600da7c650022a974be10e2f00fa07d5109e3d8826fMathieu Chartier void VisitRootIfNonNull(mirror::CompressedReference<mirror::Object>* root) const 1601723e6cee35671d2dd9aeb884dd11f6994307c01fHiroshi Yamauchi ALWAYS_INLINE 1602da7c650022a974be10e2f00fa07d5109e3d8826fMathieu Chartier SHARED_REQUIRES(Locks::mutator_lock_) { 1603da7c650022a974be10e2f00fa07d5109e3d8826fMathieu Chartier if (!root->IsNull()) { 1604da7c650022a974be10e2f00fa07d5109e3d8826fMathieu Chartier VisitRoot(root); 1605da7c650022a974be10e2f00fa07d5109e3d8826fMathieu Chartier } 1606da7c650022a974be10e2f00fa07d5109e3d8826fMathieu Chartier } 1607da7c650022a974be10e2f00fa07d5109e3d8826fMathieu Chartier 1608da7c650022a974be10e2f00fa07d5109e3d8826fMathieu Chartier void VisitRoot(mirror::CompressedReference<mirror::Object>* root) const 1609723e6cee35671d2dd9aeb884dd11f6994307c01fHiroshi Yamauchi ALWAYS_INLINE 1610da7c650022a974be10e2f00fa07d5109e3d8826fMathieu Chartier SHARED_REQUIRES(Locks::mutator_lock_) { 1611da7c650022a974be10e2f00fa07d5109e3d8826fMathieu Chartier collector_->MarkRoot(root); 1612da7c650022a974be10e2f00fa07d5109e3d8826fMathieu Chartier } 1613da7c650022a974be10e2f00fa07d5109e3d8826fMathieu Chartier 16142cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi private: 16152cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi ConcurrentCopying* const collector_; 16162cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi}; 16172cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi 16182cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi// Scan ref fields of an object. 1619723e6cee35671d2dd9aeb884dd11f6994307c01fHiroshi Yamauchiinline void ConcurrentCopying::Scan(mirror::Object* to_ref) { 16202cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi DCHECK(!region_space_->IsInFromSpace(to_ref)); 16218016bdee7ca1a066221a5d2fe5e60890de950a5bMathieu Chartier RefFieldsVisitor visitor(this); 16225496f69c0a4c2cc357a065f57b7f4ff5d9ad2fa9Hiroshi Yamauchi // Disable the read barrier for a performance reason. 16235496f69c0a4c2cc357a065f57b7f4ff5d9ad2fa9Hiroshi Yamauchi to_ref->VisitReferences</*kVisitNativeRoots*/true, kDefaultVerifyFlags, kWithoutReadBarrier>( 16245496f69c0a4c2cc357a065f57b7f4ff5d9ad2fa9Hiroshi Yamauchi visitor, visitor); 16252cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi} 16262cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi 16272cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi// Process a field. 16282cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchiinline void ConcurrentCopying::Process(mirror::Object* obj, MemberOffset offset) { 1629da7c650022a974be10e2f00fa07d5109e3d8826fMathieu Chartier mirror::Object* ref = obj->GetFieldObject< 1630da7c650022a974be10e2f00fa07d5109e3d8826fMathieu Chartier mirror::Object, kVerifyNone, kWithoutReadBarrier, false>(offset); 16312cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi mirror::Object* to_ref = Mark(ref); 16322cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi if (to_ref == ref) { 16332cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi return; 16342cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi } 16352cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi // This may fail if the mutator writes to the field at the same time. But it's ok. 16362cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi mirror::Object* expected_ref = ref; 16372cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi mirror::Object* new_ref = to_ref; 16382cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi do { 16392cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi if (expected_ref != 16402cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi obj->GetFieldObject<mirror::Object, kVerifyNone, kWithoutReadBarrier, false>(offset)) { 16412cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi // It was updated by the mutator. 16422cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi break; 16432cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi } 1644fed3e2fd4919b58f8e4f8cbc317ee101f3b9af49Hiroshi Yamauchi } while (!obj->CasFieldWeakRelaxedObjectWithoutWriteBarrier< 1645da7c650022a974be10e2f00fa07d5109e3d8826fMathieu Chartier false, false, kVerifyNone>(offset, expected_ref, new_ref)); 16462cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi} 16472cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi 1648bb87e0f1a52de656bc77cb01cb887e51a0e5198bMathieu Chartier// Process some roots. 1649723e6cee35671d2dd9aeb884dd11f6994307c01fHiroshi Yamauchiinline void ConcurrentCopying::VisitRoots( 1650bb87e0f1a52de656bc77cb01cb887e51a0e5198bMathieu Chartier mirror::Object*** roots, size_t count, const RootInfo& info ATTRIBUTE_UNUSED) { 1651bb87e0f1a52de656bc77cb01cb887e51a0e5198bMathieu Chartier for (size_t i = 0; i < count; ++i) { 1652bb87e0f1a52de656bc77cb01cb887e51a0e5198bMathieu Chartier mirror::Object** root = roots[i]; 1653bb87e0f1a52de656bc77cb01cb887e51a0e5198bMathieu Chartier mirror::Object* ref = *root; 1654bb87e0f1a52de656bc77cb01cb887e51a0e5198bMathieu Chartier mirror::Object* to_ref = Mark(ref); 1655bb87e0f1a52de656bc77cb01cb887e51a0e5198bMathieu Chartier if (to_ref == ref) { 16564809d0a8a5fca85a67dd0588ead5dfbd0f1acf96Mathieu Chartier continue; 1657bb87e0f1a52de656bc77cb01cb887e51a0e5198bMathieu Chartier } 1658bb87e0f1a52de656bc77cb01cb887e51a0e5198bMathieu Chartier Atomic<mirror::Object*>* addr = reinterpret_cast<Atomic<mirror::Object*>*>(root); 1659bb87e0f1a52de656bc77cb01cb887e51a0e5198bMathieu Chartier mirror::Object* expected_ref = ref; 1660bb87e0f1a52de656bc77cb01cb887e51a0e5198bMathieu Chartier mirror::Object* new_ref = to_ref; 1661bb87e0f1a52de656bc77cb01cb887e51a0e5198bMathieu Chartier do { 1662bb87e0f1a52de656bc77cb01cb887e51a0e5198bMathieu Chartier if (expected_ref != addr->LoadRelaxed()) { 1663bb87e0f1a52de656bc77cb01cb887e51a0e5198bMathieu Chartier // It was updated by the mutator. 1664bb87e0f1a52de656bc77cb01cb887e51a0e5198bMathieu Chartier break; 1665bb87e0f1a52de656bc77cb01cb887e51a0e5198bMathieu Chartier } 1666fed3e2fd4919b58f8e4f8cbc317ee101f3b9af49Hiroshi Yamauchi } while (!addr->CompareExchangeWeakRelaxed(expected_ref, new_ref)); 16672cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi } 1668bb87e0f1a52de656bc77cb01cb887e51a0e5198bMathieu Chartier} 1669bb87e0f1a52de656bc77cb01cb887e51a0e5198bMathieu Chartier 1670723e6cee35671d2dd9aeb884dd11f6994307c01fHiroshi Yamauchiinline void ConcurrentCopying::MarkRoot(mirror::CompressedReference<mirror::Object>* root) { 1671da7c650022a974be10e2f00fa07d5109e3d8826fMathieu Chartier DCHECK(!root->IsNull()); 1672da7c650022a974be10e2f00fa07d5109e3d8826fMathieu Chartier mirror::Object* const ref = root->AsMirrorPtr(); 1673da7c650022a974be10e2f00fa07d5109e3d8826fMathieu Chartier mirror::Object* to_ref = Mark(ref); 1674da7c650022a974be10e2f00fa07d5109e3d8826fMathieu Chartier if (to_ref != ref) { 1675bb87e0f1a52de656bc77cb01cb887e51a0e5198bMathieu Chartier auto* addr = reinterpret_cast<Atomic<mirror::CompressedReference<mirror::Object>>*>(root); 1676bb87e0f1a52de656bc77cb01cb887e51a0e5198bMathieu Chartier auto expected_ref = mirror::CompressedReference<mirror::Object>::FromMirrorPtr(ref); 1677bb87e0f1a52de656bc77cb01cb887e51a0e5198bMathieu Chartier auto new_ref = mirror::CompressedReference<mirror::Object>::FromMirrorPtr(to_ref); 1678da7c650022a974be10e2f00fa07d5109e3d8826fMathieu Chartier // If the cas fails, then it was updated by the mutator. 1679bb87e0f1a52de656bc77cb01cb887e51a0e5198bMathieu Chartier do { 1680bb87e0f1a52de656bc77cb01cb887e51a0e5198bMathieu Chartier if (ref != addr->LoadRelaxed().AsMirrorPtr()) { 1681bb87e0f1a52de656bc77cb01cb887e51a0e5198bMathieu Chartier // It was updated by the mutator. 1682bb87e0f1a52de656bc77cb01cb887e51a0e5198bMathieu Chartier break; 1683bb87e0f1a52de656bc77cb01cb887e51a0e5198bMathieu Chartier } 1684fed3e2fd4919b58f8e4f8cbc317ee101f3b9af49Hiroshi Yamauchi } while (!addr->CompareExchangeWeakRelaxed(expected_ref, new_ref)); 1685bb87e0f1a52de656bc77cb01cb887e51a0e5198bMathieu Chartier } 16862cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi} 16872cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi 1688723e6cee35671d2dd9aeb884dd11f6994307c01fHiroshi Yamauchiinline void ConcurrentCopying::VisitRoots( 1689da7c650022a974be10e2f00fa07d5109e3d8826fMathieu Chartier mirror::CompressedReference<mirror::Object>** roots, size_t count, 1690da7c650022a974be10e2f00fa07d5109e3d8826fMathieu Chartier const RootInfo& info ATTRIBUTE_UNUSED) { 1691da7c650022a974be10e2f00fa07d5109e3d8826fMathieu Chartier for (size_t i = 0; i < count; ++i) { 1692da7c650022a974be10e2f00fa07d5109e3d8826fMathieu Chartier mirror::CompressedReference<mirror::Object>* const root = roots[i]; 1693da7c650022a974be10e2f00fa07d5109e3d8826fMathieu Chartier if (!root->IsNull()) { 1694da7c650022a974be10e2f00fa07d5109e3d8826fMathieu Chartier MarkRoot(root); 1695da7c650022a974be10e2f00fa07d5109e3d8826fMathieu Chartier } 1696da7c650022a974be10e2f00fa07d5109e3d8826fMathieu Chartier } 1697da7c650022a974be10e2f00fa07d5109e3d8826fMathieu Chartier} 1698da7c650022a974be10e2f00fa07d5109e3d8826fMathieu Chartier 16992cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi// Fill the given memory block with a dummy object. Used to fill in a 17002cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi// copy of objects that was lost in race. 17012cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchivoid ConcurrentCopying::FillWithDummyObject(mirror::Object* dummy_obj, size_t byte_size) { 170214d90579f013b374638b599361970557ed4b3f09Roland Levillain CHECK_ALIGNED(byte_size, kObjectAlignment); 17032cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi memset(dummy_obj, 0, byte_size); 17042cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi mirror::Class* int_array_class = mirror::IntArray::GetArrayClass(); 17052cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi CHECK(int_array_class != nullptr); 17062cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi AssertToSpaceInvariant(nullptr, MemberOffset(0), int_array_class); 17072cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi size_t component_size = int_array_class->GetComponentSize(); 17082cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi CHECK_EQ(component_size, sizeof(int32_t)); 17092cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi size_t data_offset = mirror::Array::DataOffset(component_size).SizeValue(); 17102cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi if (data_offset > byte_size) { 17112cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi // An int array is too big. Use java.lang.Object. 17122cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi mirror::Class* java_lang_Object = WellKnownClasses::ToClass(WellKnownClasses::java_lang_Object); 17132cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi AssertToSpaceInvariant(nullptr, MemberOffset(0), java_lang_Object); 17142cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi CHECK_EQ(byte_size, java_lang_Object->GetObjectSize()); 17152cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi dummy_obj->SetClass(java_lang_Object); 17162cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi CHECK_EQ(byte_size, dummy_obj->SizeOf()); 17172cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi } else { 17182cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi // Use an int array. 17192cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi dummy_obj->SetClass(int_array_class); 17202cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi CHECK(dummy_obj->IsArrayInstance()); 17212cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi int32_t length = (byte_size - data_offset) / component_size; 17222cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi dummy_obj->AsArray()->SetLength(length); 17232cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi CHECK_EQ(dummy_obj->AsArray()->GetLength(), length) 17242cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi << "byte_size=" << byte_size << " length=" << length 17252cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi << " component_size=" << component_size << " data_offset=" << data_offset; 17262cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi CHECK_EQ(byte_size, dummy_obj->SizeOf()) 17272cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi << "byte_size=" << byte_size << " length=" << length 17282cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi << " component_size=" << component_size << " data_offset=" << data_offset; 17292cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi } 17302cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi} 17312cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi 17322cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi// Reuse the memory blocks that were copy of objects that were lost in race. 17332cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchimirror::Object* ConcurrentCopying::AllocateInSkippedBlock(size_t alloc_size) { 17342cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi // Try to reuse the blocks that were unused due to CAS failures. 173514d90579f013b374638b599361970557ed4b3f09Roland Levillain CHECK_ALIGNED(alloc_size, space::RegionSpace::kAlignment); 17362cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi Thread* self = Thread::Current(); 17372cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi size_t min_object_size = RoundUp(sizeof(mirror::Object), space::RegionSpace::kAlignment); 17382cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi MutexLock mu(self, skipped_blocks_lock_); 17392cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi auto it = skipped_blocks_map_.lower_bound(alloc_size); 17402cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi if (it == skipped_blocks_map_.end()) { 17412cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi // Not found. 17422cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi return nullptr; 17432cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi } 17442cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi { 17452cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi size_t byte_size = it->first; 17462cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi CHECK_GE(byte_size, alloc_size); 17472cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi if (byte_size > alloc_size && byte_size - alloc_size < min_object_size) { 17482cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi // If remainder would be too small for a dummy object, retry with a larger request size. 17492cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi it = skipped_blocks_map_.lower_bound(alloc_size + min_object_size); 17502cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi if (it == skipped_blocks_map_.end()) { 17512cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi // Not found. 17522cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi return nullptr; 17532cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi } 175414d90579f013b374638b599361970557ed4b3f09Roland Levillain CHECK_ALIGNED(it->first - alloc_size, space::RegionSpace::kAlignment); 17552cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi CHECK_GE(it->first - alloc_size, min_object_size) 17562cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi << "byte_size=" << byte_size << " it->first=" << it->first << " alloc_size=" << alloc_size; 17572cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi } 17582cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi } 17592cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi // Found a block. 17602cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi CHECK(it != skipped_blocks_map_.end()); 17612cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi size_t byte_size = it->first; 17622cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi uint8_t* addr = it->second; 17632cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi CHECK_GE(byte_size, alloc_size); 17642cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi CHECK(region_space_->IsInToSpace(reinterpret_cast<mirror::Object*>(addr))); 176514d90579f013b374638b599361970557ed4b3f09Roland Levillain CHECK_ALIGNED(byte_size, space::RegionSpace::kAlignment); 17662cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi if (kVerboseMode) { 17672cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi LOG(INFO) << "Reusing skipped bytes : " << reinterpret_cast<void*>(addr) << ", " << byte_size; 17682cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi } 17692cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi skipped_blocks_map_.erase(it); 17702cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi memset(addr, 0, byte_size); 17712cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi if (byte_size > alloc_size) { 17722cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi // Return the remainder to the map. 177314d90579f013b374638b599361970557ed4b3f09Roland Levillain CHECK_ALIGNED(byte_size - alloc_size, space::RegionSpace::kAlignment); 17742cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi CHECK_GE(byte_size - alloc_size, min_object_size); 17752cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi FillWithDummyObject(reinterpret_cast<mirror::Object*>(addr + alloc_size), 17762cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi byte_size - alloc_size); 17772cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi CHECK(region_space_->IsInToSpace(reinterpret_cast<mirror::Object*>(addr + alloc_size))); 17782cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi skipped_blocks_map_.insert(std::make_pair(byte_size - alloc_size, addr + alloc_size)); 17792cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi } 17802cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi return reinterpret_cast<mirror::Object*>(addr); 17812cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi} 17822cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi 17832cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchimirror::Object* ConcurrentCopying::Copy(mirror::Object* from_ref) { 17842cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi DCHECK(region_space_->IsInFromSpace(from_ref)); 17852cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi // No read barrier to avoid nested RB that might violate the to-space 17862cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi // invariant. Note that from_ref is a from space ref so the SizeOf() 17872cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi // call will access the from-space meta objects, but it's ok and necessary. 17882cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi size_t obj_size = from_ref->SizeOf<kDefaultVerifyFlags, kWithoutReadBarrier>(); 17892cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi size_t region_space_alloc_size = RoundUp(obj_size, space::RegionSpace::kAlignment); 17902cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi size_t region_space_bytes_allocated = 0U; 17912cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi size_t non_moving_space_bytes_allocated = 0U; 17922cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi size_t bytes_allocated = 0U; 17934460a84be92b5a94ecfb5c650aef4945ab849c93Hiroshi Yamauchi size_t dummy; 17942cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi mirror::Object* to_ref = region_space_->AllocNonvirtual<true>( 17954460a84be92b5a94ecfb5c650aef4945ab849c93Hiroshi Yamauchi region_space_alloc_size, ®ion_space_bytes_allocated, nullptr, &dummy); 17962cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi bytes_allocated = region_space_bytes_allocated; 17972cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi if (to_ref != nullptr) { 17982cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi DCHECK_EQ(region_space_alloc_size, region_space_bytes_allocated); 17992cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi } 18002cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi bool fall_back_to_non_moving = false; 18012cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi if (UNLIKELY(to_ref == nullptr)) { 18022cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi // Failed to allocate in the region space. Try the skipped blocks. 18032cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi to_ref = AllocateInSkippedBlock(region_space_alloc_size); 18042cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi if (to_ref != nullptr) { 18052cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi // Succeeded to allocate in a skipped block. 18062cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi if (heap_->use_tlab_) { 18072cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi // This is necessary for the tlab case as it's not accounted in the space. 18082cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi region_space_->RecordAlloc(to_ref); 18092cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi } 18102cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi bytes_allocated = region_space_alloc_size; 18112cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi } else { 18122cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi // Fall back to the non-moving space. 18132cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi fall_back_to_non_moving = true; 18142cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi if (kVerboseMode) { 18152cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi LOG(INFO) << "Out of memory in the to-space. Fall back to non-moving. skipped_bytes=" 18162cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi << to_space_bytes_skipped_.LoadSequentiallyConsistent() 18172cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi << " skipped_objects=" << to_space_objects_skipped_.LoadSequentiallyConsistent(); 18182cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi } 18192cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi fall_back_to_non_moving = true; 18202cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi to_ref = heap_->non_moving_space_->Alloc(Thread::Current(), obj_size, 18214460a84be92b5a94ecfb5c650aef4945ab849c93Hiroshi Yamauchi &non_moving_space_bytes_allocated, nullptr, &dummy); 18222cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi CHECK(to_ref != nullptr) << "Fall-back non-moving space allocation failed"; 18232cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi bytes_allocated = non_moving_space_bytes_allocated; 18242cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi // Mark it in the mark bitmap. 18252cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi accounting::ContinuousSpaceBitmap* mark_bitmap = 18262cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi heap_mark_bitmap_->GetContinuousSpaceBitmap(to_ref); 18272cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi CHECK(mark_bitmap != nullptr); 18282cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi CHECK(!mark_bitmap->AtomicTestAndSet(to_ref)); 18292cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi } 18302cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi } 18312cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi DCHECK(to_ref != nullptr); 18322cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi 18332cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi // Attempt to install the forward pointer. This is in a loop as the 18342cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi // lock word atomic write can fail. 18352cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi while (true) { 18362cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi // Copy the object. TODO: copy only the lockword in the second iteration and on? 18372cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi memcpy(to_ref, from_ref, obj_size); 18382cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi 18392cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi LockWord old_lock_word = to_ref->GetLockWord(false); 18402cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi 18412cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi if (old_lock_word.GetState() == LockWord::kForwardingAddress) { 18422cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi // Lost the race. Another thread (either GC or mutator) stored 18432cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi // the forwarding pointer first. Make the lost copy (to_ref) 18442cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi // look like a valid but dead (dummy) object and keep it for 18452cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi // future reuse. 18462cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi FillWithDummyObject(to_ref, bytes_allocated); 18472cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi if (!fall_back_to_non_moving) { 18482cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi DCHECK(region_space_->IsInToSpace(to_ref)); 18492cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi if (bytes_allocated > space::RegionSpace::kRegionSize) { 18502cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi // Free the large alloc. 18512cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi region_space_->FreeLarge(to_ref, bytes_allocated); 18522cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi } else { 18532cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi // Record the lost copy for later reuse. 18542cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi heap_->num_bytes_allocated_.FetchAndAddSequentiallyConsistent(bytes_allocated); 18552cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi to_space_bytes_skipped_.FetchAndAddSequentiallyConsistent(bytes_allocated); 18562cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi to_space_objects_skipped_.FetchAndAddSequentiallyConsistent(1); 18572cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi MutexLock mu(Thread::Current(), skipped_blocks_lock_); 18582cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi skipped_blocks_map_.insert(std::make_pair(bytes_allocated, 18592cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi reinterpret_cast<uint8_t*>(to_ref))); 18602cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi } 18612cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi } else { 18622cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi DCHECK(heap_->non_moving_space_->HasAddress(to_ref)); 18632cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi DCHECK_EQ(bytes_allocated, non_moving_space_bytes_allocated); 18642cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi // Free the non-moving-space chunk. 18652cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi accounting::ContinuousSpaceBitmap* mark_bitmap = 18662cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi heap_mark_bitmap_->GetContinuousSpaceBitmap(to_ref); 18672cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi CHECK(mark_bitmap != nullptr); 18682cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi CHECK(mark_bitmap->Clear(to_ref)); 18692cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi heap_->non_moving_space_->Free(Thread::Current(), to_ref); 18702cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi } 18712cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi 18722cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi // Get the winner's forward ptr. 18732cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi mirror::Object* lost_fwd_ptr = to_ref; 18742cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi to_ref = reinterpret_cast<mirror::Object*>(old_lock_word.ForwardingAddress()); 18752cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi CHECK(to_ref != nullptr); 18762cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi CHECK_NE(to_ref, lost_fwd_ptr); 18772cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi CHECK(region_space_->IsInToSpace(to_ref) || heap_->non_moving_space_->HasAddress(to_ref)); 18782cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi CHECK_NE(to_ref->GetLockWord(false).GetState(), LockWord::kForwardingAddress); 18792cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi return to_ref; 18802cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi } 18812cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi 188260f63f53c01cb38ca18a815603282e802a6cf918Hiroshi Yamauchi // Set the gray ptr. 188360f63f53c01cb38ca18a815603282e802a6cf918Hiroshi Yamauchi if (kUseBakerReadBarrier) { 188460f63f53c01cb38ca18a815603282e802a6cf918Hiroshi Yamauchi to_ref->SetReadBarrierPointer(ReadBarrier::GrayPtr()); 188560f63f53c01cb38ca18a815603282e802a6cf918Hiroshi Yamauchi } 188660f63f53c01cb38ca18a815603282e802a6cf918Hiroshi Yamauchi 18872cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi LockWord new_lock_word = LockWord::FromForwardingAddress(reinterpret_cast<size_t>(to_ref)); 18882cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi 18892cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi // Try to atomically write the fwd ptr. 18902cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi bool success = from_ref->CasLockWordWeakSequentiallyConsistent(old_lock_word, new_lock_word); 18912cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi if (LIKELY(success)) { 18922cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi // The CAS succeeded. 18932cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi objects_moved_.FetchAndAddSequentiallyConsistent(1); 18942cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi bytes_moved_.FetchAndAddSequentiallyConsistent(region_space_alloc_size); 18952cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi if (LIKELY(!fall_back_to_non_moving)) { 18962cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi DCHECK(region_space_->IsInToSpace(to_ref)); 18972cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi } else { 18982cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi DCHECK(heap_->non_moving_space_->HasAddress(to_ref)); 18992cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi DCHECK_EQ(bytes_allocated, non_moving_space_bytes_allocated); 19002cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi } 19012cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi if (kUseBakerReadBarrier) { 19022cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi DCHECK(to_ref->GetReadBarrierPointer() == ReadBarrier::GrayPtr()); 19032cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi } 19042cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi DCHECK(GetFwdPtr(from_ref) == to_ref); 19052cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi CHECK_NE(to_ref->GetLockWord(false).GetState(), LockWord::kForwardingAddress); 19060b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi PushOntoMarkStack(to_ref); 19072cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi return to_ref; 19082cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi } else { 19092cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi // The CAS failed. It may have lost the race or may have failed 19102cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi // due to monitor/hashcode ops. Either way, retry. 19112cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi } 19122cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi } 19132cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi} 19142cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi 19152cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchimirror::Object* ConcurrentCopying::IsMarked(mirror::Object* from_ref) { 19162cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi DCHECK(from_ref != nullptr); 1917d25f84250700c35f006d5a1d295231af174c3734Hiroshi Yamauchi space::RegionSpace::RegionType rtype = region_space_->GetRegionType(from_ref); 1918d25f84250700c35f006d5a1d295231af174c3734Hiroshi Yamauchi if (rtype == space::RegionSpace::RegionType::kRegionTypeToSpace) { 19192cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi // It's already marked. 19202cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi return from_ref; 19212cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi } 19222cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi mirror::Object* to_ref; 1923d25f84250700c35f006d5a1d295231af174c3734Hiroshi Yamauchi if (rtype == space::RegionSpace::RegionType::kRegionTypeFromSpace) { 19242cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi to_ref = GetFwdPtr(from_ref); 19252cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi DCHECK(to_ref == nullptr || region_space_->IsInToSpace(to_ref) || 19262cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi heap_->non_moving_space_->HasAddress(to_ref)) 19272cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi << "from_ref=" << from_ref << " to_ref=" << to_ref; 1928d25f84250700c35f006d5a1d295231af174c3734Hiroshi Yamauchi } else if (rtype == space::RegionSpace::RegionType::kRegionTypeUnevacFromSpace) { 19292cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi if (region_space_bitmap_->Test(from_ref)) { 19302cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi to_ref = from_ref; 19312cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi } else { 19322cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi to_ref = nullptr; 19332cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi } 19342cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi } else { 19352cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi // from_ref is in a non-moving space. 1936763a31ed7a2bfad22a9cb07f5301a71c0f97ca49Mathieu Chartier if (immune_spaces_.ContainsObject(from_ref)) { 19372cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi accounting::ContinuousSpaceBitmap* cc_bitmap = 19382cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi cc_heap_bitmap_->GetContinuousSpaceBitmap(from_ref); 19392cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi DCHECK(cc_bitmap != nullptr) 19402cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi << "An immune space object must have a bitmap"; 19412cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi if (kIsDebugBuild) { 19422cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi DCHECK(heap_mark_bitmap_->GetContinuousSpaceBitmap(from_ref)->Test(from_ref)) 19432cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi << "Immune space object must be already marked"; 19442cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi } 19452cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi if (cc_bitmap->Test(from_ref)) { 19462cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi // Already marked. 19472cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi to_ref = from_ref; 19482cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi } else { 19492cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi // Newly marked. 19502cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi to_ref = nullptr; 19512cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi } 19522cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi } else { 19532cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi // Non-immune non-moving space. Use the mark bitmap. 19542cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi accounting::ContinuousSpaceBitmap* mark_bitmap = 19552cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi heap_mark_bitmap_->GetContinuousSpaceBitmap(from_ref); 19562cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi accounting::LargeObjectBitmap* los_bitmap = 19572cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi heap_mark_bitmap_->GetLargeObjectBitmap(from_ref); 19582cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi CHECK(los_bitmap != nullptr) << "LOS bitmap covers the entire address range"; 19592cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi bool is_los = mark_bitmap == nullptr; 19602cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi if (!is_los && mark_bitmap->Test(from_ref)) { 19612cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi // Already marked. 19622cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi to_ref = from_ref; 19632cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi } else if (is_los && los_bitmap->Test(from_ref)) { 19642cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi // Already marked in LOS. 19652cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi to_ref = from_ref; 19662cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi } else { 19672cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi // Not marked. 19682cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi if (IsOnAllocStack(from_ref)) { 19692cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi // If on the allocation stack, it's considered marked. 19702cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi to_ref = from_ref; 19712cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi } else { 19722cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi // Not marked. 19732cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi to_ref = nullptr; 19742cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi } 19752cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi } 19762cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi } 19772cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi } 19782cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi return to_ref; 19792cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi} 19802cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi 19812cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchibool ConcurrentCopying::IsOnAllocStack(mirror::Object* ref) { 19822cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi QuasiAtomic::ThreadFenceAcquire(); 19832cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi accounting::ObjectStack* alloc_stack = GetAllocationStack(); 1984cb535da36915f9d10bec3880b46f1de1f7a69f22Mathieu Chartier return alloc_stack->Contains(ref); 19852cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi} 19862cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi 1987723e6cee35671d2dd9aeb884dd11f6994307c01fHiroshi Yamauchimirror::Object* ConcurrentCopying::MarkNonMoving(mirror::Object* ref) { 1988723e6cee35671d2dd9aeb884dd11f6994307c01fHiroshi Yamauchi // ref is in a non-moving space (from_ref == to_ref). 1989723e6cee35671d2dd9aeb884dd11f6994307c01fHiroshi Yamauchi DCHECK(!region_space_->HasAddress(ref)) << ref; 1990763a31ed7a2bfad22a9cb07f5301a71c0f97ca49Mathieu Chartier if (immune_spaces_.ContainsObject(ref)) { 1991723e6cee35671d2dd9aeb884dd11f6994307c01fHiroshi Yamauchi accounting::ContinuousSpaceBitmap* cc_bitmap = 1992723e6cee35671d2dd9aeb884dd11f6994307c01fHiroshi Yamauchi cc_heap_bitmap_->GetContinuousSpaceBitmap(ref); 1993723e6cee35671d2dd9aeb884dd11f6994307c01fHiroshi Yamauchi DCHECK(cc_bitmap != nullptr) 1994723e6cee35671d2dd9aeb884dd11f6994307c01fHiroshi Yamauchi << "An immune space object must have a bitmap"; 1995723e6cee35671d2dd9aeb884dd11f6994307c01fHiroshi Yamauchi if (kIsDebugBuild) { 1996723e6cee35671d2dd9aeb884dd11f6994307c01fHiroshi Yamauchi DCHECK(heap_mark_bitmap_->GetContinuousSpaceBitmap(ref)->Test(ref)) 1997723e6cee35671d2dd9aeb884dd11f6994307c01fHiroshi Yamauchi << "Immune space object must be already marked"; 19982cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi } 19992cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi // This may or may not succeed, which is ok. 20002cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi if (kUseBakerReadBarrier) { 2001723e6cee35671d2dd9aeb884dd11f6994307c01fHiroshi Yamauchi ref->AtomicSetReadBarrierPointer(ReadBarrier::WhitePtr(), ReadBarrier::GrayPtr()); 20022cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi } 2003723e6cee35671d2dd9aeb884dd11f6994307c01fHiroshi Yamauchi if (cc_bitmap->AtomicTestAndSet(ref)) { 20042cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi // Already marked. 20052cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi } else { 20062cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi // Newly marked. 20072cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi if (kUseBakerReadBarrier) { 2008723e6cee35671d2dd9aeb884dd11f6994307c01fHiroshi Yamauchi DCHECK_EQ(ref->GetReadBarrierPointer(), ReadBarrier::GrayPtr()); 20092cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi } 2010723e6cee35671d2dd9aeb884dd11f6994307c01fHiroshi Yamauchi PushOntoMarkStack(ref); 20112cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi } 20122cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi } else { 2013723e6cee35671d2dd9aeb884dd11f6994307c01fHiroshi Yamauchi // Use the mark bitmap. 2014723e6cee35671d2dd9aeb884dd11f6994307c01fHiroshi Yamauchi accounting::ContinuousSpaceBitmap* mark_bitmap = 2015723e6cee35671d2dd9aeb884dd11f6994307c01fHiroshi Yamauchi heap_mark_bitmap_->GetContinuousSpaceBitmap(ref); 2016723e6cee35671d2dd9aeb884dd11f6994307c01fHiroshi Yamauchi accounting::LargeObjectBitmap* los_bitmap = 2017723e6cee35671d2dd9aeb884dd11f6994307c01fHiroshi Yamauchi heap_mark_bitmap_->GetLargeObjectBitmap(ref); 2018723e6cee35671d2dd9aeb884dd11f6994307c01fHiroshi Yamauchi CHECK(los_bitmap != nullptr) << "LOS bitmap covers the entire address range"; 2019723e6cee35671d2dd9aeb884dd11f6994307c01fHiroshi Yamauchi bool is_los = mark_bitmap == nullptr; 2020723e6cee35671d2dd9aeb884dd11f6994307c01fHiroshi Yamauchi if (!is_los && mark_bitmap->Test(ref)) { 2021723e6cee35671d2dd9aeb884dd11f6994307c01fHiroshi Yamauchi // Already marked. 20222cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi if (kUseBakerReadBarrier) { 2023723e6cee35671d2dd9aeb884dd11f6994307c01fHiroshi Yamauchi DCHECK(ref->GetReadBarrierPointer() == ReadBarrier::GrayPtr() || 2024723e6cee35671d2dd9aeb884dd11f6994307c01fHiroshi Yamauchi ref->GetReadBarrierPointer() == ReadBarrier::BlackPtr()); 20252cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi } 2026723e6cee35671d2dd9aeb884dd11f6994307c01fHiroshi Yamauchi } else if (is_los && los_bitmap->Test(ref)) { 2027723e6cee35671d2dd9aeb884dd11f6994307c01fHiroshi Yamauchi // Already marked in LOS. 2028723e6cee35671d2dd9aeb884dd11f6994307c01fHiroshi Yamauchi if (kUseBakerReadBarrier) { 2029723e6cee35671d2dd9aeb884dd11f6994307c01fHiroshi Yamauchi DCHECK(ref->GetReadBarrierPointer() == ReadBarrier::GrayPtr() || 2030723e6cee35671d2dd9aeb884dd11f6994307c01fHiroshi Yamauchi ref->GetReadBarrierPointer() == ReadBarrier::BlackPtr()); 20312cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi } 20322cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi } else { 2033723e6cee35671d2dd9aeb884dd11f6994307c01fHiroshi Yamauchi // Not marked. 2034723e6cee35671d2dd9aeb884dd11f6994307c01fHiroshi Yamauchi if (IsOnAllocStack(ref)) { 2035723e6cee35671d2dd9aeb884dd11f6994307c01fHiroshi Yamauchi // If it's on the allocation stack, it's considered marked. Keep it white. 2036723e6cee35671d2dd9aeb884dd11f6994307c01fHiroshi Yamauchi // Objects on the allocation stack need not be marked. 2037723e6cee35671d2dd9aeb884dd11f6994307c01fHiroshi Yamauchi if (!is_los) { 2038723e6cee35671d2dd9aeb884dd11f6994307c01fHiroshi Yamauchi DCHECK(!mark_bitmap->Test(ref)); 2039723e6cee35671d2dd9aeb884dd11f6994307c01fHiroshi Yamauchi } else { 2040723e6cee35671d2dd9aeb884dd11f6994307c01fHiroshi Yamauchi DCHECK(!los_bitmap->Test(ref)); 20412cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi } 20422cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi if (kUseBakerReadBarrier) { 2043723e6cee35671d2dd9aeb884dd11f6994307c01fHiroshi Yamauchi DCHECK_EQ(ref->GetReadBarrierPointer(), ReadBarrier::WhitePtr()); 20442cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi } 20452cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi } else { 2046723e6cee35671d2dd9aeb884dd11f6994307c01fHiroshi Yamauchi // Not marked or on the allocation stack. Try to mark it. 2047723e6cee35671d2dd9aeb884dd11f6994307c01fHiroshi Yamauchi // This may or may not succeed, which is ok. 2048723e6cee35671d2dd9aeb884dd11f6994307c01fHiroshi Yamauchi if (kUseBakerReadBarrier) { 2049723e6cee35671d2dd9aeb884dd11f6994307c01fHiroshi Yamauchi ref->AtomicSetReadBarrierPointer(ReadBarrier::WhitePtr(), ReadBarrier::GrayPtr()); 2050723e6cee35671d2dd9aeb884dd11f6994307c01fHiroshi Yamauchi } 2051723e6cee35671d2dd9aeb884dd11f6994307c01fHiroshi Yamauchi if (!is_los && mark_bitmap->AtomicTestAndSet(ref)) { 2052723e6cee35671d2dd9aeb884dd11f6994307c01fHiroshi Yamauchi // Already marked. 2053723e6cee35671d2dd9aeb884dd11f6994307c01fHiroshi Yamauchi } else if (is_los && los_bitmap->AtomicTestAndSet(ref)) { 2054723e6cee35671d2dd9aeb884dd11f6994307c01fHiroshi Yamauchi // Already marked in LOS. 20552cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi } else { 2056723e6cee35671d2dd9aeb884dd11f6994307c01fHiroshi Yamauchi // Newly marked. 20572cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi if (kUseBakerReadBarrier) { 2058723e6cee35671d2dd9aeb884dd11f6994307c01fHiroshi Yamauchi DCHECK_EQ(ref->GetReadBarrierPointer(), ReadBarrier::GrayPtr()); 20592cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi } 2060723e6cee35671d2dd9aeb884dd11f6994307c01fHiroshi Yamauchi PushOntoMarkStack(ref); 20612cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi } 20622cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi } 20632cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi } 20642cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi } 2065723e6cee35671d2dd9aeb884dd11f6994307c01fHiroshi Yamauchi return ref; 20662cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi} 20672cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi 20682cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchivoid ConcurrentCopying::FinishPhase() { 2069a9d82fe8bc6960b565245b920e99107a824ca515Mathieu Chartier Thread* const self = Thread::Current(); 20700b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi { 2071a9d82fe8bc6960b565245b920e99107a824ca515Mathieu Chartier MutexLock mu(self, mark_stack_lock_); 20720b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi CHECK_EQ(pooled_mark_stacks_.size(), kMarkStackPoolSize); 20730b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi } 20742cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi region_space_ = nullptr; 20752cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi { 20762cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi MutexLock mu(Thread::Current(), skipped_blocks_lock_); 20772cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi skipped_blocks_map_.clear(); 20782cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi } 2079a9d82fe8bc6960b565245b920e99107a824ca515Mathieu Chartier ReaderMutexLock mu(self, *Locks::mutator_lock_); 2080a9d82fe8bc6960b565245b920e99107a824ca515Mathieu Chartier WriterMutexLock mu2(self, *Locks::heap_bitmap_lock_); 20812cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi heap_->ClearMarkedObjects(); 20822cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi} 20832cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi 208497509954404d031594b2ecbda607314d169d512eMathieu Chartierbool ConcurrentCopying::IsMarkedHeapReference(mirror::HeapReference<mirror::Object>* field) { 20852cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi mirror::Object* from_ref = field->AsMirrorPtr(); 208697509954404d031594b2ecbda607314d169d512eMathieu Chartier mirror::Object* to_ref = IsMarked(from_ref); 20872cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi if (to_ref == nullptr) { 20882cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi return false; 20892cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi } 20902cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi if (from_ref != to_ref) { 20912cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi QuasiAtomic::ThreadFenceRelease(); 20922cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi field->Assign(to_ref); 20932cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi QuasiAtomic::ThreadFenceSequentiallyConsistent(); 20942cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi } 20952cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi return true; 20962cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi} 20972cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi 209897509954404d031594b2ecbda607314d169d512eMathieu Chartiermirror::Object* ConcurrentCopying::MarkObject(mirror::Object* from_ref) { 209997509954404d031594b2ecbda607314d169d512eMathieu Chartier return Mark(from_ref); 21002cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi} 21012cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi 21022cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchivoid ConcurrentCopying::DelayReferenceReferent(mirror::Class* klass, mirror::Reference* reference) { 210397509954404d031594b2ecbda607314d169d512eMathieu Chartier heap_->GetReferenceProcessor()->DelayReferenceReferent(klass, reference, this); 21042cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi} 21052cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi 21060b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchivoid ConcurrentCopying::ProcessReferences(Thread* self) { 21072cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi TimingLogger::ScopedTiming split("ProcessReferences", GetTimings()); 21080b71357fb52be9bb06d35396a3042b4381b01041Hiroshi Yamauchi // We don't really need to lock the heap bitmap lock as we use CAS to mark in bitmaps. 21092cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi WriterMutexLock mu(self, *Locks::heap_bitmap_lock_); 21102cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi GetHeap()->GetReferenceProcessor()->ProcessReferences( 211197509954404d031594b2ecbda607314d169d512eMathieu Chartier true /*concurrent*/, GetTimings(), GetCurrentIteration()->GetClearSoftReferences(), this); 21122cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi} 21132cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi 21142cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchivoid ConcurrentCopying::RevokeAllThreadLocalBuffers() { 21152cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi TimingLogger::ScopedTiming t(__FUNCTION__, GetTimings()); 21162cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi region_space_->RevokeAllThreadLocalBuffers(); 21172cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi} 21182cd334ae2d4287216523882f0d298cf3901b7ab1Hiroshi Yamauchi 2119d5307ec41c8344be0c32273ec4f574064036187dHiroshi Yamauchi} // namespace collector 2120d5307ec41c8344be0c32273ec4f574064036187dHiroshi Yamauchi} // namespace gc 2121d5307ec41c8344be0c32273ec4f574064036187dHiroshi Yamauchi} // namespace art 2122