152e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier/* 252e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier * Copyright (C) 2014 The Android Open Source Project 352e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier * 452e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier * Licensed under the Apache License, Version 2.0 (the "License"); 552e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier * you may not use this file except in compliance with the License. 652e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier * You may obtain a copy of the License at 752e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier * 852e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier * http://www.apache.org/licenses/LICENSE-2.0 952e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier * 1052e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier * Unless required by applicable law or agreed to in writing, software 1152e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier * distributed under the License is distributed on an "AS IS" BASIS, 1252e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 1352e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier * See the License for the specific language governing permissions and 1452e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier * limitations under the License. 1552e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier */ 1652e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier 1752e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier#include "mark_compact.h" 1852e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier 1952e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier#include "base/logging.h" 2052e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier#include "base/mutex-inl.h" 2152e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier#include "base/timing_logger.h" 2252e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier#include "gc/accounting/heap_bitmap-inl.h" 2352e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier#include "gc/accounting/mod_union_table.h" 2452e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier#include "gc/accounting/space_bitmap-inl.h" 2552e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier#include "gc/heap.h" 2652e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier#include "gc/reference_processor.h" 2752e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier#include "gc/space/bump_pointer_space-inl.h" 2852e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier#include "gc/space/large_object_space.h" 2952e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier#include "gc/space/space-inl.h" 3052e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier#include "mirror/class-inl.h" 3152e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier#include "mirror/object-inl.h" 3252e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier#include "runtime.h" 3352e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier#include "stack.h" 3452e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier#include "thread-inl.h" 3552e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier#include "thread_list.h" 3652e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier 3752e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartiernamespace art { 3852e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartiernamespace gc { 3952e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartiernamespace collector { 4052e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier 4152e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartiervoid MarkCompact::BindBitmaps() { 42f5997b4d3f889569d5a2b724d83d764bfbb8d106Mathieu Chartier TimingLogger::ScopedTiming t(__FUNCTION__, GetTimings()); 4352e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier WriterMutexLock mu(Thread::Current(), *Locks::heap_bitmap_lock_); 4452e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier // Mark all of the spaces we never collect as immune. 4552e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier for (const auto& space : GetHeap()->GetContinuousSpaces()) { 4652e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier if (space->GetGcRetentionPolicy() == space::kGcRetentionPolicyNeverCollect || 4752e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier space->GetGcRetentionPolicy() == space::kGcRetentionPolicyFullCollect) { 48763a31ed7a2bfad22a9cb07f5301a71c0f97ca49Mathieu Chartier immune_spaces_.AddSpace(space); 4952e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier } 5052e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier } 5152e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier} 5252e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier 5352e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu ChartierMarkCompact::MarkCompact(Heap* heap, const std::string& name_prefix) 5452e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier : GarbageCollector(heap, name_prefix + (name_prefix.empty() ? "" : " ") + "mark compact"), 5597509954404d031594b2ecbda607314d169d512eMathieu Chartier space_(nullptr), collector_name_(name_), updating_references_(false) { 5652e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier} 5752e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier 5852e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartiervoid MarkCompact::RunPhases() { 5952e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier Thread* self = Thread::Current(); 6052e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier InitializePhase(); 6152e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier CHECK(!Locks::mutator_lock_->IsExclusiveHeld(self)); 6252e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier { 6352e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier ScopedPause pause(this); 6452e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier GetHeap()->PreGcVerificationPaused(this); 6552e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier GetHeap()->PrePauseRosAllocVerification(this); 6652e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier MarkingPhase(); 6752e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier ReclaimPhase(); 6852e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier } 6952e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier GetHeap()->PostGcVerification(this); 7052e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier FinishPhase(); 7152e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier} 7252e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier 7352e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartiervoid MarkCompact::ForwardObject(mirror::Object* obj) { 7452e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier const size_t alloc_size = RoundUp(obj->SizeOf(), space::BumpPointerSpace::kAlignment); 7552e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier LockWord lock_word = obj->GetLockWord(false); 7652e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier // If we have a non empty lock word, store it and restore it later. 77e15ea086439b41a805d164d2beb07b4ba96aaa97Hiroshi Yamauchi if (!LockWord::IsDefault(lock_word)) { 7852e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier // Set the bit in the bitmap so that we know to restore it later. 7952e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier objects_with_lockword_->Set(obj); 8052e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier lock_words_to_restore_.push_back(lock_word); 8152e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier } 8252e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier obj->SetLockWord(LockWord::FromForwardingAddress(reinterpret_cast<size_t>(bump_pointer_)), 8352e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier false); 8452e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier bump_pointer_ += alloc_size; 8552e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier ++live_objects_in_space_; 8652e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier} 8752e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier 8852e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartierclass CalculateObjectForwardingAddressVisitor { 8952e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier public: 9052e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier explicit CalculateObjectForwardingAddressVisitor(MarkCompact* collector) 9152e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier : collector_(collector) {} 9290443477f9a0061581c420775ce3b7eeae7468bcMathieu Chartier void operator()(mirror::Object* obj) const REQUIRES(Locks::mutator_lock_, 9352e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier Locks::heap_bitmap_lock_) { 9452e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier DCHECK_ALIGNED(obj, space::BumpPointerSpace::kAlignment); 9597509954404d031594b2ecbda607314d169d512eMathieu Chartier DCHECK(collector_->IsMarked(obj) != nullptr); 9652e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier collector_->ForwardObject(obj); 9752e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier } 9852e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier 9952e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier private: 10052e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier MarkCompact* const collector_; 10152e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier}; 10252e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier 10352e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartiervoid MarkCompact::CalculateObjectForwardingAddresses() { 104f5997b4d3f889569d5a2b724d83d764bfbb8d106Mathieu Chartier TimingLogger::ScopedTiming t(__FUNCTION__, GetTimings()); 10552e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier // The bump pointer in the space where the next forwarding address will be. 10613735955f39b3b304c37d2b2840663c131262c18Ian Rogers bump_pointer_ = reinterpret_cast<uint8_t*>(space_->Begin()); 10752e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier // Visit all the marked objects in the bitmap. 10852e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier CalculateObjectForwardingAddressVisitor visitor(this); 10952e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier objects_before_forwarding_->VisitMarkedRange(reinterpret_cast<uintptr_t>(space_->Begin()), 11052e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier reinterpret_cast<uintptr_t>(space_->End()), 11152e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier visitor); 11252e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier} 11352e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier 11452e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartiervoid MarkCompact::InitializePhase() { 115f5997b4d3f889569d5a2b724d83d764bfbb8d106Mathieu Chartier TimingLogger::ScopedTiming t(__FUNCTION__, GetTimings()); 11652e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier mark_stack_ = heap_->GetMarkStack(); 11752e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier DCHECK(mark_stack_ != nullptr); 118763a31ed7a2bfad22a9cb07f5301a71c0f97ca49Mathieu Chartier immune_spaces_.Reset(); 11952e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier CHECK(space_->CanMoveObjects()) << "Attempting compact non-movable space from " << *space_; 12052e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier // TODO: I don't think we should need heap bitmap lock to Get the mark bitmap. 12152e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier ReaderMutexLock mu(Thread::Current(), *Locks::heap_bitmap_lock_); 12252e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier mark_bitmap_ = heap_->GetMarkBitmap(); 12352e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier live_objects_in_space_ = 0; 12452e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier} 12552e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier 12652e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartiervoid MarkCompact::ProcessReferences(Thread* self) { 12752e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier WriterMutexLock mu(self, *Locks::heap_bitmap_lock_); 12852e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier heap_->GetReferenceProcessor()->ProcessReferences( 12997509954404d031594b2ecbda607314d169d512eMathieu Chartier false, GetTimings(), GetCurrentIteration()->GetClearSoftReferences(), this); 13052e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier} 13152e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier 13252e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartierclass BitmapSetSlowPathVisitor { 13352e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier public: 13403d21bc5bed887243ff6ce3531179185ffd3532cMathieu Chartier void operator()(const mirror::Object* obj) const SHARED_REQUIRES(Locks::mutator_lock_) { 13552e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier // Marking a large object, make sure its aligned as a sanity check. 13652e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier if (!IsAligned<kPageSize>(obj)) { 13752e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier Runtime::Current()->GetHeap()->DumpSpaces(LOG(ERROR)); 13852e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier LOG(FATAL) << obj; 13952e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier } 14052e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier } 14152e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier}; 14252e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier 14397509954404d031594b2ecbda607314d169d512eMathieu Chartierinline mirror::Object* MarkCompact::MarkObject(mirror::Object* obj) { 14452e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier if (obj == nullptr) { 1458118781ebc9659f806716c451bdb3fe9b77ae32bMathieu Chartier return nullptr; 14652e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier } 14752e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier if (kUseBakerOrBrooksReadBarrier) { 14852e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier // Verify all the objects have the correct forward pointer installed. 14952e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier obj->AssertReadBarrierPointer(); 15052e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier } 151763a31ed7a2bfad22a9cb07f5301a71c0f97ca49Mathieu Chartier if (!immune_spaces_.IsInImmuneRegion(obj)) { 15297509954404d031594b2ecbda607314d169d512eMathieu Chartier if (objects_before_forwarding_->HasAddress(obj)) { 15397509954404d031594b2ecbda607314d169d512eMathieu Chartier if (!objects_before_forwarding_->Set(obj)) { 15497509954404d031594b2ecbda607314d169d512eMathieu Chartier MarkStackPush(obj); // This object was not previously marked. 15597509954404d031594b2ecbda607314d169d512eMathieu Chartier } 15697509954404d031594b2ecbda607314d169d512eMathieu Chartier } else { 15797509954404d031594b2ecbda607314d169d512eMathieu Chartier DCHECK(!space_->HasAddress(obj)); 15897509954404d031594b2ecbda607314d169d512eMathieu Chartier BitmapSetSlowPathVisitor visitor; 15997509954404d031594b2ecbda607314d169d512eMathieu Chartier if (!mark_bitmap_->Set(obj, visitor)) { 16097509954404d031594b2ecbda607314d169d512eMathieu Chartier // This object was not previously marked. 16197509954404d031594b2ecbda607314d169d512eMathieu Chartier MarkStackPush(obj); 16297509954404d031594b2ecbda607314d169d512eMathieu Chartier } 16352e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier } 16452e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier } 16597509954404d031594b2ecbda607314d169d512eMathieu Chartier return obj; 16652e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier} 16752e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier 16852e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartiervoid MarkCompact::MarkingPhase() { 169f5997b4d3f889569d5a2b724d83d764bfbb8d106Mathieu Chartier TimingLogger::ScopedTiming t(__FUNCTION__, GetTimings()); 17052e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier Thread* self = Thread::Current(); 17152e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier // Bitmap which describes which objects we have to move. 17252e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier objects_before_forwarding_.reset(accounting::ContinuousSpaceBitmap::Create( 17352e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier "objects before forwarding", space_->Begin(), space_->Size())); 17452e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier // Bitmap which describes which lock words we need to restore. 17552e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier objects_with_lockword_.reset(accounting::ContinuousSpaceBitmap::Create( 17652e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier "objects with lock words", space_->Begin(), space_->Size())); 17752e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier CHECK(Locks::mutator_lock_->IsExclusiveHeld(self)); 17852e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier // Assume the cleared space is already empty. 17952e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier BindBitmaps(); 180f5997b4d3f889569d5a2b724d83d764bfbb8d106Mathieu Chartier t.NewTiming("ProcessCards"); 18152e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier // Process dirty cards and add dirty cards to mod-union tables. 1824add3b4fa38ec42bb3c71d01cf70bce8e9a9fb4eLei Li heap_->ProcessCards(GetTimings(), false, false, true); 18391d65e024846717fce3572106cffe9b957b8902cRoland Levillain // Clear the whole card table since we cannot get any additional dirty cards during the 18452e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier // paused GC. This saves memory but only works for pause the world collectors. 185f5997b4d3f889569d5a2b724d83d764bfbb8d106Mathieu Chartier t.NewTiming("ClearCardTable"); 18652e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier heap_->GetCardTable()->ClearCardTable(); 18752e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier // Need to do this before the checkpoint since we don't want any threads to add references to 18852e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier // the live stack during the recursive mark. 18952e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier if (kUseThreadLocalAllocationStack) { 190f5997b4d3f889569d5a2b724d83d764bfbb8d106Mathieu Chartier t.NewTiming("RevokeAllThreadLocalAllocationStacks"); 19152e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier heap_->RevokeAllThreadLocalAllocationStacks(self); 19252e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier } 193f5997b4d3f889569d5a2b724d83d764bfbb8d106Mathieu Chartier t.NewTiming("SwapStacks"); 194a4f6af9b1e6380b31674d7ac645b1732c846ac06Mathieu Chartier heap_->SwapStacks(); 19552e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier { 19652e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier WriterMutexLock mu(self, *Locks::heap_bitmap_lock_); 19752e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier MarkRoots(); 19852e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier // Mark roots of immune spaces. 19952e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier UpdateAndMarkModUnion(); 20052e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier // Recursively mark remaining objects. 20152e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier MarkReachableObjects(); 20252e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier } 20352e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier ProcessReferences(self); 20452e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier { 20552e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier ReaderMutexLock mu(self, *Locks::heap_bitmap_lock_); 20652e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier SweepSystemWeaks(); 20752e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier } 208951ec2c93c79c5539cbcc669566f0808d4460338Mathieu Chartier Runtime::Current()->GetClassLinker()->CleanupClassLoaders(); 20952e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier // Revoke buffers before measuring how many objects were moved since the TLABs need to be revoked 21052e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier // before they are properly counted. 21152e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier RevokeAllThreadLocalBuffers(); 21252e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier // Disabled due to an issue where we have objects in the bump pointer space which reference dead 21352e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier // objects. 21452e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier // heap_->PreSweepingGcVerification(this); 21552e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier} 21652e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier 21752e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartiervoid MarkCompact::UpdateAndMarkModUnion() { 218f5997b4d3f889569d5a2b724d83d764bfbb8d106Mathieu Chartier TimingLogger::ScopedTiming t(__FUNCTION__, GetTimings()); 21952e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier for (auto& space : heap_->GetContinuousSpaces()) { 22052e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier // If the space is immune then we need to mark the references to other spaces. 221763a31ed7a2bfad22a9cb07f5301a71c0f97ca49Mathieu Chartier if (immune_spaces_.ContainsSpace(space)) { 22252e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier accounting::ModUnionTable* table = heap_->FindModUnionTableFromSpace(space); 22352e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier if (table != nullptr) { 22452e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier // TODO: Improve naming. 225277ccbd200ea43590dfc06a93ae184a765327ad0Andreas Gampe TimingLogger::ScopedTiming t2( 22652e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier space->IsZygoteSpace() ? "UpdateAndMarkZygoteModUnionTable" : 22710fb83ad7442c8cf3356a89ec918e0786f110981Mathieu Chartier "UpdateAndMarkImageModUnionTable", GetTimings()); 22897509954404d031594b2ecbda607314d169d512eMathieu Chartier table->UpdateAndMarkReferences(this); 22952e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier } 23052e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier } 23152e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier } 23252e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier} 23352e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier 23452e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartiervoid MarkCompact::MarkReachableObjects() { 235f5997b4d3f889569d5a2b724d83d764bfbb8d106Mathieu Chartier TimingLogger::ScopedTiming t(__FUNCTION__, GetTimings()); 23652e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier accounting::ObjectStack* live_stack = heap_->GetLiveStack(); 237f5997b4d3f889569d5a2b724d83d764bfbb8d106Mathieu Chartier { 238f5997b4d3f889569d5a2b724d83d764bfbb8d106Mathieu Chartier TimingLogger::ScopedTiming t2("MarkAllocStackAsLive", GetTimings()); 239f5997b4d3f889569d5a2b724d83d764bfbb8d106Mathieu Chartier heap_->MarkAllocStackAsLive(live_stack); 240f5997b4d3f889569d5a2b724d83d764bfbb8d106Mathieu Chartier } 24152e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier live_stack->Reset(); 24252e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier // Recursively process the mark stack. 24352e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier ProcessMarkStack(); 24452e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier} 24552e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier 24652e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartiervoid MarkCompact::ReclaimPhase() { 247f5997b4d3f889569d5a2b724d83d764bfbb8d106Mathieu Chartier TimingLogger::ScopedTiming t(__FUNCTION__, GetTimings()); 24852e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier WriterMutexLock mu(Thread::Current(), *Locks::heap_bitmap_lock_); 24952e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier // Reclaim unmarked objects. 25052e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier Sweep(false); 25152e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier // Swap the live and mark bitmaps for each space which we modified space. This is an 25252e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier // optimization that enables us to not clear live bits inside of the sweep. Only swaps unbound 25352e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier // bitmaps. 25452e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier SwapBitmaps(); 25552e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier GetHeap()->UnBindBitmaps(); // Unbind the live and mark bitmaps. 25652e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier Compact(); 25752e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier} 25852e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier 25952e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartiervoid MarkCompact::ResizeMarkStack(size_t new_size) { 26097509954404d031594b2ecbda607314d169d512eMathieu Chartier std::vector<StackReference<mirror::Object>> temp(mark_stack_->Begin(), mark_stack_->End()); 26152e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier CHECK_LE(mark_stack_->Size(), new_size); 26252e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier mark_stack_->Resize(new_size); 263cb535da36915f9d10bec3880b46f1de1f7a69f22Mathieu Chartier for (auto& obj : temp) { 264cb535da36915f9d10bec3880b46f1de1f7a69f22Mathieu Chartier mark_stack_->PushBack(obj.AsMirrorPtr()); 26552e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier } 26652e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier} 26752e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier 26897509954404d031594b2ecbda607314d169d512eMathieu Chartierinline void MarkCompact::MarkStackPush(mirror::Object* obj) { 26952e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier if (UNLIKELY(mark_stack_->Size() >= mark_stack_->Capacity())) { 27052e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier ResizeMarkStack(mark_stack_->Capacity() * 2); 27152e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier } 27252e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier // The object must be pushed on to the mark stack. 27352e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier mark_stack_->PushBack(obj); 27452e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier} 27552e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier 27697509954404d031594b2ecbda607314d169d512eMathieu Chartiervoid MarkCompact::MarkHeapReference(mirror::HeapReference<mirror::Object>* obj_ptr) { 27797509954404d031594b2ecbda607314d169d512eMathieu Chartier if (updating_references_) { 27897509954404d031594b2ecbda607314d169d512eMathieu Chartier UpdateHeapReference(obj_ptr); 27997509954404d031594b2ecbda607314d169d512eMathieu Chartier } else { 28097509954404d031594b2ecbda607314d169d512eMathieu Chartier MarkObject(obj_ptr->AsMirrorPtr()); 28197509954404d031594b2ecbda607314d169d512eMathieu Chartier } 28252e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier} 28352e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier 284bb87e0f1a52de656bc77cb01cb887e51a0e5198bMathieu Chartiervoid MarkCompact::VisitRoots( 285bb87e0f1a52de656bc77cb01cb887e51a0e5198bMathieu Chartier mirror::Object*** roots, size_t count, const RootInfo& info ATTRIBUTE_UNUSED) { 286bb87e0f1a52de656bc77cb01cb887e51a0e5198bMathieu Chartier for (size_t i = 0; i < count; ++i) { 287bb87e0f1a52de656bc77cb01cb887e51a0e5198bMathieu Chartier MarkObject(*roots[i]); 288bb87e0f1a52de656bc77cb01cb887e51a0e5198bMathieu Chartier } 28952e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier} 29052e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier 291bb87e0f1a52de656bc77cb01cb887e51a0e5198bMathieu Chartiervoid MarkCompact::VisitRoots( 292bb87e0f1a52de656bc77cb01cb887e51a0e5198bMathieu Chartier mirror::CompressedReference<mirror::Object>** roots, size_t count, 293bb87e0f1a52de656bc77cb01cb887e51a0e5198bMathieu Chartier const RootInfo& info ATTRIBUTE_UNUSED) { 294bb87e0f1a52de656bc77cb01cb887e51a0e5198bMathieu Chartier for (size_t i = 0; i < count; ++i) { 295bb87e0f1a52de656bc77cb01cb887e51a0e5198bMathieu Chartier MarkObject(roots[i]->AsMirrorPtr()); 29652e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier } 29752e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier} 29852e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier 299bb87e0f1a52de656bc77cb01cb887e51a0e5198bMathieu Chartierclass UpdateRootVisitor : public RootVisitor { 300bb87e0f1a52de656bc77cb01cb887e51a0e5198bMathieu Chartier public: 301bb87e0f1a52de656bc77cb01cb887e51a0e5198bMathieu Chartier explicit UpdateRootVisitor(MarkCompact* collector) : collector_(collector) { 302bb87e0f1a52de656bc77cb01cb887e51a0e5198bMathieu Chartier } 303bb87e0f1a52de656bc77cb01cb887e51a0e5198bMathieu Chartier 304bb87e0f1a52de656bc77cb01cb887e51a0e5198bMathieu Chartier void VisitRoots(mirror::Object*** roots, size_t count, const RootInfo& info ATTRIBUTE_UNUSED) 30590443477f9a0061581c420775ce3b7eeae7468bcMathieu Chartier OVERRIDE REQUIRES(Locks::mutator_lock_) 30690443477f9a0061581c420775ce3b7eeae7468bcMathieu Chartier SHARED_REQUIRES(Locks::heap_bitmap_lock_) { 307bb87e0f1a52de656bc77cb01cb887e51a0e5198bMathieu Chartier for (size_t i = 0; i < count; ++i) { 308bb87e0f1a52de656bc77cb01cb887e51a0e5198bMathieu Chartier mirror::Object* obj = *roots[i]; 309bb87e0f1a52de656bc77cb01cb887e51a0e5198bMathieu Chartier mirror::Object* new_obj = collector_->GetMarkedForwardAddress(obj); 310bb87e0f1a52de656bc77cb01cb887e51a0e5198bMathieu Chartier if (obj != new_obj) { 311bb87e0f1a52de656bc77cb01cb887e51a0e5198bMathieu Chartier *roots[i] = new_obj; 312bb87e0f1a52de656bc77cb01cb887e51a0e5198bMathieu Chartier DCHECK(new_obj != nullptr); 313bb87e0f1a52de656bc77cb01cb887e51a0e5198bMathieu Chartier } 314bb87e0f1a52de656bc77cb01cb887e51a0e5198bMathieu Chartier } 315bb87e0f1a52de656bc77cb01cb887e51a0e5198bMathieu Chartier } 316bb87e0f1a52de656bc77cb01cb887e51a0e5198bMathieu Chartier 317bb87e0f1a52de656bc77cb01cb887e51a0e5198bMathieu Chartier void VisitRoots(mirror::CompressedReference<mirror::Object>** roots, size_t count, 318bb87e0f1a52de656bc77cb01cb887e51a0e5198bMathieu Chartier const RootInfo& info ATTRIBUTE_UNUSED) 31990443477f9a0061581c420775ce3b7eeae7468bcMathieu Chartier OVERRIDE REQUIRES(Locks::mutator_lock_) 32090443477f9a0061581c420775ce3b7eeae7468bcMathieu Chartier SHARED_REQUIRES(Locks::heap_bitmap_lock_) { 321bb87e0f1a52de656bc77cb01cb887e51a0e5198bMathieu Chartier for (size_t i = 0; i < count; ++i) { 322bb87e0f1a52de656bc77cb01cb887e51a0e5198bMathieu Chartier mirror::Object* obj = roots[i]->AsMirrorPtr(); 323bb87e0f1a52de656bc77cb01cb887e51a0e5198bMathieu Chartier mirror::Object* new_obj = collector_->GetMarkedForwardAddress(obj); 324bb87e0f1a52de656bc77cb01cb887e51a0e5198bMathieu Chartier if (obj != new_obj) { 325bb87e0f1a52de656bc77cb01cb887e51a0e5198bMathieu Chartier roots[i]->Assign(new_obj); 326bb87e0f1a52de656bc77cb01cb887e51a0e5198bMathieu Chartier DCHECK(new_obj != nullptr); 327bb87e0f1a52de656bc77cb01cb887e51a0e5198bMathieu Chartier } 328bb87e0f1a52de656bc77cb01cb887e51a0e5198bMathieu Chartier } 329bb87e0f1a52de656bc77cb01cb887e51a0e5198bMathieu Chartier } 330bb87e0f1a52de656bc77cb01cb887e51a0e5198bMathieu Chartier 331bb87e0f1a52de656bc77cb01cb887e51a0e5198bMathieu Chartier private: 332bb87e0f1a52de656bc77cb01cb887e51a0e5198bMathieu Chartier MarkCompact* const collector_; 333bb87e0f1a52de656bc77cb01cb887e51a0e5198bMathieu Chartier}; 334bb87e0f1a52de656bc77cb01cb887e51a0e5198bMathieu Chartier 33552e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartierclass UpdateObjectReferencesVisitor { 33652e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier public: 33752e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier explicit UpdateObjectReferencesVisitor(MarkCompact* collector) : collector_(collector) { 33852e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier } 33990443477f9a0061581c420775ce3b7eeae7468bcMathieu Chartier void operator()(mirror::Object* obj) const SHARED_REQUIRES(Locks::heap_bitmap_lock_) 34090443477f9a0061581c420775ce3b7eeae7468bcMathieu Chartier REQUIRES(Locks::mutator_lock_) ALWAYS_INLINE { 34152e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier collector_->UpdateObjectReferences(obj); 34252e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier } 34352e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier 34452e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier private: 34552e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier MarkCompact* const collector_; 34652e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier}; 34752e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier 34852e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartiervoid MarkCompact::UpdateReferences() { 349f5997b4d3f889569d5a2b724d83d764bfbb8d106Mathieu Chartier TimingLogger::ScopedTiming t(__FUNCTION__, GetTimings()); 35097509954404d031594b2ecbda607314d169d512eMathieu Chartier updating_references_ = true; 35152e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier Runtime* runtime = Runtime::Current(); 35252e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier // Update roots. 353bb87e0f1a52de656bc77cb01cb887e51a0e5198bMathieu Chartier UpdateRootVisitor update_root_visitor(this); 354bb87e0f1a52de656bc77cb01cb887e51a0e5198bMathieu Chartier runtime->VisitRoots(&update_root_visitor); 35552e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier // Update object references in mod union tables and spaces. 35652e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier for (const auto& space : heap_->GetContinuousSpaces()) { 35752e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier // If the space is immune then we need to mark the references to other spaces. 35852e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier accounting::ModUnionTable* table = heap_->FindModUnionTableFromSpace(space); 35952e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier if (table != nullptr) { 36052e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier // TODO: Improve naming. 361277ccbd200ea43590dfc06a93ae184a765327ad0Andreas Gampe TimingLogger::ScopedTiming t2( 36252e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier space->IsZygoteSpace() ? "UpdateZygoteModUnionTableReferences" : 36352e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier "UpdateImageModUnionTableReferences", 36410fb83ad7442c8cf3356a89ec918e0786f110981Mathieu Chartier GetTimings()); 36597509954404d031594b2ecbda607314d169d512eMathieu Chartier table->UpdateAndMarkReferences(this); 36652e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier } else { 36752e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier // No mod union table, so we need to scan the space using bitmap visit. 36852e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier // Scan the space using bitmap visit. 36952e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier accounting::ContinuousSpaceBitmap* bitmap = space->GetLiveBitmap(); 37052e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier if (bitmap != nullptr) { 37152e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier UpdateObjectReferencesVisitor visitor(this); 37252e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier bitmap->VisitMarkedRange(reinterpret_cast<uintptr_t>(space->Begin()), 37352e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier reinterpret_cast<uintptr_t>(space->End()), 37452e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier visitor); 37552e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier } 37652e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier } 37752e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier } 37852e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier CHECK(!kMovingClasses) 37952e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier << "Didn't update large object classes since they are assumed to not move."; 38052e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier // Update the system weaks, these should already have been swept. 38197509954404d031594b2ecbda607314d169d512eMathieu Chartier runtime->SweepSystemWeaks(this); 38252e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier // Update the objects in the bump pointer space last, these objects don't have a bitmap. 38352e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier UpdateObjectReferencesVisitor visitor(this); 38452e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier objects_before_forwarding_->VisitMarkedRange(reinterpret_cast<uintptr_t>(space_->Begin()), 38552e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier reinterpret_cast<uintptr_t>(space_->End()), 38652e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier visitor); 38752e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier // Update the reference processor cleared list. 38897509954404d031594b2ecbda607314d169d512eMathieu Chartier heap_->GetReferenceProcessor()->UpdateRoots(this); 38997509954404d031594b2ecbda607314d169d512eMathieu Chartier updating_references_ = false; 39052e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier} 39152e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier 39252e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartiervoid MarkCompact::Compact() { 393f5997b4d3f889569d5a2b724d83d764bfbb8d106Mathieu Chartier TimingLogger::ScopedTiming t(__FUNCTION__, GetTimings()); 39452e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier CalculateObjectForwardingAddresses(); 39552e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier UpdateReferences(); 39652e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier MoveObjects(); 39752e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier // Space 39852e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier int64_t objects_freed = space_->GetObjectsAllocated() - live_objects_in_space_; 39952e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier int64_t bytes_freed = reinterpret_cast<int64_t>(space_->End()) - 40052e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier reinterpret_cast<int64_t>(bump_pointer_); 401f5997b4d3f889569d5a2b724d83d764bfbb8d106Mathieu Chartier t.NewTiming("RecordFree"); 40252e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier space_->RecordFree(objects_freed, bytes_freed); 40310fb83ad7442c8cf3356a89ec918e0786f110981Mathieu Chartier RecordFree(ObjectBytePair(objects_freed, bytes_freed)); 40452e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier space_->SetEnd(bump_pointer_); 40552e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier // Need to zero out the memory we freed. TODO: Use madvise for pages. 40652e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier memset(bump_pointer_, 0, bytes_freed); 40752e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier} 40852e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier 40952e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier// Marks all objects in the root set. 41052e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartiervoid MarkCompact::MarkRoots() { 411f5997b4d3f889569d5a2b724d83d764bfbb8d106Mathieu Chartier TimingLogger::ScopedTiming t(__FUNCTION__, GetTimings()); 412bb87e0f1a52de656bc77cb01cb887e51a0e5198bMathieu Chartier Runtime::Current()->VisitRoots(this); 41352e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier} 41452e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier 41552e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartierinline void MarkCompact::UpdateHeapReference(mirror::HeapReference<mirror::Object>* reference) { 41652e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier mirror::Object* obj = reference->AsMirrorPtr(); 41752e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier if (obj != nullptr) { 41852e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier mirror::Object* new_obj = GetMarkedForwardAddress(obj); 41952e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier if (obj != new_obj) { 42052e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier DCHECK(new_obj != nullptr); 42152e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier reference->Assign(new_obj); 42252e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier } 42352e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier } 42452e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier} 42552e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier 42652e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartierclass UpdateReferenceVisitor { 42752e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier public: 42852e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier explicit UpdateReferenceVisitor(MarkCompact* collector) : collector_(collector) { 42952e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier } 43052e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier 43197509954404d031594b2ecbda607314d169d512eMathieu Chartier void operator()(mirror::Object* obj, MemberOffset offset, bool /*is_static*/) const 43290443477f9a0061581c420775ce3b7eeae7468bcMathieu Chartier ALWAYS_INLINE REQUIRES(Locks::mutator_lock_, Locks::heap_bitmap_lock_) { 43352e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier collector_->UpdateHeapReference(obj->GetFieldObjectReferenceAddr<kVerifyNone>(offset)); 43452e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier } 43552e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier 43652e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier void operator()(mirror::Class* /*klass*/, mirror::Reference* ref) const 43790443477f9a0061581c420775ce3b7eeae7468bcMathieu Chartier REQUIRES(Locks::mutator_lock_, Locks::heap_bitmap_lock_) { 43852e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier collector_->UpdateHeapReference( 43952e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier ref->GetFieldObjectReferenceAddr<kVerifyNone>(mirror::Reference::ReferentOffset())); 44052e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier } 44152e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier 442da7c650022a974be10e2f00fa07d5109e3d8826fMathieu Chartier // TODO: Remove NO_THREAD_SAFETY_ANALYSIS when clang better understands visitors. 443da7c650022a974be10e2f00fa07d5109e3d8826fMathieu Chartier void VisitRootIfNonNull(mirror::CompressedReference<mirror::Object>* root) const 444da7c650022a974be10e2f00fa07d5109e3d8826fMathieu Chartier NO_THREAD_SAFETY_ANALYSIS { 445da7c650022a974be10e2f00fa07d5109e3d8826fMathieu Chartier if (!root->IsNull()) { 446da7c650022a974be10e2f00fa07d5109e3d8826fMathieu Chartier VisitRoot(root); 447da7c650022a974be10e2f00fa07d5109e3d8826fMathieu Chartier } 448da7c650022a974be10e2f00fa07d5109e3d8826fMathieu Chartier } 449da7c650022a974be10e2f00fa07d5109e3d8826fMathieu Chartier 450da7c650022a974be10e2f00fa07d5109e3d8826fMathieu Chartier void VisitRoot(mirror::CompressedReference<mirror::Object>* root) const 451da7c650022a974be10e2f00fa07d5109e3d8826fMathieu Chartier NO_THREAD_SAFETY_ANALYSIS { 452da7c650022a974be10e2f00fa07d5109e3d8826fMathieu Chartier root->Assign(collector_->GetMarkedForwardAddress(root->AsMirrorPtr())); 453da7c650022a974be10e2f00fa07d5109e3d8826fMathieu Chartier } 454da7c650022a974be10e2f00fa07d5109e3d8826fMathieu Chartier 45552e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier private: 45652e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier MarkCompact* const collector_; 45752e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier}; 45852e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier 45952e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartiervoid MarkCompact::UpdateObjectReferences(mirror::Object* obj) { 46052e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier UpdateReferenceVisitor visitor(this); 461059ef3ddb2088f926ac452889e0953fdcd646a5eMathieu Chartier obj->VisitReferences(visitor, visitor); 46252e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier} 46352e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier 46497509954404d031594b2ecbda607314d169d512eMathieu Chartierinline mirror::Object* MarkCompact::GetMarkedForwardAddress(mirror::Object* obj) { 46552e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier DCHECK(obj != nullptr); 46652e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier if (objects_before_forwarding_->HasAddress(obj)) { 46752e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier DCHECK(objects_before_forwarding_->Test(obj)); 46852e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier mirror::Object* ret = 46952e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier reinterpret_cast<mirror::Object*>(obj->GetLockWord(false).ForwardingAddress()); 47052e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier DCHECK(ret != nullptr); 47152e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier return ret; 47252e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier } 47352e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier DCHECK(!space_->HasAddress(obj)); 47452e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier return obj; 47552e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier} 47652e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier 47797509954404d031594b2ecbda607314d169d512eMathieu Chartiermirror::Object* MarkCompact::IsMarked(mirror::Object* object) { 478763a31ed7a2bfad22a9cb07f5301a71c0f97ca49Mathieu Chartier if (immune_spaces_.IsInImmuneRegion(object)) { 47997509954404d031594b2ecbda607314d169d512eMathieu Chartier return object; 48097509954404d031594b2ecbda607314d169d512eMathieu Chartier } 48197509954404d031594b2ecbda607314d169d512eMathieu Chartier if (updating_references_) { 48297509954404d031594b2ecbda607314d169d512eMathieu Chartier return GetMarkedForwardAddress(object); 48352e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier } 48452e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier if (objects_before_forwarding_->HasAddress(object)) { 48597509954404d031594b2ecbda607314d169d512eMathieu Chartier return objects_before_forwarding_->Test(object) ? object : nullptr; 48652e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier } 48797509954404d031594b2ecbda607314d169d512eMathieu Chartier return mark_bitmap_->Test(object) ? object : nullptr; 48852e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier} 48952e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier 49097509954404d031594b2ecbda607314d169d512eMathieu Chartierbool MarkCompact::IsMarkedHeapReference(mirror::HeapReference<mirror::Object>* ref_ptr) { 49152e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier // Side effect free since we call this before ever moving objects. 49297509954404d031594b2ecbda607314d169d512eMathieu Chartier return IsMarked(ref_ptr->AsMirrorPtr()) != nullptr; 49352e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier} 49452e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier 49552e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartiervoid MarkCompact::SweepSystemWeaks() { 496f5997b4d3f889569d5a2b724d83d764bfbb8d106Mathieu Chartier TimingLogger::ScopedTiming t(__FUNCTION__, GetTimings()); 49797509954404d031594b2ecbda607314d169d512eMathieu Chartier Runtime::Current()->SweepSystemWeaks(this); 49852e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier} 49952e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier 50052e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartierbool MarkCompact::ShouldSweepSpace(space::ContinuousSpace* space) const { 501763a31ed7a2bfad22a9cb07f5301a71c0f97ca49Mathieu Chartier return space != space_ && !immune_spaces_.ContainsSpace(space); 50252e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier} 50352e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier 50452e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartierclass MoveObjectVisitor { 50552e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier public: 50652e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier explicit MoveObjectVisitor(MarkCompact* collector) : collector_(collector) { 50752e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier } 50890443477f9a0061581c420775ce3b7eeae7468bcMathieu Chartier void operator()(mirror::Object* obj) const SHARED_REQUIRES(Locks::heap_bitmap_lock_) 50990443477f9a0061581c420775ce3b7eeae7468bcMathieu Chartier REQUIRES(Locks::mutator_lock_) ALWAYS_INLINE { 51052e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier collector_->MoveObject(obj, obj->SizeOf()); 51152e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier } 51252e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier 51352e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier private: 51452e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier MarkCompact* const collector_; 51552e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier}; 51652e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier 51752e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartiervoid MarkCompact::MoveObject(mirror::Object* obj, size_t len) { 51852e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier // Look at the forwarding address stored in the lock word to know where to copy. 51952e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier DCHECK(space_->HasAddress(obj)) << obj; 52052e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier uintptr_t dest_addr = obj->GetLockWord(false).ForwardingAddress(); 52152e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier mirror::Object* dest_obj = reinterpret_cast<mirror::Object*>(dest_addr); 52252e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier DCHECK(space_->HasAddress(dest_obj)) << dest_obj; 52352e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier // Use memmove since there may be overlap. 52452e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier memmove(reinterpret_cast<void*>(dest_addr), reinterpret_cast<const void*>(obj), len); 52552e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier // Restore the saved lock word if needed. 526e15ea086439b41a805d164d2beb07b4ba96aaa97Hiroshi Yamauchi LockWord lock_word = LockWord::Default(); 52752e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier if (UNLIKELY(objects_with_lockword_->Test(obj))) { 52852e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier lock_word = lock_words_to_restore_.front(); 52952e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier lock_words_to_restore_.pop_front(); 53052e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier } 53152e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier dest_obj->SetLockWord(lock_word, false); 53252e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier} 53352e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier 53452e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartiervoid MarkCompact::MoveObjects() { 535f5997b4d3f889569d5a2b724d83d764bfbb8d106Mathieu Chartier TimingLogger::ScopedTiming t(__FUNCTION__, GetTimings()); 53652e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier // Move the objects in the before forwarding bitmap. 53752e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier MoveObjectVisitor visitor(this); 53852e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier objects_before_forwarding_->VisitMarkedRange(reinterpret_cast<uintptr_t>(space_->Begin()), 53952e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier reinterpret_cast<uintptr_t>(space_->End()), 54052e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier visitor); 54152e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier CHECK(lock_words_to_restore_.empty()); 54252e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier} 54352e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier 54452e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartiervoid MarkCompact::Sweep(bool swap_bitmaps) { 545f5997b4d3f889569d5a2b724d83d764bfbb8d106Mathieu Chartier TimingLogger::ScopedTiming t(__FUNCTION__, GetTimings()); 54652e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier DCHECK(mark_stack_->IsEmpty()); 54752e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier for (const auto& space : GetHeap()->GetContinuousSpaces()) { 54852e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier if (space->IsContinuousMemMapAllocSpace()) { 54952e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier space::ContinuousMemMapAllocSpace* alloc_space = space->AsContinuousMemMapAllocSpace(); 55052e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier if (!ShouldSweepSpace(alloc_space)) { 55152e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier continue; 55252e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier } 553277ccbd200ea43590dfc06a93ae184a765327ad0Andreas Gampe TimingLogger::ScopedTiming t2( 55410fb83ad7442c8cf3356a89ec918e0786f110981Mathieu Chartier alloc_space->IsZygoteSpace() ? "SweepZygoteSpace" : "SweepAllocSpace", GetTimings()); 55510fb83ad7442c8cf3356a89ec918e0786f110981Mathieu Chartier RecordFree(alloc_space->Sweep(swap_bitmaps)); 55652e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier } 55752e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier } 55852e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier SweepLargeObjects(swap_bitmaps); 55952e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier} 56052e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier 56152e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartiervoid MarkCompact::SweepLargeObjects(bool swap_bitmaps) { 5622dbe627954fd78a3659ab3cd42d2ead5b4529441Mathieu Chartier space::LargeObjectSpace* los = heap_->GetLargeObjectsSpace(); 5632dbe627954fd78a3659ab3cd42d2ead5b4529441Mathieu Chartier if (los != nullptr) { 5642dbe627954fd78a3659ab3cd42d2ead5b4529441Mathieu Chartier TimingLogger::ScopedTiming t(__FUNCTION__, GetTimings());\ 5652dbe627954fd78a3659ab3cd42d2ead5b4529441Mathieu Chartier RecordFreeLOS(los->Sweep(swap_bitmaps)); 5662dbe627954fd78a3659ab3cd42d2ead5b4529441Mathieu Chartier } 56752e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier} 56852e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier 56952e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier// Process the "referent" field in a java.lang.ref.Reference. If the referent has not yet been 57052e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier// marked, put it on the appropriate list in the heap for later processing. 57152e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartiervoid MarkCompact::DelayReferenceReferent(mirror::Class* klass, mirror::Reference* reference) { 57297509954404d031594b2ecbda607314d169d512eMathieu Chartier heap_->GetReferenceProcessor()->DelayReferenceReferent(klass, reference, this); 57352e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier} 57452e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier 57552e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartierclass MarkCompactMarkObjectVisitor { 57652e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier public: 57752e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier explicit MarkCompactMarkObjectVisitor(MarkCompact* collector) : collector_(collector) { 57852e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier } 57952e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier 58097509954404d031594b2ecbda607314d169d512eMathieu Chartier void operator()(mirror::Object* obj, MemberOffset offset, bool /*is_static*/) const ALWAYS_INLINE 58190443477f9a0061581c420775ce3b7eeae7468bcMathieu Chartier REQUIRES(Locks::mutator_lock_, Locks::heap_bitmap_lock_) { 58252e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier // Object was already verified when we scanned it. 58352e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier collector_->MarkObject(obj->GetFieldObject<mirror::Object, kVerifyNone>(offset)); 58452e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier } 58552e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier 58652e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier void operator()(mirror::Class* klass, mirror::Reference* ref) const 58790443477f9a0061581c420775ce3b7eeae7468bcMathieu Chartier SHARED_REQUIRES(Locks::mutator_lock_) 58890443477f9a0061581c420775ce3b7eeae7468bcMathieu Chartier REQUIRES(Locks::heap_bitmap_lock_) { 58952e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier collector_->DelayReferenceReferent(klass, ref); 59052e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier } 59152e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier 592da7c650022a974be10e2f00fa07d5109e3d8826fMathieu Chartier // TODO: Remove NO_THREAD_SAFETY_ANALYSIS when clang better understands visitors. 593da7c650022a974be10e2f00fa07d5109e3d8826fMathieu Chartier void VisitRootIfNonNull(mirror::CompressedReference<mirror::Object>* root) const 594da7c650022a974be10e2f00fa07d5109e3d8826fMathieu Chartier NO_THREAD_SAFETY_ANALYSIS { 595da7c650022a974be10e2f00fa07d5109e3d8826fMathieu Chartier if (!root->IsNull()) { 596da7c650022a974be10e2f00fa07d5109e3d8826fMathieu Chartier VisitRoot(root); 597da7c650022a974be10e2f00fa07d5109e3d8826fMathieu Chartier } 598da7c650022a974be10e2f00fa07d5109e3d8826fMathieu Chartier } 599da7c650022a974be10e2f00fa07d5109e3d8826fMathieu Chartier 600da7c650022a974be10e2f00fa07d5109e3d8826fMathieu Chartier void VisitRoot(mirror::CompressedReference<mirror::Object>* root) const 601da7c650022a974be10e2f00fa07d5109e3d8826fMathieu Chartier NO_THREAD_SAFETY_ANALYSIS { 602da7c650022a974be10e2f00fa07d5109e3d8826fMathieu Chartier collector_->MarkObject(root->AsMirrorPtr()); 603da7c650022a974be10e2f00fa07d5109e3d8826fMathieu Chartier } 604da7c650022a974be10e2f00fa07d5109e3d8826fMathieu Chartier 60552e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier private: 60652e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier MarkCompact* const collector_; 60752e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier}; 60852e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier 60952e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier// Visit all of the references of an object and update. 61097509954404d031594b2ecbda607314d169d512eMathieu Chartiervoid MarkCompact::ScanObject(mirror::Object* obj) { 61152e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier MarkCompactMarkObjectVisitor visitor(this); 612059ef3ddb2088f926ac452889e0953fdcd646a5eMathieu Chartier obj->VisitReferences(visitor, visitor); 61352e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier} 61452e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier 61552e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier// Scan anything that's on the mark stack. 61652e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartiervoid MarkCompact::ProcessMarkStack() { 617f5997b4d3f889569d5a2b724d83d764bfbb8d106Mathieu Chartier TimingLogger::ScopedTiming t(__FUNCTION__, GetTimings()); 61852e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier while (!mark_stack_->IsEmpty()) { 61997509954404d031594b2ecbda607314d169d512eMathieu Chartier mirror::Object* obj = mark_stack_->PopBack(); 62052e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier DCHECK(obj != nullptr); 62152e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier ScanObject(obj); 62252e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier } 62352e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier} 62452e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier 62552e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartiervoid MarkCompact::SetSpace(space::BumpPointerSpace* space) { 62652e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier DCHECK(space != nullptr); 62752e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier space_ = space; 62852e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier} 62952e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier 63052e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartiervoid MarkCompact::FinishPhase() { 631f5997b4d3f889569d5a2b724d83d764bfbb8d106Mathieu Chartier TimingLogger::ScopedTiming t(__FUNCTION__, GetTimings()); 63252e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier space_ = nullptr; 63352e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier CHECK(mark_stack_->IsEmpty()); 63452e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier mark_stack_->Reset(); 63552e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier // Clear all of the spaces' mark bitmaps. 63652e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier WriterMutexLock mu(Thread::Current(), *Locks::heap_bitmap_lock_); 63752e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier heap_->ClearMarkedObjects(); 63852e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier // Release our bitmaps. 63952e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier objects_before_forwarding_.reset(nullptr); 64052e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier objects_with_lockword_.reset(nullptr); 64152e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier} 64252e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier 64352e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartiervoid MarkCompact::RevokeAllThreadLocalBuffers() { 644f5997b4d3f889569d5a2b724d83d764bfbb8d106Mathieu Chartier TimingLogger::ScopedTiming t(__FUNCTION__, GetTimings()); 64552e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier GetHeap()->RevokeAllThreadLocalBuffers(); 64652e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier} 64752e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier 64852e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier} // namespace collector 64952e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier} // namespace gc 65052e4b43d62896b56f8c2bd041e528472bb4a0d8dMathieu Chartier} // namespace art 651