1/*
2 * Copyright (C) 2014 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *      http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17#include "mark_compact.h"
18
19#include "base/logging.h"
20#include "base/mutex-inl.h"
21#include "base/timing_logger.h"
22#include "gc/accounting/heap_bitmap-inl.h"
23#include "gc/accounting/mod_union_table.h"
24#include "gc/accounting/remembered_set.h"
25#include "gc/accounting/space_bitmap-inl.h"
26#include "gc/heap.h"
27#include "gc/reference_processor.h"
28#include "gc/space/bump_pointer_space.h"
29#include "gc/space/bump_pointer_space-inl.h"
30#include "gc/space/image_space.h"
31#include "gc/space/large_object_space.h"
32#include "gc/space/space-inl.h"
33#include "indirect_reference_table.h"
34#include "intern_table.h"
35#include "jni_internal.h"
36#include "mark_sweep-inl.h"
37#include "monitor.h"
38#include "mirror/class-inl.h"
39#include "mirror/class_loader.h"
40#include "mirror/dex_cache.h"
41#include "mirror/reference-inl.h"
42#include "mirror/object-inl.h"
43#include "mirror/object_array.h"
44#include "mirror/object_array-inl.h"
45#include "runtime.h"
46#include "stack.h"
47#include "thread-inl.h"
48#include "thread_list.h"
49
50using ::art::mirror::Object;
51
52namespace art {
53namespace gc {
54namespace collector {
55
56void MarkCompact::BindBitmaps() {
57  TimingLogger::ScopedTiming t(__FUNCTION__, GetTimings());
58  WriterMutexLock mu(Thread::Current(), *Locks::heap_bitmap_lock_);
59  // Mark all of the spaces we never collect as immune.
60  for (const auto& space : GetHeap()->GetContinuousSpaces()) {
61    if (space->GetGcRetentionPolicy() == space::kGcRetentionPolicyNeverCollect ||
62        space->GetGcRetentionPolicy() == space::kGcRetentionPolicyFullCollect) {
63      CHECK(immune_region_.AddContinuousSpace(space)) << "Failed to add space " << *space;
64    }
65  }
66}
67
68MarkCompact::MarkCompact(Heap* heap, const std::string& name_prefix)
69    : GarbageCollector(heap, name_prefix + (name_prefix.empty() ? "" : " ") + "mark compact"),
70      space_(nullptr), collector_name_(name_) {
71}
72
73void MarkCompact::RunPhases() {
74  Thread* self = Thread::Current();
75  InitializePhase();
76  CHECK(!Locks::mutator_lock_->IsExclusiveHeld(self));
77  {
78    ScopedPause pause(this);
79    GetHeap()->PreGcVerificationPaused(this);
80    GetHeap()->PrePauseRosAllocVerification(this);
81    MarkingPhase();
82    ReclaimPhase();
83  }
84  GetHeap()->PostGcVerification(this);
85  FinishPhase();
86}
87
88void MarkCompact::ForwardObject(mirror::Object* obj) {
89  const size_t alloc_size = RoundUp(obj->SizeOf(), space::BumpPointerSpace::kAlignment);
90  LockWord lock_word = obj->GetLockWord(false);
91  // If we have a non empty lock word, store it and restore it later.
92  if (!LockWord::IsDefault(lock_word)) {
93    // Set the bit in the bitmap so that we know to restore it later.
94    objects_with_lockword_->Set(obj);
95    lock_words_to_restore_.push_back(lock_word);
96  }
97  obj->SetLockWord(LockWord::FromForwardingAddress(reinterpret_cast<size_t>(bump_pointer_)),
98                   false);
99  bump_pointer_ += alloc_size;
100  ++live_objects_in_space_;
101}
102
103class CalculateObjectForwardingAddressVisitor {
104 public:
105  explicit CalculateObjectForwardingAddressVisitor(MarkCompact* collector)
106      : collector_(collector) {}
107  void operator()(mirror::Object* obj) const EXCLUSIVE_LOCKS_REQUIRED(Locks::mutator_lock_,
108                                                                      Locks::heap_bitmap_lock_) {
109    DCHECK_ALIGNED(obj, space::BumpPointerSpace::kAlignment);
110    DCHECK(collector_->IsMarked(obj));
111    collector_->ForwardObject(obj);
112  }
113
114 private:
115  MarkCompact* const collector_;
116};
117
118void MarkCompact::CalculateObjectForwardingAddresses() {
119  TimingLogger::ScopedTiming t(__FUNCTION__, GetTimings());
120  // The bump pointer in the space where the next forwarding address will be.
121  bump_pointer_ = reinterpret_cast<uint8_t*>(space_->Begin());
122  // Visit all the marked objects in the bitmap.
123  CalculateObjectForwardingAddressVisitor visitor(this);
124  objects_before_forwarding_->VisitMarkedRange(reinterpret_cast<uintptr_t>(space_->Begin()),
125                                               reinterpret_cast<uintptr_t>(space_->End()),
126                                               visitor);
127}
128
129void MarkCompact::InitializePhase() {
130  TimingLogger::ScopedTiming t(__FUNCTION__, GetTimings());
131  mark_stack_ = heap_->GetMarkStack();
132  DCHECK(mark_stack_ != nullptr);
133  immune_region_.Reset();
134  CHECK(space_->CanMoveObjects()) << "Attempting compact non-movable space from " << *space_;
135  // TODO: I don't think we should need heap bitmap lock to Get the mark bitmap.
136  ReaderMutexLock mu(Thread::Current(), *Locks::heap_bitmap_lock_);
137  mark_bitmap_ = heap_->GetMarkBitmap();
138  live_objects_in_space_ = 0;
139}
140
141void MarkCompact::ProcessReferences(Thread* self) {
142  WriterMutexLock mu(self, *Locks::heap_bitmap_lock_);
143  heap_->GetReferenceProcessor()->ProcessReferences(
144      false, GetTimings(), GetCurrentIteration()->GetClearSoftReferences(),
145      &HeapReferenceMarkedCallback, &MarkObjectCallback, &ProcessMarkStackCallback, this);
146}
147
148class BitmapSetSlowPathVisitor {
149 public:
150  void operator()(const mirror::Object* obj) const {
151    // Marking a large object, make sure its aligned as a sanity check.
152    if (!IsAligned<kPageSize>(obj)) {
153      Runtime::Current()->GetHeap()->DumpSpaces(LOG(ERROR));
154      LOG(FATAL) << obj;
155    }
156  }
157};
158
159inline void MarkCompact::MarkObject(mirror::Object* obj) {
160  if (obj == nullptr) {
161    return;
162  }
163  if (kUseBakerOrBrooksReadBarrier) {
164    // Verify all the objects have the correct forward pointer installed.
165    obj->AssertReadBarrierPointer();
166  }
167  if (immune_region_.ContainsObject(obj)) {
168    return;
169  }
170  if (objects_before_forwarding_->HasAddress(obj)) {
171    if (!objects_before_forwarding_->Set(obj)) {
172      MarkStackPush(obj);  // This object was not previously marked.
173    }
174  } else {
175    DCHECK(!space_->HasAddress(obj));
176    BitmapSetSlowPathVisitor visitor;
177    if (!mark_bitmap_->Set(obj, visitor)) {
178      // This object was not previously marked.
179      MarkStackPush(obj);
180    }
181  }
182}
183
184void MarkCompact::MarkingPhase() {
185  TimingLogger::ScopedTiming t(__FUNCTION__, GetTimings());
186  Thread* self = Thread::Current();
187  // Bitmap which describes which objects we have to move.
188  objects_before_forwarding_.reset(accounting::ContinuousSpaceBitmap::Create(
189      "objects before forwarding", space_->Begin(), space_->Size()));
190  // Bitmap which describes which lock words we need to restore.
191  objects_with_lockword_.reset(accounting::ContinuousSpaceBitmap::Create(
192      "objects with lock words", space_->Begin(), space_->Size()));
193  CHECK(Locks::mutator_lock_->IsExclusiveHeld(self));
194  // Assume the cleared space is already empty.
195  BindBitmaps();
196  t.NewTiming("ProcessCards");
197  // Process dirty cards and add dirty cards to mod-union tables.
198  heap_->ProcessCards(GetTimings(), false, false, true);
199  // Clear the whole card table since we can not Get any additional dirty cards during the
200  // paused GC. This saves memory but only works for pause the world collectors.
201  t.NewTiming("ClearCardTable");
202  heap_->GetCardTable()->ClearCardTable();
203  // Need to do this before the checkpoint since we don't want any threads to add references to
204  // the live stack during the recursive mark.
205  if (kUseThreadLocalAllocationStack) {
206    t.NewTiming("RevokeAllThreadLocalAllocationStacks");
207    heap_->RevokeAllThreadLocalAllocationStacks(self);
208  }
209  t.NewTiming("SwapStacks");
210  heap_->SwapStacks(self);
211  {
212    WriterMutexLock mu(self, *Locks::heap_bitmap_lock_);
213    MarkRoots();
214    // Mark roots of immune spaces.
215    UpdateAndMarkModUnion();
216    // Recursively mark remaining objects.
217    MarkReachableObjects();
218  }
219  ProcessReferences(self);
220  {
221    ReaderMutexLock mu(self, *Locks::heap_bitmap_lock_);
222    SweepSystemWeaks();
223  }
224  // Revoke buffers before measuring how many objects were moved since the TLABs need to be revoked
225  // before they are properly counted.
226  RevokeAllThreadLocalBuffers();
227  // Disabled due to an issue where we have objects in the bump pointer space which reference dead
228  // objects.
229  // heap_->PreSweepingGcVerification(this);
230}
231
232void MarkCompact::UpdateAndMarkModUnion() {
233  TimingLogger::ScopedTiming t(__FUNCTION__, GetTimings());
234  for (auto& space : heap_->GetContinuousSpaces()) {
235    // If the space is immune then we need to mark the references to other spaces.
236    if (immune_region_.ContainsSpace(space)) {
237      accounting::ModUnionTable* table = heap_->FindModUnionTableFromSpace(space);
238      if (table != nullptr) {
239        // TODO: Improve naming.
240        TimingLogger::ScopedTiming t2(
241            space->IsZygoteSpace() ? "UpdateAndMarkZygoteModUnionTable" :
242                                     "UpdateAndMarkImageModUnionTable", GetTimings());
243        table->UpdateAndMarkReferences(MarkHeapReferenceCallback, this);
244      }
245    }
246  }
247}
248
249void MarkCompact::MarkReachableObjects() {
250  TimingLogger::ScopedTiming t(__FUNCTION__, GetTimings());
251  accounting::ObjectStack* live_stack = heap_->GetLiveStack();
252  {
253    TimingLogger::ScopedTiming t2("MarkAllocStackAsLive", GetTimings());
254    heap_->MarkAllocStackAsLive(live_stack);
255  }
256  live_stack->Reset();
257  // Recursively process the mark stack.
258  ProcessMarkStack();
259}
260
261void MarkCompact::ReclaimPhase() {
262  TimingLogger::ScopedTiming t(__FUNCTION__, GetTimings());
263  WriterMutexLock mu(Thread::Current(), *Locks::heap_bitmap_lock_);
264  // Reclaim unmarked objects.
265  Sweep(false);
266  // Swap the live and mark bitmaps for each space which we modified space. This is an
267  // optimization that enables us to not clear live bits inside of the sweep. Only swaps unbound
268  // bitmaps.
269  SwapBitmaps();
270  GetHeap()->UnBindBitmaps();  // Unbind the live and mark bitmaps.
271  Compact();
272}
273
274void MarkCompact::ResizeMarkStack(size_t new_size) {
275  std::vector<StackReference<Object>> temp(mark_stack_->Begin(), mark_stack_->End());
276  CHECK_LE(mark_stack_->Size(), new_size);
277  mark_stack_->Resize(new_size);
278  for (auto& obj : temp) {
279    mark_stack_->PushBack(obj.AsMirrorPtr());
280  }
281}
282
283inline void MarkCompact::MarkStackPush(Object* obj) {
284  if (UNLIKELY(mark_stack_->Size() >= mark_stack_->Capacity())) {
285    ResizeMarkStack(mark_stack_->Capacity() * 2);
286  }
287  // The object must be pushed on to the mark stack.
288  mark_stack_->PushBack(obj);
289}
290
291void MarkCompact::ProcessMarkStackCallback(void* arg) {
292  reinterpret_cast<MarkCompact*>(arg)->ProcessMarkStack();
293}
294
295mirror::Object* MarkCompact::MarkObjectCallback(mirror::Object* root, void* arg) {
296  reinterpret_cast<MarkCompact*>(arg)->MarkObject(root);
297  return root;
298}
299
300void MarkCompact::MarkHeapReferenceCallback(mirror::HeapReference<mirror::Object>* obj_ptr,
301                                            void* arg) {
302  reinterpret_cast<MarkCompact*>(arg)->MarkObject(obj_ptr->AsMirrorPtr());
303}
304
305void MarkCompact::DelayReferenceReferentCallback(mirror::Class* klass, mirror::Reference* ref,
306                                                 void* arg) {
307  reinterpret_cast<MarkCompact*>(arg)->DelayReferenceReferent(klass, ref);
308}
309
310void MarkCompact::VisitRoots(
311    mirror::Object*** roots, size_t count, const RootInfo& info ATTRIBUTE_UNUSED) {
312  for (size_t i = 0; i < count; ++i) {
313    MarkObject(*roots[i]);
314  }
315}
316
317void MarkCompact::VisitRoots(
318    mirror::CompressedReference<mirror::Object>** roots, size_t count,
319    const RootInfo& info ATTRIBUTE_UNUSED) {
320  for (size_t i = 0; i < count; ++i) {
321    MarkObject(roots[i]->AsMirrorPtr());
322  }
323}
324
325class UpdateRootVisitor : public RootVisitor {
326 public:
327  explicit UpdateRootVisitor(MarkCompact* collector) : collector_(collector) {
328  }
329
330  void VisitRoots(mirror::Object*** roots, size_t count, const RootInfo& info ATTRIBUTE_UNUSED)
331      OVERRIDE EXCLUSIVE_LOCKS_REQUIRED(Locks::mutator_lock_)
332      SHARED_LOCKS_REQUIRED(Locks::heap_bitmap_lock_) {
333    for (size_t i = 0; i < count; ++i) {
334      mirror::Object* obj = *roots[i];
335      mirror::Object* new_obj = collector_->GetMarkedForwardAddress(obj);
336      if (obj != new_obj) {
337        *roots[i] = new_obj;
338        DCHECK(new_obj != nullptr);
339      }
340    }
341  }
342
343  void VisitRoots(mirror::CompressedReference<mirror::Object>** roots, size_t count,
344                  const RootInfo& info ATTRIBUTE_UNUSED)
345      OVERRIDE EXCLUSIVE_LOCKS_REQUIRED(Locks::mutator_lock_)
346      SHARED_LOCKS_REQUIRED(Locks::heap_bitmap_lock_) {
347    for (size_t i = 0; i < count; ++i) {
348      mirror::Object* obj = roots[i]->AsMirrorPtr();
349      mirror::Object* new_obj = collector_->GetMarkedForwardAddress(obj);
350      if (obj != new_obj) {
351        roots[i]->Assign(new_obj);
352        DCHECK(new_obj != nullptr);
353      }
354    }
355  }
356
357 private:
358  MarkCompact* const collector_;
359};
360
361class UpdateObjectReferencesVisitor {
362 public:
363  explicit UpdateObjectReferencesVisitor(MarkCompact* collector) : collector_(collector) {
364  }
365  void operator()(mirror::Object* obj) const SHARED_LOCKS_REQUIRED(Locks::heap_bitmap_lock_)
366          EXCLUSIVE_LOCKS_REQUIRED(Locks::mutator_lock_) ALWAYS_INLINE {
367    collector_->UpdateObjectReferences(obj);
368  }
369
370 private:
371  MarkCompact* const collector_;
372};
373
374void MarkCompact::UpdateReferences() {
375  TimingLogger::ScopedTiming t(__FUNCTION__, GetTimings());
376  Runtime* runtime = Runtime::Current();
377  // Update roots.
378  UpdateRootVisitor update_root_visitor(this);
379  runtime->VisitRoots(&update_root_visitor);
380  // Update object references in mod union tables and spaces.
381  for (const auto& space : heap_->GetContinuousSpaces()) {
382    // If the space is immune then we need to mark the references to other spaces.
383    accounting::ModUnionTable* table = heap_->FindModUnionTableFromSpace(space);
384    if (table != nullptr) {
385      // TODO: Improve naming.
386      TimingLogger::ScopedTiming t2(
387          space->IsZygoteSpace() ? "UpdateZygoteModUnionTableReferences" :
388                                   "UpdateImageModUnionTableReferences",
389                                   GetTimings());
390      table->UpdateAndMarkReferences(&UpdateHeapReferenceCallback, this);
391    } else {
392      // No mod union table, so we need to scan the space using bitmap visit.
393      // Scan the space using bitmap visit.
394      accounting::ContinuousSpaceBitmap* bitmap = space->GetLiveBitmap();
395      if (bitmap != nullptr) {
396        UpdateObjectReferencesVisitor visitor(this);
397        bitmap->VisitMarkedRange(reinterpret_cast<uintptr_t>(space->Begin()),
398                                 reinterpret_cast<uintptr_t>(space->End()),
399                                 visitor);
400      }
401    }
402  }
403  CHECK(!kMovingClasses)
404      << "Didn't update large object classes since they are assumed to not move.";
405  // Update the system weaks, these should already have been swept.
406  runtime->SweepSystemWeaks(&MarkedForwardingAddressCallback, this);
407  // Update the objects in the bump pointer space last, these objects don't have a bitmap.
408  UpdateObjectReferencesVisitor visitor(this);
409  objects_before_forwarding_->VisitMarkedRange(reinterpret_cast<uintptr_t>(space_->Begin()),
410                                               reinterpret_cast<uintptr_t>(space_->End()),
411                                               visitor);
412  // Update the reference processor cleared list.
413  heap_->GetReferenceProcessor()->UpdateRoots(&MarkedForwardingAddressCallback, this);
414}
415
416void MarkCompact::Compact() {
417  TimingLogger::ScopedTiming t(__FUNCTION__, GetTimings());
418  CalculateObjectForwardingAddresses();
419  UpdateReferences();
420  MoveObjects();
421  // Space
422  int64_t objects_freed = space_->GetObjectsAllocated() - live_objects_in_space_;
423  int64_t bytes_freed = reinterpret_cast<int64_t>(space_->End()) -
424      reinterpret_cast<int64_t>(bump_pointer_);
425  t.NewTiming("RecordFree");
426  space_->RecordFree(objects_freed, bytes_freed);
427  RecordFree(ObjectBytePair(objects_freed, bytes_freed));
428  space_->SetEnd(bump_pointer_);
429  // Need to zero out the memory we freed. TODO: Use madvise for pages.
430  memset(bump_pointer_, 0, bytes_freed);
431}
432
433// Marks all objects in the root set.
434void MarkCompact::MarkRoots() {
435  TimingLogger::ScopedTiming t(__FUNCTION__, GetTimings());
436  Runtime::Current()->VisitRoots(this);
437}
438
439mirror::Object* MarkCompact::MarkedForwardingAddressCallback(mirror::Object* obj, void* arg) {
440  return reinterpret_cast<MarkCompact*>(arg)->GetMarkedForwardAddress(obj);
441}
442
443inline void MarkCompact::UpdateHeapReference(mirror::HeapReference<mirror::Object>* reference) {
444  mirror::Object* obj = reference->AsMirrorPtr();
445  if (obj != nullptr) {
446    mirror::Object* new_obj = GetMarkedForwardAddress(obj);
447    if (obj != new_obj) {
448      DCHECK(new_obj != nullptr);
449      reference->Assign(new_obj);
450    }
451  }
452}
453
454void MarkCompact::UpdateHeapReferenceCallback(mirror::HeapReference<mirror::Object>* reference,
455                                              void* arg) {
456  reinterpret_cast<MarkCompact*>(arg)->UpdateHeapReference(reference);
457}
458
459class UpdateReferenceVisitor {
460 public:
461  explicit UpdateReferenceVisitor(MarkCompact* collector) : collector_(collector) {
462  }
463
464  void operator()(Object* obj, MemberOffset offset, bool /*is_static*/) const
465      ALWAYS_INLINE EXCLUSIVE_LOCKS_REQUIRED(Locks::mutator_lock_, Locks::heap_bitmap_lock_) {
466    collector_->UpdateHeapReference(obj->GetFieldObjectReferenceAddr<kVerifyNone>(offset));
467  }
468
469  void operator()(mirror::Class* /*klass*/, mirror::Reference* ref) const
470      EXCLUSIVE_LOCKS_REQUIRED(Locks::mutator_lock_, Locks::heap_bitmap_lock_) {
471    collector_->UpdateHeapReference(
472        ref->GetFieldObjectReferenceAddr<kVerifyNone>(mirror::Reference::ReferentOffset()));
473  }
474
475 private:
476  MarkCompact* const collector_;
477};
478
479void MarkCompact::UpdateObjectReferences(mirror::Object* obj) {
480  UpdateReferenceVisitor visitor(this);
481  obj->VisitReferences<kMovingClasses>(visitor, visitor);
482}
483
484inline mirror::Object* MarkCompact::GetMarkedForwardAddress(mirror::Object* obj) const {
485  DCHECK(obj != nullptr);
486  if (objects_before_forwarding_->HasAddress(obj)) {
487    DCHECK(objects_before_forwarding_->Test(obj));
488    mirror::Object* ret =
489        reinterpret_cast<mirror::Object*>(obj->GetLockWord(false).ForwardingAddress());
490    DCHECK(ret != nullptr);
491    return ret;
492  }
493  DCHECK(!space_->HasAddress(obj));
494  DCHECK(IsMarked(obj));
495  return obj;
496}
497
498inline bool MarkCompact::IsMarked(const Object* object) const {
499  if (immune_region_.ContainsObject(object)) {
500    return true;
501  }
502  if (objects_before_forwarding_->HasAddress(object)) {
503    return objects_before_forwarding_->Test(object);
504  }
505  return mark_bitmap_->Test(object);
506}
507
508mirror::Object* MarkCompact::IsMarkedCallback(mirror::Object* object, void* arg) {
509  return reinterpret_cast<MarkCompact*>(arg)->IsMarked(object) ? object : nullptr;
510}
511
512bool MarkCompact::HeapReferenceMarkedCallback(mirror::HeapReference<mirror::Object>* ref_ptr,
513                                              void* arg) {
514  // Side effect free since we call this before ever moving objects.
515  return reinterpret_cast<MarkCompact*>(arg)->IsMarked(ref_ptr->AsMirrorPtr());
516}
517
518void MarkCompact::SweepSystemWeaks() {
519  TimingLogger::ScopedTiming t(__FUNCTION__, GetTimings());
520  Runtime::Current()->SweepSystemWeaks(IsMarkedCallback, this);
521}
522
523bool MarkCompact::ShouldSweepSpace(space::ContinuousSpace* space) const {
524  return space != space_ && !immune_region_.ContainsSpace(space);
525}
526
527class MoveObjectVisitor {
528 public:
529  explicit MoveObjectVisitor(MarkCompact* collector) : collector_(collector) {
530  }
531  void operator()(mirror::Object* obj) const SHARED_LOCKS_REQUIRED(Locks::heap_bitmap_lock_)
532          EXCLUSIVE_LOCKS_REQUIRED(Locks::mutator_lock_) ALWAYS_INLINE {
533      collector_->MoveObject(obj, obj->SizeOf());
534  }
535
536 private:
537  MarkCompact* const collector_;
538};
539
540void MarkCompact::MoveObject(mirror::Object* obj, size_t len) {
541  // Look at the forwarding address stored in the lock word to know where to copy.
542  DCHECK(space_->HasAddress(obj)) << obj;
543  uintptr_t dest_addr = obj->GetLockWord(false).ForwardingAddress();
544  mirror::Object* dest_obj = reinterpret_cast<mirror::Object*>(dest_addr);
545  DCHECK(space_->HasAddress(dest_obj)) << dest_obj;
546  // Use memmove since there may be overlap.
547  memmove(reinterpret_cast<void*>(dest_addr), reinterpret_cast<const void*>(obj), len);
548  // Restore the saved lock word if needed.
549  LockWord lock_word = LockWord::Default();
550  if (UNLIKELY(objects_with_lockword_->Test(obj))) {
551    lock_word = lock_words_to_restore_.front();
552    lock_words_to_restore_.pop_front();
553  }
554  dest_obj->SetLockWord(lock_word, false);
555}
556
557void MarkCompact::MoveObjects() {
558  TimingLogger::ScopedTiming t(__FUNCTION__, GetTimings());
559  // Move the objects in the before forwarding bitmap.
560  MoveObjectVisitor visitor(this);
561  objects_before_forwarding_->VisitMarkedRange(reinterpret_cast<uintptr_t>(space_->Begin()),
562                                               reinterpret_cast<uintptr_t>(space_->End()),
563                                               visitor);
564  CHECK(lock_words_to_restore_.empty());
565}
566
567void MarkCompact::Sweep(bool swap_bitmaps) {
568  TimingLogger::ScopedTiming t(__FUNCTION__, GetTimings());
569  DCHECK(mark_stack_->IsEmpty());
570  for (const auto& space : GetHeap()->GetContinuousSpaces()) {
571    if (space->IsContinuousMemMapAllocSpace()) {
572      space::ContinuousMemMapAllocSpace* alloc_space = space->AsContinuousMemMapAllocSpace();
573      if (!ShouldSweepSpace(alloc_space)) {
574        continue;
575      }
576      TimingLogger::ScopedTiming t2(
577          alloc_space->IsZygoteSpace() ? "SweepZygoteSpace" : "SweepAllocSpace", GetTimings());
578      RecordFree(alloc_space->Sweep(swap_bitmaps));
579    }
580  }
581  SweepLargeObjects(swap_bitmaps);
582}
583
584void MarkCompact::SweepLargeObjects(bool swap_bitmaps) {
585  space::LargeObjectSpace* los = heap_->GetLargeObjectsSpace();
586  if (los != nullptr) {
587    TimingLogger::ScopedTiming t(__FUNCTION__, GetTimings());\
588    RecordFreeLOS(los->Sweep(swap_bitmaps));
589  }
590}
591
592// Process the "referent" field in a java.lang.ref.Reference.  If the referent has not yet been
593// marked, put it on the appropriate list in the heap for later processing.
594void MarkCompact::DelayReferenceReferent(mirror::Class* klass, mirror::Reference* reference) {
595  heap_->GetReferenceProcessor()->DelayReferenceReferent(klass, reference,
596                                                         &HeapReferenceMarkedCallback, this);
597}
598
599class MarkCompactMarkObjectVisitor {
600 public:
601  explicit MarkCompactMarkObjectVisitor(MarkCompact* collector) : collector_(collector) {
602  }
603
604  void operator()(Object* obj, MemberOffset offset, bool /*is_static*/) const ALWAYS_INLINE
605      EXCLUSIVE_LOCKS_REQUIRED(Locks::mutator_lock_, Locks::heap_bitmap_lock_) {
606    // Object was already verified when we scanned it.
607    collector_->MarkObject(obj->GetFieldObject<mirror::Object, kVerifyNone>(offset));
608  }
609
610  void operator()(mirror::Class* klass, mirror::Reference* ref) const
611      SHARED_LOCKS_REQUIRED(Locks::mutator_lock_)
612      EXCLUSIVE_LOCKS_REQUIRED(Locks::heap_bitmap_lock_) {
613    collector_->DelayReferenceReferent(klass, ref);
614  }
615
616 private:
617  MarkCompact* const collector_;
618};
619
620// Visit all of the references of an object and update.
621void MarkCompact::ScanObject(Object* obj) {
622  MarkCompactMarkObjectVisitor visitor(this);
623  obj->VisitReferences<kMovingClasses>(visitor, visitor);
624}
625
626// Scan anything that's on the mark stack.
627void MarkCompact::ProcessMarkStack() {
628  TimingLogger::ScopedTiming t(__FUNCTION__, GetTimings());
629  while (!mark_stack_->IsEmpty()) {
630    Object* obj = mark_stack_->PopBack();
631    DCHECK(obj != nullptr);
632    ScanObject(obj);
633  }
634}
635
636void MarkCompact::SetSpace(space::BumpPointerSpace* space) {
637  DCHECK(space != nullptr);
638  space_ = space;
639}
640
641void MarkCompact::FinishPhase() {
642  TimingLogger::ScopedTiming t(__FUNCTION__, GetTimings());
643  space_ = nullptr;
644  CHECK(mark_stack_->IsEmpty());
645  mark_stack_->Reset();
646  // Clear all of the spaces' mark bitmaps.
647  WriterMutexLock mu(Thread::Current(), *Locks::heap_bitmap_lock_);
648  heap_->ClearMarkedObjects();
649  // Release our bitmaps.
650  objects_before_forwarding_.reset(nullptr);
651  objects_with_lockword_.reset(nullptr);
652}
653
654void MarkCompact::RevokeAllThreadLocalBuffers() {
655  TimingLogger::ScopedTiming t(__FUNCTION__, GetTimings());
656  GetHeap()->RevokeAllThreadLocalBuffers();
657}
658
659}  // namespace collector
660}  // namespace gc
661}  // namespace art
662