1b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch// Copyright 2012 the V8 project authors. All rights reserved. 2b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch// Use of this source code is governed by a BSD-style license that can be 3b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch// found in the LICENSE file. 4b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 5b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch#include "src/v8.h" 6b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 7b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch#include "src/heap/incremental-marking.h" 8b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 9b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch#include "src/code-stubs.h" 10b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch#include "src/compilation-cache.h" 11b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch#include "src/conversions.h" 12b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch#include "src/heap/objects-visiting.h" 13b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch#include "src/heap/objects-visiting-inl.h" 143ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch 153ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdochnamespace v8 { 163ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdochnamespace internal { 173ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch 183ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch 193ef787dbeca8a5fb1086949cda830dccee07bfbdBen MurdochIncrementalMarking::IncrementalMarking(Heap* heap) 203ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch : heap_(heap), 213ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch state_(STOPPED), 223ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch marking_deque_memory_(NULL), 233ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch marking_deque_memory_committed_(false), 243ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch steps_count_(0), 253ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch old_generation_space_available_at_start_of_incremental_(0), 263ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch old_generation_space_used_at_start_of_incremental_(0), 273ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch should_hurry_(false), 28b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch marking_speed_(0), 293ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch allocated_(0), 30b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch no_marking_scope_depth_(0), 31b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch unscanned_bytes_of_large_object_(0) {} 323ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch 333ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch 34b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdochvoid IncrementalMarking::TearDown() { delete marking_deque_memory_; } 353ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch 363ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch 37b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdochvoid IncrementalMarking::RecordWriteSlow(HeapObject* obj, Object** slot, 383ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch Object* value) { 39b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch if (BaseRecordWrite(obj, slot, value) && slot != NULL) { 403ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch MarkBit obj_bit = Marking::MarkBitFrom(obj); 413ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch if (Marking::IsBlack(obj_bit)) { 423ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch // Object is not going to be rescanned we need to record the slot. 43b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch heap_->mark_compact_collector()->RecordSlot(HeapObject::RawField(obj, 0), 44b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch slot, value); 453ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch } 463ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch } 473ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch} 483ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch 493ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch 50b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdochvoid IncrementalMarking::RecordWriteFromCode(HeapObject* obj, Object** slot, 513ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch Isolate* isolate) { 52b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch DCHECK(obj->IsHeapObject()); 533ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch IncrementalMarking* marking = isolate->heap()->incremental_marking(); 543ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch 55b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch MemoryChunk* chunk = MemoryChunk::FromAddress(obj->address()); 56b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch int counter = chunk->write_barrier_counter(); 57b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch if (counter < (MemoryChunk::kWriteBarrierCounterGranularity / 2)) { 58b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch marking->write_barriers_invoked_since_last_step_ += 59b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch MemoryChunk::kWriteBarrierCounterGranularity - 60b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch chunk->write_barrier_counter(); 61b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch chunk->set_write_barrier_counter( 62b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch MemoryChunk::kWriteBarrierCounterGranularity); 63b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch } 643ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch 653ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch marking->RecordWrite(obj, slot, *slot); 663ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch} 673ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch 683ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch 69b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdochvoid IncrementalMarking::RecordCodeTargetPatch(Code* host, Address pc, 703ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch HeapObject* value) { 713ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch if (IsMarking()) { 723ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch RelocInfo rinfo(pc, RelocInfo::CODE_TARGET, 0, host); 733ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch RecordWriteIntoCode(host, &rinfo, value); 743ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch } 753ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch} 763ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch 773ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch 783ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdochvoid IncrementalMarking::RecordCodeTargetPatch(Address pc, HeapObject* value) { 793ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch if (IsMarking()) { 80b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch Code* host = heap_->isolate() 81b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch ->inner_pointer_to_code_cache() 82b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch ->GcSafeFindCodeForInnerPointer(pc); 833ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch RelocInfo rinfo(pc, RelocInfo::CODE_TARGET, 0, host); 843ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch RecordWriteIntoCode(host, &rinfo, value); 853ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch } 863ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch} 873ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch 883ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch 893ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdochvoid IncrementalMarking::RecordWriteOfCodeEntrySlow(JSFunction* host, 90b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch Object** slot, 91b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch Code* value) { 92b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch if (BaseRecordWrite(host, slot, value)) { 93b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch DCHECK(slot != NULL); 94b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch heap_->mark_compact_collector()->RecordCodeEntrySlot( 95b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch reinterpret_cast<Address>(slot), value); 963ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch } 973ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch} 983ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch 993ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch 1003ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdochvoid IncrementalMarking::RecordWriteIntoCodeSlow(HeapObject* obj, 1013ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch RelocInfo* rinfo, 1023ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch Object* value) { 1033ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch MarkBit value_bit = Marking::MarkBitFrom(HeapObject::cast(value)); 1043ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch if (Marking::IsWhite(value_bit)) { 1053ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch MarkBit obj_bit = Marking::MarkBitFrom(obj); 1063ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch if (Marking::IsBlack(obj_bit)) { 1073ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch BlackToGreyAndUnshift(obj, obj_bit); 1083ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch RestartIfNotMarking(); 1093ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch } 1103ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch // Object is either grey or white. It will be scanned if survives. 1113ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch return; 1123ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch } 1133ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch 1143ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch if (is_compacting_) { 1153ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch MarkBit obj_bit = Marking::MarkBitFrom(obj); 1163ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch if (Marking::IsBlack(obj_bit)) { 1173ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch // Object is not going to be rescanned. We need to record the slot. 1183ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch heap_->mark_compact_collector()->RecordRelocSlot(rinfo, 1193ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch Code::cast(value)); 1203ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch } 1213ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch } 1223ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch} 1233ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch 1243ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch 125b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdochstatic void MarkObjectGreyDoNotEnqueue(Object* obj) { 126b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch if (obj->IsHeapObject()) { 127b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch HeapObject* heap_obj = HeapObject::cast(obj); 128b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch MarkBit mark_bit = Marking::MarkBitFrom(HeapObject::cast(obj)); 129b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch if (Marking::IsBlack(mark_bit)) { 130b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch MemoryChunk::IncrementLiveBytesFromGC(heap_obj->address(), 131b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch -heap_obj->Size()); 1323ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch } 133b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch Marking::AnyToGrey(mark_bit); 1343ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch } 135b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch} 136b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 137b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 138b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdochstatic inline void MarkBlackOrKeepGrey(HeapObject* heap_object, 139b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch MarkBit mark_bit, int size) { 140b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch DCHECK(!Marking::IsImpossible(mark_bit)); 141b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch if (mark_bit.Get()) return; 142b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch mark_bit.Set(); 143b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch MemoryChunk::IncrementLiveBytesFromGC(heap_object->address(), size); 144b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch DCHECK(Marking::IsBlack(mark_bit)); 145b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch} 146b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 1473ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch 148b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdochstatic inline void MarkBlackOrKeepBlack(HeapObject* heap_object, 149b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch MarkBit mark_bit, int size) { 150b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch DCHECK(!Marking::IsImpossible(mark_bit)); 151b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch if (Marking::IsBlack(mark_bit)) return; 152b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch Marking::MarkBlack(mark_bit); 153b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch MemoryChunk::IncrementLiveBytesFromGC(heap_object->address(), size); 154b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch DCHECK(Marking::IsBlack(mark_bit)); 155b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch} 156b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 157b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 158b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdochclass IncrementalMarkingMarkingVisitor 159b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch : public StaticMarkingVisitor<IncrementalMarkingMarkingVisitor> { 160b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch public: 161b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch static void Initialize() { 162b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch StaticMarkingVisitor<IncrementalMarkingMarkingVisitor>::Initialize(); 163b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch table_.Register(kVisitFixedArray, &VisitFixedArrayIncremental); 164b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch table_.Register(kVisitNativeContext, &VisitNativeContextIncremental); 165b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch table_.Register(kVisitJSRegExp, &VisitJSRegExp); 166b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch } 167b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 168b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch static const int kProgressBarScanningChunk = 32 * 1024; 169b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 170b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch static void VisitFixedArrayIncremental(Map* map, HeapObject* object) { 171b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch MemoryChunk* chunk = MemoryChunk::FromAddress(object->address()); 172b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch // TODO(mstarzinger): Move setting of the flag to the allocation site of 173b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch // the array. The visitor should just check the flag. 174b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch if (FLAG_use_marking_progress_bar && 175b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch chunk->owner()->identity() == LO_SPACE) { 176b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch chunk->SetFlag(MemoryChunk::HAS_PROGRESS_BAR); 177b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch } 178b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch if (chunk->IsFlagSet(MemoryChunk::HAS_PROGRESS_BAR)) { 179b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch Heap* heap = map->GetHeap(); 180b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch // When using a progress bar for large fixed arrays, scan only a chunk of 181b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch // the array and try to push it onto the marking deque again until it is 182b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch // fully scanned. Fall back to scanning it through to the end in case this 183b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch // fails because of a full deque. 184b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch int object_size = FixedArray::BodyDescriptor::SizeOf(map, object); 185b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch int start_offset = 186b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch Max(FixedArray::BodyDescriptor::kStartOffset, chunk->progress_bar()); 187b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch int end_offset = 188b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch Min(object_size, start_offset + kProgressBarScanningChunk); 189b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch int already_scanned_offset = start_offset; 190b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch bool scan_until_end = false; 191b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch do { 192b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch VisitPointersWithAnchor(heap, HeapObject::RawField(object, 0), 193b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch HeapObject::RawField(object, start_offset), 194b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch HeapObject::RawField(object, end_offset)); 195b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch start_offset = end_offset; 196b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch end_offset = Min(object_size, end_offset + kProgressBarScanningChunk); 197b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch scan_until_end = heap->incremental_marking()->marking_deque()->IsFull(); 198b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch } while (scan_until_end && start_offset < object_size); 199b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch chunk->set_progress_bar(start_offset); 200b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch if (start_offset < object_size) { 201b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch heap->incremental_marking()->marking_deque()->UnshiftGrey(object); 202b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch heap->incremental_marking()->NotifyIncompleteScanOfObject( 203b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch object_size - (start_offset - already_scanned_offset)); 204b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch } 205b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch } else { 206b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch FixedArrayVisitor::Visit(map, object); 2073ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch } 2083ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch } 2093ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch 210b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch static void VisitNativeContextIncremental(Map* map, HeapObject* object) { 211b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch Context* context = Context::cast(object); 2123ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch 213b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch // We will mark cache black with a separate pass when we finish marking. 214b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch // Note that GC can happen when the context is not fully initialized, 215b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch // so the cache can be undefined. 216b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch Object* cache = context->get(Context::NORMALIZED_MAP_CACHE_INDEX); 217b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch if (!cache->IsUndefined()) { 218b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch MarkObjectGreyDoNotEnqueue(cache); 219b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch } 220b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch VisitNativeContext(map, context); 2213ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch } 2223ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch 223b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch INLINE(static void VisitPointer(Heap* heap, Object** p)) { 224b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch Object* obj = *p; 225b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch if (obj->IsHeapObject()) { 226b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch heap->mark_compact_collector()->RecordSlot(p, p, obj); 227b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch MarkObject(heap, obj); 2288f9999fcc44cfd4f5e1140c6678bbca4cf8ea1c7Ben Murdoch } 2298f9999fcc44cfd4f5e1140c6678bbca4cf8ea1c7Ben Murdoch } 2308f9999fcc44cfd4f5e1140c6678bbca4cf8ea1c7Ben Murdoch 231b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch INLINE(static void VisitPointers(Heap* heap, Object** start, Object** end)) { 232b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch for (Object** p = start; p < end; p++) { 233b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch Object* obj = *p; 234b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch if (obj->IsHeapObject()) { 235b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch heap->mark_compact_collector()->RecordSlot(start, p, obj); 236b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch MarkObject(heap, obj); 237b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch } 2383ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch } 2393ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch } 2403ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch 241b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch INLINE(static void VisitPointersWithAnchor(Heap* heap, Object** anchor, 242b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch Object** start, Object** end)) { 2433ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch for (Object** p = start; p < end; p++) { 2443ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch Object* obj = *p; 245b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch if (obj->IsHeapObject()) { 246b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch heap->mark_compact_collector()->RecordSlot(anchor, p, obj); 247b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch MarkObject(heap, obj); 2483ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch } 2493ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch } 2503ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch } 2513ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch 252b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch // Marks the object grey and pushes it on the marking stack. 253b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch INLINE(static void MarkObject(Heap* heap, Object* obj)) { 2543ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch HeapObject* heap_object = HeapObject::cast(obj); 2553ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch MarkBit mark_bit = Marking::MarkBitFrom(heap_object); 2563ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch if (mark_bit.data_only()) { 257b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch MarkBlackOrKeepGrey(heap_object, mark_bit, heap_object->Size()); 2583ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch } else if (Marking::IsWhite(mark_bit)) { 259b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch heap->incremental_marking()->WhiteToGreyAndPush(heap_object, mark_bit); 2603ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch } 2613ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch } 2623ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch 263b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch // Marks the object black without pushing it on the marking stack. 264b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch // Returns true if object needed marking and false otherwise. 265b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch INLINE(static bool MarkObjectWithoutPush(Heap* heap, Object* obj)) { 266b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch HeapObject* heap_object = HeapObject::cast(obj); 267b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch MarkBit mark_bit = Marking::MarkBitFrom(heap_object); 268b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch if (Marking::IsWhite(mark_bit)) { 269b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch mark_bit.Set(); 270b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch MemoryChunk::IncrementLiveBytesFromGC(heap_object->address(), 271b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch heap_object->Size()); 272b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch return true; 273b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch } 274b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch return false; 275b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch } 2763ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch}; 2773ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch 2783ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch 2793ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdochclass IncrementalMarkingRootMarkingVisitor : public ObjectVisitor { 2803ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch public: 281b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch explicit IncrementalMarkingRootMarkingVisitor( 282b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch IncrementalMarking* incremental_marking) 283b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch : incremental_marking_(incremental_marking) {} 2843ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch 285b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch void VisitPointer(Object** p) { MarkObjectByPointer(p); } 2863ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch 2873ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch void VisitPointers(Object** start, Object** end) { 2883ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch for (Object** p = start; p < end; p++) MarkObjectByPointer(p); 2893ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch } 2903ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch 2913ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch private: 2923ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch void MarkObjectByPointer(Object** p) { 2933ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch Object* obj = *p; 2943ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch if (!obj->IsHeapObject()) return; 2953ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch 2963ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch HeapObject* heap_object = HeapObject::cast(obj); 2973ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch MarkBit mark_bit = Marking::MarkBitFrom(heap_object); 2983ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch if (mark_bit.data_only()) { 299b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch MarkBlackOrKeepGrey(heap_object, mark_bit, heap_object->Size()); 3003ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch } else { 3013ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch if (Marking::IsWhite(mark_bit)) { 3023ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch incremental_marking_->WhiteToGreyAndPush(heap_object, mark_bit); 3033ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch } 3043ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch } 3053ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch } 3063ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch 3073ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch IncrementalMarking* incremental_marking_; 3083ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch}; 3093ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch 3103ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch 311b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdochvoid IncrementalMarking::Initialize() { 312b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch IncrementalMarkingMarkingVisitor::Initialize(); 313b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch} 314b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 315b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 3163ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdochvoid IncrementalMarking::SetOldSpacePageFlags(MemoryChunk* chunk, 3173ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch bool is_marking, 3183ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch bool is_compacting) { 3193ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch if (is_marking) { 3203ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch chunk->SetFlag(MemoryChunk::POINTERS_TO_HERE_ARE_INTERESTING); 3213ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch chunk->SetFlag(MemoryChunk::POINTERS_FROM_HERE_ARE_INTERESTING); 3223ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch 3233ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch // It's difficult to filter out slots recorded for large objects. 3243ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch if (chunk->owner()->identity() == LO_SPACE && 325b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch chunk->size() > static_cast<size_t>(Page::kPageSize) && is_compacting) { 3263ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch chunk->SetFlag(MemoryChunk::RESCAN_ON_EVACUATION); 3273ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch } 3283ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch } else if (chunk->owner()->identity() == CELL_SPACE || 329b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch chunk->owner()->identity() == PROPERTY_CELL_SPACE || 3303ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch chunk->scan_on_scavenge()) { 3313ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch chunk->ClearFlag(MemoryChunk::POINTERS_TO_HERE_ARE_INTERESTING); 3323ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch chunk->ClearFlag(MemoryChunk::POINTERS_FROM_HERE_ARE_INTERESTING); 3333ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch } else { 3343ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch chunk->ClearFlag(MemoryChunk::POINTERS_TO_HERE_ARE_INTERESTING); 3353ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch chunk->SetFlag(MemoryChunk::POINTERS_FROM_HERE_ARE_INTERESTING); 3363ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch } 3373ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch} 3383ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch 3393ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch 3403ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdochvoid IncrementalMarking::SetNewSpacePageFlags(NewSpacePage* chunk, 3413ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch bool is_marking) { 3423ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch chunk->SetFlag(MemoryChunk::POINTERS_TO_HERE_ARE_INTERESTING); 3433ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch if (is_marking) { 3443ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch chunk->SetFlag(MemoryChunk::POINTERS_FROM_HERE_ARE_INTERESTING); 3453ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch } else { 3463ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch chunk->ClearFlag(MemoryChunk::POINTERS_FROM_HERE_ARE_INTERESTING); 3473ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch } 3483ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch chunk->SetFlag(MemoryChunk::SCAN_ON_SCAVENGE); 3493ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch} 3503ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch 3513ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch 3523ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdochvoid IncrementalMarking::DeactivateIncrementalWriteBarrierForSpace( 3533ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch PagedSpace* space) { 3543ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch PageIterator it(space); 3553ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch while (it.has_next()) { 3563ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch Page* p = it.next(); 3573ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch SetOldSpacePageFlags(p, false, false); 3583ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch } 3593ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch} 3603ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch 3613ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch 3623ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdochvoid IncrementalMarking::DeactivateIncrementalWriteBarrierForSpace( 3633ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch NewSpace* space) { 3643ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch NewSpacePageIterator it(space); 3653ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch while (it.has_next()) { 3663ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch NewSpacePage* p = it.next(); 3673ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch SetNewSpacePageFlags(p, false); 3683ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch } 3693ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch} 3703ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch 3713ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch 3723ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdochvoid IncrementalMarking::DeactivateIncrementalWriteBarrier() { 3733ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch DeactivateIncrementalWriteBarrierForSpace(heap_->old_pointer_space()); 3743ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch DeactivateIncrementalWriteBarrierForSpace(heap_->old_data_space()); 3753ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch DeactivateIncrementalWriteBarrierForSpace(heap_->cell_space()); 376b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch DeactivateIncrementalWriteBarrierForSpace(heap_->property_cell_space()); 3773ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch DeactivateIncrementalWriteBarrierForSpace(heap_->map_space()); 3783ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch DeactivateIncrementalWriteBarrierForSpace(heap_->code_space()); 3793ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch DeactivateIncrementalWriteBarrierForSpace(heap_->new_space()); 3803ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch 3813ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch LargePage* lop = heap_->lo_space()->first_page(); 3823ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch while (lop->is_valid()) { 3833ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch SetOldSpacePageFlags(lop, false, false); 3843ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch lop = lop->next_page(); 3853ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch } 3863ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch} 3873ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch 3883ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch 3893ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdochvoid IncrementalMarking::ActivateIncrementalWriteBarrier(PagedSpace* space) { 3903ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch PageIterator it(space); 3913ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch while (it.has_next()) { 3923ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch Page* p = it.next(); 3933ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch SetOldSpacePageFlags(p, true, is_compacting_); 3943ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch } 3953ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch} 3963ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch 3973ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch 3983ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdochvoid IncrementalMarking::ActivateIncrementalWriteBarrier(NewSpace* space) { 3993ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch NewSpacePageIterator it(space->ToSpaceStart(), space->ToSpaceEnd()); 4003ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch while (it.has_next()) { 4013ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch NewSpacePage* p = it.next(); 4023ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch SetNewSpacePageFlags(p, true); 4033ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch } 4043ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch} 4053ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch 4063ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch 4073ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdochvoid IncrementalMarking::ActivateIncrementalWriteBarrier() { 4083ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch ActivateIncrementalWriteBarrier(heap_->old_pointer_space()); 4093ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch ActivateIncrementalWriteBarrier(heap_->old_data_space()); 4103ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch ActivateIncrementalWriteBarrier(heap_->cell_space()); 411b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch ActivateIncrementalWriteBarrier(heap_->property_cell_space()); 4123ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch ActivateIncrementalWriteBarrier(heap_->map_space()); 4133ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch ActivateIncrementalWriteBarrier(heap_->code_space()); 4143ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch ActivateIncrementalWriteBarrier(heap_->new_space()); 4153ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch 4163ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch LargePage* lop = heap_->lo_space()->first_page(); 4173ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch while (lop->is_valid()) { 4183ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch SetOldSpacePageFlags(lop, true, is_compacting_); 4193ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch lop = lop->next_page(); 4203ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch } 4213ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch} 4223ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch 4233ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch 424b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdochbool IncrementalMarking::ShouldActivate() { 425b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch return WorthActivating() && heap_->NextGCIsLikelyToBeFull(); 426b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch} 427b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 428b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 4293ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdochbool IncrementalMarking::WorthActivating() { 4303ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch#ifndef DEBUG 4313ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch static const intptr_t kActivationThreshold = 8 * MB; 4323ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch#else 4333ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch // TODO(gc) consider setting this to some low level so that some 4343ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch // debug tests run with incremental marking and some without. 4353ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch static const intptr_t kActivationThreshold = 0; 4363ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch#endif 437b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch // Only start incremental marking in a safe state: 1) when incremental 438b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch // marking is turned on, 2) when we are currently not in a GC, and 439b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch // 3) when we are currently not serializing or deserializing the heap. 440b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch return FLAG_incremental_marking && FLAG_incremental_marking_steps && 441b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch heap_->gc_state() == Heap::NOT_IN_GC && 442b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch !heap_->isolate()->serializer_enabled() && 443b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch heap_->isolate()->IsInitialized() && 444b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch heap_->PromotedSpaceSizeOfObjects() > kActivationThreshold; 4453ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch} 4463ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch 4473ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch 4483ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdochvoid IncrementalMarking::ActivateGeneratedStub(Code* stub) { 449b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch DCHECK(RecordWriteStub::GetMode(stub) == RecordWriteStub::STORE_BUFFER_ONLY); 4503ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch 4513ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch if (!IsMarking()) { 4523ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch // Initially stub is generated in STORE_BUFFER_ONLY mode thus 4533ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch // we don't need to do anything if incremental marking is 4543ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch // not active. 4553ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch } else if (IsCompacting()) { 4563ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch RecordWriteStub::Patch(stub, RecordWriteStub::INCREMENTAL_COMPACTION); 4573ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch } else { 4583ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch RecordWriteStub::Patch(stub, RecordWriteStub::INCREMENTAL); 4593ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch } 4603ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch} 4613ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch 4623ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch 4633ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdochstatic void PatchIncrementalMarkingRecordWriteStubs( 4643ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch Heap* heap, RecordWriteStub::Mode mode) { 4653ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch UnseededNumberDictionary* stubs = heap->code_stubs(); 4663ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch 4673ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch int capacity = stubs->Capacity(); 4683ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch for (int i = 0; i < capacity; i++) { 4693ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch Object* k = stubs->KeyAt(i); 4703ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch if (stubs->IsKey(k)) { 4713ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch uint32_t key = NumberToUint32(k); 4723ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch 473b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch if (CodeStub::MajorKeyFromKey(key) == CodeStub::RecordWrite) { 4743ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch Object* e = stubs->ValueAt(i); 4753ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch if (e->IsCode()) { 4763ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch RecordWriteStub::Patch(Code::cast(e), mode); 4773ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch } 4783ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch } 4793ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch } 4803ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch } 4813ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch} 4823ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch 4833ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch 4843ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdochvoid IncrementalMarking::EnsureMarkingDequeIsCommitted() { 4853ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch if (marking_deque_memory_ == NULL) { 486b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch marking_deque_memory_ = new base::VirtualMemory(4 * MB); 4873ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch } 4883ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch if (!marking_deque_memory_committed_) { 4893ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch bool success = marking_deque_memory_->Commit( 4903ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch reinterpret_cast<Address>(marking_deque_memory_->address()), 4913ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch marking_deque_memory_->size(), 4923ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch false); // Not executable. 4933ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch CHECK(success); 4943ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch marking_deque_memory_committed_ = true; 4953ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch } 4963ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch} 4973ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch 498b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 4993ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdochvoid IncrementalMarking::UncommitMarkingDeque() { 5003ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch if (state_ == STOPPED && marking_deque_memory_committed_) { 5013ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch bool success = marking_deque_memory_->Uncommit( 5023ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch reinterpret_cast<Address>(marking_deque_memory_->address()), 5033ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch marking_deque_memory_->size()); 5043ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch CHECK(success); 5053ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch marking_deque_memory_committed_ = false; 5063ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch } 5073ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch} 5083ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch 5093ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch 510b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdochvoid IncrementalMarking::Start(CompactionFlag flag) { 5113ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch if (FLAG_trace_incremental_marking) { 5123ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch PrintF("[IncrementalMarking] Start\n"); 5133ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch } 514b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch DCHECK(FLAG_incremental_marking); 515b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch DCHECK(FLAG_incremental_marking_steps); 516b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch DCHECK(state_ == STOPPED); 517b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch DCHECK(heap_->gc_state() == Heap::NOT_IN_GC); 518b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch DCHECK(!heap_->isolate()->serializer_enabled()); 519b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch DCHECK(heap_->isolate()->IsInitialized()); 5203ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch 5213ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch ResetStepCounters(); 5223ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch 523b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch if (!heap_->mark_compact_collector()->sweeping_in_progress()) { 524b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch StartMarking(flag); 5253ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch } else { 5263ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch if (FLAG_trace_incremental_marking) { 5273ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch PrintF("[IncrementalMarking] Start sweeping.\n"); 5283ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch } 5293ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch state_ = SWEEPING; 5303ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch } 5313ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch 5323ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch heap_->new_space()->LowerInlineAllocationLimit(kAllocatedThreshold); 5333ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch} 5343ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch 5353ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch 5363ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdochvoid IncrementalMarking::StartMarking(CompactionFlag flag) { 5373ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch if (FLAG_trace_incremental_marking) { 5383ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch PrintF("[IncrementalMarking] Start marking\n"); 5393ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch } 5403ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch 5413ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch is_compacting_ = !FLAG_never_compact && (flag == ALLOW_COMPACTION) && 542b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch heap_->mark_compact_collector()->StartCompaction( 543b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch MarkCompactCollector::INCREMENTAL_COMPACTION); 5443ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch 5453ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch state_ = MARKING; 5463ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch 547b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch RecordWriteStub::Mode mode = is_compacting_ 548b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch ? RecordWriteStub::INCREMENTAL_COMPACTION 549b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch : RecordWriteStub::INCREMENTAL; 5503ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch 5513ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch PatchIncrementalMarkingRecordWriteStubs(heap_, mode); 5523ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch 5533ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch EnsureMarkingDequeIsCommitted(); 5543ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch 5553ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch // Initialize marking stack. 5563ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch Address addr = static_cast<Address>(marking_deque_memory_->address()); 5573ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch size_t size = marking_deque_memory_->size(); 5583ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch if (FLAG_force_marking_deque_overflows) size = 64 * kPointerSize; 5593ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch marking_deque_.Initialize(addr, addr + size); 5603ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch 5613ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch ActivateIncrementalWriteBarrier(); 5623ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch 563b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch// Marking bits are cleared by the sweeper. 564b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch#ifdef VERIFY_HEAP 5653ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch if (FLAG_verify_heap) { 5663ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch heap_->mark_compact_collector()->VerifyMarkbitsAreClean(); 5673ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch } 5683ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch#endif 5693ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch 5703ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch heap_->CompletelyClearInstanceofCache(); 5713ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch heap_->isolate()->compilation_cache()->MarkCompactPrologue(); 5723ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch 5733ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch if (FLAG_cleanup_code_caches_at_gc) { 5743ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch // We will mark cache black with a separate pass 5753ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch // when we finish marking. 5763ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch MarkObjectGreyDoNotEnqueue(heap_->polymorphic_code_cache()); 5773ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch } 5783ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch 5793ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch // Mark strong roots grey. 580b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch IncrementalMarkingRootMarkingVisitor visitor(this); 5813ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch heap_->IterateStrongRoots(&visitor, VISIT_ONLY_STRONG); 5823ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch 583b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch heap_->mark_compact_collector()->MarkWeakObjectToCodeTable(); 584b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 5853ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch // Ready to start incremental marking. 5863ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch if (FLAG_trace_incremental_marking) { 5873ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch PrintF("[IncrementalMarking] Running\n"); 5883ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch } 5893ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch} 5903ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch 5913ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch 5923ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdochvoid IncrementalMarking::PrepareForScavenge() { 5933ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch if (!IsMarking()) return; 5943ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch NewSpacePageIterator it(heap_->new_space()->FromSpaceStart(), 5953ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch heap_->new_space()->FromSpaceEnd()); 5963ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch while (it.has_next()) { 5973ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch Bitmap::Clear(it.next()); 5983ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch } 5993ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch} 6003ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch 6013ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch 6023ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdochvoid IncrementalMarking::UpdateMarkingDequeAfterScavenge() { 6033ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch if (!IsMarking()) return; 6043ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch 6053ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch int current = marking_deque_.bottom(); 6063ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch int mask = marking_deque_.mask(); 6073ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch int limit = marking_deque_.top(); 6083ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch HeapObject** array = marking_deque_.array(); 6093ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch int new_top = current; 6103ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch 6113ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch Map* filler_map = heap_->one_pointer_filler_map(); 6123ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch 6133ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch while (current != limit) { 6143ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch HeapObject* obj = array[current]; 615b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch DCHECK(obj->IsHeapObject()); 6163ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch current = ((current + 1) & mask); 6173ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch if (heap_->InNewSpace(obj)) { 6183ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch MapWord map_word = obj->map_word(); 6193ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch if (map_word.IsForwardingAddress()) { 6203ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch HeapObject* dest = map_word.ToForwardingAddress(); 6213ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch array[new_top] = dest; 6223ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch new_top = ((new_top + 1) & mask); 623b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch DCHECK(new_top != marking_deque_.bottom()); 6243ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch#ifdef DEBUG 6253ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch MarkBit mark_bit = Marking::MarkBitFrom(obj); 626b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch DCHECK(Marking::IsGrey(mark_bit) || 6273ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch (obj->IsFiller() && Marking::IsWhite(mark_bit))); 6283ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch#endif 6293ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch } 6303ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch } else if (obj->map() != filler_map) { 6313ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch // Skip one word filler objects that appear on the 6323ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch // stack when we perform in place array shift. 6333ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch array[new_top] = obj; 6343ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch new_top = ((new_top + 1) & mask); 635b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch DCHECK(new_top != marking_deque_.bottom()); 6363ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch#ifdef DEBUG 637b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch MarkBit mark_bit = Marking::MarkBitFrom(obj); 638b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch MemoryChunk* chunk = MemoryChunk::FromAddress(obj->address()); 639b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch DCHECK(Marking::IsGrey(mark_bit) || 640b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch (obj->IsFiller() && Marking::IsWhite(mark_bit)) || 641b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch (chunk->IsFlagSet(MemoryChunk::HAS_PROGRESS_BAR) && 642b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch Marking::IsBlack(mark_bit))); 6433ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch#endif 6443ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch } 6453ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch } 6463ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch marking_deque_.set_top(new_top); 647b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch} 648b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 649b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 650b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdochvoid IncrementalMarking::VisitObject(Map* map, HeapObject* obj, int size) { 651b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch MarkBit map_mark_bit = Marking::MarkBitFrom(map); 652b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch if (Marking::IsWhite(map_mark_bit)) { 653b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch WhiteToGreyAndPush(map, map_mark_bit); 654b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch } 655b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 656b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch IncrementalMarkingMarkingVisitor::IterateBody(map, obj); 6573ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch 658b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch MarkBit mark_bit = Marking::MarkBitFrom(obj); 659b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch#if ENABLE_SLOW_DCHECKS 660b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch MemoryChunk* chunk = MemoryChunk::FromAddress(obj->address()); 661b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch SLOW_DCHECK(Marking::IsGrey(mark_bit) || 662b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch (obj->IsFiller() && Marking::IsWhite(mark_bit)) || 663b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch (chunk->IsFlagSet(MemoryChunk::HAS_PROGRESS_BAR) && 664b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch Marking::IsBlack(mark_bit))); 665b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch#endif 666b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch MarkBlackOrKeepBlack(obj, mark_bit, size); 6673ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch} 6683ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch 6693ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch 670b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdochintptr_t IncrementalMarking::ProcessMarkingDeque(intptr_t bytes_to_process) { 671b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch intptr_t bytes_processed = 0; 672b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch Map* filler_map = heap_->one_pointer_filler_map(); 673b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch while (!marking_deque_.IsEmpty() && bytes_processed < bytes_to_process) { 674b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch HeapObject* obj = marking_deque_.Pop(); 675b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 676b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch // Explicitly skip one word fillers. Incremental markbit patterns are 677b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch // correct only for objects that occupy at least two words. 678b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch Map* map = obj->map(); 679b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch if (map == filler_map) continue; 6803ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch 681b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch int size = obj->SizeFromMap(map); 682b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch unscanned_bytes_of_large_object_ = 0; 683b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch VisitObject(map, obj, size); 684b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch int delta = (size - unscanned_bytes_of_large_object_); 685b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch // TODO(jochen): remove after http://crbug.com/381820 is resolved. 686b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch CHECK_LT(0, delta); 687b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch bytes_processed += delta; 688b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch } 689b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch return bytes_processed; 690b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch} 691b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 692b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 693b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdochvoid IncrementalMarking::ProcessMarkingDeque() { 694b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch Map* filler_map = heap_->one_pointer_filler_map(); 695b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch while (!marking_deque_.IsEmpty()) { 696b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch HeapObject* obj = marking_deque_.Pop(); 697b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 698b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch // Explicitly skip one word fillers. Incremental markbit patterns are 699b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch // correct only for objects that occupy at least two words. 700b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch Map* map = obj->map(); 701b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch if (map == filler_map) continue; 702b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 703b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch VisitObject(map, obj, obj->SizeFromMap(map)); 7043ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch } 7053ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch} 7063ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch 7073ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch 7083ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdochvoid IncrementalMarking::Hurry() { 7093ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch if (state() == MARKING) { 7103ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch double start = 0.0; 711b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch if (FLAG_trace_incremental_marking || FLAG_print_cumulative_gc_stat) { 712b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch start = base::OS::TimeCurrentMillis(); 713b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch if (FLAG_trace_incremental_marking) { 714b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch PrintF("[IncrementalMarking] Hurry\n"); 715b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch } 7163ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch } 7173ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch // TODO(gc) hurry can mark objects it encounters black as mutator 7183ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch // was stopped. 719b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch ProcessMarkingDeque(); 7203ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch state_ = COMPLETE; 721b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch if (FLAG_trace_incremental_marking || FLAG_print_cumulative_gc_stat) { 722b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch double end = base::OS::TimeCurrentMillis(); 723b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch double delta = end - start; 724b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch heap_->tracer()->AddMarkingTime(delta); 725b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch if (FLAG_trace_incremental_marking) { 726b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch PrintF("[IncrementalMarking] Complete (hurry), spent %d ms.\n", 727b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch static_cast<int>(delta)); 728b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch } 7293ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch } 7303ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch } 7313ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch 7323ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch if (FLAG_cleanup_code_caches_at_gc) { 7333ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch PolymorphicCodeCache* poly_cache = heap_->polymorphic_code_cache(); 7343ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch Marking::GreyToBlack(Marking::MarkBitFrom(poly_cache)); 7353ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch MemoryChunk::IncrementLiveBytesFromGC(poly_cache->address(), 7363ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch PolymorphicCodeCache::kSize); 7373ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch } 7383ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch 739b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch Object* context = heap_->native_contexts_list(); 7403ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch while (!context->IsUndefined()) { 7413ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch // GC can happen when the context is not fully initialized, 7423ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch // so the cache can be undefined. 7433ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch HeapObject* cache = HeapObject::cast( 7443ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch Context::cast(context)->get(Context::NORMALIZED_MAP_CACHE_INDEX)); 7453ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch if (!cache->IsUndefined()) { 7463ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch MarkBit mark_bit = Marking::MarkBitFrom(cache); 7473ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch if (Marking::IsGrey(mark_bit)) { 7483ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch Marking::GreyToBlack(mark_bit); 7493ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch MemoryChunk::IncrementLiveBytesFromGC(cache->address(), cache->Size()); 7503ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch } 7513ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch } 7523ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch context = Context::cast(context)->get(Context::NEXT_CONTEXT_LINK); 7533ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch } 7543ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch} 7553ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch 7563ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch 7573ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdochvoid IncrementalMarking::Abort() { 7583ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch if (IsStopped()) return; 7593ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch if (FLAG_trace_incremental_marking) { 7603ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch PrintF("[IncrementalMarking] Aborting.\n"); 7613ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch } 7623ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch heap_->new_space()->LowerInlineAllocationLimit(0); 7633ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch IncrementalMarking::set_should_hurry(false); 7643ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch ResetStepCounters(); 7653ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch if (IsMarking()) { 7663ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch PatchIncrementalMarkingRecordWriteStubs(heap_, 7673ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch RecordWriteStub::STORE_BUFFER_ONLY); 7683ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch DeactivateIncrementalWriteBarrier(); 7693ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch 7703ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch if (is_compacting_) { 7713ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch LargeObjectIterator it(heap_->lo_space()); 7723ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch for (HeapObject* obj = it.Next(); obj != NULL; obj = it.Next()) { 7733ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch Page* p = Page::FromAddress(obj->address()); 7743ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch if (p->IsFlagSet(Page::RESCAN_ON_EVACUATION)) { 7753ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch p->ClearFlag(Page::RESCAN_ON_EVACUATION); 7763ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch } 7773ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch } 7783ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch } 7793ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch } 780b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch heap_->isolate()->stack_guard()->ClearGC(); 7813ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch state_ = STOPPED; 7823ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch is_compacting_ = false; 7833ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch} 7843ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch 7853ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch 7863ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdochvoid IncrementalMarking::Finalize() { 7873ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch Hurry(); 7883ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch state_ = STOPPED; 7893ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch is_compacting_ = false; 7903ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch heap_->new_space()->LowerInlineAllocationLimit(0); 7913ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch IncrementalMarking::set_should_hurry(false); 7923ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch ResetStepCounters(); 7933ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch PatchIncrementalMarkingRecordWriteStubs(heap_, 7943ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch RecordWriteStub::STORE_BUFFER_ONLY); 7953ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch DeactivateIncrementalWriteBarrier(); 796b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch DCHECK(marking_deque_.IsEmpty()); 797b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch heap_->isolate()->stack_guard()->ClearGC(); 7983ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch} 7993ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch 8003ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch 8018f9999fcc44cfd4f5e1140c6678bbca4cf8ea1c7Ben Murdochvoid IncrementalMarking::MarkingComplete(CompletionAction action) { 8023ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch state_ = COMPLETE; 8033ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch // We will set the stack guard to request a GC now. This will mean the rest 8043ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch // of the GC gets performed as soon as possible (we can't do a GC here in a 8053ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch // record-write context). If a few things get allocated between now and then 8063ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch // that shouldn't make us do a scavenge and keep being incremental, so we set 8073ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch // the should-hurry flag to indicate that there can't be much work left to do. 8083ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch set_should_hurry(true); 8093ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch if (FLAG_trace_incremental_marking) { 8103ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch PrintF("[IncrementalMarking] Complete (normal).\n"); 8113ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch } 8128f9999fcc44cfd4f5e1140c6678bbca4cf8ea1c7Ben Murdoch if (action == GC_VIA_STACK_GUARD) { 8133ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch heap_->isolate()->stack_guard()->RequestGC(); 8143ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch } 8153ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch} 8163ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch 8173ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch 818b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdochvoid IncrementalMarking::OldSpaceStep(intptr_t allocated) { 819b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch if (IsStopped() && ShouldActivate()) { 820b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch // TODO(hpayer): Let's play safe for now, but compaction should be 821b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch // in principle possible. 822b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch Start(PREVENT_COMPACTION); 823b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch } else { 824b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch Step(allocated * kFastMarking / kInitialMarkingSpeed, GC_VIA_STACK_GUARD); 8253ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch } 826b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch} 8273ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch 8283ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch 829b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdochvoid IncrementalMarking::SpeedUp() { 8303ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch bool speed_up = false; 8313ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch 832b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch if ((steps_count_ % kMarkingSpeedAccellerationInterval) == 0) { 8333ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch if (FLAG_trace_gc) { 834b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch PrintPID("Speed up marking after %d steps\n", 835b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch static_cast<int>(kMarkingSpeedAccellerationInterval)); 8363ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch } 8373ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch speed_up = true; 8383ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch } 8393ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch 8403ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch bool space_left_is_very_small = 8413ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch (old_generation_space_available_at_start_of_incremental_ < 10 * MB); 8423ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch 8433ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch bool only_1_nth_of_space_that_was_available_still_left = 844b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch (SpaceLeftInOldSpace() * (marking_speed_ + 1) < 845b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch old_generation_space_available_at_start_of_incremental_); 8463ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch 8473ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch if (space_left_is_very_small || 8483ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch only_1_nth_of_space_that_was_available_still_left) { 849b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch if (FLAG_trace_gc) PrintPID("Speed up marking because of low space left\n"); 8503ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch speed_up = true; 8513ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch } 8523ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch 8533ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch bool size_of_old_space_multiplied_by_n_during_marking = 8543ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch (heap_->PromotedTotalSize() > 855b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch (marking_speed_ + 1) * 8563ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch old_generation_space_used_at_start_of_incremental_); 8573ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch if (size_of_old_space_multiplied_by_n_during_marking) { 8583ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch speed_up = true; 8593ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch if (FLAG_trace_gc) { 860b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch PrintPID("Speed up marking because of heap size increase\n"); 8613ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch } 8623ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch } 8633ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch 864b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch int64_t promoted_during_marking = 865b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch heap_->PromotedTotalSize() - 866b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch old_generation_space_used_at_start_of_incremental_; 867b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch intptr_t delay = marking_speed_ * MB; 8683ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch intptr_t scavenge_slack = heap_->MaxSemiSpaceSize(); 8693ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch 8703ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch // We try to scan at at least twice the speed that we are allocating. 8713ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch if (promoted_during_marking > bytes_scanned_ / 2 + scavenge_slack + delay) { 8723ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch if (FLAG_trace_gc) { 873b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch PrintPID("Speed up marking because marker was not keeping up\n"); 8743ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch } 8753ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch speed_up = true; 8763ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch } 8773ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch 8783ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch if (speed_up) { 8793ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch if (state_ != MARKING) { 8803ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch if (FLAG_trace_gc) { 881b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch PrintPID("Postponing speeding up marking until marking starts\n"); 8823ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch } 8833ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch } else { 884b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch marking_speed_ += kMarkingSpeedAccelleration; 885b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch marking_speed_ = static_cast<int>( 886b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch Min(kMaxMarkingSpeed, static_cast<intptr_t>(marking_speed_ * 1.3))); 8873ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch if (FLAG_trace_gc) { 888b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch PrintPID("Marking speed increased to %d\n", marking_speed_); 8893ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch } 8903ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch } 8913ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch } 892b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch} 893b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 8943ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch 895b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdochvoid IncrementalMarking::Step(intptr_t allocated_bytes, CompletionAction action, 896b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch bool force_marking) { 897b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch if (heap_->gc_state() != Heap::NOT_IN_GC || !FLAG_incremental_marking || 898b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch !FLAG_incremental_marking_steps || 899b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch (state_ != SWEEPING && state_ != MARKING)) { 900b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch return; 901b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch } 902b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 903b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch allocated_ += allocated_bytes; 904b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 905b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch if (!force_marking && allocated_ < kAllocatedThreshold && 906b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch write_barriers_invoked_since_last_step_ < 907b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch kWriteBarriersInvokedThreshold) { 908b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch return; 909b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch } 910b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 911b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch if (state_ == MARKING && no_marking_scope_depth_ > 0) return; 912b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 913b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch { 914b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch HistogramTimerScope incremental_marking_scope( 915b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch heap_->isolate()->counters()->gc_incremental_marking()); 916b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch double start = base::OS::TimeCurrentMillis(); 917b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 918b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch // The marking speed is driven either by the allocation rate or by the rate 919b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch // at which we are having to check the color of objects in the write 920b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch // barrier. 921b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch // It is possible for a tight non-allocating loop to run a lot of write 922b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch // barriers before we get here and check them (marking can only take place 923b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch // on 924b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch // allocation), so to reduce the lumpiness we don't use the write barriers 925b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch // invoked since last step directly to determine the amount of work to do. 926b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch intptr_t bytes_to_process = 927b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch marking_speed_ * 928b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch Max(allocated_, write_barriers_invoked_since_last_step_); 929b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch allocated_ = 0; 930b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch write_barriers_invoked_since_last_step_ = 0; 931b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 932b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch bytes_scanned_ += bytes_to_process; 933b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch intptr_t bytes_processed = 0; 934b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 935b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch if (state_ == SWEEPING) { 936b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch if (heap_->mark_compact_collector()->sweeping_in_progress() && 937b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch heap_->mark_compact_collector()->IsSweepingCompleted()) { 938b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch heap_->mark_compact_collector()->EnsureSweepingCompleted(); 939b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch } 940b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch if (!heap_->mark_compact_collector()->sweeping_in_progress()) { 941b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch bytes_scanned_ = 0; 942b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch StartMarking(PREVENT_COMPACTION); 943b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch } 944b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch } else if (state_ == MARKING) { 945b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch bytes_processed = ProcessMarkingDeque(bytes_to_process); 946b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch if (marking_deque_.IsEmpty()) MarkingComplete(action); 947b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch } 948b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 949b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch steps_count_++; 950b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 951b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch // Speed up marking if we are marking too slow or if we are almost done 952b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch // with marking. 953b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch SpeedUp(); 954b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 955b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch double end = base::OS::TimeCurrentMillis(); 956b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch double duration = (end - start); 957b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch // Note that we report zero bytes here when sweeping was in progress or 958b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch // when we just started incremental marking. In these cases we did not 959b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch // process the marking deque. 960b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch heap_->tracer()->AddIncrementalMarkingStep(duration, bytes_processed); 9613ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch } 9623ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch} 9633ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch 9643ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch 9653ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdochvoid IncrementalMarking::ResetStepCounters() { 9663ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch steps_count_ = 0; 9673ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch old_generation_space_available_at_start_of_incremental_ = 9683ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch SpaceLeftInOldSpace(); 9693ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch old_generation_space_used_at_start_of_incremental_ = 9703ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch heap_->PromotedTotalSize(); 9713ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch bytes_rescanned_ = 0; 972b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch marking_speed_ = kInitialMarkingSpeed; 9733ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch bytes_scanned_ = 0; 974b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch write_barriers_invoked_since_last_step_ = 0; 9753ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch} 9763ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch 9773ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch 9783ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdochint64_t IncrementalMarking::SpaceLeftInOldSpace() { 979b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch return heap_->MaxOldGenerationSize() - heap_->PromotedSpaceSizeOfObjects(); 9803ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch} 981b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch} 982b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch} // namespace v8::internal 983