1// Copyright 2012 the V8 project authors. All rights reserved.
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5#ifndef V8_HEAP_HEAP_INL_H_
6#define V8_HEAP_HEAP_INL_H_
7
8#include <cmath>
9
10#include "src/base/platform/platform.h"
11#include "src/counters-inl.h"
12#include "src/heap/heap.h"
13#include "src/heap/incremental-marking-inl.h"
14#include "src/heap/mark-compact.h"
15#include "src/heap/object-stats.h"
16#include "src/heap/remembered-set.h"
17#include "src/heap/spaces-inl.h"
18#include "src/heap/store-buffer.h"
19#include "src/isolate.h"
20#include "src/list-inl.h"
21#include "src/log.h"
22#include "src/msan.h"
23#include "src/objects-inl.h"
24#include "src/type-feedback-vector-inl.h"
25
26namespace v8 {
27namespace internal {
28
29AllocationSpace AllocationResult::RetrySpace() {
30  DCHECK(IsRetry());
31  return static_cast<AllocationSpace>(Smi::cast(object_)->value());
32}
33
34HeapObject* AllocationResult::ToObjectChecked() {
35  CHECK(!IsRetry());
36  return HeapObject::cast(object_);
37}
38
39void PromotionQueue::insert(HeapObject* target, int32_t size,
40                            bool was_marked_black) {
41  if (emergency_stack_ != NULL) {
42    emergency_stack_->Add(Entry(target, size, was_marked_black));
43    return;
44  }
45
46  if ((rear_ - 1) < limit_) {
47    RelocateQueueHead();
48    emergency_stack_->Add(Entry(target, size, was_marked_black));
49    return;
50  }
51
52  struct Entry* entry = reinterpret_cast<struct Entry*>(--rear_);
53  entry->obj_ = target;
54  entry->size_ = size;
55  entry->was_marked_black_ = was_marked_black;
56
57// Assert no overflow into live objects.
58#ifdef DEBUG
59  SemiSpace::AssertValidRange(target->GetIsolate()->heap()->new_space()->top(),
60                              reinterpret_cast<Address>(rear_));
61#endif
62}
63
64void PromotionQueue::remove(HeapObject** target, int32_t* size,
65                            bool* was_marked_black) {
66  DCHECK(!is_empty());
67  if (front_ == rear_) {
68    Entry e = emergency_stack_->RemoveLast();
69    *target = e.obj_;
70    *size = e.size_;
71    *was_marked_black = e.was_marked_black_;
72    return;
73  }
74
75  struct Entry* entry = reinterpret_cast<struct Entry*>(--front_);
76  *target = entry->obj_;
77  *size = entry->size_;
78  *was_marked_black = entry->was_marked_black_;
79
80  // Assert no underflow.
81  SemiSpace::AssertValidRange(reinterpret_cast<Address>(rear_),
82                              reinterpret_cast<Address>(front_));
83}
84
85Page* PromotionQueue::GetHeadPage() {
86  return Page::FromAllocationAreaAddress(reinterpret_cast<Address>(rear_));
87}
88
89void PromotionQueue::SetNewLimit(Address limit) {
90  // If we are already using an emergency stack, we can ignore it.
91  if (emergency_stack_) return;
92
93  // If the limit is not on the same page, we can ignore it.
94  if (Page::FromAllocationAreaAddress(limit) != GetHeadPage()) return;
95
96  limit_ = reinterpret_cast<struct Entry*>(limit);
97
98  if (limit_ <= rear_) {
99    return;
100  }
101
102  RelocateQueueHead();
103}
104
105bool PromotionQueue::IsBelowPromotionQueue(Address to_space_top) {
106  // If an emergency stack is used, the to-space address cannot interfere
107  // with the promotion queue.
108  if (emergency_stack_) return true;
109
110  // If the given to-space top pointer and the head of the promotion queue
111  // are not on the same page, then the to-space objects are below the
112  // promotion queue.
113  if (GetHeadPage() != Page::FromAddress(to_space_top)) {
114    return true;
115  }
116  // If the to space top pointer is smaller or equal than the promotion
117  // queue head, then the to-space objects are below the promotion queue.
118  return reinterpret_cast<struct Entry*>(to_space_top) <= rear_;
119}
120
121#define ROOT_ACCESSOR(type, name, camel_name) \
122  type* Heap::name() { return type::cast(roots_[k##camel_name##RootIndex]); }
123ROOT_LIST(ROOT_ACCESSOR)
124#undef ROOT_ACCESSOR
125
126#define STRUCT_MAP_ACCESSOR(NAME, Name, name) \
127  Map* Heap::name##_map() { return Map::cast(roots_[k##Name##MapRootIndex]); }
128STRUCT_LIST(STRUCT_MAP_ACCESSOR)
129#undef STRUCT_MAP_ACCESSOR
130
131#define STRING_ACCESSOR(name, str) \
132  String* Heap::name() { return String::cast(roots_[k##name##RootIndex]); }
133INTERNALIZED_STRING_LIST(STRING_ACCESSOR)
134#undef STRING_ACCESSOR
135
136#define SYMBOL_ACCESSOR(name) \
137  Symbol* Heap::name() { return Symbol::cast(roots_[k##name##RootIndex]); }
138PRIVATE_SYMBOL_LIST(SYMBOL_ACCESSOR)
139#undef SYMBOL_ACCESSOR
140
141#define SYMBOL_ACCESSOR(name, description) \
142  Symbol* Heap::name() { return Symbol::cast(roots_[k##name##RootIndex]); }
143PUBLIC_SYMBOL_LIST(SYMBOL_ACCESSOR)
144WELL_KNOWN_SYMBOL_LIST(SYMBOL_ACCESSOR)
145#undef SYMBOL_ACCESSOR
146
147#define ROOT_ACCESSOR(type, name, camel_name)                                 \
148  void Heap::set_##name(type* value) {                                        \
149    /* The deserializer makes use of the fact that these common roots are */  \
150    /* never in new space and never on a page that is being compacted.    */  \
151    DCHECK(!deserialization_complete() ||                                     \
152           RootCanBeWrittenAfterInitialization(k##camel_name##RootIndex));    \
153    DCHECK(k##camel_name##RootIndex >= kOldSpaceRoots || !InNewSpace(value)); \
154    roots_[k##camel_name##RootIndex] = value;                                 \
155  }
156ROOT_LIST(ROOT_ACCESSOR)
157#undef ROOT_ACCESSOR
158
159PagedSpace* Heap::paged_space(int idx) {
160  DCHECK_NE(idx, LO_SPACE);
161  DCHECK_NE(idx, NEW_SPACE);
162  return static_cast<PagedSpace*>(space_[idx]);
163}
164
165Space* Heap::space(int idx) { return space_[idx]; }
166
167Address* Heap::NewSpaceAllocationTopAddress() {
168  return new_space_->allocation_top_address();
169}
170
171Address* Heap::NewSpaceAllocationLimitAddress() {
172  return new_space_->allocation_limit_address();
173}
174
175Address* Heap::OldSpaceAllocationTopAddress() {
176  return old_space_->allocation_top_address();
177}
178
179Address* Heap::OldSpaceAllocationLimitAddress() {
180  return old_space_->allocation_limit_address();
181}
182
183void Heap::UpdateNewSpaceAllocationCounter() {
184  new_space_allocation_counter_ = NewSpaceAllocationCounter();
185}
186
187size_t Heap::NewSpaceAllocationCounter() {
188  return new_space_allocation_counter_ + new_space()->AllocatedSinceLastGC();
189}
190
191template <>
192bool inline Heap::IsOneByte(Vector<const char> str, int chars) {
193  // TODO(dcarney): incorporate Latin-1 check when Latin-1 is supported?
194  return chars == str.length();
195}
196
197
198template <>
199bool inline Heap::IsOneByte(String* str, int chars) {
200  return str->IsOneByteRepresentation();
201}
202
203
204AllocationResult Heap::AllocateInternalizedStringFromUtf8(
205    Vector<const char> str, int chars, uint32_t hash_field) {
206  if (IsOneByte(str, chars)) {
207    return AllocateOneByteInternalizedString(Vector<const uint8_t>::cast(str),
208                                             hash_field);
209  }
210  return AllocateInternalizedStringImpl<false>(str, chars, hash_field);
211}
212
213
214template <typename T>
215AllocationResult Heap::AllocateInternalizedStringImpl(T t, int chars,
216                                                      uint32_t hash_field) {
217  if (IsOneByte(t, chars)) {
218    return AllocateInternalizedStringImpl<true>(t, chars, hash_field);
219  }
220  return AllocateInternalizedStringImpl<false>(t, chars, hash_field);
221}
222
223
224AllocationResult Heap::AllocateOneByteInternalizedString(
225    Vector<const uint8_t> str, uint32_t hash_field) {
226  CHECK_GE(String::kMaxLength, str.length());
227  // Compute map and object size.
228  Map* map = one_byte_internalized_string_map();
229  int size = SeqOneByteString::SizeFor(str.length());
230
231  // Allocate string.
232  HeapObject* result = nullptr;
233  {
234    AllocationResult allocation = AllocateRaw(size, OLD_SPACE);
235    if (!allocation.To(&result)) return allocation;
236  }
237
238  // String maps are all immortal immovable objects.
239  result->set_map_no_write_barrier(map);
240  // Set length and hash fields of the allocated string.
241  String* answer = String::cast(result);
242  answer->set_length(str.length());
243  answer->set_hash_field(hash_field);
244
245  DCHECK_EQ(size, answer->Size());
246
247  // Fill in the characters.
248  MemCopy(answer->address() + SeqOneByteString::kHeaderSize, str.start(),
249          str.length());
250
251  return answer;
252}
253
254
255AllocationResult Heap::AllocateTwoByteInternalizedString(Vector<const uc16> str,
256                                                         uint32_t hash_field) {
257  CHECK_GE(String::kMaxLength, str.length());
258  // Compute map and object size.
259  Map* map = internalized_string_map();
260  int size = SeqTwoByteString::SizeFor(str.length());
261
262  // Allocate string.
263  HeapObject* result = nullptr;
264  {
265    AllocationResult allocation = AllocateRaw(size, OLD_SPACE);
266    if (!allocation.To(&result)) return allocation;
267  }
268
269  result->set_map(map);
270  // Set length and hash fields of the allocated string.
271  String* answer = String::cast(result);
272  answer->set_length(str.length());
273  answer->set_hash_field(hash_field);
274
275  DCHECK_EQ(size, answer->Size());
276
277  // Fill in the characters.
278  MemCopy(answer->address() + SeqTwoByteString::kHeaderSize, str.start(),
279          str.length() * kUC16Size);
280
281  return answer;
282}
283
284AllocationResult Heap::CopyFixedArray(FixedArray* src) {
285  if (src->length() == 0) return src;
286  return CopyFixedArrayWithMap(src, src->map());
287}
288
289
290AllocationResult Heap::CopyFixedDoubleArray(FixedDoubleArray* src) {
291  if (src->length() == 0) return src;
292  return CopyFixedDoubleArrayWithMap(src, src->map());
293}
294
295
296AllocationResult Heap::AllocateRaw(int size_in_bytes, AllocationSpace space,
297                                   AllocationAlignment alignment) {
298  DCHECK(AllowHandleAllocation::IsAllowed());
299  DCHECK(AllowHeapAllocation::IsAllowed());
300  DCHECK(gc_state_ == NOT_IN_GC);
301#ifdef DEBUG
302  if (FLAG_gc_interval >= 0 && !always_allocate() &&
303      Heap::allocation_timeout_-- <= 0) {
304    return AllocationResult::Retry(space);
305  }
306  isolate_->counters()->objs_since_last_full()->Increment();
307  isolate_->counters()->objs_since_last_young()->Increment();
308#endif
309
310  bool large_object = size_in_bytes > kMaxRegularHeapObjectSize;
311  HeapObject* object = nullptr;
312  AllocationResult allocation;
313  if (NEW_SPACE == space) {
314    if (large_object) {
315      space = LO_SPACE;
316    } else {
317      allocation = new_space_->AllocateRaw(size_in_bytes, alignment);
318      if (allocation.To(&object)) {
319        OnAllocationEvent(object, size_in_bytes);
320      }
321      return allocation;
322    }
323  }
324
325  // Here we only allocate in the old generation.
326  if (OLD_SPACE == space) {
327    if (large_object) {
328      allocation = lo_space_->AllocateRaw(size_in_bytes, NOT_EXECUTABLE);
329    } else {
330      allocation = old_space_->AllocateRaw(size_in_bytes, alignment);
331    }
332  } else if (CODE_SPACE == space) {
333    if (size_in_bytes <= code_space()->AreaSize()) {
334      allocation = code_space_->AllocateRawUnaligned(size_in_bytes);
335    } else {
336      allocation = lo_space_->AllocateRaw(size_in_bytes, EXECUTABLE);
337    }
338  } else if (LO_SPACE == space) {
339    DCHECK(large_object);
340    allocation = lo_space_->AllocateRaw(size_in_bytes, NOT_EXECUTABLE);
341  } else if (MAP_SPACE == space) {
342    allocation = map_space_->AllocateRawUnaligned(size_in_bytes);
343  } else {
344    // NEW_SPACE is not allowed here.
345    UNREACHABLE();
346  }
347  if (allocation.To(&object)) {
348    OnAllocationEvent(object, size_in_bytes);
349  }
350
351  return allocation;
352}
353
354
355void Heap::OnAllocationEvent(HeapObject* object, int size_in_bytes) {
356  HeapProfiler* profiler = isolate_->heap_profiler();
357  if (profiler->is_tracking_allocations()) {
358    profiler->AllocationEvent(object->address(), size_in_bytes);
359  }
360
361  if (FLAG_verify_predictable) {
362    ++allocations_count_;
363    // Advance synthetic time by making a time request.
364    MonotonicallyIncreasingTimeInMs();
365
366    UpdateAllocationsHash(object);
367    UpdateAllocationsHash(size_in_bytes);
368
369    if (allocations_count_ % FLAG_dump_allocations_digest_at_alloc == 0) {
370      PrintAlloctionsHash();
371    }
372  }
373
374  if (FLAG_trace_allocation_stack_interval > 0) {
375    if (!FLAG_verify_predictable) ++allocations_count_;
376    if (allocations_count_ % FLAG_trace_allocation_stack_interval == 0) {
377      isolate()->PrintStack(stdout, Isolate::kPrintStackConcise);
378    }
379  }
380}
381
382
383void Heap::OnMoveEvent(HeapObject* target, HeapObject* source,
384                       int size_in_bytes) {
385  HeapProfiler* heap_profiler = isolate_->heap_profiler();
386  if (heap_profiler->is_tracking_object_moves()) {
387    heap_profiler->ObjectMoveEvent(source->address(), target->address(),
388                                   size_in_bytes);
389  }
390  if (target->IsSharedFunctionInfo()) {
391    LOG_CODE_EVENT(isolate_, SharedFunctionInfoMoveEvent(source->address(),
392                                                         target->address()));
393  }
394
395  if (FLAG_verify_predictable) {
396    ++allocations_count_;
397    // Advance synthetic time by making a time request.
398    MonotonicallyIncreasingTimeInMs();
399
400    UpdateAllocationsHash(source);
401    UpdateAllocationsHash(target);
402    UpdateAllocationsHash(size_in_bytes);
403
404    if (allocations_count_ % FLAG_dump_allocations_digest_at_alloc == 0) {
405      PrintAlloctionsHash();
406    }
407  }
408}
409
410
411void Heap::UpdateAllocationsHash(HeapObject* object) {
412  Address object_address = object->address();
413  MemoryChunk* memory_chunk = MemoryChunk::FromAddress(object_address);
414  AllocationSpace allocation_space = memory_chunk->owner()->identity();
415
416  STATIC_ASSERT(kSpaceTagSize + kPageSizeBits <= 32);
417  uint32_t value =
418      static_cast<uint32_t>(object_address - memory_chunk->address()) |
419      (static_cast<uint32_t>(allocation_space) << kPageSizeBits);
420
421  UpdateAllocationsHash(value);
422}
423
424
425void Heap::UpdateAllocationsHash(uint32_t value) {
426  uint16_t c1 = static_cast<uint16_t>(value);
427  uint16_t c2 = static_cast<uint16_t>(value >> 16);
428  raw_allocations_hash_ =
429      StringHasher::AddCharacterCore(raw_allocations_hash_, c1);
430  raw_allocations_hash_ =
431      StringHasher::AddCharacterCore(raw_allocations_hash_, c2);
432}
433
434
435void Heap::RegisterExternalString(String* string) {
436  external_string_table_.AddString(string);
437}
438
439
440void Heap::FinalizeExternalString(String* string) {
441  DCHECK(string->IsExternalString());
442  v8::String::ExternalStringResourceBase** resource_addr =
443      reinterpret_cast<v8::String::ExternalStringResourceBase**>(
444          reinterpret_cast<byte*>(string) + ExternalString::kResourceOffset -
445          kHeapObjectTag);
446
447  // Dispose of the C++ object if it has not already been disposed.
448  if (*resource_addr != NULL) {
449    (*resource_addr)->Dispose();
450    *resource_addr = NULL;
451  }
452}
453
454Address Heap::NewSpaceTop() { return new_space_->top(); }
455
456bool Heap::DeoptMaybeTenuredAllocationSites() {
457  return new_space_->IsAtMaximumCapacity() && maximum_size_scavenges_ == 0;
458}
459
460bool Heap::InNewSpace(Object* object) {
461  // Inlined check from NewSpace::Contains.
462  bool result =
463      object->IsHeapObject() &&
464      Page::FromAddress(HeapObject::cast(object)->address())->InNewSpace();
465  DCHECK(!result ||                 // Either not in new space
466         gc_state_ != NOT_IN_GC ||  // ... or in the middle of GC
467         InToSpace(object));        // ... or in to-space (where we allocate).
468  return result;
469}
470
471bool Heap::InFromSpace(Object* object) {
472  return object->IsHeapObject() &&
473         MemoryChunk::FromAddress(HeapObject::cast(object)->address())
474             ->IsFlagSet(Page::IN_FROM_SPACE);
475}
476
477
478bool Heap::InToSpace(Object* object) {
479  return object->IsHeapObject() &&
480         MemoryChunk::FromAddress(HeapObject::cast(object)->address())
481             ->IsFlagSet(Page::IN_TO_SPACE);
482}
483
484bool Heap::InOldSpace(Object* object) { return old_space_->Contains(object); }
485
486bool Heap::InNewSpaceSlow(Address address) {
487  return new_space_->ContainsSlow(address);
488}
489
490bool Heap::InOldSpaceSlow(Address address) {
491  return old_space_->ContainsSlow(address);
492}
493
494bool Heap::ShouldBePromoted(Address old_address, int object_size) {
495  Page* page = Page::FromAddress(old_address);
496  Address age_mark = new_space_->age_mark();
497  return page->IsFlagSet(MemoryChunk::NEW_SPACE_BELOW_AGE_MARK) &&
498         (!page->ContainsLimit(age_mark) || old_address < age_mark);
499}
500
501void Heap::RecordWrite(Object* object, int offset, Object* o) {
502  if (!InNewSpace(o) || !object->IsHeapObject() || InNewSpace(object)) {
503    return;
504  }
505  store_buffer()->InsertEntry(HeapObject::cast(object)->address() + offset);
506}
507
508void Heap::RecordWriteIntoCode(Code* host, RelocInfo* rinfo, Object* value) {
509  if (InNewSpace(value)) {
510    RecordWriteIntoCodeSlow(host, rinfo, value);
511  }
512}
513
514void Heap::RecordFixedArrayElements(FixedArray* array, int offset, int length) {
515  if (InNewSpace(array)) return;
516  for (int i = 0; i < length; i++) {
517    if (!InNewSpace(array->get(offset + i))) continue;
518    store_buffer()->InsertEntry(
519        reinterpret_cast<Address>(array->RawFieldOfElementAt(offset + i)));
520  }
521}
522
523Address* Heap::store_buffer_top_address() {
524  return store_buffer()->top_address();
525}
526
527bool Heap::AllowedToBeMigrated(HeapObject* obj, AllocationSpace dst) {
528  // Object migration is governed by the following rules:
529  //
530  // 1) Objects in new-space can be migrated to the old space
531  //    that matches their target space or they stay in new-space.
532  // 2) Objects in old-space stay in the same space when migrating.
533  // 3) Fillers (two or more words) can migrate due to left-trimming of
534  //    fixed arrays in new-space or old space.
535  // 4) Fillers (one word) can never migrate, they are skipped by
536  //    incremental marking explicitly to prevent invalid pattern.
537  //
538  // Since this function is used for debugging only, we do not place
539  // asserts here, but check everything explicitly.
540  if (obj->map() == one_pointer_filler_map()) return false;
541  InstanceType type = obj->map()->instance_type();
542  MemoryChunk* chunk = MemoryChunk::FromAddress(obj->address());
543  AllocationSpace src = chunk->owner()->identity();
544  switch (src) {
545    case NEW_SPACE:
546      return dst == src || dst == OLD_SPACE;
547    case OLD_SPACE:
548      return dst == src &&
549             (dst == OLD_SPACE || obj->IsFiller() || obj->IsExternalString());
550    case CODE_SPACE:
551      return dst == src && type == CODE_TYPE;
552    case MAP_SPACE:
553    case LO_SPACE:
554      return false;
555  }
556  UNREACHABLE();
557  return false;
558}
559
560void Heap::CopyBlock(Address dst, Address src, int byte_size) {
561  CopyWords(reinterpret_cast<Object**>(dst), reinterpret_cast<Object**>(src),
562            static_cast<size_t>(byte_size / kPointerSize));
563}
564
565template <Heap::FindMementoMode mode>
566AllocationMemento* Heap::FindAllocationMemento(HeapObject* object) {
567  Address object_address = object->address();
568  Address memento_address = object_address + object->Size();
569  Address last_memento_word_address = memento_address + kPointerSize;
570  // If the memento would be on another page, bail out immediately.
571  if (!Page::OnSamePage(object_address, last_memento_word_address)) {
572    return nullptr;
573  }
574  HeapObject* candidate = HeapObject::FromAddress(memento_address);
575  Map* candidate_map = candidate->map();
576  // This fast check may peek at an uninitialized word. However, the slow check
577  // below (memento_address == top) ensures that this is safe. Mark the word as
578  // initialized to silence MemorySanitizer warnings.
579  MSAN_MEMORY_IS_INITIALIZED(&candidate_map, sizeof(candidate_map));
580  if (candidate_map != allocation_memento_map()) {
581    return nullptr;
582  }
583
584  // Bail out if the memento is below the age mark, which can happen when
585  // mementos survived because a page got moved within new space.
586  Page* object_page = Page::FromAddress(object_address);
587  if (object_page->IsFlagSet(Page::NEW_SPACE_BELOW_AGE_MARK)) {
588    Address age_mark =
589        reinterpret_cast<SemiSpace*>(object_page->owner())->age_mark();
590    if (!object_page->Contains(age_mark)) {
591      return nullptr;
592    }
593    // Do an exact check in the case where the age mark is on the same page.
594    if (object_address < age_mark) {
595      return nullptr;
596    }
597  }
598
599  AllocationMemento* memento_candidate = AllocationMemento::cast(candidate);
600
601  // Depending on what the memento is used for, we might need to perform
602  // additional checks.
603  Address top;
604  switch (mode) {
605    case Heap::kForGC:
606      return memento_candidate;
607    case Heap::kForRuntime:
608      if (memento_candidate == nullptr) return nullptr;
609      // Either the object is the last object in the new space, or there is
610      // another object of at least word size (the header map word) following
611      // it, so suffices to compare ptr and top here.
612      top = NewSpaceTop();
613      DCHECK(memento_address == top ||
614             memento_address + HeapObject::kHeaderSize <= top ||
615             !Page::OnSamePage(memento_address, top - 1));
616      if ((memento_address != top) && memento_candidate->IsValid()) {
617        return memento_candidate;
618      }
619      return nullptr;
620    default:
621      UNREACHABLE();
622  }
623  UNREACHABLE();
624  return nullptr;
625}
626
627template <Heap::UpdateAllocationSiteMode mode>
628void Heap::UpdateAllocationSite(HeapObject* object,
629                                base::HashMap* pretenuring_feedback) {
630  DCHECK(InFromSpace(object) ||
631         (InToSpace(object) &&
632          Page::FromAddress(object->address())
633              ->IsFlagSet(Page::PAGE_NEW_NEW_PROMOTION)) ||
634         (!InNewSpace(object) &&
635          Page::FromAddress(object->address())
636              ->IsFlagSet(Page::PAGE_NEW_OLD_PROMOTION)));
637  if (!FLAG_allocation_site_pretenuring ||
638      !AllocationSite::CanTrack(object->map()->instance_type()))
639    return;
640  AllocationMemento* memento_candidate = FindAllocationMemento<kForGC>(object);
641  if (memento_candidate == nullptr) return;
642
643  if (mode == kGlobal) {
644    DCHECK_EQ(pretenuring_feedback, global_pretenuring_feedback_);
645    // Entering global pretenuring feedback is only used in the scavenger, where
646    // we are allowed to actually touch the allocation site.
647    if (!memento_candidate->IsValid()) return;
648    AllocationSite* site = memento_candidate->GetAllocationSite();
649    DCHECK(!site->IsZombie());
650    // For inserting in the global pretenuring storage we need to first
651    // increment the memento found count on the allocation site.
652    if (site->IncrementMementoFoundCount()) {
653      global_pretenuring_feedback_->LookupOrInsert(site,
654                                                   ObjectHash(site->address()));
655    }
656  } else {
657    DCHECK_EQ(mode, kCached);
658    DCHECK_NE(pretenuring_feedback, global_pretenuring_feedback_);
659    // Entering cached feedback is used in the parallel case. We are not allowed
660    // to dereference the allocation site and rather have to postpone all checks
661    // till actually merging the data.
662    Address key = memento_candidate->GetAllocationSiteUnchecked();
663    base::HashMap::Entry* e =
664        pretenuring_feedback->LookupOrInsert(key, ObjectHash(key));
665    DCHECK(e != nullptr);
666    (*bit_cast<intptr_t*>(&e->value))++;
667  }
668}
669
670
671void Heap::RemoveAllocationSitePretenuringFeedback(AllocationSite* site) {
672  global_pretenuring_feedback_->Remove(
673      site, static_cast<uint32_t>(bit_cast<uintptr_t>(site)));
674}
675
676bool Heap::CollectGarbage(AllocationSpace space,
677                          GarbageCollectionReason gc_reason,
678                          const v8::GCCallbackFlags callbackFlags) {
679  const char* collector_reason = NULL;
680  GarbageCollector collector = SelectGarbageCollector(space, &collector_reason);
681  return CollectGarbage(collector, gc_reason, collector_reason, callbackFlags);
682}
683
684
685Isolate* Heap::isolate() {
686  return reinterpret_cast<Isolate*>(
687      reinterpret_cast<intptr_t>(this) -
688      reinterpret_cast<size_t>(reinterpret_cast<Isolate*>(16)->heap()) + 16);
689}
690
691
692void Heap::ExternalStringTable::AddString(String* string) {
693  DCHECK(string->IsExternalString());
694  if (heap_->InNewSpace(string)) {
695    new_space_strings_.Add(string);
696  } else {
697    old_space_strings_.Add(string);
698  }
699}
700
701
702void Heap::ExternalStringTable::Iterate(ObjectVisitor* v) {
703  if (!new_space_strings_.is_empty()) {
704    Object** start = &new_space_strings_[0];
705    v->VisitPointers(start, start + new_space_strings_.length());
706  }
707  if (!old_space_strings_.is_empty()) {
708    Object** start = &old_space_strings_[0];
709    v->VisitPointers(start, start + old_space_strings_.length());
710  }
711}
712
713
714// Verify() is inline to avoid ifdef-s around its calls in release
715// mode.
716void Heap::ExternalStringTable::Verify() {
717#ifdef DEBUG
718  for (int i = 0; i < new_space_strings_.length(); ++i) {
719    Object* obj = Object::cast(new_space_strings_[i]);
720    DCHECK(heap_->InNewSpace(obj));
721    DCHECK(!obj->IsTheHole(heap_->isolate()));
722  }
723  for (int i = 0; i < old_space_strings_.length(); ++i) {
724    Object* obj = Object::cast(old_space_strings_[i]);
725    DCHECK(!heap_->InNewSpace(obj));
726    DCHECK(!obj->IsTheHole(heap_->isolate()));
727  }
728#endif
729}
730
731
732void Heap::ExternalStringTable::AddOldString(String* string) {
733  DCHECK(string->IsExternalString());
734  DCHECK(!heap_->InNewSpace(string));
735  old_space_strings_.Add(string);
736}
737
738
739void Heap::ExternalStringTable::ShrinkNewStrings(int position) {
740  new_space_strings_.Rewind(position);
741#ifdef VERIFY_HEAP
742  if (FLAG_verify_heap) {
743    Verify();
744  }
745#endif
746}
747
748void Heap::ClearInstanceofCache() { set_instanceof_cache_function(Smi::kZero); }
749
750Oddball* Heap::ToBoolean(bool condition) {
751  return condition ? true_value() : false_value();
752}
753
754
755void Heap::CompletelyClearInstanceofCache() {
756  set_instanceof_cache_map(Smi::kZero);
757  set_instanceof_cache_function(Smi::kZero);
758}
759
760
761uint32_t Heap::HashSeed() {
762  uint32_t seed = static_cast<uint32_t>(hash_seed()->value());
763  DCHECK(FLAG_randomize_hashes || seed == 0);
764  return seed;
765}
766
767
768int Heap::NextScriptId() {
769  int last_id = last_script_id()->value();
770  if (last_id == Smi::kMaxValue) {
771    last_id = 1;
772  } else {
773    last_id++;
774  }
775  set_last_script_id(Smi::FromInt(last_id));
776  return last_id;
777}
778
779void Heap::SetArgumentsAdaptorDeoptPCOffset(int pc_offset) {
780  DCHECK(arguments_adaptor_deopt_pc_offset() == Smi::kZero);
781  set_arguments_adaptor_deopt_pc_offset(Smi::FromInt(pc_offset));
782}
783
784void Heap::SetConstructStubDeoptPCOffset(int pc_offset) {
785  DCHECK(construct_stub_deopt_pc_offset() == Smi::kZero);
786  set_construct_stub_deopt_pc_offset(Smi::FromInt(pc_offset));
787}
788
789void Heap::SetGetterStubDeoptPCOffset(int pc_offset) {
790  DCHECK(getter_stub_deopt_pc_offset() == Smi::kZero);
791  set_getter_stub_deopt_pc_offset(Smi::FromInt(pc_offset));
792}
793
794void Heap::SetSetterStubDeoptPCOffset(int pc_offset) {
795  DCHECK(setter_stub_deopt_pc_offset() == Smi::kZero);
796  set_setter_stub_deopt_pc_offset(Smi::FromInt(pc_offset));
797}
798
799void Heap::SetInterpreterEntryReturnPCOffset(int pc_offset) {
800  DCHECK(interpreter_entry_return_pc_offset() == Smi::kZero);
801  set_interpreter_entry_return_pc_offset(Smi::FromInt(pc_offset));
802}
803
804int Heap::GetNextTemplateSerialNumber() {
805  int next_serial_number = next_template_serial_number()->value() + 1;
806  set_next_template_serial_number(Smi::FromInt(next_serial_number));
807  return next_serial_number;
808}
809
810void Heap::SetSerializedTemplates(FixedArray* templates) {
811  DCHECK_EQ(empty_fixed_array(), serialized_templates());
812  set_serialized_templates(templates);
813}
814
815void Heap::CreateObjectStats() {
816  if (V8_LIKELY(FLAG_gc_stats == 0)) return;
817  if (!live_object_stats_) {
818    live_object_stats_ = new ObjectStats(this);
819  }
820  if (!dead_object_stats_) {
821    dead_object_stats_ = new ObjectStats(this);
822  }
823}
824
825AlwaysAllocateScope::AlwaysAllocateScope(Isolate* isolate)
826    : heap_(isolate->heap()) {
827  heap_->always_allocate_scope_count_.Increment(1);
828}
829
830
831AlwaysAllocateScope::~AlwaysAllocateScope() {
832  heap_->always_allocate_scope_count_.Increment(-1);
833}
834
835
836void VerifyPointersVisitor::VisitPointers(Object** start, Object** end) {
837  for (Object** current = start; current < end; current++) {
838    if ((*current)->IsHeapObject()) {
839      HeapObject* object = HeapObject::cast(*current);
840      CHECK(object->GetIsolate()->heap()->Contains(object));
841      CHECK(object->map()->IsMap());
842    }
843  }
844}
845
846
847void VerifySmisVisitor::VisitPointers(Object** start, Object** end) {
848  for (Object** current = start; current < end; current++) {
849    CHECK((*current)->IsSmi());
850  }
851}
852}  // namespace internal
853}  // namespace v8
854
855#endif  // V8_HEAP_HEAP_INL_H_
856