1c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com// Copyright 2011 the V8 project authors. All rights reserved. 23484964a86451e86dcf04be9bd8c0d76ee04f081rossberg@chromium.org// Use of this source code is governed by a BSD-style license that can be 33484964a86451e86dcf04be9bd8c0d76ee04f081rossberg@chromium.org// found in the LICENSE file. 4c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com 5ca29dd85fa02449d17188f5a6ff9a7cdf2ad9680danno@chromium.org#include <algorithm> 6ca29dd85fa02449d17188f5a6ff9a7cdf2ad9680danno@chromium.org 7196eb601290dc49c3754da728dc58700dff2de1bmachenbach@chromium.org#include "src/v8.h" 81e2d50cf3d94ff48285da107b7a9da1ad0fc873dmachenbach@chromium.org 91e2d50cf3d94ff48285da107b7a9da1ad0fc873dmachenbach@chromium.org#include "src/base/atomicops.h" 10196eb601290dc49c3754da728dc58700dff2de1bmachenbach@chromium.org#include "src/counters.h" 1131c0e32e19ad3df48525fa9e7b2d1c0c07496d00machenbach@chromium.org#include "src/heap/store-buffer-inl.h" 12c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com 13c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.comnamespace v8 { 14c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.comnamespace internal { 15c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com 16c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.comStoreBuffer::StoreBuffer(Heap* heap) 17c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com : heap_(heap), 18c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com start_(NULL), 19c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com limit_(NULL), 20c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com old_start_(NULL), 21c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com old_limit_(NULL), 22c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com old_top_(NULL), 2364e3a4be4a99f31920128de34573c8ac9038de42ricow@chromium.org old_reserved_limit_(NULL), 24c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com old_buffer_is_sorted_(false), 25c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com old_buffer_is_filtered_(false), 2670d29e1ad7a6f1c163f625252ca32ecc522cb155machenbach@chromium.org during_gc_(false), 27c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com store_buffer_rebuilding_enabled_(false), 28c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com callback_(NULL), 29c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com may_move_store_buffer_entries_(true), 30c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com virtual_memory_(NULL), 31b295776b8c02617720d03b6dbcf2f912dfd2c4e7vegorov@chromium.org hash_set_1_(NULL), 32b295776b8c02617720d03b6dbcf2f912dfd2c4e7vegorov@chromium.org hash_set_2_(NULL), 3331c0e32e19ad3df48525fa9e7b2d1c0c07496d00machenbach@chromium.org hash_sets_are_empty_(true) {} 34c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com 35c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com 36f2038fb01417bcf7698b87a5dfaa4a861539618aerik.corry@gmail.comvoid StoreBuffer::SetUp() { 375de0074a922429f5e0ec2cf140c2d2989bf88140yangguo@chromium.org virtual_memory_ = new base::VirtualMemory(kStoreBufferSize * 3); 38c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com uintptr_t start_as_int = 39c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com reinterpret_cast<uintptr_t>(virtual_memory_->address()); 40c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com start_ = 41c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com reinterpret_cast<Address*>(RoundUp(start_as_int, kStoreBufferSize * 2)); 4264e3a4be4a99f31920128de34573c8ac9038de42ricow@chromium.org limit_ = start_ + (kStoreBufferSize / kPointerSize); 4364e3a4be4a99f31920128de34573c8ac9038de42ricow@chromium.org 4464e3a4be4a99f31920128de34573c8ac9038de42ricow@chromium.org old_virtual_memory_ = 4570d29e1ad7a6f1c163f625252ca32ecc522cb155machenbach@chromium.org new base::VirtualMemory(kOldStoreBufferLength * kPointerSize); 4664e3a4be4a99f31920128de34573c8ac9038de42ricow@chromium.org old_top_ = old_start_ = 4764e3a4be4a99f31920128de34573c8ac9038de42ricow@chromium.org reinterpret_cast<Address*>(old_virtual_memory_->address()); 4864e3a4be4a99f31920128de34573c8ac9038de42ricow@chromium.org // Don't know the alignment requirements of the OS, but it is certainly not 4964e3a4be4a99f31920128de34573c8ac9038de42ricow@chromium.org // less than 0xfff. 50e3c177a423baa3c30225c4e422b6f6c76d38b951machenbach@chromium.org DCHECK((reinterpret_cast<uintptr_t>(old_start_) & 0xfff) == 0); 515de0074a922429f5e0ec2cf140c2d2989bf88140yangguo@chromium.org int initial_length = 525de0074a922429f5e0ec2cf140c2d2989bf88140yangguo@chromium.org static_cast<int>(base::OS::CommitPageSize() / kPointerSize); 53e3c177a423baa3c30225c4e422b6f6c76d38b951machenbach@chromium.org DCHECK(initial_length > 0); 54e3c177a423baa3c30225c4e422b6f6c76d38b951machenbach@chromium.org DCHECK(initial_length <= kOldStoreBufferLength); 5564e3a4be4a99f31920128de34573c8ac9038de42ricow@chromium.org old_limit_ = old_start_ + initial_length; 5670d29e1ad7a6f1c163f625252ca32ecc522cb155machenbach@chromium.org old_reserved_limit_ = old_start_ + kOldStoreBufferLength; 5764e3a4be4a99f31920128de34573c8ac9038de42ricow@chromium.org 5831c0e32e19ad3df48525fa9e7b2d1c0c07496d00machenbach@chromium.org CHECK(old_virtual_memory_->Commit(reinterpret_cast<void*>(old_start_), 5931c0e32e19ad3df48525fa9e7b2d1c0c07496d00machenbach@chromium.org (old_limit_ - old_start_) * kPointerSize, 6031c0e32e19ad3df48525fa9e7b2d1c0c07496d00machenbach@chromium.org false)); 61c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com 62e3c177a423baa3c30225c4e422b6f6c76d38b951machenbach@chromium.org DCHECK(reinterpret_cast<Address>(start_) >= virtual_memory_->address()); 63e3c177a423baa3c30225c4e422b6f6c76d38b951machenbach@chromium.org DCHECK(reinterpret_cast<Address>(limit_) >= virtual_memory_->address()); 64c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com Address* vm_limit = reinterpret_cast<Address*>( 65c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com reinterpret_cast<char*>(virtual_memory_->address()) + 6631c0e32e19ad3df48525fa9e7b2d1c0c07496d00machenbach@chromium.org virtual_memory_->size()); 67e3c177a423baa3c30225c4e422b6f6c76d38b951machenbach@chromium.org DCHECK(start_ <= vm_limit); 68e3c177a423baa3c30225c4e422b6f6c76d38b951machenbach@chromium.org DCHECK(limit_ <= vm_limit); 69c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com USE(vm_limit); 70e3c177a423baa3c30225c4e422b6f6c76d38b951machenbach@chromium.org DCHECK((reinterpret_cast<uintptr_t>(limit_) & kStoreBufferOverflowBit) != 0); 71e3c177a423baa3c30225c4e422b6f6c76d38b951machenbach@chromium.org DCHECK((reinterpret_cast<uintptr_t>(limit_ - 1) & kStoreBufferOverflowBit) == 72c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com 0); 73c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com 7464e3a4be4a99f31920128de34573c8ac9038de42ricow@chromium.org CHECK(virtual_memory_->Commit(reinterpret_cast<Address>(start_), 7564e3a4be4a99f31920128de34573c8ac9038de42ricow@chromium.org kStoreBufferSize, 7664e3a4be4a99f31920128de34573c8ac9038de42ricow@chromium.org false)); // Not executable. 77c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com heap_->public_set_store_buffer_top(start_); 78c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com 79b295776b8c02617720d03b6dbcf2f912dfd2c4e7vegorov@chromium.org hash_set_1_ = new uintptr_t[kHashSetLength]; 80b295776b8c02617720d03b6dbcf2f912dfd2c4e7vegorov@chromium.org hash_set_2_ = new uintptr_t[kHashSetLength]; 81b295776b8c02617720d03b6dbcf2f912dfd2c4e7vegorov@chromium.org hash_sets_are_empty_ = false; 82c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com 83b295776b8c02617720d03b6dbcf2f912dfd2c4e7vegorov@chromium.org ClearFilteringHashSets(); 84c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com} 85c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com 86c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com 87c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.comvoid StoreBuffer::TearDown() { 88c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com delete virtual_memory_; 8964e3a4be4a99f31920128de34573c8ac9038de42ricow@chromium.org delete old_virtual_memory_; 90b295776b8c02617720d03b6dbcf2f912dfd2c4e7vegorov@chromium.org delete[] hash_set_1_; 91b295776b8c02617720d03b6dbcf2f912dfd2c4e7vegorov@chromium.org delete[] hash_set_2_; 9270d29e1ad7a6f1c163f625252ca32ecc522cb155machenbach@chromium.org old_start_ = old_top_ = old_limit_ = old_reserved_limit_ = NULL; 9370d29e1ad7a6f1c163f625252ca32ecc522cb155machenbach@chromium.org start_ = limit_ = NULL; 94c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com heap_->public_set_store_buffer_top(start_); 95c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com} 96c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com 97c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com 98c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.comvoid StoreBuffer::StoreBufferOverflow(Isolate* isolate) { 99c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com isolate->heap()->store_buffer()->Compact(); 1001510d58cbcf57c82a10e7d390bfe21a7ae68ba43mstarzinger@chromium.org isolate->counters()->store_buffer_overflows()->Increment(); 101c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com} 102c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com 103c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com 104c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.comvoid StoreBuffer::Uniq() { 105c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com // Remove adjacent duplicates and cells that do not point at new space. 106c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com Address previous = NULL; 107c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com Address* write = old_start_; 108e3c177a423baa3c30225c4e422b6f6c76d38b951machenbach@chromium.org DCHECK(may_move_store_buffer_entries_); 109c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com for (Address* read = old_start_; read < old_top_; read++) { 110c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com Address current = *read; 111c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com if (current != previous) { 112c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com if (heap_->InNewSpace(*reinterpret_cast<Object**>(current))) { 113c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com *write++ = current; 114c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com } 115c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com } 116c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com previous = current; 117c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com } 118c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com old_top_ = write; 119c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com} 120c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com 121c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com 12241728483f231e098a8dd85f109b5a30e6ccc3c39danno@chromium.orgbool StoreBuffer::SpaceAvailable(intptr_t space_needed) { 12341728483f231e098a8dd85f109b5a30e6ccc3c39danno@chromium.org return old_limit_ - old_top_ >= space_needed; 12441728483f231e098a8dd85f109b5a30e6ccc3c39danno@chromium.org} 12541728483f231e098a8dd85f109b5a30e6ccc3c39danno@chromium.org 12641728483f231e098a8dd85f109b5a30e6ccc3c39danno@chromium.org 12764e3a4be4a99f31920128de34573c8ac9038de42ricow@chromium.orgvoid StoreBuffer::EnsureSpace(intptr_t space_needed) { 12864e3a4be4a99f31920128de34573c8ac9038de42ricow@chromium.org while (old_limit_ - old_top_ < space_needed && 12970d29e1ad7a6f1c163f625252ca32ecc522cb155machenbach@chromium.org old_limit_ < old_reserved_limit_) { 13064e3a4be4a99f31920128de34573c8ac9038de42ricow@chromium.org size_t grow = old_limit_ - old_start_; // Double size. 13164e3a4be4a99f31920128de34573c8ac9038de42ricow@chromium.org CHECK(old_virtual_memory_->Commit(reinterpret_cast<void*>(old_limit_), 13231c0e32e19ad3df48525fa9e7b2d1c0c07496d00machenbach@chromium.org grow * kPointerSize, false)); 13364e3a4be4a99f31920128de34573c8ac9038de42ricow@chromium.org old_limit_ += grow; 13464e3a4be4a99f31920128de34573c8ac9038de42ricow@chromium.org } 13564e3a4be4a99f31920128de34573c8ac9038de42ricow@chromium.org 13641728483f231e098a8dd85f109b5a30e6ccc3c39danno@chromium.org if (SpaceAvailable(space_needed)) return; 13764e3a4be4a99f31920128de34573c8ac9038de42ricow@chromium.org 138c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com if (old_buffer_is_filtered_) return; 139e3c177a423baa3c30225c4e422b6f6c76d38b951machenbach@chromium.org DCHECK(may_move_store_buffer_entries_); 140c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com Compact(); 141c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com 142c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com old_buffer_is_filtered_ = true; 143c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com bool page_has_scan_on_scavenge_flag = false; 144c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com 145c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com PointerChunkIterator it(heap_); 146c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com MemoryChunk* chunk; 147c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com while ((chunk = it.next()) != NULL) { 148639bac0c5319f96e1bbe3399fb7f7f37344928bddslomov@chromium.org if (chunk->scan_on_scavenge()) { 149639bac0c5319f96e1bbe3399fb7f7f37344928bddslomov@chromium.org page_has_scan_on_scavenge_flag = true; 150639bac0c5319f96e1bbe3399fb7f7f37344928bddslomov@chromium.org break; 151639bac0c5319f96e1bbe3399fb7f7f37344928bddslomov@chromium.org } 152c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com } 153c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com 154c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com if (page_has_scan_on_scavenge_flag) { 155c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com Filter(MemoryChunk::SCAN_ON_SCAVENGE); 156c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com } 157c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com 15841728483f231e098a8dd85f109b5a30e6ccc3c39danno@chromium.org if (SpaceAvailable(space_needed)) return; 159c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com 16070d29e1ad7a6f1c163f625252ca32ecc522cb155machenbach@chromium.org // Sample 1 entry in 97 and filter out the pages where we estimate that more 16170d29e1ad7a6f1c163f625252ca32ecc522cb155machenbach@chromium.org // than 1 in 8 pointers are to new space. 16270d29e1ad7a6f1c163f625252ca32ecc522cb155machenbach@chromium.org static const int kSampleFinenesses = 5; 16370d29e1ad7a6f1c163f625252ca32ecc522cb155machenbach@chromium.org static const struct Samples { 16470d29e1ad7a6f1c163f625252ca32ecc522cb155machenbach@chromium.org int prime_sample_step; 16570d29e1ad7a6f1c163f625252ca32ecc522cb155machenbach@chromium.org int threshold; 16631c0e32e19ad3df48525fa9e7b2d1c0c07496d00machenbach@chromium.org } samples[kSampleFinenesses] = { 16731c0e32e19ad3df48525fa9e7b2d1c0c07496d00machenbach@chromium.org {97, ((Page::kPageSize / kPointerSize) / 97) / 8}, 16831c0e32e19ad3df48525fa9e7b2d1c0c07496d00machenbach@chromium.org {23, ((Page::kPageSize / kPointerSize) / 23) / 16}, 16931c0e32e19ad3df48525fa9e7b2d1c0c07496d00machenbach@chromium.org {7, ((Page::kPageSize / kPointerSize) / 7) / 32}, 17031c0e32e19ad3df48525fa9e7b2d1c0c07496d00machenbach@chromium.org {3, ((Page::kPageSize / kPointerSize) / 3) / 256}, 17131c0e32e19ad3df48525fa9e7b2d1c0c07496d00machenbach@chromium.org {1, 0}}; 17270d29e1ad7a6f1c163f625252ca32ecc522cb155machenbach@chromium.org for (int i = 0; i < kSampleFinenesses; i++) { 17370d29e1ad7a6f1c163f625252ca32ecc522cb155machenbach@chromium.org ExemptPopularPages(samples[i].prime_sample_step, samples[i].threshold); 17470d29e1ad7a6f1c163f625252ca32ecc522cb155machenbach@chromium.org // As a last resort we mark all pages as being exempt from the store buffer. 175e3c177a423baa3c30225c4e422b6f6c76d38b951machenbach@chromium.org DCHECK(i != (kSampleFinenesses - 1) || old_top_ == old_start_); 17670d29e1ad7a6f1c163f625252ca32ecc522cb155machenbach@chromium.org if (SpaceAvailable(space_needed)) return; 17770d29e1ad7a6f1c163f625252ca32ecc522cb155machenbach@chromium.org } 17870d29e1ad7a6f1c163f625252ca32ecc522cb155machenbach@chromium.org UNREACHABLE(); 179c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com} 180c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com 181c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com 182c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com// Sample the store buffer to see if some pages are taking up a lot of space 183c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com// in the store buffer. 184c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.comvoid StoreBuffer::ExemptPopularPages(int prime_sample_step, int threshold) { 185c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com PointerChunkIterator it(heap_); 186c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com MemoryChunk* chunk; 187c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com while ((chunk = it.next()) != NULL) { 188c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com chunk->set_store_buffer_counter(0); 189c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com } 190c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com bool created_new_scan_on_scavenge_pages = false; 191c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com MemoryChunk* previous_chunk = NULL; 192c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com for (Address* p = old_start_; p < old_top_; p += prime_sample_step) { 193c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com Address addr = *p; 194c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com MemoryChunk* containing_chunk = NULL; 195c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com if (previous_chunk != NULL && previous_chunk->Contains(addr)) { 196c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com containing_chunk = previous_chunk; 197c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com } else { 198c5d4971574b7a205fa0e788d8121dc79485e5e67hpayer@chromium.org containing_chunk = MemoryChunk::FromAnyPointerAddress(heap_, addr); 199c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com } 200c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com int old_counter = containing_chunk->store_buffer_counter(); 2019cbaabda8b4daeb06759ace10c926ab55bb69d7bulan@chromium.org if (old_counter >= threshold) { 202c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com containing_chunk->set_scan_on_scavenge(true); 203c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com created_new_scan_on_scavenge_pages = true; 204c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com } 205c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com containing_chunk->set_store_buffer_counter(old_counter + 1); 206c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com previous_chunk = containing_chunk; 207c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com } 208c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com if (created_new_scan_on_scavenge_pages) { 209c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com Filter(MemoryChunk::SCAN_ON_SCAVENGE); 210c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com } 211c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com old_buffer_is_filtered_ = true; 212c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com} 213c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com 214c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com 215c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.comvoid StoreBuffer::Filter(int flag) { 216c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com Address* new_top = old_start_; 217c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com MemoryChunk* previous_chunk = NULL; 218c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com for (Address* p = old_start_; p < old_top_; p++) { 219c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com Address addr = *p; 220c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com MemoryChunk* containing_chunk = NULL; 221c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com if (previous_chunk != NULL && previous_chunk->Contains(addr)) { 222c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com containing_chunk = previous_chunk; 223c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com } else { 224c5d4971574b7a205fa0e788d8121dc79485e5e67hpayer@chromium.org containing_chunk = MemoryChunk::FromAnyPointerAddress(heap_, addr); 225c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com previous_chunk = containing_chunk; 226c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com } 227c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com if (!containing_chunk->IsFlagSet(flag)) { 228c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com *new_top++ = addr; 229c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com } 230c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com } 231c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com old_top_ = new_top; 232b295776b8c02617720d03b6dbcf2f912dfd2c4e7vegorov@chromium.org 233b295776b8c02617720d03b6dbcf2f912dfd2c4e7vegorov@chromium.org // Filtering hash sets are inconsistent with the store buffer after this 234b295776b8c02617720d03b6dbcf2f912dfd2c4e7vegorov@chromium.org // operation. 235b295776b8c02617720d03b6dbcf2f912dfd2c4e7vegorov@chromium.org ClearFilteringHashSets(); 236c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com} 237c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com 238c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com 239c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.comvoid StoreBuffer::SortUniq() { 240c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com Compact(); 241c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com if (old_buffer_is_sorted_) return; 242ca29dd85fa02449d17188f5a6ff9a7cdf2ad9680danno@chromium.org std::sort(old_start_, old_top_); 243c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com Uniq(); 244c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com 245c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com old_buffer_is_sorted_ = true; 246b295776b8c02617720d03b6dbcf2f912dfd2c4e7vegorov@chromium.org 247b295776b8c02617720d03b6dbcf2f912dfd2c4e7vegorov@chromium.org // Filtering hash sets are inconsistent with the store buffer after this 248b295776b8c02617720d03b6dbcf2f912dfd2c4e7vegorov@chromium.org // operation. 249b295776b8c02617720d03b6dbcf2f912dfd2c4e7vegorov@chromium.org ClearFilteringHashSets(); 250c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com} 251c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com 252c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com 253c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.combool StoreBuffer::PrepareForIteration() { 254c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com Compact(); 255c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com PointerChunkIterator it(heap_); 256c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com MemoryChunk* chunk; 257c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com bool page_has_scan_on_scavenge_flag = false; 258c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com while ((chunk = it.next()) != NULL) { 259639bac0c5319f96e1bbe3399fb7f7f37344928bddslomov@chromium.org if (chunk->scan_on_scavenge()) { 260639bac0c5319f96e1bbe3399fb7f7f37344928bddslomov@chromium.org page_has_scan_on_scavenge_flag = true; 261639bac0c5319f96e1bbe3399fb7f7f37344928bddslomov@chromium.org break; 262639bac0c5319f96e1bbe3399fb7f7f37344928bddslomov@chromium.org } 263c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com } 264c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com 265c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com if (page_has_scan_on_scavenge_flag) { 266c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com Filter(MemoryChunk::SCAN_ON_SCAVENGE); 267c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com } 268b295776b8c02617720d03b6dbcf2f912dfd2c4e7vegorov@chromium.org 269b295776b8c02617720d03b6dbcf2f912dfd2c4e7vegorov@chromium.org // Filtering hash sets are inconsistent with the store buffer after 270b295776b8c02617720d03b6dbcf2f912dfd2c4e7vegorov@chromium.org // iteration. 271b295776b8c02617720d03b6dbcf2f912dfd2c4e7vegorov@chromium.org ClearFilteringHashSets(); 272b295776b8c02617720d03b6dbcf2f912dfd2c4e7vegorov@chromium.org 273c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com return page_has_scan_on_scavenge_flag; 274c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com} 275c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com 276c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com 277c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com#ifdef DEBUG 278c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.comvoid StoreBuffer::Clean() { 279b295776b8c02617720d03b6dbcf2f912dfd2c4e7vegorov@chromium.org ClearFilteringHashSets(); 280c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com Uniq(); // Also removes things that no longer point to new space. 28141728483f231e098a8dd85f109b5a30e6ccc3c39danno@chromium.org EnsureSpace(kStoreBufferSize / 2); 282c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com} 283c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com 284c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com 285c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.comstatic Address* in_store_buffer_1_element_cache = NULL; 286c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com 287c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com 288c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.combool StoreBuffer::CellIsInStoreBuffer(Address cell_address) { 289c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com if (!FLAG_enable_slow_asserts) return true; 290c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com if (in_store_buffer_1_element_cache != NULL && 291c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com *in_store_buffer_1_element_cache == cell_address) { 292c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com return true; 293c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com } 294c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com Address* top = reinterpret_cast<Address*>(heap_->store_buffer_top()); 295c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com for (Address* current = top - 1; current >= start_; current--) { 296c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com if (*current == cell_address) { 297c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com in_store_buffer_1_element_cache = current; 298c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com return true; 299c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com } 300c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com } 301c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com for (Address* current = old_top_ - 1; current >= old_start_; current--) { 302c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com if (*current == cell_address) { 303c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com in_store_buffer_1_element_cache = current; 304c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com return true; 305c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com } 306c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com } 307c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com return false; 308c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com} 309c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com#endif 310c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com 311c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com 312b295776b8c02617720d03b6dbcf2f912dfd2c4e7vegorov@chromium.orgvoid StoreBuffer::ClearFilteringHashSets() { 313b295776b8c02617720d03b6dbcf2f912dfd2c4e7vegorov@chromium.org if (!hash_sets_are_empty_) { 31431c0e32e19ad3df48525fa9e7b2d1c0c07496d00machenbach@chromium.org memset(reinterpret_cast<void*>(hash_set_1_), 0, 315b295776b8c02617720d03b6dbcf2f912dfd2c4e7vegorov@chromium.org sizeof(uintptr_t) * kHashSetLength); 31631c0e32e19ad3df48525fa9e7b2d1c0c07496d00machenbach@chromium.org memset(reinterpret_cast<void*>(hash_set_2_), 0, 317b295776b8c02617720d03b6dbcf2f912dfd2c4e7vegorov@chromium.org sizeof(uintptr_t) * kHashSetLength); 318b295776b8c02617720d03b6dbcf2f912dfd2c4e7vegorov@chromium.org hash_sets_are_empty_ = true; 319b295776b8c02617720d03b6dbcf2f912dfd2c4e7vegorov@chromium.org } 320c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com} 321c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com 322c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com 32370d29e1ad7a6f1c163f625252ca32ecc522cb155machenbach@chromium.orgvoid StoreBuffer::GCPrologue() { 324b295776b8c02617720d03b6dbcf2f912dfd2c4e7vegorov@chromium.org ClearFilteringHashSets(); 32570d29e1ad7a6f1c163f625252ca32ecc522cb155machenbach@chromium.org during_gc_ = true; 326c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com} 327c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com 328c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com 329c859c4f83f6ec3e010213dbbe447cc1f927d01c5svenpanne@chromium.org#ifdef VERIFY_HEAP 330c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.comvoid StoreBuffer::VerifyPointers(LargeObjectSpace* space) { 331c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com LargeObjectIterator it(space); 332c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com for (HeapObject* object = it.Next(); object != NULL; object = it.Next()) { 333c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com if (object->IsFixedArray()) { 334c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com Address slot_address = object->address(); 335c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com Address end = object->address() + object->Size(); 336c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com 337c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com while (slot_address < end) { 338c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com HeapObject** slot = reinterpret_cast<HeapObject**>(slot_address); 339c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com // When we are not in GC the Heap::InNewSpace() predicate 340c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com // checks that pointers which satisfy predicate point into 341c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com // the active semispace. 3429e41f9ecf5042292a9efcb36e264b37621199553machenbach@chromium.org Object* object = reinterpret_cast<Object*>( 3431e2d50cf3d94ff48285da107b7a9da1ad0fc873dmachenbach@chromium.org base::NoBarrier_Load(reinterpret_cast<base::AtomicWord*>(slot))); 3449e41f9ecf5042292a9efcb36e264b37621199553machenbach@chromium.org heap_->InNewSpace(object); 345c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com slot_address += kPointerSize; 346c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com } 347c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com } 348c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com } 349c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com} 350c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com#endif 351c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com 352c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com 353c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.comvoid StoreBuffer::Verify() { 354c859c4f83f6ec3e010213dbbe447cc1f927d01c5svenpanne@chromium.org#ifdef VERIFY_HEAP 355c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com VerifyPointers(heap_->lo_space()); 356c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com#endif 357c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com} 358c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com 359c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com 360c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.comvoid StoreBuffer::GCEpilogue() { 36170d29e1ad7a6f1c163f625252ca32ecc522cb155machenbach@chromium.org during_gc_ = false; 362c859c4f83f6ec3e010213dbbe447cc1f927d01c5svenpanne@chromium.org#ifdef VERIFY_HEAP 363394dbcf9009cf5203b6d85e8b515fcff072040f3erik.corry@gmail.com if (FLAG_verify_heap) { 364394dbcf9009cf5203b6d85e8b515fcff072040f3erik.corry@gmail.com Verify(); 365394dbcf9009cf5203b6d85e8b515fcff072040f3erik.corry@gmail.com } 366c859c4f83f6ec3e010213dbbe447cc1f927d01c5svenpanne@chromium.org#endif 367c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com} 368c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com 369c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com 370c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.comvoid StoreBuffer::FindPointersToNewSpaceInRegion( 37131c0e32e19ad3df48525fa9e7b2d1c0c07496d00machenbach@chromium.org Address start, Address end, ObjectSlotCallback slot_callback, 372169691d93a961c8b511f8ac8fd8ee33d081ca10fdanno@chromium.org bool clear_maps) { 37331c0e32e19ad3df48525fa9e7b2d1c0c07496d00machenbach@chromium.org for (Address slot_address = start; slot_address < end; 374c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com slot_address += kPointerSize) { 375c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com Object** slot = reinterpret_cast<Object**>(slot_address); 3769e41f9ecf5042292a9efcb36e264b37621199553machenbach@chromium.org Object* object = reinterpret_cast<Object*>( 3771e2d50cf3d94ff48285da107b7a9da1ad0fc873dmachenbach@chromium.org base::NoBarrier_Load(reinterpret_cast<base::AtomicWord*>(slot))); 3789e41f9ecf5042292a9efcb36e264b37621199553machenbach@chromium.org if (heap_->InNewSpace(object)) { 3799e41f9ecf5042292a9efcb36e264b37621199553machenbach@chromium.org HeapObject* heap_object = reinterpret_cast<HeapObject*>(object); 380e3c177a423baa3c30225c4e422b6f6c76d38b951machenbach@chromium.org DCHECK(heap_object->IsHeapObject()); 381169691d93a961c8b511f8ac8fd8ee33d081ca10fdanno@chromium.org // The new space object was not promoted if it still contains a map 382169691d93a961c8b511f8ac8fd8ee33d081ca10fdanno@chromium.org // pointer. Clear the map field now lazily. 3839e41f9ecf5042292a9efcb36e264b37621199553machenbach@chromium.org if (clear_maps) ClearDeadObject(heap_object); 3849e41f9ecf5042292a9efcb36e264b37621199553machenbach@chromium.org slot_callback(reinterpret_cast<HeapObject**>(slot), heap_object); 3859e41f9ecf5042292a9efcb36e264b37621199553machenbach@chromium.org object = reinterpret_cast<Object*>( 3861e2d50cf3d94ff48285da107b7a9da1ad0fc873dmachenbach@chromium.org base::NoBarrier_Load(reinterpret_cast<base::AtomicWord*>(slot))); 3879e41f9ecf5042292a9efcb36e264b37621199553machenbach@chromium.org if (heap_->InNewSpace(object)) { 388c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com EnterDirectlyIntoStoreBuffer(slot_address); 389c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com } 390c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com } 391c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com } 392c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com} 393c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com 394c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com 39531c0e32e19ad3df48525fa9e7b2d1c0c07496d00machenbach@chromium.orgvoid StoreBuffer::IteratePointersInStoreBuffer(ObjectSlotCallback slot_callback, 39631c0e32e19ad3df48525fa9e7b2d1c0c07496d00machenbach@chromium.org bool clear_maps) { 397c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com Address* limit = old_top_; 398c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com old_top_ = old_start_; 399c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com { 400c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com DontMoveStoreBufferEntriesScope scope(this); 401c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com for (Address* current = old_start_; current < limit; current++) { 402c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com#ifdef DEBUG 403c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com Address* saved_top = old_top_; 404c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com#endif 405c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com Object** slot = reinterpret_cast<Object**>(*current); 4069e41f9ecf5042292a9efcb36e264b37621199553machenbach@chromium.org Object* object = reinterpret_cast<Object*>( 4071e2d50cf3d94ff48285da107b7a9da1ad0fc873dmachenbach@chromium.org base::NoBarrier_Load(reinterpret_cast<base::AtomicWord*>(slot))); 408c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com if (heap_->InFromSpace(object)) { 409c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com HeapObject* heap_object = reinterpret_cast<HeapObject*>(object); 410169691d93a961c8b511f8ac8fd8ee33d081ca10fdanno@chromium.org // The new space object was not promoted if it still contains a map 411169691d93a961c8b511f8ac8fd8ee33d081ca10fdanno@chromium.org // pointer. Clear the map field now lazily. 412169691d93a961c8b511f8ac8fd8ee33d081ca10fdanno@chromium.org if (clear_maps) ClearDeadObject(heap_object); 413c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com slot_callback(reinterpret_cast<HeapObject**>(slot), heap_object); 4149e41f9ecf5042292a9efcb36e264b37621199553machenbach@chromium.org object = reinterpret_cast<Object*>( 4151e2d50cf3d94ff48285da107b7a9da1ad0fc873dmachenbach@chromium.org base::NoBarrier_Load(reinterpret_cast<base::AtomicWord*>(slot))); 4169e41f9ecf5042292a9efcb36e264b37621199553machenbach@chromium.org if (heap_->InNewSpace(object)) { 417c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com EnterDirectlyIntoStoreBuffer(reinterpret_cast<Address>(slot)); 418c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com } 419c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com } 420e3c177a423baa3c30225c4e422b6f6c76d38b951machenbach@chromium.org DCHECK(old_top_ == saved_top + 1 || old_top_ == saved_top); 421c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com } 422c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com } 423c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com} 424c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com 425c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com 426c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.comvoid StoreBuffer::IteratePointersToNewSpace(ObjectSlotCallback slot_callback) { 427169691d93a961c8b511f8ac8fd8ee33d081ca10fdanno@chromium.org IteratePointersToNewSpace(slot_callback, false); 428169691d93a961c8b511f8ac8fd8ee33d081ca10fdanno@chromium.org} 429169691d93a961c8b511f8ac8fd8ee33d081ca10fdanno@chromium.org 430169691d93a961c8b511f8ac8fd8ee33d081ca10fdanno@chromium.org 431169691d93a961c8b511f8ac8fd8ee33d081ca10fdanno@chromium.orgvoid StoreBuffer::IteratePointersToNewSpaceAndClearMaps( 432169691d93a961c8b511f8ac8fd8ee33d081ca10fdanno@chromium.org ObjectSlotCallback slot_callback) { 433169691d93a961c8b511f8ac8fd8ee33d081ca10fdanno@chromium.org IteratePointersToNewSpace(slot_callback, true); 434169691d93a961c8b511f8ac8fd8ee33d081ca10fdanno@chromium.org} 435169691d93a961c8b511f8ac8fd8ee33d081ca10fdanno@chromium.org 436169691d93a961c8b511f8ac8fd8ee33d081ca10fdanno@chromium.org 437169691d93a961c8b511f8ac8fd8ee33d081ca10fdanno@chromium.orgvoid StoreBuffer::IteratePointersToNewSpace(ObjectSlotCallback slot_callback, 438169691d93a961c8b511f8ac8fd8ee33d081ca10fdanno@chromium.org bool clear_maps) { 439c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com // We do not sort or remove duplicated entries from the store buffer because 440c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com // we expect that callback will rebuild the store buffer thus removing 441c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com // all duplicates and pointers to old space. 442c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com bool some_pages_to_scan = PrepareForIteration(); 443c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com 444c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com // TODO(gc): we want to skip slots on evacuation candidates 445c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com // but we can't simply figure that out from slot address 446c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com // because slot can belong to a large object. 447169691d93a961c8b511f8ac8fd8ee33d081ca10fdanno@chromium.org IteratePointersInStoreBuffer(slot_callback, clear_maps); 448c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com 449c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com // We are done scanning all the pointers that were in the store buffer, but 450c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com // there may be some pages marked scan_on_scavenge that have pointers to new 451c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com // space that are not in the store buffer. We must scan them now. As we 452c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com // scan, the surviving pointers to new space will be added to the store 453c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com // buffer. If there are still a lot of pointers to new space then we will 454c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com // keep the scan_on_scavenge flag on the page and discard the pointers that 455c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com // were added to the store buffer. If there are not many pointers to new 456c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com // space left on the page we will keep the pointers in the store buffer and 457c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com // remove the flag from the page. 458c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com if (some_pages_to_scan) { 459c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com if (callback_ != NULL) { 460c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com (*callback_)(heap_, NULL, kStoreBufferStartScanningPagesEvent); 461c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com } 462c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com PointerChunkIterator it(heap_); 463c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com MemoryChunk* chunk; 464c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com while ((chunk = it.next()) != NULL) { 465c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com if (chunk->scan_on_scavenge()) { 466c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com chunk->set_scan_on_scavenge(false); 467c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com if (callback_ != NULL) { 468c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com (*callback_)(heap_, chunk, kStoreBufferScanningPageEvent); 469c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com } 470c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com if (chunk->owner() == heap_->lo_space()) { 471c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com LargePage* large_page = reinterpret_cast<LargePage*>(chunk); 472c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com HeapObject* array = large_page->GetObject(); 473e3c177a423baa3c30225c4e422b6f6c76d38b951machenbach@chromium.org DCHECK(array->IsFixedArray()); 474c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com Address start = array->address(); 475c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com Address end = start + array->Size(); 476169691d93a961c8b511f8ac8fd8ee33d081ca10fdanno@chromium.org FindPointersToNewSpaceInRegion(start, end, slot_callback, clear_maps); 477c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com } else { 478c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com Page* page = reinterpret_cast<Page*>(chunk); 47970d29e1ad7a6f1c163f625252ca32ecc522cb155machenbach@chromium.org PagedSpace* owner = reinterpret_cast<PagedSpace*>(page->owner()); 48070d29e1ad7a6f1c163f625252ca32ecc522cb155machenbach@chromium.org if (owner == heap_->map_space()) { 481a8702c210b949f35c64d8e4aa01bb6d525086c85machenbach@chromium.org DCHECK(page->WasSwept()); 48270d29e1ad7a6f1c163f625252ca32ecc522cb155machenbach@chromium.org HeapObjectIterator iterator(page, NULL); 48370d29e1ad7a6f1c163f625252ca32ecc522cb155machenbach@chromium.org for (HeapObject* heap_object = iterator.Next(); heap_object != NULL; 48470d29e1ad7a6f1c163f625252ca32ecc522cb155machenbach@chromium.org heap_object = iterator.Next()) { 48570d29e1ad7a6f1c163f625252ca32ecc522cb155machenbach@chromium.org // We skip free space objects. 48670d29e1ad7a6f1c163f625252ca32ecc522cb155machenbach@chromium.org if (!heap_object->IsFiller()) { 4875e57059e20217fd540b60c237d326414afe2171emachenbach@chromium.org DCHECK(heap_object->IsMap()); 48870d29e1ad7a6f1c163f625252ca32ecc522cb155machenbach@chromium.org FindPointersToNewSpaceInRegion( 4895e57059e20217fd540b60c237d326414afe2171emachenbach@chromium.org heap_object->address() + Map::kPointerFieldsBeginOffset, 4905e57059e20217fd540b60c237d326414afe2171emachenbach@chromium.org heap_object->address() + Map::kPointerFieldsEndOffset, 4915e57059e20217fd540b60c237d326414afe2171emachenbach@chromium.org slot_callback, clear_maps); 49270d29e1ad7a6f1c163f625252ca32ecc522cb155machenbach@chromium.org } 493d0bddc653152f270a27fe32d5d7b0f5c0fa3b00cmachenbach@chromium.org } 49470d29e1ad7a6f1c163f625252ca32ecc522cb155machenbach@chromium.org } else { 4952b995c4171e67960088466af11110c6f6aeea4fcmachenbach@chromium.org if (!page->SweepingCompleted()) { 4967c0666e15d1f68f0fef5db9e5c409fab010ed816machenbach@chromium.org heap_->mark_compact_collector()->SweepInParallel(page, owner); 4972b995c4171e67960088466af11110c6f6aeea4fcmachenbach@chromium.org if (!page->SweepingCompleted()) { 4987c0666e15d1f68f0fef5db9e5c409fab010ed816machenbach@chromium.org // We were not able to sweep that page, i.e., a concurrent 4997c0666e15d1f68f0fef5db9e5c409fab010ed816machenbach@chromium.org // sweeper thread currently owns this page. 5007c0666e15d1f68f0fef5db9e5c409fab010ed816machenbach@chromium.org // TODO(hpayer): This may introduce a huge pause here. We 5017c0666e15d1f68f0fef5db9e5c409fab010ed816machenbach@chromium.org // just care about finish sweeping of the scan on scavenge page. 5027c0666e15d1f68f0fef5db9e5c409fab010ed816machenbach@chromium.org heap_->mark_compact_collector()->EnsureSweepingCompleted(); 5037c0666e15d1f68f0fef5db9e5c409fab010ed816machenbach@chromium.org } 5047c0666e15d1f68f0fef5db9e5c409fab010ed816machenbach@chromium.org } 505f2af15a6b44ea6276bdd609ee122babe52842a42machenbach@chromium.org CHECK(page->owner() == heap_->old_pointer_space()); 506a8702c210b949f35c64d8e4aa01bb6d525086c85machenbach@chromium.org HeapObjectIterator iterator(page, NULL); 507a8702c210b949f35c64d8e4aa01bb6d525086c85machenbach@chromium.org for (HeapObject* heap_object = iterator.Next(); heap_object != NULL; 508a8702c210b949f35c64d8e4aa01bb6d525086c85machenbach@chromium.org heap_object = iterator.Next()) { 509a8702c210b949f35c64d8e4aa01bb6d525086c85machenbach@chromium.org // We iterate over objects that contain new space pointers only. 5102c81ceb7f1e1ccf5f304be0646f4c1375941a7f2machenbach@chromium.org if (!heap_object->MayContainRawValues()) { 511a8702c210b949f35c64d8e4aa01bb6d525086c85machenbach@chromium.org FindPointersToNewSpaceInRegion( 512a8702c210b949f35c64d8e4aa01bb6d525086c85machenbach@chromium.org heap_object->address() + HeapObject::kHeaderSize, 513a8702c210b949f35c64d8e4aa01bb6d525086c85machenbach@chromium.org heap_object->address() + heap_object->Size(), slot_callback, 514a8702c210b949f35c64d8e4aa01bb6d525086c85machenbach@chromium.org clear_maps); 515f2af15a6b44ea6276bdd609ee122babe52842a42machenbach@chromium.org } 516f2af15a6b44ea6276bdd609ee122babe52842a42machenbach@chromium.org } 517aa107b240dc43417fae8469b5c1b0f1ec9f98400machenbach@chromium.org } 518c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com } 519c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com } 520c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com } 521c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com if (callback_ != NULL) { 522c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com (*callback_)(heap_, NULL, kStoreBufferScanningPageEvent); 523c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com } 524c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com } 525c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com} 526c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com 527c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com 528c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.comvoid StoreBuffer::Compact() { 529c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com Address* top = reinterpret_cast<Address*>(heap_->store_buffer_top()); 530c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com 531c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com if (top == start_) return; 532c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com 533c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com // There's no check of the limit in the loop below so we check here for 534c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com // the worst case (compaction doesn't eliminate any pointers). 535e3c177a423baa3c30225c4e422b6f6c76d38b951machenbach@chromium.org DCHECK(top <= limit_); 536c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com heap_->public_set_store_buffer_top(start_); 53764e3a4be4a99f31920128de34573c8ac9038de42ricow@chromium.org EnsureSpace(top - start_); 538e3c177a423baa3c30225c4e422b6f6c76d38b951machenbach@chromium.org DCHECK(may_move_store_buffer_entries_); 539c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com // Goes through the addresses in the store buffer attempting to remove 540c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com // duplicates. In the interest of speed this is a lossy operation. Some 541b295776b8c02617720d03b6dbcf2f912dfd2c4e7vegorov@chromium.org // duplicates will remain. We have two hash sets with different hash 542c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com // functions to reduce the number of unnecessary clashes. 543b295776b8c02617720d03b6dbcf2f912dfd2c4e7vegorov@chromium.org hash_sets_are_empty_ = false; // Hash sets are in use. 544c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com for (Address* current = start_; current < top; current++) { 545e3c177a423baa3c30225c4e422b6f6c76d38b951machenbach@chromium.org DCHECK(!heap_->cell_space()->Contains(*current)); 546e3c177a423baa3c30225c4e422b6f6c76d38b951machenbach@chromium.org DCHECK(!heap_->code_space()->Contains(*current)); 547e3c177a423baa3c30225c4e422b6f6c76d38b951machenbach@chromium.org DCHECK(!heap_->old_data_space()->Contains(*current)); 548c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com uintptr_t int_addr = reinterpret_cast<uintptr_t>(*current); 549c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com // Shift out the last bits including any tags. 550c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com int_addr >>= kPointerSizeLog2; 551a6bbcc801f63c451f814d6da77a1a48fba3d36c6yangguo@chromium.org // The upper part of an address is basically random because of ASLR and OS 552a6bbcc801f63c451f814d6da77a1a48fba3d36c6yangguo@chromium.org // non-determinism, so we use only the bits within a page for hashing to 553a6bbcc801f63c451f814d6da77a1a48fba3d36c6yangguo@chromium.org // make v8's behavior (more) deterministic. 554a6bbcc801f63c451f814d6da77a1a48fba3d36c6yangguo@chromium.org uintptr_t hash_addr = 555a6bbcc801f63c451f814d6da77a1a48fba3d36c6yangguo@chromium.org int_addr & (Page::kPageAlignmentMask >> kPointerSizeLog2); 556a6bbcc801f63c451f814d6da77a1a48fba3d36c6yangguo@chromium.org int hash1 = ((hash_addr ^ (hash_addr >> kHashSetLengthLog2)) & 557a6bbcc801f63c451f814d6da77a1a48fba3d36c6yangguo@chromium.org (kHashSetLength - 1)); 558b295776b8c02617720d03b6dbcf2f912dfd2c4e7vegorov@chromium.org if (hash_set_1_[hash1] == int_addr) continue; 559a6bbcc801f63c451f814d6da77a1a48fba3d36c6yangguo@chromium.org uintptr_t hash2 = (hash_addr - (hash_addr >> kHashSetLengthLog2)); 560b295776b8c02617720d03b6dbcf2f912dfd2c4e7vegorov@chromium.org hash2 ^= hash2 >> (kHashSetLengthLog2 * 2); 561fab1498f2f42726c7de46c3ed560d56dd072a8b3rossberg@chromium.org hash2 &= (kHashSetLength - 1); 562b295776b8c02617720d03b6dbcf2f912dfd2c4e7vegorov@chromium.org if (hash_set_2_[hash2] == int_addr) continue; 563b295776b8c02617720d03b6dbcf2f912dfd2c4e7vegorov@chromium.org if (hash_set_1_[hash1] == 0) { 564b295776b8c02617720d03b6dbcf2f912dfd2c4e7vegorov@chromium.org hash_set_1_[hash1] = int_addr; 565b295776b8c02617720d03b6dbcf2f912dfd2c4e7vegorov@chromium.org } else if (hash_set_2_[hash2] == 0) { 566b295776b8c02617720d03b6dbcf2f912dfd2c4e7vegorov@chromium.org hash_set_2_[hash2] = int_addr; 567c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com } else { 568c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com // Rather than slowing down we just throw away some entries. This will 569c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com // cause some duplicates to remain undetected. 570b295776b8c02617720d03b6dbcf2f912dfd2c4e7vegorov@chromium.org hash_set_1_[hash1] = int_addr; 571b295776b8c02617720d03b6dbcf2f912dfd2c4e7vegorov@chromium.org hash_set_2_[hash2] = 0; 572c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com } 573c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com old_buffer_is_sorted_ = false; 574c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com old_buffer_is_filtered_ = false; 575c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com *old_top_++ = reinterpret_cast<Address>(int_addr << kPointerSizeLog2); 576e3c177a423baa3c30225c4e422b6f6c76d38b951machenbach@chromium.org DCHECK(old_top_ <= old_limit_); 577c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com } 578c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com heap_->isolate()->counters()->store_buffer_compactions()->Increment(); 579c3b670ff19220959730d7886892bc4beb95d2ebaerik.corry@gmail.com} 58031c0e32e19ad3df48525fa9e7b2d1c0c07496d00machenbach@chromium.org} 58131c0e32e19ad3df48525fa9e7b2d1c0c07496d00machenbach@chromium.org} // namespace v8::internal 582