1cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi/* 2cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi * Copyright (C) 2013 The Android Open Source Project 3cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi * 4cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi * Licensed under the Apache License, Version 2.0 (the "License"); 5cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi * you may not use this file except in compliance with the License. 6cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi * You may obtain a copy of the License at 7cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi * 8cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi * http://www.apache.org/licenses/LICENSE-2.0 9cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi * 10cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi * Unless required by applicable law or agreed to in writing, software 11cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi * distributed under the License is distributed on an "AS IS" BASIS, 12cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi * See the License for the specific language governing permissions and 14cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi * limitations under the License. 15cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi */ 16cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi 17d7576328811e5103e99d31f834a857522cc1463fAndreas Gampe#include "rosalloc.h" 18d7576328811e5103e99d31f834a857522cc1463fAndreas Gampe 191e13374baf7dfaf442ffbf9809c37c131d681eafEvgenii Stepanov#include "base/memory_tool.h" 20cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi#include "base/mutex-inl.h" 211e13374baf7dfaf442ffbf9809c37c131d681eafEvgenii Stepanov#include "gc/space/memory_tool_settings.h" 223481ba2c4e4f3aa80d8c6d50a9f85dacb56b508bVladimir Marko#include "mem_map.h" 23a4adbfd44032d70e166e6f18096bbbed05a990baHiroshi Yamauchi#include "mirror/class-inl.h" 24a4adbfd44032d70e166e6f18096bbbed05a990baHiroshi Yamauchi#include "mirror/object.h" 25a4adbfd44032d70e166e6f18096bbbed05a990baHiroshi Yamauchi#include "mirror/object-inl.h" 26218daa2d876c5989f956e8e54b8f28f33d11b31fBrian Carlstrom#include "thread-inl.h" 27cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi#include "thread_list.h" 28cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi 29cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi#include <map> 30cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi#include <list> 31c7dd295a4e0cc1d15c0c96088e55a85389bade74Ian Rogers#include <sstream> 32cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi#include <vector> 33cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi 34cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchinamespace art { 35cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchinamespace gc { 36cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchinamespace allocator { 37cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi 3831bf42c48c4d00f0677c31264bba8d21618dae67Hiroshi Yamauchistatic constexpr bool kUsePrefetchDuringAllocRun = false; 3973d1e17b3afc7d5e56184f90bf819dc64956448aMathieu Chartierstatic constexpr bool kPrefetchNewRunDataByZeroing = false; 4073d1e17b3afc7d5e56184f90bf819dc64956448aMathieu Chartierstatic constexpr size_t kPrefetchStride = 64; 4173d1e17b3afc7d5e56184f90bf819dc64956448aMathieu Chartier 42cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchisize_t RosAlloc::bracketSizes[kNumOfSizeBrackets]; 43cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchisize_t RosAlloc::numOfPages[kNumOfSizeBrackets]; 44cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchisize_t RosAlloc::numOfSlots[kNumOfSizeBrackets]; 45cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchisize_t RosAlloc::headerSizes[kNumOfSizeBrackets]; 46cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchibool RosAlloc::initialized_ = false; 4773d1e17b3afc7d5e56184f90bf819dc64956448aMathieu Chartiersize_t RosAlloc::dedicated_full_run_storage_[kPageSize / sizeof(size_t)] = { 0 }; 4873d1e17b3afc7d5e56184f90bf819dc64956448aMathieu ChartierRosAlloc::Run* RosAlloc::dedicated_full_run_ = 4973d1e17b3afc7d5e56184f90bf819dc64956448aMathieu Chartier reinterpret_cast<RosAlloc::Run*>(dedicated_full_run_storage_); 50cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi 5126d69ffc0ebc98fbc5f316d8cd3ee6ba5b2001acHiroshi YamauchiRosAlloc::RosAlloc(void* base, size_t capacity, size_t max_capacity, 521e13374baf7dfaf442ffbf9809c37c131d681eafEvgenii Stepanov PageReleaseMode page_release_mode, bool running_on_memory_tool, 53d7576328811e5103e99d31f834a857522cc1463fAndreas Gampe size_t page_release_size_threshold) 5413735955f39b3b304c37d2b2840663c131262c18Ian Rogers : base_(reinterpret_cast<uint8_t*>(base)), footprint_(capacity), 5526d69ffc0ebc98fbc5f316d8cd3ee6ba5b2001acHiroshi Yamauchi capacity_(capacity), max_capacity_(max_capacity), 56cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi lock_("rosalloc global lock", kRosAllocGlobalLock), 57573f7d2d68e1838a0485e6b40d90c967526e00c2Hiroshi Yamauchi bulk_free_lock_("rosalloc bulk free lock", kRosAllocBulkFreeLock), 58573f7d2d68e1838a0485e6b40d90c967526e00c2Hiroshi Yamauchi page_release_mode_(page_release_mode), 59d7576328811e5103e99d31f834a857522cc1463fAndreas Gampe page_release_size_threshold_(page_release_size_threshold), 601e13374baf7dfaf442ffbf9809c37c131d681eafEvgenii Stepanov is_running_on_memory_tool_(running_on_memory_tool) { 61b5e31f3dd5f792ff60225a4daa048a57d261cdd0Hiroshi Yamauchi DCHECK_ALIGNED(base, kPageSize); 625d057056db1923947ba846b391d981759b15714aIan Rogers DCHECK_EQ(RoundUp(capacity, kPageSize), capacity); 635d057056db1923947ba846b391d981759b15714aIan Rogers DCHECK_EQ(RoundUp(max_capacity, kPageSize), max_capacity); 6426d69ffc0ebc98fbc5f316d8cd3ee6ba5b2001acHiroshi Yamauchi CHECK_LE(capacity, max_capacity); 6514d90579f013b374638b599361970557ed4b3f09Roland Levillain CHECK_ALIGNED(page_release_size_threshold_, kPageSize); 66b5e31f3dd5f792ff60225a4daa048a57d261cdd0Hiroshi Yamauchi // Zero the memory explicitly (don't rely on that the mem map is zero-initialized). 67b5e31f3dd5f792ff60225a4daa048a57d261cdd0Hiroshi Yamauchi if (!kMadviseZeroes) { 68b5e31f3dd5f792ff60225a4daa048a57d261cdd0Hiroshi Yamauchi memset(base_, 0, max_capacity); 69b5e31f3dd5f792ff60225a4daa048a57d261cdd0Hiroshi Yamauchi } 70b5e31f3dd5f792ff60225a4daa048a57d261cdd0Hiroshi Yamauchi CHECK_EQ(madvise(base_, max_capacity, MADV_DONTNEED), 0); 71cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi if (!initialized_) { 72cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi Initialize(); 73cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi } 74cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi VLOG(heap) << "RosAlloc base=" 75cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi << std::hex << (intptr_t)base_ << ", end=" 76cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi << std::hex << (intptr_t)(base_ + capacity_) 7726d69ffc0ebc98fbc5f316d8cd3ee6ba5b2001acHiroshi Yamauchi << ", capacity=" << std::dec << capacity_ 7826d69ffc0ebc98fbc5f316d8cd3ee6ba5b2001acHiroshi Yamauchi << ", max_capacity=" << std::dec << max_capacity_; 79cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi for (size_t i = 0; i < kNumOfSizeBrackets; i++) { 80f37a88b8e6db6c587fa449a12e40cb46be1689fcZuo Wang size_bracket_lock_names_[i] = 8173d1e17b3afc7d5e56184f90bf819dc64956448aMathieu Chartier StringPrintf("an rosalloc size bracket %d lock", static_cast<int>(i)); 82f37a88b8e6db6c587fa449a12e40cb46be1689fcZuo Wang size_bracket_locks_[i] = new Mutex(size_bracket_lock_names_[i].c_str(), kRosAllocBracketLock); 830651d41e41341fb2e9ef3ee41dc1f1bfc832dbbbMathieu Chartier current_runs_[i] = dedicated_full_run_; 84cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi } 8526d69ffc0ebc98fbc5f316d8cd3ee6ba5b2001acHiroshi Yamauchi DCHECK_EQ(footprint_, capacity_); 8626d69ffc0ebc98fbc5f316d8cd3ee6ba5b2001acHiroshi Yamauchi size_t num_of_pages = footprint_ / kPageSize; 8726d69ffc0ebc98fbc5f316d8cd3ee6ba5b2001acHiroshi Yamauchi size_t max_num_of_pages = max_capacity_ / kPageSize; 8826d69ffc0ebc98fbc5f316d8cd3ee6ba5b2001acHiroshi Yamauchi std::string error_msg; 895c42c29b89286e5efa4a4613132b09051ce5945bVladimir Marko page_map_mem_map_.reset(MemMap::MapAnonymous("rosalloc page map", nullptr, 905c42c29b89286e5efa4a4613132b09051ce5945bVladimir Marko RoundUp(max_num_of_pages, kPageSize), 915c42c29b89286e5efa4a4613132b09051ce5945bVladimir Marko PROT_READ | PROT_WRITE, false, false, &error_msg)); 9273d1e17b3afc7d5e56184f90bf819dc64956448aMathieu Chartier CHECK(page_map_mem_map_.get() != nullptr) << "Couldn't allocate the page map : " << error_msg; 9326d69ffc0ebc98fbc5f316d8cd3ee6ba5b2001acHiroshi Yamauchi page_map_ = page_map_mem_map_->Begin(); 9426d69ffc0ebc98fbc5f316d8cd3ee6ba5b2001acHiroshi Yamauchi page_map_size_ = num_of_pages; 9526d69ffc0ebc98fbc5f316d8cd3ee6ba5b2001acHiroshi Yamauchi max_page_map_size_ = max_num_of_pages; 96cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi free_page_run_size_map_.resize(num_of_pages); 97cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi FreePageRun* free_pages = reinterpret_cast<FreePageRun*>(base_); 98cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi if (kIsDebugBuild) { 99cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi free_pages->magic_num_ = kMagicNumFree; 100cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi } 101cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi free_pages->SetByteSize(this, capacity_); 102cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi DCHECK_EQ(capacity_ % kPageSize, static_cast<size_t>(0)); 103573f7d2d68e1838a0485e6b40d90c967526e00c2Hiroshi Yamauchi DCHECK(free_pages->IsFree()); 104cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi free_pages->ReleasePages(this); 105573f7d2d68e1838a0485e6b40d90c967526e00c2Hiroshi Yamauchi DCHECK(free_pages->IsFree()); 106cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi free_page_runs_.insert(free_pages); 107cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi if (kTraceRosAlloc) { 108cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi LOG(INFO) << "RosAlloc::RosAlloc() : Inserted run 0x" << std::hex 109cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi << reinterpret_cast<intptr_t>(free_pages) 110cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi << " into free_page_runs_"; 111cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi } 112cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi} 113cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi 114661974a5561e5ccdfbac8cb5d8df8b7e6f3483b8Mathieu ChartierRosAlloc::~RosAlloc() { 115661974a5561e5ccdfbac8cb5d8df8b7e6f3483b8Mathieu Chartier for (size_t i = 0; i < kNumOfSizeBrackets; i++) { 116661974a5561e5ccdfbac8cb5d8df8b7e6f3483b8Mathieu Chartier delete size_bracket_locks_[i]; 117661974a5561e5ccdfbac8cb5d8df8b7e6f3483b8Mathieu Chartier } 1181e13374baf7dfaf442ffbf9809c37c131d681eafEvgenii Stepanov if (is_running_on_memory_tool_) { 1191e13374baf7dfaf442ffbf9809c37c131d681eafEvgenii Stepanov MEMORY_TOOL_MAKE_DEFINED(base_, capacity_); 1201e13374baf7dfaf442ffbf9809c37c131d681eafEvgenii Stepanov } 121661974a5561e5ccdfbac8cb5d8df8b7e6f3483b8Mathieu Chartier} 122661974a5561e5ccdfbac8cb5d8df8b7e6f3483b8Mathieu Chartier 12313735955f39b3b304c37d2b2840663c131262c18Ian Rogersvoid* RosAlloc::AllocPages(Thread* self, size_t num_pages, uint8_t page_map_type) { 124cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi lock_.AssertHeld(self); 125cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi DCHECK(page_map_type == kPageMapRun || page_map_type == kPageMapLargeObject); 1262cebb24bfc3247d3e9be138a3350106737455918Mathieu Chartier FreePageRun* res = nullptr; 12773d1e17b3afc7d5e56184f90bf819dc64956448aMathieu Chartier const size_t req_byte_size = num_pages * kPageSize; 128cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi // Find the lowest address free page run that's large enough. 129cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi for (auto it = free_page_runs_.begin(); it != free_page_runs_.end(); ) { 130cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi FreePageRun* fpr = *it; 131cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi DCHECK(fpr->IsFree()); 132cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi size_t fpr_byte_size = fpr->ByteSize(this); 133cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi DCHECK_EQ(fpr_byte_size % kPageSize, static_cast<size_t>(0)); 134cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi if (req_byte_size <= fpr_byte_size) { 135cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi // Found one. 136cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi free_page_runs_.erase(it++); 137cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi if (kTraceRosAlloc) { 138cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi LOG(INFO) << "RosAlloc::AllocPages() : Erased run 0x" 139cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi << std::hex << reinterpret_cast<intptr_t>(fpr) 140cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi << " from free_page_runs_"; 141cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi } 142cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi if (req_byte_size < fpr_byte_size) { 143cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi // Split. 14413735955f39b3b304c37d2b2840663c131262c18Ian Rogers FreePageRun* remainder = reinterpret_cast<FreePageRun*>(reinterpret_cast<uint8_t*>(fpr) + req_byte_size); 145cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi if (kIsDebugBuild) { 146cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi remainder->magic_num_ = kMagicNumFree; 147cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi } 148cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi remainder->SetByteSize(this, fpr_byte_size - req_byte_size); 149cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi DCHECK_EQ(remainder->ByteSize(this) % kPageSize, static_cast<size_t>(0)); 150cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi // Don't need to call madvise on remainder here. 151cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi free_page_runs_.insert(remainder); 152cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi if (kTraceRosAlloc) { 153cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi LOG(INFO) << "RosAlloc::AllocPages() : Inserted run 0x" << std::hex 154cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi << reinterpret_cast<intptr_t>(remainder) 155cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi << " into free_page_runs_"; 156cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi } 157cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi fpr->SetByteSize(this, req_byte_size); 158cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi DCHECK_EQ(fpr->ByteSize(this) % kPageSize, static_cast<size_t>(0)); 159cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi } 160cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi res = fpr; 161cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi break; 162cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi } else { 163cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi ++it; 164cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi } 165cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi } 166cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi 167cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi // Failed to allocate pages. Grow the footprint, if possible. 1682cebb24bfc3247d3e9be138a3350106737455918Mathieu Chartier if (UNLIKELY(res == nullptr && capacity_ > footprint_)) { 1692cebb24bfc3247d3e9be138a3350106737455918Mathieu Chartier FreePageRun* last_free_page_run = nullptr; 170cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi size_t last_free_page_run_size; 171cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi auto it = free_page_runs_.rbegin(); 172cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi if (it != free_page_runs_.rend() && (last_free_page_run = *it)->End(this) == base_ + footprint_) { 173cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi // There is a free page run at the end. 174cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi DCHECK(last_free_page_run->IsFree()); 175a5b5c55c8585b7ce915f0c7e1f66d121a7f7a078Mathieu Chartier DCHECK(IsFreePage(ToPageMapIndex(last_free_page_run))); 176cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi last_free_page_run_size = last_free_page_run->ByteSize(this); 177cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi } else { 178cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi // There is no free page run at the end. 179cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi last_free_page_run_size = 0; 180cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi } 181cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi DCHECK_LT(last_free_page_run_size, req_byte_size); 182cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi if (capacity_ - footprint_ + last_free_page_run_size >= req_byte_size) { 183cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi // If we grow the heap, we can allocate it. 184cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi size_t increment = std::min(std::max(2 * MB, req_byte_size - last_free_page_run_size), 185cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi capacity_ - footprint_); 186cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi DCHECK_EQ(increment % kPageSize, static_cast<size_t>(0)); 187cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi size_t new_footprint = footprint_ + increment; 188cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi size_t new_num_of_pages = new_footprint / kPageSize; 18926d69ffc0ebc98fbc5f316d8cd3ee6ba5b2001acHiroshi Yamauchi DCHECK_LT(page_map_size_, new_num_of_pages); 190cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi DCHECK_LT(free_page_run_size_map_.size(), new_num_of_pages); 19126d69ffc0ebc98fbc5f316d8cd3ee6ba5b2001acHiroshi Yamauchi page_map_size_ = new_num_of_pages; 19226d69ffc0ebc98fbc5f316d8cd3ee6ba5b2001acHiroshi Yamauchi DCHECK_LE(page_map_size_, max_page_map_size_); 193cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi free_page_run_size_map_.resize(new_num_of_pages); 194277ccbd200ea43590dfc06a93ae184a765327ad0Andreas Gampe ArtRosAllocMoreCore(this, increment); 195cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi if (last_free_page_run_size > 0) { 196cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi // There was a free page run at the end. Expand its size. 197cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi DCHECK_EQ(last_free_page_run_size, last_free_page_run->ByteSize(this)); 198cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi last_free_page_run->SetByteSize(this, last_free_page_run_size + increment); 199cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi DCHECK_EQ(last_free_page_run->ByteSize(this) % kPageSize, static_cast<size_t>(0)); 2005d057056db1923947ba846b391d981759b15714aIan Rogers DCHECK_EQ(last_free_page_run->End(this), base_ + new_footprint); 201cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi } else { 202cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi // Otherwise, insert a new free page run at the end. 203cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi FreePageRun* new_free_page_run = reinterpret_cast<FreePageRun*>(base_ + footprint_); 204cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi if (kIsDebugBuild) { 205cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi new_free_page_run->magic_num_ = kMagicNumFree; 206cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi } 207cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi new_free_page_run->SetByteSize(this, increment); 208cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi DCHECK_EQ(new_free_page_run->ByteSize(this) % kPageSize, static_cast<size_t>(0)); 209cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi free_page_runs_.insert(new_free_page_run); 2105d057056db1923947ba846b391d981759b15714aIan Rogers DCHECK_EQ(*free_page_runs_.rbegin(), new_free_page_run); 211cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi if (kTraceRosAlloc) { 212cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi LOG(INFO) << "RosAlloc::AlloPages() : Grew the heap by inserting run 0x" 213cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi << std::hex << reinterpret_cast<intptr_t>(new_free_page_run) 214cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi << " into free_page_runs_"; 215cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi } 216cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi } 217cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi DCHECK_LE(footprint_ + increment, capacity_); 218cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi if (kTraceRosAlloc) { 219cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi LOG(INFO) << "RosAlloc::AllocPages() : increased the footprint from " 220cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi << footprint_ << " to " << new_footprint; 221cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi } 222cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi footprint_ = new_footprint; 223cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi 224cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi // And retry the last free page run. 225cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi it = free_page_runs_.rbegin(); 226cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi DCHECK(it != free_page_runs_.rend()); 227cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi FreePageRun* fpr = *it; 228cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi if (kIsDebugBuild && last_free_page_run_size > 0) { 2292cebb24bfc3247d3e9be138a3350106737455918Mathieu Chartier DCHECK(last_free_page_run != nullptr); 230cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi DCHECK_EQ(last_free_page_run, fpr); 231cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi } 232cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi size_t fpr_byte_size = fpr->ByteSize(this); 233cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi DCHECK_EQ(fpr_byte_size % kPageSize, static_cast<size_t>(0)); 234cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi DCHECK_LE(req_byte_size, fpr_byte_size); 235cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi free_page_runs_.erase(fpr); 236cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi if (kTraceRosAlloc) { 237cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi LOG(INFO) << "RosAlloc::AllocPages() : Erased run 0x" << std::hex << reinterpret_cast<intptr_t>(fpr) 238cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi << " from free_page_runs_"; 239cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi } 240cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi if (req_byte_size < fpr_byte_size) { 241cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi // Split if there's a remainder. 24213735955f39b3b304c37d2b2840663c131262c18Ian Rogers FreePageRun* remainder = reinterpret_cast<FreePageRun*>(reinterpret_cast<uint8_t*>(fpr) + req_byte_size); 243cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi if (kIsDebugBuild) { 244cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi remainder->magic_num_ = kMagicNumFree; 245cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi } 246cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi remainder->SetByteSize(this, fpr_byte_size - req_byte_size); 247cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi DCHECK_EQ(remainder->ByteSize(this) % kPageSize, static_cast<size_t>(0)); 248cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi free_page_runs_.insert(remainder); 249cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi if (kTraceRosAlloc) { 250cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi LOG(INFO) << "RosAlloc::AllocPages() : Inserted run 0x" << std::hex 251cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi << reinterpret_cast<intptr_t>(remainder) 252cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi << " into free_page_runs_"; 253cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi } 254cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi fpr->SetByteSize(this, req_byte_size); 255cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi DCHECK_EQ(fpr->ByteSize(this) % kPageSize, static_cast<size_t>(0)); 256cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi } 257cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi res = fpr; 258cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi } 259cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi } 2602cebb24bfc3247d3e9be138a3350106737455918Mathieu Chartier if (LIKELY(res != nullptr)) { 261cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi // Update the page map. 262cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi size_t page_map_idx = ToPageMapIndex(res); 263cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi for (size_t i = 0; i < num_pages; i++) { 264a5b5c55c8585b7ce915f0c7e1f66d121a7f7a078Mathieu Chartier DCHECK(IsFreePage(page_map_idx + i)); 265cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi } 266cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi switch (page_map_type) { 267cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi case kPageMapRun: 268cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi page_map_[page_map_idx] = kPageMapRun; 269cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi for (size_t i = 1; i < num_pages; i++) { 270cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi page_map_[page_map_idx + i] = kPageMapRunPart; 271cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi } 272cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi break; 273cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi case kPageMapLargeObject: 274cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi page_map_[page_map_idx] = kPageMapLargeObject; 275cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi for (size_t i = 1; i < num_pages; i++) { 276cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi page_map_[page_map_idx + i] = kPageMapLargeObjectPart; 277cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi } 278cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi break; 279cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi default: 2802fdeecb890a353d3f17407cc1cb015e0a65c2220Maxim Kazantsev LOG(FATAL) << "Unreachable - page map type: " << static_cast<int>(page_map_type); 281cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi break; 282cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi } 283cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi if (kIsDebugBuild) { 28473d1e17b3afc7d5e56184f90bf819dc64956448aMathieu Chartier // Clear the first page since it is not madvised due to the magic number. 285cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi memset(res, 0, kPageSize); 286cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi } 287cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi if (kTraceRosAlloc) { 288cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi LOG(INFO) << "RosAlloc::AllocPages() : 0x" << std::hex << reinterpret_cast<intptr_t>(res) 289cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi << "-0x" << (reinterpret_cast<intptr_t>(res) + num_pages * kPageSize) 290cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi << "(" << std::dec << (num_pages * kPageSize) << ")"; 291cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi } 292cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi return res; 293cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi } 294cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi 295cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi // Fail. 296cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi if (kTraceRosAlloc) { 2972cebb24bfc3247d3e9be138a3350106737455918Mathieu Chartier LOG(INFO) << "RosAlloc::AllocPages() : nullptr"; 298cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi } 299cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi return nullptr; 300cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi} 301cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi 30273d1e17b3afc7d5e56184f90bf819dc64956448aMathieu Chartiersize_t RosAlloc::FreePages(Thread* self, void* ptr, bool already_zero) { 303cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi lock_.AssertHeld(self); 304cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi size_t pm_idx = ToPageMapIndex(ptr); 3055d057056db1923947ba846b391d981759b15714aIan Rogers DCHECK_LT(pm_idx, page_map_size_); 30613735955f39b3b304c37d2b2840663c131262c18Ian Rogers uint8_t pm_type = page_map_[pm_idx]; 307cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi DCHECK(pm_type == kPageMapRun || pm_type == kPageMapLargeObject); 30813735955f39b3b304c37d2b2840663c131262c18Ian Rogers uint8_t pm_part_type; 309cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi switch (pm_type) { 310cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi case kPageMapRun: 311cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi pm_part_type = kPageMapRunPart; 312cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi break; 313cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi case kPageMapLargeObject: 314cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi pm_part_type = kPageMapLargeObjectPart; 315cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi break; 316cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi default: 317a5b5c55c8585b7ce915f0c7e1f66d121a7f7a078Mathieu Chartier LOG(FATAL) << "Unreachable - " << __PRETTY_FUNCTION__ << " : " << "pm_idx=" << pm_idx << ", pm_type=" 318cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi << static_cast<int>(pm_type) << ", ptr=" << std::hex 319cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi << reinterpret_cast<intptr_t>(ptr); 3208585bad7be19ee4901333f7d02d1d4d3f04877d4Mathieu Chartier return 0; 321cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi } 322cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi // Update the page map and count the number of pages. 323cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi size_t num_pages = 1; 324cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi page_map_[pm_idx] = kPageMapEmpty; 325cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi size_t idx = pm_idx + 1; 32626d69ffc0ebc98fbc5f316d8cd3ee6ba5b2001acHiroshi Yamauchi size_t end = page_map_size_; 327cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi while (idx < end && page_map_[idx] == pm_part_type) { 328cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi page_map_[idx] = kPageMapEmpty; 329cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi num_pages++; 330cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi idx++; 331cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi } 33273d1e17b3afc7d5e56184f90bf819dc64956448aMathieu Chartier const size_t byte_size = num_pages * kPageSize; 33373d1e17b3afc7d5e56184f90bf819dc64956448aMathieu Chartier if (already_zero) { 334d7576328811e5103e99d31f834a857522cc1463fAndreas Gampe if (ShouldCheckZeroMemory()) { 33513735955f39b3b304c37d2b2840663c131262c18Ian Rogers const uintptr_t* word_ptr = reinterpret_cast<uintptr_t*>(ptr); 33613735955f39b3b304c37d2b2840663c131262c18Ian Rogers for (size_t i = 0; i < byte_size / sizeof(uintptr_t); ++i) { 33773d1e17b3afc7d5e56184f90bf819dc64956448aMathieu Chartier CHECK_EQ(word_ptr[i], 0U) << "words don't match at index " << i; 33873d1e17b3afc7d5e56184f90bf819dc64956448aMathieu Chartier } 33973d1e17b3afc7d5e56184f90bf819dc64956448aMathieu Chartier } 34073d1e17b3afc7d5e56184f90bf819dc64956448aMathieu Chartier } else if (!DoesReleaseAllPages()) { 34173d1e17b3afc7d5e56184f90bf819dc64956448aMathieu Chartier memset(ptr, 0, byte_size); 34273d1e17b3afc7d5e56184f90bf819dc64956448aMathieu Chartier } 343cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi 344cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi if (kTraceRosAlloc) { 345a5b5c55c8585b7ce915f0c7e1f66d121a7f7a078Mathieu Chartier LOG(INFO) << __PRETTY_FUNCTION__ << " : 0x" << std::hex << reinterpret_cast<intptr_t>(ptr) 34673d1e17b3afc7d5e56184f90bf819dc64956448aMathieu Chartier << "-0x" << (reinterpret_cast<intptr_t>(ptr) + byte_size) 347cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi << "(" << std::dec << (num_pages * kPageSize) << ")"; 348cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi } 349cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi 350cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi // Turn it into a free run. 351cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi FreePageRun* fpr = reinterpret_cast<FreePageRun*>(ptr); 352cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi if (kIsDebugBuild) { 353cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi fpr->magic_num_ = kMagicNumFree; 354cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi } 35573d1e17b3afc7d5e56184f90bf819dc64956448aMathieu Chartier fpr->SetByteSize(this, byte_size); 35614d90579f013b374638b599361970557ed4b3f09Roland Levillain DCHECK_ALIGNED(fpr->ByteSize(this), kPageSize); 357cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi 358cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi DCHECK(free_page_runs_.find(fpr) == free_page_runs_.end()); 359cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi if (!free_page_runs_.empty()) { 360cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi // Try to coalesce in the higher address direction. 361cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi if (kTraceRosAlloc) { 362a5b5c55c8585b7ce915f0c7e1f66d121a7f7a078Mathieu Chartier LOG(INFO) << __PRETTY_FUNCTION__ << "RosAlloc::FreePages() : trying to coalesce a free page run 0x" 363cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi << std::hex << reinterpret_cast<uintptr_t>(fpr) << " [" << std::dec << pm_idx << "] -0x" 364cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi << std::hex << reinterpret_cast<uintptr_t>(fpr->End(this)) << " [" << std::dec 36526d69ffc0ebc98fbc5f316d8cd3ee6ba5b2001acHiroshi Yamauchi << (fpr->End(this) == End() ? page_map_size_ : ToPageMapIndex(fpr->End(this))) << "]"; 366cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi } 367cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi auto higher_it = free_page_runs_.upper_bound(fpr); 368cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi if (higher_it != free_page_runs_.end()) { 369cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi for (auto it = higher_it; it != free_page_runs_.end(); ) { 370cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi FreePageRun* h = *it; 371cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi DCHECK_EQ(h->ByteSize(this) % kPageSize, static_cast<size_t>(0)); 372cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi if (kTraceRosAlloc) { 373cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi LOG(INFO) << "RosAlloc::FreePages() : trying to coalesce with a higher free page run 0x" 374cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi << std::hex << reinterpret_cast<uintptr_t>(h) << " [" << std::dec << ToPageMapIndex(h) << "] -0x" 375cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi << std::hex << reinterpret_cast<uintptr_t>(h->End(this)) << " [" << std::dec 37626d69ffc0ebc98fbc5f316d8cd3ee6ba5b2001acHiroshi Yamauchi << (h->End(this) == End() ? page_map_size_ : ToPageMapIndex(h->End(this))) << "]"; 377cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi } 378cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi if (fpr->End(this) == h->Begin()) { 379cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi if (kTraceRosAlloc) { 380cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi LOG(INFO) << "Success"; 381cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi } 38273d1e17b3afc7d5e56184f90bf819dc64956448aMathieu Chartier // Clear magic num since this is no longer the start of a free page run. 38373d1e17b3afc7d5e56184f90bf819dc64956448aMathieu Chartier if (kIsDebugBuild) { 38473d1e17b3afc7d5e56184f90bf819dc64956448aMathieu Chartier h->magic_num_ = 0; 38573d1e17b3afc7d5e56184f90bf819dc64956448aMathieu Chartier } 386cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi free_page_runs_.erase(it++); 387cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi if (kTraceRosAlloc) { 388cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi LOG(INFO) << "RosAlloc::FreePages() : (coalesce) Erased run 0x" << std::hex 389cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi << reinterpret_cast<intptr_t>(h) 390cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi << " from free_page_runs_"; 391cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi } 392cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi fpr->SetByteSize(this, fpr->ByteSize(this) + h->ByteSize(this)); 393cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi DCHECK_EQ(fpr->ByteSize(this) % kPageSize, static_cast<size_t>(0)); 394cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi } else { 395cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi // Not adjacent. Stop. 396cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi if (kTraceRosAlloc) { 397cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi LOG(INFO) << "Fail"; 398cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi } 399cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi break; 400cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi } 401cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi } 402cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi } 403cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi // Try to coalesce in the lower address direction. 404cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi auto lower_it = free_page_runs_.upper_bound(fpr); 405cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi if (lower_it != free_page_runs_.begin()) { 406cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi --lower_it; 407cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi for (auto it = lower_it; ; ) { 408cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi // We want to try to coalesce with the first element but 409cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi // there's no "<=" operator for the iterator. 410cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi bool to_exit_loop = it == free_page_runs_.begin(); 411cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi 412cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi FreePageRun* l = *it; 413cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi DCHECK_EQ(l->ByteSize(this) % kPageSize, static_cast<size_t>(0)); 414cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi if (kTraceRosAlloc) { 415cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi LOG(INFO) << "RosAlloc::FreePages() : trying to coalesce with a lower free page run 0x" 416cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi << std::hex << reinterpret_cast<uintptr_t>(l) << " [" << std::dec << ToPageMapIndex(l) << "] -0x" 417cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi << std::hex << reinterpret_cast<uintptr_t>(l->End(this)) << " [" << std::dec 41826d69ffc0ebc98fbc5f316d8cd3ee6ba5b2001acHiroshi Yamauchi << (l->End(this) == End() ? page_map_size_ : ToPageMapIndex(l->End(this))) << "]"; 419cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi } 420cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi if (l->End(this) == fpr->Begin()) { 421cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi if (kTraceRosAlloc) { 422cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi LOG(INFO) << "Success"; 423cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi } 424cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi free_page_runs_.erase(it--); 425cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi if (kTraceRosAlloc) { 426cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi LOG(INFO) << "RosAlloc::FreePages() : (coalesce) Erased run 0x" << std::hex 427cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi << reinterpret_cast<intptr_t>(l) 428cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi << " from free_page_runs_"; 429cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi } 430cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi l->SetByteSize(this, l->ByteSize(this) + fpr->ByteSize(this)); 431cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi DCHECK_EQ(l->ByteSize(this) % kPageSize, static_cast<size_t>(0)); 43273d1e17b3afc7d5e56184f90bf819dc64956448aMathieu Chartier // Clear magic num since this is no longer the start of a free page run. 43373d1e17b3afc7d5e56184f90bf819dc64956448aMathieu Chartier if (kIsDebugBuild) { 43473d1e17b3afc7d5e56184f90bf819dc64956448aMathieu Chartier fpr->magic_num_ = 0; 43573d1e17b3afc7d5e56184f90bf819dc64956448aMathieu Chartier } 436cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi fpr = l; 437cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi } else { 438cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi // Not adjacent. Stop. 439cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi if (kTraceRosAlloc) { 440cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi LOG(INFO) << "Fail"; 441cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi } 442cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi break; 443cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi } 444cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi if (to_exit_loop) { 445cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi break; 446cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi } 447cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi } 448cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi } 449cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi } 450cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi 451cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi // Insert it. 452cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi DCHECK_EQ(fpr->ByteSize(this) % kPageSize, static_cast<size_t>(0)); 453cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi DCHECK(free_page_runs_.find(fpr) == free_page_runs_.end()); 454573f7d2d68e1838a0485e6b40d90c967526e00c2Hiroshi Yamauchi DCHECK(fpr->IsFree()); 455cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi fpr->ReleasePages(this); 456573f7d2d68e1838a0485e6b40d90c967526e00c2Hiroshi Yamauchi DCHECK(fpr->IsFree()); 457cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi free_page_runs_.insert(fpr); 458cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi DCHECK(free_page_runs_.find(fpr) != free_page_runs_.end()); 459cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi if (kTraceRosAlloc) { 460cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi LOG(INFO) << "RosAlloc::FreePages() : Inserted run 0x" << std::hex << reinterpret_cast<intptr_t>(fpr) 461cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi << " into free_page_runs_"; 462cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi } 46373d1e17b3afc7d5e56184f90bf819dc64956448aMathieu Chartier return byte_size; 464cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi} 465cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi 4664460a84be92b5a94ecfb5c650aef4945ab849c93Hiroshi Yamauchivoid* RosAlloc::AllocLargeObject(Thread* self, size_t size, size_t* bytes_allocated, 4674460a84be92b5a94ecfb5c650aef4945ab849c93Hiroshi Yamauchi size_t* usable_size, size_t* bytes_tl_bulk_allocated) { 4684460a84be92b5a94ecfb5c650aef4945ab849c93Hiroshi Yamauchi DCHECK(bytes_allocated != nullptr); 4694460a84be92b5a94ecfb5c650aef4945ab849c93Hiroshi Yamauchi DCHECK(usable_size != nullptr); 4705d057056db1923947ba846b391d981759b15714aIan Rogers DCHECK_GT(size, kLargeSizeThreshold); 4713c2856e939f3daa7a95a1f8cb70f47e7a621db3cHiroshi Yamauchi size_t num_pages = RoundUp(size, kPageSize) / kPageSize; 4723c2856e939f3daa7a95a1f8cb70f47e7a621db3cHiroshi Yamauchi void* r; 4733c2856e939f3daa7a95a1f8cb70f47e7a621db3cHiroshi Yamauchi { 4743c2856e939f3daa7a95a1f8cb70f47e7a621db3cHiroshi Yamauchi MutexLock mu(self, lock_); 4753c2856e939f3daa7a95a1f8cb70f47e7a621db3cHiroshi Yamauchi r = AllocPages(self, num_pages, kPageMapLargeObject); 4763c2856e939f3daa7a95a1f8cb70f47e7a621db3cHiroshi Yamauchi } 477573f7d2d68e1838a0485e6b40d90c967526e00c2Hiroshi Yamauchi if (UNLIKELY(r == nullptr)) { 478573f7d2d68e1838a0485e6b40d90c967526e00c2Hiroshi Yamauchi if (kTraceRosAlloc) { 4792cebb24bfc3247d3e9be138a3350106737455918Mathieu Chartier LOG(INFO) << "RosAlloc::AllocLargeObject() : nullptr"; 480573f7d2d68e1838a0485e6b40d90c967526e00c2Hiroshi Yamauchi } 481573f7d2d68e1838a0485e6b40d90c967526e00c2Hiroshi Yamauchi return nullptr; 482573f7d2d68e1838a0485e6b40d90c967526e00c2Hiroshi Yamauchi } 48373d1e17b3afc7d5e56184f90bf819dc64956448aMathieu Chartier const size_t total_bytes = num_pages * kPageSize; 48473d1e17b3afc7d5e56184f90bf819dc64956448aMathieu Chartier *bytes_allocated = total_bytes; 4854460a84be92b5a94ecfb5c650aef4945ab849c93Hiroshi Yamauchi *usable_size = total_bytes; 4864460a84be92b5a94ecfb5c650aef4945ab849c93Hiroshi Yamauchi *bytes_tl_bulk_allocated = total_bytes; 4873c2856e939f3daa7a95a1f8cb70f47e7a621db3cHiroshi Yamauchi if (kTraceRosAlloc) { 488573f7d2d68e1838a0485e6b40d90c967526e00c2Hiroshi Yamauchi LOG(INFO) << "RosAlloc::AllocLargeObject() : 0x" << std::hex << reinterpret_cast<intptr_t>(r) 489573f7d2d68e1838a0485e6b40d90c967526e00c2Hiroshi Yamauchi << "-0x" << (reinterpret_cast<intptr_t>(r) + num_pages * kPageSize) 490573f7d2d68e1838a0485e6b40d90c967526e00c2Hiroshi Yamauchi << "(" << std::dec << (num_pages * kPageSize) << ")"; 491573f7d2d68e1838a0485e6b40d90c967526e00c2Hiroshi Yamauchi } 492cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi // Check if the returned memory is really all zero. 493d7576328811e5103e99d31f834a857522cc1463fAndreas Gampe if (ShouldCheckZeroMemory()) { 49413735955f39b3b304c37d2b2840663c131262c18Ian Rogers CHECK_EQ(total_bytes % sizeof(uintptr_t), 0U); 49513735955f39b3b304c37d2b2840663c131262c18Ian Rogers const uintptr_t* words = reinterpret_cast<uintptr_t*>(r); 49613735955f39b3b304c37d2b2840663c131262c18Ian Rogers for (size_t i = 0; i < total_bytes / sizeof(uintptr_t); ++i) { 49773d1e17b3afc7d5e56184f90bf819dc64956448aMathieu Chartier CHECK_EQ(words[i], 0U); 498cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi } 499cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi } 5003c2856e939f3daa7a95a1f8cb70f47e7a621db3cHiroshi Yamauchi return r; 501cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi} 502cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi 5038585bad7be19ee4901333f7d02d1d4d3f04877d4Mathieu Chartiersize_t RosAlloc::FreeInternal(Thread* self, void* ptr) { 5045d057056db1923947ba846b391d981759b15714aIan Rogers DCHECK_LE(base_, ptr); 5055d057056db1923947ba846b391d981759b15714aIan Rogers DCHECK_LT(ptr, base_ + footprint_); 506cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi size_t pm_idx = RoundDownToPageMapIndex(ptr); 5078585bad7be19ee4901333f7d02d1d4d3f04877d4Mathieu Chartier Run* run = nullptr; 508cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi { 509cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi MutexLock mu(self, lock_); 5105d057056db1923947ba846b391d981759b15714aIan Rogers DCHECK_LT(pm_idx, page_map_size_); 51113735955f39b3b304c37d2b2840663c131262c18Ian Rogers uint8_t page_map_entry = page_map_[pm_idx]; 512cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi if (kTraceRosAlloc) { 513cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi LOG(INFO) << "RosAlloc::FreeInternal() : " << std::hex << ptr << ", pm_idx=" << std::dec << pm_idx 514cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi << ", page_map_entry=" << static_cast<int>(page_map_entry); 515cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi } 516cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi switch (page_map_[pm_idx]) { 517cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi case kPageMapLargeObject: 51873d1e17b3afc7d5e56184f90bf819dc64956448aMathieu Chartier return FreePages(self, ptr, false); 519cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi case kPageMapLargeObjectPart: 5202fdeecb890a353d3f17407cc1cb015e0a65c2220Maxim Kazantsev LOG(FATAL) << "Unreachable - page map type: " << static_cast<int>(page_map_[pm_idx]); 5218585bad7be19ee4901333f7d02d1d4d3f04877d4Mathieu Chartier return 0; 522cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi case kPageMapRunPart: { 523cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi // Find the beginning of the run. 524a5b5c55c8585b7ce915f0c7e1f66d121a7f7a078Mathieu Chartier do { 525a5b5c55c8585b7ce915f0c7e1f66d121a7f7a078Mathieu Chartier --pm_idx; 526a5b5c55c8585b7ce915f0c7e1f66d121a7f7a078Mathieu Chartier DCHECK_LT(pm_idx, capacity_ / kPageSize); 527a5b5c55c8585b7ce915f0c7e1f66d121a7f7a078Mathieu Chartier } while (page_map_[pm_idx] != kPageMapRun); 528fc787ecd91127b2c8458afd94e5148e2ae51a1f5Ian Rogers FALLTHROUGH_INTENDED; 529a5b5c55c8585b7ce915f0c7e1f66d121a7f7a078Mathieu Chartier case kPageMapRun: 530a5b5c55c8585b7ce915f0c7e1f66d121a7f7a078Mathieu Chartier run = reinterpret_cast<Run*>(base_ + pm_idx * kPageSize); 5315d057056db1923947ba846b391d981759b15714aIan Rogers DCHECK_EQ(run->magic_num_, kMagicNum); 532cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi break; 533a5b5c55c8585b7ce915f0c7e1f66d121a7f7a078Mathieu Chartier case kPageMapReleased: 534a5b5c55c8585b7ce915f0c7e1f66d121a7f7a078Mathieu Chartier case kPageMapEmpty: 5352fdeecb890a353d3f17407cc1cb015e0a65c2220Maxim Kazantsev LOG(FATAL) << "Unreachable - page map type: " << static_cast<int>(page_map_[pm_idx]); 536a5b5c55c8585b7ce915f0c7e1f66d121a7f7a078Mathieu Chartier return 0; 537cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi } 538cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi default: 5392fdeecb890a353d3f17407cc1cb015e0a65c2220Maxim Kazantsev LOG(FATAL) << "Unreachable - page map type: " << static_cast<int>(page_map_[pm_idx]); 5408585bad7be19ee4901333f7d02d1d4d3f04877d4Mathieu Chartier return 0; 541cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi } 542cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi } 5438585bad7be19ee4901333f7d02d1d4d3f04877d4Mathieu Chartier DCHECK(run != nullptr); 54473d1e17b3afc7d5e56184f90bf819dc64956448aMathieu Chartier return FreeFromRun(self, ptr, run); 545cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi} 546cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi 5478585bad7be19ee4901333f7d02d1d4d3f04877d4Mathieu Chartiersize_t RosAlloc::Free(Thread* self, void* ptr) { 548f5997b4d3f889569d5a2b724d83d764bfbb8d106Mathieu Chartier ReaderMutexLock rmu(self, bulk_free_lock_); 5498585bad7be19ee4901333f7d02d1d4d3f04877d4Mathieu Chartier return FreeInternal(self, ptr); 550cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi} 551cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi 55273d1e17b3afc7d5e56184f90bf819dc64956448aMathieu ChartierRosAlloc::Run* RosAlloc::AllocRun(Thread* self, size_t idx) { 55373d1e17b3afc7d5e56184f90bf819dc64956448aMathieu Chartier RosAlloc::Run* new_run = nullptr; 55473d1e17b3afc7d5e56184f90bf819dc64956448aMathieu Chartier { 55573d1e17b3afc7d5e56184f90bf819dc64956448aMathieu Chartier MutexLock mu(self, lock_); 55673d1e17b3afc7d5e56184f90bf819dc64956448aMathieu Chartier new_run = reinterpret_cast<Run*>(AllocPages(self, numOfPages[idx], kPageMapRun)); 55773d1e17b3afc7d5e56184f90bf819dc64956448aMathieu Chartier } 55873d1e17b3afc7d5e56184f90bf819dc64956448aMathieu Chartier if (LIKELY(new_run != nullptr)) { 559cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi if (kIsDebugBuild) { 560cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi new_run->magic_num_ = kMagicNum; 561cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi } 562cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi new_run->size_bracket_idx_ = idx; 56373d1e17b3afc7d5e56184f90bf819dc64956448aMathieu Chartier DCHECK(!new_run->IsThreadLocal()); 56473d1e17b3afc7d5e56184f90bf819dc64956448aMathieu Chartier DCHECK(!new_run->to_be_bulk_freed_); 5650651d41e41341fb2e9ef3ee41dc1f1bfc832dbbbMathieu Chartier if (kUsePrefetchDuringAllocRun && idx < kNumThreadLocalSizeBrackets) { 56673d1e17b3afc7d5e56184f90bf819dc64956448aMathieu Chartier // Take ownership of the cache lines if we are likely to be thread local run. 56773d1e17b3afc7d5e56184f90bf819dc64956448aMathieu Chartier if (kPrefetchNewRunDataByZeroing) { 56873d1e17b3afc7d5e56184f90bf819dc64956448aMathieu Chartier // Zeroing the data is sometimes faster than prefetching but it increases memory usage 56973d1e17b3afc7d5e56184f90bf819dc64956448aMathieu Chartier // since we end up dirtying zero pages which may have been madvised. 57073d1e17b3afc7d5e56184f90bf819dc64956448aMathieu Chartier new_run->ZeroData(); 57173d1e17b3afc7d5e56184f90bf819dc64956448aMathieu Chartier } else { 57273d1e17b3afc7d5e56184f90bf819dc64956448aMathieu Chartier const size_t num_of_slots = numOfSlots[idx]; 57373d1e17b3afc7d5e56184f90bf819dc64956448aMathieu Chartier const size_t bracket_size = bracketSizes[idx]; 57473d1e17b3afc7d5e56184f90bf819dc64956448aMathieu Chartier const size_t num_of_bytes = num_of_slots * bracket_size; 57513735955f39b3b304c37d2b2840663c131262c18Ian Rogers uint8_t* begin = reinterpret_cast<uint8_t*>(new_run) + headerSizes[idx]; 57673d1e17b3afc7d5e56184f90bf819dc64956448aMathieu Chartier for (size_t i = 0; i < num_of_bytes; i += kPrefetchStride) { 57773d1e17b3afc7d5e56184f90bf819dc64956448aMathieu Chartier __builtin_prefetch(begin + i); 57873d1e17b3afc7d5e56184f90bf819dc64956448aMathieu Chartier } 57973d1e17b3afc7d5e56184f90bf819dc64956448aMathieu Chartier } 58073d1e17b3afc7d5e56184f90bf819dc64956448aMathieu Chartier } 58131bf42c48c4d00f0677c31264bba8d21618dae67Hiroshi Yamauchi new_run->InitFreeList(); 582cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi } 583cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi return new_run; 584cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi} 585cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi 58673d1e17b3afc7d5e56184f90bf819dc64956448aMathieu ChartierRosAlloc::Run* RosAlloc::RefillRun(Thread* self, size_t idx) { 58773d1e17b3afc7d5e56184f90bf819dc64956448aMathieu Chartier // Get the lowest address non-full run from the binary tree. 58858553c7fd89ce69857017322444265469bb6af62Mathieu Chartier auto* const bt = &non_full_runs_[idx]; 58973d1e17b3afc7d5e56184f90bf819dc64956448aMathieu Chartier if (!bt->empty()) { 59073d1e17b3afc7d5e56184f90bf819dc64956448aMathieu Chartier // If there's one, use it as the current run. 59173d1e17b3afc7d5e56184f90bf819dc64956448aMathieu Chartier auto it = bt->begin(); 59273d1e17b3afc7d5e56184f90bf819dc64956448aMathieu Chartier Run* non_full_run = *it; 59373d1e17b3afc7d5e56184f90bf819dc64956448aMathieu Chartier DCHECK(non_full_run != nullptr); 59473d1e17b3afc7d5e56184f90bf819dc64956448aMathieu Chartier DCHECK(!non_full_run->IsThreadLocal()); 59573d1e17b3afc7d5e56184f90bf819dc64956448aMathieu Chartier bt->erase(it); 59673d1e17b3afc7d5e56184f90bf819dc64956448aMathieu Chartier return non_full_run; 59773d1e17b3afc7d5e56184f90bf819dc64956448aMathieu Chartier } 59873d1e17b3afc7d5e56184f90bf819dc64956448aMathieu Chartier // If there's none, allocate a new run and use it as the current run. 59973d1e17b3afc7d5e56184f90bf819dc64956448aMathieu Chartier return AllocRun(self, idx); 60073d1e17b3afc7d5e56184f90bf819dc64956448aMathieu Chartier} 60173d1e17b3afc7d5e56184f90bf819dc64956448aMathieu Chartier 60252cf5c09ab706f8bbd2a2cf2fb1ef1041f020314Hiroshi Yamauchiinline void* RosAlloc::AllocFromCurrentRunUnlocked(Thread* self, size_t idx) { 6030651d41e41341fb2e9ef3ee41dc1f1bfc832dbbbMathieu Chartier Run* current_run = current_runs_[idx]; 6040651d41e41341fb2e9ef3ee41dc1f1bfc832dbbbMathieu Chartier DCHECK(current_run != nullptr); 6050651d41e41341fb2e9ef3ee41dc1f1bfc832dbbbMathieu Chartier void* slot_addr = current_run->AllocSlot(); 6060651d41e41341fb2e9ef3ee41dc1f1bfc832dbbbMathieu Chartier if (UNLIKELY(slot_addr == nullptr)) { 6070651d41e41341fb2e9ef3ee41dc1f1bfc832dbbbMathieu Chartier // The current run got full. Try to refill it. 6080651d41e41341fb2e9ef3ee41dc1f1bfc832dbbbMathieu Chartier DCHECK(current_run->IsFull()); 6090651d41e41341fb2e9ef3ee41dc1f1bfc832dbbbMathieu Chartier if (kIsDebugBuild && current_run != dedicated_full_run_) { 6100651d41e41341fb2e9ef3ee41dc1f1bfc832dbbbMathieu Chartier full_runs_[idx].insert(current_run); 6110651d41e41341fb2e9ef3ee41dc1f1bfc832dbbbMathieu Chartier if (kTraceRosAlloc) { 612a5b5c55c8585b7ce915f0c7e1f66d121a7f7a078Mathieu Chartier LOG(INFO) << __PRETTY_FUNCTION__ << " : Inserted run 0x" << std::hex 613a5b5c55c8585b7ce915f0c7e1f66d121a7f7a078Mathieu Chartier << reinterpret_cast<intptr_t>(current_run) 6140651d41e41341fb2e9ef3ee41dc1f1bfc832dbbbMathieu Chartier << " into full_runs_[" << std::dec << idx << "]"; 6150651d41e41341fb2e9ef3ee41dc1f1bfc832dbbbMathieu Chartier } 6160651d41e41341fb2e9ef3ee41dc1f1bfc832dbbbMathieu Chartier DCHECK(non_full_runs_[idx].find(current_run) == non_full_runs_[idx].end()); 6170651d41e41341fb2e9ef3ee41dc1f1bfc832dbbbMathieu Chartier DCHECK(full_runs_[idx].find(current_run) != full_runs_[idx].end()); 6180651d41e41341fb2e9ef3ee41dc1f1bfc832dbbbMathieu Chartier } 6190651d41e41341fb2e9ef3ee41dc1f1bfc832dbbbMathieu Chartier current_run = RefillRun(self, idx); 6200651d41e41341fb2e9ef3ee41dc1f1bfc832dbbbMathieu Chartier if (UNLIKELY(current_run == nullptr)) { 6210651d41e41341fb2e9ef3ee41dc1f1bfc832dbbbMathieu Chartier // Failed to allocate a new run, make sure that it is the dedicated full run. 6220651d41e41341fb2e9ef3ee41dc1f1bfc832dbbbMathieu Chartier current_runs_[idx] = dedicated_full_run_; 6230651d41e41341fb2e9ef3ee41dc1f1bfc832dbbbMathieu Chartier return nullptr; 6240651d41e41341fb2e9ef3ee41dc1f1bfc832dbbbMathieu Chartier } 6250651d41e41341fb2e9ef3ee41dc1f1bfc832dbbbMathieu Chartier DCHECK(current_run != nullptr); 6260651d41e41341fb2e9ef3ee41dc1f1bfc832dbbbMathieu Chartier DCHECK(non_full_runs_[idx].find(current_run) == non_full_runs_[idx].end()); 6270651d41e41341fb2e9ef3ee41dc1f1bfc832dbbbMathieu Chartier DCHECK(full_runs_[idx].find(current_run) == full_runs_[idx].end()); 6280651d41e41341fb2e9ef3ee41dc1f1bfc832dbbbMathieu Chartier current_run->SetIsThreadLocal(false); 6290651d41e41341fb2e9ef3ee41dc1f1bfc832dbbbMathieu Chartier current_runs_[idx] = current_run; 6300651d41e41341fb2e9ef3ee41dc1f1bfc832dbbbMathieu Chartier DCHECK(!current_run->IsFull()); 6310651d41e41341fb2e9ef3ee41dc1f1bfc832dbbbMathieu Chartier slot_addr = current_run->AllocSlot(); 6320651d41e41341fb2e9ef3ee41dc1f1bfc832dbbbMathieu Chartier // Must succeed now with a new run. 6330651d41e41341fb2e9ef3ee41dc1f1bfc832dbbbMathieu Chartier DCHECK(slot_addr != nullptr); 6340651d41e41341fb2e9ef3ee41dc1f1bfc832dbbbMathieu Chartier } 6350651d41e41341fb2e9ef3ee41dc1f1bfc832dbbbMathieu Chartier return slot_addr; 6360651d41e41341fb2e9ef3ee41dc1f1bfc832dbbbMathieu Chartier} 6370651d41e41341fb2e9ef3ee41dc1f1bfc832dbbbMathieu Chartier 6384460a84be92b5a94ecfb5c650aef4945ab849c93Hiroshi Yamauchivoid* RosAlloc::AllocFromRunThreadUnsafe(Thread* self, size_t size, size_t* bytes_allocated, 6394460a84be92b5a94ecfb5c650aef4945ab849c93Hiroshi Yamauchi size_t* usable_size, 6404460a84be92b5a94ecfb5c650aef4945ab849c93Hiroshi Yamauchi size_t* bytes_tl_bulk_allocated) { 6414460a84be92b5a94ecfb5c650aef4945ab849c93Hiroshi Yamauchi DCHECK(bytes_allocated != nullptr); 6424460a84be92b5a94ecfb5c650aef4945ab849c93Hiroshi Yamauchi DCHECK(usable_size != nullptr); 6434460a84be92b5a94ecfb5c650aef4945ab849c93Hiroshi Yamauchi DCHECK(bytes_tl_bulk_allocated != nullptr); 6440651d41e41341fb2e9ef3ee41dc1f1bfc832dbbbMathieu Chartier DCHECK_LE(size, kLargeSizeThreshold); 6450651d41e41341fb2e9ef3ee41dc1f1bfc832dbbbMathieu Chartier size_t bracket_size; 6460651d41e41341fb2e9ef3ee41dc1f1bfc832dbbbMathieu Chartier size_t idx = SizeToIndexAndBracketSize(size, &bracket_size); 6470651d41e41341fb2e9ef3ee41dc1f1bfc832dbbbMathieu Chartier Locks::mutator_lock_->AssertExclusiveHeld(self); 6480651d41e41341fb2e9ef3ee41dc1f1bfc832dbbbMathieu Chartier void* slot_addr = AllocFromCurrentRunUnlocked(self, idx); 6490651d41e41341fb2e9ef3ee41dc1f1bfc832dbbbMathieu Chartier if (LIKELY(slot_addr != nullptr)) { 6500651d41e41341fb2e9ef3ee41dc1f1bfc832dbbbMathieu Chartier *bytes_allocated = bracket_size; 6514460a84be92b5a94ecfb5c650aef4945ab849c93Hiroshi Yamauchi *usable_size = bracket_size; 6524460a84be92b5a94ecfb5c650aef4945ab849c93Hiroshi Yamauchi *bytes_tl_bulk_allocated = bracket_size; 6530651d41e41341fb2e9ef3ee41dc1f1bfc832dbbbMathieu Chartier } 6544460a84be92b5a94ecfb5c650aef4945ab849c93Hiroshi Yamauchi // Caller verifies that it is all 0. 6550651d41e41341fb2e9ef3ee41dc1f1bfc832dbbbMathieu Chartier return slot_addr; 6560651d41e41341fb2e9ef3ee41dc1f1bfc832dbbbMathieu Chartier} 6570651d41e41341fb2e9ef3ee41dc1f1bfc832dbbbMathieu Chartier 6584460a84be92b5a94ecfb5c650aef4945ab849c93Hiroshi Yamauchivoid* RosAlloc::AllocFromRun(Thread* self, size_t size, size_t* bytes_allocated, 6594460a84be92b5a94ecfb5c650aef4945ab849c93Hiroshi Yamauchi size_t* usable_size, size_t* bytes_tl_bulk_allocated) { 6604460a84be92b5a94ecfb5c650aef4945ab849c93Hiroshi Yamauchi DCHECK(bytes_allocated != nullptr); 6614460a84be92b5a94ecfb5c650aef4945ab849c93Hiroshi Yamauchi DCHECK(usable_size != nullptr); 6624460a84be92b5a94ecfb5c650aef4945ab849c93Hiroshi Yamauchi DCHECK(bytes_tl_bulk_allocated != nullptr); 6635d057056db1923947ba846b391d981759b15714aIan Rogers DCHECK_LE(size, kLargeSizeThreshold); 664cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi size_t bracket_size; 665cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi size_t idx = SizeToIndexAndBracketSize(size, &bracket_size); 666cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi void* slot_addr; 6670651d41e41341fb2e9ef3ee41dc1f1bfc832dbbbMathieu Chartier if (LIKELY(idx < kNumThreadLocalSizeBrackets)) { 668cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi // Use a thread-local run. 669dd7624d2b9e599d57762d12031b10b89defc9807Ian Rogers Run* thread_local_run = reinterpret_cast<Run*>(self->GetRosAllocRun(idx)); 67073d1e17b3afc7d5e56184f90bf819dc64956448aMathieu Chartier // Allow invalid since this will always fail the allocation. 6714fd2050ff1da3892b5e79276f831e09b2f8e5bc9Mathieu Chartier if (kIsDebugBuild) { 6724fd2050ff1da3892b5e79276f831e09b2f8e5bc9Mathieu Chartier // Need the lock to prevent race conditions. 6734fd2050ff1da3892b5e79276f831e09b2f8e5bc9Mathieu Chartier MutexLock mu(self, *size_bracket_locks_[idx]); 6744fd2050ff1da3892b5e79276f831e09b2f8e5bc9Mathieu Chartier CHECK(non_full_runs_[idx].find(thread_local_run) == non_full_runs_[idx].end()); 6754fd2050ff1da3892b5e79276f831e09b2f8e5bc9Mathieu Chartier CHECK(full_runs_[idx].find(thread_local_run) == full_runs_[idx].end()); 6764fd2050ff1da3892b5e79276f831e09b2f8e5bc9Mathieu Chartier } 67773d1e17b3afc7d5e56184f90bf819dc64956448aMathieu Chartier DCHECK(thread_local_run != nullptr); 67873d1e17b3afc7d5e56184f90bf819dc64956448aMathieu Chartier DCHECK(thread_local_run->IsThreadLocal() || thread_local_run == dedicated_full_run_); 679cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi slot_addr = thread_local_run->AllocSlot(); 68073d1e17b3afc7d5e56184f90bf819dc64956448aMathieu Chartier // The allocation must fail if the run is invalid. 68173d1e17b3afc7d5e56184f90bf819dc64956448aMathieu Chartier DCHECK(thread_local_run != dedicated_full_run_ || slot_addr == nullptr) 68273d1e17b3afc7d5e56184f90bf819dc64956448aMathieu Chartier << "allocated from an invalid run"; 68373d1e17b3afc7d5e56184f90bf819dc64956448aMathieu Chartier if (UNLIKELY(slot_addr == nullptr)) { 684cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi // The run got full. Try to free slots. 685cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi DCHECK(thread_local_run->IsFull()); 686cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi MutexLock mu(self, *size_bracket_locks_[idx]); 687cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi bool is_all_free_after_merge; 68873d1e17b3afc7d5e56184f90bf819dc64956448aMathieu Chartier // This is safe to do for the dedicated_full_run_ since the bitmaps are empty. 68931bf42c48c4d00f0677c31264bba8d21618dae67Hiroshi Yamauchi if (thread_local_run->MergeThreadLocalFreeListToFreeList(&is_all_free_after_merge)) { 69073d1e17b3afc7d5e56184f90bf819dc64956448aMathieu Chartier DCHECK_NE(thread_local_run, dedicated_full_run_); 691cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi // Some slot got freed. Keep it. 692cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi DCHECK(!thread_local_run->IsFull()); 693cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi DCHECK_EQ(is_all_free_after_merge, thread_local_run->IsAllFree()); 694cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi } else { 695cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi // No slots got freed. Try to refill the thread-local run. 696cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi DCHECK(thread_local_run->IsFull()); 69773d1e17b3afc7d5e56184f90bf819dc64956448aMathieu Chartier if (thread_local_run != dedicated_full_run_) { 69873d1e17b3afc7d5e56184f90bf819dc64956448aMathieu Chartier thread_local_run->SetIsThreadLocal(false); 69973d1e17b3afc7d5e56184f90bf819dc64956448aMathieu Chartier if (kIsDebugBuild) { 70073d1e17b3afc7d5e56184f90bf819dc64956448aMathieu Chartier full_runs_[idx].insert(thread_local_run); 70173d1e17b3afc7d5e56184f90bf819dc64956448aMathieu Chartier if (kTraceRosAlloc) { 70273d1e17b3afc7d5e56184f90bf819dc64956448aMathieu Chartier LOG(INFO) << "RosAlloc::AllocFromRun() : Inserted run 0x" << std::hex 70373d1e17b3afc7d5e56184f90bf819dc64956448aMathieu Chartier << reinterpret_cast<intptr_t>(thread_local_run) 70473d1e17b3afc7d5e56184f90bf819dc64956448aMathieu Chartier << " into full_runs_[" << std::dec << idx << "]"; 70573d1e17b3afc7d5e56184f90bf819dc64956448aMathieu Chartier } 706cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi } 70773d1e17b3afc7d5e56184f90bf819dc64956448aMathieu Chartier DCHECK(non_full_runs_[idx].find(thread_local_run) == non_full_runs_[idx].end()); 70873d1e17b3afc7d5e56184f90bf819dc64956448aMathieu Chartier DCHECK(full_runs_[idx].find(thread_local_run) != full_runs_[idx].end()); 709cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi } 71073d1e17b3afc7d5e56184f90bf819dc64956448aMathieu Chartier 711cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi thread_local_run = RefillRun(self, idx); 7120651d41e41341fb2e9ef3ee41dc1f1bfc832dbbbMathieu Chartier if (UNLIKELY(thread_local_run == nullptr)) { 7130651d41e41341fb2e9ef3ee41dc1f1bfc832dbbbMathieu Chartier self->SetRosAllocRun(idx, dedicated_full_run_); 7140651d41e41341fb2e9ef3ee41dc1f1bfc832dbbbMathieu Chartier return nullptr; 715cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi } 716cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi DCHECK(non_full_runs_[idx].find(thread_local_run) == non_full_runs_[idx].end()); 717cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi DCHECK(full_runs_[idx].find(thread_local_run) == full_runs_[idx].end()); 71873d1e17b3afc7d5e56184f90bf819dc64956448aMathieu Chartier thread_local_run->SetIsThreadLocal(true); 719dd7624d2b9e599d57762d12031b10b89defc9807Ian Rogers self->SetRosAllocRun(idx, thread_local_run); 720cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi DCHECK(!thread_local_run->IsFull()); 721cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi } 7220651d41e41341fb2e9ef3ee41dc1f1bfc832dbbbMathieu Chartier DCHECK(thread_local_run != nullptr); 723cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi DCHECK(!thread_local_run->IsFull()); 72473d1e17b3afc7d5e56184f90bf819dc64956448aMathieu Chartier DCHECK(thread_local_run->IsThreadLocal()); 7254460a84be92b5a94ecfb5c650aef4945ab849c93Hiroshi Yamauchi // Account for all the free slots in the new or refreshed thread local run. 7264460a84be92b5a94ecfb5c650aef4945ab849c93Hiroshi Yamauchi *bytes_tl_bulk_allocated = thread_local_run->NumberOfFreeSlots() * bracket_size; 727cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi slot_addr = thread_local_run->AllocSlot(); 728cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi // Must succeed now with a new run. 7290651d41e41341fb2e9ef3ee41dc1f1bfc832dbbbMathieu Chartier DCHECK(slot_addr != nullptr); 7304460a84be92b5a94ecfb5c650aef4945ab849c93Hiroshi Yamauchi } else { 7314460a84be92b5a94ecfb5c650aef4945ab849c93Hiroshi Yamauchi // The slot is already counted. Leave it as is. 7324460a84be92b5a94ecfb5c650aef4945ab849c93Hiroshi Yamauchi *bytes_tl_bulk_allocated = 0; 733cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi } 7344460a84be92b5a94ecfb5c650aef4945ab849c93Hiroshi Yamauchi DCHECK(slot_addr != nullptr); 735cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi if (kTraceRosAlloc) { 7364460a84be92b5a94ecfb5c650aef4945ab849c93Hiroshi Yamauchi LOG(INFO) << "RosAlloc::AllocFromRun() thread-local : 0x" << std::hex 7374460a84be92b5a94ecfb5c650aef4945ab849c93Hiroshi Yamauchi << reinterpret_cast<intptr_t>(slot_addr) 738cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi << "-0x" << (reinterpret_cast<intptr_t>(slot_addr) + bracket_size) 739cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi << "(" << std::dec << (bracket_size) << ")"; 740cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi } 7414460a84be92b5a94ecfb5c650aef4945ab849c93Hiroshi Yamauchi *bytes_allocated = bracket_size; 7424460a84be92b5a94ecfb5c650aef4945ab849c93Hiroshi Yamauchi *usable_size = bracket_size; 743cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi } else { 744cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi // Use the (shared) current run. 745cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi MutexLock mu(self, *size_bracket_locks_[idx]); 7460651d41e41341fb2e9ef3ee41dc1f1bfc832dbbbMathieu Chartier slot_addr = AllocFromCurrentRunUnlocked(self, idx); 747cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi if (kTraceRosAlloc) { 7484460a84be92b5a94ecfb5c650aef4945ab849c93Hiroshi Yamauchi LOG(INFO) << "RosAlloc::AllocFromRun() : 0x" << std::hex 7494460a84be92b5a94ecfb5c650aef4945ab849c93Hiroshi Yamauchi << reinterpret_cast<intptr_t>(slot_addr) 750cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi << "-0x" << (reinterpret_cast<intptr_t>(slot_addr) + bracket_size) 751cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi << "(" << std::dec << (bracket_size) << ")"; 752cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi } 7534460a84be92b5a94ecfb5c650aef4945ab849c93Hiroshi Yamauchi if (LIKELY(slot_addr != nullptr)) { 7544460a84be92b5a94ecfb5c650aef4945ab849c93Hiroshi Yamauchi *bytes_allocated = bracket_size; 7554460a84be92b5a94ecfb5c650aef4945ab849c93Hiroshi Yamauchi *usable_size = bracket_size; 7564460a84be92b5a94ecfb5c650aef4945ab849c93Hiroshi Yamauchi *bytes_tl_bulk_allocated = bracket_size; 7574460a84be92b5a94ecfb5c650aef4945ab849c93Hiroshi Yamauchi } 758cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi } 75973d1e17b3afc7d5e56184f90bf819dc64956448aMathieu Chartier // Caller verifies that it is all 0. 760cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi return slot_addr; 761cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi} 762cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi 76373d1e17b3afc7d5e56184f90bf819dc64956448aMathieu Chartiersize_t RosAlloc::FreeFromRun(Thread* self, void* ptr, Run* run) { 7645d057056db1923947ba846b391d981759b15714aIan Rogers DCHECK_EQ(run->magic_num_, kMagicNum); 7655d057056db1923947ba846b391d981759b15714aIan Rogers DCHECK_LT(run, ptr); 7665d057056db1923947ba846b391d981759b15714aIan Rogers DCHECK_LT(ptr, run->End()); 76773d1e17b3afc7d5e56184f90bf819dc64956448aMathieu Chartier const size_t idx = run->size_bracket_idx_; 76873d1e17b3afc7d5e56184f90bf819dc64956448aMathieu Chartier const size_t bracket_size = bracketSizes[idx]; 769cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi bool run_was_full = false; 770277ccbd200ea43590dfc06a93ae184a765327ad0Andreas Gampe MutexLock brackets_mu(self, *size_bracket_locks_[idx]); 771cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi if (kIsDebugBuild) { 772cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi run_was_full = run->IsFull(); 773cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi } 774cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi if (kTraceRosAlloc) { 775cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi LOG(INFO) << "RosAlloc::FreeFromRun() : 0x" << std::hex << reinterpret_cast<intptr_t>(ptr); 776cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi } 77773d1e17b3afc7d5e56184f90bf819dc64956448aMathieu Chartier if (LIKELY(run->IsThreadLocal())) { 778cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi // It's a thread-local run. Just mark the thread-local free bit map and return. 7790651d41e41341fb2e9ef3ee41dc1f1bfc832dbbbMathieu Chartier DCHECK_LT(run->size_bracket_idx_, kNumThreadLocalSizeBrackets); 780cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi DCHECK(non_full_runs_[idx].find(run) == non_full_runs_[idx].end()); 781cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi DCHECK(full_runs_[idx].find(run) == full_runs_[idx].end()); 78231bf42c48c4d00f0677c31264bba8d21618dae67Hiroshi Yamauchi run->AddToThreadLocalFreeList(ptr); 783cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi if (kTraceRosAlloc) { 784cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi LOG(INFO) << "RosAlloc::FreeFromRun() : Freed a slot in a thread local run 0x" << std::hex 785cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi << reinterpret_cast<intptr_t>(run); 786cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi } 787cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi // A thread local run will be kept as a thread local even if it's become all free. 78873d1e17b3afc7d5e56184f90bf819dc64956448aMathieu Chartier return bracket_size; 789cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi } 790cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi // Free the slot in the run. 791cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi run->FreeSlot(ptr); 79258553c7fd89ce69857017322444265469bb6af62Mathieu Chartier auto* non_full_runs = &non_full_runs_[idx]; 793cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi if (run->IsAllFree()) { 794cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi // It has just become completely free. Free the pages of this run. 795cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi std::set<Run*>::iterator pos = non_full_runs->find(run); 796cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi if (pos != non_full_runs->end()) { 797cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi non_full_runs->erase(pos); 798cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi if (kTraceRosAlloc) { 799cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi LOG(INFO) << "RosAlloc::FreeFromRun() : Erased run 0x" << std::hex 800cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi << reinterpret_cast<intptr_t>(run) << " from non_full_runs_"; 801cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi } 802cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi } 803cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi if (run == current_runs_[idx]) { 8040651d41e41341fb2e9ef3ee41dc1f1bfc832dbbbMathieu Chartier current_runs_[idx] = dedicated_full_run_; 805cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi } 806cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi DCHECK(non_full_runs_[idx].find(run) == non_full_runs_[idx].end()); 807cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi DCHECK(full_runs_[idx].find(run) == full_runs_[idx].end()); 80831bf42c48c4d00f0677c31264bba8d21618dae67Hiroshi Yamauchi run->ZeroHeaderAndSlotHeaders(); 809cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi { 810277ccbd200ea43590dfc06a93ae184a765327ad0Andreas Gampe MutexLock lock_mu(self, lock_); 81173d1e17b3afc7d5e56184f90bf819dc64956448aMathieu Chartier FreePages(self, run, true); 812cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi } 813cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi } else { 814cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi // It is not completely free. If it wasn't the current run or 815cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi // already in the non-full run set (i.e., it was full) insert it 816cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi // into the non-full run set. 817cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi if (run != current_runs_[idx]) { 8182cebb24bfc3247d3e9be138a3350106737455918Mathieu Chartier auto* full_runs = kIsDebugBuild ? &full_runs_[idx] : nullptr; 81958553c7fd89ce69857017322444265469bb6af62Mathieu Chartier auto pos = non_full_runs->find(run); 820cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi if (pos == non_full_runs->end()) { 821cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi DCHECK(run_was_full); 822cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi DCHECK(full_runs->find(run) != full_runs->end()); 823cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi if (kIsDebugBuild) { 824cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi full_runs->erase(run); 825cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi if (kTraceRosAlloc) { 826cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi LOG(INFO) << "RosAlloc::FreeFromRun() : Erased run 0x" << std::hex 827cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi << reinterpret_cast<intptr_t>(run) << " from full_runs_"; 828cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi } 829cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi } 830cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi non_full_runs->insert(run); 831cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi DCHECK(!run->IsFull()); 832cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi if (kTraceRosAlloc) { 833cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi LOG(INFO) << "RosAlloc::FreeFromRun() : Inserted run 0x" << std::hex 834cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi << reinterpret_cast<intptr_t>(run) 835cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi << " into non_full_runs_[" << std::dec << idx << "]"; 836cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi } 837cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi } 838cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi } 839cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi } 84073d1e17b3afc7d5e56184f90bf819dc64956448aMathieu Chartier return bracket_size; 841cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi} 842cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi 84331bf42c48c4d00f0677c31264bba8d21618dae67Hiroshi Yamauchitemplate<bool kUseTail> 84431bf42c48c4d00f0677c31264bba8d21618dae67Hiroshi Yamauchistd::string RosAlloc::Run::FreeListToStr(SlotFreeList<kUseTail>* free_list) { 84531bf42c48c4d00f0677c31264bba8d21618dae67Hiroshi Yamauchi std::string free_list_str; 84631bf42c48c4d00f0677c31264bba8d21618dae67Hiroshi Yamauchi const uint8_t idx = size_bracket_idx_; 84731bf42c48c4d00f0677c31264bba8d21618dae67Hiroshi Yamauchi const size_t bracket_size = bracketSizes[idx]; 84831bf42c48c4d00f0677c31264bba8d21618dae67Hiroshi Yamauchi for (Slot* slot = free_list->Head(); slot != nullptr; slot = slot->Next()) { 84931bf42c48c4d00f0677c31264bba8d21618dae67Hiroshi Yamauchi bool is_last = slot->Next() == nullptr; 85031bf42c48c4d00f0677c31264bba8d21618dae67Hiroshi Yamauchi uintptr_t slot_offset = reinterpret_cast<uintptr_t>(slot) - 85131bf42c48c4d00f0677c31264bba8d21618dae67Hiroshi Yamauchi reinterpret_cast<uintptr_t>(FirstSlot()); 85231bf42c48c4d00f0677c31264bba8d21618dae67Hiroshi Yamauchi DCHECK_EQ(slot_offset % bracket_size, 0U); 85331bf42c48c4d00f0677c31264bba8d21618dae67Hiroshi Yamauchi uintptr_t slot_idx = slot_offset / bracket_size; 85431bf42c48c4d00f0677c31264bba8d21618dae67Hiroshi Yamauchi if (!is_last) { 85531bf42c48c4d00f0677c31264bba8d21618dae67Hiroshi Yamauchi free_list_str.append(StringPrintf("%u-", static_cast<uint32_t>(slot_idx))); 856cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi } else { 85731bf42c48c4d00f0677c31264bba8d21618dae67Hiroshi Yamauchi free_list_str.append(StringPrintf("%u", static_cast<uint32_t>(slot_idx))); 858cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi } 859cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi } 86031bf42c48c4d00f0677c31264bba8d21618dae67Hiroshi Yamauchi return free_list_str; 861a4adbfd44032d70e166e6f18096bbbed05a990baHiroshi Yamauchi} 862a4adbfd44032d70e166e6f18096bbbed05a990baHiroshi Yamauchi 863a4adbfd44032d70e166e6f18096bbbed05a990baHiroshi Yamauchistd::string RosAlloc::Run::Dump() { 864a4adbfd44032d70e166e6f18096bbbed05a990baHiroshi Yamauchi size_t idx = size_bracket_idx_; 865a4adbfd44032d70e166e6f18096bbbed05a990baHiroshi Yamauchi std::ostringstream stream; 866a4adbfd44032d70e166e6f18096bbbed05a990baHiroshi Yamauchi stream << "RosAlloc Run = " << reinterpret_cast<void*>(this) 867a4adbfd44032d70e166e6f18096bbbed05a990baHiroshi Yamauchi << "{ magic_num=" << static_cast<int>(magic_num_) 868a4adbfd44032d70e166e6f18096bbbed05a990baHiroshi Yamauchi << " size_bracket_idx=" << idx 869a4adbfd44032d70e166e6f18096bbbed05a990baHiroshi Yamauchi << " is_thread_local=" << static_cast<int>(is_thread_local_) 870a4adbfd44032d70e166e6f18096bbbed05a990baHiroshi Yamauchi << " to_be_bulk_freed=" << static_cast<int>(to_be_bulk_freed_) 87131bf42c48c4d00f0677c31264bba8d21618dae67Hiroshi Yamauchi << " free_list=" << FreeListToStr(&free_list_) 87231bf42c48c4d00f0677c31264bba8d21618dae67Hiroshi Yamauchi << " bulk_free_list=" << FreeListToStr(&bulk_free_list_) 87331bf42c48c4d00f0677c31264bba8d21618dae67Hiroshi Yamauchi << " thread_local_list=" << FreeListToStr(&thread_local_free_list_) 874a4adbfd44032d70e166e6f18096bbbed05a990baHiroshi Yamauchi << " }" << std::endl; 875a4adbfd44032d70e166e6f18096bbbed05a990baHiroshi Yamauchi return stream.str(); 876cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi} 877cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi 87831bf42c48c4d00f0677c31264bba8d21618dae67Hiroshi Yamauchivoid RosAlloc::Run::FreeSlot(void* ptr) { 87931bf42c48c4d00f0677c31264bba8d21618dae67Hiroshi Yamauchi DCHECK(!IsThreadLocal()); 88031bf42c48c4d00f0677c31264bba8d21618dae67Hiroshi Yamauchi const uint8_t idx = size_bracket_idx_; 88131bf42c48c4d00f0677c31264bba8d21618dae67Hiroshi Yamauchi const size_t bracket_size = bracketSizes[idx]; 88231bf42c48c4d00f0677c31264bba8d21618dae67Hiroshi Yamauchi Slot* slot = ToSlot(ptr); 88373d1e17b3afc7d5e56184f90bf819dc64956448aMathieu Chartier // Zero out the memory. 88473d1e17b3afc7d5e56184f90bf819dc64956448aMathieu Chartier // TODO: Investigate alternate memset since ptr is guaranteed to be aligned to 16. 88531bf42c48c4d00f0677c31264bba8d21618dae67Hiroshi Yamauchi memset(slot, 0, bracket_size); 88631bf42c48c4d00f0677c31264bba8d21618dae67Hiroshi Yamauchi free_list_.Add(slot); 887cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi if (kTraceRosAlloc) { 88831bf42c48c4d00f0677c31264bba8d21618dae67Hiroshi Yamauchi LOG(INFO) << "RosAlloc::Run::FreeSlot() : " << slot 88931bf42c48c4d00f0677c31264bba8d21618dae67Hiroshi Yamauchi << ", bracket_size=" << std::dec << bracket_size << ", slot_idx=" << SlotIndex(slot); 890cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi } 891cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi} 892cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi 89331bf42c48c4d00f0677c31264bba8d21618dae67Hiroshi Yamauchiinline bool RosAlloc::Run::MergeThreadLocalFreeListToFreeList(bool* is_all_free_after_out) { 89473d1e17b3afc7d5e56184f90bf819dc64956448aMathieu Chartier DCHECK(IsThreadLocal()); 89531bf42c48c4d00f0677c31264bba8d21618dae67Hiroshi Yamauchi // Merge the thread local free list into the free list and clear the thread local free list. 89631bf42c48c4d00f0677c31264bba8d21618dae67Hiroshi Yamauchi const uint8_t idx = size_bracket_idx_; 89731bf42c48c4d00f0677c31264bba8d21618dae67Hiroshi Yamauchi bool thread_local_free_list_size = thread_local_free_list_.Size(); 89831bf42c48c4d00f0677c31264bba8d21618dae67Hiroshi Yamauchi const size_t size_before = free_list_.Size(); 89931bf42c48c4d00f0677c31264bba8d21618dae67Hiroshi Yamauchi free_list_.Merge(&thread_local_free_list_); 90031bf42c48c4d00f0677c31264bba8d21618dae67Hiroshi Yamauchi const size_t size_after = free_list_.Size(); 90131bf42c48c4d00f0677c31264bba8d21618dae67Hiroshi Yamauchi DCHECK_EQ(size_before < size_after, thread_local_free_list_size > 0); 90231bf42c48c4d00f0677c31264bba8d21618dae67Hiroshi Yamauchi DCHECK_LE(size_before, size_after); 90331bf42c48c4d00f0677c31264bba8d21618dae67Hiroshi Yamauchi *is_all_free_after_out = free_list_.Size() == numOfSlots[idx]; 90431bf42c48c4d00f0677c31264bba8d21618dae67Hiroshi Yamauchi // Return true at least one slot was added to the free list. 90531bf42c48c4d00f0677c31264bba8d21618dae67Hiroshi Yamauchi return size_before < size_after; 906cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi} 907cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi 90831bf42c48c4d00f0677c31264bba8d21618dae67Hiroshi Yamauchiinline void RosAlloc::Run::MergeBulkFreeListToFreeList() { 90973d1e17b3afc7d5e56184f90bf819dc64956448aMathieu Chartier DCHECK(!IsThreadLocal()); 91031bf42c48c4d00f0677c31264bba8d21618dae67Hiroshi Yamauchi // Merge the bulk free list into the free list and clear the bulk free list. 91131bf42c48c4d00f0677c31264bba8d21618dae67Hiroshi Yamauchi free_list_.Merge(&bulk_free_list_); 912cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi} 913cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi 91431bf42c48c4d00f0677c31264bba8d21618dae67Hiroshi Yamauchiinline void RosAlloc::Run::MergeBulkFreeListToThreadLocalFreeList() { 91573d1e17b3afc7d5e56184f90bf819dc64956448aMathieu Chartier DCHECK(IsThreadLocal()); 91631bf42c48c4d00f0677c31264bba8d21618dae67Hiroshi Yamauchi // Merge the bulk free list into the thread local free list and clear the bulk free list. 91731bf42c48c4d00f0677c31264bba8d21618dae67Hiroshi Yamauchi thread_local_free_list_.Merge(&bulk_free_list_); 918cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi} 919cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi 92031bf42c48c4d00f0677c31264bba8d21618dae67Hiroshi Yamauchiinline void RosAlloc::Run::AddToThreadLocalFreeList(void* ptr) { 92173d1e17b3afc7d5e56184f90bf819dc64956448aMathieu Chartier DCHECK(IsThreadLocal()); 92231bf42c48c4d00f0677c31264bba8d21618dae67Hiroshi Yamauchi AddToFreeListShared(ptr, &thread_local_free_list_, __FUNCTION__); 923cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi} 924cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi 92531bf42c48c4d00f0677c31264bba8d21618dae67Hiroshi Yamauchiinline size_t RosAlloc::Run::AddToBulkFreeList(void* ptr) { 92631bf42c48c4d00f0677c31264bba8d21618dae67Hiroshi Yamauchi return AddToFreeListShared(ptr, &bulk_free_list_, __FUNCTION__); 927cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi} 928cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi 92931bf42c48c4d00f0677c31264bba8d21618dae67Hiroshi Yamauchiinline size_t RosAlloc::Run::AddToFreeListShared(void* ptr, 93031bf42c48c4d00f0677c31264bba8d21618dae67Hiroshi Yamauchi SlotFreeList<true>* free_list, 93131bf42c48c4d00f0677c31264bba8d21618dae67Hiroshi Yamauchi const char* caller_name) { 93213735955f39b3b304c37d2b2840663c131262c18Ian Rogers const uint8_t idx = size_bracket_idx_; 93373d1e17b3afc7d5e56184f90bf819dc64956448aMathieu Chartier const size_t bracket_size = bracketSizes[idx]; 93431bf42c48c4d00f0677c31264bba8d21618dae67Hiroshi Yamauchi Slot* slot = ToSlot(ptr); 93531bf42c48c4d00f0677c31264bba8d21618dae67Hiroshi Yamauchi memset(slot, 0, bracket_size); 93631bf42c48c4d00f0677c31264bba8d21618dae67Hiroshi Yamauchi free_list->Add(slot); 937cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi if (kTraceRosAlloc) { 93831bf42c48c4d00f0677c31264bba8d21618dae67Hiroshi Yamauchi LOG(INFO) << "RosAlloc::Run::" << caller_name << "() : " << ptr 93931bf42c48c4d00f0677c31264bba8d21618dae67Hiroshi Yamauchi << ", bracket_size=" << std::dec << bracket_size << ", slot_idx=" << SlotIndex(slot); 940cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi } 94173d1e17b3afc7d5e56184f90bf819dc64956448aMathieu Chartier return bracket_size; 94273d1e17b3afc7d5e56184f90bf819dc64956448aMathieu Chartier} 94373d1e17b3afc7d5e56184f90bf819dc64956448aMathieu Chartier 94431bf42c48c4d00f0677c31264bba8d21618dae67Hiroshi Yamauchiinline void RosAlloc::Run::ZeroHeaderAndSlotHeaders() { 94531bf42c48c4d00f0677c31264bba8d21618dae67Hiroshi Yamauchi DCHECK(IsAllFree()); 94613735955f39b3b304c37d2b2840663c131262c18Ian Rogers const uint8_t idx = size_bracket_idx_; 94731bf42c48c4d00f0677c31264bba8d21618dae67Hiroshi Yamauchi // Zero the slot header (next pointers). 94831bf42c48c4d00f0677c31264bba8d21618dae67Hiroshi Yamauchi for (Slot* slot = free_list_.Head(); slot != nullptr; ) { 94931bf42c48c4d00f0677c31264bba8d21618dae67Hiroshi Yamauchi Slot* next_slot = slot->Next(); 95031bf42c48c4d00f0677c31264bba8d21618dae67Hiroshi Yamauchi slot->Clear(); 95131bf42c48c4d00f0677c31264bba8d21618dae67Hiroshi Yamauchi slot = next_slot; 952a4adbfd44032d70e166e6f18096bbbed05a990baHiroshi Yamauchi } 95331bf42c48c4d00f0677c31264bba8d21618dae67Hiroshi Yamauchi // Zero the header. 95431bf42c48c4d00f0677c31264bba8d21618dae67Hiroshi Yamauchi memset(this, 0, headerSizes[idx]); 95531bf42c48c4d00f0677c31264bba8d21618dae67Hiroshi Yamauchi // Check that the entire run is all zero. 95631bf42c48c4d00f0677c31264bba8d21618dae67Hiroshi Yamauchi if (kIsDebugBuild) { 95731bf42c48c4d00f0677c31264bba8d21618dae67Hiroshi Yamauchi const size_t size = numOfPages[idx] * kPageSize; 95831bf42c48c4d00f0677c31264bba8d21618dae67Hiroshi Yamauchi const uintptr_t* word_ptr = reinterpret_cast<uintptr_t*>(this); 95931bf42c48c4d00f0677c31264bba8d21618dae67Hiroshi Yamauchi for (size_t i = 0; i < size / sizeof(uintptr_t); ++i) { 96031bf42c48c4d00f0677c31264bba8d21618dae67Hiroshi Yamauchi CHECK_EQ(word_ptr[i], 0U) << "words don't match at index " << i; 961a4adbfd44032d70e166e6f18096bbbed05a990baHiroshi Yamauchi } 962a4adbfd44032d70e166e6f18096bbbed05a990baHiroshi Yamauchi } 96373d1e17b3afc7d5e56184f90bf819dc64956448aMathieu Chartier} 96473d1e17b3afc7d5e56184f90bf819dc64956448aMathieu Chartier 96573d1e17b3afc7d5e56184f90bf819dc64956448aMathieu Chartierinline void RosAlloc::Run::ZeroData() { 96613735955f39b3b304c37d2b2840663c131262c18Ian Rogers const uint8_t idx = size_bracket_idx_; 96731bf42c48c4d00f0677c31264bba8d21618dae67Hiroshi Yamauchi uint8_t* slot_begin = reinterpret_cast<uint8_t*>(FirstSlot()); 96873d1e17b3afc7d5e56184f90bf819dc64956448aMathieu Chartier memset(slot_begin, 0, numOfSlots[idx] * bracketSizes[idx]); 96973d1e17b3afc7d5e56184f90bf819dc64956448aMathieu Chartier} 97073d1e17b3afc7d5e56184f90bf819dc64956448aMathieu Chartier 971cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchivoid RosAlloc::Run::InspectAllSlots(void (*handler)(void* start, void* end, size_t used_bytes, void* callback_arg), 972cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi void* arg) { 973cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi size_t idx = size_bracket_idx_; 97413735955f39b3b304c37d2b2840663c131262c18Ian Rogers uint8_t* slot_base = reinterpret_cast<uint8_t*>(this) + headerSizes[idx]; 975cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi size_t num_slots = numOfSlots[idx]; 976cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi size_t bracket_size = IndexToBracketSize(idx); 977c38c5ea76dd3cfd44eec21df640161046ffc3e4cMathieu Chartier DCHECK_EQ(slot_base + num_slots * bracket_size, 978c38c5ea76dd3cfd44eec21df640161046ffc3e4cMathieu Chartier reinterpret_cast<uint8_t*>(this) + numOfPages[idx] * kPageSize); 97931bf42c48c4d00f0677c31264bba8d21618dae67Hiroshi Yamauchi // Free slots are on the free list and the allocated/used slots are not. We traverse the free list 98031bf42c48c4d00f0677c31264bba8d21618dae67Hiroshi Yamauchi // to find out and record which slots are free in the is_free array. 98131bf42c48c4d00f0677c31264bba8d21618dae67Hiroshi Yamauchi std::unique_ptr<bool[]> is_free(new bool[num_slots]()); // zero initialized 98231bf42c48c4d00f0677c31264bba8d21618dae67Hiroshi Yamauchi for (Slot* slot = free_list_.Head(); slot != nullptr; slot = slot->Next()) { 98331bf42c48c4d00f0677c31264bba8d21618dae67Hiroshi Yamauchi size_t slot_idx = SlotIndex(slot); 98431bf42c48c4d00f0677c31264bba8d21618dae67Hiroshi Yamauchi DCHECK_LT(slot_idx, num_slots); 98531bf42c48c4d00f0677c31264bba8d21618dae67Hiroshi Yamauchi is_free[slot_idx] = true; 98631bf42c48c4d00f0677c31264bba8d21618dae67Hiroshi Yamauchi } 98731bf42c48c4d00f0677c31264bba8d21618dae67Hiroshi Yamauchi if (IsThreadLocal()) { 98831bf42c48c4d00f0677c31264bba8d21618dae67Hiroshi Yamauchi for (Slot* slot = thread_local_free_list_.Head(); slot != nullptr; slot = slot->Next()) { 98931bf42c48c4d00f0677c31264bba8d21618dae67Hiroshi Yamauchi size_t slot_idx = SlotIndex(slot); 99031bf42c48c4d00f0677c31264bba8d21618dae67Hiroshi Yamauchi DCHECK_LT(slot_idx, num_slots); 99131bf42c48c4d00f0677c31264bba8d21618dae67Hiroshi Yamauchi is_free[slot_idx] = true; 99231bf42c48c4d00f0677c31264bba8d21618dae67Hiroshi Yamauchi } 99331bf42c48c4d00f0677c31264bba8d21618dae67Hiroshi Yamauchi } 99431bf42c48c4d00f0677c31264bba8d21618dae67Hiroshi Yamauchi for (size_t slot_idx = 0; slot_idx < num_slots; ++slot_idx) { 99531bf42c48c4d00f0677c31264bba8d21618dae67Hiroshi Yamauchi uint8_t* slot_addr = slot_base + slot_idx * bracket_size; 99631bf42c48c4d00f0677c31264bba8d21618dae67Hiroshi Yamauchi if (!is_free[slot_idx]) { 99731bf42c48c4d00f0677c31264bba8d21618dae67Hiroshi Yamauchi handler(slot_addr, slot_addr + bracket_size, bracket_size, arg); 99831bf42c48c4d00f0677c31264bba8d21618dae67Hiroshi Yamauchi } else { 99931bf42c48c4d00f0677c31264bba8d21618dae67Hiroshi Yamauchi handler(slot_addr, slot_addr + bracket_size, 0, arg); 1000cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi } 1001cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi } 1002cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi} 1003cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi 100426d69ffc0ebc98fbc5f316d8cd3ee6ba5b2001acHiroshi Yamauchi// If true, read the page map entries in BulkFree() without using the 100526d69ffc0ebc98fbc5f316d8cd3ee6ba5b2001acHiroshi Yamauchi// lock for better performance, assuming that the existence of an 100626d69ffc0ebc98fbc5f316d8cd3ee6ba5b2001acHiroshi Yamauchi// allocated chunk/pointer being freed in BulkFree() guarantees that 100726d69ffc0ebc98fbc5f316d8cd3ee6ba5b2001acHiroshi Yamauchi// the page map entry won't change. Disabled for now. 100873d1e17b3afc7d5e56184f90bf819dc64956448aMathieu Chartierstatic constexpr bool kReadPageMapEntryWithoutLockInBulkFree = true; 100926d69ffc0ebc98fbc5f316d8cd3ee6ba5b2001acHiroshi Yamauchi 10108585bad7be19ee4901333f7d02d1d4d3f04877d4Mathieu Chartiersize_t RosAlloc::BulkFree(Thread* self, void** ptrs, size_t num_ptrs) { 10118585bad7be19ee4901333f7d02d1d4d3f04877d4Mathieu Chartier size_t freed_bytes = 0; 1012cf7f19135f0e273f7b0136315633c2abfc715343Ian Rogers if ((false)) { 1013cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi // Used only to test Free() as GC uses only BulkFree(). 1014cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi for (size_t i = 0; i < num_ptrs; ++i) { 10158585bad7be19ee4901333f7d02d1d4d3f04877d4Mathieu Chartier freed_bytes += FreeInternal(self, ptrs[i]); 1016cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi } 10178585bad7be19ee4901333f7d02d1d4d3f04877d4Mathieu Chartier return freed_bytes; 1018cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi } 1019cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi 1020cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi WriterMutexLock wmu(self, bulk_free_lock_); 1021cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi 1022cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi // First mark slots to free in the bulk free bit map without locking the 10235fcfa7d9d97246f7eb48a74356cb00ec2cbc0181Ian Rogers // size bracket locks. On host, unordered_set is faster than vector + flag. 1024c60e1b755c5632dfeb04c333489ede52ee5c945fAndreas Gampe#ifdef __ANDROID__ 1025cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi std::vector<Run*> runs; 1026cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi#else 1027700a402244a1a423da4f3ba8032459f4b65fa18fIan Rogers std::unordered_set<Run*, hash_run, eq_run> runs; 1028cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi#endif 102926d69ffc0ebc98fbc5f316d8cd3ee6ba5b2001acHiroshi Yamauchi for (size_t i = 0; i < num_ptrs; i++) { 103026d69ffc0ebc98fbc5f316d8cd3ee6ba5b2001acHiroshi Yamauchi void* ptr = ptrs[i]; 10315d057056db1923947ba846b391d981759b15714aIan Rogers DCHECK_LE(base_, ptr); 10325d057056db1923947ba846b391d981759b15714aIan Rogers DCHECK_LT(ptr, base_ + footprint_); 103326d69ffc0ebc98fbc5f316d8cd3ee6ba5b2001acHiroshi Yamauchi size_t pm_idx = RoundDownToPageMapIndex(ptr); 103473d1e17b3afc7d5e56184f90bf819dc64956448aMathieu Chartier Run* run = nullptr; 103526d69ffc0ebc98fbc5f316d8cd3ee6ba5b2001acHiroshi Yamauchi if (kReadPageMapEntryWithoutLockInBulkFree) { 103626d69ffc0ebc98fbc5f316d8cd3ee6ba5b2001acHiroshi Yamauchi // Read the page map entries without locking the lock. 103713735955f39b3b304c37d2b2840663c131262c18Ian Rogers uint8_t page_map_entry = page_map_[pm_idx]; 103826d69ffc0ebc98fbc5f316d8cd3ee6ba5b2001acHiroshi Yamauchi if (kTraceRosAlloc) { 103926d69ffc0ebc98fbc5f316d8cd3ee6ba5b2001acHiroshi Yamauchi LOG(INFO) << "RosAlloc::BulkFree() : " << std::hex << ptr << ", pm_idx=" 104026d69ffc0ebc98fbc5f316d8cd3ee6ba5b2001acHiroshi Yamauchi << std::dec << pm_idx 104126d69ffc0ebc98fbc5f316d8cd3ee6ba5b2001acHiroshi Yamauchi << ", page_map_entry=" << static_cast<int>(page_map_entry); 104226d69ffc0ebc98fbc5f316d8cd3ee6ba5b2001acHiroshi Yamauchi } 104326d69ffc0ebc98fbc5f316d8cd3ee6ba5b2001acHiroshi Yamauchi if (LIKELY(page_map_entry == kPageMapRun)) { 104426d69ffc0ebc98fbc5f316d8cd3ee6ba5b2001acHiroshi Yamauchi run = reinterpret_cast<Run*>(base_ + pm_idx * kPageSize); 104526d69ffc0ebc98fbc5f316d8cd3ee6ba5b2001acHiroshi Yamauchi } else if (LIKELY(page_map_entry == kPageMapRunPart)) { 104626d69ffc0ebc98fbc5f316d8cd3ee6ba5b2001acHiroshi Yamauchi size_t pi = pm_idx; 104726d69ffc0ebc98fbc5f316d8cd3ee6ba5b2001acHiroshi Yamauchi // Find the beginning of the run. 104873d1e17b3afc7d5e56184f90bf819dc64956448aMathieu Chartier do { 104973d1e17b3afc7d5e56184f90bf819dc64956448aMathieu Chartier --pi; 10505d057056db1923947ba846b391d981759b15714aIan Rogers DCHECK_LT(pi, capacity_ / kPageSize); 105173d1e17b3afc7d5e56184f90bf819dc64956448aMathieu Chartier } while (page_map_[pi] != kPageMapRun); 105226d69ffc0ebc98fbc5f316d8cd3ee6ba5b2001acHiroshi Yamauchi run = reinterpret_cast<Run*>(base_ + pi * kPageSize); 105326d69ffc0ebc98fbc5f316d8cd3ee6ba5b2001acHiroshi Yamauchi } else if (page_map_entry == kPageMapLargeObject) { 105426d69ffc0ebc98fbc5f316d8cd3ee6ba5b2001acHiroshi Yamauchi MutexLock mu(self, lock_); 105573d1e17b3afc7d5e56184f90bf819dc64956448aMathieu Chartier freed_bytes += FreePages(self, ptr, false); 105626d69ffc0ebc98fbc5f316d8cd3ee6ba5b2001acHiroshi Yamauchi continue; 105726d69ffc0ebc98fbc5f316d8cd3ee6ba5b2001acHiroshi Yamauchi } else { 10582fdeecb890a353d3f17407cc1cb015e0a65c2220Maxim Kazantsev LOG(FATAL) << "Unreachable - page map type: " << static_cast<int>(page_map_entry); 105926d69ffc0ebc98fbc5f316d8cd3ee6ba5b2001acHiroshi Yamauchi } 106026d69ffc0ebc98fbc5f316d8cd3ee6ba5b2001acHiroshi Yamauchi } else { 106126d69ffc0ebc98fbc5f316d8cd3ee6ba5b2001acHiroshi Yamauchi // Read the page map entries with a lock. 106273d1e17b3afc7d5e56184f90bf819dc64956448aMathieu Chartier MutexLock mu(self, lock_); 106373d1e17b3afc7d5e56184f90bf819dc64956448aMathieu Chartier DCHECK_LT(pm_idx, page_map_size_); 106413735955f39b3b304c37d2b2840663c131262c18Ian Rogers uint8_t page_map_entry = page_map_[pm_idx]; 106573d1e17b3afc7d5e56184f90bf819dc64956448aMathieu Chartier if (kTraceRosAlloc) { 106673d1e17b3afc7d5e56184f90bf819dc64956448aMathieu Chartier LOG(INFO) << "RosAlloc::BulkFree() : " << std::hex << ptr << ", pm_idx=" 106773d1e17b3afc7d5e56184f90bf819dc64956448aMathieu Chartier << std::dec << pm_idx 106873d1e17b3afc7d5e56184f90bf819dc64956448aMathieu Chartier << ", page_map_entry=" << static_cast<int>(page_map_entry); 106973d1e17b3afc7d5e56184f90bf819dc64956448aMathieu Chartier } 107073d1e17b3afc7d5e56184f90bf819dc64956448aMathieu Chartier if (LIKELY(page_map_entry == kPageMapRun)) { 107173d1e17b3afc7d5e56184f90bf819dc64956448aMathieu Chartier run = reinterpret_cast<Run*>(base_ + pm_idx * kPageSize); 107273d1e17b3afc7d5e56184f90bf819dc64956448aMathieu Chartier } else if (LIKELY(page_map_entry == kPageMapRunPart)) { 107373d1e17b3afc7d5e56184f90bf819dc64956448aMathieu Chartier size_t pi = pm_idx; 107473d1e17b3afc7d5e56184f90bf819dc64956448aMathieu Chartier // Find the beginning of the run. 107573d1e17b3afc7d5e56184f90bf819dc64956448aMathieu Chartier do { 107673d1e17b3afc7d5e56184f90bf819dc64956448aMathieu Chartier --pi; 107773d1e17b3afc7d5e56184f90bf819dc64956448aMathieu Chartier DCHECK_LT(pi, capacity_ / kPageSize); 107873d1e17b3afc7d5e56184f90bf819dc64956448aMathieu Chartier } while (page_map_[pi] != kPageMapRun); 107973d1e17b3afc7d5e56184f90bf819dc64956448aMathieu Chartier run = reinterpret_cast<Run*>(base_ + pi * kPageSize); 108073d1e17b3afc7d5e56184f90bf819dc64956448aMathieu Chartier } else if (page_map_entry == kPageMapLargeObject) { 108173d1e17b3afc7d5e56184f90bf819dc64956448aMathieu Chartier freed_bytes += FreePages(self, ptr, false); 108273d1e17b3afc7d5e56184f90bf819dc64956448aMathieu Chartier continue; 108373d1e17b3afc7d5e56184f90bf819dc64956448aMathieu Chartier } else { 10842fdeecb890a353d3f17407cc1cb015e0a65c2220Maxim Kazantsev LOG(FATAL) << "Unreachable - page map type: " << static_cast<int>(page_map_entry); 1085cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi } 108673d1e17b3afc7d5e56184f90bf819dc64956448aMathieu Chartier } 108773d1e17b3afc7d5e56184f90bf819dc64956448aMathieu Chartier DCHECK(run != nullptr); 108873d1e17b3afc7d5e56184f90bf819dc64956448aMathieu Chartier DCHECK_EQ(run->magic_num_, kMagicNum); 108973d1e17b3afc7d5e56184f90bf819dc64956448aMathieu Chartier // Set the bit in the bulk free bit map. 109031bf42c48c4d00f0677c31264bba8d21618dae67Hiroshi Yamauchi freed_bytes += run->AddToBulkFreeList(ptr); 1091c60e1b755c5632dfeb04c333489ede52ee5c945fAndreas Gampe#ifdef __ANDROID__ 109273d1e17b3afc7d5e56184f90bf819dc64956448aMathieu Chartier if (!run->to_be_bulk_freed_) { 109373d1e17b3afc7d5e56184f90bf819dc64956448aMathieu Chartier run->to_be_bulk_freed_ = true; 109473d1e17b3afc7d5e56184f90bf819dc64956448aMathieu Chartier runs.push_back(run); 109573d1e17b3afc7d5e56184f90bf819dc64956448aMathieu Chartier } 1096cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi#else 109773d1e17b3afc7d5e56184f90bf819dc64956448aMathieu Chartier runs.insert(run); 1098cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi#endif 1099cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi } 1100cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi 1101cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi // Now, iterate over the affected runs and update the alloc bit map 1102cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi // based on the bulk free bit map (for non-thread-local runs) and 1103cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi // union the bulk free bit map into the thread-local free bit map 1104cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi // (for thread-local runs.) 110573d1e17b3afc7d5e56184f90bf819dc64956448aMathieu Chartier for (Run* run : runs) { 1106c60e1b755c5632dfeb04c333489ede52ee5c945fAndreas Gampe#ifdef __ANDROID__ 1107cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi DCHECK(run->to_be_bulk_freed_); 1108cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi run->to_be_bulk_freed_ = false; 1109cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi#endif 1110cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi size_t idx = run->size_bracket_idx_; 1111277ccbd200ea43590dfc06a93ae184a765327ad0Andreas Gampe MutexLock brackets_mu(self, *size_bracket_locks_[idx]); 111273d1e17b3afc7d5e56184f90bf819dc64956448aMathieu Chartier if (run->IsThreadLocal()) { 11130651d41e41341fb2e9ef3ee41dc1f1bfc832dbbbMathieu Chartier DCHECK_LT(run->size_bracket_idx_, kNumThreadLocalSizeBrackets); 1114cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi DCHECK(non_full_runs_[idx].find(run) == non_full_runs_[idx].end()); 1115cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi DCHECK(full_runs_[idx].find(run) == full_runs_[idx].end()); 111631bf42c48c4d00f0677c31264bba8d21618dae67Hiroshi Yamauchi run->MergeBulkFreeListToThreadLocalFreeList(); 1117cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi if (kTraceRosAlloc) { 1118cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi LOG(INFO) << "RosAlloc::BulkFree() : Freed slot(s) in a thread local run 0x" 1119cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi << std::hex << reinterpret_cast<intptr_t>(run); 1120cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi } 112173d1e17b3afc7d5e56184f90bf819dc64956448aMathieu Chartier DCHECK(run->IsThreadLocal()); 1122cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi // A thread local run will be kept as a thread local even if 1123cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi // it's become all free. 1124cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi } else { 1125cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi bool run_was_full = run->IsFull(); 112631bf42c48c4d00f0677c31264bba8d21618dae67Hiroshi Yamauchi run->MergeBulkFreeListToFreeList(); 1127cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi if (kTraceRosAlloc) { 1128cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi LOG(INFO) << "RosAlloc::BulkFree() : Freed slot(s) in a run 0x" << std::hex 1129cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi << reinterpret_cast<intptr_t>(run); 1130cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi } 1131cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi // Check if the run should be moved to non_full_runs_ or 1132cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi // free_page_runs_. 113358553c7fd89ce69857017322444265469bb6af62Mathieu Chartier auto* non_full_runs = &non_full_runs_[idx]; 11342cebb24bfc3247d3e9be138a3350106737455918Mathieu Chartier auto* full_runs = kIsDebugBuild ? &full_runs_[idx] : nullptr; 1135cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi if (run->IsAllFree()) { 1136cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi // It has just become completely free. Free the pages of the 1137cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi // run. 1138cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi bool run_was_current = run == current_runs_[idx]; 1139cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi if (run_was_current) { 1140cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi DCHECK(full_runs->find(run) == full_runs->end()); 1141cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi DCHECK(non_full_runs->find(run) == non_full_runs->end()); 1142cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi // If it was a current run, reuse it. 1143cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi } else if (run_was_full) { 1144cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi // If it was full, remove it from the full run set (debug 1145cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi // only.) 1146cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi if (kIsDebugBuild) { 1147700a402244a1a423da4f3ba8032459f4b65fa18fIan Rogers std::unordered_set<Run*, hash_run, eq_run>::iterator pos = full_runs->find(run); 1148cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi DCHECK(pos != full_runs->end()); 1149cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi full_runs->erase(pos); 1150cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi if (kTraceRosAlloc) { 1151cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi LOG(INFO) << "RosAlloc::BulkFree() : Erased run 0x" << std::hex 1152cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi << reinterpret_cast<intptr_t>(run) 1153cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi << " from full_runs_"; 1154cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi } 1155cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi DCHECK(full_runs->find(run) == full_runs->end()); 1156cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi } 1157cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi } else { 1158cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi // If it was in a non full run set, remove it from the set. 1159cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi DCHECK(full_runs->find(run) == full_runs->end()); 1160cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi DCHECK(non_full_runs->find(run) != non_full_runs->end()); 1161cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi non_full_runs->erase(run); 1162cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi if (kTraceRosAlloc) { 1163cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi LOG(INFO) << "RosAlloc::BulkFree() : Erased run 0x" << std::hex 1164cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi << reinterpret_cast<intptr_t>(run) 1165cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi << " from non_full_runs_"; 1166cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi } 1167cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi DCHECK(non_full_runs->find(run) == non_full_runs->end()); 1168cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi } 1169cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi if (!run_was_current) { 117031bf42c48c4d00f0677c31264bba8d21618dae67Hiroshi Yamauchi run->ZeroHeaderAndSlotHeaders(); 1171277ccbd200ea43590dfc06a93ae184a765327ad0Andreas Gampe MutexLock lock_mu(self, lock_); 117273d1e17b3afc7d5e56184f90bf819dc64956448aMathieu Chartier FreePages(self, run, true); 1173cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi } 1174cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi } else { 1175cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi // It is not completely free. If it wasn't the current run or 1176cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi // already in the non-full run set (i.e., it was full) insert 1177cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi // it into the non-full run set. 1178cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi if (run == current_runs_[idx]) { 1179cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi DCHECK(non_full_runs->find(run) == non_full_runs->end()); 1180cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi DCHECK(full_runs->find(run) == full_runs->end()); 1181cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi // If it was a current run, keep it. 1182cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi } else if (run_was_full) { 1183cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi // If it was full, remove it from the full run set (debug 1184cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi // only) and insert into the non-full run set. 1185cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi DCHECK(full_runs->find(run) != full_runs->end()); 1186cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi DCHECK(non_full_runs->find(run) == non_full_runs->end()); 1187cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi if (kIsDebugBuild) { 1188cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi full_runs->erase(run); 1189cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi if (kTraceRosAlloc) { 1190cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi LOG(INFO) << "RosAlloc::BulkFree() : Erased run 0x" << std::hex 1191cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi << reinterpret_cast<intptr_t>(run) 1192cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi << " from full_runs_"; 1193cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi } 1194cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi } 1195cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi non_full_runs->insert(run); 1196cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi if (kTraceRosAlloc) { 1197cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi LOG(INFO) << "RosAlloc::BulkFree() : Inserted run 0x" << std::hex 1198cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi << reinterpret_cast<intptr_t>(run) 1199cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi << " into non_full_runs_[" << std::dec << idx; 1200cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi } 1201cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi } else { 1202cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi // If it was not full, so leave it in the non full run set. 1203cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi DCHECK(full_runs->find(run) == full_runs->end()); 1204cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi DCHECK(non_full_runs->find(run) != non_full_runs->end()); 1205cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi } 1206cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi } 1207cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi } 1208cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi } 12098585bad7be19ee4901333f7d02d1d4d3f04877d4Mathieu Chartier return freed_bytes; 1210cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi} 1211cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi 1212a4adbfd44032d70e166e6f18096bbbed05a990baHiroshi Yamauchistd::string RosAlloc::DumpPageMap() { 1213a4adbfd44032d70e166e6f18096bbbed05a990baHiroshi Yamauchi std::ostringstream stream; 1214a4adbfd44032d70e166e6f18096bbbed05a990baHiroshi Yamauchi stream << "RosAlloc PageMap: " << std::endl; 1215a4adbfd44032d70e166e6f18096bbbed05a990baHiroshi Yamauchi lock_.AssertHeld(Thread::Current()); 121626d69ffc0ebc98fbc5f316d8cd3ee6ba5b2001acHiroshi Yamauchi size_t end = page_map_size_; 12172cebb24bfc3247d3e9be138a3350106737455918Mathieu Chartier FreePageRun* curr_fpr = nullptr; 1218cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi size_t curr_fpr_size = 0; 1219cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi size_t remaining_curr_fpr_size = 0; 1220cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi size_t num_running_empty_pages = 0; 1221cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi for (size_t i = 0; i < end; ++i) { 122213735955f39b3b304c37d2b2840663c131262c18Ian Rogers uint8_t pm = page_map_[i]; 1223cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi switch (pm) { 1224a5b5c55c8585b7ce915f0c7e1f66d121a7f7a078Mathieu Chartier case kPageMapReleased: 1225a5b5c55c8585b7ce915f0c7e1f66d121a7f7a078Mathieu Chartier // Fall-through. 1226cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi case kPageMapEmpty: { 1227cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi FreePageRun* fpr = reinterpret_cast<FreePageRun*>(base_ + i * kPageSize); 1228cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi if (free_page_runs_.find(fpr) != free_page_runs_.end()) { 1229cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi // Encountered a fresh free page run. 1230cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi DCHECK_EQ(remaining_curr_fpr_size, static_cast<size_t>(0)); 1231cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi DCHECK(fpr->IsFree()); 12322cebb24bfc3247d3e9be138a3350106737455918Mathieu Chartier DCHECK(curr_fpr == nullptr); 1233cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi DCHECK_EQ(curr_fpr_size, static_cast<size_t>(0)); 1234cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi curr_fpr = fpr; 1235cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi curr_fpr_size = fpr->ByteSize(this); 1236cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi DCHECK_EQ(curr_fpr_size % kPageSize, static_cast<size_t>(0)); 1237cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi remaining_curr_fpr_size = curr_fpr_size - kPageSize; 1238a5b5c55c8585b7ce915f0c7e1f66d121a7f7a078Mathieu Chartier stream << "[" << i << "]=" << (pm == kPageMapReleased ? "Released" : "Empty") 1239a5b5c55c8585b7ce915f0c7e1f66d121a7f7a078Mathieu Chartier << " (FPR start) fpr_size=" << curr_fpr_size 1240a4adbfd44032d70e166e6f18096bbbed05a990baHiroshi Yamauchi << " remaining_fpr_size=" << remaining_curr_fpr_size << std::endl; 1241cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi if (remaining_curr_fpr_size == 0) { 1242cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi // Reset at the end of the current free page run. 12432cebb24bfc3247d3e9be138a3350106737455918Mathieu Chartier curr_fpr = nullptr; 1244cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi curr_fpr_size = 0; 1245cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi } 1246a4adbfd44032d70e166e6f18096bbbed05a990baHiroshi Yamauchi stream << "curr_fpr=0x" << std::hex << reinterpret_cast<intptr_t>(curr_fpr) << std::endl; 1247cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi DCHECK_EQ(num_running_empty_pages, static_cast<size_t>(0)); 1248cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi } else { 1249cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi // Still part of the current free page run. 1250cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi DCHECK_NE(num_running_empty_pages, static_cast<size_t>(0)); 12512cebb24bfc3247d3e9be138a3350106737455918Mathieu Chartier DCHECK(curr_fpr != nullptr && curr_fpr_size > 0 && remaining_curr_fpr_size > 0); 1252cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi DCHECK_EQ(remaining_curr_fpr_size % kPageSize, static_cast<size_t>(0)); 1253cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi DCHECK_GE(remaining_curr_fpr_size, static_cast<size_t>(kPageSize)); 1254cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi remaining_curr_fpr_size -= kPageSize; 1255a4adbfd44032d70e166e6f18096bbbed05a990baHiroshi Yamauchi stream << "[" << i << "]=Empty (FPR part)" 1256a4adbfd44032d70e166e6f18096bbbed05a990baHiroshi Yamauchi << " remaining_fpr_size=" << remaining_curr_fpr_size << std::endl; 1257cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi if (remaining_curr_fpr_size == 0) { 1258cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi // Reset at the end of the current free page run. 12592cebb24bfc3247d3e9be138a3350106737455918Mathieu Chartier curr_fpr = nullptr; 1260cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi curr_fpr_size = 0; 1261cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi } 1262cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi } 1263cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi num_running_empty_pages++; 1264cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi break; 1265cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi } 1266cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi case kPageMapLargeObject: { 1267cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi DCHECK_EQ(remaining_curr_fpr_size, static_cast<size_t>(0)); 1268cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi num_running_empty_pages = 0; 1269a4adbfd44032d70e166e6f18096bbbed05a990baHiroshi Yamauchi stream << "[" << i << "]=Large (start)" << std::endl; 1270cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi break; 1271cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi } 1272cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi case kPageMapLargeObjectPart: 1273cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi DCHECK_EQ(remaining_curr_fpr_size, static_cast<size_t>(0)); 1274cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi num_running_empty_pages = 0; 1275a4adbfd44032d70e166e6f18096bbbed05a990baHiroshi Yamauchi stream << "[" << i << "]=Large (part)" << std::endl; 1276cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi break; 1277cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi case kPageMapRun: { 1278cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi DCHECK_EQ(remaining_curr_fpr_size, static_cast<size_t>(0)); 1279cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi num_running_empty_pages = 0; 1280cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi Run* run = reinterpret_cast<Run*>(base_ + i * kPageSize); 1281cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi size_t idx = run->size_bracket_idx_; 1282a4adbfd44032d70e166e6f18096bbbed05a990baHiroshi Yamauchi stream << "[" << i << "]=Run (start)" 1283a4adbfd44032d70e166e6f18096bbbed05a990baHiroshi Yamauchi << " idx=" << idx 1284a4adbfd44032d70e166e6f18096bbbed05a990baHiroshi Yamauchi << " numOfPages=" << numOfPages[idx] 128573d1e17b3afc7d5e56184f90bf819dc64956448aMathieu Chartier << " is_thread_local=" << run->is_thread_local_ 1286a4adbfd44032d70e166e6f18096bbbed05a990baHiroshi Yamauchi << " is_all_free=" << (run->IsAllFree() ? 1 : 0) 1287a4adbfd44032d70e166e6f18096bbbed05a990baHiroshi Yamauchi << std::endl; 1288cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi break; 1289cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi } 1290cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi case kPageMapRunPart: 1291cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi DCHECK_EQ(remaining_curr_fpr_size, static_cast<size_t>(0)); 1292cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi num_running_empty_pages = 0; 1293a4adbfd44032d70e166e6f18096bbbed05a990baHiroshi Yamauchi stream << "[" << i << "]=Run (part)" << std::endl; 1294cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi break; 1295cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi default: 1296a4adbfd44032d70e166e6f18096bbbed05a990baHiroshi Yamauchi stream << "[" << i << "]=Unrecognizable page map type: " << pm; 1297cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi break; 1298cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi } 1299cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi } 1300a4adbfd44032d70e166e6f18096bbbed05a990baHiroshi Yamauchi return stream.str(); 1301cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi} 1302cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi 1303d7576328811e5103e99d31f834a857522cc1463fAndreas Gampesize_t RosAlloc::UsableSize(const void* ptr) { 13045d057056db1923947ba846b391d981759b15714aIan Rogers DCHECK_LE(base_, ptr); 13055d057056db1923947ba846b391d981759b15714aIan Rogers DCHECK_LT(ptr, base_ + footprint_); 1306cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi size_t pm_idx = RoundDownToPageMapIndex(ptr); 1307cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi MutexLock mu(Thread::Current(), lock_); 1308cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi switch (page_map_[pm_idx]) { 1309a5b5c55c8585b7ce915f0c7e1f66d121a7f7a078Mathieu Chartier case kPageMapReleased: 1310a5b5c55c8585b7ce915f0c7e1f66d121a7f7a078Mathieu Chartier // Fall-through. 1311a5b5c55c8585b7ce915f0c7e1f66d121a7f7a078Mathieu Chartier case kPageMapEmpty: 1312a5b5c55c8585b7ce915f0c7e1f66d121a7f7a078Mathieu Chartier LOG(FATAL) << "Unreachable - " << __PRETTY_FUNCTION__ << ": pm_idx=" << pm_idx << ", ptr=" 1313a5b5c55c8585b7ce915f0c7e1f66d121a7f7a078Mathieu Chartier << std::hex << reinterpret_cast<intptr_t>(ptr); 1314a5b5c55c8585b7ce915f0c7e1f66d121a7f7a078Mathieu Chartier break; 1315a5b5c55c8585b7ce915f0c7e1f66d121a7f7a078Mathieu Chartier case kPageMapLargeObject: { 1316a5b5c55c8585b7ce915f0c7e1f66d121a7f7a078Mathieu Chartier size_t num_pages = 1; 1317a5b5c55c8585b7ce915f0c7e1f66d121a7f7a078Mathieu Chartier size_t idx = pm_idx + 1; 1318a5b5c55c8585b7ce915f0c7e1f66d121a7f7a078Mathieu Chartier size_t end = page_map_size_; 1319a5b5c55c8585b7ce915f0c7e1f66d121a7f7a078Mathieu Chartier while (idx < end && page_map_[idx] == kPageMapLargeObjectPart) { 1320a5b5c55c8585b7ce915f0c7e1f66d121a7f7a078Mathieu Chartier num_pages++; 1321a5b5c55c8585b7ce915f0c7e1f66d121a7f7a078Mathieu Chartier idx++; 1322a5b5c55c8585b7ce915f0c7e1f66d121a7f7a078Mathieu Chartier } 1323a5b5c55c8585b7ce915f0c7e1f66d121a7f7a078Mathieu Chartier return num_pages * kPageSize; 1324a5b5c55c8585b7ce915f0c7e1f66d121a7f7a078Mathieu Chartier } 1325a5b5c55c8585b7ce915f0c7e1f66d121a7f7a078Mathieu Chartier case kPageMapLargeObjectPart: 1326a5b5c55c8585b7ce915f0c7e1f66d121a7f7a078Mathieu Chartier LOG(FATAL) << "Unreachable - " << __PRETTY_FUNCTION__ << ": pm_idx=" << pm_idx << ", ptr=" 1327a5b5c55c8585b7ce915f0c7e1f66d121a7f7a078Mathieu Chartier << std::hex << reinterpret_cast<intptr_t>(ptr); 1328a5b5c55c8585b7ce915f0c7e1f66d121a7f7a078Mathieu Chartier break; 1329a5b5c55c8585b7ce915f0c7e1f66d121a7f7a078Mathieu Chartier case kPageMapRun: 1330a5b5c55c8585b7ce915f0c7e1f66d121a7f7a078Mathieu Chartier case kPageMapRunPart: { 1331a5b5c55c8585b7ce915f0c7e1f66d121a7f7a078Mathieu Chartier // Find the beginning of the run. 1332a5b5c55c8585b7ce915f0c7e1f66d121a7f7a078Mathieu Chartier while (page_map_[pm_idx] != kPageMapRun) { 1333a5b5c55c8585b7ce915f0c7e1f66d121a7f7a078Mathieu Chartier pm_idx--; 1334a5b5c55c8585b7ce915f0c7e1f66d121a7f7a078Mathieu Chartier DCHECK_LT(pm_idx, capacity_ / kPageSize); 1335a5b5c55c8585b7ce915f0c7e1f66d121a7f7a078Mathieu Chartier } 1336a5b5c55c8585b7ce915f0c7e1f66d121a7f7a078Mathieu Chartier DCHECK_EQ(page_map_[pm_idx], kPageMapRun); 1337a5b5c55c8585b7ce915f0c7e1f66d121a7f7a078Mathieu Chartier Run* run = reinterpret_cast<Run*>(base_ + pm_idx * kPageSize); 1338a5b5c55c8585b7ce915f0c7e1f66d121a7f7a078Mathieu Chartier DCHECK_EQ(run->magic_num_, kMagicNum); 1339a5b5c55c8585b7ce915f0c7e1f66d121a7f7a078Mathieu Chartier size_t idx = run->size_bracket_idx_; 1340d7576328811e5103e99d31f834a857522cc1463fAndreas Gampe size_t offset_from_slot_base = reinterpret_cast<const uint8_t*>(ptr) 134113735955f39b3b304c37d2b2840663c131262c18Ian Rogers - (reinterpret_cast<uint8_t*>(run) + headerSizes[idx]); 1342a5b5c55c8585b7ce915f0c7e1f66d121a7f7a078Mathieu Chartier DCHECK_EQ(offset_from_slot_base % bracketSizes[idx], static_cast<size_t>(0)); 1343a5b5c55c8585b7ce915f0c7e1f66d121a7f7a078Mathieu Chartier return IndexToBracketSize(idx); 1344a5b5c55c8585b7ce915f0c7e1f66d121a7f7a078Mathieu Chartier } 1345a5b5c55c8585b7ce915f0c7e1f66d121a7f7a078Mathieu Chartier default: { 13462fdeecb890a353d3f17407cc1cb015e0a65c2220Maxim Kazantsev LOG(FATAL) << "Unreachable - page map type: " << static_cast<int>(page_map_[pm_idx]); 1347a5b5c55c8585b7ce915f0c7e1f66d121a7f7a078Mathieu Chartier break; 1348a5b5c55c8585b7ce915f0c7e1f66d121a7f7a078Mathieu Chartier } 1349cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi } 1350cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi return 0; 1351cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi} 1352cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi 1353cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchibool RosAlloc::Trim() { 1354cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi MutexLock mu(Thread::Current(), lock_); 1355cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi FreePageRun* last_free_page_run; 1356cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi DCHECK_EQ(footprint_ % kPageSize, static_cast<size_t>(0)); 1357cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi auto it = free_page_runs_.rbegin(); 1358cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi if (it != free_page_runs_.rend() && (last_free_page_run = *it)->End(this) == base_ + footprint_) { 1359cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi // Remove the last free page run, if any. 1360cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi DCHECK(last_free_page_run->IsFree()); 1361a5b5c55c8585b7ce915f0c7e1f66d121a7f7a078Mathieu Chartier DCHECK(IsFreePage(ToPageMapIndex(last_free_page_run))); 1362cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi DCHECK_EQ(last_free_page_run->ByteSize(this) % kPageSize, static_cast<size_t>(0)); 1363cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi DCHECK_EQ(last_free_page_run->End(this), base_ + footprint_); 1364cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi free_page_runs_.erase(last_free_page_run); 1365cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi size_t decrement = last_free_page_run->ByteSize(this); 1366cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi size_t new_footprint = footprint_ - decrement; 1367cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi DCHECK_EQ(new_footprint % kPageSize, static_cast<size_t>(0)); 1368cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi size_t new_num_of_pages = new_footprint / kPageSize; 136926d69ffc0ebc98fbc5f316d8cd3ee6ba5b2001acHiroshi Yamauchi DCHECK_GE(page_map_size_, new_num_of_pages); 137026d69ffc0ebc98fbc5f316d8cd3ee6ba5b2001acHiroshi Yamauchi // Zero out the tail of the page map. 137113735955f39b3b304c37d2b2840663c131262c18Ian Rogers uint8_t* zero_begin = const_cast<uint8_t*>(page_map_) + new_num_of_pages; 137213735955f39b3b304c37d2b2840663c131262c18Ian Rogers uint8_t* madvise_begin = AlignUp(zero_begin, kPageSize); 137326d69ffc0ebc98fbc5f316d8cd3ee6ba5b2001acHiroshi Yamauchi DCHECK_LE(madvise_begin, page_map_mem_map_->End()); 137426d69ffc0ebc98fbc5f316d8cd3ee6ba5b2001acHiroshi Yamauchi size_t madvise_size = page_map_mem_map_->End() - madvise_begin; 137526d69ffc0ebc98fbc5f316d8cd3ee6ba5b2001acHiroshi Yamauchi if (madvise_size > 0) { 137626d69ffc0ebc98fbc5f316d8cd3ee6ba5b2001acHiroshi Yamauchi DCHECK_ALIGNED(madvise_begin, kPageSize); 137726d69ffc0ebc98fbc5f316d8cd3ee6ba5b2001acHiroshi Yamauchi DCHECK_EQ(RoundUp(madvise_size, kPageSize), madvise_size); 1378c5f17732d8144491c642776b6b48c85dfadf4b52Ian Rogers if (!kMadviseZeroes) { 1379c5f17732d8144491c642776b6b48c85dfadf4b52Ian Rogers memset(madvise_begin, 0, madvise_size); 1380c5f17732d8144491c642776b6b48c85dfadf4b52Ian Rogers } 138126d69ffc0ebc98fbc5f316d8cd3ee6ba5b2001acHiroshi Yamauchi CHECK_EQ(madvise(madvise_begin, madvise_size, MADV_DONTNEED), 0); 138226d69ffc0ebc98fbc5f316d8cd3ee6ba5b2001acHiroshi Yamauchi } 138326d69ffc0ebc98fbc5f316d8cd3ee6ba5b2001acHiroshi Yamauchi if (madvise_begin - zero_begin) { 138426d69ffc0ebc98fbc5f316d8cd3ee6ba5b2001acHiroshi Yamauchi memset(zero_begin, 0, madvise_begin - zero_begin); 138526d69ffc0ebc98fbc5f316d8cd3ee6ba5b2001acHiroshi Yamauchi } 138626d69ffc0ebc98fbc5f316d8cd3ee6ba5b2001acHiroshi Yamauchi page_map_size_ = new_num_of_pages; 1387cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi free_page_run_size_map_.resize(new_num_of_pages); 1388cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi DCHECK_EQ(free_page_run_size_map_.size(), new_num_of_pages); 1389277ccbd200ea43590dfc06a93ae184a765327ad0Andreas Gampe ArtRosAllocMoreCore(this, -(static_cast<intptr_t>(decrement))); 1390cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi if (kTraceRosAlloc) { 1391cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi LOG(INFO) << "RosAlloc::Trim() : decreased the footprint from " 1392cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi << footprint_ << " to " << new_footprint; 1393cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi } 1394cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi DCHECK_LT(new_footprint, footprint_); 1395cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi DCHECK_LT(new_footprint, capacity_); 1396cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi footprint_ = new_footprint; 1397cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi return true; 1398cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi } 1399cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi return false; 1400cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi} 1401cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi 1402cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchivoid RosAlloc::InspectAll(void (*handler)(void* start, void* end, size_t used_bytes, void* callback_arg), 1403cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi void* arg) { 1404cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi // Note: no need to use this to release pages as we already do so in FreePages(). 14052cebb24bfc3247d3e9be138a3350106737455918Mathieu Chartier if (handler == nullptr) { 1406cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi return; 1407cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi } 1408cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi MutexLock mu(Thread::Current(), lock_); 140926d69ffc0ebc98fbc5f316d8cd3ee6ba5b2001acHiroshi Yamauchi size_t pm_end = page_map_size_; 1410cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi size_t i = 0; 1411cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi while (i < pm_end) { 141213735955f39b3b304c37d2b2840663c131262c18Ian Rogers uint8_t pm = page_map_[i]; 1413cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi switch (pm) { 1414a5b5c55c8585b7ce915f0c7e1f66d121a7f7a078Mathieu Chartier case kPageMapReleased: 1415a5b5c55c8585b7ce915f0c7e1f66d121a7f7a078Mathieu Chartier // Fall-through. 1416cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi case kPageMapEmpty: { 1417cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi // The start of a free page run. 1418cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi FreePageRun* fpr = reinterpret_cast<FreePageRun*>(base_ + i * kPageSize); 1419cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi DCHECK(free_page_runs_.find(fpr) != free_page_runs_.end()); 1420cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi size_t fpr_size = fpr->ByteSize(this); 142114d90579f013b374638b599361970557ed4b3f09Roland Levillain DCHECK_ALIGNED(fpr_size, kPageSize); 1422cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi void* start = fpr; 1423573f7d2d68e1838a0485e6b40d90c967526e00c2Hiroshi Yamauchi if (kIsDebugBuild) { 1424573f7d2d68e1838a0485e6b40d90c967526e00c2Hiroshi Yamauchi // In the debug build, the first page of a free page run 1425573f7d2d68e1838a0485e6b40d90c967526e00c2Hiroshi Yamauchi // contains a magic number for debugging. Exclude it. 142613735955f39b3b304c37d2b2840663c131262c18Ian Rogers start = reinterpret_cast<uint8_t*>(fpr) + kPageSize; 1427573f7d2d68e1838a0485e6b40d90c967526e00c2Hiroshi Yamauchi } 142813735955f39b3b304c37d2b2840663c131262c18Ian Rogers void* end = reinterpret_cast<uint8_t*>(fpr) + fpr_size; 1429cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi handler(start, end, 0, arg); 1430cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi size_t num_pages = fpr_size / kPageSize; 1431cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi if (kIsDebugBuild) { 1432cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi for (size_t j = i + 1; j < i + num_pages; ++j) { 1433a5b5c55c8585b7ce915f0c7e1f66d121a7f7a078Mathieu Chartier DCHECK(IsFreePage(j)); 1434cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi } 1435cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi } 1436cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi i += fpr_size / kPageSize; 1437cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi DCHECK_LE(i, pm_end); 1438cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi break; 1439cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi } 1440cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi case kPageMapLargeObject: { 1441cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi // The start of a large object. 1442cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi size_t num_pages = 1; 1443cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi size_t idx = i + 1; 1444cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi while (idx < pm_end && page_map_[idx] == kPageMapLargeObjectPart) { 1445cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi num_pages++; 1446cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi idx++; 1447cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi } 1448cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi void* start = base_ + i * kPageSize; 1449cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi void* end = base_ + (i + num_pages) * kPageSize; 1450cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi size_t used_bytes = num_pages * kPageSize; 1451cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi handler(start, end, used_bytes, arg); 1452cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi if (kIsDebugBuild) { 1453cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi for (size_t j = i + 1; j < i + num_pages; ++j) { 1454cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi DCHECK_EQ(page_map_[j], kPageMapLargeObjectPart); 1455cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi } 1456cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi } 1457cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi i += num_pages; 1458cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi DCHECK_LE(i, pm_end); 1459cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi break; 1460cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi } 1461cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi case kPageMapLargeObjectPart: 14622fdeecb890a353d3f17407cc1cb015e0a65c2220Maxim Kazantsev LOG(FATAL) << "Unreachable - page map type: " << static_cast<int>(pm); 1463cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi break; 1464cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi case kPageMapRun: { 1465cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi // The start of a run. 1466cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi Run* run = reinterpret_cast<Run*>(base_ + i * kPageSize); 14675d057056db1923947ba846b391d981759b15714aIan Rogers DCHECK_EQ(run->magic_num_, kMagicNum); 146873d1e17b3afc7d5e56184f90bf819dc64956448aMathieu Chartier // The dedicated full run doesn't contain any real allocations, don't visit the slots in 146973d1e17b3afc7d5e56184f90bf819dc64956448aMathieu Chartier // there. 1470cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi run->InspectAllSlots(handler, arg); 1471cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi size_t num_pages = numOfPages[run->size_bracket_idx_]; 1472cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi if (kIsDebugBuild) { 1473cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi for (size_t j = i + 1; j < i + num_pages; ++j) { 1474cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi DCHECK_EQ(page_map_[j], kPageMapRunPart); 1475cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi } 1476cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi } 1477cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi i += num_pages; 1478cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi DCHECK_LE(i, pm_end); 1479cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi break; 1480cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi } 1481cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi case kPageMapRunPart: 14822fdeecb890a353d3f17407cc1cb015e0a65c2220Maxim Kazantsev LOG(FATAL) << "Unreachable - page map type: " << static_cast<int>(pm); 1483cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi break; 1484cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi default: 14852fdeecb890a353d3f17407cc1cb015e0a65c2220Maxim Kazantsev LOG(FATAL) << "Unreachable - page map type: " << static_cast<int>(pm); 1486cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi break; 1487cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi } 1488cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi } 1489cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi} 1490cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi 1491cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchisize_t RosAlloc::Footprint() { 1492cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi MutexLock mu(Thread::Current(), lock_); 1493cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi return footprint_; 1494cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi} 1495cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi 1496cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchisize_t RosAlloc::FootprintLimit() { 1497cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi MutexLock mu(Thread::Current(), lock_); 1498cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi return capacity_; 1499cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi} 1500cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi 1501cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchivoid RosAlloc::SetFootprintLimit(size_t new_capacity) { 1502cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi MutexLock mu(Thread::Current(), lock_); 1503cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi DCHECK_EQ(RoundUp(new_capacity, kPageSize), new_capacity); 1504cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi // Only growing is supported here. But Trim() is supported. 1505cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi if (capacity_ < new_capacity) { 150626d69ffc0ebc98fbc5f316d8cd3ee6ba5b2001acHiroshi Yamauchi CHECK_LE(new_capacity, max_capacity_); 1507cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi capacity_ = new_capacity; 1508cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi VLOG(heap) << "new capacity=" << capacity_; 1509cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi } 1510cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi} 1511cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi 1512578462103b548484e45079a749cb684f6a7e635cLei Li// Below may be called by mutator itself just before thread termination. 15134460a84be92b5a94ecfb5c650aef4945ab849c93Hiroshi Yamauchisize_t RosAlloc::RevokeThreadLocalRuns(Thread* thread) { 1514cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi Thread* self = Thread::Current(); 15154460a84be92b5a94ecfb5c650aef4945ab849c93Hiroshi Yamauchi size_t free_bytes = 0U; 15160651d41e41341fb2e9ef3ee41dc1f1bfc832dbbbMathieu Chartier for (size_t idx = 0; idx < kNumThreadLocalSizeBrackets; idx++) { 1517cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi MutexLock mu(self, *size_bracket_locks_[idx]); 1518dd7624d2b9e599d57762d12031b10b89defc9807Ian Rogers Run* thread_local_run = reinterpret_cast<Run*>(thread->GetRosAllocRun(idx)); 151973d1e17b3afc7d5e56184f90bf819dc64956448aMathieu Chartier CHECK(thread_local_run != nullptr); 152073d1e17b3afc7d5e56184f90bf819dc64956448aMathieu Chartier // Invalid means already revoked. 152173d1e17b3afc7d5e56184f90bf819dc64956448aMathieu Chartier DCHECK(thread_local_run->IsThreadLocal()); 152273d1e17b3afc7d5e56184f90bf819dc64956448aMathieu Chartier if (thread_local_run != dedicated_full_run_) { 15234460a84be92b5a94ecfb5c650aef4945ab849c93Hiroshi Yamauchi // Note the thread local run may not be full here. 152473d1e17b3afc7d5e56184f90bf819dc64956448aMathieu Chartier thread->SetRosAllocRun(idx, dedicated_full_run_); 1525cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi DCHECK_EQ(thread_local_run->magic_num_, kMagicNum); 15264460a84be92b5a94ecfb5c650aef4945ab849c93Hiroshi Yamauchi // Count the number of free slots left. 15274460a84be92b5a94ecfb5c650aef4945ab849c93Hiroshi Yamauchi size_t num_free_slots = thread_local_run->NumberOfFreeSlots(); 15284460a84be92b5a94ecfb5c650aef4945ab849c93Hiroshi Yamauchi free_bytes += num_free_slots * bracketSizes[idx]; 1529578462103b548484e45079a749cb684f6a7e635cLei Li // The above bracket index lock guards thread local free list to avoid race condition 1530578462103b548484e45079a749cb684f6a7e635cLei Li // with unioning bulk free list to thread local free list by GC thread in BulkFree. 1531578462103b548484e45079a749cb684f6a7e635cLei Li // If thread local run is true, GC thread will help update thread local free list 1532578462103b548484e45079a749cb684f6a7e635cLei Li // in BulkFree. And the latest thread local free list will be merged to free list 1533578462103b548484e45079a749cb684f6a7e635cLei Li // either when this thread local run is full or when revoking this run here. In this 1534578462103b548484e45079a749cb684f6a7e635cLei Li // case the free list wll be updated. If thread local run is false, GC thread will help 1535578462103b548484e45079a749cb684f6a7e635cLei Li // merge bulk free list in next BulkFree. 1536578462103b548484e45079a749cb684f6a7e635cLei Li // Thus no need to merge bulk free list to free list again here. 1537cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi bool dont_care; 153831bf42c48c4d00f0677c31264bba8d21618dae67Hiroshi Yamauchi thread_local_run->MergeThreadLocalFreeListToFreeList(&dont_care); 153973d1e17b3afc7d5e56184f90bf819dc64956448aMathieu Chartier thread_local_run->SetIsThreadLocal(false); 1540cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi DCHECK(non_full_runs_[idx].find(thread_local_run) == non_full_runs_[idx].end()); 1541cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi DCHECK(full_runs_[idx].find(thread_local_run) == full_runs_[idx].end()); 15420651d41e41341fb2e9ef3ee41dc1f1bfc832dbbbMathieu Chartier RevokeRun(self, idx, thread_local_run); 15430651d41e41341fb2e9ef3ee41dc1f1bfc832dbbbMathieu Chartier } 15440651d41e41341fb2e9ef3ee41dc1f1bfc832dbbbMathieu Chartier } 15454460a84be92b5a94ecfb5c650aef4945ab849c93Hiroshi Yamauchi return free_bytes; 15460651d41e41341fb2e9ef3ee41dc1f1bfc832dbbbMathieu Chartier} 15470651d41e41341fb2e9ef3ee41dc1f1bfc832dbbbMathieu Chartier 15480651d41e41341fb2e9ef3ee41dc1f1bfc832dbbbMathieu Chartiervoid RosAlloc::RevokeRun(Thread* self, size_t idx, Run* run) { 15490651d41e41341fb2e9ef3ee41dc1f1bfc832dbbbMathieu Chartier size_bracket_locks_[idx]->AssertHeld(self); 15500651d41e41341fb2e9ef3ee41dc1f1bfc832dbbbMathieu Chartier DCHECK(run != dedicated_full_run_); 15510651d41e41341fb2e9ef3ee41dc1f1bfc832dbbbMathieu Chartier if (run->IsFull()) { 15520651d41e41341fb2e9ef3ee41dc1f1bfc832dbbbMathieu Chartier if (kIsDebugBuild) { 15530651d41e41341fb2e9ef3ee41dc1f1bfc832dbbbMathieu Chartier full_runs_[idx].insert(run); 15540651d41e41341fb2e9ef3ee41dc1f1bfc832dbbbMathieu Chartier DCHECK(full_runs_[idx].find(run) != full_runs_[idx].end()); 15550651d41e41341fb2e9ef3ee41dc1f1bfc832dbbbMathieu Chartier if (kTraceRosAlloc) { 1556a5b5c55c8585b7ce915f0c7e1f66d121a7f7a078Mathieu Chartier LOG(INFO) << __PRETTY_FUNCTION__ << " : Inserted run 0x" << std::hex 15570651d41e41341fb2e9ef3ee41dc1f1bfc832dbbbMathieu Chartier << reinterpret_cast<intptr_t>(run) 15580651d41e41341fb2e9ef3ee41dc1f1bfc832dbbbMathieu Chartier << " into full_runs_[" << std::dec << idx << "]"; 1559cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi } 1560cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi } 15610651d41e41341fb2e9ef3ee41dc1f1bfc832dbbbMathieu Chartier } else if (run->IsAllFree()) { 156231bf42c48c4d00f0677c31264bba8d21618dae67Hiroshi Yamauchi run->ZeroHeaderAndSlotHeaders(); 15630651d41e41341fb2e9ef3ee41dc1f1bfc832dbbbMathieu Chartier MutexLock mu(self, lock_); 15640651d41e41341fb2e9ef3ee41dc1f1bfc832dbbbMathieu Chartier FreePages(self, run, true); 15650651d41e41341fb2e9ef3ee41dc1f1bfc832dbbbMathieu Chartier } else { 15660651d41e41341fb2e9ef3ee41dc1f1bfc832dbbbMathieu Chartier non_full_runs_[idx].insert(run); 15670651d41e41341fb2e9ef3ee41dc1f1bfc832dbbbMathieu Chartier DCHECK(non_full_runs_[idx].find(run) != non_full_runs_[idx].end()); 15680651d41e41341fb2e9ef3ee41dc1f1bfc832dbbbMathieu Chartier if (kTraceRosAlloc) { 1569a5b5c55c8585b7ce915f0c7e1f66d121a7f7a078Mathieu Chartier LOG(INFO) << __PRETTY_FUNCTION__ << " : Inserted run 0x" << std::hex 15700651d41e41341fb2e9ef3ee41dc1f1bfc832dbbbMathieu Chartier << reinterpret_cast<intptr_t>(run) 15710651d41e41341fb2e9ef3ee41dc1f1bfc832dbbbMathieu Chartier << " into non_full_runs_[" << std::dec << idx << "]"; 15720651d41e41341fb2e9ef3ee41dc1f1bfc832dbbbMathieu Chartier } 15730651d41e41341fb2e9ef3ee41dc1f1bfc832dbbbMathieu Chartier } 15740651d41e41341fb2e9ef3ee41dc1f1bfc832dbbbMathieu Chartier} 15750651d41e41341fb2e9ef3ee41dc1f1bfc832dbbbMathieu Chartier 15760651d41e41341fb2e9ef3ee41dc1f1bfc832dbbbMathieu Chartiervoid RosAlloc::RevokeThreadUnsafeCurrentRuns() { 15770651d41e41341fb2e9ef3ee41dc1f1bfc832dbbbMathieu Chartier // Revoke the current runs which share the same idx as thread local runs. 15780651d41e41341fb2e9ef3ee41dc1f1bfc832dbbbMathieu Chartier Thread* self = Thread::Current(); 15790651d41e41341fb2e9ef3ee41dc1f1bfc832dbbbMathieu Chartier for (size_t idx = 0; idx < kNumThreadLocalSizeBrackets; ++idx) { 15800651d41e41341fb2e9ef3ee41dc1f1bfc832dbbbMathieu Chartier MutexLock mu(self, *size_bracket_locks_[idx]); 15810651d41e41341fb2e9ef3ee41dc1f1bfc832dbbbMathieu Chartier if (current_runs_[idx] != dedicated_full_run_) { 15820651d41e41341fb2e9ef3ee41dc1f1bfc832dbbbMathieu Chartier RevokeRun(self, idx, current_runs_[idx]); 15830651d41e41341fb2e9ef3ee41dc1f1bfc832dbbbMathieu Chartier current_runs_[idx] = dedicated_full_run_; 15840651d41e41341fb2e9ef3ee41dc1f1bfc832dbbbMathieu Chartier } 1585cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi } 1586cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi} 1587cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi 15884460a84be92b5a94ecfb5c650aef4945ab849c93Hiroshi Yamauchisize_t RosAlloc::RevokeAllThreadLocalRuns() { 1589cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi // This is called when a mutator thread won't allocate such as at 1590cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi // the Zygote creation time or during the GC pause. 1591f5b0e20b5b31f5f5465784adcf2a204dcd69c7fdHiroshi Yamauchi MutexLock mu(Thread::Current(), *Locks::runtime_shutdown_lock_); 1592f5b0e20b5b31f5f5465784adcf2a204dcd69c7fdHiroshi Yamauchi MutexLock mu2(Thread::Current(), *Locks::thread_list_lock_); 1593cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi std::list<Thread*> thread_list = Runtime::Current()->GetThreadList()->GetList(); 15944460a84be92b5a94ecfb5c650aef4945ab849c93Hiroshi Yamauchi size_t free_bytes = 0U; 159573d1e17b3afc7d5e56184f90bf819dc64956448aMathieu Chartier for (Thread* thread : thread_list) { 15964460a84be92b5a94ecfb5c650aef4945ab849c93Hiroshi Yamauchi free_bytes += RevokeThreadLocalRuns(thread); 1597cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi } 15980651d41e41341fb2e9ef3ee41dc1f1bfc832dbbbMathieu Chartier RevokeThreadUnsafeCurrentRuns(); 15994460a84be92b5a94ecfb5c650aef4945ab849c93Hiroshi Yamauchi return free_bytes; 1600cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi} 1601cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi 1602c93c530efc175954160c3834c93961a1a946a35aHiroshi Yamauchivoid RosAlloc::AssertThreadLocalRunsAreRevoked(Thread* thread) { 1603c93c530efc175954160c3834c93961a1a946a35aHiroshi Yamauchi if (kIsDebugBuild) { 1604c93c530efc175954160c3834c93961a1a946a35aHiroshi Yamauchi Thread* self = Thread::Current(); 1605c93c530efc175954160c3834c93961a1a946a35aHiroshi Yamauchi // Avoid race conditions on the bulk free bit maps with BulkFree() (GC). 1606a1c1c71e24c93a720bbf13de129c75a9a0bde37aMathieu Chartier ReaderMutexLock wmu(self, bulk_free_lock_); 16070651d41e41341fb2e9ef3ee41dc1f1bfc832dbbbMathieu Chartier for (size_t idx = 0; idx < kNumThreadLocalSizeBrackets; idx++) { 1608c93c530efc175954160c3834c93961a1a946a35aHiroshi Yamauchi MutexLock mu(self, *size_bracket_locks_[idx]); 1609dd7624d2b9e599d57762d12031b10b89defc9807Ian Rogers Run* thread_local_run = reinterpret_cast<Run*>(thread->GetRosAllocRun(idx)); 161073d1e17b3afc7d5e56184f90bf819dc64956448aMathieu Chartier DCHECK(thread_local_run == nullptr || thread_local_run == dedicated_full_run_); 1611c93c530efc175954160c3834c93961a1a946a35aHiroshi Yamauchi } 1612c93c530efc175954160c3834c93961a1a946a35aHiroshi Yamauchi } 1613c93c530efc175954160c3834c93961a1a946a35aHiroshi Yamauchi} 1614c93c530efc175954160c3834c93961a1a946a35aHiroshi Yamauchi 1615c93c530efc175954160c3834c93961a1a946a35aHiroshi Yamauchivoid RosAlloc::AssertAllThreadLocalRunsAreRevoked() { 1616c93c530efc175954160c3834c93961a1a946a35aHiroshi Yamauchi if (kIsDebugBuild) { 16170651d41e41341fb2e9ef3ee41dc1f1bfc832dbbbMathieu Chartier Thread* self = Thread::Current(); 1618277ccbd200ea43590dfc06a93ae184a765327ad0Andreas Gampe MutexLock shutdown_mu(self, *Locks::runtime_shutdown_lock_); 1619277ccbd200ea43590dfc06a93ae184a765327ad0Andreas Gampe MutexLock thread_list_mu(self, *Locks::thread_list_lock_); 1620c93c530efc175954160c3834c93961a1a946a35aHiroshi Yamauchi std::list<Thread*> thread_list = Runtime::Current()->GetThreadList()->GetList(); 1621c93c530efc175954160c3834c93961a1a946a35aHiroshi Yamauchi for (Thread* t : thread_list) { 1622c93c530efc175954160c3834c93961a1a946a35aHiroshi Yamauchi AssertThreadLocalRunsAreRevoked(t); 1623c93c530efc175954160c3834c93961a1a946a35aHiroshi Yamauchi } 16240651d41e41341fb2e9ef3ee41dc1f1bfc832dbbbMathieu Chartier for (size_t idx = 0; idx < kNumThreadLocalSizeBrackets; ++idx) { 1625277ccbd200ea43590dfc06a93ae184a765327ad0Andreas Gampe MutexLock brackets_mu(self, *size_bracket_locks_[idx]); 16260651d41e41341fb2e9ef3ee41dc1f1bfc832dbbbMathieu Chartier CHECK_EQ(current_runs_[idx], dedicated_full_run_); 16270651d41e41341fb2e9ef3ee41dc1f1bfc832dbbbMathieu Chartier } 1628c93c530efc175954160c3834c93961a1a946a35aHiroshi Yamauchi } 1629c93c530efc175954160c3834c93961a1a946a35aHiroshi Yamauchi} 1630c93c530efc175954160c3834c93961a1a946a35aHiroshi Yamauchi 1631cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchivoid RosAlloc::Initialize() { 1632cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi // bracketSizes. 16337ed9c561048d79083b6d0576c71a986a3123bca6Hiroshi Yamauchi static_assert(kNumRegularSizeBrackets == kNumOfSizeBrackets - 2, 16347ed9c561048d79083b6d0576c71a986a3123bca6Hiroshi Yamauchi "There should be two non-regular brackets"); 1635cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi for (size_t i = 0; i < kNumOfSizeBrackets; i++) { 16367ed9c561048d79083b6d0576c71a986a3123bca6Hiroshi Yamauchi if (i < kNumThreadLocalSizeBrackets) { 16377ed9c561048d79083b6d0576c71a986a3123bca6Hiroshi Yamauchi bracketSizes[i] = kThreadLocalBracketQuantumSize * (i + 1); 16387ed9c561048d79083b6d0576c71a986a3123bca6Hiroshi Yamauchi } else if (i < kNumRegularSizeBrackets) { 16397ed9c561048d79083b6d0576c71a986a3123bca6Hiroshi Yamauchi bracketSizes[i] = kBracketQuantumSize * (i - kNumThreadLocalSizeBrackets + 1) + 16407ed9c561048d79083b6d0576c71a986a3123bca6Hiroshi Yamauchi (kThreadLocalBracketQuantumSize * kNumThreadLocalSizeBrackets); 1641cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi } else if (i == kNumOfSizeBrackets - 2) { 1642cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi bracketSizes[i] = 1 * KB; 1643cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi } else { 16445d057056db1923947ba846b391d981759b15714aIan Rogers DCHECK_EQ(i, kNumOfSizeBrackets - 1); 1645cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi bracketSizes[i] = 2 * KB; 1646cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi } 1647cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi if (kTraceRosAlloc) { 1648cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi LOG(INFO) << "bracketSizes[" << i << "]=" << bracketSizes[i]; 1649cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi } 1650cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi } 1651cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi // numOfPages. 1652cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi for (size_t i = 0; i < kNumOfSizeBrackets; i++) { 16537ed9c561048d79083b6d0576c71a986a3123bca6Hiroshi Yamauchi if (i < kNumThreadLocalSizeBrackets) { 1654cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi numOfPages[i] = 1; 16557ed9c561048d79083b6d0576c71a986a3123bca6Hiroshi Yamauchi } else if (i < (kNumThreadLocalSizeBrackets + kNumRegularSizeBrackets) / 2) { 1656a16ff3c138747fc7ff62e11f1c85a3698c966559Hiroshi Yamauchi numOfPages[i] = 1; 16577ed9c561048d79083b6d0576c71a986a3123bca6Hiroshi Yamauchi } else if (i < kNumRegularSizeBrackets) { 1658a16ff3c138747fc7ff62e11f1c85a3698c966559Hiroshi Yamauchi numOfPages[i] = 1; 16597ed9c561048d79083b6d0576c71a986a3123bca6Hiroshi Yamauchi } else if (i == kNumOfSizeBrackets - 2) { 1660a16ff3c138747fc7ff62e11f1c85a3698c966559Hiroshi Yamauchi numOfPages[i] = 2; 1661cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi } else { 16625d057056db1923947ba846b391d981759b15714aIan Rogers DCHECK_EQ(i, kNumOfSizeBrackets - 1); 1663a16ff3c138747fc7ff62e11f1c85a3698c966559Hiroshi Yamauchi numOfPages[i] = 4; 1664cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi } 1665cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi if (kTraceRosAlloc) { 1666cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi LOG(INFO) << "numOfPages[" << i << "]=" << numOfPages[i]; 1667cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi } 1668cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi } 1669cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi // Compute numOfSlots and slotOffsets. 1670cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi for (size_t i = 0; i < kNumOfSizeBrackets; i++) { 1671cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi size_t bracket_size = bracketSizes[i]; 1672cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi size_t run_size = kPageSize * numOfPages[i]; 1673cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi size_t max_num_of_slots = run_size / bracket_size; 1674cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi // Compute the actual number of slots by taking the header and 1675cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi // alignment into account. 167631bf42c48c4d00f0677c31264bba8d21618dae67Hiroshi Yamauchi size_t fixed_header_size = RoundUp(Run::fixed_header_size(), sizeof(uint64_t)); 167731bf42c48c4d00f0677c31264bba8d21618dae67Hiroshi Yamauchi DCHECK_EQ(fixed_header_size, 80U); 1678cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi size_t header_size = 0; 1679cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi size_t num_of_slots = 0; 1680cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi // Search for the maximum number of slots that allows enough space 168131bf42c48c4d00f0677c31264bba8d21618dae67Hiroshi Yamauchi // for the header. 1682cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi for (int s = max_num_of_slots; s >= 0; s--) { 1683cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi size_t tmp_slots_size = bracket_size * s; 168431bf42c48c4d00f0677c31264bba8d21618dae67Hiroshi Yamauchi size_t tmp_unaligned_header_size = fixed_header_size; 1685cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi // Align up the unaligned header size. bracket_size may not be a power of two. 1686cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi size_t tmp_header_size = (tmp_unaligned_header_size % bracket_size == 0) ? 1687cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi tmp_unaligned_header_size : 1688cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi tmp_unaligned_header_size + (bracket_size - tmp_unaligned_header_size % bracket_size); 16897ed9c561048d79083b6d0576c71a986a3123bca6Hiroshi Yamauchi DCHECK_EQ(tmp_header_size % bracket_size, 0U); 16907ed9c561048d79083b6d0576c71a986a3123bca6Hiroshi Yamauchi DCHECK_EQ(tmp_header_size % sizeof(uint64_t), 0U); 1691cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi if (tmp_slots_size + tmp_header_size <= run_size) { 1692cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi // Found the right number of slots, that is, there was enough 1693cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi // space for the header (including the bit maps.) 1694cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi num_of_slots = s; 1695cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi header_size = tmp_header_size; 1696cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi break; 1697cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi } 1698cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi } 16997ed9c561048d79083b6d0576c71a986a3123bca6Hiroshi Yamauchi DCHECK_GT(num_of_slots, 0U) << i; 17007ed9c561048d79083b6d0576c71a986a3123bca6Hiroshi Yamauchi DCHECK_GT(header_size, 0U) << i; 1701cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi // Add the padding for the alignment remainder. 1702cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi header_size += run_size % bracket_size; 17035d057056db1923947ba846b391d981759b15714aIan Rogers DCHECK_EQ(header_size + num_of_slots * bracket_size, run_size); 1704cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi numOfSlots[i] = num_of_slots; 1705cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi headerSizes[i] = header_size; 1706cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi if (kTraceRosAlloc) { 1707cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi LOG(INFO) << "numOfSlots[" << i << "]=" << numOfSlots[i] 170831bf42c48c4d00f0677c31264bba8d21618dae67Hiroshi Yamauchi << ", headerSizes[" << i << "]=" << headerSizes[i]; 1709cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi } 1710cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi } 17117ed9c561048d79083b6d0576c71a986a3123bca6Hiroshi Yamauchi // Set up the dedicated full run so that nobody can successfully allocate from it. 171273d1e17b3afc7d5e56184f90bf819dc64956448aMathieu Chartier if (kIsDebugBuild) { 171373d1e17b3afc7d5e56184f90bf819dc64956448aMathieu Chartier dedicated_full_run_->magic_num_ = kMagicNum; 171473d1e17b3afc7d5e56184f90bf819dc64956448aMathieu Chartier } 171573d1e17b3afc7d5e56184f90bf819dc64956448aMathieu Chartier // It doesn't matter which size bracket we use since the main goal is to have the allocation 171673d1e17b3afc7d5e56184f90bf819dc64956448aMathieu Chartier // fail 100% of the time you attempt to allocate into the dedicated full run. 171773d1e17b3afc7d5e56184f90bf819dc64956448aMathieu Chartier dedicated_full_run_->size_bracket_idx_ = 0; 171831bf42c48c4d00f0677c31264bba8d21618dae67Hiroshi Yamauchi DCHECK_EQ(dedicated_full_run_->FreeList()->Size(), 0U); // It looks full. 171973d1e17b3afc7d5e56184f90bf819dc64956448aMathieu Chartier dedicated_full_run_->SetIsThreadLocal(true); 172031bf42c48c4d00f0677c31264bba8d21618dae67Hiroshi Yamauchi 172131bf42c48c4d00f0677c31264bba8d21618dae67Hiroshi Yamauchi // The smallest bracket size must be at least as large as the sizeof(Slot). 172231bf42c48c4d00f0677c31264bba8d21618dae67Hiroshi Yamauchi DCHECK_LE(sizeof(Slot), bracketSizes[0]) << "sizeof(Slot) <= the smallest bracket size"; 17237ed9c561048d79083b6d0576c71a986a3123bca6Hiroshi Yamauchi // Check the invariants between the max bracket sizes and the number of brackets. 17247ed9c561048d79083b6d0576c71a986a3123bca6Hiroshi Yamauchi DCHECK_EQ(kMaxThreadLocalBracketSize, bracketSizes[kNumThreadLocalSizeBrackets - 1]); 17257ed9c561048d79083b6d0576c71a986a3123bca6Hiroshi Yamauchi DCHECK_EQ(kMaxRegularBracketSize, bracketSizes[kNumRegularSizeBrackets - 1]); 1726cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi} 1727cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi 17286a3c1fcb4ba42ad4d5d142c17a3712a6ddd3866fIan Rogersvoid RosAlloc::BytesAllocatedCallback(void* start ATTRIBUTE_UNUSED, void* end ATTRIBUTE_UNUSED, 17296a3c1fcb4ba42ad4d5d142c17a3712a6ddd3866fIan Rogers size_t used_bytes, void* arg) { 1730cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi if (used_bytes == 0) { 1731cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi return; 1732cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi } 1733cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi size_t* bytes_allocated = reinterpret_cast<size_t*>(arg); 1734cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi *bytes_allocated += used_bytes; 1735cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi} 1736cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi 17376a3c1fcb4ba42ad4d5d142c17a3712a6ddd3866fIan Rogersvoid RosAlloc::ObjectsAllocatedCallback(void* start ATTRIBUTE_UNUSED, void* end ATTRIBUTE_UNUSED, 17386a3c1fcb4ba42ad4d5d142c17a3712a6ddd3866fIan Rogers size_t used_bytes, void* arg) { 1739cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi if (used_bytes == 0) { 1740cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi return; 1741cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi } 1742cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi size_t* objects_allocated = reinterpret_cast<size_t*>(arg); 1743cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi ++(*objects_allocated); 1744cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi} 1745cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi 1746a4adbfd44032d70e166e6f18096bbbed05a990baHiroshi Yamauchivoid RosAlloc::Verify() { 1747a4adbfd44032d70e166e6f18096bbbed05a990baHiroshi Yamauchi Thread* self = Thread::Current(); 1748a4adbfd44032d70e166e6f18096bbbed05a990baHiroshi Yamauchi CHECK(Locks::mutator_lock_->IsExclusiveHeld(self)) 1749a5b5c55c8585b7ce915f0c7e1f66d121a7f7a078Mathieu Chartier << "The mutator locks isn't exclusively locked at " << __PRETTY_FUNCTION__; 1750277ccbd200ea43590dfc06a93ae184a765327ad0Andreas Gampe MutexLock thread_list_mu(self, *Locks::thread_list_lock_); 1751a1c1c71e24c93a720bbf13de129c75a9a0bde37aMathieu Chartier ReaderMutexLock wmu(self, bulk_free_lock_); 1752a4adbfd44032d70e166e6f18096bbbed05a990baHiroshi Yamauchi std::vector<Run*> runs; 1753a4adbfd44032d70e166e6f18096bbbed05a990baHiroshi Yamauchi { 1754277ccbd200ea43590dfc06a93ae184a765327ad0Andreas Gampe MutexLock lock_mu(self, lock_); 175526d69ffc0ebc98fbc5f316d8cd3ee6ba5b2001acHiroshi Yamauchi size_t pm_end = page_map_size_; 1756a4adbfd44032d70e166e6f18096bbbed05a990baHiroshi Yamauchi size_t i = 0; 17571e13374baf7dfaf442ffbf9809c37c131d681eafEvgenii Stepanov size_t memory_tool_modifier = is_running_on_memory_tool_ ? 17581e13374baf7dfaf442ffbf9809c37c131d681eafEvgenii Stepanov 2 * ::art::gc::space::kDefaultMemoryToolRedZoneBytes : // Redzones before and after. 1759fef16adc09a26aa8081c3e803bbc66e1947a97a0Andreas Gampe 0; 1760a4adbfd44032d70e166e6f18096bbbed05a990baHiroshi Yamauchi while (i < pm_end) { 176113735955f39b3b304c37d2b2840663c131262c18Ian Rogers uint8_t pm = page_map_[i]; 1762a4adbfd44032d70e166e6f18096bbbed05a990baHiroshi Yamauchi switch (pm) { 1763a5b5c55c8585b7ce915f0c7e1f66d121a7f7a078Mathieu Chartier case kPageMapReleased: 1764a5b5c55c8585b7ce915f0c7e1f66d121a7f7a078Mathieu Chartier // Fall-through. 1765a4adbfd44032d70e166e6f18096bbbed05a990baHiroshi Yamauchi case kPageMapEmpty: { 1766a4adbfd44032d70e166e6f18096bbbed05a990baHiroshi Yamauchi // The start of a free page run. 1767a4adbfd44032d70e166e6f18096bbbed05a990baHiroshi Yamauchi FreePageRun* fpr = reinterpret_cast<FreePageRun*>(base_ + i * kPageSize); 17685d057056db1923947ba846b391d981759b15714aIan Rogers DCHECK_EQ(fpr->magic_num_, kMagicNumFree); 1769a4adbfd44032d70e166e6f18096bbbed05a990baHiroshi Yamauchi CHECK(free_page_runs_.find(fpr) != free_page_runs_.end()) 1770a4adbfd44032d70e166e6f18096bbbed05a990baHiroshi Yamauchi << "An empty page must belong to the free page run set"; 1771a4adbfd44032d70e166e6f18096bbbed05a990baHiroshi Yamauchi size_t fpr_size = fpr->ByteSize(this); 177214d90579f013b374638b599361970557ed4b3f09Roland Levillain CHECK_ALIGNED(fpr_size, kPageSize) 1773a4adbfd44032d70e166e6f18096bbbed05a990baHiroshi Yamauchi << "A free page run size isn't page-aligned : " << fpr_size; 1774a4adbfd44032d70e166e6f18096bbbed05a990baHiroshi Yamauchi size_t num_pages = fpr_size / kPageSize; 1775a4adbfd44032d70e166e6f18096bbbed05a990baHiroshi Yamauchi CHECK_GT(num_pages, static_cast<uintptr_t>(0)) 1776a4adbfd44032d70e166e6f18096bbbed05a990baHiroshi Yamauchi << "A free page run size must be > 0 : " << fpr_size; 1777a4adbfd44032d70e166e6f18096bbbed05a990baHiroshi Yamauchi for (size_t j = i + 1; j < i + num_pages; ++j) { 1778a5b5c55c8585b7ce915f0c7e1f66d121a7f7a078Mathieu Chartier CHECK(IsFreePage(j)) 1779a4adbfd44032d70e166e6f18096bbbed05a990baHiroshi Yamauchi << "A mismatch between the page map table for kPageMapEmpty " 1780a4adbfd44032d70e166e6f18096bbbed05a990baHiroshi Yamauchi << " at page index " << j 1781a4adbfd44032d70e166e6f18096bbbed05a990baHiroshi Yamauchi << " and the free page run size : page index range : " 1782a4adbfd44032d70e166e6f18096bbbed05a990baHiroshi Yamauchi << i << " to " << (i + num_pages) << std::endl << DumpPageMap(); 1783a4adbfd44032d70e166e6f18096bbbed05a990baHiroshi Yamauchi } 1784a4adbfd44032d70e166e6f18096bbbed05a990baHiroshi Yamauchi i += num_pages; 1785a4adbfd44032d70e166e6f18096bbbed05a990baHiroshi Yamauchi CHECK_LE(i, pm_end) << "Page map index " << i << " out of range < " << pm_end 1786a4adbfd44032d70e166e6f18096bbbed05a990baHiroshi Yamauchi << std::endl << DumpPageMap(); 1787a4adbfd44032d70e166e6f18096bbbed05a990baHiroshi Yamauchi break; 1788a4adbfd44032d70e166e6f18096bbbed05a990baHiroshi Yamauchi } 1789a4adbfd44032d70e166e6f18096bbbed05a990baHiroshi Yamauchi case kPageMapLargeObject: { 1790a4adbfd44032d70e166e6f18096bbbed05a990baHiroshi Yamauchi // The start of a large object. 1791a4adbfd44032d70e166e6f18096bbbed05a990baHiroshi Yamauchi size_t num_pages = 1; 1792a4adbfd44032d70e166e6f18096bbbed05a990baHiroshi Yamauchi size_t idx = i + 1; 1793a4adbfd44032d70e166e6f18096bbbed05a990baHiroshi Yamauchi while (idx < pm_end && page_map_[idx] == kPageMapLargeObjectPart) { 1794a4adbfd44032d70e166e6f18096bbbed05a990baHiroshi Yamauchi num_pages++; 1795a4adbfd44032d70e166e6f18096bbbed05a990baHiroshi Yamauchi idx++; 1796a4adbfd44032d70e166e6f18096bbbed05a990baHiroshi Yamauchi } 1797d7576328811e5103e99d31f834a857522cc1463fAndreas Gampe uint8_t* start = base_ + i * kPageSize; 17981e13374baf7dfaf442ffbf9809c37c131d681eafEvgenii Stepanov if (is_running_on_memory_tool_) { 17991e13374baf7dfaf442ffbf9809c37c131d681eafEvgenii Stepanov start += ::art::gc::space::kDefaultMemoryToolRedZoneBytes; 1800d7576328811e5103e99d31f834a857522cc1463fAndreas Gampe } 1801a4adbfd44032d70e166e6f18096bbbed05a990baHiroshi Yamauchi mirror::Object* obj = reinterpret_cast<mirror::Object*>(start); 1802a4adbfd44032d70e166e6f18096bbbed05a990baHiroshi Yamauchi size_t obj_size = obj->SizeOf(); 18031e13374baf7dfaf442ffbf9809c37c131d681eafEvgenii Stepanov CHECK_GT(obj_size + memory_tool_modifier, kLargeSizeThreshold) 1804a4adbfd44032d70e166e6f18096bbbed05a990baHiroshi Yamauchi << "A rosalloc large object size must be > " << kLargeSizeThreshold; 18051e13374baf7dfaf442ffbf9809c37c131d681eafEvgenii Stepanov CHECK_EQ(num_pages, RoundUp(obj_size + memory_tool_modifier, kPageSize) / kPageSize) 18061e13374baf7dfaf442ffbf9809c37c131d681eafEvgenii Stepanov << "A rosalloc large object size " << obj_size + memory_tool_modifier 1807a4adbfd44032d70e166e6f18096bbbed05a990baHiroshi Yamauchi << " does not match the page map table " << (num_pages * kPageSize) 1808a4adbfd44032d70e166e6f18096bbbed05a990baHiroshi Yamauchi << std::endl << DumpPageMap(); 1809a4adbfd44032d70e166e6f18096bbbed05a990baHiroshi Yamauchi i += num_pages; 1810a4adbfd44032d70e166e6f18096bbbed05a990baHiroshi Yamauchi CHECK_LE(i, pm_end) << "Page map index " << i << " out of range < " << pm_end 1811a4adbfd44032d70e166e6f18096bbbed05a990baHiroshi Yamauchi << std::endl << DumpPageMap(); 1812a4adbfd44032d70e166e6f18096bbbed05a990baHiroshi Yamauchi break; 1813a4adbfd44032d70e166e6f18096bbbed05a990baHiroshi Yamauchi } 1814a4adbfd44032d70e166e6f18096bbbed05a990baHiroshi Yamauchi case kPageMapLargeObjectPart: 18152fdeecb890a353d3f17407cc1cb015e0a65c2220Maxim Kazantsev LOG(FATAL) << "Unreachable - page map type: " << static_cast<int>(pm) << std::endl << DumpPageMap(); 1816a4adbfd44032d70e166e6f18096bbbed05a990baHiroshi Yamauchi break; 1817a4adbfd44032d70e166e6f18096bbbed05a990baHiroshi Yamauchi case kPageMapRun: { 1818a4adbfd44032d70e166e6f18096bbbed05a990baHiroshi Yamauchi // The start of a run. 1819a4adbfd44032d70e166e6f18096bbbed05a990baHiroshi Yamauchi Run* run = reinterpret_cast<Run*>(base_ + i * kPageSize); 18205d057056db1923947ba846b391d981759b15714aIan Rogers DCHECK_EQ(run->magic_num_, kMagicNum); 1821a4adbfd44032d70e166e6f18096bbbed05a990baHiroshi Yamauchi size_t idx = run->size_bracket_idx_; 18225d057056db1923947ba846b391d981759b15714aIan Rogers CHECK_LT(idx, kNumOfSizeBrackets) << "Out of range size bracket index : " << idx; 1823a4adbfd44032d70e166e6f18096bbbed05a990baHiroshi Yamauchi size_t num_pages = numOfPages[idx]; 1824a4adbfd44032d70e166e6f18096bbbed05a990baHiroshi Yamauchi CHECK_GT(num_pages, static_cast<uintptr_t>(0)) 1825a4adbfd44032d70e166e6f18096bbbed05a990baHiroshi Yamauchi << "Run size must be > 0 : " << num_pages; 1826a4adbfd44032d70e166e6f18096bbbed05a990baHiroshi Yamauchi for (size_t j = i + 1; j < i + num_pages; ++j) { 1827a4adbfd44032d70e166e6f18096bbbed05a990baHiroshi Yamauchi CHECK_EQ(page_map_[j], kPageMapRunPart) 1828a4adbfd44032d70e166e6f18096bbbed05a990baHiroshi Yamauchi << "A mismatch between the page map table for kPageMapRunPart " 1829a4adbfd44032d70e166e6f18096bbbed05a990baHiroshi Yamauchi << " at page index " << j 1830a4adbfd44032d70e166e6f18096bbbed05a990baHiroshi Yamauchi << " and the run size : page index range " << i << " to " << (i + num_pages) 1831a4adbfd44032d70e166e6f18096bbbed05a990baHiroshi Yamauchi << std::endl << DumpPageMap(); 1832a4adbfd44032d70e166e6f18096bbbed05a990baHiroshi Yamauchi } 183373d1e17b3afc7d5e56184f90bf819dc64956448aMathieu Chartier // Don't verify the dedicated_full_run_ since it doesn't have any real allocations. 1834a4adbfd44032d70e166e6f18096bbbed05a990baHiroshi Yamauchi runs.push_back(run); 1835a4adbfd44032d70e166e6f18096bbbed05a990baHiroshi Yamauchi i += num_pages; 1836a4adbfd44032d70e166e6f18096bbbed05a990baHiroshi Yamauchi CHECK_LE(i, pm_end) << "Page map index " << i << " out of range < " << pm_end 1837a4adbfd44032d70e166e6f18096bbbed05a990baHiroshi Yamauchi << std::endl << DumpPageMap(); 1838a4adbfd44032d70e166e6f18096bbbed05a990baHiroshi Yamauchi break; 1839a4adbfd44032d70e166e6f18096bbbed05a990baHiroshi Yamauchi } 1840a4adbfd44032d70e166e6f18096bbbed05a990baHiroshi Yamauchi case kPageMapRunPart: 18410651d41e41341fb2e9ef3ee41dc1f1bfc832dbbbMathieu Chartier // Fall-through. 1842a4adbfd44032d70e166e6f18096bbbed05a990baHiroshi Yamauchi default: 18432fdeecb890a353d3f17407cc1cb015e0a65c2220Maxim Kazantsev LOG(FATAL) << "Unreachable - page map type: " << static_cast<int>(pm) << std::endl << DumpPageMap(); 1844a4adbfd44032d70e166e6f18096bbbed05a990baHiroshi Yamauchi break; 1845a4adbfd44032d70e166e6f18096bbbed05a990baHiroshi Yamauchi } 1846a4adbfd44032d70e166e6f18096bbbed05a990baHiroshi Yamauchi } 1847a4adbfd44032d70e166e6f18096bbbed05a990baHiroshi Yamauchi } 18480651d41e41341fb2e9ef3ee41dc1f1bfc832dbbbMathieu Chartier std::list<Thread*> threads = Runtime::Current()->GetThreadList()->GetList(); 18490651d41e41341fb2e9ef3ee41dc1f1bfc832dbbbMathieu Chartier for (Thread* thread : threads) { 18500651d41e41341fb2e9ef3ee41dc1f1bfc832dbbbMathieu Chartier for (size_t i = 0; i < kNumThreadLocalSizeBrackets; ++i) { 1851277ccbd200ea43590dfc06a93ae184a765327ad0Andreas Gampe MutexLock brackets_mu(self, *size_bracket_locks_[i]); 18520651d41e41341fb2e9ef3ee41dc1f1bfc832dbbbMathieu Chartier Run* thread_local_run = reinterpret_cast<Run*>(thread->GetRosAllocRun(i)); 18530651d41e41341fb2e9ef3ee41dc1f1bfc832dbbbMathieu Chartier CHECK(thread_local_run != nullptr); 18540651d41e41341fb2e9ef3ee41dc1f1bfc832dbbbMathieu Chartier CHECK(thread_local_run->IsThreadLocal()); 18550651d41e41341fb2e9ef3ee41dc1f1bfc832dbbbMathieu Chartier CHECK(thread_local_run == dedicated_full_run_ || 18560651d41e41341fb2e9ef3ee41dc1f1bfc832dbbbMathieu Chartier thread_local_run->size_bracket_idx_ == i); 18570651d41e41341fb2e9ef3ee41dc1f1bfc832dbbbMathieu Chartier } 18580651d41e41341fb2e9ef3ee41dc1f1bfc832dbbbMathieu Chartier } 18590651d41e41341fb2e9ef3ee41dc1f1bfc832dbbbMathieu Chartier for (size_t i = 0; i < kNumOfSizeBrackets; i++) { 1860277ccbd200ea43590dfc06a93ae184a765327ad0Andreas Gampe MutexLock brackets_mu(self, *size_bracket_locks_[i]); 18610651d41e41341fb2e9ef3ee41dc1f1bfc832dbbbMathieu Chartier Run* current_run = current_runs_[i]; 18620651d41e41341fb2e9ef3ee41dc1f1bfc832dbbbMathieu Chartier CHECK(current_run != nullptr); 18630651d41e41341fb2e9ef3ee41dc1f1bfc832dbbbMathieu Chartier if (current_run != dedicated_full_run_) { 18640651d41e41341fb2e9ef3ee41dc1f1bfc832dbbbMathieu Chartier // The dedicated full run is currently marked as thread local. 18650651d41e41341fb2e9ef3ee41dc1f1bfc832dbbbMathieu Chartier CHECK(!current_run->IsThreadLocal()); 18660651d41e41341fb2e9ef3ee41dc1f1bfc832dbbbMathieu Chartier CHECK_EQ(current_run->size_bracket_idx_, i); 18670651d41e41341fb2e9ef3ee41dc1f1bfc832dbbbMathieu Chartier } 18680651d41e41341fb2e9ef3ee41dc1f1bfc832dbbbMathieu Chartier } 1869a4adbfd44032d70e166e6f18096bbbed05a990baHiroshi Yamauchi // Call Verify() here for the lock order. 1870a4adbfd44032d70e166e6f18096bbbed05a990baHiroshi Yamauchi for (auto& run : runs) { 18711e13374baf7dfaf442ffbf9809c37c131d681eafEvgenii Stepanov run->Verify(self, this, is_running_on_memory_tool_); 1872a4adbfd44032d70e166e6f18096bbbed05a990baHiroshi Yamauchi } 1873a4adbfd44032d70e166e6f18096bbbed05a990baHiroshi Yamauchi} 1874a4adbfd44032d70e166e6f18096bbbed05a990baHiroshi Yamauchi 18751e13374baf7dfaf442ffbf9809c37c131d681eafEvgenii Stepanovvoid RosAlloc::Run::Verify(Thread* self, RosAlloc* rosalloc, bool running_on_memory_tool) { 18765d057056db1923947ba846b391d981759b15714aIan Rogers DCHECK_EQ(magic_num_, kMagicNum) << "Bad magic number : " << Dump(); 187773d1e17b3afc7d5e56184f90bf819dc64956448aMathieu Chartier const size_t idx = size_bracket_idx_; 18785d057056db1923947ba846b391d981759b15714aIan Rogers CHECK_LT(idx, kNumOfSizeBrackets) << "Out of range size bracket index : " << Dump(); 187913735955f39b3b304c37d2b2840663c131262c18Ian Rogers uint8_t* slot_base = reinterpret_cast<uint8_t*>(this) + headerSizes[idx]; 188073d1e17b3afc7d5e56184f90bf819dc64956448aMathieu Chartier const size_t num_slots = numOfSlots[idx]; 1881a4adbfd44032d70e166e6f18096bbbed05a990baHiroshi Yamauchi size_t bracket_size = IndexToBracketSize(idx); 1882a4adbfd44032d70e166e6f18096bbbed05a990baHiroshi Yamauchi CHECK_EQ(slot_base + num_slots * bracket_size, 188313735955f39b3b304c37d2b2840663c131262c18Ian Rogers reinterpret_cast<uint8_t*>(this) + numOfPages[idx] * kPageSize) 1884a4adbfd44032d70e166e6f18096bbbed05a990baHiroshi Yamauchi << "Mismatch in the end address of the run " << Dump(); 188531bf42c48c4d00f0677c31264bba8d21618dae67Hiroshi Yamauchi // Check that the bulk free list is empty. It's only used during BulkFree(). 188631bf42c48c4d00f0677c31264bba8d21618dae67Hiroshi Yamauchi CHECK(IsBulkFreeListEmpty()) << "The bulk free isn't empty " << Dump(); 1887a4adbfd44032d70e166e6f18096bbbed05a990baHiroshi Yamauchi // Check the thread local runs, the current runs, and the run sets. 188873d1e17b3afc7d5e56184f90bf819dc64956448aMathieu Chartier if (IsThreadLocal()) { 1889a4adbfd44032d70e166e6f18096bbbed05a990baHiroshi Yamauchi // If it's a thread local run, then it must be pointed to by an owner thread. 1890a4adbfd44032d70e166e6f18096bbbed05a990baHiroshi Yamauchi bool owner_found = false; 1891a4adbfd44032d70e166e6f18096bbbed05a990baHiroshi Yamauchi std::list<Thread*> thread_list = Runtime::Current()->GetThreadList()->GetList(); 1892a4adbfd44032d70e166e6f18096bbbed05a990baHiroshi Yamauchi for (auto it = thread_list.begin(); it != thread_list.end(); ++it) { 1893a4adbfd44032d70e166e6f18096bbbed05a990baHiroshi Yamauchi Thread* thread = *it; 18940651d41e41341fb2e9ef3ee41dc1f1bfc832dbbbMathieu Chartier for (size_t i = 0; i < kNumThreadLocalSizeBrackets; i++) { 1895a4adbfd44032d70e166e6f18096bbbed05a990baHiroshi Yamauchi MutexLock mu(self, *rosalloc->size_bracket_locks_[i]); 1896dd7624d2b9e599d57762d12031b10b89defc9807Ian Rogers Run* thread_local_run = reinterpret_cast<Run*>(thread->GetRosAllocRun(i)); 1897a4adbfd44032d70e166e6f18096bbbed05a990baHiroshi Yamauchi if (thread_local_run == this) { 1898a4adbfd44032d70e166e6f18096bbbed05a990baHiroshi Yamauchi CHECK(!owner_found) 1899a4adbfd44032d70e166e6f18096bbbed05a990baHiroshi Yamauchi << "A thread local run has more than one owner thread " << Dump(); 1900a4adbfd44032d70e166e6f18096bbbed05a990baHiroshi Yamauchi CHECK_EQ(i, idx) 1901a4adbfd44032d70e166e6f18096bbbed05a990baHiroshi Yamauchi << "A mismatching size bracket index in a thread local run " << Dump(); 1902a4adbfd44032d70e166e6f18096bbbed05a990baHiroshi Yamauchi owner_found = true; 1903a4adbfd44032d70e166e6f18096bbbed05a990baHiroshi Yamauchi } 1904a4adbfd44032d70e166e6f18096bbbed05a990baHiroshi Yamauchi } 1905a4adbfd44032d70e166e6f18096bbbed05a990baHiroshi Yamauchi } 1906a4adbfd44032d70e166e6f18096bbbed05a990baHiroshi Yamauchi CHECK(owner_found) << "A thread local run has no owner thread " << Dump(); 1907a4adbfd44032d70e166e6f18096bbbed05a990baHiroshi Yamauchi } else { 190831bf42c48c4d00f0677c31264bba8d21618dae67Hiroshi Yamauchi // If it's not thread local, check that the thread local free list is empty. 190931bf42c48c4d00f0677c31264bba8d21618dae67Hiroshi Yamauchi CHECK(IsThreadLocalFreeListEmpty()) 191031bf42c48c4d00f0677c31264bba8d21618dae67Hiroshi Yamauchi << "A non-thread-local run's thread local free list isn't empty " 1911a4adbfd44032d70e166e6f18096bbbed05a990baHiroshi Yamauchi << Dump(); 191231bf42c48c4d00f0677c31264bba8d21618dae67Hiroshi Yamauchi // Check if it's a current run for the size bracket. 1913a4adbfd44032d70e166e6f18096bbbed05a990baHiroshi Yamauchi bool is_current_run = false; 1914a4adbfd44032d70e166e6f18096bbbed05a990baHiroshi Yamauchi for (size_t i = 0; i < kNumOfSizeBrackets; i++) { 1915a4adbfd44032d70e166e6f18096bbbed05a990baHiroshi Yamauchi MutexLock mu(self, *rosalloc->size_bracket_locks_[i]); 1916a4adbfd44032d70e166e6f18096bbbed05a990baHiroshi Yamauchi Run* current_run = rosalloc->current_runs_[i]; 1917a4adbfd44032d70e166e6f18096bbbed05a990baHiroshi Yamauchi if (idx == i) { 1918a4adbfd44032d70e166e6f18096bbbed05a990baHiroshi Yamauchi if (this == current_run) { 1919a4adbfd44032d70e166e6f18096bbbed05a990baHiroshi Yamauchi is_current_run = true; 1920a4adbfd44032d70e166e6f18096bbbed05a990baHiroshi Yamauchi } 1921a4adbfd44032d70e166e6f18096bbbed05a990baHiroshi Yamauchi } else { 1922a4adbfd44032d70e166e6f18096bbbed05a990baHiroshi Yamauchi // If the size bucket index does not match, then it must not 1923a4adbfd44032d70e166e6f18096bbbed05a990baHiroshi Yamauchi // be a current run. 1924a4adbfd44032d70e166e6f18096bbbed05a990baHiroshi Yamauchi CHECK_NE(this, current_run) 1925a4adbfd44032d70e166e6f18096bbbed05a990baHiroshi Yamauchi << "A current run points to a run with a wrong size bracket index " << Dump(); 1926a4adbfd44032d70e166e6f18096bbbed05a990baHiroshi Yamauchi } 1927a4adbfd44032d70e166e6f18096bbbed05a990baHiroshi Yamauchi } 1928a4adbfd44032d70e166e6f18096bbbed05a990baHiroshi Yamauchi // If it's neither a thread local or current run, then it must be 1929a4adbfd44032d70e166e6f18096bbbed05a990baHiroshi Yamauchi // in a run set. 1930a4adbfd44032d70e166e6f18096bbbed05a990baHiroshi Yamauchi if (!is_current_run) { 1931a4adbfd44032d70e166e6f18096bbbed05a990baHiroshi Yamauchi MutexLock mu(self, rosalloc->lock_); 193258553c7fd89ce69857017322444265469bb6af62Mathieu Chartier auto& non_full_runs = rosalloc->non_full_runs_[idx]; 1933a4adbfd44032d70e166e6f18096bbbed05a990baHiroshi Yamauchi // If it's all free, it must be a free page run rather than a run. 1934a4adbfd44032d70e166e6f18096bbbed05a990baHiroshi Yamauchi CHECK(!IsAllFree()) << "A free run must be in a free page run set " << Dump(); 1935a4adbfd44032d70e166e6f18096bbbed05a990baHiroshi Yamauchi if (!IsFull()) { 1936a4adbfd44032d70e166e6f18096bbbed05a990baHiroshi Yamauchi // If it's not full, it must in the non-full run set. 1937a4adbfd44032d70e166e6f18096bbbed05a990baHiroshi Yamauchi CHECK(non_full_runs.find(this) != non_full_runs.end()) 1938a4adbfd44032d70e166e6f18096bbbed05a990baHiroshi Yamauchi << "A non-full run isn't in the non-full run set " << Dump(); 1939a4adbfd44032d70e166e6f18096bbbed05a990baHiroshi Yamauchi } else { 1940a4adbfd44032d70e166e6f18096bbbed05a990baHiroshi Yamauchi // If it's full, it must in the full run set (debug build only.) 1941a4adbfd44032d70e166e6f18096bbbed05a990baHiroshi Yamauchi if (kIsDebugBuild) { 194258553c7fd89ce69857017322444265469bb6af62Mathieu Chartier auto& full_runs = rosalloc->full_runs_[idx]; 1943a4adbfd44032d70e166e6f18096bbbed05a990baHiroshi Yamauchi CHECK(full_runs.find(this) != full_runs.end()) 1944a4adbfd44032d70e166e6f18096bbbed05a990baHiroshi Yamauchi << " A full run isn't in the full run set " << Dump(); 1945a4adbfd44032d70e166e6f18096bbbed05a990baHiroshi Yamauchi } 1946a4adbfd44032d70e166e6f18096bbbed05a990baHiroshi Yamauchi } 1947a4adbfd44032d70e166e6f18096bbbed05a990baHiroshi Yamauchi } 1948a4adbfd44032d70e166e6f18096bbbed05a990baHiroshi Yamauchi } 1949a4adbfd44032d70e166e6f18096bbbed05a990baHiroshi Yamauchi // Check each slot. 19501e13374baf7dfaf442ffbf9809c37c131d681eafEvgenii Stepanov size_t memory_tool_modifier = running_on_memory_tool ? 19511e13374baf7dfaf442ffbf9809c37c131d681eafEvgenii Stepanov 2 * ::art::gc::space::kDefaultMemoryToolRedZoneBytes : 1952d7576328811e5103e99d31f834a857522cc1463fAndreas Gampe 0U; 195331bf42c48c4d00f0677c31264bba8d21618dae67Hiroshi Yamauchi // TODO: reuse InspectAllSlots(). 195431bf42c48c4d00f0677c31264bba8d21618dae67Hiroshi Yamauchi std::unique_ptr<bool[]> is_free(new bool[num_slots]()); // zero initialized 195531bf42c48c4d00f0677c31264bba8d21618dae67Hiroshi Yamauchi // Mark the free slots and the remaining ones are allocated. 195631bf42c48c4d00f0677c31264bba8d21618dae67Hiroshi Yamauchi for (Slot* slot = free_list_.Head(); slot != nullptr; slot = slot->Next()) { 195731bf42c48c4d00f0677c31264bba8d21618dae67Hiroshi Yamauchi size_t slot_idx = SlotIndex(slot); 195831bf42c48c4d00f0677c31264bba8d21618dae67Hiroshi Yamauchi DCHECK_LT(slot_idx, num_slots); 195931bf42c48c4d00f0677c31264bba8d21618dae67Hiroshi Yamauchi is_free[slot_idx] = true; 196031bf42c48c4d00f0677c31264bba8d21618dae67Hiroshi Yamauchi } 196131bf42c48c4d00f0677c31264bba8d21618dae67Hiroshi Yamauchi if (IsThreadLocal()) { 196231bf42c48c4d00f0677c31264bba8d21618dae67Hiroshi Yamauchi for (Slot* slot = thread_local_free_list_.Head(); slot != nullptr; slot = slot->Next()) { 196331bf42c48c4d00f0677c31264bba8d21618dae67Hiroshi Yamauchi size_t slot_idx = SlotIndex(slot); 196431bf42c48c4d00f0677c31264bba8d21618dae67Hiroshi Yamauchi DCHECK_LT(slot_idx, num_slots); 196531bf42c48c4d00f0677c31264bba8d21618dae67Hiroshi Yamauchi is_free[slot_idx] = true; 196631bf42c48c4d00f0677c31264bba8d21618dae67Hiroshi Yamauchi } 196731bf42c48c4d00f0677c31264bba8d21618dae67Hiroshi Yamauchi } 196831bf42c48c4d00f0677c31264bba8d21618dae67Hiroshi Yamauchi for (size_t slot_idx = 0; slot_idx < num_slots; ++slot_idx) { 196931bf42c48c4d00f0677c31264bba8d21618dae67Hiroshi Yamauchi uint8_t* slot_addr = slot_base + slot_idx * bracket_size; 197031bf42c48c4d00f0677c31264bba8d21618dae67Hiroshi Yamauchi if (running_on_memory_tool) { 197131bf42c48c4d00f0677c31264bba8d21618dae67Hiroshi Yamauchi slot_addr += ::art::gc::space::kDefaultMemoryToolRedZoneBytes; 197231bf42c48c4d00f0677c31264bba8d21618dae67Hiroshi Yamauchi } 197331bf42c48c4d00f0677c31264bba8d21618dae67Hiroshi Yamauchi if (!is_free[slot_idx]) { 197431bf42c48c4d00f0677c31264bba8d21618dae67Hiroshi Yamauchi // The slot is allocated 197531bf42c48c4d00f0677c31264bba8d21618dae67Hiroshi Yamauchi mirror::Object* obj = reinterpret_cast<mirror::Object*>(slot_addr); 197631bf42c48c4d00f0677c31264bba8d21618dae67Hiroshi Yamauchi size_t obj_size = obj->SizeOf(); 197731bf42c48c4d00f0677c31264bba8d21618dae67Hiroshi Yamauchi CHECK_LE(obj_size + memory_tool_modifier, kLargeSizeThreshold) 197831bf42c48c4d00f0677c31264bba8d21618dae67Hiroshi Yamauchi << "A run slot contains a large object " << Dump(); 197931bf42c48c4d00f0677c31264bba8d21618dae67Hiroshi Yamauchi CHECK_EQ(SizeToIndex(obj_size + memory_tool_modifier), idx) 198031bf42c48c4d00f0677c31264bba8d21618dae67Hiroshi Yamauchi << PrettyTypeOf(obj) << " " 198131bf42c48c4d00f0677c31264bba8d21618dae67Hiroshi Yamauchi << "obj_size=" << obj_size << "(" << obj_size + memory_tool_modifier << "), idx=" << idx 198231bf42c48c4d00f0677c31264bba8d21618dae67Hiroshi Yamauchi << " A run slot contains an object with wrong size " << Dump(); 1983a4adbfd44032d70e166e6f18096bbbed05a990baHiroshi Yamauchi } 1984a4adbfd44032d70e166e6f18096bbbed05a990baHiroshi Yamauchi } 1985a4adbfd44032d70e166e6f18096bbbed05a990baHiroshi Yamauchi} 1986a4adbfd44032d70e166e6f18096bbbed05a990baHiroshi Yamauchi 1987d9a88de76de4c81ad75340b824df64a68c739351Hiroshi Yamauchisize_t RosAlloc::ReleasePages() { 1988d9a88de76de4c81ad75340b824df64a68c739351Hiroshi Yamauchi VLOG(heap) << "RosAlloc::ReleasePages()"; 1989d9a88de76de4c81ad75340b824df64a68c739351Hiroshi Yamauchi DCHECK(!DoesReleaseAllPages()); 1990d9a88de76de4c81ad75340b824df64a68c739351Hiroshi Yamauchi Thread* self = Thread::Current(); 1991d9a88de76de4c81ad75340b824df64a68c739351Hiroshi Yamauchi size_t reclaimed_bytes = 0; 1992d9a88de76de4c81ad75340b824df64a68c739351Hiroshi Yamauchi size_t i = 0; 1993a5b5c55c8585b7ce915f0c7e1f66d121a7f7a078Mathieu Chartier // Check the page map size which might have changed due to grow/shrink. 1994a5b5c55c8585b7ce915f0c7e1f66d121a7f7a078Mathieu Chartier while (i < page_map_size_) { 1995a5b5c55c8585b7ce915f0c7e1f66d121a7f7a078Mathieu Chartier // Reading the page map without a lock is racy but the race is benign since it should only 1996a5b5c55c8585b7ce915f0c7e1f66d121a7f7a078Mathieu Chartier // result in occasionally not releasing pages which we could release. 199713735955f39b3b304c37d2b2840663c131262c18Ian Rogers uint8_t pm = page_map_[i]; 1998d9a88de76de4c81ad75340b824df64a68c739351Hiroshi Yamauchi switch (pm) { 1999e28ed99750554e0314b893dd2aa88ff9c3ca4500Mathieu Chartier case kPageMapReleased: 2000e28ed99750554e0314b893dd2aa88ff9c3ca4500Mathieu Chartier // Fall through. 2001d9a88de76de4c81ad75340b824df64a68c739351Hiroshi Yamauchi case kPageMapEmpty: { 2002e28ed99750554e0314b893dd2aa88ff9c3ca4500Mathieu Chartier // This is currently the start of a free page run. 2003e28ed99750554e0314b893dd2aa88ff9c3ca4500Mathieu Chartier // Acquire the lock to prevent other threads racing in and modifying the page map. 2004a5b5c55c8585b7ce915f0c7e1f66d121a7f7a078Mathieu Chartier MutexLock mu(self, lock_); 2005a5b5c55c8585b7ce915f0c7e1f66d121a7f7a078Mathieu Chartier // Check that it's still empty after we acquired the lock since another thread could have 2006a5b5c55c8585b7ce915f0c7e1f66d121a7f7a078Mathieu Chartier // raced in and placed an allocation here. 2007e28ed99750554e0314b893dd2aa88ff9c3ca4500Mathieu Chartier if (IsFreePage(i)) { 2008e28ed99750554e0314b893dd2aa88ff9c3ca4500Mathieu Chartier // Free page runs can start with a released page if we coalesced a released page free 2009e28ed99750554e0314b893dd2aa88ff9c3ca4500Mathieu Chartier // page run with an empty page run. 2010a5b5c55c8585b7ce915f0c7e1f66d121a7f7a078Mathieu Chartier FreePageRun* fpr = reinterpret_cast<FreePageRun*>(base_ + i * kPageSize); 2011e28ed99750554e0314b893dd2aa88ff9c3ca4500Mathieu Chartier // There is a race condition where FreePage can coalesce fpr with the previous 2012e28ed99750554e0314b893dd2aa88ff9c3ca4500Mathieu Chartier // free page run before we acquire lock_. In that case free_page_runs_.find will not find 2013e28ed99750554e0314b893dd2aa88ff9c3ca4500Mathieu Chartier // a run starting at fpr. To handle this race, we skip reclaiming the page range and go 2014e28ed99750554e0314b893dd2aa88ff9c3ca4500Mathieu Chartier // to the next page. 2015e28ed99750554e0314b893dd2aa88ff9c3ca4500Mathieu Chartier if (free_page_runs_.find(fpr) != free_page_runs_.end()) { 2016e28ed99750554e0314b893dd2aa88ff9c3ca4500Mathieu Chartier size_t fpr_size = fpr->ByteSize(this); 201714d90579f013b374638b599361970557ed4b3f09Roland Levillain DCHECK_ALIGNED(fpr_size, kPageSize); 201813735955f39b3b304c37d2b2840663c131262c18Ian Rogers uint8_t* start = reinterpret_cast<uint8_t*>(fpr); 2019e28ed99750554e0314b893dd2aa88ff9c3ca4500Mathieu Chartier reclaimed_bytes += ReleasePageRange(start, start + fpr_size); 2020e28ed99750554e0314b893dd2aa88ff9c3ca4500Mathieu Chartier size_t pages = fpr_size / kPageSize; 2021e28ed99750554e0314b893dd2aa88ff9c3ca4500Mathieu Chartier CHECK_GT(pages, 0U) << "Infinite loop probable"; 2022e28ed99750554e0314b893dd2aa88ff9c3ca4500Mathieu Chartier i += pages; 2023e28ed99750554e0314b893dd2aa88ff9c3ca4500Mathieu Chartier DCHECK_LE(i, page_map_size_); 2024e28ed99750554e0314b893dd2aa88ff9c3ca4500Mathieu Chartier break; 2025e28ed99750554e0314b893dd2aa88ff9c3ca4500Mathieu Chartier } 2026d9a88de76de4c81ad75340b824df64a68c739351Hiroshi Yamauchi } 2027fc787ecd91127b2c8458afd94e5148e2ae51a1f5Ian Rogers FALLTHROUGH_INTENDED; 2028d9a88de76de4c81ad75340b824df64a68c739351Hiroshi Yamauchi } 2029d9a88de76de4c81ad75340b824df64a68c739351Hiroshi Yamauchi case kPageMapLargeObject: // Fall through. 2030d9a88de76de4c81ad75340b824df64a68c739351Hiroshi Yamauchi case kPageMapLargeObjectPart: // Fall through. 2031d9a88de76de4c81ad75340b824df64a68c739351Hiroshi Yamauchi case kPageMapRun: // Fall through. 2032d9a88de76de4c81ad75340b824df64a68c739351Hiroshi Yamauchi case kPageMapRunPart: // Fall through. 2033d9a88de76de4c81ad75340b824df64a68c739351Hiroshi Yamauchi ++i; 2034d9a88de76de4c81ad75340b824df64a68c739351Hiroshi Yamauchi break; // Skip. 2035d9a88de76de4c81ad75340b824df64a68c739351Hiroshi Yamauchi default: 20362fdeecb890a353d3f17407cc1cb015e0a65c2220Maxim Kazantsev LOG(FATAL) << "Unreachable - page map type: " << static_cast<int>(pm); 2037d9a88de76de4c81ad75340b824df64a68c739351Hiroshi Yamauchi break; 2038d9a88de76de4c81ad75340b824df64a68c739351Hiroshi Yamauchi } 2039d9a88de76de4c81ad75340b824df64a68c739351Hiroshi Yamauchi } 2040d9a88de76de4c81ad75340b824df64a68c739351Hiroshi Yamauchi return reclaimed_bytes; 2041d9a88de76de4c81ad75340b824df64a68c739351Hiroshi Yamauchi} 2042d9a88de76de4c81ad75340b824df64a68c739351Hiroshi Yamauchi 204313735955f39b3b304c37d2b2840663c131262c18Ian Rogerssize_t RosAlloc::ReleasePageRange(uint8_t* start, uint8_t* end) { 2044a5b5c55c8585b7ce915f0c7e1f66d121a7f7a078Mathieu Chartier DCHECK_ALIGNED(start, kPageSize); 2045a5b5c55c8585b7ce915f0c7e1f66d121a7f7a078Mathieu Chartier DCHECK_ALIGNED(end, kPageSize); 2046a5b5c55c8585b7ce915f0c7e1f66d121a7f7a078Mathieu Chartier DCHECK_LT(start, end); 2047a5b5c55c8585b7ce915f0c7e1f66d121a7f7a078Mathieu Chartier if (kIsDebugBuild) { 2048a5b5c55c8585b7ce915f0c7e1f66d121a7f7a078Mathieu Chartier // In the debug build, the first page of a free page run 2049a5b5c55c8585b7ce915f0c7e1f66d121a7f7a078Mathieu Chartier // contains a magic number for debugging. Exclude it. 2050a5b5c55c8585b7ce915f0c7e1f66d121a7f7a078Mathieu Chartier start += kPageSize; 2051d7576328811e5103e99d31f834a857522cc1463fAndreas Gampe 2052d7576328811e5103e99d31f834a857522cc1463fAndreas Gampe // Single pages won't be released. 2053d7576328811e5103e99d31f834a857522cc1463fAndreas Gampe if (start == end) { 2054d7576328811e5103e99d31f834a857522cc1463fAndreas Gampe return 0; 2055d7576328811e5103e99d31f834a857522cc1463fAndreas Gampe } 2056a5b5c55c8585b7ce915f0c7e1f66d121a7f7a078Mathieu Chartier } 2057a5b5c55c8585b7ce915f0c7e1f66d121a7f7a078Mathieu Chartier if (!kMadviseZeroes) { 2058a5b5c55c8585b7ce915f0c7e1f66d121a7f7a078Mathieu Chartier // TODO: Do this when we resurrect the page instead. 2059a5b5c55c8585b7ce915f0c7e1f66d121a7f7a078Mathieu Chartier memset(start, 0, end - start); 2060a5b5c55c8585b7ce915f0c7e1f66d121a7f7a078Mathieu Chartier } 2061a5b5c55c8585b7ce915f0c7e1f66d121a7f7a078Mathieu Chartier CHECK_EQ(madvise(start, end - start, MADV_DONTNEED), 0); 2062a5b5c55c8585b7ce915f0c7e1f66d121a7f7a078Mathieu Chartier size_t pm_idx = ToPageMapIndex(start); 2063a5b5c55c8585b7ce915f0c7e1f66d121a7f7a078Mathieu Chartier size_t reclaimed_bytes = 0; 2064a5b5c55c8585b7ce915f0c7e1f66d121a7f7a078Mathieu Chartier // Calculate reclaimed bytes and upate page map. 2065a5b5c55c8585b7ce915f0c7e1f66d121a7f7a078Mathieu Chartier const size_t max_idx = pm_idx + (end - start) / kPageSize; 2066a5b5c55c8585b7ce915f0c7e1f66d121a7f7a078Mathieu Chartier for (; pm_idx < max_idx; ++pm_idx) { 2067a5b5c55c8585b7ce915f0c7e1f66d121a7f7a078Mathieu Chartier DCHECK(IsFreePage(pm_idx)); 2068a5b5c55c8585b7ce915f0c7e1f66d121a7f7a078Mathieu Chartier if (page_map_[pm_idx] == kPageMapEmpty) { 2069a5b5c55c8585b7ce915f0c7e1f66d121a7f7a078Mathieu Chartier // Mark the page as released and update how many bytes we released. 2070a5b5c55c8585b7ce915f0c7e1f66d121a7f7a078Mathieu Chartier reclaimed_bytes += kPageSize; 2071a5b5c55c8585b7ce915f0c7e1f66d121a7f7a078Mathieu Chartier page_map_[pm_idx] = kPageMapReleased; 2072a5b5c55c8585b7ce915f0c7e1f66d121a7f7a078Mathieu Chartier } 2073a5b5c55c8585b7ce915f0c7e1f66d121a7f7a078Mathieu Chartier } 2074a5b5c55c8585b7ce915f0c7e1f66d121a7f7a078Mathieu Chartier return reclaimed_bytes; 2075a5b5c55c8585b7ce915f0c7e1f66d121a7f7a078Mathieu Chartier} 2076a5b5c55c8585b7ce915f0c7e1f66d121a7f7a078Mathieu Chartier 2077654dd48e2230e16bfaa225decce72b52642e2f78Hiroshi Yamauchivoid RosAlloc::LogFragmentationAllocFailure(std::ostream& os, size_t failed_alloc_bytes) { 2078654dd48e2230e16bfaa225decce72b52642e2f78Hiroshi Yamauchi Thread* self = Thread::Current(); 2079654dd48e2230e16bfaa225decce72b52642e2f78Hiroshi Yamauchi size_t largest_continuous_free_pages = 0; 2080654dd48e2230e16bfaa225decce72b52642e2f78Hiroshi Yamauchi WriterMutexLock wmu(self, bulk_free_lock_); 2081654dd48e2230e16bfaa225decce72b52642e2f78Hiroshi Yamauchi MutexLock mu(self, lock_); 2082654dd48e2230e16bfaa225decce72b52642e2f78Hiroshi Yamauchi for (FreePageRun* fpr : free_page_runs_) { 2083654dd48e2230e16bfaa225decce72b52642e2f78Hiroshi Yamauchi largest_continuous_free_pages = std::max(largest_continuous_free_pages, 2084654dd48e2230e16bfaa225decce72b52642e2f78Hiroshi Yamauchi fpr->ByteSize(this)); 2085654dd48e2230e16bfaa225decce72b52642e2f78Hiroshi Yamauchi } 2086654dd48e2230e16bfaa225decce72b52642e2f78Hiroshi Yamauchi if (failed_alloc_bytes > kLargeSizeThreshold) { 2087654dd48e2230e16bfaa225decce72b52642e2f78Hiroshi Yamauchi // Large allocation. 2088654dd48e2230e16bfaa225decce72b52642e2f78Hiroshi Yamauchi size_t required_bytes = RoundUp(failed_alloc_bytes, kPageSize); 2089654dd48e2230e16bfaa225decce72b52642e2f78Hiroshi Yamauchi if (required_bytes > largest_continuous_free_pages) { 2090654dd48e2230e16bfaa225decce72b52642e2f78Hiroshi Yamauchi os << "; failed due to fragmentation (required continguous free " 2091654dd48e2230e16bfaa225decce72b52642e2f78Hiroshi Yamauchi << required_bytes << " bytes where largest contiguous free " 2092654dd48e2230e16bfaa225decce72b52642e2f78Hiroshi Yamauchi << largest_continuous_free_pages << " bytes)"; 2093654dd48e2230e16bfaa225decce72b52642e2f78Hiroshi Yamauchi } 2094654dd48e2230e16bfaa225decce72b52642e2f78Hiroshi Yamauchi } else { 2095654dd48e2230e16bfaa225decce72b52642e2f78Hiroshi Yamauchi // Non-large allocation. 2096654dd48e2230e16bfaa225decce72b52642e2f78Hiroshi Yamauchi size_t required_bytes = numOfPages[SizeToIndex(failed_alloc_bytes)] * kPageSize; 2097654dd48e2230e16bfaa225decce72b52642e2f78Hiroshi Yamauchi if (required_bytes > largest_continuous_free_pages) { 2098654dd48e2230e16bfaa225decce72b52642e2f78Hiroshi Yamauchi os << "; failed due to fragmentation (required continguous free " 2099654dd48e2230e16bfaa225decce72b52642e2f78Hiroshi Yamauchi << required_bytes << " bytes for a new buffer where largest contiguous free " 2100654dd48e2230e16bfaa225decce72b52642e2f78Hiroshi Yamauchi << largest_continuous_free_pages << " bytes)"; 2101654dd48e2230e16bfaa225decce72b52642e2f78Hiroshi Yamauchi } 2102654dd48e2230e16bfaa225decce72b52642e2f78Hiroshi Yamauchi } 2103654dd48e2230e16bfaa225decce72b52642e2f78Hiroshi Yamauchi} 2104654dd48e2230e16bfaa225decce72b52642e2f78Hiroshi Yamauchi 2105565c2d9bce43c430d4267c82f5702160d971e712Hiroshi Yamauchivoid RosAlloc::DumpStats(std::ostream& os) { 2106565c2d9bce43c430d4267c82f5702160d971e712Hiroshi Yamauchi Thread* self = Thread::Current(); 2107565c2d9bce43c430d4267c82f5702160d971e712Hiroshi Yamauchi CHECK(Locks::mutator_lock_->IsExclusiveHeld(self)) 2108565c2d9bce43c430d4267c82f5702160d971e712Hiroshi Yamauchi << "The mutator locks isn't exclusively locked at " << __PRETTY_FUNCTION__; 2109565c2d9bce43c430d4267c82f5702160d971e712Hiroshi Yamauchi size_t num_large_objects = 0; 2110565c2d9bce43c430d4267c82f5702160d971e712Hiroshi Yamauchi size_t num_pages_large_objects = 0; 2111565c2d9bce43c430d4267c82f5702160d971e712Hiroshi Yamauchi // These arrays are zero initialized. 2112565c2d9bce43c430d4267c82f5702160d971e712Hiroshi Yamauchi std::unique_ptr<size_t[]> num_runs(new size_t[kNumOfSizeBrackets]()); 2113565c2d9bce43c430d4267c82f5702160d971e712Hiroshi Yamauchi std::unique_ptr<size_t[]> num_pages_runs(new size_t[kNumOfSizeBrackets]()); 2114565c2d9bce43c430d4267c82f5702160d971e712Hiroshi Yamauchi std::unique_ptr<size_t[]> num_slots(new size_t[kNumOfSizeBrackets]()); 2115565c2d9bce43c430d4267c82f5702160d971e712Hiroshi Yamauchi std::unique_ptr<size_t[]> num_used_slots(new size_t[kNumOfSizeBrackets]()); 2116565c2d9bce43c430d4267c82f5702160d971e712Hiroshi Yamauchi std::unique_ptr<size_t[]> num_metadata_bytes(new size_t[kNumOfSizeBrackets]()); 2117565c2d9bce43c430d4267c82f5702160d971e712Hiroshi Yamauchi ReaderMutexLock rmu(self, bulk_free_lock_); 2118565c2d9bce43c430d4267c82f5702160d971e712Hiroshi Yamauchi MutexLock lock_mu(self, lock_); 2119565c2d9bce43c430d4267c82f5702160d971e712Hiroshi Yamauchi for (size_t i = 0; i < page_map_size_; ) { 2120565c2d9bce43c430d4267c82f5702160d971e712Hiroshi Yamauchi uint8_t pm = page_map_[i]; 2121565c2d9bce43c430d4267c82f5702160d971e712Hiroshi Yamauchi switch (pm) { 2122565c2d9bce43c430d4267c82f5702160d971e712Hiroshi Yamauchi case kPageMapReleased: 2123565c2d9bce43c430d4267c82f5702160d971e712Hiroshi Yamauchi case kPageMapEmpty: 2124565c2d9bce43c430d4267c82f5702160d971e712Hiroshi Yamauchi ++i; 2125565c2d9bce43c430d4267c82f5702160d971e712Hiroshi Yamauchi break; 2126565c2d9bce43c430d4267c82f5702160d971e712Hiroshi Yamauchi case kPageMapLargeObject: { 2127565c2d9bce43c430d4267c82f5702160d971e712Hiroshi Yamauchi size_t num_pages = 1; 2128565c2d9bce43c430d4267c82f5702160d971e712Hiroshi Yamauchi size_t idx = i + 1; 2129565c2d9bce43c430d4267c82f5702160d971e712Hiroshi Yamauchi while (idx < page_map_size_ && page_map_[idx] == kPageMapLargeObjectPart) { 2130565c2d9bce43c430d4267c82f5702160d971e712Hiroshi Yamauchi num_pages++; 2131565c2d9bce43c430d4267c82f5702160d971e712Hiroshi Yamauchi idx++; 2132565c2d9bce43c430d4267c82f5702160d971e712Hiroshi Yamauchi } 2133565c2d9bce43c430d4267c82f5702160d971e712Hiroshi Yamauchi num_large_objects++; 2134565c2d9bce43c430d4267c82f5702160d971e712Hiroshi Yamauchi num_pages_large_objects += num_pages; 2135565c2d9bce43c430d4267c82f5702160d971e712Hiroshi Yamauchi i += num_pages; 2136565c2d9bce43c430d4267c82f5702160d971e712Hiroshi Yamauchi break; 2137565c2d9bce43c430d4267c82f5702160d971e712Hiroshi Yamauchi } 2138565c2d9bce43c430d4267c82f5702160d971e712Hiroshi Yamauchi case kPageMapLargeObjectPart: 2139565c2d9bce43c430d4267c82f5702160d971e712Hiroshi Yamauchi LOG(FATAL) << "Unreachable - page map type: " << static_cast<int>(pm) << std::endl 2140565c2d9bce43c430d4267c82f5702160d971e712Hiroshi Yamauchi << DumpPageMap(); 2141565c2d9bce43c430d4267c82f5702160d971e712Hiroshi Yamauchi break; 2142565c2d9bce43c430d4267c82f5702160d971e712Hiroshi Yamauchi case kPageMapRun: { 2143565c2d9bce43c430d4267c82f5702160d971e712Hiroshi Yamauchi Run* run = reinterpret_cast<Run*>(base_ + i * kPageSize); 2144565c2d9bce43c430d4267c82f5702160d971e712Hiroshi Yamauchi size_t idx = run->size_bracket_idx_; 2145565c2d9bce43c430d4267c82f5702160d971e712Hiroshi Yamauchi size_t num_pages = numOfPages[idx]; 2146565c2d9bce43c430d4267c82f5702160d971e712Hiroshi Yamauchi num_runs[idx]++; 2147565c2d9bce43c430d4267c82f5702160d971e712Hiroshi Yamauchi num_pages_runs[idx] += num_pages; 2148565c2d9bce43c430d4267c82f5702160d971e712Hiroshi Yamauchi num_slots[idx] += numOfSlots[idx]; 2149565c2d9bce43c430d4267c82f5702160d971e712Hiroshi Yamauchi size_t num_free_slots = run->NumberOfFreeSlots(); 2150565c2d9bce43c430d4267c82f5702160d971e712Hiroshi Yamauchi num_used_slots[idx] += numOfSlots[idx] - num_free_slots; 2151565c2d9bce43c430d4267c82f5702160d971e712Hiroshi Yamauchi num_metadata_bytes[idx] += headerSizes[idx]; 2152565c2d9bce43c430d4267c82f5702160d971e712Hiroshi Yamauchi i += num_pages; 2153565c2d9bce43c430d4267c82f5702160d971e712Hiroshi Yamauchi break; 2154565c2d9bce43c430d4267c82f5702160d971e712Hiroshi Yamauchi } 2155565c2d9bce43c430d4267c82f5702160d971e712Hiroshi Yamauchi case kPageMapRunPart: 2156565c2d9bce43c430d4267c82f5702160d971e712Hiroshi Yamauchi // Fall-through. 2157565c2d9bce43c430d4267c82f5702160d971e712Hiroshi Yamauchi default: 2158565c2d9bce43c430d4267c82f5702160d971e712Hiroshi Yamauchi LOG(FATAL) << "Unreachable - page map type: " << static_cast<int>(pm) << std::endl 2159565c2d9bce43c430d4267c82f5702160d971e712Hiroshi Yamauchi << DumpPageMap(); 2160565c2d9bce43c430d4267c82f5702160d971e712Hiroshi Yamauchi break; 2161565c2d9bce43c430d4267c82f5702160d971e712Hiroshi Yamauchi } 2162565c2d9bce43c430d4267c82f5702160d971e712Hiroshi Yamauchi } 2163565c2d9bce43c430d4267c82f5702160d971e712Hiroshi Yamauchi os << "RosAlloc stats:\n"; 2164565c2d9bce43c430d4267c82f5702160d971e712Hiroshi Yamauchi for (size_t i = 0; i < kNumOfSizeBrackets; ++i) { 2165565c2d9bce43c430d4267c82f5702160d971e712Hiroshi Yamauchi os << "Bracket " << i << " (" << bracketSizes[i] << "):" 2166565c2d9bce43c430d4267c82f5702160d971e712Hiroshi Yamauchi << " #runs=" << num_runs[i] 2167565c2d9bce43c430d4267c82f5702160d971e712Hiroshi Yamauchi << " #pages=" << num_pages_runs[i] 2168565c2d9bce43c430d4267c82f5702160d971e712Hiroshi Yamauchi << " (" << PrettySize(num_pages_runs[i] * kPageSize) << ")" 2169565c2d9bce43c430d4267c82f5702160d971e712Hiroshi Yamauchi << " #metadata_bytes=" << PrettySize(num_metadata_bytes[i]) 2170565c2d9bce43c430d4267c82f5702160d971e712Hiroshi Yamauchi << " #slots=" << num_slots[i] << " (" << PrettySize(num_slots[i] * bracketSizes[i]) << ")" 2171565c2d9bce43c430d4267c82f5702160d971e712Hiroshi Yamauchi << " #used_slots=" << num_used_slots[i] 2172565c2d9bce43c430d4267c82f5702160d971e712Hiroshi Yamauchi << " (" << PrettySize(num_used_slots[i] * bracketSizes[i]) << ")\n"; 2173565c2d9bce43c430d4267c82f5702160d971e712Hiroshi Yamauchi } 2174565c2d9bce43c430d4267c82f5702160d971e712Hiroshi Yamauchi os << "Large #allocations=" << num_large_objects 2175565c2d9bce43c430d4267c82f5702160d971e712Hiroshi Yamauchi << " #pages=" << num_pages_large_objects 2176565c2d9bce43c430d4267c82f5702160d971e712Hiroshi Yamauchi << " (" << PrettySize(num_pages_large_objects * kPageSize) << ")\n"; 2177565c2d9bce43c430d4267c82f5702160d971e712Hiroshi Yamauchi size_t total_num_pages = 0; 2178565c2d9bce43c430d4267c82f5702160d971e712Hiroshi Yamauchi size_t total_metadata_bytes = 0; 2179565c2d9bce43c430d4267c82f5702160d971e712Hiroshi Yamauchi size_t total_allocated_bytes = 0; 2180565c2d9bce43c430d4267c82f5702160d971e712Hiroshi Yamauchi for (size_t i = 0; i < kNumOfSizeBrackets; ++i) { 2181565c2d9bce43c430d4267c82f5702160d971e712Hiroshi Yamauchi total_num_pages += num_pages_runs[i]; 2182565c2d9bce43c430d4267c82f5702160d971e712Hiroshi Yamauchi total_metadata_bytes += num_metadata_bytes[i]; 2183565c2d9bce43c430d4267c82f5702160d971e712Hiroshi Yamauchi total_allocated_bytes += num_used_slots[i] * bracketSizes[i]; 2184565c2d9bce43c430d4267c82f5702160d971e712Hiroshi Yamauchi } 2185565c2d9bce43c430d4267c82f5702160d971e712Hiroshi Yamauchi total_num_pages += num_pages_large_objects; 2186565c2d9bce43c430d4267c82f5702160d971e712Hiroshi Yamauchi total_allocated_bytes += num_pages_large_objects * kPageSize; 2187565c2d9bce43c430d4267c82f5702160d971e712Hiroshi Yamauchi os << "Total #total_bytes=" << PrettySize(total_num_pages * kPageSize) 2188565c2d9bce43c430d4267c82f5702160d971e712Hiroshi Yamauchi << " #metadata_bytes=" << PrettySize(total_metadata_bytes) 2189565c2d9bce43c430d4267c82f5702160d971e712Hiroshi Yamauchi << " #used_bytes=" << PrettySize(total_allocated_bytes) << "\n"; 2190565c2d9bce43c430d4267c82f5702160d971e712Hiroshi Yamauchi os << "\n"; 2191565c2d9bce43c430d4267c82f5702160d971e712Hiroshi Yamauchi} 2192565c2d9bce43c430d4267c82f5702160d971e712Hiroshi Yamauchi 2193cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi} // namespace allocator 2194cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi} // namespace gc 2195cf58d4adf461eb9b8e84baa8019054c88cd8acc6Hiroshi Yamauchi} // namespace art 2196