13c2856e939f3daa7a95a1f8cb70f47e7a621db3cHiroshi Yamauchi/* 23c2856e939f3daa7a95a1f8cb70f47e7a621db3cHiroshi Yamauchi * Copyright (C) 2013 The Android Open Source Project 33c2856e939f3daa7a95a1f8cb70f47e7a621db3cHiroshi Yamauchi * 43c2856e939f3daa7a95a1f8cb70f47e7a621db3cHiroshi Yamauchi * Licensed under the Apache License, Version 2.0 (the "License"); 53c2856e939f3daa7a95a1f8cb70f47e7a621db3cHiroshi Yamauchi * you may not use this file except in compliance with the License. 63c2856e939f3daa7a95a1f8cb70f47e7a621db3cHiroshi Yamauchi * You may obtain a copy of the License at 73c2856e939f3daa7a95a1f8cb70f47e7a621db3cHiroshi Yamauchi * 83c2856e939f3daa7a95a1f8cb70f47e7a621db3cHiroshi Yamauchi * http://www.apache.org/licenses/LICENSE-2.0 93c2856e939f3daa7a95a1f8cb70f47e7a621db3cHiroshi Yamauchi * 103c2856e939f3daa7a95a1f8cb70f47e7a621db3cHiroshi Yamauchi * Unless required by applicable law or agreed to in writing, software 113c2856e939f3daa7a95a1f8cb70f47e7a621db3cHiroshi Yamauchi * distributed under the License is distributed on an "AS IS" BASIS, 123c2856e939f3daa7a95a1f8cb70f47e7a621db3cHiroshi Yamauchi * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 133c2856e939f3daa7a95a1f8cb70f47e7a621db3cHiroshi Yamauchi * See the License for the specific language governing permissions and 143c2856e939f3daa7a95a1f8cb70f47e7a621db3cHiroshi Yamauchi * limitations under the License. 153c2856e939f3daa7a95a1f8cb70f47e7a621db3cHiroshi Yamauchi */ 163c2856e939f3daa7a95a1f8cb70f47e7a621db3cHiroshi Yamauchi 173c2856e939f3daa7a95a1f8cb70f47e7a621db3cHiroshi Yamauchi#ifndef ART_RUNTIME_GC_ALLOCATOR_ROSALLOC_INL_H_ 183c2856e939f3daa7a95a1f8cb70f47e7a621db3cHiroshi Yamauchi#define ART_RUNTIME_GC_ALLOCATOR_ROSALLOC_INL_H_ 193c2856e939f3daa7a95a1f8cb70f47e7a621db3cHiroshi Yamauchi 203c2856e939f3daa7a95a1f8cb70f47e7a621db3cHiroshi Yamauchi#include "rosalloc.h" 213c2856e939f3daa7a95a1f8cb70f47e7a621db3cHiroshi Yamauchi 223c2856e939f3daa7a95a1f8cb70f47e7a621db3cHiroshi Yamauchinamespace art { 233c2856e939f3daa7a95a1f8cb70f47e7a621db3cHiroshi Yamauchinamespace gc { 243c2856e939f3daa7a95a1f8cb70f47e7a621db3cHiroshi Yamauchinamespace allocator { 253c2856e939f3daa7a95a1f8cb70f47e7a621db3cHiroshi Yamauchi 26d7576328811e5103e99d31f834a857522cc1463fAndreas Gampeinline ALWAYS_INLINE bool RosAlloc::ShouldCheckZeroMemory() { 271e13374baf7dfaf442ffbf9809c37c131d681eafEvgenii Stepanov return kCheckZeroMemory && !is_running_on_memory_tool_; 28d7576328811e5103e99d31f834a857522cc1463fAndreas Gampe} 29d7576328811e5103e99d31f834a857522cc1463fAndreas Gampe 300651d41e41341fb2e9ef3ee41dc1f1bfc832dbbbMathieu Chartiertemplate<bool kThreadSafe> 314460a84be92b5a94ecfb5c650aef4945ab849c93Hiroshi Yamauchiinline ALWAYS_INLINE void* RosAlloc::Alloc(Thread* self, size_t size, size_t* bytes_allocated, 324460a84be92b5a94ecfb5c650aef4945ab849c93Hiroshi Yamauchi size_t* usable_size, 334460a84be92b5a94ecfb5c650aef4945ab849c93Hiroshi Yamauchi size_t* bytes_tl_bulk_allocated) { 343c2856e939f3daa7a95a1f8cb70f47e7a621db3cHiroshi Yamauchi if (UNLIKELY(size > kLargeSizeThreshold)) { 354460a84be92b5a94ecfb5c650aef4945ab849c93Hiroshi Yamauchi return AllocLargeObject(self, size, bytes_allocated, usable_size, 364460a84be92b5a94ecfb5c650aef4945ab849c93Hiroshi Yamauchi bytes_tl_bulk_allocated); 373c2856e939f3daa7a95a1f8cb70f47e7a621db3cHiroshi Yamauchi } 380651d41e41341fb2e9ef3ee41dc1f1bfc832dbbbMathieu Chartier void* m; 390651d41e41341fb2e9ef3ee41dc1f1bfc832dbbbMathieu Chartier if (kThreadSafe) { 404460a84be92b5a94ecfb5c650aef4945ab849c93Hiroshi Yamauchi m = AllocFromRun(self, size, bytes_allocated, usable_size, bytes_tl_bulk_allocated); 410651d41e41341fb2e9ef3ee41dc1f1bfc832dbbbMathieu Chartier } else { 424460a84be92b5a94ecfb5c650aef4945ab849c93Hiroshi Yamauchi m = AllocFromRunThreadUnsafe(self, size, bytes_allocated, usable_size, 434460a84be92b5a94ecfb5c650aef4945ab849c93Hiroshi Yamauchi bytes_tl_bulk_allocated); 440651d41e41341fb2e9ef3ee41dc1f1bfc832dbbbMathieu Chartier } 453c2856e939f3daa7a95a1f8cb70f47e7a621db3cHiroshi Yamauchi // Check if the returned memory is really all zero. 46d7576328811e5103e99d31f834a857522cc1463fAndreas Gampe if (ShouldCheckZeroMemory() && m != nullptr) { 4713735955f39b3b304c37d2b2840663c131262c18Ian Rogers uint8_t* bytes = reinterpret_cast<uint8_t*>(m); 483c2856e939f3daa7a95a1f8cb70f47e7a621db3cHiroshi Yamauchi for (size_t i = 0; i < size; ++i) { 493c2856e939f3daa7a95a1f8cb70f47e7a621db3cHiroshi Yamauchi DCHECK_EQ(bytes[i], 0); 503c2856e939f3daa7a95a1f8cb70f47e7a621db3cHiroshi Yamauchi } 513c2856e939f3daa7a95a1f8cb70f47e7a621db3cHiroshi Yamauchi } 523c2856e939f3daa7a95a1f8cb70f47e7a621db3cHiroshi Yamauchi return m; 533c2856e939f3daa7a95a1f8cb70f47e7a621db3cHiroshi Yamauchi} 543c2856e939f3daa7a95a1f8cb70f47e7a621db3cHiroshi Yamauchi 554460a84be92b5a94ecfb5c650aef4945ab849c93Hiroshi Yamauchiinline bool RosAlloc::Run::IsFull() { 5631bf42c48c4d00f0677c31264bba8d21618dae67Hiroshi Yamauchi return free_list_.Size() == 0; 574460a84be92b5a94ecfb5c650aef4945ab849c93Hiroshi Yamauchi} 584460a84be92b5a94ecfb5c650aef4945ab849c93Hiroshi Yamauchi 594460a84be92b5a94ecfb5c650aef4945ab849c93Hiroshi Yamauchiinline bool RosAlloc::CanAllocFromThreadLocalRun(Thread* self, size_t size) { 604460a84be92b5a94ecfb5c650aef4945ab849c93Hiroshi Yamauchi if (UNLIKELY(!IsSizeForThreadLocal(size))) { 614460a84be92b5a94ecfb5c650aef4945ab849c93Hiroshi Yamauchi return false; 624460a84be92b5a94ecfb5c650aef4945ab849c93Hiroshi Yamauchi } 634460a84be92b5a94ecfb5c650aef4945ab849c93Hiroshi Yamauchi size_t bracket_size; 644460a84be92b5a94ecfb5c650aef4945ab849c93Hiroshi Yamauchi size_t idx = SizeToIndexAndBracketSize(size, &bracket_size); 654460a84be92b5a94ecfb5c650aef4945ab849c93Hiroshi Yamauchi DCHECK_LT(idx, kNumThreadLocalSizeBrackets); 664460a84be92b5a94ecfb5c650aef4945ab849c93Hiroshi Yamauchi Run* thread_local_run = reinterpret_cast<Run*>(self->GetRosAllocRun(idx)); 674460a84be92b5a94ecfb5c650aef4945ab849c93Hiroshi Yamauchi if (kIsDebugBuild) { 684460a84be92b5a94ecfb5c650aef4945ab849c93Hiroshi Yamauchi // Need the lock to prevent race conditions. 694460a84be92b5a94ecfb5c650aef4945ab849c93Hiroshi Yamauchi MutexLock mu(self, *size_bracket_locks_[idx]); 704460a84be92b5a94ecfb5c650aef4945ab849c93Hiroshi Yamauchi CHECK(non_full_runs_[idx].find(thread_local_run) == non_full_runs_[idx].end()); 714460a84be92b5a94ecfb5c650aef4945ab849c93Hiroshi Yamauchi CHECK(full_runs_[idx].find(thread_local_run) == full_runs_[idx].end()); 724460a84be92b5a94ecfb5c650aef4945ab849c93Hiroshi Yamauchi } 734460a84be92b5a94ecfb5c650aef4945ab849c93Hiroshi Yamauchi DCHECK(thread_local_run != nullptr); 744460a84be92b5a94ecfb5c650aef4945ab849c93Hiroshi Yamauchi DCHECK(thread_local_run->IsThreadLocal() || thread_local_run == dedicated_full_run_); 754460a84be92b5a94ecfb5c650aef4945ab849c93Hiroshi Yamauchi return !thread_local_run->IsFull(); 764460a84be92b5a94ecfb5c650aef4945ab849c93Hiroshi Yamauchi} 774460a84be92b5a94ecfb5c650aef4945ab849c93Hiroshi Yamauchi 784460a84be92b5a94ecfb5c650aef4945ab849c93Hiroshi Yamauchiinline void* RosAlloc::AllocFromThreadLocalRun(Thread* self, size_t size, 794460a84be92b5a94ecfb5c650aef4945ab849c93Hiroshi Yamauchi size_t* bytes_allocated) { 804460a84be92b5a94ecfb5c650aef4945ab849c93Hiroshi Yamauchi DCHECK(bytes_allocated != nullptr); 814460a84be92b5a94ecfb5c650aef4945ab849c93Hiroshi Yamauchi if (UNLIKELY(!IsSizeForThreadLocal(size))) { 824460a84be92b5a94ecfb5c650aef4945ab849c93Hiroshi Yamauchi return nullptr; 834460a84be92b5a94ecfb5c650aef4945ab849c93Hiroshi Yamauchi } 844460a84be92b5a94ecfb5c650aef4945ab849c93Hiroshi Yamauchi size_t bracket_size; 854460a84be92b5a94ecfb5c650aef4945ab849c93Hiroshi Yamauchi size_t idx = SizeToIndexAndBracketSize(size, &bracket_size); 864460a84be92b5a94ecfb5c650aef4945ab849c93Hiroshi Yamauchi Run* thread_local_run = reinterpret_cast<Run*>(self->GetRosAllocRun(idx)); 874460a84be92b5a94ecfb5c650aef4945ab849c93Hiroshi Yamauchi if (kIsDebugBuild) { 884460a84be92b5a94ecfb5c650aef4945ab849c93Hiroshi Yamauchi // Need the lock to prevent race conditions. 894460a84be92b5a94ecfb5c650aef4945ab849c93Hiroshi Yamauchi MutexLock mu(self, *size_bracket_locks_[idx]); 904460a84be92b5a94ecfb5c650aef4945ab849c93Hiroshi Yamauchi CHECK(non_full_runs_[idx].find(thread_local_run) == non_full_runs_[idx].end()); 914460a84be92b5a94ecfb5c650aef4945ab849c93Hiroshi Yamauchi CHECK(full_runs_[idx].find(thread_local_run) == full_runs_[idx].end()); 924460a84be92b5a94ecfb5c650aef4945ab849c93Hiroshi Yamauchi } 934460a84be92b5a94ecfb5c650aef4945ab849c93Hiroshi Yamauchi DCHECK(thread_local_run != nullptr); 944460a84be92b5a94ecfb5c650aef4945ab849c93Hiroshi Yamauchi DCHECK(thread_local_run->IsThreadLocal() || thread_local_run == dedicated_full_run_); 954460a84be92b5a94ecfb5c650aef4945ab849c93Hiroshi Yamauchi void* slot_addr = thread_local_run->AllocSlot(); 964460a84be92b5a94ecfb5c650aef4945ab849c93Hiroshi Yamauchi if (LIKELY(slot_addr != nullptr)) { 974460a84be92b5a94ecfb5c650aef4945ab849c93Hiroshi Yamauchi *bytes_allocated = bracket_size; 984460a84be92b5a94ecfb5c650aef4945ab849c93Hiroshi Yamauchi } 994460a84be92b5a94ecfb5c650aef4945ab849c93Hiroshi Yamauchi return slot_addr; 1004460a84be92b5a94ecfb5c650aef4945ab849c93Hiroshi Yamauchi} 1014460a84be92b5a94ecfb5c650aef4945ab849c93Hiroshi Yamauchi 1024460a84be92b5a94ecfb5c650aef4945ab849c93Hiroshi Yamauchiinline size_t RosAlloc::MaxBytesBulkAllocatedFor(size_t size) { 1034460a84be92b5a94ecfb5c650aef4945ab849c93Hiroshi Yamauchi if (UNLIKELY(!IsSizeForThreadLocal(size))) { 1044460a84be92b5a94ecfb5c650aef4945ab849c93Hiroshi Yamauchi return size; 1054460a84be92b5a94ecfb5c650aef4945ab849c93Hiroshi Yamauchi } 1064460a84be92b5a94ecfb5c650aef4945ab849c93Hiroshi Yamauchi size_t bracket_size; 1074460a84be92b5a94ecfb5c650aef4945ab849c93Hiroshi Yamauchi size_t idx = SizeToIndexAndBracketSize(size, &bracket_size); 1084460a84be92b5a94ecfb5c650aef4945ab849c93Hiroshi Yamauchi return numOfSlots[idx] * bracket_size; 1094460a84be92b5a94ecfb5c650aef4945ab849c93Hiroshi Yamauchi} 1104460a84be92b5a94ecfb5c650aef4945ab849c93Hiroshi Yamauchi 1114460a84be92b5a94ecfb5c650aef4945ab849c93Hiroshi Yamauchiinline void* RosAlloc::Run::AllocSlot() { 11231bf42c48c4d00f0677c31264bba8d21618dae67Hiroshi Yamauchi Slot* slot = free_list_.Remove(); 11331bf42c48c4d00f0677c31264bba8d21618dae67Hiroshi Yamauchi if (kTraceRosAlloc && slot != nullptr) { 11431bf42c48c4d00f0677c31264bba8d21618dae67Hiroshi Yamauchi const uint8_t idx = size_bracket_idx_; 11531bf42c48c4d00f0677c31264bba8d21618dae67Hiroshi Yamauchi LOG(INFO) << "RosAlloc::Run::AllocSlot() : " << slot 11631bf42c48c4d00f0677c31264bba8d21618dae67Hiroshi Yamauchi << ", bracket_size=" << std::dec << bracketSizes[idx] 11731bf42c48c4d00f0677c31264bba8d21618dae67Hiroshi Yamauchi << ", slot_idx=" << SlotIndex(slot); 1184460a84be92b5a94ecfb5c650aef4945ab849c93Hiroshi Yamauchi } 11931bf42c48c4d00f0677c31264bba8d21618dae67Hiroshi Yamauchi return slot; 1204460a84be92b5a94ecfb5c650aef4945ab849c93Hiroshi Yamauchi} 1214460a84be92b5a94ecfb5c650aef4945ab849c93Hiroshi Yamauchi 1223c2856e939f3daa7a95a1f8cb70f47e7a621db3cHiroshi Yamauchi} // namespace allocator 1233c2856e939f3daa7a95a1f8cb70f47e7a621db3cHiroshi Yamauchi} // namespace gc 1243c2856e939f3daa7a95a1f8cb70f47e7a621db3cHiroshi Yamauchi} // namespace art 1253c2856e939f3daa7a95a1f8cb70f47e7a621db3cHiroshi Yamauchi 1263c2856e939f3daa7a95a1f8cb70f47e7a621db3cHiroshi Yamauchi#endif // ART_RUNTIME_GC_ALLOCATOR_ROSALLOC_INL_H_ 127