1
2/*
3 * Copyright (C) 2013 The Android Open Source Project
4 *
5 * Licensed under the Apache License, Version 2.0 (the "License");
6 * you may not use this file except in compliance with the License.
7 * You may obtain a copy of the License at
8 *
9 *      http://www.apache.org/licenses/LICENSE-2.0
10 *
11 * Unless required by applicable law or agreed to in writing, software
12 * distributed under the License is distributed on an "AS IS" BASIS,
13 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 * See the License for the specific language governing permissions and
15 * limitations under the License.
16 */
17
18#include "rosalloc_space-inl.h"
19
20#include "gc/accounting/card_table.h"
21#include "gc/accounting/space_bitmap-inl.h"
22#include "gc/heap.h"
23#include "mirror/class-inl.h"
24#include "mirror/object-inl.h"
25#include "runtime.h"
26#include "thread.h"
27#include "thread_list.h"
28#include "utils.h"
29#include "valgrind_malloc_space-inl.h"
30
31namespace art {
32namespace gc {
33namespace space {
34
35static constexpr bool kPrefetchDuringRosAllocFreeList = false;
36static constexpr size_t kPrefetchLookAhead = 8;
37// Use this only for verification, it is not safe to use since the class of the object may have
38// been freed.
39static constexpr bool kVerifyFreedBytes = false;
40
41// TODO: Fix
42// template class ValgrindMallocSpace<RosAllocSpace, allocator::RosAlloc*>;
43
44RosAllocSpace::RosAllocSpace(const std::string& name, MemMap* mem_map,
45                             art::gc::allocator::RosAlloc* rosalloc, byte* begin, byte* end,
46                             byte* limit, size_t growth_limit, bool can_move_objects,
47                             size_t starting_size, size_t initial_size, bool low_memory_mode)
48    : MallocSpace(name, mem_map, begin, end, limit, growth_limit, true, can_move_objects,
49                  starting_size, initial_size),
50      rosalloc_(rosalloc), low_memory_mode_(low_memory_mode) {
51  CHECK(rosalloc != nullptr);
52}
53
54RosAllocSpace* RosAllocSpace::CreateFromMemMap(MemMap* mem_map, const std::string& name,
55                                               size_t starting_size, size_t initial_size,
56                                               size_t growth_limit, size_t capacity,
57                                               bool low_memory_mode, bool can_move_objects) {
58  DCHECK(mem_map != nullptr);
59  allocator::RosAlloc* rosalloc = CreateRosAlloc(mem_map->Begin(), starting_size, initial_size,
60                                                 capacity, low_memory_mode);
61  if (rosalloc == NULL) {
62    LOG(ERROR) << "Failed to initialize rosalloc for alloc space (" << name << ")";
63    return NULL;
64  }
65
66  // Protect memory beyond the starting size. MoreCore will add r/w permissions when necessory
67  byte* end = mem_map->Begin() + starting_size;
68  if (capacity - starting_size > 0) {
69    CHECK_MEMORY_CALL(mprotect, (end, capacity - starting_size, PROT_NONE), name);
70  }
71
72  // Everything is set so record in immutable structure and leave
73  byte* begin = mem_map->Begin();
74  // TODO: Fix RosAllocSpace to support valgrind. There is currently some issues with
75  // AllocationSize caused by redzones. b/12944686
76  if (false && Runtime::Current()->GetHeap()->RunningOnValgrind()) {
77    LOG(FATAL) << "Unimplemented";
78  } else {
79    return new RosAllocSpace(name, mem_map, rosalloc, begin, end, begin + capacity, growth_limit,
80                             can_move_objects, starting_size, initial_size, low_memory_mode);
81  }
82}
83
84RosAllocSpace::~RosAllocSpace() {
85  delete rosalloc_;
86}
87
88RosAllocSpace* RosAllocSpace::Create(const std::string& name, size_t initial_size,
89                                     size_t growth_limit, size_t capacity, byte* requested_begin,
90                                     bool low_memory_mode, bool can_move_objects) {
91  uint64_t start_time = 0;
92  if (VLOG_IS_ON(heap) || VLOG_IS_ON(startup)) {
93    start_time = NanoTime();
94    VLOG(startup) << "RosAllocSpace::Create entering " << name
95                  << " initial_size=" << PrettySize(initial_size)
96                  << " growth_limit=" << PrettySize(growth_limit)
97                  << " capacity=" << PrettySize(capacity)
98                  << " requested_begin=" << reinterpret_cast<void*>(requested_begin);
99  }
100
101  // Memory we promise to rosalloc before it asks for morecore.
102  // Note: making this value large means that large allocations are unlikely to succeed as rosalloc
103  // will ask for this memory from sys_alloc which will fail as the footprint (this value plus the
104  // size of the large allocation) will be greater than the footprint limit.
105  size_t starting_size = Heap::kDefaultStartingSize;
106  MemMap* mem_map = CreateMemMap(name, starting_size, &initial_size, &growth_limit, &capacity,
107                                 requested_begin);
108  if (mem_map == NULL) {
109    LOG(ERROR) << "Failed to create mem map for alloc space (" << name << ") of size "
110               << PrettySize(capacity);
111    return NULL;
112  }
113
114  RosAllocSpace* space = CreateFromMemMap(mem_map, name, starting_size, initial_size,
115                                          growth_limit, capacity, low_memory_mode,
116                                          can_move_objects);
117  // We start out with only the initial size possibly containing objects.
118  if (VLOG_IS_ON(heap) || VLOG_IS_ON(startup)) {
119    LOG(INFO) << "RosAllocSpace::Create exiting (" << PrettyDuration(NanoTime() - start_time)
120        << " ) " << *space;
121  }
122  return space;
123}
124
125allocator::RosAlloc* RosAllocSpace::CreateRosAlloc(void* begin, size_t morecore_start,
126                                                   size_t initial_size,
127                                                   size_t maximum_size, bool low_memory_mode) {
128  // clear errno to allow PLOG on error
129  errno = 0;
130  // create rosalloc using our backing storage starting at begin and
131  // with a footprint of morecore_start. When morecore_start bytes of
132  // memory is exhaused morecore will be called.
133  allocator::RosAlloc* rosalloc = new art::gc::allocator::RosAlloc(
134      begin, morecore_start, maximum_size,
135      low_memory_mode ?
136          art::gc::allocator::RosAlloc::kPageReleaseModeAll :
137          art::gc::allocator::RosAlloc::kPageReleaseModeSizeAndEnd);
138  if (rosalloc != NULL) {
139    rosalloc->SetFootprintLimit(initial_size);
140  } else {
141    PLOG(ERROR) << "RosAlloc::Create failed";
142  }
143  return rosalloc;
144}
145
146mirror::Object* RosAllocSpace::AllocWithGrowth(Thread* self, size_t num_bytes,
147                                               size_t* bytes_allocated, size_t* usable_size) {
148  mirror::Object* result;
149  {
150    MutexLock mu(self, lock_);
151    // Grow as much as possible within the space.
152    size_t max_allowed = Capacity();
153    rosalloc_->SetFootprintLimit(max_allowed);
154    // Try the allocation.
155    result = AllocCommon(self, num_bytes, bytes_allocated, usable_size);
156    // Shrink back down as small as possible.
157    size_t footprint = rosalloc_->Footprint();
158    rosalloc_->SetFootprintLimit(footprint);
159  }
160  // Note RosAlloc zeroes memory internally.
161  // Return the new allocation or NULL.
162  CHECK(!kDebugSpaces || result == nullptr || Contains(result));
163  return result;
164}
165
166MallocSpace* RosAllocSpace::CreateInstance(const std::string& name, MemMap* mem_map, void* allocator,
167                                           byte* begin, byte* end, byte* limit, size_t growth_limit,
168                                           bool can_move_objects) {
169  return new RosAllocSpace(name, mem_map, reinterpret_cast<allocator::RosAlloc*>(allocator),
170                           begin, end, limit, growth_limit, can_move_objects, starting_size_,
171                           initial_size_, low_memory_mode_);
172}
173
174size_t RosAllocSpace::Free(Thread* self, mirror::Object* ptr) {
175  if (kDebugSpaces) {
176    CHECK(ptr != NULL);
177    CHECK(Contains(ptr)) << "Free (" << ptr << ") not in bounds of heap " << *this;
178  }
179  if (kRecentFreeCount > 0) {
180    MutexLock mu(self, lock_);
181    RegisterRecentFree(ptr);
182  }
183  return rosalloc_->Free(self, ptr);
184}
185
186size_t RosAllocSpace::FreeList(Thread* self, size_t num_ptrs, mirror::Object** ptrs) {
187  DCHECK(ptrs != nullptr);
188
189  size_t verify_bytes = 0;
190  for (size_t i = 0; i < num_ptrs; i++) {
191    if (kPrefetchDuringRosAllocFreeList && i + kPrefetchLookAhead < num_ptrs) {
192      __builtin_prefetch(reinterpret_cast<char*>(ptrs[i + kPrefetchLookAhead]));
193    }
194    if (kVerifyFreedBytes) {
195      verify_bytes += AllocationSizeNonvirtual(ptrs[i], nullptr);
196    }
197  }
198
199  if (kRecentFreeCount > 0) {
200    MutexLock mu(self, lock_);
201    for (size_t i = 0; i < num_ptrs; i++) {
202      RegisterRecentFree(ptrs[i]);
203    }
204  }
205
206  if (kDebugSpaces) {
207    size_t num_broken_ptrs = 0;
208    for (size_t i = 0; i < num_ptrs; i++) {
209      if (!Contains(ptrs[i])) {
210        num_broken_ptrs++;
211        LOG(ERROR) << "FreeList[" << i << "] (" << ptrs[i] << ") not in bounds of heap " << *this;
212      } else {
213        size_t size = rosalloc_->UsableSize(ptrs[i]);
214        memset(ptrs[i], 0xEF, size);
215      }
216    }
217    CHECK_EQ(num_broken_ptrs, 0u);
218  }
219
220  const size_t bytes_freed = rosalloc_->BulkFree(self, reinterpret_cast<void**>(ptrs), num_ptrs);
221  if (kVerifyFreedBytes) {
222    CHECK_EQ(verify_bytes, bytes_freed);
223  }
224  return bytes_freed;
225}
226
227// Callback from rosalloc when it needs to increase the footprint
228extern "C" void* art_heap_rosalloc_morecore(allocator::RosAlloc* rosalloc, intptr_t increment) {
229  Heap* heap = Runtime::Current()->GetHeap();
230  RosAllocSpace* rosalloc_space = heap->GetRosAllocSpace(rosalloc);
231  DCHECK(rosalloc_space != nullptr);
232  DCHECK_EQ(rosalloc_space->GetRosAlloc(), rosalloc);
233  return rosalloc_space->MoreCore(increment);
234}
235
236size_t RosAllocSpace::Trim() {
237  VLOG(heap) << "RosAllocSpace::Trim() ";
238  {
239    MutexLock mu(Thread::Current(), lock_);
240    // Trim to release memory at the end of the space.
241    rosalloc_->Trim();
242  }
243  // Attempt to release pages if it does not release all empty pages.
244  if (!rosalloc_->DoesReleaseAllPages()) {
245    return rosalloc_->ReleasePages();
246  }
247  return 0;
248}
249
250void RosAllocSpace::Walk(void(*callback)(void *start, void *end, size_t num_bytes, void* callback_arg),
251                         void* arg) {
252  InspectAllRosAlloc(callback, arg, true);
253}
254
255size_t RosAllocSpace::GetFootprint() {
256  MutexLock mu(Thread::Current(), lock_);
257  return rosalloc_->Footprint();
258}
259
260size_t RosAllocSpace::GetFootprintLimit() {
261  MutexLock mu(Thread::Current(), lock_);
262  return rosalloc_->FootprintLimit();
263}
264
265void RosAllocSpace::SetFootprintLimit(size_t new_size) {
266  MutexLock mu(Thread::Current(), lock_);
267  VLOG(heap) << "RosAllocSpace::SetFootprintLimit " << PrettySize(new_size);
268  // Compare against the actual footprint, rather than the Size(), because the heap may not have
269  // grown all the way to the allowed size yet.
270  size_t current_space_size = rosalloc_->Footprint();
271  if (new_size < current_space_size) {
272    // Don't let the space grow any more.
273    new_size = current_space_size;
274  }
275  rosalloc_->SetFootprintLimit(new_size);
276}
277
278uint64_t RosAllocSpace::GetBytesAllocated() {
279  size_t bytes_allocated = 0;
280  InspectAllRosAlloc(art::gc::allocator::RosAlloc::BytesAllocatedCallback, &bytes_allocated, false);
281  return bytes_allocated;
282}
283
284uint64_t RosAllocSpace::GetObjectsAllocated() {
285  size_t objects_allocated = 0;
286  InspectAllRosAlloc(art::gc::allocator::RosAlloc::ObjectsAllocatedCallback, &objects_allocated, false);
287  return objects_allocated;
288}
289
290void RosAllocSpace::InspectAllRosAllocWithSuspendAll(
291    void (*callback)(void *start, void *end, size_t num_bytes, void* callback_arg),
292    void* arg, bool do_null_callback_at_end) NO_THREAD_SAFETY_ANALYSIS {
293  // TODO: NO_THREAD_SAFETY_ANALYSIS.
294  Thread* self = Thread::Current();
295  ThreadList* tl = Runtime::Current()->GetThreadList();
296  tl->SuspendAll();
297  {
298    MutexLock mu(self, *Locks::runtime_shutdown_lock_);
299    MutexLock mu2(self, *Locks::thread_list_lock_);
300    rosalloc_->InspectAll(callback, arg);
301    if (do_null_callback_at_end) {
302      callback(NULL, NULL, 0, arg);  // Indicate end of a space.
303    }
304  }
305  tl->ResumeAll();
306}
307
308void RosAllocSpace::InspectAllRosAlloc(void (*callback)(void *start, void *end, size_t num_bytes, void* callback_arg),
309                                       void* arg, bool do_null_callback_at_end) NO_THREAD_SAFETY_ANALYSIS {
310  // TODO: NO_THREAD_SAFETY_ANALYSIS.
311  Thread* self = Thread::Current();
312  if (Locks::mutator_lock_->IsExclusiveHeld(self)) {
313    // The mutators are already suspended. For example, a call path
314    // from SignalCatcher::HandleSigQuit().
315    rosalloc_->InspectAll(callback, arg);
316    if (do_null_callback_at_end) {
317      callback(NULL, NULL, 0, arg);  // Indicate end of a space.
318    }
319  } else if (Locks::mutator_lock_->IsSharedHeld(self)) {
320    // The mutators are not suspended yet and we have a shared access
321    // to the mutator lock. Temporarily release the shared access by
322    // transitioning to the suspend state, and suspend the mutators.
323    self->TransitionFromRunnableToSuspended(kSuspended);
324    InspectAllRosAllocWithSuspendAll(callback, arg, do_null_callback_at_end);
325    self->TransitionFromSuspendedToRunnable();
326    Locks::mutator_lock_->AssertSharedHeld(self);
327  } else {
328    // The mutators are not suspended yet. Suspend the mutators.
329    InspectAllRosAllocWithSuspendAll(callback, arg, do_null_callback_at_end);
330  }
331}
332
333void RosAllocSpace::RevokeThreadLocalBuffers(Thread* thread) {
334  rosalloc_->RevokeThreadLocalRuns(thread);
335}
336
337void RosAllocSpace::RevokeAllThreadLocalBuffers() {
338  rosalloc_->RevokeAllThreadLocalRuns();
339}
340
341void RosAllocSpace::AssertAllThreadLocalBuffersAreRevoked() {
342  if (kIsDebugBuild) {
343    rosalloc_->AssertAllThreadLocalRunsAreRevoked();
344  }
345}
346
347void RosAllocSpace::Clear() {
348  size_t footprint_limit = GetFootprintLimit();
349  madvise(GetMemMap()->Begin(), GetMemMap()->Size(), MADV_DONTNEED);
350  live_bitmap_->Clear();
351  mark_bitmap_->Clear();
352  SetEnd(begin_ + starting_size_);
353  delete rosalloc_;
354  rosalloc_ = CreateRosAlloc(mem_map_->Begin(), starting_size_, initial_size_, Capacity(),
355                             low_memory_mode_);
356  SetFootprintLimit(footprint_limit);
357}
358
359}  // namespace space
360}  // namespace gc
361}  // namespace art
362