1/*
2 * Copyright (C) 2012 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *      http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17#include "entrypoints/quick/quick_alloc_entrypoints.h"
18
19#include "art_method-inl.h"
20#include "callee_save_frame.h"
21#include "entrypoints/entrypoint_utils-inl.h"
22#include "mirror/class-inl.h"
23#include "mirror/object_array-inl.h"
24#include "mirror/object-inl.h"
25
26namespace art {
27
28static constexpr bool kUseTlabFastPath = true;
29
30#define GENERATE_ENTRYPOINTS_FOR_ALLOCATOR_INST(suffix, suffix2, instrumented_bool, allocator_type) \
31extern "C" mirror::Object* artAllocObjectFromCode ##suffix##suffix2( \
32    uint32_t type_idx, ArtMethod* method, Thread* self) \
33    SHARED_REQUIRES(Locks::mutator_lock_) { \
34  ScopedQuickEntrypointChecks sqec(self); \
35  if (kUseTlabFastPath && !instrumented_bool && allocator_type == gc::kAllocatorTypeTLAB) { \
36    mirror::Class* klass = method->GetDexCacheResolvedType<false>(type_idx, sizeof(void*)); \
37    if (LIKELY(klass != nullptr && klass->IsInitialized() && !klass->IsFinalizable())) { \
38      size_t byte_count = klass->GetObjectSize(); \
39      byte_count = RoundUp(byte_count, gc::space::BumpPointerSpace::kAlignment); \
40      mirror::Object* obj; \
41      if (LIKELY(byte_count < self->TlabSize())) { \
42        obj = self->AllocTlab(byte_count); \
43        DCHECK(obj != nullptr) << "AllocTlab can't fail"; \
44        obj->SetClass(klass); \
45        if (kUseBakerOrBrooksReadBarrier) { \
46          if (kUseBrooksReadBarrier) { \
47            obj->SetReadBarrierPointer(obj); \
48          } \
49          obj->AssertReadBarrierPointer(); \
50        } \
51        QuasiAtomic::ThreadFenceForConstructor(); \
52        return obj; \
53      } \
54    } \
55  } \
56  return AllocObjectFromCode<false, instrumented_bool>(type_idx, method, self, allocator_type); \
57} \
58extern "C" mirror::Object* artAllocObjectFromCodeResolved##suffix##suffix2( \
59    mirror::Class* klass, ArtMethod* method ATTRIBUTE_UNUSED, Thread* self) \
60    SHARED_REQUIRES(Locks::mutator_lock_) { \
61  ScopedQuickEntrypointChecks sqec(self); \
62  if (kUseTlabFastPath && !instrumented_bool && allocator_type == gc::kAllocatorTypeTLAB) { \
63    if (LIKELY(klass->IsInitialized())) { \
64      size_t byte_count = klass->GetObjectSize(); \
65      byte_count = RoundUp(byte_count, gc::space::BumpPointerSpace::kAlignment); \
66      mirror::Object* obj; \
67      if (LIKELY(byte_count < self->TlabSize())) { \
68        obj = self->AllocTlab(byte_count); \
69        DCHECK(obj != nullptr) << "AllocTlab can't fail"; \
70        obj->SetClass(klass); \
71        if (kUseBakerOrBrooksReadBarrier) { \
72          if (kUseBrooksReadBarrier) { \
73            obj->SetReadBarrierPointer(obj); \
74          } \
75          obj->AssertReadBarrierPointer(); \
76        } \
77        QuasiAtomic::ThreadFenceForConstructor(); \
78        return obj; \
79      } \
80    } \
81  } \
82  return AllocObjectFromCodeResolved<instrumented_bool>(klass, self, allocator_type); \
83} \
84extern "C" mirror::Object* artAllocObjectFromCodeInitialized##suffix##suffix2( \
85    mirror::Class* klass, ArtMethod* method ATTRIBUTE_UNUSED, Thread* self) \
86    SHARED_REQUIRES(Locks::mutator_lock_) { \
87  ScopedQuickEntrypointChecks sqec(self); \
88  if (kUseTlabFastPath && !instrumented_bool && allocator_type == gc::kAllocatorTypeTLAB) { \
89    size_t byte_count = klass->GetObjectSize(); \
90    byte_count = RoundUp(byte_count, gc::space::BumpPointerSpace::kAlignment); \
91    mirror::Object* obj; \
92    if (LIKELY(byte_count < self->TlabSize())) { \
93      obj = self->AllocTlab(byte_count); \
94      DCHECK(obj != nullptr) << "AllocTlab can't fail"; \
95      obj->SetClass(klass); \
96      if (kUseBakerOrBrooksReadBarrier) { \
97        if (kUseBrooksReadBarrier) { \
98          obj->SetReadBarrierPointer(obj); \
99        } \
100        obj->AssertReadBarrierPointer(); \
101      } \
102      QuasiAtomic::ThreadFenceForConstructor(); \
103      return obj; \
104    } \
105  } \
106  return AllocObjectFromCodeInitialized<instrumented_bool>(klass, self, allocator_type); \
107} \
108extern "C" mirror::Object* artAllocObjectFromCodeWithAccessCheck##suffix##suffix2( \
109    uint32_t type_idx, ArtMethod* method, Thread* self) \
110    SHARED_REQUIRES(Locks::mutator_lock_) { \
111  ScopedQuickEntrypointChecks sqec(self); \
112  return AllocObjectFromCode<true, instrumented_bool>(type_idx, method, self, allocator_type); \
113} \
114extern "C" mirror::Array* artAllocArrayFromCode##suffix##suffix2( \
115    uint32_t type_idx, int32_t component_count, ArtMethod* method, Thread* self) \
116    SHARED_REQUIRES(Locks::mutator_lock_) { \
117  ScopedQuickEntrypointChecks sqec(self); \
118  return AllocArrayFromCode<false, instrumented_bool>(type_idx, component_count, method, self, \
119                                                      allocator_type); \
120} \
121extern "C" mirror::Array* artAllocArrayFromCodeResolved##suffix##suffix2( \
122    mirror::Class* klass, int32_t component_count, ArtMethod* method, Thread* self) \
123    SHARED_REQUIRES(Locks::mutator_lock_) { \
124  ScopedQuickEntrypointChecks sqec(self); \
125  return AllocArrayFromCodeResolved<false, instrumented_bool>(klass, component_count, method, self, \
126                                                              allocator_type); \
127} \
128extern "C" mirror::Array* artAllocArrayFromCodeWithAccessCheck##suffix##suffix2( \
129    uint32_t type_idx, int32_t component_count, ArtMethod* method, Thread* self) \
130    SHARED_REQUIRES(Locks::mutator_lock_) { \
131  ScopedQuickEntrypointChecks sqec(self); \
132  return AllocArrayFromCode<true, instrumented_bool>(type_idx, component_count, method, self, \
133                                                     allocator_type); \
134} \
135extern "C" mirror::Array* artCheckAndAllocArrayFromCode##suffix##suffix2( \
136    uint32_t type_idx, int32_t component_count, ArtMethod* method, Thread* self) \
137    SHARED_REQUIRES(Locks::mutator_lock_) { \
138  ScopedQuickEntrypointChecks sqec(self); \
139  if (!instrumented_bool) { \
140    return CheckAndAllocArrayFromCode(type_idx, component_count, method, self, false, allocator_type); \
141  } else { \
142    return CheckAndAllocArrayFromCodeInstrumented(type_idx, component_count, method, self, false, allocator_type); \
143  } \
144} \
145extern "C" mirror::Array* artCheckAndAllocArrayFromCodeWithAccessCheck##suffix##suffix2( \
146    uint32_t type_idx, int32_t component_count, ArtMethod* method, Thread* self) \
147    SHARED_REQUIRES(Locks::mutator_lock_) { \
148  ScopedQuickEntrypointChecks sqec(self); \
149  if (!instrumented_bool) { \
150    return CheckAndAllocArrayFromCode(type_idx, component_count, method, self, true, allocator_type); \
151  } else { \
152    return CheckAndAllocArrayFromCodeInstrumented(type_idx, component_count, method, self, true, allocator_type); \
153  } \
154} \
155extern "C" mirror::String* artAllocStringFromBytesFromCode##suffix##suffix2( \
156    mirror::ByteArray* byte_array, int32_t high, int32_t offset, int32_t byte_count, \
157    Thread* self) \
158    SHARED_REQUIRES(Locks::mutator_lock_) { \
159  ScopedQuickEntrypointChecks sqec(self); \
160  StackHandleScope<1> hs(self); \
161  Handle<mirror::ByteArray> handle_array(hs.NewHandle(byte_array)); \
162  return mirror::String::AllocFromByteArray<instrumented_bool>(self, byte_count, handle_array, \
163                                                               offset, high, allocator_type); \
164} \
165extern "C" mirror::String* artAllocStringFromCharsFromCode##suffix##suffix2( \
166    int32_t offset, int32_t char_count, mirror::CharArray* char_array, Thread* self) \
167    SHARED_REQUIRES(Locks::mutator_lock_) { \
168  StackHandleScope<1> hs(self); \
169  Handle<mirror::CharArray> handle_array(hs.NewHandle(char_array)); \
170  return mirror::String::AllocFromCharArray<instrumented_bool>(self, char_count, handle_array, \
171                                                               offset, allocator_type); \
172} \
173extern "C" mirror::String* artAllocStringFromStringFromCode##suffix##suffix2( \
174    mirror::String* string, Thread* self) \
175    SHARED_REQUIRES(Locks::mutator_lock_) { \
176  StackHandleScope<1> hs(self); \
177  Handle<mirror::String> handle_string(hs.NewHandle(string)); \
178  return mirror::String::AllocFromString<instrumented_bool>(self, handle_string->GetLength(), \
179                                                            handle_string, 0, allocator_type); \
180}
181
182#define GENERATE_ENTRYPOINTS_FOR_ALLOCATOR(suffix, allocator_type) \
183    GENERATE_ENTRYPOINTS_FOR_ALLOCATOR_INST(suffix, Instrumented, true, allocator_type) \
184    GENERATE_ENTRYPOINTS_FOR_ALLOCATOR_INST(suffix, , false, allocator_type)
185
186GENERATE_ENTRYPOINTS_FOR_ALLOCATOR(DlMalloc, gc::kAllocatorTypeDlMalloc)
187GENERATE_ENTRYPOINTS_FOR_ALLOCATOR(RosAlloc, gc::kAllocatorTypeRosAlloc)
188GENERATE_ENTRYPOINTS_FOR_ALLOCATOR(BumpPointer, gc::kAllocatorTypeBumpPointer)
189GENERATE_ENTRYPOINTS_FOR_ALLOCATOR(TLAB, gc::kAllocatorTypeTLAB)
190GENERATE_ENTRYPOINTS_FOR_ALLOCATOR(Region, gc::kAllocatorTypeRegion)
191GENERATE_ENTRYPOINTS_FOR_ALLOCATOR(RegionTLAB, gc::kAllocatorTypeRegionTLAB)
192
193#define GENERATE_ENTRYPOINTS(suffix) \
194extern "C" void* art_quick_alloc_array##suffix(uint32_t, int32_t, ArtMethod* ref); \
195extern "C" void* art_quick_alloc_array_resolved##suffix(mirror::Class* klass, int32_t, ArtMethod* ref); \
196extern "C" void* art_quick_alloc_array_with_access_check##suffix(uint32_t, int32_t, ArtMethod* ref); \
197extern "C" void* art_quick_alloc_object##suffix(uint32_t type_idx, ArtMethod* ref); \
198extern "C" void* art_quick_alloc_object_resolved##suffix(mirror::Class* klass, ArtMethod* ref); \
199extern "C" void* art_quick_alloc_object_initialized##suffix(mirror::Class* klass, ArtMethod* ref); \
200extern "C" void* art_quick_alloc_object_with_access_check##suffix(uint32_t type_idx, ArtMethod* ref); \
201extern "C" void* art_quick_check_and_alloc_array##suffix(uint32_t, int32_t, ArtMethod* ref); \
202extern "C" void* art_quick_check_and_alloc_array_with_access_check##suffix(uint32_t, int32_t, ArtMethod* ref); \
203extern "C" void* art_quick_alloc_string_from_bytes##suffix(void*, int32_t, int32_t, int32_t); \
204extern "C" void* art_quick_alloc_string_from_chars##suffix(int32_t, int32_t, void*); \
205extern "C" void* art_quick_alloc_string_from_string##suffix(void*); \
206extern "C" void* art_quick_alloc_array##suffix##_instrumented(uint32_t, int32_t, ArtMethod* ref); \
207extern "C" void* art_quick_alloc_array_resolved##suffix##_instrumented(mirror::Class* klass, int32_t, ArtMethod* ref); \
208extern "C" void* art_quick_alloc_array_with_access_check##suffix##_instrumented(uint32_t, int32_t, ArtMethod* ref); \
209extern "C" void* art_quick_alloc_object##suffix##_instrumented(uint32_t type_idx, ArtMethod* ref); \
210extern "C" void* art_quick_alloc_object_resolved##suffix##_instrumented(mirror::Class* klass, ArtMethod* ref); \
211extern "C" void* art_quick_alloc_object_initialized##suffix##_instrumented(mirror::Class* klass, ArtMethod* ref); \
212extern "C" void* art_quick_alloc_object_with_access_check##suffix##_instrumented(uint32_t type_idx, ArtMethod* ref); \
213extern "C" void* art_quick_check_and_alloc_array##suffix##_instrumented(uint32_t, int32_t, ArtMethod* ref); \
214extern "C" void* art_quick_check_and_alloc_array_with_access_check##suffix##_instrumented(uint32_t, int32_t, ArtMethod* ref); \
215extern "C" void* art_quick_alloc_string_from_bytes##suffix##_instrumented(void*, int32_t, int32_t, int32_t); \
216extern "C" void* art_quick_alloc_string_from_chars##suffix##_instrumented(int32_t, int32_t, void*); \
217extern "C" void* art_quick_alloc_string_from_string##suffix##_instrumented(void*); \
218void SetQuickAllocEntryPoints##suffix(QuickEntryPoints* qpoints, bool instrumented) { \
219  if (instrumented) { \
220    qpoints->pAllocArray = art_quick_alloc_array##suffix##_instrumented; \
221    qpoints->pAllocArrayResolved = art_quick_alloc_array_resolved##suffix##_instrumented; \
222    qpoints->pAllocArrayWithAccessCheck = art_quick_alloc_array_with_access_check##suffix##_instrumented; \
223    qpoints->pAllocObject = art_quick_alloc_object##suffix##_instrumented; \
224    qpoints->pAllocObjectResolved = art_quick_alloc_object_resolved##suffix##_instrumented; \
225    qpoints->pAllocObjectInitialized = art_quick_alloc_object_initialized##suffix##_instrumented; \
226    qpoints->pAllocObjectWithAccessCheck = art_quick_alloc_object_with_access_check##suffix##_instrumented; \
227    qpoints->pCheckAndAllocArray = art_quick_check_and_alloc_array##suffix##_instrumented; \
228    qpoints->pCheckAndAllocArrayWithAccessCheck = art_quick_check_and_alloc_array_with_access_check##suffix##_instrumented; \
229    qpoints->pAllocStringFromBytes = art_quick_alloc_string_from_bytes##suffix##_instrumented; \
230    qpoints->pAllocStringFromChars = art_quick_alloc_string_from_chars##suffix##_instrumented; \
231    qpoints->pAllocStringFromString = art_quick_alloc_string_from_string##suffix##_instrumented; \
232  } else { \
233    qpoints->pAllocArray = art_quick_alloc_array##suffix; \
234    qpoints->pAllocArrayResolved = art_quick_alloc_array_resolved##suffix; \
235    qpoints->pAllocArrayWithAccessCheck = art_quick_alloc_array_with_access_check##suffix; \
236    qpoints->pAllocObject = art_quick_alloc_object##suffix; \
237    qpoints->pAllocObjectResolved = art_quick_alloc_object_resolved##suffix; \
238    qpoints->pAllocObjectInitialized = art_quick_alloc_object_initialized##suffix; \
239    qpoints->pAllocObjectWithAccessCheck = art_quick_alloc_object_with_access_check##suffix; \
240    qpoints->pCheckAndAllocArray = art_quick_check_and_alloc_array##suffix; \
241    qpoints->pCheckAndAllocArrayWithAccessCheck = art_quick_check_and_alloc_array_with_access_check##suffix; \
242    qpoints->pAllocStringFromBytes = art_quick_alloc_string_from_bytes##suffix; \
243    qpoints->pAllocStringFromChars = art_quick_alloc_string_from_chars##suffix; \
244    qpoints->pAllocStringFromString = art_quick_alloc_string_from_string##suffix; \
245  } \
246}
247
248// Generate the entrypoint functions.
249#if !defined(__APPLE__) || !defined(__LP64__)
250GENERATE_ENTRYPOINTS(_dlmalloc)
251GENERATE_ENTRYPOINTS(_rosalloc)
252GENERATE_ENTRYPOINTS(_bump_pointer)
253GENERATE_ENTRYPOINTS(_tlab)
254GENERATE_ENTRYPOINTS(_region)
255GENERATE_ENTRYPOINTS(_region_tlab)
256#endif
257
258static bool entry_points_instrumented = false;
259static gc::AllocatorType entry_points_allocator = gc::kAllocatorTypeDlMalloc;
260
261void SetQuickAllocEntryPointsAllocator(gc::AllocatorType allocator) {
262  entry_points_allocator = allocator;
263}
264
265void SetQuickAllocEntryPointsInstrumented(bool instrumented) {
266  entry_points_instrumented = instrumented;
267}
268
269void ResetQuickAllocEntryPoints(QuickEntryPoints* qpoints) {
270#if !defined(__APPLE__) || !defined(__LP64__)
271  switch (entry_points_allocator) {
272    case gc::kAllocatorTypeDlMalloc: {
273      SetQuickAllocEntryPoints_dlmalloc(qpoints, entry_points_instrumented);
274      return;
275    }
276    case gc::kAllocatorTypeRosAlloc: {
277      SetQuickAllocEntryPoints_rosalloc(qpoints, entry_points_instrumented);
278      return;
279    }
280    case gc::kAllocatorTypeBumpPointer: {
281      CHECK(kMovingCollector);
282      SetQuickAllocEntryPoints_bump_pointer(qpoints, entry_points_instrumented);
283      return;
284    }
285    case gc::kAllocatorTypeTLAB: {
286      CHECK(kMovingCollector);
287      SetQuickAllocEntryPoints_tlab(qpoints, entry_points_instrumented);
288      return;
289    }
290    case gc::kAllocatorTypeRegion: {
291      CHECK(kMovingCollector);
292      SetQuickAllocEntryPoints_region(qpoints, entry_points_instrumented);
293      return;
294    }
295    case gc::kAllocatorTypeRegionTLAB: {
296      CHECK(kMovingCollector);
297      SetQuickAllocEntryPoints_region_tlab(qpoints, entry_points_instrumented);
298      return;
299    }
300    default:
301      break;
302  }
303#else
304  UNUSED(qpoints);
305#endif
306  UNIMPLEMENTED(FATAL);
307  UNREACHABLE();
308}
309
310}  // namespace art
311