1 /*
2 * Copyright (C) 2012 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 #include "entrypoints/quick/quick_alloc_entrypoints.h"
18
19 #include "art_method-inl.h"
20 #include "callee_save_frame.h"
21 #include "entrypoints/entrypoint_utils-inl.h"
22 #include "mirror/class-inl.h"
23 #include "mirror/object_array-inl.h"
24 #include "mirror/object-inl.h"
25
26 namespace art {
27
28 static constexpr bool kUseTlabFastPath = true;
29
30 #define GENERATE_ENTRYPOINTS_FOR_ALLOCATOR_INST(suffix, suffix2, instrumented_bool, allocator_type) \
31 extern "C" mirror::Object* artAllocObjectFromCode ##suffix##suffix2( \
32 uint32_t type_idx, ArtMethod* method, Thread* self) \
33 SHARED_REQUIRES(Locks::mutator_lock_) { \
34 ScopedQuickEntrypointChecks sqec(self); \
35 if (kUseTlabFastPath && !instrumented_bool && allocator_type == gc::kAllocatorTypeTLAB) { \
36 mirror::Class* klass = method->GetDexCacheResolvedType<false>(type_idx, sizeof(void*)); \
37 if (LIKELY(klass != nullptr && klass->IsInitialized() && !klass->IsFinalizable())) { \
38 size_t byte_count = klass->GetObjectSize(); \
39 byte_count = RoundUp(byte_count, gc::space::BumpPointerSpace::kAlignment); \
40 mirror::Object* obj; \
41 if (LIKELY(byte_count < self->TlabSize())) { \
42 obj = self->AllocTlab(byte_count); \
43 DCHECK(obj != nullptr) << "AllocTlab can't fail"; \
44 obj->SetClass(klass); \
45 if (kUseBakerOrBrooksReadBarrier) { \
46 if (kUseBrooksReadBarrier) { \
47 obj->SetReadBarrierPointer(obj); \
48 } \
49 obj->AssertReadBarrierPointer(); \
50 } \
51 QuasiAtomic::ThreadFenceForConstructor(); \
52 return obj; \
53 } \
54 } \
55 } \
56 return AllocObjectFromCode<false, instrumented_bool>(type_idx, method, self, allocator_type); \
57 } \
58 extern "C" mirror::Object* artAllocObjectFromCodeResolved##suffix##suffix2( \
59 mirror::Class* klass, ArtMethod* method ATTRIBUTE_UNUSED, Thread* self) \
60 SHARED_REQUIRES(Locks::mutator_lock_) { \
61 ScopedQuickEntrypointChecks sqec(self); \
62 if (kUseTlabFastPath && !instrumented_bool && allocator_type == gc::kAllocatorTypeTLAB) { \
63 if (LIKELY(klass->IsInitialized())) { \
64 size_t byte_count = klass->GetObjectSize(); \
65 byte_count = RoundUp(byte_count, gc::space::BumpPointerSpace::kAlignment); \
66 mirror::Object* obj; \
67 if (LIKELY(byte_count < self->TlabSize())) { \
68 obj = self->AllocTlab(byte_count); \
69 DCHECK(obj != nullptr) << "AllocTlab can't fail"; \
70 obj->SetClass(klass); \
71 if (kUseBakerOrBrooksReadBarrier) { \
72 if (kUseBrooksReadBarrier) { \
73 obj->SetReadBarrierPointer(obj); \
74 } \
75 obj->AssertReadBarrierPointer(); \
76 } \
77 QuasiAtomic::ThreadFenceForConstructor(); \
78 return obj; \
79 } \
80 } \
81 } \
82 return AllocObjectFromCodeResolved<instrumented_bool>(klass, self, allocator_type); \
83 } \
84 extern "C" mirror::Object* artAllocObjectFromCodeInitialized##suffix##suffix2( \
85 mirror::Class* klass, ArtMethod* method ATTRIBUTE_UNUSED, Thread* self) \
86 SHARED_REQUIRES(Locks::mutator_lock_) { \
87 ScopedQuickEntrypointChecks sqec(self); \
88 if (kUseTlabFastPath && !instrumented_bool && allocator_type == gc::kAllocatorTypeTLAB) { \
89 size_t byte_count = klass->GetObjectSize(); \
90 byte_count = RoundUp(byte_count, gc::space::BumpPointerSpace::kAlignment); \
91 mirror::Object* obj; \
92 if (LIKELY(byte_count < self->TlabSize())) { \
93 obj = self->AllocTlab(byte_count); \
94 DCHECK(obj != nullptr) << "AllocTlab can't fail"; \
95 obj->SetClass(klass); \
96 if (kUseBakerOrBrooksReadBarrier) { \
97 if (kUseBrooksReadBarrier) { \
98 obj->SetReadBarrierPointer(obj); \
99 } \
100 obj->AssertReadBarrierPointer(); \
101 } \
102 QuasiAtomic::ThreadFenceForConstructor(); \
103 return obj; \
104 } \
105 } \
106 return AllocObjectFromCodeInitialized<instrumented_bool>(klass, self, allocator_type); \
107 } \
108 extern "C" mirror::Object* artAllocObjectFromCodeWithAccessCheck##suffix##suffix2( \
109 uint32_t type_idx, ArtMethod* method, Thread* self) \
110 SHARED_REQUIRES(Locks::mutator_lock_) { \
111 ScopedQuickEntrypointChecks sqec(self); \
112 return AllocObjectFromCode<true, instrumented_bool>(type_idx, method, self, allocator_type); \
113 } \
114 extern "C" mirror::Array* artAllocArrayFromCode##suffix##suffix2( \
115 uint32_t type_idx, int32_t component_count, ArtMethod* method, Thread* self) \
116 SHARED_REQUIRES(Locks::mutator_lock_) { \
117 ScopedQuickEntrypointChecks sqec(self); \
118 return AllocArrayFromCode<false, instrumented_bool>(type_idx, component_count, method, self, \
119 allocator_type); \
120 } \
121 extern "C" mirror::Array* artAllocArrayFromCodeResolved##suffix##suffix2( \
122 mirror::Class* klass, int32_t component_count, ArtMethod* method, Thread* self) \
123 SHARED_REQUIRES(Locks::mutator_lock_) { \
124 ScopedQuickEntrypointChecks sqec(self); \
125 return AllocArrayFromCodeResolved<false, instrumented_bool>(klass, component_count, method, self, \
126 allocator_type); \
127 } \
128 extern "C" mirror::Array* artAllocArrayFromCodeWithAccessCheck##suffix##suffix2( \
129 uint32_t type_idx, int32_t component_count, ArtMethod* method, Thread* self) \
130 SHARED_REQUIRES(Locks::mutator_lock_) { \
131 ScopedQuickEntrypointChecks sqec(self); \
132 return AllocArrayFromCode<true, instrumented_bool>(type_idx, component_count, method, self, \
133 allocator_type); \
134 } \
135 extern "C" mirror::Array* artCheckAndAllocArrayFromCode##suffix##suffix2( \
136 uint32_t type_idx, int32_t component_count, ArtMethod* method, Thread* self) \
137 SHARED_REQUIRES(Locks::mutator_lock_) { \
138 ScopedQuickEntrypointChecks sqec(self); \
139 if (!instrumented_bool) { \
140 return CheckAndAllocArrayFromCode(type_idx, component_count, method, self, false, allocator_type); \
141 } else { \
142 return CheckAndAllocArrayFromCodeInstrumented(type_idx, component_count, method, self, false, allocator_type); \
143 } \
144 } \
145 extern "C" mirror::Array* artCheckAndAllocArrayFromCodeWithAccessCheck##suffix##suffix2( \
146 uint32_t type_idx, int32_t component_count, ArtMethod* method, Thread* self) \
147 SHARED_REQUIRES(Locks::mutator_lock_) { \
148 ScopedQuickEntrypointChecks sqec(self); \
149 if (!instrumented_bool) { \
150 return CheckAndAllocArrayFromCode(type_idx, component_count, method, self, true, allocator_type); \
151 } else { \
152 return CheckAndAllocArrayFromCodeInstrumented(type_idx, component_count, method, self, true, allocator_type); \
153 } \
154 } \
155 extern "C" mirror::String* artAllocStringFromBytesFromCode##suffix##suffix2( \
156 mirror::ByteArray* byte_array, int32_t high, int32_t offset, int32_t byte_count, \
157 Thread* self) \
158 SHARED_REQUIRES(Locks::mutator_lock_) { \
159 ScopedQuickEntrypointChecks sqec(self); \
160 StackHandleScope<1> hs(self); \
161 Handle<mirror::ByteArray> handle_array(hs.NewHandle(byte_array)); \
162 return mirror::String::AllocFromByteArray<instrumented_bool>(self, byte_count, handle_array, \
163 offset, high, allocator_type); \
164 } \
165 extern "C" mirror::String* artAllocStringFromCharsFromCode##suffix##suffix2( \
166 int32_t offset, int32_t char_count, mirror::CharArray* char_array, Thread* self) \
167 SHARED_REQUIRES(Locks::mutator_lock_) { \
168 StackHandleScope<1> hs(self); \
169 Handle<mirror::CharArray> handle_array(hs.NewHandle(char_array)); \
170 return mirror::String::AllocFromCharArray<instrumented_bool>(self, char_count, handle_array, \
171 offset, allocator_type); \
172 } \
173 extern "C" mirror::String* artAllocStringFromStringFromCode##suffix##suffix2( \
174 mirror::String* string, Thread* self) \
175 SHARED_REQUIRES(Locks::mutator_lock_) { \
176 StackHandleScope<1> hs(self); \
177 Handle<mirror::String> handle_string(hs.NewHandle(string)); \
178 return mirror::String::AllocFromString<instrumented_bool>(self, handle_string->GetLength(), \
179 handle_string, 0, allocator_type); \
180 }
181
182 #define GENERATE_ENTRYPOINTS_FOR_ALLOCATOR(suffix, allocator_type) \
183 GENERATE_ENTRYPOINTS_FOR_ALLOCATOR_INST(suffix, Instrumented, true, allocator_type) \
184 GENERATE_ENTRYPOINTS_FOR_ALLOCATOR_INST(suffix, , false, allocator_type)
185
186 GENERATE_ENTRYPOINTS_FOR_ALLOCATOR(DlMalloc, gc::kAllocatorTypeDlMalloc)
187 GENERATE_ENTRYPOINTS_FOR_ALLOCATOR(RosAlloc, gc::kAllocatorTypeRosAlloc)
188 GENERATE_ENTRYPOINTS_FOR_ALLOCATOR(BumpPointer, gc::kAllocatorTypeBumpPointer)
189 GENERATE_ENTRYPOINTS_FOR_ALLOCATOR(TLAB, gc::kAllocatorTypeTLAB)
190 GENERATE_ENTRYPOINTS_FOR_ALLOCATOR(Region, gc::kAllocatorTypeRegion)
191 GENERATE_ENTRYPOINTS_FOR_ALLOCATOR(RegionTLAB, gc::kAllocatorTypeRegionTLAB)
192
193 #define GENERATE_ENTRYPOINTS(suffix) \
194 extern "C" void* art_quick_alloc_array##suffix(uint32_t, int32_t, ArtMethod* ref); \
195 extern "C" void* art_quick_alloc_array_resolved##suffix(mirror::Class* klass, int32_t, ArtMethod* ref); \
196 extern "C" void* art_quick_alloc_array_with_access_check##suffix(uint32_t, int32_t, ArtMethod* ref); \
197 extern "C" void* art_quick_alloc_object##suffix(uint32_t type_idx, ArtMethod* ref); \
198 extern "C" void* art_quick_alloc_object_resolved##suffix(mirror::Class* klass, ArtMethod* ref); \
199 extern "C" void* art_quick_alloc_object_initialized##suffix(mirror::Class* klass, ArtMethod* ref); \
200 extern "C" void* art_quick_alloc_object_with_access_check##suffix(uint32_t type_idx, ArtMethod* ref); \
201 extern "C" void* art_quick_check_and_alloc_array##suffix(uint32_t, int32_t, ArtMethod* ref); \
202 extern "C" void* art_quick_check_and_alloc_array_with_access_check##suffix(uint32_t, int32_t, ArtMethod* ref); \
203 extern "C" void* art_quick_alloc_string_from_bytes##suffix(void*, int32_t, int32_t, int32_t); \
204 extern "C" void* art_quick_alloc_string_from_chars##suffix(int32_t, int32_t, void*); \
205 extern "C" void* art_quick_alloc_string_from_string##suffix(void*); \
206 extern "C" void* art_quick_alloc_array##suffix##_instrumented(uint32_t, int32_t, ArtMethod* ref); \
207 extern "C" void* art_quick_alloc_array_resolved##suffix##_instrumented(mirror::Class* klass, int32_t, ArtMethod* ref); \
208 extern "C" void* art_quick_alloc_array_with_access_check##suffix##_instrumented(uint32_t, int32_t, ArtMethod* ref); \
209 extern "C" void* art_quick_alloc_object##suffix##_instrumented(uint32_t type_idx, ArtMethod* ref); \
210 extern "C" void* art_quick_alloc_object_resolved##suffix##_instrumented(mirror::Class* klass, ArtMethod* ref); \
211 extern "C" void* art_quick_alloc_object_initialized##suffix##_instrumented(mirror::Class* klass, ArtMethod* ref); \
212 extern "C" void* art_quick_alloc_object_with_access_check##suffix##_instrumented(uint32_t type_idx, ArtMethod* ref); \
213 extern "C" void* art_quick_check_and_alloc_array##suffix##_instrumented(uint32_t, int32_t, ArtMethod* ref); \
214 extern "C" void* art_quick_check_and_alloc_array_with_access_check##suffix##_instrumented(uint32_t, int32_t, ArtMethod* ref); \
215 extern "C" void* art_quick_alloc_string_from_bytes##suffix##_instrumented(void*, int32_t, int32_t, int32_t); \
216 extern "C" void* art_quick_alloc_string_from_chars##suffix##_instrumented(int32_t, int32_t, void*); \
217 extern "C" void* art_quick_alloc_string_from_string##suffix##_instrumented(void*); \
218 void SetQuickAllocEntryPoints##suffix(QuickEntryPoints* qpoints, bool instrumented) { \
219 if (instrumented) { \
220 qpoints->pAllocArray = art_quick_alloc_array##suffix##_instrumented; \
221 qpoints->pAllocArrayResolved = art_quick_alloc_array_resolved##suffix##_instrumented; \
222 qpoints->pAllocArrayWithAccessCheck = art_quick_alloc_array_with_access_check##suffix##_instrumented; \
223 qpoints->pAllocObject = art_quick_alloc_object##suffix##_instrumented; \
224 qpoints->pAllocObjectResolved = art_quick_alloc_object_resolved##suffix##_instrumented; \
225 qpoints->pAllocObjectInitialized = art_quick_alloc_object_initialized##suffix##_instrumented; \
226 qpoints->pAllocObjectWithAccessCheck = art_quick_alloc_object_with_access_check##suffix##_instrumented; \
227 qpoints->pCheckAndAllocArray = art_quick_check_and_alloc_array##suffix##_instrumented; \
228 qpoints->pCheckAndAllocArrayWithAccessCheck = art_quick_check_and_alloc_array_with_access_check##suffix##_instrumented; \
229 qpoints->pAllocStringFromBytes = art_quick_alloc_string_from_bytes##suffix##_instrumented; \
230 qpoints->pAllocStringFromChars = art_quick_alloc_string_from_chars##suffix##_instrumented; \
231 qpoints->pAllocStringFromString = art_quick_alloc_string_from_string##suffix##_instrumented; \
232 } else { \
233 qpoints->pAllocArray = art_quick_alloc_array##suffix; \
234 qpoints->pAllocArrayResolved = art_quick_alloc_array_resolved##suffix; \
235 qpoints->pAllocArrayWithAccessCheck = art_quick_alloc_array_with_access_check##suffix; \
236 qpoints->pAllocObject = art_quick_alloc_object##suffix; \
237 qpoints->pAllocObjectResolved = art_quick_alloc_object_resolved##suffix; \
238 qpoints->pAllocObjectInitialized = art_quick_alloc_object_initialized##suffix; \
239 qpoints->pAllocObjectWithAccessCheck = art_quick_alloc_object_with_access_check##suffix; \
240 qpoints->pCheckAndAllocArray = art_quick_check_and_alloc_array##suffix; \
241 qpoints->pCheckAndAllocArrayWithAccessCheck = art_quick_check_and_alloc_array_with_access_check##suffix; \
242 qpoints->pAllocStringFromBytes = art_quick_alloc_string_from_bytes##suffix; \
243 qpoints->pAllocStringFromChars = art_quick_alloc_string_from_chars##suffix; \
244 qpoints->pAllocStringFromString = art_quick_alloc_string_from_string##suffix; \
245 } \
246 }
247
248 // Generate the entrypoint functions.
249 #if !defined(__APPLE__) || !defined(__LP64__)
250 GENERATE_ENTRYPOINTS(_dlmalloc)
251 GENERATE_ENTRYPOINTS(_rosalloc)
252 GENERATE_ENTRYPOINTS(_bump_pointer)
253 GENERATE_ENTRYPOINTS(_tlab)
254 GENERATE_ENTRYPOINTS(_region)
255 GENERATE_ENTRYPOINTS(_region_tlab)
256 #endif
257
258 static bool entry_points_instrumented = false;
259 static gc::AllocatorType entry_points_allocator = gc::kAllocatorTypeDlMalloc;
260
SetQuickAllocEntryPointsAllocator(gc::AllocatorType allocator)261 void SetQuickAllocEntryPointsAllocator(gc::AllocatorType allocator) {
262 entry_points_allocator = allocator;
263 }
264
SetQuickAllocEntryPointsInstrumented(bool instrumented)265 void SetQuickAllocEntryPointsInstrumented(bool instrumented) {
266 entry_points_instrumented = instrumented;
267 }
268
ResetQuickAllocEntryPoints(QuickEntryPoints * qpoints)269 void ResetQuickAllocEntryPoints(QuickEntryPoints* qpoints) {
270 #if !defined(__APPLE__) || !defined(__LP64__)
271 switch (entry_points_allocator) {
272 case gc::kAllocatorTypeDlMalloc: {
273 SetQuickAllocEntryPoints_dlmalloc(qpoints, entry_points_instrumented);
274 return;
275 }
276 case gc::kAllocatorTypeRosAlloc: {
277 SetQuickAllocEntryPoints_rosalloc(qpoints, entry_points_instrumented);
278 return;
279 }
280 case gc::kAllocatorTypeBumpPointer: {
281 CHECK(kMovingCollector);
282 SetQuickAllocEntryPoints_bump_pointer(qpoints, entry_points_instrumented);
283 return;
284 }
285 case gc::kAllocatorTypeTLAB: {
286 CHECK(kMovingCollector);
287 SetQuickAllocEntryPoints_tlab(qpoints, entry_points_instrumented);
288 return;
289 }
290 case gc::kAllocatorTypeRegion: {
291 CHECK(kMovingCollector);
292 SetQuickAllocEntryPoints_region(qpoints, entry_points_instrumented);
293 return;
294 }
295 case gc::kAllocatorTypeRegionTLAB: {
296 CHECK(kMovingCollector);
297 SetQuickAllocEntryPoints_region_tlab(qpoints, entry_points_instrumented);
298 return;
299 }
300 default:
301 break;
302 }
303 #else
304 UNUSED(qpoints);
305 #endif
306 UNIMPLEMENTED(FATAL);
307 UNREACHABLE();
308 }
309
310 } // namespace art
311