1 /*
2 * Copyright (C) 2013 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 #ifndef ART_RUNTIME_GC_HEAP_INL_H_
18 #define ART_RUNTIME_GC_HEAP_INL_H_
19
20 #include "heap.h"
21
22 #include "allocation_listener.h"
23 #include "base/quasi_atomic.h"
24 #include "base/time_utils.h"
25 #include "gc/accounting/atomic_stack.h"
26 #include "gc/accounting/card_table-inl.h"
27 #include "gc/allocation_record.h"
28 #include "gc/collector/semi_space.h"
29 #include "gc/space/bump_pointer_space-inl.h"
30 #include "gc/space/dlmalloc_space-inl.h"
31 #include "gc/space/large_object_space.h"
32 #include "gc/space/region_space-inl.h"
33 #include "gc/space/rosalloc_space-inl.h"
34 #include "handle_scope-inl.h"
35 #include "obj_ptr-inl.h"
36 #include "runtime.h"
37 #include "thread-inl.h"
38 #include "verify_object.h"
39 #include "write_barrier-inl.h"
40
41 namespace art HIDDEN {
42 namespace gc {
43
44 template <bool kInstrumented, bool kCheckLargeObject, typename PreFenceVisitor>
AllocObjectWithAllocator(Thread * self,ObjPtr<mirror::Class> klass,size_t byte_count,AllocatorType allocator,const PreFenceVisitor & pre_fence_visitor)45 inline mirror::Object* Heap::AllocObjectWithAllocator(Thread* self,
46 ObjPtr<mirror::Class> klass,
47 size_t byte_count,
48 AllocatorType allocator,
49 const PreFenceVisitor& pre_fence_visitor) {
50 auto no_suspend_pre_fence_visitor =
51 [&pre_fence_visitor](auto... x) REQUIRES_SHARED(Locks::mutator_lock_) {
52 ScopedAssertNoThreadSuspension sants("No thread suspension during pre-fence visitor");
53 pre_fence_visitor(x...);
54 };
55
56 if (kIsDebugBuild) {
57 CheckPreconditionsForAllocObject(klass, byte_count);
58 // Since allocation can cause a GC which will need to SuspendAll, make sure all allocations are
59 // done in the runnable state where suspension is expected.
60 CHECK_EQ(self->GetState(), ThreadState::kRunnable);
61 self->AssertThreadSuspensionIsAllowable();
62 self->AssertNoPendingException();
63 // Make sure to preserve klass.
64 StackHandleScope<1> hs(self);
65 HandleWrapperObjPtr<mirror::Class> h = hs.NewHandleWrapper(&klass);
66 self->PoisonObjectPointers();
67 }
68 auto pre_object_allocated = [&]() REQUIRES_SHARED(Locks::mutator_lock_)
69 REQUIRES(!Roles::uninterruptible_ /* only suspends if kInstrumented */) {
70 if constexpr (kInstrumented) {
71 AllocationListener* l = alloc_listener_.load(std::memory_order_seq_cst);
72 if (UNLIKELY(l != nullptr) && UNLIKELY(l->HasPreAlloc())) {
73 StackHandleScope<1> hs(self);
74 HandleWrapperObjPtr<mirror::Class> h_klass(hs.NewHandleWrapper(&klass));
75 l->PreObjectAllocated(self, h_klass, &byte_count);
76 }
77 }
78 };
79 ObjPtr<mirror::Object> obj;
80 // bytes allocated for the (individual) object.
81 size_t bytes_allocated;
82 size_t usable_size;
83 size_t new_num_bytes_allocated = 0;
84 bool need_gc = false;
85 uint32_t starting_gc_num; // o.w. GC number at which we observed need for GC.
86 {
87 // Bytes allocated that includes bulk thread-local buffer allocations in addition to direct
88 // non-TLAB object allocations. Only set for non-thread-local allocation,
89 size_t bytes_tl_bulk_allocated = 0u;
90 // Do the initial pre-alloc
91 // TODO: Consider what happens if the allocator is switched while suspended here.
92 pre_object_allocated();
93
94 // Need to check that we aren't the large object allocator since the large object allocation
95 // code path includes this function. If we didn't check we would have an infinite loop.
96 if (kCheckLargeObject && UNLIKELY(ShouldAllocLargeObject(klass, byte_count))) {
97 // AllocLargeObject can suspend and will recall PreObjectAllocated if needed.
98 obj = AllocLargeObject<kInstrumented, PreFenceVisitor>(self, &klass, byte_count,
99 pre_fence_visitor);
100 if (obj != nullptr) {
101 return obj.Ptr();
102 }
103 // There should be an OOM exception, since we are retrying, clear it.
104 self->ClearException();
105
106 // If the large object allocation failed, try to use the normal spaces (main space,
107 // non moving space). This can happen if there is significant virtual address space
108 // fragmentation.
109 // kInstrumented may be out of date, so recurse without large object checking, rather than
110 // continue.
111 return AllocObjectWithAllocator</*kInstrumented=*/ true, /*kCheckLargeObject=*/ false>
112 (self, klass, byte_count, GetUpdatedAllocator(allocator), pre_fence_visitor);
113 }
114 ScopedAssertNoThreadSuspension ants("Called PreObjectAllocated, no suspend until alloc");
115 if (IsTLABAllocator(allocator)) {
116 byte_count = RoundUp(byte_count, space::BumpPointerSpace::kAlignment);
117 }
118 // If we have a thread local allocation we don't need to update bytes allocated.
119 if (IsTLABAllocator(allocator) && byte_count <= self->TlabSize()) {
120 obj = self->AllocTlab(byte_count);
121 DCHECK(obj != nullptr) << "AllocTlab can't fail";
122 obj->SetClass(klass);
123 if (kUseBakerReadBarrier) {
124 obj->AssertReadBarrierState();
125 }
126 bytes_allocated = byte_count;
127 usable_size = bytes_allocated;
128 no_suspend_pre_fence_visitor(obj, usable_size);
129 QuasiAtomic::ThreadFenceForConstructor();
130 } else if (
131 !kInstrumented && allocator == kAllocatorTypeRosAlloc &&
132 (obj = rosalloc_space_->AllocThreadLocal(self, byte_count, &bytes_allocated)) != nullptr &&
133 LIKELY(obj != nullptr)) {
134 DCHECK(!is_running_on_memory_tool_);
135 obj->SetClass(klass);
136 if (kUseBakerReadBarrier) {
137 obj->AssertReadBarrierState();
138 }
139 usable_size = bytes_allocated;
140 no_suspend_pre_fence_visitor(obj, usable_size);
141 QuasiAtomic::ThreadFenceForConstructor();
142 } else {
143 obj = TryToAllocate<kInstrumented, false>(self, allocator, byte_count, &bytes_allocated,
144 &usable_size, &bytes_tl_bulk_allocated);
145 if (UNLIKELY(obj == nullptr)) {
146 // AllocateInternalWithGc internally re-allows, and can cause, thread suspension, if
147 // someone instruments the entrypoints or changes the allocator in a suspend point here,
148 // we need to retry the allocation. It will send the pre-alloc event again.
149 obj = AllocateInternalWithGc(self,
150 allocator,
151 kInstrumented,
152 byte_count,
153 &bytes_allocated,
154 &usable_size,
155 &bytes_tl_bulk_allocated,
156 &klass);
157 if (obj == nullptr) {
158 // The only way that we can get a null return if there is no pending exception is if the
159 // allocator or instrumentation changed.
160 if (!self->IsExceptionPending()) {
161 // Since we are restarting, allow thread suspension.
162 ScopedAllowThreadSuspension ats;
163 // Get the new class size in case class redefinition changed the class size since alloc
164 // started.
165 int new_byte_count = klass->IsVariableSize()? byte_count : klass->GetObjectSize();
166 // AllocObject will pick up the new allocator type, and instrumented as true is the safe
167 // default.
168 return AllocObjectWithAllocator</*kInstrumented=*/true>(self,
169 klass,
170 new_byte_count,
171 GetUpdatedAllocator(allocator),
172 pre_fence_visitor);
173 }
174 return nullptr;
175 }
176 // Non-null result implies neither instrumentation nor allocator changed.
177 }
178 DCHECK_GT(bytes_allocated, 0u);
179 DCHECK_GT(usable_size, 0u);
180 obj->SetClass(klass);
181 if (kUseBakerReadBarrier) {
182 obj->AssertReadBarrierState();
183 }
184 if (collector::SemiSpace::kUseRememberedSet &&
185 UNLIKELY(allocator == kAllocatorTypeNonMoving)) {
186 // (Note this if statement will be constant folded away for the fast-path quick entry
187 // points.) Because SetClass() has no write barrier, the GC may need a write barrier in the
188 // case the object is non movable and points to a recently allocated movable class.
189 WriteBarrier::ForFieldWrite(obj, mirror::Object::ClassOffset(), klass);
190 }
191 no_suspend_pre_fence_visitor(obj, usable_size);
192 QuasiAtomic::ThreadFenceForConstructor();
193 }
194 if (bytes_tl_bulk_allocated > 0) {
195 starting_gc_num = GetCurrentGcNum();
196 size_t num_bytes_allocated_before = AddBytesAllocated(bytes_tl_bulk_allocated);
197 new_num_bytes_allocated = num_bytes_allocated_before + bytes_tl_bulk_allocated;
198 // Only trace when we get an increase in the number of bytes allocated. This happens when
199 // obtaining a new TLAB and isn't often enough to hurt performance according to golem.
200 if (region_space_) {
201 // With CC collector, during a GC cycle, the heap usage increases as
202 // there are two copies of evacuated objects. Therefore, add evac-bytes
203 // to the heap size. When the GC cycle is not running, evac-bytes
204 // are 0, as required.
205 TraceHeapSize(new_num_bytes_allocated + region_space_->EvacBytes());
206 } else {
207 TraceHeapSize(new_num_bytes_allocated);
208 }
209 // IsGcConcurrent() isn't known at compile time so we can optimize by not checking it for the
210 // BumpPointer or TLAB allocators. This is nice since it allows the entire if statement to be
211 // optimized out.
212 if (IsGcConcurrent() && UNLIKELY(ShouldConcurrentGCForJava(new_num_bytes_allocated))) {
213 need_gc = true;
214 }
215 GetMetrics()->TotalBytesAllocated()->Add(bytes_tl_bulk_allocated);
216 GetMetrics()->TotalBytesAllocatedDelta()->Add(bytes_tl_bulk_allocated);
217 }
218 }
219 if (kIsDebugBuild && Runtime::Current()->IsStarted()) {
220 CHECK_LE(obj->SizeOf(), usable_size);
221 }
222 // TODO: Deprecate.
223 if (kInstrumented) {
224 if (Runtime::Current()->HasStatsEnabled()) {
225 RuntimeStats* thread_stats = self->GetStats();
226 ++thread_stats->allocated_objects;
227 thread_stats->allocated_bytes += bytes_allocated;
228 RuntimeStats* global_stats = Runtime::Current()->GetStats();
229 ++global_stats->allocated_objects;
230 global_stats->allocated_bytes += bytes_allocated;
231 }
232 } else {
233 DCHECK(!Runtime::Current()->HasStatsEnabled());
234 }
235 if (kInstrumented) {
236 if (IsAllocTrackingEnabled()) {
237 // allocation_records_ is not null since it never becomes null after allocation tracking is
238 // enabled.
239 DCHECK(allocation_records_ != nullptr);
240 allocation_records_->RecordAllocation(self, &obj, bytes_allocated);
241 }
242 AllocationListener* l = alloc_listener_.load(std::memory_order_seq_cst);
243 if (l != nullptr) {
244 // Same as above. We assume that a listener that was once stored will never be deleted.
245 // Otherwise we'd have to perform this under a lock.
246 l->ObjectAllocated(self, &obj, bytes_allocated);
247 }
248 } else {
249 DCHECK(!IsAllocTrackingEnabled());
250 }
251 if (AllocatorHasAllocationStack(allocator)) {
252 PushOnAllocationStack(self, &obj);
253 }
254 if (kInstrumented) {
255 if (gc_stress_mode_) {
256 CheckGcStressMode(self, &obj);
257 }
258 } else {
259 DCHECK(!gc_stress_mode_);
260 }
261 if (need_gc) {
262 // Do this only once thread suspension is allowed again, and we're done with kInstrumented.
263 RequestConcurrentGCAndSaveObject(self, /*force_full=*/ false, starting_gc_num, &obj);
264 }
265 VerifyObject(obj);
266 self->VerifyStack();
267 return obj.Ptr();
268 }
269
270 // The size of a thread-local allocation stack in the number of references.
271 static constexpr size_t kThreadLocalAllocationStackSize = 128;
272
PushOnAllocationStack(Thread * self,ObjPtr<mirror::Object> * obj)273 inline void Heap::PushOnAllocationStack(Thread* self, ObjPtr<mirror::Object>* obj) {
274 if (kUseThreadLocalAllocationStack) {
275 if (UNLIKELY(!self->PushOnThreadLocalAllocationStack(obj->Ptr()))) {
276 PushOnThreadLocalAllocationStackWithInternalGC(self, obj);
277 }
278 } else if (UNLIKELY(!allocation_stack_->AtomicPushBack(obj->Ptr()))) {
279 PushOnAllocationStackWithInternalGC(self, obj);
280 }
281 }
282
283 template <bool kInstrumented, typename PreFenceVisitor>
AllocLargeObject(Thread * self,ObjPtr<mirror::Class> * klass,size_t byte_count,const PreFenceVisitor & pre_fence_visitor)284 inline mirror::Object* Heap::AllocLargeObject(Thread* self,
285 ObjPtr<mirror::Class>* klass,
286 size_t byte_count,
287 const PreFenceVisitor& pre_fence_visitor) {
288 // Save and restore the class in case it moves.
289 StackHandleScope<1> hs(self);
290 auto klass_wrapper = hs.NewHandleWrapper(klass);
291 mirror::Object* obj = AllocObjectWithAllocator<kInstrumented, false, PreFenceVisitor>
292 (self, *klass, byte_count, kAllocatorTypeLOS, pre_fence_visitor);
293 // Java Heap Profiler check and sample allocation.
294 if (GetHeapSampler().IsEnabled()) {
295 JHPCheckNonTlabSampleAllocation(self, obj, byte_count);
296 }
297 return obj;
298 }
299
300 template <const bool kInstrumented, const bool kGrow>
TryToAllocate(Thread * self,AllocatorType allocator_type,size_t alloc_size,size_t * bytes_allocated,size_t * usable_size,size_t * bytes_tl_bulk_allocated)301 inline mirror::Object* Heap::TryToAllocate(Thread* self,
302 AllocatorType allocator_type,
303 size_t alloc_size,
304 size_t* bytes_allocated,
305 size_t* usable_size,
306 size_t* bytes_tl_bulk_allocated) {
307 if (allocator_type != kAllocatorTypeRegionTLAB &&
308 allocator_type != kAllocatorTypeTLAB &&
309 allocator_type != kAllocatorTypeRosAlloc &&
310 UNLIKELY(IsOutOfMemoryOnAllocation(allocator_type, alloc_size, kGrow))) {
311 return nullptr;
312 }
313 mirror::Object* ret;
314 switch (allocator_type) {
315 case kAllocatorTypeBumpPointer: {
316 DCHECK(bump_pointer_space_ != nullptr);
317 alloc_size = RoundUp(alloc_size, space::BumpPointerSpace::kAlignment);
318 ret = bump_pointer_space_->AllocNonvirtual(alloc_size);
319 if (LIKELY(ret != nullptr)) {
320 *bytes_allocated = alloc_size;
321 *usable_size = alloc_size;
322 *bytes_tl_bulk_allocated = alloc_size;
323 }
324 break;
325 }
326 case kAllocatorTypeRosAlloc: {
327 if (kInstrumented && UNLIKELY(is_running_on_memory_tool_)) {
328 // If running on ASan, we should be using the instrumented path.
329 size_t max_bytes_tl_bulk_allocated = rosalloc_space_->MaxBytesBulkAllocatedFor(alloc_size);
330 if (UNLIKELY(IsOutOfMemoryOnAllocation(allocator_type,
331 max_bytes_tl_bulk_allocated,
332 kGrow))) {
333 return nullptr;
334 }
335 ret = rosalloc_space_->Alloc(self, alloc_size, bytes_allocated, usable_size,
336 bytes_tl_bulk_allocated);
337 } else {
338 DCHECK(!is_running_on_memory_tool_);
339 size_t max_bytes_tl_bulk_allocated =
340 rosalloc_space_->MaxBytesBulkAllocatedForNonvirtual(alloc_size);
341 if (UNLIKELY(IsOutOfMemoryOnAllocation(allocator_type,
342 max_bytes_tl_bulk_allocated,
343 kGrow))) {
344 return nullptr;
345 }
346 if (!kInstrumented) {
347 DCHECK(!rosalloc_space_->CanAllocThreadLocal(self, alloc_size));
348 }
349 ret = rosalloc_space_->AllocNonvirtual(self,
350 alloc_size,
351 bytes_allocated,
352 usable_size,
353 bytes_tl_bulk_allocated);
354 }
355 break;
356 }
357 case kAllocatorTypeDlMalloc: {
358 if (kInstrumented && UNLIKELY(is_running_on_memory_tool_)) {
359 // If running on ASan, we should be using the instrumented path.
360 ret = dlmalloc_space_->Alloc(self,
361 alloc_size,
362 bytes_allocated,
363 usable_size,
364 bytes_tl_bulk_allocated);
365 } else {
366 DCHECK(!is_running_on_memory_tool_);
367 ret = dlmalloc_space_->AllocNonvirtual(self,
368 alloc_size,
369 bytes_allocated,
370 usable_size,
371 bytes_tl_bulk_allocated);
372 }
373 break;
374 }
375 case kAllocatorTypeNonMoving: {
376 ret = non_moving_space_->Alloc(self,
377 alloc_size,
378 bytes_allocated,
379 usable_size,
380 bytes_tl_bulk_allocated);
381 break;
382 }
383 case kAllocatorTypeLOS: {
384 ret = large_object_space_->Alloc(self,
385 alloc_size,
386 bytes_allocated,
387 usable_size,
388 bytes_tl_bulk_allocated);
389 // Note that the bump pointer spaces aren't necessarily next to
390 // the other continuous spaces like the non-moving alloc space or
391 // the zygote space.
392 DCHECK(ret == nullptr || large_object_space_->Contains(ret));
393 break;
394 }
395 case kAllocatorTypeRegion: {
396 DCHECK(region_space_ != nullptr);
397 alloc_size = RoundUp(alloc_size, space::RegionSpace::kAlignment);
398 ret = region_space_->AllocNonvirtual<false>(alloc_size,
399 bytes_allocated,
400 usable_size,
401 bytes_tl_bulk_allocated);
402 break;
403 }
404 case kAllocatorTypeTLAB:
405 FALLTHROUGH_INTENDED;
406 case kAllocatorTypeRegionTLAB: {
407 DCHECK_ALIGNED(alloc_size, kObjectAlignment);
408 static_assert(space::RegionSpace::kAlignment == space::BumpPointerSpace::kAlignment,
409 "mismatched alignments");
410 static_assert(kObjectAlignment == space::BumpPointerSpace::kAlignment,
411 "mismatched alignments");
412 if (UNLIKELY(self->TlabSize() < alloc_size)) {
413 return AllocWithNewTLAB(self,
414 allocator_type,
415 alloc_size,
416 kGrow,
417 bytes_allocated,
418 usable_size,
419 bytes_tl_bulk_allocated);
420 }
421 // The allocation can't fail.
422 ret = self->AllocTlab(alloc_size);
423 DCHECK(ret != nullptr);
424 *bytes_allocated = alloc_size;
425 *bytes_tl_bulk_allocated = 0; // Allocated in an existing buffer.
426 *usable_size = alloc_size;
427 break;
428 }
429 default: {
430 LOG(FATAL) << "Invalid allocator type";
431 ret = nullptr;
432 }
433 }
434 return ret;
435 }
436
ShouldAllocLargeObject(ObjPtr<mirror::Class> c,size_t byte_count)437 inline bool Heap::ShouldAllocLargeObject(ObjPtr<mirror::Class> c, size_t byte_count) const {
438 // We need to have a zygote space or else our newly allocated large object can end up in the
439 // Zygote resulting in it being prematurely freed.
440 // We can only do this for primitive objects since large objects will not be within the card table
441 // range. This also means that we rely on SetClass not dirtying the object's card.
442 return byte_count >= large_object_threshold_ && (c->IsPrimitiveArray() || c->IsStringClass());
443 }
444
IsOutOfMemoryOnAllocation(AllocatorType allocator_type,size_t alloc_size,bool grow)445 inline bool Heap::IsOutOfMemoryOnAllocation([[maybe_unused]] AllocatorType allocator_type,
446 size_t alloc_size,
447 bool grow) {
448 size_t old_target = target_footprint_.load(std::memory_order_relaxed);
449 while (true) {
450 size_t old_allocated = num_bytes_allocated_.load(std::memory_order_relaxed);
451 size_t new_footprint = old_allocated + alloc_size;
452 // Tests against heap limits are inherently approximate, since multiple allocations may
453 // race, and this is not atomic with the allocation.
454 if (UNLIKELY(new_footprint <= old_target)) {
455 return false;
456 } else if (UNLIKELY(new_footprint > growth_limit_)) {
457 return true;
458 }
459 // We are between target_footprint_ and growth_limit_ .
460 if (IsGcConcurrent()) {
461 return false;
462 } else {
463 if (grow) {
464 if (target_footprint_.compare_exchange_weak(/*inout ref*/old_target, new_footprint,
465 std::memory_order_relaxed)) {
466 VlogHeapGrowth(old_target, new_footprint, alloc_size);
467 return false;
468 } // else try again.
469 } else {
470 return true;
471 }
472 }
473 }
474 }
475
ShouldConcurrentGCForJava(size_t new_num_bytes_allocated)476 inline bool Heap::ShouldConcurrentGCForJava(size_t new_num_bytes_allocated) {
477 // For a Java allocation, we only check whether the number of Java allocated bytes excceeds a
478 // threshold. By not considering native allocation here, we (a) ensure that Java heap bounds are
479 // maintained, and (b) reduce the cost of the check here.
480 return new_num_bytes_allocated >= concurrent_start_bytes_;
481 }
482
483 } // namespace gc
484 } // namespace art
485
486 #endif // ART_RUNTIME_GC_HEAP_INL_H_
487