1 /*
2 * Copyright (C) 2011 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 #ifndef ART_RUNTIME_MIRROR_OBJECT_READBARRIER_INL_H_
18 #define ART_RUNTIME_MIRROR_OBJECT_READBARRIER_INL_H_
19
20 #include "object.h"
21
22 #include "base/atomic.h"
23 #include "heap_poisoning.h"
24 #include "lock_word-inl.h"
25 #include "object_reference-inl.h"
26 #include "read_barrier.h"
27 #include "runtime.h"
28
29 namespace art {
30 namespace mirror {
31
32 template<VerifyObjectFlags kVerifyFlags>
GetLockWord(bool as_volatile)33 inline LockWord Object::GetLockWord(bool as_volatile) {
34 if (as_volatile) {
35 return LockWord(GetField32Volatile<kVerifyFlags>(MonitorOffset()));
36 }
37 return LockWord(GetField32<kVerifyFlags>(MonitorOffset()));
38 }
39
40 template<bool kTransactionActive, bool kCheckTransaction, VerifyObjectFlags kVerifyFlags>
CasField32(MemberOffset field_offset,int32_t old_value,int32_t new_value,CASMode mode,std::memory_order memory_order)41 inline bool Object::CasField32(MemberOffset field_offset,
42 int32_t old_value,
43 int32_t new_value,
44 CASMode mode,
45 std::memory_order memory_order) {
46 if (kCheckTransaction) {
47 DCHECK_EQ(kTransactionActive, Runtime::Current()->IsActiveTransaction());
48 }
49 if (kTransactionActive) {
50 Runtime::Current()->RecordWriteField32(this, field_offset, old_value, true);
51 }
52 if (kVerifyFlags & kVerifyThis) {
53 VerifyObject(this);
54 }
55 uint8_t* raw_addr = reinterpret_cast<uint8_t*>(this) + field_offset.Int32Value();
56 AtomicInteger* atomic_addr = reinterpret_cast<AtomicInteger*>(raw_addr);
57
58 return atomic_addr->CompareAndSet(old_value, new_value, mode, memory_order);
59 }
60
CasLockWord(LockWord old_val,LockWord new_val,CASMode mode,std::memory_order memory_order)61 inline bool Object::CasLockWord(LockWord old_val,
62 LockWord new_val,
63 CASMode mode,
64 std::memory_order memory_order) {
65 // Force use of non-transactional mode and do not check.
66 return CasField32<false, false>(MonitorOffset(),
67 old_val.GetValue(),
68 new_val.GetValue(),
69 mode,
70 memory_order);
71 }
72
GetReadBarrierState(uintptr_t * fake_address_dependency)73 inline uint32_t Object::GetReadBarrierState(uintptr_t* fake_address_dependency) {
74 if (!kUseBakerReadBarrier) {
75 LOG(FATAL) << "Unreachable";
76 UNREACHABLE();
77 }
78 #if defined(__arm__)
79 uintptr_t obj = reinterpret_cast<uintptr_t>(this);
80 uintptr_t result;
81 DCHECK_EQ(OFFSETOF_MEMBER(Object, monitor_), 4U);
82 // Use inline assembly to prevent the compiler from optimizing away the false dependency.
83 __asm__ __volatile__(
84 "ldr %[result], [%[obj], #4]\n\t"
85 // This instruction is enough to "fool the compiler and the CPU" by having `fad` always be
86 // null, without them being able to assume that fact.
87 "eor %[fad], %[result], %[result]\n\t"
88 : [result] "+r" (result), [fad] "=r" (*fake_address_dependency)
89 : [obj] "r" (obj));
90 DCHECK_EQ(*fake_address_dependency, 0U);
91 LockWord lw(static_cast<uint32_t>(result));
92 uint32_t rb_state = lw.ReadBarrierState();
93 return rb_state;
94 #elif defined(__aarch64__)
95 uintptr_t obj = reinterpret_cast<uintptr_t>(this);
96 uintptr_t result;
97 DCHECK_EQ(OFFSETOF_MEMBER(Object, monitor_), 4U);
98 // Use inline assembly to prevent the compiler from optimizing away the false dependency.
99 __asm__ __volatile__(
100 "ldr %w[result], [%[obj], #4]\n\t"
101 // This instruction is enough to "fool the compiler and the CPU" by having `fad` always be
102 // null, without them being able to assume that fact.
103 "eor %[fad], %[result], %[result]\n\t"
104 : [result] "+r" (result), [fad] "=r" (*fake_address_dependency)
105 : [obj] "r" (obj));
106 DCHECK_EQ(*fake_address_dependency, 0U);
107 LockWord lw(static_cast<uint32_t>(result));
108 uint32_t rb_state = lw.ReadBarrierState();
109 return rb_state;
110 #elif defined(__i386__) || defined(__x86_64__)
111 LockWord lw = GetLockWord(false);
112 // i386/x86_64 don't need fake address dependency. Use a compiler fence to avoid compiler
113 // reordering.
114 *fake_address_dependency = 0;
115 std::atomic_signal_fence(std::memory_order_acquire);
116 uint32_t rb_state = lw.ReadBarrierState();
117 return rb_state;
118 #else
119 // MIPS32/MIPS64: use a memory barrier to prevent load-load reordering.
120 LockWord lw = GetLockWord(false);
121 *fake_address_dependency = 0;
122 std::atomic_thread_fence(std::memory_order_acquire);
123 uint32_t rb_state = lw.ReadBarrierState();
124 return rb_state;
125 #endif
126 }
127
GetReadBarrierState()128 inline uint32_t Object::GetReadBarrierState() {
129 if (!kUseBakerReadBarrier) {
130 LOG(FATAL) << "Unreachable";
131 UNREACHABLE();
132 }
133 DCHECK(kUseBakerReadBarrier);
134 LockWord lw(GetFieldPrimitive<uint32_t, /*kIsVolatile=*/false>(MonitorOffset()));
135 uint32_t rb_state = lw.ReadBarrierState();
136 DCHECK(ReadBarrier::IsValidReadBarrierState(rb_state)) << rb_state;
137 return rb_state;
138 }
139
GetReadBarrierStateAcquire()140 inline uint32_t Object::GetReadBarrierStateAcquire() {
141 if (!kUseBakerReadBarrier) {
142 LOG(FATAL) << "Unreachable";
143 UNREACHABLE();
144 }
145 LockWord lw(GetFieldAcquire<uint32_t>(MonitorOffset()));
146 uint32_t rb_state = lw.ReadBarrierState();
147 DCHECK(ReadBarrier::IsValidReadBarrierState(rb_state)) << rb_state;
148 return rb_state;
149 }
150
151 template<std::memory_order kMemoryOrder>
AtomicSetReadBarrierState(uint32_t expected_rb_state,uint32_t rb_state)152 inline bool Object::AtomicSetReadBarrierState(uint32_t expected_rb_state, uint32_t rb_state) {
153 if (!kUseBakerReadBarrier) {
154 LOG(FATAL) << "Unreachable";
155 UNREACHABLE();
156 }
157 DCHECK(ReadBarrier::IsValidReadBarrierState(expected_rb_state)) << expected_rb_state;
158 DCHECK(ReadBarrier::IsValidReadBarrierState(rb_state)) << rb_state;
159 LockWord expected_lw;
160 LockWord new_lw;
161 do {
162 LockWord lw = GetLockWord(false);
163 if (UNLIKELY(lw.ReadBarrierState() != expected_rb_state)) {
164 // Lost the race.
165 return false;
166 }
167 expected_lw = lw;
168 expected_lw.SetReadBarrierState(expected_rb_state);
169 new_lw = lw;
170 new_lw.SetReadBarrierState(rb_state);
171 // ConcurrentCopying::ProcessMarkStackRef uses this with
172 // `kMemoryOrder` == `std::memory_order_release`.
173 // If `kMemoryOrder` == `std::memory_order_release`, use a CAS release so that when GC updates
174 // all the fields of an object and then changes the object from gray to black (non-gray), the
175 // field updates (stores) will be visible (won't be reordered after this CAS.)
176 } while (!CasLockWord(expected_lw, new_lw, CASMode::kWeak, kMemoryOrder));
177 return true;
178 }
179
AtomicSetMarkBit(uint32_t expected_mark_bit,uint32_t mark_bit)180 inline bool Object::AtomicSetMarkBit(uint32_t expected_mark_bit, uint32_t mark_bit) {
181 LockWord expected_lw;
182 LockWord new_lw;
183 do {
184 LockWord lw = GetLockWord(false);
185 if (UNLIKELY(lw.MarkBitState() != expected_mark_bit)) {
186 // Lost the race.
187 return false;
188 }
189 expected_lw = lw;
190 new_lw = lw;
191 new_lw.SetMarkBitState(mark_bit);
192 // Since this is only set from the mutator, we can use the non-release CAS.
193 } while (!CasLockWord(expected_lw, new_lw, CASMode::kWeak, std::memory_order_relaxed));
194 return true;
195 }
196
197 } // namespace mirror
198 } // namespace art
199
200 #endif // ART_RUNTIME_MIRROR_OBJECT_READBARRIER_INL_H_
201