1 /*
2  * Copyright (C) 2011 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 #ifndef ART_RUNTIME_MIRROR_OBJECT_READBARRIER_INL_H_
18 #define ART_RUNTIME_MIRROR_OBJECT_READBARRIER_INL_H_
19 
20 #include "object.h"
21 
22 #include "base/atomic.h"
23 #include "heap_poisoning.h"
24 #include "lock_word-inl.h"
25 #include "object_reference-inl.h"
26 #include "read_barrier.h"
27 #include "runtime.h"
28 
29 namespace art {
30 namespace mirror {
31 
32 template<VerifyObjectFlags kVerifyFlags>
GetLockWord(bool as_volatile)33 inline LockWord Object::GetLockWord(bool as_volatile) {
34   if (as_volatile) {
35     return LockWord(GetField32Volatile<kVerifyFlags>(OFFSET_OF_OBJECT_MEMBER(Object, monitor_)));
36   }
37   return LockWord(GetField32<kVerifyFlags>(OFFSET_OF_OBJECT_MEMBER(Object, monitor_)));
38 }
39 
40 template<bool kTransactionActive, bool kCheckTransaction, VerifyObjectFlags kVerifyFlags>
CasFieldWeakRelaxed32(MemberOffset field_offset,int32_t old_value,int32_t new_value)41 inline bool Object::CasFieldWeakRelaxed32(MemberOffset field_offset,
42                                           int32_t old_value, int32_t new_value) {
43   if (kCheckTransaction) {
44     DCHECK_EQ(kTransactionActive, Runtime::Current()->IsActiveTransaction());
45   }
46   if (kTransactionActive) {
47     Runtime::Current()->RecordWriteField32(this, field_offset, old_value, true);
48   }
49   if (kVerifyFlags & kVerifyThis) {
50     VerifyObject(this);
51   }
52   uint8_t* raw_addr = reinterpret_cast<uint8_t*>(this) + field_offset.Int32Value();
53   AtomicInteger* atomic_addr = reinterpret_cast<AtomicInteger*>(raw_addr);
54 
55   return atomic_addr->CompareAndSetWeakRelaxed(old_value, new_value);
56 }
57 
CasLockWordWeakRelaxed(LockWord old_val,LockWord new_val)58 inline bool Object::CasLockWordWeakRelaxed(LockWord old_val, LockWord new_val) {
59   // Force use of non-transactional mode and do not check.
60   return CasFieldWeakRelaxed32<false, false>(
61       OFFSET_OF_OBJECT_MEMBER(Object, monitor_), old_val.GetValue(), new_val.GetValue());
62 }
63 
CasLockWordWeakRelease(LockWord old_val,LockWord new_val)64 inline bool Object::CasLockWordWeakRelease(LockWord old_val, LockWord new_val) {
65   // Force use of non-transactional mode and do not check.
66   return CasFieldWeakRelease32<false, false>(
67       OFFSET_OF_OBJECT_MEMBER(Object, monitor_), old_val.GetValue(), new_val.GetValue());
68 }
69 
GetReadBarrierState(uintptr_t * fake_address_dependency)70 inline uint32_t Object::GetReadBarrierState(uintptr_t* fake_address_dependency) {
71   if (!kUseBakerReadBarrier) {
72     LOG(FATAL) << "Unreachable";
73     UNREACHABLE();
74   }
75 #if defined(__arm__)
76   uintptr_t obj = reinterpret_cast<uintptr_t>(this);
77   uintptr_t result;
78   DCHECK_EQ(OFFSETOF_MEMBER(Object, monitor_), 4U);
79   // Use inline assembly to prevent the compiler from optimizing away the false dependency.
80   __asm__ __volatile__(
81       "ldr %[result], [%[obj], #4]\n\t"
82       // This instruction is enough to "fool the compiler and the CPU" by having `fad` always be
83       // null, without them being able to assume that fact.
84       "eor %[fad], %[result], %[result]\n\t"
85       : [result] "+r" (result), [fad] "=r" (*fake_address_dependency)
86       : [obj] "r" (obj));
87   DCHECK_EQ(*fake_address_dependency, 0U);
88   LockWord lw(static_cast<uint32_t>(result));
89   uint32_t rb_state = lw.ReadBarrierState();
90   return rb_state;
91 #elif defined(__aarch64__)
92   uintptr_t obj = reinterpret_cast<uintptr_t>(this);
93   uintptr_t result;
94   DCHECK_EQ(OFFSETOF_MEMBER(Object, monitor_), 4U);
95   // Use inline assembly to prevent the compiler from optimizing away the false dependency.
96   __asm__ __volatile__(
97       "ldr %w[result], [%[obj], #4]\n\t"
98       // This instruction is enough to "fool the compiler and the CPU" by having `fad` always be
99       // null, without them being able to assume that fact.
100       "eor %[fad], %[result], %[result]\n\t"
101       : [result] "+r" (result), [fad] "=r" (*fake_address_dependency)
102       : [obj] "r" (obj));
103   DCHECK_EQ(*fake_address_dependency, 0U);
104   LockWord lw(static_cast<uint32_t>(result));
105   uint32_t rb_state = lw.ReadBarrierState();
106   return rb_state;
107 #elif defined(__i386__) || defined(__x86_64__)
108   LockWord lw = GetLockWord(false);
109   // i386/x86_64 don't need fake address dependency. Use a compiler fence to avoid compiler
110   // reordering.
111   *fake_address_dependency = 0;
112   std::atomic_signal_fence(std::memory_order_acquire);
113   uint32_t rb_state = lw.ReadBarrierState();
114   return rb_state;
115 #else
116   // MIPS32/MIPS64: use a memory barrier to prevent load-load reordering.
117   LockWord lw = GetLockWord(false);
118   *fake_address_dependency = 0;
119   std::atomic_thread_fence(std::memory_order_acquire);
120   uint32_t rb_state = lw.ReadBarrierState();
121   return rb_state;
122 #endif
123 }
124 
GetReadBarrierState()125 inline uint32_t Object::GetReadBarrierState() {
126   if (!kUseBakerReadBarrier) {
127     LOG(FATAL) << "Unreachable";
128     UNREACHABLE();
129   }
130   DCHECK(kUseBakerReadBarrier);
131   LockWord lw(GetField<uint32_t, /*kIsVolatile*/false>(OFFSET_OF_OBJECT_MEMBER(Object, monitor_)));
132   uint32_t rb_state = lw.ReadBarrierState();
133   DCHECK(ReadBarrier::IsValidReadBarrierState(rb_state)) << rb_state;
134   return rb_state;
135 }
136 
GetReadBarrierStateAcquire()137 inline uint32_t Object::GetReadBarrierStateAcquire() {
138   if (!kUseBakerReadBarrier) {
139     LOG(FATAL) << "Unreachable";
140     UNREACHABLE();
141   }
142   LockWord lw(GetFieldAcquire<uint32_t>(OFFSET_OF_OBJECT_MEMBER(Object, monitor_)));
143   uint32_t rb_state = lw.ReadBarrierState();
144   DCHECK(ReadBarrier::IsValidReadBarrierState(rb_state)) << rb_state;
145   return rb_state;
146 }
147 
148 template<bool kCasRelease>
AtomicSetReadBarrierState(uint32_t expected_rb_state,uint32_t rb_state)149 inline bool Object::AtomicSetReadBarrierState(uint32_t expected_rb_state, uint32_t rb_state) {
150   if (!kUseBakerReadBarrier) {
151     LOG(FATAL) << "Unreachable";
152     UNREACHABLE();
153   }
154   DCHECK(ReadBarrier::IsValidReadBarrierState(expected_rb_state)) << expected_rb_state;
155   DCHECK(ReadBarrier::IsValidReadBarrierState(rb_state)) << rb_state;
156   LockWord expected_lw;
157   LockWord new_lw;
158   do {
159     LockWord lw = GetLockWord(false);
160     if (UNLIKELY(lw.ReadBarrierState() != expected_rb_state)) {
161       // Lost the race.
162       return false;
163     }
164     expected_lw = lw;
165     expected_lw.SetReadBarrierState(expected_rb_state);
166     new_lw = lw;
167     new_lw.SetReadBarrierState(rb_state);
168     // ConcurrentCopying::ProcessMarkStackRef uses this with kCasRelease == true.
169     // If kCasRelease == true, use a CAS release so that when GC updates all the fields of
170     // an object and then changes the object from gray to black, the field updates (stores) will be
171     // visible (won't be reordered after this CAS.)
172   } while (!(kCasRelease ?
173              CasLockWordWeakRelease(expected_lw, new_lw) :
174              CasLockWordWeakRelaxed(expected_lw, new_lw)));
175   return true;
176 }
177 
AtomicSetMarkBit(uint32_t expected_mark_bit,uint32_t mark_bit)178 inline bool Object::AtomicSetMarkBit(uint32_t expected_mark_bit, uint32_t mark_bit) {
179   LockWord expected_lw;
180   LockWord new_lw;
181   do {
182     LockWord lw = GetLockWord(false);
183     if (UNLIKELY(lw.MarkBitState() != expected_mark_bit)) {
184       // Lost the race.
185       return false;
186     }
187     expected_lw = lw;
188     new_lw = lw;
189     new_lw.SetMarkBitState(mark_bit);
190     // Since this is only set from the mutator, we can use the non-release CAS.
191   } while (!CasLockWordWeakRelaxed(expected_lw, new_lw));
192   return true;
193 }
194 
195 template<bool kTransactionActive, bool kCheckTransaction, VerifyObjectFlags kVerifyFlags>
CasFieldStrongRelaxedObjectWithoutWriteBarrier(MemberOffset field_offset,ObjPtr<Object> old_value,ObjPtr<Object> new_value)196 inline bool Object::CasFieldStrongRelaxedObjectWithoutWriteBarrier(
197     MemberOffset field_offset,
198     ObjPtr<Object> old_value,
199     ObjPtr<Object> new_value) {
200   if (kCheckTransaction) {
201     DCHECK_EQ(kTransactionActive, Runtime::Current()->IsActiveTransaction());
202   }
203   if (kVerifyFlags & kVerifyThis) {
204     VerifyObject(this);
205   }
206   if (kVerifyFlags & kVerifyWrites) {
207     VerifyObject(new_value);
208   }
209   if (kVerifyFlags & kVerifyReads) {
210     VerifyObject(old_value);
211   }
212   if (kTransactionActive) {
213     Runtime::Current()->RecordWriteFieldReference(this, field_offset, old_value, true);
214   }
215   uint32_t old_ref(PtrCompression<kPoisonHeapReferences, Object>::Compress(old_value));
216   uint32_t new_ref(PtrCompression<kPoisonHeapReferences, Object>::Compress(new_value));
217   uint8_t* raw_addr = reinterpret_cast<uint8_t*>(this) + field_offset.Int32Value();
218   Atomic<uint32_t>* atomic_addr = reinterpret_cast<Atomic<uint32_t>*>(raw_addr);
219 
220   bool success = atomic_addr->CompareAndSetStrongRelaxed(old_ref, new_ref);
221   return success;
222 }
223 
224 template<bool kTransactionActive, bool kCheckTransaction, VerifyObjectFlags kVerifyFlags>
CasFieldStrongReleaseObjectWithoutWriteBarrier(MemberOffset field_offset,ObjPtr<Object> old_value,ObjPtr<Object> new_value)225 inline bool Object::CasFieldStrongReleaseObjectWithoutWriteBarrier(
226     MemberOffset field_offset,
227     ObjPtr<Object> old_value,
228     ObjPtr<Object> new_value) {
229   if (kCheckTransaction) {
230     DCHECK_EQ(kTransactionActive, Runtime::Current()->IsActiveTransaction());
231   }
232   if (kVerifyFlags & kVerifyThis) {
233     VerifyObject(this);
234   }
235   if (kVerifyFlags & kVerifyWrites) {
236     VerifyObject(new_value);
237   }
238   if (kVerifyFlags & kVerifyReads) {
239     VerifyObject(old_value);
240   }
241   if (kTransactionActive) {
242     Runtime::Current()->RecordWriteFieldReference(this, field_offset, old_value, true);
243   }
244   uint32_t old_ref(PtrCompression<kPoisonHeapReferences, Object>::Compress(old_value));
245   uint32_t new_ref(PtrCompression<kPoisonHeapReferences, Object>::Compress(new_value));
246   uint8_t* raw_addr = reinterpret_cast<uint8_t*>(this) + field_offset.Int32Value();
247   Atomic<uint32_t>* atomic_addr = reinterpret_cast<Atomic<uint32_t>*>(raw_addr);
248 
249   bool success = atomic_addr->CompareAndSetStrongRelease(old_ref, new_ref);
250   return success;
251 }
252 
253 }  // namespace mirror
254 }  // namespace art
255 
256 #endif  // ART_RUNTIME_MIRROR_OBJECT_READBARRIER_INL_H_
257