1 /*
2 * Copyright (C) 2015 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 #include "linker/arm64/relative_patcher_arm64.h"
18
19 #include "arch/arm64/instruction_set_features_arm64.h"
20 #include "base/casts.h"
21 #include "driver/compiler_options.h"
22 #include "linker/relative_patcher_test.h"
23 #include "lock_word.h"
24 #include "mirror/array-inl.h"
25 #include "mirror/object.h"
26 #include "oat/oat_quick_method_header.h"
27 #include "optimizing/code_generator_arm64.h"
28 #include "optimizing/optimizing_unit_test.h"
29
30 namespace art {
31 namespace linker {
32
33 class Arm64RelativePatcherTest : public RelativePatcherTest {
34 public:
Arm64RelativePatcherTest(const std::string & variant)35 explicit Arm64RelativePatcherTest(const std::string& variant)
36 : RelativePatcherTest(InstructionSet::kArm64, variant) { }
37
38 protected:
39 static const uint8_t kCallRawCode[];
40 static const ArrayRef<const uint8_t> kCallCode;
41 static const uint8_t kNopRawCode[];
42 static const ArrayRef<const uint8_t> kNopCode;
43
44 // NOP instruction.
45 static constexpr uint32_t kNopInsn = 0xd503201f;
46
47 // All branches can be created from kBlPlus0 or kBPlus0 by adding the low 26 bits.
48 static constexpr uint32_t kBlPlus0 = 0x94000000u;
49 static constexpr uint32_t kBPlus0 = 0x14000000u;
50
51 // Special BL values.
52 static constexpr uint32_t kBlPlusMax = 0x95ffffffu;
53 static constexpr uint32_t kBlMinusMax = 0x96000000u;
54
55 // LDR immediate, 32-bit, unsigned offset.
56 static constexpr uint32_t kLdrWInsn = 0xb9400000u;
57
58 // LDR register, 32-bit, LSL #2.
59 static constexpr uint32_t kLdrWLsl2Insn = 0xb8607800u;
60
61 // LDUR, 32-bit.
62 static constexpr uint32_t kLdurWInsn = 0xb8400000u;
63
64 // ADD/ADDS/SUB/SUBS immediate, 64-bit.
65 static constexpr uint32_t kAddXInsn = 0x91000000u;
66 static constexpr uint32_t kAddsXInsn = 0xb1000000u;
67 static constexpr uint32_t kSubXInsn = 0xd1000000u;
68 static constexpr uint32_t kSubsXInsn = 0xf1000000u;
69
70 // LDUR x2, [sp, #4], i.e. unaligned load crossing 64-bit boundary (assuming aligned sp).
71 static constexpr uint32_t kLdurInsn = 0xf840405fu;
72
73 // LDR w12, <label> and LDR x12, <label>. Bits 5-23 contain label displacement in 4-byte units.
74 static constexpr uint32_t kLdrWPcRelInsn = 0x1800000cu;
75 static constexpr uint32_t kLdrXPcRelInsn = 0x5800000cu;
76
77 // LDR w13, [SP, #<pimm>] and LDR x13, [SP, #<pimm>]. Bits 10-21 contain displacement from SP
78 // in units of 4-bytes (for 32-bit load) or 8-bytes (for 64-bit load).
79 static constexpr uint32_t kLdrWSpRelInsn = 0xb94003edu;
80 static constexpr uint32_t kLdrXSpRelInsn = 0xf94003edu;
81
82 // CBNZ x17, +0. Bits 5-23 are a placeholder for target offset from PC in units of 4-bytes.
83 static constexpr uint32_t kCbnzIP1Plus0Insn = 0xb5000011u;
84
InsertInsn(std::vector<uint8_t> * code,size_t pos,uint32_t insn)85 static void InsertInsn(std::vector<uint8_t>* code, size_t pos, uint32_t insn) {
86 CHECK_LE(pos, code->size());
87 const uint8_t insn_code[] = {
88 static_cast<uint8_t>(insn),
89 static_cast<uint8_t>(insn >> 8),
90 static_cast<uint8_t>(insn >> 16),
91 static_cast<uint8_t>(insn >> 24),
92 };
93 static_assert(sizeof(insn_code) == 4u, "Invalid sizeof(insn_code).");
94 code->insert(code->begin() + pos, insn_code, insn_code + sizeof(insn_code));
95 }
96
PushBackInsn(std::vector<uint8_t> * code,uint32_t insn)97 static void PushBackInsn(std::vector<uint8_t>* code, uint32_t insn) {
98 InsertInsn(code, code->size(), insn);
99 }
100
RawCode(std::initializer_list<uint32_t> insns)101 static std::vector<uint8_t> RawCode(std::initializer_list<uint32_t> insns) {
102 std::vector<uint8_t> raw_code;
103 raw_code.reserve(insns.size() * 4u);
104 for (uint32_t insn : insns) {
105 PushBackInsn(&raw_code, insn);
106 }
107 return raw_code;
108 }
109
Create2MethodsWithGap(const ArrayRef<const uint8_t> & method1_code,const ArrayRef<const LinkerPatch> & method1_patches,const ArrayRef<const uint8_t> & last_method_code,const ArrayRef<const LinkerPatch> & last_method_patches,uint32_t distance_without_thunks)110 uint32_t Create2MethodsWithGap(const ArrayRef<const uint8_t>& method1_code,
111 const ArrayRef<const LinkerPatch>& method1_patches,
112 const ArrayRef<const uint8_t>& last_method_code,
113 const ArrayRef<const LinkerPatch>& last_method_patches,
114 uint32_t distance_without_thunks) {
115 CHECK_EQ(distance_without_thunks % kArm64CodeAlignment, 0u);
116 uint32_t method1_offset =
117 kTrampolineSize + CodeAlignmentSize(kTrampolineSize) + sizeof(OatQuickMethodHeader);
118 AddCompiledMethod(MethodRef(1u), method1_code, method1_patches);
119 const uint32_t gap_start = method1_offset + method1_code.size();
120
121 // We want to put the last method at a very precise offset.
122 const uint32_t last_method_offset = method1_offset + distance_without_thunks;
123 CHECK_ALIGNED(last_method_offset, kArm64CodeAlignment);
124 const uint32_t gap_end = last_method_offset - sizeof(OatQuickMethodHeader);
125
126 // Fill the gap with intermediate methods in chunks of 2MiB and the first in [2MiB, 4MiB).
127 // (This allows deduplicating the small chunks to avoid using 256MiB of memory for +-128MiB
128 // offsets by this test. Making the first chunk bigger makes it easy to give all intermediate
129 // methods the same alignment of the end, so the thunk insertion adds a predictable size as
130 // long as it's after the first chunk.)
131 uint32_t method_idx = 2u;
132 constexpr uint32_t kSmallChunkSize = 2 * MB;
133 std::vector<uint8_t> gap_code;
134 uint32_t gap_size = gap_end - gap_start;
135 uint32_t num_small_chunks = std::max(gap_size / kSmallChunkSize, 1u) - 1u;
136 uint32_t chunk_start = gap_start;
137 uint32_t chunk_size = gap_size - num_small_chunks * kSmallChunkSize;
138 for (uint32_t i = 0; i <= num_small_chunks; ++i) { // num_small_chunks+1 iterations.
139 uint32_t chunk_code_size =
140 chunk_size - CodeAlignmentSize(chunk_start) - sizeof(OatQuickMethodHeader);
141 gap_code.resize(chunk_code_size, 0u);
142 AddCompiledMethod(MethodRef(method_idx), ArrayRef<const uint8_t>(gap_code));
143 method_idx += 1u;
144 chunk_start += chunk_size;
145 chunk_size = kSmallChunkSize; // For all but the first chunk.
146 DCHECK_EQ(CodeAlignmentSize(gap_end), CodeAlignmentSize(chunk_start));
147 }
148
149 // Add the last method and link
150 AddCompiledMethod(MethodRef(method_idx), last_method_code, last_method_patches);
151 Link();
152
153 // Check assumptions.
154 CHECK_EQ(GetMethodOffset(1), method1_offset);
155 auto last_result = method_offset_map_.FindMethodOffset(MethodRef(method_idx));
156 CHECK(last_result.first);
157 // There may be a thunk before method2.
158 if (last_result.second != last_method_offset) {
159 // Thunk present. Check that there's only one.
160 uint32_t thunk_end =
161 CompiledCode::AlignCode(gap_end, InstructionSet::kArm64) + MethodCallThunkSize();
162 uint32_t header_offset = thunk_end + CodeAlignmentSize(thunk_end);
163 CHECK_EQ(last_result.second, header_offset + sizeof(OatQuickMethodHeader));
164 }
165 return method_idx;
166 }
167
GetMethodOffset(uint32_t method_idx)168 uint32_t GetMethodOffset(uint32_t method_idx) {
169 auto result = method_offset_map_.FindMethodOffset(MethodRef(method_idx));
170 CHECK(result.first);
171 CHECK_ALIGNED(result.second, 4u);
172 return result.second;
173 }
174
CompileThunk(const LinkerPatch & patch,std::string * debug_name=nullptr)175 std::vector<uint8_t> CompileThunk(const LinkerPatch& patch,
176 /*out*/ std::string* debug_name = nullptr) {
177 OptimizingUnitTestHelper helper;
178 HGraph* graph = helper.CreateGraph();
179 CompilerOptions compiler_options;
180
181 // Set isa to arm64.
182 compiler_options.instruction_set_ = instruction_set_;
183 compiler_options.instruction_set_features_ =
184 InstructionSetFeatures::FromBitmap(instruction_set_, instruction_set_features_->AsBitmap());
185 CHECK(compiler_options.instruction_set_features_->Equals(instruction_set_features_.get()));
186
187 // If a test requests that implicit null checks are enabled or disabled,
188 // apply that option, otherwise use the default from `CompilerOptions`.
189 if (implicit_null_checks_.has_value()) {
190 compiler_options.implicit_null_checks_ = implicit_null_checks_.value();
191 }
192
193 arm64::CodeGeneratorARM64 codegen(graph, compiler_options);
194 ArenaVector<uint8_t> code(helper.GetAllocator()->Adapter());
195 codegen.EmitThunkCode(patch, &code, debug_name);
196 return std::vector<uint8_t>(code.begin(), code.end());
197 }
198
AddCompiledMethod(MethodReference method_ref,const ArrayRef<const uint8_t> & code,const ArrayRef<const LinkerPatch> & patches=ArrayRef<const LinkerPatch> ())199 void AddCompiledMethod(
200 MethodReference method_ref,
201 const ArrayRef<const uint8_t>& code,
202 const ArrayRef<const LinkerPatch>& patches = ArrayRef<const LinkerPatch>()) {
203 RelativePatcherTest::AddCompiledMethod(method_ref, code, patches);
204
205 // Make sure the ThunkProvider has all the necessary thunks.
206 for (const LinkerPatch& patch : patches) {
207 if (patch.GetType() == LinkerPatch::Type::kCallEntrypoint ||
208 patch.GetType() == LinkerPatch::Type::kBakerReadBarrierBranch ||
209 patch.GetType() == LinkerPatch::Type::kCallRelative) {
210 std::string debug_name;
211 std::vector<uint8_t> thunk_code = CompileThunk(patch, &debug_name);
212 thunk_provider_.SetThunkCode(patch, ArrayRef<const uint8_t>(thunk_code), debug_name);
213 }
214 }
215 }
216
CompileMethodCallThunk()217 std::vector<uint8_t> CompileMethodCallThunk() {
218 LinkerPatch patch = LinkerPatch::RelativeCodePatch(/* literal_offset */ 0u,
219 /* target_dex_file*/ nullptr,
220 /* target_method_idx */ 0u);
221 return CompileThunk(patch);
222 }
223
MethodCallThunkSize()224 uint32_t MethodCallThunkSize() {
225 return CompileMethodCallThunk().size();
226 }
227
CheckThunk(uint32_t thunk_offset)228 bool CheckThunk(uint32_t thunk_offset) {
229 const std::vector<uint8_t> expected_code = CompileMethodCallThunk();
230 if (output_.size() < thunk_offset + expected_code.size()) {
231 LOG(ERROR) << "output_.size() == " << output_.size() << " < "
232 << "thunk_offset + expected_code.size() == " << (thunk_offset + expected_code.size());
233 return false;
234 }
235 ArrayRef<const uint8_t> linked_code(&output_[thunk_offset], expected_code.size());
236 if (linked_code == ArrayRef<const uint8_t>(expected_code)) {
237 return true;
238 }
239 // Log failure info.
240 DumpDiff(ArrayRef<const uint8_t>(expected_code), linked_code);
241 return false;
242 }
243
GenNops(size_t num_nops)244 static std::vector<uint8_t> GenNops(size_t num_nops) {
245 std::vector<uint8_t> result;
246 result.reserve(num_nops * 4u);
247 for (size_t i = 0; i != num_nops; ++i) {
248 PushBackInsn(&result, kNopInsn);
249 }
250 return result;
251 }
252
GenNopsAndBl(size_t num_nops,uint32_t bl)253 static std::vector<uint8_t> GenNopsAndBl(size_t num_nops, uint32_t bl) {
254 std::vector<uint8_t> result;
255 result.reserve(num_nops * 4u + 4u);
256 for (size_t i = 0; i != num_nops; ++i) {
257 PushBackInsn(&result, kNopInsn);
258 }
259 PushBackInsn(&result, bl);
260 return result;
261 }
262
GenNopsAndAdrpAndUse(size_t num_nops,uint32_t method_offset,uint32_t target_offset,uint32_t use_insn)263 std::vector<uint8_t> GenNopsAndAdrpAndUse(size_t num_nops,
264 uint32_t method_offset,
265 uint32_t target_offset,
266 uint32_t use_insn) {
267 std::vector<uint8_t> result;
268 result.reserve(num_nops * 4u + 8u);
269 for (size_t i = 0; i != num_nops; ++i) {
270 PushBackInsn(&result, kNopInsn);
271 }
272 CHECK_ALIGNED(method_offset, 4u);
273 CHECK_ALIGNED(target_offset, 4u);
274 uint32_t adrp_offset = method_offset + num_nops * 4u;
275 uint32_t disp = target_offset - (adrp_offset & ~0xfffu);
276 if (use_insn == kLdrWInsn) {
277 DCHECK_ALIGNED(disp, 1u << 2);
278 use_insn |= 1 | // LDR x1, [x0, #(imm12 << 2)]
279 ((disp & 0xfffu) << (10 - 2)); // imm12 = ((disp & 0xfffu) >> 2) is at bit 10.
280 } else if (use_insn == kAddXInsn) {
281 use_insn |= 1 | // ADD x1, x0, #imm
282 (disp & 0xfffu) << 10; // imm12 = (disp & 0xfffu) is at bit 10.
283 } else {
284 LOG(FATAL) << "Unexpected instruction: 0x" << std::hex << use_insn;
285 }
286 uint32_t adrp = 0x90000000u | // ADRP x0, +SignExtend(immhi:immlo:Zeros(12), 64)
287 ((disp & 0x3000u) << (29 - 12)) | // immlo = ((disp & 0x3000u) >> 12) is at bit 29,
288 ((disp & 0xffffc000) >> (14 - 5)) | // immhi = (disp >> 14) is at bit 5,
289 // We take the sign bit from the disp, limiting disp to +- 2GiB.
290 ((disp & 0x80000000) >> (31 - 23)); // sign bit in immhi is at bit 23.
291 PushBackInsn(&result, adrp);
292 PushBackInsn(&result, use_insn);
293 return result;
294 }
295
GenNopsAndAdrpLdr(size_t num_nops,uint32_t method_offset,uint32_t target_offset)296 std::vector<uint8_t> GenNopsAndAdrpLdr(size_t num_nops,
297 uint32_t method_offset,
298 uint32_t target_offset) {
299 return GenNopsAndAdrpAndUse(num_nops, method_offset, target_offset, kLdrWInsn);
300 }
301
TestNopsAdrpLdr(size_t num_nops,uint32_t bss_begin,uint32_t string_entry_offset)302 void TestNopsAdrpLdr(size_t num_nops, uint32_t bss_begin, uint32_t string_entry_offset) {
303 constexpr uint32_t kStringIndex = 1u;
304 string_index_to_offset_map_.Put(kStringIndex, string_entry_offset);
305 bss_begin_ = bss_begin;
306 auto code = GenNopsAndAdrpLdr(num_nops, 0u, 0u); // Unpatched.
307 const LinkerPatch patches[] = {
308 LinkerPatch::StringBssEntryPatch(num_nops * 4u , nullptr, num_nops * 4u, kStringIndex),
309 LinkerPatch::StringBssEntryPatch(num_nops * 4u + 4u, nullptr, num_nops * 4u, kStringIndex),
310 };
311 AddCompiledMethod(MethodRef(1u),
312 ArrayRef<const uint8_t>(code),
313 ArrayRef<const LinkerPatch>(patches));
314 Link();
315
316 uint32_t method1_offset = GetMethodOffset(1u);
317 uint32_t target_offset = bss_begin_ + string_entry_offset;
318 auto expected_code = GenNopsAndAdrpLdr(num_nops, method1_offset, target_offset);
319 EXPECT_TRUE(CheckLinkedMethod(MethodRef(1u), ArrayRef<const uint8_t>(expected_code)));
320 }
321
GenNopsAndAdrpAdd(size_t num_nops,uint32_t method_offset,uint32_t target_offset)322 std::vector<uint8_t> GenNopsAndAdrpAdd(size_t num_nops,
323 uint32_t method_offset,
324 uint32_t target_offset) {
325 return GenNopsAndAdrpAndUse(num_nops, method_offset, target_offset, kAddXInsn);
326 }
327
TestNopsAdrpAdd(size_t num_nops,uint32_t string_offset)328 void TestNopsAdrpAdd(size_t num_nops, uint32_t string_offset) {
329 constexpr uint32_t kStringIndex = 1u;
330 string_index_to_offset_map_.Put(kStringIndex, string_offset);
331 auto code = GenNopsAndAdrpAdd(num_nops, 0u, 0u); // Unpatched.
332 const LinkerPatch patches[] = {
333 LinkerPatch::RelativeStringPatch(num_nops * 4u , nullptr, num_nops * 4u, kStringIndex),
334 LinkerPatch::RelativeStringPatch(num_nops * 4u + 4u, nullptr, num_nops * 4u, kStringIndex),
335 };
336 AddCompiledMethod(MethodRef(1u),
337 ArrayRef<const uint8_t>(code),
338 ArrayRef<const LinkerPatch>(patches));
339 Link();
340
341 uint32_t method1_offset = GetMethodOffset(1u);
342 auto expected_code = GenNopsAndAdrpAdd(num_nops, method1_offset, string_offset);
343 EXPECT_TRUE(CheckLinkedMethod(MethodRef(1u), ArrayRef<const uint8_t>(expected_code)));
344 }
345
PrepareNopsAdrpInsn2Ldr(size_t num_nops,uint32_t insn2,uint32_t bss_begin,uint32_t string_entry_offset)346 void PrepareNopsAdrpInsn2Ldr(size_t num_nops,
347 uint32_t insn2,
348 uint32_t bss_begin,
349 uint32_t string_entry_offset) {
350 constexpr uint32_t kStringIndex = 1u;
351 string_index_to_offset_map_.Put(kStringIndex, string_entry_offset);
352 bss_begin_ = bss_begin;
353 auto code = GenNopsAndAdrpLdr(num_nops, 0u, 0u); // Unpatched.
354 InsertInsn(&code, num_nops * 4u + 4u, insn2);
355 const LinkerPatch patches[] = {
356 LinkerPatch::StringBssEntryPatch(num_nops * 4u , nullptr, num_nops * 4u, kStringIndex),
357 LinkerPatch::StringBssEntryPatch(num_nops * 4u + 8u, nullptr, num_nops * 4u, kStringIndex),
358 };
359 AddCompiledMethod(MethodRef(1u),
360 ArrayRef<const uint8_t>(code),
361 ArrayRef<const LinkerPatch>(patches));
362 Link();
363 }
364
PrepareNopsAdrpInsn2Add(size_t num_nops,uint32_t insn2,uint32_t string_offset)365 void PrepareNopsAdrpInsn2Add(size_t num_nops, uint32_t insn2, uint32_t string_offset) {
366 constexpr uint32_t kStringIndex = 1u;
367 string_index_to_offset_map_.Put(kStringIndex, string_offset);
368 auto code = GenNopsAndAdrpAdd(num_nops, 0u, 0u); // Unpatched.
369 InsertInsn(&code, num_nops * 4u + 4u, insn2);
370 const LinkerPatch patches[] = {
371 LinkerPatch::RelativeStringPatch(num_nops * 4u , nullptr, num_nops * 4u, kStringIndex),
372 LinkerPatch::RelativeStringPatch(num_nops * 4u + 8u, nullptr, num_nops * 4u, kStringIndex),
373 };
374 AddCompiledMethod(MethodRef(1u),
375 ArrayRef<const uint8_t>(code),
376 ArrayRef<const LinkerPatch>(patches));
377 Link();
378 }
379
TestNopsAdrpInsn2AndUse(size_t num_nops,uint32_t insn2,uint32_t target_offset,uint32_t use_insn)380 void TestNopsAdrpInsn2AndUse(size_t num_nops,
381 uint32_t insn2,
382 uint32_t target_offset,
383 uint32_t use_insn) {
384 uint32_t method1_offset = GetMethodOffset(1u);
385 auto expected_code = GenNopsAndAdrpAndUse(num_nops, method1_offset, target_offset, use_insn);
386 InsertInsn(&expected_code, num_nops * 4u + 4u, insn2);
387 EXPECT_TRUE(CheckLinkedMethod(MethodRef(1u), ArrayRef<const uint8_t>(expected_code)));
388 }
389
TestNopsAdrpInsn2AndUseHasThunk(size_t num_nops,uint32_t insn2,uint32_t target_offset,uint32_t use_insn)390 void TestNopsAdrpInsn2AndUseHasThunk(size_t num_nops,
391 uint32_t insn2,
392 uint32_t target_offset,
393 uint32_t use_insn) {
394 uint32_t method1_offset = GetMethodOffset(1u);
395 CHECK(!compiled_method_refs_.empty());
396 CHECK_EQ(compiled_method_refs_[0].index, 1u);
397 CHECK_EQ(compiled_method_refs_.size(), compiled_methods_.size());
398 uint32_t method1_size = compiled_methods_[0]->GetQuickCode().size();
399 uint32_t thunk_offset =
400 CompiledCode::AlignCode(method1_offset + method1_size, InstructionSet::kArm64);
401 uint32_t b_diff = thunk_offset - (method1_offset + num_nops * 4u);
402 CHECK_ALIGNED(b_diff, 4u);
403 ASSERT_LT(b_diff, 128 * MB);
404 uint32_t b_out = kBPlus0 + ((b_diff >> 2) & 0x03ffffffu);
405 uint32_t b_in = kBPlus0 + ((-b_diff >> 2) & 0x03ffffffu);
406
407 auto expected_code = GenNopsAndAdrpAndUse(num_nops, method1_offset, target_offset, use_insn);
408 InsertInsn(&expected_code, num_nops * 4u + 4u, insn2);
409 // Replace adrp with bl.
410 expected_code.erase(expected_code.begin() + num_nops * 4u,
411 expected_code.begin() + num_nops * 4u + 4u);
412 InsertInsn(&expected_code, num_nops * 4u, b_out);
413 EXPECT_TRUE(CheckLinkedMethod(MethodRef(1u), ArrayRef<const uint8_t>(expected_code)));
414
415 auto expected_thunk_code = GenNopsAndAdrpLdr(0u, thunk_offset, target_offset);
416 ASSERT_EQ(expected_thunk_code.size(), 8u);
417 expected_thunk_code.erase(expected_thunk_code.begin() + 4u, expected_thunk_code.begin() + 8u);
418 InsertInsn(&expected_thunk_code, 4u, b_in);
419 ASSERT_EQ(expected_thunk_code.size(), 8u);
420
421 uint32_t thunk_size = MethodCallThunkSize();
422 ASSERT_EQ(thunk_offset + thunk_size, output_.size());
423 ASSERT_EQ(thunk_size, expected_thunk_code.size());
424 ArrayRef<const uint8_t> thunk_code(&output_[thunk_offset], thunk_size);
425 if (ArrayRef<const uint8_t>(expected_thunk_code) != thunk_code) {
426 DumpDiff(ArrayRef<const uint8_t>(expected_thunk_code), thunk_code);
427 FAIL();
428 }
429 }
430
TestAdrpInsn2Ldr(uint32_t insn2,uint32_t adrp_offset,bool has_thunk,uint32_t bss_begin,uint32_t string_entry_offset)431 void TestAdrpInsn2Ldr(uint32_t insn2,
432 uint32_t adrp_offset,
433 bool has_thunk,
434 uint32_t bss_begin,
435 uint32_t string_entry_offset) {
436 uint32_t method1_offset =
437 kTrampolineSize + CodeAlignmentSize(kTrampolineSize) + sizeof(OatQuickMethodHeader);
438 ASSERT_LT(method1_offset, adrp_offset);
439 CHECK_ALIGNED(adrp_offset, 4u);
440 uint32_t num_nops = (adrp_offset - method1_offset) / 4u;
441 PrepareNopsAdrpInsn2Ldr(num_nops, insn2, bss_begin, string_entry_offset);
442 uint32_t target_offset = bss_begin_ + string_entry_offset;
443 if (has_thunk) {
444 TestNopsAdrpInsn2AndUseHasThunk(num_nops, insn2, target_offset, kLdrWInsn);
445 } else {
446 TestNopsAdrpInsn2AndUse(num_nops, insn2, target_offset, kLdrWInsn);
447 }
448 ASSERT_EQ(method1_offset, GetMethodOffset(1u)); // If this fails, num_nops is wrong.
449 }
450
TestAdrpLdurLdr(uint32_t adrp_offset,bool has_thunk,uint32_t bss_begin,uint32_t string_entry_offset)451 void TestAdrpLdurLdr(uint32_t adrp_offset,
452 bool has_thunk,
453 uint32_t bss_begin,
454 uint32_t string_entry_offset) {
455 TestAdrpInsn2Ldr(kLdurInsn, adrp_offset, has_thunk, bss_begin, string_entry_offset);
456 }
457
TestAdrpLdrPcRelLdr(uint32_t pcrel_ldr_insn,int32_t pcrel_disp,uint32_t adrp_offset,bool has_thunk,uint32_t bss_begin,uint32_t string_entry_offset)458 void TestAdrpLdrPcRelLdr(uint32_t pcrel_ldr_insn,
459 int32_t pcrel_disp,
460 uint32_t adrp_offset,
461 bool has_thunk,
462 uint32_t bss_begin,
463 uint32_t string_entry_offset) {
464 ASSERT_LT(pcrel_disp, 0x100000);
465 ASSERT_GE(pcrel_disp, -0x100000);
466 ASSERT_EQ(pcrel_disp & 0x3, 0);
467 uint32_t insn2 = pcrel_ldr_insn | (((static_cast<uint32_t>(pcrel_disp) >> 2) & 0x7ffffu) << 5);
468 TestAdrpInsn2Ldr(insn2, adrp_offset, has_thunk, bss_begin, string_entry_offset);
469 }
470
TestAdrpLdrSpRelLdr(uint32_t sprel_ldr_insn,uint32_t sprel_disp_in_load_units,uint32_t adrp_offset,bool has_thunk,uint32_t bss_begin,uint32_t string_entry_offset)471 void TestAdrpLdrSpRelLdr(uint32_t sprel_ldr_insn,
472 uint32_t sprel_disp_in_load_units,
473 uint32_t adrp_offset,
474 bool has_thunk,
475 uint32_t bss_begin,
476 uint32_t string_entry_offset) {
477 ASSERT_LT(sprel_disp_in_load_units, 0x1000u);
478 uint32_t insn2 = sprel_ldr_insn | ((sprel_disp_in_load_units & 0xfffu) << 10);
479 TestAdrpInsn2Ldr(insn2, adrp_offset, has_thunk, bss_begin, string_entry_offset);
480 }
481
TestAdrpInsn2Add(uint32_t insn2,uint32_t adrp_offset,bool has_thunk,uint32_t string_offset)482 void TestAdrpInsn2Add(uint32_t insn2,
483 uint32_t adrp_offset,
484 bool has_thunk,
485 uint32_t string_offset) {
486 uint32_t method1_offset =
487 kTrampolineSize + CodeAlignmentSize(kTrampolineSize) + sizeof(OatQuickMethodHeader);
488 ASSERT_LT(method1_offset, adrp_offset);
489 CHECK_ALIGNED(adrp_offset, 4u);
490 uint32_t num_nops = (adrp_offset - method1_offset) / 4u;
491 PrepareNopsAdrpInsn2Add(num_nops, insn2, string_offset);
492 if (has_thunk) {
493 TestNopsAdrpInsn2AndUseHasThunk(num_nops, insn2, string_offset, kAddXInsn);
494 } else {
495 TestNopsAdrpInsn2AndUse(num_nops, insn2, string_offset, kAddXInsn);
496 }
497 ASSERT_EQ(method1_offset, GetMethodOffset(1u)); // If this fails, num_nops is wrong.
498 }
499
TestAdrpLdurAdd(uint32_t adrp_offset,bool has_thunk,uint32_t string_offset)500 void TestAdrpLdurAdd(uint32_t adrp_offset, bool has_thunk, uint32_t string_offset) {
501 TestAdrpInsn2Add(kLdurInsn, adrp_offset, has_thunk, string_offset);
502 }
503
TestAdrpLdrPcRelAdd(uint32_t pcrel_ldr_insn,int32_t pcrel_disp,uint32_t adrp_offset,bool has_thunk,uint32_t string_offset)504 void TestAdrpLdrPcRelAdd(uint32_t pcrel_ldr_insn,
505 int32_t pcrel_disp,
506 uint32_t adrp_offset,
507 bool has_thunk,
508 uint32_t string_offset) {
509 ASSERT_LT(pcrel_disp, 0x100000);
510 ASSERT_GE(pcrel_disp, -0x100000);
511 ASSERT_EQ(pcrel_disp & 0x3, 0);
512 uint32_t insn2 = pcrel_ldr_insn | (((static_cast<uint32_t>(pcrel_disp) >> 2) & 0x7ffffu) << 5);
513 TestAdrpInsn2Add(insn2, adrp_offset, has_thunk, string_offset);
514 }
515
TestAdrpLdrSpRelAdd(uint32_t sprel_ldr_insn,uint32_t sprel_disp_in_load_units,uint32_t adrp_offset,bool has_thunk,uint32_t string_offset)516 void TestAdrpLdrSpRelAdd(uint32_t sprel_ldr_insn,
517 uint32_t sprel_disp_in_load_units,
518 uint32_t adrp_offset,
519 bool has_thunk,
520 uint32_t string_offset) {
521 ASSERT_LT(sprel_disp_in_load_units, 0x1000u);
522 uint32_t insn2 = sprel_ldr_insn | ((sprel_disp_in_load_units & 0xfffu) << 10);
523 TestAdrpInsn2Add(insn2, adrp_offset, has_thunk, string_offset);
524 }
525
EncodeBakerReadBarrierFieldData(uint32_t base_reg,uint32_t holder_reg)526 static uint32_t EncodeBakerReadBarrierFieldData(uint32_t base_reg, uint32_t holder_reg) {
527 return arm64::CodeGeneratorARM64::EncodeBakerReadBarrierFieldData(base_reg, holder_reg);
528 }
529
EncodeBakerReadBarrierArrayData(uint32_t base_reg)530 static uint32_t EncodeBakerReadBarrierArrayData(uint32_t base_reg) {
531 return arm64::CodeGeneratorARM64::EncodeBakerReadBarrierArrayData(base_reg);
532 }
533
EncodeBakerReadBarrierGcRootData(uint32_t root_reg)534 static uint32_t EncodeBakerReadBarrierGcRootData(uint32_t root_reg) {
535 return arm64::CodeGeneratorARM64::EncodeBakerReadBarrierGcRootData(root_reg);
536 }
537
CompileBakerOffsetThunk(uint32_t base_reg,uint32_t holder_reg)538 std::vector<uint8_t> CompileBakerOffsetThunk(uint32_t base_reg, uint32_t holder_reg) {
539 const LinkerPatch patch = LinkerPatch::BakerReadBarrierBranchPatch(
540 /* literal_offset */ 0u, EncodeBakerReadBarrierFieldData(base_reg, holder_reg));
541 return CompileThunk(patch);
542 }
543
CompileBakerArrayThunk(uint32_t base_reg)544 std::vector<uint8_t> CompileBakerArrayThunk(uint32_t base_reg) {
545 LinkerPatch patch = LinkerPatch::BakerReadBarrierBranchPatch(
546 /* literal_offset */ 0u, EncodeBakerReadBarrierArrayData(base_reg));
547 return CompileThunk(patch);
548 }
549
CompileBakerGcRootThunk(uint32_t root_reg)550 std::vector<uint8_t> CompileBakerGcRootThunk(uint32_t root_reg) {
551 LinkerPatch patch = LinkerPatch::BakerReadBarrierBranchPatch(
552 /* literal_offset */ 0u, EncodeBakerReadBarrierGcRootData(root_reg));
553 return CompileThunk(patch);
554 }
555
GetOutputInsn(uint32_t offset)556 uint32_t GetOutputInsn(uint32_t offset) {
557 CHECK_LE(offset, output_.size());
558 CHECK_GE(output_.size() - offset, 4u);
559 return (static_cast<uint32_t>(output_[offset]) << 0) |
560 (static_cast<uint32_t>(output_[offset + 1]) << 8) |
561 (static_cast<uint32_t>(output_[offset + 2]) << 16) |
562 (static_cast<uint32_t>(output_[offset + 3]) << 24);
563 }
564
565 void TestBakerField(uint32_t offset, uint32_t ref_reg, bool implicit_null_checks);
566
Reset()567 void Reset() final {
568 RelativePatcherTest::Reset();
569 implicit_null_checks_ = std::nullopt;
570 }
571
572 private:
573 std::optional<bool> implicit_null_checks_ = std::nullopt;
574 };
575
576 const uint8_t Arm64RelativePatcherTest::kCallRawCode[] = {
577 0x00, 0x00, 0x00, 0x94
578 };
579
580 const ArrayRef<const uint8_t> Arm64RelativePatcherTest::kCallCode(kCallRawCode);
581
582 const uint8_t Arm64RelativePatcherTest::kNopRawCode[] = {
583 0x1f, 0x20, 0x03, 0xd5
584 };
585
586 const ArrayRef<const uint8_t> Arm64RelativePatcherTest::kNopCode(kNopRawCode);
587
588 class Arm64RelativePatcherTestDefault : public Arm64RelativePatcherTest {
589 public:
Arm64RelativePatcherTestDefault()590 Arm64RelativePatcherTestDefault() : Arm64RelativePatcherTest("default") { }
591 };
592
TEST_F(Arm64RelativePatcherTestDefault,CallSelf)593 TEST_F(Arm64RelativePatcherTestDefault, CallSelf) {
594 const LinkerPatch patches[] = {
595 LinkerPatch::RelativeCodePatch(0u, nullptr, 1u),
596 };
597 AddCompiledMethod(MethodRef(1u), kCallCode, ArrayRef<const LinkerPatch>(patches));
598 Link();
599
600 const std::vector<uint8_t> expected_code = RawCode({kBlPlus0});
601 EXPECT_TRUE(CheckLinkedMethod(MethodRef(1u), ArrayRef<const uint8_t>(expected_code)));
602 }
603
TEST_F(Arm64RelativePatcherTestDefault,CallOther)604 TEST_F(Arm64RelativePatcherTestDefault, CallOther) {
605 const LinkerPatch method1_patches[] = {
606 LinkerPatch::RelativeCodePatch(0u, nullptr, 2u),
607 };
608 AddCompiledMethod(MethodRef(1u), kCallCode, ArrayRef<const LinkerPatch>(method1_patches));
609 const LinkerPatch method2_patches[] = {
610 LinkerPatch::RelativeCodePatch(0u, nullptr, 1u),
611 };
612 AddCompiledMethod(MethodRef(2u), kCallCode, ArrayRef<const LinkerPatch>(method2_patches));
613 Link();
614
615 uint32_t method1_offset = GetMethodOffset(1u);
616 uint32_t method2_offset = GetMethodOffset(2u);
617 uint32_t diff_after = method2_offset - method1_offset;
618 CHECK_ALIGNED(diff_after, 4u);
619 ASSERT_LT(diff_after >> 2, 1u << 8); // Simple encoding, (diff_after >> 2) fits into 8 bits.
620 const std::vector<uint8_t> method1_expected_code = RawCode({kBlPlus0 + (diff_after >> 2)});
621 EXPECT_TRUE(CheckLinkedMethod(MethodRef(1u), ArrayRef<const uint8_t>(method1_expected_code)));
622 uint32_t diff_before = method1_offset - method2_offset;
623 CHECK_ALIGNED(diff_before, 4u);
624 ASSERT_GE(diff_before, -1u << 27);
625 auto method2_expected_code = GenNopsAndBl(0u, kBlPlus0 | ((diff_before >> 2) & 0x03ffffffu));
626 EXPECT_TRUE(CheckLinkedMethod(MethodRef(2u), ArrayRef<const uint8_t>(method2_expected_code)));
627 }
628
TEST_F(Arm64RelativePatcherTestDefault,CallTrampoline)629 TEST_F(Arm64RelativePatcherTestDefault, CallTrampoline) {
630 const LinkerPatch patches[] = {
631 LinkerPatch::RelativeCodePatch(0u, nullptr, 2u),
632 };
633 AddCompiledMethod(MethodRef(1u), kCallCode, ArrayRef<const LinkerPatch>(patches));
634 Link();
635
636 uint32_t method1_offset = GetMethodOffset(1u);
637 uint32_t diff = kTrampolineOffset - method1_offset;
638 ASSERT_EQ(diff & 1u, 0u);
639 ASSERT_GE(diff, -1u << 9); // Simple encoding, -256 <= (diff >> 1) < 0 (checked as unsigned).
640 auto expected_code = GenNopsAndBl(0u, kBlPlus0 | ((diff >> 2) & 0x03ffffffu));
641 EXPECT_TRUE(CheckLinkedMethod(MethodRef(1u), ArrayRef<const uint8_t>(expected_code)));
642 }
643
TEST_F(Arm64RelativePatcherTestDefault,CallTrampolineTooFar)644 TEST_F(Arm64RelativePatcherTestDefault, CallTrampolineTooFar) {
645 constexpr uint32_t missing_method_index = 1024u;
646 auto last_method_raw_code = GenNopsAndBl(1u, kBlPlus0);
647 constexpr uint32_t bl_offset_in_last_method = 1u * 4u; // After NOPs.
648 ArrayRef<const uint8_t> last_method_code(last_method_raw_code);
649 ASSERT_EQ(bl_offset_in_last_method + 4u, last_method_code.size());
650 const LinkerPatch last_method_patches[] = {
651 LinkerPatch::RelativeCodePatch(bl_offset_in_last_method, nullptr, missing_method_index),
652 };
653
654 constexpr uint32_t just_over_max_negative_disp = 128 * MB + 4;
655 uint32_t last_method_idx = Create2MethodsWithGap(
656 kNopCode,
657 ArrayRef<const LinkerPatch>(),
658 last_method_code,
659 ArrayRef<const LinkerPatch>(last_method_patches),
660 just_over_max_negative_disp - bl_offset_in_last_method);
661 uint32_t method1_offset = GetMethodOffset(1u);
662 uint32_t last_method_offset = GetMethodOffset(last_method_idx);
663 ASSERT_EQ(method1_offset,
664 last_method_offset + bl_offset_in_last_method - just_over_max_negative_disp);
665 ASSERT_FALSE(method_offset_map_.FindMethodOffset(MethodRef(missing_method_index)).first);
666
667 // Check linked code.
668 uint32_t thunk_offset =
669 CompiledCode::AlignCode(last_method_offset + last_method_code.size(), InstructionSet::kArm64);
670 uint32_t diff = thunk_offset - (last_method_offset + bl_offset_in_last_method);
671 ASSERT_TRUE(IsAligned<4u>(diff));
672 ASSERT_LT(diff, 128 * MB);
673 auto expected_code = GenNopsAndBl(1u, kBlPlus0 | (diff >> 2));
674 EXPECT_TRUE(CheckLinkedMethod(MethodRef(last_method_idx),
675 ArrayRef<const uint8_t>(expected_code)));
676 EXPECT_TRUE(CheckThunk(thunk_offset));
677 }
678
TEST_F(Arm64RelativePatcherTestDefault,CallOtherAlmostTooFarAfter)679 TEST_F(Arm64RelativePatcherTestDefault, CallOtherAlmostTooFarAfter) {
680 auto method1_raw_code = GenNopsAndBl(1u, kBlPlus0);
681 constexpr uint32_t bl_offset_in_method1 = 1u * 4u; // After NOPs.
682 ArrayRef<const uint8_t> method1_code(method1_raw_code);
683 ASSERT_EQ(bl_offset_in_method1 + 4u, method1_code.size());
684 const uint32_t kExpectedLastMethodIdx = 65u; // Based on 2MiB chunks in Create2MethodsWithGap().
685 const LinkerPatch method1_patches[] = {
686 LinkerPatch::RelativeCodePatch(bl_offset_in_method1, nullptr, kExpectedLastMethodIdx),
687 };
688
689 constexpr uint32_t max_positive_disp = 128 * MB - 4u;
690 uint32_t last_method_idx = Create2MethodsWithGap(method1_code,
691 ArrayRef<const LinkerPatch>(method1_patches),
692 kNopCode,
693 ArrayRef<const LinkerPatch>(),
694 bl_offset_in_method1 + max_positive_disp);
695 ASSERT_EQ(kExpectedLastMethodIdx, last_method_idx);
696
697 uint32_t method1_offset = GetMethodOffset(1u);
698 uint32_t last_method_offset = GetMethodOffset(last_method_idx);
699 ASSERT_EQ(method1_offset + bl_offset_in_method1 + max_positive_disp, last_method_offset);
700
701 // Check linked code.
702 auto expected_code = GenNopsAndBl(1u, kBlPlusMax);
703 EXPECT_TRUE(CheckLinkedMethod(MethodRef(1u), ArrayRef<const uint8_t>(expected_code)));
704 }
705
TEST_F(Arm64RelativePatcherTestDefault,CallOtherAlmostTooFarBefore)706 TEST_F(Arm64RelativePatcherTestDefault, CallOtherAlmostTooFarBefore) {
707 auto last_method_raw_code = GenNopsAndBl(0u, kBlPlus0);
708 constexpr uint32_t bl_offset_in_last_method = 0u * 4u; // After NOPs.
709 ArrayRef<const uint8_t> last_method_code(last_method_raw_code);
710 ASSERT_EQ(bl_offset_in_last_method + 4u, last_method_code.size());
711 const LinkerPatch last_method_patches[] = {
712 LinkerPatch::RelativeCodePatch(bl_offset_in_last_method, nullptr, 1u),
713 };
714
715 constexpr uint32_t max_negative_disp = 128 * MB;
716 uint32_t last_method_idx = Create2MethodsWithGap(kNopCode,
717 ArrayRef<const LinkerPatch>(),
718 last_method_code,
719 ArrayRef<const LinkerPatch>(last_method_patches),
720 max_negative_disp - bl_offset_in_last_method);
721 uint32_t method1_offset = GetMethodOffset(1u);
722 uint32_t last_method_offset = GetMethodOffset(last_method_idx);
723 ASSERT_EQ(method1_offset, last_method_offset + bl_offset_in_last_method - max_negative_disp);
724
725 // Check linked code.
726 auto expected_code = GenNopsAndBl(0u, kBlMinusMax);
727 EXPECT_TRUE(CheckLinkedMethod(MethodRef(last_method_idx),
728 ArrayRef<const uint8_t>(expected_code)));
729 }
730
TEST_F(Arm64RelativePatcherTestDefault,CallOtherJustTooFarAfter)731 TEST_F(Arm64RelativePatcherTestDefault, CallOtherJustTooFarAfter) {
732 auto method1_raw_code = GenNopsAndBl(0u, kBlPlus0);
733 constexpr uint32_t bl_offset_in_method1 = 0u * 4u; // After NOPs.
734 ArrayRef<const uint8_t> method1_code(method1_raw_code);
735 ASSERT_EQ(bl_offset_in_method1 + 4u, method1_code.size());
736 const uint32_t kExpectedLastMethodIdx = 65u; // Based on 2MiB chunks in Create2MethodsWithGap().
737 const LinkerPatch method1_patches[] = {
738 LinkerPatch::RelativeCodePatch(bl_offset_in_method1, nullptr, kExpectedLastMethodIdx),
739 };
740
741 constexpr uint32_t just_over_max_positive_disp = 128 * MB;
742 uint32_t last_method_idx = Create2MethodsWithGap(
743 method1_code,
744 ArrayRef<const LinkerPatch>(method1_patches),
745 kNopCode,
746 ArrayRef<const LinkerPatch>(),
747 bl_offset_in_method1 + just_over_max_positive_disp);
748 ASSERT_EQ(kExpectedLastMethodIdx, last_method_idx);
749 uint32_t method_after_thunk_idx = last_method_idx;
750 if (sizeof(OatQuickMethodHeader) < kArm64CodeAlignment) {
751 // The thunk needs to start on a kArm64CodeAlignment-aligned address before the address where
752 // the last method would have been if there was no thunk. If the size of the
753 // OatQuickMethodHeader is at least kArm64CodeAlignment, the thunk start shall fit between the
754 // previous filler method and that address. Otherwise, it shall be inserted before that filler
755 // method.
756 method_after_thunk_idx -= 1u;
757 }
758
759 uint32_t method1_offset = GetMethodOffset(1u);
760 uint32_t method_after_thunk_offset = GetMethodOffset(method_after_thunk_idx);
761 ASSERT_TRUE(IsAligned<kArm64CodeAlignment>(method_after_thunk_offset));
762 uint32_t method_after_thunk_header_offset =
763 method_after_thunk_offset - sizeof(OatQuickMethodHeader);
764 uint32_t thunk_size = MethodCallThunkSize();
765 uint32_t thunk_offset = RoundDown(
766 method_after_thunk_header_offset - thunk_size, kArm64CodeAlignment);
767 DCHECK_EQ(thunk_offset + thunk_size + CodeAlignmentSize(thunk_offset + thunk_size),
768 method_after_thunk_header_offset);
769 ASSERT_TRUE(IsAligned<kArm64CodeAlignment>(thunk_offset));
770 uint32_t diff = thunk_offset - (method1_offset + bl_offset_in_method1);
771 ASSERT_TRUE(IsAligned<4u>(diff));
772 ASSERT_LT(diff, 128 * MB);
773 auto expected_code = GenNopsAndBl(0u, kBlPlus0 | (diff >> 2));
774 EXPECT_TRUE(CheckLinkedMethod(MethodRef(1u), ArrayRef<const uint8_t>(expected_code)));
775 CheckThunk(thunk_offset);
776 }
777
TEST_F(Arm64RelativePatcherTestDefault,CallOtherJustTooFarBefore)778 TEST_F(Arm64RelativePatcherTestDefault, CallOtherJustTooFarBefore) {
779 auto last_method_raw_code = GenNopsAndBl(1u, kBlPlus0);
780 constexpr uint32_t bl_offset_in_last_method = 1u * 4u; // After NOPs.
781 ArrayRef<const uint8_t> last_method_code(last_method_raw_code);
782 ASSERT_EQ(bl_offset_in_last_method + 4u, last_method_code.size());
783 const LinkerPatch last_method_patches[] = {
784 LinkerPatch::RelativeCodePatch(bl_offset_in_last_method, nullptr, 1u),
785 };
786
787 constexpr uint32_t just_over_max_negative_disp = 128 * MB + 4;
788 uint32_t last_method_idx = Create2MethodsWithGap(
789 kNopCode, ArrayRef<const LinkerPatch>(), last_method_code,
790 ArrayRef<const LinkerPatch>(last_method_patches),
791 just_over_max_negative_disp - bl_offset_in_last_method);
792 uint32_t method1_offset = GetMethodOffset(1u);
793 uint32_t last_method_offset = GetMethodOffset(last_method_idx);
794 ASSERT_EQ(method1_offset,
795 last_method_offset + bl_offset_in_last_method - just_over_max_negative_disp);
796
797 // Check linked code.
798 uint32_t thunk_offset =
799 CompiledCode::AlignCode(last_method_offset + last_method_code.size(), InstructionSet::kArm64);
800 uint32_t diff = thunk_offset - (last_method_offset + bl_offset_in_last_method);
801 ASSERT_TRUE(IsAligned<4u>(diff));
802 ASSERT_LT(diff, 128 * MB);
803 auto expected_code = GenNopsAndBl(1u, kBlPlus0 | (diff >> 2));
804 EXPECT_TRUE(CheckLinkedMethod(MethodRef(last_method_idx),
805 ArrayRef<const uint8_t>(expected_code)));
806 EXPECT_TRUE(CheckThunk(thunk_offset));
807 }
808
TEST_F(Arm64RelativePatcherTestDefault,StringBssEntry)809 TEST_F(Arm64RelativePatcherTestDefault, StringBssEntry) {
810 struct TestCase {
811 uint32_t bss_begin;
812 uint32_t string_entry_offset;
813 };
814 static const TestCase test_cases[] = {
815 { 0x12345678u, 0x1234u },
816 { -0x12345678u, 0x4444u },
817 { 0x12345000u, 0x3ffcu },
818 { 0x12345000u, 0x4000u }
819 };
820 for (const TestCase& test_case : test_cases) {
821 Reset();
822 TestNopsAdrpLdr(/*num_nops=*/ 0u, test_case.bss_begin, test_case.string_entry_offset);
823 }
824 }
825
TEST_F(Arm64RelativePatcherTestDefault,StringReference)826 TEST_F(Arm64RelativePatcherTestDefault, StringReference) {
827 for (uint32_t string_offset : { 0x12345678u, -0x12345678u, 0x12345000u, 0x12345ffcu}) {
828 Reset();
829 TestNopsAdrpAdd(/*num_nops=*/ 0u, string_offset);
830 }
831 }
832
833 template <typename Test>
TestForAdrpOffsets(Test test,std::initializer_list<uint32_t> args)834 void TestForAdrpOffsets(Test test, std::initializer_list<uint32_t> args) {
835 for (uint32_t adrp_offset : { 0xff4u, 0xff8u, 0xffcu, 0x1000u }) {
836 for (uint32_t arg : args) {
837 test(adrp_offset, arg);
838 }
839 }
840 }
841
TEST_F(Arm64RelativePatcherTestDefault,StringBssEntryLdur)842 TEST_F(Arm64RelativePatcherTestDefault, StringBssEntryLdur) {
843 TestForAdrpOffsets(
844 [&](uint32_t adrp_offset, uint32_t string_entry_offset) {
845 Reset();
846 bool has_thunk = ((adrp_offset) == 0xff8u || (adrp_offset) == 0xffcu);
847 TestAdrpLdurLdr(adrp_offset, has_thunk, /*bss_begin=*/ 0x12345678u, string_entry_offset);
848 },
849 { 0x1234u, 0x1238u });
850 }
851
852 // LDR <Wt>, <label> is always aligned. We should never have to use a fixup.
TEST_F(Arm64RelativePatcherTestDefault,StringBssEntryWPcRel)853 TEST_F(Arm64RelativePatcherTestDefault, StringBssEntryWPcRel) {
854 TestForAdrpOffsets(
855 [&](uint32_t adrp_offset, uint32_t pcrel_disp) {
856 Reset();
857 TestAdrpLdrPcRelLdr(kLdrWPcRelInsn,
858 pcrel_disp,
859 adrp_offset,
860 /*has_thunk=*/ false,
861 /*bss_begin=*/ 0x12345678u,
862 /*string_entry_offset=*/ 0x1234u);
863 },
864 { 0x1234u, 0x1238u });
865 }
866
867 // LDR <Xt>, <label> is aligned when offset + displacement is a multiple of 8.
TEST_F(Arm64RelativePatcherTestDefault,StringBssEntryXPcRel)868 TEST_F(Arm64RelativePatcherTestDefault, StringBssEntryXPcRel) {
869 TestForAdrpOffsets(
870 [&](uint32_t adrp_offset, uint32_t pcrel_disp) {
871 Reset();
872 bool unaligned = !IsAligned<8u>((adrp_offset) + 4u + static_cast<uint32_t>(pcrel_disp));
873 bool has_thunk = ((adrp_offset) == 0xff8u || (adrp_offset) == 0xffcu) && unaligned;
874 TestAdrpLdrPcRelLdr(kLdrXPcRelInsn,
875 pcrel_disp,
876 adrp_offset,
877 has_thunk,
878 /*bss_begin=*/ 0x12345678u,
879 /*string_entry_offset=*/ 0x1234u);
880 },
881 { 0x1234u, 0x1238u });
882 }
883
884 // LDR <Wt>, [SP, #<pimm>] and LDR <Xt>, [SP, #<pimm>] are always aligned. No fixup needed.
TEST_F(Arm64RelativePatcherTestDefault,StringBssEntryWSpRel)885 TEST_F(Arm64RelativePatcherTestDefault, StringBssEntryWSpRel) {
886 TestForAdrpOffsets(
887 [&](uint32_t adrp_offset, uint32_t disp) {
888 Reset();
889 TestAdrpLdrSpRelLdr(kLdrWSpRelInsn,
890 /*sprel_disp_in_load_units=*/ disp >> 2,
891 adrp_offset,
892 /*has_thunk=*/ false,
893 /*bss_begin=*/ 0x12345678u,
894 /*string_entry_offset=*/ 0x1234u);
895 },
896 { 0u, 4u });
897 }
898
TEST_F(Arm64RelativePatcherTestDefault,StringBssEntryXSpRel)899 TEST_F(Arm64RelativePatcherTestDefault, StringBssEntryXSpRel) {
900 TestForAdrpOffsets(
901 [&](uint32_t adrp_offset, uint32_t disp) {
902 Reset();
903 TestAdrpLdrSpRelLdr(kLdrXSpRelInsn,
904 /*sprel_disp_in_load_units=*/ (disp) >> 3,
905 adrp_offset,
906 /*has_thunk=*/ false,
907 /*bss_begin=*/ 0x12345678u,
908 /*string_entry_offset=*/ 0x1234u);
909 },
910 { 0u, 8u });
911 }
912
TEST_F(Arm64RelativePatcherTestDefault,StringReferenceLdur)913 TEST_F(Arm64RelativePatcherTestDefault, StringReferenceLdur) {
914 TestForAdrpOffsets(
915 [&](uint32_t adrp_offset, uint32_t string_offset) {
916 Reset();
917 bool has_thunk = ((adrp_offset) == 0xff8u || (adrp_offset) == 0xffcu);
918 TestAdrpLdurAdd(adrp_offset, has_thunk, string_offset);
919 },
920 { 0x12345678u, 0xffffc840u });
921 }
922
TEST_F(Arm64RelativePatcherTestDefault,StringReferenceSubX3X2)923 TEST_F(Arm64RelativePatcherTestDefault, StringReferenceSubX3X2) {
924 TestForAdrpOffsets(
925 [&](uint32_t adrp_offset, uint32_t string_offset) {
926 Reset();
927 /* SUB unrelated to "ADRP x0, addr". */ \
928 uint32_t sub = kSubXInsn | (100 << 10) | (2u << 5) | 3u; /* SUB x3, x2, #100 */
929 TestAdrpInsn2Add(sub, adrp_offset, /*has_thunk=*/ false, string_offset);
930 },
931 { 0x12345678u, 0xffffc840u });
932 }
933
TEST_F(Arm64RelativePatcherTestDefault,StringReferenceSubsX3X0)934 TEST_F(Arm64RelativePatcherTestDefault, StringReferenceSubsX3X0) {
935 TestForAdrpOffsets(
936 [&](uint32_t adrp_offset, uint32_t string_offset) {
937 Reset();
938 /* SUBS that uses the result of "ADRP x0, addr". */ \
939 uint32_t subs = kSubsXInsn | (100 << 10) | (0u << 5) | 3u; /* SUBS x3, x0, #100 */
940 TestAdrpInsn2Add(subs, adrp_offset, /*has_thunk=*/ false, string_offset);
941 },
942 { 0x12345678u, 0xffffc840u });
943 }
944
TEST_F(Arm64RelativePatcherTestDefault,StringReferenceAddX0X0)945 TEST_F(Arm64RelativePatcherTestDefault, StringReferenceAddX0X0) {
946 TestForAdrpOffsets(
947 [&](uint32_t adrp_offset, uint32_t string_offset) {
948 Reset();
949 /* ADD that uses the result register of "ADRP x0, addr" as both source and destination. */
950 uint32_t add = kAddXInsn | (100 << 10) | (0u << 5) | 0u; /* ADD x0, x0, #100 */
951 TestAdrpInsn2Add(add, adrp_offset, /*has_thunk=*/ false, string_offset);
952 },
953 { 0x12345678u, 0xffffc840 });
954 }
955
TEST_F(Arm64RelativePatcherTestDefault,StringReferenceAddsX0X2)956 TEST_F(Arm64RelativePatcherTestDefault, StringReferenceAddsX0X2) {
957 TestForAdrpOffsets(
958 [&](uint32_t adrp_offset, uint32_t string_offset) {
959 Reset();
960 /* ADDS that does not use the result of "ADRP x0, addr" but overwrites that register. */
961 uint32_t adds = kAddsXInsn | (100 << 10) | (2u << 5) | 0u; /* ADDS x0, x2, #100 */
962 bool has_thunk = ((adrp_offset) == 0xff8u || (adrp_offset) == 0xffcu);
963 TestAdrpInsn2Add(adds, adrp_offset, has_thunk, string_offset);
964 },
965 { 0x12345678u, 0xffffc840u });
966 }
967
968 // LDR <Wt>, <label> is always aligned. We should never have to use a fixup.
TEST_F(Arm64RelativePatcherTestDefault,StringReferenceWPcRel)969 TEST_F(Arm64RelativePatcherTestDefault, StringReferenceWPcRel) {
970 TestForAdrpOffsets(
971 [&](uint32_t adrp_offset, uint32_t pcrel_disp) {
972 Reset();
973 TestAdrpLdrPcRelAdd(kLdrWPcRelInsn,
974 pcrel_disp,
975 adrp_offset,
976 /*has_thunk=*/ false,
977 /*string_offset=*/ 0x12345678u);
978 },
979 { 0x1234u, 0x1238u });
980 }
981
982 // LDR <Xt>, <label> is aligned when offset + displacement is a multiple of 8.
TEST_F(Arm64RelativePatcherTestDefault,StringReferenceXPcRel)983 TEST_F(Arm64RelativePatcherTestDefault, StringReferenceXPcRel) {
984 TestForAdrpOffsets(
985 [&](uint32_t adrp_offset, uint32_t pcrel_disp) {
986 Reset();
987 bool unaligned = !IsAligned<8u>((adrp_offset) + 4u + static_cast<uint32_t>(pcrel_disp));
988 bool has_thunk = ((adrp_offset) == 0xff8u || (adrp_offset) == 0xffcu) && unaligned;
989 TestAdrpLdrPcRelAdd(kLdrXPcRelInsn,
990 pcrel_disp,
991 adrp_offset,
992 has_thunk,
993 /*string_offset=*/ 0x12345678u);
994 },
995 { 0x1234u, 0x1238u });
996 }
997
998 // LDR <Wt>, [SP, #<pimm>] and LDR <Xt>, [SP, #<pimm>] are always aligned. No fixup needed.
TEST_F(Arm64RelativePatcherTestDefault,StringReferenceWSpRel)999 TEST_F(Arm64RelativePatcherTestDefault, StringReferenceWSpRel) {
1000 TestForAdrpOffsets(
1001 [&](uint32_t adrp_offset, uint32_t disp) {
1002 Reset();
1003 TestAdrpLdrSpRelAdd(kLdrWSpRelInsn,
1004 /*sprel_disp_in_load_units=*/ (disp) >> 2,
1005 adrp_offset,
1006 /*has_thunk=*/ false,
1007 /*string_offset=*/ 0x12345678u);
1008 },
1009 { 0u, 4u });
1010 }
1011
TEST_F(Arm64RelativePatcherTestDefault,StringReferenceXSpRel)1012 TEST_F(Arm64RelativePatcherTestDefault, StringReferenceXSpRel) {
1013 TestForAdrpOffsets(
1014 [&](uint32_t adrp_offset, uint32_t disp) {
1015 Reset();
1016 TestAdrpLdrSpRelAdd(kLdrXSpRelInsn,
1017 /*sprel_disp_in_load_units=*/ (disp) >> 3,
1018 adrp_offset,
1019 /*has_thunk=*/ false,
1020 /*string_offset=*/ 0x12345678u);
1021 },
1022 { 0u, 8u });
1023 }
1024
TEST_F(Arm64RelativePatcherTestDefault,EntrypointCall)1025 TEST_F(Arm64RelativePatcherTestDefault, EntrypointCall) {
1026 constexpr uint32_t kEntrypointOffset = 512;
1027 const LinkerPatch patches[] = {
1028 LinkerPatch::CallEntrypointPatch(0u, kEntrypointOffset),
1029 };
1030 AddCompiledMethod(MethodRef(1u), kCallCode, ArrayRef<const LinkerPatch>(patches));
1031 Link();
1032
1033 uint32_t method_offset = GetMethodOffset(1u);
1034 uint32_t thunk_offset = CompiledCode::AlignCode(method_offset + kCallCode.size(),
1035 InstructionSet::kArm64);
1036 uint32_t diff = thunk_offset - method_offset;
1037 ASSERT_TRUE(IsAligned<4u>(diff));
1038 ASSERT_LT(diff, 128 * MB);
1039 auto expected_code = RawCode({kBlPlus0 | (diff >> 2)});
1040 EXPECT_TRUE(CheckLinkedMethod(MethodRef(1u), ArrayRef<const uint8_t>(expected_code)));
1041
1042 // Verify the thunk.
1043 uint32_t ldr_ip0_tr_offset =
1044 0xf9400000 | // LDR Xt, [Xn, #<simm>]
1045 ((kEntrypointOffset >> 3) << 10) | // imm12 = (simm >> scale), scale = 3
1046 (/* tr */ 19 << 5) | // Xn = TR
1047 /* ip0 */ 16; // Xt = ip0
1048 uint32_t br_ip0 = 0xd61f0000 | (/* ip0 */ 16 << 5);
1049 auto expected_thunk = RawCode({ ldr_ip0_tr_offset, br_ip0 });
1050 ASSERT_LE(8u, output_.size() - thunk_offset);
1051 EXPECT_EQ(ldr_ip0_tr_offset, GetOutputInsn(thunk_offset));
1052 EXPECT_EQ(br_ip0, GetOutputInsn(thunk_offset + 4u));
1053 }
1054
TestBakerField(uint32_t offset,uint32_t ref_reg,bool implicit_null_checks)1055 void Arm64RelativePatcherTest::TestBakerField(uint32_t offset,
1056 uint32_t ref_reg,
1057 bool implicit_null_checks) {
1058 implicit_null_checks_ = implicit_null_checks;
1059 uint32_t valid_regs[] = {
1060 0, 1, 2, 3, 4, 5, 6, 7, 8, 9,
1061 10, 11, 12, 13, 14, 15, 18, 19, // IP0 and IP1 are reserved.
1062 20, 21, 22, 23, 24, 25, 26, 27, 28, 29,
1063 // LR and SP/ZR are reserved.
1064 };
1065 DCHECK_ALIGNED(offset, 4u);
1066 DCHECK_LT(offset, 16 * KB);
1067 constexpr size_t kMethodCodeSize = 8u;
1068 constexpr size_t kLiteralOffset = 0u;
1069 uint32_t method_idx = 0u;
1070 for (uint32_t base_reg : valid_regs) {
1071 for (uint32_t holder_reg : valid_regs) {
1072 uint32_t ldr = kLdrWInsn | (offset << (10 - 2)) | (base_reg << 5) | ref_reg;
1073 const std::vector<uint8_t> raw_code = RawCode({kCbnzIP1Plus0Insn, ldr});
1074 ASSERT_EQ(kMethodCodeSize, raw_code.size());
1075 ArrayRef<const uint8_t> code(raw_code);
1076 uint32_t encoded_data = EncodeBakerReadBarrierFieldData(base_reg, holder_reg);
1077 const LinkerPatch patches[] = {
1078 LinkerPatch::BakerReadBarrierBranchPatch(kLiteralOffset, encoded_data),
1079 };
1080 ++method_idx;
1081 AddCompiledMethod(MethodRef(method_idx), code, ArrayRef<const LinkerPatch>(patches));
1082 }
1083 }
1084 Link();
1085
1086 // All thunks are at the end.
1087 uint32_t thunk_offset =
1088 GetMethodOffset(method_idx) + RoundUp(kMethodCodeSize, kArm64CodeAlignment);
1089 method_idx = 0u;
1090 for (uint32_t base_reg : valid_regs) {
1091 for (uint32_t holder_reg : valid_regs) {
1092 ++method_idx;
1093 uint32_t cbnz_offset = thunk_offset - (GetMethodOffset(method_idx) + kLiteralOffset);
1094 uint32_t cbnz = kCbnzIP1Plus0Insn | (cbnz_offset << (5 - 2));
1095 uint32_t ldr = kLdrWInsn | (offset << (10 - 2)) | (base_reg << 5) | ref_reg;
1096 const std::vector<uint8_t> expected_code = RawCode({cbnz, ldr});
1097 ASSERT_EQ(kMethodCodeSize, expected_code.size());
1098 ASSERT_TRUE(
1099 CheckLinkedMethod(MethodRef(method_idx), ArrayRef<const uint8_t>(expected_code)));
1100
1101 std::vector<uint8_t> expected_thunk = CompileBakerOffsetThunk(base_reg, holder_reg);
1102 ASSERT_GT(output_.size(), thunk_offset);
1103 ASSERT_GE(output_.size() - thunk_offset, expected_thunk.size());
1104 ArrayRef<const uint8_t> compiled_thunk(output_.data() + thunk_offset,
1105 expected_thunk.size());
1106 if (ArrayRef<const uint8_t>(expected_thunk) != compiled_thunk) {
1107 DumpDiff(ArrayRef<const uint8_t>(expected_thunk), compiled_thunk);
1108 ASSERT_TRUE(false);
1109 }
1110
1111 size_t gray_check_offset = thunk_offset;
1112 if (implicit_null_checks && holder_reg == base_reg) {
1113 // Verify that the null-check CBZ uses the correct register, i.e. holder_reg.
1114 ASSERT_GE(output_.size() - gray_check_offset, 4u);
1115 ASSERT_EQ(0x34000000u | holder_reg, GetOutputInsn(thunk_offset) & 0xff00001fu);
1116 gray_check_offset +=4u;
1117 }
1118 // Verify that the lock word for gray bit check is loaded from the holder address.
1119 static constexpr size_t kGrayCheckInsns = 5;
1120 ASSERT_GE(output_.size() - gray_check_offset, 4u * kGrayCheckInsns);
1121 const uint32_t load_lock_word =
1122 kLdrWInsn |
1123 (mirror::Object::MonitorOffset().Uint32Value() << (10 - 2)) |
1124 (holder_reg << 5) |
1125 /* ip0 */ 16;
1126 EXPECT_EQ(load_lock_word, GetOutputInsn(gray_check_offset));
1127 // Verify the gray bit check.
1128 const uint32_t check_gray_bit_without_offset =
1129 0x37000000u | (LockWord::kReadBarrierStateShift << 19) | /* ip0 */ 16;
1130 EXPECT_EQ(check_gray_bit_without_offset, GetOutputInsn(gray_check_offset + 4u) & 0xfff8001fu);
1131 // Verify the fake dependency.
1132 const uint32_t fake_dependency =
1133 0x8b408000u | // ADD Xd, Xn, Xm, LSR 32
1134 (/* ip0 */ 16 << 16) | // Xm = ip0
1135 (base_reg << 5) | // Xn = base_reg
1136 base_reg; // Xd = base_reg
1137 EXPECT_EQ(fake_dependency, GetOutputInsn(gray_check_offset + 12u));
1138 // Do not check the rest of the implementation.
1139
1140 // The next thunk follows on the next aligned offset.
1141 thunk_offset += RoundUp(expected_thunk.size(), kArm64CodeAlignment);
1142 }
1143 }
1144 }
1145
TEST_F(Arm64RelativePatcherTestDefault,BakerOffset)1146 TEST_F(Arm64RelativePatcherTestDefault, BakerOffset) {
1147 struct TestCase {
1148 uint32_t offset;
1149 uint32_t ref_reg;
1150 };
1151 static const TestCase test_cases[] = {
1152 { 0u, 0u },
1153 { 8u, 15u},
1154 { 0x3ffcu, 29u },
1155 };
1156 for (const TestCase& test_case : test_cases) {
1157 Reset();
1158 TestBakerField(test_case.offset, test_case.ref_reg, /*implicit_null_checks=*/ true);
1159 Reset();
1160 TestBakerField(test_case.offset, test_case.ref_reg, /*implicit_null_checks=*/ false);
1161 }
1162 }
1163
1164
TEST_F(Arm64RelativePatcherTestDefault,BakerOffsetThunkInTheMiddle)1165 TEST_F(Arm64RelativePatcherTestDefault, BakerOffsetThunkInTheMiddle) {
1166 // One thunk in the middle with maximum distance branches to it from both sides.
1167 // Use offset = 0, base_reg = 0, ref_reg = 0, the LDR is simply `kLdrWInsn`.
1168 constexpr uint32_t kLiteralOffset1 = 4;
1169 const std::vector<uint8_t> raw_code1 = RawCode({kNopInsn, kCbnzIP1Plus0Insn, kLdrWInsn});
1170 ArrayRef<const uint8_t> code1(raw_code1);
1171 uint32_t encoded_data = EncodeBakerReadBarrierFieldData(/* base_reg */ 0, /* holder_reg */ 0);
1172 const LinkerPatch patches1[] = {
1173 LinkerPatch::BakerReadBarrierBranchPatch(kLiteralOffset1, encoded_data),
1174 };
1175 AddCompiledMethod(MethodRef(1u), code1, ArrayRef<const LinkerPatch>(patches1));
1176
1177 // Allow thunk at 1MiB offset from the start of the method above. Literal offset being 4
1178 // allows the branch to reach that thunk.
1179 size_t filler1_size =
1180 1 * MB - RoundUp(raw_code1.size() + sizeof(OatQuickMethodHeader), kArm64CodeAlignment);
1181 std::vector<uint8_t> raw_filler1_code = GenNops(filler1_size / 4u);
1182 ArrayRef<const uint8_t> filler1_code(raw_filler1_code);
1183 AddCompiledMethod(MethodRef(2u), filler1_code);
1184
1185 // Enforce thunk reservation with a tiny method.
1186 AddCompiledMethod(MethodRef(3u), kNopCode);
1187
1188 // Allow reaching the thunk from the very beginning of a method 1MiB away. Backward branch
1189 // reaches the full 1MiB. Things to subtract:
1190 // - thunk size and method 3 pre-header, rounded up (padding in between if needed)
1191 // - method 3 code and method 4 pre-header, rounded up (padding in between if needed)
1192 // - method 4 header (let there be no padding between method 4 code and method 5 pre-header).
1193 size_t thunk_size = CompileBakerOffsetThunk(/* base_reg */ 0, /* holder_reg */ 0).size();
1194 size_t filler2_size =
1195 1 * MB - RoundUp(thunk_size + sizeof(OatQuickMethodHeader), kArm64CodeAlignment)
1196 - RoundUp(kNopCode.size() + sizeof(OatQuickMethodHeader), kArm64CodeAlignment)
1197 - sizeof(OatQuickMethodHeader);
1198 std::vector<uint8_t> raw_filler2_code = GenNops(filler2_size / 4u);
1199 ArrayRef<const uint8_t> filler2_code(raw_filler2_code);
1200 AddCompiledMethod(MethodRef(4u), filler2_code);
1201
1202 constexpr uint32_t kLiteralOffset2 = 0;
1203 const std::vector<uint8_t> raw_code2 = RawCode({kCbnzIP1Plus0Insn, kLdrWInsn});
1204 ArrayRef<const uint8_t> code2(raw_code2);
1205 const LinkerPatch patches2[] = {
1206 LinkerPatch::BakerReadBarrierBranchPatch(kLiteralOffset2, encoded_data),
1207 };
1208 AddCompiledMethod(MethodRef(5u), code2, ArrayRef<const LinkerPatch>(patches2));
1209
1210 Link();
1211
1212 uint32_t first_method_offset = GetMethodOffset(1u);
1213 uint32_t last_method_offset = GetMethodOffset(5u);
1214 EXPECT_EQ(2 * MB, last_method_offset - first_method_offset);
1215
1216 const uint32_t cbnz_max_forward = kCbnzIP1Plus0Insn | 0x007fffe0;
1217 const uint32_t cbnz_max_backward = kCbnzIP1Plus0Insn | 0x00800000;
1218 const std::vector<uint8_t> expected_code1 = RawCode({kNopInsn, cbnz_max_forward, kLdrWInsn});
1219 const std::vector<uint8_t> expected_code2 = RawCode({cbnz_max_backward, kLdrWInsn});
1220 ASSERT_TRUE(CheckLinkedMethod(MethodRef(1), ArrayRef<const uint8_t>(expected_code1)));
1221 ASSERT_TRUE(CheckLinkedMethod(MethodRef(5), ArrayRef<const uint8_t>(expected_code2)));
1222 }
1223
TEST_F(Arm64RelativePatcherTestDefault,BakerOffsetThunkBeforeFiller)1224 TEST_F(Arm64RelativePatcherTestDefault, BakerOffsetThunkBeforeFiller) {
1225 // Based on the first part of BakerOffsetThunkInTheMiddle but the CBNZ is one instruction
1226 // earlier, so the thunk is emitted before the filler.
1227 // Use offset = 0, base_reg = 0, ref_reg = 0, the LDR is simply `kLdrWInsn`.
1228 constexpr uint32_t kLiteralOffset1 = 0;
1229 const std::vector<uint8_t> raw_code1 = RawCode({kCbnzIP1Plus0Insn, kLdrWInsn, kNopInsn});
1230 ArrayRef<const uint8_t> code1(raw_code1);
1231 uint32_t encoded_data = EncodeBakerReadBarrierFieldData(/* base_reg */ 0, /* holder_reg */ 0);
1232 const LinkerPatch patches1[] = {
1233 LinkerPatch::BakerReadBarrierBranchPatch(kLiteralOffset1, encoded_data),
1234 };
1235 AddCompiledMethod(MethodRef(1u), code1, ArrayRef<const LinkerPatch>(patches1));
1236
1237 // Allow thunk at 1MiB offset from the start of the method above. Literal offset being 4
1238 // allows the branch to reach that thunk.
1239 size_t filler1_size =
1240 1 * MB - RoundUp(raw_code1.size() + sizeof(OatQuickMethodHeader), kArm64CodeAlignment);
1241 std::vector<uint8_t> raw_filler1_code = GenNops(filler1_size / 4u);
1242 ArrayRef<const uint8_t> filler1_code(raw_filler1_code);
1243 AddCompiledMethod(MethodRef(2u), filler1_code);
1244
1245 Link();
1246
1247 const uint32_t cbnz_offset = RoundUp(raw_code1.size(), kArm64CodeAlignment) - kLiteralOffset1;
1248 const uint32_t cbnz = kCbnzIP1Plus0Insn | (cbnz_offset << (5 - 2));
1249 const std::vector<uint8_t> expected_code1 = RawCode({cbnz, kLdrWInsn, kNopInsn});
1250 ASSERT_TRUE(CheckLinkedMethod(MethodRef(1), ArrayRef<const uint8_t>(expected_code1)));
1251 }
1252
TEST_F(Arm64RelativePatcherTestDefault,BakerOffsetThunkInTheMiddleUnreachableFromLast)1253 TEST_F(Arm64RelativePatcherTestDefault, BakerOffsetThunkInTheMiddleUnreachableFromLast) {
1254 // Based on the BakerOffsetThunkInTheMiddle but the CBNZ in the last method is preceded
1255 // by NOP and cannot reach the thunk in the middle, so we emit an extra thunk at the end.
1256 // Use offset = 0, base_reg = 0, ref_reg = 0, the LDR is simply `kLdrWInsn`.
1257 constexpr uint32_t kLiteralOffset1 = 4;
1258 const std::vector<uint8_t> raw_code1 = RawCode({kNopInsn, kCbnzIP1Plus0Insn, kLdrWInsn});
1259 ArrayRef<const uint8_t> code1(raw_code1);
1260 uint32_t encoded_data = EncodeBakerReadBarrierFieldData(/* base_reg */ 0, /* holder_reg */ 0);
1261 const LinkerPatch patches1[] = {
1262 LinkerPatch::BakerReadBarrierBranchPatch(kLiteralOffset1, encoded_data),
1263 };
1264 AddCompiledMethod(MethodRef(1u), code1, ArrayRef<const LinkerPatch>(patches1));
1265
1266 // Allow thunk at 1MiB offset from the start of the method above. Literal offset being 4
1267 // allows the branch to reach that thunk.
1268 size_t filler1_size =
1269 1 * MB - RoundUp(raw_code1.size() + sizeof(OatQuickMethodHeader), kArm64CodeAlignment);
1270 std::vector<uint8_t> raw_filler1_code = GenNops(filler1_size / 4u);
1271 ArrayRef<const uint8_t> filler1_code(raw_filler1_code);
1272 AddCompiledMethod(MethodRef(2u), filler1_code);
1273
1274 // Enforce thunk reservation with a tiny method.
1275 AddCompiledMethod(MethodRef(3u), kNopCode);
1276
1277 // If not for the extra NOP, this would allow reaching the thunk from the very beginning
1278 // of a method 1MiB away. Backward branch reaches the full 1MiB. Things to subtract:
1279 // - thunk size and method 3 pre-header, rounded up (padding in between if needed)
1280 // - method 3 code and method 4 pre-header, rounded up (padding in between if needed)
1281 // - method 4 header (let there be no padding between method 4 code and method 5 pre-header).
1282 size_t thunk_size = CompileBakerOffsetThunk(/* base_reg */ 0, /* holder_reg */ 0).size();
1283 size_t filler2_size =
1284 1 * MB - RoundUp(thunk_size + sizeof(OatQuickMethodHeader), kArm64CodeAlignment)
1285 - RoundUp(kNopCode.size() + sizeof(OatQuickMethodHeader), kArm64CodeAlignment)
1286 - sizeof(OatQuickMethodHeader);
1287 std::vector<uint8_t> raw_filler2_code = GenNops(filler2_size / 4u);
1288 ArrayRef<const uint8_t> filler2_code(raw_filler2_code);
1289 AddCompiledMethod(MethodRef(4u), filler2_code);
1290
1291 // Extra NOP compared to BakerOffsetThunkInTheMiddle.
1292 constexpr uint32_t kLiteralOffset2 = 4;
1293 const std::vector<uint8_t> raw_code2 = RawCode({kNopInsn, kCbnzIP1Plus0Insn, kLdrWInsn});
1294 ArrayRef<const uint8_t> code2(raw_code2);
1295 const LinkerPatch patches2[] = {
1296 LinkerPatch::BakerReadBarrierBranchPatch(kLiteralOffset2, encoded_data),
1297 };
1298 AddCompiledMethod(MethodRef(5u), code2, ArrayRef<const LinkerPatch>(patches2));
1299
1300 Link();
1301
1302 const uint32_t cbnz_max_forward = kCbnzIP1Plus0Insn | 0x007fffe0;
1303 const uint32_t cbnz_last_offset =
1304 RoundUp(raw_code2.size(), kArm64CodeAlignment) - kLiteralOffset2;
1305 const uint32_t cbnz_last = kCbnzIP1Plus0Insn | (cbnz_last_offset << (5 - 2));
1306 const std::vector<uint8_t> expected_code1 = RawCode({kNopInsn, cbnz_max_forward, kLdrWInsn});
1307 const std::vector<uint8_t> expected_code2 = RawCode({kNopInsn, cbnz_last, kLdrWInsn});
1308 ASSERT_TRUE(CheckLinkedMethod(MethodRef(1), ArrayRef<const uint8_t>(expected_code1)));
1309 ASSERT_TRUE(CheckLinkedMethod(MethodRef(5), ArrayRef<const uint8_t>(expected_code2)));
1310 }
1311
TEST_F(Arm64RelativePatcherTestDefault,BakerArray)1312 TEST_F(Arm64RelativePatcherTestDefault, BakerArray) {
1313 uint32_t valid_regs[] = {
1314 0, 1, 2, 3, 4, 5, 6, 7, 8, 9,
1315 10, 11, 12, 13, 14, 15, 18, 19, // IP0 and IP1 are reserved.
1316 20, 21, 22, 23, 24, 25, 26, 27, 28, 29,
1317 // LR and SP/ZR are reserved.
1318 };
1319 auto ldr = [](uint32_t base_reg) {
1320 uint32_t index_reg = (base_reg == 0u) ? 1u : 0u;
1321 uint32_t ref_reg = (base_reg == 2) ? 3u : 2u;
1322 return kLdrWLsl2Insn | (index_reg << 16) | (base_reg << 5) | ref_reg;
1323 };
1324 constexpr size_t kMethodCodeSize = 8u;
1325 constexpr size_t kLiteralOffset = 0u;
1326 uint32_t method_idx = 0u;
1327 for (uint32_t base_reg : valid_regs) {
1328 ++method_idx;
1329 const std::vector<uint8_t> raw_code = RawCode({kCbnzIP1Plus0Insn, ldr(base_reg)});
1330 ASSERT_EQ(kMethodCodeSize, raw_code.size());
1331 ArrayRef<const uint8_t> code(raw_code);
1332 const LinkerPatch patches[] = {
1333 LinkerPatch::BakerReadBarrierBranchPatch(
1334 kLiteralOffset, EncodeBakerReadBarrierArrayData(base_reg)),
1335 };
1336 AddCompiledMethod(MethodRef(method_idx), code, ArrayRef<const LinkerPatch>(patches));
1337 }
1338 Link();
1339
1340 // All thunks are at the end.
1341 uint32_t thunk_offset =
1342 GetMethodOffset(method_idx) + RoundUp(kMethodCodeSize, kArm64CodeAlignment);
1343 method_idx = 0u;
1344 for (uint32_t base_reg : valid_regs) {
1345 ++method_idx;
1346 uint32_t cbnz_offset = thunk_offset - (GetMethodOffset(method_idx) + kLiteralOffset);
1347 uint32_t cbnz = kCbnzIP1Plus0Insn | (cbnz_offset << (5 - 2));
1348 const std::vector<uint8_t> expected_code = RawCode({cbnz, ldr(base_reg)});
1349 ASSERT_EQ(kMethodCodeSize, expected_code.size());
1350 EXPECT_TRUE(CheckLinkedMethod(MethodRef(method_idx), ArrayRef<const uint8_t>(expected_code)));
1351
1352 std::vector<uint8_t> expected_thunk = CompileBakerArrayThunk(base_reg);
1353 ASSERT_GT(output_.size(), thunk_offset);
1354 ASSERT_GE(output_.size() - thunk_offset, expected_thunk.size());
1355 ArrayRef<const uint8_t> compiled_thunk(output_.data() + thunk_offset,
1356 expected_thunk.size());
1357 if (ArrayRef<const uint8_t>(expected_thunk) != compiled_thunk) {
1358 DumpDiff(ArrayRef<const uint8_t>(expected_thunk), compiled_thunk);
1359 ASSERT_TRUE(false);
1360 }
1361
1362 // Verify that the lock word for gray bit check is loaded from the correct address
1363 // before the base_reg which points to the array data.
1364 static constexpr size_t kGrayCheckInsns = 5;
1365 ASSERT_GE(output_.size() - thunk_offset, 4u * kGrayCheckInsns);
1366 int32_t data_offset =
1367 mirror::Array::DataOffset(Primitive::ComponentSize(Primitive::kPrimNot)).Int32Value();
1368 int32_t offset = mirror::Object::MonitorOffset().Int32Value() - data_offset;
1369 ASSERT_LT(offset, 0);
1370 const uint32_t load_lock_word =
1371 kLdurWInsn |
1372 ((offset & 0x1ffu) << 12) |
1373 (base_reg << 5) |
1374 /* ip0 */ 16;
1375 EXPECT_EQ(load_lock_word, GetOutputInsn(thunk_offset));
1376 // Verify the gray bit check.
1377 const uint32_t check_gray_bit_without_offset =
1378 0x37000000u | (LockWord::kReadBarrierStateShift << 19) | /* ip0 */ 16;
1379 EXPECT_EQ(check_gray_bit_without_offset, GetOutputInsn(thunk_offset + 4u) & 0xfff8001fu);
1380 // Verify the fake dependency.
1381 const uint32_t fake_dependency =
1382 0x8b408000u | // ADD Xd, Xn, Xm, LSR 32
1383 (/* ip0 */ 16 << 16) | // Xm = ip0
1384 (base_reg << 5) | // Xn = base_reg
1385 base_reg; // Xd = base_reg
1386 EXPECT_EQ(fake_dependency, GetOutputInsn(thunk_offset + 12u));
1387 // Do not check the rest of the implementation.
1388
1389 // The next thunk follows on the next aligned offset.
1390 thunk_offset += RoundUp(expected_thunk.size(), kArm64CodeAlignment);
1391 }
1392 }
1393
TEST_F(Arm64RelativePatcherTestDefault,BakerGcRoot)1394 TEST_F(Arm64RelativePatcherTestDefault, BakerGcRoot) {
1395 uint32_t valid_regs[] = {
1396 0, 1, 2, 3, 4, 5, 6, 7, 8, 9,
1397 10, 11, 12, 13, 14, 15, 18, 19, // IP0 and IP1 are reserved.
1398 20, 21, 22, 23, 24, 25, 26, 27, 28, 29,
1399 // LR and SP/ZR are reserved.
1400 };
1401 constexpr size_t kMethodCodeSize = 8u;
1402 constexpr size_t kLiteralOffset = 4u;
1403 uint32_t method_idx = 0u;
1404 for (uint32_t root_reg : valid_regs) {
1405 ++method_idx;
1406 uint32_t ldr = kLdrWInsn | (/* offset */ 8 << (10 - 2)) | (/* base_reg */ 0 << 5) | root_reg;
1407 const std::vector<uint8_t> raw_code = RawCode({ldr, kCbnzIP1Plus0Insn});
1408 ASSERT_EQ(kMethodCodeSize, raw_code.size());
1409 ArrayRef<const uint8_t> code(raw_code);
1410 const LinkerPatch patches[] = {
1411 LinkerPatch::BakerReadBarrierBranchPatch(
1412 kLiteralOffset, EncodeBakerReadBarrierGcRootData(root_reg)),
1413 };
1414 AddCompiledMethod(MethodRef(method_idx), code, ArrayRef<const LinkerPatch>(patches));
1415 }
1416 Link();
1417
1418 // All thunks are at the end.
1419 uint32_t thunk_offset =
1420 GetMethodOffset(method_idx) + RoundUp(kMethodCodeSize, kArm64CodeAlignment);
1421 method_idx = 0u;
1422 for (uint32_t root_reg : valid_regs) {
1423 ++method_idx;
1424 uint32_t cbnz_offset = thunk_offset - (GetMethodOffset(method_idx) + kLiteralOffset);
1425 uint32_t cbnz = kCbnzIP1Plus0Insn | (cbnz_offset << (5 - 2));
1426 uint32_t ldr = kLdrWInsn | (/* offset */ 8 << (10 - 2)) | (/* base_reg */ 0 << 5) | root_reg;
1427 const std::vector<uint8_t> expected_code = RawCode({ldr, cbnz});
1428 ASSERT_EQ(kMethodCodeSize, expected_code.size());
1429 EXPECT_TRUE(CheckLinkedMethod(MethodRef(method_idx), ArrayRef<const uint8_t>(expected_code)));
1430
1431 std::vector<uint8_t> expected_thunk = CompileBakerGcRootThunk(root_reg);
1432 ASSERT_GT(output_.size(), thunk_offset);
1433 ASSERT_GE(output_.size() - thunk_offset, expected_thunk.size());
1434 ArrayRef<const uint8_t> compiled_thunk(output_.data() + thunk_offset,
1435 expected_thunk.size());
1436 if (ArrayRef<const uint8_t>(expected_thunk) != compiled_thunk) {
1437 DumpDiff(ArrayRef<const uint8_t>(expected_thunk), compiled_thunk);
1438 ASSERT_TRUE(false);
1439 }
1440
1441 // Verify that the fast-path null-check CBZ uses the correct register, i.e. root_reg.
1442 ASSERT_GE(output_.size() - thunk_offset, 4u);
1443 ASSERT_EQ(0x34000000u | root_reg, GetOutputInsn(thunk_offset) & 0xff00001fu);
1444 // Do not check the rest of the implementation.
1445
1446 // The next thunk follows on the next aligned offset.
1447 thunk_offset += RoundUp(expected_thunk.size(), kArm64CodeAlignment);
1448 }
1449 }
1450
TEST_F(Arm64RelativePatcherTestDefault,BakerAndMethodCallInteraction)1451 TEST_F(Arm64RelativePatcherTestDefault, BakerAndMethodCallInteraction) {
1452 // During development, there was a `DCHECK_LE(MaxNextOffset(), next_thunk.MaxNextOffset());`
1453 // in `ArmBaseRelativePatcher::ThunkData::MakeSpaceBefore()` which does not necessarily
1454 // hold when we're reserving thunks of different sizes. This test exposes the situation
1455 // by using Baker thunks and a method call thunk.
1456
1457 // Add a method call patch that can reach to method 1 offset + 128MiB.
1458 uint32_t method_idx = 0u;
1459 constexpr size_t kMethodCallLiteralOffset = 4u;
1460 constexpr uint32_t kMissingMethodIdx = 2u;
1461 const std::vector<uint8_t> raw_code1 = RawCode({kNopInsn, kBlPlus0});
1462 const LinkerPatch method1_patches[] = {
1463 LinkerPatch::RelativeCodePatch(kMethodCallLiteralOffset, nullptr, 2u),
1464 };
1465 ArrayRef<const uint8_t> code1(raw_code1);
1466 ++method_idx;
1467 AddCompiledMethod(MethodRef(1u), code1, ArrayRef<const LinkerPatch>(method1_patches));
1468
1469 // Skip kMissingMethodIdx.
1470 ++method_idx;
1471 ASSERT_EQ(kMissingMethodIdx, method_idx);
1472 // Add a method with the right size that the method code for the next one starts 1MiB
1473 // after code for method 1.
1474 size_t filler_size =
1475 1 * MB - RoundUp(raw_code1.size() + sizeof(OatQuickMethodHeader), kArm64CodeAlignment)
1476 - sizeof(OatQuickMethodHeader);
1477 std::vector<uint8_t> filler_code = GenNops(filler_size / 4u);
1478 ++method_idx;
1479 AddCompiledMethod(MethodRef(method_idx), ArrayRef<const uint8_t>(filler_code));
1480 // Add 126 methods with 1MiB code+header, making the code for the next method start 1MiB
1481 // before the currently scheduled MaxNextOffset() for the method call thunk.
1482 for (uint32_t i = 0; i != 126; ++i) {
1483 filler_size = 1 * MB - sizeof(OatQuickMethodHeader);
1484 filler_code = GenNops(filler_size / 4u);
1485 ++method_idx;
1486 AddCompiledMethod(MethodRef(method_idx), ArrayRef<const uint8_t>(filler_code));
1487 }
1488
1489 // Add 2 Baker GC root patches to the last method, one that would allow the thunk at
1490 // 1MiB + kArm64CodeAlignment, i.e. kArm64CodeAlignment after the method call thunk, and the
1491 // second that needs it kArm64CodeAlignment after that. Given the size of the GC root thunk
1492 // is more than the space required by the method call thunk plus kArm64CodeAlignment,
1493 // this pushes the first GC root thunk's pending MaxNextOffset() before the method call
1494 // thunk's pending MaxNextOffset() which needs to be adjusted.
1495 ASSERT_LT(RoundUp(CompileMethodCallThunk().size(), kArm64CodeAlignment) + kArm64CodeAlignment,
1496 CompileBakerGcRootThunk(/* root_reg */ 0).size());
1497 static_assert(kArm64CodeAlignment == 16, "Code below assumes kArm64CodeAlignment == 16");
1498 constexpr size_t kBakerLiteralOffset1 = 4u + kArm64CodeAlignment;
1499 constexpr size_t kBakerLiteralOffset2 = 4u + 2 * kArm64CodeAlignment;
1500 // Use offset = 0, base_reg = 0, the LDR is simply `kLdrWInsn | root_reg`.
1501 const uint32_t ldr1 = kLdrWInsn | /* root_reg */ 1;
1502 const uint32_t ldr2 = kLdrWInsn | /* root_reg */ 2;
1503 const std::vector<uint8_t> last_method_raw_code = RawCode({
1504 kNopInsn, kNopInsn, kNopInsn, kNopInsn, // Padding before first GC root read barrier.
1505 ldr1, kCbnzIP1Plus0Insn, // First GC root LDR with read barrier.
1506 kNopInsn, kNopInsn, // Padding before second GC root read barrier.
1507 ldr2, kCbnzIP1Plus0Insn, // Second GC root LDR with read barrier.
1508 });
1509 uint32_t encoded_data1 = EncodeBakerReadBarrierGcRootData(/* root_reg */ 1);
1510 uint32_t encoded_data2 = EncodeBakerReadBarrierGcRootData(/* root_reg */ 2);
1511 const LinkerPatch last_method_patches[] = {
1512 LinkerPatch::BakerReadBarrierBranchPatch(kBakerLiteralOffset1, encoded_data1),
1513 LinkerPatch::BakerReadBarrierBranchPatch(kBakerLiteralOffset2, encoded_data2),
1514 };
1515 ++method_idx;
1516 AddCompiledMethod(MethodRef(method_idx),
1517 ArrayRef<const uint8_t>(last_method_raw_code),
1518 ArrayRef<const LinkerPatch>(last_method_patches));
1519
1520 // The main purpose of the test is to check that Link() does not cause a crash.
1521 Link();
1522
1523 ASSERT_EQ(127 * MB, GetMethodOffset(method_idx) - GetMethodOffset(1u));
1524 }
1525
1526 } // namespace linker
1527 } // namespace art
1528