1 /*
2 * Copyright (C) 2014 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 #ifndef ART_COMPILER_OPTIMIZING_CODE_GENERATOR_ARM64_H_
18 #define ART_COMPILER_OPTIMIZING_CODE_GENERATOR_ARM64_H_
19
20 #include "base/bit_field.h"
21 #include "class_root.h"
22 #include "code_generator.h"
23 #include "common_arm64.h"
24 #include "dex/dex_file_types.h"
25 #include "dex/string_reference.h"
26 #include "dex/type_reference.h"
27 #include "driver/compiler_options.h"
28 #include "nodes.h"
29 #include "parallel_move_resolver.h"
30 #include "utils/arm64/assembler_arm64.h"
31
32 // TODO(VIXL): Make VIXL compile with -Wshadow.
33 #pragma GCC diagnostic push
34 #pragma GCC diagnostic ignored "-Wshadow"
35 #include "aarch64/disasm-aarch64.h"
36 #include "aarch64/macro-assembler-aarch64.h"
37 #pragma GCC diagnostic pop
38
39 namespace art {
40
41 namespace linker {
42 class Arm64RelativePatcherTest;
43 } // namespace linker
44
45 namespace arm64 {
46
47 class CodeGeneratorARM64;
48
49 // Use a local definition to prevent copying mistakes.
50 static constexpr size_t kArm64WordSize = static_cast<size_t>(kArm64PointerSize);
51
52 // These constants are used as an approximate margin when emission of veneer and literal pools
53 // must be blocked.
54 static constexpr int kMaxMacroInstructionSizeInBytes = 15 * vixl::aarch64::kInstructionSize;
55 static constexpr int kInvokeCodeMarginSizeInBytes = 6 * kMaxMacroInstructionSizeInBytes;
56
57 static const vixl::aarch64::Register kParameterCoreRegisters[] = {
58 vixl::aarch64::x1,
59 vixl::aarch64::x2,
60 vixl::aarch64::x3,
61 vixl::aarch64::x4,
62 vixl::aarch64::x5,
63 vixl::aarch64::x6,
64 vixl::aarch64::x7
65 };
66 static constexpr size_t kParameterCoreRegistersLength = arraysize(kParameterCoreRegisters);
67 static const vixl::aarch64::VRegister kParameterFPRegisters[] = {
68 vixl::aarch64::d0,
69 vixl::aarch64::d1,
70 vixl::aarch64::d2,
71 vixl::aarch64::d3,
72 vixl::aarch64::d4,
73 vixl::aarch64::d5,
74 vixl::aarch64::d6,
75 vixl::aarch64::d7
76 };
77 static constexpr size_t kParameterFPRegistersLength = arraysize(kParameterFPRegisters);
78
79 // Thread Register.
80 const vixl::aarch64::Register tr = vixl::aarch64::x19;
81 // Marking Register.
82 const vixl::aarch64::Register mr = vixl::aarch64::x20;
83 // Method register on invoke.
84 static const vixl::aarch64::Register kArtMethodRegister = vixl::aarch64::x0;
85 const vixl::aarch64::CPURegList vixl_reserved_core_registers(vixl::aarch64::ip0,
86 vixl::aarch64::ip1);
87 const vixl::aarch64::CPURegList vixl_reserved_fp_registers(vixl::aarch64::d31);
88
89 const vixl::aarch64::CPURegList runtime_reserved_core_registers =
90 vixl::aarch64::CPURegList(
91 tr,
92 // Reserve X20 as Marking Register when emitting Baker read barriers.
93 ((kEmitCompilerReadBarrier && kUseBakerReadBarrier) ? mr : vixl::aarch64::NoCPUReg),
94 vixl::aarch64::lr);
95
96 // Some instructions have special requirements for a temporary, for example
97 // LoadClass/kBssEntry and LoadString/kBssEntry for Baker read barrier require
98 // temp that's not an R0 (to avoid an extra move) and Baker read barrier field
99 // loads with large offsets need a fixed register to limit the number of link-time
100 // thunks we generate. For these and similar cases, we want to reserve a specific
101 // register that's neither callee-save nor an argument register. We choose x15.
FixedTempLocation()102 inline Location FixedTempLocation() {
103 return Location::RegisterLocation(vixl::aarch64::x15.GetCode());
104 }
105
106 // Callee-save registers AAPCS64, without x19 (Thread Register) (nor
107 // x20 (Marking Register) when emitting Baker read barriers).
108 const vixl::aarch64::CPURegList callee_saved_core_registers(
109 vixl::aarch64::CPURegister::kRegister,
110 vixl::aarch64::kXRegSize,
111 ((kEmitCompilerReadBarrier && kUseBakerReadBarrier)
112 ? vixl::aarch64::x21.GetCode()
113 : vixl::aarch64::x20.GetCode()),
114 vixl::aarch64::x30.GetCode());
115 const vixl::aarch64::CPURegList callee_saved_fp_registers(vixl::aarch64::CPURegister::kVRegister,
116 vixl::aarch64::kDRegSize,
117 vixl::aarch64::d8.GetCode(),
118 vixl::aarch64::d15.GetCode());
119 Location ARM64ReturnLocation(DataType::Type return_type);
120
121 class SlowPathCodeARM64 : public SlowPathCode {
122 public:
SlowPathCodeARM64(HInstruction * instruction)123 explicit SlowPathCodeARM64(HInstruction* instruction)
124 : SlowPathCode(instruction), entry_label_(), exit_label_() {}
125
GetEntryLabel()126 vixl::aarch64::Label* GetEntryLabel() { return &entry_label_; }
GetExitLabel()127 vixl::aarch64::Label* GetExitLabel() { return &exit_label_; }
128
129 void SaveLiveRegisters(CodeGenerator* codegen, LocationSummary* locations) override;
130 void RestoreLiveRegisters(CodeGenerator* codegen, LocationSummary* locations) override;
131
132 private:
133 vixl::aarch64::Label entry_label_;
134 vixl::aarch64::Label exit_label_;
135
136 DISALLOW_COPY_AND_ASSIGN(SlowPathCodeARM64);
137 };
138
139 class JumpTableARM64 : public DeletableArenaObject<kArenaAllocSwitchTable> {
140 public:
JumpTableARM64(HPackedSwitch * switch_instr)141 explicit JumpTableARM64(HPackedSwitch* switch_instr)
142 : switch_instr_(switch_instr), table_start_() {}
143
GetTableStartLabel()144 vixl::aarch64::Label* GetTableStartLabel() { return &table_start_; }
145
146 void EmitTable(CodeGeneratorARM64* codegen);
147
148 private:
149 HPackedSwitch* const switch_instr_;
150 vixl::aarch64::Label table_start_;
151
152 DISALLOW_COPY_AND_ASSIGN(JumpTableARM64);
153 };
154
155 static const vixl::aarch64::Register kRuntimeParameterCoreRegisters[] =
156 { vixl::aarch64::x0,
157 vixl::aarch64::x1,
158 vixl::aarch64::x2,
159 vixl::aarch64::x3,
160 vixl::aarch64::x4,
161 vixl::aarch64::x5,
162 vixl::aarch64::x6,
163 vixl::aarch64::x7 };
164 static constexpr size_t kRuntimeParameterCoreRegistersLength =
165 arraysize(kRuntimeParameterCoreRegisters);
166 static const vixl::aarch64::VRegister kRuntimeParameterFpuRegisters[] =
167 { vixl::aarch64::d0,
168 vixl::aarch64::d1,
169 vixl::aarch64::d2,
170 vixl::aarch64::d3,
171 vixl::aarch64::d4,
172 vixl::aarch64::d5,
173 vixl::aarch64::d6,
174 vixl::aarch64::d7 };
175 static constexpr size_t kRuntimeParameterFpuRegistersLength =
176 arraysize(kRuntimeParameterCoreRegisters);
177
178 class InvokeRuntimeCallingConvention : public CallingConvention<vixl::aarch64::Register,
179 vixl::aarch64::VRegister> {
180 public:
181 static constexpr size_t kParameterCoreRegistersLength = arraysize(kParameterCoreRegisters);
182
InvokeRuntimeCallingConvention()183 InvokeRuntimeCallingConvention()
184 : CallingConvention(kRuntimeParameterCoreRegisters,
185 kRuntimeParameterCoreRegistersLength,
186 kRuntimeParameterFpuRegisters,
187 kRuntimeParameterFpuRegistersLength,
188 kArm64PointerSize) {}
189
190 Location GetReturnLocation(DataType::Type return_type);
191
192 private:
193 DISALLOW_COPY_AND_ASSIGN(InvokeRuntimeCallingConvention);
194 };
195
196 class InvokeDexCallingConvention : public CallingConvention<vixl::aarch64::Register,
197 vixl::aarch64::VRegister> {
198 public:
InvokeDexCallingConvention()199 InvokeDexCallingConvention()
200 : CallingConvention(kParameterCoreRegisters,
201 kParameterCoreRegistersLength,
202 kParameterFPRegisters,
203 kParameterFPRegistersLength,
204 kArm64PointerSize) {}
205
GetReturnLocation(DataType::Type return_type)206 Location GetReturnLocation(DataType::Type return_type) const {
207 return ARM64ReturnLocation(return_type);
208 }
209
210
211 private:
212 DISALLOW_COPY_AND_ASSIGN(InvokeDexCallingConvention);
213 };
214
215 class InvokeDexCallingConventionVisitorARM64 : public InvokeDexCallingConventionVisitor {
216 public:
InvokeDexCallingConventionVisitorARM64()217 InvokeDexCallingConventionVisitorARM64() {}
~InvokeDexCallingConventionVisitorARM64()218 virtual ~InvokeDexCallingConventionVisitorARM64() {}
219
220 Location GetNextLocation(DataType::Type type) override;
GetReturnLocation(DataType::Type return_type)221 Location GetReturnLocation(DataType::Type return_type) const override {
222 return calling_convention.GetReturnLocation(return_type);
223 }
224 Location GetMethodLocation() const override;
225
226 private:
227 InvokeDexCallingConvention calling_convention;
228
229 DISALLOW_COPY_AND_ASSIGN(InvokeDexCallingConventionVisitorARM64);
230 };
231
232 class CriticalNativeCallingConventionVisitorARM64 : public InvokeDexCallingConventionVisitor {
233 public:
CriticalNativeCallingConventionVisitorARM64(bool for_register_allocation)234 explicit CriticalNativeCallingConventionVisitorARM64(bool for_register_allocation)
235 : for_register_allocation_(for_register_allocation) {}
236
~CriticalNativeCallingConventionVisitorARM64()237 virtual ~CriticalNativeCallingConventionVisitorARM64() {}
238
239 Location GetNextLocation(DataType::Type type) override;
240 Location GetReturnLocation(DataType::Type type) const override;
241 Location GetMethodLocation() const override;
242
GetStackOffset()243 size_t GetStackOffset() const { return stack_offset_; }
244
245 private:
246 // Register allocator does not support adjusting frame size, so we cannot provide final locations
247 // of stack arguments for register allocation. We ask the register allocator for any location and
248 // move these arguments to the right place after adjusting the SP when generating the call.
249 const bool for_register_allocation_;
250 size_t gpr_index_ = 0u;
251 size_t fpr_index_ = 0u;
252 size_t stack_offset_ = 0u;
253
254 DISALLOW_COPY_AND_ASSIGN(CriticalNativeCallingConventionVisitorARM64);
255 };
256
257 class FieldAccessCallingConventionARM64 : public FieldAccessCallingConvention {
258 public:
FieldAccessCallingConventionARM64()259 FieldAccessCallingConventionARM64() {}
260
GetObjectLocation()261 Location GetObjectLocation() const override {
262 return helpers::LocationFrom(vixl::aarch64::x1);
263 }
GetFieldIndexLocation()264 Location GetFieldIndexLocation() const override {
265 return helpers::LocationFrom(vixl::aarch64::x0);
266 }
GetReturnLocation(DataType::Type type ATTRIBUTE_UNUSED)267 Location GetReturnLocation(DataType::Type type ATTRIBUTE_UNUSED) const override {
268 return helpers::LocationFrom(vixl::aarch64::x0);
269 }
GetSetValueLocation(DataType::Type type ATTRIBUTE_UNUSED,bool is_instance)270 Location GetSetValueLocation(DataType::Type type ATTRIBUTE_UNUSED,
271 bool is_instance) const override {
272 return is_instance
273 ? helpers::LocationFrom(vixl::aarch64::x2)
274 : helpers::LocationFrom(vixl::aarch64::x1);
275 }
GetFpuLocation(DataType::Type type ATTRIBUTE_UNUSED)276 Location GetFpuLocation(DataType::Type type ATTRIBUTE_UNUSED) const override {
277 return helpers::LocationFrom(vixl::aarch64::d0);
278 }
279
280 private:
281 DISALLOW_COPY_AND_ASSIGN(FieldAccessCallingConventionARM64);
282 };
283
284 class InstructionCodeGeneratorARM64 : public InstructionCodeGenerator {
285 public:
286 InstructionCodeGeneratorARM64(HGraph* graph, CodeGeneratorARM64* codegen);
287
288 #define DECLARE_VISIT_INSTRUCTION(name, super) \
289 void Visit##name(H##name* instr) override;
290
291 FOR_EACH_CONCRETE_INSTRUCTION_SCALAR_COMMON(DECLARE_VISIT_INSTRUCTION)
FOR_EACH_CONCRETE_INSTRUCTION_ARM64(DECLARE_VISIT_INSTRUCTION)292 FOR_EACH_CONCRETE_INSTRUCTION_ARM64(DECLARE_VISIT_INSTRUCTION)
293 FOR_EACH_CONCRETE_INSTRUCTION_SHARED(DECLARE_VISIT_INSTRUCTION)
294
295 #undef DECLARE_VISIT_INSTRUCTION
296
297 void VisitInstruction(HInstruction* instruction) override {
298 LOG(FATAL) << "Unreachable instruction " << instruction->DebugName()
299 << " (id " << instruction->GetId() << ")";
300 }
301
GetAssembler()302 Arm64Assembler* GetAssembler() const { return assembler_; }
GetVIXLAssembler()303 vixl::aarch64::MacroAssembler* GetVIXLAssembler() { return GetAssembler()->GetVIXLAssembler(); }
304
305 // SIMD helpers.
306 virtual Location AllocateSIMDScratchLocation(vixl::aarch64::UseScratchRegisterScope* scope) = 0;
307 virtual void FreeSIMDScratchLocation(Location loc,
308 vixl::aarch64::UseScratchRegisterScope* scope) = 0;
309 virtual void LoadSIMDRegFromStack(Location destination, Location source) = 0;
310 virtual void MoveSIMDRegToSIMDReg(Location destination, Location source) = 0;
311 virtual void MoveToSIMDStackSlot(Location destination, Location source) = 0;
312 virtual void SaveLiveRegistersHelper(LocationSummary* locations,
313 int64_t spill_offset) = 0;
314 virtual void RestoreLiveRegistersHelper(LocationSummary* locations,
315 int64_t spill_offset) = 0;
316
317 protected:
318 void GenerateClassInitializationCheck(SlowPathCodeARM64* slow_path,
319 vixl::aarch64::Register class_reg);
320 void GenerateBitstringTypeCheckCompare(HTypeCheckInstruction* check,
321 vixl::aarch64::Register temp);
322 void GenerateSuspendCheck(HSuspendCheck* instruction, HBasicBlock* successor);
323 void HandleBinaryOp(HBinaryOperation* instr);
324
325 void HandleFieldSet(HInstruction* instruction,
326 const FieldInfo& field_info,
327 bool value_can_be_null);
328 void HandleFieldGet(HInstruction* instruction, const FieldInfo& field_info);
329 void HandleCondition(HCondition* instruction);
330
331 // Generate a heap reference load using one register `out`:
332 //
333 // out <- *(out + offset)
334 //
335 // while honoring heap poisoning and/or read barriers (if any).
336 //
337 // Location `maybe_temp` is used when generating a read barrier and
338 // shall be a register in that case; it may be an invalid location
339 // otherwise.
340 void GenerateReferenceLoadOneRegister(HInstruction* instruction,
341 Location out,
342 uint32_t offset,
343 Location maybe_temp,
344 ReadBarrierOption read_barrier_option);
345 // Generate a heap reference load using two different registers
346 // `out` and `obj`:
347 //
348 // out <- *(obj + offset)
349 //
350 // while honoring heap poisoning and/or read barriers (if any).
351 //
352 // Location `maybe_temp` is used when generating a Baker's (fast
353 // path) read barrier and shall be a register in that case; it may
354 // be an invalid location otherwise.
355 void GenerateReferenceLoadTwoRegisters(HInstruction* instruction,
356 Location out,
357 Location obj,
358 uint32_t offset,
359 Location maybe_temp,
360 ReadBarrierOption read_barrier_option);
361
362 // Generate a floating-point comparison.
363 void GenerateFcmp(HInstruction* instruction);
364
365 void HandleShift(HBinaryOperation* instr);
366 void GenerateTestAndBranch(HInstruction* instruction,
367 size_t condition_input_index,
368 vixl::aarch64::Label* true_target,
369 vixl::aarch64::Label* false_target);
370 void DivRemOneOrMinusOne(HBinaryOperation* instruction);
371 void DivRemByPowerOfTwo(HBinaryOperation* instruction);
372 void GenerateIncrementNegativeByOne(vixl::aarch64::Register out,
373 vixl::aarch64::Register in, bool use_cond_inc);
374 void GenerateResultRemWithAnyConstant(vixl::aarch64::Register out,
375 vixl::aarch64::Register dividend,
376 vixl::aarch64::Register quotient,
377 int64_t divisor,
378 // This function may acquire a scratch register.
379 vixl::aarch64::UseScratchRegisterScope* temps_scope);
380 void GenerateInt64UnsignedDivRemWithAnyPositiveConstant(HBinaryOperation* instruction);
381 void GenerateInt64DivRemWithAnyConstant(HBinaryOperation* instruction);
382 void GenerateInt32DivRemWithAnyConstant(HBinaryOperation* instruction);
383 void GenerateDivRemWithAnyConstant(HBinaryOperation* instruction, int64_t divisor);
384 void GenerateIntDiv(HDiv* instruction);
385 void GenerateIntDivForConstDenom(HDiv *instruction);
386 void GenerateIntDivForPower2Denom(HDiv *instruction);
387 void GenerateIntRem(HRem* instruction);
388 void GenerateIntRemForConstDenom(HRem *instruction);
389 void GenerateIntRemForPower2Denom(HRem *instruction);
390 void HandleGoto(HInstruction* got, HBasicBlock* successor);
391
392 // Helpers to set up locations for vector memory operations. Returns the memory operand and,
393 // if used, sets the output parameter scratch to a temporary register used in this operand,
394 // so that the client can release it right after the memory operand use.
395 // Neon version.
396 vixl::aarch64::MemOperand VecNEONAddress(
397 HVecMemoryOperation* instruction,
398 // This function may acquire a scratch register.
399 vixl::aarch64::UseScratchRegisterScope* temps_scope,
400 size_t size,
401 bool is_string_char_at,
402 /*out*/ vixl::aarch64::Register* scratch);
403 // SVE version.
404 vixl::aarch64::SVEMemOperand VecSVEAddress(
405 HVecMemoryOperation* instruction,
406 // This function may acquire a scratch register.
407 vixl::aarch64::UseScratchRegisterScope* temps_scope,
408 size_t size,
409 bool is_string_char_at,
410 /*out*/ vixl::aarch64::Register* scratch);
411
412 Arm64Assembler* const assembler_;
413 CodeGeneratorARM64* const codegen_;
414
415 DISALLOW_COPY_AND_ASSIGN(InstructionCodeGeneratorARM64);
416 };
417
418 class LocationsBuilderARM64 : public HGraphVisitor {
419 public:
LocationsBuilderARM64(HGraph * graph,CodeGeneratorARM64 * codegen)420 LocationsBuilderARM64(HGraph* graph, CodeGeneratorARM64* codegen)
421 : HGraphVisitor(graph), codegen_(codegen) {}
422
423 #define DECLARE_VISIT_INSTRUCTION(name, super) \
424 void Visit##name(H##name* instr) override;
425
426 FOR_EACH_CONCRETE_INSTRUCTION_SCALAR_COMMON(DECLARE_VISIT_INSTRUCTION)
FOR_EACH_CONCRETE_INSTRUCTION_ARM64(DECLARE_VISIT_INSTRUCTION)427 FOR_EACH_CONCRETE_INSTRUCTION_ARM64(DECLARE_VISIT_INSTRUCTION)
428 FOR_EACH_CONCRETE_INSTRUCTION_SHARED(DECLARE_VISIT_INSTRUCTION)
429
430 #undef DECLARE_VISIT_INSTRUCTION
431
432 void VisitInstruction(HInstruction* instruction) override {
433 LOG(FATAL) << "Unreachable instruction " << instruction->DebugName()
434 << " (id " << instruction->GetId() << ")";
435 }
436
437 protected:
438 void HandleBinaryOp(HBinaryOperation* instr);
439 void HandleFieldSet(HInstruction* instruction);
440 void HandleFieldGet(HInstruction* instruction, const FieldInfo& field_info);
441 void HandleInvoke(HInvoke* instr);
442 void HandleCondition(HCondition* instruction);
443 void HandleShift(HBinaryOperation* instr);
444
445 CodeGeneratorARM64* const codegen_;
446 InvokeDexCallingConventionVisitorARM64 parameter_visitor_;
447
448 DISALLOW_COPY_AND_ASSIGN(LocationsBuilderARM64);
449 };
450
451 class InstructionCodeGeneratorARM64Neon : public InstructionCodeGeneratorARM64 {
452 public:
InstructionCodeGeneratorARM64Neon(HGraph * graph,CodeGeneratorARM64 * codegen)453 InstructionCodeGeneratorARM64Neon(HGraph* graph, CodeGeneratorARM64* codegen) :
454 InstructionCodeGeneratorARM64(graph, codegen) {}
455
456 #define DECLARE_VISIT_INSTRUCTION(name, super) \
457 void Visit##name(H##name* instr) override;
458
459 FOR_EACH_CONCRETE_INSTRUCTION_VECTOR_COMMON(DECLARE_VISIT_INSTRUCTION)
460
461 #undef DECLARE_VISIT_INSTRUCTION
462
463 Location AllocateSIMDScratchLocation(vixl::aarch64::UseScratchRegisterScope* scope) override;
464 void FreeSIMDScratchLocation(Location loc,
465 vixl::aarch64::UseScratchRegisterScope* scope) override;
466 void LoadSIMDRegFromStack(Location destination, Location source) override;
467 void MoveSIMDRegToSIMDReg(Location destination, Location source) override;
468 void MoveToSIMDStackSlot(Location destination, Location source) override;
469 void SaveLiveRegistersHelper(LocationSummary* locations, int64_t spill_offset) override;
470 void RestoreLiveRegistersHelper(LocationSummary* locations, int64_t spill_offset) override;
471 };
472
473 class LocationsBuilderARM64Neon : public LocationsBuilderARM64 {
474 public:
LocationsBuilderARM64Neon(HGraph * graph,CodeGeneratorARM64 * codegen)475 LocationsBuilderARM64Neon(HGraph* graph, CodeGeneratorARM64* codegen) :
476 LocationsBuilderARM64(graph, codegen) {}
477
478 #define DECLARE_VISIT_INSTRUCTION(name, super) \
479 void Visit##name(H##name* instr) override;
480
481 FOR_EACH_CONCRETE_INSTRUCTION_VECTOR_COMMON(DECLARE_VISIT_INSTRUCTION)
482
483 #undef DECLARE_VISIT_INSTRUCTION
484 };
485
486 class InstructionCodeGeneratorARM64Sve : public InstructionCodeGeneratorARM64 {
487 public:
InstructionCodeGeneratorARM64Sve(HGraph * graph,CodeGeneratorARM64 * codegen)488 InstructionCodeGeneratorARM64Sve(HGraph* graph, CodeGeneratorARM64* codegen) :
489 InstructionCodeGeneratorARM64(graph, codegen) {}
490
491 #define DECLARE_VISIT_INSTRUCTION(name, super) \
492 void Visit##name(H##name* instr) override;
493
494 FOR_EACH_CONCRETE_INSTRUCTION_VECTOR_COMMON(DECLARE_VISIT_INSTRUCTION)
495
496 #undef DECLARE_VISIT_INSTRUCTION
497
498 Location AllocateSIMDScratchLocation(vixl::aarch64::UseScratchRegisterScope* scope) override;
499 void FreeSIMDScratchLocation(Location loc,
500 vixl::aarch64::UseScratchRegisterScope* scope) override;
501 void LoadSIMDRegFromStack(Location destination, Location source) override;
502 void MoveSIMDRegToSIMDReg(Location destination, Location source) override;
503 void MoveToSIMDStackSlot(Location destination, Location source) override;
504 void SaveLiveRegistersHelper(LocationSummary* locations, int64_t spill_offset) override;
505 void RestoreLiveRegistersHelper(LocationSummary* locations, int64_t spill_offset) override;
506
507 private:
508 // Validate that instruction vector length and packed type are compliant with the SIMD
509 // register size (full SIMD register is used).
510 void ValidateVectorLength(HVecOperation* instr) const;
511
512 // Returns default predicate register which is used as governing vector predicate
513 // to implement predicated loop execution.
514 //
515 // TODO: This is a hack to be addressed when register allocator supports SIMD types.
LoopPReg()516 static vixl::aarch64::PRegister LoopPReg() {
517 return vixl::aarch64::p0;
518 }
519 };
520
521 class LocationsBuilderARM64Sve : public LocationsBuilderARM64 {
522 public:
LocationsBuilderARM64Sve(HGraph * graph,CodeGeneratorARM64 * codegen)523 LocationsBuilderARM64Sve(HGraph* graph, CodeGeneratorARM64* codegen) :
524 LocationsBuilderARM64(graph, codegen) {}
525
526 #define DECLARE_VISIT_INSTRUCTION(name, super) \
527 void Visit##name(H##name* instr) override;
528
529 FOR_EACH_CONCRETE_INSTRUCTION_VECTOR_COMMON(DECLARE_VISIT_INSTRUCTION)
530
531 #undef DECLARE_VISIT_INSTRUCTION
532 };
533
534 class ParallelMoveResolverARM64 : public ParallelMoveResolverNoSwap {
535 public:
ParallelMoveResolverARM64(ArenaAllocator * allocator,CodeGeneratorARM64 * codegen)536 ParallelMoveResolverARM64(ArenaAllocator* allocator, CodeGeneratorARM64* codegen)
537 : ParallelMoveResolverNoSwap(allocator), codegen_(codegen), vixl_temps_() {}
538
539 protected:
540 void PrepareForEmitNativeCode() override;
541 void FinishEmitNativeCode() override;
542 Location AllocateScratchLocationFor(Location::Kind kind) override;
543 void FreeScratchLocation(Location loc) override;
544 void EmitMove(size_t index) override;
545
546 private:
547 Arm64Assembler* GetAssembler() const;
GetVIXLAssembler()548 vixl::aarch64::MacroAssembler* GetVIXLAssembler() const {
549 return GetAssembler()->GetVIXLAssembler();
550 }
551
552 CodeGeneratorARM64* const codegen_;
553 vixl::aarch64::UseScratchRegisterScope vixl_temps_;
554
555 DISALLOW_COPY_AND_ASSIGN(ParallelMoveResolverARM64);
556 };
557
558 class CodeGeneratorARM64 : public CodeGenerator {
559 public:
560 CodeGeneratorARM64(HGraph* graph,
561 const CompilerOptions& compiler_options,
562 OptimizingCompilerStats* stats = nullptr);
~CodeGeneratorARM64()563 virtual ~CodeGeneratorARM64() {}
564
565 void GenerateFrameEntry() override;
566 void GenerateFrameExit() override;
567
568 vixl::aarch64::CPURegList GetFramePreservedCoreRegisters() const;
569 vixl::aarch64::CPURegList GetFramePreservedFPRegisters() const;
570
571 void Bind(HBasicBlock* block) override;
572
GetLabelOf(HBasicBlock * block)573 vixl::aarch64::Label* GetLabelOf(HBasicBlock* block) {
574 block = FirstNonEmptyBlock(block);
575 return &(block_labels_[block->GetBlockId()]);
576 }
577
GetWordSize()578 size_t GetWordSize() const override {
579 return kArm64WordSize;
580 }
581
SupportsPredicatedSIMD()582 bool SupportsPredicatedSIMD() const override { return ShouldUseSVE(); }
583
GetSlowPathFPWidth()584 size_t GetSlowPathFPWidth() const override {
585 return GetGraph()->HasSIMD()
586 ? GetSIMDRegisterWidth()
587 : vixl::aarch64::kDRegSizeInBytes;
588 }
589
GetCalleePreservedFPWidth()590 size_t GetCalleePreservedFPWidth() const override {
591 return vixl::aarch64::kDRegSizeInBytes;
592 }
593
594 size_t GetSIMDRegisterWidth() const override;
595
GetAddressOf(HBasicBlock * block)596 uintptr_t GetAddressOf(HBasicBlock* block) override {
597 vixl::aarch64::Label* block_entry_label = GetLabelOf(block);
598 DCHECK(block_entry_label->IsBound());
599 return block_entry_label->GetLocation();
600 }
601
GetLocationBuilder()602 HGraphVisitor* GetLocationBuilder() override { return location_builder_; }
GetInstructionCodeGeneratorArm64()603 InstructionCodeGeneratorARM64* GetInstructionCodeGeneratorArm64() {
604 return instruction_visitor_;
605 }
GetInstructionVisitor()606 HGraphVisitor* GetInstructionVisitor() override { return GetInstructionCodeGeneratorArm64(); }
GetAssembler()607 Arm64Assembler* GetAssembler() override { return &assembler_; }
GetAssembler()608 const Arm64Assembler& GetAssembler() const override { return assembler_; }
GetVIXLAssembler()609 vixl::aarch64::MacroAssembler* GetVIXLAssembler() { return GetAssembler()->GetVIXLAssembler(); }
610
611 // Emit a write barrier.
612 void MarkGCCard(vixl::aarch64::Register object,
613 vixl::aarch64::Register value,
614 bool value_can_be_null);
615
616 void GenerateMemoryBarrier(MemBarrierKind kind);
617
618 // Register allocation.
619
620 void SetupBlockedRegisters() const override;
621
622 size_t SaveCoreRegister(size_t stack_index, uint32_t reg_id) override;
623 size_t RestoreCoreRegister(size_t stack_index, uint32_t reg_id) override;
624 size_t SaveFloatingPointRegister(size_t stack_index, uint32_t reg_id) override;
625 size_t RestoreFloatingPointRegister(size_t stack_index, uint32_t reg_id) override;
626
627 // The number of registers that can be allocated. The register allocator may
628 // decide to reserve and not use a few of them.
629 // We do not consider registers sp, xzr, wzr. They are either not allocatable
630 // (xzr, wzr), or make for poor allocatable registers (sp alignment
631 // requirements, etc.). This also facilitates our task as all other registers
632 // can easily be mapped via to or from their type and index or code.
633 static const int kNumberOfAllocatableRegisters = vixl::aarch64::kNumberOfRegisters - 1;
634 static const int kNumberOfAllocatableFPRegisters = vixl::aarch64::kNumberOfVRegisters;
635 static constexpr int kNumberOfAllocatableRegisterPairs = 0;
636
637 void DumpCoreRegister(std::ostream& stream, int reg) const override;
638 void DumpFloatingPointRegister(std::ostream& stream, int reg) const override;
639
GetInstructionSet()640 InstructionSet GetInstructionSet() const override {
641 return InstructionSet::kArm64;
642 }
643
644 const Arm64InstructionSetFeatures& GetInstructionSetFeatures() const;
645
Initialize()646 void Initialize() override {
647 block_labels_.resize(GetGraph()->GetBlocks().size());
648 }
649
650 // We want to use the STP and LDP instructions to spill and restore registers for slow paths.
651 // These instructions can only encode offsets that are multiples of the register size accessed.
GetPreferredSlotsAlignment()652 uint32_t GetPreferredSlotsAlignment() const override { return vixl::aarch64::kXRegSizeInBytes; }
653
CreateJumpTable(HPackedSwitch * switch_instr)654 JumpTableARM64* CreateJumpTable(HPackedSwitch* switch_instr) {
655 jump_tables_.emplace_back(new (GetGraph()->GetAllocator()) JumpTableARM64(switch_instr));
656 return jump_tables_.back().get();
657 }
658
659 void Finalize(CodeAllocator* allocator) override;
660
661 // Code generation helpers.
662 void MoveConstant(vixl::aarch64::CPURegister destination, HConstant* constant);
663 void MoveConstant(Location destination, int32_t value) override;
664 void MoveLocation(Location dst, Location src, DataType::Type dst_type) override;
665 void AddLocationAsTemp(Location location, LocationSummary* locations) override;
666
667 void Load(DataType::Type type,
668 vixl::aarch64::CPURegister dst,
669 const vixl::aarch64::MemOperand& src);
670 void Store(DataType::Type type,
671 vixl::aarch64::CPURegister src,
672 const vixl::aarch64::MemOperand& dst);
673 void LoadAcquire(HInstruction* instruction,
674 DataType::Type type,
675 vixl::aarch64::CPURegister dst,
676 const vixl::aarch64::MemOperand& src,
677 bool needs_null_check);
678 void StoreRelease(HInstruction* instruction,
679 DataType::Type type,
680 vixl::aarch64::CPURegister src,
681 const vixl::aarch64::MemOperand& dst,
682 bool needs_null_check);
683
684 // Generate code to invoke a runtime entry point.
685 void InvokeRuntime(QuickEntrypointEnum entrypoint,
686 HInstruction* instruction,
687 uint32_t dex_pc,
688 SlowPathCode* slow_path = nullptr) override;
689
690 // Generate code to invoke a runtime entry point, but do not record
691 // PC-related information in a stack map.
692 void InvokeRuntimeWithoutRecordingPcInfo(int32_t entry_point_offset,
693 HInstruction* instruction,
694 SlowPathCode* slow_path);
695
GetMoveResolver()696 ParallelMoveResolverARM64* GetMoveResolver() override { return &move_resolver_; }
697
NeedsTwoRegisters(DataType::Type type ATTRIBUTE_UNUSED)698 bool NeedsTwoRegisters(DataType::Type type ATTRIBUTE_UNUSED) const override {
699 return false;
700 }
701
702 // Check if the desired_string_load_kind is supported. If it is, return it,
703 // otherwise return a fall-back kind that should be used instead.
704 HLoadString::LoadKind GetSupportedLoadStringKind(
705 HLoadString::LoadKind desired_string_load_kind) override;
706
707 // Check if the desired_class_load_kind is supported. If it is, return it,
708 // otherwise return a fall-back kind that should be used instead.
709 HLoadClass::LoadKind GetSupportedLoadClassKind(
710 HLoadClass::LoadKind desired_class_load_kind) override;
711
712 // Check if the desired_dispatch_info is supported. If it is, return it,
713 // otherwise return a fall-back info that should be used instead.
714 HInvokeStaticOrDirect::DispatchInfo GetSupportedInvokeStaticOrDirectDispatch(
715 const HInvokeStaticOrDirect::DispatchInfo& desired_dispatch_info,
716 ArtMethod* method) override;
717
718 void LoadMethod(MethodLoadKind load_kind, Location temp, HInvoke* invoke);
719 void GenerateStaticOrDirectCall(
720 HInvokeStaticOrDirect* invoke, Location temp, SlowPathCode* slow_path = nullptr) override;
721 void GenerateVirtualCall(
722 HInvokeVirtual* invoke, Location temp, SlowPathCode* slow_path = nullptr) override;
723
724 void MoveFromReturnRegister(Location trg, DataType::Type type) override;
725
726 // Add a new boot image intrinsic patch for an instruction and return the label
727 // to be bound before the instruction. The instruction will be either the
728 // ADRP (pass `adrp_label = null`) or the ADD (pass `adrp_label` pointing
729 // to the associated ADRP patch label).
730 vixl::aarch64::Label* NewBootImageIntrinsicPatch(uint32_t intrinsic_data,
731 vixl::aarch64::Label* adrp_label = nullptr);
732
733 // Add a new boot image relocation patch for an instruction and return the label
734 // to be bound before the instruction. The instruction will be either the
735 // ADRP (pass `adrp_label = null`) or the LDR (pass `adrp_label` pointing
736 // to the associated ADRP patch label).
737 vixl::aarch64::Label* NewBootImageRelRoPatch(uint32_t boot_image_offset,
738 vixl::aarch64::Label* adrp_label = nullptr);
739
740 // Add a new boot image method patch for an instruction and return the label
741 // to be bound before the instruction. The instruction will be either the
742 // ADRP (pass `adrp_label = null`) or the ADD (pass `adrp_label` pointing
743 // to the associated ADRP patch label).
744 vixl::aarch64::Label* NewBootImageMethodPatch(MethodReference target_method,
745 vixl::aarch64::Label* adrp_label = nullptr);
746
747 // Add a new .bss entry method patch for an instruction and return
748 // the label to be bound before the instruction. The instruction will be
749 // either the ADRP (pass `adrp_label = null`) or the LDR (pass `adrp_label`
750 // pointing to the associated ADRP patch label).
751 vixl::aarch64::Label* NewMethodBssEntryPatch(MethodReference target_method,
752 vixl::aarch64::Label* adrp_label = nullptr);
753
754 // Add a new boot image type patch for an instruction and return the label
755 // to be bound before the instruction. The instruction will be either the
756 // ADRP (pass `adrp_label = null`) or the ADD (pass `adrp_label` pointing
757 // to the associated ADRP patch label).
758 vixl::aarch64::Label* NewBootImageTypePatch(const DexFile& dex_file,
759 dex::TypeIndex type_index,
760 vixl::aarch64::Label* adrp_label = nullptr);
761
762 // Add a new .bss entry type patch for an instruction and return the label
763 // to be bound before the instruction. The instruction will be either the
764 // ADRP (pass `adrp_label = null`) or the ADD (pass `adrp_label` pointing
765 // to the associated ADRP patch label).
766 vixl::aarch64::Label* NewBssEntryTypePatch(HLoadClass* load_class,
767 vixl::aarch64::Label* adrp_label = nullptr);
768
769 // Add a new boot image string patch for an instruction and return the label
770 // to be bound before the instruction. The instruction will be either the
771 // ADRP (pass `adrp_label = null`) or the ADD (pass `adrp_label` pointing
772 // to the associated ADRP patch label).
773 vixl::aarch64::Label* NewBootImageStringPatch(const DexFile& dex_file,
774 dex::StringIndex string_index,
775 vixl::aarch64::Label* adrp_label = nullptr);
776
777 // Add a new .bss entry string patch for an instruction and return the label
778 // to be bound before the instruction. The instruction will be either the
779 // ADRP (pass `adrp_label = null`) or the ADD (pass `adrp_label` pointing
780 // to the associated ADRP patch label).
781 vixl::aarch64::Label* NewStringBssEntryPatch(const DexFile& dex_file,
782 dex::StringIndex string_index,
783 vixl::aarch64::Label* adrp_label = nullptr);
784
785 // Add a new boot image JNI entrypoint patch for an instruction and return the label
786 // to be bound before the instruction. The instruction will be either the
787 // ADRP (pass `adrp_label = null`) or the LDR (pass `adrp_label` pointing
788 // to the associated ADRP patch label).
789 vixl::aarch64::Label* NewBootImageJniEntrypointPatch(MethodReference target_method,
790 vixl::aarch64::Label* adrp_label = nullptr);
791
792 // Emit the BL instruction for entrypoint thunk call and record the associated patch for AOT.
793 void EmitEntrypointThunkCall(ThreadOffset64 entrypoint_offset);
794
795 // Emit the CBNZ instruction for baker read barrier and record
796 // the associated patch for AOT or slow path for JIT.
797 void EmitBakerReadBarrierCbnz(uint32_t custom_data);
798
799 vixl::aarch64::Literal<uint32_t>* DeduplicateBootImageAddressLiteral(uint64_t address);
800 vixl::aarch64::Literal<uint32_t>* DeduplicateJitStringLiteral(const DexFile& dex_file,
801 dex::StringIndex string_index,
802 Handle<mirror::String> handle);
803 vixl::aarch64::Literal<uint32_t>* DeduplicateJitClassLiteral(const DexFile& dex_file,
804 dex::TypeIndex string_index,
805 Handle<mirror::Class> handle);
806
807 void EmitAdrpPlaceholder(vixl::aarch64::Label* fixup_label, vixl::aarch64::Register reg);
808 void EmitAddPlaceholder(vixl::aarch64::Label* fixup_label,
809 vixl::aarch64::Register out,
810 vixl::aarch64::Register base);
811 void EmitLdrOffsetPlaceholder(vixl::aarch64::Label* fixup_label,
812 vixl::aarch64::Register out,
813 vixl::aarch64::Register base);
814
815 void LoadBootImageAddress(vixl::aarch64::Register reg, uint32_t boot_image_reference);
816 void LoadTypeForBootImageIntrinsic(vixl::aarch64::Register reg, TypeReference type_reference);
817 void LoadIntrinsicDeclaringClass(vixl::aarch64::Register reg, HInvoke* invoke);
818 void LoadClassRootForIntrinsic(vixl::aarch64::Register reg, ClassRoot class_root);
819
820 void EmitLinkerPatches(ArenaVector<linker::LinkerPatch>* linker_patches) override;
821 bool NeedsThunkCode(const linker::LinkerPatch& patch) const override;
822 void EmitThunkCode(const linker::LinkerPatch& patch,
823 /*out*/ ArenaVector<uint8_t>* code,
824 /*out*/ std::string* debug_name) override;
825
826 void EmitJitRootPatches(uint8_t* code, const uint8_t* roots_data) override;
827
828 // Generate a GC root reference load:
829 //
830 // root <- *(obj + offset)
831 //
832 // while honoring read barriers based on read_barrier_option.
833 void GenerateGcRootFieldLoad(HInstruction* instruction,
834 Location root,
835 vixl::aarch64::Register obj,
836 uint32_t offset,
837 vixl::aarch64::Label* fixup_label,
838 ReadBarrierOption read_barrier_option);
839 // Generate MOV for the `old_value` in intrinsic CAS and mark it with Baker read barrier.
840 void GenerateIntrinsicCasMoveWithBakerReadBarrier(vixl::aarch64::Register marked_old_value,
841 vixl::aarch64::Register old_value);
842 // Fast path implementation of ReadBarrier::Barrier for a heap
843 // reference field load when Baker's read barriers are used.
844 // Overload suitable for Unsafe.getObject/-Volatile() intrinsic.
845 void GenerateFieldLoadWithBakerReadBarrier(HInstruction* instruction,
846 Location ref,
847 vixl::aarch64::Register obj,
848 const vixl::aarch64::MemOperand& src,
849 bool needs_null_check,
850 bool use_load_acquire);
851 // Fast path implementation of ReadBarrier::Barrier for a heap
852 // reference field load when Baker's read barriers are used.
853 void GenerateFieldLoadWithBakerReadBarrier(HInstruction* instruction,
854 Location ref,
855 vixl::aarch64::Register obj,
856 uint32_t offset,
857 Location maybe_temp,
858 bool needs_null_check,
859 bool use_load_acquire);
860 // Fast path implementation of ReadBarrier::Barrier for a heap
861 // reference array load when Baker's read barriers are used.
862 void GenerateArrayLoadWithBakerReadBarrier(HArrayGet* instruction,
863 Location ref,
864 vixl::aarch64::Register obj,
865 uint32_t data_offset,
866 Location index,
867 bool needs_null_check);
868
869 // Emit code checking the status of the Marking Register, and
870 // aborting the program if MR does not match the value stored in the
871 // art::Thread object. Code is only emitted in debug mode and if
872 // CompilerOptions::EmitRunTimeChecksInDebugMode returns true.
873 //
874 // Argument `code` is used to identify the different occurrences of
875 // MaybeGenerateMarkingRegisterCheck in the code generator, and is
876 // passed to the BRK instruction.
877 //
878 // If `temp_loc` is a valid location, it is expected to be a
879 // register and will be used as a temporary to generate code;
880 // otherwise, a temporary will be fetched from the core register
881 // scratch pool.
882 virtual void MaybeGenerateMarkingRegisterCheck(int code,
883 Location temp_loc = Location::NoLocation());
884
885 // Create slow path for a read barrier for a heap reference within `instruction`.
886 //
887 // This is a helper function for GenerateReadBarrierSlow() that has the same
888 // arguments. The creation and adding of the slow path is exposed for intrinsics
889 // that cannot use GenerateReadBarrierSlow() from their own slow paths.
890 SlowPathCodeARM64* AddReadBarrierSlowPath(HInstruction* instruction,
891 Location out,
892 Location ref,
893 Location obj,
894 uint32_t offset,
895 Location index);
896
897 // Generate a read barrier for a heap reference within `instruction`
898 // using a slow path.
899 //
900 // A read barrier for an object reference read from the heap is
901 // implemented as a call to the artReadBarrierSlow runtime entry
902 // point, which is passed the values in locations `ref`, `obj`, and
903 // `offset`:
904 //
905 // mirror::Object* artReadBarrierSlow(mirror::Object* ref,
906 // mirror::Object* obj,
907 // uint32_t offset);
908 //
909 // The `out` location contains the value returned by
910 // artReadBarrierSlow.
911 //
912 // When `index` is provided (i.e. for array accesses), the offset
913 // value passed to artReadBarrierSlow is adjusted to take `index`
914 // into account.
915 void GenerateReadBarrierSlow(HInstruction* instruction,
916 Location out,
917 Location ref,
918 Location obj,
919 uint32_t offset,
920 Location index = Location::NoLocation());
921
922 // If read barriers are enabled, generate a read barrier for a heap
923 // reference using a slow path. If heap poisoning is enabled, also
924 // unpoison the reference in `out`.
925 void MaybeGenerateReadBarrierSlow(HInstruction* instruction,
926 Location out,
927 Location ref,
928 Location obj,
929 uint32_t offset,
930 Location index = Location::NoLocation());
931
932 // Generate a read barrier for a GC root within `instruction` using
933 // a slow path.
934 //
935 // A read barrier for an object reference GC root is implemented as
936 // a call to the artReadBarrierForRootSlow runtime entry point,
937 // which is passed the value in location `root`:
938 //
939 // mirror::Object* artReadBarrierForRootSlow(GcRoot<mirror::Object>* root);
940 //
941 // The `out` location contains the value returned by
942 // artReadBarrierForRootSlow.
943 void GenerateReadBarrierForRootSlow(HInstruction* instruction, Location out, Location root);
944
945 void IncreaseFrame(size_t adjustment) override;
946 void DecreaseFrame(size_t adjustment) override;
947
948 void GenerateNop() override;
949
950 void GenerateImplicitNullCheck(HNullCheck* instruction) override;
951 void GenerateExplicitNullCheck(HNullCheck* instruction) override;
952
MaybeRecordImplicitNullCheck(HInstruction * instr)953 void MaybeRecordImplicitNullCheck(HInstruction* instr) final {
954 // The function must be only called within special scopes
955 // (EmissionCheckScope, ExactAssemblyScope) which prevent generation of
956 // veneer/literal pools by VIXL assembler.
957 CHECK_EQ(GetVIXLAssembler()->ArePoolsBlocked(), true)
958 << "The function must only be called within EmissionCheckScope or ExactAssemblyScope";
959 CodeGenerator::MaybeRecordImplicitNullCheck(instr);
960 }
961
962 void MaybeGenerateInlineCacheCheck(HInstruction* instruction, vixl::aarch64::Register klass);
963 void MaybeIncrementHotness(bool is_frame_entry);
964
965 private:
966 // Encoding of thunk type and data for link-time generated thunks for Baker read barriers.
967
968 enum class BakerReadBarrierKind : uint8_t {
969 kField, // Field get or array get with constant offset (i.e. constant index).
970 kAcquire, // Volatile field get.
971 kArray, // Array get with index in register.
972 kGcRoot, // GC root load.
973 kLast = kGcRoot
974 };
975
976 static constexpr uint32_t kBakerReadBarrierInvalidEncodedReg = /* sp/zr is invalid */ 31u;
977
978 static constexpr size_t kBitsForBakerReadBarrierKind =
979 MinimumBitsToStore(static_cast<size_t>(BakerReadBarrierKind::kLast));
980 static constexpr size_t kBakerReadBarrierBitsForRegister =
981 MinimumBitsToStore(kBakerReadBarrierInvalidEncodedReg);
982 using BakerReadBarrierKindField =
983 BitField<BakerReadBarrierKind, 0, kBitsForBakerReadBarrierKind>;
984 using BakerReadBarrierFirstRegField =
985 BitField<uint32_t, kBitsForBakerReadBarrierKind, kBakerReadBarrierBitsForRegister>;
986 using BakerReadBarrierSecondRegField =
987 BitField<uint32_t,
988 kBitsForBakerReadBarrierKind + kBakerReadBarrierBitsForRegister,
989 kBakerReadBarrierBitsForRegister>;
990
CheckValidReg(uint32_t reg)991 static void CheckValidReg(uint32_t reg) {
992 DCHECK(reg < vixl::aarch64::lr.GetCode() &&
993 reg != vixl::aarch64::ip0.GetCode() &&
994 reg != vixl::aarch64::ip1.GetCode()) << reg;
995 }
996
EncodeBakerReadBarrierFieldData(uint32_t base_reg,uint32_t holder_reg)997 static inline uint32_t EncodeBakerReadBarrierFieldData(uint32_t base_reg, uint32_t holder_reg) {
998 CheckValidReg(base_reg);
999 CheckValidReg(holder_reg);
1000 return BakerReadBarrierKindField::Encode(BakerReadBarrierKind::kField) |
1001 BakerReadBarrierFirstRegField::Encode(base_reg) |
1002 BakerReadBarrierSecondRegField::Encode(holder_reg);
1003 }
1004
EncodeBakerReadBarrierAcquireData(uint32_t base_reg,uint32_t holder_reg)1005 static inline uint32_t EncodeBakerReadBarrierAcquireData(uint32_t base_reg, uint32_t holder_reg) {
1006 CheckValidReg(base_reg);
1007 CheckValidReg(holder_reg);
1008 DCHECK_NE(base_reg, holder_reg);
1009 return BakerReadBarrierKindField::Encode(BakerReadBarrierKind::kAcquire) |
1010 BakerReadBarrierFirstRegField::Encode(base_reg) |
1011 BakerReadBarrierSecondRegField::Encode(holder_reg);
1012 }
1013
EncodeBakerReadBarrierArrayData(uint32_t base_reg)1014 static inline uint32_t EncodeBakerReadBarrierArrayData(uint32_t base_reg) {
1015 CheckValidReg(base_reg);
1016 return BakerReadBarrierKindField::Encode(BakerReadBarrierKind::kArray) |
1017 BakerReadBarrierFirstRegField::Encode(base_reg) |
1018 BakerReadBarrierSecondRegField::Encode(kBakerReadBarrierInvalidEncodedReg);
1019 }
1020
EncodeBakerReadBarrierGcRootData(uint32_t root_reg)1021 static inline uint32_t EncodeBakerReadBarrierGcRootData(uint32_t root_reg) {
1022 CheckValidReg(root_reg);
1023 return BakerReadBarrierKindField::Encode(BakerReadBarrierKind::kGcRoot) |
1024 BakerReadBarrierFirstRegField::Encode(root_reg) |
1025 BakerReadBarrierSecondRegField::Encode(kBakerReadBarrierInvalidEncodedReg);
1026 }
1027
1028 void CompileBakerReadBarrierThunk(Arm64Assembler& assembler,
1029 uint32_t encoded_data,
1030 /*out*/ std::string* debug_name);
1031
1032 using Uint64ToLiteralMap = ArenaSafeMap<uint64_t, vixl::aarch64::Literal<uint64_t>*>;
1033 using Uint32ToLiteralMap = ArenaSafeMap<uint32_t, vixl::aarch64::Literal<uint32_t>*>;
1034 using StringToLiteralMap = ArenaSafeMap<StringReference,
1035 vixl::aarch64::Literal<uint32_t>*,
1036 StringReferenceValueComparator>;
1037 using TypeToLiteralMap = ArenaSafeMap<TypeReference,
1038 vixl::aarch64::Literal<uint32_t>*,
1039 TypeReferenceValueComparator>;
1040
1041 vixl::aarch64::Literal<uint32_t>* DeduplicateUint32Literal(uint32_t value);
1042 vixl::aarch64::Literal<uint64_t>* DeduplicateUint64Literal(uint64_t value);
1043
1044 // The PcRelativePatchInfo is used for PC-relative addressing of methods/strings/types,
1045 // whether through .data.bimg.rel.ro, .bss, or directly in the boot image.
1046 struct PcRelativePatchInfo : PatchInfo<vixl::aarch64::Label> {
PcRelativePatchInfoPcRelativePatchInfo1047 PcRelativePatchInfo(const DexFile* dex_file, uint32_t off_or_idx)
1048 : PatchInfo<vixl::aarch64::Label>(dex_file, off_or_idx), pc_insn_label() { }
1049
1050 vixl::aarch64::Label* pc_insn_label;
1051 };
1052
1053 struct BakerReadBarrierPatchInfo {
BakerReadBarrierPatchInfoBakerReadBarrierPatchInfo1054 explicit BakerReadBarrierPatchInfo(uint32_t data) : label(), custom_data(data) { }
1055
1056 vixl::aarch64::Label label;
1057 uint32_t custom_data;
1058 };
1059
1060 vixl::aarch64::Label* NewPcRelativePatch(const DexFile* dex_file,
1061 uint32_t offset_or_index,
1062 vixl::aarch64::Label* adrp_label,
1063 ArenaDeque<PcRelativePatchInfo>* patches);
1064
1065 void EmitJumpTables();
1066
1067 template <linker::LinkerPatch (*Factory)(size_t, const DexFile*, uint32_t, uint32_t)>
1068 static void EmitPcRelativeLinkerPatches(const ArenaDeque<PcRelativePatchInfo>& infos,
1069 ArenaVector<linker::LinkerPatch>* linker_patches);
1070
1071 // Returns whether SVE features are supported and should be used.
1072 bool ShouldUseSVE() const;
1073
1074 // Labels for each block that will be compiled.
1075 // We use a deque so that the `vixl::aarch64::Label` objects do not move in memory.
1076 ArenaDeque<vixl::aarch64::Label> block_labels_; // Indexed by block id.
1077 vixl::aarch64::Label frame_entry_label_;
1078 ArenaVector<std::unique_ptr<JumpTableARM64>> jump_tables_;
1079
1080 LocationsBuilderARM64Neon location_builder_neon_;
1081 InstructionCodeGeneratorARM64Neon instruction_visitor_neon_;
1082 LocationsBuilderARM64Sve location_builder_sve_;
1083 InstructionCodeGeneratorARM64Sve instruction_visitor_sve_;
1084
1085 LocationsBuilderARM64* location_builder_;
1086 InstructionCodeGeneratorARM64* instruction_visitor_;
1087 ParallelMoveResolverARM64 move_resolver_;
1088 Arm64Assembler assembler_;
1089
1090 // PC-relative method patch info for kBootImageLinkTimePcRelative.
1091 ArenaDeque<PcRelativePatchInfo> boot_image_method_patches_;
1092 // PC-relative method patch info for kBssEntry.
1093 ArenaDeque<PcRelativePatchInfo> method_bss_entry_patches_;
1094 // PC-relative type patch info for kBootImageLinkTimePcRelative.
1095 ArenaDeque<PcRelativePatchInfo> boot_image_type_patches_;
1096 // PC-relative type patch info for kBssEntry.
1097 ArenaDeque<PcRelativePatchInfo> type_bss_entry_patches_;
1098 // PC-relative public type patch info for kBssEntryPublic.
1099 ArenaDeque<PcRelativePatchInfo> public_type_bss_entry_patches_;
1100 // PC-relative package type patch info for kBssEntryPackage.
1101 ArenaDeque<PcRelativePatchInfo> package_type_bss_entry_patches_;
1102 // PC-relative String patch info for kBootImageLinkTimePcRelative.
1103 ArenaDeque<PcRelativePatchInfo> boot_image_string_patches_;
1104 // PC-relative String patch info for kBssEntry.
1105 ArenaDeque<PcRelativePatchInfo> string_bss_entry_patches_;
1106 // PC-relative method patch info for kBootImageLinkTimePcRelative+kCallCriticalNative.
1107 ArenaDeque<PcRelativePatchInfo> boot_image_jni_entrypoint_patches_;
1108 // PC-relative patch info for IntrinsicObjects for the boot image,
1109 // and for method/type/string patches for kBootImageRelRo otherwise.
1110 ArenaDeque<PcRelativePatchInfo> boot_image_other_patches_;
1111 // Patch info for calls to entrypoint dispatch thunks. Used for slow paths.
1112 ArenaDeque<PatchInfo<vixl::aarch64::Label>> call_entrypoint_patches_;
1113 // Baker read barrier patch info.
1114 ArenaDeque<BakerReadBarrierPatchInfo> baker_read_barrier_patches_;
1115
1116 // Deduplication map for 32-bit literals, used for JIT for boot image addresses.
1117 Uint32ToLiteralMap uint32_literals_;
1118 // Deduplication map for 64-bit literals, used for JIT for method address or method code.
1119 Uint64ToLiteralMap uint64_literals_;
1120 // Patches for string literals in JIT compiled code.
1121 StringToLiteralMap jit_string_patches_;
1122 // Patches for class literals in JIT compiled code.
1123 TypeToLiteralMap jit_class_patches_;
1124
1125 // Baker read barrier slow paths, mapping custom data (uint32_t) to label.
1126 // Wrap the label to work around vixl::aarch64::Label being non-copyable
1127 // and non-moveable and as such unusable in ArenaSafeMap<>.
1128 struct LabelWrapper {
LabelWrapperLabelWrapper1129 LabelWrapper(const LabelWrapper& src)
1130 : label() {
1131 DCHECK(!src.label.IsLinked() && !src.label.IsBound());
1132 }
1133 LabelWrapper() = default;
1134 vixl::aarch64::Label label;
1135 };
1136 ArenaSafeMap<uint32_t, LabelWrapper> jit_baker_read_barrier_slow_paths_;
1137
1138 friend class linker::Arm64RelativePatcherTest;
1139 DISALLOW_COPY_AND_ASSIGN(CodeGeneratorARM64);
1140 };
1141
GetAssembler()1142 inline Arm64Assembler* ParallelMoveResolverARM64::GetAssembler() const {
1143 return codegen_->GetAssembler();
1144 }
1145
1146 } // namespace arm64
1147 } // namespace art
1148
1149 #endif // ART_COMPILER_OPTIMIZING_CODE_GENERATOR_ARM64_H_
1150