1 /*
2  * Copyright (C) 2014 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 #ifndef ART_COMPILER_OPTIMIZING_CODE_GENERATOR_ARM64_H_
18 #define ART_COMPILER_OPTIMIZING_CODE_GENERATOR_ARM64_H_
19 
20 #include "base/bit_field.h"
21 #include "class_root.h"
22 #include "code_generator.h"
23 #include "common_arm64.h"
24 #include "dex/dex_file_types.h"
25 #include "dex/string_reference.h"
26 #include "dex/type_reference.h"
27 #include "driver/compiler_options.h"
28 #include "nodes.h"
29 #include "parallel_move_resolver.h"
30 #include "utils/arm64/assembler_arm64.h"
31 
32 // TODO(VIXL): Make VIXL compile with -Wshadow.
33 #pragma GCC diagnostic push
34 #pragma GCC diagnostic ignored "-Wshadow"
35 #include "aarch64/disasm-aarch64.h"
36 #include "aarch64/macro-assembler-aarch64.h"
37 #pragma GCC diagnostic pop
38 
39 namespace art {
40 
41 namespace linker {
42 class Arm64RelativePatcherTest;
43 }  // namespace linker
44 
45 namespace arm64 {
46 
47 class CodeGeneratorARM64;
48 
49 // Use a local definition to prevent copying mistakes.
50 static constexpr size_t kArm64WordSize = static_cast<size_t>(kArm64PointerSize);
51 
52 // These constants are used as an approximate margin when emission of veneer and literal pools
53 // must be blocked.
54 static constexpr int kMaxMacroInstructionSizeInBytes = 15 * vixl::aarch64::kInstructionSize;
55 static constexpr int kInvokeCodeMarginSizeInBytes = 6 * kMaxMacroInstructionSizeInBytes;
56 
57 static const vixl::aarch64::Register kParameterCoreRegisters[] = {
58   vixl::aarch64::x1,
59   vixl::aarch64::x2,
60   vixl::aarch64::x3,
61   vixl::aarch64::x4,
62   vixl::aarch64::x5,
63   vixl::aarch64::x6,
64   vixl::aarch64::x7
65 };
66 static constexpr size_t kParameterCoreRegistersLength = arraysize(kParameterCoreRegisters);
67 static const vixl::aarch64::VRegister kParameterFPRegisters[] = {
68   vixl::aarch64::d0,
69   vixl::aarch64::d1,
70   vixl::aarch64::d2,
71   vixl::aarch64::d3,
72   vixl::aarch64::d4,
73   vixl::aarch64::d5,
74   vixl::aarch64::d6,
75   vixl::aarch64::d7
76 };
77 static constexpr size_t kParameterFPRegistersLength = arraysize(kParameterFPRegisters);
78 
79 // Thread Register.
80 const vixl::aarch64::Register tr = vixl::aarch64::x19;
81 // Marking Register.
82 const vixl::aarch64::Register mr = vixl::aarch64::x20;
83 // Method register on invoke.
84 static const vixl::aarch64::Register kArtMethodRegister = vixl::aarch64::x0;
85 const vixl::aarch64::CPURegList vixl_reserved_core_registers(vixl::aarch64::ip0,
86                                                              vixl::aarch64::ip1);
87 const vixl::aarch64::CPURegList vixl_reserved_fp_registers(vixl::aarch64::d31);
88 
89 const vixl::aarch64::CPURegList runtime_reserved_core_registers =
90     vixl::aarch64::CPURegList(
91         tr,
92         // Reserve X20 as Marking Register when emitting Baker read barriers.
93         ((kEmitCompilerReadBarrier && kUseBakerReadBarrier) ? mr : vixl::aarch64::NoCPUReg),
94         vixl::aarch64::lr);
95 
96 // Some instructions have special requirements for a temporary, for example
97 // LoadClass/kBssEntry and LoadString/kBssEntry for Baker read barrier require
98 // temp that's not an R0 (to avoid an extra move) and Baker read barrier field
99 // loads with large offsets need a fixed register to limit the number of link-time
100 // thunks we generate. For these and similar cases, we want to reserve a specific
101 // register that's neither callee-save nor an argument register. We choose x15.
FixedTempLocation()102 inline Location FixedTempLocation() {
103   return Location::RegisterLocation(vixl::aarch64::x15.GetCode());
104 }
105 
106 // Callee-save registers AAPCS64, without x19 (Thread Register) (nor
107 // x20 (Marking Register) when emitting Baker read barriers).
108 const vixl::aarch64::CPURegList callee_saved_core_registers(
109     vixl::aarch64::CPURegister::kRegister,
110     vixl::aarch64::kXRegSize,
111     ((kEmitCompilerReadBarrier && kUseBakerReadBarrier)
112          ? vixl::aarch64::x21.GetCode()
113          : vixl::aarch64::x20.GetCode()),
114      vixl::aarch64::x30.GetCode());
115 const vixl::aarch64::CPURegList callee_saved_fp_registers(vixl::aarch64::CPURegister::kVRegister,
116                                                           vixl::aarch64::kDRegSize,
117                                                           vixl::aarch64::d8.GetCode(),
118                                                           vixl::aarch64::d15.GetCode());
119 Location ARM64ReturnLocation(DataType::Type return_type);
120 
121 class SlowPathCodeARM64 : public SlowPathCode {
122  public:
SlowPathCodeARM64(HInstruction * instruction)123   explicit SlowPathCodeARM64(HInstruction* instruction)
124       : SlowPathCode(instruction), entry_label_(), exit_label_() {}
125 
GetEntryLabel()126   vixl::aarch64::Label* GetEntryLabel() { return &entry_label_; }
GetExitLabel()127   vixl::aarch64::Label* GetExitLabel() { return &exit_label_; }
128 
129   void SaveLiveRegisters(CodeGenerator* codegen, LocationSummary* locations) override;
130   void RestoreLiveRegisters(CodeGenerator* codegen, LocationSummary* locations) override;
131 
132  private:
133   vixl::aarch64::Label entry_label_;
134   vixl::aarch64::Label exit_label_;
135 
136   DISALLOW_COPY_AND_ASSIGN(SlowPathCodeARM64);
137 };
138 
139 class JumpTableARM64 : public DeletableArenaObject<kArenaAllocSwitchTable> {
140  public:
JumpTableARM64(HPackedSwitch * switch_instr)141   explicit JumpTableARM64(HPackedSwitch* switch_instr)
142     : switch_instr_(switch_instr), table_start_() {}
143 
GetTableStartLabel()144   vixl::aarch64::Label* GetTableStartLabel() { return &table_start_; }
145 
146   void EmitTable(CodeGeneratorARM64* codegen);
147 
148  private:
149   HPackedSwitch* const switch_instr_;
150   vixl::aarch64::Label table_start_;
151 
152   DISALLOW_COPY_AND_ASSIGN(JumpTableARM64);
153 };
154 
155 static const vixl::aarch64::Register kRuntimeParameterCoreRegisters[] =
156     { vixl::aarch64::x0,
157       vixl::aarch64::x1,
158       vixl::aarch64::x2,
159       vixl::aarch64::x3,
160       vixl::aarch64::x4,
161       vixl::aarch64::x5,
162       vixl::aarch64::x6,
163       vixl::aarch64::x7 };
164 static constexpr size_t kRuntimeParameterCoreRegistersLength =
165     arraysize(kRuntimeParameterCoreRegisters);
166 static const vixl::aarch64::VRegister kRuntimeParameterFpuRegisters[] =
167     { vixl::aarch64::d0,
168       vixl::aarch64::d1,
169       vixl::aarch64::d2,
170       vixl::aarch64::d3,
171       vixl::aarch64::d4,
172       vixl::aarch64::d5,
173       vixl::aarch64::d6,
174       vixl::aarch64::d7 };
175 static constexpr size_t kRuntimeParameterFpuRegistersLength =
176     arraysize(kRuntimeParameterCoreRegisters);
177 
178 class InvokeRuntimeCallingConvention : public CallingConvention<vixl::aarch64::Register,
179                                                                 vixl::aarch64::VRegister> {
180  public:
181   static constexpr size_t kParameterCoreRegistersLength = arraysize(kParameterCoreRegisters);
182 
InvokeRuntimeCallingConvention()183   InvokeRuntimeCallingConvention()
184       : CallingConvention(kRuntimeParameterCoreRegisters,
185                           kRuntimeParameterCoreRegistersLength,
186                           kRuntimeParameterFpuRegisters,
187                           kRuntimeParameterFpuRegistersLength,
188                           kArm64PointerSize) {}
189 
190   Location GetReturnLocation(DataType::Type return_type);
191 
192  private:
193   DISALLOW_COPY_AND_ASSIGN(InvokeRuntimeCallingConvention);
194 };
195 
196 class InvokeDexCallingConvention : public CallingConvention<vixl::aarch64::Register,
197                                                             vixl::aarch64::VRegister> {
198  public:
InvokeDexCallingConvention()199   InvokeDexCallingConvention()
200       : CallingConvention(kParameterCoreRegisters,
201                           kParameterCoreRegistersLength,
202                           kParameterFPRegisters,
203                           kParameterFPRegistersLength,
204                           kArm64PointerSize) {}
205 
GetReturnLocation(DataType::Type return_type)206   Location GetReturnLocation(DataType::Type return_type) const {
207     return ARM64ReturnLocation(return_type);
208   }
209 
210 
211  private:
212   DISALLOW_COPY_AND_ASSIGN(InvokeDexCallingConvention);
213 };
214 
215 class InvokeDexCallingConventionVisitorARM64 : public InvokeDexCallingConventionVisitor {
216  public:
InvokeDexCallingConventionVisitorARM64()217   InvokeDexCallingConventionVisitorARM64() {}
~InvokeDexCallingConventionVisitorARM64()218   virtual ~InvokeDexCallingConventionVisitorARM64() {}
219 
220   Location GetNextLocation(DataType::Type type) override;
GetReturnLocation(DataType::Type return_type)221   Location GetReturnLocation(DataType::Type return_type) const override {
222     return calling_convention.GetReturnLocation(return_type);
223   }
224   Location GetMethodLocation() const override;
225 
226  private:
227   InvokeDexCallingConvention calling_convention;
228 
229   DISALLOW_COPY_AND_ASSIGN(InvokeDexCallingConventionVisitorARM64);
230 };
231 
232 class CriticalNativeCallingConventionVisitorARM64 : public InvokeDexCallingConventionVisitor {
233  public:
CriticalNativeCallingConventionVisitorARM64(bool for_register_allocation)234   explicit CriticalNativeCallingConventionVisitorARM64(bool for_register_allocation)
235       : for_register_allocation_(for_register_allocation) {}
236 
~CriticalNativeCallingConventionVisitorARM64()237   virtual ~CriticalNativeCallingConventionVisitorARM64() {}
238 
239   Location GetNextLocation(DataType::Type type) override;
240   Location GetReturnLocation(DataType::Type type) const override;
241   Location GetMethodLocation() const override;
242 
GetStackOffset()243   size_t GetStackOffset() const { return stack_offset_; }
244 
245  private:
246   // Register allocator does not support adjusting frame size, so we cannot provide final locations
247   // of stack arguments for register allocation. We ask the register allocator for any location and
248   // move these arguments to the right place after adjusting the SP when generating the call.
249   const bool for_register_allocation_;
250   size_t gpr_index_ = 0u;
251   size_t fpr_index_ = 0u;
252   size_t stack_offset_ = 0u;
253 
254   DISALLOW_COPY_AND_ASSIGN(CriticalNativeCallingConventionVisitorARM64);
255 };
256 
257 class FieldAccessCallingConventionARM64 : public FieldAccessCallingConvention {
258  public:
FieldAccessCallingConventionARM64()259   FieldAccessCallingConventionARM64() {}
260 
GetObjectLocation()261   Location GetObjectLocation() const override {
262     return helpers::LocationFrom(vixl::aarch64::x1);
263   }
GetFieldIndexLocation()264   Location GetFieldIndexLocation() const override {
265     return helpers::LocationFrom(vixl::aarch64::x0);
266   }
GetReturnLocation(DataType::Type type ATTRIBUTE_UNUSED)267   Location GetReturnLocation(DataType::Type type ATTRIBUTE_UNUSED) const override {
268     return helpers::LocationFrom(vixl::aarch64::x0);
269   }
GetSetValueLocation(DataType::Type type ATTRIBUTE_UNUSED,bool is_instance)270   Location GetSetValueLocation(DataType::Type type ATTRIBUTE_UNUSED,
271                                bool is_instance) const override {
272     return is_instance
273         ? helpers::LocationFrom(vixl::aarch64::x2)
274         : helpers::LocationFrom(vixl::aarch64::x1);
275   }
GetFpuLocation(DataType::Type type ATTRIBUTE_UNUSED)276   Location GetFpuLocation(DataType::Type type ATTRIBUTE_UNUSED) const override {
277     return helpers::LocationFrom(vixl::aarch64::d0);
278   }
279 
280  private:
281   DISALLOW_COPY_AND_ASSIGN(FieldAccessCallingConventionARM64);
282 };
283 
284 class InstructionCodeGeneratorARM64 : public InstructionCodeGenerator {
285  public:
286   InstructionCodeGeneratorARM64(HGraph* graph, CodeGeneratorARM64* codegen);
287 
288 #define DECLARE_VISIT_INSTRUCTION(name, super) \
289   void Visit##name(H##name* instr) override;
290 
291   FOR_EACH_CONCRETE_INSTRUCTION_SCALAR_COMMON(DECLARE_VISIT_INSTRUCTION)
FOR_EACH_CONCRETE_INSTRUCTION_ARM64(DECLARE_VISIT_INSTRUCTION)292   FOR_EACH_CONCRETE_INSTRUCTION_ARM64(DECLARE_VISIT_INSTRUCTION)
293   FOR_EACH_CONCRETE_INSTRUCTION_SHARED(DECLARE_VISIT_INSTRUCTION)
294 
295 #undef DECLARE_VISIT_INSTRUCTION
296 
297   void VisitInstruction(HInstruction* instruction) override {
298     LOG(FATAL) << "Unreachable instruction " << instruction->DebugName()
299                << " (id " << instruction->GetId() << ")";
300   }
301 
GetAssembler()302   Arm64Assembler* GetAssembler() const { return assembler_; }
GetVIXLAssembler()303   vixl::aarch64::MacroAssembler* GetVIXLAssembler() { return GetAssembler()->GetVIXLAssembler(); }
304 
305   // SIMD helpers.
306   virtual Location AllocateSIMDScratchLocation(vixl::aarch64::UseScratchRegisterScope* scope) = 0;
307   virtual void FreeSIMDScratchLocation(Location loc,
308                                        vixl::aarch64::UseScratchRegisterScope* scope)  = 0;
309   virtual void LoadSIMDRegFromStack(Location destination, Location source) = 0;
310   virtual void MoveSIMDRegToSIMDReg(Location destination, Location source) = 0;
311   virtual void MoveToSIMDStackSlot(Location destination, Location source) = 0;
312   virtual void SaveLiveRegistersHelper(LocationSummary* locations,
313                                        int64_t spill_offset) = 0;
314   virtual void RestoreLiveRegistersHelper(LocationSummary* locations,
315                                           int64_t spill_offset) = 0;
316 
317  protected:
318   void GenerateClassInitializationCheck(SlowPathCodeARM64* slow_path,
319                                         vixl::aarch64::Register class_reg);
320   void GenerateBitstringTypeCheckCompare(HTypeCheckInstruction* check,
321                                          vixl::aarch64::Register temp);
322   void GenerateSuspendCheck(HSuspendCheck* instruction, HBasicBlock* successor);
323   void HandleBinaryOp(HBinaryOperation* instr);
324 
325   void HandleFieldSet(HInstruction* instruction,
326                       const FieldInfo& field_info,
327                       bool value_can_be_null);
328   void HandleFieldGet(HInstruction* instruction, const FieldInfo& field_info);
329   void HandleCondition(HCondition* instruction);
330 
331   // Generate a heap reference load using one register `out`:
332   //
333   //   out <- *(out + offset)
334   //
335   // while honoring heap poisoning and/or read barriers (if any).
336   //
337   // Location `maybe_temp` is used when generating a read barrier and
338   // shall be a register in that case; it may be an invalid location
339   // otherwise.
340   void GenerateReferenceLoadOneRegister(HInstruction* instruction,
341                                         Location out,
342                                         uint32_t offset,
343                                         Location maybe_temp,
344                                         ReadBarrierOption read_barrier_option);
345   // Generate a heap reference load using two different registers
346   // `out` and `obj`:
347   //
348   //   out <- *(obj + offset)
349   //
350   // while honoring heap poisoning and/or read barriers (if any).
351   //
352   // Location `maybe_temp` is used when generating a Baker's (fast
353   // path) read barrier and shall be a register in that case; it may
354   // be an invalid location otherwise.
355   void GenerateReferenceLoadTwoRegisters(HInstruction* instruction,
356                                          Location out,
357                                          Location obj,
358                                          uint32_t offset,
359                                          Location maybe_temp,
360                                          ReadBarrierOption read_barrier_option);
361 
362   // Generate a floating-point comparison.
363   void GenerateFcmp(HInstruction* instruction);
364 
365   void HandleShift(HBinaryOperation* instr);
366   void GenerateTestAndBranch(HInstruction* instruction,
367                              size_t condition_input_index,
368                              vixl::aarch64::Label* true_target,
369                              vixl::aarch64::Label* false_target);
370   void DivRemOneOrMinusOne(HBinaryOperation* instruction);
371   void DivRemByPowerOfTwo(HBinaryOperation* instruction);
372   void GenerateIncrementNegativeByOne(vixl::aarch64::Register out,
373                                       vixl::aarch64::Register in, bool use_cond_inc);
374   void GenerateResultRemWithAnyConstant(vixl::aarch64::Register out,
375                                         vixl::aarch64::Register dividend,
376                                         vixl::aarch64::Register quotient,
377                                         int64_t divisor,
378                                         // This function may acquire a scratch register.
379                                         vixl::aarch64::UseScratchRegisterScope* temps_scope);
380   void GenerateInt64UnsignedDivRemWithAnyPositiveConstant(HBinaryOperation* instruction);
381   void GenerateInt64DivRemWithAnyConstant(HBinaryOperation* instruction);
382   void GenerateInt32DivRemWithAnyConstant(HBinaryOperation* instruction);
383   void GenerateDivRemWithAnyConstant(HBinaryOperation* instruction, int64_t divisor);
384   void GenerateIntDiv(HDiv* instruction);
385   void GenerateIntDivForConstDenom(HDiv *instruction);
386   void GenerateIntDivForPower2Denom(HDiv *instruction);
387   void GenerateIntRem(HRem* instruction);
388   void GenerateIntRemForConstDenom(HRem *instruction);
389   void GenerateIntRemForPower2Denom(HRem *instruction);
390   void HandleGoto(HInstruction* got, HBasicBlock* successor);
391 
392   // Helpers to set up locations for vector memory operations. Returns the memory operand and,
393   // if used, sets the output parameter scratch to a temporary register used in this operand,
394   // so that the client can release it right after the memory operand use.
395   // Neon version.
396   vixl::aarch64::MemOperand VecNEONAddress(
397       HVecMemoryOperation* instruction,
398       // This function may acquire a scratch register.
399       vixl::aarch64::UseScratchRegisterScope* temps_scope,
400       size_t size,
401       bool is_string_char_at,
402       /*out*/ vixl::aarch64::Register* scratch);
403   // SVE version.
404   vixl::aarch64::SVEMemOperand VecSVEAddress(
405       HVecMemoryOperation* instruction,
406       // This function may acquire a scratch register.
407       vixl::aarch64::UseScratchRegisterScope* temps_scope,
408       size_t size,
409       bool is_string_char_at,
410       /*out*/ vixl::aarch64::Register* scratch);
411 
412   Arm64Assembler* const assembler_;
413   CodeGeneratorARM64* const codegen_;
414 
415   DISALLOW_COPY_AND_ASSIGN(InstructionCodeGeneratorARM64);
416 };
417 
418 class LocationsBuilderARM64 : public HGraphVisitor {
419  public:
LocationsBuilderARM64(HGraph * graph,CodeGeneratorARM64 * codegen)420   LocationsBuilderARM64(HGraph* graph, CodeGeneratorARM64* codegen)
421       : HGraphVisitor(graph), codegen_(codegen) {}
422 
423 #define DECLARE_VISIT_INSTRUCTION(name, super) \
424   void Visit##name(H##name* instr) override;
425 
426   FOR_EACH_CONCRETE_INSTRUCTION_SCALAR_COMMON(DECLARE_VISIT_INSTRUCTION)
FOR_EACH_CONCRETE_INSTRUCTION_ARM64(DECLARE_VISIT_INSTRUCTION)427   FOR_EACH_CONCRETE_INSTRUCTION_ARM64(DECLARE_VISIT_INSTRUCTION)
428   FOR_EACH_CONCRETE_INSTRUCTION_SHARED(DECLARE_VISIT_INSTRUCTION)
429 
430 #undef DECLARE_VISIT_INSTRUCTION
431 
432   void VisitInstruction(HInstruction* instruction) override {
433     LOG(FATAL) << "Unreachable instruction " << instruction->DebugName()
434                << " (id " << instruction->GetId() << ")";
435   }
436 
437  protected:
438   void HandleBinaryOp(HBinaryOperation* instr);
439   void HandleFieldSet(HInstruction* instruction);
440   void HandleFieldGet(HInstruction* instruction, const FieldInfo& field_info);
441   void HandleInvoke(HInvoke* instr);
442   void HandleCondition(HCondition* instruction);
443   void HandleShift(HBinaryOperation* instr);
444 
445   CodeGeneratorARM64* const codegen_;
446   InvokeDexCallingConventionVisitorARM64 parameter_visitor_;
447 
448   DISALLOW_COPY_AND_ASSIGN(LocationsBuilderARM64);
449 };
450 
451 class InstructionCodeGeneratorARM64Neon : public InstructionCodeGeneratorARM64 {
452  public:
InstructionCodeGeneratorARM64Neon(HGraph * graph,CodeGeneratorARM64 * codegen)453   InstructionCodeGeneratorARM64Neon(HGraph* graph, CodeGeneratorARM64* codegen) :
454       InstructionCodeGeneratorARM64(graph, codegen) {}
455 
456 #define DECLARE_VISIT_INSTRUCTION(name, super) \
457   void Visit##name(H##name* instr) override;
458 
459   FOR_EACH_CONCRETE_INSTRUCTION_VECTOR_COMMON(DECLARE_VISIT_INSTRUCTION)
460 
461 #undef DECLARE_VISIT_INSTRUCTION
462 
463   Location AllocateSIMDScratchLocation(vixl::aarch64::UseScratchRegisterScope* scope) override;
464   void FreeSIMDScratchLocation(Location loc,
465                                vixl::aarch64::UseScratchRegisterScope* scope) override;
466   void LoadSIMDRegFromStack(Location destination, Location source) override;
467   void MoveSIMDRegToSIMDReg(Location destination, Location source) override;
468   void MoveToSIMDStackSlot(Location destination, Location source) override;
469   void SaveLiveRegistersHelper(LocationSummary* locations, int64_t spill_offset) override;
470   void RestoreLiveRegistersHelper(LocationSummary* locations, int64_t spill_offset) override;
471 };
472 
473 class LocationsBuilderARM64Neon : public LocationsBuilderARM64 {
474  public:
LocationsBuilderARM64Neon(HGraph * graph,CodeGeneratorARM64 * codegen)475   LocationsBuilderARM64Neon(HGraph* graph, CodeGeneratorARM64* codegen) :
476       LocationsBuilderARM64(graph, codegen) {}
477 
478 #define DECLARE_VISIT_INSTRUCTION(name, super) \
479   void Visit##name(H##name* instr) override;
480 
481   FOR_EACH_CONCRETE_INSTRUCTION_VECTOR_COMMON(DECLARE_VISIT_INSTRUCTION)
482 
483 #undef DECLARE_VISIT_INSTRUCTION
484 };
485 
486 class InstructionCodeGeneratorARM64Sve : public InstructionCodeGeneratorARM64 {
487  public:
InstructionCodeGeneratorARM64Sve(HGraph * graph,CodeGeneratorARM64 * codegen)488   InstructionCodeGeneratorARM64Sve(HGraph* graph, CodeGeneratorARM64* codegen) :
489       InstructionCodeGeneratorARM64(graph, codegen) {}
490 
491 #define DECLARE_VISIT_INSTRUCTION(name, super) \
492   void Visit##name(H##name* instr) override;
493 
494   FOR_EACH_CONCRETE_INSTRUCTION_VECTOR_COMMON(DECLARE_VISIT_INSTRUCTION)
495 
496 #undef DECLARE_VISIT_INSTRUCTION
497 
498   Location AllocateSIMDScratchLocation(vixl::aarch64::UseScratchRegisterScope* scope) override;
499   void FreeSIMDScratchLocation(Location loc,
500                                vixl::aarch64::UseScratchRegisterScope* scope) override;
501   void LoadSIMDRegFromStack(Location destination, Location source) override;
502   void MoveSIMDRegToSIMDReg(Location destination, Location source) override;
503   void MoveToSIMDStackSlot(Location destination, Location source) override;
504   void SaveLiveRegistersHelper(LocationSummary* locations, int64_t spill_offset) override;
505   void RestoreLiveRegistersHelper(LocationSummary* locations, int64_t spill_offset) override;
506 
507  private:
508   // Validate that instruction vector length and packed type are compliant with the SIMD
509   // register size (full SIMD register is used).
510   void ValidateVectorLength(HVecOperation* instr) const;
511 
512   // Returns default predicate register which is used as governing vector predicate
513   // to implement predicated loop execution.
514   //
515   // TODO: This is a hack to be addressed when register allocator supports SIMD types.
LoopPReg()516   static vixl::aarch64::PRegister LoopPReg() {
517     return vixl::aarch64::p0;
518   }
519 };
520 
521 class LocationsBuilderARM64Sve : public LocationsBuilderARM64 {
522  public:
LocationsBuilderARM64Sve(HGraph * graph,CodeGeneratorARM64 * codegen)523   LocationsBuilderARM64Sve(HGraph* graph, CodeGeneratorARM64* codegen) :
524       LocationsBuilderARM64(graph, codegen) {}
525 
526 #define DECLARE_VISIT_INSTRUCTION(name, super) \
527   void Visit##name(H##name* instr) override;
528 
529   FOR_EACH_CONCRETE_INSTRUCTION_VECTOR_COMMON(DECLARE_VISIT_INSTRUCTION)
530 
531 #undef DECLARE_VISIT_INSTRUCTION
532 };
533 
534 class ParallelMoveResolverARM64 : public ParallelMoveResolverNoSwap {
535  public:
ParallelMoveResolverARM64(ArenaAllocator * allocator,CodeGeneratorARM64 * codegen)536   ParallelMoveResolverARM64(ArenaAllocator* allocator, CodeGeneratorARM64* codegen)
537       : ParallelMoveResolverNoSwap(allocator), codegen_(codegen), vixl_temps_() {}
538 
539  protected:
540   void PrepareForEmitNativeCode() override;
541   void FinishEmitNativeCode() override;
542   Location AllocateScratchLocationFor(Location::Kind kind) override;
543   void FreeScratchLocation(Location loc) override;
544   void EmitMove(size_t index) override;
545 
546  private:
547   Arm64Assembler* GetAssembler() const;
GetVIXLAssembler()548   vixl::aarch64::MacroAssembler* GetVIXLAssembler() const {
549     return GetAssembler()->GetVIXLAssembler();
550   }
551 
552   CodeGeneratorARM64* const codegen_;
553   vixl::aarch64::UseScratchRegisterScope vixl_temps_;
554 
555   DISALLOW_COPY_AND_ASSIGN(ParallelMoveResolverARM64);
556 };
557 
558 class CodeGeneratorARM64 : public CodeGenerator {
559  public:
560   CodeGeneratorARM64(HGraph* graph,
561                      const CompilerOptions& compiler_options,
562                      OptimizingCompilerStats* stats = nullptr);
~CodeGeneratorARM64()563   virtual ~CodeGeneratorARM64() {}
564 
565   void GenerateFrameEntry() override;
566   void GenerateFrameExit() override;
567 
568   vixl::aarch64::CPURegList GetFramePreservedCoreRegisters() const;
569   vixl::aarch64::CPURegList GetFramePreservedFPRegisters() const;
570 
571   void Bind(HBasicBlock* block) override;
572 
GetLabelOf(HBasicBlock * block)573   vixl::aarch64::Label* GetLabelOf(HBasicBlock* block) {
574     block = FirstNonEmptyBlock(block);
575     return &(block_labels_[block->GetBlockId()]);
576   }
577 
GetWordSize()578   size_t GetWordSize() const override {
579     return kArm64WordSize;
580   }
581 
SupportsPredicatedSIMD()582   bool SupportsPredicatedSIMD() const override { return ShouldUseSVE(); }
583 
GetSlowPathFPWidth()584   size_t GetSlowPathFPWidth() const override {
585     return GetGraph()->HasSIMD()
586         ? GetSIMDRegisterWidth()
587         : vixl::aarch64::kDRegSizeInBytes;
588   }
589 
GetCalleePreservedFPWidth()590   size_t GetCalleePreservedFPWidth() const override {
591     return vixl::aarch64::kDRegSizeInBytes;
592   }
593 
594   size_t GetSIMDRegisterWidth() const override;
595 
GetAddressOf(HBasicBlock * block)596   uintptr_t GetAddressOf(HBasicBlock* block) override {
597     vixl::aarch64::Label* block_entry_label = GetLabelOf(block);
598     DCHECK(block_entry_label->IsBound());
599     return block_entry_label->GetLocation();
600   }
601 
GetLocationBuilder()602   HGraphVisitor* GetLocationBuilder() override { return location_builder_; }
GetInstructionCodeGeneratorArm64()603   InstructionCodeGeneratorARM64* GetInstructionCodeGeneratorArm64() {
604     return instruction_visitor_;
605   }
GetInstructionVisitor()606   HGraphVisitor* GetInstructionVisitor() override { return GetInstructionCodeGeneratorArm64(); }
GetAssembler()607   Arm64Assembler* GetAssembler() override { return &assembler_; }
GetAssembler()608   const Arm64Assembler& GetAssembler() const override { return assembler_; }
GetVIXLAssembler()609   vixl::aarch64::MacroAssembler* GetVIXLAssembler() { return GetAssembler()->GetVIXLAssembler(); }
610 
611   // Emit a write barrier.
612   void MarkGCCard(vixl::aarch64::Register object,
613                   vixl::aarch64::Register value,
614                   bool value_can_be_null);
615 
616   void GenerateMemoryBarrier(MemBarrierKind kind);
617 
618   // Register allocation.
619 
620   void SetupBlockedRegisters() const override;
621 
622   size_t SaveCoreRegister(size_t stack_index, uint32_t reg_id) override;
623   size_t RestoreCoreRegister(size_t stack_index, uint32_t reg_id) override;
624   size_t SaveFloatingPointRegister(size_t stack_index, uint32_t reg_id) override;
625   size_t RestoreFloatingPointRegister(size_t stack_index, uint32_t reg_id) override;
626 
627   // The number of registers that can be allocated. The register allocator may
628   // decide to reserve and not use a few of them.
629   // We do not consider registers sp, xzr, wzr. They are either not allocatable
630   // (xzr, wzr), or make for poor allocatable registers (sp alignment
631   // requirements, etc.). This also facilitates our task as all other registers
632   // can easily be mapped via to or from their type and index or code.
633   static const int kNumberOfAllocatableRegisters = vixl::aarch64::kNumberOfRegisters - 1;
634   static const int kNumberOfAllocatableFPRegisters = vixl::aarch64::kNumberOfVRegisters;
635   static constexpr int kNumberOfAllocatableRegisterPairs = 0;
636 
637   void DumpCoreRegister(std::ostream& stream, int reg) const override;
638   void DumpFloatingPointRegister(std::ostream& stream, int reg) const override;
639 
GetInstructionSet()640   InstructionSet GetInstructionSet() const override {
641     return InstructionSet::kArm64;
642   }
643 
644   const Arm64InstructionSetFeatures& GetInstructionSetFeatures() const;
645 
Initialize()646   void Initialize() override {
647     block_labels_.resize(GetGraph()->GetBlocks().size());
648   }
649 
650   // We want to use the STP and LDP instructions to spill and restore registers for slow paths.
651   // These instructions can only encode offsets that are multiples of the register size accessed.
GetPreferredSlotsAlignment()652   uint32_t GetPreferredSlotsAlignment() const override { return vixl::aarch64::kXRegSizeInBytes; }
653 
CreateJumpTable(HPackedSwitch * switch_instr)654   JumpTableARM64* CreateJumpTable(HPackedSwitch* switch_instr) {
655     jump_tables_.emplace_back(new (GetGraph()->GetAllocator()) JumpTableARM64(switch_instr));
656     return jump_tables_.back().get();
657   }
658 
659   void Finalize(CodeAllocator* allocator) override;
660 
661   // Code generation helpers.
662   void MoveConstant(vixl::aarch64::CPURegister destination, HConstant* constant);
663   void MoveConstant(Location destination, int32_t value) override;
664   void MoveLocation(Location dst, Location src, DataType::Type dst_type) override;
665   void AddLocationAsTemp(Location location, LocationSummary* locations) override;
666 
667   void Load(DataType::Type type,
668             vixl::aarch64::CPURegister dst,
669             const vixl::aarch64::MemOperand& src);
670   void Store(DataType::Type type,
671              vixl::aarch64::CPURegister src,
672              const vixl::aarch64::MemOperand& dst);
673   void LoadAcquire(HInstruction* instruction,
674                    DataType::Type type,
675                    vixl::aarch64::CPURegister dst,
676                    const vixl::aarch64::MemOperand& src,
677                    bool needs_null_check);
678   void StoreRelease(HInstruction* instruction,
679                     DataType::Type type,
680                     vixl::aarch64::CPURegister src,
681                     const vixl::aarch64::MemOperand& dst,
682                     bool needs_null_check);
683 
684   // Generate code to invoke a runtime entry point.
685   void InvokeRuntime(QuickEntrypointEnum entrypoint,
686                      HInstruction* instruction,
687                      uint32_t dex_pc,
688                      SlowPathCode* slow_path = nullptr) override;
689 
690   // Generate code to invoke a runtime entry point, but do not record
691   // PC-related information in a stack map.
692   void InvokeRuntimeWithoutRecordingPcInfo(int32_t entry_point_offset,
693                                            HInstruction* instruction,
694                                            SlowPathCode* slow_path);
695 
GetMoveResolver()696   ParallelMoveResolverARM64* GetMoveResolver() override { return &move_resolver_; }
697 
NeedsTwoRegisters(DataType::Type type ATTRIBUTE_UNUSED)698   bool NeedsTwoRegisters(DataType::Type type ATTRIBUTE_UNUSED) const override {
699     return false;
700   }
701 
702   // Check if the desired_string_load_kind is supported. If it is, return it,
703   // otherwise return a fall-back kind that should be used instead.
704   HLoadString::LoadKind GetSupportedLoadStringKind(
705       HLoadString::LoadKind desired_string_load_kind) override;
706 
707   // Check if the desired_class_load_kind is supported. If it is, return it,
708   // otherwise return a fall-back kind that should be used instead.
709   HLoadClass::LoadKind GetSupportedLoadClassKind(
710       HLoadClass::LoadKind desired_class_load_kind) override;
711 
712   // Check if the desired_dispatch_info is supported. If it is, return it,
713   // otherwise return a fall-back info that should be used instead.
714   HInvokeStaticOrDirect::DispatchInfo GetSupportedInvokeStaticOrDirectDispatch(
715       const HInvokeStaticOrDirect::DispatchInfo& desired_dispatch_info,
716       ArtMethod* method) override;
717 
718   void LoadMethod(MethodLoadKind load_kind, Location temp, HInvoke* invoke);
719   void GenerateStaticOrDirectCall(
720       HInvokeStaticOrDirect* invoke, Location temp, SlowPathCode* slow_path = nullptr) override;
721   void GenerateVirtualCall(
722       HInvokeVirtual* invoke, Location temp, SlowPathCode* slow_path = nullptr) override;
723 
724   void MoveFromReturnRegister(Location trg, DataType::Type type) override;
725 
726   // Add a new boot image intrinsic patch for an instruction and return the label
727   // to be bound before the instruction. The instruction will be either the
728   // ADRP (pass `adrp_label = null`) or the ADD (pass `adrp_label` pointing
729   // to the associated ADRP patch label).
730   vixl::aarch64::Label* NewBootImageIntrinsicPatch(uint32_t intrinsic_data,
731                                                    vixl::aarch64::Label* adrp_label = nullptr);
732 
733   // Add a new boot image relocation patch for an instruction and return the label
734   // to be bound before the instruction. The instruction will be either the
735   // ADRP (pass `adrp_label = null`) or the LDR (pass `adrp_label` pointing
736   // to the associated ADRP patch label).
737   vixl::aarch64::Label* NewBootImageRelRoPatch(uint32_t boot_image_offset,
738                                                vixl::aarch64::Label* adrp_label = nullptr);
739 
740   // Add a new boot image method patch for an instruction and return the label
741   // to be bound before the instruction. The instruction will be either the
742   // ADRP (pass `adrp_label = null`) or the ADD (pass `adrp_label` pointing
743   // to the associated ADRP patch label).
744   vixl::aarch64::Label* NewBootImageMethodPatch(MethodReference target_method,
745                                                 vixl::aarch64::Label* adrp_label = nullptr);
746 
747   // Add a new .bss entry method patch for an instruction and return
748   // the label to be bound before the instruction. The instruction will be
749   // either the ADRP (pass `adrp_label = null`) or the LDR (pass `adrp_label`
750   // pointing to the associated ADRP patch label).
751   vixl::aarch64::Label* NewMethodBssEntryPatch(MethodReference target_method,
752                                                vixl::aarch64::Label* adrp_label = nullptr);
753 
754   // Add a new boot image type patch for an instruction and return the label
755   // to be bound before the instruction. The instruction will be either the
756   // ADRP (pass `adrp_label = null`) or the ADD (pass `adrp_label` pointing
757   // to the associated ADRP patch label).
758   vixl::aarch64::Label* NewBootImageTypePatch(const DexFile& dex_file,
759                                               dex::TypeIndex type_index,
760                                               vixl::aarch64::Label* adrp_label = nullptr);
761 
762   // Add a new .bss entry type patch for an instruction and return the label
763   // to be bound before the instruction. The instruction will be either the
764   // ADRP (pass `adrp_label = null`) or the ADD (pass `adrp_label` pointing
765   // to the associated ADRP patch label).
766   vixl::aarch64::Label* NewBssEntryTypePatch(HLoadClass* load_class,
767                                              vixl::aarch64::Label* adrp_label = nullptr);
768 
769   // Add a new boot image string patch for an instruction and return the label
770   // to be bound before the instruction. The instruction will be either the
771   // ADRP (pass `adrp_label = null`) or the ADD (pass `adrp_label` pointing
772   // to the associated ADRP patch label).
773   vixl::aarch64::Label* NewBootImageStringPatch(const DexFile& dex_file,
774                                                 dex::StringIndex string_index,
775                                                 vixl::aarch64::Label* adrp_label = nullptr);
776 
777   // Add a new .bss entry string patch for an instruction and return the label
778   // to be bound before the instruction. The instruction will be either the
779   // ADRP (pass `adrp_label = null`) or the ADD (pass `adrp_label` pointing
780   // to the associated ADRP patch label).
781   vixl::aarch64::Label* NewStringBssEntryPatch(const DexFile& dex_file,
782                                                dex::StringIndex string_index,
783                                                vixl::aarch64::Label* adrp_label = nullptr);
784 
785   // Add a new boot image JNI entrypoint patch for an instruction and return the label
786   // to be bound before the instruction. The instruction will be either the
787   // ADRP (pass `adrp_label = null`) or the LDR (pass `adrp_label` pointing
788   // to the associated ADRP patch label).
789   vixl::aarch64::Label* NewBootImageJniEntrypointPatch(MethodReference target_method,
790                                                        vixl::aarch64::Label* adrp_label = nullptr);
791 
792   // Emit the BL instruction for entrypoint thunk call and record the associated patch for AOT.
793   void EmitEntrypointThunkCall(ThreadOffset64 entrypoint_offset);
794 
795   // Emit the CBNZ instruction for baker read barrier and record
796   // the associated patch for AOT or slow path for JIT.
797   void EmitBakerReadBarrierCbnz(uint32_t custom_data);
798 
799   vixl::aarch64::Literal<uint32_t>* DeduplicateBootImageAddressLiteral(uint64_t address);
800   vixl::aarch64::Literal<uint32_t>* DeduplicateJitStringLiteral(const DexFile& dex_file,
801                                                                 dex::StringIndex string_index,
802                                                                 Handle<mirror::String> handle);
803   vixl::aarch64::Literal<uint32_t>* DeduplicateJitClassLiteral(const DexFile& dex_file,
804                                                                dex::TypeIndex string_index,
805                                                                Handle<mirror::Class> handle);
806 
807   void EmitAdrpPlaceholder(vixl::aarch64::Label* fixup_label, vixl::aarch64::Register reg);
808   void EmitAddPlaceholder(vixl::aarch64::Label* fixup_label,
809                           vixl::aarch64::Register out,
810                           vixl::aarch64::Register base);
811   void EmitLdrOffsetPlaceholder(vixl::aarch64::Label* fixup_label,
812                                 vixl::aarch64::Register out,
813                                 vixl::aarch64::Register base);
814 
815   void LoadBootImageAddress(vixl::aarch64::Register reg, uint32_t boot_image_reference);
816   void LoadTypeForBootImageIntrinsic(vixl::aarch64::Register reg, TypeReference type_reference);
817   void LoadIntrinsicDeclaringClass(vixl::aarch64::Register reg, HInvoke* invoke);
818   void LoadClassRootForIntrinsic(vixl::aarch64::Register reg, ClassRoot class_root);
819 
820   void EmitLinkerPatches(ArenaVector<linker::LinkerPatch>* linker_patches) override;
821   bool NeedsThunkCode(const linker::LinkerPatch& patch) const override;
822   void EmitThunkCode(const linker::LinkerPatch& patch,
823                      /*out*/ ArenaVector<uint8_t>* code,
824                      /*out*/ std::string* debug_name) override;
825 
826   void EmitJitRootPatches(uint8_t* code, const uint8_t* roots_data) override;
827 
828   // Generate a GC root reference load:
829   //
830   //   root <- *(obj + offset)
831   //
832   // while honoring read barriers based on read_barrier_option.
833   void GenerateGcRootFieldLoad(HInstruction* instruction,
834                                Location root,
835                                vixl::aarch64::Register obj,
836                                uint32_t offset,
837                                vixl::aarch64::Label* fixup_label,
838                                ReadBarrierOption read_barrier_option);
839   // Generate MOV for the `old_value` in intrinsic CAS and mark it with Baker read barrier.
840   void GenerateIntrinsicCasMoveWithBakerReadBarrier(vixl::aarch64::Register marked_old_value,
841                                                     vixl::aarch64::Register old_value);
842   // Fast path implementation of ReadBarrier::Barrier for a heap
843   // reference field load when Baker's read barriers are used.
844   // Overload suitable for Unsafe.getObject/-Volatile() intrinsic.
845   void GenerateFieldLoadWithBakerReadBarrier(HInstruction* instruction,
846                                              Location ref,
847                                              vixl::aarch64::Register obj,
848                                              const vixl::aarch64::MemOperand& src,
849                                              bool needs_null_check,
850                                              bool use_load_acquire);
851   // Fast path implementation of ReadBarrier::Barrier for a heap
852   // reference field load when Baker's read barriers are used.
853   void GenerateFieldLoadWithBakerReadBarrier(HInstruction* instruction,
854                                              Location ref,
855                                              vixl::aarch64::Register obj,
856                                              uint32_t offset,
857                                              Location maybe_temp,
858                                              bool needs_null_check,
859                                              bool use_load_acquire);
860   // Fast path implementation of ReadBarrier::Barrier for a heap
861   // reference array load when Baker's read barriers are used.
862   void GenerateArrayLoadWithBakerReadBarrier(HArrayGet* instruction,
863                                              Location ref,
864                                              vixl::aarch64::Register obj,
865                                              uint32_t data_offset,
866                                              Location index,
867                                              bool needs_null_check);
868 
869   // Emit code checking the status of the Marking Register, and
870   // aborting the program if MR does not match the value stored in the
871   // art::Thread object. Code is only emitted in debug mode and if
872   // CompilerOptions::EmitRunTimeChecksInDebugMode returns true.
873   //
874   // Argument `code` is used to identify the different occurrences of
875   // MaybeGenerateMarkingRegisterCheck in the code generator, and is
876   // passed to the BRK instruction.
877   //
878   // If `temp_loc` is a valid location, it is expected to be a
879   // register and will be used as a temporary to generate code;
880   // otherwise, a temporary will be fetched from the core register
881   // scratch pool.
882   virtual void MaybeGenerateMarkingRegisterCheck(int code,
883                                                  Location temp_loc = Location::NoLocation());
884 
885   // Create slow path for a read barrier for a heap reference within `instruction`.
886   //
887   // This is a helper function for GenerateReadBarrierSlow() that has the same
888   // arguments. The creation and adding of the slow path is exposed for intrinsics
889   // that cannot use GenerateReadBarrierSlow() from their own slow paths.
890   SlowPathCodeARM64* AddReadBarrierSlowPath(HInstruction* instruction,
891                                             Location out,
892                                             Location ref,
893                                             Location obj,
894                                             uint32_t offset,
895                                             Location index);
896 
897   // Generate a read barrier for a heap reference within `instruction`
898   // using a slow path.
899   //
900   // A read barrier for an object reference read from the heap is
901   // implemented as a call to the artReadBarrierSlow runtime entry
902   // point, which is passed the values in locations `ref`, `obj`, and
903   // `offset`:
904   //
905   //   mirror::Object* artReadBarrierSlow(mirror::Object* ref,
906   //                                      mirror::Object* obj,
907   //                                      uint32_t offset);
908   //
909   // The `out` location contains the value returned by
910   // artReadBarrierSlow.
911   //
912   // When `index` is provided (i.e. for array accesses), the offset
913   // value passed to artReadBarrierSlow is adjusted to take `index`
914   // into account.
915   void GenerateReadBarrierSlow(HInstruction* instruction,
916                                Location out,
917                                Location ref,
918                                Location obj,
919                                uint32_t offset,
920                                Location index = Location::NoLocation());
921 
922   // If read barriers are enabled, generate a read barrier for a heap
923   // reference using a slow path. If heap poisoning is enabled, also
924   // unpoison the reference in `out`.
925   void MaybeGenerateReadBarrierSlow(HInstruction* instruction,
926                                     Location out,
927                                     Location ref,
928                                     Location obj,
929                                     uint32_t offset,
930                                     Location index = Location::NoLocation());
931 
932   // Generate a read barrier for a GC root within `instruction` using
933   // a slow path.
934   //
935   // A read barrier for an object reference GC root is implemented as
936   // a call to the artReadBarrierForRootSlow runtime entry point,
937   // which is passed the value in location `root`:
938   //
939   //   mirror::Object* artReadBarrierForRootSlow(GcRoot<mirror::Object>* root);
940   //
941   // The `out` location contains the value returned by
942   // artReadBarrierForRootSlow.
943   void GenerateReadBarrierForRootSlow(HInstruction* instruction, Location out, Location root);
944 
945   void IncreaseFrame(size_t adjustment) override;
946   void DecreaseFrame(size_t adjustment) override;
947 
948   void GenerateNop() override;
949 
950   void GenerateImplicitNullCheck(HNullCheck* instruction) override;
951   void GenerateExplicitNullCheck(HNullCheck* instruction) override;
952 
MaybeRecordImplicitNullCheck(HInstruction * instr)953   void MaybeRecordImplicitNullCheck(HInstruction* instr) final {
954     // The function must be only called within special scopes
955     // (EmissionCheckScope, ExactAssemblyScope) which prevent generation of
956     // veneer/literal pools by VIXL assembler.
957     CHECK_EQ(GetVIXLAssembler()->ArePoolsBlocked(), true)
958         << "The function must only be called within EmissionCheckScope or ExactAssemblyScope";
959     CodeGenerator::MaybeRecordImplicitNullCheck(instr);
960   }
961 
962   void MaybeGenerateInlineCacheCheck(HInstruction* instruction, vixl::aarch64::Register klass);
963   void MaybeIncrementHotness(bool is_frame_entry);
964 
965  private:
966   // Encoding of thunk type and data for link-time generated thunks for Baker read barriers.
967 
968   enum class BakerReadBarrierKind : uint8_t {
969     kField,     // Field get or array get with constant offset (i.e. constant index).
970     kAcquire,   // Volatile field get.
971     kArray,     // Array get with index in register.
972     kGcRoot,    // GC root load.
973     kLast = kGcRoot
974   };
975 
976   static constexpr uint32_t kBakerReadBarrierInvalidEncodedReg = /* sp/zr is invalid */ 31u;
977 
978   static constexpr size_t kBitsForBakerReadBarrierKind =
979       MinimumBitsToStore(static_cast<size_t>(BakerReadBarrierKind::kLast));
980   static constexpr size_t kBakerReadBarrierBitsForRegister =
981       MinimumBitsToStore(kBakerReadBarrierInvalidEncodedReg);
982   using BakerReadBarrierKindField =
983       BitField<BakerReadBarrierKind, 0, kBitsForBakerReadBarrierKind>;
984   using BakerReadBarrierFirstRegField =
985       BitField<uint32_t, kBitsForBakerReadBarrierKind, kBakerReadBarrierBitsForRegister>;
986   using BakerReadBarrierSecondRegField =
987       BitField<uint32_t,
988                kBitsForBakerReadBarrierKind + kBakerReadBarrierBitsForRegister,
989                kBakerReadBarrierBitsForRegister>;
990 
CheckValidReg(uint32_t reg)991   static void CheckValidReg(uint32_t reg) {
992     DCHECK(reg < vixl::aarch64::lr.GetCode() &&
993            reg != vixl::aarch64::ip0.GetCode() &&
994            reg != vixl::aarch64::ip1.GetCode()) << reg;
995   }
996 
EncodeBakerReadBarrierFieldData(uint32_t base_reg,uint32_t holder_reg)997   static inline uint32_t EncodeBakerReadBarrierFieldData(uint32_t base_reg, uint32_t holder_reg) {
998     CheckValidReg(base_reg);
999     CheckValidReg(holder_reg);
1000     return BakerReadBarrierKindField::Encode(BakerReadBarrierKind::kField) |
1001            BakerReadBarrierFirstRegField::Encode(base_reg) |
1002            BakerReadBarrierSecondRegField::Encode(holder_reg);
1003   }
1004 
EncodeBakerReadBarrierAcquireData(uint32_t base_reg,uint32_t holder_reg)1005   static inline uint32_t EncodeBakerReadBarrierAcquireData(uint32_t base_reg, uint32_t holder_reg) {
1006     CheckValidReg(base_reg);
1007     CheckValidReg(holder_reg);
1008     DCHECK_NE(base_reg, holder_reg);
1009     return BakerReadBarrierKindField::Encode(BakerReadBarrierKind::kAcquire) |
1010            BakerReadBarrierFirstRegField::Encode(base_reg) |
1011            BakerReadBarrierSecondRegField::Encode(holder_reg);
1012   }
1013 
EncodeBakerReadBarrierArrayData(uint32_t base_reg)1014   static inline uint32_t EncodeBakerReadBarrierArrayData(uint32_t base_reg) {
1015     CheckValidReg(base_reg);
1016     return BakerReadBarrierKindField::Encode(BakerReadBarrierKind::kArray) |
1017            BakerReadBarrierFirstRegField::Encode(base_reg) |
1018            BakerReadBarrierSecondRegField::Encode(kBakerReadBarrierInvalidEncodedReg);
1019   }
1020 
EncodeBakerReadBarrierGcRootData(uint32_t root_reg)1021   static inline uint32_t EncodeBakerReadBarrierGcRootData(uint32_t root_reg) {
1022     CheckValidReg(root_reg);
1023     return BakerReadBarrierKindField::Encode(BakerReadBarrierKind::kGcRoot) |
1024            BakerReadBarrierFirstRegField::Encode(root_reg) |
1025            BakerReadBarrierSecondRegField::Encode(kBakerReadBarrierInvalidEncodedReg);
1026   }
1027 
1028   void CompileBakerReadBarrierThunk(Arm64Assembler& assembler,
1029                                     uint32_t encoded_data,
1030                                     /*out*/ std::string* debug_name);
1031 
1032   using Uint64ToLiteralMap = ArenaSafeMap<uint64_t, vixl::aarch64::Literal<uint64_t>*>;
1033   using Uint32ToLiteralMap = ArenaSafeMap<uint32_t, vixl::aarch64::Literal<uint32_t>*>;
1034   using StringToLiteralMap = ArenaSafeMap<StringReference,
1035                                           vixl::aarch64::Literal<uint32_t>*,
1036                                           StringReferenceValueComparator>;
1037   using TypeToLiteralMap = ArenaSafeMap<TypeReference,
1038                                         vixl::aarch64::Literal<uint32_t>*,
1039                                         TypeReferenceValueComparator>;
1040 
1041   vixl::aarch64::Literal<uint32_t>* DeduplicateUint32Literal(uint32_t value);
1042   vixl::aarch64::Literal<uint64_t>* DeduplicateUint64Literal(uint64_t value);
1043 
1044   // The PcRelativePatchInfo is used for PC-relative addressing of methods/strings/types,
1045   // whether through .data.bimg.rel.ro, .bss, or directly in the boot image.
1046   struct PcRelativePatchInfo : PatchInfo<vixl::aarch64::Label> {
PcRelativePatchInfoPcRelativePatchInfo1047     PcRelativePatchInfo(const DexFile* dex_file, uint32_t off_or_idx)
1048         : PatchInfo<vixl::aarch64::Label>(dex_file, off_or_idx), pc_insn_label() { }
1049 
1050     vixl::aarch64::Label* pc_insn_label;
1051   };
1052 
1053   struct BakerReadBarrierPatchInfo {
BakerReadBarrierPatchInfoBakerReadBarrierPatchInfo1054     explicit BakerReadBarrierPatchInfo(uint32_t data) : label(), custom_data(data) { }
1055 
1056     vixl::aarch64::Label label;
1057     uint32_t custom_data;
1058   };
1059 
1060   vixl::aarch64::Label* NewPcRelativePatch(const DexFile* dex_file,
1061                                            uint32_t offset_or_index,
1062                                            vixl::aarch64::Label* adrp_label,
1063                                            ArenaDeque<PcRelativePatchInfo>* patches);
1064 
1065   void EmitJumpTables();
1066 
1067   template <linker::LinkerPatch (*Factory)(size_t, const DexFile*, uint32_t, uint32_t)>
1068   static void EmitPcRelativeLinkerPatches(const ArenaDeque<PcRelativePatchInfo>& infos,
1069                                           ArenaVector<linker::LinkerPatch>* linker_patches);
1070 
1071   // Returns whether SVE features are supported and should be used.
1072   bool ShouldUseSVE() const;
1073 
1074   // Labels for each block that will be compiled.
1075   // We use a deque so that the `vixl::aarch64::Label` objects do not move in memory.
1076   ArenaDeque<vixl::aarch64::Label> block_labels_;  // Indexed by block id.
1077   vixl::aarch64::Label frame_entry_label_;
1078   ArenaVector<std::unique_ptr<JumpTableARM64>> jump_tables_;
1079 
1080   LocationsBuilderARM64Neon location_builder_neon_;
1081   InstructionCodeGeneratorARM64Neon instruction_visitor_neon_;
1082   LocationsBuilderARM64Sve location_builder_sve_;
1083   InstructionCodeGeneratorARM64Sve instruction_visitor_sve_;
1084 
1085   LocationsBuilderARM64* location_builder_;
1086   InstructionCodeGeneratorARM64* instruction_visitor_;
1087   ParallelMoveResolverARM64 move_resolver_;
1088   Arm64Assembler assembler_;
1089 
1090   // PC-relative method patch info for kBootImageLinkTimePcRelative.
1091   ArenaDeque<PcRelativePatchInfo> boot_image_method_patches_;
1092   // PC-relative method patch info for kBssEntry.
1093   ArenaDeque<PcRelativePatchInfo> method_bss_entry_patches_;
1094   // PC-relative type patch info for kBootImageLinkTimePcRelative.
1095   ArenaDeque<PcRelativePatchInfo> boot_image_type_patches_;
1096   // PC-relative type patch info for kBssEntry.
1097   ArenaDeque<PcRelativePatchInfo> type_bss_entry_patches_;
1098   // PC-relative public type patch info for kBssEntryPublic.
1099   ArenaDeque<PcRelativePatchInfo> public_type_bss_entry_patches_;
1100   // PC-relative package type patch info for kBssEntryPackage.
1101   ArenaDeque<PcRelativePatchInfo> package_type_bss_entry_patches_;
1102   // PC-relative String patch info for kBootImageLinkTimePcRelative.
1103   ArenaDeque<PcRelativePatchInfo> boot_image_string_patches_;
1104   // PC-relative String patch info for kBssEntry.
1105   ArenaDeque<PcRelativePatchInfo> string_bss_entry_patches_;
1106   // PC-relative method patch info for kBootImageLinkTimePcRelative+kCallCriticalNative.
1107   ArenaDeque<PcRelativePatchInfo> boot_image_jni_entrypoint_patches_;
1108   // PC-relative patch info for IntrinsicObjects for the boot image,
1109   // and for method/type/string patches for kBootImageRelRo otherwise.
1110   ArenaDeque<PcRelativePatchInfo> boot_image_other_patches_;
1111   // Patch info for calls to entrypoint dispatch thunks. Used for slow paths.
1112   ArenaDeque<PatchInfo<vixl::aarch64::Label>> call_entrypoint_patches_;
1113   // Baker read barrier patch info.
1114   ArenaDeque<BakerReadBarrierPatchInfo> baker_read_barrier_patches_;
1115 
1116   // Deduplication map for 32-bit literals, used for JIT for boot image addresses.
1117   Uint32ToLiteralMap uint32_literals_;
1118   // Deduplication map for 64-bit literals, used for JIT for method address or method code.
1119   Uint64ToLiteralMap uint64_literals_;
1120   // Patches for string literals in JIT compiled code.
1121   StringToLiteralMap jit_string_patches_;
1122   // Patches for class literals in JIT compiled code.
1123   TypeToLiteralMap jit_class_patches_;
1124 
1125   // Baker read barrier slow paths, mapping custom data (uint32_t) to label.
1126   // Wrap the label to work around vixl::aarch64::Label being non-copyable
1127   // and non-moveable and as such unusable in ArenaSafeMap<>.
1128   struct LabelWrapper {
LabelWrapperLabelWrapper1129     LabelWrapper(const LabelWrapper& src)
1130         : label() {
1131       DCHECK(!src.label.IsLinked() && !src.label.IsBound());
1132     }
1133     LabelWrapper() = default;
1134     vixl::aarch64::Label label;
1135   };
1136   ArenaSafeMap<uint32_t, LabelWrapper> jit_baker_read_barrier_slow_paths_;
1137 
1138   friend class linker::Arm64RelativePatcherTest;
1139   DISALLOW_COPY_AND_ASSIGN(CodeGeneratorARM64);
1140 };
1141 
GetAssembler()1142 inline Arm64Assembler* ParallelMoveResolverARM64::GetAssembler() const {
1143   return codegen_->GetAssembler();
1144 }
1145 
1146 }  // namespace arm64
1147 }  // namespace art
1148 
1149 #endif  // ART_COMPILER_OPTIMIZING_CODE_GENERATOR_ARM64_H_
1150