1 /*
2  * Copyright (C) 2014 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 #ifndef ART_COMPILER_OPTIMIZING_CODE_GENERATOR_ARM64_H_
18 #define ART_COMPILER_OPTIMIZING_CODE_GENERATOR_ARM64_H_
19 
20 #include "base/bit_field.h"
21 #include "base/macros.h"
22 #include "class_root.h"
23 #include "code_generator.h"
24 #include "common_arm64.h"
25 #include "dex/dex_file_types.h"
26 #include "dex/string_reference.h"
27 #include "dex/type_reference.h"
28 #include "driver/compiler_options.h"
29 #include "jit_patches_arm64.h"
30 #include "nodes.h"
31 #include "parallel_move_resolver.h"
32 #include "utils/arm64/assembler_arm64.h"
33 
34 // TODO(VIXL): Make VIXL compile with -Wshadow.
35 #pragma GCC diagnostic push
36 #pragma GCC diagnostic ignored "-Wshadow"
37 #include "aarch64/disasm-aarch64.h"
38 #include "aarch64/macro-assembler-aarch64.h"
39 #pragma GCC diagnostic pop
40 
41 namespace art HIDDEN {
42 
43 namespace linker {
44 class Arm64RelativePatcherTest;
45 }  // namespace linker
46 
47 namespace arm64 {
48 
49 class CodeGeneratorARM64;
50 
51 // Use a local definition to prevent copying mistakes.
52 static constexpr size_t kArm64WordSize = static_cast<size_t>(kArm64PointerSize);
53 
54 // This constant is used as an approximate margin when emission of veneer and literal pools
55 // must be blocked.
56 static constexpr int kMaxMacroInstructionSizeInBytes = 15 * vixl::aarch64::kInstructionSize;
57 
58 static const vixl::aarch64::Register kParameterCoreRegisters[] = {
59     vixl::aarch64::x1,
60     vixl::aarch64::x2,
61     vixl::aarch64::x3,
62     vixl::aarch64::x4,
63     vixl::aarch64::x5,
64     vixl::aarch64::x6,
65     vixl::aarch64::x7
66 };
67 static constexpr size_t kParameterCoreRegistersLength = arraysize(kParameterCoreRegisters);
68 static const vixl::aarch64::VRegister kParameterFPRegisters[] = {
69     vixl::aarch64::d0,
70     vixl::aarch64::d1,
71     vixl::aarch64::d2,
72     vixl::aarch64::d3,
73     vixl::aarch64::d4,
74     vixl::aarch64::d5,
75     vixl::aarch64::d6,
76     vixl::aarch64::d7
77 };
78 static constexpr size_t kParameterFPRegistersLength = arraysize(kParameterFPRegisters);
79 
80 // Thread Register.
81 const vixl::aarch64::Register tr = vixl::aarch64::x19;
82 // Marking Register.
83 const vixl::aarch64::Register mr = vixl::aarch64::x20;
84 // Implicit suspend check register.
85 const vixl::aarch64::Register kImplicitSuspendCheckRegister = vixl::aarch64::x21;
86 // Method register on invoke.
87 static const vixl::aarch64::Register kArtMethodRegister = vixl::aarch64::x0;
88 const vixl::aarch64::CPURegList vixl_reserved_core_registers(vixl::aarch64::ip0,
89                                                              vixl::aarch64::ip1);
90 const vixl::aarch64::CPURegList vixl_reserved_fp_registers(vixl::aarch64::d31);
91 
92 const vixl::aarch64::CPURegList runtime_reserved_core_registers =
93     vixl::aarch64::CPURegList(
94         tr,
95         // Reserve X20 as Marking Register when emitting Baker read barriers.
96         // TODO: We don't need to reserve marking-register for userfaultfd GC. But
97         // that would require some work in the assembler code as the right GC is
98         // chosen at load-time and not compile time.
99         (kReserveMarkingRegister ? mr : vixl::aarch64::NoCPUReg),
100         kImplicitSuspendCheckRegister,
101         vixl::aarch64::lr);
102 
103 // Some instructions have special requirements for a temporary, for example
104 // LoadClass/kBssEntry and LoadString/kBssEntry for Baker read barrier require
105 // temp that's not an R0 (to avoid an extra move) and Baker read barrier field
106 // loads with large offsets need a fixed register to limit the number of link-time
107 // thunks we generate. For these and similar cases, we want to reserve a specific
108 // register that's neither callee-save nor an argument register. We choose x15.
FixedTempLocation()109 inline Location FixedTempLocation() {
110   return Location::RegisterLocation(vixl::aarch64::x15.GetCode());
111 }
112 
113 // Callee-save registers AAPCS64, without x19 (Thread Register) (nor
114 // x20 (Marking Register) when emitting Baker read barriers).
115 const vixl::aarch64::CPURegList callee_saved_core_registers(
116     vixl::aarch64::CPURegister::kRegister,
117     vixl::aarch64::kXRegSize,
118     (kReserveMarkingRegister ? vixl::aarch64::x21.GetCode() : vixl::aarch64::x20.GetCode()),
119     vixl::aarch64::x30.GetCode());
120 const vixl::aarch64::CPURegList callee_saved_fp_registers(vixl::aarch64::CPURegister::kVRegister,
121                                                           vixl::aarch64::kDRegSize,
122                                                           vixl::aarch64::d8.GetCode(),
123                                                           vixl::aarch64::d15.GetCode());
124 Location ARM64ReturnLocation(DataType::Type return_type);
125 
126 #define UNIMPLEMENTED_INTRINSIC_LIST_ARM64(V) \
127   V(IntegerRemainderUnsigned)                 \
128   V(LongRemainderUnsigned)                    \
129   V(StringStringIndexOf)                      \
130   V(StringStringIndexOfAfter)                 \
131   V(StringBufferAppend)                       \
132   V(StringBufferLength)                       \
133   V(StringBufferToString)                     \
134   V(StringBuilderAppendObject)                \
135   V(StringBuilderAppendString)                \
136   V(StringBuilderAppendCharSequence)          \
137   V(StringBuilderAppendCharArray)             \
138   V(StringBuilderAppendBoolean)               \
139   V(StringBuilderAppendChar)                  \
140   V(StringBuilderAppendInt)                   \
141   V(StringBuilderAppendLong)                  \
142   V(StringBuilderAppendFloat)                 \
143   V(StringBuilderAppendDouble)                \
144   V(StringBuilderLength)                      \
145   V(StringBuilderToString)                    \
146   V(SystemArrayCopyByte)                      \
147   V(SystemArrayCopyInt)                       \
148   /* 1.8 */                                   \
149   V(MethodHandleInvokeExact)                  \
150   V(MethodHandleInvoke)
151 
152 class SlowPathCodeARM64 : public SlowPathCode {
153  public:
SlowPathCodeARM64(HInstruction * instruction)154   explicit SlowPathCodeARM64(HInstruction* instruction)
155       : SlowPathCode(instruction), entry_label_(), exit_label_() {}
156 
GetEntryLabel()157   vixl::aarch64::Label* GetEntryLabel() { return &entry_label_; }
GetExitLabel()158   vixl::aarch64::Label* GetExitLabel() { return &exit_label_; }
159 
160   void SaveLiveRegisters(CodeGenerator* codegen, LocationSummary* locations) override;
161   void RestoreLiveRegisters(CodeGenerator* codegen, LocationSummary* locations) override;
162 
163  private:
164   vixl::aarch64::Label entry_label_;
165   vixl::aarch64::Label exit_label_;
166 
167   DISALLOW_COPY_AND_ASSIGN(SlowPathCodeARM64);
168 };
169 
170 class JumpTableARM64 : public DeletableArenaObject<kArenaAllocSwitchTable> {
171  public:
JumpTableARM64(HPackedSwitch * switch_instr)172   explicit JumpTableARM64(HPackedSwitch* switch_instr)
173     : switch_instr_(switch_instr), table_start_() {}
174 
GetTableStartLabel()175   vixl::aarch64::Label* GetTableStartLabel() { return &table_start_; }
176 
177   void EmitTable(CodeGeneratorARM64* codegen);
178 
179  private:
180   HPackedSwitch* const switch_instr_;
181   vixl::aarch64::Label table_start_;
182 
183   DISALLOW_COPY_AND_ASSIGN(JumpTableARM64);
184 };
185 
186 static const vixl::aarch64::Register kRuntimeParameterCoreRegisters[] = {
187     vixl::aarch64::x0,
188     vixl::aarch64::x1,
189     vixl::aarch64::x2,
190     vixl::aarch64::x3,
191     vixl::aarch64::x4,
192     vixl::aarch64::x5,
193     vixl::aarch64::x6,
194     vixl::aarch64::x7
195 };
196 static constexpr size_t kRuntimeParameterCoreRegistersLength =
197     arraysize(kRuntimeParameterCoreRegisters);
198 static const vixl::aarch64::VRegister kRuntimeParameterFpuRegisters[] = {
199     vixl::aarch64::d0,
200     vixl::aarch64::d1,
201     vixl::aarch64::d2,
202     vixl::aarch64::d3,
203     vixl::aarch64::d4,
204     vixl::aarch64::d5,
205     vixl::aarch64::d6,
206     vixl::aarch64::d7
207 };
208 static constexpr size_t kRuntimeParameterFpuRegistersLength =
209     arraysize(kRuntimeParameterCoreRegisters);
210 
211 class InvokeRuntimeCallingConvention : public CallingConvention<vixl::aarch64::Register,
212                                                                 vixl::aarch64::VRegister> {
213  public:
InvokeRuntimeCallingConvention()214   InvokeRuntimeCallingConvention()
215       : CallingConvention(kRuntimeParameterCoreRegisters,
216                           kRuntimeParameterCoreRegistersLength,
217                           kRuntimeParameterFpuRegisters,
218                           kRuntimeParameterFpuRegistersLength,
219                           kArm64PointerSize) {}
220 
221   Location GetReturnLocation(DataType::Type return_type);
222 
223  private:
224   DISALLOW_COPY_AND_ASSIGN(InvokeRuntimeCallingConvention);
225 };
226 
227 class InvokeDexCallingConvention : public CallingConvention<vixl::aarch64::Register,
228                                                             vixl::aarch64::VRegister> {
229  public:
InvokeDexCallingConvention()230   InvokeDexCallingConvention()
231       : CallingConvention(kParameterCoreRegisters,
232                           kParameterCoreRegistersLength,
233                           kParameterFPRegisters,
234                           kParameterFPRegistersLength,
235                           kArm64PointerSize) {}
236 
GetReturnLocation(DataType::Type return_type)237   Location GetReturnLocation(DataType::Type return_type) const {
238     return ARM64ReturnLocation(return_type);
239   }
240 
241 
242  private:
243   DISALLOW_COPY_AND_ASSIGN(InvokeDexCallingConvention);
244 };
245 
246 class InvokeDexCallingConventionVisitorARM64 : public InvokeDexCallingConventionVisitor {
247  public:
InvokeDexCallingConventionVisitorARM64()248   InvokeDexCallingConventionVisitorARM64() {}
~InvokeDexCallingConventionVisitorARM64()249   virtual ~InvokeDexCallingConventionVisitorARM64() {}
250 
251   Location GetNextLocation(DataType::Type type) override;
GetReturnLocation(DataType::Type return_type)252   Location GetReturnLocation(DataType::Type return_type) const override {
253     return calling_convention.GetReturnLocation(return_type);
254   }
255   Location GetMethodLocation() const override;
256 
257  private:
258   InvokeDexCallingConvention calling_convention;
259 
260   DISALLOW_COPY_AND_ASSIGN(InvokeDexCallingConventionVisitorARM64);
261 };
262 
263 class CriticalNativeCallingConventionVisitorARM64 : public InvokeDexCallingConventionVisitor {
264  public:
CriticalNativeCallingConventionVisitorARM64(bool for_register_allocation)265   explicit CriticalNativeCallingConventionVisitorARM64(bool for_register_allocation)
266       : for_register_allocation_(for_register_allocation) {}
267 
~CriticalNativeCallingConventionVisitorARM64()268   virtual ~CriticalNativeCallingConventionVisitorARM64() {}
269 
270   Location GetNextLocation(DataType::Type type) override;
271   Location GetReturnLocation(DataType::Type type) const override;
272   Location GetMethodLocation() const override;
273 
GetStackOffset()274   size_t GetStackOffset() const { return stack_offset_; }
275 
276  private:
277   // Register allocator does not support adjusting frame size, so we cannot provide final locations
278   // of stack arguments for register allocation. We ask the register allocator for any location and
279   // move these arguments to the right place after adjusting the SP when generating the call.
280   const bool for_register_allocation_;
281   size_t gpr_index_ = 0u;
282   size_t fpr_index_ = 0u;
283   size_t stack_offset_ = 0u;
284 
285   DISALLOW_COPY_AND_ASSIGN(CriticalNativeCallingConventionVisitorARM64);
286 };
287 
288 class FieldAccessCallingConventionARM64 : public FieldAccessCallingConvention {
289  public:
FieldAccessCallingConventionARM64()290   FieldAccessCallingConventionARM64() {}
291 
GetObjectLocation()292   Location GetObjectLocation() const override {
293     return helpers::LocationFrom(vixl::aarch64::x1);
294   }
GetFieldIndexLocation()295   Location GetFieldIndexLocation() const override {
296     return helpers::LocationFrom(vixl::aarch64::x0);
297   }
GetReturnLocation(DataType::Type type)298   Location GetReturnLocation([[maybe_unused]] DataType::Type type) const override {
299     return helpers::LocationFrom(vixl::aarch64::x0);
300   }
GetSetValueLocation(DataType::Type type,bool is_instance)301   Location GetSetValueLocation([[maybe_unused]] DataType::Type type,
302                                bool is_instance) const override {
303     return is_instance
304         ? helpers::LocationFrom(vixl::aarch64::x2)
305         : helpers::LocationFrom(vixl::aarch64::x1);
306   }
GetFpuLocation(DataType::Type type)307   Location GetFpuLocation([[maybe_unused]] DataType::Type type) const override {
308     return helpers::LocationFrom(vixl::aarch64::d0);
309   }
310 
311  private:
312   DISALLOW_COPY_AND_ASSIGN(FieldAccessCallingConventionARM64);
313 };
314 
315 class InstructionCodeGeneratorARM64 : public InstructionCodeGenerator {
316  public:
317   InstructionCodeGeneratorARM64(HGraph* graph, CodeGeneratorARM64* codegen);
318 
319 #define DECLARE_VISIT_INSTRUCTION(name, super) \
320   void Visit##name(H##name* instr) override;
321 
322   FOR_EACH_CONCRETE_INSTRUCTION_SCALAR_COMMON(DECLARE_VISIT_INSTRUCTION)
FOR_EACH_CONCRETE_INSTRUCTION_ARM64(DECLARE_VISIT_INSTRUCTION)323   FOR_EACH_CONCRETE_INSTRUCTION_ARM64(DECLARE_VISIT_INSTRUCTION)
324   FOR_EACH_CONCRETE_INSTRUCTION_SHARED(DECLARE_VISIT_INSTRUCTION)
325 
326 #undef DECLARE_VISIT_INSTRUCTION
327 
328   void VisitInstruction(HInstruction* instruction) override {
329     LOG(FATAL) << "Unreachable instruction " << instruction->DebugName()
330                << " (id " << instruction->GetId() << ")";
331   }
332 
GetAssembler()333   Arm64Assembler* GetAssembler() const { return assembler_; }
GetVIXLAssembler()334   vixl::aarch64::MacroAssembler* GetVIXLAssembler() { return GetAssembler()->GetVIXLAssembler(); }
335 
336   // SIMD helpers.
337   virtual Location AllocateSIMDScratchLocation(vixl::aarch64::UseScratchRegisterScope* scope) = 0;
338   virtual void FreeSIMDScratchLocation(Location loc,
339                                        vixl::aarch64::UseScratchRegisterScope* scope)  = 0;
340   virtual void LoadSIMDRegFromStack(Location destination, Location source) = 0;
341   virtual void MoveSIMDRegToSIMDReg(Location destination, Location source) = 0;
342   virtual void MoveToSIMDStackSlot(Location destination, Location source) = 0;
343   virtual void SaveLiveRegistersHelper(LocationSummary* locations,
344                                        int64_t spill_offset) = 0;
345   virtual void RestoreLiveRegistersHelper(LocationSummary* locations,
346                                           int64_t spill_offset) = 0;
347 
348  protected:
349   void GenerateClassInitializationCheck(SlowPathCodeARM64* slow_path,
350                                         vixl::aarch64::Register class_reg);
351   void GenerateBitstringTypeCheckCompare(HTypeCheckInstruction* check,
352                                          vixl::aarch64::Register temp);
353   void GenerateSuspendCheck(HSuspendCheck* instruction, HBasicBlock* successor);
354   void HandleBinaryOp(HBinaryOperation* instr);
355 
356   void HandleFieldSet(HInstruction* instruction,
357                       const FieldInfo& field_info,
358                       bool value_can_be_null,
359                       WriteBarrierKind write_barrier_kind);
360   void HandleFieldGet(HInstruction* instruction, const FieldInfo& field_info);
361   void HandleCondition(HCondition* instruction);
362 
363   // Generate a heap reference load using one register `out`:
364   //
365   //   out <- *(out + offset)
366   //
367   // while honoring heap poisoning and/or read barriers (if any).
368   //
369   // Location `maybe_temp` is used when generating a read barrier and
370   // shall be a register in that case; it may be an invalid location
371   // otherwise.
372   void GenerateReferenceLoadOneRegister(HInstruction* instruction,
373                                         Location out,
374                                         uint32_t offset,
375                                         Location maybe_temp,
376                                         ReadBarrierOption read_barrier_option);
377   // Generate a heap reference load using two different registers
378   // `out` and `obj`:
379   //
380   //   out <- *(obj + offset)
381   //
382   // while honoring heap poisoning and/or read barriers (if any).
383   //
384   // Location `maybe_temp` is used when generating a Baker's (fast
385   // path) read barrier and shall be a register in that case; it may
386   // be an invalid location otherwise.
387   void GenerateReferenceLoadTwoRegisters(HInstruction* instruction,
388                                          Location out,
389                                          Location obj,
390                                          uint32_t offset,
391                                          Location maybe_temp,
392                                          ReadBarrierOption read_barrier_option);
393 
394   // Generate a floating-point comparison.
395   void GenerateFcmp(HInstruction* instruction);
396 
397   void HandleShift(HBinaryOperation* instr);
398   void GenerateTestAndBranch(HInstruction* instruction,
399                              size_t condition_input_index,
400                              vixl::aarch64::Label* true_target,
401                              vixl::aarch64::Label* false_target);
402   void DivRemOneOrMinusOne(HBinaryOperation* instruction);
403   void DivRemByPowerOfTwo(HBinaryOperation* instruction);
404   void GenerateIncrementNegativeByOne(vixl::aarch64::Register out,
405                                       vixl::aarch64::Register in, bool use_cond_inc);
406   void GenerateResultRemWithAnyConstant(vixl::aarch64::Register out,
407                                         vixl::aarch64::Register dividend,
408                                         vixl::aarch64::Register quotient,
409                                         int64_t divisor,
410                                         // This function may acquire a scratch register.
411                                         vixl::aarch64::UseScratchRegisterScope* temps_scope);
412   void GenerateInt64UnsignedDivRemWithAnyPositiveConstant(HBinaryOperation* instruction);
413   void GenerateInt64DivRemWithAnyConstant(HBinaryOperation* instruction);
414   void GenerateInt32DivRemWithAnyConstant(HBinaryOperation* instruction);
415   void GenerateDivRemWithAnyConstant(HBinaryOperation* instruction, int64_t divisor);
416   void GenerateIntDiv(HDiv* instruction);
417   void GenerateIntDivForConstDenom(HDiv *instruction);
418   void GenerateIntDivForPower2Denom(HDiv *instruction);
419   void GenerateIntRem(HRem* instruction);
420   void GenerateIntRemForConstDenom(HRem *instruction);
421   void GenerateIntRemForPower2Denom(HRem *instruction);
422   void HandleGoto(HInstruction* got, HBasicBlock* successor);
423   void GenerateMethodEntryExitHook(HInstruction* instruction);
424 
425   // Helpers to set up locations for vector memory operations. Returns the memory operand and,
426   // if used, sets the output parameter scratch to a temporary register used in this operand,
427   // so that the client can release it right after the memory operand use.
428   // Neon version.
429   vixl::aarch64::MemOperand VecNEONAddress(
430       HVecMemoryOperation* instruction,
431       // This function may acquire a scratch register.
432       vixl::aarch64::UseScratchRegisterScope* temps_scope,
433       size_t size,
434       bool is_string_char_at,
435       /*out*/ vixl::aarch64::Register* scratch);
436   // SVE version.
437   vixl::aarch64::SVEMemOperand VecSVEAddress(
438       HVecMemoryOperation* instruction,
439       // This function may acquire a scratch register.
440       vixl::aarch64::UseScratchRegisterScope* temps_scope,
441       size_t size,
442       bool is_string_char_at,
443       /*out*/ vixl::aarch64::Register* scratch);
444 
445   Arm64Assembler* const assembler_;
446   CodeGeneratorARM64* const codegen_;
447 
448   DISALLOW_COPY_AND_ASSIGN(InstructionCodeGeneratorARM64);
449 };
450 
451 class LocationsBuilderARM64 : public HGraphVisitor {
452  public:
LocationsBuilderARM64(HGraph * graph,CodeGeneratorARM64 * codegen)453   LocationsBuilderARM64(HGraph* graph, CodeGeneratorARM64* codegen)
454       : HGraphVisitor(graph), codegen_(codegen) {}
455 
456 #define DECLARE_VISIT_INSTRUCTION(name, super) \
457   void Visit##name(H##name* instr) override;
458 
459   FOR_EACH_CONCRETE_INSTRUCTION_SCALAR_COMMON(DECLARE_VISIT_INSTRUCTION)
FOR_EACH_CONCRETE_INSTRUCTION_ARM64(DECLARE_VISIT_INSTRUCTION)460   FOR_EACH_CONCRETE_INSTRUCTION_ARM64(DECLARE_VISIT_INSTRUCTION)
461   FOR_EACH_CONCRETE_INSTRUCTION_SHARED(DECLARE_VISIT_INSTRUCTION)
462 
463 #undef DECLARE_VISIT_INSTRUCTION
464 
465   void VisitInstruction(HInstruction* instruction) override {
466     LOG(FATAL) << "Unreachable instruction " << instruction->DebugName()
467                << " (id " << instruction->GetId() << ")";
468   }
469 
470  protected:
471   void HandleBinaryOp(HBinaryOperation* instr);
472   void HandleFieldSet(HInstruction* instruction);
473   void HandleFieldGet(HInstruction* instruction, const FieldInfo& field_info);
474   void HandleInvoke(HInvoke* instr);
475   void HandleCondition(HCondition* instruction);
476   void HandleShift(HBinaryOperation* instr);
477 
478   CodeGeneratorARM64* const codegen_;
479   InvokeDexCallingConventionVisitorARM64 parameter_visitor_;
480 
481   DISALLOW_COPY_AND_ASSIGN(LocationsBuilderARM64);
482 };
483 
484 class InstructionCodeGeneratorARM64Neon : public InstructionCodeGeneratorARM64 {
485  public:
InstructionCodeGeneratorARM64Neon(HGraph * graph,CodeGeneratorARM64 * codegen)486   InstructionCodeGeneratorARM64Neon(HGraph* graph, CodeGeneratorARM64* codegen) :
487       InstructionCodeGeneratorARM64(graph, codegen) {}
488 
489 #define DECLARE_VISIT_INSTRUCTION(name, super) \
490   void Visit##name(H##name* instr) override;
491 
492   FOR_EACH_CONCRETE_INSTRUCTION_VECTOR_COMMON(DECLARE_VISIT_INSTRUCTION)
493 
494 #undef DECLARE_VISIT_INSTRUCTION
495 
496   Location AllocateSIMDScratchLocation(vixl::aarch64::UseScratchRegisterScope* scope) override;
497   void FreeSIMDScratchLocation(Location loc,
498                                vixl::aarch64::UseScratchRegisterScope* scope) override;
499   void LoadSIMDRegFromStack(Location destination, Location source) override;
500   void MoveSIMDRegToSIMDReg(Location destination, Location source) override;
501   void MoveToSIMDStackSlot(Location destination, Location source) override;
502   void SaveLiveRegistersHelper(LocationSummary* locations, int64_t spill_offset) override;
503   void RestoreLiveRegistersHelper(LocationSummary* locations, int64_t spill_offset) override;
504 };
505 
506 class LocationsBuilderARM64Neon : public LocationsBuilderARM64 {
507  public:
LocationsBuilderARM64Neon(HGraph * graph,CodeGeneratorARM64 * codegen)508   LocationsBuilderARM64Neon(HGraph* graph, CodeGeneratorARM64* codegen) :
509       LocationsBuilderARM64(graph, codegen) {}
510 
511 #define DECLARE_VISIT_INSTRUCTION(name, super) \
512   void Visit##name(H##name* instr) override;
513 
514   FOR_EACH_CONCRETE_INSTRUCTION_VECTOR_COMMON(DECLARE_VISIT_INSTRUCTION)
515 
516 #undef DECLARE_VISIT_INSTRUCTION
517 };
518 
519 class InstructionCodeGeneratorARM64Sve : public InstructionCodeGeneratorARM64 {
520  public:
InstructionCodeGeneratorARM64Sve(HGraph * graph,CodeGeneratorARM64 * codegen)521   InstructionCodeGeneratorARM64Sve(HGraph* graph, CodeGeneratorARM64* codegen) :
522       InstructionCodeGeneratorARM64(graph, codegen) {}
523 
524 #define DECLARE_VISIT_INSTRUCTION(name, super) \
525   void Visit##name(H##name* instr) override;
526 
527   FOR_EACH_CONCRETE_INSTRUCTION_VECTOR_COMMON(DECLARE_VISIT_INSTRUCTION)
528 
529 #undef DECLARE_VISIT_INSTRUCTION
530 
531   Location AllocateSIMDScratchLocation(vixl::aarch64::UseScratchRegisterScope* scope) override;
532   void FreeSIMDScratchLocation(Location loc,
533                                vixl::aarch64::UseScratchRegisterScope* scope) override;
534   void LoadSIMDRegFromStack(Location destination, Location source) override;
535   void MoveSIMDRegToSIMDReg(Location destination, Location source) override;
536   void MoveToSIMDStackSlot(Location destination, Location source) override;
537   void SaveLiveRegistersHelper(LocationSummary* locations, int64_t spill_offset) override;
538   void RestoreLiveRegistersHelper(LocationSummary* locations, int64_t spill_offset) override;
539 
540  private:
541   // Validate that instruction vector length and packed type are compliant with the SIMD
542   // register size (full SIMD register is used).
543   void ValidateVectorLength(HVecOperation* instr) const;
544 
GetVecGoverningPReg(HVecOperation * instr)545   vixl::aarch64::PRegister GetVecGoverningPReg(HVecOperation* instr) {
546     return GetVecPredSetFixedOutPReg(instr->GetGoverningPredicate());
547   }
548 
549   // Returns a fixed p-reg for predicate setting instruction.
550   //
551   // Currently we only support diamond CF loops for predicated vectorization; also we don't have
552   // register allocator support for vector predicates. Thus we use fixed P-regs for loop main,
553   // True and False predicates as a temporary solution.
554   //
555   // TODO: Support SIMD types and registers in ART.
GetVecPredSetFixedOutPReg(HVecPredSetOperation * instr)556   static vixl::aarch64::PRegister GetVecPredSetFixedOutPReg(HVecPredSetOperation* instr) {
557     if (instr->IsVecPredWhile() || instr->IsVecPredSetAll()) {
558       // VecPredWhile and VecPredSetAll live ranges never overlap due to the current vectorization
559       // scheme: the former only is live inside a vectorized loop and the later is never in a
560       // loop and never spans across loops.
561       return vixl::aarch64::p0;
562     } else if (instr->IsVecPredNot()) {
563       // This relies on the fact that we only use PredNot manually in the autovectorizer,
564       // so there is only one of them in each loop.
565       return vixl::aarch64::p1;
566     } else {
567       DCHECK(instr->IsVecCondition());
568       return vixl::aarch64::p2;
569     }
570   }
571 };
572 
573 class LocationsBuilderARM64Sve : public LocationsBuilderARM64 {
574  public:
LocationsBuilderARM64Sve(HGraph * graph,CodeGeneratorARM64 * codegen)575   LocationsBuilderARM64Sve(HGraph* graph, CodeGeneratorARM64* codegen) :
576       LocationsBuilderARM64(graph, codegen) {}
577 
578 #define DECLARE_VISIT_INSTRUCTION(name, super) \
579   void Visit##name(H##name* instr) override;
580 
581   FOR_EACH_CONCRETE_INSTRUCTION_VECTOR_COMMON(DECLARE_VISIT_INSTRUCTION)
582 
583 #undef DECLARE_VISIT_INSTRUCTION
584 };
585 
586 class ParallelMoveResolverARM64 : public ParallelMoveResolverNoSwap {
587  public:
ParallelMoveResolverARM64(ArenaAllocator * allocator,CodeGeneratorARM64 * codegen)588   ParallelMoveResolverARM64(ArenaAllocator* allocator, CodeGeneratorARM64* codegen)
589       : ParallelMoveResolverNoSwap(allocator), codegen_(codegen), vixl_temps_() {}
590 
591  protected:
592   void PrepareForEmitNativeCode() override;
593   void FinishEmitNativeCode() override;
594   Location AllocateScratchLocationFor(Location::Kind kind) override;
595   void FreeScratchLocation(Location loc) override;
596   void EmitMove(size_t index) override;
597 
598  private:
599   Arm64Assembler* GetAssembler() const;
GetVIXLAssembler()600   vixl::aarch64::MacroAssembler* GetVIXLAssembler() const {
601     return GetAssembler()->GetVIXLAssembler();
602   }
603 
604   CodeGeneratorARM64* const codegen_;
605   vixl::aarch64::UseScratchRegisterScope vixl_temps_;
606 
607   DISALLOW_COPY_AND_ASSIGN(ParallelMoveResolverARM64);
608 };
609 
610 class CodeGeneratorARM64 : public CodeGenerator {
611  public:
612   CodeGeneratorARM64(HGraph* graph,
613                      const CompilerOptions& compiler_options,
614                      OptimizingCompilerStats* stats = nullptr);
~CodeGeneratorARM64()615   virtual ~CodeGeneratorARM64() {}
616 
617   void GenerateFrameEntry() override;
618   void GenerateFrameExit() override;
619 
620   vixl::aarch64::CPURegList GetFramePreservedCoreRegisters() const;
621   vixl::aarch64::CPURegList GetFramePreservedFPRegisters() const;
622 
623   void Bind(HBasicBlock* block) override;
624 
GetLabelOf(HBasicBlock * block)625   vixl::aarch64::Label* GetLabelOf(HBasicBlock* block) {
626     block = FirstNonEmptyBlock(block);
627     return &(block_labels_[block->GetBlockId()]);
628   }
629 
GetWordSize()630   size_t GetWordSize() const override {
631     return kArm64WordSize;
632   }
633 
SupportsPredicatedSIMD()634   bool SupportsPredicatedSIMD() const override { return ShouldUseSVE(); }
635 
GetSlowPathFPWidth()636   size_t GetSlowPathFPWidth() const override {
637     return GetGraph()->HasSIMD()
638         ? GetSIMDRegisterWidth()
639         : vixl::aarch64::kDRegSizeInBytes;
640   }
641 
GetCalleePreservedFPWidth()642   size_t GetCalleePreservedFPWidth() const override {
643     return vixl::aarch64::kDRegSizeInBytes;
644   }
645 
646   size_t GetSIMDRegisterWidth() const override;
647 
GetAddressOf(HBasicBlock * block)648   uintptr_t GetAddressOf(HBasicBlock* block) override {
649     vixl::aarch64::Label* block_entry_label = GetLabelOf(block);
650     DCHECK(block_entry_label->IsBound());
651     return block_entry_label->GetLocation();
652   }
653 
GetLocationBuilder()654   HGraphVisitor* GetLocationBuilder() override { return location_builder_; }
GetInstructionCodeGeneratorArm64()655   InstructionCodeGeneratorARM64* GetInstructionCodeGeneratorArm64() {
656     return instruction_visitor_;
657   }
GetInstructionVisitor()658   HGraphVisitor* GetInstructionVisitor() override { return GetInstructionCodeGeneratorArm64(); }
GetAssembler()659   Arm64Assembler* GetAssembler() override { return &assembler_; }
GetAssembler()660   const Arm64Assembler& GetAssembler() const override { return assembler_; }
GetVIXLAssembler()661   vixl::aarch64::MacroAssembler* GetVIXLAssembler() { return GetAssembler()->GetVIXLAssembler(); }
662 
663   // Emit a write barrier if:
664   // A) emit_null_check is false
665   // B) emit_null_check is true, and value is not null.
666   void MaybeMarkGCCard(vixl::aarch64::Register object,
667                        vixl::aarch64::Register value,
668                        bool emit_null_check);
669 
670   // Emit a write barrier unconditionally.
671   void MarkGCCard(vixl::aarch64::Register object);
672 
673   // Crash if the card table is not valid. This check is only emitted for the CC GC. We assert
674   // `(!clean || !self->is_gc_marking)`, since the card table should not be set to clean when the CC
675   // GC is marking for eliminated write barriers.
676   void CheckGCCardIsValid(vixl::aarch64::Register object);
677 
678   void GenerateMemoryBarrier(MemBarrierKind kind);
679 
680   // Register allocation.
681 
682   void SetupBlockedRegisters() const override;
683 
684   size_t SaveCoreRegister(size_t stack_index, uint32_t reg_id) override;
685   size_t RestoreCoreRegister(size_t stack_index, uint32_t reg_id) override;
686   size_t SaveFloatingPointRegister(size_t stack_index, uint32_t reg_id) override;
687   size_t RestoreFloatingPointRegister(size_t stack_index, uint32_t reg_id) override;
688 
689   // The number of registers that can be allocated. The register allocator may
690   // decide to reserve and not use a few of them.
691   // We do not consider registers sp, xzr, wzr. They are either not allocatable
692   // (xzr, wzr), or make for poor allocatable registers (sp alignment
693   // requirements, etc.). This also facilitates our task as all other registers
694   // can easily be mapped via to or from their type and index or code.
695   static const int kNumberOfAllocatableRegisters = vixl::aarch64::kNumberOfRegisters - 1;
696   static const int kNumberOfAllocatableFPRegisters = vixl::aarch64::kNumberOfVRegisters;
697   static constexpr int kNumberOfAllocatableRegisterPairs = 0;
698 
699   void DumpCoreRegister(std::ostream& stream, int reg) const override;
700   void DumpFloatingPointRegister(std::ostream& stream, int reg) const override;
701 
GetInstructionSet()702   InstructionSet GetInstructionSet() const override {
703     return InstructionSet::kArm64;
704   }
705 
706   const Arm64InstructionSetFeatures& GetInstructionSetFeatures() const;
707 
Initialize()708   void Initialize() override {
709     block_labels_.resize(GetGraph()->GetBlocks().size());
710   }
711 
712   // We want to use the STP and LDP instructions to spill and restore registers for slow paths.
713   // These instructions can only encode offsets that are multiples of the register size accessed.
GetPreferredSlotsAlignment()714   uint32_t GetPreferredSlotsAlignment() const override { return vixl::aarch64::kXRegSizeInBytes; }
715 
CreateJumpTable(HPackedSwitch * switch_instr)716   JumpTableARM64* CreateJumpTable(HPackedSwitch* switch_instr) {
717     jump_tables_.emplace_back(new (GetGraph()->GetAllocator()) JumpTableARM64(switch_instr));
718     return jump_tables_.back().get();
719   }
720 
721   void Finalize() override;
722 
723   // Code generation helpers.
724   void MoveConstant(vixl::aarch64::CPURegister destination, HConstant* constant);
725   void MoveConstant(Location destination, int32_t value) override;
726   void MoveLocation(Location dst, Location src, DataType::Type dst_type) override;
727   void AddLocationAsTemp(Location location, LocationSummary* locations) override;
728 
729   void Load(DataType::Type type,
730             vixl::aarch64::CPURegister dst,
731             const vixl::aarch64::MemOperand& src);
732   void Store(DataType::Type type,
733              vixl::aarch64::CPURegister src,
734              const vixl::aarch64::MemOperand& dst);
735   void LoadAcquire(HInstruction* instruction,
736                    DataType::Type type,
737                    vixl::aarch64::CPURegister dst,
738                    const vixl::aarch64::MemOperand& src,
739                    bool needs_null_check);
740   void StoreRelease(HInstruction* instruction,
741                     DataType::Type type,
742                     vixl::aarch64::CPURegister src,
743                     const vixl::aarch64::MemOperand& dst,
744                     bool needs_null_check);
745 
746   // Generate code to invoke a runtime entry point.
747   void InvokeRuntime(QuickEntrypointEnum entrypoint,
748                      HInstruction* instruction,
749                      uint32_t dex_pc,
750                      SlowPathCode* slow_path = nullptr) override;
751 
752   // Generate code to invoke a runtime entry point, but do not record
753   // PC-related information in a stack map.
754   void InvokeRuntimeWithoutRecordingPcInfo(int32_t entry_point_offset,
755                                            HInstruction* instruction,
756                                            SlowPathCode* slow_path);
757 
GetMoveResolver()758   ParallelMoveResolverARM64* GetMoveResolver() override { return &move_resolver_; }
759 
NeedsTwoRegisters(DataType::Type type)760   bool NeedsTwoRegisters([[maybe_unused]] DataType::Type type) const override { return false; }
761 
762   // Check if the desired_string_load_kind is supported. If it is, return it,
763   // otherwise return a fall-back kind that should be used instead.
764   HLoadString::LoadKind GetSupportedLoadStringKind(
765       HLoadString::LoadKind desired_string_load_kind) override;
766 
767   // Check if the desired_class_load_kind is supported. If it is, return it,
768   // otherwise return a fall-back kind that should be used instead.
769   HLoadClass::LoadKind GetSupportedLoadClassKind(
770       HLoadClass::LoadKind desired_class_load_kind) override;
771 
772   // Check if the desired_dispatch_info is supported. If it is, return it,
773   // otherwise return a fall-back info that should be used instead.
774   HInvokeStaticOrDirect::DispatchInfo GetSupportedInvokeStaticOrDirectDispatch(
775       const HInvokeStaticOrDirect::DispatchInfo& desired_dispatch_info,
776       ArtMethod* method) override;
777 
778   void LoadMethod(MethodLoadKind load_kind, Location temp, HInvoke* invoke);
779   void GenerateStaticOrDirectCall(
780       HInvokeStaticOrDirect* invoke, Location temp, SlowPathCode* slow_path = nullptr) override;
781   void GenerateVirtualCall(
782       HInvokeVirtual* invoke, Location temp, SlowPathCode* slow_path = nullptr) override;
783 
784   void MoveFromReturnRegister(Location trg, DataType::Type type) override;
785 
786   // Add a new boot image intrinsic patch for an instruction and return the label
787   // to be bound before the instruction. The instruction will be either the
788   // ADRP (pass `adrp_label = null`) or the ADD (pass `adrp_label` pointing
789   // to the associated ADRP patch label).
790   vixl::aarch64::Label* NewBootImageIntrinsicPatch(uint32_t intrinsic_data,
791                                                    vixl::aarch64::Label* adrp_label = nullptr);
792 
793   // Add a new boot image relocation patch for an instruction and return the label
794   // to be bound before the instruction. The instruction will be either the
795   // ADRP (pass `adrp_label = null`) or the LDR (pass `adrp_label` pointing
796   // to the associated ADRP patch label).
797   vixl::aarch64::Label* NewBootImageRelRoPatch(uint32_t boot_image_offset,
798                                                vixl::aarch64::Label* adrp_label = nullptr);
799 
800   // Add a new boot image method patch for an instruction and return the label
801   // to be bound before the instruction. The instruction will be either the
802   // ADRP (pass `adrp_label = null`) or the ADD (pass `adrp_label` pointing
803   // to the associated ADRP patch label).
804   vixl::aarch64::Label* NewBootImageMethodPatch(MethodReference target_method,
805                                                 vixl::aarch64::Label* adrp_label = nullptr);
806 
807   // Add a new .bss entry method patch for an instruction and return
808   // the label to be bound before the instruction. The instruction will be
809   // either the ADRP (pass `adrp_label = null`) or the LDR (pass `adrp_label`
810   // pointing to the associated ADRP patch label).
811   vixl::aarch64::Label* NewMethodBssEntryPatch(MethodReference target_method,
812                                                vixl::aarch64::Label* adrp_label = nullptr);
813 
814   // Add a new boot image type patch for an instruction and return the label
815   // to be bound before the instruction. The instruction will be either the
816   // ADRP (pass `adrp_label = null`) or the ADD (pass `adrp_label` pointing
817   // to the associated ADRP patch label).
818   vixl::aarch64::Label* NewBootImageTypePatch(const DexFile& dex_file,
819                                               dex::TypeIndex type_index,
820                                               vixl::aarch64::Label* adrp_label = nullptr);
821 
822   // Add a new app image type patch for an instruction and return the label
823   // to be bound before the instruction. The instruction will be either the
824   // ADRP (pass `adrp_label = null`) or the LDR (pass `adrp_label` pointing
825   // to the associated ADRP patch label).
826   vixl::aarch64::Label* NewAppImageTypePatch(const DexFile& dex_file,
827                                              dex::TypeIndex type_index,
828                                              vixl::aarch64::Label* adrp_label = nullptr);
829 
830   // Add a new .bss entry type patch for an instruction and return the label
831   // to be bound before the instruction. The instruction will be either the
832   // ADRP (pass `adrp_label = null`) or the ADD (pass `adrp_label` pointing
833   // to the associated ADRP patch label).
834   vixl::aarch64::Label* NewBssEntryTypePatch(HLoadClass* load_class,
835                                              vixl::aarch64::Label* adrp_label = nullptr);
836 
837   // Add a new boot image string patch for an instruction and return the label
838   // to be bound before the instruction. The instruction will be either the
839   // ADRP (pass `adrp_label = null`) or the ADD (pass `adrp_label` pointing
840   // to the associated ADRP patch label).
841   vixl::aarch64::Label* NewBootImageStringPatch(const DexFile& dex_file,
842                                                 dex::StringIndex string_index,
843                                                 vixl::aarch64::Label* adrp_label = nullptr);
844 
845   // Add a new .bss entry string patch for an instruction and return the label
846   // to be bound before the instruction. The instruction will be either the
847   // ADRP (pass `adrp_label = null`) or the ADD (pass `adrp_label` pointing
848   // to the associated ADRP patch label).
849   vixl::aarch64::Label* NewStringBssEntryPatch(const DexFile& dex_file,
850                                                dex::StringIndex string_index,
851                                                vixl::aarch64::Label* adrp_label = nullptr);
852 
853   // Add a new boot image JNI entrypoint patch for an instruction and return the label
854   // to be bound before the instruction. The instruction will be either the
855   // ADRP (pass `adrp_label = null`) or the LDR (pass `adrp_label` pointing
856   // to the associated ADRP patch label).
857   vixl::aarch64::Label* NewBootImageJniEntrypointPatch(MethodReference target_method,
858                                                        vixl::aarch64::Label* adrp_label = nullptr);
859 
860   // Emit the BL instruction for entrypoint thunk call and record the associated patch for AOT.
861   void EmitEntrypointThunkCall(ThreadOffset64 entrypoint_offset);
862 
863   // Emit the CBNZ instruction for baker read barrier and record
864   // the associated patch for AOT or slow path for JIT.
865   void EmitBakerReadBarrierCbnz(uint32_t custom_data);
866 
DeduplicateBootImageAddressLiteral(uint64_t address)867   vixl::aarch64::Literal<uint32_t>* DeduplicateBootImageAddressLiteral(uint64_t address) {
868     return jit_patches_.DeduplicateBootImageAddressLiteral(address);
869   }
DeduplicateJitStringLiteral(const DexFile & dex_file,dex::StringIndex string_index,Handle<mirror::String> handle)870   vixl::aarch64::Literal<uint32_t>* DeduplicateJitStringLiteral(const DexFile& dex_file,
871                                                                 dex::StringIndex string_index,
872                                                                 Handle<mirror::String> handle) {
873     return jit_patches_.DeduplicateJitStringLiteral(
874         dex_file, string_index, handle, GetCodeGenerationData());
875   }
DeduplicateJitClassLiteral(const DexFile & dex_file,dex::TypeIndex class_index,Handle<mirror::Class> handle)876   vixl::aarch64::Literal<uint32_t>* DeduplicateJitClassLiteral(const DexFile& dex_file,
877                                                                dex::TypeIndex class_index,
878                                                                Handle<mirror::Class> handle) {
879     return jit_patches_.DeduplicateJitClassLiteral(
880         dex_file, class_index, handle, GetCodeGenerationData());
881   }
882 
883   void EmitAdrpPlaceholder(vixl::aarch64::Label* fixup_label, vixl::aarch64::Register reg);
884   void EmitAddPlaceholder(vixl::aarch64::Label* fixup_label,
885                           vixl::aarch64::Register out,
886                           vixl::aarch64::Register base);
887   void EmitLdrOffsetPlaceholder(vixl::aarch64::Label* fixup_label,
888                                 vixl::aarch64::Register out,
889                                 vixl::aarch64::Register base);
890 
891   void LoadBootImageRelRoEntry(vixl::aarch64::Register reg, uint32_t boot_image_offset);
892   void LoadBootImageAddress(vixl::aarch64::Register reg, uint32_t boot_image_reference);
893   void LoadTypeForBootImageIntrinsic(vixl::aarch64::Register reg, TypeReference type_reference);
894   void LoadIntrinsicDeclaringClass(vixl::aarch64::Register reg, HInvoke* invoke);
895   void LoadClassRootForIntrinsic(vixl::aarch64::Register reg, ClassRoot class_root);
896 
897   void EmitLinkerPatches(ArenaVector<linker::LinkerPatch>* linker_patches) override;
898   bool NeedsThunkCode(const linker::LinkerPatch& patch) const override;
899   void EmitThunkCode(const linker::LinkerPatch& patch,
900                      /*out*/ ArenaVector<uint8_t>* code,
901                      /*out*/ std::string* debug_name) override;
902 
903   void EmitJitRootPatches(uint8_t* code, const uint8_t* roots_data) override;
904 
905   // Generate a GC root reference load:
906   //
907   //   root <- *(obj + offset)
908   //
909   // while honoring read barriers based on read_barrier_option.
910   void GenerateGcRootFieldLoad(HInstruction* instruction,
911                                Location root,
912                                vixl::aarch64::Register obj,
913                                uint32_t offset,
914                                vixl::aarch64::Label* fixup_label,
915                                ReadBarrierOption read_barrier_option);
916   // Generate MOV for the `old_value` in intrinsic and mark it with Baker read barrier.
917   void GenerateIntrinsicMoveWithBakerReadBarrier(vixl::aarch64::Register marked_old_value,
918                                                  vixl::aarch64::Register old_value);
919   // Fast path implementation of ReadBarrier::Barrier for a heap
920   // reference field load when Baker's read barriers are used.
921   // Overload suitable for Unsafe.getObject/-Volatile() intrinsic.
922   void GenerateFieldLoadWithBakerReadBarrier(HInstruction* instruction,
923                                              Location ref,
924                                              vixl::aarch64::Register obj,
925                                              const vixl::aarch64::MemOperand& src,
926                                              bool needs_null_check,
927                                              bool use_load_acquire);
928   // Fast path implementation of ReadBarrier::Barrier for a heap
929   // reference field load when Baker's read barriers are used.
930   void GenerateFieldLoadWithBakerReadBarrier(HInstruction* instruction,
931                                              Location ref,
932                                              vixl::aarch64::Register obj,
933                                              uint32_t offset,
934                                              Location maybe_temp,
935                                              bool needs_null_check,
936                                              bool use_load_acquire);
937   // Fast path implementation of ReadBarrier::Barrier for a heap
938   // reference array load when Baker's read barriers are used.
939   void GenerateArrayLoadWithBakerReadBarrier(HArrayGet* instruction,
940                                              Location ref,
941                                              vixl::aarch64::Register obj,
942                                              uint32_t data_offset,
943                                              Location index,
944                                              bool needs_null_check);
945 
946   // Emit code checking the status of the Marking Register, and
947   // aborting the program if MR does not match the value stored in the
948   // art::Thread object. Code is only emitted in debug mode and if
949   // CompilerOptions::EmitRunTimeChecksInDebugMode returns true.
950   //
951   // Argument `code` is used to identify the different occurrences of
952   // MaybeGenerateMarkingRegisterCheck in the code generator, and is
953   // passed to the BRK instruction.
954   //
955   // If `temp_loc` is a valid location, it is expected to be a
956   // register and will be used as a temporary to generate code;
957   // otherwise, a temporary will be fetched from the core register
958   // scratch pool.
959   virtual void MaybeGenerateMarkingRegisterCheck(int code,
960                                                  Location temp_loc = Location::NoLocation());
961 
962   // Create slow path for a read barrier for a heap reference within `instruction`.
963   //
964   // This is a helper function for GenerateReadBarrierSlow() that has the same
965   // arguments. The creation and adding of the slow path is exposed for intrinsics
966   // that cannot use GenerateReadBarrierSlow() from their own slow paths.
967   SlowPathCodeARM64* AddReadBarrierSlowPath(HInstruction* instruction,
968                                             Location out,
969                                             Location ref,
970                                             Location obj,
971                                             uint32_t offset,
972                                             Location index);
973 
974   // Generate a read barrier for a heap reference within `instruction`
975   // using a slow path.
976   //
977   // A read barrier for an object reference read from the heap is
978   // implemented as a call to the artReadBarrierSlow runtime entry
979   // point, which is passed the values in locations `ref`, `obj`, and
980   // `offset`:
981   //
982   //   mirror::Object* artReadBarrierSlow(mirror::Object* ref,
983   //                                      mirror::Object* obj,
984   //                                      uint32_t offset);
985   //
986   // The `out` location contains the value returned by
987   // artReadBarrierSlow.
988   //
989   // When `index` is provided (i.e. for array accesses), the offset
990   // value passed to artReadBarrierSlow is adjusted to take `index`
991   // into account.
992   void GenerateReadBarrierSlow(HInstruction* instruction,
993                                Location out,
994                                Location ref,
995                                Location obj,
996                                uint32_t offset,
997                                Location index = Location::NoLocation());
998 
999   // If read barriers are enabled, generate a read barrier for a heap
1000   // reference using a slow path. If heap poisoning is enabled, also
1001   // unpoison the reference in `out`.
1002   void MaybeGenerateReadBarrierSlow(HInstruction* instruction,
1003                                     Location out,
1004                                     Location ref,
1005                                     Location obj,
1006                                     uint32_t offset,
1007                                     Location index = Location::NoLocation());
1008 
1009   // Generate a read barrier for a GC root within `instruction` using
1010   // a slow path.
1011   //
1012   // A read barrier for an object reference GC root is implemented as
1013   // a call to the artReadBarrierForRootSlow runtime entry point,
1014   // which is passed the value in location `root`:
1015   //
1016   //   mirror::Object* artReadBarrierForRootSlow(GcRoot<mirror::Object>* root);
1017   //
1018   // The `out` location contains the value returned by
1019   // artReadBarrierForRootSlow.
1020   void GenerateReadBarrierForRootSlow(HInstruction* instruction, Location out, Location root);
1021 
1022   void IncreaseFrame(size_t adjustment) override;
1023   void DecreaseFrame(size_t adjustment) override;
1024 
1025   void GenerateNop() override;
1026 
1027   void GenerateImplicitNullCheck(HNullCheck* instruction) override;
1028   void GenerateExplicitNullCheck(HNullCheck* instruction) override;
1029 
MaybeRecordImplicitNullCheck(HInstruction * instr)1030   void MaybeRecordImplicitNullCheck(HInstruction* instr) final {
1031     // The function must be only called within special scopes
1032     // (EmissionCheckScope, ExactAssemblyScope) which prevent generation of
1033     // veneer/literal pools by VIXL assembler.
1034     CHECK_EQ(GetVIXLAssembler()->ArePoolsBlocked(), true)
1035         << "The function must only be called within EmissionCheckScope or ExactAssemblyScope";
1036     CodeGenerator::MaybeRecordImplicitNullCheck(instr);
1037   }
1038 
1039   void MaybeGenerateInlineCacheCheck(HInstruction* instruction, vixl::aarch64::Register klass);
1040   void MaybeIncrementHotness(HSuspendCheck* suspend_check, bool is_frame_entry);
1041 
1042   bool CanUseImplicitSuspendCheck() const;
1043 
1044  private:
1045   // Encoding of thunk type and data for link-time generated thunks for Baker read barriers.
1046 
1047   enum class BakerReadBarrierKind : uint8_t {
1048     kField,     // Field get or array get with constant offset (i.e. constant index).
1049     kAcquire,   // Volatile field get.
1050     kArray,     // Array get with index in register.
1051     kGcRoot,    // GC root load.
1052     kLast = kGcRoot
1053   };
1054 
1055   static constexpr uint32_t kBakerReadBarrierInvalidEncodedReg = /* sp/zr is invalid */ 31u;
1056 
1057   static constexpr size_t kBitsForBakerReadBarrierKind =
1058       MinimumBitsToStore(static_cast<size_t>(BakerReadBarrierKind::kLast));
1059   static constexpr size_t kBakerReadBarrierBitsForRegister =
1060       MinimumBitsToStore(kBakerReadBarrierInvalidEncodedReg);
1061   using BakerReadBarrierKindField =
1062       BitField<BakerReadBarrierKind, 0, kBitsForBakerReadBarrierKind>;
1063   using BakerReadBarrierFirstRegField =
1064       BitField<uint32_t, kBitsForBakerReadBarrierKind, kBakerReadBarrierBitsForRegister>;
1065   using BakerReadBarrierSecondRegField =
1066       BitField<uint32_t,
1067                kBitsForBakerReadBarrierKind + kBakerReadBarrierBitsForRegister,
1068                kBakerReadBarrierBitsForRegister>;
1069 
CheckValidReg(uint32_t reg)1070   static void CheckValidReg(uint32_t reg) {
1071     DCHECK(reg < vixl::aarch64::lr.GetCode() &&
1072            reg != vixl::aarch64::ip0.GetCode() &&
1073            reg != vixl::aarch64::ip1.GetCode()) << reg;
1074   }
1075 
EncodeBakerReadBarrierFieldData(uint32_t base_reg,uint32_t holder_reg)1076   static inline uint32_t EncodeBakerReadBarrierFieldData(uint32_t base_reg, uint32_t holder_reg) {
1077     CheckValidReg(base_reg);
1078     CheckValidReg(holder_reg);
1079     return BakerReadBarrierKindField::Encode(BakerReadBarrierKind::kField) |
1080            BakerReadBarrierFirstRegField::Encode(base_reg) |
1081            BakerReadBarrierSecondRegField::Encode(holder_reg);
1082   }
1083 
EncodeBakerReadBarrierAcquireData(uint32_t base_reg,uint32_t holder_reg)1084   static inline uint32_t EncodeBakerReadBarrierAcquireData(uint32_t base_reg, uint32_t holder_reg) {
1085     CheckValidReg(base_reg);
1086     CheckValidReg(holder_reg);
1087     DCHECK_NE(base_reg, holder_reg);
1088     return BakerReadBarrierKindField::Encode(BakerReadBarrierKind::kAcquire) |
1089            BakerReadBarrierFirstRegField::Encode(base_reg) |
1090            BakerReadBarrierSecondRegField::Encode(holder_reg);
1091   }
1092 
EncodeBakerReadBarrierArrayData(uint32_t base_reg)1093   static inline uint32_t EncodeBakerReadBarrierArrayData(uint32_t base_reg) {
1094     CheckValidReg(base_reg);
1095     return BakerReadBarrierKindField::Encode(BakerReadBarrierKind::kArray) |
1096            BakerReadBarrierFirstRegField::Encode(base_reg) |
1097            BakerReadBarrierSecondRegField::Encode(kBakerReadBarrierInvalidEncodedReg);
1098   }
1099 
EncodeBakerReadBarrierGcRootData(uint32_t root_reg)1100   static inline uint32_t EncodeBakerReadBarrierGcRootData(uint32_t root_reg) {
1101     CheckValidReg(root_reg);
1102     return BakerReadBarrierKindField::Encode(BakerReadBarrierKind::kGcRoot) |
1103            BakerReadBarrierFirstRegField::Encode(root_reg) |
1104            BakerReadBarrierSecondRegField::Encode(kBakerReadBarrierInvalidEncodedReg);
1105   }
1106 
1107   void CompileBakerReadBarrierThunk(Arm64Assembler& assembler,
1108                                     uint32_t encoded_data,
1109                                     /*out*/ std::string* debug_name);
1110 
1111   // The PcRelativePatchInfo is used for PC-relative addressing of methods/strings/types,
1112   // whether through .data.img.rel.ro, .bss, or directly in the boot image.
1113   struct PcRelativePatchInfo : PatchInfo<vixl::aarch64::Label> {
PcRelativePatchInfoPcRelativePatchInfo1114     PcRelativePatchInfo(const DexFile* dex_file, uint32_t off_or_idx)
1115         : PatchInfo<vixl::aarch64::Label>(dex_file, off_or_idx), pc_insn_label() { }
1116 
1117     vixl::aarch64::Label* pc_insn_label;
1118   };
1119 
1120   struct BakerReadBarrierPatchInfo {
BakerReadBarrierPatchInfoBakerReadBarrierPatchInfo1121     explicit BakerReadBarrierPatchInfo(uint32_t data) : label(), custom_data(data) { }
1122 
1123     vixl::aarch64::Label label;
1124     uint32_t custom_data;
1125   };
1126 
1127   vixl::aarch64::Label* NewPcRelativePatch(const DexFile* dex_file,
1128                                            uint32_t offset_or_index,
1129                                            vixl::aarch64::Label* adrp_label,
1130                                            ArenaDeque<PcRelativePatchInfo>* patches);
1131 
1132   void EmitJumpTables();
1133 
1134   template <linker::LinkerPatch (*Factory)(size_t, const DexFile*, uint32_t, uint32_t)>
1135   static void EmitPcRelativeLinkerPatches(const ArenaDeque<PcRelativePatchInfo>& infos,
1136                                           ArenaVector<linker::LinkerPatch>* linker_patches);
1137 
1138   // Returns whether SVE features are supported and should be used.
1139   bool ShouldUseSVE() const;
1140 
1141   // Labels for each block that will be compiled.
1142   // We use a deque so that the `vixl::aarch64::Label` objects do not move in memory.
1143   ArenaDeque<vixl::aarch64::Label> block_labels_;  // Indexed by block id.
1144   vixl::aarch64::Label frame_entry_label_;
1145   ArenaVector<std::unique_ptr<JumpTableARM64>> jump_tables_;
1146 
1147   LocationsBuilderARM64Neon location_builder_neon_;
1148   InstructionCodeGeneratorARM64Neon instruction_visitor_neon_;
1149   LocationsBuilderARM64Sve location_builder_sve_;
1150   InstructionCodeGeneratorARM64Sve instruction_visitor_sve_;
1151 
1152   LocationsBuilderARM64* location_builder_;
1153   InstructionCodeGeneratorARM64* instruction_visitor_;
1154   ParallelMoveResolverARM64 move_resolver_;
1155   Arm64Assembler assembler_;
1156 
1157   // PC-relative method patch info for kBootImageLinkTimePcRelative.
1158   ArenaDeque<PcRelativePatchInfo> boot_image_method_patches_;
1159   // PC-relative method patch info for kBssEntry.
1160   ArenaDeque<PcRelativePatchInfo> method_bss_entry_patches_;
1161   // PC-relative type patch info for kBootImageLinkTimePcRelative.
1162   ArenaDeque<PcRelativePatchInfo> boot_image_type_patches_;
1163   // PC-relative type patch info for kAppImageRelRo.
1164   ArenaDeque<PcRelativePatchInfo> app_image_type_patches_;
1165   // PC-relative type patch info for kBssEntry.
1166   ArenaDeque<PcRelativePatchInfo> type_bss_entry_patches_;
1167   // PC-relative public type patch info for kBssEntryPublic.
1168   ArenaDeque<PcRelativePatchInfo> public_type_bss_entry_patches_;
1169   // PC-relative package type patch info for kBssEntryPackage.
1170   ArenaDeque<PcRelativePatchInfo> package_type_bss_entry_patches_;
1171   // PC-relative String patch info for kBootImageLinkTimePcRelative.
1172   ArenaDeque<PcRelativePatchInfo> boot_image_string_patches_;
1173   // PC-relative String patch info for kBssEntry.
1174   ArenaDeque<PcRelativePatchInfo> string_bss_entry_patches_;
1175   // PC-relative method patch info for kBootImageLinkTimePcRelative+kCallCriticalNative.
1176   ArenaDeque<PcRelativePatchInfo> boot_image_jni_entrypoint_patches_;
1177   // PC-relative patch info for IntrinsicObjects for the boot image,
1178   // and for method/type/string patches for kBootImageRelRo otherwise.
1179   ArenaDeque<PcRelativePatchInfo> boot_image_other_patches_;
1180   // Patch info for calls to entrypoint dispatch thunks. Used for slow paths.
1181   ArenaDeque<PatchInfo<vixl::aarch64::Label>> call_entrypoint_patches_;
1182   // Baker read barrier patch info.
1183   ArenaDeque<BakerReadBarrierPatchInfo> baker_read_barrier_patches_;
1184 
1185   JitPatchesARM64 jit_patches_;
1186 
1187   // Baker read barrier slow paths, mapping custom data (uint32_t) to label.
1188   // Wrap the label to work around vixl::aarch64::Label being non-copyable
1189   // and non-moveable and as such unusable in ArenaSafeMap<>.
1190   struct LabelWrapper {
LabelWrapperLabelWrapper1191     LabelWrapper(const LabelWrapper& src)
1192         : label() {
1193       DCHECK(!src.label.IsLinked() && !src.label.IsBound());
1194     }
1195     LabelWrapper() = default;
1196     vixl::aarch64::Label label;
1197   };
1198   ArenaSafeMap<uint32_t, LabelWrapper> jit_baker_read_barrier_slow_paths_;
1199 
1200   friend class linker::Arm64RelativePatcherTest;
1201   DISALLOW_COPY_AND_ASSIGN(CodeGeneratorARM64);
1202 };
1203 
GetAssembler()1204 inline Arm64Assembler* ParallelMoveResolverARM64::GetAssembler() const {
1205   return codegen_->GetAssembler();
1206 }
1207 
1208 }  // namespace arm64
1209 }  // namespace art
1210 
1211 #endif  // ART_COMPILER_OPTIMIZING_CODE_GENERATOR_ARM64_H_
1212