1 /*
2 * Copyright (C) 2014 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 #ifndef ART_COMPILER_OPTIMIZING_CODE_GENERATOR_ARM64_H_
18 #define ART_COMPILER_OPTIMIZING_CODE_GENERATOR_ARM64_H_
19
20 #include "arch/arm64/quick_method_frame_info_arm64.h"
21 #include "code_generator.h"
22 #include "common_arm64.h"
23 #include "dex_file_types.h"
24 #include "driver/compiler_options.h"
25 #include "nodes.h"
26 #include "parallel_move_resolver.h"
27 #include "string_reference.h"
28 #include "utils/arm64/assembler_arm64.h"
29 #include "utils/type_reference.h"
30
31 // TODO(VIXL): Make VIXL compile with -Wshadow.
32 #pragma GCC diagnostic push
33 #pragma GCC diagnostic ignored "-Wshadow"
34 #include "aarch64/disasm-aarch64.h"
35 #include "aarch64/macro-assembler-aarch64.h"
36 #pragma GCC diagnostic pop
37
38 namespace art {
39 namespace arm64 {
40
41 class CodeGeneratorARM64;
42
43 // Use a local definition to prevent copying mistakes.
44 static constexpr size_t kArm64WordSize = static_cast<size_t>(kArm64PointerSize);
45
46 // These constants are used as an approximate margin when emission of veneer and literal pools
47 // must be blocked.
48 static constexpr int kMaxMacroInstructionSizeInBytes = 15 * vixl::aarch64::kInstructionSize;
49 static constexpr int kInvokeCodeMarginSizeInBytes = 6 * kMaxMacroInstructionSizeInBytes;
50
51 static const vixl::aarch64::Register kParameterCoreRegisters[] = {
52 vixl::aarch64::x1,
53 vixl::aarch64::x2,
54 vixl::aarch64::x3,
55 vixl::aarch64::x4,
56 vixl::aarch64::x5,
57 vixl::aarch64::x6,
58 vixl::aarch64::x7
59 };
60 static constexpr size_t kParameterCoreRegistersLength = arraysize(kParameterCoreRegisters);
61 static const vixl::aarch64::FPRegister kParameterFPRegisters[] = {
62 vixl::aarch64::d0,
63 vixl::aarch64::d1,
64 vixl::aarch64::d2,
65 vixl::aarch64::d3,
66 vixl::aarch64::d4,
67 vixl::aarch64::d5,
68 vixl::aarch64::d6,
69 vixl::aarch64::d7
70 };
71 static constexpr size_t kParameterFPRegistersLength = arraysize(kParameterFPRegisters);
72
73 // Thread Register
74 const vixl::aarch64::Register tr = vixl::aarch64::x19;
75 // Method register on invoke.
76 static const vixl::aarch64::Register kArtMethodRegister = vixl::aarch64::x0;
77 const vixl::aarch64::CPURegList vixl_reserved_core_registers(vixl::aarch64::ip0,
78 vixl::aarch64::ip1);
79 const vixl::aarch64::CPURegList vixl_reserved_fp_registers(vixl::aarch64::d31);
80
81 const vixl::aarch64::CPURegList runtime_reserved_core_registers(tr, vixl::aarch64::lr);
82
83 // Callee-saved registers AAPCS64 (without x19 - Thread Register)
84 const vixl::aarch64::CPURegList callee_saved_core_registers(vixl::aarch64::CPURegister::kRegister,
85 vixl::aarch64::kXRegSize,
86 vixl::aarch64::x20.GetCode(),
87 vixl::aarch64::x30.GetCode());
88 const vixl::aarch64::CPURegList callee_saved_fp_registers(vixl::aarch64::CPURegister::kFPRegister,
89 vixl::aarch64::kDRegSize,
90 vixl::aarch64::d8.GetCode(),
91 vixl::aarch64::d15.GetCode());
92 Location ARM64ReturnLocation(Primitive::Type return_type);
93
94 class SlowPathCodeARM64 : public SlowPathCode {
95 public:
SlowPathCodeARM64(HInstruction * instruction)96 explicit SlowPathCodeARM64(HInstruction* instruction)
97 : SlowPathCode(instruction), entry_label_(), exit_label_() {}
98
GetEntryLabel()99 vixl::aarch64::Label* GetEntryLabel() { return &entry_label_; }
GetExitLabel()100 vixl::aarch64::Label* GetExitLabel() { return &exit_label_; }
101
102 void SaveLiveRegisters(CodeGenerator* codegen, LocationSummary* locations) OVERRIDE;
103 void RestoreLiveRegisters(CodeGenerator* codegen, LocationSummary* locations) OVERRIDE;
104
105 private:
106 vixl::aarch64::Label entry_label_;
107 vixl::aarch64::Label exit_label_;
108
109 DISALLOW_COPY_AND_ASSIGN(SlowPathCodeARM64);
110 };
111
112 class JumpTableARM64 : public DeletableArenaObject<kArenaAllocSwitchTable> {
113 public:
JumpTableARM64(HPackedSwitch * switch_instr)114 explicit JumpTableARM64(HPackedSwitch* switch_instr)
115 : switch_instr_(switch_instr), table_start_() {}
116
GetTableStartLabel()117 vixl::aarch64::Label* GetTableStartLabel() { return &table_start_; }
118
119 void EmitTable(CodeGeneratorARM64* codegen);
120
121 private:
122 HPackedSwitch* const switch_instr_;
123 vixl::aarch64::Label table_start_;
124
125 DISALLOW_COPY_AND_ASSIGN(JumpTableARM64);
126 };
127
128 static const vixl::aarch64::Register kRuntimeParameterCoreRegisters[] =
129 { vixl::aarch64::x0,
130 vixl::aarch64::x1,
131 vixl::aarch64::x2,
132 vixl::aarch64::x3,
133 vixl::aarch64::x4,
134 vixl::aarch64::x5,
135 vixl::aarch64::x6,
136 vixl::aarch64::x7 };
137 static constexpr size_t kRuntimeParameterCoreRegistersLength =
138 arraysize(kRuntimeParameterCoreRegisters);
139 static const vixl::aarch64::FPRegister kRuntimeParameterFpuRegisters[] =
140 { vixl::aarch64::d0,
141 vixl::aarch64::d1,
142 vixl::aarch64::d2,
143 vixl::aarch64::d3,
144 vixl::aarch64::d4,
145 vixl::aarch64::d5,
146 vixl::aarch64::d6,
147 vixl::aarch64::d7 };
148 static constexpr size_t kRuntimeParameterFpuRegistersLength =
149 arraysize(kRuntimeParameterCoreRegisters);
150
151 class InvokeRuntimeCallingConvention : public CallingConvention<vixl::aarch64::Register,
152 vixl::aarch64::FPRegister> {
153 public:
154 static constexpr size_t kParameterCoreRegistersLength = arraysize(kParameterCoreRegisters);
155
InvokeRuntimeCallingConvention()156 InvokeRuntimeCallingConvention()
157 : CallingConvention(kRuntimeParameterCoreRegisters,
158 kRuntimeParameterCoreRegistersLength,
159 kRuntimeParameterFpuRegisters,
160 kRuntimeParameterFpuRegistersLength,
161 kArm64PointerSize) {}
162
163 Location GetReturnLocation(Primitive::Type return_type);
164
165 private:
166 DISALLOW_COPY_AND_ASSIGN(InvokeRuntimeCallingConvention);
167 };
168
169 class InvokeDexCallingConvention : public CallingConvention<vixl::aarch64::Register,
170 vixl::aarch64::FPRegister> {
171 public:
InvokeDexCallingConvention()172 InvokeDexCallingConvention()
173 : CallingConvention(kParameterCoreRegisters,
174 kParameterCoreRegistersLength,
175 kParameterFPRegisters,
176 kParameterFPRegistersLength,
177 kArm64PointerSize) {}
178
GetReturnLocation(Primitive::Type return_type)179 Location GetReturnLocation(Primitive::Type return_type) const {
180 return ARM64ReturnLocation(return_type);
181 }
182
183
184 private:
185 DISALLOW_COPY_AND_ASSIGN(InvokeDexCallingConvention);
186 };
187
188 class InvokeDexCallingConventionVisitorARM64 : public InvokeDexCallingConventionVisitor {
189 public:
InvokeDexCallingConventionVisitorARM64()190 InvokeDexCallingConventionVisitorARM64() {}
~InvokeDexCallingConventionVisitorARM64()191 virtual ~InvokeDexCallingConventionVisitorARM64() {}
192
193 Location GetNextLocation(Primitive::Type type) OVERRIDE;
GetReturnLocation(Primitive::Type return_type)194 Location GetReturnLocation(Primitive::Type return_type) const OVERRIDE {
195 return calling_convention.GetReturnLocation(return_type);
196 }
197 Location GetMethodLocation() const OVERRIDE;
198
199 private:
200 InvokeDexCallingConvention calling_convention;
201
202 DISALLOW_COPY_AND_ASSIGN(InvokeDexCallingConventionVisitorARM64);
203 };
204
205 class FieldAccessCallingConventionARM64 : public FieldAccessCallingConvention {
206 public:
FieldAccessCallingConventionARM64()207 FieldAccessCallingConventionARM64() {}
208
GetObjectLocation()209 Location GetObjectLocation() const OVERRIDE {
210 return helpers::LocationFrom(vixl::aarch64::x1);
211 }
GetFieldIndexLocation()212 Location GetFieldIndexLocation() const OVERRIDE {
213 return helpers::LocationFrom(vixl::aarch64::x0);
214 }
GetReturnLocation(Primitive::Type type ATTRIBUTE_UNUSED)215 Location GetReturnLocation(Primitive::Type type ATTRIBUTE_UNUSED) const OVERRIDE {
216 return helpers::LocationFrom(vixl::aarch64::x0);
217 }
GetSetValueLocation(Primitive::Type type ATTRIBUTE_UNUSED,bool is_instance)218 Location GetSetValueLocation(Primitive::Type type ATTRIBUTE_UNUSED,
219 bool is_instance) const OVERRIDE {
220 return is_instance
221 ? helpers::LocationFrom(vixl::aarch64::x2)
222 : helpers::LocationFrom(vixl::aarch64::x1);
223 }
GetFpuLocation(Primitive::Type type ATTRIBUTE_UNUSED)224 Location GetFpuLocation(Primitive::Type type ATTRIBUTE_UNUSED) const OVERRIDE {
225 return helpers::LocationFrom(vixl::aarch64::d0);
226 }
227
228 private:
229 DISALLOW_COPY_AND_ASSIGN(FieldAccessCallingConventionARM64);
230 };
231
232 class InstructionCodeGeneratorARM64 : public InstructionCodeGenerator {
233 public:
234 InstructionCodeGeneratorARM64(HGraph* graph, CodeGeneratorARM64* codegen);
235
236 #define DECLARE_VISIT_INSTRUCTION(name, super) \
237 void Visit##name(H##name* instr) OVERRIDE;
238
239 FOR_EACH_CONCRETE_INSTRUCTION_COMMON(DECLARE_VISIT_INSTRUCTION)
FOR_EACH_CONCRETE_INSTRUCTION_ARM64(DECLARE_VISIT_INSTRUCTION)240 FOR_EACH_CONCRETE_INSTRUCTION_ARM64(DECLARE_VISIT_INSTRUCTION)
241 FOR_EACH_CONCRETE_INSTRUCTION_SHARED(DECLARE_VISIT_INSTRUCTION)
242
243 #undef DECLARE_VISIT_INSTRUCTION
244
245 void VisitInstruction(HInstruction* instruction) OVERRIDE {
246 LOG(FATAL) << "Unreachable instruction " << instruction->DebugName()
247 << " (id " << instruction->GetId() << ")";
248 }
249
GetAssembler()250 Arm64Assembler* GetAssembler() const { return assembler_; }
GetVIXLAssembler()251 vixl::aarch64::MacroAssembler* GetVIXLAssembler() { return GetAssembler()->GetVIXLAssembler(); }
252
253 private:
254 void GenerateClassInitializationCheck(SlowPathCodeARM64* slow_path,
255 vixl::aarch64::Register class_reg);
256 void GenerateSuspendCheck(HSuspendCheck* instruction, HBasicBlock* successor);
257 void HandleBinaryOp(HBinaryOperation* instr);
258
259 void HandleFieldSet(HInstruction* instruction,
260 const FieldInfo& field_info,
261 bool value_can_be_null);
262 void HandleFieldGet(HInstruction* instruction, const FieldInfo& field_info);
263 void HandleCondition(HCondition* instruction);
264
265 // Generate a heap reference load using one register `out`:
266 //
267 // out <- *(out + offset)
268 //
269 // while honoring heap poisoning and/or read barriers (if any).
270 //
271 // Location `maybe_temp` is used when generating a read barrier and
272 // shall be a register in that case; it may be an invalid location
273 // otherwise.
274 void GenerateReferenceLoadOneRegister(HInstruction* instruction,
275 Location out,
276 uint32_t offset,
277 Location maybe_temp,
278 ReadBarrierOption read_barrier_option);
279 // Generate a heap reference load using two different registers
280 // `out` and `obj`:
281 //
282 // out <- *(obj + offset)
283 //
284 // while honoring heap poisoning and/or read barriers (if any).
285 //
286 // Location `maybe_temp` is used when generating a Baker's (fast
287 // path) read barrier and shall be a register in that case; it may
288 // be an invalid location otherwise.
289 void GenerateReferenceLoadTwoRegisters(HInstruction* instruction,
290 Location out,
291 Location obj,
292 uint32_t offset,
293 Location maybe_temp,
294 ReadBarrierOption read_barrier_option);
295 // Generate a GC root reference load:
296 //
297 // root <- *(obj + offset)
298 //
299 // while honoring read barriers based on read_barrier_option.
300 void GenerateGcRootFieldLoad(HInstruction* instruction,
301 Location root,
302 vixl::aarch64::Register obj,
303 uint32_t offset,
304 vixl::aarch64::Label* fixup_label,
305 ReadBarrierOption read_barrier_option);
306
307 // Generate a floating-point comparison.
308 void GenerateFcmp(HInstruction* instruction);
309
310 void HandleShift(HBinaryOperation* instr);
311 void GenerateTestAndBranch(HInstruction* instruction,
312 size_t condition_input_index,
313 vixl::aarch64::Label* true_target,
314 vixl::aarch64::Label* false_target);
315 void DivRemOneOrMinusOne(HBinaryOperation* instruction);
316 void DivRemByPowerOfTwo(HBinaryOperation* instruction);
317 void GenerateDivRemWithAnyConstant(HBinaryOperation* instruction);
318 void GenerateDivRemIntegral(HBinaryOperation* instruction);
319 void HandleGoto(HInstruction* got, HBasicBlock* successor);
320
321 vixl::aarch64::MemOperand CreateVecMemRegisters(
322 HVecMemoryOperation* instruction,
323 Location* reg_loc,
324 bool is_load,
325 // This function may acquire a scratch register.
326 vixl::aarch64::UseScratchRegisterScope* temps_scope);
327
328 Arm64Assembler* const assembler_;
329 CodeGeneratorARM64* const codegen_;
330
331 DISALLOW_COPY_AND_ASSIGN(InstructionCodeGeneratorARM64);
332 };
333
334 class LocationsBuilderARM64 : public HGraphVisitor {
335 public:
LocationsBuilderARM64(HGraph * graph,CodeGeneratorARM64 * codegen)336 LocationsBuilderARM64(HGraph* graph, CodeGeneratorARM64* codegen)
337 : HGraphVisitor(graph), codegen_(codegen) {}
338
339 #define DECLARE_VISIT_INSTRUCTION(name, super) \
340 void Visit##name(H##name* instr) OVERRIDE;
341
342 FOR_EACH_CONCRETE_INSTRUCTION_COMMON(DECLARE_VISIT_INSTRUCTION)
FOR_EACH_CONCRETE_INSTRUCTION_ARM64(DECLARE_VISIT_INSTRUCTION)343 FOR_EACH_CONCRETE_INSTRUCTION_ARM64(DECLARE_VISIT_INSTRUCTION)
344 FOR_EACH_CONCRETE_INSTRUCTION_SHARED(DECLARE_VISIT_INSTRUCTION)
345
346 #undef DECLARE_VISIT_INSTRUCTION
347
348 void VisitInstruction(HInstruction* instruction) OVERRIDE {
349 LOG(FATAL) << "Unreachable instruction " << instruction->DebugName()
350 << " (id " << instruction->GetId() << ")";
351 }
352
353 private:
354 void HandleBinaryOp(HBinaryOperation* instr);
355 void HandleFieldSet(HInstruction* instruction);
356 void HandleFieldGet(HInstruction* instruction, const FieldInfo& field_info);
357 void HandleInvoke(HInvoke* instr);
358 void HandleCondition(HCondition* instruction);
359 void HandleShift(HBinaryOperation* instr);
360
361 CodeGeneratorARM64* const codegen_;
362 InvokeDexCallingConventionVisitorARM64 parameter_visitor_;
363
364 DISALLOW_COPY_AND_ASSIGN(LocationsBuilderARM64);
365 };
366
367 class ParallelMoveResolverARM64 : public ParallelMoveResolverNoSwap {
368 public:
ParallelMoveResolverARM64(ArenaAllocator * allocator,CodeGeneratorARM64 * codegen)369 ParallelMoveResolverARM64(ArenaAllocator* allocator, CodeGeneratorARM64* codegen)
370 : ParallelMoveResolverNoSwap(allocator), codegen_(codegen), vixl_temps_() {}
371
372 protected:
373 void PrepareForEmitNativeCode() OVERRIDE;
374 void FinishEmitNativeCode() OVERRIDE;
375 Location AllocateScratchLocationFor(Location::Kind kind) OVERRIDE;
376 void FreeScratchLocation(Location loc) OVERRIDE;
377 void EmitMove(size_t index) OVERRIDE;
378
379 private:
380 Arm64Assembler* GetAssembler() const;
GetVIXLAssembler()381 vixl::aarch64::MacroAssembler* GetVIXLAssembler() const {
382 return GetAssembler()->GetVIXLAssembler();
383 }
384
385 CodeGeneratorARM64* const codegen_;
386 vixl::aarch64::UseScratchRegisterScope vixl_temps_;
387
388 DISALLOW_COPY_AND_ASSIGN(ParallelMoveResolverARM64);
389 };
390
391 class CodeGeneratorARM64 : public CodeGenerator {
392 public:
393 CodeGeneratorARM64(HGraph* graph,
394 const Arm64InstructionSetFeatures& isa_features,
395 const CompilerOptions& compiler_options,
396 OptimizingCompilerStats* stats = nullptr);
~CodeGeneratorARM64()397 virtual ~CodeGeneratorARM64() {}
398
399 void GenerateFrameEntry() OVERRIDE;
400 void GenerateFrameExit() OVERRIDE;
401
402 vixl::aarch64::CPURegList GetFramePreservedCoreRegisters() const;
403 vixl::aarch64::CPURegList GetFramePreservedFPRegisters() const;
404
405 void Bind(HBasicBlock* block) OVERRIDE;
406
GetLabelOf(HBasicBlock * block)407 vixl::aarch64::Label* GetLabelOf(HBasicBlock* block) {
408 block = FirstNonEmptyBlock(block);
409 return &(block_labels_[block->GetBlockId()]);
410 }
411
GetWordSize()412 size_t GetWordSize() const OVERRIDE {
413 return kArm64WordSize;
414 }
415
GetFloatingPointSpillSlotSize()416 size_t GetFloatingPointSpillSlotSize() const OVERRIDE {
417 return GetGraph()->HasSIMD()
418 ? 2 * kArm64WordSize // 16 bytes == 2 arm64 words for each spill
419 : 1 * kArm64WordSize; // 8 bytes == 1 arm64 words for each spill
420 }
421
GetAddressOf(HBasicBlock * block)422 uintptr_t GetAddressOf(HBasicBlock* block) OVERRIDE {
423 vixl::aarch64::Label* block_entry_label = GetLabelOf(block);
424 DCHECK(block_entry_label->IsBound());
425 return block_entry_label->GetLocation();
426 }
427
GetLocationBuilder()428 HGraphVisitor* GetLocationBuilder() OVERRIDE { return &location_builder_; }
GetInstructionVisitor()429 HGraphVisitor* GetInstructionVisitor() OVERRIDE { return &instruction_visitor_; }
GetAssembler()430 Arm64Assembler* GetAssembler() OVERRIDE { return &assembler_; }
GetAssembler()431 const Arm64Assembler& GetAssembler() const OVERRIDE { return assembler_; }
GetVIXLAssembler()432 vixl::aarch64::MacroAssembler* GetVIXLAssembler() { return GetAssembler()->GetVIXLAssembler(); }
433
434 // Emit a write barrier.
435 void MarkGCCard(vixl::aarch64::Register object,
436 vixl::aarch64::Register value,
437 bool value_can_be_null);
438
439 void GenerateMemoryBarrier(MemBarrierKind kind);
440
441 // Register allocation.
442
443 void SetupBlockedRegisters() const OVERRIDE;
444
445 size_t SaveCoreRegister(size_t stack_index, uint32_t reg_id) OVERRIDE;
446 size_t RestoreCoreRegister(size_t stack_index, uint32_t reg_id) OVERRIDE;
447 size_t SaveFloatingPointRegister(size_t stack_index, uint32_t reg_id) OVERRIDE;
448 size_t RestoreFloatingPointRegister(size_t stack_index, uint32_t reg_id) OVERRIDE;
449
450 // The number of registers that can be allocated. The register allocator may
451 // decide to reserve and not use a few of them.
452 // We do not consider registers sp, xzr, wzr. They are either not allocatable
453 // (xzr, wzr), or make for poor allocatable registers (sp alignment
454 // requirements, etc.). This also facilitates our task as all other registers
455 // can easily be mapped via to or from their type and index or code.
456 static const int kNumberOfAllocatableRegisters = vixl::aarch64::kNumberOfRegisters - 1;
457 static const int kNumberOfAllocatableFPRegisters = vixl::aarch64::kNumberOfFPRegisters;
458 static constexpr int kNumberOfAllocatableRegisterPairs = 0;
459
460 void DumpCoreRegister(std::ostream& stream, int reg) const OVERRIDE;
461 void DumpFloatingPointRegister(std::ostream& stream, int reg) const OVERRIDE;
462
GetInstructionSet()463 InstructionSet GetInstructionSet() const OVERRIDE {
464 return InstructionSet::kArm64;
465 }
466
GetInstructionSetFeatures()467 const Arm64InstructionSetFeatures& GetInstructionSetFeatures() const {
468 return isa_features_;
469 }
470
Initialize()471 void Initialize() OVERRIDE {
472 block_labels_.resize(GetGraph()->GetBlocks().size());
473 }
474
475 // We want to use the STP and LDP instructions to spill and restore registers for slow paths.
476 // These instructions can only encode offsets that are multiples of the register size accessed.
GetPreferredSlotsAlignment()477 uint32_t GetPreferredSlotsAlignment() const OVERRIDE { return vixl::aarch64::kXRegSizeInBytes; }
478
CreateJumpTable(HPackedSwitch * switch_instr)479 JumpTableARM64* CreateJumpTable(HPackedSwitch* switch_instr) {
480 jump_tables_.emplace_back(new (GetGraph()->GetArena()) JumpTableARM64(switch_instr));
481 return jump_tables_.back().get();
482 }
483
484 void Finalize(CodeAllocator* allocator) OVERRIDE;
485
486 // Code generation helpers.
487 void MoveConstant(vixl::aarch64::CPURegister destination, HConstant* constant);
488 void MoveConstant(Location destination, int32_t value) OVERRIDE;
489 void MoveLocation(Location dst, Location src, Primitive::Type dst_type) OVERRIDE;
490 void AddLocationAsTemp(Location location, LocationSummary* locations) OVERRIDE;
491
492 void Load(Primitive::Type type,
493 vixl::aarch64::CPURegister dst,
494 const vixl::aarch64::MemOperand& src);
495 void Store(Primitive::Type type,
496 vixl::aarch64::CPURegister src,
497 const vixl::aarch64::MemOperand& dst);
498 void LoadAcquire(HInstruction* instruction,
499 vixl::aarch64::CPURegister dst,
500 const vixl::aarch64::MemOperand& src,
501 bool needs_null_check);
502 void StoreRelease(HInstruction* instruction,
503 Primitive::Type type,
504 vixl::aarch64::CPURegister src,
505 const vixl::aarch64::MemOperand& dst,
506 bool needs_null_check);
507
508 // Generate code to invoke a runtime entry point.
509 void InvokeRuntime(QuickEntrypointEnum entrypoint,
510 HInstruction* instruction,
511 uint32_t dex_pc,
512 SlowPathCode* slow_path = nullptr) OVERRIDE;
513
514 // Generate code to invoke a runtime entry point, but do not record
515 // PC-related information in a stack map.
516 void InvokeRuntimeWithoutRecordingPcInfo(int32_t entry_point_offset,
517 HInstruction* instruction,
518 SlowPathCode* slow_path);
519
GetMoveResolver()520 ParallelMoveResolverARM64* GetMoveResolver() OVERRIDE { return &move_resolver_; }
521
NeedsTwoRegisters(Primitive::Type type ATTRIBUTE_UNUSED)522 bool NeedsTwoRegisters(Primitive::Type type ATTRIBUTE_UNUSED) const OVERRIDE {
523 return false;
524 }
525
526 // Check if the desired_string_load_kind is supported. If it is, return it,
527 // otherwise return a fall-back kind that should be used instead.
528 HLoadString::LoadKind GetSupportedLoadStringKind(
529 HLoadString::LoadKind desired_string_load_kind) OVERRIDE;
530
531 // Check if the desired_class_load_kind is supported. If it is, return it,
532 // otherwise return a fall-back kind that should be used instead.
533 HLoadClass::LoadKind GetSupportedLoadClassKind(
534 HLoadClass::LoadKind desired_class_load_kind) OVERRIDE;
535
536 // Check if the desired_dispatch_info is supported. If it is, return it,
537 // otherwise return a fall-back info that should be used instead.
538 HInvokeStaticOrDirect::DispatchInfo GetSupportedInvokeStaticOrDirectDispatch(
539 const HInvokeStaticOrDirect::DispatchInfo& desired_dispatch_info,
540 HInvokeStaticOrDirect* invoke) OVERRIDE;
541
542 Location GenerateCalleeMethodStaticOrDirectCall(HInvokeStaticOrDirect* invoke, Location temp);
543 void GenerateStaticOrDirectCall(HInvokeStaticOrDirect* invoke, Location temp) OVERRIDE;
544 void GenerateVirtualCall(HInvokeVirtual* invoke, Location temp) OVERRIDE;
545
MoveFromReturnRegister(Location trg ATTRIBUTE_UNUSED,Primitive::Type type ATTRIBUTE_UNUSED)546 void MoveFromReturnRegister(Location trg ATTRIBUTE_UNUSED,
547 Primitive::Type type ATTRIBUTE_UNUSED) OVERRIDE {
548 UNIMPLEMENTED(FATAL);
549 }
550
551 // Add a new PC-relative string patch for an instruction and return the label
552 // to be bound before the instruction. The instruction will be either the
553 // ADRP (pass `adrp_label = null`) or the ADD (pass `adrp_label` pointing
554 // to the associated ADRP patch label).
555 vixl::aarch64::Label* NewPcRelativeStringPatch(const DexFile& dex_file,
556 dex::StringIndex string_index,
557 vixl::aarch64::Label* adrp_label = nullptr);
558
559 // Add a new PC-relative type patch for an instruction and return the label
560 // to be bound before the instruction. The instruction will be either the
561 // ADRP (pass `adrp_label = null`) or the ADD (pass `adrp_label` pointing
562 // to the associated ADRP patch label).
563 vixl::aarch64::Label* NewPcRelativeTypePatch(const DexFile& dex_file,
564 dex::TypeIndex type_index,
565 vixl::aarch64::Label* adrp_label = nullptr);
566
567 // Add a new .bss entry type patch for an instruction and return the label
568 // to be bound before the instruction. The instruction will be either the
569 // ADRP (pass `adrp_label = null`) or the ADD (pass `adrp_label` pointing
570 // to the associated ADRP patch label).
571 vixl::aarch64::Label* NewBssEntryTypePatch(const DexFile& dex_file,
572 dex::TypeIndex type_index,
573 vixl::aarch64::Label* adrp_label = nullptr);
574
575 // Add a new PC-relative dex cache array patch for an instruction and return
576 // the label to be bound before the instruction. The instruction will be
577 // either the ADRP (pass `adrp_label = null`) or the LDR (pass `adrp_label`
578 // pointing to the associated ADRP patch label).
579 vixl::aarch64::Label* NewPcRelativeDexCacheArrayPatch(
580 const DexFile& dex_file,
581 uint32_t element_offset,
582 vixl::aarch64::Label* adrp_label = nullptr);
583
584 // Add a new baker read barrier patch and return the label to be bound
585 // before the CBNZ instruction.
586 vixl::aarch64::Label* NewBakerReadBarrierPatch(uint32_t custom_data);
587
588 vixl::aarch64::Literal<uint32_t>* DeduplicateBootImageStringLiteral(
589 const DexFile& dex_file,
590 dex::StringIndex string_index);
591 vixl::aarch64::Literal<uint32_t>* DeduplicateBootImageTypeLiteral(const DexFile& dex_file,
592 dex::TypeIndex type_index);
593 vixl::aarch64::Literal<uint32_t>* DeduplicateBootImageAddressLiteral(uint64_t address);
594 vixl::aarch64::Literal<uint32_t>* DeduplicateJitStringLiteral(const DexFile& dex_file,
595 dex::StringIndex string_index,
596 Handle<mirror::String> handle);
597 vixl::aarch64::Literal<uint32_t>* DeduplicateJitClassLiteral(const DexFile& dex_file,
598 dex::TypeIndex string_index,
599 Handle<mirror::Class> handle);
600
601 void EmitAdrpPlaceholder(vixl::aarch64::Label* fixup_label, vixl::aarch64::Register reg);
602 void EmitAddPlaceholder(vixl::aarch64::Label* fixup_label,
603 vixl::aarch64::Register out,
604 vixl::aarch64::Register base);
605 void EmitLdrOffsetPlaceholder(vixl::aarch64::Label* fixup_label,
606 vixl::aarch64::Register out,
607 vixl::aarch64::Register base);
608
609 void EmitLinkerPatches(ArenaVector<LinkerPatch>* linker_patches) OVERRIDE;
610
611 void EmitJitRootPatches(uint8_t* code, const uint8_t* roots_data) OVERRIDE;
612
613 // Fast path implementation of ReadBarrier::Barrier for a heap
614 // reference field load when Baker's read barriers are used.
615 void GenerateFieldLoadWithBakerReadBarrier(HInstruction* instruction,
616 Location ref,
617 vixl::aarch64::Register obj,
618 uint32_t offset,
619 Location maybe_temp,
620 bool needs_null_check,
621 bool use_load_acquire);
622 // Fast path implementation of ReadBarrier::Barrier for a heap
623 // reference array load when Baker's read barriers are used.
624 void GenerateArrayLoadWithBakerReadBarrier(HInstruction* instruction,
625 Location ref,
626 vixl::aarch64::Register obj,
627 uint32_t data_offset,
628 Location index,
629 vixl::aarch64::Register temp,
630 bool needs_null_check);
631 // Factored implementation, used by GenerateFieldLoadWithBakerReadBarrier,
632 // GenerateArrayLoadWithBakerReadBarrier and some intrinsics.
633 //
634 // Load the object reference located at the address
635 // `obj + offset + (index << scale_factor)`, held by object `obj`, into
636 // `ref`, and mark it if needed.
637 //
638 // If `always_update_field` is true, the value of the reference is
639 // atomically updated in the holder (`obj`).
640 void GenerateReferenceLoadWithBakerReadBarrier(HInstruction* instruction,
641 Location ref,
642 vixl::aarch64::Register obj,
643 uint32_t offset,
644 Location index,
645 size_t scale_factor,
646 vixl::aarch64::Register temp,
647 bool needs_null_check,
648 bool use_load_acquire,
649 bool always_update_field = false);
650
651 // Generate a heap reference load (with no read barrier).
652 void GenerateRawReferenceLoad(HInstruction* instruction,
653 Location ref,
654 vixl::aarch64::Register obj,
655 uint32_t offset,
656 Location index,
657 size_t scale_factor,
658 bool needs_null_check,
659 bool use_load_acquire);
660
661 // Generate a read barrier for a heap reference within `instruction`
662 // using a slow path.
663 //
664 // A read barrier for an object reference read from the heap is
665 // implemented as a call to the artReadBarrierSlow runtime entry
666 // point, which is passed the values in locations `ref`, `obj`, and
667 // `offset`:
668 //
669 // mirror::Object* artReadBarrierSlow(mirror::Object* ref,
670 // mirror::Object* obj,
671 // uint32_t offset);
672 //
673 // The `out` location contains the value returned by
674 // artReadBarrierSlow.
675 //
676 // When `index` is provided (i.e. for array accesses), the offset
677 // value passed to artReadBarrierSlow is adjusted to take `index`
678 // into account.
679 void GenerateReadBarrierSlow(HInstruction* instruction,
680 Location out,
681 Location ref,
682 Location obj,
683 uint32_t offset,
684 Location index = Location::NoLocation());
685
686 // If read barriers are enabled, generate a read barrier for a heap
687 // reference using a slow path. If heap poisoning is enabled, also
688 // unpoison the reference in `out`.
689 void MaybeGenerateReadBarrierSlow(HInstruction* instruction,
690 Location out,
691 Location ref,
692 Location obj,
693 uint32_t offset,
694 Location index = Location::NoLocation());
695
696 // Generate a read barrier for a GC root within `instruction` using
697 // a slow path.
698 //
699 // A read barrier for an object reference GC root is implemented as
700 // a call to the artReadBarrierForRootSlow runtime entry point,
701 // which is passed the value in location `root`:
702 //
703 // mirror::Object* artReadBarrierForRootSlow(GcRoot<mirror::Object>* root);
704 //
705 // The `out` location contains the value returned by
706 // artReadBarrierForRootSlow.
707 void GenerateReadBarrierForRootSlow(HInstruction* instruction, Location out, Location root);
708
709 void GenerateNop() OVERRIDE;
710
711 void GenerateImplicitNullCheck(HNullCheck* instruction) OVERRIDE;
712 void GenerateExplicitNullCheck(HNullCheck* instruction) OVERRIDE;
713
714 private:
715 using Uint64ToLiteralMap = ArenaSafeMap<uint64_t, vixl::aarch64::Literal<uint64_t>*>;
716 using Uint32ToLiteralMap = ArenaSafeMap<uint32_t, vixl::aarch64::Literal<uint32_t>*>;
717 using MethodToLiteralMap = ArenaSafeMap<MethodReference,
718 vixl::aarch64::Literal<uint64_t>*,
719 MethodReferenceComparator>;
720 using StringToLiteralMap = ArenaSafeMap<StringReference,
721 vixl::aarch64::Literal<uint32_t>*,
722 StringReferenceValueComparator>;
723 using TypeToLiteralMap = ArenaSafeMap<TypeReference,
724 vixl::aarch64::Literal<uint32_t>*,
725 TypeReferenceValueComparator>;
726
727 vixl::aarch64::Literal<uint32_t>* DeduplicateUint32Literal(uint32_t value,
728 Uint32ToLiteralMap* map);
729 vixl::aarch64::Literal<uint64_t>* DeduplicateUint64Literal(uint64_t value);
730 vixl::aarch64::Literal<uint64_t>* DeduplicateMethodLiteral(MethodReference target_method,
731 MethodToLiteralMap* map);
732
733 // The PcRelativePatchInfo is used for PC-relative addressing of dex cache arrays
734 // and boot image strings/types. The only difference is the interpretation of the
735 // offset_or_index.
736 struct PcRelativePatchInfo {
PcRelativePatchInfoPcRelativePatchInfo737 PcRelativePatchInfo(const DexFile& dex_file, uint32_t off_or_idx)
738 : target_dex_file(dex_file), offset_or_index(off_or_idx), label(), pc_insn_label() { }
739
740 const DexFile& target_dex_file;
741 // Either the dex cache array element offset or the string/type index.
742 uint32_t offset_or_index;
743 vixl::aarch64::Label label;
744 vixl::aarch64::Label* pc_insn_label;
745 };
746
747 struct BakerReadBarrierPatchInfo {
BakerReadBarrierPatchInfoBakerReadBarrierPatchInfo748 explicit BakerReadBarrierPatchInfo(uint32_t data) : label(), custom_data(data) { }
749
750 vixl::aarch64::Label label;
751 uint32_t custom_data;
752 };
753
754 vixl::aarch64::Label* NewPcRelativePatch(const DexFile& dex_file,
755 uint32_t offset_or_index,
756 vixl::aarch64::Label* adrp_label,
757 ArenaDeque<PcRelativePatchInfo>* patches);
758
759 void EmitJumpTables();
760
761 template <LinkerPatch (*Factory)(size_t, const DexFile*, uint32_t, uint32_t)>
762 static void EmitPcRelativeLinkerPatches(const ArenaDeque<PcRelativePatchInfo>& infos,
763 ArenaVector<LinkerPatch>* linker_patches);
764
765 // Labels for each block that will be compiled.
766 // We use a deque so that the `vixl::aarch64::Label` objects do not move in memory.
767 ArenaDeque<vixl::aarch64::Label> block_labels_; // Indexed by block id.
768 vixl::aarch64::Label frame_entry_label_;
769 ArenaVector<std::unique_ptr<JumpTableARM64>> jump_tables_;
770
771 LocationsBuilderARM64 location_builder_;
772 InstructionCodeGeneratorARM64 instruction_visitor_;
773 ParallelMoveResolverARM64 move_resolver_;
774 Arm64Assembler assembler_;
775 const Arm64InstructionSetFeatures& isa_features_;
776
777 // Deduplication map for 32-bit literals, used for non-patchable boot image addresses.
778 Uint32ToLiteralMap uint32_literals_;
779 // Deduplication map for 64-bit literals, used for non-patchable method address or method code.
780 Uint64ToLiteralMap uint64_literals_;
781 // PC-relative DexCache access info.
782 ArenaDeque<PcRelativePatchInfo> pc_relative_dex_cache_patches_;
783 // Deduplication map for boot string literals for kBootImageLinkTimeAddress.
784 StringToLiteralMap boot_image_string_patches_;
785 // PC-relative String patch info; type depends on configuration (app .bss or boot image PIC).
786 ArenaDeque<PcRelativePatchInfo> pc_relative_string_patches_;
787 // Deduplication map for boot type literals for kBootImageLinkTimeAddress.
788 TypeToLiteralMap boot_image_type_patches_;
789 // PC-relative type patch info for kBootImageLinkTimePcRelative.
790 ArenaDeque<PcRelativePatchInfo> pc_relative_type_patches_;
791 // PC-relative type patch info for kBssEntry.
792 ArenaDeque<PcRelativePatchInfo> type_bss_entry_patches_;
793 // Baker read barrier patch info.
794 ArenaDeque<BakerReadBarrierPatchInfo> baker_read_barrier_patches_;
795
796 // Patches for string literals in JIT compiled code.
797 StringToLiteralMap jit_string_patches_;
798 // Patches for class literals in JIT compiled code.
799 TypeToLiteralMap jit_class_patches_;
800
801 DISALLOW_COPY_AND_ASSIGN(CodeGeneratorARM64);
802 };
803
GetAssembler()804 inline Arm64Assembler* ParallelMoveResolverARM64::GetAssembler() const {
805 return codegen_->GetAssembler();
806 }
807
808 } // namespace arm64
809 } // namespace art
810
811 #endif // ART_COMPILER_OPTIMIZING_CODE_GENERATOR_ARM64_H_
812