1 /*
2 * Copyright (C) 2014 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 #include "code_generator_arm.h"
18
19 #include "arch/arm/instruction_set_features_arm.h"
20 #include "art_method.h"
21 #include "code_generator_utils.h"
22 #include "compiled_method.h"
23 #include "entrypoints/quick/quick_entrypoints.h"
24 #include "gc/accounting/card_table.h"
25 #include "intrinsics.h"
26 #include "intrinsics_arm.h"
27 #include "mirror/array-inl.h"
28 #include "mirror/class-inl.h"
29 #include "thread.h"
30 #include "utils/arm/assembler_arm.h"
31 #include "utils/arm/managed_register_arm.h"
32 #include "utils/assembler.h"
33 #include "utils/stack_checks.h"
34
35 namespace art {
36
37 template<class MirrorType>
38 class GcRoot;
39
40 namespace arm {
41
ExpectedPairLayout(Location location)42 static bool ExpectedPairLayout(Location location) {
43 // We expected this for both core and fpu register pairs.
44 return ((location.low() & 1) == 0) && (location.low() + 1 == location.high());
45 }
46
47 static constexpr int kCurrentMethodStackOffset = 0;
48 static constexpr Register kMethodRegisterArgument = R0;
49
50 static constexpr Register kCoreAlwaysSpillRegister = R5;
51 static constexpr Register kCoreCalleeSaves[] =
52 { R5, R6, R7, R8, R10, R11, LR };
53 static constexpr SRegister kFpuCalleeSaves[] =
54 { S16, S17, S18, S19, S20, S21, S22, S23, S24, S25, S26, S27, S28, S29, S30, S31 };
55
56 // D31 cannot be split into two S registers, and the register allocator only works on
57 // S registers. Therefore there is no need to block it.
58 static constexpr DRegister DTMP = D31;
59
60 static constexpr uint32_t kPackedSwitchCompareJumpThreshold = 7;
61
62 #define __ down_cast<ArmAssembler*>(codegen->GetAssembler())->
63 #define QUICK_ENTRY_POINT(x) QUICK_ENTRYPOINT_OFFSET(kArmWordSize, x).Int32Value()
64
65 class NullCheckSlowPathARM : public SlowPathCode {
66 public:
NullCheckSlowPathARM(HNullCheck * instruction)67 explicit NullCheckSlowPathARM(HNullCheck* instruction) : SlowPathCode(instruction) {}
68
EmitNativeCode(CodeGenerator * codegen)69 void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
70 CodeGeneratorARM* arm_codegen = down_cast<CodeGeneratorARM*>(codegen);
71 __ Bind(GetEntryLabel());
72 if (instruction_->CanThrowIntoCatchBlock()) {
73 // Live registers will be restored in the catch block if caught.
74 SaveLiveRegisters(codegen, instruction_->GetLocations());
75 }
76 arm_codegen->InvokeRuntime(
77 QUICK_ENTRY_POINT(pThrowNullPointer), instruction_, instruction_->GetDexPc(), this);
78 CheckEntrypointTypes<kQuickThrowNullPointer, void, void>();
79 }
80
IsFatal() const81 bool IsFatal() const OVERRIDE { return true; }
82
GetDescription() const83 const char* GetDescription() const OVERRIDE { return "NullCheckSlowPathARM"; }
84
85 private:
86 DISALLOW_COPY_AND_ASSIGN(NullCheckSlowPathARM);
87 };
88
89 class DivZeroCheckSlowPathARM : public SlowPathCode {
90 public:
DivZeroCheckSlowPathARM(HDivZeroCheck * instruction)91 explicit DivZeroCheckSlowPathARM(HDivZeroCheck* instruction) : SlowPathCode(instruction) {}
92
EmitNativeCode(CodeGenerator * codegen)93 void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
94 CodeGeneratorARM* arm_codegen = down_cast<CodeGeneratorARM*>(codegen);
95 __ Bind(GetEntryLabel());
96 if (instruction_->CanThrowIntoCatchBlock()) {
97 // Live registers will be restored in the catch block if caught.
98 SaveLiveRegisters(codegen, instruction_->GetLocations());
99 }
100 arm_codegen->InvokeRuntime(
101 QUICK_ENTRY_POINT(pThrowDivZero), instruction_, instruction_->GetDexPc(), this);
102 CheckEntrypointTypes<kQuickThrowDivZero, void, void>();
103 }
104
IsFatal() const105 bool IsFatal() const OVERRIDE { return true; }
106
GetDescription() const107 const char* GetDescription() const OVERRIDE { return "DivZeroCheckSlowPathARM"; }
108
109 private:
110 DISALLOW_COPY_AND_ASSIGN(DivZeroCheckSlowPathARM);
111 };
112
113 class SuspendCheckSlowPathARM : public SlowPathCode {
114 public:
SuspendCheckSlowPathARM(HSuspendCheck * instruction,HBasicBlock * successor)115 SuspendCheckSlowPathARM(HSuspendCheck* instruction, HBasicBlock* successor)
116 : SlowPathCode(instruction), successor_(successor) {}
117
EmitNativeCode(CodeGenerator * codegen)118 void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
119 CodeGeneratorARM* arm_codegen = down_cast<CodeGeneratorARM*>(codegen);
120 __ Bind(GetEntryLabel());
121 SaveLiveRegisters(codegen, instruction_->GetLocations());
122 arm_codegen->InvokeRuntime(
123 QUICK_ENTRY_POINT(pTestSuspend), instruction_, instruction_->GetDexPc(), this);
124 CheckEntrypointTypes<kQuickTestSuspend, void, void>();
125 RestoreLiveRegisters(codegen, instruction_->GetLocations());
126 if (successor_ == nullptr) {
127 __ b(GetReturnLabel());
128 } else {
129 __ b(arm_codegen->GetLabelOf(successor_));
130 }
131 }
132
GetReturnLabel()133 Label* GetReturnLabel() {
134 DCHECK(successor_ == nullptr);
135 return &return_label_;
136 }
137
GetSuccessor() const138 HBasicBlock* GetSuccessor() const {
139 return successor_;
140 }
141
GetDescription() const142 const char* GetDescription() const OVERRIDE { return "SuspendCheckSlowPathARM"; }
143
144 private:
145 // If not null, the block to branch to after the suspend check.
146 HBasicBlock* const successor_;
147
148 // If `successor_` is null, the label to branch to after the suspend check.
149 Label return_label_;
150
151 DISALLOW_COPY_AND_ASSIGN(SuspendCheckSlowPathARM);
152 };
153
154 class BoundsCheckSlowPathARM : public SlowPathCode {
155 public:
BoundsCheckSlowPathARM(HBoundsCheck * instruction)156 explicit BoundsCheckSlowPathARM(HBoundsCheck* instruction)
157 : SlowPathCode(instruction) {}
158
EmitNativeCode(CodeGenerator * codegen)159 void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
160 CodeGeneratorARM* arm_codegen = down_cast<CodeGeneratorARM*>(codegen);
161 LocationSummary* locations = instruction_->GetLocations();
162
163 __ Bind(GetEntryLabel());
164 if (instruction_->CanThrowIntoCatchBlock()) {
165 // Live registers will be restored in the catch block if caught.
166 SaveLiveRegisters(codegen, instruction_->GetLocations());
167 }
168 // We're moving two locations to locations that could overlap, so we need a parallel
169 // move resolver.
170 InvokeRuntimeCallingConvention calling_convention;
171 codegen->EmitParallelMoves(
172 locations->InAt(0),
173 Location::RegisterLocation(calling_convention.GetRegisterAt(0)),
174 Primitive::kPrimInt,
175 locations->InAt(1),
176 Location::RegisterLocation(calling_convention.GetRegisterAt(1)),
177 Primitive::kPrimInt);
178 arm_codegen->InvokeRuntime(
179 QUICK_ENTRY_POINT(pThrowArrayBounds), instruction_, instruction_->GetDexPc(), this);
180 CheckEntrypointTypes<kQuickThrowArrayBounds, void, int32_t, int32_t>();
181 }
182
IsFatal() const183 bool IsFatal() const OVERRIDE { return true; }
184
GetDescription() const185 const char* GetDescription() const OVERRIDE { return "BoundsCheckSlowPathARM"; }
186
187 private:
188 DISALLOW_COPY_AND_ASSIGN(BoundsCheckSlowPathARM);
189 };
190
191 class LoadClassSlowPathARM : public SlowPathCode {
192 public:
LoadClassSlowPathARM(HLoadClass * cls,HInstruction * at,uint32_t dex_pc,bool do_clinit)193 LoadClassSlowPathARM(HLoadClass* cls,
194 HInstruction* at,
195 uint32_t dex_pc,
196 bool do_clinit)
197 : SlowPathCode(at), cls_(cls), at_(at), dex_pc_(dex_pc), do_clinit_(do_clinit) {
198 DCHECK(at->IsLoadClass() || at->IsClinitCheck());
199 }
200
EmitNativeCode(CodeGenerator * codegen)201 void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
202 LocationSummary* locations = at_->GetLocations();
203
204 CodeGeneratorARM* arm_codegen = down_cast<CodeGeneratorARM*>(codegen);
205 __ Bind(GetEntryLabel());
206 SaveLiveRegisters(codegen, locations);
207
208 InvokeRuntimeCallingConvention calling_convention;
209 __ LoadImmediate(calling_convention.GetRegisterAt(0), cls_->GetTypeIndex());
210 int32_t entry_point_offset = do_clinit_
211 ? QUICK_ENTRY_POINT(pInitializeStaticStorage)
212 : QUICK_ENTRY_POINT(pInitializeType);
213 arm_codegen->InvokeRuntime(entry_point_offset, at_, dex_pc_, this);
214 if (do_clinit_) {
215 CheckEntrypointTypes<kQuickInitializeStaticStorage, void*, uint32_t>();
216 } else {
217 CheckEntrypointTypes<kQuickInitializeType, void*, uint32_t>();
218 }
219
220 // Move the class to the desired location.
221 Location out = locations->Out();
222 if (out.IsValid()) {
223 DCHECK(out.IsRegister() && !locations->GetLiveRegisters()->ContainsCoreRegister(out.reg()));
224 arm_codegen->Move32(locations->Out(), Location::RegisterLocation(R0));
225 }
226 RestoreLiveRegisters(codegen, locations);
227 __ b(GetExitLabel());
228 }
229
GetDescription() const230 const char* GetDescription() const OVERRIDE { return "LoadClassSlowPathARM"; }
231
232 private:
233 // The class this slow path will load.
234 HLoadClass* const cls_;
235
236 // The instruction where this slow path is happening.
237 // (Might be the load class or an initialization check).
238 HInstruction* const at_;
239
240 // The dex PC of `at_`.
241 const uint32_t dex_pc_;
242
243 // Whether to initialize the class.
244 const bool do_clinit_;
245
246 DISALLOW_COPY_AND_ASSIGN(LoadClassSlowPathARM);
247 };
248
249 class LoadStringSlowPathARM : public SlowPathCode {
250 public:
LoadStringSlowPathARM(HLoadString * instruction)251 explicit LoadStringSlowPathARM(HLoadString* instruction) : SlowPathCode(instruction) {}
252
EmitNativeCode(CodeGenerator * codegen)253 void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
254 LocationSummary* locations = instruction_->GetLocations();
255 DCHECK(!locations->GetLiveRegisters()->ContainsCoreRegister(locations->Out().reg()));
256
257 CodeGeneratorARM* arm_codegen = down_cast<CodeGeneratorARM*>(codegen);
258 __ Bind(GetEntryLabel());
259 SaveLiveRegisters(codegen, locations);
260
261 InvokeRuntimeCallingConvention calling_convention;
262 const uint32_t string_index = instruction_->AsLoadString()->GetStringIndex();
263 __ LoadImmediate(calling_convention.GetRegisterAt(0), string_index);
264 arm_codegen->InvokeRuntime(
265 QUICK_ENTRY_POINT(pResolveString), instruction_, instruction_->GetDexPc(), this);
266 CheckEntrypointTypes<kQuickResolveString, void*, uint32_t>();
267 arm_codegen->Move32(locations->Out(), Location::RegisterLocation(R0));
268
269 RestoreLiveRegisters(codegen, locations);
270 __ b(GetExitLabel());
271 }
272
GetDescription() const273 const char* GetDescription() const OVERRIDE { return "LoadStringSlowPathARM"; }
274
275 private:
276 DISALLOW_COPY_AND_ASSIGN(LoadStringSlowPathARM);
277 };
278
279 class TypeCheckSlowPathARM : public SlowPathCode {
280 public:
TypeCheckSlowPathARM(HInstruction * instruction,bool is_fatal)281 TypeCheckSlowPathARM(HInstruction* instruction, bool is_fatal)
282 : SlowPathCode(instruction), is_fatal_(is_fatal) {}
283
EmitNativeCode(CodeGenerator * codegen)284 void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
285 LocationSummary* locations = instruction_->GetLocations();
286 Location object_class = instruction_->IsCheckCast() ? locations->GetTemp(0)
287 : locations->Out();
288 DCHECK(instruction_->IsCheckCast()
289 || !locations->GetLiveRegisters()->ContainsCoreRegister(locations->Out().reg()));
290
291 CodeGeneratorARM* arm_codegen = down_cast<CodeGeneratorARM*>(codegen);
292 __ Bind(GetEntryLabel());
293
294 if (!is_fatal_) {
295 SaveLiveRegisters(codegen, locations);
296 }
297
298 // We're moving two locations to locations that could overlap, so we need a parallel
299 // move resolver.
300 InvokeRuntimeCallingConvention calling_convention;
301 codegen->EmitParallelMoves(
302 locations->InAt(1),
303 Location::RegisterLocation(calling_convention.GetRegisterAt(0)),
304 Primitive::kPrimNot,
305 object_class,
306 Location::RegisterLocation(calling_convention.GetRegisterAt(1)),
307 Primitive::kPrimNot);
308
309 if (instruction_->IsInstanceOf()) {
310 arm_codegen->InvokeRuntime(QUICK_ENTRY_POINT(pInstanceofNonTrivial),
311 instruction_,
312 instruction_->GetDexPc(),
313 this);
314 CheckEntrypointTypes<
315 kQuickInstanceofNonTrivial, uint32_t, const mirror::Class*, const mirror::Class*>();
316 arm_codegen->Move32(locations->Out(), Location::RegisterLocation(R0));
317 } else {
318 DCHECK(instruction_->IsCheckCast());
319 arm_codegen->InvokeRuntime(QUICK_ENTRY_POINT(pCheckCast),
320 instruction_,
321 instruction_->GetDexPc(),
322 this);
323 CheckEntrypointTypes<kQuickCheckCast, void, const mirror::Class*, const mirror::Class*>();
324 }
325
326 if (!is_fatal_) {
327 RestoreLiveRegisters(codegen, locations);
328 __ b(GetExitLabel());
329 }
330 }
331
GetDescription() const332 const char* GetDescription() const OVERRIDE { return "TypeCheckSlowPathARM"; }
333
IsFatal() const334 bool IsFatal() const OVERRIDE { return is_fatal_; }
335
336 private:
337 const bool is_fatal_;
338
339 DISALLOW_COPY_AND_ASSIGN(TypeCheckSlowPathARM);
340 };
341
342 class DeoptimizationSlowPathARM : public SlowPathCode {
343 public:
DeoptimizationSlowPathARM(HDeoptimize * instruction)344 explicit DeoptimizationSlowPathARM(HDeoptimize* instruction)
345 : SlowPathCode(instruction) {}
346
EmitNativeCode(CodeGenerator * codegen)347 void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
348 CodeGeneratorARM* arm_codegen = down_cast<CodeGeneratorARM*>(codegen);
349 __ Bind(GetEntryLabel());
350 SaveLiveRegisters(codegen, instruction_->GetLocations());
351 arm_codegen->InvokeRuntime(QUICK_ENTRY_POINT(pDeoptimize),
352 instruction_,
353 instruction_->GetDexPc(),
354 this);
355 CheckEntrypointTypes<kQuickDeoptimize, void, void>();
356 }
357
GetDescription() const358 const char* GetDescription() const OVERRIDE { return "DeoptimizationSlowPathARM"; }
359
360 private:
361 DISALLOW_COPY_AND_ASSIGN(DeoptimizationSlowPathARM);
362 };
363
364 class ArraySetSlowPathARM : public SlowPathCode {
365 public:
ArraySetSlowPathARM(HInstruction * instruction)366 explicit ArraySetSlowPathARM(HInstruction* instruction) : SlowPathCode(instruction) {}
367
EmitNativeCode(CodeGenerator * codegen)368 void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
369 LocationSummary* locations = instruction_->GetLocations();
370 __ Bind(GetEntryLabel());
371 SaveLiveRegisters(codegen, locations);
372
373 InvokeRuntimeCallingConvention calling_convention;
374 HParallelMove parallel_move(codegen->GetGraph()->GetArena());
375 parallel_move.AddMove(
376 locations->InAt(0),
377 Location::RegisterLocation(calling_convention.GetRegisterAt(0)),
378 Primitive::kPrimNot,
379 nullptr);
380 parallel_move.AddMove(
381 locations->InAt(1),
382 Location::RegisterLocation(calling_convention.GetRegisterAt(1)),
383 Primitive::kPrimInt,
384 nullptr);
385 parallel_move.AddMove(
386 locations->InAt(2),
387 Location::RegisterLocation(calling_convention.GetRegisterAt(2)),
388 Primitive::kPrimNot,
389 nullptr);
390 codegen->GetMoveResolver()->EmitNativeCode(¶llel_move);
391
392 CodeGeneratorARM* arm_codegen = down_cast<CodeGeneratorARM*>(codegen);
393 arm_codegen->InvokeRuntime(QUICK_ENTRY_POINT(pAputObject),
394 instruction_,
395 instruction_->GetDexPc(),
396 this);
397 CheckEntrypointTypes<kQuickAputObject, void, mirror::Array*, int32_t, mirror::Object*>();
398 RestoreLiveRegisters(codegen, locations);
399 __ b(GetExitLabel());
400 }
401
GetDescription() const402 const char* GetDescription() const OVERRIDE { return "ArraySetSlowPathARM"; }
403
404 private:
405 DISALLOW_COPY_AND_ASSIGN(ArraySetSlowPathARM);
406 };
407
408 // Slow path marking an object during a read barrier.
409 class ReadBarrierMarkSlowPathARM : public SlowPathCode {
410 public:
ReadBarrierMarkSlowPathARM(HInstruction * instruction,Location out,Location obj)411 ReadBarrierMarkSlowPathARM(HInstruction* instruction, Location out, Location obj)
412 : SlowPathCode(instruction), out_(out), obj_(obj) {
413 DCHECK(kEmitCompilerReadBarrier);
414 }
415
GetDescription() const416 const char* GetDescription() const OVERRIDE { return "ReadBarrierMarkSlowPathARM"; }
417
EmitNativeCode(CodeGenerator * codegen)418 void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
419 LocationSummary* locations = instruction_->GetLocations();
420 Register reg_out = out_.AsRegister<Register>();
421 DCHECK(locations->CanCall());
422 DCHECK(!locations->GetLiveRegisters()->ContainsCoreRegister(reg_out));
423 DCHECK(instruction_->IsInstanceFieldGet() ||
424 instruction_->IsStaticFieldGet() ||
425 instruction_->IsArrayGet() ||
426 instruction_->IsLoadClass() ||
427 instruction_->IsLoadString() ||
428 instruction_->IsInstanceOf() ||
429 instruction_->IsCheckCast())
430 << "Unexpected instruction in read barrier marking slow path: "
431 << instruction_->DebugName();
432
433 __ Bind(GetEntryLabel());
434 SaveLiveRegisters(codegen, locations);
435
436 InvokeRuntimeCallingConvention calling_convention;
437 CodeGeneratorARM* arm_codegen = down_cast<CodeGeneratorARM*>(codegen);
438 arm_codegen->Move32(Location::RegisterLocation(calling_convention.GetRegisterAt(0)), obj_);
439 arm_codegen->InvokeRuntime(QUICK_ENTRY_POINT(pReadBarrierMark),
440 instruction_,
441 instruction_->GetDexPc(),
442 this);
443 CheckEntrypointTypes<kQuickReadBarrierMark, mirror::Object*, mirror::Object*>();
444 arm_codegen->Move32(out_, Location::RegisterLocation(R0));
445
446 RestoreLiveRegisters(codegen, locations);
447 __ b(GetExitLabel());
448 }
449
450 private:
451 const Location out_;
452 const Location obj_;
453
454 DISALLOW_COPY_AND_ASSIGN(ReadBarrierMarkSlowPathARM);
455 };
456
457 // Slow path generating a read barrier for a heap reference.
458 class ReadBarrierForHeapReferenceSlowPathARM : public SlowPathCode {
459 public:
ReadBarrierForHeapReferenceSlowPathARM(HInstruction * instruction,Location out,Location ref,Location obj,uint32_t offset,Location index)460 ReadBarrierForHeapReferenceSlowPathARM(HInstruction* instruction,
461 Location out,
462 Location ref,
463 Location obj,
464 uint32_t offset,
465 Location index)
466 : SlowPathCode(instruction),
467 out_(out),
468 ref_(ref),
469 obj_(obj),
470 offset_(offset),
471 index_(index) {
472 DCHECK(kEmitCompilerReadBarrier);
473 // If `obj` is equal to `out` or `ref`, it means the initial object
474 // has been overwritten by (or after) the heap object reference load
475 // to be instrumented, e.g.:
476 //
477 // __ LoadFromOffset(kLoadWord, out, out, offset);
478 // codegen_->GenerateReadBarrierSlow(instruction, out_loc, out_loc, out_loc, offset);
479 //
480 // In that case, we have lost the information about the original
481 // object, and the emitted read barrier cannot work properly.
482 DCHECK(!obj.Equals(out)) << "obj=" << obj << " out=" << out;
483 DCHECK(!obj.Equals(ref)) << "obj=" << obj << " ref=" << ref;
484 }
485
EmitNativeCode(CodeGenerator * codegen)486 void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
487 CodeGeneratorARM* arm_codegen = down_cast<CodeGeneratorARM*>(codegen);
488 LocationSummary* locations = instruction_->GetLocations();
489 Register reg_out = out_.AsRegister<Register>();
490 DCHECK(locations->CanCall());
491 DCHECK(!locations->GetLiveRegisters()->ContainsCoreRegister(reg_out));
492 DCHECK(!instruction_->IsInvoke() ||
493 (instruction_->IsInvokeStaticOrDirect() &&
494 instruction_->GetLocations()->Intrinsified()))
495 << "Unexpected instruction in read barrier for heap reference slow path: "
496 << instruction_->DebugName();
497
498 __ Bind(GetEntryLabel());
499 SaveLiveRegisters(codegen, locations);
500
501 // We may have to change the index's value, but as `index_` is a
502 // constant member (like other "inputs" of this slow path),
503 // introduce a copy of it, `index`.
504 Location index = index_;
505 if (index_.IsValid()) {
506 // Handle `index_` for HArrayGet and intrinsic UnsafeGetObject.
507 if (instruction_->IsArrayGet()) {
508 // Compute the actual memory offset and store it in `index`.
509 Register index_reg = index_.AsRegister<Register>();
510 DCHECK(locations->GetLiveRegisters()->ContainsCoreRegister(index_reg));
511 if (codegen->IsCoreCalleeSaveRegister(index_reg)) {
512 // We are about to change the value of `index_reg` (see the
513 // calls to art::arm::Thumb2Assembler::Lsl and
514 // art::arm::Thumb2Assembler::AddConstant below), but it has
515 // not been saved by the previous call to
516 // art::SlowPathCode::SaveLiveRegisters, as it is a
517 // callee-save register --
518 // art::SlowPathCode::SaveLiveRegisters does not consider
519 // callee-save registers, as it has been designed with the
520 // assumption that callee-save registers are supposed to be
521 // handled by the called function. So, as a callee-save
522 // register, `index_reg` _would_ eventually be saved onto
523 // the stack, but it would be too late: we would have
524 // changed its value earlier. Therefore, we manually save
525 // it here into another freely available register,
526 // `free_reg`, chosen of course among the caller-save
527 // registers (as a callee-save `free_reg` register would
528 // exhibit the same problem).
529 //
530 // Note we could have requested a temporary register from
531 // the register allocator instead; but we prefer not to, as
532 // this is a slow path, and we know we can find a
533 // caller-save register that is available.
534 Register free_reg = FindAvailableCallerSaveRegister(codegen);
535 __ Mov(free_reg, index_reg);
536 index_reg = free_reg;
537 index = Location::RegisterLocation(index_reg);
538 } else {
539 // The initial register stored in `index_` has already been
540 // saved in the call to art::SlowPathCode::SaveLiveRegisters
541 // (as it is not a callee-save register), so we can freely
542 // use it.
543 }
544 // Shifting the index value contained in `index_reg` by the scale
545 // factor (2) cannot overflow in practice, as the runtime is
546 // unable to allocate object arrays with a size larger than
547 // 2^26 - 1 (that is, 2^28 - 4 bytes).
548 __ Lsl(index_reg, index_reg, TIMES_4);
549 static_assert(
550 sizeof(mirror::HeapReference<mirror::Object>) == sizeof(int32_t),
551 "art::mirror::HeapReference<art::mirror::Object> and int32_t have different sizes.");
552 __ AddConstant(index_reg, index_reg, offset_);
553 } else {
554 DCHECK(instruction_->IsInvoke());
555 DCHECK(instruction_->GetLocations()->Intrinsified());
556 DCHECK((instruction_->AsInvoke()->GetIntrinsic() == Intrinsics::kUnsafeGetObject) ||
557 (instruction_->AsInvoke()->GetIntrinsic() == Intrinsics::kUnsafeGetObjectVolatile))
558 << instruction_->AsInvoke()->GetIntrinsic();
559 DCHECK_EQ(offset_, 0U);
560 DCHECK(index_.IsRegisterPair());
561 // UnsafeGet's offset location is a register pair, the low
562 // part contains the correct offset.
563 index = index_.ToLow();
564 }
565 }
566
567 // We're moving two or three locations to locations that could
568 // overlap, so we need a parallel move resolver.
569 InvokeRuntimeCallingConvention calling_convention;
570 HParallelMove parallel_move(codegen->GetGraph()->GetArena());
571 parallel_move.AddMove(ref_,
572 Location::RegisterLocation(calling_convention.GetRegisterAt(0)),
573 Primitive::kPrimNot,
574 nullptr);
575 parallel_move.AddMove(obj_,
576 Location::RegisterLocation(calling_convention.GetRegisterAt(1)),
577 Primitive::kPrimNot,
578 nullptr);
579 if (index.IsValid()) {
580 parallel_move.AddMove(index,
581 Location::RegisterLocation(calling_convention.GetRegisterAt(2)),
582 Primitive::kPrimInt,
583 nullptr);
584 codegen->GetMoveResolver()->EmitNativeCode(¶llel_move);
585 } else {
586 codegen->GetMoveResolver()->EmitNativeCode(¶llel_move);
587 __ LoadImmediate(calling_convention.GetRegisterAt(2), offset_);
588 }
589 arm_codegen->InvokeRuntime(QUICK_ENTRY_POINT(pReadBarrierSlow),
590 instruction_,
591 instruction_->GetDexPc(),
592 this);
593 CheckEntrypointTypes<
594 kQuickReadBarrierSlow, mirror::Object*, mirror::Object*, mirror::Object*, uint32_t>();
595 arm_codegen->Move32(out_, Location::RegisterLocation(R0));
596
597 RestoreLiveRegisters(codegen, locations);
598 __ b(GetExitLabel());
599 }
600
GetDescription() const601 const char* GetDescription() const OVERRIDE { return "ReadBarrierForHeapReferenceSlowPathARM"; }
602
603 private:
FindAvailableCallerSaveRegister(CodeGenerator * codegen)604 Register FindAvailableCallerSaveRegister(CodeGenerator* codegen) {
605 size_t ref = static_cast<int>(ref_.AsRegister<Register>());
606 size_t obj = static_cast<int>(obj_.AsRegister<Register>());
607 for (size_t i = 0, e = codegen->GetNumberOfCoreRegisters(); i < e; ++i) {
608 if (i != ref && i != obj && !codegen->IsCoreCalleeSaveRegister(i)) {
609 return static_cast<Register>(i);
610 }
611 }
612 // We shall never fail to find a free caller-save register, as
613 // there are more than two core caller-save registers on ARM
614 // (meaning it is possible to find one which is different from
615 // `ref` and `obj`).
616 DCHECK_GT(codegen->GetNumberOfCoreCallerSaveRegisters(), 2u);
617 LOG(FATAL) << "Could not find a free caller-save register";
618 UNREACHABLE();
619 }
620
621 const Location out_;
622 const Location ref_;
623 const Location obj_;
624 const uint32_t offset_;
625 // An additional location containing an index to an array.
626 // Only used for HArrayGet and the UnsafeGetObject &
627 // UnsafeGetObjectVolatile intrinsics.
628 const Location index_;
629
630 DISALLOW_COPY_AND_ASSIGN(ReadBarrierForHeapReferenceSlowPathARM);
631 };
632
633 // Slow path generating a read barrier for a GC root.
634 class ReadBarrierForRootSlowPathARM : public SlowPathCode {
635 public:
ReadBarrierForRootSlowPathARM(HInstruction * instruction,Location out,Location root)636 ReadBarrierForRootSlowPathARM(HInstruction* instruction, Location out, Location root)
637 : SlowPathCode(instruction), out_(out), root_(root) {
638 DCHECK(kEmitCompilerReadBarrier);
639 }
640
EmitNativeCode(CodeGenerator * codegen)641 void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
642 LocationSummary* locations = instruction_->GetLocations();
643 Register reg_out = out_.AsRegister<Register>();
644 DCHECK(locations->CanCall());
645 DCHECK(!locations->GetLiveRegisters()->ContainsCoreRegister(reg_out));
646 DCHECK(instruction_->IsLoadClass() || instruction_->IsLoadString())
647 << "Unexpected instruction in read barrier for GC root slow path: "
648 << instruction_->DebugName();
649
650 __ Bind(GetEntryLabel());
651 SaveLiveRegisters(codegen, locations);
652
653 InvokeRuntimeCallingConvention calling_convention;
654 CodeGeneratorARM* arm_codegen = down_cast<CodeGeneratorARM*>(codegen);
655 arm_codegen->Move32(Location::RegisterLocation(calling_convention.GetRegisterAt(0)), root_);
656 arm_codegen->InvokeRuntime(QUICK_ENTRY_POINT(pReadBarrierForRootSlow),
657 instruction_,
658 instruction_->GetDexPc(),
659 this);
660 CheckEntrypointTypes<kQuickReadBarrierForRootSlow, mirror::Object*, GcRoot<mirror::Object>*>();
661 arm_codegen->Move32(out_, Location::RegisterLocation(R0));
662
663 RestoreLiveRegisters(codegen, locations);
664 __ b(GetExitLabel());
665 }
666
GetDescription() const667 const char* GetDescription() const OVERRIDE { return "ReadBarrierForRootSlowPathARM"; }
668
669 private:
670 const Location out_;
671 const Location root_;
672
673 DISALLOW_COPY_AND_ASSIGN(ReadBarrierForRootSlowPathARM);
674 };
675
676 #undef __
677 #define __ down_cast<ArmAssembler*>(GetAssembler())->
678
ARMCondition(IfCondition cond)679 inline Condition ARMCondition(IfCondition cond) {
680 switch (cond) {
681 case kCondEQ: return EQ;
682 case kCondNE: return NE;
683 case kCondLT: return LT;
684 case kCondLE: return LE;
685 case kCondGT: return GT;
686 case kCondGE: return GE;
687 case kCondB: return LO;
688 case kCondBE: return LS;
689 case kCondA: return HI;
690 case kCondAE: return HS;
691 }
692 LOG(FATAL) << "Unreachable";
693 UNREACHABLE();
694 }
695
696 // Maps signed condition to unsigned condition.
ARMUnsignedCondition(IfCondition cond)697 inline Condition ARMUnsignedCondition(IfCondition cond) {
698 switch (cond) {
699 case kCondEQ: return EQ;
700 case kCondNE: return NE;
701 // Signed to unsigned.
702 case kCondLT: return LO;
703 case kCondLE: return LS;
704 case kCondGT: return HI;
705 case kCondGE: return HS;
706 // Unsigned remain unchanged.
707 case kCondB: return LO;
708 case kCondBE: return LS;
709 case kCondA: return HI;
710 case kCondAE: return HS;
711 }
712 LOG(FATAL) << "Unreachable";
713 UNREACHABLE();
714 }
715
ARMFPCondition(IfCondition cond,bool gt_bias)716 inline Condition ARMFPCondition(IfCondition cond, bool gt_bias) {
717 // The ARM condition codes can express all the necessary branches, see the
718 // "Meaning (floating-point)" column in the table A8-1 of the ARMv7 reference manual.
719 // There is no dex instruction or HIR that would need the missing conditions
720 // "equal or unordered" or "not equal".
721 switch (cond) {
722 case kCondEQ: return EQ;
723 case kCondNE: return NE /* unordered */;
724 case kCondLT: return gt_bias ? CC : LT /* unordered */;
725 case kCondLE: return gt_bias ? LS : LE /* unordered */;
726 case kCondGT: return gt_bias ? HI /* unordered */ : GT;
727 case kCondGE: return gt_bias ? CS /* unordered */ : GE;
728 default:
729 LOG(FATAL) << "UNREACHABLE";
730 UNREACHABLE();
731 }
732 }
733
DumpCoreRegister(std::ostream & stream,int reg) const734 void CodeGeneratorARM::DumpCoreRegister(std::ostream& stream, int reg) const {
735 stream << Register(reg);
736 }
737
DumpFloatingPointRegister(std::ostream & stream,int reg) const738 void CodeGeneratorARM::DumpFloatingPointRegister(std::ostream& stream, int reg) const {
739 stream << SRegister(reg);
740 }
741
SaveCoreRegister(size_t stack_index,uint32_t reg_id)742 size_t CodeGeneratorARM::SaveCoreRegister(size_t stack_index, uint32_t reg_id) {
743 __ StoreToOffset(kStoreWord, static_cast<Register>(reg_id), SP, stack_index);
744 return kArmWordSize;
745 }
746
RestoreCoreRegister(size_t stack_index,uint32_t reg_id)747 size_t CodeGeneratorARM::RestoreCoreRegister(size_t stack_index, uint32_t reg_id) {
748 __ LoadFromOffset(kLoadWord, static_cast<Register>(reg_id), SP, stack_index);
749 return kArmWordSize;
750 }
751
SaveFloatingPointRegister(size_t stack_index,uint32_t reg_id)752 size_t CodeGeneratorARM::SaveFloatingPointRegister(size_t stack_index, uint32_t reg_id) {
753 __ StoreSToOffset(static_cast<SRegister>(reg_id), SP, stack_index);
754 return kArmWordSize;
755 }
756
RestoreFloatingPointRegister(size_t stack_index,uint32_t reg_id)757 size_t CodeGeneratorARM::RestoreFloatingPointRegister(size_t stack_index, uint32_t reg_id) {
758 __ LoadSFromOffset(static_cast<SRegister>(reg_id), SP, stack_index);
759 return kArmWordSize;
760 }
761
CodeGeneratorARM(HGraph * graph,const ArmInstructionSetFeatures & isa_features,const CompilerOptions & compiler_options,OptimizingCompilerStats * stats)762 CodeGeneratorARM::CodeGeneratorARM(HGraph* graph,
763 const ArmInstructionSetFeatures& isa_features,
764 const CompilerOptions& compiler_options,
765 OptimizingCompilerStats* stats)
766 : CodeGenerator(graph,
767 kNumberOfCoreRegisters,
768 kNumberOfSRegisters,
769 kNumberOfRegisterPairs,
770 ComputeRegisterMask(reinterpret_cast<const int*>(kCoreCalleeSaves),
771 arraysize(kCoreCalleeSaves)),
772 ComputeRegisterMask(reinterpret_cast<const int*>(kFpuCalleeSaves),
773 arraysize(kFpuCalleeSaves)),
774 compiler_options,
775 stats),
776 block_labels_(nullptr),
777 location_builder_(graph, this),
778 instruction_visitor_(graph, this),
779 move_resolver_(graph->GetArena(), this),
780 assembler_(graph->GetArena()),
781 isa_features_(isa_features),
782 uint32_literals_(std::less<uint32_t>(),
783 graph->GetArena()->Adapter(kArenaAllocCodeGenerator)),
784 method_patches_(MethodReferenceComparator(),
785 graph->GetArena()->Adapter(kArenaAllocCodeGenerator)),
786 call_patches_(MethodReferenceComparator(),
787 graph->GetArena()->Adapter(kArenaAllocCodeGenerator)),
788 relative_call_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)),
789 pc_relative_dex_cache_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)),
790 boot_image_string_patches_(StringReferenceValueComparator(),
791 graph->GetArena()->Adapter(kArenaAllocCodeGenerator)),
792 pc_relative_string_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)),
793 boot_image_address_patches_(std::less<uint32_t>(),
794 graph->GetArena()->Adapter(kArenaAllocCodeGenerator)) {
795 // Always save the LR register to mimic Quick.
796 AddAllocatedRegister(Location::RegisterLocation(LR));
797 }
798
Finalize(CodeAllocator * allocator)799 void CodeGeneratorARM::Finalize(CodeAllocator* allocator) {
800 // Ensure that we fix up branches and literal loads and emit the literal pool.
801 __ FinalizeCode();
802
803 // Adjust native pc offsets in stack maps.
804 for (size_t i = 0, num = stack_map_stream_.GetNumberOfStackMaps(); i != num; ++i) {
805 uint32_t old_position = stack_map_stream_.GetStackMap(i).native_pc_offset;
806 uint32_t new_position = __ GetAdjustedPosition(old_position);
807 stack_map_stream_.SetStackMapNativePcOffset(i, new_position);
808 }
809 // Adjust pc offsets for the disassembly information.
810 if (disasm_info_ != nullptr) {
811 GeneratedCodeInterval* frame_entry_interval = disasm_info_->GetFrameEntryInterval();
812 frame_entry_interval->start = __ GetAdjustedPosition(frame_entry_interval->start);
813 frame_entry_interval->end = __ GetAdjustedPosition(frame_entry_interval->end);
814 for (auto& it : *disasm_info_->GetInstructionIntervals()) {
815 it.second.start = __ GetAdjustedPosition(it.second.start);
816 it.second.end = __ GetAdjustedPosition(it.second.end);
817 }
818 for (auto& it : *disasm_info_->GetSlowPathIntervals()) {
819 it.code_interval.start = __ GetAdjustedPosition(it.code_interval.start);
820 it.code_interval.end = __ GetAdjustedPosition(it.code_interval.end);
821 }
822 }
823
824 CodeGenerator::Finalize(allocator);
825 }
826
SetupBlockedRegisters() const827 void CodeGeneratorARM::SetupBlockedRegisters() const {
828 // Don't allocate the dalvik style register pair passing.
829 blocked_register_pairs_[R1_R2] = true;
830
831 // Stack register, LR and PC are always reserved.
832 blocked_core_registers_[SP] = true;
833 blocked_core_registers_[LR] = true;
834 blocked_core_registers_[PC] = true;
835
836 // Reserve thread register.
837 blocked_core_registers_[TR] = true;
838
839 // Reserve temp register.
840 blocked_core_registers_[IP] = true;
841
842 if (GetGraph()->IsDebuggable()) {
843 // Stubs do not save callee-save floating point registers. If the graph
844 // is debuggable, we need to deal with these registers differently. For
845 // now, just block them.
846 for (size_t i = 0; i < arraysize(kFpuCalleeSaves); ++i) {
847 blocked_fpu_registers_[kFpuCalleeSaves[i]] = true;
848 }
849 }
850
851 UpdateBlockedPairRegisters();
852 }
853
UpdateBlockedPairRegisters() const854 void CodeGeneratorARM::UpdateBlockedPairRegisters() const {
855 for (int i = 0; i < kNumberOfRegisterPairs; i++) {
856 ArmManagedRegister current =
857 ArmManagedRegister::FromRegisterPair(static_cast<RegisterPair>(i));
858 if (blocked_core_registers_[current.AsRegisterPairLow()]
859 || blocked_core_registers_[current.AsRegisterPairHigh()]) {
860 blocked_register_pairs_[i] = true;
861 }
862 }
863 }
864
InstructionCodeGeneratorARM(HGraph * graph,CodeGeneratorARM * codegen)865 InstructionCodeGeneratorARM::InstructionCodeGeneratorARM(HGraph* graph, CodeGeneratorARM* codegen)
866 : InstructionCodeGenerator(graph, codegen),
867 assembler_(codegen->GetAssembler()),
868 codegen_(codegen) {}
869
ComputeSpillMask()870 void CodeGeneratorARM::ComputeSpillMask() {
871 core_spill_mask_ = allocated_registers_.GetCoreRegisters() & core_callee_save_mask_;
872 DCHECK_NE(core_spill_mask_, 0u) << "At least the return address register must be saved";
873 // There is no easy instruction to restore just the PC on thumb2. We spill and
874 // restore another arbitrary register.
875 core_spill_mask_ |= (1 << kCoreAlwaysSpillRegister);
876 fpu_spill_mask_ = allocated_registers_.GetFloatingPointRegisters() & fpu_callee_save_mask_;
877 // We use vpush and vpop for saving and restoring floating point registers, which take
878 // a SRegister and the number of registers to save/restore after that SRegister. We
879 // therefore update the `fpu_spill_mask_` to also contain those registers not allocated,
880 // but in the range.
881 if (fpu_spill_mask_ != 0) {
882 uint32_t least_significant_bit = LeastSignificantBit(fpu_spill_mask_);
883 uint32_t most_significant_bit = MostSignificantBit(fpu_spill_mask_);
884 for (uint32_t i = least_significant_bit + 1 ; i < most_significant_bit; ++i) {
885 fpu_spill_mask_ |= (1 << i);
886 }
887 }
888 }
889
DWARFReg(Register reg)890 static dwarf::Reg DWARFReg(Register reg) {
891 return dwarf::Reg::ArmCore(static_cast<int>(reg));
892 }
893
DWARFReg(SRegister reg)894 static dwarf::Reg DWARFReg(SRegister reg) {
895 return dwarf::Reg::ArmFp(static_cast<int>(reg));
896 }
897
GenerateFrameEntry()898 void CodeGeneratorARM::GenerateFrameEntry() {
899 bool skip_overflow_check =
900 IsLeafMethod() && !FrameNeedsStackCheck(GetFrameSize(), InstructionSet::kArm);
901 DCHECK(GetCompilerOptions().GetImplicitStackOverflowChecks());
902 __ Bind(&frame_entry_label_);
903
904 if (HasEmptyFrame()) {
905 return;
906 }
907
908 if (!skip_overflow_check) {
909 __ AddConstant(IP, SP, -static_cast<int32_t>(GetStackOverflowReservedBytes(kArm)));
910 __ LoadFromOffset(kLoadWord, IP, IP, 0);
911 RecordPcInfo(nullptr, 0);
912 }
913
914 __ PushList(core_spill_mask_);
915 __ cfi().AdjustCFAOffset(kArmWordSize * POPCOUNT(core_spill_mask_));
916 __ cfi().RelOffsetForMany(DWARFReg(kMethodRegisterArgument), 0, core_spill_mask_, kArmWordSize);
917 if (fpu_spill_mask_ != 0) {
918 SRegister start_register = SRegister(LeastSignificantBit(fpu_spill_mask_));
919 __ vpushs(start_register, POPCOUNT(fpu_spill_mask_));
920 __ cfi().AdjustCFAOffset(kArmWordSize * POPCOUNT(fpu_spill_mask_));
921 __ cfi().RelOffsetForMany(DWARFReg(S0), 0, fpu_spill_mask_, kArmWordSize);
922 }
923 int adjust = GetFrameSize() - FrameEntrySpillSize();
924 __ AddConstant(SP, -adjust);
925 __ cfi().AdjustCFAOffset(adjust);
926 __ StoreToOffset(kStoreWord, kMethodRegisterArgument, SP, 0);
927 }
928
GenerateFrameExit()929 void CodeGeneratorARM::GenerateFrameExit() {
930 if (HasEmptyFrame()) {
931 __ bx(LR);
932 return;
933 }
934 __ cfi().RememberState();
935 int adjust = GetFrameSize() - FrameEntrySpillSize();
936 __ AddConstant(SP, adjust);
937 __ cfi().AdjustCFAOffset(-adjust);
938 if (fpu_spill_mask_ != 0) {
939 SRegister start_register = SRegister(LeastSignificantBit(fpu_spill_mask_));
940 __ vpops(start_register, POPCOUNT(fpu_spill_mask_));
941 __ cfi().AdjustCFAOffset(-kArmPointerSize * POPCOUNT(fpu_spill_mask_));
942 __ cfi().RestoreMany(DWARFReg(SRegister(0)), fpu_spill_mask_);
943 }
944 // Pop LR into PC to return.
945 DCHECK_NE(core_spill_mask_ & (1 << LR), 0U);
946 uint32_t pop_mask = (core_spill_mask_ & (~(1 << LR))) | 1 << PC;
947 __ PopList(pop_mask);
948 __ cfi().RestoreState();
949 __ cfi().DefCFAOffset(GetFrameSize());
950 }
951
Bind(HBasicBlock * block)952 void CodeGeneratorARM::Bind(HBasicBlock* block) {
953 Label* label = GetLabelOf(block);
954 __ BindTrackedLabel(label);
955 }
956
GetNextLocation(Primitive::Type type)957 Location InvokeDexCallingConventionVisitorARM::GetNextLocation(Primitive::Type type) {
958 switch (type) {
959 case Primitive::kPrimBoolean:
960 case Primitive::kPrimByte:
961 case Primitive::kPrimChar:
962 case Primitive::kPrimShort:
963 case Primitive::kPrimInt:
964 case Primitive::kPrimNot: {
965 uint32_t index = gp_index_++;
966 uint32_t stack_index = stack_index_++;
967 if (index < calling_convention.GetNumberOfRegisters()) {
968 return Location::RegisterLocation(calling_convention.GetRegisterAt(index));
969 } else {
970 return Location::StackSlot(calling_convention.GetStackOffsetOf(stack_index));
971 }
972 }
973
974 case Primitive::kPrimLong: {
975 uint32_t index = gp_index_;
976 uint32_t stack_index = stack_index_;
977 gp_index_ += 2;
978 stack_index_ += 2;
979 if (index + 1 < calling_convention.GetNumberOfRegisters()) {
980 if (calling_convention.GetRegisterAt(index) == R1) {
981 // Skip R1, and use R2_R3 instead.
982 gp_index_++;
983 index++;
984 }
985 }
986 if (index + 1 < calling_convention.GetNumberOfRegisters()) {
987 DCHECK_EQ(calling_convention.GetRegisterAt(index) + 1,
988 calling_convention.GetRegisterAt(index + 1));
989
990 return Location::RegisterPairLocation(calling_convention.GetRegisterAt(index),
991 calling_convention.GetRegisterAt(index + 1));
992 } else {
993 return Location::DoubleStackSlot(calling_convention.GetStackOffsetOf(stack_index));
994 }
995 }
996
997 case Primitive::kPrimFloat: {
998 uint32_t stack_index = stack_index_++;
999 if (float_index_ % 2 == 0) {
1000 float_index_ = std::max(double_index_, float_index_);
1001 }
1002 if (float_index_ < calling_convention.GetNumberOfFpuRegisters()) {
1003 return Location::FpuRegisterLocation(calling_convention.GetFpuRegisterAt(float_index_++));
1004 } else {
1005 return Location::StackSlot(calling_convention.GetStackOffsetOf(stack_index));
1006 }
1007 }
1008
1009 case Primitive::kPrimDouble: {
1010 double_index_ = std::max(double_index_, RoundUp(float_index_, 2));
1011 uint32_t stack_index = stack_index_;
1012 stack_index_ += 2;
1013 if (double_index_ + 1 < calling_convention.GetNumberOfFpuRegisters()) {
1014 uint32_t index = double_index_;
1015 double_index_ += 2;
1016 Location result = Location::FpuRegisterPairLocation(
1017 calling_convention.GetFpuRegisterAt(index),
1018 calling_convention.GetFpuRegisterAt(index + 1));
1019 DCHECK(ExpectedPairLayout(result));
1020 return result;
1021 } else {
1022 return Location::DoubleStackSlot(calling_convention.GetStackOffsetOf(stack_index));
1023 }
1024 }
1025
1026 case Primitive::kPrimVoid:
1027 LOG(FATAL) << "Unexpected parameter type " << type;
1028 break;
1029 }
1030 return Location::NoLocation();
1031 }
1032
GetReturnLocation(Primitive::Type type) const1033 Location InvokeDexCallingConventionVisitorARM::GetReturnLocation(Primitive::Type type) const {
1034 switch (type) {
1035 case Primitive::kPrimBoolean:
1036 case Primitive::kPrimByte:
1037 case Primitive::kPrimChar:
1038 case Primitive::kPrimShort:
1039 case Primitive::kPrimInt:
1040 case Primitive::kPrimNot: {
1041 return Location::RegisterLocation(R0);
1042 }
1043
1044 case Primitive::kPrimFloat: {
1045 return Location::FpuRegisterLocation(S0);
1046 }
1047
1048 case Primitive::kPrimLong: {
1049 return Location::RegisterPairLocation(R0, R1);
1050 }
1051
1052 case Primitive::kPrimDouble: {
1053 return Location::FpuRegisterPairLocation(S0, S1);
1054 }
1055
1056 case Primitive::kPrimVoid:
1057 return Location::NoLocation();
1058 }
1059
1060 UNREACHABLE();
1061 }
1062
GetMethodLocation() const1063 Location InvokeDexCallingConventionVisitorARM::GetMethodLocation() const {
1064 return Location::RegisterLocation(kMethodRegisterArgument);
1065 }
1066
Move32(Location destination,Location source)1067 void CodeGeneratorARM::Move32(Location destination, Location source) {
1068 if (source.Equals(destination)) {
1069 return;
1070 }
1071 if (destination.IsRegister()) {
1072 if (source.IsRegister()) {
1073 __ Mov(destination.AsRegister<Register>(), source.AsRegister<Register>());
1074 } else if (source.IsFpuRegister()) {
1075 __ vmovrs(destination.AsRegister<Register>(), source.AsFpuRegister<SRegister>());
1076 } else {
1077 __ LoadFromOffset(kLoadWord, destination.AsRegister<Register>(), SP, source.GetStackIndex());
1078 }
1079 } else if (destination.IsFpuRegister()) {
1080 if (source.IsRegister()) {
1081 __ vmovsr(destination.AsFpuRegister<SRegister>(), source.AsRegister<Register>());
1082 } else if (source.IsFpuRegister()) {
1083 __ vmovs(destination.AsFpuRegister<SRegister>(), source.AsFpuRegister<SRegister>());
1084 } else {
1085 __ LoadSFromOffset(destination.AsFpuRegister<SRegister>(), SP, source.GetStackIndex());
1086 }
1087 } else {
1088 DCHECK(destination.IsStackSlot()) << destination;
1089 if (source.IsRegister()) {
1090 __ StoreToOffset(kStoreWord, source.AsRegister<Register>(), SP, destination.GetStackIndex());
1091 } else if (source.IsFpuRegister()) {
1092 __ StoreSToOffset(source.AsFpuRegister<SRegister>(), SP, destination.GetStackIndex());
1093 } else {
1094 DCHECK(source.IsStackSlot()) << source;
1095 __ LoadFromOffset(kLoadWord, IP, SP, source.GetStackIndex());
1096 __ StoreToOffset(kStoreWord, IP, SP, destination.GetStackIndex());
1097 }
1098 }
1099 }
1100
Move64(Location destination,Location source)1101 void CodeGeneratorARM::Move64(Location destination, Location source) {
1102 if (source.Equals(destination)) {
1103 return;
1104 }
1105 if (destination.IsRegisterPair()) {
1106 if (source.IsRegisterPair()) {
1107 EmitParallelMoves(
1108 Location::RegisterLocation(source.AsRegisterPairHigh<Register>()),
1109 Location::RegisterLocation(destination.AsRegisterPairHigh<Register>()),
1110 Primitive::kPrimInt,
1111 Location::RegisterLocation(source.AsRegisterPairLow<Register>()),
1112 Location::RegisterLocation(destination.AsRegisterPairLow<Register>()),
1113 Primitive::kPrimInt);
1114 } else if (source.IsFpuRegister()) {
1115 UNIMPLEMENTED(FATAL);
1116 } else if (source.IsFpuRegisterPair()) {
1117 __ vmovrrd(destination.AsRegisterPairLow<Register>(),
1118 destination.AsRegisterPairHigh<Register>(),
1119 FromLowSToD(source.AsFpuRegisterPairLow<SRegister>()));
1120 } else {
1121 DCHECK(source.IsDoubleStackSlot());
1122 DCHECK(ExpectedPairLayout(destination));
1123 __ LoadFromOffset(kLoadWordPair, destination.AsRegisterPairLow<Register>(),
1124 SP, source.GetStackIndex());
1125 }
1126 } else if (destination.IsFpuRegisterPair()) {
1127 if (source.IsDoubleStackSlot()) {
1128 __ LoadDFromOffset(FromLowSToD(destination.AsFpuRegisterPairLow<SRegister>()),
1129 SP,
1130 source.GetStackIndex());
1131 } else if (source.IsRegisterPair()) {
1132 __ vmovdrr(FromLowSToD(destination.AsFpuRegisterPairLow<SRegister>()),
1133 source.AsRegisterPairLow<Register>(),
1134 source.AsRegisterPairHigh<Register>());
1135 } else {
1136 UNIMPLEMENTED(FATAL);
1137 }
1138 } else {
1139 DCHECK(destination.IsDoubleStackSlot());
1140 if (source.IsRegisterPair()) {
1141 // No conflict possible, so just do the moves.
1142 if (source.AsRegisterPairLow<Register>() == R1) {
1143 DCHECK_EQ(source.AsRegisterPairHigh<Register>(), R2);
1144 __ StoreToOffset(kStoreWord, R1, SP, destination.GetStackIndex());
1145 __ StoreToOffset(kStoreWord, R2, SP, destination.GetHighStackIndex(kArmWordSize));
1146 } else {
1147 __ StoreToOffset(kStoreWordPair, source.AsRegisterPairLow<Register>(),
1148 SP, destination.GetStackIndex());
1149 }
1150 } else if (source.IsFpuRegisterPair()) {
1151 __ StoreDToOffset(FromLowSToD(source.AsFpuRegisterPairLow<SRegister>()),
1152 SP,
1153 destination.GetStackIndex());
1154 } else {
1155 DCHECK(source.IsDoubleStackSlot());
1156 EmitParallelMoves(
1157 Location::StackSlot(source.GetStackIndex()),
1158 Location::StackSlot(destination.GetStackIndex()),
1159 Primitive::kPrimInt,
1160 Location::StackSlot(source.GetHighStackIndex(kArmWordSize)),
1161 Location::StackSlot(destination.GetHighStackIndex(kArmWordSize)),
1162 Primitive::kPrimInt);
1163 }
1164 }
1165 }
1166
MoveConstant(Location location,int32_t value)1167 void CodeGeneratorARM::MoveConstant(Location location, int32_t value) {
1168 DCHECK(location.IsRegister());
1169 __ LoadImmediate(location.AsRegister<Register>(), value);
1170 }
1171
MoveLocation(Location dst,Location src,Primitive::Type dst_type)1172 void CodeGeneratorARM::MoveLocation(Location dst, Location src, Primitive::Type dst_type) {
1173 HParallelMove move(GetGraph()->GetArena());
1174 move.AddMove(src, dst, dst_type, nullptr);
1175 GetMoveResolver()->EmitNativeCode(&move);
1176 }
1177
AddLocationAsTemp(Location location,LocationSummary * locations)1178 void CodeGeneratorARM::AddLocationAsTemp(Location location, LocationSummary* locations) {
1179 if (location.IsRegister()) {
1180 locations->AddTemp(location);
1181 } else if (location.IsRegisterPair()) {
1182 locations->AddTemp(Location::RegisterLocation(location.AsRegisterPairLow<Register>()));
1183 locations->AddTemp(Location::RegisterLocation(location.AsRegisterPairHigh<Register>()));
1184 } else {
1185 UNIMPLEMENTED(FATAL) << "AddLocationAsTemp not implemented for location " << location;
1186 }
1187 }
1188
InvokeRuntime(QuickEntrypointEnum entrypoint,HInstruction * instruction,uint32_t dex_pc,SlowPathCode * slow_path)1189 void CodeGeneratorARM::InvokeRuntime(QuickEntrypointEnum entrypoint,
1190 HInstruction* instruction,
1191 uint32_t dex_pc,
1192 SlowPathCode* slow_path) {
1193 InvokeRuntime(GetThreadOffset<kArmWordSize>(entrypoint).Int32Value(),
1194 instruction,
1195 dex_pc,
1196 slow_path);
1197 }
1198
InvokeRuntime(int32_t entry_point_offset,HInstruction * instruction,uint32_t dex_pc,SlowPathCode * slow_path)1199 void CodeGeneratorARM::InvokeRuntime(int32_t entry_point_offset,
1200 HInstruction* instruction,
1201 uint32_t dex_pc,
1202 SlowPathCode* slow_path) {
1203 ValidateInvokeRuntime(instruction, slow_path);
1204 __ LoadFromOffset(kLoadWord, LR, TR, entry_point_offset);
1205 __ blx(LR);
1206 RecordPcInfo(instruction, dex_pc, slow_path);
1207 }
1208
HandleGoto(HInstruction * got,HBasicBlock * successor)1209 void InstructionCodeGeneratorARM::HandleGoto(HInstruction* got, HBasicBlock* successor) {
1210 DCHECK(!successor->IsExitBlock());
1211
1212 HBasicBlock* block = got->GetBlock();
1213 HInstruction* previous = got->GetPrevious();
1214
1215 HLoopInformation* info = block->GetLoopInformation();
1216 if (info != nullptr && info->IsBackEdge(*block) && info->HasSuspendCheck()) {
1217 codegen_->ClearSpillSlotsFromLoopPhisInStackMap(info->GetSuspendCheck());
1218 GenerateSuspendCheck(info->GetSuspendCheck(), successor);
1219 return;
1220 }
1221
1222 if (block->IsEntryBlock() && (previous != nullptr) && previous->IsSuspendCheck()) {
1223 GenerateSuspendCheck(previous->AsSuspendCheck(), nullptr);
1224 }
1225 if (!codegen_->GoesToNextBlock(got->GetBlock(), successor)) {
1226 __ b(codegen_->GetLabelOf(successor));
1227 }
1228 }
1229
VisitGoto(HGoto * got)1230 void LocationsBuilderARM::VisitGoto(HGoto* got) {
1231 got->SetLocations(nullptr);
1232 }
1233
VisitGoto(HGoto * got)1234 void InstructionCodeGeneratorARM::VisitGoto(HGoto* got) {
1235 HandleGoto(got, got->GetSuccessor());
1236 }
1237
VisitTryBoundary(HTryBoundary * try_boundary)1238 void LocationsBuilderARM::VisitTryBoundary(HTryBoundary* try_boundary) {
1239 try_boundary->SetLocations(nullptr);
1240 }
1241
VisitTryBoundary(HTryBoundary * try_boundary)1242 void InstructionCodeGeneratorARM::VisitTryBoundary(HTryBoundary* try_boundary) {
1243 HBasicBlock* successor = try_boundary->GetNormalFlowSuccessor();
1244 if (!successor->IsExitBlock()) {
1245 HandleGoto(try_boundary, successor);
1246 }
1247 }
1248
VisitExit(HExit * exit)1249 void LocationsBuilderARM::VisitExit(HExit* exit) {
1250 exit->SetLocations(nullptr);
1251 }
1252
VisitExit(HExit * exit ATTRIBUTE_UNUSED)1253 void InstructionCodeGeneratorARM::VisitExit(HExit* exit ATTRIBUTE_UNUSED) {
1254 }
1255
GenerateFPJumps(HCondition * cond,Label * true_label,Label * false_label ATTRIBUTE_UNUSED)1256 void InstructionCodeGeneratorARM::GenerateFPJumps(HCondition* cond,
1257 Label* true_label,
1258 Label* false_label ATTRIBUTE_UNUSED) {
1259 __ vmstat(); // transfer FP status register to ARM APSR.
1260 __ b(true_label, ARMFPCondition(cond->GetCondition(), cond->IsGtBias()));
1261 }
1262
GenerateLongComparesAndJumps(HCondition * cond,Label * true_label,Label * false_label)1263 void InstructionCodeGeneratorARM::GenerateLongComparesAndJumps(HCondition* cond,
1264 Label* true_label,
1265 Label* false_label) {
1266 LocationSummary* locations = cond->GetLocations();
1267 Location left = locations->InAt(0);
1268 Location right = locations->InAt(1);
1269 IfCondition if_cond = cond->GetCondition();
1270
1271 Register left_high = left.AsRegisterPairHigh<Register>();
1272 Register left_low = left.AsRegisterPairLow<Register>();
1273 IfCondition true_high_cond = if_cond;
1274 IfCondition false_high_cond = cond->GetOppositeCondition();
1275 Condition final_condition = ARMUnsignedCondition(if_cond); // unsigned on lower part
1276
1277 // Set the conditions for the test, remembering that == needs to be
1278 // decided using the low words.
1279 // TODO: consider avoiding jumps with temporary and CMP low+SBC high
1280 switch (if_cond) {
1281 case kCondEQ:
1282 case kCondNE:
1283 // Nothing to do.
1284 break;
1285 case kCondLT:
1286 false_high_cond = kCondGT;
1287 break;
1288 case kCondLE:
1289 true_high_cond = kCondLT;
1290 break;
1291 case kCondGT:
1292 false_high_cond = kCondLT;
1293 break;
1294 case kCondGE:
1295 true_high_cond = kCondGT;
1296 break;
1297 case kCondB:
1298 false_high_cond = kCondA;
1299 break;
1300 case kCondBE:
1301 true_high_cond = kCondB;
1302 break;
1303 case kCondA:
1304 false_high_cond = kCondB;
1305 break;
1306 case kCondAE:
1307 true_high_cond = kCondA;
1308 break;
1309 }
1310 if (right.IsConstant()) {
1311 int64_t value = right.GetConstant()->AsLongConstant()->GetValue();
1312 int32_t val_low = Low32Bits(value);
1313 int32_t val_high = High32Bits(value);
1314
1315 __ CmpConstant(left_high, val_high);
1316 if (if_cond == kCondNE) {
1317 __ b(true_label, ARMCondition(true_high_cond));
1318 } else if (if_cond == kCondEQ) {
1319 __ b(false_label, ARMCondition(false_high_cond));
1320 } else {
1321 __ b(true_label, ARMCondition(true_high_cond));
1322 __ b(false_label, ARMCondition(false_high_cond));
1323 }
1324 // Must be equal high, so compare the lows.
1325 __ CmpConstant(left_low, val_low);
1326 } else {
1327 Register right_high = right.AsRegisterPairHigh<Register>();
1328 Register right_low = right.AsRegisterPairLow<Register>();
1329
1330 __ cmp(left_high, ShifterOperand(right_high));
1331 if (if_cond == kCondNE) {
1332 __ b(true_label, ARMCondition(true_high_cond));
1333 } else if (if_cond == kCondEQ) {
1334 __ b(false_label, ARMCondition(false_high_cond));
1335 } else {
1336 __ b(true_label, ARMCondition(true_high_cond));
1337 __ b(false_label, ARMCondition(false_high_cond));
1338 }
1339 // Must be equal high, so compare the lows.
1340 __ cmp(left_low, ShifterOperand(right_low));
1341 }
1342 // The last comparison might be unsigned.
1343 // TODO: optimize cases where this is always true/false
1344 __ b(true_label, final_condition);
1345 }
1346
GenerateCompareTestAndBranch(HCondition * condition,Label * true_target_in,Label * false_target_in)1347 void InstructionCodeGeneratorARM::GenerateCompareTestAndBranch(HCondition* condition,
1348 Label* true_target_in,
1349 Label* false_target_in) {
1350 // Generated branching requires both targets to be explicit. If either of the
1351 // targets is nullptr (fallthrough) use and bind `fallthrough_target` instead.
1352 Label fallthrough_target;
1353 Label* true_target = true_target_in == nullptr ? &fallthrough_target : true_target_in;
1354 Label* false_target = false_target_in == nullptr ? &fallthrough_target : false_target_in;
1355
1356 LocationSummary* locations = condition->GetLocations();
1357 Location left = locations->InAt(0);
1358 Location right = locations->InAt(1);
1359
1360 Primitive::Type type = condition->InputAt(0)->GetType();
1361 switch (type) {
1362 case Primitive::kPrimLong:
1363 GenerateLongComparesAndJumps(condition, true_target, false_target);
1364 break;
1365 case Primitive::kPrimFloat:
1366 __ vcmps(left.AsFpuRegister<SRegister>(), right.AsFpuRegister<SRegister>());
1367 GenerateFPJumps(condition, true_target, false_target);
1368 break;
1369 case Primitive::kPrimDouble:
1370 __ vcmpd(FromLowSToD(left.AsFpuRegisterPairLow<SRegister>()),
1371 FromLowSToD(right.AsFpuRegisterPairLow<SRegister>()));
1372 GenerateFPJumps(condition, true_target, false_target);
1373 break;
1374 default:
1375 LOG(FATAL) << "Unexpected compare type " << type;
1376 }
1377
1378 if (false_target != &fallthrough_target) {
1379 __ b(false_target);
1380 }
1381
1382 if (fallthrough_target.IsLinked()) {
1383 __ Bind(&fallthrough_target);
1384 }
1385 }
1386
GenerateTestAndBranch(HInstruction * instruction,size_t condition_input_index,Label * true_target,Label * false_target)1387 void InstructionCodeGeneratorARM::GenerateTestAndBranch(HInstruction* instruction,
1388 size_t condition_input_index,
1389 Label* true_target,
1390 Label* false_target) {
1391 HInstruction* cond = instruction->InputAt(condition_input_index);
1392
1393 if (true_target == nullptr && false_target == nullptr) {
1394 // Nothing to do. The code always falls through.
1395 return;
1396 } else if (cond->IsIntConstant()) {
1397 // Constant condition, statically compared against "true" (integer value 1).
1398 if (cond->AsIntConstant()->IsTrue()) {
1399 if (true_target != nullptr) {
1400 __ b(true_target);
1401 }
1402 } else {
1403 DCHECK(cond->AsIntConstant()->IsFalse()) << cond->AsIntConstant()->GetValue();
1404 if (false_target != nullptr) {
1405 __ b(false_target);
1406 }
1407 }
1408 return;
1409 }
1410
1411 // The following code generates these patterns:
1412 // (1) true_target == nullptr && false_target != nullptr
1413 // - opposite condition true => branch to false_target
1414 // (2) true_target != nullptr && false_target == nullptr
1415 // - condition true => branch to true_target
1416 // (3) true_target != nullptr && false_target != nullptr
1417 // - condition true => branch to true_target
1418 // - branch to false_target
1419 if (IsBooleanValueOrMaterializedCondition(cond)) {
1420 // Condition has been materialized, compare the output to 0.
1421 Location cond_val = instruction->GetLocations()->InAt(condition_input_index);
1422 DCHECK(cond_val.IsRegister());
1423 if (true_target == nullptr) {
1424 __ CompareAndBranchIfZero(cond_val.AsRegister<Register>(), false_target);
1425 } else {
1426 __ CompareAndBranchIfNonZero(cond_val.AsRegister<Register>(), true_target);
1427 }
1428 } else {
1429 // Condition has not been materialized. Use its inputs as the comparison and
1430 // its condition as the branch condition.
1431 HCondition* condition = cond->AsCondition();
1432
1433 // If this is a long or FP comparison that has been folded into
1434 // the HCondition, generate the comparison directly.
1435 Primitive::Type type = condition->InputAt(0)->GetType();
1436 if (type == Primitive::kPrimLong || Primitive::IsFloatingPointType(type)) {
1437 GenerateCompareTestAndBranch(condition, true_target, false_target);
1438 return;
1439 }
1440
1441 LocationSummary* locations = cond->GetLocations();
1442 DCHECK(locations->InAt(0).IsRegister());
1443 Register left = locations->InAt(0).AsRegister<Register>();
1444 Location right = locations->InAt(1);
1445 if (right.IsRegister()) {
1446 __ cmp(left, ShifterOperand(right.AsRegister<Register>()));
1447 } else {
1448 DCHECK(right.IsConstant());
1449 __ CmpConstant(left, CodeGenerator::GetInt32ValueOf(right.GetConstant()));
1450 }
1451 if (true_target == nullptr) {
1452 __ b(false_target, ARMCondition(condition->GetOppositeCondition()));
1453 } else {
1454 __ b(true_target, ARMCondition(condition->GetCondition()));
1455 }
1456 }
1457
1458 // If neither branch falls through (case 3), the conditional branch to `true_target`
1459 // was already emitted (case 2) and we need to emit a jump to `false_target`.
1460 if (true_target != nullptr && false_target != nullptr) {
1461 __ b(false_target);
1462 }
1463 }
1464
VisitIf(HIf * if_instr)1465 void LocationsBuilderARM::VisitIf(HIf* if_instr) {
1466 LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(if_instr);
1467 if (IsBooleanValueOrMaterializedCondition(if_instr->InputAt(0))) {
1468 locations->SetInAt(0, Location::RequiresRegister());
1469 }
1470 }
1471
VisitIf(HIf * if_instr)1472 void InstructionCodeGeneratorARM::VisitIf(HIf* if_instr) {
1473 HBasicBlock* true_successor = if_instr->IfTrueSuccessor();
1474 HBasicBlock* false_successor = if_instr->IfFalseSuccessor();
1475 Label* true_target = codegen_->GoesToNextBlock(if_instr->GetBlock(), true_successor) ?
1476 nullptr : codegen_->GetLabelOf(true_successor);
1477 Label* false_target = codegen_->GoesToNextBlock(if_instr->GetBlock(), false_successor) ?
1478 nullptr : codegen_->GetLabelOf(false_successor);
1479 GenerateTestAndBranch(if_instr, /* condition_input_index */ 0, true_target, false_target);
1480 }
1481
VisitDeoptimize(HDeoptimize * deoptimize)1482 void LocationsBuilderARM::VisitDeoptimize(HDeoptimize* deoptimize) {
1483 LocationSummary* locations = new (GetGraph()->GetArena())
1484 LocationSummary(deoptimize, LocationSummary::kCallOnSlowPath);
1485 if (IsBooleanValueOrMaterializedCondition(deoptimize->InputAt(0))) {
1486 locations->SetInAt(0, Location::RequiresRegister());
1487 }
1488 }
1489
VisitDeoptimize(HDeoptimize * deoptimize)1490 void InstructionCodeGeneratorARM::VisitDeoptimize(HDeoptimize* deoptimize) {
1491 SlowPathCode* slow_path = deopt_slow_paths_.NewSlowPath<DeoptimizationSlowPathARM>(deoptimize);
1492 GenerateTestAndBranch(deoptimize,
1493 /* condition_input_index */ 0,
1494 slow_path->GetEntryLabel(),
1495 /* false_target */ nullptr);
1496 }
1497
VisitSelect(HSelect * select)1498 void LocationsBuilderARM::VisitSelect(HSelect* select) {
1499 LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(select);
1500 if (Primitive::IsFloatingPointType(select->GetType())) {
1501 locations->SetInAt(0, Location::RequiresFpuRegister());
1502 locations->SetInAt(1, Location::RequiresFpuRegister());
1503 } else {
1504 locations->SetInAt(0, Location::RequiresRegister());
1505 locations->SetInAt(1, Location::RequiresRegister());
1506 }
1507 if (IsBooleanValueOrMaterializedCondition(select->GetCondition())) {
1508 locations->SetInAt(2, Location::RequiresRegister());
1509 }
1510 locations->SetOut(Location::SameAsFirstInput());
1511 }
1512
VisitSelect(HSelect * select)1513 void InstructionCodeGeneratorARM::VisitSelect(HSelect* select) {
1514 LocationSummary* locations = select->GetLocations();
1515 Label false_target;
1516 GenerateTestAndBranch(select,
1517 /* condition_input_index */ 2,
1518 /* true_target */ nullptr,
1519 &false_target);
1520 codegen_->MoveLocation(locations->Out(), locations->InAt(1), select->GetType());
1521 __ Bind(&false_target);
1522 }
1523
VisitNativeDebugInfo(HNativeDebugInfo * info)1524 void LocationsBuilderARM::VisitNativeDebugInfo(HNativeDebugInfo* info) {
1525 new (GetGraph()->GetArena()) LocationSummary(info);
1526 }
1527
VisitNativeDebugInfo(HNativeDebugInfo *)1528 void InstructionCodeGeneratorARM::VisitNativeDebugInfo(HNativeDebugInfo*) {
1529 // MaybeRecordNativeDebugInfo is already called implicitly in CodeGenerator::Compile.
1530 }
1531
GenerateNop()1532 void CodeGeneratorARM::GenerateNop() {
1533 __ nop();
1534 }
1535
HandleCondition(HCondition * cond)1536 void LocationsBuilderARM::HandleCondition(HCondition* cond) {
1537 LocationSummary* locations =
1538 new (GetGraph()->GetArena()) LocationSummary(cond, LocationSummary::kNoCall);
1539 // Handle the long/FP comparisons made in instruction simplification.
1540 switch (cond->InputAt(0)->GetType()) {
1541 case Primitive::kPrimLong:
1542 locations->SetInAt(0, Location::RequiresRegister());
1543 locations->SetInAt(1, Location::RegisterOrConstant(cond->InputAt(1)));
1544 if (!cond->IsEmittedAtUseSite()) {
1545 locations->SetOut(Location::RequiresRegister(), Location::kOutputOverlap);
1546 }
1547 break;
1548
1549 case Primitive::kPrimFloat:
1550 case Primitive::kPrimDouble:
1551 locations->SetInAt(0, Location::RequiresFpuRegister());
1552 locations->SetInAt(1, Location::RequiresFpuRegister());
1553 if (!cond->IsEmittedAtUseSite()) {
1554 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
1555 }
1556 break;
1557
1558 default:
1559 locations->SetInAt(0, Location::RequiresRegister());
1560 locations->SetInAt(1, Location::RegisterOrConstant(cond->InputAt(1)));
1561 if (!cond->IsEmittedAtUseSite()) {
1562 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
1563 }
1564 }
1565 }
1566
HandleCondition(HCondition * cond)1567 void InstructionCodeGeneratorARM::HandleCondition(HCondition* cond) {
1568 if (cond->IsEmittedAtUseSite()) {
1569 return;
1570 }
1571
1572 LocationSummary* locations = cond->GetLocations();
1573 Location left = locations->InAt(0);
1574 Location right = locations->InAt(1);
1575 Register out = locations->Out().AsRegister<Register>();
1576 Label true_label, false_label;
1577
1578 switch (cond->InputAt(0)->GetType()) {
1579 default: {
1580 // Integer case.
1581 if (right.IsRegister()) {
1582 __ cmp(left.AsRegister<Register>(), ShifterOperand(right.AsRegister<Register>()));
1583 } else {
1584 DCHECK(right.IsConstant());
1585 __ CmpConstant(left.AsRegister<Register>(),
1586 CodeGenerator::GetInt32ValueOf(right.GetConstant()));
1587 }
1588 __ it(ARMCondition(cond->GetCondition()), kItElse);
1589 __ mov(locations->Out().AsRegister<Register>(), ShifterOperand(1),
1590 ARMCondition(cond->GetCondition()));
1591 __ mov(locations->Out().AsRegister<Register>(), ShifterOperand(0),
1592 ARMCondition(cond->GetOppositeCondition()));
1593 return;
1594 }
1595 case Primitive::kPrimLong:
1596 GenerateLongComparesAndJumps(cond, &true_label, &false_label);
1597 break;
1598 case Primitive::kPrimFloat:
1599 __ vcmps(left.AsFpuRegister<SRegister>(), right.AsFpuRegister<SRegister>());
1600 GenerateFPJumps(cond, &true_label, &false_label);
1601 break;
1602 case Primitive::kPrimDouble:
1603 __ vcmpd(FromLowSToD(left.AsFpuRegisterPairLow<SRegister>()),
1604 FromLowSToD(right.AsFpuRegisterPairLow<SRegister>()));
1605 GenerateFPJumps(cond, &true_label, &false_label);
1606 break;
1607 }
1608
1609 // Convert the jumps into the result.
1610 Label done_label;
1611
1612 // False case: result = 0.
1613 __ Bind(&false_label);
1614 __ LoadImmediate(out, 0);
1615 __ b(&done_label);
1616
1617 // True case: result = 1.
1618 __ Bind(&true_label);
1619 __ LoadImmediate(out, 1);
1620 __ Bind(&done_label);
1621 }
1622
VisitEqual(HEqual * comp)1623 void LocationsBuilderARM::VisitEqual(HEqual* comp) {
1624 HandleCondition(comp);
1625 }
1626
VisitEqual(HEqual * comp)1627 void InstructionCodeGeneratorARM::VisitEqual(HEqual* comp) {
1628 HandleCondition(comp);
1629 }
1630
VisitNotEqual(HNotEqual * comp)1631 void LocationsBuilderARM::VisitNotEqual(HNotEqual* comp) {
1632 HandleCondition(comp);
1633 }
1634
VisitNotEqual(HNotEqual * comp)1635 void InstructionCodeGeneratorARM::VisitNotEqual(HNotEqual* comp) {
1636 HandleCondition(comp);
1637 }
1638
VisitLessThan(HLessThan * comp)1639 void LocationsBuilderARM::VisitLessThan(HLessThan* comp) {
1640 HandleCondition(comp);
1641 }
1642
VisitLessThan(HLessThan * comp)1643 void InstructionCodeGeneratorARM::VisitLessThan(HLessThan* comp) {
1644 HandleCondition(comp);
1645 }
1646
VisitLessThanOrEqual(HLessThanOrEqual * comp)1647 void LocationsBuilderARM::VisitLessThanOrEqual(HLessThanOrEqual* comp) {
1648 HandleCondition(comp);
1649 }
1650
VisitLessThanOrEqual(HLessThanOrEqual * comp)1651 void InstructionCodeGeneratorARM::VisitLessThanOrEqual(HLessThanOrEqual* comp) {
1652 HandleCondition(comp);
1653 }
1654
VisitGreaterThan(HGreaterThan * comp)1655 void LocationsBuilderARM::VisitGreaterThan(HGreaterThan* comp) {
1656 HandleCondition(comp);
1657 }
1658
VisitGreaterThan(HGreaterThan * comp)1659 void InstructionCodeGeneratorARM::VisitGreaterThan(HGreaterThan* comp) {
1660 HandleCondition(comp);
1661 }
1662
VisitGreaterThanOrEqual(HGreaterThanOrEqual * comp)1663 void LocationsBuilderARM::VisitGreaterThanOrEqual(HGreaterThanOrEqual* comp) {
1664 HandleCondition(comp);
1665 }
1666
VisitGreaterThanOrEqual(HGreaterThanOrEqual * comp)1667 void InstructionCodeGeneratorARM::VisitGreaterThanOrEqual(HGreaterThanOrEqual* comp) {
1668 HandleCondition(comp);
1669 }
1670
VisitBelow(HBelow * comp)1671 void LocationsBuilderARM::VisitBelow(HBelow* comp) {
1672 HandleCondition(comp);
1673 }
1674
VisitBelow(HBelow * comp)1675 void InstructionCodeGeneratorARM::VisitBelow(HBelow* comp) {
1676 HandleCondition(comp);
1677 }
1678
VisitBelowOrEqual(HBelowOrEqual * comp)1679 void LocationsBuilderARM::VisitBelowOrEqual(HBelowOrEqual* comp) {
1680 HandleCondition(comp);
1681 }
1682
VisitBelowOrEqual(HBelowOrEqual * comp)1683 void InstructionCodeGeneratorARM::VisitBelowOrEqual(HBelowOrEqual* comp) {
1684 HandleCondition(comp);
1685 }
1686
VisitAbove(HAbove * comp)1687 void LocationsBuilderARM::VisitAbove(HAbove* comp) {
1688 HandleCondition(comp);
1689 }
1690
VisitAbove(HAbove * comp)1691 void InstructionCodeGeneratorARM::VisitAbove(HAbove* comp) {
1692 HandleCondition(comp);
1693 }
1694
VisitAboveOrEqual(HAboveOrEqual * comp)1695 void LocationsBuilderARM::VisitAboveOrEqual(HAboveOrEqual* comp) {
1696 HandleCondition(comp);
1697 }
1698
VisitAboveOrEqual(HAboveOrEqual * comp)1699 void InstructionCodeGeneratorARM::VisitAboveOrEqual(HAboveOrEqual* comp) {
1700 HandleCondition(comp);
1701 }
1702
VisitIntConstant(HIntConstant * constant)1703 void LocationsBuilderARM::VisitIntConstant(HIntConstant* constant) {
1704 LocationSummary* locations =
1705 new (GetGraph()->GetArena()) LocationSummary(constant, LocationSummary::kNoCall);
1706 locations->SetOut(Location::ConstantLocation(constant));
1707 }
1708
VisitIntConstant(HIntConstant * constant ATTRIBUTE_UNUSED)1709 void InstructionCodeGeneratorARM::VisitIntConstant(HIntConstant* constant ATTRIBUTE_UNUSED) {
1710 // Will be generated at use site.
1711 }
1712
VisitNullConstant(HNullConstant * constant)1713 void LocationsBuilderARM::VisitNullConstant(HNullConstant* constant) {
1714 LocationSummary* locations =
1715 new (GetGraph()->GetArena()) LocationSummary(constant, LocationSummary::kNoCall);
1716 locations->SetOut(Location::ConstantLocation(constant));
1717 }
1718
VisitNullConstant(HNullConstant * constant ATTRIBUTE_UNUSED)1719 void InstructionCodeGeneratorARM::VisitNullConstant(HNullConstant* constant ATTRIBUTE_UNUSED) {
1720 // Will be generated at use site.
1721 }
1722
VisitLongConstant(HLongConstant * constant)1723 void LocationsBuilderARM::VisitLongConstant(HLongConstant* constant) {
1724 LocationSummary* locations =
1725 new (GetGraph()->GetArena()) LocationSummary(constant, LocationSummary::kNoCall);
1726 locations->SetOut(Location::ConstantLocation(constant));
1727 }
1728
VisitLongConstant(HLongConstant * constant ATTRIBUTE_UNUSED)1729 void InstructionCodeGeneratorARM::VisitLongConstant(HLongConstant* constant ATTRIBUTE_UNUSED) {
1730 // Will be generated at use site.
1731 }
1732
VisitFloatConstant(HFloatConstant * constant)1733 void LocationsBuilderARM::VisitFloatConstant(HFloatConstant* constant) {
1734 LocationSummary* locations =
1735 new (GetGraph()->GetArena()) LocationSummary(constant, LocationSummary::kNoCall);
1736 locations->SetOut(Location::ConstantLocation(constant));
1737 }
1738
VisitFloatConstant(HFloatConstant * constant ATTRIBUTE_UNUSED)1739 void InstructionCodeGeneratorARM::VisitFloatConstant(HFloatConstant* constant ATTRIBUTE_UNUSED) {
1740 // Will be generated at use site.
1741 }
1742
VisitDoubleConstant(HDoubleConstant * constant)1743 void LocationsBuilderARM::VisitDoubleConstant(HDoubleConstant* constant) {
1744 LocationSummary* locations =
1745 new (GetGraph()->GetArena()) LocationSummary(constant, LocationSummary::kNoCall);
1746 locations->SetOut(Location::ConstantLocation(constant));
1747 }
1748
VisitDoubleConstant(HDoubleConstant * constant ATTRIBUTE_UNUSED)1749 void InstructionCodeGeneratorARM::VisitDoubleConstant(HDoubleConstant* constant ATTRIBUTE_UNUSED) {
1750 // Will be generated at use site.
1751 }
1752
VisitMemoryBarrier(HMemoryBarrier * memory_barrier)1753 void LocationsBuilderARM::VisitMemoryBarrier(HMemoryBarrier* memory_barrier) {
1754 memory_barrier->SetLocations(nullptr);
1755 }
1756
VisitMemoryBarrier(HMemoryBarrier * memory_barrier)1757 void InstructionCodeGeneratorARM::VisitMemoryBarrier(HMemoryBarrier* memory_barrier) {
1758 codegen_->GenerateMemoryBarrier(memory_barrier->GetBarrierKind());
1759 }
1760
VisitReturnVoid(HReturnVoid * ret)1761 void LocationsBuilderARM::VisitReturnVoid(HReturnVoid* ret) {
1762 ret->SetLocations(nullptr);
1763 }
1764
VisitReturnVoid(HReturnVoid * ret ATTRIBUTE_UNUSED)1765 void InstructionCodeGeneratorARM::VisitReturnVoid(HReturnVoid* ret ATTRIBUTE_UNUSED) {
1766 codegen_->GenerateFrameExit();
1767 }
1768
VisitReturn(HReturn * ret)1769 void LocationsBuilderARM::VisitReturn(HReturn* ret) {
1770 LocationSummary* locations =
1771 new (GetGraph()->GetArena()) LocationSummary(ret, LocationSummary::kNoCall);
1772 locations->SetInAt(0, parameter_visitor_.GetReturnLocation(ret->InputAt(0)->GetType()));
1773 }
1774
VisitReturn(HReturn * ret ATTRIBUTE_UNUSED)1775 void InstructionCodeGeneratorARM::VisitReturn(HReturn* ret ATTRIBUTE_UNUSED) {
1776 codegen_->GenerateFrameExit();
1777 }
1778
VisitInvokeUnresolved(HInvokeUnresolved * invoke)1779 void LocationsBuilderARM::VisitInvokeUnresolved(HInvokeUnresolved* invoke) {
1780 // The trampoline uses the same calling convention as dex calling conventions,
1781 // except instead of loading arg0/r0 with the target Method*, arg0/r0 will contain
1782 // the method_idx.
1783 HandleInvoke(invoke);
1784 }
1785
VisitInvokeUnresolved(HInvokeUnresolved * invoke)1786 void InstructionCodeGeneratorARM::VisitInvokeUnresolved(HInvokeUnresolved* invoke) {
1787 codegen_->GenerateInvokeUnresolvedRuntimeCall(invoke);
1788 }
1789
VisitInvokeStaticOrDirect(HInvokeStaticOrDirect * invoke)1790 void LocationsBuilderARM::VisitInvokeStaticOrDirect(HInvokeStaticOrDirect* invoke) {
1791 // Explicit clinit checks triggered by static invokes must have been pruned by
1792 // art::PrepareForRegisterAllocation.
1793 DCHECK(!invoke->IsStaticWithExplicitClinitCheck());
1794
1795 IntrinsicLocationsBuilderARM intrinsic(GetGraph()->GetArena(),
1796 codegen_->GetAssembler(),
1797 codegen_->GetInstructionSetFeatures());
1798 if (intrinsic.TryDispatch(invoke)) {
1799 if (invoke->GetLocations()->CanCall() && invoke->HasPcRelativeDexCache()) {
1800 invoke->GetLocations()->SetInAt(invoke->GetSpecialInputIndex(), Location::Any());
1801 }
1802 return;
1803 }
1804
1805 HandleInvoke(invoke);
1806
1807 // For PC-relative dex cache the invoke has an extra input, the PC-relative address base.
1808 if (invoke->HasPcRelativeDexCache()) {
1809 invoke->GetLocations()->SetInAt(invoke->GetSpecialInputIndex(), Location::RequiresRegister());
1810 }
1811 }
1812
TryGenerateIntrinsicCode(HInvoke * invoke,CodeGeneratorARM * codegen)1813 static bool TryGenerateIntrinsicCode(HInvoke* invoke, CodeGeneratorARM* codegen) {
1814 if (invoke->GetLocations()->Intrinsified()) {
1815 IntrinsicCodeGeneratorARM intrinsic(codegen);
1816 intrinsic.Dispatch(invoke);
1817 return true;
1818 }
1819 return false;
1820 }
1821
VisitInvokeStaticOrDirect(HInvokeStaticOrDirect * invoke)1822 void InstructionCodeGeneratorARM::VisitInvokeStaticOrDirect(HInvokeStaticOrDirect* invoke) {
1823 // Explicit clinit checks triggered by static invokes must have been pruned by
1824 // art::PrepareForRegisterAllocation.
1825 DCHECK(!invoke->IsStaticWithExplicitClinitCheck());
1826
1827 if (TryGenerateIntrinsicCode(invoke, codegen_)) {
1828 return;
1829 }
1830
1831 LocationSummary* locations = invoke->GetLocations();
1832 codegen_->GenerateStaticOrDirectCall(
1833 invoke, locations->HasTemps() ? locations->GetTemp(0) : Location::NoLocation());
1834 codegen_->RecordPcInfo(invoke, invoke->GetDexPc());
1835 }
1836
HandleInvoke(HInvoke * invoke)1837 void LocationsBuilderARM::HandleInvoke(HInvoke* invoke) {
1838 InvokeDexCallingConventionVisitorARM calling_convention_visitor;
1839 CodeGenerator::CreateCommonInvokeLocationSummary(invoke, &calling_convention_visitor);
1840 }
1841
VisitInvokeVirtual(HInvokeVirtual * invoke)1842 void LocationsBuilderARM::VisitInvokeVirtual(HInvokeVirtual* invoke) {
1843 IntrinsicLocationsBuilderARM intrinsic(GetGraph()->GetArena(),
1844 codegen_->GetAssembler(),
1845 codegen_->GetInstructionSetFeatures());
1846 if (intrinsic.TryDispatch(invoke)) {
1847 return;
1848 }
1849
1850 HandleInvoke(invoke);
1851 }
1852
VisitInvokeVirtual(HInvokeVirtual * invoke)1853 void InstructionCodeGeneratorARM::VisitInvokeVirtual(HInvokeVirtual* invoke) {
1854 if (TryGenerateIntrinsicCode(invoke, codegen_)) {
1855 return;
1856 }
1857
1858 codegen_->GenerateVirtualCall(invoke, invoke->GetLocations()->GetTemp(0));
1859 DCHECK(!codegen_->IsLeafMethod());
1860 codegen_->RecordPcInfo(invoke, invoke->GetDexPc());
1861 }
1862
VisitInvokeInterface(HInvokeInterface * invoke)1863 void LocationsBuilderARM::VisitInvokeInterface(HInvokeInterface* invoke) {
1864 HandleInvoke(invoke);
1865 // Add the hidden argument.
1866 invoke->GetLocations()->AddTemp(Location::RegisterLocation(R12));
1867 }
1868
VisitInvokeInterface(HInvokeInterface * invoke)1869 void InstructionCodeGeneratorARM::VisitInvokeInterface(HInvokeInterface* invoke) {
1870 // TODO: b/18116999, our IMTs can miss an IncompatibleClassChangeError.
1871 LocationSummary* locations = invoke->GetLocations();
1872 Register temp = locations->GetTemp(0).AsRegister<Register>();
1873 Register hidden_reg = locations->GetTemp(1).AsRegister<Register>();
1874 uint32_t method_offset = mirror::Class::EmbeddedImTableEntryOffset(
1875 invoke->GetImtIndex() % mirror::Class::kImtSize, kArmPointerSize).Uint32Value();
1876 Location receiver = locations->InAt(0);
1877 uint32_t class_offset = mirror::Object::ClassOffset().Int32Value();
1878
1879 // Set the hidden argument. This is safe to do this here, as R12
1880 // won't be modified thereafter, before the `blx` (call) instruction.
1881 DCHECK_EQ(R12, hidden_reg);
1882 __ LoadImmediate(hidden_reg, invoke->GetDexMethodIndex());
1883
1884 if (receiver.IsStackSlot()) {
1885 __ LoadFromOffset(kLoadWord, temp, SP, receiver.GetStackIndex());
1886 // /* HeapReference<Class> */ temp = temp->klass_
1887 __ LoadFromOffset(kLoadWord, temp, temp, class_offset);
1888 } else {
1889 // /* HeapReference<Class> */ temp = receiver->klass_
1890 __ LoadFromOffset(kLoadWord, temp, receiver.AsRegister<Register>(), class_offset);
1891 }
1892 codegen_->MaybeRecordImplicitNullCheck(invoke);
1893 // Instead of simply (possibly) unpoisoning `temp` here, we should
1894 // emit a read barrier for the previous class reference load.
1895 // However this is not required in practice, as this is an
1896 // intermediate/temporary reference and because the current
1897 // concurrent copying collector keeps the from-space memory
1898 // intact/accessible until the end of the marking phase (the
1899 // concurrent copying collector may not in the future).
1900 __ MaybeUnpoisonHeapReference(temp);
1901 // temp = temp->GetImtEntryAt(method_offset);
1902 uint32_t entry_point =
1903 ArtMethod::EntryPointFromQuickCompiledCodeOffset(kArmWordSize).Int32Value();
1904 __ LoadFromOffset(kLoadWord, temp, temp, method_offset);
1905 // LR = temp->GetEntryPoint();
1906 __ LoadFromOffset(kLoadWord, LR, temp, entry_point);
1907 // LR();
1908 __ blx(LR);
1909 DCHECK(!codegen_->IsLeafMethod());
1910 codegen_->RecordPcInfo(invoke, invoke->GetDexPc());
1911 }
1912
VisitNeg(HNeg * neg)1913 void LocationsBuilderARM::VisitNeg(HNeg* neg) {
1914 LocationSummary* locations =
1915 new (GetGraph()->GetArena()) LocationSummary(neg, LocationSummary::kNoCall);
1916 switch (neg->GetResultType()) {
1917 case Primitive::kPrimInt: {
1918 locations->SetInAt(0, Location::RequiresRegister());
1919 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
1920 break;
1921 }
1922 case Primitive::kPrimLong: {
1923 locations->SetInAt(0, Location::RequiresRegister());
1924 locations->SetOut(Location::RequiresRegister(), Location::kOutputOverlap);
1925 break;
1926 }
1927
1928 case Primitive::kPrimFloat:
1929 case Primitive::kPrimDouble:
1930 locations->SetInAt(0, Location::RequiresFpuRegister());
1931 locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap);
1932 break;
1933
1934 default:
1935 LOG(FATAL) << "Unexpected neg type " << neg->GetResultType();
1936 }
1937 }
1938
VisitNeg(HNeg * neg)1939 void InstructionCodeGeneratorARM::VisitNeg(HNeg* neg) {
1940 LocationSummary* locations = neg->GetLocations();
1941 Location out = locations->Out();
1942 Location in = locations->InAt(0);
1943 switch (neg->GetResultType()) {
1944 case Primitive::kPrimInt:
1945 DCHECK(in.IsRegister());
1946 __ rsb(out.AsRegister<Register>(), in.AsRegister<Register>(), ShifterOperand(0));
1947 break;
1948
1949 case Primitive::kPrimLong:
1950 DCHECK(in.IsRegisterPair());
1951 // out.lo = 0 - in.lo (and update the carry/borrow (C) flag)
1952 __ rsbs(out.AsRegisterPairLow<Register>(),
1953 in.AsRegisterPairLow<Register>(),
1954 ShifterOperand(0));
1955 // We cannot emit an RSC (Reverse Subtract with Carry)
1956 // instruction here, as it does not exist in the Thumb-2
1957 // instruction set. We use the following approach
1958 // using SBC and SUB instead.
1959 //
1960 // out.hi = -C
1961 __ sbc(out.AsRegisterPairHigh<Register>(),
1962 out.AsRegisterPairHigh<Register>(),
1963 ShifterOperand(out.AsRegisterPairHigh<Register>()));
1964 // out.hi = out.hi - in.hi
1965 __ sub(out.AsRegisterPairHigh<Register>(),
1966 out.AsRegisterPairHigh<Register>(),
1967 ShifterOperand(in.AsRegisterPairHigh<Register>()));
1968 break;
1969
1970 case Primitive::kPrimFloat:
1971 DCHECK(in.IsFpuRegister());
1972 __ vnegs(out.AsFpuRegister<SRegister>(), in.AsFpuRegister<SRegister>());
1973 break;
1974
1975 case Primitive::kPrimDouble:
1976 DCHECK(in.IsFpuRegisterPair());
1977 __ vnegd(FromLowSToD(out.AsFpuRegisterPairLow<SRegister>()),
1978 FromLowSToD(in.AsFpuRegisterPairLow<SRegister>()));
1979 break;
1980
1981 default:
1982 LOG(FATAL) << "Unexpected neg type " << neg->GetResultType();
1983 }
1984 }
1985
VisitTypeConversion(HTypeConversion * conversion)1986 void LocationsBuilderARM::VisitTypeConversion(HTypeConversion* conversion) {
1987 Primitive::Type result_type = conversion->GetResultType();
1988 Primitive::Type input_type = conversion->GetInputType();
1989 DCHECK_NE(result_type, input_type);
1990
1991 // The float-to-long, double-to-long and long-to-float type conversions
1992 // rely on a call to the runtime.
1993 LocationSummary::CallKind call_kind =
1994 (((input_type == Primitive::kPrimFloat || input_type == Primitive::kPrimDouble)
1995 && result_type == Primitive::kPrimLong)
1996 || (input_type == Primitive::kPrimLong && result_type == Primitive::kPrimFloat))
1997 ? LocationSummary::kCall
1998 : LocationSummary::kNoCall;
1999 LocationSummary* locations =
2000 new (GetGraph()->GetArena()) LocationSummary(conversion, call_kind);
2001
2002 // The Java language does not allow treating boolean as an integral type but
2003 // our bit representation makes it safe.
2004
2005 switch (result_type) {
2006 case Primitive::kPrimByte:
2007 switch (input_type) {
2008 case Primitive::kPrimLong:
2009 // Type conversion from long to byte is a result of code transformations.
2010 case Primitive::kPrimBoolean:
2011 // Boolean input is a result of code transformations.
2012 case Primitive::kPrimShort:
2013 case Primitive::kPrimInt:
2014 case Primitive::kPrimChar:
2015 // Processing a Dex `int-to-byte' instruction.
2016 locations->SetInAt(0, Location::RequiresRegister());
2017 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
2018 break;
2019
2020 default:
2021 LOG(FATAL) << "Unexpected type conversion from " << input_type
2022 << " to " << result_type;
2023 }
2024 break;
2025
2026 case Primitive::kPrimShort:
2027 switch (input_type) {
2028 case Primitive::kPrimLong:
2029 // Type conversion from long to short is a result of code transformations.
2030 case Primitive::kPrimBoolean:
2031 // Boolean input is a result of code transformations.
2032 case Primitive::kPrimByte:
2033 case Primitive::kPrimInt:
2034 case Primitive::kPrimChar:
2035 // Processing a Dex `int-to-short' instruction.
2036 locations->SetInAt(0, Location::RequiresRegister());
2037 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
2038 break;
2039
2040 default:
2041 LOG(FATAL) << "Unexpected type conversion from " << input_type
2042 << " to " << result_type;
2043 }
2044 break;
2045
2046 case Primitive::kPrimInt:
2047 switch (input_type) {
2048 case Primitive::kPrimLong:
2049 // Processing a Dex `long-to-int' instruction.
2050 locations->SetInAt(0, Location::Any());
2051 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
2052 break;
2053
2054 case Primitive::kPrimFloat:
2055 // Processing a Dex `float-to-int' instruction.
2056 locations->SetInAt(0, Location::RequiresFpuRegister());
2057 locations->SetOut(Location::RequiresRegister());
2058 locations->AddTemp(Location::RequiresFpuRegister());
2059 break;
2060
2061 case Primitive::kPrimDouble:
2062 // Processing a Dex `double-to-int' instruction.
2063 locations->SetInAt(0, Location::RequiresFpuRegister());
2064 locations->SetOut(Location::RequiresRegister());
2065 locations->AddTemp(Location::RequiresFpuRegister());
2066 break;
2067
2068 default:
2069 LOG(FATAL) << "Unexpected type conversion from " << input_type
2070 << " to " << result_type;
2071 }
2072 break;
2073
2074 case Primitive::kPrimLong:
2075 switch (input_type) {
2076 case Primitive::kPrimBoolean:
2077 // Boolean input is a result of code transformations.
2078 case Primitive::kPrimByte:
2079 case Primitive::kPrimShort:
2080 case Primitive::kPrimInt:
2081 case Primitive::kPrimChar:
2082 // Processing a Dex `int-to-long' instruction.
2083 locations->SetInAt(0, Location::RequiresRegister());
2084 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
2085 break;
2086
2087 case Primitive::kPrimFloat: {
2088 // Processing a Dex `float-to-long' instruction.
2089 InvokeRuntimeCallingConvention calling_convention;
2090 locations->SetInAt(0, Location::FpuRegisterLocation(
2091 calling_convention.GetFpuRegisterAt(0)));
2092 locations->SetOut(Location::RegisterPairLocation(R0, R1));
2093 break;
2094 }
2095
2096 case Primitive::kPrimDouble: {
2097 // Processing a Dex `double-to-long' instruction.
2098 InvokeRuntimeCallingConvention calling_convention;
2099 locations->SetInAt(0, Location::FpuRegisterPairLocation(
2100 calling_convention.GetFpuRegisterAt(0),
2101 calling_convention.GetFpuRegisterAt(1)));
2102 locations->SetOut(Location::RegisterPairLocation(R0, R1));
2103 break;
2104 }
2105
2106 default:
2107 LOG(FATAL) << "Unexpected type conversion from " << input_type
2108 << " to " << result_type;
2109 }
2110 break;
2111
2112 case Primitive::kPrimChar:
2113 switch (input_type) {
2114 case Primitive::kPrimLong:
2115 // Type conversion from long to char is a result of code transformations.
2116 case Primitive::kPrimBoolean:
2117 // Boolean input is a result of code transformations.
2118 case Primitive::kPrimByte:
2119 case Primitive::kPrimShort:
2120 case Primitive::kPrimInt:
2121 // Processing a Dex `int-to-char' instruction.
2122 locations->SetInAt(0, Location::RequiresRegister());
2123 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
2124 break;
2125
2126 default:
2127 LOG(FATAL) << "Unexpected type conversion from " << input_type
2128 << " to " << result_type;
2129 }
2130 break;
2131
2132 case Primitive::kPrimFloat:
2133 switch (input_type) {
2134 case Primitive::kPrimBoolean:
2135 // Boolean input is a result of code transformations.
2136 case Primitive::kPrimByte:
2137 case Primitive::kPrimShort:
2138 case Primitive::kPrimInt:
2139 case Primitive::kPrimChar:
2140 // Processing a Dex `int-to-float' instruction.
2141 locations->SetInAt(0, Location::RequiresRegister());
2142 locations->SetOut(Location::RequiresFpuRegister());
2143 break;
2144
2145 case Primitive::kPrimLong: {
2146 // Processing a Dex `long-to-float' instruction.
2147 InvokeRuntimeCallingConvention calling_convention;
2148 locations->SetInAt(0, Location::RegisterPairLocation(
2149 calling_convention.GetRegisterAt(0), calling_convention.GetRegisterAt(1)));
2150 locations->SetOut(Location::FpuRegisterLocation(calling_convention.GetFpuRegisterAt(0)));
2151 break;
2152 }
2153
2154 case Primitive::kPrimDouble:
2155 // Processing a Dex `double-to-float' instruction.
2156 locations->SetInAt(0, Location::RequiresFpuRegister());
2157 locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap);
2158 break;
2159
2160 default:
2161 LOG(FATAL) << "Unexpected type conversion from " << input_type
2162 << " to " << result_type;
2163 };
2164 break;
2165
2166 case Primitive::kPrimDouble:
2167 switch (input_type) {
2168 case Primitive::kPrimBoolean:
2169 // Boolean input is a result of code transformations.
2170 case Primitive::kPrimByte:
2171 case Primitive::kPrimShort:
2172 case Primitive::kPrimInt:
2173 case Primitive::kPrimChar:
2174 // Processing a Dex `int-to-double' instruction.
2175 locations->SetInAt(0, Location::RequiresRegister());
2176 locations->SetOut(Location::RequiresFpuRegister());
2177 break;
2178
2179 case Primitive::kPrimLong:
2180 // Processing a Dex `long-to-double' instruction.
2181 locations->SetInAt(0, Location::RequiresRegister());
2182 locations->SetOut(Location::RequiresFpuRegister());
2183 locations->AddTemp(Location::RequiresFpuRegister());
2184 locations->AddTemp(Location::RequiresFpuRegister());
2185 break;
2186
2187 case Primitive::kPrimFloat:
2188 // Processing a Dex `float-to-double' instruction.
2189 locations->SetInAt(0, Location::RequiresFpuRegister());
2190 locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap);
2191 break;
2192
2193 default:
2194 LOG(FATAL) << "Unexpected type conversion from " << input_type
2195 << " to " << result_type;
2196 };
2197 break;
2198
2199 default:
2200 LOG(FATAL) << "Unexpected type conversion from " << input_type
2201 << " to " << result_type;
2202 }
2203 }
2204
VisitTypeConversion(HTypeConversion * conversion)2205 void InstructionCodeGeneratorARM::VisitTypeConversion(HTypeConversion* conversion) {
2206 LocationSummary* locations = conversion->GetLocations();
2207 Location out = locations->Out();
2208 Location in = locations->InAt(0);
2209 Primitive::Type result_type = conversion->GetResultType();
2210 Primitive::Type input_type = conversion->GetInputType();
2211 DCHECK_NE(result_type, input_type);
2212 switch (result_type) {
2213 case Primitive::kPrimByte:
2214 switch (input_type) {
2215 case Primitive::kPrimLong:
2216 // Type conversion from long to byte is a result of code transformations.
2217 __ sbfx(out.AsRegister<Register>(), in.AsRegisterPairLow<Register>(), 0, 8);
2218 break;
2219 case Primitive::kPrimBoolean:
2220 // Boolean input is a result of code transformations.
2221 case Primitive::kPrimShort:
2222 case Primitive::kPrimInt:
2223 case Primitive::kPrimChar:
2224 // Processing a Dex `int-to-byte' instruction.
2225 __ sbfx(out.AsRegister<Register>(), in.AsRegister<Register>(), 0, 8);
2226 break;
2227
2228 default:
2229 LOG(FATAL) << "Unexpected type conversion from " << input_type
2230 << " to " << result_type;
2231 }
2232 break;
2233
2234 case Primitive::kPrimShort:
2235 switch (input_type) {
2236 case Primitive::kPrimLong:
2237 // Type conversion from long to short is a result of code transformations.
2238 __ sbfx(out.AsRegister<Register>(), in.AsRegisterPairLow<Register>(), 0, 16);
2239 break;
2240 case Primitive::kPrimBoolean:
2241 // Boolean input is a result of code transformations.
2242 case Primitive::kPrimByte:
2243 case Primitive::kPrimInt:
2244 case Primitive::kPrimChar:
2245 // Processing a Dex `int-to-short' instruction.
2246 __ sbfx(out.AsRegister<Register>(), in.AsRegister<Register>(), 0, 16);
2247 break;
2248
2249 default:
2250 LOG(FATAL) << "Unexpected type conversion from " << input_type
2251 << " to " << result_type;
2252 }
2253 break;
2254
2255 case Primitive::kPrimInt:
2256 switch (input_type) {
2257 case Primitive::kPrimLong:
2258 // Processing a Dex `long-to-int' instruction.
2259 DCHECK(out.IsRegister());
2260 if (in.IsRegisterPair()) {
2261 __ Mov(out.AsRegister<Register>(), in.AsRegisterPairLow<Register>());
2262 } else if (in.IsDoubleStackSlot()) {
2263 __ LoadFromOffset(kLoadWord, out.AsRegister<Register>(), SP, in.GetStackIndex());
2264 } else {
2265 DCHECK(in.IsConstant());
2266 DCHECK(in.GetConstant()->IsLongConstant());
2267 int64_t value = in.GetConstant()->AsLongConstant()->GetValue();
2268 __ LoadImmediate(out.AsRegister<Register>(), static_cast<int32_t>(value));
2269 }
2270 break;
2271
2272 case Primitive::kPrimFloat: {
2273 // Processing a Dex `float-to-int' instruction.
2274 SRegister temp = locations->GetTemp(0).AsFpuRegisterPairLow<SRegister>();
2275 __ vmovs(temp, in.AsFpuRegister<SRegister>());
2276 __ vcvtis(temp, temp);
2277 __ vmovrs(out.AsRegister<Register>(), temp);
2278 break;
2279 }
2280
2281 case Primitive::kPrimDouble: {
2282 // Processing a Dex `double-to-int' instruction.
2283 SRegister temp_s = locations->GetTemp(0).AsFpuRegisterPairLow<SRegister>();
2284 DRegister temp_d = FromLowSToD(temp_s);
2285 __ vmovd(temp_d, FromLowSToD(in.AsFpuRegisterPairLow<SRegister>()));
2286 __ vcvtid(temp_s, temp_d);
2287 __ vmovrs(out.AsRegister<Register>(), temp_s);
2288 break;
2289 }
2290
2291 default:
2292 LOG(FATAL) << "Unexpected type conversion from " << input_type
2293 << " to " << result_type;
2294 }
2295 break;
2296
2297 case Primitive::kPrimLong:
2298 switch (input_type) {
2299 case Primitive::kPrimBoolean:
2300 // Boolean input is a result of code transformations.
2301 case Primitive::kPrimByte:
2302 case Primitive::kPrimShort:
2303 case Primitive::kPrimInt:
2304 case Primitive::kPrimChar:
2305 // Processing a Dex `int-to-long' instruction.
2306 DCHECK(out.IsRegisterPair());
2307 DCHECK(in.IsRegister());
2308 __ Mov(out.AsRegisterPairLow<Register>(), in.AsRegister<Register>());
2309 // Sign extension.
2310 __ Asr(out.AsRegisterPairHigh<Register>(),
2311 out.AsRegisterPairLow<Register>(),
2312 31);
2313 break;
2314
2315 case Primitive::kPrimFloat:
2316 // Processing a Dex `float-to-long' instruction.
2317 codegen_->InvokeRuntime(QUICK_ENTRY_POINT(pF2l),
2318 conversion,
2319 conversion->GetDexPc(),
2320 nullptr);
2321 CheckEntrypointTypes<kQuickF2l, int64_t, float>();
2322 break;
2323
2324 case Primitive::kPrimDouble:
2325 // Processing a Dex `double-to-long' instruction.
2326 codegen_->InvokeRuntime(QUICK_ENTRY_POINT(pD2l),
2327 conversion,
2328 conversion->GetDexPc(),
2329 nullptr);
2330 CheckEntrypointTypes<kQuickD2l, int64_t, double>();
2331 break;
2332
2333 default:
2334 LOG(FATAL) << "Unexpected type conversion from " << input_type
2335 << " to " << result_type;
2336 }
2337 break;
2338
2339 case Primitive::kPrimChar:
2340 switch (input_type) {
2341 case Primitive::kPrimLong:
2342 // Type conversion from long to char is a result of code transformations.
2343 __ ubfx(out.AsRegister<Register>(), in.AsRegisterPairLow<Register>(), 0, 16);
2344 break;
2345 case Primitive::kPrimBoolean:
2346 // Boolean input is a result of code transformations.
2347 case Primitive::kPrimByte:
2348 case Primitive::kPrimShort:
2349 case Primitive::kPrimInt:
2350 // Processing a Dex `int-to-char' instruction.
2351 __ ubfx(out.AsRegister<Register>(), in.AsRegister<Register>(), 0, 16);
2352 break;
2353
2354 default:
2355 LOG(FATAL) << "Unexpected type conversion from " << input_type
2356 << " to " << result_type;
2357 }
2358 break;
2359
2360 case Primitive::kPrimFloat:
2361 switch (input_type) {
2362 case Primitive::kPrimBoolean:
2363 // Boolean input is a result of code transformations.
2364 case Primitive::kPrimByte:
2365 case Primitive::kPrimShort:
2366 case Primitive::kPrimInt:
2367 case Primitive::kPrimChar: {
2368 // Processing a Dex `int-to-float' instruction.
2369 __ vmovsr(out.AsFpuRegister<SRegister>(), in.AsRegister<Register>());
2370 __ vcvtsi(out.AsFpuRegister<SRegister>(), out.AsFpuRegister<SRegister>());
2371 break;
2372 }
2373
2374 case Primitive::kPrimLong:
2375 // Processing a Dex `long-to-float' instruction.
2376 codegen_->InvokeRuntime(QUICK_ENTRY_POINT(pL2f),
2377 conversion,
2378 conversion->GetDexPc(),
2379 nullptr);
2380 CheckEntrypointTypes<kQuickL2f, float, int64_t>();
2381 break;
2382
2383 case Primitive::kPrimDouble:
2384 // Processing a Dex `double-to-float' instruction.
2385 __ vcvtsd(out.AsFpuRegister<SRegister>(),
2386 FromLowSToD(in.AsFpuRegisterPairLow<SRegister>()));
2387 break;
2388
2389 default:
2390 LOG(FATAL) << "Unexpected type conversion from " << input_type
2391 << " to " << result_type;
2392 };
2393 break;
2394
2395 case Primitive::kPrimDouble:
2396 switch (input_type) {
2397 case Primitive::kPrimBoolean:
2398 // Boolean input is a result of code transformations.
2399 case Primitive::kPrimByte:
2400 case Primitive::kPrimShort:
2401 case Primitive::kPrimInt:
2402 case Primitive::kPrimChar: {
2403 // Processing a Dex `int-to-double' instruction.
2404 __ vmovsr(out.AsFpuRegisterPairLow<SRegister>(), in.AsRegister<Register>());
2405 __ vcvtdi(FromLowSToD(out.AsFpuRegisterPairLow<SRegister>()),
2406 out.AsFpuRegisterPairLow<SRegister>());
2407 break;
2408 }
2409
2410 case Primitive::kPrimLong: {
2411 // Processing a Dex `long-to-double' instruction.
2412 Register low = in.AsRegisterPairLow<Register>();
2413 Register high = in.AsRegisterPairHigh<Register>();
2414 SRegister out_s = out.AsFpuRegisterPairLow<SRegister>();
2415 DRegister out_d = FromLowSToD(out_s);
2416 SRegister temp_s = locations->GetTemp(0).AsFpuRegisterPairLow<SRegister>();
2417 DRegister temp_d = FromLowSToD(temp_s);
2418 SRegister constant_s = locations->GetTemp(1).AsFpuRegisterPairLow<SRegister>();
2419 DRegister constant_d = FromLowSToD(constant_s);
2420
2421 // temp_d = int-to-double(high)
2422 __ vmovsr(temp_s, high);
2423 __ vcvtdi(temp_d, temp_s);
2424 // constant_d = k2Pow32EncodingForDouble
2425 __ LoadDImmediate(constant_d, bit_cast<double, int64_t>(k2Pow32EncodingForDouble));
2426 // out_d = unsigned-to-double(low)
2427 __ vmovsr(out_s, low);
2428 __ vcvtdu(out_d, out_s);
2429 // out_d += temp_d * constant_d
2430 __ vmlad(out_d, temp_d, constant_d);
2431 break;
2432 }
2433
2434 case Primitive::kPrimFloat:
2435 // Processing a Dex `float-to-double' instruction.
2436 __ vcvtds(FromLowSToD(out.AsFpuRegisterPairLow<SRegister>()),
2437 in.AsFpuRegister<SRegister>());
2438 break;
2439
2440 default:
2441 LOG(FATAL) << "Unexpected type conversion from " << input_type
2442 << " to " << result_type;
2443 };
2444 break;
2445
2446 default:
2447 LOG(FATAL) << "Unexpected type conversion from " << input_type
2448 << " to " << result_type;
2449 }
2450 }
2451
VisitAdd(HAdd * add)2452 void LocationsBuilderARM::VisitAdd(HAdd* add) {
2453 LocationSummary* locations =
2454 new (GetGraph()->GetArena()) LocationSummary(add, LocationSummary::kNoCall);
2455 switch (add->GetResultType()) {
2456 case Primitive::kPrimInt: {
2457 locations->SetInAt(0, Location::RequiresRegister());
2458 locations->SetInAt(1, Location::RegisterOrConstant(add->InputAt(1)));
2459 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
2460 break;
2461 }
2462
2463 case Primitive::kPrimLong: {
2464 locations->SetInAt(0, Location::RequiresRegister());
2465 locations->SetInAt(1, Location::RequiresRegister());
2466 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
2467 break;
2468 }
2469
2470 case Primitive::kPrimFloat:
2471 case Primitive::kPrimDouble: {
2472 locations->SetInAt(0, Location::RequiresFpuRegister());
2473 locations->SetInAt(1, Location::RequiresFpuRegister());
2474 locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap);
2475 break;
2476 }
2477
2478 default:
2479 LOG(FATAL) << "Unexpected add type " << add->GetResultType();
2480 }
2481 }
2482
VisitAdd(HAdd * add)2483 void InstructionCodeGeneratorARM::VisitAdd(HAdd* add) {
2484 LocationSummary* locations = add->GetLocations();
2485 Location out = locations->Out();
2486 Location first = locations->InAt(0);
2487 Location second = locations->InAt(1);
2488 switch (add->GetResultType()) {
2489 case Primitive::kPrimInt:
2490 if (second.IsRegister()) {
2491 __ add(out.AsRegister<Register>(),
2492 first.AsRegister<Register>(),
2493 ShifterOperand(second.AsRegister<Register>()));
2494 } else {
2495 __ AddConstant(out.AsRegister<Register>(),
2496 first.AsRegister<Register>(),
2497 second.GetConstant()->AsIntConstant()->GetValue());
2498 }
2499 break;
2500
2501 case Primitive::kPrimLong: {
2502 DCHECK(second.IsRegisterPair());
2503 __ adds(out.AsRegisterPairLow<Register>(),
2504 first.AsRegisterPairLow<Register>(),
2505 ShifterOperand(second.AsRegisterPairLow<Register>()));
2506 __ adc(out.AsRegisterPairHigh<Register>(),
2507 first.AsRegisterPairHigh<Register>(),
2508 ShifterOperand(second.AsRegisterPairHigh<Register>()));
2509 break;
2510 }
2511
2512 case Primitive::kPrimFloat:
2513 __ vadds(out.AsFpuRegister<SRegister>(),
2514 first.AsFpuRegister<SRegister>(),
2515 second.AsFpuRegister<SRegister>());
2516 break;
2517
2518 case Primitive::kPrimDouble:
2519 __ vaddd(FromLowSToD(out.AsFpuRegisterPairLow<SRegister>()),
2520 FromLowSToD(first.AsFpuRegisterPairLow<SRegister>()),
2521 FromLowSToD(second.AsFpuRegisterPairLow<SRegister>()));
2522 break;
2523
2524 default:
2525 LOG(FATAL) << "Unexpected add type " << add->GetResultType();
2526 }
2527 }
2528
VisitSub(HSub * sub)2529 void LocationsBuilderARM::VisitSub(HSub* sub) {
2530 LocationSummary* locations =
2531 new (GetGraph()->GetArena()) LocationSummary(sub, LocationSummary::kNoCall);
2532 switch (sub->GetResultType()) {
2533 case Primitive::kPrimInt: {
2534 locations->SetInAt(0, Location::RequiresRegister());
2535 locations->SetInAt(1, Location::RegisterOrConstant(sub->InputAt(1)));
2536 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
2537 break;
2538 }
2539
2540 case Primitive::kPrimLong: {
2541 locations->SetInAt(0, Location::RequiresRegister());
2542 locations->SetInAt(1, Location::RequiresRegister());
2543 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
2544 break;
2545 }
2546 case Primitive::kPrimFloat:
2547 case Primitive::kPrimDouble: {
2548 locations->SetInAt(0, Location::RequiresFpuRegister());
2549 locations->SetInAt(1, Location::RequiresFpuRegister());
2550 locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap);
2551 break;
2552 }
2553 default:
2554 LOG(FATAL) << "Unexpected sub type " << sub->GetResultType();
2555 }
2556 }
2557
VisitSub(HSub * sub)2558 void InstructionCodeGeneratorARM::VisitSub(HSub* sub) {
2559 LocationSummary* locations = sub->GetLocations();
2560 Location out = locations->Out();
2561 Location first = locations->InAt(0);
2562 Location second = locations->InAt(1);
2563 switch (sub->GetResultType()) {
2564 case Primitive::kPrimInt: {
2565 if (second.IsRegister()) {
2566 __ sub(out.AsRegister<Register>(),
2567 first.AsRegister<Register>(),
2568 ShifterOperand(second.AsRegister<Register>()));
2569 } else {
2570 __ AddConstant(out.AsRegister<Register>(),
2571 first.AsRegister<Register>(),
2572 -second.GetConstant()->AsIntConstant()->GetValue());
2573 }
2574 break;
2575 }
2576
2577 case Primitive::kPrimLong: {
2578 DCHECK(second.IsRegisterPair());
2579 __ subs(out.AsRegisterPairLow<Register>(),
2580 first.AsRegisterPairLow<Register>(),
2581 ShifterOperand(second.AsRegisterPairLow<Register>()));
2582 __ sbc(out.AsRegisterPairHigh<Register>(),
2583 first.AsRegisterPairHigh<Register>(),
2584 ShifterOperand(second.AsRegisterPairHigh<Register>()));
2585 break;
2586 }
2587
2588 case Primitive::kPrimFloat: {
2589 __ vsubs(out.AsFpuRegister<SRegister>(),
2590 first.AsFpuRegister<SRegister>(),
2591 second.AsFpuRegister<SRegister>());
2592 break;
2593 }
2594
2595 case Primitive::kPrimDouble: {
2596 __ vsubd(FromLowSToD(out.AsFpuRegisterPairLow<SRegister>()),
2597 FromLowSToD(first.AsFpuRegisterPairLow<SRegister>()),
2598 FromLowSToD(second.AsFpuRegisterPairLow<SRegister>()));
2599 break;
2600 }
2601
2602
2603 default:
2604 LOG(FATAL) << "Unexpected sub type " << sub->GetResultType();
2605 }
2606 }
2607
VisitMul(HMul * mul)2608 void LocationsBuilderARM::VisitMul(HMul* mul) {
2609 LocationSummary* locations =
2610 new (GetGraph()->GetArena()) LocationSummary(mul, LocationSummary::kNoCall);
2611 switch (mul->GetResultType()) {
2612 case Primitive::kPrimInt:
2613 case Primitive::kPrimLong: {
2614 locations->SetInAt(0, Location::RequiresRegister());
2615 locations->SetInAt(1, Location::RequiresRegister());
2616 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
2617 break;
2618 }
2619
2620 case Primitive::kPrimFloat:
2621 case Primitive::kPrimDouble: {
2622 locations->SetInAt(0, Location::RequiresFpuRegister());
2623 locations->SetInAt(1, Location::RequiresFpuRegister());
2624 locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap);
2625 break;
2626 }
2627
2628 default:
2629 LOG(FATAL) << "Unexpected mul type " << mul->GetResultType();
2630 }
2631 }
2632
VisitMul(HMul * mul)2633 void InstructionCodeGeneratorARM::VisitMul(HMul* mul) {
2634 LocationSummary* locations = mul->GetLocations();
2635 Location out = locations->Out();
2636 Location first = locations->InAt(0);
2637 Location second = locations->InAt(1);
2638 switch (mul->GetResultType()) {
2639 case Primitive::kPrimInt: {
2640 __ mul(out.AsRegister<Register>(),
2641 first.AsRegister<Register>(),
2642 second.AsRegister<Register>());
2643 break;
2644 }
2645 case Primitive::kPrimLong: {
2646 Register out_hi = out.AsRegisterPairHigh<Register>();
2647 Register out_lo = out.AsRegisterPairLow<Register>();
2648 Register in1_hi = first.AsRegisterPairHigh<Register>();
2649 Register in1_lo = first.AsRegisterPairLow<Register>();
2650 Register in2_hi = second.AsRegisterPairHigh<Register>();
2651 Register in2_lo = second.AsRegisterPairLow<Register>();
2652
2653 // Extra checks to protect caused by the existence of R1_R2.
2654 // The algorithm is wrong if out.hi is either in1.lo or in2.lo:
2655 // (e.g. in1=r0_r1, in2=r2_r3 and out=r1_r2);
2656 DCHECK_NE(out_hi, in1_lo);
2657 DCHECK_NE(out_hi, in2_lo);
2658
2659 // input: in1 - 64 bits, in2 - 64 bits
2660 // output: out
2661 // formula: out.hi : out.lo = (in1.lo * in2.hi + in1.hi * in2.lo)* 2^32 + in1.lo * in2.lo
2662 // parts: out.hi = in1.lo * in2.hi + in1.hi * in2.lo + (in1.lo * in2.lo)[63:32]
2663 // parts: out.lo = (in1.lo * in2.lo)[31:0]
2664
2665 // IP <- in1.lo * in2.hi
2666 __ mul(IP, in1_lo, in2_hi);
2667 // out.hi <- in1.lo * in2.hi + in1.hi * in2.lo
2668 __ mla(out_hi, in1_hi, in2_lo, IP);
2669 // out.lo <- (in1.lo * in2.lo)[31:0];
2670 __ umull(out_lo, IP, in1_lo, in2_lo);
2671 // out.hi <- in2.hi * in1.lo + in2.lo * in1.hi + (in1.lo * in2.lo)[63:32]
2672 __ add(out_hi, out_hi, ShifterOperand(IP));
2673 break;
2674 }
2675
2676 case Primitive::kPrimFloat: {
2677 __ vmuls(out.AsFpuRegister<SRegister>(),
2678 first.AsFpuRegister<SRegister>(),
2679 second.AsFpuRegister<SRegister>());
2680 break;
2681 }
2682
2683 case Primitive::kPrimDouble: {
2684 __ vmuld(FromLowSToD(out.AsFpuRegisterPairLow<SRegister>()),
2685 FromLowSToD(first.AsFpuRegisterPairLow<SRegister>()),
2686 FromLowSToD(second.AsFpuRegisterPairLow<SRegister>()));
2687 break;
2688 }
2689
2690 default:
2691 LOG(FATAL) << "Unexpected mul type " << mul->GetResultType();
2692 }
2693 }
2694
DivRemOneOrMinusOne(HBinaryOperation * instruction)2695 void InstructionCodeGeneratorARM::DivRemOneOrMinusOne(HBinaryOperation* instruction) {
2696 DCHECK(instruction->IsDiv() || instruction->IsRem());
2697 DCHECK(instruction->GetResultType() == Primitive::kPrimInt);
2698
2699 LocationSummary* locations = instruction->GetLocations();
2700 Location second = locations->InAt(1);
2701 DCHECK(second.IsConstant());
2702
2703 Register out = locations->Out().AsRegister<Register>();
2704 Register dividend = locations->InAt(0).AsRegister<Register>();
2705 int32_t imm = second.GetConstant()->AsIntConstant()->GetValue();
2706 DCHECK(imm == 1 || imm == -1);
2707
2708 if (instruction->IsRem()) {
2709 __ LoadImmediate(out, 0);
2710 } else {
2711 if (imm == 1) {
2712 __ Mov(out, dividend);
2713 } else {
2714 __ rsb(out, dividend, ShifterOperand(0));
2715 }
2716 }
2717 }
2718
DivRemByPowerOfTwo(HBinaryOperation * instruction)2719 void InstructionCodeGeneratorARM::DivRemByPowerOfTwo(HBinaryOperation* instruction) {
2720 DCHECK(instruction->IsDiv() || instruction->IsRem());
2721 DCHECK(instruction->GetResultType() == Primitive::kPrimInt);
2722
2723 LocationSummary* locations = instruction->GetLocations();
2724 Location second = locations->InAt(1);
2725 DCHECK(second.IsConstant());
2726
2727 Register out = locations->Out().AsRegister<Register>();
2728 Register dividend = locations->InAt(0).AsRegister<Register>();
2729 Register temp = locations->GetTemp(0).AsRegister<Register>();
2730 int32_t imm = second.GetConstant()->AsIntConstant()->GetValue();
2731 uint32_t abs_imm = static_cast<uint32_t>(AbsOrMin(imm));
2732 int ctz_imm = CTZ(abs_imm);
2733
2734 if (ctz_imm == 1) {
2735 __ Lsr(temp, dividend, 32 - ctz_imm);
2736 } else {
2737 __ Asr(temp, dividend, 31);
2738 __ Lsr(temp, temp, 32 - ctz_imm);
2739 }
2740 __ add(out, temp, ShifterOperand(dividend));
2741
2742 if (instruction->IsDiv()) {
2743 __ Asr(out, out, ctz_imm);
2744 if (imm < 0) {
2745 __ rsb(out, out, ShifterOperand(0));
2746 }
2747 } else {
2748 __ ubfx(out, out, 0, ctz_imm);
2749 __ sub(out, out, ShifterOperand(temp));
2750 }
2751 }
2752
GenerateDivRemWithAnyConstant(HBinaryOperation * instruction)2753 void InstructionCodeGeneratorARM::GenerateDivRemWithAnyConstant(HBinaryOperation* instruction) {
2754 DCHECK(instruction->IsDiv() || instruction->IsRem());
2755 DCHECK(instruction->GetResultType() == Primitive::kPrimInt);
2756
2757 LocationSummary* locations = instruction->GetLocations();
2758 Location second = locations->InAt(1);
2759 DCHECK(second.IsConstant());
2760
2761 Register out = locations->Out().AsRegister<Register>();
2762 Register dividend = locations->InAt(0).AsRegister<Register>();
2763 Register temp1 = locations->GetTemp(0).AsRegister<Register>();
2764 Register temp2 = locations->GetTemp(1).AsRegister<Register>();
2765 int64_t imm = second.GetConstant()->AsIntConstant()->GetValue();
2766
2767 int64_t magic;
2768 int shift;
2769 CalculateMagicAndShiftForDivRem(imm, false /* is_long */, &magic, &shift);
2770
2771 __ LoadImmediate(temp1, magic);
2772 __ smull(temp2, temp1, dividend, temp1);
2773
2774 if (imm > 0 && magic < 0) {
2775 __ add(temp1, temp1, ShifterOperand(dividend));
2776 } else if (imm < 0 && magic > 0) {
2777 __ sub(temp1, temp1, ShifterOperand(dividend));
2778 }
2779
2780 if (shift != 0) {
2781 __ Asr(temp1, temp1, shift);
2782 }
2783
2784 if (instruction->IsDiv()) {
2785 __ sub(out, temp1, ShifterOperand(temp1, ASR, 31));
2786 } else {
2787 __ sub(temp1, temp1, ShifterOperand(temp1, ASR, 31));
2788 // TODO: Strength reduction for mls.
2789 __ LoadImmediate(temp2, imm);
2790 __ mls(out, temp1, temp2, dividend);
2791 }
2792 }
2793
GenerateDivRemConstantIntegral(HBinaryOperation * instruction)2794 void InstructionCodeGeneratorARM::GenerateDivRemConstantIntegral(HBinaryOperation* instruction) {
2795 DCHECK(instruction->IsDiv() || instruction->IsRem());
2796 DCHECK(instruction->GetResultType() == Primitive::kPrimInt);
2797
2798 LocationSummary* locations = instruction->GetLocations();
2799 Location second = locations->InAt(1);
2800 DCHECK(second.IsConstant());
2801
2802 int32_t imm = second.GetConstant()->AsIntConstant()->GetValue();
2803 if (imm == 0) {
2804 // Do not generate anything. DivZeroCheck would prevent any code to be executed.
2805 } else if (imm == 1 || imm == -1) {
2806 DivRemOneOrMinusOne(instruction);
2807 } else if (IsPowerOfTwo(AbsOrMin(imm))) {
2808 DivRemByPowerOfTwo(instruction);
2809 } else {
2810 DCHECK(imm <= -2 || imm >= 2);
2811 GenerateDivRemWithAnyConstant(instruction);
2812 }
2813 }
2814
VisitDiv(HDiv * div)2815 void LocationsBuilderARM::VisitDiv(HDiv* div) {
2816 LocationSummary::CallKind call_kind = LocationSummary::kNoCall;
2817 if (div->GetResultType() == Primitive::kPrimLong) {
2818 // pLdiv runtime call.
2819 call_kind = LocationSummary::kCall;
2820 } else if (div->GetResultType() == Primitive::kPrimInt && div->InputAt(1)->IsConstant()) {
2821 // sdiv will be replaced by other instruction sequence.
2822 } else if (div->GetResultType() == Primitive::kPrimInt &&
2823 !codegen_->GetInstructionSetFeatures().HasDivideInstruction()) {
2824 // pIdivmod runtime call.
2825 call_kind = LocationSummary::kCall;
2826 }
2827
2828 LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(div, call_kind);
2829
2830 switch (div->GetResultType()) {
2831 case Primitive::kPrimInt: {
2832 if (div->InputAt(1)->IsConstant()) {
2833 locations->SetInAt(0, Location::RequiresRegister());
2834 locations->SetInAt(1, Location::ConstantLocation(div->InputAt(1)->AsConstant()));
2835 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
2836 int32_t value = div->InputAt(1)->AsIntConstant()->GetValue();
2837 if (value == 1 || value == 0 || value == -1) {
2838 // No temp register required.
2839 } else {
2840 locations->AddTemp(Location::RequiresRegister());
2841 if (!IsPowerOfTwo(AbsOrMin(value))) {
2842 locations->AddTemp(Location::RequiresRegister());
2843 }
2844 }
2845 } else if (codegen_->GetInstructionSetFeatures().HasDivideInstruction()) {
2846 locations->SetInAt(0, Location::RequiresRegister());
2847 locations->SetInAt(1, Location::RequiresRegister());
2848 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
2849 } else {
2850 InvokeRuntimeCallingConvention calling_convention;
2851 locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(0)));
2852 locations->SetInAt(1, Location::RegisterLocation(calling_convention.GetRegisterAt(1)));
2853 // Note: divrem will compute both the quotient and the remainder as the pair R0 and R1, but
2854 // we only need the former.
2855 locations->SetOut(Location::RegisterLocation(R0));
2856 }
2857 break;
2858 }
2859 case Primitive::kPrimLong: {
2860 InvokeRuntimeCallingConvention calling_convention;
2861 locations->SetInAt(0, Location::RegisterPairLocation(
2862 calling_convention.GetRegisterAt(0), calling_convention.GetRegisterAt(1)));
2863 locations->SetInAt(1, Location::RegisterPairLocation(
2864 calling_convention.GetRegisterAt(2), calling_convention.GetRegisterAt(3)));
2865 locations->SetOut(Location::RegisterPairLocation(R0, R1));
2866 break;
2867 }
2868 case Primitive::kPrimFloat:
2869 case Primitive::kPrimDouble: {
2870 locations->SetInAt(0, Location::RequiresFpuRegister());
2871 locations->SetInAt(1, Location::RequiresFpuRegister());
2872 locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap);
2873 break;
2874 }
2875
2876 default:
2877 LOG(FATAL) << "Unexpected div type " << div->GetResultType();
2878 }
2879 }
2880
VisitDiv(HDiv * div)2881 void InstructionCodeGeneratorARM::VisitDiv(HDiv* div) {
2882 LocationSummary* locations = div->GetLocations();
2883 Location out = locations->Out();
2884 Location first = locations->InAt(0);
2885 Location second = locations->InAt(1);
2886
2887 switch (div->GetResultType()) {
2888 case Primitive::kPrimInt: {
2889 if (second.IsConstant()) {
2890 GenerateDivRemConstantIntegral(div);
2891 } else if (codegen_->GetInstructionSetFeatures().HasDivideInstruction()) {
2892 __ sdiv(out.AsRegister<Register>(),
2893 first.AsRegister<Register>(),
2894 second.AsRegister<Register>());
2895 } else {
2896 InvokeRuntimeCallingConvention calling_convention;
2897 DCHECK_EQ(calling_convention.GetRegisterAt(0), first.AsRegister<Register>());
2898 DCHECK_EQ(calling_convention.GetRegisterAt(1), second.AsRegister<Register>());
2899 DCHECK_EQ(R0, out.AsRegister<Register>());
2900
2901 codegen_->InvokeRuntime(QUICK_ENTRY_POINT(pIdivmod), div, div->GetDexPc(), nullptr);
2902 CheckEntrypointTypes<kQuickIdivmod, int32_t, int32_t, int32_t>();
2903 }
2904 break;
2905 }
2906
2907 case Primitive::kPrimLong: {
2908 InvokeRuntimeCallingConvention calling_convention;
2909 DCHECK_EQ(calling_convention.GetRegisterAt(0), first.AsRegisterPairLow<Register>());
2910 DCHECK_EQ(calling_convention.GetRegisterAt(1), first.AsRegisterPairHigh<Register>());
2911 DCHECK_EQ(calling_convention.GetRegisterAt(2), second.AsRegisterPairLow<Register>());
2912 DCHECK_EQ(calling_convention.GetRegisterAt(3), second.AsRegisterPairHigh<Register>());
2913 DCHECK_EQ(R0, out.AsRegisterPairLow<Register>());
2914 DCHECK_EQ(R1, out.AsRegisterPairHigh<Register>());
2915
2916 codegen_->InvokeRuntime(QUICK_ENTRY_POINT(pLdiv), div, div->GetDexPc(), nullptr);
2917 CheckEntrypointTypes<kQuickLdiv, int64_t, int64_t, int64_t>();
2918 break;
2919 }
2920
2921 case Primitive::kPrimFloat: {
2922 __ vdivs(out.AsFpuRegister<SRegister>(),
2923 first.AsFpuRegister<SRegister>(),
2924 second.AsFpuRegister<SRegister>());
2925 break;
2926 }
2927
2928 case Primitive::kPrimDouble: {
2929 __ vdivd(FromLowSToD(out.AsFpuRegisterPairLow<SRegister>()),
2930 FromLowSToD(first.AsFpuRegisterPairLow<SRegister>()),
2931 FromLowSToD(second.AsFpuRegisterPairLow<SRegister>()));
2932 break;
2933 }
2934
2935 default:
2936 LOG(FATAL) << "Unexpected div type " << div->GetResultType();
2937 }
2938 }
2939
VisitRem(HRem * rem)2940 void LocationsBuilderARM::VisitRem(HRem* rem) {
2941 Primitive::Type type = rem->GetResultType();
2942
2943 // Most remainders are implemented in the runtime.
2944 LocationSummary::CallKind call_kind = LocationSummary::kCall;
2945 if (rem->GetResultType() == Primitive::kPrimInt && rem->InputAt(1)->IsConstant()) {
2946 // sdiv will be replaced by other instruction sequence.
2947 call_kind = LocationSummary::kNoCall;
2948 } else if ((rem->GetResultType() == Primitive::kPrimInt)
2949 && codegen_->GetInstructionSetFeatures().HasDivideInstruction()) {
2950 // Have hardware divide instruction for int, do it with three instructions.
2951 call_kind = LocationSummary::kNoCall;
2952 }
2953
2954 LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(rem, call_kind);
2955
2956 switch (type) {
2957 case Primitive::kPrimInt: {
2958 if (rem->InputAt(1)->IsConstant()) {
2959 locations->SetInAt(0, Location::RequiresRegister());
2960 locations->SetInAt(1, Location::ConstantLocation(rem->InputAt(1)->AsConstant()));
2961 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
2962 int32_t value = rem->InputAt(1)->AsIntConstant()->GetValue();
2963 if (value == 1 || value == 0 || value == -1) {
2964 // No temp register required.
2965 } else {
2966 locations->AddTemp(Location::RequiresRegister());
2967 if (!IsPowerOfTwo(AbsOrMin(value))) {
2968 locations->AddTemp(Location::RequiresRegister());
2969 }
2970 }
2971 } else if (codegen_->GetInstructionSetFeatures().HasDivideInstruction()) {
2972 locations->SetInAt(0, Location::RequiresRegister());
2973 locations->SetInAt(1, Location::RequiresRegister());
2974 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
2975 locations->AddTemp(Location::RequiresRegister());
2976 } else {
2977 InvokeRuntimeCallingConvention calling_convention;
2978 locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(0)));
2979 locations->SetInAt(1, Location::RegisterLocation(calling_convention.GetRegisterAt(1)));
2980 // Note: divrem will compute both the quotient and the remainder as the pair R0 and R1, but
2981 // we only need the latter.
2982 locations->SetOut(Location::RegisterLocation(R1));
2983 }
2984 break;
2985 }
2986 case Primitive::kPrimLong: {
2987 InvokeRuntimeCallingConvention calling_convention;
2988 locations->SetInAt(0, Location::RegisterPairLocation(
2989 calling_convention.GetRegisterAt(0), calling_convention.GetRegisterAt(1)));
2990 locations->SetInAt(1, Location::RegisterPairLocation(
2991 calling_convention.GetRegisterAt(2), calling_convention.GetRegisterAt(3)));
2992 // The runtime helper puts the output in R2,R3.
2993 locations->SetOut(Location::RegisterPairLocation(R2, R3));
2994 break;
2995 }
2996 case Primitive::kPrimFloat: {
2997 InvokeRuntimeCallingConvention calling_convention;
2998 locations->SetInAt(0, Location::FpuRegisterLocation(calling_convention.GetFpuRegisterAt(0)));
2999 locations->SetInAt(1, Location::FpuRegisterLocation(calling_convention.GetFpuRegisterAt(1)));
3000 locations->SetOut(Location::FpuRegisterLocation(S0));
3001 break;
3002 }
3003
3004 case Primitive::kPrimDouble: {
3005 InvokeRuntimeCallingConvention calling_convention;
3006 locations->SetInAt(0, Location::FpuRegisterPairLocation(
3007 calling_convention.GetFpuRegisterAt(0), calling_convention.GetFpuRegisterAt(1)));
3008 locations->SetInAt(1, Location::FpuRegisterPairLocation(
3009 calling_convention.GetFpuRegisterAt(2), calling_convention.GetFpuRegisterAt(3)));
3010 locations->SetOut(Location::Location::FpuRegisterPairLocation(S0, S1));
3011 break;
3012 }
3013
3014 default:
3015 LOG(FATAL) << "Unexpected rem type " << type;
3016 }
3017 }
3018
VisitRem(HRem * rem)3019 void InstructionCodeGeneratorARM::VisitRem(HRem* rem) {
3020 LocationSummary* locations = rem->GetLocations();
3021 Location out = locations->Out();
3022 Location first = locations->InAt(0);
3023 Location second = locations->InAt(1);
3024
3025 Primitive::Type type = rem->GetResultType();
3026 switch (type) {
3027 case Primitive::kPrimInt: {
3028 if (second.IsConstant()) {
3029 GenerateDivRemConstantIntegral(rem);
3030 } else if (codegen_->GetInstructionSetFeatures().HasDivideInstruction()) {
3031 Register reg1 = first.AsRegister<Register>();
3032 Register reg2 = second.AsRegister<Register>();
3033 Register temp = locations->GetTemp(0).AsRegister<Register>();
3034
3035 // temp = reg1 / reg2 (integer division)
3036 // dest = reg1 - temp * reg2
3037 __ sdiv(temp, reg1, reg2);
3038 __ mls(out.AsRegister<Register>(), temp, reg2, reg1);
3039 } else {
3040 InvokeRuntimeCallingConvention calling_convention;
3041 DCHECK_EQ(calling_convention.GetRegisterAt(0), first.AsRegister<Register>());
3042 DCHECK_EQ(calling_convention.GetRegisterAt(1), second.AsRegister<Register>());
3043 DCHECK_EQ(R1, out.AsRegister<Register>());
3044
3045 codegen_->InvokeRuntime(QUICK_ENTRY_POINT(pIdivmod), rem, rem->GetDexPc(), nullptr);
3046 CheckEntrypointTypes<kQuickIdivmod, int32_t, int32_t, int32_t>();
3047 }
3048 break;
3049 }
3050
3051 case Primitive::kPrimLong: {
3052 codegen_->InvokeRuntime(QUICK_ENTRY_POINT(pLmod), rem, rem->GetDexPc(), nullptr);
3053 CheckEntrypointTypes<kQuickLmod, int64_t, int64_t, int64_t>();
3054 break;
3055 }
3056
3057 case Primitive::kPrimFloat: {
3058 codegen_->InvokeRuntime(QUICK_ENTRY_POINT(pFmodf), rem, rem->GetDexPc(), nullptr);
3059 CheckEntrypointTypes<kQuickFmodf, float, float, float>();
3060 break;
3061 }
3062
3063 case Primitive::kPrimDouble: {
3064 codegen_->InvokeRuntime(QUICK_ENTRY_POINT(pFmod), rem, rem->GetDexPc(), nullptr);
3065 CheckEntrypointTypes<kQuickFmod, double, double, double>();
3066 break;
3067 }
3068
3069 default:
3070 LOG(FATAL) << "Unexpected rem type " << type;
3071 }
3072 }
3073
VisitDivZeroCheck(HDivZeroCheck * instruction)3074 void LocationsBuilderARM::VisitDivZeroCheck(HDivZeroCheck* instruction) {
3075 LocationSummary::CallKind call_kind = instruction->CanThrowIntoCatchBlock()
3076 ? LocationSummary::kCallOnSlowPath
3077 : LocationSummary::kNoCall;
3078 LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction, call_kind);
3079 locations->SetInAt(0, Location::RegisterOrConstant(instruction->InputAt(0)));
3080 if (instruction->HasUses()) {
3081 locations->SetOut(Location::SameAsFirstInput());
3082 }
3083 }
3084
VisitDivZeroCheck(HDivZeroCheck * instruction)3085 void InstructionCodeGeneratorARM::VisitDivZeroCheck(HDivZeroCheck* instruction) {
3086 SlowPathCode* slow_path = new (GetGraph()->GetArena()) DivZeroCheckSlowPathARM(instruction);
3087 codegen_->AddSlowPath(slow_path);
3088
3089 LocationSummary* locations = instruction->GetLocations();
3090 Location value = locations->InAt(0);
3091
3092 switch (instruction->GetType()) {
3093 case Primitive::kPrimBoolean:
3094 case Primitive::kPrimByte:
3095 case Primitive::kPrimChar:
3096 case Primitive::kPrimShort:
3097 case Primitive::kPrimInt: {
3098 if (value.IsRegister()) {
3099 __ CompareAndBranchIfZero(value.AsRegister<Register>(), slow_path->GetEntryLabel());
3100 } else {
3101 DCHECK(value.IsConstant()) << value;
3102 if (value.GetConstant()->AsIntConstant()->GetValue() == 0) {
3103 __ b(slow_path->GetEntryLabel());
3104 }
3105 }
3106 break;
3107 }
3108 case Primitive::kPrimLong: {
3109 if (value.IsRegisterPair()) {
3110 __ orrs(IP,
3111 value.AsRegisterPairLow<Register>(),
3112 ShifterOperand(value.AsRegisterPairHigh<Register>()));
3113 __ b(slow_path->GetEntryLabel(), EQ);
3114 } else {
3115 DCHECK(value.IsConstant()) << value;
3116 if (value.GetConstant()->AsLongConstant()->GetValue() == 0) {
3117 __ b(slow_path->GetEntryLabel());
3118 }
3119 }
3120 break;
3121 default:
3122 LOG(FATAL) << "Unexpected type for HDivZeroCheck " << instruction->GetType();
3123 }
3124 }
3125 }
3126
HandleIntegerRotate(LocationSummary * locations)3127 void InstructionCodeGeneratorARM::HandleIntegerRotate(LocationSummary* locations) {
3128 Register in = locations->InAt(0).AsRegister<Register>();
3129 Location rhs = locations->InAt(1);
3130 Register out = locations->Out().AsRegister<Register>();
3131
3132 if (rhs.IsConstant()) {
3133 // Arm32 and Thumb2 assemblers require a rotation on the interval [1,31],
3134 // so map all rotations to a +ve. equivalent in that range.
3135 // (e.g. left *or* right by -2 bits == 30 bits in the same direction.)
3136 uint32_t rot = CodeGenerator::GetInt32ValueOf(rhs.GetConstant()) & 0x1F;
3137 if (rot) {
3138 // Rotate, mapping left rotations to right equivalents if necessary.
3139 // (e.g. left by 2 bits == right by 30.)
3140 __ Ror(out, in, rot);
3141 } else if (out != in) {
3142 __ Mov(out, in);
3143 }
3144 } else {
3145 __ Ror(out, in, rhs.AsRegister<Register>());
3146 }
3147 }
3148
3149 // Gain some speed by mapping all Long rotates onto equivalent pairs of Integer
3150 // rotates by swapping input regs (effectively rotating by the first 32-bits of
3151 // a larger rotation) or flipping direction (thus treating larger right/left
3152 // rotations as sub-word sized rotations in the other direction) as appropriate.
HandleLongRotate(LocationSummary * locations)3153 void InstructionCodeGeneratorARM::HandleLongRotate(LocationSummary* locations) {
3154 Register in_reg_lo = locations->InAt(0).AsRegisterPairLow<Register>();
3155 Register in_reg_hi = locations->InAt(0).AsRegisterPairHigh<Register>();
3156 Location rhs = locations->InAt(1);
3157 Register out_reg_lo = locations->Out().AsRegisterPairLow<Register>();
3158 Register out_reg_hi = locations->Out().AsRegisterPairHigh<Register>();
3159
3160 if (rhs.IsConstant()) {
3161 uint64_t rot = CodeGenerator::GetInt64ValueOf(rhs.GetConstant());
3162 // Map all rotations to +ve. equivalents on the interval [0,63].
3163 rot &= kMaxLongShiftDistance;
3164 // For rotates over a word in size, 'pre-rotate' by 32-bits to keep rotate
3165 // logic below to a simple pair of binary orr.
3166 // (e.g. 34 bits == in_reg swap + 2 bits right.)
3167 if (rot >= kArmBitsPerWord) {
3168 rot -= kArmBitsPerWord;
3169 std::swap(in_reg_hi, in_reg_lo);
3170 }
3171 // Rotate, or mov to out for zero or word size rotations.
3172 if (rot != 0u) {
3173 __ Lsr(out_reg_hi, in_reg_hi, rot);
3174 __ orr(out_reg_hi, out_reg_hi, ShifterOperand(in_reg_lo, arm::LSL, kArmBitsPerWord - rot));
3175 __ Lsr(out_reg_lo, in_reg_lo, rot);
3176 __ orr(out_reg_lo, out_reg_lo, ShifterOperand(in_reg_hi, arm::LSL, kArmBitsPerWord - rot));
3177 } else {
3178 __ Mov(out_reg_lo, in_reg_lo);
3179 __ Mov(out_reg_hi, in_reg_hi);
3180 }
3181 } else {
3182 Register shift_right = locations->GetTemp(0).AsRegister<Register>();
3183 Register shift_left = locations->GetTemp(1).AsRegister<Register>();
3184 Label end;
3185 Label shift_by_32_plus_shift_right;
3186
3187 __ and_(shift_right, rhs.AsRegister<Register>(), ShifterOperand(0x1F));
3188 __ Lsrs(shift_left, rhs.AsRegister<Register>(), 6);
3189 __ rsb(shift_left, shift_right, ShifterOperand(kArmBitsPerWord), AL, kCcKeep);
3190 __ b(&shift_by_32_plus_shift_right, CC);
3191
3192 // out_reg_hi = (reg_hi << shift_left) | (reg_lo >> shift_right).
3193 // out_reg_lo = (reg_lo << shift_left) | (reg_hi >> shift_right).
3194 __ Lsl(out_reg_hi, in_reg_hi, shift_left);
3195 __ Lsr(out_reg_lo, in_reg_lo, shift_right);
3196 __ add(out_reg_hi, out_reg_hi, ShifterOperand(out_reg_lo));
3197 __ Lsl(out_reg_lo, in_reg_lo, shift_left);
3198 __ Lsr(shift_left, in_reg_hi, shift_right);
3199 __ add(out_reg_lo, out_reg_lo, ShifterOperand(shift_left));
3200 __ b(&end);
3201
3202 __ Bind(&shift_by_32_plus_shift_right); // Shift by 32+shift_right.
3203 // out_reg_hi = (reg_hi >> shift_right) | (reg_lo << shift_left).
3204 // out_reg_lo = (reg_lo >> shift_right) | (reg_hi << shift_left).
3205 __ Lsr(out_reg_hi, in_reg_hi, shift_right);
3206 __ Lsl(out_reg_lo, in_reg_lo, shift_left);
3207 __ add(out_reg_hi, out_reg_hi, ShifterOperand(out_reg_lo));
3208 __ Lsr(out_reg_lo, in_reg_lo, shift_right);
3209 __ Lsl(shift_right, in_reg_hi, shift_left);
3210 __ add(out_reg_lo, out_reg_lo, ShifterOperand(shift_right));
3211
3212 __ Bind(&end);
3213 }
3214 }
3215
VisitRor(HRor * ror)3216 void LocationsBuilderARM::VisitRor(HRor* ror) {
3217 LocationSummary* locations =
3218 new (GetGraph()->GetArena()) LocationSummary(ror, LocationSummary::kNoCall);
3219 switch (ror->GetResultType()) {
3220 case Primitive::kPrimInt: {
3221 locations->SetInAt(0, Location::RequiresRegister());
3222 locations->SetInAt(1, Location::RegisterOrConstant(ror->InputAt(1)));
3223 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
3224 break;
3225 }
3226 case Primitive::kPrimLong: {
3227 locations->SetInAt(0, Location::RequiresRegister());
3228 if (ror->InputAt(1)->IsConstant()) {
3229 locations->SetInAt(1, Location::ConstantLocation(ror->InputAt(1)->AsConstant()));
3230 } else {
3231 locations->SetInAt(1, Location::RequiresRegister());
3232 locations->AddTemp(Location::RequiresRegister());
3233 locations->AddTemp(Location::RequiresRegister());
3234 }
3235 locations->SetOut(Location::RequiresRegister(), Location::kOutputOverlap);
3236 break;
3237 }
3238 default:
3239 LOG(FATAL) << "Unexpected operation type " << ror->GetResultType();
3240 }
3241 }
3242
VisitRor(HRor * ror)3243 void InstructionCodeGeneratorARM::VisitRor(HRor* ror) {
3244 LocationSummary* locations = ror->GetLocations();
3245 Primitive::Type type = ror->GetResultType();
3246 switch (type) {
3247 case Primitive::kPrimInt: {
3248 HandleIntegerRotate(locations);
3249 break;
3250 }
3251 case Primitive::kPrimLong: {
3252 HandleLongRotate(locations);
3253 break;
3254 }
3255 default:
3256 LOG(FATAL) << "Unexpected operation type " << type;
3257 UNREACHABLE();
3258 }
3259 }
3260
HandleShift(HBinaryOperation * op)3261 void LocationsBuilderARM::HandleShift(HBinaryOperation* op) {
3262 DCHECK(op->IsShl() || op->IsShr() || op->IsUShr());
3263
3264 LocationSummary* locations =
3265 new (GetGraph()->GetArena()) LocationSummary(op, LocationSummary::kNoCall);
3266
3267 switch (op->GetResultType()) {
3268 case Primitive::kPrimInt: {
3269 locations->SetInAt(0, Location::RequiresRegister());
3270 if (op->InputAt(1)->IsConstant()) {
3271 locations->SetInAt(1, Location::ConstantLocation(op->InputAt(1)->AsConstant()));
3272 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
3273 } else {
3274 locations->SetInAt(1, Location::RequiresRegister());
3275 // Make the output overlap, as it will be used to hold the masked
3276 // second input.
3277 locations->SetOut(Location::RequiresRegister(), Location::kOutputOverlap);
3278 }
3279 break;
3280 }
3281 case Primitive::kPrimLong: {
3282 locations->SetInAt(0, Location::RequiresRegister());
3283 if (op->InputAt(1)->IsConstant()) {
3284 locations->SetInAt(1, Location::ConstantLocation(op->InputAt(1)->AsConstant()));
3285 // For simplicity, use kOutputOverlap even though we only require that low registers
3286 // don't clash with high registers which the register allocator currently guarantees.
3287 locations->SetOut(Location::RequiresRegister(), Location::kOutputOverlap);
3288 } else {
3289 locations->SetInAt(1, Location::RequiresRegister());
3290 locations->AddTemp(Location::RequiresRegister());
3291 locations->SetOut(Location::RequiresRegister(), Location::kOutputOverlap);
3292 }
3293 break;
3294 }
3295 default:
3296 LOG(FATAL) << "Unexpected operation type " << op->GetResultType();
3297 }
3298 }
3299
HandleShift(HBinaryOperation * op)3300 void InstructionCodeGeneratorARM::HandleShift(HBinaryOperation* op) {
3301 DCHECK(op->IsShl() || op->IsShr() || op->IsUShr());
3302
3303 LocationSummary* locations = op->GetLocations();
3304 Location out = locations->Out();
3305 Location first = locations->InAt(0);
3306 Location second = locations->InAt(1);
3307
3308 Primitive::Type type = op->GetResultType();
3309 switch (type) {
3310 case Primitive::kPrimInt: {
3311 Register out_reg = out.AsRegister<Register>();
3312 Register first_reg = first.AsRegister<Register>();
3313 if (second.IsRegister()) {
3314 Register second_reg = second.AsRegister<Register>();
3315 // ARM doesn't mask the shift count so we need to do it ourselves.
3316 __ and_(out_reg, second_reg, ShifterOperand(kMaxIntShiftDistance));
3317 if (op->IsShl()) {
3318 __ Lsl(out_reg, first_reg, out_reg);
3319 } else if (op->IsShr()) {
3320 __ Asr(out_reg, first_reg, out_reg);
3321 } else {
3322 __ Lsr(out_reg, first_reg, out_reg);
3323 }
3324 } else {
3325 int32_t cst = second.GetConstant()->AsIntConstant()->GetValue();
3326 uint32_t shift_value = cst & kMaxIntShiftDistance;
3327 if (shift_value == 0) { // ARM does not support shifting with 0 immediate.
3328 __ Mov(out_reg, first_reg);
3329 } else if (op->IsShl()) {
3330 __ Lsl(out_reg, first_reg, shift_value);
3331 } else if (op->IsShr()) {
3332 __ Asr(out_reg, first_reg, shift_value);
3333 } else {
3334 __ Lsr(out_reg, first_reg, shift_value);
3335 }
3336 }
3337 break;
3338 }
3339 case Primitive::kPrimLong: {
3340 Register o_h = out.AsRegisterPairHigh<Register>();
3341 Register o_l = out.AsRegisterPairLow<Register>();
3342
3343 Register high = first.AsRegisterPairHigh<Register>();
3344 Register low = first.AsRegisterPairLow<Register>();
3345
3346 if (second.IsRegister()) {
3347 Register temp = locations->GetTemp(0).AsRegister<Register>();
3348
3349 Register second_reg = second.AsRegister<Register>();
3350
3351 if (op->IsShl()) {
3352 __ and_(o_l, second_reg, ShifterOperand(kMaxLongShiftDistance));
3353 // Shift the high part
3354 __ Lsl(o_h, high, o_l);
3355 // Shift the low part and `or` what overflew on the high part
3356 __ rsb(temp, o_l, ShifterOperand(kArmBitsPerWord));
3357 __ Lsr(temp, low, temp);
3358 __ orr(o_h, o_h, ShifterOperand(temp));
3359 // If the shift is > 32 bits, override the high part
3360 __ subs(temp, o_l, ShifterOperand(kArmBitsPerWord));
3361 __ it(PL);
3362 __ Lsl(o_h, low, temp, PL);
3363 // Shift the low part
3364 __ Lsl(o_l, low, o_l);
3365 } else if (op->IsShr()) {
3366 __ and_(o_h, second_reg, ShifterOperand(kMaxLongShiftDistance));
3367 // Shift the low part
3368 __ Lsr(o_l, low, o_h);
3369 // Shift the high part and `or` what underflew on the low part
3370 __ rsb(temp, o_h, ShifterOperand(kArmBitsPerWord));
3371 __ Lsl(temp, high, temp);
3372 __ orr(o_l, o_l, ShifterOperand(temp));
3373 // If the shift is > 32 bits, override the low part
3374 __ subs(temp, o_h, ShifterOperand(kArmBitsPerWord));
3375 __ it(PL);
3376 __ Asr(o_l, high, temp, PL);
3377 // Shift the high part
3378 __ Asr(o_h, high, o_h);
3379 } else {
3380 __ and_(o_h, second_reg, ShifterOperand(kMaxLongShiftDistance));
3381 // same as Shr except we use `Lsr`s and not `Asr`s
3382 __ Lsr(o_l, low, o_h);
3383 __ rsb(temp, o_h, ShifterOperand(kArmBitsPerWord));
3384 __ Lsl(temp, high, temp);
3385 __ orr(o_l, o_l, ShifterOperand(temp));
3386 __ subs(temp, o_h, ShifterOperand(kArmBitsPerWord));
3387 __ it(PL);
3388 __ Lsr(o_l, high, temp, PL);
3389 __ Lsr(o_h, high, o_h);
3390 }
3391 } else {
3392 // Register allocator doesn't create partial overlap.
3393 DCHECK_NE(o_l, high);
3394 DCHECK_NE(o_h, low);
3395 int32_t cst = second.GetConstant()->AsIntConstant()->GetValue();
3396 uint32_t shift_value = cst & kMaxLongShiftDistance;
3397 if (shift_value > 32) {
3398 if (op->IsShl()) {
3399 __ Lsl(o_h, low, shift_value - 32);
3400 __ LoadImmediate(o_l, 0);
3401 } else if (op->IsShr()) {
3402 __ Asr(o_l, high, shift_value - 32);
3403 __ Asr(o_h, high, 31);
3404 } else {
3405 __ Lsr(o_l, high, shift_value - 32);
3406 __ LoadImmediate(o_h, 0);
3407 }
3408 } else if (shift_value == 32) {
3409 if (op->IsShl()) {
3410 __ mov(o_h, ShifterOperand(low));
3411 __ LoadImmediate(o_l, 0);
3412 } else if (op->IsShr()) {
3413 __ mov(o_l, ShifterOperand(high));
3414 __ Asr(o_h, high, 31);
3415 } else {
3416 __ mov(o_l, ShifterOperand(high));
3417 __ LoadImmediate(o_h, 0);
3418 }
3419 } else if (shift_value == 1) {
3420 if (op->IsShl()) {
3421 __ Lsls(o_l, low, 1);
3422 __ adc(o_h, high, ShifterOperand(high));
3423 } else if (op->IsShr()) {
3424 __ Asrs(o_h, high, 1);
3425 __ Rrx(o_l, low);
3426 } else {
3427 __ Lsrs(o_h, high, 1);
3428 __ Rrx(o_l, low);
3429 }
3430 } else {
3431 DCHECK(2 <= shift_value && shift_value < 32) << shift_value;
3432 if (op->IsShl()) {
3433 __ Lsl(o_h, high, shift_value);
3434 __ orr(o_h, o_h, ShifterOperand(low, LSR, 32 - shift_value));
3435 __ Lsl(o_l, low, shift_value);
3436 } else if (op->IsShr()) {
3437 __ Lsr(o_l, low, shift_value);
3438 __ orr(o_l, o_l, ShifterOperand(high, LSL, 32 - shift_value));
3439 __ Asr(o_h, high, shift_value);
3440 } else {
3441 __ Lsr(o_l, low, shift_value);
3442 __ orr(o_l, o_l, ShifterOperand(high, LSL, 32 - shift_value));
3443 __ Lsr(o_h, high, shift_value);
3444 }
3445 }
3446 }
3447 break;
3448 }
3449 default:
3450 LOG(FATAL) << "Unexpected operation type " << type;
3451 UNREACHABLE();
3452 }
3453 }
3454
VisitShl(HShl * shl)3455 void LocationsBuilderARM::VisitShl(HShl* shl) {
3456 HandleShift(shl);
3457 }
3458
VisitShl(HShl * shl)3459 void InstructionCodeGeneratorARM::VisitShl(HShl* shl) {
3460 HandleShift(shl);
3461 }
3462
VisitShr(HShr * shr)3463 void LocationsBuilderARM::VisitShr(HShr* shr) {
3464 HandleShift(shr);
3465 }
3466
VisitShr(HShr * shr)3467 void InstructionCodeGeneratorARM::VisitShr(HShr* shr) {
3468 HandleShift(shr);
3469 }
3470
VisitUShr(HUShr * ushr)3471 void LocationsBuilderARM::VisitUShr(HUShr* ushr) {
3472 HandleShift(ushr);
3473 }
3474
VisitUShr(HUShr * ushr)3475 void InstructionCodeGeneratorARM::VisitUShr(HUShr* ushr) {
3476 HandleShift(ushr);
3477 }
3478
VisitNewInstance(HNewInstance * instruction)3479 void LocationsBuilderARM::VisitNewInstance(HNewInstance* instruction) {
3480 LocationSummary* locations =
3481 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCall);
3482 if (instruction->IsStringAlloc()) {
3483 locations->AddTemp(Location::RegisterLocation(kMethodRegisterArgument));
3484 } else {
3485 InvokeRuntimeCallingConvention calling_convention;
3486 locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(0)));
3487 locations->SetInAt(1, Location::RegisterLocation(calling_convention.GetRegisterAt(1)));
3488 }
3489 locations->SetOut(Location::RegisterLocation(R0));
3490 }
3491
VisitNewInstance(HNewInstance * instruction)3492 void InstructionCodeGeneratorARM::VisitNewInstance(HNewInstance* instruction) {
3493 // Note: if heap poisoning is enabled, the entry point takes cares
3494 // of poisoning the reference.
3495 if (instruction->IsStringAlloc()) {
3496 // String is allocated through StringFactory. Call NewEmptyString entry point.
3497 Register temp = instruction->GetLocations()->GetTemp(0).AsRegister<Register>();
3498 MemberOffset code_offset = ArtMethod::EntryPointFromQuickCompiledCodeOffset(kArmWordSize);
3499 __ LoadFromOffset(kLoadWord, temp, TR, QUICK_ENTRY_POINT(pNewEmptyString));
3500 __ LoadFromOffset(kLoadWord, LR, temp, code_offset.Int32Value());
3501 __ blx(LR);
3502 codegen_->RecordPcInfo(instruction, instruction->GetDexPc());
3503 } else {
3504 codegen_->InvokeRuntime(instruction->GetEntrypoint(),
3505 instruction,
3506 instruction->GetDexPc(),
3507 nullptr);
3508 CheckEntrypointTypes<kQuickAllocObjectWithAccessCheck, void*, uint32_t, ArtMethod*>();
3509 }
3510 }
3511
VisitNewArray(HNewArray * instruction)3512 void LocationsBuilderARM::VisitNewArray(HNewArray* instruction) {
3513 LocationSummary* locations =
3514 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCall);
3515 InvokeRuntimeCallingConvention calling_convention;
3516 locations->AddTemp(Location::RegisterLocation(calling_convention.GetRegisterAt(0)));
3517 locations->SetOut(Location::RegisterLocation(R0));
3518 locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(1)));
3519 locations->SetInAt(1, Location::RegisterLocation(calling_convention.GetRegisterAt(2)));
3520 }
3521
VisitNewArray(HNewArray * instruction)3522 void InstructionCodeGeneratorARM::VisitNewArray(HNewArray* instruction) {
3523 InvokeRuntimeCallingConvention calling_convention;
3524 __ LoadImmediate(calling_convention.GetRegisterAt(0), instruction->GetTypeIndex());
3525 // Note: if heap poisoning is enabled, the entry point takes cares
3526 // of poisoning the reference.
3527 codegen_->InvokeRuntime(instruction->GetEntrypoint(),
3528 instruction,
3529 instruction->GetDexPc(),
3530 nullptr);
3531 CheckEntrypointTypes<kQuickAllocArrayWithAccessCheck, void*, uint32_t, int32_t, ArtMethod*>();
3532 }
3533
VisitParameterValue(HParameterValue * instruction)3534 void LocationsBuilderARM::VisitParameterValue(HParameterValue* instruction) {
3535 LocationSummary* locations =
3536 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
3537 Location location = parameter_visitor_.GetNextLocation(instruction->GetType());
3538 if (location.IsStackSlot()) {
3539 location = Location::StackSlot(location.GetStackIndex() + codegen_->GetFrameSize());
3540 } else if (location.IsDoubleStackSlot()) {
3541 location = Location::DoubleStackSlot(location.GetStackIndex() + codegen_->GetFrameSize());
3542 }
3543 locations->SetOut(location);
3544 }
3545
VisitParameterValue(HParameterValue * instruction ATTRIBUTE_UNUSED)3546 void InstructionCodeGeneratorARM::VisitParameterValue(
3547 HParameterValue* instruction ATTRIBUTE_UNUSED) {
3548 // Nothing to do, the parameter is already at its location.
3549 }
3550
VisitCurrentMethod(HCurrentMethod * instruction)3551 void LocationsBuilderARM::VisitCurrentMethod(HCurrentMethod* instruction) {
3552 LocationSummary* locations =
3553 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
3554 locations->SetOut(Location::RegisterLocation(kMethodRegisterArgument));
3555 }
3556
VisitCurrentMethod(HCurrentMethod * instruction ATTRIBUTE_UNUSED)3557 void InstructionCodeGeneratorARM::VisitCurrentMethod(HCurrentMethod* instruction ATTRIBUTE_UNUSED) {
3558 // Nothing to do, the method is already at its location.
3559 }
3560
VisitNot(HNot * not_)3561 void LocationsBuilderARM::VisitNot(HNot* not_) {
3562 LocationSummary* locations =
3563 new (GetGraph()->GetArena()) LocationSummary(not_, LocationSummary::kNoCall);
3564 locations->SetInAt(0, Location::RequiresRegister());
3565 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
3566 }
3567
VisitNot(HNot * not_)3568 void InstructionCodeGeneratorARM::VisitNot(HNot* not_) {
3569 LocationSummary* locations = not_->GetLocations();
3570 Location out = locations->Out();
3571 Location in = locations->InAt(0);
3572 switch (not_->GetResultType()) {
3573 case Primitive::kPrimInt:
3574 __ mvn(out.AsRegister<Register>(), ShifterOperand(in.AsRegister<Register>()));
3575 break;
3576
3577 case Primitive::kPrimLong:
3578 __ mvn(out.AsRegisterPairLow<Register>(),
3579 ShifterOperand(in.AsRegisterPairLow<Register>()));
3580 __ mvn(out.AsRegisterPairHigh<Register>(),
3581 ShifterOperand(in.AsRegisterPairHigh<Register>()));
3582 break;
3583
3584 default:
3585 LOG(FATAL) << "Unimplemented type for not operation " << not_->GetResultType();
3586 }
3587 }
3588
VisitBooleanNot(HBooleanNot * bool_not)3589 void LocationsBuilderARM::VisitBooleanNot(HBooleanNot* bool_not) {
3590 LocationSummary* locations =
3591 new (GetGraph()->GetArena()) LocationSummary(bool_not, LocationSummary::kNoCall);
3592 locations->SetInAt(0, Location::RequiresRegister());
3593 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
3594 }
3595
VisitBooleanNot(HBooleanNot * bool_not)3596 void InstructionCodeGeneratorARM::VisitBooleanNot(HBooleanNot* bool_not) {
3597 LocationSummary* locations = bool_not->GetLocations();
3598 Location out = locations->Out();
3599 Location in = locations->InAt(0);
3600 __ eor(out.AsRegister<Register>(), in.AsRegister<Register>(), ShifterOperand(1));
3601 }
3602
VisitCompare(HCompare * compare)3603 void LocationsBuilderARM::VisitCompare(HCompare* compare) {
3604 LocationSummary* locations =
3605 new (GetGraph()->GetArena()) LocationSummary(compare, LocationSummary::kNoCall);
3606 switch (compare->InputAt(0)->GetType()) {
3607 case Primitive::kPrimBoolean:
3608 case Primitive::kPrimByte:
3609 case Primitive::kPrimShort:
3610 case Primitive::kPrimChar:
3611 case Primitive::kPrimInt:
3612 case Primitive::kPrimLong: {
3613 locations->SetInAt(0, Location::RequiresRegister());
3614 locations->SetInAt(1, Location::RequiresRegister());
3615 // Output overlaps because it is written before doing the low comparison.
3616 locations->SetOut(Location::RequiresRegister(), Location::kOutputOverlap);
3617 break;
3618 }
3619 case Primitive::kPrimFloat:
3620 case Primitive::kPrimDouble: {
3621 locations->SetInAt(0, Location::RequiresFpuRegister());
3622 locations->SetInAt(1, Location::RequiresFpuRegister());
3623 locations->SetOut(Location::RequiresRegister());
3624 break;
3625 }
3626 default:
3627 LOG(FATAL) << "Unexpected type for compare operation " << compare->InputAt(0)->GetType();
3628 }
3629 }
3630
VisitCompare(HCompare * compare)3631 void InstructionCodeGeneratorARM::VisitCompare(HCompare* compare) {
3632 LocationSummary* locations = compare->GetLocations();
3633 Register out = locations->Out().AsRegister<Register>();
3634 Location left = locations->InAt(0);
3635 Location right = locations->InAt(1);
3636
3637 Label less, greater, done;
3638 Primitive::Type type = compare->InputAt(0)->GetType();
3639 Condition less_cond;
3640 switch (type) {
3641 case Primitive::kPrimBoolean:
3642 case Primitive::kPrimByte:
3643 case Primitive::kPrimShort:
3644 case Primitive::kPrimChar:
3645 case Primitive::kPrimInt: {
3646 __ LoadImmediate(out, 0);
3647 __ cmp(left.AsRegister<Register>(),
3648 ShifterOperand(right.AsRegister<Register>())); // Signed compare.
3649 less_cond = LT;
3650 break;
3651 }
3652 case Primitive::kPrimLong: {
3653 __ cmp(left.AsRegisterPairHigh<Register>(),
3654 ShifterOperand(right.AsRegisterPairHigh<Register>())); // Signed compare.
3655 __ b(&less, LT);
3656 __ b(&greater, GT);
3657 // Do LoadImmediate before the last `cmp`, as LoadImmediate might affect the status flags.
3658 __ LoadImmediate(out, 0);
3659 __ cmp(left.AsRegisterPairLow<Register>(),
3660 ShifterOperand(right.AsRegisterPairLow<Register>())); // Unsigned compare.
3661 less_cond = LO;
3662 break;
3663 }
3664 case Primitive::kPrimFloat:
3665 case Primitive::kPrimDouble: {
3666 __ LoadImmediate(out, 0);
3667 if (type == Primitive::kPrimFloat) {
3668 __ vcmps(left.AsFpuRegister<SRegister>(), right.AsFpuRegister<SRegister>());
3669 } else {
3670 __ vcmpd(FromLowSToD(left.AsFpuRegisterPairLow<SRegister>()),
3671 FromLowSToD(right.AsFpuRegisterPairLow<SRegister>()));
3672 }
3673 __ vmstat(); // transfer FP status register to ARM APSR.
3674 less_cond = ARMFPCondition(kCondLT, compare->IsGtBias());
3675 break;
3676 }
3677 default:
3678 LOG(FATAL) << "Unexpected compare type " << type;
3679 UNREACHABLE();
3680 }
3681
3682 __ b(&done, EQ);
3683 __ b(&less, less_cond);
3684
3685 __ Bind(&greater);
3686 __ LoadImmediate(out, 1);
3687 __ b(&done);
3688
3689 __ Bind(&less);
3690 __ LoadImmediate(out, -1);
3691
3692 __ Bind(&done);
3693 }
3694
VisitPhi(HPhi * instruction)3695 void LocationsBuilderARM::VisitPhi(HPhi* instruction) {
3696 LocationSummary* locations =
3697 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
3698 for (size_t i = 0, e = instruction->InputCount(); i < e; ++i) {
3699 locations->SetInAt(i, Location::Any());
3700 }
3701 locations->SetOut(Location::Any());
3702 }
3703
VisitPhi(HPhi * instruction ATTRIBUTE_UNUSED)3704 void InstructionCodeGeneratorARM::VisitPhi(HPhi* instruction ATTRIBUTE_UNUSED) {
3705 LOG(FATAL) << "Unreachable";
3706 }
3707
GenerateMemoryBarrier(MemBarrierKind kind)3708 void CodeGeneratorARM::GenerateMemoryBarrier(MemBarrierKind kind) {
3709 // TODO (ported from quick): revisit ARM barrier kinds.
3710 DmbOptions flavor = DmbOptions::ISH; // Quiet C++ warnings.
3711 switch (kind) {
3712 case MemBarrierKind::kAnyStore:
3713 case MemBarrierKind::kLoadAny:
3714 case MemBarrierKind::kAnyAny: {
3715 flavor = DmbOptions::ISH;
3716 break;
3717 }
3718 case MemBarrierKind::kStoreStore: {
3719 flavor = DmbOptions::ISHST;
3720 break;
3721 }
3722 default:
3723 LOG(FATAL) << "Unexpected memory barrier " << kind;
3724 }
3725 __ dmb(flavor);
3726 }
3727
GenerateWideAtomicLoad(Register addr,uint32_t offset,Register out_lo,Register out_hi)3728 void InstructionCodeGeneratorARM::GenerateWideAtomicLoad(Register addr,
3729 uint32_t offset,
3730 Register out_lo,
3731 Register out_hi) {
3732 if (offset != 0) {
3733 // Ensure `out_lo` is different from `addr`, so that loading
3734 // `offset` into `out_lo` does not clutter `addr`.
3735 DCHECK_NE(out_lo, addr);
3736 __ LoadImmediate(out_lo, offset);
3737 __ add(IP, addr, ShifterOperand(out_lo));
3738 addr = IP;
3739 }
3740 __ ldrexd(out_lo, out_hi, addr);
3741 }
3742
GenerateWideAtomicStore(Register addr,uint32_t offset,Register value_lo,Register value_hi,Register temp1,Register temp2,HInstruction * instruction)3743 void InstructionCodeGeneratorARM::GenerateWideAtomicStore(Register addr,
3744 uint32_t offset,
3745 Register value_lo,
3746 Register value_hi,
3747 Register temp1,
3748 Register temp2,
3749 HInstruction* instruction) {
3750 Label fail;
3751 if (offset != 0) {
3752 __ LoadImmediate(temp1, offset);
3753 __ add(IP, addr, ShifterOperand(temp1));
3754 addr = IP;
3755 }
3756 __ Bind(&fail);
3757 // We need a load followed by store. (The address used in a STREX instruction must
3758 // be the same as the address in the most recently executed LDREX instruction.)
3759 __ ldrexd(temp1, temp2, addr);
3760 codegen_->MaybeRecordImplicitNullCheck(instruction);
3761 __ strexd(temp1, value_lo, value_hi, addr);
3762 __ CompareAndBranchIfNonZero(temp1, &fail);
3763 }
3764
HandleFieldSet(HInstruction * instruction,const FieldInfo & field_info)3765 void LocationsBuilderARM::HandleFieldSet(HInstruction* instruction, const FieldInfo& field_info) {
3766 DCHECK(instruction->IsInstanceFieldSet() || instruction->IsStaticFieldSet());
3767
3768 LocationSummary* locations =
3769 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
3770 locations->SetInAt(0, Location::RequiresRegister());
3771
3772 Primitive::Type field_type = field_info.GetFieldType();
3773 if (Primitive::IsFloatingPointType(field_type)) {
3774 locations->SetInAt(1, Location::RequiresFpuRegister());
3775 } else {
3776 locations->SetInAt(1, Location::RequiresRegister());
3777 }
3778
3779 bool is_wide = field_type == Primitive::kPrimLong || field_type == Primitive::kPrimDouble;
3780 bool generate_volatile = field_info.IsVolatile()
3781 && is_wide
3782 && !codegen_->GetInstructionSetFeatures().HasAtomicLdrdAndStrd();
3783 bool needs_write_barrier =
3784 CodeGenerator::StoreNeedsWriteBarrier(field_type, instruction->InputAt(1));
3785 // Temporary registers for the write barrier.
3786 // TODO: consider renaming StoreNeedsWriteBarrier to StoreNeedsGCMark.
3787 if (needs_write_barrier) {
3788 locations->AddTemp(Location::RequiresRegister()); // Possibly used for reference poisoning too.
3789 locations->AddTemp(Location::RequiresRegister());
3790 } else if (generate_volatile) {
3791 // ARM encoding have some additional constraints for ldrexd/strexd:
3792 // - registers need to be consecutive
3793 // - the first register should be even but not R14.
3794 // We don't test for ARM yet, and the assertion makes sure that we
3795 // revisit this if we ever enable ARM encoding.
3796 DCHECK_EQ(InstructionSet::kThumb2, codegen_->GetInstructionSet());
3797
3798 locations->AddTemp(Location::RequiresRegister());
3799 locations->AddTemp(Location::RequiresRegister());
3800 if (field_type == Primitive::kPrimDouble) {
3801 // For doubles we need two more registers to copy the value.
3802 locations->AddTemp(Location::RegisterLocation(R2));
3803 locations->AddTemp(Location::RegisterLocation(R3));
3804 }
3805 }
3806 }
3807
HandleFieldSet(HInstruction * instruction,const FieldInfo & field_info,bool value_can_be_null)3808 void InstructionCodeGeneratorARM::HandleFieldSet(HInstruction* instruction,
3809 const FieldInfo& field_info,
3810 bool value_can_be_null) {
3811 DCHECK(instruction->IsInstanceFieldSet() || instruction->IsStaticFieldSet());
3812
3813 LocationSummary* locations = instruction->GetLocations();
3814 Register base = locations->InAt(0).AsRegister<Register>();
3815 Location value = locations->InAt(1);
3816
3817 bool is_volatile = field_info.IsVolatile();
3818 bool atomic_ldrd_strd = codegen_->GetInstructionSetFeatures().HasAtomicLdrdAndStrd();
3819 Primitive::Type field_type = field_info.GetFieldType();
3820 uint32_t offset = field_info.GetFieldOffset().Uint32Value();
3821 bool needs_write_barrier =
3822 CodeGenerator::StoreNeedsWriteBarrier(field_type, instruction->InputAt(1));
3823
3824 if (is_volatile) {
3825 codegen_->GenerateMemoryBarrier(MemBarrierKind::kAnyStore);
3826 }
3827
3828 switch (field_type) {
3829 case Primitive::kPrimBoolean:
3830 case Primitive::kPrimByte: {
3831 __ StoreToOffset(kStoreByte, value.AsRegister<Register>(), base, offset);
3832 break;
3833 }
3834
3835 case Primitive::kPrimShort:
3836 case Primitive::kPrimChar: {
3837 __ StoreToOffset(kStoreHalfword, value.AsRegister<Register>(), base, offset);
3838 break;
3839 }
3840
3841 case Primitive::kPrimInt:
3842 case Primitive::kPrimNot: {
3843 if (kPoisonHeapReferences && needs_write_barrier) {
3844 // Note that in the case where `value` is a null reference,
3845 // we do not enter this block, as a null reference does not
3846 // need poisoning.
3847 DCHECK_EQ(field_type, Primitive::kPrimNot);
3848 Register temp = locations->GetTemp(0).AsRegister<Register>();
3849 __ Mov(temp, value.AsRegister<Register>());
3850 __ PoisonHeapReference(temp);
3851 __ StoreToOffset(kStoreWord, temp, base, offset);
3852 } else {
3853 __ StoreToOffset(kStoreWord, value.AsRegister<Register>(), base, offset);
3854 }
3855 break;
3856 }
3857
3858 case Primitive::kPrimLong: {
3859 if (is_volatile && !atomic_ldrd_strd) {
3860 GenerateWideAtomicStore(base, offset,
3861 value.AsRegisterPairLow<Register>(),
3862 value.AsRegisterPairHigh<Register>(),
3863 locations->GetTemp(0).AsRegister<Register>(),
3864 locations->GetTemp(1).AsRegister<Register>(),
3865 instruction);
3866 } else {
3867 __ StoreToOffset(kStoreWordPair, value.AsRegisterPairLow<Register>(), base, offset);
3868 codegen_->MaybeRecordImplicitNullCheck(instruction);
3869 }
3870 break;
3871 }
3872
3873 case Primitive::kPrimFloat: {
3874 __ StoreSToOffset(value.AsFpuRegister<SRegister>(), base, offset);
3875 break;
3876 }
3877
3878 case Primitive::kPrimDouble: {
3879 DRegister value_reg = FromLowSToD(value.AsFpuRegisterPairLow<SRegister>());
3880 if (is_volatile && !atomic_ldrd_strd) {
3881 Register value_reg_lo = locations->GetTemp(0).AsRegister<Register>();
3882 Register value_reg_hi = locations->GetTemp(1).AsRegister<Register>();
3883
3884 __ vmovrrd(value_reg_lo, value_reg_hi, value_reg);
3885
3886 GenerateWideAtomicStore(base, offset,
3887 value_reg_lo,
3888 value_reg_hi,
3889 locations->GetTemp(2).AsRegister<Register>(),
3890 locations->GetTemp(3).AsRegister<Register>(),
3891 instruction);
3892 } else {
3893 __ StoreDToOffset(value_reg, base, offset);
3894 codegen_->MaybeRecordImplicitNullCheck(instruction);
3895 }
3896 break;
3897 }
3898
3899 case Primitive::kPrimVoid:
3900 LOG(FATAL) << "Unreachable type " << field_type;
3901 UNREACHABLE();
3902 }
3903
3904 // Longs and doubles are handled in the switch.
3905 if (field_type != Primitive::kPrimLong && field_type != Primitive::kPrimDouble) {
3906 codegen_->MaybeRecordImplicitNullCheck(instruction);
3907 }
3908
3909 if (CodeGenerator::StoreNeedsWriteBarrier(field_type, instruction->InputAt(1))) {
3910 Register temp = locations->GetTemp(0).AsRegister<Register>();
3911 Register card = locations->GetTemp(1).AsRegister<Register>();
3912 codegen_->MarkGCCard(
3913 temp, card, base, value.AsRegister<Register>(), value_can_be_null);
3914 }
3915
3916 if (is_volatile) {
3917 codegen_->GenerateMemoryBarrier(MemBarrierKind::kAnyAny);
3918 }
3919 }
3920
HandleFieldGet(HInstruction * instruction,const FieldInfo & field_info)3921 void LocationsBuilderARM::HandleFieldGet(HInstruction* instruction, const FieldInfo& field_info) {
3922 DCHECK(instruction->IsInstanceFieldGet() || instruction->IsStaticFieldGet());
3923
3924 bool object_field_get_with_read_barrier =
3925 kEmitCompilerReadBarrier && (field_info.GetFieldType() == Primitive::kPrimNot);
3926 LocationSummary* locations =
3927 new (GetGraph()->GetArena()) LocationSummary(instruction,
3928 object_field_get_with_read_barrier ?
3929 LocationSummary::kCallOnSlowPath :
3930 LocationSummary::kNoCall);
3931 locations->SetInAt(0, Location::RequiresRegister());
3932
3933 bool volatile_for_double = field_info.IsVolatile()
3934 && (field_info.GetFieldType() == Primitive::kPrimDouble)
3935 && !codegen_->GetInstructionSetFeatures().HasAtomicLdrdAndStrd();
3936 // The output overlaps in case of volatile long: we don't want the
3937 // code generated by GenerateWideAtomicLoad to overwrite the
3938 // object's location. Likewise, in the case of an object field get
3939 // with read barriers enabled, we do not want the load to overwrite
3940 // the object's location, as we need it to emit the read barrier.
3941 bool overlap = (field_info.IsVolatile() && (field_info.GetFieldType() == Primitive::kPrimLong)) ||
3942 object_field_get_with_read_barrier;
3943
3944 if (Primitive::IsFloatingPointType(instruction->GetType())) {
3945 locations->SetOut(Location::RequiresFpuRegister());
3946 } else {
3947 locations->SetOut(Location::RequiresRegister(),
3948 (overlap ? Location::kOutputOverlap : Location::kNoOutputOverlap));
3949 }
3950 if (volatile_for_double) {
3951 // ARM encoding have some additional constraints for ldrexd/strexd:
3952 // - registers need to be consecutive
3953 // - the first register should be even but not R14.
3954 // We don't test for ARM yet, and the assertion makes sure that we
3955 // revisit this if we ever enable ARM encoding.
3956 DCHECK_EQ(InstructionSet::kThumb2, codegen_->GetInstructionSet());
3957 locations->AddTemp(Location::RequiresRegister());
3958 locations->AddTemp(Location::RequiresRegister());
3959 } else if (object_field_get_with_read_barrier && kUseBakerReadBarrier) {
3960 // We need a temporary register for the read barrier marking slow
3961 // path in CodeGeneratorARM::GenerateFieldLoadWithBakerReadBarrier.
3962 locations->AddTemp(Location::RequiresRegister());
3963 }
3964 }
3965
ArmEncodableConstantOrRegister(HInstruction * constant,Opcode opcode)3966 Location LocationsBuilderARM::ArmEncodableConstantOrRegister(HInstruction* constant,
3967 Opcode opcode) {
3968 DCHECK(!Primitive::IsFloatingPointType(constant->GetType()));
3969 if (constant->IsConstant() &&
3970 CanEncodeConstantAsImmediate(constant->AsConstant(), opcode)) {
3971 return Location::ConstantLocation(constant->AsConstant());
3972 }
3973 return Location::RequiresRegister();
3974 }
3975
CanEncodeConstantAsImmediate(HConstant * input_cst,Opcode opcode)3976 bool LocationsBuilderARM::CanEncodeConstantAsImmediate(HConstant* input_cst,
3977 Opcode opcode) {
3978 uint64_t value = static_cast<uint64_t>(Int64FromConstant(input_cst));
3979 if (Primitive::Is64BitType(input_cst->GetType())) {
3980 return CanEncodeConstantAsImmediate(Low32Bits(value), opcode) &&
3981 CanEncodeConstantAsImmediate(High32Bits(value), opcode);
3982 } else {
3983 return CanEncodeConstantAsImmediate(Low32Bits(value), opcode);
3984 }
3985 }
3986
CanEncodeConstantAsImmediate(uint32_t value,Opcode opcode)3987 bool LocationsBuilderARM::CanEncodeConstantAsImmediate(uint32_t value, Opcode opcode) {
3988 ShifterOperand so;
3989 ArmAssembler* assembler = codegen_->GetAssembler();
3990 if (assembler->ShifterOperandCanHold(kNoRegister, kNoRegister, opcode, value, &so)) {
3991 return true;
3992 }
3993 Opcode neg_opcode = kNoOperand;
3994 switch (opcode) {
3995 case AND:
3996 neg_opcode = BIC;
3997 break;
3998 case ORR:
3999 neg_opcode = ORN;
4000 break;
4001 default:
4002 return false;
4003 }
4004 return assembler->ShifterOperandCanHold(kNoRegister, kNoRegister, neg_opcode, ~value, &so);
4005 }
4006
HandleFieldGet(HInstruction * instruction,const FieldInfo & field_info)4007 void InstructionCodeGeneratorARM::HandleFieldGet(HInstruction* instruction,
4008 const FieldInfo& field_info) {
4009 DCHECK(instruction->IsInstanceFieldGet() || instruction->IsStaticFieldGet());
4010
4011 LocationSummary* locations = instruction->GetLocations();
4012 Location base_loc = locations->InAt(0);
4013 Register base = base_loc.AsRegister<Register>();
4014 Location out = locations->Out();
4015 bool is_volatile = field_info.IsVolatile();
4016 bool atomic_ldrd_strd = codegen_->GetInstructionSetFeatures().HasAtomicLdrdAndStrd();
4017 Primitive::Type field_type = field_info.GetFieldType();
4018 uint32_t offset = field_info.GetFieldOffset().Uint32Value();
4019
4020 switch (field_type) {
4021 case Primitive::kPrimBoolean:
4022 __ LoadFromOffset(kLoadUnsignedByte, out.AsRegister<Register>(), base, offset);
4023 break;
4024
4025 case Primitive::kPrimByte:
4026 __ LoadFromOffset(kLoadSignedByte, out.AsRegister<Register>(), base, offset);
4027 break;
4028
4029 case Primitive::kPrimShort:
4030 __ LoadFromOffset(kLoadSignedHalfword, out.AsRegister<Register>(), base, offset);
4031 break;
4032
4033 case Primitive::kPrimChar:
4034 __ LoadFromOffset(kLoadUnsignedHalfword, out.AsRegister<Register>(), base, offset);
4035 break;
4036
4037 case Primitive::kPrimInt:
4038 __ LoadFromOffset(kLoadWord, out.AsRegister<Register>(), base, offset);
4039 break;
4040
4041 case Primitive::kPrimNot: {
4042 // /* HeapReference<Object> */ out = *(base + offset)
4043 if (kEmitCompilerReadBarrier && kUseBakerReadBarrier) {
4044 Location temp_loc = locations->GetTemp(0);
4045 // Note that a potential implicit null check is handled in this
4046 // CodeGeneratorARM::GenerateFieldLoadWithBakerReadBarrier call.
4047 codegen_->GenerateFieldLoadWithBakerReadBarrier(
4048 instruction, out, base, offset, temp_loc, /* needs_null_check */ true);
4049 if (is_volatile) {
4050 codegen_->GenerateMemoryBarrier(MemBarrierKind::kLoadAny);
4051 }
4052 } else {
4053 __ LoadFromOffset(kLoadWord, out.AsRegister<Register>(), base, offset);
4054 codegen_->MaybeRecordImplicitNullCheck(instruction);
4055 if (is_volatile) {
4056 codegen_->GenerateMemoryBarrier(MemBarrierKind::kLoadAny);
4057 }
4058 // If read barriers are enabled, emit read barriers other than
4059 // Baker's using a slow path (and also unpoison the loaded
4060 // reference, if heap poisoning is enabled).
4061 codegen_->MaybeGenerateReadBarrierSlow(instruction, out, out, base_loc, offset);
4062 }
4063 break;
4064 }
4065
4066 case Primitive::kPrimLong:
4067 if (is_volatile && !atomic_ldrd_strd) {
4068 GenerateWideAtomicLoad(base, offset,
4069 out.AsRegisterPairLow<Register>(),
4070 out.AsRegisterPairHigh<Register>());
4071 } else {
4072 __ LoadFromOffset(kLoadWordPair, out.AsRegisterPairLow<Register>(), base, offset);
4073 }
4074 break;
4075
4076 case Primitive::kPrimFloat:
4077 __ LoadSFromOffset(out.AsFpuRegister<SRegister>(), base, offset);
4078 break;
4079
4080 case Primitive::kPrimDouble: {
4081 DRegister out_reg = FromLowSToD(out.AsFpuRegisterPairLow<SRegister>());
4082 if (is_volatile && !atomic_ldrd_strd) {
4083 Register lo = locations->GetTemp(0).AsRegister<Register>();
4084 Register hi = locations->GetTemp(1).AsRegister<Register>();
4085 GenerateWideAtomicLoad(base, offset, lo, hi);
4086 codegen_->MaybeRecordImplicitNullCheck(instruction);
4087 __ vmovdrr(out_reg, lo, hi);
4088 } else {
4089 __ LoadDFromOffset(out_reg, base, offset);
4090 codegen_->MaybeRecordImplicitNullCheck(instruction);
4091 }
4092 break;
4093 }
4094
4095 case Primitive::kPrimVoid:
4096 LOG(FATAL) << "Unreachable type " << field_type;
4097 UNREACHABLE();
4098 }
4099
4100 if (field_type == Primitive::kPrimNot || field_type == Primitive::kPrimDouble) {
4101 // Potential implicit null checks, in the case of reference or
4102 // double fields, are handled in the previous switch statement.
4103 } else {
4104 codegen_->MaybeRecordImplicitNullCheck(instruction);
4105 }
4106
4107 if (is_volatile) {
4108 if (field_type == Primitive::kPrimNot) {
4109 // Memory barriers, in the case of references, are also handled
4110 // in the previous switch statement.
4111 } else {
4112 codegen_->GenerateMemoryBarrier(MemBarrierKind::kLoadAny);
4113 }
4114 }
4115 }
4116
VisitInstanceFieldSet(HInstanceFieldSet * instruction)4117 void LocationsBuilderARM::VisitInstanceFieldSet(HInstanceFieldSet* instruction) {
4118 HandleFieldSet(instruction, instruction->GetFieldInfo());
4119 }
4120
VisitInstanceFieldSet(HInstanceFieldSet * instruction)4121 void InstructionCodeGeneratorARM::VisitInstanceFieldSet(HInstanceFieldSet* instruction) {
4122 HandleFieldSet(instruction, instruction->GetFieldInfo(), instruction->GetValueCanBeNull());
4123 }
4124
VisitInstanceFieldGet(HInstanceFieldGet * instruction)4125 void LocationsBuilderARM::VisitInstanceFieldGet(HInstanceFieldGet* instruction) {
4126 HandleFieldGet(instruction, instruction->GetFieldInfo());
4127 }
4128
VisitInstanceFieldGet(HInstanceFieldGet * instruction)4129 void InstructionCodeGeneratorARM::VisitInstanceFieldGet(HInstanceFieldGet* instruction) {
4130 HandleFieldGet(instruction, instruction->GetFieldInfo());
4131 }
4132
VisitStaticFieldGet(HStaticFieldGet * instruction)4133 void LocationsBuilderARM::VisitStaticFieldGet(HStaticFieldGet* instruction) {
4134 HandleFieldGet(instruction, instruction->GetFieldInfo());
4135 }
4136
VisitStaticFieldGet(HStaticFieldGet * instruction)4137 void InstructionCodeGeneratorARM::VisitStaticFieldGet(HStaticFieldGet* instruction) {
4138 HandleFieldGet(instruction, instruction->GetFieldInfo());
4139 }
4140
VisitStaticFieldSet(HStaticFieldSet * instruction)4141 void LocationsBuilderARM::VisitStaticFieldSet(HStaticFieldSet* instruction) {
4142 HandleFieldSet(instruction, instruction->GetFieldInfo());
4143 }
4144
VisitStaticFieldSet(HStaticFieldSet * instruction)4145 void InstructionCodeGeneratorARM::VisitStaticFieldSet(HStaticFieldSet* instruction) {
4146 HandleFieldSet(instruction, instruction->GetFieldInfo(), instruction->GetValueCanBeNull());
4147 }
4148
VisitUnresolvedInstanceFieldGet(HUnresolvedInstanceFieldGet * instruction)4149 void LocationsBuilderARM::VisitUnresolvedInstanceFieldGet(
4150 HUnresolvedInstanceFieldGet* instruction) {
4151 FieldAccessCallingConventionARM calling_convention;
4152 codegen_->CreateUnresolvedFieldLocationSummary(
4153 instruction, instruction->GetFieldType(), calling_convention);
4154 }
4155
VisitUnresolvedInstanceFieldGet(HUnresolvedInstanceFieldGet * instruction)4156 void InstructionCodeGeneratorARM::VisitUnresolvedInstanceFieldGet(
4157 HUnresolvedInstanceFieldGet* instruction) {
4158 FieldAccessCallingConventionARM calling_convention;
4159 codegen_->GenerateUnresolvedFieldAccess(instruction,
4160 instruction->GetFieldType(),
4161 instruction->GetFieldIndex(),
4162 instruction->GetDexPc(),
4163 calling_convention);
4164 }
4165
VisitUnresolvedInstanceFieldSet(HUnresolvedInstanceFieldSet * instruction)4166 void LocationsBuilderARM::VisitUnresolvedInstanceFieldSet(
4167 HUnresolvedInstanceFieldSet* instruction) {
4168 FieldAccessCallingConventionARM calling_convention;
4169 codegen_->CreateUnresolvedFieldLocationSummary(
4170 instruction, instruction->GetFieldType(), calling_convention);
4171 }
4172
VisitUnresolvedInstanceFieldSet(HUnresolvedInstanceFieldSet * instruction)4173 void InstructionCodeGeneratorARM::VisitUnresolvedInstanceFieldSet(
4174 HUnresolvedInstanceFieldSet* instruction) {
4175 FieldAccessCallingConventionARM calling_convention;
4176 codegen_->GenerateUnresolvedFieldAccess(instruction,
4177 instruction->GetFieldType(),
4178 instruction->GetFieldIndex(),
4179 instruction->GetDexPc(),
4180 calling_convention);
4181 }
4182
VisitUnresolvedStaticFieldGet(HUnresolvedStaticFieldGet * instruction)4183 void LocationsBuilderARM::VisitUnresolvedStaticFieldGet(
4184 HUnresolvedStaticFieldGet* instruction) {
4185 FieldAccessCallingConventionARM calling_convention;
4186 codegen_->CreateUnresolvedFieldLocationSummary(
4187 instruction, instruction->GetFieldType(), calling_convention);
4188 }
4189
VisitUnresolvedStaticFieldGet(HUnresolvedStaticFieldGet * instruction)4190 void InstructionCodeGeneratorARM::VisitUnresolvedStaticFieldGet(
4191 HUnresolvedStaticFieldGet* instruction) {
4192 FieldAccessCallingConventionARM calling_convention;
4193 codegen_->GenerateUnresolvedFieldAccess(instruction,
4194 instruction->GetFieldType(),
4195 instruction->GetFieldIndex(),
4196 instruction->GetDexPc(),
4197 calling_convention);
4198 }
4199
VisitUnresolvedStaticFieldSet(HUnresolvedStaticFieldSet * instruction)4200 void LocationsBuilderARM::VisitUnresolvedStaticFieldSet(
4201 HUnresolvedStaticFieldSet* instruction) {
4202 FieldAccessCallingConventionARM calling_convention;
4203 codegen_->CreateUnresolvedFieldLocationSummary(
4204 instruction, instruction->GetFieldType(), calling_convention);
4205 }
4206
VisitUnresolvedStaticFieldSet(HUnresolvedStaticFieldSet * instruction)4207 void InstructionCodeGeneratorARM::VisitUnresolvedStaticFieldSet(
4208 HUnresolvedStaticFieldSet* instruction) {
4209 FieldAccessCallingConventionARM calling_convention;
4210 codegen_->GenerateUnresolvedFieldAccess(instruction,
4211 instruction->GetFieldType(),
4212 instruction->GetFieldIndex(),
4213 instruction->GetDexPc(),
4214 calling_convention);
4215 }
4216
VisitNullCheck(HNullCheck * instruction)4217 void LocationsBuilderARM::VisitNullCheck(HNullCheck* instruction) {
4218 LocationSummary::CallKind call_kind = instruction->CanThrowIntoCatchBlock()
4219 ? LocationSummary::kCallOnSlowPath
4220 : LocationSummary::kNoCall;
4221 LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction, call_kind);
4222 locations->SetInAt(0, Location::RequiresRegister());
4223 if (instruction->HasUses()) {
4224 locations->SetOut(Location::SameAsFirstInput());
4225 }
4226 }
4227
GenerateImplicitNullCheck(HNullCheck * instruction)4228 void CodeGeneratorARM::GenerateImplicitNullCheck(HNullCheck* instruction) {
4229 if (CanMoveNullCheckToUser(instruction)) {
4230 return;
4231 }
4232 Location obj = instruction->GetLocations()->InAt(0);
4233
4234 __ LoadFromOffset(kLoadWord, IP, obj.AsRegister<Register>(), 0);
4235 RecordPcInfo(instruction, instruction->GetDexPc());
4236 }
4237
GenerateExplicitNullCheck(HNullCheck * instruction)4238 void CodeGeneratorARM::GenerateExplicitNullCheck(HNullCheck* instruction) {
4239 SlowPathCode* slow_path = new (GetGraph()->GetArena()) NullCheckSlowPathARM(instruction);
4240 AddSlowPath(slow_path);
4241
4242 LocationSummary* locations = instruction->GetLocations();
4243 Location obj = locations->InAt(0);
4244
4245 __ CompareAndBranchIfZero(obj.AsRegister<Register>(), slow_path->GetEntryLabel());
4246 }
4247
VisitNullCheck(HNullCheck * instruction)4248 void InstructionCodeGeneratorARM::VisitNullCheck(HNullCheck* instruction) {
4249 codegen_->GenerateNullCheck(instruction);
4250 }
4251
VisitArrayGet(HArrayGet * instruction)4252 void LocationsBuilderARM::VisitArrayGet(HArrayGet* instruction) {
4253 bool object_array_get_with_read_barrier =
4254 kEmitCompilerReadBarrier && (instruction->GetType() == Primitive::kPrimNot);
4255 LocationSummary* locations =
4256 new (GetGraph()->GetArena()) LocationSummary(instruction,
4257 object_array_get_with_read_barrier ?
4258 LocationSummary::kCallOnSlowPath :
4259 LocationSummary::kNoCall);
4260 locations->SetInAt(0, Location::RequiresRegister());
4261 locations->SetInAt(1, Location::RegisterOrConstant(instruction->InputAt(1)));
4262 if (Primitive::IsFloatingPointType(instruction->GetType())) {
4263 locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap);
4264 } else {
4265 // The output overlaps in the case of an object array get with
4266 // read barriers enabled: we do not want the move to overwrite the
4267 // array's location, as we need it to emit the read barrier.
4268 locations->SetOut(
4269 Location::RequiresRegister(),
4270 object_array_get_with_read_barrier ? Location::kOutputOverlap : Location::kNoOutputOverlap);
4271 }
4272 // We need a temporary register for the read barrier marking slow
4273 // path in CodeGeneratorARM::GenerateArrayLoadWithBakerReadBarrier.
4274 if (object_array_get_with_read_barrier && kUseBakerReadBarrier) {
4275 locations->AddTemp(Location::RequiresRegister());
4276 }
4277 }
4278
VisitArrayGet(HArrayGet * instruction)4279 void InstructionCodeGeneratorARM::VisitArrayGet(HArrayGet* instruction) {
4280 LocationSummary* locations = instruction->GetLocations();
4281 Location obj_loc = locations->InAt(0);
4282 Register obj = obj_loc.AsRegister<Register>();
4283 Location index = locations->InAt(1);
4284 Location out_loc = locations->Out();
4285
4286 Primitive::Type type = instruction->GetType();
4287 switch (type) {
4288 case Primitive::kPrimBoolean: {
4289 uint32_t data_offset = mirror::Array::DataOffset(sizeof(uint8_t)).Uint32Value();
4290 Register out = out_loc.AsRegister<Register>();
4291 if (index.IsConstant()) {
4292 size_t offset =
4293 (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_1) + data_offset;
4294 __ LoadFromOffset(kLoadUnsignedByte, out, obj, offset);
4295 } else {
4296 __ add(IP, obj, ShifterOperand(index.AsRegister<Register>()));
4297 __ LoadFromOffset(kLoadUnsignedByte, out, IP, data_offset);
4298 }
4299 break;
4300 }
4301
4302 case Primitive::kPrimByte: {
4303 uint32_t data_offset = mirror::Array::DataOffset(sizeof(int8_t)).Uint32Value();
4304 Register out = out_loc.AsRegister<Register>();
4305 if (index.IsConstant()) {
4306 size_t offset =
4307 (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_1) + data_offset;
4308 __ LoadFromOffset(kLoadSignedByte, out, obj, offset);
4309 } else {
4310 __ add(IP, obj, ShifterOperand(index.AsRegister<Register>()));
4311 __ LoadFromOffset(kLoadSignedByte, out, IP, data_offset);
4312 }
4313 break;
4314 }
4315
4316 case Primitive::kPrimShort: {
4317 uint32_t data_offset = mirror::Array::DataOffset(sizeof(int16_t)).Uint32Value();
4318 Register out = out_loc.AsRegister<Register>();
4319 if (index.IsConstant()) {
4320 size_t offset =
4321 (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_2) + data_offset;
4322 __ LoadFromOffset(kLoadSignedHalfword, out, obj, offset);
4323 } else {
4324 __ add(IP, obj, ShifterOperand(index.AsRegister<Register>(), LSL, TIMES_2));
4325 __ LoadFromOffset(kLoadSignedHalfword, out, IP, data_offset);
4326 }
4327 break;
4328 }
4329
4330 case Primitive::kPrimChar: {
4331 uint32_t data_offset = mirror::Array::DataOffset(sizeof(uint16_t)).Uint32Value();
4332 Register out = out_loc.AsRegister<Register>();
4333 if (index.IsConstant()) {
4334 size_t offset =
4335 (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_2) + data_offset;
4336 __ LoadFromOffset(kLoadUnsignedHalfword, out, obj, offset);
4337 } else {
4338 __ add(IP, obj, ShifterOperand(index.AsRegister<Register>(), LSL, TIMES_2));
4339 __ LoadFromOffset(kLoadUnsignedHalfword, out, IP, data_offset);
4340 }
4341 break;
4342 }
4343
4344 case Primitive::kPrimInt: {
4345 uint32_t data_offset = mirror::Array::DataOffset(sizeof(int32_t)).Uint32Value();
4346 Register out = out_loc.AsRegister<Register>();
4347 if (index.IsConstant()) {
4348 size_t offset =
4349 (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4) + data_offset;
4350 __ LoadFromOffset(kLoadWord, out, obj, offset);
4351 } else {
4352 __ add(IP, obj, ShifterOperand(index.AsRegister<Register>(), LSL, TIMES_4));
4353 __ LoadFromOffset(kLoadWord, out, IP, data_offset);
4354 }
4355 break;
4356 }
4357
4358 case Primitive::kPrimNot: {
4359 static_assert(
4360 sizeof(mirror::HeapReference<mirror::Object>) == sizeof(int32_t),
4361 "art::mirror::HeapReference<art::mirror::Object> and int32_t have different sizes.");
4362 uint32_t data_offset = mirror::Array::DataOffset(sizeof(int32_t)).Uint32Value();
4363 // /* HeapReference<Object> */ out =
4364 // *(obj + data_offset + index * sizeof(HeapReference<Object>))
4365 if (kEmitCompilerReadBarrier && kUseBakerReadBarrier) {
4366 Location temp = locations->GetTemp(0);
4367 // Note that a potential implicit null check is handled in this
4368 // CodeGeneratorARM::GenerateArrayLoadWithBakerReadBarrier call.
4369 codegen_->GenerateArrayLoadWithBakerReadBarrier(
4370 instruction, out_loc, obj, data_offset, index, temp, /* needs_null_check */ true);
4371 } else {
4372 Register out = out_loc.AsRegister<Register>();
4373 if (index.IsConstant()) {
4374 size_t offset =
4375 (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4) + data_offset;
4376 __ LoadFromOffset(kLoadWord, out, obj, offset);
4377 codegen_->MaybeRecordImplicitNullCheck(instruction);
4378 // If read barriers are enabled, emit read barriers other than
4379 // Baker's using a slow path (and also unpoison the loaded
4380 // reference, if heap poisoning is enabled).
4381 codegen_->MaybeGenerateReadBarrierSlow(instruction, out_loc, out_loc, obj_loc, offset);
4382 } else {
4383 __ add(IP, obj, ShifterOperand(index.AsRegister<Register>(), LSL, TIMES_4));
4384 __ LoadFromOffset(kLoadWord, out, IP, data_offset);
4385 codegen_->MaybeRecordImplicitNullCheck(instruction);
4386 // If read barriers are enabled, emit read barriers other than
4387 // Baker's using a slow path (and also unpoison the loaded
4388 // reference, if heap poisoning is enabled).
4389 codegen_->MaybeGenerateReadBarrierSlow(
4390 instruction, out_loc, out_loc, obj_loc, data_offset, index);
4391 }
4392 }
4393 break;
4394 }
4395
4396 case Primitive::kPrimLong: {
4397 uint32_t data_offset = mirror::Array::DataOffset(sizeof(int64_t)).Uint32Value();
4398 if (index.IsConstant()) {
4399 size_t offset =
4400 (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_8) + data_offset;
4401 __ LoadFromOffset(kLoadWordPair, out_loc.AsRegisterPairLow<Register>(), obj, offset);
4402 } else {
4403 __ add(IP, obj, ShifterOperand(index.AsRegister<Register>(), LSL, TIMES_8));
4404 __ LoadFromOffset(kLoadWordPair, out_loc.AsRegisterPairLow<Register>(), IP, data_offset);
4405 }
4406 break;
4407 }
4408
4409 case Primitive::kPrimFloat: {
4410 uint32_t data_offset = mirror::Array::DataOffset(sizeof(float)).Uint32Value();
4411 SRegister out = out_loc.AsFpuRegister<SRegister>();
4412 if (index.IsConstant()) {
4413 size_t offset = (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4) + data_offset;
4414 __ LoadSFromOffset(out, obj, offset);
4415 } else {
4416 __ add(IP, obj, ShifterOperand(index.AsRegister<Register>(), LSL, TIMES_4));
4417 __ LoadSFromOffset(out, IP, data_offset);
4418 }
4419 break;
4420 }
4421
4422 case Primitive::kPrimDouble: {
4423 uint32_t data_offset = mirror::Array::DataOffset(sizeof(double)).Uint32Value();
4424 SRegister out = out_loc.AsFpuRegisterPairLow<SRegister>();
4425 if (index.IsConstant()) {
4426 size_t offset = (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_8) + data_offset;
4427 __ LoadDFromOffset(FromLowSToD(out), obj, offset);
4428 } else {
4429 __ add(IP, obj, ShifterOperand(index.AsRegister<Register>(), LSL, TIMES_8));
4430 __ LoadDFromOffset(FromLowSToD(out), IP, data_offset);
4431 }
4432 break;
4433 }
4434
4435 case Primitive::kPrimVoid:
4436 LOG(FATAL) << "Unreachable type " << type;
4437 UNREACHABLE();
4438 }
4439
4440 if (type == Primitive::kPrimNot) {
4441 // Potential implicit null checks, in the case of reference
4442 // arrays, are handled in the previous switch statement.
4443 } else {
4444 codegen_->MaybeRecordImplicitNullCheck(instruction);
4445 }
4446 }
4447
VisitArraySet(HArraySet * instruction)4448 void LocationsBuilderARM::VisitArraySet(HArraySet* instruction) {
4449 Primitive::Type value_type = instruction->GetComponentType();
4450
4451 bool needs_write_barrier =
4452 CodeGenerator::StoreNeedsWriteBarrier(value_type, instruction->GetValue());
4453 bool may_need_runtime_call_for_type_check = instruction->NeedsTypeCheck();
4454 bool object_array_set_with_read_barrier =
4455 kEmitCompilerReadBarrier && (value_type == Primitive::kPrimNot);
4456
4457 LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(
4458 instruction,
4459 (may_need_runtime_call_for_type_check || object_array_set_with_read_barrier) ?
4460 LocationSummary::kCallOnSlowPath :
4461 LocationSummary::kNoCall);
4462
4463 locations->SetInAt(0, Location::RequiresRegister());
4464 locations->SetInAt(1, Location::RegisterOrConstant(instruction->InputAt(1)));
4465 if (Primitive::IsFloatingPointType(value_type)) {
4466 locations->SetInAt(2, Location::RequiresFpuRegister());
4467 } else {
4468 locations->SetInAt(2, Location::RequiresRegister());
4469 }
4470 if (needs_write_barrier) {
4471 // Temporary registers for the write barrier.
4472 locations->AddTemp(Location::RequiresRegister()); // Possibly used for ref. poisoning too.
4473 locations->AddTemp(Location::RequiresRegister());
4474 }
4475 }
4476
VisitArraySet(HArraySet * instruction)4477 void InstructionCodeGeneratorARM::VisitArraySet(HArraySet* instruction) {
4478 LocationSummary* locations = instruction->GetLocations();
4479 Location array_loc = locations->InAt(0);
4480 Register array = array_loc.AsRegister<Register>();
4481 Location index = locations->InAt(1);
4482 Primitive::Type value_type = instruction->GetComponentType();
4483 bool may_need_runtime_call_for_type_check = instruction->NeedsTypeCheck();
4484 bool needs_write_barrier =
4485 CodeGenerator::StoreNeedsWriteBarrier(value_type, instruction->GetValue());
4486
4487 switch (value_type) {
4488 case Primitive::kPrimBoolean:
4489 case Primitive::kPrimByte: {
4490 uint32_t data_offset = mirror::Array::DataOffset(sizeof(uint8_t)).Uint32Value();
4491 Register value = locations->InAt(2).AsRegister<Register>();
4492 if (index.IsConstant()) {
4493 size_t offset =
4494 (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_1) + data_offset;
4495 __ StoreToOffset(kStoreByte, value, array, offset);
4496 } else {
4497 __ add(IP, array, ShifterOperand(index.AsRegister<Register>()));
4498 __ StoreToOffset(kStoreByte, value, IP, data_offset);
4499 }
4500 break;
4501 }
4502
4503 case Primitive::kPrimShort:
4504 case Primitive::kPrimChar: {
4505 uint32_t data_offset = mirror::Array::DataOffset(sizeof(uint16_t)).Uint32Value();
4506 Register value = locations->InAt(2).AsRegister<Register>();
4507 if (index.IsConstant()) {
4508 size_t offset =
4509 (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_2) + data_offset;
4510 __ StoreToOffset(kStoreHalfword, value, array, offset);
4511 } else {
4512 __ add(IP, array, ShifterOperand(index.AsRegister<Register>(), LSL, TIMES_2));
4513 __ StoreToOffset(kStoreHalfword, value, IP, data_offset);
4514 }
4515 break;
4516 }
4517
4518 case Primitive::kPrimNot: {
4519 uint32_t data_offset = mirror::Array::DataOffset(sizeof(int32_t)).Uint32Value();
4520 Location value_loc = locations->InAt(2);
4521 Register value = value_loc.AsRegister<Register>();
4522 Register source = value;
4523
4524 if (instruction->InputAt(2)->IsNullConstant()) {
4525 // Just setting null.
4526 if (index.IsConstant()) {
4527 size_t offset =
4528 (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4) + data_offset;
4529 __ StoreToOffset(kStoreWord, source, array, offset);
4530 } else {
4531 DCHECK(index.IsRegister()) << index;
4532 __ add(IP, array, ShifterOperand(index.AsRegister<Register>(), LSL, TIMES_4));
4533 __ StoreToOffset(kStoreWord, source, IP, data_offset);
4534 }
4535 codegen_->MaybeRecordImplicitNullCheck(instruction);
4536 DCHECK(!needs_write_barrier);
4537 DCHECK(!may_need_runtime_call_for_type_check);
4538 break;
4539 }
4540
4541 DCHECK(needs_write_barrier);
4542 Register temp1 = locations->GetTemp(0).AsRegister<Register>();
4543 Register temp2 = locations->GetTemp(1).AsRegister<Register>();
4544 uint32_t class_offset = mirror::Object::ClassOffset().Int32Value();
4545 uint32_t super_offset = mirror::Class::SuperClassOffset().Int32Value();
4546 uint32_t component_offset = mirror::Class::ComponentTypeOffset().Int32Value();
4547 Label done;
4548 SlowPathCode* slow_path = nullptr;
4549
4550 if (may_need_runtime_call_for_type_check) {
4551 slow_path = new (GetGraph()->GetArena()) ArraySetSlowPathARM(instruction);
4552 codegen_->AddSlowPath(slow_path);
4553 if (instruction->GetValueCanBeNull()) {
4554 Label non_zero;
4555 __ CompareAndBranchIfNonZero(value, &non_zero);
4556 if (index.IsConstant()) {
4557 size_t offset =
4558 (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4) + data_offset;
4559 __ StoreToOffset(kStoreWord, value, array, offset);
4560 } else {
4561 DCHECK(index.IsRegister()) << index;
4562 __ add(IP, array, ShifterOperand(index.AsRegister<Register>(), LSL, TIMES_4));
4563 __ StoreToOffset(kStoreWord, value, IP, data_offset);
4564 }
4565 codegen_->MaybeRecordImplicitNullCheck(instruction);
4566 __ b(&done);
4567 __ Bind(&non_zero);
4568 }
4569
4570 if (kEmitCompilerReadBarrier) {
4571 // When read barriers are enabled, the type checking
4572 // instrumentation requires two read barriers:
4573 //
4574 // __ Mov(temp2, temp1);
4575 // // /* HeapReference<Class> */ temp1 = temp1->component_type_
4576 // __ LoadFromOffset(kLoadWord, temp1, temp1, component_offset);
4577 // codegen_->GenerateReadBarrierSlow(
4578 // instruction, temp1_loc, temp1_loc, temp2_loc, component_offset);
4579 //
4580 // // /* HeapReference<Class> */ temp2 = value->klass_
4581 // __ LoadFromOffset(kLoadWord, temp2, value, class_offset);
4582 // codegen_->GenerateReadBarrierSlow(
4583 // instruction, temp2_loc, temp2_loc, value_loc, class_offset, temp1_loc);
4584 //
4585 // __ cmp(temp1, ShifterOperand(temp2));
4586 //
4587 // However, the second read barrier may trash `temp`, as it
4588 // is a temporary register, and as such would not be saved
4589 // along with live registers before calling the runtime (nor
4590 // restored afterwards). So in this case, we bail out and
4591 // delegate the work to the array set slow path.
4592 //
4593 // TODO: Extend the register allocator to support a new
4594 // "(locally) live temp" location so as to avoid always
4595 // going into the slow path when read barriers are enabled.
4596 __ b(slow_path->GetEntryLabel());
4597 } else {
4598 // /* HeapReference<Class> */ temp1 = array->klass_
4599 __ LoadFromOffset(kLoadWord, temp1, array, class_offset);
4600 codegen_->MaybeRecordImplicitNullCheck(instruction);
4601 __ MaybeUnpoisonHeapReference(temp1);
4602
4603 // /* HeapReference<Class> */ temp1 = temp1->component_type_
4604 __ LoadFromOffset(kLoadWord, temp1, temp1, component_offset);
4605 // /* HeapReference<Class> */ temp2 = value->klass_
4606 __ LoadFromOffset(kLoadWord, temp2, value, class_offset);
4607 // If heap poisoning is enabled, no need to unpoison `temp1`
4608 // nor `temp2`, as we are comparing two poisoned references.
4609 __ cmp(temp1, ShifterOperand(temp2));
4610
4611 if (instruction->StaticTypeOfArrayIsObjectArray()) {
4612 Label do_put;
4613 __ b(&do_put, EQ);
4614 // If heap poisoning is enabled, the `temp1` reference has
4615 // not been unpoisoned yet; unpoison it now.
4616 __ MaybeUnpoisonHeapReference(temp1);
4617
4618 // /* HeapReference<Class> */ temp1 = temp1->super_class_
4619 __ LoadFromOffset(kLoadWord, temp1, temp1, super_offset);
4620 // If heap poisoning is enabled, no need to unpoison
4621 // `temp1`, as we are comparing against null below.
4622 __ CompareAndBranchIfNonZero(temp1, slow_path->GetEntryLabel());
4623 __ Bind(&do_put);
4624 } else {
4625 __ b(slow_path->GetEntryLabel(), NE);
4626 }
4627 }
4628 }
4629
4630 if (kPoisonHeapReferences) {
4631 // Note that in the case where `value` is a null reference,
4632 // we do not enter this block, as a null reference does not
4633 // need poisoning.
4634 DCHECK_EQ(value_type, Primitive::kPrimNot);
4635 __ Mov(temp1, value);
4636 __ PoisonHeapReference(temp1);
4637 source = temp1;
4638 }
4639
4640 if (index.IsConstant()) {
4641 size_t offset =
4642 (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4) + data_offset;
4643 __ StoreToOffset(kStoreWord, source, array, offset);
4644 } else {
4645 DCHECK(index.IsRegister()) << index;
4646 __ add(IP, array, ShifterOperand(index.AsRegister<Register>(), LSL, TIMES_4));
4647 __ StoreToOffset(kStoreWord, source, IP, data_offset);
4648 }
4649
4650 if (!may_need_runtime_call_for_type_check) {
4651 codegen_->MaybeRecordImplicitNullCheck(instruction);
4652 }
4653
4654 codegen_->MarkGCCard(temp1, temp2, array, value, instruction->GetValueCanBeNull());
4655
4656 if (done.IsLinked()) {
4657 __ Bind(&done);
4658 }
4659
4660 if (slow_path != nullptr) {
4661 __ Bind(slow_path->GetExitLabel());
4662 }
4663
4664 break;
4665 }
4666
4667 case Primitive::kPrimInt: {
4668 uint32_t data_offset = mirror::Array::DataOffset(sizeof(int32_t)).Uint32Value();
4669 Register value = locations->InAt(2).AsRegister<Register>();
4670 if (index.IsConstant()) {
4671 size_t offset =
4672 (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4) + data_offset;
4673 __ StoreToOffset(kStoreWord, value, array, offset);
4674 } else {
4675 DCHECK(index.IsRegister()) << index;
4676 __ add(IP, array, ShifterOperand(index.AsRegister<Register>(), LSL, TIMES_4));
4677 __ StoreToOffset(kStoreWord, value, IP, data_offset);
4678 }
4679 break;
4680 }
4681
4682 case Primitive::kPrimLong: {
4683 uint32_t data_offset = mirror::Array::DataOffset(sizeof(int64_t)).Uint32Value();
4684 Location value = locations->InAt(2);
4685 if (index.IsConstant()) {
4686 size_t offset =
4687 (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_8) + data_offset;
4688 __ StoreToOffset(kStoreWordPair, value.AsRegisterPairLow<Register>(), array, offset);
4689 } else {
4690 __ add(IP, array, ShifterOperand(index.AsRegister<Register>(), LSL, TIMES_8));
4691 __ StoreToOffset(kStoreWordPair, value.AsRegisterPairLow<Register>(), IP, data_offset);
4692 }
4693 break;
4694 }
4695
4696 case Primitive::kPrimFloat: {
4697 uint32_t data_offset = mirror::Array::DataOffset(sizeof(float)).Uint32Value();
4698 Location value = locations->InAt(2);
4699 DCHECK(value.IsFpuRegister());
4700 if (index.IsConstant()) {
4701 size_t offset = (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4) + data_offset;
4702 __ StoreSToOffset(value.AsFpuRegister<SRegister>(), array, offset);
4703 } else {
4704 __ add(IP, array, ShifterOperand(index.AsRegister<Register>(), LSL, TIMES_4));
4705 __ StoreSToOffset(value.AsFpuRegister<SRegister>(), IP, data_offset);
4706 }
4707 break;
4708 }
4709
4710 case Primitive::kPrimDouble: {
4711 uint32_t data_offset = mirror::Array::DataOffset(sizeof(double)).Uint32Value();
4712 Location value = locations->InAt(2);
4713 DCHECK(value.IsFpuRegisterPair());
4714 if (index.IsConstant()) {
4715 size_t offset = (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_8) + data_offset;
4716 __ StoreDToOffset(FromLowSToD(value.AsFpuRegisterPairLow<SRegister>()), array, offset);
4717 } else {
4718 __ add(IP, array, ShifterOperand(index.AsRegister<Register>(), LSL, TIMES_8));
4719 __ StoreDToOffset(FromLowSToD(value.AsFpuRegisterPairLow<SRegister>()), IP, data_offset);
4720 }
4721
4722 break;
4723 }
4724
4725 case Primitive::kPrimVoid:
4726 LOG(FATAL) << "Unreachable type " << value_type;
4727 UNREACHABLE();
4728 }
4729
4730 // Objects are handled in the switch.
4731 if (value_type != Primitive::kPrimNot) {
4732 codegen_->MaybeRecordImplicitNullCheck(instruction);
4733 }
4734 }
4735
VisitArrayLength(HArrayLength * instruction)4736 void LocationsBuilderARM::VisitArrayLength(HArrayLength* instruction) {
4737 LocationSummary* locations =
4738 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
4739 locations->SetInAt(0, Location::RequiresRegister());
4740 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
4741 }
4742
VisitArrayLength(HArrayLength * instruction)4743 void InstructionCodeGeneratorARM::VisitArrayLength(HArrayLength* instruction) {
4744 LocationSummary* locations = instruction->GetLocations();
4745 uint32_t offset = mirror::Array::LengthOffset().Uint32Value();
4746 Register obj = locations->InAt(0).AsRegister<Register>();
4747 Register out = locations->Out().AsRegister<Register>();
4748 __ LoadFromOffset(kLoadWord, out, obj, offset);
4749 codegen_->MaybeRecordImplicitNullCheck(instruction);
4750 }
4751
VisitBoundsCheck(HBoundsCheck * instruction)4752 void LocationsBuilderARM::VisitBoundsCheck(HBoundsCheck* instruction) {
4753 LocationSummary::CallKind call_kind = instruction->CanThrowIntoCatchBlock()
4754 ? LocationSummary::kCallOnSlowPath
4755 : LocationSummary::kNoCall;
4756 LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction, call_kind);
4757 locations->SetInAt(0, Location::RequiresRegister());
4758 locations->SetInAt(1, Location::RequiresRegister());
4759 if (instruction->HasUses()) {
4760 locations->SetOut(Location::SameAsFirstInput());
4761 }
4762 }
4763
VisitBoundsCheck(HBoundsCheck * instruction)4764 void InstructionCodeGeneratorARM::VisitBoundsCheck(HBoundsCheck* instruction) {
4765 LocationSummary* locations = instruction->GetLocations();
4766 SlowPathCode* slow_path =
4767 new (GetGraph()->GetArena()) BoundsCheckSlowPathARM(instruction);
4768 codegen_->AddSlowPath(slow_path);
4769
4770 Register index = locations->InAt(0).AsRegister<Register>();
4771 Register length = locations->InAt(1).AsRegister<Register>();
4772
4773 __ cmp(index, ShifterOperand(length));
4774 __ b(slow_path->GetEntryLabel(), HS);
4775 }
4776
MarkGCCard(Register temp,Register card,Register object,Register value,bool can_be_null)4777 void CodeGeneratorARM::MarkGCCard(Register temp,
4778 Register card,
4779 Register object,
4780 Register value,
4781 bool can_be_null) {
4782 Label is_null;
4783 if (can_be_null) {
4784 __ CompareAndBranchIfZero(value, &is_null);
4785 }
4786 __ LoadFromOffset(kLoadWord, card, TR, Thread::CardTableOffset<kArmWordSize>().Int32Value());
4787 __ Lsr(temp, object, gc::accounting::CardTable::kCardShift);
4788 __ strb(card, Address(card, temp));
4789 if (can_be_null) {
4790 __ Bind(&is_null);
4791 }
4792 }
4793
VisitParallelMove(HParallelMove * instruction ATTRIBUTE_UNUSED)4794 void LocationsBuilderARM::VisitParallelMove(HParallelMove* instruction ATTRIBUTE_UNUSED) {
4795 LOG(FATAL) << "Unreachable";
4796 }
4797
VisitParallelMove(HParallelMove * instruction)4798 void InstructionCodeGeneratorARM::VisitParallelMove(HParallelMove* instruction) {
4799 codegen_->GetMoveResolver()->EmitNativeCode(instruction);
4800 }
4801
VisitSuspendCheck(HSuspendCheck * instruction)4802 void LocationsBuilderARM::VisitSuspendCheck(HSuspendCheck* instruction) {
4803 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCallOnSlowPath);
4804 }
4805
VisitSuspendCheck(HSuspendCheck * instruction)4806 void InstructionCodeGeneratorARM::VisitSuspendCheck(HSuspendCheck* instruction) {
4807 HBasicBlock* block = instruction->GetBlock();
4808 if (block->GetLoopInformation() != nullptr) {
4809 DCHECK(block->GetLoopInformation()->GetSuspendCheck() == instruction);
4810 // The back edge will generate the suspend check.
4811 return;
4812 }
4813 if (block->IsEntryBlock() && instruction->GetNext()->IsGoto()) {
4814 // The goto will generate the suspend check.
4815 return;
4816 }
4817 GenerateSuspendCheck(instruction, nullptr);
4818 }
4819
GenerateSuspendCheck(HSuspendCheck * instruction,HBasicBlock * successor)4820 void InstructionCodeGeneratorARM::GenerateSuspendCheck(HSuspendCheck* instruction,
4821 HBasicBlock* successor) {
4822 SuspendCheckSlowPathARM* slow_path =
4823 down_cast<SuspendCheckSlowPathARM*>(instruction->GetSlowPath());
4824 if (slow_path == nullptr) {
4825 slow_path = new (GetGraph()->GetArena()) SuspendCheckSlowPathARM(instruction, successor);
4826 instruction->SetSlowPath(slow_path);
4827 codegen_->AddSlowPath(slow_path);
4828 if (successor != nullptr) {
4829 DCHECK(successor->IsLoopHeader());
4830 codegen_->ClearSpillSlotsFromLoopPhisInStackMap(instruction);
4831 }
4832 } else {
4833 DCHECK_EQ(slow_path->GetSuccessor(), successor);
4834 }
4835
4836 __ LoadFromOffset(
4837 kLoadUnsignedHalfword, IP, TR, Thread::ThreadFlagsOffset<kArmWordSize>().Int32Value());
4838 if (successor == nullptr) {
4839 __ CompareAndBranchIfNonZero(IP, slow_path->GetEntryLabel());
4840 __ Bind(slow_path->GetReturnLabel());
4841 } else {
4842 __ CompareAndBranchIfZero(IP, codegen_->GetLabelOf(successor));
4843 __ b(slow_path->GetEntryLabel());
4844 }
4845 }
4846
GetAssembler() const4847 ArmAssembler* ParallelMoveResolverARM::GetAssembler() const {
4848 return codegen_->GetAssembler();
4849 }
4850
EmitMove(size_t index)4851 void ParallelMoveResolverARM::EmitMove(size_t index) {
4852 MoveOperands* move = moves_[index];
4853 Location source = move->GetSource();
4854 Location destination = move->GetDestination();
4855
4856 if (source.IsRegister()) {
4857 if (destination.IsRegister()) {
4858 __ Mov(destination.AsRegister<Register>(), source.AsRegister<Register>());
4859 } else if (destination.IsFpuRegister()) {
4860 __ vmovsr(destination.AsFpuRegister<SRegister>(), source.AsRegister<Register>());
4861 } else {
4862 DCHECK(destination.IsStackSlot());
4863 __ StoreToOffset(kStoreWord, source.AsRegister<Register>(),
4864 SP, destination.GetStackIndex());
4865 }
4866 } else if (source.IsStackSlot()) {
4867 if (destination.IsRegister()) {
4868 __ LoadFromOffset(kLoadWord, destination.AsRegister<Register>(),
4869 SP, source.GetStackIndex());
4870 } else if (destination.IsFpuRegister()) {
4871 __ LoadSFromOffset(destination.AsFpuRegister<SRegister>(), SP, source.GetStackIndex());
4872 } else {
4873 DCHECK(destination.IsStackSlot());
4874 __ LoadFromOffset(kLoadWord, IP, SP, source.GetStackIndex());
4875 __ StoreToOffset(kStoreWord, IP, SP, destination.GetStackIndex());
4876 }
4877 } else if (source.IsFpuRegister()) {
4878 if (destination.IsRegister()) {
4879 __ vmovrs(destination.AsRegister<Register>(), source.AsFpuRegister<SRegister>());
4880 } else if (destination.IsFpuRegister()) {
4881 __ vmovs(destination.AsFpuRegister<SRegister>(), source.AsFpuRegister<SRegister>());
4882 } else {
4883 DCHECK(destination.IsStackSlot());
4884 __ StoreSToOffset(source.AsFpuRegister<SRegister>(), SP, destination.GetStackIndex());
4885 }
4886 } else if (source.IsDoubleStackSlot()) {
4887 if (destination.IsDoubleStackSlot()) {
4888 __ LoadDFromOffset(DTMP, SP, source.GetStackIndex());
4889 __ StoreDToOffset(DTMP, SP, destination.GetStackIndex());
4890 } else if (destination.IsRegisterPair()) {
4891 DCHECK(ExpectedPairLayout(destination));
4892 __ LoadFromOffset(
4893 kLoadWordPair, destination.AsRegisterPairLow<Register>(), SP, source.GetStackIndex());
4894 } else {
4895 DCHECK(destination.IsFpuRegisterPair()) << destination;
4896 __ LoadDFromOffset(FromLowSToD(destination.AsFpuRegisterPairLow<SRegister>()),
4897 SP,
4898 source.GetStackIndex());
4899 }
4900 } else if (source.IsRegisterPair()) {
4901 if (destination.IsRegisterPair()) {
4902 __ Mov(destination.AsRegisterPairLow<Register>(), source.AsRegisterPairLow<Register>());
4903 __ Mov(destination.AsRegisterPairHigh<Register>(), source.AsRegisterPairHigh<Register>());
4904 } else if (destination.IsFpuRegisterPair()) {
4905 __ vmovdrr(FromLowSToD(destination.AsFpuRegisterPairLow<SRegister>()),
4906 source.AsRegisterPairLow<Register>(),
4907 source.AsRegisterPairHigh<Register>());
4908 } else {
4909 DCHECK(destination.IsDoubleStackSlot()) << destination;
4910 DCHECK(ExpectedPairLayout(source));
4911 __ StoreToOffset(
4912 kStoreWordPair, source.AsRegisterPairLow<Register>(), SP, destination.GetStackIndex());
4913 }
4914 } else if (source.IsFpuRegisterPair()) {
4915 if (destination.IsRegisterPair()) {
4916 __ vmovrrd(destination.AsRegisterPairLow<Register>(),
4917 destination.AsRegisterPairHigh<Register>(),
4918 FromLowSToD(source.AsFpuRegisterPairLow<SRegister>()));
4919 } else if (destination.IsFpuRegisterPair()) {
4920 __ vmovd(FromLowSToD(destination.AsFpuRegisterPairLow<SRegister>()),
4921 FromLowSToD(source.AsFpuRegisterPairLow<SRegister>()));
4922 } else {
4923 DCHECK(destination.IsDoubleStackSlot()) << destination;
4924 __ StoreDToOffset(FromLowSToD(source.AsFpuRegisterPairLow<SRegister>()),
4925 SP,
4926 destination.GetStackIndex());
4927 }
4928 } else {
4929 DCHECK(source.IsConstant()) << source;
4930 HConstant* constant = source.GetConstant();
4931 if (constant->IsIntConstant() || constant->IsNullConstant()) {
4932 int32_t value = CodeGenerator::GetInt32ValueOf(constant);
4933 if (destination.IsRegister()) {
4934 __ LoadImmediate(destination.AsRegister<Register>(), value);
4935 } else {
4936 DCHECK(destination.IsStackSlot());
4937 __ LoadImmediate(IP, value);
4938 __ StoreToOffset(kStoreWord, IP, SP, destination.GetStackIndex());
4939 }
4940 } else if (constant->IsLongConstant()) {
4941 int64_t value = constant->AsLongConstant()->GetValue();
4942 if (destination.IsRegisterPair()) {
4943 __ LoadImmediate(destination.AsRegisterPairLow<Register>(), Low32Bits(value));
4944 __ LoadImmediate(destination.AsRegisterPairHigh<Register>(), High32Bits(value));
4945 } else {
4946 DCHECK(destination.IsDoubleStackSlot()) << destination;
4947 __ LoadImmediate(IP, Low32Bits(value));
4948 __ StoreToOffset(kStoreWord, IP, SP, destination.GetStackIndex());
4949 __ LoadImmediate(IP, High32Bits(value));
4950 __ StoreToOffset(kStoreWord, IP, SP, destination.GetHighStackIndex(kArmWordSize));
4951 }
4952 } else if (constant->IsDoubleConstant()) {
4953 double value = constant->AsDoubleConstant()->GetValue();
4954 if (destination.IsFpuRegisterPair()) {
4955 __ LoadDImmediate(FromLowSToD(destination.AsFpuRegisterPairLow<SRegister>()), value);
4956 } else {
4957 DCHECK(destination.IsDoubleStackSlot()) << destination;
4958 uint64_t int_value = bit_cast<uint64_t, double>(value);
4959 __ LoadImmediate(IP, Low32Bits(int_value));
4960 __ StoreToOffset(kStoreWord, IP, SP, destination.GetStackIndex());
4961 __ LoadImmediate(IP, High32Bits(int_value));
4962 __ StoreToOffset(kStoreWord, IP, SP, destination.GetHighStackIndex(kArmWordSize));
4963 }
4964 } else {
4965 DCHECK(constant->IsFloatConstant()) << constant->DebugName();
4966 float value = constant->AsFloatConstant()->GetValue();
4967 if (destination.IsFpuRegister()) {
4968 __ LoadSImmediate(destination.AsFpuRegister<SRegister>(), value);
4969 } else {
4970 DCHECK(destination.IsStackSlot());
4971 __ LoadImmediate(IP, bit_cast<int32_t, float>(value));
4972 __ StoreToOffset(kStoreWord, IP, SP, destination.GetStackIndex());
4973 }
4974 }
4975 }
4976 }
4977
Exchange(Register reg,int mem)4978 void ParallelMoveResolverARM::Exchange(Register reg, int mem) {
4979 __ Mov(IP, reg);
4980 __ LoadFromOffset(kLoadWord, reg, SP, mem);
4981 __ StoreToOffset(kStoreWord, IP, SP, mem);
4982 }
4983
Exchange(int mem1,int mem2)4984 void ParallelMoveResolverARM::Exchange(int mem1, int mem2) {
4985 ScratchRegisterScope ensure_scratch(this, IP, R0, codegen_->GetNumberOfCoreRegisters());
4986 int stack_offset = ensure_scratch.IsSpilled() ? kArmWordSize : 0;
4987 __ LoadFromOffset(kLoadWord, static_cast<Register>(ensure_scratch.GetRegister()),
4988 SP, mem1 + stack_offset);
4989 __ LoadFromOffset(kLoadWord, IP, SP, mem2 + stack_offset);
4990 __ StoreToOffset(kStoreWord, static_cast<Register>(ensure_scratch.GetRegister()),
4991 SP, mem2 + stack_offset);
4992 __ StoreToOffset(kStoreWord, IP, SP, mem1 + stack_offset);
4993 }
4994
EmitSwap(size_t index)4995 void ParallelMoveResolverARM::EmitSwap(size_t index) {
4996 MoveOperands* move = moves_[index];
4997 Location source = move->GetSource();
4998 Location destination = move->GetDestination();
4999
5000 if (source.IsRegister() && destination.IsRegister()) {
5001 DCHECK_NE(source.AsRegister<Register>(), IP);
5002 DCHECK_NE(destination.AsRegister<Register>(), IP);
5003 __ Mov(IP, source.AsRegister<Register>());
5004 __ Mov(source.AsRegister<Register>(), destination.AsRegister<Register>());
5005 __ Mov(destination.AsRegister<Register>(), IP);
5006 } else if (source.IsRegister() && destination.IsStackSlot()) {
5007 Exchange(source.AsRegister<Register>(), destination.GetStackIndex());
5008 } else if (source.IsStackSlot() && destination.IsRegister()) {
5009 Exchange(destination.AsRegister<Register>(), source.GetStackIndex());
5010 } else if (source.IsStackSlot() && destination.IsStackSlot()) {
5011 Exchange(source.GetStackIndex(), destination.GetStackIndex());
5012 } else if (source.IsFpuRegister() && destination.IsFpuRegister()) {
5013 __ vmovrs(IP, source.AsFpuRegister<SRegister>());
5014 __ vmovs(source.AsFpuRegister<SRegister>(), destination.AsFpuRegister<SRegister>());
5015 __ vmovsr(destination.AsFpuRegister<SRegister>(), IP);
5016 } else if (source.IsRegisterPair() && destination.IsRegisterPair()) {
5017 __ vmovdrr(DTMP, source.AsRegisterPairLow<Register>(), source.AsRegisterPairHigh<Register>());
5018 __ Mov(source.AsRegisterPairLow<Register>(), destination.AsRegisterPairLow<Register>());
5019 __ Mov(source.AsRegisterPairHigh<Register>(), destination.AsRegisterPairHigh<Register>());
5020 __ vmovrrd(destination.AsRegisterPairLow<Register>(),
5021 destination.AsRegisterPairHigh<Register>(),
5022 DTMP);
5023 } else if (source.IsRegisterPair() || destination.IsRegisterPair()) {
5024 Register low_reg = source.IsRegisterPair()
5025 ? source.AsRegisterPairLow<Register>()
5026 : destination.AsRegisterPairLow<Register>();
5027 int mem = source.IsRegisterPair()
5028 ? destination.GetStackIndex()
5029 : source.GetStackIndex();
5030 DCHECK(ExpectedPairLayout(source.IsRegisterPair() ? source : destination));
5031 __ vmovdrr(DTMP, low_reg, static_cast<Register>(low_reg + 1));
5032 __ LoadFromOffset(kLoadWordPair, low_reg, SP, mem);
5033 __ StoreDToOffset(DTMP, SP, mem);
5034 } else if (source.IsFpuRegisterPair() && destination.IsFpuRegisterPair()) {
5035 DRegister first = FromLowSToD(source.AsFpuRegisterPairLow<SRegister>());
5036 DRegister second = FromLowSToD(destination.AsFpuRegisterPairLow<SRegister>());
5037 __ vmovd(DTMP, first);
5038 __ vmovd(first, second);
5039 __ vmovd(second, DTMP);
5040 } else if (source.IsFpuRegisterPair() || destination.IsFpuRegisterPair()) {
5041 DRegister reg = source.IsFpuRegisterPair()
5042 ? FromLowSToD(source.AsFpuRegisterPairLow<SRegister>())
5043 : FromLowSToD(destination.AsFpuRegisterPairLow<SRegister>());
5044 int mem = source.IsFpuRegisterPair()
5045 ? destination.GetStackIndex()
5046 : source.GetStackIndex();
5047 __ vmovd(DTMP, reg);
5048 __ LoadDFromOffset(reg, SP, mem);
5049 __ StoreDToOffset(DTMP, SP, mem);
5050 } else if (source.IsFpuRegister() || destination.IsFpuRegister()) {
5051 SRegister reg = source.IsFpuRegister() ? source.AsFpuRegister<SRegister>()
5052 : destination.AsFpuRegister<SRegister>();
5053 int mem = source.IsFpuRegister()
5054 ? destination.GetStackIndex()
5055 : source.GetStackIndex();
5056
5057 __ vmovrs(IP, reg);
5058 __ LoadSFromOffset(reg, SP, mem);
5059 __ StoreToOffset(kStoreWord, IP, SP, mem);
5060 } else if (source.IsDoubleStackSlot() && destination.IsDoubleStackSlot()) {
5061 Exchange(source.GetStackIndex(), destination.GetStackIndex());
5062 Exchange(source.GetHighStackIndex(kArmWordSize), destination.GetHighStackIndex(kArmWordSize));
5063 } else {
5064 LOG(FATAL) << "Unimplemented" << source << " <-> " << destination;
5065 }
5066 }
5067
SpillScratch(int reg)5068 void ParallelMoveResolverARM::SpillScratch(int reg) {
5069 __ Push(static_cast<Register>(reg));
5070 }
5071
RestoreScratch(int reg)5072 void ParallelMoveResolverARM::RestoreScratch(int reg) {
5073 __ Pop(static_cast<Register>(reg));
5074 }
5075
VisitLoadClass(HLoadClass * cls)5076 void LocationsBuilderARM::VisitLoadClass(HLoadClass* cls) {
5077 InvokeRuntimeCallingConvention calling_convention;
5078 CodeGenerator::CreateLoadClassLocationSummary(
5079 cls,
5080 Location::RegisterLocation(calling_convention.GetRegisterAt(0)),
5081 Location::RegisterLocation(R0),
5082 /* code_generator_supports_read_barrier */ true);
5083 }
5084
VisitLoadClass(HLoadClass * cls)5085 void InstructionCodeGeneratorARM::VisitLoadClass(HLoadClass* cls) {
5086 LocationSummary* locations = cls->GetLocations();
5087 if (cls->NeedsAccessCheck()) {
5088 codegen_->MoveConstant(locations->GetTemp(0), cls->GetTypeIndex());
5089 codegen_->InvokeRuntime(QUICK_ENTRY_POINT(pInitializeTypeAndVerifyAccess),
5090 cls,
5091 cls->GetDexPc(),
5092 nullptr);
5093 CheckEntrypointTypes<kQuickInitializeTypeAndVerifyAccess, void*, uint32_t>();
5094 return;
5095 }
5096
5097 Location out_loc = locations->Out();
5098 Register out = out_loc.AsRegister<Register>();
5099 Register current_method = locations->InAt(0).AsRegister<Register>();
5100
5101 if (cls->IsReferrersClass()) {
5102 DCHECK(!cls->CanCallRuntime());
5103 DCHECK(!cls->MustGenerateClinitCheck());
5104 // /* GcRoot<mirror::Class> */ out = current_method->declaring_class_
5105 GenerateGcRootFieldLoad(
5106 cls, out_loc, current_method, ArtMethod::DeclaringClassOffset().Int32Value());
5107 } else {
5108 // /* GcRoot<mirror::Class>[] */ out =
5109 // current_method.ptr_sized_fields_->dex_cache_resolved_types_
5110 __ LoadFromOffset(kLoadWord,
5111 out,
5112 current_method,
5113 ArtMethod::DexCacheResolvedTypesOffset(kArmPointerSize).Int32Value());
5114 // /* GcRoot<mirror::Class> */ out = out[type_index]
5115 GenerateGcRootFieldLoad(cls, out_loc, out, CodeGenerator::GetCacheOffset(cls->GetTypeIndex()));
5116
5117 if (!cls->IsInDexCache() || cls->MustGenerateClinitCheck()) {
5118 DCHECK(cls->CanCallRuntime());
5119 SlowPathCode* slow_path = new (GetGraph()->GetArena()) LoadClassSlowPathARM(
5120 cls, cls, cls->GetDexPc(), cls->MustGenerateClinitCheck());
5121 codegen_->AddSlowPath(slow_path);
5122 if (!cls->IsInDexCache()) {
5123 __ CompareAndBranchIfZero(out, slow_path->GetEntryLabel());
5124 }
5125 if (cls->MustGenerateClinitCheck()) {
5126 GenerateClassInitializationCheck(slow_path, out);
5127 } else {
5128 __ Bind(slow_path->GetExitLabel());
5129 }
5130 }
5131 }
5132 }
5133
VisitClinitCheck(HClinitCheck * check)5134 void LocationsBuilderARM::VisitClinitCheck(HClinitCheck* check) {
5135 LocationSummary* locations =
5136 new (GetGraph()->GetArena()) LocationSummary(check, LocationSummary::kCallOnSlowPath);
5137 locations->SetInAt(0, Location::RequiresRegister());
5138 if (check->HasUses()) {
5139 locations->SetOut(Location::SameAsFirstInput());
5140 }
5141 }
5142
VisitClinitCheck(HClinitCheck * check)5143 void InstructionCodeGeneratorARM::VisitClinitCheck(HClinitCheck* check) {
5144 // We assume the class is not null.
5145 SlowPathCode* slow_path = new (GetGraph()->GetArena()) LoadClassSlowPathARM(
5146 check->GetLoadClass(), check, check->GetDexPc(), true);
5147 codegen_->AddSlowPath(slow_path);
5148 GenerateClassInitializationCheck(slow_path,
5149 check->GetLocations()->InAt(0).AsRegister<Register>());
5150 }
5151
GenerateClassInitializationCheck(SlowPathCode * slow_path,Register class_reg)5152 void InstructionCodeGeneratorARM::GenerateClassInitializationCheck(
5153 SlowPathCode* slow_path, Register class_reg) {
5154 __ LoadFromOffset(kLoadWord, IP, class_reg, mirror::Class::StatusOffset().Int32Value());
5155 __ cmp(IP, ShifterOperand(mirror::Class::kStatusInitialized));
5156 __ b(slow_path->GetEntryLabel(), LT);
5157 // Even if the initialized flag is set, we may be in a situation where caches are not synced
5158 // properly. Therefore, we do a memory fence.
5159 __ dmb(ISH);
5160 __ Bind(slow_path->GetExitLabel());
5161 }
5162
GetSupportedLoadStringKind(HLoadString::LoadKind desired_string_load_kind)5163 HLoadString::LoadKind CodeGeneratorARM::GetSupportedLoadStringKind(
5164 HLoadString::LoadKind desired_string_load_kind) {
5165 if (kEmitCompilerReadBarrier) {
5166 switch (desired_string_load_kind) {
5167 case HLoadString::LoadKind::kBootImageLinkTimeAddress:
5168 case HLoadString::LoadKind::kBootImageLinkTimePcRelative:
5169 case HLoadString::LoadKind::kBootImageAddress:
5170 // TODO: Implement for read barrier.
5171 return HLoadString::LoadKind::kDexCacheViaMethod;
5172 default:
5173 break;
5174 }
5175 }
5176 switch (desired_string_load_kind) {
5177 case HLoadString::LoadKind::kBootImageLinkTimeAddress:
5178 DCHECK(!GetCompilerOptions().GetCompilePic());
5179 break;
5180 case HLoadString::LoadKind::kBootImageLinkTimePcRelative:
5181 DCHECK(GetCompilerOptions().GetCompilePic());
5182 break;
5183 case HLoadString::LoadKind::kBootImageAddress:
5184 break;
5185 case HLoadString::LoadKind::kDexCacheAddress:
5186 DCHECK(Runtime::Current()->UseJitCompilation());
5187 break;
5188 case HLoadString::LoadKind::kDexCachePcRelative:
5189 DCHECK(!Runtime::Current()->UseJitCompilation());
5190 // We disable pc-relative load when there is an irreducible loop, as the optimization
5191 // is incompatible with it.
5192 // TODO: Create as many ArmDexCacheArraysBase instructions as needed for methods
5193 // with irreducible loops.
5194 if (GetGraph()->HasIrreducibleLoops()) {
5195 return HLoadString::LoadKind::kDexCacheViaMethod;
5196 }
5197 break;
5198 case HLoadString::LoadKind::kDexCacheViaMethod:
5199 break;
5200 }
5201 return desired_string_load_kind;
5202 }
5203
VisitLoadString(HLoadString * load)5204 void LocationsBuilderARM::VisitLoadString(HLoadString* load) {
5205 LocationSummary::CallKind call_kind = (load->NeedsEnvironment() || kEmitCompilerReadBarrier)
5206 ? LocationSummary::kCallOnSlowPath
5207 : LocationSummary::kNoCall;
5208 LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(load, call_kind);
5209 HLoadString::LoadKind load_kind = load->GetLoadKind();
5210 if (load_kind == HLoadString::LoadKind::kDexCacheViaMethod ||
5211 load_kind == HLoadString::LoadKind::kDexCachePcRelative) {
5212 locations->SetInAt(0, Location::RequiresRegister());
5213 }
5214 locations->SetOut(Location::RequiresRegister());
5215 }
5216
VisitLoadString(HLoadString * load)5217 void InstructionCodeGeneratorARM::VisitLoadString(HLoadString* load) {
5218 LocationSummary* locations = load->GetLocations();
5219 Location out_loc = locations->Out();
5220 Register out = out_loc.AsRegister<Register>();
5221
5222 switch (load->GetLoadKind()) {
5223 case HLoadString::LoadKind::kBootImageLinkTimeAddress: {
5224 DCHECK(!kEmitCompilerReadBarrier);
5225 __ LoadLiteral(out, codegen_->DeduplicateBootImageStringLiteral(load->GetDexFile(),
5226 load->GetStringIndex()));
5227 return; // No dex cache slow path.
5228 }
5229 case HLoadString::LoadKind::kBootImageLinkTimePcRelative: {
5230 DCHECK(!kEmitCompilerReadBarrier);
5231 CodeGeneratorARM::PcRelativePatchInfo* labels =
5232 codegen_->NewPcRelativeStringPatch(load->GetDexFile(), load->GetStringIndex());
5233 __ BindTrackedLabel(&labels->movw_label);
5234 __ movw(out, /* placeholder */ 0u);
5235 __ BindTrackedLabel(&labels->movt_label);
5236 __ movt(out, /* placeholder */ 0u);
5237 __ BindTrackedLabel(&labels->add_pc_label);
5238 __ add(out, out, ShifterOperand(PC));
5239 return; // No dex cache slow path.
5240 }
5241 case HLoadString::LoadKind::kBootImageAddress: {
5242 DCHECK(!kEmitCompilerReadBarrier);
5243 DCHECK_NE(load->GetAddress(), 0u);
5244 uint32_t address = dchecked_integral_cast<uint32_t>(load->GetAddress());
5245 __ LoadLiteral(out, codegen_->DeduplicateBootImageAddressLiteral(address));
5246 return; // No dex cache slow path.
5247 }
5248 case HLoadString::LoadKind::kDexCacheAddress: {
5249 DCHECK_NE(load->GetAddress(), 0u);
5250 uint32_t address = dchecked_integral_cast<uint32_t>(load->GetAddress());
5251 // 16-bit LDR immediate has a 5-bit offset multiplied by the size and that gives
5252 // a 128B range. To try and reduce the number of literals if we load multiple strings,
5253 // simply split the dex cache address to a 128B aligned base loaded from a literal
5254 // and the remaining offset embedded in the load.
5255 static_assert(sizeof(GcRoot<mirror::String>) == 4u, "Expected GC root to be 4 bytes.");
5256 DCHECK_ALIGNED(load->GetAddress(), 4u);
5257 constexpr size_t offset_bits = /* encoded bits */ 5 + /* scale */ 2;
5258 uint32_t base_address = address & ~MaxInt<uint32_t>(offset_bits);
5259 uint32_t offset = address & MaxInt<uint32_t>(offset_bits);
5260 __ LoadLiteral(out, codegen_->DeduplicateDexCacheAddressLiteral(base_address));
5261 GenerateGcRootFieldLoad(load, out_loc, out, offset);
5262 break;
5263 }
5264 case HLoadString::LoadKind::kDexCachePcRelative: {
5265 Register base_reg = locations->InAt(0).AsRegister<Register>();
5266 HArmDexCacheArraysBase* base = load->InputAt(0)->AsArmDexCacheArraysBase();
5267 int32_t offset = load->GetDexCacheElementOffset() - base->GetElementOffset();
5268 GenerateGcRootFieldLoad(load, out_loc, base_reg, offset);
5269 break;
5270 }
5271 case HLoadString::LoadKind::kDexCacheViaMethod: {
5272 Register current_method = locations->InAt(0).AsRegister<Register>();
5273
5274 // /* GcRoot<mirror::Class> */ out = current_method->declaring_class_
5275 GenerateGcRootFieldLoad(
5276 load, out_loc, current_method, ArtMethod::DeclaringClassOffset().Int32Value());
5277 // /* GcRoot<mirror::String>[] */ out = out->dex_cache_strings_
5278 __ LoadFromOffset(kLoadWord, out, out, mirror::Class::DexCacheStringsOffset().Int32Value());
5279 // /* GcRoot<mirror::String> */ out = out[string_index]
5280 GenerateGcRootFieldLoad(
5281 load, out_loc, out, CodeGenerator::GetCacheOffset(load->GetStringIndex()));
5282 break;
5283 }
5284 default:
5285 LOG(FATAL) << "Unexpected load kind: " << load->GetLoadKind();
5286 UNREACHABLE();
5287 }
5288
5289 if (!load->IsInDexCache()) {
5290 SlowPathCode* slow_path = new (GetGraph()->GetArena()) LoadStringSlowPathARM(load);
5291 codegen_->AddSlowPath(slow_path);
5292 __ CompareAndBranchIfZero(out, slow_path->GetEntryLabel());
5293 __ Bind(slow_path->GetExitLabel());
5294 }
5295 }
5296
GetExceptionTlsOffset()5297 static int32_t GetExceptionTlsOffset() {
5298 return Thread::ExceptionOffset<kArmWordSize>().Int32Value();
5299 }
5300
VisitLoadException(HLoadException * load)5301 void LocationsBuilderARM::VisitLoadException(HLoadException* load) {
5302 LocationSummary* locations =
5303 new (GetGraph()->GetArena()) LocationSummary(load, LocationSummary::kNoCall);
5304 locations->SetOut(Location::RequiresRegister());
5305 }
5306
VisitLoadException(HLoadException * load)5307 void InstructionCodeGeneratorARM::VisitLoadException(HLoadException* load) {
5308 Register out = load->GetLocations()->Out().AsRegister<Register>();
5309 __ LoadFromOffset(kLoadWord, out, TR, GetExceptionTlsOffset());
5310 }
5311
VisitClearException(HClearException * clear)5312 void LocationsBuilderARM::VisitClearException(HClearException* clear) {
5313 new (GetGraph()->GetArena()) LocationSummary(clear, LocationSummary::kNoCall);
5314 }
5315
VisitClearException(HClearException * clear ATTRIBUTE_UNUSED)5316 void InstructionCodeGeneratorARM::VisitClearException(HClearException* clear ATTRIBUTE_UNUSED) {
5317 __ LoadImmediate(IP, 0);
5318 __ StoreToOffset(kStoreWord, IP, TR, GetExceptionTlsOffset());
5319 }
5320
VisitThrow(HThrow * instruction)5321 void LocationsBuilderARM::VisitThrow(HThrow* instruction) {
5322 LocationSummary* locations =
5323 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCall);
5324 InvokeRuntimeCallingConvention calling_convention;
5325 locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(0)));
5326 }
5327
VisitThrow(HThrow * instruction)5328 void InstructionCodeGeneratorARM::VisitThrow(HThrow* instruction) {
5329 codegen_->InvokeRuntime(
5330 QUICK_ENTRY_POINT(pDeliverException), instruction, instruction->GetDexPc(), nullptr);
5331 CheckEntrypointTypes<kQuickDeliverException, void, mirror::Object*>();
5332 }
5333
TypeCheckNeedsATemporary(TypeCheckKind type_check_kind)5334 static bool TypeCheckNeedsATemporary(TypeCheckKind type_check_kind) {
5335 return kEmitCompilerReadBarrier &&
5336 (kUseBakerReadBarrier ||
5337 type_check_kind == TypeCheckKind::kAbstractClassCheck ||
5338 type_check_kind == TypeCheckKind::kClassHierarchyCheck ||
5339 type_check_kind == TypeCheckKind::kArrayObjectCheck);
5340 }
5341
VisitInstanceOf(HInstanceOf * instruction)5342 void LocationsBuilderARM::VisitInstanceOf(HInstanceOf* instruction) {
5343 LocationSummary::CallKind call_kind = LocationSummary::kNoCall;
5344 TypeCheckKind type_check_kind = instruction->GetTypeCheckKind();
5345 switch (type_check_kind) {
5346 case TypeCheckKind::kExactCheck:
5347 case TypeCheckKind::kAbstractClassCheck:
5348 case TypeCheckKind::kClassHierarchyCheck:
5349 case TypeCheckKind::kArrayObjectCheck:
5350 call_kind =
5351 kEmitCompilerReadBarrier ? LocationSummary::kCallOnSlowPath : LocationSummary::kNoCall;
5352 break;
5353 case TypeCheckKind::kArrayCheck:
5354 case TypeCheckKind::kUnresolvedCheck:
5355 case TypeCheckKind::kInterfaceCheck:
5356 call_kind = LocationSummary::kCallOnSlowPath;
5357 break;
5358 }
5359
5360 LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction, call_kind);
5361 locations->SetInAt(0, Location::RequiresRegister());
5362 locations->SetInAt(1, Location::RequiresRegister());
5363 // The "out" register is used as a temporary, so it overlaps with the inputs.
5364 // Note that TypeCheckSlowPathARM uses this register too.
5365 locations->SetOut(Location::RequiresRegister(), Location::kOutputOverlap);
5366 // When read barriers are enabled, we need a temporary register for
5367 // some cases.
5368 if (TypeCheckNeedsATemporary(type_check_kind)) {
5369 locations->AddTemp(Location::RequiresRegister());
5370 }
5371 }
5372
VisitInstanceOf(HInstanceOf * instruction)5373 void InstructionCodeGeneratorARM::VisitInstanceOf(HInstanceOf* instruction) {
5374 TypeCheckKind type_check_kind = instruction->GetTypeCheckKind();
5375 LocationSummary* locations = instruction->GetLocations();
5376 Location obj_loc = locations->InAt(0);
5377 Register obj = obj_loc.AsRegister<Register>();
5378 Register cls = locations->InAt(1).AsRegister<Register>();
5379 Location out_loc = locations->Out();
5380 Register out = out_loc.AsRegister<Register>();
5381 Location maybe_temp_loc = TypeCheckNeedsATemporary(type_check_kind) ?
5382 locations->GetTemp(0) :
5383 Location::NoLocation();
5384 uint32_t class_offset = mirror::Object::ClassOffset().Int32Value();
5385 uint32_t super_offset = mirror::Class::SuperClassOffset().Int32Value();
5386 uint32_t component_offset = mirror::Class::ComponentTypeOffset().Int32Value();
5387 uint32_t primitive_offset = mirror::Class::PrimitiveTypeOffset().Int32Value();
5388 Label done, zero;
5389 SlowPathCode* slow_path = nullptr;
5390
5391 // Return 0 if `obj` is null.
5392 // avoid null check if we know obj is not null.
5393 if (instruction->MustDoNullCheck()) {
5394 __ CompareAndBranchIfZero(obj, &zero);
5395 }
5396
5397 // /* HeapReference<Class> */ out = obj->klass_
5398 GenerateReferenceLoadTwoRegisters(instruction, out_loc, obj_loc, class_offset, maybe_temp_loc);
5399
5400 switch (type_check_kind) {
5401 case TypeCheckKind::kExactCheck: {
5402 __ cmp(out, ShifterOperand(cls));
5403 // Classes must be equal for the instanceof to succeed.
5404 __ b(&zero, NE);
5405 __ LoadImmediate(out, 1);
5406 __ b(&done);
5407 break;
5408 }
5409
5410 case TypeCheckKind::kAbstractClassCheck: {
5411 // If the class is abstract, we eagerly fetch the super class of the
5412 // object to avoid doing a comparison we know will fail.
5413 Label loop;
5414 __ Bind(&loop);
5415 // /* HeapReference<Class> */ out = out->super_class_
5416 GenerateReferenceLoadOneRegister(instruction, out_loc, super_offset, maybe_temp_loc);
5417 // If `out` is null, we use it for the result, and jump to `done`.
5418 __ CompareAndBranchIfZero(out, &done);
5419 __ cmp(out, ShifterOperand(cls));
5420 __ b(&loop, NE);
5421 __ LoadImmediate(out, 1);
5422 if (zero.IsLinked()) {
5423 __ b(&done);
5424 }
5425 break;
5426 }
5427
5428 case TypeCheckKind::kClassHierarchyCheck: {
5429 // Walk over the class hierarchy to find a match.
5430 Label loop, success;
5431 __ Bind(&loop);
5432 __ cmp(out, ShifterOperand(cls));
5433 __ b(&success, EQ);
5434 // /* HeapReference<Class> */ out = out->super_class_
5435 GenerateReferenceLoadOneRegister(instruction, out_loc, super_offset, maybe_temp_loc);
5436 __ CompareAndBranchIfNonZero(out, &loop);
5437 // If `out` is null, we use it for the result, and jump to `done`.
5438 __ b(&done);
5439 __ Bind(&success);
5440 __ LoadImmediate(out, 1);
5441 if (zero.IsLinked()) {
5442 __ b(&done);
5443 }
5444 break;
5445 }
5446
5447 case TypeCheckKind::kArrayObjectCheck: {
5448 // Do an exact check.
5449 Label exact_check;
5450 __ cmp(out, ShifterOperand(cls));
5451 __ b(&exact_check, EQ);
5452 // Otherwise, we need to check that the object's class is a non-primitive array.
5453 // /* HeapReference<Class> */ out = out->component_type_
5454 GenerateReferenceLoadOneRegister(instruction, out_loc, component_offset, maybe_temp_loc);
5455 // If `out` is null, we use it for the result, and jump to `done`.
5456 __ CompareAndBranchIfZero(out, &done);
5457 __ LoadFromOffset(kLoadUnsignedHalfword, out, out, primitive_offset);
5458 static_assert(Primitive::kPrimNot == 0, "Expected 0 for kPrimNot");
5459 __ CompareAndBranchIfNonZero(out, &zero);
5460 __ Bind(&exact_check);
5461 __ LoadImmediate(out, 1);
5462 __ b(&done);
5463 break;
5464 }
5465
5466 case TypeCheckKind::kArrayCheck: {
5467 __ cmp(out, ShifterOperand(cls));
5468 DCHECK(locations->OnlyCallsOnSlowPath());
5469 slow_path = new (GetGraph()->GetArena()) TypeCheckSlowPathARM(instruction,
5470 /* is_fatal */ false);
5471 codegen_->AddSlowPath(slow_path);
5472 __ b(slow_path->GetEntryLabel(), NE);
5473 __ LoadImmediate(out, 1);
5474 if (zero.IsLinked()) {
5475 __ b(&done);
5476 }
5477 break;
5478 }
5479
5480 case TypeCheckKind::kUnresolvedCheck:
5481 case TypeCheckKind::kInterfaceCheck: {
5482 // Note that we indeed only call on slow path, but we always go
5483 // into the slow path for the unresolved and interface check
5484 // cases.
5485 //
5486 // We cannot directly call the InstanceofNonTrivial runtime
5487 // entry point without resorting to a type checking slow path
5488 // here (i.e. by calling InvokeRuntime directly), as it would
5489 // require to assign fixed registers for the inputs of this
5490 // HInstanceOf instruction (following the runtime calling
5491 // convention), which might be cluttered by the potential first
5492 // read barrier emission at the beginning of this method.
5493 //
5494 // TODO: Introduce a new runtime entry point taking the object
5495 // to test (instead of its class) as argument, and let it deal
5496 // with the read barrier issues. This will let us refactor this
5497 // case of the `switch` code as it was previously (with a direct
5498 // call to the runtime not using a type checking slow path).
5499 // This should also be beneficial for the other cases above.
5500 DCHECK(locations->OnlyCallsOnSlowPath());
5501 slow_path = new (GetGraph()->GetArena()) TypeCheckSlowPathARM(instruction,
5502 /* is_fatal */ false);
5503 codegen_->AddSlowPath(slow_path);
5504 __ b(slow_path->GetEntryLabel());
5505 if (zero.IsLinked()) {
5506 __ b(&done);
5507 }
5508 break;
5509 }
5510 }
5511
5512 if (zero.IsLinked()) {
5513 __ Bind(&zero);
5514 __ LoadImmediate(out, 0);
5515 }
5516
5517 if (done.IsLinked()) {
5518 __ Bind(&done);
5519 }
5520
5521 if (slow_path != nullptr) {
5522 __ Bind(slow_path->GetExitLabel());
5523 }
5524 }
5525
VisitCheckCast(HCheckCast * instruction)5526 void LocationsBuilderARM::VisitCheckCast(HCheckCast* instruction) {
5527 LocationSummary::CallKind call_kind = LocationSummary::kNoCall;
5528 bool throws_into_catch = instruction->CanThrowIntoCatchBlock();
5529
5530 TypeCheckKind type_check_kind = instruction->GetTypeCheckKind();
5531 switch (type_check_kind) {
5532 case TypeCheckKind::kExactCheck:
5533 case TypeCheckKind::kAbstractClassCheck:
5534 case TypeCheckKind::kClassHierarchyCheck:
5535 case TypeCheckKind::kArrayObjectCheck:
5536 call_kind = (throws_into_catch || kEmitCompilerReadBarrier) ?
5537 LocationSummary::kCallOnSlowPath :
5538 LocationSummary::kNoCall; // In fact, call on a fatal (non-returning) slow path.
5539 break;
5540 case TypeCheckKind::kArrayCheck:
5541 case TypeCheckKind::kUnresolvedCheck:
5542 case TypeCheckKind::kInterfaceCheck:
5543 call_kind = LocationSummary::kCallOnSlowPath;
5544 break;
5545 }
5546
5547 LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction, call_kind);
5548 locations->SetInAt(0, Location::RequiresRegister());
5549 locations->SetInAt(1, Location::RequiresRegister());
5550 // Note that TypeCheckSlowPathARM uses this "temp" register too.
5551 locations->AddTemp(Location::RequiresRegister());
5552 // When read barriers are enabled, we need an additional temporary
5553 // register for some cases.
5554 if (TypeCheckNeedsATemporary(type_check_kind)) {
5555 locations->AddTemp(Location::RequiresRegister());
5556 }
5557 }
5558
VisitCheckCast(HCheckCast * instruction)5559 void InstructionCodeGeneratorARM::VisitCheckCast(HCheckCast* instruction) {
5560 TypeCheckKind type_check_kind = instruction->GetTypeCheckKind();
5561 LocationSummary* locations = instruction->GetLocations();
5562 Location obj_loc = locations->InAt(0);
5563 Register obj = obj_loc.AsRegister<Register>();
5564 Register cls = locations->InAt(1).AsRegister<Register>();
5565 Location temp_loc = locations->GetTemp(0);
5566 Register temp = temp_loc.AsRegister<Register>();
5567 Location maybe_temp2_loc = TypeCheckNeedsATemporary(type_check_kind) ?
5568 locations->GetTemp(1) :
5569 Location::NoLocation();
5570 uint32_t class_offset = mirror::Object::ClassOffset().Int32Value();
5571 uint32_t super_offset = mirror::Class::SuperClassOffset().Int32Value();
5572 uint32_t component_offset = mirror::Class::ComponentTypeOffset().Int32Value();
5573 uint32_t primitive_offset = mirror::Class::PrimitiveTypeOffset().Int32Value();
5574
5575 bool is_type_check_slow_path_fatal =
5576 (type_check_kind == TypeCheckKind::kExactCheck ||
5577 type_check_kind == TypeCheckKind::kAbstractClassCheck ||
5578 type_check_kind == TypeCheckKind::kClassHierarchyCheck ||
5579 type_check_kind == TypeCheckKind::kArrayObjectCheck) &&
5580 !instruction->CanThrowIntoCatchBlock();
5581 SlowPathCode* type_check_slow_path =
5582 new (GetGraph()->GetArena()) TypeCheckSlowPathARM(instruction,
5583 is_type_check_slow_path_fatal);
5584 codegen_->AddSlowPath(type_check_slow_path);
5585
5586 Label done;
5587 // Avoid null check if we know obj is not null.
5588 if (instruction->MustDoNullCheck()) {
5589 __ CompareAndBranchIfZero(obj, &done);
5590 }
5591
5592 // /* HeapReference<Class> */ temp = obj->klass_
5593 GenerateReferenceLoadTwoRegisters(instruction, temp_loc, obj_loc, class_offset, maybe_temp2_loc);
5594
5595 switch (type_check_kind) {
5596 case TypeCheckKind::kExactCheck:
5597 case TypeCheckKind::kArrayCheck: {
5598 __ cmp(temp, ShifterOperand(cls));
5599 // Jump to slow path for throwing the exception or doing a
5600 // more involved array check.
5601 __ b(type_check_slow_path->GetEntryLabel(), NE);
5602 break;
5603 }
5604
5605 case TypeCheckKind::kAbstractClassCheck: {
5606 // If the class is abstract, we eagerly fetch the super class of the
5607 // object to avoid doing a comparison we know will fail.
5608 Label loop, compare_classes;
5609 __ Bind(&loop);
5610 // /* HeapReference<Class> */ temp = temp->super_class_
5611 GenerateReferenceLoadOneRegister(instruction, temp_loc, super_offset, maybe_temp2_loc);
5612
5613 // If the class reference currently in `temp` is not null, jump
5614 // to the `compare_classes` label to compare it with the checked
5615 // class.
5616 __ CompareAndBranchIfNonZero(temp, &compare_classes);
5617 // Otherwise, jump to the slow path to throw the exception.
5618 //
5619 // But before, move back the object's class into `temp` before
5620 // going into the slow path, as it has been overwritten in the
5621 // meantime.
5622 // /* HeapReference<Class> */ temp = obj->klass_
5623 GenerateReferenceLoadTwoRegisters(
5624 instruction, temp_loc, obj_loc, class_offset, maybe_temp2_loc);
5625 __ b(type_check_slow_path->GetEntryLabel());
5626
5627 __ Bind(&compare_classes);
5628 __ cmp(temp, ShifterOperand(cls));
5629 __ b(&loop, NE);
5630 break;
5631 }
5632
5633 case TypeCheckKind::kClassHierarchyCheck: {
5634 // Walk over the class hierarchy to find a match.
5635 Label loop;
5636 __ Bind(&loop);
5637 __ cmp(temp, ShifterOperand(cls));
5638 __ b(&done, EQ);
5639
5640 // /* HeapReference<Class> */ temp = temp->super_class_
5641 GenerateReferenceLoadOneRegister(instruction, temp_loc, super_offset, maybe_temp2_loc);
5642
5643 // If the class reference currently in `temp` is not null, jump
5644 // back at the beginning of the loop.
5645 __ CompareAndBranchIfNonZero(temp, &loop);
5646 // Otherwise, jump to the slow path to throw the exception.
5647 //
5648 // But before, move back the object's class into `temp` before
5649 // going into the slow path, as it has been overwritten in the
5650 // meantime.
5651 // /* HeapReference<Class> */ temp = obj->klass_
5652 GenerateReferenceLoadTwoRegisters(
5653 instruction, temp_loc, obj_loc, class_offset, maybe_temp2_loc);
5654 __ b(type_check_slow_path->GetEntryLabel());
5655 break;
5656 }
5657
5658 case TypeCheckKind::kArrayObjectCheck: {
5659 // Do an exact check.
5660 Label check_non_primitive_component_type;
5661 __ cmp(temp, ShifterOperand(cls));
5662 __ b(&done, EQ);
5663
5664 // Otherwise, we need to check that the object's class is a non-primitive array.
5665 // /* HeapReference<Class> */ temp = temp->component_type_
5666 GenerateReferenceLoadOneRegister(instruction, temp_loc, component_offset, maybe_temp2_loc);
5667
5668 // If the component type is not null (i.e. the object is indeed
5669 // an array), jump to label `check_non_primitive_component_type`
5670 // to further check that this component type is not a primitive
5671 // type.
5672 __ CompareAndBranchIfNonZero(temp, &check_non_primitive_component_type);
5673 // Otherwise, jump to the slow path to throw the exception.
5674 //
5675 // But before, move back the object's class into `temp` before
5676 // going into the slow path, as it has been overwritten in the
5677 // meantime.
5678 // /* HeapReference<Class> */ temp = obj->klass_
5679 GenerateReferenceLoadTwoRegisters(
5680 instruction, temp_loc, obj_loc, class_offset, maybe_temp2_loc);
5681 __ b(type_check_slow_path->GetEntryLabel());
5682
5683 __ Bind(&check_non_primitive_component_type);
5684 __ LoadFromOffset(kLoadUnsignedHalfword, temp, temp, primitive_offset);
5685 static_assert(Primitive::kPrimNot == 0, "Expected 0 for art::Primitive::kPrimNot");
5686 __ CompareAndBranchIfZero(temp, &done);
5687 // Same comment as above regarding `temp` and the slow path.
5688 // /* HeapReference<Class> */ temp = obj->klass_
5689 GenerateReferenceLoadTwoRegisters(
5690 instruction, temp_loc, obj_loc, class_offset, maybe_temp2_loc);
5691 __ b(type_check_slow_path->GetEntryLabel());
5692 break;
5693 }
5694
5695 case TypeCheckKind::kUnresolvedCheck:
5696 case TypeCheckKind::kInterfaceCheck:
5697 // We always go into the type check slow path for the unresolved
5698 // and interface check cases.
5699 //
5700 // We cannot directly call the CheckCast runtime entry point
5701 // without resorting to a type checking slow path here (i.e. by
5702 // calling InvokeRuntime directly), as it would require to
5703 // assign fixed registers for the inputs of this HInstanceOf
5704 // instruction (following the runtime calling convention), which
5705 // might be cluttered by the potential first read barrier
5706 // emission at the beginning of this method.
5707 //
5708 // TODO: Introduce a new runtime entry point taking the object
5709 // to test (instead of its class) as argument, and let it deal
5710 // with the read barrier issues. This will let us refactor this
5711 // case of the `switch` code as it was previously (with a direct
5712 // call to the runtime not using a type checking slow path).
5713 // This should also be beneficial for the other cases above.
5714 __ b(type_check_slow_path->GetEntryLabel());
5715 break;
5716 }
5717 __ Bind(&done);
5718
5719 __ Bind(type_check_slow_path->GetExitLabel());
5720 }
5721
VisitMonitorOperation(HMonitorOperation * instruction)5722 void LocationsBuilderARM::VisitMonitorOperation(HMonitorOperation* instruction) {
5723 LocationSummary* locations =
5724 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCall);
5725 InvokeRuntimeCallingConvention calling_convention;
5726 locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(0)));
5727 }
5728
VisitMonitorOperation(HMonitorOperation * instruction)5729 void InstructionCodeGeneratorARM::VisitMonitorOperation(HMonitorOperation* instruction) {
5730 codegen_->InvokeRuntime(instruction->IsEnter()
5731 ? QUICK_ENTRY_POINT(pLockObject) : QUICK_ENTRY_POINT(pUnlockObject),
5732 instruction,
5733 instruction->GetDexPc(),
5734 nullptr);
5735 if (instruction->IsEnter()) {
5736 CheckEntrypointTypes<kQuickLockObject, void, mirror::Object*>();
5737 } else {
5738 CheckEntrypointTypes<kQuickUnlockObject, void, mirror::Object*>();
5739 }
5740 }
5741
VisitAnd(HAnd * instruction)5742 void LocationsBuilderARM::VisitAnd(HAnd* instruction) { HandleBitwiseOperation(instruction, AND); }
VisitOr(HOr * instruction)5743 void LocationsBuilderARM::VisitOr(HOr* instruction) { HandleBitwiseOperation(instruction, ORR); }
VisitXor(HXor * instruction)5744 void LocationsBuilderARM::VisitXor(HXor* instruction) { HandleBitwiseOperation(instruction, EOR); }
5745
HandleBitwiseOperation(HBinaryOperation * instruction,Opcode opcode)5746 void LocationsBuilderARM::HandleBitwiseOperation(HBinaryOperation* instruction, Opcode opcode) {
5747 LocationSummary* locations =
5748 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
5749 DCHECK(instruction->GetResultType() == Primitive::kPrimInt
5750 || instruction->GetResultType() == Primitive::kPrimLong);
5751 // Note: GVN reorders commutative operations to have the constant on the right hand side.
5752 locations->SetInAt(0, Location::RequiresRegister());
5753 locations->SetInAt(1, ArmEncodableConstantOrRegister(instruction->InputAt(1), opcode));
5754 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
5755 }
5756
VisitAnd(HAnd * instruction)5757 void InstructionCodeGeneratorARM::VisitAnd(HAnd* instruction) {
5758 HandleBitwiseOperation(instruction);
5759 }
5760
VisitOr(HOr * instruction)5761 void InstructionCodeGeneratorARM::VisitOr(HOr* instruction) {
5762 HandleBitwiseOperation(instruction);
5763 }
5764
VisitXor(HXor * instruction)5765 void InstructionCodeGeneratorARM::VisitXor(HXor* instruction) {
5766 HandleBitwiseOperation(instruction);
5767 }
5768
5769
VisitBitwiseNegatedRight(HBitwiseNegatedRight * instruction)5770 void LocationsBuilderARM::VisitBitwiseNegatedRight(HBitwiseNegatedRight* instruction) {
5771 LocationSummary* locations =
5772 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
5773 DCHECK(instruction->GetResultType() == Primitive::kPrimInt
5774 || instruction->GetResultType() == Primitive::kPrimLong);
5775
5776 locations->SetInAt(0, Location::RequiresRegister());
5777 locations->SetInAt(1, Location::RequiresRegister());
5778 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
5779 }
5780
VisitBitwiseNegatedRight(HBitwiseNegatedRight * instruction)5781 void InstructionCodeGeneratorARM::VisitBitwiseNegatedRight(HBitwiseNegatedRight* instruction) {
5782 LocationSummary* locations = instruction->GetLocations();
5783 Location first = locations->InAt(0);
5784 Location second = locations->InAt(1);
5785 Location out = locations->Out();
5786
5787 if (instruction->GetResultType() == Primitive::kPrimInt) {
5788 Register first_reg = first.AsRegister<Register>();
5789 ShifterOperand second_reg(second.AsRegister<Register>());
5790 Register out_reg = out.AsRegister<Register>();
5791
5792 switch (instruction->GetOpKind()) {
5793 case HInstruction::kAnd:
5794 __ bic(out_reg, first_reg, second_reg);
5795 break;
5796 case HInstruction::kOr:
5797 __ orn(out_reg, first_reg, second_reg);
5798 break;
5799 // There is no EON on arm.
5800 case HInstruction::kXor:
5801 default:
5802 LOG(FATAL) << "Unexpected instruction " << instruction->DebugName();
5803 UNREACHABLE();
5804 }
5805 return;
5806
5807 } else {
5808 DCHECK_EQ(instruction->GetResultType(), Primitive::kPrimLong);
5809 Register first_low = first.AsRegisterPairLow<Register>();
5810 Register first_high = first.AsRegisterPairHigh<Register>();
5811 ShifterOperand second_low(second.AsRegisterPairLow<Register>());
5812 ShifterOperand second_high(second.AsRegisterPairHigh<Register>());
5813 Register out_low = out.AsRegisterPairLow<Register>();
5814 Register out_high = out.AsRegisterPairHigh<Register>();
5815
5816 switch (instruction->GetOpKind()) {
5817 case HInstruction::kAnd:
5818 __ bic(out_low, first_low, second_low);
5819 __ bic(out_high, first_high, second_high);
5820 break;
5821 case HInstruction::kOr:
5822 __ orn(out_low, first_low, second_low);
5823 __ orn(out_high, first_high, second_high);
5824 break;
5825 // There is no EON on arm.
5826 case HInstruction::kXor:
5827 default:
5828 LOG(FATAL) << "Unexpected instruction " << instruction->DebugName();
5829 UNREACHABLE();
5830 }
5831 }
5832 }
5833
GenerateAndConst(Register out,Register first,uint32_t value)5834 void InstructionCodeGeneratorARM::GenerateAndConst(Register out, Register first, uint32_t value) {
5835 // Optimize special cases for individual halfs of `and-long` (`and` is simplified earlier).
5836 if (value == 0xffffffffu) {
5837 if (out != first) {
5838 __ mov(out, ShifterOperand(first));
5839 }
5840 return;
5841 }
5842 if (value == 0u) {
5843 __ mov(out, ShifterOperand(0));
5844 return;
5845 }
5846 ShifterOperand so;
5847 if (__ ShifterOperandCanHold(kNoRegister, kNoRegister, AND, value, &so)) {
5848 __ and_(out, first, so);
5849 } else {
5850 DCHECK(__ ShifterOperandCanHold(kNoRegister, kNoRegister, BIC, ~value, &so));
5851 __ bic(out, first, ShifterOperand(~value));
5852 }
5853 }
5854
GenerateOrrConst(Register out,Register first,uint32_t value)5855 void InstructionCodeGeneratorARM::GenerateOrrConst(Register out, Register first, uint32_t value) {
5856 // Optimize special cases for individual halfs of `or-long` (`or` is simplified earlier).
5857 if (value == 0u) {
5858 if (out != first) {
5859 __ mov(out, ShifterOperand(first));
5860 }
5861 return;
5862 }
5863 if (value == 0xffffffffu) {
5864 __ mvn(out, ShifterOperand(0));
5865 return;
5866 }
5867 ShifterOperand so;
5868 if (__ ShifterOperandCanHold(kNoRegister, kNoRegister, ORR, value, &so)) {
5869 __ orr(out, first, so);
5870 } else {
5871 DCHECK(__ ShifterOperandCanHold(kNoRegister, kNoRegister, ORN, ~value, &so));
5872 __ orn(out, first, ShifterOperand(~value));
5873 }
5874 }
5875
GenerateEorConst(Register out,Register first,uint32_t value)5876 void InstructionCodeGeneratorARM::GenerateEorConst(Register out, Register first, uint32_t value) {
5877 // Optimize special case for individual halfs of `xor-long` (`xor` is simplified earlier).
5878 if (value == 0u) {
5879 if (out != first) {
5880 __ mov(out, ShifterOperand(first));
5881 }
5882 return;
5883 }
5884 __ eor(out, first, ShifterOperand(value));
5885 }
5886
HandleBitwiseOperation(HBinaryOperation * instruction)5887 void InstructionCodeGeneratorARM::HandleBitwiseOperation(HBinaryOperation* instruction) {
5888 LocationSummary* locations = instruction->GetLocations();
5889 Location first = locations->InAt(0);
5890 Location second = locations->InAt(1);
5891 Location out = locations->Out();
5892
5893 if (second.IsConstant()) {
5894 uint64_t value = static_cast<uint64_t>(Int64FromConstant(second.GetConstant()));
5895 uint32_t value_low = Low32Bits(value);
5896 if (instruction->GetResultType() == Primitive::kPrimInt) {
5897 Register first_reg = first.AsRegister<Register>();
5898 Register out_reg = out.AsRegister<Register>();
5899 if (instruction->IsAnd()) {
5900 GenerateAndConst(out_reg, first_reg, value_low);
5901 } else if (instruction->IsOr()) {
5902 GenerateOrrConst(out_reg, first_reg, value_low);
5903 } else {
5904 DCHECK(instruction->IsXor());
5905 GenerateEorConst(out_reg, first_reg, value_low);
5906 }
5907 } else {
5908 DCHECK_EQ(instruction->GetResultType(), Primitive::kPrimLong);
5909 uint32_t value_high = High32Bits(value);
5910 Register first_low = first.AsRegisterPairLow<Register>();
5911 Register first_high = first.AsRegisterPairHigh<Register>();
5912 Register out_low = out.AsRegisterPairLow<Register>();
5913 Register out_high = out.AsRegisterPairHigh<Register>();
5914 if (instruction->IsAnd()) {
5915 GenerateAndConst(out_low, first_low, value_low);
5916 GenerateAndConst(out_high, first_high, value_high);
5917 } else if (instruction->IsOr()) {
5918 GenerateOrrConst(out_low, first_low, value_low);
5919 GenerateOrrConst(out_high, first_high, value_high);
5920 } else {
5921 DCHECK(instruction->IsXor());
5922 GenerateEorConst(out_low, first_low, value_low);
5923 GenerateEorConst(out_high, first_high, value_high);
5924 }
5925 }
5926 return;
5927 }
5928
5929 if (instruction->GetResultType() == Primitive::kPrimInt) {
5930 Register first_reg = first.AsRegister<Register>();
5931 ShifterOperand second_reg(second.AsRegister<Register>());
5932 Register out_reg = out.AsRegister<Register>();
5933 if (instruction->IsAnd()) {
5934 __ and_(out_reg, first_reg, second_reg);
5935 } else if (instruction->IsOr()) {
5936 __ orr(out_reg, first_reg, second_reg);
5937 } else {
5938 DCHECK(instruction->IsXor());
5939 __ eor(out_reg, first_reg, second_reg);
5940 }
5941 } else {
5942 DCHECK_EQ(instruction->GetResultType(), Primitive::kPrimLong);
5943 Register first_low = first.AsRegisterPairLow<Register>();
5944 Register first_high = first.AsRegisterPairHigh<Register>();
5945 ShifterOperand second_low(second.AsRegisterPairLow<Register>());
5946 ShifterOperand second_high(second.AsRegisterPairHigh<Register>());
5947 Register out_low = out.AsRegisterPairLow<Register>();
5948 Register out_high = out.AsRegisterPairHigh<Register>();
5949 if (instruction->IsAnd()) {
5950 __ and_(out_low, first_low, second_low);
5951 __ and_(out_high, first_high, second_high);
5952 } else if (instruction->IsOr()) {
5953 __ orr(out_low, first_low, second_low);
5954 __ orr(out_high, first_high, second_high);
5955 } else {
5956 DCHECK(instruction->IsXor());
5957 __ eor(out_low, first_low, second_low);
5958 __ eor(out_high, first_high, second_high);
5959 }
5960 }
5961 }
5962
GenerateReferenceLoadOneRegister(HInstruction * instruction,Location out,uint32_t offset,Location maybe_temp)5963 void InstructionCodeGeneratorARM::GenerateReferenceLoadOneRegister(HInstruction* instruction,
5964 Location out,
5965 uint32_t offset,
5966 Location maybe_temp) {
5967 Register out_reg = out.AsRegister<Register>();
5968 if (kEmitCompilerReadBarrier) {
5969 DCHECK(maybe_temp.IsRegister()) << maybe_temp;
5970 if (kUseBakerReadBarrier) {
5971 // Load with fast path based Baker's read barrier.
5972 // /* HeapReference<Object> */ out = *(out + offset)
5973 codegen_->GenerateFieldLoadWithBakerReadBarrier(
5974 instruction, out, out_reg, offset, maybe_temp, /* needs_null_check */ false);
5975 } else {
5976 // Load with slow path based read barrier.
5977 // Save the value of `out` into `maybe_temp` before overwriting it
5978 // in the following move operation, as we will need it for the
5979 // read barrier below.
5980 __ Mov(maybe_temp.AsRegister<Register>(), out_reg);
5981 // /* HeapReference<Object> */ out = *(out + offset)
5982 __ LoadFromOffset(kLoadWord, out_reg, out_reg, offset);
5983 codegen_->GenerateReadBarrierSlow(instruction, out, out, maybe_temp, offset);
5984 }
5985 } else {
5986 // Plain load with no read barrier.
5987 // /* HeapReference<Object> */ out = *(out + offset)
5988 __ LoadFromOffset(kLoadWord, out_reg, out_reg, offset);
5989 __ MaybeUnpoisonHeapReference(out_reg);
5990 }
5991 }
5992
GenerateReferenceLoadTwoRegisters(HInstruction * instruction,Location out,Location obj,uint32_t offset,Location maybe_temp)5993 void InstructionCodeGeneratorARM::GenerateReferenceLoadTwoRegisters(HInstruction* instruction,
5994 Location out,
5995 Location obj,
5996 uint32_t offset,
5997 Location maybe_temp) {
5998 Register out_reg = out.AsRegister<Register>();
5999 Register obj_reg = obj.AsRegister<Register>();
6000 if (kEmitCompilerReadBarrier) {
6001 if (kUseBakerReadBarrier) {
6002 DCHECK(maybe_temp.IsRegister()) << maybe_temp;
6003 // Load with fast path based Baker's read barrier.
6004 // /* HeapReference<Object> */ out = *(obj + offset)
6005 codegen_->GenerateFieldLoadWithBakerReadBarrier(
6006 instruction, out, obj_reg, offset, maybe_temp, /* needs_null_check */ false);
6007 } else {
6008 // Load with slow path based read barrier.
6009 // /* HeapReference<Object> */ out = *(obj + offset)
6010 __ LoadFromOffset(kLoadWord, out_reg, obj_reg, offset);
6011 codegen_->GenerateReadBarrierSlow(instruction, out, out, obj, offset);
6012 }
6013 } else {
6014 // Plain load with no read barrier.
6015 // /* HeapReference<Object> */ out = *(obj + offset)
6016 __ LoadFromOffset(kLoadWord, out_reg, obj_reg, offset);
6017 __ MaybeUnpoisonHeapReference(out_reg);
6018 }
6019 }
6020
GenerateGcRootFieldLoad(HInstruction * instruction,Location root,Register obj,uint32_t offset)6021 void InstructionCodeGeneratorARM::GenerateGcRootFieldLoad(HInstruction* instruction,
6022 Location root,
6023 Register obj,
6024 uint32_t offset) {
6025 Register root_reg = root.AsRegister<Register>();
6026 if (kEmitCompilerReadBarrier) {
6027 if (kUseBakerReadBarrier) {
6028 // Fast path implementation of art::ReadBarrier::BarrierForRoot when
6029 // Baker's read barrier are used:
6030 //
6031 // root = obj.field;
6032 // if (Thread::Current()->GetIsGcMarking()) {
6033 // root = ReadBarrier::Mark(root)
6034 // }
6035
6036 // /* GcRoot<mirror::Object> */ root = *(obj + offset)
6037 __ LoadFromOffset(kLoadWord, root_reg, obj, offset);
6038 static_assert(
6039 sizeof(mirror::CompressedReference<mirror::Object>) == sizeof(GcRoot<mirror::Object>),
6040 "art::mirror::CompressedReference<mirror::Object> and art::GcRoot<mirror::Object> "
6041 "have different sizes.");
6042 static_assert(sizeof(mirror::CompressedReference<mirror::Object>) == sizeof(int32_t),
6043 "art::mirror::CompressedReference<mirror::Object> and int32_t "
6044 "have different sizes.");
6045
6046 // Slow path used to mark the GC root `root`.
6047 SlowPathCode* slow_path =
6048 new (GetGraph()->GetArena()) ReadBarrierMarkSlowPathARM(instruction, root, root);
6049 codegen_->AddSlowPath(slow_path);
6050
6051 // IP = Thread::Current()->GetIsGcMarking()
6052 __ LoadFromOffset(
6053 kLoadWord, IP, TR, Thread::IsGcMarkingOffset<kArmWordSize>().Int32Value());
6054 __ CompareAndBranchIfNonZero(IP, slow_path->GetEntryLabel());
6055 __ Bind(slow_path->GetExitLabel());
6056 } else {
6057 // GC root loaded through a slow path for read barriers other
6058 // than Baker's.
6059 // /* GcRoot<mirror::Object>* */ root = obj + offset
6060 __ AddConstant(root_reg, obj, offset);
6061 // /* mirror::Object* */ root = root->Read()
6062 codegen_->GenerateReadBarrierForRootSlow(instruction, root, root);
6063 }
6064 } else {
6065 // Plain GC root load with no read barrier.
6066 // /* GcRoot<mirror::Object> */ root = *(obj + offset)
6067 __ LoadFromOffset(kLoadWord, root_reg, obj, offset);
6068 // Note that GC roots are not affected by heap poisoning, thus we
6069 // do not have to unpoison `root_reg` here.
6070 }
6071 }
6072
GenerateFieldLoadWithBakerReadBarrier(HInstruction * instruction,Location ref,Register obj,uint32_t offset,Location temp,bool needs_null_check)6073 void CodeGeneratorARM::GenerateFieldLoadWithBakerReadBarrier(HInstruction* instruction,
6074 Location ref,
6075 Register obj,
6076 uint32_t offset,
6077 Location temp,
6078 bool needs_null_check) {
6079 DCHECK(kEmitCompilerReadBarrier);
6080 DCHECK(kUseBakerReadBarrier);
6081
6082 // /* HeapReference<Object> */ ref = *(obj + offset)
6083 Location no_index = Location::NoLocation();
6084 GenerateReferenceLoadWithBakerReadBarrier(
6085 instruction, ref, obj, offset, no_index, temp, needs_null_check);
6086 }
6087
GenerateArrayLoadWithBakerReadBarrier(HInstruction * instruction,Location ref,Register obj,uint32_t data_offset,Location index,Location temp,bool needs_null_check)6088 void CodeGeneratorARM::GenerateArrayLoadWithBakerReadBarrier(HInstruction* instruction,
6089 Location ref,
6090 Register obj,
6091 uint32_t data_offset,
6092 Location index,
6093 Location temp,
6094 bool needs_null_check) {
6095 DCHECK(kEmitCompilerReadBarrier);
6096 DCHECK(kUseBakerReadBarrier);
6097
6098 // /* HeapReference<Object> */ ref =
6099 // *(obj + data_offset + index * sizeof(HeapReference<Object>))
6100 GenerateReferenceLoadWithBakerReadBarrier(
6101 instruction, ref, obj, data_offset, index, temp, needs_null_check);
6102 }
6103
GenerateReferenceLoadWithBakerReadBarrier(HInstruction * instruction,Location ref,Register obj,uint32_t offset,Location index,Location temp,bool needs_null_check)6104 void CodeGeneratorARM::GenerateReferenceLoadWithBakerReadBarrier(HInstruction* instruction,
6105 Location ref,
6106 Register obj,
6107 uint32_t offset,
6108 Location index,
6109 Location temp,
6110 bool needs_null_check) {
6111 DCHECK(kEmitCompilerReadBarrier);
6112 DCHECK(kUseBakerReadBarrier);
6113
6114 // In slow path based read barriers, the read barrier call is
6115 // inserted after the original load. However, in fast path based
6116 // Baker's read barriers, we need to perform the load of
6117 // mirror::Object::monitor_ *before* the original reference load.
6118 // This load-load ordering is required by the read barrier.
6119 // The fast path/slow path (for Baker's algorithm) should look like:
6120 //
6121 // uint32_t rb_state = Lockword(obj->monitor_).ReadBarrierState();
6122 // lfence; // Load fence or artificial data dependency to prevent load-load reordering
6123 // HeapReference<Object> ref = *src; // Original reference load.
6124 // bool is_gray = (rb_state == ReadBarrier::gray_ptr_);
6125 // if (is_gray) {
6126 // ref = ReadBarrier::Mark(ref); // Performed by runtime entrypoint slow path.
6127 // }
6128 //
6129 // Note: the original implementation in ReadBarrier::Barrier is
6130 // slightly more complex as it performs additional checks that we do
6131 // not do here for performance reasons.
6132
6133 Register ref_reg = ref.AsRegister<Register>();
6134 Register temp_reg = temp.AsRegister<Register>();
6135 uint32_t monitor_offset = mirror::Object::MonitorOffset().Int32Value();
6136
6137 // /* int32_t */ monitor = obj->monitor_
6138 __ LoadFromOffset(kLoadWord, temp_reg, obj, monitor_offset);
6139 if (needs_null_check) {
6140 MaybeRecordImplicitNullCheck(instruction);
6141 }
6142 // /* LockWord */ lock_word = LockWord(monitor)
6143 static_assert(sizeof(LockWord) == sizeof(int32_t),
6144 "art::LockWord and int32_t have different sizes.");
6145 // /* uint32_t */ rb_state = lock_word.ReadBarrierState()
6146 __ Lsr(temp_reg, temp_reg, LockWord::kReadBarrierStateShift);
6147 __ and_(temp_reg, temp_reg, ShifterOperand(LockWord::kReadBarrierStateMask));
6148 static_assert(
6149 LockWord::kReadBarrierStateMask == ReadBarrier::rb_ptr_mask_,
6150 "art::LockWord::kReadBarrierStateMask is not equal to art::ReadBarrier::rb_ptr_mask_.");
6151
6152 // Introduce a dependency on the high bits of rb_state, which shall
6153 // be all zeroes, to prevent load-load reordering, and without using
6154 // a memory barrier (which would be more expensive).
6155 // IP = rb_state & ~LockWord::kReadBarrierStateMask = 0
6156 __ bic(IP, temp_reg, ShifterOperand(LockWord::kReadBarrierStateMask));
6157 // obj is unchanged by this operation, but its value now depends on
6158 // IP, which depends on temp_reg.
6159 __ add(obj, obj, ShifterOperand(IP));
6160
6161 // The actual reference load.
6162 if (index.IsValid()) {
6163 static_assert(
6164 sizeof(mirror::HeapReference<mirror::Object>) == sizeof(int32_t),
6165 "art::mirror::HeapReference<art::mirror::Object> and int32_t have different sizes.");
6166 // /* HeapReference<Object> */ ref =
6167 // *(obj + offset + index * sizeof(HeapReference<Object>))
6168 if (index.IsConstant()) {
6169 size_t computed_offset =
6170 (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4) + offset;
6171 __ LoadFromOffset(kLoadWord, ref_reg, obj, computed_offset);
6172 } else {
6173 __ add(IP, obj, ShifterOperand(index.AsRegister<Register>(), LSL, TIMES_4));
6174 __ LoadFromOffset(kLoadWord, ref_reg, IP, offset);
6175 }
6176 } else {
6177 // /* HeapReference<Object> */ ref = *(obj + offset)
6178 __ LoadFromOffset(kLoadWord, ref_reg, obj, offset);
6179 }
6180
6181 // Object* ref = ref_addr->AsMirrorPtr()
6182 __ MaybeUnpoisonHeapReference(ref_reg);
6183
6184 // Slow path used to mark the object `ref` when it is gray.
6185 SlowPathCode* slow_path =
6186 new (GetGraph()->GetArena()) ReadBarrierMarkSlowPathARM(instruction, ref, ref);
6187 AddSlowPath(slow_path);
6188
6189 // if (rb_state == ReadBarrier::gray_ptr_)
6190 // ref = ReadBarrier::Mark(ref);
6191 __ cmp(temp_reg, ShifterOperand(ReadBarrier::gray_ptr_));
6192 __ b(slow_path->GetEntryLabel(), EQ);
6193 __ Bind(slow_path->GetExitLabel());
6194 }
6195
GenerateReadBarrierSlow(HInstruction * instruction,Location out,Location ref,Location obj,uint32_t offset,Location index)6196 void CodeGeneratorARM::GenerateReadBarrierSlow(HInstruction* instruction,
6197 Location out,
6198 Location ref,
6199 Location obj,
6200 uint32_t offset,
6201 Location index) {
6202 DCHECK(kEmitCompilerReadBarrier);
6203
6204 // Insert a slow path based read barrier *after* the reference load.
6205 //
6206 // If heap poisoning is enabled, the unpoisoning of the loaded
6207 // reference will be carried out by the runtime within the slow
6208 // path.
6209 //
6210 // Note that `ref` currently does not get unpoisoned (when heap
6211 // poisoning is enabled), which is alright as the `ref` argument is
6212 // not used by the artReadBarrierSlow entry point.
6213 //
6214 // TODO: Unpoison `ref` when it is used by artReadBarrierSlow.
6215 SlowPathCode* slow_path = new (GetGraph()->GetArena())
6216 ReadBarrierForHeapReferenceSlowPathARM(instruction, out, ref, obj, offset, index);
6217 AddSlowPath(slow_path);
6218
6219 __ b(slow_path->GetEntryLabel());
6220 __ Bind(slow_path->GetExitLabel());
6221 }
6222
MaybeGenerateReadBarrierSlow(HInstruction * instruction,Location out,Location ref,Location obj,uint32_t offset,Location index)6223 void CodeGeneratorARM::MaybeGenerateReadBarrierSlow(HInstruction* instruction,
6224 Location out,
6225 Location ref,
6226 Location obj,
6227 uint32_t offset,
6228 Location index) {
6229 if (kEmitCompilerReadBarrier) {
6230 // Baker's read barriers shall be handled by the fast path
6231 // (CodeGeneratorARM::GenerateReferenceLoadWithBakerReadBarrier).
6232 DCHECK(!kUseBakerReadBarrier);
6233 // If heap poisoning is enabled, unpoisoning will be taken care of
6234 // by the runtime within the slow path.
6235 GenerateReadBarrierSlow(instruction, out, ref, obj, offset, index);
6236 } else if (kPoisonHeapReferences) {
6237 __ UnpoisonHeapReference(out.AsRegister<Register>());
6238 }
6239 }
6240
GenerateReadBarrierForRootSlow(HInstruction * instruction,Location out,Location root)6241 void CodeGeneratorARM::GenerateReadBarrierForRootSlow(HInstruction* instruction,
6242 Location out,
6243 Location root) {
6244 DCHECK(kEmitCompilerReadBarrier);
6245
6246 // Insert a slow path based read barrier *after* the GC root load.
6247 //
6248 // Note that GC roots are not affected by heap poisoning, so we do
6249 // not need to do anything special for this here.
6250 SlowPathCode* slow_path =
6251 new (GetGraph()->GetArena()) ReadBarrierForRootSlowPathARM(instruction, out, root);
6252 AddSlowPath(slow_path);
6253
6254 __ b(slow_path->GetEntryLabel());
6255 __ Bind(slow_path->GetExitLabel());
6256 }
6257
GetSupportedInvokeStaticOrDirectDispatch(const HInvokeStaticOrDirect::DispatchInfo & desired_dispatch_info,MethodReference target_method)6258 HInvokeStaticOrDirect::DispatchInfo CodeGeneratorARM::GetSupportedInvokeStaticOrDirectDispatch(
6259 const HInvokeStaticOrDirect::DispatchInfo& desired_dispatch_info,
6260 MethodReference target_method) {
6261 HInvokeStaticOrDirect::DispatchInfo dispatch_info = desired_dispatch_info;
6262 // We disable pc-relative load when there is an irreducible loop, as the optimization
6263 // is incompatible with it.
6264 // TODO: Create as many ArmDexCacheArraysBase instructions as needed for methods
6265 // with irreducible loops.
6266 if (GetGraph()->HasIrreducibleLoops() &&
6267 (dispatch_info.method_load_kind ==
6268 HInvokeStaticOrDirect::MethodLoadKind::kDexCachePcRelative)) {
6269 dispatch_info.method_load_kind = HInvokeStaticOrDirect::MethodLoadKind::kDexCacheViaMethod;
6270 }
6271
6272 if (dispatch_info.code_ptr_location == HInvokeStaticOrDirect::CodePtrLocation::kCallPCRelative) {
6273 const DexFile& outer_dex_file = GetGraph()->GetDexFile();
6274 if (&outer_dex_file != target_method.dex_file) {
6275 // Calls across dex files are more likely to exceed the available BL range,
6276 // so use absolute patch with fixup if available and kCallArtMethod otherwise.
6277 HInvokeStaticOrDirect::CodePtrLocation code_ptr_location =
6278 (desired_dispatch_info.method_load_kind ==
6279 HInvokeStaticOrDirect::MethodLoadKind::kDirectAddressWithFixup)
6280 ? HInvokeStaticOrDirect::CodePtrLocation::kCallDirectWithFixup
6281 : HInvokeStaticOrDirect::CodePtrLocation::kCallArtMethod;
6282 return HInvokeStaticOrDirect::DispatchInfo {
6283 dispatch_info.method_load_kind,
6284 code_ptr_location,
6285 dispatch_info.method_load_data,
6286 0u
6287 };
6288 }
6289 }
6290 return dispatch_info;
6291 }
6292
GetInvokeStaticOrDirectExtraParameter(HInvokeStaticOrDirect * invoke,Register temp)6293 Register CodeGeneratorARM::GetInvokeStaticOrDirectExtraParameter(HInvokeStaticOrDirect* invoke,
6294 Register temp) {
6295 DCHECK_EQ(invoke->InputCount(), invoke->GetNumberOfArguments() + 1u);
6296 Location location = invoke->GetLocations()->InAt(invoke->GetSpecialInputIndex());
6297 if (!invoke->GetLocations()->Intrinsified()) {
6298 return location.AsRegister<Register>();
6299 }
6300 // For intrinsics we allow any location, so it may be on the stack.
6301 if (!location.IsRegister()) {
6302 __ LoadFromOffset(kLoadWord, temp, SP, location.GetStackIndex());
6303 return temp;
6304 }
6305 // For register locations, check if the register was saved. If so, get it from the stack.
6306 // Note: There is a chance that the register was saved but not overwritten, so we could
6307 // save one load. However, since this is just an intrinsic slow path we prefer this
6308 // simple and more robust approach rather that trying to determine if that's the case.
6309 SlowPathCode* slow_path = GetCurrentSlowPath();
6310 DCHECK(slow_path != nullptr); // For intrinsified invokes the call is emitted on the slow path.
6311 if (slow_path->IsCoreRegisterSaved(location.AsRegister<Register>())) {
6312 int stack_offset = slow_path->GetStackOffsetOfCoreRegister(location.AsRegister<Register>());
6313 __ LoadFromOffset(kLoadWord, temp, SP, stack_offset);
6314 return temp;
6315 }
6316 return location.AsRegister<Register>();
6317 }
6318
GenerateStaticOrDirectCall(HInvokeStaticOrDirect * invoke,Location temp)6319 void CodeGeneratorARM::GenerateStaticOrDirectCall(HInvokeStaticOrDirect* invoke, Location temp) {
6320 // For better instruction scheduling we load the direct code pointer before the method pointer.
6321 switch (invoke->GetCodePtrLocation()) {
6322 case HInvokeStaticOrDirect::CodePtrLocation::kCallDirectWithFixup:
6323 // LR = code address from literal pool with link-time patch.
6324 __ LoadLiteral(LR, DeduplicateMethodCodeLiteral(invoke->GetTargetMethod()));
6325 break;
6326 case HInvokeStaticOrDirect::CodePtrLocation::kCallDirect:
6327 // LR = invoke->GetDirectCodePtr();
6328 __ LoadImmediate(LR, invoke->GetDirectCodePtr());
6329 break;
6330 default:
6331 break;
6332 }
6333
6334 Location callee_method = temp; // For all kinds except kRecursive, callee will be in temp.
6335 switch (invoke->GetMethodLoadKind()) {
6336 case HInvokeStaticOrDirect::MethodLoadKind::kStringInit:
6337 // temp = thread->string_init_entrypoint
6338 __ LoadFromOffset(kLoadWord, temp.AsRegister<Register>(), TR, invoke->GetStringInitOffset());
6339 break;
6340 case HInvokeStaticOrDirect::MethodLoadKind::kRecursive:
6341 callee_method = invoke->GetLocations()->InAt(invoke->GetSpecialInputIndex());
6342 break;
6343 case HInvokeStaticOrDirect::MethodLoadKind::kDirectAddress:
6344 __ LoadImmediate(temp.AsRegister<Register>(), invoke->GetMethodAddress());
6345 break;
6346 case HInvokeStaticOrDirect::MethodLoadKind::kDirectAddressWithFixup:
6347 __ LoadLiteral(temp.AsRegister<Register>(),
6348 DeduplicateMethodAddressLiteral(invoke->GetTargetMethod()));
6349 break;
6350 case HInvokeStaticOrDirect::MethodLoadKind::kDexCachePcRelative: {
6351 HArmDexCacheArraysBase* base =
6352 invoke->InputAt(invoke->GetSpecialInputIndex())->AsArmDexCacheArraysBase();
6353 Register base_reg = GetInvokeStaticOrDirectExtraParameter(invoke,
6354 temp.AsRegister<Register>());
6355 int32_t offset = invoke->GetDexCacheArrayOffset() - base->GetElementOffset();
6356 __ LoadFromOffset(kLoadWord, temp.AsRegister<Register>(), base_reg, offset);
6357 break;
6358 }
6359 case HInvokeStaticOrDirect::MethodLoadKind::kDexCacheViaMethod: {
6360 Location current_method = invoke->GetLocations()->InAt(invoke->GetSpecialInputIndex());
6361 Register method_reg;
6362 Register reg = temp.AsRegister<Register>();
6363 if (current_method.IsRegister()) {
6364 method_reg = current_method.AsRegister<Register>();
6365 } else {
6366 DCHECK(invoke->GetLocations()->Intrinsified());
6367 DCHECK(!current_method.IsValid());
6368 method_reg = reg;
6369 __ LoadFromOffset(kLoadWord, reg, SP, kCurrentMethodStackOffset);
6370 }
6371 // /* ArtMethod*[] */ temp = temp.ptr_sized_fields_->dex_cache_resolved_methods_;
6372 __ LoadFromOffset(kLoadWord,
6373 reg,
6374 method_reg,
6375 ArtMethod::DexCacheResolvedMethodsOffset(kArmPointerSize).Int32Value());
6376 // temp = temp[index_in_cache];
6377 // Note: Don't use invoke->GetTargetMethod() as it may point to a different dex file.
6378 uint32_t index_in_cache = invoke->GetDexMethodIndex();
6379 __ LoadFromOffset(kLoadWord, reg, reg, CodeGenerator::GetCachePointerOffset(index_in_cache));
6380 break;
6381 }
6382 }
6383
6384 switch (invoke->GetCodePtrLocation()) {
6385 case HInvokeStaticOrDirect::CodePtrLocation::kCallSelf:
6386 __ bl(GetFrameEntryLabel());
6387 break;
6388 case HInvokeStaticOrDirect::CodePtrLocation::kCallPCRelative:
6389 relative_call_patches_.emplace_back(invoke->GetTargetMethod());
6390 __ BindTrackedLabel(&relative_call_patches_.back().label);
6391 // Arbitrarily branch to the BL itself, override at link time.
6392 __ bl(&relative_call_patches_.back().label);
6393 break;
6394 case HInvokeStaticOrDirect::CodePtrLocation::kCallDirectWithFixup:
6395 case HInvokeStaticOrDirect::CodePtrLocation::kCallDirect:
6396 // LR prepared above for better instruction scheduling.
6397 // LR()
6398 __ blx(LR);
6399 break;
6400 case HInvokeStaticOrDirect::CodePtrLocation::kCallArtMethod:
6401 // LR = callee_method->entry_point_from_quick_compiled_code_
6402 __ LoadFromOffset(
6403 kLoadWord, LR, callee_method.AsRegister<Register>(),
6404 ArtMethod::EntryPointFromQuickCompiledCodeOffset(kArmWordSize).Int32Value());
6405 // LR()
6406 __ blx(LR);
6407 break;
6408 }
6409
6410 DCHECK(!IsLeafMethod());
6411 }
6412
GenerateVirtualCall(HInvokeVirtual * invoke,Location temp_location)6413 void CodeGeneratorARM::GenerateVirtualCall(HInvokeVirtual* invoke, Location temp_location) {
6414 Register temp = temp_location.AsRegister<Register>();
6415 uint32_t method_offset = mirror::Class::EmbeddedVTableEntryOffset(
6416 invoke->GetVTableIndex(), kArmPointerSize).Uint32Value();
6417
6418 // Use the calling convention instead of the location of the receiver, as
6419 // intrinsics may have put the receiver in a different register. In the intrinsics
6420 // slow path, the arguments have been moved to the right place, so here we are
6421 // guaranteed that the receiver is the first register of the calling convention.
6422 InvokeDexCallingConvention calling_convention;
6423 Register receiver = calling_convention.GetRegisterAt(0);
6424 uint32_t class_offset = mirror::Object::ClassOffset().Int32Value();
6425 // /* HeapReference<Class> */ temp = receiver->klass_
6426 __ LoadFromOffset(kLoadWord, temp, receiver, class_offset);
6427 MaybeRecordImplicitNullCheck(invoke);
6428 // Instead of simply (possibly) unpoisoning `temp` here, we should
6429 // emit a read barrier for the previous class reference load.
6430 // However this is not required in practice, as this is an
6431 // intermediate/temporary reference and because the current
6432 // concurrent copying collector keeps the from-space memory
6433 // intact/accessible until the end of the marking phase (the
6434 // concurrent copying collector may not in the future).
6435 __ MaybeUnpoisonHeapReference(temp);
6436 // temp = temp->GetMethodAt(method_offset);
6437 uint32_t entry_point = ArtMethod::EntryPointFromQuickCompiledCodeOffset(
6438 kArmWordSize).Int32Value();
6439 __ LoadFromOffset(kLoadWord, temp, temp, method_offset);
6440 // LR = temp->GetEntryPoint();
6441 __ LoadFromOffset(kLoadWord, LR, temp, entry_point);
6442 // LR();
6443 __ blx(LR);
6444 }
6445
NewPcRelativeStringPatch(const DexFile & dex_file,uint32_t string_index)6446 CodeGeneratorARM::PcRelativePatchInfo* CodeGeneratorARM::NewPcRelativeStringPatch(
6447 const DexFile& dex_file, uint32_t string_index) {
6448 return NewPcRelativePatch(dex_file, string_index, &pc_relative_string_patches_);
6449 }
6450
NewPcRelativeDexCacheArrayPatch(const DexFile & dex_file,uint32_t element_offset)6451 CodeGeneratorARM::PcRelativePatchInfo* CodeGeneratorARM::NewPcRelativeDexCacheArrayPatch(
6452 const DexFile& dex_file, uint32_t element_offset) {
6453 return NewPcRelativePatch(dex_file, element_offset, &pc_relative_dex_cache_patches_);
6454 }
6455
NewPcRelativePatch(const DexFile & dex_file,uint32_t offset_or_index,ArenaDeque<PcRelativePatchInfo> * patches)6456 CodeGeneratorARM::PcRelativePatchInfo* CodeGeneratorARM::NewPcRelativePatch(
6457 const DexFile& dex_file, uint32_t offset_or_index, ArenaDeque<PcRelativePatchInfo>* patches) {
6458 patches->emplace_back(dex_file, offset_or_index);
6459 return &patches->back();
6460 }
6461
DeduplicateBootImageStringLiteral(const DexFile & dex_file,uint32_t string_index)6462 Literal* CodeGeneratorARM::DeduplicateBootImageStringLiteral(const DexFile& dex_file,
6463 uint32_t string_index) {
6464 return boot_image_string_patches_.GetOrCreate(
6465 StringReference(&dex_file, string_index),
6466 [this]() { return __ NewLiteral<uint32_t>(/* placeholder */ 0u); });
6467 }
6468
DeduplicateBootImageAddressLiteral(uint32_t address)6469 Literal* CodeGeneratorARM::DeduplicateBootImageAddressLiteral(uint32_t address) {
6470 bool needs_patch = GetCompilerOptions().GetIncludePatchInformation();
6471 Uint32ToLiteralMap* map = needs_patch ? &boot_image_address_patches_ : &uint32_literals_;
6472 return DeduplicateUint32Literal(dchecked_integral_cast<uint32_t>(address), map);
6473 }
6474
DeduplicateDexCacheAddressLiteral(uint32_t address)6475 Literal* CodeGeneratorARM::DeduplicateDexCacheAddressLiteral(uint32_t address) {
6476 return DeduplicateUint32Literal(address, &uint32_literals_);
6477 }
6478
EmitLinkerPatches(ArenaVector<LinkerPatch> * linker_patches)6479 void CodeGeneratorARM::EmitLinkerPatches(ArenaVector<LinkerPatch>* linker_patches) {
6480 DCHECK(linker_patches->empty());
6481 size_t size =
6482 method_patches_.size() +
6483 call_patches_.size() +
6484 relative_call_patches_.size() +
6485 /* MOVW+MOVT for each base */ 2u * pc_relative_dex_cache_patches_.size() +
6486 boot_image_string_patches_.size() +
6487 /* MOVW+MOVT for each base */ 2u * pc_relative_string_patches_.size() +
6488 boot_image_address_patches_.size();
6489 linker_patches->reserve(size);
6490 for (const auto& entry : method_patches_) {
6491 const MethodReference& target_method = entry.first;
6492 Literal* literal = entry.second;
6493 DCHECK(literal->GetLabel()->IsBound());
6494 uint32_t literal_offset = literal->GetLabel()->Position();
6495 linker_patches->push_back(LinkerPatch::MethodPatch(literal_offset,
6496 target_method.dex_file,
6497 target_method.dex_method_index));
6498 }
6499 for (const auto& entry : call_patches_) {
6500 const MethodReference& target_method = entry.first;
6501 Literal* literal = entry.second;
6502 DCHECK(literal->GetLabel()->IsBound());
6503 uint32_t literal_offset = literal->GetLabel()->Position();
6504 linker_patches->push_back(LinkerPatch::CodePatch(literal_offset,
6505 target_method.dex_file,
6506 target_method.dex_method_index));
6507 }
6508 for (const MethodPatchInfo<Label>& info : relative_call_patches_) {
6509 uint32_t literal_offset = info.label.Position();
6510 linker_patches->push_back(LinkerPatch::RelativeCodePatch(literal_offset,
6511 info.target_method.dex_file,
6512 info.target_method.dex_method_index));
6513 }
6514 for (const PcRelativePatchInfo& info : pc_relative_dex_cache_patches_) {
6515 const DexFile& dex_file = info.target_dex_file;
6516 size_t base_element_offset = info.offset_or_index;
6517 DCHECK(info.add_pc_label.IsBound());
6518 uint32_t add_pc_offset = dchecked_integral_cast<uint32_t>(info.add_pc_label.Position());
6519 // Add MOVW patch.
6520 DCHECK(info.movw_label.IsBound());
6521 uint32_t movw_offset = dchecked_integral_cast<uint32_t>(info.movw_label.Position());
6522 linker_patches->push_back(LinkerPatch::DexCacheArrayPatch(movw_offset,
6523 &dex_file,
6524 add_pc_offset,
6525 base_element_offset));
6526 // Add MOVT patch.
6527 DCHECK(info.movt_label.IsBound());
6528 uint32_t movt_offset = dchecked_integral_cast<uint32_t>(info.movt_label.Position());
6529 linker_patches->push_back(LinkerPatch::DexCacheArrayPatch(movt_offset,
6530 &dex_file,
6531 add_pc_offset,
6532 base_element_offset));
6533 }
6534 for (const auto& entry : boot_image_string_patches_) {
6535 const StringReference& target_string = entry.first;
6536 Literal* literal = entry.second;
6537 DCHECK(literal->GetLabel()->IsBound());
6538 uint32_t literal_offset = literal->GetLabel()->Position();
6539 linker_patches->push_back(LinkerPatch::StringPatch(literal_offset,
6540 target_string.dex_file,
6541 target_string.string_index));
6542 }
6543 for (const PcRelativePatchInfo& info : pc_relative_string_patches_) {
6544 const DexFile& dex_file = info.target_dex_file;
6545 uint32_t string_index = info.offset_or_index;
6546 DCHECK(info.add_pc_label.IsBound());
6547 uint32_t add_pc_offset = dchecked_integral_cast<uint32_t>(info.add_pc_label.Position());
6548 // Add MOVW patch.
6549 DCHECK(info.movw_label.IsBound());
6550 uint32_t movw_offset = dchecked_integral_cast<uint32_t>(info.movw_label.Position());
6551 linker_patches->push_back(LinkerPatch::RelativeStringPatch(movw_offset,
6552 &dex_file,
6553 add_pc_offset,
6554 string_index));
6555 // Add MOVT patch.
6556 DCHECK(info.movt_label.IsBound());
6557 uint32_t movt_offset = dchecked_integral_cast<uint32_t>(info.movt_label.Position());
6558 linker_patches->push_back(LinkerPatch::RelativeStringPatch(movt_offset,
6559 &dex_file,
6560 add_pc_offset,
6561 string_index));
6562 }
6563 for (const auto& entry : boot_image_address_patches_) {
6564 DCHECK(GetCompilerOptions().GetIncludePatchInformation());
6565 Literal* literal = entry.second;
6566 DCHECK(literal->GetLabel()->IsBound());
6567 uint32_t literal_offset = literal->GetLabel()->Position();
6568 linker_patches->push_back(LinkerPatch::RecordPosition(literal_offset));
6569 }
6570 }
6571
DeduplicateUint32Literal(uint32_t value,Uint32ToLiteralMap * map)6572 Literal* CodeGeneratorARM::DeduplicateUint32Literal(uint32_t value, Uint32ToLiteralMap* map) {
6573 return map->GetOrCreate(
6574 value,
6575 [this, value]() { return __ NewLiteral<uint32_t>(value); });
6576 }
6577
DeduplicateMethodLiteral(MethodReference target_method,MethodToLiteralMap * map)6578 Literal* CodeGeneratorARM::DeduplicateMethodLiteral(MethodReference target_method,
6579 MethodToLiteralMap* map) {
6580 return map->GetOrCreate(
6581 target_method,
6582 [this]() { return __ NewLiteral<uint32_t>(/* placeholder */ 0u); });
6583 }
6584
DeduplicateMethodAddressLiteral(MethodReference target_method)6585 Literal* CodeGeneratorARM::DeduplicateMethodAddressLiteral(MethodReference target_method) {
6586 return DeduplicateMethodLiteral(target_method, &method_patches_);
6587 }
6588
DeduplicateMethodCodeLiteral(MethodReference target_method)6589 Literal* CodeGeneratorARM::DeduplicateMethodCodeLiteral(MethodReference target_method) {
6590 return DeduplicateMethodLiteral(target_method, &call_patches_);
6591 }
6592
VisitMultiplyAccumulate(HMultiplyAccumulate * instr)6593 void LocationsBuilderARM::VisitMultiplyAccumulate(HMultiplyAccumulate* instr) {
6594 LocationSummary* locations =
6595 new (GetGraph()->GetArena()) LocationSummary(instr, LocationSummary::kNoCall);
6596 locations->SetInAt(HMultiplyAccumulate::kInputAccumulatorIndex,
6597 Location::RequiresRegister());
6598 locations->SetInAt(HMultiplyAccumulate::kInputMulLeftIndex, Location::RequiresRegister());
6599 locations->SetInAt(HMultiplyAccumulate::kInputMulRightIndex, Location::RequiresRegister());
6600 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
6601 }
6602
VisitMultiplyAccumulate(HMultiplyAccumulate * instr)6603 void InstructionCodeGeneratorARM::VisitMultiplyAccumulate(HMultiplyAccumulate* instr) {
6604 LocationSummary* locations = instr->GetLocations();
6605 Register res = locations->Out().AsRegister<Register>();
6606 Register accumulator =
6607 locations->InAt(HMultiplyAccumulate::kInputAccumulatorIndex).AsRegister<Register>();
6608 Register mul_left =
6609 locations->InAt(HMultiplyAccumulate::kInputMulLeftIndex).AsRegister<Register>();
6610 Register mul_right =
6611 locations->InAt(HMultiplyAccumulate::kInputMulRightIndex).AsRegister<Register>();
6612
6613 if (instr->GetOpKind() == HInstruction::kAdd) {
6614 __ mla(res, mul_left, mul_right, accumulator);
6615 } else {
6616 __ mls(res, mul_left, mul_right, accumulator);
6617 }
6618 }
6619
VisitBoundType(HBoundType * instruction ATTRIBUTE_UNUSED)6620 void LocationsBuilderARM::VisitBoundType(HBoundType* instruction ATTRIBUTE_UNUSED) {
6621 // Nothing to do, this should be removed during prepare for register allocator.
6622 LOG(FATAL) << "Unreachable";
6623 }
6624
VisitBoundType(HBoundType * instruction ATTRIBUTE_UNUSED)6625 void InstructionCodeGeneratorARM::VisitBoundType(HBoundType* instruction ATTRIBUTE_UNUSED) {
6626 // Nothing to do, this should be removed during prepare for register allocator.
6627 LOG(FATAL) << "Unreachable";
6628 }
6629
6630 // Simple implementation of packed switch - generate cascaded compare/jumps.
VisitPackedSwitch(HPackedSwitch * switch_instr)6631 void LocationsBuilderARM::VisitPackedSwitch(HPackedSwitch* switch_instr) {
6632 LocationSummary* locations =
6633 new (GetGraph()->GetArena()) LocationSummary(switch_instr, LocationSummary::kNoCall);
6634 locations->SetInAt(0, Location::RequiresRegister());
6635 if (switch_instr->GetNumEntries() > kPackedSwitchCompareJumpThreshold &&
6636 codegen_->GetAssembler()->IsThumb()) {
6637 locations->AddTemp(Location::RequiresRegister()); // We need a temp for the table base.
6638 if (switch_instr->GetStartValue() != 0) {
6639 locations->AddTemp(Location::RequiresRegister()); // We need a temp for the bias.
6640 }
6641 }
6642 }
6643
VisitPackedSwitch(HPackedSwitch * switch_instr)6644 void InstructionCodeGeneratorARM::VisitPackedSwitch(HPackedSwitch* switch_instr) {
6645 int32_t lower_bound = switch_instr->GetStartValue();
6646 uint32_t num_entries = switch_instr->GetNumEntries();
6647 LocationSummary* locations = switch_instr->GetLocations();
6648 Register value_reg = locations->InAt(0).AsRegister<Register>();
6649 HBasicBlock* default_block = switch_instr->GetDefaultBlock();
6650
6651 if (num_entries <= kPackedSwitchCompareJumpThreshold || !codegen_->GetAssembler()->IsThumb()) {
6652 // Create a series of compare/jumps.
6653 Register temp_reg = IP;
6654 // Note: It is fine for the below AddConstantSetFlags() using IP register to temporarily store
6655 // the immediate, because IP is used as the destination register. For the other
6656 // AddConstantSetFlags() and GenerateCompareWithImmediate(), the immediate values are constant,
6657 // and they can be encoded in the instruction without making use of IP register.
6658 __ AddConstantSetFlags(temp_reg, value_reg, -lower_bound);
6659
6660 const ArenaVector<HBasicBlock*>& successors = switch_instr->GetBlock()->GetSuccessors();
6661 // Jump to successors[0] if value == lower_bound.
6662 __ b(codegen_->GetLabelOf(successors[0]), EQ);
6663 int32_t last_index = 0;
6664 for (; num_entries - last_index > 2; last_index += 2) {
6665 __ AddConstantSetFlags(temp_reg, temp_reg, -2);
6666 // Jump to successors[last_index + 1] if value < case_value[last_index + 2].
6667 __ b(codegen_->GetLabelOf(successors[last_index + 1]), LO);
6668 // Jump to successors[last_index + 2] if value == case_value[last_index + 2].
6669 __ b(codegen_->GetLabelOf(successors[last_index + 2]), EQ);
6670 }
6671 if (num_entries - last_index == 2) {
6672 // The last missing case_value.
6673 __ CmpConstant(temp_reg, 1);
6674 __ b(codegen_->GetLabelOf(successors[last_index + 1]), EQ);
6675 }
6676
6677 // And the default for any other value.
6678 if (!codegen_->GoesToNextBlock(switch_instr->GetBlock(), default_block)) {
6679 __ b(codegen_->GetLabelOf(default_block));
6680 }
6681 } else {
6682 // Create a table lookup.
6683 Register temp_reg = locations->GetTemp(0).AsRegister<Register>();
6684
6685 // Materialize a pointer to the switch table
6686 std::vector<Label*> labels(num_entries);
6687 const ArenaVector<HBasicBlock*>& successors = switch_instr->GetBlock()->GetSuccessors();
6688 for (uint32_t i = 0; i < num_entries; i++) {
6689 labels[i] = codegen_->GetLabelOf(successors[i]);
6690 }
6691 JumpTable* table = __ CreateJumpTable(std::move(labels), temp_reg);
6692
6693 // Remove the bias.
6694 Register key_reg;
6695 if (lower_bound != 0) {
6696 key_reg = locations->GetTemp(1).AsRegister<Register>();
6697 __ AddConstant(key_reg, value_reg, -lower_bound);
6698 } else {
6699 key_reg = value_reg;
6700 }
6701
6702 // Check whether the value is in the table, jump to default block if not.
6703 __ CmpConstant(key_reg, num_entries - 1);
6704 __ b(codegen_->GetLabelOf(default_block), Condition::HI);
6705
6706 // Load the displacement from the table.
6707 __ ldr(temp_reg, Address(temp_reg, key_reg, Shift::LSL, 2));
6708
6709 // Dispatch is a direct add to the PC (for Thumb2).
6710 __ EmitJumpTableDispatch(table, temp_reg);
6711 }
6712 }
6713
VisitArmDexCacheArraysBase(HArmDexCacheArraysBase * base)6714 void LocationsBuilderARM::VisitArmDexCacheArraysBase(HArmDexCacheArraysBase* base) {
6715 LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(base);
6716 locations->SetOut(Location::RequiresRegister());
6717 }
6718
VisitArmDexCacheArraysBase(HArmDexCacheArraysBase * base)6719 void InstructionCodeGeneratorARM::VisitArmDexCacheArraysBase(HArmDexCacheArraysBase* base) {
6720 Register base_reg = base->GetLocations()->Out().AsRegister<Register>();
6721 CodeGeneratorARM::PcRelativePatchInfo* labels =
6722 codegen_->NewPcRelativeDexCacheArrayPatch(base->GetDexFile(), base->GetElementOffset());
6723 __ BindTrackedLabel(&labels->movw_label);
6724 __ movw(base_reg, /* placeholder */ 0u);
6725 __ BindTrackedLabel(&labels->movt_label);
6726 __ movt(base_reg, /* placeholder */ 0u);
6727 __ BindTrackedLabel(&labels->add_pc_label);
6728 __ add(base_reg, base_reg, ShifterOperand(PC));
6729 }
6730
MoveFromReturnRegister(Location trg,Primitive::Type type)6731 void CodeGeneratorARM::MoveFromReturnRegister(Location trg, Primitive::Type type) {
6732 if (!trg.IsValid()) {
6733 DCHECK_EQ(type, Primitive::kPrimVoid);
6734 return;
6735 }
6736
6737 DCHECK_NE(type, Primitive::kPrimVoid);
6738
6739 Location return_loc = InvokeDexCallingConventionVisitorARM().GetReturnLocation(type);
6740 if (return_loc.Equals(trg)) {
6741 return;
6742 }
6743
6744 // TODO: Consider pairs in the parallel move resolver, then this could be nicely merged
6745 // with the last branch.
6746 if (type == Primitive::kPrimLong) {
6747 HParallelMove parallel_move(GetGraph()->GetArena());
6748 parallel_move.AddMove(return_loc.ToLow(), trg.ToLow(), Primitive::kPrimInt, nullptr);
6749 parallel_move.AddMove(return_loc.ToHigh(), trg.ToHigh(), Primitive::kPrimInt, nullptr);
6750 GetMoveResolver()->EmitNativeCode(¶llel_move);
6751 } else if (type == Primitive::kPrimDouble) {
6752 HParallelMove parallel_move(GetGraph()->GetArena());
6753 parallel_move.AddMove(return_loc.ToLow(), trg.ToLow(), Primitive::kPrimFloat, nullptr);
6754 parallel_move.AddMove(return_loc.ToHigh(), trg.ToHigh(), Primitive::kPrimFloat, nullptr);
6755 GetMoveResolver()->EmitNativeCode(¶llel_move);
6756 } else {
6757 // Let the parallel move resolver take care of all of this.
6758 HParallelMove parallel_move(GetGraph()->GetArena());
6759 parallel_move.AddMove(return_loc, trg, type, nullptr);
6760 GetMoveResolver()->EmitNativeCode(¶llel_move);
6761 }
6762 }
6763
VisitClassTableGet(HClassTableGet * instruction)6764 void LocationsBuilderARM::VisitClassTableGet(HClassTableGet* instruction) {
6765 LocationSummary* locations =
6766 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
6767 locations->SetInAt(0, Location::RequiresRegister());
6768 locations->SetOut(Location::RequiresRegister());
6769 }
6770
VisitClassTableGet(HClassTableGet * instruction)6771 void InstructionCodeGeneratorARM::VisitClassTableGet(HClassTableGet* instruction) {
6772 LocationSummary* locations = instruction->GetLocations();
6773 uint32_t method_offset = 0;
6774 if (instruction->GetTableKind() == HClassTableGet::TableKind::kVTable) {
6775 method_offset = mirror::Class::EmbeddedVTableEntryOffset(
6776 instruction->GetIndex(), kArmPointerSize).SizeValue();
6777 } else {
6778 method_offset = mirror::Class::EmbeddedImTableEntryOffset(
6779 instruction->GetIndex() % mirror::Class::kImtSize, kArmPointerSize).Uint32Value();
6780 }
6781 __ LoadFromOffset(kLoadWord,
6782 locations->Out().AsRegister<Register>(),
6783 locations->InAt(0).AsRegister<Register>(),
6784 method_offset);
6785 }
6786
6787 #undef __
6788 #undef QUICK_ENTRY_POINT
6789
6790 } // namespace arm
6791 } // namespace art
6792