1 /*
2  * Copyright (C) 2014 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 #include "code_generator_x86_64.h"
18 
19 #include "art_method.h"
20 #include "code_generator_utils.h"
21 #include "entrypoints/quick/quick_entrypoints.h"
22 #include "gc/accounting/card_table.h"
23 #include "intrinsics.h"
24 #include "intrinsics_x86_64.h"
25 #include "mirror/array-inl.h"
26 #include "mirror/class-inl.h"
27 #include "mirror/object_reference.h"
28 #include "thread.h"
29 #include "utils/assembler.h"
30 #include "utils/stack_checks.h"
31 #include "utils/x86_64/assembler_x86_64.h"
32 #include "utils/x86_64/managed_register_x86_64.h"
33 
34 namespace art {
35 
36 namespace x86_64 {
37 
38 // Some x86_64 instructions require a register to be available as temp.
39 static constexpr Register TMP = R11;
40 
41 static constexpr int kCurrentMethodStackOffset = 0;
42 
43 static constexpr Register kCoreCalleeSaves[] = { RBX, RBP, R12, R13, R14, R15 };
44 static constexpr FloatRegister kFpuCalleeSaves[] = { XMM12, XMM13, XMM14, XMM15 };
45 
46 static constexpr int kC2ConditionMask = 0x400;
47 
48 
49 #define __ reinterpret_cast<X86_64Assembler*>(codegen->GetAssembler())->
50 
51 class NullCheckSlowPathX86_64 : public SlowPathCodeX86_64 {
52  public:
NullCheckSlowPathX86_64(HNullCheck * instruction)53   explicit NullCheckSlowPathX86_64(HNullCheck* instruction) : instruction_(instruction) {}
54 
EmitNativeCode(CodeGenerator * codegen)55   void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
56     __ Bind(GetEntryLabel());
57     __ gs()->call(
58         Address::Absolute(QUICK_ENTRYPOINT_OFFSET(kX86_64WordSize, pThrowNullPointer), true));
59     RecordPcInfo(codegen, instruction_, instruction_->GetDexPc());
60   }
61 
62  private:
63   HNullCheck* const instruction_;
64   DISALLOW_COPY_AND_ASSIGN(NullCheckSlowPathX86_64);
65 };
66 
67 class DivZeroCheckSlowPathX86_64 : public SlowPathCodeX86_64 {
68  public:
DivZeroCheckSlowPathX86_64(HDivZeroCheck * instruction)69   explicit DivZeroCheckSlowPathX86_64(HDivZeroCheck* instruction) : instruction_(instruction) {}
70 
EmitNativeCode(CodeGenerator * codegen)71   void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
72     __ Bind(GetEntryLabel());
73     __ gs()->call(
74         Address::Absolute(QUICK_ENTRYPOINT_OFFSET(kX86_64WordSize, pThrowDivZero), true));
75     RecordPcInfo(codegen, instruction_, instruction_->GetDexPc());
76   }
77 
78  private:
79   HDivZeroCheck* const instruction_;
80   DISALLOW_COPY_AND_ASSIGN(DivZeroCheckSlowPathX86_64);
81 };
82 
83 class DivRemMinusOneSlowPathX86_64 : public SlowPathCodeX86_64 {
84  public:
DivRemMinusOneSlowPathX86_64(Register reg,Primitive::Type type,bool is_div)85   explicit DivRemMinusOneSlowPathX86_64(Register reg, Primitive::Type type, bool is_div)
86       : cpu_reg_(CpuRegister(reg)), type_(type), is_div_(is_div) {}
87 
EmitNativeCode(CodeGenerator * codegen)88   void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
89     __ Bind(GetEntryLabel());
90     if (type_ == Primitive::kPrimInt) {
91       if (is_div_) {
92         __ negl(cpu_reg_);
93       } else {
94         __ movl(cpu_reg_, Immediate(0));
95       }
96 
97     } else {
98       DCHECK_EQ(Primitive::kPrimLong, type_);
99       if (is_div_) {
100         __ negq(cpu_reg_);
101       } else {
102         __ xorl(cpu_reg_, cpu_reg_);
103       }
104     }
105     __ jmp(GetExitLabel());
106   }
107 
108  private:
109   const CpuRegister cpu_reg_;
110   const Primitive::Type type_;
111   const bool is_div_;
112   DISALLOW_COPY_AND_ASSIGN(DivRemMinusOneSlowPathX86_64);
113 };
114 
115 class SuspendCheckSlowPathX86_64 : public SlowPathCodeX86_64 {
116  public:
SuspendCheckSlowPathX86_64(HSuspendCheck * instruction,HBasicBlock * successor)117   explicit SuspendCheckSlowPathX86_64(HSuspendCheck* instruction, HBasicBlock* successor)
118       : instruction_(instruction), successor_(successor) {}
119 
EmitNativeCode(CodeGenerator * codegen)120   void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
121     CodeGeneratorX86_64* x64_codegen = down_cast<CodeGeneratorX86_64*>(codegen);
122     __ Bind(GetEntryLabel());
123     SaveLiveRegisters(codegen, instruction_->GetLocations());
124     __ gs()->call(Address::Absolute(QUICK_ENTRYPOINT_OFFSET(kX86_64WordSize, pTestSuspend), true));
125     RecordPcInfo(codegen, instruction_, instruction_->GetDexPc());
126     RestoreLiveRegisters(codegen, instruction_->GetLocations());
127     if (successor_ == nullptr) {
128       __ jmp(GetReturnLabel());
129     } else {
130       __ jmp(x64_codegen->GetLabelOf(successor_));
131     }
132   }
133 
GetReturnLabel()134   Label* GetReturnLabel() {
135     DCHECK(successor_ == nullptr);
136     return &return_label_;
137   }
138 
GetSuccessor() const139   HBasicBlock* GetSuccessor() const {
140     return successor_;
141   }
142 
143  private:
144   HSuspendCheck* const instruction_;
145   HBasicBlock* const successor_;
146   Label return_label_;
147 
148   DISALLOW_COPY_AND_ASSIGN(SuspendCheckSlowPathX86_64);
149 };
150 
151 class BoundsCheckSlowPathX86_64 : public SlowPathCodeX86_64 {
152  public:
BoundsCheckSlowPathX86_64(HBoundsCheck * instruction,Location index_location,Location length_location)153   BoundsCheckSlowPathX86_64(HBoundsCheck* instruction,
154                             Location index_location,
155                             Location length_location)
156       : instruction_(instruction),
157         index_location_(index_location),
158         length_location_(length_location) {}
159 
EmitNativeCode(CodeGenerator * codegen)160   void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
161     __ Bind(GetEntryLabel());
162     // We're moving two locations to locations that could overlap, so we need a parallel
163     // move resolver.
164     InvokeRuntimeCallingConvention calling_convention;
165     codegen->EmitParallelMoves(
166         index_location_,
167         Location::RegisterLocation(calling_convention.GetRegisterAt(0)),
168         Primitive::kPrimInt,
169         length_location_,
170         Location::RegisterLocation(calling_convention.GetRegisterAt(1)),
171         Primitive::kPrimInt);
172     __ gs()->call(Address::Absolute(
173         QUICK_ENTRYPOINT_OFFSET(kX86_64WordSize, pThrowArrayBounds), true));
174     RecordPcInfo(codegen, instruction_, instruction_->GetDexPc());
175   }
176 
177  private:
178   HBoundsCheck* const instruction_;
179   const Location index_location_;
180   const Location length_location_;
181 
182   DISALLOW_COPY_AND_ASSIGN(BoundsCheckSlowPathX86_64);
183 };
184 
185 class LoadClassSlowPathX86_64 : public SlowPathCodeX86_64 {
186  public:
LoadClassSlowPathX86_64(HLoadClass * cls,HInstruction * at,uint32_t dex_pc,bool do_clinit)187   LoadClassSlowPathX86_64(HLoadClass* cls,
188                           HInstruction* at,
189                           uint32_t dex_pc,
190                           bool do_clinit)
191       : cls_(cls), at_(at), dex_pc_(dex_pc), do_clinit_(do_clinit) {
192     DCHECK(at->IsLoadClass() || at->IsClinitCheck());
193   }
194 
EmitNativeCode(CodeGenerator * codegen)195   void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
196     LocationSummary* locations = at_->GetLocations();
197     CodeGeneratorX86_64* x64_codegen = down_cast<CodeGeneratorX86_64*>(codegen);
198     __ Bind(GetEntryLabel());
199 
200     SaveLiveRegisters(codegen, locations);
201 
202     InvokeRuntimeCallingConvention calling_convention;
203     __ movl(CpuRegister(calling_convention.GetRegisterAt(0)), Immediate(cls_->GetTypeIndex()));
204     __ gs()->call(Address::Absolute((do_clinit_
205           ? QUICK_ENTRYPOINT_OFFSET(kX86_64WordSize, pInitializeStaticStorage)
206           : QUICK_ENTRYPOINT_OFFSET(kX86_64WordSize, pInitializeType)) , true));
207     RecordPcInfo(codegen, at_, dex_pc_);
208 
209     Location out = locations->Out();
210     // Move the class to the desired location.
211     if (out.IsValid()) {
212       DCHECK(out.IsRegister() && !locations->GetLiveRegisters()->ContainsCoreRegister(out.reg()));
213       x64_codegen->Move(out, Location::RegisterLocation(RAX));
214     }
215 
216     RestoreLiveRegisters(codegen, locations);
217     __ jmp(GetExitLabel());
218   }
219 
220  private:
221   // The class this slow path will load.
222   HLoadClass* const cls_;
223 
224   // The instruction where this slow path is happening.
225   // (Might be the load class or an initialization check).
226   HInstruction* const at_;
227 
228   // The dex PC of `at_`.
229   const uint32_t dex_pc_;
230 
231   // Whether to initialize the class.
232   const bool do_clinit_;
233 
234   DISALLOW_COPY_AND_ASSIGN(LoadClassSlowPathX86_64);
235 };
236 
237 class LoadStringSlowPathX86_64 : public SlowPathCodeX86_64 {
238  public:
LoadStringSlowPathX86_64(HLoadString * instruction)239   explicit LoadStringSlowPathX86_64(HLoadString* instruction) : instruction_(instruction) {}
240 
EmitNativeCode(CodeGenerator * codegen)241   void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
242     LocationSummary* locations = instruction_->GetLocations();
243     DCHECK(!locations->GetLiveRegisters()->ContainsCoreRegister(locations->Out().reg()));
244 
245     CodeGeneratorX86_64* x64_codegen = down_cast<CodeGeneratorX86_64*>(codegen);
246     __ Bind(GetEntryLabel());
247     SaveLiveRegisters(codegen, locations);
248 
249     InvokeRuntimeCallingConvention calling_convention;
250     __ movl(CpuRegister(calling_convention.GetRegisterAt(0)),
251             Immediate(instruction_->GetStringIndex()));
252     __ gs()->call(Address::Absolute(
253         QUICK_ENTRYPOINT_OFFSET(kX86_64WordSize, pResolveString), true));
254     RecordPcInfo(codegen, instruction_, instruction_->GetDexPc());
255     x64_codegen->Move(locations->Out(), Location::RegisterLocation(RAX));
256     RestoreLiveRegisters(codegen, locations);
257     __ jmp(GetExitLabel());
258   }
259 
260  private:
261   HLoadString* const instruction_;
262 
263   DISALLOW_COPY_AND_ASSIGN(LoadStringSlowPathX86_64);
264 };
265 
266 class TypeCheckSlowPathX86_64 : public SlowPathCodeX86_64 {
267  public:
TypeCheckSlowPathX86_64(HInstruction * instruction,Location class_to_check,Location object_class,uint32_t dex_pc)268   TypeCheckSlowPathX86_64(HInstruction* instruction,
269                           Location class_to_check,
270                           Location object_class,
271                           uint32_t dex_pc)
272       : instruction_(instruction),
273         class_to_check_(class_to_check),
274         object_class_(object_class),
275         dex_pc_(dex_pc) {}
276 
EmitNativeCode(CodeGenerator * codegen)277   void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
278     LocationSummary* locations = instruction_->GetLocations();
279     DCHECK(instruction_->IsCheckCast()
280            || !locations->GetLiveRegisters()->ContainsCoreRegister(locations->Out().reg()));
281 
282     CodeGeneratorX86_64* x64_codegen = down_cast<CodeGeneratorX86_64*>(codegen);
283     __ Bind(GetEntryLabel());
284     SaveLiveRegisters(codegen, locations);
285 
286     // We're moving two locations to locations that could overlap, so we need a parallel
287     // move resolver.
288     InvokeRuntimeCallingConvention calling_convention;
289     codegen->EmitParallelMoves(
290         class_to_check_,
291         Location::RegisterLocation(calling_convention.GetRegisterAt(0)),
292         Primitive::kPrimNot,
293         object_class_,
294         Location::RegisterLocation(calling_convention.GetRegisterAt(1)),
295         Primitive::kPrimNot);
296 
297     if (instruction_->IsInstanceOf()) {
298       __ gs()->call(
299           Address::Absolute(QUICK_ENTRYPOINT_OFFSET(kX86_64WordSize, pInstanceofNonTrivial), true));
300     } else {
301       DCHECK(instruction_->IsCheckCast());
302       __ gs()->call(
303           Address::Absolute(QUICK_ENTRYPOINT_OFFSET(kX86_64WordSize, pCheckCast), true));
304     }
305     RecordPcInfo(codegen, instruction_, dex_pc_);
306 
307     if (instruction_->IsInstanceOf()) {
308       x64_codegen->Move(locations->Out(), Location::RegisterLocation(RAX));
309     }
310 
311     RestoreLiveRegisters(codegen, locations);
312     __ jmp(GetExitLabel());
313   }
314 
315  private:
316   HInstruction* const instruction_;
317   const Location class_to_check_;
318   const Location object_class_;
319   const uint32_t dex_pc_;
320 
321   DISALLOW_COPY_AND_ASSIGN(TypeCheckSlowPathX86_64);
322 };
323 
324 class DeoptimizationSlowPathX86_64 : public SlowPathCodeX86_64 {
325  public:
DeoptimizationSlowPathX86_64(HInstruction * instruction)326   explicit DeoptimizationSlowPathX86_64(HInstruction* instruction)
327       : instruction_(instruction) {}
328 
EmitNativeCode(CodeGenerator * codegen)329   void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
330     __ Bind(GetEntryLabel());
331     SaveLiveRegisters(codegen, instruction_->GetLocations());
332     __ gs()->call(
333         Address::Absolute(QUICK_ENTRYPOINT_OFFSET(kX86_64WordSize, pDeoptimize), true));
334     DCHECK(instruction_->IsDeoptimize());
335     HDeoptimize* deoptimize = instruction_->AsDeoptimize();
336     uint32_t dex_pc = deoptimize->GetDexPc();
337     codegen->RecordPcInfo(instruction_, dex_pc, this);
338   }
339 
340  private:
341   HInstruction* const instruction_;
342   DISALLOW_COPY_AND_ASSIGN(DeoptimizationSlowPathX86_64);
343 };
344 
345 #undef __
346 #define __ reinterpret_cast<X86_64Assembler*>(GetAssembler())->
347 
X86_64Condition(IfCondition cond)348 inline Condition X86_64Condition(IfCondition cond) {
349   switch (cond) {
350     case kCondEQ: return kEqual;
351     case kCondNE: return kNotEqual;
352     case kCondLT: return kLess;
353     case kCondLE: return kLessEqual;
354     case kCondGT: return kGreater;
355     case kCondGE: return kGreaterEqual;
356     default:
357       LOG(FATAL) << "Unknown if condition";
358   }
359   return kEqual;
360 }
361 
GenerateStaticOrDirectCall(HInvokeStaticOrDirect * invoke,CpuRegister temp)362 void CodeGeneratorX86_64::GenerateStaticOrDirectCall(HInvokeStaticOrDirect* invoke,
363                                                      CpuRegister temp) {
364   // All registers are assumed to be correctly set up.
365 
366   // TODO: Implement all kinds of calls:
367   // 1) boot -> boot
368   // 2) app -> boot
369   // 3) app -> app
370   //
371   // Currently we implement the app -> app logic, which looks up in the resolve cache.
372 
373   if (invoke->IsStringInit()) {
374     // temp = thread->string_init_entrypoint
375     __ gs()->movq(temp, Address::Absolute(invoke->GetStringInitOffset(), true));
376     // (temp + offset_of_quick_compiled_code)()
377     __ call(Address(temp, ArtMethod::EntryPointFromQuickCompiledCodeOffset(
378         kX86_64WordSize).SizeValue()));
379   } else {
380     // temp = method;
381     LoadCurrentMethod(temp);
382     if (!invoke->IsRecursive()) {
383       // temp = temp->dex_cache_resolved_methods_;
384       __ movl(temp, Address(temp, ArtMethod::DexCacheResolvedMethodsOffset().SizeValue()));
385       // temp = temp[index_in_cache]
386       __ movq(temp, Address(
387           temp, CodeGenerator::GetCachePointerOffset(invoke->GetDexMethodIndex())));
388       // (temp + offset_of_quick_compiled_code)()
389       __ call(Address(temp, ArtMethod::EntryPointFromQuickCompiledCodeOffset(
390           kX86_64WordSize).SizeValue()));
391     } else {
392       __ call(&frame_entry_label_);
393     }
394   }
395 
396   DCHECK(!IsLeafMethod());
397 }
398 
DumpCoreRegister(std::ostream & stream,int reg) const399 void CodeGeneratorX86_64::DumpCoreRegister(std::ostream& stream, int reg) const {
400   stream << X86_64ManagedRegister::FromCpuRegister(Register(reg));
401 }
402 
DumpFloatingPointRegister(std::ostream & stream,int reg) const403 void CodeGeneratorX86_64::DumpFloatingPointRegister(std::ostream& stream, int reg) const {
404   stream << X86_64ManagedRegister::FromXmmRegister(FloatRegister(reg));
405 }
406 
SaveCoreRegister(size_t stack_index,uint32_t reg_id)407 size_t CodeGeneratorX86_64::SaveCoreRegister(size_t stack_index, uint32_t reg_id) {
408   __ movq(Address(CpuRegister(RSP), stack_index), CpuRegister(reg_id));
409   return kX86_64WordSize;
410 }
411 
RestoreCoreRegister(size_t stack_index,uint32_t reg_id)412 size_t CodeGeneratorX86_64::RestoreCoreRegister(size_t stack_index, uint32_t reg_id) {
413   __ movq(CpuRegister(reg_id), Address(CpuRegister(RSP), stack_index));
414   return kX86_64WordSize;
415 }
416 
SaveFloatingPointRegister(size_t stack_index,uint32_t reg_id)417 size_t CodeGeneratorX86_64::SaveFloatingPointRegister(size_t stack_index, uint32_t reg_id) {
418   __ movsd(Address(CpuRegister(RSP), stack_index), XmmRegister(reg_id));
419   return kX86_64WordSize;
420 }
421 
RestoreFloatingPointRegister(size_t stack_index,uint32_t reg_id)422 size_t CodeGeneratorX86_64::RestoreFloatingPointRegister(size_t stack_index, uint32_t reg_id) {
423   __ movsd(XmmRegister(reg_id), Address(CpuRegister(RSP), stack_index));
424   return kX86_64WordSize;
425 }
426 
427 static constexpr int kNumberOfCpuRegisterPairs = 0;
428 // Use a fake return address register to mimic Quick.
429 static constexpr Register kFakeReturnRegister = Register(kLastCpuRegister + 1);
CodeGeneratorX86_64(HGraph * graph,const X86_64InstructionSetFeatures & isa_features,const CompilerOptions & compiler_options)430 CodeGeneratorX86_64::CodeGeneratorX86_64(HGraph* graph,
431                 const X86_64InstructionSetFeatures& isa_features,
432                 const CompilerOptions& compiler_options)
433       : CodeGenerator(graph,
434                       kNumberOfCpuRegisters,
435                       kNumberOfFloatRegisters,
436                       kNumberOfCpuRegisterPairs,
437                       ComputeRegisterMask(reinterpret_cast<const int*>(kCoreCalleeSaves),
438                                           arraysize(kCoreCalleeSaves))
439                           | (1 << kFakeReturnRegister),
440                       ComputeRegisterMask(reinterpret_cast<const int*>(kFpuCalleeSaves),
441                                           arraysize(kFpuCalleeSaves)),
442                       compiler_options),
443         block_labels_(graph->GetArena(), 0),
444         location_builder_(graph, this),
445         instruction_visitor_(graph, this),
446         move_resolver_(graph->GetArena(), this),
447         isa_features_(isa_features),
448         constant_area_start_(0) {
449   AddAllocatedRegister(Location::RegisterLocation(kFakeReturnRegister));
450 }
451 
InstructionCodeGeneratorX86_64(HGraph * graph,CodeGeneratorX86_64 * codegen)452 InstructionCodeGeneratorX86_64::InstructionCodeGeneratorX86_64(HGraph* graph,
453                                                                CodeGeneratorX86_64* codegen)
454       : HGraphVisitor(graph),
455         assembler_(codegen->GetAssembler()),
456         codegen_(codegen) {}
457 
AllocateFreeRegister(Primitive::Type type) const458 Location CodeGeneratorX86_64::AllocateFreeRegister(Primitive::Type type) const {
459   switch (type) {
460     case Primitive::kPrimLong:
461     case Primitive::kPrimByte:
462     case Primitive::kPrimBoolean:
463     case Primitive::kPrimChar:
464     case Primitive::kPrimShort:
465     case Primitive::kPrimInt:
466     case Primitive::kPrimNot: {
467       size_t reg = FindFreeEntry(blocked_core_registers_, kNumberOfCpuRegisters);
468       return Location::RegisterLocation(reg);
469     }
470 
471     case Primitive::kPrimFloat:
472     case Primitive::kPrimDouble: {
473       size_t reg = FindFreeEntry(blocked_fpu_registers_, kNumberOfFloatRegisters);
474       return Location::FpuRegisterLocation(reg);
475     }
476 
477     case Primitive::kPrimVoid:
478       LOG(FATAL) << "Unreachable type " << type;
479   }
480 
481   return Location();
482 }
483 
SetupBlockedRegisters(bool is_baseline) const484 void CodeGeneratorX86_64::SetupBlockedRegisters(bool is_baseline) const {
485   // Stack register is always reserved.
486   blocked_core_registers_[RSP] = true;
487 
488   // Block the register used as TMP.
489   blocked_core_registers_[TMP] = true;
490 
491   if (is_baseline) {
492     for (size_t i = 0; i < arraysize(kCoreCalleeSaves); ++i) {
493       blocked_core_registers_[kCoreCalleeSaves[i]] = true;
494     }
495     for (size_t i = 0; i < arraysize(kFpuCalleeSaves); ++i) {
496       blocked_fpu_registers_[kFpuCalleeSaves[i]] = true;
497     }
498   }
499 }
500 
DWARFReg(Register reg)501 static dwarf::Reg DWARFReg(Register reg) {
502   return dwarf::Reg::X86_64Core(static_cast<int>(reg));
503 }
504 
DWARFReg(FloatRegister reg)505 static dwarf::Reg DWARFReg(FloatRegister reg) {
506   return dwarf::Reg::X86_64Fp(static_cast<int>(reg));
507 }
508 
GenerateFrameEntry()509 void CodeGeneratorX86_64::GenerateFrameEntry() {
510   __ cfi().SetCurrentCFAOffset(kX86_64WordSize);  // return address
511   __ Bind(&frame_entry_label_);
512   bool skip_overflow_check = IsLeafMethod()
513       && !FrameNeedsStackCheck(GetFrameSize(), InstructionSet::kX86_64);
514   DCHECK(GetCompilerOptions().GetImplicitStackOverflowChecks());
515 
516   if (!skip_overflow_check) {
517     __ testq(CpuRegister(RAX), Address(
518         CpuRegister(RSP), -static_cast<int32_t>(GetStackOverflowReservedBytes(kX86_64))));
519     RecordPcInfo(nullptr, 0);
520   }
521 
522   if (HasEmptyFrame()) {
523     return;
524   }
525 
526   for (int i = arraysize(kCoreCalleeSaves) - 1; i >= 0; --i) {
527     Register reg = kCoreCalleeSaves[i];
528     if (allocated_registers_.ContainsCoreRegister(reg)) {
529       __ pushq(CpuRegister(reg));
530       __ cfi().AdjustCFAOffset(kX86_64WordSize);
531       __ cfi().RelOffset(DWARFReg(reg), 0);
532     }
533   }
534 
535   int adjust = GetFrameSize() - GetCoreSpillSize();
536   __ subq(CpuRegister(RSP), Immediate(adjust));
537   __ cfi().AdjustCFAOffset(adjust);
538   uint32_t xmm_spill_location = GetFpuSpillStart();
539   size_t xmm_spill_slot_size = GetFloatingPointSpillSlotSize();
540 
541   for (int i = arraysize(kFpuCalleeSaves) - 1; i >= 0; --i) {
542     if (allocated_registers_.ContainsFloatingPointRegister(kFpuCalleeSaves[i])) {
543       int offset = xmm_spill_location + (xmm_spill_slot_size * i);
544       __ movsd(Address(CpuRegister(RSP), offset), XmmRegister(kFpuCalleeSaves[i]));
545       __ cfi().RelOffset(DWARFReg(kFpuCalleeSaves[i]), offset);
546     }
547   }
548 
549   __ movq(Address(CpuRegister(RSP), kCurrentMethodStackOffset), CpuRegister(RDI));
550 }
551 
GenerateFrameExit()552 void CodeGeneratorX86_64::GenerateFrameExit() {
553   __ cfi().RememberState();
554   if (!HasEmptyFrame()) {
555     uint32_t xmm_spill_location = GetFpuSpillStart();
556     size_t xmm_spill_slot_size = GetFloatingPointSpillSlotSize();
557     for (size_t i = 0; i < arraysize(kFpuCalleeSaves); ++i) {
558       if (allocated_registers_.ContainsFloatingPointRegister(kFpuCalleeSaves[i])) {
559         int offset = xmm_spill_location + (xmm_spill_slot_size * i);
560         __ movsd(XmmRegister(kFpuCalleeSaves[i]), Address(CpuRegister(RSP), offset));
561         __ cfi().Restore(DWARFReg(kFpuCalleeSaves[i]));
562       }
563     }
564 
565     int adjust = GetFrameSize() - GetCoreSpillSize();
566     __ addq(CpuRegister(RSP), Immediate(adjust));
567     __ cfi().AdjustCFAOffset(-adjust);
568 
569     for (size_t i = 0; i < arraysize(kCoreCalleeSaves); ++i) {
570       Register reg = kCoreCalleeSaves[i];
571       if (allocated_registers_.ContainsCoreRegister(reg)) {
572         __ popq(CpuRegister(reg));
573         __ cfi().AdjustCFAOffset(-static_cast<int>(kX86_64WordSize));
574         __ cfi().Restore(DWARFReg(reg));
575       }
576     }
577   }
578   __ ret();
579   __ cfi().RestoreState();
580   __ cfi().DefCFAOffset(GetFrameSize());
581 }
582 
Bind(HBasicBlock * block)583 void CodeGeneratorX86_64::Bind(HBasicBlock* block) {
584   __ Bind(GetLabelOf(block));
585 }
586 
LoadCurrentMethod(CpuRegister reg)587 void CodeGeneratorX86_64::LoadCurrentMethod(CpuRegister reg) {
588   DCHECK(RequiresCurrentMethod());
589   __ movq(reg, Address(CpuRegister(RSP), kCurrentMethodStackOffset));
590 }
591 
GetStackLocation(HLoadLocal * load) const592 Location CodeGeneratorX86_64::GetStackLocation(HLoadLocal* load) const {
593   switch (load->GetType()) {
594     case Primitive::kPrimLong:
595     case Primitive::kPrimDouble:
596       return Location::DoubleStackSlot(GetStackSlot(load->GetLocal()));
597 
598     case Primitive::kPrimInt:
599     case Primitive::kPrimNot:
600     case Primitive::kPrimFloat:
601       return Location::StackSlot(GetStackSlot(load->GetLocal()));
602 
603     case Primitive::kPrimBoolean:
604     case Primitive::kPrimByte:
605     case Primitive::kPrimChar:
606     case Primitive::kPrimShort:
607     case Primitive::kPrimVoid:
608       LOG(FATAL) << "Unexpected type " << load->GetType();
609       UNREACHABLE();
610   }
611 
612   LOG(FATAL) << "Unreachable";
613   UNREACHABLE();
614 }
615 
Move(Location destination,Location source)616 void CodeGeneratorX86_64::Move(Location destination, Location source) {
617   if (source.Equals(destination)) {
618     return;
619   }
620   if (destination.IsRegister()) {
621     if (source.IsRegister()) {
622       __ movq(destination.AsRegister<CpuRegister>(), source.AsRegister<CpuRegister>());
623     } else if (source.IsFpuRegister()) {
624       __ movd(destination.AsRegister<CpuRegister>(), source.AsFpuRegister<XmmRegister>());
625     } else if (source.IsStackSlot()) {
626       __ movl(destination.AsRegister<CpuRegister>(),
627               Address(CpuRegister(RSP), source.GetStackIndex()));
628     } else {
629       DCHECK(source.IsDoubleStackSlot());
630       __ movq(destination.AsRegister<CpuRegister>(),
631               Address(CpuRegister(RSP), source.GetStackIndex()));
632     }
633   } else if (destination.IsFpuRegister()) {
634     if (source.IsRegister()) {
635       __ movd(destination.AsFpuRegister<XmmRegister>(), source.AsRegister<CpuRegister>());
636     } else if (source.IsFpuRegister()) {
637       __ movaps(destination.AsFpuRegister<XmmRegister>(), source.AsFpuRegister<XmmRegister>());
638     } else if (source.IsStackSlot()) {
639       __ movss(destination.AsFpuRegister<XmmRegister>(),
640               Address(CpuRegister(RSP), source.GetStackIndex()));
641     } else {
642       DCHECK(source.IsDoubleStackSlot());
643       __ movsd(destination.AsFpuRegister<XmmRegister>(),
644                Address(CpuRegister(RSP), source.GetStackIndex()));
645     }
646   } else if (destination.IsStackSlot()) {
647     if (source.IsRegister()) {
648       __ movl(Address(CpuRegister(RSP), destination.GetStackIndex()),
649               source.AsRegister<CpuRegister>());
650     } else if (source.IsFpuRegister()) {
651       __ movss(Address(CpuRegister(RSP), destination.GetStackIndex()),
652                source.AsFpuRegister<XmmRegister>());
653     } else if (source.IsConstant()) {
654       HConstant* constant = source.GetConstant();
655       int32_t value = GetInt32ValueOf(constant);
656       __ movl(Address(CpuRegister(RSP), destination.GetStackIndex()), Immediate(value));
657     } else {
658       DCHECK(source.IsStackSlot()) << source;
659       __ movl(CpuRegister(TMP), Address(CpuRegister(RSP), source.GetStackIndex()));
660       __ movl(Address(CpuRegister(RSP), destination.GetStackIndex()), CpuRegister(TMP));
661     }
662   } else {
663     DCHECK(destination.IsDoubleStackSlot());
664     if (source.IsRegister()) {
665       __ movq(Address(CpuRegister(RSP), destination.GetStackIndex()),
666               source.AsRegister<CpuRegister>());
667     } else if (source.IsFpuRegister()) {
668       __ movsd(Address(CpuRegister(RSP), destination.GetStackIndex()),
669                source.AsFpuRegister<XmmRegister>());
670     } else if (source.IsConstant()) {
671       HConstant* constant = source.GetConstant();
672       int64_t value;
673       if (constant->IsDoubleConstant()) {
674         value = bit_cast<int64_t, double>(constant->AsDoubleConstant()->GetValue());
675       } else {
676         DCHECK(constant->IsLongConstant());
677         value = constant->AsLongConstant()->GetValue();
678       }
679       Load64BitValue(CpuRegister(TMP), value);
680       __ movq(Address(CpuRegister(RSP), destination.GetStackIndex()), CpuRegister(TMP));
681     } else {
682       DCHECK(source.IsDoubleStackSlot());
683       __ movq(CpuRegister(TMP), Address(CpuRegister(RSP), source.GetStackIndex()));
684       __ movq(Address(CpuRegister(RSP), destination.GetStackIndex()), CpuRegister(TMP));
685     }
686   }
687 }
688 
Move(HInstruction * instruction,Location location,HInstruction * move_for)689 void CodeGeneratorX86_64::Move(HInstruction* instruction,
690                                Location location,
691                                HInstruction* move_for) {
692   LocationSummary* locations = instruction->GetLocations();
693   if (locations != nullptr && locations->Out().Equals(location)) {
694     return;
695   }
696 
697   if (locations != nullptr && locations->Out().IsConstant()) {
698     HConstant* const_to_move = locations->Out().GetConstant();
699     if (const_to_move->IsIntConstant() || const_to_move->IsNullConstant()) {
700       Immediate imm(GetInt32ValueOf(const_to_move));
701       if (location.IsRegister()) {
702         __ movl(location.AsRegister<CpuRegister>(), imm);
703       } else if (location.IsStackSlot()) {
704         __ movl(Address(CpuRegister(RSP), location.GetStackIndex()), imm);
705       } else {
706         DCHECK(location.IsConstant());
707         DCHECK_EQ(location.GetConstant(), const_to_move);
708       }
709     } else if (const_to_move->IsLongConstant()) {
710       int64_t value = const_to_move->AsLongConstant()->GetValue();
711       if (location.IsRegister()) {
712         Load64BitValue(location.AsRegister<CpuRegister>(), value);
713       } else if (location.IsDoubleStackSlot()) {
714         Load64BitValue(CpuRegister(TMP), value);
715         __ movq(Address(CpuRegister(RSP), location.GetStackIndex()), CpuRegister(TMP));
716       } else {
717         DCHECK(location.IsConstant());
718         DCHECK_EQ(location.GetConstant(), const_to_move);
719       }
720     }
721   } else if (instruction->IsLoadLocal()) {
722     switch (instruction->GetType()) {
723       case Primitive::kPrimBoolean:
724       case Primitive::kPrimByte:
725       case Primitive::kPrimChar:
726       case Primitive::kPrimShort:
727       case Primitive::kPrimInt:
728       case Primitive::kPrimNot:
729       case Primitive::kPrimFloat:
730         Move(location, Location::StackSlot(GetStackSlot(instruction->AsLoadLocal()->GetLocal())));
731         break;
732 
733       case Primitive::kPrimLong:
734       case Primitive::kPrimDouble:
735         Move(location,
736              Location::DoubleStackSlot(GetStackSlot(instruction->AsLoadLocal()->GetLocal())));
737         break;
738 
739       default:
740         LOG(FATAL) << "Unexpected local type " << instruction->GetType();
741     }
742   } else if (instruction->IsTemporary()) {
743     Location temp_location = GetTemporaryLocation(instruction->AsTemporary());
744     Move(location, temp_location);
745   } else {
746     DCHECK((instruction->GetNext() == move_for) || instruction->GetNext()->IsTemporary());
747     switch (instruction->GetType()) {
748       case Primitive::kPrimBoolean:
749       case Primitive::kPrimByte:
750       case Primitive::kPrimChar:
751       case Primitive::kPrimShort:
752       case Primitive::kPrimInt:
753       case Primitive::kPrimNot:
754       case Primitive::kPrimLong:
755       case Primitive::kPrimFloat:
756       case Primitive::kPrimDouble:
757         Move(location, locations->Out());
758         break;
759 
760       default:
761         LOG(FATAL) << "Unexpected type " << instruction->GetType();
762     }
763   }
764 }
765 
VisitGoto(HGoto * got)766 void LocationsBuilderX86_64::VisitGoto(HGoto* got) {
767   got->SetLocations(nullptr);
768 }
769 
VisitGoto(HGoto * got)770 void InstructionCodeGeneratorX86_64::VisitGoto(HGoto* got) {
771   HBasicBlock* successor = got->GetSuccessor();
772   DCHECK(!successor->IsExitBlock());
773 
774   HBasicBlock* block = got->GetBlock();
775   HInstruction* previous = got->GetPrevious();
776 
777   HLoopInformation* info = block->GetLoopInformation();
778   if (info != nullptr && info->IsBackEdge(*block) && info->HasSuspendCheck()) {
779     GenerateSuspendCheck(info->GetSuspendCheck(), successor);
780     return;
781   }
782 
783   if (block->IsEntryBlock() && (previous != nullptr) && previous->IsSuspendCheck()) {
784     GenerateSuspendCheck(previous->AsSuspendCheck(), nullptr);
785   }
786   if (!codegen_->GoesToNextBlock(got->GetBlock(), successor)) {
787     __ jmp(codegen_->GetLabelOf(successor));
788   }
789 }
790 
VisitExit(HExit * exit)791 void LocationsBuilderX86_64::VisitExit(HExit* exit) {
792   exit->SetLocations(nullptr);
793 }
794 
VisitExit(HExit * exit)795 void InstructionCodeGeneratorX86_64::VisitExit(HExit* exit) {
796   UNUSED(exit);
797 }
798 
GenerateTestAndBranch(HInstruction * instruction,Label * true_target,Label * false_target,Label * always_true_target)799 void InstructionCodeGeneratorX86_64::GenerateTestAndBranch(HInstruction* instruction,
800                                                            Label* true_target,
801                                                            Label* false_target,
802                                                            Label* always_true_target) {
803   HInstruction* cond = instruction->InputAt(0);
804   if (cond->IsIntConstant()) {
805     // Constant condition, statically compared against 1.
806     int32_t cond_value = cond->AsIntConstant()->GetValue();
807     if (cond_value == 1) {
808       if (always_true_target != nullptr) {
809         __ jmp(always_true_target);
810       }
811       return;
812     } else {
813       DCHECK_EQ(cond_value, 0);
814     }
815   } else {
816     bool materialized =
817         !cond->IsCondition() || cond->AsCondition()->NeedsMaterialization();
818     // Moves do not affect the eflags register, so if the condition is
819     // evaluated just before the if, we don't need to evaluate it
820     // again.
821     bool eflags_set = cond->IsCondition()
822         && cond->AsCondition()->IsBeforeWhenDisregardMoves(instruction);
823     if (materialized) {
824       if (!eflags_set) {
825         // Materialized condition, compare against 0.
826         Location lhs = instruction->GetLocations()->InAt(0);
827         if (lhs.IsRegister()) {
828           __ testl(lhs.AsRegister<CpuRegister>(), lhs.AsRegister<CpuRegister>());
829         } else {
830           __ cmpl(Address(CpuRegister(RSP), lhs.GetStackIndex()),
831                   Immediate(0));
832         }
833         __ j(kNotEqual, true_target);
834       } else {
835         __ j(X86_64Condition(cond->AsCondition()->GetCondition()), true_target);
836       }
837     } else {
838       Location lhs = cond->GetLocations()->InAt(0);
839       Location rhs = cond->GetLocations()->InAt(1);
840       if (rhs.IsRegister()) {
841         __ cmpl(lhs.AsRegister<CpuRegister>(), rhs.AsRegister<CpuRegister>());
842       } else if (rhs.IsConstant()) {
843         int32_t constant = CodeGenerator::GetInt32ValueOf(rhs.GetConstant());
844         if (constant == 0) {
845           __ testl(lhs.AsRegister<CpuRegister>(), lhs.AsRegister<CpuRegister>());
846         } else {
847           __ cmpl(lhs.AsRegister<CpuRegister>(), Immediate(constant));
848         }
849       } else {
850         __ cmpl(lhs.AsRegister<CpuRegister>(),
851                 Address(CpuRegister(RSP), rhs.GetStackIndex()));
852       }
853       __ j(X86_64Condition(cond->AsCondition()->GetCondition()), true_target);
854     }
855   }
856   if (false_target != nullptr) {
857     __ jmp(false_target);
858   }
859 }
860 
VisitIf(HIf * if_instr)861 void LocationsBuilderX86_64::VisitIf(HIf* if_instr) {
862   LocationSummary* locations =
863       new (GetGraph()->GetArena()) LocationSummary(if_instr, LocationSummary::kNoCall);
864   HInstruction* cond = if_instr->InputAt(0);
865   if (!cond->IsCondition() || cond->AsCondition()->NeedsMaterialization()) {
866     locations->SetInAt(0, Location::Any());
867   }
868 }
869 
VisitIf(HIf * if_instr)870 void InstructionCodeGeneratorX86_64::VisitIf(HIf* if_instr) {
871   Label* true_target = codegen_->GetLabelOf(if_instr->IfTrueSuccessor());
872   Label* false_target = codegen_->GetLabelOf(if_instr->IfFalseSuccessor());
873   Label* always_true_target = true_target;
874   if (codegen_->GoesToNextBlock(if_instr->GetBlock(),
875                                 if_instr->IfTrueSuccessor())) {
876     always_true_target = nullptr;
877   }
878   if (codegen_->GoesToNextBlock(if_instr->GetBlock(),
879                                 if_instr->IfFalseSuccessor())) {
880     false_target = nullptr;
881   }
882   GenerateTestAndBranch(if_instr, true_target, false_target, always_true_target);
883 }
884 
VisitDeoptimize(HDeoptimize * deoptimize)885 void LocationsBuilderX86_64::VisitDeoptimize(HDeoptimize* deoptimize) {
886   LocationSummary* locations = new (GetGraph()->GetArena())
887       LocationSummary(deoptimize, LocationSummary::kCallOnSlowPath);
888   HInstruction* cond = deoptimize->InputAt(0);
889   DCHECK(cond->IsCondition());
890   if (cond->AsCondition()->NeedsMaterialization()) {
891     locations->SetInAt(0, Location::Any());
892   }
893 }
894 
VisitDeoptimize(HDeoptimize * deoptimize)895 void InstructionCodeGeneratorX86_64::VisitDeoptimize(HDeoptimize* deoptimize) {
896   SlowPathCodeX86_64* slow_path = new (GetGraph()->GetArena())
897       DeoptimizationSlowPathX86_64(deoptimize);
898   codegen_->AddSlowPath(slow_path);
899   Label* slow_path_entry = slow_path->GetEntryLabel();
900   GenerateTestAndBranch(deoptimize, slow_path_entry, nullptr, slow_path_entry);
901 }
902 
VisitLocal(HLocal * local)903 void LocationsBuilderX86_64::VisitLocal(HLocal* local) {
904   local->SetLocations(nullptr);
905 }
906 
VisitLocal(HLocal * local)907 void InstructionCodeGeneratorX86_64::VisitLocal(HLocal* local) {
908   DCHECK_EQ(local->GetBlock(), GetGraph()->GetEntryBlock());
909 }
910 
VisitLoadLocal(HLoadLocal * local)911 void LocationsBuilderX86_64::VisitLoadLocal(HLoadLocal* local) {
912   local->SetLocations(nullptr);
913 }
914 
VisitLoadLocal(HLoadLocal * load)915 void InstructionCodeGeneratorX86_64::VisitLoadLocal(HLoadLocal* load) {
916   // Nothing to do, this is driven by the code generator.
917   UNUSED(load);
918 }
919 
VisitStoreLocal(HStoreLocal * store)920 void LocationsBuilderX86_64::VisitStoreLocal(HStoreLocal* store) {
921   LocationSummary* locations =
922       new (GetGraph()->GetArena()) LocationSummary(store, LocationSummary::kNoCall);
923   switch (store->InputAt(1)->GetType()) {
924     case Primitive::kPrimBoolean:
925     case Primitive::kPrimByte:
926     case Primitive::kPrimChar:
927     case Primitive::kPrimShort:
928     case Primitive::kPrimInt:
929     case Primitive::kPrimNot:
930     case Primitive::kPrimFloat:
931       locations->SetInAt(1, Location::StackSlot(codegen_->GetStackSlot(store->GetLocal())));
932       break;
933 
934     case Primitive::kPrimLong:
935     case Primitive::kPrimDouble:
936       locations->SetInAt(1, Location::DoubleStackSlot(codegen_->GetStackSlot(store->GetLocal())));
937       break;
938 
939     default:
940       LOG(FATAL) << "Unexpected local type " << store->InputAt(1)->GetType();
941   }
942 }
943 
VisitStoreLocal(HStoreLocal * store)944 void InstructionCodeGeneratorX86_64::VisitStoreLocal(HStoreLocal* store) {
945   UNUSED(store);
946 }
947 
VisitCondition(HCondition * comp)948 void LocationsBuilderX86_64::VisitCondition(HCondition* comp) {
949   LocationSummary* locations =
950       new (GetGraph()->GetArena()) LocationSummary(comp, LocationSummary::kNoCall);
951   locations->SetInAt(0, Location::RequiresRegister());
952   locations->SetInAt(1, Location::Any());
953   if (comp->NeedsMaterialization()) {
954     locations->SetOut(Location::RequiresRegister());
955   }
956 }
957 
VisitCondition(HCondition * comp)958 void InstructionCodeGeneratorX86_64::VisitCondition(HCondition* comp) {
959   if (comp->NeedsMaterialization()) {
960     LocationSummary* locations = comp->GetLocations();
961     CpuRegister reg = locations->Out().AsRegister<CpuRegister>();
962     // Clear register: setcc only sets the low byte.
963     __ xorl(reg, reg);
964     Location lhs = locations->InAt(0);
965     Location rhs = locations->InAt(1);
966     if (rhs.IsRegister()) {
967       __ cmpl(lhs.AsRegister<CpuRegister>(), rhs.AsRegister<CpuRegister>());
968     } else if (rhs.IsConstant()) {
969       int32_t constant = CodeGenerator::GetInt32ValueOf(rhs.GetConstant());
970       if (constant == 0) {
971         __ testl(lhs.AsRegister<CpuRegister>(), lhs.AsRegister<CpuRegister>());
972       } else {
973         __ cmpl(lhs.AsRegister<CpuRegister>(), Immediate(constant));
974       }
975     } else {
976       __ cmpl(lhs.AsRegister<CpuRegister>(), Address(CpuRegister(RSP), rhs.GetStackIndex()));
977     }
978     __ setcc(X86_64Condition(comp->GetCondition()), reg);
979   }
980 }
981 
VisitEqual(HEqual * comp)982 void LocationsBuilderX86_64::VisitEqual(HEqual* comp) {
983   VisitCondition(comp);
984 }
985 
VisitEqual(HEqual * comp)986 void InstructionCodeGeneratorX86_64::VisitEqual(HEqual* comp) {
987   VisitCondition(comp);
988 }
989 
VisitNotEqual(HNotEqual * comp)990 void LocationsBuilderX86_64::VisitNotEqual(HNotEqual* comp) {
991   VisitCondition(comp);
992 }
993 
VisitNotEqual(HNotEqual * comp)994 void InstructionCodeGeneratorX86_64::VisitNotEqual(HNotEqual* comp) {
995   VisitCondition(comp);
996 }
997 
VisitLessThan(HLessThan * comp)998 void LocationsBuilderX86_64::VisitLessThan(HLessThan* comp) {
999   VisitCondition(comp);
1000 }
1001 
VisitLessThan(HLessThan * comp)1002 void InstructionCodeGeneratorX86_64::VisitLessThan(HLessThan* comp) {
1003   VisitCondition(comp);
1004 }
1005 
VisitLessThanOrEqual(HLessThanOrEqual * comp)1006 void LocationsBuilderX86_64::VisitLessThanOrEqual(HLessThanOrEqual* comp) {
1007   VisitCondition(comp);
1008 }
1009 
VisitLessThanOrEqual(HLessThanOrEqual * comp)1010 void InstructionCodeGeneratorX86_64::VisitLessThanOrEqual(HLessThanOrEqual* comp) {
1011   VisitCondition(comp);
1012 }
1013 
VisitGreaterThan(HGreaterThan * comp)1014 void LocationsBuilderX86_64::VisitGreaterThan(HGreaterThan* comp) {
1015   VisitCondition(comp);
1016 }
1017 
VisitGreaterThan(HGreaterThan * comp)1018 void InstructionCodeGeneratorX86_64::VisitGreaterThan(HGreaterThan* comp) {
1019   VisitCondition(comp);
1020 }
1021 
VisitGreaterThanOrEqual(HGreaterThanOrEqual * comp)1022 void LocationsBuilderX86_64::VisitGreaterThanOrEqual(HGreaterThanOrEqual* comp) {
1023   VisitCondition(comp);
1024 }
1025 
VisitGreaterThanOrEqual(HGreaterThanOrEqual * comp)1026 void InstructionCodeGeneratorX86_64::VisitGreaterThanOrEqual(HGreaterThanOrEqual* comp) {
1027   VisitCondition(comp);
1028 }
1029 
VisitCompare(HCompare * compare)1030 void LocationsBuilderX86_64::VisitCompare(HCompare* compare) {
1031   LocationSummary* locations =
1032       new (GetGraph()->GetArena()) LocationSummary(compare, LocationSummary::kNoCall);
1033   switch (compare->InputAt(0)->GetType()) {
1034     case Primitive::kPrimLong: {
1035       locations->SetInAt(0, Location::RequiresRegister());
1036       locations->SetInAt(1, Location::Any());
1037       locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
1038       break;
1039     }
1040     case Primitive::kPrimFloat:
1041     case Primitive::kPrimDouble: {
1042       locations->SetInAt(0, Location::RequiresFpuRegister());
1043       locations->SetInAt(1, Location::Any());
1044       locations->SetOut(Location::RequiresRegister());
1045       break;
1046     }
1047     default:
1048       LOG(FATAL) << "Unexpected type for compare operation " << compare->InputAt(0)->GetType();
1049   }
1050 }
1051 
VisitCompare(HCompare * compare)1052 void InstructionCodeGeneratorX86_64::VisitCompare(HCompare* compare) {
1053   LocationSummary* locations = compare->GetLocations();
1054   CpuRegister out = locations->Out().AsRegister<CpuRegister>();
1055   Location left = locations->InAt(0);
1056   Location right = locations->InAt(1);
1057 
1058   Label less, greater, done;
1059   Primitive::Type type = compare->InputAt(0)->GetType();
1060   switch (type) {
1061     case Primitive::kPrimLong: {
1062       CpuRegister left_reg = left.AsRegister<CpuRegister>();
1063       if (right.IsConstant()) {
1064         int64_t value = right.GetConstant()->AsLongConstant()->GetValue();
1065         if (IsInt<32>(value)) {
1066           if (value == 0) {
1067             __ testq(left_reg, left_reg);
1068           } else {
1069             __ cmpq(left_reg, Immediate(static_cast<int32_t>(value)));
1070           }
1071         } else {
1072           // Value won't fit in an int.
1073           __ cmpq(left_reg, codegen_->LiteralInt64Address(value));
1074         }
1075       } else if (right.IsDoubleStackSlot()) {
1076         __ cmpq(left_reg, Address(CpuRegister(RSP), right.GetStackIndex()));
1077       } else {
1078         __ cmpq(left_reg, right.AsRegister<CpuRegister>());
1079       }
1080       break;
1081     }
1082     case Primitive::kPrimFloat: {
1083       XmmRegister left_reg = left.AsFpuRegister<XmmRegister>();
1084       if (right.IsConstant()) {
1085         float value = right.GetConstant()->AsFloatConstant()->GetValue();
1086         __ ucomiss(left_reg, codegen_->LiteralFloatAddress(value));
1087       } else if (right.IsStackSlot()) {
1088         __ ucomiss(left_reg, Address(CpuRegister(RSP), right.GetStackIndex()));
1089       } else {
1090         __ ucomiss(left_reg, right.AsFpuRegister<XmmRegister>());
1091       }
1092       __ j(kUnordered, compare->IsGtBias() ? &greater : &less);
1093       break;
1094     }
1095     case Primitive::kPrimDouble: {
1096       XmmRegister left_reg = left.AsFpuRegister<XmmRegister>();
1097       if (right.IsConstant()) {
1098         double value = right.GetConstant()->AsDoubleConstant()->GetValue();
1099         __ ucomisd(left_reg, codegen_->LiteralDoubleAddress(value));
1100       } else if (right.IsDoubleStackSlot()) {
1101         __ ucomisd(left_reg, Address(CpuRegister(RSP), right.GetStackIndex()));
1102       } else {
1103         __ ucomisd(left_reg, right.AsFpuRegister<XmmRegister>());
1104       }
1105       __ j(kUnordered, compare->IsGtBias() ? &greater : &less);
1106       break;
1107     }
1108     default:
1109       LOG(FATAL) << "Unexpected compare type " << type;
1110   }
1111   __ movl(out, Immediate(0));
1112   __ j(kEqual, &done);
1113   __ j(type == Primitive::kPrimLong ? kLess : kBelow, &less);  //  ucomis{s,d} sets CF (kBelow)
1114 
1115   __ Bind(&greater);
1116   __ movl(out, Immediate(1));
1117   __ jmp(&done);
1118 
1119   __ Bind(&less);
1120   __ movl(out, Immediate(-1));
1121 
1122   __ Bind(&done);
1123 }
1124 
VisitIntConstant(HIntConstant * constant)1125 void LocationsBuilderX86_64::VisitIntConstant(HIntConstant* constant) {
1126   LocationSummary* locations =
1127       new (GetGraph()->GetArena()) LocationSummary(constant, LocationSummary::kNoCall);
1128   locations->SetOut(Location::ConstantLocation(constant));
1129 }
1130 
VisitIntConstant(HIntConstant * constant)1131 void InstructionCodeGeneratorX86_64::VisitIntConstant(HIntConstant* constant) {
1132   // Will be generated at use site.
1133   UNUSED(constant);
1134 }
1135 
VisitNullConstant(HNullConstant * constant)1136 void LocationsBuilderX86_64::VisitNullConstant(HNullConstant* constant) {
1137   LocationSummary* locations =
1138       new (GetGraph()->GetArena()) LocationSummary(constant, LocationSummary::kNoCall);
1139   locations->SetOut(Location::ConstantLocation(constant));
1140 }
1141 
VisitNullConstant(HNullConstant * constant)1142 void InstructionCodeGeneratorX86_64::VisitNullConstant(HNullConstant* constant) {
1143   // Will be generated at use site.
1144   UNUSED(constant);
1145 }
1146 
VisitLongConstant(HLongConstant * constant)1147 void LocationsBuilderX86_64::VisitLongConstant(HLongConstant* constant) {
1148   LocationSummary* locations =
1149       new (GetGraph()->GetArena()) LocationSummary(constant, LocationSummary::kNoCall);
1150   locations->SetOut(Location::ConstantLocation(constant));
1151 }
1152 
VisitLongConstant(HLongConstant * constant)1153 void InstructionCodeGeneratorX86_64::VisitLongConstant(HLongConstant* constant) {
1154   // Will be generated at use site.
1155   UNUSED(constant);
1156 }
1157 
VisitFloatConstant(HFloatConstant * constant)1158 void LocationsBuilderX86_64::VisitFloatConstant(HFloatConstant* constant) {
1159   LocationSummary* locations =
1160       new (GetGraph()->GetArena()) LocationSummary(constant, LocationSummary::kNoCall);
1161   locations->SetOut(Location::ConstantLocation(constant));
1162 }
1163 
VisitFloatConstant(HFloatConstant * constant)1164 void InstructionCodeGeneratorX86_64::VisitFloatConstant(HFloatConstant* constant) {
1165   // Will be generated at use site.
1166   UNUSED(constant);
1167 }
1168 
VisitDoubleConstant(HDoubleConstant * constant)1169 void LocationsBuilderX86_64::VisitDoubleConstant(HDoubleConstant* constant) {
1170   LocationSummary* locations =
1171       new (GetGraph()->GetArena()) LocationSummary(constant, LocationSummary::kNoCall);
1172   locations->SetOut(Location::ConstantLocation(constant));
1173 }
1174 
VisitDoubleConstant(HDoubleConstant * constant)1175 void InstructionCodeGeneratorX86_64::VisitDoubleConstant(HDoubleConstant* constant) {
1176   // Will be generated at use site.
1177   UNUSED(constant);
1178 }
1179 
VisitMemoryBarrier(HMemoryBarrier * memory_barrier)1180 void LocationsBuilderX86_64::VisitMemoryBarrier(HMemoryBarrier* memory_barrier) {
1181   memory_barrier->SetLocations(nullptr);
1182 }
1183 
VisitMemoryBarrier(HMemoryBarrier * memory_barrier)1184 void InstructionCodeGeneratorX86_64::VisitMemoryBarrier(HMemoryBarrier* memory_barrier) {
1185   GenerateMemoryBarrier(memory_barrier->GetBarrierKind());
1186 }
1187 
VisitReturnVoid(HReturnVoid * ret)1188 void LocationsBuilderX86_64::VisitReturnVoid(HReturnVoid* ret) {
1189   ret->SetLocations(nullptr);
1190 }
1191 
VisitReturnVoid(HReturnVoid * ret)1192 void InstructionCodeGeneratorX86_64::VisitReturnVoid(HReturnVoid* ret) {
1193   UNUSED(ret);
1194   codegen_->GenerateFrameExit();
1195 }
1196 
VisitReturn(HReturn * ret)1197 void LocationsBuilderX86_64::VisitReturn(HReturn* ret) {
1198   LocationSummary* locations =
1199       new (GetGraph()->GetArena()) LocationSummary(ret, LocationSummary::kNoCall);
1200   switch (ret->InputAt(0)->GetType()) {
1201     case Primitive::kPrimBoolean:
1202     case Primitive::kPrimByte:
1203     case Primitive::kPrimChar:
1204     case Primitive::kPrimShort:
1205     case Primitive::kPrimInt:
1206     case Primitive::kPrimNot:
1207     case Primitive::kPrimLong:
1208       locations->SetInAt(0, Location::RegisterLocation(RAX));
1209       break;
1210 
1211     case Primitive::kPrimFloat:
1212     case Primitive::kPrimDouble:
1213       locations->SetInAt(0, Location::FpuRegisterLocation(XMM0));
1214       break;
1215 
1216     default:
1217       LOG(FATAL) << "Unexpected return type " << ret->InputAt(0)->GetType();
1218   }
1219 }
1220 
VisitReturn(HReturn * ret)1221 void InstructionCodeGeneratorX86_64::VisitReturn(HReturn* ret) {
1222   if (kIsDebugBuild) {
1223     switch (ret->InputAt(0)->GetType()) {
1224       case Primitive::kPrimBoolean:
1225       case Primitive::kPrimByte:
1226       case Primitive::kPrimChar:
1227       case Primitive::kPrimShort:
1228       case Primitive::kPrimInt:
1229       case Primitive::kPrimNot:
1230       case Primitive::kPrimLong:
1231         DCHECK_EQ(ret->GetLocations()->InAt(0).AsRegister<CpuRegister>().AsRegister(), RAX);
1232         break;
1233 
1234       case Primitive::kPrimFloat:
1235       case Primitive::kPrimDouble:
1236         DCHECK_EQ(ret->GetLocations()->InAt(0).AsFpuRegister<XmmRegister>().AsFloatRegister(),
1237                   XMM0);
1238         break;
1239 
1240       default:
1241         LOG(FATAL) << "Unexpected return type " << ret->InputAt(0)->GetType();
1242     }
1243   }
1244   codegen_->GenerateFrameExit();
1245 }
1246 
GetNextLocation(Primitive::Type type)1247 Location InvokeDexCallingConventionVisitorX86_64::GetNextLocation(Primitive::Type type) {
1248   switch (type) {
1249     case Primitive::kPrimBoolean:
1250     case Primitive::kPrimByte:
1251     case Primitive::kPrimChar:
1252     case Primitive::kPrimShort:
1253     case Primitive::kPrimInt:
1254     case Primitive::kPrimNot: {
1255       uint32_t index = gp_index_++;
1256       stack_index_++;
1257       if (index < calling_convention.GetNumberOfRegisters()) {
1258         return Location::RegisterLocation(calling_convention.GetRegisterAt(index));
1259       } else {
1260         return Location::StackSlot(calling_convention.GetStackOffsetOf(stack_index_ - 1));
1261       }
1262     }
1263 
1264     case Primitive::kPrimLong: {
1265       uint32_t index = gp_index_;
1266       stack_index_ += 2;
1267       if (index < calling_convention.GetNumberOfRegisters()) {
1268         gp_index_ += 1;
1269         return Location::RegisterLocation(calling_convention.GetRegisterAt(index));
1270       } else {
1271         gp_index_ += 2;
1272         return Location::DoubleStackSlot(calling_convention.GetStackOffsetOf(stack_index_ - 2));
1273       }
1274     }
1275 
1276     case Primitive::kPrimFloat: {
1277       uint32_t index = float_index_++;
1278       stack_index_++;
1279       if (index < calling_convention.GetNumberOfFpuRegisters()) {
1280         return Location::FpuRegisterLocation(calling_convention.GetFpuRegisterAt(index));
1281       } else {
1282         return Location::StackSlot(calling_convention.GetStackOffsetOf(stack_index_ - 1));
1283       }
1284     }
1285 
1286     case Primitive::kPrimDouble: {
1287       uint32_t index = float_index_++;
1288       stack_index_ += 2;
1289       if (index < calling_convention.GetNumberOfFpuRegisters()) {
1290         return Location::FpuRegisterLocation(calling_convention.GetFpuRegisterAt(index));
1291       } else {
1292         return Location::DoubleStackSlot(calling_convention.GetStackOffsetOf(stack_index_ - 2));
1293       }
1294     }
1295 
1296     case Primitive::kPrimVoid:
1297       LOG(FATAL) << "Unexpected parameter type " << type;
1298       break;
1299   }
1300   return Location();
1301 }
1302 
VisitInvokeStaticOrDirect(HInvokeStaticOrDirect * invoke)1303 void LocationsBuilderX86_64::VisitInvokeStaticOrDirect(HInvokeStaticOrDirect* invoke) {
1304   // When we do not run baseline, explicit clinit checks triggered by static
1305   // invokes must have been pruned by art::PrepareForRegisterAllocation.
1306   DCHECK(codegen_->IsBaseline() || !invoke->IsStaticWithExplicitClinitCheck());
1307 
1308   IntrinsicLocationsBuilderX86_64 intrinsic(codegen_);
1309   if (intrinsic.TryDispatch(invoke)) {
1310     return;
1311   }
1312 
1313   HandleInvoke(invoke);
1314 }
1315 
TryGenerateIntrinsicCode(HInvoke * invoke,CodeGeneratorX86_64 * codegen)1316 static bool TryGenerateIntrinsicCode(HInvoke* invoke, CodeGeneratorX86_64* codegen) {
1317   if (invoke->GetLocations()->Intrinsified()) {
1318     IntrinsicCodeGeneratorX86_64 intrinsic(codegen);
1319     intrinsic.Dispatch(invoke);
1320     return true;
1321   }
1322   return false;
1323 }
1324 
VisitInvokeStaticOrDirect(HInvokeStaticOrDirect * invoke)1325 void InstructionCodeGeneratorX86_64::VisitInvokeStaticOrDirect(HInvokeStaticOrDirect* invoke) {
1326   // When we do not run baseline, explicit clinit checks triggered by static
1327   // invokes must have been pruned by art::PrepareForRegisterAllocation.
1328   DCHECK(codegen_->IsBaseline() || !invoke->IsStaticWithExplicitClinitCheck());
1329 
1330   if (TryGenerateIntrinsicCode(invoke, codegen_)) {
1331     return;
1332   }
1333 
1334   codegen_->GenerateStaticOrDirectCall(
1335       invoke,
1336       invoke->GetLocations()->GetTemp(0).AsRegister<CpuRegister>());
1337   codegen_->RecordPcInfo(invoke, invoke->GetDexPc());
1338 }
1339 
HandleInvoke(HInvoke * invoke)1340 void LocationsBuilderX86_64::HandleInvoke(HInvoke* invoke) {
1341   LocationSummary* locations =
1342       new (GetGraph()->GetArena()) LocationSummary(invoke, LocationSummary::kCall);
1343   locations->AddTemp(Location::RegisterLocation(RDI));
1344 
1345   InvokeDexCallingConventionVisitorX86_64 calling_convention_visitor;
1346   for (size_t i = 0; i < invoke->GetNumberOfArguments(); i++) {
1347     HInstruction* input = invoke->InputAt(i);
1348     locations->SetInAt(i, calling_convention_visitor.GetNextLocation(input->GetType()));
1349   }
1350 
1351   switch (invoke->GetType()) {
1352     case Primitive::kPrimBoolean:
1353     case Primitive::kPrimByte:
1354     case Primitive::kPrimChar:
1355     case Primitive::kPrimShort:
1356     case Primitive::kPrimInt:
1357     case Primitive::kPrimNot:
1358     case Primitive::kPrimLong:
1359       locations->SetOut(Location::RegisterLocation(RAX));
1360       break;
1361 
1362     case Primitive::kPrimVoid:
1363       break;
1364 
1365     case Primitive::kPrimDouble:
1366     case Primitive::kPrimFloat:
1367       locations->SetOut(Location::FpuRegisterLocation(XMM0));
1368       break;
1369   }
1370 }
1371 
VisitInvokeVirtual(HInvokeVirtual * invoke)1372 void LocationsBuilderX86_64::VisitInvokeVirtual(HInvokeVirtual* invoke) {
1373   IntrinsicLocationsBuilderX86_64 intrinsic(codegen_);
1374   if (intrinsic.TryDispatch(invoke)) {
1375     return;
1376   }
1377 
1378   HandleInvoke(invoke);
1379 }
1380 
VisitInvokeVirtual(HInvokeVirtual * invoke)1381 void InstructionCodeGeneratorX86_64::VisitInvokeVirtual(HInvokeVirtual* invoke) {
1382   if (TryGenerateIntrinsicCode(invoke, codegen_)) {
1383     return;
1384   }
1385 
1386   CpuRegister temp = invoke->GetLocations()->GetTemp(0).AsRegister<CpuRegister>();
1387   size_t method_offset = mirror::Class::EmbeddedVTableEntryOffset(
1388       invoke->GetVTableIndex(), kX86_64PointerSize).SizeValue();
1389   LocationSummary* locations = invoke->GetLocations();
1390   Location receiver = locations->InAt(0);
1391   size_t class_offset = mirror::Object::ClassOffset().SizeValue();
1392   // temp = object->GetClass();
1393   if (receiver.IsStackSlot()) {
1394     __ movl(temp, Address(CpuRegister(RSP), receiver.GetStackIndex()));
1395     __ movl(temp, Address(temp, class_offset));
1396   } else {
1397     __ movl(temp, Address(receiver.AsRegister<CpuRegister>(), class_offset));
1398   }
1399   codegen_->MaybeRecordImplicitNullCheck(invoke);
1400   // temp = temp->GetMethodAt(method_offset);
1401   __ movq(temp, Address(temp, method_offset));
1402   // call temp->GetEntryPoint();
1403   __ call(Address(temp, ArtMethod::EntryPointFromQuickCompiledCodeOffset(
1404       kX86_64WordSize).SizeValue()));
1405 
1406   DCHECK(!codegen_->IsLeafMethod());
1407   codegen_->RecordPcInfo(invoke, invoke->GetDexPc());
1408 }
1409 
VisitInvokeInterface(HInvokeInterface * invoke)1410 void LocationsBuilderX86_64::VisitInvokeInterface(HInvokeInterface* invoke) {
1411   HandleInvoke(invoke);
1412   // Add the hidden argument.
1413   invoke->GetLocations()->AddTemp(Location::RegisterLocation(RAX));
1414 }
1415 
VisitInvokeInterface(HInvokeInterface * invoke)1416 void InstructionCodeGeneratorX86_64::VisitInvokeInterface(HInvokeInterface* invoke) {
1417   // TODO: b/18116999, our IMTs can miss an IncompatibleClassChangeError.
1418   CpuRegister temp = invoke->GetLocations()->GetTemp(0).AsRegister<CpuRegister>();
1419   uint32_t method_offset = mirror::Class::EmbeddedImTableEntryOffset(
1420       invoke->GetImtIndex() % mirror::Class::kImtSize, kX86_64PointerSize).Uint32Value();
1421   LocationSummary* locations = invoke->GetLocations();
1422   Location receiver = locations->InAt(0);
1423   size_t class_offset = mirror::Object::ClassOffset().SizeValue();
1424 
1425   // Set the hidden argument.
1426   CpuRegister hidden_reg = invoke->GetLocations()->GetTemp(1).AsRegister<CpuRegister>();
1427   codegen_->Load64BitValue(hidden_reg, invoke->GetDexMethodIndex());
1428 
1429   // temp = object->GetClass();
1430   if (receiver.IsStackSlot()) {
1431     __ movl(temp, Address(CpuRegister(RSP), receiver.GetStackIndex()));
1432     __ movl(temp, Address(temp, class_offset));
1433   } else {
1434     __ movl(temp, Address(receiver.AsRegister<CpuRegister>(), class_offset));
1435   }
1436   codegen_->MaybeRecordImplicitNullCheck(invoke);
1437   // temp = temp->GetImtEntryAt(method_offset);
1438   __ movq(temp, Address(temp, method_offset));
1439   // call temp->GetEntryPoint();
1440   __ call(Address(temp, ArtMethod::EntryPointFromQuickCompiledCodeOffset(
1441       kX86_64WordSize).SizeValue()));
1442 
1443   DCHECK(!codegen_->IsLeafMethod());
1444   codegen_->RecordPcInfo(invoke, invoke->GetDexPc());
1445 }
1446 
VisitNeg(HNeg * neg)1447 void LocationsBuilderX86_64::VisitNeg(HNeg* neg) {
1448   LocationSummary* locations =
1449       new (GetGraph()->GetArena()) LocationSummary(neg, LocationSummary::kNoCall);
1450   switch (neg->GetResultType()) {
1451     case Primitive::kPrimInt:
1452     case Primitive::kPrimLong:
1453       locations->SetInAt(0, Location::RequiresRegister());
1454       locations->SetOut(Location::SameAsFirstInput());
1455       break;
1456 
1457     case Primitive::kPrimFloat:
1458     case Primitive::kPrimDouble:
1459       locations->SetInAt(0, Location::RequiresFpuRegister());
1460       locations->SetOut(Location::SameAsFirstInput());
1461       locations->AddTemp(Location::RequiresFpuRegister());
1462       break;
1463 
1464     default:
1465       LOG(FATAL) << "Unexpected neg type " << neg->GetResultType();
1466   }
1467 }
1468 
VisitNeg(HNeg * neg)1469 void InstructionCodeGeneratorX86_64::VisitNeg(HNeg* neg) {
1470   LocationSummary* locations = neg->GetLocations();
1471   Location out = locations->Out();
1472   Location in = locations->InAt(0);
1473   switch (neg->GetResultType()) {
1474     case Primitive::kPrimInt:
1475       DCHECK(in.IsRegister());
1476       DCHECK(in.Equals(out));
1477       __ negl(out.AsRegister<CpuRegister>());
1478       break;
1479 
1480     case Primitive::kPrimLong:
1481       DCHECK(in.IsRegister());
1482       DCHECK(in.Equals(out));
1483       __ negq(out.AsRegister<CpuRegister>());
1484       break;
1485 
1486     case Primitive::kPrimFloat: {
1487       DCHECK(in.Equals(out));
1488       XmmRegister mask = locations->GetTemp(0).AsFpuRegister<XmmRegister>();
1489       // Implement float negation with an exclusive or with value
1490       // 0x80000000 (mask for bit 31, representing the sign of a
1491       // single-precision floating-point number).
1492       __ movss(mask, codegen_->LiteralInt32Address(0x80000000));
1493       __ xorps(out.AsFpuRegister<XmmRegister>(), mask);
1494       break;
1495     }
1496 
1497     case Primitive::kPrimDouble: {
1498       DCHECK(in.Equals(out));
1499       XmmRegister mask = locations->GetTemp(0).AsFpuRegister<XmmRegister>();
1500       // Implement double negation with an exclusive or with value
1501       // 0x8000000000000000 (mask for bit 63, representing the sign of
1502       // a double-precision floating-point number).
1503       __ movsd(mask, codegen_->LiteralInt64Address(INT64_C(0x8000000000000000)));
1504       __ xorpd(out.AsFpuRegister<XmmRegister>(), mask);
1505       break;
1506     }
1507 
1508     default:
1509       LOG(FATAL) << "Unexpected neg type " << neg->GetResultType();
1510   }
1511 }
1512 
VisitTypeConversion(HTypeConversion * conversion)1513 void LocationsBuilderX86_64::VisitTypeConversion(HTypeConversion* conversion) {
1514   LocationSummary* locations =
1515       new (GetGraph()->GetArena()) LocationSummary(conversion, LocationSummary::kNoCall);
1516   Primitive::Type result_type = conversion->GetResultType();
1517   Primitive::Type input_type = conversion->GetInputType();
1518   DCHECK_NE(result_type, input_type);
1519 
1520   // The Java language does not allow treating boolean as an integral type but
1521   // our bit representation makes it safe.
1522 
1523   switch (result_type) {
1524     case Primitive::kPrimByte:
1525       switch (input_type) {
1526         case Primitive::kPrimBoolean:
1527           // Boolean input is a result of code transformations.
1528         case Primitive::kPrimShort:
1529         case Primitive::kPrimInt:
1530         case Primitive::kPrimChar:
1531           // Processing a Dex `int-to-byte' instruction.
1532           locations->SetInAt(0, Location::Any());
1533           locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
1534           break;
1535 
1536         default:
1537           LOG(FATAL) << "Unexpected type conversion from " << input_type
1538                      << " to " << result_type;
1539       }
1540       break;
1541 
1542     case Primitive::kPrimShort:
1543       switch (input_type) {
1544         case Primitive::kPrimBoolean:
1545           // Boolean input is a result of code transformations.
1546         case Primitive::kPrimByte:
1547         case Primitive::kPrimInt:
1548         case Primitive::kPrimChar:
1549           // Processing a Dex `int-to-short' instruction.
1550           locations->SetInAt(0, Location::Any());
1551           locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
1552           break;
1553 
1554         default:
1555           LOG(FATAL) << "Unexpected type conversion from " << input_type
1556                      << " to " << result_type;
1557       }
1558       break;
1559 
1560     case Primitive::kPrimInt:
1561       switch (input_type) {
1562         case Primitive::kPrimLong:
1563           // Processing a Dex `long-to-int' instruction.
1564           locations->SetInAt(0, Location::Any());
1565           locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
1566           break;
1567 
1568         case Primitive::kPrimFloat:
1569           // Processing a Dex `float-to-int' instruction.
1570           locations->SetInAt(0, Location::RequiresFpuRegister());
1571           locations->SetOut(Location::RequiresRegister());
1572           locations->AddTemp(Location::RequiresFpuRegister());
1573           break;
1574 
1575         case Primitive::kPrimDouble:
1576           // Processing a Dex `double-to-int' instruction.
1577           locations->SetInAt(0, Location::RequiresFpuRegister());
1578           locations->SetOut(Location::RequiresRegister());
1579           locations->AddTemp(Location::RequiresFpuRegister());
1580           break;
1581 
1582         default:
1583           LOG(FATAL) << "Unexpected type conversion from " << input_type
1584                      << " to " << result_type;
1585       }
1586       break;
1587 
1588     case Primitive::kPrimLong:
1589       switch (input_type) {
1590         case Primitive::kPrimBoolean:
1591           // Boolean input is a result of code transformations.
1592         case Primitive::kPrimByte:
1593         case Primitive::kPrimShort:
1594         case Primitive::kPrimInt:
1595         case Primitive::kPrimChar:
1596           // Processing a Dex `int-to-long' instruction.
1597           // TODO: We would benefit from a (to-be-implemented)
1598           // Location::RegisterOrStackSlot requirement for this input.
1599           locations->SetInAt(0, Location::RequiresRegister());
1600           locations->SetOut(Location::RequiresRegister());
1601           break;
1602 
1603         case Primitive::kPrimFloat:
1604           // Processing a Dex `float-to-long' instruction.
1605           locations->SetInAt(0, Location::RequiresFpuRegister());
1606           locations->SetOut(Location::RequiresRegister());
1607           locations->AddTemp(Location::RequiresFpuRegister());
1608           break;
1609 
1610         case Primitive::kPrimDouble:
1611           // Processing a Dex `double-to-long' instruction.
1612           locations->SetInAt(0, Location::RequiresFpuRegister());
1613           locations->SetOut(Location::RequiresRegister());
1614           locations->AddTemp(Location::RequiresFpuRegister());
1615           break;
1616 
1617         default:
1618           LOG(FATAL) << "Unexpected type conversion from " << input_type
1619                      << " to " << result_type;
1620       }
1621       break;
1622 
1623     case Primitive::kPrimChar:
1624       switch (input_type) {
1625         case Primitive::kPrimBoolean:
1626           // Boolean input is a result of code transformations.
1627         case Primitive::kPrimByte:
1628         case Primitive::kPrimShort:
1629         case Primitive::kPrimInt:
1630           // Processing a Dex `int-to-char' instruction.
1631           locations->SetInAt(0, Location::Any());
1632           locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
1633           break;
1634 
1635         default:
1636           LOG(FATAL) << "Unexpected type conversion from " << input_type
1637                      << " to " << result_type;
1638       }
1639       break;
1640 
1641     case Primitive::kPrimFloat:
1642       switch (input_type) {
1643         case Primitive::kPrimBoolean:
1644           // Boolean input is a result of code transformations.
1645         case Primitive::kPrimByte:
1646         case Primitive::kPrimShort:
1647         case Primitive::kPrimInt:
1648         case Primitive::kPrimChar:
1649           // Processing a Dex `int-to-float' instruction.
1650           locations->SetInAt(0, Location::Any());
1651           locations->SetOut(Location::RequiresFpuRegister());
1652           break;
1653 
1654         case Primitive::kPrimLong:
1655           // Processing a Dex `long-to-float' instruction.
1656           locations->SetInAt(0, Location::Any());
1657           locations->SetOut(Location::RequiresFpuRegister());
1658           break;
1659 
1660         case Primitive::kPrimDouble:
1661           // Processing a Dex `double-to-float' instruction.
1662           locations->SetInAt(0, Location::Any());
1663           locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap);
1664           break;
1665 
1666         default:
1667           LOG(FATAL) << "Unexpected type conversion from " << input_type
1668                      << " to " << result_type;
1669       };
1670       break;
1671 
1672     case Primitive::kPrimDouble:
1673       switch (input_type) {
1674         case Primitive::kPrimBoolean:
1675           // Boolean input is a result of code transformations.
1676         case Primitive::kPrimByte:
1677         case Primitive::kPrimShort:
1678         case Primitive::kPrimInt:
1679         case Primitive::kPrimChar:
1680           // Processing a Dex `int-to-double' instruction.
1681           locations->SetInAt(0, Location::Any());
1682           locations->SetOut(Location::RequiresFpuRegister());
1683           break;
1684 
1685         case Primitive::kPrimLong:
1686           // Processing a Dex `long-to-double' instruction.
1687           locations->SetInAt(0, Location::Any());
1688           locations->SetOut(Location::RequiresFpuRegister());
1689           break;
1690 
1691         case Primitive::kPrimFloat:
1692           // Processing a Dex `float-to-double' instruction.
1693           locations->SetInAt(0, Location::Any());
1694           locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap);
1695           break;
1696 
1697         default:
1698           LOG(FATAL) << "Unexpected type conversion from " << input_type
1699                      << " to " << result_type;
1700       }
1701       break;
1702 
1703     default:
1704       LOG(FATAL) << "Unexpected type conversion from " << input_type
1705                  << " to " << result_type;
1706   }
1707 }
1708 
VisitTypeConversion(HTypeConversion * conversion)1709 void InstructionCodeGeneratorX86_64::VisitTypeConversion(HTypeConversion* conversion) {
1710   LocationSummary* locations = conversion->GetLocations();
1711   Location out = locations->Out();
1712   Location in = locations->InAt(0);
1713   Primitive::Type result_type = conversion->GetResultType();
1714   Primitive::Type input_type = conversion->GetInputType();
1715   DCHECK_NE(result_type, input_type);
1716   switch (result_type) {
1717     case Primitive::kPrimByte:
1718       switch (input_type) {
1719         case Primitive::kPrimBoolean:
1720           // Boolean input is a result of code transformations.
1721         case Primitive::kPrimShort:
1722         case Primitive::kPrimInt:
1723         case Primitive::kPrimChar:
1724           // Processing a Dex `int-to-byte' instruction.
1725           if (in.IsRegister()) {
1726             __ movsxb(out.AsRegister<CpuRegister>(), in.AsRegister<CpuRegister>());
1727           } else if (in.IsStackSlot()) {
1728             __ movsxb(out.AsRegister<CpuRegister>(),
1729                       Address(CpuRegister(RSP), in.GetStackIndex()));
1730           } else {
1731             DCHECK(in.GetConstant()->IsIntConstant());
1732             __ movl(out.AsRegister<CpuRegister>(),
1733                     Immediate(static_cast<int8_t>(in.GetConstant()->AsIntConstant()->GetValue())));
1734           }
1735           break;
1736 
1737         default:
1738           LOG(FATAL) << "Unexpected type conversion from " << input_type
1739                      << " to " << result_type;
1740       }
1741       break;
1742 
1743     case Primitive::kPrimShort:
1744       switch (input_type) {
1745         case Primitive::kPrimBoolean:
1746           // Boolean input is a result of code transformations.
1747         case Primitive::kPrimByte:
1748         case Primitive::kPrimInt:
1749         case Primitive::kPrimChar:
1750           // Processing a Dex `int-to-short' instruction.
1751           if (in.IsRegister()) {
1752             __ movsxw(out.AsRegister<CpuRegister>(), in.AsRegister<CpuRegister>());
1753           } else if (in.IsStackSlot()) {
1754             __ movsxw(out.AsRegister<CpuRegister>(),
1755                       Address(CpuRegister(RSP), in.GetStackIndex()));
1756           } else {
1757             DCHECK(in.GetConstant()->IsIntConstant());
1758             __ movl(out.AsRegister<CpuRegister>(),
1759                     Immediate(static_cast<int16_t>(in.GetConstant()->AsIntConstant()->GetValue())));
1760           }
1761           break;
1762 
1763         default:
1764           LOG(FATAL) << "Unexpected type conversion from " << input_type
1765                      << " to " << result_type;
1766       }
1767       break;
1768 
1769     case Primitive::kPrimInt:
1770       switch (input_type) {
1771         case Primitive::kPrimLong:
1772           // Processing a Dex `long-to-int' instruction.
1773           if (in.IsRegister()) {
1774             __ movl(out.AsRegister<CpuRegister>(), in.AsRegister<CpuRegister>());
1775           } else if (in.IsDoubleStackSlot()) {
1776             __ movl(out.AsRegister<CpuRegister>(),
1777                     Address(CpuRegister(RSP), in.GetStackIndex()));
1778           } else {
1779             DCHECK(in.IsConstant());
1780             DCHECK(in.GetConstant()->IsLongConstant());
1781             int64_t value = in.GetConstant()->AsLongConstant()->GetValue();
1782             __ movl(out.AsRegister<CpuRegister>(), Immediate(static_cast<int32_t>(value)));
1783           }
1784           break;
1785 
1786         case Primitive::kPrimFloat: {
1787           // Processing a Dex `float-to-int' instruction.
1788           XmmRegister input = in.AsFpuRegister<XmmRegister>();
1789           CpuRegister output = out.AsRegister<CpuRegister>();
1790           XmmRegister temp = locations->GetTemp(0).AsFpuRegister<XmmRegister>();
1791           Label done, nan;
1792 
1793           __ movl(output, Immediate(kPrimIntMax));
1794           // temp = int-to-float(output)
1795           __ cvtsi2ss(temp, output, false);
1796           // if input >= temp goto done
1797           __ comiss(input, temp);
1798           __ j(kAboveEqual, &done);
1799           // if input == NaN goto nan
1800           __ j(kUnordered, &nan);
1801           // output = float-to-int-truncate(input)
1802           __ cvttss2si(output, input, false);
1803           __ jmp(&done);
1804           __ Bind(&nan);
1805           //  output = 0
1806           __ xorl(output, output);
1807           __ Bind(&done);
1808           break;
1809         }
1810 
1811         case Primitive::kPrimDouble: {
1812           // Processing a Dex `double-to-int' instruction.
1813           XmmRegister input = in.AsFpuRegister<XmmRegister>();
1814           CpuRegister output = out.AsRegister<CpuRegister>();
1815           XmmRegister temp = locations->GetTemp(0).AsFpuRegister<XmmRegister>();
1816           Label done, nan;
1817 
1818           __ movl(output, Immediate(kPrimIntMax));
1819           // temp = int-to-double(output)
1820           __ cvtsi2sd(temp, output);
1821           // if input >= temp goto done
1822           __ comisd(input, temp);
1823           __ j(kAboveEqual, &done);
1824           // if input == NaN goto nan
1825           __ j(kUnordered, &nan);
1826           // output = double-to-int-truncate(input)
1827           __ cvttsd2si(output, input);
1828           __ jmp(&done);
1829           __ Bind(&nan);
1830           //  output = 0
1831           __ xorl(output, output);
1832           __ Bind(&done);
1833           break;
1834         }
1835 
1836         default:
1837           LOG(FATAL) << "Unexpected type conversion from " << input_type
1838                      << " to " << result_type;
1839       }
1840       break;
1841 
1842     case Primitive::kPrimLong:
1843       switch (input_type) {
1844         DCHECK(out.IsRegister());
1845         case Primitive::kPrimBoolean:
1846           // Boolean input is a result of code transformations.
1847         case Primitive::kPrimByte:
1848         case Primitive::kPrimShort:
1849         case Primitive::kPrimInt:
1850         case Primitive::kPrimChar:
1851           // Processing a Dex `int-to-long' instruction.
1852           DCHECK(in.IsRegister());
1853           __ movsxd(out.AsRegister<CpuRegister>(), in.AsRegister<CpuRegister>());
1854           break;
1855 
1856         case Primitive::kPrimFloat: {
1857           // Processing a Dex `float-to-long' instruction.
1858           XmmRegister input = in.AsFpuRegister<XmmRegister>();
1859           CpuRegister output = out.AsRegister<CpuRegister>();
1860           XmmRegister temp = locations->GetTemp(0).AsFpuRegister<XmmRegister>();
1861           Label done, nan;
1862 
1863           codegen_->Load64BitValue(output, kPrimLongMax);
1864           // temp = long-to-float(output)
1865           __ cvtsi2ss(temp, output, true);
1866           // if input >= temp goto done
1867           __ comiss(input, temp);
1868           __ j(kAboveEqual, &done);
1869           // if input == NaN goto nan
1870           __ j(kUnordered, &nan);
1871           // output = float-to-long-truncate(input)
1872           __ cvttss2si(output, input, true);
1873           __ jmp(&done);
1874           __ Bind(&nan);
1875           //  output = 0
1876           __ xorl(output, output);
1877           __ Bind(&done);
1878           break;
1879         }
1880 
1881         case Primitive::kPrimDouble: {
1882           // Processing a Dex `double-to-long' instruction.
1883           XmmRegister input = in.AsFpuRegister<XmmRegister>();
1884           CpuRegister output = out.AsRegister<CpuRegister>();
1885           XmmRegister temp = locations->GetTemp(0).AsFpuRegister<XmmRegister>();
1886           Label done, nan;
1887 
1888           codegen_->Load64BitValue(output, kPrimLongMax);
1889           // temp = long-to-double(output)
1890           __ cvtsi2sd(temp, output, true);
1891           // if input >= temp goto done
1892           __ comisd(input, temp);
1893           __ j(kAboveEqual, &done);
1894           // if input == NaN goto nan
1895           __ j(kUnordered, &nan);
1896           // output = double-to-long-truncate(input)
1897           __ cvttsd2si(output, input, true);
1898           __ jmp(&done);
1899           __ Bind(&nan);
1900           //  output = 0
1901           __ xorl(output, output);
1902           __ Bind(&done);
1903           break;
1904         }
1905 
1906         default:
1907           LOG(FATAL) << "Unexpected type conversion from " << input_type
1908                      << " to " << result_type;
1909       }
1910       break;
1911 
1912     case Primitive::kPrimChar:
1913       switch (input_type) {
1914         case Primitive::kPrimBoolean:
1915           // Boolean input is a result of code transformations.
1916         case Primitive::kPrimByte:
1917         case Primitive::kPrimShort:
1918         case Primitive::kPrimInt:
1919           // Processing a Dex `int-to-char' instruction.
1920           if (in.IsRegister()) {
1921             __ movzxw(out.AsRegister<CpuRegister>(), in.AsRegister<CpuRegister>());
1922           } else if (in.IsStackSlot()) {
1923             __ movzxw(out.AsRegister<CpuRegister>(),
1924                       Address(CpuRegister(RSP), in.GetStackIndex()));
1925           } else {
1926             DCHECK(in.GetConstant()->IsIntConstant());
1927             __ movl(out.AsRegister<CpuRegister>(),
1928                     Immediate(static_cast<uint16_t>(in.GetConstant()->AsIntConstant()->GetValue())));
1929           }
1930           break;
1931 
1932         default:
1933           LOG(FATAL) << "Unexpected type conversion from " << input_type
1934                      << " to " << result_type;
1935       }
1936       break;
1937 
1938     case Primitive::kPrimFloat:
1939       switch (input_type) {
1940         case Primitive::kPrimBoolean:
1941           // Boolean input is a result of code transformations.
1942         case Primitive::kPrimByte:
1943         case Primitive::kPrimShort:
1944         case Primitive::kPrimInt:
1945         case Primitive::kPrimChar:
1946           // Processing a Dex `int-to-float' instruction.
1947           if (in.IsRegister()) {
1948             __ cvtsi2ss(out.AsFpuRegister<XmmRegister>(), in.AsRegister<CpuRegister>(), false);
1949           } else if (in.IsConstant()) {
1950             int32_t v = in.GetConstant()->AsIntConstant()->GetValue();
1951             XmmRegister dest = out.AsFpuRegister<XmmRegister>();
1952             if (v == 0) {
1953               __ xorps(dest, dest);
1954             } else {
1955               __ movss(dest, codegen_->LiteralFloatAddress(static_cast<float>(v)));
1956             }
1957           } else {
1958             __ cvtsi2ss(out.AsFpuRegister<XmmRegister>(),
1959                         Address(CpuRegister(RSP), in.GetStackIndex()), false);
1960           }
1961           break;
1962 
1963         case Primitive::kPrimLong:
1964           // Processing a Dex `long-to-float' instruction.
1965           if (in.IsRegister()) {
1966             __ cvtsi2ss(out.AsFpuRegister<XmmRegister>(), in.AsRegister<CpuRegister>(), true);
1967           } else if (in.IsConstant()) {
1968             int64_t v = in.GetConstant()->AsLongConstant()->GetValue();
1969             XmmRegister dest = out.AsFpuRegister<XmmRegister>();
1970             if (v == 0) {
1971               __ xorps(dest, dest);
1972             } else {
1973               __ movss(dest, codegen_->LiteralFloatAddress(static_cast<float>(v)));
1974             }
1975           } else {
1976             __ cvtsi2ss(out.AsFpuRegister<XmmRegister>(),
1977                         Address(CpuRegister(RSP), in.GetStackIndex()), true);
1978           }
1979           break;
1980 
1981         case Primitive::kPrimDouble:
1982           // Processing a Dex `double-to-float' instruction.
1983           if (in.IsFpuRegister()) {
1984             __ cvtsd2ss(out.AsFpuRegister<XmmRegister>(), in.AsFpuRegister<XmmRegister>());
1985           } else if (in.IsConstant()) {
1986             double v = in.GetConstant()->AsDoubleConstant()->GetValue();
1987             XmmRegister dest = out.AsFpuRegister<XmmRegister>();
1988             if (bit_cast<int64_t, double>(v) == 0) {
1989               __ xorps(dest, dest);
1990             } else {
1991               __ movss(dest, codegen_->LiteralFloatAddress(static_cast<float>(v)));
1992             }
1993           } else {
1994             __ cvtsd2ss(out.AsFpuRegister<XmmRegister>(),
1995                         Address(CpuRegister(RSP), in.GetStackIndex()));
1996           }
1997           break;
1998 
1999         default:
2000           LOG(FATAL) << "Unexpected type conversion from " << input_type
2001                      << " to " << result_type;
2002       };
2003       break;
2004 
2005     case Primitive::kPrimDouble:
2006       switch (input_type) {
2007         case Primitive::kPrimBoolean:
2008           // Boolean input is a result of code transformations.
2009         case Primitive::kPrimByte:
2010         case Primitive::kPrimShort:
2011         case Primitive::kPrimInt:
2012         case Primitive::kPrimChar:
2013           // Processing a Dex `int-to-double' instruction.
2014           if (in.IsRegister()) {
2015             __ cvtsi2sd(out.AsFpuRegister<XmmRegister>(), in.AsRegister<CpuRegister>(), false);
2016           } else if (in.IsConstant()) {
2017             int32_t v = in.GetConstant()->AsIntConstant()->GetValue();
2018             XmmRegister dest = out.AsFpuRegister<XmmRegister>();
2019             if (v == 0) {
2020               __ xorpd(dest, dest);
2021             } else {
2022               __ movsd(dest, codegen_->LiteralDoubleAddress(static_cast<double>(v)));
2023             }
2024           } else {
2025             __ cvtsi2sd(out.AsFpuRegister<XmmRegister>(),
2026                         Address(CpuRegister(RSP), in.GetStackIndex()), false);
2027           }
2028           break;
2029 
2030         case Primitive::kPrimLong:
2031           // Processing a Dex `long-to-double' instruction.
2032           if (in.IsRegister()) {
2033             __ cvtsi2sd(out.AsFpuRegister<XmmRegister>(), in.AsRegister<CpuRegister>(), true);
2034           } else if (in.IsConstant()) {
2035             int64_t v = in.GetConstant()->AsLongConstant()->GetValue();
2036             XmmRegister dest = out.AsFpuRegister<XmmRegister>();
2037             if (v == 0) {
2038               __ xorpd(dest, dest);
2039             } else {
2040               __ movsd(dest, codegen_->LiteralDoubleAddress(static_cast<double>(v)));
2041             }
2042           } else {
2043             __ cvtsi2sd(out.AsFpuRegister<XmmRegister>(),
2044                         Address(CpuRegister(RSP), in.GetStackIndex()), true);
2045           }
2046           break;
2047 
2048         case Primitive::kPrimFloat:
2049           // Processing a Dex `float-to-double' instruction.
2050           if (in.IsFpuRegister()) {
2051             __ cvtss2sd(out.AsFpuRegister<XmmRegister>(), in.AsFpuRegister<XmmRegister>());
2052           } else if (in.IsConstant()) {
2053             float v = in.GetConstant()->AsFloatConstant()->GetValue();
2054             XmmRegister dest = out.AsFpuRegister<XmmRegister>();
2055             if (bit_cast<int32_t, float>(v) == 0) {
2056               __ xorpd(dest, dest);
2057             } else {
2058               __ movsd(dest, codegen_->LiteralDoubleAddress(static_cast<double>(v)));
2059             }
2060           } else {
2061             __ cvtss2sd(out.AsFpuRegister<XmmRegister>(),
2062                         Address(CpuRegister(RSP), in.GetStackIndex()));
2063           }
2064           break;
2065 
2066         default:
2067           LOG(FATAL) << "Unexpected type conversion from " << input_type
2068                      << " to " << result_type;
2069       };
2070       break;
2071 
2072     default:
2073       LOG(FATAL) << "Unexpected type conversion from " << input_type
2074                  << " to " << result_type;
2075   }
2076 }
2077 
VisitAdd(HAdd * add)2078 void LocationsBuilderX86_64::VisitAdd(HAdd* add) {
2079   LocationSummary* locations =
2080       new (GetGraph()->GetArena()) LocationSummary(add, LocationSummary::kNoCall);
2081   switch (add->GetResultType()) {
2082     case Primitive::kPrimInt: {
2083       locations->SetInAt(0, Location::RequiresRegister());
2084       locations->SetInAt(1, Location::RegisterOrConstant(add->InputAt(1)));
2085       locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
2086       break;
2087     }
2088 
2089     case Primitive::kPrimLong: {
2090       locations->SetInAt(0, Location::RequiresRegister());
2091       // We can use a leaq or addq if the constant can fit in an immediate.
2092       locations->SetInAt(1, Location::RegisterOrInt32LongConstant(add->InputAt(1)));
2093       locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
2094       break;
2095     }
2096 
2097     case Primitive::kPrimDouble:
2098     case Primitive::kPrimFloat: {
2099       locations->SetInAt(0, Location::RequiresFpuRegister());
2100       locations->SetInAt(1, Location::Any());
2101       locations->SetOut(Location::SameAsFirstInput());
2102       break;
2103     }
2104 
2105     default:
2106       LOG(FATAL) << "Unexpected add type " << add->GetResultType();
2107   }
2108 }
2109 
VisitAdd(HAdd * add)2110 void InstructionCodeGeneratorX86_64::VisitAdd(HAdd* add) {
2111   LocationSummary* locations = add->GetLocations();
2112   Location first = locations->InAt(0);
2113   Location second = locations->InAt(1);
2114   Location out = locations->Out();
2115 
2116   switch (add->GetResultType()) {
2117     case Primitive::kPrimInt: {
2118       if (second.IsRegister()) {
2119         if (out.AsRegister<Register>() == first.AsRegister<Register>()) {
2120           __ addl(out.AsRegister<CpuRegister>(), second.AsRegister<CpuRegister>());
2121         } else {
2122           __ leal(out.AsRegister<CpuRegister>(), Address(
2123               first.AsRegister<CpuRegister>(), second.AsRegister<CpuRegister>(), TIMES_1, 0));
2124         }
2125       } else if (second.IsConstant()) {
2126         if (out.AsRegister<Register>() == first.AsRegister<Register>()) {
2127           __ addl(out.AsRegister<CpuRegister>(),
2128                   Immediate(second.GetConstant()->AsIntConstant()->GetValue()));
2129         } else {
2130           __ leal(out.AsRegister<CpuRegister>(), Address(
2131               first.AsRegister<CpuRegister>(), second.GetConstant()->AsIntConstant()->GetValue()));
2132         }
2133       } else {
2134         DCHECK(first.Equals(locations->Out()));
2135         __ addl(first.AsRegister<CpuRegister>(), Address(CpuRegister(RSP), second.GetStackIndex()));
2136       }
2137       break;
2138     }
2139 
2140     case Primitive::kPrimLong: {
2141       if (second.IsRegister()) {
2142         if (out.AsRegister<Register>() == first.AsRegister<Register>()) {
2143           __ addq(out.AsRegister<CpuRegister>(), second.AsRegister<CpuRegister>());
2144         } else {
2145           __ leaq(out.AsRegister<CpuRegister>(), Address(
2146               first.AsRegister<CpuRegister>(), second.AsRegister<CpuRegister>(), TIMES_1, 0));
2147         }
2148       } else {
2149         DCHECK(second.IsConstant());
2150         int64_t value = second.GetConstant()->AsLongConstant()->GetValue();
2151         int32_t int32_value = Low32Bits(value);
2152         DCHECK_EQ(int32_value, value);
2153         if (out.AsRegister<Register>() == first.AsRegister<Register>()) {
2154           __ addq(out.AsRegister<CpuRegister>(), Immediate(int32_value));
2155         } else {
2156           __ leaq(out.AsRegister<CpuRegister>(), Address(
2157               first.AsRegister<CpuRegister>(), int32_value));
2158         }
2159       }
2160       break;
2161     }
2162 
2163     case Primitive::kPrimFloat: {
2164       if (second.IsFpuRegister()) {
2165         __ addss(first.AsFpuRegister<XmmRegister>(), second.AsFpuRegister<XmmRegister>());
2166       } else if (second.IsConstant()) {
2167         __ addss(first.AsFpuRegister<XmmRegister>(),
2168                  codegen_->LiteralFloatAddress(second.GetConstant()->AsFloatConstant()->GetValue()));
2169       } else {
2170         DCHECK(second.IsStackSlot());
2171         __ addss(first.AsFpuRegister<XmmRegister>(),
2172                  Address(CpuRegister(RSP), second.GetStackIndex()));
2173       }
2174       break;
2175     }
2176 
2177     case Primitive::kPrimDouble: {
2178       if (second.IsFpuRegister()) {
2179         __ addsd(first.AsFpuRegister<XmmRegister>(), second.AsFpuRegister<XmmRegister>());
2180       } else if (second.IsConstant()) {
2181         __ addsd(first.AsFpuRegister<XmmRegister>(),
2182                  codegen_->LiteralDoubleAddress(second.GetConstant()->AsDoubleConstant()->GetValue()));
2183       } else {
2184         DCHECK(second.IsDoubleStackSlot());
2185         __ addsd(first.AsFpuRegister<XmmRegister>(),
2186                  Address(CpuRegister(RSP), second.GetStackIndex()));
2187       }
2188       break;
2189     }
2190 
2191     default:
2192       LOG(FATAL) << "Unexpected add type " << add->GetResultType();
2193   }
2194 }
2195 
VisitSub(HSub * sub)2196 void LocationsBuilderX86_64::VisitSub(HSub* sub) {
2197   LocationSummary* locations =
2198       new (GetGraph()->GetArena()) LocationSummary(sub, LocationSummary::kNoCall);
2199   switch (sub->GetResultType()) {
2200     case Primitive::kPrimInt: {
2201       locations->SetInAt(0, Location::RequiresRegister());
2202       locations->SetInAt(1, Location::Any());
2203       locations->SetOut(Location::SameAsFirstInput());
2204       break;
2205     }
2206     case Primitive::kPrimLong: {
2207       locations->SetInAt(0, Location::RequiresRegister());
2208       locations->SetInAt(1, Location::RegisterOrInt32LongConstant(sub->InputAt(1)));
2209       locations->SetOut(Location::SameAsFirstInput());
2210       break;
2211     }
2212     case Primitive::kPrimFloat:
2213     case Primitive::kPrimDouble: {
2214       locations->SetInAt(0, Location::RequiresFpuRegister());
2215       locations->SetInAt(1, Location::Any());
2216       locations->SetOut(Location::SameAsFirstInput());
2217       break;
2218     }
2219     default:
2220       LOG(FATAL) << "Unexpected sub type " << sub->GetResultType();
2221   }
2222 }
2223 
VisitSub(HSub * sub)2224 void InstructionCodeGeneratorX86_64::VisitSub(HSub* sub) {
2225   LocationSummary* locations = sub->GetLocations();
2226   Location first = locations->InAt(0);
2227   Location second = locations->InAt(1);
2228   DCHECK(first.Equals(locations->Out()));
2229   switch (sub->GetResultType()) {
2230     case Primitive::kPrimInt: {
2231       if (second.IsRegister()) {
2232         __ subl(first.AsRegister<CpuRegister>(), second.AsRegister<CpuRegister>());
2233       } else if (second.IsConstant()) {
2234         Immediate imm(second.GetConstant()->AsIntConstant()->GetValue());
2235         __ subl(first.AsRegister<CpuRegister>(), imm);
2236       } else {
2237         __ subl(first.AsRegister<CpuRegister>(), Address(CpuRegister(RSP), second.GetStackIndex()));
2238       }
2239       break;
2240     }
2241     case Primitive::kPrimLong: {
2242       if (second.IsConstant()) {
2243         int64_t value = second.GetConstant()->AsLongConstant()->GetValue();
2244         DCHECK(IsInt<32>(value));
2245         __ subq(first.AsRegister<CpuRegister>(), Immediate(static_cast<int32_t>(value)));
2246       } else {
2247         __ subq(first.AsRegister<CpuRegister>(), second.AsRegister<CpuRegister>());
2248       }
2249       break;
2250     }
2251 
2252     case Primitive::kPrimFloat: {
2253       if (second.IsFpuRegister()) {
2254         __ subss(first.AsFpuRegister<XmmRegister>(), second.AsFpuRegister<XmmRegister>());
2255       } else if (second.IsConstant()) {
2256         __ subss(first.AsFpuRegister<XmmRegister>(),
2257                  codegen_->LiteralFloatAddress(second.GetConstant()->AsFloatConstant()->GetValue()));
2258       } else {
2259         DCHECK(second.IsStackSlot());
2260         __ subss(first.AsFpuRegister<XmmRegister>(),
2261                  Address(CpuRegister(RSP), second.GetStackIndex()));
2262       }
2263       break;
2264     }
2265 
2266     case Primitive::kPrimDouble: {
2267       if (second.IsFpuRegister()) {
2268         __ subsd(first.AsFpuRegister<XmmRegister>(), second.AsFpuRegister<XmmRegister>());
2269       } else if (second.IsConstant()) {
2270         __ subsd(first.AsFpuRegister<XmmRegister>(),
2271                  codegen_->LiteralDoubleAddress(second.GetConstant()->AsDoubleConstant()->GetValue()));
2272       } else {
2273         DCHECK(second.IsDoubleStackSlot());
2274         __ subsd(first.AsFpuRegister<XmmRegister>(),
2275                  Address(CpuRegister(RSP), second.GetStackIndex()));
2276       }
2277       break;
2278     }
2279 
2280     default:
2281       LOG(FATAL) << "Unexpected sub type " << sub->GetResultType();
2282   }
2283 }
2284 
VisitMul(HMul * mul)2285 void LocationsBuilderX86_64::VisitMul(HMul* mul) {
2286   LocationSummary* locations =
2287       new (GetGraph()->GetArena()) LocationSummary(mul, LocationSummary::kNoCall);
2288   switch (mul->GetResultType()) {
2289     case Primitive::kPrimInt: {
2290       locations->SetInAt(0, Location::RequiresRegister());
2291       locations->SetInAt(1, Location::Any());
2292       locations->SetOut(Location::SameAsFirstInput());
2293       break;
2294     }
2295     case Primitive::kPrimLong: {
2296       locations->SetInAt(0, Location::RequiresRegister());
2297       locations->SetInAt(1, Location::RegisterOrInt32LongConstant(mul->InputAt(1)));
2298       if (locations->InAt(1).IsConstant()) {
2299         // Can use 3 operand multiply.
2300         locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
2301       } else {
2302         locations->SetOut(Location::SameAsFirstInput());
2303       }
2304       break;
2305     }
2306     case Primitive::kPrimFloat:
2307     case Primitive::kPrimDouble: {
2308       locations->SetInAt(0, Location::RequiresFpuRegister());
2309       locations->SetInAt(1, Location::Any());
2310       locations->SetOut(Location::SameAsFirstInput());
2311       break;
2312     }
2313 
2314     default:
2315       LOG(FATAL) << "Unexpected mul type " << mul->GetResultType();
2316   }
2317 }
2318 
VisitMul(HMul * mul)2319 void InstructionCodeGeneratorX86_64::VisitMul(HMul* mul) {
2320   LocationSummary* locations = mul->GetLocations();
2321   Location first = locations->InAt(0);
2322   Location second = locations->InAt(1);
2323   switch (mul->GetResultType()) {
2324     case Primitive::kPrimInt: {
2325       DCHECK(first.Equals(locations->Out()));
2326       if (second.IsRegister()) {
2327         __ imull(first.AsRegister<CpuRegister>(), second.AsRegister<CpuRegister>());
2328       } else if (second.IsConstant()) {
2329         Immediate imm(second.GetConstant()->AsIntConstant()->GetValue());
2330         __ imull(first.AsRegister<CpuRegister>(), imm);
2331       } else {
2332         DCHECK(second.IsStackSlot());
2333         __ imull(first.AsRegister<CpuRegister>(),
2334                  Address(CpuRegister(RSP), second.GetStackIndex()));
2335       }
2336       break;
2337     }
2338     case Primitive::kPrimLong: {
2339       if (second.IsConstant()) {
2340         int64_t value = second.GetConstant()->AsLongConstant()->GetValue();
2341         DCHECK(IsInt<32>(value));
2342         __ imulq(locations->Out().AsRegister<CpuRegister>(),
2343                  first.AsRegister<CpuRegister>(),
2344                  Immediate(static_cast<int32_t>(value)));
2345       } else {
2346         DCHECK(first.Equals(locations->Out()));
2347         __ imulq(first.AsRegister<CpuRegister>(), second.AsRegister<CpuRegister>());
2348       }
2349       break;
2350     }
2351 
2352     case Primitive::kPrimFloat: {
2353       DCHECK(first.Equals(locations->Out()));
2354       if (second.IsFpuRegister()) {
2355         __ mulss(first.AsFpuRegister<XmmRegister>(), second.AsFpuRegister<XmmRegister>());
2356       } else if (second.IsConstant()) {
2357         __ mulss(first.AsFpuRegister<XmmRegister>(),
2358                  codegen_->LiteralFloatAddress(second.GetConstant()->AsFloatConstant()->GetValue()));
2359       } else {
2360         DCHECK(second.IsStackSlot());
2361         __ mulss(first.AsFpuRegister<XmmRegister>(),
2362                  Address(CpuRegister(RSP), second.GetStackIndex()));
2363       }
2364       break;
2365     }
2366 
2367     case Primitive::kPrimDouble: {
2368       DCHECK(first.Equals(locations->Out()));
2369       if (second.IsFpuRegister()) {
2370         __ mulsd(first.AsFpuRegister<XmmRegister>(), second.AsFpuRegister<XmmRegister>());
2371       } else if (second.IsConstant()) {
2372         __ mulsd(first.AsFpuRegister<XmmRegister>(),
2373                  codegen_->LiteralDoubleAddress(second.GetConstant()->AsDoubleConstant()->GetValue()));
2374       } else {
2375         DCHECK(second.IsDoubleStackSlot());
2376         __ mulsd(first.AsFpuRegister<XmmRegister>(),
2377                  Address(CpuRegister(RSP), second.GetStackIndex()));
2378       }
2379       break;
2380     }
2381 
2382     default:
2383       LOG(FATAL) << "Unexpected mul type " << mul->GetResultType();
2384   }
2385 }
2386 
PushOntoFPStack(Location source,uint32_t temp_offset,uint32_t stack_adjustment,bool is_float)2387 void InstructionCodeGeneratorX86_64::PushOntoFPStack(Location source, uint32_t temp_offset,
2388                                                      uint32_t stack_adjustment, bool is_float) {
2389   if (source.IsStackSlot()) {
2390     DCHECK(is_float);
2391     __ flds(Address(CpuRegister(RSP), source.GetStackIndex() + stack_adjustment));
2392   } else if (source.IsDoubleStackSlot()) {
2393     DCHECK(!is_float);
2394     __ fldl(Address(CpuRegister(RSP), source.GetStackIndex() + stack_adjustment));
2395   } else {
2396     // Write the value to the temporary location on the stack and load to FP stack.
2397     if (is_float) {
2398       Location stack_temp = Location::StackSlot(temp_offset);
2399       codegen_->Move(stack_temp, source);
2400       __ flds(Address(CpuRegister(RSP), temp_offset));
2401     } else {
2402       Location stack_temp = Location::DoubleStackSlot(temp_offset);
2403       codegen_->Move(stack_temp, source);
2404       __ fldl(Address(CpuRegister(RSP), temp_offset));
2405     }
2406   }
2407 }
2408 
GenerateRemFP(HRem * rem)2409 void InstructionCodeGeneratorX86_64::GenerateRemFP(HRem *rem) {
2410   Primitive::Type type = rem->GetResultType();
2411   bool is_float = type == Primitive::kPrimFloat;
2412   size_t elem_size = Primitive::ComponentSize(type);
2413   LocationSummary* locations = rem->GetLocations();
2414   Location first = locations->InAt(0);
2415   Location second = locations->InAt(1);
2416   Location out = locations->Out();
2417 
2418   // Create stack space for 2 elements.
2419   // TODO: enhance register allocator to ask for stack temporaries.
2420   __ subq(CpuRegister(RSP), Immediate(2 * elem_size));
2421 
2422   // Load the values to the FP stack in reverse order, using temporaries if needed.
2423   PushOntoFPStack(second, elem_size, 2 * elem_size, is_float);
2424   PushOntoFPStack(first, 0, 2 * elem_size, is_float);
2425 
2426   // Loop doing FPREM until we stabilize.
2427   Label retry;
2428   __ Bind(&retry);
2429   __ fprem();
2430 
2431   // Move FP status to AX.
2432   __ fstsw();
2433 
2434   // And see if the argument reduction is complete. This is signaled by the
2435   // C2 FPU flag bit set to 0.
2436   __ andl(CpuRegister(RAX), Immediate(kC2ConditionMask));
2437   __ j(kNotEqual, &retry);
2438 
2439   // We have settled on the final value. Retrieve it into an XMM register.
2440   // Store FP top of stack to real stack.
2441   if (is_float) {
2442     __ fsts(Address(CpuRegister(RSP), 0));
2443   } else {
2444     __ fstl(Address(CpuRegister(RSP), 0));
2445   }
2446 
2447   // Pop the 2 items from the FP stack.
2448   __ fucompp();
2449 
2450   // Load the value from the stack into an XMM register.
2451   DCHECK(out.IsFpuRegister()) << out;
2452   if (is_float) {
2453     __ movss(out.AsFpuRegister<XmmRegister>(), Address(CpuRegister(RSP), 0));
2454   } else {
2455     __ movsd(out.AsFpuRegister<XmmRegister>(), Address(CpuRegister(RSP), 0));
2456   }
2457 
2458   // And remove the temporary stack space we allocated.
2459   __ addq(CpuRegister(RSP), Immediate(2 * elem_size));
2460 }
2461 
DivRemOneOrMinusOne(HBinaryOperation * instruction)2462 void InstructionCodeGeneratorX86_64::DivRemOneOrMinusOne(HBinaryOperation* instruction) {
2463   DCHECK(instruction->IsDiv() || instruction->IsRem());
2464 
2465   LocationSummary* locations = instruction->GetLocations();
2466   Location second = locations->InAt(1);
2467   DCHECK(second.IsConstant());
2468 
2469   CpuRegister output_register = locations->Out().AsRegister<CpuRegister>();
2470   CpuRegister input_register = locations->InAt(0).AsRegister<CpuRegister>();
2471   int64_t imm = Int64FromConstant(second.GetConstant());
2472 
2473   DCHECK(imm == 1 || imm == -1);
2474 
2475   switch (instruction->GetResultType()) {
2476     case Primitive::kPrimInt: {
2477       if (instruction->IsRem()) {
2478         __ xorl(output_register, output_register);
2479       } else {
2480         __ movl(output_register, input_register);
2481         if (imm == -1) {
2482           __ negl(output_register);
2483         }
2484       }
2485       break;
2486     }
2487 
2488     case Primitive::kPrimLong: {
2489       if (instruction->IsRem()) {
2490         __ xorl(output_register, output_register);
2491       } else {
2492         __ movq(output_register, input_register);
2493         if (imm == -1) {
2494           __ negq(output_register);
2495         }
2496       }
2497       break;
2498     }
2499 
2500     default:
2501       LOG(FATAL) << "Unexpected type for div by (-)1 " << instruction->GetResultType();
2502   }
2503 }
2504 
DivByPowerOfTwo(HDiv * instruction)2505 void InstructionCodeGeneratorX86_64::DivByPowerOfTwo(HDiv* instruction) {
2506   LocationSummary* locations = instruction->GetLocations();
2507   Location second = locations->InAt(1);
2508 
2509   CpuRegister output_register = locations->Out().AsRegister<CpuRegister>();
2510   CpuRegister numerator = locations->InAt(0).AsRegister<CpuRegister>();
2511 
2512   int64_t imm = Int64FromConstant(second.GetConstant());
2513 
2514   DCHECK(IsPowerOfTwo(std::abs(imm)));
2515 
2516   CpuRegister tmp = locations->GetTemp(0).AsRegister<CpuRegister>();
2517 
2518   if (instruction->GetResultType() == Primitive::kPrimInt) {
2519     __ leal(tmp, Address(numerator, std::abs(imm) - 1));
2520     __ testl(numerator, numerator);
2521     __ cmov(kGreaterEqual, tmp, numerator);
2522     int shift = CTZ(imm);
2523     __ sarl(tmp, Immediate(shift));
2524 
2525     if (imm < 0) {
2526       __ negl(tmp);
2527     }
2528 
2529     __ movl(output_register, tmp);
2530   } else {
2531     DCHECK_EQ(instruction->GetResultType(), Primitive::kPrimLong);
2532     CpuRegister rdx = locations->GetTemp(0).AsRegister<CpuRegister>();
2533 
2534     codegen_->Load64BitValue(rdx, std::abs(imm) - 1);
2535     __ addq(rdx, numerator);
2536     __ testq(numerator, numerator);
2537     __ cmov(kGreaterEqual, rdx, numerator);
2538     int shift = CTZ(imm);
2539     __ sarq(rdx, Immediate(shift));
2540 
2541     if (imm < 0) {
2542       __ negq(rdx);
2543     }
2544 
2545     __ movq(output_register, rdx);
2546   }
2547 }
2548 
GenerateDivRemWithAnyConstant(HBinaryOperation * instruction)2549 void InstructionCodeGeneratorX86_64::GenerateDivRemWithAnyConstant(HBinaryOperation* instruction) {
2550   DCHECK(instruction->IsDiv() || instruction->IsRem());
2551 
2552   LocationSummary* locations = instruction->GetLocations();
2553   Location second = locations->InAt(1);
2554 
2555   CpuRegister numerator = instruction->IsDiv() ? locations->GetTemp(1).AsRegister<CpuRegister>()
2556       : locations->GetTemp(0).AsRegister<CpuRegister>();
2557   CpuRegister eax = locations->InAt(0).AsRegister<CpuRegister>();
2558   CpuRegister edx = instruction->IsDiv() ? locations->GetTemp(0).AsRegister<CpuRegister>()
2559       : locations->Out().AsRegister<CpuRegister>();
2560   CpuRegister out = locations->Out().AsRegister<CpuRegister>();
2561 
2562   DCHECK_EQ(RAX, eax.AsRegister());
2563   DCHECK_EQ(RDX, edx.AsRegister());
2564   if (instruction->IsDiv()) {
2565     DCHECK_EQ(RAX, out.AsRegister());
2566   } else {
2567     DCHECK_EQ(RDX, out.AsRegister());
2568   }
2569 
2570   int64_t magic;
2571   int shift;
2572 
2573   // TODO: can these branches be written as one?
2574   if (instruction->GetResultType() == Primitive::kPrimInt) {
2575     int imm = second.GetConstant()->AsIntConstant()->GetValue();
2576 
2577     CalculateMagicAndShiftForDivRem(imm, false /* is_long */, &magic, &shift);
2578 
2579     __ movl(numerator, eax);
2580 
2581     Label no_div;
2582     Label end;
2583     __ testl(eax, eax);
2584     __ j(kNotEqual, &no_div);
2585 
2586     __ xorl(out, out);
2587     __ jmp(&end);
2588 
2589     __ Bind(&no_div);
2590 
2591     __ movl(eax, Immediate(magic));
2592     __ imull(numerator);
2593 
2594     if (imm > 0 && magic < 0) {
2595       __ addl(edx, numerator);
2596     } else if (imm < 0 && magic > 0) {
2597       __ subl(edx, numerator);
2598     }
2599 
2600     if (shift != 0) {
2601       __ sarl(edx, Immediate(shift));
2602     }
2603 
2604     __ movl(eax, edx);
2605     __ shrl(edx, Immediate(31));
2606     __ addl(edx, eax);
2607 
2608     if (instruction->IsRem()) {
2609       __ movl(eax, numerator);
2610       __ imull(edx, Immediate(imm));
2611       __ subl(eax, edx);
2612       __ movl(edx, eax);
2613     } else {
2614       __ movl(eax, edx);
2615     }
2616     __ Bind(&end);
2617   } else {
2618     int64_t imm = second.GetConstant()->AsLongConstant()->GetValue();
2619 
2620     DCHECK_EQ(instruction->GetResultType(), Primitive::kPrimLong);
2621 
2622     CpuRegister rax = eax;
2623     CpuRegister rdx = edx;
2624 
2625     CalculateMagicAndShiftForDivRem(imm, true /* is_long */, &magic, &shift);
2626 
2627     // Save the numerator.
2628     __ movq(numerator, rax);
2629 
2630     // RAX = magic
2631     codegen_->Load64BitValue(rax, magic);
2632 
2633     // RDX:RAX = magic * numerator
2634     __ imulq(numerator);
2635 
2636     if (imm > 0 && magic < 0) {
2637       // RDX += numerator
2638       __ addq(rdx, numerator);
2639     } else if (imm < 0 && magic > 0) {
2640       // RDX -= numerator
2641       __ subq(rdx, numerator);
2642     }
2643 
2644     // Shift if needed.
2645     if (shift != 0) {
2646       __ sarq(rdx, Immediate(shift));
2647     }
2648 
2649     // RDX += 1 if RDX < 0
2650     __ movq(rax, rdx);
2651     __ shrq(rdx, Immediate(63));
2652     __ addq(rdx, rax);
2653 
2654     if (instruction->IsRem()) {
2655       __ movq(rax, numerator);
2656 
2657       if (IsInt<32>(imm)) {
2658         __ imulq(rdx, Immediate(static_cast<int32_t>(imm)));
2659       } else {
2660         __ imulq(rdx, codegen_->LiteralInt64Address(imm));
2661       }
2662 
2663       __ subq(rax, rdx);
2664       __ movq(rdx, rax);
2665     } else {
2666       __ movq(rax, rdx);
2667     }
2668   }
2669 }
2670 
GenerateDivRemIntegral(HBinaryOperation * instruction)2671 void InstructionCodeGeneratorX86_64::GenerateDivRemIntegral(HBinaryOperation* instruction) {
2672   DCHECK(instruction->IsDiv() || instruction->IsRem());
2673   Primitive::Type type = instruction->GetResultType();
2674   DCHECK(type == Primitive::kPrimInt || Primitive::kPrimLong);
2675 
2676   bool is_div = instruction->IsDiv();
2677   LocationSummary* locations = instruction->GetLocations();
2678 
2679   CpuRegister out = locations->Out().AsRegister<CpuRegister>();
2680   Location second = locations->InAt(1);
2681 
2682   DCHECK_EQ(RAX, locations->InAt(0).AsRegister<CpuRegister>().AsRegister());
2683   DCHECK_EQ(is_div ? RAX : RDX, out.AsRegister());
2684 
2685   if (second.IsConstant()) {
2686     int64_t imm = Int64FromConstant(second.GetConstant());
2687 
2688     if (imm == 0) {
2689       // Do not generate anything. DivZeroCheck would prevent any code to be executed.
2690     } else if (imm == 1 || imm == -1) {
2691       DivRemOneOrMinusOne(instruction);
2692     } else if (instruction->IsDiv() && IsPowerOfTwo(std::abs(imm))) {
2693       DivByPowerOfTwo(instruction->AsDiv());
2694     } else {
2695       DCHECK(imm <= -2 || imm >= 2);
2696       GenerateDivRemWithAnyConstant(instruction);
2697     }
2698   } else {
2699     SlowPathCodeX86_64* slow_path =
2700         new (GetGraph()->GetArena()) DivRemMinusOneSlowPathX86_64(
2701             out.AsRegister(), type, is_div);
2702     codegen_->AddSlowPath(slow_path);
2703 
2704     CpuRegister second_reg = second.AsRegister<CpuRegister>();
2705     // 0x80000000(00000000)/-1 triggers an arithmetic exception!
2706     // Dividing by -1 is actually negation and -0x800000000(00000000) = 0x80000000(00000000)
2707     // so it's safe to just use negl instead of more complex comparisons.
2708     if (type == Primitive::kPrimInt) {
2709       __ cmpl(second_reg, Immediate(-1));
2710       __ j(kEqual, slow_path->GetEntryLabel());
2711       // edx:eax <- sign-extended of eax
2712       __ cdq();
2713       // eax = quotient, edx = remainder
2714       __ idivl(second_reg);
2715     } else {
2716       __ cmpq(second_reg, Immediate(-1));
2717       __ j(kEqual, slow_path->GetEntryLabel());
2718       // rdx:rax <- sign-extended of rax
2719       __ cqo();
2720       // rax = quotient, rdx = remainder
2721       __ idivq(second_reg);
2722     }
2723     __ Bind(slow_path->GetExitLabel());
2724   }
2725 }
2726 
VisitDiv(HDiv * div)2727 void LocationsBuilderX86_64::VisitDiv(HDiv* div) {
2728   LocationSummary* locations =
2729       new (GetGraph()->GetArena()) LocationSummary(div, LocationSummary::kNoCall);
2730   switch (div->GetResultType()) {
2731     case Primitive::kPrimInt:
2732     case Primitive::kPrimLong: {
2733       locations->SetInAt(0, Location::RegisterLocation(RAX));
2734       locations->SetInAt(1, Location::RegisterOrConstant(div->InputAt(1)));
2735       locations->SetOut(Location::SameAsFirstInput());
2736       // Intel uses edx:eax as the dividend.
2737       locations->AddTemp(Location::RegisterLocation(RDX));
2738       // We need to save the numerator while we tweak rax and rdx. As we are using imul in a way
2739       // which enforces results to be in RAX and RDX, things are simpler if we use RDX also as
2740       // output and request another temp.
2741       if (div->InputAt(1)->IsConstant()) {
2742         locations->AddTemp(Location::RequiresRegister());
2743       }
2744       break;
2745     }
2746 
2747     case Primitive::kPrimFloat:
2748     case Primitive::kPrimDouble: {
2749       locations->SetInAt(0, Location::RequiresFpuRegister());
2750       locations->SetInAt(1, Location::Any());
2751       locations->SetOut(Location::SameAsFirstInput());
2752       break;
2753     }
2754 
2755     default:
2756       LOG(FATAL) << "Unexpected div type " << div->GetResultType();
2757   }
2758 }
2759 
VisitDiv(HDiv * div)2760 void InstructionCodeGeneratorX86_64::VisitDiv(HDiv* div) {
2761   LocationSummary* locations = div->GetLocations();
2762   Location first = locations->InAt(0);
2763   Location second = locations->InAt(1);
2764   DCHECK(first.Equals(locations->Out()));
2765 
2766   Primitive::Type type = div->GetResultType();
2767   switch (type) {
2768     case Primitive::kPrimInt:
2769     case Primitive::kPrimLong: {
2770       GenerateDivRemIntegral(div);
2771       break;
2772     }
2773 
2774     case Primitive::kPrimFloat: {
2775       if (second.IsFpuRegister()) {
2776         __ divss(first.AsFpuRegister<XmmRegister>(), second.AsFpuRegister<XmmRegister>());
2777       } else if (second.IsConstant()) {
2778         __ divss(first.AsFpuRegister<XmmRegister>(),
2779                  codegen_->LiteralFloatAddress(second.GetConstant()->AsFloatConstant()->GetValue()));
2780       } else {
2781         DCHECK(second.IsStackSlot());
2782         __ divss(first.AsFpuRegister<XmmRegister>(),
2783                  Address(CpuRegister(RSP), second.GetStackIndex()));
2784       }
2785       break;
2786     }
2787 
2788     case Primitive::kPrimDouble: {
2789       if (second.IsFpuRegister()) {
2790         __ divsd(first.AsFpuRegister<XmmRegister>(), second.AsFpuRegister<XmmRegister>());
2791       } else if (second.IsConstant()) {
2792         __ divsd(first.AsFpuRegister<XmmRegister>(),
2793                  codegen_->LiteralDoubleAddress(second.GetConstant()->AsDoubleConstant()->GetValue()));
2794       } else {
2795         DCHECK(second.IsDoubleStackSlot());
2796         __ divsd(first.AsFpuRegister<XmmRegister>(),
2797                  Address(CpuRegister(RSP), second.GetStackIndex()));
2798       }
2799       break;
2800     }
2801 
2802     default:
2803       LOG(FATAL) << "Unexpected div type " << div->GetResultType();
2804   }
2805 }
2806 
VisitRem(HRem * rem)2807 void LocationsBuilderX86_64::VisitRem(HRem* rem) {
2808   Primitive::Type type = rem->GetResultType();
2809   LocationSummary* locations =
2810     new (GetGraph()->GetArena()) LocationSummary(rem, LocationSummary::kNoCall);
2811 
2812   switch (type) {
2813     case Primitive::kPrimInt:
2814     case Primitive::kPrimLong: {
2815       locations->SetInAt(0, Location::RegisterLocation(RAX));
2816       locations->SetInAt(1, Location::RegisterOrConstant(rem->InputAt(1)));
2817       // Intel uses rdx:rax as the dividend and puts the remainder in rdx
2818       locations->SetOut(Location::RegisterLocation(RDX));
2819       // We need to save the numerator while we tweak eax and edx. As we are using imul in a way
2820       // which enforces results to be in RAX and RDX, things are simpler if we use EAX also as
2821       // output and request another temp.
2822       if (rem->InputAt(1)->IsConstant()) {
2823         locations->AddTemp(Location::RequiresRegister());
2824       }
2825       break;
2826     }
2827 
2828     case Primitive::kPrimFloat:
2829     case Primitive::kPrimDouble: {
2830       locations->SetInAt(0, Location::Any());
2831       locations->SetInAt(1, Location::Any());
2832       locations->SetOut(Location::RequiresFpuRegister());
2833       locations->AddTemp(Location::RegisterLocation(RAX));
2834       break;
2835     }
2836 
2837     default:
2838       LOG(FATAL) << "Unexpected rem type " << type;
2839   }
2840 }
2841 
VisitRem(HRem * rem)2842 void InstructionCodeGeneratorX86_64::VisitRem(HRem* rem) {
2843   Primitive::Type type = rem->GetResultType();
2844   switch (type) {
2845     case Primitive::kPrimInt:
2846     case Primitive::kPrimLong: {
2847       GenerateDivRemIntegral(rem);
2848       break;
2849     }
2850     case Primitive::kPrimFloat:
2851     case Primitive::kPrimDouble: {
2852       GenerateRemFP(rem);
2853       break;
2854     }
2855     default:
2856       LOG(FATAL) << "Unexpected rem type " << rem->GetResultType();
2857   }
2858 }
2859 
VisitDivZeroCheck(HDivZeroCheck * instruction)2860 void LocationsBuilderX86_64::VisitDivZeroCheck(HDivZeroCheck* instruction) {
2861   LocationSummary* locations =
2862       new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
2863   locations->SetInAt(0, Location::Any());
2864   if (instruction->HasUses()) {
2865     locations->SetOut(Location::SameAsFirstInput());
2866   }
2867 }
2868 
VisitDivZeroCheck(HDivZeroCheck * instruction)2869 void InstructionCodeGeneratorX86_64::VisitDivZeroCheck(HDivZeroCheck* instruction) {
2870   SlowPathCodeX86_64* slow_path =
2871       new (GetGraph()->GetArena()) DivZeroCheckSlowPathX86_64(instruction);
2872   codegen_->AddSlowPath(slow_path);
2873 
2874   LocationSummary* locations = instruction->GetLocations();
2875   Location value = locations->InAt(0);
2876 
2877   switch (instruction->GetType()) {
2878     case Primitive::kPrimInt: {
2879       if (value.IsRegister()) {
2880         __ testl(value.AsRegister<CpuRegister>(), value.AsRegister<CpuRegister>());
2881         __ j(kEqual, slow_path->GetEntryLabel());
2882       } else if (value.IsStackSlot()) {
2883         __ cmpl(Address(CpuRegister(RSP), value.GetStackIndex()), Immediate(0));
2884         __ j(kEqual, slow_path->GetEntryLabel());
2885       } else {
2886         DCHECK(value.IsConstant()) << value;
2887         if (value.GetConstant()->AsIntConstant()->GetValue() == 0) {
2888         __ jmp(slow_path->GetEntryLabel());
2889         }
2890       }
2891       break;
2892     }
2893     case Primitive::kPrimLong: {
2894       if (value.IsRegister()) {
2895         __ testq(value.AsRegister<CpuRegister>(), value.AsRegister<CpuRegister>());
2896         __ j(kEqual, slow_path->GetEntryLabel());
2897       } else if (value.IsDoubleStackSlot()) {
2898         __ cmpq(Address(CpuRegister(RSP), value.GetStackIndex()), Immediate(0));
2899         __ j(kEqual, slow_path->GetEntryLabel());
2900       } else {
2901         DCHECK(value.IsConstant()) << value;
2902         if (value.GetConstant()->AsLongConstant()->GetValue() == 0) {
2903         __ jmp(slow_path->GetEntryLabel());
2904         }
2905       }
2906       break;
2907     }
2908     default:
2909       LOG(FATAL) << "Unexpected type for HDivZeroCheck " << instruction->GetType();
2910   }
2911 }
2912 
HandleShift(HBinaryOperation * op)2913 void LocationsBuilderX86_64::HandleShift(HBinaryOperation* op) {
2914   DCHECK(op->IsShl() || op->IsShr() || op->IsUShr());
2915 
2916   LocationSummary* locations =
2917       new (GetGraph()->GetArena()) LocationSummary(op, LocationSummary::kNoCall);
2918 
2919   switch (op->GetResultType()) {
2920     case Primitive::kPrimInt:
2921     case Primitive::kPrimLong: {
2922       locations->SetInAt(0, Location::RequiresRegister());
2923       // The shift count needs to be in CL.
2924       locations->SetInAt(1, Location::ByteRegisterOrConstant(RCX, op->InputAt(1)));
2925       locations->SetOut(Location::SameAsFirstInput());
2926       break;
2927     }
2928     default:
2929       LOG(FATAL) << "Unexpected operation type " << op->GetResultType();
2930   }
2931 }
2932 
HandleShift(HBinaryOperation * op)2933 void InstructionCodeGeneratorX86_64::HandleShift(HBinaryOperation* op) {
2934   DCHECK(op->IsShl() || op->IsShr() || op->IsUShr());
2935 
2936   LocationSummary* locations = op->GetLocations();
2937   CpuRegister first_reg = locations->InAt(0).AsRegister<CpuRegister>();
2938   Location second = locations->InAt(1);
2939 
2940   switch (op->GetResultType()) {
2941     case Primitive::kPrimInt: {
2942       if (second.IsRegister()) {
2943         CpuRegister second_reg = second.AsRegister<CpuRegister>();
2944         if (op->IsShl()) {
2945           __ shll(first_reg, second_reg);
2946         } else if (op->IsShr()) {
2947           __ sarl(first_reg, second_reg);
2948         } else {
2949           __ shrl(first_reg, second_reg);
2950         }
2951       } else {
2952         Immediate imm(second.GetConstant()->AsIntConstant()->GetValue() & kMaxIntShiftValue);
2953         if (op->IsShl()) {
2954           __ shll(first_reg, imm);
2955         } else if (op->IsShr()) {
2956           __ sarl(first_reg, imm);
2957         } else {
2958           __ shrl(first_reg, imm);
2959         }
2960       }
2961       break;
2962     }
2963     case Primitive::kPrimLong: {
2964       if (second.IsRegister()) {
2965         CpuRegister second_reg = second.AsRegister<CpuRegister>();
2966         if (op->IsShl()) {
2967           __ shlq(first_reg, second_reg);
2968         } else if (op->IsShr()) {
2969           __ sarq(first_reg, second_reg);
2970         } else {
2971           __ shrq(first_reg, second_reg);
2972         }
2973       } else {
2974         Immediate imm(second.GetConstant()->AsIntConstant()->GetValue() & kMaxLongShiftValue);
2975         if (op->IsShl()) {
2976           __ shlq(first_reg, imm);
2977         } else if (op->IsShr()) {
2978           __ sarq(first_reg, imm);
2979         } else {
2980           __ shrq(first_reg, imm);
2981         }
2982       }
2983       break;
2984     }
2985     default:
2986       LOG(FATAL) << "Unexpected operation type " << op->GetResultType();
2987   }
2988 }
2989 
VisitShl(HShl * shl)2990 void LocationsBuilderX86_64::VisitShl(HShl* shl) {
2991   HandleShift(shl);
2992 }
2993 
VisitShl(HShl * shl)2994 void InstructionCodeGeneratorX86_64::VisitShl(HShl* shl) {
2995   HandleShift(shl);
2996 }
2997 
VisitShr(HShr * shr)2998 void LocationsBuilderX86_64::VisitShr(HShr* shr) {
2999   HandleShift(shr);
3000 }
3001 
VisitShr(HShr * shr)3002 void InstructionCodeGeneratorX86_64::VisitShr(HShr* shr) {
3003   HandleShift(shr);
3004 }
3005 
VisitUShr(HUShr * ushr)3006 void LocationsBuilderX86_64::VisitUShr(HUShr* ushr) {
3007   HandleShift(ushr);
3008 }
3009 
VisitUShr(HUShr * ushr)3010 void InstructionCodeGeneratorX86_64::VisitUShr(HUShr* ushr) {
3011   HandleShift(ushr);
3012 }
3013 
VisitNewInstance(HNewInstance * instruction)3014 void LocationsBuilderX86_64::VisitNewInstance(HNewInstance* instruction) {
3015   LocationSummary* locations =
3016       new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCall);
3017   InvokeRuntimeCallingConvention calling_convention;
3018   locations->AddTemp(Location::RegisterLocation(calling_convention.GetRegisterAt(0)));
3019   locations->AddTemp(Location::RegisterLocation(calling_convention.GetRegisterAt(1)));
3020   locations->SetOut(Location::RegisterLocation(RAX));
3021 }
3022 
VisitNewInstance(HNewInstance * instruction)3023 void InstructionCodeGeneratorX86_64::VisitNewInstance(HNewInstance* instruction) {
3024   InvokeRuntimeCallingConvention calling_convention;
3025   codegen_->LoadCurrentMethod(CpuRegister(calling_convention.GetRegisterAt(1)));
3026   codegen_->Load64BitValue(CpuRegister(calling_convention.GetRegisterAt(0)),
3027                            instruction->GetTypeIndex());
3028   __ gs()->call(
3029       Address::Absolute(GetThreadOffset<kX86_64WordSize>(instruction->GetEntrypoint()), true));
3030 
3031   DCHECK(!codegen_->IsLeafMethod());
3032   codegen_->RecordPcInfo(instruction, instruction->GetDexPc());
3033 }
3034 
VisitNewArray(HNewArray * instruction)3035 void LocationsBuilderX86_64::VisitNewArray(HNewArray* instruction) {
3036   LocationSummary* locations =
3037       new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCall);
3038   InvokeRuntimeCallingConvention calling_convention;
3039   locations->AddTemp(Location::RegisterLocation(calling_convention.GetRegisterAt(0)));
3040   locations->AddTemp(Location::RegisterLocation(calling_convention.GetRegisterAt(2)));
3041   locations->SetOut(Location::RegisterLocation(RAX));
3042   locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(1)));
3043 }
3044 
VisitNewArray(HNewArray * instruction)3045 void InstructionCodeGeneratorX86_64::VisitNewArray(HNewArray* instruction) {
3046   InvokeRuntimeCallingConvention calling_convention;
3047   codegen_->LoadCurrentMethod(CpuRegister(calling_convention.GetRegisterAt(2)));
3048   codegen_->Load64BitValue(CpuRegister(calling_convention.GetRegisterAt(0)),
3049                            instruction->GetTypeIndex());
3050 
3051   __ gs()->call(
3052       Address::Absolute(GetThreadOffset<kX86_64WordSize>(instruction->GetEntrypoint()), true));
3053 
3054   DCHECK(!codegen_->IsLeafMethod());
3055   codegen_->RecordPcInfo(instruction, instruction->GetDexPc());
3056 }
3057 
VisitParameterValue(HParameterValue * instruction)3058 void LocationsBuilderX86_64::VisitParameterValue(HParameterValue* instruction) {
3059   LocationSummary* locations =
3060       new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
3061   Location location = parameter_visitor_.GetNextLocation(instruction->GetType());
3062   if (location.IsStackSlot()) {
3063     location = Location::StackSlot(location.GetStackIndex() + codegen_->GetFrameSize());
3064   } else if (location.IsDoubleStackSlot()) {
3065     location = Location::DoubleStackSlot(location.GetStackIndex() + codegen_->GetFrameSize());
3066   }
3067   locations->SetOut(location);
3068 }
3069 
VisitParameterValue(HParameterValue * instruction)3070 void InstructionCodeGeneratorX86_64::VisitParameterValue(HParameterValue* instruction) {
3071   // Nothing to do, the parameter is already at its location.
3072   UNUSED(instruction);
3073 }
3074 
VisitNot(HNot * not_)3075 void LocationsBuilderX86_64::VisitNot(HNot* not_) {
3076   LocationSummary* locations =
3077       new (GetGraph()->GetArena()) LocationSummary(not_, LocationSummary::kNoCall);
3078   locations->SetInAt(0, Location::RequiresRegister());
3079   locations->SetOut(Location::SameAsFirstInput());
3080 }
3081 
VisitNot(HNot * not_)3082 void InstructionCodeGeneratorX86_64::VisitNot(HNot* not_) {
3083   LocationSummary* locations = not_->GetLocations();
3084   DCHECK_EQ(locations->InAt(0).AsRegister<CpuRegister>().AsRegister(),
3085             locations->Out().AsRegister<CpuRegister>().AsRegister());
3086   Location out = locations->Out();
3087   switch (not_->GetResultType()) {
3088     case Primitive::kPrimInt:
3089       __ notl(out.AsRegister<CpuRegister>());
3090       break;
3091 
3092     case Primitive::kPrimLong:
3093       __ notq(out.AsRegister<CpuRegister>());
3094       break;
3095 
3096     default:
3097       LOG(FATAL) << "Unimplemented type for not operation " << not_->GetResultType();
3098   }
3099 }
3100 
VisitBooleanNot(HBooleanNot * bool_not)3101 void LocationsBuilderX86_64::VisitBooleanNot(HBooleanNot* bool_not) {
3102   LocationSummary* locations =
3103       new (GetGraph()->GetArena()) LocationSummary(bool_not, LocationSummary::kNoCall);
3104   locations->SetInAt(0, Location::RequiresRegister());
3105   locations->SetOut(Location::SameAsFirstInput());
3106 }
3107 
VisitBooleanNot(HBooleanNot * bool_not)3108 void InstructionCodeGeneratorX86_64::VisitBooleanNot(HBooleanNot* bool_not) {
3109   LocationSummary* locations = bool_not->GetLocations();
3110   DCHECK_EQ(locations->InAt(0).AsRegister<CpuRegister>().AsRegister(),
3111             locations->Out().AsRegister<CpuRegister>().AsRegister());
3112   Location out = locations->Out();
3113   __ xorl(out.AsRegister<CpuRegister>(), Immediate(1));
3114 }
3115 
VisitPhi(HPhi * instruction)3116 void LocationsBuilderX86_64::VisitPhi(HPhi* instruction) {
3117   LocationSummary* locations =
3118       new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
3119   for (size_t i = 0, e = instruction->InputCount(); i < e; ++i) {
3120     locations->SetInAt(i, Location::Any());
3121   }
3122   locations->SetOut(Location::Any());
3123 }
3124 
VisitPhi(HPhi * instruction)3125 void InstructionCodeGeneratorX86_64::VisitPhi(HPhi* instruction) {
3126   UNUSED(instruction);
3127   LOG(FATAL) << "Unimplemented";
3128 }
3129 
GenerateMemoryBarrier(MemBarrierKind kind)3130 void InstructionCodeGeneratorX86_64::GenerateMemoryBarrier(MemBarrierKind kind) {
3131   /*
3132    * According to the JSR-133 Cookbook, for x86 only StoreLoad/AnyAny barriers need memory fence.
3133    * All other barriers (LoadAny, AnyStore, StoreStore) are nops due to the x86 memory model.
3134    * For those cases, all we need to ensure is that there is a scheduling barrier in place.
3135    */
3136   switch (kind) {
3137     case MemBarrierKind::kAnyAny: {
3138       __ mfence();
3139       break;
3140     }
3141     case MemBarrierKind::kAnyStore:
3142     case MemBarrierKind::kLoadAny:
3143     case MemBarrierKind::kStoreStore: {
3144       // nop
3145       break;
3146     }
3147     default:
3148       LOG(FATAL) << "Unexpected memory barier " << kind;
3149   }
3150 }
3151 
HandleFieldGet(HInstruction * instruction)3152 void LocationsBuilderX86_64::HandleFieldGet(HInstruction* instruction) {
3153   DCHECK(instruction->IsInstanceFieldGet() || instruction->IsStaticFieldGet());
3154 
3155   LocationSummary* locations =
3156       new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
3157   locations->SetInAt(0, Location::RequiresRegister());
3158   if (Primitive::IsFloatingPointType(instruction->GetType())) {
3159     locations->SetOut(Location::RequiresFpuRegister());
3160   } else {
3161     locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
3162   }
3163 }
3164 
HandleFieldGet(HInstruction * instruction,const FieldInfo & field_info)3165 void InstructionCodeGeneratorX86_64::HandleFieldGet(HInstruction* instruction,
3166                                                     const FieldInfo& field_info) {
3167   DCHECK(instruction->IsInstanceFieldGet() || instruction->IsStaticFieldGet());
3168 
3169   LocationSummary* locations = instruction->GetLocations();
3170   CpuRegister base = locations->InAt(0).AsRegister<CpuRegister>();
3171   Location out = locations->Out();
3172   bool is_volatile = field_info.IsVolatile();
3173   Primitive::Type field_type = field_info.GetFieldType();
3174   uint32_t offset = field_info.GetFieldOffset().Uint32Value();
3175 
3176   switch (field_type) {
3177     case Primitive::kPrimBoolean: {
3178       __ movzxb(out.AsRegister<CpuRegister>(), Address(base, offset));
3179       break;
3180     }
3181 
3182     case Primitive::kPrimByte: {
3183       __ movsxb(out.AsRegister<CpuRegister>(), Address(base, offset));
3184       break;
3185     }
3186 
3187     case Primitive::kPrimShort: {
3188       __ movsxw(out.AsRegister<CpuRegister>(), Address(base, offset));
3189       break;
3190     }
3191 
3192     case Primitive::kPrimChar: {
3193       __ movzxw(out.AsRegister<CpuRegister>(), Address(base, offset));
3194       break;
3195     }
3196 
3197     case Primitive::kPrimInt:
3198     case Primitive::kPrimNot: {
3199       __ movl(out.AsRegister<CpuRegister>(), Address(base, offset));
3200       break;
3201     }
3202 
3203     case Primitive::kPrimLong: {
3204       __ movq(out.AsRegister<CpuRegister>(), Address(base, offset));
3205       break;
3206     }
3207 
3208     case Primitive::kPrimFloat: {
3209       __ movss(out.AsFpuRegister<XmmRegister>(), Address(base, offset));
3210       break;
3211     }
3212 
3213     case Primitive::kPrimDouble: {
3214       __ movsd(out.AsFpuRegister<XmmRegister>(), Address(base, offset));
3215       break;
3216     }
3217 
3218     case Primitive::kPrimVoid:
3219       LOG(FATAL) << "Unreachable type " << field_type;
3220       UNREACHABLE();
3221   }
3222 
3223   codegen_->MaybeRecordImplicitNullCheck(instruction);
3224 
3225   if (is_volatile) {
3226     GenerateMemoryBarrier(MemBarrierKind::kLoadAny);
3227   }
3228 }
3229 
HandleFieldSet(HInstruction * instruction,const FieldInfo & field_info)3230 void LocationsBuilderX86_64::HandleFieldSet(HInstruction* instruction,
3231                                             const FieldInfo& field_info) {
3232   DCHECK(instruction->IsInstanceFieldSet() || instruction->IsStaticFieldSet());
3233 
3234   LocationSummary* locations =
3235       new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
3236   bool needs_write_barrier =
3237       CodeGenerator::StoreNeedsWriteBarrier(field_info.GetFieldType(), instruction->InputAt(1));
3238 
3239   locations->SetInAt(0, Location::RequiresRegister());
3240   if (Primitive::IsFloatingPointType(instruction->InputAt(1)->GetType())) {
3241     locations->SetInAt(1, Location::RequiresFpuRegister());
3242   } else {
3243     locations->SetInAt(1, Location::RegisterOrInt32LongConstant(instruction->InputAt(1)));
3244   }
3245   if (needs_write_barrier) {
3246     // Temporary registers for the write barrier.
3247     locations->AddTemp(Location::RequiresRegister());
3248     locations->AddTemp(Location::RequiresRegister());
3249   }
3250 }
3251 
HandleFieldSet(HInstruction * instruction,const FieldInfo & field_info)3252 void InstructionCodeGeneratorX86_64::HandleFieldSet(HInstruction* instruction,
3253                                                     const FieldInfo& field_info) {
3254   DCHECK(instruction->IsInstanceFieldSet() || instruction->IsStaticFieldSet());
3255 
3256   LocationSummary* locations = instruction->GetLocations();
3257   CpuRegister base = locations->InAt(0).AsRegister<CpuRegister>();
3258   Location value = locations->InAt(1);
3259   bool is_volatile = field_info.IsVolatile();
3260   Primitive::Type field_type = field_info.GetFieldType();
3261   uint32_t offset = field_info.GetFieldOffset().Uint32Value();
3262 
3263   if (is_volatile) {
3264     GenerateMemoryBarrier(MemBarrierKind::kAnyStore);
3265   }
3266 
3267   switch (field_type) {
3268     case Primitive::kPrimBoolean:
3269     case Primitive::kPrimByte: {
3270       if (value.IsConstant()) {
3271         int32_t v = CodeGenerator::GetInt32ValueOf(value.GetConstant());
3272         __ movb(Address(base, offset), Immediate(v));
3273       } else {
3274         __ movb(Address(base, offset), value.AsRegister<CpuRegister>());
3275       }
3276       break;
3277     }
3278 
3279     case Primitive::kPrimShort:
3280     case Primitive::kPrimChar: {
3281       if (value.IsConstant()) {
3282         int32_t v = CodeGenerator::GetInt32ValueOf(value.GetConstant());
3283         __ movw(Address(base, offset), Immediate(v));
3284       } else {
3285         __ movw(Address(base, offset), value.AsRegister<CpuRegister>());
3286       }
3287       break;
3288     }
3289 
3290     case Primitive::kPrimInt:
3291     case Primitive::kPrimNot: {
3292       if (value.IsConstant()) {
3293         int32_t v = CodeGenerator::GetInt32ValueOf(value.GetConstant());
3294         __ movw(Address(base, offset), Immediate(v));
3295       } else {
3296         __ movl(Address(base, offset), value.AsRegister<CpuRegister>());
3297       }
3298       break;
3299     }
3300 
3301     case Primitive::kPrimLong: {
3302       if (value.IsConstant()) {
3303         int64_t v = value.GetConstant()->AsLongConstant()->GetValue();
3304         DCHECK(IsInt<32>(v));
3305         int32_t v_32 = v;
3306         __ movq(Address(base, offset), Immediate(v_32));
3307       } else {
3308         __ movq(Address(base, offset), value.AsRegister<CpuRegister>());
3309       }
3310       break;
3311     }
3312 
3313     case Primitive::kPrimFloat: {
3314       __ movss(Address(base, offset), value.AsFpuRegister<XmmRegister>());
3315       break;
3316     }
3317 
3318     case Primitive::kPrimDouble: {
3319       __ movsd(Address(base, offset), value.AsFpuRegister<XmmRegister>());
3320       break;
3321     }
3322 
3323     case Primitive::kPrimVoid:
3324       LOG(FATAL) << "Unreachable type " << field_type;
3325       UNREACHABLE();
3326   }
3327 
3328   codegen_->MaybeRecordImplicitNullCheck(instruction);
3329 
3330   if (CodeGenerator::StoreNeedsWriteBarrier(field_type, instruction->InputAt(1))) {
3331     CpuRegister temp = locations->GetTemp(0).AsRegister<CpuRegister>();
3332     CpuRegister card = locations->GetTemp(1).AsRegister<CpuRegister>();
3333     codegen_->MarkGCCard(temp, card, base, value.AsRegister<CpuRegister>());
3334   }
3335 
3336   if (is_volatile) {
3337     GenerateMemoryBarrier(MemBarrierKind::kAnyAny);
3338   }
3339 }
3340 
VisitInstanceFieldSet(HInstanceFieldSet * instruction)3341 void LocationsBuilderX86_64::VisitInstanceFieldSet(HInstanceFieldSet* instruction) {
3342   HandleFieldSet(instruction, instruction->GetFieldInfo());
3343 }
3344 
VisitInstanceFieldSet(HInstanceFieldSet * instruction)3345 void InstructionCodeGeneratorX86_64::VisitInstanceFieldSet(HInstanceFieldSet* instruction) {
3346   HandleFieldSet(instruction, instruction->GetFieldInfo());
3347 }
3348 
VisitInstanceFieldGet(HInstanceFieldGet * instruction)3349 void LocationsBuilderX86_64::VisitInstanceFieldGet(HInstanceFieldGet* instruction) {
3350   HandleFieldGet(instruction);
3351 }
3352 
VisitInstanceFieldGet(HInstanceFieldGet * instruction)3353 void InstructionCodeGeneratorX86_64::VisitInstanceFieldGet(HInstanceFieldGet* instruction) {
3354   HandleFieldGet(instruction, instruction->GetFieldInfo());
3355 }
3356 
VisitStaticFieldGet(HStaticFieldGet * instruction)3357 void LocationsBuilderX86_64::VisitStaticFieldGet(HStaticFieldGet* instruction) {
3358   HandleFieldGet(instruction);
3359 }
3360 
VisitStaticFieldGet(HStaticFieldGet * instruction)3361 void InstructionCodeGeneratorX86_64::VisitStaticFieldGet(HStaticFieldGet* instruction) {
3362   HandleFieldGet(instruction, instruction->GetFieldInfo());
3363 }
3364 
VisitStaticFieldSet(HStaticFieldSet * instruction)3365 void LocationsBuilderX86_64::VisitStaticFieldSet(HStaticFieldSet* instruction) {
3366   HandleFieldSet(instruction, instruction->GetFieldInfo());
3367 }
3368 
VisitStaticFieldSet(HStaticFieldSet * instruction)3369 void InstructionCodeGeneratorX86_64::VisitStaticFieldSet(HStaticFieldSet* instruction) {
3370   HandleFieldSet(instruction, instruction->GetFieldInfo());
3371 }
3372 
VisitNullCheck(HNullCheck * instruction)3373 void LocationsBuilderX86_64::VisitNullCheck(HNullCheck* instruction) {
3374   LocationSummary* locations =
3375       new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
3376   Location loc = codegen_->GetCompilerOptions().GetImplicitNullChecks()
3377       ? Location::RequiresRegister()
3378       : Location::Any();
3379   locations->SetInAt(0, loc);
3380   if (instruction->HasUses()) {
3381     locations->SetOut(Location::SameAsFirstInput());
3382   }
3383 }
3384 
GenerateImplicitNullCheck(HNullCheck * instruction)3385 void InstructionCodeGeneratorX86_64::GenerateImplicitNullCheck(HNullCheck* instruction) {
3386   if (codegen_->CanMoveNullCheckToUser(instruction)) {
3387     return;
3388   }
3389   LocationSummary* locations = instruction->GetLocations();
3390   Location obj = locations->InAt(0);
3391 
3392   __ testl(CpuRegister(RAX), Address(obj.AsRegister<CpuRegister>(), 0));
3393   codegen_->RecordPcInfo(instruction, instruction->GetDexPc());
3394 }
3395 
GenerateExplicitNullCheck(HNullCheck * instruction)3396 void InstructionCodeGeneratorX86_64::GenerateExplicitNullCheck(HNullCheck* instruction) {
3397   SlowPathCodeX86_64* slow_path = new (GetGraph()->GetArena()) NullCheckSlowPathX86_64(instruction);
3398   codegen_->AddSlowPath(slow_path);
3399 
3400   LocationSummary* locations = instruction->GetLocations();
3401   Location obj = locations->InAt(0);
3402 
3403   if (obj.IsRegister()) {
3404     __ testl(obj.AsRegister<CpuRegister>(), obj.AsRegister<CpuRegister>());
3405   } else if (obj.IsStackSlot()) {
3406     __ cmpl(Address(CpuRegister(RSP), obj.GetStackIndex()), Immediate(0));
3407   } else {
3408     DCHECK(obj.IsConstant()) << obj;
3409     DCHECK_EQ(obj.GetConstant()->AsIntConstant()->GetValue(), 0);
3410     __ jmp(slow_path->GetEntryLabel());
3411     return;
3412   }
3413   __ j(kEqual, slow_path->GetEntryLabel());
3414 }
3415 
VisitNullCheck(HNullCheck * instruction)3416 void InstructionCodeGeneratorX86_64::VisitNullCheck(HNullCheck* instruction) {
3417   if (codegen_->GetCompilerOptions().GetImplicitNullChecks()) {
3418     GenerateImplicitNullCheck(instruction);
3419   } else {
3420     GenerateExplicitNullCheck(instruction);
3421   }
3422 }
3423 
VisitArrayGet(HArrayGet * instruction)3424 void LocationsBuilderX86_64::VisitArrayGet(HArrayGet* instruction) {
3425   LocationSummary* locations =
3426       new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
3427   locations->SetInAt(0, Location::RequiresRegister());
3428   locations->SetInAt(1, Location::RegisterOrConstant(instruction->InputAt(1)));
3429   if (Primitive::IsFloatingPointType(instruction->GetType())) {
3430     locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap);
3431   } else {
3432     locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
3433   }
3434 }
3435 
VisitArrayGet(HArrayGet * instruction)3436 void InstructionCodeGeneratorX86_64::VisitArrayGet(HArrayGet* instruction) {
3437   LocationSummary* locations = instruction->GetLocations();
3438   CpuRegister obj = locations->InAt(0).AsRegister<CpuRegister>();
3439   Location index = locations->InAt(1);
3440 
3441   switch (instruction->GetType()) {
3442     case Primitive::kPrimBoolean: {
3443       uint32_t data_offset = mirror::Array::DataOffset(sizeof(uint8_t)).Uint32Value();
3444       CpuRegister out = locations->Out().AsRegister<CpuRegister>();
3445       if (index.IsConstant()) {
3446         __ movzxb(out, Address(obj,
3447             (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_1) + data_offset));
3448       } else {
3449         __ movzxb(out, Address(obj, index.AsRegister<CpuRegister>(), TIMES_1, data_offset));
3450       }
3451       break;
3452     }
3453 
3454     case Primitive::kPrimByte: {
3455       uint32_t data_offset = mirror::Array::DataOffset(sizeof(int8_t)).Uint32Value();
3456       CpuRegister out = locations->Out().AsRegister<CpuRegister>();
3457       if (index.IsConstant()) {
3458         __ movsxb(out, Address(obj,
3459             (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_1) + data_offset));
3460       } else {
3461         __ movsxb(out, Address(obj, index.AsRegister<CpuRegister>(), TIMES_1, data_offset));
3462       }
3463       break;
3464     }
3465 
3466     case Primitive::kPrimShort: {
3467       uint32_t data_offset = mirror::Array::DataOffset(sizeof(int16_t)).Uint32Value();
3468       CpuRegister out = locations->Out().AsRegister<CpuRegister>();
3469       if (index.IsConstant()) {
3470         __ movsxw(out, Address(obj,
3471             (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_2) + data_offset));
3472       } else {
3473         __ movsxw(out, Address(obj, index.AsRegister<CpuRegister>(), TIMES_2, data_offset));
3474       }
3475       break;
3476     }
3477 
3478     case Primitive::kPrimChar: {
3479       uint32_t data_offset = mirror::Array::DataOffset(sizeof(uint16_t)).Uint32Value();
3480       CpuRegister out = locations->Out().AsRegister<CpuRegister>();
3481       if (index.IsConstant()) {
3482         __ movzxw(out, Address(obj,
3483             (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_2) + data_offset));
3484       } else {
3485         __ movzxw(out, Address(obj, index.AsRegister<CpuRegister>(), TIMES_2, data_offset));
3486       }
3487       break;
3488     }
3489 
3490     case Primitive::kPrimInt:
3491     case Primitive::kPrimNot: {
3492       DCHECK_EQ(sizeof(mirror::HeapReference<mirror::Object>), sizeof(int32_t));
3493       uint32_t data_offset = mirror::Array::DataOffset(sizeof(int32_t)).Uint32Value();
3494       CpuRegister out = locations->Out().AsRegister<CpuRegister>();
3495       if (index.IsConstant()) {
3496         __ movl(out, Address(obj,
3497             (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4) + data_offset));
3498       } else {
3499         __ movl(out, Address(obj, index.AsRegister<CpuRegister>(), TIMES_4, data_offset));
3500       }
3501       break;
3502     }
3503 
3504     case Primitive::kPrimLong: {
3505       uint32_t data_offset = mirror::Array::DataOffset(sizeof(int64_t)).Uint32Value();
3506       CpuRegister out = locations->Out().AsRegister<CpuRegister>();
3507       if (index.IsConstant()) {
3508         __ movq(out, Address(obj,
3509             (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_8) + data_offset));
3510       } else {
3511         __ movq(out, Address(obj, index.AsRegister<CpuRegister>(), TIMES_8, data_offset));
3512       }
3513       break;
3514     }
3515 
3516     case Primitive::kPrimFloat: {
3517       uint32_t data_offset = mirror::Array::DataOffset(sizeof(float)).Uint32Value();
3518       XmmRegister out = locations->Out().AsFpuRegister<XmmRegister>();
3519       if (index.IsConstant()) {
3520         __ movss(out, Address(obj,
3521             (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4) + data_offset));
3522       } else {
3523         __ movss(out, Address(obj, index.AsRegister<CpuRegister>(), TIMES_4, data_offset));
3524       }
3525       break;
3526     }
3527 
3528     case Primitive::kPrimDouble: {
3529       uint32_t data_offset = mirror::Array::DataOffset(sizeof(double)).Uint32Value();
3530       XmmRegister out = locations->Out().AsFpuRegister<XmmRegister>();
3531       if (index.IsConstant()) {
3532         __ movsd(out, Address(obj,
3533             (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_8) + data_offset));
3534       } else {
3535         __ movsd(out, Address(obj, index.AsRegister<CpuRegister>(), TIMES_8, data_offset));
3536       }
3537       break;
3538     }
3539 
3540     case Primitive::kPrimVoid:
3541       LOG(FATAL) << "Unreachable type " << instruction->GetType();
3542       UNREACHABLE();
3543   }
3544   codegen_->MaybeRecordImplicitNullCheck(instruction);
3545 }
3546 
VisitArraySet(HArraySet * instruction)3547 void LocationsBuilderX86_64::VisitArraySet(HArraySet* instruction) {
3548   Primitive::Type value_type = instruction->GetComponentType();
3549 
3550   bool needs_write_barrier =
3551       CodeGenerator::StoreNeedsWriteBarrier(value_type, instruction->GetValue());
3552   bool needs_runtime_call = instruction->NeedsTypeCheck();
3553 
3554   LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(
3555       instruction, needs_runtime_call ? LocationSummary::kCall : LocationSummary::kNoCall);
3556   if (needs_runtime_call) {
3557     InvokeRuntimeCallingConvention calling_convention;
3558     locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(0)));
3559     locations->SetInAt(1, Location::RegisterLocation(calling_convention.GetRegisterAt(1)));
3560     locations->SetInAt(2, Location::RegisterLocation(calling_convention.GetRegisterAt(2)));
3561   } else {
3562     locations->SetInAt(0, Location::RequiresRegister());
3563     locations->SetInAt(
3564         1, Location::RegisterOrConstant(instruction->InputAt(1)));
3565     locations->SetInAt(2, Location::RequiresRegister());
3566     if (value_type == Primitive::kPrimLong) {
3567       locations->SetInAt(2, Location::RegisterOrInt32LongConstant(instruction->InputAt(2)));
3568     } else if (value_type == Primitive::kPrimFloat || value_type == Primitive::kPrimDouble) {
3569       locations->SetInAt(2, Location::RequiresFpuRegister());
3570     } else {
3571       locations->SetInAt(2, Location::RegisterOrConstant(instruction->InputAt(2)));
3572     }
3573 
3574     if (needs_write_barrier) {
3575       // Temporary registers for the write barrier.
3576       locations->AddTemp(Location::RequiresRegister());
3577       locations->AddTemp(Location::RequiresRegister());
3578     }
3579   }
3580 }
3581 
VisitArraySet(HArraySet * instruction)3582 void InstructionCodeGeneratorX86_64::VisitArraySet(HArraySet* instruction) {
3583   LocationSummary* locations = instruction->GetLocations();
3584   CpuRegister obj = locations->InAt(0).AsRegister<CpuRegister>();
3585   Location index = locations->InAt(1);
3586   Location value = locations->InAt(2);
3587   Primitive::Type value_type = instruction->GetComponentType();
3588   bool needs_runtime_call = locations->WillCall();
3589   bool needs_write_barrier =
3590       CodeGenerator::StoreNeedsWriteBarrier(value_type, instruction->GetValue());
3591 
3592   switch (value_type) {
3593     case Primitive::kPrimBoolean:
3594     case Primitive::kPrimByte: {
3595       uint32_t data_offset = mirror::Array::DataOffset(sizeof(uint8_t)).Uint32Value();
3596       if (index.IsConstant()) {
3597         size_t offset = (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_1) + data_offset;
3598         if (value.IsRegister()) {
3599           __ movb(Address(obj, offset), value.AsRegister<CpuRegister>());
3600         } else {
3601           __ movb(Address(obj, offset),
3602                   Immediate(value.GetConstant()->AsIntConstant()->GetValue()));
3603         }
3604       } else {
3605         if (value.IsRegister()) {
3606           __ movb(Address(obj, index.AsRegister<CpuRegister>(), TIMES_1, data_offset),
3607                   value.AsRegister<CpuRegister>());
3608         } else {
3609           __ movb(Address(obj, index.AsRegister<CpuRegister>(), TIMES_1, data_offset),
3610                   Immediate(value.GetConstant()->AsIntConstant()->GetValue()));
3611         }
3612       }
3613       codegen_->MaybeRecordImplicitNullCheck(instruction);
3614       break;
3615     }
3616 
3617     case Primitive::kPrimShort:
3618     case Primitive::kPrimChar: {
3619       uint32_t data_offset = mirror::Array::DataOffset(sizeof(uint16_t)).Uint32Value();
3620       if (index.IsConstant()) {
3621         size_t offset = (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_2) + data_offset;
3622         if (value.IsRegister()) {
3623           __ movw(Address(obj, offset), value.AsRegister<CpuRegister>());
3624         } else {
3625           DCHECK(value.IsConstant()) << value;
3626           __ movw(Address(obj, offset),
3627                   Immediate(value.GetConstant()->AsIntConstant()->GetValue()));
3628         }
3629       } else {
3630         DCHECK(index.IsRegister()) << index;
3631         if (value.IsRegister()) {
3632           __ movw(Address(obj, index.AsRegister<CpuRegister>(), TIMES_2, data_offset),
3633                   value.AsRegister<CpuRegister>());
3634         } else {
3635           DCHECK(value.IsConstant()) << value;
3636           __ movw(Address(obj, index.AsRegister<CpuRegister>(), TIMES_2, data_offset),
3637                   Immediate(value.GetConstant()->AsIntConstant()->GetValue()));
3638         }
3639       }
3640       codegen_->MaybeRecordImplicitNullCheck(instruction);
3641       break;
3642     }
3643 
3644     case Primitive::kPrimInt:
3645     case Primitive::kPrimNot: {
3646       if (!needs_runtime_call) {
3647         uint32_t data_offset = mirror::Array::DataOffset(sizeof(int32_t)).Uint32Value();
3648         if (index.IsConstant()) {
3649           size_t offset =
3650               (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4) + data_offset;
3651           if (value.IsRegister()) {
3652             __ movl(Address(obj, offset), value.AsRegister<CpuRegister>());
3653           } else {
3654             DCHECK(value.IsConstant()) << value;
3655             int32_t v = CodeGenerator::GetInt32ValueOf(value.GetConstant());
3656             __ movl(Address(obj, offset), Immediate(v));
3657           }
3658         } else {
3659           DCHECK(index.IsRegister()) << index;
3660           if (value.IsRegister()) {
3661             __ movl(Address(obj, index.AsRegister<CpuRegister>(), TIMES_4, data_offset),
3662                     value.AsRegister<CpuRegister>());
3663           } else {
3664             DCHECK(value.IsConstant()) << value;
3665             int32_t v = CodeGenerator::GetInt32ValueOf(value.GetConstant());
3666             __ movl(Address(obj, index.AsRegister<CpuRegister>(), TIMES_4, data_offset),
3667                     Immediate(v));
3668           }
3669         }
3670         codegen_->MaybeRecordImplicitNullCheck(instruction);
3671         if (needs_write_barrier) {
3672           DCHECK_EQ(value_type, Primitive::kPrimNot);
3673           CpuRegister temp = locations->GetTemp(0).AsRegister<CpuRegister>();
3674           CpuRegister card = locations->GetTemp(1).AsRegister<CpuRegister>();
3675           codegen_->MarkGCCard(temp, card, obj, value.AsRegister<CpuRegister>());
3676         }
3677       } else {
3678         DCHECK_EQ(value_type, Primitive::kPrimNot);
3679         __ gs()->call(Address::Absolute(QUICK_ENTRYPOINT_OFFSET(kX86_64WordSize, pAputObject),
3680                                         true));
3681         DCHECK(!codegen_->IsLeafMethod());
3682         codegen_->RecordPcInfo(instruction, instruction->GetDexPc());
3683       }
3684       break;
3685     }
3686 
3687     case Primitive::kPrimLong: {
3688       uint32_t data_offset = mirror::Array::DataOffset(sizeof(int64_t)).Uint32Value();
3689       if (index.IsConstant()) {
3690         size_t offset = (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_8) + data_offset;
3691         if (value.IsRegister()) {
3692           __ movq(Address(obj, offset), value.AsRegister<CpuRegister>());
3693         } else {
3694           int64_t v = value.GetConstant()->AsLongConstant()->GetValue();
3695           DCHECK(IsInt<32>(v));
3696           int32_t v_32 = v;
3697           __ movq(Address(obj, offset), Immediate(v_32));
3698         }
3699       } else {
3700         if (value.IsRegister()) {
3701           __ movq(Address(obj, index.AsRegister<CpuRegister>(), TIMES_8, data_offset),
3702                   value.AsRegister<CpuRegister>());
3703         } else {
3704           int64_t v = value.GetConstant()->AsLongConstant()->GetValue();
3705           DCHECK(IsInt<32>(v));
3706           int32_t v_32 = v;
3707           __ movq(Address(obj, index.AsRegister<CpuRegister>(), TIMES_8, data_offset),
3708                   Immediate(v_32));
3709         }
3710       }
3711       codegen_->MaybeRecordImplicitNullCheck(instruction);
3712       break;
3713     }
3714 
3715     case Primitive::kPrimFloat: {
3716       uint32_t data_offset = mirror::Array::DataOffset(sizeof(float)).Uint32Value();
3717       if (index.IsConstant()) {
3718         size_t offset = (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4) + data_offset;
3719         DCHECK(value.IsFpuRegister());
3720         __ movss(Address(obj, offset), value.AsFpuRegister<XmmRegister>());
3721       } else {
3722         DCHECK(value.IsFpuRegister());
3723         __ movss(Address(obj, index.AsRegister<CpuRegister>(), TIMES_4, data_offset),
3724                 value.AsFpuRegister<XmmRegister>());
3725       }
3726       codegen_->MaybeRecordImplicitNullCheck(instruction);
3727       break;
3728     }
3729 
3730     case Primitive::kPrimDouble: {
3731       uint32_t data_offset = mirror::Array::DataOffset(sizeof(double)).Uint32Value();
3732       if (index.IsConstant()) {
3733         size_t offset = (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_8) + data_offset;
3734         DCHECK(value.IsFpuRegister());
3735         __ movsd(Address(obj, offset), value.AsFpuRegister<XmmRegister>());
3736       } else {
3737         DCHECK(value.IsFpuRegister());
3738         __ movsd(Address(obj, index.AsRegister<CpuRegister>(), TIMES_8, data_offset),
3739                 value.AsFpuRegister<XmmRegister>());
3740       }
3741       codegen_->MaybeRecordImplicitNullCheck(instruction);
3742       break;
3743     }
3744 
3745     case Primitive::kPrimVoid:
3746       LOG(FATAL) << "Unreachable type " << instruction->GetType();
3747       UNREACHABLE();
3748   }
3749 }
3750 
VisitArrayLength(HArrayLength * instruction)3751 void LocationsBuilderX86_64::VisitArrayLength(HArrayLength* instruction) {
3752   LocationSummary* locations =
3753       new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
3754   locations->SetInAt(0, Location::RequiresRegister());
3755   locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
3756 }
3757 
VisitArrayLength(HArrayLength * instruction)3758 void InstructionCodeGeneratorX86_64::VisitArrayLength(HArrayLength* instruction) {
3759   LocationSummary* locations = instruction->GetLocations();
3760   uint32_t offset = mirror::Array::LengthOffset().Uint32Value();
3761   CpuRegister obj = locations->InAt(0).AsRegister<CpuRegister>();
3762   CpuRegister out = locations->Out().AsRegister<CpuRegister>();
3763   __ movl(out, Address(obj, offset));
3764   codegen_->MaybeRecordImplicitNullCheck(instruction);
3765 }
3766 
VisitBoundsCheck(HBoundsCheck * instruction)3767 void LocationsBuilderX86_64::VisitBoundsCheck(HBoundsCheck* instruction) {
3768   LocationSummary* locations =
3769       new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
3770   locations->SetInAt(0, Location::RegisterOrConstant(instruction->InputAt(0)));
3771   locations->SetInAt(1, Location::RegisterOrConstant(instruction->InputAt(1)));
3772   if (instruction->HasUses()) {
3773     locations->SetOut(Location::SameAsFirstInput());
3774   }
3775 }
3776 
VisitBoundsCheck(HBoundsCheck * instruction)3777 void InstructionCodeGeneratorX86_64::VisitBoundsCheck(HBoundsCheck* instruction) {
3778   LocationSummary* locations = instruction->GetLocations();
3779   Location index_loc = locations->InAt(0);
3780   Location length_loc = locations->InAt(1);
3781   SlowPathCodeX86_64* slow_path =
3782     new (GetGraph()->GetArena()) BoundsCheckSlowPathX86_64(instruction, index_loc, length_loc);
3783 
3784   if (length_loc.IsConstant()) {
3785     int32_t length = CodeGenerator::GetInt32ValueOf(length_loc.GetConstant());
3786     if (index_loc.IsConstant()) {
3787       // BCE will remove the bounds check if we are guarenteed to pass.
3788       int32_t index = CodeGenerator::GetInt32ValueOf(index_loc.GetConstant());
3789       if (index < 0 || index >= length) {
3790         codegen_->AddSlowPath(slow_path);
3791         __ jmp(slow_path->GetEntryLabel());
3792       } else {
3793         // Some optimization after BCE may have generated this, and we should not
3794         // generate a bounds check if it is a valid range.
3795       }
3796       return;
3797     }
3798 
3799     // We have to reverse the jump condition because the length is the constant.
3800     CpuRegister index_reg = index_loc.AsRegister<CpuRegister>();
3801     __ cmpl(index_reg, Immediate(length));
3802     codegen_->AddSlowPath(slow_path);
3803     __ j(kAboveEqual, slow_path->GetEntryLabel());
3804   } else {
3805     CpuRegister length = length_loc.AsRegister<CpuRegister>();
3806     if (index_loc.IsConstant()) {
3807       int32_t value = CodeGenerator::GetInt32ValueOf(index_loc.GetConstant());
3808       __ cmpl(length, Immediate(value));
3809     } else {
3810       __ cmpl(length, index_loc.AsRegister<CpuRegister>());
3811     }
3812     codegen_->AddSlowPath(slow_path);
3813     __ j(kBelowEqual, slow_path->GetEntryLabel());
3814   }
3815 }
3816 
MarkGCCard(CpuRegister temp,CpuRegister card,CpuRegister object,CpuRegister value)3817 void CodeGeneratorX86_64::MarkGCCard(CpuRegister temp,
3818                                      CpuRegister card,
3819                                      CpuRegister object,
3820                                      CpuRegister value) {
3821   Label is_null;
3822   __ testl(value, value);
3823   __ j(kEqual, &is_null);
3824   __ gs()->movq(card, Address::Absolute(
3825       Thread::CardTableOffset<kX86_64WordSize>().Int32Value(), true));
3826   __ movq(temp, object);
3827   __ shrq(temp, Immediate(gc::accounting::CardTable::kCardShift));
3828   __ movb(Address(temp, card, TIMES_1, 0),  card);
3829   __ Bind(&is_null);
3830 }
3831 
VisitTemporary(HTemporary * temp)3832 void LocationsBuilderX86_64::VisitTemporary(HTemporary* temp) {
3833   temp->SetLocations(nullptr);
3834 }
3835 
VisitTemporary(HTemporary * temp)3836 void InstructionCodeGeneratorX86_64::VisitTemporary(HTemporary* temp) {
3837   // Nothing to do, this is driven by the code generator.
3838   UNUSED(temp);
3839 }
3840 
VisitParallelMove(HParallelMove * instruction)3841 void LocationsBuilderX86_64::VisitParallelMove(HParallelMove* instruction) {
3842   UNUSED(instruction);
3843   LOG(FATAL) << "Unimplemented";
3844 }
3845 
VisitParallelMove(HParallelMove * instruction)3846 void InstructionCodeGeneratorX86_64::VisitParallelMove(HParallelMove* instruction) {
3847   codegen_->GetMoveResolver()->EmitNativeCode(instruction);
3848 }
3849 
VisitSuspendCheck(HSuspendCheck * instruction)3850 void LocationsBuilderX86_64::VisitSuspendCheck(HSuspendCheck* instruction) {
3851   new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCallOnSlowPath);
3852 }
3853 
VisitSuspendCheck(HSuspendCheck * instruction)3854 void InstructionCodeGeneratorX86_64::VisitSuspendCheck(HSuspendCheck* instruction) {
3855   HBasicBlock* block = instruction->GetBlock();
3856   if (block->GetLoopInformation() != nullptr) {
3857     DCHECK(block->GetLoopInformation()->GetSuspendCheck() == instruction);
3858     // The back edge will generate the suspend check.
3859     return;
3860   }
3861   if (block->IsEntryBlock() && instruction->GetNext()->IsGoto()) {
3862     // The goto will generate the suspend check.
3863     return;
3864   }
3865   GenerateSuspendCheck(instruction, nullptr);
3866 }
3867 
GenerateSuspendCheck(HSuspendCheck * instruction,HBasicBlock * successor)3868 void InstructionCodeGeneratorX86_64::GenerateSuspendCheck(HSuspendCheck* instruction,
3869                                                           HBasicBlock* successor) {
3870   SuspendCheckSlowPathX86_64* slow_path =
3871       down_cast<SuspendCheckSlowPathX86_64*>(instruction->GetSlowPath());
3872   if (slow_path == nullptr) {
3873     slow_path = new (GetGraph()->GetArena()) SuspendCheckSlowPathX86_64(instruction, successor);
3874     instruction->SetSlowPath(slow_path);
3875     codegen_->AddSlowPath(slow_path);
3876     if (successor != nullptr) {
3877       DCHECK(successor->IsLoopHeader());
3878       codegen_->ClearSpillSlotsFromLoopPhisInStackMap(instruction);
3879     }
3880   } else {
3881     DCHECK_EQ(slow_path->GetSuccessor(), successor);
3882   }
3883 
3884   __ gs()->cmpw(Address::Absolute(
3885       Thread::ThreadFlagsOffset<kX86_64WordSize>().Int32Value(), true), Immediate(0));
3886   if (successor == nullptr) {
3887     __ j(kNotEqual, slow_path->GetEntryLabel());
3888     __ Bind(slow_path->GetReturnLabel());
3889   } else {
3890     __ j(kEqual, codegen_->GetLabelOf(successor));
3891     __ jmp(slow_path->GetEntryLabel());
3892   }
3893 }
3894 
GetAssembler() const3895 X86_64Assembler* ParallelMoveResolverX86_64::GetAssembler() const {
3896   return codegen_->GetAssembler();
3897 }
3898 
EmitMove(size_t index)3899 void ParallelMoveResolverX86_64::EmitMove(size_t index) {
3900   MoveOperands* move = moves_.Get(index);
3901   Location source = move->GetSource();
3902   Location destination = move->GetDestination();
3903 
3904   if (source.IsRegister()) {
3905     if (destination.IsRegister()) {
3906       __ movq(destination.AsRegister<CpuRegister>(), source.AsRegister<CpuRegister>());
3907     } else if (destination.IsStackSlot()) {
3908       __ movl(Address(CpuRegister(RSP), destination.GetStackIndex()),
3909               source.AsRegister<CpuRegister>());
3910     } else {
3911       DCHECK(destination.IsDoubleStackSlot());
3912       __ movq(Address(CpuRegister(RSP), destination.GetStackIndex()),
3913               source.AsRegister<CpuRegister>());
3914     }
3915   } else if (source.IsStackSlot()) {
3916     if (destination.IsRegister()) {
3917       __ movl(destination.AsRegister<CpuRegister>(),
3918               Address(CpuRegister(RSP), source.GetStackIndex()));
3919     } else if (destination.IsFpuRegister()) {
3920       __ movss(destination.AsFpuRegister<XmmRegister>(),
3921               Address(CpuRegister(RSP), source.GetStackIndex()));
3922     } else {
3923       DCHECK(destination.IsStackSlot());
3924       __ movl(CpuRegister(TMP), Address(CpuRegister(RSP), source.GetStackIndex()));
3925       __ movl(Address(CpuRegister(RSP), destination.GetStackIndex()), CpuRegister(TMP));
3926     }
3927   } else if (source.IsDoubleStackSlot()) {
3928     if (destination.IsRegister()) {
3929       __ movq(destination.AsRegister<CpuRegister>(),
3930               Address(CpuRegister(RSP), source.GetStackIndex()));
3931     } else if (destination.IsFpuRegister()) {
3932       __ movsd(destination.AsFpuRegister<XmmRegister>(),
3933                Address(CpuRegister(RSP), source.GetStackIndex()));
3934     } else {
3935       DCHECK(destination.IsDoubleStackSlot()) << destination;
3936       __ movq(CpuRegister(TMP), Address(CpuRegister(RSP), source.GetStackIndex()));
3937       __ movq(Address(CpuRegister(RSP), destination.GetStackIndex()), CpuRegister(TMP));
3938     }
3939   } else if (source.IsConstant()) {
3940     HConstant* constant = source.GetConstant();
3941     if (constant->IsIntConstant() || constant->IsNullConstant()) {
3942       int32_t value = CodeGenerator::GetInt32ValueOf(constant);
3943       if (destination.IsRegister()) {
3944         if (value == 0) {
3945           __ xorl(destination.AsRegister<CpuRegister>(), destination.AsRegister<CpuRegister>());
3946         } else {
3947           __ movl(destination.AsRegister<CpuRegister>(), Immediate(value));
3948         }
3949       } else {
3950         DCHECK(destination.IsStackSlot()) << destination;
3951         __ movl(Address(CpuRegister(RSP), destination.GetStackIndex()), Immediate(value));
3952       }
3953     } else if (constant->IsLongConstant()) {
3954       int64_t value = constant->AsLongConstant()->GetValue();
3955       if (destination.IsRegister()) {
3956         codegen_->Load64BitValue(destination.AsRegister<CpuRegister>(), value);
3957       } else {
3958         DCHECK(destination.IsDoubleStackSlot()) << destination;
3959         codegen_->Load64BitValue(CpuRegister(TMP), value);
3960         __ movq(Address(CpuRegister(RSP), destination.GetStackIndex()), CpuRegister(TMP));
3961       }
3962     } else if (constant->IsFloatConstant()) {
3963       float fp_value = constant->AsFloatConstant()->GetValue();
3964       int32_t value = bit_cast<int32_t, float>(fp_value);
3965       if (destination.IsFpuRegister()) {
3966         XmmRegister dest = destination.AsFpuRegister<XmmRegister>();
3967         if (value == 0) {
3968           // easy FP 0.0.
3969           __ xorps(dest, dest);
3970         } else {
3971           __ movss(dest, codegen_->LiteralFloatAddress(fp_value));
3972         }
3973       } else {
3974         DCHECK(destination.IsStackSlot()) << destination;
3975         Immediate imm(value);
3976         __ movl(Address(CpuRegister(RSP), destination.GetStackIndex()), imm);
3977       }
3978     } else {
3979       DCHECK(constant->IsDoubleConstant()) << constant->DebugName();
3980       double fp_value =  constant->AsDoubleConstant()->GetValue();
3981       int64_t value = bit_cast<int64_t, double>(fp_value);
3982       if (destination.IsFpuRegister()) {
3983         XmmRegister dest = destination.AsFpuRegister<XmmRegister>();
3984         if (value == 0) {
3985           __ xorpd(dest, dest);
3986         } else {
3987           __ movsd(dest, codegen_->LiteralDoubleAddress(fp_value));
3988         }
3989       } else {
3990         DCHECK(destination.IsDoubleStackSlot()) << destination;
3991         codegen_->Load64BitValue(CpuRegister(TMP), value);
3992         __ movq(Address(CpuRegister(RSP), destination.GetStackIndex()), CpuRegister(TMP));
3993       }
3994     }
3995   } else if (source.IsFpuRegister()) {
3996     if (destination.IsFpuRegister()) {
3997       __ movaps(destination.AsFpuRegister<XmmRegister>(), source.AsFpuRegister<XmmRegister>());
3998     } else if (destination.IsStackSlot()) {
3999       __ movss(Address(CpuRegister(RSP), destination.GetStackIndex()),
4000                source.AsFpuRegister<XmmRegister>());
4001     } else {
4002       DCHECK(destination.IsDoubleStackSlot()) << destination;
4003       __ movsd(Address(CpuRegister(RSP), destination.GetStackIndex()),
4004                source.AsFpuRegister<XmmRegister>());
4005     }
4006   }
4007 }
4008 
Exchange32(CpuRegister reg,int mem)4009 void ParallelMoveResolverX86_64::Exchange32(CpuRegister reg, int mem) {
4010   __ movl(CpuRegister(TMP), Address(CpuRegister(RSP), mem));
4011   __ movl(Address(CpuRegister(RSP), mem), reg);
4012   __ movl(reg, CpuRegister(TMP));
4013 }
4014 
Exchange32(int mem1,int mem2)4015 void ParallelMoveResolverX86_64::Exchange32(int mem1, int mem2) {
4016   ScratchRegisterScope ensure_scratch(
4017       this, TMP, RAX, codegen_->GetNumberOfCoreRegisters());
4018 
4019   int stack_offset = ensure_scratch.IsSpilled() ? kX86_64WordSize : 0;
4020   __ movl(CpuRegister(TMP), Address(CpuRegister(RSP), mem1 + stack_offset));
4021   __ movl(CpuRegister(ensure_scratch.GetRegister()),
4022           Address(CpuRegister(RSP), mem2 + stack_offset));
4023   __ movl(Address(CpuRegister(RSP), mem2 + stack_offset), CpuRegister(TMP));
4024   __ movl(Address(CpuRegister(RSP), mem1 + stack_offset),
4025           CpuRegister(ensure_scratch.GetRegister()));
4026 }
4027 
Exchange64(CpuRegister reg,int mem)4028 void ParallelMoveResolverX86_64::Exchange64(CpuRegister reg, int mem) {
4029   __ movq(CpuRegister(TMP), Address(CpuRegister(RSP), mem));
4030   __ movq(Address(CpuRegister(RSP), mem), reg);
4031   __ movq(reg, CpuRegister(TMP));
4032 }
4033 
Exchange64(int mem1,int mem2)4034 void ParallelMoveResolverX86_64::Exchange64(int mem1, int mem2) {
4035   ScratchRegisterScope ensure_scratch(
4036       this, TMP, RAX, codegen_->GetNumberOfCoreRegisters());
4037 
4038   int stack_offset = ensure_scratch.IsSpilled() ? kX86_64WordSize : 0;
4039   __ movq(CpuRegister(TMP), Address(CpuRegister(RSP), mem1 + stack_offset));
4040   __ movq(CpuRegister(ensure_scratch.GetRegister()),
4041           Address(CpuRegister(RSP), mem2 + stack_offset));
4042   __ movq(Address(CpuRegister(RSP), mem2 + stack_offset), CpuRegister(TMP));
4043   __ movq(Address(CpuRegister(RSP), mem1 + stack_offset),
4044           CpuRegister(ensure_scratch.GetRegister()));
4045 }
4046 
Exchange32(XmmRegister reg,int mem)4047 void ParallelMoveResolverX86_64::Exchange32(XmmRegister reg, int mem) {
4048   __ movl(CpuRegister(TMP), Address(CpuRegister(RSP), mem));
4049   __ movss(Address(CpuRegister(RSP), mem), reg);
4050   __ movd(reg, CpuRegister(TMP));
4051 }
4052 
Exchange64(XmmRegister reg,int mem)4053 void ParallelMoveResolverX86_64::Exchange64(XmmRegister reg, int mem) {
4054   __ movq(CpuRegister(TMP), Address(CpuRegister(RSP), mem));
4055   __ movsd(Address(CpuRegister(RSP), mem), reg);
4056   __ movd(reg, CpuRegister(TMP));
4057 }
4058 
EmitSwap(size_t index)4059 void ParallelMoveResolverX86_64::EmitSwap(size_t index) {
4060   MoveOperands* move = moves_.Get(index);
4061   Location source = move->GetSource();
4062   Location destination = move->GetDestination();
4063 
4064   if (source.IsRegister() && destination.IsRegister()) {
4065     __ xchgq(destination.AsRegister<CpuRegister>(), source.AsRegister<CpuRegister>());
4066   } else if (source.IsRegister() && destination.IsStackSlot()) {
4067     Exchange32(source.AsRegister<CpuRegister>(), destination.GetStackIndex());
4068   } else if (source.IsStackSlot() && destination.IsRegister()) {
4069     Exchange32(destination.AsRegister<CpuRegister>(), source.GetStackIndex());
4070   } else if (source.IsStackSlot() && destination.IsStackSlot()) {
4071     Exchange32(destination.GetStackIndex(), source.GetStackIndex());
4072   } else if (source.IsRegister() && destination.IsDoubleStackSlot()) {
4073     Exchange64(source.AsRegister<CpuRegister>(), destination.GetStackIndex());
4074   } else if (source.IsDoubleStackSlot() && destination.IsRegister()) {
4075     Exchange64(destination.AsRegister<CpuRegister>(), source.GetStackIndex());
4076   } else if (source.IsDoubleStackSlot() && destination.IsDoubleStackSlot()) {
4077     Exchange64(destination.GetStackIndex(), source.GetStackIndex());
4078   } else if (source.IsFpuRegister() && destination.IsFpuRegister()) {
4079     __ movd(CpuRegister(TMP), source.AsFpuRegister<XmmRegister>());
4080     __ movaps(source.AsFpuRegister<XmmRegister>(), destination.AsFpuRegister<XmmRegister>());
4081     __ movd(destination.AsFpuRegister<XmmRegister>(), CpuRegister(TMP));
4082   } else if (source.IsFpuRegister() && destination.IsStackSlot()) {
4083     Exchange32(source.AsFpuRegister<XmmRegister>(), destination.GetStackIndex());
4084   } else if (source.IsStackSlot() && destination.IsFpuRegister()) {
4085     Exchange32(destination.AsFpuRegister<XmmRegister>(), source.GetStackIndex());
4086   } else if (source.IsFpuRegister() && destination.IsDoubleStackSlot()) {
4087     Exchange64(source.AsFpuRegister<XmmRegister>(), destination.GetStackIndex());
4088   } else if (source.IsDoubleStackSlot() && destination.IsFpuRegister()) {
4089     Exchange64(destination.AsFpuRegister<XmmRegister>(), source.GetStackIndex());
4090   } else {
4091     LOG(FATAL) << "Unimplemented swap between " << source << " and " << destination;
4092   }
4093 }
4094 
4095 
SpillScratch(int reg)4096 void ParallelMoveResolverX86_64::SpillScratch(int reg) {
4097   __ pushq(CpuRegister(reg));
4098 }
4099 
4100 
RestoreScratch(int reg)4101 void ParallelMoveResolverX86_64::RestoreScratch(int reg) {
4102   __ popq(CpuRegister(reg));
4103 }
4104 
GenerateClassInitializationCheck(SlowPathCodeX86_64 * slow_path,CpuRegister class_reg)4105 void InstructionCodeGeneratorX86_64::GenerateClassInitializationCheck(
4106     SlowPathCodeX86_64* slow_path, CpuRegister class_reg) {
4107   __ cmpl(Address(class_reg,  mirror::Class::StatusOffset().Int32Value()),
4108           Immediate(mirror::Class::kStatusInitialized));
4109   __ j(kLess, slow_path->GetEntryLabel());
4110   __ Bind(slow_path->GetExitLabel());
4111   // No need for memory fence, thanks to the X86_64 memory model.
4112 }
4113 
VisitLoadClass(HLoadClass * cls)4114 void LocationsBuilderX86_64::VisitLoadClass(HLoadClass* cls) {
4115   LocationSummary::CallKind call_kind = cls->CanCallRuntime()
4116       ? LocationSummary::kCallOnSlowPath
4117       : LocationSummary::kNoCall;
4118   LocationSummary* locations =
4119       new (GetGraph()->GetArena()) LocationSummary(cls, call_kind);
4120   locations->SetOut(Location::RequiresRegister());
4121 }
4122 
VisitLoadClass(HLoadClass * cls)4123 void InstructionCodeGeneratorX86_64::VisitLoadClass(HLoadClass* cls) {
4124   CpuRegister out = cls->GetLocations()->Out().AsRegister<CpuRegister>();
4125   if (cls->IsReferrersClass()) {
4126     DCHECK(!cls->CanCallRuntime());
4127     DCHECK(!cls->MustGenerateClinitCheck());
4128     codegen_->LoadCurrentMethod(out);
4129     __ movl(out, Address(out, ArtMethod::DeclaringClassOffset().Int32Value()));
4130   } else {
4131     DCHECK(cls->CanCallRuntime());
4132     codegen_->LoadCurrentMethod(out);
4133     __ movl(out, Address(out, ArtMethod::DexCacheResolvedTypesOffset().Int32Value()));
4134     __ movl(out, Address(out, CodeGenerator::GetCacheOffset(cls->GetTypeIndex())));
4135     SlowPathCodeX86_64* slow_path = new (GetGraph()->GetArena()) LoadClassSlowPathX86_64(
4136         cls, cls, cls->GetDexPc(), cls->MustGenerateClinitCheck());
4137     codegen_->AddSlowPath(slow_path);
4138     __ testl(out, out);
4139     __ j(kEqual, slow_path->GetEntryLabel());
4140     if (cls->MustGenerateClinitCheck()) {
4141       GenerateClassInitializationCheck(slow_path, out);
4142     } else {
4143       __ Bind(slow_path->GetExitLabel());
4144     }
4145   }
4146 }
4147 
VisitClinitCheck(HClinitCheck * check)4148 void LocationsBuilderX86_64::VisitClinitCheck(HClinitCheck* check) {
4149   LocationSummary* locations =
4150       new (GetGraph()->GetArena()) LocationSummary(check, LocationSummary::kCallOnSlowPath);
4151   locations->SetInAt(0, Location::RequiresRegister());
4152   if (check->HasUses()) {
4153     locations->SetOut(Location::SameAsFirstInput());
4154   }
4155 }
4156 
VisitClinitCheck(HClinitCheck * check)4157 void InstructionCodeGeneratorX86_64::VisitClinitCheck(HClinitCheck* check) {
4158   // We assume the class to not be null.
4159   SlowPathCodeX86_64* slow_path = new (GetGraph()->GetArena()) LoadClassSlowPathX86_64(
4160       check->GetLoadClass(), check, check->GetDexPc(), true);
4161   codegen_->AddSlowPath(slow_path);
4162   GenerateClassInitializationCheck(slow_path,
4163                                    check->GetLocations()->InAt(0).AsRegister<CpuRegister>());
4164 }
4165 
VisitLoadString(HLoadString * load)4166 void LocationsBuilderX86_64::VisitLoadString(HLoadString* load) {
4167   LocationSummary* locations =
4168       new (GetGraph()->GetArena()) LocationSummary(load, LocationSummary::kCallOnSlowPath);
4169   locations->SetOut(Location::RequiresRegister());
4170 }
4171 
VisitLoadString(HLoadString * load)4172 void InstructionCodeGeneratorX86_64::VisitLoadString(HLoadString* load) {
4173   SlowPathCodeX86_64* slow_path = new (GetGraph()->GetArena()) LoadStringSlowPathX86_64(load);
4174   codegen_->AddSlowPath(slow_path);
4175 
4176   CpuRegister out = load->GetLocations()->Out().AsRegister<CpuRegister>();
4177   codegen_->LoadCurrentMethod(CpuRegister(out));
4178   __ movl(out, Address(out, ArtMethod::DeclaringClassOffset().Int32Value()));
4179   __ movl(out, Address(out, mirror::Class::DexCacheStringsOffset().Int32Value()));
4180   __ movl(out, Address(out, CodeGenerator::GetCacheOffset(load->GetStringIndex())));
4181   __ testl(out, out);
4182   __ j(kEqual, slow_path->GetEntryLabel());
4183   __ Bind(slow_path->GetExitLabel());
4184 }
4185 
VisitLoadException(HLoadException * load)4186 void LocationsBuilderX86_64::VisitLoadException(HLoadException* load) {
4187   LocationSummary* locations =
4188       new (GetGraph()->GetArena()) LocationSummary(load, LocationSummary::kNoCall);
4189   locations->SetOut(Location::RequiresRegister());
4190 }
4191 
VisitLoadException(HLoadException * load)4192 void InstructionCodeGeneratorX86_64::VisitLoadException(HLoadException* load) {
4193   Address address = Address::Absolute(
4194       Thread::ExceptionOffset<kX86_64WordSize>().Int32Value(), true);
4195   __ gs()->movl(load->GetLocations()->Out().AsRegister<CpuRegister>(), address);
4196   __ gs()->movl(address, Immediate(0));
4197 }
4198 
VisitThrow(HThrow * instruction)4199 void LocationsBuilderX86_64::VisitThrow(HThrow* instruction) {
4200   LocationSummary* locations =
4201       new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCall);
4202   InvokeRuntimeCallingConvention calling_convention;
4203   locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(0)));
4204 }
4205 
VisitThrow(HThrow * instruction)4206 void InstructionCodeGeneratorX86_64::VisitThrow(HThrow* instruction) {
4207   __ gs()->call(
4208         Address::Absolute(QUICK_ENTRYPOINT_OFFSET(kX86_64WordSize, pDeliverException), true));
4209   codegen_->RecordPcInfo(instruction, instruction->GetDexPc());
4210 }
4211 
VisitInstanceOf(HInstanceOf * instruction)4212 void LocationsBuilderX86_64::VisitInstanceOf(HInstanceOf* instruction) {
4213   LocationSummary::CallKind call_kind = instruction->IsClassFinal()
4214       ? LocationSummary::kNoCall
4215       : LocationSummary::kCallOnSlowPath;
4216   LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction, call_kind);
4217   locations->SetInAt(0, Location::RequiresRegister());
4218   locations->SetInAt(1, Location::Any());
4219   locations->SetOut(Location::RequiresRegister());
4220 }
4221 
VisitInstanceOf(HInstanceOf * instruction)4222 void InstructionCodeGeneratorX86_64::VisitInstanceOf(HInstanceOf* instruction) {
4223   LocationSummary* locations = instruction->GetLocations();
4224   CpuRegister obj = locations->InAt(0).AsRegister<CpuRegister>();
4225   Location cls = locations->InAt(1);
4226   CpuRegister out = locations->Out().AsRegister<CpuRegister>();
4227   uint32_t class_offset = mirror::Object::ClassOffset().Int32Value();
4228   Label done, zero;
4229   SlowPathCodeX86_64* slow_path = nullptr;
4230 
4231   // Return 0 if `obj` is null.
4232   // Avoid null check if we know obj is not null.
4233   if (instruction->MustDoNullCheck()) {
4234     __ testl(obj, obj);
4235     __ j(kEqual, &zero);
4236   }
4237   // Compare the class of `obj` with `cls`.
4238   __ movl(out, Address(obj, class_offset));
4239   if (cls.IsRegister()) {
4240     __ cmpl(out, cls.AsRegister<CpuRegister>());
4241   } else {
4242     DCHECK(cls.IsStackSlot()) << cls;
4243     __ cmpl(out, Address(CpuRegister(RSP), cls.GetStackIndex()));
4244   }
4245   if (instruction->IsClassFinal()) {
4246     // Classes must be equal for the instanceof to succeed.
4247     __ j(kNotEqual, &zero);
4248     __ movl(out, Immediate(1));
4249     __ jmp(&done);
4250   } else {
4251     // If the classes are not equal, we go into a slow path.
4252     DCHECK(locations->OnlyCallsOnSlowPath());
4253     slow_path = new (GetGraph()->GetArena()) TypeCheckSlowPathX86_64(
4254         instruction, locations->InAt(1), locations->Out(), instruction->GetDexPc());
4255     codegen_->AddSlowPath(slow_path);
4256     __ j(kNotEqual, slow_path->GetEntryLabel());
4257     __ movl(out, Immediate(1));
4258     __ jmp(&done);
4259   }
4260 
4261   if (instruction->MustDoNullCheck() || instruction->IsClassFinal()) {
4262     __ Bind(&zero);
4263     __ movl(out, Immediate(0));
4264   }
4265 
4266   if (slow_path != nullptr) {
4267     __ Bind(slow_path->GetExitLabel());
4268   }
4269   __ Bind(&done);
4270 }
4271 
VisitCheckCast(HCheckCast * instruction)4272 void LocationsBuilderX86_64::VisitCheckCast(HCheckCast* instruction) {
4273   LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(
4274       instruction, LocationSummary::kCallOnSlowPath);
4275   locations->SetInAt(0, Location::RequiresRegister());
4276   locations->SetInAt(1, Location::Any());
4277   locations->AddTemp(Location::RequiresRegister());
4278 }
4279 
VisitCheckCast(HCheckCast * instruction)4280 void InstructionCodeGeneratorX86_64::VisitCheckCast(HCheckCast* instruction) {
4281   LocationSummary* locations = instruction->GetLocations();
4282   CpuRegister obj = locations->InAt(0).AsRegister<CpuRegister>();
4283   Location cls = locations->InAt(1);
4284   CpuRegister temp = locations->GetTemp(0).AsRegister<CpuRegister>();
4285   uint32_t class_offset = mirror::Object::ClassOffset().Int32Value();
4286   SlowPathCodeX86_64* slow_path = new (GetGraph()->GetArena()) TypeCheckSlowPathX86_64(
4287       instruction, locations->InAt(1), locations->GetTemp(0), instruction->GetDexPc());
4288   codegen_->AddSlowPath(slow_path);
4289 
4290   // Avoid null check if we know obj is not null.
4291   if (instruction->MustDoNullCheck()) {
4292     __ testl(obj, obj);
4293     __ j(kEqual, slow_path->GetExitLabel());
4294   }
4295   // Compare the class of `obj` with `cls`.
4296   __ movl(temp, Address(obj, class_offset));
4297   if (cls.IsRegister()) {
4298     __ cmpl(temp, cls.AsRegister<CpuRegister>());
4299   } else {
4300     DCHECK(cls.IsStackSlot()) << cls;
4301     __ cmpl(temp, Address(CpuRegister(RSP), cls.GetStackIndex()));
4302   }
4303   // Classes must be equal for the checkcast to succeed.
4304   __ j(kNotEqual, slow_path->GetEntryLabel());
4305   __ Bind(slow_path->GetExitLabel());
4306 }
4307 
VisitMonitorOperation(HMonitorOperation * instruction)4308 void LocationsBuilderX86_64::VisitMonitorOperation(HMonitorOperation* instruction) {
4309   LocationSummary* locations =
4310       new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCall);
4311   InvokeRuntimeCallingConvention calling_convention;
4312   locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(0)));
4313 }
4314 
VisitMonitorOperation(HMonitorOperation * instruction)4315 void InstructionCodeGeneratorX86_64::VisitMonitorOperation(HMonitorOperation* instruction) {
4316   __ gs()->call(Address::Absolute(instruction->IsEnter()
4317         ? QUICK_ENTRYPOINT_OFFSET(kX86_64WordSize, pLockObject)
4318         : QUICK_ENTRYPOINT_OFFSET(kX86_64WordSize, pUnlockObject),
4319       true));
4320   codegen_->RecordPcInfo(instruction, instruction->GetDexPc());
4321 }
4322 
VisitAnd(HAnd * instruction)4323 void LocationsBuilderX86_64::VisitAnd(HAnd* instruction) { HandleBitwiseOperation(instruction); }
VisitOr(HOr * instruction)4324 void LocationsBuilderX86_64::VisitOr(HOr* instruction) { HandleBitwiseOperation(instruction); }
VisitXor(HXor * instruction)4325 void LocationsBuilderX86_64::VisitXor(HXor* instruction) { HandleBitwiseOperation(instruction); }
4326 
HandleBitwiseOperation(HBinaryOperation * instruction)4327 void LocationsBuilderX86_64::HandleBitwiseOperation(HBinaryOperation* instruction) {
4328   LocationSummary* locations =
4329       new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
4330   DCHECK(instruction->GetResultType() == Primitive::kPrimInt
4331          || instruction->GetResultType() == Primitive::kPrimLong);
4332   locations->SetInAt(0, Location::RequiresRegister());
4333   locations->SetInAt(1, Location::Any());
4334   locations->SetOut(Location::SameAsFirstInput());
4335 }
4336 
VisitAnd(HAnd * instruction)4337 void InstructionCodeGeneratorX86_64::VisitAnd(HAnd* instruction) {
4338   HandleBitwiseOperation(instruction);
4339 }
4340 
VisitOr(HOr * instruction)4341 void InstructionCodeGeneratorX86_64::VisitOr(HOr* instruction) {
4342   HandleBitwiseOperation(instruction);
4343 }
4344 
VisitXor(HXor * instruction)4345 void InstructionCodeGeneratorX86_64::VisitXor(HXor* instruction) {
4346   HandleBitwiseOperation(instruction);
4347 }
4348 
HandleBitwiseOperation(HBinaryOperation * instruction)4349 void InstructionCodeGeneratorX86_64::HandleBitwiseOperation(HBinaryOperation* instruction) {
4350   LocationSummary* locations = instruction->GetLocations();
4351   Location first = locations->InAt(0);
4352   Location second = locations->InAt(1);
4353   DCHECK(first.Equals(locations->Out()));
4354 
4355   if (instruction->GetResultType() == Primitive::kPrimInt) {
4356     if (second.IsRegister()) {
4357       if (instruction->IsAnd()) {
4358         __ andl(first.AsRegister<CpuRegister>(), second.AsRegister<CpuRegister>());
4359       } else if (instruction->IsOr()) {
4360         __ orl(first.AsRegister<CpuRegister>(), second.AsRegister<CpuRegister>());
4361       } else {
4362         DCHECK(instruction->IsXor());
4363         __ xorl(first.AsRegister<CpuRegister>(), second.AsRegister<CpuRegister>());
4364       }
4365     } else if (second.IsConstant()) {
4366       Immediate imm(second.GetConstant()->AsIntConstant()->GetValue());
4367       if (instruction->IsAnd()) {
4368         __ andl(first.AsRegister<CpuRegister>(), imm);
4369       } else if (instruction->IsOr()) {
4370         __ orl(first.AsRegister<CpuRegister>(), imm);
4371       } else {
4372         DCHECK(instruction->IsXor());
4373         __ xorl(first.AsRegister<CpuRegister>(), imm);
4374       }
4375     } else {
4376       Address address(CpuRegister(RSP), second.GetStackIndex());
4377       if (instruction->IsAnd()) {
4378         __ andl(first.AsRegister<CpuRegister>(), address);
4379       } else if (instruction->IsOr()) {
4380         __ orl(first.AsRegister<CpuRegister>(), address);
4381       } else {
4382         DCHECK(instruction->IsXor());
4383         __ xorl(first.AsRegister<CpuRegister>(), address);
4384       }
4385     }
4386   } else {
4387     DCHECK_EQ(instruction->GetResultType(), Primitive::kPrimLong);
4388     CpuRegister first_reg = first.AsRegister<CpuRegister>();
4389     bool second_is_constant = false;
4390     int64_t value = 0;
4391     if (second.IsConstant()) {
4392       second_is_constant = true;
4393       value = second.GetConstant()->AsLongConstant()->GetValue();
4394     }
4395     bool is_int32_value = IsInt<32>(value);
4396 
4397     if (instruction->IsAnd()) {
4398       if (second_is_constant) {
4399         if (is_int32_value) {
4400           __ andq(first_reg, Immediate(static_cast<int32_t>(value)));
4401         } else {
4402           __ andq(first_reg, codegen_->LiteralInt64Address(value));
4403         }
4404       } else if (second.IsDoubleStackSlot()) {
4405         __ andq(first_reg, Address(CpuRegister(RSP), second.GetStackIndex()));
4406       } else {
4407         __ andq(first_reg, second.AsRegister<CpuRegister>());
4408       }
4409     } else if (instruction->IsOr()) {
4410       if (second_is_constant) {
4411         if (is_int32_value) {
4412           __ orq(first_reg, Immediate(static_cast<int32_t>(value)));
4413         } else {
4414           __ orq(first_reg, codegen_->LiteralInt64Address(value));
4415         }
4416       } else if (second.IsDoubleStackSlot()) {
4417         __ orq(first_reg, Address(CpuRegister(RSP), second.GetStackIndex()));
4418       } else {
4419         __ orq(first_reg, second.AsRegister<CpuRegister>());
4420       }
4421     } else {
4422       DCHECK(instruction->IsXor());
4423       if (second_is_constant) {
4424         if (is_int32_value) {
4425           __ xorq(first_reg, Immediate(static_cast<int32_t>(value)));
4426         } else {
4427           __ xorq(first_reg, codegen_->LiteralInt64Address(value));
4428         }
4429       } else if (second.IsDoubleStackSlot()) {
4430         __ xorq(first_reg, Address(CpuRegister(RSP), second.GetStackIndex()));
4431       } else {
4432         __ xorq(first_reg, second.AsRegister<CpuRegister>());
4433       }
4434     }
4435   }
4436 }
4437 
VisitBoundType(HBoundType * instruction)4438 void LocationsBuilderX86_64::VisitBoundType(HBoundType* instruction) {
4439   // Nothing to do, this should be removed during prepare for register allocator.
4440   UNUSED(instruction);
4441   LOG(FATAL) << "Unreachable";
4442 }
4443 
VisitBoundType(HBoundType * instruction)4444 void InstructionCodeGeneratorX86_64::VisitBoundType(HBoundType* instruction) {
4445   // Nothing to do, this should be removed during prepare for register allocator.
4446   UNUSED(instruction);
4447   LOG(FATAL) << "Unreachable";
4448 }
4449 
Load64BitValue(CpuRegister dest,int64_t value)4450 void CodeGeneratorX86_64::Load64BitValue(CpuRegister dest, int64_t value) {
4451   if (value == 0) {
4452     __ xorl(dest, dest);
4453   } else if (value > 0 && IsInt<32>(value)) {
4454     // We can use a 32 bit move, as it will zero-extend and is one byte shorter.
4455     __ movl(dest, Immediate(static_cast<int32_t>(value)));
4456   } else {
4457     __ movq(dest, Immediate(value));
4458   }
4459 }
4460 
Finalize(CodeAllocator * allocator)4461 void CodeGeneratorX86_64::Finalize(CodeAllocator* allocator) {
4462   // Generate the constant area if needed.
4463   X86_64Assembler* assembler = GetAssembler();
4464   if (!assembler->IsConstantAreaEmpty()) {
4465     // Align to 4 byte boundary to reduce cache misses, as the data is 4 and 8
4466     // byte values.  If used for vectors at a later time, this will need to be
4467     // updated to 16 bytes with the appropriate offset.
4468     assembler->Align(4, 0);
4469     constant_area_start_ = assembler->CodeSize();
4470     assembler->AddConstantArea();
4471   }
4472 
4473   // And finish up.
4474   CodeGenerator::Finalize(allocator);
4475 }
4476 
4477 /**
4478  * Class to handle late fixup of offsets into constant area.
4479  */
4480 class RIPFixup : public AssemblerFixup, public ArenaObject<kArenaAllocMisc> {
4481   public:
RIPFixup(const CodeGeneratorX86_64 & codegen,int offset)4482     RIPFixup(const CodeGeneratorX86_64& codegen, int offset)
4483       : codegen_(codegen), offset_into_constant_area_(offset) {}
4484 
4485   private:
Process(const MemoryRegion & region,int pos)4486     void Process(const MemoryRegion& region, int pos) OVERRIDE {
4487       // Patch the correct offset for the instruction.  We use the address of the
4488       // 'next' instruction, which is 'pos' (patch the 4 bytes before).
4489       int constant_offset = codegen_.ConstantAreaStart() + offset_into_constant_area_;
4490       int relative_position = constant_offset - pos;
4491 
4492       // Patch in the right value.
4493       region.StoreUnaligned<int32_t>(pos - 4, relative_position);
4494     }
4495 
4496     const CodeGeneratorX86_64& codegen_;
4497 
4498     // Location in constant area that the fixup refers to.
4499     int offset_into_constant_area_;
4500 };
4501 
LiteralDoubleAddress(double v)4502 Address CodeGeneratorX86_64::LiteralDoubleAddress(double v) {
4503   AssemblerFixup* fixup = new (GetGraph()->GetArena()) RIPFixup(*this, __ AddDouble(v));
4504   return Address::RIP(fixup);
4505 }
4506 
LiteralFloatAddress(float v)4507 Address CodeGeneratorX86_64::LiteralFloatAddress(float v) {
4508   AssemblerFixup* fixup = new (GetGraph()->GetArena()) RIPFixup(*this, __ AddFloat(v));
4509   return Address::RIP(fixup);
4510 }
4511 
LiteralInt32Address(int32_t v)4512 Address CodeGeneratorX86_64::LiteralInt32Address(int32_t v) {
4513   AssemblerFixup* fixup = new (GetGraph()->GetArena()) RIPFixup(*this, __ AddInt32(v));
4514   return Address::RIP(fixup);
4515 }
4516 
LiteralInt64Address(int64_t v)4517 Address CodeGeneratorX86_64::LiteralInt64Address(int64_t v) {
4518   AssemblerFixup* fixup = new (GetGraph()->GetArena()) RIPFixup(*this, __ AddInt64(v));
4519   return Address::RIP(fixup);
4520 }
4521 
4522 }  // namespace x86_64
4523 }  // namespace art
4524