1 /*
2 * Copyright (C) 2014 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 #include "code_generator_x86.h"
18
19 #include "art_method.h"
20 #include "code_generator_utils.h"
21 #include "compiled_method.h"
22 #include "entrypoints/quick/quick_entrypoints.h"
23 #include "entrypoints/quick/quick_entrypoints_enum.h"
24 #include "gc/accounting/card_table.h"
25 #include "intrinsics.h"
26 #include "intrinsics_x86.h"
27 #include "mirror/array-inl.h"
28 #include "mirror/class-inl.h"
29 #include "thread.h"
30 #include "utils/assembler.h"
31 #include "utils/stack_checks.h"
32 #include "utils/x86/assembler_x86.h"
33 #include "utils/x86/managed_register_x86.h"
34
35 namespace art {
36
37 template<class MirrorType>
38 class GcRoot;
39
40 namespace x86 {
41
42 static constexpr int kCurrentMethodStackOffset = 0;
43 static constexpr Register kMethodRegisterArgument = EAX;
44 static constexpr Register kCoreCalleeSaves[] = { EBP, ESI, EDI };
45
46 static constexpr int kC2ConditionMask = 0x400;
47
48 static constexpr int kFakeReturnRegister = Register(8);
49
50 #define __ down_cast<X86Assembler*>(codegen->GetAssembler())->
51 #define QUICK_ENTRY_POINT(x) QUICK_ENTRYPOINT_OFFSET(kX86WordSize, x).Int32Value()
52
53 class NullCheckSlowPathX86 : public SlowPathCode {
54 public:
NullCheckSlowPathX86(HNullCheck * instruction)55 explicit NullCheckSlowPathX86(HNullCheck* instruction) : SlowPathCode(instruction) {}
56
EmitNativeCode(CodeGenerator * codegen)57 void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
58 CodeGeneratorX86* x86_codegen = down_cast<CodeGeneratorX86*>(codegen);
59 __ Bind(GetEntryLabel());
60 if (instruction_->CanThrowIntoCatchBlock()) {
61 // Live registers will be restored in the catch block if caught.
62 SaveLiveRegisters(codegen, instruction_->GetLocations());
63 }
64 x86_codegen->InvokeRuntime(QUICK_ENTRY_POINT(pThrowNullPointer),
65 instruction_,
66 instruction_->GetDexPc(),
67 this);
68 CheckEntrypointTypes<kQuickThrowNullPointer, void, void>();
69 }
70
IsFatal() const71 bool IsFatal() const OVERRIDE { return true; }
72
GetDescription() const73 const char* GetDescription() const OVERRIDE { return "NullCheckSlowPathX86"; }
74
75 private:
76 DISALLOW_COPY_AND_ASSIGN(NullCheckSlowPathX86);
77 };
78
79 class DivZeroCheckSlowPathX86 : public SlowPathCode {
80 public:
DivZeroCheckSlowPathX86(HDivZeroCheck * instruction)81 explicit DivZeroCheckSlowPathX86(HDivZeroCheck* instruction) : SlowPathCode(instruction) {}
82
EmitNativeCode(CodeGenerator * codegen)83 void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
84 CodeGeneratorX86* x86_codegen = down_cast<CodeGeneratorX86*>(codegen);
85 __ Bind(GetEntryLabel());
86 if (instruction_->CanThrowIntoCatchBlock()) {
87 // Live registers will be restored in the catch block if caught.
88 SaveLiveRegisters(codegen, instruction_->GetLocations());
89 }
90 x86_codegen->InvokeRuntime(QUICK_ENTRY_POINT(pThrowDivZero),
91 instruction_,
92 instruction_->GetDexPc(),
93 this);
94 CheckEntrypointTypes<kQuickThrowDivZero, void, void>();
95 }
96
IsFatal() const97 bool IsFatal() const OVERRIDE { return true; }
98
GetDescription() const99 const char* GetDescription() const OVERRIDE { return "DivZeroCheckSlowPathX86"; }
100
101 private:
102 DISALLOW_COPY_AND_ASSIGN(DivZeroCheckSlowPathX86);
103 };
104
105 class DivRemMinusOneSlowPathX86 : public SlowPathCode {
106 public:
DivRemMinusOneSlowPathX86(HInstruction * instruction,Register reg,bool is_div)107 DivRemMinusOneSlowPathX86(HInstruction* instruction, Register reg, bool is_div)
108 : SlowPathCode(instruction), reg_(reg), is_div_(is_div) {}
109
EmitNativeCode(CodeGenerator * codegen)110 void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
111 __ Bind(GetEntryLabel());
112 if (is_div_) {
113 __ negl(reg_);
114 } else {
115 __ movl(reg_, Immediate(0));
116 }
117 __ jmp(GetExitLabel());
118 }
119
GetDescription() const120 const char* GetDescription() const OVERRIDE { return "DivRemMinusOneSlowPathX86"; }
121
122 private:
123 Register reg_;
124 bool is_div_;
125 DISALLOW_COPY_AND_ASSIGN(DivRemMinusOneSlowPathX86);
126 };
127
128 class BoundsCheckSlowPathX86 : public SlowPathCode {
129 public:
BoundsCheckSlowPathX86(HBoundsCheck * instruction)130 explicit BoundsCheckSlowPathX86(HBoundsCheck* instruction) : SlowPathCode(instruction) {}
131
EmitNativeCode(CodeGenerator * codegen)132 void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
133 LocationSummary* locations = instruction_->GetLocations();
134 CodeGeneratorX86* x86_codegen = down_cast<CodeGeneratorX86*>(codegen);
135 __ Bind(GetEntryLabel());
136 // We're moving two locations to locations that could overlap, so we need a parallel
137 // move resolver.
138 if (instruction_->CanThrowIntoCatchBlock()) {
139 // Live registers will be restored in the catch block if caught.
140 SaveLiveRegisters(codegen, instruction_->GetLocations());
141 }
142 InvokeRuntimeCallingConvention calling_convention;
143 x86_codegen->EmitParallelMoves(
144 locations->InAt(0),
145 Location::RegisterLocation(calling_convention.GetRegisterAt(0)),
146 Primitive::kPrimInt,
147 locations->InAt(1),
148 Location::RegisterLocation(calling_convention.GetRegisterAt(1)),
149 Primitive::kPrimInt);
150 x86_codegen->InvokeRuntime(QUICK_ENTRY_POINT(pThrowArrayBounds),
151 instruction_,
152 instruction_->GetDexPc(),
153 this);
154 CheckEntrypointTypes<kQuickThrowArrayBounds, void, int32_t, int32_t>();
155 }
156
IsFatal() const157 bool IsFatal() const OVERRIDE { return true; }
158
GetDescription() const159 const char* GetDescription() const OVERRIDE { return "BoundsCheckSlowPathX86"; }
160
161 private:
162 DISALLOW_COPY_AND_ASSIGN(BoundsCheckSlowPathX86);
163 };
164
165 class SuspendCheckSlowPathX86 : public SlowPathCode {
166 public:
SuspendCheckSlowPathX86(HSuspendCheck * instruction,HBasicBlock * successor)167 SuspendCheckSlowPathX86(HSuspendCheck* instruction, HBasicBlock* successor)
168 : SlowPathCode(instruction), successor_(successor) {}
169
EmitNativeCode(CodeGenerator * codegen)170 void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
171 CodeGeneratorX86* x86_codegen = down_cast<CodeGeneratorX86*>(codegen);
172 __ Bind(GetEntryLabel());
173 SaveLiveRegisters(codegen, instruction_->GetLocations());
174 x86_codegen->InvokeRuntime(QUICK_ENTRY_POINT(pTestSuspend),
175 instruction_,
176 instruction_->GetDexPc(),
177 this);
178 CheckEntrypointTypes<kQuickTestSuspend, void, void>();
179 RestoreLiveRegisters(codegen, instruction_->GetLocations());
180 if (successor_ == nullptr) {
181 __ jmp(GetReturnLabel());
182 } else {
183 __ jmp(x86_codegen->GetLabelOf(successor_));
184 }
185 }
186
GetReturnLabel()187 Label* GetReturnLabel() {
188 DCHECK(successor_ == nullptr);
189 return &return_label_;
190 }
191
GetSuccessor() const192 HBasicBlock* GetSuccessor() const {
193 return successor_;
194 }
195
GetDescription() const196 const char* GetDescription() const OVERRIDE { return "SuspendCheckSlowPathX86"; }
197
198 private:
199 HBasicBlock* const successor_;
200 Label return_label_;
201
202 DISALLOW_COPY_AND_ASSIGN(SuspendCheckSlowPathX86);
203 };
204
205 class LoadStringSlowPathX86 : public SlowPathCode {
206 public:
LoadStringSlowPathX86(HLoadString * instruction)207 explicit LoadStringSlowPathX86(HLoadString* instruction): SlowPathCode(instruction) {}
208
EmitNativeCode(CodeGenerator * codegen)209 void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
210 LocationSummary* locations = instruction_->GetLocations();
211 DCHECK(!locations->GetLiveRegisters()->ContainsCoreRegister(locations->Out().reg()));
212
213 CodeGeneratorX86* x86_codegen = down_cast<CodeGeneratorX86*>(codegen);
214 __ Bind(GetEntryLabel());
215 SaveLiveRegisters(codegen, locations);
216
217 InvokeRuntimeCallingConvention calling_convention;
218 const uint32_t string_index = instruction_->AsLoadString()->GetStringIndex();
219 __ movl(calling_convention.GetRegisterAt(0), Immediate(string_index));
220 x86_codegen->InvokeRuntime(QUICK_ENTRY_POINT(pResolveString),
221 instruction_,
222 instruction_->GetDexPc(),
223 this);
224 CheckEntrypointTypes<kQuickResolveString, void*, uint32_t>();
225 x86_codegen->Move32(locations->Out(), Location::RegisterLocation(EAX));
226 RestoreLiveRegisters(codegen, locations);
227
228 __ jmp(GetExitLabel());
229 }
230
GetDescription() const231 const char* GetDescription() const OVERRIDE { return "LoadStringSlowPathX86"; }
232
233 private:
234 DISALLOW_COPY_AND_ASSIGN(LoadStringSlowPathX86);
235 };
236
237 class LoadClassSlowPathX86 : public SlowPathCode {
238 public:
LoadClassSlowPathX86(HLoadClass * cls,HInstruction * at,uint32_t dex_pc,bool do_clinit)239 LoadClassSlowPathX86(HLoadClass* cls,
240 HInstruction* at,
241 uint32_t dex_pc,
242 bool do_clinit)
243 : SlowPathCode(at), cls_(cls), at_(at), dex_pc_(dex_pc), do_clinit_(do_clinit) {
244 DCHECK(at->IsLoadClass() || at->IsClinitCheck());
245 }
246
EmitNativeCode(CodeGenerator * codegen)247 void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
248 LocationSummary* locations = at_->GetLocations();
249 CodeGeneratorX86* x86_codegen = down_cast<CodeGeneratorX86*>(codegen);
250 __ Bind(GetEntryLabel());
251 SaveLiveRegisters(codegen, locations);
252
253 InvokeRuntimeCallingConvention calling_convention;
254 __ movl(calling_convention.GetRegisterAt(0), Immediate(cls_->GetTypeIndex()));
255 x86_codegen->InvokeRuntime(do_clinit_ ? QUICK_ENTRY_POINT(pInitializeStaticStorage)
256 : QUICK_ENTRY_POINT(pInitializeType),
257 at_, dex_pc_, this);
258 if (do_clinit_) {
259 CheckEntrypointTypes<kQuickInitializeStaticStorage, void*, uint32_t>();
260 } else {
261 CheckEntrypointTypes<kQuickInitializeType, void*, uint32_t>();
262 }
263
264 // Move the class to the desired location.
265 Location out = locations->Out();
266 if (out.IsValid()) {
267 DCHECK(out.IsRegister() && !locations->GetLiveRegisters()->ContainsCoreRegister(out.reg()));
268 x86_codegen->Move32(out, Location::RegisterLocation(EAX));
269 }
270
271 RestoreLiveRegisters(codegen, locations);
272 __ jmp(GetExitLabel());
273 }
274
GetDescription() const275 const char* GetDescription() const OVERRIDE { return "LoadClassSlowPathX86"; }
276
277 private:
278 // The class this slow path will load.
279 HLoadClass* const cls_;
280
281 // The instruction where this slow path is happening.
282 // (Might be the load class or an initialization check).
283 HInstruction* const at_;
284
285 // The dex PC of `at_`.
286 const uint32_t dex_pc_;
287
288 // Whether to initialize the class.
289 const bool do_clinit_;
290
291 DISALLOW_COPY_AND_ASSIGN(LoadClassSlowPathX86);
292 };
293
294 class TypeCheckSlowPathX86 : public SlowPathCode {
295 public:
TypeCheckSlowPathX86(HInstruction * instruction,bool is_fatal)296 TypeCheckSlowPathX86(HInstruction* instruction, bool is_fatal)
297 : SlowPathCode(instruction), is_fatal_(is_fatal) {}
298
EmitNativeCode(CodeGenerator * codegen)299 void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
300 LocationSummary* locations = instruction_->GetLocations();
301 Location object_class = instruction_->IsCheckCast() ? locations->GetTemp(0)
302 : locations->Out();
303 DCHECK(instruction_->IsCheckCast()
304 || !locations->GetLiveRegisters()->ContainsCoreRegister(locations->Out().reg()));
305
306 CodeGeneratorX86* x86_codegen = down_cast<CodeGeneratorX86*>(codegen);
307 __ Bind(GetEntryLabel());
308
309 if (!is_fatal_) {
310 SaveLiveRegisters(codegen, locations);
311 }
312
313 // We're moving two locations to locations that could overlap, so we need a parallel
314 // move resolver.
315 InvokeRuntimeCallingConvention calling_convention;
316 x86_codegen->EmitParallelMoves(
317 locations->InAt(1),
318 Location::RegisterLocation(calling_convention.GetRegisterAt(0)),
319 Primitive::kPrimNot,
320 object_class,
321 Location::RegisterLocation(calling_convention.GetRegisterAt(1)),
322 Primitive::kPrimNot);
323
324 if (instruction_->IsInstanceOf()) {
325 x86_codegen->InvokeRuntime(QUICK_ENTRY_POINT(pInstanceofNonTrivial),
326 instruction_,
327 instruction_->GetDexPc(),
328 this);
329 CheckEntrypointTypes<
330 kQuickInstanceofNonTrivial, uint32_t, const mirror::Class*, const mirror::Class*>();
331 } else {
332 DCHECK(instruction_->IsCheckCast());
333 x86_codegen->InvokeRuntime(QUICK_ENTRY_POINT(pCheckCast),
334 instruction_,
335 instruction_->GetDexPc(),
336 this);
337 CheckEntrypointTypes<kQuickCheckCast, void, const mirror::Class*, const mirror::Class*>();
338 }
339
340 if (!is_fatal_) {
341 if (instruction_->IsInstanceOf()) {
342 x86_codegen->Move32(locations->Out(), Location::RegisterLocation(EAX));
343 }
344 RestoreLiveRegisters(codegen, locations);
345
346 __ jmp(GetExitLabel());
347 }
348 }
349
GetDescription() const350 const char* GetDescription() const OVERRIDE { return "TypeCheckSlowPathX86"; }
IsFatal() const351 bool IsFatal() const OVERRIDE { return is_fatal_; }
352
353 private:
354 const bool is_fatal_;
355
356 DISALLOW_COPY_AND_ASSIGN(TypeCheckSlowPathX86);
357 };
358
359 class DeoptimizationSlowPathX86 : public SlowPathCode {
360 public:
DeoptimizationSlowPathX86(HDeoptimize * instruction)361 explicit DeoptimizationSlowPathX86(HDeoptimize* instruction)
362 : SlowPathCode(instruction) {}
363
EmitNativeCode(CodeGenerator * codegen)364 void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
365 CodeGeneratorX86* x86_codegen = down_cast<CodeGeneratorX86*>(codegen);
366 __ Bind(GetEntryLabel());
367 SaveLiveRegisters(codegen, instruction_->GetLocations());
368 x86_codegen->InvokeRuntime(QUICK_ENTRY_POINT(pDeoptimize),
369 instruction_,
370 instruction_->GetDexPc(),
371 this);
372 CheckEntrypointTypes<kQuickDeoptimize, void, void>();
373 }
374
GetDescription() const375 const char* GetDescription() const OVERRIDE { return "DeoptimizationSlowPathX86"; }
376
377 private:
378 DISALLOW_COPY_AND_ASSIGN(DeoptimizationSlowPathX86);
379 };
380
381 class ArraySetSlowPathX86 : public SlowPathCode {
382 public:
ArraySetSlowPathX86(HInstruction * instruction)383 explicit ArraySetSlowPathX86(HInstruction* instruction) : SlowPathCode(instruction) {}
384
EmitNativeCode(CodeGenerator * codegen)385 void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
386 LocationSummary* locations = instruction_->GetLocations();
387 __ Bind(GetEntryLabel());
388 SaveLiveRegisters(codegen, locations);
389
390 InvokeRuntimeCallingConvention calling_convention;
391 HParallelMove parallel_move(codegen->GetGraph()->GetArena());
392 parallel_move.AddMove(
393 locations->InAt(0),
394 Location::RegisterLocation(calling_convention.GetRegisterAt(0)),
395 Primitive::kPrimNot,
396 nullptr);
397 parallel_move.AddMove(
398 locations->InAt(1),
399 Location::RegisterLocation(calling_convention.GetRegisterAt(1)),
400 Primitive::kPrimInt,
401 nullptr);
402 parallel_move.AddMove(
403 locations->InAt(2),
404 Location::RegisterLocation(calling_convention.GetRegisterAt(2)),
405 Primitive::kPrimNot,
406 nullptr);
407 codegen->GetMoveResolver()->EmitNativeCode(¶llel_move);
408
409 CodeGeneratorX86* x86_codegen = down_cast<CodeGeneratorX86*>(codegen);
410 x86_codegen->InvokeRuntime(QUICK_ENTRY_POINT(pAputObject),
411 instruction_,
412 instruction_->GetDexPc(),
413 this);
414 CheckEntrypointTypes<kQuickAputObject, void, mirror::Array*, int32_t, mirror::Object*>();
415 RestoreLiveRegisters(codegen, locations);
416 __ jmp(GetExitLabel());
417 }
418
GetDescription() const419 const char* GetDescription() const OVERRIDE { return "ArraySetSlowPathX86"; }
420
421 private:
422 DISALLOW_COPY_AND_ASSIGN(ArraySetSlowPathX86);
423 };
424
425 // Slow path marking an object during a read barrier.
426 class ReadBarrierMarkSlowPathX86 : public SlowPathCode {
427 public:
ReadBarrierMarkSlowPathX86(HInstruction * instruction,Location out,Location obj)428 ReadBarrierMarkSlowPathX86(HInstruction* instruction, Location out, Location obj)
429 : SlowPathCode(instruction), out_(out), obj_(obj) {
430 DCHECK(kEmitCompilerReadBarrier);
431 }
432
GetDescription() const433 const char* GetDescription() const OVERRIDE { return "ReadBarrierMarkSlowPathX86"; }
434
EmitNativeCode(CodeGenerator * codegen)435 void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
436 LocationSummary* locations = instruction_->GetLocations();
437 Register reg_out = out_.AsRegister<Register>();
438 DCHECK(locations->CanCall());
439 DCHECK(!locations->GetLiveRegisters()->ContainsCoreRegister(reg_out));
440 DCHECK(instruction_->IsInstanceFieldGet() ||
441 instruction_->IsStaticFieldGet() ||
442 instruction_->IsArrayGet() ||
443 instruction_->IsLoadClass() ||
444 instruction_->IsLoadString() ||
445 instruction_->IsInstanceOf() ||
446 instruction_->IsCheckCast())
447 << "Unexpected instruction in read barrier marking slow path: "
448 << instruction_->DebugName();
449
450 __ Bind(GetEntryLabel());
451 SaveLiveRegisters(codegen, locations);
452
453 InvokeRuntimeCallingConvention calling_convention;
454 CodeGeneratorX86* x86_codegen = down_cast<CodeGeneratorX86*>(codegen);
455 x86_codegen->Move32(Location::RegisterLocation(calling_convention.GetRegisterAt(0)), obj_);
456 x86_codegen->InvokeRuntime(QUICK_ENTRY_POINT(pReadBarrierMark),
457 instruction_,
458 instruction_->GetDexPc(),
459 this);
460 CheckEntrypointTypes<kQuickReadBarrierMark, mirror::Object*, mirror::Object*>();
461 x86_codegen->Move32(out_, Location::RegisterLocation(EAX));
462
463 RestoreLiveRegisters(codegen, locations);
464 __ jmp(GetExitLabel());
465 }
466
467 private:
468 const Location out_;
469 const Location obj_;
470
471 DISALLOW_COPY_AND_ASSIGN(ReadBarrierMarkSlowPathX86);
472 };
473
474 // Slow path generating a read barrier for a heap reference.
475 class ReadBarrierForHeapReferenceSlowPathX86 : public SlowPathCode {
476 public:
ReadBarrierForHeapReferenceSlowPathX86(HInstruction * instruction,Location out,Location ref,Location obj,uint32_t offset,Location index)477 ReadBarrierForHeapReferenceSlowPathX86(HInstruction* instruction,
478 Location out,
479 Location ref,
480 Location obj,
481 uint32_t offset,
482 Location index)
483 : SlowPathCode(instruction),
484 out_(out),
485 ref_(ref),
486 obj_(obj),
487 offset_(offset),
488 index_(index) {
489 DCHECK(kEmitCompilerReadBarrier);
490 // If `obj` is equal to `out` or `ref`, it means the initial object
491 // has been overwritten by (or after) the heap object reference load
492 // to be instrumented, e.g.:
493 //
494 // __ movl(out, Address(out, offset));
495 // codegen_->GenerateReadBarrierSlow(instruction, out_loc, out_loc, out_loc, offset);
496 //
497 // In that case, we have lost the information about the original
498 // object, and the emitted read barrier cannot work properly.
499 DCHECK(!obj.Equals(out)) << "obj=" << obj << " out=" << out;
500 DCHECK(!obj.Equals(ref)) << "obj=" << obj << " ref=" << ref;
501 }
502
EmitNativeCode(CodeGenerator * codegen)503 void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
504 CodeGeneratorX86* x86_codegen = down_cast<CodeGeneratorX86*>(codegen);
505 LocationSummary* locations = instruction_->GetLocations();
506 Register reg_out = out_.AsRegister<Register>();
507 DCHECK(locations->CanCall());
508 DCHECK(!locations->GetLiveRegisters()->ContainsCoreRegister(reg_out));
509 DCHECK(!instruction_->IsInvoke() ||
510 (instruction_->IsInvokeStaticOrDirect() &&
511 instruction_->GetLocations()->Intrinsified()))
512 << "Unexpected instruction in read barrier for heap reference slow path: "
513 << instruction_->DebugName();
514
515 __ Bind(GetEntryLabel());
516 SaveLiveRegisters(codegen, locations);
517
518 // We may have to change the index's value, but as `index_` is a
519 // constant member (like other "inputs" of this slow path),
520 // introduce a copy of it, `index`.
521 Location index = index_;
522 if (index_.IsValid()) {
523 // Handle `index_` for HArrayGet and intrinsic UnsafeGetObject.
524 if (instruction_->IsArrayGet()) {
525 // Compute the actual memory offset and store it in `index`.
526 Register index_reg = index_.AsRegister<Register>();
527 DCHECK(locations->GetLiveRegisters()->ContainsCoreRegister(index_reg));
528 if (codegen->IsCoreCalleeSaveRegister(index_reg)) {
529 // We are about to change the value of `index_reg` (see the
530 // calls to art::x86::X86Assembler::shll and
531 // art::x86::X86Assembler::AddImmediate below), but it has
532 // not been saved by the previous call to
533 // art::SlowPathCode::SaveLiveRegisters, as it is a
534 // callee-save register --
535 // art::SlowPathCode::SaveLiveRegisters does not consider
536 // callee-save registers, as it has been designed with the
537 // assumption that callee-save registers are supposed to be
538 // handled by the called function. So, as a callee-save
539 // register, `index_reg` _would_ eventually be saved onto
540 // the stack, but it would be too late: we would have
541 // changed its value earlier. Therefore, we manually save
542 // it here into another freely available register,
543 // `free_reg`, chosen of course among the caller-save
544 // registers (as a callee-save `free_reg` register would
545 // exhibit the same problem).
546 //
547 // Note we could have requested a temporary register from
548 // the register allocator instead; but we prefer not to, as
549 // this is a slow path, and we know we can find a
550 // caller-save register that is available.
551 Register free_reg = FindAvailableCallerSaveRegister(codegen);
552 __ movl(free_reg, index_reg);
553 index_reg = free_reg;
554 index = Location::RegisterLocation(index_reg);
555 } else {
556 // The initial register stored in `index_` has already been
557 // saved in the call to art::SlowPathCode::SaveLiveRegisters
558 // (as it is not a callee-save register), so we can freely
559 // use it.
560 }
561 // Shifting the index value contained in `index_reg` by the scale
562 // factor (2) cannot overflow in practice, as the runtime is
563 // unable to allocate object arrays with a size larger than
564 // 2^26 - 1 (that is, 2^28 - 4 bytes).
565 __ shll(index_reg, Immediate(TIMES_4));
566 static_assert(
567 sizeof(mirror::HeapReference<mirror::Object>) == sizeof(int32_t),
568 "art::mirror::HeapReference<art::mirror::Object> and int32_t have different sizes.");
569 __ AddImmediate(index_reg, Immediate(offset_));
570 } else {
571 DCHECK(instruction_->IsInvoke());
572 DCHECK(instruction_->GetLocations()->Intrinsified());
573 DCHECK((instruction_->AsInvoke()->GetIntrinsic() == Intrinsics::kUnsafeGetObject) ||
574 (instruction_->AsInvoke()->GetIntrinsic() == Intrinsics::kUnsafeGetObjectVolatile))
575 << instruction_->AsInvoke()->GetIntrinsic();
576 DCHECK_EQ(offset_, 0U);
577 DCHECK(index_.IsRegisterPair());
578 // UnsafeGet's offset location is a register pair, the low
579 // part contains the correct offset.
580 index = index_.ToLow();
581 }
582 }
583
584 // We're moving two or three locations to locations that could
585 // overlap, so we need a parallel move resolver.
586 InvokeRuntimeCallingConvention calling_convention;
587 HParallelMove parallel_move(codegen->GetGraph()->GetArena());
588 parallel_move.AddMove(ref_,
589 Location::RegisterLocation(calling_convention.GetRegisterAt(0)),
590 Primitive::kPrimNot,
591 nullptr);
592 parallel_move.AddMove(obj_,
593 Location::RegisterLocation(calling_convention.GetRegisterAt(1)),
594 Primitive::kPrimNot,
595 nullptr);
596 if (index.IsValid()) {
597 parallel_move.AddMove(index,
598 Location::RegisterLocation(calling_convention.GetRegisterAt(2)),
599 Primitive::kPrimInt,
600 nullptr);
601 codegen->GetMoveResolver()->EmitNativeCode(¶llel_move);
602 } else {
603 codegen->GetMoveResolver()->EmitNativeCode(¶llel_move);
604 __ movl(calling_convention.GetRegisterAt(2), Immediate(offset_));
605 }
606 x86_codegen->InvokeRuntime(QUICK_ENTRY_POINT(pReadBarrierSlow),
607 instruction_,
608 instruction_->GetDexPc(),
609 this);
610 CheckEntrypointTypes<
611 kQuickReadBarrierSlow, mirror::Object*, mirror::Object*, mirror::Object*, uint32_t>();
612 x86_codegen->Move32(out_, Location::RegisterLocation(EAX));
613
614 RestoreLiveRegisters(codegen, locations);
615 __ jmp(GetExitLabel());
616 }
617
GetDescription() const618 const char* GetDescription() const OVERRIDE { return "ReadBarrierForHeapReferenceSlowPathX86"; }
619
620 private:
FindAvailableCallerSaveRegister(CodeGenerator * codegen)621 Register FindAvailableCallerSaveRegister(CodeGenerator* codegen) {
622 size_t ref = static_cast<int>(ref_.AsRegister<Register>());
623 size_t obj = static_cast<int>(obj_.AsRegister<Register>());
624 for (size_t i = 0, e = codegen->GetNumberOfCoreRegisters(); i < e; ++i) {
625 if (i != ref && i != obj && !codegen->IsCoreCalleeSaveRegister(i)) {
626 return static_cast<Register>(i);
627 }
628 }
629 // We shall never fail to find a free caller-save register, as
630 // there are more than two core caller-save registers on x86
631 // (meaning it is possible to find one which is different from
632 // `ref` and `obj`).
633 DCHECK_GT(codegen->GetNumberOfCoreCallerSaveRegisters(), 2u);
634 LOG(FATAL) << "Could not find a free caller-save register";
635 UNREACHABLE();
636 }
637
638 const Location out_;
639 const Location ref_;
640 const Location obj_;
641 const uint32_t offset_;
642 // An additional location containing an index to an array.
643 // Only used for HArrayGet and the UnsafeGetObject &
644 // UnsafeGetObjectVolatile intrinsics.
645 const Location index_;
646
647 DISALLOW_COPY_AND_ASSIGN(ReadBarrierForHeapReferenceSlowPathX86);
648 };
649
650 // Slow path generating a read barrier for a GC root.
651 class ReadBarrierForRootSlowPathX86 : public SlowPathCode {
652 public:
ReadBarrierForRootSlowPathX86(HInstruction * instruction,Location out,Location root)653 ReadBarrierForRootSlowPathX86(HInstruction* instruction, Location out, Location root)
654 : SlowPathCode(instruction), out_(out), root_(root) {
655 DCHECK(kEmitCompilerReadBarrier);
656 }
657
EmitNativeCode(CodeGenerator * codegen)658 void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
659 LocationSummary* locations = instruction_->GetLocations();
660 Register reg_out = out_.AsRegister<Register>();
661 DCHECK(locations->CanCall());
662 DCHECK(!locations->GetLiveRegisters()->ContainsCoreRegister(reg_out));
663 DCHECK(instruction_->IsLoadClass() || instruction_->IsLoadString())
664 << "Unexpected instruction in read barrier for GC root slow path: "
665 << instruction_->DebugName();
666
667 __ Bind(GetEntryLabel());
668 SaveLiveRegisters(codegen, locations);
669
670 InvokeRuntimeCallingConvention calling_convention;
671 CodeGeneratorX86* x86_codegen = down_cast<CodeGeneratorX86*>(codegen);
672 x86_codegen->Move32(Location::RegisterLocation(calling_convention.GetRegisterAt(0)), root_);
673 x86_codegen->InvokeRuntime(QUICK_ENTRY_POINT(pReadBarrierForRootSlow),
674 instruction_,
675 instruction_->GetDexPc(),
676 this);
677 CheckEntrypointTypes<kQuickReadBarrierForRootSlow, mirror::Object*, GcRoot<mirror::Object>*>();
678 x86_codegen->Move32(out_, Location::RegisterLocation(EAX));
679
680 RestoreLiveRegisters(codegen, locations);
681 __ jmp(GetExitLabel());
682 }
683
GetDescription() const684 const char* GetDescription() const OVERRIDE { return "ReadBarrierForRootSlowPathX86"; }
685
686 private:
687 const Location out_;
688 const Location root_;
689
690 DISALLOW_COPY_AND_ASSIGN(ReadBarrierForRootSlowPathX86);
691 };
692
693 #undef __
694 #define __ down_cast<X86Assembler*>(GetAssembler())->
695
X86Condition(IfCondition cond)696 inline Condition X86Condition(IfCondition cond) {
697 switch (cond) {
698 case kCondEQ: return kEqual;
699 case kCondNE: return kNotEqual;
700 case kCondLT: return kLess;
701 case kCondLE: return kLessEqual;
702 case kCondGT: return kGreater;
703 case kCondGE: return kGreaterEqual;
704 case kCondB: return kBelow;
705 case kCondBE: return kBelowEqual;
706 case kCondA: return kAbove;
707 case kCondAE: return kAboveEqual;
708 }
709 LOG(FATAL) << "Unreachable";
710 UNREACHABLE();
711 }
712
713 // Maps signed condition to unsigned condition and FP condition to x86 name.
X86UnsignedOrFPCondition(IfCondition cond)714 inline Condition X86UnsignedOrFPCondition(IfCondition cond) {
715 switch (cond) {
716 case kCondEQ: return kEqual;
717 case kCondNE: return kNotEqual;
718 // Signed to unsigned, and FP to x86 name.
719 case kCondLT: return kBelow;
720 case kCondLE: return kBelowEqual;
721 case kCondGT: return kAbove;
722 case kCondGE: return kAboveEqual;
723 // Unsigned remain unchanged.
724 case kCondB: return kBelow;
725 case kCondBE: return kBelowEqual;
726 case kCondA: return kAbove;
727 case kCondAE: return kAboveEqual;
728 }
729 LOG(FATAL) << "Unreachable";
730 UNREACHABLE();
731 }
732
DumpCoreRegister(std::ostream & stream,int reg) const733 void CodeGeneratorX86::DumpCoreRegister(std::ostream& stream, int reg) const {
734 stream << Register(reg);
735 }
736
DumpFloatingPointRegister(std::ostream & stream,int reg) const737 void CodeGeneratorX86::DumpFloatingPointRegister(std::ostream& stream, int reg) const {
738 stream << XmmRegister(reg);
739 }
740
SaveCoreRegister(size_t stack_index,uint32_t reg_id)741 size_t CodeGeneratorX86::SaveCoreRegister(size_t stack_index, uint32_t reg_id) {
742 __ movl(Address(ESP, stack_index), static_cast<Register>(reg_id));
743 return kX86WordSize;
744 }
745
RestoreCoreRegister(size_t stack_index,uint32_t reg_id)746 size_t CodeGeneratorX86::RestoreCoreRegister(size_t stack_index, uint32_t reg_id) {
747 __ movl(static_cast<Register>(reg_id), Address(ESP, stack_index));
748 return kX86WordSize;
749 }
750
SaveFloatingPointRegister(size_t stack_index,uint32_t reg_id)751 size_t CodeGeneratorX86::SaveFloatingPointRegister(size_t stack_index, uint32_t reg_id) {
752 __ movsd(Address(ESP, stack_index), XmmRegister(reg_id));
753 return GetFloatingPointSpillSlotSize();
754 }
755
RestoreFloatingPointRegister(size_t stack_index,uint32_t reg_id)756 size_t CodeGeneratorX86::RestoreFloatingPointRegister(size_t stack_index, uint32_t reg_id) {
757 __ movsd(XmmRegister(reg_id), Address(ESP, stack_index));
758 return GetFloatingPointSpillSlotSize();
759 }
760
InvokeRuntime(QuickEntrypointEnum entrypoint,HInstruction * instruction,uint32_t dex_pc,SlowPathCode * slow_path)761 void CodeGeneratorX86::InvokeRuntime(QuickEntrypointEnum entrypoint,
762 HInstruction* instruction,
763 uint32_t dex_pc,
764 SlowPathCode* slow_path) {
765 InvokeRuntime(GetThreadOffset<kX86WordSize>(entrypoint).Int32Value(),
766 instruction,
767 dex_pc,
768 slow_path);
769 }
770
InvokeRuntime(int32_t entry_point_offset,HInstruction * instruction,uint32_t dex_pc,SlowPathCode * slow_path)771 void CodeGeneratorX86::InvokeRuntime(int32_t entry_point_offset,
772 HInstruction* instruction,
773 uint32_t dex_pc,
774 SlowPathCode* slow_path) {
775 ValidateInvokeRuntime(instruction, slow_path);
776 __ fs()->call(Address::Absolute(entry_point_offset));
777 RecordPcInfo(instruction, dex_pc, slow_path);
778 }
779
CodeGeneratorX86(HGraph * graph,const X86InstructionSetFeatures & isa_features,const CompilerOptions & compiler_options,OptimizingCompilerStats * stats)780 CodeGeneratorX86::CodeGeneratorX86(HGraph* graph,
781 const X86InstructionSetFeatures& isa_features,
782 const CompilerOptions& compiler_options,
783 OptimizingCompilerStats* stats)
784 : CodeGenerator(graph,
785 kNumberOfCpuRegisters,
786 kNumberOfXmmRegisters,
787 kNumberOfRegisterPairs,
788 ComputeRegisterMask(reinterpret_cast<const int*>(kCoreCalleeSaves),
789 arraysize(kCoreCalleeSaves))
790 | (1 << kFakeReturnRegister),
791 0,
792 compiler_options,
793 stats),
794 block_labels_(nullptr),
795 location_builder_(graph, this),
796 instruction_visitor_(graph, this),
797 move_resolver_(graph->GetArena(), this),
798 assembler_(graph->GetArena()),
799 isa_features_(isa_features),
800 method_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)),
801 relative_call_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)),
802 pc_relative_dex_cache_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)),
803 simple_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)),
804 string_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)),
805 constant_area_start_(-1),
806 fixups_to_jump_tables_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)),
807 method_address_offset_(-1) {
808 // Use a fake return address register to mimic Quick.
809 AddAllocatedRegister(Location::RegisterLocation(kFakeReturnRegister));
810 }
811
SetupBlockedRegisters() const812 void CodeGeneratorX86::SetupBlockedRegisters() const {
813 // Don't allocate the dalvik style register pair passing.
814 blocked_register_pairs_[ECX_EDX] = true;
815
816 // Stack register is always reserved.
817 blocked_core_registers_[ESP] = true;
818
819 UpdateBlockedPairRegisters();
820 }
821
UpdateBlockedPairRegisters() const822 void CodeGeneratorX86::UpdateBlockedPairRegisters() const {
823 for (int i = 0; i < kNumberOfRegisterPairs; i++) {
824 X86ManagedRegister current =
825 X86ManagedRegister::FromRegisterPair(static_cast<RegisterPair>(i));
826 if (blocked_core_registers_[current.AsRegisterPairLow()]
827 || blocked_core_registers_[current.AsRegisterPairHigh()]) {
828 blocked_register_pairs_[i] = true;
829 }
830 }
831 }
832
InstructionCodeGeneratorX86(HGraph * graph,CodeGeneratorX86 * codegen)833 InstructionCodeGeneratorX86::InstructionCodeGeneratorX86(HGraph* graph, CodeGeneratorX86* codegen)
834 : InstructionCodeGenerator(graph, codegen),
835 assembler_(codegen->GetAssembler()),
836 codegen_(codegen) {}
837
DWARFReg(Register reg)838 static dwarf::Reg DWARFReg(Register reg) {
839 return dwarf::Reg::X86Core(static_cast<int>(reg));
840 }
841
GenerateFrameEntry()842 void CodeGeneratorX86::GenerateFrameEntry() {
843 __ cfi().SetCurrentCFAOffset(kX86WordSize); // return address
844 __ Bind(&frame_entry_label_);
845 bool skip_overflow_check =
846 IsLeafMethod() && !FrameNeedsStackCheck(GetFrameSize(), InstructionSet::kX86);
847 DCHECK(GetCompilerOptions().GetImplicitStackOverflowChecks());
848
849 if (!skip_overflow_check) {
850 __ testl(EAX, Address(ESP, -static_cast<int32_t>(GetStackOverflowReservedBytes(kX86))));
851 RecordPcInfo(nullptr, 0);
852 }
853
854 if (HasEmptyFrame()) {
855 return;
856 }
857
858 for (int i = arraysize(kCoreCalleeSaves) - 1; i >= 0; --i) {
859 Register reg = kCoreCalleeSaves[i];
860 if (allocated_registers_.ContainsCoreRegister(reg)) {
861 __ pushl(reg);
862 __ cfi().AdjustCFAOffset(kX86WordSize);
863 __ cfi().RelOffset(DWARFReg(reg), 0);
864 }
865 }
866
867 int adjust = GetFrameSize() - FrameEntrySpillSize();
868 __ subl(ESP, Immediate(adjust));
869 __ cfi().AdjustCFAOffset(adjust);
870 __ movl(Address(ESP, kCurrentMethodStackOffset), kMethodRegisterArgument);
871 }
872
GenerateFrameExit()873 void CodeGeneratorX86::GenerateFrameExit() {
874 __ cfi().RememberState();
875 if (!HasEmptyFrame()) {
876 int adjust = GetFrameSize() - FrameEntrySpillSize();
877 __ addl(ESP, Immediate(adjust));
878 __ cfi().AdjustCFAOffset(-adjust);
879
880 for (size_t i = 0; i < arraysize(kCoreCalleeSaves); ++i) {
881 Register reg = kCoreCalleeSaves[i];
882 if (allocated_registers_.ContainsCoreRegister(reg)) {
883 __ popl(reg);
884 __ cfi().AdjustCFAOffset(-static_cast<int>(kX86WordSize));
885 __ cfi().Restore(DWARFReg(reg));
886 }
887 }
888 }
889 __ ret();
890 __ cfi().RestoreState();
891 __ cfi().DefCFAOffset(GetFrameSize());
892 }
893
Bind(HBasicBlock * block)894 void CodeGeneratorX86::Bind(HBasicBlock* block) {
895 __ Bind(GetLabelOf(block));
896 }
897
GetReturnLocation(Primitive::Type type) const898 Location InvokeDexCallingConventionVisitorX86::GetReturnLocation(Primitive::Type type) const {
899 switch (type) {
900 case Primitive::kPrimBoolean:
901 case Primitive::kPrimByte:
902 case Primitive::kPrimChar:
903 case Primitive::kPrimShort:
904 case Primitive::kPrimInt:
905 case Primitive::kPrimNot:
906 return Location::RegisterLocation(EAX);
907
908 case Primitive::kPrimLong:
909 return Location::RegisterPairLocation(EAX, EDX);
910
911 case Primitive::kPrimVoid:
912 return Location::NoLocation();
913
914 case Primitive::kPrimDouble:
915 case Primitive::kPrimFloat:
916 return Location::FpuRegisterLocation(XMM0);
917 }
918
919 UNREACHABLE();
920 }
921
GetMethodLocation() const922 Location InvokeDexCallingConventionVisitorX86::GetMethodLocation() const {
923 return Location::RegisterLocation(kMethodRegisterArgument);
924 }
925
GetNextLocation(Primitive::Type type)926 Location InvokeDexCallingConventionVisitorX86::GetNextLocation(Primitive::Type type) {
927 switch (type) {
928 case Primitive::kPrimBoolean:
929 case Primitive::kPrimByte:
930 case Primitive::kPrimChar:
931 case Primitive::kPrimShort:
932 case Primitive::kPrimInt:
933 case Primitive::kPrimNot: {
934 uint32_t index = gp_index_++;
935 stack_index_++;
936 if (index < calling_convention.GetNumberOfRegisters()) {
937 return Location::RegisterLocation(calling_convention.GetRegisterAt(index));
938 } else {
939 return Location::StackSlot(calling_convention.GetStackOffsetOf(stack_index_ - 1));
940 }
941 }
942
943 case Primitive::kPrimLong: {
944 uint32_t index = gp_index_;
945 gp_index_ += 2;
946 stack_index_ += 2;
947 if (index + 1 < calling_convention.GetNumberOfRegisters()) {
948 X86ManagedRegister pair = X86ManagedRegister::FromRegisterPair(
949 calling_convention.GetRegisterPairAt(index));
950 return Location::RegisterPairLocation(pair.AsRegisterPairLow(), pair.AsRegisterPairHigh());
951 } else {
952 return Location::DoubleStackSlot(calling_convention.GetStackOffsetOf(stack_index_ - 2));
953 }
954 }
955
956 case Primitive::kPrimFloat: {
957 uint32_t index = float_index_++;
958 stack_index_++;
959 if (index < calling_convention.GetNumberOfFpuRegisters()) {
960 return Location::FpuRegisterLocation(calling_convention.GetFpuRegisterAt(index));
961 } else {
962 return Location::StackSlot(calling_convention.GetStackOffsetOf(stack_index_ - 1));
963 }
964 }
965
966 case Primitive::kPrimDouble: {
967 uint32_t index = float_index_++;
968 stack_index_ += 2;
969 if (index < calling_convention.GetNumberOfFpuRegisters()) {
970 return Location::FpuRegisterLocation(calling_convention.GetFpuRegisterAt(index));
971 } else {
972 return Location::DoubleStackSlot(calling_convention.GetStackOffsetOf(stack_index_ - 2));
973 }
974 }
975
976 case Primitive::kPrimVoid:
977 LOG(FATAL) << "Unexpected parameter type " << type;
978 break;
979 }
980 return Location::NoLocation();
981 }
982
Move32(Location destination,Location source)983 void CodeGeneratorX86::Move32(Location destination, Location source) {
984 if (source.Equals(destination)) {
985 return;
986 }
987 if (destination.IsRegister()) {
988 if (source.IsRegister()) {
989 __ movl(destination.AsRegister<Register>(), source.AsRegister<Register>());
990 } else if (source.IsFpuRegister()) {
991 __ movd(destination.AsRegister<Register>(), source.AsFpuRegister<XmmRegister>());
992 } else {
993 DCHECK(source.IsStackSlot());
994 __ movl(destination.AsRegister<Register>(), Address(ESP, source.GetStackIndex()));
995 }
996 } else if (destination.IsFpuRegister()) {
997 if (source.IsRegister()) {
998 __ movd(destination.AsFpuRegister<XmmRegister>(), source.AsRegister<Register>());
999 } else if (source.IsFpuRegister()) {
1000 __ movaps(destination.AsFpuRegister<XmmRegister>(), source.AsFpuRegister<XmmRegister>());
1001 } else {
1002 DCHECK(source.IsStackSlot());
1003 __ movss(destination.AsFpuRegister<XmmRegister>(), Address(ESP, source.GetStackIndex()));
1004 }
1005 } else {
1006 DCHECK(destination.IsStackSlot()) << destination;
1007 if (source.IsRegister()) {
1008 __ movl(Address(ESP, destination.GetStackIndex()), source.AsRegister<Register>());
1009 } else if (source.IsFpuRegister()) {
1010 __ movss(Address(ESP, destination.GetStackIndex()), source.AsFpuRegister<XmmRegister>());
1011 } else if (source.IsConstant()) {
1012 HConstant* constant = source.GetConstant();
1013 int32_t value = GetInt32ValueOf(constant);
1014 __ movl(Address(ESP, destination.GetStackIndex()), Immediate(value));
1015 } else {
1016 DCHECK(source.IsStackSlot());
1017 __ pushl(Address(ESP, source.GetStackIndex()));
1018 __ popl(Address(ESP, destination.GetStackIndex()));
1019 }
1020 }
1021 }
1022
Move64(Location destination,Location source)1023 void CodeGeneratorX86::Move64(Location destination, Location source) {
1024 if (source.Equals(destination)) {
1025 return;
1026 }
1027 if (destination.IsRegisterPair()) {
1028 if (source.IsRegisterPair()) {
1029 EmitParallelMoves(
1030 Location::RegisterLocation(source.AsRegisterPairHigh<Register>()),
1031 Location::RegisterLocation(destination.AsRegisterPairHigh<Register>()),
1032 Primitive::kPrimInt,
1033 Location::RegisterLocation(source.AsRegisterPairLow<Register>()),
1034 Location::RegisterLocation(destination.AsRegisterPairLow<Register>()),
1035 Primitive::kPrimInt);
1036 } else if (source.IsFpuRegister()) {
1037 XmmRegister src_reg = source.AsFpuRegister<XmmRegister>();
1038 __ movd(destination.AsRegisterPairLow<Register>(), src_reg);
1039 __ psrlq(src_reg, Immediate(32));
1040 __ movd(destination.AsRegisterPairHigh<Register>(), src_reg);
1041 } else {
1042 // No conflict possible, so just do the moves.
1043 DCHECK(source.IsDoubleStackSlot());
1044 __ movl(destination.AsRegisterPairLow<Register>(), Address(ESP, source.GetStackIndex()));
1045 __ movl(destination.AsRegisterPairHigh<Register>(),
1046 Address(ESP, source.GetHighStackIndex(kX86WordSize)));
1047 }
1048 } else if (destination.IsFpuRegister()) {
1049 if (source.IsFpuRegister()) {
1050 __ movaps(destination.AsFpuRegister<XmmRegister>(), source.AsFpuRegister<XmmRegister>());
1051 } else if (source.IsDoubleStackSlot()) {
1052 __ movsd(destination.AsFpuRegister<XmmRegister>(), Address(ESP, source.GetStackIndex()));
1053 } else if (source.IsRegisterPair()) {
1054 size_t elem_size = Primitive::ComponentSize(Primitive::kPrimInt);
1055 // Create stack space for 2 elements.
1056 __ subl(ESP, Immediate(2 * elem_size));
1057 __ movl(Address(ESP, 0), source.AsRegisterPairLow<Register>());
1058 __ movl(Address(ESP, elem_size), source.AsRegisterPairHigh<Register>());
1059 __ movsd(destination.AsFpuRegister<XmmRegister>(), Address(ESP, 0));
1060 // And remove the temporary stack space we allocated.
1061 __ addl(ESP, Immediate(2 * elem_size));
1062 } else {
1063 LOG(FATAL) << "Unimplemented";
1064 }
1065 } else {
1066 DCHECK(destination.IsDoubleStackSlot()) << destination;
1067 if (source.IsRegisterPair()) {
1068 // No conflict possible, so just do the moves.
1069 __ movl(Address(ESP, destination.GetStackIndex()), source.AsRegisterPairLow<Register>());
1070 __ movl(Address(ESP, destination.GetHighStackIndex(kX86WordSize)),
1071 source.AsRegisterPairHigh<Register>());
1072 } else if (source.IsFpuRegister()) {
1073 __ movsd(Address(ESP, destination.GetStackIndex()), source.AsFpuRegister<XmmRegister>());
1074 } else if (source.IsConstant()) {
1075 HConstant* constant = source.GetConstant();
1076 int64_t value;
1077 if (constant->IsLongConstant()) {
1078 value = constant->AsLongConstant()->GetValue();
1079 } else {
1080 DCHECK(constant->IsDoubleConstant());
1081 value = bit_cast<int64_t, double>(constant->AsDoubleConstant()->GetValue());
1082 }
1083 __ movl(Address(ESP, destination.GetStackIndex()), Immediate(Low32Bits(value)));
1084 __ movl(Address(ESP, destination.GetHighStackIndex(kX86WordSize)), Immediate(High32Bits(value)));
1085 } else {
1086 DCHECK(source.IsDoubleStackSlot()) << source;
1087 EmitParallelMoves(
1088 Location::StackSlot(source.GetStackIndex()),
1089 Location::StackSlot(destination.GetStackIndex()),
1090 Primitive::kPrimInt,
1091 Location::StackSlot(source.GetHighStackIndex(kX86WordSize)),
1092 Location::StackSlot(destination.GetHighStackIndex(kX86WordSize)),
1093 Primitive::kPrimInt);
1094 }
1095 }
1096 }
1097
MoveConstant(Location location,int32_t value)1098 void CodeGeneratorX86::MoveConstant(Location location, int32_t value) {
1099 DCHECK(location.IsRegister());
1100 __ movl(location.AsRegister<Register>(), Immediate(value));
1101 }
1102
MoveLocation(Location dst,Location src,Primitive::Type dst_type)1103 void CodeGeneratorX86::MoveLocation(Location dst, Location src, Primitive::Type dst_type) {
1104 HParallelMove move(GetGraph()->GetArena());
1105 if (dst_type == Primitive::kPrimLong && !src.IsConstant() && !src.IsFpuRegister()) {
1106 move.AddMove(src.ToLow(), dst.ToLow(), Primitive::kPrimInt, nullptr);
1107 move.AddMove(src.ToHigh(), dst.ToHigh(), Primitive::kPrimInt, nullptr);
1108 } else {
1109 move.AddMove(src, dst, dst_type, nullptr);
1110 }
1111 GetMoveResolver()->EmitNativeCode(&move);
1112 }
1113
AddLocationAsTemp(Location location,LocationSummary * locations)1114 void CodeGeneratorX86::AddLocationAsTemp(Location location, LocationSummary* locations) {
1115 if (location.IsRegister()) {
1116 locations->AddTemp(location);
1117 } else if (location.IsRegisterPair()) {
1118 locations->AddTemp(Location::RegisterLocation(location.AsRegisterPairLow<Register>()));
1119 locations->AddTemp(Location::RegisterLocation(location.AsRegisterPairHigh<Register>()));
1120 } else {
1121 UNIMPLEMENTED(FATAL) << "AddLocationAsTemp not implemented for location " << location;
1122 }
1123 }
1124
HandleGoto(HInstruction * got,HBasicBlock * successor)1125 void InstructionCodeGeneratorX86::HandleGoto(HInstruction* got, HBasicBlock* successor) {
1126 DCHECK(!successor->IsExitBlock());
1127
1128 HBasicBlock* block = got->GetBlock();
1129 HInstruction* previous = got->GetPrevious();
1130
1131 HLoopInformation* info = block->GetLoopInformation();
1132 if (info != nullptr && info->IsBackEdge(*block) && info->HasSuspendCheck()) {
1133 GenerateSuspendCheck(info->GetSuspendCheck(), successor);
1134 return;
1135 }
1136
1137 if (block->IsEntryBlock() && (previous != nullptr) && previous->IsSuspendCheck()) {
1138 GenerateSuspendCheck(previous->AsSuspendCheck(), nullptr);
1139 }
1140 if (!codegen_->GoesToNextBlock(got->GetBlock(), successor)) {
1141 __ jmp(codegen_->GetLabelOf(successor));
1142 }
1143 }
1144
VisitGoto(HGoto * got)1145 void LocationsBuilderX86::VisitGoto(HGoto* got) {
1146 got->SetLocations(nullptr);
1147 }
1148
VisitGoto(HGoto * got)1149 void InstructionCodeGeneratorX86::VisitGoto(HGoto* got) {
1150 HandleGoto(got, got->GetSuccessor());
1151 }
1152
VisitTryBoundary(HTryBoundary * try_boundary)1153 void LocationsBuilderX86::VisitTryBoundary(HTryBoundary* try_boundary) {
1154 try_boundary->SetLocations(nullptr);
1155 }
1156
VisitTryBoundary(HTryBoundary * try_boundary)1157 void InstructionCodeGeneratorX86::VisitTryBoundary(HTryBoundary* try_boundary) {
1158 HBasicBlock* successor = try_boundary->GetNormalFlowSuccessor();
1159 if (!successor->IsExitBlock()) {
1160 HandleGoto(try_boundary, successor);
1161 }
1162 }
1163
VisitExit(HExit * exit)1164 void LocationsBuilderX86::VisitExit(HExit* exit) {
1165 exit->SetLocations(nullptr);
1166 }
1167
VisitExit(HExit * exit ATTRIBUTE_UNUSED)1168 void InstructionCodeGeneratorX86::VisitExit(HExit* exit ATTRIBUTE_UNUSED) {
1169 }
1170
1171 template<class LabelType>
GenerateFPJumps(HCondition * cond,LabelType * true_label,LabelType * false_label)1172 void InstructionCodeGeneratorX86::GenerateFPJumps(HCondition* cond,
1173 LabelType* true_label,
1174 LabelType* false_label) {
1175 if (cond->IsFPConditionTrueIfNaN()) {
1176 __ j(kUnordered, true_label);
1177 } else if (cond->IsFPConditionFalseIfNaN()) {
1178 __ j(kUnordered, false_label);
1179 }
1180 __ j(X86UnsignedOrFPCondition(cond->GetCondition()), true_label);
1181 }
1182
1183 template<class LabelType>
GenerateLongComparesAndJumps(HCondition * cond,LabelType * true_label,LabelType * false_label)1184 void InstructionCodeGeneratorX86::GenerateLongComparesAndJumps(HCondition* cond,
1185 LabelType* true_label,
1186 LabelType* false_label) {
1187 LocationSummary* locations = cond->GetLocations();
1188 Location left = locations->InAt(0);
1189 Location right = locations->InAt(1);
1190 IfCondition if_cond = cond->GetCondition();
1191
1192 Register left_high = left.AsRegisterPairHigh<Register>();
1193 Register left_low = left.AsRegisterPairLow<Register>();
1194 IfCondition true_high_cond = if_cond;
1195 IfCondition false_high_cond = cond->GetOppositeCondition();
1196 Condition final_condition = X86UnsignedOrFPCondition(if_cond); // unsigned on lower part
1197
1198 // Set the conditions for the test, remembering that == needs to be
1199 // decided using the low words.
1200 switch (if_cond) {
1201 case kCondEQ:
1202 case kCondNE:
1203 // Nothing to do.
1204 break;
1205 case kCondLT:
1206 false_high_cond = kCondGT;
1207 break;
1208 case kCondLE:
1209 true_high_cond = kCondLT;
1210 break;
1211 case kCondGT:
1212 false_high_cond = kCondLT;
1213 break;
1214 case kCondGE:
1215 true_high_cond = kCondGT;
1216 break;
1217 case kCondB:
1218 false_high_cond = kCondA;
1219 break;
1220 case kCondBE:
1221 true_high_cond = kCondB;
1222 break;
1223 case kCondA:
1224 false_high_cond = kCondB;
1225 break;
1226 case kCondAE:
1227 true_high_cond = kCondA;
1228 break;
1229 }
1230
1231 if (right.IsConstant()) {
1232 int64_t value = right.GetConstant()->AsLongConstant()->GetValue();
1233 int32_t val_high = High32Bits(value);
1234 int32_t val_low = Low32Bits(value);
1235
1236 codegen_->Compare32BitValue(left_high, val_high);
1237 if (if_cond == kCondNE) {
1238 __ j(X86Condition(true_high_cond), true_label);
1239 } else if (if_cond == kCondEQ) {
1240 __ j(X86Condition(false_high_cond), false_label);
1241 } else {
1242 __ j(X86Condition(true_high_cond), true_label);
1243 __ j(X86Condition(false_high_cond), false_label);
1244 }
1245 // Must be equal high, so compare the lows.
1246 codegen_->Compare32BitValue(left_low, val_low);
1247 } else if (right.IsRegisterPair()) {
1248 Register right_high = right.AsRegisterPairHigh<Register>();
1249 Register right_low = right.AsRegisterPairLow<Register>();
1250
1251 __ cmpl(left_high, right_high);
1252 if (if_cond == kCondNE) {
1253 __ j(X86Condition(true_high_cond), true_label);
1254 } else if (if_cond == kCondEQ) {
1255 __ j(X86Condition(false_high_cond), false_label);
1256 } else {
1257 __ j(X86Condition(true_high_cond), true_label);
1258 __ j(X86Condition(false_high_cond), false_label);
1259 }
1260 // Must be equal high, so compare the lows.
1261 __ cmpl(left_low, right_low);
1262 } else {
1263 DCHECK(right.IsDoubleStackSlot());
1264 __ cmpl(left_high, Address(ESP, right.GetHighStackIndex(kX86WordSize)));
1265 if (if_cond == kCondNE) {
1266 __ j(X86Condition(true_high_cond), true_label);
1267 } else if (if_cond == kCondEQ) {
1268 __ j(X86Condition(false_high_cond), false_label);
1269 } else {
1270 __ j(X86Condition(true_high_cond), true_label);
1271 __ j(X86Condition(false_high_cond), false_label);
1272 }
1273 // Must be equal high, so compare the lows.
1274 __ cmpl(left_low, Address(ESP, right.GetStackIndex()));
1275 }
1276 // The last comparison might be unsigned.
1277 __ j(final_condition, true_label);
1278 }
1279
GenerateFPCompare(Location lhs,Location rhs,HInstruction * insn,bool is_double)1280 void InstructionCodeGeneratorX86::GenerateFPCompare(Location lhs,
1281 Location rhs,
1282 HInstruction* insn,
1283 bool is_double) {
1284 HX86LoadFromConstantTable* const_area = insn->InputAt(1)->AsX86LoadFromConstantTable();
1285 if (is_double) {
1286 if (rhs.IsFpuRegister()) {
1287 __ ucomisd(lhs.AsFpuRegister<XmmRegister>(), rhs.AsFpuRegister<XmmRegister>());
1288 } else if (const_area != nullptr) {
1289 DCHECK(const_area->IsEmittedAtUseSite());
1290 __ ucomisd(lhs.AsFpuRegister<XmmRegister>(),
1291 codegen_->LiteralDoubleAddress(
1292 const_area->GetConstant()->AsDoubleConstant()->GetValue(),
1293 const_area->GetLocations()->InAt(0).AsRegister<Register>()));
1294 } else {
1295 DCHECK(rhs.IsDoubleStackSlot());
1296 __ ucomisd(lhs.AsFpuRegister<XmmRegister>(), Address(ESP, rhs.GetStackIndex()));
1297 }
1298 } else {
1299 if (rhs.IsFpuRegister()) {
1300 __ ucomiss(lhs.AsFpuRegister<XmmRegister>(), rhs.AsFpuRegister<XmmRegister>());
1301 } else if (const_area != nullptr) {
1302 DCHECK(const_area->IsEmittedAtUseSite());
1303 __ ucomiss(lhs.AsFpuRegister<XmmRegister>(),
1304 codegen_->LiteralFloatAddress(
1305 const_area->GetConstant()->AsFloatConstant()->GetValue(),
1306 const_area->GetLocations()->InAt(0).AsRegister<Register>()));
1307 } else {
1308 DCHECK(rhs.IsStackSlot());
1309 __ ucomiss(lhs.AsFpuRegister<XmmRegister>(), Address(ESP, rhs.GetStackIndex()));
1310 }
1311 }
1312 }
1313
1314 template<class LabelType>
GenerateCompareTestAndBranch(HCondition * condition,LabelType * true_target_in,LabelType * false_target_in)1315 void InstructionCodeGeneratorX86::GenerateCompareTestAndBranch(HCondition* condition,
1316 LabelType* true_target_in,
1317 LabelType* false_target_in) {
1318 // Generated branching requires both targets to be explicit. If either of the
1319 // targets is nullptr (fallthrough) use and bind `fallthrough_target` instead.
1320 LabelType fallthrough_target;
1321 LabelType* true_target = true_target_in == nullptr ? &fallthrough_target : true_target_in;
1322 LabelType* false_target = false_target_in == nullptr ? &fallthrough_target : false_target_in;
1323
1324 LocationSummary* locations = condition->GetLocations();
1325 Location left = locations->InAt(0);
1326 Location right = locations->InAt(1);
1327
1328 Primitive::Type type = condition->InputAt(0)->GetType();
1329 switch (type) {
1330 case Primitive::kPrimLong:
1331 GenerateLongComparesAndJumps(condition, true_target, false_target);
1332 break;
1333 case Primitive::kPrimFloat:
1334 GenerateFPCompare(left, right, condition, false);
1335 GenerateFPJumps(condition, true_target, false_target);
1336 break;
1337 case Primitive::kPrimDouble:
1338 GenerateFPCompare(left, right, condition, true);
1339 GenerateFPJumps(condition, true_target, false_target);
1340 break;
1341 default:
1342 LOG(FATAL) << "Unexpected compare type " << type;
1343 }
1344
1345 if (false_target != &fallthrough_target) {
1346 __ jmp(false_target);
1347 }
1348
1349 if (fallthrough_target.IsLinked()) {
1350 __ Bind(&fallthrough_target);
1351 }
1352 }
1353
AreEflagsSetFrom(HInstruction * cond,HInstruction * branch)1354 static bool AreEflagsSetFrom(HInstruction* cond, HInstruction* branch) {
1355 // Moves may affect the eflags register (move zero uses xorl), so the EFLAGS
1356 // are set only strictly before `branch`. We can't use the eflags on long/FP
1357 // conditions if they are materialized due to the complex branching.
1358 return cond->IsCondition() &&
1359 cond->GetNext() == branch &&
1360 cond->InputAt(0)->GetType() != Primitive::kPrimLong &&
1361 !Primitive::IsFloatingPointType(cond->InputAt(0)->GetType());
1362 }
1363
1364 template<class LabelType>
GenerateTestAndBranch(HInstruction * instruction,size_t condition_input_index,LabelType * true_target,LabelType * false_target)1365 void InstructionCodeGeneratorX86::GenerateTestAndBranch(HInstruction* instruction,
1366 size_t condition_input_index,
1367 LabelType* true_target,
1368 LabelType* false_target) {
1369 HInstruction* cond = instruction->InputAt(condition_input_index);
1370
1371 if (true_target == nullptr && false_target == nullptr) {
1372 // Nothing to do. The code always falls through.
1373 return;
1374 } else if (cond->IsIntConstant()) {
1375 // Constant condition, statically compared against "true" (integer value 1).
1376 if (cond->AsIntConstant()->IsTrue()) {
1377 if (true_target != nullptr) {
1378 __ jmp(true_target);
1379 }
1380 } else {
1381 DCHECK(cond->AsIntConstant()->IsFalse()) << cond->AsIntConstant()->GetValue();
1382 if (false_target != nullptr) {
1383 __ jmp(false_target);
1384 }
1385 }
1386 return;
1387 }
1388
1389 // The following code generates these patterns:
1390 // (1) true_target == nullptr && false_target != nullptr
1391 // - opposite condition true => branch to false_target
1392 // (2) true_target != nullptr && false_target == nullptr
1393 // - condition true => branch to true_target
1394 // (3) true_target != nullptr && false_target != nullptr
1395 // - condition true => branch to true_target
1396 // - branch to false_target
1397 if (IsBooleanValueOrMaterializedCondition(cond)) {
1398 if (AreEflagsSetFrom(cond, instruction)) {
1399 if (true_target == nullptr) {
1400 __ j(X86Condition(cond->AsCondition()->GetOppositeCondition()), false_target);
1401 } else {
1402 __ j(X86Condition(cond->AsCondition()->GetCondition()), true_target);
1403 }
1404 } else {
1405 // Materialized condition, compare against 0.
1406 Location lhs = instruction->GetLocations()->InAt(condition_input_index);
1407 if (lhs.IsRegister()) {
1408 __ testl(lhs.AsRegister<Register>(), lhs.AsRegister<Register>());
1409 } else {
1410 __ cmpl(Address(ESP, lhs.GetStackIndex()), Immediate(0));
1411 }
1412 if (true_target == nullptr) {
1413 __ j(kEqual, false_target);
1414 } else {
1415 __ j(kNotEqual, true_target);
1416 }
1417 }
1418 } else {
1419 // Condition has not been materialized, use its inputs as the comparison and
1420 // its condition as the branch condition.
1421 HCondition* condition = cond->AsCondition();
1422
1423 // If this is a long or FP comparison that has been folded into
1424 // the HCondition, generate the comparison directly.
1425 Primitive::Type type = condition->InputAt(0)->GetType();
1426 if (type == Primitive::kPrimLong || Primitive::IsFloatingPointType(type)) {
1427 GenerateCompareTestAndBranch(condition, true_target, false_target);
1428 return;
1429 }
1430
1431 Location lhs = condition->GetLocations()->InAt(0);
1432 Location rhs = condition->GetLocations()->InAt(1);
1433 // LHS is guaranteed to be in a register (see LocationsBuilderX86::HandleCondition).
1434 if (rhs.IsRegister()) {
1435 __ cmpl(lhs.AsRegister<Register>(), rhs.AsRegister<Register>());
1436 } else if (rhs.IsConstant()) {
1437 int32_t constant = CodeGenerator::GetInt32ValueOf(rhs.GetConstant());
1438 codegen_->Compare32BitValue(lhs.AsRegister<Register>(), constant);
1439 } else {
1440 __ cmpl(lhs.AsRegister<Register>(), Address(ESP, rhs.GetStackIndex()));
1441 }
1442 if (true_target == nullptr) {
1443 __ j(X86Condition(condition->GetOppositeCondition()), false_target);
1444 } else {
1445 __ j(X86Condition(condition->GetCondition()), true_target);
1446 }
1447 }
1448
1449 // If neither branch falls through (case 3), the conditional branch to `true_target`
1450 // was already emitted (case 2) and we need to emit a jump to `false_target`.
1451 if (true_target != nullptr && false_target != nullptr) {
1452 __ jmp(false_target);
1453 }
1454 }
1455
VisitIf(HIf * if_instr)1456 void LocationsBuilderX86::VisitIf(HIf* if_instr) {
1457 LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(if_instr);
1458 if (IsBooleanValueOrMaterializedCondition(if_instr->InputAt(0))) {
1459 locations->SetInAt(0, Location::Any());
1460 }
1461 }
1462
VisitIf(HIf * if_instr)1463 void InstructionCodeGeneratorX86::VisitIf(HIf* if_instr) {
1464 HBasicBlock* true_successor = if_instr->IfTrueSuccessor();
1465 HBasicBlock* false_successor = if_instr->IfFalseSuccessor();
1466 Label* true_target = codegen_->GoesToNextBlock(if_instr->GetBlock(), true_successor) ?
1467 nullptr : codegen_->GetLabelOf(true_successor);
1468 Label* false_target = codegen_->GoesToNextBlock(if_instr->GetBlock(), false_successor) ?
1469 nullptr : codegen_->GetLabelOf(false_successor);
1470 GenerateTestAndBranch(if_instr, /* condition_input_index */ 0, true_target, false_target);
1471 }
1472
VisitDeoptimize(HDeoptimize * deoptimize)1473 void LocationsBuilderX86::VisitDeoptimize(HDeoptimize* deoptimize) {
1474 LocationSummary* locations = new (GetGraph()->GetArena())
1475 LocationSummary(deoptimize, LocationSummary::kCallOnSlowPath);
1476 if (IsBooleanValueOrMaterializedCondition(deoptimize->InputAt(0))) {
1477 locations->SetInAt(0, Location::Any());
1478 }
1479 }
1480
VisitDeoptimize(HDeoptimize * deoptimize)1481 void InstructionCodeGeneratorX86::VisitDeoptimize(HDeoptimize* deoptimize) {
1482 SlowPathCode* slow_path = deopt_slow_paths_.NewSlowPath<DeoptimizationSlowPathX86>(deoptimize);
1483 GenerateTestAndBranch<Label>(deoptimize,
1484 /* condition_input_index */ 0,
1485 slow_path->GetEntryLabel(),
1486 /* false_target */ nullptr);
1487 }
1488
SelectCanUseCMOV(HSelect * select)1489 static bool SelectCanUseCMOV(HSelect* select) {
1490 // There are no conditional move instructions for XMMs.
1491 if (Primitive::IsFloatingPointType(select->GetType())) {
1492 return false;
1493 }
1494
1495 // A FP condition doesn't generate the single CC that we need.
1496 // In 32 bit mode, a long condition doesn't generate a single CC either.
1497 HInstruction* condition = select->GetCondition();
1498 if (condition->IsCondition()) {
1499 Primitive::Type compare_type = condition->InputAt(0)->GetType();
1500 if (compare_type == Primitive::kPrimLong ||
1501 Primitive::IsFloatingPointType(compare_type)) {
1502 return false;
1503 }
1504 }
1505
1506 // We can generate a CMOV for this Select.
1507 return true;
1508 }
1509
VisitSelect(HSelect * select)1510 void LocationsBuilderX86::VisitSelect(HSelect* select) {
1511 LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(select);
1512 if (Primitive::IsFloatingPointType(select->GetType())) {
1513 locations->SetInAt(0, Location::RequiresFpuRegister());
1514 locations->SetInAt(1, Location::Any());
1515 } else {
1516 locations->SetInAt(0, Location::RequiresRegister());
1517 if (SelectCanUseCMOV(select)) {
1518 if (select->InputAt(1)->IsConstant()) {
1519 // Cmov can't handle a constant value.
1520 locations->SetInAt(1, Location::RequiresRegister());
1521 } else {
1522 locations->SetInAt(1, Location::Any());
1523 }
1524 } else {
1525 locations->SetInAt(1, Location::Any());
1526 }
1527 }
1528 if (IsBooleanValueOrMaterializedCondition(select->GetCondition())) {
1529 locations->SetInAt(2, Location::RequiresRegister());
1530 }
1531 locations->SetOut(Location::SameAsFirstInput());
1532 }
1533
GenerateIntCompare(Location lhs,Location rhs)1534 void InstructionCodeGeneratorX86::GenerateIntCompare(Location lhs, Location rhs) {
1535 Register lhs_reg = lhs.AsRegister<Register>();
1536 if (rhs.IsConstant()) {
1537 int32_t value = CodeGenerator::GetInt32ValueOf(rhs.GetConstant());
1538 codegen_->Compare32BitValue(lhs_reg, value);
1539 } else if (rhs.IsStackSlot()) {
1540 __ cmpl(lhs_reg, Address(ESP, rhs.GetStackIndex()));
1541 } else {
1542 __ cmpl(lhs_reg, rhs.AsRegister<Register>());
1543 }
1544 }
1545
VisitSelect(HSelect * select)1546 void InstructionCodeGeneratorX86::VisitSelect(HSelect* select) {
1547 LocationSummary* locations = select->GetLocations();
1548 DCHECK(locations->InAt(0).Equals(locations->Out()));
1549 if (SelectCanUseCMOV(select)) {
1550 // If both the condition and the source types are integer, we can generate
1551 // a CMOV to implement Select.
1552
1553 HInstruction* select_condition = select->GetCondition();
1554 Condition cond = kNotEqual;
1555
1556 // Figure out how to test the 'condition'.
1557 if (select_condition->IsCondition()) {
1558 HCondition* condition = select_condition->AsCondition();
1559 if (!condition->IsEmittedAtUseSite()) {
1560 // This was a previously materialized condition.
1561 // Can we use the existing condition code?
1562 if (AreEflagsSetFrom(condition, select)) {
1563 // Materialization was the previous instruction. Condition codes are right.
1564 cond = X86Condition(condition->GetCondition());
1565 } else {
1566 // No, we have to recreate the condition code.
1567 Register cond_reg = locations->InAt(2).AsRegister<Register>();
1568 __ testl(cond_reg, cond_reg);
1569 }
1570 } else {
1571 // We can't handle FP or long here.
1572 DCHECK_NE(condition->InputAt(0)->GetType(), Primitive::kPrimLong);
1573 DCHECK(!Primitive::IsFloatingPointType(condition->InputAt(0)->GetType()));
1574 LocationSummary* cond_locations = condition->GetLocations();
1575 GenerateIntCompare(cond_locations->InAt(0), cond_locations->InAt(1));
1576 cond = X86Condition(condition->GetCondition());
1577 }
1578 } else {
1579 // Must be a boolean condition, which needs to be compared to 0.
1580 Register cond_reg = locations->InAt(2).AsRegister<Register>();
1581 __ testl(cond_reg, cond_reg);
1582 }
1583
1584 // If the condition is true, overwrite the output, which already contains false.
1585 Location false_loc = locations->InAt(0);
1586 Location true_loc = locations->InAt(1);
1587 if (select->GetType() == Primitive::kPrimLong) {
1588 // 64 bit conditional move.
1589 Register false_high = false_loc.AsRegisterPairHigh<Register>();
1590 Register false_low = false_loc.AsRegisterPairLow<Register>();
1591 if (true_loc.IsRegisterPair()) {
1592 __ cmovl(cond, false_high, true_loc.AsRegisterPairHigh<Register>());
1593 __ cmovl(cond, false_low, true_loc.AsRegisterPairLow<Register>());
1594 } else {
1595 __ cmovl(cond, false_high, Address(ESP, true_loc.GetHighStackIndex(kX86WordSize)));
1596 __ cmovl(cond, false_low, Address(ESP, true_loc.GetStackIndex()));
1597 }
1598 } else {
1599 // 32 bit conditional move.
1600 Register false_reg = false_loc.AsRegister<Register>();
1601 if (true_loc.IsRegister()) {
1602 __ cmovl(cond, false_reg, true_loc.AsRegister<Register>());
1603 } else {
1604 __ cmovl(cond, false_reg, Address(ESP, true_loc.GetStackIndex()));
1605 }
1606 }
1607 } else {
1608 NearLabel false_target;
1609 GenerateTestAndBranch<NearLabel>(
1610 select, /* condition_input_index */ 2, /* true_target */ nullptr, &false_target);
1611 codegen_->MoveLocation(locations->Out(), locations->InAt(1), select->GetType());
1612 __ Bind(&false_target);
1613 }
1614 }
1615
VisitNativeDebugInfo(HNativeDebugInfo * info)1616 void LocationsBuilderX86::VisitNativeDebugInfo(HNativeDebugInfo* info) {
1617 new (GetGraph()->GetArena()) LocationSummary(info);
1618 }
1619
VisitNativeDebugInfo(HNativeDebugInfo *)1620 void InstructionCodeGeneratorX86::VisitNativeDebugInfo(HNativeDebugInfo*) {
1621 // MaybeRecordNativeDebugInfo is already called implicitly in CodeGenerator::Compile.
1622 }
1623
GenerateNop()1624 void CodeGeneratorX86::GenerateNop() {
1625 __ nop();
1626 }
1627
HandleCondition(HCondition * cond)1628 void LocationsBuilderX86::HandleCondition(HCondition* cond) {
1629 LocationSummary* locations =
1630 new (GetGraph()->GetArena()) LocationSummary(cond, LocationSummary::kNoCall);
1631 // Handle the long/FP comparisons made in instruction simplification.
1632 switch (cond->InputAt(0)->GetType()) {
1633 case Primitive::kPrimLong: {
1634 locations->SetInAt(0, Location::RequiresRegister());
1635 locations->SetInAt(1, Location::Any());
1636 if (!cond->IsEmittedAtUseSite()) {
1637 locations->SetOut(Location::RequiresRegister());
1638 }
1639 break;
1640 }
1641 case Primitive::kPrimFloat:
1642 case Primitive::kPrimDouble: {
1643 locations->SetInAt(0, Location::RequiresFpuRegister());
1644 if (cond->InputAt(1)->IsX86LoadFromConstantTable()) {
1645 DCHECK(cond->InputAt(1)->IsEmittedAtUseSite());
1646 } else if (cond->InputAt(1)->IsConstant()) {
1647 locations->SetInAt(1, Location::RequiresFpuRegister());
1648 } else {
1649 locations->SetInAt(1, Location::Any());
1650 }
1651 if (!cond->IsEmittedAtUseSite()) {
1652 locations->SetOut(Location::RequiresRegister());
1653 }
1654 break;
1655 }
1656 default:
1657 locations->SetInAt(0, Location::RequiresRegister());
1658 locations->SetInAt(1, Location::Any());
1659 if (!cond->IsEmittedAtUseSite()) {
1660 // We need a byte register.
1661 locations->SetOut(Location::RegisterLocation(ECX));
1662 }
1663 break;
1664 }
1665 }
1666
HandleCondition(HCondition * cond)1667 void InstructionCodeGeneratorX86::HandleCondition(HCondition* cond) {
1668 if (cond->IsEmittedAtUseSite()) {
1669 return;
1670 }
1671
1672 LocationSummary* locations = cond->GetLocations();
1673 Location lhs = locations->InAt(0);
1674 Location rhs = locations->InAt(1);
1675 Register reg = locations->Out().AsRegister<Register>();
1676 NearLabel true_label, false_label;
1677
1678 switch (cond->InputAt(0)->GetType()) {
1679 default: {
1680 // Integer case.
1681
1682 // Clear output register: setb only sets the low byte.
1683 __ xorl(reg, reg);
1684 GenerateIntCompare(lhs, rhs);
1685 __ setb(X86Condition(cond->GetCondition()), reg);
1686 return;
1687 }
1688 case Primitive::kPrimLong:
1689 GenerateLongComparesAndJumps(cond, &true_label, &false_label);
1690 break;
1691 case Primitive::kPrimFloat:
1692 GenerateFPCompare(lhs, rhs, cond, false);
1693 GenerateFPJumps(cond, &true_label, &false_label);
1694 break;
1695 case Primitive::kPrimDouble:
1696 GenerateFPCompare(lhs, rhs, cond, true);
1697 GenerateFPJumps(cond, &true_label, &false_label);
1698 break;
1699 }
1700
1701 // Convert the jumps into the result.
1702 NearLabel done_label;
1703
1704 // False case: result = 0.
1705 __ Bind(&false_label);
1706 __ xorl(reg, reg);
1707 __ jmp(&done_label);
1708
1709 // True case: result = 1.
1710 __ Bind(&true_label);
1711 __ movl(reg, Immediate(1));
1712 __ Bind(&done_label);
1713 }
1714
VisitEqual(HEqual * comp)1715 void LocationsBuilderX86::VisitEqual(HEqual* comp) {
1716 HandleCondition(comp);
1717 }
1718
VisitEqual(HEqual * comp)1719 void InstructionCodeGeneratorX86::VisitEqual(HEqual* comp) {
1720 HandleCondition(comp);
1721 }
1722
VisitNotEqual(HNotEqual * comp)1723 void LocationsBuilderX86::VisitNotEqual(HNotEqual* comp) {
1724 HandleCondition(comp);
1725 }
1726
VisitNotEqual(HNotEqual * comp)1727 void InstructionCodeGeneratorX86::VisitNotEqual(HNotEqual* comp) {
1728 HandleCondition(comp);
1729 }
1730
VisitLessThan(HLessThan * comp)1731 void LocationsBuilderX86::VisitLessThan(HLessThan* comp) {
1732 HandleCondition(comp);
1733 }
1734
VisitLessThan(HLessThan * comp)1735 void InstructionCodeGeneratorX86::VisitLessThan(HLessThan* comp) {
1736 HandleCondition(comp);
1737 }
1738
VisitLessThanOrEqual(HLessThanOrEqual * comp)1739 void LocationsBuilderX86::VisitLessThanOrEqual(HLessThanOrEqual* comp) {
1740 HandleCondition(comp);
1741 }
1742
VisitLessThanOrEqual(HLessThanOrEqual * comp)1743 void InstructionCodeGeneratorX86::VisitLessThanOrEqual(HLessThanOrEqual* comp) {
1744 HandleCondition(comp);
1745 }
1746
VisitGreaterThan(HGreaterThan * comp)1747 void LocationsBuilderX86::VisitGreaterThan(HGreaterThan* comp) {
1748 HandleCondition(comp);
1749 }
1750
VisitGreaterThan(HGreaterThan * comp)1751 void InstructionCodeGeneratorX86::VisitGreaterThan(HGreaterThan* comp) {
1752 HandleCondition(comp);
1753 }
1754
VisitGreaterThanOrEqual(HGreaterThanOrEqual * comp)1755 void LocationsBuilderX86::VisitGreaterThanOrEqual(HGreaterThanOrEqual* comp) {
1756 HandleCondition(comp);
1757 }
1758
VisitGreaterThanOrEqual(HGreaterThanOrEqual * comp)1759 void InstructionCodeGeneratorX86::VisitGreaterThanOrEqual(HGreaterThanOrEqual* comp) {
1760 HandleCondition(comp);
1761 }
1762
VisitBelow(HBelow * comp)1763 void LocationsBuilderX86::VisitBelow(HBelow* comp) {
1764 HandleCondition(comp);
1765 }
1766
VisitBelow(HBelow * comp)1767 void InstructionCodeGeneratorX86::VisitBelow(HBelow* comp) {
1768 HandleCondition(comp);
1769 }
1770
VisitBelowOrEqual(HBelowOrEqual * comp)1771 void LocationsBuilderX86::VisitBelowOrEqual(HBelowOrEqual* comp) {
1772 HandleCondition(comp);
1773 }
1774
VisitBelowOrEqual(HBelowOrEqual * comp)1775 void InstructionCodeGeneratorX86::VisitBelowOrEqual(HBelowOrEqual* comp) {
1776 HandleCondition(comp);
1777 }
1778
VisitAbove(HAbove * comp)1779 void LocationsBuilderX86::VisitAbove(HAbove* comp) {
1780 HandleCondition(comp);
1781 }
1782
VisitAbove(HAbove * comp)1783 void InstructionCodeGeneratorX86::VisitAbove(HAbove* comp) {
1784 HandleCondition(comp);
1785 }
1786
VisitAboveOrEqual(HAboveOrEqual * comp)1787 void LocationsBuilderX86::VisitAboveOrEqual(HAboveOrEqual* comp) {
1788 HandleCondition(comp);
1789 }
1790
VisitAboveOrEqual(HAboveOrEqual * comp)1791 void InstructionCodeGeneratorX86::VisitAboveOrEqual(HAboveOrEqual* comp) {
1792 HandleCondition(comp);
1793 }
1794
VisitIntConstant(HIntConstant * constant)1795 void LocationsBuilderX86::VisitIntConstant(HIntConstant* constant) {
1796 LocationSummary* locations =
1797 new (GetGraph()->GetArena()) LocationSummary(constant, LocationSummary::kNoCall);
1798 locations->SetOut(Location::ConstantLocation(constant));
1799 }
1800
VisitIntConstant(HIntConstant * constant ATTRIBUTE_UNUSED)1801 void InstructionCodeGeneratorX86::VisitIntConstant(HIntConstant* constant ATTRIBUTE_UNUSED) {
1802 // Will be generated at use site.
1803 }
1804
VisitNullConstant(HNullConstant * constant)1805 void LocationsBuilderX86::VisitNullConstant(HNullConstant* constant) {
1806 LocationSummary* locations =
1807 new (GetGraph()->GetArena()) LocationSummary(constant, LocationSummary::kNoCall);
1808 locations->SetOut(Location::ConstantLocation(constant));
1809 }
1810
VisitNullConstant(HNullConstant * constant ATTRIBUTE_UNUSED)1811 void InstructionCodeGeneratorX86::VisitNullConstant(HNullConstant* constant ATTRIBUTE_UNUSED) {
1812 // Will be generated at use site.
1813 }
1814
VisitLongConstant(HLongConstant * constant)1815 void LocationsBuilderX86::VisitLongConstant(HLongConstant* constant) {
1816 LocationSummary* locations =
1817 new (GetGraph()->GetArena()) LocationSummary(constant, LocationSummary::kNoCall);
1818 locations->SetOut(Location::ConstantLocation(constant));
1819 }
1820
VisitLongConstant(HLongConstant * constant ATTRIBUTE_UNUSED)1821 void InstructionCodeGeneratorX86::VisitLongConstant(HLongConstant* constant ATTRIBUTE_UNUSED) {
1822 // Will be generated at use site.
1823 }
1824
VisitFloatConstant(HFloatConstant * constant)1825 void LocationsBuilderX86::VisitFloatConstant(HFloatConstant* constant) {
1826 LocationSummary* locations =
1827 new (GetGraph()->GetArena()) LocationSummary(constant, LocationSummary::kNoCall);
1828 locations->SetOut(Location::ConstantLocation(constant));
1829 }
1830
VisitFloatConstant(HFloatConstant * constant ATTRIBUTE_UNUSED)1831 void InstructionCodeGeneratorX86::VisitFloatConstant(HFloatConstant* constant ATTRIBUTE_UNUSED) {
1832 // Will be generated at use site.
1833 }
1834
VisitDoubleConstant(HDoubleConstant * constant)1835 void LocationsBuilderX86::VisitDoubleConstant(HDoubleConstant* constant) {
1836 LocationSummary* locations =
1837 new (GetGraph()->GetArena()) LocationSummary(constant, LocationSummary::kNoCall);
1838 locations->SetOut(Location::ConstantLocation(constant));
1839 }
1840
VisitDoubleConstant(HDoubleConstant * constant ATTRIBUTE_UNUSED)1841 void InstructionCodeGeneratorX86::VisitDoubleConstant(HDoubleConstant* constant ATTRIBUTE_UNUSED) {
1842 // Will be generated at use site.
1843 }
1844
VisitMemoryBarrier(HMemoryBarrier * memory_barrier)1845 void LocationsBuilderX86::VisitMemoryBarrier(HMemoryBarrier* memory_barrier) {
1846 memory_barrier->SetLocations(nullptr);
1847 }
1848
VisitMemoryBarrier(HMemoryBarrier * memory_barrier)1849 void InstructionCodeGeneratorX86::VisitMemoryBarrier(HMemoryBarrier* memory_barrier) {
1850 codegen_->GenerateMemoryBarrier(memory_barrier->GetBarrierKind());
1851 }
1852
VisitReturnVoid(HReturnVoid * ret)1853 void LocationsBuilderX86::VisitReturnVoid(HReturnVoid* ret) {
1854 ret->SetLocations(nullptr);
1855 }
1856
VisitReturnVoid(HReturnVoid * ret ATTRIBUTE_UNUSED)1857 void InstructionCodeGeneratorX86::VisitReturnVoid(HReturnVoid* ret ATTRIBUTE_UNUSED) {
1858 codegen_->GenerateFrameExit();
1859 }
1860
VisitReturn(HReturn * ret)1861 void LocationsBuilderX86::VisitReturn(HReturn* ret) {
1862 LocationSummary* locations =
1863 new (GetGraph()->GetArena()) LocationSummary(ret, LocationSummary::kNoCall);
1864 switch (ret->InputAt(0)->GetType()) {
1865 case Primitive::kPrimBoolean:
1866 case Primitive::kPrimByte:
1867 case Primitive::kPrimChar:
1868 case Primitive::kPrimShort:
1869 case Primitive::kPrimInt:
1870 case Primitive::kPrimNot:
1871 locations->SetInAt(0, Location::RegisterLocation(EAX));
1872 break;
1873
1874 case Primitive::kPrimLong:
1875 locations->SetInAt(
1876 0, Location::RegisterPairLocation(EAX, EDX));
1877 break;
1878
1879 case Primitive::kPrimFloat:
1880 case Primitive::kPrimDouble:
1881 locations->SetInAt(
1882 0, Location::FpuRegisterLocation(XMM0));
1883 break;
1884
1885 default:
1886 LOG(FATAL) << "Unknown return type " << ret->InputAt(0)->GetType();
1887 }
1888 }
1889
VisitReturn(HReturn * ret)1890 void InstructionCodeGeneratorX86::VisitReturn(HReturn* ret) {
1891 if (kIsDebugBuild) {
1892 switch (ret->InputAt(0)->GetType()) {
1893 case Primitive::kPrimBoolean:
1894 case Primitive::kPrimByte:
1895 case Primitive::kPrimChar:
1896 case Primitive::kPrimShort:
1897 case Primitive::kPrimInt:
1898 case Primitive::kPrimNot:
1899 DCHECK_EQ(ret->GetLocations()->InAt(0).AsRegister<Register>(), EAX);
1900 break;
1901
1902 case Primitive::kPrimLong:
1903 DCHECK_EQ(ret->GetLocations()->InAt(0).AsRegisterPairLow<Register>(), EAX);
1904 DCHECK_EQ(ret->GetLocations()->InAt(0).AsRegisterPairHigh<Register>(), EDX);
1905 break;
1906
1907 case Primitive::kPrimFloat:
1908 case Primitive::kPrimDouble:
1909 DCHECK_EQ(ret->GetLocations()->InAt(0).AsFpuRegister<XmmRegister>(), XMM0);
1910 break;
1911
1912 default:
1913 LOG(FATAL) << "Unknown return type " << ret->InputAt(0)->GetType();
1914 }
1915 }
1916 codegen_->GenerateFrameExit();
1917 }
1918
VisitInvokeUnresolved(HInvokeUnresolved * invoke)1919 void LocationsBuilderX86::VisitInvokeUnresolved(HInvokeUnresolved* invoke) {
1920 // The trampoline uses the same calling convention as dex calling conventions,
1921 // except instead of loading arg0/r0 with the target Method*, arg0/r0 will contain
1922 // the method_idx.
1923 HandleInvoke(invoke);
1924 }
1925
VisitInvokeUnresolved(HInvokeUnresolved * invoke)1926 void InstructionCodeGeneratorX86::VisitInvokeUnresolved(HInvokeUnresolved* invoke) {
1927 codegen_->GenerateInvokeUnresolvedRuntimeCall(invoke);
1928 }
1929
VisitInvokeStaticOrDirect(HInvokeStaticOrDirect * invoke)1930 void LocationsBuilderX86::VisitInvokeStaticOrDirect(HInvokeStaticOrDirect* invoke) {
1931 // Explicit clinit checks triggered by static invokes must have been pruned by
1932 // art::PrepareForRegisterAllocation.
1933 DCHECK(!invoke->IsStaticWithExplicitClinitCheck());
1934
1935 IntrinsicLocationsBuilderX86 intrinsic(codegen_);
1936 if (intrinsic.TryDispatch(invoke)) {
1937 if (invoke->GetLocations()->CanCall() && invoke->HasPcRelativeDexCache()) {
1938 invoke->GetLocations()->SetInAt(invoke->GetSpecialInputIndex(), Location::Any());
1939 }
1940 return;
1941 }
1942
1943 HandleInvoke(invoke);
1944
1945 // For PC-relative dex cache the invoke has an extra input, the PC-relative address base.
1946 if (invoke->HasPcRelativeDexCache()) {
1947 invoke->GetLocations()->SetInAt(invoke->GetSpecialInputIndex(), Location::RequiresRegister());
1948 }
1949 }
1950
TryGenerateIntrinsicCode(HInvoke * invoke,CodeGeneratorX86 * codegen)1951 static bool TryGenerateIntrinsicCode(HInvoke* invoke, CodeGeneratorX86* codegen) {
1952 if (invoke->GetLocations()->Intrinsified()) {
1953 IntrinsicCodeGeneratorX86 intrinsic(codegen);
1954 intrinsic.Dispatch(invoke);
1955 return true;
1956 }
1957 return false;
1958 }
1959
VisitInvokeStaticOrDirect(HInvokeStaticOrDirect * invoke)1960 void InstructionCodeGeneratorX86::VisitInvokeStaticOrDirect(HInvokeStaticOrDirect* invoke) {
1961 // Explicit clinit checks triggered by static invokes must have been pruned by
1962 // art::PrepareForRegisterAllocation.
1963 DCHECK(!invoke->IsStaticWithExplicitClinitCheck());
1964
1965 if (TryGenerateIntrinsicCode(invoke, codegen_)) {
1966 return;
1967 }
1968
1969 LocationSummary* locations = invoke->GetLocations();
1970 codegen_->GenerateStaticOrDirectCall(
1971 invoke, locations->HasTemps() ? locations->GetTemp(0) : Location::NoLocation());
1972 codegen_->RecordPcInfo(invoke, invoke->GetDexPc());
1973 }
1974
VisitInvokeVirtual(HInvokeVirtual * invoke)1975 void LocationsBuilderX86::VisitInvokeVirtual(HInvokeVirtual* invoke) {
1976 IntrinsicLocationsBuilderX86 intrinsic(codegen_);
1977 if (intrinsic.TryDispatch(invoke)) {
1978 return;
1979 }
1980
1981 HandleInvoke(invoke);
1982 }
1983
HandleInvoke(HInvoke * invoke)1984 void LocationsBuilderX86::HandleInvoke(HInvoke* invoke) {
1985 InvokeDexCallingConventionVisitorX86 calling_convention_visitor;
1986 CodeGenerator::CreateCommonInvokeLocationSummary(invoke, &calling_convention_visitor);
1987 }
1988
VisitInvokeVirtual(HInvokeVirtual * invoke)1989 void InstructionCodeGeneratorX86::VisitInvokeVirtual(HInvokeVirtual* invoke) {
1990 if (TryGenerateIntrinsicCode(invoke, codegen_)) {
1991 return;
1992 }
1993
1994 codegen_->GenerateVirtualCall(invoke, invoke->GetLocations()->GetTemp(0));
1995 DCHECK(!codegen_->IsLeafMethod());
1996 codegen_->RecordPcInfo(invoke, invoke->GetDexPc());
1997 }
1998
VisitInvokeInterface(HInvokeInterface * invoke)1999 void LocationsBuilderX86::VisitInvokeInterface(HInvokeInterface* invoke) {
2000 // This call to HandleInvoke allocates a temporary (core) register
2001 // which is also used to transfer the hidden argument from FP to
2002 // core register.
2003 HandleInvoke(invoke);
2004 // Add the hidden argument.
2005 invoke->GetLocations()->AddTemp(Location::FpuRegisterLocation(XMM7));
2006 }
2007
VisitInvokeInterface(HInvokeInterface * invoke)2008 void InstructionCodeGeneratorX86::VisitInvokeInterface(HInvokeInterface* invoke) {
2009 // TODO: b/18116999, our IMTs can miss an IncompatibleClassChangeError.
2010 LocationSummary* locations = invoke->GetLocations();
2011 Register temp = locations->GetTemp(0).AsRegister<Register>();
2012 XmmRegister hidden_reg = locations->GetTemp(1).AsFpuRegister<XmmRegister>();
2013 uint32_t method_offset = mirror::Class::EmbeddedImTableEntryOffset(
2014 invoke->GetImtIndex() % mirror::Class::kImtSize, kX86PointerSize).Uint32Value();
2015 Location receiver = locations->InAt(0);
2016 uint32_t class_offset = mirror::Object::ClassOffset().Int32Value();
2017
2018 // Set the hidden argument. This is safe to do this here, as XMM7
2019 // won't be modified thereafter, before the `call` instruction.
2020 DCHECK_EQ(XMM7, hidden_reg);
2021 __ movl(temp, Immediate(invoke->GetDexMethodIndex()));
2022 __ movd(hidden_reg, temp);
2023
2024 if (receiver.IsStackSlot()) {
2025 __ movl(temp, Address(ESP, receiver.GetStackIndex()));
2026 // /* HeapReference<Class> */ temp = temp->klass_
2027 __ movl(temp, Address(temp, class_offset));
2028 } else {
2029 // /* HeapReference<Class> */ temp = receiver->klass_
2030 __ movl(temp, Address(receiver.AsRegister<Register>(), class_offset));
2031 }
2032 codegen_->MaybeRecordImplicitNullCheck(invoke);
2033 // Instead of simply (possibly) unpoisoning `temp` here, we should
2034 // emit a read barrier for the previous class reference load.
2035 // However this is not required in practice, as this is an
2036 // intermediate/temporary reference and because the current
2037 // concurrent copying collector keeps the from-space memory
2038 // intact/accessible until the end of the marking phase (the
2039 // concurrent copying collector may not in the future).
2040 __ MaybeUnpoisonHeapReference(temp);
2041 // temp = temp->GetImtEntryAt(method_offset);
2042 __ movl(temp, Address(temp, method_offset));
2043 // call temp->GetEntryPoint();
2044 __ call(Address(temp,
2045 ArtMethod::EntryPointFromQuickCompiledCodeOffset(kX86WordSize).Int32Value()));
2046
2047 DCHECK(!codegen_->IsLeafMethod());
2048 codegen_->RecordPcInfo(invoke, invoke->GetDexPc());
2049 }
2050
VisitNeg(HNeg * neg)2051 void LocationsBuilderX86::VisitNeg(HNeg* neg) {
2052 LocationSummary* locations =
2053 new (GetGraph()->GetArena()) LocationSummary(neg, LocationSummary::kNoCall);
2054 switch (neg->GetResultType()) {
2055 case Primitive::kPrimInt:
2056 case Primitive::kPrimLong:
2057 locations->SetInAt(0, Location::RequiresRegister());
2058 locations->SetOut(Location::SameAsFirstInput());
2059 break;
2060
2061 case Primitive::kPrimFloat:
2062 locations->SetInAt(0, Location::RequiresFpuRegister());
2063 locations->SetOut(Location::SameAsFirstInput());
2064 locations->AddTemp(Location::RequiresRegister());
2065 locations->AddTemp(Location::RequiresFpuRegister());
2066 break;
2067
2068 case Primitive::kPrimDouble:
2069 locations->SetInAt(0, Location::RequiresFpuRegister());
2070 locations->SetOut(Location::SameAsFirstInput());
2071 locations->AddTemp(Location::RequiresFpuRegister());
2072 break;
2073
2074 default:
2075 LOG(FATAL) << "Unexpected neg type " << neg->GetResultType();
2076 }
2077 }
2078
VisitNeg(HNeg * neg)2079 void InstructionCodeGeneratorX86::VisitNeg(HNeg* neg) {
2080 LocationSummary* locations = neg->GetLocations();
2081 Location out = locations->Out();
2082 Location in = locations->InAt(0);
2083 switch (neg->GetResultType()) {
2084 case Primitive::kPrimInt:
2085 DCHECK(in.IsRegister());
2086 DCHECK(in.Equals(out));
2087 __ negl(out.AsRegister<Register>());
2088 break;
2089
2090 case Primitive::kPrimLong:
2091 DCHECK(in.IsRegisterPair());
2092 DCHECK(in.Equals(out));
2093 __ negl(out.AsRegisterPairLow<Register>());
2094 // Negation is similar to subtraction from zero. The least
2095 // significant byte triggers a borrow when it is different from
2096 // zero; to take it into account, add 1 to the most significant
2097 // byte if the carry flag (CF) is set to 1 after the first NEGL
2098 // operation.
2099 __ adcl(out.AsRegisterPairHigh<Register>(), Immediate(0));
2100 __ negl(out.AsRegisterPairHigh<Register>());
2101 break;
2102
2103 case Primitive::kPrimFloat: {
2104 DCHECK(in.Equals(out));
2105 Register constant = locations->GetTemp(0).AsRegister<Register>();
2106 XmmRegister mask = locations->GetTemp(1).AsFpuRegister<XmmRegister>();
2107 // Implement float negation with an exclusive or with value
2108 // 0x80000000 (mask for bit 31, representing the sign of a
2109 // single-precision floating-point number).
2110 __ movl(constant, Immediate(INT32_C(0x80000000)));
2111 __ movd(mask, constant);
2112 __ xorps(out.AsFpuRegister<XmmRegister>(), mask);
2113 break;
2114 }
2115
2116 case Primitive::kPrimDouble: {
2117 DCHECK(in.Equals(out));
2118 XmmRegister mask = locations->GetTemp(0).AsFpuRegister<XmmRegister>();
2119 // Implement double negation with an exclusive or with value
2120 // 0x8000000000000000 (mask for bit 63, representing the sign of
2121 // a double-precision floating-point number).
2122 __ LoadLongConstant(mask, INT64_C(0x8000000000000000));
2123 __ xorpd(out.AsFpuRegister<XmmRegister>(), mask);
2124 break;
2125 }
2126
2127 default:
2128 LOG(FATAL) << "Unexpected neg type " << neg->GetResultType();
2129 }
2130 }
2131
VisitX86FPNeg(HX86FPNeg * neg)2132 void LocationsBuilderX86::VisitX86FPNeg(HX86FPNeg* neg) {
2133 LocationSummary* locations =
2134 new (GetGraph()->GetArena()) LocationSummary(neg, LocationSummary::kNoCall);
2135 DCHECK(Primitive::IsFloatingPointType(neg->GetType()));
2136 locations->SetInAt(0, Location::RequiresFpuRegister());
2137 locations->SetInAt(1, Location::RequiresRegister());
2138 locations->SetOut(Location::SameAsFirstInput());
2139 locations->AddTemp(Location::RequiresFpuRegister());
2140 }
2141
VisitX86FPNeg(HX86FPNeg * neg)2142 void InstructionCodeGeneratorX86::VisitX86FPNeg(HX86FPNeg* neg) {
2143 LocationSummary* locations = neg->GetLocations();
2144 Location out = locations->Out();
2145 DCHECK(locations->InAt(0).Equals(out));
2146
2147 Register constant_area = locations->InAt(1).AsRegister<Register>();
2148 XmmRegister mask = locations->GetTemp(0).AsFpuRegister<XmmRegister>();
2149 if (neg->GetType() == Primitive::kPrimFloat) {
2150 __ movss(mask, codegen_->LiteralInt32Address(INT32_C(0x80000000), constant_area));
2151 __ xorps(out.AsFpuRegister<XmmRegister>(), mask);
2152 } else {
2153 __ movsd(mask, codegen_->LiteralInt64Address(INT64_C(0x8000000000000000), constant_area));
2154 __ xorpd(out.AsFpuRegister<XmmRegister>(), mask);
2155 }
2156 }
2157
VisitTypeConversion(HTypeConversion * conversion)2158 void LocationsBuilderX86::VisitTypeConversion(HTypeConversion* conversion) {
2159 Primitive::Type result_type = conversion->GetResultType();
2160 Primitive::Type input_type = conversion->GetInputType();
2161 DCHECK_NE(result_type, input_type);
2162
2163 // The float-to-long and double-to-long type conversions rely on a
2164 // call to the runtime.
2165 LocationSummary::CallKind call_kind =
2166 ((input_type == Primitive::kPrimFloat || input_type == Primitive::kPrimDouble)
2167 && result_type == Primitive::kPrimLong)
2168 ? LocationSummary::kCall
2169 : LocationSummary::kNoCall;
2170 LocationSummary* locations =
2171 new (GetGraph()->GetArena()) LocationSummary(conversion, call_kind);
2172
2173 // The Java language does not allow treating boolean as an integral type but
2174 // our bit representation makes it safe.
2175
2176 switch (result_type) {
2177 case Primitive::kPrimByte:
2178 switch (input_type) {
2179 case Primitive::kPrimLong: {
2180 // Type conversion from long to byte is a result of code transformations.
2181 HInstruction* input = conversion->InputAt(0);
2182 Location input_location = input->IsConstant()
2183 ? Location::ConstantLocation(input->AsConstant())
2184 : Location::RegisterPairLocation(EAX, EDX);
2185 locations->SetInAt(0, input_location);
2186 // Make the output overlap to please the register allocator. This greatly simplifies
2187 // the validation of the linear scan implementation
2188 locations->SetOut(Location::RequiresRegister(), Location::kOutputOverlap);
2189 break;
2190 }
2191 case Primitive::kPrimBoolean:
2192 // Boolean input is a result of code transformations.
2193 case Primitive::kPrimShort:
2194 case Primitive::kPrimInt:
2195 case Primitive::kPrimChar:
2196 // Processing a Dex `int-to-byte' instruction.
2197 locations->SetInAt(0, Location::ByteRegisterOrConstant(ECX, conversion->InputAt(0)));
2198 // Make the output overlap to please the register allocator. This greatly simplifies
2199 // the validation of the linear scan implementation
2200 locations->SetOut(Location::RequiresRegister(), Location::kOutputOverlap);
2201 break;
2202
2203 default:
2204 LOG(FATAL) << "Unexpected type conversion from " << input_type
2205 << " to " << result_type;
2206 }
2207 break;
2208
2209 case Primitive::kPrimShort:
2210 switch (input_type) {
2211 case Primitive::kPrimLong:
2212 // Type conversion from long to short is a result of code transformations.
2213 case Primitive::kPrimBoolean:
2214 // Boolean input is a result of code transformations.
2215 case Primitive::kPrimByte:
2216 case Primitive::kPrimInt:
2217 case Primitive::kPrimChar:
2218 // Processing a Dex `int-to-short' instruction.
2219 locations->SetInAt(0, Location::Any());
2220 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
2221 break;
2222
2223 default:
2224 LOG(FATAL) << "Unexpected type conversion from " << input_type
2225 << " to " << result_type;
2226 }
2227 break;
2228
2229 case Primitive::kPrimInt:
2230 switch (input_type) {
2231 case Primitive::kPrimLong:
2232 // Processing a Dex `long-to-int' instruction.
2233 locations->SetInAt(0, Location::Any());
2234 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
2235 break;
2236
2237 case Primitive::kPrimFloat:
2238 // Processing a Dex `float-to-int' instruction.
2239 locations->SetInAt(0, Location::RequiresFpuRegister());
2240 locations->SetOut(Location::RequiresRegister());
2241 locations->AddTemp(Location::RequiresFpuRegister());
2242 break;
2243
2244 case Primitive::kPrimDouble:
2245 // Processing a Dex `double-to-int' instruction.
2246 locations->SetInAt(0, Location::RequiresFpuRegister());
2247 locations->SetOut(Location::RequiresRegister());
2248 locations->AddTemp(Location::RequiresFpuRegister());
2249 break;
2250
2251 default:
2252 LOG(FATAL) << "Unexpected type conversion from " << input_type
2253 << " to " << result_type;
2254 }
2255 break;
2256
2257 case Primitive::kPrimLong:
2258 switch (input_type) {
2259 case Primitive::kPrimBoolean:
2260 // Boolean input is a result of code transformations.
2261 case Primitive::kPrimByte:
2262 case Primitive::kPrimShort:
2263 case Primitive::kPrimInt:
2264 case Primitive::kPrimChar:
2265 // Processing a Dex `int-to-long' instruction.
2266 locations->SetInAt(0, Location::RegisterLocation(EAX));
2267 locations->SetOut(Location::RegisterPairLocation(EAX, EDX));
2268 break;
2269
2270 case Primitive::kPrimFloat:
2271 case Primitive::kPrimDouble: {
2272 // Processing a Dex `float-to-long' or 'double-to-long' instruction.
2273 InvokeRuntimeCallingConvention calling_convention;
2274 XmmRegister parameter = calling_convention.GetFpuRegisterAt(0);
2275 locations->SetInAt(0, Location::FpuRegisterLocation(parameter));
2276
2277 // The runtime helper puts the result in EAX, EDX.
2278 locations->SetOut(Location::RegisterPairLocation(EAX, EDX));
2279 }
2280 break;
2281
2282 default:
2283 LOG(FATAL) << "Unexpected type conversion from " << input_type
2284 << " to " << result_type;
2285 }
2286 break;
2287
2288 case Primitive::kPrimChar:
2289 switch (input_type) {
2290 case Primitive::kPrimLong:
2291 // Type conversion from long to char is a result of code transformations.
2292 case Primitive::kPrimBoolean:
2293 // Boolean input is a result of code transformations.
2294 case Primitive::kPrimByte:
2295 case Primitive::kPrimShort:
2296 case Primitive::kPrimInt:
2297 // Processing a Dex `int-to-char' instruction.
2298 locations->SetInAt(0, Location::Any());
2299 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
2300 break;
2301
2302 default:
2303 LOG(FATAL) << "Unexpected type conversion from " << input_type
2304 << " to " << result_type;
2305 }
2306 break;
2307
2308 case Primitive::kPrimFloat:
2309 switch (input_type) {
2310 case Primitive::kPrimBoolean:
2311 // Boolean input is a result of code transformations.
2312 case Primitive::kPrimByte:
2313 case Primitive::kPrimShort:
2314 case Primitive::kPrimInt:
2315 case Primitive::kPrimChar:
2316 // Processing a Dex `int-to-float' instruction.
2317 locations->SetInAt(0, Location::RequiresRegister());
2318 locations->SetOut(Location::RequiresFpuRegister());
2319 break;
2320
2321 case Primitive::kPrimLong:
2322 // Processing a Dex `long-to-float' instruction.
2323 locations->SetInAt(0, Location::Any());
2324 locations->SetOut(Location::Any());
2325 break;
2326
2327 case Primitive::kPrimDouble:
2328 // Processing a Dex `double-to-float' instruction.
2329 locations->SetInAt(0, Location::RequiresFpuRegister());
2330 locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap);
2331 break;
2332
2333 default:
2334 LOG(FATAL) << "Unexpected type conversion from " << input_type
2335 << " to " << result_type;
2336 };
2337 break;
2338
2339 case Primitive::kPrimDouble:
2340 switch (input_type) {
2341 case Primitive::kPrimBoolean:
2342 // Boolean input is a result of code transformations.
2343 case Primitive::kPrimByte:
2344 case Primitive::kPrimShort:
2345 case Primitive::kPrimInt:
2346 case Primitive::kPrimChar:
2347 // Processing a Dex `int-to-double' instruction.
2348 locations->SetInAt(0, Location::RequiresRegister());
2349 locations->SetOut(Location::RequiresFpuRegister());
2350 break;
2351
2352 case Primitive::kPrimLong:
2353 // Processing a Dex `long-to-double' instruction.
2354 locations->SetInAt(0, Location::Any());
2355 locations->SetOut(Location::Any());
2356 break;
2357
2358 case Primitive::kPrimFloat:
2359 // Processing a Dex `float-to-double' instruction.
2360 locations->SetInAt(0, Location::RequiresFpuRegister());
2361 locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap);
2362 break;
2363
2364 default:
2365 LOG(FATAL) << "Unexpected type conversion from " << input_type
2366 << " to " << result_type;
2367 }
2368 break;
2369
2370 default:
2371 LOG(FATAL) << "Unexpected type conversion from " << input_type
2372 << " to " << result_type;
2373 }
2374 }
2375
VisitTypeConversion(HTypeConversion * conversion)2376 void InstructionCodeGeneratorX86::VisitTypeConversion(HTypeConversion* conversion) {
2377 LocationSummary* locations = conversion->GetLocations();
2378 Location out = locations->Out();
2379 Location in = locations->InAt(0);
2380 Primitive::Type result_type = conversion->GetResultType();
2381 Primitive::Type input_type = conversion->GetInputType();
2382 DCHECK_NE(result_type, input_type);
2383 switch (result_type) {
2384 case Primitive::kPrimByte:
2385 switch (input_type) {
2386 case Primitive::kPrimLong:
2387 // Type conversion from long to byte is a result of code transformations.
2388 if (in.IsRegisterPair()) {
2389 __ movsxb(out.AsRegister<Register>(), in.AsRegisterPairLow<ByteRegister>());
2390 } else {
2391 DCHECK(in.GetConstant()->IsLongConstant());
2392 int64_t value = in.GetConstant()->AsLongConstant()->GetValue();
2393 __ movl(out.AsRegister<Register>(), Immediate(static_cast<int8_t>(value)));
2394 }
2395 break;
2396 case Primitive::kPrimBoolean:
2397 // Boolean input is a result of code transformations.
2398 case Primitive::kPrimShort:
2399 case Primitive::kPrimInt:
2400 case Primitive::kPrimChar:
2401 // Processing a Dex `int-to-byte' instruction.
2402 if (in.IsRegister()) {
2403 __ movsxb(out.AsRegister<Register>(), in.AsRegister<ByteRegister>());
2404 } else {
2405 DCHECK(in.GetConstant()->IsIntConstant());
2406 int32_t value = in.GetConstant()->AsIntConstant()->GetValue();
2407 __ movl(out.AsRegister<Register>(), Immediate(static_cast<int8_t>(value)));
2408 }
2409 break;
2410
2411 default:
2412 LOG(FATAL) << "Unexpected type conversion from " << input_type
2413 << " to " << result_type;
2414 }
2415 break;
2416
2417 case Primitive::kPrimShort:
2418 switch (input_type) {
2419 case Primitive::kPrimLong:
2420 // Type conversion from long to short is a result of code transformations.
2421 if (in.IsRegisterPair()) {
2422 __ movsxw(out.AsRegister<Register>(), in.AsRegisterPairLow<Register>());
2423 } else if (in.IsDoubleStackSlot()) {
2424 __ movsxw(out.AsRegister<Register>(), Address(ESP, in.GetStackIndex()));
2425 } else {
2426 DCHECK(in.GetConstant()->IsLongConstant());
2427 int64_t value = in.GetConstant()->AsLongConstant()->GetValue();
2428 __ movl(out.AsRegister<Register>(), Immediate(static_cast<int16_t>(value)));
2429 }
2430 break;
2431 case Primitive::kPrimBoolean:
2432 // Boolean input is a result of code transformations.
2433 case Primitive::kPrimByte:
2434 case Primitive::kPrimInt:
2435 case Primitive::kPrimChar:
2436 // Processing a Dex `int-to-short' instruction.
2437 if (in.IsRegister()) {
2438 __ movsxw(out.AsRegister<Register>(), in.AsRegister<Register>());
2439 } else if (in.IsStackSlot()) {
2440 __ movsxw(out.AsRegister<Register>(), Address(ESP, in.GetStackIndex()));
2441 } else {
2442 DCHECK(in.GetConstant()->IsIntConstant());
2443 int32_t value = in.GetConstant()->AsIntConstant()->GetValue();
2444 __ movl(out.AsRegister<Register>(), Immediate(static_cast<int16_t>(value)));
2445 }
2446 break;
2447
2448 default:
2449 LOG(FATAL) << "Unexpected type conversion from " << input_type
2450 << " to " << result_type;
2451 }
2452 break;
2453
2454 case Primitive::kPrimInt:
2455 switch (input_type) {
2456 case Primitive::kPrimLong:
2457 // Processing a Dex `long-to-int' instruction.
2458 if (in.IsRegisterPair()) {
2459 __ movl(out.AsRegister<Register>(), in.AsRegisterPairLow<Register>());
2460 } else if (in.IsDoubleStackSlot()) {
2461 __ movl(out.AsRegister<Register>(), Address(ESP, in.GetStackIndex()));
2462 } else {
2463 DCHECK(in.IsConstant());
2464 DCHECK(in.GetConstant()->IsLongConstant());
2465 int64_t value = in.GetConstant()->AsLongConstant()->GetValue();
2466 __ movl(out.AsRegister<Register>(), Immediate(static_cast<int32_t>(value)));
2467 }
2468 break;
2469
2470 case Primitive::kPrimFloat: {
2471 // Processing a Dex `float-to-int' instruction.
2472 XmmRegister input = in.AsFpuRegister<XmmRegister>();
2473 Register output = out.AsRegister<Register>();
2474 XmmRegister temp = locations->GetTemp(0).AsFpuRegister<XmmRegister>();
2475 NearLabel done, nan;
2476
2477 __ movl(output, Immediate(kPrimIntMax));
2478 // temp = int-to-float(output)
2479 __ cvtsi2ss(temp, output);
2480 // if input >= temp goto done
2481 __ comiss(input, temp);
2482 __ j(kAboveEqual, &done);
2483 // if input == NaN goto nan
2484 __ j(kUnordered, &nan);
2485 // output = float-to-int-truncate(input)
2486 __ cvttss2si(output, input);
2487 __ jmp(&done);
2488 __ Bind(&nan);
2489 // output = 0
2490 __ xorl(output, output);
2491 __ Bind(&done);
2492 break;
2493 }
2494
2495 case Primitive::kPrimDouble: {
2496 // Processing a Dex `double-to-int' instruction.
2497 XmmRegister input = in.AsFpuRegister<XmmRegister>();
2498 Register output = out.AsRegister<Register>();
2499 XmmRegister temp = locations->GetTemp(0).AsFpuRegister<XmmRegister>();
2500 NearLabel done, nan;
2501
2502 __ movl(output, Immediate(kPrimIntMax));
2503 // temp = int-to-double(output)
2504 __ cvtsi2sd(temp, output);
2505 // if input >= temp goto done
2506 __ comisd(input, temp);
2507 __ j(kAboveEqual, &done);
2508 // if input == NaN goto nan
2509 __ j(kUnordered, &nan);
2510 // output = double-to-int-truncate(input)
2511 __ cvttsd2si(output, input);
2512 __ jmp(&done);
2513 __ Bind(&nan);
2514 // output = 0
2515 __ xorl(output, output);
2516 __ Bind(&done);
2517 break;
2518 }
2519
2520 default:
2521 LOG(FATAL) << "Unexpected type conversion from " << input_type
2522 << " to " << result_type;
2523 }
2524 break;
2525
2526 case Primitive::kPrimLong:
2527 switch (input_type) {
2528 case Primitive::kPrimBoolean:
2529 // Boolean input is a result of code transformations.
2530 case Primitive::kPrimByte:
2531 case Primitive::kPrimShort:
2532 case Primitive::kPrimInt:
2533 case Primitive::kPrimChar:
2534 // Processing a Dex `int-to-long' instruction.
2535 DCHECK_EQ(out.AsRegisterPairLow<Register>(), EAX);
2536 DCHECK_EQ(out.AsRegisterPairHigh<Register>(), EDX);
2537 DCHECK_EQ(in.AsRegister<Register>(), EAX);
2538 __ cdq();
2539 break;
2540
2541 case Primitive::kPrimFloat:
2542 // Processing a Dex `float-to-long' instruction.
2543 codegen_->InvokeRuntime(QUICK_ENTRY_POINT(pF2l),
2544 conversion,
2545 conversion->GetDexPc(),
2546 nullptr);
2547 CheckEntrypointTypes<kQuickF2l, int64_t, float>();
2548 break;
2549
2550 case Primitive::kPrimDouble:
2551 // Processing a Dex `double-to-long' instruction.
2552 codegen_->InvokeRuntime(QUICK_ENTRY_POINT(pD2l),
2553 conversion,
2554 conversion->GetDexPc(),
2555 nullptr);
2556 CheckEntrypointTypes<kQuickD2l, int64_t, double>();
2557 break;
2558
2559 default:
2560 LOG(FATAL) << "Unexpected type conversion from " << input_type
2561 << " to " << result_type;
2562 }
2563 break;
2564
2565 case Primitive::kPrimChar:
2566 switch (input_type) {
2567 case Primitive::kPrimLong:
2568 // Type conversion from long to short is a result of code transformations.
2569 if (in.IsRegisterPair()) {
2570 __ movzxw(out.AsRegister<Register>(), in.AsRegisterPairLow<Register>());
2571 } else if (in.IsDoubleStackSlot()) {
2572 __ movzxw(out.AsRegister<Register>(), Address(ESP, in.GetStackIndex()));
2573 } else {
2574 DCHECK(in.GetConstant()->IsLongConstant());
2575 int64_t value = in.GetConstant()->AsLongConstant()->GetValue();
2576 __ movl(out.AsRegister<Register>(), Immediate(static_cast<uint16_t>(value)));
2577 }
2578 break;
2579 case Primitive::kPrimBoolean:
2580 // Boolean input is a result of code transformations.
2581 case Primitive::kPrimByte:
2582 case Primitive::kPrimShort:
2583 case Primitive::kPrimInt:
2584 // Processing a Dex `Process a Dex `int-to-char'' instruction.
2585 if (in.IsRegister()) {
2586 __ movzxw(out.AsRegister<Register>(), in.AsRegister<Register>());
2587 } else if (in.IsStackSlot()) {
2588 __ movzxw(out.AsRegister<Register>(), Address(ESP, in.GetStackIndex()));
2589 } else {
2590 DCHECK(in.GetConstant()->IsIntConstant());
2591 int32_t value = in.GetConstant()->AsIntConstant()->GetValue();
2592 __ movl(out.AsRegister<Register>(), Immediate(static_cast<uint16_t>(value)));
2593 }
2594 break;
2595
2596 default:
2597 LOG(FATAL) << "Unexpected type conversion from " << input_type
2598 << " to " << result_type;
2599 }
2600 break;
2601
2602 case Primitive::kPrimFloat:
2603 switch (input_type) {
2604 case Primitive::kPrimBoolean:
2605 // Boolean input is a result of code transformations.
2606 case Primitive::kPrimByte:
2607 case Primitive::kPrimShort:
2608 case Primitive::kPrimInt:
2609 case Primitive::kPrimChar:
2610 // Processing a Dex `int-to-float' instruction.
2611 __ cvtsi2ss(out.AsFpuRegister<XmmRegister>(), in.AsRegister<Register>());
2612 break;
2613
2614 case Primitive::kPrimLong: {
2615 // Processing a Dex `long-to-float' instruction.
2616 size_t adjustment = 0;
2617
2618 // Create stack space for the call to
2619 // InstructionCodeGeneratorX86::PushOntoFPStack and/or X86Assembler::fstps below.
2620 // TODO: enhance register allocator to ask for stack temporaries.
2621 if (!in.IsDoubleStackSlot() || !out.IsStackSlot()) {
2622 adjustment = Primitive::ComponentSize(Primitive::kPrimLong);
2623 __ subl(ESP, Immediate(adjustment));
2624 }
2625
2626 // Load the value to the FP stack, using temporaries if needed.
2627 PushOntoFPStack(in, 0, adjustment, false, true);
2628
2629 if (out.IsStackSlot()) {
2630 __ fstps(Address(ESP, out.GetStackIndex() + adjustment));
2631 } else {
2632 __ fstps(Address(ESP, 0));
2633 Location stack_temp = Location::StackSlot(0);
2634 codegen_->Move32(out, stack_temp);
2635 }
2636
2637 // Remove the temporary stack space we allocated.
2638 if (adjustment != 0) {
2639 __ addl(ESP, Immediate(adjustment));
2640 }
2641 break;
2642 }
2643
2644 case Primitive::kPrimDouble:
2645 // Processing a Dex `double-to-float' instruction.
2646 __ cvtsd2ss(out.AsFpuRegister<XmmRegister>(), in.AsFpuRegister<XmmRegister>());
2647 break;
2648
2649 default:
2650 LOG(FATAL) << "Unexpected type conversion from " << input_type
2651 << " to " << result_type;
2652 };
2653 break;
2654
2655 case Primitive::kPrimDouble:
2656 switch (input_type) {
2657 case Primitive::kPrimBoolean:
2658 // Boolean input is a result of code transformations.
2659 case Primitive::kPrimByte:
2660 case Primitive::kPrimShort:
2661 case Primitive::kPrimInt:
2662 case Primitive::kPrimChar:
2663 // Processing a Dex `int-to-double' instruction.
2664 __ cvtsi2sd(out.AsFpuRegister<XmmRegister>(), in.AsRegister<Register>());
2665 break;
2666
2667 case Primitive::kPrimLong: {
2668 // Processing a Dex `long-to-double' instruction.
2669 size_t adjustment = 0;
2670
2671 // Create stack space for the call to
2672 // InstructionCodeGeneratorX86::PushOntoFPStack and/or X86Assembler::fstpl below.
2673 // TODO: enhance register allocator to ask for stack temporaries.
2674 if (!in.IsDoubleStackSlot() || !out.IsDoubleStackSlot()) {
2675 adjustment = Primitive::ComponentSize(Primitive::kPrimLong);
2676 __ subl(ESP, Immediate(adjustment));
2677 }
2678
2679 // Load the value to the FP stack, using temporaries if needed.
2680 PushOntoFPStack(in, 0, adjustment, false, true);
2681
2682 if (out.IsDoubleStackSlot()) {
2683 __ fstpl(Address(ESP, out.GetStackIndex() + adjustment));
2684 } else {
2685 __ fstpl(Address(ESP, 0));
2686 Location stack_temp = Location::DoubleStackSlot(0);
2687 codegen_->Move64(out, stack_temp);
2688 }
2689
2690 // Remove the temporary stack space we allocated.
2691 if (adjustment != 0) {
2692 __ addl(ESP, Immediate(adjustment));
2693 }
2694 break;
2695 }
2696
2697 case Primitive::kPrimFloat:
2698 // Processing a Dex `float-to-double' instruction.
2699 __ cvtss2sd(out.AsFpuRegister<XmmRegister>(), in.AsFpuRegister<XmmRegister>());
2700 break;
2701
2702 default:
2703 LOG(FATAL) << "Unexpected type conversion from " << input_type
2704 << " to " << result_type;
2705 };
2706 break;
2707
2708 default:
2709 LOG(FATAL) << "Unexpected type conversion from " << input_type
2710 << " to " << result_type;
2711 }
2712 }
2713
VisitAdd(HAdd * add)2714 void LocationsBuilderX86::VisitAdd(HAdd* add) {
2715 LocationSummary* locations =
2716 new (GetGraph()->GetArena()) LocationSummary(add, LocationSummary::kNoCall);
2717 switch (add->GetResultType()) {
2718 case Primitive::kPrimInt: {
2719 locations->SetInAt(0, Location::RequiresRegister());
2720 locations->SetInAt(1, Location::RegisterOrConstant(add->InputAt(1)));
2721 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
2722 break;
2723 }
2724
2725 case Primitive::kPrimLong: {
2726 locations->SetInAt(0, Location::RequiresRegister());
2727 locations->SetInAt(1, Location::Any());
2728 locations->SetOut(Location::SameAsFirstInput());
2729 break;
2730 }
2731
2732 case Primitive::kPrimFloat:
2733 case Primitive::kPrimDouble: {
2734 locations->SetInAt(0, Location::RequiresFpuRegister());
2735 if (add->InputAt(1)->IsX86LoadFromConstantTable()) {
2736 DCHECK(add->InputAt(1)->IsEmittedAtUseSite());
2737 } else if (add->InputAt(1)->IsConstant()) {
2738 locations->SetInAt(1, Location::RequiresFpuRegister());
2739 } else {
2740 locations->SetInAt(1, Location::Any());
2741 }
2742 locations->SetOut(Location::SameAsFirstInput());
2743 break;
2744 }
2745
2746 default:
2747 LOG(FATAL) << "Unexpected add type " << add->GetResultType();
2748 break;
2749 }
2750 }
2751
VisitAdd(HAdd * add)2752 void InstructionCodeGeneratorX86::VisitAdd(HAdd* add) {
2753 LocationSummary* locations = add->GetLocations();
2754 Location first = locations->InAt(0);
2755 Location second = locations->InAt(1);
2756 Location out = locations->Out();
2757
2758 switch (add->GetResultType()) {
2759 case Primitive::kPrimInt: {
2760 if (second.IsRegister()) {
2761 if (out.AsRegister<Register>() == first.AsRegister<Register>()) {
2762 __ addl(out.AsRegister<Register>(), second.AsRegister<Register>());
2763 } else if (out.AsRegister<Register>() == second.AsRegister<Register>()) {
2764 __ addl(out.AsRegister<Register>(), first.AsRegister<Register>());
2765 } else {
2766 __ leal(out.AsRegister<Register>(), Address(
2767 first.AsRegister<Register>(), second.AsRegister<Register>(), TIMES_1, 0));
2768 }
2769 } else if (second.IsConstant()) {
2770 int32_t value = second.GetConstant()->AsIntConstant()->GetValue();
2771 if (out.AsRegister<Register>() == first.AsRegister<Register>()) {
2772 __ addl(out.AsRegister<Register>(), Immediate(value));
2773 } else {
2774 __ leal(out.AsRegister<Register>(), Address(first.AsRegister<Register>(), value));
2775 }
2776 } else {
2777 DCHECK(first.Equals(locations->Out()));
2778 __ addl(first.AsRegister<Register>(), Address(ESP, second.GetStackIndex()));
2779 }
2780 break;
2781 }
2782
2783 case Primitive::kPrimLong: {
2784 if (second.IsRegisterPair()) {
2785 __ addl(first.AsRegisterPairLow<Register>(), second.AsRegisterPairLow<Register>());
2786 __ adcl(first.AsRegisterPairHigh<Register>(), second.AsRegisterPairHigh<Register>());
2787 } else if (second.IsDoubleStackSlot()) {
2788 __ addl(first.AsRegisterPairLow<Register>(), Address(ESP, second.GetStackIndex()));
2789 __ adcl(first.AsRegisterPairHigh<Register>(),
2790 Address(ESP, second.GetHighStackIndex(kX86WordSize)));
2791 } else {
2792 DCHECK(second.IsConstant()) << second;
2793 int64_t value = second.GetConstant()->AsLongConstant()->GetValue();
2794 __ addl(first.AsRegisterPairLow<Register>(), Immediate(Low32Bits(value)));
2795 __ adcl(first.AsRegisterPairHigh<Register>(), Immediate(High32Bits(value)));
2796 }
2797 break;
2798 }
2799
2800 case Primitive::kPrimFloat: {
2801 if (second.IsFpuRegister()) {
2802 __ addss(first.AsFpuRegister<XmmRegister>(), second.AsFpuRegister<XmmRegister>());
2803 } else if (add->InputAt(1)->IsX86LoadFromConstantTable()) {
2804 HX86LoadFromConstantTable* const_area = add->InputAt(1)->AsX86LoadFromConstantTable();
2805 DCHECK(const_area->IsEmittedAtUseSite());
2806 __ addss(first.AsFpuRegister<XmmRegister>(),
2807 codegen_->LiteralFloatAddress(
2808 const_area->GetConstant()->AsFloatConstant()->GetValue(),
2809 const_area->GetLocations()->InAt(0).AsRegister<Register>()));
2810 } else {
2811 DCHECK(second.IsStackSlot());
2812 __ addss(first.AsFpuRegister<XmmRegister>(), Address(ESP, second.GetStackIndex()));
2813 }
2814 break;
2815 }
2816
2817 case Primitive::kPrimDouble: {
2818 if (second.IsFpuRegister()) {
2819 __ addsd(first.AsFpuRegister<XmmRegister>(), second.AsFpuRegister<XmmRegister>());
2820 } else if (add->InputAt(1)->IsX86LoadFromConstantTable()) {
2821 HX86LoadFromConstantTable* const_area = add->InputAt(1)->AsX86LoadFromConstantTable();
2822 DCHECK(const_area->IsEmittedAtUseSite());
2823 __ addsd(first.AsFpuRegister<XmmRegister>(),
2824 codegen_->LiteralDoubleAddress(
2825 const_area->GetConstant()->AsDoubleConstant()->GetValue(),
2826 const_area->GetLocations()->InAt(0).AsRegister<Register>()));
2827 } else {
2828 DCHECK(second.IsDoubleStackSlot());
2829 __ addsd(first.AsFpuRegister<XmmRegister>(), Address(ESP, second.GetStackIndex()));
2830 }
2831 break;
2832 }
2833
2834 default:
2835 LOG(FATAL) << "Unexpected add type " << add->GetResultType();
2836 }
2837 }
2838
VisitSub(HSub * sub)2839 void LocationsBuilderX86::VisitSub(HSub* sub) {
2840 LocationSummary* locations =
2841 new (GetGraph()->GetArena()) LocationSummary(sub, LocationSummary::kNoCall);
2842 switch (sub->GetResultType()) {
2843 case Primitive::kPrimInt:
2844 case Primitive::kPrimLong: {
2845 locations->SetInAt(0, Location::RequiresRegister());
2846 locations->SetInAt(1, Location::Any());
2847 locations->SetOut(Location::SameAsFirstInput());
2848 break;
2849 }
2850 case Primitive::kPrimFloat:
2851 case Primitive::kPrimDouble: {
2852 locations->SetInAt(0, Location::RequiresFpuRegister());
2853 if (sub->InputAt(1)->IsX86LoadFromConstantTable()) {
2854 DCHECK(sub->InputAt(1)->IsEmittedAtUseSite());
2855 } else if (sub->InputAt(1)->IsConstant()) {
2856 locations->SetInAt(1, Location::RequiresFpuRegister());
2857 } else {
2858 locations->SetInAt(1, Location::Any());
2859 }
2860 locations->SetOut(Location::SameAsFirstInput());
2861 break;
2862 }
2863
2864 default:
2865 LOG(FATAL) << "Unexpected sub type " << sub->GetResultType();
2866 }
2867 }
2868
VisitSub(HSub * sub)2869 void InstructionCodeGeneratorX86::VisitSub(HSub* sub) {
2870 LocationSummary* locations = sub->GetLocations();
2871 Location first = locations->InAt(0);
2872 Location second = locations->InAt(1);
2873 DCHECK(first.Equals(locations->Out()));
2874 switch (sub->GetResultType()) {
2875 case Primitive::kPrimInt: {
2876 if (second.IsRegister()) {
2877 __ subl(first.AsRegister<Register>(), second.AsRegister<Register>());
2878 } else if (second.IsConstant()) {
2879 __ subl(first.AsRegister<Register>(),
2880 Immediate(second.GetConstant()->AsIntConstant()->GetValue()));
2881 } else {
2882 __ subl(first.AsRegister<Register>(), Address(ESP, second.GetStackIndex()));
2883 }
2884 break;
2885 }
2886
2887 case Primitive::kPrimLong: {
2888 if (second.IsRegisterPair()) {
2889 __ subl(first.AsRegisterPairLow<Register>(), second.AsRegisterPairLow<Register>());
2890 __ sbbl(first.AsRegisterPairHigh<Register>(), second.AsRegisterPairHigh<Register>());
2891 } else if (second.IsDoubleStackSlot()) {
2892 __ subl(first.AsRegisterPairLow<Register>(), Address(ESP, second.GetStackIndex()));
2893 __ sbbl(first.AsRegisterPairHigh<Register>(),
2894 Address(ESP, second.GetHighStackIndex(kX86WordSize)));
2895 } else {
2896 DCHECK(second.IsConstant()) << second;
2897 int64_t value = second.GetConstant()->AsLongConstant()->GetValue();
2898 __ subl(first.AsRegisterPairLow<Register>(), Immediate(Low32Bits(value)));
2899 __ sbbl(first.AsRegisterPairHigh<Register>(), Immediate(High32Bits(value)));
2900 }
2901 break;
2902 }
2903
2904 case Primitive::kPrimFloat: {
2905 if (second.IsFpuRegister()) {
2906 __ subss(first.AsFpuRegister<XmmRegister>(), second.AsFpuRegister<XmmRegister>());
2907 } else if (sub->InputAt(1)->IsX86LoadFromConstantTable()) {
2908 HX86LoadFromConstantTable* const_area = sub->InputAt(1)->AsX86LoadFromConstantTable();
2909 DCHECK(const_area->IsEmittedAtUseSite());
2910 __ subss(first.AsFpuRegister<XmmRegister>(),
2911 codegen_->LiteralFloatAddress(
2912 const_area->GetConstant()->AsFloatConstant()->GetValue(),
2913 const_area->GetLocations()->InAt(0).AsRegister<Register>()));
2914 } else {
2915 DCHECK(second.IsStackSlot());
2916 __ subss(first.AsFpuRegister<XmmRegister>(), Address(ESP, second.GetStackIndex()));
2917 }
2918 break;
2919 }
2920
2921 case Primitive::kPrimDouble: {
2922 if (second.IsFpuRegister()) {
2923 __ subsd(first.AsFpuRegister<XmmRegister>(), second.AsFpuRegister<XmmRegister>());
2924 } else if (sub->InputAt(1)->IsX86LoadFromConstantTable()) {
2925 HX86LoadFromConstantTable* const_area = sub->InputAt(1)->AsX86LoadFromConstantTable();
2926 DCHECK(const_area->IsEmittedAtUseSite());
2927 __ subsd(first.AsFpuRegister<XmmRegister>(),
2928 codegen_->LiteralDoubleAddress(
2929 const_area->GetConstant()->AsDoubleConstant()->GetValue(),
2930 const_area->GetLocations()->InAt(0).AsRegister<Register>()));
2931 } else {
2932 DCHECK(second.IsDoubleStackSlot());
2933 __ subsd(first.AsFpuRegister<XmmRegister>(), Address(ESP, second.GetStackIndex()));
2934 }
2935 break;
2936 }
2937
2938 default:
2939 LOG(FATAL) << "Unexpected sub type " << sub->GetResultType();
2940 }
2941 }
2942
VisitMul(HMul * mul)2943 void LocationsBuilderX86::VisitMul(HMul* mul) {
2944 LocationSummary* locations =
2945 new (GetGraph()->GetArena()) LocationSummary(mul, LocationSummary::kNoCall);
2946 switch (mul->GetResultType()) {
2947 case Primitive::kPrimInt:
2948 locations->SetInAt(0, Location::RequiresRegister());
2949 locations->SetInAt(1, Location::Any());
2950 if (mul->InputAt(1)->IsIntConstant()) {
2951 // Can use 3 operand multiply.
2952 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
2953 } else {
2954 locations->SetOut(Location::SameAsFirstInput());
2955 }
2956 break;
2957 case Primitive::kPrimLong: {
2958 locations->SetInAt(0, Location::RequiresRegister());
2959 locations->SetInAt(1, Location::Any());
2960 locations->SetOut(Location::SameAsFirstInput());
2961 // Needed for imul on 32bits with 64bits output.
2962 locations->AddTemp(Location::RegisterLocation(EAX));
2963 locations->AddTemp(Location::RegisterLocation(EDX));
2964 break;
2965 }
2966 case Primitive::kPrimFloat:
2967 case Primitive::kPrimDouble: {
2968 locations->SetInAt(0, Location::RequiresFpuRegister());
2969 if (mul->InputAt(1)->IsX86LoadFromConstantTable()) {
2970 DCHECK(mul->InputAt(1)->IsEmittedAtUseSite());
2971 } else if (mul->InputAt(1)->IsConstant()) {
2972 locations->SetInAt(1, Location::RequiresFpuRegister());
2973 } else {
2974 locations->SetInAt(1, Location::Any());
2975 }
2976 locations->SetOut(Location::SameAsFirstInput());
2977 break;
2978 }
2979
2980 default:
2981 LOG(FATAL) << "Unexpected mul type " << mul->GetResultType();
2982 }
2983 }
2984
VisitMul(HMul * mul)2985 void InstructionCodeGeneratorX86::VisitMul(HMul* mul) {
2986 LocationSummary* locations = mul->GetLocations();
2987 Location first = locations->InAt(0);
2988 Location second = locations->InAt(1);
2989 Location out = locations->Out();
2990
2991 switch (mul->GetResultType()) {
2992 case Primitive::kPrimInt:
2993 // The constant may have ended up in a register, so test explicitly to avoid
2994 // problems where the output may not be the same as the first operand.
2995 if (mul->InputAt(1)->IsIntConstant()) {
2996 Immediate imm(mul->InputAt(1)->AsIntConstant()->GetValue());
2997 __ imull(out.AsRegister<Register>(), first.AsRegister<Register>(), imm);
2998 } else if (second.IsRegister()) {
2999 DCHECK(first.Equals(out));
3000 __ imull(first.AsRegister<Register>(), second.AsRegister<Register>());
3001 } else {
3002 DCHECK(second.IsStackSlot());
3003 DCHECK(first.Equals(out));
3004 __ imull(first.AsRegister<Register>(), Address(ESP, second.GetStackIndex()));
3005 }
3006 break;
3007
3008 case Primitive::kPrimLong: {
3009 Register in1_hi = first.AsRegisterPairHigh<Register>();
3010 Register in1_lo = first.AsRegisterPairLow<Register>();
3011 Register eax = locations->GetTemp(0).AsRegister<Register>();
3012 Register edx = locations->GetTemp(1).AsRegister<Register>();
3013
3014 DCHECK_EQ(EAX, eax);
3015 DCHECK_EQ(EDX, edx);
3016
3017 // input: in1 - 64 bits, in2 - 64 bits.
3018 // output: in1
3019 // formula: in1.hi : in1.lo = (in1.lo * in2.hi + in1.hi * in2.lo)* 2^32 + in1.lo * in2.lo
3020 // parts: in1.hi = in1.lo * in2.hi + in1.hi * in2.lo + (in1.lo * in2.lo)[63:32]
3021 // parts: in1.lo = (in1.lo * in2.lo)[31:0]
3022 if (second.IsConstant()) {
3023 DCHECK(second.GetConstant()->IsLongConstant());
3024
3025 int64_t value = second.GetConstant()->AsLongConstant()->GetValue();
3026 int32_t low_value = Low32Bits(value);
3027 int32_t high_value = High32Bits(value);
3028 Immediate low(low_value);
3029 Immediate high(high_value);
3030
3031 __ movl(eax, high);
3032 // eax <- in1.lo * in2.hi
3033 __ imull(eax, in1_lo);
3034 // in1.hi <- in1.hi * in2.lo
3035 __ imull(in1_hi, low);
3036 // in1.hi <- in1.lo * in2.hi + in1.hi * in2.lo
3037 __ addl(in1_hi, eax);
3038 // move in2_lo to eax to prepare for double precision
3039 __ movl(eax, low);
3040 // edx:eax <- in1.lo * in2.lo
3041 __ mull(in1_lo);
3042 // in1.hi <- in2.hi * in1.lo + in2.lo * in1.hi + (in1.lo * in2.lo)[63:32]
3043 __ addl(in1_hi, edx);
3044 // in1.lo <- (in1.lo * in2.lo)[31:0];
3045 __ movl(in1_lo, eax);
3046 } else if (second.IsRegisterPair()) {
3047 Register in2_hi = second.AsRegisterPairHigh<Register>();
3048 Register in2_lo = second.AsRegisterPairLow<Register>();
3049
3050 __ movl(eax, in2_hi);
3051 // eax <- in1.lo * in2.hi
3052 __ imull(eax, in1_lo);
3053 // in1.hi <- in1.hi * in2.lo
3054 __ imull(in1_hi, in2_lo);
3055 // in1.hi <- in1.lo * in2.hi + in1.hi * in2.lo
3056 __ addl(in1_hi, eax);
3057 // move in1_lo to eax to prepare for double precision
3058 __ movl(eax, in1_lo);
3059 // edx:eax <- in1.lo * in2.lo
3060 __ mull(in2_lo);
3061 // in1.hi <- in2.hi * in1.lo + in2.lo * in1.hi + (in1.lo * in2.lo)[63:32]
3062 __ addl(in1_hi, edx);
3063 // in1.lo <- (in1.lo * in2.lo)[31:0];
3064 __ movl(in1_lo, eax);
3065 } else {
3066 DCHECK(second.IsDoubleStackSlot()) << second;
3067 Address in2_hi(ESP, second.GetHighStackIndex(kX86WordSize));
3068 Address in2_lo(ESP, second.GetStackIndex());
3069
3070 __ movl(eax, in2_hi);
3071 // eax <- in1.lo * in2.hi
3072 __ imull(eax, in1_lo);
3073 // in1.hi <- in1.hi * in2.lo
3074 __ imull(in1_hi, in2_lo);
3075 // in1.hi <- in1.lo * in2.hi + in1.hi * in2.lo
3076 __ addl(in1_hi, eax);
3077 // move in1_lo to eax to prepare for double precision
3078 __ movl(eax, in1_lo);
3079 // edx:eax <- in1.lo * in2.lo
3080 __ mull(in2_lo);
3081 // in1.hi <- in2.hi * in1.lo + in2.lo * in1.hi + (in1.lo * in2.lo)[63:32]
3082 __ addl(in1_hi, edx);
3083 // in1.lo <- (in1.lo * in2.lo)[31:0];
3084 __ movl(in1_lo, eax);
3085 }
3086
3087 break;
3088 }
3089
3090 case Primitive::kPrimFloat: {
3091 DCHECK(first.Equals(locations->Out()));
3092 if (second.IsFpuRegister()) {
3093 __ mulss(first.AsFpuRegister<XmmRegister>(), second.AsFpuRegister<XmmRegister>());
3094 } else if (mul->InputAt(1)->IsX86LoadFromConstantTable()) {
3095 HX86LoadFromConstantTable* const_area = mul->InputAt(1)->AsX86LoadFromConstantTable();
3096 DCHECK(const_area->IsEmittedAtUseSite());
3097 __ mulss(first.AsFpuRegister<XmmRegister>(),
3098 codegen_->LiteralFloatAddress(
3099 const_area->GetConstant()->AsFloatConstant()->GetValue(),
3100 const_area->GetLocations()->InAt(0).AsRegister<Register>()));
3101 } else {
3102 DCHECK(second.IsStackSlot());
3103 __ mulss(first.AsFpuRegister<XmmRegister>(), Address(ESP, second.GetStackIndex()));
3104 }
3105 break;
3106 }
3107
3108 case Primitive::kPrimDouble: {
3109 DCHECK(first.Equals(locations->Out()));
3110 if (second.IsFpuRegister()) {
3111 __ mulsd(first.AsFpuRegister<XmmRegister>(), second.AsFpuRegister<XmmRegister>());
3112 } else if (mul->InputAt(1)->IsX86LoadFromConstantTable()) {
3113 HX86LoadFromConstantTable* const_area = mul->InputAt(1)->AsX86LoadFromConstantTable();
3114 DCHECK(const_area->IsEmittedAtUseSite());
3115 __ mulsd(first.AsFpuRegister<XmmRegister>(),
3116 codegen_->LiteralDoubleAddress(
3117 const_area->GetConstant()->AsDoubleConstant()->GetValue(),
3118 const_area->GetLocations()->InAt(0).AsRegister<Register>()));
3119 } else {
3120 DCHECK(second.IsDoubleStackSlot());
3121 __ mulsd(first.AsFpuRegister<XmmRegister>(), Address(ESP, second.GetStackIndex()));
3122 }
3123 break;
3124 }
3125
3126 default:
3127 LOG(FATAL) << "Unexpected mul type " << mul->GetResultType();
3128 }
3129 }
3130
PushOntoFPStack(Location source,uint32_t temp_offset,uint32_t stack_adjustment,bool is_fp,bool is_wide)3131 void InstructionCodeGeneratorX86::PushOntoFPStack(Location source,
3132 uint32_t temp_offset,
3133 uint32_t stack_adjustment,
3134 bool is_fp,
3135 bool is_wide) {
3136 if (source.IsStackSlot()) {
3137 DCHECK(!is_wide);
3138 if (is_fp) {
3139 __ flds(Address(ESP, source.GetStackIndex() + stack_adjustment));
3140 } else {
3141 __ filds(Address(ESP, source.GetStackIndex() + stack_adjustment));
3142 }
3143 } else if (source.IsDoubleStackSlot()) {
3144 DCHECK(is_wide);
3145 if (is_fp) {
3146 __ fldl(Address(ESP, source.GetStackIndex() + stack_adjustment));
3147 } else {
3148 __ fildl(Address(ESP, source.GetStackIndex() + stack_adjustment));
3149 }
3150 } else {
3151 // Write the value to the temporary location on the stack and load to FP stack.
3152 if (!is_wide) {
3153 Location stack_temp = Location::StackSlot(temp_offset);
3154 codegen_->Move32(stack_temp, source);
3155 if (is_fp) {
3156 __ flds(Address(ESP, temp_offset));
3157 } else {
3158 __ filds(Address(ESP, temp_offset));
3159 }
3160 } else {
3161 Location stack_temp = Location::DoubleStackSlot(temp_offset);
3162 codegen_->Move64(stack_temp, source);
3163 if (is_fp) {
3164 __ fldl(Address(ESP, temp_offset));
3165 } else {
3166 __ fildl(Address(ESP, temp_offset));
3167 }
3168 }
3169 }
3170 }
3171
GenerateRemFP(HRem * rem)3172 void InstructionCodeGeneratorX86::GenerateRemFP(HRem *rem) {
3173 Primitive::Type type = rem->GetResultType();
3174 bool is_float = type == Primitive::kPrimFloat;
3175 size_t elem_size = Primitive::ComponentSize(type);
3176 LocationSummary* locations = rem->GetLocations();
3177 Location first = locations->InAt(0);
3178 Location second = locations->InAt(1);
3179 Location out = locations->Out();
3180
3181 // Create stack space for 2 elements.
3182 // TODO: enhance register allocator to ask for stack temporaries.
3183 __ subl(ESP, Immediate(2 * elem_size));
3184
3185 // Load the values to the FP stack in reverse order, using temporaries if needed.
3186 const bool is_wide = !is_float;
3187 PushOntoFPStack(second, elem_size, 2 * elem_size, /* is_fp */ true, is_wide);
3188 PushOntoFPStack(first, 0, 2 * elem_size, /* is_fp */ true, is_wide);
3189
3190 // Loop doing FPREM until we stabilize.
3191 NearLabel retry;
3192 __ Bind(&retry);
3193 __ fprem();
3194
3195 // Move FP status to AX.
3196 __ fstsw();
3197
3198 // And see if the argument reduction is complete. This is signaled by the
3199 // C2 FPU flag bit set to 0.
3200 __ andl(EAX, Immediate(kC2ConditionMask));
3201 __ j(kNotEqual, &retry);
3202
3203 // We have settled on the final value. Retrieve it into an XMM register.
3204 // Store FP top of stack to real stack.
3205 if (is_float) {
3206 __ fsts(Address(ESP, 0));
3207 } else {
3208 __ fstl(Address(ESP, 0));
3209 }
3210
3211 // Pop the 2 items from the FP stack.
3212 __ fucompp();
3213
3214 // Load the value from the stack into an XMM register.
3215 DCHECK(out.IsFpuRegister()) << out;
3216 if (is_float) {
3217 __ movss(out.AsFpuRegister<XmmRegister>(), Address(ESP, 0));
3218 } else {
3219 __ movsd(out.AsFpuRegister<XmmRegister>(), Address(ESP, 0));
3220 }
3221
3222 // And remove the temporary stack space we allocated.
3223 __ addl(ESP, Immediate(2 * elem_size));
3224 }
3225
3226
DivRemOneOrMinusOne(HBinaryOperation * instruction)3227 void InstructionCodeGeneratorX86::DivRemOneOrMinusOne(HBinaryOperation* instruction) {
3228 DCHECK(instruction->IsDiv() || instruction->IsRem());
3229
3230 LocationSummary* locations = instruction->GetLocations();
3231 DCHECK(locations->InAt(1).IsConstant());
3232 DCHECK(locations->InAt(1).GetConstant()->IsIntConstant());
3233
3234 Register out_register = locations->Out().AsRegister<Register>();
3235 Register input_register = locations->InAt(0).AsRegister<Register>();
3236 int32_t imm = locations->InAt(1).GetConstant()->AsIntConstant()->GetValue();
3237
3238 DCHECK(imm == 1 || imm == -1);
3239
3240 if (instruction->IsRem()) {
3241 __ xorl(out_register, out_register);
3242 } else {
3243 __ movl(out_register, input_register);
3244 if (imm == -1) {
3245 __ negl(out_register);
3246 }
3247 }
3248 }
3249
3250
DivByPowerOfTwo(HDiv * instruction)3251 void InstructionCodeGeneratorX86::DivByPowerOfTwo(HDiv* instruction) {
3252 LocationSummary* locations = instruction->GetLocations();
3253
3254 Register out_register = locations->Out().AsRegister<Register>();
3255 Register input_register = locations->InAt(0).AsRegister<Register>();
3256 int32_t imm = locations->InAt(1).GetConstant()->AsIntConstant()->GetValue();
3257 DCHECK(IsPowerOfTwo(AbsOrMin(imm)));
3258 uint32_t abs_imm = static_cast<uint32_t>(AbsOrMin(imm));
3259
3260 Register num = locations->GetTemp(0).AsRegister<Register>();
3261
3262 __ leal(num, Address(input_register, abs_imm - 1));
3263 __ testl(input_register, input_register);
3264 __ cmovl(kGreaterEqual, num, input_register);
3265 int shift = CTZ(imm);
3266 __ sarl(num, Immediate(shift));
3267
3268 if (imm < 0) {
3269 __ negl(num);
3270 }
3271
3272 __ movl(out_register, num);
3273 }
3274
GenerateDivRemWithAnyConstant(HBinaryOperation * instruction)3275 void InstructionCodeGeneratorX86::GenerateDivRemWithAnyConstant(HBinaryOperation* instruction) {
3276 DCHECK(instruction->IsDiv() || instruction->IsRem());
3277
3278 LocationSummary* locations = instruction->GetLocations();
3279 int imm = locations->InAt(1).GetConstant()->AsIntConstant()->GetValue();
3280
3281 Register eax = locations->InAt(0).AsRegister<Register>();
3282 Register out = locations->Out().AsRegister<Register>();
3283 Register num;
3284 Register edx;
3285
3286 if (instruction->IsDiv()) {
3287 edx = locations->GetTemp(0).AsRegister<Register>();
3288 num = locations->GetTemp(1).AsRegister<Register>();
3289 } else {
3290 edx = locations->Out().AsRegister<Register>();
3291 num = locations->GetTemp(0).AsRegister<Register>();
3292 }
3293
3294 DCHECK_EQ(EAX, eax);
3295 DCHECK_EQ(EDX, edx);
3296 if (instruction->IsDiv()) {
3297 DCHECK_EQ(EAX, out);
3298 } else {
3299 DCHECK_EQ(EDX, out);
3300 }
3301
3302 int64_t magic;
3303 int shift;
3304 CalculateMagicAndShiftForDivRem(imm, false /* is_long */, &magic, &shift);
3305
3306 NearLabel ndiv;
3307 NearLabel end;
3308 // If numerator is 0, the result is 0, no computation needed.
3309 __ testl(eax, eax);
3310 __ j(kNotEqual, &ndiv);
3311
3312 __ xorl(out, out);
3313 __ jmp(&end);
3314
3315 __ Bind(&ndiv);
3316
3317 // Save the numerator.
3318 __ movl(num, eax);
3319
3320 // EAX = magic
3321 __ movl(eax, Immediate(magic));
3322
3323 // EDX:EAX = magic * numerator
3324 __ imull(num);
3325
3326 if (imm > 0 && magic < 0) {
3327 // EDX += num
3328 __ addl(edx, num);
3329 } else if (imm < 0 && magic > 0) {
3330 __ subl(edx, num);
3331 }
3332
3333 // Shift if needed.
3334 if (shift != 0) {
3335 __ sarl(edx, Immediate(shift));
3336 }
3337
3338 // EDX += 1 if EDX < 0
3339 __ movl(eax, edx);
3340 __ shrl(edx, Immediate(31));
3341 __ addl(edx, eax);
3342
3343 if (instruction->IsRem()) {
3344 __ movl(eax, num);
3345 __ imull(edx, Immediate(imm));
3346 __ subl(eax, edx);
3347 __ movl(edx, eax);
3348 } else {
3349 __ movl(eax, edx);
3350 }
3351 __ Bind(&end);
3352 }
3353
GenerateDivRemIntegral(HBinaryOperation * instruction)3354 void InstructionCodeGeneratorX86::GenerateDivRemIntegral(HBinaryOperation* instruction) {
3355 DCHECK(instruction->IsDiv() || instruction->IsRem());
3356
3357 LocationSummary* locations = instruction->GetLocations();
3358 Location out = locations->Out();
3359 Location first = locations->InAt(0);
3360 Location second = locations->InAt(1);
3361 bool is_div = instruction->IsDiv();
3362
3363 switch (instruction->GetResultType()) {
3364 case Primitive::kPrimInt: {
3365 DCHECK_EQ(EAX, first.AsRegister<Register>());
3366 DCHECK_EQ(is_div ? EAX : EDX, out.AsRegister<Register>());
3367
3368 if (second.IsConstant()) {
3369 int32_t imm = second.GetConstant()->AsIntConstant()->GetValue();
3370
3371 if (imm == 0) {
3372 // Do not generate anything for 0. DivZeroCheck would forbid any generated code.
3373 } else if (imm == 1 || imm == -1) {
3374 DivRemOneOrMinusOne(instruction);
3375 } else if (is_div && IsPowerOfTwo(AbsOrMin(imm))) {
3376 DivByPowerOfTwo(instruction->AsDiv());
3377 } else {
3378 DCHECK(imm <= -2 || imm >= 2);
3379 GenerateDivRemWithAnyConstant(instruction);
3380 }
3381 } else {
3382 SlowPathCode* slow_path = new (GetGraph()->GetArena()) DivRemMinusOneSlowPathX86(
3383 instruction, out.AsRegister<Register>(), is_div);
3384 codegen_->AddSlowPath(slow_path);
3385
3386 Register second_reg = second.AsRegister<Register>();
3387 // 0x80000000/-1 triggers an arithmetic exception!
3388 // Dividing by -1 is actually negation and -0x800000000 = 0x80000000 so
3389 // it's safe to just use negl instead of more complex comparisons.
3390
3391 __ cmpl(second_reg, Immediate(-1));
3392 __ j(kEqual, slow_path->GetEntryLabel());
3393
3394 // edx:eax <- sign-extended of eax
3395 __ cdq();
3396 // eax = quotient, edx = remainder
3397 __ idivl(second_reg);
3398 __ Bind(slow_path->GetExitLabel());
3399 }
3400 break;
3401 }
3402
3403 case Primitive::kPrimLong: {
3404 InvokeRuntimeCallingConvention calling_convention;
3405 DCHECK_EQ(calling_convention.GetRegisterAt(0), first.AsRegisterPairLow<Register>());
3406 DCHECK_EQ(calling_convention.GetRegisterAt(1), first.AsRegisterPairHigh<Register>());
3407 DCHECK_EQ(calling_convention.GetRegisterAt(2), second.AsRegisterPairLow<Register>());
3408 DCHECK_EQ(calling_convention.GetRegisterAt(3), second.AsRegisterPairHigh<Register>());
3409 DCHECK_EQ(EAX, out.AsRegisterPairLow<Register>());
3410 DCHECK_EQ(EDX, out.AsRegisterPairHigh<Register>());
3411
3412 if (is_div) {
3413 codegen_->InvokeRuntime(QUICK_ENTRY_POINT(pLdiv),
3414 instruction,
3415 instruction->GetDexPc(),
3416 nullptr);
3417 CheckEntrypointTypes<kQuickLdiv, int64_t, int64_t, int64_t>();
3418 } else {
3419 codegen_->InvokeRuntime(QUICK_ENTRY_POINT(pLmod),
3420 instruction,
3421 instruction->GetDexPc(),
3422 nullptr);
3423 CheckEntrypointTypes<kQuickLmod, int64_t, int64_t, int64_t>();
3424 }
3425 break;
3426 }
3427
3428 default:
3429 LOG(FATAL) << "Unexpected type for GenerateDivRemIntegral " << instruction->GetResultType();
3430 }
3431 }
3432
VisitDiv(HDiv * div)3433 void LocationsBuilderX86::VisitDiv(HDiv* div) {
3434 LocationSummary::CallKind call_kind = (div->GetResultType() == Primitive::kPrimLong)
3435 ? LocationSummary::kCall
3436 : LocationSummary::kNoCall;
3437 LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(div, call_kind);
3438
3439 switch (div->GetResultType()) {
3440 case Primitive::kPrimInt: {
3441 locations->SetInAt(0, Location::RegisterLocation(EAX));
3442 locations->SetInAt(1, Location::RegisterOrConstant(div->InputAt(1)));
3443 locations->SetOut(Location::SameAsFirstInput());
3444 // Intel uses edx:eax as the dividend.
3445 locations->AddTemp(Location::RegisterLocation(EDX));
3446 // We need to save the numerator while we tweak eax and edx. As we are using imul in a way
3447 // which enforces results to be in EAX and EDX, things are simpler if we use EAX also as
3448 // output and request another temp.
3449 if (div->InputAt(1)->IsIntConstant()) {
3450 locations->AddTemp(Location::RequiresRegister());
3451 }
3452 break;
3453 }
3454 case Primitive::kPrimLong: {
3455 InvokeRuntimeCallingConvention calling_convention;
3456 locations->SetInAt(0, Location::RegisterPairLocation(
3457 calling_convention.GetRegisterAt(0), calling_convention.GetRegisterAt(1)));
3458 locations->SetInAt(1, Location::RegisterPairLocation(
3459 calling_convention.GetRegisterAt(2), calling_convention.GetRegisterAt(3)));
3460 // Runtime helper puts the result in EAX, EDX.
3461 locations->SetOut(Location::RegisterPairLocation(EAX, EDX));
3462 break;
3463 }
3464 case Primitive::kPrimFloat:
3465 case Primitive::kPrimDouble: {
3466 locations->SetInAt(0, Location::RequiresFpuRegister());
3467 if (div->InputAt(1)->IsX86LoadFromConstantTable()) {
3468 DCHECK(div->InputAt(1)->IsEmittedAtUseSite());
3469 } else if (div->InputAt(1)->IsConstant()) {
3470 locations->SetInAt(1, Location::RequiresFpuRegister());
3471 } else {
3472 locations->SetInAt(1, Location::Any());
3473 }
3474 locations->SetOut(Location::SameAsFirstInput());
3475 break;
3476 }
3477
3478 default:
3479 LOG(FATAL) << "Unexpected div type " << div->GetResultType();
3480 }
3481 }
3482
VisitDiv(HDiv * div)3483 void InstructionCodeGeneratorX86::VisitDiv(HDiv* div) {
3484 LocationSummary* locations = div->GetLocations();
3485 Location first = locations->InAt(0);
3486 Location second = locations->InAt(1);
3487
3488 switch (div->GetResultType()) {
3489 case Primitive::kPrimInt:
3490 case Primitive::kPrimLong: {
3491 GenerateDivRemIntegral(div);
3492 break;
3493 }
3494
3495 case Primitive::kPrimFloat: {
3496 if (second.IsFpuRegister()) {
3497 __ divss(first.AsFpuRegister<XmmRegister>(), second.AsFpuRegister<XmmRegister>());
3498 } else if (div->InputAt(1)->IsX86LoadFromConstantTable()) {
3499 HX86LoadFromConstantTable* const_area = div->InputAt(1)->AsX86LoadFromConstantTable();
3500 DCHECK(const_area->IsEmittedAtUseSite());
3501 __ divss(first.AsFpuRegister<XmmRegister>(),
3502 codegen_->LiteralFloatAddress(
3503 const_area->GetConstant()->AsFloatConstant()->GetValue(),
3504 const_area->GetLocations()->InAt(0).AsRegister<Register>()));
3505 } else {
3506 DCHECK(second.IsStackSlot());
3507 __ divss(first.AsFpuRegister<XmmRegister>(), Address(ESP, second.GetStackIndex()));
3508 }
3509 break;
3510 }
3511
3512 case Primitive::kPrimDouble: {
3513 if (second.IsFpuRegister()) {
3514 __ divsd(first.AsFpuRegister<XmmRegister>(), second.AsFpuRegister<XmmRegister>());
3515 } else if (div->InputAt(1)->IsX86LoadFromConstantTable()) {
3516 HX86LoadFromConstantTable* const_area = div->InputAt(1)->AsX86LoadFromConstantTable();
3517 DCHECK(const_area->IsEmittedAtUseSite());
3518 __ divsd(first.AsFpuRegister<XmmRegister>(),
3519 codegen_->LiteralDoubleAddress(
3520 const_area->GetConstant()->AsDoubleConstant()->GetValue(),
3521 const_area->GetLocations()->InAt(0).AsRegister<Register>()));
3522 } else {
3523 DCHECK(second.IsDoubleStackSlot());
3524 __ divsd(first.AsFpuRegister<XmmRegister>(), Address(ESP, second.GetStackIndex()));
3525 }
3526 break;
3527 }
3528
3529 default:
3530 LOG(FATAL) << "Unexpected div type " << div->GetResultType();
3531 }
3532 }
3533
VisitRem(HRem * rem)3534 void LocationsBuilderX86::VisitRem(HRem* rem) {
3535 Primitive::Type type = rem->GetResultType();
3536
3537 LocationSummary::CallKind call_kind = (rem->GetResultType() == Primitive::kPrimLong)
3538 ? LocationSummary::kCall
3539 : LocationSummary::kNoCall;
3540 LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(rem, call_kind);
3541
3542 switch (type) {
3543 case Primitive::kPrimInt: {
3544 locations->SetInAt(0, Location::RegisterLocation(EAX));
3545 locations->SetInAt(1, Location::RegisterOrConstant(rem->InputAt(1)));
3546 locations->SetOut(Location::RegisterLocation(EDX));
3547 // We need to save the numerator while we tweak eax and edx. As we are using imul in a way
3548 // which enforces results to be in EAX and EDX, things are simpler if we use EDX also as
3549 // output and request another temp.
3550 if (rem->InputAt(1)->IsIntConstant()) {
3551 locations->AddTemp(Location::RequiresRegister());
3552 }
3553 break;
3554 }
3555 case Primitive::kPrimLong: {
3556 InvokeRuntimeCallingConvention calling_convention;
3557 locations->SetInAt(0, Location::RegisterPairLocation(
3558 calling_convention.GetRegisterAt(0), calling_convention.GetRegisterAt(1)));
3559 locations->SetInAt(1, Location::RegisterPairLocation(
3560 calling_convention.GetRegisterAt(2), calling_convention.GetRegisterAt(3)));
3561 // Runtime helper puts the result in EAX, EDX.
3562 locations->SetOut(Location::RegisterPairLocation(EAX, EDX));
3563 break;
3564 }
3565 case Primitive::kPrimDouble:
3566 case Primitive::kPrimFloat: {
3567 locations->SetInAt(0, Location::Any());
3568 locations->SetInAt(1, Location::Any());
3569 locations->SetOut(Location::RequiresFpuRegister());
3570 locations->AddTemp(Location::RegisterLocation(EAX));
3571 break;
3572 }
3573
3574 default:
3575 LOG(FATAL) << "Unexpected rem type " << type;
3576 }
3577 }
3578
VisitRem(HRem * rem)3579 void InstructionCodeGeneratorX86::VisitRem(HRem* rem) {
3580 Primitive::Type type = rem->GetResultType();
3581 switch (type) {
3582 case Primitive::kPrimInt:
3583 case Primitive::kPrimLong: {
3584 GenerateDivRemIntegral(rem);
3585 break;
3586 }
3587 case Primitive::kPrimFloat:
3588 case Primitive::kPrimDouble: {
3589 GenerateRemFP(rem);
3590 break;
3591 }
3592 default:
3593 LOG(FATAL) << "Unexpected rem type " << type;
3594 }
3595 }
3596
VisitDivZeroCheck(HDivZeroCheck * instruction)3597 void LocationsBuilderX86::VisitDivZeroCheck(HDivZeroCheck* instruction) {
3598 LocationSummary::CallKind call_kind = instruction->CanThrowIntoCatchBlock()
3599 ? LocationSummary::kCallOnSlowPath
3600 : LocationSummary::kNoCall;
3601 LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction, call_kind);
3602 switch (instruction->GetType()) {
3603 case Primitive::kPrimBoolean:
3604 case Primitive::kPrimByte:
3605 case Primitive::kPrimChar:
3606 case Primitive::kPrimShort:
3607 case Primitive::kPrimInt: {
3608 locations->SetInAt(0, Location::Any());
3609 break;
3610 }
3611 case Primitive::kPrimLong: {
3612 locations->SetInAt(0, Location::RegisterOrConstant(instruction->InputAt(0)));
3613 if (!instruction->IsConstant()) {
3614 locations->AddTemp(Location::RequiresRegister());
3615 }
3616 break;
3617 }
3618 default:
3619 LOG(FATAL) << "Unexpected type for HDivZeroCheck " << instruction->GetType();
3620 }
3621 if (instruction->HasUses()) {
3622 locations->SetOut(Location::SameAsFirstInput());
3623 }
3624 }
3625
VisitDivZeroCheck(HDivZeroCheck * instruction)3626 void InstructionCodeGeneratorX86::VisitDivZeroCheck(HDivZeroCheck* instruction) {
3627 SlowPathCode* slow_path = new (GetGraph()->GetArena()) DivZeroCheckSlowPathX86(instruction);
3628 codegen_->AddSlowPath(slow_path);
3629
3630 LocationSummary* locations = instruction->GetLocations();
3631 Location value = locations->InAt(0);
3632
3633 switch (instruction->GetType()) {
3634 case Primitive::kPrimBoolean:
3635 case Primitive::kPrimByte:
3636 case Primitive::kPrimChar:
3637 case Primitive::kPrimShort:
3638 case Primitive::kPrimInt: {
3639 if (value.IsRegister()) {
3640 __ testl(value.AsRegister<Register>(), value.AsRegister<Register>());
3641 __ j(kEqual, slow_path->GetEntryLabel());
3642 } else if (value.IsStackSlot()) {
3643 __ cmpl(Address(ESP, value.GetStackIndex()), Immediate(0));
3644 __ j(kEqual, slow_path->GetEntryLabel());
3645 } else {
3646 DCHECK(value.IsConstant()) << value;
3647 if (value.GetConstant()->AsIntConstant()->GetValue() == 0) {
3648 __ jmp(slow_path->GetEntryLabel());
3649 }
3650 }
3651 break;
3652 }
3653 case Primitive::kPrimLong: {
3654 if (value.IsRegisterPair()) {
3655 Register temp = locations->GetTemp(0).AsRegister<Register>();
3656 __ movl(temp, value.AsRegisterPairLow<Register>());
3657 __ orl(temp, value.AsRegisterPairHigh<Register>());
3658 __ j(kEqual, slow_path->GetEntryLabel());
3659 } else {
3660 DCHECK(value.IsConstant()) << value;
3661 if (value.GetConstant()->AsLongConstant()->GetValue() == 0) {
3662 __ jmp(slow_path->GetEntryLabel());
3663 }
3664 }
3665 break;
3666 }
3667 default:
3668 LOG(FATAL) << "Unexpected type for HDivZeroCheck" << instruction->GetType();
3669 }
3670 }
3671
HandleShift(HBinaryOperation * op)3672 void LocationsBuilderX86::HandleShift(HBinaryOperation* op) {
3673 DCHECK(op->IsShl() || op->IsShr() || op->IsUShr());
3674
3675 LocationSummary* locations =
3676 new (GetGraph()->GetArena()) LocationSummary(op, LocationSummary::kNoCall);
3677
3678 switch (op->GetResultType()) {
3679 case Primitive::kPrimInt:
3680 case Primitive::kPrimLong: {
3681 // Can't have Location::Any() and output SameAsFirstInput()
3682 locations->SetInAt(0, Location::RequiresRegister());
3683 // The shift count needs to be in CL or a constant.
3684 locations->SetInAt(1, Location::ByteRegisterOrConstant(ECX, op->InputAt(1)));
3685 locations->SetOut(Location::SameAsFirstInput());
3686 break;
3687 }
3688 default:
3689 LOG(FATAL) << "Unexpected op type " << op->GetResultType();
3690 }
3691 }
3692
HandleShift(HBinaryOperation * op)3693 void InstructionCodeGeneratorX86::HandleShift(HBinaryOperation* op) {
3694 DCHECK(op->IsShl() || op->IsShr() || op->IsUShr());
3695
3696 LocationSummary* locations = op->GetLocations();
3697 Location first = locations->InAt(0);
3698 Location second = locations->InAt(1);
3699 DCHECK(first.Equals(locations->Out()));
3700
3701 switch (op->GetResultType()) {
3702 case Primitive::kPrimInt: {
3703 DCHECK(first.IsRegister());
3704 Register first_reg = first.AsRegister<Register>();
3705 if (second.IsRegister()) {
3706 Register second_reg = second.AsRegister<Register>();
3707 DCHECK_EQ(ECX, second_reg);
3708 if (op->IsShl()) {
3709 __ shll(first_reg, second_reg);
3710 } else if (op->IsShr()) {
3711 __ sarl(first_reg, second_reg);
3712 } else {
3713 __ shrl(first_reg, second_reg);
3714 }
3715 } else {
3716 int32_t shift = second.GetConstant()->AsIntConstant()->GetValue() & kMaxIntShiftDistance;
3717 if (shift == 0) {
3718 return;
3719 }
3720 Immediate imm(shift);
3721 if (op->IsShl()) {
3722 __ shll(first_reg, imm);
3723 } else if (op->IsShr()) {
3724 __ sarl(first_reg, imm);
3725 } else {
3726 __ shrl(first_reg, imm);
3727 }
3728 }
3729 break;
3730 }
3731 case Primitive::kPrimLong: {
3732 if (second.IsRegister()) {
3733 Register second_reg = second.AsRegister<Register>();
3734 DCHECK_EQ(ECX, second_reg);
3735 if (op->IsShl()) {
3736 GenerateShlLong(first, second_reg);
3737 } else if (op->IsShr()) {
3738 GenerateShrLong(first, second_reg);
3739 } else {
3740 GenerateUShrLong(first, second_reg);
3741 }
3742 } else {
3743 // Shift by a constant.
3744 int32_t shift = second.GetConstant()->AsIntConstant()->GetValue() & kMaxLongShiftDistance;
3745 // Nothing to do if the shift is 0, as the input is already the output.
3746 if (shift != 0) {
3747 if (op->IsShl()) {
3748 GenerateShlLong(first, shift);
3749 } else if (op->IsShr()) {
3750 GenerateShrLong(first, shift);
3751 } else {
3752 GenerateUShrLong(first, shift);
3753 }
3754 }
3755 }
3756 break;
3757 }
3758 default:
3759 LOG(FATAL) << "Unexpected op type " << op->GetResultType();
3760 }
3761 }
3762
GenerateShlLong(const Location & loc,int shift)3763 void InstructionCodeGeneratorX86::GenerateShlLong(const Location& loc, int shift) {
3764 Register low = loc.AsRegisterPairLow<Register>();
3765 Register high = loc.AsRegisterPairHigh<Register>();
3766 if (shift == 1) {
3767 // This is just an addition.
3768 __ addl(low, low);
3769 __ adcl(high, high);
3770 } else if (shift == 32) {
3771 // Shift by 32 is easy. High gets low, and low gets 0.
3772 codegen_->EmitParallelMoves(
3773 loc.ToLow(),
3774 loc.ToHigh(),
3775 Primitive::kPrimInt,
3776 Location::ConstantLocation(GetGraph()->GetIntConstant(0)),
3777 loc.ToLow(),
3778 Primitive::kPrimInt);
3779 } else if (shift > 32) {
3780 // Low part becomes 0. High part is low part << (shift-32).
3781 __ movl(high, low);
3782 __ shll(high, Immediate(shift - 32));
3783 __ xorl(low, low);
3784 } else {
3785 // Between 1 and 31.
3786 __ shld(high, low, Immediate(shift));
3787 __ shll(low, Immediate(shift));
3788 }
3789 }
3790
GenerateShlLong(const Location & loc,Register shifter)3791 void InstructionCodeGeneratorX86::GenerateShlLong(const Location& loc, Register shifter) {
3792 NearLabel done;
3793 __ shld(loc.AsRegisterPairHigh<Register>(), loc.AsRegisterPairLow<Register>(), shifter);
3794 __ shll(loc.AsRegisterPairLow<Register>(), shifter);
3795 __ testl(shifter, Immediate(32));
3796 __ j(kEqual, &done);
3797 __ movl(loc.AsRegisterPairHigh<Register>(), loc.AsRegisterPairLow<Register>());
3798 __ movl(loc.AsRegisterPairLow<Register>(), Immediate(0));
3799 __ Bind(&done);
3800 }
3801
GenerateShrLong(const Location & loc,int shift)3802 void InstructionCodeGeneratorX86::GenerateShrLong(const Location& loc, int shift) {
3803 Register low = loc.AsRegisterPairLow<Register>();
3804 Register high = loc.AsRegisterPairHigh<Register>();
3805 if (shift == 32) {
3806 // Need to copy the sign.
3807 DCHECK_NE(low, high);
3808 __ movl(low, high);
3809 __ sarl(high, Immediate(31));
3810 } else if (shift > 32) {
3811 DCHECK_NE(low, high);
3812 // High part becomes sign. Low part is shifted by shift - 32.
3813 __ movl(low, high);
3814 __ sarl(high, Immediate(31));
3815 __ sarl(low, Immediate(shift - 32));
3816 } else {
3817 // Between 1 and 31.
3818 __ shrd(low, high, Immediate(shift));
3819 __ sarl(high, Immediate(shift));
3820 }
3821 }
3822
GenerateShrLong(const Location & loc,Register shifter)3823 void InstructionCodeGeneratorX86::GenerateShrLong(const Location& loc, Register shifter) {
3824 NearLabel done;
3825 __ shrd(loc.AsRegisterPairLow<Register>(), loc.AsRegisterPairHigh<Register>(), shifter);
3826 __ sarl(loc.AsRegisterPairHigh<Register>(), shifter);
3827 __ testl(shifter, Immediate(32));
3828 __ j(kEqual, &done);
3829 __ movl(loc.AsRegisterPairLow<Register>(), loc.AsRegisterPairHigh<Register>());
3830 __ sarl(loc.AsRegisterPairHigh<Register>(), Immediate(31));
3831 __ Bind(&done);
3832 }
3833
GenerateUShrLong(const Location & loc,int shift)3834 void InstructionCodeGeneratorX86::GenerateUShrLong(const Location& loc, int shift) {
3835 Register low = loc.AsRegisterPairLow<Register>();
3836 Register high = loc.AsRegisterPairHigh<Register>();
3837 if (shift == 32) {
3838 // Shift by 32 is easy. Low gets high, and high gets 0.
3839 codegen_->EmitParallelMoves(
3840 loc.ToHigh(),
3841 loc.ToLow(),
3842 Primitive::kPrimInt,
3843 Location::ConstantLocation(GetGraph()->GetIntConstant(0)),
3844 loc.ToHigh(),
3845 Primitive::kPrimInt);
3846 } else if (shift > 32) {
3847 // Low part is high >> (shift - 32). High part becomes 0.
3848 __ movl(low, high);
3849 __ shrl(low, Immediate(shift - 32));
3850 __ xorl(high, high);
3851 } else {
3852 // Between 1 and 31.
3853 __ shrd(low, high, Immediate(shift));
3854 __ shrl(high, Immediate(shift));
3855 }
3856 }
3857
GenerateUShrLong(const Location & loc,Register shifter)3858 void InstructionCodeGeneratorX86::GenerateUShrLong(const Location& loc, Register shifter) {
3859 NearLabel done;
3860 __ shrd(loc.AsRegisterPairLow<Register>(), loc.AsRegisterPairHigh<Register>(), shifter);
3861 __ shrl(loc.AsRegisterPairHigh<Register>(), shifter);
3862 __ testl(shifter, Immediate(32));
3863 __ j(kEqual, &done);
3864 __ movl(loc.AsRegisterPairLow<Register>(), loc.AsRegisterPairHigh<Register>());
3865 __ movl(loc.AsRegisterPairHigh<Register>(), Immediate(0));
3866 __ Bind(&done);
3867 }
3868
VisitRor(HRor * ror)3869 void LocationsBuilderX86::VisitRor(HRor* ror) {
3870 LocationSummary* locations =
3871 new (GetGraph()->GetArena()) LocationSummary(ror, LocationSummary::kNoCall);
3872
3873 switch (ror->GetResultType()) {
3874 case Primitive::kPrimLong:
3875 // Add the temporary needed.
3876 locations->AddTemp(Location::RequiresRegister());
3877 FALLTHROUGH_INTENDED;
3878 case Primitive::kPrimInt:
3879 locations->SetInAt(0, Location::RequiresRegister());
3880 // The shift count needs to be in CL (unless it is a constant).
3881 locations->SetInAt(1, Location::ByteRegisterOrConstant(ECX, ror->InputAt(1)));
3882 locations->SetOut(Location::SameAsFirstInput());
3883 break;
3884 default:
3885 LOG(FATAL) << "Unexpected operation type " << ror->GetResultType();
3886 UNREACHABLE();
3887 }
3888 }
3889
VisitRor(HRor * ror)3890 void InstructionCodeGeneratorX86::VisitRor(HRor* ror) {
3891 LocationSummary* locations = ror->GetLocations();
3892 Location first = locations->InAt(0);
3893 Location second = locations->InAt(1);
3894
3895 if (ror->GetResultType() == Primitive::kPrimInt) {
3896 Register first_reg = first.AsRegister<Register>();
3897 if (second.IsRegister()) {
3898 Register second_reg = second.AsRegister<Register>();
3899 __ rorl(first_reg, second_reg);
3900 } else {
3901 Immediate imm(second.GetConstant()->AsIntConstant()->GetValue() & kMaxIntShiftDistance);
3902 __ rorl(first_reg, imm);
3903 }
3904 return;
3905 }
3906
3907 DCHECK_EQ(ror->GetResultType(), Primitive::kPrimLong);
3908 Register first_reg_lo = first.AsRegisterPairLow<Register>();
3909 Register first_reg_hi = first.AsRegisterPairHigh<Register>();
3910 Register temp_reg = locations->GetTemp(0).AsRegister<Register>();
3911 if (second.IsRegister()) {
3912 Register second_reg = second.AsRegister<Register>();
3913 DCHECK_EQ(second_reg, ECX);
3914 __ movl(temp_reg, first_reg_hi);
3915 __ shrd(first_reg_hi, first_reg_lo, second_reg);
3916 __ shrd(first_reg_lo, temp_reg, second_reg);
3917 __ movl(temp_reg, first_reg_hi);
3918 __ testl(second_reg, Immediate(32));
3919 __ cmovl(kNotEqual, first_reg_hi, first_reg_lo);
3920 __ cmovl(kNotEqual, first_reg_lo, temp_reg);
3921 } else {
3922 int32_t shift_amt = second.GetConstant()->AsIntConstant()->GetValue() & kMaxLongShiftDistance;
3923 if (shift_amt == 0) {
3924 // Already fine.
3925 return;
3926 }
3927 if (shift_amt == 32) {
3928 // Just swap.
3929 __ movl(temp_reg, first_reg_lo);
3930 __ movl(first_reg_lo, first_reg_hi);
3931 __ movl(first_reg_hi, temp_reg);
3932 return;
3933 }
3934
3935 Immediate imm(shift_amt);
3936 // Save the constents of the low value.
3937 __ movl(temp_reg, first_reg_lo);
3938
3939 // Shift right into low, feeding bits from high.
3940 __ shrd(first_reg_lo, first_reg_hi, imm);
3941
3942 // Shift right into high, feeding bits from the original low.
3943 __ shrd(first_reg_hi, temp_reg, imm);
3944
3945 // Swap if needed.
3946 if (shift_amt > 32) {
3947 __ movl(temp_reg, first_reg_lo);
3948 __ movl(first_reg_lo, first_reg_hi);
3949 __ movl(first_reg_hi, temp_reg);
3950 }
3951 }
3952 }
3953
VisitShl(HShl * shl)3954 void LocationsBuilderX86::VisitShl(HShl* shl) {
3955 HandleShift(shl);
3956 }
3957
VisitShl(HShl * shl)3958 void InstructionCodeGeneratorX86::VisitShl(HShl* shl) {
3959 HandleShift(shl);
3960 }
3961
VisitShr(HShr * shr)3962 void LocationsBuilderX86::VisitShr(HShr* shr) {
3963 HandleShift(shr);
3964 }
3965
VisitShr(HShr * shr)3966 void InstructionCodeGeneratorX86::VisitShr(HShr* shr) {
3967 HandleShift(shr);
3968 }
3969
VisitUShr(HUShr * ushr)3970 void LocationsBuilderX86::VisitUShr(HUShr* ushr) {
3971 HandleShift(ushr);
3972 }
3973
VisitUShr(HUShr * ushr)3974 void InstructionCodeGeneratorX86::VisitUShr(HUShr* ushr) {
3975 HandleShift(ushr);
3976 }
3977
VisitNewInstance(HNewInstance * instruction)3978 void LocationsBuilderX86::VisitNewInstance(HNewInstance* instruction) {
3979 LocationSummary* locations =
3980 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCall);
3981 locations->SetOut(Location::RegisterLocation(EAX));
3982 if (instruction->IsStringAlloc()) {
3983 locations->AddTemp(Location::RegisterLocation(kMethodRegisterArgument));
3984 } else {
3985 InvokeRuntimeCallingConvention calling_convention;
3986 locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(0)));
3987 locations->SetInAt(1, Location::RegisterLocation(calling_convention.GetRegisterAt(1)));
3988 }
3989 }
3990
VisitNewInstance(HNewInstance * instruction)3991 void InstructionCodeGeneratorX86::VisitNewInstance(HNewInstance* instruction) {
3992 // Note: if heap poisoning is enabled, the entry point takes cares
3993 // of poisoning the reference.
3994 if (instruction->IsStringAlloc()) {
3995 // String is allocated through StringFactory. Call NewEmptyString entry point.
3996 Register temp = instruction->GetLocations()->GetTemp(0).AsRegister<Register>();
3997 MemberOffset code_offset = ArtMethod::EntryPointFromQuickCompiledCodeOffset(kX86WordSize);
3998 __ fs()->movl(temp, Address::Absolute(QUICK_ENTRY_POINT(pNewEmptyString)));
3999 __ call(Address(temp, code_offset.Int32Value()));
4000 codegen_->RecordPcInfo(instruction, instruction->GetDexPc());
4001 } else {
4002 codegen_->InvokeRuntime(instruction->GetEntrypoint(),
4003 instruction,
4004 instruction->GetDexPc(),
4005 nullptr);
4006 CheckEntrypointTypes<kQuickAllocObjectWithAccessCheck, void*, uint32_t, ArtMethod*>();
4007 DCHECK(!codegen_->IsLeafMethod());
4008 }
4009 }
4010
VisitNewArray(HNewArray * instruction)4011 void LocationsBuilderX86::VisitNewArray(HNewArray* instruction) {
4012 LocationSummary* locations =
4013 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCall);
4014 locations->SetOut(Location::RegisterLocation(EAX));
4015 InvokeRuntimeCallingConvention calling_convention;
4016 locations->AddTemp(Location::RegisterLocation(calling_convention.GetRegisterAt(0)));
4017 locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(1)));
4018 locations->SetInAt(1, Location::RegisterLocation(calling_convention.GetRegisterAt(2)));
4019 }
4020
VisitNewArray(HNewArray * instruction)4021 void InstructionCodeGeneratorX86::VisitNewArray(HNewArray* instruction) {
4022 InvokeRuntimeCallingConvention calling_convention;
4023 __ movl(calling_convention.GetRegisterAt(0), Immediate(instruction->GetTypeIndex()));
4024 // Note: if heap poisoning is enabled, the entry point takes cares
4025 // of poisoning the reference.
4026 codegen_->InvokeRuntime(instruction->GetEntrypoint(),
4027 instruction,
4028 instruction->GetDexPc(),
4029 nullptr);
4030 CheckEntrypointTypes<kQuickAllocArrayWithAccessCheck, void*, uint32_t, int32_t, ArtMethod*>();
4031 DCHECK(!codegen_->IsLeafMethod());
4032 }
4033
VisitParameterValue(HParameterValue * instruction)4034 void LocationsBuilderX86::VisitParameterValue(HParameterValue* instruction) {
4035 LocationSummary* locations =
4036 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
4037 Location location = parameter_visitor_.GetNextLocation(instruction->GetType());
4038 if (location.IsStackSlot()) {
4039 location = Location::StackSlot(location.GetStackIndex() + codegen_->GetFrameSize());
4040 } else if (location.IsDoubleStackSlot()) {
4041 location = Location::DoubleStackSlot(location.GetStackIndex() + codegen_->GetFrameSize());
4042 }
4043 locations->SetOut(location);
4044 }
4045
VisitParameterValue(HParameterValue * instruction ATTRIBUTE_UNUSED)4046 void InstructionCodeGeneratorX86::VisitParameterValue(
4047 HParameterValue* instruction ATTRIBUTE_UNUSED) {
4048 }
4049
VisitCurrentMethod(HCurrentMethod * instruction)4050 void LocationsBuilderX86::VisitCurrentMethod(HCurrentMethod* instruction) {
4051 LocationSummary* locations =
4052 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
4053 locations->SetOut(Location::RegisterLocation(kMethodRegisterArgument));
4054 }
4055
VisitCurrentMethod(HCurrentMethod * instruction ATTRIBUTE_UNUSED)4056 void InstructionCodeGeneratorX86::VisitCurrentMethod(HCurrentMethod* instruction ATTRIBUTE_UNUSED) {
4057 }
4058
VisitClassTableGet(HClassTableGet * instruction)4059 void LocationsBuilderX86::VisitClassTableGet(HClassTableGet* instruction) {
4060 LocationSummary* locations =
4061 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
4062 locations->SetInAt(0, Location::RequiresRegister());
4063 locations->SetOut(Location::RequiresRegister());
4064 }
4065
VisitClassTableGet(HClassTableGet * instruction)4066 void InstructionCodeGeneratorX86::VisitClassTableGet(HClassTableGet* instruction) {
4067 LocationSummary* locations = instruction->GetLocations();
4068 uint32_t method_offset = 0;
4069 if (instruction->GetTableKind() == HClassTableGet::TableKind::kVTable) {
4070 method_offset = mirror::Class::EmbeddedVTableEntryOffset(
4071 instruction->GetIndex(), kX86PointerSize).SizeValue();
4072 } else {
4073 method_offset = mirror::Class::EmbeddedImTableEntryOffset(
4074 instruction->GetIndex() % mirror::Class::kImtSize, kX86PointerSize).Uint32Value();
4075 }
4076 __ movl(locations->Out().AsRegister<Register>(),
4077 Address(locations->InAt(0).AsRegister<Register>(), method_offset));
4078 }
4079
VisitNot(HNot * not_)4080 void LocationsBuilderX86::VisitNot(HNot* not_) {
4081 LocationSummary* locations =
4082 new (GetGraph()->GetArena()) LocationSummary(not_, LocationSummary::kNoCall);
4083 locations->SetInAt(0, Location::RequiresRegister());
4084 locations->SetOut(Location::SameAsFirstInput());
4085 }
4086
VisitNot(HNot * not_)4087 void InstructionCodeGeneratorX86::VisitNot(HNot* not_) {
4088 LocationSummary* locations = not_->GetLocations();
4089 Location in = locations->InAt(0);
4090 Location out = locations->Out();
4091 DCHECK(in.Equals(out));
4092 switch (not_->GetResultType()) {
4093 case Primitive::kPrimInt:
4094 __ notl(out.AsRegister<Register>());
4095 break;
4096
4097 case Primitive::kPrimLong:
4098 __ notl(out.AsRegisterPairLow<Register>());
4099 __ notl(out.AsRegisterPairHigh<Register>());
4100 break;
4101
4102 default:
4103 LOG(FATAL) << "Unimplemented type for not operation " << not_->GetResultType();
4104 }
4105 }
4106
VisitBooleanNot(HBooleanNot * bool_not)4107 void LocationsBuilderX86::VisitBooleanNot(HBooleanNot* bool_not) {
4108 LocationSummary* locations =
4109 new (GetGraph()->GetArena()) LocationSummary(bool_not, LocationSummary::kNoCall);
4110 locations->SetInAt(0, Location::RequiresRegister());
4111 locations->SetOut(Location::SameAsFirstInput());
4112 }
4113
VisitBooleanNot(HBooleanNot * bool_not)4114 void InstructionCodeGeneratorX86::VisitBooleanNot(HBooleanNot* bool_not) {
4115 LocationSummary* locations = bool_not->GetLocations();
4116 Location in = locations->InAt(0);
4117 Location out = locations->Out();
4118 DCHECK(in.Equals(out));
4119 __ xorl(out.AsRegister<Register>(), Immediate(1));
4120 }
4121
VisitCompare(HCompare * compare)4122 void LocationsBuilderX86::VisitCompare(HCompare* compare) {
4123 LocationSummary* locations =
4124 new (GetGraph()->GetArena()) LocationSummary(compare, LocationSummary::kNoCall);
4125 switch (compare->InputAt(0)->GetType()) {
4126 case Primitive::kPrimBoolean:
4127 case Primitive::kPrimByte:
4128 case Primitive::kPrimShort:
4129 case Primitive::kPrimChar:
4130 case Primitive::kPrimInt:
4131 case Primitive::kPrimLong: {
4132 locations->SetInAt(0, Location::RequiresRegister());
4133 locations->SetInAt(1, Location::Any());
4134 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
4135 break;
4136 }
4137 case Primitive::kPrimFloat:
4138 case Primitive::kPrimDouble: {
4139 locations->SetInAt(0, Location::RequiresFpuRegister());
4140 if (compare->InputAt(1)->IsX86LoadFromConstantTable()) {
4141 DCHECK(compare->InputAt(1)->IsEmittedAtUseSite());
4142 } else if (compare->InputAt(1)->IsConstant()) {
4143 locations->SetInAt(1, Location::RequiresFpuRegister());
4144 } else {
4145 locations->SetInAt(1, Location::Any());
4146 }
4147 locations->SetOut(Location::RequiresRegister());
4148 break;
4149 }
4150 default:
4151 LOG(FATAL) << "Unexpected type for compare operation " << compare->InputAt(0)->GetType();
4152 }
4153 }
4154
VisitCompare(HCompare * compare)4155 void InstructionCodeGeneratorX86::VisitCompare(HCompare* compare) {
4156 LocationSummary* locations = compare->GetLocations();
4157 Register out = locations->Out().AsRegister<Register>();
4158 Location left = locations->InAt(0);
4159 Location right = locations->InAt(1);
4160
4161 NearLabel less, greater, done;
4162 Condition less_cond = kLess;
4163
4164 switch (compare->InputAt(0)->GetType()) {
4165 case Primitive::kPrimBoolean:
4166 case Primitive::kPrimByte:
4167 case Primitive::kPrimShort:
4168 case Primitive::kPrimChar:
4169 case Primitive::kPrimInt: {
4170 GenerateIntCompare(left, right);
4171 break;
4172 }
4173 case Primitive::kPrimLong: {
4174 Register left_low = left.AsRegisterPairLow<Register>();
4175 Register left_high = left.AsRegisterPairHigh<Register>();
4176 int32_t val_low = 0;
4177 int32_t val_high = 0;
4178 bool right_is_const = false;
4179
4180 if (right.IsConstant()) {
4181 DCHECK(right.GetConstant()->IsLongConstant());
4182 right_is_const = true;
4183 int64_t val = right.GetConstant()->AsLongConstant()->GetValue();
4184 val_low = Low32Bits(val);
4185 val_high = High32Bits(val);
4186 }
4187
4188 if (right.IsRegisterPair()) {
4189 __ cmpl(left_high, right.AsRegisterPairHigh<Register>());
4190 } else if (right.IsDoubleStackSlot()) {
4191 __ cmpl(left_high, Address(ESP, right.GetHighStackIndex(kX86WordSize)));
4192 } else {
4193 DCHECK(right_is_const) << right;
4194 codegen_->Compare32BitValue(left_high, val_high);
4195 }
4196 __ j(kLess, &less); // Signed compare.
4197 __ j(kGreater, &greater); // Signed compare.
4198 if (right.IsRegisterPair()) {
4199 __ cmpl(left_low, right.AsRegisterPairLow<Register>());
4200 } else if (right.IsDoubleStackSlot()) {
4201 __ cmpl(left_low, Address(ESP, right.GetStackIndex()));
4202 } else {
4203 DCHECK(right_is_const) << right;
4204 codegen_->Compare32BitValue(left_low, val_low);
4205 }
4206 less_cond = kBelow; // for CF (unsigned).
4207 break;
4208 }
4209 case Primitive::kPrimFloat: {
4210 GenerateFPCompare(left, right, compare, false);
4211 __ j(kUnordered, compare->IsGtBias() ? &greater : &less);
4212 less_cond = kBelow; // for CF (floats).
4213 break;
4214 }
4215 case Primitive::kPrimDouble: {
4216 GenerateFPCompare(left, right, compare, true);
4217 __ j(kUnordered, compare->IsGtBias() ? &greater : &less);
4218 less_cond = kBelow; // for CF (floats).
4219 break;
4220 }
4221 default:
4222 LOG(FATAL) << "Unexpected type for compare operation " << compare->InputAt(0)->GetType();
4223 }
4224
4225 __ movl(out, Immediate(0));
4226 __ j(kEqual, &done);
4227 __ j(less_cond, &less);
4228
4229 __ Bind(&greater);
4230 __ movl(out, Immediate(1));
4231 __ jmp(&done);
4232
4233 __ Bind(&less);
4234 __ movl(out, Immediate(-1));
4235
4236 __ Bind(&done);
4237 }
4238
VisitPhi(HPhi * instruction)4239 void LocationsBuilderX86::VisitPhi(HPhi* instruction) {
4240 LocationSummary* locations =
4241 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
4242 for (size_t i = 0, e = instruction->InputCount(); i < e; ++i) {
4243 locations->SetInAt(i, Location::Any());
4244 }
4245 locations->SetOut(Location::Any());
4246 }
4247
VisitPhi(HPhi * instruction ATTRIBUTE_UNUSED)4248 void InstructionCodeGeneratorX86::VisitPhi(HPhi* instruction ATTRIBUTE_UNUSED) {
4249 LOG(FATAL) << "Unreachable";
4250 }
4251
GenerateMemoryBarrier(MemBarrierKind kind)4252 void CodeGeneratorX86::GenerateMemoryBarrier(MemBarrierKind kind) {
4253 /*
4254 * According to the JSR-133 Cookbook, for x86 only StoreLoad/AnyAny barriers need memory fence.
4255 * All other barriers (LoadAny, AnyStore, StoreStore) are nops due to the x86 memory model.
4256 * For those cases, all we need to ensure is that there is a scheduling barrier in place.
4257 */
4258 switch (kind) {
4259 case MemBarrierKind::kAnyAny: {
4260 MemoryFence();
4261 break;
4262 }
4263 case MemBarrierKind::kAnyStore:
4264 case MemBarrierKind::kLoadAny:
4265 case MemBarrierKind::kStoreStore: {
4266 // nop
4267 break;
4268 }
4269 default:
4270 LOG(FATAL) << "Unexpected memory barrier " << kind;
4271 }
4272 }
4273
GetSupportedInvokeStaticOrDirectDispatch(const HInvokeStaticOrDirect::DispatchInfo & desired_dispatch_info,MethodReference target_method ATTRIBUTE_UNUSED)4274 HInvokeStaticOrDirect::DispatchInfo CodeGeneratorX86::GetSupportedInvokeStaticOrDirectDispatch(
4275 const HInvokeStaticOrDirect::DispatchInfo& desired_dispatch_info,
4276 MethodReference target_method ATTRIBUTE_UNUSED) {
4277 HInvokeStaticOrDirect::DispatchInfo dispatch_info = desired_dispatch_info;
4278
4279 // We disable pc-relative load when there is an irreducible loop, as the optimization
4280 // is incompatible with it.
4281 // TODO: Create as many X86ComputeBaseMethodAddress instructions
4282 // as needed for methods with irreducible loops.
4283 if (GetGraph()->HasIrreducibleLoops() &&
4284 (dispatch_info.method_load_kind ==
4285 HInvokeStaticOrDirect::MethodLoadKind::kDexCachePcRelative)) {
4286 dispatch_info.method_load_kind = HInvokeStaticOrDirect::MethodLoadKind::kDexCacheViaMethod;
4287 }
4288 switch (dispatch_info.code_ptr_location) {
4289 case HInvokeStaticOrDirect::CodePtrLocation::kCallDirectWithFixup:
4290 case HInvokeStaticOrDirect::CodePtrLocation::kCallDirect:
4291 // For direct code, we actually prefer to call via the code pointer from ArtMethod*.
4292 // (Though the direct CALL ptr16:32 is available for consideration).
4293 return HInvokeStaticOrDirect::DispatchInfo {
4294 dispatch_info.method_load_kind,
4295 HInvokeStaticOrDirect::CodePtrLocation::kCallArtMethod,
4296 dispatch_info.method_load_data,
4297 0u
4298 };
4299 default:
4300 return dispatch_info;
4301 }
4302 }
4303
GetInvokeStaticOrDirectExtraParameter(HInvokeStaticOrDirect * invoke,Register temp)4304 Register CodeGeneratorX86::GetInvokeStaticOrDirectExtraParameter(HInvokeStaticOrDirect* invoke,
4305 Register temp) {
4306 DCHECK_EQ(invoke->InputCount(), invoke->GetNumberOfArguments() + 1u);
4307 Location location = invoke->GetLocations()->InAt(invoke->GetSpecialInputIndex());
4308 if (!invoke->GetLocations()->Intrinsified()) {
4309 return location.AsRegister<Register>();
4310 }
4311 // For intrinsics we allow any location, so it may be on the stack.
4312 if (!location.IsRegister()) {
4313 __ movl(temp, Address(ESP, location.GetStackIndex()));
4314 return temp;
4315 }
4316 // For register locations, check if the register was saved. If so, get it from the stack.
4317 // Note: There is a chance that the register was saved but not overwritten, so we could
4318 // save one load. However, since this is just an intrinsic slow path we prefer this
4319 // simple and more robust approach rather that trying to determine if that's the case.
4320 SlowPathCode* slow_path = GetCurrentSlowPath();
4321 DCHECK(slow_path != nullptr); // For intrinsified invokes the call is emitted on the slow path.
4322 if (slow_path->IsCoreRegisterSaved(location.AsRegister<Register>())) {
4323 int stack_offset = slow_path->GetStackOffsetOfCoreRegister(location.AsRegister<Register>());
4324 __ movl(temp, Address(ESP, stack_offset));
4325 return temp;
4326 }
4327 return location.AsRegister<Register>();
4328 }
4329
GenerateStaticOrDirectCall(HInvokeStaticOrDirect * invoke,Location temp)4330 void CodeGeneratorX86::GenerateStaticOrDirectCall(HInvokeStaticOrDirect* invoke, Location temp) {
4331 Location callee_method = temp; // For all kinds except kRecursive, callee will be in temp.
4332 switch (invoke->GetMethodLoadKind()) {
4333 case HInvokeStaticOrDirect::MethodLoadKind::kStringInit:
4334 // temp = thread->string_init_entrypoint
4335 __ fs()->movl(temp.AsRegister<Register>(), Address::Absolute(invoke->GetStringInitOffset()));
4336 break;
4337 case HInvokeStaticOrDirect::MethodLoadKind::kRecursive:
4338 callee_method = invoke->GetLocations()->InAt(invoke->GetSpecialInputIndex());
4339 break;
4340 case HInvokeStaticOrDirect::MethodLoadKind::kDirectAddress:
4341 __ movl(temp.AsRegister<Register>(), Immediate(invoke->GetMethodAddress()));
4342 break;
4343 case HInvokeStaticOrDirect::MethodLoadKind::kDirectAddressWithFixup:
4344 __ movl(temp.AsRegister<Register>(), Immediate(/* placeholder */ 0));
4345 method_patches_.emplace_back(invoke->GetTargetMethod());
4346 __ Bind(&method_patches_.back().label); // Bind the label at the end of the "movl" insn.
4347 break;
4348 case HInvokeStaticOrDirect::MethodLoadKind::kDexCachePcRelative: {
4349 Register base_reg = GetInvokeStaticOrDirectExtraParameter(invoke,
4350 temp.AsRegister<Register>());
4351 __ movl(temp.AsRegister<Register>(), Address(base_reg, kDummy32BitOffset));
4352 // Bind a new fixup label at the end of the "movl" insn.
4353 uint32_t offset = invoke->GetDexCacheArrayOffset();
4354 __ Bind(NewPcRelativeDexCacheArrayPatch(*invoke->GetTargetMethod().dex_file, offset));
4355 break;
4356 }
4357 case HInvokeStaticOrDirect::MethodLoadKind::kDexCacheViaMethod: {
4358 Location current_method = invoke->GetLocations()->InAt(invoke->GetSpecialInputIndex());
4359 Register method_reg;
4360 Register reg = temp.AsRegister<Register>();
4361 if (current_method.IsRegister()) {
4362 method_reg = current_method.AsRegister<Register>();
4363 } else {
4364 DCHECK(invoke->GetLocations()->Intrinsified());
4365 DCHECK(!current_method.IsValid());
4366 method_reg = reg;
4367 __ movl(reg, Address(ESP, kCurrentMethodStackOffset));
4368 }
4369 // /* ArtMethod*[] */ temp = temp.ptr_sized_fields_->dex_cache_resolved_methods_;
4370 __ movl(reg, Address(method_reg,
4371 ArtMethod::DexCacheResolvedMethodsOffset(kX86PointerSize).Int32Value()));
4372 // temp = temp[index_in_cache];
4373 // Note: Don't use invoke->GetTargetMethod() as it may point to a different dex file.
4374 uint32_t index_in_cache = invoke->GetDexMethodIndex();
4375 __ movl(reg, Address(reg, CodeGenerator::GetCachePointerOffset(index_in_cache)));
4376 break;
4377 }
4378 }
4379
4380 switch (invoke->GetCodePtrLocation()) {
4381 case HInvokeStaticOrDirect::CodePtrLocation::kCallSelf:
4382 __ call(GetFrameEntryLabel());
4383 break;
4384 case HInvokeStaticOrDirect::CodePtrLocation::kCallPCRelative: {
4385 relative_call_patches_.emplace_back(invoke->GetTargetMethod());
4386 Label* label = &relative_call_patches_.back().label;
4387 __ call(label); // Bind to the patch label, override at link time.
4388 __ Bind(label); // Bind the label at the end of the "call" insn.
4389 break;
4390 }
4391 case HInvokeStaticOrDirect::CodePtrLocation::kCallDirectWithFixup:
4392 case HInvokeStaticOrDirect::CodePtrLocation::kCallDirect:
4393 // Filtered out by GetSupportedInvokeStaticOrDirectDispatch().
4394 LOG(FATAL) << "Unsupported";
4395 UNREACHABLE();
4396 case HInvokeStaticOrDirect::CodePtrLocation::kCallArtMethod:
4397 // (callee_method + offset_of_quick_compiled_code)()
4398 __ call(Address(callee_method.AsRegister<Register>(),
4399 ArtMethod::EntryPointFromQuickCompiledCodeOffset(
4400 kX86WordSize).Int32Value()));
4401 break;
4402 }
4403
4404 DCHECK(!IsLeafMethod());
4405 }
4406
GenerateVirtualCall(HInvokeVirtual * invoke,Location temp_in)4407 void CodeGeneratorX86::GenerateVirtualCall(HInvokeVirtual* invoke, Location temp_in) {
4408 Register temp = temp_in.AsRegister<Register>();
4409 uint32_t method_offset = mirror::Class::EmbeddedVTableEntryOffset(
4410 invoke->GetVTableIndex(), kX86PointerSize).Uint32Value();
4411
4412 // Use the calling convention instead of the location of the receiver, as
4413 // intrinsics may have put the receiver in a different register. In the intrinsics
4414 // slow path, the arguments have been moved to the right place, so here we are
4415 // guaranteed that the receiver is the first register of the calling convention.
4416 InvokeDexCallingConvention calling_convention;
4417 Register receiver = calling_convention.GetRegisterAt(0);
4418 uint32_t class_offset = mirror::Object::ClassOffset().Int32Value();
4419 // /* HeapReference<Class> */ temp = receiver->klass_
4420 __ movl(temp, Address(receiver, class_offset));
4421 MaybeRecordImplicitNullCheck(invoke);
4422 // Instead of simply (possibly) unpoisoning `temp` here, we should
4423 // emit a read barrier for the previous class reference load.
4424 // However this is not required in practice, as this is an
4425 // intermediate/temporary reference and because the current
4426 // concurrent copying collector keeps the from-space memory
4427 // intact/accessible until the end of the marking phase (the
4428 // concurrent copying collector may not in the future).
4429 __ MaybeUnpoisonHeapReference(temp);
4430 // temp = temp->GetMethodAt(method_offset);
4431 __ movl(temp, Address(temp, method_offset));
4432 // call temp->GetEntryPoint();
4433 __ call(Address(
4434 temp, ArtMethod::EntryPointFromQuickCompiledCodeOffset(kX86WordSize).Int32Value()));
4435 }
4436
RecordSimplePatch()4437 void CodeGeneratorX86::RecordSimplePatch() {
4438 if (GetCompilerOptions().GetIncludePatchInformation()) {
4439 simple_patches_.emplace_back();
4440 __ Bind(&simple_patches_.back());
4441 }
4442 }
4443
RecordStringPatch(HLoadString * load_string)4444 void CodeGeneratorX86::RecordStringPatch(HLoadString* load_string) {
4445 string_patches_.emplace_back(load_string->GetDexFile(), load_string->GetStringIndex());
4446 __ Bind(&string_patches_.back().label);
4447 }
4448
NewPcRelativeDexCacheArrayPatch(const DexFile & dex_file,uint32_t element_offset)4449 Label* CodeGeneratorX86::NewPcRelativeDexCacheArrayPatch(const DexFile& dex_file,
4450 uint32_t element_offset) {
4451 // Add the patch entry and bind its label at the end of the instruction.
4452 pc_relative_dex_cache_patches_.emplace_back(dex_file, element_offset);
4453 return &pc_relative_dex_cache_patches_.back().label;
4454 }
4455
EmitLinkerPatches(ArenaVector<LinkerPatch> * linker_patches)4456 void CodeGeneratorX86::EmitLinkerPatches(ArenaVector<LinkerPatch>* linker_patches) {
4457 DCHECK(linker_patches->empty());
4458 size_t size =
4459 method_patches_.size() +
4460 relative_call_patches_.size() +
4461 pc_relative_dex_cache_patches_.size() +
4462 simple_patches_.size() +
4463 string_patches_.size();
4464 linker_patches->reserve(size);
4465 // The label points to the end of the "movl" insn but the literal offset for method
4466 // patch needs to point to the embedded constant which occupies the last 4 bytes.
4467 constexpr uint32_t kLabelPositionToLiteralOffsetAdjustment = 4u;
4468 for (const MethodPatchInfo<Label>& info : method_patches_) {
4469 uint32_t literal_offset = info.label.Position() - kLabelPositionToLiteralOffsetAdjustment;
4470 linker_patches->push_back(LinkerPatch::MethodPatch(literal_offset,
4471 info.target_method.dex_file,
4472 info.target_method.dex_method_index));
4473 }
4474 for (const MethodPatchInfo<Label>& info : relative_call_patches_) {
4475 uint32_t literal_offset = info.label.Position() - kLabelPositionToLiteralOffsetAdjustment;
4476 linker_patches->push_back(LinkerPatch::RelativeCodePatch(literal_offset,
4477 info.target_method.dex_file,
4478 info.target_method.dex_method_index));
4479 }
4480 for (const PcRelativeDexCacheAccessInfo& info : pc_relative_dex_cache_patches_) {
4481 uint32_t literal_offset = info.label.Position() - kLabelPositionToLiteralOffsetAdjustment;
4482 linker_patches->push_back(LinkerPatch::DexCacheArrayPatch(literal_offset,
4483 &info.target_dex_file,
4484 GetMethodAddressOffset(),
4485 info.element_offset));
4486 }
4487 for (const Label& label : simple_patches_) {
4488 uint32_t literal_offset = label.Position() - kLabelPositionToLiteralOffsetAdjustment;
4489 linker_patches->push_back(LinkerPatch::RecordPosition(literal_offset));
4490 }
4491 if (GetCompilerOptions().GetCompilePic()) {
4492 for (const StringPatchInfo<Label>& info : string_patches_) {
4493 uint32_t literal_offset = info.label.Position() - kLabelPositionToLiteralOffsetAdjustment;
4494 linker_patches->push_back(LinkerPatch::RelativeStringPatch(literal_offset,
4495 &info.dex_file,
4496 GetMethodAddressOffset(),
4497 info.string_index));
4498 }
4499 } else {
4500 for (const StringPatchInfo<Label>& info : string_patches_) {
4501 uint32_t literal_offset = info.label.Position() - kLabelPositionToLiteralOffsetAdjustment;
4502 linker_patches->push_back(LinkerPatch::StringPatch(literal_offset,
4503 &info.dex_file,
4504 info.string_index));
4505 }
4506 }
4507 }
4508
MarkGCCard(Register temp,Register card,Register object,Register value,bool value_can_be_null)4509 void CodeGeneratorX86::MarkGCCard(Register temp,
4510 Register card,
4511 Register object,
4512 Register value,
4513 bool value_can_be_null) {
4514 NearLabel is_null;
4515 if (value_can_be_null) {
4516 __ testl(value, value);
4517 __ j(kEqual, &is_null);
4518 }
4519 __ fs()->movl(card, Address::Absolute(Thread::CardTableOffset<kX86WordSize>().Int32Value()));
4520 __ movl(temp, object);
4521 __ shrl(temp, Immediate(gc::accounting::CardTable::kCardShift));
4522 __ movb(Address(temp, card, TIMES_1, 0),
4523 X86ManagedRegister::FromCpuRegister(card).AsByteRegister());
4524 if (value_can_be_null) {
4525 __ Bind(&is_null);
4526 }
4527 }
4528
HandleFieldGet(HInstruction * instruction,const FieldInfo & field_info)4529 void LocationsBuilderX86::HandleFieldGet(HInstruction* instruction, const FieldInfo& field_info) {
4530 DCHECK(instruction->IsInstanceFieldGet() || instruction->IsStaticFieldGet());
4531
4532 bool object_field_get_with_read_barrier =
4533 kEmitCompilerReadBarrier && (instruction->GetType() == Primitive::kPrimNot);
4534 LocationSummary* locations =
4535 new (GetGraph()->GetArena()) LocationSummary(instruction,
4536 kEmitCompilerReadBarrier ?
4537 LocationSummary::kCallOnSlowPath :
4538 LocationSummary::kNoCall);
4539 locations->SetInAt(0, Location::RequiresRegister());
4540
4541 if (Primitive::IsFloatingPointType(instruction->GetType())) {
4542 locations->SetOut(Location::RequiresFpuRegister());
4543 } else {
4544 // The output overlaps in case of long: we don't want the low move
4545 // to overwrite the object's location. Likewise, in the case of
4546 // an object field get with read barriers enabled, we do not want
4547 // the move to overwrite the object's location, as we need it to emit
4548 // the read barrier.
4549 locations->SetOut(
4550 Location::RequiresRegister(),
4551 (object_field_get_with_read_barrier || instruction->GetType() == Primitive::kPrimLong) ?
4552 Location::kOutputOverlap :
4553 Location::kNoOutputOverlap);
4554 }
4555
4556 if (field_info.IsVolatile() && (field_info.GetFieldType() == Primitive::kPrimLong)) {
4557 // Long values can be loaded atomically into an XMM using movsd.
4558 // So we use an XMM register as a temp to achieve atomicity (first
4559 // load the temp into the XMM and then copy the XMM into the
4560 // output, 32 bits at a time).
4561 locations->AddTemp(Location::RequiresFpuRegister());
4562 } else if (object_field_get_with_read_barrier && kUseBakerReadBarrier) {
4563 // We need a temporary register for the read barrier marking slow
4564 // path in CodeGeneratorX86::GenerateFieldLoadWithBakerReadBarrier.
4565 locations->AddTemp(Location::RequiresRegister());
4566 }
4567 }
4568
HandleFieldGet(HInstruction * instruction,const FieldInfo & field_info)4569 void InstructionCodeGeneratorX86::HandleFieldGet(HInstruction* instruction,
4570 const FieldInfo& field_info) {
4571 DCHECK(instruction->IsInstanceFieldGet() || instruction->IsStaticFieldGet());
4572
4573 LocationSummary* locations = instruction->GetLocations();
4574 Location base_loc = locations->InAt(0);
4575 Register base = base_loc.AsRegister<Register>();
4576 Location out = locations->Out();
4577 bool is_volatile = field_info.IsVolatile();
4578 Primitive::Type field_type = field_info.GetFieldType();
4579 uint32_t offset = field_info.GetFieldOffset().Uint32Value();
4580
4581 switch (field_type) {
4582 case Primitive::kPrimBoolean: {
4583 __ movzxb(out.AsRegister<Register>(), Address(base, offset));
4584 break;
4585 }
4586
4587 case Primitive::kPrimByte: {
4588 __ movsxb(out.AsRegister<Register>(), Address(base, offset));
4589 break;
4590 }
4591
4592 case Primitive::kPrimShort: {
4593 __ movsxw(out.AsRegister<Register>(), Address(base, offset));
4594 break;
4595 }
4596
4597 case Primitive::kPrimChar: {
4598 __ movzxw(out.AsRegister<Register>(), Address(base, offset));
4599 break;
4600 }
4601
4602 case Primitive::kPrimInt:
4603 __ movl(out.AsRegister<Register>(), Address(base, offset));
4604 break;
4605
4606 case Primitive::kPrimNot: {
4607 // /* HeapReference<Object> */ out = *(base + offset)
4608 if (kEmitCompilerReadBarrier && kUseBakerReadBarrier) {
4609 Location temp_loc = locations->GetTemp(0);
4610 // Note that a potential implicit null check is handled in this
4611 // CodeGeneratorX86::GenerateFieldLoadWithBakerReadBarrier call.
4612 codegen_->GenerateFieldLoadWithBakerReadBarrier(
4613 instruction, out, base, offset, temp_loc, /* needs_null_check */ true);
4614 if (is_volatile) {
4615 codegen_->GenerateMemoryBarrier(MemBarrierKind::kLoadAny);
4616 }
4617 } else {
4618 __ movl(out.AsRegister<Register>(), Address(base, offset));
4619 codegen_->MaybeRecordImplicitNullCheck(instruction);
4620 if (is_volatile) {
4621 codegen_->GenerateMemoryBarrier(MemBarrierKind::kLoadAny);
4622 }
4623 // If read barriers are enabled, emit read barriers other than
4624 // Baker's using a slow path (and also unpoison the loaded
4625 // reference, if heap poisoning is enabled).
4626 codegen_->MaybeGenerateReadBarrierSlow(instruction, out, out, base_loc, offset);
4627 }
4628 break;
4629 }
4630
4631 case Primitive::kPrimLong: {
4632 if (is_volatile) {
4633 XmmRegister temp = locations->GetTemp(0).AsFpuRegister<XmmRegister>();
4634 __ movsd(temp, Address(base, offset));
4635 codegen_->MaybeRecordImplicitNullCheck(instruction);
4636 __ movd(out.AsRegisterPairLow<Register>(), temp);
4637 __ psrlq(temp, Immediate(32));
4638 __ movd(out.AsRegisterPairHigh<Register>(), temp);
4639 } else {
4640 DCHECK_NE(base, out.AsRegisterPairLow<Register>());
4641 __ movl(out.AsRegisterPairLow<Register>(), Address(base, offset));
4642 codegen_->MaybeRecordImplicitNullCheck(instruction);
4643 __ movl(out.AsRegisterPairHigh<Register>(), Address(base, kX86WordSize + offset));
4644 }
4645 break;
4646 }
4647
4648 case Primitive::kPrimFloat: {
4649 __ movss(out.AsFpuRegister<XmmRegister>(), Address(base, offset));
4650 break;
4651 }
4652
4653 case Primitive::kPrimDouble: {
4654 __ movsd(out.AsFpuRegister<XmmRegister>(), Address(base, offset));
4655 break;
4656 }
4657
4658 case Primitive::kPrimVoid:
4659 LOG(FATAL) << "Unreachable type " << field_type;
4660 UNREACHABLE();
4661 }
4662
4663 if (field_type == Primitive::kPrimNot || field_type == Primitive::kPrimLong) {
4664 // Potential implicit null checks, in the case of reference or
4665 // long fields, are handled in the previous switch statement.
4666 } else {
4667 codegen_->MaybeRecordImplicitNullCheck(instruction);
4668 }
4669
4670 if (is_volatile) {
4671 if (field_type == Primitive::kPrimNot) {
4672 // Memory barriers, in the case of references, are also handled
4673 // in the previous switch statement.
4674 } else {
4675 codegen_->GenerateMemoryBarrier(MemBarrierKind::kLoadAny);
4676 }
4677 }
4678 }
4679
HandleFieldSet(HInstruction * instruction,const FieldInfo & field_info)4680 void LocationsBuilderX86::HandleFieldSet(HInstruction* instruction, const FieldInfo& field_info) {
4681 DCHECK(instruction->IsInstanceFieldSet() || instruction->IsStaticFieldSet());
4682
4683 LocationSummary* locations =
4684 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
4685 locations->SetInAt(0, Location::RequiresRegister());
4686 bool is_volatile = field_info.IsVolatile();
4687 Primitive::Type field_type = field_info.GetFieldType();
4688 bool is_byte_type = (field_type == Primitive::kPrimBoolean)
4689 || (field_type == Primitive::kPrimByte);
4690
4691 // The register allocator does not support multiple
4692 // inputs that die at entry with one in a specific register.
4693 if (is_byte_type) {
4694 // Ensure the value is in a byte register.
4695 locations->SetInAt(1, Location::RegisterLocation(EAX));
4696 } else if (Primitive::IsFloatingPointType(field_type)) {
4697 if (is_volatile && field_type == Primitive::kPrimDouble) {
4698 // In order to satisfy the semantics of volatile, this must be a single instruction store.
4699 locations->SetInAt(1, Location::RequiresFpuRegister());
4700 } else {
4701 locations->SetInAt(1, Location::FpuRegisterOrConstant(instruction->InputAt(1)));
4702 }
4703 } else if (is_volatile && field_type == Primitive::kPrimLong) {
4704 // In order to satisfy the semantics of volatile, this must be a single instruction store.
4705 locations->SetInAt(1, Location::RequiresRegister());
4706
4707 // 64bits value can be atomically written to an address with movsd and an XMM register.
4708 // We need two XMM registers because there's no easier way to (bit) copy a register pair
4709 // into a single XMM register (we copy each pair part into the XMMs and then interleave them).
4710 // NB: We could make the register allocator understand fp_reg <-> core_reg moves but given the
4711 // isolated cases when we need this it isn't worth adding the extra complexity.
4712 locations->AddTemp(Location::RequiresFpuRegister());
4713 locations->AddTemp(Location::RequiresFpuRegister());
4714 } else {
4715 locations->SetInAt(1, Location::RegisterOrConstant(instruction->InputAt(1)));
4716
4717 if (CodeGenerator::StoreNeedsWriteBarrier(field_type, instruction->InputAt(1))) {
4718 // Temporary registers for the write barrier.
4719 locations->AddTemp(Location::RequiresRegister()); // May be used for reference poisoning too.
4720 // Ensure the card is in a byte register.
4721 locations->AddTemp(Location::RegisterLocation(ECX));
4722 }
4723 }
4724 }
4725
HandleFieldSet(HInstruction * instruction,const FieldInfo & field_info,bool value_can_be_null)4726 void InstructionCodeGeneratorX86::HandleFieldSet(HInstruction* instruction,
4727 const FieldInfo& field_info,
4728 bool value_can_be_null) {
4729 DCHECK(instruction->IsInstanceFieldSet() || instruction->IsStaticFieldSet());
4730
4731 LocationSummary* locations = instruction->GetLocations();
4732 Register base = locations->InAt(0).AsRegister<Register>();
4733 Location value = locations->InAt(1);
4734 bool is_volatile = field_info.IsVolatile();
4735 Primitive::Type field_type = field_info.GetFieldType();
4736 uint32_t offset = field_info.GetFieldOffset().Uint32Value();
4737 bool needs_write_barrier =
4738 CodeGenerator::StoreNeedsWriteBarrier(field_type, instruction->InputAt(1));
4739
4740 if (is_volatile) {
4741 codegen_->GenerateMemoryBarrier(MemBarrierKind::kAnyStore);
4742 }
4743
4744 bool maybe_record_implicit_null_check_done = false;
4745
4746 switch (field_type) {
4747 case Primitive::kPrimBoolean:
4748 case Primitive::kPrimByte: {
4749 __ movb(Address(base, offset), value.AsRegister<ByteRegister>());
4750 break;
4751 }
4752
4753 case Primitive::kPrimShort:
4754 case Primitive::kPrimChar: {
4755 if (value.IsConstant()) {
4756 int16_t v = CodeGenerator::GetInt32ValueOf(value.GetConstant());
4757 __ movw(Address(base, offset), Immediate(v));
4758 } else {
4759 __ movw(Address(base, offset), value.AsRegister<Register>());
4760 }
4761 break;
4762 }
4763
4764 case Primitive::kPrimInt:
4765 case Primitive::kPrimNot: {
4766 if (kPoisonHeapReferences && needs_write_barrier) {
4767 // Note that in the case where `value` is a null reference,
4768 // we do not enter this block, as the reference does not
4769 // need poisoning.
4770 DCHECK_EQ(field_type, Primitive::kPrimNot);
4771 Register temp = locations->GetTemp(0).AsRegister<Register>();
4772 __ movl(temp, value.AsRegister<Register>());
4773 __ PoisonHeapReference(temp);
4774 __ movl(Address(base, offset), temp);
4775 } else if (value.IsConstant()) {
4776 int32_t v = CodeGenerator::GetInt32ValueOf(value.GetConstant());
4777 __ movl(Address(base, offset), Immediate(v));
4778 } else {
4779 DCHECK(value.IsRegister()) << value;
4780 __ movl(Address(base, offset), value.AsRegister<Register>());
4781 }
4782 break;
4783 }
4784
4785 case Primitive::kPrimLong: {
4786 if (is_volatile) {
4787 XmmRegister temp1 = locations->GetTemp(0).AsFpuRegister<XmmRegister>();
4788 XmmRegister temp2 = locations->GetTemp(1).AsFpuRegister<XmmRegister>();
4789 __ movd(temp1, value.AsRegisterPairLow<Register>());
4790 __ movd(temp2, value.AsRegisterPairHigh<Register>());
4791 __ punpckldq(temp1, temp2);
4792 __ movsd(Address(base, offset), temp1);
4793 codegen_->MaybeRecordImplicitNullCheck(instruction);
4794 } else if (value.IsConstant()) {
4795 int64_t v = CodeGenerator::GetInt64ValueOf(value.GetConstant());
4796 __ movl(Address(base, offset), Immediate(Low32Bits(v)));
4797 codegen_->MaybeRecordImplicitNullCheck(instruction);
4798 __ movl(Address(base, kX86WordSize + offset), Immediate(High32Bits(v)));
4799 } else {
4800 __ movl(Address(base, offset), value.AsRegisterPairLow<Register>());
4801 codegen_->MaybeRecordImplicitNullCheck(instruction);
4802 __ movl(Address(base, kX86WordSize + offset), value.AsRegisterPairHigh<Register>());
4803 }
4804 maybe_record_implicit_null_check_done = true;
4805 break;
4806 }
4807
4808 case Primitive::kPrimFloat: {
4809 if (value.IsConstant()) {
4810 int32_t v = CodeGenerator::GetInt32ValueOf(value.GetConstant());
4811 __ movl(Address(base, offset), Immediate(v));
4812 } else {
4813 __ movss(Address(base, offset), value.AsFpuRegister<XmmRegister>());
4814 }
4815 break;
4816 }
4817
4818 case Primitive::kPrimDouble: {
4819 if (value.IsConstant()) {
4820 int64_t v = CodeGenerator::GetInt64ValueOf(value.GetConstant());
4821 __ movl(Address(base, offset), Immediate(Low32Bits(v)));
4822 codegen_->MaybeRecordImplicitNullCheck(instruction);
4823 __ movl(Address(base, kX86WordSize + offset), Immediate(High32Bits(v)));
4824 maybe_record_implicit_null_check_done = true;
4825 } else {
4826 __ movsd(Address(base, offset), value.AsFpuRegister<XmmRegister>());
4827 }
4828 break;
4829 }
4830
4831 case Primitive::kPrimVoid:
4832 LOG(FATAL) << "Unreachable type " << field_type;
4833 UNREACHABLE();
4834 }
4835
4836 if (!maybe_record_implicit_null_check_done) {
4837 codegen_->MaybeRecordImplicitNullCheck(instruction);
4838 }
4839
4840 if (needs_write_barrier) {
4841 Register temp = locations->GetTemp(0).AsRegister<Register>();
4842 Register card = locations->GetTemp(1).AsRegister<Register>();
4843 codegen_->MarkGCCard(temp, card, base, value.AsRegister<Register>(), value_can_be_null);
4844 }
4845
4846 if (is_volatile) {
4847 codegen_->GenerateMemoryBarrier(MemBarrierKind::kAnyAny);
4848 }
4849 }
4850
VisitStaticFieldGet(HStaticFieldGet * instruction)4851 void LocationsBuilderX86::VisitStaticFieldGet(HStaticFieldGet* instruction) {
4852 HandleFieldGet(instruction, instruction->GetFieldInfo());
4853 }
4854
VisitStaticFieldGet(HStaticFieldGet * instruction)4855 void InstructionCodeGeneratorX86::VisitStaticFieldGet(HStaticFieldGet* instruction) {
4856 HandleFieldGet(instruction, instruction->GetFieldInfo());
4857 }
4858
VisitStaticFieldSet(HStaticFieldSet * instruction)4859 void LocationsBuilderX86::VisitStaticFieldSet(HStaticFieldSet* instruction) {
4860 HandleFieldSet(instruction, instruction->GetFieldInfo());
4861 }
4862
VisitStaticFieldSet(HStaticFieldSet * instruction)4863 void InstructionCodeGeneratorX86::VisitStaticFieldSet(HStaticFieldSet* instruction) {
4864 HandleFieldSet(instruction, instruction->GetFieldInfo(), instruction->GetValueCanBeNull());
4865 }
4866
VisitInstanceFieldSet(HInstanceFieldSet * instruction)4867 void LocationsBuilderX86::VisitInstanceFieldSet(HInstanceFieldSet* instruction) {
4868 HandleFieldSet(instruction, instruction->GetFieldInfo());
4869 }
4870
VisitInstanceFieldSet(HInstanceFieldSet * instruction)4871 void InstructionCodeGeneratorX86::VisitInstanceFieldSet(HInstanceFieldSet* instruction) {
4872 HandleFieldSet(instruction, instruction->GetFieldInfo(), instruction->GetValueCanBeNull());
4873 }
4874
VisitInstanceFieldGet(HInstanceFieldGet * instruction)4875 void LocationsBuilderX86::VisitInstanceFieldGet(HInstanceFieldGet* instruction) {
4876 HandleFieldGet(instruction, instruction->GetFieldInfo());
4877 }
4878
VisitInstanceFieldGet(HInstanceFieldGet * instruction)4879 void InstructionCodeGeneratorX86::VisitInstanceFieldGet(HInstanceFieldGet* instruction) {
4880 HandleFieldGet(instruction, instruction->GetFieldInfo());
4881 }
4882
VisitUnresolvedInstanceFieldGet(HUnresolvedInstanceFieldGet * instruction)4883 void LocationsBuilderX86::VisitUnresolvedInstanceFieldGet(
4884 HUnresolvedInstanceFieldGet* instruction) {
4885 FieldAccessCallingConventionX86 calling_convention;
4886 codegen_->CreateUnresolvedFieldLocationSummary(
4887 instruction, instruction->GetFieldType(), calling_convention);
4888 }
4889
VisitUnresolvedInstanceFieldGet(HUnresolvedInstanceFieldGet * instruction)4890 void InstructionCodeGeneratorX86::VisitUnresolvedInstanceFieldGet(
4891 HUnresolvedInstanceFieldGet* instruction) {
4892 FieldAccessCallingConventionX86 calling_convention;
4893 codegen_->GenerateUnresolvedFieldAccess(instruction,
4894 instruction->GetFieldType(),
4895 instruction->GetFieldIndex(),
4896 instruction->GetDexPc(),
4897 calling_convention);
4898 }
4899
VisitUnresolvedInstanceFieldSet(HUnresolvedInstanceFieldSet * instruction)4900 void LocationsBuilderX86::VisitUnresolvedInstanceFieldSet(
4901 HUnresolvedInstanceFieldSet* instruction) {
4902 FieldAccessCallingConventionX86 calling_convention;
4903 codegen_->CreateUnresolvedFieldLocationSummary(
4904 instruction, instruction->GetFieldType(), calling_convention);
4905 }
4906
VisitUnresolvedInstanceFieldSet(HUnresolvedInstanceFieldSet * instruction)4907 void InstructionCodeGeneratorX86::VisitUnresolvedInstanceFieldSet(
4908 HUnresolvedInstanceFieldSet* instruction) {
4909 FieldAccessCallingConventionX86 calling_convention;
4910 codegen_->GenerateUnresolvedFieldAccess(instruction,
4911 instruction->GetFieldType(),
4912 instruction->GetFieldIndex(),
4913 instruction->GetDexPc(),
4914 calling_convention);
4915 }
4916
VisitUnresolvedStaticFieldGet(HUnresolvedStaticFieldGet * instruction)4917 void LocationsBuilderX86::VisitUnresolvedStaticFieldGet(
4918 HUnresolvedStaticFieldGet* instruction) {
4919 FieldAccessCallingConventionX86 calling_convention;
4920 codegen_->CreateUnresolvedFieldLocationSummary(
4921 instruction, instruction->GetFieldType(), calling_convention);
4922 }
4923
VisitUnresolvedStaticFieldGet(HUnresolvedStaticFieldGet * instruction)4924 void InstructionCodeGeneratorX86::VisitUnresolvedStaticFieldGet(
4925 HUnresolvedStaticFieldGet* instruction) {
4926 FieldAccessCallingConventionX86 calling_convention;
4927 codegen_->GenerateUnresolvedFieldAccess(instruction,
4928 instruction->GetFieldType(),
4929 instruction->GetFieldIndex(),
4930 instruction->GetDexPc(),
4931 calling_convention);
4932 }
4933
VisitUnresolvedStaticFieldSet(HUnresolvedStaticFieldSet * instruction)4934 void LocationsBuilderX86::VisitUnresolvedStaticFieldSet(
4935 HUnresolvedStaticFieldSet* instruction) {
4936 FieldAccessCallingConventionX86 calling_convention;
4937 codegen_->CreateUnresolvedFieldLocationSummary(
4938 instruction, instruction->GetFieldType(), calling_convention);
4939 }
4940
VisitUnresolvedStaticFieldSet(HUnresolvedStaticFieldSet * instruction)4941 void InstructionCodeGeneratorX86::VisitUnresolvedStaticFieldSet(
4942 HUnresolvedStaticFieldSet* instruction) {
4943 FieldAccessCallingConventionX86 calling_convention;
4944 codegen_->GenerateUnresolvedFieldAccess(instruction,
4945 instruction->GetFieldType(),
4946 instruction->GetFieldIndex(),
4947 instruction->GetDexPc(),
4948 calling_convention);
4949 }
4950
VisitNullCheck(HNullCheck * instruction)4951 void LocationsBuilderX86::VisitNullCheck(HNullCheck* instruction) {
4952 LocationSummary::CallKind call_kind = instruction->CanThrowIntoCatchBlock()
4953 ? LocationSummary::kCallOnSlowPath
4954 : LocationSummary::kNoCall;
4955 LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction, call_kind);
4956 Location loc = codegen_->IsImplicitNullCheckAllowed(instruction)
4957 ? Location::RequiresRegister()
4958 : Location::Any();
4959 locations->SetInAt(0, loc);
4960 if (instruction->HasUses()) {
4961 locations->SetOut(Location::SameAsFirstInput());
4962 }
4963 }
4964
GenerateImplicitNullCheck(HNullCheck * instruction)4965 void CodeGeneratorX86::GenerateImplicitNullCheck(HNullCheck* instruction) {
4966 if (CanMoveNullCheckToUser(instruction)) {
4967 return;
4968 }
4969 LocationSummary* locations = instruction->GetLocations();
4970 Location obj = locations->InAt(0);
4971
4972 __ testl(EAX, Address(obj.AsRegister<Register>(), 0));
4973 RecordPcInfo(instruction, instruction->GetDexPc());
4974 }
4975
GenerateExplicitNullCheck(HNullCheck * instruction)4976 void CodeGeneratorX86::GenerateExplicitNullCheck(HNullCheck* instruction) {
4977 SlowPathCode* slow_path = new (GetGraph()->GetArena()) NullCheckSlowPathX86(instruction);
4978 AddSlowPath(slow_path);
4979
4980 LocationSummary* locations = instruction->GetLocations();
4981 Location obj = locations->InAt(0);
4982
4983 if (obj.IsRegister()) {
4984 __ testl(obj.AsRegister<Register>(), obj.AsRegister<Register>());
4985 } else if (obj.IsStackSlot()) {
4986 __ cmpl(Address(ESP, obj.GetStackIndex()), Immediate(0));
4987 } else {
4988 DCHECK(obj.IsConstant()) << obj;
4989 DCHECK(obj.GetConstant()->IsNullConstant());
4990 __ jmp(slow_path->GetEntryLabel());
4991 return;
4992 }
4993 __ j(kEqual, slow_path->GetEntryLabel());
4994 }
4995
VisitNullCheck(HNullCheck * instruction)4996 void InstructionCodeGeneratorX86::VisitNullCheck(HNullCheck* instruction) {
4997 codegen_->GenerateNullCheck(instruction);
4998 }
4999
VisitArrayGet(HArrayGet * instruction)5000 void LocationsBuilderX86::VisitArrayGet(HArrayGet* instruction) {
5001 bool object_array_get_with_read_barrier =
5002 kEmitCompilerReadBarrier && (instruction->GetType() == Primitive::kPrimNot);
5003 LocationSummary* locations =
5004 new (GetGraph()->GetArena()) LocationSummary(instruction,
5005 object_array_get_with_read_barrier ?
5006 LocationSummary::kCallOnSlowPath :
5007 LocationSummary::kNoCall);
5008 locations->SetInAt(0, Location::RequiresRegister());
5009 locations->SetInAt(1, Location::RegisterOrConstant(instruction->InputAt(1)));
5010 if (Primitive::IsFloatingPointType(instruction->GetType())) {
5011 locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap);
5012 } else {
5013 // The output overlaps in case of long: we don't want the low move
5014 // to overwrite the array's location. Likewise, in the case of an
5015 // object array get with read barriers enabled, we do not want the
5016 // move to overwrite the array's location, as we need it to emit
5017 // the read barrier.
5018 locations->SetOut(
5019 Location::RequiresRegister(),
5020 (instruction->GetType() == Primitive::kPrimLong || object_array_get_with_read_barrier) ?
5021 Location::kOutputOverlap :
5022 Location::kNoOutputOverlap);
5023 }
5024 // We need a temporary register for the read barrier marking slow
5025 // path in CodeGeneratorX86::GenerateArrayLoadWithBakerReadBarrier.
5026 if (object_array_get_with_read_barrier && kUseBakerReadBarrier) {
5027 locations->AddTemp(Location::RequiresRegister());
5028 }
5029 }
5030
VisitArrayGet(HArrayGet * instruction)5031 void InstructionCodeGeneratorX86::VisitArrayGet(HArrayGet* instruction) {
5032 LocationSummary* locations = instruction->GetLocations();
5033 Location obj_loc = locations->InAt(0);
5034 Register obj = obj_loc.AsRegister<Register>();
5035 Location index = locations->InAt(1);
5036 Location out_loc = locations->Out();
5037
5038 Primitive::Type type = instruction->GetType();
5039 switch (type) {
5040 case Primitive::kPrimBoolean: {
5041 uint32_t data_offset = mirror::Array::DataOffset(sizeof(uint8_t)).Uint32Value();
5042 Register out = out_loc.AsRegister<Register>();
5043 if (index.IsConstant()) {
5044 __ movzxb(out, Address(obj,
5045 (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_1) + data_offset));
5046 } else {
5047 __ movzxb(out, Address(obj, index.AsRegister<Register>(), TIMES_1, data_offset));
5048 }
5049 break;
5050 }
5051
5052 case Primitive::kPrimByte: {
5053 uint32_t data_offset = mirror::Array::DataOffset(sizeof(int8_t)).Uint32Value();
5054 Register out = out_loc.AsRegister<Register>();
5055 if (index.IsConstant()) {
5056 __ movsxb(out, Address(obj,
5057 (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_1) + data_offset));
5058 } else {
5059 __ movsxb(out, Address(obj, index.AsRegister<Register>(), TIMES_1, data_offset));
5060 }
5061 break;
5062 }
5063
5064 case Primitive::kPrimShort: {
5065 uint32_t data_offset = mirror::Array::DataOffset(sizeof(int16_t)).Uint32Value();
5066 Register out = out_loc.AsRegister<Register>();
5067 if (index.IsConstant()) {
5068 __ movsxw(out, Address(obj,
5069 (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_2) + data_offset));
5070 } else {
5071 __ movsxw(out, Address(obj, index.AsRegister<Register>(), TIMES_2, data_offset));
5072 }
5073 break;
5074 }
5075
5076 case Primitive::kPrimChar: {
5077 uint32_t data_offset = mirror::Array::DataOffset(sizeof(uint16_t)).Uint32Value();
5078 Register out = out_loc.AsRegister<Register>();
5079 if (index.IsConstant()) {
5080 __ movzxw(out, Address(obj,
5081 (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_2) + data_offset));
5082 } else {
5083 __ movzxw(out, Address(obj, index.AsRegister<Register>(), TIMES_2, data_offset));
5084 }
5085 break;
5086 }
5087
5088 case Primitive::kPrimInt: {
5089 uint32_t data_offset = mirror::Array::DataOffset(sizeof(int32_t)).Uint32Value();
5090 Register out = out_loc.AsRegister<Register>();
5091 if (index.IsConstant()) {
5092 __ movl(out, Address(obj,
5093 (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4) + data_offset));
5094 } else {
5095 __ movl(out, Address(obj, index.AsRegister<Register>(), TIMES_4, data_offset));
5096 }
5097 break;
5098 }
5099
5100 case Primitive::kPrimNot: {
5101 static_assert(
5102 sizeof(mirror::HeapReference<mirror::Object>) == sizeof(int32_t),
5103 "art::mirror::HeapReference<art::mirror::Object> and int32_t have different sizes.");
5104 uint32_t data_offset = mirror::Array::DataOffset(sizeof(int32_t)).Uint32Value();
5105 // /* HeapReference<Object> */ out =
5106 // *(obj + data_offset + index * sizeof(HeapReference<Object>))
5107 if (kEmitCompilerReadBarrier && kUseBakerReadBarrier) {
5108 Location temp = locations->GetTemp(0);
5109 // Note that a potential implicit null check is handled in this
5110 // CodeGeneratorX86::GenerateArrayLoadWithBakerReadBarrier call.
5111 codegen_->GenerateArrayLoadWithBakerReadBarrier(
5112 instruction, out_loc, obj, data_offset, index, temp, /* needs_null_check */ true);
5113 } else {
5114 Register out = out_loc.AsRegister<Register>();
5115 if (index.IsConstant()) {
5116 uint32_t offset =
5117 (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4) + data_offset;
5118 __ movl(out, Address(obj, offset));
5119 codegen_->MaybeRecordImplicitNullCheck(instruction);
5120 // If read barriers are enabled, emit read barriers other than
5121 // Baker's using a slow path (and also unpoison the loaded
5122 // reference, if heap poisoning is enabled).
5123 codegen_->MaybeGenerateReadBarrierSlow(instruction, out_loc, out_loc, obj_loc, offset);
5124 } else {
5125 __ movl(out, Address(obj, index.AsRegister<Register>(), TIMES_4, data_offset));
5126 codegen_->MaybeRecordImplicitNullCheck(instruction);
5127 // If read barriers are enabled, emit read barriers other than
5128 // Baker's using a slow path (and also unpoison the loaded
5129 // reference, if heap poisoning is enabled).
5130 codegen_->MaybeGenerateReadBarrierSlow(
5131 instruction, out_loc, out_loc, obj_loc, data_offset, index);
5132 }
5133 }
5134 break;
5135 }
5136
5137 case Primitive::kPrimLong: {
5138 uint32_t data_offset = mirror::Array::DataOffset(sizeof(int64_t)).Uint32Value();
5139 DCHECK_NE(obj, out_loc.AsRegisterPairLow<Register>());
5140 if (index.IsConstant()) {
5141 size_t offset = (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_8) + data_offset;
5142 __ movl(out_loc.AsRegisterPairLow<Register>(), Address(obj, offset));
5143 codegen_->MaybeRecordImplicitNullCheck(instruction);
5144 __ movl(out_loc.AsRegisterPairHigh<Register>(), Address(obj, offset + kX86WordSize));
5145 } else {
5146 __ movl(out_loc.AsRegisterPairLow<Register>(),
5147 Address(obj, index.AsRegister<Register>(), TIMES_8, data_offset));
5148 codegen_->MaybeRecordImplicitNullCheck(instruction);
5149 __ movl(out_loc.AsRegisterPairHigh<Register>(),
5150 Address(obj, index.AsRegister<Register>(), TIMES_8, data_offset + kX86WordSize));
5151 }
5152 break;
5153 }
5154
5155 case Primitive::kPrimFloat: {
5156 uint32_t data_offset = mirror::Array::DataOffset(sizeof(float)).Uint32Value();
5157 XmmRegister out = out_loc.AsFpuRegister<XmmRegister>();
5158 if (index.IsConstant()) {
5159 __ movss(out, Address(obj,
5160 (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4) + data_offset));
5161 } else {
5162 __ movss(out, Address(obj, index.AsRegister<Register>(), TIMES_4, data_offset));
5163 }
5164 break;
5165 }
5166
5167 case Primitive::kPrimDouble: {
5168 uint32_t data_offset = mirror::Array::DataOffset(sizeof(double)).Uint32Value();
5169 XmmRegister out = out_loc.AsFpuRegister<XmmRegister>();
5170 if (index.IsConstant()) {
5171 __ movsd(out, Address(obj,
5172 (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_8) + data_offset));
5173 } else {
5174 __ movsd(out, Address(obj, index.AsRegister<Register>(), TIMES_8, data_offset));
5175 }
5176 break;
5177 }
5178
5179 case Primitive::kPrimVoid:
5180 LOG(FATAL) << "Unreachable type " << type;
5181 UNREACHABLE();
5182 }
5183
5184 if (type == Primitive::kPrimNot || type == Primitive::kPrimLong) {
5185 // Potential implicit null checks, in the case of reference or
5186 // long arrays, are handled in the previous switch statement.
5187 } else {
5188 codegen_->MaybeRecordImplicitNullCheck(instruction);
5189 }
5190 }
5191
VisitArraySet(HArraySet * instruction)5192 void LocationsBuilderX86::VisitArraySet(HArraySet* instruction) {
5193 Primitive::Type value_type = instruction->GetComponentType();
5194
5195 bool needs_write_barrier =
5196 CodeGenerator::StoreNeedsWriteBarrier(value_type, instruction->GetValue());
5197 bool may_need_runtime_call_for_type_check = instruction->NeedsTypeCheck();
5198 bool object_array_set_with_read_barrier =
5199 kEmitCompilerReadBarrier && (value_type == Primitive::kPrimNot);
5200
5201 LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(
5202 instruction,
5203 (may_need_runtime_call_for_type_check || object_array_set_with_read_barrier) ?
5204 LocationSummary::kCallOnSlowPath :
5205 LocationSummary::kNoCall);
5206
5207 bool is_byte_type = (value_type == Primitive::kPrimBoolean)
5208 || (value_type == Primitive::kPrimByte);
5209 // We need the inputs to be different than the output in case of long operation.
5210 // In case of a byte operation, the register allocator does not support multiple
5211 // inputs that die at entry with one in a specific register.
5212 locations->SetInAt(0, Location::RequiresRegister());
5213 locations->SetInAt(1, Location::RegisterOrConstant(instruction->InputAt(1)));
5214 if (is_byte_type) {
5215 // Ensure the value is in a byte register.
5216 locations->SetInAt(2, Location::ByteRegisterOrConstant(EAX, instruction->InputAt(2)));
5217 } else if (Primitive::IsFloatingPointType(value_type)) {
5218 locations->SetInAt(2, Location::FpuRegisterOrConstant(instruction->InputAt(2)));
5219 } else {
5220 locations->SetInAt(2, Location::RegisterOrConstant(instruction->InputAt(2)));
5221 }
5222 if (needs_write_barrier) {
5223 // Temporary registers for the write barrier.
5224 locations->AddTemp(Location::RequiresRegister()); // Possibly used for ref. poisoning too.
5225 // Ensure the card is in a byte register.
5226 locations->AddTemp(Location::RegisterLocation(ECX));
5227 }
5228 }
5229
VisitArraySet(HArraySet * instruction)5230 void InstructionCodeGeneratorX86::VisitArraySet(HArraySet* instruction) {
5231 LocationSummary* locations = instruction->GetLocations();
5232 Location array_loc = locations->InAt(0);
5233 Register array = array_loc.AsRegister<Register>();
5234 Location index = locations->InAt(1);
5235 Location value = locations->InAt(2);
5236 Primitive::Type value_type = instruction->GetComponentType();
5237 uint32_t class_offset = mirror::Object::ClassOffset().Int32Value();
5238 uint32_t super_offset = mirror::Class::SuperClassOffset().Int32Value();
5239 uint32_t component_offset = mirror::Class::ComponentTypeOffset().Int32Value();
5240 bool may_need_runtime_call_for_type_check = instruction->NeedsTypeCheck();
5241 bool needs_write_barrier =
5242 CodeGenerator::StoreNeedsWriteBarrier(value_type, instruction->GetValue());
5243
5244 switch (value_type) {
5245 case Primitive::kPrimBoolean:
5246 case Primitive::kPrimByte: {
5247 uint32_t offset = mirror::Array::DataOffset(sizeof(uint8_t)).Uint32Value();
5248 Address address = index.IsConstant()
5249 ? Address(array, (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_1) + offset)
5250 : Address(array, index.AsRegister<Register>(), TIMES_1, offset);
5251 if (value.IsRegister()) {
5252 __ movb(address, value.AsRegister<ByteRegister>());
5253 } else {
5254 __ movb(address, Immediate(value.GetConstant()->AsIntConstant()->GetValue()));
5255 }
5256 codegen_->MaybeRecordImplicitNullCheck(instruction);
5257 break;
5258 }
5259
5260 case Primitive::kPrimShort:
5261 case Primitive::kPrimChar: {
5262 uint32_t offset = mirror::Array::DataOffset(sizeof(uint16_t)).Uint32Value();
5263 Address address = index.IsConstant()
5264 ? Address(array, (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_2) + offset)
5265 : Address(array, index.AsRegister<Register>(), TIMES_2, offset);
5266 if (value.IsRegister()) {
5267 __ movw(address, value.AsRegister<Register>());
5268 } else {
5269 __ movw(address, Immediate(value.GetConstant()->AsIntConstant()->GetValue()));
5270 }
5271 codegen_->MaybeRecordImplicitNullCheck(instruction);
5272 break;
5273 }
5274
5275 case Primitive::kPrimNot: {
5276 uint32_t offset = mirror::Array::DataOffset(sizeof(int32_t)).Uint32Value();
5277 Address address = index.IsConstant()
5278 ? Address(array, (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4) + offset)
5279 : Address(array, index.AsRegister<Register>(), TIMES_4, offset);
5280
5281 if (!value.IsRegister()) {
5282 // Just setting null.
5283 DCHECK(instruction->InputAt(2)->IsNullConstant());
5284 DCHECK(value.IsConstant()) << value;
5285 __ movl(address, Immediate(0));
5286 codegen_->MaybeRecordImplicitNullCheck(instruction);
5287 DCHECK(!needs_write_barrier);
5288 DCHECK(!may_need_runtime_call_for_type_check);
5289 break;
5290 }
5291
5292 DCHECK(needs_write_barrier);
5293 Register register_value = value.AsRegister<Register>();
5294 NearLabel done, not_null, do_put;
5295 SlowPathCode* slow_path = nullptr;
5296 Register temp = locations->GetTemp(0).AsRegister<Register>();
5297 if (may_need_runtime_call_for_type_check) {
5298 slow_path = new (GetGraph()->GetArena()) ArraySetSlowPathX86(instruction);
5299 codegen_->AddSlowPath(slow_path);
5300 if (instruction->GetValueCanBeNull()) {
5301 __ testl(register_value, register_value);
5302 __ j(kNotEqual, ¬_null);
5303 __ movl(address, Immediate(0));
5304 codegen_->MaybeRecordImplicitNullCheck(instruction);
5305 __ jmp(&done);
5306 __ Bind(¬_null);
5307 }
5308
5309 if (kEmitCompilerReadBarrier) {
5310 // When read barriers are enabled, the type checking
5311 // instrumentation requires two read barriers:
5312 //
5313 // __ movl(temp2, temp);
5314 // // /* HeapReference<Class> */ temp = temp->component_type_
5315 // __ movl(temp, Address(temp, component_offset));
5316 // codegen_->GenerateReadBarrierSlow(
5317 // instruction, temp_loc, temp_loc, temp2_loc, component_offset);
5318 //
5319 // // /* HeapReference<Class> */ temp2 = register_value->klass_
5320 // __ movl(temp2, Address(register_value, class_offset));
5321 // codegen_->GenerateReadBarrierSlow(
5322 // instruction, temp2_loc, temp2_loc, value, class_offset, temp_loc);
5323 //
5324 // __ cmpl(temp, temp2);
5325 //
5326 // However, the second read barrier may trash `temp`, as it
5327 // is a temporary register, and as such would not be saved
5328 // along with live registers before calling the runtime (nor
5329 // restored afterwards). So in this case, we bail out and
5330 // delegate the work to the array set slow path.
5331 //
5332 // TODO: Extend the register allocator to support a new
5333 // "(locally) live temp" location so as to avoid always
5334 // going into the slow path when read barriers are enabled.
5335 __ jmp(slow_path->GetEntryLabel());
5336 } else {
5337 // /* HeapReference<Class> */ temp = array->klass_
5338 __ movl(temp, Address(array, class_offset));
5339 codegen_->MaybeRecordImplicitNullCheck(instruction);
5340 __ MaybeUnpoisonHeapReference(temp);
5341
5342 // /* HeapReference<Class> */ temp = temp->component_type_
5343 __ movl(temp, Address(temp, component_offset));
5344 // If heap poisoning is enabled, no need to unpoison `temp`
5345 // nor the object reference in `register_value->klass`, as
5346 // we are comparing two poisoned references.
5347 __ cmpl(temp, Address(register_value, class_offset));
5348
5349 if (instruction->StaticTypeOfArrayIsObjectArray()) {
5350 __ j(kEqual, &do_put);
5351 // If heap poisoning is enabled, the `temp` reference has
5352 // not been unpoisoned yet; unpoison it now.
5353 __ MaybeUnpoisonHeapReference(temp);
5354
5355 // /* HeapReference<Class> */ temp = temp->super_class_
5356 __ movl(temp, Address(temp, super_offset));
5357 // If heap poisoning is enabled, no need to unpoison
5358 // `temp`, as we are comparing against null below.
5359 __ testl(temp, temp);
5360 __ j(kNotEqual, slow_path->GetEntryLabel());
5361 __ Bind(&do_put);
5362 } else {
5363 __ j(kNotEqual, slow_path->GetEntryLabel());
5364 }
5365 }
5366 }
5367
5368 if (kPoisonHeapReferences) {
5369 __ movl(temp, register_value);
5370 __ PoisonHeapReference(temp);
5371 __ movl(address, temp);
5372 } else {
5373 __ movl(address, register_value);
5374 }
5375 if (!may_need_runtime_call_for_type_check) {
5376 codegen_->MaybeRecordImplicitNullCheck(instruction);
5377 }
5378
5379 Register card = locations->GetTemp(1).AsRegister<Register>();
5380 codegen_->MarkGCCard(
5381 temp, card, array, value.AsRegister<Register>(), instruction->GetValueCanBeNull());
5382 __ Bind(&done);
5383
5384 if (slow_path != nullptr) {
5385 __ Bind(slow_path->GetExitLabel());
5386 }
5387
5388 break;
5389 }
5390
5391 case Primitive::kPrimInt: {
5392 uint32_t offset = mirror::Array::DataOffset(sizeof(int32_t)).Uint32Value();
5393 Address address = index.IsConstant()
5394 ? Address(array, (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4) + offset)
5395 : Address(array, index.AsRegister<Register>(), TIMES_4, offset);
5396 if (value.IsRegister()) {
5397 __ movl(address, value.AsRegister<Register>());
5398 } else {
5399 DCHECK(value.IsConstant()) << value;
5400 int32_t v = CodeGenerator::GetInt32ValueOf(value.GetConstant());
5401 __ movl(address, Immediate(v));
5402 }
5403 codegen_->MaybeRecordImplicitNullCheck(instruction);
5404 break;
5405 }
5406
5407 case Primitive::kPrimLong: {
5408 uint32_t data_offset = mirror::Array::DataOffset(sizeof(int64_t)).Uint32Value();
5409 if (index.IsConstant()) {
5410 size_t offset = (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_8) + data_offset;
5411 if (value.IsRegisterPair()) {
5412 __ movl(Address(array, offset), value.AsRegisterPairLow<Register>());
5413 codegen_->MaybeRecordImplicitNullCheck(instruction);
5414 __ movl(Address(array, offset + kX86WordSize), value.AsRegisterPairHigh<Register>());
5415 } else {
5416 DCHECK(value.IsConstant());
5417 int64_t val = value.GetConstant()->AsLongConstant()->GetValue();
5418 __ movl(Address(array, offset), Immediate(Low32Bits(val)));
5419 codegen_->MaybeRecordImplicitNullCheck(instruction);
5420 __ movl(Address(array, offset + kX86WordSize), Immediate(High32Bits(val)));
5421 }
5422 } else {
5423 if (value.IsRegisterPair()) {
5424 __ movl(Address(array, index.AsRegister<Register>(), TIMES_8, data_offset),
5425 value.AsRegisterPairLow<Register>());
5426 codegen_->MaybeRecordImplicitNullCheck(instruction);
5427 __ movl(Address(array, index.AsRegister<Register>(), TIMES_8, data_offset + kX86WordSize),
5428 value.AsRegisterPairHigh<Register>());
5429 } else {
5430 DCHECK(value.IsConstant());
5431 int64_t val = value.GetConstant()->AsLongConstant()->GetValue();
5432 __ movl(Address(array, index.AsRegister<Register>(), TIMES_8, data_offset),
5433 Immediate(Low32Bits(val)));
5434 codegen_->MaybeRecordImplicitNullCheck(instruction);
5435 __ movl(Address(array, index.AsRegister<Register>(), TIMES_8, data_offset + kX86WordSize),
5436 Immediate(High32Bits(val)));
5437 }
5438 }
5439 break;
5440 }
5441
5442 case Primitive::kPrimFloat: {
5443 uint32_t offset = mirror::Array::DataOffset(sizeof(float)).Uint32Value();
5444 Address address = index.IsConstant()
5445 ? Address(array, (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4) + offset)
5446 : Address(array, index.AsRegister<Register>(), TIMES_4, offset);
5447 if (value.IsFpuRegister()) {
5448 __ movss(address, value.AsFpuRegister<XmmRegister>());
5449 } else {
5450 DCHECK(value.IsConstant());
5451 int32_t v = bit_cast<int32_t, float>(value.GetConstant()->AsFloatConstant()->GetValue());
5452 __ movl(address, Immediate(v));
5453 }
5454 codegen_->MaybeRecordImplicitNullCheck(instruction);
5455 break;
5456 }
5457
5458 case Primitive::kPrimDouble: {
5459 uint32_t offset = mirror::Array::DataOffset(sizeof(double)).Uint32Value();
5460 Address address = index.IsConstant()
5461 ? Address(array, (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_8) + offset)
5462 : Address(array, index.AsRegister<Register>(), TIMES_8, offset);
5463 if (value.IsFpuRegister()) {
5464 __ movsd(address, value.AsFpuRegister<XmmRegister>());
5465 } else {
5466 DCHECK(value.IsConstant());
5467 Address address_hi = index.IsConstant() ?
5468 Address(array, (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_8) +
5469 offset + kX86WordSize) :
5470 Address(array, index.AsRegister<Register>(), TIMES_8, offset + kX86WordSize);
5471 int64_t v = bit_cast<int64_t, double>(value.GetConstant()->AsDoubleConstant()->GetValue());
5472 __ movl(address, Immediate(Low32Bits(v)));
5473 codegen_->MaybeRecordImplicitNullCheck(instruction);
5474 __ movl(address_hi, Immediate(High32Bits(v)));
5475 }
5476 break;
5477 }
5478
5479 case Primitive::kPrimVoid:
5480 LOG(FATAL) << "Unreachable type " << instruction->GetType();
5481 UNREACHABLE();
5482 }
5483 }
5484
VisitArrayLength(HArrayLength * instruction)5485 void LocationsBuilderX86::VisitArrayLength(HArrayLength* instruction) {
5486 LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction);
5487 locations->SetInAt(0, Location::RequiresRegister());
5488 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
5489 }
5490
VisitArrayLength(HArrayLength * instruction)5491 void InstructionCodeGeneratorX86::VisitArrayLength(HArrayLength* instruction) {
5492 LocationSummary* locations = instruction->GetLocations();
5493 uint32_t offset = mirror::Array::LengthOffset().Uint32Value();
5494 Register obj = locations->InAt(0).AsRegister<Register>();
5495 Register out = locations->Out().AsRegister<Register>();
5496 __ movl(out, Address(obj, offset));
5497 codegen_->MaybeRecordImplicitNullCheck(instruction);
5498 }
5499
VisitBoundsCheck(HBoundsCheck * instruction)5500 void LocationsBuilderX86::VisitBoundsCheck(HBoundsCheck* instruction) {
5501 LocationSummary::CallKind call_kind = instruction->CanThrowIntoCatchBlock()
5502 ? LocationSummary::kCallOnSlowPath
5503 : LocationSummary::kNoCall;
5504 LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction, call_kind);
5505 locations->SetInAt(0, Location::RegisterOrConstant(instruction->InputAt(0)));
5506 locations->SetInAt(1, Location::RegisterOrConstant(instruction->InputAt(1)));
5507 if (instruction->HasUses()) {
5508 locations->SetOut(Location::SameAsFirstInput());
5509 }
5510 }
5511
VisitBoundsCheck(HBoundsCheck * instruction)5512 void InstructionCodeGeneratorX86::VisitBoundsCheck(HBoundsCheck* instruction) {
5513 LocationSummary* locations = instruction->GetLocations();
5514 Location index_loc = locations->InAt(0);
5515 Location length_loc = locations->InAt(1);
5516 SlowPathCode* slow_path =
5517 new (GetGraph()->GetArena()) BoundsCheckSlowPathX86(instruction);
5518
5519 if (length_loc.IsConstant()) {
5520 int32_t length = CodeGenerator::GetInt32ValueOf(length_loc.GetConstant());
5521 if (index_loc.IsConstant()) {
5522 // BCE will remove the bounds check if we are guarenteed to pass.
5523 int32_t index = CodeGenerator::GetInt32ValueOf(index_loc.GetConstant());
5524 if (index < 0 || index >= length) {
5525 codegen_->AddSlowPath(slow_path);
5526 __ jmp(slow_path->GetEntryLabel());
5527 } else {
5528 // Some optimization after BCE may have generated this, and we should not
5529 // generate a bounds check if it is a valid range.
5530 }
5531 return;
5532 }
5533
5534 // We have to reverse the jump condition because the length is the constant.
5535 Register index_reg = index_loc.AsRegister<Register>();
5536 __ cmpl(index_reg, Immediate(length));
5537 codegen_->AddSlowPath(slow_path);
5538 __ j(kAboveEqual, slow_path->GetEntryLabel());
5539 } else {
5540 Register length = length_loc.AsRegister<Register>();
5541 if (index_loc.IsConstant()) {
5542 int32_t value = CodeGenerator::GetInt32ValueOf(index_loc.GetConstant());
5543 __ cmpl(length, Immediate(value));
5544 } else {
5545 __ cmpl(length, index_loc.AsRegister<Register>());
5546 }
5547 codegen_->AddSlowPath(slow_path);
5548 __ j(kBelowEqual, slow_path->GetEntryLabel());
5549 }
5550 }
5551
VisitParallelMove(HParallelMove * instruction ATTRIBUTE_UNUSED)5552 void LocationsBuilderX86::VisitParallelMove(HParallelMove* instruction ATTRIBUTE_UNUSED) {
5553 LOG(FATAL) << "Unreachable";
5554 }
5555
VisitParallelMove(HParallelMove * instruction)5556 void InstructionCodeGeneratorX86::VisitParallelMove(HParallelMove* instruction) {
5557 codegen_->GetMoveResolver()->EmitNativeCode(instruction);
5558 }
5559
VisitSuspendCheck(HSuspendCheck * instruction)5560 void LocationsBuilderX86::VisitSuspendCheck(HSuspendCheck* instruction) {
5561 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCallOnSlowPath);
5562 }
5563
VisitSuspendCheck(HSuspendCheck * instruction)5564 void InstructionCodeGeneratorX86::VisitSuspendCheck(HSuspendCheck* instruction) {
5565 HBasicBlock* block = instruction->GetBlock();
5566 if (block->GetLoopInformation() != nullptr) {
5567 DCHECK(block->GetLoopInformation()->GetSuspendCheck() == instruction);
5568 // The back edge will generate the suspend check.
5569 return;
5570 }
5571 if (block->IsEntryBlock() && instruction->GetNext()->IsGoto()) {
5572 // The goto will generate the suspend check.
5573 return;
5574 }
5575 GenerateSuspendCheck(instruction, nullptr);
5576 }
5577
GenerateSuspendCheck(HSuspendCheck * instruction,HBasicBlock * successor)5578 void InstructionCodeGeneratorX86::GenerateSuspendCheck(HSuspendCheck* instruction,
5579 HBasicBlock* successor) {
5580 SuspendCheckSlowPathX86* slow_path =
5581 down_cast<SuspendCheckSlowPathX86*>(instruction->GetSlowPath());
5582 if (slow_path == nullptr) {
5583 slow_path = new (GetGraph()->GetArena()) SuspendCheckSlowPathX86(instruction, successor);
5584 instruction->SetSlowPath(slow_path);
5585 codegen_->AddSlowPath(slow_path);
5586 if (successor != nullptr) {
5587 DCHECK(successor->IsLoopHeader());
5588 codegen_->ClearSpillSlotsFromLoopPhisInStackMap(instruction);
5589 }
5590 } else {
5591 DCHECK_EQ(slow_path->GetSuccessor(), successor);
5592 }
5593
5594 __ fs()->cmpw(Address::Absolute(Thread::ThreadFlagsOffset<kX86WordSize>().Int32Value()),
5595 Immediate(0));
5596 if (successor == nullptr) {
5597 __ j(kNotEqual, slow_path->GetEntryLabel());
5598 __ Bind(slow_path->GetReturnLabel());
5599 } else {
5600 __ j(kEqual, codegen_->GetLabelOf(successor));
5601 __ jmp(slow_path->GetEntryLabel());
5602 }
5603 }
5604
GetAssembler() const5605 X86Assembler* ParallelMoveResolverX86::GetAssembler() const {
5606 return codegen_->GetAssembler();
5607 }
5608
MoveMemoryToMemory32(int dst,int src)5609 void ParallelMoveResolverX86::MoveMemoryToMemory32(int dst, int src) {
5610 ScratchRegisterScope ensure_scratch(
5611 this, kNoRegister, EAX, codegen_->GetNumberOfCoreRegisters());
5612 Register temp_reg = static_cast<Register>(ensure_scratch.GetRegister());
5613 int stack_offset = ensure_scratch.IsSpilled() ? kX86WordSize : 0;
5614 __ movl(temp_reg, Address(ESP, src + stack_offset));
5615 __ movl(Address(ESP, dst + stack_offset), temp_reg);
5616 }
5617
MoveMemoryToMemory64(int dst,int src)5618 void ParallelMoveResolverX86::MoveMemoryToMemory64(int dst, int src) {
5619 ScratchRegisterScope ensure_scratch(
5620 this, kNoRegister, EAX, codegen_->GetNumberOfCoreRegisters());
5621 Register temp_reg = static_cast<Register>(ensure_scratch.GetRegister());
5622 int stack_offset = ensure_scratch.IsSpilled() ? kX86WordSize : 0;
5623 __ movl(temp_reg, Address(ESP, src + stack_offset));
5624 __ movl(Address(ESP, dst + stack_offset), temp_reg);
5625 __ movl(temp_reg, Address(ESP, src + stack_offset + kX86WordSize));
5626 __ movl(Address(ESP, dst + stack_offset + kX86WordSize), temp_reg);
5627 }
5628
EmitMove(size_t index)5629 void ParallelMoveResolverX86::EmitMove(size_t index) {
5630 MoveOperands* move = moves_[index];
5631 Location source = move->GetSource();
5632 Location destination = move->GetDestination();
5633
5634 if (source.IsRegister()) {
5635 if (destination.IsRegister()) {
5636 __ movl(destination.AsRegister<Register>(), source.AsRegister<Register>());
5637 } else if (destination.IsFpuRegister()) {
5638 __ movd(destination.AsFpuRegister<XmmRegister>(), source.AsRegister<Register>());
5639 } else {
5640 DCHECK(destination.IsStackSlot());
5641 __ movl(Address(ESP, destination.GetStackIndex()), source.AsRegister<Register>());
5642 }
5643 } else if (source.IsRegisterPair()) {
5644 size_t elem_size = Primitive::ComponentSize(Primitive::kPrimInt);
5645 // Create stack space for 2 elements.
5646 __ subl(ESP, Immediate(2 * elem_size));
5647 __ movl(Address(ESP, 0), source.AsRegisterPairLow<Register>());
5648 __ movl(Address(ESP, elem_size), source.AsRegisterPairHigh<Register>());
5649 __ movsd(destination.AsFpuRegister<XmmRegister>(), Address(ESP, 0));
5650 // And remove the temporary stack space we allocated.
5651 __ addl(ESP, Immediate(2 * elem_size));
5652 } else if (source.IsFpuRegister()) {
5653 if (destination.IsRegister()) {
5654 __ movd(destination.AsRegister<Register>(), source.AsFpuRegister<XmmRegister>());
5655 } else if (destination.IsFpuRegister()) {
5656 __ movaps(destination.AsFpuRegister<XmmRegister>(), source.AsFpuRegister<XmmRegister>());
5657 } else if (destination.IsRegisterPair()) {
5658 XmmRegister src_reg = source.AsFpuRegister<XmmRegister>();
5659 __ movd(destination.AsRegisterPairLow<Register>(), src_reg);
5660 __ psrlq(src_reg, Immediate(32));
5661 __ movd(destination.AsRegisterPairHigh<Register>(), src_reg);
5662 } else if (destination.IsStackSlot()) {
5663 __ movss(Address(ESP, destination.GetStackIndex()), source.AsFpuRegister<XmmRegister>());
5664 } else {
5665 DCHECK(destination.IsDoubleStackSlot());
5666 __ movsd(Address(ESP, destination.GetStackIndex()), source.AsFpuRegister<XmmRegister>());
5667 }
5668 } else if (source.IsStackSlot()) {
5669 if (destination.IsRegister()) {
5670 __ movl(destination.AsRegister<Register>(), Address(ESP, source.GetStackIndex()));
5671 } else if (destination.IsFpuRegister()) {
5672 __ movss(destination.AsFpuRegister<XmmRegister>(), Address(ESP, source.GetStackIndex()));
5673 } else {
5674 DCHECK(destination.IsStackSlot());
5675 MoveMemoryToMemory32(destination.GetStackIndex(), source.GetStackIndex());
5676 }
5677 } else if (source.IsDoubleStackSlot()) {
5678 if (destination.IsRegisterPair()) {
5679 __ movl(destination.AsRegisterPairLow<Register>(), Address(ESP, source.GetStackIndex()));
5680 __ movl(destination.AsRegisterPairHigh<Register>(),
5681 Address(ESP, source.GetHighStackIndex(kX86WordSize)));
5682 } else if (destination.IsFpuRegister()) {
5683 __ movsd(destination.AsFpuRegister<XmmRegister>(), Address(ESP, source.GetStackIndex()));
5684 } else {
5685 DCHECK(destination.IsDoubleStackSlot()) << destination;
5686 MoveMemoryToMemory64(destination.GetStackIndex(), source.GetStackIndex());
5687 }
5688 } else if (source.IsConstant()) {
5689 HConstant* constant = source.GetConstant();
5690 if (constant->IsIntConstant() || constant->IsNullConstant()) {
5691 int32_t value = CodeGenerator::GetInt32ValueOf(constant);
5692 if (destination.IsRegister()) {
5693 if (value == 0) {
5694 __ xorl(destination.AsRegister<Register>(), destination.AsRegister<Register>());
5695 } else {
5696 __ movl(destination.AsRegister<Register>(), Immediate(value));
5697 }
5698 } else {
5699 DCHECK(destination.IsStackSlot()) << destination;
5700 __ movl(Address(ESP, destination.GetStackIndex()), Immediate(value));
5701 }
5702 } else if (constant->IsFloatConstant()) {
5703 float fp_value = constant->AsFloatConstant()->GetValue();
5704 int32_t value = bit_cast<int32_t, float>(fp_value);
5705 Immediate imm(value);
5706 if (destination.IsFpuRegister()) {
5707 XmmRegister dest = destination.AsFpuRegister<XmmRegister>();
5708 if (value == 0) {
5709 // Easy handling of 0.0.
5710 __ xorps(dest, dest);
5711 } else {
5712 ScratchRegisterScope ensure_scratch(
5713 this, kNoRegister, EAX, codegen_->GetNumberOfCoreRegisters());
5714 Register temp = static_cast<Register>(ensure_scratch.GetRegister());
5715 __ movl(temp, Immediate(value));
5716 __ movd(dest, temp);
5717 }
5718 } else {
5719 DCHECK(destination.IsStackSlot()) << destination;
5720 __ movl(Address(ESP, destination.GetStackIndex()), imm);
5721 }
5722 } else if (constant->IsLongConstant()) {
5723 int64_t value = constant->AsLongConstant()->GetValue();
5724 int32_t low_value = Low32Bits(value);
5725 int32_t high_value = High32Bits(value);
5726 Immediate low(low_value);
5727 Immediate high(high_value);
5728 if (destination.IsDoubleStackSlot()) {
5729 __ movl(Address(ESP, destination.GetStackIndex()), low);
5730 __ movl(Address(ESP, destination.GetHighStackIndex(kX86WordSize)), high);
5731 } else {
5732 __ movl(destination.AsRegisterPairLow<Register>(), low);
5733 __ movl(destination.AsRegisterPairHigh<Register>(), high);
5734 }
5735 } else {
5736 DCHECK(constant->IsDoubleConstant());
5737 double dbl_value = constant->AsDoubleConstant()->GetValue();
5738 int64_t value = bit_cast<int64_t, double>(dbl_value);
5739 int32_t low_value = Low32Bits(value);
5740 int32_t high_value = High32Bits(value);
5741 Immediate low(low_value);
5742 Immediate high(high_value);
5743 if (destination.IsFpuRegister()) {
5744 XmmRegister dest = destination.AsFpuRegister<XmmRegister>();
5745 if (value == 0) {
5746 // Easy handling of 0.0.
5747 __ xorpd(dest, dest);
5748 } else {
5749 __ pushl(high);
5750 __ pushl(low);
5751 __ movsd(dest, Address(ESP, 0));
5752 __ addl(ESP, Immediate(8));
5753 }
5754 } else {
5755 DCHECK(destination.IsDoubleStackSlot()) << destination;
5756 __ movl(Address(ESP, destination.GetStackIndex()), low);
5757 __ movl(Address(ESP, destination.GetHighStackIndex(kX86WordSize)), high);
5758 }
5759 }
5760 } else {
5761 LOG(FATAL) << "Unimplemented move: " << destination << " <- " << source;
5762 }
5763 }
5764
Exchange(Register reg,int mem)5765 void ParallelMoveResolverX86::Exchange(Register reg, int mem) {
5766 Register suggested_scratch = reg == EAX ? EBX : EAX;
5767 ScratchRegisterScope ensure_scratch(
5768 this, reg, suggested_scratch, codegen_->GetNumberOfCoreRegisters());
5769
5770 int stack_offset = ensure_scratch.IsSpilled() ? kX86WordSize : 0;
5771 __ movl(static_cast<Register>(ensure_scratch.GetRegister()), Address(ESP, mem + stack_offset));
5772 __ movl(Address(ESP, mem + stack_offset), reg);
5773 __ movl(reg, static_cast<Register>(ensure_scratch.GetRegister()));
5774 }
5775
Exchange32(XmmRegister reg,int mem)5776 void ParallelMoveResolverX86::Exchange32(XmmRegister reg, int mem) {
5777 ScratchRegisterScope ensure_scratch(
5778 this, kNoRegister, EAX, codegen_->GetNumberOfCoreRegisters());
5779
5780 Register temp_reg = static_cast<Register>(ensure_scratch.GetRegister());
5781 int stack_offset = ensure_scratch.IsSpilled() ? kX86WordSize : 0;
5782 __ movl(temp_reg, Address(ESP, mem + stack_offset));
5783 __ movss(Address(ESP, mem + stack_offset), reg);
5784 __ movd(reg, temp_reg);
5785 }
5786
Exchange(int mem1,int mem2)5787 void ParallelMoveResolverX86::Exchange(int mem1, int mem2) {
5788 ScratchRegisterScope ensure_scratch1(
5789 this, kNoRegister, EAX, codegen_->GetNumberOfCoreRegisters());
5790
5791 Register suggested_scratch = ensure_scratch1.GetRegister() == EAX ? EBX : EAX;
5792 ScratchRegisterScope ensure_scratch2(
5793 this, ensure_scratch1.GetRegister(), suggested_scratch, codegen_->GetNumberOfCoreRegisters());
5794
5795 int stack_offset = ensure_scratch1.IsSpilled() ? kX86WordSize : 0;
5796 stack_offset += ensure_scratch2.IsSpilled() ? kX86WordSize : 0;
5797 __ movl(static_cast<Register>(ensure_scratch1.GetRegister()), Address(ESP, mem1 + stack_offset));
5798 __ movl(static_cast<Register>(ensure_scratch2.GetRegister()), Address(ESP, mem2 + stack_offset));
5799 __ movl(Address(ESP, mem2 + stack_offset), static_cast<Register>(ensure_scratch1.GetRegister()));
5800 __ movl(Address(ESP, mem1 + stack_offset), static_cast<Register>(ensure_scratch2.GetRegister()));
5801 }
5802
EmitSwap(size_t index)5803 void ParallelMoveResolverX86::EmitSwap(size_t index) {
5804 MoveOperands* move = moves_[index];
5805 Location source = move->GetSource();
5806 Location destination = move->GetDestination();
5807
5808 if (source.IsRegister() && destination.IsRegister()) {
5809 // Use XOR swap algorithm to avoid serializing XCHG instruction or using a temporary.
5810 DCHECK_NE(destination.AsRegister<Register>(), source.AsRegister<Register>());
5811 __ xorl(destination.AsRegister<Register>(), source.AsRegister<Register>());
5812 __ xorl(source.AsRegister<Register>(), destination.AsRegister<Register>());
5813 __ xorl(destination.AsRegister<Register>(), source.AsRegister<Register>());
5814 } else if (source.IsRegister() && destination.IsStackSlot()) {
5815 Exchange(source.AsRegister<Register>(), destination.GetStackIndex());
5816 } else if (source.IsStackSlot() && destination.IsRegister()) {
5817 Exchange(destination.AsRegister<Register>(), source.GetStackIndex());
5818 } else if (source.IsStackSlot() && destination.IsStackSlot()) {
5819 Exchange(destination.GetStackIndex(), source.GetStackIndex());
5820 } else if (source.IsFpuRegister() && destination.IsFpuRegister()) {
5821 // Use XOR Swap algorithm to avoid a temporary.
5822 DCHECK_NE(source.reg(), destination.reg());
5823 __ xorpd(destination.AsFpuRegister<XmmRegister>(), source.AsFpuRegister<XmmRegister>());
5824 __ xorpd(source.AsFpuRegister<XmmRegister>(), destination.AsFpuRegister<XmmRegister>());
5825 __ xorpd(destination.AsFpuRegister<XmmRegister>(), source.AsFpuRegister<XmmRegister>());
5826 } else if (source.IsFpuRegister() && destination.IsStackSlot()) {
5827 Exchange32(source.AsFpuRegister<XmmRegister>(), destination.GetStackIndex());
5828 } else if (destination.IsFpuRegister() && source.IsStackSlot()) {
5829 Exchange32(destination.AsFpuRegister<XmmRegister>(), source.GetStackIndex());
5830 } else if (source.IsFpuRegister() && destination.IsDoubleStackSlot()) {
5831 // Take advantage of the 16 bytes in the XMM register.
5832 XmmRegister reg = source.AsFpuRegister<XmmRegister>();
5833 Address stack(ESP, destination.GetStackIndex());
5834 // Load the double into the high doubleword.
5835 __ movhpd(reg, stack);
5836
5837 // Store the low double into the destination.
5838 __ movsd(stack, reg);
5839
5840 // Move the high double to the low double.
5841 __ psrldq(reg, Immediate(8));
5842 } else if (destination.IsFpuRegister() && source.IsDoubleStackSlot()) {
5843 // Take advantage of the 16 bytes in the XMM register.
5844 XmmRegister reg = destination.AsFpuRegister<XmmRegister>();
5845 Address stack(ESP, source.GetStackIndex());
5846 // Load the double into the high doubleword.
5847 __ movhpd(reg, stack);
5848
5849 // Store the low double into the destination.
5850 __ movsd(stack, reg);
5851
5852 // Move the high double to the low double.
5853 __ psrldq(reg, Immediate(8));
5854 } else if (destination.IsDoubleStackSlot() && source.IsDoubleStackSlot()) {
5855 Exchange(destination.GetStackIndex(), source.GetStackIndex());
5856 Exchange(destination.GetHighStackIndex(kX86WordSize), source.GetHighStackIndex(kX86WordSize));
5857 } else {
5858 LOG(FATAL) << "Unimplemented: source: " << source << ", destination: " << destination;
5859 }
5860 }
5861
SpillScratch(int reg)5862 void ParallelMoveResolverX86::SpillScratch(int reg) {
5863 __ pushl(static_cast<Register>(reg));
5864 }
5865
RestoreScratch(int reg)5866 void ParallelMoveResolverX86::RestoreScratch(int reg) {
5867 __ popl(static_cast<Register>(reg));
5868 }
5869
VisitLoadClass(HLoadClass * cls)5870 void LocationsBuilderX86::VisitLoadClass(HLoadClass* cls) {
5871 InvokeRuntimeCallingConvention calling_convention;
5872 CodeGenerator::CreateLoadClassLocationSummary(
5873 cls,
5874 Location::RegisterLocation(calling_convention.GetRegisterAt(0)),
5875 Location::RegisterLocation(EAX),
5876 /* code_generator_supports_read_barrier */ true);
5877 }
5878
VisitLoadClass(HLoadClass * cls)5879 void InstructionCodeGeneratorX86::VisitLoadClass(HLoadClass* cls) {
5880 LocationSummary* locations = cls->GetLocations();
5881 if (cls->NeedsAccessCheck()) {
5882 codegen_->MoveConstant(locations->GetTemp(0), cls->GetTypeIndex());
5883 codegen_->InvokeRuntime(QUICK_ENTRY_POINT(pInitializeTypeAndVerifyAccess),
5884 cls,
5885 cls->GetDexPc(),
5886 nullptr);
5887 CheckEntrypointTypes<kQuickInitializeTypeAndVerifyAccess, void*, uint32_t>();
5888 return;
5889 }
5890
5891 Location out_loc = locations->Out();
5892 Register out = out_loc.AsRegister<Register>();
5893 Register current_method = locations->InAt(0).AsRegister<Register>();
5894
5895 if (cls->IsReferrersClass()) {
5896 DCHECK(!cls->CanCallRuntime());
5897 DCHECK(!cls->MustGenerateClinitCheck());
5898 // /* GcRoot<mirror::Class> */ out = current_method->declaring_class_
5899 GenerateGcRootFieldLoad(
5900 cls, out_loc, Address(current_method, ArtMethod::DeclaringClassOffset().Int32Value()));
5901 } else {
5902 // /* GcRoot<mirror::Class>[] */ out =
5903 // current_method.ptr_sized_fields_->dex_cache_resolved_types_
5904 __ movl(out, Address(current_method,
5905 ArtMethod::DexCacheResolvedTypesOffset(kX86PointerSize).Int32Value()));
5906 // /* GcRoot<mirror::Class> */ out = out[type_index]
5907 GenerateGcRootFieldLoad(
5908 cls, out_loc, Address(out, CodeGenerator::GetCacheOffset(cls->GetTypeIndex())));
5909
5910 if (!cls->IsInDexCache() || cls->MustGenerateClinitCheck()) {
5911 DCHECK(cls->CanCallRuntime());
5912 SlowPathCode* slow_path = new (GetGraph()->GetArena()) LoadClassSlowPathX86(
5913 cls, cls, cls->GetDexPc(), cls->MustGenerateClinitCheck());
5914 codegen_->AddSlowPath(slow_path);
5915
5916 if (!cls->IsInDexCache()) {
5917 __ testl(out, out);
5918 __ j(kEqual, slow_path->GetEntryLabel());
5919 }
5920
5921 if (cls->MustGenerateClinitCheck()) {
5922 GenerateClassInitializationCheck(slow_path, out);
5923 } else {
5924 __ Bind(slow_path->GetExitLabel());
5925 }
5926 }
5927 }
5928 }
5929
VisitClinitCheck(HClinitCheck * check)5930 void LocationsBuilderX86::VisitClinitCheck(HClinitCheck* check) {
5931 LocationSummary* locations =
5932 new (GetGraph()->GetArena()) LocationSummary(check, LocationSummary::kCallOnSlowPath);
5933 locations->SetInAt(0, Location::RequiresRegister());
5934 if (check->HasUses()) {
5935 locations->SetOut(Location::SameAsFirstInput());
5936 }
5937 }
5938
VisitClinitCheck(HClinitCheck * check)5939 void InstructionCodeGeneratorX86::VisitClinitCheck(HClinitCheck* check) {
5940 // We assume the class to not be null.
5941 SlowPathCode* slow_path = new (GetGraph()->GetArena()) LoadClassSlowPathX86(
5942 check->GetLoadClass(), check, check->GetDexPc(), true);
5943 codegen_->AddSlowPath(slow_path);
5944 GenerateClassInitializationCheck(slow_path,
5945 check->GetLocations()->InAt(0).AsRegister<Register>());
5946 }
5947
GenerateClassInitializationCheck(SlowPathCode * slow_path,Register class_reg)5948 void InstructionCodeGeneratorX86::GenerateClassInitializationCheck(
5949 SlowPathCode* slow_path, Register class_reg) {
5950 __ cmpl(Address(class_reg, mirror::Class::StatusOffset().Int32Value()),
5951 Immediate(mirror::Class::kStatusInitialized));
5952 __ j(kLess, slow_path->GetEntryLabel());
5953 __ Bind(slow_path->GetExitLabel());
5954 // No need for memory fence, thanks to the X86 memory model.
5955 }
5956
GetSupportedLoadStringKind(HLoadString::LoadKind desired_string_load_kind)5957 HLoadString::LoadKind CodeGeneratorX86::GetSupportedLoadStringKind(
5958 HLoadString::LoadKind desired_string_load_kind) {
5959 if (kEmitCompilerReadBarrier) {
5960 switch (desired_string_load_kind) {
5961 case HLoadString::LoadKind::kBootImageLinkTimeAddress:
5962 case HLoadString::LoadKind::kBootImageLinkTimePcRelative:
5963 case HLoadString::LoadKind::kBootImageAddress:
5964 // TODO: Implement for read barrier.
5965 return HLoadString::LoadKind::kDexCacheViaMethod;
5966 default:
5967 break;
5968 }
5969 }
5970 switch (desired_string_load_kind) {
5971 case HLoadString::LoadKind::kBootImageLinkTimeAddress:
5972 DCHECK(!GetCompilerOptions().GetCompilePic());
5973 break;
5974 case HLoadString::LoadKind::kBootImageLinkTimePcRelative:
5975 DCHECK(GetCompilerOptions().GetCompilePic());
5976 FALLTHROUGH_INTENDED;
5977 case HLoadString::LoadKind::kDexCachePcRelative:
5978 DCHECK(!Runtime::Current()->UseJitCompilation()); // Note: boot image is also non-JIT.
5979 // We disable pc-relative load when there is an irreducible loop, as the optimization
5980 // is incompatible with it.
5981 // TODO: Create as many X86ComputeBaseMethodAddress instructions as needed for methods
5982 // with irreducible loops.
5983 if (GetGraph()->HasIrreducibleLoops()) {
5984 return HLoadString::LoadKind::kDexCacheViaMethod;
5985 }
5986 break;
5987 case HLoadString::LoadKind::kBootImageAddress:
5988 break;
5989 case HLoadString::LoadKind::kDexCacheAddress:
5990 DCHECK(Runtime::Current()->UseJitCompilation());
5991 break;
5992 case HLoadString::LoadKind::kDexCacheViaMethod:
5993 break;
5994 }
5995 return desired_string_load_kind;
5996 }
5997
VisitLoadString(HLoadString * load)5998 void LocationsBuilderX86::VisitLoadString(HLoadString* load) {
5999 LocationSummary::CallKind call_kind = (load->NeedsEnvironment() || kEmitCompilerReadBarrier)
6000 ? LocationSummary::kCallOnSlowPath
6001 : LocationSummary::kNoCall;
6002 LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(load, call_kind);
6003 HLoadString::LoadKind load_kind = load->GetLoadKind();
6004 if (load_kind == HLoadString::LoadKind::kDexCacheViaMethod ||
6005 load_kind == HLoadString::LoadKind::kBootImageLinkTimePcRelative ||
6006 load_kind == HLoadString::LoadKind::kDexCachePcRelative) {
6007 locations->SetInAt(0, Location::RequiresRegister());
6008 }
6009 locations->SetOut(Location::RequiresRegister());
6010 }
6011
VisitLoadString(HLoadString * load)6012 void InstructionCodeGeneratorX86::VisitLoadString(HLoadString* load) {
6013 LocationSummary* locations = load->GetLocations();
6014 Location out_loc = locations->Out();
6015 Register out = out_loc.AsRegister<Register>();
6016
6017 switch (load->GetLoadKind()) {
6018 case HLoadString::LoadKind::kBootImageLinkTimeAddress: {
6019 DCHECK(!kEmitCompilerReadBarrier);
6020 __ movl(out, Immediate(/* placeholder */ 0));
6021 codegen_->RecordStringPatch(load);
6022 return; // No dex cache slow path.
6023 }
6024 case HLoadString::LoadKind::kBootImageLinkTimePcRelative: {
6025 DCHECK(!kEmitCompilerReadBarrier);
6026 Register method_address = locations->InAt(0).AsRegister<Register>();
6027 __ leal(out, Address(method_address, CodeGeneratorX86::kDummy32BitOffset));
6028 codegen_->RecordStringPatch(load);
6029 return; // No dex cache slow path.
6030 }
6031 case HLoadString::LoadKind::kBootImageAddress: {
6032 DCHECK(!kEmitCompilerReadBarrier);
6033 DCHECK_NE(load->GetAddress(), 0u);
6034 uint32_t address = dchecked_integral_cast<uint32_t>(load->GetAddress());
6035 __ movl(out, Immediate(address));
6036 codegen_->RecordSimplePatch();
6037 return; // No dex cache slow path.
6038 }
6039 case HLoadString::LoadKind::kDexCacheAddress: {
6040 DCHECK_NE(load->GetAddress(), 0u);
6041 uint32_t address = dchecked_integral_cast<uint32_t>(load->GetAddress());
6042 GenerateGcRootFieldLoad(load, out_loc, Address::Absolute(address));
6043 break;
6044 }
6045 case HLoadString::LoadKind::kDexCachePcRelative: {
6046 Register base_reg = locations->InAt(0).AsRegister<Register>();
6047 uint32_t offset = load->GetDexCacheElementOffset();
6048 Label* fixup_label = codegen_->NewPcRelativeDexCacheArrayPatch(load->GetDexFile(), offset);
6049 GenerateGcRootFieldLoad(
6050 load, out_loc, Address(base_reg, CodeGeneratorX86::kDummy32BitOffset), fixup_label);
6051 break;
6052 }
6053 case HLoadString::LoadKind::kDexCacheViaMethod: {
6054 Register current_method = locations->InAt(0).AsRegister<Register>();
6055
6056 // /* GcRoot<mirror::Class> */ out = current_method->declaring_class_
6057 GenerateGcRootFieldLoad(
6058 load, out_loc, Address(current_method, ArtMethod::DeclaringClassOffset().Int32Value()));
6059
6060 // /* GcRoot<mirror::String>[] */ out = out->dex_cache_strings_
6061 __ movl(out, Address(out, mirror::Class::DexCacheStringsOffset().Int32Value()));
6062 // /* GcRoot<mirror::String> */ out = out[string_index]
6063 GenerateGcRootFieldLoad(
6064 load, out_loc, Address(out, CodeGenerator::GetCacheOffset(load->GetStringIndex())));
6065 break;
6066 }
6067 default:
6068 LOG(FATAL) << "Unexpected load kind: " << load->GetLoadKind();
6069 UNREACHABLE();
6070 }
6071
6072 if (!load->IsInDexCache()) {
6073 SlowPathCode* slow_path = new (GetGraph()->GetArena()) LoadStringSlowPathX86(load);
6074 codegen_->AddSlowPath(slow_path);
6075 __ testl(out, out);
6076 __ j(kEqual, slow_path->GetEntryLabel());
6077 __ Bind(slow_path->GetExitLabel());
6078 }
6079 }
6080
GetExceptionTlsAddress()6081 static Address GetExceptionTlsAddress() {
6082 return Address::Absolute(Thread::ExceptionOffset<kX86WordSize>().Int32Value());
6083 }
6084
VisitLoadException(HLoadException * load)6085 void LocationsBuilderX86::VisitLoadException(HLoadException* load) {
6086 LocationSummary* locations =
6087 new (GetGraph()->GetArena()) LocationSummary(load, LocationSummary::kNoCall);
6088 locations->SetOut(Location::RequiresRegister());
6089 }
6090
VisitLoadException(HLoadException * load)6091 void InstructionCodeGeneratorX86::VisitLoadException(HLoadException* load) {
6092 __ fs()->movl(load->GetLocations()->Out().AsRegister<Register>(), GetExceptionTlsAddress());
6093 }
6094
VisitClearException(HClearException * clear)6095 void LocationsBuilderX86::VisitClearException(HClearException* clear) {
6096 new (GetGraph()->GetArena()) LocationSummary(clear, LocationSummary::kNoCall);
6097 }
6098
VisitClearException(HClearException * clear ATTRIBUTE_UNUSED)6099 void InstructionCodeGeneratorX86::VisitClearException(HClearException* clear ATTRIBUTE_UNUSED) {
6100 __ fs()->movl(GetExceptionTlsAddress(), Immediate(0));
6101 }
6102
VisitThrow(HThrow * instruction)6103 void LocationsBuilderX86::VisitThrow(HThrow* instruction) {
6104 LocationSummary* locations =
6105 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCall);
6106 InvokeRuntimeCallingConvention calling_convention;
6107 locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(0)));
6108 }
6109
VisitThrow(HThrow * instruction)6110 void InstructionCodeGeneratorX86::VisitThrow(HThrow* instruction) {
6111 codegen_->InvokeRuntime(QUICK_ENTRY_POINT(pDeliverException),
6112 instruction,
6113 instruction->GetDexPc(),
6114 nullptr);
6115 CheckEntrypointTypes<kQuickDeliverException, void, mirror::Object*>();
6116 }
6117
TypeCheckNeedsATemporary(TypeCheckKind type_check_kind)6118 static bool TypeCheckNeedsATemporary(TypeCheckKind type_check_kind) {
6119 return kEmitCompilerReadBarrier &&
6120 (kUseBakerReadBarrier ||
6121 type_check_kind == TypeCheckKind::kAbstractClassCheck ||
6122 type_check_kind == TypeCheckKind::kClassHierarchyCheck ||
6123 type_check_kind == TypeCheckKind::kArrayObjectCheck);
6124 }
6125
VisitInstanceOf(HInstanceOf * instruction)6126 void LocationsBuilderX86::VisitInstanceOf(HInstanceOf* instruction) {
6127 LocationSummary::CallKind call_kind = LocationSummary::kNoCall;
6128 TypeCheckKind type_check_kind = instruction->GetTypeCheckKind();
6129 switch (type_check_kind) {
6130 case TypeCheckKind::kExactCheck:
6131 case TypeCheckKind::kAbstractClassCheck:
6132 case TypeCheckKind::kClassHierarchyCheck:
6133 case TypeCheckKind::kArrayObjectCheck:
6134 call_kind =
6135 kEmitCompilerReadBarrier ? LocationSummary::kCallOnSlowPath : LocationSummary::kNoCall;
6136 break;
6137 case TypeCheckKind::kArrayCheck:
6138 case TypeCheckKind::kUnresolvedCheck:
6139 case TypeCheckKind::kInterfaceCheck:
6140 call_kind = LocationSummary::kCallOnSlowPath;
6141 break;
6142 }
6143
6144 LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction, call_kind);
6145 locations->SetInAt(0, Location::RequiresRegister());
6146 locations->SetInAt(1, Location::Any());
6147 // Note that TypeCheckSlowPathX86 uses this "out" register too.
6148 locations->SetOut(Location::RequiresRegister());
6149 // When read barriers are enabled, we need a temporary register for
6150 // some cases.
6151 if (TypeCheckNeedsATemporary(type_check_kind)) {
6152 locations->AddTemp(Location::RequiresRegister());
6153 }
6154 }
6155
VisitInstanceOf(HInstanceOf * instruction)6156 void InstructionCodeGeneratorX86::VisitInstanceOf(HInstanceOf* instruction) {
6157 TypeCheckKind type_check_kind = instruction->GetTypeCheckKind();
6158 LocationSummary* locations = instruction->GetLocations();
6159 Location obj_loc = locations->InAt(0);
6160 Register obj = obj_loc.AsRegister<Register>();
6161 Location cls = locations->InAt(1);
6162 Location out_loc = locations->Out();
6163 Register out = out_loc.AsRegister<Register>();
6164 Location maybe_temp_loc = TypeCheckNeedsATemporary(type_check_kind) ?
6165 locations->GetTemp(0) :
6166 Location::NoLocation();
6167 uint32_t class_offset = mirror::Object::ClassOffset().Int32Value();
6168 uint32_t super_offset = mirror::Class::SuperClassOffset().Int32Value();
6169 uint32_t component_offset = mirror::Class::ComponentTypeOffset().Int32Value();
6170 uint32_t primitive_offset = mirror::Class::PrimitiveTypeOffset().Int32Value();
6171 SlowPathCode* slow_path = nullptr;
6172 NearLabel done, zero;
6173
6174 // Return 0 if `obj` is null.
6175 // Avoid null check if we know obj is not null.
6176 if (instruction->MustDoNullCheck()) {
6177 __ testl(obj, obj);
6178 __ j(kEqual, &zero);
6179 }
6180
6181 // /* HeapReference<Class> */ out = obj->klass_
6182 GenerateReferenceLoadTwoRegisters(instruction, out_loc, obj_loc, class_offset, maybe_temp_loc);
6183
6184 switch (type_check_kind) {
6185 case TypeCheckKind::kExactCheck: {
6186 if (cls.IsRegister()) {
6187 __ cmpl(out, cls.AsRegister<Register>());
6188 } else {
6189 DCHECK(cls.IsStackSlot()) << cls;
6190 __ cmpl(out, Address(ESP, cls.GetStackIndex()));
6191 }
6192
6193 // Classes must be equal for the instanceof to succeed.
6194 __ j(kNotEqual, &zero);
6195 __ movl(out, Immediate(1));
6196 __ jmp(&done);
6197 break;
6198 }
6199
6200 case TypeCheckKind::kAbstractClassCheck: {
6201 // If the class is abstract, we eagerly fetch the super class of the
6202 // object to avoid doing a comparison we know will fail.
6203 NearLabel loop;
6204 __ Bind(&loop);
6205 // /* HeapReference<Class> */ out = out->super_class_
6206 GenerateReferenceLoadOneRegister(instruction, out_loc, super_offset, maybe_temp_loc);
6207 __ testl(out, out);
6208 // If `out` is null, we use it for the result, and jump to `done`.
6209 __ j(kEqual, &done);
6210 if (cls.IsRegister()) {
6211 __ cmpl(out, cls.AsRegister<Register>());
6212 } else {
6213 DCHECK(cls.IsStackSlot()) << cls;
6214 __ cmpl(out, Address(ESP, cls.GetStackIndex()));
6215 }
6216 __ j(kNotEqual, &loop);
6217 __ movl(out, Immediate(1));
6218 if (zero.IsLinked()) {
6219 __ jmp(&done);
6220 }
6221 break;
6222 }
6223
6224 case TypeCheckKind::kClassHierarchyCheck: {
6225 // Walk over the class hierarchy to find a match.
6226 NearLabel loop, success;
6227 __ Bind(&loop);
6228 if (cls.IsRegister()) {
6229 __ cmpl(out, cls.AsRegister<Register>());
6230 } else {
6231 DCHECK(cls.IsStackSlot()) << cls;
6232 __ cmpl(out, Address(ESP, cls.GetStackIndex()));
6233 }
6234 __ j(kEqual, &success);
6235 // /* HeapReference<Class> */ out = out->super_class_
6236 GenerateReferenceLoadOneRegister(instruction, out_loc, super_offset, maybe_temp_loc);
6237 __ testl(out, out);
6238 __ j(kNotEqual, &loop);
6239 // If `out` is null, we use it for the result, and jump to `done`.
6240 __ jmp(&done);
6241 __ Bind(&success);
6242 __ movl(out, Immediate(1));
6243 if (zero.IsLinked()) {
6244 __ jmp(&done);
6245 }
6246 break;
6247 }
6248
6249 case TypeCheckKind::kArrayObjectCheck: {
6250 // Do an exact check.
6251 NearLabel exact_check;
6252 if (cls.IsRegister()) {
6253 __ cmpl(out, cls.AsRegister<Register>());
6254 } else {
6255 DCHECK(cls.IsStackSlot()) << cls;
6256 __ cmpl(out, Address(ESP, cls.GetStackIndex()));
6257 }
6258 __ j(kEqual, &exact_check);
6259 // Otherwise, we need to check that the object's class is a non-primitive array.
6260 // /* HeapReference<Class> */ out = out->component_type_
6261 GenerateReferenceLoadOneRegister(instruction, out_loc, component_offset, maybe_temp_loc);
6262 __ testl(out, out);
6263 // If `out` is null, we use it for the result, and jump to `done`.
6264 __ j(kEqual, &done);
6265 __ cmpw(Address(out, primitive_offset), Immediate(Primitive::kPrimNot));
6266 __ j(kNotEqual, &zero);
6267 __ Bind(&exact_check);
6268 __ movl(out, Immediate(1));
6269 __ jmp(&done);
6270 break;
6271 }
6272
6273 case TypeCheckKind::kArrayCheck: {
6274 if (cls.IsRegister()) {
6275 __ cmpl(out, cls.AsRegister<Register>());
6276 } else {
6277 DCHECK(cls.IsStackSlot()) << cls;
6278 __ cmpl(out, Address(ESP, cls.GetStackIndex()));
6279 }
6280 DCHECK(locations->OnlyCallsOnSlowPath());
6281 slow_path = new (GetGraph()->GetArena()) TypeCheckSlowPathX86(instruction,
6282 /* is_fatal */ false);
6283 codegen_->AddSlowPath(slow_path);
6284 __ j(kNotEqual, slow_path->GetEntryLabel());
6285 __ movl(out, Immediate(1));
6286 if (zero.IsLinked()) {
6287 __ jmp(&done);
6288 }
6289 break;
6290 }
6291
6292 case TypeCheckKind::kUnresolvedCheck:
6293 case TypeCheckKind::kInterfaceCheck: {
6294 // Note that we indeed only call on slow path, but we always go
6295 // into the slow path for the unresolved and interface check
6296 // cases.
6297 //
6298 // We cannot directly call the InstanceofNonTrivial runtime
6299 // entry point without resorting to a type checking slow path
6300 // here (i.e. by calling InvokeRuntime directly), as it would
6301 // require to assign fixed registers for the inputs of this
6302 // HInstanceOf instruction (following the runtime calling
6303 // convention), which might be cluttered by the potential first
6304 // read barrier emission at the beginning of this method.
6305 //
6306 // TODO: Introduce a new runtime entry point taking the object
6307 // to test (instead of its class) as argument, and let it deal
6308 // with the read barrier issues. This will let us refactor this
6309 // case of the `switch` code as it was previously (with a direct
6310 // call to the runtime not using a type checking slow path).
6311 // This should also be beneficial for the other cases above.
6312 DCHECK(locations->OnlyCallsOnSlowPath());
6313 slow_path = new (GetGraph()->GetArena()) TypeCheckSlowPathX86(instruction,
6314 /* is_fatal */ false);
6315 codegen_->AddSlowPath(slow_path);
6316 __ jmp(slow_path->GetEntryLabel());
6317 if (zero.IsLinked()) {
6318 __ jmp(&done);
6319 }
6320 break;
6321 }
6322 }
6323
6324 if (zero.IsLinked()) {
6325 __ Bind(&zero);
6326 __ xorl(out, out);
6327 }
6328
6329 if (done.IsLinked()) {
6330 __ Bind(&done);
6331 }
6332
6333 if (slow_path != nullptr) {
6334 __ Bind(slow_path->GetExitLabel());
6335 }
6336 }
6337
VisitCheckCast(HCheckCast * instruction)6338 void LocationsBuilderX86::VisitCheckCast(HCheckCast* instruction) {
6339 LocationSummary::CallKind call_kind = LocationSummary::kNoCall;
6340 bool throws_into_catch = instruction->CanThrowIntoCatchBlock();
6341 TypeCheckKind type_check_kind = instruction->GetTypeCheckKind();
6342 switch (type_check_kind) {
6343 case TypeCheckKind::kExactCheck:
6344 case TypeCheckKind::kAbstractClassCheck:
6345 case TypeCheckKind::kClassHierarchyCheck:
6346 case TypeCheckKind::kArrayObjectCheck:
6347 call_kind = (throws_into_catch || kEmitCompilerReadBarrier) ?
6348 LocationSummary::kCallOnSlowPath :
6349 LocationSummary::kNoCall; // In fact, call on a fatal (non-returning) slow path.
6350 break;
6351 case TypeCheckKind::kArrayCheck:
6352 case TypeCheckKind::kUnresolvedCheck:
6353 case TypeCheckKind::kInterfaceCheck:
6354 call_kind = LocationSummary::kCallOnSlowPath;
6355 break;
6356 }
6357 LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction, call_kind);
6358 locations->SetInAt(0, Location::RequiresRegister());
6359 locations->SetInAt(1, Location::Any());
6360 // Note that TypeCheckSlowPathX86 uses this "temp" register too.
6361 locations->AddTemp(Location::RequiresRegister());
6362 // When read barriers are enabled, we need an additional temporary
6363 // register for some cases.
6364 if (TypeCheckNeedsATemporary(type_check_kind)) {
6365 locations->AddTemp(Location::RequiresRegister());
6366 }
6367 }
6368
VisitCheckCast(HCheckCast * instruction)6369 void InstructionCodeGeneratorX86::VisitCheckCast(HCheckCast* instruction) {
6370 TypeCheckKind type_check_kind = instruction->GetTypeCheckKind();
6371 LocationSummary* locations = instruction->GetLocations();
6372 Location obj_loc = locations->InAt(0);
6373 Register obj = obj_loc.AsRegister<Register>();
6374 Location cls = locations->InAt(1);
6375 Location temp_loc = locations->GetTemp(0);
6376 Register temp = temp_loc.AsRegister<Register>();
6377 Location maybe_temp2_loc = TypeCheckNeedsATemporary(type_check_kind) ?
6378 locations->GetTemp(1) :
6379 Location::NoLocation();
6380 uint32_t class_offset = mirror::Object::ClassOffset().Int32Value();
6381 uint32_t super_offset = mirror::Class::SuperClassOffset().Int32Value();
6382 uint32_t component_offset = mirror::Class::ComponentTypeOffset().Int32Value();
6383 uint32_t primitive_offset = mirror::Class::PrimitiveTypeOffset().Int32Value();
6384
6385 bool is_type_check_slow_path_fatal =
6386 (type_check_kind == TypeCheckKind::kExactCheck ||
6387 type_check_kind == TypeCheckKind::kAbstractClassCheck ||
6388 type_check_kind == TypeCheckKind::kClassHierarchyCheck ||
6389 type_check_kind == TypeCheckKind::kArrayObjectCheck) &&
6390 !instruction->CanThrowIntoCatchBlock();
6391 SlowPathCode* type_check_slow_path =
6392 new (GetGraph()->GetArena()) TypeCheckSlowPathX86(instruction,
6393 is_type_check_slow_path_fatal);
6394 codegen_->AddSlowPath(type_check_slow_path);
6395
6396 NearLabel done;
6397 // Avoid null check if we know obj is not null.
6398 if (instruction->MustDoNullCheck()) {
6399 __ testl(obj, obj);
6400 __ j(kEqual, &done);
6401 }
6402
6403 // /* HeapReference<Class> */ temp = obj->klass_
6404 GenerateReferenceLoadTwoRegisters(instruction, temp_loc, obj_loc, class_offset, maybe_temp2_loc);
6405
6406 switch (type_check_kind) {
6407 case TypeCheckKind::kExactCheck:
6408 case TypeCheckKind::kArrayCheck: {
6409 if (cls.IsRegister()) {
6410 __ cmpl(temp, cls.AsRegister<Register>());
6411 } else {
6412 DCHECK(cls.IsStackSlot()) << cls;
6413 __ cmpl(temp, Address(ESP, cls.GetStackIndex()));
6414 }
6415 // Jump to slow path for throwing the exception or doing a
6416 // more involved array check.
6417 __ j(kNotEqual, type_check_slow_path->GetEntryLabel());
6418 break;
6419 }
6420
6421 case TypeCheckKind::kAbstractClassCheck: {
6422 // If the class is abstract, we eagerly fetch the super class of the
6423 // object to avoid doing a comparison we know will fail.
6424 NearLabel loop, compare_classes;
6425 __ Bind(&loop);
6426 // /* HeapReference<Class> */ temp = temp->super_class_
6427 GenerateReferenceLoadOneRegister(instruction, temp_loc, super_offset, maybe_temp2_loc);
6428
6429 // If the class reference currently in `temp` is not null, jump
6430 // to the `compare_classes` label to compare it with the checked
6431 // class.
6432 __ testl(temp, temp);
6433 __ j(kNotEqual, &compare_classes);
6434 // Otherwise, jump to the slow path to throw the exception.
6435 //
6436 // But before, move back the object's class into `temp` before
6437 // going into the slow path, as it has been overwritten in the
6438 // meantime.
6439 // /* HeapReference<Class> */ temp = obj->klass_
6440 GenerateReferenceLoadTwoRegisters(
6441 instruction, temp_loc, obj_loc, class_offset, maybe_temp2_loc);
6442 __ jmp(type_check_slow_path->GetEntryLabel());
6443
6444 __ Bind(&compare_classes);
6445 if (cls.IsRegister()) {
6446 __ cmpl(temp, cls.AsRegister<Register>());
6447 } else {
6448 DCHECK(cls.IsStackSlot()) << cls;
6449 __ cmpl(temp, Address(ESP, cls.GetStackIndex()));
6450 }
6451 __ j(kNotEqual, &loop);
6452 break;
6453 }
6454
6455 case TypeCheckKind::kClassHierarchyCheck: {
6456 // Walk over the class hierarchy to find a match.
6457 NearLabel loop;
6458 __ Bind(&loop);
6459 if (cls.IsRegister()) {
6460 __ cmpl(temp, cls.AsRegister<Register>());
6461 } else {
6462 DCHECK(cls.IsStackSlot()) << cls;
6463 __ cmpl(temp, Address(ESP, cls.GetStackIndex()));
6464 }
6465 __ j(kEqual, &done);
6466
6467 // /* HeapReference<Class> */ temp = temp->super_class_
6468 GenerateReferenceLoadOneRegister(instruction, temp_loc, super_offset, maybe_temp2_loc);
6469
6470 // If the class reference currently in `temp` is not null, jump
6471 // back at the beginning of the loop.
6472 __ testl(temp, temp);
6473 __ j(kNotEqual, &loop);
6474 // Otherwise, jump to the slow path to throw the exception.
6475 //
6476 // But before, move back the object's class into `temp` before
6477 // going into the slow path, as it has been overwritten in the
6478 // meantime.
6479 // /* HeapReference<Class> */ temp = obj->klass_
6480 GenerateReferenceLoadTwoRegisters(
6481 instruction, temp_loc, obj_loc, class_offset, maybe_temp2_loc);
6482 __ jmp(type_check_slow_path->GetEntryLabel());
6483 break;
6484 }
6485
6486 case TypeCheckKind::kArrayObjectCheck: {
6487 // Do an exact check.
6488 NearLabel check_non_primitive_component_type;
6489 if (cls.IsRegister()) {
6490 __ cmpl(temp, cls.AsRegister<Register>());
6491 } else {
6492 DCHECK(cls.IsStackSlot()) << cls;
6493 __ cmpl(temp, Address(ESP, cls.GetStackIndex()));
6494 }
6495 __ j(kEqual, &done);
6496
6497 // Otherwise, we need to check that the object's class is a non-primitive array.
6498 // /* HeapReference<Class> */ temp = temp->component_type_
6499 GenerateReferenceLoadOneRegister(instruction, temp_loc, component_offset, maybe_temp2_loc);
6500
6501 // If the component type is not null (i.e. the object is indeed
6502 // an array), jump to label `check_non_primitive_component_type`
6503 // to further check that this component type is not a primitive
6504 // type.
6505 __ testl(temp, temp);
6506 __ j(kNotEqual, &check_non_primitive_component_type);
6507 // Otherwise, jump to the slow path to throw the exception.
6508 //
6509 // But before, move back the object's class into `temp` before
6510 // going into the slow path, as it has been overwritten in the
6511 // meantime.
6512 // /* HeapReference<Class> */ temp = obj->klass_
6513 GenerateReferenceLoadTwoRegisters(
6514 instruction, temp_loc, obj_loc, class_offset, maybe_temp2_loc);
6515 __ jmp(type_check_slow_path->GetEntryLabel());
6516
6517 __ Bind(&check_non_primitive_component_type);
6518 __ cmpw(Address(temp, primitive_offset), Immediate(Primitive::kPrimNot));
6519 __ j(kEqual, &done);
6520 // Same comment as above regarding `temp` and the slow path.
6521 // /* HeapReference<Class> */ temp = obj->klass_
6522 GenerateReferenceLoadTwoRegisters(
6523 instruction, temp_loc, obj_loc, class_offset, maybe_temp2_loc);
6524 __ jmp(type_check_slow_path->GetEntryLabel());
6525 break;
6526 }
6527
6528 case TypeCheckKind::kUnresolvedCheck:
6529 case TypeCheckKind::kInterfaceCheck:
6530 // We always go into the type check slow path for the unresolved
6531 // and interface check cases.
6532 //
6533 // We cannot directly call the CheckCast runtime entry point
6534 // without resorting to a type checking slow path here (i.e. by
6535 // calling InvokeRuntime directly), as it would require to
6536 // assign fixed registers for the inputs of this HInstanceOf
6537 // instruction (following the runtime calling convention), which
6538 // might be cluttered by the potential first read barrier
6539 // emission at the beginning of this method.
6540 //
6541 // TODO: Introduce a new runtime entry point taking the object
6542 // to test (instead of its class) as argument, and let it deal
6543 // with the read barrier issues. This will let us refactor this
6544 // case of the `switch` code as it was previously (with a direct
6545 // call to the runtime not using a type checking slow path).
6546 // This should also be beneficial for the other cases above.
6547 __ jmp(type_check_slow_path->GetEntryLabel());
6548 break;
6549 }
6550 __ Bind(&done);
6551
6552 __ Bind(type_check_slow_path->GetExitLabel());
6553 }
6554
VisitMonitorOperation(HMonitorOperation * instruction)6555 void LocationsBuilderX86::VisitMonitorOperation(HMonitorOperation* instruction) {
6556 LocationSummary* locations =
6557 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCall);
6558 InvokeRuntimeCallingConvention calling_convention;
6559 locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(0)));
6560 }
6561
VisitMonitorOperation(HMonitorOperation * instruction)6562 void InstructionCodeGeneratorX86::VisitMonitorOperation(HMonitorOperation* instruction) {
6563 codegen_->InvokeRuntime(instruction->IsEnter() ? QUICK_ENTRY_POINT(pLockObject)
6564 : QUICK_ENTRY_POINT(pUnlockObject),
6565 instruction,
6566 instruction->GetDexPc(),
6567 nullptr);
6568 if (instruction->IsEnter()) {
6569 CheckEntrypointTypes<kQuickLockObject, void, mirror::Object*>();
6570 } else {
6571 CheckEntrypointTypes<kQuickUnlockObject, void, mirror::Object*>();
6572 }
6573 }
6574
VisitAnd(HAnd * instruction)6575 void LocationsBuilderX86::VisitAnd(HAnd* instruction) { HandleBitwiseOperation(instruction); }
VisitOr(HOr * instruction)6576 void LocationsBuilderX86::VisitOr(HOr* instruction) { HandleBitwiseOperation(instruction); }
VisitXor(HXor * instruction)6577 void LocationsBuilderX86::VisitXor(HXor* instruction) { HandleBitwiseOperation(instruction); }
6578
HandleBitwiseOperation(HBinaryOperation * instruction)6579 void LocationsBuilderX86::HandleBitwiseOperation(HBinaryOperation* instruction) {
6580 LocationSummary* locations =
6581 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
6582 DCHECK(instruction->GetResultType() == Primitive::kPrimInt
6583 || instruction->GetResultType() == Primitive::kPrimLong);
6584 locations->SetInAt(0, Location::RequiresRegister());
6585 locations->SetInAt(1, Location::Any());
6586 locations->SetOut(Location::SameAsFirstInput());
6587 }
6588
VisitAnd(HAnd * instruction)6589 void InstructionCodeGeneratorX86::VisitAnd(HAnd* instruction) {
6590 HandleBitwiseOperation(instruction);
6591 }
6592
VisitOr(HOr * instruction)6593 void InstructionCodeGeneratorX86::VisitOr(HOr* instruction) {
6594 HandleBitwiseOperation(instruction);
6595 }
6596
VisitXor(HXor * instruction)6597 void InstructionCodeGeneratorX86::VisitXor(HXor* instruction) {
6598 HandleBitwiseOperation(instruction);
6599 }
6600
HandleBitwiseOperation(HBinaryOperation * instruction)6601 void InstructionCodeGeneratorX86::HandleBitwiseOperation(HBinaryOperation* instruction) {
6602 LocationSummary* locations = instruction->GetLocations();
6603 Location first = locations->InAt(0);
6604 Location second = locations->InAt(1);
6605 DCHECK(first.Equals(locations->Out()));
6606
6607 if (instruction->GetResultType() == Primitive::kPrimInt) {
6608 if (second.IsRegister()) {
6609 if (instruction->IsAnd()) {
6610 __ andl(first.AsRegister<Register>(), second.AsRegister<Register>());
6611 } else if (instruction->IsOr()) {
6612 __ orl(first.AsRegister<Register>(), second.AsRegister<Register>());
6613 } else {
6614 DCHECK(instruction->IsXor());
6615 __ xorl(first.AsRegister<Register>(), second.AsRegister<Register>());
6616 }
6617 } else if (second.IsConstant()) {
6618 if (instruction->IsAnd()) {
6619 __ andl(first.AsRegister<Register>(),
6620 Immediate(second.GetConstant()->AsIntConstant()->GetValue()));
6621 } else if (instruction->IsOr()) {
6622 __ orl(first.AsRegister<Register>(),
6623 Immediate(second.GetConstant()->AsIntConstant()->GetValue()));
6624 } else {
6625 DCHECK(instruction->IsXor());
6626 __ xorl(first.AsRegister<Register>(),
6627 Immediate(second.GetConstant()->AsIntConstant()->GetValue()));
6628 }
6629 } else {
6630 if (instruction->IsAnd()) {
6631 __ andl(first.AsRegister<Register>(), Address(ESP, second.GetStackIndex()));
6632 } else if (instruction->IsOr()) {
6633 __ orl(first.AsRegister<Register>(), Address(ESP, second.GetStackIndex()));
6634 } else {
6635 DCHECK(instruction->IsXor());
6636 __ xorl(first.AsRegister<Register>(), Address(ESP, second.GetStackIndex()));
6637 }
6638 }
6639 } else {
6640 DCHECK_EQ(instruction->GetResultType(), Primitive::kPrimLong);
6641 if (second.IsRegisterPair()) {
6642 if (instruction->IsAnd()) {
6643 __ andl(first.AsRegisterPairLow<Register>(), second.AsRegisterPairLow<Register>());
6644 __ andl(first.AsRegisterPairHigh<Register>(), second.AsRegisterPairHigh<Register>());
6645 } else if (instruction->IsOr()) {
6646 __ orl(first.AsRegisterPairLow<Register>(), second.AsRegisterPairLow<Register>());
6647 __ orl(first.AsRegisterPairHigh<Register>(), second.AsRegisterPairHigh<Register>());
6648 } else {
6649 DCHECK(instruction->IsXor());
6650 __ xorl(first.AsRegisterPairLow<Register>(), second.AsRegisterPairLow<Register>());
6651 __ xorl(first.AsRegisterPairHigh<Register>(), second.AsRegisterPairHigh<Register>());
6652 }
6653 } else if (second.IsDoubleStackSlot()) {
6654 if (instruction->IsAnd()) {
6655 __ andl(first.AsRegisterPairLow<Register>(), Address(ESP, second.GetStackIndex()));
6656 __ andl(first.AsRegisterPairHigh<Register>(),
6657 Address(ESP, second.GetHighStackIndex(kX86WordSize)));
6658 } else if (instruction->IsOr()) {
6659 __ orl(first.AsRegisterPairLow<Register>(), Address(ESP, second.GetStackIndex()));
6660 __ orl(first.AsRegisterPairHigh<Register>(),
6661 Address(ESP, second.GetHighStackIndex(kX86WordSize)));
6662 } else {
6663 DCHECK(instruction->IsXor());
6664 __ xorl(first.AsRegisterPairLow<Register>(), Address(ESP, second.GetStackIndex()));
6665 __ xorl(first.AsRegisterPairHigh<Register>(),
6666 Address(ESP, second.GetHighStackIndex(kX86WordSize)));
6667 }
6668 } else {
6669 DCHECK(second.IsConstant()) << second;
6670 int64_t value = second.GetConstant()->AsLongConstant()->GetValue();
6671 int32_t low_value = Low32Bits(value);
6672 int32_t high_value = High32Bits(value);
6673 Immediate low(low_value);
6674 Immediate high(high_value);
6675 Register first_low = first.AsRegisterPairLow<Register>();
6676 Register first_high = first.AsRegisterPairHigh<Register>();
6677 if (instruction->IsAnd()) {
6678 if (low_value == 0) {
6679 __ xorl(first_low, first_low);
6680 } else if (low_value != -1) {
6681 __ andl(first_low, low);
6682 }
6683 if (high_value == 0) {
6684 __ xorl(first_high, first_high);
6685 } else if (high_value != -1) {
6686 __ andl(first_high, high);
6687 }
6688 } else if (instruction->IsOr()) {
6689 if (low_value != 0) {
6690 __ orl(first_low, low);
6691 }
6692 if (high_value != 0) {
6693 __ orl(first_high, high);
6694 }
6695 } else {
6696 DCHECK(instruction->IsXor());
6697 if (low_value != 0) {
6698 __ xorl(first_low, low);
6699 }
6700 if (high_value != 0) {
6701 __ xorl(first_high, high);
6702 }
6703 }
6704 }
6705 }
6706 }
6707
GenerateReferenceLoadOneRegister(HInstruction * instruction,Location out,uint32_t offset,Location maybe_temp)6708 void InstructionCodeGeneratorX86::GenerateReferenceLoadOneRegister(HInstruction* instruction,
6709 Location out,
6710 uint32_t offset,
6711 Location maybe_temp) {
6712 Register out_reg = out.AsRegister<Register>();
6713 if (kEmitCompilerReadBarrier) {
6714 DCHECK(maybe_temp.IsRegister()) << maybe_temp;
6715 if (kUseBakerReadBarrier) {
6716 // Load with fast path based Baker's read barrier.
6717 // /* HeapReference<Object> */ out = *(out + offset)
6718 codegen_->GenerateFieldLoadWithBakerReadBarrier(
6719 instruction, out, out_reg, offset, maybe_temp, /* needs_null_check */ false);
6720 } else {
6721 // Load with slow path based read barrier.
6722 // Save the value of `out` into `maybe_temp` before overwriting it
6723 // in the following move operation, as we will need it for the
6724 // read barrier below.
6725 __ movl(maybe_temp.AsRegister<Register>(), out_reg);
6726 // /* HeapReference<Object> */ out = *(out + offset)
6727 __ movl(out_reg, Address(out_reg, offset));
6728 codegen_->GenerateReadBarrierSlow(instruction, out, out, maybe_temp, offset);
6729 }
6730 } else {
6731 // Plain load with no read barrier.
6732 // /* HeapReference<Object> */ out = *(out + offset)
6733 __ movl(out_reg, Address(out_reg, offset));
6734 __ MaybeUnpoisonHeapReference(out_reg);
6735 }
6736 }
6737
GenerateReferenceLoadTwoRegisters(HInstruction * instruction,Location out,Location obj,uint32_t offset,Location maybe_temp)6738 void InstructionCodeGeneratorX86::GenerateReferenceLoadTwoRegisters(HInstruction* instruction,
6739 Location out,
6740 Location obj,
6741 uint32_t offset,
6742 Location maybe_temp) {
6743 Register out_reg = out.AsRegister<Register>();
6744 Register obj_reg = obj.AsRegister<Register>();
6745 if (kEmitCompilerReadBarrier) {
6746 if (kUseBakerReadBarrier) {
6747 DCHECK(maybe_temp.IsRegister()) << maybe_temp;
6748 // Load with fast path based Baker's read barrier.
6749 // /* HeapReference<Object> */ out = *(obj + offset)
6750 codegen_->GenerateFieldLoadWithBakerReadBarrier(
6751 instruction, out, obj_reg, offset, maybe_temp, /* needs_null_check */ false);
6752 } else {
6753 // Load with slow path based read barrier.
6754 // /* HeapReference<Object> */ out = *(obj + offset)
6755 __ movl(out_reg, Address(obj_reg, offset));
6756 codegen_->GenerateReadBarrierSlow(instruction, out, out, obj, offset);
6757 }
6758 } else {
6759 // Plain load with no read barrier.
6760 // /* HeapReference<Object> */ out = *(obj + offset)
6761 __ movl(out_reg, Address(obj_reg, offset));
6762 __ MaybeUnpoisonHeapReference(out_reg);
6763 }
6764 }
6765
GenerateGcRootFieldLoad(HInstruction * instruction,Location root,const Address & address,Label * fixup_label)6766 void InstructionCodeGeneratorX86::GenerateGcRootFieldLoad(HInstruction* instruction,
6767 Location root,
6768 const Address& address,
6769 Label* fixup_label) {
6770 Register root_reg = root.AsRegister<Register>();
6771 if (kEmitCompilerReadBarrier) {
6772 if (kUseBakerReadBarrier) {
6773 // Fast path implementation of art::ReadBarrier::BarrierForRoot when
6774 // Baker's read barrier are used:
6775 //
6776 // root = *address;
6777 // if (Thread::Current()->GetIsGcMarking()) {
6778 // root = ReadBarrier::Mark(root)
6779 // }
6780
6781 // /* GcRoot<mirror::Object> */ root = *address
6782 __ movl(root_reg, address);
6783 if (fixup_label != nullptr) {
6784 __ Bind(fixup_label);
6785 }
6786 static_assert(
6787 sizeof(mirror::CompressedReference<mirror::Object>) == sizeof(GcRoot<mirror::Object>),
6788 "art::mirror::CompressedReference<mirror::Object> and art::GcRoot<mirror::Object> "
6789 "have different sizes.");
6790 static_assert(sizeof(mirror::CompressedReference<mirror::Object>) == sizeof(int32_t),
6791 "art::mirror::CompressedReference<mirror::Object> and int32_t "
6792 "have different sizes.");
6793
6794 // Slow path used to mark the GC root `root`.
6795 SlowPathCode* slow_path =
6796 new (GetGraph()->GetArena()) ReadBarrierMarkSlowPathX86(instruction, root, root);
6797 codegen_->AddSlowPath(slow_path);
6798
6799 __ fs()->cmpl(Address::Absolute(Thread::IsGcMarkingOffset<kX86WordSize>().Int32Value()),
6800 Immediate(0));
6801 __ j(kNotEqual, slow_path->GetEntryLabel());
6802 __ Bind(slow_path->GetExitLabel());
6803 } else {
6804 // GC root loaded through a slow path for read barriers other
6805 // than Baker's.
6806 // /* GcRoot<mirror::Object>* */ root = address
6807 __ leal(root_reg, address);
6808 if (fixup_label != nullptr) {
6809 __ Bind(fixup_label);
6810 }
6811 // /* mirror::Object* */ root = root->Read()
6812 codegen_->GenerateReadBarrierForRootSlow(instruction, root, root);
6813 }
6814 } else {
6815 // Plain GC root load with no read barrier.
6816 // /* GcRoot<mirror::Object> */ root = *address
6817 __ movl(root_reg, address);
6818 if (fixup_label != nullptr) {
6819 __ Bind(fixup_label);
6820 }
6821 // Note that GC roots are not affected by heap poisoning, thus we
6822 // do not have to unpoison `root_reg` here.
6823 }
6824 }
6825
GenerateFieldLoadWithBakerReadBarrier(HInstruction * instruction,Location ref,Register obj,uint32_t offset,Location temp,bool needs_null_check)6826 void CodeGeneratorX86::GenerateFieldLoadWithBakerReadBarrier(HInstruction* instruction,
6827 Location ref,
6828 Register obj,
6829 uint32_t offset,
6830 Location temp,
6831 bool needs_null_check) {
6832 DCHECK(kEmitCompilerReadBarrier);
6833 DCHECK(kUseBakerReadBarrier);
6834
6835 // /* HeapReference<Object> */ ref = *(obj + offset)
6836 Address src(obj, offset);
6837 GenerateReferenceLoadWithBakerReadBarrier(instruction, ref, obj, src, temp, needs_null_check);
6838 }
6839
GenerateArrayLoadWithBakerReadBarrier(HInstruction * instruction,Location ref,Register obj,uint32_t data_offset,Location index,Location temp,bool needs_null_check)6840 void CodeGeneratorX86::GenerateArrayLoadWithBakerReadBarrier(HInstruction* instruction,
6841 Location ref,
6842 Register obj,
6843 uint32_t data_offset,
6844 Location index,
6845 Location temp,
6846 bool needs_null_check) {
6847 DCHECK(kEmitCompilerReadBarrier);
6848 DCHECK(kUseBakerReadBarrier);
6849
6850 // /* HeapReference<Object> */ ref =
6851 // *(obj + data_offset + index * sizeof(HeapReference<Object>))
6852 Address src = index.IsConstant() ?
6853 Address(obj, (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4) + data_offset) :
6854 Address(obj, index.AsRegister<Register>(), TIMES_4, data_offset);
6855 GenerateReferenceLoadWithBakerReadBarrier(instruction, ref, obj, src, temp, needs_null_check);
6856 }
6857
GenerateReferenceLoadWithBakerReadBarrier(HInstruction * instruction,Location ref,Register obj,const Address & src,Location temp,bool needs_null_check)6858 void CodeGeneratorX86::GenerateReferenceLoadWithBakerReadBarrier(HInstruction* instruction,
6859 Location ref,
6860 Register obj,
6861 const Address& src,
6862 Location temp,
6863 bool needs_null_check) {
6864 DCHECK(kEmitCompilerReadBarrier);
6865 DCHECK(kUseBakerReadBarrier);
6866
6867 // In slow path based read barriers, the read barrier call is
6868 // inserted after the original load. However, in fast path based
6869 // Baker's read barriers, we need to perform the load of
6870 // mirror::Object::monitor_ *before* the original reference load.
6871 // This load-load ordering is required by the read barrier.
6872 // The fast path/slow path (for Baker's algorithm) should look like:
6873 //
6874 // uint32_t rb_state = Lockword(obj->monitor_).ReadBarrierState();
6875 // lfence; // Load fence or artificial data dependency to prevent load-load reordering
6876 // HeapReference<Object> ref = *src; // Original reference load.
6877 // bool is_gray = (rb_state == ReadBarrier::gray_ptr_);
6878 // if (is_gray) {
6879 // ref = ReadBarrier::Mark(ref); // Performed by runtime entrypoint slow path.
6880 // }
6881 //
6882 // Note: the original implementation in ReadBarrier::Barrier is
6883 // slightly more complex as:
6884 // - it implements the load-load fence using a data dependency on
6885 // the high-bits of rb_state, which are expected to be all zeroes
6886 // (we use CodeGeneratorX86::GenerateMemoryBarrier instead here,
6887 // which is a no-op thanks to the x86 memory model);
6888 // - it performs additional checks that we do not do here for
6889 // performance reasons.
6890
6891 Register ref_reg = ref.AsRegister<Register>();
6892 Register temp_reg = temp.AsRegister<Register>();
6893 uint32_t monitor_offset = mirror::Object::MonitorOffset().Int32Value();
6894
6895 // /* int32_t */ monitor = obj->monitor_
6896 __ movl(temp_reg, Address(obj, monitor_offset));
6897 if (needs_null_check) {
6898 MaybeRecordImplicitNullCheck(instruction);
6899 }
6900 // /* LockWord */ lock_word = LockWord(monitor)
6901 static_assert(sizeof(LockWord) == sizeof(int32_t),
6902 "art::LockWord and int32_t have different sizes.");
6903 // /* uint32_t */ rb_state = lock_word.ReadBarrierState()
6904 __ shrl(temp_reg, Immediate(LockWord::kReadBarrierStateShift));
6905 __ andl(temp_reg, Immediate(LockWord::kReadBarrierStateMask));
6906 static_assert(
6907 LockWord::kReadBarrierStateMask == ReadBarrier::rb_ptr_mask_,
6908 "art::LockWord::kReadBarrierStateMask is not equal to art::ReadBarrier::rb_ptr_mask_.");
6909
6910 // Load fence to prevent load-load reordering.
6911 // Note that this is a no-op, thanks to the x86 memory model.
6912 GenerateMemoryBarrier(MemBarrierKind::kLoadAny);
6913
6914 // The actual reference load.
6915 // /* HeapReference<Object> */ ref = *src
6916 __ movl(ref_reg, src);
6917
6918 // Object* ref = ref_addr->AsMirrorPtr()
6919 __ MaybeUnpoisonHeapReference(ref_reg);
6920
6921 // Slow path used to mark the object `ref` when it is gray.
6922 SlowPathCode* slow_path =
6923 new (GetGraph()->GetArena()) ReadBarrierMarkSlowPathX86(instruction, ref, ref);
6924 AddSlowPath(slow_path);
6925
6926 // if (rb_state == ReadBarrier::gray_ptr_)
6927 // ref = ReadBarrier::Mark(ref);
6928 __ cmpl(temp_reg, Immediate(ReadBarrier::gray_ptr_));
6929 __ j(kEqual, slow_path->GetEntryLabel());
6930 __ Bind(slow_path->GetExitLabel());
6931 }
6932
GenerateReadBarrierSlow(HInstruction * instruction,Location out,Location ref,Location obj,uint32_t offset,Location index)6933 void CodeGeneratorX86::GenerateReadBarrierSlow(HInstruction* instruction,
6934 Location out,
6935 Location ref,
6936 Location obj,
6937 uint32_t offset,
6938 Location index) {
6939 DCHECK(kEmitCompilerReadBarrier);
6940
6941 // Insert a slow path based read barrier *after* the reference load.
6942 //
6943 // If heap poisoning is enabled, the unpoisoning of the loaded
6944 // reference will be carried out by the runtime within the slow
6945 // path.
6946 //
6947 // Note that `ref` currently does not get unpoisoned (when heap
6948 // poisoning is enabled), which is alright as the `ref` argument is
6949 // not used by the artReadBarrierSlow entry point.
6950 //
6951 // TODO: Unpoison `ref` when it is used by artReadBarrierSlow.
6952 SlowPathCode* slow_path = new (GetGraph()->GetArena())
6953 ReadBarrierForHeapReferenceSlowPathX86(instruction, out, ref, obj, offset, index);
6954 AddSlowPath(slow_path);
6955
6956 __ jmp(slow_path->GetEntryLabel());
6957 __ Bind(slow_path->GetExitLabel());
6958 }
6959
MaybeGenerateReadBarrierSlow(HInstruction * instruction,Location out,Location ref,Location obj,uint32_t offset,Location index)6960 void CodeGeneratorX86::MaybeGenerateReadBarrierSlow(HInstruction* instruction,
6961 Location out,
6962 Location ref,
6963 Location obj,
6964 uint32_t offset,
6965 Location index) {
6966 if (kEmitCompilerReadBarrier) {
6967 // Baker's read barriers shall be handled by the fast path
6968 // (CodeGeneratorX86::GenerateReferenceLoadWithBakerReadBarrier).
6969 DCHECK(!kUseBakerReadBarrier);
6970 // If heap poisoning is enabled, unpoisoning will be taken care of
6971 // by the runtime within the slow path.
6972 GenerateReadBarrierSlow(instruction, out, ref, obj, offset, index);
6973 } else if (kPoisonHeapReferences) {
6974 __ UnpoisonHeapReference(out.AsRegister<Register>());
6975 }
6976 }
6977
GenerateReadBarrierForRootSlow(HInstruction * instruction,Location out,Location root)6978 void CodeGeneratorX86::GenerateReadBarrierForRootSlow(HInstruction* instruction,
6979 Location out,
6980 Location root) {
6981 DCHECK(kEmitCompilerReadBarrier);
6982
6983 // Insert a slow path based read barrier *after* the GC root load.
6984 //
6985 // Note that GC roots are not affected by heap poisoning, so we do
6986 // not need to do anything special for this here.
6987 SlowPathCode* slow_path =
6988 new (GetGraph()->GetArena()) ReadBarrierForRootSlowPathX86(instruction, out, root);
6989 AddSlowPath(slow_path);
6990
6991 __ jmp(slow_path->GetEntryLabel());
6992 __ Bind(slow_path->GetExitLabel());
6993 }
6994
VisitBoundType(HBoundType * instruction ATTRIBUTE_UNUSED)6995 void LocationsBuilderX86::VisitBoundType(HBoundType* instruction ATTRIBUTE_UNUSED) {
6996 // Nothing to do, this should be removed during prepare for register allocator.
6997 LOG(FATAL) << "Unreachable";
6998 }
6999
VisitBoundType(HBoundType * instruction ATTRIBUTE_UNUSED)7000 void InstructionCodeGeneratorX86::VisitBoundType(HBoundType* instruction ATTRIBUTE_UNUSED) {
7001 // Nothing to do, this should be removed during prepare for register allocator.
7002 LOG(FATAL) << "Unreachable";
7003 }
7004
7005 // Simple implementation of packed switch - generate cascaded compare/jumps.
VisitPackedSwitch(HPackedSwitch * switch_instr)7006 void LocationsBuilderX86::VisitPackedSwitch(HPackedSwitch* switch_instr) {
7007 LocationSummary* locations =
7008 new (GetGraph()->GetArena()) LocationSummary(switch_instr, LocationSummary::kNoCall);
7009 locations->SetInAt(0, Location::RequiresRegister());
7010 }
7011
GenPackedSwitchWithCompares(Register value_reg,int32_t lower_bound,uint32_t num_entries,HBasicBlock * switch_block,HBasicBlock * default_block)7012 void InstructionCodeGeneratorX86::GenPackedSwitchWithCompares(Register value_reg,
7013 int32_t lower_bound,
7014 uint32_t num_entries,
7015 HBasicBlock* switch_block,
7016 HBasicBlock* default_block) {
7017 // Figure out the correct compare values and jump conditions.
7018 // Handle the first compare/branch as a special case because it might
7019 // jump to the default case.
7020 DCHECK_GT(num_entries, 2u);
7021 Condition first_condition;
7022 uint32_t index;
7023 const ArenaVector<HBasicBlock*>& successors = switch_block->GetSuccessors();
7024 if (lower_bound != 0) {
7025 first_condition = kLess;
7026 __ cmpl(value_reg, Immediate(lower_bound));
7027 __ j(first_condition, codegen_->GetLabelOf(default_block));
7028 __ j(kEqual, codegen_->GetLabelOf(successors[0]));
7029
7030 index = 1;
7031 } else {
7032 // Handle all the compare/jumps below.
7033 first_condition = kBelow;
7034 index = 0;
7035 }
7036
7037 // Handle the rest of the compare/jumps.
7038 for (; index + 1 < num_entries; index += 2) {
7039 int32_t compare_to_value = lower_bound + index + 1;
7040 __ cmpl(value_reg, Immediate(compare_to_value));
7041 // Jump to successors[index] if value < case_value[index].
7042 __ j(first_condition, codegen_->GetLabelOf(successors[index]));
7043 // Jump to successors[index + 1] if value == case_value[index + 1].
7044 __ j(kEqual, codegen_->GetLabelOf(successors[index + 1]));
7045 }
7046
7047 if (index != num_entries) {
7048 // There are an odd number of entries. Handle the last one.
7049 DCHECK_EQ(index + 1, num_entries);
7050 __ cmpl(value_reg, Immediate(lower_bound + index));
7051 __ j(kEqual, codegen_->GetLabelOf(successors[index]));
7052 }
7053
7054 // And the default for any other value.
7055 if (!codegen_->GoesToNextBlock(switch_block, default_block)) {
7056 __ jmp(codegen_->GetLabelOf(default_block));
7057 }
7058 }
7059
VisitPackedSwitch(HPackedSwitch * switch_instr)7060 void InstructionCodeGeneratorX86::VisitPackedSwitch(HPackedSwitch* switch_instr) {
7061 int32_t lower_bound = switch_instr->GetStartValue();
7062 uint32_t num_entries = switch_instr->GetNumEntries();
7063 LocationSummary* locations = switch_instr->GetLocations();
7064 Register value_reg = locations->InAt(0).AsRegister<Register>();
7065
7066 GenPackedSwitchWithCompares(value_reg,
7067 lower_bound,
7068 num_entries,
7069 switch_instr->GetBlock(),
7070 switch_instr->GetDefaultBlock());
7071 }
7072
VisitX86PackedSwitch(HX86PackedSwitch * switch_instr)7073 void LocationsBuilderX86::VisitX86PackedSwitch(HX86PackedSwitch* switch_instr) {
7074 LocationSummary* locations =
7075 new (GetGraph()->GetArena()) LocationSummary(switch_instr, LocationSummary::kNoCall);
7076 locations->SetInAt(0, Location::RequiresRegister());
7077
7078 // Constant area pointer.
7079 locations->SetInAt(1, Location::RequiresRegister());
7080
7081 // And the temporary we need.
7082 locations->AddTemp(Location::RequiresRegister());
7083 }
7084
VisitX86PackedSwitch(HX86PackedSwitch * switch_instr)7085 void InstructionCodeGeneratorX86::VisitX86PackedSwitch(HX86PackedSwitch* switch_instr) {
7086 int32_t lower_bound = switch_instr->GetStartValue();
7087 uint32_t num_entries = switch_instr->GetNumEntries();
7088 LocationSummary* locations = switch_instr->GetLocations();
7089 Register value_reg = locations->InAt(0).AsRegister<Register>();
7090 HBasicBlock* default_block = switch_instr->GetDefaultBlock();
7091
7092 if (num_entries <= kPackedSwitchJumpTableThreshold) {
7093 GenPackedSwitchWithCompares(value_reg,
7094 lower_bound,
7095 num_entries,
7096 switch_instr->GetBlock(),
7097 default_block);
7098 return;
7099 }
7100
7101 // Optimizing has a jump area.
7102 Register temp_reg = locations->GetTemp(0).AsRegister<Register>();
7103 Register constant_area = locations->InAt(1).AsRegister<Register>();
7104
7105 // Remove the bias, if needed.
7106 if (lower_bound != 0) {
7107 __ leal(temp_reg, Address(value_reg, -lower_bound));
7108 value_reg = temp_reg;
7109 }
7110
7111 // Is the value in range?
7112 DCHECK_GE(num_entries, 1u);
7113 __ cmpl(value_reg, Immediate(num_entries - 1));
7114 __ j(kAbove, codegen_->GetLabelOf(default_block));
7115
7116 // We are in the range of the table.
7117 // Load (target-constant_area) from the jump table, indexing by the value.
7118 __ movl(temp_reg, codegen_->LiteralCaseTable(switch_instr, constant_area, value_reg));
7119
7120 // Compute the actual target address by adding in constant_area.
7121 __ addl(temp_reg, constant_area);
7122
7123 // And jump.
7124 __ jmp(temp_reg);
7125 }
7126
VisitX86ComputeBaseMethodAddress(HX86ComputeBaseMethodAddress * insn)7127 void LocationsBuilderX86::VisitX86ComputeBaseMethodAddress(
7128 HX86ComputeBaseMethodAddress* insn) {
7129 LocationSummary* locations =
7130 new (GetGraph()->GetArena()) LocationSummary(insn, LocationSummary::kNoCall);
7131 locations->SetOut(Location::RequiresRegister());
7132 }
7133
VisitX86ComputeBaseMethodAddress(HX86ComputeBaseMethodAddress * insn)7134 void InstructionCodeGeneratorX86::VisitX86ComputeBaseMethodAddress(
7135 HX86ComputeBaseMethodAddress* insn) {
7136 LocationSummary* locations = insn->GetLocations();
7137 Register reg = locations->Out().AsRegister<Register>();
7138
7139 // Generate call to next instruction.
7140 Label next_instruction;
7141 __ call(&next_instruction);
7142 __ Bind(&next_instruction);
7143
7144 // Remember this offset for later use with constant area.
7145 codegen_->SetMethodAddressOffset(GetAssembler()->CodeSize());
7146
7147 // Grab the return address off the stack.
7148 __ popl(reg);
7149 }
7150
VisitX86LoadFromConstantTable(HX86LoadFromConstantTable * insn)7151 void LocationsBuilderX86::VisitX86LoadFromConstantTable(
7152 HX86LoadFromConstantTable* insn) {
7153 LocationSummary* locations =
7154 new (GetGraph()->GetArena()) LocationSummary(insn, LocationSummary::kNoCall);
7155
7156 locations->SetInAt(0, Location::RequiresRegister());
7157 locations->SetInAt(1, Location::ConstantLocation(insn->GetConstant()));
7158
7159 // If we don't need to be materialized, we only need the inputs to be set.
7160 if (insn->IsEmittedAtUseSite()) {
7161 return;
7162 }
7163
7164 switch (insn->GetType()) {
7165 case Primitive::kPrimFloat:
7166 case Primitive::kPrimDouble:
7167 locations->SetOut(Location::RequiresFpuRegister());
7168 break;
7169
7170 case Primitive::kPrimInt:
7171 locations->SetOut(Location::RequiresRegister());
7172 break;
7173
7174 default:
7175 LOG(FATAL) << "Unsupported x86 constant area type " << insn->GetType();
7176 }
7177 }
7178
VisitX86LoadFromConstantTable(HX86LoadFromConstantTable * insn)7179 void InstructionCodeGeneratorX86::VisitX86LoadFromConstantTable(HX86LoadFromConstantTable* insn) {
7180 if (insn->IsEmittedAtUseSite()) {
7181 return;
7182 }
7183
7184 LocationSummary* locations = insn->GetLocations();
7185 Location out = locations->Out();
7186 Register const_area = locations->InAt(0).AsRegister<Register>();
7187 HConstant *value = insn->GetConstant();
7188
7189 switch (insn->GetType()) {
7190 case Primitive::kPrimFloat:
7191 __ movss(out.AsFpuRegister<XmmRegister>(),
7192 codegen_->LiteralFloatAddress(value->AsFloatConstant()->GetValue(), const_area));
7193 break;
7194
7195 case Primitive::kPrimDouble:
7196 __ movsd(out.AsFpuRegister<XmmRegister>(),
7197 codegen_->LiteralDoubleAddress(value->AsDoubleConstant()->GetValue(), const_area));
7198 break;
7199
7200 case Primitive::kPrimInt:
7201 __ movl(out.AsRegister<Register>(),
7202 codegen_->LiteralInt32Address(value->AsIntConstant()->GetValue(), const_area));
7203 break;
7204
7205 default:
7206 LOG(FATAL) << "Unsupported x86 constant area type " << insn->GetType();
7207 }
7208 }
7209
7210 /**
7211 * Class to handle late fixup of offsets into constant area.
7212 */
7213 class RIPFixup : public AssemblerFixup, public ArenaObject<kArenaAllocCodeGenerator> {
7214 public:
RIPFixup(CodeGeneratorX86 & codegen,size_t offset)7215 RIPFixup(CodeGeneratorX86& codegen, size_t offset)
7216 : codegen_(&codegen), offset_into_constant_area_(offset) {}
7217
7218 protected:
SetOffset(size_t offset)7219 void SetOffset(size_t offset) { offset_into_constant_area_ = offset; }
7220
7221 CodeGeneratorX86* codegen_;
7222
7223 private:
Process(const MemoryRegion & region,int pos)7224 void Process(const MemoryRegion& region, int pos) OVERRIDE {
7225 // Patch the correct offset for the instruction. The place to patch is the
7226 // last 4 bytes of the instruction.
7227 // The value to patch is the distance from the offset in the constant area
7228 // from the address computed by the HX86ComputeBaseMethodAddress instruction.
7229 int32_t constant_offset = codegen_->ConstantAreaStart() + offset_into_constant_area_;
7230 int32_t relative_position = constant_offset - codegen_->GetMethodAddressOffset();;
7231
7232 // Patch in the right value.
7233 region.StoreUnaligned<int32_t>(pos - 4, relative_position);
7234 }
7235
7236 // Location in constant area that the fixup refers to.
7237 int32_t offset_into_constant_area_;
7238 };
7239
7240 /**
7241 * Class to handle late fixup of offsets to a jump table that will be created in the
7242 * constant area.
7243 */
7244 class JumpTableRIPFixup : public RIPFixup {
7245 public:
JumpTableRIPFixup(CodeGeneratorX86 & codegen,HX86PackedSwitch * switch_instr)7246 JumpTableRIPFixup(CodeGeneratorX86& codegen, HX86PackedSwitch* switch_instr)
7247 : RIPFixup(codegen, static_cast<size_t>(-1)), switch_instr_(switch_instr) {}
7248
CreateJumpTable()7249 void CreateJumpTable() {
7250 X86Assembler* assembler = codegen_->GetAssembler();
7251
7252 // Ensure that the reference to the jump table has the correct offset.
7253 const int32_t offset_in_constant_table = assembler->ConstantAreaSize();
7254 SetOffset(offset_in_constant_table);
7255
7256 // The label values in the jump table are computed relative to the
7257 // instruction addressing the constant area.
7258 const int32_t relative_offset = codegen_->GetMethodAddressOffset();
7259
7260 // Populate the jump table with the correct values for the jump table.
7261 int32_t num_entries = switch_instr_->GetNumEntries();
7262 HBasicBlock* block = switch_instr_->GetBlock();
7263 const ArenaVector<HBasicBlock*>& successors = block->GetSuccessors();
7264 // The value that we want is the target offset - the position of the table.
7265 for (int32_t i = 0; i < num_entries; i++) {
7266 HBasicBlock* b = successors[i];
7267 Label* l = codegen_->GetLabelOf(b);
7268 DCHECK(l->IsBound());
7269 int32_t offset_to_block = l->Position() - relative_offset;
7270 assembler->AppendInt32(offset_to_block);
7271 }
7272 }
7273
7274 private:
7275 const HX86PackedSwitch* switch_instr_;
7276 };
7277
Finalize(CodeAllocator * allocator)7278 void CodeGeneratorX86::Finalize(CodeAllocator* allocator) {
7279 // Generate the constant area if needed.
7280 X86Assembler* assembler = GetAssembler();
7281 if (!assembler->IsConstantAreaEmpty() || !fixups_to_jump_tables_.empty()) {
7282 // Align to 4 byte boundary to reduce cache misses, as the data is 4 and 8
7283 // byte values.
7284 assembler->Align(4, 0);
7285 constant_area_start_ = assembler->CodeSize();
7286
7287 // Populate any jump tables.
7288 for (auto jump_table : fixups_to_jump_tables_) {
7289 jump_table->CreateJumpTable();
7290 }
7291
7292 // And now add the constant area to the generated code.
7293 assembler->AddConstantArea();
7294 }
7295
7296 // And finish up.
7297 CodeGenerator::Finalize(allocator);
7298 }
7299
LiteralDoubleAddress(double v,Register reg)7300 Address CodeGeneratorX86::LiteralDoubleAddress(double v, Register reg) {
7301 AssemblerFixup* fixup = new (GetGraph()->GetArena()) RIPFixup(*this, __ AddDouble(v));
7302 return Address(reg, kDummy32BitOffset, fixup);
7303 }
7304
LiteralFloatAddress(float v,Register reg)7305 Address CodeGeneratorX86::LiteralFloatAddress(float v, Register reg) {
7306 AssemblerFixup* fixup = new (GetGraph()->GetArena()) RIPFixup(*this, __ AddFloat(v));
7307 return Address(reg, kDummy32BitOffset, fixup);
7308 }
7309
LiteralInt32Address(int32_t v,Register reg)7310 Address CodeGeneratorX86::LiteralInt32Address(int32_t v, Register reg) {
7311 AssemblerFixup* fixup = new (GetGraph()->GetArena()) RIPFixup(*this, __ AddInt32(v));
7312 return Address(reg, kDummy32BitOffset, fixup);
7313 }
7314
LiteralInt64Address(int64_t v,Register reg)7315 Address CodeGeneratorX86::LiteralInt64Address(int64_t v, Register reg) {
7316 AssemblerFixup* fixup = new (GetGraph()->GetArena()) RIPFixup(*this, __ AddInt64(v));
7317 return Address(reg, kDummy32BitOffset, fixup);
7318 }
7319
Load32BitValue(Register dest,int32_t value)7320 void CodeGeneratorX86::Load32BitValue(Register dest, int32_t value) {
7321 if (value == 0) {
7322 __ xorl(dest, dest);
7323 } else {
7324 __ movl(dest, Immediate(value));
7325 }
7326 }
7327
Compare32BitValue(Register dest,int32_t value)7328 void CodeGeneratorX86::Compare32BitValue(Register dest, int32_t value) {
7329 if (value == 0) {
7330 __ testl(dest, dest);
7331 } else {
7332 __ cmpl(dest, Immediate(value));
7333 }
7334 }
7335
LiteralCaseTable(HX86PackedSwitch * switch_instr,Register reg,Register value)7336 Address CodeGeneratorX86::LiteralCaseTable(HX86PackedSwitch* switch_instr,
7337 Register reg,
7338 Register value) {
7339 // Create a fixup to be used to create and address the jump table.
7340 JumpTableRIPFixup* table_fixup =
7341 new (GetGraph()->GetArena()) JumpTableRIPFixup(*this, switch_instr);
7342
7343 // We have to populate the jump tables.
7344 fixups_to_jump_tables_.push_back(table_fixup);
7345
7346 // We want a scaled address, as we are extracting the correct offset from the table.
7347 return Address(reg, value, TIMES_4, kDummy32BitOffset, table_fixup);
7348 }
7349
7350 // TODO: target as memory.
MoveFromReturnRegister(Location target,Primitive::Type type)7351 void CodeGeneratorX86::MoveFromReturnRegister(Location target, Primitive::Type type) {
7352 if (!target.IsValid()) {
7353 DCHECK_EQ(type, Primitive::kPrimVoid);
7354 return;
7355 }
7356
7357 DCHECK_NE(type, Primitive::kPrimVoid);
7358
7359 Location return_loc = InvokeDexCallingConventionVisitorX86().GetReturnLocation(type);
7360 if (target.Equals(return_loc)) {
7361 return;
7362 }
7363
7364 // TODO: Consider pairs in the parallel move resolver, then this could be nicely merged
7365 // with the else branch.
7366 if (type == Primitive::kPrimLong) {
7367 HParallelMove parallel_move(GetGraph()->GetArena());
7368 parallel_move.AddMove(return_loc.ToLow(), target.ToLow(), Primitive::kPrimInt, nullptr);
7369 parallel_move.AddMove(return_loc.ToHigh(), target.ToHigh(), Primitive::kPrimInt, nullptr);
7370 GetMoveResolver()->EmitNativeCode(¶llel_move);
7371 } else {
7372 // Let the parallel move resolver take care of all of this.
7373 HParallelMove parallel_move(GetGraph()->GetArena());
7374 parallel_move.AddMove(return_loc, target, type, nullptr);
7375 GetMoveResolver()->EmitNativeCode(¶llel_move);
7376 }
7377 }
7378
7379 #undef __
7380
7381 } // namespace x86
7382 } // namespace art
7383