1 // Copyright 2013 the V8 project authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4
5 #include "src/compiler/code-generator.h"
6
7 #include <limits>
8
9 #include "src/compilation-info.h"
10 #include "src/compiler/code-generator-impl.h"
11 #include "src/compiler/gap-resolver.h"
12 #include "src/compiler/node-matchers.h"
13 #include "src/compiler/osr.h"
14 #include "src/wasm/wasm-module.h"
15 #include "src/x64/assembler-x64.h"
16 #include "src/x64/macro-assembler-x64.h"
17
18 namespace v8 {
19 namespace internal {
20 namespace compiler {
21
22 #define __ masm()->
23
24 // Adds X64 specific methods for decoding operands.
25 class X64OperandConverter : public InstructionOperandConverter {
26 public:
X64OperandConverter(CodeGenerator * gen,Instruction * instr)27 X64OperandConverter(CodeGenerator* gen, Instruction* instr)
28 : InstructionOperandConverter(gen, instr) {}
29
InputImmediate(size_t index)30 Immediate InputImmediate(size_t index) {
31 return ToImmediate(instr_->InputAt(index));
32 }
33
InputOperand(size_t index,int extra=0)34 Operand InputOperand(size_t index, int extra = 0) {
35 return ToOperand(instr_->InputAt(index), extra);
36 }
37
OutputOperand()38 Operand OutputOperand() { return ToOperand(instr_->Output()); }
39
ToImmediate(InstructionOperand * operand)40 Immediate ToImmediate(InstructionOperand* operand) {
41 Constant constant = ToConstant(operand);
42 if (constant.type() == Constant::kFloat64) {
43 DCHECK_EQ(0, bit_cast<int64_t>(constant.ToFloat64()));
44 return Immediate(0);
45 }
46 if (constant.rmode() == RelocInfo::WASM_MEMORY_REFERENCE ||
47 constant.rmode() == RelocInfo::WASM_MEMORY_SIZE_REFERENCE ||
48 constant.rmode() == RelocInfo::WASM_GLOBAL_REFERENCE) {
49 return Immediate(constant.ToInt32(), constant.rmode());
50 }
51 return Immediate(constant.ToInt32());
52 }
53
ToOperand(InstructionOperand * op,int extra=0)54 Operand ToOperand(InstructionOperand* op, int extra = 0) {
55 DCHECK(op->IsStackSlot() || op->IsFPStackSlot());
56 return SlotToOperand(AllocatedOperand::cast(op)->index(), extra);
57 }
58
SlotToOperand(int slot_index,int extra=0)59 Operand SlotToOperand(int slot_index, int extra = 0) {
60 FrameOffset offset = frame_access_state()->GetFrameOffset(slot_index);
61 return Operand(offset.from_stack_pointer() ? rsp : rbp,
62 offset.offset() + extra);
63 }
64
NextOffset(size_t * offset)65 static size_t NextOffset(size_t* offset) {
66 size_t i = *offset;
67 (*offset)++;
68 return i;
69 }
70
ScaleFor(AddressingMode one,AddressingMode mode)71 static ScaleFactor ScaleFor(AddressingMode one, AddressingMode mode) {
72 STATIC_ASSERT(0 == static_cast<int>(times_1));
73 STATIC_ASSERT(1 == static_cast<int>(times_2));
74 STATIC_ASSERT(2 == static_cast<int>(times_4));
75 STATIC_ASSERT(3 == static_cast<int>(times_8));
76 int scale = static_cast<int>(mode - one);
77 DCHECK(scale >= 0 && scale < 4);
78 return static_cast<ScaleFactor>(scale);
79 }
80
MemoryOperand(size_t * offset)81 Operand MemoryOperand(size_t* offset) {
82 AddressingMode mode = AddressingModeField::decode(instr_->opcode());
83 switch (mode) {
84 case kMode_MR: {
85 Register base = InputRegister(NextOffset(offset));
86 int32_t disp = 0;
87 return Operand(base, disp);
88 }
89 case kMode_MRI: {
90 Register base = InputRegister(NextOffset(offset));
91 int32_t disp = InputInt32(NextOffset(offset));
92 return Operand(base, disp);
93 }
94 case kMode_MR1:
95 case kMode_MR2:
96 case kMode_MR4:
97 case kMode_MR8: {
98 Register base = InputRegister(NextOffset(offset));
99 Register index = InputRegister(NextOffset(offset));
100 ScaleFactor scale = ScaleFor(kMode_MR1, mode);
101 int32_t disp = 0;
102 return Operand(base, index, scale, disp);
103 }
104 case kMode_MR1I:
105 case kMode_MR2I:
106 case kMode_MR4I:
107 case kMode_MR8I: {
108 Register base = InputRegister(NextOffset(offset));
109 Register index = InputRegister(NextOffset(offset));
110 ScaleFactor scale = ScaleFor(kMode_MR1I, mode);
111 int32_t disp = InputInt32(NextOffset(offset));
112 return Operand(base, index, scale, disp);
113 }
114 case kMode_M1: {
115 Register base = InputRegister(NextOffset(offset));
116 int32_t disp = 0;
117 return Operand(base, disp);
118 }
119 case kMode_M2:
120 UNREACHABLE(); // Should use kModeMR with more compact encoding instead
121 return Operand(no_reg, 0);
122 case kMode_M4:
123 case kMode_M8: {
124 Register index = InputRegister(NextOffset(offset));
125 ScaleFactor scale = ScaleFor(kMode_M1, mode);
126 int32_t disp = 0;
127 return Operand(index, scale, disp);
128 }
129 case kMode_M1I:
130 case kMode_M2I:
131 case kMode_M4I:
132 case kMode_M8I: {
133 Register index = InputRegister(NextOffset(offset));
134 ScaleFactor scale = ScaleFor(kMode_M1I, mode);
135 int32_t disp = InputInt32(NextOffset(offset));
136 return Operand(index, scale, disp);
137 }
138 case kMode_Root: {
139 Register base = kRootRegister;
140 int32_t disp = InputInt32(NextOffset(offset));
141 return Operand(base, disp);
142 }
143 case kMode_None:
144 UNREACHABLE();
145 return Operand(no_reg, 0);
146 }
147 UNREACHABLE();
148 return Operand(no_reg, 0);
149 }
150
MemoryOperand(size_t first_input=0)151 Operand MemoryOperand(size_t first_input = 0) {
152 return MemoryOperand(&first_input);
153 }
154 };
155
156
157 namespace {
158
HasImmediateInput(Instruction * instr,size_t index)159 bool HasImmediateInput(Instruction* instr, size_t index) {
160 return instr->InputAt(index)->IsImmediate();
161 }
162
163
164 class OutOfLineLoadZero final : public OutOfLineCode {
165 public:
OutOfLineLoadZero(CodeGenerator * gen,Register result)166 OutOfLineLoadZero(CodeGenerator* gen, Register result)
167 : OutOfLineCode(gen), result_(result) {}
168
Generate()169 void Generate() final { __ xorl(result_, result_); }
170
171 private:
172 Register const result_;
173 };
174
175 class OutOfLineLoadFloat32NaN final : public OutOfLineCode {
176 public:
OutOfLineLoadFloat32NaN(CodeGenerator * gen,XMMRegister result)177 OutOfLineLoadFloat32NaN(CodeGenerator* gen, XMMRegister result)
178 : OutOfLineCode(gen), result_(result) {}
179
Generate()180 void Generate() final {
181 __ Xorps(result_, result_);
182 __ Divss(result_, result_);
183 }
184
185 private:
186 XMMRegister const result_;
187 };
188
189 class OutOfLineLoadFloat64NaN final : public OutOfLineCode {
190 public:
OutOfLineLoadFloat64NaN(CodeGenerator * gen,XMMRegister result)191 OutOfLineLoadFloat64NaN(CodeGenerator* gen, XMMRegister result)
192 : OutOfLineCode(gen), result_(result) {}
193
Generate()194 void Generate() final {
195 __ Xorpd(result_, result_);
196 __ Divsd(result_, result_);
197 }
198
199 private:
200 XMMRegister const result_;
201 };
202
203 class OutOfLineTruncateDoubleToI final : public OutOfLineCode {
204 public:
OutOfLineTruncateDoubleToI(CodeGenerator * gen,Register result,XMMRegister input,UnwindingInfoWriter * unwinding_info_writer)205 OutOfLineTruncateDoubleToI(CodeGenerator* gen, Register result,
206 XMMRegister input,
207 UnwindingInfoWriter* unwinding_info_writer)
208 : OutOfLineCode(gen),
209 result_(result),
210 input_(input),
211 unwinding_info_writer_(unwinding_info_writer) {}
212
Generate()213 void Generate() final {
214 __ subp(rsp, Immediate(kDoubleSize));
215 unwinding_info_writer_->MaybeIncreaseBaseOffsetAt(__ pc_offset(),
216 kDoubleSize);
217 __ Movsd(MemOperand(rsp, 0), input_);
218 __ SlowTruncateToI(result_, rsp, 0);
219 __ addp(rsp, Immediate(kDoubleSize));
220 unwinding_info_writer_->MaybeIncreaseBaseOffsetAt(__ pc_offset(),
221 -kDoubleSize);
222 }
223
224 private:
225 Register const result_;
226 XMMRegister const input_;
227 UnwindingInfoWriter* const unwinding_info_writer_;
228 };
229
230
231 class OutOfLineRecordWrite final : public OutOfLineCode {
232 public:
OutOfLineRecordWrite(CodeGenerator * gen,Register object,Operand operand,Register value,Register scratch0,Register scratch1,RecordWriteMode mode)233 OutOfLineRecordWrite(CodeGenerator* gen, Register object, Operand operand,
234 Register value, Register scratch0, Register scratch1,
235 RecordWriteMode mode)
236 : OutOfLineCode(gen),
237 object_(object),
238 operand_(operand),
239 value_(value),
240 scratch0_(scratch0),
241 scratch1_(scratch1),
242 mode_(mode) {}
243
Generate()244 void Generate() final {
245 if (mode_ > RecordWriteMode::kValueIsPointer) {
246 __ JumpIfSmi(value_, exit());
247 }
248 __ CheckPageFlag(value_, scratch0_,
249 MemoryChunk::kPointersToHereAreInterestingMask, zero,
250 exit());
251 RememberedSetAction const remembered_set_action =
252 mode_ > RecordWriteMode::kValueIsMap ? EMIT_REMEMBERED_SET
253 : OMIT_REMEMBERED_SET;
254 SaveFPRegsMode const save_fp_mode =
255 frame()->DidAllocateDoubleRegisters() ? kSaveFPRegs : kDontSaveFPRegs;
256 RecordWriteStub stub(isolate(), object_, scratch0_, scratch1_,
257 remembered_set_action, save_fp_mode);
258 __ leap(scratch1_, operand_);
259 __ CallStub(&stub);
260 }
261
262 private:
263 Register const object_;
264 Operand const operand_;
265 Register const value_;
266 Register const scratch0_;
267 Register const scratch1_;
268 RecordWriteMode const mode_;
269 };
270
271 class WasmOutOfLineTrap final : public OutOfLineCode {
272 public:
WasmOutOfLineTrap(CodeGenerator * gen,Address pc,bool frame_elided,Register context,int32_t position)273 WasmOutOfLineTrap(CodeGenerator* gen, Address pc, bool frame_elided,
274 Register context, int32_t position)
275 : OutOfLineCode(gen),
276 pc_(pc),
277 frame_elided_(frame_elided),
278 context_(context),
279 position_(position) {}
280
Generate()281 void Generate() final {
282 // TODO(eholk): record pc_ and the current pc in a table so that
283 // the signal handler can find it.
284 USE(pc_);
285
286 if (frame_elided_) {
287 __ EnterFrame(StackFrame::WASM);
288 }
289
290 wasm::TrapReason trap_id = wasm::kTrapMemOutOfBounds;
291 int trap_reason = wasm::WasmOpcodes::TrapReasonToMessageId(trap_id);
292 __ Push(Smi::FromInt(trap_reason));
293 __ Push(Smi::FromInt(position_));
294 __ Move(rsi, context_);
295 __ CallRuntime(Runtime::kThrowWasmError);
296 }
297
298 private:
299 Address pc_;
300 bool frame_elided_;
301 Register context_;
302 int32_t position_;
303 };
304
305 } // namespace
306
307
308 #define ASSEMBLE_UNOP(asm_instr) \
309 do { \
310 if (instr->Output()->IsRegister()) { \
311 __ asm_instr(i.OutputRegister()); \
312 } else { \
313 __ asm_instr(i.OutputOperand()); \
314 } \
315 } while (0)
316
317
318 #define ASSEMBLE_BINOP(asm_instr) \
319 do { \
320 if (HasImmediateInput(instr, 1)) { \
321 if (instr->InputAt(0)->IsRegister()) { \
322 __ asm_instr(i.InputRegister(0), i.InputImmediate(1)); \
323 } else { \
324 __ asm_instr(i.InputOperand(0), i.InputImmediate(1)); \
325 } \
326 } else { \
327 if (instr->InputAt(1)->IsRegister()) { \
328 __ asm_instr(i.InputRegister(0), i.InputRegister(1)); \
329 } else { \
330 __ asm_instr(i.InputRegister(0), i.InputOperand(1)); \
331 } \
332 } \
333 } while (0)
334
335 #define ASSEMBLE_COMPARE(asm_instr) \
336 do { \
337 if (AddressingModeField::decode(instr->opcode()) != kMode_None) { \
338 size_t index = 0; \
339 Operand left = i.MemoryOperand(&index); \
340 if (HasImmediateInput(instr, index)) { \
341 __ asm_instr(left, i.InputImmediate(index)); \
342 } else { \
343 __ asm_instr(left, i.InputRegister(index)); \
344 } \
345 } else { \
346 if (HasImmediateInput(instr, 1)) { \
347 if (instr->InputAt(0)->IsRegister()) { \
348 __ asm_instr(i.InputRegister(0), i.InputImmediate(1)); \
349 } else { \
350 __ asm_instr(i.InputOperand(0), i.InputImmediate(1)); \
351 } \
352 } else { \
353 if (instr->InputAt(1)->IsRegister()) { \
354 __ asm_instr(i.InputRegister(0), i.InputRegister(1)); \
355 } else { \
356 __ asm_instr(i.InputRegister(0), i.InputOperand(1)); \
357 } \
358 } \
359 } \
360 } while (0)
361
362 #define ASSEMBLE_MULT(asm_instr) \
363 do { \
364 if (HasImmediateInput(instr, 1)) { \
365 if (instr->InputAt(0)->IsRegister()) { \
366 __ asm_instr(i.OutputRegister(), i.InputRegister(0), \
367 i.InputImmediate(1)); \
368 } else { \
369 __ asm_instr(i.OutputRegister(), i.InputOperand(0), \
370 i.InputImmediate(1)); \
371 } \
372 } else { \
373 if (instr->InputAt(1)->IsRegister()) { \
374 __ asm_instr(i.OutputRegister(), i.InputRegister(1)); \
375 } else { \
376 __ asm_instr(i.OutputRegister(), i.InputOperand(1)); \
377 } \
378 } \
379 } while (0)
380
381
382 #define ASSEMBLE_SHIFT(asm_instr, width) \
383 do { \
384 if (HasImmediateInput(instr, 1)) { \
385 if (instr->Output()->IsRegister()) { \
386 __ asm_instr(i.OutputRegister(), Immediate(i.InputInt##width(1))); \
387 } else { \
388 __ asm_instr(i.OutputOperand(), Immediate(i.InputInt##width(1))); \
389 } \
390 } else { \
391 if (instr->Output()->IsRegister()) { \
392 __ asm_instr##_cl(i.OutputRegister()); \
393 } else { \
394 __ asm_instr##_cl(i.OutputOperand()); \
395 } \
396 } \
397 } while (0)
398
399
400 #define ASSEMBLE_MOVX(asm_instr) \
401 do { \
402 if (instr->addressing_mode() != kMode_None) { \
403 __ asm_instr(i.OutputRegister(), i.MemoryOperand()); \
404 } else if (instr->InputAt(0)->IsRegister()) { \
405 __ asm_instr(i.OutputRegister(), i.InputRegister(0)); \
406 } else { \
407 __ asm_instr(i.OutputRegister(), i.InputOperand(0)); \
408 } \
409 } while (0)
410
411 #define ASSEMBLE_SSE_BINOP(asm_instr) \
412 do { \
413 if (instr->InputAt(1)->IsFPRegister()) { \
414 __ asm_instr(i.InputDoubleRegister(0), i.InputDoubleRegister(1)); \
415 } else { \
416 __ asm_instr(i.InputDoubleRegister(0), i.InputOperand(1)); \
417 } \
418 } while (0)
419
420 #define ASSEMBLE_SSE_UNOP(asm_instr) \
421 do { \
422 if (instr->InputAt(0)->IsFPRegister()) { \
423 __ asm_instr(i.OutputDoubleRegister(), i.InputDoubleRegister(0)); \
424 } else { \
425 __ asm_instr(i.OutputDoubleRegister(), i.InputOperand(0)); \
426 } \
427 } while (0)
428
429 #define ASSEMBLE_AVX_BINOP(asm_instr) \
430 do { \
431 CpuFeatureScope avx_scope(masm(), AVX); \
432 if (instr->InputAt(1)->IsFPRegister()) { \
433 __ asm_instr(i.OutputDoubleRegister(), i.InputDoubleRegister(0), \
434 i.InputDoubleRegister(1)); \
435 } else { \
436 __ asm_instr(i.OutputDoubleRegister(), i.InputDoubleRegister(0), \
437 i.InputOperand(1)); \
438 } \
439 } while (0)
440
441 #define ASSEMBLE_CHECKED_LOAD_FLOAT(asm_instr, OutOfLineLoadNaN) \
442 do { \
443 auto result = i.OutputDoubleRegister(); \
444 auto buffer = i.InputRegister(0); \
445 auto index1 = i.InputRegister(1); \
446 auto index2 = i.InputUint32(2); \
447 OutOfLineCode* ool; \
448 if (instr->InputAt(3)->IsRegister()) { \
449 auto length = i.InputRegister(3); \
450 DCHECK_EQ(0u, index2); \
451 __ cmpl(index1, length); \
452 ool = new (zone()) OutOfLineLoadNaN(this, result); \
453 } else { \
454 auto length = i.InputUint32(3); \
455 RelocInfo::Mode rmode = i.ToConstant(instr->InputAt(3)).rmode(); \
456 DCHECK_LE(index2, length); \
457 __ cmpl(index1, Immediate(length - index2, rmode)); \
458 class OutOfLineLoadFloat final : public OutOfLineCode { \
459 public: \
460 OutOfLineLoadFloat(CodeGenerator* gen, XMMRegister result, \
461 Register buffer, Register index1, int32_t index2, \
462 int32_t length, RelocInfo::Mode rmode) \
463 : OutOfLineCode(gen), \
464 result_(result), \
465 buffer_(buffer), \
466 index1_(index1), \
467 index2_(index2), \
468 length_(length), \
469 rmode_(rmode) {} \
470 \
471 void Generate() final { \
472 __ leal(kScratchRegister, Operand(index1_, index2_)); \
473 __ Pcmpeqd(result_, result_); \
474 __ cmpl(kScratchRegister, Immediate(length_, rmode_)); \
475 __ j(above_equal, exit()); \
476 __ asm_instr(result_, \
477 Operand(buffer_, kScratchRegister, times_1, 0)); \
478 } \
479 \
480 private: \
481 XMMRegister const result_; \
482 Register const buffer_; \
483 Register const index1_; \
484 int32_t const index2_; \
485 int32_t const length_; \
486 RelocInfo::Mode rmode_; \
487 }; \
488 ool = new (zone()) OutOfLineLoadFloat(this, result, buffer, index1, \
489 index2, length, rmode); \
490 } \
491 __ j(above_equal, ool->entry()); \
492 __ asm_instr(result, Operand(buffer, index1, times_1, index2)); \
493 __ bind(ool->exit()); \
494 } while (false)
495
496 #define ASSEMBLE_CHECKED_LOAD_INTEGER(asm_instr) \
497 do { \
498 auto result = i.OutputRegister(); \
499 auto buffer = i.InputRegister(0); \
500 auto index1 = i.InputRegister(1); \
501 auto index2 = i.InputUint32(2); \
502 OutOfLineCode* ool; \
503 if (instr->InputAt(3)->IsRegister()) { \
504 auto length = i.InputRegister(3); \
505 DCHECK_EQ(0u, index2); \
506 __ cmpl(index1, length); \
507 ool = new (zone()) OutOfLineLoadZero(this, result); \
508 } else { \
509 auto length = i.InputUint32(3); \
510 RelocInfo::Mode rmode = i.ToConstant(instr->InputAt(3)).rmode(); \
511 DCHECK_LE(index2, length); \
512 __ cmpl(index1, Immediate(length - index2, rmode)); \
513 class OutOfLineLoadInteger final : public OutOfLineCode { \
514 public: \
515 OutOfLineLoadInteger(CodeGenerator* gen, Register result, \
516 Register buffer, Register index1, int32_t index2, \
517 int32_t length, RelocInfo::Mode rmode) \
518 : OutOfLineCode(gen), \
519 result_(result), \
520 buffer_(buffer), \
521 index1_(index1), \
522 index2_(index2), \
523 length_(length), \
524 rmode_(rmode) {} \
525 \
526 void Generate() final { \
527 Label oob; \
528 __ leal(kScratchRegister, Operand(index1_, index2_)); \
529 __ cmpl(kScratchRegister, Immediate(length_, rmode_)); \
530 __ j(above_equal, &oob, Label::kNear); \
531 __ asm_instr(result_, \
532 Operand(buffer_, kScratchRegister, times_1, 0)); \
533 __ jmp(exit()); \
534 __ bind(&oob); \
535 __ xorl(result_, result_); \
536 } \
537 \
538 private: \
539 Register const result_; \
540 Register const buffer_; \
541 Register const index1_; \
542 int32_t const index2_; \
543 int32_t const length_; \
544 RelocInfo::Mode const rmode_; \
545 }; \
546 ool = new (zone()) OutOfLineLoadInteger(this, result, buffer, index1, \
547 index2, length, rmode); \
548 } \
549 __ j(above_equal, ool->entry()); \
550 __ asm_instr(result, Operand(buffer, index1, times_1, index2)); \
551 __ bind(ool->exit()); \
552 } while (false)
553
554 #define ASSEMBLE_CHECKED_STORE_FLOAT(asm_instr) \
555 do { \
556 auto buffer = i.InputRegister(0); \
557 auto index1 = i.InputRegister(1); \
558 auto index2 = i.InputUint32(2); \
559 auto value = i.InputDoubleRegister(4); \
560 if (instr->InputAt(3)->IsRegister()) { \
561 auto length = i.InputRegister(3); \
562 DCHECK_EQ(0u, index2); \
563 Label done; \
564 __ cmpl(index1, length); \
565 __ j(above_equal, &done, Label::kNear); \
566 __ asm_instr(Operand(buffer, index1, times_1, index2), value); \
567 __ bind(&done); \
568 } else { \
569 auto length = i.InputUint32(3); \
570 RelocInfo::Mode rmode = i.ToConstant(instr->InputAt(3)).rmode(); \
571 DCHECK_LE(index2, length); \
572 __ cmpl(index1, Immediate(length - index2, rmode)); \
573 class OutOfLineStoreFloat final : public OutOfLineCode { \
574 public: \
575 OutOfLineStoreFloat(CodeGenerator* gen, Register buffer, \
576 Register index1, int32_t index2, int32_t length, \
577 XMMRegister value, RelocInfo::Mode rmode) \
578 : OutOfLineCode(gen), \
579 buffer_(buffer), \
580 index1_(index1), \
581 index2_(index2), \
582 length_(length), \
583 value_(value), \
584 rmode_(rmode) {} \
585 \
586 void Generate() final { \
587 __ leal(kScratchRegister, Operand(index1_, index2_)); \
588 __ cmpl(kScratchRegister, Immediate(length_, rmode_)); \
589 __ j(above_equal, exit()); \
590 __ asm_instr(Operand(buffer_, kScratchRegister, times_1, 0), \
591 value_); \
592 } \
593 \
594 private: \
595 Register const buffer_; \
596 Register const index1_; \
597 int32_t const index2_; \
598 int32_t const length_; \
599 XMMRegister const value_; \
600 RelocInfo::Mode rmode_; \
601 }; \
602 auto ool = new (zone()) OutOfLineStoreFloat( \
603 this, buffer, index1, index2, length, value, rmode); \
604 __ j(above_equal, ool->entry()); \
605 __ asm_instr(Operand(buffer, index1, times_1, index2), value); \
606 __ bind(ool->exit()); \
607 } \
608 } while (false)
609
610 #define ASSEMBLE_CHECKED_STORE_INTEGER_IMPL(asm_instr, Value) \
611 do { \
612 auto buffer = i.InputRegister(0); \
613 auto index1 = i.InputRegister(1); \
614 auto index2 = i.InputUint32(2); \
615 if (instr->InputAt(3)->IsRegister()) { \
616 auto length = i.InputRegister(3); \
617 DCHECK_EQ(0u, index2); \
618 Label done; \
619 __ cmpl(index1, length); \
620 __ j(above_equal, &done, Label::kNear); \
621 __ asm_instr(Operand(buffer, index1, times_1, index2), value); \
622 __ bind(&done); \
623 } else { \
624 auto length = i.InputUint32(3); \
625 RelocInfo::Mode rmode = i.ToConstant(instr->InputAt(3)).rmode(); \
626 DCHECK_LE(index2, length); \
627 __ cmpl(index1, Immediate(length - index2, rmode)); \
628 class OutOfLineStoreInteger final : public OutOfLineCode { \
629 public: \
630 OutOfLineStoreInteger(CodeGenerator* gen, Register buffer, \
631 Register index1, int32_t index2, int32_t length, \
632 Value value, RelocInfo::Mode rmode) \
633 : OutOfLineCode(gen), \
634 buffer_(buffer), \
635 index1_(index1), \
636 index2_(index2), \
637 length_(length), \
638 value_(value), \
639 rmode_(rmode) {} \
640 \
641 void Generate() final { \
642 __ leal(kScratchRegister, Operand(index1_, index2_)); \
643 __ cmpl(kScratchRegister, Immediate(length_, rmode_)); \
644 __ j(above_equal, exit()); \
645 __ asm_instr(Operand(buffer_, kScratchRegister, times_1, 0), \
646 value_); \
647 } \
648 \
649 private: \
650 Register const buffer_; \
651 Register const index1_; \
652 int32_t const index2_; \
653 int32_t const length_; \
654 Value const value_; \
655 RelocInfo::Mode rmode_; \
656 }; \
657 auto ool = new (zone()) OutOfLineStoreInteger( \
658 this, buffer, index1, index2, length, value, rmode); \
659 __ j(above_equal, ool->entry()); \
660 __ asm_instr(Operand(buffer, index1, times_1, index2), value); \
661 __ bind(ool->exit()); \
662 } \
663 } while (false)
664
665 #define ASSEMBLE_CHECKED_STORE_INTEGER(asm_instr) \
666 do { \
667 if (instr->InputAt(4)->IsRegister()) { \
668 Register value = i.InputRegister(4); \
669 ASSEMBLE_CHECKED_STORE_INTEGER_IMPL(asm_instr, Register); \
670 } else { \
671 Immediate value = i.InputImmediate(4); \
672 ASSEMBLE_CHECKED_STORE_INTEGER_IMPL(asm_instr, Immediate); \
673 } \
674 } while (false)
675
676 #define ASSEMBLE_IEEE754_BINOP(name) \
677 do { \
678 __ PrepareCallCFunction(2); \
679 __ CallCFunction(ExternalReference::ieee754_##name##_function(isolate()), \
680 2); \
681 } while (false)
682
683 #define ASSEMBLE_IEEE754_UNOP(name) \
684 do { \
685 __ PrepareCallCFunction(1); \
686 __ CallCFunction(ExternalReference::ieee754_##name##_function(isolate()), \
687 1); \
688 } while (false)
689
AssembleDeconstructFrame()690 void CodeGenerator::AssembleDeconstructFrame() {
691 unwinding_info_writer_.MarkFrameDeconstructed(__ pc_offset());
692 __ movq(rsp, rbp);
693 __ popq(rbp);
694 }
695
AssemblePrepareTailCall()696 void CodeGenerator::AssemblePrepareTailCall() {
697 if (frame_access_state()->has_frame()) {
698 __ movq(rbp, MemOperand(rbp, 0));
699 }
700 frame_access_state()->SetFrameAccessToSP();
701 }
702
AssemblePopArgumentsAdaptorFrame(Register args_reg,Register scratch1,Register scratch2,Register scratch3)703 void CodeGenerator::AssemblePopArgumentsAdaptorFrame(Register args_reg,
704 Register scratch1,
705 Register scratch2,
706 Register scratch3) {
707 DCHECK(!AreAliased(args_reg, scratch1, scratch2, scratch3));
708 Label done;
709
710 // Check if current frame is an arguments adaptor frame.
711 __ Cmp(Operand(rbp, StandardFrameConstants::kContextOffset),
712 Smi::FromInt(StackFrame::ARGUMENTS_ADAPTOR));
713 __ j(not_equal, &done, Label::kNear);
714
715 // Load arguments count from current arguments adaptor frame (note, it
716 // does not include receiver).
717 Register caller_args_count_reg = scratch1;
718 __ SmiToInteger32(
719 caller_args_count_reg,
720 Operand(rbp, ArgumentsAdaptorFrameConstants::kLengthOffset));
721
722 ParameterCount callee_args_count(args_reg);
723 __ PrepareForTailCall(callee_args_count, caller_args_count_reg, scratch2,
724 scratch3, ReturnAddressState::kOnStack);
725 __ bind(&done);
726 }
727
728 namespace {
729
AdjustStackPointerForTailCall(MacroAssembler * masm,FrameAccessState * state,int new_slot_above_sp,bool allow_shrinkage=true)730 void AdjustStackPointerForTailCall(MacroAssembler* masm,
731 FrameAccessState* state,
732 int new_slot_above_sp,
733 bool allow_shrinkage = true) {
734 int current_sp_offset = state->GetSPToFPSlotCount() +
735 StandardFrameConstants::kFixedSlotCountAboveFp;
736 int stack_slot_delta = new_slot_above_sp - current_sp_offset;
737 if (stack_slot_delta > 0) {
738 masm->subq(rsp, Immediate(stack_slot_delta * kPointerSize));
739 state->IncreaseSPDelta(stack_slot_delta);
740 } else if (allow_shrinkage && stack_slot_delta < 0) {
741 masm->addq(rsp, Immediate(-stack_slot_delta * kPointerSize));
742 state->IncreaseSPDelta(stack_slot_delta);
743 }
744 }
745
746 } // namespace
747
AssembleTailCallBeforeGap(Instruction * instr,int first_unused_stack_slot)748 void CodeGenerator::AssembleTailCallBeforeGap(Instruction* instr,
749 int first_unused_stack_slot) {
750 CodeGenerator::PushTypeFlags flags(kImmediatePush | kScalarPush);
751 ZoneVector<MoveOperands*> pushes(zone());
752 GetPushCompatibleMoves(instr, flags, &pushes);
753
754 if (!pushes.empty() &&
755 (LocationOperand::cast(pushes.back()->destination()).index() + 1 ==
756 first_unused_stack_slot)) {
757 X64OperandConverter g(this, instr);
758 for (auto move : pushes) {
759 LocationOperand destination_location(
760 LocationOperand::cast(move->destination()));
761 InstructionOperand source(move->source());
762 AdjustStackPointerForTailCall(masm(), frame_access_state(),
763 destination_location.index());
764 if (source.IsStackSlot()) {
765 LocationOperand source_location(LocationOperand::cast(source));
766 __ Push(g.SlotToOperand(source_location.index()));
767 } else if (source.IsRegister()) {
768 LocationOperand source_location(LocationOperand::cast(source));
769 __ Push(source_location.GetRegister());
770 } else if (source.IsImmediate()) {
771 __ Push(Immediate(ImmediateOperand::cast(source).inline_value()));
772 } else {
773 // Pushes of non-scalar data types is not supported.
774 UNIMPLEMENTED();
775 }
776 frame_access_state()->IncreaseSPDelta(1);
777 move->Eliminate();
778 }
779 }
780 AdjustStackPointerForTailCall(masm(), frame_access_state(),
781 first_unused_stack_slot, false);
782 }
783
AssembleTailCallAfterGap(Instruction * instr,int first_unused_stack_slot)784 void CodeGenerator::AssembleTailCallAfterGap(Instruction* instr,
785 int first_unused_stack_slot) {
786 AdjustStackPointerForTailCall(masm(), frame_access_state(),
787 first_unused_stack_slot);
788 }
789
790 // Assembles an instruction after register allocation, producing machine code.
AssembleArchInstruction(Instruction * instr)791 CodeGenerator::CodeGenResult CodeGenerator::AssembleArchInstruction(
792 Instruction* instr) {
793 X64OperandConverter i(this, instr);
794 InstructionCode opcode = instr->opcode();
795 ArchOpcode arch_opcode = ArchOpcodeField::decode(opcode);
796 switch (arch_opcode) {
797 case kArchCallCodeObject: {
798 EnsureSpaceForLazyDeopt();
799 if (HasImmediateInput(instr, 0)) {
800 Handle<Code> code = Handle<Code>::cast(i.InputHeapObject(0));
801 __ Call(code, RelocInfo::CODE_TARGET);
802 } else {
803 Register reg = i.InputRegister(0);
804 __ addp(reg, Immediate(Code::kHeaderSize - kHeapObjectTag));
805 __ call(reg);
806 }
807 RecordCallPosition(instr);
808 frame_access_state()->ClearSPDelta();
809 break;
810 }
811 case kArchTailCallCodeObjectFromJSFunction:
812 case kArchTailCallCodeObject: {
813 if (arch_opcode == kArchTailCallCodeObjectFromJSFunction) {
814 AssemblePopArgumentsAdaptorFrame(kJavaScriptCallArgCountRegister,
815 i.TempRegister(0), i.TempRegister(1),
816 i.TempRegister(2));
817 }
818 if (HasImmediateInput(instr, 0)) {
819 Handle<Code> code = Handle<Code>::cast(i.InputHeapObject(0));
820 __ jmp(code, RelocInfo::CODE_TARGET);
821 } else {
822 Register reg = i.InputRegister(0);
823 __ addp(reg, Immediate(Code::kHeaderSize - kHeapObjectTag));
824 __ jmp(reg);
825 }
826 unwinding_info_writer_.MarkBlockWillExit();
827 frame_access_state()->ClearSPDelta();
828 frame_access_state()->SetFrameAccessToDefault();
829 break;
830 }
831 case kArchTailCallAddress: {
832 CHECK(!HasImmediateInput(instr, 0));
833 Register reg = i.InputRegister(0);
834 __ jmp(reg);
835 unwinding_info_writer_.MarkBlockWillExit();
836 frame_access_state()->ClearSPDelta();
837 frame_access_state()->SetFrameAccessToDefault();
838 break;
839 }
840 case kArchCallJSFunction: {
841 EnsureSpaceForLazyDeopt();
842 Register func = i.InputRegister(0);
843 if (FLAG_debug_code) {
844 // Check the function's context matches the context argument.
845 __ cmpp(rsi, FieldOperand(func, JSFunction::kContextOffset));
846 __ Assert(equal, kWrongFunctionContext);
847 }
848 __ Call(FieldOperand(func, JSFunction::kCodeEntryOffset));
849 frame_access_state()->ClearSPDelta();
850 RecordCallPosition(instr);
851 break;
852 }
853 case kArchTailCallJSFunctionFromJSFunction: {
854 Register func = i.InputRegister(0);
855 if (FLAG_debug_code) {
856 // Check the function's context matches the context argument.
857 __ cmpp(rsi, FieldOperand(func, JSFunction::kContextOffset));
858 __ Assert(equal, kWrongFunctionContext);
859 }
860 AssemblePopArgumentsAdaptorFrame(kJavaScriptCallArgCountRegister,
861 i.TempRegister(0), i.TempRegister(1),
862 i.TempRegister(2));
863 __ jmp(FieldOperand(func, JSFunction::kCodeEntryOffset));
864 frame_access_state()->ClearSPDelta();
865 frame_access_state()->SetFrameAccessToDefault();
866 break;
867 }
868 case kArchPrepareCallCFunction: {
869 // Frame alignment requires using FP-relative frame addressing.
870 frame_access_state()->SetFrameAccessToFP();
871 int const num_parameters = MiscField::decode(instr->opcode());
872 __ PrepareCallCFunction(num_parameters);
873 break;
874 }
875 case kArchPrepareTailCall:
876 AssemblePrepareTailCall();
877 break;
878 case kArchCallCFunction: {
879 int const num_parameters = MiscField::decode(instr->opcode());
880 if (HasImmediateInput(instr, 0)) {
881 ExternalReference ref = i.InputExternalReference(0);
882 __ CallCFunction(ref, num_parameters);
883 } else {
884 Register func = i.InputRegister(0);
885 __ CallCFunction(func, num_parameters);
886 }
887 frame_access_state()->SetFrameAccessToDefault();
888 frame_access_state()->ClearSPDelta();
889 break;
890 }
891 case kArchJmp:
892 AssembleArchJump(i.InputRpo(0));
893 break;
894 case kArchLookupSwitch:
895 AssembleArchLookupSwitch(instr);
896 break;
897 case kArchTableSwitch:
898 AssembleArchTableSwitch(instr);
899 break;
900 case kArchComment: {
901 Address comment_string = i.InputExternalReference(0).address();
902 __ RecordComment(reinterpret_cast<const char*>(comment_string));
903 break;
904 }
905 case kArchDebugBreak:
906 __ int3();
907 break;
908 case kArchNop:
909 case kArchThrowTerminator:
910 // don't emit code for nops.
911 break;
912 case kArchDeoptimize: {
913 int deopt_state_id =
914 BuildTranslation(instr, -1, 0, OutputFrameStateCombine::Ignore());
915 Deoptimizer::BailoutType bailout_type =
916 Deoptimizer::BailoutType(MiscField::decode(instr->opcode()));
917 CodeGenResult result = AssembleDeoptimizerCall(
918 deopt_state_id, bailout_type, current_source_position_);
919 if (result != kSuccess) return result;
920 break;
921 }
922 case kArchRet:
923 AssembleReturn(instr->InputAt(0));
924 break;
925 case kArchStackPointer:
926 __ movq(i.OutputRegister(), rsp);
927 break;
928 case kArchFramePointer:
929 __ movq(i.OutputRegister(), rbp);
930 break;
931 case kArchParentFramePointer:
932 if (frame_access_state()->has_frame()) {
933 __ movq(i.OutputRegister(), Operand(rbp, 0));
934 } else {
935 __ movq(i.OutputRegister(), rbp);
936 }
937 break;
938 case kArchTruncateDoubleToI: {
939 auto result = i.OutputRegister();
940 auto input = i.InputDoubleRegister(0);
941 auto ool = new (zone()) OutOfLineTruncateDoubleToI(
942 this, result, input, &unwinding_info_writer_);
943 // We use Cvttsd2siq instead of Cvttsd2si due to performance reasons. The
944 // use of Cvttsd2siq requires the movl below to avoid sign extension.
945 __ Cvttsd2siq(result, input);
946 __ cmpq(result, Immediate(1));
947 __ j(overflow, ool->entry());
948 __ bind(ool->exit());
949 __ movl(result, result);
950 break;
951 }
952 case kArchStoreWithWriteBarrier: {
953 RecordWriteMode mode =
954 static_cast<RecordWriteMode>(MiscField::decode(instr->opcode()));
955 Register object = i.InputRegister(0);
956 size_t index = 0;
957 Operand operand = i.MemoryOperand(&index);
958 Register value = i.InputRegister(index);
959 Register scratch0 = i.TempRegister(0);
960 Register scratch1 = i.TempRegister(1);
961 auto ool = new (zone()) OutOfLineRecordWrite(this, object, operand, value,
962 scratch0, scratch1, mode);
963 __ movp(operand, value);
964 __ CheckPageFlag(object, scratch0,
965 MemoryChunk::kPointersFromHereAreInterestingMask,
966 not_zero, ool->entry());
967 __ bind(ool->exit());
968 break;
969 }
970 case kArchStackSlot: {
971 FrameOffset offset =
972 frame_access_state()->GetFrameOffset(i.InputInt32(0));
973 Register base;
974 if (offset.from_stack_pointer()) {
975 base = rsp;
976 } else {
977 base = rbp;
978 }
979 __ leaq(i.OutputRegister(), Operand(base, offset.offset()));
980 break;
981 }
982 case kIeee754Float64Acos:
983 ASSEMBLE_IEEE754_UNOP(acos);
984 break;
985 case kIeee754Float64Acosh:
986 ASSEMBLE_IEEE754_UNOP(acosh);
987 break;
988 case kIeee754Float64Asin:
989 ASSEMBLE_IEEE754_UNOP(asin);
990 break;
991 case kIeee754Float64Asinh:
992 ASSEMBLE_IEEE754_UNOP(asinh);
993 break;
994 case kIeee754Float64Atan:
995 ASSEMBLE_IEEE754_UNOP(atan);
996 break;
997 case kIeee754Float64Atanh:
998 ASSEMBLE_IEEE754_UNOP(atanh);
999 break;
1000 case kIeee754Float64Atan2:
1001 ASSEMBLE_IEEE754_BINOP(atan2);
1002 break;
1003 case kIeee754Float64Cbrt:
1004 ASSEMBLE_IEEE754_UNOP(cbrt);
1005 break;
1006 case kIeee754Float64Cos:
1007 ASSEMBLE_IEEE754_UNOP(cos);
1008 break;
1009 case kIeee754Float64Cosh:
1010 ASSEMBLE_IEEE754_UNOP(cosh);
1011 break;
1012 case kIeee754Float64Exp:
1013 ASSEMBLE_IEEE754_UNOP(exp);
1014 break;
1015 case kIeee754Float64Expm1:
1016 ASSEMBLE_IEEE754_UNOP(expm1);
1017 break;
1018 case kIeee754Float64Log:
1019 ASSEMBLE_IEEE754_UNOP(log);
1020 break;
1021 case kIeee754Float64Log1p:
1022 ASSEMBLE_IEEE754_UNOP(log1p);
1023 break;
1024 case kIeee754Float64Log2:
1025 ASSEMBLE_IEEE754_UNOP(log2);
1026 break;
1027 case kIeee754Float64Log10:
1028 ASSEMBLE_IEEE754_UNOP(log10);
1029 break;
1030 case kIeee754Float64Pow: {
1031 // TODO(bmeurer): Improve integration of the stub.
1032 __ Movsd(xmm2, xmm0);
1033 MathPowStub stub(isolate(), MathPowStub::DOUBLE);
1034 __ CallStub(&stub);
1035 __ Movsd(xmm0, xmm3);
1036 break;
1037 }
1038 case kIeee754Float64Sin:
1039 ASSEMBLE_IEEE754_UNOP(sin);
1040 break;
1041 case kIeee754Float64Sinh:
1042 ASSEMBLE_IEEE754_UNOP(sinh);
1043 break;
1044 case kIeee754Float64Tan:
1045 ASSEMBLE_IEEE754_UNOP(tan);
1046 break;
1047 case kIeee754Float64Tanh:
1048 ASSEMBLE_IEEE754_UNOP(tanh);
1049 break;
1050 case kX64Add32:
1051 ASSEMBLE_BINOP(addl);
1052 break;
1053 case kX64Add:
1054 ASSEMBLE_BINOP(addq);
1055 break;
1056 case kX64Sub32:
1057 ASSEMBLE_BINOP(subl);
1058 break;
1059 case kX64Sub:
1060 ASSEMBLE_BINOP(subq);
1061 break;
1062 case kX64And32:
1063 ASSEMBLE_BINOP(andl);
1064 break;
1065 case kX64And:
1066 ASSEMBLE_BINOP(andq);
1067 break;
1068 case kX64Cmp8:
1069 ASSEMBLE_COMPARE(cmpb);
1070 break;
1071 case kX64Cmp16:
1072 ASSEMBLE_COMPARE(cmpw);
1073 break;
1074 case kX64Cmp32:
1075 ASSEMBLE_COMPARE(cmpl);
1076 break;
1077 case kX64Cmp:
1078 ASSEMBLE_COMPARE(cmpq);
1079 break;
1080 case kX64Test8:
1081 ASSEMBLE_COMPARE(testb);
1082 break;
1083 case kX64Test16:
1084 ASSEMBLE_COMPARE(testw);
1085 break;
1086 case kX64Test32:
1087 ASSEMBLE_COMPARE(testl);
1088 break;
1089 case kX64Test:
1090 ASSEMBLE_COMPARE(testq);
1091 break;
1092 case kX64Imul32:
1093 ASSEMBLE_MULT(imull);
1094 break;
1095 case kX64Imul:
1096 ASSEMBLE_MULT(imulq);
1097 break;
1098 case kX64ImulHigh32:
1099 if (instr->InputAt(1)->IsRegister()) {
1100 __ imull(i.InputRegister(1));
1101 } else {
1102 __ imull(i.InputOperand(1));
1103 }
1104 break;
1105 case kX64UmulHigh32:
1106 if (instr->InputAt(1)->IsRegister()) {
1107 __ mull(i.InputRegister(1));
1108 } else {
1109 __ mull(i.InputOperand(1));
1110 }
1111 break;
1112 case kX64Idiv32:
1113 __ cdq();
1114 __ idivl(i.InputRegister(1));
1115 break;
1116 case kX64Idiv:
1117 __ cqo();
1118 __ idivq(i.InputRegister(1));
1119 break;
1120 case kX64Udiv32:
1121 __ xorl(rdx, rdx);
1122 __ divl(i.InputRegister(1));
1123 break;
1124 case kX64Udiv:
1125 __ xorq(rdx, rdx);
1126 __ divq(i.InputRegister(1));
1127 break;
1128 case kX64Not:
1129 ASSEMBLE_UNOP(notq);
1130 break;
1131 case kX64Not32:
1132 ASSEMBLE_UNOP(notl);
1133 break;
1134 case kX64Neg:
1135 ASSEMBLE_UNOP(negq);
1136 break;
1137 case kX64Neg32:
1138 ASSEMBLE_UNOP(negl);
1139 break;
1140 case kX64Or32:
1141 ASSEMBLE_BINOP(orl);
1142 break;
1143 case kX64Or:
1144 ASSEMBLE_BINOP(orq);
1145 break;
1146 case kX64Xor32:
1147 ASSEMBLE_BINOP(xorl);
1148 break;
1149 case kX64Xor:
1150 ASSEMBLE_BINOP(xorq);
1151 break;
1152 case kX64Shl32:
1153 ASSEMBLE_SHIFT(shll, 5);
1154 break;
1155 case kX64Shl:
1156 ASSEMBLE_SHIFT(shlq, 6);
1157 break;
1158 case kX64Shr32:
1159 ASSEMBLE_SHIFT(shrl, 5);
1160 break;
1161 case kX64Shr:
1162 ASSEMBLE_SHIFT(shrq, 6);
1163 break;
1164 case kX64Sar32:
1165 ASSEMBLE_SHIFT(sarl, 5);
1166 break;
1167 case kX64Sar:
1168 ASSEMBLE_SHIFT(sarq, 6);
1169 break;
1170 case kX64Ror32:
1171 ASSEMBLE_SHIFT(rorl, 5);
1172 break;
1173 case kX64Ror:
1174 ASSEMBLE_SHIFT(rorq, 6);
1175 break;
1176 case kX64Lzcnt:
1177 if (instr->InputAt(0)->IsRegister()) {
1178 __ Lzcntq(i.OutputRegister(), i.InputRegister(0));
1179 } else {
1180 __ Lzcntq(i.OutputRegister(), i.InputOperand(0));
1181 }
1182 break;
1183 case kX64Lzcnt32:
1184 if (instr->InputAt(0)->IsRegister()) {
1185 __ Lzcntl(i.OutputRegister(), i.InputRegister(0));
1186 } else {
1187 __ Lzcntl(i.OutputRegister(), i.InputOperand(0));
1188 }
1189 break;
1190 case kX64Tzcnt:
1191 if (instr->InputAt(0)->IsRegister()) {
1192 __ Tzcntq(i.OutputRegister(), i.InputRegister(0));
1193 } else {
1194 __ Tzcntq(i.OutputRegister(), i.InputOperand(0));
1195 }
1196 break;
1197 case kX64Tzcnt32:
1198 if (instr->InputAt(0)->IsRegister()) {
1199 __ Tzcntl(i.OutputRegister(), i.InputRegister(0));
1200 } else {
1201 __ Tzcntl(i.OutputRegister(), i.InputOperand(0));
1202 }
1203 break;
1204 case kX64Popcnt:
1205 if (instr->InputAt(0)->IsRegister()) {
1206 __ Popcntq(i.OutputRegister(), i.InputRegister(0));
1207 } else {
1208 __ Popcntq(i.OutputRegister(), i.InputOperand(0));
1209 }
1210 break;
1211 case kX64Popcnt32:
1212 if (instr->InputAt(0)->IsRegister()) {
1213 __ Popcntl(i.OutputRegister(), i.InputRegister(0));
1214 } else {
1215 __ Popcntl(i.OutputRegister(), i.InputOperand(0));
1216 }
1217 break;
1218 case kSSEFloat32Cmp:
1219 ASSEMBLE_SSE_BINOP(Ucomiss);
1220 break;
1221 case kSSEFloat32Add:
1222 ASSEMBLE_SSE_BINOP(addss);
1223 break;
1224 case kSSEFloat32Sub:
1225 ASSEMBLE_SSE_BINOP(subss);
1226 break;
1227 case kSSEFloat32Mul:
1228 ASSEMBLE_SSE_BINOP(mulss);
1229 break;
1230 case kSSEFloat32Div:
1231 ASSEMBLE_SSE_BINOP(divss);
1232 // Don't delete this mov. It may improve performance on some CPUs,
1233 // when there is a (v)mulss depending on the result.
1234 __ movaps(i.OutputDoubleRegister(), i.OutputDoubleRegister());
1235 break;
1236 case kSSEFloat32Abs: {
1237 // TODO(bmeurer): Use RIP relative 128-bit constants.
1238 __ pcmpeqd(kScratchDoubleReg, kScratchDoubleReg);
1239 __ psrlq(kScratchDoubleReg, 33);
1240 __ andps(i.OutputDoubleRegister(), kScratchDoubleReg);
1241 break;
1242 }
1243 case kSSEFloat32Neg: {
1244 // TODO(bmeurer): Use RIP relative 128-bit constants.
1245 __ pcmpeqd(kScratchDoubleReg, kScratchDoubleReg);
1246 __ psllq(kScratchDoubleReg, 31);
1247 __ xorps(i.OutputDoubleRegister(), kScratchDoubleReg);
1248 break;
1249 }
1250 case kSSEFloat32Sqrt:
1251 ASSEMBLE_SSE_UNOP(sqrtss);
1252 break;
1253 case kSSEFloat32ToFloat64:
1254 ASSEMBLE_SSE_UNOP(Cvtss2sd);
1255 break;
1256 case kSSEFloat32Round: {
1257 CpuFeatureScope sse_scope(masm(), SSE4_1);
1258 RoundingMode const mode =
1259 static_cast<RoundingMode>(MiscField::decode(instr->opcode()));
1260 __ Roundss(i.OutputDoubleRegister(), i.InputDoubleRegister(0), mode);
1261 break;
1262 }
1263 case kSSEFloat32ToInt32:
1264 if (instr->InputAt(0)->IsFPRegister()) {
1265 __ Cvttss2si(i.OutputRegister(), i.InputDoubleRegister(0));
1266 } else {
1267 __ Cvttss2si(i.OutputRegister(), i.InputOperand(0));
1268 }
1269 break;
1270 case kSSEFloat32ToUint32: {
1271 if (instr->InputAt(0)->IsFPRegister()) {
1272 __ Cvttss2siq(i.OutputRegister(), i.InputDoubleRegister(0));
1273 } else {
1274 __ Cvttss2siq(i.OutputRegister(), i.InputOperand(0));
1275 }
1276 break;
1277 }
1278 case kSSEFloat64Cmp:
1279 ASSEMBLE_SSE_BINOP(Ucomisd);
1280 break;
1281 case kSSEFloat64Add:
1282 ASSEMBLE_SSE_BINOP(addsd);
1283 break;
1284 case kSSEFloat64Sub:
1285 ASSEMBLE_SSE_BINOP(subsd);
1286 break;
1287 case kSSEFloat64Mul:
1288 ASSEMBLE_SSE_BINOP(mulsd);
1289 break;
1290 case kSSEFloat64Div:
1291 ASSEMBLE_SSE_BINOP(divsd);
1292 // Don't delete this mov. It may improve performance on some CPUs,
1293 // when there is a (v)mulsd depending on the result.
1294 __ Movapd(i.OutputDoubleRegister(), i.OutputDoubleRegister());
1295 break;
1296 case kSSEFloat64Mod: {
1297 __ subq(rsp, Immediate(kDoubleSize));
1298 unwinding_info_writer_.MaybeIncreaseBaseOffsetAt(__ pc_offset(),
1299 kDoubleSize);
1300 // Move values to st(0) and st(1).
1301 __ Movsd(Operand(rsp, 0), i.InputDoubleRegister(1));
1302 __ fld_d(Operand(rsp, 0));
1303 __ Movsd(Operand(rsp, 0), i.InputDoubleRegister(0));
1304 __ fld_d(Operand(rsp, 0));
1305 // Loop while fprem isn't done.
1306 Label mod_loop;
1307 __ bind(&mod_loop);
1308 // This instructions traps on all kinds inputs, but we are assuming the
1309 // floating point control word is set to ignore them all.
1310 __ fprem();
1311 // The following 2 instruction implicitly use rax.
1312 __ fnstsw_ax();
1313 if (CpuFeatures::IsSupported(SAHF)) {
1314 CpuFeatureScope sahf_scope(masm(), SAHF);
1315 __ sahf();
1316 } else {
1317 __ shrl(rax, Immediate(8));
1318 __ andl(rax, Immediate(0xFF));
1319 __ pushq(rax);
1320 unwinding_info_writer_.MaybeIncreaseBaseOffsetAt(__ pc_offset(),
1321 kPointerSize);
1322 __ popfq();
1323 unwinding_info_writer_.MaybeIncreaseBaseOffsetAt(__ pc_offset(),
1324 -kPointerSize);
1325 }
1326 __ j(parity_even, &mod_loop);
1327 // Move output to stack and clean up.
1328 __ fstp(1);
1329 __ fstp_d(Operand(rsp, 0));
1330 __ Movsd(i.OutputDoubleRegister(), Operand(rsp, 0));
1331 __ addq(rsp, Immediate(kDoubleSize));
1332 unwinding_info_writer_.MaybeIncreaseBaseOffsetAt(__ pc_offset(),
1333 -kDoubleSize);
1334 break;
1335 }
1336 case kSSEFloat32Max: {
1337 Label compare_nan, compare_swap, done_compare;
1338 if (instr->InputAt(1)->IsFPRegister()) {
1339 __ Ucomiss(i.InputDoubleRegister(0), i.InputDoubleRegister(1));
1340 } else {
1341 __ Ucomiss(i.InputDoubleRegister(0), i.InputOperand(1));
1342 }
1343 auto ool =
1344 new (zone()) OutOfLineLoadFloat32NaN(this, i.OutputDoubleRegister());
1345 __ j(parity_even, ool->entry());
1346 __ j(above, &done_compare, Label::kNear);
1347 __ j(below, &compare_swap, Label::kNear);
1348 __ Movmskps(kScratchRegister, i.InputDoubleRegister(0));
1349 __ testl(kScratchRegister, Immediate(1));
1350 __ j(zero, &done_compare, Label::kNear);
1351 __ bind(&compare_swap);
1352 if (instr->InputAt(1)->IsFPRegister()) {
1353 __ Movss(i.InputDoubleRegister(0), i.InputDoubleRegister(1));
1354 } else {
1355 __ Movss(i.InputDoubleRegister(0), i.InputOperand(1));
1356 }
1357 __ bind(&done_compare);
1358 __ bind(ool->exit());
1359 break;
1360 }
1361 case kSSEFloat32Min: {
1362 Label compare_swap, done_compare;
1363 if (instr->InputAt(1)->IsFPRegister()) {
1364 __ Ucomiss(i.InputDoubleRegister(0), i.InputDoubleRegister(1));
1365 } else {
1366 __ Ucomiss(i.InputDoubleRegister(0), i.InputOperand(1));
1367 }
1368 auto ool =
1369 new (zone()) OutOfLineLoadFloat32NaN(this, i.OutputDoubleRegister());
1370 __ j(parity_even, ool->entry());
1371 __ j(below, &done_compare, Label::kNear);
1372 __ j(above, &compare_swap, Label::kNear);
1373 if (instr->InputAt(1)->IsFPRegister()) {
1374 __ Movmskps(kScratchRegister, i.InputDoubleRegister(1));
1375 } else {
1376 __ Movss(kScratchDoubleReg, i.InputOperand(1));
1377 __ Movmskps(kScratchRegister, kScratchDoubleReg);
1378 }
1379 __ testl(kScratchRegister, Immediate(1));
1380 __ j(zero, &done_compare, Label::kNear);
1381 __ bind(&compare_swap);
1382 if (instr->InputAt(1)->IsFPRegister()) {
1383 __ Movss(i.InputDoubleRegister(0), i.InputDoubleRegister(1));
1384 } else {
1385 __ Movss(i.InputDoubleRegister(0), i.InputOperand(1));
1386 }
1387 __ bind(&done_compare);
1388 __ bind(ool->exit());
1389 break;
1390 }
1391 case kSSEFloat64Max: {
1392 Label compare_nan, compare_swap, done_compare;
1393 if (instr->InputAt(1)->IsFPRegister()) {
1394 __ Ucomisd(i.InputDoubleRegister(0), i.InputDoubleRegister(1));
1395 } else {
1396 __ Ucomisd(i.InputDoubleRegister(0), i.InputOperand(1));
1397 }
1398 auto ool =
1399 new (zone()) OutOfLineLoadFloat64NaN(this, i.OutputDoubleRegister());
1400 __ j(parity_even, ool->entry());
1401 __ j(above, &done_compare, Label::kNear);
1402 __ j(below, &compare_swap, Label::kNear);
1403 __ Movmskpd(kScratchRegister, i.InputDoubleRegister(0));
1404 __ testl(kScratchRegister, Immediate(1));
1405 __ j(zero, &done_compare, Label::kNear);
1406 __ bind(&compare_swap);
1407 if (instr->InputAt(1)->IsFPRegister()) {
1408 __ Movsd(i.InputDoubleRegister(0), i.InputDoubleRegister(1));
1409 } else {
1410 __ Movsd(i.InputDoubleRegister(0), i.InputOperand(1));
1411 }
1412 __ bind(&done_compare);
1413 __ bind(ool->exit());
1414 break;
1415 }
1416 case kSSEFloat64Min: {
1417 Label compare_swap, done_compare;
1418 if (instr->InputAt(1)->IsFPRegister()) {
1419 __ Ucomisd(i.InputDoubleRegister(0), i.InputDoubleRegister(1));
1420 } else {
1421 __ Ucomisd(i.InputDoubleRegister(0), i.InputOperand(1));
1422 }
1423 auto ool =
1424 new (zone()) OutOfLineLoadFloat64NaN(this, i.OutputDoubleRegister());
1425 __ j(parity_even, ool->entry());
1426 __ j(below, &done_compare, Label::kNear);
1427 __ j(above, &compare_swap, Label::kNear);
1428 if (instr->InputAt(1)->IsFPRegister()) {
1429 __ Movmskpd(kScratchRegister, i.InputDoubleRegister(1));
1430 } else {
1431 __ Movsd(kScratchDoubleReg, i.InputOperand(1));
1432 __ Movmskpd(kScratchRegister, kScratchDoubleReg);
1433 }
1434 __ testl(kScratchRegister, Immediate(1));
1435 __ j(zero, &done_compare, Label::kNear);
1436 __ bind(&compare_swap);
1437 if (instr->InputAt(1)->IsFPRegister()) {
1438 __ Movsd(i.InputDoubleRegister(0), i.InputDoubleRegister(1));
1439 } else {
1440 __ Movsd(i.InputDoubleRegister(0), i.InputOperand(1));
1441 }
1442 __ bind(&done_compare);
1443 __ bind(ool->exit());
1444 break;
1445 }
1446 case kSSEFloat64Abs: {
1447 // TODO(bmeurer): Use RIP relative 128-bit constants.
1448 __ pcmpeqd(kScratchDoubleReg, kScratchDoubleReg);
1449 __ psrlq(kScratchDoubleReg, 1);
1450 __ andpd(i.OutputDoubleRegister(), kScratchDoubleReg);
1451 break;
1452 }
1453 case kSSEFloat64Neg: {
1454 // TODO(bmeurer): Use RIP relative 128-bit constants.
1455 __ pcmpeqd(kScratchDoubleReg, kScratchDoubleReg);
1456 __ psllq(kScratchDoubleReg, 63);
1457 __ xorpd(i.OutputDoubleRegister(), kScratchDoubleReg);
1458 break;
1459 }
1460 case kSSEFloat64Sqrt:
1461 ASSEMBLE_SSE_UNOP(Sqrtsd);
1462 break;
1463 case kSSEFloat64Round: {
1464 CpuFeatureScope sse_scope(masm(), SSE4_1);
1465 RoundingMode const mode =
1466 static_cast<RoundingMode>(MiscField::decode(instr->opcode()));
1467 __ Roundsd(i.OutputDoubleRegister(), i.InputDoubleRegister(0), mode);
1468 break;
1469 }
1470 case kSSEFloat64ToFloat32:
1471 ASSEMBLE_SSE_UNOP(Cvtsd2ss);
1472 break;
1473 case kSSEFloat64ToInt32:
1474 if (instr->InputAt(0)->IsFPRegister()) {
1475 __ Cvttsd2si(i.OutputRegister(), i.InputDoubleRegister(0));
1476 } else {
1477 __ Cvttsd2si(i.OutputRegister(), i.InputOperand(0));
1478 }
1479 break;
1480 case kSSEFloat64ToUint32: {
1481 if (instr->InputAt(0)->IsFPRegister()) {
1482 __ Cvttsd2siq(i.OutputRegister(), i.InputDoubleRegister(0));
1483 } else {
1484 __ Cvttsd2siq(i.OutputRegister(), i.InputOperand(0));
1485 }
1486 if (MiscField::decode(instr->opcode())) {
1487 __ AssertZeroExtended(i.OutputRegister());
1488 }
1489 break;
1490 }
1491 case kSSEFloat32ToInt64:
1492 if (instr->InputAt(0)->IsFPRegister()) {
1493 __ Cvttss2siq(i.OutputRegister(), i.InputDoubleRegister(0));
1494 } else {
1495 __ Cvttss2siq(i.OutputRegister(), i.InputOperand(0));
1496 }
1497 if (instr->OutputCount() > 1) {
1498 __ Set(i.OutputRegister(1), 1);
1499 Label done;
1500 Label fail;
1501 __ Move(kScratchDoubleReg, static_cast<float>(INT64_MIN));
1502 if (instr->InputAt(0)->IsFPRegister()) {
1503 __ Ucomiss(kScratchDoubleReg, i.InputDoubleRegister(0));
1504 } else {
1505 __ Ucomiss(kScratchDoubleReg, i.InputOperand(0));
1506 }
1507 // If the input is NaN, then the conversion fails.
1508 __ j(parity_even, &fail);
1509 // If the input is INT64_MIN, then the conversion succeeds.
1510 __ j(equal, &done);
1511 __ cmpq(i.OutputRegister(0), Immediate(1));
1512 // If the conversion results in INT64_MIN, but the input was not
1513 // INT64_MIN, then the conversion fails.
1514 __ j(no_overflow, &done);
1515 __ bind(&fail);
1516 __ Set(i.OutputRegister(1), 0);
1517 __ bind(&done);
1518 }
1519 break;
1520 case kSSEFloat64ToInt64:
1521 if (instr->InputAt(0)->IsFPRegister()) {
1522 __ Cvttsd2siq(i.OutputRegister(0), i.InputDoubleRegister(0));
1523 } else {
1524 __ Cvttsd2siq(i.OutputRegister(0), i.InputOperand(0));
1525 }
1526 if (instr->OutputCount() > 1) {
1527 __ Set(i.OutputRegister(1), 1);
1528 Label done;
1529 Label fail;
1530 __ Move(kScratchDoubleReg, static_cast<double>(INT64_MIN));
1531 if (instr->InputAt(0)->IsFPRegister()) {
1532 __ Ucomisd(kScratchDoubleReg, i.InputDoubleRegister(0));
1533 } else {
1534 __ Ucomisd(kScratchDoubleReg, i.InputOperand(0));
1535 }
1536 // If the input is NaN, then the conversion fails.
1537 __ j(parity_even, &fail);
1538 // If the input is INT64_MIN, then the conversion succeeds.
1539 __ j(equal, &done);
1540 __ cmpq(i.OutputRegister(0), Immediate(1));
1541 // If the conversion results in INT64_MIN, but the input was not
1542 // INT64_MIN, then the conversion fails.
1543 __ j(no_overflow, &done);
1544 __ bind(&fail);
1545 __ Set(i.OutputRegister(1), 0);
1546 __ bind(&done);
1547 }
1548 break;
1549 case kSSEFloat32ToUint64: {
1550 Label done;
1551 Label success;
1552 if (instr->OutputCount() > 1) {
1553 __ Set(i.OutputRegister(1), 0);
1554 }
1555 // There does not exist a Float32ToUint64 instruction, so we have to use
1556 // the Float32ToInt64 instruction.
1557 if (instr->InputAt(0)->IsFPRegister()) {
1558 __ Cvttss2siq(i.OutputRegister(), i.InputDoubleRegister(0));
1559 } else {
1560 __ Cvttss2siq(i.OutputRegister(), i.InputOperand(0));
1561 }
1562 // Check if the result of the Float32ToInt64 conversion is positive, we
1563 // are already done.
1564 __ testq(i.OutputRegister(), i.OutputRegister());
1565 __ j(positive, &success);
1566 // The result of the first conversion was negative, which means that the
1567 // input value was not within the positive int64 range. We subtract 2^64
1568 // and convert it again to see if it is within the uint64 range.
1569 __ Move(kScratchDoubleReg, -9223372036854775808.0f);
1570 if (instr->InputAt(0)->IsFPRegister()) {
1571 __ addss(kScratchDoubleReg, i.InputDoubleRegister(0));
1572 } else {
1573 __ addss(kScratchDoubleReg, i.InputOperand(0));
1574 }
1575 __ Cvttss2siq(i.OutputRegister(), kScratchDoubleReg);
1576 __ testq(i.OutputRegister(), i.OutputRegister());
1577 // The only possible negative value here is 0x80000000000000000, which is
1578 // used on x64 to indicate an integer overflow.
1579 __ j(negative, &done);
1580 // The input value is within uint64 range and the second conversion worked
1581 // successfully, but we still have to undo the subtraction we did
1582 // earlier.
1583 __ Set(kScratchRegister, 0x8000000000000000);
1584 __ orq(i.OutputRegister(), kScratchRegister);
1585 __ bind(&success);
1586 if (instr->OutputCount() > 1) {
1587 __ Set(i.OutputRegister(1), 1);
1588 }
1589 __ bind(&done);
1590 break;
1591 }
1592 case kSSEFloat64ToUint64: {
1593 Label done;
1594 Label success;
1595 if (instr->OutputCount() > 1) {
1596 __ Set(i.OutputRegister(1), 0);
1597 }
1598 // There does not exist a Float64ToUint64 instruction, so we have to use
1599 // the Float64ToInt64 instruction.
1600 if (instr->InputAt(0)->IsFPRegister()) {
1601 __ Cvttsd2siq(i.OutputRegister(), i.InputDoubleRegister(0));
1602 } else {
1603 __ Cvttsd2siq(i.OutputRegister(), i.InputOperand(0));
1604 }
1605 // Check if the result of the Float64ToInt64 conversion is positive, we
1606 // are already done.
1607 __ testq(i.OutputRegister(), i.OutputRegister());
1608 __ j(positive, &success);
1609 // The result of the first conversion was negative, which means that the
1610 // input value was not within the positive int64 range. We subtract 2^64
1611 // and convert it again to see if it is within the uint64 range.
1612 __ Move(kScratchDoubleReg, -9223372036854775808.0);
1613 if (instr->InputAt(0)->IsFPRegister()) {
1614 __ addsd(kScratchDoubleReg, i.InputDoubleRegister(0));
1615 } else {
1616 __ addsd(kScratchDoubleReg, i.InputOperand(0));
1617 }
1618 __ Cvttsd2siq(i.OutputRegister(), kScratchDoubleReg);
1619 __ testq(i.OutputRegister(), i.OutputRegister());
1620 // The only possible negative value here is 0x80000000000000000, which is
1621 // used on x64 to indicate an integer overflow.
1622 __ j(negative, &done);
1623 // The input value is within uint64 range and the second conversion worked
1624 // successfully, but we still have to undo the subtraction we did
1625 // earlier.
1626 __ Set(kScratchRegister, 0x8000000000000000);
1627 __ orq(i.OutputRegister(), kScratchRegister);
1628 __ bind(&success);
1629 if (instr->OutputCount() > 1) {
1630 __ Set(i.OutputRegister(1), 1);
1631 }
1632 __ bind(&done);
1633 break;
1634 }
1635 case kSSEInt32ToFloat64:
1636 if (instr->InputAt(0)->IsRegister()) {
1637 __ Cvtlsi2sd(i.OutputDoubleRegister(), i.InputRegister(0));
1638 } else {
1639 __ Cvtlsi2sd(i.OutputDoubleRegister(), i.InputOperand(0));
1640 }
1641 break;
1642 case kSSEInt32ToFloat32:
1643 if (instr->InputAt(0)->IsRegister()) {
1644 __ Cvtlsi2ss(i.OutputDoubleRegister(), i.InputRegister(0));
1645 } else {
1646 __ Cvtlsi2ss(i.OutputDoubleRegister(), i.InputOperand(0));
1647 }
1648 break;
1649 case kSSEInt64ToFloat32:
1650 if (instr->InputAt(0)->IsRegister()) {
1651 __ Cvtqsi2ss(i.OutputDoubleRegister(), i.InputRegister(0));
1652 } else {
1653 __ Cvtqsi2ss(i.OutputDoubleRegister(), i.InputOperand(0));
1654 }
1655 break;
1656 case kSSEInt64ToFloat64:
1657 if (instr->InputAt(0)->IsRegister()) {
1658 __ Cvtqsi2sd(i.OutputDoubleRegister(), i.InputRegister(0));
1659 } else {
1660 __ Cvtqsi2sd(i.OutputDoubleRegister(), i.InputOperand(0));
1661 }
1662 break;
1663 case kSSEUint64ToFloat32:
1664 if (instr->InputAt(0)->IsRegister()) {
1665 __ movq(kScratchRegister, i.InputRegister(0));
1666 } else {
1667 __ movq(kScratchRegister, i.InputOperand(0));
1668 }
1669 __ Cvtqui2ss(i.OutputDoubleRegister(), kScratchRegister,
1670 i.TempRegister(0));
1671 break;
1672 case kSSEUint64ToFloat64:
1673 if (instr->InputAt(0)->IsRegister()) {
1674 __ movq(kScratchRegister, i.InputRegister(0));
1675 } else {
1676 __ movq(kScratchRegister, i.InputOperand(0));
1677 }
1678 __ Cvtqui2sd(i.OutputDoubleRegister(), kScratchRegister,
1679 i.TempRegister(0));
1680 break;
1681 case kSSEUint32ToFloat64:
1682 if (instr->InputAt(0)->IsRegister()) {
1683 __ movl(kScratchRegister, i.InputRegister(0));
1684 } else {
1685 __ movl(kScratchRegister, i.InputOperand(0));
1686 }
1687 __ Cvtqsi2sd(i.OutputDoubleRegister(), kScratchRegister);
1688 break;
1689 case kSSEUint32ToFloat32:
1690 if (instr->InputAt(0)->IsRegister()) {
1691 __ movl(kScratchRegister, i.InputRegister(0));
1692 } else {
1693 __ movl(kScratchRegister, i.InputOperand(0));
1694 }
1695 __ Cvtqsi2ss(i.OutputDoubleRegister(), kScratchRegister);
1696 break;
1697 case kSSEFloat64ExtractLowWord32:
1698 if (instr->InputAt(0)->IsFPStackSlot()) {
1699 __ movl(i.OutputRegister(), i.InputOperand(0));
1700 } else {
1701 __ Movd(i.OutputRegister(), i.InputDoubleRegister(0));
1702 }
1703 break;
1704 case kSSEFloat64ExtractHighWord32:
1705 if (instr->InputAt(0)->IsFPStackSlot()) {
1706 __ movl(i.OutputRegister(), i.InputOperand(0, kDoubleSize / 2));
1707 } else {
1708 __ Pextrd(i.OutputRegister(), i.InputDoubleRegister(0), 1);
1709 }
1710 break;
1711 case kSSEFloat64InsertLowWord32:
1712 if (instr->InputAt(1)->IsRegister()) {
1713 __ Pinsrd(i.OutputDoubleRegister(), i.InputRegister(1), 0);
1714 } else {
1715 __ Pinsrd(i.OutputDoubleRegister(), i.InputOperand(1), 0);
1716 }
1717 break;
1718 case kSSEFloat64InsertHighWord32:
1719 if (instr->InputAt(1)->IsRegister()) {
1720 __ Pinsrd(i.OutputDoubleRegister(), i.InputRegister(1), 1);
1721 } else {
1722 __ Pinsrd(i.OutputDoubleRegister(), i.InputOperand(1), 1);
1723 }
1724 break;
1725 case kSSEFloat64LoadLowWord32:
1726 if (instr->InputAt(0)->IsRegister()) {
1727 __ Movd(i.OutputDoubleRegister(), i.InputRegister(0));
1728 } else {
1729 __ Movd(i.OutputDoubleRegister(), i.InputOperand(0));
1730 }
1731 break;
1732 case kAVXFloat32Cmp: {
1733 CpuFeatureScope avx_scope(masm(), AVX);
1734 if (instr->InputAt(1)->IsFPRegister()) {
1735 __ vucomiss(i.InputDoubleRegister(0), i.InputDoubleRegister(1));
1736 } else {
1737 __ vucomiss(i.InputDoubleRegister(0), i.InputOperand(1));
1738 }
1739 break;
1740 }
1741 case kAVXFloat32Add:
1742 ASSEMBLE_AVX_BINOP(vaddss);
1743 break;
1744 case kAVXFloat32Sub:
1745 ASSEMBLE_AVX_BINOP(vsubss);
1746 break;
1747 case kAVXFloat32Mul:
1748 ASSEMBLE_AVX_BINOP(vmulss);
1749 break;
1750 case kAVXFloat32Div:
1751 ASSEMBLE_AVX_BINOP(vdivss);
1752 // Don't delete this mov. It may improve performance on some CPUs,
1753 // when there is a (v)mulss depending on the result.
1754 __ Movaps(i.OutputDoubleRegister(), i.OutputDoubleRegister());
1755 break;
1756 case kAVXFloat64Cmp: {
1757 CpuFeatureScope avx_scope(masm(), AVX);
1758 if (instr->InputAt(1)->IsFPRegister()) {
1759 __ vucomisd(i.InputDoubleRegister(0), i.InputDoubleRegister(1));
1760 } else {
1761 __ vucomisd(i.InputDoubleRegister(0), i.InputOperand(1));
1762 }
1763 break;
1764 }
1765 case kAVXFloat64Add:
1766 ASSEMBLE_AVX_BINOP(vaddsd);
1767 break;
1768 case kAVXFloat64Sub:
1769 ASSEMBLE_AVX_BINOP(vsubsd);
1770 break;
1771 case kAVXFloat64Mul:
1772 ASSEMBLE_AVX_BINOP(vmulsd);
1773 break;
1774 case kAVXFloat64Div:
1775 ASSEMBLE_AVX_BINOP(vdivsd);
1776 // Don't delete this mov. It may improve performance on some CPUs,
1777 // when there is a (v)mulsd depending on the result.
1778 __ Movapd(i.OutputDoubleRegister(), i.OutputDoubleRegister());
1779 break;
1780 case kAVXFloat32Abs: {
1781 // TODO(bmeurer): Use RIP relative 128-bit constants.
1782 CpuFeatureScope avx_scope(masm(), AVX);
1783 __ vpcmpeqd(kScratchDoubleReg, kScratchDoubleReg, kScratchDoubleReg);
1784 __ vpsrlq(kScratchDoubleReg, kScratchDoubleReg, 33);
1785 if (instr->InputAt(0)->IsFPRegister()) {
1786 __ vandps(i.OutputDoubleRegister(), kScratchDoubleReg,
1787 i.InputDoubleRegister(0));
1788 } else {
1789 __ vandps(i.OutputDoubleRegister(), kScratchDoubleReg,
1790 i.InputOperand(0));
1791 }
1792 break;
1793 }
1794 case kAVXFloat32Neg: {
1795 // TODO(bmeurer): Use RIP relative 128-bit constants.
1796 CpuFeatureScope avx_scope(masm(), AVX);
1797 __ vpcmpeqd(kScratchDoubleReg, kScratchDoubleReg, kScratchDoubleReg);
1798 __ vpsllq(kScratchDoubleReg, kScratchDoubleReg, 31);
1799 if (instr->InputAt(0)->IsFPRegister()) {
1800 __ vxorps(i.OutputDoubleRegister(), kScratchDoubleReg,
1801 i.InputDoubleRegister(0));
1802 } else {
1803 __ vxorps(i.OutputDoubleRegister(), kScratchDoubleReg,
1804 i.InputOperand(0));
1805 }
1806 break;
1807 }
1808 case kAVXFloat64Abs: {
1809 // TODO(bmeurer): Use RIP relative 128-bit constants.
1810 CpuFeatureScope avx_scope(masm(), AVX);
1811 __ vpcmpeqd(kScratchDoubleReg, kScratchDoubleReg, kScratchDoubleReg);
1812 __ vpsrlq(kScratchDoubleReg, kScratchDoubleReg, 1);
1813 if (instr->InputAt(0)->IsFPRegister()) {
1814 __ vandpd(i.OutputDoubleRegister(), kScratchDoubleReg,
1815 i.InputDoubleRegister(0));
1816 } else {
1817 __ vandpd(i.OutputDoubleRegister(), kScratchDoubleReg,
1818 i.InputOperand(0));
1819 }
1820 break;
1821 }
1822 case kAVXFloat64Neg: {
1823 // TODO(bmeurer): Use RIP relative 128-bit constants.
1824 CpuFeatureScope avx_scope(masm(), AVX);
1825 __ vpcmpeqd(kScratchDoubleReg, kScratchDoubleReg, kScratchDoubleReg);
1826 __ vpsllq(kScratchDoubleReg, kScratchDoubleReg, 63);
1827 if (instr->InputAt(0)->IsFPRegister()) {
1828 __ vxorpd(i.OutputDoubleRegister(), kScratchDoubleReg,
1829 i.InputDoubleRegister(0));
1830 } else {
1831 __ vxorpd(i.OutputDoubleRegister(), kScratchDoubleReg,
1832 i.InputOperand(0));
1833 }
1834 break;
1835 }
1836 case kSSEFloat64SilenceNaN:
1837 __ Xorpd(kScratchDoubleReg, kScratchDoubleReg);
1838 __ Subsd(i.InputDoubleRegister(0), kScratchDoubleReg);
1839 break;
1840 case kX64Movsxbl:
1841 ASSEMBLE_MOVX(movsxbl);
1842 __ AssertZeroExtended(i.OutputRegister());
1843 break;
1844 case kX64Movzxbl:
1845 ASSEMBLE_MOVX(movzxbl);
1846 __ AssertZeroExtended(i.OutputRegister());
1847 break;
1848 case kX64Movsxbq:
1849 ASSEMBLE_MOVX(movsxbq);
1850 break;
1851 case kX64Movzxbq:
1852 ASSEMBLE_MOVX(movzxbq);
1853 __ AssertZeroExtended(i.OutputRegister());
1854 break;
1855 case kX64Movb: {
1856 size_t index = 0;
1857 Operand operand = i.MemoryOperand(&index);
1858 if (HasImmediateInput(instr, index)) {
1859 __ movb(operand, Immediate(i.InputInt8(index)));
1860 } else {
1861 __ movb(operand, i.InputRegister(index));
1862 }
1863 break;
1864 }
1865 case kX64Movsxwl:
1866 ASSEMBLE_MOVX(movsxwl);
1867 __ AssertZeroExtended(i.OutputRegister());
1868 break;
1869 case kX64Movzxwl:
1870 ASSEMBLE_MOVX(movzxwl);
1871 __ AssertZeroExtended(i.OutputRegister());
1872 break;
1873 case kX64Movsxwq:
1874 ASSEMBLE_MOVX(movsxwq);
1875 break;
1876 case kX64Movzxwq:
1877 ASSEMBLE_MOVX(movzxwq);
1878 __ AssertZeroExtended(i.OutputRegister());
1879 break;
1880 case kX64Movw: {
1881 size_t index = 0;
1882 Operand operand = i.MemoryOperand(&index);
1883 if (HasImmediateInput(instr, index)) {
1884 __ movw(operand, Immediate(i.InputInt16(index)));
1885 } else {
1886 __ movw(operand, i.InputRegister(index));
1887 }
1888 break;
1889 }
1890 case kX64Movl:
1891 case kX64TrapMovl:
1892 if (instr->HasOutput()) {
1893 if (instr->addressing_mode() == kMode_None) {
1894 if (instr->InputAt(0)->IsRegister()) {
1895 __ movl(i.OutputRegister(), i.InputRegister(0));
1896 } else {
1897 __ movl(i.OutputRegister(), i.InputOperand(0));
1898 }
1899 } else {
1900 Address pc = __ pc();
1901 __ movl(i.OutputRegister(), i.MemoryOperand());
1902
1903 if (arch_opcode == kX64TrapMovl) {
1904 bool frame_elided = !frame_access_state()->has_frame();
1905 new (zone()) WasmOutOfLineTrap(this, pc, frame_elided,
1906 i.InputRegister(2), i.InputInt32(3));
1907 }
1908 }
1909 __ AssertZeroExtended(i.OutputRegister());
1910 } else {
1911 size_t index = 0;
1912 Operand operand = i.MemoryOperand(&index);
1913 if (HasImmediateInput(instr, index)) {
1914 __ movl(operand, i.InputImmediate(index));
1915 } else {
1916 __ movl(operand, i.InputRegister(index));
1917 }
1918 }
1919 break;
1920 case kX64Movsxlq:
1921 ASSEMBLE_MOVX(movsxlq);
1922 break;
1923 case kX64Movq:
1924 if (instr->HasOutput()) {
1925 __ movq(i.OutputRegister(), i.MemoryOperand());
1926 } else {
1927 size_t index = 0;
1928 Operand operand = i.MemoryOperand(&index);
1929 if (HasImmediateInput(instr, index)) {
1930 __ movq(operand, i.InputImmediate(index));
1931 } else {
1932 __ movq(operand, i.InputRegister(index));
1933 }
1934 }
1935 break;
1936 case kX64Movss:
1937 if (instr->HasOutput()) {
1938 __ movss(i.OutputDoubleRegister(), i.MemoryOperand());
1939 } else {
1940 size_t index = 0;
1941 Operand operand = i.MemoryOperand(&index);
1942 __ movss(operand, i.InputDoubleRegister(index));
1943 }
1944 break;
1945 case kX64Movsd:
1946 if (instr->HasOutput()) {
1947 __ Movsd(i.OutputDoubleRegister(), i.MemoryOperand());
1948 } else {
1949 size_t index = 0;
1950 Operand operand = i.MemoryOperand(&index);
1951 __ Movsd(operand, i.InputDoubleRegister(index));
1952 }
1953 break;
1954 case kX64BitcastFI:
1955 if (instr->InputAt(0)->IsFPStackSlot()) {
1956 __ movl(i.OutputRegister(), i.InputOperand(0));
1957 } else {
1958 __ Movd(i.OutputRegister(), i.InputDoubleRegister(0));
1959 }
1960 break;
1961 case kX64BitcastDL:
1962 if (instr->InputAt(0)->IsFPStackSlot()) {
1963 __ movq(i.OutputRegister(), i.InputOperand(0));
1964 } else {
1965 __ Movq(i.OutputRegister(), i.InputDoubleRegister(0));
1966 }
1967 break;
1968 case kX64BitcastIF:
1969 if (instr->InputAt(0)->IsRegister()) {
1970 __ Movd(i.OutputDoubleRegister(), i.InputRegister(0));
1971 } else {
1972 __ movss(i.OutputDoubleRegister(), i.InputOperand(0));
1973 }
1974 break;
1975 case kX64BitcastLD:
1976 if (instr->InputAt(0)->IsRegister()) {
1977 __ Movq(i.OutputDoubleRegister(), i.InputRegister(0));
1978 } else {
1979 __ Movsd(i.OutputDoubleRegister(), i.InputOperand(0));
1980 }
1981 break;
1982 case kX64Lea32: {
1983 AddressingMode mode = AddressingModeField::decode(instr->opcode());
1984 // Shorten "leal" to "addl", "subl" or "shll" if the register allocation
1985 // and addressing mode just happens to work out. The "addl"/"subl" forms
1986 // in these cases are faster based on measurements.
1987 if (i.InputRegister(0).is(i.OutputRegister())) {
1988 if (mode == kMode_MRI) {
1989 int32_t constant_summand = i.InputInt32(1);
1990 if (constant_summand > 0) {
1991 __ addl(i.OutputRegister(), Immediate(constant_summand));
1992 } else if (constant_summand < 0) {
1993 __ subl(i.OutputRegister(), Immediate(-constant_summand));
1994 }
1995 } else if (mode == kMode_MR1) {
1996 if (i.InputRegister(1).is(i.OutputRegister())) {
1997 __ shll(i.OutputRegister(), Immediate(1));
1998 } else {
1999 __ addl(i.OutputRegister(), i.InputRegister(1));
2000 }
2001 } else if (mode == kMode_M2) {
2002 __ shll(i.OutputRegister(), Immediate(1));
2003 } else if (mode == kMode_M4) {
2004 __ shll(i.OutputRegister(), Immediate(2));
2005 } else if (mode == kMode_M8) {
2006 __ shll(i.OutputRegister(), Immediate(3));
2007 } else {
2008 __ leal(i.OutputRegister(), i.MemoryOperand());
2009 }
2010 } else if (mode == kMode_MR1 &&
2011 i.InputRegister(1).is(i.OutputRegister())) {
2012 __ addl(i.OutputRegister(), i.InputRegister(0));
2013 } else {
2014 __ leal(i.OutputRegister(), i.MemoryOperand());
2015 }
2016 __ AssertZeroExtended(i.OutputRegister());
2017 break;
2018 }
2019 case kX64Lea: {
2020 AddressingMode mode = AddressingModeField::decode(instr->opcode());
2021 // Shorten "leaq" to "addq", "subq" or "shlq" if the register allocation
2022 // and addressing mode just happens to work out. The "addq"/"subq" forms
2023 // in these cases are faster based on measurements.
2024 if (i.InputRegister(0).is(i.OutputRegister())) {
2025 if (mode == kMode_MRI) {
2026 int32_t constant_summand = i.InputInt32(1);
2027 if (constant_summand > 0) {
2028 __ addq(i.OutputRegister(), Immediate(constant_summand));
2029 } else if (constant_summand < 0) {
2030 __ subq(i.OutputRegister(), Immediate(-constant_summand));
2031 }
2032 } else if (mode == kMode_MR1) {
2033 if (i.InputRegister(1).is(i.OutputRegister())) {
2034 __ shlq(i.OutputRegister(), Immediate(1));
2035 } else {
2036 __ addq(i.OutputRegister(), i.InputRegister(1));
2037 }
2038 } else if (mode == kMode_M2) {
2039 __ shlq(i.OutputRegister(), Immediate(1));
2040 } else if (mode == kMode_M4) {
2041 __ shlq(i.OutputRegister(), Immediate(2));
2042 } else if (mode == kMode_M8) {
2043 __ shlq(i.OutputRegister(), Immediate(3));
2044 } else {
2045 __ leaq(i.OutputRegister(), i.MemoryOperand());
2046 }
2047 } else if (mode == kMode_MR1 &&
2048 i.InputRegister(1).is(i.OutputRegister())) {
2049 __ addq(i.OutputRegister(), i.InputRegister(0));
2050 } else {
2051 __ leaq(i.OutputRegister(), i.MemoryOperand());
2052 }
2053 break;
2054 }
2055 case kX64Dec32:
2056 __ decl(i.OutputRegister());
2057 break;
2058 case kX64Inc32:
2059 __ incl(i.OutputRegister());
2060 break;
2061 case kX64Push:
2062 if (HasImmediateInput(instr, 0)) {
2063 __ pushq(i.InputImmediate(0));
2064 frame_access_state()->IncreaseSPDelta(1);
2065 unwinding_info_writer_.MaybeIncreaseBaseOffsetAt(__ pc_offset(),
2066 kPointerSize);
2067 } else {
2068 if (instr->InputAt(0)->IsRegister()) {
2069 __ pushq(i.InputRegister(0));
2070 frame_access_state()->IncreaseSPDelta(1);
2071 unwinding_info_writer_.MaybeIncreaseBaseOffsetAt(__ pc_offset(),
2072 kPointerSize);
2073 } else if (instr->InputAt(0)->IsFPRegister()) {
2074 // TODO(titzer): use another machine instruction?
2075 __ subq(rsp, Immediate(kDoubleSize));
2076 frame_access_state()->IncreaseSPDelta(kDoubleSize / kPointerSize);
2077 unwinding_info_writer_.MaybeIncreaseBaseOffsetAt(__ pc_offset(),
2078 kDoubleSize);
2079 __ Movsd(Operand(rsp, 0), i.InputDoubleRegister(0));
2080 } else {
2081 __ pushq(i.InputOperand(0));
2082 frame_access_state()->IncreaseSPDelta(1);
2083 unwinding_info_writer_.MaybeIncreaseBaseOffsetAt(__ pc_offset(),
2084 kPointerSize);
2085 }
2086 }
2087 break;
2088 case kX64Poke: {
2089 int const slot = MiscField::decode(instr->opcode());
2090 if (HasImmediateInput(instr, 0)) {
2091 __ movq(Operand(rsp, slot * kPointerSize), i.InputImmediate(0));
2092 } else {
2093 __ movq(Operand(rsp, slot * kPointerSize), i.InputRegister(0));
2094 }
2095 break;
2096 }
2097 case kX64Xchgb: {
2098 size_t index = 0;
2099 Operand operand = i.MemoryOperand(&index);
2100 __ xchgb(i.InputRegister(index), operand);
2101 break;
2102 }
2103 case kX64Xchgw: {
2104 size_t index = 0;
2105 Operand operand = i.MemoryOperand(&index);
2106 __ xchgw(i.InputRegister(index), operand);
2107 break;
2108 }
2109 case kX64Xchgl: {
2110 size_t index = 0;
2111 Operand operand = i.MemoryOperand(&index);
2112 __ xchgl(i.InputRegister(index), operand);
2113 break;
2114 }
2115 case kX64Int32x4Create: {
2116 CpuFeatureScope sse_scope(masm(), SSE4_1);
2117 XMMRegister dst = i.OutputSimd128Register();
2118 __ Movd(dst, i.InputRegister(0));
2119 __ shufps(dst, dst, 0x0);
2120 break;
2121 }
2122 case kX64Int32x4ExtractLane: {
2123 CpuFeatureScope sse_scope(masm(), SSE4_1);
2124 __ Pextrd(i.OutputRegister(), i.InputSimd128Register(0), i.InputInt8(1));
2125 break;
2126 }
2127 case kCheckedLoadInt8:
2128 ASSEMBLE_CHECKED_LOAD_INTEGER(movsxbl);
2129 break;
2130 case kCheckedLoadUint8:
2131 ASSEMBLE_CHECKED_LOAD_INTEGER(movzxbl);
2132 break;
2133 case kCheckedLoadInt16:
2134 ASSEMBLE_CHECKED_LOAD_INTEGER(movsxwl);
2135 break;
2136 case kCheckedLoadUint16:
2137 ASSEMBLE_CHECKED_LOAD_INTEGER(movzxwl);
2138 break;
2139 case kCheckedLoadWord32:
2140 ASSEMBLE_CHECKED_LOAD_INTEGER(movl);
2141 break;
2142 case kCheckedLoadWord64:
2143 ASSEMBLE_CHECKED_LOAD_INTEGER(movq);
2144 break;
2145 case kCheckedLoadFloat32:
2146 ASSEMBLE_CHECKED_LOAD_FLOAT(Movss, OutOfLineLoadFloat32NaN);
2147 break;
2148 case kCheckedLoadFloat64:
2149 ASSEMBLE_CHECKED_LOAD_FLOAT(Movsd, OutOfLineLoadFloat64NaN);
2150 break;
2151 case kCheckedStoreWord8:
2152 ASSEMBLE_CHECKED_STORE_INTEGER(movb);
2153 break;
2154 case kCheckedStoreWord16:
2155 ASSEMBLE_CHECKED_STORE_INTEGER(movw);
2156 break;
2157 case kCheckedStoreWord32:
2158 ASSEMBLE_CHECKED_STORE_INTEGER(movl);
2159 break;
2160 case kCheckedStoreWord64:
2161 ASSEMBLE_CHECKED_STORE_INTEGER(movq);
2162 break;
2163 case kCheckedStoreFloat32:
2164 ASSEMBLE_CHECKED_STORE_FLOAT(Movss);
2165 break;
2166 case kCheckedStoreFloat64:
2167 ASSEMBLE_CHECKED_STORE_FLOAT(Movsd);
2168 break;
2169 case kX64StackCheck:
2170 __ CompareRoot(rsp, Heap::kStackLimitRootIndex);
2171 break;
2172 case kAtomicLoadInt8:
2173 case kAtomicLoadUint8:
2174 case kAtomicLoadInt16:
2175 case kAtomicLoadUint16:
2176 case kAtomicLoadWord32:
2177 case kAtomicStoreWord8:
2178 case kAtomicStoreWord16:
2179 case kAtomicStoreWord32:
2180 UNREACHABLE(); // Won't be generated by instruction selector.
2181 break;
2182 }
2183 return kSuccess;
2184 } // NOLINT(readability/fn_size)
2185
2186
2187 // Assembles branches after this instruction.
AssembleArchBranch(Instruction * instr,BranchInfo * branch)2188 void CodeGenerator::AssembleArchBranch(Instruction* instr, BranchInfo* branch) {
2189 X64OperandConverter i(this, instr);
2190 Label::Distance flabel_distance =
2191 branch->fallthru ? Label::kNear : Label::kFar;
2192 Label* tlabel = branch->true_label;
2193 Label* flabel = branch->false_label;
2194 switch (branch->condition) {
2195 case kUnorderedEqual:
2196 __ j(parity_even, flabel, flabel_distance);
2197 // Fall through.
2198 case kEqual:
2199 __ j(equal, tlabel);
2200 break;
2201 case kUnorderedNotEqual:
2202 __ j(parity_even, tlabel);
2203 // Fall through.
2204 case kNotEqual:
2205 __ j(not_equal, tlabel);
2206 break;
2207 case kSignedLessThan:
2208 __ j(less, tlabel);
2209 break;
2210 case kSignedGreaterThanOrEqual:
2211 __ j(greater_equal, tlabel);
2212 break;
2213 case kSignedLessThanOrEqual:
2214 __ j(less_equal, tlabel);
2215 break;
2216 case kSignedGreaterThan:
2217 __ j(greater, tlabel);
2218 break;
2219 case kUnsignedLessThan:
2220 __ j(below, tlabel);
2221 break;
2222 case kUnsignedGreaterThanOrEqual:
2223 __ j(above_equal, tlabel);
2224 break;
2225 case kUnsignedLessThanOrEqual:
2226 __ j(below_equal, tlabel);
2227 break;
2228 case kUnsignedGreaterThan:
2229 __ j(above, tlabel);
2230 break;
2231 case kOverflow:
2232 __ j(overflow, tlabel);
2233 break;
2234 case kNotOverflow:
2235 __ j(no_overflow, tlabel);
2236 break;
2237 default:
2238 UNREACHABLE();
2239 break;
2240 }
2241 if (!branch->fallthru) __ jmp(flabel, flabel_distance);
2242 }
2243
2244
AssembleArchJump(RpoNumber target)2245 void CodeGenerator::AssembleArchJump(RpoNumber target) {
2246 if (!IsNextInAssemblyOrder(target)) __ jmp(GetLabel(target));
2247 }
2248
2249
2250 // Assembles boolean materializations after this instruction.
AssembleArchBoolean(Instruction * instr,FlagsCondition condition)2251 void CodeGenerator::AssembleArchBoolean(Instruction* instr,
2252 FlagsCondition condition) {
2253 X64OperandConverter i(this, instr);
2254 Label done;
2255
2256 // Materialize a full 64-bit 1 or 0 value. The result register is always the
2257 // last output of the instruction.
2258 Label check;
2259 DCHECK_NE(0u, instr->OutputCount());
2260 Register reg = i.OutputRegister(instr->OutputCount() - 1);
2261 Condition cc = no_condition;
2262 switch (condition) {
2263 case kUnorderedEqual:
2264 __ j(parity_odd, &check, Label::kNear);
2265 __ movl(reg, Immediate(0));
2266 __ jmp(&done, Label::kNear);
2267 // Fall through.
2268 case kEqual:
2269 cc = equal;
2270 break;
2271 case kUnorderedNotEqual:
2272 __ j(parity_odd, &check, Label::kNear);
2273 __ movl(reg, Immediate(1));
2274 __ jmp(&done, Label::kNear);
2275 // Fall through.
2276 case kNotEqual:
2277 cc = not_equal;
2278 break;
2279 case kSignedLessThan:
2280 cc = less;
2281 break;
2282 case kSignedGreaterThanOrEqual:
2283 cc = greater_equal;
2284 break;
2285 case kSignedLessThanOrEqual:
2286 cc = less_equal;
2287 break;
2288 case kSignedGreaterThan:
2289 cc = greater;
2290 break;
2291 case kUnsignedLessThan:
2292 cc = below;
2293 break;
2294 case kUnsignedGreaterThanOrEqual:
2295 cc = above_equal;
2296 break;
2297 case kUnsignedLessThanOrEqual:
2298 cc = below_equal;
2299 break;
2300 case kUnsignedGreaterThan:
2301 cc = above;
2302 break;
2303 case kOverflow:
2304 cc = overflow;
2305 break;
2306 case kNotOverflow:
2307 cc = no_overflow;
2308 break;
2309 default:
2310 UNREACHABLE();
2311 break;
2312 }
2313 __ bind(&check);
2314 __ setcc(cc, reg);
2315 __ movzxbl(reg, reg);
2316 __ bind(&done);
2317 }
2318
2319
AssembleArchLookupSwitch(Instruction * instr)2320 void CodeGenerator::AssembleArchLookupSwitch(Instruction* instr) {
2321 X64OperandConverter i(this, instr);
2322 Register input = i.InputRegister(0);
2323 for (size_t index = 2; index < instr->InputCount(); index += 2) {
2324 __ cmpl(input, Immediate(i.InputInt32(index + 0)));
2325 __ j(equal, GetLabel(i.InputRpo(index + 1)));
2326 }
2327 AssembleArchJump(i.InputRpo(1));
2328 }
2329
2330
AssembleArchTableSwitch(Instruction * instr)2331 void CodeGenerator::AssembleArchTableSwitch(Instruction* instr) {
2332 X64OperandConverter i(this, instr);
2333 Register input = i.InputRegister(0);
2334 int32_t const case_count = static_cast<int32_t>(instr->InputCount() - 2);
2335 Label** cases = zone()->NewArray<Label*>(case_count);
2336 for (int32_t index = 0; index < case_count; ++index) {
2337 cases[index] = GetLabel(i.InputRpo(index + 2));
2338 }
2339 Label* const table = AddJumpTable(cases, case_count);
2340 __ cmpl(input, Immediate(case_count));
2341 __ j(above_equal, GetLabel(i.InputRpo(1)));
2342 __ leaq(kScratchRegister, Operand(table));
2343 __ jmp(Operand(kScratchRegister, input, times_8, 0));
2344 }
2345
AssembleDeoptimizerCall(int deoptimization_id,Deoptimizer::BailoutType bailout_type,SourcePosition pos)2346 CodeGenerator::CodeGenResult CodeGenerator::AssembleDeoptimizerCall(
2347 int deoptimization_id, Deoptimizer::BailoutType bailout_type,
2348 SourcePosition pos) {
2349 Address deopt_entry = Deoptimizer::GetDeoptimizationEntry(
2350 isolate(), deoptimization_id, bailout_type);
2351 if (deopt_entry == nullptr) return kTooManyDeoptimizationBailouts;
2352 DeoptimizeReason deoptimization_reason =
2353 GetDeoptimizationReason(deoptimization_id);
2354 __ RecordDeoptReason(deoptimization_reason, pos, deoptimization_id);
2355 __ call(deopt_entry, RelocInfo::RUNTIME_ENTRY);
2356 return kSuccess;
2357 }
2358
2359
2360 namespace {
2361
2362 static const int kQuadWordSize = 16;
2363
2364 } // namespace
2365
FinishFrame(Frame * frame)2366 void CodeGenerator::FinishFrame(Frame* frame) {
2367 CallDescriptor* descriptor = linkage()->GetIncomingDescriptor();
2368
2369 const RegList saves_fp = descriptor->CalleeSavedFPRegisters();
2370 if (saves_fp != 0) {
2371 frame->AlignSavedCalleeRegisterSlots();
2372 if (saves_fp != 0) { // Save callee-saved XMM registers.
2373 const uint32_t saves_fp_count = base::bits::CountPopulation32(saves_fp);
2374 frame->AllocateSavedCalleeRegisterSlots(saves_fp_count *
2375 (kQuadWordSize / kPointerSize));
2376 }
2377 }
2378 const RegList saves = descriptor->CalleeSavedRegisters();
2379 if (saves != 0) { // Save callee-saved registers.
2380 int count = 0;
2381 for (int i = Register::kNumRegisters - 1; i >= 0; i--) {
2382 if (((1 << i) & saves)) {
2383 ++count;
2384 }
2385 }
2386 frame->AllocateSavedCalleeRegisterSlots(count);
2387 }
2388 }
2389
AssembleConstructFrame()2390 void CodeGenerator::AssembleConstructFrame() {
2391 CallDescriptor* descriptor = linkage()->GetIncomingDescriptor();
2392 if (frame_access_state()->has_frame()) {
2393 int pc_base = __ pc_offset();
2394
2395 if (descriptor->IsCFunctionCall()) {
2396 __ pushq(rbp);
2397 __ movq(rbp, rsp);
2398 } else if (descriptor->IsJSFunctionCall()) {
2399 __ Prologue(this->info()->GeneratePreagedPrologue());
2400 if (descriptor->PushArgumentCount()) {
2401 __ pushq(kJavaScriptCallArgCountRegister);
2402 }
2403 } else {
2404 __ StubPrologue(info()->GetOutputStackFrameType());
2405 }
2406
2407 if (!descriptor->IsJSFunctionCall() || !info()->GeneratePreagedPrologue()) {
2408 unwinding_info_writer_.MarkFrameConstructed(pc_base);
2409 }
2410 }
2411 int shrink_slots =
2412 frame()->GetTotalFrameSlotCount() - descriptor->CalculateFixedFrameSize();
2413
2414 if (info()->is_osr()) {
2415 // TurboFan OSR-compiled functions cannot be entered directly.
2416 __ Abort(kShouldNotDirectlyEnterOsrFunction);
2417
2418 // Unoptimized code jumps directly to this entrypoint while the unoptimized
2419 // frame is still on the stack. Optimized code uses OSR values directly from
2420 // the unoptimized frame. Thus, all that needs to be done is to allocate the
2421 // remaining stack slots.
2422 if (FLAG_code_comments) __ RecordComment("-- OSR entrypoint --");
2423 osr_pc_offset_ = __ pc_offset();
2424 shrink_slots -= static_cast<int>(OsrHelper(info()).UnoptimizedFrameSlots());
2425 }
2426
2427 const RegList saves_fp = descriptor->CalleeSavedFPRegisters();
2428 if (shrink_slots > 0) {
2429 __ subq(rsp, Immediate(shrink_slots * kPointerSize));
2430 }
2431
2432 if (saves_fp != 0) { // Save callee-saved XMM registers.
2433 const uint32_t saves_fp_count = base::bits::CountPopulation32(saves_fp);
2434 const int stack_size = saves_fp_count * kQuadWordSize;
2435 // Adjust the stack pointer.
2436 __ subp(rsp, Immediate(stack_size));
2437 // Store the registers on the stack.
2438 int slot_idx = 0;
2439 for (int i = 0; i < XMMRegister::kMaxNumRegisters; i++) {
2440 if (!((1 << i) & saves_fp)) continue;
2441 __ movdqu(Operand(rsp, kQuadWordSize * slot_idx),
2442 XMMRegister::from_code(i));
2443 slot_idx++;
2444 }
2445 }
2446
2447 const RegList saves = descriptor->CalleeSavedRegisters();
2448 if (saves != 0) { // Save callee-saved registers.
2449 for (int i = Register::kNumRegisters - 1; i >= 0; i--) {
2450 if (!((1 << i) & saves)) continue;
2451 __ pushq(Register::from_code(i));
2452 }
2453 }
2454 }
2455
AssembleReturn(InstructionOperand * pop)2456 void CodeGenerator::AssembleReturn(InstructionOperand* pop) {
2457 CallDescriptor* descriptor = linkage()->GetIncomingDescriptor();
2458
2459 // Restore registers.
2460 const RegList saves = descriptor->CalleeSavedRegisters();
2461 if (saves != 0) {
2462 for (int i = 0; i < Register::kNumRegisters; i++) {
2463 if (!((1 << i) & saves)) continue;
2464 __ popq(Register::from_code(i));
2465 }
2466 }
2467 const RegList saves_fp = descriptor->CalleeSavedFPRegisters();
2468 if (saves_fp != 0) {
2469 const uint32_t saves_fp_count = base::bits::CountPopulation32(saves_fp);
2470 const int stack_size = saves_fp_count * kQuadWordSize;
2471 // Load the registers from the stack.
2472 int slot_idx = 0;
2473 for (int i = 0; i < XMMRegister::kMaxNumRegisters; i++) {
2474 if (!((1 << i) & saves_fp)) continue;
2475 __ movdqu(XMMRegister::from_code(i),
2476 Operand(rsp, kQuadWordSize * slot_idx));
2477 slot_idx++;
2478 }
2479 // Adjust the stack pointer.
2480 __ addp(rsp, Immediate(stack_size));
2481 }
2482
2483 unwinding_info_writer_.MarkBlockWillExit();
2484
2485 // Might need rcx for scratch if pop_size is too big or if there is a variable
2486 // pop count.
2487 DCHECK_EQ(0u, descriptor->CalleeSavedRegisters() & rcx.bit());
2488 DCHECK_EQ(0u, descriptor->CalleeSavedRegisters() & rdx.bit());
2489 size_t pop_size = descriptor->StackParameterCount() * kPointerSize;
2490 X64OperandConverter g(this, nullptr);
2491 if (descriptor->IsCFunctionCall()) {
2492 AssembleDeconstructFrame();
2493 } else if (frame_access_state()->has_frame()) {
2494 if (pop->IsImmediate() && g.ToConstant(pop).ToInt32() == 0) {
2495 // Canonicalize JSFunction return sites for now.
2496 if (return_label_.is_bound()) {
2497 __ jmp(&return_label_);
2498 return;
2499 } else {
2500 __ bind(&return_label_);
2501 AssembleDeconstructFrame();
2502 }
2503 } else {
2504 AssembleDeconstructFrame();
2505 }
2506 }
2507
2508 if (pop->IsImmediate()) {
2509 DCHECK_EQ(Constant::kInt32, g.ToConstant(pop).type());
2510 pop_size += g.ToConstant(pop).ToInt32() * kPointerSize;
2511 CHECK_LT(pop_size, static_cast<size_t>(std::numeric_limits<int>::max()));
2512 __ Ret(static_cast<int>(pop_size), rcx);
2513 } else {
2514 Register pop_reg = g.ToRegister(pop);
2515 Register scratch_reg = pop_reg.is(rcx) ? rdx : rcx;
2516 __ popq(scratch_reg);
2517 __ leaq(rsp, Operand(rsp, pop_reg, times_8, static_cast<int>(pop_size)));
2518 __ jmp(scratch_reg);
2519 }
2520 }
2521
2522
AssembleMove(InstructionOperand * source,InstructionOperand * destination)2523 void CodeGenerator::AssembleMove(InstructionOperand* source,
2524 InstructionOperand* destination) {
2525 X64OperandConverter g(this, nullptr);
2526 // Dispatch on the source and destination operand kinds. Not all
2527 // combinations are possible.
2528 if (source->IsRegister()) {
2529 DCHECK(destination->IsRegister() || destination->IsStackSlot());
2530 Register src = g.ToRegister(source);
2531 if (destination->IsRegister()) {
2532 __ movq(g.ToRegister(destination), src);
2533 } else {
2534 __ movq(g.ToOperand(destination), src);
2535 }
2536 } else if (source->IsStackSlot()) {
2537 DCHECK(destination->IsRegister() || destination->IsStackSlot());
2538 Operand src = g.ToOperand(source);
2539 if (destination->IsRegister()) {
2540 Register dst = g.ToRegister(destination);
2541 __ movq(dst, src);
2542 } else {
2543 // Spill on demand to use a temporary register for memory-to-memory
2544 // moves.
2545 Register tmp = kScratchRegister;
2546 Operand dst = g.ToOperand(destination);
2547 __ movq(tmp, src);
2548 __ movq(dst, tmp);
2549 }
2550 } else if (source->IsConstant()) {
2551 ConstantOperand* constant_source = ConstantOperand::cast(source);
2552 Constant src = g.ToConstant(constant_source);
2553 if (destination->IsRegister() || destination->IsStackSlot()) {
2554 Register dst = destination->IsRegister() ? g.ToRegister(destination)
2555 : kScratchRegister;
2556 switch (src.type()) {
2557 case Constant::kInt32: {
2558 if (src.rmode() == RelocInfo::WASM_MEMORY_REFERENCE ||
2559 src.rmode() == RelocInfo::WASM_GLOBAL_REFERENCE) {
2560 __ movq(dst, src.ToInt64(), src.rmode());
2561 } else {
2562 // TODO(dcarney): don't need scratch in this case.
2563 int32_t value = src.ToInt32();
2564 if (value == 0) {
2565 __ xorl(dst, dst);
2566 } else {
2567 if (src.rmode() == RelocInfo::WASM_MEMORY_SIZE_REFERENCE) {
2568 __ movl(dst, Immediate(value, src.rmode()));
2569 } else {
2570 __ movl(dst, Immediate(value));
2571 }
2572 }
2573 }
2574 break;
2575 }
2576 case Constant::kInt64:
2577 if (src.rmode() == RelocInfo::WASM_MEMORY_REFERENCE ||
2578 src.rmode() == RelocInfo::WASM_GLOBAL_REFERENCE) {
2579 __ movq(dst, src.ToInt64(), src.rmode());
2580 } else {
2581 DCHECK(src.rmode() != RelocInfo::WASM_MEMORY_SIZE_REFERENCE);
2582 __ Set(dst, src.ToInt64());
2583 }
2584 break;
2585 case Constant::kFloat32:
2586 __ Move(dst,
2587 isolate()->factory()->NewNumber(src.ToFloat32(), TENURED));
2588 break;
2589 case Constant::kFloat64:
2590 __ Move(dst,
2591 isolate()->factory()->NewNumber(src.ToFloat64(), TENURED));
2592 break;
2593 case Constant::kExternalReference:
2594 __ Move(dst, src.ToExternalReference());
2595 break;
2596 case Constant::kHeapObject: {
2597 Handle<HeapObject> src_object = src.ToHeapObject();
2598 Heap::RootListIndex index;
2599 if (IsMaterializableFromRoot(src_object, &index)) {
2600 __ LoadRoot(dst, index);
2601 } else {
2602 __ Move(dst, src_object);
2603 }
2604 break;
2605 }
2606 case Constant::kRpoNumber:
2607 UNREACHABLE(); // TODO(dcarney): load of labels on x64.
2608 break;
2609 }
2610 if (destination->IsStackSlot()) {
2611 __ movq(g.ToOperand(destination), kScratchRegister);
2612 }
2613 } else if (src.type() == Constant::kFloat32) {
2614 // TODO(turbofan): Can we do better here?
2615 uint32_t src_const = bit_cast<uint32_t>(src.ToFloat32());
2616 if (destination->IsFPRegister()) {
2617 __ Move(g.ToDoubleRegister(destination), src_const);
2618 } else {
2619 DCHECK(destination->IsFPStackSlot());
2620 Operand dst = g.ToOperand(destination);
2621 __ movl(dst, Immediate(src_const));
2622 }
2623 } else {
2624 DCHECK_EQ(Constant::kFloat64, src.type());
2625 uint64_t src_const = bit_cast<uint64_t>(src.ToFloat64());
2626 if (destination->IsFPRegister()) {
2627 __ Move(g.ToDoubleRegister(destination), src_const);
2628 } else {
2629 DCHECK(destination->IsFPStackSlot());
2630 __ movq(kScratchRegister, src_const);
2631 __ movq(g.ToOperand(destination), kScratchRegister);
2632 }
2633 }
2634 } else if (source->IsFPRegister()) {
2635 XMMRegister src = g.ToDoubleRegister(source);
2636 if (destination->IsFPRegister()) {
2637 XMMRegister dst = g.ToDoubleRegister(destination);
2638 __ Movapd(dst, src);
2639 } else {
2640 DCHECK(destination->IsFPStackSlot());
2641 Operand dst = g.ToOperand(destination);
2642 MachineRepresentation rep =
2643 LocationOperand::cast(source)->representation();
2644 if (rep != MachineRepresentation::kSimd128) {
2645 __ Movsd(dst, src);
2646 } else {
2647 __ Movups(dst, src);
2648 }
2649 }
2650 } else if (source->IsFPStackSlot()) {
2651 DCHECK(destination->IsFPRegister() || destination->IsFPStackSlot());
2652 Operand src = g.ToOperand(source);
2653 MachineRepresentation rep = LocationOperand::cast(source)->representation();
2654 if (destination->IsFPRegister()) {
2655 XMMRegister dst = g.ToDoubleRegister(destination);
2656 if (rep != MachineRepresentation::kSimd128) {
2657 __ Movsd(dst, src);
2658 } else {
2659 __ Movups(dst, src);
2660 }
2661 } else {
2662 Operand dst = g.ToOperand(destination);
2663 if (rep != MachineRepresentation::kSimd128) {
2664 __ Movsd(kScratchDoubleReg, src);
2665 __ Movsd(dst, kScratchDoubleReg);
2666 } else {
2667 __ Movups(kScratchDoubleReg, src);
2668 __ Movups(dst, kScratchDoubleReg);
2669 }
2670 }
2671 } else {
2672 UNREACHABLE();
2673 }
2674 }
2675
2676
AssembleSwap(InstructionOperand * source,InstructionOperand * destination)2677 void CodeGenerator::AssembleSwap(InstructionOperand* source,
2678 InstructionOperand* destination) {
2679 X64OperandConverter g(this, nullptr);
2680 // Dispatch on the source and destination operand kinds. Not all
2681 // combinations are possible.
2682 if (source->IsRegister() && destination->IsRegister()) {
2683 // Register-register.
2684 Register src = g.ToRegister(source);
2685 Register dst = g.ToRegister(destination);
2686 __ movq(kScratchRegister, src);
2687 __ movq(src, dst);
2688 __ movq(dst, kScratchRegister);
2689 } else if (source->IsRegister() && destination->IsStackSlot()) {
2690 Register src = g.ToRegister(source);
2691 __ pushq(src);
2692 frame_access_state()->IncreaseSPDelta(1);
2693 unwinding_info_writer_.MaybeIncreaseBaseOffsetAt(__ pc_offset(),
2694 kPointerSize);
2695 Operand dst = g.ToOperand(destination);
2696 __ movq(src, dst);
2697 frame_access_state()->IncreaseSPDelta(-1);
2698 dst = g.ToOperand(destination);
2699 __ popq(dst);
2700 unwinding_info_writer_.MaybeIncreaseBaseOffsetAt(__ pc_offset(),
2701 -kPointerSize);
2702 } else if ((source->IsStackSlot() && destination->IsStackSlot()) ||
2703 (source->IsFPStackSlot() && destination->IsFPStackSlot())) {
2704 // Memory-memory.
2705 Operand src = g.ToOperand(source);
2706 Operand dst = g.ToOperand(destination);
2707 MachineRepresentation rep = LocationOperand::cast(source)->representation();
2708 if (rep != MachineRepresentation::kSimd128) {
2709 Register tmp = kScratchRegister;
2710 __ movq(tmp, dst);
2711 __ pushq(src);
2712 unwinding_info_writer_.MaybeIncreaseBaseOffsetAt(__ pc_offset(),
2713 kPointerSize);
2714 frame_access_state()->IncreaseSPDelta(1);
2715 src = g.ToOperand(source);
2716 __ movq(src, tmp);
2717 frame_access_state()->IncreaseSPDelta(-1);
2718 dst = g.ToOperand(destination);
2719 __ popq(dst);
2720 unwinding_info_writer_.MaybeIncreaseBaseOffsetAt(__ pc_offset(),
2721 -kPointerSize);
2722 } else {
2723 // Use the XOR trick to swap without a temporary.
2724 __ Movups(kScratchDoubleReg, src);
2725 __ Xorps(kScratchDoubleReg, dst); // scratch contains src ^ dst.
2726 __ Movups(src, kScratchDoubleReg);
2727 __ Xorps(kScratchDoubleReg, dst); // scratch contains src.
2728 __ Movups(dst, kScratchDoubleReg);
2729 __ Xorps(kScratchDoubleReg, src); // scratch contains dst.
2730 __ Movups(src, kScratchDoubleReg);
2731 }
2732 } else if (source->IsFPRegister() && destination->IsFPRegister()) {
2733 // XMM register-register swap.
2734 XMMRegister src = g.ToDoubleRegister(source);
2735 XMMRegister dst = g.ToDoubleRegister(destination);
2736 __ Movapd(kScratchDoubleReg, src);
2737 __ Movapd(src, dst);
2738 __ Movapd(dst, kScratchDoubleReg);
2739 } else if (source->IsFPRegister() && destination->IsFPStackSlot()) {
2740 // XMM register-memory swap.
2741 XMMRegister src = g.ToDoubleRegister(source);
2742 Operand dst = g.ToOperand(destination);
2743 MachineRepresentation rep = LocationOperand::cast(source)->representation();
2744 if (rep != MachineRepresentation::kSimd128) {
2745 __ Movsd(kScratchDoubleReg, src);
2746 __ Movsd(src, dst);
2747 __ Movsd(dst, kScratchDoubleReg);
2748 } else {
2749 __ Movups(kScratchDoubleReg, src);
2750 __ Movups(src, dst);
2751 __ Movups(dst, kScratchDoubleReg);
2752 }
2753 } else {
2754 // No other combinations are possible.
2755 UNREACHABLE();
2756 }
2757 }
2758
2759
AssembleJumpTable(Label ** targets,size_t target_count)2760 void CodeGenerator::AssembleJumpTable(Label** targets, size_t target_count) {
2761 for (size_t index = 0; index < target_count; ++index) {
2762 __ dq(targets[index]);
2763 }
2764 }
2765
2766
EnsureSpaceForLazyDeopt()2767 void CodeGenerator::EnsureSpaceForLazyDeopt() {
2768 if (!info()->ShouldEnsureSpaceForLazyDeopt()) {
2769 return;
2770 }
2771
2772 int space_needed = Deoptimizer::patch_size();
2773 // Ensure that we have enough space after the previous lazy-bailout
2774 // instruction for patching the code here.
2775 int current_pc = __ pc_offset();
2776 if (current_pc < last_lazy_deopt_pc_ + space_needed) {
2777 int padding_size = last_lazy_deopt_pc_ + space_needed - current_pc;
2778 __ Nop(padding_size);
2779 }
2780 }
2781
2782 #undef __
2783
2784 } // namespace compiler
2785 } // namespace internal
2786 } // namespace v8
2787