1 // Copyright 2017 the V8 project authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4 
5 #ifndef V8_WASM_BASELINE_X64_LIFTOFF_ASSEMBLER_X64_H_
6 #define V8_WASM_BASELINE_X64_LIFTOFF_ASSEMBLER_X64_H_
7 
8 #include "src/wasm/baseline/liftoff-assembler.h"
9 
10 #include "src/assembler.h"
11 #include "src/wasm/value-type.h"
12 
13 namespace v8 {
14 namespace internal {
15 namespace wasm {
16 
17 #define REQUIRE_CPU_FEATURE(name, ...)   \
18   if (!CpuFeatures::IsSupported(name)) { \
19     bailout("no " #name);                \
20     return __VA_ARGS__;                  \
21   }                                      \
22   CpuFeatureScope feature(this, name);
23 
24 namespace liftoff {
25 
26 static_assert((kLiftoffAssemblerGpCacheRegs &
27                Register::ListOf<kScratchRegister>()) == 0,
28               "scratch register must not be used as cache registers");
29 
30 constexpr DoubleRegister kScratchDoubleReg2 = xmm14;
31 static_assert(kScratchDoubleReg != kScratchDoubleReg2, "collision");
32 static_assert(
33     (kLiftoffAssemblerFpCacheRegs &
34      DoubleRegister::ListOf<kScratchDoubleReg, kScratchDoubleReg2>()) == 0,
35     "scratch registers must not be used as cache registers");
36 
37 // rbp-8 holds the stack marker, rbp-16 is the instance parameter, first stack
38 // slot is located at rbp-24.
39 constexpr int32_t kConstantStackSpace = 16;
40 constexpr int32_t kFirstStackSlotOffset =
41     kConstantStackSpace + LiftoffAssembler::kStackSlotSize;
42 
GetStackSlot(uint32_t index)43 inline Operand GetStackSlot(uint32_t index) {
44   int32_t offset = index * LiftoffAssembler::kStackSlotSize;
45   return Operand(rbp, -kFirstStackSlotOffset - offset);
46 }
47 
48 // TODO(clemensh): Make this a constexpr variable once Operand is constexpr.
GetInstanceOperand()49 inline Operand GetInstanceOperand() { return Operand(rbp, -16); }
50 
GetMemOp(LiftoffAssembler * assm,Register addr,Register offset,uint32_t offset_imm)51 inline Operand GetMemOp(LiftoffAssembler* assm, Register addr, Register offset,
52                         uint32_t offset_imm) {
53   if (is_uint31(offset_imm)) {
54     if (offset == no_reg) return Operand(addr, offset_imm);
55     return Operand(addr, offset, times_1, offset_imm);
56   }
57   // Offset immediate does not fit in 31 bits.
58   Register scratch = kScratchRegister;
59   assm->movl(scratch, Immediate(offset_imm));
60   if (offset != no_reg) {
61     assm->addq(scratch, offset);
62   }
63   return Operand(addr, scratch, times_1, 0);
64 }
65 
Load(LiftoffAssembler * assm,LiftoffRegister dst,Operand src,ValueType type)66 inline void Load(LiftoffAssembler* assm, LiftoffRegister dst, Operand src,
67                  ValueType type) {
68   switch (type) {
69     case kWasmI32:
70       assm->movl(dst.gp(), src);
71       break;
72     case kWasmI64:
73       assm->movq(dst.gp(), src);
74       break;
75     case kWasmF32:
76       assm->Movss(dst.fp(), src);
77       break;
78     case kWasmF64:
79       assm->Movsd(dst.fp(), src);
80       break;
81     default:
82       UNREACHABLE();
83   }
84 }
85 
Store(LiftoffAssembler * assm,Operand dst,LiftoffRegister src,ValueType type)86 inline void Store(LiftoffAssembler* assm, Operand dst, LiftoffRegister src,
87                   ValueType type) {
88   switch (type) {
89     case kWasmI32:
90       assm->movl(dst, src.gp());
91       break;
92     case kWasmI64:
93       assm->movq(dst, src.gp());
94       break;
95     case kWasmF32:
96       assm->Movss(dst, src.fp());
97       break;
98     case kWasmF64:
99       assm->Movsd(dst, src.fp());
100       break;
101     default:
102       UNREACHABLE();
103   }
104 }
105 
push(LiftoffAssembler * assm,LiftoffRegister reg,ValueType type)106 inline void push(LiftoffAssembler* assm, LiftoffRegister reg, ValueType type) {
107   switch (type) {
108     case kWasmI32:
109     case kWasmI64:
110       assm->pushq(reg.gp());
111       break;
112     case kWasmF32:
113       assm->subp(rsp, Immediate(kPointerSize));
114       assm->Movss(Operand(rsp, 0), reg.fp());
115       break;
116     case kWasmF64:
117       assm->subp(rsp, Immediate(kPointerSize));
118       assm->Movsd(Operand(rsp, 0), reg.fp());
119       break;
120     default:
121       UNREACHABLE();
122   }
123 }
124 
125 template <typename... Regs>
SpillRegisters(LiftoffAssembler * assm,Regs...regs)126 inline void SpillRegisters(LiftoffAssembler* assm, Regs... regs) {
127   for (LiftoffRegister r : {LiftoffRegister(regs)...}) {
128     if (assm->cache_state()->is_used(r)) assm->SpillRegister(r);
129   }
130 }
131 
132 }  // namespace liftoff
133 
PrepareStackFrame()134 int LiftoffAssembler::PrepareStackFrame() {
135   int offset = pc_offset();
136   sub_sp_32(0);
137   return offset;
138 }
139 
PatchPrepareStackFrame(int offset,uint32_t stack_slots)140 void LiftoffAssembler::PatchPrepareStackFrame(int offset,
141                                               uint32_t stack_slots) {
142   uint32_t bytes = liftoff::kConstantStackSpace + kStackSlotSize * stack_slots;
143   DCHECK_LE(bytes, kMaxInt);
144   // We can't run out of space, just pass anything big enough to not cause the
145   // assembler to try to grow the buffer.
146   constexpr int kAvailableSpace = 64;
147   Assembler patching_assembler(AssemblerOptions{}, buffer_ + offset,
148                                kAvailableSpace);
149   patching_assembler.sub_sp_32(bytes);
150 }
151 
FinishCode()152 void LiftoffAssembler::FinishCode() {}
153 
AbortCompilation()154 void LiftoffAssembler::AbortCompilation() {}
155 
LoadConstant(LiftoffRegister reg,WasmValue value,RelocInfo::Mode rmode)156 void LiftoffAssembler::LoadConstant(LiftoffRegister reg, WasmValue value,
157                                     RelocInfo::Mode rmode) {
158   switch (value.type()) {
159     case kWasmI32:
160       if (value.to_i32() == 0 && RelocInfo::IsNone(rmode)) {
161         xorl(reg.gp(), reg.gp());
162       } else {
163         movl(reg.gp(), Immediate(value.to_i32(), rmode));
164       }
165       break;
166     case kWasmI64:
167       if (RelocInfo::IsNone(rmode)) {
168         TurboAssembler::Set(reg.gp(), value.to_i64());
169       } else {
170         movq(reg.gp(), value.to_i64(), rmode);
171       }
172       break;
173     case kWasmF32:
174       TurboAssembler::Move(reg.fp(), value.to_f32_boxed().get_bits());
175       break;
176     case kWasmF64:
177       TurboAssembler::Move(reg.fp(), value.to_f64_boxed().get_bits());
178       break;
179     default:
180       UNREACHABLE();
181   }
182 }
183 
LoadFromInstance(Register dst,uint32_t offset,int size)184 void LiftoffAssembler::LoadFromInstance(Register dst, uint32_t offset,
185                                         int size) {
186   DCHECK_LE(offset, kMaxInt);
187   movp(dst, liftoff::GetInstanceOperand());
188   DCHECK(size == 4 || size == 8);
189   if (size == 4) {
190     movl(dst, Operand(dst, offset));
191   } else {
192     movq(dst, Operand(dst, offset));
193   }
194 }
195 
SpillInstance(Register instance)196 void LiftoffAssembler::SpillInstance(Register instance) {
197   movp(liftoff::GetInstanceOperand(), instance);
198 }
199 
FillInstanceInto(Register dst)200 void LiftoffAssembler::FillInstanceInto(Register dst) {
201   movp(dst, liftoff::GetInstanceOperand());
202 }
203 
Load(LiftoffRegister dst,Register src_addr,Register offset_reg,uint32_t offset_imm,LoadType type,LiftoffRegList pinned,uint32_t * protected_load_pc,bool is_load_mem)204 void LiftoffAssembler::Load(LiftoffRegister dst, Register src_addr,
205                             Register offset_reg, uint32_t offset_imm,
206                             LoadType type, LiftoffRegList pinned,
207                             uint32_t* protected_load_pc, bool is_load_mem) {
208   if (emit_debug_code() && offset_reg != no_reg) {
209     AssertZeroExtended(offset_reg);
210   }
211   Operand src_op = liftoff::GetMemOp(this, src_addr, offset_reg, offset_imm);
212   if (protected_load_pc) *protected_load_pc = pc_offset();
213   switch (type.value()) {
214     case LoadType::kI32Load8U:
215     case LoadType::kI64Load8U:
216       movzxbl(dst.gp(), src_op);
217       break;
218     case LoadType::kI32Load8S:
219       movsxbl(dst.gp(), src_op);
220       break;
221     case LoadType::kI64Load8S:
222       movsxbq(dst.gp(), src_op);
223       break;
224     case LoadType::kI32Load16U:
225     case LoadType::kI64Load16U:
226       movzxwl(dst.gp(), src_op);
227       break;
228     case LoadType::kI32Load16S:
229       movsxwl(dst.gp(), src_op);
230       break;
231     case LoadType::kI64Load16S:
232       movsxwq(dst.gp(), src_op);
233       break;
234     case LoadType::kI32Load:
235     case LoadType::kI64Load32U:
236       movl(dst.gp(), src_op);
237       break;
238     case LoadType::kI64Load32S:
239       movsxlq(dst.gp(), src_op);
240       break;
241     case LoadType::kI64Load:
242       movq(dst.gp(), src_op);
243       break;
244     case LoadType::kF32Load:
245       Movss(dst.fp(), src_op);
246       break;
247     case LoadType::kF64Load:
248       Movsd(dst.fp(), src_op);
249       break;
250     default:
251       UNREACHABLE();
252   }
253 }
254 
Store(Register dst_addr,Register offset_reg,uint32_t offset_imm,LiftoffRegister src,StoreType type,LiftoffRegList pinned,uint32_t * protected_store_pc,bool is_store_mem)255 void LiftoffAssembler::Store(Register dst_addr, Register offset_reg,
256                              uint32_t offset_imm, LiftoffRegister src,
257                              StoreType type, LiftoffRegList pinned,
258                              uint32_t* protected_store_pc, bool is_store_mem) {
259   if (emit_debug_code() && offset_reg != no_reg) {
260     AssertZeroExtended(offset_reg);
261   }
262   Operand dst_op = liftoff::GetMemOp(this, dst_addr, offset_reg, offset_imm);
263   if (protected_store_pc) *protected_store_pc = pc_offset();
264   switch (type.value()) {
265     case StoreType::kI32Store8:
266     case StoreType::kI64Store8:
267       movb(dst_op, src.gp());
268       break;
269     case StoreType::kI32Store16:
270     case StoreType::kI64Store16:
271       movw(dst_op, src.gp());
272       break;
273     case StoreType::kI32Store:
274     case StoreType::kI64Store32:
275       movl(dst_op, src.gp());
276       break;
277     case StoreType::kI64Store:
278       movq(dst_op, src.gp());
279       break;
280     case StoreType::kF32Store:
281       Movss(dst_op, src.fp());
282       break;
283     case StoreType::kF64Store:
284       Movsd(dst_op, src.fp());
285       break;
286     default:
287       UNREACHABLE();
288   }
289 }
290 
LoadCallerFrameSlot(LiftoffRegister dst,uint32_t caller_slot_idx,ValueType type)291 void LiftoffAssembler::LoadCallerFrameSlot(LiftoffRegister dst,
292                                            uint32_t caller_slot_idx,
293                                            ValueType type) {
294   Operand src(rbp, kPointerSize * (caller_slot_idx + 1));
295   liftoff::Load(this, dst, src, type);
296 }
297 
MoveStackValue(uint32_t dst_index,uint32_t src_index,ValueType type)298 void LiftoffAssembler::MoveStackValue(uint32_t dst_index, uint32_t src_index,
299                                       ValueType type) {
300   DCHECK_NE(dst_index, src_index);
301   if (cache_state_.has_unused_register(kGpReg)) {
302     Fill(LiftoffRegister{kScratchRegister}, src_index, type);
303     Spill(dst_index, LiftoffRegister{kScratchRegister}, type);
304   } else {
305     pushq(liftoff::GetStackSlot(src_index));
306     popq(liftoff::GetStackSlot(dst_index));
307   }
308 }
309 
Move(Register dst,Register src,ValueType type)310 void LiftoffAssembler::Move(Register dst, Register src, ValueType type) {
311   DCHECK_NE(dst, src);
312   if (type == kWasmI32) {
313     movl(dst, src);
314   } else {
315     DCHECK_EQ(kWasmI64, type);
316     movq(dst, src);
317   }
318 }
319 
Move(DoubleRegister dst,DoubleRegister src,ValueType type)320 void LiftoffAssembler::Move(DoubleRegister dst, DoubleRegister src,
321                             ValueType type) {
322   DCHECK_NE(dst, src);
323   if (type == kWasmF32) {
324     Movss(dst, src);
325   } else {
326     DCHECK_EQ(kWasmF64, type);
327     Movsd(dst, src);
328   }
329 }
330 
Spill(uint32_t index,LiftoffRegister reg,ValueType type)331 void LiftoffAssembler::Spill(uint32_t index, LiftoffRegister reg,
332                              ValueType type) {
333   RecordUsedSpillSlot(index);
334   Operand dst = liftoff::GetStackSlot(index);
335   switch (type) {
336     case kWasmI32:
337       movl(dst, reg.gp());
338       break;
339     case kWasmI64:
340       movq(dst, reg.gp());
341       break;
342     case kWasmF32:
343       Movss(dst, reg.fp());
344       break;
345     case kWasmF64:
346       Movsd(dst, reg.fp());
347       break;
348     default:
349       UNREACHABLE();
350   }
351 }
352 
Spill(uint32_t index,WasmValue value)353 void LiftoffAssembler::Spill(uint32_t index, WasmValue value) {
354   RecordUsedSpillSlot(index);
355   Operand dst = liftoff::GetStackSlot(index);
356   switch (value.type()) {
357     case kWasmI32:
358       movl(dst, Immediate(value.to_i32()));
359       break;
360     case kWasmI64: {
361       if (is_int32(value.to_i64())) {
362         // Sign extend low word.
363         movq(dst, Immediate(static_cast<int32_t>(value.to_i64())));
364       } else if (is_uint32(value.to_i64())) {
365         // Zero extend low word.
366         movl(kScratchRegister, Immediate(static_cast<int32_t>(value.to_i64())));
367         movq(dst, kScratchRegister);
368       } else {
369         movq(kScratchRegister, value.to_i64());
370         movq(dst, kScratchRegister);
371       }
372       break;
373     }
374     default:
375       // We do not track f32 and f64 constants, hence they are unreachable.
376       UNREACHABLE();
377   }
378 }
379 
Fill(LiftoffRegister reg,uint32_t index,ValueType type)380 void LiftoffAssembler::Fill(LiftoffRegister reg, uint32_t index,
381                             ValueType type) {
382   Operand src = liftoff::GetStackSlot(index);
383   switch (type) {
384     case kWasmI32:
385       movl(reg.gp(), src);
386       break;
387     case kWasmI64:
388       movq(reg.gp(), src);
389       break;
390     case kWasmF32:
391       Movss(reg.fp(), src);
392       break;
393     case kWasmF64:
394       Movsd(reg.fp(), src);
395       break;
396     default:
397       UNREACHABLE();
398   }
399 }
400 
FillI64Half(Register,uint32_t half_index)401 void LiftoffAssembler::FillI64Half(Register, uint32_t half_index) {
402   UNREACHABLE();
403 }
404 
emit_i32_add(Register dst,Register lhs,Register rhs)405 void LiftoffAssembler::emit_i32_add(Register dst, Register lhs, Register rhs) {
406   if (lhs != dst) {
407     leal(dst, Operand(lhs, rhs, times_1, 0));
408   } else {
409     addl(dst, rhs);
410   }
411 }
412 
emit_i32_sub(Register dst,Register lhs,Register rhs)413 void LiftoffAssembler::emit_i32_sub(Register dst, Register lhs, Register rhs) {
414   if (dst == rhs) {
415     negl(dst);
416     addl(dst, lhs);
417   } else {
418     if (dst != lhs) movl(dst, lhs);
419     subl(dst, rhs);
420   }
421 }
422 
423 namespace liftoff {
424 template <void (Assembler::*op)(Register, Register),
425           void (Assembler::*mov)(Register, Register)>
EmitCommutativeBinOp(LiftoffAssembler * assm,Register dst,Register lhs,Register rhs)426 void EmitCommutativeBinOp(LiftoffAssembler* assm, Register dst, Register lhs,
427                           Register rhs) {
428   if (dst == rhs) {
429     (assm->*op)(dst, lhs);
430   } else {
431     if (dst != lhs) (assm->*mov)(dst, lhs);
432     (assm->*op)(dst, rhs);
433   }
434 }
435 }  // namespace liftoff
436 
emit_i32_mul(Register dst,Register lhs,Register rhs)437 void LiftoffAssembler::emit_i32_mul(Register dst, Register lhs, Register rhs) {
438   liftoff::EmitCommutativeBinOp<&Assembler::imull, &Assembler::movl>(this, dst,
439                                                                      lhs, rhs);
440 }
441 
442 namespace liftoff {
443 enum class DivOrRem : uint8_t { kDiv, kRem };
444 template <typename type, DivOrRem div_or_rem>
EmitIntDivOrRem(LiftoffAssembler * assm,Register dst,Register lhs,Register rhs,Label * trap_div_by_zero,Label * trap_div_unrepresentable)445 void EmitIntDivOrRem(LiftoffAssembler* assm, Register dst, Register lhs,
446                      Register rhs, Label* trap_div_by_zero,
447                      Label* trap_div_unrepresentable) {
448   constexpr bool needs_unrepresentable_check =
449       std::is_signed<type>::value && div_or_rem == DivOrRem::kDiv;
450   constexpr bool special_case_minus_1 =
451       std::is_signed<type>::value && div_or_rem == DivOrRem::kRem;
452   DCHECK_EQ(needs_unrepresentable_check, trap_div_unrepresentable != nullptr);
453 
454 #define iop(name, ...)            \
455   do {                            \
456     if (sizeof(type) == 4) {      \
457       assm->name##l(__VA_ARGS__); \
458     } else {                      \
459       assm->name##q(__VA_ARGS__); \
460     }                             \
461   } while (false)
462 
463   // For division, the lhs is always taken from {edx:eax}. Thus, make sure that
464   // these registers are unused. If {rhs} is stored in one of them, move it to
465   // another temporary register.
466   // Do all this before any branch, such that the code is executed
467   // unconditionally, as the cache state will also be modified unconditionally.
468   liftoff::SpillRegisters(assm, rdx, rax);
469   if (rhs == rax || rhs == rdx) {
470     iop(mov, kScratchRegister, rhs);
471     rhs = kScratchRegister;
472   }
473 
474   // Check for division by zero.
475   iop(test, rhs, rhs);
476   assm->j(zero, trap_div_by_zero);
477 
478   Label done;
479   if (needs_unrepresentable_check) {
480     // Check for {kMinInt / -1}. This is unrepresentable.
481     Label do_div;
482     iop(cmp, rhs, Immediate(-1));
483     assm->j(not_equal, &do_div);
484     // {lhs} is min int if {lhs - 1} overflows.
485     iop(cmp, lhs, Immediate(1));
486     assm->j(overflow, trap_div_unrepresentable);
487     assm->bind(&do_div);
488   } else if (special_case_minus_1) {
489     // {lhs % -1} is always 0 (needs to be special cased because {kMinInt / -1}
490     // cannot be computed).
491     Label do_rem;
492     iop(cmp, rhs, Immediate(-1));
493     assm->j(not_equal, &do_rem);
494     // clang-format off
495     // (conflicts with presubmit checks because it is confused about "xor")
496     iop(xor, dst, dst);
497     // clang-format on
498     assm->jmp(&done);
499     assm->bind(&do_rem);
500   }
501 
502   // Now move {lhs} into {eax}, then zero-extend or sign-extend into {edx}, then
503   // do the division.
504   if (lhs != rax) iop(mov, rax, lhs);
505   if (std::is_same<int32_t, type>::value) {  // i32
506     assm->cdq();
507     assm->idivl(rhs);
508   } else if (std::is_same<uint32_t, type>::value) {  // u32
509     assm->xorl(rdx, rdx);
510     assm->divl(rhs);
511   } else if (std::is_same<int64_t, type>::value) {  // i64
512     assm->cqo();
513     assm->idivq(rhs);
514   } else {  // u64
515     assm->xorq(rdx, rdx);
516     assm->divq(rhs);
517   }
518 
519   // Move back the result (in {eax} or {edx}) into the {dst} register.
520   constexpr Register kResultReg = div_or_rem == DivOrRem::kDiv ? rax : rdx;
521   if (dst != kResultReg) {
522     iop(mov, dst, kResultReg);
523   }
524   if (special_case_minus_1) assm->bind(&done);
525 }
526 }  // namespace liftoff
527 
emit_i32_divs(Register dst,Register lhs,Register rhs,Label * trap_div_by_zero,Label * trap_div_unrepresentable)528 void LiftoffAssembler::emit_i32_divs(Register dst, Register lhs, Register rhs,
529                                      Label* trap_div_by_zero,
530                                      Label* trap_div_unrepresentable) {
531   liftoff::EmitIntDivOrRem<int32_t, liftoff::DivOrRem::kDiv>(
532       this, dst, lhs, rhs, trap_div_by_zero, trap_div_unrepresentable);
533 }
534 
emit_i32_divu(Register dst,Register lhs,Register rhs,Label * trap_div_by_zero)535 void LiftoffAssembler::emit_i32_divu(Register dst, Register lhs, Register rhs,
536                                      Label* trap_div_by_zero) {
537   liftoff::EmitIntDivOrRem<uint32_t, liftoff::DivOrRem::kDiv>(
538       this, dst, lhs, rhs, trap_div_by_zero, nullptr);
539 }
540 
emit_i32_rems(Register dst,Register lhs,Register rhs,Label * trap_div_by_zero)541 void LiftoffAssembler::emit_i32_rems(Register dst, Register lhs, Register rhs,
542                                      Label* trap_div_by_zero) {
543   liftoff::EmitIntDivOrRem<int32_t, liftoff::DivOrRem::kRem>(
544       this, dst, lhs, rhs, trap_div_by_zero, nullptr);
545 }
546 
emit_i32_remu(Register dst,Register lhs,Register rhs,Label * trap_div_by_zero)547 void LiftoffAssembler::emit_i32_remu(Register dst, Register lhs, Register rhs,
548                                      Label* trap_div_by_zero) {
549   liftoff::EmitIntDivOrRem<uint32_t, liftoff::DivOrRem::kRem>(
550       this, dst, lhs, rhs, trap_div_by_zero, nullptr);
551 }
552 
emit_i32_and(Register dst,Register lhs,Register rhs)553 void LiftoffAssembler::emit_i32_and(Register dst, Register lhs, Register rhs) {
554   liftoff::EmitCommutativeBinOp<&Assembler::andl, &Assembler::movl>(this, dst,
555                                                                     lhs, rhs);
556 }
557 
emit_i32_or(Register dst,Register lhs,Register rhs)558 void LiftoffAssembler::emit_i32_or(Register dst, Register lhs, Register rhs) {
559   liftoff::EmitCommutativeBinOp<&Assembler::orl, &Assembler::movl>(this, dst,
560                                                                    lhs, rhs);
561 }
562 
emit_i32_xor(Register dst,Register lhs,Register rhs)563 void LiftoffAssembler::emit_i32_xor(Register dst, Register lhs, Register rhs) {
564   liftoff::EmitCommutativeBinOp<&Assembler::xorl, &Assembler::movl>(this, dst,
565                                                                     lhs, rhs);
566 }
567 
568 namespace liftoff {
569 template <ValueType type>
EmitShiftOperation(LiftoffAssembler * assm,Register dst,Register src,Register amount,void (Assembler::* emit_shift)(Register),LiftoffRegList pinned)570 inline void EmitShiftOperation(LiftoffAssembler* assm, Register dst,
571                                Register src, Register amount,
572                                void (Assembler::*emit_shift)(Register),
573                                LiftoffRegList pinned) {
574   // If dst is rcx, compute into the scratch register first, then move to rcx.
575   if (dst == rcx) {
576     assm->Move(kScratchRegister, src, type);
577     if (amount != rcx) assm->Move(rcx, amount, type);
578     (assm->*emit_shift)(kScratchRegister);
579     assm->Move(rcx, kScratchRegister, type);
580     return;
581   }
582 
583   // Move amount into rcx. If rcx is in use, move its content into the scratch
584   // register. If src is rcx, src is now the scratch register.
585   bool use_scratch = false;
586   if (amount != rcx) {
587     use_scratch = src == rcx ||
588                   assm->cache_state()->is_used(LiftoffRegister(rcx)) ||
589                   pinned.has(LiftoffRegister(rcx));
590     if (use_scratch) assm->movq(kScratchRegister, rcx);
591     if (src == rcx) src = kScratchRegister;
592     assm->Move(rcx, amount, type);
593   }
594 
595   // Do the actual shift.
596   if (dst != src) assm->Move(dst, src, type);
597   (assm->*emit_shift)(dst);
598 
599   // Restore rcx if needed.
600   if (use_scratch) assm->movq(rcx, kScratchRegister);
601 }
602 }  // namespace liftoff
603 
emit_i32_shl(Register dst,Register src,Register amount,LiftoffRegList pinned)604 void LiftoffAssembler::emit_i32_shl(Register dst, Register src, Register amount,
605                                     LiftoffRegList pinned) {
606   liftoff::EmitShiftOperation<kWasmI32>(this, dst, src, amount,
607                                         &Assembler::shll_cl, pinned);
608 }
609 
emit_i32_sar(Register dst,Register src,Register amount,LiftoffRegList pinned)610 void LiftoffAssembler::emit_i32_sar(Register dst, Register src, Register amount,
611                                     LiftoffRegList pinned) {
612   liftoff::EmitShiftOperation<kWasmI32>(this, dst, src, amount,
613                                         &Assembler::sarl_cl, pinned);
614 }
615 
emit_i32_shr(Register dst,Register src,Register amount,LiftoffRegList pinned)616 void LiftoffAssembler::emit_i32_shr(Register dst, Register src, Register amount,
617                                     LiftoffRegList pinned) {
618   liftoff::EmitShiftOperation<kWasmI32>(this, dst, src, amount,
619                                         &Assembler::shrl_cl, pinned);
620 }
621 
emit_i32_clz(Register dst,Register src)622 bool LiftoffAssembler::emit_i32_clz(Register dst, Register src) {
623   Label nonzero_input;
624   Label continuation;
625   testl(src, src);
626   j(not_zero, &nonzero_input, Label::kNear);
627   movl(dst, Immediate(32));
628   jmp(&continuation, Label::kNear);
629 
630   bind(&nonzero_input);
631   // Get most significant bit set (MSBS).
632   bsrl(dst, src);
633   // CLZ = 31 - MSBS = MSBS ^ 31.
634   xorl(dst, Immediate(31));
635 
636   bind(&continuation);
637   return true;
638 }
639 
emit_i32_ctz(Register dst,Register src)640 bool LiftoffAssembler::emit_i32_ctz(Register dst, Register src) {
641   Label nonzero_input;
642   Label continuation;
643   testl(src, src);
644   j(not_zero, &nonzero_input, Label::kNear);
645   movl(dst, Immediate(32));
646   jmp(&continuation, Label::kNear);
647 
648   bind(&nonzero_input);
649   // Get least significant bit set, which equals number of trailing zeros.
650   bsfl(dst, src);
651 
652   bind(&continuation);
653   return true;
654 }
655 
emit_i32_popcnt(Register dst,Register src)656 bool LiftoffAssembler::emit_i32_popcnt(Register dst, Register src) {
657   if (!CpuFeatures::IsSupported(POPCNT)) return false;
658   CpuFeatureScope scope(this, POPCNT);
659   popcntl(dst, src);
660   return true;
661 }
662 
emit_i64_add(LiftoffRegister dst,LiftoffRegister lhs,LiftoffRegister rhs)663 void LiftoffAssembler::emit_i64_add(LiftoffRegister dst, LiftoffRegister lhs,
664                                     LiftoffRegister rhs) {
665   if (lhs.gp() != dst.gp()) {
666     leap(dst.gp(), Operand(lhs.gp(), rhs.gp(), times_1, 0));
667   } else {
668     addp(dst.gp(), rhs.gp());
669   }
670 }
671 
emit_i64_sub(LiftoffRegister dst,LiftoffRegister lhs,LiftoffRegister rhs)672 void LiftoffAssembler::emit_i64_sub(LiftoffRegister dst, LiftoffRegister lhs,
673                                     LiftoffRegister rhs) {
674   if (dst.gp() == rhs.gp()) {
675     negq(dst.gp());
676     addq(dst.gp(), lhs.gp());
677   } else {
678     if (dst.gp() != lhs.gp()) movq(dst.gp(), lhs.gp());
679     subq(dst.gp(), rhs.gp());
680   }
681 }
682 
emit_i64_mul(LiftoffRegister dst,LiftoffRegister lhs,LiftoffRegister rhs)683 void LiftoffAssembler::emit_i64_mul(LiftoffRegister dst, LiftoffRegister lhs,
684                                     LiftoffRegister rhs) {
685   liftoff::EmitCommutativeBinOp<&Assembler::imulq, &Assembler::movq>(
686       this, dst.gp(), lhs.gp(), rhs.gp());
687 }
688 
emit_i64_divs(LiftoffRegister dst,LiftoffRegister lhs,LiftoffRegister rhs,Label * trap_div_by_zero,Label * trap_div_unrepresentable)689 bool LiftoffAssembler::emit_i64_divs(LiftoffRegister dst, LiftoffRegister lhs,
690                                      LiftoffRegister rhs,
691                                      Label* trap_div_by_zero,
692                                      Label* trap_div_unrepresentable) {
693   liftoff::EmitIntDivOrRem<int64_t, liftoff::DivOrRem::kDiv>(
694       this, dst.gp(), lhs.gp(), rhs.gp(), trap_div_by_zero,
695       trap_div_unrepresentable);
696   return true;
697 }
698 
emit_i64_divu(LiftoffRegister dst,LiftoffRegister lhs,LiftoffRegister rhs,Label * trap_div_by_zero)699 bool LiftoffAssembler::emit_i64_divu(LiftoffRegister dst, LiftoffRegister lhs,
700                                      LiftoffRegister rhs,
701                                      Label* trap_div_by_zero) {
702   liftoff::EmitIntDivOrRem<uint64_t, liftoff::DivOrRem::kDiv>(
703       this, dst.gp(), lhs.gp(), rhs.gp(), trap_div_by_zero, nullptr);
704   return true;
705 }
706 
emit_i64_rems(LiftoffRegister dst,LiftoffRegister lhs,LiftoffRegister rhs,Label * trap_div_by_zero)707 bool LiftoffAssembler::emit_i64_rems(LiftoffRegister dst, LiftoffRegister lhs,
708                                      LiftoffRegister rhs,
709                                      Label* trap_div_by_zero) {
710   liftoff::EmitIntDivOrRem<int64_t, liftoff::DivOrRem::kRem>(
711       this, dst.gp(), lhs.gp(), rhs.gp(), trap_div_by_zero, nullptr);
712   return true;
713 }
714 
emit_i64_remu(LiftoffRegister dst,LiftoffRegister lhs,LiftoffRegister rhs,Label * trap_div_by_zero)715 bool LiftoffAssembler::emit_i64_remu(LiftoffRegister dst, LiftoffRegister lhs,
716                                      LiftoffRegister rhs,
717                                      Label* trap_div_by_zero) {
718   liftoff::EmitIntDivOrRem<uint64_t, liftoff::DivOrRem::kRem>(
719       this, dst.gp(), lhs.gp(), rhs.gp(), trap_div_by_zero, nullptr);
720   return true;
721 }
722 
emit_i64_and(LiftoffRegister dst,LiftoffRegister lhs,LiftoffRegister rhs)723 void LiftoffAssembler::emit_i64_and(LiftoffRegister dst, LiftoffRegister lhs,
724                                     LiftoffRegister rhs) {
725   liftoff::EmitCommutativeBinOp<&Assembler::andq, &Assembler::movq>(
726       this, dst.gp(), lhs.gp(), rhs.gp());
727 }
728 
emit_i64_or(LiftoffRegister dst,LiftoffRegister lhs,LiftoffRegister rhs)729 void LiftoffAssembler::emit_i64_or(LiftoffRegister dst, LiftoffRegister lhs,
730                                    LiftoffRegister rhs) {
731   liftoff::EmitCommutativeBinOp<&Assembler::orq, &Assembler::movq>(
732       this, dst.gp(), lhs.gp(), rhs.gp());
733 }
734 
emit_i64_xor(LiftoffRegister dst,LiftoffRegister lhs,LiftoffRegister rhs)735 void LiftoffAssembler::emit_i64_xor(LiftoffRegister dst, LiftoffRegister lhs,
736                                     LiftoffRegister rhs) {
737   liftoff::EmitCommutativeBinOp<&Assembler::xorq, &Assembler::movq>(
738       this, dst.gp(), lhs.gp(), rhs.gp());
739 }
740 
emit_i64_shl(LiftoffRegister dst,LiftoffRegister src,Register amount,LiftoffRegList pinned)741 void LiftoffAssembler::emit_i64_shl(LiftoffRegister dst, LiftoffRegister src,
742                                     Register amount, LiftoffRegList pinned) {
743   liftoff::EmitShiftOperation<kWasmI64>(this, dst.gp(), src.gp(), amount,
744                                         &Assembler::shlq_cl, pinned);
745 }
746 
emit_i64_sar(LiftoffRegister dst,LiftoffRegister src,Register amount,LiftoffRegList pinned)747 void LiftoffAssembler::emit_i64_sar(LiftoffRegister dst, LiftoffRegister src,
748                                     Register amount, LiftoffRegList pinned) {
749   liftoff::EmitShiftOperation<kWasmI64>(this, dst.gp(), src.gp(), amount,
750                                         &Assembler::sarq_cl, pinned);
751 }
752 
emit_i64_shr(LiftoffRegister dst,LiftoffRegister src,Register amount,LiftoffRegList pinned)753 void LiftoffAssembler::emit_i64_shr(LiftoffRegister dst, LiftoffRegister src,
754                                     Register amount, LiftoffRegList pinned) {
755   liftoff::EmitShiftOperation<kWasmI64>(this, dst.gp(), src.gp(), amount,
756                                         &Assembler::shrq_cl, pinned);
757 }
758 
emit_i32_to_intptr(Register dst,Register src)759 void LiftoffAssembler::emit_i32_to_intptr(Register dst, Register src) {
760   movsxlq(dst, src);
761 }
762 
emit_f32_add(DoubleRegister dst,DoubleRegister lhs,DoubleRegister rhs)763 void LiftoffAssembler::emit_f32_add(DoubleRegister dst, DoubleRegister lhs,
764                                     DoubleRegister rhs) {
765   if (CpuFeatures::IsSupported(AVX)) {
766     CpuFeatureScope scope(this, AVX);
767     vaddss(dst, lhs, rhs);
768   } else if (dst == rhs) {
769     addss(dst, lhs);
770   } else {
771     if (dst != lhs) movss(dst, lhs);
772     addss(dst, rhs);
773   }
774 }
775 
emit_f32_sub(DoubleRegister dst,DoubleRegister lhs,DoubleRegister rhs)776 void LiftoffAssembler::emit_f32_sub(DoubleRegister dst, DoubleRegister lhs,
777                                     DoubleRegister rhs) {
778   if (CpuFeatures::IsSupported(AVX)) {
779     CpuFeatureScope scope(this, AVX);
780     vsubss(dst, lhs, rhs);
781   } else if (dst == rhs) {
782     movss(kScratchDoubleReg, rhs);
783     movss(dst, lhs);
784     subss(dst, kScratchDoubleReg);
785   } else {
786     if (dst != lhs) movss(dst, lhs);
787     subss(dst, rhs);
788   }
789 }
790 
emit_f32_mul(DoubleRegister dst,DoubleRegister lhs,DoubleRegister rhs)791 void LiftoffAssembler::emit_f32_mul(DoubleRegister dst, DoubleRegister lhs,
792                                     DoubleRegister rhs) {
793   if (CpuFeatures::IsSupported(AVX)) {
794     CpuFeatureScope scope(this, AVX);
795     vmulss(dst, lhs, rhs);
796   } else if (dst == rhs) {
797     mulss(dst, lhs);
798   } else {
799     if (dst != lhs) movss(dst, lhs);
800     mulss(dst, rhs);
801   }
802 }
803 
emit_f32_div(DoubleRegister dst,DoubleRegister lhs,DoubleRegister rhs)804 void LiftoffAssembler::emit_f32_div(DoubleRegister dst, DoubleRegister lhs,
805                                     DoubleRegister rhs) {
806   if (CpuFeatures::IsSupported(AVX)) {
807     CpuFeatureScope scope(this, AVX);
808     vdivss(dst, lhs, rhs);
809   } else if (dst == rhs) {
810     movss(kScratchDoubleReg, rhs);
811     movss(dst, lhs);
812     divss(dst, kScratchDoubleReg);
813   } else {
814     if (dst != lhs) movss(dst, lhs);
815     divss(dst, rhs);
816   }
817 }
818 
819 namespace liftoff {
820 enum class MinOrMax : uint8_t { kMin, kMax };
821 template <typename type>
EmitFloatMinOrMax(LiftoffAssembler * assm,DoubleRegister dst,DoubleRegister lhs,DoubleRegister rhs,MinOrMax min_or_max)822 inline void EmitFloatMinOrMax(LiftoffAssembler* assm, DoubleRegister dst,
823                               DoubleRegister lhs, DoubleRegister rhs,
824                               MinOrMax min_or_max) {
825   Label is_nan;
826   Label lhs_below_rhs;
827   Label lhs_above_rhs;
828   Label done;
829 
830 #define dop(name, ...)            \
831   do {                            \
832     if (sizeof(type) == 4) {      \
833       assm->name##s(__VA_ARGS__); \
834     } else {                      \
835       assm->name##d(__VA_ARGS__); \
836     }                             \
837   } while (false)
838 
839   // Check the easy cases first: nan (e.g. unordered), smaller and greater.
840   // NaN has to be checked first, because PF=1 implies CF=1.
841   dop(Ucomis, lhs, rhs);
842   assm->j(parity_even, &is_nan, Label::kNear);   // PF=1
843   assm->j(below, &lhs_below_rhs, Label::kNear);  // CF=1
844   assm->j(above, &lhs_above_rhs, Label::kNear);  // CF=0 && ZF=0
845 
846   // If we get here, then either
847   // a) {lhs == rhs},
848   // b) {lhs == -0.0} and {rhs == 0.0}, or
849   // c) {lhs == 0.0} and {rhs == -0.0}.
850   // For a), it does not matter whether we return {lhs} or {rhs}. Check the sign
851   // bit of {rhs} to differentiate b) and c).
852   dop(Movmskp, kScratchRegister, rhs);
853   assm->testl(kScratchRegister, Immediate(1));
854   assm->j(zero, &lhs_below_rhs, Label::kNear);
855   assm->jmp(&lhs_above_rhs, Label::kNear);
856 
857   assm->bind(&is_nan);
858   // Create a NaN output.
859   dop(Xorp, dst, dst);
860   dop(Divs, dst, dst);
861   assm->jmp(&done, Label::kNear);
862 
863   assm->bind(&lhs_below_rhs);
864   DoubleRegister lhs_below_rhs_src = min_or_max == MinOrMax::kMin ? lhs : rhs;
865   if (dst != lhs_below_rhs_src) dop(Movs, dst, lhs_below_rhs_src);
866   assm->jmp(&done, Label::kNear);
867 
868   assm->bind(&lhs_above_rhs);
869   DoubleRegister lhs_above_rhs_src = min_or_max == MinOrMax::kMin ? rhs : lhs;
870   if (dst != lhs_above_rhs_src) dop(Movs, dst, lhs_above_rhs_src);
871 
872   assm->bind(&done);
873 }
874 }  // namespace liftoff
875 
emit_f32_min(DoubleRegister dst,DoubleRegister lhs,DoubleRegister rhs)876 void LiftoffAssembler::emit_f32_min(DoubleRegister dst, DoubleRegister lhs,
877                                     DoubleRegister rhs) {
878   liftoff::EmitFloatMinOrMax<float>(this, dst, lhs, rhs,
879                                     liftoff::MinOrMax::kMin);
880 }
881 
emit_f32_max(DoubleRegister dst,DoubleRegister lhs,DoubleRegister rhs)882 void LiftoffAssembler::emit_f32_max(DoubleRegister dst, DoubleRegister lhs,
883                                     DoubleRegister rhs) {
884   liftoff::EmitFloatMinOrMax<float>(this, dst, lhs, rhs,
885                                     liftoff::MinOrMax::kMax);
886 }
887 
emit_f32_abs(DoubleRegister dst,DoubleRegister src)888 void LiftoffAssembler::emit_f32_abs(DoubleRegister dst, DoubleRegister src) {
889   static constexpr uint32_t kSignBit = uint32_t{1} << 31;
890   if (dst == src) {
891     TurboAssembler::Move(kScratchDoubleReg, kSignBit - 1);
892     Andps(dst, kScratchDoubleReg);
893   } else {
894     TurboAssembler::Move(dst, kSignBit - 1);
895     Andps(dst, src);
896   }
897 }
898 
emit_f32_neg(DoubleRegister dst,DoubleRegister src)899 void LiftoffAssembler::emit_f32_neg(DoubleRegister dst, DoubleRegister src) {
900   static constexpr uint32_t kSignBit = uint32_t{1} << 31;
901   if (dst == src) {
902     TurboAssembler::Move(kScratchDoubleReg, kSignBit);
903     Xorps(dst, kScratchDoubleReg);
904   } else {
905     TurboAssembler::Move(dst, kSignBit);
906     Xorps(dst, src);
907   }
908 }
909 
emit_f32_ceil(DoubleRegister dst,DoubleRegister src)910 void LiftoffAssembler::emit_f32_ceil(DoubleRegister dst, DoubleRegister src) {
911   REQUIRE_CPU_FEATURE(SSE4_1);
912   Roundss(dst, src, kRoundUp);
913 }
914 
emit_f32_floor(DoubleRegister dst,DoubleRegister src)915 void LiftoffAssembler::emit_f32_floor(DoubleRegister dst, DoubleRegister src) {
916   REQUIRE_CPU_FEATURE(SSE4_1);
917   Roundss(dst, src, kRoundDown);
918 }
919 
emit_f32_trunc(DoubleRegister dst,DoubleRegister src)920 void LiftoffAssembler::emit_f32_trunc(DoubleRegister dst, DoubleRegister src) {
921   REQUIRE_CPU_FEATURE(SSE4_1);
922   Roundss(dst, src, kRoundToZero);
923 }
924 
emit_f32_nearest_int(DoubleRegister dst,DoubleRegister src)925 void LiftoffAssembler::emit_f32_nearest_int(DoubleRegister dst,
926                                             DoubleRegister src) {
927   REQUIRE_CPU_FEATURE(SSE4_1);
928   Roundss(dst, src, kRoundToNearest);
929 }
930 
emit_f32_sqrt(DoubleRegister dst,DoubleRegister src)931 void LiftoffAssembler::emit_f32_sqrt(DoubleRegister dst, DoubleRegister src) {
932   Sqrtss(dst, src);
933 }
934 
emit_f64_add(DoubleRegister dst,DoubleRegister lhs,DoubleRegister rhs)935 void LiftoffAssembler::emit_f64_add(DoubleRegister dst, DoubleRegister lhs,
936                                     DoubleRegister rhs) {
937   if (CpuFeatures::IsSupported(AVX)) {
938     CpuFeatureScope scope(this, AVX);
939     vaddsd(dst, lhs, rhs);
940   } else if (dst == rhs) {
941     addsd(dst, lhs);
942   } else {
943     if (dst != lhs) movsd(dst, lhs);
944     addsd(dst, rhs);
945   }
946 }
947 
emit_f64_sub(DoubleRegister dst,DoubleRegister lhs,DoubleRegister rhs)948 void LiftoffAssembler::emit_f64_sub(DoubleRegister dst, DoubleRegister lhs,
949                                     DoubleRegister rhs) {
950   if (CpuFeatures::IsSupported(AVX)) {
951     CpuFeatureScope scope(this, AVX);
952     vsubsd(dst, lhs, rhs);
953   } else if (dst == rhs) {
954     movsd(kScratchDoubleReg, rhs);
955     movsd(dst, lhs);
956     subsd(dst, kScratchDoubleReg);
957   } else {
958     if (dst != lhs) movsd(dst, lhs);
959     subsd(dst, rhs);
960   }
961 }
962 
emit_f64_mul(DoubleRegister dst,DoubleRegister lhs,DoubleRegister rhs)963 void LiftoffAssembler::emit_f64_mul(DoubleRegister dst, DoubleRegister lhs,
964                                     DoubleRegister rhs) {
965   if (CpuFeatures::IsSupported(AVX)) {
966     CpuFeatureScope scope(this, AVX);
967     vmulsd(dst, lhs, rhs);
968   } else if (dst == rhs) {
969     mulsd(dst, lhs);
970   } else {
971     if (dst != lhs) movsd(dst, lhs);
972     mulsd(dst, rhs);
973   }
974 }
975 
emit_f64_div(DoubleRegister dst,DoubleRegister lhs,DoubleRegister rhs)976 void LiftoffAssembler::emit_f64_div(DoubleRegister dst, DoubleRegister lhs,
977                                     DoubleRegister rhs) {
978   if (CpuFeatures::IsSupported(AVX)) {
979     CpuFeatureScope scope(this, AVX);
980     vdivsd(dst, lhs, rhs);
981   } else if (dst == rhs) {
982     movsd(kScratchDoubleReg, rhs);
983     movsd(dst, lhs);
984     divsd(dst, kScratchDoubleReg);
985   } else {
986     if (dst != lhs) movsd(dst, lhs);
987     divsd(dst, rhs);
988   }
989 }
990 
emit_f64_min(DoubleRegister dst,DoubleRegister lhs,DoubleRegister rhs)991 void LiftoffAssembler::emit_f64_min(DoubleRegister dst, DoubleRegister lhs,
992                                     DoubleRegister rhs) {
993   liftoff::EmitFloatMinOrMax<double>(this, dst, lhs, rhs,
994                                      liftoff::MinOrMax::kMin);
995 }
996 
emit_f64_max(DoubleRegister dst,DoubleRegister lhs,DoubleRegister rhs)997 void LiftoffAssembler::emit_f64_max(DoubleRegister dst, DoubleRegister lhs,
998                                     DoubleRegister rhs) {
999   liftoff::EmitFloatMinOrMax<double>(this, dst, lhs, rhs,
1000                                      liftoff::MinOrMax::kMax);
1001 }
1002 
emit_f64_abs(DoubleRegister dst,DoubleRegister src)1003 void LiftoffAssembler::emit_f64_abs(DoubleRegister dst, DoubleRegister src) {
1004   static constexpr uint64_t kSignBit = uint64_t{1} << 63;
1005   if (dst == src) {
1006     TurboAssembler::Move(kScratchDoubleReg, kSignBit - 1);
1007     Andpd(dst, kScratchDoubleReg);
1008   } else {
1009     TurboAssembler::Move(dst, kSignBit - 1);
1010     Andpd(dst, src);
1011   }
1012 }
1013 
emit_f64_neg(DoubleRegister dst,DoubleRegister src)1014 void LiftoffAssembler::emit_f64_neg(DoubleRegister dst, DoubleRegister src) {
1015   static constexpr uint64_t kSignBit = uint64_t{1} << 63;
1016   if (dst == src) {
1017     TurboAssembler::Move(kScratchDoubleReg, kSignBit);
1018     Xorpd(dst, kScratchDoubleReg);
1019   } else {
1020     TurboAssembler::Move(dst, kSignBit);
1021     Xorpd(dst, src);
1022   }
1023 }
1024 
emit_f64_ceil(DoubleRegister dst,DoubleRegister src)1025 bool LiftoffAssembler::emit_f64_ceil(DoubleRegister dst, DoubleRegister src) {
1026   REQUIRE_CPU_FEATURE(SSE4_1, true);
1027   Roundsd(dst, src, kRoundUp);
1028   return true;
1029 }
1030 
emit_f64_floor(DoubleRegister dst,DoubleRegister src)1031 bool LiftoffAssembler::emit_f64_floor(DoubleRegister dst, DoubleRegister src) {
1032   REQUIRE_CPU_FEATURE(SSE4_1, true);
1033   Roundsd(dst, src, kRoundDown);
1034   return true;
1035 }
1036 
emit_f64_trunc(DoubleRegister dst,DoubleRegister src)1037 bool LiftoffAssembler::emit_f64_trunc(DoubleRegister dst, DoubleRegister src) {
1038   REQUIRE_CPU_FEATURE(SSE4_1, true);
1039   Roundsd(dst, src, kRoundToZero);
1040   return true;
1041 }
1042 
emit_f64_nearest_int(DoubleRegister dst,DoubleRegister src)1043 bool LiftoffAssembler::emit_f64_nearest_int(DoubleRegister dst,
1044                                             DoubleRegister src) {
1045   REQUIRE_CPU_FEATURE(SSE4_1, true);
1046   Roundsd(dst, src, kRoundToNearest);
1047   return true;
1048 }
1049 
emit_f64_sqrt(DoubleRegister dst,DoubleRegister src)1050 void LiftoffAssembler::emit_f64_sqrt(DoubleRegister dst, DoubleRegister src) {
1051   Sqrtsd(dst, src);
1052 }
1053 
1054 namespace liftoff {
1055 // Used for float to int conversions. If the value in {converted_back} equals
1056 // {src} afterwards, the conversion succeeded.
1057 template <typename dst_type, typename src_type>
ConvertFloatToIntAndBack(LiftoffAssembler * assm,Register dst,DoubleRegister src,DoubleRegister converted_back)1058 inline void ConvertFloatToIntAndBack(LiftoffAssembler* assm, Register dst,
1059                                      DoubleRegister src,
1060                                      DoubleRegister converted_back) {
1061   if (std::is_same<double, src_type>::value) {  // f64
1062     if (std::is_same<int32_t, dst_type>::value) {  // f64 -> i32
1063       assm->Cvttsd2si(dst, src);
1064       assm->Cvtlsi2sd(converted_back, dst);
1065     } else if (std::is_same<uint32_t, dst_type>::value) {  // f64 -> u32
1066       assm->Cvttsd2siq(dst, src);
1067       assm->movl(dst, dst);
1068       assm->Cvtqsi2sd(converted_back, dst);
1069     } else if (std::is_same<int64_t, dst_type>::value) {  // f64 -> i64
1070       assm->Cvttsd2siq(dst, src);
1071       assm->Cvtqsi2sd(converted_back, dst);
1072     } else {
1073       UNREACHABLE();
1074     }
1075   } else {                                  // f32
1076     if (std::is_same<int32_t, dst_type>::value) {  // f32 -> i32
1077       assm->Cvttss2si(dst, src);
1078       assm->Cvtlsi2ss(converted_back, dst);
1079     } else if (std::is_same<uint32_t, dst_type>::value) {  // f32 -> u32
1080       assm->Cvttss2siq(dst, src);
1081       assm->movl(dst, dst);
1082       assm->Cvtqsi2ss(converted_back, dst);
1083     } else if (std::is_same<int64_t, dst_type>::value) {  // f32 -> i64
1084       assm->Cvttss2siq(dst, src);
1085       assm->Cvtqsi2ss(converted_back, dst);
1086     } else {
1087       UNREACHABLE();
1088     }
1089   }
1090 }
1091 
1092 template <typename dst_type, typename src_type>
EmitTruncateFloatToInt(LiftoffAssembler * assm,Register dst,DoubleRegister src,Label * trap)1093 inline bool EmitTruncateFloatToInt(LiftoffAssembler* assm, Register dst,
1094                                    DoubleRegister src, Label* trap) {
1095   if (!CpuFeatures::IsSupported(SSE4_1)) {
1096     assm->bailout("no SSE4.1");
1097     return true;
1098   }
1099   CpuFeatureScope feature(assm, SSE4_1);
1100 
1101   DoubleRegister rounded = kScratchDoubleReg;
1102   DoubleRegister converted_back = kScratchDoubleReg2;
1103 
1104   if (std::is_same<double, src_type>::value) {  // f64
1105     assm->Roundsd(rounded, src, kRoundToZero);
1106   } else {  // f32
1107     assm->Roundss(rounded, src, kRoundToZero);
1108   }
1109   ConvertFloatToIntAndBack<dst_type, src_type>(assm, dst, rounded,
1110                                                converted_back);
1111   if (std::is_same<double, src_type>::value) {  // f64
1112     assm->Ucomisd(converted_back, rounded);
1113   } else {  // f32
1114     assm->Ucomiss(converted_back, rounded);
1115   }
1116 
1117   // Jump to trap if PF is 0 (one of the operands was NaN) or they are not
1118   // equal.
1119   assm->j(parity_even, trap);
1120   assm->j(not_equal, trap);
1121   return true;
1122 }
1123 }  // namespace liftoff
1124 
emit_type_conversion(WasmOpcode opcode,LiftoffRegister dst,LiftoffRegister src,Label * trap)1125 bool LiftoffAssembler::emit_type_conversion(WasmOpcode opcode,
1126                                             LiftoffRegister dst,
1127                                             LiftoffRegister src, Label* trap) {
1128   switch (opcode) {
1129     case kExprI32ConvertI64:
1130       movl(dst.gp(), src.gp());
1131       return true;
1132     case kExprI32SConvertF32:
1133       return liftoff::EmitTruncateFloatToInt<int32_t, float>(this, dst.gp(),
1134                                                              src.fp(), trap);
1135     case kExprI32UConvertF32:
1136       return liftoff::EmitTruncateFloatToInt<uint32_t, float>(this, dst.gp(),
1137                                                               src.fp(), trap);
1138     case kExprI32SConvertF64:
1139       return liftoff::EmitTruncateFloatToInt<int32_t, double>(this, dst.gp(),
1140                                                               src.fp(), trap);
1141     case kExprI32UConvertF64:
1142       return liftoff::EmitTruncateFloatToInt<uint32_t, double>(this, dst.gp(),
1143                                                                src.fp(), trap);
1144     case kExprI32ReinterpretF32:
1145       Movd(dst.gp(), src.fp());
1146       return true;
1147     case kExprI64SConvertI32:
1148       movsxlq(dst.gp(), src.gp());
1149       return true;
1150     case kExprI64SConvertF32:
1151       return liftoff::EmitTruncateFloatToInt<int64_t, float>(this, dst.gp(),
1152                                                              src.fp(), trap);
1153     case kExprI64UConvertF32: {
1154       REQUIRE_CPU_FEATURE(SSE4_1, true);
1155       Cvttss2uiq(dst.gp(), src.fp(), trap);
1156       return true;
1157     }
1158     case kExprI64SConvertF64:
1159       return liftoff::EmitTruncateFloatToInt<int64_t, double>(this, dst.gp(),
1160                                                               src.fp(), trap);
1161     case kExprI64UConvertF64: {
1162       REQUIRE_CPU_FEATURE(SSE4_1, true);
1163       Cvttsd2uiq(dst.gp(), src.fp(), trap);
1164       return true;
1165     }
1166     case kExprI64UConvertI32:
1167       AssertZeroExtended(src.gp());
1168       if (dst.gp() != src.gp()) movl(dst.gp(), src.gp());
1169       return true;
1170     case kExprI64ReinterpretF64:
1171       Movq(dst.gp(), src.fp());
1172       return true;
1173     case kExprF32SConvertI32:
1174       Cvtlsi2ss(dst.fp(), src.gp());
1175       return true;
1176     case kExprF32UConvertI32:
1177       movl(kScratchRegister, src.gp());
1178       Cvtqsi2ss(dst.fp(), kScratchRegister);
1179       return true;
1180     case kExprF32SConvertI64:
1181       Cvtqsi2ss(dst.fp(), src.gp());
1182       return true;
1183     case kExprF32UConvertI64:
1184       Cvtqui2ss(dst.fp(), src.gp());
1185       return true;
1186     case kExprF32ConvertF64:
1187       Cvtsd2ss(dst.fp(), src.fp());
1188       return true;
1189     case kExprF32ReinterpretI32:
1190       Movd(dst.fp(), src.gp());
1191       return true;
1192     case kExprF64SConvertI32:
1193       Cvtlsi2sd(dst.fp(), src.gp());
1194       return true;
1195     case kExprF64UConvertI32:
1196       movl(kScratchRegister, src.gp());
1197       Cvtqsi2sd(dst.fp(), kScratchRegister);
1198       return true;
1199     case kExprF64SConvertI64:
1200       Cvtqsi2sd(dst.fp(), src.gp());
1201       return true;
1202     case kExprF64UConvertI64:
1203       Cvtqui2sd(dst.fp(), src.gp());
1204       return true;
1205     case kExprF64ConvertF32:
1206       Cvtss2sd(dst.fp(), src.fp());
1207       return true;
1208     case kExprF64ReinterpretI64:
1209       Movq(dst.fp(), src.gp());
1210       return true;
1211     default:
1212       UNREACHABLE();
1213   }
1214 }
1215 
emit_jump(Label * label)1216 void LiftoffAssembler::emit_jump(Label* label) { jmp(label); }
1217 
emit_jump(Register target)1218 void LiftoffAssembler::emit_jump(Register target) { jmp(target); }
1219 
emit_cond_jump(Condition cond,Label * label,ValueType type,Register lhs,Register rhs)1220 void LiftoffAssembler::emit_cond_jump(Condition cond, Label* label,
1221                                       ValueType type, Register lhs,
1222                                       Register rhs) {
1223   if (rhs != no_reg) {
1224     switch (type) {
1225       case kWasmI32:
1226         cmpl(lhs, rhs);
1227         break;
1228       case kWasmI64:
1229         cmpq(lhs, rhs);
1230         break;
1231       default:
1232         UNREACHABLE();
1233     }
1234   } else {
1235     DCHECK_EQ(type, kWasmI32);
1236     testl(lhs, lhs);
1237   }
1238 
1239   j(cond, label);
1240 }
1241 
emit_i32_eqz(Register dst,Register src)1242 void LiftoffAssembler::emit_i32_eqz(Register dst, Register src) {
1243   testl(src, src);
1244   setcc(equal, dst);
1245   movzxbl(dst, dst);
1246 }
1247 
emit_i32_set_cond(Condition cond,Register dst,Register lhs,Register rhs)1248 void LiftoffAssembler::emit_i32_set_cond(Condition cond, Register dst,
1249                                          Register lhs, Register rhs) {
1250   cmpl(lhs, rhs);
1251   setcc(cond, dst);
1252   movzxbl(dst, dst);
1253 }
1254 
emit_i64_eqz(Register dst,LiftoffRegister src)1255 void LiftoffAssembler::emit_i64_eqz(Register dst, LiftoffRegister src) {
1256   testq(src.gp(), src.gp());
1257   setcc(equal, dst);
1258   movzxbl(dst, dst);
1259 }
1260 
emit_i64_set_cond(Condition cond,Register dst,LiftoffRegister lhs,LiftoffRegister rhs)1261 void LiftoffAssembler::emit_i64_set_cond(Condition cond, Register dst,
1262                                          LiftoffRegister lhs,
1263                                          LiftoffRegister rhs) {
1264   cmpq(lhs.gp(), rhs.gp());
1265   setcc(cond, dst);
1266   movzxbl(dst, dst);
1267 }
1268 
1269 namespace liftoff {
1270 template <void (TurboAssembler::*cmp_op)(DoubleRegister, DoubleRegister)>
EmitFloatSetCond(LiftoffAssembler * assm,Condition cond,Register dst,DoubleRegister lhs,DoubleRegister rhs)1271 void EmitFloatSetCond(LiftoffAssembler* assm, Condition cond, Register dst,
1272                       DoubleRegister lhs, DoubleRegister rhs) {
1273   Label cont;
1274   Label not_nan;
1275 
1276   (assm->*cmp_op)(lhs, rhs);
1277   // If PF is one, one of the operands was NaN. This needs special handling.
1278   assm->j(parity_odd, &not_nan, Label::kNear);
1279   // Return 1 for f32.ne, 0 for all other cases.
1280   if (cond == not_equal) {
1281     assm->movl(dst, Immediate(1));
1282   } else {
1283     assm->xorl(dst, dst);
1284   }
1285   assm->jmp(&cont, Label::kNear);
1286   assm->bind(&not_nan);
1287 
1288   assm->setcc(cond, dst);
1289   assm->movzxbl(dst, dst);
1290   assm->bind(&cont);
1291 }
1292 }  // namespace liftoff
1293 
emit_f32_set_cond(Condition cond,Register dst,DoubleRegister lhs,DoubleRegister rhs)1294 void LiftoffAssembler::emit_f32_set_cond(Condition cond, Register dst,
1295                                          DoubleRegister lhs,
1296                                          DoubleRegister rhs) {
1297   liftoff::EmitFloatSetCond<&TurboAssembler::Ucomiss>(this, cond, dst, lhs,
1298                                                       rhs);
1299 }
1300 
emit_f64_set_cond(Condition cond,Register dst,DoubleRegister lhs,DoubleRegister rhs)1301 void LiftoffAssembler::emit_f64_set_cond(Condition cond, Register dst,
1302                                          DoubleRegister lhs,
1303                                          DoubleRegister rhs) {
1304   liftoff::EmitFloatSetCond<&TurboAssembler::Ucomisd>(this, cond, dst, lhs,
1305                                                       rhs);
1306 }
1307 
StackCheck(Label * ool_code,Register limit_address)1308 void LiftoffAssembler::StackCheck(Label* ool_code, Register limit_address) {
1309   cmpp(rsp, Operand(limit_address, 0));
1310   j(below_equal, ool_code);
1311 }
1312 
CallTrapCallbackForTesting()1313 void LiftoffAssembler::CallTrapCallbackForTesting() {
1314   PrepareCallCFunction(0);
1315   CallCFunction(ExternalReference::wasm_call_trap_callback_for_testing(), 0);
1316 }
1317 
AssertUnreachable(AbortReason reason)1318 void LiftoffAssembler::AssertUnreachable(AbortReason reason) {
1319   TurboAssembler::AssertUnreachable(reason);
1320 }
1321 
PushRegisters(LiftoffRegList regs)1322 void LiftoffAssembler::PushRegisters(LiftoffRegList regs) {
1323   LiftoffRegList gp_regs = regs & kGpCacheRegList;
1324   while (!gp_regs.is_empty()) {
1325     LiftoffRegister reg = gp_regs.GetFirstRegSet();
1326     pushq(reg.gp());
1327     gp_regs.clear(reg);
1328   }
1329   LiftoffRegList fp_regs = regs & kFpCacheRegList;
1330   unsigned num_fp_regs = fp_regs.GetNumRegsSet();
1331   if (num_fp_regs) {
1332     subp(rsp, Immediate(num_fp_regs * kStackSlotSize));
1333     unsigned offset = 0;
1334     while (!fp_regs.is_empty()) {
1335       LiftoffRegister reg = fp_regs.GetFirstRegSet();
1336       Movsd(Operand(rsp, offset), reg.fp());
1337       fp_regs.clear(reg);
1338       offset += sizeof(double);
1339     }
1340     DCHECK_EQ(offset, num_fp_regs * sizeof(double));
1341   }
1342 }
1343 
PopRegisters(LiftoffRegList regs)1344 void LiftoffAssembler::PopRegisters(LiftoffRegList regs) {
1345   LiftoffRegList fp_regs = regs & kFpCacheRegList;
1346   unsigned fp_offset = 0;
1347   while (!fp_regs.is_empty()) {
1348     LiftoffRegister reg = fp_regs.GetFirstRegSet();
1349     Movsd(reg.fp(), Operand(rsp, fp_offset));
1350     fp_regs.clear(reg);
1351     fp_offset += sizeof(double);
1352   }
1353   if (fp_offset) addp(rsp, Immediate(fp_offset));
1354   LiftoffRegList gp_regs = regs & kGpCacheRegList;
1355   while (!gp_regs.is_empty()) {
1356     LiftoffRegister reg = gp_regs.GetLastRegSet();
1357     popq(reg.gp());
1358     gp_regs.clear(reg);
1359   }
1360 }
1361 
DropStackSlotsAndRet(uint32_t num_stack_slots)1362 void LiftoffAssembler::DropStackSlotsAndRet(uint32_t num_stack_slots) {
1363   DCHECK_LT(num_stack_slots, (1 << 16) / kPointerSize);  // 16 bit immediate
1364   ret(static_cast<int>(num_stack_slots * kPointerSize));
1365 }
1366 
CallC(wasm::FunctionSig * sig,const LiftoffRegister * args,const LiftoffRegister * rets,ValueType out_argument_type,int stack_bytes,ExternalReference ext_ref)1367 void LiftoffAssembler::CallC(wasm::FunctionSig* sig,
1368                              const LiftoffRegister* args,
1369                              const LiftoffRegister* rets,
1370                              ValueType out_argument_type, int stack_bytes,
1371                              ExternalReference ext_ref) {
1372   subp(rsp, Immediate(stack_bytes));
1373 
1374   int arg_bytes = 0;
1375   for (ValueType param_type : sig->parameters()) {
1376     liftoff::Store(this, Operand(rsp, arg_bytes), *args++, param_type);
1377     arg_bytes += ValueTypes::MemSize(param_type);
1378   }
1379   DCHECK_LE(arg_bytes, stack_bytes);
1380 
1381   // Pass a pointer to the buffer with the arguments to the C function.
1382   movp(arg_reg_1, rsp);
1383 
1384   constexpr int kNumCCallArgs = 1;
1385 
1386   // Now call the C function.
1387   PrepareCallCFunction(kNumCCallArgs);
1388   CallCFunction(ext_ref, kNumCCallArgs);
1389 
1390   // Move return value to the right register.
1391   const LiftoffRegister* next_result_reg = rets;
1392   if (sig->return_count() > 0) {
1393     DCHECK_EQ(1, sig->return_count());
1394     constexpr Register kReturnReg = rax;
1395     if (kReturnReg != next_result_reg->gp()) {
1396       Move(*next_result_reg, LiftoffRegister(kReturnReg), sig->GetReturn(0));
1397     }
1398     ++next_result_reg;
1399   }
1400 
1401   // Load potential output value from the buffer on the stack.
1402   if (out_argument_type != kWasmStmt) {
1403     liftoff::Load(this, *next_result_reg, Operand(rsp, 0), out_argument_type);
1404   }
1405 
1406   addp(rsp, Immediate(stack_bytes));
1407 }
1408 
CallNativeWasmCode(Address addr)1409 void LiftoffAssembler::CallNativeWasmCode(Address addr) {
1410   near_call(addr, RelocInfo::WASM_CALL);
1411 }
1412 
CallIndirect(wasm::FunctionSig * sig,compiler::CallDescriptor * call_descriptor,Register target)1413 void LiftoffAssembler::CallIndirect(wasm::FunctionSig* sig,
1414                                     compiler::CallDescriptor* call_descriptor,
1415                                     Register target) {
1416   if (target == no_reg) {
1417     popq(kScratchRegister);
1418     target = kScratchRegister;
1419   }
1420   if (FLAG_untrusted_code_mitigations) {
1421     RetpolineCall(target);
1422   } else {
1423     call(target);
1424   }
1425 }
1426 
CallRuntimeStub(WasmCode::RuntimeStubId sid)1427 void LiftoffAssembler::CallRuntimeStub(WasmCode::RuntimeStubId sid) {
1428   // A direct call to a wasm runtime stub defined in this module.
1429   // Just encode the stub index. This will be patched at relocation.
1430   near_call(static_cast<Address>(sid), RelocInfo::WASM_STUB_CALL);
1431 }
1432 
AllocateStackSlot(Register addr,uint32_t size)1433 void LiftoffAssembler::AllocateStackSlot(Register addr, uint32_t size) {
1434   subp(rsp, Immediate(size));
1435   movp(addr, rsp);
1436 }
1437 
DeallocateStackSlot(uint32_t size)1438 void LiftoffAssembler::DeallocateStackSlot(uint32_t size) {
1439   addp(rsp, Immediate(size));
1440 }
1441 
Construct()1442 void LiftoffStackSlots::Construct() {
1443   for (auto& slot : slots_) {
1444     const LiftoffAssembler::VarState& src = slot.src_;
1445     switch (src.loc()) {
1446       case LiftoffAssembler::VarState::kStack:
1447         if (src.type() == kWasmI32) {
1448           // Load i32 values to a register first to ensure they are zero
1449           // extended.
1450           asm_->movl(kScratchRegister, liftoff::GetStackSlot(slot.src_index_));
1451           asm_->pushq(kScratchRegister);
1452         } else {
1453           // For all other types, just push the whole (8-byte) stack slot.
1454           // This is also ok for f32 values (even though we copy 4 uninitialized
1455           // bytes), because f32 and f64 values are clearly distinguished in
1456           // Turbofan, so the uninitialized bytes are never accessed.
1457           asm_->pushq(liftoff::GetStackSlot(slot.src_index_));
1458         }
1459         break;
1460       case LiftoffAssembler::VarState::kRegister:
1461         liftoff::push(asm_, src.reg(), src.type());
1462         break;
1463       case LiftoffAssembler::VarState::KIntConst:
1464         asm_->pushq(Immediate(src.i32_const()));
1465         break;
1466     }
1467   }
1468 }
1469 
1470 #undef REQUIRE_CPU_FEATURE
1471 
1472 }  // namespace wasm
1473 }  // namespace internal
1474 }  // namespace v8
1475 
1476 #endif  // V8_WASM_BASELINE_X64_LIFTOFF_ASSEMBLER_X64_H_
1477