1 // Copyright 2012 the V8 project authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4
5 #include "src/x64/assembler-x64.h"
6
7 #include <cstring>
8
9 #if V8_TARGET_ARCH_X64
10
11 #if V8_LIBC_MSVCRT
12 #include <intrin.h> // _xgetbv()
13 #endif
14 #if V8_OS_MACOSX
15 #include <sys/sysctl.h>
16 #endif
17
18 #include "src/base/bits.h"
19 #include "src/macro-assembler.h"
20 #include "src/v8.h"
21
22 namespace v8 {
23 namespace internal {
24
25 // -----------------------------------------------------------------------------
26 // Implementation of CpuFeatures
27
28 namespace {
29
30 #if !V8_LIBC_MSVCRT
31
_xgetbv(unsigned int xcr)32 V8_INLINE uint64_t _xgetbv(unsigned int xcr) {
33 unsigned eax, edx;
34 // Check xgetbv; this uses a .byte sequence instead of the instruction
35 // directly because older assemblers do not include support for xgetbv and
36 // there is no easy way to conditionally compile based on the assembler
37 // used.
38 __asm__ volatile(".byte 0x0f, 0x01, 0xd0" : "=a"(eax), "=d"(edx) : "c"(xcr));
39 return static_cast<uint64_t>(eax) | (static_cast<uint64_t>(edx) << 32);
40 }
41
42 #define _XCR_XFEATURE_ENABLED_MASK 0
43
44 #endif // !V8_LIBC_MSVCRT
45
46
OSHasAVXSupport()47 bool OSHasAVXSupport() {
48 #if V8_OS_MACOSX
49 // Mac OS X up to 10.9 has a bug where AVX transitions were indeed being
50 // caused by ISRs, so we detect that here and disable AVX in that case.
51 char buffer[128];
52 size_t buffer_size = arraysize(buffer);
53 int ctl_name[] = {CTL_KERN, KERN_OSRELEASE};
54 if (sysctl(ctl_name, 2, buffer, &buffer_size, nullptr, 0) != 0) {
55 V8_Fatal(__FILE__, __LINE__, "V8 failed to get kernel version");
56 }
57 // The buffer now contains a string of the form XX.YY.ZZ, where
58 // XX is the major kernel version component.
59 char* period_pos = strchr(buffer, '.');
60 DCHECK_NOT_NULL(period_pos);
61 *period_pos = '\0';
62 long kernel_version_major = strtol(buffer, nullptr, 10); // NOLINT
63 if (kernel_version_major <= 13) return false;
64 #endif // V8_OS_MACOSX
65 // Check whether OS claims to support AVX.
66 uint64_t feature_mask = _xgetbv(_XCR_XFEATURE_ENABLED_MASK);
67 return (feature_mask & 0x6) == 0x6;
68 }
69
70 } // namespace
71
72
ProbeImpl(bool cross_compile)73 void CpuFeatures::ProbeImpl(bool cross_compile) {
74 base::CPU cpu;
75 CHECK(cpu.has_sse2()); // SSE2 support is mandatory.
76 CHECK(cpu.has_cmov()); // CMOV support is mandatory.
77
78 // Only use statically determined features for cross compile (snapshot).
79 if (cross_compile) return;
80
81 if (cpu.has_sse41() && FLAG_enable_sse4_1) supported_ |= 1u << SSE4_1;
82 if (cpu.has_ssse3() && FLAG_enable_ssse3) supported_ |= 1u << SSSE3;
83 if (cpu.has_sse3() && FLAG_enable_sse3) supported_ |= 1u << SSE3;
84 // SAHF is not generally available in long mode.
85 if (cpu.has_sahf() && FLAG_enable_sahf) supported_ |= 1u << SAHF;
86 if (cpu.has_avx() && FLAG_enable_avx && cpu.has_osxsave() &&
87 OSHasAVXSupport()) {
88 supported_ |= 1u << AVX;
89 }
90 if (cpu.has_fma3() && FLAG_enable_fma3 && cpu.has_osxsave() &&
91 OSHasAVXSupport()) {
92 supported_ |= 1u << FMA3;
93 }
94 if (cpu.has_bmi1() && FLAG_enable_bmi1) supported_ |= 1u << BMI1;
95 if (cpu.has_bmi2() && FLAG_enable_bmi2) supported_ |= 1u << BMI2;
96 if (cpu.has_lzcnt() && FLAG_enable_lzcnt) supported_ |= 1u << LZCNT;
97 if (cpu.has_popcnt() && FLAG_enable_popcnt) supported_ |= 1u << POPCNT;
98 if (strcmp(FLAG_mcpu, "auto") == 0) {
99 if (cpu.is_atom()) supported_ |= 1u << ATOM;
100 } else if (strcmp(FLAG_mcpu, "atom") == 0) {
101 supported_ |= 1u << ATOM;
102 }
103 }
104
105
PrintTarget()106 void CpuFeatures::PrintTarget() { }
PrintFeatures()107 void CpuFeatures::PrintFeatures() {
108 printf(
109 "SSE3=%d SSSE3=%d SSE4_1=%d SAHF=%d AVX=%d FMA3=%d BMI1=%d BMI2=%d "
110 "LZCNT=%d "
111 "POPCNT=%d ATOM=%d\n",
112 CpuFeatures::IsSupported(SSE3), CpuFeatures::IsSupported(SSSE3),
113 CpuFeatures::IsSupported(SSE4_1), CpuFeatures::IsSupported(SAHF),
114 CpuFeatures::IsSupported(AVX), CpuFeatures::IsSupported(FMA3),
115 CpuFeatures::IsSupported(BMI1), CpuFeatures::IsSupported(BMI2),
116 CpuFeatures::IsSupported(LZCNT), CpuFeatures::IsSupported(POPCNT),
117 CpuFeatures::IsSupported(ATOM));
118 }
119
120 // -----------------------------------------------------------------------------
121 // Implementation of RelocInfo
122
wasm_memory_reference()123 Address RelocInfo::wasm_memory_reference() {
124 DCHECK(IsWasmMemoryReference(rmode_));
125 return Memory::Address_at(pc_);
126 }
127
wasm_global_reference()128 Address RelocInfo::wasm_global_reference() {
129 DCHECK(IsWasmGlobalReference(rmode_));
130 return Memory::Address_at(pc_);
131 }
132
wasm_memory_size_reference()133 uint32_t RelocInfo::wasm_memory_size_reference() {
134 DCHECK(IsWasmMemorySizeReference(rmode_));
135 return Memory::uint32_at(pc_);
136 }
137
unchecked_update_wasm_memory_reference(Address address,ICacheFlushMode flush_mode)138 void RelocInfo::unchecked_update_wasm_memory_reference(
139 Address address, ICacheFlushMode flush_mode) {
140 Memory::Address_at(pc_) = address;
141 }
142
unchecked_update_wasm_memory_size(uint32_t size,ICacheFlushMode flush_mode)143 void RelocInfo::unchecked_update_wasm_memory_size(uint32_t size,
144 ICacheFlushMode flush_mode) {
145 Memory::uint32_at(pc_) = size;
146 }
147
148 // -----------------------------------------------------------------------------
149 // Implementation of Operand
150
Operand(Register base,int32_t disp)151 Operand::Operand(Register base, int32_t disp) : rex_(0) {
152 len_ = 1;
153 if (base.is(rsp) || base.is(r12)) {
154 // SIB byte is needed to encode (rsp + offset) or (r12 + offset).
155 set_sib(times_1, rsp, base);
156 }
157
158 if (disp == 0 && !base.is(rbp) && !base.is(r13)) {
159 set_modrm(0, base);
160 } else if (is_int8(disp)) {
161 set_modrm(1, base);
162 set_disp8(disp);
163 } else {
164 set_modrm(2, base);
165 set_disp32(disp);
166 }
167 }
168
169
Operand(Register base,Register index,ScaleFactor scale,int32_t disp)170 Operand::Operand(Register base,
171 Register index,
172 ScaleFactor scale,
173 int32_t disp) : rex_(0) {
174 DCHECK(!index.is(rsp));
175 len_ = 1;
176 set_sib(scale, index, base);
177 if (disp == 0 && !base.is(rbp) && !base.is(r13)) {
178 // This call to set_modrm doesn't overwrite the REX.B (or REX.X) bits
179 // possibly set by set_sib.
180 set_modrm(0, rsp);
181 } else if (is_int8(disp)) {
182 set_modrm(1, rsp);
183 set_disp8(disp);
184 } else {
185 set_modrm(2, rsp);
186 set_disp32(disp);
187 }
188 }
189
190
Operand(Register index,ScaleFactor scale,int32_t disp)191 Operand::Operand(Register index,
192 ScaleFactor scale,
193 int32_t disp) : rex_(0) {
194 DCHECK(!index.is(rsp));
195 len_ = 1;
196 set_modrm(0, rsp);
197 set_sib(scale, index, rbp);
198 set_disp32(disp);
199 }
200
201
Operand(Label * label)202 Operand::Operand(Label* label) : rex_(0), len_(1) {
203 DCHECK_NOT_NULL(label);
204 set_modrm(0, rbp);
205 set_disp64(reinterpret_cast<intptr_t>(label));
206 }
207
208
Operand(const Operand & operand,int32_t offset)209 Operand::Operand(const Operand& operand, int32_t offset) {
210 DCHECK(operand.len_ >= 1);
211 // Operand encodes REX ModR/M [SIB] [Disp].
212 byte modrm = operand.buf_[0];
213 DCHECK(modrm < 0xC0); // Disallow mode 3 (register target).
214 bool has_sib = ((modrm & 0x07) == 0x04);
215 byte mode = modrm & 0xC0;
216 int disp_offset = has_sib ? 2 : 1;
217 int base_reg = (has_sib ? operand.buf_[1] : modrm) & 0x07;
218 // Mode 0 with rbp/r13 as ModR/M or SIB base register always has a 32-bit
219 // displacement.
220 bool is_baseless = (mode == 0) && (base_reg == 0x05); // No base or RIP base.
221 int32_t disp_value = 0;
222 if (mode == 0x80 || is_baseless) {
223 // Mode 2 or mode 0 with rbp/r13 as base: Word displacement.
224 disp_value = *bit_cast<const int32_t*>(&operand.buf_[disp_offset]);
225 } else if (mode == 0x40) {
226 // Mode 1: Byte displacement.
227 disp_value = static_cast<signed char>(operand.buf_[disp_offset]);
228 }
229
230 // Write new operand with same registers, but with modified displacement.
231 DCHECK(offset >= 0 ? disp_value + offset > disp_value
232 : disp_value + offset < disp_value); // No overflow.
233 disp_value += offset;
234 rex_ = operand.rex_;
235 if (!is_int8(disp_value) || is_baseless) {
236 // Need 32 bits of displacement, mode 2 or mode 1 with register rbp/r13.
237 buf_[0] = (modrm & 0x3f) | (is_baseless ? 0x00 : 0x80);
238 len_ = disp_offset + 4;
239 Memory::int32_at(&buf_[disp_offset]) = disp_value;
240 } else if (disp_value != 0 || (base_reg == 0x05)) {
241 // Need 8 bits of displacement.
242 buf_[0] = (modrm & 0x3f) | 0x40; // Mode 1.
243 len_ = disp_offset + 1;
244 buf_[disp_offset] = static_cast<byte>(disp_value);
245 } else {
246 // Need no displacement.
247 buf_[0] = (modrm & 0x3f); // Mode 0.
248 len_ = disp_offset;
249 }
250 if (has_sib) {
251 buf_[1] = operand.buf_[1];
252 }
253 }
254
255
AddressUsesRegister(Register reg) const256 bool Operand::AddressUsesRegister(Register reg) const {
257 int code = reg.code();
258 DCHECK((buf_[0] & 0xC0) != 0xC0); // Always a memory operand.
259 // Start with only low three bits of base register. Initial decoding doesn't
260 // distinguish on the REX.B bit.
261 int base_code = buf_[0] & 0x07;
262 if (base_code == rsp.code()) {
263 // SIB byte present in buf_[1].
264 // Check the index register from the SIB byte + REX.X prefix.
265 int index_code = ((buf_[1] >> 3) & 0x07) | ((rex_ & 0x02) << 2);
266 // Index code (including REX.X) of 0x04 (rsp) means no index register.
267 if (index_code != rsp.code() && index_code == code) return true;
268 // Add REX.B to get the full base register code.
269 base_code = (buf_[1] & 0x07) | ((rex_ & 0x01) << 3);
270 // A base register of 0x05 (rbp) with mod = 0 means no base register.
271 if (base_code == rbp.code() && ((buf_[0] & 0xC0) == 0)) return false;
272 return code == base_code;
273 } else {
274 // A base register with low bits of 0x05 (rbp or r13) and mod = 0 means
275 // no base register.
276 if (base_code == rbp.code() && ((buf_[0] & 0xC0) == 0)) return false;
277 base_code |= ((rex_ & 0x01) << 3);
278 return code == base_code;
279 }
280 }
281
282
283 // -----------------------------------------------------------------------------
284 // Implementation of Assembler.
285
Assembler(Isolate * isolate,void * buffer,int buffer_size)286 Assembler::Assembler(Isolate* isolate, void* buffer, int buffer_size)
287 : AssemblerBase(isolate, buffer, buffer_size), code_targets_(100) {
288 // Clear the buffer in debug mode unless it was provided by the
289 // caller in which case we can't be sure it's okay to overwrite
290 // existing code in it.
291 #ifdef DEBUG
292 if (own_buffer_) {
293 memset(buffer_, 0xCC, buffer_size_); // int3
294 }
295 #endif
296
297 reloc_info_writer.Reposition(buffer_ + buffer_size_, pc_);
298 }
299
300
GetCode(CodeDesc * desc)301 void Assembler::GetCode(CodeDesc* desc) {
302 // Finalize code (at this point overflow() may be true, but the gap ensures
303 // that we are still not overlapping instructions and relocation info).
304 DCHECK(pc_ <= reloc_info_writer.pos()); // No overlap.
305 // Set up code descriptor.
306 desc->buffer = buffer_;
307 desc->buffer_size = buffer_size_;
308 desc->instr_size = pc_offset();
309 DCHECK(desc->instr_size > 0); // Zero-size code objects upset the system.
310 desc->reloc_size =
311 static_cast<int>((buffer_ + buffer_size_) - reloc_info_writer.pos());
312 desc->origin = this;
313 desc->constant_pool_size = 0;
314 desc->unwinding_info_size = 0;
315 desc->unwinding_info = nullptr;
316 }
317
318
Align(int m)319 void Assembler::Align(int m) {
320 DCHECK(base::bits::IsPowerOfTwo32(m));
321 int delta = (m - (pc_offset() & (m - 1))) & (m - 1);
322 Nop(delta);
323 }
324
325
CodeTargetAlign()326 void Assembler::CodeTargetAlign() {
327 Align(16); // Preferred alignment of jump targets on x64.
328 }
329
330
IsNop(Address addr)331 bool Assembler::IsNop(Address addr) {
332 Address a = addr;
333 while (*a == 0x66) a++;
334 if (*a == 0x90) return true;
335 if (a[0] == 0xf && a[1] == 0x1f) return true;
336 return false;
337 }
338
339
bind_to(Label * L,int pos)340 void Assembler::bind_to(Label* L, int pos) {
341 DCHECK(!L->is_bound()); // Label may only be bound once.
342 DCHECK(0 <= pos && pos <= pc_offset()); // Position must be valid.
343 if (L->is_linked()) {
344 int current = L->pos();
345 int next = long_at(current);
346 while (next != current) {
347 if (current >= 4 && long_at(current - 4) == 0) {
348 // Absolute address.
349 intptr_t imm64 = reinterpret_cast<intptr_t>(buffer_ + pos);
350 *reinterpret_cast<intptr_t*>(addr_at(current - 4)) = imm64;
351 internal_reference_positions_.push_back(current - 4);
352 } else {
353 // Relative address, relative to point after address.
354 int imm32 = pos - (current + sizeof(int32_t));
355 long_at_put(current, imm32);
356 }
357 current = next;
358 next = long_at(next);
359 }
360 // Fix up last fixup on linked list.
361 if (current >= 4 && long_at(current - 4) == 0) {
362 // Absolute address.
363 intptr_t imm64 = reinterpret_cast<intptr_t>(buffer_ + pos);
364 *reinterpret_cast<intptr_t*>(addr_at(current - 4)) = imm64;
365 internal_reference_positions_.push_back(current - 4);
366 } else {
367 // Relative address, relative to point after address.
368 int imm32 = pos - (current + sizeof(int32_t));
369 long_at_put(current, imm32);
370 }
371 }
372 while (L->is_near_linked()) {
373 int fixup_pos = L->near_link_pos();
374 int offset_to_next =
375 static_cast<int>(*reinterpret_cast<int8_t*>(addr_at(fixup_pos)));
376 DCHECK(offset_to_next <= 0);
377 int disp = pos - (fixup_pos + sizeof(int8_t));
378 CHECK(is_int8(disp));
379 set_byte_at(fixup_pos, disp);
380 if (offset_to_next < 0) {
381 L->link_to(fixup_pos + offset_to_next, Label::kNear);
382 } else {
383 L->UnuseNear();
384 }
385 }
386 L->bind_to(pos);
387 }
388
389
bind(Label * L)390 void Assembler::bind(Label* L) {
391 bind_to(L, pc_offset());
392 }
393
394
GrowBuffer()395 void Assembler::GrowBuffer() {
396 DCHECK(buffer_overflow());
397 if (!own_buffer_) FATAL("external code buffer is too small");
398
399 // Compute new buffer size.
400 CodeDesc desc; // the new buffer
401 desc.buffer_size = 2 * buffer_size_;
402
403 // Some internal data structures overflow for very large buffers,
404 // they must ensure that kMaximalBufferSize is not too large.
405 if (desc.buffer_size > kMaximalBufferSize ||
406 static_cast<size_t>(desc.buffer_size) >
407 isolate()->heap()->MaxOldGenerationSize()) {
408 V8::FatalProcessOutOfMemory("Assembler::GrowBuffer");
409 }
410
411 // Set up new buffer.
412 desc.buffer = NewArray<byte>(desc.buffer_size);
413 desc.origin = this;
414 desc.instr_size = pc_offset();
415 desc.reloc_size =
416 static_cast<int>((buffer_ + buffer_size_) - (reloc_info_writer.pos()));
417
418 // Clear the buffer in debug mode. Use 'int3' instructions to make
419 // sure to get into problems if we ever run uninitialized code.
420 #ifdef DEBUG
421 memset(desc.buffer, 0xCC, desc.buffer_size);
422 #endif
423
424 // Copy the data.
425 intptr_t pc_delta = desc.buffer - buffer_;
426 intptr_t rc_delta = (desc.buffer + desc.buffer_size) -
427 (buffer_ + buffer_size_);
428 MemMove(desc.buffer, buffer_, desc.instr_size);
429 MemMove(rc_delta + reloc_info_writer.pos(), reloc_info_writer.pos(),
430 desc.reloc_size);
431
432 // Switch buffers.
433 DeleteArray(buffer_);
434 buffer_ = desc.buffer;
435 buffer_size_ = desc.buffer_size;
436 pc_ += pc_delta;
437 reloc_info_writer.Reposition(reloc_info_writer.pos() + rc_delta,
438 reloc_info_writer.last_pc() + pc_delta);
439
440 // Relocate internal references.
441 for (auto pos : internal_reference_positions_) {
442 intptr_t* p = reinterpret_cast<intptr_t*>(buffer_ + pos);
443 *p += pc_delta;
444 }
445
446 DCHECK(!buffer_overflow());
447 }
448
449
emit_operand(int code,const Operand & adr)450 void Assembler::emit_operand(int code, const Operand& adr) {
451 DCHECK(is_uint3(code));
452 const unsigned length = adr.len_;
453 DCHECK(length > 0);
454
455 // Emit updated ModR/M byte containing the given register.
456 DCHECK((adr.buf_[0] & 0x38) == 0);
457 *pc_++ = adr.buf_[0] | code << 3;
458
459 // Recognize RIP relative addressing.
460 if (adr.buf_[0] == 5) {
461 DCHECK_EQ(9u, length);
462 Label* label = *bit_cast<Label* const*>(&adr.buf_[1]);
463 if (label->is_bound()) {
464 int offset = label->pos() - pc_offset() - sizeof(int32_t);
465 DCHECK_GE(0, offset);
466 emitl(offset);
467 } else if (label->is_linked()) {
468 emitl(label->pos());
469 label->link_to(pc_offset() - sizeof(int32_t));
470 } else {
471 DCHECK(label->is_unused());
472 int32_t current = pc_offset();
473 emitl(current);
474 label->link_to(current);
475 }
476 } else {
477 // Emit the rest of the encoded operand.
478 for (unsigned i = 1; i < length; i++) *pc_++ = adr.buf_[i];
479 }
480 }
481
482
483 // Assembler Instruction implementations.
484
arithmetic_op(byte opcode,Register reg,const Operand & op,int size)485 void Assembler::arithmetic_op(byte opcode,
486 Register reg,
487 const Operand& op,
488 int size) {
489 EnsureSpace ensure_space(this);
490 emit_rex(reg, op, size);
491 emit(opcode);
492 emit_operand(reg, op);
493 }
494
495
arithmetic_op(byte opcode,Register reg,Register rm_reg,int size)496 void Assembler::arithmetic_op(byte opcode,
497 Register reg,
498 Register rm_reg,
499 int size) {
500 EnsureSpace ensure_space(this);
501 DCHECK((opcode & 0xC6) == 2);
502 if (rm_reg.low_bits() == 4) { // Forces SIB byte.
503 // Swap reg and rm_reg and change opcode operand order.
504 emit_rex(rm_reg, reg, size);
505 emit(opcode ^ 0x02);
506 emit_modrm(rm_reg, reg);
507 } else {
508 emit_rex(reg, rm_reg, size);
509 emit(opcode);
510 emit_modrm(reg, rm_reg);
511 }
512 }
513
514
arithmetic_op_16(byte opcode,Register reg,Register rm_reg)515 void Assembler::arithmetic_op_16(byte opcode, Register reg, Register rm_reg) {
516 EnsureSpace ensure_space(this);
517 DCHECK((opcode & 0xC6) == 2);
518 if (rm_reg.low_bits() == 4) { // Forces SIB byte.
519 // Swap reg and rm_reg and change opcode operand order.
520 emit(0x66);
521 emit_optional_rex_32(rm_reg, reg);
522 emit(opcode ^ 0x02);
523 emit_modrm(rm_reg, reg);
524 } else {
525 emit(0x66);
526 emit_optional_rex_32(reg, rm_reg);
527 emit(opcode);
528 emit_modrm(reg, rm_reg);
529 }
530 }
531
532
arithmetic_op_16(byte opcode,Register reg,const Operand & rm_reg)533 void Assembler::arithmetic_op_16(byte opcode,
534 Register reg,
535 const Operand& rm_reg) {
536 EnsureSpace ensure_space(this);
537 emit(0x66);
538 emit_optional_rex_32(reg, rm_reg);
539 emit(opcode);
540 emit_operand(reg, rm_reg);
541 }
542
543
arithmetic_op_8(byte opcode,Register reg,const Operand & op)544 void Assembler::arithmetic_op_8(byte opcode, Register reg, const Operand& op) {
545 EnsureSpace ensure_space(this);
546 if (!reg.is_byte_register()) {
547 emit_rex_32(reg, op);
548 } else {
549 emit_optional_rex_32(reg, op);
550 }
551 emit(opcode);
552 emit_operand(reg, op);
553 }
554
555
arithmetic_op_8(byte opcode,Register reg,Register rm_reg)556 void Assembler::arithmetic_op_8(byte opcode, Register reg, Register rm_reg) {
557 EnsureSpace ensure_space(this);
558 DCHECK((opcode & 0xC6) == 2);
559 if (rm_reg.low_bits() == 4) { // Forces SIB byte.
560 // Swap reg and rm_reg and change opcode operand order.
561 if (!rm_reg.is_byte_register() || !reg.is_byte_register()) {
562 // Register is not one of al, bl, cl, dl. Its encoding needs REX.
563 emit_rex_32(rm_reg, reg);
564 }
565 emit(opcode ^ 0x02);
566 emit_modrm(rm_reg, reg);
567 } else {
568 if (!reg.is_byte_register() || !rm_reg.is_byte_register()) {
569 // Register is not one of al, bl, cl, dl. Its encoding needs REX.
570 emit_rex_32(reg, rm_reg);
571 }
572 emit(opcode);
573 emit_modrm(reg, rm_reg);
574 }
575 }
576
577
immediate_arithmetic_op(byte subcode,Register dst,Immediate src,int size)578 void Assembler::immediate_arithmetic_op(byte subcode,
579 Register dst,
580 Immediate src,
581 int size) {
582 EnsureSpace ensure_space(this);
583 emit_rex(dst, size);
584 if (is_int8(src.value_) && RelocInfo::IsNone(src.rmode_)) {
585 emit(0x83);
586 emit_modrm(subcode, dst);
587 emit(src.value_);
588 } else if (dst.is(rax)) {
589 emit(0x05 | (subcode << 3));
590 emit(src);
591 } else {
592 emit(0x81);
593 emit_modrm(subcode, dst);
594 emit(src);
595 }
596 }
597
immediate_arithmetic_op(byte subcode,const Operand & dst,Immediate src,int size)598 void Assembler::immediate_arithmetic_op(byte subcode,
599 const Operand& dst,
600 Immediate src,
601 int size) {
602 EnsureSpace ensure_space(this);
603 emit_rex(dst, size);
604 if (is_int8(src.value_)) {
605 emit(0x83);
606 emit_operand(subcode, dst);
607 if (!RelocInfo::IsNone(src.rmode_)) {
608 RecordRelocInfo(src.rmode_);
609 }
610 emit(src.value_);
611 } else {
612 emit(0x81);
613 emit_operand(subcode, dst);
614 emit(src);
615 }
616 }
617
618
immediate_arithmetic_op_16(byte subcode,Register dst,Immediate src)619 void Assembler::immediate_arithmetic_op_16(byte subcode,
620 Register dst,
621 Immediate src) {
622 EnsureSpace ensure_space(this);
623 emit(0x66); // Operand size override prefix.
624 emit_optional_rex_32(dst);
625 if (is_int8(src.value_)) {
626 emit(0x83);
627 emit_modrm(subcode, dst);
628 emit(src.value_);
629 } else if (dst.is(rax)) {
630 emit(0x05 | (subcode << 3));
631 emitw(src.value_);
632 } else {
633 emit(0x81);
634 emit_modrm(subcode, dst);
635 emitw(src.value_);
636 }
637 }
638
639
immediate_arithmetic_op_16(byte subcode,const Operand & dst,Immediate src)640 void Assembler::immediate_arithmetic_op_16(byte subcode,
641 const Operand& dst,
642 Immediate src) {
643 EnsureSpace ensure_space(this);
644 emit(0x66); // Operand size override prefix.
645 emit_optional_rex_32(dst);
646 if (is_int8(src.value_)) {
647 emit(0x83);
648 emit_operand(subcode, dst);
649 emit(src.value_);
650 } else {
651 emit(0x81);
652 emit_operand(subcode, dst);
653 emitw(src.value_);
654 }
655 }
656
657
immediate_arithmetic_op_8(byte subcode,const Operand & dst,Immediate src)658 void Assembler::immediate_arithmetic_op_8(byte subcode,
659 const Operand& dst,
660 Immediate src) {
661 EnsureSpace ensure_space(this);
662 emit_optional_rex_32(dst);
663 DCHECK(is_int8(src.value_) || is_uint8(src.value_));
664 emit(0x80);
665 emit_operand(subcode, dst);
666 emit(src.value_);
667 }
668
669
immediate_arithmetic_op_8(byte subcode,Register dst,Immediate src)670 void Assembler::immediate_arithmetic_op_8(byte subcode,
671 Register dst,
672 Immediate src) {
673 EnsureSpace ensure_space(this);
674 if (!dst.is_byte_register()) {
675 // Register is not one of al, bl, cl, dl. Its encoding needs REX.
676 emit_rex_32(dst);
677 }
678 DCHECK(is_int8(src.value_) || is_uint8(src.value_));
679 emit(0x80);
680 emit_modrm(subcode, dst);
681 emit(src.value_);
682 }
683
684
shift(Register dst,Immediate shift_amount,int subcode,int size)685 void Assembler::shift(Register dst,
686 Immediate shift_amount,
687 int subcode,
688 int size) {
689 EnsureSpace ensure_space(this);
690 DCHECK(size == kInt64Size ? is_uint6(shift_amount.value_)
691 : is_uint5(shift_amount.value_));
692 if (shift_amount.value_ == 1) {
693 emit_rex(dst, size);
694 emit(0xD1);
695 emit_modrm(subcode, dst);
696 } else {
697 emit_rex(dst, size);
698 emit(0xC1);
699 emit_modrm(subcode, dst);
700 emit(shift_amount.value_);
701 }
702 }
703
704
shift(Operand dst,Immediate shift_amount,int subcode,int size)705 void Assembler::shift(Operand dst, Immediate shift_amount, int subcode,
706 int size) {
707 EnsureSpace ensure_space(this);
708 DCHECK(size == kInt64Size ? is_uint6(shift_amount.value_)
709 : is_uint5(shift_amount.value_));
710 if (shift_amount.value_ == 1) {
711 emit_rex(dst, size);
712 emit(0xD1);
713 emit_operand(subcode, dst);
714 } else {
715 emit_rex(dst, size);
716 emit(0xC1);
717 emit_operand(subcode, dst);
718 emit(shift_amount.value_);
719 }
720 }
721
722
shift(Register dst,int subcode,int size)723 void Assembler::shift(Register dst, int subcode, int size) {
724 EnsureSpace ensure_space(this);
725 emit_rex(dst, size);
726 emit(0xD3);
727 emit_modrm(subcode, dst);
728 }
729
730
shift(Operand dst,int subcode,int size)731 void Assembler::shift(Operand dst, int subcode, int size) {
732 EnsureSpace ensure_space(this);
733 emit_rex(dst, size);
734 emit(0xD3);
735 emit_operand(subcode, dst);
736 }
737
738
bt(const Operand & dst,Register src)739 void Assembler::bt(const Operand& dst, Register src) {
740 EnsureSpace ensure_space(this);
741 emit_rex_64(src, dst);
742 emit(0x0F);
743 emit(0xA3);
744 emit_operand(src, dst);
745 }
746
747
bts(const Operand & dst,Register src)748 void Assembler::bts(const Operand& dst, Register src) {
749 EnsureSpace ensure_space(this);
750 emit_rex_64(src, dst);
751 emit(0x0F);
752 emit(0xAB);
753 emit_operand(src, dst);
754 }
755
756
bsrl(Register dst,Register src)757 void Assembler::bsrl(Register dst, Register src) {
758 EnsureSpace ensure_space(this);
759 emit_optional_rex_32(dst, src);
760 emit(0x0F);
761 emit(0xBD);
762 emit_modrm(dst, src);
763 }
764
765
bsrl(Register dst,const Operand & src)766 void Assembler::bsrl(Register dst, const Operand& src) {
767 EnsureSpace ensure_space(this);
768 emit_optional_rex_32(dst, src);
769 emit(0x0F);
770 emit(0xBD);
771 emit_operand(dst, src);
772 }
773
774
bsrq(Register dst,Register src)775 void Assembler::bsrq(Register dst, Register src) {
776 EnsureSpace ensure_space(this);
777 emit_rex_64(dst, src);
778 emit(0x0F);
779 emit(0xBD);
780 emit_modrm(dst, src);
781 }
782
783
bsrq(Register dst,const Operand & src)784 void Assembler::bsrq(Register dst, const Operand& src) {
785 EnsureSpace ensure_space(this);
786 emit_rex_64(dst, src);
787 emit(0x0F);
788 emit(0xBD);
789 emit_operand(dst, src);
790 }
791
792
bsfl(Register dst,Register src)793 void Assembler::bsfl(Register dst, Register src) {
794 EnsureSpace ensure_space(this);
795 emit_optional_rex_32(dst, src);
796 emit(0x0F);
797 emit(0xBC);
798 emit_modrm(dst, src);
799 }
800
801
bsfl(Register dst,const Operand & src)802 void Assembler::bsfl(Register dst, const Operand& src) {
803 EnsureSpace ensure_space(this);
804 emit_optional_rex_32(dst, src);
805 emit(0x0F);
806 emit(0xBC);
807 emit_operand(dst, src);
808 }
809
810
bsfq(Register dst,Register src)811 void Assembler::bsfq(Register dst, Register src) {
812 EnsureSpace ensure_space(this);
813 emit_rex_64(dst, src);
814 emit(0x0F);
815 emit(0xBC);
816 emit_modrm(dst, src);
817 }
818
819
bsfq(Register dst,const Operand & src)820 void Assembler::bsfq(Register dst, const Operand& src) {
821 EnsureSpace ensure_space(this);
822 emit_rex_64(dst, src);
823 emit(0x0F);
824 emit(0xBC);
825 emit_operand(dst, src);
826 }
827
828
call(Label * L)829 void Assembler::call(Label* L) {
830 EnsureSpace ensure_space(this);
831 // 1110 1000 #32-bit disp.
832 emit(0xE8);
833 if (L->is_bound()) {
834 int offset = L->pos() - pc_offset() - sizeof(int32_t);
835 DCHECK(offset <= 0);
836 emitl(offset);
837 } else if (L->is_linked()) {
838 emitl(L->pos());
839 L->link_to(pc_offset() - sizeof(int32_t));
840 } else {
841 DCHECK(L->is_unused());
842 int32_t current = pc_offset();
843 emitl(current);
844 L->link_to(current);
845 }
846 }
847
848
call(Address entry,RelocInfo::Mode rmode)849 void Assembler::call(Address entry, RelocInfo::Mode rmode) {
850 DCHECK(RelocInfo::IsRuntimeEntry(rmode));
851 EnsureSpace ensure_space(this);
852 // 1110 1000 #32-bit disp.
853 emit(0xE8);
854 emit_runtime_entry(entry, rmode);
855 }
856
857
call(Handle<Code> target,RelocInfo::Mode rmode,TypeFeedbackId ast_id)858 void Assembler::call(Handle<Code> target,
859 RelocInfo::Mode rmode,
860 TypeFeedbackId ast_id) {
861 EnsureSpace ensure_space(this);
862 // 1110 1000 #32-bit disp.
863 emit(0xE8);
864 emit_code_target(target, rmode, ast_id);
865 }
866
867
call(Register adr)868 void Assembler::call(Register adr) {
869 EnsureSpace ensure_space(this);
870 // Opcode: FF /2 r64.
871 emit_optional_rex_32(adr);
872 emit(0xFF);
873 emit_modrm(0x2, adr);
874 }
875
876
call(const Operand & op)877 void Assembler::call(const Operand& op) {
878 EnsureSpace ensure_space(this);
879 // Opcode: FF /2 m64.
880 emit_optional_rex_32(op);
881 emit(0xFF);
882 emit_operand(0x2, op);
883 }
884
885
886 // Calls directly to the given address using a relative offset.
887 // Should only ever be used in Code objects for calls within the
888 // same Code object. Should not be used when generating new code (use labels),
889 // but only when patching existing code.
call(Address target)890 void Assembler::call(Address target) {
891 EnsureSpace ensure_space(this);
892 // 1110 1000 #32-bit disp.
893 emit(0xE8);
894 Address source = pc_ + 4;
895 intptr_t displacement = target - source;
896 DCHECK(is_int32(displacement));
897 emitl(static_cast<int32_t>(displacement));
898 }
899
900
clc()901 void Assembler::clc() {
902 EnsureSpace ensure_space(this);
903 emit(0xF8);
904 }
905
906
cld()907 void Assembler::cld() {
908 EnsureSpace ensure_space(this);
909 emit(0xFC);
910 }
911
912
cdq()913 void Assembler::cdq() {
914 EnsureSpace ensure_space(this);
915 emit(0x99);
916 }
917
918
cmovq(Condition cc,Register dst,Register src)919 void Assembler::cmovq(Condition cc, Register dst, Register src) {
920 if (cc == always) {
921 movq(dst, src);
922 } else if (cc == never) {
923 return;
924 }
925 // No need to check CpuInfo for CMOV support, it's a required part of the
926 // 64-bit architecture.
927 DCHECK(cc >= 0); // Use mov for unconditional moves.
928 EnsureSpace ensure_space(this);
929 // Opcode: REX.W 0f 40 + cc /r.
930 emit_rex_64(dst, src);
931 emit(0x0f);
932 emit(0x40 + cc);
933 emit_modrm(dst, src);
934 }
935
936
cmovq(Condition cc,Register dst,const Operand & src)937 void Assembler::cmovq(Condition cc, Register dst, const Operand& src) {
938 if (cc == always) {
939 movq(dst, src);
940 } else if (cc == never) {
941 return;
942 }
943 DCHECK(cc >= 0);
944 EnsureSpace ensure_space(this);
945 // Opcode: REX.W 0f 40 + cc /r.
946 emit_rex_64(dst, src);
947 emit(0x0f);
948 emit(0x40 + cc);
949 emit_operand(dst, src);
950 }
951
952
cmovl(Condition cc,Register dst,Register src)953 void Assembler::cmovl(Condition cc, Register dst, Register src) {
954 if (cc == always) {
955 movl(dst, src);
956 } else if (cc == never) {
957 return;
958 }
959 DCHECK(cc >= 0);
960 EnsureSpace ensure_space(this);
961 // Opcode: 0f 40 + cc /r.
962 emit_optional_rex_32(dst, src);
963 emit(0x0f);
964 emit(0x40 + cc);
965 emit_modrm(dst, src);
966 }
967
968
cmovl(Condition cc,Register dst,const Operand & src)969 void Assembler::cmovl(Condition cc, Register dst, const Operand& src) {
970 if (cc == always) {
971 movl(dst, src);
972 } else if (cc == never) {
973 return;
974 }
975 DCHECK(cc >= 0);
976 EnsureSpace ensure_space(this);
977 // Opcode: 0f 40 + cc /r.
978 emit_optional_rex_32(dst, src);
979 emit(0x0f);
980 emit(0x40 + cc);
981 emit_operand(dst, src);
982 }
983
984
cmpb_al(Immediate imm8)985 void Assembler::cmpb_al(Immediate imm8) {
986 DCHECK(is_int8(imm8.value_) || is_uint8(imm8.value_));
987 EnsureSpace ensure_space(this);
988 emit(0x3c);
989 emit(imm8.value_);
990 }
991
lock()992 void Assembler::lock() {
993 EnsureSpace ensure_space(this);
994 emit(0xf0);
995 }
996
cmpxchgb(const Operand & dst,Register src)997 void Assembler::cmpxchgb(const Operand& dst, Register src) {
998 EnsureSpace ensure_space(this);
999 if (!src.is_byte_register()) {
1000 // Register is not one of al, bl, cl, dl. Its encoding needs REX.
1001 emit_rex_32(src, dst);
1002 } else {
1003 emit_optional_rex_32(src, dst);
1004 }
1005 emit(0x0f);
1006 emit(0xb0);
1007 emit_operand(src, dst);
1008 }
1009
cmpxchgw(const Operand & dst,Register src)1010 void Assembler::cmpxchgw(const Operand& dst, Register src) {
1011 EnsureSpace ensure_space(this);
1012 emit(0x66);
1013 emit_optional_rex_32(src, dst);
1014 emit(0x0f);
1015 emit(0xb1);
1016 emit_operand(src, dst);
1017 }
1018
emit_cmpxchg(const Operand & dst,Register src,int size)1019 void Assembler::emit_cmpxchg(const Operand& dst, Register src, int size) {
1020 EnsureSpace ensure_space(this);
1021 emit_rex(src, dst, size);
1022 emit(0x0f);
1023 emit(0xb1);
1024 emit_operand(src, dst);
1025 }
1026
cpuid()1027 void Assembler::cpuid() {
1028 EnsureSpace ensure_space(this);
1029 emit(0x0F);
1030 emit(0xA2);
1031 }
1032
1033
cqo()1034 void Assembler::cqo() {
1035 EnsureSpace ensure_space(this);
1036 emit_rex_64();
1037 emit(0x99);
1038 }
1039
1040
emit_dec(Register dst,int size)1041 void Assembler::emit_dec(Register dst, int size) {
1042 EnsureSpace ensure_space(this);
1043 emit_rex(dst, size);
1044 emit(0xFF);
1045 emit_modrm(0x1, dst);
1046 }
1047
1048
emit_dec(const Operand & dst,int size)1049 void Assembler::emit_dec(const Operand& dst, int size) {
1050 EnsureSpace ensure_space(this);
1051 emit_rex(dst, size);
1052 emit(0xFF);
1053 emit_operand(1, dst);
1054 }
1055
1056
decb(Register dst)1057 void Assembler::decb(Register dst) {
1058 EnsureSpace ensure_space(this);
1059 if (!dst.is_byte_register()) {
1060 // Register is not one of al, bl, cl, dl. Its encoding needs REX.
1061 emit_rex_32(dst);
1062 }
1063 emit(0xFE);
1064 emit_modrm(0x1, dst);
1065 }
1066
1067
decb(const Operand & dst)1068 void Assembler::decb(const Operand& dst) {
1069 EnsureSpace ensure_space(this);
1070 emit_optional_rex_32(dst);
1071 emit(0xFE);
1072 emit_operand(1, dst);
1073 }
1074
1075
enter(Immediate size)1076 void Assembler::enter(Immediate size) {
1077 EnsureSpace ensure_space(this);
1078 emit(0xC8);
1079 emitw(size.value_); // 16 bit operand, always.
1080 emit(0);
1081 }
1082
1083
hlt()1084 void Assembler::hlt() {
1085 EnsureSpace ensure_space(this);
1086 emit(0xF4);
1087 }
1088
1089
emit_idiv(Register src,int size)1090 void Assembler::emit_idiv(Register src, int size) {
1091 EnsureSpace ensure_space(this);
1092 emit_rex(src, size);
1093 emit(0xF7);
1094 emit_modrm(0x7, src);
1095 }
1096
1097
emit_div(Register src,int size)1098 void Assembler::emit_div(Register src, int size) {
1099 EnsureSpace ensure_space(this);
1100 emit_rex(src, size);
1101 emit(0xF7);
1102 emit_modrm(0x6, src);
1103 }
1104
1105
emit_imul(Register src,int size)1106 void Assembler::emit_imul(Register src, int size) {
1107 EnsureSpace ensure_space(this);
1108 emit_rex(src, size);
1109 emit(0xF7);
1110 emit_modrm(0x5, src);
1111 }
1112
1113
emit_imul(const Operand & src,int size)1114 void Assembler::emit_imul(const Operand& src, int size) {
1115 EnsureSpace ensure_space(this);
1116 emit_rex(src, size);
1117 emit(0xF7);
1118 emit_operand(0x5, src);
1119 }
1120
1121
emit_imul(Register dst,Register src,int size)1122 void Assembler::emit_imul(Register dst, Register src, int size) {
1123 EnsureSpace ensure_space(this);
1124 emit_rex(dst, src, size);
1125 emit(0x0F);
1126 emit(0xAF);
1127 emit_modrm(dst, src);
1128 }
1129
1130
emit_imul(Register dst,const Operand & src,int size)1131 void Assembler::emit_imul(Register dst, const Operand& src, int size) {
1132 EnsureSpace ensure_space(this);
1133 emit_rex(dst, src, size);
1134 emit(0x0F);
1135 emit(0xAF);
1136 emit_operand(dst, src);
1137 }
1138
1139
emit_imul(Register dst,Register src,Immediate imm,int size)1140 void Assembler::emit_imul(Register dst, Register src, Immediate imm, int size) {
1141 EnsureSpace ensure_space(this);
1142 emit_rex(dst, src, size);
1143 if (is_int8(imm.value_)) {
1144 emit(0x6B);
1145 emit_modrm(dst, src);
1146 emit(imm.value_);
1147 } else {
1148 emit(0x69);
1149 emit_modrm(dst, src);
1150 emitl(imm.value_);
1151 }
1152 }
1153
1154
emit_imul(Register dst,const Operand & src,Immediate imm,int size)1155 void Assembler::emit_imul(Register dst, const Operand& src, Immediate imm,
1156 int size) {
1157 EnsureSpace ensure_space(this);
1158 emit_rex(dst, src, size);
1159 if (is_int8(imm.value_)) {
1160 emit(0x6B);
1161 emit_operand(dst, src);
1162 emit(imm.value_);
1163 } else {
1164 emit(0x69);
1165 emit_operand(dst, src);
1166 emitl(imm.value_);
1167 }
1168 }
1169
1170
emit_inc(Register dst,int size)1171 void Assembler::emit_inc(Register dst, int size) {
1172 EnsureSpace ensure_space(this);
1173 emit_rex(dst, size);
1174 emit(0xFF);
1175 emit_modrm(0x0, dst);
1176 }
1177
1178
emit_inc(const Operand & dst,int size)1179 void Assembler::emit_inc(const Operand& dst, int size) {
1180 EnsureSpace ensure_space(this);
1181 emit_rex(dst, size);
1182 emit(0xFF);
1183 emit_operand(0, dst);
1184 }
1185
1186
int3()1187 void Assembler::int3() {
1188 EnsureSpace ensure_space(this);
1189 emit(0xCC);
1190 }
1191
1192
j(Condition cc,Label * L,Label::Distance distance)1193 void Assembler::j(Condition cc, Label* L, Label::Distance distance) {
1194 if (cc == always) {
1195 jmp(L);
1196 return;
1197 } else if (cc == never) {
1198 return;
1199 }
1200 EnsureSpace ensure_space(this);
1201 DCHECK(is_uint4(cc));
1202 if (L->is_bound()) {
1203 const int short_size = 2;
1204 const int long_size = 6;
1205 int offs = L->pos() - pc_offset();
1206 DCHECK(offs <= 0);
1207 // Determine whether we can use 1-byte offsets for backwards branches,
1208 // which have a max range of 128 bytes.
1209
1210 // We also need to check predictable_code_size() flag here, because on x64,
1211 // when the full code generator recompiles code for debugging, some places
1212 // need to be padded out to a certain size. The debugger is keeping track of
1213 // how often it did this so that it can adjust return addresses on the
1214 // stack, but if the size of jump instructions can also change, that's not
1215 // enough and the calculated offsets would be incorrect.
1216 if (is_int8(offs - short_size) && !predictable_code_size()) {
1217 // 0111 tttn #8-bit disp.
1218 emit(0x70 | cc);
1219 emit((offs - short_size) & 0xFF);
1220 } else {
1221 // 0000 1111 1000 tttn #32-bit disp.
1222 emit(0x0F);
1223 emit(0x80 | cc);
1224 emitl(offs - long_size);
1225 }
1226 } else if (distance == Label::kNear) {
1227 // 0111 tttn #8-bit disp
1228 emit(0x70 | cc);
1229 byte disp = 0x00;
1230 if (L->is_near_linked()) {
1231 int offset = L->near_link_pos() - pc_offset();
1232 DCHECK(is_int8(offset));
1233 disp = static_cast<byte>(offset & 0xFF);
1234 }
1235 L->link_to(pc_offset(), Label::kNear);
1236 emit(disp);
1237 } else if (L->is_linked()) {
1238 // 0000 1111 1000 tttn #32-bit disp.
1239 emit(0x0F);
1240 emit(0x80 | cc);
1241 emitl(L->pos());
1242 L->link_to(pc_offset() - sizeof(int32_t));
1243 } else {
1244 DCHECK(L->is_unused());
1245 emit(0x0F);
1246 emit(0x80 | cc);
1247 int32_t current = pc_offset();
1248 emitl(current);
1249 L->link_to(current);
1250 }
1251 }
1252
1253
j(Condition cc,Address entry,RelocInfo::Mode rmode)1254 void Assembler::j(Condition cc, Address entry, RelocInfo::Mode rmode) {
1255 DCHECK(RelocInfo::IsRuntimeEntry(rmode));
1256 EnsureSpace ensure_space(this);
1257 DCHECK(is_uint4(cc));
1258 emit(0x0F);
1259 emit(0x80 | cc);
1260 emit_runtime_entry(entry, rmode);
1261 }
1262
1263
j(Condition cc,Handle<Code> target,RelocInfo::Mode rmode)1264 void Assembler::j(Condition cc,
1265 Handle<Code> target,
1266 RelocInfo::Mode rmode) {
1267 EnsureSpace ensure_space(this);
1268 DCHECK(is_uint4(cc));
1269 // 0000 1111 1000 tttn #32-bit disp.
1270 emit(0x0F);
1271 emit(0x80 | cc);
1272 emit_code_target(target, rmode);
1273 }
1274
1275
jmp(Label * L,Label::Distance distance)1276 void Assembler::jmp(Label* L, Label::Distance distance) {
1277 EnsureSpace ensure_space(this);
1278 const int short_size = sizeof(int8_t);
1279 const int long_size = sizeof(int32_t);
1280 if (L->is_bound()) {
1281 int offs = L->pos() - pc_offset() - 1;
1282 DCHECK(offs <= 0);
1283 if (is_int8(offs - short_size) && !predictable_code_size()) {
1284 // 1110 1011 #8-bit disp.
1285 emit(0xEB);
1286 emit((offs - short_size) & 0xFF);
1287 } else {
1288 // 1110 1001 #32-bit disp.
1289 emit(0xE9);
1290 emitl(offs - long_size);
1291 }
1292 } else if (distance == Label::kNear) {
1293 emit(0xEB);
1294 byte disp = 0x00;
1295 if (L->is_near_linked()) {
1296 int offset = L->near_link_pos() - pc_offset();
1297 DCHECK(is_int8(offset));
1298 disp = static_cast<byte>(offset & 0xFF);
1299 }
1300 L->link_to(pc_offset(), Label::kNear);
1301 emit(disp);
1302 } else if (L->is_linked()) {
1303 // 1110 1001 #32-bit disp.
1304 emit(0xE9);
1305 emitl(L->pos());
1306 L->link_to(pc_offset() - long_size);
1307 } else {
1308 // 1110 1001 #32-bit disp.
1309 DCHECK(L->is_unused());
1310 emit(0xE9);
1311 int32_t current = pc_offset();
1312 emitl(current);
1313 L->link_to(current);
1314 }
1315 }
1316
1317
jmp(Handle<Code> target,RelocInfo::Mode rmode)1318 void Assembler::jmp(Handle<Code> target, RelocInfo::Mode rmode) {
1319 EnsureSpace ensure_space(this);
1320 // 1110 1001 #32-bit disp.
1321 emit(0xE9);
1322 emit_code_target(target, rmode);
1323 }
1324
1325
jmp(Address entry,RelocInfo::Mode rmode)1326 void Assembler::jmp(Address entry, RelocInfo::Mode rmode) {
1327 DCHECK(RelocInfo::IsRuntimeEntry(rmode));
1328 EnsureSpace ensure_space(this);
1329 DCHECK(RelocInfo::IsRuntimeEntry(rmode));
1330 emit(0xE9);
1331 emit_runtime_entry(entry, rmode);
1332 }
1333
1334
jmp(Register target)1335 void Assembler::jmp(Register target) {
1336 EnsureSpace ensure_space(this);
1337 // Opcode FF/4 r64.
1338 emit_optional_rex_32(target);
1339 emit(0xFF);
1340 emit_modrm(0x4, target);
1341 }
1342
1343
jmp(const Operand & src)1344 void Assembler::jmp(const Operand& src) {
1345 EnsureSpace ensure_space(this);
1346 // Opcode FF/4 m64.
1347 emit_optional_rex_32(src);
1348 emit(0xFF);
1349 emit_operand(0x4, src);
1350 }
1351
1352
emit_lea(Register dst,const Operand & src,int size)1353 void Assembler::emit_lea(Register dst, const Operand& src, int size) {
1354 EnsureSpace ensure_space(this);
1355 emit_rex(dst, src, size);
1356 emit(0x8D);
1357 emit_operand(dst, src);
1358 }
1359
1360
load_rax(void * value,RelocInfo::Mode mode)1361 void Assembler::load_rax(void* value, RelocInfo::Mode mode) {
1362 EnsureSpace ensure_space(this);
1363 if (kPointerSize == kInt64Size) {
1364 emit(0x48); // REX.W
1365 emit(0xA1);
1366 emitp(value, mode);
1367 } else {
1368 DCHECK(kPointerSize == kInt32Size);
1369 emit(0xA1);
1370 emitp(value, mode);
1371 // In 64-bit mode, need to zero extend the operand to 8 bytes.
1372 // See 2.2.1.4 in Intel64 and IA32 Architectures Software
1373 // Developer's Manual Volume 2.
1374 emitl(0);
1375 }
1376 }
1377
1378
load_rax(ExternalReference ref)1379 void Assembler::load_rax(ExternalReference ref) {
1380 load_rax(ref.address(), RelocInfo::EXTERNAL_REFERENCE);
1381 }
1382
1383
leave()1384 void Assembler::leave() {
1385 EnsureSpace ensure_space(this);
1386 emit(0xC9);
1387 }
1388
1389
movb(Register dst,const Operand & src)1390 void Assembler::movb(Register dst, const Operand& src) {
1391 EnsureSpace ensure_space(this);
1392 if (!dst.is_byte_register()) {
1393 // Register is not one of al, bl, cl, dl. Its encoding needs REX.
1394 emit_rex_32(dst, src);
1395 } else {
1396 emit_optional_rex_32(dst, src);
1397 }
1398 emit(0x8A);
1399 emit_operand(dst, src);
1400 }
1401
1402
movb(Register dst,Immediate imm)1403 void Assembler::movb(Register dst, Immediate imm) {
1404 EnsureSpace ensure_space(this);
1405 if (!dst.is_byte_register()) {
1406 // Register is not one of al, bl, cl, dl. Its encoding needs REX.
1407 emit_rex_32(dst);
1408 }
1409 emit(0xB0 + dst.low_bits());
1410 emit(imm.value_);
1411 }
1412
1413
movb(const Operand & dst,Register src)1414 void Assembler::movb(const Operand& dst, Register src) {
1415 EnsureSpace ensure_space(this);
1416 if (!src.is_byte_register()) {
1417 // Register is not one of al, bl, cl, dl. Its encoding needs REX.
1418 emit_rex_32(src, dst);
1419 } else {
1420 emit_optional_rex_32(src, dst);
1421 }
1422 emit(0x88);
1423 emit_operand(src, dst);
1424 }
1425
1426
movb(const Operand & dst,Immediate imm)1427 void Assembler::movb(const Operand& dst, Immediate imm) {
1428 EnsureSpace ensure_space(this);
1429 emit_optional_rex_32(dst);
1430 emit(0xC6);
1431 emit_operand(0x0, dst);
1432 emit(static_cast<byte>(imm.value_));
1433 }
1434
1435
movw(Register dst,const Operand & src)1436 void Assembler::movw(Register dst, const Operand& src) {
1437 EnsureSpace ensure_space(this);
1438 emit(0x66);
1439 emit_optional_rex_32(dst, src);
1440 emit(0x8B);
1441 emit_operand(dst, src);
1442 }
1443
1444
movw(const Operand & dst,Register src)1445 void Assembler::movw(const Operand& dst, Register src) {
1446 EnsureSpace ensure_space(this);
1447 emit(0x66);
1448 emit_optional_rex_32(src, dst);
1449 emit(0x89);
1450 emit_operand(src, dst);
1451 }
1452
1453
movw(const Operand & dst,Immediate imm)1454 void Assembler::movw(const Operand& dst, Immediate imm) {
1455 EnsureSpace ensure_space(this);
1456 emit(0x66);
1457 emit_optional_rex_32(dst);
1458 emit(0xC7);
1459 emit_operand(0x0, dst);
1460 emit(static_cast<byte>(imm.value_ & 0xff));
1461 emit(static_cast<byte>(imm.value_ >> 8));
1462 }
1463
1464
emit_mov(Register dst,const Operand & src,int size)1465 void Assembler::emit_mov(Register dst, const Operand& src, int size) {
1466 EnsureSpace ensure_space(this);
1467 emit_rex(dst, src, size);
1468 emit(0x8B);
1469 emit_operand(dst, src);
1470 }
1471
1472
emit_mov(Register dst,Register src,int size)1473 void Assembler::emit_mov(Register dst, Register src, int size) {
1474 EnsureSpace ensure_space(this);
1475 if (src.low_bits() == 4) {
1476 emit_rex(src, dst, size);
1477 emit(0x89);
1478 emit_modrm(src, dst);
1479 } else {
1480 emit_rex(dst, src, size);
1481 emit(0x8B);
1482 emit_modrm(dst, src);
1483 }
1484 }
1485
1486
emit_mov(const Operand & dst,Register src,int size)1487 void Assembler::emit_mov(const Operand& dst, Register src, int size) {
1488 EnsureSpace ensure_space(this);
1489 emit_rex(src, dst, size);
1490 emit(0x89);
1491 emit_operand(src, dst);
1492 }
1493
1494
emit_mov(Register dst,Immediate value,int size)1495 void Assembler::emit_mov(Register dst, Immediate value, int size) {
1496 EnsureSpace ensure_space(this);
1497 emit_rex(dst, size);
1498 if (size == kInt64Size) {
1499 emit(0xC7);
1500 emit_modrm(0x0, dst);
1501 } else {
1502 DCHECK(size == kInt32Size);
1503 emit(0xB8 + dst.low_bits());
1504 }
1505 emit(value);
1506 }
1507
1508
emit_mov(const Operand & dst,Immediate value,int size)1509 void Assembler::emit_mov(const Operand& dst, Immediate value, int size) {
1510 EnsureSpace ensure_space(this);
1511 emit_rex(dst, size);
1512 emit(0xC7);
1513 emit_operand(0x0, dst);
1514 emit(value);
1515 }
1516
1517
movp(Register dst,void * value,RelocInfo::Mode rmode)1518 void Assembler::movp(Register dst, void* value, RelocInfo::Mode rmode) {
1519 EnsureSpace ensure_space(this);
1520 emit_rex(dst, kPointerSize);
1521 emit(0xB8 | dst.low_bits());
1522 emitp(value, rmode);
1523 }
1524
movq(Register dst,int64_t value,RelocInfo::Mode rmode)1525 void Assembler::movq(Register dst, int64_t value, RelocInfo::Mode rmode) {
1526 EnsureSpace ensure_space(this);
1527 emit_rex_64(dst);
1528 emit(0xB8 | dst.low_bits());
1529 if (!RelocInfo::IsNone(rmode)) {
1530 RecordRelocInfo(rmode, value);
1531 }
1532 emitq(value);
1533 }
1534
movq(Register dst,uint64_t value,RelocInfo::Mode rmode)1535 void Assembler::movq(Register dst, uint64_t value, RelocInfo::Mode rmode) {
1536 movq(dst, static_cast<int64_t>(value), rmode);
1537 }
1538
1539 // Loads the ip-relative location of the src label into the target location
1540 // (as a 32-bit offset sign extended to 64-bit).
movl(const Operand & dst,Label * src)1541 void Assembler::movl(const Operand& dst, Label* src) {
1542 EnsureSpace ensure_space(this);
1543 emit_optional_rex_32(dst);
1544 emit(0xC7);
1545 emit_operand(0, dst);
1546 if (src->is_bound()) {
1547 int offset = src->pos() - pc_offset() - sizeof(int32_t);
1548 DCHECK(offset <= 0);
1549 emitl(offset);
1550 } else if (src->is_linked()) {
1551 emitl(src->pos());
1552 src->link_to(pc_offset() - sizeof(int32_t));
1553 } else {
1554 DCHECK(src->is_unused());
1555 int32_t current = pc_offset();
1556 emitl(current);
1557 src->link_to(current);
1558 }
1559 }
1560
1561
movsxbl(Register dst,Register src)1562 void Assembler::movsxbl(Register dst, Register src) {
1563 EnsureSpace ensure_space(this);
1564 if (!src.is_byte_register()) {
1565 // Register is not one of al, bl, cl, dl. Its encoding needs REX.
1566 emit_rex_32(dst, src);
1567 } else {
1568 emit_optional_rex_32(dst, src);
1569 }
1570 emit(0x0F);
1571 emit(0xBE);
1572 emit_modrm(dst, src);
1573 }
1574
1575
movsxbl(Register dst,const Operand & src)1576 void Assembler::movsxbl(Register dst, const Operand& src) {
1577 EnsureSpace ensure_space(this);
1578 emit_optional_rex_32(dst, src);
1579 emit(0x0F);
1580 emit(0xBE);
1581 emit_operand(dst, src);
1582 }
1583
1584
movsxbq(Register dst,const Operand & src)1585 void Assembler::movsxbq(Register dst, const Operand& src) {
1586 EnsureSpace ensure_space(this);
1587 emit_rex_64(dst, src);
1588 emit(0x0F);
1589 emit(0xBE);
1590 emit_operand(dst, src);
1591 }
1592
movsxbq(Register dst,Register src)1593 void Assembler::movsxbq(Register dst, Register src) {
1594 EnsureSpace ensure_space(this);
1595 emit_rex_64(dst, src);
1596 emit(0x0F);
1597 emit(0xBE);
1598 emit_modrm(dst, src);
1599 }
1600
movsxwl(Register dst,Register src)1601 void Assembler::movsxwl(Register dst, Register src) {
1602 EnsureSpace ensure_space(this);
1603 emit_optional_rex_32(dst, src);
1604 emit(0x0F);
1605 emit(0xBF);
1606 emit_modrm(dst, src);
1607 }
1608
1609
movsxwl(Register dst,const Operand & src)1610 void Assembler::movsxwl(Register dst, const Operand& src) {
1611 EnsureSpace ensure_space(this);
1612 emit_optional_rex_32(dst, src);
1613 emit(0x0F);
1614 emit(0xBF);
1615 emit_operand(dst, src);
1616 }
1617
1618
movsxwq(Register dst,const Operand & src)1619 void Assembler::movsxwq(Register dst, const Operand& src) {
1620 EnsureSpace ensure_space(this);
1621 emit_rex_64(dst, src);
1622 emit(0x0F);
1623 emit(0xBF);
1624 emit_operand(dst, src);
1625 }
1626
movsxwq(Register dst,Register src)1627 void Assembler::movsxwq(Register dst, Register src) {
1628 EnsureSpace ensure_space(this);
1629 emit_rex_64(dst, src);
1630 emit(0x0F);
1631 emit(0xBF);
1632 emit_modrm(dst, src);
1633 }
1634
movsxlq(Register dst,Register src)1635 void Assembler::movsxlq(Register dst, Register src) {
1636 EnsureSpace ensure_space(this);
1637 emit_rex_64(dst, src);
1638 emit(0x63);
1639 emit_modrm(dst, src);
1640 }
1641
1642
movsxlq(Register dst,const Operand & src)1643 void Assembler::movsxlq(Register dst, const Operand& src) {
1644 EnsureSpace ensure_space(this);
1645 emit_rex_64(dst, src);
1646 emit(0x63);
1647 emit_operand(dst, src);
1648 }
1649
1650
emit_movzxb(Register dst,const Operand & src,int size)1651 void Assembler::emit_movzxb(Register dst, const Operand& src, int size) {
1652 EnsureSpace ensure_space(this);
1653 // 32 bit operations zero the top 32 bits of 64 bit registers. Therefore
1654 // there is no need to make this a 64 bit operation.
1655 emit_optional_rex_32(dst, src);
1656 emit(0x0F);
1657 emit(0xB6);
1658 emit_operand(dst, src);
1659 }
1660
1661
emit_movzxb(Register dst,Register src,int size)1662 void Assembler::emit_movzxb(Register dst, Register src, int size) {
1663 EnsureSpace ensure_space(this);
1664 // 32 bit operations zero the top 32 bits of 64 bit registers. Therefore
1665 // there is no need to make this a 64 bit operation.
1666 if (!src.is_byte_register()) {
1667 // Register is not one of al, bl, cl, dl. Its encoding needs REX.
1668 emit_rex_32(dst, src);
1669 } else {
1670 emit_optional_rex_32(dst, src);
1671 }
1672 emit(0x0F);
1673 emit(0xB6);
1674 emit_modrm(dst, src);
1675 }
1676
1677
emit_movzxw(Register dst,const Operand & src,int size)1678 void Assembler::emit_movzxw(Register dst, const Operand& src, int size) {
1679 EnsureSpace ensure_space(this);
1680 // 32 bit operations zero the top 32 bits of 64 bit registers. Therefore
1681 // there is no need to make this a 64 bit operation.
1682 emit_optional_rex_32(dst, src);
1683 emit(0x0F);
1684 emit(0xB7);
1685 emit_operand(dst, src);
1686 }
1687
1688
emit_movzxw(Register dst,Register src,int size)1689 void Assembler::emit_movzxw(Register dst, Register src, int size) {
1690 EnsureSpace ensure_space(this);
1691 // 32 bit operations zero the top 32 bits of 64 bit registers. Therefore
1692 // there is no need to make this a 64 bit operation.
1693 emit_optional_rex_32(dst, src);
1694 emit(0x0F);
1695 emit(0xB7);
1696 emit_modrm(dst, src);
1697 }
1698
1699
repmovsb()1700 void Assembler::repmovsb() {
1701 EnsureSpace ensure_space(this);
1702 emit(0xF3);
1703 emit(0xA4);
1704 }
1705
1706
repmovsw()1707 void Assembler::repmovsw() {
1708 EnsureSpace ensure_space(this);
1709 emit(0x66); // Operand size override.
1710 emit(0xF3);
1711 emit(0xA4);
1712 }
1713
1714
emit_repmovs(int size)1715 void Assembler::emit_repmovs(int size) {
1716 EnsureSpace ensure_space(this);
1717 emit(0xF3);
1718 emit_rex(size);
1719 emit(0xA5);
1720 }
1721
1722
mull(Register src)1723 void Assembler::mull(Register src) {
1724 EnsureSpace ensure_space(this);
1725 emit_optional_rex_32(src);
1726 emit(0xF7);
1727 emit_modrm(0x4, src);
1728 }
1729
1730
mull(const Operand & src)1731 void Assembler::mull(const Operand& src) {
1732 EnsureSpace ensure_space(this);
1733 emit_optional_rex_32(src);
1734 emit(0xF7);
1735 emit_operand(0x4, src);
1736 }
1737
1738
mulq(Register src)1739 void Assembler::mulq(Register src) {
1740 EnsureSpace ensure_space(this);
1741 emit_rex_64(src);
1742 emit(0xF7);
1743 emit_modrm(0x4, src);
1744 }
1745
1746
emit_neg(Register dst,int size)1747 void Assembler::emit_neg(Register dst, int size) {
1748 EnsureSpace ensure_space(this);
1749 emit_rex(dst, size);
1750 emit(0xF7);
1751 emit_modrm(0x3, dst);
1752 }
1753
1754
emit_neg(const Operand & dst,int size)1755 void Assembler::emit_neg(const Operand& dst, int size) {
1756 EnsureSpace ensure_space(this);
1757 emit_rex_64(dst);
1758 emit(0xF7);
1759 emit_operand(3, dst);
1760 }
1761
1762
nop()1763 void Assembler::nop() {
1764 EnsureSpace ensure_space(this);
1765 emit(0x90);
1766 }
1767
1768
emit_not(Register dst,int size)1769 void Assembler::emit_not(Register dst, int size) {
1770 EnsureSpace ensure_space(this);
1771 emit_rex(dst, size);
1772 emit(0xF7);
1773 emit_modrm(0x2, dst);
1774 }
1775
1776
emit_not(const Operand & dst,int size)1777 void Assembler::emit_not(const Operand& dst, int size) {
1778 EnsureSpace ensure_space(this);
1779 emit_rex(dst, size);
1780 emit(0xF7);
1781 emit_operand(2, dst);
1782 }
1783
1784
Nop(int n)1785 void Assembler::Nop(int n) {
1786 // The recommended muti-byte sequences of NOP instructions from the Intel 64
1787 // and IA-32 Architectures Software Developer's Manual.
1788 //
1789 // Length Assembly Byte Sequence
1790 // 2 bytes 66 NOP 66 90H
1791 // 3 bytes NOP DWORD ptr [EAX] 0F 1F 00H
1792 // 4 bytes NOP DWORD ptr [EAX + 00H] 0F 1F 40 00H
1793 // 5 bytes NOP DWORD ptr [EAX + EAX*1 + 00H] 0F 1F 44 00 00H
1794 // 6 bytes 66 NOP DWORD ptr [EAX + EAX*1 + 00H] 66 0F 1F 44 00 00H
1795 // 7 bytes NOP DWORD ptr [EAX + 00000000H] 0F 1F 80 00 00 00 00H
1796 // 8 bytes NOP DWORD ptr [EAX + EAX*1 + 00000000H] 0F 1F 84 00 00 00 00 00H
1797 // 9 bytes 66 NOP DWORD ptr [EAX + EAX*1 + 66 0F 1F 84 00 00 00 00
1798 // 00000000H] 00H
1799
1800 EnsureSpace ensure_space(this);
1801 while (n > 0) {
1802 switch (n) {
1803 case 2:
1804 emit(0x66);
1805 case 1:
1806 emit(0x90);
1807 return;
1808 case 3:
1809 emit(0x0f);
1810 emit(0x1f);
1811 emit(0x00);
1812 return;
1813 case 4:
1814 emit(0x0f);
1815 emit(0x1f);
1816 emit(0x40);
1817 emit(0x00);
1818 return;
1819 case 6:
1820 emit(0x66);
1821 case 5:
1822 emit(0x0f);
1823 emit(0x1f);
1824 emit(0x44);
1825 emit(0x00);
1826 emit(0x00);
1827 return;
1828 case 7:
1829 emit(0x0f);
1830 emit(0x1f);
1831 emit(0x80);
1832 emit(0x00);
1833 emit(0x00);
1834 emit(0x00);
1835 emit(0x00);
1836 return;
1837 default:
1838 case 11:
1839 emit(0x66);
1840 n--;
1841 case 10:
1842 emit(0x66);
1843 n--;
1844 case 9:
1845 emit(0x66);
1846 n--;
1847 case 8:
1848 emit(0x0f);
1849 emit(0x1f);
1850 emit(0x84);
1851 emit(0x00);
1852 emit(0x00);
1853 emit(0x00);
1854 emit(0x00);
1855 emit(0x00);
1856 n -= 8;
1857 }
1858 }
1859 }
1860
1861
popq(Register dst)1862 void Assembler::popq(Register dst) {
1863 EnsureSpace ensure_space(this);
1864 emit_optional_rex_32(dst);
1865 emit(0x58 | dst.low_bits());
1866 }
1867
1868
popq(const Operand & dst)1869 void Assembler::popq(const Operand& dst) {
1870 EnsureSpace ensure_space(this);
1871 emit_optional_rex_32(dst);
1872 emit(0x8F);
1873 emit_operand(0, dst);
1874 }
1875
1876
popfq()1877 void Assembler::popfq() {
1878 EnsureSpace ensure_space(this);
1879 emit(0x9D);
1880 }
1881
1882
pushq(Register src)1883 void Assembler::pushq(Register src) {
1884 EnsureSpace ensure_space(this);
1885 emit_optional_rex_32(src);
1886 emit(0x50 | src.low_bits());
1887 }
1888
1889
pushq(const Operand & src)1890 void Assembler::pushq(const Operand& src) {
1891 EnsureSpace ensure_space(this);
1892 emit_optional_rex_32(src);
1893 emit(0xFF);
1894 emit_operand(6, src);
1895 }
1896
1897
pushq(Immediate value)1898 void Assembler::pushq(Immediate value) {
1899 EnsureSpace ensure_space(this);
1900 if (is_int8(value.value_)) {
1901 emit(0x6A);
1902 emit(value.value_); // Emit low byte of value.
1903 } else {
1904 emit(0x68);
1905 emitl(value.value_);
1906 }
1907 }
1908
1909
pushq_imm32(int32_t imm32)1910 void Assembler::pushq_imm32(int32_t imm32) {
1911 EnsureSpace ensure_space(this);
1912 emit(0x68);
1913 emitl(imm32);
1914 }
1915
1916
pushfq()1917 void Assembler::pushfq() {
1918 EnsureSpace ensure_space(this);
1919 emit(0x9C);
1920 }
1921
1922
ret(int imm16)1923 void Assembler::ret(int imm16) {
1924 EnsureSpace ensure_space(this);
1925 DCHECK(is_uint16(imm16));
1926 if (imm16 == 0) {
1927 emit(0xC3);
1928 } else {
1929 emit(0xC2);
1930 emit(imm16 & 0xFF);
1931 emit((imm16 >> 8) & 0xFF);
1932 }
1933 }
1934
1935
ud2()1936 void Assembler::ud2() {
1937 EnsureSpace ensure_space(this);
1938 emit(0x0F);
1939 emit(0x0B);
1940 }
1941
1942
setcc(Condition cc,Register reg)1943 void Assembler::setcc(Condition cc, Register reg) {
1944 if (cc > last_condition) {
1945 movb(reg, Immediate(cc == always ? 1 : 0));
1946 return;
1947 }
1948 EnsureSpace ensure_space(this);
1949 DCHECK(is_uint4(cc));
1950 if (!reg.is_byte_register()) {
1951 // Register is not one of al, bl, cl, dl. Its encoding needs REX.
1952 emit_rex_32(reg);
1953 }
1954 emit(0x0F);
1955 emit(0x90 | cc);
1956 emit_modrm(0x0, reg);
1957 }
1958
1959
shld(Register dst,Register src)1960 void Assembler::shld(Register dst, Register src) {
1961 EnsureSpace ensure_space(this);
1962 emit_rex_64(src, dst);
1963 emit(0x0F);
1964 emit(0xA5);
1965 emit_modrm(src, dst);
1966 }
1967
1968
shrd(Register dst,Register src)1969 void Assembler::shrd(Register dst, Register src) {
1970 EnsureSpace ensure_space(this);
1971 emit_rex_64(src, dst);
1972 emit(0x0F);
1973 emit(0xAD);
1974 emit_modrm(src, dst);
1975 }
1976
xchgb(Register reg,const Operand & op)1977 void Assembler::xchgb(Register reg, const Operand& op) {
1978 EnsureSpace ensure_space(this);
1979 if (!reg.is_byte_register()) {
1980 // Register is not one of al, bl, cl, dl. Its encoding needs REX.
1981 emit_rex_32(reg, op);
1982 } else {
1983 emit_optional_rex_32(reg, op);
1984 }
1985 emit(0x86);
1986 emit_operand(reg, op);
1987 }
1988
xchgw(Register reg,const Operand & op)1989 void Assembler::xchgw(Register reg, const Operand& op) {
1990 EnsureSpace ensure_space(this);
1991 emit(0x66);
1992 emit_optional_rex_32(reg, op);
1993 emit(0x87);
1994 emit_operand(reg, op);
1995 }
1996
emit_xchg(Register dst,Register src,int size)1997 void Assembler::emit_xchg(Register dst, Register src, int size) {
1998 EnsureSpace ensure_space(this);
1999 if (src.is(rax) || dst.is(rax)) { // Single-byte encoding
2000 Register other = src.is(rax) ? dst : src;
2001 emit_rex(other, size);
2002 emit(0x90 | other.low_bits());
2003 } else if (dst.low_bits() == 4) {
2004 emit_rex(dst, src, size);
2005 emit(0x87);
2006 emit_modrm(dst, src);
2007 } else {
2008 emit_rex(src, dst, size);
2009 emit(0x87);
2010 emit_modrm(src, dst);
2011 }
2012 }
2013
2014
emit_xchg(Register dst,const Operand & src,int size)2015 void Assembler::emit_xchg(Register dst, const Operand& src, int size) {
2016 EnsureSpace ensure_space(this);
2017 emit_rex(dst, src, size);
2018 emit(0x87);
2019 emit_operand(dst, src);
2020 }
2021
2022
store_rax(void * dst,RelocInfo::Mode mode)2023 void Assembler::store_rax(void* dst, RelocInfo::Mode mode) {
2024 EnsureSpace ensure_space(this);
2025 if (kPointerSize == kInt64Size) {
2026 emit(0x48); // REX.W
2027 emit(0xA3);
2028 emitp(dst, mode);
2029 } else {
2030 DCHECK(kPointerSize == kInt32Size);
2031 emit(0xA3);
2032 emitp(dst, mode);
2033 // In 64-bit mode, need to zero extend the operand to 8 bytes.
2034 // See 2.2.1.4 in Intel64 and IA32 Architectures Software
2035 // Developer's Manual Volume 2.
2036 emitl(0);
2037 }
2038 }
2039
2040
store_rax(ExternalReference ref)2041 void Assembler::store_rax(ExternalReference ref) {
2042 store_rax(ref.address(), RelocInfo::EXTERNAL_REFERENCE);
2043 }
2044
2045
testb(Register dst,Register src)2046 void Assembler::testb(Register dst, Register src) {
2047 EnsureSpace ensure_space(this);
2048 if (src.low_bits() == 4) {
2049 emit_rex_32(src, dst);
2050 emit(0x84);
2051 emit_modrm(src, dst);
2052 } else {
2053 if (!dst.is_byte_register() || !src.is_byte_register()) {
2054 // Register is not one of al, bl, cl, dl. Its encoding needs REX.
2055 emit_rex_32(dst, src);
2056 }
2057 emit(0x84);
2058 emit_modrm(dst, src);
2059 }
2060 }
2061
2062
testb(Register reg,Immediate mask)2063 void Assembler::testb(Register reg, Immediate mask) {
2064 DCHECK(is_int8(mask.value_) || is_uint8(mask.value_));
2065 EnsureSpace ensure_space(this);
2066 if (reg.is(rax)) {
2067 emit(0xA8);
2068 emit(mask.value_); // Low byte emitted.
2069 } else {
2070 if (!reg.is_byte_register()) {
2071 // Register is not one of al, bl, cl, dl. Its encoding needs REX.
2072 emit_rex_32(reg);
2073 }
2074 emit(0xF6);
2075 emit_modrm(0x0, reg);
2076 emit(mask.value_); // Low byte emitted.
2077 }
2078 }
2079
2080
testb(const Operand & op,Immediate mask)2081 void Assembler::testb(const Operand& op, Immediate mask) {
2082 DCHECK(is_int8(mask.value_) || is_uint8(mask.value_));
2083 EnsureSpace ensure_space(this);
2084 emit_optional_rex_32(rax, op);
2085 emit(0xF6);
2086 emit_operand(rax, op); // Operation code 0
2087 emit(mask.value_); // Low byte emitted.
2088 }
2089
2090
testb(const Operand & op,Register reg)2091 void Assembler::testb(const Operand& op, Register reg) {
2092 EnsureSpace ensure_space(this);
2093 if (!reg.is_byte_register()) {
2094 // Register is not one of al, bl, cl, dl. Its encoding needs REX.
2095 emit_rex_32(reg, op);
2096 } else {
2097 emit_optional_rex_32(reg, op);
2098 }
2099 emit(0x84);
2100 emit_operand(reg, op);
2101 }
2102
testw(Register dst,Register src)2103 void Assembler::testw(Register dst, Register src) {
2104 EnsureSpace ensure_space(this);
2105 emit(0x66);
2106 if (src.low_bits() == 4) {
2107 emit_rex_32(src, dst);
2108 }
2109 emit(0x85);
2110 emit_modrm(src, dst);
2111 }
2112
testw(Register reg,Immediate mask)2113 void Assembler::testw(Register reg, Immediate mask) {
2114 DCHECK(is_int16(mask.value_) || is_uint16(mask.value_));
2115 EnsureSpace ensure_space(this);
2116 emit(0x66);
2117 if (reg.is(rax)) {
2118 emit(0xA9);
2119 emitw(mask.value_);
2120 } else {
2121 if (reg.low_bits() == 4) {
2122 emit_rex_32(reg);
2123 }
2124 emit(0xF7);
2125 emit_modrm(0x0, reg);
2126 emitw(mask.value_);
2127 }
2128 }
2129
testw(const Operand & op,Immediate mask)2130 void Assembler::testw(const Operand& op, Immediate mask) {
2131 DCHECK(is_int16(mask.value_) || is_uint16(mask.value_));
2132 EnsureSpace ensure_space(this);
2133 emit(0x66);
2134 emit_optional_rex_32(rax, op);
2135 emit(0xF7);
2136 emit_operand(rax, op);
2137 emitw(mask.value_);
2138 }
2139
testw(const Operand & op,Register reg)2140 void Assembler::testw(const Operand& op, Register reg) {
2141 EnsureSpace ensure_space(this);
2142 emit(0x66);
2143 emit_optional_rex_32(reg, op);
2144 emit(0x85);
2145 emit_operand(rax, op);
2146 }
2147
emit_test(Register dst,Register src,int size)2148 void Assembler::emit_test(Register dst, Register src, int size) {
2149 EnsureSpace ensure_space(this);
2150 if (src.low_bits() == 4) {
2151 emit_rex(src, dst, size);
2152 emit(0x85);
2153 emit_modrm(src, dst);
2154 } else {
2155 emit_rex(dst, src, size);
2156 emit(0x85);
2157 emit_modrm(dst, src);
2158 }
2159 }
2160
2161
emit_test(Register reg,Immediate mask,int size)2162 void Assembler::emit_test(Register reg, Immediate mask, int size) {
2163 // testl with a mask that fits in the low byte is exactly testb.
2164 if (is_uint8(mask.value_)) {
2165 testb(reg, mask);
2166 return;
2167 }
2168 EnsureSpace ensure_space(this);
2169 if (reg.is(rax)) {
2170 emit_rex(rax, size);
2171 emit(0xA9);
2172 emit(mask);
2173 } else {
2174 emit_rex(reg, size);
2175 emit(0xF7);
2176 emit_modrm(0x0, reg);
2177 emit(mask);
2178 }
2179 }
2180
2181
emit_test(const Operand & op,Immediate mask,int size)2182 void Assembler::emit_test(const Operand& op, Immediate mask, int size) {
2183 // testl with a mask that fits in the low byte is exactly testb.
2184 if (is_uint8(mask.value_)) {
2185 testb(op, mask);
2186 return;
2187 }
2188 EnsureSpace ensure_space(this);
2189 emit_rex(rax, op, size);
2190 emit(0xF7);
2191 emit_operand(rax, op); // Operation code 0
2192 emit(mask);
2193 }
2194
2195
emit_test(const Operand & op,Register reg,int size)2196 void Assembler::emit_test(const Operand& op, Register reg, int size) {
2197 EnsureSpace ensure_space(this);
2198 emit_rex(reg, op, size);
2199 emit(0x85);
2200 emit_operand(reg, op);
2201 }
2202
2203
2204 // FPU instructions.
2205
2206
fld(int i)2207 void Assembler::fld(int i) {
2208 EnsureSpace ensure_space(this);
2209 emit_farith(0xD9, 0xC0, i);
2210 }
2211
2212
fld1()2213 void Assembler::fld1() {
2214 EnsureSpace ensure_space(this);
2215 emit(0xD9);
2216 emit(0xE8);
2217 }
2218
2219
fldz()2220 void Assembler::fldz() {
2221 EnsureSpace ensure_space(this);
2222 emit(0xD9);
2223 emit(0xEE);
2224 }
2225
2226
fldpi()2227 void Assembler::fldpi() {
2228 EnsureSpace ensure_space(this);
2229 emit(0xD9);
2230 emit(0xEB);
2231 }
2232
2233
fldln2()2234 void Assembler::fldln2() {
2235 EnsureSpace ensure_space(this);
2236 emit(0xD9);
2237 emit(0xED);
2238 }
2239
2240
fld_s(const Operand & adr)2241 void Assembler::fld_s(const Operand& adr) {
2242 EnsureSpace ensure_space(this);
2243 emit_optional_rex_32(adr);
2244 emit(0xD9);
2245 emit_operand(0, adr);
2246 }
2247
2248
fld_d(const Operand & adr)2249 void Assembler::fld_d(const Operand& adr) {
2250 EnsureSpace ensure_space(this);
2251 emit_optional_rex_32(adr);
2252 emit(0xDD);
2253 emit_operand(0, adr);
2254 }
2255
2256
fstp_s(const Operand & adr)2257 void Assembler::fstp_s(const Operand& adr) {
2258 EnsureSpace ensure_space(this);
2259 emit_optional_rex_32(adr);
2260 emit(0xD9);
2261 emit_operand(3, adr);
2262 }
2263
2264
fstp_d(const Operand & adr)2265 void Assembler::fstp_d(const Operand& adr) {
2266 EnsureSpace ensure_space(this);
2267 emit_optional_rex_32(adr);
2268 emit(0xDD);
2269 emit_operand(3, adr);
2270 }
2271
2272
fstp(int index)2273 void Assembler::fstp(int index) {
2274 DCHECK(is_uint3(index));
2275 EnsureSpace ensure_space(this);
2276 emit_farith(0xDD, 0xD8, index);
2277 }
2278
2279
fild_s(const Operand & adr)2280 void Assembler::fild_s(const Operand& adr) {
2281 EnsureSpace ensure_space(this);
2282 emit_optional_rex_32(adr);
2283 emit(0xDB);
2284 emit_operand(0, adr);
2285 }
2286
2287
fild_d(const Operand & adr)2288 void Assembler::fild_d(const Operand& adr) {
2289 EnsureSpace ensure_space(this);
2290 emit_optional_rex_32(adr);
2291 emit(0xDF);
2292 emit_operand(5, adr);
2293 }
2294
2295
fistp_s(const Operand & adr)2296 void Assembler::fistp_s(const Operand& adr) {
2297 EnsureSpace ensure_space(this);
2298 emit_optional_rex_32(adr);
2299 emit(0xDB);
2300 emit_operand(3, adr);
2301 }
2302
2303
fisttp_s(const Operand & adr)2304 void Assembler::fisttp_s(const Operand& adr) {
2305 DCHECK(IsEnabled(SSE3));
2306 EnsureSpace ensure_space(this);
2307 emit_optional_rex_32(adr);
2308 emit(0xDB);
2309 emit_operand(1, adr);
2310 }
2311
2312
fisttp_d(const Operand & adr)2313 void Assembler::fisttp_d(const Operand& adr) {
2314 DCHECK(IsEnabled(SSE3));
2315 EnsureSpace ensure_space(this);
2316 emit_optional_rex_32(adr);
2317 emit(0xDD);
2318 emit_operand(1, adr);
2319 }
2320
2321
fist_s(const Operand & adr)2322 void Assembler::fist_s(const Operand& adr) {
2323 EnsureSpace ensure_space(this);
2324 emit_optional_rex_32(adr);
2325 emit(0xDB);
2326 emit_operand(2, adr);
2327 }
2328
2329
fistp_d(const Operand & adr)2330 void Assembler::fistp_d(const Operand& adr) {
2331 EnsureSpace ensure_space(this);
2332 emit_optional_rex_32(adr);
2333 emit(0xDF);
2334 emit_operand(7, adr);
2335 }
2336
2337
fabs()2338 void Assembler::fabs() {
2339 EnsureSpace ensure_space(this);
2340 emit(0xD9);
2341 emit(0xE1);
2342 }
2343
2344
fchs()2345 void Assembler::fchs() {
2346 EnsureSpace ensure_space(this);
2347 emit(0xD9);
2348 emit(0xE0);
2349 }
2350
2351
fcos()2352 void Assembler::fcos() {
2353 EnsureSpace ensure_space(this);
2354 emit(0xD9);
2355 emit(0xFF);
2356 }
2357
2358
fsin()2359 void Assembler::fsin() {
2360 EnsureSpace ensure_space(this);
2361 emit(0xD9);
2362 emit(0xFE);
2363 }
2364
2365
fptan()2366 void Assembler::fptan() {
2367 EnsureSpace ensure_space(this);
2368 emit(0xD9);
2369 emit(0xF2);
2370 }
2371
2372
fyl2x()2373 void Assembler::fyl2x() {
2374 EnsureSpace ensure_space(this);
2375 emit(0xD9);
2376 emit(0xF1);
2377 }
2378
2379
f2xm1()2380 void Assembler::f2xm1() {
2381 EnsureSpace ensure_space(this);
2382 emit(0xD9);
2383 emit(0xF0);
2384 }
2385
2386
fscale()2387 void Assembler::fscale() {
2388 EnsureSpace ensure_space(this);
2389 emit(0xD9);
2390 emit(0xFD);
2391 }
2392
2393
fninit()2394 void Assembler::fninit() {
2395 EnsureSpace ensure_space(this);
2396 emit(0xDB);
2397 emit(0xE3);
2398 }
2399
2400
fadd(int i)2401 void Assembler::fadd(int i) {
2402 EnsureSpace ensure_space(this);
2403 emit_farith(0xDC, 0xC0, i);
2404 }
2405
2406
fsub(int i)2407 void Assembler::fsub(int i) {
2408 EnsureSpace ensure_space(this);
2409 emit_farith(0xDC, 0xE8, i);
2410 }
2411
2412
fisub_s(const Operand & adr)2413 void Assembler::fisub_s(const Operand& adr) {
2414 EnsureSpace ensure_space(this);
2415 emit_optional_rex_32(adr);
2416 emit(0xDA);
2417 emit_operand(4, adr);
2418 }
2419
2420
fmul(int i)2421 void Assembler::fmul(int i) {
2422 EnsureSpace ensure_space(this);
2423 emit_farith(0xDC, 0xC8, i);
2424 }
2425
2426
fdiv(int i)2427 void Assembler::fdiv(int i) {
2428 EnsureSpace ensure_space(this);
2429 emit_farith(0xDC, 0xF8, i);
2430 }
2431
2432
faddp(int i)2433 void Assembler::faddp(int i) {
2434 EnsureSpace ensure_space(this);
2435 emit_farith(0xDE, 0xC0, i);
2436 }
2437
2438
fsubp(int i)2439 void Assembler::fsubp(int i) {
2440 EnsureSpace ensure_space(this);
2441 emit_farith(0xDE, 0xE8, i);
2442 }
2443
2444
fsubrp(int i)2445 void Assembler::fsubrp(int i) {
2446 EnsureSpace ensure_space(this);
2447 emit_farith(0xDE, 0xE0, i);
2448 }
2449
2450
fmulp(int i)2451 void Assembler::fmulp(int i) {
2452 EnsureSpace ensure_space(this);
2453 emit_farith(0xDE, 0xC8, i);
2454 }
2455
2456
fdivp(int i)2457 void Assembler::fdivp(int i) {
2458 EnsureSpace ensure_space(this);
2459 emit_farith(0xDE, 0xF8, i);
2460 }
2461
2462
fprem()2463 void Assembler::fprem() {
2464 EnsureSpace ensure_space(this);
2465 emit(0xD9);
2466 emit(0xF8);
2467 }
2468
2469
fprem1()2470 void Assembler::fprem1() {
2471 EnsureSpace ensure_space(this);
2472 emit(0xD9);
2473 emit(0xF5);
2474 }
2475
2476
fxch(int i)2477 void Assembler::fxch(int i) {
2478 EnsureSpace ensure_space(this);
2479 emit_farith(0xD9, 0xC8, i);
2480 }
2481
2482
fincstp()2483 void Assembler::fincstp() {
2484 EnsureSpace ensure_space(this);
2485 emit(0xD9);
2486 emit(0xF7);
2487 }
2488
2489
ffree(int i)2490 void Assembler::ffree(int i) {
2491 EnsureSpace ensure_space(this);
2492 emit_farith(0xDD, 0xC0, i);
2493 }
2494
2495
ftst()2496 void Assembler::ftst() {
2497 EnsureSpace ensure_space(this);
2498 emit(0xD9);
2499 emit(0xE4);
2500 }
2501
2502
fucomp(int i)2503 void Assembler::fucomp(int i) {
2504 EnsureSpace ensure_space(this);
2505 emit_farith(0xDD, 0xE8, i);
2506 }
2507
2508
fucompp()2509 void Assembler::fucompp() {
2510 EnsureSpace ensure_space(this);
2511 emit(0xDA);
2512 emit(0xE9);
2513 }
2514
2515
fucomi(int i)2516 void Assembler::fucomi(int i) {
2517 EnsureSpace ensure_space(this);
2518 emit(0xDB);
2519 emit(0xE8 + i);
2520 }
2521
2522
fucomip()2523 void Assembler::fucomip() {
2524 EnsureSpace ensure_space(this);
2525 emit(0xDF);
2526 emit(0xE9);
2527 }
2528
2529
fcompp()2530 void Assembler::fcompp() {
2531 EnsureSpace ensure_space(this);
2532 emit(0xDE);
2533 emit(0xD9);
2534 }
2535
2536
fnstsw_ax()2537 void Assembler::fnstsw_ax() {
2538 EnsureSpace ensure_space(this);
2539 emit(0xDF);
2540 emit(0xE0);
2541 }
2542
2543
fwait()2544 void Assembler::fwait() {
2545 EnsureSpace ensure_space(this);
2546 emit(0x9B);
2547 }
2548
2549
frndint()2550 void Assembler::frndint() {
2551 EnsureSpace ensure_space(this);
2552 emit(0xD9);
2553 emit(0xFC);
2554 }
2555
2556
fnclex()2557 void Assembler::fnclex() {
2558 EnsureSpace ensure_space(this);
2559 emit(0xDB);
2560 emit(0xE2);
2561 }
2562
2563
sahf()2564 void Assembler::sahf() {
2565 // TODO(X64): Test for presence. Not all 64-bit intel CPU's have sahf
2566 // in 64-bit mode. Test CpuID.
2567 DCHECK(IsEnabled(SAHF));
2568 EnsureSpace ensure_space(this);
2569 emit(0x9E);
2570 }
2571
2572
emit_farith(int b1,int b2,int i)2573 void Assembler::emit_farith(int b1, int b2, int i) {
2574 DCHECK(is_uint8(b1) && is_uint8(b2)); // wrong opcode
2575 DCHECK(is_uint3(i)); // illegal stack offset
2576 emit(b1);
2577 emit(b2 + i);
2578 }
2579
2580
2581 // SSE operations.
2582
andps(XMMRegister dst,XMMRegister src)2583 void Assembler::andps(XMMRegister dst, XMMRegister src) {
2584 EnsureSpace ensure_space(this);
2585 emit_optional_rex_32(dst, src);
2586 emit(0x0F);
2587 emit(0x54);
2588 emit_sse_operand(dst, src);
2589 }
2590
2591
andps(XMMRegister dst,const Operand & src)2592 void Assembler::andps(XMMRegister dst, const Operand& src) {
2593 EnsureSpace ensure_space(this);
2594 emit_optional_rex_32(dst, src);
2595 emit(0x0F);
2596 emit(0x54);
2597 emit_sse_operand(dst, src);
2598 }
2599
2600
orps(XMMRegister dst,XMMRegister src)2601 void Assembler::orps(XMMRegister dst, XMMRegister src) {
2602 EnsureSpace ensure_space(this);
2603 emit_optional_rex_32(dst, src);
2604 emit(0x0F);
2605 emit(0x56);
2606 emit_sse_operand(dst, src);
2607 }
2608
2609
orps(XMMRegister dst,const Operand & src)2610 void Assembler::orps(XMMRegister dst, const Operand& src) {
2611 EnsureSpace ensure_space(this);
2612 emit_optional_rex_32(dst, src);
2613 emit(0x0F);
2614 emit(0x56);
2615 emit_sse_operand(dst, src);
2616 }
2617
2618
xorps(XMMRegister dst,XMMRegister src)2619 void Assembler::xorps(XMMRegister dst, XMMRegister src) {
2620 DCHECK(!IsEnabled(AVX));
2621 EnsureSpace ensure_space(this);
2622 emit_optional_rex_32(dst, src);
2623 emit(0x0F);
2624 emit(0x57);
2625 emit_sse_operand(dst, src);
2626 }
2627
2628
xorps(XMMRegister dst,const Operand & src)2629 void Assembler::xorps(XMMRegister dst, const Operand& src) {
2630 DCHECK(!IsEnabled(AVX));
2631 EnsureSpace ensure_space(this);
2632 emit_optional_rex_32(dst, src);
2633 emit(0x0F);
2634 emit(0x57);
2635 emit_sse_operand(dst, src);
2636 }
2637
2638
addps(XMMRegister dst,XMMRegister src)2639 void Assembler::addps(XMMRegister dst, XMMRegister src) {
2640 EnsureSpace ensure_space(this);
2641 emit_optional_rex_32(dst, src);
2642 emit(0x0F);
2643 emit(0x58);
2644 emit_sse_operand(dst, src);
2645 }
2646
2647
addps(XMMRegister dst,const Operand & src)2648 void Assembler::addps(XMMRegister dst, const Operand& src) {
2649 EnsureSpace ensure_space(this);
2650 emit_optional_rex_32(dst, src);
2651 emit(0x0F);
2652 emit(0x58);
2653 emit_sse_operand(dst, src);
2654 }
2655
2656
subps(XMMRegister dst,XMMRegister src)2657 void Assembler::subps(XMMRegister dst, XMMRegister src) {
2658 EnsureSpace ensure_space(this);
2659 emit_optional_rex_32(dst, src);
2660 emit(0x0F);
2661 emit(0x5C);
2662 emit_sse_operand(dst, src);
2663 }
2664
2665
subps(XMMRegister dst,const Operand & src)2666 void Assembler::subps(XMMRegister dst, const Operand& src) {
2667 EnsureSpace ensure_space(this);
2668 emit_optional_rex_32(dst, src);
2669 emit(0x0F);
2670 emit(0x5C);
2671 emit_sse_operand(dst, src);
2672 }
2673
2674
mulps(XMMRegister dst,XMMRegister src)2675 void Assembler::mulps(XMMRegister dst, XMMRegister src) {
2676 EnsureSpace ensure_space(this);
2677 emit_optional_rex_32(dst, src);
2678 emit(0x0F);
2679 emit(0x59);
2680 emit_sse_operand(dst, src);
2681 }
2682
2683
mulps(XMMRegister dst,const Operand & src)2684 void Assembler::mulps(XMMRegister dst, const Operand& src) {
2685 EnsureSpace ensure_space(this);
2686 emit_optional_rex_32(dst, src);
2687 emit(0x0F);
2688 emit(0x59);
2689 emit_sse_operand(dst, src);
2690 }
2691
2692
divps(XMMRegister dst,XMMRegister src)2693 void Assembler::divps(XMMRegister dst, XMMRegister src) {
2694 EnsureSpace ensure_space(this);
2695 emit_optional_rex_32(dst, src);
2696 emit(0x0F);
2697 emit(0x5E);
2698 emit_sse_operand(dst, src);
2699 }
2700
2701
divps(XMMRegister dst,const Operand & src)2702 void Assembler::divps(XMMRegister dst, const Operand& src) {
2703 EnsureSpace ensure_space(this);
2704 emit_optional_rex_32(dst, src);
2705 emit(0x0F);
2706 emit(0x5E);
2707 emit_sse_operand(dst, src);
2708 }
2709
2710
2711 // SSE 2 operations.
2712
movd(XMMRegister dst,Register src)2713 void Assembler::movd(XMMRegister dst, Register src) {
2714 DCHECK(!IsEnabled(AVX));
2715 EnsureSpace ensure_space(this);
2716 emit(0x66);
2717 emit_optional_rex_32(dst, src);
2718 emit(0x0F);
2719 emit(0x6E);
2720 emit_sse_operand(dst, src);
2721 }
2722
2723
movd(XMMRegister dst,const Operand & src)2724 void Assembler::movd(XMMRegister dst, const Operand& src) {
2725 DCHECK(!IsEnabled(AVX));
2726 EnsureSpace ensure_space(this);
2727 emit(0x66);
2728 emit_optional_rex_32(dst, src);
2729 emit(0x0F);
2730 emit(0x6E);
2731 emit_sse_operand(dst, src);
2732 }
2733
2734
movd(Register dst,XMMRegister src)2735 void Assembler::movd(Register dst, XMMRegister src) {
2736 DCHECK(!IsEnabled(AVX));
2737 EnsureSpace ensure_space(this);
2738 emit(0x66);
2739 emit_optional_rex_32(src, dst);
2740 emit(0x0F);
2741 emit(0x7E);
2742 emit_sse_operand(src, dst);
2743 }
2744
2745
movq(XMMRegister dst,Register src)2746 void Assembler::movq(XMMRegister dst, Register src) {
2747 DCHECK(!IsEnabled(AVX));
2748 EnsureSpace ensure_space(this);
2749 emit(0x66);
2750 emit_rex_64(dst, src);
2751 emit(0x0F);
2752 emit(0x6E);
2753 emit_sse_operand(dst, src);
2754 }
2755
2756
movq(Register dst,XMMRegister src)2757 void Assembler::movq(Register dst, XMMRegister src) {
2758 DCHECK(!IsEnabled(AVX));
2759 EnsureSpace ensure_space(this);
2760 emit(0x66);
2761 emit_rex_64(src, dst);
2762 emit(0x0F);
2763 emit(0x7E);
2764 emit_sse_operand(src, dst);
2765 }
2766
2767
movq(XMMRegister dst,XMMRegister src)2768 void Assembler::movq(XMMRegister dst, XMMRegister src) {
2769 DCHECK(!IsEnabled(AVX));
2770 EnsureSpace ensure_space(this);
2771 if (dst.low_bits() == 4) {
2772 // Avoid unnecessary SIB byte.
2773 emit(0xf3);
2774 emit_optional_rex_32(dst, src);
2775 emit(0x0F);
2776 emit(0x7e);
2777 emit_sse_operand(dst, src);
2778 } else {
2779 emit(0x66);
2780 emit_optional_rex_32(src, dst);
2781 emit(0x0F);
2782 emit(0xD6);
2783 emit_sse_operand(src, dst);
2784 }
2785 }
2786
2787
movdqa(const Operand & dst,XMMRegister src)2788 void Assembler::movdqa(const Operand& dst, XMMRegister src) {
2789 EnsureSpace ensure_space(this);
2790 emit(0x66);
2791 emit_rex_64(src, dst);
2792 emit(0x0F);
2793 emit(0x7F);
2794 emit_sse_operand(src, dst);
2795 }
2796
2797
movdqa(XMMRegister dst,const Operand & src)2798 void Assembler::movdqa(XMMRegister dst, const Operand& src) {
2799 EnsureSpace ensure_space(this);
2800 emit(0x66);
2801 emit_rex_64(dst, src);
2802 emit(0x0F);
2803 emit(0x6F);
2804 emit_sse_operand(dst, src);
2805 }
2806
2807
movdqu(const Operand & dst,XMMRegister src)2808 void Assembler::movdqu(const Operand& dst, XMMRegister src) {
2809 EnsureSpace ensure_space(this);
2810 emit(0xF3);
2811 emit_rex_64(src, dst);
2812 emit(0x0F);
2813 emit(0x7F);
2814 emit_sse_operand(src, dst);
2815 }
2816
2817
movdqu(XMMRegister dst,const Operand & src)2818 void Assembler::movdqu(XMMRegister dst, const Operand& src) {
2819 EnsureSpace ensure_space(this);
2820 emit(0xF3);
2821 emit_rex_64(dst, src);
2822 emit(0x0F);
2823 emit(0x6F);
2824 emit_sse_operand(dst, src);
2825 }
2826
2827
extractps(Register dst,XMMRegister src,byte imm8)2828 void Assembler::extractps(Register dst, XMMRegister src, byte imm8) {
2829 DCHECK(IsEnabled(SSE4_1));
2830 DCHECK(is_uint8(imm8));
2831 EnsureSpace ensure_space(this);
2832 emit(0x66);
2833 emit_optional_rex_32(src, dst);
2834 emit(0x0F);
2835 emit(0x3A);
2836 emit(0x17);
2837 emit_sse_operand(src, dst);
2838 emit(imm8);
2839 }
2840
pextrb(Register dst,XMMRegister src,int8_t imm8)2841 void Assembler::pextrb(Register dst, XMMRegister src, int8_t imm8) {
2842 DCHECK(IsEnabled(SSE4_1));
2843 DCHECK(is_uint8(imm8));
2844 EnsureSpace ensure_space(this);
2845 emit(0x66);
2846 emit_optional_rex_32(src, dst);
2847 emit(0x0F);
2848 emit(0x3A);
2849 emit(0x14);
2850 emit_sse_operand(src, dst);
2851 emit(imm8);
2852 }
2853
pextrb(const Operand & dst,XMMRegister src,int8_t imm8)2854 void Assembler::pextrb(const Operand& dst, XMMRegister src, int8_t imm8) {
2855 DCHECK(IsEnabled(SSE4_1));
2856 DCHECK(is_uint8(imm8));
2857 EnsureSpace ensure_space(this);
2858 emit(0x66);
2859 emit_optional_rex_32(src, dst);
2860 emit(0x0F);
2861 emit(0x3A);
2862 emit(0x14);
2863 emit_sse_operand(src, dst);
2864 emit(imm8);
2865 }
2866
pinsrw(XMMRegister dst,Register src,int8_t imm8)2867 void Assembler::pinsrw(XMMRegister dst, Register src, int8_t imm8) {
2868 DCHECK(is_uint8(imm8));
2869 EnsureSpace ensure_space(this);
2870 emit(0x66);
2871 emit_optional_rex_32(dst, src);
2872 emit(0x0F);
2873 emit(0xC4);
2874 emit_sse_operand(dst, src);
2875 emit(imm8);
2876 }
2877
pinsrw(XMMRegister dst,const Operand & src,int8_t imm8)2878 void Assembler::pinsrw(XMMRegister dst, const Operand& src, int8_t imm8) {
2879 DCHECK(is_uint8(imm8));
2880 EnsureSpace ensure_space(this);
2881 emit(0x66);
2882 emit_optional_rex_32(dst, src);
2883 emit(0x0F);
2884 emit(0xC4);
2885 emit_sse_operand(dst, src);
2886 emit(imm8);
2887 }
2888
pextrw(Register dst,XMMRegister src,int8_t imm8)2889 void Assembler::pextrw(Register dst, XMMRegister src, int8_t imm8) {
2890 DCHECK(is_uint8(imm8));
2891 EnsureSpace ensure_space(this);
2892 emit(0x66);
2893 emit_optional_rex_32(src, dst);
2894 emit(0x0F);
2895 emit(0xC5);
2896 emit_sse_operand(src, dst);
2897 emit(imm8);
2898 }
2899
pextrw(const Operand & dst,XMMRegister src,int8_t imm8)2900 void Assembler::pextrw(const Operand& dst, XMMRegister src, int8_t imm8) {
2901 DCHECK(IsEnabled(SSE4_1));
2902 DCHECK(is_uint8(imm8));
2903 EnsureSpace ensure_space(this);
2904 emit(0x66);
2905 emit_optional_rex_32(src, dst);
2906 emit(0x0F);
2907 emit(0x3A);
2908 emit(0x15);
2909 emit_sse_operand(src, dst);
2910 emit(imm8);
2911 }
2912
pextrd(Register dst,XMMRegister src,int8_t imm8)2913 void Assembler::pextrd(Register dst, XMMRegister src, int8_t imm8) {
2914 DCHECK(IsEnabled(SSE4_1));
2915 EnsureSpace ensure_space(this);
2916 emit(0x66);
2917 emit_optional_rex_32(src, dst);
2918 emit(0x0F);
2919 emit(0x3A);
2920 emit(0x16);
2921 emit_sse_operand(src, dst);
2922 emit(imm8);
2923 }
2924
pextrd(const Operand & dst,XMMRegister src,int8_t imm8)2925 void Assembler::pextrd(const Operand& dst, XMMRegister src, int8_t imm8) {
2926 DCHECK(IsEnabled(SSE4_1));
2927 EnsureSpace ensure_space(this);
2928 emit(0x66);
2929 emit_optional_rex_32(src, dst);
2930 emit(0x0F);
2931 emit(0x3A);
2932 emit(0x16);
2933 emit_sse_operand(src, dst);
2934 emit(imm8);
2935 }
2936
pinsrd(XMMRegister dst,Register src,int8_t imm8)2937 void Assembler::pinsrd(XMMRegister dst, Register src, int8_t imm8) {
2938 DCHECK(IsEnabled(SSE4_1));
2939 EnsureSpace ensure_space(this);
2940 emit(0x66);
2941 emit_optional_rex_32(dst, src);
2942 emit(0x0F);
2943 emit(0x3A);
2944 emit(0x22);
2945 emit_sse_operand(dst, src);
2946 emit(imm8);
2947 }
2948
2949
pinsrd(XMMRegister dst,const Operand & src,int8_t imm8)2950 void Assembler::pinsrd(XMMRegister dst, const Operand& src, int8_t imm8) {
2951 DCHECK(IsEnabled(SSE4_1));
2952 EnsureSpace ensure_space(this);
2953 emit(0x66);
2954 emit_optional_rex_32(dst, src);
2955 emit(0x0F);
2956 emit(0x3A);
2957 emit(0x22);
2958 emit_sse_operand(dst, src);
2959 emit(imm8);
2960 }
2961
pinsrb(XMMRegister dst,Register src,int8_t imm8)2962 void Assembler::pinsrb(XMMRegister dst, Register src, int8_t imm8) {
2963 DCHECK(IsEnabled(SSE4_1));
2964 EnsureSpace ensure_space(this);
2965 emit(0x66);
2966 emit_optional_rex_32(dst, src);
2967 emit(0x0F);
2968 emit(0x3A);
2969 emit(0x20);
2970 emit_sse_operand(dst, src);
2971 emit(imm8);
2972 }
2973
pinsrb(XMMRegister dst,const Operand & src,int8_t imm8)2974 void Assembler::pinsrb(XMMRegister dst, const Operand& src, int8_t imm8) {
2975 DCHECK(IsEnabled(SSE4_1));
2976 EnsureSpace ensure_space(this);
2977 emit(0x66);
2978 emit_optional_rex_32(dst, src);
2979 emit(0x0F);
2980 emit(0x3A);
2981 emit(0x20);
2982 emit_sse_operand(dst, src);
2983 emit(imm8);
2984 }
2985
insertps(XMMRegister dst,XMMRegister src,byte imm8)2986 void Assembler::insertps(XMMRegister dst, XMMRegister src, byte imm8) {
2987 DCHECK(CpuFeatures::IsSupported(SSE4_1));
2988 DCHECK(is_uint8(imm8));
2989 EnsureSpace ensure_space(this);
2990 emit(0x66);
2991 emit_optional_rex_32(dst, src);
2992 emit(0x0F);
2993 emit(0x3A);
2994 emit(0x21);
2995 emit_sse_operand(dst, src);
2996 emit(imm8);
2997 }
2998
movsd(const Operand & dst,XMMRegister src)2999 void Assembler::movsd(const Operand& dst, XMMRegister src) {
3000 DCHECK(!IsEnabled(AVX));
3001 EnsureSpace ensure_space(this);
3002 emit(0xF2); // double
3003 emit_optional_rex_32(src, dst);
3004 emit(0x0F);
3005 emit(0x11); // store
3006 emit_sse_operand(src, dst);
3007 }
3008
3009
movsd(XMMRegister dst,XMMRegister src)3010 void Assembler::movsd(XMMRegister dst, XMMRegister src) {
3011 DCHECK(!IsEnabled(AVX));
3012 EnsureSpace ensure_space(this);
3013 emit(0xF2); // double
3014 emit_optional_rex_32(dst, src);
3015 emit(0x0F);
3016 emit(0x10); // load
3017 emit_sse_operand(dst, src);
3018 }
3019
3020
movsd(XMMRegister dst,const Operand & src)3021 void Assembler::movsd(XMMRegister dst, const Operand& src) {
3022 DCHECK(!IsEnabled(AVX));
3023 EnsureSpace ensure_space(this);
3024 emit(0xF2); // double
3025 emit_optional_rex_32(dst, src);
3026 emit(0x0F);
3027 emit(0x10); // load
3028 emit_sse_operand(dst, src);
3029 }
3030
3031
movaps(XMMRegister dst,XMMRegister src)3032 void Assembler::movaps(XMMRegister dst, XMMRegister src) {
3033 DCHECK(!IsEnabled(AVX));
3034 EnsureSpace ensure_space(this);
3035 if (src.low_bits() == 4) {
3036 // Try to avoid an unnecessary SIB byte.
3037 emit_optional_rex_32(src, dst);
3038 emit(0x0F);
3039 emit(0x29);
3040 emit_sse_operand(src, dst);
3041 } else {
3042 emit_optional_rex_32(dst, src);
3043 emit(0x0F);
3044 emit(0x28);
3045 emit_sse_operand(dst, src);
3046 }
3047 }
3048
3049
shufps(XMMRegister dst,XMMRegister src,byte imm8)3050 void Assembler::shufps(XMMRegister dst, XMMRegister src, byte imm8) {
3051 DCHECK(is_uint8(imm8));
3052 EnsureSpace ensure_space(this);
3053 emit_optional_rex_32(src, dst);
3054 emit(0x0F);
3055 emit(0xC6);
3056 emit_sse_operand(dst, src);
3057 emit(imm8);
3058 }
3059
3060
movapd(XMMRegister dst,XMMRegister src)3061 void Assembler::movapd(XMMRegister dst, XMMRegister src) {
3062 DCHECK(!IsEnabled(AVX));
3063 EnsureSpace ensure_space(this);
3064 if (src.low_bits() == 4) {
3065 // Try to avoid an unnecessary SIB byte.
3066 emit(0x66);
3067 emit_optional_rex_32(src, dst);
3068 emit(0x0F);
3069 emit(0x29);
3070 emit_sse_operand(src, dst);
3071 } else {
3072 emit(0x66);
3073 emit_optional_rex_32(dst, src);
3074 emit(0x0F);
3075 emit(0x28);
3076 emit_sse_operand(dst, src);
3077 }
3078 }
3079
3080
movupd(XMMRegister dst,const Operand & src)3081 void Assembler::movupd(XMMRegister dst, const Operand& src) {
3082 EnsureSpace ensure_space(this);
3083 emit(0x66);
3084 emit_optional_rex_32(dst, src);
3085 emit(0x0F);
3086 emit(0x10);
3087 emit_sse_operand(dst, src);
3088 }
3089
movupd(const Operand & dst,XMMRegister src)3090 void Assembler::movupd(const Operand& dst, XMMRegister src) {
3091 EnsureSpace ensure_space(this);
3092 emit(0x66);
3093 emit_optional_rex_32(src, dst);
3094 emit(0x0F);
3095 emit(0x11);
3096 emit_sse_operand(src, dst);
3097 }
3098
addss(XMMRegister dst,XMMRegister src)3099 void Assembler::addss(XMMRegister dst, XMMRegister src) {
3100 EnsureSpace ensure_space(this);
3101 emit(0xF3);
3102 emit_optional_rex_32(dst, src);
3103 emit(0x0F);
3104 emit(0x58);
3105 emit_sse_operand(dst, src);
3106 }
3107
3108
addss(XMMRegister dst,const Operand & src)3109 void Assembler::addss(XMMRegister dst, const Operand& src) {
3110 EnsureSpace ensure_space(this);
3111 emit(0xF3);
3112 emit_optional_rex_32(dst, src);
3113 emit(0x0F);
3114 emit(0x58);
3115 emit_sse_operand(dst, src);
3116 }
3117
3118
subss(XMMRegister dst,XMMRegister src)3119 void Assembler::subss(XMMRegister dst, XMMRegister src) {
3120 EnsureSpace ensure_space(this);
3121 emit(0xF3);
3122 emit_optional_rex_32(dst, src);
3123 emit(0x0F);
3124 emit(0x5C);
3125 emit_sse_operand(dst, src);
3126 }
3127
3128
subss(XMMRegister dst,const Operand & src)3129 void Assembler::subss(XMMRegister dst, const Operand& src) {
3130 EnsureSpace ensure_space(this);
3131 emit(0xF3);
3132 emit_optional_rex_32(dst, src);
3133 emit(0x0F);
3134 emit(0x5C);
3135 emit_sse_operand(dst, src);
3136 }
3137
3138
mulss(XMMRegister dst,XMMRegister src)3139 void Assembler::mulss(XMMRegister dst, XMMRegister src) {
3140 EnsureSpace ensure_space(this);
3141 emit(0xF3);
3142 emit_optional_rex_32(dst, src);
3143 emit(0x0F);
3144 emit(0x59);
3145 emit_sse_operand(dst, src);
3146 }
3147
3148
mulss(XMMRegister dst,const Operand & src)3149 void Assembler::mulss(XMMRegister dst, const Operand& src) {
3150 EnsureSpace ensure_space(this);
3151 emit(0xF3);
3152 emit_optional_rex_32(dst, src);
3153 emit(0x0F);
3154 emit(0x59);
3155 emit_sse_operand(dst, src);
3156 }
3157
3158
divss(XMMRegister dst,XMMRegister src)3159 void Assembler::divss(XMMRegister dst, XMMRegister src) {
3160 EnsureSpace ensure_space(this);
3161 emit(0xF3);
3162 emit_optional_rex_32(dst, src);
3163 emit(0x0F);
3164 emit(0x5E);
3165 emit_sse_operand(dst, src);
3166 }
3167
3168
divss(XMMRegister dst,const Operand & src)3169 void Assembler::divss(XMMRegister dst, const Operand& src) {
3170 EnsureSpace ensure_space(this);
3171 emit(0xF3);
3172 emit_optional_rex_32(dst, src);
3173 emit(0x0F);
3174 emit(0x5E);
3175 emit_sse_operand(dst, src);
3176 }
3177
3178
maxss(XMMRegister dst,XMMRegister src)3179 void Assembler::maxss(XMMRegister dst, XMMRegister src) {
3180 EnsureSpace ensure_space(this);
3181 emit(0xF3);
3182 emit_optional_rex_32(dst, src);
3183 emit(0x0F);
3184 emit(0x5F);
3185 emit_sse_operand(dst, src);
3186 }
3187
3188
maxss(XMMRegister dst,const Operand & src)3189 void Assembler::maxss(XMMRegister dst, const Operand& src) {
3190 EnsureSpace ensure_space(this);
3191 emit(0xF3);
3192 emit_optional_rex_32(dst, src);
3193 emit(0x0F);
3194 emit(0x5F);
3195 emit_sse_operand(dst, src);
3196 }
3197
3198
minss(XMMRegister dst,XMMRegister src)3199 void Assembler::minss(XMMRegister dst, XMMRegister src) {
3200 EnsureSpace ensure_space(this);
3201 emit(0xF3);
3202 emit_optional_rex_32(dst, src);
3203 emit(0x0F);
3204 emit(0x5D);
3205 emit_sse_operand(dst, src);
3206 }
3207
3208
minss(XMMRegister dst,const Operand & src)3209 void Assembler::minss(XMMRegister dst, const Operand& src) {
3210 EnsureSpace ensure_space(this);
3211 emit(0xF3);
3212 emit_optional_rex_32(dst, src);
3213 emit(0x0F);
3214 emit(0x5D);
3215 emit_sse_operand(dst, src);
3216 }
3217
3218
sqrtss(XMMRegister dst,XMMRegister src)3219 void Assembler::sqrtss(XMMRegister dst, XMMRegister src) {
3220 EnsureSpace ensure_space(this);
3221 emit(0xF3);
3222 emit_optional_rex_32(dst, src);
3223 emit(0x0F);
3224 emit(0x51);
3225 emit_sse_operand(dst, src);
3226 }
3227
3228
sqrtss(XMMRegister dst,const Operand & src)3229 void Assembler::sqrtss(XMMRegister dst, const Operand& src) {
3230 EnsureSpace ensure_space(this);
3231 emit(0xF3);
3232 emit_optional_rex_32(dst, src);
3233 emit(0x0F);
3234 emit(0x51);
3235 emit_sse_operand(dst, src);
3236 }
3237
3238
ucomiss(XMMRegister dst,XMMRegister src)3239 void Assembler::ucomiss(XMMRegister dst, XMMRegister src) {
3240 DCHECK(!IsEnabled(AVX));
3241 EnsureSpace ensure_space(this);
3242 emit_optional_rex_32(dst, src);
3243 emit(0x0f);
3244 emit(0x2e);
3245 emit_sse_operand(dst, src);
3246 }
3247
3248
ucomiss(XMMRegister dst,const Operand & src)3249 void Assembler::ucomiss(XMMRegister dst, const Operand& src) {
3250 DCHECK(!IsEnabled(AVX));
3251 EnsureSpace ensure_space(this);
3252 emit_optional_rex_32(dst, src);
3253 emit(0x0f);
3254 emit(0x2e);
3255 emit_sse_operand(dst, src);
3256 }
3257
3258
movss(XMMRegister dst,XMMRegister src)3259 void Assembler::movss(XMMRegister dst, XMMRegister src) {
3260 DCHECK(!IsEnabled(AVX));
3261 EnsureSpace ensure_space(this);
3262 emit(0xF3); // single
3263 emit_optional_rex_32(dst, src);
3264 emit(0x0F);
3265 emit(0x10); // load
3266 emit_sse_operand(dst, src);
3267 }
3268
3269
movss(XMMRegister dst,const Operand & src)3270 void Assembler::movss(XMMRegister dst, const Operand& src) {
3271 DCHECK(!IsEnabled(AVX));
3272 EnsureSpace ensure_space(this);
3273 emit(0xF3); // single
3274 emit_optional_rex_32(dst, src);
3275 emit(0x0F);
3276 emit(0x10); // load
3277 emit_sse_operand(dst, src);
3278 }
3279
3280
movss(const Operand & src,XMMRegister dst)3281 void Assembler::movss(const Operand& src, XMMRegister dst) {
3282 DCHECK(!IsEnabled(AVX));
3283 EnsureSpace ensure_space(this);
3284 emit(0xF3); // single
3285 emit_optional_rex_32(dst, src);
3286 emit(0x0F);
3287 emit(0x11); // store
3288 emit_sse_operand(dst, src);
3289 }
3290
3291
psllq(XMMRegister reg,byte imm8)3292 void Assembler::psllq(XMMRegister reg, byte imm8) {
3293 DCHECK(!IsEnabled(AVX));
3294 EnsureSpace ensure_space(this);
3295 emit(0x66);
3296 emit_optional_rex_32(reg);
3297 emit(0x0F);
3298 emit(0x73);
3299 emit_sse_operand(rsi, reg); // rsi == 6
3300 emit(imm8);
3301 }
3302
3303
psrlq(XMMRegister reg,byte imm8)3304 void Assembler::psrlq(XMMRegister reg, byte imm8) {
3305 DCHECK(!IsEnabled(AVX));
3306 EnsureSpace ensure_space(this);
3307 emit(0x66);
3308 emit_optional_rex_32(reg);
3309 emit(0x0F);
3310 emit(0x73);
3311 emit_sse_operand(rdx, reg); // rdx == 2
3312 emit(imm8);
3313 }
3314
psllw(XMMRegister reg,byte imm8)3315 void Assembler::psllw(XMMRegister reg, byte imm8) {
3316 EnsureSpace ensure_space(this);
3317 emit(0x66);
3318 emit_optional_rex_32(reg);
3319 emit(0x0F);
3320 emit(0x71);
3321 emit_sse_operand(rsi, reg); // rsi == 6
3322 emit(imm8);
3323 }
3324
pslld(XMMRegister reg,byte imm8)3325 void Assembler::pslld(XMMRegister reg, byte imm8) {
3326 EnsureSpace ensure_space(this);
3327 emit(0x66);
3328 emit_optional_rex_32(reg);
3329 emit(0x0F);
3330 emit(0x72);
3331 emit_sse_operand(rsi, reg); // rsi == 6
3332 emit(imm8);
3333 }
3334
psrlw(XMMRegister reg,byte imm8)3335 void Assembler::psrlw(XMMRegister reg, byte imm8) {
3336 EnsureSpace ensure_space(this);
3337 emit(0x66);
3338 emit_optional_rex_32(reg);
3339 emit(0x0F);
3340 emit(0x71);
3341 emit_sse_operand(rdx, reg); // rdx == 2
3342 emit(imm8);
3343 }
3344
psrld(XMMRegister reg,byte imm8)3345 void Assembler::psrld(XMMRegister reg, byte imm8) {
3346 EnsureSpace ensure_space(this);
3347 emit(0x66);
3348 emit_optional_rex_32(reg);
3349 emit(0x0F);
3350 emit(0x72);
3351 emit_sse_operand(rdx, reg); // rdx == 2
3352 emit(imm8);
3353 }
3354
psraw(XMMRegister reg,byte imm8)3355 void Assembler::psraw(XMMRegister reg, byte imm8) {
3356 EnsureSpace ensure_space(this);
3357 emit(0x66);
3358 emit_optional_rex_32(reg);
3359 emit(0x0F);
3360 emit(0x71);
3361 emit_sse_operand(rsp, reg); // rsp == 4
3362 emit(imm8);
3363 }
3364
psrad(XMMRegister reg,byte imm8)3365 void Assembler::psrad(XMMRegister reg, byte imm8) {
3366 EnsureSpace ensure_space(this);
3367 emit(0x66);
3368 emit_optional_rex_32(reg);
3369 emit(0x0F);
3370 emit(0x72);
3371 emit_sse_operand(rsp, reg); // rsp == 4
3372 emit(imm8);
3373 }
3374
cmpps(XMMRegister dst,XMMRegister src,int8_t cmp)3375 void Assembler::cmpps(XMMRegister dst, XMMRegister src, int8_t cmp) {
3376 EnsureSpace ensure_space(this);
3377 emit_optional_rex_32(dst, src);
3378 emit(0x0F);
3379 emit(0xC2);
3380 emit_sse_operand(dst, src);
3381 emit(cmp);
3382 }
3383
cmpps(XMMRegister dst,const Operand & src,int8_t cmp)3384 void Assembler::cmpps(XMMRegister dst, const Operand& src, int8_t cmp) {
3385 EnsureSpace ensure_space(this);
3386 emit_optional_rex_32(dst, src);
3387 emit(0x0F);
3388 emit(0xC2);
3389 emit_sse_operand(dst, src);
3390 emit(cmp);
3391 }
3392
cmppd(XMMRegister dst,XMMRegister src,int8_t cmp)3393 void Assembler::cmppd(XMMRegister dst, XMMRegister src, int8_t cmp) {
3394 EnsureSpace ensure_space(this);
3395 emit_optional_rex_32(dst, src);
3396 emit(0x66);
3397 emit(0x0F);
3398 emit(0xC2);
3399 emit_sse_operand(dst, src);
3400 emit(cmp);
3401 }
3402
cmppd(XMMRegister dst,const Operand & src,int8_t cmp)3403 void Assembler::cmppd(XMMRegister dst, const Operand& src, int8_t cmp) {
3404 EnsureSpace ensure_space(this);
3405 emit_optional_rex_32(dst, src);
3406 emit(0x66);
3407 emit(0x0F);
3408 emit(0xC2);
3409 emit_sse_operand(dst, src);
3410 emit(cmp);
3411 }
3412
cvttss2si(Register dst,const Operand & src)3413 void Assembler::cvttss2si(Register dst, const Operand& src) {
3414 DCHECK(!IsEnabled(AVX));
3415 EnsureSpace ensure_space(this);
3416 emit(0xF3);
3417 emit_optional_rex_32(dst, src);
3418 emit(0x0F);
3419 emit(0x2C);
3420 emit_operand(dst, src);
3421 }
3422
3423
cvttss2si(Register dst,XMMRegister src)3424 void Assembler::cvttss2si(Register dst, XMMRegister src) {
3425 DCHECK(!IsEnabled(AVX));
3426 EnsureSpace ensure_space(this);
3427 emit(0xF3);
3428 emit_optional_rex_32(dst, src);
3429 emit(0x0F);
3430 emit(0x2C);
3431 emit_sse_operand(dst, src);
3432 }
3433
3434
cvttsd2si(Register dst,const Operand & src)3435 void Assembler::cvttsd2si(Register dst, const Operand& src) {
3436 DCHECK(!IsEnabled(AVX));
3437 EnsureSpace ensure_space(this);
3438 emit(0xF2);
3439 emit_optional_rex_32(dst, src);
3440 emit(0x0F);
3441 emit(0x2C);
3442 emit_operand(dst, src);
3443 }
3444
3445
cvttsd2si(Register dst,XMMRegister src)3446 void Assembler::cvttsd2si(Register dst, XMMRegister src) {
3447 DCHECK(!IsEnabled(AVX));
3448 EnsureSpace ensure_space(this);
3449 emit(0xF2);
3450 emit_optional_rex_32(dst, src);
3451 emit(0x0F);
3452 emit(0x2C);
3453 emit_sse_operand(dst, src);
3454 }
3455
3456
cvttss2siq(Register dst,XMMRegister src)3457 void Assembler::cvttss2siq(Register dst, XMMRegister src) {
3458 DCHECK(!IsEnabled(AVX));
3459 EnsureSpace ensure_space(this);
3460 emit(0xF3);
3461 emit_rex_64(dst, src);
3462 emit(0x0F);
3463 emit(0x2C);
3464 emit_sse_operand(dst, src);
3465 }
3466
3467
cvttss2siq(Register dst,const Operand & src)3468 void Assembler::cvttss2siq(Register dst, const Operand& src) {
3469 DCHECK(!IsEnabled(AVX));
3470 EnsureSpace ensure_space(this);
3471 emit(0xF3);
3472 emit_rex_64(dst, src);
3473 emit(0x0F);
3474 emit(0x2C);
3475 emit_sse_operand(dst, src);
3476 }
3477
3478
cvttsd2siq(Register dst,XMMRegister src)3479 void Assembler::cvttsd2siq(Register dst, XMMRegister src) {
3480 DCHECK(!IsEnabled(AVX));
3481 EnsureSpace ensure_space(this);
3482 emit(0xF2);
3483 emit_rex_64(dst, src);
3484 emit(0x0F);
3485 emit(0x2C);
3486 emit_sse_operand(dst, src);
3487 }
3488
3489
cvttsd2siq(Register dst,const Operand & src)3490 void Assembler::cvttsd2siq(Register dst, const Operand& src) {
3491 DCHECK(!IsEnabled(AVX));
3492 EnsureSpace ensure_space(this);
3493 emit(0xF2);
3494 emit_rex_64(dst, src);
3495 emit(0x0F);
3496 emit(0x2C);
3497 emit_sse_operand(dst, src);
3498 }
3499
3500
cvtlsi2sd(XMMRegister dst,const Operand & src)3501 void Assembler::cvtlsi2sd(XMMRegister dst, const Operand& src) {
3502 DCHECK(!IsEnabled(AVX));
3503 EnsureSpace ensure_space(this);
3504 emit(0xF2);
3505 emit_optional_rex_32(dst, src);
3506 emit(0x0F);
3507 emit(0x2A);
3508 emit_sse_operand(dst, src);
3509 }
3510
3511
cvtlsi2sd(XMMRegister dst,Register src)3512 void Assembler::cvtlsi2sd(XMMRegister dst, Register src) {
3513 DCHECK(!IsEnabled(AVX));
3514 EnsureSpace ensure_space(this);
3515 emit(0xF2);
3516 emit_optional_rex_32(dst, src);
3517 emit(0x0F);
3518 emit(0x2A);
3519 emit_sse_operand(dst, src);
3520 }
3521
3522
cvtlsi2ss(XMMRegister dst,const Operand & src)3523 void Assembler::cvtlsi2ss(XMMRegister dst, const Operand& src) {
3524 DCHECK(!IsEnabled(AVX));
3525 EnsureSpace ensure_space(this);
3526 emit(0xF3);
3527 emit_optional_rex_32(dst, src);
3528 emit(0x0F);
3529 emit(0x2A);
3530 emit_sse_operand(dst, src);
3531 }
3532
3533
cvtlsi2ss(XMMRegister dst,Register src)3534 void Assembler::cvtlsi2ss(XMMRegister dst, Register src) {
3535 EnsureSpace ensure_space(this);
3536 emit(0xF3);
3537 emit_optional_rex_32(dst, src);
3538 emit(0x0F);
3539 emit(0x2A);
3540 emit_sse_operand(dst, src);
3541 }
3542
3543
cvtqsi2ss(XMMRegister dst,const Operand & src)3544 void Assembler::cvtqsi2ss(XMMRegister dst, const Operand& src) {
3545 DCHECK(!IsEnabled(AVX));
3546 EnsureSpace ensure_space(this);
3547 emit(0xF3);
3548 emit_rex_64(dst, src);
3549 emit(0x0F);
3550 emit(0x2A);
3551 emit_sse_operand(dst, src);
3552 }
3553
3554
cvtqsi2ss(XMMRegister dst,Register src)3555 void Assembler::cvtqsi2ss(XMMRegister dst, Register src) {
3556 DCHECK(!IsEnabled(AVX));
3557 EnsureSpace ensure_space(this);
3558 emit(0xF3);
3559 emit_rex_64(dst, src);
3560 emit(0x0F);
3561 emit(0x2A);
3562 emit_sse_operand(dst, src);
3563 }
3564
3565
cvtqsi2sd(XMMRegister dst,const Operand & src)3566 void Assembler::cvtqsi2sd(XMMRegister dst, const Operand& src) {
3567 DCHECK(!IsEnabled(AVX));
3568 EnsureSpace ensure_space(this);
3569 emit(0xF2);
3570 emit_rex_64(dst, src);
3571 emit(0x0F);
3572 emit(0x2A);
3573 emit_sse_operand(dst, src);
3574 }
3575
3576
cvtqsi2sd(XMMRegister dst,Register src)3577 void Assembler::cvtqsi2sd(XMMRegister dst, Register src) {
3578 DCHECK(!IsEnabled(AVX));
3579 EnsureSpace ensure_space(this);
3580 emit(0xF2);
3581 emit_rex_64(dst, src);
3582 emit(0x0F);
3583 emit(0x2A);
3584 emit_sse_operand(dst, src);
3585 }
3586
3587
cvtss2sd(XMMRegister dst,XMMRegister src)3588 void Assembler::cvtss2sd(XMMRegister dst, XMMRegister src) {
3589 DCHECK(!IsEnabled(AVX));
3590 EnsureSpace ensure_space(this);
3591 emit(0xF3);
3592 emit_optional_rex_32(dst, src);
3593 emit(0x0F);
3594 emit(0x5A);
3595 emit_sse_operand(dst, src);
3596 }
3597
3598
cvtss2sd(XMMRegister dst,const Operand & src)3599 void Assembler::cvtss2sd(XMMRegister dst, const Operand& src) {
3600 DCHECK(!IsEnabled(AVX));
3601 EnsureSpace ensure_space(this);
3602 emit(0xF3);
3603 emit_optional_rex_32(dst, src);
3604 emit(0x0F);
3605 emit(0x5A);
3606 emit_sse_operand(dst, src);
3607 }
3608
3609
cvtsd2ss(XMMRegister dst,XMMRegister src)3610 void Assembler::cvtsd2ss(XMMRegister dst, XMMRegister src) {
3611 DCHECK(!IsEnabled(AVX));
3612 EnsureSpace ensure_space(this);
3613 emit(0xF2);
3614 emit_optional_rex_32(dst, src);
3615 emit(0x0F);
3616 emit(0x5A);
3617 emit_sse_operand(dst, src);
3618 }
3619
3620
cvtsd2ss(XMMRegister dst,const Operand & src)3621 void Assembler::cvtsd2ss(XMMRegister dst, const Operand& src) {
3622 DCHECK(!IsEnabled(AVX));
3623 EnsureSpace ensure_space(this);
3624 emit(0xF2);
3625 emit_optional_rex_32(dst, src);
3626 emit(0x0F);
3627 emit(0x5A);
3628 emit_sse_operand(dst, src);
3629 }
3630
3631
cvtsd2si(Register dst,XMMRegister src)3632 void Assembler::cvtsd2si(Register dst, XMMRegister src) {
3633 DCHECK(!IsEnabled(AVX));
3634 EnsureSpace ensure_space(this);
3635 emit(0xF2);
3636 emit_optional_rex_32(dst, src);
3637 emit(0x0F);
3638 emit(0x2D);
3639 emit_sse_operand(dst, src);
3640 }
3641
3642
cvtsd2siq(Register dst,XMMRegister src)3643 void Assembler::cvtsd2siq(Register dst, XMMRegister src) {
3644 DCHECK(!IsEnabled(AVX));
3645 EnsureSpace ensure_space(this);
3646 emit(0xF2);
3647 emit_rex_64(dst, src);
3648 emit(0x0F);
3649 emit(0x2D);
3650 emit_sse_operand(dst, src);
3651 }
3652
3653
addsd(XMMRegister dst,XMMRegister src)3654 void Assembler::addsd(XMMRegister dst, XMMRegister src) {
3655 EnsureSpace ensure_space(this);
3656 emit(0xF2);
3657 emit_optional_rex_32(dst, src);
3658 emit(0x0F);
3659 emit(0x58);
3660 emit_sse_operand(dst, src);
3661 }
3662
3663
addsd(XMMRegister dst,const Operand & src)3664 void Assembler::addsd(XMMRegister dst, const Operand& src) {
3665 EnsureSpace ensure_space(this);
3666 emit(0xF2);
3667 emit_optional_rex_32(dst, src);
3668 emit(0x0F);
3669 emit(0x58);
3670 emit_sse_operand(dst, src);
3671 }
3672
3673
mulsd(XMMRegister dst,XMMRegister src)3674 void Assembler::mulsd(XMMRegister dst, XMMRegister src) {
3675 EnsureSpace ensure_space(this);
3676 emit(0xF2);
3677 emit_optional_rex_32(dst, src);
3678 emit(0x0F);
3679 emit(0x59);
3680 emit_sse_operand(dst, src);
3681 }
3682
3683
mulsd(XMMRegister dst,const Operand & src)3684 void Assembler::mulsd(XMMRegister dst, const Operand& src) {
3685 EnsureSpace ensure_space(this);
3686 emit(0xF2);
3687 emit_optional_rex_32(dst, src);
3688 emit(0x0F);
3689 emit(0x59);
3690 emit_sse_operand(dst, src);
3691 }
3692
3693
subsd(XMMRegister dst,XMMRegister src)3694 void Assembler::subsd(XMMRegister dst, XMMRegister src) {
3695 EnsureSpace ensure_space(this);
3696 emit(0xF2);
3697 emit_optional_rex_32(dst, src);
3698 emit(0x0F);
3699 emit(0x5C);
3700 emit_sse_operand(dst, src);
3701 }
3702
3703
subsd(XMMRegister dst,const Operand & src)3704 void Assembler::subsd(XMMRegister dst, const Operand& src) {
3705 EnsureSpace ensure_space(this);
3706 emit(0xF2);
3707 emit_optional_rex_32(dst, src);
3708 emit(0x0F);
3709 emit(0x5C);
3710 emit_sse_operand(dst, src);
3711 }
3712
3713
divsd(XMMRegister dst,XMMRegister src)3714 void Assembler::divsd(XMMRegister dst, XMMRegister src) {
3715 EnsureSpace ensure_space(this);
3716 emit(0xF2);
3717 emit_optional_rex_32(dst, src);
3718 emit(0x0F);
3719 emit(0x5E);
3720 emit_sse_operand(dst, src);
3721 }
3722
3723
divsd(XMMRegister dst,const Operand & src)3724 void Assembler::divsd(XMMRegister dst, const Operand& src) {
3725 EnsureSpace ensure_space(this);
3726 emit(0xF2);
3727 emit_optional_rex_32(dst, src);
3728 emit(0x0F);
3729 emit(0x5E);
3730 emit_sse_operand(dst, src);
3731 }
3732
3733
maxsd(XMMRegister dst,XMMRegister src)3734 void Assembler::maxsd(XMMRegister dst, XMMRegister src) {
3735 EnsureSpace ensure_space(this);
3736 emit(0xF2);
3737 emit_optional_rex_32(dst, src);
3738 emit(0x0F);
3739 emit(0x5F);
3740 emit_sse_operand(dst, src);
3741 }
3742
3743
maxsd(XMMRegister dst,const Operand & src)3744 void Assembler::maxsd(XMMRegister dst, const Operand& src) {
3745 EnsureSpace ensure_space(this);
3746 emit(0xF2);
3747 emit_optional_rex_32(dst, src);
3748 emit(0x0F);
3749 emit(0x5F);
3750 emit_sse_operand(dst, src);
3751 }
3752
3753
minsd(XMMRegister dst,XMMRegister src)3754 void Assembler::minsd(XMMRegister dst, XMMRegister src) {
3755 EnsureSpace ensure_space(this);
3756 emit(0xF2);
3757 emit_optional_rex_32(dst, src);
3758 emit(0x0F);
3759 emit(0x5D);
3760 emit_sse_operand(dst, src);
3761 }
3762
3763
minsd(XMMRegister dst,const Operand & src)3764 void Assembler::minsd(XMMRegister dst, const Operand& src) {
3765 EnsureSpace ensure_space(this);
3766 emit(0xF2);
3767 emit_optional_rex_32(dst, src);
3768 emit(0x0F);
3769 emit(0x5D);
3770 emit_sse_operand(dst, src);
3771 }
3772
3773
andpd(XMMRegister dst,XMMRegister src)3774 void Assembler::andpd(XMMRegister dst, XMMRegister src) {
3775 EnsureSpace ensure_space(this);
3776 emit(0x66);
3777 emit_optional_rex_32(dst, src);
3778 emit(0x0F);
3779 emit(0x54);
3780 emit_sse_operand(dst, src);
3781 }
3782
3783
andpd(XMMRegister dst,const Operand & src)3784 void Assembler::andpd(XMMRegister dst, const Operand& src) {
3785 EnsureSpace ensure_space(this);
3786 emit(0x66);
3787 emit_optional_rex_32(dst, src);
3788 emit(0x0F);
3789 emit(0x54);
3790 emit_sse_operand(dst, src);
3791 }
3792
3793
orpd(XMMRegister dst,XMMRegister src)3794 void Assembler::orpd(XMMRegister dst, XMMRegister src) {
3795 EnsureSpace ensure_space(this);
3796 emit(0x66);
3797 emit_optional_rex_32(dst, src);
3798 emit(0x0F);
3799 emit(0x56);
3800 emit_sse_operand(dst, src);
3801 }
3802
3803
orpd(XMMRegister dst,const Operand & src)3804 void Assembler::orpd(XMMRegister dst, const Operand& src) {
3805 EnsureSpace ensure_space(this);
3806 emit(0x66);
3807 emit_optional_rex_32(dst, src);
3808 emit(0x0F);
3809 emit(0x56);
3810 emit_sse_operand(dst, src);
3811 }
3812
3813
xorpd(XMMRegister dst,XMMRegister src)3814 void Assembler::xorpd(XMMRegister dst, XMMRegister src) {
3815 DCHECK(!IsEnabled(AVX));
3816 EnsureSpace ensure_space(this);
3817 emit(0x66);
3818 emit_optional_rex_32(dst, src);
3819 emit(0x0F);
3820 emit(0x57);
3821 emit_sse_operand(dst, src);
3822 }
3823
3824
xorpd(XMMRegister dst,const Operand & src)3825 void Assembler::xorpd(XMMRegister dst, const Operand& src) {
3826 DCHECK(!IsEnabled(AVX));
3827 EnsureSpace ensure_space(this);
3828 emit(0x66);
3829 emit_optional_rex_32(dst, src);
3830 emit(0x0F);
3831 emit(0x57);
3832 emit_sse_operand(dst, src);
3833 }
3834
3835
sqrtsd(XMMRegister dst,XMMRegister src)3836 void Assembler::sqrtsd(XMMRegister dst, XMMRegister src) {
3837 DCHECK(!IsEnabled(AVX));
3838 EnsureSpace ensure_space(this);
3839 emit(0xF2);
3840 emit_optional_rex_32(dst, src);
3841 emit(0x0F);
3842 emit(0x51);
3843 emit_sse_operand(dst, src);
3844 }
3845
3846
sqrtsd(XMMRegister dst,const Operand & src)3847 void Assembler::sqrtsd(XMMRegister dst, const Operand& src) {
3848 DCHECK(!IsEnabled(AVX));
3849 EnsureSpace ensure_space(this);
3850 emit(0xF2);
3851 emit_optional_rex_32(dst, src);
3852 emit(0x0F);
3853 emit(0x51);
3854 emit_sse_operand(dst, src);
3855 }
3856
3857
ucomisd(XMMRegister dst,XMMRegister src)3858 void Assembler::ucomisd(XMMRegister dst, XMMRegister src) {
3859 DCHECK(!IsEnabled(AVX));
3860 EnsureSpace ensure_space(this);
3861 emit(0x66);
3862 emit_optional_rex_32(dst, src);
3863 emit(0x0f);
3864 emit(0x2e);
3865 emit_sse_operand(dst, src);
3866 }
3867
3868
ucomisd(XMMRegister dst,const Operand & src)3869 void Assembler::ucomisd(XMMRegister dst, const Operand& src) {
3870 DCHECK(!IsEnabled(AVX));
3871 EnsureSpace ensure_space(this);
3872 emit(0x66);
3873 emit_optional_rex_32(dst, src);
3874 emit(0x0f);
3875 emit(0x2e);
3876 emit_sse_operand(dst, src);
3877 }
3878
3879
cmpltsd(XMMRegister dst,XMMRegister src)3880 void Assembler::cmpltsd(XMMRegister dst, XMMRegister src) {
3881 EnsureSpace ensure_space(this);
3882 emit(0xF2);
3883 emit_optional_rex_32(dst, src);
3884 emit(0x0F);
3885 emit(0xC2);
3886 emit_sse_operand(dst, src);
3887 emit(0x01); // LT == 1
3888 }
3889
3890
roundss(XMMRegister dst,XMMRegister src,RoundingMode mode)3891 void Assembler::roundss(XMMRegister dst, XMMRegister src, RoundingMode mode) {
3892 DCHECK(!IsEnabled(AVX));
3893 DCHECK(IsEnabled(SSE4_1));
3894 EnsureSpace ensure_space(this);
3895 emit(0x66);
3896 emit_optional_rex_32(dst, src);
3897 emit(0x0f);
3898 emit(0x3a);
3899 emit(0x0a);
3900 emit_sse_operand(dst, src);
3901 // Mask precision exception.
3902 emit(static_cast<byte>(mode) | 0x8);
3903 }
3904
3905
roundsd(XMMRegister dst,XMMRegister src,RoundingMode mode)3906 void Assembler::roundsd(XMMRegister dst, XMMRegister src, RoundingMode mode) {
3907 DCHECK(!IsEnabled(AVX));
3908 DCHECK(IsEnabled(SSE4_1));
3909 EnsureSpace ensure_space(this);
3910 emit(0x66);
3911 emit_optional_rex_32(dst, src);
3912 emit(0x0f);
3913 emit(0x3a);
3914 emit(0x0b);
3915 emit_sse_operand(dst, src);
3916 // Mask precision exception.
3917 emit(static_cast<byte>(mode) | 0x8);
3918 }
3919
3920
movmskpd(Register dst,XMMRegister src)3921 void Assembler::movmskpd(Register dst, XMMRegister src) {
3922 EnsureSpace ensure_space(this);
3923 emit(0x66);
3924 emit_optional_rex_32(dst, src);
3925 emit(0x0f);
3926 emit(0x50);
3927 emit_sse_operand(dst, src);
3928 }
3929
3930
movmskps(Register dst,XMMRegister src)3931 void Assembler::movmskps(Register dst, XMMRegister src) {
3932 EnsureSpace ensure_space(this);
3933 emit_optional_rex_32(dst, src);
3934 emit(0x0f);
3935 emit(0x50);
3936 emit_sse_operand(dst, src);
3937 }
3938
3939
punpckldq(XMMRegister dst,XMMRegister src)3940 void Assembler::punpckldq(XMMRegister dst, XMMRegister src) {
3941 EnsureSpace ensure_space(this);
3942 emit(0x66);
3943 emit_optional_rex_32(dst, src);
3944 emit(0x0F);
3945 emit(0x62);
3946 emit_sse_operand(dst, src);
3947 }
3948
punpckldq(XMMRegister dst,const Operand & src)3949 void Assembler::punpckldq(XMMRegister dst, const Operand& src) {
3950 EnsureSpace ensure_space(this);
3951 emit(0x66);
3952 emit_optional_rex_32(dst, src);
3953 emit(0x0F);
3954 emit(0x62);
3955 emit_sse_operand(dst, src);
3956 }
3957
punpckhdq(XMMRegister dst,XMMRegister src)3958 void Assembler::punpckhdq(XMMRegister dst, XMMRegister src) {
3959 EnsureSpace ensure_space(this);
3960 emit(0x66);
3961 emit_optional_rex_32(dst, src);
3962 emit(0x0F);
3963 emit(0x6A);
3964 emit_sse_operand(dst, src);
3965 }
3966
3967
3968 // AVX instructions
vfmasd(byte op,XMMRegister dst,XMMRegister src1,XMMRegister src2)3969 void Assembler::vfmasd(byte op, XMMRegister dst, XMMRegister src1,
3970 XMMRegister src2) {
3971 DCHECK(IsEnabled(FMA3));
3972 EnsureSpace ensure_space(this);
3973 emit_vex_prefix(dst, src1, src2, kLIG, k66, k0F38, kW1);
3974 emit(op);
3975 emit_sse_operand(dst, src2);
3976 }
3977
3978
vfmasd(byte op,XMMRegister dst,XMMRegister src1,const Operand & src2)3979 void Assembler::vfmasd(byte op, XMMRegister dst, XMMRegister src1,
3980 const Operand& src2) {
3981 DCHECK(IsEnabled(FMA3));
3982 EnsureSpace ensure_space(this);
3983 emit_vex_prefix(dst, src1, src2, kLIG, k66, k0F38, kW1);
3984 emit(op);
3985 emit_sse_operand(dst, src2);
3986 }
3987
3988
vfmass(byte op,XMMRegister dst,XMMRegister src1,XMMRegister src2)3989 void Assembler::vfmass(byte op, XMMRegister dst, XMMRegister src1,
3990 XMMRegister src2) {
3991 DCHECK(IsEnabled(FMA3));
3992 EnsureSpace ensure_space(this);
3993 emit_vex_prefix(dst, src1, src2, kLIG, k66, k0F38, kW0);
3994 emit(op);
3995 emit_sse_operand(dst, src2);
3996 }
3997
3998
vfmass(byte op,XMMRegister dst,XMMRegister src1,const Operand & src2)3999 void Assembler::vfmass(byte op, XMMRegister dst, XMMRegister src1,
4000 const Operand& src2) {
4001 DCHECK(IsEnabled(FMA3));
4002 EnsureSpace ensure_space(this);
4003 emit_vex_prefix(dst, src1, src2, kLIG, k66, k0F38, kW0);
4004 emit(op);
4005 emit_sse_operand(dst, src2);
4006 }
4007
4008
vmovd(XMMRegister dst,Register src)4009 void Assembler::vmovd(XMMRegister dst, Register src) {
4010 DCHECK(IsEnabled(AVX));
4011 EnsureSpace ensure_space(this);
4012 XMMRegister isrc = {src.code()};
4013 emit_vex_prefix(dst, xmm0, isrc, kL128, k66, k0F, kW0);
4014 emit(0x6e);
4015 emit_sse_operand(dst, src);
4016 }
4017
4018
vmovd(XMMRegister dst,const Operand & src)4019 void Assembler::vmovd(XMMRegister dst, const Operand& src) {
4020 DCHECK(IsEnabled(AVX));
4021 EnsureSpace ensure_space(this);
4022 emit_vex_prefix(dst, xmm0, src, kL128, k66, k0F, kW0);
4023 emit(0x6e);
4024 emit_sse_operand(dst, src);
4025 }
4026
4027
vmovd(Register dst,XMMRegister src)4028 void Assembler::vmovd(Register dst, XMMRegister src) {
4029 DCHECK(IsEnabled(AVX));
4030 EnsureSpace ensure_space(this);
4031 XMMRegister idst = {dst.code()};
4032 emit_vex_prefix(src, xmm0, idst, kL128, k66, k0F, kW0);
4033 emit(0x7e);
4034 emit_sse_operand(src, dst);
4035 }
4036
4037
vmovq(XMMRegister dst,Register src)4038 void Assembler::vmovq(XMMRegister dst, Register src) {
4039 DCHECK(IsEnabled(AVX));
4040 EnsureSpace ensure_space(this);
4041 XMMRegister isrc = {src.code()};
4042 emit_vex_prefix(dst, xmm0, isrc, kL128, k66, k0F, kW1);
4043 emit(0x6e);
4044 emit_sse_operand(dst, src);
4045 }
4046
4047
vmovq(XMMRegister dst,const Operand & src)4048 void Assembler::vmovq(XMMRegister dst, const Operand& src) {
4049 DCHECK(IsEnabled(AVX));
4050 EnsureSpace ensure_space(this);
4051 emit_vex_prefix(dst, xmm0, src, kL128, k66, k0F, kW1);
4052 emit(0x6e);
4053 emit_sse_operand(dst, src);
4054 }
4055
4056
vmovq(Register dst,XMMRegister src)4057 void Assembler::vmovq(Register dst, XMMRegister src) {
4058 DCHECK(IsEnabled(AVX));
4059 EnsureSpace ensure_space(this);
4060 XMMRegister idst = {dst.code()};
4061 emit_vex_prefix(src, xmm0, idst, kL128, k66, k0F, kW1);
4062 emit(0x7e);
4063 emit_sse_operand(src, dst);
4064 }
4065
vinstr(byte op,XMMRegister dst,XMMRegister src1,XMMRegister src2,SIMDPrefix pp,LeadingOpcode m,VexW w)4066 void Assembler::vinstr(byte op, XMMRegister dst, XMMRegister src1,
4067 XMMRegister src2, SIMDPrefix pp, LeadingOpcode m,
4068 VexW w) {
4069 DCHECK(IsEnabled(AVX));
4070 EnsureSpace ensure_space(this);
4071 emit_vex_prefix(dst, src1, src2, kLIG, pp, m, w);
4072 emit(op);
4073 emit_sse_operand(dst, src2);
4074 }
4075
vinstr(byte op,XMMRegister dst,XMMRegister src1,const Operand & src2,SIMDPrefix pp,LeadingOpcode m,VexW w)4076 void Assembler::vinstr(byte op, XMMRegister dst, XMMRegister src1,
4077 const Operand& src2, SIMDPrefix pp, LeadingOpcode m,
4078 VexW w) {
4079 DCHECK(IsEnabled(AVX));
4080 EnsureSpace ensure_space(this);
4081 emit_vex_prefix(dst, src1, src2, kLIG, pp, m, w);
4082 emit(op);
4083 emit_sse_operand(dst, src2);
4084 }
4085
4086
vps(byte op,XMMRegister dst,XMMRegister src1,XMMRegister src2)4087 void Assembler::vps(byte op, XMMRegister dst, XMMRegister src1,
4088 XMMRegister src2) {
4089 DCHECK(IsEnabled(AVX));
4090 EnsureSpace ensure_space(this);
4091 emit_vex_prefix(dst, src1, src2, kL128, kNone, k0F, kWIG);
4092 emit(op);
4093 emit_sse_operand(dst, src2);
4094 }
4095
4096
vps(byte op,XMMRegister dst,XMMRegister src1,const Operand & src2)4097 void Assembler::vps(byte op, XMMRegister dst, XMMRegister src1,
4098 const Operand& src2) {
4099 DCHECK(IsEnabled(AVX));
4100 EnsureSpace ensure_space(this);
4101 emit_vex_prefix(dst, src1, src2, kL128, kNone, k0F, kWIG);
4102 emit(op);
4103 emit_sse_operand(dst, src2);
4104 }
4105
4106
vpd(byte op,XMMRegister dst,XMMRegister src1,XMMRegister src2)4107 void Assembler::vpd(byte op, XMMRegister dst, XMMRegister src1,
4108 XMMRegister src2) {
4109 DCHECK(IsEnabled(AVX));
4110 EnsureSpace ensure_space(this);
4111 emit_vex_prefix(dst, src1, src2, kL128, k66, k0F, kWIG);
4112 emit(op);
4113 emit_sse_operand(dst, src2);
4114 }
4115
4116
vpd(byte op,XMMRegister dst,XMMRegister src1,const Operand & src2)4117 void Assembler::vpd(byte op, XMMRegister dst, XMMRegister src1,
4118 const Operand& src2) {
4119 DCHECK(IsEnabled(AVX));
4120 EnsureSpace ensure_space(this);
4121 emit_vex_prefix(dst, src1, src2, kL128, k66, k0F, kWIG);
4122 emit(op);
4123 emit_sse_operand(dst, src2);
4124 }
4125
4126
vucomiss(XMMRegister dst,XMMRegister src)4127 void Assembler::vucomiss(XMMRegister dst, XMMRegister src) {
4128 DCHECK(IsEnabled(AVX));
4129 EnsureSpace ensure_space(this);
4130 emit_vex_prefix(dst, xmm0, src, kLIG, kNone, k0F, kWIG);
4131 emit(0x2e);
4132 emit_sse_operand(dst, src);
4133 }
4134
4135
vucomiss(XMMRegister dst,const Operand & src)4136 void Assembler::vucomiss(XMMRegister dst, const Operand& src) {
4137 DCHECK(IsEnabled(AVX));
4138 EnsureSpace ensure_space(this);
4139 emit_vex_prefix(dst, xmm0, src, kLIG, kNone, k0F, kWIG);
4140 emit(0x2e);
4141 emit_sse_operand(dst, src);
4142 }
4143
4144
vss(byte op,XMMRegister dst,XMMRegister src1,XMMRegister src2)4145 void Assembler::vss(byte op, XMMRegister dst, XMMRegister src1,
4146 XMMRegister src2) {
4147 DCHECK(IsEnabled(AVX));
4148 EnsureSpace ensure_space(this);
4149 emit_vex_prefix(dst, src1, src2, kLIG, kF3, k0F, kWIG);
4150 emit(op);
4151 emit_sse_operand(dst, src2);
4152 }
4153
4154
vss(byte op,XMMRegister dst,XMMRegister src1,const Operand & src2)4155 void Assembler::vss(byte op, XMMRegister dst, XMMRegister src1,
4156 const Operand& src2) {
4157 DCHECK(IsEnabled(AVX));
4158 EnsureSpace ensure_space(this);
4159 emit_vex_prefix(dst, src1, src2, kLIG, kF3, k0F, kWIG);
4160 emit(op);
4161 emit_sse_operand(dst, src2);
4162 }
4163
4164
bmi1q(byte op,Register reg,Register vreg,Register rm)4165 void Assembler::bmi1q(byte op, Register reg, Register vreg, Register rm) {
4166 DCHECK(IsEnabled(BMI1));
4167 EnsureSpace ensure_space(this);
4168 emit_vex_prefix(reg, vreg, rm, kLZ, kNone, k0F38, kW1);
4169 emit(op);
4170 emit_modrm(reg, rm);
4171 }
4172
4173
bmi1q(byte op,Register reg,Register vreg,const Operand & rm)4174 void Assembler::bmi1q(byte op, Register reg, Register vreg, const Operand& rm) {
4175 DCHECK(IsEnabled(BMI1));
4176 EnsureSpace ensure_space(this);
4177 emit_vex_prefix(reg, vreg, rm, kLZ, kNone, k0F38, kW1);
4178 emit(op);
4179 emit_operand(reg, rm);
4180 }
4181
4182
bmi1l(byte op,Register reg,Register vreg,Register rm)4183 void Assembler::bmi1l(byte op, Register reg, Register vreg, Register rm) {
4184 DCHECK(IsEnabled(BMI1));
4185 EnsureSpace ensure_space(this);
4186 emit_vex_prefix(reg, vreg, rm, kLZ, kNone, k0F38, kW0);
4187 emit(op);
4188 emit_modrm(reg, rm);
4189 }
4190
4191
bmi1l(byte op,Register reg,Register vreg,const Operand & rm)4192 void Assembler::bmi1l(byte op, Register reg, Register vreg, const Operand& rm) {
4193 DCHECK(IsEnabled(BMI1));
4194 EnsureSpace ensure_space(this);
4195 emit_vex_prefix(reg, vreg, rm, kLZ, kNone, k0F38, kW0);
4196 emit(op);
4197 emit_operand(reg, rm);
4198 }
4199
4200
tzcntq(Register dst,Register src)4201 void Assembler::tzcntq(Register dst, Register src) {
4202 DCHECK(IsEnabled(BMI1));
4203 EnsureSpace ensure_space(this);
4204 emit(0xF3);
4205 emit_rex_64(dst, src);
4206 emit(0x0F);
4207 emit(0xBC);
4208 emit_modrm(dst, src);
4209 }
4210
4211
tzcntq(Register dst,const Operand & src)4212 void Assembler::tzcntq(Register dst, const Operand& src) {
4213 DCHECK(IsEnabled(BMI1));
4214 EnsureSpace ensure_space(this);
4215 emit(0xF3);
4216 emit_rex_64(dst, src);
4217 emit(0x0F);
4218 emit(0xBC);
4219 emit_operand(dst, src);
4220 }
4221
4222
tzcntl(Register dst,Register src)4223 void Assembler::tzcntl(Register dst, Register src) {
4224 DCHECK(IsEnabled(BMI1));
4225 EnsureSpace ensure_space(this);
4226 emit(0xF3);
4227 emit_optional_rex_32(dst, src);
4228 emit(0x0F);
4229 emit(0xBC);
4230 emit_modrm(dst, src);
4231 }
4232
4233
tzcntl(Register dst,const Operand & src)4234 void Assembler::tzcntl(Register dst, const Operand& src) {
4235 DCHECK(IsEnabled(BMI1));
4236 EnsureSpace ensure_space(this);
4237 emit(0xF3);
4238 emit_optional_rex_32(dst, src);
4239 emit(0x0F);
4240 emit(0xBC);
4241 emit_operand(dst, src);
4242 }
4243
4244
lzcntq(Register dst,Register src)4245 void Assembler::lzcntq(Register dst, Register src) {
4246 DCHECK(IsEnabled(LZCNT));
4247 EnsureSpace ensure_space(this);
4248 emit(0xF3);
4249 emit_rex_64(dst, src);
4250 emit(0x0F);
4251 emit(0xBD);
4252 emit_modrm(dst, src);
4253 }
4254
4255
lzcntq(Register dst,const Operand & src)4256 void Assembler::lzcntq(Register dst, const Operand& src) {
4257 DCHECK(IsEnabled(LZCNT));
4258 EnsureSpace ensure_space(this);
4259 emit(0xF3);
4260 emit_rex_64(dst, src);
4261 emit(0x0F);
4262 emit(0xBD);
4263 emit_operand(dst, src);
4264 }
4265
4266
lzcntl(Register dst,Register src)4267 void Assembler::lzcntl(Register dst, Register src) {
4268 DCHECK(IsEnabled(LZCNT));
4269 EnsureSpace ensure_space(this);
4270 emit(0xF3);
4271 emit_optional_rex_32(dst, src);
4272 emit(0x0F);
4273 emit(0xBD);
4274 emit_modrm(dst, src);
4275 }
4276
4277
lzcntl(Register dst,const Operand & src)4278 void Assembler::lzcntl(Register dst, const Operand& src) {
4279 DCHECK(IsEnabled(LZCNT));
4280 EnsureSpace ensure_space(this);
4281 emit(0xF3);
4282 emit_optional_rex_32(dst, src);
4283 emit(0x0F);
4284 emit(0xBD);
4285 emit_operand(dst, src);
4286 }
4287
4288
popcntq(Register dst,Register src)4289 void Assembler::popcntq(Register dst, Register src) {
4290 DCHECK(IsEnabled(POPCNT));
4291 EnsureSpace ensure_space(this);
4292 emit(0xF3);
4293 emit_rex_64(dst, src);
4294 emit(0x0F);
4295 emit(0xB8);
4296 emit_modrm(dst, src);
4297 }
4298
4299
popcntq(Register dst,const Operand & src)4300 void Assembler::popcntq(Register dst, const Operand& src) {
4301 DCHECK(IsEnabled(POPCNT));
4302 EnsureSpace ensure_space(this);
4303 emit(0xF3);
4304 emit_rex_64(dst, src);
4305 emit(0x0F);
4306 emit(0xB8);
4307 emit_operand(dst, src);
4308 }
4309
4310
popcntl(Register dst,Register src)4311 void Assembler::popcntl(Register dst, Register src) {
4312 DCHECK(IsEnabled(POPCNT));
4313 EnsureSpace ensure_space(this);
4314 emit(0xF3);
4315 emit_optional_rex_32(dst, src);
4316 emit(0x0F);
4317 emit(0xB8);
4318 emit_modrm(dst, src);
4319 }
4320
4321
popcntl(Register dst,const Operand & src)4322 void Assembler::popcntl(Register dst, const Operand& src) {
4323 DCHECK(IsEnabled(POPCNT));
4324 EnsureSpace ensure_space(this);
4325 emit(0xF3);
4326 emit_optional_rex_32(dst, src);
4327 emit(0x0F);
4328 emit(0xB8);
4329 emit_operand(dst, src);
4330 }
4331
4332
bmi2q(SIMDPrefix pp,byte op,Register reg,Register vreg,Register rm)4333 void Assembler::bmi2q(SIMDPrefix pp, byte op, Register reg, Register vreg,
4334 Register rm) {
4335 DCHECK(IsEnabled(BMI2));
4336 EnsureSpace ensure_space(this);
4337 emit_vex_prefix(reg, vreg, rm, kLZ, pp, k0F38, kW1);
4338 emit(op);
4339 emit_modrm(reg, rm);
4340 }
4341
4342
bmi2q(SIMDPrefix pp,byte op,Register reg,Register vreg,const Operand & rm)4343 void Assembler::bmi2q(SIMDPrefix pp, byte op, Register reg, Register vreg,
4344 const Operand& rm) {
4345 DCHECK(IsEnabled(BMI2));
4346 EnsureSpace ensure_space(this);
4347 emit_vex_prefix(reg, vreg, rm, kLZ, pp, k0F38, kW1);
4348 emit(op);
4349 emit_operand(reg, rm);
4350 }
4351
4352
bmi2l(SIMDPrefix pp,byte op,Register reg,Register vreg,Register rm)4353 void Assembler::bmi2l(SIMDPrefix pp, byte op, Register reg, Register vreg,
4354 Register rm) {
4355 DCHECK(IsEnabled(BMI2));
4356 EnsureSpace ensure_space(this);
4357 emit_vex_prefix(reg, vreg, rm, kLZ, pp, k0F38, kW0);
4358 emit(op);
4359 emit_modrm(reg, rm);
4360 }
4361
4362
bmi2l(SIMDPrefix pp,byte op,Register reg,Register vreg,const Operand & rm)4363 void Assembler::bmi2l(SIMDPrefix pp, byte op, Register reg, Register vreg,
4364 const Operand& rm) {
4365 DCHECK(IsEnabled(BMI2));
4366 EnsureSpace ensure_space(this);
4367 emit_vex_prefix(reg, vreg, rm, kLZ, pp, k0F38, kW0);
4368 emit(op);
4369 emit_operand(reg, rm);
4370 }
4371
4372
rorxq(Register dst,Register src,byte imm8)4373 void Assembler::rorxq(Register dst, Register src, byte imm8) {
4374 DCHECK(IsEnabled(BMI2));
4375 DCHECK(is_uint8(imm8));
4376 Register vreg = {0}; // VEX.vvvv unused
4377 EnsureSpace ensure_space(this);
4378 emit_vex_prefix(dst, vreg, src, kLZ, kF2, k0F3A, kW1);
4379 emit(0xF0);
4380 emit_modrm(dst, src);
4381 emit(imm8);
4382 }
4383
4384
rorxq(Register dst,const Operand & src,byte imm8)4385 void Assembler::rorxq(Register dst, const Operand& src, byte imm8) {
4386 DCHECK(IsEnabled(BMI2));
4387 DCHECK(is_uint8(imm8));
4388 Register vreg = {0}; // VEX.vvvv unused
4389 EnsureSpace ensure_space(this);
4390 emit_vex_prefix(dst, vreg, src, kLZ, kF2, k0F3A, kW1);
4391 emit(0xF0);
4392 emit_operand(dst, src);
4393 emit(imm8);
4394 }
4395
4396
rorxl(Register dst,Register src,byte imm8)4397 void Assembler::rorxl(Register dst, Register src, byte imm8) {
4398 DCHECK(IsEnabled(BMI2));
4399 DCHECK(is_uint8(imm8));
4400 Register vreg = {0}; // VEX.vvvv unused
4401 EnsureSpace ensure_space(this);
4402 emit_vex_prefix(dst, vreg, src, kLZ, kF2, k0F3A, kW0);
4403 emit(0xF0);
4404 emit_modrm(dst, src);
4405 emit(imm8);
4406 }
4407
4408
rorxl(Register dst,const Operand & src,byte imm8)4409 void Assembler::rorxl(Register dst, const Operand& src, byte imm8) {
4410 DCHECK(IsEnabled(BMI2));
4411 DCHECK(is_uint8(imm8));
4412 Register vreg = {0}; // VEX.vvvv unused
4413 EnsureSpace ensure_space(this);
4414 emit_vex_prefix(dst, vreg, src, kLZ, kF2, k0F3A, kW0);
4415 emit(0xF0);
4416 emit_operand(dst, src);
4417 emit(imm8);
4418 }
4419
minps(XMMRegister dst,XMMRegister src)4420 void Assembler::minps(XMMRegister dst, XMMRegister src) {
4421 EnsureSpace ensure_space(this);
4422 emit_optional_rex_32(dst, src);
4423 emit(0x0F);
4424 emit(0x5D);
4425 emit_sse_operand(dst, src);
4426 }
4427
minps(XMMRegister dst,const Operand & src)4428 void Assembler::minps(XMMRegister dst, const Operand& src) {
4429 EnsureSpace ensure_space(this);
4430 emit_optional_rex_32(dst, src);
4431 emit(0x0F);
4432 emit(0x5D);
4433 emit_sse_operand(dst, src);
4434 }
4435
maxps(XMMRegister dst,XMMRegister src)4436 void Assembler::maxps(XMMRegister dst, XMMRegister src) {
4437 EnsureSpace ensure_space(this);
4438 emit_optional_rex_32(dst, src);
4439 emit(0x0F);
4440 emit(0x5F);
4441 emit_sse_operand(dst, src);
4442 }
4443
maxps(XMMRegister dst,const Operand & src)4444 void Assembler::maxps(XMMRegister dst, const Operand& src) {
4445 EnsureSpace ensure_space(this);
4446 emit_optional_rex_32(dst, src);
4447 emit(0x0F);
4448 emit(0x5F);
4449 emit_sse_operand(dst, src);
4450 }
4451
rcpps(XMMRegister dst,XMMRegister src)4452 void Assembler::rcpps(XMMRegister dst, XMMRegister src) {
4453 EnsureSpace ensure_space(this);
4454 emit_optional_rex_32(dst, src);
4455 emit(0x0F);
4456 emit(0x53);
4457 emit_sse_operand(dst, src);
4458 }
4459
rcpps(XMMRegister dst,const Operand & src)4460 void Assembler::rcpps(XMMRegister dst, const Operand& src) {
4461 EnsureSpace ensure_space(this);
4462 emit_optional_rex_32(dst, src);
4463 emit(0x0F);
4464 emit(0x53);
4465 emit_sse_operand(dst, src);
4466 }
4467
rsqrtps(XMMRegister dst,XMMRegister src)4468 void Assembler::rsqrtps(XMMRegister dst, XMMRegister src) {
4469 EnsureSpace ensure_space(this);
4470 emit_optional_rex_32(dst, src);
4471 emit(0x0F);
4472 emit(0x52);
4473 emit_sse_operand(dst, src);
4474 }
4475
rsqrtps(XMMRegister dst,const Operand & src)4476 void Assembler::rsqrtps(XMMRegister dst, const Operand& src) {
4477 EnsureSpace ensure_space(this);
4478 emit_optional_rex_32(dst, src);
4479 emit(0x0F);
4480 emit(0x52);
4481 emit_sse_operand(dst, src);
4482 }
4483
sqrtps(XMMRegister dst,XMMRegister src)4484 void Assembler::sqrtps(XMMRegister dst, XMMRegister src) {
4485 EnsureSpace ensure_space(this);
4486 emit_optional_rex_32(dst, src);
4487 emit(0x0F);
4488 emit(0x51);
4489 emit_sse_operand(dst, src);
4490 }
4491
sqrtps(XMMRegister dst,const Operand & src)4492 void Assembler::sqrtps(XMMRegister dst, const Operand& src) {
4493 EnsureSpace ensure_space(this);
4494 emit_optional_rex_32(dst, src);
4495 emit(0x0F);
4496 emit(0x51);
4497 emit_sse_operand(dst, src);
4498 }
4499
cvtdq2ps(XMMRegister dst,XMMRegister src)4500 void Assembler::cvtdq2ps(XMMRegister dst, XMMRegister src) {
4501 EnsureSpace ensure_space(this);
4502 emit_optional_rex_32(dst, src);
4503 emit(0x0F);
4504 emit(0x5B);
4505 emit_sse_operand(dst, src);
4506 }
4507
cvtdq2ps(XMMRegister dst,const Operand & src)4508 void Assembler::cvtdq2ps(XMMRegister dst, const Operand& src) {
4509 EnsureSpace ensure_space(this);
4510 emit_optional_rex_32(dst, src);
4511 emit(0x0F);
4512 emit(0x5B);
4513 emit_sse_operand(dst, src);
4514 }
4515
movups(XMMRegister dst,XMMRegister src)4516 void Assembler::movups(XMMRegister dst, XMMRegister src) {
4517 EnsureSpace ensure_space(this);
4518 if (src.low_bits() == 4) {
4519 // Try to avoid an unnecessary SIB byte.
4520 emit_optional_rex_32(src, dst);
4521 emit(0x0F);
4522 emit(0x11);
4523 emit_sse_operand(src, dst);
4524 } else {
4525 emit_optional_rex_32(dst, src);
4526 emit(0x0F);
4527 emit(0x10);
4528 emit_sse_operand(dst, src);
4529 }
4530 }
4531
movups(XMMRegister dst,const Operand & src)4532 void Assembler::movups(XMMRegister dst, const Operand& src) {
4533 EnsureSpace ensure_space(this);
4534 emit_optional_rex_32(dst, src);
4535 emit(0x0F);
4536 emit(0x10);
4537 emit_sse_operand(dst, src);
4538 }
4539
movups(const Operand & dst,XMMRegister src)4540 void Assembler::movups(const Operand& dst, XMMRegister src) {
4541 EnsureSpace ensure_space(this);
4542 emit_optional_rex_32(src, dst);
4543 emit(0x0F);
4544 emit(0x11);
4545 emit_sse_operand(src, dst);
4546 }
4547
sse2_instr(XMMRegister dst,XMMRegister src,byte prefix,byte escape,byte opcode)4548 void Assembler::sse2_instr(XMMRegister dst, XMMRegister src, byte prefix,
4549 byte escape, byte opcode) {
4550 EnsureSpace ensure_space(this);
4551 emit(prefix);
4552 emit_optional_rex_32(dst, src);
4553 emit(escape);
4554 emit(opcode);
4555 emit_sse_operand(dst, src);
4556 }
4557
sse2_instr(XMMRegister dst,const Operand & src,byte prefix,byte escape,byte opcode)4558 void Assembler::sse2_instr(XMMRegister dst, const Operand& src, byte prefix,
4559 byte escape, byte opcode) {
4560 EnsureSpace ensure_space(this);
4561 emit(prefix);
4562 emit_optional_rex_32(dst, src);
4563 emit(escape);
4564 emit(opcode);
4565 emit_sse_operand(dst, src);
4566 }
4567
ssse3_instr(XMMRegister dst,XMMRegister src,byte prefix,byte escape1,byte escape2,byte opcode)4568 void Assembler::ssse3_instr(XMMRegister dst, XMMRegister src, byte prefix,
4569 byte escape1, byte escape2, byte opcode) {
4570 DCHECK(IsEnabled(SSSE3));
4571 EnsureSpace ensure_space(this);
4572 emit(prefix);
4573 emit_optional_rex_32(dst, src);
4574 emit(escape1);
4575 emit(escape2);
4576 emit(opcode);
4577 emit_sse_operand(dst, src);
4578 }
4579
ssse3_instr(XMMRegister dst,const Operand & src,byte prefix,byte escape1,byte escape2,byte opcode)4580 void Assembler::ssse3_instr(XMMRegister dst, const Operand& src, byte prefix,
4581 byte escape1, byte escape2, byte opcode) {
4582 DCHECK(IsEnabled(SSSE3));
4583 EnsureSpace ensure_space(this);
4584 emit(prefix);
4585 emit_optional_rex_32(dst, src);
4586 emit(escape1);
4587 emit(escape2);
4588 emit(opcode);
4589 emit_sse_operand(dst, src);
4590 }
4591
sse4_instr(XMMRegister dst,XMMRegister src,byte prefix,byte escape1,byte escape2,byte opcode)4592 void Assembler::sse4_instr(XMMRegister dst, XMMRegister src, byte prefix,
4593 byte escape1, byte escape2, byte opcode) {
4594 DCHECK(IsEnabled(SSE4_1));
4595 EnsureSpace ensure_space(this);
4596 emit(prefix);
4597 emit_optional_rex_32(dst, src);
4598 emit(escape1);
4599 emit(escape2);
4600 emit(opcode);
4601 emit_sse_operand(dst, src);
4602 }
4603
sse4_instr(XMMRegister dst,const Operand & src,byte prefix,byte escape1,byte escape2,byte opcode)4604 void Assembler::sse4_instr(XMMRegister dst, const Operand& src, byte prefix,
4605 byte escape1, byte escape2, byte opcode) {
4606 DCHECK(IsEnabled(SSE4_1));
4607 EnsureSpace ensure_space(this);
4608 emit(prefix);
4609 emit_optional_rex_32(dst, src);
4610 emit(escape1);
4611 emit(escape2);
4612 emit(opcode);
4613 emit_sse_operand(dst, src);
4614 }
4615
lddqu(XMMRegister dst,const Operand & src)4616 void Assembler::lddqu(XMMRegister dst, const Operand& src) {
4617 DCHECK(IsEnabled(SSE3));
4618 EnsureSpace ensure_space(this);
4619 emit(0xF2);
4620 emit_optional_rex_32(dst, src);
4621 emit(0x0F);
4622 emit(0xF0);
4623 emit_sse_operand(dst, src);
4624 }
4625
psrldq(XMMRegister dst,uint8_t shift)4626 void Assembler::psrldq(XMMRegister dst, uint8_t shift) {
4627 EnsureSpace ensure_space(this);
4628 emit(0x66);
4629 emit_optional_rex_32(dst);
4630 emit(0x0F);
4631 emit(0x73);
4632 emit_sse_operand(dst);
4633 emit(shift);
4634 }
4635
pshufd(XMMRegister dst,XMMRegister src,uint8_t shuffle)4636 void Assembler::pshufd(XMMRegister dst, XMMRegister src, uint8_t shuffle) {
4637 EnsureSpace ensure_space(this);
4638 emit(0x66);
4639 emit_optional_rex_32(dst, src);
4640 emit(0x0F);
4641 emit(0x70);
4642 emit_sse_operand(dst, src);
4643 emit(shuffle);
4644 }
4645
pshufd(XMMRegister dst,const Operand & src,uint8_t shuffle)4646 void Assembler::pshufd(XMMRegister dst, const Operand& src, uint8_t shuffle) {
4647 EnsureSpace ensure_space(this);
4648 emit(0x66);
4649 emit_optional_rex_32(dst, src);
4650 emit(0x0F);
4651 emit(0x70);
4652 emit_sse_operand(dst, src);
4653 emit(shuffle);
4654 }
4655
emit_sse_operand(XMMRegister reg,const Operand & adr)4656 void Assembler::emit_sse_operand(XMMRegister reg, const Operand& adr) {
4657 Register ireg = { reg.code() };
4658 emit_operand(ireg, adr);
4659 }
4660
4661
emit_sse_operand(Register reg,const Operand & adr)4662 void Assembler::emit_sse_operand(Register reg, const Operand& adr) {
4663 Register ireg = {reg.code()};
4664 emit_operand(ireg, adr);
4665 }
4666
4667
emit_sse_operand(XMMRegister dst,XMMRegister src)4668 void Assembler::emit_sse_operand(XMMRegister dst, XMMRegister src) {
4669 emit(0xC0 | (dst.low_bits() << 3) | src.low_bits());
4670 }
4671
4672
emit_sse_operand(XMMRegister dst,Register src)4673 void Assembler::emit_sse_operand(XMMRegister dst, Register src) {
4674 emit(0xC0 | (dst.low_bits() << 3) | src.low_bits());
4675 }
4676
4677
emit_sse_operand(Register dst,XMMRegister src)4678 void Assembler::emit_sse_operand(Register dst, XMMRegister src) {
4679 emit(0xC0 | (dst.low_bits() << 3) | src.low_bits());
4680 }
4681
emit_sse_operand(XMMRegister dst)4682 void Assembler::emit_sse_operand(XMMRegister dst) {
4683 emit(0xD8 | dst.low_bits());
4684 }
4685
4686
db(uint8_t data)4687 void Assembler::db(uint8_t data) {
4688 EnsureSpace ensure_space(this);
4689 emit(data);
4690 }
4691
4692
dd(uint32_t data)4693 void Assembler::dd(uint32_t data) {
4694 EnsureSpace ensure_space(this);
4695 emitl(data);
4696 }
4697
4698
dq(uint64_t data)4699 void Assembler::dq(uint64_t data) {
4700 EnsureSpace ensure_space(this);
4701 emitq(data);
4702 }
4703
4704
dq(Label * label)4705 void Assembler::dq(Label* label) {
4706 EnsureSpace ensure_space(this);
4707 if (label->is_bound()) {
4708 internal_reference_positions_.push_back(pc_offset());
4709 emitp(buffer_ + label->pos(), RelocInfo::INTERNAL_REFERENCE);
4710 } else {
4711 RecordRelocInfo(RelocInfo::INTERNAL_REFERENCE);
4712 emitl(0); // Zero for the first 32bit marks it as 64bit absolute address.
4713 if (label->is_linked()) {
4714 emitl(label->pos());
4715 label->link_to(pc_offset() - sizeof(int32_t));
4716 } else {
4717 DCHECK(label->is_unused());
4718 int32_t current = pc_offset();
4719 emitl(current);
4720 label->link_to(current);
4721 }
4722 }
4723 }
4724
4725
4726 // Relocation information implementations.
4727
RecordRelocInfo(RelocInfo::Mode rmode,intptr_t data)4728 void Assembler::RecordRelocInfo(RelocInfo::Mode rmode, intptr_t data) {
4729 DCHECK(!RelocInfo::IsNone(rmode));
4730 // Don't record external references unless the heap will be serialized.
4731 if (rmode == RelocInfo::EXTERNAL_REFERENCE &&
4732 !serializer_enabled() && !emit_debug_code()) {
4733 return;
4734 } else if (rmode == RelocInfo::CODE_AGE_SEQUENCE) {
4735 // Don't record psuedo relocation info for code age sequence mode.
4736 return;
4737 }
4738 RelocInfo rinfo(isolate(), pc_, rmode, data, NULL);
4739 reloc_info_writer.Write(&rinfo);
4740 }
4741
4742
4743 const int RelocInfo::kApplyMask = RelocInfo::kCodeTargetMask |
4744 1 << RelocInfo::RUNTIME_ENTRY |
4745 1 << RelocInfo::INTERNAL_REFERENCE |
4746 1 << RelocInfo::CODE_AGE_SEQUENCE;
4747
4748
IsCodedSpecially()4749 bool RelocInfo::IsCodedSpecially() {
4750 // The deserializer needs to know whether a pointer is specially coded. Being
4751 // specially coded on x64 means that it is a relative 32 bit address, as used
4752 // by branch instructions.
4753 return (1 << rmode_) & kApplyMask;
4754 }
4755
4756
IsInConstantPool()4757 bool RelocInfo::IsInConstantPool() {
4758 return false;
4759 }
4760
4761
4762 } // namespace internal
4763 } // namespace v8
4764
4765 #endif // V8_TARGET_ARCH_X64
4766