1 // Copyright 2012 the V8 project authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4
5 #include "src/x64/assembler-x64.h"
6
7 #include <cstring>
8
9 #if V8_TARGET_ARCH_X64
10
11 #if V8_LIBC_MSVCRT
12 #include <intrin.h> // _xgetbv()
13 #endif
14 #if V8_OS_MACOSX
15 #include <sys/sysctl.h>
16 #endif
17
18 #include "src/base/bits.h"
19 #include "src/macro-assembler.h"
20 #include "src/v8.h"
21
22 namespace v8 {
23 namespace internal {
24
25 // -----------------------------------------------------------------------------
26 // Implementation of CpuFeatures
27
28 namespace {
29
30 #if !V8_LIBC_MSVCRT
31
_xgetbv(unsigned int xcr)32 V8_INLINE uint64_t _xgetbv(unsigned int xcr) {
33 unsigned eax, edx;
34 // Check xgetbv; this uses a .byte sequence instead of the instruction
35 // directly because older assemblers do not include support for xgetbv and
36 // there is no easy way to conditionally compile based on the assembler
37 // used.
38 __asm__ volatile(".byte 0x0f, 0x01, 0xd0" : "=a"(eax), "=d"(edx) : "c"(xcr));
39 return static_cast<uint64_t>(eax) | (static_cast<uint64_t>(edx) << 32);
40 }
41
42 #define _XCR_XFEATURE_ENABLED_MASK 0
43
44 #endif // !V8_LIBC_MSVCRT
45
46
OSHasAVXSupport()47 bool OSHasAVXSupport() {
48 #if V8_OS_MACOSX
49 // Mac OS X up to 10.9 has a bug where AVX transitions were indeed being
50 // caused by ISRs, so we detect that here and disable AVX in that case.
51 char buffer[128];
52 size_t buffer_size = arraysize(buffer);
53 int ctl_name[] = {CTL_KERN, KERN_OSRELEASE};
54 if (sysctl(ctl_name, 2, buffer, &buffer_size, nullptr, 0) != 0) {
55 V8_Fatal(__FILE__, __LINE__, "V8 failed to get kernel version");
56 }
57 // The buffer now contains a string of the form XX.YY.ZZ, where
58 // XX is the major kernel version component.
59 char* period_pos = strchr(buffer, '.');
60 DCHECK_NOT_NULL(period_pos);
61 *period_pos = '\0';
62 long kernel_version_major = strtol(buffer, nullptr, 10); // NOLINT
63 if (kernel_version_major <= 13) return false;
64 #endif // V8_OS_MACOSX
65 // Check whether OS claims to support AVX.
66 uint64_t feature_mask = _xgetbv(_XCR_XFEATURE_ENABLED_MASK);
67 return (feature_mask & 0x6) == 0x6;
68 }
69
70 } // namespace
71
72
ProbeImpl(bool cross_compile)73 void CpuFeatures::ProbeImpl(bool cross_compile) {
74 base::CPU cpu;
75 CHECK(cpu.has_sse2()); // SSE2 support is mandatory.
76 CHECK(cpu.has_cmov()); // CMOV support is mandatory.
77
78 // Only use statically determined features for cross compile (snapshot).
79 if (cross_compile) return;
80
81 if (cpu.has_sse41() && FLAG_enable_sse4_1) supported_ |= 1u << SSE4_1;
82 if (cpu.has_sse3() && FLAG_enable_sse3) supported_ |= 1u << SSE3;
83 // SAHF is not generally available in long mode.
84 if (cpu.has_sahf() && FLAG_enable_sahf) supported_ |= 1u << SAHF;
85 if (cpu.has_avx() && FLAG_enable_avx && cpu.has_osxsave() &&
86 OSHasAVXSupport()) {
87 supported_ |= 1u << AVX;
88 }
89 if (cpu.has_fma3() && FLAG_enable_fma3 && cpu.has_osxsave() &&
90 OSHasAVXSupport()) {
91 supported_ |= 1u << FMA3;
92 }
93 if (cpu.has_bmi1() && FLAG_enable_bmi1) supported_ |= 1u << BMI1;
94 if (cpu.has_bmi2() && FLAG_enable_bmi2) supported_ |= 1u << BMI2;
95 if (cpu.has_lzcnt() && FLAG_enable_lzcnt) supported_ |= 1u << LZCNT;
96 if (cpu.has_popcnt() && FLAG_enable_popcnt) supported_ |= 1u << POPCNT;
97 if (strcmp(FLAG_mcpu, "auto") == 0) {
98 if (cpu.is_atom()) supported_ |= 1u << ATOM;
99 } else if (strcmp(FLAG_mcpu, "atom") == 0) {
100 supported_ |= 1u << ATOM;
101 }
102 }
103
104
PrintTarget()105 void CpuFeatures::PrintTarget() { }
PrintFeatures()106 void CpuFeatures::PrintFeatures() {
107 printf(
108 "SSE3=%d SSE4_1=%d SAHF=%d AVX=%d FMA3=%d BMI1=%d BMI2=%d LZCNT=%d "
109 "POPCNT=%d ATOM=%d\n",
110 CpuFeatures::IsSupported(SSE3), CpuFeatures::IsSupported(SSE4_1),
111 CpuFeatures::IsSupported(SAHF), CpuFeatures::IsSupported(AVX),
112 CpuFeatures::IsSupported(FMA3), CpuFeatures::IsSupported(BMI1),
113 CpuFeatures::IsSupported(BMI2), CpuFeatures::IsSupported(LZCNT),
114 CpuFeatures::IsSupported(POPCNT), CpuFeatures::IsSupported(ATOM));
115 }
116
117
118 // -----------------------------------------------------------------------------
119 // Implementation of Operand
120
Operand(Register base,int32_t disp)121 Operand::Operand(Register base, int32_t disp) : rex_(0) {
122 len_ = 1;
123 if (base.is(rsp) || base.is(r12)) {
124 // SIB byte is needed to encode (rsp + offset) or (r12 + offset).
125 set_sib(times_1, rsp, base);
126 }
127
128 if (disp == 0 && !base.is(rbp) && !base.is(r13)) {
129 set_modrm(0, base);
130 } else if (is_int8(disp)) {
131 set_modrm(1, base);
132 set_disp8(disp);
133 } else {
134 set_modrm(2, base);
135 set_disp32(disp);
136 }
137 }
138
139
Operand(Register base,Register index,ScaleFactor scale,int32_t disp)140 Operand::Operand(Register base,
141 Register index,
142 ScaleFactor scale,
143 int32_t disp) : rex_(0) {
144 DCHECK(!index.is(rsp));
145 len_ = 1;
146 set_sib(scale, index, base);
147 if (disp == 0 && !base.is(rbp) && !base.is(r13)) {
148 // This call to set_modrm doesn't overwrite the REX.B (or REX.X) bits
149 // possibly set by set_sib.
150 set_modrm(0, rsp);
151 } else if (is_int8(disp)) {
152 set_modrm(1, rsp);
153 set_disp8(disp);
154 } else {
155 set_modrm(2, rsp);
156 set_disp32(disp);
157 }
158 }
159
160
Operand(Register index,ScaleFactor scale,int32_t disp)161 Operand::Operand(Register index,
162 ScaleFactor scale,
163 int32_t disp) : rex_(0) {
164 DCHECK(!index.is(rsp));
165 len_ = 1;
166 set_modrm(0, rsp);
167 set_sib(scale, index, rbp);
168 set_disp32(disp);
169 }
170
171
Operand(Label * label)172 Operand::Operand(Label* label) : rex_(0), len_(1) {
173 DCHECK_NOT_NULL(label);
174 set_modrm(0, rbp);
175 set_disp64(reinterpret_cast<intptr_t>(label));
176 }
177
178
Operand(const Operand & operand,int32_t offset)179 Operand::Operand(const Operand& operand, int32_t offset) {
180 DCHECK(operand.len_ >= 1);
181 // Operand encodes REX ModR/M [SIB] [Disp].
182 byte modrm = operand.buf_[0];
183 DCHECK(modrm < 0xC0); // Disallow mode 3 (register target).
184 bool has_sib = ((modrm & 0x07) == 0x04);
185 byte mode = modrm & 0xC0;
186 int disp_offset = has_sib ? 2 : 1;
187 int base_reg = (has_sib ? operand.buf_[1] : modrm) & 0x07;
188 // Mode 0 with rbp/r13 as ModR/M or SIB base register always has a 32-bit
189 // displacement.
190 bool is_baseless = (mode == 0) && (base_reg == 0x05); // No base or RIP base.
191 int32_t disp_value = 0;
192 if (mode == 0x80 || is_baseless) {
193 // Mode 2 or mode 0 with rbp/r13 as base: Word displacement.
194 disp_value = *bit_cast<const int32_t*>(&operand.buf_[disp_offset]);
195 } else if (mode == 0x40) {
196 // Mode 1: Byte displacement.
197 disp_value = static_cast<signed char>(operand.buf_[disp_offset]);
198 }
199
200 // Write new operand with same registers, but with modified displacement.
201 DCHECK(offset >= 0 ? disp_value + offset > disp_value
202 : disp_value + offset < disp_value); // No overflow.
203 disp_value += offset;
204 rex_ = operand.rex_;
205 if (!is_int8(disp_value) || is_baseless) {
206 // Need 32 bits of displacement, mode 2 or mode 1 with register rbp/r13.
207 buf_[0] = (modrm & 0x3f) | (is_baseless ? 0x00 : 0x80);
208 len_ = disp_offset + 4;
209 Memory::int32_at(&buf_[disp_offset]) = disp_value;
210 } else if (disp_value != 0 || (base_reg == 0x05)) {
211 // Need 8 bits of displacement.
212 buf_[0] = (modrm & 0x3f) | 0x40; // Mode 1.
213 len_ = disp_offset + 1;
214 buf_[disp_offset] = static_cast<byte>(disp_value);
215 } else {
216 // Need no displacement.
217 buf_[0] = (modrm & 0x3f); // Mode 0.
218 len_ = disp_offset;
219 }
220 if (has_sib) {
221 buf_[1] = operand.buf_[1];
222 }
223 }
224
225
AddressUsesRegister(Register reg) const226 bool Operand::AddressUsesRegister(Register reg) const {
227 int code = reg.code();
228 DCHECK((buf_[0] & 0xC0) != 0xC0); // Always a memory operand.
229 // Start with only low three bits of base register. Initial decoding doesn't
230 // distinguish on the REX.B bit.
231 int base_code = buf_[0] & 0x07;
232 if (base_code == rsp.code()) {
233 // SIB byte present in buf_[1].
234 // Check the index register from the SIB byte + REX.X prefix.
235 int index_code = ((buf_[1] >> 3) & 0x07) | ((rex_ & 0x02) << 2);
236 // Index code (including REX.X) of 0x04 (rsp) means no index register.
237 if (index_code != rsp.code() && index_code == code) return true;
238 // Add REX.B to get the full base register code.
239 base_code = (buf_[1] & 0x07) | ((rex_ & 0x01) << 3);
240 // A base register of 0x05 (rbp) with mod = 0 means no base register.
241 if (base_code == rbp.code() && ((buf_[0] & 0xC0) == 0)) return false;
242 return code == base_code;
243 } else {
244 // A base register with low bits of 0x05 (rbp or r13) and mod = 0 means
245 // no base register.
246 if (base_code == rbp.code() && ((buf_[0] & 0xC0) == 0)) return false;
247 base_code |= ((rex_ & 0x01) << 3);
248 return code == base_code;
249 }
250 }
251
252
253 // -----------------------------------------------------------------------------
254 // Implementation of Assembler.
255
256 #ifdef GENERATED_CODE_COVERAGE
257 static void InitCoverageLog();
258 #endif
259
Assembler(Isolate * isolate,void * buffer,int buffer_size)260 Assembler::Assembler(Isolate* isolate, void* buffer, int buffer_size)
261 : AssemblerBase(isolate, buffer, buffer_size),
262 code_targets_(100),
263 positions_recorder_(this) {
264 // Clear the buffer in debug mode unless it was provided by the
265 // caller in which case we can't be sure it's okay to overwrite
266 // existing code in it.
267 #ifdef DEBUG
268 if (own_buffer_) {
269 memset(buffer_, 0xCC, buffer_size_); // int3
270 }
271 #endif
272
273 reloc_info_writer.Reposition(buffer_ + buffer_size_, pc_);
274
275
276 #ifdef GENERATED_CODE_COVERAGE
277 InitCoverageLog();
278 #endif
279 }
280
281
GetCode(CodeDesc * desc)282 void Assembler::GetCode(CodeDesc* desc) {
283 // Finalize code (at this point overflow() may be true, but the gap ensures
284 // that we are still not overlapping instructions and relocation info).
285 reloc_info_writer.Finish();
286 DCHECK(pc_ <= reloc_info_writer.pos()); // No overlap.
287 // Set up code descriptor.
288 desc->buffer = buffer_;
289 desc->buffer_size = buffer_size_;
290 desc->instr_size = pc_offset();
291 DCHECK(desc->instr_size > 0); // Zero-size code objects upset the system.
292 desc->reloc_size =
293 static_cast<int>((buffer_ + buffer_size_) - reloc_info_writer.pos());
294 desc->origin = this;
295 desc->constant_pool_size = 0;
296 }
297
298
Align(int m)299 void Assembler::Align(int m) {
300 DCHECK(base::bits::IsPowerOfTwo32(m));
301 int delta = (m - (pc_offset() & (m - 1))) & (m - 1);
302 Nop(delta);
303 }
304
305
CodeTargetAlign()306 void Assembler::CodeTargetAlign() {
307 Align(16); // Preferred alignment of jump targets on x64.
308 }
309
310
IsNop(Address addr)311 bool Assembler::IsNop(Address addr) {
312 Address a = addr;
313 while (*a == 0x66) a++;
314 if (*a == 0x90) return true;
315 if (a[0] == 0xf && a[1] == 0x1f) return true;
316 return false;
317 }
318
319
bind_to(Label * L,int pos)320 void Assembler::bind_to(Label* L, int pos) {
321 DCHECK(!L->is_bound()); // Label may only be bound once.
322 DCHECK(0 <= pos && pos <= pc_offset()); // Position must be valid.
323 if (L->is_linked()) {
324 int current = L->pos();
325 int next = long_at(current);
326 while (next != current) {
327 if (current >= 4 && long_at(current - 4) == 0) {
328 // Absolute address.
329 intptr_t imm64 = reinterpret_cast<intptr_t>(buffer_ + pos);
330 *reinterpret_cast<intptr_t*>(addr_at(current - 4)) = imm64;
331 internal_reference_positions_.push_back(current - 4);
332 } else {
333 // Relative address, relative to point after address.
334 int imm32 = pos - (current + sizeof(int32_t));
335 long_at_put(current, imm32);
336 }
337 current = next;
338 next = long_at(next);
339 }
340 // Fix up last fixup on linked list.
341 if (current >= 4 && long_at(current - 4) == 0) {
342 // Absolute address.
343 intptr_t imm64 = reinterpret_cast<intptr_t>(buffer_ + pos);
344 *reinterpret_cast<intptr_t*>(addr_at(current - 4)) = imm64;
345 internal_reference_positions_.push_back(current - 4);
346 } else {
347 // Relative address, relative to point after address.
348 int imm32 = pos - (current + sizeof(int32_t));
349 long_at_put(current, imm32);
350 }
351 }
352 while (L->is_near_linked()) {
353 int fixup_pos = L->near_link_pos();
354 int offset_to_next =
355 static_cast<int>(*reinterpret_cast<int8_t*>(addr_at(fixup_pos)));
356 DCHECK(offset_to_next <= 0);
357 int disp = pos - (fixup_pos + sizeof(int8_t));
358 CHECK(is_int8(disp));
359 set_byte_at(fixup_pos, disp);
360 if (offset_to_next < 0) {
361 L->link_to(fixup_pos + offset_to_next, Label::kNear);
362 } else {
363 L->UnuseNear();
364 }
365 }
366 L->bind_to(pos);
367 }
368
369
bind(Label * L)370 void Assembler::bind(Label* L) {
371 bind_to(L, pc_offset());
372 }
373
374
GrowBuffer()375 void Assembler::GrowBuffer() {
376 DCHECK(buffer_overflow());
377 if (!own_buffer_) FATAL("external code buffer is too small");
378
379 // Compute new buffer size.
380 CodeDesc desc; // the new buffer
381 desc.buffer_size = 2 * buffer_size_;
382
383 // Some internal data structures overflow for very large buffers,
384 // they must ensure that kMaximalBufferSize is not too large.
385 if ((desc.buffer_size > kMaximalBufferSize) ||
386 (desc.buffer_size > isolate()->heap()->MaxOldGenerationSize())) {
387 V8::FatalProcessOutOfMemory("Assembler::GrowBuffer");
388 }
389
390 // Set up new buffer.
391 desc.buffer = NewArray<byte>(desc.buffer_size);
392 desc.origin = this;
393 desc.instr_size = pc_offset();
394 desc.reloc_size =
395 static_cast<int>((buffer_ + buffer_size_) - (reloc_info_writer.pos()));
396
397 // Clear the buffer in debug mode. Use 'int3' instructions to make
398 // sure to get into problems if we ever run uninitialized code.
399 #ifdef DEBUG
400 memset(desc.buffer, 0xCC, desc.buffer_size);
401 #endif
402
403 // Copy the data.
404 intptr_t pc_delta = desc.buffer - buffer_;
405 intptr_t rc_delta = (desc.buffer + desc.buffer_size) -
406 (buffer_ + buffer_size_);
407 MemMove(desc.buffer, buffer_, desc.instr_size);
408 MemMove(rc_delta + reloc_info_writer.pos(), reloc_info_writer.pos(),
409 desc.reloc_size);
410
411 // Switch buffers.
412 DeleteArray(buffer_);
413 buffer_ = desc.buffer;
414 buffer_size_ = desc.buffer_size;
415 pc_ += pc_delta;
416 reloc_info_writer.Reposition(reloc_info_writer.pos() + rc_delta,
417 reloc_info_writer.last_pc() + pc_delta);
418
419 // Relocate internal references.
420 for (auto pos : internal_reference_positions_) {
421 intptr_t* p = reinterpret_cast<intptr_t*>(buffer_ + pos);
422 *p += pc_delta;
423 }
424
425 DCHECK(!buffer_overflow());
426 }
427
428
emit_operand(int code,const Operand & adr)429 void Assembler::emit_operand(int code, const Operand& adr) {
430 DCHECK(is_uint3(code));
431 const unsigned length = adr.len_;
432 DCHECK(length > 0);
433
434 // Emit updated ModR/M byte containing the given register.
435 DCHECK((adr.buf_[0] & 0x38) == 0);
436 *pc_++ = adr.buf_[0] | code << 3;
437
438 // Recognize RIP relative addressing.
439 if (adr.buf_[0] == 5) {
440 DCHECK_EQ(9u, length);
441 Label* label = *bit_cast<Label* const*>(&adr.buf_[1]);
442 if (label->is_bound()) {
443 int offset = label->pos() - pc_offset() - sizeof(int32_t);
444 DCHECK_GE(0, offset);
445 emitl(offset);
446 } else if (label->is_linked()) {
447 emitl(label->pos());
448 label->link_to(pc_offset() - sizeof(int32_t));
449 } else {
450 DCHECK(label->is_unused());
451 int32_t current = pc_offset();
452 emitl(current);
453 label->link_to(current);
454 }
455 } else {
456 // Emit the rest of the encoded operand.
457 for (unsigned i = 1; i < length; i++) *pc_++ = adr.buf_[i];
458 }
459 }
460
461
462 // Assembler Instruction implementations.
463
arithmetic_op(byte opcode,Register reg,const Operand & op,int size)464 void Assembler::arithmetic_op(byte opcode,
465 Register reg,
466 const Operand& op,
467 int size) {
468 EnsureSpace ensure_space(this);
469 emit_rex(reg, op, size);
470 emit(opcode);
471 emit_operand(reg, op);
472 }
473
474
arithmetic_op(byte opcode,Register reg,Register rm_reg,int size)475 void Assembler::arithmetic_op(byte opcode,
476 Register reg,
477 Register rm_reg,
478 int size) {
479 EnsureSpace ensure_space(this);
480 DCHECK((opcode & 0xC6) == 2);
481 if (rm_reg.low_bits() == 4) { // Forces SIB byte.
482 // Swap reg and rm_reg and change opcode operand order.
483 emit_rex(rm_reg, reg, size);
484 emit(opcode ^ 0x02);
485 emit_modrm(rm_reg, reg);
486 } else {
487 emit_rex(reg, rm_reg, size);
488 emit(opcode);
489 emit_modrm(reg, rm_reg);
490 }
491 }
492
493
arithmetic_op_16(byte opcode,Register reg,Register rm_reg)494 void Assembler::arithmetic_op_16(byte opcode, Register reg, Register rm_reg) {
495 EnsureSpace ensure_space(this);
496 DCHECK((opcode & 0xC6) == 2);
497 if (rm_reg.low_bits() == 4) { // Forces SIB byte.
498 // Swap reg and rm_reg and change opcode operand order.
499 emit(0x66);
500 emit_optional_rex_32(rm_reg, reg);
501 emit(opcode ^ 0x02);
502 emit_modrm(rm_reg, reg);
503 } else {
504 emit(0x66);
505 emit_optional_rex_32(reg, rm_reg);
506 emit(opcode);
507 emit_modrm(reg, rm_reg);
508 }
509 }
510
511
arithmetic_op_16(byte opcode,Register reg,const Operand & rm_reg)512 void Assembler::arithmetic_op_16(byte opcode,
513 Register reg,
514 const Operand& rm_reg) {
515 EnsureSpace ensure_space(this);
516 emit(0x66);
517 emit_optional_rex_32(reg, rm_reg);
518 emit(opcode);
519 emit_operand(reg, rm_reg);
520 }
521
522
arithmetic_op_8(byte opcode,Register reg,const Operand & op)523 void Assembler::arithmetic_op_8(byte opcode, Register reg, const Operand& op) {
524 EnsureSpace ensure_space(this);
525 if (!reg.is_byte_register()) {
526 // Register is not one of al, bl, cl, dl. Its encoding needs REX.
527 emit_rex_32(reg);
528 }
529 emit(opcode);
530 emit_operand(reg, op);
531 }
532
533
arithmetic_op_8(byte opcode,Register reg,Register rm_reg)534 void Assembler::arithmetic_op_8(byte opcode, Register reg, Register rm_reg) {
535 EnsureSpace ensure_space(this);
536 DCHECK((opcode & 0xC6) == 2);
537 if (rm_reg.low_bits() == 4) { // Forces SIB byte.
538 // Swap reg and rm_reg and change opcode operand order.
539 if (!rm_reg.is_byte_register() || !reg.is_byte_register()) {
540 // Register is not one of al, bl, cl, dl. Its encoding needs REX.
541 emit_rex_32(rm_reg, reg);
542 }
543 emit(opcode ^ 0x02);
544 emit_modrm(rm_reg, reg);
545 } else {
546 if (!reg.is_byte_register() || !rm_reg.is_byte_register()) {
547 // Register is not one of al, bl, cl, dl. Its encoding needs REX.
548 emit_rex_32(reg, rm_reg);
549 }
550 emit(opcode);
551 emit_modrm(reg, rm_reg);
552 }
553 }
554
555
immediate_arithmetic_op(byte subcode,Register dst,Immediate src,int size)556 void Assembler::immediate_arithmetic_op(byte subcode,
557 Register dst,
558 Immediate src,
559 int size) {
560 EnsureSpace ensure_space(this);
561 emit_rex(dst, size);
562 if (is_int8(src.value_)) {
563 emit(0x83);
564 emit_modrm(subcode, dst);
565 emit(src.value_);
566 } else if (dst.is(rax)) {
567 emit(0x05 | (subcode << 3));
568 emitl(src.value_);
569 } else {
570 emit(0x81);
571 emit_modrm(subcode, dst);
572 emitl(src.value_);
573 }
574 }
575
immediate_arithmetic_op(byte subcode,const Operand & dst,Immediate src,int size)576 void Assembler::immediate_arithmetic_op(byte subcode,
577 const Operand& dst,
578 Immediate src,
579 int size) {
580 EnsureSpace ensure_space(this);
581 emit_rex(dst, size);
582 if (is_int8(src.value_)) {
583 emit(0x83);
584 emit_operand(subcode, dst);
585 emit(src.value_);
586 } else {
587 emit(0x81);
588 emit_operand(subcode, dst);
589 emitl(src.value_);
590 }
591 }
592
593
immediate_arithmetic_op_16(byte subcode,Register dst,Immediate src)594 void Assembler::immediate_arithmetic_op_16(byte subcode,
595 Register dst,
596 Immediate src) {
597 EnsureSpace ensure_space(this);
598 emit(0x66); // Operand size override prefix.
599 emit_optional_rex_32(dst);
600 if (is_int8(src.value_)) {
601 emit(0x83);
602 emit_modrm(subcode, dst);
603 emit(src.value_);
604 } else if (dst.is(rax)) {
605 emit(0x05 | (subcode << 3));
606 emitw(src.value_);
607 } else {
608 emit(0x81);
609 emit_modrm(subcode, dst);
610 emitw(src.value_);
611 }
612 }
613
614
immediate_arithmetic_op_16(byte subcode,const Operand & dst,Immediate src)615 void Assembler::immediate_arithmetic_op_16(byte subcode,
616 const Operand& dst,
617 Immediate src) {
618 EnsureSpace ensure_space(this);
619 emit(0x66); // Operand size override prefix.
620 emit_optional_rex_32(dst);
621 if (is_int8(src.value_)) {
622 emit(0x83);
623 emit_operand(subcode, dst);
624 emit(src.value_);
625 } else {
626 emit(0x81);
627 emit_operand(subcode, dst);
628 emitw(src.value_);
629 }
630 }
631
632
immediate_arithmetic_op_8(byte subcode,const Operand & dst,Immediate src)633 void Assembler::immediate_arithmetic_op_8(byte subcode,
634 const Operand& dst,
635 Immediate src) {
636 EnsureSpace ensure_space(this);
637 emit_optional_rex_32(dst);
638 DCHECK(is_int8(src.value_) || is_uint8(src.value_));
639 emit(0x80);
640 emit_operand(subcode, dst);
641 emit(src.value_);
642 }
643
644
immediate_arithmetic_op_8(byte subcode,Register dst,Immediate src)645 void Assembler::immediate_arithmetic_op_8(byte subcode,
646 Register dst,
647 Immediate src) {
648 EnsureSpace ensure_space(this);
649 if (!dst.is_byte_register()) {
650 // Register is not one of al, bl, cl, dl. Its encoding needs REX.
651 emit_rex_32(dst);
652 }
653 DCHECK(is_int8(src.value_) || is_uint8(src.value_));
654 emit(0x80);
655 emit_modrm(subcode, dst);
656 emit(src.value_);
657 }
658
659
shift(Register dst,Immediate shift_amount,int subcode,int size)660 void Assembler::shift(Register dst,
661 Immediate shift_amount,
662 int subcode,
663 int size) {
664 EnsureSpace ensure_space(this);
665 DCHECK(size == kInt64Size ? is_uint6(shift_amount.value_)
666 : is_uint5(shift_amount.value_));
667 if (shift_amount.value_ == 1) {
668 emit_rex(dst, size);
669 emit(0xD1);
670 emit_modrm(subcode, dst);
671 } else {
672 emit_rex(dst, size);
673 emit(0xC1);
674 emit_modrm(subcode, dst);
675 emit(shift_amount.value_);
676 }
677 }
678
679
shift(Operand dst,Immediate shift_amount,int subcode,int size)680 void Assembler::shift(Operand dst, Immediate shift_amount, int subcode,
681 int size) {
682 EnsureSpace ensure_space(this);
683 DCHECK(size == kInt64Size ? is_uint6(shift_amount.value_)
684 : is_uint5(shift_amount.value_));
685 if (shift_amount.value_ == 1) {
686 emit_rex(dst, size);
687 emit(0xD1);
688 emit_operand(subcode, dst);
689 } else {
690 emit_rex(dst, size);
691 emit(0xC1);
692 emit_operand(subcode, dst);
693 emit(shift_amount.value_);
694 }
695 }
696
697
shift(Register dst,int subcode,int size)698 void Assembler::shift(Register dst, int subcode, int size) {
699 EnsureSpace ensure_space(this);
700 emit_rex(dst, size);
701 emit(0xD3);
702 emit_modrm(subcode, dst);
703 }
704
705
shift(Operand dst,int subcode,int size)706 void Assembler::shift(Operand dst, int subcode, int size) {
707 EnsureSpace ensure_space(this);
708 emit_rex(dst, size);
709 emit(0xD3);
710 emit_operand(subcode, dst);
711 }
712
713
bt(const Operand & dst,Register src)714 void Assembler::bt(const Operand& dst, Register src) {
715 EnsureSpace ensure_space(this);
716 emit_rex_64(src, dst);
717 emit(0x0F);
718 emit(0xA3);
719 emit_operand(src, dst);
720 }
721
722
bts(const Operand & dst,Register src)723 void Assembler::bts(const Operand& dst, Register src) {
724 EnsureSpace ensure_space(this);
725 emit_rex_64(src, dst);
726 emit(0x0F);
727 emit(0xAB);
728 emit_operand(src, dst);
729 }
730
731
bsrl(Register dst,Register src)732 void Assembler::bsrl(Register dst, Register src) {
733 EnsureSpace ensure_space(this);
734 emit_optional_rex_32(dst, src);
735 emit(0x0F);
736 emit(0xBD);
737 emit_modrm(dst, src);
738 }
739
740
bsrl(Register dst,const Operand & src)741 void Assembler::bsrl(Register dst, const Operand& src) {
742 EnsureSpace ensure_space(this);
743 emit_optional_rex_32(dst, src);
744 emit(0x0F);
745 emit(0xBD);
746 emit_operand(dst, src);
747 }
748
749
bsrq(Register dst,Register src)750 void Assembler::bsrq(Register dst, Register src) {
751 EnsureSpace ensure_space(this);
752 emit_rex_64(dst, src);
753 emit(0x0F);
754 emit(0xBD);
755 emit_modrm(dst, src);
756 }
757
758
bsrq(Register dst,const Operand & src)759 void Assembler::bsrq(Register dst, const Operand& src) {
760 EnsureSpace ensure_space(this);
761 emit_rex_64(dst, src);
762 emit(0x0F);
763 emit(0xBD);
764 emit_operand(dst, src);
765 }
766
767
bsfl(Register dst,Register src)768 void Assembler::bsfl(Register dst, Register src) {
769 EnsureSpace ensure_space(this);
770 emit_optional_rex_32(dst, src);
771 emit(0x0F);
772 emit(0xBC);
773 emit_modrm(dst, src);
774 }
775
776
bsfl(Register dst,const Operand & src)777 void Assembler::bsfl(Register dst, const Operand& src) {
778 EnsureSpace ensure_space(this);
779 emit_optional_rex_32(dst, src);
780 emit(0x0F);
781 emit(0xBC);
782 emit_operand(dst, src);
783 }
784
785
bsfq(Register dst,Register src)786 void Assembler::bsfq(Register dst, Register src) {
787 EnsureSpace ensure_space(this);
788 emit_rex_64(dst, src);
789 emit(0x0F);
790 emit(0xBC);
791 emit_modrm(dst, src);
792 }
793
794
bsfq(Register dst,const Operand & src)795 void Assembler::bsfq(Register dst, const Operand& src) {
796 EnsureSpace ensure_space(this);
797 emit_rex_64(dst, src);
798 emit(0x0F);
799 emit(0xBC);
800 emit_operand(dst, src);
801 }
802
803
call(Label * L)804 void Assembler::call(Label* L) {
805 positions_recorder()->WriteRecordedPositions();
806 EnsureSpace ensure_space(this);
807 // 1110 1000 #32-bit disp.
808 emit(0xE8);
809 if (L->is_bound()) {
810 int offset = L->pos() - pc_offset() - sizeof(int32_t);
811 DCHECK(offset <= 0);
812 emitl(offset);
813 } else if (L->is_linked()) {
814 emitl(L->pos());
815 L->link_to(pc_offset() - sizeof(int32_t));
816 } else {
817 DCHECK(L->is_unused());
818 int32_t current = pc_offset();
819 emitl(current);
820 L->link_to(current);
821 }
822 }
823
824
call(Address entry,RelocInfo::Mode rmode)825 void Assembler::call(Address entry, RelocInfo::Mode rmode) {
826 DCHECK(RelocInfo::IsRuntimeEntry(rmode));
827 positions_recorder()->WriteRecordedPositions();
828 EnsureSpace ensure_space(this);
829 // 1110 1000 #32-bit disp.
830 emit(0xE8);
831 emit_runtime_entry(entry, rmode);
832 }
833
834
call(Handle<Code> target,RelocInfo::Mode rmode,TypeFeedbackId ast_id)835 void Assembler::call(Handle<Code> target,
836 RelocInfo::Mode rmode,
837 TypeFeedbackId ast_id) {
838 positions_recorder()->WriteRecordedPositions();
839 EnsureSpace ensure_space(this);
840 // 1110 1000 #32-bit disp.
841 emit(0xE8);
842 emit_code_target(target, rmode, ast_id);
843 }
844
845
call(Register adr)846 void Assembler::call(Register adr) {
847 positions_recorder()->WriteRecordedPositions();
848 EnsureSpace ensure_space(this);
849 // Opcode: FF /2 r64.
850 emit_optional_rex_32(adr);
851 emit(0xFF);
852 emit_modrm(0x2, adr);
853 }
854
855
call(const Operand & op)856 void Assembler::call(const Operand& op) {
857 positions_recorder()->WriteRecordedPositions();
858 EnsureSpace ensure_space(this);
859 // Opcode: FF /2 m64.
860 emit_optional_rex_32(op);
861 emit(0xFF);
862 emit_operand(0x2, op);
863 }
864
865
866 // Calls directly to the given address using a relative offset.
867 // Should only ever be used in Code objects for calls within the
868 // same Code object. Should not be used when generating new code (use labels),
869 // but only when patching existing code.
call(Address target)870 void Assembler::call(Address target) {
871 positions_recorder()->WriteRecordedPositions();
872 EnsureSpace ensure_space(this);
873 // 1110 1000 #32-bit disp.
874 emit(0xE8);
875 Address source = pc_ + 4;
876 intptr_t displacement = target - source;
877 DCHECK(is_int32(displacement));
878 emitl(static_cast<int32_t>(displacement));
879 }
880
881
clc()882 void Assembler::clc() {
883 EnsureSpace ensure_space(this);
884 emit(0xF8);
885 }
886
887
cld()888 void Assembler::cld() {
889 EnsureSpace ensure_space(this);
890 emit(0xFC);
891 }
892
893
cdq()894 void Assembler::cdq() {
895 EnsureSpace ensure_space(this);
896 emit(0x99);
897 }
898
899
cmovq(Condition cc,Register dst,Register src)900 void Assembler::cmovq(Condition cc, Register dst, Register src) {
901 if (cc == always) {
902 movq(dst, src);
903 } else if (cc == never) {
904 return;
905 }
906 // No need to check CpuInfo for CMOV support, it's a required part of the
907 // 64-bit architecture.
908 DCHECK(cc >= 0); // Use mov for unconditional moves.
909 EnsureSpace ensure_space(this);
910 // Opcode: REX.W 0f 40 + cc /r.
911 emit_rex_64(dst, src);
912 emit(0x0f);
913 emit(0x40 + cc);
914 emit_modrm(dst, src);
915 }
916
917
cmovq(Condition cc,Register dst,const Operand & src)918 void Assembler::cmovq(Condition cc, Register dst, const Operand& src) {
919 if (cc == always) {
920 movq(dst, src);
921 } else if (cc == never) {
922 return;
923 }
924 DCHECK(cc >= 0);
925 EnsureSpace ensure_space(this);
926 // Opcode: REX.W 0f 40 + cc /r.
927 emit_rex_64(dst, src);
928 emit(0x0f);
929 emit(0x40 + cc);
930 emit_operand(dst, src);
931 }
932
933
cmovl(Condition cc,Register dst,Register src)934 void Assembler::cmovl(Condition cc, Register dst, Register src) {
935 if (cc == always) {
936 movl(dst, src);
937 } else if (cc == never) {
938 return;
939 }
940 DCHECK(cc >= 0);
941 EnsureSpace ensure_space(this);
942 // Opcode: 0f 40 + cc /r.
943 emit_optional_rex_32(dst, src);
944 emit(0x0f);
945 emit(0x40 + cc);
946 emit_modrm(dst, src);
947 }
948
949
cmovl(Condition cc,Register dst,const Operand & src)950 void Assembler::cmovl(Condition cc, Register dst, const Operand& src) {
951 if (cc == always) {
952 movl(dst, src);
953 } else if (cc == never) {
954 return;
955 }
956 DCHECK(cc >= 0);
957 EnsureSpace ensure_space(this);
958 // Opcode: 0f 40 + cc /r.
959 emit_optional_rex_32(dst, src);
960 emit(0x0f);
961 emit(0x40 + cc);
962 emit_operand(dst, src);
963 }
964
965
cmpb_al(Immediate imm8)966 void Assembler::cmpb_al(Immediate imm8) {
967 DCHECK(is_int8(imm8.value_) || is_uint8(imm8.value_));
968 EnsureSpace ensure_space(this);
969 emit(0x3c);
970 emit(imm8.value_);
971 }
972
973
cpuid()974 void Assembler::cpuid() {
975 EnsureSpace ensure_space(this);
976 emit(0x0F);
977 emit(0xA2);
978 }
979
980
cqo()981 void Assembler::cqo() {
982 EnsureSpace ensure_space(this);
983 emit_rex_64();
984 emit(0x99);
985 }
986
987
emit_dec(Register dst,int size)988 void Assembler::emit_dec(Register dst, int size) {
989 EnsureSpace ensure_space(this);
990 emit_rex(dst, size);
991 emit(0xFF);
992 emit_modrm(0x1, dst);
993 }
994
995
emit_dec(const Operand & dst,int size)996 void Assembler::emit_dec(const Operand& dst, int size) {
997 EnsureSpace ensure_space(this);
998 emit_rex(dst, size);
999 emit(0xFF);
1000 emit_operand(1, dst);
1001 }
1002
1003
decb(Register dst)1004 void Assembler::decb(Register dst) {
1005 EnsureSpace ensure_space(this);
1006 if (!dst.is_byte_register()) {
1007 // Register is not one of al, bl, cl, dl. Its encoding needs REX.
1008 emit_rex_32(dst);
1009 }
1010 emit(0xFE);
1011 emit_modrm(0x1, dst);
1012 }
1013
1014
decb(const Operand & dst)1015 void Assembler::decb(const Operand& dst) {
1016 EnsureSpace ensure_space(this);
1017 emit_optional_rex_32(dst);
1018 emit(0xFE);
1019 emit_operand(1, dst);
1020 }
1021
1022
enter(Immediate size)1023 void Assembler::enter(Immediate size) {
1024 EnsureSpace ensure_space(this);
1025 emit(0xC8);
1026 emitw(size.value_); // 16 bit operand, always.
1027 emit(0);
1028 }
1029
1030
hlt()1031 void Assembler::hlt() {
1032 EnsureSpace ensure_space(this);
1033 emit(0xF4);
1034 }
1035
1036
emit_idiv(Register src,int size)1037 void Assembler::emit_idiv(Register src, int size) {
1038 EnsureSpace ensure_space(this);
1039 emit_rex(src, size);
1040 emit(0xF7);
1041 emit_modrm(0x7, src);
1042 }
1043
1044
emit_div(Register src,int size)1045 void Assembler::emit_div(Register src, int size) {
1046 EnsureSpace ensure_space(this);
1047 emit_rex(src, size);
1048 emit(0xF7);
1049 emit_modrm(0x6, src);
1050 }
1051
1052
emit_imul(Register src,int size)1053 void Assembler::emit_imul(Register src, int size) {
1054 EnsureSpace ensure_space(this);
1055 emit_rex(src, size);
1056 emit(0xF7);
1057 emit_modrm(0x5, src);
1058 }
1059
1060
emit_imul(const Operand & src,int size)1061 void Assembler::emit_imul(const Operand& src, int size) {
1062 EnsureSpace ensure_space(this);
1063 emit_rex(src, size);
1064 emit(0xF7);
1065 emit_operand(0x5, src);
1066 }
1067
1068
emit_imul(Register dst,Register src,int size)1069 void Assembler::emit_imul(Register dst, Register src, int size) {
1070 EnsureSpace ensure_space(this);
1071 emit_rex(dst, src, size);
1072 emit(0x0F);
1073 emit(0xAF);
1074 emit_modrm(dst, src);
1075 }
1076
1077
emit_imul(Register dst,const Operand & src,int size)1078 void Assembler::emit_imul(Register dst, const Operand& src, int size) {
1079 EnsureSpace ensure_space(this);
1080 emit_rex(dst, src, size);
1081 emit(0x0F);
1082 emit(0xAF);
1083 emit_operand(dst, src);
1084 }
1085
1086
emit_imul(Register dst,Register src,Immediate imm,int size)1087 void Assembler::emit_imul(Register dst, Register src, Immediate imm, int size) {
1088 EnsureSpace ensure_space(this);
1089 emit_rex(dst, src, size);
1090 if (is_int8(imm.value_)) {
1091 emit(0x6B);
1092 emit_modrm(dst, src);
1093 emit(imm.value_);
1094 } else {
1095 emit(0x69);
1096 emit_modrm(dst, src);
1097 emitl(imm.value_);
1098 }
1099 }
1100
1101
emit_imul(Register dst,const Operand & src,Immediate imm,int size)1102 void Assembler::emit_imul(Register dst, const Operand& src, Immediate imm,
1103 int size) {
1104 EnsureSpace ensure_space(this);
1105 emit_rex(dst, src, size);
1106 if (is_int8(imm.value_)) {
1107 emit(0x6B);
1108 emit_operand(dst, src);
1109 emit(imm.value_);
1110 } else {
1111 emit(0x69);
1112 emit_operand(dst, src);
1113 emitl(imm.value_);
1114 }
1115 }
1116
1117
emit_inc(Register dst,int size)1118 void Assembler::emit_inc(Register dst, int size) {
1119 EnsureSpace ensure_space(this);
1120 emit_rex(dst, size);
1121 emit(0xFF);
1122 emit_modrm(0x0, dst);
1123 }
1124
1125
emit_inc(const Operand & dst,int size)1126 void Assembler::emit_inc(const Operand& dst, int size) {
1127 EnsureSpace ensure_space(this);
1128 emit_rex(dst, size);
1129 emit(0xFF);
1130 emit_operand(0, dst);
1131 }
1132
1133
int3()1134 void Assembler::int3() {
1135 EnsureSpace ensure_space(this);
1136 emit(0xCC);
1137 }
1138
1139
j(Condition cc,Label * L,Label::Distance distance)1140 void Assembler::j(Condition cc, Label* L, Label::Distance distance) {
1141 if (cc == always) {
1142 jmp(L);
1143 return;
1144 } else if (cc == never) {
1145 return;
1146 }
1147 EnsureSpace ensure_space(this);
1148 DCHECK(is_uint4(cc));
1149 if (L->is_bound()) {
1150 const int short_size = 2;
1151 const int long_size = 6;
1152 int offs = L->pos() - pc_offset();
1153 DCHECK(offs <= 0);
1154 // Determine whether we can use 1-byte offsets for backwards branches,
1155 // which have a max range of 128 bytes.
1156
1157 // We also need to check predictable_code_size() flag here, because on x64,
1158 // when the full code generator recompiles code for debugging, some places
1159 // need to be padded out to a certain size. The debugger is keeping track of
1160 // how often it did this so that it can adjust return addresses on the
1161 // stack, but if the size of jump instructions can also change, that's not
1162 // enough and the calculated offsets would be incorrect.
1163 if (is_int8(offs - short_size) && !predictable_code_size()) {
1164 // 0111 tttn #8-bit disp.
1165 emit(0x70 | cc);
1166 emit((offs - short_size) & 0xFF);
1167 } else {
1168 // 0000 1111 1000 tttn #32-bit disp.
1169 emit(0x0F);
1170 emit(0x80 | cc);
1171 emitl(offs - long_size);
1172 }
1173 } else if (distance == Label::kNear) {
1174 // 0111 tttn #8-bit disp
1175 emit(0x70 | cc);
1176 byte disp = 0x00;
1177 if (L->is_near_linked()) {
1178 int offset = L->near_link_pos() - pc_offset();
1179 DCHECK(is_int8(offset));
1180 disp = static_cast<byte>(offset & 0xFF);
1181 }
1182 L->link_to(pc_offset(), Label::kNear);
1183 emit(disp);
1184 } else if (L->is_linked()) {
1185 // 0000 1111 1000 tttn #32-bit disp.
1186 emit(0x0F);
1187 emit(0x80 | cc);
1188 emitl(L->pos());
1189 L->link_to(pc_offset() - sizeof(int32_t));
1190 } else {
1191 DCHECK(L->is_unused());
1192 emit(0x0F);
1193 emit(0x80 | cc);
1194 int32_t current = pc_offset();
1195 emitl(current);
1196 L->link_to(current);
1197 }
1198 }
1199
1200
j(Condition cc,Address entry,RelocInfo::Mode rmode)1201 void Assembler::j(Condition cc, Address entry, RelocInfo::Mode rmode) {
1202 DCHECK(RelocInfo::IsRuntimeEntry(rmode));
1203 EnsureSpace ensure_space(this);
1204 DCHECK(is_uint4(cc));
1205 emit(0x0F);
1206 emit(0x80 | cc);
1207 emit_runtime_entry(entry, rmode);
1208 }
1209
1210
j(Condition cc,Handle<Code> target,RelocInfo::Mode rmode)1211 void Assembler::j(Condition cc,
1212 Handle<Code> target,
1213 RelocInfo::Mode rmode) {
1214 EnsureSpace ensure_space(this);
1215 DCHECK(is_uint4(cc));
1216 // 0000 1111 1000 tttn #32-bit disp.
1217 emit(0x0F);
1218 emit(0x80 | cc);
1219 emit_code_target(target, rmode);
1220 }
1221
1222
jmp(Label * L,Label::Distance distance)1223 void Assembler::jmp(Label* L, Label::Distance distance) {
1224 EnsureSpace ensure_space(this);
1225 const int short_size = sizeof(int8_t);
1226 const int long_size = sizeof(int32_t);
1227 if (L->is_bound()) {
1228 int offs = L->pos() - pc_offset() - 1;
1229 DCHECK(offs <= 0);
1230 if (is_int8(offs - short_size) && !predictable_code_size()) {
1231 // 1110 1011 #8-bit disp.
1232 emit(0xEB);
1233 emit((offs - short_size) & 0xFF);
1234 } else {
1235 // 1110 1001 #32-bit disp.
1236 emit(0xE9);
1237 emitl(offs - long_size);
1238 }
1239 } else if (distance == Label::kNear) {
1240 emit(0xEB);
1241 byte disp = 0x00;
1242 if (L->is_near_linked()) {
1243 int offset = L->near_link_pos() - pc_offset();
1244 DCHECK(is_int8(offset));
1245 disp = static_cast<byte>(offset & 0xFF);
1246 }
1247 L->link_to(pc_offset(), Label::kNear);
1248 emit(disp);
1249 } else if (L->is_linked()) {
1250 // 1110 1001 #32-bit disp.
1251 emit(0xE9);
1252 emitl(L->pos());
1253 L->link_to(pc_offset() - long_size);
1254 } else {
1255 // 1110 1001 #32-bit disp.
1256 DCHECK(L->is_unused());
1257 emit(0xE9);
1258 int32_t current = pc_offset();
1259 emitl(current);
1260 L->link_to(current);
1261 }
1262 }
1263
1264
jmp(Handle<Code> target,RelocInfo::Mode rmode)1265 void Assembler::jmp(Handle<Code> target, RelocInfo::Mode rmode) {
1266 EnsureSpace ensure_space(this);
1267 // 1110 1001 #32-bit disp.
1268 emit(0xE9);
1269 emit_code_target(target, rmode);
1270 }
1271
1272
jmp(Address entry,RelocInfo::Mode rmode)1273 void Assembler::jmp(Address entry, RelocInfo::Mode rmode) {
1274 DCHECK(RelocInfo::IsRuntimeEntry(rmode));
1275 EnsureSpace ensure_space(this);
1276 DCHECK(RelocInfo::IsRuntimeEntry(rmode));
1277 emit(0xE9);
1278 emit_runtime_entry(entry, rmode);
1279 }
1280
1281
jmp(Register target)1282 void Assembler::jmp(Register target) {
1283 EnsureSpace ensure_space(this);
1284 // Opcode FF/4 r64.
1285 emit_optional_rex_32(target);
1286 emit(0xFF);
1287 emit_modrm(0x4, target);
1288 }
1289
1290
jmp(const Operand & src)1291 void Assembler::jmp(const Operand& src) {
1292 EnsureSpace ensure_space(this);
1293 // Opcode FF/4 m64.
1294 emit_optional_rex_32(src);
1295 emit(0xFF);
1296 emit_operand(0x4, src);
1297 }
1298
1299
emit_lea(Register dst,const Operand & src,int size)1300 void Assembler::emit_lea(Register dst, const Operand& src, int size) {
1301 EnsureSpace ensure_space(this);
1302 emit_rex(dst, src, size);
1303 emit(0x8D);
1304 emit_operand(dst, src);
1305 }
1306
1307
load_rax(void * value,RelocInfo::Mode mode)1308 void Assembler::load_rax(void* value, RelocInfo::Mode mode) {
1309 EnsureSpace ensure_space(this);
1310 if (kPointerSize == kInt64Size) {
1311 emit(0x48); // REX.W
1312 emit(0xA1);
1313 emitp(value, mode);
1314 } else {
1315 DCHECK(kPointerSize == kInt32Size);
1316 emit(0xA1);
1317 emitp(value, mode);
1318 // In 64-bit mode, need to zero extend the operand to 8 bytes.
1319 // See 2.2.1.4 in Intel64 and IA32 Architectures Software
1320 // Developer's Manual Volume 2.
1321 emitl(0);
1322 }
1323 }
1324
1325
load_rax(ExternalReference ref)1326 void Assembler::load_rax(ExternalReference ref) {
1327 load_rax(ref.address(), RelocInfo::EXTERNAL_REFERENCE);
1328 }
1329
1330
leave()1331 void Assembler::leave() {
1332 EnsureSpace ensure_space(this);
1333 emit(0xC9);
1334 }
1335
1336
movb(Register dst,const Operand & src)1337 void Assembler::movb(Register dst, const Operand& src) {
1338 EnsureSpace ensure_space(this);
1339 if (!dst.is_byte_register()) {
1340 // Register is not one of al, bl, cl, dl. Its encoding needs REX.
1341 emit_rex_32(dst, src);
1342 } else {
1343 emit_optional_rex_32(dst, src);
1344 }
1345 emit(0x8A);
1346 emit_operand(dst, src);
1347 }
1348
1349
movb(Register dst,Immediate imm)1350 void Assembler::movb(Register dst, Immediate imm) {
1351 EnsureSpace ensure_space(this);
1352 if (!dst.is_byte_register()) {
1353 // Register is not one of al, bl, cl, dl. Its encoding needs REX.
1354 emit_rex_32(dst);
1355 }
1356 emit(0xB0 + dst.low_bits());
1357 emit(imm.value_);
1358 }
1359
1360
movb(const Operand & dst,Register src)1361 void Assembler::movb(const Operand& dst, Register src) {
1362 EnsureSpace ensure_space(this);
1363 if (!src.is_byte_register()) {
1364 // Register is not one of al, bl, cl, dl. Its encoding needs REX.
1365 emit_rex_32(src, dst);
1366 } else {
1367 emit_optional_rex_32(src, dst);
1368 }
1369 emit(0x88);
1370 emit_operand(src, dst);
1371 }
1372
1373
movb(const Operand & dst,Immediate imm)1374 void Assembler::movb(const Operand& dst, Immediate imm) {
1375 EnsureSpace ensure_space(this);
1376 emit_optional_rex_32(dst);
1377 emit(0xC6);
1378 emit_operand(0x0, dst);
1379 emit(static_cast<byte>(imm.value_));
1380 }
1381
1382
movw(Register dst,const Operand & src)1383 void Assembler::movw(Register dst, const Operand& src) {
1384 EnsureSpace ensure_space(this);
1385 emit(0x66);
1386 emit_optional_rex_32(dst, src);
1387 emit(0x8B);
1388 emit_operand(dst, src);
1389 }
1390
1391
movw(const Operand & dst,Register src)1392 void Assembler::movw(const Operand& dst, Register src) {
1393 EnsureSpace ensure_space(this);
1394 emit(0x66);
1395 emit_optional_rex_32(src, dst);
1396 emit(0x89);
1397 emit_operand(src, dst);
1398 }
1399
1400
movw(const Operand & dst,Immediate imm)1401 void Assembler::movw(const Operand& dst, Immediate imm) {
1402 EnsureSpace ensure_space(this);
1403 emit(0x66);
1404 emit_optional_rex_32(dst);
1405 emit(0xC7);
1406 emit_operand(0x0, dst);
1407 emit(static_cast<byte>(imm.value_ & 0xff));
1408 emit(static_cast<byte>(imm.value_ >> 8));
1409 }
1410
1411
emit_mov(Register dst,const Operand & src,int size)1412 void Assembler::emit_mov(Register dst, const Operand& src, int size) {
1413 EnsureSpace ensure_space(this);
1414 emit_rex(dst, src, size);
1415 emit(0x8B);
1416 emit_operand(dst, src);
1417 }
1418
1419
emit_mov(Register dst,Register src,int size)1420 void Assembler::emit_mov(Register dst, Register src, int size) {
1421 EnsureSpace ensure_space(this);
1422 if (src.low_bits() == 4) {
1423 emit_rex(src, dst, size);
1424 emit(0x89);
1425 emit_modrm(src, dst);
1426 } else {
1427 emit_rex(dst, src, size);
1428 emit(0x8B);
1429 emit_modrm(dst, src);
1430 }
1431 }
1432
1433
emit_mov(const Operand & dst,Register src,int size)1434 void Assembler::emit_mov(const Operand& dst, Register src, int size) {
1435 EnsureSpace ensure_space(this);
1436 emit_rex(src, dst, size);
1437 emit(0x89);
1438 emit_operand(src, dst);
1439 }
1440
1441
emit_mov(Register dst,Immediate value,int size)1442 void Assembler::emit_mov(Register dst, Immediate value, int size) {
1443 EnsureSpace ensure_space(this);
1444 emit_rex(dst, size);
1445 if (size == kInt64Size) {
1446 emit(0xC7);
1447 emit_modrm(0x0, dst);
1448 } else {
1449 DCHECK(size == kInt32Size);
1450 emit(0xB8 + dst.low_bits());
1451 }
1452 emit(value);
1453 }
1454
1455
emit_mov(const Operand & dst,Immediate value,int size)1456 void Assembler::emit_mov(const Operand& dst, Immediate value, int size) {
1457 EnsureSpace ensure_space(this);
1458 emit_rex(dst, size);
1459 emit(0xC7);
1460 emit_operand(0x0, dst);
1461 emit(value);
1462 }
1463
1464
movp(Register dst,void * value,RelocInfo::Mode rmode)1465 void Assembler::movp(Register dst, void* value, RelocInfo::Mode rmode) {
1466 EnsureSpace ensure_space(this);
1467 emit_rex(dst, kPointerSize);
1468 emit(0xB8 | dst.low_bits());
1469 emitp(value, rmode);
1470 }
1471
1472
movq(Register dst,int64_t value)1473 void Assembler::movq(Register dst, int64_t value) {
1474 EnsureSpace ensure_space(this);
1475 emit_rex_64(dst);
1476 emit(0xB8 | dst.low_bits());
1477 emitq(value);
1478 }
1479
1480
movq(Register dst,uint64_t value)1481 void Assembler::movq(Register dst, uint64_t value) {
1482 movq(dst, static_cast<int64_t>(value));
1483 }
1484
1485
1486 // Loads the ip-relative location of the src label into the target location
1487 // (as a 32-bit offset sign extended to 64-bit).
movl(const Operand & dst,Label * src)1488 void Assembler::movl(const Operand& dst, Label* src) {
1489 EnsureSpace ensure_space(this);
1490 emit_optional_rex_32(dst);
1491 emit(0xC7);
1492 emit_operand(0, dst);
1493 if (src->is_bound()) {
1494 int offset = src->pos() - pc_offset() - sizeof(int32_t);
1495 DCHECK(offset <= 0);
1496 emitl(offset);
1497 } else if (src->is_linked()) {
1498 emitl(src->pos());
1499 src->link_to(pc_offset() - sizeof(int32_t));
1500 } else {
1501 DCHECK(src->is_unused());
1502 int32_t current = pc_offset();
1503 emitl(current);
1504 src->link_to(current);
1505 }
1506 }
1507
1508
movsxbl(Register dst,Register src)1509 void Assembler::movsxbl(Register dst, Register src) {
1510 EnsureSpace ensure_space(this);
1511 if (!src.is_byte_register()) {
1512 // Register is not one of al, bl, cl, dl. Its encoding needs REX.
1513 emit_rex_32(dst, src);
1514 } else {
1515 emit_optional_rex_32(dst, src);
1516 }
1517 emit(0x0F);
1518 emit(0xBE);
1519 emit_modrm(dst, src);
1520 }
1521
1522
movsxbl(Register dst,const Operand & src)1523 void Assembler::movsxbl(Register dst, const Operand& src) {
1524 EnsureSpace ensure_space(this);
1525 emit_optional_rex_32(dst, src);
1526 emit(0x0F);
1527 emit(0xBE);
1528 emit_operand(dst, src);
1529 }
1530
1531
movsxbq(Register dst,const Operand & src)1532 void Assembler::movsxbq(Register dst, const Operand& src) {
1533 EnsureSpace ensure_space(this);
1534 emit_rex_64(dst, src);
1535 emit(0x0F);
1536 emit(0xBE);
1537 emit_operand(dst, src);
1538 }
1539
1540
movsxwl(Register dst,Register src)1541 void Assembler::movsxwl(Register dst, Register src) {
1542 EnsureSpace ensure_space(this);
1543 emit_optional_rex_32(dst, src);
1544 emit(0x0F);
1545 emit(0xBF);
1546 emit_modrm(dst, src);
1547 }
1548
1549
movsxwl(Register dst,const Operand & src)1550 void Assembler::movsxwl(Register dst, const Operand& src) {
1551 EnsureSpace ensure_space(this);
1552 emit_optional_rex_32(dst, src);
1553 emit(0x0F);
1554 emit(0xBF);
1555 emit_operand(dst, src);
1556 }
1557
1558
movsxwq(Register dst,const Operand & src)1559 void Assembler::movsxwq(Register dst, const Operand& src) {
1560 EnsureSpace ensure_space(this);
1561 emit_rex_64(dst, src);
1562 emit(0x0F);
1563 emit(0xBF);
1564 emit_operand(dst, src);
1565 }
1566
1567
movsxlq(Register dst,Register src)1568 void Assembler::movsxlq(Register dst, Register src) {
1569 EnsureSpace ensure_space(this);
1570 emit_rex_64(dst, src);
1571 emit(0x63);
1572 emit_modrm(dst, src);
1573 }
1574
1575
movsxlq(Register dst,const Operand & src)1576 void Assembler::movsxlq(Register dst, const Operand& src) {
1577 EnsureSpace ensure_space(this);
1578 emit_rex_64(dst, src);
1579 emit(0x63);
1580 emit_operand(dst, src);
1581 }
1582
1583
emit_movzxb(Register dst,const Operand & src,int size)1584 void Assembler::emit_movzxb(Register dst, const Operand& src, int size) {
1585 EnsureSpace ensure_space(this);
1586 // 32 bit operations zero the top 32 bits of 64 bit registers. Therefore
1587 // there is no need to make this a 64 bit operation.
1588 emit_optional_rex_32(dst, src);
1589 emit(0x0F);
1590 emit(0xB6);
1591 emit_operand(dst, src);
1592 }
1593
1594
emit_movzxb(Register dst,Register src,int size)1595 void Assembler::emit_movzxb(Register dst, Register src, int size) {
1596 EnsureSpace ensure_space(this);
1597 // 32 bit operations zero the top 32 bits of 64 bit registers. Therefore
1598 // there is no need to make this a 64 bit operation.
1599 if (!src.is_byte_register()) {
1600 // Register is not one of al, bl, cl, dl. Its encoding needs REX.
1601 emit_rex_32(dst, src);
1602 } else {
1603 emit_optional_rex_32(dst, src);
1604 }
1605 emit(0x0F);
1606 emit(0xB6);
1607 emit_modrm(dst, src);
1608 }
1609
1610
emit_movzxw(Register dst,const Operand & src,int size)1611 void Assembler::emit_movzxw(Register dst, const Operand& src, int size) {
1612 EnsureSpace ensure_space(this);
1613 // 32 bit operations zero the top 32 bits of 64 bit registers. Therefore
1614 // there is no need to make this a 64 bit operation.
1615 emit_optional_rex_32(dst, src);
1616 emit(0x0F);
1617 emit(0xB7);
1618 emit_operand(dst, src);
1619 }
1620
1621
emit_movzxw(Register dst,Register src,int size)1622 void Assembler::emit_movzxw(Register dst, Register src, int size) {
1623 EnsureSpace ensure_space(this);
1624 // 32 bit operations zero the top 32 bits of 64 bit registers. Therefore
1625 // there is no need to make this a 64 bit operation.
1626 emit_optional_rex_32(dst, src);
1627 emit(0x0F);
1628 emit(0xB7);
1629 emit_modrm(dst, src);
1630 }
1631
1632
repmovsb()1633 void Assembler::repmovsb() {
1634 EnsureSpace ensure_space(this);
1635 emit(0xF3);
1636 emit(0xA4);
1637 }
1638
1639
repmovsw()1640 void Assembler::repmovsw() {
1641 EnsureSpace ensure_space(this);
1642 emit(0x66); // Operand size override.
1643 emit(0xF3);
1644 emit(0xA4);
1645 }
1646
1647
emit_repmovs(int size)1648 void Assembler::emit_repmovs(int size) {
1649 EnsureSpace ensure_space(this);
1650 emit(0xF3);
1651 emit_rex(size);
1652 emit(0xA5);
1653 }
1654
1655
mull(Register src)1656 void Assembler::mull(Register src) {
1657 EnsureSpace ensure_space(this);
1658 emit_optional_rex_32(src);
1659 emit(0xF7);
1660 emit_modrm(0x4, src);
1661 }
1662
1663
mull(const Operand & src)1664 void Assembler::mull(const Operand& src) {
1665 EnsureSpace ensure_space(this);
1666 emit_optional_rex_32(src);
1667 emit(0xF7);
1668 emit_operand(0x4, src);
1669 }
1670
1671
mulq(Register src)1672 void Assembler::mulq(Register src) {
1673 EnsureSpace ensure_space(this);
1674 emit_rex_64(src);
1675 emit(0xF7);
1676 emit_modrm(0x4, src);
1677 }
1678
1679
emit_neg(Register dst,int size)1680 void Assembler::emit_neg(Register dst, int size) {
1681 EnsureSpace ensure_space(this);
1682 emit_rex(dst, size);
1683 emit(0xF7);
1684 emit_modrm(0x3, dst);
1685 }
1686
1687
emit_neg(const Operand & dst,int size)1688 void Assembler::emit_neg(const Operand& dst, int size) {
1689 EnsureSpace ensure_space(this);
1690 emit_rex_64(dst);
1691 emit(0xF7);
1692 emit_operand(3, dst);
1693 }
1694
1695
nop()1696 void Assembler::nop() {
1697 EnsureSpace ensure_space(this);
1698 emit(0x90);
1699 }
1700
1701
emit_not(Register dst,int size)1702 void Assembler::emit_not(Register dst, int size) {
1703 EnsureSpace ensure_space(this);
1704 emit_rex(dst, size);
1705 emit(0xF7);
1706 emit_modrm(0x2, dst);
1707 }
1708
1709
emit_not(const Operand & dst,int size)1710 void Assembler::emit_not(const Operand& dst, int size) {
1711 EnsureSpace ensure_space(this);
1712 emit_rex(dst, size);
1713 emit(0xF7);
1714 emit_operand(2, dst);
1715 }
1716
1717
Nop(int n)1718 void Assembler::Nop(int n) {
1719 // The recommended muti-byte sequences of NOP instructions from the Intel 64
1720 // and IA-32 Architectures Software Developer's Manual.
1721 //
1722 // Length Assembly Byte Sequence
1723 // 2 bytes 66 NOP 66 90H
1724 // 3 bytes NOP DWORD ptr [EAX] 0F 1F 00H
1725 // 4 bytes NOP DWORD ptr [EAX + 00H] 0F 1F 40 00H
1726 // 5 bytes NOP DWORD ptr [EAX + EAX*1 + 00H] 0F 1F 44 00 00H
1727 // 6 bytes 66 NOP DWORD ptr [EAX + EAX*1 + 00H] 66 0F 1F 44 00 00H
1728 // 7 bytes NOP DWORD ptr [EAX + 00000000H] 0F 1F 80 00 00 00 00H
1729 // 8 bytes NOP DWORD ptr [EAX + EAX*1 + 00000000H] 0F 1F 84 00 00 00 00 00H
1730 // 9 bytes 66 NOP DWORD ptr [EAX + EAX*1 + 66 0F 1F 84 00 00 00 00
1731 // 00000000H] 00H
1732
1733 EnsureSpace ensure_space(this);
1734 while (n > 0) {
1735 switch (n) {
1736 case 2:
1737 emit(0x66);
1738 case 1:
1739 emit(0x90);
1740 return;
1741 case 3:
1742 emit(0x0f);
1743 emit(0x1f);
1744 emit(0x00);
1745 return;
1746 case 4:
1747 emit(0x0f);
1748 emit(0x1f);
1749 emit(0x40);
1750 emit(0x00);
1751 return;
1752 case 6:
1753 emit(0x66);
1754 case 5:
1755 emit(0x0f);
1756 emit(0x1f);
1757 emit(0x44);
1758 emit(0x00);
1759 emit(0x00);
1760 return;
1761 case 7:
1762 emit(0x0f);
1763 emit(0x1f);
1764 emit(0x80);
1765 emit(0x00);
1766 emit(0x00);
1767 emit(0x00);
1768 emit(0x00);
1769 return;
1770 default:
1771 case 11:
1772 emit(0x66);
1773 n--;
1774 case 10:
1775 emit(0x66);
1776 n--;
1777 case 9:
1778 emit(0x66);
1779 n--;
1780 case 8:
1781 emit(0x0f);
1782 emit(0x1f);
1783 emit(0x84);
1784 emit(0x00);
1785 emit(0x00);
1786 emit(0x00);
1787 emit(0x00);
1788 emit(0x00);
1789 n -= 8;
1790 }
1791 }
1792 }
1793
1794
popq(Register dst)1795 void Assembler::popq(Register dst) {
1796 EnsureSpace ensure_space(this);
1797 emit_optional_rex_32(dst);
1798 emit(0x58 | dst.low_bits());
1799 }
1800
1801
popq(const Operand & dst)1802 void Assembler::popq(const Operand& dst) {
1803 EnsureSpace ensure_space(this);
1804 emit_optional_rex_32(dst);
1805 emit(0x8F);
1806 emit_operand(0, dst);
1807 }
1808
1809
popfq()1810 void Assembler::popfq() {
1811 EnsureSpace ensure_space(this);
1812 emit(0x9D);
1813 }
1814
1815
pushq(Register src)1816 void Assembler::pushq(Register src) {
1817 EnsureSpace ensure_space(this);
1818 emit_optional_rex_32(src);
1819 emit(0x50 | src.low_bits());
1820 }
1821
1822
pushq(const Operand & src)1823 void Assembler::pushq(const Operand& src) {
1824 EnsureSpace ensure_space(this);
1825 emit_optional_rex_32(src);
1826 emit(0xFF);
1827 emit_operand(6, src);
1828 }
1829
1830
pushq(Immediate value)1831 void Assembler::pushq(Immediate value) {
1832 EnsureSpace ensure_space(this);
1833 if (is_int8(value.value_)) {
1834 emit(0x6A);
1835 emit(value.value_); // Emit low byte of value.
1836 } else {
1837 emit(0x68);
1838 emitl(value.value_);
1839 }
1840 }
1841
1842
pushq_imm32(int32_t imm32)1843 void Assembler::pushq_imm32(int32_t imm32) {
1844 EnsureSpace ensure_space(this);
1845 emit(0x68);
1846 emitl(imm32);
1847 }
1848
1849
pushfq()1850 void Assembler::pushfq() {
1851 EnsureSpace ensure_space(this);
1852 emit(0x9C);
1853 }
1854
1855
ret(int imm16)1856 void Assembler::ret(int imm16) {
1857 EnsureSpace ensure_space(this);
1858 DCHECK(is_uint16(imm16));
1859 if (imm16 == 0) {
1860 emit(0xC3);
1861 } else {
1862 emit(0xC2);
1863 emit(imm16 & 0xFF);
1864 emit((imm16 >> 8) & 0xFF);
1865 }
1866 }
1867
1868
ud2()1869 void Assembler::ud2() {
1870 EnsureSpace ensure_space(this);
1871 emit(0x0F);
1872 emit(0x0B);
1873 }
1874
1875
setcc(Condition cc,Register reg)1876 void Assembler::setcc(Condition cc, Register reg) {
1877 if (cc > last_condition) {
1878 movb(reg, Immediate(cc == always ? 1 : 0));
1879 return;
1880 }
1881 EnsureSpace ensure_space(this);
1882 DCHECK(is_uint4(cc));
1883 if (!reg.is_byte_register()) {
1884 // Register is not one of al, bl, cl, dl. Its encoding needs REX.
1885 emit_rex_32(reg);
1886 }
1887 emit(0x0F);
1888 emit(0x90 | cc);
1889 emit_modrm(0x0, reg);
1890 }
1891
1892
shld(Register dst,Register src)1893 void Assembler::shld(Register dst, Register src) {
1894 EnsureSpace ensure_space(this);
1895 emit_rex_64(src, dst);
1896 emit(0x0F);
1897 emit(0xA5);
1898 emit_modrm(src, dst);
1899 }
1900
1901
shrd(Register dst,Register src)1902 void Assembler::shrd(Register dst, Register src) {
1903 EnsureSpace ensure_space(this);
1904 emit_rex_64(src, dst);
1905 emit(0x0F);
1906 emit(0xAD);
1907 emit_modrm(src, dst);
1908 }
1909
1910
emit_xchg(Register dst,Register src,int size)1911 void Assembler::emit_xchg(Register dst, Register src, int size) {
1912 EnsureSpace ensure_space(this);
1913 if (src.is(rax) || dst.is(rax)) { // Single-byte encoding
1914 Register other = src.is(rax) ? dst : src;
1915 emit_rex(other, size);
1916 emit(0x90 | other.low_bits());
1917 } else if (dst.low_bits() == 4) {
1918 emit_rex(dst, src, size);
1919 emit(0x87);
1920 emit_modrm(dst, src);
1921 } else {
1922 emit_rex(src, dst, size);
1923 emit(0x87);
1924 emit_modrm(src, dst);
1925 }
1926 }
1927
1928
emit_xchg(Register dst,const Operand & src,int size)1929 void Assembler::emit_xchg(Register dst, const Operand& src, int size) {
1930 EnsureSpace ensure_space(this);
1931 emit_rex(dst, src, size);
1932 emit(0x87);
1933 emit_operand(dst, src);
1934 }
1935
1936
store_rax(void * dst,RelocInfo::Mode mode)1937 void Assembler::store_rax(void* dst, RelocInfo::Mode mode) {
1938 EnsureSpace ensure_space(this);
1939 if (kPointerSize == kInt64Size) {
1940 emit(0x48); // REX.W
1941 emit(0xA3);
1942 emitp(dst, mode);
1943 } else {
1944 DCHECK(kPointerSize == kInt32Size);
1945 emit(0xA3);
1946 emitp(dst, mode);
1947 // In 64-bit mode, need to zero extend the operand to 8 bytes.
1948 // See 2.2.1.4 in Intel64 and IA32 Architectures Software
1949 // Developer's Manual Volume 2.
1950 emitl(0);
1951 }
1952 }
1953
1954
store_rax(ExternalReference ref)1955 void Assembler::store_rax(ExternalReference ref) {
1956 store_rax(ref.address(), RelocInfo::EXTERNAL_REFERENCE);
1957 }
1958
1959
testb(Register dst,Register src)1960 void Assembler::testb(Register dst, Register src) {
1961 EnsureSpace ensure_space(this);
1962 if (src.low_bits() == 4) {
1963 emit_rex_32(src, dst);
1964 emit(0x84);
1965 emit_modrm(src, dst);
1966 } else {
1967 if (!dst.is_byte_register() || !src.is_byte_register()) {
1968 // Register is not one of al, bl, cl, dl. Its encoding needs REX.
1969 emit_rex_32(dst, src);
1970 }
1971 emit(0x84);
1972 emit_modrm(dst, src);
1973 }
1974 }
1975
1976
testb(Register reg,Immediate mask)1977 void Assembler::testb(Register reg, Immediate mask) {
1978 DCHECK(is_int8(mask.value_) || is_uint8(mask.value_));
1979 EnsureSpace ensure_space(this);
1980 if (reg.is(rax)) {
1981 emit(0xA8);
1982 emit(mask.value_); // Low byte emitted.
1983 } else {
1984 if (!reg.is_byte_register()) {
1985 // Register is not one of al, bl, cl, dl. Its encoding needs REX.
1986 emit_rex_32(reg);
1987 }
1988 emit(0xF6);
1989 emit_modrm(0x0, reg);
1990 emit(mask.value_); // Low byte emitted.
1991 }
1992 }
1993
1994
testb(const Operand & op,Immediate mask)1995 void Assembler::testb(const Operand& op, Immediate mask) {
1996 DCHECK(is_int8(mask.value_) || is_uint8(mask.value_));
1997 EnsureSpace ensure_space(this);
1998 emit_optional_rex_32(rax, op);
1999 emit(0xF6);
2000 emit_operand(rax, op); // Operation code 0
2001 emit(mask.value_); // Low byte emitted.
2002 }
2003
2004
testb(const Operand & op,Register reg)2005 void Assembler::testb(const Operand& op, Register reg) {
2006 EnsureSpace ensure_space(this);
2007 if (!reg.is_byte_register()) {
2008 // Register is not one of al, bl, cl, dl. Its encoding needs REX.
2009 emit_rex_32(reg, op);
2010 } else {
2011 emit_optional_rex_32(reg, op);
2012 }
2013 emit(0x84);
2014 emit_operand(reg, op);
2015 }
2016
2017
emit_test(Register dst,Register src,int size)2018 void Assembler::emit_test(Register dst, Register src, int size) {
2019 EnsureSpace ensure_space(this);
2020 if (src.low_bits() == 4) {
2021 emit_rex(src, dst, size);
2022 emit(0x85);
2023 emit_modrm(src, dst);
2024 } else {
2025 emit_rex(dst, src, size);
2026 emit(0x85);
2027 emit_modrm(dst, src);
2028 }
2029 }
2030
2031
emit_test(Register reg,Immediate mask,int size)2032 void Assembler::emit_test(Register reg, Immediate mask, int size) {
2033 // testl with a mask that fits in the low byte is exactly testb.
2034 if (is_uint8(mask.value_)) {
2035 testb(reg, mask);
2036 return;
2037 }
2038 EnsureSpace ensure_space(this);
2039 if (reg.is(rax)) {
2040 emit_rex(rax, size);
2041 emit(0xA9);
2042 emit(mask);
2043 } else {
2044 emit_rex(reg, size);
2045 emit(0xF7);
2046 emit_modrm(0x0, reg);
2047 emit(mask);
2048 }
2049 }
2050
2051
emit_test(const Operand & op,Immediate mask,int size)2052 void Assembler::emit_test(const Operand& op, Immediate mask, int size) {
2053 // testl with a mask that fits in the low byte is exactly testb.
2054 if (is_uint8(mask.value_)) {
2055 testb(op, mask);
2056 return;
2057 }
2058 EnsureSpace ensure_space(this);
2059 emit_rex(rax, op, size);
2060 emit(0xF7);
2061 emit_operand(rax, op); // Operation code 0
2062 emit(mask);
2063 }
2064
2065
emit_test(const Operand & op,Register reg,int size)2066 void Assembler::emit_test(const Operand& op, Register reg, int size) {
2067 EnsureSpace ensure_space(this);
2068 emit_rex(reg, op, size);
2069 emit(0x85);
2070 emit_operand(reg, op);
2071 }
2072
2073
2074 // FPU instructions.
2075
2076
fld(int i)2077 void Assembler::fld(int i) {
2078 EnsureSpace ensure_space(this);
2079 emit_farith(0xD9, 0xC0, i);
2080 }
2081
2082
fld1()2083 void Assembler::fld1() {
2084 EnsureSpace ensure_space(this);
2085 emit(0xD9);
2086 emit(0xE8);
2087 }
2088
2089
fldz()2090 void Assembler::fldz() {
2091 EnsureSpace ensure_space(this);
2092 emit(0xD9);
2093 emit(0xEE);
2094 }
2095
2096
fldpi()2097 void Assembler::fldpi() {
2098 EnsureSpace ensure_space(this);
2099 emit(0xD9);
2100 emit(0xEB);
2101 }
2102
2103
fldln2()2104 void Assembler::fldln2() {
2105 EnsureSpace ensure_space(this);
2106 emit(0xD9);
2107 emit(0xED);
2108 }
2109
2110
fld_s(const Operand & adr)2111 void Assembler::fld_s(const Operand& adr) {
2112 EnsureSpace ensure_space(this);
2113 emit_optional_rex_32(adr);
2114 emit(0xD9);
2115 emit_operand(0, adr);
2116 }
2117
2118
fld_d(const Operand & adr)2119 void Assembler::fld_d(const Operand& adr) {
2120 EnsureSpace ensure_space(this);
2121 emit_optional_rex_32(adr);
2122 emit(0xDD);
2123 emit_operand(0, adr);
2124 }
2125
2126
fstp_s(const Operand & adr)2127 void Assembler::fstp_s(const Operand& adr) {
2128 EnsureSpace ensure_space(this);
2129 emit_optional_rex_32(adr);
2130 emit(0xD9);
2131 emit_operand(3, adr);
2132 }
2133
2134
fstp_d(const Operand & adr)2135 void Assembler::fstp_d(const Operand& adr) {
2136 EnsureSpace ensure_space(this);
2137 emit_optional_rex_32(adr);
2138 emit(0xDD);
2139 emit_operand(3, adr);
2140 }
2141
2142
fstp(int index)2143 void Assembler::fstp(int index) {
2144 DCHECK(is_uint3(index));
2145 EnsureSpace ensure_space(this);
2146 emit_farith(0xDD, 0xD8, index);
2147 }
2148
2149
fild_s(const Operand & adr)2150 void Assembler::fild_s(const Operand& adr) {
2151 EnsureSpace ensure_space(this);
2152 emit_optional_rex_32(adr);
2153 emit(0xDB);
2154 emit_operand(0, adr);
2155 }
2156
2157
fild_d(const Operand & adr)2158 void Assembler::fild_d(const Operand& adr) {
2159 EnsureSpace ensure_space(this);
2160 emit_optional_rex_32(adr);
2161 emit(0xDF);
2162 emit_operand(5, adr);
2163 }
2164
2165
fistp_s(const Operand & adr)2166 void Assembler::fistp_s(const Operand& adr) {
2167 EnsureSpace ensure_space(this);
2168 emit_optional_rex_32(adr);
2169 emit(0xDB);
2170 emit_operand(3, adr);
2171 }
2172
2173
fisttp_s(const Operand & adr)2174 void Assembler::fisttp_s(const Operand& adr) {
2175 DCHECK(IsEnabled(SSE3));
2176 EnsureSpace ensure_space(this);
2177 emit_optional_rex_32(adr);
2178 emit(0xDB);
2179 emit_operand(1, adr);
2180 }
2181
2182
fisttp_d(const Operand & adr)2183 void Assembler::fisttp_d(const Operand& adr) {
2184 DCHECK(IsEnabled(SSE3));
2185 EnsureSpace ensure_space(this);
2186 emit_optional_rex_32(adr);
2187 emit(0xDD);
2188 emit_operand(1, adr);
2189 }
2190
2191
fist_s(const Operand & adr)2192 void Assembler::fist_s(const Operand& adr) {
2193 EnsureSpace ensure_space(this);
2194 emit_optional_rex_32(adr);
2195 emit(0xDB);
2196 emit_operand(2, adr);
2197 }
2198
2199
fistp_d(const Operand & adr)2200 void Assembler::fistp_d(const Operand& adr) {
2201 EnsureSpace ensure_space(this);
2202 emit_optional_rex_32(adr);
2203 emit(0xDF);
2204 emit_operand(7, adr);
2205 }
2206
2207
fabs()2208 void Assembler::fabs() {
2209 EnsureSpace ensure_space(this);
2210 emit(0xD9);
2211 emit(0xE1);
2212 }
2213
2214
fchs()2215 void Assembler::fchs() {
2216 EnsureSpace ensure_space(this);
2217 emit(0xD9);
2218 emit(0xE0);
2219 }
2220
2221
fcos()2222 void Assembler::fcos() {
2223 EnsureSpace ensure_space(this);
2224 emit(0xD9);
2225 emit(0xFF);
2226 }
2227
2228
fsin()2229 void Assembler::fsin() {
2230 EnsureSpace ensure_space(this);
2231 emit(0xD9);
2232 emit(0xFE);
2233 }
2234
2235
fptan()2236 void Assembler::fptan() {
2237 EnsureSpace ensure_space(this);
2238 emit(0xD9);
2239 emit(0xF2);
2240 }
2241
2242
fyl2x()2243 void Assembler::fyl2x() {
2244 EnsureSpace ensure_space(this);
2245 emit(0xD9);
2246 emit(0xF1);
2247 }
2248
2249
f2xm1()2250 void Assembler::f2xm1() {
2251 EnsureSpace ensure_space(this);
2252 emit(0xD9);
2253 emit(0xF0);
2254 }
2255
2256
fscale()2257 void Assembler::fscale() {
2258 EnsureSpace ensure_space(this);
2259 emit(0xD9);
2260 emit(0xFD);
2261 }
2262
2263
fninit()2264 void Assembler::fninit() {
2265 EnsureSpace ensure_space(this);
2266 emit(0xDB);
2267 emit(0xE3);
2268 }
2269
2270
fadd(int i)2271 void Assembler::fadd(int i) {
2272 EnsureSpace ensure_space(this);
2273 emit_farith(0xDC, 0xC0, i);
2274 }
2275
2276
fsub(int i)2277 void Assembler::fsub(int i) {
2278 EnsureSpace ensure_space(this);
2279 emit_farith(0xDC, 0xE8, i);
2280 }
2281
2282
fisub_s(const Operand & adr)2283 void Assembler::fisub_s(const Operand& adr) {
2284 EnsureSpace ensure_space(this);
2285 emit_optional_rex_32(adr);
2286 emit(0xDA);
2287 emit_operand(4, adr);
2288 }
2289
2290
fmul(int i)2291 void Assembler::fmul(int i) {
2292 EnsureSpace ensure_space(this);
2293 emit_farith(0xDC, 0xC8, i);
2294 }
2295
2296
fdiv(int i)2297 void Assembler::fdiv(int i) {
2298 EnsureSpace ensure_space(this);
2299 emit_farith(0xDC, 0xF8, i);
2300 }
2301
2302
faddp(int i)2303 void Assembler::faddp(int i) {
2304 EnsureSpace ensure_space(this);
2305 emit_farith(0xDE, 0xC0, i);
2306 }
2307
2308
fsubp(int i)2309 void Assembler::fsubp(int i) {
2310 EnsureSpace ensure_space(this);
2311 emit_farith(0xDE, 0xE8, i);
2312 }
2313
2314
fsubrp(int i)2315 void Assembler::fsubrp(int i) {
2316 EnsureSpace ensure_space(this);
2317 emit_farith(0xDE, 0xE0, i);
2318 }
2319
2320
fmulp(int i)2321 void Assembler::fmulp(int i) {
2322 EnsureSpace ensure_space(this);
2323 emit_farith(0xDE, 0xC8, i);
2324 }
2325
2326
fdivp(int i)2327 void Assembler::fdivp(int i) {
2328 EnsureSpace ensure_space(this);
2329 emit_farith(0xDE, 0xF8, i);
2330 }
2331
2332
fprem()2333 void Assembler::fprem() {
2334 EnsureSpace ensure_space(this);
2335 emit(0xD9);
2336 emit(0xF8);
2337 }
2338
2339
fprem1()2340 void Assembler::fprem1() {
2341 EnsureSpace ensure_space(this);
2342 emit(0xD9);
2343 emit(0xF5);
2344 }
2345
2346
fxch(int i)2347 void Assembler::fxch(int i) {
2348 EnsureSpace ensure_space(this);
2349 emit_farith(0xD9, 0xC8, i);
2350 }
2351
2352
fincstp()2353 void Assembler::fincstp() {
2354 EnsureSpace ensure_space(this);
2355 emit(0xD9);
2356 emit(0xF7);
2357 }
2358
2359
ffree(int i)2360 void Assembler::ffree(int i) {
2361 EnsureSpace ensure_space(this);
2362 emit_farith(0xDD, 0xC0, i);
2363 }
2364
2365
ftst()2366 void Assembler::ftst() {
2367 EnsureSpace ensure_space(this);
2368 emit(0xD9);
2369 emit(0xE4);
2370 }
2371
2372
fucomp(int i)2373 void Assembler::fucomp(int i) {
2374 EnsureSpace ensure_space(this);
2375 emit_farith(0xDD, 0xE8, i);
2376 }
2377
2378
fucompp()2379 void Assembler::fucompp() {
2380 EnsureSpace ensure_space(this);
2381 emit(0xDA);
2382 emit(0xE9);
2383 }
2384
2385
fucomi(int i)2386 void Assembler::fucomi(int i) {
2387 EnsureSpace ensure_space(this);
2388 emit(0xDB);
2389 emit(0xE8 + i);
2390 }
2391
2392
fucomip()2393 void Assembler::fucomip() {
2394 EnsureSpace ensure_space(this);
2395 emit(0xDF);
2396 emit(0xE9);
2397 }
2398
2399
fcompp()2400 void Assembler::fcompp() {
2401 EnsureSpace ensure_space(this);
2402 emit(0xDE);
2403 emit(0xD9);
2404 }
2405
2406
fnstsw_ax()2407 void Assembler::fnstsw_ax() {
2408 EnsureSpace ensure_space(this);
2409 emit(0xDF);
2410 emit(0xE0);
2411 }
2412
2413
fwait()2414 void Assembler::fwait() {
2415 EnsureSpace ensure_space(this);
2416 emit(0x9B);
2417 }
2418
2419
frndint()2420 void Assembler::frndint() {
2421 EnsureSpace ensure_space(this);
2422 emit(0xD9);
2423 emit(0xFC);
2424 }
2425
2426
fnclex()2427 void Assembler::fnclex() {
2428 EnsureSpace ensure_space(this);
2429 emit(0xDB);
2430 emit(0xE2);
2431 }
2432
2433
sahf()2434 void Assembler::sahf() {
2435 // TODO(X64): Test for presence. Not all 64-bit intel CPU's have sahf
2436 // in 64-bit mode. Test CpuID.
2437 DCHECK(IsEnabled(SAHF));
2438 EnsureSpace ensure_space(this);
2439 emit(0x9E);
2440 }
2441
2442
emit_farith(int b1,int b2,int i)2443 void Assembler::emit_farith(int b1, int b2, int i) {
2444 DCHECK(is_uint8(b1) && is_uint8(b2)); // wrong opcode
2445 DCHECK(is_uint3(i)); // illegal stack offset
2446 emit(b1);
2447 emit(b2 + i);
2448 }
2449
2450
2451 // SSE operations.
2452
andps(XMMRegister dst,XMMRegister src)2453 void Assembler::andps(XMMRegister dst, XMMRegister src) {
2454 EnsureSpace ensure_space(this);
2455 emit_optional_rex_32(dst, src);
2456 emit(0x0F);
2457 emit(0x54);
2458 emit_sse_operand(dst, src);
2459 }
2460
2461
andps(XMMRegister dst,const Operand & src)2462 void Assembler::andps(XMMRegister dst, const Operand& src) {
2463 EnsureSpace ensure_space(this);
2464 emit_optional_rex_32(dst, src);
2465 emit(0x0F);
2466 emit(0x54);
2467 emit_sse_operand(dst, src);
2468 }
2469
2470
orps(XMMRegister dst,XMMRegister src)2471 void Assembler::orps(XMMRegister dst, XMMRegister src) {
2472 EnsureSpace ensure_space(this);
2473 emit_optional_rex_32(dst, src);
2474 emit(0x0F);
2475 emit(0x56);
2476 emit_sse_operand(dst, src);
2477 }
2478
2479
orps(XMMRegister dst,const Operand & src)2480 void Assembler::orps(XMMRegister dst, const Operand& src) {
2481 EnsureSpace ensure_space(this);
2482 emit_optional_rex_32(dst, src);
2483 emit(0x0F);
2484 emit(0x56);
2485 emit_sse_operand(dst, src);
2486 }
2487
2488
xorps(XMMRegister dst,XMMRegister src)2489 void Assembler::xorps(XMMRegister dst, XMMRegister src) {
2490 DCHECK(!IsEnabled(AVX));
2491 EnsureSpace ensure_space(this);
2492 emit_optional_rex_32(dst, src);
2493 emit(0x0F);
2494 emit(0x57);
2495 emit_sse_operand(dst, src);
2496 }
2497
2498
xorps(XMMRegister dst,const Operand & src)2499 void Assembler::xorps(XMMRegister dst, const Operand& src) {
2500 DCHECK(!IsEnabled(AVX));
2501 EnsureSpace ensure_space(this);
2502 emit_optional_rex_32(dst, src);
2503 emit(0x0F);
2504 emit(0x57);
2505 emit_sse_operand(dst, src);
2506 }
2507
2508
addps(XMMRegister dst,XMMRegister src)2509 void Assembler::addps(XMMRegister dst, XMMRegister src) {
2510 EnsureSpace ensure_space(this);
2511 emit_optional_rex_32(dst, src);
2512 emit(0x0F);
2513 emit(0x58);
2514 emit_sse_operand(dst, src);
2515 }
2516
2517
addps(XMMRegister dst,const Operand & src)2518 void Assembler::addps(XMMRegister dst, const Operand& src) {
2519 EnsureSpace ensure_space(this);
2520 emit_optional_rex_32(dst, src);
2521 emit(0x0F);
2522 emit(0x58);
2523 emit_sse_operand(dst, src);
2524 }
2525
2526
subps(XMMRegister dst,XMMRegister src)2527 void Assembler::subps(XMMRegister dst, XMMRegister src) {
2528 EnsureSpace ensure_space(this);
2529 emit_optional_rex_32(dst, src);
2530 emit(0x0F);
2531 emit(0x5C);
2532 emit_sse_operand(dst, src);
2533 }
2534
2535
subps(XMMRegister dst,const Operand & src)2536 void Assembler::subps(XMMRegister dst, const Operand& src) {
2537 EnsureSpace ensure_space(this);
2538 emit_optional_rex_32(dst, src);
2539 emit(0x0F);
2540 emit(0x5C);
2541 emit_sse_operand(dst, src);
2542 }
2543
2544
mulps(XMMRegister dst,XMMRegister src)2545 void Assembler::mulps(XMMRegister dst, XMMRegister src) {
2546 EnsureSpace ensure_space(this);
2547 emit_optional_rex_32(dst, src);
2548 emit(0x0F);
2549 emit(0x59);
2550 emit_sse_operand(dst, src);
2551 }
2552
2553
mulps(XMMRegister dst,const Operand & src)2554 void Assembler::mulps(XMMRegister dst, const Operand& src) {
2555 EnsureSpace ensure_space(this);
2556 emit_optional_rex_32(dst, src);
2557 emit(0x0F);
2558 emit(0x59);
2559 emit_sse_operand(dst, src);
2560 }
2561
2562
divps(XMMRegister dst,XMMRegister src)2563 void Assembler::divps(XMMRegister dst, XMMRegister src) {
2564 EnsureSpace ensure_space(this);
2565 emit_optional_rex_32(dst, src);
2566 emit(0x0F);
2567 emit(0x5E);
2568 emit_sse_operand(dst, src);
2569 }
2570
2571
divps(XMMRegister dst,const Operand & src)2572 void Assembler::divps(XMMRegister dst, const Operand& src) {
2573 EnsureSpace ensure_space(this);
2574 emit_optional_rex_32(dst, src);
2575 emit(0x0F);
2576 emit(0x5E);
2577 emit_sse_operand(dst, src);
2578 }
2579
2580
2581 // SSE 2 operations.
2582
movd(XMMRegister dst,Register src)2583 void Assembler::movd(XMMRegister dst, Register src) {
2584 DCHECK(!IsEnabled(AVX));
2585 EnsureSpace ensure_space(this);
2586 emit(0x66);
2587 emit_optional_rex_32(dst, src);
2588 emit(0x0F);
2589 emit(0x6E);
2590 emit_sse_operand(dst, src);
2591 }
2592
2593
movd(XMMRegister dst,const Operand & src)2594 void Assembler::movd(XMMRegister dst, const Operand& src) {
2595 DCHECK(!IsEnabled(AVX));
2596 EnsureSpace ensure_space(this);
2597 emit(0x66);
2598 emit_optional_rex_32(dst, src);
2599 emit(0x0F);
2600 emit(0x6E);
2601 emit_sse_operand(dst, src);
2602 }
2603
2604
movd(Register dst,XMMRegister src)2605 void Assembler::movd(Register dst, XMMRegister src) {
2606 DCHECK(!IsEnabled(AVX));
2607 EnsureSpace ensure_space(this);
2608 emit(0x66);
2609 emit_optional_rex_32(src, dst);
2610 emit(0x0F);
2611 emit(0x7E);
2612 emit_sse_operand(src, dst);
2613 }
2614
2615
movq(XMMRegister dst,Register src)2616 void Assembler::movq(XMMRegister dst, Register src) {
2617 DCHECK(!IsEnabled(AVX));
2618 EnsureSpace ensure_space(this);
2619 emit(0x66);
2620 emit_rex_64(dst, src);
2621 emit(0x0F);
2622 emit(0x6E);
2623 emit_sse_operand(dst, src);
2624 }
2625
2626
movq(Register dst,XMMRegister src)2627 void Assembler::movq(Register dst, XMMRegister src) {
2628 DCHECK(!IsEnabled(AVX));
2629 EnsureSpace ensure_space(this);
2630 emit(0x66);
2631 emit_rex_64(src, dst);
2632 emit(0x0F);
2633 emit(0x7E);
2634 emit_sse_operand(src, dst);
2635 }
2636
2637
movq(XMMRegister dst,XMMRegister src)2638 void Assembler::movq(XMMRegister dst, XMMRegister src) {
2639 DCHECK(!IsEnabled(AVX));
2640 EnsureSpace ensure_space(this);
2641 if (dst.low_bits() == 4) {
2642 // Avoid unnecessary SIB byte.
2643 emit(0xf3);
2644 emit_optional_rex_32(dst, src);
2645 emit(0x0F);
2646 emit(0x7e);
2647 emit_sse_operand(dst, src);
2648 } else {
2649 emit(0x66);
2650 emit_optional_rex_32(src, dst);
2651 emit(0x0F);
2652 emit(0xD6);
2653 emit_sse_operand(src, dst);
2654 }
2655 }
2656
2657
movdqa(const Operand & dst,XMMRegister src)2658 void Assembler::movdqa(const Operand& dst, XMMRegister src) {
2659 EnsureSpace ensure_space(this);
2660 emit(0x66);
2661 emit_rex_64(src, dst);
2662 emit(0x0F);
2663 emit(0x7F);
2664 emit_sse_operand(src, dst);
2665 }
2666
2667
movdqa(XMMRegister dst,const Operand & src)2668 void Assembler::movdqa(XMMRegister dst, const Operand& src) {
2669 EnsureSpace ensure_space(this);
2670 emit(0x66);
2671 emit_rex_64(dst, src);
2672 emit(0x0F);
2673 emit(0x6F);
2674 emit_sse_operand(dst, src);
2675 }
2676
2677
movdqu(const Operand & dst,XMMRegister src)2678 void Assembler::movdqu(const Operand& dst, XMMRegister src) {
2679 EnsureSpace ensure_space(this);
2680 emit(0xF3);
2681 emit_rex_64(src, dst);
2682 emit(0x0F);
2683 emit(0x7F);
2684 emit_sse_operand(src, dst);
2685 }
2686
2687
movdqu(XMMRegister dst,const Operand & src)2688 void Assembler::movdqu(XMMRegister dst, const Operand& src) {
2689 EnsureSpace ensure_space(this);
2690 emit(0xF3);
2691 emit_rex_64(dst, src);
2692 emit(0x0F);
2693 emit(0x6F);
2694 emit_sse_operand(dst, src);
2695 }
2696
2697
extractps(Register dst,XMMRegister src,byte imm8)2698 void Assembler::extractps(Register dst, XMMRegister src, byte imm8) {
2699 DCHECK(IsEnabled(SSE4_1));
2700 DCHECK(is_uint8(imm8));
2701 EnsureSpace ensure_space(this);
2702 emit(0x66);
2703 emit_optional_rex_32(src, dst);
2704 emit(0x0F);
2705 emit(0x3A);
2706 emit(0x17);
2707 emit_sse_operand(src, dst);
2708 emit(imm8);
2709 }
2710
2711
pextrd(Register dst,XMMRegister src,int8_t imm8)2712 void Assembler::pextrd(Register dst, XMMRegister src, int8_t imm8) {
2713 DCHECK(IsEnabled(SSE4_1));
2714 EnsureSpace ensure_space(this);
2715 emit(0x66);
2716 emit_optional_rex_32(src, dst);
2717 emit(0x0F);
2718 emit(0x3A);
2719 emit(0x16);
2720 emit_sse_operand(src, dst);
2721 emit(imm8);
2722 }
2723
2724
pinsrd(XMMRegister dst,Register src,int8_t imm8)2725 void Assembler::pinsrd(XMMRegister dst, Register src, int8_t imm8) {
2726 DCHECK(IsEnabled(SSE4_1));
2727 EnsureSpace ensure_space(this);
2728 emit(0x66);
2729 emit_optional_rex_32(dst, src);
2730 emit(0x0F);
2731 emit(0x3A);
2732 emit(0x22);
2733 emit_sse_operand(dst, src);
2734 emit(imm8);
2735 }
2736
2737
pinsrd(XMMRegister dst,const Operand & src,int8_t imm8)2738 void Assembler::pinsrd(XMMRegister dst, const Operand& src, int8_t imm8) {
2739 DCHECK(IsEnabled(SSE4_1));
2740 EnsureSpace ensure_space(this);
2741 emit(0x66);
2742 emit_optional_rex_32(dst, src);
2743 emit(0x0F);
2744 emit(0x3A);
2745 emit(0x22);
2746 emit_sse_operand(dst, src);
2747 emit(imm8);
2748 }
2749
2750
movsd(const Operand & dst,XMMRegister src)2751 void Assembler::movsd(const Operand& dst, XMMRegister src) {
2752 DCHECK(!IsEnabled(AVX));
2753 EnsureSpace ensure_space(this);
2754 emit(0xF2); // double
2755 emit_optional_rex_32(src, dst);
2756 emit(0x0F);
2757 emit(0x11); // store
2758 emit_sse_operand(src, dst);
2759 }
2760
2761
movsd(XMMRegister dst,XMMRegister src)2762 void Assembler::movsd(XMMRegister dst, XMMRegister src) {
2763 DCHECK(!IsEnabled(AVX));
2764 EnsureSpace ensure_space(this);
2765 emit(0xF2); // double
2766 emit_optional_rex_32(dst, src);
2767 emit(0x0F);
2768 emit(0x10); // load
2769 emit_sse_operand(dst, src);
2770 }
2771
2772
movsd(XMMRegister dst,const Operand & src)2773 void Assembler::movsd(XMMRegister dst, const Operand& src) {
2774 DCHECK(!IsEnabled(AVX));
2775 EnsureSpace ensure_space(this);
2776 emit(0xF2); // double
2777 emit_optional_rex_32(dst, src);
2778 emit(0x0F);
2779 emit(0x10); // load
2780 emit_sse_operand(dst, src);
2781 }
2782
2783
movaps(XMMRegister dst,XMMRegister src)2784 void Assembler::movaps(XMMRegister dst, XMMRegister src) {
2785 DCHECK(!IsEnabled(AVX));
2786 EnsureSpace ensure_space(this);
2787 if (src.low_bits() == 4) {
2788 // Try to avoid an unnecessary SIB byte.
2789 emit_optional_rex_32(src, dst);
2790 emit(0x0F);
2791 emit(0x29);
2792 emit_sse_operand(src, dst);
2793 } else {
2794 emit_optional_rex_32(dst, src);
2795 emit(0x0F);
2796 emit(0x28);
2797 emit_sse_operand(dst, src);
2798 }
2799 }
2800
2801
shufps(XMMRegister dst,XMMRegister src,byte imm8)2802 void Assembler::shufps(XMMRegister dst, XMMRegister src, byte imm8) {
2803 DCHECK(is_uint8(imm8));
2804 EnsureSpace ensure_space(this);
2805 emit_optional_rex_32(src, dst);
2806 emit(0x0F);
2807 emit(0xC6);
2808 emit_sse_operand(dst, src);
2809 emit(imm8);
2810 }
2811
2812
movapd(XMMRegister dst,XMMRegister src)2813 void Assembler::movapd(XMMRegister dst, XMMRegister src) {
2814 DCHECK(!IsEnabled(AVX));
2815 EnsureSpace ensure_space(this);
2816 if (src.low_bits() == 4) {
2817 // Try to avoid an unnecessary SIB byte.
2818 emit(0x66);
2819 emit_optional_rex_32(src, dst);
2820 emit(0x0F);
2821 emit(0x29);
2822 emit_sse_operand(src, dst);
2823 } else {
2824 emit(0x66);
2825 emit_optional_rex_32(dst, src);
2826 emit(0x0F);
2827 emit(0x28);
2828 emit_sse_operand(dst, src);
2829 }
2830 }
2831
2832
addss(XMMRegister dst,XMMRegister src)2833 void Assembler::addss(XMMRegister dst, XMMRegister src) {
2834 EnsureSpace ensure_space(this);
2835 emit(0xF3);
2836 emit_optional_rex_32(dst, src);
2837 emit(0x0F);
2838 emit(0x58);
2839 emit_sse_operand(dst, src);
2840 }
2841
2842
addss(XMMRegister dst,const Operand & src)2843 void Assembler::addss(XMMRegister dst, const Operand& src) {
2844 EnsureSpace ensure_space(this);
2845 emit(0xF3);
2846 emit_optional_rex_32(dst, src);
2847 emit(0x0F);
2848 emit(0x58);
2849 emit_sse_operand(dst, src);
2850 }
2851
2852
subss(XMMRegister dst,XMMRegister src)2853 void Assembler::subss(XMMRegister dst, XMMRegister src) {
2854 EnsureSpace ensure_space(this);
2855 emit(0xF3);
2856 emit_optional_rex_32(dst, src);
2857 emit(0x0F);
2858 emit(0x5C);
2859 emit_sse_operand(dst, src);
2860 }
2861
2862
subss(XMMRegister dst,const Operand & src)2863 void Assembler::subss(XMMRegister dst, const Operand& src) {
2864 EnsureSpace ensure_space(this);
2865 emit(0xF3);
2866 emit_optional_rex_32(dst, src);
2867 emit(0x0F);
2868 emit(0x5C);
2869 emit_sse_operand(dst, src);
2870 }
2871
2872
mulss(XMMRegister dst,XMMRegister src)2873 void Assembler::mulss(XMMRegister dst, XMMRegister src) {
2874 EnsureSpace ensure_space(this);
2875 emit(0xF3);
2876 emit_optional_rex_32(dst, src);
2877 emit(0x0F);
2878 emit(0x59);
2879 emit_sse_operand(dst, src);
2880 }
2881
2882
mulss(XMMRegister dst,const Operand & src)2883 void Assembler::mulss(XMMRegister dst, const Operand& src) {
2884 EnsureSpace ensure_space(this);
2885 emit(0xF3);
2886 emit_optional_rex_32(dst, src);
2887 emit(0x0F);
2888 emit(0x59);
2889 emit_sse_operand(dst, src);
2890 }
2891
2892
divss(XMMRegister dst,XMMRegister src)2893 void Assembler::divss(XMMRegister dst, XMMRegister src) {
2894 EnsureSpace ensure_space(this);
2895 emit(0xF3);
2896 emit_optional_rex_32(dst, src);
2897 emit(0x0F);
2898 emit(0x5E);
2899 emit_sse_operand(dst, src);
2900 }
2901
2902
divss(XMMRegister dst,const Operand & src)2903 void Assembler::divss(XMMRegister dst, const Operand& src) {
2904 EnsureSpace ensure_space(this);
2905 emit(0xF3);
2906 emit_optional_rex_32(dst, src);
2907 emit(0x0F);
2908 emit(0x5E);
2909 emit_sse_operand(dst, src);
2910 }
2911
2912
maxss(XMMRegister dst,XMMRegister src)2913 void Assembler::maxss(XMMRegister dst, XMMRegister src) {
2914 EnsureSpace ensure_space(this);
2915 emit(0xF3);
2916 emit_optional_rex_32(dst, src);
2917 emit(0x0F);
2918 emit(0x5F);
2919 emit_sse_operand(dst, src);
2920 }
2921
2922
maxss(XMMRegister dst,const Operand & src)2923 void Assembler::maxss(XMMRegister dst, const Operand& src) {
2924 EnsureSpace ensure_space(this);
2925 emit(0xF3);
2926 emit_optional_rex_32(dst, src);
2927 emit(0x0F);
2928 emit(0x5F);
2929 emit_sse_operand(dst, src);
2930 }
2931
2932
minss(XMMRegister dst,XMMRegister src)2933 void Assembler::minss(XMMRegister dst, XMMRegister src) {
2934 EnsureSpace ensure_space(this);
2935 emit(0xF3);
2936 emit_optional_rex_32(dst, src);
2937 emit(0x0F);
2938 emit(0x5D);
2939 emit_sse_operand(dst, src);
2940 }
2941
2942
minss(XMMRegister dst,const Operand & src)2943 void Assembler::minss(XMMRegister dst, const Operand& src) {
2944 EnsureSpace ensure_space(this);
2945 emit(0xF3);
2946 emit_optional_rex_32(dst, src);
2947 emit(0x0F);
2948 emit(0x5D);
2949 emit_sse_operand(dst, src);
2950 }
2951
2952
sqrtss(XMMRegister dst,XMMRegister src)2953 void Assembler::sqrtss(XMMRegister dst, XMMRegister src) {
2954 EnsureSpace ensure_space(this);
2955 emit(0xF3);
2956 emit_optional_rex_32(dst, src);
2957 emit(0x0F);
2958 emit(0x51);
2959 emit_sse_operand(dst, src);
2960 }
2961
2962
sqrtss(XMMRegister dst,const Operand & src)2963 void Assembler::sqrtss(XMMRegister dst, const Operand& src) {
2964 EnsureSpace ensure_space(this);
2965 emit(0xF3);
2966 emit_optional_rex_32(dst, src);
2967 emit(0x0F);
2968 emit(0x51);
2969 emit_sse_operand(dst, src);
2970 }
2971
2972
ucomiss(XMMRegister dst,XMMRegister src)2973 void Assembler::ucomiss(XMMRegister dst, XMMRegister src) {
2974 DCHECK(!IsEnabled(AVX));
2975 EnsureSpace ensure_space(this);
2976 emit_optional_rex_32(dst, src);
2977 emit(0x0f);
2978 emit(0x2e);
2979 emit_sse_operand(dst, src);
2980 }
2981
2982
ucomiss(XMMRegister dst,const Operand & src)2983 void Assembler::ucomiss(XMMRegister dst, const Operand& src) {
2984 DCHECK(!IsEnabled(AVX));
2985 EnsureSpace ensure_space(this);
2986 emit_optional_rex_32(dst, src);
2987 emit(0x0f);
2988 emit(0x2e);
2989 emit_sse_operand(dst, src);
2990 }
2991
2992
movss(XMMRegister dst,XMMRegister src)2993 void Assembler::movss(XMMRegister dst, XMMRegister src) {
2994 DCHECK(!IsEnabled(AVX));
2995 EnsureSpace ensure_space(this);
2996 emit(0xF3); // single
2997 emit_optional_rex_32(dst, src);
2998 emit(0x0F);
2999 emit(0x10); // load
3000 emit_sse_operand(dst, src);
3001 }
3002
3003
movss(XMMRegister dst,const Operand & src)3004 void Assembler::movss(XMMRegister dst, const Operand& src) {
3005 DCHECK(!IsEnabled(AVX));
3006 EnsureSpace ensure_space(this);
3007 emit(0xF3); // single
3008 emit_optional_rex_32(dst, src);
3009 emit(0x0F);
3010 emit(0x10); // load
3011 emit_sse_operand(dst, src);
3012 }
3013
3014
movss(const Operand & src,XMMRegister dst)3015 void Assembler::movss(const Operand& src, XMMRegister dst) {
3016 DCHECK(!IsEnabled(AVX));
3017 EnsureSpace ensure_space(this);
3018 emit(0xF3); // single
3019 emit_optional_rex_32(dst, src);
3020 emit(0x0F);
3021 emit(0x11); // store
3022 emit_sse_operand(dst, src);
3023 }
3024
3025
psllq(XMMRegister reg,byte imm8)3026 void Assembler::psllq(XMMRegister reg, byte imm8) {
3027 DCHECK(!IsEnabled(AVX));
3028 EnsureSpace ensure_space(this);
3029 emit(0x66);
3030 emit_optional_rex_32(reg);
3031 emit(0x0F);
3032 emit(0x73);
3033 emit_sse_operand(rsi, reg); // rsi == 6
3034 emit(imm8);
3035 }
3036
3037
psrlq(XMMRegister reg,byte imm8)3038 void Assembler::psrlq(XMMRegister reg, byte imm8) {
3039 DCHECK(!IsEnabled(AVX));
3040 EnsureSpace ensure_space(this);
3041 emit(0x66);
3042 emit_optional_rex_32(reg);
3043 emit(0x0F);
3044 emit(0x73);
3045 emit_sse_operand(rdx, reg); // rdx == 2
3046 emit(imm8);
3047 }
3048
3049
pslld(XMMRegister reg,byte imm8)3050 void Assembler::pslld(XMMRegister reg, byte imm8) {
3051 EnsureSpace ensure_space(this);
3052 emit(0x66);
3053 emit_optional_rex_32(reg);
3054 emit(0x0F);
3055 emit(0x72);
3056 emit_sse_operand(rsi, reg); // rsi == 6
3057 emit(imm8);
3058 }
3059
3060
psrld(XMMRegister reg,byte imm8)3061 void Assembler::psrld(XMMRegister reg, byte imm8) {
3062 EnsureSpace ensure_space(this);
3063 emit(0x66);
3064 emit_optional_rex_32(reg);
3065 emit(0x0F);
3066 emit(0x72);
3067 emit_sse_operand(rdx, reg); // rdx == 2
3068 emit(imm8);
3069 }
3070
3071
cvttss2si(Register dst,const Operand & src)3072 void Assembler::cvttss2si(Register dst, const Operand& src) {
3073 DCHECK(!IsEnabled(AVX));
3074 EnsureSpace ensure_space(this);
3075 emit(0xF3);
3076 emit_optional_rex_32(dst, src);
3077 emit(0x0F);
3078 emit(0x2C);
3079 emit_operand(dst, src);
3080 }
3081
3082
cvttss2si(Register dst,XMMRegister src)3083 void Assembler::cvttss2si(Register dst, XMMRegister src) {
3084 DCHECK(!IsEnabled(AVX));
3085 EnsureSpace ensure_space(this);
3086 emit(0xF3);
3087 emit_optional_rex_32(dst, src);
3088 emit(0x0F);
3089 emit(0x2C);
3090 emit_sse_operand(dst, src);
3091 }
3092
3093
cvttsd2si(Register dst,const Operand & src)3094 void Assembler::cvttsd2si(Register dst, const Operand& src) {
3095 DCHECK(!IsEnabled(AVX));
3096 EnsureSpace ensure_space(this);
3097 emit(0xF2);
3098 emit_optional_rex_32(dst, src);
3099 emit(0x0F);
3100 emit(0x2C);
3101 emit_operand(dst, src);
3102 }
3103
3104
cvttsd2si(Register dst,XMMRegister src)3105 void Assembler::cvttsd2si(Register dst, XMMRegister src) {
3106 DCHECK(!IsEnabled(AVX));
3107 EnsureSpace ensure_space(this);
3108 emit(0xF2);
3109 emit_optional_rex_32(dst, src);
3110 emit(0x0F);
3111 emit(0x2C);
3112 emit_sse_operand(dst, src);
3113 }
3114
3115
cvttss2siq(Register dst,XMMRegister src)3116 void Assembler::cvttss2siq(Register dst, XMMRegister src) {
3117 DCHECK(!IsEnabled(AVX));
3118 EnsureSpace ensure_space(this);
3119 emit(0xF3);
3120 emit_rex_64(dst, src);
3121 emit(0x0F);
3122 emit(0x2C);
3123 emit_sse_operand(dst, src);
3124 }
3125
3126
cvttss2siq(Register dst,const Operand & src)3127 void Assembler::cvttss2siq(Register dst, const Operand& src) {
3128 DCHECK(!IsEnabled(AVX));
3129 EnsureSpace ensure_space(this);
3130 emit(0xF3);
3131 emit_rex_64(dst, src);
3132 emit(0x0F);
3133 emit(0x2C);
3134 emit_sse_operand(dst, src);
3135 }
3136
3137
cvttsd2siq(Register dst,XMMRegister src)3138 void Assembler::cvttsd2siq(Register dst, XMMRegister src) {
3139 DCHECK(!IsEnabled(AVX));
3140 EnsureSpace ensure_space(this);
3141 emit(0xF2);
3142 emit_rex_64(dst, src);
3143 emit(0x0F);
3144 emit(0x2C);
3145 emit_sse_operand(dst, src);
3146 }
3147
3148
cvttsd2siq(Register dst,const Operand & src)3149 void Assembler::cvttsd2siq(Register dst, const Operand& src) {
3150 DCHECK(!IsEnabled(AVX));
3151 EnsureSpace ensure_space(this);
3152 emit(0xF2);
3153 emit_rex_64(dst, src);
3154 emit(0x0F);
3155 emit(0x2C);
3156 emit_sse_operand(dst, src);
3157 }
3158
3159
cvtlsi2sd(XMMRegister dst,const Operand & src)3160 void Assembler::cvtlsi2sd(XMMRegister dst, const Operand& src) {
3161 DCHECK(!IsEnabled(AVX));
3162 EnsureSpace ensure_space(this);
3163 emit(0xF2);
3164 emit_optional_rex_32(dst, src);
3165 emit(0x0F);
3166 emit(0x2A);
3167 emit_sse_operand(dst, src);
3168 }
3169
3170
cvtlsi2sd(XMMRegister dst,Register src)3171 void Assembler::cvtlsi2sd(XMMRegister dst, Register src) {
3172 DCHECK(!IsEnabled(AVX));
3173 EnsureSpace ensure_space(this);
3174 emit(0xF2);
3175 emit_optional_rex_32(dst, src);
3176 emit(0x0F);
3177 emit(0x2A);
3178 emit_sse_operand(dst, src);
3179 }
3180
3181
cvtlsi2ss(XMMRegister dst,Register src)3182 void Assembler::cvtlsi2ss(XMMRegister dst, Register src) {
3183 EnsureSpace ensure_space(this);
3184 emit(0xF3);
3185 emit_optional_rex_32(dst, src);
3186 emit(0x0F);
3187 emit(0x2A);
3188 emit_sse_operand(dst, src);
3189 }
3190
3191
cvtqsi2ss(XMMRegister dst,const Operand & src)3192 void Assembler::cvtqsi2ss(XMMRegister dst, const Operand& src) {
3193 DCHECK(!IsEnabled(AVX));
3194 EnsureSpace ensure_space(this);
3195 emit(0xF3);
3196 emit_rex_64(dst, src);
3197 emit(0x0F);
3198 emit(0x2A);
3199 emit_sse_operand(dst, src);
3200 }
3201
3202
cvtqsi2ss(XMMRegister dst,Register src)3203 void Assembler::cvtqsi2ss(XMMRegister dst, Register src) {
3204 DCHECK(!IsEnabled(AVX));
3205 EnsureSpace ensure_space(this);
3206 emit(0xF3);
3207 emit_rex_64(dst, src);
3208 emit(0x0F);
3209 emit(0x2A);
3210 emit_sse_operand(dst, src);
3211 }
3212
3213
cvtqsi2sd(XMMRegister dst,const Operand & src)3214 void Assembler::cvtqsi2sd(XMMRegister dst, const Operand& src) {
3215 DCHECK(!IsEnabled(AVX));
3216 EnsureSpace ensure_space(this);
3217 emit(0xF2);
3218 emit_rex_64(dst, src);
3219 emit(0x0F);
3220 emit(0x2A);
3221 emit_sse_operand(dst, src);
3222 }
3223
3224
cvtqsi2sd(XMMRegister dst,Register src)3225 void Assembler::cvtqsi2sd(XMMRegister dst, Register src) {
3226 DCHECK(!IsEnabled(AVX));
3227 EnsureSpace ensure_space(this);
3228 emit(0xF2);
3229 emit_rex_64(dst, src);
3230 emit(0x0F);
3231 emit(0x2A);
3232 emit_sse_operand(dst, src);
3233 }
3234
3235
cvtss2sd(XMMRegister dst,XMMRegister src)3236 void Assembler::cvtss2sd(XMMRegister dst, XMMRegister src) {
3237 DCHECK(!IsEnabled(AVX));
3238 EnsureSpace ensure_space(this);
3239 emit(0xF3);
3240 emit_optional_rex_32(dst, src);
3241 emit(0x0F);
3242 emit(0x5A);
3243 emit_sse_operand(dst, src);
3244 }
3245
3246
cvtss2sd(XMMRegister dst,const Operand & src)3247 void Assembler::cvtss2sd(XMMRegister dst, const Operand& src) {
3248 DCHECK(!IsEnabled(AVX));
3249 EnsureSpace ensure_space(this);
3250 emit(0xF3);
3251 emit_optional_rex_32(dst, src);
3252 emit(0x0F);
3253 emit(0x5A);
3254 emit_sse_operand(dst, src);
3255 }
3256
3257
cvtsd2ss(XMMRegister dst,XMMRegister src)3258 void Assembler::cvtsd2ss(XMMRegister dst, XMMRegister src) {
3259 DCHECK(!IsEnabled(AVX));
3260 EnsureSpace ensure_space(this);
3261 emit(0xF2);
3262 emit_optional_rex_32(dst, src);
3263 emit(0x0F);
3264 emit(0x5A);
3265 emit_sse_operand(dst, src);
3266 }
3267
3268
cvtsd2ss(XMMRegister dst,const Operand & src)3269 void Assembler::cvtsd2ss(XMMRegister dst, const Operand& src) {
3270 DCHECK(!IsEnabled(AVX));
3271 EnsureSpace ensure_space(this);
3272 emit(0xF2);
3273 emit_optional_rex_32(dst, src);
3274 emit(0x0F);
3275 emit(0x5A);
3276 emit_sse_operand(dst, src);
3277 }
3278
3279
cvtsd2si(Register dst,XMMRegister src)3280 void Assembler::cvtsd2si(Register dst, XMMRegister src) {
3281 DCHECK(!IsEnabled(AVX));
3282 EnsureSpace ensure_space(this);
3283 emit(0xF2);
3284 emit_optional_rex_32(dst, src);
3285 emit(0x0F);
3286 emit(0x2D);
3287 emit_sse_operand(dst, src);
3288 }
3289
3290
cvtsd2siq(Register dst,XMMRegister src)3291 void Assembler::cvtsd2siq(Register dst, XMMRegister src) {
3292 DCHECK(!IsEnabled(AVX));
3293 EnsureSpace ensure_space(this);
3294 emit(0xF2);
3295 emit_rex_64(dst, src);
3296 emit(0x0F);
3297 emit(0x2D);
3298 emit_sse_operand(dst, src);
3299 }
3300
3301
addsd(XMMRegister dst,XMMRegister src)3302 void Assembler::addsd(XMMRegister dst, XMMRegister src) {
3303 EnsureSpace ensure_space(this);
3304 emit(0xF2);
3305 emit_optional_rex_32(dst, src);
3306 emit(0x0F);
3307 emit(0x58);
3308 emit_sse_operand(dst, src);
3309 }
3310
3311
addsd(XMMRegister dst,const Operand & src)3312 void Assembler::addsd(XMMRegister dst, const Operand& src) {
3313 EnsureSpace ensure_space(this);
3314 emit(0xF2);
3315 emit_optional_rex_32(dst, src);
3316 emit(0x0F);
3317 emit(0x58);
3318 emit_sse_operand(dst, src);
3319 }
3320
3321
mulsd(XMMRegister dst,XMMRegister src)3322 void Assembler::mulsd(XMMRegister dst, XMMRegister src) {
3323 EnsureSpace ensure_space(this);
3324 emit(0xF2);
3325 emit_optional_rex_32(dst, src);
3326 emit(0x0F);
3327 emit(0x59);
3328 emit_sse_operand(dst, src);
3329 }
3330
3331
mulsd(XMMRegister dst,const Operand & src)3332 void Assembler::mulsd(XMMRegister dst, const Operand& src) {
3333 EnsureSpace ensure_space(this);
3334 emit(0xF2);
3335 emit_optional_rex_32(dst, src);
3336 emit(0x0F);
3337 emit(0x59);
3338 emit_sse_operand(dst, src);
3339 }
3340
3341
subsd(XMMRegister dst,XMMRegister src)3342 void Assembler::subsd(XMMRegister dst, XMMRegister src) {
3343 EnsureSpace ensure_space(this);
3344 emit(0xF2);
3345 emit_optional_rex_32(dst, src);
3346 emit(0x0F);
3347 emit(0x5C);
3348 emit_sse_operand(dst, src);
3349 }
3350
3351
subsd(XMMRegister dst,const Operand & src)3352 void Assembler::subsd(XMMRegister dst, const Operand& src) {
3353 EnsureSpace ensure_space(this);
3354 emit(0xF2);
3355 emit_optional_rex_32(dst, src);
3356 emit(0x0F);
3357 emit(0x5C);
3358 emit_sse_operand(dst, src);
3359 }
3360
3361
divsd(XMMRegister dst,XMMRegister src)3362 void Assembler::divsd(XMMRegister dst, XMMRegister src) {
3363 EnsureSpace ensure_space(this);
3364 emit(0xF2);
3365 emit_optional_rex_32(dst, src);
3366 emit(0x0F);
3367 emit(0x5E);
3368 emit_sse_operand(dst, src);
3369 }
3370
3371
divsd(XMMRegister dst,const Operand & src)3372 void Assembler::divsd(XMMRegister dst, const Operand& src) {
3373 EnsureSpace ensure_space(this);
3374 emit(0xF2);
3375 emit_optional_rex_32(dst, src);
3376 emit(0x0F);
3377 emit(0x5E);
3378 emit_sse_operand(dst, src);
3379 }
3380
3381
maxsd(XMMRegister dst,XMMRegister src)3382 void Assembler::maxsd(XMMRegister dst, XMMRegister src) {
3383 EnsureSpace ensure_space(this);
3384 emit(0xF2);
3385 emit_optional_rex_32(dst, src);
3386 emit(0x0F);
3387 emit(0x5F);
3388 emit_sse_operand(dst, src);
3389 }
3390
3391
maxsd(XMMRegister dst,const Operand & src)3392 void Assembler::maxsd(XMMRegister dst, const Operand& src) {
3393 EnsureSpace ensure_space(this);
3394 emit(0xF2);
3395 emit_optional_rex_32(dst, src);
3396 emit(0x0F);
3397 emit(0x5F);
3398 emit_sse_operand(dst, src);
3399 }
3400
3401
minsd(XMMRegister dst,XMMRegister src)3402 void Assembler::minsd(XMMRegister dst, XMMRegister src) {
3403 EnsureSpace ensure_space(this);
3404 emit(0xF2);
3405 emit_optional_rex_32(dst, src);
3406 emit(0x0F);
3407 emit(0x5D);
3408 emit_sse_operand(dst, src);
3409 }
3410
3411
minsd(XMMRegister dst,const Operand & src)3412 void Assembler::minsd(XMMRegister dst, const Operand& src) {
3413 EnsureSpace ensure_space(this);
3414 emit(0xF2);
3415 emit_optional_rex_32(dst, src);
3416 emit(0x0F);
3417 emit(0x5D);
3418 emit_sse_operand(dst, src);
3419 }
3420
3421
andpd(XMMRegister dst,XMMRegister src)3422 void Assembler::andpd(XMMRegister dst, XMMRegister src) {
3423 EnsureSpace ensure_space(this);
3424 emit(0x66);
3425 emit_optional_rex_32(dst, src);
3426 emit(0x0F);
3427 emit(0x54);
3428 emit_sse_operand(dst, src);
3429 }
3430
3431
orpd(XMMRegister dst,XMMRegister src)3432 void Assembler::orpd(XMMRegister dst, XMMRegister src) {
3433 EnsureSpace ensure_space(this);
3434 emit(0x66);
3435 emit_optional_rex_32(dst, src);
3436 emit(0x0F);
3437 emit(0x56);
3438 emit_sse_operand(dst, src);
3439 }
3440
3441
xorpd(XMMRegister dst,XMMRegister src)3442 void Assembler::xorpd(XMMRegister dst, XMMRegister src) {
3443 DCHECK(!IsEnabled(AVX));
3444 EnsureSpace ensure_space(this);
3445 emit(0x66);
3446 emit_optional_rex_32(dst, src);
3447 emit(0x0F);
3448 emit(0x57);
3449 emit_sse_operand(dst, src);
3450 }
3451
3452
sqrtsd(XMMRegister dst,XMMRegister src)3453 void Assembler::sqrtsd(XMMRegister dst, XMMRegister src) {
3454 DCHECK(!IsEnabled(AVX));
3455 EnsureSpace ensure_space(this);
3456 emit(0xF2);
3457 emit_optional_rex_32(dst, src);
3458 emit(0x0F);
3459 emit(0x51);
3460 emit_sse_operand(dst, src);
3461 }
3462
3463
sqrtsd(XMMRegister dst,const Operand & src)3464 void Assembler::sqrtsd(XMMRegister dst, const Operand& src) {
3465 DCHECK(!IsEnabled(AVX));
3466 EnsureSpace ensure_space(this);
3467 emit(0xF2);
3468 emit_optional_rex_32(dst, src);
3469 emit(0x0F);
3470 emit(0x51);
3471 emit_sse_operand(dst, src);
3472 }
3473
3474
ucomisd(XMMRegister dst,XMMRegister src)3475 void Assembler::ucomisd(XMMRegister dst, XMMRegister src) {
3476 DCHECK(!IsEnabled(AVX));
3477 EnsureSpace ensure_space(this);
3478 emit(0x66);
3479 emit_optional_rex_32(dst, src);
3480 emit(0x0f);
3481 emit(0x2e);
3482 emit_sse_operand(dst, src);
3483 }
3484
3485
ucomisd(XMMRegister dst,const Operand & src)3486 void Assembler::ucomisd(XMMRegister dst, const Operand& src) {
3487 DCHECK(!IsEnabled(AVX));
3488 EnsureSpace ensure_space(this);
3489 emit(0x66);
3490 emit_optional_rex_32(dst, src);
3491 emit(0x0f);
3492 emit(0x2e);
3493 emit_sse_operand(dst, src);
3494 }
3495
3496
cmpltsd(XMMRegister dst,XMMRegister src)3497 void Assembler::cmpltsd(XMMRegister dst, XMMRegister src) {
3498 EnsureSpace ensure_space(this);
3499 emit(0xF2);
3500 emit_optional_rex_32(dst, src);
3501 emit(0x0F);
3502 emit(0xC2);
3503 emit_sse_operand(dst, src);
3504 emit(0x01); // LT == 1
3505 }
3506
3507
roundss(XMMRegister dst,XMMRegister src,RoundingMode mode)3508 void Assembler::roundss(XMMRegister dst, XMMRegister src, RoundingMode mode) {
3509 DCHECK(!IsEnabled(AVX));
3510 DCHECK(IsEnabled(SSE4_1));
3511 EnsureSpace ensure_space(this);
3512 emit(0x66);
3513 emit_optional_rex_32(dst, src);
3514 emit(0x0f);
3515 emit(0x3a);
3516 emit(0x0a);
3517 emit_sse_operand(dst, src);
3518 // Mask precision exception.
3519 emit(static_cast<byte>(mode) | 0x8);
3520 }
3521
3522
roundsd(XMMRegister dst,XMMRegister src,RoundingMode mode)3523 void Assembler::roundsd(XMMRegister dst, XMMRegister src, RoundingMode mode) {
3524 DCHECK(!IsEnabled(AVX));
3525 DCHECK(IsEnabled(SSE4_1));
3526 EnsureSpace ensure_space(this);
3527 emit(0x66);
3528 emit_optional_rex_32(dst, src);
3529 emit(0x0f);
3530 emit(0x3a);
3531 emit(0x0b);
3532 emit_sse_operand(dst, src);
3533 // Mask precision exception.
3534 emit(static_cast<byte>(mode) | 0x8);
3535 }
3536
3537
movmskpd(Register dst,XMMRegister src)3538 void Assembler::movmskpd(Register dst, XMMRegister src) {
3539 EnsureSpace ensure_space(this);
3540 emit(0x66);
3541 emit_optional_rex_32(dst, src);
3542 emit(0x0f);
3543 emit(0x50);
3544 emit_sse_operand(dst, src);
3545 }
3546
3547
movmskps(Register dst,XMMRegister src)3548 void Assembler::movmskps(Register dst, XMMRegister src) {
3549 EnsureSpace ensure_space(this);
3550 emit_optional_rex_32(dst, src);
3551 emit(0x0f);
3552 emit(0x50);
3553 emit_sse_operand(dst, src);
3554 }
3555
3556
pcmpeqd(XMMRegister dst,XMMRegister src)3557 void Assembler::pcmpeqd(XMMRegister dst, XMMRegister src) {
3558 DCHECK(!IsEnabled(AVX));
3559 EnsureSpace ensure_space(this);
3560 emit(0x66);
3561 emit_optional_rex_32(dst, src);
3562 emit(0x0F);
3563 emit(0x76);
3564 emit_sse_operand(dst, src);
3565 }
3566
3567
punpckldq(XMMRegister dst,XMMRegister src)3568 void Assembler::punpckldq(XMMRegister dst, XMMRegister src) {
3569 EnsureSpace ensure_space(this);
3570 emit(0x66);
3571 emit_optional_rex_32(dst, src);
3572 emit(0x0F);
3573 emit(0x62);
3574 emit_sse_operand(dst, src);
3575 }
3576
3577
punpckhdq(XMMRegister dst,XMMRegister src)3578 void Assembler::punpckhdq(XMMRegister dst, XMMRegister src) {
3579 EnsureSpace ensure_space(this);
3580 emit(0x66);
3581 emit_optional_rex_32(dst, src);
3582 emit(0x0F);
3583 emit(0x6A);
3584 emit_sse_operand(dst, src);
3585 }
3586
3587
3588 // AVX instructions
vfmasd(byte op,XMMRegister dst,XMMRegister src1,XMMRegister src2)3589 void Assembler::vfmasd(byte op, XMMRegister dst, XMMRegister src1,
3590 XMMRegister src2) {
3591 DCHECK(IsEnabled(FMA3));
3592 EnsureSpace ensure_space(this);
3593 emit_vex_prefix(dst, src1, src2, kLIG, k66, k0F38, kW1);
3594 emit(op);
3595 emit_sse_operand(dst, src2);
3596 }
3597
3598
vfmasd(byte op,XMMRegister dst,XMMRegister src1,const Operand & src2)3599 void Assembler::vfmasd(byte op, XMMRegister dst, XMMRegister src1,
3600 const Operand& src2) {
3601 DCHECK(IsEnabled(FMA3));
3602 EnsureSpace ensure_space(this);
3603 emit_vex_prefix(dst, src1, src2, kLIG, k66, k0F38, kW1);
3604 emit(op);
3605 emit_sse_operand(dst, src2);
3606 }
3607
3608
vfmass(byte op,XMMRegister dst,XMMRegister src1,XMMRegister src2)3609 void Assembler::vfmass(byte op, XMMRegister dst, XMMRegister src1,
3610 XMMRegister src2) {
3611 DCHECK(IsEnabled(FMA3));
3612 EnsureSpace ensure_space(this);
3613 emit_vex_prefix(dst, src1, src2, kLIG, k66, k0F38, kW0);
3614 emit(op);
3615 emit_sse_operand(dst, src2);
3616 }
3617
3618
vfmass(byte op,XMMRegister dst,XMMRegister src1,const Operand & src2)3619 void Assembler::vfmass(byte op, XMMRegister dst, XMMRegister src1,
3620 const Operand& src2) {
3621 DCHECK(IsEnabled(FMA3));
3622 EnsureSpace ensure_space(this);
3623 emit_vex_prefix(dst, src1, src2, kLIG, k66, k0F38, kW0);
3624 emit(op);
3625 emit_sse_operand(dst, src2);
3626 }
3627
3628
vmovd(XMMRegister dst,Register src)3629 void Assembler::vmovd(XMMRegister dst, Register src) {
3630 DCHECK(IsEnabled(AVX));
3631 EnsureSpace ensure_space(this);
3632 XMMRegister isrc = {src.code()};
3633 emit_vex_prefix(dst, xmm0, isrc, kL128, k66, k0F, kW0);
3634 emit(0x6e);
3635 emit_sse_operand(dst, src);
3636 }
3637
3638
vmovd(XMMRegister dst,const Operand & src)3639 void Assembler::vmovd(XMMRegister dst, const Operand& src) {
3640 DCHECK(IsEnabled(AVX));
3641 EnsureSpace ensure_space(this);
3642 emit_vex_prefix(dst, xmm0, src, kL128, k66, k0F, kW0);
3643 emit(0x6e);
3644 emit_sse_operand(dst, src);
3645 }
3646
3647
vmovd(Register dst,XMMRegister src)3648 void Assembler::vmovd(Register dst, XMMRegister src) {
3649 DCHECK(IsEnabled(AVX));
3650 EnsureSpace ensure_space(this);
3651 XMMRegister idst = {dst.code()};
3652 emit_vex_prefix(src, xmm0, idst, kL128, k66, k0F, kW0);
3653 emit(0x7e);
3654 emit_sse_operand(src, dst);
3655 }
3656
3657
vmovq(XMMRegister dst,Register src)3658 void Assembler::vmovq(XMMRegister dst, Register src) {
3659 DCHECK(IsEnabled(AVX));
3660 EnsureSpace ensure_space(this);
3661 XMMRegister isrc = {src.code()};
3662 emit_vex_prefix(dst, xmm0, isrc, kL128, k66, k0F, kW1);
3663 emit(0x6e);
3664 emit_sse_operand(dst, src);
3665 }
3666
3667
vmovq(XMMRegister dst,const Operand & src)3668 void Assembler::vmovq(XMMRegister dst, const Operand& src) {
3669 DCHECK(IsEnabled(AVX));
3670 EnsureSpace ensure_space(this);
3671 emit_vex_prefix(dst, xmm0, src, kL128, k66, k0F, kW1);
3672 emit(0x6e);
3673 emit_sse_operand(dst, src);
3674 }
3675
3676
vmovq(Register dst,XMMRegister src)3677 void Assembler::vmovq(Register dst, XMMRegister src) {
3678 DCHECK(IsEnabled(AVX));
3679 EnsureSpace ensure_space(this);
3680 XMMRegister idst = {dst.code()};
3681 emit_vex_prefix(src, xmm0, idst, kL128, k66, k0F, kW1);
3682 emit(0x7e);
3683 emit_sse_operand(src, dst);
3684 }
3685
3686
vsd(byte op,XMMRegister dst,XMMRegister src1,XMMRegister src2,SIMDPrefix pp,LeadingOpcode m,VexW w)3687 void Assembler::vsd(byte op, XMMRegister dst, XMMRegister src1,
3688 XMMRegister src2, SIMDPrefix pp, LeadingOpcode m, VexW w) {
3689 DCHECK(IsEnabled(AVX));
3690 EnsureSpace ensure_space(this);
3691 emit_vex_prefix(dst, src1, src2, kLIG, pp, m, w);
3692 emit(op);
3693 emit_sse_operand(dst, src2);
3694 }
3695
3696
vsd(byte op,XMMRegister dst,XMMRegister src1,const Operand & src2,SIMDPrefix pp,LeadingOpcode m,VexW w)3697 void Assembler::vsd(byte op, XMMRegister dst, XMMRegister src1,
3698 const Operand& src2, SIMDPrefix pp, LeadingOpcode m,
3699 VexW w) {
3700 DCHECK(IsEnabled(AVX));
3701 EnsureSpace ensure_space(this);
3702 emit_vex_prefix(dst, src1, src2, kLIG, pp, m, w);
3703 emit(op);
3704 emit_sse_operand(dst, src2);
3705 }
3706
3707
vps(byte op,XMMRegister dst,XMMRegister src1,XMMRegister src2)3708 void Assembler::vps(byte op, XMMRegister dst, XMMRegister src1,
3709 XMMRegister src2) {
3710 DCHECK(IsEnabled(AVX));
3711 EnsureSpace ensure_space(this);
3712 emit_vex_prefix(dst, src1, src2, kL128, kNone, k0F, kWIG);
3713 emit(op);
3714 emit_sse_operand(dst, src2);
3715 }
3716
3717
vps(byte op,XMMRegister dst,XMMRegister src1,const Operand & src2)3718 void Assembler::vps(byte op, XMMRegister dst, XMMRegister src1,
3719 const Operand& src2) {
3720 DCHECK(IsEnabled(AVX));
3721 EnsureSpace ensure_space(this);
3722 emit_vex_prefix(dst, src1, src2, kL128, kNone, k0F, kWIG);
3723 emit(op);
3724 emit_sse_operand(dst, src2);
3725 }
3726
3727
vpd(byte op,XMMRegister dst,XMMRegister src1,XMMRegister src2)3728 void Assembler::vpd(byte op, XMMRegister dst, XMMRegister src1,
3729 XMMRegister src2) {
3730 DCHECK(IsEnabled(AVX));
3731 EnsureSpace ensure_space(this);
3732 emit_vex_prefix(dst, src1, src2, kL128, k66, k0F, kWIG);
3733 emit(op);
3734 emit_sse_operand(dst, src2);
3735 }
3736
3737
vpd(byte op,XMMRegister dst,XMMRegister src1,const Operand & src2)3738 void Assembler::vpd(byte op, XMMRegister dst, XMMRegister src1,
3739 const Operand& src2) {
3740 DCHECK(IsEnabled(AVX));
3741 EnsureSpace ensure_space(this);
3742 emit_vex_prefix(dst, src1, src2, kL128, k66, k0F, kWIG);
3743 emit(op);
3744 emit_sse_operand(dst, src2);
3745 }
3746
3747
vucomiss(XMMRegister dst,XMMRegister src)3748 void Assembler::vucomiss(XMMRegister dst, XMMRegister src) {
3749 DCHECK(IsEnabled(AVX));
3750 EnsureSpace ensure_space(this);
3751 emit_vex_prefix(dst, xmm0, src, kLIG, kNone, k0F, kWIG);
3752 emit(0x2e);
3753 emit_sse_operand(dst, src);
3754 }
3755
3756
vucomiss(XMMRegister dst,const Operand & src)3757 void Assembler::vucomiss(XMMRegister dst, const Operand& src) {
3758 DCHECK(IsEnabled(AVX));
3759 EnsureSpace ensure_space(this);
3760 emit_vex_prefix(dst, xmm0, src, kLIG, kNone, k0F, kWIG);
3761 emit(0x2e);
3762 emit_sse_operand(dst, src);
3763 }
3764
3765
vss(byte op,XMMRegister dst,XMMRegister src1,XMMRegister src2)3766 void Assembler::vss(byte op, XMMRegister dst, XMMRegister src1,
3767 XMMRegister src2) {
3768 DCHECK(IsEnabled(AVX));
3769 EnsureSpace ensure_space(this);
3770 emit_vex_prefix(dst, src1, src2, kLIG, kF3, k0F, kWIG);
3771 emit(op);
3772 emit_sse_operand(dst, src2);
3773 }
3774
3775
vss(byte op,XMMRegister dst,XMMRegister src1,const Operand & src2)3776 void Assembler::vss(byte op, XMMRegister dst, XMMRegister src1,
3777 const Operand& src2) {
3778 DCHECK(IsEnabled(AVX));
3779 EnsureSpace ensure_space(this);
3780 emit_vex_prefix(dst, src1, src2, kLIG, kF3, k0F, kWIG);
3781 emit(op);
3782 emit_sse_operand(dst, src2);
3783 }
3784
3785
bmi1q(byte op,Register reg,Register vreg,Register rm)3786 void Assembler::bmi1q(byte op, Register reg, Register vreg, Register rm) {
3787 DCHECK(IsEnabled(BMI1));
3788 EnsureSpace ensure_space(this);
3789 emit_vex_prefix(reg, vreg, rm, kLZ, kNone, k0F38, kW1);
3790 emit(op);
3791 emit_modrm(reg, rm);
3792 }
3793
3794
bmi1q(byte op,Register reg,Register vreg,const Operand & rm)3795 void Assembler::bmi1q(byte op, Register reg, Register vreg, const Operand& rm) {
3796 DCHECK(IsEnabled(BMI1));
3797 EnsureSpace ensure_space(this);
3798 emit_vex_prefix(reg, vreg, rm, kLZ, kNone, k0F38, kW1);
3799 emit(op);
3800 emit_operand(reg, rm);
3801 }
3802
3803
bmi1l(byte op,Register reg,Register vreg,Register rm)3804 void Assembler::bmi1l(byte op, Register reg, Register vreg, Register rm) {
3805 DCHECK(IsEnabled(BMI1));
3806 EnsureSpace ensure_space(this);
3807 emit_vex_prefix(reg, vreg, rm, kLZ, kNone, k0F38, kW0);
3808 emit(op);
3809 emit_modrm(reg, rm);
3810 }
3811
3812
bmi1l(byte op,Register reg,Register vreg,const Operand & rm)3813 void Assembler::bmi1l(byte op, Register reg, Register vreg, const Operand& rm) {
3814 DCHECK(IsEnabled(BMI1));
3815 EnsureSpace ensure_space(this);
3816 emit_vex_prefix(reg, vreg, rm, kLZ, kNone, k0F38, kW0);
3817 emit(op);
3818 emit_operand(reg, rm);
3819 }
3820
3821
tzcntq(Register dst,Register src)3822 void Assembler::tzcntq(Register dst, Register src) {
3823 DCHECK(IsEnabled(BMI1));
3824 EnsureSpace ensure_space(this);
3825 emit(0xF3);
3826 emit_rex_64(dst, src);
3827 emit(0x0F);
3828 emit(0xBC);
3829 emit_modrm(dst, src);
3830 }
3831
3832
tzcntq(Register dst,const Operand & src)3833 void Assembler::tzcntq(Register dst, const Operand& src) {
3834 DCHECK(IsEnabled(BMI1));
3835 EnsureSpace ensure_space(this);
3836 emit(0xF3);
3837 emit_rex_64(dst, src);
3838 emit(0x0F);
3839 emit(0xBC);
3840 emit_operand(dst, src);
3841 }
3842
3843
tzcntl(Register dst,Register src)3844 void Assembler::tzcntl(Register dst, Register src) {
3845 DCHECK(IsEnabled(BMI1));
3846 EnsureSpace ensure_space(this);
3847 emit(0xF3);
3848 emit_optional_rex_32(dst, src);
3849 emit(0x0F);
3850 emit(0xBC);
3851 emit_modrm(dst, src);
3852 }
3853
3854
tzcntl(Register dst,const Operand & src)3855 void Assembler::tzcntl(Register dst, const Operand& src) {
3856 DCHECK(IsEnabled(BMI1));
3857 EnsureSpace ensure_space(this);
3858 emit(0xF3);
3859 emit_optional_rex_32(dst, src);
3860 emit(0x0F);
3861 emit(0xBC);
3862 emit_operand(dst, src);
3863 }
3864
3865
lzcntq(Register dst,Register src)3866 void Assembler::lzcntq(Register dst, Register src) {
3867 DCHECK(IsEnabled(LZCNT));
3868 EnsureSpace ensure_space(this);
3869 emit(0xF3);
3870 emit_rex_64(dst, src);
3871 emit(0x0F);
3872 emit(0xBD);
3873 emit_modrm(dst, src);
3874 }
3875
3876
lzcntq(Register dst,const Operand & src)3877 void Assembler::lzcntq(Register dst, const Operand& src) {
3878 DCHECK(IsEnabled(LZCNT));
3879 EnsureSpace ensure_space(this);
3880 emit(0xF3);
3881 emit_rex_64(dst, src);
3882 emit(0x0F);
3883 emit(0xBD);
3884 emit_operand(dst, src);
3885 }
3886
3887
lzcntl(Register dst,Register src)3888 void Assembler::lzcntl(Register dst, Register src) {
3889 DCHECK(IsEnabled(LZCNT));
3890 EnsureSpace ensure_space(this);
3891 emit(0xF3);
3892 emit_optional_rex_32(dst, src);
3893 emit(0x0F);
3894 emit(0xBD);
3895 emit_modrm(dst, src);
3896 }
3897
3898
lzcntl(Register dst,const Operand & src)3899 void Assembler::lzcntl(Register dst, const Operand& src) {
3900 DCHECK(IsEnabled(LZCNT));
3901 EnsureSpace ensure_space(this);
3902 emit(0xF3);
3903 emit_optional_rex_32(dst, src);
3904 emit(0x0F);
3905 emit(0xBD);
3906 emit_operand(dst, src);
3907 }
3908
3909
popcntq(Register dst,Register src)3910 void Assembler::popcntq(Register dst, Register src) {
3911 DCHECK(IsEnabled(POPCNT));
3912 EnsureSpace ensure_space(this);
3913 emit(0xF3);
3914 emit_rex_64(dst, src);
3915 emit(0x0F);
3916 emit(0xB8);
3917 emit_modrm(dst, src);
3918 }
3919
3920
popcntq(Register dst,const Operand & src)3921 void Assembler::popcntq(Register dst, const Operand& src) {
3922 DCHECK(IsEnabled(POPCNT));
3923 EnsureSpace ensure_space(this);
3924 emit(0xF3);
3925 emit_rex_64(dst, src);
3926 emit(0x0F);
3927 emit(0xB8);
3928 emit_operand(dst, src);
3929 }
3930
3931
popcntl(Register dst,Register src)3932 void Assembler::popcntl(Register dst, Register src) {
3933 DCHECK(IsEnabled(POPCNT));
3934 EnsureSpace ensure_space(this);
3935 emit(0xF3);
3936 emit_optional_rex_32(dst, src);
3937 emit(0x0F);
3938 emit(0xB8);
3939 emit_modrm(dst, src);
3940 }
3941
3942
popcntl(Register dst,const Operand & src)3943 void Assembler::popcntl(Register dst, const Operand& src) {
3944 DCHECK(IsEnabled(POPCNT));
3945 EnsureSpace ensure_space(this);
3946 emit(0xF3);
3947 emit_optional_rex_32(dst, src);
3948 emit(0x0F);
3949 emit(0xB8);
3950 emit_operand(dst, src);
3951 }
3952
3953
bmi2q(SIMDPrefix pp,byte op,Register reg,Register vreg,Register rm)3954 void Assembler::bmi2q(SIMDPrefix pp, byte op, Register reg, Register vreg,
3955 Register rm) {
3956 DCHECK(IsEnabled(BMI2));
3957 EnsureSpace ensure_space(this);
3958 emit_vex_prefix(reg, vreg, rm, kLZ, pp, k0F38, kW1);
3959 emit(op);
3960 emit_modrm(reg, rm);
3961 }
3962
3963
bmi2q(SIMDPrefix pp,byte op,Register reg,Register vreg,const Operand & rm)3964 void Assembler::bmi2q(SIMDPrefix pp, byte op, Register reg, Register vreg,
3965 const Operand& rm) {
3966 DCHECK(IsEnabled(BMI2));
3967 EnsureSpace ensure_space(this);
3968 emit_vex_prefix(reg, vreg, rm, kLZ, pp, k0F38, kW1);
3969 emit(op);
3970 emit_operand(reg, rm);
3971 }
3972
3973
bmi2l(SIMDPrefix pp,byte op,Register reg,Register vreg,Register rm)3974 void Assembler::bmi2l(SIMDPrefix pp, byte op, Register reg, Register vreg,
3975 Register rm) {
3976 DCHECK(IsEnabled(BMI2));
3977 EnsureSpace ensure_space(this);
3978 emit_vex_prefix(reg, vreg, rm, kLZ, pp, k0F38, kW0);
3979 emit(op);
3980 emit_modrm(reg, rm);
3981 }
3982
3983
bmi2l(SIMDPrefix pp,byte op,Register reg,Register vreg,const Operand & rm)3984 void Assembler::bmi2l(SIMDPrefix pp, byte op, Register reg, Register vreg,
3985 const Operand& rm) {
3986 DCHECK(IsEnabled(BMI2));
3987 EnsureSpace ensure_space(this);
3988 emit_vex_prefix(reg, vreg, rm, kLZ, pp, k0F38, kW0);
3989 emit(op);
3990 emit_operand(reg, rm);
3991 }
3992
3993
rorxq(Register dst,Register src,byte imm8)3994 void Assembler::rorxq(Register dst, Register src, byte imm8) {
3995 DCHECK(IsEnabled(BMI2));
3996 DCHECK(is_uint8(imm8));
3997 Register vreg = {0}; // VEX.vvvv unused
3998 EnsureSpace ensure_space(this);
3999 emit_vex_prefix(dst, vreg, src, kLZ, kF2, k0F3A, kW1);
4000 emit(0xF0);
4001 emit_modrm(dst, src);
4002 emit(imm8);
4003 }
4004
4005
rorxq(Register dst,const Operand & src,byte imm8)4006 void Assembler::rorxq(Register dst, const Operand& src, byte imm8) {
4007 DCHECK(IsEnabled(BMI2));
4008 DCHECK(is_uint8(imm8));
4009 Register vreg = {0}; // VEX.vvvv unused
4010 EnsureSpace ensure_space(this);
4011 emit_vex_prefix(dst, vreg, src, kLZ, kF2, k0F3A, kW1);
4012 emit(0xF0);
4013 emit_operand(dst, src);
4014 emit(imm8);
4015 }
4016
4017
rorxl(Register dst,Register src,byte imm8)4018 void Assembler::rorxl(Register dst, Register src, byte imm8) {
4019 DCHECK(IsEnabled(BMI2));
4020 DCHECK(is_uint8(imm8));
4021 Register vreg = {0}; // VEX.vvvv unused
4022 EnsureSpace ensure_space(this);
4023 emit_vex_prefix(dst, vreg, src, kLZ, kF2, k0F3A, kW0);
4024 emit(0xF0);
4025 emit_modrm(dst, src);
4026 emit(imm8);
4027 }
4028
4029
rorxl(Register dst,const Operand & src,byte imm8)4030 void Assembler::rorxl(Register dst, const Operand& src, byte imm8) {
4031 DCHECK(IsEnabled(BMI2));
4032 DCHECK(is_uint8(imm8));
4033 Register vreg = {0}; // VEX.vvvv unused
4034 EnsureSpace ensure_space(this);
4035 emit_vex_prefix(dst, vreg, src, kLZ, kF2, k0F3A, kW0);
4036 emit(0xF0);
4037 emit_operand(dst, src);
4038 emit(imm8);
4039 }
4040
4041
emit_sse_operand(XMMRegister reg,const Operand & adr)4042 void Assembler::emit_sse_operand(XMMRegister reg, const Operand& adr) {
4043 Register ireg = { reg.code() };
4044 emit_operand(ireg, adr);
4045 }
4046
4047
emit_sse_operand(Register reg,const Operand & adr)4048 void Assembler::emit_sse_operand(Register reg, const Operand& adr) {
4049 Register ireg = {reg.code()};
4050 emit_operand(ireg, adr);
4051 }
4052
4053
emit_sse_operand(XMMRegister dst,XMMRegister src)4054 void Assembler::emit_sse_operand(XMMRegister dst, XMMRegister src) {
4055 emit(0xC0 | (dst.low_bits() << 3) | src.low_bits());
4056 }
4057
4058
emit_sse_operand(XMMRegister dst,Register src)4059 void Assembler::emit_sse_operand(XMMRegister dst, Register src) {
4060 emit(0xC0 | (dst.low_bits() << 3) | src.low_bits());
4061 }
4062
4063
emit_sse_operand(Register dst,XMMRegister src)4064 void Assembler::emit_sse_operand(Register dst, XMMRegister src) {
4065 emit(0xC0 | (dst.low_bits() << 3) | src.low_bits());
4066 }
4067
4068
db(uint8_t data)4069 void Assembler::db(uint8_t data) {
4070 EnsureSpace ensure_space(this);
4071 emit(data);
4072 }
4073
4074
dd(uint32_t data)4075 void Assembler::dd(uint32_t data) {
4076 EnsureSpace ensure_space(this);
4077 emitl(data);
4078 }
4079
4080
dq(uint64_t data)4081 void Assembler::dq(uint64_t data) {
4082 EnsureSpace ensure_space(this);
4083 emitq(data);
4084 }
4085
4086
dq(Label * label)4087 void Assembler::dq(Label* label) {
4088 EnsureSpace ensure_space(this);
4089 if (label->is_bound()) {
4090 internal_reference_positions_.push_back(pc_offset());
4091 emitp(buffer_ + label->pos(), RelocInfo::INTERNAL_REFERENCE);
4092 } else {
4093 RecordRelocInfo(RelocInfo::INTERNAL_REFERENCE);
4094 emitl(0); // Zero for the first 32bit marks it as 64bit absolute address.
4095 if (label->is_linked()) {
4096 emitl(label->pos());
4097 label->link_to(pc_offset() - sizeof(int32_t));
4098 } else {
4099 DCHECK(label->is_unused());
4100 int32_t current = pc_offset();
4101 emitl(current);
4102 label->link_to(current);
4103 }
4104 }
4105 }
4106
4107
4108 // Relocation information implementations.
4109
RecordRelocInfo(RelocInfo::Mode rmode,intptr_t data)4110 void Assembler::RecordRelocInfo(RelocInfo::Mode rmode, intptr_t data) {
4111 DCHECK(!RelocInfo::IsNone(rmode));
4112 // Don't record external references unless the heap will be serialized.
4113 if (rmode == RelocInfo::EXTERNAL_REFERENCE &&
4114 !serializer_enabled() && !emit_debug_code()) {
4115 return;
4116 } else if (rmode == RelocInfo::CODE_AGE_SEQUENCE) {
4117 // Don't record psuedo relocation info for code age sequence mode.
4118 return;
4119 }
4120 RelocInfo rinfo(isolate(), pc_, rmode, data, NULL);
4121 reloc_info_writer.Write(&rinfo);
4122 }
4123
4124
4125 const int RelocInfo::kApplyMask = RelocInfo::kCodeTargetMask |
4126 1 << RelocInfo::RUNTIME_ENTRY |
4127 1 << RelocInfo::INTERNAL_REFERENCE |
4128 1 << RelocInfo::CODE_AGE_SEQUENCE;
4129
4130
IsCodedSpecially()4131 bool RelocInfo::IsCodedSpecially() {
4132 // The deserializer needs to know whether a pointer is specially coded. Being
4133 // specially coded on x64 means that it is a relative 32 bit address, as used
4134 // by branch instructions.
4135 return (1 << rmode_) & kApplyMask;
4136 }
4137
4138
IsInConstantPool()4139 bool RelocInfo::IsInConstantPool() {
4140 return false;
4141 }
4142
4143
4144 } // namespace internal
4145 } // namespace v8
4146
4147 #endif // V8_TARGET_ARCH_X64
4148