1 /*
2  * Copyright (C) 2014 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 #include "assembler_x86_64.h"
18 
19 #include "base/casts.h"
20 #include "entrypoints/quick/quick_entrypoints.h"
21 #include "memory_region.h"
22 #include "thread.h"
23 
24 namespace art {
25 namespace x86_64 {
26 
operator <<(std::ostream & os,const CpuRegister & reg)27 std::ostream& operator<<(std::ostream& os, const CpuRegister& reg) {
28   return os << reg.AsRegister();
29 }
30 
operator <<(std::ostream & os,const XmmRegister & reg)31 std::ostream& operator<<(std::ostream& os, const XmmRegister& reg) {
32   return os << reg.AsFloatRegister();
33 }
34 
operator <<(std::ostream & os,const X87Register & reg)35 std::ostream& operator<<(std::ostream& os, const X87Register& reg) {
36   return os << "ST" << static_cast<int>(reg);
37 }
38 
operator <<(std::ostream & os,const Address & addr)39 std::ostream& operator<<(std::ostream& os, const Address& addr) {
40   switch (addr.mod()) {
41     case 0:
42       if (addr.rm() != RSP || addr.cpu_index().AsRegister() == RSP) {
43         return os << "(%" << addr.cpu_rm() << ")";
44       } else if (addr.base() == RBP) {
45         return os << static_cast<int>(addr.disp32()) << "(,%" << addr.cpu_index()
46                   << "," << (1 << addr.scale()) << ")";
47       }
48       return os << "(%" << addr.cpu_base() << ",%"
49                 << addr.cpu_index() << "," << (1 << addr.scale()) << ")";
50     case 1:
51       if (addr.rm() != RSP || addr.cpu_index().AsRegister() == RSP) {
52         return os << static_cast<int>(addr.disp8()) << "(%" << addr.cpu_rm() << ")";
53       }
54       return os << static_cast<int>(addr.disp8()) << "(%" << addr.cpu_base() << ",%"
55                 << addr.cpu_index() << "," << (1 << addr.scale()) << ")";
56     case 2:
57       if (addr.rm() != RSP || addr.cpu_index().AsRegister() == RSP) {
58         return os << static_cast<int>(addr.disp32()) << "(%" << addr.cpu_rm() << ")";
59       }
60       return os << static_cast<int>(addr.disp32()) << "(%" << addr.cpu_base() << ",%"
61                 << addr.cpu_index() << "," << (1 << addr.scale()) << ")";
62     default:
63       return os << "<address?>";
64   }
65 }
66 
call(CpuRegister reg)67 void X86_64Assembler::call(CpuRegister reg) {
68   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
69   EmitOptionalRex32(reg);
70   EmitUint8(0xFF);
71   EmitRegisterOperand(2, reg.LowBits());
72 }
73 
74 
call(const Address & address)75 void X86_64Assembler::call(const Address& address) {
76   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
77   EmitOptionalRex32(address);
78   EmitUint8(0xFF);
79   EmitOperand(2, address);
80 }
81 
82 
call(Label * label)83 void X86_64Assembler::call(Label* label) {
84   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
85   EmitUint8(0xE8);
86   static const int kSize = 5;
87   // Offset by one because we already have emitted the opcode.
88   EmitLabel(label, kSize - 1);
89 }
90 
pushq(CpuRegister reg)91 void X86_64Assembler::pushq(CpuRegister reg) {
92   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
93   EmitOptionalRex32(reg);
94   EmitUint8(0x50 + reg.LowBits());
95 }
96 
97 
pushq(const Address & address)98 void X86_64Assembler::pushq(const Address& address) {
99   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
100   EmitOptionalRex32(address);
101   EmitUint8(0xFF);
102   EmitOperand(6, address);
103 }
104 
105 
pushq(const Immediate & imm)106 void X86_64Assembler::pushq(const Immediate& imm) {
107   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
108   CHECK(imm.is_int32());  // pushq only supports 32b immediate.
109   if (imm.is_int8()) {
110     EmitUint8(0x6A);
111     EmitUint8(imm.value() & 0xFF);
112   } else {
113     EmitUint8(0x68);
114     EmitImmediate(imm);
115   }
116 }
117 
118 
popq(CpuRegister reg)119 void X86_64Assembler::popq(CpuRegister reg) {
120   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
121   EmitOptionalRex32(reg);
122   EmitUint8(0x58 + reg.LowBits());
123 }
124 
125 
popq(const Address & address)126 void X86_64Assembler::popq(const Address& address) {
127   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
128   EmitOptionalRex32(address);
129   EmitUint8(0x8F);
130   EmitOperand(0, address);
131 }
132 
133 
movq(CpuRegister dst,const Immediate & imm)134 void X86_64Assembler::movq(CpuRegister dst, const Immediate& imm) {
135   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
136   if (imm.is_int32()) {
137     // 32 bit. Note: sign-extends.
138     EmitRex64(dst);
139     EmitUint8(0xC7);
140     EmitRegisterOperand(0, dst.LowBits());
141     EmitInt32(static_cast<int32_t>(imm.value()));
142   } else {
143     EmitRex64(dst);
144     EmitUint8(0xB8 + dst.LowBits());
145     EmitInt64(imm.value());
146   }
147 }
148 
149 
movl(CpuRegister dst,const Immediate & imm)150 void X86_64Assembler::movl(CpuRegister dst, const Immediate& imm) {
151   CHECK(imm.is_int32());
152   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
153   EmitOptionalRex32(dst);
154   EmitUint8(0xB8 + dst.LowBits());
155   EmitImmediate(imm);
156 }
157 
158 
movq(const Address & dst,const Immediate & imm)159 void X86_64Assembler::movq(const Address& dst, const Immediate& imm) {
160   CHECK(imm.is_int32());
161   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
162   EmitRex64(dst);
163   EmitUint8(0xC7);
164   EmitOperand(0, dst);
165   EmitImmediate(imm);
166 }
167 
168 
movq(CpuRegister dst,CpuRegister src)169 void X86_64Assembler::movq(CpuRegister dst, CpuRegister src) {
170   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
171   // 0x89 is movq r/m64 <- r64, with op1 in r/m and op2 in reg: so reverse EmitRex64
172   EmitRex64(src, dst);
173   EmitUint8(0x89);
174   EmitRegisterOperand(src.LowBits(), dst.LowBits());
175 }
176 
177 
movl(CpuRegister dst,CpuRegister src)178 void X86_64Assembler::movl(CpuRegister dst, CpuRegister src) {
179   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
180   EmitOptionalRex32(dst, src);
181   EmitUint8(0x8B);
182   EmitRegisterOperand(dst.LowBits(), src.LowBits());
183 }
184 
185 
movq(CpuRegister dst,const Address & src)186 void X86_64Assembler::movq(CpuRegister dst, const Address& src) {
187   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
188   EmitRex64(dst, src);
189   EmitUint8(0x8B);
190   EmitOperand(dst.LowBits(), src);
191 }
192 
193 
movl(CpuRegister dst,const Address & src)194 void X86_64Assembler::movl(CpuRegister dst, const Address& src) {
195   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
196   EmitOptionalRex32(dst, src);
197   EmitUint8(0x8B);
198   EmitOperand(dst.LowBits(), src);
199 }
200 
201 
movq(const Address & dst,CpuRegister src)202 void X86_64Assembler::movq(const Address& dst, CpuRegister src) {
203   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
204   EmitRex64(src, dst);
205   EmitUint8(0x89);
206   EmitOperand(src.LowBits(), dst);
207 }
208 
209 
movl(const Address & dst,CpuRegister src)210 void X86_64Assembler::movl(const Address& dst, CpuRegister src) {
211   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
212   EmitOptionalRex32(src, dst);
213   EmitUint8(0x89);
214   EmitOperand(src.LowBits(), dst);
215 }
216 
movl(const Address & dst,const Immediate & imm)217 void X86_64Assembler::movl(const Address& dst, const Immediate& imm) {
218   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
219   EmitOptionalRex32(dst);
220   EmitUint8(0xC7);
221   EmitOperand(0, dst);
222   EmitImmediate(imm);
223 }
224 
movntl(const Address & dst,CpuRegister src)225 void X86_64Assembler::movntl(const Address& dst, CpuRegister src) {
226   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
227   EmitOptionalRex32(src, dst);
228   EmitUint8(0x0F);
229   EmitUint8(0xC3);
230   EmitOperand(src.LowBits(), dst);
231 }
232 
movntq(const Address & dst,CpuRegister src)233 void X86_64Assembler::movntq(const Address& dst, CpuRegister src) {
234   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
235   EmitRex64(src, dst);
236   EmitUint8(0x0F);
237   EmitUint8(0xC3);
238   EmitOperand(src.LowBits(), dst);
239 }
240 
cmov(Condition c,CpuRegister dst,CpuRegister src)241 void X86_64Assembler::cmov(Condition c, CpuRegister dst, CpuRegister src) {
242   cmov(c, dst, src, true);
243 }
244 
cmov(Condition c,CpuRegister dst,CpuRegister src,bool is64bit)245 void X86_64Assembler::cmov(Condition c, CpuRegister dst, CpuRegister src, bool is64bit) {
246   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
247   EmitOptionalRex(false, is64bit, dst.NeedsRex(), false, src.NeedsRex());
248   EmitUint8(0x0F);
249   EmitUint8(0x40 + c);
250   EmitRegisterOperand(dst.LowBits(), src.LowBits());
251 }
252 
253 
cmov(Condition c,CpuRegister dst,const Address & src,bool is64bit)254 void X86_64Assembler::cmov(Condition c, CpuRegister dst, const Address& src, bool is64bit) {
255   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
256   if (is64bit) {
257     EmitRex64(dst, src);
258   } else {
259     EmitOptionalRex32(dst, src);
260   }
261   EmitUint8(0x0F);
262   EmitUint8(0x40 + c);
263   EmitOperand(dst.LowBits(), src);
264 }
265 
266 
movzxb(CpuRegister dst,CpuRegister src)267 void X86_64Assembler::movzxb(CpuRegister dst, CpuRegister src) {
268   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
269   EmitOptionalByteRegNormalizingRex32(dst, src);
270   EmitUint8(0x0F);
271   EmitUint8(0xB6);
272   EmitRegisterOperand(dst.LowBits(), src.LowBits());
273 }
274 
275 
movzxb(CpuRegister dst,const Address & src)276 void X86_64Assembler::movzxb(CpuRegister dst, const Address& src) {
277   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
278   // Byte register is only in the source register form, so we don't use
279   // EmitOptionalByteRegNormalizingRex32(dst, src);
280   EmitOptionalRex32(dst, src);
281   EmitUint8(0x0F);
282   EmitUint8(0xB6);
283   EmitOperand(dst.LowBits(), src);
284 }
285 
286 
movsxb(CpuRegister dst,CpuRegister src)287 void X86_64Assembler::movsxb(CpuRegister dst, CpuRegister src) {
288   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
289   EmitOptionalByteRegNormalizingRex32(dst, src);
290   EmitUint8(0x0F);
291   EmitUint8(0xBE);
292   EmitRegisterOperand(dst.LowBits(), src.LowBits());
293 }
294 
295 
movsxb(CpuRegister dst,const Address & src)296 void X86_64Assembler::movsxb(CpuRegister dst, const Address& src) {
297   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
298   // Byte register is only in the source register form, so we don't use
299   // EmitOptionalByteRegNormalizingRex32(dst, src);
300   EmitOptionalRex32(dst, src);
301   EmitUint8(0x0F);
302   EmitUint8(0xBE);
303   EmitOperand(dst.LowBits(), src);
304 }
305 
306 
movb(CpuRegister,const Address &)307 void X86_64Assembler::movb(CpuRegister /*dst*/, const Address& /*src*/) {
308   LOG(FATAL) << "Use movzxb or movsxb instead.";
309 }
310 
311 
movb(const Address & dst,CpuRegister src)312 void X86_64Assembler::movb(const Address& dst, CpuRegister src) {
313   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
314   EmitOptionalByteRegNormalizingRex32(src, dst);
315   EmitUint8(0x88);
316   EmitOperand(src.LowBits(), dst);
317 }
318 
319 
movb(const Address & dst,const Immediate & imm)320 void X86_64Assembler::movb(const Address& dst, const Immediate& imm) {
321   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
322   EmitOptionalRex32(dst);
323   EmitUint8(0xC6);
324   EmitOperand(Register::RAX, dst);
325   CHECK(imm.is_int8());
326   EmitUint8(imm.value() & 0xFF);
327 }
328 
329 
movzxw(CpuRegister dst,CpuRegister src)330 void X86_64Assembler::movzxw(CpuRegister dst, CpuRegister src) {
331   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
332   EmitOptionalRex32(dst, src);
333   EmitUint8(0x0F);
334   EmitUint8(0xB7);
335   EmitRegisterOperand(dst.LowBits(), src.LowBits());
336 }
337 
338 
movzxw(CpuRegister dst,const Address & src)339 void X86_64Assembler::movzxw(CpuRegister dst, const Address& src) {
340   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
341   EmitOptionalRex32(dst, src);
342   EmitUint8(0x0F);
343   EmitUint8(0xB7);
344   EmitOperand(dst.LowBits(), src);
345 }
346 
347 
movsxw(CpuRegister dst,CpuRegister src)348 void X86_64Assembler::movsxw(CpuRegister dst, CpuRegister src) {
349   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
350   EmitOptionalRex32(dst, src);
351   EmitUint8(0x0F);
352   EmitUint8(0xBF);
353   EmitRegisterOperand(dst.LowBits(), src.LowBits());
354 }
355 
356 
movsxw(CpuRegister dst,const Address & src)357 void X86_64Assembler::movsxw(CpuRegister dst, const Address& src) {
358   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
359   EmitOptionalRex32(dst, src);
360   EmitUint8(0x0F);
361   EmitUint8(0xBF);
362   EmitOperand(dst.LowBits(), src);
363 }
364 
365 
movw(CpuRegister,const Address &)366 void X86_64Assembler::movw(CpuRegister /*dst*/, const Address& /*src*/) {
367   LOG(FATAL) << "Use movzxw or movsxw instead.";
368 }
369 
370 
movw(const Address & dst,CpuRegister src)371 void X86_64Assembler::movw(const Address& dst, CpuRegister src) {
372   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
373   EmitOperandSizeOverride();
374   EmitOptionalRex32(src, dst);
375   EmitUint8(0x89);
376   EmitOperand(src.LowBits(), dst);
377 }
378 
379 
movw(const Address & dst,const Immediate & imm)380 void X86_64Assembler::movw(const Address& dst, const Immediate& imm) {
381   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
382   EmitOperandSizeOverride();
383   EmitOptionalRex32(dst);
384   EmitUint8(0xC7);
385   EmitOperand(Register::RAX, dst);
386   CHECK(imm.is_uint16() || imm.is_int16());
387   EmitUint8(imm.value() & 0xFF);
388   EmitUint8(imm.value() >> 8);
389 }
390 
391 
leaq(CpuRegister dst,const Address & src)392 void X86_64Assembler::leaq(CpuRegister dst, const Address& src) {
393   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
394   EmitRex64(dst, src);
395   EmitUint8(0x8D);
396   EmitOperand(dst.LowBits(), src);
397 }
398 
399 
leal(CpuRegister dst,const Address & src)400 void X86_64Assembler::leal(CpuRegister dst, const Address& src) {
401   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
402   EmitOptionalRex32(dst, src);
403   EmitUint8(0x8D);
404   EmitOperand(dst.LowBits(), src);
405 }
406 
407 
movaps(XmmRegister dst,XmmRegister src)408 void X86_64Assembler::movaps(XmmRegister dst, XmmRegister src) {
409   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
410   EmitOptionalRex32(dst, src);
411   EmitUint8(0x0F);
412   EmitUint8(0x28);
413   EmitXmmRegisterOperand(dst.LowBits(), src);
414 }
415 
416 
movaps(XmmRegister dst,const Address & src)417 void X86_64Assembler::movaps(XmmRegister dst, const Address& src) {
418   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
419   EmitOptionalRex32(dst, src);
420   EmitUint8(0x0F);
421   EmitUint8(0x28);
422   EmitOperand(dst.LowBits(), src);
423 }
424 
425 
movups(XmmRegister dst,const Address & src)426 void X86_64Assembler::movups(XmmRegister dst, const Address& src) {
427   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
428   EmitOptionalRex32(dst, src);
429   EmitUint8(0x0F);
430   EmitUint8(0x10);
431   EmitOperand(dst.LowBits(), src);
432 }
433 
434 
movaps(const Address & dst,XmmRegister src)435 void X86_64Assembler::movaps(const Address& dst, XmmRegister src) {
436   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
437   EmitOptionalRex32(src, dst);
438   EmitUint8(0x0F);
439   EmitUint8(0x29);
440   EmitOperand(src.LowBits(), dst);
441 }
442 
443 
movups(const Address & dst,XmmRegister src)444 void X86_64Assembler::movups(const Address& dst, XmmRegister src) {
445   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
446   EmitOptionalRex32(src, dst);
447   EmitUint8(0x0F);
448   EmitUint8(0x11);
449   EmitOperand(src.LowBits(), dst);
450 }
451 
452 
movss(XmmRegister dst,const Address & src)453 void X86_64Assembler::movss(XmmRegister dst, const Address& src) {
454   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
455   EmitUint8(0xF3);
456   EmitOptionalRex32(dst, src);
457   EmitUint8(0x0F);
458   EmitUint8(0x10);
459   EmitOperand(dst.LowBits(), src);
460 }
461 
462 
movss(const Address & dst,XmmRegister src)463 void X86_64Assembler::movss(const Address& dst, XmmRegister src) {
464   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
465   EmitUint8(0xF3);
466   EmitOptionalRex32(src, dst);
467   EmitUint8(0x0F);
468   EmitUint8(0x11);
469   EmitOperand(src.LowBits(), dst);
470 }
471 
472 
movss(XmmRegister dst,XmmRegister src)473 void X86_64Assembler::movss(XmmRegister dst, XmmRegister src) {
474   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
475   EmitUint8(0xF3);
476   EmitOptionalRex32(src, dst);  // Movss is MR encoding instead of the usual RM.
477   EmitUint8(0x0F);
478   EmitUint8(0x11);
479   EmitXmmRegisterOperand(src.LowBits(), dst);
480 }
481 
482 
movsxd(CpuRegister dst,CpuRegister src)483 void X86_64Assembler::movsxd(CpuRegister dst, CpuRegister src) {
484   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
485   EmitRex64(dst, src);
486   EmitUint8(0x63);
487   EmitRegisterOperand(dst.LowBits(), src.LowBits());
488 }
489 
490 
movsxd(CpuRegister dst,const Address & src)491 void X86_64Assembler::movsxd(CpuRegister dst, const Address& src) {
492   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
493   EmitRex64(dst, src);
494   EmitUint8(0x63);
495   EmitOperand(dst.LowBits(), src);
496 }
497 
498 
movd(XmmRegister dst,CpuRegister src)499 void X86_64Assembler::movd(XmmRegister dst, CpuRegister src) {
500   movd(dst, src, true);
501 }
502 
movd(CpuRegister dst,XmmRegister src)503 void X86_64Assembler::movd(CpuRegister dst, XmmRegister src) {
504   movd(dst, src, true);
505 }
506 
movd(XmmRegister dst,CpuRegister src,bool is64bit)507 void X86_64Assembler::movd(XmmRegister dst, CpuRegister src, bool is64bit) {
508   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
509   EmitUint8(0x66);
510   EmitOptionalRex(false, is64bit, dst.NeedsRex(), false, src.NeedsRex());
511   EmitUint8(0x0F);
512   EmitUint8(0x6E);
513   EmitOperand(dst.LowBits(), Operand(src));
514 }
515 
movd(CpuRegister dst,XmmRegister src,bool is64bit)516 void X86_64Assembler::movd(CpuRegister dst, XmmRegister src, bool is64bit) {
517   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
518   EmitUint8(0x66);
519   EmitOptionalRex(false, is64bit, src.NeedsRex(), false, dst.NeedsRex());
520   EmitUint8(0x0F);
521   EmitUint8(0x7E);
522   EmitOperand(src.LowBits(), Operand(dst));
523 }
524 
525 
addss(XmmRegister dst,XmmRegister src)526 void X86_64Assembler::addss(XmmRegister dst, XmmRegister src) {
527   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
528   EmitUint8(0xF3);
529   EmitOptionalRex32(dst, src);
530   EmitUint8(0x0F);
531   EmitUint8(0x58);
532   EmitXmmRegisterOperand(dst.LowBits(), src);
533 }
534 
535 
addss(XmmRegister dst,const Address & src)536 void X86_64Assembler::addss(XmmRegister dst, const Address& src) {
537   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
538   EmitUint8(0xF3);
539   EmitOptionalRex32(dst, src);
540   EmitUint8(0x0F);
541   EmitUint8(0x58);
542   EmitOperand(dst.LowBits(), src);
543 }
544 
545 
subss(XmmRegister dst,XmmRegister src)546 void X86_64Assembler::subss(XmmRegister dst, XmmRegister src) {
547   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
548   EmitUint8(0xF3);
549   EmitOptionalRex32(dst, src);
550   EmitUint8(0x0F);
551   EmitUint8(0x5C);
552   EmitXmmRegisterOperand(dst.LowBits(), src);
553 }
554 
555 
subss(XmmRegister dst,const Address & src)556 void X86_64Assembler::subss(XmmRegister dst, const Address& src) {
557   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
558   EmitUint8(0xF3);
559   EmitOptionalRex32(dst, src);
560   EmitUint8(0x0F);
561   EmitUint8(0x5C);
562   EmitOperand(dst.LowBits(), src);
563 }
564 
565 
mulss(XmmRegister dst,XmmRegister src)566 void X86_64Assembler::mulss(XmmRegister dst, XmmRegister src) {
567   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
568   EmitUint8(0xF3);
569   EmitOptionalRex32(dst, src);
570   EmitUint8(0x0F);
571   EmitUint8(0x59);
572   EmitXmmRegisterOperand(dst.LowBits(), src);
573 }
574 
575 
mulss(XmmRegister dst,const Address & src)576 void X86_64Assembler::mulss(XmmRegister dst, const Address& src) {
577   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
578   EmitUint8(0xF3);
579   EmitOptionalRex32(dst, src);
580   EmitUint8(0x0F);
581   EmitUint8(0x59);
582   EmitOperand(dst.LowBits(), src);
583 }
584 
585 
divss(XmmRegister dst,XmmRegister src)586 void X86_64Assembler::divss(XmmRegister dst, XmmRegister src) {
587   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
588   EmitUint8(0xF3);
589   EmitOptionalRex32(dst, src);
590   EmitUint8(0x0F);
591   EmitUint8(0x5E);
592   EmitXmmRegisterOperand(dst.LowBits(), src);
593 }
594 
595 
divss(XmmRegister dst,const Address & src)596 void X86_64Assembler::divss(XmmRegister dst, const Address& src) {
597   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
598   EmitUint8(0xF3);
599   EmitOptionalRex32(dst, src);
600   EmitUint8(0x0F);
601   EmitUint8(0x5E);
602   EmitOperand(dst.LowBits(), src);
603 }
604 
605 
addps(XmmRegister dst,XmmRegister src)606 void X86_64Assembler::addps(XmmRegister dst, XmmRegister src) {
607   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
608   EmitOptionalRex32(dst, src);
609   EmitUint8(0x0F);
610   EmitUint8(0x58);
611   EmitXmmRegisterOperand(dst.LowBits(), src);
612 }
613 
614 
subps(XmmRegister dst,XmmRegister src)615 void X86_64Assembler::subps(XmmRegister dst, XmmRegister src) {
616   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
617   EmitOptionalRex32(dst, src);
618   EmitUint8(0x0F);
619   EmitUint8(0x5C);
620   EmitXmmRegisterOperand(dst.LowBits(), src);
621 }
622 
623 
mulps(XmmRegister dst,XmmRegister src)624 void X86_64Assembler::mulps(XmmRegister dst, XmmRegister src) {
625   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
626   EmitOptionalRex32(dst, src);
627   EmitUint8(0x0F);
628   EmitUint8(0x59);
629   EmitXmmRegisterOperand(dst.LowBits(), src);
630 }
631 
632 
divps(XmmRegister dst,XmmRegister src)633 void X86_64Assembler::divps(XmmRegister dst, XmmRegister src) {
634   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
635   EmitOptionalRex32(dst, src);
636   EmitUint8(0x0F);
637   EmitUint8(0x5E);
638   EmitXmmRegisterOperand(dst.LowBits(), src);
639 }
640 
641 
flds(const Address & src)642 void X86_64Assembler::flds(const Address& src) {
643   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
644   EmitUint8(0xD9);
645   EmitOperand(0, src);
646 }
647 
648 
fsts(const Address & dst)649 void X86_64Assembler::fsts(const Address& dst) {
650   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
651   EmitUint8(0xD9);
652   EmitOperand(2, dst);
653 }
654 
655 
fstps(const Address & dst)656 void X86_64Assembler::fstps(const Address& dst) {
657   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
658   EmitUint8(0xD9);
659   EmitOperand(3, dst);
660 }
661 
662 
movapd(XmmRegister dst,XmmRegister src)663 void X86_64Assembler::movapd(XmmRegister dst, XmmRegister src) {
664   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
665   EmitUint8(0x66);
666   EmitOptionalRex32(dst, src);
667   EmitUint8(0x0F);
668   EmitUint8(0x28);
669   EmitXmmRegisterOperand(dst.LowBits(), src);
670 }
671 
672 
movapd(XmmRegister dst,const Address & src)673 void X86_64Assembler::movapd(XmmRegister dst, const Address& src) {
674   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
675   EmitUint8(0x66);
676   EmitOptionalRex32(dst, src);
677   EmitUint8(0x0F);
678   EmitUint8(0x28);
679   EmitOperand(dst.LowBits(), src);
680 }
681 
682 
movupd(XmmRegister dst,const Address & src)683 void X86_64Assembler::movupd(XmmRegister dst, const Address& src) {
684   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
685   EmitUint8(0x66);
686   EmitOptionalRex32(dst, src);
687   EmitUint8(0x0F);
688   EmitUint8(0x10);
689   EmitOperand(dst.LowBits(), src);
690 }
691 
692 
movapd(const Address & dst,XmmRegister src)693 void X86_64Assembler::movapd(const Address& dst, XmmRegister src) {
694   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
695   EmitUint8(0x66);
696   EmitOptionalRex32(src, dst);
697   EmitUint8(0x0F);
698   EmitUint8(0x29);
699   EmitOperand(src.LowBits(), dst);
700 }
701 
702 
movupd(const Address & dst,XmmRegister src)703 void X86_64Assembler::movupd(const Address& dst, XmmRegister src) {
704   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
705   EmitUint8(0x66);
706   EmitOptionalRex32(src, dst);
707   EmitUint8(0x0F);
708   EmitUint8(0x11);
709   EmitOperand(src.LowBits(), dst);
710 }
711 
712 
movsd(XmmRegister dst,const Address & src)713 void X86_64Assembler::movsd(XmmRegister dst, const Address& src) {
714   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
715   EmitUint8(0xF2);
716   EmitOptionalRex32(dst, src);
717   EmitUint8(0x0F);
718   EmitUint8(0x10);
719   EmitOperand(dst.LowBits(), src);
720 }
721 
722 
movsd(const Address & dst,XmmRegister src)723 void X86_64Assembler::movsd(const Address& dst, XmmRegister src) {
724   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
725   EmitUint8(0xF2);
726   EmitOptionalRex32(src, dst);
727   EmitUint8(0x0F);
728   EmitUint8(0x11);
729   EmitOperand(src.LowBits(), dst);
730 }
731 
732 
movsd(XmmRegister dst,XmmRegister src)733 void X86_64Assembler::movsd(XmmRegister dst, XmmRegister src) {
734   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
735   EmitUint8(0xF2);
736   EmitOptionalRex32(src, dst);  // Movsd is MR encoding instead of the usual RM.
737   EmitUint8(0x0F);
738   EmitUint8(0x11);
739   EmitXmmRegisterOperand(src.LowBits(), dst);
740 }
741 
742 
addsd(XmmRegister dst,XmmRegister src)743 void X86_64Assembler::addsd(XmmRegister dst, XmmRegister src) {
744   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
745   EmitUint8(0xF2);
746   EmitOptionalRex32(dst, src);
747   EmitUint8(0x0F);
748   EmitUint8(0x58);
749   EmitXmmRegisterOperand(dst.LowBits(), src);
750 }
751 
752 
addsd(XmmRegister dst,const Address & src)753 void X86_64Assembler::addsd(XmmRegister dst, const Address& src) {
754   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
755   EmitUint8(0xF2);
756   EmitOptionalRex32(dst, src);
757   EmitUint8(0x0F);
758   EmitUint8(0x58);
759   EmitOperand(dst.LowBits(), src);
760 }
761 
762 
subsd(XmmRegister dst,XmmRegister src)763 void X86_64Assembler::subsd(XmmRegister dst, XmmRegister src) {
764   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
765   EmitUint8(0xF2);
766   EmitOptionalRex32(dst, src);
767   EmitUint8(0x0F);
768   EmitUint8(0x5C);
769   EmitXmmRegisterOperand(dst.LowBits(), src);
770 }
771 
772 
subsd(XmmRegister dst,const Address & src)773 void X86_64Assembler::subsd(XmmRegister dst, const Address& src) {
774   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
775   EmitUint8(0xF2);
776   EmitOptionalRex32(dst, src);
777   EmitUint8(0x0F);
778   EmitUint8(0x5C);
779   EmitOperand(dst.LowBits(), src);
780 }
781 
782 
mulsd(XmmRegister dst,XmmRegister src)783 void X86_64Assembler::mulsd(XmmRegister dst, XmmRegister src) {
784   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
785   EmitUint8(0xF2);
786   EmitOptionalRex32(dst, src);
787   EmitUint8(0x0F);
788   EmitUint8(0x59);
789   EmitXmmRegisterOperand(dst.LowBits(), src);
790 }
791 
792 
mulsd(XmmRegister dst,const Address & src)793 void X86_64Assembler::mulsd(XmmRegister dst, const Address& src) {
794   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
795   EmitUint8(0xF2);
796   EmitOptionalRex32(dst, src);
797   EmitUint8(0x0F);
798   EmitUint8(0x59);
799   EmitOperand(dst.LowBits(), src);
800 }
801 
802 
divsd(XmmRegister dst,XmmRegister src)803 void X86_64Assembler::divsd(XmmRegister dst, XmmRegister src) {
804   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
805   EmitUint8(0xF2);
806   EmitOptionalRex32(dst, src);
807   EmitUint8(0x0F);
808   EmitUint8(0x5E);
809   EmitXmmRegisterOperand(dst.LowBits(), src);
810 }
811 
812 
divsd(XmmRegister dst,const Address & src)813 void X86_64Assembler::divsd(XmmRegister dst, const Address& src) {
814   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
815   EmitUint8(0xF2);
816   EmitOptionalRex32(dst, src);
817   EmitUint8(0x0F);
818   EmitUint8(0x5E);
819   EmitOperand(dst.LowBits(), src);
820 }
821 
822 
addpd(XmmRegister dst,XmmRegister src)823 void X86_64Assembler::addpd(XmmRegister dst, XmmRegister src) {
824   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
825   EmitUint8(0x66);
826   EmitOptionalRex32(dst, src);
827   EmitUint8(0x0F);
828   EmitUint8(0x58);
829   EmitXmmRegisterOperand(dst.LowBits(), src);
830 }
831 
832 
subpd(XmmRegister dst,XmmRegister src)833 void X86_64Assembler::subpd(XmmRegister dst, XmmRegister src) {
834   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
835   EmitUint8(0x66);
836   EmitOptionalRex32(dst, src);
837   EmitUint8(0x0F);
838   EmitUint8(0x5C);
839   EmitXmmRegisterOperand(dst.LowBits(), src);
840 }
841 
842 
mulpd(XmmRegister dst,XmmRegister src)843 void X86_64Assembler::mulpd(XmmRegister dst, XmmRegister src) {
844   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
845   EmitUint8(0x66);
846   EmitOptionalRex32(dst, src);
847   EmitUint8(0x0F);
848   EmitUint8(0x59);
849   EmitXmmRegisterOperand(dst.LowBits(), src);
850 }
851 
852 
divpd(XmmRegister dst,XmmRegister src)853 void X86_64Assembler::divpd(XmmRegister dst, XmmRegister src) {
854   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
855   EmitUint8(0x66);
856   EmitOptionalRex32(dst, src);
857   EmitUint8(0x0F);
858   EmitUint8(0x5E);
859   EmitXmmRegisterOperand(dst.LowBits(), src);
860 }
861 
862 
movdqa(XmmRegister dst,XmmRegister src)863 void X86_64Assembler::movdqa(XmmRegister dst, XmmRegister src) {
864   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
865   EmitUint8(0x66);
866   EmitOptionalRex32(dst, src);
867   EmitUint8(0x0F);
868   EmitUint8(0x6F);
869   EmitXmmRegisterOperand(dst.LowBits(), src);
870 }
871 
872 
movdqa(XmmRegister dst,const Address & src)873 void X86_64Assembler::movdqa(XmmRegister dst, const Address& src) {
874   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
875   EmitUint8(0x66);
876   EmitOptionalRex32(dst, src);
877   EmitUint8(0x0F);
878   EmitUint8(0x6F);
879   EmitOperand(dst.LowBits(), src);
880 }
881 
882 
movdqu(XmmRegister dst,const Address & src)883 void X86_64Assembler::movdqu(XmmRegister dst, const Address& src) {
884   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
885   EmitUint8(0xF3);
886   EmitOptionalRex32(dst, src);
887   EmitUint8(0x0F);
888   EmitUint8(0x6F);
889   EmitOperand(dst.LowBits(), src);
890 }
891 
892 
movdqa(const Address & dst,XmmRegister src)893 void X86_64Assembler::movdqa(const Address& dst, XmmRegister src) {
894   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
895   EmitUint8(0x66);
896   EmitOptionalRex32(src, dst);
897   EmitUint8(0x0F);
898   EmitUint8(0x7F);
899   EmitOperand(src.LowBits(), dst);
900 }
901 
902 
movdqu(const Address & dst,XmmRegister src)903 void X86_64Assembler::movdqu(const Address& dst, XmmRegister src) {
904   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
905   EmitUint8(0xF3);
906   EmitOptionalRex32(src, dst);
907   EmitUint8(0x0F);
908   EmitUint8(0x7F);
909   EmitOperand(src.LowBits(), dst);
910 }
911 
912 
paddb(XmmRegister dst,XmmRegister src)913 void X86_64Assembler::paddb(XmmRegister dst, XmmRegister src) {
914   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
915   EmitUint8(0x66);
916   EmitOptionalRex32(dst, src);
917   EmitUint8(0x0F);
918   EmitUint8(0xFC);
919   EmitXmmRegisterOperand(dst.LowBits(), src);
920 }
921 
922 
psubb(XmmRegister dst,XmmRegister src)923 void X86_64Assembler::psubb(XmmRegister dst, XmmRegister src) {
924   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
925   EmitUint8(0x66);
926   EmitOptionalRex32(dst, src);
927   EmitUint8(0x0F);
928   EmitUint8(0xF8);
929   EmitXmmRegisterOperand(dst.LowBits(), src);
930 }
931 
932 
paddw(XmmRegister dst,XmmRegister src)933 void X86_64Assembler::paddw(XmmRegister dst, XmmRegister src) {
934   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
935   EmitUint8(0x66);
936   EmitOptionalRex32(dst, src);
937   EmitUint8(0x0F);
938   EmitUint8(0xFD);
939   EmitXmmRegisterOperand(dst.LowBits(), src);
940 }
941 
942 
psubw(XmmRegister dst,XmmRegister src)943 void X86_64Assembler::psubw(XmmRegister dst, XmmRegister src) {
944   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
945   EmitUint8(0x66);
946   EmitOptionalRex32(dst, src);
947   EmitUint8(0x0F);
948   EmitUint8(0xF9);
949   EmitXmmRegisterOperand(dst.LowBits(), src);
950 }
951 
952 
pmullw(XmmRegister dst,XmmRegister src)953 void X86_64Assembler::pmullw(XmmRegister dst, XmmRegister src) {
954   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
955   EmitUint8(0x66);
956   EmitOptionalRex32(dst, src);
957   EmitUint8(0x0F);
958   EmitUint8(0xD5);
959   EmitXmmRegisterOperand(dst.LowBits(), src);
960 }
961 
962 
paddd(XmmRegister dst,XmmRegister src)963 void X86_64Assembler::paddd(XmmRegister dst, XmmRegister src) {
964   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
965   EmitUint8(0x66);
966   EmitOptionalRex32(dst, src);
967   EmitUint8(0x0F);
968   EmitUint8(0xFE);
969   EmitXmmRegisterOperand(dst.LowBits(), src);
970 }
971 
972 
psubd(XmmRegister dst,XmmRegister src)973 void X86_64Assembler::psubd(XmmRegister dst, XmmRegister src) {
974   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
975   EmitUint8(0x66);
976   EmitOptionalRex32(dst, src);
977   EmitUint8(0x0F);
978   EmitUint8(0xFA);
979   EmitXmmRegisterOperand(dst.LowBits(), src);
980 }
981 
982 
pmulld(XmmRegister dst,XmmRegister src)983 void X86_64Assembler::pmulld(XmmRegister dst, XmmRegister src) {
984   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
985   EmitUint8(0x66);
986   EmitOptionalRex32(dst, src);
987   EmitUint8(0x0F);
988   EmitUint8(0x38);
989   EmitUint8(0x40);
990   EmitXmmRegisterOperand(dst.LowBits(), src);
991 }
992 
993 
paddq(XmmRegister dst,XmmRegister src)994 void X86_64Assembler::paddq(XmmRegister dst, XmmRegister src) {
995   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
996   EmitUint8(0x66);
997   EmitOptionalRex32(dst, src);
998   EmitUint8(0x0F);
999   EmitUint8(0xD4);
1000   EmitXmmRegisterOperand(dst.LowBits(), src);
1001 }
1002 
1003 
psubq(XmmRegister dst,XmmRegister src)1004 void X86_64Assembler::psubq(XmmRegister dst, XmmRegister src) {
1005   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1006   EmitUint8(0x66);
1007   EmitOptionalRex32(dst, src);
1008   EmitUint8(0x0F);
1009   EmitUint8(0xFB);
1010   EmitXmmRegisterOperand(dst.LowBits(), src);
1011 }
1012 
1013 
cvtsi2ss(XmmRegister dst,CpuRegister src)1014 void X86_64Assembler::cvtsi2ss(XmmRegister dst, CpuRegister src) {
1015   cvtsi2ss(dst, src, false);
1016 }
1017 
1018 
cvtsi2ss(XmmRegister dst,CpuRegister src,bool is64bit)1019 void X86_64Assembler::cvtsi2ss(XmmRegister dst, CpuRegister src, bool is64bit) {
1020   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1021   EmitUint8(0xF3);
1022   if (is64bit) {
1023     // Emit a REX.W prefix if the operand size is 64 bits.
1024     EmitRex64(dst, src);
1025   } else {
1026     EmitOptionalRex32(dst, src);
1027   }
1028   EmitUint8(0x0F);
1029   EmitUint8(0x2A);
1030   EmitOperand(dst.LowBits(), Operand(src));
1031 }
1032 
1033 
cvtsi2ss(XmmRegister dst,const Address & src,bool is64bit)1034 void X86_64Assembler::cvtsi2ss(XmmRegister dst, const Address& src, bool is64bit) {
1035   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1036   EmitUint8(0xF3);
1037   if (is64bit) {
1038     // Emit a REX.W prefix if the operand size is 64 bits.
1039     EmitRex64(dst, src);
1040   } else {
1041     EmitOptionalRex32(dst, src);
1042   }
1043   EmitUint8(0x0F);
1044   EmitUint8(0x2A);
1045   EmitOperand(dst.LowBits(), src);
1046 }
1047 
1048 
cvtsi2sd(XmmRegister dst,CpuRegister src)1049 void X86_64Assembler::cvtsi2sd(XmmRegister dst, CpuRegister src) {
1050   cvtsi2sd(dst, src, false);
1051 }
1052 
1053 
cvtsi2sd(XmmRegister dst,CpuRegister src,bool is64bit)1054 void X86_64Assembler::cvtsi2sd(XmmRegister dst, CpuRegister src, bool is64bit) {
1055   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1056   EmitUint8(0xF2);
1057   if (is64bit) {
1058     // Emit a REX.W prefix if the operand size is 64 bits.
1059     EmitRex64(dst, src);
1060   } else {
1061     EmitOptionalRex32(dst, src);
1062   }
1063   EmitUint8(0x0F);
1064   EmitUint8(0x2A);
1065   EmitOperand(dst.LowBits(), Operand(src));
1066 }
1067 
1068 
cvtsi2sd(XmmRegister dst,const Address & src,bool is64bit)1069 void X86_64Assembler::cvtsi2sd(XmmRegister dst, const Address& src, bool is64bit) {
1070   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1071   EmitUint8(0xF2);
1072   if (is64bit) {
1073     // Emit a REX.W prefix if the operand size is 64 bits.
1074     EmitRex64(dst, src);
1075   } else {
1076     EmitOptionalRex32(dst, src);
1077   }
1078   EmitUint8(0x0F);
1079   EmitUint8(0x2A);
1080   EmitOperand(dst.LowBits(), src);
1081 }
1082 
1083 
cvtss2si(CpuRegister dst,XmmRegister src)1084 void X86_64Assembler::cvtss2si(CpuRegister dst, XmmRegister src) {
1085   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1086   EmitUint8(0xF3);
1087   EmitOptionalRex32(dst, src);
1088   EmitUint8(0x0F);
1089   EmitUint8(0x2D);
1090   EmitXmmRegisterOperand(dst.LowBits(), src);
1091 }
1092 
1093 
cvtss2sd(XmmRegister dst,XmmRegister src)1094 void X86_64Assembler::cvtss2sd(XmmRegister dst, XmmRegister src) {
1095   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1096   EmitUint8(0xF3);
1097   EmitOptionalRex32(dst, src);
1098   EmitUint8(0x0F);
1099   EmitUint8(0x5A);
1100   EmitXmmRegisterOperand(dst.LowBits(), src);
1101 }
1102 
1103 
cvtss2sd(XmmRegister dst,const Address & src)1104 void X86_64Assembler::cvtss2sd(XmmRegister dst, const Address& src) {
1105   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1106   EmitUint8(0xF3);
1107   EmitOptionalRex32(dst, src);
1108   EmitUint8(0x0F);
1109   EmitUint8(0x5A);
1110   EmitOperand(dst.LowBits(), src);
1111 }
1112 
1113 
cvtsd2si(CpuRegister dst,XmmRegister src)1114 void X86_64Assembler::cvtsd2si(CpuRegister dst, XmmRegister src) {
1115   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1116   EmitUint8(0xF2);
1117   EmitOptionalRex32(dst, src);
1118   EmitUint8(0x0F);
1119   EmitUint8(0x2D);
1120   EmitXmmRegisterOperand(dst.LowBits(), src);
1121 }
1122 
1123 
cvttss2si(CpuRegister dst,XmmRegister src)1124 void X86_64Assembler::cvttss2si(CpuRegister dst, XmmRegister src) {
1125   cvttss2si(dst, src, false);
1126 }
1127 
1128 
cvttss2si(CpuRegister dst,XmmRegister src,bool is64bit)1129 void X86_64Assembler::cvttss2si(CpuRegister dst, XmmRegister src, bool is64bit) {
1130   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1131   EmitUint8(0xF3);
1132   if (is64bit) {
1133     // Emit a REX.W prefix if the operand size is 64 bits.
1134     EmitRex64(dst, src);
1135   } else {
1136     EmitOptionalRex32(dst, src);
1137   }
1138   EmitUint8(0x0F);
1139   EmitUint8(0x2C);
1140   EmitXmmRegisterOperand(dst.LowBits(), src);
1141 }
1142 
1143 
cvttsd2si(CpuRegister dst,XmmRegister src)1144 void X86_64Assembler::cvttsd2si(CpuRegister dst, XmmRegister src) {
1145   cvttsd2si(dst, src, false);
1146 }
1147 
1148 
cvttsd2si(CpuRegister dst,XmmRegister src,bool is64bit)1149 void X86_64Assembler::cvttsd2si(CpuRegister dst, XmmRegister src, bool is64bit) {
1150   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1151   EmitUint8(0xF2);
1152   if (is64bit) {
1153     // Emit a REX.W prefix if the operand size is 64 bits.
1154     EmitRex64(dst, src);
1155   } else {
1156     EmitOptionalRex32(dst, src);
1157   }
1158   EmitUint8(0x0F);
1159   EmitUint8(0x2C);
1160   EmitXmmRegisterOperand(dst.LowBits(), src);
1161 }
1162 
1163 
cvtsd2ss(XmmRegister dst,XmmRegister src)1164 void X86_64Assembler::cvtsd2ss(XmmRegister dst, XmmRegister src) {
1165   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1166   EmitUint8(0xF2);
1167   EmitOptionalRex32(dst, src);
1168   EmitUint8(0x0F);
1169   EmitUint8(0x5A);
1170   EmitXmmRegisterOperand(dst.LowBits(), src);
1171 }
1172 
1173 
cvtsd2ss(XmmRegister dst,const Address & src)1174 void X86_64Assembler::cvtsd2ss(XmmRegister dst, const Address& src) {
1175   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1176   EmitUint8(0xF2);
1177   EmitOptionalRex32(dst, src);
1178   EmitUint8(0x0F);
1179   EmitUint8(0x5A);
1180   EmitOperand(dst.LowBits(), src);
1181 }
1182 
1183 
cvtdq2ps(XmmRegister dst,XmmRegister src)1184 void X86_64Assembler::cvtdq2ps(XmmRegister dst, XmmRegister src) {
1185   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1186   EmitOptionalRex32(dst, src);
1187   EmitUint8(0x0F);
1188   EmitUint8(0x5B);
1189   EmitXmmRegisterOperand(dst.LowBits(), src);
1190 }
1191 
1192 
cvtdq2pd(XmmRegister dst,XmmRegister src)1193 void X86_64Assembler::cvtdq2pd(XmmRegister dst, XmmRegister src) {
1194   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1195   EmitUint8(0xF3);
1196   EmitOptionalRex32(dst, src);
1197   EmitUint8(0x0F);
1198   EmitUint8(0xE6);
1199   EmitXmmRegisterOperand(dst.LowBits(), src);
1200 }
1201 
1202 
comiss(XmmRegister a,XmmRegister b)1203 void X86_64Assembler::comiss(XmmRegister a, XmmRegister b) {
1204   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1205   EmitOptionalRex32(a, b);
1206   EmitUint8(0x0F);
1207   EmitUint8(0x2F);
1208   EmitXmmRegisterOperand(a.LowBits(), b);
1209 }
1210 
1211 
comiss(XmmRegister a,const Address & b)1212 void X86_64Assembler::comiss(XmmRegister a, const Address& b) {
1213   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1214   EmitOptionalRex32(a, b);
1215   EmitUint8(0x0F);
1216   EmitUint8(0x2F);
1217   EmitOperand(a.LowBits(), b);
1218 }
1219 
1220 
comisd(XmmRegister a,XmmRegister b)1221 void X86_64Assembler::comisd(XmmRegister a, XmmRegister b) {
1222   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1223   EmitUint8(0x66);
1224   EmitOptionalRex32(a, b);
1225   EmitUint8(0x0F);
1226   EmitUint8(0x2F);
1227   EmitXmmRegisterOperand(a.LowBits(), b);
1228 }
1229 
1230 
comisd(XmmRegister a,const Address & b)1231 void X86_64Assembler::comisd(XmmRegister a, const Address& b) {
1232   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1233   EmitUint8(0x66);
1234   EmitOptionalRex32(a, b);
1235   EmitUint8(0x0F);
1236   EmitUint8(0x2F);
1237   EmitOperand(a.LowBits(), b);
1238 }
1239 
1240 
ucomiss(XmmRegister a,XmmRegister b)1241 void X86_64Assembler::ucomiss(XmmRegister a, XmmRegister b) {
1242   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1243   EmitOptionalRex32(a, b);
1244   EmitUint8(0x0F);
1245   EmitUint8(0x2E);
1246   EmitXmmRegisterOperand(a.LowBits(), b);
1247 }
1248 
1249 
ucomiss(XmmRegister a,const Address & b)1250 void X86_64Assembler::ucomiss(XmmRegister a, const Address& b) {
1251   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1252   EmitOptionalRex32(a, b);
1253   EmitUint8(0x0F);
1254   EmitUint8(0x2E);
1255   EmitOperand(a.LowBits(), b);
1256 }
1257 
1258 
ucomisd(XmmRegister a,XmmRegister b)1259 void X86_64Assembler::ucomisd(XmmRegister a, XmmRegister b) {
1260   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1261   EmitUint8(0x66);
1262   EmitOptionalRex32(a, b);
1263   EmitUint8(0x0F);
1264   EmitUint8(0x2E);
1265   EmitXmmRegisterOperand(a.LowBits(), b);
1266 }
1267 
1268 
ucomisd(XmmRegister a,const Address & b)1269 void X86_64Assembler::ucomisd(XmmRegister a, const Address& b) {
1270   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1271   EmitUint8(0x66);
1272   EmitOptionalRex32(a, b);
1273   EmitUint8(0x0F);
1274   EmitUint8(0x2E);
1275   EmitOperand(a.LowBits(), b);
1276 }
1277 
1278 
roundsd(XmmRegister dst,XmmRegister src,const Immediate & imm)1279 void X86_64Assembler::roundsd(XmmRegister dst, XmmRegister src, const Immediate& imm) {
1280   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1281   EmitUint8(0x66);
1282   EmitOptionalRex32(dst, src);
1283   EmitUint8(0x0F);
1284   EmitUint8(0x3A);
1285   EmitUint8(0x0B);
1286   EmitXmmRegisterOperand(dst.LowBits(), src);
1287   EmitUint8(imm.value());
1288 }
1289 
1290 
roundss(XmmRegister dst,XmmRegister src,const Immediate & imm)1291 void X86_64Assembler::roundss(XmmRegister dst, XmmRegister src, const Immediate& imm) {
1292   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1293   EmitUint8(0x66);
1294   EmitOptionalRex32(dst, src);
1295   EmitUint8(0x0F);
1296   EmitUint8(0x3A);
1297   EmitUint8(0x0A);
1298   EmitXmmRegisterOperand(dst.LowBits(), src);
1299   EmitUint8(imm.value());
1300 }
1301 
1302 
sqrtsd(XmmRegister dst,XmmRegister src)1303 void X86_64Assembler::sqrtsd(XmmRegister dst, XmmRegister src) {
1304   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1305   EmitUint8(0xF2);
1306   EmitOptionalRex32(dst, src);
1307   EmitUint8(0x0F);
1308   EmitUint8(0x51);
1309   EmitXmmRegisterOperand(dst.LowBits(), src);
1310 }
1311 
1312 
sqrtss(XmmRegister dst,XmmRegister src)1313 void X86_64Assembler::sqrtss(XmmRegister dst, XmmRegister src) {
1314   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1315   EmitUint8(0xF3);
1316   EmitOptionalRex32(dst, src);
1317   EmitUint8(0x0F);
1318   EmitUint8(0x51);
1319   EmitXmmRegisterOperand(dst.LowBits(), src);
1320 }
1321 
1322 
xorpd(XmmRegister dst,const Address & src)1323 void X86_64Assembler::xorpd(XmmRegister dst, const Address& src) {
1324   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1325   EmitUint8(0x66);
1326   EmitOptionalRex32(dst, src);
1327   EmitUint8(0x0F);
1328   EmitUint8(0x57);
1329   EmitOperand(dst.LowBits(), src);
1330 }
1331 
1332 
xorpd(XmmRegister dst,XmmRegister src)1333 void X86_64Assembler::xorpd(XmmRegister dst, XmmRegister src) {
1334   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1335   EmitUint8(0x66);
1336   EmitOptionalRex32(dst, src);
1337   EmitUint8(0x0F);
1338   EmitUint8(0x57);
1339   EmitXmmRegisterOperand(dst.LowBits(), src);
1340 }
1341 
1342 
xorps(XmmRegister dst,const Address & src)1343 void X86_64Assembler::xorps(XmmRegister dst, const Address& src) {
1344   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1345   EmitOptionalRex32(dst, src);
1346   EmitUint8(0x0F);
1347   EmitUint8(0x57);
1348   EmitOperand(dst.LowBits(), src);
1349 }
1350 
1351 
xorps(XmmRegister dst,XmmRegister src)1352 void X86_64Assembler::xorps(XmmRegister dst, XmmRegister src) {
1353   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1354   EmitOptionalRex32(dst, src);
1355   EmitUint8(0x0F);
1356   EmitUint8(0x57);
1357   EmitXmmRegisterOperand(dst.LowBits(), src);
1358 }
1359 
1360 
pxor(XmmRegister dst,XmmRegister src)1361 void X86_64Assembler::pxor(XmmRegister dst, XmmRegister src) {
1362   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1363   EmitUint8(0x66);
1364   EmitOptionalRex32(dst, src);
1365   EmitUint8(0x0F);
1366   EmitUint8(0xEF);
1367   EmitXmmRegisterOperand(dst.LowBits(), src);
1368 }
1369 
1370 
andpd(XmmRegister dst,const Address & src)1371 void X86_64Assembler::andpd(XmmRegister dst, const Address& src) {
1372   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1373   EmitUint8(0x66);
1374   EmitOptionalRex32(dst, src);
1375   EmitUint8(0x0F);
1376   EmitUint8(0x54);
1377   EmitOperand(dst.LowBits(), src);
1378 }
1379 
andpd(XmmRegister dst,XmmRegister src)1380 void X86_64Assembler::andpd(XmmRegister dst, XmmRegister src) {
1381   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1382   EmitUint8(0x66);
1383   EmitOptionalRex32(dst, src);
1384   EmitUint8(0x0F);
1385   EmitUint8(0x54);
1386   EmitXmmRegisterOperand(dst.LowBits(), src);
1387 }
1388 
andps(XmmRegister dst,XmmRegister src)1389 void X86_64Assembler::andps(XmmRegister dst, XmmRegister src) {
1390   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1391   EmitOptionalRex32(dst, src);
1392   EmitUint8(0x0F);
1393   EmitUint8(0x54);
1394   EmitXmmRegisterOperand(dst.LowBits(), src);
1395 }
1396 
pand(XmmRegister dst,XmmRegister src)1397 void X86_64Assembler::pand(XmmRegister dst, XmmRegister src) {
1398   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1399   EmitUint8(0x66);
1400   EmitOptionalRex32(dst, src);
1401   EmitUint8(0x0F);
1402   EmitUint8(0xDB);
1403   EmitXmmRegisterOperand(dst.LowBits(), src);
1404 }
1405 
andnpd(XmmRegister dst,XmmRegister src)1406 void X86_64Assembler::andnpd(XmmRegister dst, XmmRegister src) {
1407   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1408   EmitUint8(0x66);
1409   EmitOptionalRex32(dst, src);
1410   EmitUint8(0x0F);
1411   EmitUint8(0x55);
1412   EmitXmmRegisterOperand(dst.LowBits(), src);
1413 }
1414 
andnps(XmmRegister dst,XmmRegister src)1415 void X86_64Assembler::andnps(XmmRegister dst, XmmRegister src) {
1416   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1417   EmitOptionalRex32(dst, src);
1418   EmitUint8(0x0F);
1419   EmitUint8(0x55);
1420   EmitXmmRegisterOperand(dst.LowBits(), src);
1421 }
1422 
pandn(XmmRegister dst,XmmRegister src)1423 void X86_64Assembler::pandn(XmmRegister dst, XmmRegister src) {
1424   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1425   EmitUint8(0x66);
1426   EmitOptionalRex32(dst, src);
1427   EmitUint8(0x0F);
1428   EmitUint8(0xDF);
1429   EmitXmmRegisterOperand(dst.LowBits(), src);
1430 }
1431 
orpd(XmmRegister dst,XmmRegister src)1432 void X86_64Assembler::orpd(XmmRegister dst, XmmRegister src) {
1433   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1434   EmitUint8(0x66);
1435   EmitOptionalRex32(dst, src);
1436   EmitUint8(0x0F);
1437   EmitUint8(0x56);
1438   EmitXmmRegisterOperand(dst.LowBits(), src);
1439 }
1440 
orps(XmmRegister dst,XmmRegister src)1441 void X86_64Assembler::orps(XmmRegister dst, XmmRegister src) {
1442   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1443   EmitOptionalRex32(dst, src);
1444   EmitUint8(0x0F);
1445   EmitUint8(0x56);
1446   EmitXmmRegisterOperand(dst.LowBits(), src);
1447 }
1448 
por(XmmRegister dst,XmmRegister src)1449 void X86_64Assembler::por(XmmRegister dst, XmmRegister src) {
1450   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1451   EmitUint8(0x66);
1452   EmitOptionalRex32(dst, src);
1453   EmitUint8(0x0F);
1454   EmitUint8(0xEB);
1455   EmitXmmRegisterOperand(dst.LowBits(), src);
1456 }
1457 
pavgb(XmmRegister dst,XmmRegister src)1458 void X86_64Assembler::pavgb(XmmRegister dst, XmmRegister src) {
1459   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1460   EmitUint8(0x66);
1461   EmitOptionalRex32(dst, src);
1462   EmitUint8(0x0F);
1463   EmitUint8(0xE0);
1464   EmitXmmRegisterOperand(dst.LowBits(), src);
1465 }
1466 
pavgw(XmmRegister dst,XmmRegister src)1467 void X86_64Assembler::pavgw(XmmRegister dst, XmmRegister src) {
1468   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1469   EmitUint8(0x66);
1470   EmitOptionalRex32(dst, src);
1471   EmitUint8(0x0F);
1472   EmitUint8(0xE3);
1473   EmitXmmRegisterOperand(dst.LowBits(), src);
1474 }
1475 
psadbw(XmmRegister dst,XmmRegister src)1476 void X86_64Assembler::psadbw(XmmRegister dst, XmmRegister src) {
1477   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1478   EmitUint8(0x66);
1479   EmitOptionalRex32(dst, src);
1480   EmitUint8(0x0F);
1481   EmitUint8(0xF6);
1482   EmitXmmRegisterOperand(dst.LowBits(), src);
1483 }
1484 
pmaddwd(XmmRegister dst,XmmRegister src)1485 void X86_64Assembler::pmaddwd(XmmRegister dst, XmmRegister src) {
1486   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1487   EmitUint8(0x66);
1488   EmitOptionalRex32(dst, src);
1489   EmitUint8(0x0F);
1490   EmitUint8(0xF5);
1491   EmitXmmRegisterOperand(dst.LowBits(), src);
1492 }
1493 
phaddw(XmmRegister dst,XmmRegister src)1494 void X86_64Assembler::phaddw(XmmRegister dst, XmmRegister src) {
1495   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1496   EmitUint8(0x66);
1497   EmitOptionalRex32(dst, src);
1498   EmitUint8(0x0F);
1499   EmitUint8(0x38);
1500   EmitUint8(0x01);
1501   EmitXmmRegisterOperand(dst.LowBits(), src);
1502 }
1503 
phaddd(XmmRegister dst,XmmRegister src)1504 void X86_64Assembler::phaddd(XmmRegister dst, XmmRegister src) {
1505   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1506   EmitUint8(0x66);
1507   EmitOptionalRex32(dst, src);
1508   EmitUint8(0x0F);
1509   EmitUint8(0x38);
1510   EmitUint8(0x02);
1511   EmitXmmRegisterOperand(dst.LowBits(), src);
1512 }
1513 
haddps(XmmRegister dst,XmmRegister src)1514 void X86_64Assembler::haddps(XmmRegister dst, XmmRegister src) {
1515   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1516   EmitUint8(0xF2);
1517   EmitOptionalRex32(dst, src);
1518   EmitUint8(0x0F);
1519   EmitUint8(0x7C);
1520   EmitXmmRegisterOperand(dst.LowBits(), src);
1521 }
1522 
haddpd(XmmRegister dst,XmmRegister src)1523 void X86_64Assembler::haddpd(XmmRegister dst, XmmRegister src) {
1524   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1525   EmitUint8(0x66);
1526   EmitOptionalRex32(dst, src);
1527   EmitUint8(0x0F);
1528   EmitUint8(0x7C);
1529   EmitXmmRegisterOperand(dst.LowBits(), src);
1530 }
1531 
phsubw(XmmRegister dst,XmmRegister src)1532 void X86_64Assembler::phsubw(XmmRegister dst, XmmRegister src) {
1533   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1534   EmitUint8(0x66);
1535   EmitOptionalRex32(dst, src);
1536   EmitUint8(0x0F);
1537   EmitUint8(0x38);
1538   EmitUint8(0x05);
1539   EmitXmmRegisterOperand(dst.LowBits(), src);
1540 }
1541 
phsubd(XmmRegister dst,XmmRegister src)1542 void X86_64Assembler::phsubd(XmmRegister dst, XmmRegister src) {
1543   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1544   EmitUint8(0x66);
1545   EmitOptionalRex32(dst, src);
1546   EmitUint8(0x0F);
1547   EmitUint8(0x38);
1548   EmitUint8(0x06);
1549   EmitXmmRegisterOperand(dst.LowBits(), src);
1550 }
1551 
hsubps(XmmRegister dst,XmmRegister src)1552 void X86_64Assembler::hsubps(XmmRegister dst, XmmRegister src) {
1553   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1554   EmitUint8(0xF2);
1555   EmitOptionalRex32(dst, src);
1556   EmitUint8(0x0F);
1557   EmitUint8(0x7D);
1558   EmitXmmRegisterOperand(dst.LowBits(), src);
1559 }
1560 
hsubpd(XmmRegister dst,XmmRegister src)1561 void X86_64Assembler::hsubpd(XmmRegister dst, XmmRegister src) {
1562   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1563   EmitUint8(0x66);
1564   EmitOptionalRex32(dst, src);
1565   EmitUint8(0x0F);
1566   EmitUint8(0x7D);
1567   EmitXmmRegisterOperand(dst.LowBits(), src);
1568 }
1569 
pminsb(XmmRegister dst,XmmRegister src)1570 void X86_64Assembler::pminsb(XmmRegister dst, XmmRegister src) {
1571   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1572   EmitUint8(0x66);
1573   EmitOptionalRex32(dst, src);
1574   EmitUint8(0x0F);
1575   EmitUint8(0x38);
1576   EmitUint8(0x38);
1577   EmitXmmRegisterOperand(dst.LowBits(), src);
1578 }
1579 
pmaxsb(XmmRegister dst,XmmRegister src)1580 void X86_64Assembler::pmaxsb(XmmRegister dst, XmmRegister src) {
1581   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1582   EmitUint8(0x66);
1583   EmitOptionalRex32(dst, src);
1584   EmitUint8(0x0F);
1585   EmitUint8(0x38);
1586   EmitUint8(0x3C);
1587   EmitXmmRegisterOperand(dst.LowBits(), src);
1588 }
1589 
pminsw(XmmRegister dst,XmmRegister src)1590 void X86_64Assembler::pminsw(XmmRegister dst, XmmRegister src) {
1591   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1592   EmitUint8(0x66);
1593   EmitOptionalRex32(dst, src);
1594   EmitUint8(0x0F);
1595   EmitUint8(0xEA);
1596   EmitXmmRegisterOperand(dst.LowBits(), src);
1597 }
1598 
pmaxsw(XmmRegister dst,XmmRegister src)1599 void X86_64Assembler::pmaxsw(XmmRegister dst, XmmRegister src) {
1600   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1601   EmitUint8(0x66);
1602   EmitOptionalRex32(dst, src);
1603   EmitUint8(0x0F);
1604   EmitUint8(0xEE);
1605   EmitXmmRegisterOperand(dst.LowBits(), src);
1606 }
1607 
pminsd(XmmRegister dst,XmmRegister src)1608 void X86_64Assembler::pminsd(XmmRegister dst, XmmRegister src) {
1609   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1610   EmitUint8(0x66);
1611   EmitOptionalRex32(dst, src);
1612   EmitUint8(0x0F);
1613   EmitUint8(0x38);
1614   EmitUint8(0x39);
1615   EmitXmmRegisterOperand(dst.LowBits(), src);
1616 }
1617 
pmaxsd(XmmRegister dst,XmmRegister src)1618 void X86_64Assembler::pmaxsd(XmmRegister dst, XmmRegister src) {
1619   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1620   EmitUint8(0x66);
1621   EmitOptionalRex32(dst, src);
1622   EmitUint8(0x0F);
1623   EmitUint8(0x38);
1624   EmitUint8(0x3D);
1625   EmitXmmRegisterOperand(dst.LowBits(), src);
1626 }
1627 
pminub(XmmRegister dst,XmmRegister src)1628 void X86_64Assembler::pminub(XmmRegister dst, XmmRegister src) {
1629   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1630   EmitUint8(0x66);
1631   EmitOptionalRex32(dst, src);
1632   EmitUint8(0x0F);
1633   EmitUint8(0xDA);
1634   EmitXmmRegisterOperand(dst.LowBits(), src);
1635 }
1636 
pmaxub(XmmRegister dst,XmmRegister src)1637 void X86_64Assembler::pmaxub(XmmRegister dst, XmmRegister src) {
1638   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1639   EmitUint8(0x66);
1640   EmitOptionalRex32(dst, src);
1641   EmitUint8(0x0F);
1642   EmitUint8(0xDE);
1643   EmitXmmRegisterOperand(dst.LowBits(), src);
1644 }
1645 
pminuw(XmmRegister dst,XmmRegister src)1646 void X86_64Assembler::pminuw(XmmRegister dst, XmmRegister src) {
1647   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1648   EmitUint8(0x66);
1649   EmitOptionalRex32(dst, src);
1650   EmitUint8(0x0F);
1651   EmitUint8(0x38);
1652   EmitUint8(0x3A);
1653   EmitXmmRegisterOperand(dst.LowBits(), src);
1654 }
1655 
pmaxuw(XmmRegister dst,XmmRegister src)1656 void X86_64Assembler::pmaxuw(XmmRegister dst, XmmRegister src) {
1657   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1658   EmitUint8(0x66);
1659   EmitOptionalRex32(dst, src);
1660   EmitUint8(0x0F);
1661   EmitUint8(0x38);
1662   EmitUint8(0x3E);
1663   EmitXmmRegisterOperand(dst.LowBits(), src);
1664 }
1665 
pminud(XmmRegister dst,XmmRegister src)1666 void X86_64Assembler::pminud(XmmRegister dst, XmmRegister src) {
1667   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1668   EmitUint8(0x66);
1669   EmitOptionalRex32(dst, src);
1670   EmitUint8(0x0F);
1671   EmitUint8(0x38);
1672   EmitUint8(0x3B);
1673   EmitXmmRegisterOperand(dst.LowBits(), src);
1674 }
1675 
pmaxud(XmmRegister dst,XmmRegister src)1676 void X86_64Assembler::pmaxud(XmmRegister dst, XmmRegister src) {
1677   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1678   EmitUint8(0x66);
1679   EmitOptionalRex32(dst, src);
1680   EmitUint8(0x0F);
1681   EmitUint8(0x38);
1682   EmitUint8(0x3F);
1683   EmitXmmRegisterOperand(dst.LowBits(), src);
1684 }
1685 
minps(XmmRegister dst,XmmRegister src)1686 void X86_64Assembler::minps(XmmRegister dst, XmmRegister src) {
1687   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1688   EmitOptionalRex32(dst, src);
1689   EmitUint8(0x0F);
1690   EmitUint8(0x5D);
1691   EmitXmmRegisterOperand(dst.LowBits(), src);
1692 }
1693 
maxps(XmmRegister dst,XmmRegister src)1694 void X86_64Assembler::maxps(XmmRegister dst, XmmRegister src) {
1695   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1696   EmitOptionalRex32(dst, src);
1697   EmitUint8(0x0F);
1698   EmitUint8(0x5F);
1699   EmitXmmRegisterOperand(dst.LowBits(), src);
1700 }
1701 
minpd(XmmRegister dst,XmmRegister src)1702 void X86_64Assembler::minpd(XmmRegister dst, XmmRegister src) {
1703   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1704   EmitUint8(0x66);
1705   EmitOptionalRex32(dst, src);
1706   EmitUint8(0x0F);
1707   EmitUint8(0x5D);
1708   EmitXmmRegisterOperand(dst.LowBits(), src);
1709 }
1710 
maxpd(XmmRegister dst,XmmRegister src)1711 void X86_64Assembler::maxpd(XmmRegister dst, XmmRegister src) {
1712   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1713   EmitUint8(0x66);
1714   EmitOptionalRex32(dst, src);
1715   EmitUint8(0x0F);
1716   EmitUint8(0x5F);
1717   EmitXmmRegisterOperand(dst.LowBits(), src);
1718 }
1719 
pcmpeqb(XmmRegister dst,XmmRegister src)1720 void X86_64Assembler::pcmpeqb(XmmRegister dst, XmmRegister src) {
1721   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1722   EmitUint8(0x66);
1723   EmitOptionalRex32(dst, src);
1724   EmitUint8(0x0F);
1725   EmitUint8(0x74);
1726   EmitXmmRegisterOperand(dst.LowBits(), src);
1727 }
1728 
pcmpeqw(XmmRegister dst,XmmRegister src)1729 void X86_64Assembler::pcmpeqw(XmmRegister dst, XmmRegister src) {
1730   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1731   EmitUint8(0x66);
1732   EmitOptionalRex32(dst, src);
1733   EmitUint8(0x0F);
1734   EmitUint8(0x75);
1735   EmitXmmRegisterOperand(dst.LowBits(), src);
1736 }
1737 
pcmpeqd(XmmRegister dst,XmmRegister src)1738 void X86_64Assembler::pcmpeqd(XmmRegister dst, XmmRegister src) {
1739   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1740   EmitUint8(0x66);
1741   EmitOptionalRex32(dst, src);
1742   EmitUint8(0x0F);
1743   EmitUint8(0x76);
1744   EmitXmmRegisterOperand(dst.LowBits(), src);
1745 }
1746 
pcmpeqq(XmmRegister dst,XmmRegister src)1747 void X86_64Assembler::pcmpeqq(XmmRegister dst, XmmRegister src) {
1748   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1749   EmitUint8(0x66);
1750   EmitOptionalRex32(dst, src);
1751   EmitUint8(0x0F);
1752   EmitUint8(0x38);
1753   EmitUint8(0x29);
1754   EmitXmmRegisterOperand(dst.LowBits(), src);
1755 }
1756 
pcmpgtb(XmmRegister dst,XmmRegister src)1757 void X86_64Assembler::pcmpgtb(XmmRegister dst, XmmRegister src) {
1758   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1759   EmitUint8(0x66);
1760   EmitOptionalRex32(dst, src);
1761   EmitUint8(0x0F);
1762   EmitUint8(0x64);
1763   EmitXmmRegisterOperand(dst.LowBits(), src);
1764 }
1765 
pcmpgtw(XmmRegister dst,XmmRegister src)1766 void X86_64Assembler::pcmpgtw(XmmRegister dst, XmmRegister src) {
1767   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1768   EmitUint8(0x66);
1769   EmitOptionalRex32(dst, src);
1770   EmitUint8(0x0F);
1771   EmitUint8(0x65);
1772   EmitXmmRegisterOperand(dst.LowBits(), src);
1773 }
1774 
pcmpgtd(XmmRegister dst,XmmRegister src)1775 void X86_64Assembler::pcmpgtd(XmmRegister dst, XmmRegister src) {
1776   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1777   EmitUint8(0x66);
1778   EmitOptionalRex32(dst, src);
1779   EmitUint8(0x0F);
1780   EmitUint8(0x66);
1781   EmitXmmRegisterOperand(dst.LowBits(), src);
1782 }
1783 
pcmpgtq(XmmRegister dst,XmmRegister src)1784 void X86_64Assembler::pcmpgtq(XmmRegister dst, XmmRegister src) {
1785   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1786   EmitUint8(0x66);
1787   EmitOptionalRex32(dst, src);
1788   EmitUint8(0x0F);
1789   EmitUint8(0x38);
1790   EmitUint8(0x37);
1791   EmitXmmRegisterOperand(dst.LowBits(), src);
1792 }
1793 
shufpd(XmmRegister dst,XmmRegister src,const Immediate & imm)1794 void X86_64Assembler::shufpd(XmmRegister dst, XmmRegister src, const Immediate& imm) {
1795   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1796   EmitUint8(0x66);
1797   EmitOptionalRex32(dst, src);
1798   EmitUint8(0x0F);
1799   EmitUint8(0xC6);
1800   EmitXmmRegisterOperand(dst.LowBits(), src);
1801   EmitUint8(imm.value());
1802 }
1803 
1804 
shufps(XmmRegister dst,XmmRegister src,const Immediate & imm)1805 void X86_64Assembler::shufps(XmmRegister dst, XmmRegister src, const Immediate& imm) {
1806   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1807   EmitOptionalRex32(dst, src);
1808   EmitUint8(0x0F);
1809   EmitUint8(0xC6);
1810   EmitXmmRegisterOperand(dst.LowBits(), src);
1811   EmitUint8(imm.value());
1812 }
1813 
1814 
pshufd(XmmRegister dst,XmmRegister src,const Immediate & imm)1815 void X86_64Assembler::pshufd(XmmRegister dst, XmmRegister src, const Immediate& imm) {
1816   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1817   EmitUint8(0x66);
1818   EmitOptionalRex32(dst, src);
1819   EmitUint8(0x0F);
1820   EmitUint8(0x70);
1821   EmitXmmRegisterOperand(dst.LowBits(), src);
1822   EmitUint8(imm.value());
1823 }
1824 
1825 
punpcklbw(XmmRegister dst,XmmRegister src)1826 void X86_64Assembler::punpcklbw(XmmRegister dst, XmmRegister src) {
1827   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1828   EmitUint8(0x66);
1829   EmitOptionalRex32(dst, src);
1830   EmitUint8(0x0F);
1831   EmitUint8(0x60);
1832   EmitXmmRegisterOperand(dst.LowBits(), src);
1833 }
1834 
1835 
punpcklwd(XmmRegister dst,XmmRegister src)1836 void X86_64Assembler::punpcklwd(XmmRegister dst, XmmRegister src) {
1837   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1838   EmitUint8(0x66);
1839   EmitOptionalRex32(dst, src);
1840   EmitUint8(0x0F);
1841   EmitUint8(0x61);
1842   EmitXmmRegisterOperand(dst.LowBits(), src);
1843 }
1844 
1845 
punpckldq(XmmRegister dst,XmmRegister src)1846 void X86_64Assembler::punpckldq(XmmRegister dst, XmmRegister src) {
1847   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1848   EmitUint8(0x66);
1849   EmitOptionalRex32(dst, src);
1850   EmitUint8(0x0F);
1851   EmitUint8(0x62);
1852   EmitXmmRegisterOperand(dst.LowBits(), src);
1853 }
1854 
1855 
punpcklqdq(XmmRegister dst,XmmRegister src)1856 void X86_64Assembler::punpcklqdq(XmmRegister dst, XmmRegister src) {
1857   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1858   EmitUint8(0x66);
1859   EmitOptionalRex32(dst, src);
1860   EmitUint8(0x0F);
1861   EmitUint8(0x6C);
1862   EmitXmmRegisterOperand(dst.LowBits(), src);
1863 }
1864 
1865 
punpckhbw(XmmRegister dst,XmmRegister src)1866 void X86_64Assembler::punpckhbw(XmmRegister dst, XmmRegister src) {
1867   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1868   EmitUint8(0x66);
1869   EmitOptionalRex32(dst, src);
1870   EmitUint8(0x0F);
1871   EmitUint8(0x68);
1872   EmitXmmRegisterOperand(dst.LowBits(), src);
1873 }
1874 
1875 
punpckhwd(XmmRegister dst,XmmRegister src)1876 void X86_64Assembler::punpckhwd(XmmRegister dst, XmmRegister src) {
1877   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1878   EmitUint8(0x66);
1879   EmitOptionalRex32(dst, src);
1880   EmitUint8(0x0F);
1881   EmitUint8(0x69);
1882   EmitXmmRegisterOperand(dst.LowBits(), src);
1883 }
1884 
1885 
punpckhdq(XmmRegister dst,XmmRegister src)1886 void X86_64Assembler::punpckhdq(XmmRegister dst, XmmRegister src) {
1887   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1888   EmitUint8(0x66);
1889   EmitOptionalRex32(dst, src);
1890   EmitUint8(0x0F);
1891   EmitUint8(0x6A);
1892   EmitXmmRegisterOperand(dst.LowBits(), src);
1893 }
1894 
1895 
punpckhqdq(XmmRegister dst,XmmRegister src)1896 void X86_64Assembler::punpckhqdq(XmmRegister dst, XmmRegister src) {
1897   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1898   EmitUint8(0x66);
1899   EmitOptionalRex32(dst, src);
1900   EmitUint8(0x0F);
1901   EmitUint8(0x6D);
1902   EmitXmmRegisterOperand(dst.LowBits(), src);
1903 }
1904 
1905 
psllw(XmmRegister reg,const Immediate & shift_count)1906 void X86_64Assembler::psllw(XmmRegister reg, const Immediate& shift_count) {
1907   DCHECK(shift_count.is_uint8());
1908   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1909   EmitUint8(0x66);
1910   EmitOptionalRex(false, false, false, false, reg.NeedsRex());
1911   EmitUint8(0x0F);
1912   EmitUint8(0x71);
1913   EmitXmmRegisterOperand(6, reg);
1914   EmitUint8(shift_count.value());
1915 }
1916 
1917 
pslld(XmmRegister reg,const Immediate & shift_count)1918 void X86_64Assembler::pslld(XmmRegister reg, const Immediate& shift_count) {
1919   DCHECK(shift_count.is_uint8());
1920   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1921   EmitUint8(0x66);
1922   EmitOptionalRex(false, false, false, false, reg.NeedsRex());
1923   EmitUint8(0x0F);
1924   EmitUint8(0x72);
1925   EmitXmmRegisterOperand(6, reg);
1926   EmitUint8(shift_count.value());
1927 }
1928 
1929 
psllq(XmmRegister reg,const Immediate & shift_count)1930 void X86_64Assembler::psllq(XmmRegister reg, const Immediate& shift_count) {
1931   DCHECK(shift_count.is_uint8());
1932   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1933   EmitUint8(0x66);
1934   EmitOptionalRex(false, false, false, false, reg.NeedsRex());
1935   EmitUint8(0x0F);
1936   EmitUint8(0x73);
1937   EmitXmmRegisterOperand(6, reg);
1938   EmitUint8(shift_count.value());
1939 }
1940 
1941 
psraw(XmmRegister reg,const Immediate & shift_count)1942 void X86_64Assembler::psraw(XmmRegister reg, const Immediate& shift_count) {
1943   DCHECK(shift_count.is_uint8());
1944   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1945   EmitUint8(0x66);
1946   EmitOptionalRex(false, false, false, false, reg.NeedsRex());
1947   EmitUint8(0x0F);
1948   EmitUint8(0x71);
1949   EmitXmmRegisterOperand(4, reg);
1950   EmitUint8(shift_count.value());
1951 }
1952 
1953 
psrad(XmmRegister reg,const Immediate & shift_count)1954 void X86_64Assembler::psrad(XmmRegister reg, const Immediate& shift_count) {
1955   DCHECK(shift_count.is_uint8());
1956   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1957   EmitUint8(0x66);
1958   EmitOptionalRex(false, false, false, false, reg.NeedsRex());
1959   EmitUint8(0x0F);
1960   EmitUint8(0x72);
1961   EmitXmmRegisterOperand(4, reg);
1962   EmitUint8(shift_count.value());
1963 }
1964 
1965 
psrlw(XmmRegister reg,const Immediate & shift_count)1966 void X86_64Assembler::psrlw(XmmRegister reg, const Immediate& shift_count) {
1967   DCHECK(shift_count.is_uint8());
1968   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1969   EmitUint8(0x66);
1970   EmitOptionalRex(false, false, false, false, reg.NeedsRex());
1971   EmitUint8(0x0F);
1972   EmitUint8(0x71);
1973   EmitXmmRegisterOperand(2, reg);
1974   EmitUint8(shift_count.value());
1975 }
1976 
1977 
psrld(XmmRegister reg,const Immediate & shift_count)1978 void X86_64Assembler::psrld(XmmRegister reg, const Immediate& shift_count) {
1979   DCHECK(shift_count.is_uint8());
1980   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1981   EmitUint8(0x66);
1982   EmitOptionalRex(false, false, false, false, reg.NeedsRex());
1983   EmitUint8(0x0F);
1984   EmitUint8(0x72);
1985   EmitXmmRegisterOperand(2, reg);
1986   EmitUint8(shift_count.value());
1987 }
1988 
1989 
psrlq(XmmRegister reg,const Immediate & shift_count)1990 void X86_64Assembler::psrlq(XmmRegister reg, const Immediate& shift_count) {
1991   DCHECK(shift_count.is_uint8());
1992   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1993   EmitUint8(0x66);
1994   EmitOptionalRex(false, false, false, false, reg.NeedsRex());
1995   EmitUint8(0x0F);
1996   EmitUint8(0x73);
1997   EmitXmmRegisterOperand(2, reg);
1998   EmitUint8(shift_count.value());
1999 }
2000 
2001 
psrldq(XmmRegister reg,const Immediate & shift_count)2002 void X86_64Assembler::psrldq(XmmRegister reg, const Immediate& shift_count) {
2003   DCHECK(shift_count.is_uint8());
2004   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
2005   EmitUint8(0x66);
2006   EmitOptionalRex(false, false, false, false, reg.NeedsRex());
2007   EmitUint8(0x0F);
2008   EmitUint8(0x73);
2009   EmitXmmRegisterOperand(3, reg);
2010   EmitUint8(shift_count.value());
2011 }
2012 
2013 
fldl(const Address & src)2014 void X86_64Assembler::fldl(const Address& src) {
2015   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
2016   EmitUint8(0xDD);
2017   EmitOperand(0, src);
2018 }
2019 
2020 
fstl(const Address & dst)2021 void X86_64Assembler::fstl(const Address& dst) {
2022   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
2023   EmitUint8(0xDD);
2024   EmitOperand(2, dst);
2025 }
2026 
2027 
fstpl(const Address & dst)2028 void X86_64Assembler::fstpl(const Address& dst) {
2029   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
2030   EmitUint8(0xDD);
2031   EmitOperand(3, dst);
2032 }
2033 
2034 
fstsw()2035 void X86_64Assembler::fstsw() {
2036   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
2037   EmitUint8(0x9B);
2038   EmitUint8(0xDF);
2039   EmitUint8(0xE0);
2040 }
2041 
2042 
fnstcw(const Address & dst)2043 void X86_64Assembler::fnstcw(const Address& dst) {
2044   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
2045   EmitUint8(0xD9);
2046   EmitOperand(7, dst);
2047 }
2048 
2049 
fldcw(const Address & src)2050 void X86_64Assembler::fldcw(const Address& src) {
2051   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
2052   EmitUint8(0xD9);
2053   EmitOperand(5, src);
2054 }
2055 
2056 
fistpl(const Address & dst)2057 void X86_64Assembler::fistpl(const Address& dst) {
2058   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
2059   EmitUint8(0xDF);
2060   EmitOperand(7, dst);
2061 }
2062 
2063 
fistps(const Address & dst)2064 void X86_64Assembler::fistps(const Address& dst) {
2065   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
2066   EmitUint8(0xDB);
2067   EmitOperand(3, dst);
2068 }
2069 
2070 
fildl(const Address & src)2071 void X86_64Assembler::fildl(const Address& src) {
2072   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
2073   EmitUint8(0xDF);
2074   EmitOperand(5, src);
2075 }
2076 
2077 
filds(const Address & src)2078 void X86_64Assembler::filds(const Address& src) {
2079   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
2080   EmitUint8(0xDB);
2081   EmitOperand(0, src);
2082 }
2083 
2084 
fincstp()2085 void X86_64Assembler::fincstp() {
2086   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
2087   EmitUint8(0xD9);
2088   EmitUint8(0xF7);
2089 }
2090 
2091 
ffree(const Immediate & index)2092 void X86_64Assembler::ffree(const Immediate& index) {
2093   CHECK_LT(index.value(), 7);
2094   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
2095   EmitUint8(0xDD);
2096   EmitUint8(0xC0 + index.value());
2097 }
2098 
2099 
fsin()2100 void X86_64Assembler::fsin() {
2101   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
2102   EmitUint8(0xD9);
2103   EmitUint8(0xFE);
2104 }
2105 
2106 
fcos()2107 void X86_64Assembler::fcos() {
2108   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
2109   EmitUint8(0xD9);
2110   EmitUint8(0xFF);
2111 }
2112 
2113 
fptan()2114 void X86_64Assembler::fptan() {
2115   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
2116   EmitUint8(0xD9);
2117   EmitUint8(0xF2);
2118 }
2119 
fucompp()2120 void X86_64Assembler::fucompp() {
2121   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
2122   EmitUint8(0xDA);
2123   EmitUint8(0xE9);
2124 }
2125 
2126 
fprem()2127 void X86_64Assembler::fprem() {
2128   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
2129   EmitUint8(0xD9);
2130   EmitUint8(0xF8);
2131 }
2132 
2133 
xchgl(CpuRegister dst,CpuRegister src)2134 void X86_64Assembler::xchgl(CpuRegister dst, CpuRegister src) {
2135   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
2136   // There is a short version for rax.
2137   // It's a bit awkward, as CpuRegister has a const field, so assignment and thus swapping doesn't
2138   // work.
2139   const bool src_rax = src.AsRegister() == RAX;
2140   const bool dst_rax = dst.AsRegister() == RAX;
2141   if (src_rax || dst_rax) {
2142     EmitOptionalRex32(src_rax ? dst : src);
2143     EmitUint8(0x90 + (src_rax ? dst.LowBits() : src.LowBits()));
2144     return;
2145   }
2146 
2147   // General case.
2148   EmitOptionalRex32(src, dst);
2149   EmitUint8(0x87);
2150   EmitRegisterOperand(src.LowBits(), dst.LowBits());
2151 }
2152 
2153 
xchgq(CpuRegister dst,CpuRegister src)2154 void X86_64Assembler::xchgq(CpuRegister dst, CpuRegister src) {
2155   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
2156   // There is a short version for rax.
2157   // It's a bit awkward, as CpuRegister has a const field, so assignment and thus swapping doesn't
2158   // work.
2159   const bool src_rax = src.AsRegister() == RAX;
2160   const bool dst_rax = dst.AsRegister() == RAX;
2161   if (src_rax || dst_rax) {
2162     // If src == target, emit a nop instead.
2163     if (src_rax && dst_rax) {
2164       EmitUint8(0x90);
2165     } else {
2166       EmitRex64(src_rax ? dst : src);
2167       EmitUint8(0x90 + (src_rax ? dst.LowBits() : src.LowBits()));
2168     }
2169     return;
2170   }
2171 
2172   // General case.
2173   EmitRex64(src, dst);
2174   EmitUint8(0x87);
2175   EmitRegisterOperand(src.LowBits(), dst.LowBits());
2176 }
2177 
2178 
xchgl(CpuRegister reg,const Address & address)2179 void X86_64Assembler::xchgl(CpuRegister reg, const Address& address) {
2180   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
2181   EmitOptionalRex32(reg, address);
2182   EmitUint8(0x87);
2183   EmitOperand(reg.LowBits(), address);
2184 }
2185 
2186 
cmpb(const Address & address,const Immediate & imm)2187 void X86_64Assembler::cmpb(const Address& address, const Immediate& imm) {
2188   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
2189   CHECK(imm.is_int32());
2190   EmitOptionalRex32(address);
2191   EmitUint8(0x80);
2192   EmitOperand(7, address);
2193   EmitUint8(imm.value() & 0xFF);
2194 }
2195 
2196 
cmpw(const Address & address,const Immediate & imm)2197 void X86_64Assembler::cmpw(const Address& address, const Immediate& imm) {
2198   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
2199   CHECK(imm.is_int32());
2200   EmitOperandSizeOverride();
2201   EmitOptionalRex32(address);
2202   EmitComplex(7, address, imm, /* is_16_op */ true);
2203 }
2204 
2205 
cmpl(CpuRegister reg,const Immediate & imm)2206 void X86_64Assembler::cmpl(CpuRegister reg, const Immediate& imm) {
2207   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
2208   CHECK(imm.is_int32());
2209   EmitOptionalRex32(reg);
2210   EmitComplex(7, Operand(reg), imm);
2211 }
2212 
2213 
cmpl(CpuRegister reg0,CpuRegister reg1)2214 void X86_64Assembler::cmpl(CpuRegister reg0, CpuRegister reg1) {
2215   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
2216   EmitOptionalRex32(reg0, reg1);
2217   EmitUint8(0x3B);
2218   EmitOperand(reg0.LowBits(), Operand(reg1));
2219 }
2220 
2221 
cmpl(CpuRegister reg,const Address & address)2222 void X86_64Assembler::cmpl(CpuRegister reg, const Address& address) {
2223   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
2224   EmitOptionalRex32(reg, address);
2225   EmitUint8(0x3B);
2226   EmitOperand(reg.LowBits(), address);
2227 }
2228 
2229 
cmpl(const Address & address,CpuRegister reg)2230 void X86_64Assembler::cmpl(const Address& address, CpuRegister reg) {
2231   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
2232   EmitOptionalRex32(reg, address);
2233   EmitUint8(0x39);
2234   EmitOperand(reg.LowBits(), address);
2235 }
2236 
2237 
cmpl(const Address & address,const Immediate & imm)2238 void X86_64Assembler::cmpl(const Address& address, const Immediate& imm) {
2239   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
2240   CHECK(imm.is_int32());
2241   EmitOptionalRex32(address);
2242   EmitComplex(7, address, imm);
2243 }
2244 
2245 
cmpq(CpuRegister reg0,CpuRegister reg1)2246 void X86_64Assembler::cmpq(CpuRegister reg0, CpuRegister reg1) {
2247   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
2248   EmitRex64(reg0, reg1);
2249   EmitUint8(0x3B);
2250   EmitOperand(reg0.LowBits(), Operand(reg1));
2251 }
2252 
2253 
cmpq(CpuRegister reg,const Immediate & imm)2254 void X86_64Assembler::cmpq(CpuRegister reg, const Immediate& imm) {
2255   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
2256   CHECK(imm.is_int32());  // cmpq only supports 32b immediate.
2257   EmitRex64(reg);
2258   EmitComplex(7, Operand(reg), imm);
2259 }
2260 
2261 
cmpq(CpuRegister reg,const Address & address)2262 void X86_64Assembler::cmpq(CpuRegister reg, const Address& address) {
2263   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
2264   EmitRex64(reg, address);
2265   EmitUint8(0x3B);
2266   EmitOperand(reg.LowBits(), address);
2267 }
2268 
2269 
cmpq(const Address & address,const Immediate & imm)2270 void X86_64Assembler::cmpq(const Address& address, const Immediate& imm) {
2271   CHECK(imm.is_int32());  // cmpq only supports 32b immediate.
2272   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
2273   EmitRex64(address);
2274   EmitComplex(7, address, imm);
2275 }
2276 
2277 
addl(CpuRegister dst,CpuRegister src)2278 void X86_64Assembler::addl(CpuRegister dst, CpuRegister src) {
2279   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
2280   EmitOptionalRex32(dst, src);
2281   EmitUint8(0x03);
2282   EmitRegisterOperand(dst.LowBits(), src.LowBits());
2283 }
2284 
2285 
addl(CpuRegister reg,const Address & address)2286 void X86_64Assembler::addl(CpuRegister reg, const Address& address) {
2287   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
2288   EmitOptionalRex32(reg, address);
2289   EmitUint8(0x03);
2290   EmitOperand(reg.LowBits(), address);
2291 }
2292 
2293 
testl(CpuRegister reg1,CpuRegister reg2)2294 void X86_64Assembler::testl(CpuRegister reg1, CpuRegister reg2) {
2295   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
2296   EmitOptionalRex32(reg1, reg2);
2297   EmitUint8(0x85);
2298   EmitRegisterOperand(reg1.LowBits(), reg2.LowBits());
2299 }
2300 
2301 
testl(CpuRegister reg,const Address & address)2302 void X86_64Assembler::testl(CpuRegister reg, const Address& address) {
2303   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
2304   EmitOptionalRex32(reg, address);
2305   EmitUint8(0x85);
2306   EmitOperand(reg.LowBits(), address);
2307 }
2308 
2309 
testl(CpuRegister reg,const Immediate & immediate)2310 void X86_64Assembler::testl(CpuRegister reg, const Immediate& immediate) {
2311   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
2312   // For registers that have a byte variant (RAX, RBX, RCX, and RDX)
2313   // we only test the byte CpuRegister to keep the encoding short.
2314   if (immediate.is_uint8() && reg.AsRegister() < 4) {
2315     // Use zero-extended 8-bit immediate.
2316     if (reg.AsRegister() == RAX) {
2317       EmitUint8(0xA8);
2318     } else {
2319       EmitUint8(0xF6);
2320       EmitUint8(0xC0 + reg.AsRegister());
2321     }
2322     EmitUint8(immediate.value() & 0xFF);
2323   } else if (reg.AsRegister() == RAX) {
2324     // Use short form if the destination is RAX.
2325     EmitUint8(0xA9);
2326     EmitImmediate(immediate);
2327   } else {
2328     EmitOptionalRex32(reg);
2329     EmitUint8(0xF7);
2330     EmitOperand(0, Operand(reg));
2331     EmitImmediate(immediate);
2332   }
2333 }
2334 
2335 
testq(CpuRegister reg1,CpuRegister reg2)2336 void X86_64Assembler::testq(CpuRegister reg1, CpuRegister reg2) {
2337   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
2338   EmitRex64(reg1, reg2);
2339   EmitUint8(0x85);
2340   EmitRegisterOperand(reg1.LowBits(), reg2.LowBits());
2341 }
2342 
2343 
testq(CpuRegister reg,const Address & address)2344 void X86_64Assembler::testq(CpuRegister reg, const Address& address) {
2345   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
2346   EmitRex64(reg, address);
2347   EmitUint8(0x85);
2348   EmitOperand(reg.LowBits(), address);
2349 }
2350 
2351 
testb(const Address & dst,const Immediate & imm)2352 void X86_64Assembler::testb(const Address& dst, const Immediate& imm) {
2353   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
2354   EmitOptionalRex32(dst);
2355   EmitUint8(0xF6);
2356   EmitOperand(Register::RAX, dst);
2357   CHECK(imm.is_int8());
2358   EmitUint8(imm.value() & 0xFF);
2359 }
2360 
2361 
testl(const Address & dst,const Immediate & imm)2362 void X86_64Assembler::testl(const Address& dst, const Immediate& imm) {
2363   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
2364   EmitOptionalRex32(dst);
2365   EmitUint8(0xF7);
2366   EmitOperand(0, dst);
2367   EmitImmediate(imm);
2368 }
2369 
2370 
andl(CpuRegister dst,CpuRegister src)2371 void X86_64Assembler::andl(CpuRegister dst, CpuRegister src) {
2372   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
2373   EmitOptionalRex32(dst, src);
2374   EmitUint8(0x23);
2375   EmitOperand(dst.LowBits(), Operand(src));
2376 }
2377 
2378 
andl(CpuRegister reg,const Address & address)2379 void X86_64Assembler::andl(CpuRegister reg, const Address& address) {
2380   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
2381   EmitOptionalRex32(reg, address);
2382   EmitUint8(0x23);
2383   EmitOperand(reg.LowBits(), address);
2384 }
2385 
2386 
andl(CpuRegister dst,const Immediate & imm)2387 void X86_64Assembler::andl(CpuRegister dst, const Immediate& imm) {
2388   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
2389   EmitOptionalRex32(dst);
2390   EmitComplex(4, Operand(dst), imm);
2391 }
2392 
2393 
andq(CpuRegister reg,const Immediate & imm)2394 void X86_64Assembler::andq(CpuRegister reg, const Immediate& imm) {
2395   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
2396   CHECK(imm.is_int32());  // andq only supports 32b immediate.
2397   EmitRex64(reg);
2398   EmitComplex(4, Operand(reg), imm);
2399 }
2400 
2401 
andq(CpuRegister dst,CpuRegister src)2402 void X86_64Assembler::andq(CpuRegister dst, CpuRegister src) {
2403   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
2404   EmitRex64(dst, src);
2405   EmitUint8(0x23);
2406   EmitOperand(dst.LowBits(), Operand(src));
2407 }
2408 
2409 
andq(CpuRegister dst,const Address & src)2410 void X86_64Assembler::andq(CpuRegister dst, const Address& src) {
2411   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
2412   EmitRex64(dst, src);
2413   EmitUint8(0x23);
2414   EmitOperand(dst.LowBits(), src);
2415 }
2416 
2417 
orl(CpuRegister dst,CpuRegister src)2418 void X86_64Assembler::orl(CpuRegister dst, CpuRegister src) {
2419   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
2420   EmitOptionalRex32(dst, src);
2421   EmitUint8(0x0B);
2422   EmitOperand(dst.LowBits(), Operand(src));
2423 }
2424 
2425 
orl(CpuRegister reg,const Address & address)2426 void X86_64Assembler::orl(CpuRegister reg, const Address& address) {
2427   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
2428   EmitOptionalRex32(reg, address);
2429   EmitUint8(0x0B);
2430   EmitOperand(reg.LowBits(), address);
2431 }
2432 
2433 
orl(CpuRegister dst,const Immediate & imm)2434 void X86_64Assembler::orl(CpuRegister dst, const Immediate& imm) {
2435   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
2436   EmitOptionalRex32(dst);
2437   EmitComplex(1, Operand(dst), imm);
2438 }
2439 
2440 
orq(CpuRegister dst,const Immediate & imm)2441 void X86_64Assembler::orq(CpuRegister dst, const Immediate& imm) {
2442   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
2443   CHECK(imm.is_int32());  // orq only supports 32b immediate.
2444   EmitRex64(dst);
2445   EmitComplex(1, Operand(dst), imm);
2446 }
2447 
2448 
orq(CpuRegister dst,CpuRegister src)2449 void X86_64Assembler::orq(CpuRegister dst, CpuRegister src) {
2450   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
2451   EmitRex64(dst, src);
2452   EmitUint8(0x0B);
2453   EmitOperand(dst.LowBits(), Operand(src));
2454 }
2455 
2456 
orq(CpuRegister dst,const Address & src)2457 void X86_64Assembler::orq(CpuRegister dst, const Address& src) {
2458   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
2459   EmitRex64(dst, src);
2460   EmitUint8(0x0B);
2461   EmitOperand(dst.LowBits(), src);
2462 }
2463 
2464 
xorl(CpuRegister dst,CpuRegister src)2465 void X86_64Assembler::xorl(CpuRegister dst, CpuRegister src) {
2466   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
2467   EmitOptionalRex32(dst, src);
2468   EmitUint8(0x33);
2469   EmitOperand(dst.LowBits(), Operand(src));
2470 }
2471 
2472 
xorl(CpuRegister reg,const Address & address)2473 void X86_64Assembler::xorl(CpuRegister reg, const Address& address) {
2474   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
2475   EmitOptionalRex32(reg, address);
2476   EmitUint8(0x33);
2477   EmitOperand(reg.LowBits(), address);
2478 }
2479 
2480 
xorl(CpuRegister dst,const Immediate & imm)2481 void X86_64Assembler::xorl(CpuRegister dst, const Immediate& imm) {
2482   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
2483   EmitOptionalRex32(dst);
2484   EmitComplex(6, Operand(dst), imm);
2485 }
2486 
2487 
xorq(CpuRegister dst,CpuRegister src)2488 void X86_64Assembler::xorq(CpuRegister dst, CpuRegister src) {
2489   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
2490   EmitRex64(dst, src);
2491   EmitUint8(0x33);
2492   EmitOperand(dst.LowBits(), Operand(src));
2493 }
2494 
2495 
xorq(CpuRegister dst,const Immediate & imm)2496 void X86_64Assembler::xorq(CpuRegister dst, const Immediate& imm) {
2497   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
2498   CHECK(imm.is_int32());  // xorq only supports 32b immediate.
2499   EmitRex64(dst);
2500   EmitComplex(6, Operand(dst), imm);
2501 }
2502 
xorq(CpuRegister dst,const Address & src)2503 void X86_64Assembler::xorq(CpuRegister dst, const Address& src) {
2504   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
2505   EmitRex64(dst, src);
2506   EmitUint8(0x33);
2507   EmitOperand(dst.LowBits(), src);
2508 }
2509 
2510 
2511 #if 0
2512 void X86_64Assembler::rex(bool force, bool w, Register* r, Register* x, Register* b) {
2513   // REX.WRXB
2514   // W - 64-bit operand
2515   // R - MODRM.reg
2516   // X - SIB.index
2517   // B - MODRM.rm/SIB.base
2518   uint8_t rex = force ? 0x40 : 0;
2519   if (w) {
2520     rex |= 0x48;  // REX.W000
2521   }
2522   if (r != nullptr && *r >= Register::R8 && *r < Register::kNumberOfCpuRegisters) {
2523     rex |= 0x44;  // REX.0R00
2524     *r = static_cast<Register>(*r - 8);
2525   }
2526   if (x != nullptr && *x >= Register::R8 && *x < Register::kNumberOfCpuRegisters) {
2527     rex |= 0x42;  // REX.00X0
2528     *x = static_cast<Register>(*x - 8);
2529   }
2530   if (b != nullptr && *b >= Register::R8 && *b < Register::kNumberOfCpuRegisters) {
2531     rex |= 0x41;  // REX.000B
2532     *b = static_cast<Register>(*b - 8);
2533   }
2534   if (rex != 0) {
2535     EmitUint8(rex);
2536   }
2537 }
2538 
2539 void X86_64Assembler::rex_reg_mem(bool force, bool w, Register* dst, const Address& mem) {
2540   // REX.WRXB
2541   // W - 64-bit operand
2542   // R - MODRM.reg
2543   // X - SIB.index
2544   // B - MODRM.rm/SIB.base
2545   uint8_t rex = mem->rex();
2546   if (force) {
2547     rex |= 0x40;  // REX.0000
2548   }
2549   if (w) {
2550     rex |= 0x48;  // REX.W000
2551   }
2552   if (dst != nullptr && *dst >= Register::R8 && *dst < Register::kNumberOfCpuRegisters) {
2553     rex |= 0x44;  // REX.0R00
2554     *dst = static_cast<Register>(*dst - 8);
2555   }
2556   if (rex != 0) {
2557     EmitUint8(rex);
2558   }
2559 }
2560 
2561 void rex_mem_reg(bool force, bool w, Address* mem, Register* src);
2562 #endif
2563 
addl(CpuRegister reg,const Immediate & imm)2564 void X86_64Assembler::addl(CpuRegister reg, const Immediate& imm) {
2565   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
2566   EmitOptionalRex32(reg);
2567   EmitComplex(0, Operand(reg), imm);
2568 }
2569 
2570 
addq(CpuRegister reg,const Immediate & imm)2571 void X86_64Assembler::addq(CpuRegister reg, const Immediate& imm) {
2572   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
2573   CHECK(imm.is_int32());  // addq only supports 32b immediate.
2574   EmitRex64(reg);
2575   EmitComplex(0, Operand(reg), imm);
2576 }
2577 
2578 
addq(CpuRegister dst,const Address & address)2579 void X86_64Assembler::addq(CpuRegister dst, const Address& address) {
2580   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
2581   EmitRex64(dst, address);
2582   EmitUint8(0x03);
2583   EmitOperand(dst.LowBits(), address);
2584 }
2585 
2586 
addq(CpuRegister dst,CpuRegister src)2587 void X86_64Assembler::addq(CpuRegister dst, CpuRegister src) {
2588   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
2589   // 0x01 is addq r/m64 <- r/m64 + r64, with op1 in r/m and op2 in reg: so reverse EmitRex64
2590   EmitRex64(src, dst);
2591   EmitUint8(0x01);
2592   EmitRegisterOperand(src.LowBits(), dst.LowBits());
2593 }
2594 
2595 
addl(const Address & address,CpuRegister reg)2596 void X86_64Assembler::addl(const Address& address, CpuRegister reg) {
2597   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
2598   EmitOptionalRex32(reg, address);
2599   EmitUint8(0x01);
2600   EmitOperand(reg.LowBits(), address);
2601 }
2602 
2603 
addl(const Address & address,const Immediate & imm)2604 void X86_64Assembler::addl(const Address& address, const Immediate& imm) {
2605   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
2606   EmitOptionalRex32(address);
2607   EmitComplex(0, address, imm);
2608 }
2609 
2610 
addw(const Address & address,const Immediate & imm)2611 void X86_64Assembler::addw(const Address& address, const Immediate& imm) {
2612   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
2613   CHECK(imm.is_uint16() || imm.is_int16()) << imm.value();
2614   EmitUint8(0x66);
2615   EmitOptionalRex32(address);
2616   EmitComplex(0, address, imm, /* is_16_op */ true);
2617 }
2618 
2619 
subl(CpuRegister dst,CpuRegister src)2620 void X86_64Assembler::subl(CpuRegister dst, CpuRegister src) {
2621   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
2622   EmitOptionalRex32(dst, src);
2623   EmitUint8(0x2B);
2624   EmitOperand(dst.LowBits(), Operand(src));
2625 }
2626 
2627 
subl(CpuRegister reg,const Immediate & imm)2628 void X86_64Assembler::subl(CpuRegister reg, const Immediate& imm) {
2629   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
2630   EmitOptionalRex32(reg);
2631   EmitComplex(5, Operand(reg), imm);
2632 }
2633 
2634 
subq(CpuRegister reg,const Immediate & imm)2635 void X86_64Assembler::subq(CpuRegister reg, const Immediate& imm) {
2636   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
2637   CHECK(imm.is_int32());  // subq only supports 32b immediate.
2638   EmitRex64(reg);
2639   EmitComplex(5, Operand(reg), imm);
2640 }
2641 
2642 
subq(CpuRegister dst,CpuRegister src)2643 void X86_64Assembler::subq(CpuRegister dst, CpuRegister src) {
2644   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
2645   EmitRex64(dst, src);
2646   EmitUint8(0x2B);
2647   EmitRegisterOperand(dst.LowBits(), src.LowBits());
2648 }
2649 
2650 
subq(CpuRegister reg,const Address & address)2651 void X86_64Assembler::subq(CpuRegister reg, const Address& address) {
2652   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
2653   EmitRex64(reg, address);
2654   EmitUint8(0x2B);
2655   EmitOperand(reg.LowBits() & 7, address);
2656 }
2657 
2658 
subl(CpuRegister reg,const Address & address)2659 void X86_64Assembler::subl(CpuRegister reg, const Address& address) {
2660   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
2661   EmitOptionalRex32(reg, address);
2662   EmitUint8(0x2B);
2663   EmitOperand(reg.LowBits(), address);
2664 }
2665 
2666 
cdq()2667 void X86_64Assembler::cdq() {
2668   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
2669   EmitUint8(0x99);
2670 }
2671 
2672 
cqo()2673 void X86_64Assembler::cqo() {
2674   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
2675   EmitRex64();
2676   EmitUint8(0x99);
2677 }
2678 
2679 
idivl(CpuRegister reg)2680 void X86_64Assembler::idivl(CpuRegister reg) {
2681   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
2682   EmitOptionalRex32(reg);
2683   EmitUint8(0xF7);
2684   EmitUint8(0xF8 | reg.LowBits());
2685 }
2686 
2687 
idivq(CpuRegister reg)2688 void X86_64Assembler::idivq(CpuRegister reg) {
2689   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
2690   EmitRex64(reg);
2691   EmitUint8(0xF7);
2692   EmitUint8(0xF8 | reg.LowBits());
2693 }
2694 
2695 
imull(CpuRegister dst,CpuRegister src)2696 void X86_64Assembler::imull(CpuRegister dst, CpuRegister src) {
2697   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
2698   EmitOptionalRex32(dst, src);
2699   EmitUint8(0x0F);
2700   EmitUint8(0xAF);
2701   EmitOperand(dst.LowBits(), Operand(src));
2702 }
2703 
imull(CpuRegister dst,CpuRegister src,const Immediate & imm)2704 void X86_64Assembler::imull(CpuRegister dst, CpuRegister src, const Immediate& imm) {
2705   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
2706   CHECK(imm.is_int32());  // imull only supports 32b immediate.
2707 
2708   EmitOptionalRex32(dst, src);
2709 
2710   // See whether imm can be represented as a sign-extended 8bit value.
2711   int32_t v32 = static_cast<int32_t>(imm.value());
2712   if (IsInt<8>(v32)) {
2713     // Sign-extension works.
2714     EmitUint8(0x6B);
2715     EmitOperand(dst.LowBits(), Operand(src));
2716     EmitUint8(static_cast<uint8_t>(v32 & 0xFF));
2717   } else {
2718     // Not representable, use full immediate.
2719     EmitUint8(0x69);
2720     EmitOperand(dst.LowBits(), Operand(src));
2721     EmitImmediate(imm);
2722   }
2723 }
2724 
2725 
imull(CpuRegister reg,const Immediate & imm)2726 void X86_64Assembler::imull(CpuRegister reg, const Immediate& imm) {
2727   imull(reg, reg, imm);
2728 }
2729 
2730 
imull(CpuRegister reg,const Address & address)2731 void X86_64Assembler::imull(CpuRegister reg, const Address& address) {
2732   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
2733   EmitOptionalRex32(reg, address);
2734   EmitUint8(0x0F);
2735   EmitUint8(0xAF);
2736   EmitOperand(reg.LowBits(), address);
2737 }
2738 
2739 
imulq(CpuRegister dst,CpuRegister src)2740 void X86_64Assembler::imulq(CpuRegister dst, CpuRegister src) {
2741   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
2742   EmitRex64(dst, src);
2743   EmitUint8(0x0F);
2744   EmitUint8(0xAF);
2745   EmitRegisterOperand(dst.LowBits(), src.LowBits());
2746 }
2747 
2748 
imulq(CpuRegister reg,const Immediate & imm)2749 void X86_64Assembler::imulq(CpuRegister reg, const Immediate& imm) {
2750   imulq(reg, reg, imm);
2751 }
2752 
imulq(CpuRegister dst,CpuRegister reg,const Immediate & imm)2753 void X86_64Assembler::imulq(CpuRegister dst, CpuRegister reg, const Immediate& imm) {
2754   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
2755   CHECK(imm.is_int32());  // imulq only supports 32b immediate.
2756 
2757   EmitRex64(dst, reg);
2758 
2759   // See whether imm can be represented as a sign-extended 8bit value.
2760   int64_t v64 = imm.value();
2761   if (IsInt<8>(v64)) {
2762     // Sign-extension works.
2763     EmitUint8(0x6B);
2764     EmitOperand(dst.LowBits(), Operand(reg));
2765     EmitUint8(static_cast<uint8_t>(v64 & 0xFF));
2766   } else {
2767     // Not representable, use full immediate.
2768     EmitUint8(0x69);
2769     EmitOperand(dst.LowBits(), Operand(reg));
2770     EmitImmediate(imm);
2771   }
2772 }
2773 
imulq(CpuRegister reg,const Address & address)2774 void X86_64Assembler::imulq(CpuRegister reg, const Address& address) {
2775   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
2776   EmitRex64(reg, address);
2777   EmitUint8(0x0F);
2778   EmitUint8(0xAF);
2779   EmitOperand(reg.LowBits(), address);
2780 }
2781 
2782 
imull(CpuRegister reg)2783 void X86_64Assembler::imull(CpuRegister reg) {
2784   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
2785   EmitOptionalRex32(reg);
2786   EmitUint8(0xF7);
2787   EmitOperand(5, Operand(reg));
2788 }
2789 
2790 
imulq(CpuRegister reg)2791 void X86_64Assembler::imulq(CpuRegister reg) {
2792   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
2793   EmitRex64(reg);
2794   EmitUint8(0xF7);
2795   EmitOperand(5, Operand(reg));
2796 }
2797 
2798 
imull(const Address & address)2799 void X86_64Assembler::imull(const Address& address) {
2800   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
2801   EmitOptionalRex32(address);
2802   EmitUint8(0xF7);
2803   EmitOperand(5, address);
2804 }
2805 
2806 
mull(CpuRegister reg)2807 void X86_64Assembler::mull(CpuRegister reg) {
2808   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
2809   EmitOptionalRex32(reg);
2810   EmitUint8(0xF7);
2811   EmitOperand(4, Operand(reg));
2812 }
2813 
2814 
mull(const Address & address)2815 void X86_64Assembler::mull(const Address& address) {
2816   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
2817   EmitOptionalRex32(address);
2818   EmitUint8(0xF7);
2819   EmitOperand(4, address);
2820 }
2821 
2822 
shll(CpuRegister reg,const Immediate & imm)2823 void X86_64Assembler::shll(CpuRegister reg, const Immediate& imm) {
2824   EmitGenericShift(false, 4, reg, imm);
2825 }
2826 
2827 
shlq(CpuRegister reg,const Immediate & imm)2828 void X86_64Assembler::shlq(CpuRegister reg, const Immediate& imm) {
2829   EmitGenericShift(true, 4, reg, imm);
2830 }
2831 
2832 
shll(CpuRegister operand,CpuRegister shifter)2833 void X86_64Assembler::shll(CpuRegister operand, CpuRegister shifter) {
2834   EmitGenericShift(false, 4, operand, shifter);
2835 }
2836 
2837 
shlq(CpuRegister operand,CpuRegister shifter)2838 void X86_64Assembler::shlq(CpuRegister operand, CpuRegister shifter) {
2839   EmitGenericShift(true, 4, operand, shifter);
2840 }
2841 
2842 
shrl(CpuRegister reg,const Immediate & imm)2843 void X86_64Assembler::shrl(CpuRegister reg, const Immediate& imm) {
2844   EmitGenericShift(false, 5, reg, imm);
2845 }
2846 
2847 
shrq(CpuRegister reg,const Immediate & imm)2848 void X86_64Assembler::shrq(CpuRegister reg, const Immediate& imm) {
2849   EmitGenericShift(true, 5, reg, imm);
2850 }
2851 
2852 
shrl(CpuRegister operand,CpuRegister shifter)2853 void X86_64Assembler::shrl(CpuRegister operand, CpuRegister shifter) {
2854   EmitGenericShift(false, 5, operand, shifter);
2855 }
2856 
2857 
shrq(CpuRegister operand,CpuRegister shifter)2858 void X86_64Assembler::shrq(CpuRegister operand, CpuRegister shifter) {
2859   EmitGenericShift(true, 5, operand, shifter);
2860 }
2861 
2862 
sarl(CpuRegister reg,const Immediate & imm)2863 void X86_64Assembler::sarl(CpuRegister reg, const Immediate& imm) {
2864   EmitGenericShift(false, 7, reg, imm);
2865 }
2866 
2867 
sarl(CpuRegister operand,CpuRegister shifter)2868 void X86_64Assembler::sarl(CpuRegister operand, CpuRegister shifter) {
2869   EmitGenericShift(false, 7, operand, shifter);
2870 }
2871 
2872 
sarq(CpuRegister reg,const Immediate & imm)2873 void X86_64Assembler::sarq(CpuRegister reg, const Immediate& imm) {
2874   EmitGenericShift(true, 7, reg, imm);
2875 }
2876 
2877 
sarq(CpuRegister operand,CpuRegister shifter)2878 void X86_64Assembler::sarq(CpuRegister operand, CpuRegister shifter) {
2879   EmitGenericShift(true, 7, operand, shifter);
2880 }
2881 
2882 
roll(CpuRegister reg,const Immediate & imm)2883 void X86_64Assembler::roll(CpuRegister reg, const Immediate& imm) {
2884   EmitGenericShift(false, 0, reg, imm);
2885 }
2886 
2887 
roll(CpuRegister operand,CpuRegister shifter)2888 void X86_64Assembler::roll(CpuRegister operand, CpuRegister shifter) {
2889   EmitGenericShift(false, 0, operand, shifter);
2890 }
2891 
2892 
rorl(CpuRegister reg,const Immediate & imm)2893 void X86_64Assembler::rorl(CpuRegister reg, const Immediate& imm) {
2894   EmitGenericShift(false, 1, reg, imm);
2895 }
2896 
2897 
rorl(CpuRegister operand,CpuRegister shifter)2898 void X86_64Assembler::rorl(CpuRegister operand, CpuRegister shifter) {
2899   EmitGenericShift(false, 1, operand, shifter);
2900 }
2901 
2902 
rolq(CpuRegister reg,const Immediate & imm)2903 void X86_64Assembler::rolq(CpuRegister reg, const Immediate& imm) {
2904   EmitGenericShift(true, 0, reg, imm);
2905 }
2906 
2907 
rolq(CpuRegister operand,CpuRegister shifter)2908 void X86_64Assembler::rolq(CpuRegister operand, CpuRegister shifter) {
2909   EmitGenericShift(true, 0, operand, shifter);
2910 }
2911 
2912 
rorq(CpuRegister reg,const Immediate & imm)2913 void X86_64Assembler::rorq(CpuRegister reg, const Immediate& imm) {
2914   EmitGenericShift(true, 1, reg, imm);
2915 }
2916 
2917 
rorq(CpuRegister operand,CpuRegister shifter)2918 void X86_64Assembler::rorq(CpuRegister operand, CpuRegister shifter) {
2919   EmitGenericShift(true, 1, operand, shifter);
2920 }
2921 
2922 
negl(CpuRegister reg)2923 void X86_64Assembler::negl(CpuRegister reg) {
2924   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
2925   EmitOptionalRex32(reg);
2926   EmitUint8(0xF7);
2927   EmitOperand(3, Operand(reg));
2928 }
2929 
2930 
negq(CpuRegister reg)2931 void X86_64Assembler::negq(CpuRegister reg) {
2932   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
2933   EmitRex64(reg);
2934   EmitUint8(0xF7);
2935   EmitOperand(3, Operand(reg));
2936 }
2937 
2938 
notl(CpuRegister reg)2939 void X86_64Assembler::notl(CpuRegister reg) {
2940   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
2941   EmitOptionalRex32(reg);
2942   EmitUint8(0xF7);
2943   EmitUint8(0xD0 | reg.LowBits());
2944 }
2945 
2946 
notq(CpuRegister reg)2947 void X86_64Assembler::notq(CpuRegister reg) {
2948   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
2949   EmitRex64(reg);
2950   EmitUint8(0xF7);
2951   EmitOperand(2, Operand(reg));
2952 }
2953 
2954 
enter(const Immediate & imm)2955 void X86_64Assembler::enter(const Immediate& imm) {
2956   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
2957   EmitUint8(0xC8);
2958   CHECK(imm.is_uint16()) << imm.value();
2959   EmitUint8(imm.value() & 0xFF);
2960   EmitUint8((imm.value() >> 8) & 0xFF);
2961   EmitUint8(0x00);
2962 }
2963 
2964 
leave()2965 void X86_64Assembler::leave() {
2966   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
2967   EmitUint8(0xC9);
2968 }
2969 
2970 
ret()2971 void X86_64Assembler::ret() {
2972   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
2973   EmitUint8(0xC3);
2974 }
2975 
2976 
ret(const Immediate & imm)2977 void X86_64Assembler::ret(const Immediate& imm) {
2978   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
2979   EmitUint8(0xC2);
2980   CHECK(imm.is_uint16());
2981   EmitUint8(imm.value() & 0xFF);
2982   EmitUint8((imm.value() >> 8) & 0xFF);
2983 }
2984 
2985 
2986 
nop()2987 void X86_64Assembler::nop() {
2988   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
2989   EmitUint8(0x90);
2990 }
2991 
2992 
int3()2993 void X86_64Assembler::int3() {
2994   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
2995   EmitUint8(0xCC);
2996 }
2997 
2998 
hlt()2999 void X86_64Assembler::hlt() {
3000   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
3001   EmitUint8(0xF4);
3002 }
3003 
3004 
j(Condition condition,Label * label)3005 void X86_64Assembler::j(Condition condition, Label* label) {
3006   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
3007   if (label->IsBound()) {
3008     static const int kShortSize = 2;
3009     static const int kLongSize = 6;
3010     int offset = label->Position() - buffer_.Size();
3011     CHECK_LE(offset, 0);
3012     if (IsInt<8>(offset - kShortSize)) {
3013       EmitUint8(0x70 + condition);
3014       EmitUint8((offset - kShortSize) & 0xFF);
3015     } else {
3016       EmitUint8(0x0F);
3017       EmitUint8(0x80 + condition);
3018       EmitInt32(offset - kLongSize);
3019     }
3020   } else {
3021     EmitUint8(0x0F);
3022     EmitUint8(0x80 + condition);
3023     EmitLabelLink(label);
3024   }
3025 }
3026 
3027 
j(Condition condition,NearLabel * label)3028 void X86_64Assembler::j(Condition condition, NearLabel* label) {
3029   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
3030   if (label->IsBound()) {
3031     static const int kShortSize = 2;
3032     int offset = label->Position() - buffer_.Size();
3033     CHECK_LE(offset, 0);
3034     CHECK(IsInt<8>(offset - kShortSize));
3035     EmitUint8(0x70 + condition);
3036     EmitUint8((offset - kShortSize) & 0xFF);
3037   } else {
3038     EmitUint8(0x70 + condition);
3039     EmitLabelLink(label);
3040   }
3041 }
3042 
3043 
jrcxz(NearLabel * label)3044 void X86_64Assembler::jrcxz(NearLabel* label) {
3045   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
3046   if (label->IsBound()) {
3047     static const int kShortSize = 2;
3048     int offset = label->Position() - buffer_.Size();
3049     CHECK_LE(offset, 0);
3050     CHECK(IsInt<8>(offset - kShortSize));
3051     EmitUint8(0xE3);
3052     EmitUint8((offset - kShortSize) & 0xFF);
3053   } else {
3054     EmitUint8(0xE3);
3055     EmitLabelLink(label);
3056   }
3057 }
3058 
3059 
jmp(CpuRegister reg)3060 void X86_64Assembler::jmp(CpuRegister reg) {
3061   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
3062   EmitOptionalRex32(reg);
3063   EmitUint8(0xFF);
3064   EmitRegisterOperand(4, reg.LowBits());
3065 }
3066 
jmp(const Address & address)3067 void X86_64Assembler::jmp(const Address& address) {
3068   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
3069   EmitOptionalRex32(address);
3070   EmitUint8(0xFF);
3071   EmitOperand(4, address);
3072 }
3073 
jmp(Label * label)3074 void X86_64Assembler::jmp(Label* label) {
3075   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
3076   if (label->IsBound()) {
3077     static const int kShortSize = 2;
3078     static const int kLongSize = 5;
3079     int offset = label->Position() - buffer_.Size();
3080     CHECK_LE(offset, 0);
3081     if (IsInt<8>(offset - kShortSize)) {
3082       EmitUint8(0xEB);
3083       EmitUint8((offset - kShortSize) & 0xFF);
3084     } else {
3085       EmitUint8(0xE9);
3086       EmitInt32(offset - kLongSize);
3087     }
3088   } else {
3089     EmitUint8(0xE9);
3090     EmitLabelLink(label);
3091   }
3092 }
3093 
3094 
jmp(NearLabel * label)3095 void X86_64Assembler::jmp(NearLabel* label) {
3096   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
3097   if (label->IsBound()) {
3098     static const int kShortSize = 2;
3099     int offset = label->Position() - buffer_.Size();
3100     CHECK_LE(offset, 0);
3101     CHECK(IsInt<8>(offset - kShortSize));
3102     EmitUint8(0xEB);
3103     EmitUint8((offset - kShortSize) & 0xFF);
3104   } else {
3105     EmitUint8(0xEB);
3106     EmitLabelLink(label);
3107   }
3108 }
3109 
3110 
rep_movsw()3111 void X86_64Assembler::rep_movsw() {
3112   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
3113   EmitUint8(0x66);
3114   EmitUint8(0xF3);
3115   EmitUint8(0xA5);
3116 }
3117 
3118 
lock()3119 X86_64Assembler* X86_64Assembler::lock() {
3120   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
3121   EmitUint8(0xF0);
3122   return this;
3123 }
3124 
3125 
cmpxchgl(const Address & address,CpuRegister reg)3126 void X86_64Assembler::cmpxchgl(const Address& address, CpuRegister reg) {
3127   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
3128   EmitOptionalRex32(reg, address);
3129   EmitUint8(0x0F);
3130   EmitUint8(0xB1);
3131   EmitOperand(reg.LowBits(), address);
3132 }
3133 
3134 
cmpxchgq(const Address & address,CpuRegister reg)3135 void X86_64Assembler::cmpxchgq(const Address& address, CpuRegister reg) {
3136   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
3137   EmitRex64(reg, address);
3138   EmitUint8(0x0F);
3139   EmitUint8(0xB1);
3140   EmitOperand(reg.LowBits(), address);
3141 }
3142 
3143 
mfence()3144 void X86_64Assembler::mfence() {
3145   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
3146   EmitUint8(0x0F);
3147   EmitUint8(0xAE);
3148   EmitUint8(0xF0);
3149 }
3150 
3151 
gs()3152 X86_64Assembler* X86_64Assembler::gs() {
3153   // TODO: gs is a prefix and not an instruction
3154   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
3155   EmitUint8(0x65);
3156   return this;
3157 }
3158 
3159 
AddImmediate(CpuRegister reg,const Immediate & imm)3160 void X86_64Assembler::AddImmediate(CpuRegister reg, const Immediate& imm) {
3161   int value = imm.value();
3162   if (value != 0) {
3163     if (value > 0) {
3164       addl(reg, imm);
3165     } else {
3166       subl(reg, Immediate(value));
3167     }
3168   }
3169 }
3170 
3171 
setcc(Condition condition,CpuRegister dst)3172 void X86_64Assembler::setcc(Condition condition, CpuRegister dst) {
3173   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
3174   // RSP, RBP, RDI, RSI need rex prefix (else the pattern encodes ah/bh/ch/dh).
3175   if (dst.NeedsRex() || dst.AsRegister() > 3) {
3176     EmitOptionalRex(true, false, false, false, dst.NeedsRex());
3177   }
3178   EmitUint8(0x0F);
3179   EmitUint8(0x90 + condition);
3180   EmitUint8(0xC0 + dst.LowBits());
3181 }
3182 
bswapl(CpuRegister dst)3183 void X86_64Assembler::bswapl(CpuRegister dst) {
3184   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
3185   EmitOptionalRex(false, false, false, false, dst.NeedsRex());
3186   EmitUint8(0x0F);
3187   EmitUint8(0xC8 + dst.LowBits());
3188 }
3189 
bswapq(CpuRegister dst)3190 void X86_64Assembler::bswapq(CpuRegister dst) {
3191   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
3192   EmitOptionalRex(false, true, false, false, dst.NeedsRex());
3193   EmitUint8(0x0F);
3194   EmitUint8(0xC8 + dst.LowBits());
3195 }
3196 
bsfl(CpuRegister dst,CpuRegister src)3197 void X86_64Assembler::bsfl(CpuRegister dst, CpuRegister src) {
3198   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
3199   EmitOptionalRex32(dst, src);
3200   EmitUint8(0x0F);
3201   EmitUint8(0xBC);
3202   EmitRegisterOperand(dst.LowBits(), src.LowBits());
3203 }
3204 
bsfl(CpuRegister dst,const Address & src)3205 void X86_64Assembler::bsfl(CpuRegister dst, const Address& src) {
3206   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
3207   EmitOptionalRex32(dst, src);
3208   EmitUint8(0x0F);
3209   EmitUint8(0xBC);
3210   EmitOperand(dst.LowBits(), src);
3211 }
3212 
bsfq(CpuRegister dst,CpuRegister src)3213 void X86_64Assembler::bsfq(CpuRegister dst, CpuRegister src) {
3214   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
3215   EmitRex64(dst, src);
3216   EmitUint8(0x0F);
3217   EmitUint8(0xBC);
3218   EmitRegisterOperand(dst.LowBits(), src.LowBits());
3219 }
3220 
bsfq(CpuRegister dst,const Address & src)3221 void X86_64Assembler::bsfq(CpuRegister dst, const Address& src) {
3222   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
3223   EmitRex64(dst, src);
3224   EmitUint8(0x0F);
3225   EmitUint8(0xBC);
3226   EmitOperand(dst.LowBits(), src);
3227 }
3228 
bsrl(CpuRegister dst,CpuRegister src)3229 void X86_64Assembler::bsrl(CpuRegister dst, CpuRegister src) {
3230   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
3231   EmitOptionalRex32(dst, src);
3232   EmitUint8(0x0F);
3233   EmitUint8(0xBD);
3234   EmitRegisterOperand(dst.LowBits(), src.LowBits());
3235 }
3236 
bsrl(CpuRegister dst,const Address & src)3237 void X86_64Assembler::bsrl(CpuRegister dst, const Address& src) {
3238   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
3239   EmitOptionalRex32(dst, src);
3240   EmitUint8(0x0F);
3241   EmitUint8(0xBD);
3242   EmitOperand(dst.LowBits(), src);
3243 }
3244 
bsrq(CpuRegister dst,CpuRegister src)3245 void X86_64Assembler::bsrq(CpuRegister dst, CpuRegister src) {
3246   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
3247   EmitRex64(dst, src);
3248   EmitUint8(0x0F);
3249   EmitUint8(0xBD);
3250   EmitRegisterOperand(dst.LowBits(), src.LowBits());
3251 }
3252 
bsrq(CpuRegister dst,const Address & src)3253 void X86_64Assembler::bsrq(CpuRegister dst, const Address& src) {
3254   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
3255   EmitRex64(dst, src);
3256   EmitUint8(0x0F);
3257   EmitUint8(0xBD);
3258   EmitOperand(dst.LowBits(), src);
3259 }
3260 
popcntl(CpuRegister dst,CpuRegister src)3261 void X86_64Assembler::popcntl(CpuRegister dst, CpuRegister src) {
3262   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
3263   EmitUint8(0xF3);
3264   EmitOptionalRex32(dst, src);
3265   EmitUint8(0x0F);
3266   EmitUint8(0xB8);
3267   EmitRegisterOperand(dst.LowBits(), src.LowBits());
3268 }
3269 
popcntl(CpuRegister dst,const Address & src)3270 void X86_64Assembler::popcntl(CpuRegister dst, const Address& src) {
3271   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
3272   EmitUint8(0xF3);
3273   EmitOptionalRex32(dst, src);
3274   EmitUint8(0x0F);
3275   EmitUint8(0xB8);
3276   EmitOperand(dst.LowBits(), src);
3277 }
3278 
popcntq(CpuRegister dst,CpuRegister src)3279 void X86_64Assembler::popcntq(CpuRegister dst, CpuRegister src) {
3280   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
3281   EmitUint8(0xF3);
3282   EmitRex64(dst, src);
3283   EmitUint8(0x0F);
3284   EmitUint8(0xB8);
3285   EmitRegisterOperand(dst.LowBits(), src.LowBits());
3286 }
3287 
popcntq(CpuRegister dst,const Address & src)3288 void X86_64Assembler::popcntq(CpuRegister dst, const Address& src) {
3289   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
3290   EmitUint8(0xF3);
3291   EmitRex64(dst, src);
3292   EmitUint8(0x0F);
3293   EmitUint8(0xB8);
3294   EmitOperand(dst.LowBits(), src);
3295 }
3296 
repne_scasb()3297 void X86_64Assembler::repne_scasb() {
3298   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
3299   EmitUint8(0xF2);
3300   EmitUint8(0xAE);
3301 }
3302 
repne_scasw()3303 void X86_64Assembler::repne_scasw() {
3304   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
3305   EmitUint8(0x66);
3306   EmitUint8(0xF2);
3307   EmitUint8(0xAF);
3308 }
3309 
repe_cmpsw()3310 void X86_64Assembler::repe_cmpsw() {
3311   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
3312   EmitUint8(0x66);
3313   EmitUint8(0xF3);
3314   EmitUint8(0xA7);
3315 }
3316 
3317 
repe_cmpsl()3318 void X86_64Assembler::repe_cmpsl() {
3319   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
3320   EmitUint8(0xF3);
3321   EmitUint8(0xA7);
3322 }
3323 
3324 
repe_cmpsq()3325 void X86_64Assembler::repe_cmpsq() {
3326   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
3327   EmitUint8(0xF3);
3328   EmitRex64();
3329   EmitUint8(0xA7);
3330 }
3331 
3332 
LoadDoubleConstant(XmmRegister dst,double value)3333 void X86_64Assembler::LoadDoubleConstant(XmmRegister dst, double value) {
3334   // TODO: Need to have a code constants table.
3335   int64_t constant = bit_cast<int64_t, double>(value);
3336   pushq(Immediate(High32Bits(constant)));
3337   pushq(Immediate(Low32Bits(constant)));
3338   movsd(dst, Address(CpuRegister(RSP), 0));
3339   addq(CpuRegister(RSP), Immediate(2 * sizeof(intptr_t)));
3340 }
3341 
3342 
Align(int alignment,int offset)3343 void X86_64Assembler::Align(int alignment, int offset) {
3344   CHECK(IsPowerOfTwo(alignment));
3345   // Emit nop instruction until the real position is aligned.
3346   while (((offset + buffer_.GetPosition()) & (alignment-1)) != 0) {
3347     nop();
3348   }
3349 }
3350 
3351 
Bind(Label * label)3352 void X86_64Assembler::Bind(Label* label) {
3353   int bound = buffer_.Size();
3354   CHECK(!label->IsBound());  // Labels can only be bound once.
3355   while (label->IsLinked()) {
3356     int position = label->LinkPosition();
3357     int next = buffer_.Load<int32_t>(position);
3358     buffer_.Store<int32_t>(position, bound - (position + 4));
3359     label->position_ = next;
3360   }
3361   label->BindTo(bound);
3362 }
3363 
3364 
Bind(NearLabel * label)3365 void X86_64Assembler::Bind(NearLabel* label) {
3366   int bound = buffer_.Size();
3367   CHECK(!label->IsBound());  // Labels can only be bound once.
3368   while (label->IsLinked()) {
3369     int position = label->LinkPosition();
3370     uint8_t delta = buffer_.Load<uint8_t>(position);
3371     int offset = bound - (position + 1);
3372     CHECK(IsInt<8>(offset));
3373     buffer_.Store<int8_t>(position, offset);
3374     label->position_ = delta != 0u ? label->position_ - delta : 0;
3375   }
3376   label->BindTo(bound);
3377 }
3378 
3379 
EmitOperand(uint8_t reg_or_opcode,const Operand & operand)3380 void X86_64Assembler::EmitOperand(uint8_t reg_or_opcode, const Operand& operand) {
3381   CHECK_GE(reg_or_opcode, 0);
3382   CHECK_LT(reg_or_opcode, 8);
3383   const int length = operand.length_;
3384   CHECK_GT(length, 0);
3385   // Emit the ModRM byte updated with the given reg value.
3386   CHECK_EQ(operand.encoding_[0] & 0x38, 0);
3387   EmitUint8(operand.encoding_[0] + (reg_or_opcode << 3));
3388   // Emit the rest of the encoded operand.
3389   for (int i = 1; i < length; i++) {
3390     EmitUint8(operand.encoding_[i]);
3391   }
3392   AssemblerFixup* fixup = operand.GetFixup();
3393   if (fixup != nullptr) {
3394     EmitFixup(fixup);
3395   }
3396 }
3397 
3398 
EmitImmediate(const Immediate & imm,bool is_16_op)3399 void X86_64Assembler::EmitImmediate(const Immediate& imm, bool is_16_op) {
3400   if (is_16_op) {
3401     EmitUint8(imm.value() & 0xFF);
3402     EmitUint8(imm.value() >> 8);
3403   } else if (imm.is_int32()) {
3404     EmitInt32(static_cast<int32_t>(imm.value()));
3405   } else {
3406     EmitInt64(imm.value());
3407   }
3408 }
3409 
3410 
EmitComplex(uint8_t reg_or_opcode,const Operand & operand,const Immediate & immediate,bool is_16_op)3411 void X86_64Assembler::EmitComplex(uint8_t reg_or_opcode,
3412                                   const Operand& operand,
3413                                   const Immediate& immediate,
3414                                   bool is_16_op) {
3415   CHECK_GE(reg_or_opcode, 0);
3416   CHECK_LT(reg_or_opcode, 8);
3417   if (immediate.is_int8()) {
3418     // Use sign-extended 8-bit immediate.
3419     EmitUint8(0x83);
3420     EmitOperand(reg_or_opcode, operand);
3421     EmitUint8(immediate.value() & 0xFF);
3422   } else if (operand.IsRegister(CpuRegister(RAX))) {
3423     // Use short form if the destination is eax.
3424     EmitUint8(0x05 + (reg_or_opcode << 3));
3425     EmitImmediate(immediate, is_16_op);
3426   } else {
3427     EmitUint8(0x81);
3428     EmitOperand(reg_or_opcode, operand);
3429     EmitImmediate(immediate, is_16_op);
3430   }
3431 }
3432 
3433 
EmitLabel(Label * label,int instruction_size)3434 void X86_64Assembler::EmitLabel(Label* label, int instruction_size) {
3435   if (label->IsBound()) {
3436     int offset = label->Position() - buffer_.Size();
3437     CHECK_LE(offset, 0);
3438     EmitInt32(offset - instruction_size);
3439   } else {
3440     EmitLabelLink(label);
3441   }
3442 }
3443 
3444 
EmitLabelLink(Label * label)3445 void X86_64Assembler::EmitLabelLink(Label* label) {
3446   CHECK(!label->IsBound());
3447   int position = buffer_.Size();
3448   EmitInt32(label->position_);
3449   label->LinkTo(position);
3450 }
3451 
3452 
EmitLabelLink(NearLabel * label)3453 void X86_64Assembler::EmitLabelLink(NearLabel* label) {
3454   CHECK(!label->IsBound());
3455   int position = buffer_.Size();
3456   if (label->IsLinked()) {
3457     // Save the delta in the byte that we have to play with.
3458     uint32_t delta = position - label->LinkPosition();
3459     CHECK(IsUint<8>(delta));
3460     EmitUint8(delta & 0xFF);
3461   } else {
3462     EmitUint8(0);
3463   }
3464   label->LinkTo(position);
3465 }
3466 
3467 
EmitGenericShift(bool wide,int reg_or_opcode,CpuRegister reg,const Immediate & imm)3468 void X86_64Assembler::EmitGenericShift(bool wide,
3469                                        int reg_or_opcode,
3470                                        CpuRegister reg,
3471                                        const Immediate& imm) {
3472   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
3473   CHECK(imm.is_int8());
3474   if (wide) {
3475     EmitRex64(reg);
3476   } else {
3477     EmitOptionalRex32(reg);
3478   }
3479   if (imm.value() == 1) {
3480     EmitUint8(0xD1);
3481     EmitOperand(reg_or_opcode, Operand(reg));
3482   } else {
3483     EmitUint8(0xC1);
3484     EmitOperand(reg_or_opcode, Operand(reg));
3485     EmitUint8(imm.value() & 0xFF);
3486   }
3487 }
3488 
3489 
EmitGenericShift(bool wide,int reg_or_opcode,CpuRegister operand,CpuRegister shifter)3490 void X86_64Assembler::EmitGenericShift(bool wide,
3491                                        int reg_or_opcode,
3492                                        CpuRegister operand,
3493                                        CpuRegister shifter) {
3494   AssemblerBuffer::EnsureCapacity ensured(&buffer_);
3495   CHECK_EQ(shifter.AsRegister(), RCX);
3496   if (wide) {
3497     EmitRex64(operand);
3498   } else {
3499     EmitOptionalRex32(operand);
3500   }
3501   EmitUint8(0xD3);
3502   EmitOperand(reg_or_opcode, Operand(operand));
3503 }
3504 
EmitOptionalRex(bool force,bool w,bool r,bool x,bool b)3505 void X86_64Assembler::EmitOptionalRex(bool force, bool w, bool r, bool x, bool b) {
3506   // REX.WRXB
3507   // W - 64-bit operand
3508   // R - MODRM.reg
3509   // X - SIB.index
3510   // B - MODRM.rm/SIB.base
3511   uint8_t rex = force ? 0x40 : 0;
3512   if (w) {
3513     rex |= 0x48;  // REX.W000
3514   }
3515   if (r) {
3516     rex |= 0x44;  // REX.0R00
3517   }
3518   if (x) {
3519     rex |= 0x42;  // REX.00X0
3520   }
3521   if (b) {
3522     rex |= 0x41;  // REX.000B
3523   }
3524   if (rex != 0) {
3525     EmitUint8(rex);
3526   }
3527 }
3528 
EmitOptionalRex32(CpuRegister reg)3529 void X86_64Assembler::EmitOptionalRex32(CpuRegister reg) {
3530   EmitOptionalRex(false, false, false, false, reg.NeedsRex());
3531 }
3532 
EmitOptionalRex32(CpuRegister dst,CpuRegister src)3533 void X86_64Assembler::EmitOptionalRex32(CpuRegister dst, CpuRegister src) {
3534   EmitOptionalRex(false, false, dst.NeedsRex(), false, src.NeedsRex());
3535 }
3536 
EmitOptionalRex32(XmmRegister dst,XmmRegister src)3537 void X86_64Assembler::EmitOptionalRex32(XmmRegister dst, XmmRegister src) {
3538   EmitOptionalRex(false, false, dst.NeedsRex(), false, src.NeedsRex());
3539 }
3540 
EmitOptionalRex32(CpuRegister dst,XmmRegister src)3541 void X86_64Assembler::EmitOptionalRex32(CpuRegister dst, XmmRegister src) {
3542   EmitOptionalRex(false, false, dst.NeedsRex(), false, src.NeedsRex());
3543 }
3544 
EmitOptionalRex32(XmmRegister dst,CpuRegister src)3545 void X86_64Assembler::EmitOptionalRex32(XmmRegister dst, CpuRegister src) {
3546   EmitOptionalRex(false, false, dst.NeedsRex(), false, src.NeedsRex());
3547 }
3548 
EmitOptionalRex32(const Operand & operand)3549 void X86_64Assembler::EmitOptionalRex32(const Operand& operand) {
3550   uint8_t rex = operand.rex();
3551   if (rex != 0) {
3552     EmitUint8(rex);
3553   }
3554 }
3555 
EmitOptionalRex32(CpuRegister dst,const Operand & operand)3556 void X86_64Assembler::EmitOptionalRex32(CpuRegister dst, const Operand& operand) {
3557   uint8_t rex = operand.rex();
3558   if (dst.NeedsRex()) {
3559     rex |= 0x44;  // REX.0R00
3560   }
3561   if (rex != 0) {
3562     EmitUint8(rex);
3563   }
3564 }
3565 
EmitOptionalRex32(XmmRegister dst,const Operand & operand)3566 void X86_64Assembler::EmitOptionalRex32(XmmRegister dst, const Operand& operand) {
3567   uint8_t rex = operand.rex();
3568   if (dst.NeedsRex()) {
3569     rex |= 0x44;  // REX.0R00
3570   }
3571   if (rex != 0) {
3572     EmitUint8(rex);
3573   }
3574 }
3575 
EmitRex64()3576 void X86_64Assembler::EmitRex64() {
3577   EmitOptionalRex(false, true, false, false, false);
3578 }
3579 
EmitRex64(CpuRegister reg)3580 void X86_64Assembler::EmitRex64(CpuRegister reg) {
3581   EmitOptionalRex(false, true, false, false, reg.NeedsRex());
3582 }
3583 
EmitRex64(const Operand & operand)3584 void X86_64Assembler::EmitRex64(const Operand& operand) {
3585   uint8_t rex = operand.rex();
3586   rex |= 0x48;  // REX.W000
3587   EmitUint8(rex);
3588 }
3589 
EmitRex64(CpuRegister dst,CpuRegister src)3590 void X86_64Assembler::EmitRex64(CpuRegister dst, CpuRegister src) {
3591   EmitOptionalRex(false, true, dst.NeedsRex(), false, src.NeedsRex());
3592 }
3593 
EmitRex64(XmmRegister dst,CpuRegister src)3594 void X86_64Assembler::EmitRex64(XmmRegister dst, CpuRegister src) {
3595   EmitOptionalRex(false, true, dst.NeedsRex(), false, src.NeedsRex());
3596 }
3597 
EmitRex64(CpuRegister dst,XmmRegister src)3598 void X86_64Assembler::EmitRex64(CpuRegister dst, XmmRegister src) {
3599   EmitOptionalRex(false, true, dst.NeedsRex(), false, src.NeedsRex());
3600 }
3601 
EmitRex64(CpuRegister dst,const Operand & operand)3602 void X86_64Assembler::EmitRex64(CpuRegister dst, const Operand& operand) {
3603   uint8_t rex = 0x48 | operand.rex();  // REX.W000
3604   if (dst.NeedsRex()) {
3605     rex |= 0x44;  // REX.0R00
3606   }
3607   EmitUint8(rex);
3608 }
3609 
EmitRex64(XmmRegister dst,const Operand & operand)3610 void X86_64Assembler::EmitRex64(XmmRegister dst, const Operand& operand) {
3611   uint8_t rex = 0x48 | operand.rex();  // REX.W000
3612   if (dst.NeedsRex()) {
3613     rex |= 0x44;  // REX.0R00
3614   }
3615   EmitUint8(rex);
3616 }
3617 
EmitOptionalByteRegNormalizingRex32(CpuRegister dst,CpuRegister src)3618 void X86_64Assembler::EmitOptionalByteRegNormalizingRex32(CpuRegister dst, CpuRegister src) {
3619   // For src, SPL, BPL, SIL, DIL need the rex prefix.
3620   bool force = src.AsRegister() > 3;
3621   EmitOptionalRex(force, false, dst.NeedsRex(), false, src.NeedsRex());
3622 }
3623 
EmitOptionalByteRegNormalizingRex32(CpuRegister dst,const Operand & operand)3624 void X86_64Assembler::EmitOptionalByteRegNormalizingRex32(CpuRegister dst, const Operand& operand) {
3625   uint8_t rex = operand.rex();
3626   // For dst, SPL, BPL, SIL, DIL need the rex prefix.
3627   bool force = dst.AsRegister() > 3;
3628   if (force) {
3629     rex |= 0x40;  // REX.0000
3630   }
3631   if (dst.NeedsRex()) {
3632     rex |= 0x44;  // REX.0R00
3633   }
3634   if (rex != 0) {
3635     EmitUint8(rex);
3636   }
3637 }
3638 
AddConstantArea()3639 void X86_64Assembler::AddConstantArea() {
3640   ArrayRef<const int32_t> area = constant_area_.GetBuffer();
3641   for (size_t i = 0, e = area.size(); i < e; i++) {
3642     AssemblerBuffer::EnsureCapacity ensured(&buffer_);
3643     EmitInt32(area[i]);
3644   }
3645 }
3646 
AppendInt32(int32_t v)3647 size_t ConstantArea::AppendInt32(int32_t v) {
3648   size_t result = buffer_.size() * elem_size_;
3649   buffer_.push_back(v);
3650   return result;
3651 }
3652 
AddInt32(int32_t v)3653 size_t ConstantArea::AddInt32(int32_t v) {
3654   // Look for an existing match.
3655   for (size_t i = 0, e = buffer_.size(); i < e; i++) {
3656     if (v == buffer_[i]) {
3657       return i * elem_size_;
3658     }
3659   }
3660 
3661   // Didn't match anything.
3662   return AppendInt32(v);
3663 }
3664 
AddInt64(int64_t v)3665 size_t ConstantArea::AddInt64(int64_t v) {
3666   int32_t v_low = v;
3667   int32_t v_high = v >> 32;
3668   if (buffer_.size() > 1) {
3669     // Ensure we don't pass the end of the buffer.
3670     for (size_t i = 0, e = buffer_.size() - 1; i < e; i++) {
3671       if (v_low == buffer_[i] && v_high == buffer_[i + 1]) {
3672         return i * elem_size_;
3673       }
3674     }
3675   }
3676 
3677   // Didn't match anything.
3678   size_t result = buffer_.size() * elem_size_;
3679   buffer_.push_back(v_low);
3680   buffer_.push_back(v_high);
3681   return result;
3682 }
3683 
AddDouble(double v)3684 size_t ConstantArea::AddDouble(double v) {
3685   // Treat the value as a 64-bit integer value.
3686   return AddInt64(bit_cast<int64_t, double>(v));
3687 }
3688 
AddFloat(float v)3689 size_t ConstantArea::AddFloat(float v) {
3690   // Treat the value as a 32-bit integer value.
3691   return AddInt32(bit_cast<int32_t, float>(v));
3692 }
3693 
3694 }  // namespace x86_64
3695 }  // namespace art
3696