1 /*
2 * Copyright (C) 2014 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 #include "assembler_x86_64.h"
18
19 #include "base/casts.h"
20 #include "entrypoints/quick/quick_entrypoints.h"
21 #include "memory_region.h"
22 #include "thread.h"
23
24 namespace art {
25 namespace x86_64 {
26
operator <<(std::ostream & os,const CpuRegister & reg)27 std::ostream& operator<<(std::ostream& os, const CpuRegister& reg) {
28 return os << reg.AsRegister();
29 }
30
operator <<(std::ostream & os,const XmmRegister & reg)31 std::ostream& operator<<(std::ostream& os, const XmmRegister& reg) {
32 return os << reg.AsFloatRegister();
33 }
34
operator <<(std::ostream & os,const X87Register & reg)35 std::ostream& operator<<(std::ostream& os, const X87Register& reg) {
36 return os << "ST" << static_cast<int>(reg);
37 }
38
call(CpuRegister reg)39 void X86_64Assembler::call(CpuRegister reg) {
40 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
41 EmitOptionalRex32(reg);
42 EmitUint8(0xFF);
43 EmitRegisterOperand(2, reg.LowBits());
44 }
45
46
call(const Address & address)47 void X86_64Assembler::call(const Address& address) {
48 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
49 EmitOptionalRex32(address);
50 EmitUint8(0xFF);
51 EmitOperand(2, address);
52 }
53
54
call(Label * label)55 void X86_64Assembler::call(Label* label) {
56 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
57 EmitUint8(0xE8);
58 static const int kSize = 5;
59 // Offset by one because we already have emitted the opcode.
60 EmitLabel(label, kSize - 1);
61 }
62
pushq(CpuRegister reg)63 void X86_64Assembler::pushq(CpuRegister reg) {
64 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
65 EmitOptionalRex32(reg);
66 EmitUint8(0x50 + reg.LowBits());
67 }
68
69
pushq(const Address & address)70 void X86_64Assembler::pushq(const Address& address) {
71 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
72 EmitOptionalRex32(address);
73 EmitUint8(0xFF);
74 EmitOperand(6, address);
75 }
76
77
pushq(const Immediate & imm)78 void X86_64Assembler::pushq(const Immediate& imm) {
79 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
80 CHECK(imm.is_int32()); // pushq only supports 32b immediate.
81 if (imm.is_int8()) {
82 EmitUint8(0x6A);
83 EmitUint8(imm.value() & 0xFF);
84 } else {
85 EmitUint8(0x68);
86 EmitImmediate(imm);
87 }
88 }
89
90
popq(CpuRegister reg)91 void X86_64Assembler::popq(CpuRegister reg) {
92 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
93 EmitOptionalRex32(reg);
94 EmitUint8(0x58 + reg.LowBits());
95 }
96
97
popq(const Address & address)98 void X86_64Assembler::popq(const Address& address) {
99 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
100 EmitOptionalRex32(address);
101 EmitUint8(0x8F);
102 EmitOperand(0, address);
103 }
104
105
movq(CpuRegister dst,const Immediate & imm)106 void X86_64Assembler::movq(CpuRegister dst, const Immediate& imm) {
107 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
108 if (imm.is_int32()) {
109 // 32 bit. Note: sign-extends.
110 EmitRex64(dst);
111 EmitUint8(0xC7);
112 EmitRegisterOperand(0, dst.LowBits());
113 EmitInt32(static_cast<int32_t>(imm.value()));
114 } else {
115 EmitRex64(dst);
116 EmitUint8(0xB8 + dst.LowBits());
117 EmitInt64(imm.value());
118 }
119 }
120
121
movl(CpuRegister dst,const Immediate & imm)122 void X86_64Assembler::movl(CpuRegister dst, const Immediate& imm) {
123 CHECK(imm.is_int32());
124 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
125 EmitOptionalRex32(dst);
126 EmitUint8(0xB8 + dst.LowBits());
127 EmitImmediate(imm);
128 }
129
130
movq(const Address & dst,const Immediate & imm)131 void X86_64Assembler::movq(const Address& dst, const Immediate& imm) {
132 CHECK(imm.is_int32());
133 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
134 EmitRex64(dst);
135 EmitUint8(0xC7);
136 EmitOperand(0, dst);
137 EmitImmediate(imm);
138 }
139
140
movq(CpuRegister dst,CpuRegister src)141 void X86_64Assembler::movq(CpuRegister dst, CpuRegister src) {
142 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
143 // 0x89 is movq r/m64 <- r64, with op1 in r/m and op2 in reg: so reverse EmitRex64
144 EmitRex64(src, dst);
145 EmitUint8(0x89);
146 EmitRegisterOperand(src.LowBits(), dst.LowBits());
147 }
148
149
movl(CpuRegister dst,CpuRegister src)150 void X86_64Assembler::movl(CpuRegister dst, CpuRegister src) {
151 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
152 EmitOptionalRex32(dst, src);
153 EmitUint8(0x8B);
154 EmitRegisterOperand(dst.LowBits(), src.LowBits());
155 }
156
157
movq(CpuRegister dst,const Address & src)158 void X86_64Assembler::movq(CpuRegister dst, const Address& src) {
159 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
160 EmitRex64(dst, src);
161 EmitUint8(0x8B);
162 EmitOperand(dst.LowBits(), src);
163 }
164
165
movl(CpuRegister dst,const Address & src)166 void X86_64Assembler::movl(CpuRegister dst, const Address& src) {
167 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
168 EmitOptionalRex32(dst, src);
169 EmitUint8(0x8B);
170 EmitOperand(dst.LowBits(), src);
171 }
172
173
movq(const Address & dst,CpuRegister src)174 void X86_64Assembler::movq(const Address& dst, CpuRegister src) {
175 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
176 EmitRex64(src, dst);
177 EmitUint8(0x89);
178 EmitOperand(src.LowBits(), dst);
179 }
180
181
movl(const Address & dst,CpuRegister src)182 void X86_64Assembler::movl(const Address& dst, CpuRegister src) {
183 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
184 EmitOptionalRex32(src, dst);
185 EmitUint8(0x89);
186 EmitOperand(src.LowBits(), dst);
187 }
188
movl(const Address & dst,const Immediate & imm)189 void X86_64Assembler::movl(const Address& dst, const Immediate& imm) {
190 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
191 EmitOptionalRex32(dst);
192 EmitUint8(0xC7);
193 EmitOperand(0, dst);
194 EmitImmediate(imm);
195 }
196
197
cmov(Condition c,CpuRegister dst,CpuRegister src)198 void X86_64Assembler::cmov(Condition c, CpuRegister dst, CpuRegister src) {
199 cmov(c, dst, src, true);
200 }
201
cmov(Condition c,CpuRegister dst,CpuRegister src,bool is64bit)202 void X86_64Assembler::cmov(Condition c, CpuRegister dst, CpuRegister src, bool is64bit) {
203 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
204 EmitOptionalRex(false, is64bit, dst.NeedsRex(), false, src.NeedsRex());
205 EmitUint8(0x0F);
206 EmitUint8(0x40 + c);
207 EmitRegisterOperand(dst.LowBits(), src.LowBits());
208 }
209
210
movzxb(CpuRegister dst,CpuRegister src)211 void X86_64Assembler::movzxb(CpuRegister dst, CpuRegister src) {
212 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
213 EmitOptionalByteRegNormalizingRex32(dst, src);
214 EmitUint8(0x0F);
215 EmitUint8(0xB6);
216 EmitRegisterOperand(dst.LowBits(), src.LowBits());
217 }
218
219
movzxb(CpuRegister dst,const Address & src)220 void X86_64Assembler::movzxb(CpuRegister dst, const Address& src) {
221 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
222 // Byte register is only in the source register form, so we don't use
223 // EmitOptionalByteRegNormalizingRex32(dst, src);
224 EmitOptionalRex32(dst, src);
225 EmitUint8(0x0F);
226 EmitUint8(0xB6);
227 EmitOperand(dst.LowBits(), src);
228 }
229
230
movsxb(CpuRegister dst,CpuRegister src)231 void X86_64Assembler::movsxb(CpuRegister dst, CpuRegister src) {
232 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
233 EmitOptionalByteRegNormalizingRex32(dst, src);
234 EmitUint8(0x0F);
235 EmitUint8(0xBE);
236 EmitRegisterOperand(dst.LowBits(), src.LowBits());
237 }
238
239
movsxb(CpuRegister dst,const Address & src)240 void X86_64Assembler::movsxb(CpuRegister dst, const Address& src) {
241 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
242 // Byte register is only in the source register form, so we don't use
243 // EmitOptionalByteRegNormalizingRex32(dst, src);
244 EmitOptionalRex32(dst, src);
245 EmitUint8(0x0F);
246 EmitUint8(0xBE);
247 EmitOperand(dst.LowBits(), src);
248 }
249
250
movb(CpuRegister,const Address &)251 void X86_64Assembler::movb(CpuRegister /*dst*/, const Address& /*src*/) {
252 LOG(FATAL) << "Use movzxb or movsxb instead.";
253 }
254
255
movb(const Address & dst,CpuRegister src)256 void X86_64Assembler::movb(const Address& dst, CpuRegister src) {
257 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
258 EmitOptionalByteRegNormalizingRex32(src, dst);
259 EmitUint8(0x88);
260 EmitOperand(src.LowBits(), dst);
261 }
262
263
movb(const Address & dst,const Immediate & imm)264 void X86_64Assembler::movb(const Address& dst, const Immediate& imm) {
265 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
266 EmitOptionalRex32(dst);
267 EmitUint8(0xC6);
268 EmitOperand(Register::RAX, dst);
269 CHECK(imm.is_int8());
270 EmitUint8(imm.value() & 0xFF);
271 }
272
273
movzxw(CpuRegister dst,CpuRegister src)274 void X86_64Assembler::movzxw(CpuRegister dst, CpuRegister src) {
275 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
276 EmitOptionalRex32(dst, src);
277 EmitUint8(0x0F);
278 EmitUint8(0xB7);
279 EmitRegisterOperand(dst.LowBits(), src.LowBits());
280 }
281
282
movzxw(CpuRegister dst,const Address & src)283 void X86_64Assembler::movzxw(CpuRegister dst, const Address& src) {
284 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
285 EmitOptionalRex32(dst, src);
286 EmitUint8(0x0F);
287 EmitUint8(0xB7);
288 EmitOperand(dst.LowBits(), src);
289 }
290
291
movsxw(CpuRegister dst,CpuRegister src)292 void X86_64Assembler::movsxw(CpuRegister dst, CpuRegister src) {
293 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
294 EmitOptionalRex32(dst, src);
295 EmitUint8(0x0F);
296 EmitUint8(0xBF);
297 EmitRegisterOperand(dst.LowBits(), src.LowBits());
298 }
299
300
movsxw(CpuRegister dst,const Address & src)301 void X86_64Assembler::movsxw(CpuRegister dst, const Address& src) {
302 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
303 EmitOptionalRex32(dst, src);
304 EmitUint8(0x0F);
305 EmitUint8(0xBF);
306 EmitOperand(dst.LowBits(), src);
307 }
308
309
movw(CpuRegister,const Address &)310 void X86_64Assembler::movw(CpuRegister /*dst*/, const Address& /*src*/) {
311 LOG(FATAL) << "Use movzxw or movsxw instead.";
312 }
313
314
movw(const Address & dst,CpuRegister src)315 void X86_64Assembler::movw(const Address& dst, CpuRegister src) {
316 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
317 EmitOperandSizeOverride();
318 EmitOptionalRex32(src, dst);
319 EmitUint8(0x89);
320 EmitOperand(src.LowBits(), dst);
321 }
322
323
movw(const Address & dst,const Immediate & imm)324 void X86_64Assembler::movw(const Address& dst, const Immediate& imm) {
325 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
326 EmitOperandSizeOverride();
327 EmitOptionalRex32(dst);
328 EmitUint8(0xC7);
329 EmitOperand(Register::RAX, dst);
330 CHECK(imm.is_uint16() || imm.is_int16());
331 EmitUint8(imm.value() & 0xFF);
332 EmitUint8(imm.value() >> 8);
333 }
334
335
leaq(CpuRegister dst,const Address & src)336 void X86_64Assembler::leaq(CpuRegister dst, const Address& src) {
337 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
338 EmitRex64(dst, src);
339 EmitUint8(0x8D);
340 EmitOperand(dst.LowBits(), src);
341 }
342
343
leal(CpuRegister dst,const Address & src)344 void X86_64Assembler::leal(CpuRegister dst, const Address& src) {
345 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
346 EmitOptionalRex32(dst, src);
347 EmitUint8(0x8D);
348 EmitOperand(dst.LowBits(), src);
349 }
350
351
movaps(XmmRegister dst,XmmRegister src)352 void X86_64Assembler::movaps(XmmRegister dst, XmmRegister src) {
353 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
354 EmitOptionalRex32(dst, src);
355 EmitUint8(0x0F);
356 EmitUint8(0x28);
357 EmitXmmRegisterOperand(dst.LowBits(), src);
358 }
359
360
movss(XmmRegister dst,const Address & src)361 void X86_64Assembler::movss(XmmRegister dst, const Address& src) {
362 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
363 EmitUint8(0xF3);
364 EmitOptionalRex32(dst, src);
365 EmitUint8(0x0F);
366 EmitUint8(0x10);
367 EmitOperand(dst.LowBits(), src);
368 }
369
370
movss(const Address & dst,XmmRegister src)371 void X86_64Assembler::movss(const Address& dst, XmmRegister src) {
372 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
373 EmitUint8(0xF3);
374 EmitOptionalRex32(src, dst);
375 EmitUint8(0x0F);
376 EmitUint8(0x11);
377 EmitOperand(src.LowBits(), dst);
378 }
379
380
movss(XmmRegister dst,XmmRegister src)381 void X86_64Assembler::movss(XmmRegister dst, XmmRegister src) {
382 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
383 EmitUint8(0xF3);
384 EmitOptionalRex32(src, dst); // Movss is MR encoding instead of the usual RM.
385 EmitUint8(0x0F);
386 EmitUint8(0x11);
387 EmitXmmRegisterOperand(src.LowBits(), dst);
388 }
389
390
movsxd(CpuRegister dst,CpuRegister src)391 void X86_64Assembler::movsxd(CpuRegister dst, CpuRegister src) {
392 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
393 EmitRex64(dst, src);
394 EmitUint8(0x63);
395 EmitRegisterOperand(dst.LowBits(), src.LowBits());
396 }
397
398
movsxd(CpuRegister dst,const Address & src)399 void X86_64Assembler::movsxd(CpuRegister dst, const Address& src) {
400 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
401 EmitRex64(dst, src);
402 EmitUint8(0x63);
403 EmitOperand(dst.LowBits(), src);
404 }
405
406
movd(XmmRegister dst,CpuRegister src)407 void X86_64Assembler::movd(XmmRegister dst, CpuRegister src) {
408 movd(dst, src, true);
409 }
410
movd(CpuRegister dst,XmmRegister src)411 void X86_64Assembler::movd(CpuRegister dst, XmmRegister src) {
412 movd(dst, src, true);
413 }
414
movd(XmmRegister dst,CpuRegister src,bool is64bit)415 void X86_64Assembler::movd(XmmRegister dst, CpuRegister src, bool is64bit) {
416 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
417 EmitUint8(0x66);
418 EmitOptionalRex(false, is64bit, dst.NeedsRex(), false, src.NeedsRex());
419 EmitUint8(0x0F);
420 EmitUint8(0x6E);
421 EmitOperand(dst.LowBits(), Operand(src));
422 }
423
movd(CpuRegister dst,XmmRegister src,bool is64bit)424 void X86_64Assembler::movd(CpuRegister dst, XmmRegister src, bool is64bit) {
425 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
426 EmitUint8(0x66);
427 EmitOptionalRex(false, is64bit, src.NeedsRex(), false, dst.NeedsRex());
428 EmitUint8(0x0F);
429 EmitUint8(0x7E);
430 EmitOperand(src.LowBits(), Operand(dst));
431 }
432
433
addss(XmmRegister dst,XmmRegister src)434 void X86_64Assembler::addss(XmmRegister dst, XmmRegister src) {
435 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
436 EmitUint8(0xF3);
437 EmitOptionalRex32(dst, src);
438 EmitUint8(0x0F);
439 EmitUint8(0x58);
440 EmitXmmRegisterOperand(dst.LowBits(), src);
441 }
442
443
addss(XmmRegister dst,const Address & src)444 void X86_64Assembler::addss(XmmRegister dst, const Address& src) {
445 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
446 EmitUint8(0xF3);
447 EmitOptionalRex32(dst, src);
448 EmitUint8(0x0F);
449 EmitUint8(0x58);
450 EmitOperand(dst.LowBits(), src);
451 }
452
453
subss(XmmRegister dst,XmmRegister src)454 void X86_64Assembler::subss(XmmRegister dst, XmmRegister src) {
455 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
456 EmitUint8(0xF3);
457 EmitOptionalRex32(dst, src);
458 EmitUint8(0x0F);
459 EmitUint8(0x5C);
460 EmitXmmRegisterOperand(dst.LowBits(), src);
461 }
462
463
subss(XmmRegister dst,const Address & src)464 void X86_64Assembler::subss(XmmRegister dst, const Address& src) {
465 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
466 EmitUint8(0xF3);
467 EmitOptionalRex32(dst, src);
468 EmitUint8(0x0F);
469 EmitUint8(0x5C);
470 EmitOperand(dst.LowBits(), src);
471 }
472
473
mulss(XmmRegister dst,XmmRegister src)474 void X86_64Assembler::mulss(XmmRegister dst, XmmRegister src) {
475 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
476 EmitUint8(0xF3);
477 EmitOptionalRex32(dst, src);
478 EmitUint8(0x0F);
479 EmitUint8(0x59);
480 EmitXmmRegisterOperand(dst.LowBits(), src);
481 }
482
483
mulss(XmmRegister dst,const Address & src)484 void X86_64Assembler::mulss(XmmRegister dst, const Address& src) {
485 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
486 EmitUint8(0xF3);
487 EmitOptionalRex32(dst, src);
488 EmitUint8(0x0F);
489 EmitUint8(0x59);
490 EmitOperand(dst.LowBits(), src);
491 }
492
493
divss(XmmRegister dst,XmmRegister src)494 void X86_64Assembler::divss(XmmRegister dst, XmmRegister src) {
495 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
496 EmitUint8(0xF3);
497 EmitOptionalRex32(dst, src);
498 EmitUint8(0x0F);
499 EmitUint8(0x5E);
500 EmitXmmRegisterOperand(dst.LowBits(), src);
501 }
502
503
divss(XmmRegister dst,const Address & src)504 void X86_64Assembler::divss(XmmRegister dst, const Address& src) {
505 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
506 EmitUint8(0xF3);
507 EmitOptionalRex32(dst, src);
508 EmitUint8(0x0F);
509 EmitUint8(0x5E);
510 EmitOperand(dst.LowBits(), src);
511 }
512
513
flds(const Address & src)514 void X86_64Assembler::flds(const Address& src) {
515 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
516 EmitUint8(0xD9);
517 EmitOperand(0, src);
518 }
519
520
fsts(const Address & dst)521 void X86_64Assembler::fsts(const Address& dst) {
522 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
523 EmitUint8(0xD9);
524 EmitOperand(2, dst);
525 }
526
527
fstps(const Address & dst)528 void X86_64Assembler::fstps(const Address& dst) {
529 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
530 EmitUint8(0xD9);
531 EmitOperand(3, dst);
532 }
533
534
movsd(XmmRegister dst,const Address & src)535 void X86_64Assembler::movsd(XmmRegister dst, const Address& src) {
536 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
537 EmitUint8(0xF2);
538 EmitOptionalRex32(dst, src);
539 EmitUint8(0x0F);
540 EmitUint8(0x10);
541 EmitOperand(dst.LowBits(), src);
542 }
543
544
movsd(const Address & dst,XmmRegister src)545 void X86_64Assembler::movsd(const Address& dst, XmmRegister src) {
546 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
547 EmitUint8(0xF2);
548 EmitOptionalRex32(src, dst);
549 EmitUint8(0x0F);
550 EmitUint8(0x11);
551 EmitOperand(src.LowBits(), dst);
552 }
553
554
movsd(XmmRegister dst,XmmRegister src)555 void X86_64Assembler::movsd(XmmRegister dst, XmmRegister src) {
556 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
557 EmitUint8(0xF2);
558 EmitOptionalRex32(src, dst); // Movsd is MR encoding instead of the usual RM.
559 EmitUint8(0x0F);
560 EmitUint8(0x11);
561 EmitXmmRegisterOperand(src.LowBits(), dst);
562 }
563
564
addsd(XmmRegister dst,XmmRegister src)565 void X86_64Assembler::addsd(XmmRegister dst, XmmRegister src) {
566 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
567 EmitUint8(0xF2);
568 EmitOptionalRex32(dst, src);
569 EmitUint8(0x0F);
570 EmitUint8(0x58);
571 EmitXmmRegisterOperand(dst.LowBits(), src);
572 }
573
574
addsd(XmmRegister dst,const Address & src)575 void X86_64Assembler::addsd(XmmRegister dst, const Address& src) {
576 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
577 EmitUint8(0xF2);
578 EmitOptionalRex32(dst, src);
579 EmitUint8(0x0F);
580 EmitUint8(0x58);
581 EmitOperand(dst.LowBits(), src);
582 }
583
584
subsd(XmmRegister dst,XmmRegister src)585 void X86_64Assembler::subsd(XmmRegister dst, XmmRegister src) {
586 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
587 EmitUint8(0xF2);
588 EmitOptionalRex32(dst, src);
589 EmitUint8(0x0F);
590 EmitUint8(0x5C);
591 EmitXmmRegisterOperand(dst.LowBits(), src);
592 }
593
594
subsd(XmmRegister dst,const Address & src)595 void X86_64Assembler::subsd(XmmRegister dst, const Address& src) {
596 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
597 EmitUint8(0xF2);
598 EmitOptionalRex32(dst, src);
599 EmitUint8(0x0F);
600 EmitUint8(0x5C);
601 EmitOperand(dst.LowBits(), src);
602 }
603
604
mulsd(XmmRegister dst,XmmRegister src)605 void X86_64Assembler::mulsd(XmmRegister dst, XmmRegister src) {
606 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
607 EmitUint8(0xF2);
608 EmitOptionalRex32(dst, src);
609 EmitUint8(0x0F);
610 EmitUint8(0x59);
611 EmitXmmRegisterOperand(dst.LowBits(), src);
612 }
613
614
mulsd(XmmRegister dst,const Address & src)615 void X86_64Assembler::mulsd(XmmRegister dst, const Address& src) {
616 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
617 EmitUint8(0xF2);
618 EmitOptionalRex32(dst, src);
619 EmitUint8(0x0F);
620 EmitUint8(0x59);
621 EmitOperand(dst.LowBits(), src);
622 }
623
624
divsd(XmmRegister dst,XmmRegister src)625 void X86_64Assembler::divsd(XmmRegister dst, XmmRegister src) {
626 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
627 EmitUint8(0xF2);
628 EmitOptionalRex32(dst, src);
629 EmitUint8(0x0F);
630 EmitUint8(0x5E);
631 EmitXmmRegisterOperand(dst.LowBits(), src);
632 }
633
634
divsd(XmmRegister dst,const Address & src)635 void X86_64Assembler::divsd(XmmRegister dst, const Address& src) {
636 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
637 EmitUint8(0xF2);
638 EmitOptionalRex32(dst, src);
639 EmitUint8(0x0F);
640 EmitUint8(0x5E);
641 EmitOperand(dst.LowBits(), src);
642 }
643
644
cvtsi2ss(XmmRegister dst,CpuRegister src)645 void X86_64Assembler::cvtsi2ss(XmmRegister dst, CpuRegister src) {
646 cvtsi2ss(dst, src, false);
647 }
648
649
cvtsi2ss(XmmRegister dst,CpuRegister src,bool is64bit)650 void X86_64Assembler::cvtsi2ss(XmmRegister dst, CpuRegister src, bool is64bit) {
651 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
652 EmitUint8(0xF3);
653 if (is64bit) {
654 // Emit a REX.W prefix if the operand size is 64 bits.
655 EmitRex64(dst, src);
656 } else {
657 EmitOptionalRex32(dst, src);
658 }
659 EmitUint8(0x0F);
660 EmitUint8(0x2A);
661 EmitOperand(dst.LowBits(), Operand(src));
662 }
663
664
cvtsi2ss(XmmRegister dst,const Address & src,bool is64bit)665 void X86_64Assembler::cvtsi2ss(XmmRegister dst, const Address& src, bool is64bit) {
666 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
667 EmitUint8(0xF3);
668 if (is64bit) {
669 // Emit a REX.W prefix if the operand size is 64 bits.
670 EmitRex64(dst, src);
671 } else {
672 EmitOptionalRex32(dst, src);
673 }
674 EmitUint8(0x0F);
675 EmitUint8(0x2A);
676 EmitOperand(dst.LowBits(), src);
677 }
678
679
cvtsi2sd(XmmRegister dst,CpuRegister src)680 void X86_64Assembler::cvtsi2sd(XmmRegister dst, CpuRegister src) {
681 cvtsi2sd(dst, src, false);
682 }
683
684
cvtsi2sd(XmmRegister dst,CpuRegister src,bool is64bit)685 void X86_64Assembler::cvtsi2sd(XmmRegister dst, CpuRegister src, bool is64bit) {
686 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
687 EmitUint8(0xF2);
688 if (is64bit) {
689 // Emit a REX.W prefix if the operand size is 64 bits.
690 EmitRex64(dst, src);
691 } else {
692 EmitOptionalRex32(dst, src);
693 }
694 EmitUint8(0x0F);
695 EmitUint8(0x2A);
696 EmitOperand(dst.LowBits(), Operand(src));
697 }
698
699
cvtsi2sd(XmmRegister dst,const Address & src,bool is64bit)700 void X86_64Assembler::cvtsi2sd(XmmRegister dst, const Address& src, bool is64bit) {
701 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
702 EmitUint8(0xF2);
703 if (is64bit) {
704 // Emit a REX.W prefix if the operand size is 64 bits.
705 EmitRex64(dst, src);
706 } else {
707 EmitOptionalRex32(dst, src);
708 }
709 EmitUint8(0x0F);
710 EmitUint8(0x2A);
711 EmitOperand(dst.LowBits(), src);
712 }
713
714
cvtss2si(CpuRegister dst,XmmRegister src)715 void X86_64Assembler::cvtss2si(CpuRegister dst, XmmRegister src) {
716 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
717 EmitUint8(0xF3);
718 EmitOptionalRex32(dst, src);
719 EmitUint8(0x0F);
720 EmitUint8(0x2D);
721 EmitXmmRegisterOperand(dst.LowBits(), src);
722 }
723
724
cvtss2sd(XmmRegister dst,XmmRegister src)725 void X86_64Assembler::cvtss2sd(XmmRegister dst, XmmRegister src) {
726 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
727 EmitUint8(0xF3);
728 EmitOptionalRex32(dst, src);
729 EmitUint8(0x0F);
730 EmitUint8(0x5A);
731 EmitXmmRegisterOperand(dst.LowBits(), src);
732 }
733
734
cvtss2sd(XmmRegister dst,const Address & src)735 void X86_64Assembler::cvtss2sd(XmmRegister dst, const Address& src) {
736 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
737 EmitUint8(0xF3);
738 EmitOptionalRex32(dst, src);
739 EmitUint8(0x0F);
740 EmitUint8(0x5A);
741 EmitOperand(dst.LowBits(), src);
742 }
743
744
cvtsd2si(CpuRegister dst,XmmRegister src)745 void X86_64Assembler::cvtsd2si(CpuRegister dst, XmmRegister src) {
746 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
747 EmitUint8(0xF2);
748 EmitOptionalRex32(dst, src);
749 EmitUint8(0x0F);
750 EmitUint8(0x2D);
751 EmitXmmRegisterOperand(dst.LowBits(), src);
752 }
753
754
cvttss2si(CpuRegister dst,XmmRegister src)755 void X86_64Assembler::cvttss2si(CpuRegister dst, XmmRegister src) {
756 cvttss2si(dst, src, false);
757 }
758
759
cvttss2si(CpuRegister dst,XmmRegister src,bool is64bit)760 void X86_64Assembler::cvttss2si(CpuRegister dst, XmmRegister src, bool is64bit) {
761 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
762 EmitUint8(0xF3);
763 if (is64bit) {
764 // Emit a REX.W prefix if the operand size is 64 bits.
765 EmitRex64(dst, src);
766 } else {
767 EmitOptionalRex32(dst, src);
768 }
769 EmitUint8(0x0F);
770 EmitUint8(0x2C);
771 EmitXmmRegisterOperand(dst.LowBits(), src);
772 }
773
774
cvttsd2si(CpuRegister dst,XmmRegister src)775 void X86_64Assembler::cvttsd2si(CpuRegister dst, XmmRegister src) {
776 cvttsd2si(dst, src, false);
777 }
778
779
cvttsd2si(CpuRegister dst,XmmRegister src,bool is64bit)780 void X86_64Assembler::cvttsd2si(CpuRegister dst, XmmRegister src, bool is64bit) {
781 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
782 EmitUint8(0xF2);
783 if (is64bit) {
784 // Emit a REX.W prefix if the operand size is 64 bits.
785 EmitRex64(dst, src);
786 } else {
787 EmitOptionalRex32(dst, src);
788 }
789 EmitUint8(0x0F);
790 EmitUint8(0x2C);
791 EmitXmmRegisterOperand(dst.LowBits(), src);
792 }
793
794
cvtsd2ss(XmmRegister dst,XmmRegister src)795 void X86_64Assembler::cvtsd2ss(XmmRegister dst, XmmRegister src) {
796 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
797 EmitUint8(0xF2);
798 EmitOptionalRex32(dst, src);
799 EmitUint8(0x0F);
800 EmitUint8(0x5A);
801 EmitXmmRegisterOperand(dst.LowBits(), src);
802 }
803
804
cvtsd2ss(XmmRegister dst,const Address & src)805 void X86_64Assembler::cvtsd2ss(XmmRegister dst, const Address& src) {
806 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
807 EmitUint8(0xF2);
808 EmitOptionalRex32(dst, src);
809 EmitUint8(0x0F);
810 EmitUint8(0x5A);
811 EmitOperand(dst.LowBits(), src);
812 }
813
814
cvtdq2pd(XmmRegister dst,XmmRegister src)815 void X86_64Assembler::cvtdq2pd(XmmRegister dst, XmmRegister src) {
816 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
817 EmitUint8(0xF3);
818 EmitOptionalRex32(dst, src);
819 EmitUint8(0x0F);
820 EmitUint8(0xE6);
821 EmitXmmRegisterOperand(dst.LowBits(), src);
822 }
823
824
comiss(XmmRegister a,XmmRegister b)825 void X86_64Assembler::comiss(XmmRegister a, XmmRegister b) {
826 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
827 EmitOptionalRex32(a, b);
828 EmitUint8(0x0F);
829 EmitUint8(0x2F);
830 EmitXmmRegisterOperand(a.LowBits(), b);
831 }
832
833
comiss(XmmRegister a,const Address & b)834 void X86_64Assembler::comiss(XmmRegister a, const Address& b) {
835 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
836 EmitOptionalRex32(a, b);
837 EmitUint8(0x0F);
838 EmitUint8(0x2F);
839 EmitOperand(a.LowBits(), b);
840 }
841
842
comisd(XmmRegister a,XmmRegister b)843 void X86_64Assembler::comisd(XmmRegister a, XmmRegister b) {
844 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
845 EmitUint8(0x66);
846 EmitOptionalRex32(a, b);
847 EmitUint8(0x0F);
848 EmitUint8(0x2F);
849 EmitXmmRegisterOperand(a.LowBits(), b);
850 }
851
852
comisd(XmmRegister a,const Address & b)853 void X86_64Assembler::comisd(XmmRegister a, const Address& b) {
854 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
855 EmitUint8(0x66);
856 EmitOptionalRex32(a, b);
857 EmitUint8(0x0F);
858 EmitUint8(0x2F);
859 EmitOperand(a.LowBits(), b);
860 }
861
862
ucomiss(XmmRegister a,XmmRegister b)863 void X86_64Assembler::ucomiss(XmmRegister a, XmmRegister b) {
864 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
865 EmitOptionalRex32(a, b);
866 EmitUint8(0x0F);
867 EmitUint8(0x2E);
868 EmitXmmRegisterOperand(a.LowBits(), b);
869 }
870
871
ucomiss(XmmRegister a,const Address & b)872 void X86_64Assembler::ucomiss(XmmRegister a, const Address& b) {
873 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
874 EmitOptionalRex32(a, b);
875 EmitUint8(0x0F);
876 EmitUint8(0x2E);
877 EmitOperand(a.LowBits(), b);
878 }
879
880
ucomisd(XmmRegister a,XmmRegister b)881 void X86_64Assembler::ucomisd(XmmRegister a, XmmRegister b) {
882 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
883 EmitUint8(0x66);
884 EmitOptionalRex32(a, b);
885 EmitUint8(0x0F);
886 EmitUint8(0x2E);
887 EmitXmmRegisterOperand(a.LowBits(), b);
888 }
889
890
ucomisd(XmmRegister a,const Address & b)891 void X86_64Assembler::ucomisd(XmmRegister a, const Address& b) {
892 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
893 EmitUint8(0x66);
894 EmitOptionalRex32(a, b);
895 EmitUint8(0x0F);
896 EmitUint8(0x2E);
897 EmitOperand(a.LowBits(), b);
898 }
899
900
roundsd(XmmRegister dst,XmmRegister src,const Immediate & imm)901 void X86_64Assembler::roundsd(XmmRegister dst, XmmRegister src, const Immediate& imm) {
902 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
903 EmitUint8(0x66);
904 EmitOptionalRex32(dst, src);
905 EmitUint8(0x0F);
906 EmitUint8(0x3A);
907 EmitUint8(0x0B);
908 EmitXmmRegisterOperand(dst.LowBits(), src);
909 EmitUint8(imm.value());
910 }
911
912
roundss(XmmRegister dst,XmmRegister src,const Immediate & imm)913 void X86_64Assembler::roundss(XmmRegister dst, XmmRegister src, const Immediate& imm) {
914 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
915 EmitUint8(0x66);
916 EmitOptionalRex32(dst, src);
917 EmitUint8(0x0F);
918 EmitUint8(0x3A);
919 EmitUint8(0x0A);
920 EmitXmmRegisterOperand(dst.LowBits(), src);
921 EmitUint8(imm.value());
922 }
923
924
sqrtsd(XmmRegister dst,XmmRegister src)925 void X86_64Assembler::sqrtsd(XmmRegister dst, XmmRegister src) {
926 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
927 EmitUint8(0xF2);
928 EmitOptionalRex32(dst, src);
929 EmitUint8(0x0F);
930 EmitUint8(0x51);
931 EmitXmmRegisterOperand(dst.LowBits(), src);
932 }
933
934
sqrtss(XmmRegister dst,XmmRegister src)935 void X86_64Assembler::sqrtss(XmmRegister dst, XmmRegister src) {
936 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
937 EmitUint8(0xF3);
938 EmitOptionalRex32(dst, src);
939 EmitUint8(0x0F);
940 EmitUint8(0x51);
941 EmitXmmRegisterOperand(dst.LowBits(), src);
942 }
943
944
xorpd(XmmRegister dst,const Address & src)945 void X86_64Assembler::xorpd(XmmRegister dst, const Address& src) {
946 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
947 EmitUint8(0x66);
948 EmitOptionalRex32(dst, src);
949 EmitUint8(0x0F);
950 EmitUint8(0x57);
951 EmitOperand(dst.LowBits(), src);
952 }
953
954
xorpd(XmmRegister dst,XmmRegister src)955 void X86_64Assembler::xorpd(XmmRegister dst, XmmRegister src) {
956 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
957 EmitUint8(0x66);
958 EmitOptionalRex32(dst, src);
959 EmitUint8(0x0F);
960 EmitUint8(0x57);
961 EmitXmmRegisterOperand(dst.LowBits(), src);
962 }
963
964
xorps(XmmRegister dst,const Address & src)965 void X86_64Assembler::xorps(XmmRegister dst, const Address& src) {
966 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
967 EmitOptionalRex32(dst, src);
968 EmitUint8(0x0F);
969 EmitUint8(0x57);
970 EmitOperand(dst.LowBits(), src);
971 }
972
973
xorps(XmmRegister dst,XmmRegister src)974 void X86_64Assembler::xorps(XmmRegister dst, XmmRegister src) {
975 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
976 EmitOptionalRex32(dst, src);
977 EmitUint8(0x0F);
978 EmitUint8(0x57);
979 EmitXmmRegisterOperand(dst.LowBits(), src);
980 }
981
982
andpd(XmmRegister dst,const Address & src)983 void X86_64Assembler::andpd(XmmRegister dst, const Address& src) {
984 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
985 EmitUint8(0x66);
986 EmitOptionalRex32(dst, src);
987 EmitUint8(0x0F);
988 EmitUint8(0x54);
989 EmitOperand(dst.LowBits(), src);
990 }
991
andpd(XmmRegister dst,XmmRegister src)992 void X86_64Assembler::andpd(XmmRegister dst, XmmRegister src) {
993 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
994 EmitUint8(0x66);
995 EmitOptionalRex32(dst, src);
996 EmitUint8(0x0F);
997 EmitUint8(0x54);
998 EmitXmmRegisterOperand(dst.LowBits(), src);
999 }
1000
andps(XmmRegister dst,XmmRegister src)1001 void X86_64Assembler::andps(XmmRegister dst, XmmRegister src) {
1002 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1003 EmitOptionalRex32(dst, src);
1004 EmitUint8(0x0F);
1005 EmitUint8(0x54);
1006 EmitXmmRegisterOperand(dst.LowBits(), src);
1007 }
1008
orpd(XmmRegister dst,XmmRegister src)1009 void X86_64Assembler::orpd(XmmRegister dst, XmmRegister src) {
1010 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1011 EmitUint8(0x66);
1012 EmitOptionalRex32(dst, src);
1013 EmitUint8(0x0F);
1014 EmitUint8(0x56);
1015 EmitXmmRegisterOperand(dst.LowBits(), src);
1016 }
1017
orps(XmmRegister dst,XmmRegister src)1018 void X86_64Assembler::orps(XmmRegister dst, XmmRegister src) {
1019 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1020 EmitOptionalRex32(dst, src);
1021 EmitUint8(0x0F);
1022 EmitUint8(0x56);
1023 EmitXmmRegisterOperand(dst.LowBits(), src);
1024 }
1025
fldl(const Address & src)1026 void X86_64Assembler::fldl(const Address& src) {
1027 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1028 EmitUint8(0xDD);
1029 EmitOperand(0, src);
1030 }
1031
1032
fstl(const Address & dst)1033 void X86_64Assembler::fstl(const Address& dst) {
1034 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1035 EmitUint8(0xDD);
1036 EmitOperand(2, dst);
1037 }
1038
1039
fstpl(const Address & dst)1040 void X86_64Assembler::fstpl(const Address& dst) {
1041 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1042 EmitUint8(0xDD);
1043 EmitOperand(3, dst);
1044 }
1045
1046
fstsw()1047 void X86_64Assembler::fstsw() {
1048 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1049 EmitUint8(0x9B);
1050 EmitUint8(0xDF);
1051 EmitUint8(0xE0);
1052 }
1053
1054
fnstcw(const Address & dst)1055 void X86_64Assembler::fnstcw(const Address& dst) {
1056 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1057 EmitUint8(0xD9);
1058 EmitOperand(7, dst);
1059 }
1060
1061
fldcw(const Address & src)1062 void X86_64Assembler::fldcw(const Address& src) {
1063 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1064 EmitUint8(0xD9);
1065 EmitOperand(5, src);
1066 }
1067
1068
fistpl(const Address & dst)1069 void X86_64Assembler::fistpl(const Address& dst) {
1070 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1071 EmitUint8(0xDF);
1072 EmitOperand(7, dst);
1073 }
1074
1075
fistps(const Address & dst)1076 void X86_64Assembler::fistps(const Address& dst) {
1077 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1078 EmitUint8(0xDB);
1079 EmitOperand(3, dst);
1080 }
1081
1082
fildl(const Address & src)1083 void X86_64Assembler::fildl(const Address& src) {
1084 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1085 EmitUint8(0xDF);
1086 EmitOperand(5, src);
1087 }
1088
1089
filds(const Address & src)1090 void X86_64Assembler::filds(const Address& src) {
1091 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1092 EmitUint8(0xDB);
1093 EmitOperand(0, src);
1094 }
1095
1096
fincstp()1097 void X86_64Assembler::fincstp() {
1098 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1099 EmitUint8(0xD9);
1100 EmitUint8(0xF7);
1101 }
1102
1103
ffree(const Immediate & index)1104 void X86_64Assembler::ffree(const Immediate& index) {
1105 CHECK_LT(index.value(), 7);
1106 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1107 EmitUint8(0xDD);
1108 EmitUint8(0xC0 + index.value());
1109 }
1110
1111
fsin()1112 void X86_64Assembler::fsin() {
1113 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1114 EmitUint8(0xD9);
1115 EmitUint8(0xFE);
1116 }
1117
1118
fcos()1119 void X86_64Assembler::fcos() {
1120 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1121 EmitUint8(0xD9);
1122 EmitUint8(0xFF);
1123 }
1124
1125
fptan()1126 void X86_64Assembler::fptan() {
1127 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1128 EmitUint8(0xD9);
1129 EmitUint8(0xF2);
1130 }
1131
fucompp()1132 void X86_64Assembler::fucompp() {
1133 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1134 EmitUint8(0xDA);
1135 EmitUint8(0xE9);
1136 }
1137
1138
fprem()1139 void X86_64Assembler::fprem() {
1140 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1141 EmitUint8(0xD9);
1142 EmitUint8(0xF8);
1143 }
1144
1145
xchgl(CpuRegister dst,CpuRegister src)1146 void X86_64Assembler::xchgl(CpuRegister dst, CpuRegister src) {
1147 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1148 // There is a short version for rax.
1149 // It's a bit awkward, as CpuRegister has a const field, so assignment and thus swapping doesn't
1150 // work.
1151 const bool src_rax = src.AsRegister() == RAX;
1152 const bool dst_rax = dst.AsRegister() == RAX;
1153 if (src_rax || dst_rax) {
1154 EmitOptionalRex32(src_rax ? dst : src);
1155 EmitUint8(0x90 + (src_rax ? dst.LowBits() : src.LowBits()));
1156 return;
1157 }
1158
1159 // General case.
1160 EmitOptionalRex32(src, dst);
1161 EmitUint8(0x87);
1162 EmitRegisterOperand(src.LowBits(), dst.LowBits());
1163 }
1164
1165
xchgq(CpuRegister dst,CpuRegister src)1166 void X86_64Assembler::xchgq(CpuRegister dst, CpuRegister src) {
1167 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1168 // There is a short version for rax.
1169 // It's a bit awkward, as CpuRegister has a const field, so assignment and thus swapping doesn't
1170 // work.
1171 const bool src_rax = src.AsRegister() == RAX;
1172 const bool dst_rax = dst.AsRegister() == RAX;
1173 if (src_rax || dst_rax) {
1174 // If src == target, emit a nop instead.
1175 if (src_rax && dst_rax) {
1176 EmitUint8(0x90);
1177 } else {
1178 EmitRex64(src_rax ? dst : src);
1179 EmitUint8(0x90 + (src_rax ? dst.LowBits() : src.LowBits()));
1180 }
1181 return;
1182 }
1183
1184 // General case.
1185 EmitRex64(src, dst);
1186 EmitUint8(0x87);
1187 EmitRegisterOperand(src.LowBits(), dst.LowBits());
1188 }
1189
1190
xchgl(CpuRegister reg,const Address & address)1191 void X86_64Assembler::xchgl(CpuRegister reg, const Address& address) {
1192 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1193 EmitOptionalRex32(reg, address);
1194 EmitUint8(0x87);
1195 EmitOperand(reg.LowBits(), address);
1196 }
1197
1198
cmpw(const Address & address,const Immediate & imm)1199 void X86_64Assembler::cmpw(const Address& address, const Immediate& imm) {
1200 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1201 EmitOptionalRex32(address);
1202 EmitUint8(0x66);
1203 EmitComplex(7, address, imm);
1204 }
1205
1206
cmpl(CpuRegister reg,const Immediate & imm)1207 void X86_64Assembler::cmpl(CpuRegister reg, const Immediate& imm) {
1208 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1209 EmitOptionalRex32(reg);
1210 EmitComplex(7, Operand(reg), imm);
1211 }
1212
1213
cmpl(CpuRegister reg0,CpuRegister reg1)1214 void X86_64Assembler::cmpl(CpuRegister reg0, CpuRegister reg1) {
1215 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1216 EmitOptionalRex32(reg0, reg1);
1217 EmitUint8(0x3B);
1218 EmitOperand(reg0.LowBits(), Operand(reg1));
1219 }
1220
1221
cmpl(CpuRegister reg,const Address & address)1222 void X86_64Assembler::cmpl(CpuRegister reg, const Address& address) {
1223 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1224 EmitOptionalRex32(reg, address);
1225 EmitUint8(0x3B);
1226 EmitOperand(reg.LowBits(), address);
1227 }
1228
1229
cmpl(const Address & address,CpuRegister reg)1230 void X86_64Assembler::cmpl(const Address& address, CpuRegister reg) {
1231 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1232 EmitOptionalRex32(reg, address);
1233 EmitUint8(0x39);
1234 EmitOperand(reg.LowBits(), address);
1235 }
1236
1237
cmpl(const Address & address,const Immediate & imm)1238 void X86_64Assembler::cmpl(const Address& address, const Immediate& imm) {
1239 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1240 EmitOptionalRex32(address);
1241 EmitComplex(7, address, imm);
1242 }
1243
1244
cmpq(CpuRegister reg0,CpuRegister reg1)1245 void X86_64Assembler::cmpq(CpuRegister reg0, CpuRegister reg1) {
1246 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1247 EmitRex64(reg0, reg1);
1248 EmitUint8(0x3B);
1249 EmitOperand(reg0.LowBits(), Operand(reg1));
1250 }
1251
1252
cmpq(CpuRegister reg,const Immediate & imm)1253 void X86_64Assembler::cmpq(CpuRegister reg, const Immediate& imm) {
1254 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1255 CHECK(imm.is_int32()); // cmpq only supports 32b immediate.
1256 EmitRex64(reg);
1257 EmitComplex(7, Operand(reg), imm);
1258 }
1259
1260
cmpq(CpuRegister reg,const Address & address)1261 void X86_64Assembler::cmpq(CpuRegister reg, const Address& address) {
1262 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1263 EmitRex64(reg, address);
1264 EmitUint8(0x3B);
1265 EmitOperand(reg.LowBits(), address);
1266 }
1267
1268
cmpq(const Address & address,const Immediate & imm)1269 void X86_64Assembler::cmpq(const Address& address, const Immediate& imm) {
1270 CHECK(imm.is_int32()); // cmpq only supports 32b immediate.
1271 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1272 EmitRex64(address);
1273 EmitComplex(7, address, imm);
1274 }
1275
1276
addl(CpuRegister dst,CpuRegister src)1277 void X86_64Assembler::addl(CpuRegister dst, CpuRegister src) {
1278 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1279 EmitOptionalRex32(dst, src);
1280 EmitUint8(0x03);
1281 EmitRegisterOperand(dst.LowBits(), src.LowBits());
1282 }
1283
1284
addl(CpuRegister reg,const Address & address)1285 void X86_64Assembler::addl(CpuRegister reg, const Address& address) {
1286 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1287 EmitOptionalRex32(reg, address);
1288 EmitUint8(0x03);
1289 EmitOperand(reg.LowBits(), address);
1290 }
1291
1292
testl(CpuRegister reg1,CpuRegister reg2)1293 void X86_64Assembler::testl(CpuRegister reg1, CpuRegister reg2) {
1294 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1295 EmitOptionalRex32(reg1, reg2);
1296 EmitUint8(0x85);
1297 EmitRegisterOperand(reg1.LowBits(), reg2.LowBits());
1298 }
1299
1300
testl(CpuRegister reg,const Address & address)1301 void X86_64Assembler::testl(CpuRegister reg, const Address& address) {
1302 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1303 EmitOptionalRex32(reg, address);
1304 EmitUint8(0x85);
1305 EmitOperand(reg.LowBits(), address);
1306 }
1307
1308
testl(CpuRegister reg,const Immediate & immediate)1309 void X86_64Assembler::testl(CpuRegister reg, const Immediate& immediate) {
1310 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1311 // For registers that have a byte variant (RAX, RBX, RCX, and RDX)
1312 // we only test the byte CpuRegister to keep the encoding short.
1313 if (immediate.is_uint8() && reg.AsRegister() < 4) {
1314 // Use zero-extended 8-bit immediate.
1315 if (reg.AsRegister() == RAX) {
1316 EmitUint8(0xA8);
1317 } else {
1318 EmitUint8(0xF6);
1319 EmitUint8(0xC0 + reg.AsRegister());
1320 }
1321 EmitUint8(immediate.value() & 0xFF);
1322 } else if (reg.AsRegister() == RAX) {
1323 // Use short form if the destination is RAX.
1324 EmitUint8(0xA9);
1325 EmitImmediate(immediate);
1326 } else {
1327 EmitOptionalRex32(reg);
1328 EmitUint8(0xF7);
1329 EmitOperand(0, Operand(reg));
1330 EmitImmediate(immediate);
1331 }
1332 }
1333
1334
testq(CpuRegister reg1,CpuRegister reg2)1335 void X86_64Assembler::testq(CpuRegister reg1, CpuRegister reg2) {
1336 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1337 EmitRex64(reg1, reg2);
1338 EmitUint8(0x85);
1339 EmitRegisterOperand(reg1.LowBits(), reg2.LowBits());
1340 }
1341
1342
testq(CpuRegister reg,const Address & address)1343 void X86_64Assembler::testq(CpuRegister reg, const Address& address) {
1344 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1345 EmitRex64(reg, address);
1346 EmitUint8(0x85);
1347 EmitOperand(reg.LowBits(), address);
1348 }
1349
1350
andl(CpuRegister dst,CpuRegister src)1351 void X86_64Assembler::andl(CpuRegister dst, CpuRegister src) {
1352 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1353 EmitOptionalRex32(dst, src);
1354 EmitUint8(0x23);
1355 EmitOperand(dst.LowBits(), Operand(src));
1356 }
1357
1358
andl(CpuRegister reg,const Address & address)1359 void X86_64Assembler::andl(CpuRegister reg, const Address& address) {
1360 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1361 EmitOptionalRex32(reg, address);
1362 EmitUint8(0x23);
1363 EmitOperand(reg.LowBits(), address);
1364 }
1365
1366
andl(CpuRegister dst,const Immediate & imm)1367 void X86_64Assembler::andl(CpuRegister dst, const Immediate& imm) {
1368 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1369 EmitOptionalRex32(dst);
1370 EmitComplex(4, Operand(dst), imm);
1371 }
1372
1373
andq(CpuRegister reg,const Immediate & imm)1374 void X86_64Assembler::andq(CpuRegister reg, const Immediate& imm) {
1375 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1376 CHECK(imm.is_int32()); // andq only supports 32b immediate.
1377 EmitRex64(reg);
1378 EmitComplex(4, Operand(reg), imm);
1379 }
1380
1381
andq(CpuRegister dst,CpuRegister src)1382 void X86_64Assembler::andq(CpuRegister dst, CpuRegister src) {
1383 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1384 EmitRex64(dst, src);
1385 EmitUint8(0x23);
1386 EmitOperand(dst.LowBits(), Operand(src));
1387 }
1388
1389
andq(CpuRegister dst,const Address & src)1390 void X86_64Assembler::andq(CpuRegister dst, const Address& src) {
1391 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1392 EmitRex64(dst, src);
1393 EmitUint8(0x23);
1394 EmitOperand(dst.LowBits(), src);
1395 }
1396
1397
orl(CpuRegister dst,CpuRegister src)1398 void X86_64Assembler::orl(CpuRegister dst, CpuRegister src) {
1399 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1400 EmitOptionalRex32(dst, src);
1401 EmitUint8(0x0B);
1402 EmitOperand(dst.LowBits(), Operand(src));
1403 }
1404
1405
orl(CpuRegister reg,const Address & address)1406 void X86_64Assembler::orl(CpuRegister reg, const Address& address) {
1407 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1408 EmitOptionalRex32(reg, address);
1409 EmitUint8(0x0B);
1410 EmitOperand(reg.LowBits(), address);
1411 }
1412
1413
orl(CpuRegister dst,const Immediate & imm)1414 void X86_64Assembler::orl(CpuRegister dst, const Immediate& imm) {
1415 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1416 EmitOptionalRex32(dst);
1417 EmitComplex(1, Operand(dst), imm);
1418 }
1419
1420
orq(CpuRegister dst,const Immediate & imm)1421 void X86_64Assembler::orq(CpuRegister dst, const Immediate& imm) {
1422 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1423 CHECK(imm.is_int32()); // orq only supports 32b immediate.
1424 EmitRex64(dst);
1425 EmitComplex(1, Operand(dst), imm);
1426 }
1427
1428
orq(CpuRegister dst,CpuRegister src)1429 void X86_64Assembler::orq(CpuRegister dst, CpuRegister src) {
1430 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1431 EmitRex64(dst, src);
1432 EmitUint8(0x0B);
1433 EmitOperand(dst.LowBits(), Operand(src));
1434 }
1435
1436
orq(CpuRegister dst,const Address & src)1437 void X86_64Assembler::orq(CpuRegister dst, const Address& src) {
1438 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1439 EmitRex64(dst, src);
1440 EmitUint8(0x0B);
1441 EmitOperand(dst.LowBits(), src);
1442 }
1443
1444
xorl(CpuRegister dst,CpuRegister src)1445 void X86_64Assembler::xorl(CpuRegister dst, CpuRegister src) {
1446 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1447 EmitOptionalRex32(dst, src);
1448 EmitUint8(0x33);
1449 EmitOperand(dst.LowBits(), Operand(src));
1450 }
1451
1452
xorl(CpuRegister reg,const Address & address)1453 void X86_64Assembler::xorl(CpuRegister reg, const Address& address) {
1454 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1455 EmitOptionalRex32(reg, address);
1456 EmitUint8(0x33);
1457 EmitOperand(reg.LowBits(), address);
1458 }
1459
1460
xorl(CpuRegister dst,const Immediate & imm)1461 void X86_64Assembler::xorl(CpuRegister dst, const Immediate& imm) {
1462 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1463 EmitOptionalRex32(dst);
1464 EmitComplex(6, Operand(dst), imm);
1465 }
1466
1467
xorq(CpuRegister dst,CpuRegister src)1468 void X86_64Assembler::xorq(CpuRegister dst, CpuRegister src) {
1469 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1470 EmitRex64(dst, src);
1471 EmitUint8(0x33);
1472 EmitOperand(dst.LowBits(), Operand(src));
1473 }
1474
1475
xorq(CpuRegister dst,const Immediate & imm)1476 void X86_64Assembler::xorq(CpuRegister dst, const Immediate& imm) {
1477 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1478 CHECK(imm.is_int32()); // xorq only supports 32b immediate.
1479 EmitRex64(dst);
1480 EmitComplex(6, Operand(dst), imm);
1481 }
1482
xorq(CpuRegister dst,const Address & src)1483 void X86_64Assembler::xorq(CpuRegister dst, const Address& src) {
1484 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1485 EmitRex64(dst, src);
1486 EmitUint8(0x33);
1487 EmitOperand(dst.LowBits(), src);
1488 }
1489
1490
1491 #if 0
1492 void X86_64Assembler::rex(bool force, bool w, Register* r, Register* x, Register* b) {
1493 // REX.WRXB
1494 // W - 64-bit operand
1495 // R - MODRM.reg
1496 // X - SIB.index
1497 // B - MODRM.rm/SIB.base
1498 uint8_t rex = force ? 0x40 : 0;
1499 if (w) {
1500 rex |= 0x48; // REX.W000
1501 }
1502 if (r != nullptr && *r >= Register::R8 && *r < Register::kNumberOfCpuRegisters) {
1503 rex |= 0x44; // REX.0R00
1504 *r = static_cast<Register>(*r - 8);
1505 }
1506 if (x != nullptr && *x >= Register::R8 && *x < Register::kNumberOfCpuRegisters) {
1507 rex |= 0x42; // REX.00X0
1508 *x = static_cast<Register>(*x - 8);
1509 }
1510 if (b != nullptr && *b >= Register::R8 && *b < Register::kNumberOfCpuRegisters) {
1511 rex |= 0x41; // REX.000B
1512 *b = static_cast<Register>(*b - 8);
1513 }
1514 if (rex != 0) {
1515 EmitUint8(rex);
1516 }
1517 }
1518
1519 void X86_64Assembler::rex_reg_mem(bool force, bool w, Register* dst, const Address& mem) {
1520 // REX.WRXB
1521 // W - 64-bit operand
1522 // R - MODRM.reg
1523 // X - SIB.index
1524 // B - MODRM.rm/SIB.base
1525 uint8_t rex = mem->rex();
1526 if (force) {
1527 rex |= 0x40; // REX.0000
1528 }
1529 if (w) {
1530 rex |= 0x48; // REX.W000
1531 }
1532 if (dst != nullptr && *dst >= Register::R8 && *dst < Register::kNumberOfCpuRegisters) {
1533 rex |= 0x44; // REX.0R00
1534 *dst = static_cast<Register>(*dst - 8);
1535 }
1536 if (rex != 0) {
1537 EmitUint8(rex);
1538 }
1539 }
1540
1541 void rex_mem_reg(bool force, bool w, Address* mem, Register* src);
1542 #endif
1543
addl(CpuRegister reg,const Immediate & imm)1544 void X86_64Assembler::addl(CpuRegister reg, const Immediate& imm) {
1545 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1546 EmitOptionalRex32(reg);
1547 EmitComplex(0, Operand(reg), imm);
1548 }
1549
1550
addq(CpuRegister reg,const Immediate & imm)1551 void X86_64Assembler::addq(CpuRegister reg, const Immediate& imm) {
1552 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1553 CHECK(imm.is_int32()); // addq only supports 32b immediate.
1554 EmitRex64(reg);
1555 EmitComplex(0, Operand(reg), imm);
1556 }
1557
1558
addq(CpuRegister dst,const Address & address)1559 void X86_64Assembler::addq(CpuRegister dst, const Address& address) {
1560 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1561 EmitRex64(dst, address);
1562 EmitUint8(0x03);
1563 EmitOperand(dst.LowBits(), address);
1564 }
1565
1566
addq(CpuRegister dst,CpuRegister src)1567 void X86_64Assembler::addq(CpuRegister dst, CpuRegister src) {
1568 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1569 // 0x01 is addq r/m64 <- r/m64 + r64, with op1 in r/m and op2 in reg: so reverse EmitRex64
1570 EmitRex64(src, dst);
1571 EmitUint8(0x01);
1572 EmitRegisterOperand(src.LowBits(), dst.LowBits());
1573 }
1574
1575
addl(const Address & address,CpuRegister reg)1576 void X86_64Assembler::addl(const Address& address, CpuRegister reg) {
1577 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1578 EmitOptionalRex32(reg, address);
1579 EmitUint8(0x01);
1580 EmitOperand(reg.LowBits(), address);
1581 }
1582
1583
addl(const Address & address,const Immediate & imm)1584 void X86_64Assembler::addl(const Address& address, const Immediate& imm) {
1585 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1586 EmitOptionalRex32(address);
1587 EmitComplex(0, address, imm);
1588 }
1589
1590
subl(CpuRegister dst,CpuRegister src)1591 void X86_64Assembler::subl(CpuRegister dst, CpuRegister src) {
1592 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1593 EmitOptionalRex32(dst, src);
1594 EmitUint8(0x2B);
1595 EmitOperand(dst.LowBits(), Operand(src));
1596 }
1597
1598
subl(CpuRegister reg,const Immediate & imm)1599 void X86_64Assembler::subl(CpuRegister reg, const Immediate& imm) {
1600 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1601 EmitOptionalRex32(reg);
1602 EmitComplex(5, Operand(reg), imm);
1603 }
1604
1605
subq(CpuRegister reg,const Immediate & imm)1606 void X86_64Assembler::subq(CpuRegister reg, const Immediate& imm) {
1607 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1608 CHECK(imm.is_int32()); // subq only supports 32b immediate.
1609 EmitRex64(reg);
1610 EmitComplex(5, Operand(reg), imm);
1611 }
1612
1613
subq(CpuRegister dst,CpuRegister src)1614 void X86_64Assembler::subq(CpuRegister dst, CpuRegister src) {
1615 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1616 EmitRex64(dst, src);
1617 EmitUint8(0x2B);
1618 EmitRegisterOperand(dst.LowBits(), src.LowBits());
1619 }
1620
1621
subq(CpuRegister reg,const Address & address)1622 void X86_64Assembler::subq(CpuRegister reg, const Address& address) {
1623 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1624 EmitRex64(reg, address);
1625 EmitUint8(0x2B);
1626 EmitOperand(reg.LowBits() & 7, address);
1627 }
1628
1629
subl(CpuRegister reg,const Address & address)1630 void X86_64Assembler::subl(CpuRegister reg, const Address& address) {
1631 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1632 EmitOptionalRex32(reg, address);
1633 EmitUint8(0x2B);
1634 EmitOperand(reg.LowBits(), address);
1635 }
1636
1637
cdq()1638 void X86_64Assembler::cdq() {
1639 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1640 EmitUint8(0x99);
1641 }
1642
1643
cqo()1644 void X86_64Assembler::cqo() {
1645 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1646 EmitRex64();
1647 EmitUint8(0x99);
1648 }
1649
1650
idivl(CpuRegister reg)1651 void X86_64Assembler::idivl(CpuRegister reg) {
1652 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1653 EmitOptionalRex32(reg);
1654 EmitUint8(0xF7);
1655 EmitUint8(0xF8 | reg.LowBits());
1656 }
1657
1658
idivq(CpuRegister reg)1659 void X86_64Assembler::idivq(CpuRegister reg) {
1660 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1661 EmitRex64(reg);
1662 EmitUint8(0xF7);
1663 EmitUint8(0xF8 | reg.LowBits());
1664 }
1665
1666
imull(CpuRegister dst,CpuRegister src)1667 void X86_64Assembler::imull(CpuRegister dst, CpuRegister src) {
1668 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1669 EmitOptionalRex32(dst, src);
1670 EmitUint8(0x0F);
1671 EmitUint8(0xAF);
1672 EmitOperand(dst.LowBits(), Operand(src));
1673 }
1674
imull(CpuRegister reg,const Immediate & imm)1675 void X86_64Assembler::imull(CpuRegister reg, const Immediate& imm) {
1676 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1677 CHECK(imm.is_int32()); // imull only supports 32b immediate.
1678
1679 EmitOptionalRex32(reg, reg);
1680
1681 // See whether imm can be represented as a sign-extended 8bit value.
1682 int32_t v32 = static_cast<int32_t>(imm.value());
1683 if (IsInt<8>(v32)) {
1684 // Sign-extension works.
1685 EmitUint8(0x6B);
1686 EmitOperand(reg.LowBits(), Operand(reg));
1687 EmitUint8(static_cast<uint8_t>(v32 & 0xFF));
1688 } else {
1689 // Not representable, use full immediate.
1690 EmitUint8(0x69);
1691 EmitOperand(reg.LowBits(), Operand(reg));
1692 EmitImmediate(imm);
1693 }
1694 }
1695
1696
imull(CpuRegister reg,const Address & address)1697 void X86_64Assembler::imull(CpuRegister reg, const Address& address) {
1698 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1699 EmitOptionalRex32(reg, address);
1700 EmitUint8(0x0F);
1701 EmitUint8(0xAF);
1702 EmitOperand(reg.LowBits(), address);
1703 }
1704
1705
imulq(CpuRegister dst,CpuRegister src)1706 void X86_64Assembler::imulq(CpuRegister dst, CpuRegister src) {
1707 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1708 EmitRex64(dst, src);
1709 EmitUint8(0x0F);
1710 EmitUint8(0xAF);
1711 EmitRegisterOperand(dst.LowBits(), src.LowBits());
1712 }
1713
1714
imulq(CpuRegister reg,const Immediate & imm)1715 void X86_64Assembler::imulq(CpuRegister reg, const Immediate& imm) {
1716 imulq(reg, reg, imm);
1717 }
1718
imulq(CpuRegister dst,CpuRegister reg,const Immediate & imm)1719 void X86_64Assembler::imulq(CpuRegister dst, CpuRegister reg, const Immediate& imm) {
1720 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1721 CHECK(imm.is_int32()); // imulq only supports 32b immediate.
1722
1723 EmitRex64(dst, reg);
1724
1725 // See whether imm can be represented as a sign-extended 8bit value.
1726 int64_t v64 = imm.value();
1727 if (IsInt<8>(v64)) {
1728 // Sign-extension works.
1729 EmitUint8(0x6B);
1730 EmitOperand(dst.LowBits(), Operand(reg));
1731 EmitUint8(static_cast<uint8_t>(v64 & 0xFF));
1732 } else {
1733 // Not representable, use full immediate.
1734 EmitUint8(0x69);
1735 EmitOperand(dst.LowBits(), Operand(reg));
1736 EmitImmediate(imm);
1737 }
1738 }
1739
imulq(CpuRegister reg,const Address & address)1740 void X86_64Assembler::imulq(CpuRegister reg, const Address& address) {
1741 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1742 EmitRex64(reg, address);
1743 EmitUint8(0x0F);
1744 EmitUint8(0xAF);
1745 EmitOperand(reg.LowBits(), address);
1746 }
1747
1748
imull(CpuRegister reg)1749 void X86_64Assembler::imull(CpuRegister reg) {
1750 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1751 EmitOptionalRex32(reg);
1752 EmitUint8(0xF7);
1753 EmitOperand(5, Operand(reg));
1754 }
1755
1756
imulq(CpuRegister reg)1757 void X86_64Assembler::imulq(CpuRegister reg) {
1758 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1759 EmitRex64(reg);
1760 EmitUint8(0xF7);
1761 EmitOperand(5, Operand(reg));
1762 }
1763
1764
imull(const Address & address)1765 void X86_64Assembler::imull(const Address& address) {
1766 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1767 EmitOptionalRex32(address);
1768 EmitUint8(0xF7);
1769 EmitOperand(5, address);
1770 }
1771
1772
mull(CpuRegister reg)1773 void X86_64Assembler::mull(CpuRegister reg) {
1774 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1775 EmitOptionalRex32(reg);
1776 EmitUint8(0xF7);
1777 EmitOperand(4, Operand(reg));
1778 }
1779
1780
mull(const Address & address)1781 void X86_64Assembler::mull(const Address& address) {
1782 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1783 EmitOptionalRex32(address);
1784 EmitUint8(0xF7);
1785 EmitOperand(4, address);
1786 }
1787
1788
shll(CpuRegister reg,const Immediate & imm)1789 void X86_64Assembler::shll(CpuRegister reg, const Immediate& imm) {
1790 EmitGenericShift(false, 4, reg, imm);
1791 }
1792
1793
shlq(CpuRegister reg,const Immediate & imm)1794 void X86_64Assembler::shlq(CpuRegister reg, const Immediate& imm) {
1795 EmitGenericShift(true, 4, reg, imm);
1796 }
1797
1798
shll(CpuRegister operand,CpuRegister shifter)1799 void X86_64Assembler::shll(CpuRegister operand, CpuRegister shifter) {
1800 EmitGenericShift(false, 4, operand, shifter);
1801 }
1802
1803
shlq(CpuRegister operand,CpuRegister shifter)1804 void X86_64Assembler::shlq(CpuRegister operand, CpuRegister shifter) {
1805 EmitGenericShift(true, 4, operand, shifter);
1806 }
1807
1808
shrl(CpuRegister reg,const Immediate & imm)1809 void X86_64Assembler::shrl(CpuRegister reg, const Immediate& imm) {
1810 EmitGenericShift(false, 5, reg, imm);
1811 }
1812
1813
shrq(CpuRegister reg,const Immediate & imm)1814 void X86_64Assembler::shrq(CpuRegister reg, const Immediate& imm) {
1815 EmitGenericShift(true, 5, reg, imm);
1816 }
1817
1818
shrl(CpuRegister operand,CpuRegister shifter)1819 void X86_64Assembler::shrl(CpuRegister operand, CpuRegister shifter) {
1820 EmitGenericShift(false, 5, operand, shifter);
1821 }
1822
1823
shrq(CpuRegister operand,CpuRegister shifter)1824 void X86_64Assembler::shrq(CpuRegister operand, CpuRegister shifter) {
1825 EmitGenericShift(true, 5, operand, shifter);
1826 }
1827
1828
sarl(CpuRegister reg,const Immediate & imm)1829 void X86_64Assembler::sarl(CpuRegister reg, const Immediate& imm) {
1830 EmitGenericShift(false, 7, reg, imm);
1831 }
1832
1833
sarl(CpuRegister operand,CpuRegister shifter)1834 void X86_64Assembler::sarl(CpuRegister operand, CpuRegister shifter) {
1835 EmitGenericShift(false, 7, operand, shifter);
1836 }
1837
1838
sarq(CpuRegister reg,const Immediate & imm)1839 void X86_64Assembler::sarq(CpuRegister reg, const Immediate& imm) {
1840 EmitGenericShift(true, 7, reg, imm);
1841 }
1842
1843
sarq(CpuRegister operand,CpuRegister shifter)1844 void X86_64Assembler::sarq(CpuRegister operand, CpuRegister shifter) {
1845 EmitGenericShift(true, 7, operand, shifter);
1846 }
1847
1848
negl(CpuRegister reg)1849 void X86_64Assembler::negl(CpuRegister reg) {
1850 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1851 EmitOptionalRex32(reg);
1852 EmitUint8(0xF7);
1853 EmitOperand(3, Operand(reg));
1854 }
1855
1856
negq(CpuRegister reg)1857 void X86_64Assembler::negq(CpuRegister reg) {
1858 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1859 EmitRex64(reg);
1860 EmitUint8(0xF7);
1861 EmitOperand(3, Operand(reg));
1862 }
1863
1864
notl(CpuRegister reg)1865 void X86_64Assembler::notl(CpuRegister reg) {
1866 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1867 EmitOptionalRex32(reg);
1868 EmitUint8(0xF7);
1869 EmitUint8(0xD0 | reg.LowBits());
1870 }
1871
1872
notq(CpuRegister reg)1873 void X86_64Assembler::notq(CpuRegister reg) {
1874 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1875 EmitRex64(reg);
1876 EmitUint8(0xF7);
1877 EmitOperand(2, Operand(reg));
1878 }
1879
1880
enter(const Immediate & imm)1881 void X86_64Assembler::enter(const Immediate& imm) {
1882 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1883 EmitUint8(0xC8);
1884 CHECK(imm.is_uint16()) << imm.value();
1885 EmitUint8(imm.value() & 0xFF);
1886 EmitUint8((imm.value() >> 8) & 0xFF);
1887 EmitUint8(0x00);
1888 }
1889
1890
leave()1891 void X86_64Assembler::leave() {
1892 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1893 EmitUint8(0xC9);
1894 }
1895
1896
ret()1897 void X86_64Assembler::ret() {
1898 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1899 EmitUint8(0xC3);
1900 }
1901
1902
ret(const Immediate & imm)1903 void X86_64Assembler::ret(const Immediate& imm) {
1904 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1905 EmitUint8(0xC2);
1906 CHECK(imm.is_uint16());
1907 EmitUint8(imm.value() & 0xFF);
1908 EmitUint8((imm.value() >> 8) & 0xFF);
1909 }
1910
1911
1912
nop()1913 void X86_64Assembler::nop() {
1914 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1915 EmitUint8(0x90);
1916 }
1917
1918
int3()1919 void X86_64Assembler::int3() {
1920 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1921 EmitUint8(0xCC);
1922 }
1923
1924
hlt()1925 void X86_64Assembler::hlt() {
1926 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1927 EmitUint8(0xF4);
1928 }
1929
1930
j(Condition condition,Label * label)1931 void X86_64Assembler::j(Condition condition, Label* label) {
1932 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1933 if (label->IsBound()) {
1934 static const int kShortSize = 2;
1935 static const int kLongSize = 6;
1936 int offset = label->Position() - buffer_.Size();
1937 CHECK_LE(offset, 0);
1938 if (IsInt<8>(offset - kShortSize)) {
1939 EmitUint8(0x70 + condition);
1940 EmitUint8((offset - kShortSize) & 0xFF);
1941 } else {
1942 EmitUint8(0x0F);
1943 EmitUint8(0x80 + condition);
1944 EmitInt32(offset - kLongSize);
1945 }
1946 } else {
1947 EmitUint8(0x0F);
1948 EmitUint8(0x80 + condition);
1949 EmitLabelLink(label);
1950 }
1951 }
1952
1953
jmp(CpuRegister reg)1954 void X86_64Assembler::jmp(CpuRegister reg) {
1955 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1956 EmitOptionalRex32(reg);
1957 EmitUint8(0xFF);
1958 EmitRegisterOperand(4, reg.LowBits());
1959 }
1960
jmp(const Address & address)1961 void X86_64Assembler::jmp(const Address& address) {
1962 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1963 EmitOptionalRex32(address);
1964 EmitUint8(0xFF);
1965 EmitOperand(4, address);
1966 }
1967
jmp(Label * label)1968 void X86_64Assembler::jmp(Label* label) {
1969 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1970 if (label->IsBound()) {
1971 static const int kShortSize = 2;
1972 static const int kLongSize = 5;
1973 int offset = label->Position() - buffer_.Size();
1974 CHECK_LE(offset, 0);
1975 if (IsInt<8>(offset - kShortSize)) {
1976 EmitUint8(0xEB);
1977 EmitUint8((offset - kShortSize) & 0xFF);
1978 } else {
1979 EmitUint8(0xE9);
1980 EmitInt32(offset - kLongSize);
1981 }
1982 } else {
1983 EmitUint8(0xE9);
1984 EmitLabelLink(label);
1985 }
1986 }
1987
1988
lock()1989 X86_64Assembler* X86_64Assembler::lock() {
1990 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1991 EmitUint8(0xF0);
1992 return this;
1993 }
1994
1995
cmpxchgl(const Address & address,CpuRegister reg)1996 void X86_64Assembler::cmpxchgl(const Address& address, CpuRegister reg) {
1997 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1998 EmitOptionalRex32(reg, address);
1999 EmitUint8(0x0F);
2000 EmitUint8(0xB1);
2001 EmitOperand(reg.LowBits(), address);
2002 }
2003
2004
cmpxchgq(const Address & address,CpuRegister reg)2005 void X86_64Assembler::cmpxchgq(const Address& address, CpuRegister reg) {
2006 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
2007 EmitRex64(reg, address);
2008 EmitUint8(0x0F);
2009 EmitUint8(0xB1);
2010 EmitOperand(reg.LowBits(), address);
2011 }
2012
2013
mfence()2014 void X86_64Assembler::mfence() {
2015 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
2016 EmitUint8(0x0F);
2017 EmitUint8(0xAE);
2018 EmitUint8(0xF0);
2019 }
2020
2021
gs()2022 X86_64Assembler* X86_64Assembler::gs() {
2023 // TODO: gs is a prefix and not an instruction
2024 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
2025 EmitUint8(0x65);
2026 return this;
2027 }
2028
2029
AddImmediate(CpuRegister reg,const Immediate & imm)2030 void X86_64Assembler::AddImmediate(CpuRegister reg, const Immediate& imm) {
2031 int value = imm.value();
2032 if (value != 0) {
2033 if (value > 0) {
2034 addl(reg, imm);
2035 } else {
2036 subl(reg, Immediate(value));
2037 }
2038 }
2039 }
2040
2041
setcc(Condition condition,CpuRegister dst)2042 void X86_64Assembler::setcc(Condition condition, CpuRegister dst) {
2043 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
2044 // RSP, RBP, RDI, RSI need rex prefix (else the pattern encodes ah/bh/ch/dh).
2045 if (dst.NeedsRex() || dst.AsRegister() > 3) {
2046 EmitOptionalRex(true, false, false, false, dst.NeedsRex());
2047 }
2048 EmitUint8(0x0F);
2049 EmitUint8(0x90 + condition);
2050 EmitUint8(0xC0 + dst.LowBits());
2051 }
2052
bswapl(CpuRegister dst)2053 void X86_64Assembler::bswapl(CpuRegister dst) {
2054 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
2055 EmitOptionalRex(false, false, false, false, dst.NeedsRex());
2056 EmitUint8(0x0F);
2057 EmitUint8(0xC8 + dst.LowBits());
2058 }
2059
bswapq(CpuRegister dst)2060 void X86_64Assembler::bswapq(CpuRegister dst) {
2061 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
2062 EmitOptionalRex(false, true, false, false, dst.NeedsRex());
2063 EmitUint8(0x0F);
2064 EmitUint8(0xC8 + dst.LowBits());
2065 }
2066
2067
repne_scasw()2068 void X86_64Assembler::repne_scasw() {
2069 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
2070 EmitUint8(0x66);
2071 EmitUint8(0xF2);
2072 EmitUint8(0xAF);
2073 }
2074
2075
LoadDoubleConstant(XmmRegister dst,double value)2076 void X86_64Assembler::LoadDoubleConstant(XmmRegister dst, double value) {
2077 // TODO: Need to have a code constants table.
2078 int64_t constant = bit_cast<int64_t, double>(value);
2079 pushq(Immediate(High32Bits(constant)));
2080 pushq(Immediate(Low32Bits(constant)));
2081 movsd(dst, Address(CpuRegister(RSP), 0));
2082 addq(CpuRegister(RSP), Immediate(2 * sizeof(intptr_t)));
2083 }
2084
2085
Align(int alignment,int offset)2086 void X86_64Assembler::Align(int alignment, int offset) {
2087 CHECK(IsPowerOfTwo(alignment));
2088 // Emit nop instruction until the real position is aligned.
2089 while (((offset + buffer_.GetPosition()) & (alignment-1)) != 0) {
2090 nop();
2091 }
2092 }
2093
2094
Bind(Label * label)2095 void X86_64Assembler::Bind(Label* label) {
2096 int bound = buffer_.Size();
2097 CHECK(!label->IsBound()); // Labels can only be bound once.
2098 while (label->IsLinked()) {
2099 int position = label->LinkPosition();
2100 int next = buffer_.Load<int32_t>(position);
2101 buffer_.Store<int32_t>(position, bound - (position + 4));
2102 label->position_ = next;
2103 }
2104 label->BindTo(bound);
2105 }
2106
2107
EmitOperand(uint8_t reg_or_opcode,const Operand & operand)2108 void X86_64Assembler::EmitOperand(uint8_t reg_or_opcode, const Operand& operand) {
2109 CHECK_GE(reg_or_opcode, 0);
2110 CHECK_LT(reg_or_opcode, 8);
2111 const int length = operand.length_;
2112 CHECK_GT(length, 0);
2113 // Emit the ModRM byte updated with the given reg value.
2114 CHECK_EQ(operand.encoding_[0] & 0x38, 0);
2115 EmitUint8(operand.encoding_[0] + (reg_or_opcode << 3));
2116 // Emit the rest of the encoded operand.
2117 for (int i = 1; i < length; i++) {
2118 EmitUint8(operand.encoding_[i]);
2119 }
2120 AssemblerFixup* fixup = operand.GetFixup();
2121 if (fixup != nullptr) {
2122 EmitFixup(fixup);
2123 }
2124 }
2125
2126
EmitImmediate(const Immediate & imm)2127 void X86_64Assembler::EmitImmediate(const Immediate& imm) {
2128 if (imm.is_int32()) {
2129 EmitInt32(static_cast<int32_t>(imm.value()));
2130 } else {
2131 EmitInt64(imm.value());
2132 }
2133 }
2134
2135
EmitComplex(uint8_t reg_or_opcode,const Operand & operand,const Immediate & immediate)2136 void X86_64Assembler::EmitComplex(uint8_t reg_or_opcode,
2137 const Operand& operand,
2138 const Immediate& immediate) {
2139 CHECK_GE(reg_or_opcode, 0);
2140 CHECK_LT(reg_or_opcode, 8);
2141 if (immediate.is_int8()) {
2142 // Use sign-extended 8-bit immediate.
2143 EmitUint8(0x83);
2144 EmitOperand(reg_or_opcode, operand);
2145 EmitUint8(immediate.value() & 0xFF);
2146 } else if (operand.IsRegister(CpuRegister(RAX))) {
2147 // Use short form if the destination is eax.
2148 EmitUint8(0x05 + (reg_or_opcode << 3));
2149 EmitImmediate(immediate);
2150 } else {
2151 EmitUint8(0x81);
2152 EmitOperand(reg_or_opcode, operand);
2153 EmitImmediate(immediate);
2154 }
2155 }
2156
2157
EmitLabel(Label * label,int instruction_size)2158 void X86_64Assembler::EmitLabel(Label* label, int instruction_size) {
2159 if (label->IsBound()) {
2160 int offset = label->Position() - buffer_.Size();
2161 CHECK_LE(offset, 0);
2162 EmitInt32(offset - instruction_size);
2163 } else {
2164 EmitLabelLink(label);
2165 }
2166 }
2167
2168
EmitLabelLink(Label * label)2169 void X86_64Assembler::EmitLabelLink(Label* label) {
2170 CHECK(!label->IsBound());
2171 int position = buffer_.Size();
2172 EmitInt32(label->position_);
2173 label->LinkTo(position);
2174 }
2175
2176
EmitGenericShift(bool wide,int reg_or_opcode,CpuRegister reg,const Immediate & imm)2177 void X86_64Assembler::EmitGenericShift(bool wide,
2178 int reg_or_opcode,
2179 CpuRegister reg,
2180 const Immediate& imm) {
2181 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
2182 CHECK(imm.is_int8());
2183 if (wide) {
2184 EmitRex64(reg);
2185 } else {
2186 EmitOptionalRex32(reg);
2187 }
2188 if (imm.value() == 1) {
2189 EmitUint8(0xD1);
2190 EmitOperand(reg_or_opcode, Operand(reg));
2191 } else {
2192 EmitUint8(0xC1);
2193 EmitOperand(reg_or_opcode, Operand(reg));
2194 EmitUint8(imm.value() & 0xFF);
2195 }
2196 }
2197
2198
EmitGenericShift(bool wide,int reg_or_opcode,CpuRegister operand,CpuRegister shifter)2199 void X86_64Assembler::EmitGenericShift(bool wide,
2200 int reg_or_opcode,
2201 CpuRegister operand,
2202 CpuRegister shifter) {
2203 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
2204 CHECK_EQ(shifter.AsRegister(), RCX);
2205 if (wide) {
2206 EmitRex64(operand);
2207 } else {
2208 EmitOptionalRex32(operand);
2209 }
2210 EmitUint8(0xD3);
2211 EmitOperand(reg_or_opcode, Operand(operand));
2212 }
2213
EmitOptionalRex(bool force,bool w,bool r,bool x,bool b)2214 void X86_64Assembler::EmitOptionalRex(bool force, bool w, bool r, bool x, bool b) {
2215 // REX.WRXB
2216 // W - 64-bit operand
2217 // R - MODRM.reg
2218 // X - SIB.index
2219 // B - MODRM.rm/SIB.base
2220 uint8_t rex = force ? 0x40 : 0;
2221 if (w) {
2222 rex |= 0x48; // REX.W000
2223 }
2224 if (r) {
2225 rex |= 0x44; // REX.0R00
2226 }
2227 if (x) {
2228 rex |= 0x42; // REX.00X0
2229 }
2230 if (b) {
2231 rex |= 0x41; // REX.000B
2232 }
2233 if (rex != 0) {
2234 EmitUint8(rex);
2235 }
2236 }
2237
EmitOptionalRex32(CpuRegister reg)2238 void X86_64Assembler::EmitOptionalRex32(CpuRegister reg) {
2239 EmitOptionalRex(false, false, false, false, reg.NeedsRex());
2240 }
2241
EmitOptionalRex32(CpuRegister dst,CpuRegister src)2242 void X86_64Assembler::EmitOptionalRex32(CpuRegister dst, CpuRegister src) {
2243 EmitOptionalRex(false, false, dst.NeedsRex(), false, src.NeedsRex());
2244 }
2245
EmitOptionalRex32(XmmRegister dst,XmmRegister src)2246 void X86_64Assembler::EmitOptionalRex32(XmmRegister dst, XmmRegister src) {
2247 EmitOptionalRex(false, false, dst.NeedsRex(), false, src.NeedsRex());
2248 }
2249
EmitOptionalRex32(CpuRegister dst,XmmRegister src)2250 void X86_64Assembler::EmitOptionalRex32(CpuRegister dst, XmmRegister src) {
2251 EmitOptionalRex(false, false, dst.NeedsRex(), false, src.NeedsRex());
2252 }
2253
EmitOptionalRex32(XmmRegister dst,CpuRegister src)2254 void X86_64Assembler::EmitOptionalRex32(XmmRegister dst, CpuRegister src) {
2255 EmitOptionalRex(false, false, dst.NeedsRex(), false, src.NeedsRex());
2256 }
2257
EmitOptionalRex32(const Operand & operand)2258 void X86_64Assembler::EmitOptionalRex32(const Operand& operand) {
2259 uint8_t rex = operand.rex();
2260 if (rex != 0) {
2261 EmitUint8(rex);
2262 }
2263 }
2264
EmitOptionalRex32(CpuRegister dst,const Operand & operand)2265 void X86_64Assembler::EmitOptionalRex32(CpuRegister dst, const Operand& operand) {
2266 uint8_t rex = operand.rex();
2267 if (dst.NeedsRex()) {
2268 rex |= 0x44; // REX.0R00
2269 }
2270 if (rex != 0) {
2271 EmitUint8(rex);
2272 }
2273 }
2274
EmitOptionalRex32(XmmRegister dst,const Operand & operand)2275 void X86_64Assembler::EmitOptionalRex32(XmmRegister dst, const Operand& operand) {
2276 uint8_t rex = operand.rex();
2277 if (dst.NeedsRex()) {
2278 rex |= 0x44; // REX.0R00
2279 }
2280 if (rex != 0) {
2281 EmitUint8(rex);
2282 }
2283 }
2284
EmitRex64()2285 void X86_64Assembler::EmitRex64() {
2286 EmitOptionalRex(false, true, false, false, false);
2287 }
2288
EmitRex64(CpuRegister reg)2289 void X86_64Assembler::EmitRex64(CpuRegister reg) {
2290 EmitOptionalRex(false, true, false, false, reg.NeedsRex());
2291 }
2292
EmitRex64(const Operand & operand)2293 void X86_64Assembler::EmitRex64(const Operand& operand) {
2294 uint8_t rex = operand.rex();
2295 rex |= 0x48; // REX.W000
2296 EmitUint8(rex);
2297 }
2298
EmitRex64(CpuRegister dst,CpuRegister src)2299 void X86_64Assembler::EmitRex64(CpuRegister dst, CpuRegister src) {
2300 EmitOptionalRex(false, true, dst.NeedsRex(), false, src.NeedsRex());
2301 }
2302
EmitRex64(XmmRegister dst,CpuRegister src)2303 void X86_64Assembler::EmitRex64(XmmRegister dst, CpuRegister src) {
2304 EmitOptionalRex(false, true, dst.NeedsRex(), false, src.NeedsRex());
2305 }
2306
EmitRex64(CpuRegister dst,XmmRegister src)2307 void X86_64Assembler::EmitRex64(CpuRegister dst, XmmRegister src) {
2308 EmitOptionalRex(false, true, dst.NeedsRex(), false, src.NeedsRex());
2309 }
2310
EmitRex64(CpuRegister dst,const Operand & operand)2311 void X86_64Assembler::EmitRex64(CpuRegister dst, const Operand& operand) {
2312 uint8_t rex = 0x48 | operand.rex(); // REX.W000
2313 if (dst.NeedsRex()) {
2314 rex |= 0x44; // REX.0R00
2315 }
2316 EmitUint8(rex);
2317 }
2318
EmitRex64(XmmRegister dst,const Operand & operand)2319 void X86_64Assembler::EmitRex64(XmmRegister dst, const Operand& operand) {
2320 uint8_t rex = 0x48 | operand.rex(); // REX.W000
2321 if (dst.NeedsRex()) {
2322 rex |= 0x44; // REX.0R00
2323 }
2324 EmitUint8(rex);
2325 }
2326
EmitOptionalByteRegNormalizingRex32(CpuRegister dst,CpuRegister src)2327 void X86_64Assembler::EmitOptionalByteRegNormalizingRex32(CpuRegister dst, CpuRegister src) {
2328 // For src, SPL, BPL, SIL, DIL need the rex prefix.
2329 bool force = src.AsRegister() > 3;
2330 EmitOptionalRex(force, false, dst.NeedsRex(), false, src.NeedsRex());
2331 }
2332
EmitOptionalByteRegNormalizingRex32(CpuRegister dst,const Operand & operand)2333 void X86_64Assembler::EmitOptionalByteRegNormalizingRex32(CpuRegister dst, const Operand& operand) {
2334 uint8_t rex = operand.rex();
2335 // For dst, SPL, BPL, SIL, DIL need the rex prefix.
2336 bool force = dst.AsRegister() > 3;
2337 if (force) {
2338 rex |= 0x40; // REX.0000
2339 }
2340 if (dst.NeedsRex()) {
2341 rex |= 0x44; // REX.0R00
2342 }
2343 if (rex != 0) {
2344 EmitUint8(rex);
2345 }
2346 }
2347
DWARFReg(Register reg)2348 static dwarf::Reg DWARFReg(Register reg) {
2349 return dwarf::Reg::X86_64Core(static_cast<int>(reg));
2350 }
DWARFReg(FloatRegister reg)2351 static dwarf::Reg DWARFReg(FloatRegister reg) {
2352 return dwarf::Reg::X86_64Fp(static_cast<int>(reg));
2353 }
2354
2355 constexpr size_t kFramePointerSize = 8;
2356
BuildFrame(size_t frame_size,ManagedRegister method_reg,const std::vector<ManagedRegister> & spill_regs,const ManagedRegisterEntrySpills & entry_spills)2357 void X86_64Assembler::BuildFrame(size_t frame_size, ManagedRegister method_reg,
2358 const std::vector<ManagedRegister>& spill_regs,
2359 const ManagedRegisterEntrySpills& entry_spills) {
2360 DCHECK_EQ(buffer_.Size(), 0U); // Nothing emitted yet.
2361 cfi_.SetCurrentCFAOffset(8); // Return address on stack.
2362 CHECK_ALIGNED(frame_size, kStackAlignment);
2363 int gpr_count = 0;
2364 for (int i = spill_regs.size() - 1; i >= 0; --i) {
2365 x86_64::X86_64ManagedRegister spill = spill_regs.at(i).AsX86_64();
2366 if (spill.IsCpuRegister()) {
2367 pushq(spill.AsCpuRegister());
2368 gpr_count++;
2369 cfi_.AdjustCFAOffset(kFramePointerSize);
2370 cfi_.RelOffset(DWARFReg(spill.AsCpuRegister().AsRegister()), 0);
2371 }
2372 }
2373 // return address then method on stack.
2374 int64_t rest_of_frame = static_cast<int64_t>(frame_size)
2375 - (gpr_count * kFramePointerSize)
2376 - kFramePointerSize /*return address*/;
2377 subq(CpuRegister(RSP), Immediate(rest_of_frame));
2378 cfi_.AdjustCFAOffset(rest_of_frame);
2379
2380 // spill xmms
2381 int64_t offset = rest_of_frame;
2382 for (int i = spill_regs.size() - 1; i >= 0; --i) {
2383 x86_64::X86_64ManagedRegister spill = spill_regs.at(i).AsX86_64();
2384 if (spill.IsXmmRegister()) {
2385 offset -= sizeof(double);
2386 movsd(Address(CpuRegister(RSP), offset), spill.AsXmmRegister());
2387 cfi_.RelOffset(DWARFReg(spill.AsXmmRegister().AsFloatRegister()), offset);
2388 }
2389 }
2390
2391 DCHECK_EQ(kX86_64PointerSize, kFramePointerSize);
2392
2393 movq(Address(CpuRegister(RSP), 0), method_reg.AsX86_64().AsCpuRegister());
2394
2395 for (size_t i = 0; i < entry_spills.size(); ++i) {
2396 ManagedRegisterSpill spill = entry_spills.at(i);
2397 if (spill.AsX86_64().IsCpuRegister()) {
2398 if (spill.getSize() == 8) {
2399 movq(Address(CpuRegister(RSP), frame_size + spill.getSpillOffset()),
2400 spill.AsX86_64().AsCpuRegister());
2401 } else {
2402 CHECK_EQ(spill.getSize(), 4);
2403 movl(Address(CpuRegister(RSP), frame_size + spill.getSpillOffset()), spill.AsX86_64().AsCpuRegister());
2404 }
2405 } else {
2406 if (spill.getSize() == 8) {
2407 movsd(Address(CpuRegister(RSP), frame_size + spill.getSpillOffset()), spill.AsX86_64().AsXmmRegister());
2408 } else {
2409 CHECK_EQ(spill.getSize(), 4);
2410 movss(Address(CpuRegister(RSP), frame_size + spill.getSpillOffset()), spill.AsX86_64().AsXmmRegister());
2411 }
2412 }
2413 }
2414 }
2415
RemoveFrame(size_t frame_size,const std::vector<ManagedRegister> & spill_regs)2416 void X86_64Assembler::RemoveFrame(size_t frame_size,
2417 const std::vector<ManagedRegister>& spill_regs) {
2418 CHECK_ALIGNED(frame_size, kStackAlignment);
2419 cfi_.RememberState();
2420 int gpr_count = 0;
2421 // unspill xmms
2422 int64_t offset = static_cast<int64_t>(frame_size) - (spill_regs.size() * kFramePointerSize) - 2 * kFramePointerSize;
2423 for (size_t i = 0; i < spill_regs.size(); ++i) {
2424 x86_64::X86_64ManagedRegister spill = spill_regs.at(i).AsX86_64();
2425 if (spill.IsXmmRegister()) {
2426 offset += sizeof(double);
2427 movsd(spill.AsXmmRegister(), Address(CpuRegister(RSP), offset));
2428 cfi_.Restore(DWARFReg(spill.AsXmmRegister().AsFloatRegister()));
2429 } else {
2430 gpr_count++;
2431 }
2432 }
2433 int adjust = static_cast<int>(frame_size) - (gpr_count * kFramePointerSize) - kFramePointerSize;
2434 addq(CpuRegister(RSP), Immediate(adjust));
2435 cfi_.AdjustCFAOffset(-adjust);
2436 for (size_t i = 0; i < spill_regs.size(); ++i) {
2437 x86_64::X86_64ManagedRegister spill = spill_regs.at(i).AsX86_64();
2438 if (spill.IsCpuRegister()) {
2439 popq(spill.AsCpuRegister());
2440 cfi_.AdjustCFAOffset(-static_cast<int>(kFramePointerSize));
2441 cfi_.Restore(DWARFReg(spill.AsCpuRegister().AsRegister()));
2442 }
2443 }
2444 ret();
2445 // The CFI should be restored for any code that follows the exit block.
2446 cfi_.RestoreState();
2447 cfi_.DefCFAOffset(frame_size);
2448 }
2449
IncreaseFrameSize(size_t adjust)2450 void X86_64Assembler::IncreaseFrameSize(size_t adjust) {
2451 CHECK_ALIGNED(adjust, kStackAlignment);
2452 addq(CpuRegister(RSP), Immediate(-static_cast<int64_t>(adjust)));
2453 cfi_.AdjustCFAOffset(adjust);
2454 }
2455
DecreaseFrameSize(size_t adjust)2456 void X86_64Assembler::DecreaseFrameSize(size_t adjust) {
2457 CHECK_ALIGNED(adjust, kStackAlignment);
2458 addq(CpuRegister(RSP), Immediate(adjust));
2459 cfi_.AdjustCFAOffset(-adjust);
2460 }
2461
Store(FrameOffset offs,ManagedRegister msrc,size_t size)2462 void X86_64Assembler::Store(FrameOffset offs, ManagedRegister msrc, size_t size) {
2463 X86_64ManagedRegister src = msrc.AsX86_64();
2464 if (src.IsNoRegister()) {
2465 CHECK_EQ(0u, size);
2466 } else if (src.IsCpuRegister()) {
2467 if (size == 4) {
2468 CHECK_EQ(4u, size);
2469 movl(Address(CpuRegister(RSP), offs), src.AsCpuRegister());
2470 } else {
2471 CHECK_EQ(8u, size);
2472 movq(Address(CpuRegister(RSP), offs), src.AsCpuRegister());
2473 }
2474 } else if (src.IsRegisterPair()) {
2475 CHECK_EQ(0u, size);
2476 movq(Address(CpuRegister(RSP), offs), src.AsRegisterPairLow());
2477 movq(Address(CpuRegister(RSP), FrameOffset(offs.Int32Value()+4)),
2478 src.AsRegisterPairHigh());
2479 } else if (src.IsX87Register()) {
2480 if (size == 4) {
2481 fstps(Address(CpuRegister(RSP), offs));
2482 } else {
2483 fstpl(Address(CpuRegister(RSP), offs));
2484 }
2485 } else {
2486 CHECK(src.IsXmmRegister());
2487 if (size == 4) {
2488 movss(Address(CpuRegister(RSP), offs), src.AsXmmRegister());
2489 } else {
2490 movsd(Address(CpuRegister(RSP), offs), src.AsXmmRegister());
2491 }
2492 }
2493 }
2494
StoreRef(FrameOffset dest,ManagedRegister msrc)2495 void X86_64Assembler::StoreRef(FrameOffset dest, ManagedRegister msrc) {
2496 X86_64ManagedRegister src = msrc.AsX86_64();
2497 CHECK(src.IsCpuRegister());
2498 movl(Address(CpuRegister(RSP), dest), src.AsCpuRegister());
2499 }
2500
StoreRawPtr(FrameOffset dest,ManagedRegister msrc)2501 void X86_64Assembler::StoreRawPtr(FrameOffset dest, ManagedRegister msrc) {
2502 X86_64ManagedRegister src = msrc.AsX86_64();
2503 CHECK(src.IsCpuRegister());
2504 movq(Address(CpuRegister(RSP), dest), src.AsCpuRegister());
2505 }
2506
StoreImmediateToFrame(FrameOffset dest,uint32_t imm,ManagedRegister)2507 void X86_64Assembler::StoreImmediateToFrame(FrameOffset dest, uint32_t imm,
2508 ManagedRegister) {
2509 movl(Address(CpuRegister(RSP), dest), Immediate(imm)); // TODO(64) movq?
2510 }
2511
StoreImmediateToThread64(ThreadOffset<8> dest,uint32_t imm,ManagedRegister)2512 void X86_64Assembler::StoreImmediateToThread64(ThreadOffset<8> dest, uint32_t imm,
2513 ManagedRegister) {
2514 gs()->movl(Address::Absolute(dest, true), Immediate(imm)); // TODO(64) movq?
2515 }
2516
StoreStackOffsetToThread64(ThreadOffset<8> thr_offs,FrameOffset fr_offs,ManagedRegister mscratch)2517 void X86_64Assembler::StoreStackOffsetToThread64(ThreadOffset<8> thr_offs,
2518 FrameOffset fr_offs,
2519 ManagedRegister mscratch) {
2520 X86_64ManagedRegister scratch = mscratch.AsX86_64();
2521 CHECK(scratch.IsCpuRegister());
2522 leaq(scratch.AsCpuRegister(), Address(CpuRegister(RSP), fr_offs));
2523 gs()->movq(Address::Absolute(thr_offs, true), scratch.AsCpuRegister());
2524 }
2525
StoreStackPointerToThread64(ThreadOffset<8> thr_offs)2526 void X86_64Assembler::StoreStackPointerToThread64(ThreadOffset<8> thr_offs) {
2527 gs()->movq(Address::Absolute(thr_offs, true), CpuRegister(RSP));
2528 }
2529
StoreSpanning(FrameOffset,ManagedRegister,FrameOffset,ManagedRegister)2530 void X86_64Assembler::StoreSpanning(FrameOffset /*dst*/, ManagedRegister /*src*/,
2531 FrameOffset /*in_off*/, ManagedRegister /*scratch*/) {
2532 UNIMPLEMENTED(FATAL); // this case only currently exists for ARM
2533 }
2534
Load(ManagedRegister mdest,FrameOffset src,size_t size)2535 void X86_64Assembler::Load(ManagedRegister mdest, FrameOffset src, size_t size) {
2536 X86_64ManagedRegister dest = mdest.AsX86_64();
2537 if (dest.IsNoRegister()) {
2538 CHECK_EQ(0u, size);
2539 } else if (dest.IsCpuRegister()) {
2540 if (size == 4) {
2541 CHECK_EQ(4u, size);
2542 movl(dest.AsCpuRegister(), Address(CpuRegister(RSP), src));
2543 } else {
2544 CHECK_EQ(8u, size);
2545 movq(dest.AsCpuRegister(), Address(CpuRegister(RSP), src));
2546 }
2547 } else if (dest.IsRegisterPair()) {
2548 CHECK_EQ(0u, size);
2549 movq(dest.AsRegisterPairLow(), Address(CpuRegister(RSP), src));
2550 movq(dest.AsRegisterPairHigh(), Address(CpuRegister(RSP), FrameOffset(src.Int32Value()+4)));
2551 } else if (dest.IsX87Register()) {
2552 if (size == 4) {
2553 flds(Address(CpuRegister(RSP), src));
2554 } else {
2555 fldl(Address(CpuRegister(RSP), src));
2556 }
2557 } else {
2558 CHECK(dest.IsXmmRegister());
2559 if (size == 4) {
2560 movss(dest.AsXmmRegister(), Address(CpuRegister(RSP), src));
2561 } else {
2562 movsd(dest.AsXmmRegister(), Address(CpuRegister(RSP), src));
2563 }
2564 }
2565 }
2566
LoadFromThread64(ManagedRegister mdest,ThreadOffset<8> src,size_t size)2567 void X86_64Assembler::LoadFromThread64(ManagedRegister mdest, ThreadOffset<8> src, size_t size) {
2568 X86_64ManagedRegister dest = mdest.AsX86_64();
2569 if (dest.IsNoRegister()) {
2570 CHECK_EQ(0u, size);
2571 } else if (dest.IsCpuRegister()) {
2572 CHECK_EQ(4u, size);
2573 gs()->movl(dest.AsCpuRegister(), Address::Absolute(src, true));
2574 } else if (dest.IsRegisterPair()) {
2575 CHECK_EQ(8u, size);
2576 gs()->movq(dest.AsRegisterPairLow(), Address::Absolute(src, true));
2577 } else if (dest.IsX87Register()) {
2578 if (size == 4) {
2579 gs()->flds(Address::Absolute(src, true));
2580 } else {
2581 gs()->fldl(Address::Absolute(src, true));
2582 }
2583 } else {
2584 CHECK(dest.IsXmmRegister());
2585 if (size == 4) {
2586 gs()->movss(dest.AsXmmRegister(), Address::Absolute(src, true));
2587 } else {
2588 gs()->movsd(dest.AsXmmRegister(), Address::Absolute(src, true));
2589 }
2590 }
2591 }
2592
LoadRef(ManagedRegister mdest,FrameOffset src)2593 void X86_64Assembler::LoadRef(ManagedRegister mdest, FrameOffset src) {
2594 X86_64ManagedRegister dest = mdest.AsX86_64();
2595 CHECK(dest.IsCpuRegister());
2596 movq(dest.AsCpuRegister(), Address(CpuRegister(RSP), src));
2597 }
2598
LoadRef(ManagedRegister mdest,ManagedRegister base,MemberOffset offs,bool poison_reference)2599 void X86_64Assembler::LoadRef(ManagedRegister mdest, ManagedRegister base, MemberOffset offs,
2600 bool poison_reference) {
2601 X86_64ManagedRegister dest = mdest.AsX86_64();
2602 CHECK(dest.IsCpuRegister() && dest.IsCpuRegister());
2603 movl(dest.AsCpuRegister(), Address(base.AsX86_64().AsCpuRegister(), offs));
2604 if (kPoisonHeapReferences && poison_reference) {
2605 negl(dest.AsCpuRegister());
2606 }
2607 }
2608
LoadRawPtr(ManagedRegister mdest,ManagedRegister base,Offset offs)2609 void X86_64Assembler::LoadRawPtr(ManagedRegister mdest, ManagedRegister base,
2610 Offset offs) {
2611 X86_64ManagedRegister dest = mdest.AsX86_64();
2612 CHECK(dest.IsCpuRegister() && dest.IsCpuRegister());
2613 movq(dest.AsCpuRegister(), Address(base.AsX86_64().AsCpuRegister(), offs));
2614 }
2615
LoadRawPtrFromThread64(ManagedRegister mdest,ThreadOffset<8> offs)2616 void X86_64Assembler::LoadRawPtrFromThread64(ManagedRegister mdest, ThreadOffset<8> offs) {
2617 X86_64ManagedRegister dest = mdest.AsX86_64();
2618 CHECK(dest.IsCpuRegister());
2619 gs()->movq(dest.AsCpuRegister(), Address::Absolute(offs, true));
2620 }
2621
SignExtend(ManagedRegister mreg,size_t size)2622 void X86_64Assembler::SignExtend(ManagedRegister mreg, size_t size) {
2623 X86_64ManagedRegister reg = mreg.AsX86_64();
2624 CHECK(size == 1 || size == 2) << size;
2625 CHECK(reg.IsCpuRegister()) << reg;
2626 if (size == 1) {
2627 movsxb(reg.AsCpuRegister(), reg.AsCpuRegister());
2628 } else {
2629 movsxw(reg.AsCpuRegister(), reg.AsCpuRegister());
2630 }
2631 }
2632
ZeroExtend(ManagedRegister mreg,size_t size)2633 void X86_64Assembler::ZeroExtend(ManagedRegister mreg, size_t size) {
2634 X86_64ManagedRegister reg = mreg.AsX86_64();
2635 CHECK(size == 1 || size == 2) << size;
2636 CHECK(reg.IsCpuRegister()) << reg;
2637 if (size == 1) {
2638 movzxb(reg.AsCpuRegister(), reg.AsCpuRegister());
2639 } else {
2640 movzxw(reg.AsCpuRegister(), reg.AsCpuRegister());
2641 }
2642 }
2643
Move(ManagedRegister mdest,ManagedRegister msrc,size_t size)2644 void X86_64Assembler::Move(ManagedRegister mdest, ManagedRegister msrc, size_t size) {
2645 X86_64ManagedRegister dest = mdest.AsX86_64();
2646 X86_64ManagedRegister src = msrc.AsX86_64();
2647 if (!dest.Equals(src)) {
2648 if (dest.IsCpuRegister() && src.IsCpuRegister()) {
2649 movq(dest.AsCpuRegister(), src.AsCpuRegister());
2650 } else if (src.IsX87Register() && dest.IsXmmRegister()) {
2651 // Pass via stack and pop X87 register
2652 subl(CpuRegister(RSP), Immediate(16));
2653 if (size == 4) {
2654 CHECK_EQ(src.AsX87Register(), ST0);
2655 fstps(Address(CpuRegister(RSP), 0));
2656 movss(dest.AsXmmRegister(), Address(CpuRegister(RSP), 0));
2657 } else {
2658 CHECK_EQ(src.AsX87Register(), ST0);
2659 fstpl(Address(CpuRegister(RSP), 0));
2660 movsd(dest.AsXmmRegister(), Address(CpuRegister(RSP), 0));
2661 }
2662 addq(CpuRegister(RSP), Immediate(16));
2663 } else {
2664 // TODO: x87, SSE
2665 UNIMPLEMENTED(FATAL) << ": Move " << dest << ", " << src;
2666 }
2667 }
2668 }
2669
CopyRef(FrameOffset dest,FrameOffset src,ManagedRegister mscratch)2670 void X86_64Assembler::CopyRef(FrameOffset dest, FrameOffset src, ManagedRegister mscratch) {
2671 X86_64ManagedRegister scratch = mscratch.AsX86_64();
2672 CHECK(scratch.IsCpuRegister());
2673 movl(scratch.AsCpuRegister(), Address(CpuRegister(RSP), src));
2674 movl(Address(CpuRegister(RSP), dest), scratch.AsCpuRegister());
2675 }
2676
CopyRawPtrFromThread64(FrameOffset fr_offs,ThreadOffset<8> thr_offs,ManagedRegister mscratch)2677 void X86_64Assembler::CopyRawPtrFromThread64(FrameOffset fr_offs,
2678 ThreadOffset<8> thr_offs,
2679 ManagedRegister mscratch) {
2680 X86_64ManagedRegister scratch = mscratch.AsX86_64();
2681 CHECK(scratch.IsCpuRegister());
2682 gs()->movq(scratch.AsCpuRegister(), Address::Absolute(thr_offs, true));
2683 Store(fr_offs, scratch, 8);
2684 }
2685
CopyRawPtrToThread64(ThreadOffset<8> thr_offs,FrameOffset fr_offs,ManagedRegister mscratch)2686 void X86_64Assembler::CopyRawPtrToThread64(ThreadOffset<8> thr_offs,
2687 FrameOffset fr_offs,
2688 ManagedRegister mscratch) {
2689 X86_64ManagedRegister scratch = mscratch.AsX86_64();
2690 CHECK(scratch.IsCpuRegister());
2691 Load(scratch, fr_offs, 8);
2692 gs()->movq(Address::Absolute(thr_offs, true), scratch.AsCpuRegister());
2693 }
2694
Copy(FrameOffset dest,FrameOffset src,ManagedRegister mscratch,size_t size)2695 void X86_64Assembler::Copy(FrameOffset dest, FrameOffset src, ManagedRegister mscratch,
2696 size_t size) {
2697 X86_64ManagedRegister scratch = mscratch.AsX86_64();
2698 if (scratch.IsCpuRegister() && size == 8) {
2699 Load(scratch, src, 4);
2700 Store(dest, scratch, 4);
2701 Load(scratch, FrameOffset(src.Int32Value() + 4), 4);
2702 Store(FrameOffset(dest.Int32Value() + 4), scratch, 4);
2703 } else {
2704 Load(scratch, src, size);
2705 Store(dest, scratch, size);
2706 }
2707 }
2708
Copy(FrameOffset,ManagedRegister,Offset,ManagedRegister,size_t)2709 void X86_64Assembler::Copy(FrameOffset /*dst*/, ManagedRegister /*src_base*/, Offset /*src_offset*/,
2710 ManagedRegister /*scratch*/, size_t /*size*/) {
2711 UNIMPLEMENTED(FATAL);
2712 }
2713
Copy(ManagedRegister dest_base,Offset dest_offset,FrameOffset src,ManagedRegister scratch,size_t size)2714 void X86_64Assembler::Copy(ManagedRegister dest_base, Offset dest_offset, FrameOffset src,
2715 ManagedRegister scratch, size_t size) {
2716 CHECK(scratch.IsNoRegister());
2717 CHECK_EQ(size, 4u);
2718 pushq(Address(CpuRegister(RSP), src));
2719 popq(Address(dest_base.AsX86_64().AsCpuRegister(), dest_offset));
2720 }
2721
Copy(FrameOffset dest,FrameOffset src_base,Offset src_offset,ManagedRegister mscratch,size_t size)2722 void X86_64Assembler::Copy(FrameOffset dest, FrameOffset src_base, Offset src_offset,
2723 ManagedRegister mscratch, size_t size) {
2724 CpuRegister scratch = mscratch.AsX86_64().AsCpuRegister();
2725 CHECK_EQ(size, 4u);
2726 movq(scratch, Address(CpuRegister(RSP), src_base));
2727 movq(scratch, Address(scratch, src_offset));
2728 movq(Address(CpuRegister(RSP), dest), scratch);
2729 }
2730
Copy(ManagedRegister dest,Offset dest_offset,ManagedRegister src,Offset src_offset,ManagedRegister scratch,size_t size)2731 void X86_64Assembler::Copy(ManagedRegister dest, Offset dest_offset,
2732 ManagedRegister src, Offset src_offset,
2733 ManagedRegister scratch, size_t size) {
2734 CHECK_EQ(size, 4u);
2735 CHECK(scratch.IsNoRegister());
2736 pushq(Address(src.AsX86_64().AsCpuRegister(), src_offset));
2737 popq(Address(dest.AsX86_64().AsCpuRegister(), dest_offset));
2738 }
2739
Copy(FrameOffset dest,Offset dest_offset,FrameOffset src,Offset src_offset,ManagedRegister mscratch,size_t size)2740 void X86_64Assembler::Copy(FrameOffset dest, Offset dest_offset, FrameOffset src, Offset src_offset,
2741 ManagedRegister mscratch, size_t size) {
2742 CpuRegister scratch = mscratch.AsX86_64().AsCpuRegister();
2743 CHECK_EQ(size, 4u);
2744 CHECK_EQ(dest.Int32Value(), src.Int32Value());
2745 movq(scratch, Address(CpuRegister(RSP), src));
2746 pushq(Address(scratch, src_offset));
2747 popq(Address(scratch, dest_offset));
2748 }
2749
MemoryBarrier(ManagedRegister)2750 void X86_64Assembler::MemoryBarrier(ManagedRegister) {
2751 mfence();
2752 }
2753
CreateHandleScopeEntry(ManagedRegister mout_reg,FrameOffset handle_scope_offset,ManagedRegister min_reg,bool null_allowed)2754 void X86_64Assembler::CreateHandleScopeEntry(ManagedRegister mout_reg,
2755 FrameOffset handle_scope_offset,
2756 ManagedRegister min_reg, bool null_allowed) {
2757 X86_64ManagedRegister out_reg = mout_reg.AsX86_64();
2758 X86_64ManagedRegister in_reg = min_reg.AsX86_64();
2759 if (in_reg.IsNoRegister()) { // TODO(64): && null_allowed
2760 // Use out_reg as indicator of null.
2761 in_reg = out_reg;
2762 // TODO: movzwl
2763 movl(in_reg.AsCpuRegister(), Address(CpuRegister(RSP), handle_scope_offset));
2764 }
2765 CHECK(in_reg.IsCpuRegister());
2766 CHECK(out_reg.IsCpuRegister());
2767 VerifyObject(in_reg, null_allowed);
2768 if (null_allowed) {
2769 Label null_arg;
2770 if (!out_reg.Equals(in_reg)) {
2771 xorl(out_reg.AsCpuRegister(), out_reg.AsCpuRegister());
2772 }
2773 testl(in_reg.AsCpuRegister(), in_reg.AsCpuRegister());
2774 j(kZero, &null_arg);
2775 leaq(out_reg.AsCpuRegister(), Address(CpuRegister(RSP), handle_scope_offset));
2776 Bind(&null_arg);
2777 } else {
2778 leaq(out_reg.AsCpuRegister(), Address(CpuRegister(RSP), handle_scope_offset));
2779 }
2780 }
2781
CreateHandleScopeEntry(FrameOffset out_off,FrameOffset handle_scope_offset,ManagedRegister mscratch,bool null_allowed)2782 void X86_64Assembler::CreateHandleScopeEntry(FrameOffset out_off,
2783 FrameOffset handle_scope_offset,
2784 ManagedRegister mscratch,
2785 bool null_allowed) {
2786 X86_64ManagedRegister scratch = mscratch.AsX86_64();
2787 CHECK(scratch.IsCpuRegister());
2788 if (null_allowed) {
2789 Label null_arg;
2790 movl(scratch.AsCpuRegister(), Address(CpuRegister(RSP), handle_scope_offset));
2791 testl(scratch.AsCpuRegister(), scratch.AsCpuRegister());
2792 j(kZero, &null_arg);
2793 leaq(scratch.AsCpuRegister(), Address(CpuRegister(RSP), handle_scope_offset));
2794 Bind(&null_arg);
2795 } else {
2796 leaq(scratch.AsCpuRegister(), Address(CpuRegister(RSP), handle_scope_offset));
2797 }
2798 Store(out_off, scratch, 8);
2799 }
2800
2801 // Given a handle scope entry, load the associated reference.
LoadReferenceFromHandleScope(ManagedRegister mout_reg,ManagedRegister min_reg)2802 void X86_64Assembler::LoadReferenceFromHandleScope(ManagedRegister mout_reg,
2803 ManagedRegister min_reg) {
2804 X86_64ManagedRegister out_reg = mout_reg.AsX86_64();
2805 X86_64ManagedRegister in_reg = min_reg.AsX86_64();
2806 CHECK(out_reg.IsCpuRegister());
2807 CHECK(in_reg.IsCpuRegister());
2808 Label null_arg;
2809 if (!out_reg.Equals(in_reg)) {
2810 xorl(out_reg.AsCpuRegister(), out_reg.AsCpuRegister());
2811 }
2812 testl(in_reg.AsCpuRegister(), in_reg.AsCpuRegister());
2813 j(kZero, &null_arg);
2814 movq(out_reg.AsCpuRegister(), Address(in_reg.AsCpuRegister(), 0));
2815 Bind(&null_arg);
2816 }
2817
VerifyObject(ManagedRegister,bool)2818 void X86_64Assembler::VerifyObject(ManagedRegister /*src*/, bool /*could_be_null*/) {
2819 // TODO: not validating references
2820 }
2821
VerifyObject(FrameOffset,bool)2822 void X86_64Assembler::VerifyObject(FrameOffset /*src*/, bool /*could_be_null*/) {
2823 // TODO: not validating references
2824 }
2825
Call(ManagedRegister mbase,Offset offset,ManagedRegister)2826 void X86_64Assembler::Call(ManagedRegister mbase, Offset offset, ManagedRegister) {
2827 X86_64ManagedRegister base = mbase.AsX86_64();
2828 CHECK(base.IsCpuRegister());
2829 call(Address(base.AsCpuRegister(), offset.Int32Value()));
2830 // TODO: place reference map on call
2831 }
2832
Call(FrameOffset base,Offset offset,ManagedRegister mscratch)2833 void X86_64Assembler::Call(FrameOffset base, Offset offset, ManagedRegister mscratch) {
2834 CpuRegister scratch = mscratch.AsX86_64().AsCpuRegister();
2835 movq(scratch, Address(CpuRegister(RSP), base));
2836 call(Address(scratch, offset));
2837 }
2838
CallFromThread64(ThreadOffset<8> offset,ManagedRegister)2839 void X86_64Assembler::CallFromThread64(ThreadOffset<8> offset, ManagedRegister /*mscratch*/) {
2840 gs()->call(Address::Absolute(offset, true));
2841 }
2842
GetCurrentThread(ManagedRegister tr)2843 void X86_64Assembler::GetCurrentThread(ManagedRegister tr) {
2844 gs()->movq(tr.AsX86_64().AsCpuRegister(), Address::Absolute(Thread::SelfOffset<8>(), true));
2845 }
2846
GetCurrentThread(FrameOffset offset,ManagedRegister mscratch)2847 void X86_64Assembler::GetCurrentThread(FrameOffset offset, ManagedRegister mscratch) {
2848 X86_64ManagedRegister scratch = mscratch.AsX86_64();
2849 gs()->movq(scratch.AsCpuRegister(), Address::Absolute(Thread::SelfOffset<8>(), true));
2850 movq(Address(CpuRegister(RSP), offset), scratch.AsCpuRegister());
2851 }
2852
2853 // Slowpath entered when Thread::Current()->_exception is non-null
2854 class X86_64ExceptionSlowPath FINAL : public SlowPath {
2855 public:
X86_64ExceptionSlowPath(size_t stack_adjust)2856 explicit X86_64ExceptionSlowPath(size_t stack_adjust) : stack_adjust_(stack_adjust) {}
2857 virtual void Emit(Assembler *sp_asm) OVERRIDE;
2858 private:
2859 const size_t stack_adjust_;
2860 };
2861
ExceptionPoll(ManagedRegister,size_t stack_adjust)2862 void X86_64Assembler::ExceptionPoll(ManagedRegister /*scratch*/, size_t stack_adjust) {
2863 X86_64ExceptionSlowPath* slow = new X86_64ExceptionSlowPath(stack_adjust);
2864 buffer_.EnqueueSlowPath(slow);
2865 gs()->cmpl(Address::Absolute(Thread::ExceptionOffset<8>(), true), Immediate(0));
2866 j(kNotEqual, slow->Entry());
2867 }
2868
Emit(Assembler * sasm)2869 void X86_64ExceptionSlowPath::Emit(Assembler *sasm) {
2870 X86_64Assembler* sp_asm = down_cast<X86_64Assembler*>(sasm);
2871 #define __ sp_asm->
2872 __ Bind(&entry_);
2873 // Note: the return value is dead
2874 if (stack_adjust_ != 0) { // Fix up the frame.
2875 __ DecreaseFrameSize(stack_adjust_);
2876 }
2877 // Pass exception as argument in RDI
2878 __ gs()->movq(CpuRegister(RDI), Address::Absolute(Thread::ExceptionOffset<8>(), true));
2879 __ gs()->call(Address::Absolute(QUICK_ENTRYPOINT_OFFSET(8, pDeliverException), true));
2880 // this call should never return
2881 __ int3();
2882 #undef __
2883 }
2884
AddConstantArea()2885 void X86_64Assembler::AddConstantArea() {
2886 const std::vector<int32_t>& area = constant_area_.GetBuffer();
2887 for (size_t i = 0, e = area.size(); i < e; i++) {
2888 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
2889 EmitInt32(area[i]);
2890 }
2891 }
2892
AddInt32(int32_t v)2893 int ConstantArea::AddInt32(int32_t v) {
2894 for (size_t i = 0, e = buffer_.size(); i < e; i++) {
2895 if (v == buffer_[i]) {
2896 return i * elem_size_;
2897 }
2898 }
2899
2900 // Didn't match anything.
2901 int result = buffer_.size() * elem_size_;
2902 buffer_.push_back(v);
2903 return result;
2904 }
2905
AddInt64(int64_t v)2906 int ConstantArea::AddInt64(int64_t v) {
2907 int32_t v_low = v;
2908 int32_t v_high = v >> 32;
2909 if (buffer_.size() > 1) {
2910 // Ensure we don't pass the end of the buffer.
2911 for (size_t i = 0, e = buffer_.size() - 1; i < e; i++) {
2912 if (v_low == buffer_[i] && v_high == buffer_[i + 1]) {
2913 return i * elem_size_;
2914 }
2915 }
2916 }
2917
2918 // Didn't match anything.
2919 int result = buffer_.size() * elem_size_;
2920 buffer_.push_back(v_low);
2921 buffer_.push_back(v_high);
2922 return result;
2923 }
2924
AddDouble(double v)2925 int ConstantArea::AddDouble(double v) {
2926 // Treat the value as a 64-bit integer value.
2927 return AddInt64(bit_cast<int64_t, double>(v));
2928 }
2929
AddFloat(float v)2930 int ConstantArea::AddFloat(float v) {
2931 // Treat the value as a 32-bit integer value.
2932 return AddInt32(bit_cast<int32_t, float>(v));
2933 }
2934
2935 } // namespace x86_64
2936 } // namespace art
2937