1 /*
2 * Copyright (C) 2014 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 #include "assembler_x86_64.h"
18
19 #include "base/casts.h"
20 #include "entrypoints/quick/quick_entrypoints.h"
21 #include "memory_region.h"
22 #include "thread.h"
23
24 namespace art {
25 namespace x86_64 {
26
operator <<(std::ostream & os,const CpuRegister & reg)27 std::ostream& operator<<(std::ostream& os, const CpuRegister& reg) {
28 return os << reg.AsRegister();
29 }
30
operator <<(std::ostream & os,const XmmRegister & reg)31 std::ostream& operator<<(std::ostream& os, const XmmRegister& reg) {
32 return os << reg.AsFloatRegister();
33 }
34
operator <<(std::ostream & os,const X87Register & reg)35 std::ostream& operator<<(std::ostream& os, const X87Register& reg) {
36 return os << "ST" << static_cast<int>(reg);
37 }
38
call(CpuRegister reg)39 void X86_64Assembler::call(CpuRegister reg) {
40 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
41 EmitOptionalRex32(reg);
42 EmitUint8(0xFF);
43 EmitRegisterOperand(2, reg.LowBits());
44 }
45
46
call(const Address & address)47 void X86_64Assembler::call(const Address& address) {
48 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
49 EmitOptionalRex32(address);
50 EmitUint8(0xFF);
51 EmitOperand(2, address);
52 }
53
54
call(Label * label)55 void X86_64Assembler::call(Label* label) {
56 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
57 EmitUint8(0xE8);
58 static const int kSize = 5;
59 // Offset by one because we already have emitted the opcode.
60 EmitLabel(label, kSize - 1);
61 }
62
pushq(CpuRegister reg)63 void X86_64Assembler::pushq(CpuRegister reg) {
64 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
65 EmitOptionalRex32(reg);
66 EmitUint8(0x50 + reg.LowBits());
67 }
68
69
pushq(const Address & address)70 void X86_64Assembler::pushq(const Address& address) {
71 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
72 EmitOptionalRex32(address);
73 EmitUint8(0xFF);
74 EmitOperand(6, address);
75 }
76
77
pushq(const Immediate & imm)78 void X86_64Assembler::pushq(const Immediate& imm) {
79 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
80 CHECK(imm.is_int32()); // pushq only supports 32b immediate.
81 if (imm.is_int8()) {
82 EmitUint8(0x6A);
83 EmitUint8(imm.value() & 0xFF);
84 } else {
85 EmitUint8(0x68);
86 EmitImmediate(imm);
87 }
88 }
89
90
popq(CpuRegister reg)91 void X86_64Assembler::popq(CpuRegister reg) {
92 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
93 EmitOptionalRex32(reg);
94 EmitUint8(0x58 + reg.LowBits());
95 }
96
97
popq(const Address & address)98 void X86_64Assembler::popq(const Address& address) {
99 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
100 EmitOptionalRex32(address);
101 EmitUint8(0x8F);
102 EmitOperand(0, address);
103 }
104
105
movq(CpuRegister dst,const Immediate & imm)106 void X86_64Assembler::movq(CpuRegister dst, const Immediate& imm) {
107 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
108 if (imm.is_int32()) {
109 // 32 bit. Note: sign-extends.
110 EmitRex64(dst);
111 EmitUint8(0xC7);
112 EmitRegisterOperand(0, dst.LowBits());
113 EmitInt32(static_cast<int32_t>(imm.value()));
114 } else {
115 EmitRex64(dst);
116 EmitUint8(0xB8 + dst.LowBits());
117 EmitInt64(imm.value());
118 }
119 }
120
121
movl(CpuRegister dst,const Immediate & imm)122 void X86_64Assembler::movl(CpuRegister dst, const Immediate& imm) {
123 CHECK(imm.is_int32());
124 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
125 EmitOptionalRex32(dst);
126 EmitUint8(0xB8 + dst.LowBits());
127 EmitImmediate(imm);
128 }
129
130
movq(const Address & dst,const Immediate & imm)131 void X86_64Assembler::movq(const Address& dst, const Immediate& imm) {
132 CHECK(imm.is_int32());
133 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
134 EmitRex64(dst);
135 EmitUint8(0xC7);
136 EmitOperand(0, dst);
137 EmitImmediate(imm);
138 }
139
140
movq(CpuRegister dst,CpuRegister src)141 void X86_64Assembler::movq(CpuRegister dst, CpuRegister src) {
142 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
143 // 0x89 is movq r/m64 <- r64, with op1 in r/m and op2 in reg: so reverse EmitRex64
144 EmitRex64(src, dst);
145 EmitUint8(0x89);
146 EmitRegisterOperand(src.LowBits(), dst.LowBits());
147 }
148
149
movl(CpuRegister dst,CpuRegister src)150 void X86_64Assembler::movl(CpuRegister dst, CpuRegister src) {
151 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
152 EmitOptionalRex32(dst, src);
153 EmitUint8(0x8B);
154 EmitRegisterOperand(dst.LowBits(), src.LowBits());
155 }
156
157
movq(CpuRegister dst,const Address & src)158 void X86_64Assembler::movq(CpuRegister dst, const Address& src) {
159 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
160 EmitRex64(dst, src);
161 EmitUint8(0x8B);
162 EmitOperand(dst.LowBits(), src);
163 }
164
165
movl(CpuRegister dst,const Address & src)166 void X86_64Assembler::movl(CpuRegister dst, const Address& src) {
167 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
168 EmitOptionalRex32(dst, src);
169 EmitUint8(0x8B);
170 EmitOperand(dst.LowBits(), src);
171 }
172
173
movq(const Address & dst,CpuRegister src)174 void X86_64Assembler::movq(const Address& dst, CpuRegister src) {
175 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
176 EmitRex64(src, dst);
177 EmitUint8(0x89);
178 EmitOperand(src.LowBits(), dst);
179 }
180
181
movl(const Address & dst,CpuRegister src)182 void X86_64Assembler::movl(const Address& dst, CpuRegister src) {
183 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
184 EmitOptionalRex32(src, dst);
185 EmitUint8(0x89);
186 EmitOperand(src.LowBits(), dst);
187 }
188
movl(const Address & dst,const Immediate & imm)189 void X86_64Assembler::movl(const Address& dst, const Immediate& imm) {
190 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
191 EmitOptionalRex32(dst);
192 EmitUint8(0xC7);
193 EmitOperand(0, dst);
194 EmitImmediate(imm);
195 }
196
movntl(const Address & dst,CpuRegister src)197 void X86_64Assembler::movntl(const Address& dst, CpuRegister src) {
198 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
199 EmitOptionalRex32(src, dst);
200 EmitUint8(0x0F);
201 EmitUint8(0xC3);
202 EmitOperand(src.LowBits(), dst);
203 }
204
movntq(const Address & dst,CpuRegister src)205 void X86_64Assembler::movntq(const Address& dst, CpuRegister src) {
206 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
207 EmitRex64(src, dst);
208 EmitUint8(0x0F);
209 EmitUint8(0xC3);
210 EmitOperand(src.LowBits(), dst);
211 }
212
cmov(Condition c,CpuRegister dst,CpuRegister src)213 void X86_64Assembler::cmov(Condition c, CpuRegister dst, CpuRegister src) {
214 cmov(c, dst, src, true);
215 }
216
cmov(Condition c,CpuRegister dst,CpuRegister src,bool is64bit)217 void X86_64Assembler::cmov(Condition c, CpuRegister dst, CpuRegister src, bool is64bit) {
218 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
219 EmitOptionalRex(false, is64bit, dst.NeedsRex(), false, src.NeedsRex());
220 EmitUint8(0x0F);
221 EmitUint8(0x40 + c);
222 EmitRegisterOperand(dst.LowBits(), src.LowBits());
223 }
224
225
cmov(Condition c,CpuRegister dst,const Address & src,bool is64bit)226 void X86_64Assembler::cmov(Condition c, CpuRegister dst, const Address& src, bool is64bit) {
227 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
228 if (is64bit) {
229 EmitRex64(dst, src);
230 } else {
231 EmitOptionalRex32(dst, src);
232 }
233 EmitUint8(0x0F);
234 EmitUint8(0x40 + c);
235 EmitOperand(dst.LowBits(), src);
236 }
237
238
movzxb(CpuRegister dst,CpuRegister src)239 void X86_64Assembler::movzxb(CpuRegister dst, CpuRegister src) {
240 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
241 EmitOptionalByteRegNormalizingRex32(dst, src);
242 EmitUint8(0x0F);
243 EmitUint8(0xB6);
244 EmitRegisterOperand(dst.LowBits(), src.LowBits());
245 }
246
247
movzxb(CpuRegister dst,const Address & src)248 void X86_64Assembler::movzxb(CpuRegister dst, const Address& src) {
249 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
250 // Byte register is only in the source register form, so we don't use
251 // EmitOptionalByteRegNormalizingRex32(dst, src);
252 EmitOptionalRex32(dst, src);
253 EmitUint8(0x0F);
254 EmitUint8(0xB6);
255 EmitOperand(dst.LowBits(), src);
256 }
257
258
movsxb(CpuRegister dst,CpuRegister src)259 void X86_64Assembler::movsxb(CpuRegister dst, CpuRegister src) {
260 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
261 EmitOptionalByteRegNormalizingRex32(dst, src);
262 EmitUint8(0x0F);
263 EmitUint8(0xBE);
264 EmitRegisterOperand(dst.LowBits(), src.LowBits());
265 }
266
267
movsxb(CpuRegister dst,const Address & src)268 void X86_64Assembler::movsxb(CpuRegister dst, const Address& src) {
269 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
270 // Byte register is only in the source register form, so we don't use
271 // EmitOptionalByteRegNormalizingRex32(dst, src);
272 EmitOptionalRex32(dst, src);
273 EmitUint8(0x0F);
274 EmitUint8(0xBE);
275 EmitOperand(dst.LowBits(), src);
276 }
277
278
movb(CpuRegister,const Address &)279 void X86_64Assembler::movb(CpuRegister /*dst*/, const Address& /*src*/) {
280 LOG(FATAL) << "Use movzxb or movsxb instead.";
281 }
282
283
movb(const Address & dst,CpuRegister src)284 void X86_64Assembler::movb(const Address& dst, CpuRegister src) {
285 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
286 EmitOptionalByteRegNormalizingRex32(src, dst);
287 EmitUint8(0x88);
288 EmitOperand(src.LowBits(), dst);
289 }
290
291
movb(const Address & dst,const Immediate & imm)292 void X86_64Assembler::movb(const Address& dst, const Immediate& imm) {
293 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
294 EmitOptionalRex32(dst);
295 EmitUint8(0xC6);
296 EmitOperand(Register::RAX, dst);
297 CHECK(imm.is_int8());
298 EmitUint8(imm.value() & 0xFF);
299 }
300
301
movzxw(CpuRegister dst,CpuRegister src)302 void X86_64Assembler::movzxw(CpuRegister dst, CpuRegister src) {
303 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
304 EmitOptionalRex32(dst, src);
305 EmitUint8(0x0F);
306 EmitUint8(0xB7);
307 EmitRegisterOperand(dst.LowBits(), src.LowBits());
308 }
309
310
movzxw(CpuRegister dst,const Address & src)311 void X86_64Assembler::movzxw(CpuRegister dst, const Address& src) {
312 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
313 EmitOptionalRex32(dst, src);
314 EmitUint8(0x0F);
315 EmitUint8(0xB7);
316 EmitOperand(dst.LowBits(), src);
317 }
318
319
movsxw(CpuRegister dst,CpuRegister src)320 void X86_64Assembler::movsxw(CpuRegister dst, CpuRegister src) {
321 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
322 EmitOptionalRex32(dst, src);
323 EmitUint8(0x0F);
324 EmitUint8(0xBF);
325 EmitRegisterOperand(dst.LowBits(), src.LowBits());
326 }
327
328
movsxw(CpuRegister dst,const Address & src)329 void X86_64Assembler::movsxw(CpuRegister dst, const Address& src) {
330 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
331 EmitOptionalRex32(dst, src);
332 EmitUint8(0x0F);
333 EmitUint8(0xBF);
334 EmitOperand(dst.LowBits(), src);
335 }
336
337
movw(CpuRegister,const Address &)338 void X86_64Assembler::movw(CpuRegister /*dst*/, const Address& /*src*/) {
339 LOG(FATAL) << "Use movzxw or movsxw instead.";
340 }
341
342
movw(const Address & dst,CpuRegister src)343 void X86_64Assembler::movw(const Address& dst, CpuRegister src) {
344 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
345 EmitOperandSizeOverride();
346 EmitOptionalRex32(src, dst);
347 EmitUint8(0x89);
348 EmitOperand(src.LowBits(), dst);
349 }
350
351
movw(const Address & dst,const Immediate & imm)352 void X86_64Assembler::movw(const Address& dst, const Immediate& imm) {
353 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
354 EmitOperandSizeOverride();
355 EmitOptionalRex32(dst);
356 EmitUint8(0xC7);
357 EmitOperand(Register::RAX, dst);
358 CHECK(imm.is_uint16() || imm.is_int16());
359 EmitUint8(imm.value() & 0xFF);
360 EmitUint8(imm.value() >> 8);
361 }
362
363
leaq(CpuRegister dst,const Address & src)364 void X86_64Assembler::leaq(CpuRegister dst, const Address& src) {
365 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
366 EmitRex64(dst, src);
367 EmitUint8(0x8D);
368 EmitOperand(dst.LowBits(), src);
369 }
370
371
leal(CpuRegister dst,const Address & src)372 void X86_64Assembler::leal(CpuRegister dst, const Address& src) {
373 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
374 EmitOptionalRex32(dst, src);
375 EmitUint8(0x8D);
376 EmitOperand(dst.LowBits(), src);
377 }
378
379
movaps(XmmRegister dst,XmmRegister src)380 void X86_64Assembler::movaps(XmmRegister dst, XmmRegister src) {
381 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
382 EmitOptionalRex32(dst, src);
383 EmitUint8(0x0F);
384 EmitUint8(0x28);
385 EmitXmmRegisterOperand(dst.LowBits(), src);
386 }
387
388
movss(XmmRegister dst,const Address & src)389 void X86_64Assembler::movss(XmmRegister dst, const Address& src) {
390 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
391 EmitUint8(0xF3);
392 EmitOptionalRex32(dst, src);
393 EmitUint8(0x0F);
394 EmitUint8(0x10);
395 EmitOperand(dst.LowBits(), src);
396 }
397
398
movss(const Address & dst,XmmRegister src)399 void X86_64Assembler::movss(const Address& dst, XmmRegister src) {
400 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
401 EmitUint8(0xF3);
402 EmitOptionalRex32(src, dst);
403 EmitUint8(0x0F);
404 EmitUint8(0x11);
405 EmitOperand(src.LowBits(), dst);
406 }
407
408
movss(XmmRegister dst,XmmRegister src)409 void X86_64Assembler::movss(XmmRegister dst, XmmRegister src) {
410 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
411 EmitUint8(0xF3);
412 EmitOptionalRex32(src, dst); // Movss is MR encoding instead of the usual RM.
413 EmitUint8(0x0F);
414 EmitUint8(0x11);
415 EmitXmmRegisterOperand(src.LowBits(), dst);
416 }
417
418
movsxd(CpuRegister dst,CpuRegister src)419 void X86_64Assembler::movsxd(CpuRegister dst, CpuRegister src) {
420 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
421 EmitRex64(dst, src);
422 EmitUint8(0x63);
423 EmitRegisterOperand(dst.LowBits(), src.LowBits());
424 }
425
426
movsxd(CpuRegister dst,const Address & src)427 void X86_64Assembler::movsxd(CpuRegister dst, const Address& src) {
428 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
429 EmitRex64(dst, src);
430 EmitUint8(0x63);
431 EmitOperand(dst.LowBits(), src);
432 }
433
434
movd(XmmRegister dst,CpuRegister src)435 void X86_64Assembler::movd(XmmRegister dst, CpuRegister src) {
436 movd(dst, src, true);
437 }
438
movd(CpuRegister dst,XmmRegister src)439 void X86_64Assembler::movd(CpuRegister dst, XmmRegister src) {
440 movd(dst, src, true);
441 }
442
movd(XmmRegister dst,CpuRegister src,bool is64bit)443 void X86_64Assembler::movd(XmmRegister dst, CpuRegister src, bool is64bit) {
444 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
445 EmitUint8(0x66);
446 EmitOptionalRex(false, is64bit, dst.NeedsRex(), false, src.NeedsRex());
447 EmitUint8(0x0F);
448 EmitUint8(0x6E);
449 EmitOperand(dst.LowBits(), Operand(src));
450 }
451
movd(CpuRegister dst,XmmRegister src,bool is64bit)452 void X86_64Assembler::movd(CpuRegister dst, XmmRegister src, bool is64bit) {
453 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
454 EmitUint8(0x66);
455 EmitOptionalRex(false, is64bit, src.NeedsRex(), false, dst.NeedsRex());
456 EmitUint8(0x0F);
457 EmitUint8(0x7E);
458 EmitOperand(src.LowBits(), Operand(dst));
459 }
460
461
addss(XmmRegister dst,XmmRegister src)462 void X86_64Assembler::addss(XmmRegister dst, XmmRegister src) {
463 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
464 EmitUint8(0xF3);
465 EmitOptionalRex32(dst, src);
466 EmitUint8(0x0F);
467 EmitUint8(0x58);
468 EmitXmmRegisterOperand(dst.LowBits(), src);
469 }
470
471
addss(XmmRegister dst,const Address & src)472 void X86_64Assembler::addss(XmmRegister dst, const Address& src) {
473 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
474 EmitUint8(0xF3);
475 EmitOptionalRex32(dst, src);
476 EmitUint8(0x0F);
477 EmitUint8(0x58);
478 EmitOperand(dst.LowBits(), src);
479 }
480
481
subss(XmmRegister dst,XmmRegister src)482 void X86_64Assembler::subss(XmmRegister dst, XmmRegister src) {
483 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
484 EmitUint8(0xF3);
485 EmitOptionalRex32(dst, src);
486 EmitUint8(0x0F);
487 EmitUint8(0x5C);
488 EmitXmmRegisterOperand(dst.LowBits(), src);
489 }
490
491
subss(XmmRegister dst,const Address & src)492 void X86_64Assembler::subss(XmmRegister dst, const Address& src) {
493 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
494 EmitUint8(0xF3);
495 EmitOptionalRex32(dst, src);
496 EmitUint8(0x0F);
497 EmitUint8(0x5C);
498 EmitOperand(dst.LowBits(), src);
499 }
500
501
mulss(XmmRegister dst,XmmRegister src)502 void X86_64Assembler::mulss(XmmRegister dst, XmmRegister src) {
503 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
504 EmitUint8(0xF3);
505 EmitOptionalRex32(dst, src);
506 EmitUint8(0x0F);
507 EmitUint8(0x59);
508 EmitXmmRegisterOperand(dst.LowBits(), src);
509 }
510
511
mulss(XmmRegister dst,const Address & src)512 void X86_64Assembler::mulss(XmmRegister dst, const Address& src) {
513 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
514 EmitUint8(0xF3);
515 EmitOptionalRex32(dst, src);
516 EmitUint8(0x0F);
517 EmitUint8(0x59);
518 EmitOperand(dst.LowBits(), src);
519 }
520
521
divss(XmmRegister dst,XmmRegister src)522 void X86_64Assembler::divss(XmmRegister dst, XmmRegister src) {
523 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
524 EmitUint8(0xF3);
525 EmitOptionalRex32(dst, src);
526 EmitUint8(0x0F);
527 EmitUint8(0x5E);
528 EmitXmmRegisterOperand(dst.LowBits(), src);
529 }
530
531
divss(XmmRegister dst,const Address & src)532 void X86_64Assembler::divss(XmmRegister dst, const Address& src) {
533 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
534 EmitUint8(0xF3);
535 EmitOptionalRex32(dst, src);
536 EmitUint8(0x0F);
537 EmitUint8(0x5E);
538 EmitOperand(dst.LowBits(), src);
539 }
540
541
flds(const Address & src)542 void X86_64Assembler::flds(const Address& src) {
543 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
544 EmitUint8(0xD9);
545 EmitOperand(0, src);
546 }
547
548
fsts(const Address & dst)549 void X86_64Assembler::fsts(const Address& dst) {
550 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
551 EmitUint8(0xD9);
552 EmitOperand(2, dst);
553 }
554
555
fstps(const Address & dst)556 void X86_64Assembler::fstps(const Address& dst) {
557 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
558 EmitUint8(0xD9);
559 EmitOperand(3, dst);
560 }
561
562
movsd(XmmRegister dst,const Address & src)563 void X86_64Assembler::movsd(XmmRegister dst, const Address& src) {
564 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
565 EmitUint8(0xF2);
566 EmitOptionalRex32(dst, src);
567 EmitUint8(0x0F);
568 EmitUint8(0x10);
569 EmitOperand(dst.LowBits(), src);
570 }
571
572
movsd(const Address & dst,XmmRegister src)573 void X86_64Assembler::movsd(const Address& dst, XmmRegister src) {
574 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
575 EmitUint8(0xF2);
576 EmitOptionalRex32(src, dst);
577 EmitUint8(0x0F);
578 EmitUint8(0x11);
579 EmitOperand(src.LowBits(), dst);
580 }
581
582
movsd(XmmRegister dst,XmmRegister src)583 void X86_64Assembler::movsd(XmmRegister dst, XmmRegister src) {
584 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
585 EmitUint8(0xF2);
586 EmitOptionalRex32(src, dst); // Movsd is MR encoding instead of the usual RM.
587 EmitUint8(0x0F);
588 EmitUint8(0x11);
589 EmitXmmRegisterOperand(src.LowBits(), dst);
590 }
591
592
addsd(XmmRegister dst,XmmRegister src)593 void X86_64Assembler::addsd(XmmRegister dst, XmmRegister src) {
594 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
595 EmitUint8(0xF2);
596 EmitOptionalRex32(dst, src);
597 EmitUint8(0x0F);
598 EmitUint8(0x58);
599 EmitXmmRegisterOperand(dst.LowBits(), src);
600 }
601
602
addsd(XmmRegister dst,const Address & src)603 void X86_64Assembler::addsd(XmmRegister dst, const Address& src) {
604 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
605 EmitUint8(0xF2);
606 EmitOptionalRex32(dst, src);
607 EmitUint8(0x0F);
608 EmitUint8(0x58);
609 EmitOperand(dst.LowBits(), src);
610 }
611
612
subsd(XmmRegister dst,XmmRegister src)613 void X86_64Assembler::subsd(XmmRegister dst, XmmRegister src) {
614 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
615 EmitUint8(0xF2);
616 EmitOptionalRex32(dst, src);
617 EmitUint8(0x0F);
618 EmitUint8(0x5C);
619 EmitXmmRegisterOperand(dst.LowBits(), src);
620 }
621
622
subsd(XmmRegister dst,const Address & src)623 void X86_64Assembler::subsd(XmmRegister dst, const Address& src) {
624 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
625 EmitUint8(0xF2);
626 EmitOptionalRex32(dst, src);
627 EmitUint8(0x0F);
628 EmitUint8(0x5C);
629 EmitOperand(dst.LowBits(), src);
630 }
631
632
mulsd(XmmRegister dst,XmmRegister src)633 void X86_64Assembler::mulsd(XmmRegister dst, XmmRegister src) {
634 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
635 EmitUint8(0xF2);
636 EmitOptionalRex32(dst, src);
637 EmitUint8(0x0F);
638 EmitUint8(0x59);
639 EmitXmmRegisterOperand(dst.LowBits(), src);
640 }
641
642
mulsd(XmmRegister dst,const Address & src)643 void X86_64Assembler::mulsd(XmmRegister dst, const Address& src) {
644 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
645 EmitUint8(0xF2);
646 EmitOptionalRex32(dst, src);
647 EmitUint8(0x0F);
648 EmitUint8(0x59);
649 EmitOperand(dst.LowBits(), src);
650 }
651
652
divsd(XmmRegister dst,XmmRegister src)653 void X86_64Assembler::divsd(XmmRegister dst, XmmRegister src) {
654 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
655 EmitUint8(0xF2);
656 EmitOptionalRex32(dst, src);
657 EmitUint8(0x0F);
658 EmitUint8(0x5E);
659 EmitXmmRegisterOperand(dst.LowBits(), src);
660 }
661
662
divsd(XmmRegister dst,const Address & src)663 void X86_64Assembler::divsd(XmmRegister dst, const Address& src) {
664 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
665 EmitUint8(0xF2);
666 EmitOptionalRex32(dst, src);
667 EmitUint8(0x0F);
668 EmitUint8(0x5E);
669 EmitOperand(dst.LowBits(), src);
670 }
671
672
cvtsi2ss(XmmRegister dst,CpuRegister src)673 void X86_64Assembler::cvtsi2ss(XmmRegister dst, CpuRegister src) {
674 cvtsi2ss(dst, src, false);
675 }
676
677
cvtsi2ss(XmmRegister dst,CpuRegister src,bool is64bit)678 void X86_64Assembler::cvtsi2ss(XmmRegister dst, CpuRegister src, bool is64bit) {
679 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
680 EmitUint8(0xF3);
681 if (is64bit) {
682 // Emit a REX.W prefix if the operand size is 64 bits.
683 EmitRex64(dst, src);
684 } else {
685 EmitOptionalRex32(dst, src);
686 }
687 EmitUint8(0x0F);
688 EmitUint8(0x2A);
689 EmitOperand(dst.LowBits(), Operand(src));
690 }
691
692
cvtsi2ss(XmmRegister dst,const Address & src,bool is64bit)693 void X86_64Assembler::cvtsi2ss(XmmRegister dst, const Address& src, bool is64bit) {
694 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
695 EmitUint8(0xF3);
696 if (is64bit) {
697 // Emit a REX.W prefix if the operand size is 64 bits.
698 EmitRex64(dst, src);
699 } else {
700 EmitOptionalRex32(dst, src);
701 }
702 EmitUint8(0x0F);
703 EmitUint8(0x2A);
704 EmitOperand(dst.LowBits(), src);
705 }
706
707
cvtsi2sd(XmmRegister dst,CpuRegister src)708 void X86_64Assembler::cvtsi2sd(XmmRegister dst, CpuRegister src) {
709 cvtsi2sd(dst, src, false);
710 }
711
712
cvtsi2sd(XmmRegister dst,CpuRegister src,bool is64bit)713 void X86_64Assembler::cvtsi2sd(XmmRegister dst, CpuRegister src, bool is64bit) {
714 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
715 EmitUint8(0xF2);
716 if (is64bit) {
717 // Emit a REX.W prefix if the operand size is 64 bits.
718 EmitRex64(dst, src);
719 } else {
720 EmitOptionalRex32(dst, src);
721 }
722 EmitUint8(0x0F);
723 EmitUint8(0x2A);
724 EmitOperand(dst.LowBits(), Operand(src));
725 }
726
727
cvtsi2sd(XmmRegister dst,const Address & src,bool is64bit)728 void X86_64Assembler::cvtsi2sd(XmmRegister dst, const Address& src, bool is64bit) {
729 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
730 EmitUint8(0xF2);
731 if (is64bit) {
732 // Emit a REX.W prefix if the operand size is 64 bits.
733 EmitRex64(dst, src);
734 } else {
735 EmitOptionalRex32(dst, src);
736 }
737 EmitUint8(0x0F);
738 EmitUint8(0x2A);
739 EmitOperand(dst.LowBits(), src);
740 }
741
742
cvtss2si(CpuRegister dst,XmmRegister src)743 void X86_64Assembler::cvtss2si(CpuRegister dst, XmmRegister src) {
744 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
745 EmitUint8(0xF3);
746 EmitOptionalRex32(dst, src);
747 EmitUint8(0x0F);
748 EmitUint8(0x2D);
749 EmitXmmRegisterOperand(dst.LowBits(), src);
750 }
751
752
cvtss2sd(XmmRegister dst,XmmRegister src)753 void X86_64Assembler::cvtss2sd(XmmRegister dst, XmmRegister src) {
754 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
755 EmitUint8(0xF3);
756 EmitOptionalRex32(dst, src);
757 EmitUint8(0x0F);
758 EmitUint8(0x5A);
759 EmitXmmRegisterOperand(dst.LowBits(), src);
760 }
761
762
cvtss2sd(XmmRegister dst,const Address & src)763 void X86_64Assembler::cvtss2sd(XmmRegister dst, const Address& src) {
764 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
765 EmitUint8(0xF3);
766 EmitOptionalRex32(dst, src);
767 EmitUint8(0x0F);
768 EmitUint8(0x5A);
769 EmitOperand(dst.LowBits(), src);
770 }
771
772
cvtsd2si(CpuRegister dst,XmmRegister src)773 void X86_64Assembler::cvtsd2si(CpuRegister dst, XmmRegister src) {
774 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
775 EmitUint8(0xF2);
776 EmitOptionalRex32(dst, src);
777 EmitUint8(0x0F);
778 EmitUint8(0x2D);
779 EmitXmmRegisterOperand(dst.LowBits(), src);
780 }
781
782
cvttss2si(CpuRegister dst,XmmRegister src)783 void X86_64Assembler::cvttss2si(CpuRegister dst, XmmRegister src) {
784 cvttss2si(dst, src, false);
785 }
786
787
cvttss2si(CpuRegister dst,XmmRegister src,bool is64bit)788 void X86_64Assembler::cvttss2si(CpuRegister dst, XmmRegister src, bool is64bit) {
789 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
790 EmitUint8(0xF3);
791 if (is64bit) {
792 // Emit a REX.W prefix if the operand size is 64 bits.
793 EmitRex64(dst, src);
794 } else {
795 EmitOptionalRex32(dst, src);
796 }
797 EmitUint8(0x0F);
798 EmitUint8(0x2C);
799 EmitXmmRegisterOperand(dst.LowBits(), src);
800 }
801
802
cvttsd2si(CpuRegister dst,XmmRegister src)803 void X86_64Assembler::cvttsd2si(CpuRegister dst, XmmRegister src) {
804 cvttsd2si(dst, src, false);
805 }
806
807
cvttsd2si(CpuRegister dst,XmmRegister src,bool is64bit)808 void X86_64Assembler::cvttsd2si(CpuRegister dst, XmmRegister src, bool is64bit) {
809 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
810 EmitUint8(0xF2);
811 if (is64bit) {
812 // Emit a REX.W prefix if the operand size is 64 bits.
813 EmitRex64(dst, src);
814 } else {
815 EmitOptionalRex32(dst, src);
816 }
817 EmitUint8(0x0F);
818 EmitUint8(0x2C);
819 EmitXmmRegisterOperand(dst.LowBits(), src);
820 }
821
822
cvtsd2ss(XmmRegister dst,XmmRegister src)823 void X86_64Assembler::cvtsd2ss(XmmRegister dst, XmmRegister src) {
824 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
825 EmitUint8(0xF2);
826 EmitOptionalRex32(dst, src);
827 EmitUint8(0x0F);
828 EmitUint8(0x5A);
829 EmitXmmRegisterOperand(dst.LowBits(), src);
830 }
831
832
cvtsd2ss(XmmRegister dst,const Address & src)833 void X86_64Assembler::cvtsd2ss(XmmRegister dst, const Address& src) {
834 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
835 EmitUint8(0xF2);
836 EmitOptionalRex32(dst, src);
837 EmitUint8(0x0F);
838 EmitUint8(0x5A);
839 EmitOperand(dst.LowBits(), src);
840 }
841
842
cvtdq2pd(XmmRegister dst,XmmRegister src)843 void X86_64Assembler::cvtdq2pd(XmmRegister dst, XmmRegister src) {
844 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
845 EmitUint8(0xF3);
846 EmitOptionalRex32(dst, src);
847 EmitUint8(0x0F);
848 EmitUint8(0xE6);
849 EmitXmmRegisterOperand(dst.LowBits(), src);
850 }
851
852
comiss(XmmRegister a,XmmRegister b)853 void X86_64Assembler::comiss(XmmRegister a, XmmRegister b) {
854 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
855 EmitOptionalRex32(a, b);
856 EmitUint8(0x0F);
857 EmitUint8(0x2F);
858 EmitXmmRegisterOperand(a.LowBits(), b);
859 }
860
861
comiss(XmmRegister a,const Address & b)862 void X86_64Assembler::comiss(XmmRegister a, const Address& b) {
863 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
864 EmitOptionalRex32(a, b);
865 EmitUint8(0x0F);
866 EmitUint8(0x2F);
867 EmitOperand(a.LowBits(), b);
868 }
869
870
comisd(XmmRegister a,XmmRegister b)871 void X86_64Assembler::comisd(XmmRegister a, XmmRegister b) {
872 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
873 EmitUint8(0x66);
874 EmitOptionalRex32(a, b);
875 EmitUint8(0x0F);
876 EmitUint8(0x2F);
877 EmitXmmRegisterOperand(a.LowBits(), b);
878 }
879
880
comisd(XmmRegister a,const Address & b)881 void X86_64Assembler::comisd(XmmRegister a, const Address& b) {
882 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
883 EmitUint8(0x66);
884 EmitOptionalRex32(a, b);
885 EmitUint8(0x0F);
886 EmitUint8(0x2F);
887 EmitOperand(a.LowBits(), b);
888 }
889
890
ucomiss(XmmRegister a,XmmRegister b)891 void X86_64Assembler::ucomiss(XmmRegister a, XmmRegister b) {
892 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
893 EmitOptionalRex32(a, b);
894 EmitUint8(0x0F);
895 EmitUint8(0x2E);
896 EmitXmmRegisterOperand(a.LowBits(), b);
897 }
898
899
ucomiss(XmmRegister a,const Address & b)900 void X86_64Assembler::ucomiss(XmmRegister a, const Address& b) {
901 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
902 EmitOptionalRex32(a, b);
903 EmitUint8(0x0F);
904 EmitUint8(0x2E);
905 EmitOperand(a.LowBits(), b);
906 }
907
908
ucomisd(XmmRegister a,XmmRegister b)909 void X86_64Assembler::ucomisd(XmmRegister a, XmmRegister b) {
910 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
911 EmitUint8(0x66);
912 EmitOptionalRex32(a, b);
913 EmitUint8(0x0F);
914 EmitUint8(0x2E);
915 EmitXmmRegisterOperand(a.LowBits(), b);
916 }
917
918
ucomisd(XmmRegister a,const Address & b)919 void X86_64Assembler::ucomisd(XmmRegister a, const Address& b) {
920 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
921 EmitUint8(0x66);
922 EmitOptionalRex32(a, b);
923 EmitUint8(0x0F);
924 EmitUint8(0x2E);
925 EmitOperand(a.LowBits(), b);
926 }
927
928
roundsd(XmmRegister dst,XmmRegister src,const Immediate & imm)929 void X86_64Assembler::roundsd(XmmRegister dst, XmmRegister src, const Immediate& imm) {
930 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
931 EmitUint8(0x66);
932 EmitOptionalRex32(dst, src);
933 EmitUint8(0x0F);
934 EmitUint8(0x3A);
935 EmitUint8(0x0B);
936 EmitXmmRegisterOperand(dst.LowBits(), src);
937 EmitUint8(imm.value());
938 }
939
940
roundss(XmmRegister dst,XmmRegister src,const Immediate & imm)941 void X86_64Assembler::roundss(XmmRegister dst, XmmRegister src, const Immediate& imm) {
942 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
943 EmitUint8(0x66);
944 EmitOptionalRex32(dst, src);
945 EmitUint8(0x0F);
946 EmitUint8(0x3A);
947 EmitUint8(0x0A);
948 EmitXmmRegisterOperand(dst.LowBits(), src);
949 EmitUint8(imm.value());
950 }
951
952
sqrtsd(XmmRegister dst,XmmRegister src)953 void X86_64Assembler::sqrtsd(XmmRegister dst, XmmRegister src) {
954 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
955 EmitUint8(0xF2);
956 EmitOptionalRex32(dst, src);
957 EmitUint8(0x0F);
958 EmitUint8(0x51);
959 EmitXmmRegisterOperand(dst.LowBits(), src);
960 }
961
962
sqrtss(XmmRegister dst,XmmRegister src)963 void X86_64Assembler::sqrtss(XmmRegister dst, XmmRegister src) {
964 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
965 EmitUint8(0xF3);
966 EmitOptionalRex32(dst, src);
967 EmitUint8(0x0F);
968 EmitUint8(0x51);
969 EmitXmmRegisterOperand(dst.LowBits(), src);
970 }
971
972
xorpd(XmmRegister dst,const Address & src)973 void X86_64Assembler::xorpd(XmmRegister dst, const Address& src) {
974 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
975 EmitUint8(0x66);
976 EmitOptionalRex32(dst, src);
977 EmitUint8(0x0F);
978 EmitUint8(0x57);
979 EmitOperand(dst.LowBits(), src);
980 }
981
982
xorpd(XmmRegister dst,XmmRegister src)983 void X86_64Assembler::xorpd(XmmRegister dst, XmmRegister src) {
984 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
985 EmitUint8(0x66);
986 EmitOptionalRex32(dst, src);
987 EmitUint8(0x0F);
988 EmitUint8(0x57);
989 EmitXmmRegisterOperand(dst.LowBits(), src);
990 }
991
992
xorps(XmmRegister dst,const Address & src)993 void X86_64Assembler::xorps(XmmRegister dst, const Address& src) {
994 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
995 EmitOptionalRex32(dst, src);
996 EmitUint8(0x0F);
997 EmitUint8(0x57);
998 EmitOperand(dst.LowBits(), src);
999 }
1000
1001
xorps(XmmRegister dst,XmmRegister src)1002 void X86_64Assembler::xorps(XmmRegister dst, XmmRegister src) {
1003 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1004 EmitOptionalRex32(dst, src);
1005 EmitUint8(0x0F);
1006 EmitUint8(0x57);
1007 EmitXmmRegisterOperand(dst.LowBits(), src);
1008 }
1009
1010
andpd(XmmRegister dst,const Address & src)1011 void X86_64Assembler::andpd(XmmRegister dst, const Address& src) {
1012 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1013 EmitUint8(0x66);
1014 EmitOptionalRex32(dst, src);
1015 EmitUint8(0x0F);
1016 EmitUint8(0x54);
1017 EmitOperand(dst.LowBits(), src);
1018 }
1019
andpd(XmmRegister dst,XmmRegister src)1020 void X86_64Assembler::andpd(XmmRegister dst, XmmRegister src) {
1021 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1022 EmitUint8(0x66);
1023 EmitOptionalRex32(dst, src);
1024 EmitUint8(0x0F);
1025 EmitUint8(0x54);
1026 EmitXmmRegisterOperand(dst.LowBits(), src);
1027 }
1028
andps(XmmRegister dst,XmmRegister src)1029 void X86_64Assembler::andps(XmmRegister dst, XmmRegister src) {
1030 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1031 EmitOptionalRex32(dst, src);
1032 EmitUint8(0x0F);
1033 EmitUint8(0x54);
1034 EmitXmmRegisterOperand(dst.LowBits(), src);
1035 }
1036
orpd(XmmRegister dst,XmmRegister src)1037 void X86_64Assembler::orpd(XmmRegister dst, XmmRegister src) {
1038 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1039 EmitUint8(0x66);
1040 EmitOptionalRex32(dst, src);
1041 EmitUint8(0x0F);
1042 EmitUint8(0x56);
1043 EmitXmmRegisterOperand(dst.LowBits(), src);
1044 }
1045
orps(XmmRegister dst,XmmRegister src)1046 void X86_64Assembler::orps(XmmRegister dst, XmmRegister src) {
1047 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1048 EmitOptionalRex32(dst, src);
1049 EmitUint8(0x0F);
1050 EmitUint8(0x56);
1051 EmitXmmRegisterOperand(dst.LowBits(), src);
1052 }
1053
fldl(const Address & src)1054 void X86_64Assembler::fldl(const Address& src) {
1055 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1056 EmitUint8(0xDD);
1057 EmitOperand(0, src);
1058 }
1059
1060
fstl(const Address & dst)1061 void X86_64Assembler::fstl(const Address& dst) {
1062 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1063 EmitUint8(0xDD);
1064 EmitOperand(2, dst);
1065 }
1066
1067
fstpl(const Address & dst)1068 void X86_64Assembler::fstpl(const Address& dst) {
1069 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1070 EmitUint8(0xDD);
1071 EmitOperand(3, dst);
1072 }
1073
1074
fstsw()1075 void X86_64Assembler::fstsw() {
1076 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1077 EmitUint8(0x9B);
1078 EmitUint8(0xDF);
1079 EmitUint8(0xE0);
1080 }
1081
1082
fnstcw(const Address & dst)1083 void X86_64Assembler::fnstcw(const Address& dst) {
1084 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1085 EmitUint8(0xD9);
1086 EmitOperand(7, dst);
1087 }
1088
1089
fldcw(const Address & src)1090 void X86_64Assembler::fldcw(const Address& src) {
1091 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1092 EmitUint8(0xD9);
1093 EmitOperand(5, src);
1094 }
1095
1096
fistpl(const Address & dst)1097 void X86_64Assembler::fistpl(const Address& dst) {
1098 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1099 EmitUint8(0xDF);
1100 EmitOperand(7, dst);
1101 }
1102
1103
fistps(const Address & dst)1104 void X86_64Assembler::fistps(const Address& dst) {
1105 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1106 EmitUint8(0xDB);
1107 EmitOperand(3, dst);
1108 }
1109
1110
fildl(const Address & src)1111 void X86_64Assembler::fildl(const Address& src) {
1112 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1113 EmitUint8(0xDF);
1114 EmitOperand(5, src);
1115 }
1116
1117
filds(const Address & src)1118 void X86_64Assembler::filds(const Address& src) {
1119 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1120 EmitUint8(0xDB);
1121 EmitOperand(0, src);
1122 }
1123
1124
fincstp()1125 void X86_64Assembler::fincstp() {
1126 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1127 EmitUint8(0xD9);
1128 EmitUint8(0xF7);
1129 }
1130
1131
ffree(const Immediate & index)1132 void X86_64Assembler::ffree(const Immediate& index) {
1133 CHECK_LT(index.value(), 7);
1134 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1135 EmitUint8(0xDD);
1136 EmitUint8(0xC0 + index.value());
1137 }
1138
1139
fsin()1140 void X86_64Assembler::fsin() {
1141 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1142 EmitUint8(0xD9);
1143 EmitUint8(0xFE);
1144 }
1145
1146
fcos()1147 void X86_64Assembler::fcos() {
1148 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1149 EmitUint8(0xD9);
1150 EmitUint8(0xFF);
1151 }
1152
1153
fptan()1154 void X86_64Assembler::fptan() {
1155 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1156 EmitUint8(0xD9);
1157 EmitUint8(0xF2);
1158 }
1159
fucompp()1160 void X86_64Assembler::fucompp() {
1161 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1162 EmitUint8(0xDA);
1163 EmitUint8(0xE9);
1164 }
1165
1166
fprem()1167 void X86_64Assembler::fprem() {
1168 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1169 EmitUint8(0xD9);
1170 EmitUint8(0xF8);
1171 }
1172
1173
xchgl(CpuRegister dst,CpuRegister src)1174 void X86_64Assembler::xchgl(CpuRegister dst, CpuRegister src) {
1175 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1176 // There is a short version for rax.
1177 // It's a bit awkward, as CpuRegister has a const field, so assignment and thus swapping doesn't
1178 // work.
1179 const bool src_rax = src.AsRegister() == RAX;
1180 const bool dst_rax = dst.AsRegister() == RAX;
1181 if (src_rax || dst_rax) {
1182 EmitOptionalRex32(src_rax ? dst : src);
1183 EmitUint8(0x90 + (src_rax ? dst.LowBits() : src.LowBits()));
1184 return;
1185 }
1186
1187 // General case.
1188 EmitOptionalRex32(src, dst);
1189 EmitUint8(0x87);
1190 EmitRegisterOperand(src.LowBits(), dst.LowBits());
1191 }
1192
1193
xchgq(CpuRegister dst,CpuRegister src)1194 void X86_64Assembler::xchgq(CpuRegister dst, CpuRegister src) {
1195 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1196 // There is a short version for rax.
1197 // It's a bit awkward, as CpuRegister has a const field, so assignment and thus swapping doesn't
1198 // work.
1199 const bool src_rax = src.AsRegister() == RAX;
1200 const bool dst_rax = dst.AsRegister() == RAX;
1201 if (src_rax || dst_rax) {
1202 // If src == target, emit a nop instead.
1203 if (src_rax && dst_rax) {
1204 EmitUint8(0x90);
1205 } else {
1206 EmitRex64(src_rax ? dst : src);
1207 EmitUint8(0x90 + (src_rax ? dst.LowBits() : src.LowBits()));
1208 }
1209 return;
1210 }
1211
1212 // General case.
1213 EmitRex64(src, dst);
1214 EmitUint8(0x87);
1215 EmitRegisterOperand(src.LowBits(), dst.LowBits());
1216 }
1217
1218
xchgl(CpuRegister reg,const Address & address)1219 void X86_64Assembler::xchgl(CpuRegister reg, const Address& address) {
1220 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1221 EmitOptionalRex32(reg, address);
1222 EmitUint8(0x87);
1223 EmitOperand(reg.LowBits(), address);
1224 }
1225
1226
cmpw(const Address & address,const Immediate & imm)1227 void X86_64Assembler::cmpw(const Address& address, const Immediate& imm) {
1228 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1229 CHECK(imm.is_int32());
1230 EmitOperandSizeOverride();
1231 EmitOptionalRex32(address);
1232 EmitComplex(7, address, imm);
1233 }
1234
1235
cmpl(CpuRegister reg,const Immediate & imm)1236 void X86_64Assembler::cmpl(CpuRegister reg, const Immediate& imm) {
1237 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1238 CHECK(imm.is_int32());
1239 EmitOptionalRex32(reg);
1240 EmitComplex(7, Operand(reg), imm);
1241 }
1242
1243
cmpl(CpuRegister reg0,CpuRegister reg1)1244 void X86_64Assembler::cmpl(CpuRegister reg0, CpuRegister reg1) {
1245 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1246 EmitOptionalRex32(reg0, reg1);
1247 EmitUint8(0x3B);
1248 EmitOperand(reg0.LowBits(), Operand(reg1));
1249 }
1250
1251
cmpl(CpuRegister reg,const Address & address)1252 void X86_64Assembler::cmpl(CpuRegister reg, const Address& address) {
1253 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1254 EmitOptionalRex32(reg, address);
1255 EmitUint8(0x3B);
1256 EmitOperand(reg.LowBits(), address);
1257 }
1258
1259
cmpl(const Address & address,CpuRegister reg)1260 void X86_64Assembler::cmpl(const Address& address, CpuRegister reg) {
1261 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1262 EmitOptionalRex32(reg, address);
1263 EmitUint8(0x39);
1264 EmitOperand(reg.LowBits(), address);
1265 }
1266
1267
cmpl(const Address & address,const Immediate & imm)1268 void X86_64Assembler::cmpl(const Address& address, const Immediate& imm) {
1269 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1270 CHECK(imm.is_int32());
1271 EmitOptionalRex32(address);
1272 EmitComplex(7, address, imm);
1273 }
1274
1275
cmpq(CpuRegister reg0,CpuRegister reg1)1276 void X86_64Assembler::cmpq(CpuRegister reg0, CpuRegister reg1) {
1277 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1278 EmitRex64(reg0, reg1);
1279 EmitUint8(0x3B);
1280 EmitOperand(reg0.LowBits(), Operand(reg1));
1281 }
1282
1283
cmpq(CpuRegister reg,const Immediate & imm)1284 void X86_64Assembler::cmpq(CpuRegister reg, const Immediate& imm) {
1285 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1286 CHECK(imm.is_int32()); // cmpq only supports 32b immediate.
1287 EmitRex64(reg);
1288 EmitComplex(7, Operand(reg), imm);
1289 }
1290
1291
cmpq(CpuRegister reg,const Address & address)1292 void X86_64Assembler::cmpq(CpuRegister reg, const Address& address) {
1293 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1294 EmitRex64(reg, address);
1295 EmitUint8(0x3B);
1296 EmitOperand(reg.LowBits(), address);
1297 }
1298
1299
cmpq(const Address & address,const Immediate & imm)1300 void X86_64Assembler::cmpq(const Address& address, const Immediate& imm) {
1301 CHECK(imm.is_int32()); // cmpq only supports 32b immediate.
1302 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1303 EmitRex64(address);
1304 EmitComplex(7, address, imm);
1305 }
1306
1307
addl(CpuRegister dst,CpuRegister src)1308 void X86_64Assembler::addl(CpuRegister dst, CpuRegister src) {
1309 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1310 EmitOptionalRex32(dst, src);
1311 EmitUint8(0x03);
1312 EmitRegisterOperand(dst.LowBits(), src.LowBits());
1313 }
1314
1315
addl(CpuRegister reg,const Address & address)1316 void X86_64Assembler::addl(CpuRegister reg, const Address& address) {
1317 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1318 EmitOptionalRex32(reg, address);
1319 EmitUint8(0x03);
1320 EmitOperand(reg.LowBits(), address);
1321 }
1322
1323
testl(CpuRegister reg1,CpuRegister reg2)1324 void X86_64Assembler::testl(CpuRegister reg1, CpuRegister reg2) {
1325 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1326 EmitOptionalRex32(reg1, reg2);
1327 EmitUint8(0x85);
1328 EmitRegisterOperand(reg1.LowBits(), reg2.LowBits());
1329 }
1330
1331
testl(CpuRegister reg,const Address & address)1332 void X86_64Assembler::testl(CpuRegister reg, const Address& address) {
1333 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1334 EmitOptionalRex32(reg, address);
1335 EmitUint8(0x85);
1336 EmitOperand(reg.LowBits(), address);
1337 }
1338
1339
testl(CpuRegister reg,const Immediate & immediate)1340 void X86_64Assembler::testl(CpuRegister reg, const Immediate& immediate) {
1341 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1342 // For registers that have a byte variant (RAX, RBX, RCX, and RDX)
1343 // we only test the byte CpuRegister to keep the encoding short.
1344 if (immediate.is_uint8() && reg.AsRegister() < 4) {
1345 // Use zero-extended 8-bit immediate.
1346 if (reg.AsRegister() == RAX) {
1347 EmitUint8(0xA8);
1348 } else {
1349 EmitUint8(0xF6);
1350 EmitUint8(0xC0 + reg.AsRegister());
1351 }
1352 EmitUint8(immediate.value() & 0xFF);
1353 } else if (reg.AsRegister() == RAX) {
1354 // Use short form if the destination is RAX.
1355 EmitUint8(0xA9);
1356 EmitImmediate(immediate);
1357 } else {
1358 EmitOptionalRex32(reg);
1359 EmitUint8(0xF7);
1360 EmitOperand(0, Operand(reg));
1361 EmitImmediate(immediate);
1362 }
1363 }
1364
1365
testq(CpuRegister reg1,CpuRegister reg2)1366 void X86_64Assembler::testq(CpuRegister reg1, CpuRegister reg2) {
1367 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1368 EmitRex64(reg1, reg2);
1369 EmitUint8(0x85);
1370 EmitRegisterOperand(reg1.LowBits(), reg2.LowBits());
1371 }
1372
1373
testq(CpuRegister reg,const Address & address)1374 void X86_64Assembler::testq(CpuRegister reg, const Address& address) {
1375 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1376 EmitRex64(reg, address);
1377 EmitUint8(0x85);
1378 EmitOperand(reg.LowBits(), address);
1379 }
1380
1381
andl(CpuRegister dst,CpuRegister src)1382 void X86_64Assembler::andl(CpuRegister dst, CpuRegister src) {
1383 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1384 EmitOptionalRex32(dst, src);
1385 EmitUint8(0x23);
1386 EmitOperand(dst.LowBits(), Operand(src));
1387 }
1388
1389
andl(CpuRegister reg,const Address & address)1390 void X86_64Assembler::andl(CpuRegister reg, const Address& address) {
1391 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1392 EmitOptionalRex32(reg, address);
1393 EmitUint8(0x23);
1394 EmitOperand(reg.LowBits(), address);
1395 }
1396
1397
andl(CpuRegister dst,const Immediate & imm)1398 void X86_64Assembler::andl(CpuRegister dst, const Immediate& imm) {
1399 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1400 EmitOptionalRex32(dst);
1401 EmitComplex(4, Operand(dst), imm);
1402 }
1403
1404
andq(CpuRegister reg,const Immediate & imm)1405 void X86_64Assembler::andq(CpuRegister reg, const Immediate& imm) {
1406 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1407 CHECK(imm.is_int32()); // andq only supports 32b immediate.
1408 EmitRex64(reg);
1409 EmitComplex(4, Operand(reg), imm);
1410 }
1411
1412
andq(CpuRegister dst,CpuRegister src)1413 void X86_64Assembler::andq(CpuRegister dst, CpuRegister src) {
1414 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1415 EmitRex64(dst, src);
1416 EmitUint8(0x23);
1417 EmitOperand(dst.LowBits(), Operand(src));
1418 }
1419
1420
andq(CpuRegister dst,const Address & src)1421 void X86_64Assembler::andq(CpuRegister dst, const Address& src) {
1422 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1423 EmitRex64(dst, src);
1424 EmitUint8(0x23);
1425 EmitOperand(dst.LowBits(), src);
1426 }
1427
1428
orl(CpuRegister dst,CpuRegister src)1429 void X86_64Assembler::orl(CpuRegister dst, CpuRegister src) {
1430 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1431 EmitOptionalRex32(dst, src);
1432 EmitUint8(0x0B);
1433 EmitOperand(dst.LowBits(), Operand(src));
1434 }
1435
1436
orl(CpuRegister reg,const Address & address)1437 void X86_64Assembler::orl(CpuRegister reg, const Address& address) {
1438 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1439 EmitOptionalRex32(reg, address);
1440 EmitUint8(0x0B);
1441 EmitOperand(reg.LowBits(), address);
1442 }
1443
1444
orl(CpuRegister dst,const Immediate & imm)1445 void X86_64Assembler::orl(CpuRegister dst, const Immediate& imm) {
1446 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1447 EmitOptionalRex32(dst);
1448 EmitComplex(1, Operand(dst), imm);
1449 }
1450
1451
orq(CpuRegister dst,const Immediate & imm)1452 void X86_64Assembler::orq(CpuRegister dst, const Immediate& imm) {
1453 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1454 CHECK(imm.is_int32()); // orq only supports 32b immediate.
1455 EmitRex64(dst);
1456 EmitComplex(1, Operand(dst), imm);
1457 }
1458
1459
orq(CpuRegister dst,CpuRegister src)1460 void X86_64Assembler::orq(CpuRegister dst, CpuRegister src) {
1461 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1462 EmitRex64(dst, src);
1463 EmitUint8(0x0B);
1464 EmitOperand(dst.LowBits(), Operand(src));
1465 }
1466
1467
orq(CpuRegister dst,const Address & src)1468 void X86_64Assembler::orq(CpuRegister dst, const Address& src) {
1469 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1470 EmitRex64(dst, src);
1471 EmitUint8(0x0B);
1472 EmitOperand(dst.LowBits(), src);
1473 }
1474
1475
xorl(CpuRegister dst,CpuRegister src)1476 void X86_64Assembler::xorl(CpuRegister dst, CpuRegister src) {
1477 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1478 EmitOptionalRex32(dst, src);
1479 EmitUint8(0x33);
1480 EmitOperand(dst.LowBits(), Operand(src));
1481 }
1482
1483
xorl(CpuRegister reg,const Address & address)1484 void X86_64Assembler::xorl(CpuRegister reg, const Address& address) {
1485 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1486 EmitOptionalRex32(reg, address);
1487 EmitUint8(0x33);
1488 EmitOperand(reg.LowBits(), address);
1489 }
1490
1491
xorl(CpuRegister dst,const Immediate & imm)1492 void X86_64Assembler::xorl(CpuRegister dst, const Immediate& imm) {
1493 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1494 EmitOptionalRex32(dst);
1495 EmitComplex(6, Operand(dst), imm);
1496 }
1497
1498
xorq(CpuRegister dst,CpuRegister src)1499 void X86_64Assembler::xorq(CpuRegister dst, CpuRegister src) {
1500 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1501 EmitRex64(dst, src);
1502 EmitUint8(0x33);
1503 EmitOperand(dst.LowBits(), Operand(src));
1504 }
1505
1506
xorq(CpuRegister dst,const Immediate & imm)1507 void X86_64Assembler::xorq(CpuRegister dst, const Immediate& imm) {
1508 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1509 CHECK(imm.is_int32()); // xorq only supports 32b immediate.
1510 EmitRex64(dst);
1511 EmitComplex(6, Operand(dst), imm);
1512 }
1513
xorq(CpuRegister dst,const Address & src)1514 void X86_64Assembler::xorq(CpuRegister dst, const Address& src) {
1515 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1516 EmitRex64(dst, src);
1517 EmitUint8(0x33);
1518 EmitOperand(dst.LowBits(), src);
1519 }
1520
1521
1522 #if 0
1523 void X86_64Assembler::rex(bool force, bool w, Register* r, Register* x, Register* b) {
1524 // REX.WRXB
1525 // W - 64-bit operand
1526 // R - MODRM.reg
1527 // X - SIB.index
1528 // B - MODRM.rm/SIB.base
1529 uint8_t rex = force ? 0x40 : 0;
1530 if (w) {
1531 rex |= 0x48; // REX.W000
1532 }
1533 if (r != nullptr && *r >= Register::R8 && *r < Register::kNumberOfCpuRegisters) {
1534 rex |= 0x44; // REX.0R00
1535 *r = static_cast<Register>(*r - 8);
1536 }
1537 if (x != nullptr && *x >= Register::R8 && *x < Register::kNumberOfCpuRegisters) {
1538 rex |= 0x42; // REX.00X0
1539 *x = static_cast<Register>(*x - 8);
1540 }
1541 if (b != nullptr && *b >= Register::R8 && *b < Register::kNumberOfCpuRegisters) {
1542 rex |= 0x41; // REX.000B
1543 *b = static_cast<Register>(*b - 8);
1544 }
1545 if (rex != 0) {
1546 EmitUint8(rex);
1547 }
1548 }
1549
1550 void X86_64Assembler::rex_reg_mem(bool force, bool w, Register* dst, const Address& mem) {
1551 // REX.WRXB
1552 // W - 64-bit operand
1553 // R - MODRM.reg
1554 // X - SIB.index
1555 // B - MODRM.rm/SIB.base
1556 uint8_t rex = mem->rex();
1557 if (force) {
1558 rex |= 0x40; // REX.0000
1559 }
1560 if (w) {
1561 rex |= 0x48; // REX.W000
1562 }
1563 if (dst != nullptr && *dst >= Register::R8 && *dst < Register::kNumberOfCpuRegisters) {
1564 rex |= 0x44; // REX.0R00
1565 *dst = static_cast<Register>(*dst - 8);
1566 }
1567 if (rex != 0) {
1568 EmitUint8(rex);
1569 }
1570 }
1571
1572 void rex_mem_reg(bool force, bool w, Address* mem, Register* src);
1573 #endif
1574
addl(CpuRegister reg,const Immediate & imm)1575 void X86_64Assembler::addl(CpuRegister reg, const Immediate& imm) {
1576 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1577 EmitOptionalRex32(reg);
1578 EmitComplex(0, Operand(reg), imm);
1579 }
1580
1581
addq(CpuRegister reg,const Immediate & imm)1582 void X86_64Assembler::addq(CpuRegister reg, const Immediate& imm) {
1583 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1584 CHECK(imm.is_int32()); // addq only supports 32b immediate.
1585 EmitRex64(reg);
1586 EmitComplex(0, Operand(reg), imm);
1587 }
1588
1589
addq(CpuRegister dst,const Address & address)1590 void X86_64Assembler::addq(CpuRegister dst, const Address& address) {
1591 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1592 EmitRex64(dst, address);
1593 EmitUint8(0x03);
1594 EmitOperand(dst.LowBits(), address);
1595 }
1596
1597
addq(CpuRegister dst,CpuRegister src)1598 void X86_64Assembler::addq(CpuRegister dst, CpuRegister src) {
1599 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1600 // 0x01 is addq r/m64 <- r/m64 + r64, with op1 in r/m and op2 in reg: so reverse EmitRex64
1601 EmitRex64(src, dst);
1602 EmitUint8(0x01);
1603 EmitRegisterOperand(src.LowBits(), dst.LowBits());
1604 }
1605
1606
addl(const Address & address,CpuRegister reg)1607 void X86_64Assembler::addl(const Address& address, CpuRegister reg) {
1608 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1609 EmitOptionalRex32(reg, address);
1610 EmitUint8(0x01);
1611 EmitOperand(reg.LowBits(), address);
1612 }
1613
1614
addl(const Address & address,const Immediate & imm)1615 void X86_64Assembler::addl(const Address& address, const Immediate& imm) {
1616 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1617 EmitOptionalRex32(address);
1618 EmitComplex(0, address, imm);
1619 }
1620
1621
subl(CpuRegister dst,CpuRegister src)1622 void X86_64Assembler::subl(CpuRegister dst, CpuRegister src) {
1623 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1624 EmitOptionalRex32(dst, src);
1625 EmitUint8(0x2B);
1626 EmitOperand(dst.LowBits(), Operand(src));
1627 }
1628
1629
subl(CpuRegister reg,const Immediate & imm)1630 void X86_64Assembler::subl(CpuRegister reg, const Immediate& imm) {
1631 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1632 EmitOptionalRex32(reg);
1633 EmitComplex(5, Operand(reg), imm);
1634 }
1635
1636
subq(CpuRegister reg,const Immediate & imm)1637 void X86_64Assembler::subq(CpuRegister reg, const Immediate& imm) {
1638 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1639 CHECK(imm.is_int32()); // subq only supports 32b immediate.
1640 EmitRex64(reg);
1641 EmitComplex(5, Operand(reg), imm);
1642 }
1643
1644
subq(CpuRegister dst,CpuRegister src)1645 void X86_64Assembler::subq(CpuRegister dst, CpuRegister src) {
1646 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1647 EmitRex64(dst, src);
1648 EmitUint8(0x2B);
1649 EmitRegisterOperand(dst.LowBits(), src.LowBits());
1650 }
1651
1652
subq(CpuRegister reg,const Address & address)1653 void X86_64Assembler::subq(CpuRegister reg, const Address& address) {
1654 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1655 EmitRex64(reg, address);
1656 EmitUint8(0x2B);
1657 EmitOperand(reg.LowBits() & 7, address);
1658 }
1659
1660
subl(CpuRegister reg,const Address & address)1661 void X86_64Assembler::subl(CpuRegister reg, const Address& address) {
1662 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1663 EmitOptionalRex32(reg, address);
1664 EmitUint8(0x2B);
1665 EmitOperand(reg.LowBits(), address);
1666 }
1667
1668
cdq()1669 void X86_64Assembler::cdq() {
1670 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1671 EmitUint8(0x99);
1672 }
1673
1674
cqo()1675 void X86_64Assembler::cqo() {
1676 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1677 EmitRex64();
1678 EmitUint8(0x99);
1679 }
1680
1681
idivl(CpuRegister reg)1682 void X86_64Assembler::idivl(CpuRegister reg) {
1683 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1684 EmitOptionalRex32(reg);
1685 EmitUint8(0xF7);
1686 EmitUint8(0xF8 | reg.LowBits());
1687 }
1688
1689
idivq(CpuRegister reg)1690 void X86_64Assembler::idivq(CpuRegister reg) {
1691 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1692 EmitRex64(reg);
1693 EmitUint8(0xF7);
1694 EmitUint8(0xF8 | reg.LowBits());
1695 }
1696
1697
imull(CpuRegister dst,CpuRegister src)1698 void X86_64Assembler::imull(CpuRegister dst, CpuRegister src) {
1699 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1700 EmitOptionalRex32(dst, src);
1701 EmitUint8(0x0F);
1702 EmitUint8(0xAF);
1703 EmitOperand(dst.LowBits(), Operand(src));
1704 }
1705
imull(CpuRegister dst,CpuRegister src,const Immediate & imm)1706 void X86_64Assembler::imull(CpuRegister dst, CpuRegister src, const Immediate& imm) {
1707 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1708 CHECK(imm.is_int32()); // imull only supports 32b immediate.
1709
1710 EmitOptionalRex32(dst, src);
1711
1712 // See whether imm can be represented as a sign-extended 8bit value.
1713 int32_t v32 = static_cast<int32_t>(imm.value());
1714 if (IsInt<8>(v32)) {
1715 // Sign-extension works.
1716 EmitUint8(0x6B);
1717 EmitOperand(dst.LowBits(), Operand(src));
1718 EmitUint8(static_cast<uint8_t>(v32 & 0xFF));
1719 } else {
1720 // Not representable, use full immediate.
1721 EmitUint8(0x69);
1722 EmitOperand(dst.LowBits(), Operand(src));
1723 EmitImmediate(imm);
1724 }
1725 }
1726
1727
imull(CpuRegister reg,const Immediate & imm)1728 void X86_64Assembler::imull(CpuRegister reg, const Immediate& imm) {
1729 imull(reg, reg, imm);
1730 }
1731
1732
imull(CpuRegister reg,const Address & address)1733 void X86_64Assembler::imull(CpuRegister reg, const Address& address) {
1734 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1735 EmitOptionalRex32(reg, address);
1736 EmitUint8(0x0F);
1737 EmitUint8(0xAF);
1738 EmitOperand(reg.LowBits(), address);
1739 }
1740
1741
imulq(CpuRegister dst,CpuRegister src)1742 void X86_64Assembler::imulq(CpuRegister dst, CpuRegister src) {
1743 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1744 EmitRex64(dst, src);
1745 EmitUint8(0x0F);
1746 EmitUint8(0xAF);
1747 EmitRegisterOperand(dst.LowBits(), src.LowBits());
1748 }
1749
1750
imulq(CpuRegister reg,const Immediate & imm)1751 void X86_64Assembler::imulq(CpuRegister reg, const Immediate& imm) {
1752 imulq(reg, reg, imm);
1753 }
1754
imulq(CpuRegister dst,CpuRegister reg,const Immediate & imm)1755 void X86_64Assembler::imulq(CpuRegister dst, CpuRegister reg, const Immediate& imm) {
1756 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1757 CHECK(imm.is_int32()); // imulq only supports 32b immediate.
1758
1759 EmitRex64(dst, reg);
1760
1761 // See whether imm can be represented as a sign-extended 8bit value.
1762 int64_t v64 = imm.value();
1763 if (IsInt<8>(v64)) {
1764 // Sign-extension works.
1765 EmitUint8(0x6B);
1766 EmitOperand(dst.LowBits(), Operand(reg));
1767 EmitUint8(static_cast<uint8_t>(v64 & 0xFF));
1768 } else {
1769 // Not representable, use full immediate.
1770 EmitUint8(0x69);
1771 EmitOperand(dst.LowBits(), Operand(reg));
1772 EmitImmediate(imm);
1773 }
1774 }
1775
imulq(CpuRegister reg,const Address & address)1776 void X86_64Assembler::imulq(CpuRegister reg, const Address& address) {
1777 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1778 EmitRex64(reg, address);
1779 EmitUint8(0x0F);
1780 EmitUint8(0xAF);
1781 EmitOperand(reg.LowBits(), address);
1782 }
1783
1784
imull(CpuRegister reg)1785 void X86_64Assembler::imull(CpuRegister reg) {
1786 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1787 EmitOptionalRex32(reg);
1788 EmitUint8(0xF7);
1789 EmitOperand(5, Operand(reg));
1790 }
1791
1792
imulq(CpuRegister reg)1793 void X86_64Assembler::imulq(CpuRegister reg) {
1794 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1795 EmitRex64(reg);
1796 EmitUint8(0xF7);
1797 EmitOperand(5, Operand(reg));
1798 }
1799
1800
imull(const Address & address)1801 void X86_64Assembler::imull(const Address& address) {
1802 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1803 EmitOptionalRex32(address);
1804 EmitUint8(0xF7);
1805 EmitOperand(5, address);
1806 }
1807
1808
mull(CpuRegister reg)1809 void X86_64Assembler::mull(CpuRegister reg) {
1810 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1811 EmitOptionalRex32(reg);
1812 EmitUint8(0xF7);
1813 EmitOperand(4, Operand(reg));
1814 }
1815
1816
mull(const Address & address)1817 void X86_64Assembler::mull(const Address& address) {
1818 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1819 EmitOptionalRex32(address);
1820 EmitUint8(0xF7);
1821 EmitOperand(4, address);
1822 }
1823
1824
shll(CpuRegister reg,const Immediate & imm)1825 void X86_64Assembler::shll(CpuRegister reg, const Immediate& imm) {
1826 EmitGenericShift(false, 4, reg, imm);
1827 }
1828
1829
shlq(CpuRegister reg,const Immediate & imm)1830 void X86_64Assembler::shlq(CpuRegister reg, const Immediate& imm) {
1831 EmitGenericShift(true, 4, reg, imm);
1832 }
1833
1834
shll(CpuRegister operand,CpuRegister shifter)1835 void X86_64Assembler::shll(CpuRegister operand, CpuRegister shifter) {
1836 EmitGenericShift(false, 4, operand, shifter);
1837 }
1838
1839
shlq(CpuRegister operand,CpuRegister shifter)1840 void X86_64Assembler::shlq(CpuRegister operand, CpuRegister shifter) {
1841 EmitGenericShift(true, 4, operand, shifter);
1842 }
1843
1844
shrl(CpuRegister reg,const Immediate & imm)1845 void X86_64Assembler::shrl(CpuRegister reg, const Immediate& imm) {
1846 EmitGenericShift(false, 5, reg, imm);
1847 }
1848
1849
shrq(CpuRegister reg,const Immediate & imm)1850 void X86_64Assembler::shrq(CpuRegister reg, const Immediate& imm) {
1851 EmitGenericShift(true, 5, reg, imm);
1852 }
1853
1854
shrl(CpuRegister operand,CpuRegister shifter)1855 void X86_64Assembler::shrl(CpuRegister operand, CpuRegister shifter) {
1856 EmitGenericShift(false, 5, operand, shifter);
1857 }
1858
1859
shrq(CpuRegister operand,CpuRegister shifter)1860 void X86_64Assembler::shrq(CpuRegister operand, CpuRegister shifter) {
1861 EmitGenericShift(true, 5, operand, shifter);
1862 }
1863
1864
sarl(CpuRegister reg,const Immediate & imm)1865 void X86_64Assembler::sarl(CpuRegister reg, const Immediate& imm) {
1866 EmitGenericShift(false, 7, reg, imm);
1867 }
1868
1869
sarl(CpuRegister operand,CpuRegister shifter)1870 void X86_64Assembler::sarl(CpuRegister operand, CpuRegister shifter) {
1871 EmitGenericShift(false, 7, operand, shifter);
1872 }
1873
1874
sarq(CpuRegister reg,const Immediate & imm)1875 void X86_64Assembler::sarq(CpuRegister reg, const Immediate& imm) {
1876 EmitGenericShift(true, 7, reg, imm);
1877 }
1878
1879
sarq(CpuRegister operand,CpuRegister shifter)1880 void X86_64Assembler::sarq(CpuRegister operand, CpuRegister shifter) {
1881 EmitGenericShift(true, 7, operand, shifter);
1882 }
1883
1884
roll(CpuRegister reg,const Immediate & imm)1885 void X86_64Assembler::roll(CpuRegister reg, const Immediate& imm) {
1886 EmitGenericShift(false, 0, reg, imm);
1887 }
1888
1889
roll(CpuRegister operand,CpuRegister shifter)1890 void X86_64Assembler::roll(CpuRegister operand, CpuRegister shifter) {
1891 EmitGenericShift(false, 0, operand, shifter);
1892 }
1893
1894
rorl(CpuRegister reg,const Immediate & imm)1895 void X86_64Assembler::rorl(CpuRegister reg, const Immediate& imm) {
1896 EmitGenericShift(false, 1, reg, imm);
1897 }
1898
1899
rorl(CpuRegister operand,CpuRegister shifter)1900 void X86_64Assembler::rorl(CpuRegister operand, CpuRegister shifter) {
1901 EmitGenericShift(false, 1, operand, shifter);
1902 }
1903
1904
rolq(CpuRegister reg,const Immediate & imm)1905 void X86_64Assembler::rolq(CpuRegister reg, const Immediate& imm) {
1906 EmitGenericShift(true, 0, reg, imm);
1907 }
1908
1909
rolq(CpuRegister operand,CpuRegister shifter)1910 void X86_64Assembler::rolq(CpuRegister operand, CpuRegister shifter) {
1911 EmitGenericShift(true, 0, operand, shifter);
1912 }
1913
1914
rorq(CpuRegister reg,const Immediate & imm)1915 void X86_64Assembler::rorq(CpuRegister reg, const Immediate& imm) {
1916 EmitGenericShift(true, 1, reg, imm);
1917 }
1918
1919
rorq(CpuRegister operand,CpuRegister shifter)1920 void X86_64Assembler::rorq(CpuRegister operand, CpuRegister shifter) {
1921 EmitGenericShift(true, 1, operand, shifter);
1922 }
1923
1924
negl(CpuRegister reg)1925 void X86_64Assembler::negl(CpuRegister reg) {
1926 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1927 EmitOptionalRex32(reg);
1928 EmitUint8(0xF7);
1929 EmitOperand(3, Operand(reg));
1930 }
1931
1932
negq(CpuRegister reg)1933 void X86_64Assembler::negq(CpuRegister reg) {
1934 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1935 EmitRex64(reg);
1936 EmitUint8(0xF7);
1937 EmitOperand(3, Operand(reg));
1938 }
1939
1940
notl(CpuRegister reg)1941 void X86_64Assembler::notl(CpuRegister reg) {
1942 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1943 EmitOptionalRex32(reg);
1944 EmitUint8(0xF7);
1945 EmitUint8(0xD0 | reg.LowBits());
1946 }
1947
1948
notq(CpuRegister reg)1949 void X86_64Assembler::notq(CpuRegister reg) {
1950 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1951 EmitRex64(reg);
1952 EmitUint8(0xF7);
1953 EmitOperand(2, Operand(reg));
1954 }
1955
1956
enter(const Immediate & imm)1957 void X86_64Assembler::enter(const Immediate& imm) {
1958 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1959 EmitUint8(0xC8);
1960 CHECK(imm.is_uint16()) << imm.value();
1961 EmitUint8(imm.value() & 0xFF);
1962 EmitUint8((imm.value() >> 8) & 0xFF);
1963 EmitUint8(0x00);
1964 }
1965
1966
leave()1967 void X86_64Assembler::leave() {
1968 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1969 EmitUint8(0xC9);
1970 }
1971
1972
ret()1973 void X86_64Assembler::ret() {
1974 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1975 EmitUint8(0xC3);
1976 }
1977
1978
ret(const Immediate & imm)1979 void X86_64Assembler::ret(const Immediate& imm) {
1980 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1981 EmitUint8(0xC2);
1982 CHECK(imm.is_uint16());
1983 EmitUint8(imm.value() & 0xFF);
1984 EmitUint8((imm.value() >> 8) & 0xFF);
1985 }
1986
1987
1988
nop()1989 void X86_64Assembler::nop() {
1990 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1991 EmitUint8(0x90);
1992 }
1993
1994
int3()1995 void X86_64Assembler::int3() {
1996 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
1997 EmitUint8(0xCC);
1998 }
1999
2000
hlt()2001 void X86_64Assembler::hlt() {
2002 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
2003 EmitUint8(0xF4);
2004 }
2005
2006
j(Condition condition,Label * label)2007 void X86_64Assembler::j(Condition condition, Label* label) {
2008 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
2009 if (label->IsBound()) {
2010 static const int kShortSize = 2;
2011 static const int kLongSize = 6;
2012 int offset = label->Position() - buffer_.Size();
2013 CHECK_LE(offset, 0);
2014 if (IsInt<8>(offset - kShortSize)) {
2015 EmitUint8(0x70 + condition);
2016 EmitUint8((offset - kShortSize) & 0xFF);
2017 } else {
2018 EmitUint8(0x0F);
2019 EmitUint8(0x80 + condition);
2020 EmitInt32(offset - kLongSize);
2021 }
2022 } else {
2023 EmitUint8(0x0F);
2024 EmitUint8(0x80 + condition);
2025 EmitLabelLink(label);
2026 }
2027 }
2028
2029
j(Condition condition,NearLabel * label)2030 void X86_64Assembler::j(Condition condition, NearLabel* label) {
2031 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
2032 if (label->IsBound()) {
2033 static const int kShortSize = 2;
2034 int offset = label->Position() - buffer_.Size();
2035 CHECK_LE(offset, 0);
2036 CHECK(IsInt<8>(offset - kShortSize));
2037 EmitUint8(0x70 + condition);
2038 EmitUint8((offset - kShortSize) & 0xFF);
2039 } else {
2040 EmitUint8(0x70 + condition);
2041 EmitLabelLink(label);
2042 }
2043 }
2044
2045
jrcxz(NearLabel * label)2046 void X86_64Assembler::jrcxz(NearLabel* label) {
2047 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
2048 if (label->IsBound()) {
2049 static const int kShortSize = 2;
2050 int offset = label->Position() - buffer_.Size();
2051 CHECK_LE(offset, 0);
2052 CHECK(IsInt<8>(offset - kShortSize));
2053 EmitUint8(0xE3);
2054 EmitUint8((offset - kShortSize) & 0xFF);
2055 } else {
2056 EmitUint8(0xE3);
2057 EmitLabelLink(label);
2058 }
2059 }
2060
2061
jmp(CpuRegister reg)2062 void X86_64Assembler::jmp(CpuRegister reg) {
2063 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
2064 EmitOptionalRex32(reg);
2065 EmitUint8(0xFF);
2066 EmitRegisterOperand(4, reg.LowBits());
2067 }
2068
jmp(const Address & address)2069 void X86_64Assembler::jmp(const Address& address) {
2070 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
2071 EmitOptionalRex32(address);
2072 EmitUint8(0xFF);
2073 EmitOperand(4, address);
2074 }
2075
jmp(Label * label)2076 void X86_64Assembler::jmp(Label* label) {
2077 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
2078 if (label->IsBound()) {
2079 static const int kShortSize = 2;
2080 static const int kLongSize = 5;
2081 int offset = label->Position() - buffer_.Size();
2082 CHECK_LE(offset, 0);
2083 if (IsInt<8>(offset - kShortSize)) {
2084 EmitUint8(0xEB);
2085 EmitUint8((offset - kShortSize) & 0xFF);
2086 } else {
2087 EmitUint8(0xE9);
2088 EmitInt32(offset - kLongSize);
2089 }
2090 } else {
2091 EmitUint8(0xE9);
2092 EmitLabelLink(label);
2093 }
2094 }
2095
2096
jmp(NearLabel * label)2097 void X86_64Assembler::jmp(NearLabel* label) {
2098 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
2099 if (label->IsBound()) {
2100 static const int kShortSize = 2;
2101 int offset = label->Position() - buffer_.Size();
2102 CHECK_LE(offset, 0);
2103 CHECK(IsInt<8>(offset - kShortSize));
2104 EmitUint8(0xEB);
2105 EmitUint8((offset - kShortSize) & 0xFF);
2106 } else {
2107 EmitUint8(0xEB);
2108 EmitLabelLink(label);
2109 }
2110 }
2111
2112
rep_movsw()2113 void X86_64Assembler::rep_movsw() {
2114 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
2115 EmitUint8(0x66);
2116 EmitUint8(0xF3);
2117 EmitUint8(0xA5);
2118 }
2119
2120
lock()2121 X86_64Assembler* X86_64Assembler::lock() {
2122 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
2123 EmitUint8(0xF0);
2124 return this;
2125 }
2126
2127
cmpxchgl(const Address & address,CpuRegister reg)2128 void X86_64Assembler::cmpxchgl(const Address& address, CpuRegister reg) {
2129 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
2130 EmitOptionalRex32(reg, address);
2131 EmitUint8(0x0F);
2132 EmitUint8(0xB1);
2133 EmitOperand(reg.LowBits(), address);
2134 }
2135
2136
cmpxchgq(const Address & address,CpuRegister reg)2137 void X86_64Assembler::cmpxchgq(const Address& address, CpuRegister reg) {
2138 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
2139 EmitRex64(reg, address);
2140 EmitUint8(0x0F);
2141 EmitUint8(0xB1);
2142 EmitOperand(reg.LowBits(), address);
2143 }
2144
2145
mfence()2146 void X86_64Assembler::mfence() {
2147 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
2148 EmitUint8(0x0F);
2149 EmitUint8(0xAE);
2150 EmitUint8(0xF0);
2151 }
2152
2153
gs()2154 X86_64Assembler* X86_64Assembler::gs() {
2155 // TODO: gs is a prefix and not an instruction
2156 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
2157 EmitUint8(0x65);
2158 return this;
2159 }
2160
2161
AddImmediate(CpuRegister reg,const Immediate & imm)2162 void X86_64Assembler::AddImmediate(CpuRegister reg, const Immediate& imm) {
2163 int value = imm.value();
2164 if (value != 0) {
2165 if (value > 0) {
2166 addl(reg, imm);
2167 } else {
2168 subl(reg, Immediate(value));
2169 }
2170 }
2171 }
2172
2173
setcc(Condition condition,CpuRegister dst)2174 void X86_64Assembler::setcc(Condition condition, CpuRegister dst) {
2175 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
2176 // RSP, RBP, RDI, RSI need rex prefix (else the pattern encodes ah/bh/ch/dh).
2177 if (dst.NeedsRex() || dst.AsRegister() > 3) {
2178 EmitOptionalRex(true, false, false, false, dst.NeedsRex());
2179 }
2180 EmitUint8(0x0F);
2181 EmitUint8(0x90 + condition);
2182 EmitUint8(0xC0 + dst.LowBits());
2183 }
2184
bswapl(CpuRegister dst)2185 void X86_64Assembler::bswapl(CpuRegister dst) {
2186 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
2187 EmitOptionalRex(false, false, false, false, dst.NeedsRex());
2188 EmitUint8(0x0F);
2189 EmitUint8(0xC8 + dst.LowBits());
2190 }
2191
bswapq(CpuRegister dst)2192 void X86_64Assembler::bswapq(CpuRegister dst) {
2193 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
2194 EmitOptionalRex(false, true, false, false, dst.NeedsRex());
2195 EmitUint8(0x0F);
2196 EmitUint8(0xC8 + dst.LowBits());
2197 }
2198
bsfl(CpuRegister dst,CpuRegister src)2199 void X86_64Assembler::bsfl(CpuRegister dst, CpuRegister src) {
2200 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
2201 EmitOptionalRex32(dst, src);
2202 EmitUint8(0x0F);
2203 EmitUint8(0xBC);
2204 EmitRegisterOperand(dst.LowBits(), src.LowBits());
2205 }
2206
bsfl(CpuRegister dst,const Address & src)2207 void X86_64Assembler::bsfl(CpuRegister dst, const Address& src) {
2208 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
2209 EmitOptionalRex32(dst, src);
2210 EmitUint8(0x0F);
2211 EmitUint8(0xBC);
2212 EmitOperand(dst.LowBits(), src);
2213 }
2214
bsfq(CpuRegister dst,CpuRegister src)2215 void X86_64Assembler::bsfq(CpuRegister dst, CpuRegister src) {
2216 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
2217 EmitRex64(dst, src);
2218 EmitUint8(0x0F);
2219 EmitUint8(0xBC);
2220 EmitRegisterOperand(dst.LowBits(), src.LowBits());
2221 }
2222
bsfq(CpuRegister dst,const Address & src)2223 void X86_64Assembler::bsfq(CpuRegister dst, const Address& src) {
2224 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
2225 EmitRex64(dst, src);
2226 EmitUint8(0x0F);
2227 EmitUint8(0xBC);
2228 EmitOperand(dst.LowBits(), src);
2229 }
2230
bsrl(CpuRegister dst,CpuRegister src)2231 void X86_64Assembler::bsrl(CpuRegister dst, CpuRegister src) {
2232 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
2233 EmitOptionalRex32(dst, src);
2234 EmitUint8(0x0F);
2235 EmitUint8(0xBD);
2236 EmitRegisterOperand(dst.LowBits(), src.LowBits());
2237 }
2238
bsrl(CpuRegister dst,const Address & src)2239 void X86_64Assembler::bsrl(CpuRegister dst, const Address& src) {
2240 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
2241 EmitOptionalRex32(dst, src);
2242 EmitUint8(0x0F);
2243 EmitUint8(0xBD);
2244 EmitOperand(dst.LowBits(), src);
2245 }
2246
bsrq(CpuRegister dst,CpuRegister src)2247 void X86_64Assembler::bsrq(CpuRegister dst, CpuRegister src) {
2248 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
2249 EmitRex64(dst, src);
2250 EmitUint8(0x0F);
2251 EmitUint8(0xBD);
2252 EmitRegisterOperand(dst.LowBits(), src.LowBits());
2253 }
2254
bsrq(CpuRegister dst,const Address & src)2255 void X86_64Assembler::bsrq(CpuRegister dst, const Address& src) {
2256 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
2257 EmitRex64(dst, src);
2258 EmitUint8(0x0F);
2259 EmitUint8(0xBD);
2260 EmitOperand(dst.LowBits(), src);
2261 }
2262
popcntl(CpuRegister dst,CpuRegister src)2263 void X86_64Assembler::popcntl(CpuRegister dst, CpuRegister src) {
2264 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
2265 EmitUint8(0xF3);
2266 EmitOptionalRex32(dst, src);
2267 EmitUint8(0x0F);
2268 EmitUint8(0xB8);
2269 EmitRegisterOperand(dst.LowBits(), src.LowBits());
2270 }
2271
popcntl(CpuRegister dst,const Address & src)2272 void X86_64Assembler::popcntl(CpuRegister dst, const Address& src) {
2273 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
2274 EmitUint8(0xF3);
2275 EmitOptionalRex32(dst, src);
2276 EmitUint8(0x0F);
2277 EmitUint8(0xB8);
2278 EmitOperand(dst.LowBits(), src);
2279 }
2280
popcntq(CpuRegister dst,CpuRegister src)2281 void X86_64Assembler::popcntq(CpuRegister dst, CpuRegister src) {
2282 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
2283 EmitUint8(0xF3);
2284 EmitRex64(dst, src);
2285 EmitUint8(0x0F);
2286 EmitUint8(0xB8);
2287 EmitRegisterOperand(dst.LowBits(), src.LowBits());
2288 }
2289
popcntq(CpuRegister dst,const Address & src)2290 void X86_64Assembler::popcntq(CpuRegister dst, const Address& src) {
2291 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
2292 EmitUint8(0xF3);
2293 EmitRex64(dst, src);
2294 EmitUint8(0x0F);
2295 EmitUint8(0xB8);
2296 EmitOperand(dst.LowBits(), src);
2297 }
2298
repne_scasw()2299 void X86_64Assembler::repne_scasw() {
2300 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
2301 EmitUint8(0x66);
2302 EmitUint8(0xF2);
2303 EmitUint8(0xAF);
2304 }
2305
2306
repe_cmpsw()2307 void X86_64Assembler::repe_cmpsw() {
2308 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
2309 EmitUint8(0x66);
2310 EmitUint8(0xF3);
2311 EmitUint8(0xA7);
2312 }
2313
2314
repe_cmpsl()2315 void X86_64Assembler::repe_cmpsl() {
2316 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
2317 EmitUint8(0xF3);
2318 EmitUint8(0xA7);
2319 }
2320
2321
repe_cmpsq()2322 void X86_64Assembler::repe_cmpsq() {
2323 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
2324 EmitUint8(0xF3);
2325 EmitRex64();
2326 EmitUint8(0xA7);
2327 }
2328
2329
LoadDoubleConstant(XmmRegister dst,double value)2330 void X86_64Assembler::LoadDoubleConstant(XmmRegister dst, double value) {
2331 // TODO: Need to have a code constants table.
2332 int64_t constant = bit_cast<int64_t, double>(value);
2333 pushq(Immediate(High32Bits(constant)));
2334 pushq(Immediate(Low32Bits(constant)));
2335 movsd(dst, Address(CpuRegister(RSP), 0));
2336 addq(CpuRegister(RSP), Immediate(2 * sizeof(intptr_t)));
2337 }
2338
2339
Align(int alignment,int offset)2340 void X86_64Assembler::Align(int alignment, int offset) {
2341 CHECK(IsPowerOfTwo(alignment));
2342 // Emit nop instruction until the real position is aligned.
2343 while (((offset + buffer_.GetPosition()) & (alignment-1)) != 0) {
2344 nop();
2345 }
2346 }
2347
2348
Bind(Label * label)2349 void X86_64Assembler::Bind(Label* label) {
2350 int bound = buffer_.Size();
2351 CHECK(!label->IsBound()); // Labels can only be bound once.
2352 while (label->IsLinked()) {
2353 int position = label->LinkPosition();
2354 int next = buffer_.Load<int32_t>(position);
2355 buffer_.Store<int32_t>(position, bound - (position + 4));
2356 label->position_ = next;
2357 }
2358 label->BindTo(bound);
2359 }
2360
2361
Bind(NearLabel * label)2362 void X86_64Assembler::Bind(NearLabel* label) {
2363 int bound = buffer_.Size();
2364 CHECK(!label->IsBound()); // Labels can only be bound once.
2365 while (label->IsLinked()) {
2366 int position = label->LinkPosition();
2367 uint8_t delta = buffer_.Load<uint8_t>(position);
2368 int offset = bound - (position + 1);
2369 CHECK(IsInt<8>(offset));
2370 buffer_.Store<int8_t>(position, offset);
2371 label->position_ = delta != 0u ? label->position_ - delta : 0;
2372 }
2373 label->BindTo(bound);
2374 }
2375
2376
EmitOperand(uint8_t reg_or_opcode,const Operand & operand)2377 void X86_64Assembler::EmitOperand(uint8_t reg_or_opcode, const Operand& operand) {
2378 CHECK_GE(reg_or_opcode, 0);
2379 CHECK_LT(reg_or_opcode, 8);
2380 const int length = operand.length_;
2381 CHECK_GT(length, 0);
2382 // Emit the ModRM byte updated with the given reg value.
2383 CHECK_EQ(operand.encoding_[0] & 0x38, 0);
2384 EmitUint8(operand.encoding_[0] + (reg_or_opcode << 3));
2385 // Emit the rest of the encoded operand.
2386 for (int i = 1; i < length; i++) {
2387 EmitUint8(operand.encoding_[i]);
2388 }
2389 AssemblerFixup* fixup = operand.GetFixup();
2390 if (fixup != nullptr) {
2391 EmitFixup(fixup);
2392 }
2393 }
2394
2395
EmitImmediate(const Immediate & imm)2396 void X86_64Assembler::EmitImmediate(const Immediate& imm) {
2397 if (imm.is_int32()) {
2398 EmitInt32(static_cast<int32_t>(imm.value()));
2399 } else {
2400 EmitInt64(imm.value());
2401 }
2402 }
2403
2404
EmitComplex(uint8_t reg_or_opcode,const Operand & operand,const Immediate & immediate)2405 void X86_64Assembler::EmitComplex(uint8_t reg_or_opcode,
2406 const Operand& operand,
2407 const Immediate& immediate) {
2408 CHECK_GE(reg_or_opcode, 0);
2409 CHECK_LT(reg_or_opcode, 8);
2410 if (immediate.is_int8()) {
2411 // Use sign-extended 8-bit immediate.
2412 EmitUint8(0x83);
2413 EmitOperand(reg_or_opcode, operand);
2414 EmitUint8(immediate.value() & 0xFF);
2415 } else if (operand.IsRegister(CpuRegister(RAX))) {
2416 // Use short form if the destination is eax.
2417 EmitUint8(0x05 + (reg_or_opcode << 3));
2418 EmitImmediate(immediate);
2419 } else {
2420 EmitUint8(0x81);
2421 EmitOperand(reg_or_opcode, operand);
2422 EmitImmediate(immediate);
2423 }
2424 }
2425
2426
EmitLabel(Label * label,int instruction_size)2427 void X86_64Assembler::EmitLabel(Label* label, int instruction_size) {
2428 if (label->IsBound()) {
2429 int offset = label->Position() - buffer_.Size();
2430 CHECK_LE(offset, 0);
2431 EmitInt32(offset - instruction_size);
2432 } else {
2433 EmitLabelLink(label);
2434 }
2435 }
2436
2437
EmitLabelLink(Label * label)2438 void X86_64Assembler::EmitLabelLink(Label* label) {
2439 CHECK(!label->IsBound());
2440 int position = buffer_.Size();
2441 EmitInt32(label->position_);
2442 label->LinkTo(position);
2443 }
2444
2445
EmitLabelLink(NearLabel * label)2446 void X86_64Assembler::EmitLabelLink(NearLabel* label) {
2447 CHECK(!label->IsBound());
2448 int position = buffer_.Size();
2449 if (label->IsLinked()) {
2450 // Save the delta in the byte that we have to play with.
2451 uint32_t delta = position - label->LinkPosition();
2452 CHECK(IsUint<8>(delta));
2453 EmitUint8(delta & 0xFF);
2454 } else {
2455 EmitUint8(0);
2456 }
2457 label->LinkTo(position);
2458 }
2459
2460
EmitGenericShift(bool wide,int reg_or_opcode,CpuRegister reg,const Immediate & imm)2461 void X86_64Assembler::EmitGenericShift(bool wide,
2462 int reg_or_opcode,
2463 CpuRegister reg,
2464 const Immediate& imm) {
2465 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
2466 CHECK(imm.is_int8());
2467 if (wide) {
2468 EmitRex64(reg);
2469 } else {
2470 EmitOptionalRex32(reg);
2471 }
2472 if (imm.value() == 1) {
2473 EmitUint8(0xD1);
2474 EmitOperand(reg_or_opcode, Operand(reg));
2475 } else {
2476 EmitUint8(0xC1);
2477 EmitOperand(reg_or_opcode, Operand(reg));
2478 EmitUint8(imm.value() & 0xFF);
2479 }
2480 }
2481
2482
EmitGenericShift(bool wide,int reg_or_opcode,CpuRegister operand,CpuRegister shifter)2483 void X86_64Assembler::EmitGenericShift(bool wide,
2484 int reg_or_opcode,
2485 CpuRegister operand,
2486 CpuRegister shifter) {
2487 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
2488 CHECK_EQ(shifter.AsRegister(), RCX);
2489 if (wide) {
2490 EmitRex64(operand);
2491 } else {
2492 EmitOptionalRex32(operand);
2493 }
2494 EmitUint8(0xD3);
2495 EmitOperand(reg_or_opcode, Operand(operand));
2496 }
2497
EmitOptionalRex(bool force,bool w,bool r,bool x,bool b)2498 void X86_64Assembler::EmitOptionalRex(bool force, bool w, bool r, bool x, bool b) {
2499 // REX.WRXB
2500 // W - 64-bit operand
2501 // R - MODRM.reg
2502 // X - SIB.index
2503 // B - MODRM.rm/SIB.base
2504 uint8_t rex = force ? 0x40 : 0;
2505 if (w) {
2506 rex |= 0x48; // REX.W000
2507 }
2508 if (r) {
2509 rex |= 0x44; // REX.0R00
2510 }
2511 if (x) {
2512 rex |= 0x42; // REX.00X0
2513 }
2514 if (b) {
2515 rex |= 0x41; // REX.000B
2516 }
2517 if (rex != 0) {
2518 EmitUint8(rex);
2519 }
2520 }
2521
EmitOptionalRex32(CpuRegister reg)2522 void X86_64Assembler::EmitOptionalRex32(CpuRegister reg) {
2523 EmitOptionalRex(false, false, false, false, reg.NeedsRex());
2524 }
2525
EmitOptionalRex32(CpuRegister dst,CpuRegister src)2526 void X86_64Assembler::EmitOptionalRex32(CpuRegister dst, CpuRegister src) {
2527 EmitOptionalRex(false, false, dst.NeedsRex(), false, src.NeedsRex());
2528 }
2529
EmitOptionalRex32(XmmRegister dst,XmmRegister src)2530 void X86_64Assembler::EmitOptionalRex32(XmmRegister dst, XmmRegister src) {
2531 EmitOptionalRex(false, false, dst.NeedsRex(), false, src.NeedsRex());
2532 }
2533
EmitOptionalRex32(CpuRegister dst,XmmRegister src)2534 void X86_64Assembler::EmitOptionalRex32(CpuRegister dst, XmmRegister src) {
2535 EmitOptionalRex(false, false, dst.NeedsRex(), false, src.NeedsRex());
2536 }
2537
EmitOptionalRex32(XmmRegister dst,CpuRegister src)2538 void X86_64Assembler::EmitOptionalRex32(XmmRegister dst, CpuRegister src) {
2539 EmitOptionalRex(false, false, dst.NeedsRex(), false, src.NeedsRex());
2540 }
2541
EmitOptionalRex32(const Operand & operand)2542 void X86_64Assembler::EmitOptionalRex32(const Operand& operand) {
2543 uint8_t rex = operand.rex();
2544 if (rex != 0) {
2545 EmitUint8(rex);
2546 }
2547 }
2548
EmitOptionalRex32(CpuRegister dst,const Operand & operand)2549 void X86_64Assembler::EmitOptionalRex32(CpuRegister dst, const Operand& operand) {
2550 uint8_t rex = operand.rex();
2551 if (dst.NeedsRex()) {
2552 rex |= 0x44; // REX.0R00
2553 }
2554 if (rex != 0) {
2555 EmitUint8(rex);
2556 }
2557 }
2558
EmitOptionalRex32(XmmRegister dst,const Operand & operand)2559 void X86_64Assembler::EmitOptionalRex32(XmmRegister dst, const Operand& operand) {
2560 uint8_t rex = operand.rex();
2561 if (dst.NeedsRex()) {
2562 rex |= 0x44; // REX.0R00
2563 }
2564 if (rex != 0) {
2565 EmitUint8(rex);
2566 }
2567 }
2568
EmitRex64()2569 void X86_64Assembler::EmitRex64() {
2570 EmitOptionalRex(false, true, false, false, false);
2571 }
2572
EmitRex64(CpuRegister reg)2573 void X86_64Assembler::EmitRex64(CpuRegister reg) {
2574 EmitOptionalRex(false, true, false, false, reg.NeedsRex());
2575 }
2576
EmitRex64(const Operand & operand)2577 void X86_64Assembler::EmitRex64(const Operand& operand) {
2578 uint8_t rex = operand.rex();
2579 rex |= 0x48; // REX.W000
2580 EmitUint8(rex);
2581 }
2582
EmitRex64(CpuRegister dst,CpuRegister src)2583 void X86_64Assembler::EmitRex64(CpuRegister dst, CpuRegister src) {
2584 EmitOptionalRex(false, true, dst.NeedsRex(), false, src.NeedsRex());
2585 }
2586
EmitRex64(XmmRegister dst,CpuRegister src)2587 void X86_64Assembler::EmitRex64(XmmRegister dst, CpuRegister src) {
2588 EmitOptionalRex(false, true, dst.NeedsRex(), false, src.NeedsRex());
2589 }
2590
EmitRex64(CpuRegister dst,XmmRegister src)2591 void X86_64Assembler::EmitRex64(CpuRegister dst, XmmRegister src) {
2592 EmitOptionalRex(false, true, dst.NeedsRex(), false, src.NeedsRex());
2593 }
2594
EmitRex64(CpuRegister dst,const Operand & operand)2595 void X86_64Assembler::EmitRex64(CpuRegister dst, const Operand& operand) {
2596 uint8_t rex = 0x48 | operand.rex(); // REX.W000
2597 if (dst.NeedsRex()) {
2598 rex |= 0x44; // REX.0R00
2599 }
2600 EmitUint8(rex);
2601 }
2602
EmitRex64(XmmRegister dst,const Operand & operand)2603 void X86_64Assembler::EmitRex64(XmmRegister dst, const Operand& operand) {
2604 uint8_t rex = 0x48 | operand.rex(); // REX.W000
2605 if (dst.NeedsRex()) {
2606 rex |= 0x44; // REX.0R00
2607 }
2608 EmitUint8(rex);
2609 }
2610
EmitOptionalByteRegNormalizingRex32(CpuRegister dst,CpuRegister src)2611 void X86_64Assembler::EmitOptionalByteRegNormalizingRex32(CpuRegister dst, CpuRegister src) {
2612 // For src, SPL, BPL, SIL, DIL need the rex prefix.
2613 bool force = src.AsRegister() > 3;
2614 EmitOptionalRex(force, false, dst.NeedsRex(), false, src.NeedsRex());
2615 }
2616
EmitOptionalByteRegNormalizingRex32(CpuRegister dst,const Operand & operand)2617 void X86_64Assembler::EmitOptionalByteRegNormalizingRex32(CpuRegister dst, const Operand& operand) {
2618 uint8_t rex = operand.rex();
2619 // For dst, SPL, BPL, SIL, DIL need the rex prefix.
2620 bool force = dst.AsRegister() > 3;
2621 if (force) {
2622 rex |= 0x40; // REX.0000
2623 }
2624 if (dst.NeedsRex()) {
2625 rex |= 0x44; // REX.0R00
2626 }
2627 if (rex != 0) {
2628 EmitUint8(rex);
2629 }
2630 }
2631
DWARFReg(Register reg)2632 static dwarf::Reg DWARFReg(Register reg) {
2633 return dwarf::Reg::X86_64Core(static_cast<int>(reg));
2634 }
DWARFReg(FloatRegister reg)2635 static dwarf::Reg DWARFReg(FloatRegister reg) {
2636 return dwarf::Reg::X86_64Fp(static_cast<int>(reg));
2637 }
2638
2639 constexpr size_t kFramePointerSize = 8;
2640
BuildFrame(size_t frame_size,ManagedRegister method_reg,const std::vector<ManagedRegister> & spill_regs,const ManagedRegisterEntrySpills & entry_spills)2641 void X86_64Assembler::BuildFrame(size_t frame_size, ManagedRegister method_reg,
2642 const std::vector<ManagedRegister>& spill_regs,
2643 const ManagedRegisterEntrySpills& entry_spills) {
2644 DCHECK_EQ(buffer_.Size(), 0U); // Nothing emitted yet.
2645 cfi_.SetCurrentCFAOffset(8); // Return address on stack.
2646 CHECK_ALIGNED(frame_size, kStackAlignment);
2647 int gpr_count = 0;
2648 for (int i = spill_regs.size() - 1; i >= 0; --i) {
2649 x86_64::X86_64ManagedRegister spill = spill_regs.at(i).AsX86_64();
2650 if (spill.IsCpuRegister()) {
2651 pushq(spill.AsCpuRegister());
2652 gpr_count++;
2653 cfi_.AdjustCFAOffset(kFramePointerSize);
2654 cfi_.RelOffset(DWARFReg(spill.AsCpuRegister().AsRegister()), 0);
2655 }
2656 }
2657 // return address then method on stack.
2658 int64_t rest_of_frame = static_cast<int64_t>(frame_size)
2659 - (gpr_count * kFramePointerSize)
2660 - kFramePointerSize /*return address*/;
2661 subq(CpuRegister(RSP), Immediate(rest_of_frame));
2662 cfi_.AdjustCFAOffset(rest_of_frame);
2663
2664 // spill xmms
2665 int64_t offset = rest_of_frame;
2666 for (int i = spill_regs.size() - 1; i >= 0; --i) {
2667 x86_64::X86_64ManagedRegister spill = spill_regs.at(i).AsX86_64();
2668 if (spill.IsXmmRegister()) {
2669 offset -= sizeof(double);
2670 movsd(Address(CpuRegister(RSP), offset), spill.AsXmmRegister());
2671 cfi_.RelOffset(DWARFReg(spill.AsXmmRegister().AsFloatRegister()), offset);
2672 }
2673 }
2674
2675 DCHECK_EQ(kX86_64PointerSize, kFramePointerSize);
2676
2677 movq(Address(CpuRegister(RSP), 0), method_reg.AsX86_64().AsCpuRegister());
2678
2679 for (size_t i = 0; i < entry_spills.size(); ++i) {
2680 ManagedRegisterSpill spill = entry_spills.at(i);
2681 if (spill.AsX86_64().IsCpuRegister()) {
2682 if (spill.getSize() == 8) {
2683 movq(Address(CpuRegister(RSP), frame_size + spill.getSpillOffset()),
2684 spill.AsX86_64().AsCpuRegister());
2685 } else {
2686 CHECK_EQ(spill.getSize(), 4);
2687 movl(Address(CpuRegister(RSP), frame_size + spill.getSpillOffset()), spill.AsX86_64().AsCpuRegister());
2688 }
2689 } else {
2690 if (spill.getSize() == 8) {
2691 movsd(Address(CpuRegister(RSP), frame_size + spill.getSpillOffset()), spill.AsX86_64().AsXmmRegister());
2692 } else {
2693 CHECK_EQ(spill.getSize(), 4);
2694 movss(Address(CpuRegister(RSP), frame_size + spill.getSpillOffset()), spill.AsX86_64().AsXmmRegister());
2695 }
2696 }
2697 }
2698 }
2699
RemoveFrame(size_t frame_size,const std::vector<ManagedRegister> & spill_regs)2700 void X86_64Assembler::RemoveFrame(size_t frame_size,
2701 const std::vector<ManagedRegister>& spill_regs) {
2702 CHECK_ALIGNED(frame_size, kStackAlignment);
2703 cfi_.RememberState();
2704 int gpr_count = 0;
2705 // unspill xmms
2706 int64_t offset = static_cast<int64_t>(frame_size) - (spill_regs.size() * kFramePointerSize) - 2 * kFramePointerSize;
2707 for (size_t i = 0; i < spill_regs.size(); ++i) {
2708 x86_64::X86_64ManagedRegister spill = spill_regs.at(i).AsX86_64();
2709 if (spill.IsXmmRegister()) {
2710 offset += sizeof(double);
2711 movsd(spill.AsXmmRegister(), Address(CpuRegister(RSP), offset));
2712 cfi_.Restore(DWARFReg(spill.AsXmmRegister().AsFloatRegister()));
2713 } else {
2714 gpr_count++;
2715 }
2716 }
2717 int adjust = static_cast<int>(frame_size) - (gpr_count * kFramePointerSize) - kFramePointerSize;
2718 addq(CpuRegister(RSP), Immediate(adjust));
2719 cfi_.AdjustCFAOffset(-adjust);
2720 for (size_t i = 0; i < spill_regs.size(); ++i) {
2721 x86_64::X86_64ManagedRegister spill = spill_regs.at(i).AsX86_64();
2722 if (spill.IsCpuRegister()) {
2723 popq(spill.AsCpuRegister());
2724 cfi_.AdjustCFAOffset(-static_cast<int>(kFramePointerSize));
2725 cfi_.Restore(DWARFReg(spill.AsCpuRegister().AsRegister()));
2726 }
2727 }
2728 ret();
2729 // The CFI should be restored for any code that follows the exit block.
2730 cfi_.RestoreState();
2731 cfi_.DefCFAOffset(frame_size);
2732 }
2733
IncreaseFrameSize(size_t adjust)2734 void X86_64Assembler::IncreaseFrameSize(size_t adjust) {
2735 CHECK_ALIGNED(adjust, kStackAlignment);
2736 addq(CpuRegister(RSP), Immediate(-static_cast<int64_t>(adjust)));
2737 cfi_.AdjustCFAOffset(adjust);
2738 }
2739
DecreaseFrameSize(size_t adjust)2740 void X86_64Assembler::DecreaseFrameSize(size_t adjust) {
2741 CHECK_ALIGNED(adjust, kStackAlignment);
2742 addq(CpuRegister(RSP), Immediate(adjust));
2743 cfi_.AdjustCFAOffset(-adjust);
2744 }
2745
Store(FrameOffset offs,ManagedRegister msrc,size_t size)2746 void X86_64Assembler::Store(FrameOffset offs, ManagedRegister msrc, size_t size) {
2747 X86_64ManagedRegister src = msrc.AsX86_64();
2748 if (src.IsNoRegister()) {
2749 CHECK_EQ(0u, size);
2750 } else if (src.IsCpuRegister()) {
2751 if (size == 4) {
2752 CHECK_EQ(4u, size);
2753 movl(Address(CpuRegister(RSP), offs), src.AsCpuRegister());
2754 } else {
2755 CHECK_EQ(8u, size);
2756 movq(Address(CpuRegister(RSP), offs), src.AsCpuRegister());
2757 }
2758 } else if (src.IsRegisterPair()) {
2759 CHECK_EQ(0u, size);
2760 movq(Address(CpuRegister(RSP), offs), src.AsRegisterPairLow());
2761 movq(Address(CpuRegister(RSP), FrameOffset(offs.Int32Value()+4)),
2762 src.AsRegisterPairHigh());
2763 } else if (src.IsX87Register()) {
2764 if (size == 4) {
2765 fstps(Address(CpuRegister(RSP), offs));
2766 } else {
2767 fstpl(Address(CpuRegister(RSP), offs));
2768 }
2769 } else {
2770 CHECK(src.IsXmmRegister());
2771 if (size == 4) {
2772 movss(Address(CpuRegister(RSP), offs), src.AsXmmRegister());
2773 } else {
2774 movsd(Address(CpuRegister(RSP), offs), src.AsXmmRegister());
2775 }
2776 }
2777 }
2778
StoreRef(FrameOffset dest,ManagedRegister msrc)2779 void X86_64Assembler::StoreRef(FrameOffset dest, ManagedRegister msrc) {
2780 X86_64ManagedRegister src = msrc.AsX86_64();
2781 CHECK(src.IsCpuRegister());
2782 movl(Address(CpuRegister(RSP), dest), src.AsCpuRegister());
2783 }
2784
StoreRawPtr(FrameOffset dest,ManagedRegister msrc)2785 void X86_64Assembler::StoreRawPtr(FrameOffset dest, ManagedRegister msrc) {
2786 X86_64ManagedRegister src = msrc.AsX86_64();
2787 CHECK(src.IsCpuRegister());
2788 movq(Address(CpuRegister(RSP), dest), src.AsCpuRegister());
2789 }
2790
StoreImmediateToFrame(FrameOffset dest,uint32_t imm,ManagedRegister)2791 void X86_64Assembler::StoreImmediateToFrame(FrameOffset dest, uint32_t imm,
2792 ManagedRegister) {
2793 movl(Address(CpuRegister(RSP), dest), Immediate(imm)); // TODO(64) movq?
2794 }
2795
StoreImmediateToThread64(ThreadOffset<8> dest,uint32_t imm,ManagedRegister)2796 void X86_64Assembler::StoreImmediateToThread64(ThreadOffset<8> dest, uint32_t imm,
2797 ManagedRegister) {
2798 gs()->movl(Address::Absolute(dest, true), Immediate(imm)); // TODO(64) movq?
2799 }
2800
StoreStackOffsetToThread64(ThreadOffset<8> thr_offs,FrameOffset fr_offs,ManagedRegister mscratch)2801 void X86_64Assembler::StoreStackOffsetToThread64(ThreadOffset<8> thr_offs,
2802 FrameOffset fr_offs,
2803 ManagedRegister mscratch) {
2804 X86_64ManagedRegister scratch = mscratch.AsX86_64();
2805 CHECK(scratch.IsCpuRegister());
2806 leaq(scratch.AsCpuRegister(), Address(CpuRegister(RSP), fr_offs));
2807 gs()->movq(Address::Absolute(thr_offs, true), scratch.AsCpuRegister());
2808 }
2809
StoreStackPointerToThread64(ThreadOffset<8> thr_offs)2810 void X86_64Assembler::StoreStackPointerToThread64(ThreadOffset<8> thr_offs) {
2811 gs()->movq(Address::Absolute(thr_offs, true), CpuRegister(RSP));
2812 }
2813
StoreSpanning(FrameOffset,ManagedRegister,FrameOffset,ManagedRegister)2814 void X86_64Assembler::StoreSpanning(FrameOffset /*dst*/, ManagedRegister /*src*/,
2815 FrameOffset /*in_off*/, ManagedRegister /*scratch*/) {
2816 UNIMPLEMENTED(FATAL); // this case only currently exists for ARM
2817 }
2818
Load(ManagedRegister mdest,FrameOffset src,size_t size)2819 void X86_64Assembler::Load(ManagedRegister mdest, FrameOffset src, size_t size) {
2820 X86_64ManagedRegister dest = mdest.AsX86_64();
2821 if (dest.IsNoRegister()) {
2822 CHECK_EQ(0u, size);
2823 } else if (dest.IsCpuRegister()) {
2824 if (size == 4) {
2825 CHECK_EQ(4u, size);
2826 movl(dest.AsCpuRegister(), Address(CpuRegister(RSP), src));
2827 } else {
2828 CHECK_EQ(8u, size);
2829 movq(dest.AsCpuRegister(), Address(CpuRegister(RSP), src));
2830 }
2831 } else if (dest.IsRegisterPair()) {
2832 CHECK_EQ(0u, size);
2833 movq(dest.AsRegisterPairLow(), Address(CpuRegister(RSP), src));
2834 movq(dest.AsRegisterPairHigh(), Address(CpuRegister(RSP), FrameOffset(src.Int32Value()+4)));
2835 } else if (dest.IsX87Register()) {
2836 if (size == 4) {
2837 flds(Address(CpuRegister(RSP), src));
2838 } else {
2839 fldl(Address(CpuRegister(RSP), src));
2840 }
2841 } else {
2842 CHECK(dest.IsXmmRegister());
2843 if (size == 4) {
2844 movss(dest.AsXmmRegister(), Address(CpuRegister(RSP), src));
2845 } else {
2846 movsd(dest.AsXmmRegister(), Address(CpuRegister(RSP), src));
2847 }
2848 }
2849 }
2850
LoadFromThread64(ManagedRegister mdest,ThreadOffset<8> src,size_t size)2851 void X86_64Assembler::LoadFromThread64(ManagedRegister mdest, ThreadOffset<8> src, size_t size) {
2852 X86_64ManagedRegister dest = mdest.AsX86_64();
2853 if (dest.IsNoRegister()) {
2854 CHECK_EQ(0u, size);
2855 } else if (dest.IsCpuRegister()) {
2856 CHECK_EQ(4u, size);
2857 gs()->movl(dest.AsCpuRegister(), Address::Absolute(src, true));
2858 } else if (dest.IsRegisterPair()) {
2859 CHECK_EQ(8u, size);
2860 gs()->movq(dest.AsRegisterPairLow(), Address::Absolute(src, true));
2861 } else if (dest.IsX87Register()) {
2862 if (size == 4) {
2863 gs()->flds(Address::Absolute(src, true));
2864 } else {
2865 gs()->fldl(Address::Absolute(src, true));
2866 }
2867 } else {
2868 CHECK(dest.IsXmmRegister());
2869 if (size == 4) {
2870 gs()->movss(dest.AsXmmRegister(), Address::Absolute(src, true));
2871 } else {
2872 gs()->movsd(dest.AsXmmRegister(), Address::Absolute(src, true));
2873 }
2874 }
2875 }
2876
LoadRef(ManagedRegister mdest,FrameOffset src)2877 void X86_64Assembler::LoadRef(ManagedRegister mdest, FrameOffset src) {
2878 X86_64ManagedRegister dest = mdest.AsX86_64();
2879 CHECK(dest.IsCpuRegister());
2880 movq(dest.AsCpuRegister(), Address(CpuRegister(RSP), src));
2881 }
2882
LoadRef(ManagedRegister mdest,ManagedRegister base,MemberOffset offs,bool unpoison_reference)2883 void X86_64Assembler::LoadRef(ManagedRegister mdest, ManagedRegister base, MemberOffset offs,
2884 bool unpoison_reference) {
2885 X86_64ManagedRegister dest = mdest.AsX86_64();
2886 CHECK(dest.IsCpuRegister() && dest.IsCpuRegister());
2887 movl(dest.AsCpuRegister(), Address(base.AsX86_64().AsCpuRegister(), offs));
2888 if (unpoison_reference) {
2889 MaybeUnpoisonHeapReference(dest.AsCpuRegister());
2890 }
2891 }
2892
LoadRawPtr(ManagedRegister mdest,ManagedRegister base,Offset offs)2893 void X86_64Assembler::LoadRawPtr(ManagedRegister mdest, ManagedRegister base,
2894 Offset offs) {
2895 X86_64ManagedRegister dest = mdest.AsX86_64();
2896 CHECK(dest.IsCpuRegister() && dest.IsCpuRegister());
2897 movq(dest.AsCpuRegister(), Address(base.AsX86_64().AsCpuRegister(), offs));
2898 }
2899
LoadRawPtrFromThread64(ManagedRegister mdest,ThreadOffset<8> offs)2900 void X86_64Assembler::LoadRawPtrFromThread64(ManagedRegister mdest, ThreadOffset<8> offs) {
2901 X86_64ManagedRegister dest = mdest.AsX86_64();
2902 CHECK(dest.IsCpuRegister());
2903 gs()->movq(dest.AsCpuRegister(), Address::Absolute(offs, true));
2904 }
2905
SignExtend(ManagedRegister mreg,size_t size)2906 void X86_64Assembler::SignExtend(ManagedRegister mreg, size_t size) {
2907 X86_64ManagedRegister reg = mreg.AsX86_64();
2908 CHECK(size == 1 || size == 2) << size;
2909 CHECK(reg.IsCpuRegister()) << reg;
2910 if (size == 1) {
2911 movsxb(reg.AsCpuRegister(), reg.AsCpuRegister());
2912 } else {
2913 movsxw(reg.AsCpuRegister(), reg.AsCpuRegister());
2914 }
2915 }
2916
ZeroExtend(ManagedRegister mreg,size_t size)2917 void X86_64Assembler::ZeroExtend(ManagedRegister mreg, size_t size) {
2918 X86_64ManagedRegister reg = mreg.AsX86_64();
2919 CHECK(size == 1 || size == 2) << size;
2920 CHECK(reg.IsCpuRegister()) << reg;
2921 if (size == 1) {
2922 movzxb(reg.AsCpuRegister(), reg.AsCpuRegister());
2923 } else {
2924 movzxw(reg.AsCpuRegister(), reg.AsCpuRegister());
2925 }
2926 }
2927
Move(ManagedRegister mdest,ManagedRegister msrc,size_t size)2928 void X86_64Assembler::Move(ManagedRegister mdest, ManagedRegister msrc, size_t size) {
2929 X86_64ManagedRegister dest = mdest.AsX86_64();
2930 X86_64ManagedRegister src = msrc.AsX86_64();
2931 if (!dest.Equals(src)) {
2932 if (dest.IsCpuRegister() && src.IsCpuRegister()) {
2933 movq(dest.AsCpuRegister(), src.AsCpuRegister());
2934 } else if (src.IsX87Register() && dest.IsXmmRegister()) {
2935 // Pass via stack and pop X87 register
2936 subl(CpuRegister(RSP), Immediate(16));
2937 if (size == 4) {
2938 CHECK_EQ(src.AsX87Register(), ST0);
2939 fstps(Address(CpuRegister(RSP), 0));
2940 movss(dest.AsXmmRegister(), Address(CpuRegister(RSP), 0));
2941 } else {
2942 CHECK_EQ(src.AsX87Register(), ST0);
2943 fstpl(Address(CpuRegister(RSP), 0));
2944 movsd(dest.AsXmmRegister(), Address(CpuRegister(RSP), 0));
2945 }
2946 addq(CpuRegister(RSP), Immediate(16));
2947 } else {
2948 // TODO: x87, SSE
2949 UNIMPLEMENTED(FATAL) << ": Move " << dest << ", " << src;
2950 }
2951 }
2952 }
2953
CopyRef(FrameOffset dest,FrameOffset src,ManagedRegister mscratch)2954 void X86_64Assembler::CopyRef(FrameOffset dest, FrameOffset src, ManagedRegister mscratch) {
2955 X86_64ManagedRegister scratch = mscratch.AsX86_64();
2956 CHECK(scratch.IsCpuRegister());
2957 movl(scratch.AsCpuRegister(), Address(CpuRegister(RSP), src));
2958 movl(Address(CpuRegister(RSP), dest), scratch.AsCpuRegister());
2959 }
2960
CopyRawPtrFromThread64(FrameOffset fr_offs,ThreadOffset<8> thr_offs,ManagedRegister mscratch)2961 void X86_64Assembler::CopyRawPtrFromThread64(FrameOffset fr_offs,
2962 ThreadOffset<8> thr_offs,
2963 ManagedRegister mscratch) {
2964 X86_64ManagedRegister scratch = mscratch.AsX86_64();
2965 CHECK(scratch.IsCpuRegister());
2966 gs()->movq(scratch.AsCpuRegister(), Address::Absolute(thr_offs, true));
2967 Store(fr_offs, scratch, 8);
2968 }
2969
CopyRawPtrToThread64(ThreadOffset<8> thr_offs,FrameOffset fr_offs,ManagedRegister mscratch)2970 void X86_64Assembler::CopyRawPtrToThread64(ThreadOffset<8> thr_offs,
2971 FrameOffset fr_offs,
2972 ManagedRegister mscratch) {
2973 X86_64ManagedRegister scratch = mscratch.AsX86_64();
2974 CHECK(scratch.IsCpuRegister());
2975 Load(scratch, fr_offs, 8);
2976 gs()->movq(Address::Absolute(thr_offs, true), scratch.AsCpuRegister());
2977 }
2978
Copy(FrameOffset dest,FrameOffset src,ManagedRegister mscratch,size_t size)2979 void X86_64Assembler::Copy(FrameOffset dest, FrameOffset src, ManagedRegister mscratch,
2980 size_t size) {
2981 X86_64ManagedRegister scratch = mscratch.AsX86_64();
2982 if (scratch.IsCpuRegister() && size == 8) {
2983 Load(scratch, src, 4);
2984 Store(dest, scratch, 4);
2985 Load(scratch, FrameOffset(src.Int32Value() + 4), 4);
2986 Store(FrameOffset(dest.Int32Value() + 4), scratch, 4);
2987 } else {
2988 Load(scratch, src, size);
2989 Store(dest, scratch, size);
2990 }
2991 }
2992
Copy(FrameOffset,ManagedRegister,Offset,ManagedRegister,size_t)2993 void X86_64Assembler::Copy(FrameOffset /*dst*/, ManagedRegister /*src_base*/, Offset /*src_offset*/,
2994 ManagedRegister /*scratch*/, size_t /*size*/) {
2995 UNIMPLEMENTED(FATAL);
2996 }
2997
Copy(ManagedRegister dest_base,Offset dest_offset,FrameOffset src,ManagedRegister scratch,size_t size)2998 void X86_64Assembler::Copy(ManagedRegister dest_base, Offset dest_offset, FrameOffset src,
2999 ManagedRegister scratch, size_t size) {
3000 CHECK(scratch.IsNoRegister());
3001 CHECK_EQ(size, 4u);
3002 pushq(Address(CpuRegister(RSP), src));
3003 popq(Address(dest_base.AsX86_64().AsCpuRegister(), dest_offset));
3004 }
3005
Copy(FrameOffset dest,FrameOffset src_base,Offset src_offset,ManagedRegister mscratch,size_t size)3006 void X86_64Assembler::Copy(FrameOffset dest, FrameOffset src_base, Offset src_offset,
3007 ManagedRegister mscratch, size_t size) {
3008 CpuRegister scratch = mscratch.AsX86_64().AsCpuRegister();
3009 CHECK_EQ(size, 4u);
3010 movq(scratch, Address(CpuRegister(RSP), src_base));
3011 movq(scratch, Address(scratch, src_offset));
3012 movq(Address(CpuRegister(RSP), dest), scratch);
3013 }
3014
Copy(ManagedRegister dest,Offset dest_offset,ManagedRegister src,Offset src_offset,ManagedRegister scratch,size_t size)3015 void X86_64Assembler::Copy(ManagedRegister dest, Offset dest_offset,
3016 ManagedRegister src, Offset src_offset,
3017 ManagedRegister scratch, size_t size) {
3018 CHECK_EQ(size, 4u);
3019 CHECK(scratch.IsNoRegister());
3020 pushq(Address(src.AsX86_64().AsCpuRegister(), src_offset));
3021 popq(Address(dest.AsX86_64().AsCpuRegister(), dest_offset));
3022 }
3023
Copy(FrameOffset dest,Offset dest_offset,FrameOffset src,Offset src_offset,ManagedRegister mscratch,size_t size)3024 void X86_64Assembler::Copy(FrameOffset dest, Offset dest_offset, FrameOffset src, Offset src_offset,
3025 ManagedRegister mscratch, size_t size) {
3026 CpuRegister scratch = mscratch.AsX86_64().AsCpuRegister();
3027 CHECK_EQ(size, 4u);
3028 CHECK_EQ(dest.Int32Value(), src.Int32Value());
3029 movq(scratch, Address(CpuRegister(RSP), src));
3030 pushq(Address(scratch, src_offset));
3031 popq(Address(scratch, dest_offset));
3032 }
3033
MemoryBarrier(ManagedRegister)3034 void X86_64Assembler::MemoryBarrier(ManagedRegister) {
3035 mfence();
3036 }
3037
CreateHandleScopeEntry(ManagedRegister mout_reg,FrameOffset handle_scope_offset,ManagedRegister min_reg,bool null_allowed)3038 void X86_64Assembler::CreateHandleScopeEntry(ManagedRegister mout_reg,
3039 FrameOffset handle_scope_offset,
3040 ManagedRegister min_reg, bool null_allowed) {
3041 X86_64ManagedRegister out_reg = mout_reg.AsX86_64();
3042 X86_64ManagedRegister in_reg = min_reg.AsX86_64();
3043 if (in_reg.IsNoRegister()) { // TODO(64): && null_allowed
3044 // Use out_reg as indicator of null.
3045 in_reg = out_reg;
3046 // TODO: movzwl
3047 movl(in_reg.AsCpuRegister(), Address(CpuRegister(RSP), handle_scope_offset));
3048 }
3049 CHECK(in_reg.IsCpuRegister());
3050 CHECK(out_reg.IsCpuRegister());
3051 VerifyObject(in_reg, null_allowed);
3052 if (null_allowed) {
3053 Label null_arg;
3054 if (!out_reg.Equals(in_reg)) {
3055 xorl(out_reg.AsCpuRegister(), out_reg.AsCpuRegister());
3056 }
3057 testl(in_reg.AsCpuRegister(), in_reg.AsCpuRegister());
3058 j(kZero, &null_arg);
3059 leaq(out_reg.AsCpuRegister(), Address(CpuRegister(RSP), handle_scope_offset));
3060 Bind(&null_arg);
3061 } else {
3062 leaq(out_reg.AsCpuRegister(), Address(CpuRegister(RSP), handle_scope_offset));
3063 }
3064 }
3065
CreateHandleScopeEntry(FrameOffset out_off,FrameOffset handle_scope_offset,ManagedRegister mscratch,bool null_allowed)3066 void X86_64Assembler::CreateHandleScopeEntry(FrameOffset out_off,
3067 FrameOffset handle_scope_offset,
3068 ManagedRegister mscratch,
3069 bool null_allowed) {
3070 X86_64ManagedRegister scratch = mscratch.AsX86_64();
3071 CHECK(scratch.IsCpuRegister());
3072 if (null_allowed) {
3073 Label null_arg;
3074 movl(scratch.AsCpuRegister(), Address(CpuRegister(RSP), handle_scope_offset));
3075 testl(scratch.AsCpuRegister(), scratch.AsCpuRegister());
3076 j(kZero, &null_arg);
3077 leaq(scratch.AsCpuRegister(), Address(CpuRegister(RSP), handle_scope_offset));
3078 Bind(&null_arg);
3079 } else {
3080 leaq(scratch.AsCpuRegister(), Address(CpuRegister(RSP), handle_scope_offset));
3081 }
3082 Store(out_off, scratch, 8);
3083 }
3084
3085 // Given a handle scope entry, load the associated reference.
LoadReferenceFromHandleScope(ManagedRegister mout_reg,ManagedRegister min_reg)3086 void X86_64Assembler::LoadReferenceFromHandleScope(ManagedRegister mout_reg,
3087 ManagedRegister min_reg) {
3088 X86_64ManagedRegister out_reg = mout_reg.AsX86_64();
3089 X86_64ManagedRegister in_reg = min_reg.AsX86_64();
3090 CHECK(out_reg.IsCpuRegister());
3091 CHECK(in_reg.IsCpuRegister());
3092 Label null_arg;
3093 if (!out_reg.Equals(in_reg)) {
3094 xorl(out_reg.AsCpuRegister(), out_reg.AsCpuRegister());
3095 }
3096 testl(in_reg.AsCpuRegister(), in_reg.AsCpuRegister());
3097 j(kZero, &null_arg);
3098 movq(out_reg.AsCpuRegister(), Address(in_reg.AsCpuRegister(), 0));
3099 Bind(&null_arg);
3100 }
3101
VerifyObject(ManagedRegister,bool)3102 void X86_64Assembler::VerifyObject(ManagedRegister /*src*/, bool /*could_be_null*/) {
3103 // TODO: not validating references
3104 }
3105
VerifyObject(FrameOffset,bool)3106 void X86_64Assembler::VerifyObject(FrameOffset /*src*/, bool /*could_be_null*/) {
3107 // TODO: not validating references
3108 }
3109
Call(ManagedRegister mbase,Offset offset,ManagedRegister)3110 void X86_64Assembler::Call(ManagedRegister mbase, Offset offset, ManagedRegister) {
3111 X86_64ManagedRegister base = mbase.AsX86_64();
3112 CHECK(base.IsCpuRegister());
3113 call(Address(base.AsCpuRegister(), offset.Int32Value()));
3114 // TODO: place reference map on call
3115 }
3116
Call(FrameOffset base,Offset offset,ManagedRegister mscratch)3117 void X86_64Assembler::Call(FrameOffset base, Offset offset, ManagedRegister mscratch) {
3118 CpuRegister scratch = mscratch.AsX86_64().AsCpuRegister();
3119 movq(scratch, Address(CpuRegister(RSP), base));
3120 call(Address(scratch, offset));
3121 }
3122
CallFromThread64(ThreadOffset<8> offset,ManagedRegister)3123 void X86_64Assembler::CallFromThread64(ThreadOffset<8> offset, ManagedRegister /*mscratch*/) {
3124 gs()->call(Address::Absolute(offset, true));
3125 }
3126
GetCurrentThread(ManagedRegister tr)3127 void X86_64Assembler::GetCurrentThread(ManagedRegister tr) {
3128 gs()->movq(tr.AsX86_64().AsCpuRegister(), Address::Absolute(Thread::SelfOffset<8>(), true));
3129 }
3130
GetCurrentThread(FrameOffset offset,ManagedRegister mscratch)3131 void X86_64Assembler::GetCurrentThread(FrameOffset offset, ManagedRegister mscratch) {
3132 X86_64ManagedRegister scratch = mscratch.AsX86_64();
3133 gs()->movq(scratch.AsCpuRegister(), Address::Absolute(Thread::SelfOffset<8>(), true));
3134 movq(Address(CpuRegister(RSP), offset), scratch.AsCpuRegister());
3135 }
3136
3137 // Slowpath entered when Thread::Current()->_exception is non-null
3138 class X86_64ExceptionSlowPath FINAL : public SlowPath {
3139 public:
X86_64ExceptionSlowPath(size_t stack_adjust)3140 explicit X86_64ExceptionSlowPath(size_t stack_adjust) : stack_adjust_(stack_adjust) {}
3141 virtual void Emit(Assembler *sp_asm) OVERRIDE;
3142 private:
3143 const size_t stack_adjust_;
3144 };
3145
ExceptionPoll(ManagedRegister,size_t stack_adjust)3146 void X86_64Assembler::ExceptionPoll(ManagedRegister /*scratch*/, size_t stack_adjust) {
3147 X86_64ExceptionSlowPath* slow = new (GetArena()) X86_64ExceptionSlowPath(stack_adjust);
3148 buffer_.EnqueueSlowPath(slow);
3149 gs()->cmpl(Address::Absolute(Thread::ExceptionOffset<8>(), true), Immediate(0));
3150 j(kNotEqual, slow->Entry());
3151 }
3152
Emit(Assembler * sasm)3153 void X86_64ExceptionSlowPath::Emit(Assembler *sasm) {
3154 X86_64Assembler* sp_asm = down_cast<X86_64Assembler*>(sasm);
3155 #define __ sp_asm->
3156 __ Bind(&entry_);
3157 // Note: the return value is dead
3158 if (stack_adjust_ != 0) { // Fix up the frame.
3159 __ DecreaseFrameSize(stack_adjust_);
3160 }
3161 // Pass exception as argument in RDI
3162 __ gs()->movq(CpuRegister(RDI), Address::Absolute(Thread::ExceptionOffset<8>(), true));
3163 __ gs()->call(Address::Absolute(QUICK_ENTRYPOINT_OFFSET(8, pDeliverException), true));
3164 // this call should never return
3165 __ int3();
3166 #undef __
3167 }
3168
AddConstantArea()3169 void X86_64Assembler::AddConstantArea() {
3170 ArrayRef<const int32_t> area = constant_area_.GetBuffer();
3171 for (size_t i = 0, e = area.size(); i < e; i++) {
3172 AssemblerBuffer::EnsureCapacity ensured(&buffer_);
3173 EmitInt32(area[i]);
3174 }
3175 }
3176
AppendInt32(int32_t v)3177 size_t ConstantArea::AppendInt32(int32_t v) {
3178 size_t result = buffer_.size() * elem_size_;
3179 buffer_.push_back(v);
3180 return result;
3181 }
3182
AddInt32(int32_t v)3183 size_t ConstantArea::AddInt32(int32_t v) {
3184 // Look for an existing match.
3185 for (size_t i = 0, e = buffer_.size(); i < e; i++) {
3186 if (v == buffer_[i]) {
3187 return i * elem_size_;
3188 }
3189 }
3190
3191 // Didn't match anything.
3192 return AppendInt32(v);
3193 }
3194
AddInt64(int64_t v)3195 size_t ConstantArea::AddInt64(int64_t v) {
3196 int32_t v_low = v;
3197 int32_t v_high = v >> 32;
3198 if (buffer_.size() > 1) {
3199 // Ensure we don't pass the end of the buffer.
3200 for (size_t i = 0, e = buffer_.size() - 1; i < e; i++) {
3201 if (v_low == buffer_[i] && v_high == buffer_[i + 1]) {
3202 return i * elem_size_;
3203 }
3204 }
3205 }
3206
3207 // Didn't match anything.
3208 size_t result = buffer_.size() * elem_size_;
3209 buffer_.push_back(v_low);
3210 buffer_.push_back(v_high);
3211 return result;
3212 }
3213
AddDouble(double v)3214 size_t ConstantArea::AddDouble(double v) {
3215 // Treat the value as a 64-bit integer value.
3216 return AddInt64(bit_cast<int64_t, double>(v));
3217 }
3218
AddFloat(float v)3219 size_t ConstantArea::AddFloat(float v) {
3220 // Treat the value as a 32-bit integer value.
3221 return AddInt32(bit_cast<int32_t, float>(v));
3222 }
3223
3224 } // namespace x86_64
3225 } // namespace art
3226