1 /*
2 * Stack-less Just-In-Time compiler
3 *
4 * Copyright 2009-2012 Zoltan Herczeg (hzmester@freemail.hu). All rights reserved.
5 *
6 * Redistribution and use in source and binary forms, with or without modification, are
7 * permitted provided that the following conditions are met:
8 *
9 * 1. Redistributions of source code must retain the above copyright notice, this list of
10 * conditions and the following disclaimer.
11 *
12 * 2. Redistributions in binary form must reproduce the above copyright notice, this list
13 * of conditions and the following disclaimer in the documentation and/or other materials
14 * provided with the distribution.
15 *
16 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER(S) AND CONTRIBUTORS ``AS IS'' AND ANY
17 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
18 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT
19 * SHALL THE COPYRIGHT HOLDER(S) OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
20 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
21 * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
22 * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
23 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
24 * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
25 */
26
27 /* x86 64-bit arch dependent functions. */
28
emit_load_imm64(struct sljit_compiler * compiler,sljit_si reg,sljit_sw imm)29 static sljit_si emit_load_imm64(struct sljit_compiler *compiler, sljit_si reg, sljit_sw imm)
30 {
31 sljit_ub *inst;
32
33 inst = (sljit_ub*)ensure_buf(compiler, 1 + 2 + sizeof(sljit_sw));
34 FAIL_IF(!inst);
35 INC_SIZE(2 + sizeof(sljit_sw));
36 *inst++ = REX_W | ((reg_map[reg] <= 7) ? 0 : REX_B);
37 *inst++ = MOV_r_i32 + (reg_map[reg] & 0x7);
38 *(sljit_sw*)inst = imm;
39 return SLJIT_SUCCESS;
40 }
41
generate_far_jump_code(struct sljit_jump * jump,sljit_ub * code_ptr,sljit_si type)42 static sljit_ub* generate_far_jump_code(struct sljit_jump *jump, sljit_ub *code_ptr, sljit_si type)
43 {
44 if (type < SLJIT_JUMP) {
45 /* Invert type. */
46 *code_ptr++ = get_jump_code(type ^ 0x1) - 0x10;
47 *code_ptr++ = 10 + 3;
48 }
49
50 SLJIT_COMPILE_ASSERT(reg_map[TMP_REG3] == 9, tmp3_is_9_first);
51 *code_ptr++ = REX_W | REX_B;
52 *code_ptr++ = MOV_r_i32 + 1;
53 jump->addr = (sljit_uw)code_ptr;
54
55 if (jump->flags & JUMP_LABEL)
56 jump->flags |= PATCH_MD;
57 else
58 *(sljit_sw*)code_ptr = jump->u.target;
59
60 code_ptr += sizeof(sljit_sw);
61 *code_ptr++ = REX_B;
62 *code_ptr++ = GROUP_FF;
63 *code_ptr++ = (type >= SLJIT_FAST_CALL) ? (MOD_REG | CALL_rm | 1) : (MOD_REG | JMP_rm | 1);
64
65 return code_ptr;
66 }
67
generate_fixed_jump(sljit_ub * code_ptr,sljit_sw addr,sljit_si type)68 static sljit_ub* generate_fixed_jump(sljit_ub *code_ptr, sljit_sw addr, sljit_si type)
69 {
70 sljit_sw delta = addr - ((sljit_sw)code_ptr + 1 + sizeof(sljit_si));
71
72 if (delta <= HALFWORD_MAX && delta >= HALFWORD_MIN) {
73 *code_ptr++ = (type == 2) ? CALL_i32 : JMP_i32;
74 *(sljit_sw*)code_ptr = delta;
75 }
76 else {
77 SLJIT_COMPILE_ASSERT(reg_map[TMP_REG3] == 9, tmp3_is_9_second);
78 *code_ptr++ = REX_W | REX_B;
79 *code_ptr++ = MOV_r_i32 + 1;
80 *(sljit_sw*)code_ptr = addr;
81 code_ptr += sizeof(sljit_sw);
82 *code_ptr++ = REX_B;
83 *code_ptr++ = GROUP_FF;
84 *code_ptr++ = (type == 2) ? (MOD_REG | CALL_rm | 1) : (MOD_REG | JMP_rm | 1);
85 }
86
87 return code_ptr;
88 }
89
sljit_emit_enter(struct sljit_compiler * compiler,sljit_si options,sljit_si args,sljit_si scratches,sljit_si saveds,sljit_si fscratches,sljit_si fsaveds,sljit_si local_size)90 SLJIT_API_FUNC_ATTRIBUTE sljit_si sljit_emit_enter(struct sljit_compiler *compiler,
91 sljit_si options, sljit_si args, sljit_si scratches, sljit_si saveds,
92 sljit_si fscratches, sljit_si fsaveds, sljit_si local_size)
93 {
94 sljit_si i, tmp, size, saved_register_size;
95 sljit_ub *inst;
96
97 CHECK_ERROR();
98 check_sljit_emit_enter(compiler, options, args, scratches, saveds, fscratches, fsaveds, local_size);
99
100 compiler->options = options;
101 compiler->scratches = scratches;
102 compiler->saveds = saveds;
103 compiler->fscratches = fscratches;
104 compiler->fsaveds = fsaveds;
105 compiler->flags_saved = 0;
106 #if (defined SLJIT_DEBUG && SLJIT_DEBUG)
107 compiler->logical_local_size = local_size;
108 #endif
109
110 /* Including the return address saved by the call instruction. */
111 saved_register_size = GET_SAVED_REGISTERS_SIZE(scratches, saveds, 1);
112
113 tmp = saveds < SLJIT_NUMBER_OF_SAVED_REGISTERS ? (SLJIT_S0 + 1 - saveds) : SLJIT_FIRST_SAVED_REG;
114 for (i = SLJIT_S0; i >= tmp; i--) {
115 size = reg_map[i] >= 8 ? 2 : 1;
116 inst = (sljit_ub*)ensure_buf(compiler, 1 + size);
117 FAIL_IF(!inst);
118 INC_SIZE(size);
119 if (reg_map[i] >= 8)
120 *inst++ = REX_B;
121 PUSH_REG(reg_lmap[i]);
122 }
123
124 for (i = scratches; i >= SLJIT_FIRST_SAVED_REG; i--) {
125 size = reg_map[i] >= 8 ? 2 : 1;
126 inst = (sljit_ub*)ensure_buf(compiler, 1 + size);
127 FAIL_IF(!inst);
128 INC_SIZE(size);
129 if (reg_map[i] >= 8)
130 *inst++ = REX_B;
131 PUSH_REG(reg_lmap[i]);
132 }
133
134 if (args > 0) {
135 size = args * 3;
136 inst = (sljit_ub*)ensure_buf(compiler, 1 + size);
137 FAIL_IF(!inst);
138
139 INC_SIZE(size);
140
141 #ifndef _WIN64
142 if (args > 0) {
143 *inst++ = REX_W;
144 *inst++ = MOV_r_rm;
145 *inst++ = MOD_REG | (reg_map[SLJIT_S0] << 3) | 0x7 /* rdi */;
146 }
147 if (args > 1) {
148 *inst++ = REX_W | REX_R;
149 *inst++ = MOV_r_rm;
150 *inst++ = MOD_REG | (reg_lmap[SLJIT_S1] << 3) | 0x6 /* rsi */;
151 }
152 if (args > 2) {
153 *inst++ = REX_W | REX_R;
154 *inst++ = MOV_r_rm;
155 *inst++ = MOD_REG | (reg_lmap[SLJIT_S2] << 3) | 0x2 /* rdx */;
156 }
157 #else
158 if (args > 0) {
159 *inst++ = REX_W;
160 *inst++ = MOV_r_rm;
161 *inst++ = MOD_REG | (reg_map[SLJIT_S0] << 3) | 0x1 /* rcx */;
162 }
163 if (args > 1) {
164 *inst++ = REX_W;
165 *inst++ = MOV_r_rm;
166 *inst++ = MOD_REG | (reg_map[SLJIT_S1] << 3) | 0x2 /* rdx */;
167 }
168 if (args > 2) {
169 *inst++ = REX_W | REX_B;
170 *inst++ = MOV_r_rm;
171 *inst++ = MOD_REG | (reg_map[SLJIT_S2] << 3) | 0x0 /* r8 */;
172 }
173 #endif
174 }
175
176 local_size = ((local_size + FIXED_LOCALS_OFFSET + saved_register_size + 16 - 1) & ~(16 - 1)) - saved_register_size;
177 compiler->local_size = local_size;
178
179 #ifdef _WIN64
180 if (local_size > 1024) {
181 /* Allocate stack for the callback, which grows the stack. */
182 inst = (sljit_ub*)ensure_buf(compiler, 1 + 4 + (3 + sizeof(sljit_si)));
183 FAIL_IF(!inst);
184 INC_SIZE(4 + (3 + sizeof(sljit_si)));
185 *inst++ = REX_W;
186 *inst++ = GROUP_BINARY_83;
187 *inst++ = MOD_REG | SUB | 4;
188 /* Allocated size for registers must be divisible by 8. */
189 SLJIT_ASSERT(!(saved_register_size & 0x7));
190 /* Aligned to 16 byte. */
191 if (saved_register_size & 0x8) {
192 *inst++ = 5 * sizeof(sljit_sw);
193 local_size -= 5 * sizeof(sljit_sw);
194 } else {
195 *inst++ = 4 * sizeof(sljit_sw);
196 local_size -= 4 * sizeof(sljit_sw);
197 }
198 /* Second instruction */
199 SLJIT_COMPILE_ASSERT(reg_map[SLJIT_R0] < 8, temporary_reg1_is_loreg);
200 *inst++ = REX_W;
201 *inst++ = MOV_rm_i32;
202 *inst++ = MOD_REG | reg_lmap[SLJIT_R0];
203 *(sljit_si*)inst = local_size;
204 #if (defined SLJIT_VERBOSE && SLJIT_VERBOSE) || (defined SLJIT_DEBUG && SLJIT_DEBUG)
205 compiler->skip_checks = 1;
206 #endif
207 FAIL_IF(sljit_emit_ijump(compiler, SLJIT_CALL1, SLJIT_IMM, SLJIT_FUNC_OFFSET(sljit_grow_stack)));
208 }
209 #endif
210
211 SLJIT_ASSERT(local_size > 0);
212 if (local_size <= 127) {
213 inst = (sljit_ub*)ensure_buf(compiler, 1 + 4);
214 FAIL_IF(!inst);
215 INC_SIZE(4);
216 *inst++ = REX_W;
217 *inst++ = GROUP_BINARY_83;
218 *inst++ = MOD_REG | SUB | 4;
219 *inst++ = local_size;
220 }
221 else {
222 inst = (sljit_ub*)ensure_buf(compiler, 1 + 7);
223 FAIL_IF(!inst);
224 INC_SIZE(7);
225 *inst++ = REX_W;
226 *inst++ = GROUP_BINARY_81;
227 *inst++ = MOD_REG | SUB | 4;
228 *(sljit_si*)inst = local_size;
229 inst += sizeof(sljit_si);
230 }
231
232 #ifdef _WIN64
233 /* Save xmm6 register: movaps [rsp + 0x20], xmm6 */
234 if (fscratches >= 6 || fsaveds >= 1) {
235 inst = (sljit_ub*)ensure_buf(compiler, 1 + 5);
236 FAIL_IF(!inst);
237 INC_SIZE(5);
238 *inst++ = GROUP_0F;
239 *(sljit_si*)inst = 0x20247429;
240 }
241 #endif
242
243 return SLJIT_SUCCESS;
244 }
245
sljit_set_context(struct sljit_compiler * compiler,sljit_si options,sljit_si args,sljit_si scratches,sljit_si saveds,sljit_si fscratches,sljit_si fsaveds,sljit_si local_size)246 SLJIT_API_FUNC_ATTRIBUTE void sljit_set_context(struct sljit_compiler *compiler,
247 sljit_si options, sljit_si args, sljit_si scratches, sljit_si saveds,
248 sljit_si fscratches, sljit_si fsaveds, sljit_si local_size)
249 {
250 sljit_si saved_register_size;
251
252 CHECK_ERROR_VOID();
253 check_sljit_set_context(compiler, options, args, scratches, saveds, fscratches, fsaveds, local_size);
254
255 compiler->options = options;
256 compiler->scratches = scratches;
257 compiler->saveds = saveds;
258 compiler->fscratches = fscratches;
259 compiler->fsaveds = fsaveds;
260 #if (defined SLJIT_DEBUG && SLJIT_DEBUG)
261 compiler->logical_local_size = local_size;
262 #endif
263
264 /* Including the return address saved by the call instruction. */
265 saved_register_size = GET_SAVED_REGISTERS_SIZE(scratches, saveds, 1);
266 compiler->local_size = ((local_size + FIXED_LOCALS_OFFSET + saved_register_size + 16 - 1) & ~(16 - 1)) - saved_register_size;
267 }
268
sljit_emit_return(struct sljit_compiler * compiler,sljit_si op,sljit_si src,sljit_sw srcw)269 SLJIT_API_FUNC_ATTRIBUTE sljit_si sljit_emit_return(struct sljit_compiler *compiler, sljit_si op, sljit_si src, sljit_sw srcw)
270 {
271 sljit_si i, tmp, size;
272 sljit_ub *inst;
273
274 CHECK_ERROR();
275 check_sljit_emit_return(compiler, op, src, srcw);
276
277 compiler->flags_saved = 0;
278 FAIL_IF(emit_mov_before_return(compiler, op, src, srcw));
279
280 #ifdef _WIN64
281 /* Restore xmm6 register: movaps xmm6, [rsp + 0x20] */
282 if (compiler->fscratches >= 6 || compiler->fsaveds >= 1) {
283 inst = (sljit_ub*)ensure_buf(compiler, 1 + 5);
284 FAIL_IF(!inst);
285 INC_SIZE(5);
286 *inst++ = GROUP_0F;
287 *(sljit_si*)inst = 0x20247428;
288 }
289 #endif
290
291 SLJIT_ASSERT(compiler->local_size > 0);
292 if (compiler->local_size <= 127) {
293 inst = (sljit_ub*)ensure_buf(compiler, 1 + 4);
294 FAIL_IF(!inst);
295 INC_SIZE(4);
296 *inst++ = REX_W;
297 *inst++ = GROUP_BINARY_83;
298 *inst++ = MOD_REG | ADD | 4;
299 *inst = compiler->local_size;
300 }
301 else {
302 inst = (sljit_ub*)ensure_buf(compiler, 1 + 7);
303 FAIL_IF(!inst);
304 INC_SIZE(7);
305 *inst++ = REX_W;
306 *inst++ = GROUP_BINARY_81;
307 *inst++ = MOD_REG | ADD | 4;
308 *(sljit_si*)inst = compiler->local_size;
309 }
310
311 tmp = compiler->scratches;
312 for (i = SLJIT_FIRST_SAVED_REG; i <= tmp; i++) {
313 size = reg_map[i] >= 8 ? 2 : 1;
314 inst = (sljit_ub*)ensure_buf(compiler, 1 + size);
315 FAIL_IF(!inst);
316 INC_SIZE(size);
317 if (reg_map[i] >= 8)
318 *inst++ = REX_B;
319 POP_REG(reg_lmap[i]);
320 }
321
322 tmp = compiler->saveds < SLJIT_NUMBER_OF_SAVED_REGISTERS ? (SLJIT_S0 + 1 - compiler->saveds) : SLJIT_FIRST_SAVED_REG;
323 for (i = tmp; i <= SLJIT_S0; i++) {
324 size = reg_map[i] >= 8 ? 2 : 1;
325 inst = (sljit_ub*)ensure_buf(compiler, 1 + size);
326 FAIL_IF(!inst);
327 INC_SIZE(size);
328 if (reg_map[i] >= 8)
329 *inst++ = REX_B;
330 POP_REG(reg_lmap[i]);
331 }
332
333 inst = (sljit_ub*)ensure_buf(compiler, 1 + 1);
334 FAIL_IF(!inst);
335 INC_SIZE(1);
336 RET();
337 return SLJIT_SUCCESS;
338 }
339
340 /* --------------------------------------------------------------------- */
341 /* Operators */
342 /* --------------------------------------------------------------------- */
343
emit_do_imm32(struct sljit_compiler * compiler,sljit_ub rex,sljit_ub opcode,sljit_sw imm)344 static sljit_si emit_do_imm32(struct sljit_compiler *compiler, sljit_ub rex, sljit_ub opcode, sljit_sw imm)
345 {
346 sljit_ub *inst;
347 sljit_si length = 1 + (rex ? 1 : 0) + sizeof(sljit_si);
348
349 inst = (sljit_ub*)ensure_buf(compiler, 1 + length);
350 FAIL_IF(!inst);
351 INC_SIZE(length);
352 if (rex)
353 *inst++ = rex;
354 *inst++ = opcode;
355 *(sljit_si*)inst = imm;
356 return SLJIT_SUCCESS;
357 }
358
emit_x86_instruction(struct sljit_compiler * compiler,sljit_si size,sljit_si a,sljit_sw imma,sljit_si b,sljit_sw immb)359 static sljit_ub* emit_x86_instruction(struct sljit_compiler *compiler, sljit_si size,
360 /* The register or immediate operand. */
361 sljit_si a, sljit_sw imma,
362 /* The general operand (not immediate). */
363 sljit_si b, sljit_sw immb)
364 {
365 sljit_ub *inst;
366 sljit_ub *buf_ptr;
367 sljit_ub rex = 0;
368 sljit_si flags = size & ~0xf;
369 sljit_si inst_size;
370
371 /* The immediate operand must be 32 bit. */
372 SLJIT_ASSERT(!(a & SLJIT_IMM) || compiler->mode32 || IS_HALFWORD(imma));
373 /* Both cannot be switched on. */
374 SLJIT_ASSERT((flags & (EX86_BIN_INS | EX86_SHIFT_INS)) != (EX86_BIN_INS | EX86_SHIFT_INS));
375 /* Size flags not allowed for typed instructions. */
376 SLJIT_ASSERT(!(flags & (EX86_BIN_INS | EX86_SHIFT_INS)) || (flags & (EX86_BYTE_ARG | EX86_HALF_ARG)) == 0);
377 /* Both size flags cannot be switched on. */
378 SLJIT_ASSERT((flags & (EX86_BYTE_ARG | EX86_HALF_ARG)) != (EX86_BYTE_ARG | EX86_HALF_ARG));
379 /* SSE2 and immediate is not possible. */
380 SLJIT_ASSERT(!(a & SLJIT_IMM) || !(flags & EX86_SSE2));
381 SLJIT_ASSERT((flags & (EX86_PREF_F2 | EX86_PREF_F3)) != (EX86_PREF_F2 | EX86_PREF_F3)
382 && (flags & (EX86_PREF_F2 | EX86_PREF_66)) != (EX86_PREF_F2 | EX86_PREF_66)
383 && (flags & (EX86_PREF_F3 | EX86_PREF_66)) != (EX86_PREF_F3 | EX86_PREF_66));
384
385 size &= 0xf;
386 inst_size = size;
387
388 if (!compiler->mode32 && !(flags & EX86_NO_REXW))
389 rex |= REX_W;
390 else if (flags & EX86_REX)
391 rex |= REX;
392
393 if (flags & (EX86_PREF_F2 | EX86_PREF_F3))
394 inst_size++;
395 if (flags & EX86_PREF_66)
396 inst_size++;
397
398 /* Calculate size of b. */
399 inst_size += 1; /* mod r/m byte. */
400 if (b & SLJIT_MEM) {
401 if (!(b & OFFS_REG_MASK)) {
402 if (NOT_HALFWORD(immb)) {
403 if (emit_load_imm64(compiler, TMP_REG3, immb))
404 return NULL;
405 immb = 0;
406 if (b & REG_MASK)
407 b |= TO_OFFS_REG(TMP_REG3);
408 else
409 b |= TMP_REG3;
410 }
411 else if (reg_lmap[b & REG_MASK] == 4)
412 b |= TO_OFFS_REG(SLJIT_SP);
413 }
414
415 if ((b & REG_MASK) == SLJIT_UNUSED)
416 inst_size += 1 + sizeof(sljit_si); /* SIB byte required to avoid RIP based addressing. */
417 else {
418 if (reg_map[b & REG_MASK] >= 8)
419 rex |= REX_B;
420
421 if (immb != 0 && (!(b & OFFS_REG_MASK) || (b & OFFS_REG_MASK) == TO_OFFS_REG(SLJIT_SP))) {
422 /* Immediate operand. */
423 if (immb <= 127 && immb >= -128)
424 inst_size += sizeof(sljit_sb);
425 else
426 inst_size += sizeof(sljit_si);
427 }
428 else if (reg_lmap[b & REG_MASK] == 5)
429 inst_size += sizeof(sljit_sb);
430
431 if ((b & OFFS_REG_MASK) != SLJIT_UNUSED) {
432 inst_size += 1; /* SIB byte. */
433 if (reg_map[OFFS_REG(b)] >= 8)
434 rex |= REX_X;
435 }
436 }
437 }
438 else if (!(flags & EX86_SSE2_OP2) && reg_map[b] >= 8)
439 rex |= REX_B;
440
441 if (a & SLJIT_IMM) {
442 if (flags & EX86_BIN_INS) {
443 if (imma <= 127 && imma >= -128) {
444 inst_size += 1;
445 flags |= EX86_BYTE_ARG;
446 } else
447 inst_size += 4;
448 }
449 else if (flags & EX86_SHIFT_INS) {
450 imma &= compiler->mode32 ? 0x1f : 0x3f;
451 if (imma != 1) {
452 inst_size ++;
453 flags |= EX86_BYTE_ARG;
454 }
455 } else if (flags & EX86_BYTE_ARG)
456 inst_size++;
457 else if (flags & EX86_HALF_ARG)
458 inst_size += sizeof(short);
459 else
460 inst_size += sizeof(sljit_si);
461 }
462 else {
463 SLJIT_ASSERT(!(flags & EX86_SHIFT_INS) || a == SLJIT_PREF_SHIFT_REG);
464 /* reg_map[SLJIT_PREF_SHIFT_REG] is less than 8. */
465 if (!(flags & EX86_SSE2_OP1) && reg_map[a] >= 8)
466 rex |= REX_R;
467 }
468
469 if (rex)
470 inst_size++;
471
472 inst = (sljit_ub*)ensure_buf(compiler, 1 + inst_size);
473 PTR_FAIL_IF(!inst);
474
475 /* Encoding the byte. */
476 INC_SIZE(inst_size);
477 if (flags & EX86_PREF_F2)
478 *inst++ = 0xf2;
479 if (flags & EX86_PREF_F3)
480 *inst++ = 0xf3;
481 if (flags & EX86_PREF_66)
482 *inst++ = 0x66;
483 if (rex)
484 *inst++ = rex;
485 buf_ptr = inst + size;
486
487 /* Encode mod/rm byte. */
488 if (!(flags & EX86_SHIFT_INS)) {
489 if ((flags & EX86_BIN_INS) && (a & SLJIT_IMM))
490 *inst = (flags & EX86_BYTE_ARG) ? GROUP_BINARY_83 : GROUP_BINARY_81;
491
492 if ((a & SLJIT_IMM) || (a == 0))
493 *buf_ptr = 0;
494 else if (!(flags & EX86_SSE2_OP1))
495 *buf_ptr = reg_lmap[a] << 3;
496 else
497 *buf_ptr = a << 3;
498 }
499 else {
500 if (a & SLJIT_IMM) {
501 if (imma == 1)
502 *inst = GROUP_SHIFT_1;
503 else
504 *inst = GROUP_SHIFT_N;
505 } else
506 *inst = GROUP_SHIFT_CL;
507 *buf_ptr = 0;
508 }
509
510 if (!(b & SLJIT_MEM))
511 *buf_ptr++ |= MOD_REG + ((!(flags & EX86_SSE2_OP2)) ? reg_lmap[b] : b);
512 else if ((b & REG_MASK) != SLJIT_UNUSED) {
513 if ((b & OFFS_REG_MASK) == SLJIT_UNUSED || (b & OFFS_REG_MASK) == TO_OFFS_REG(SLJIT_SP)) {
514 if (immb != 0 || reg_lmap[b & REG_MASK] == 5) {
515 if (immb <= 127 && immb >= -128)
516 *buf_ptr |= 0x40;
517 else
518 *buf_ptr |= 0x80;
519 }
520
521 if ((b & OFFS_REG_MASK) == SLJIT_UNUSED)
522 *buf_ptr++ |= reg_lmap[b & REG_MASK];
523 else {
524 *buf_ptr++ |= 0x04;
525 *buf_ptr++ = reg_lmap[b & REG_MASK] | (reg_lmap[OFFS_REG(b)] << 3);
526 }
527
528 if (immb != 0 || reg_lmap[b & REG_MASK] == 5) {
529 if (immb <= 127 && immb >= -128)
530 *buf_ptr++ = immb; /* 8 bit displacement. */
531 else {
532 *(sljit_si*)buf_ptr = immb; /* 32 bit displacement. */
533 buf_ptr += sizeof(sljit_si);
534 }
535 }
536 }
537 else {
538 if (reg_lmap[b & REG_MASK] == 5)
539 *buf_ptr |= 0x40;
540 *buf_ptr++ |= 0x04;
541 *buf_ptr++ = reg_lmap[b & REG_MASK] | (reg_lmap[OFFS_REG(b)] << 3) | (immb << 6);
542 if (reg_lmap[b & REG_MASK] == 5)
543 *buf_ptr++ = 0;
544 }
545 }
546 else {
547 *buf_ptr++ |= 0x04;
548 *buf_ptr++ = 0x25;
549 *(sljit_si*)buf_ptr = immb; /* 32 bit displacement. */
550 buf_ptr += sizeof(sljit_si);
551 }
552
553 if (a & SLJIT_IMM) {
554 if (flags & EX86_BYTE_ARG)
555 *buf_ptr = imma;
556 else if (flags & EX86_HALF_ARG)
557 *(short*)buf_ptr = imma;
558 else if (!(flags & EX86_SHIFT_INS))
559 *(sljit_si*)buf_ptr = imma;
560 }
561
562 return !(flags & EX86_SHIFT_INS) ? inst : (inst + 1);
563 }
564
565 /* --------------------------------------------------------------------- */
566 /* Call / return instructions */
567 /* --------------------------------------------------------------------- */
568
call_with_args(struct sljit_compiler * compiler,sljit_si type)569 static SLJIT_INLINE sljit_si call_with_args(struct sljit_compiler *compiler, sljit_si type)
570 {
571 sljit_ub *inst;
572
573 #ifndef _WIN64
574 SLJIT_COMPILE_ASSERT(reg_map[SLJIT_R1] == 6 && reg_map[SLJIT_R0] < 8 && reg_map[SLJIT_R2] < 8, args_registers);
575
576 inst = (sljit_ub*)ensure_buf(compiler, 1 + ((type < SLJIT_CALL3) ? 3 : 6));
577 FAIL_IF(!inst);
578 INC_SIZE((type < SLJIT_CALL3) ? 3 : 6);
579 if (type >= SLJIT_CALL3) {
580 *inst++ = REX_W;
581 *inst++ = MOV_r_rm;
582 *inst++ = MOD_REG | (0x2 /* rdx */ << 3) | reg_lmap[SLJIT_R2];
583 }
584 *inst++ = REX_W;
585 *inst++ = MOV_r_rm;
586 *inst++ = MOD_REG | (0x7 /* rdi */ << 3) | reg_lmap[SLJIT_R0];
587 #else
588 SLJIT_COMPILE_ASSERT(reg_map[SLJIT_R1] == 2 && reg_map[SLJIT_R0] < 8 && reg_map[SLJIT_R2] < 8, args_registers);
589
590 inst = (sljit_ub*)ensure_buf(compiler, 1 + ((type < SLJIT_CALL3) ? 3 : 6));
591 FAIL_IF(!inst);
592 INC_SIZE((type < SLJIT_CALL3) ? 3 : 6);
593 if (type >= SLJIT_CALL3) {
594 *inst++ = REX_W | REX_R;
595 *inst++ = MOV_r_rm;
596 *inst++ = MOD_REG | (0x0 /* r8 */ << 3) | reg_lmap[SLJIT_R2];
597 }
598 *inst++ = REX_W;
599 *inst++ = MOV_r_rm;
600 *inst++ = MOD_REG | (0x1 /* rcx */ << 3) | reg_lmap[SLJIT_R0];
601 #endif
602 return SLJIT_SUCCESS;
603 }
604
sljit_emit_fast_enter(struct sljit_compiler * compiler,sljit_si dst,sljit_sw dstw)605 SLJIT_API_FUNC_ATTRIBUTE sljit_si sljit_emit_fast_enter(struct sljit_compiler *compiler, sljit_si dst, sljit_sw dstw)
606 {
607 sljit_ub *inst;
608
609 CHECK_ERROR();
610 check_sljit_emit_fast_enter(compiler, dst, dstw);
611 ADJUST_LOCAL_OFFSET(dst, dstw);
612
613 /* For UNUSED dst. Uncommon, but possible. */
614 if (dst == SLJIT_UNUSED)
615 dst = TMP_REG1;
616
617 if (FAST_IS_REG(dst)) {
618 if (reg_map[dst] < 8) {
619 inst = (sljit_ub*)ensure_buf(compiler, 1 + 1);
620 FAIL_IF(!inst);
621 INC_SIZE(1);
622 POP_REG(reg_lmap[dst]);
623 return SLJIT_SUCCESS;
624 }
625
626 inst = (sljit_ub*)ensure_buf(compiler, 1 + 2);
627 FAIL_IF(!inst);
628 INC_SIZE(2);
629 *inst++ = REX_B;
630 POP_REG(reg_lmap[dst]);
631 return SLJIT_SUCCESS;
632 }
633
634 /* REX_W is not necessary (src is not immediate). */
635 compiler->mode32 = 1;
636 inst = emit_x86_instruction(compiler, 1, 0, 0, dst, dstw);
637 FAIL_IF(!inst);
638 *inst++ = POP_rm;
639 return SLJIT_SUCCESS;
640 }
641
sljit_emit_fast_return(struct sljit_compiler * compiler,sljit_si src,sljit_sw srcw)642 SLJIT_API_FUNC_ATTRIBUTE sljit_si sljit_emit_fast_return(struct sljit_compiler *compiler, sljit_si src, sljit_sw srcw)
643 {
644 sljit_ub *inst;
645
646 CHECK_ERROR();
647 check_sljit_emit_fast_return(compiler, src, srcw);
648 ADJUST_LOCAL_OFFSET(src, srcw);
649
650 if ((src & SLJIT_IMM) && NOT_HALFWORD(srcw)) {
651 FAIL_IF(emit_load_imm64(compiler, TMP_REG1, srcw));
652 src = TMP_REG1;
653 }
654
655 if (FAST_IS_REG(src)) {
656 if (reg_map[src] < 8) {
657 inst = (sljit_ub*)ensure_buf(compiler, 1 + 1 + 1);
658 FAIL_IF(!inst);
659
660 INC_SIZE(1 + 1);
661 PUSH_REG(reg_lmap[src]);
662 }
663 else {
664 inst = (sljit_ub*)ensure_buf(compiler, 1 + 2 + 1);
665 FAIL_IF(!inst);
666
667 INC_SIZE(2 + 1);
668 *inst++ = REX_B;
669 PUSH_REG(reg_lmap[src]);
670 }
671 }
672 else if (src & SLJIT_MEM) {
673 /* REX_W is not necessary (src is not immediate). */
674 compiler->mode32 = 1;
675 inst = emit_x86_instruction(compiler, 1, 0, 0, src, srcw);
676 FAIL_IF(!inst);
677 *inst++ = GROUP_FF;
678 *inst |= PUSH_rm;
679
680 inst = (sljit_ub*)ensure_buf(compiler, 1 + 1);
681 FAIL_IF(!inst);
682 INC_SIZE(1);
683 }
684 else {
685 SLJIT_ASSERT(IS_HALFWORD(srcw));
686 /* SLJIT_IMM. */
687 inst = (sljit_ub*)ensure_buf(compiler, 1 + 5 + 1);
688 FAIL_IF(!inst);
689
690 INC_SIZE(5 + 1);
691 *inst++ = PUSH_i32;
692 *(sljit_si*)inst = srcw;
693 inst += sizeof(sljit_si);
694 }
695
696 RET();
697 return SLJIT_SUCCESS;
698 }
699
700
701 /* --------------------------------------------------------------------- */
702 /* Extend input */
703 /* --------------------------------------------------------------------- */
704
emit_mov_int(struct sljit_compiler * compiler,sljit_si sign,sljit_si dst,sljit_sw dstw,sljit_si src,sljit_sw srcw)705 static sljit_si emit_mov_int(struct sljit_compiler *compiler, sljit_si sign,
706 sljit_si dst, sljit_sw dstw,
707 sljit_si src, sljit_sw srcw)
708 {
709 sljit_ub* inst;
710 sljit_si dst_r;
711
712 compiler->mode32 = 0;
713
714 if (dst == SLJIT_UNUSED && !(src & SLJIT_MEM))
715 return SLJIT_SUCCESS; /* Empty instruction. */
716
717 if (src & SLJIT_IMM) {
718 if (FAST_IS_REG(dst)) {
719 if (sign || ((sljit_uw)srcw <= 0x7fffffff)) {
720 inst = emit_x86_instruction(compiler, 1, SLJIT_IMM, (sljit_sw)(sljit_si)srcw, dst, dstw);
721 FAIL_IF(!inst);
722 *inst = MOV_rm_i32;
723 return SLJIT_SUCCESS;
724 }
725 return emit_load_imm64(compiler, dst, srcw);
726 }
727 compiler->mode32 = 1;
728 inst = emit_x86_instruction(compiler, 1, SLJIT_IMM, (sljit_sw)(sljit_si)srcw, dst, dstw);
729 FAIL_IF(!inst);
730 *inst = MOV_rm_i32;
731 compiler->mode32 = 0;
732 return SLJIT_SUCCESS;
733 }
734
735 dst_r = FAST_IS_REG(dst) ? dst : TMP_REG1;
736
737 if ((dst & SLJIT_MEM) && FAST_IS_REG(src))
738 dst_r = src;
739 else {
740 if (sign) {
741 inst = emit_x86_instruction(compiler, 1, dst_r, 0, src, srcw);
742 FAIL_IF(!inst);
743 *inst++ = MOVSXD_r_rm;
744 } else {
745 compiler->mode32 = 1;
746 FAIL_IF(emit_mov(compiler, dst_r, 0, src, srcw));
747 compiler->mode32 = 0;
748 }
749 }
750
751 if (dst & SLJIT_MEM) {
752 compiler->mode32 = 1;
753 inst = emit_x86_instruction(compiler, 1, dst_r, 0, dst, dstw);
754 FAIL_IF(!inst);
755 *inst = MOV_rm_r;
756 compiler->mode32 = 0;
757 }
758
759 return SLJIT_SUCCESS;
760 }
761