1/* 2 * Stack-less Just-In-Time compiler 3 * 4 * Copyright 2009-2012 Zoltan Herczeg (hzmester@freemail.hu). All rights reserved. 5 * 6 * Redistribution and use in source and binary forms, with or without modification, are 7 * permitted provided that the following conditions are met: 8 * 9 * 1. Redistributions of source code must retain the above copyright notice, this list of 10 * conditions and the following disclaimer. 11 * 12 * 2. Redistributions in binary form must reproduce the above copyright notice, this list 13 * of conditions and the following disclaimer in the documentation and/or other materials 14 * provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER(S) AND CONTRIBUTORS ``AS IS'' AND ANY 17 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 18 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT 19 * SHALL THE COPYRIGHT HOLDER(S) OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, 20 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED 21 * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR 22 * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 23 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN 24 * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 25 */ 26 27/* x86 64-bit arch dependent functions. */ 28 29static sljit_si emit_load_imm64(struct sljit_compiler *compiler, sljit_si reg, sljit_sw imm) 30{ 31 sljit_ub *inst; 32 33 inst = (sljit_ub*)ensure_buf(compiler, 1 + 2 + sizeof(sljit_sw)); 34 FAIL_IF(!inst); 35 INC_SIZE(2 + sizeof(sljit_sw)); 36 *inst++ = REX_W | ((reg_map[reg] <= 7) ? 0 : REX_B); 37 *inst++ = MOV_r_i32 + (reg_map[reg] & 0x7); 38 *(sljit_sw*)inst = imm; 39 return SLJIT_SUCCESS; 40} 41 42static sljit_ub* generate_far_jump_code(struct sljit_jump *jump, sljit_ub *code_ptr, sljit_si type) 43{ 44 if (type < SLJIT_JUMP) { 45 /* Invert type. */ 46 *code_ptr++ = get_jump_code(type ^ 0x1) - 0x10; 47 *code_ptr++ = 10 + 3; 48 } 49 50 SLJIT_COMPILE_ASSERT(reg_map[TMP_REG3] == 9, tmp3_is_9_first); 51 *code_ptr++ = REX_W | REX_B; 52 *code_ptr++ = MOV_r_i32 + 1; 53 jump->addr = (sljit_uw)code_ptr; 54 55 if (jump->flags & JUMP_LABEL) 56 jump->flags |= PATCH_MD; 57 else 58 *(sljit_sw*)code_ptr = jump->u.target; 59 60 code_ptr += sizeof(sljit_sw); 61 *code_ptr++ = REX_B; 62 *code_ptr++ = GROUP_FF; 63 *code_ptr++ = (type >= SLJIT_FAST_CALL) ? (MOD_REG | CALL_rm | 1) : (MOD_REG | JMP_rm | 1); 64 65 return code_ptr; 66} 67 68static sljit_ub* generate_fixed_jump(sljit_ub *code_ptr, sljit_sw addr, sljit_si type) 69{ 70 sljit_sw delta = addr - ((sljit_sw)code_ptr + 1 + sizeof(sljit_si)); 71 72 if (delta <= HALFWORD_MAX && delta >= HALFWORD_MIN) { 73 *code_ptr++ = (type == 2) ? CALL_i32 : JMP_i32; 74 *(sljit_sw*)code_ptr = delta; 75 } 76 else { 77 SLJIT_COMPILE_ASSERT(reg_map[TMP_REG3] == 9, tmp3_is_9_second); 78 *code_ptr++ = REX_W | REX_B; 79 *code_ptr++ = MOV_r_i32 + 1; 80 *(sljit_sw*)code_ptr = addr; 81 code_ptr += sizeof(sljit_sw); 82 *code_ptr++ = REX_B; 83 *code_ptr++ = GROUP_FF; 84 *code_ptr++ = (type == 2) ? (MOD_REG | CALL_rm | 1) : (MOD_REG | JMP_rm | 1); 85 } 86 87 return code_ptr; 88} 89 90SLJIT_API_FUNC_ATTRIBUTE sljit_si sljit_emit_enter(struct sljit_compiler *compiler, 91 sljit_si options, sljit_si args, sljit_si scratches, sljit_si saveds, 92 sljit_si fscratches, sljit_si fsaveds, sljit_si local_size) 93{ 94 sljit_si i, tmp, size, saved_register_size; 95 sljit_ub *inst; 96 97 CHECK_ERROR(); 98 check_sljit_emit_enter(compiler, options, args, scratches, saveds, fscratches, fsaveds, local_size); 99 100 compiler->options = options; 101 compiler->scratches = scratches; 102 compiler->saveds = saveds; 103 compiler->fscratches = fscratches; 104 compiler->fsaveds = fsaveds; 105 compiler->flags_saved = 0; 106#if (defined SLJIT_DEBUG && SLJIT_DEBUG) 107 compiler->logical_local_size = local_size; 108#endif 109 110 /* Including the return address saved by the call instruction. */ 111 saved_register_size = GET_SAVED_REGISTERS_SIZE(scratches, saveds, 1); 112 113 tmp = saveds < SLJIT_NUMBER_OF_SAVED_REGISTERS ? (SLJIT_S0 + 1 - saveds) : SLJIT_FIRST_SAVED_REG; 114 for (i = SLJIT_S0; i >= tmp; i--) { 115 size = reg_map[i] >= 8 ? 2 : 1; 116 inst = (sljit_ub*)ensure_buf(compiler, 1 + size); 117 FAIL_IF(!inst); 118 INC_SIZE(size); 119 if (reg_map[i] >= 8) 120 *inst++ = REX_B; 121 PUSH_REG(reg_lmap[i]); 122 } 123 124 for (i = scratches; i >= SLJIT_FIRST_SAVED_REG; i--) { 125 size = reg_map[i] >= 8 ? 2 : 1; 126 inst = (sljit_ub*)ensure_buf(compiler, 1 + size); 127 FAIL_IF(!inst); 128 INC_SIZE(size); 129 if (reg_map[i] >= 8) 130 *inst++ = REX_B; 131 PUSH_REG(reg_lmap[i]); 132 } 133 134 if (args > 0) { 135 size = args * 3; 136 inst = (sljit_ub*)ensure_buf(compiler, 1 + size); 137 FAIL_IF(!inst); 138 139 INC_SIZE(size); 140 141#ifndef _WIN64 142 if (args > 0) { 143 *inst++ = REX_W; 144 *inst++ = MOV_r_rm; 145 *inst++ = MOD_REG | (reg_map[SLJIT_S0] << 3) | 0x7 /* rdi */; 146 } 147 if (args > 1) { 148 *inst++ = REX_W | REX_R; 149 *inst++ = MOV_r_rm; 150 *inst++ = MOD_REG | (reg_lmap[SLJIT_S1] << 3) | 0x6 /* rsi */; 151 } 152 if (args > 2) { 153 *inst++ = REX_W | REX_R; 154 *inst++ = MOV_r_rm; 155 *inst++ = MOD_REG | (reg_lmap[SLJIT_S2] << 3) | 0x2 /* rdx */; 156 } 157#else 158 if (args > 0) { 159 *inst++ = REX_W; 160 *inst++ = MOV_r_rm; 161 *inst++ = MOD_REG | (reg_map[SLJIT_S0] << 3) | 0x1 /* rcx */; 162 } 163 if (args > 1) { 164 *inst++ = REX_W; 165 *inst++ = MOV_r_rm; 166 *inst++ = MOD_REG | (reg_map[SLJIT_S1] << 3) | 0x2 /* rdx */; 167 } 168 if (args > 2) { 169 *inst++ = REX_W | REX_B; 170 *inst++ = MOV_r_rm; 171 *inst++ = MOD_REG | (reg_map[SLJIT_S2] << 3) | 0x0 /* r8 */; 172 } 173#endif 174 } 175 176 local_size = ((local_size + FIXED_LOCALS_OFFSET + saved_register_size + 16 - 1) & ~(16 - 1)) - saved_register_size; 177 compiler->local_size = local_size; 178 179#ifdef _WIN64 180 if (local_size > 1024) { 181 /* Allocate stack for the callback, which grows the stack. */ 182 inst = (sljit_ub*)ensure_buf(compiler, 1 + 4 + (3 + sizeof(sljit_si))); 183 FAIL_IF(!inst); 184 INC_SIZE(4 + (3 + sizeof(sljit_si))); 185 *inst++ = REX_W; 186 *inst++ = GROUP_BINARY_83; 187 *inst++ = MOD_REG | SUB | 4; 188 /* Allocated size for registers must be divisible by 8. */ 189 SLJIT_ASSERT(!(saved_register_size & 0x7)); 190 /* Aligned to 16 byte. */ 191 if (saved_register_size & 0x8) { 192 *inst++ = 5 * sizeof(sljit_sw); 193 local_size -= 5 * sizeof(sljit_sw); 194 } else { 195 *inst++ = 4 * sizeof(sljit_sw); 196 local_size -= 4 * sizeof(sljit_sw); 197 } 198 /* Second instruction */ 199 SLJIT_COMPILE_ASSERT(reg_map[SLJIT_R0] < 8, temporary_reg1_is_loreg); 200 *inst++ = REX_W; 201 *inst++ = MOV_rm_i32; 202 *inst++ = MOD_REG | reg_lmap[SLJIT_R0]; 203 *(sljit_si*)inst = local_size; 204#if (defined SLJIT_VERBOSE && SLJIT_VERBOSE) || (defined SLJIT_DEBUG && SLJIT_DEBUG) 205 compiler->skip_checks = 1; 206#endif 207 FAIL_IF(sljit_emit_ijump(compiler, SLJIT_CALL1, SLJIT_IMM, SLJIT_FUNC_OFFSET(sljit_grow_stack))); 208 } 209#endif 210 211 SLJIT_ASSERT(local_size > 0); 212 if (local_size <= 127) { 213 inst = (sljit_ub*)ensure_buf(compiler, 1 + 4); 214 FAIL_IF(!inst); 215 INC_SIZE(4); 216 *inst++ = REX_W; 217 *inst++ = GROUP_BINARY_83; 218 *inst++ = MOD_REG | SUB | 4; 219 *inst++ = local_size; 220 } 221 else { 222 inst = (sljit_ub*)ensure_buf(compiler, 1 + 7); 223 FAIL_IF(!inst); 224 INC_SIZE(7); 225 *inst++ = REX_W; 226 *inst++ = GROUP_BINARY_81; 227 *inst++ = MOD_REG | SUB | 4; 228 *(sljit_si*)inst = local_size; 229 inst += sizeof(sljit_si); 230 } 231 232#ifdef _WIN64 233 /* Save xmm6 register: movaps [rsp + 0x20], xmm6 */ 234 if (fscratches >= 6 || fsaveds >= 1) { 235 inst = (sljit_ub*)ensure_buf(compiler, 1 + 5); 236 FAIL_IF(!inst); 237 INC_SIZE(5); 238 *inst++ = GROUP_0F; 239 *(sljit_si*)inst = 0x20247429; 240 } 241#endif 242 243 return SLJIT_SUCCESS; 244} 245 246SLJIT_API_FUNC_ATTRIBUTE void sljit_set_context(struct sljit_compiler *compiler, 247 sljit_si options, sljit_si args, sljit_si scratches, sljit_si saveds, 248 sljit_si fscratches, sljit_si fsaveds, sljit_si local_size) 249{ 250 sljit_si saved_register_size; 251 252 CHECK_ERROR_VOID(); 253 check_sljit_set_context(compiler, options, args, scratches, saveds, fscratches, fsaveds, local_size); 254 255 compiler->options = options; 256 compiler->scratches = scratches; 257 compiler->saveds = saveds; 258 compiler->fscratches = fscratches; 259 compiler->fsaveds = fsaveds; 260#if (defined SLJIT_DEBUG && SLJIT_DEBUG) 261 compiler->logical_local_size = local_size; 262#endif 263 264 /* Including the return address saved by the call instruction. */ 265 saved_register_size = GET_SAVED_REGISTERS_SIZE(scratches, saveds, 1); 266 compiler->local_size = ((local_size + FIXED_LOCALS_OFFSET + saved_register_size + 16 - 1) & ~(16 - 1)) - saved_register_size; 267} 268 269SLJIT_API_FUNC_ATTRIBUTE sljit_si sljit_emit_return(struct sljit_compiler *compiler, sljit_si op, sljit_si src, sljit_sw srcw) 270{ 271 sljit_si i, tmp, size; 272 sljit_ub *inst; 273 274 CHECK_ERROR(); 275 check_sljit_emit_return(compiler, op, src, srcw); 276 277 compiler->flags_saved = 0; 278 FAIL_IF(emit_mov_before_return(compiler, op, src, srcw)); 279 280#ifdef _WIN64 281 /* Restore xmm6 register: movaps xmm6, [rsp + 0x20] */ 282 if (compiler->fscratches >= 6 || compiler->fsaveds >= 1) { 283 inst = (sljit_ub*)ensure_buf(compiler, 1 + 5); 284 FAIL_IF(!inst); 285 INC_SIZE(5); 286 *inst++ = GROUP_0F; 287 *(sljit_si*)inst = 0x20247428; 288 } 289#endif 290 291 SLJIT_ASSERT(compiler->local_size > 0); 292 if (compiler->local_size <= 127) { 293 inst = (sljit_ub*)ensure_buf(compiler, 1 + 4); 294 FAIL_IF(!inst); 295 INC_SIZE(4); 296 *inst++ = REX_W; 297 *inst++ = GROUP_BINARY_83; 298 *inst++ = MOD_REG | ADD | 4; 299 *inst = compiler->local_size; 300 } 301 else { 302 inst = (sljit_ub*)ensure_buf(compiler, 1 + 7); 303 FAIL_IF(!inst); 304 INC_SIZE(7); 305 *inst++ = REX_W; 306 *inst++ = GROUP_BINARY_81; 307 *inst++ = MOD_REG | ADD | 4; 308 *(sljit_si*)inst = compiler->local_size; 309 } 310 311 tmp = compiler->scratches; 312 for (i = SLJIT_FIRST_SAVED_REG; i <= tmp; i++) { 313 size = reg_map[i] >= 8 ? 2 : 1; 314 inst = (sljit_ub*)ensure_buf(compiler, 1 + size); 315 FAIL_IF(!inst); 316 INC_SIZE(size); 317 if (reg_map[i] >= 8) 318 *inst++ = REX_B; 319 POP_REG(reg_lmap[i]); 320 } 321 322 tmp = compiler->saveds < SLJIT_NUMBER_OF_SAVED_REGISTERS ? (SLJIT_S0 + 1 - compiler->saveds) : SLJIT_FIRST_SAVED_REG; 323 for (i = tmp; i <= SLJIT_S0; i++) { 324 size = reg_map[i] >= 8 ? 2 : 1; 325 inst = (sljit_ub*)ensure_buf(compiler, 1 + size); 326 FAIL_IF(!inst); 327 INC_SIZE(size); 328 if (reg_map[i] >= 8) 329 *inst++ = REX_B; 330 POP_REG(reg_lmap[i]); 331 } 332 333 inst = (sljit_ub*)ensure_buf(compiler, 1 + 1); 334 FAIL_IF(!inst); 335 INC_SIZE(1); 336 RET(); 337 return SLJIT_SUCCESS; 338} 339 340/* --------------------------------------------------------------------- */ 341/* Operators */ 342/* --------------------------------------------------------------------- */ 343 344static sljit_si emit_do_imm32(struct sljit_compiler *compiler, sljit_ub rex, sljit_ub opcode, sljit_sw imm) 345{ 346 sljit_ub *inst; 347 sljit_si length = 1 + (rex ? 1 : 0) + sizeof(sljit_si); 348 349 inst = (sljit_ub*)ensure_buf(compiler, 1 + length); 350 FAIL_IF(!inst); 351 INC_SIZE(length); 352 if (rex) 353 *inst++ = rex; 354 *inst++ = opcode; 355 *(sljit_si*)inst = imm; 356 return SLJIT_SUCCESS; 357} 358 359static sljit_ub* emit_x86_instruction(struct sljit_compiler *compiler, sljit_si size, 360 /* The register or immediate operand. */ 361 sljit_si a, sljit_sw imma, 362 /* The general operand (not immediate). */ 363 sljit_si b, sljit_sw immb) 364{ 365 sljit_ub *inst; 366 sljit_ub *buf_ptr; 367 sljit_ub rex = 0; 368 sljit_si flags = size & ~0xf; 369 sljit_si inst_size; 370 371 /* The immediate operand must be 32 bit. */ 372 SLJIT_ASSERT(!(a & SLJIT_IMM) || compiler->mode32 || IS_HALFWORD(imma)); 373 /* Both cannot be switched on. */ 374 SLJIT_ASSERT((flags & (EX86_BIN_INS | EX86_SHIFT_INS)) != (EX86_BIN_INS | EX86_SHIFT_INS)); 375 /* Size flags not allowed for typed instructions. */ 376 SLJIT_ASSERT(!(flags & (EX86_BIN_INS | EX86_SHIFT_INS)) || (flags & (EX86_BYTE_ARG | EX86_HALF_ARG)) == 0); 377 /* Both size flags cannot be switched on. */ 378 SLJIT_ASSERT((flags & (EX86_BYTE_ARG | EX86_HALF_ARG)) != (EX86_BYTE_ARG | EX86_HALF_ARG)); 379 /* SSE2 and immediate is not possible. */ 380 SLJIT_ASSERT(!(a & SLJIT_IMM) || !(flags & EX86_SSE2)); 381 SLJIT_ASSERT((flags & (EX86_PREF_F2 | EX86_PREF_F3)) != (EX86_PREF_F2 | EX86_PREF_F3) 382 && (flags & (EX86_PREF_F2 | EX86_PREF_66)) != (EX86_PREF_F2 | EX86_PREF_66) 383 && (flags & (EX86_PREF_F3 | EX86_PREF_66)) != (EX86_PREF_F3 | EX86_PREF_66)); 384 385 size &= 0xf; 386 inst_size = size; 387 388 if (!compiler->mode32 && !(flags & EX86_NO_REXW)) 389 rex |= REX_W; 390 else if (flags & EX86_REX) 391 rex |= REX; 392 393 if (flags & (EX86_PREF_F2 | EX86_PREF_F3)) 394 inst_size++; 395 if (flags & EX86_PREF_66) 396 inst_size++; 397 398 /* Calculate size of b. */ 399 inst_size += 1; /* mod r/m byte. */ 400 if (b & SLJIT_MEM) { 401 if (!(b & OFFS_REG_MASK)) { 402 if (NOT_HALFWORD(immb)) { 403 if (emit_load_imm64(compiler, TMP_REG3, immb)) 404 return NULL; 405 immb = 0; 406 if (b & REG_MASK) 407 b |= TO_OFFS_REG(TMP_REG3); 408 else 409 b |= TMP_REG3; 410 } 411 else if (reg_lmap[b & REG_MASK] == 4) 412 b |= TO_OFFS_REG(SLJIT_SP); 413 } 414 415 if ((b & REG_MASK) == SLJIT_UNUSED) 416 inst_size += 1 + sizeof(sljit_si); /* SIB byte required to avoid RIP based addressing. */ 417 else { 418 if (reg_map[b & REG_MASK] >= 8) 419 rex |= REX_B; 420 421 if (immb != 0 && (!(b & OFFS_REG_MASK) || (b & OFFS_REG_MASK) == TO_OFFS_REG(SLJIT_SP))) { 422 /* Immediate operand. */ 423 if (immb <= 127 && immb >= -128) 424 inst_size += sizeof(sljit_sb); 425 else 426 inst_size += sizeof(sljit_si); 427 } 428 else if (reg_lmap[b & REG_MASK] == 5) 429 inst_size += sizeof(sljit_sb); 430 431 if ((b & OFFS_REG_MASK) != SLJIT_UNUSED) { 432 inst_size += 1; /* SIB byte. */ 433 if (reg_map[OFFS_REG(b)] >= 8) 434 rex |= REX_X; 435 } 436 } 437 } 438 else if (!(flags & EX86_SSE2_OP2) && reg_map[b] >= 8) 439 rex |= REX_B; 440 441 if (a & SLJIT_IMM) { 442 if (flags & EX86_BIN_INS) { 443 if (imma <= 127 && imma >= -128) { 444 inst_size += 1; 445 flags |= EX86_BYTE_ARG; 446 } else 447 inst_size += 4; 448 } 449 else if (flags & EX86_SHIFT_INS) { 450 imma &= compiler->mode32 ? 0x1f : 0x3f; 451 if (imma != 1) { 452 inst_size ++; 453 flags |= EX86_BYTE_ARG; 454 } 455 } else if (flags & EX86_BYTE_ARG) 456 inst_size++; 457 else if (flags & EX86_HALF_ARG) 458 inst_size += sizeof(short); 459 else 460 inst_size += sizeof(sljit_si); 461 } 462 else { 463 SLJIT_ASSERT(!(flags & EX86_SHIFT_INS) || a == SLJIT_PREF_SHIFT_REG); 464 /* reg_map[SLJIT_PREF_SHIFT_REG] is less than 8. */ 465 if (!(flags & EX86_SSE2_OP1) && reg_map[a] >= 8) 466 rex |= REX_R; 467 } 468 469 if (rex) 470 inst_size++; 471 472 inst = (sljit_ub*)ensure_buf(compiler, 1 + inst_size); 473 PTR_FAIL_IF(!inst); 474 475 /* Encoding the byte. */ 476 INC_SIZE(inst_size); 477 if (flags & EX86_PREF_F2) 478 *inst++ = 0xf2; 479 if (flags & EX86_PREF_F3) 480 *inst++ = 0xf3; 481 if (flags & EX86_PREF_66) 482 *inst++ = 0x66; 483 if (rex) 484 *inst++ = rex; 485 buf_ptr = inst + size; 486 487 /* Encode mod/rm byte. */ 488 if (!(flags & EX86_SHIFT_INS)) { 489 if ((flags & EX86_BIN_INS) && (a & SLJIT_IMM)) 490 *inst = (flags & EX86_BYTE_ARG) ? GROUP_BINARY_83 : GROUP_BINARY_81; 491 492 if ((a & SLJIT_IMM) || (a == 0)) 493 *buf_ptr = 0; 494 else if (!(flags & EX86_SSE2_OP1)) 495 *buf_ptr = reg_lmap[a] << 3; 496 else 497 *buf_ptr = a << 3; 498 } 499 else { 500 if (a & SLJIT_IMM) { 501 if (imma == 1) 502 *inst = GROUP_SHIFT_1; 503 else 504 *inst = GROUP_SHIFT_N; 505 } else 506 *inst = GROUP_SHIFT_CL; 507 *buf_ptr = 0; 508 } 509 510 if (!(b & SLJIT_MEM)) 511 *buf_ptr++ |= MOD_REG + ((!(flags & EX86_SSE2_OP2)) ? reg_lmap[b] : b); 512 else if ((b & REG_MASK) != SLJIT_UNUSED) { 513 if ((b & OFFS_REG_MASK) == SLJIT_UNUSED || (b & OFFS_REG_MASK) == TO_OFFS_REG(SLJIT_SP)) { 514 if (immb != 0 || reg_lmap[b & REG_MASK] == 5) { 515 if (immb <= 127 && immb >= -128) 516 *buf_ptr |= 0x40; 517 else 518 *buf_ptr |= 0x80; 519 } 520 521 if ((b & OFFS_REG_MASK) == SLJIT_UNUSED) 522 *buf_ptr++ |= reg_lmap[b & REG_MASK]; 523 else { 524 *buf_ptr++ |= 0x04; 525 *buf_ptr++ = reg_lmap[b & REG_MASK] | (reg_lmap[OFFS_REG(b)] << 3); 526 } 527 528 if (immb != 0 || reg_lmap[b & REG_MASK] == 5) { 529 if (immb <= 127 && immb >= -128) 530 *buf_ptr++ = immb; /* 8 bit displacement. */ 531 else { 532 *(sljit_si*)buf_ptr = immb; /* 32 bit displacement. */ 533 buf_ptr += sizeof(sljit_si); 534 } 535 } 536 } 537 else { 538 if (reg_lmap[b & REG_MASK] == 5) 539 *buf_ptr |= 0x40; 540 *buf_ptr++ |= 0x04; 541 *buf_ptr++ = reg_lmap[b & REG_MASK] | (reg_lmap[OFFS_REG(b)] << 3) | (immb << 6); 542 if (reg_lmap[b & REG_MASK] == 5) 543 *buf_ptr++ = 0; 544 } 545 } 546 else { 547 *buf_ptr++ |= 0x04; 548 *buf_ptr++ = 0x25; 549 *(sljit_si*)buf_ptr = immb; /* 32 bit displacement. */ 550 buf_ptr += sizeof(sljit_si); 551 } 552 553 if (a & SLJIT_IMM) { 554 if (flags & EX86_BYTE_ARG) 555 *buf_ptr = imma; 556 else if (flags & EX86_HALF_ARG) 557 *(short*)buf_ptr = imma; 558 else if (!(flags & EX86_SHIFT_INS)) 559 *(sljit_si*)buf_ptr = imma; 560 } 561 562 return !(flags & EX86_SHIFT_INS) ? inst : (inst + 1); 563} 564 565/* --------------------------------------------------------------------- */ 566/* Call / return instructions */ 567/* --------------------------------------------------------------------- */ 568 569static SLJIT_INLINE sljit_si call_with_args(struct sljit_compiler *compiler, sljit_si type) 570{ 571 sljit_ub *inst; 572 573#ifndef _WIN64 574 SLJIT_COMPILE_ASSERT(reg_map[SLJIT_R1] == 6 && reg_map[SLJIT_R0] < 8 && reg_map[SLJIT_R2] < 8, args_registers); 575 576 inst = (sljit_ub*)ensure_buf(compiler, 1 + ((type < SLJIT_CALL3) ? 3 : 6)); 577 FAIL_IF(!inst); 578 INC_SIZE((type < SLJIT_CALL3) ? 3 : 6); 579 if (type >= SLJIT_CALL3) { 580 *inst++ = REX_W; 581 *inst++ = MOV_r_rm; 582 *inst++ = MOD_REG | (0x2 /* rdx */ << 3) | reg_lmap[SLJIT_R2]; 583 } 584 *inst++ = REX_W; 585 *inst++ = MOV_r_rm; 586 *inst++ = MOD_REG | (0x7 /* rdi */ << 3) | reg_lmap[SLJIT_R0]; 587#else 588 SLJIT_COMPILE_ASSERT(reg_map[SLJIT_R1] == 2 && reg_map[SLJIT_R0] < 8 && reg_map[SLJIT_R2] < 8, args_registers); 589 590 inst = (sljit_ub*)ensure_buf(compiler, 1 + ((type < SLJIT_CALL3) ? 3 : 6)); 591 FAIL_IF(!inst); 592 INC_SIZE((type < SLJIT_CALL3) ? 3 : 6); 593 if (type >= SLJIT_CALL3) { 594 *inst++ = REX_W | REX_R; 595 *inst++ = MOV_r_rm; 596 *inst++ = MOD_REG | (0x0 /* r8 */ << 3) | reg_lmap[SLJIT_R2]; 597 } 598 *inst++ = REX_W; 599 *inst++ = MOV_r_rm; 600 *inst++ = MOD_REG | (0x1 /* rcx */ << 3) | reg_lmap[SLJIT_R0]; 601#endif 602 return SLJIT_SUCCESS; 603} 604 605SLJIT_API_FUNC_ATTRIBUTE sljit_si sljit_emit_fast_enter(struct sljit_compiler *compiler, sljit_si dst, sljit_sw dstw) 606{ 607 sljit_ub *inst; 608 609 CHECK_ERROR(); 610 check_sljit_emit_fast_enter(compiler, dst, dstw); 611 ADJUST_LOCAL_OFFSET(dst, dstw); 612 613 /* For UNUSED dst. Uncommon, but possible. */ 614 if (dst == SLJIT_UNUSED) 615 dst = TMP_REG1; 616 617 if (FAST_IS_REG(dst)) { 618 if (reg_map[dst] < 8) { 619 inst = (sljit_ub*)ensure_buf(compiler, 1 + 1); 620 FAIL_IF(!inst); 621 INC_SIZE(1); 622 POP_REG(reg_lmap[dst]); 623 return SLJIT_SUCCESS; 624 } 625 626 inst = (sljit_ub*)ensure_buf(compiler, 1 + 2); 627 FAIL_IF(!inst); 628 INC_SIZE(2); 629 *inst++ = REX_B; 630 POP_REG(reg_lmap[dst]); 631 return SLJIT_SUCCESS; 632 } 633 634 /* REX_W is not necessary (src is not immediate). */ 635 compiler->mode32 = 1; 636 inst = emit_x86_instruction(compiler, 1, 0, 0, dst, dstw); 637 FAIL_IF(!inst); 638 *inst++ = POP_rm; 639 return SLJIT_SUCCESS; 640} 641 642SLJIT_API_FUNC_ATTRIBUTE sljit_si sljit_emit_fast_return(struct sljit_compiler *compiler, sljit_si src, sljit_sw srcw) 643{ 644 sljit_ub *inst; 645 646 CHECK_ERROR(); 647 check_sljit_emit_fast_return(compiler, src, srcw); 648 ADJUST_LOCAL_OFFSET(src, srcw); 649 650 if ((src & SLJIT_IMM) && NOT_HALFWORD(srcw)) { 651 FAIL_IF(emit_load_imm64(compiler, TMP_REG1, srcw)); 652 src = TMP_REG1; 653 } 654 655 if (FAST_IS_REG(src)) { 656 if (reg_map[src] < 8) { 657 inst = (sljit_ub*)ensure_buf(compiler, 1 + 1 + 1); 658 FAIL_IF(!inst); 659 660 INC_SIZE(1 + 1); 661 PUSH_REG(reg_lmap[src]); 662 } 663 else { 664 inst = (sljit_ub*)ensure_buf(compiler, 1 + 2 + 1); 665 FAIL_IF(!inst); 666 667 INC_SIZE(2 + 1); 668 *inst++ = REX_B; 669 PUSH_REG(reg_lmap[src]); 670 } 671 } 672 else if (src & SLJIT_MEM) { 673 /* REX_W is not necessary (src is not immediate). */ 674 compiler->mode32 = 1; 675 inst = emit_x86_instruction(compiler, 1, 0, 0, src, srcw); 676 FAIL_IF(!inst); 677 *inst++ = GROUP_FF; 678 *inst |= PUSH_rm; 679 680 inst = (sljit_ub*)ensure_buf(compiler, 1 + 1); 681 FAIL_IF(!inst); 682 INC_SIZE(1); 683 } 684 else { 685 SLJIT_ASSERT(IS_HALFWORD(srcw)); 686 /* SLJIT_IMM. */ 687 inst = (sljit_ub*)ensure_buf(compiler, 1 + 5 + 1); 688 FAIL_IF(!inst); 689 690 INC_SIZE(5 + 1); 691 *inst++ = PUSH_i32; 692 *(sljit_si*)inst = srcw; 693 inst += sizeof(sljit_si); 694 } 695 696 RET(); 697 return SLJIT_SUCCESS; 698} 699 700 701/* --------------------------------------------------------------------- */ 702/* Extend input */ 703/* --------------------------------------------------------------------- */ 704 705static sljit_si emit_mov_int(struct sljit_compiler *compiler, sljit_si sign, 706 sljit_si dst, sljit_sw dstw, 707 sljit_si src, sljit_sw srcw) 708{ 709 sljit_ub* inst; 710 sljit_si dst_r; 711 712 compiler->mode32 = 0; 713 714 if (dst == SLJIT_UNUSED && !(src & SLJIT_MEM)) 715 return SLJIT_SUCCESS; /* Empty instruction. */ 716 717 if (src & SLJIT_IMM) { 718 if (FAST_IS_REG(dst)) { 719 if (sign || ((sljit_uw)srcw <= 0x7fffffff)) { 720 inst = emit_x86_instruction(compiler, 1, SLJIT_IMM, (sljit_sw)(sljit_si)srcw, dst, dstw); 721 FAIL_IF(!inst); 722 *inst = MOV_rm_i32; 723 return SLJIT_SUCCESS; 724 } 725 return emit_load_imm64(compiler, dst, srcw); 726 } 727 compiler->mode32 = 1; 728 inst = emit_x86_instruction(compiler, 1, SLJIT_IMM, (sljit_sw)(sljit_si)srcw, dst, dstw); 729 FAIL_IF(!inst); 730 *inst = MOV_rm_i32; 731 compiler->mode32 = 0; 732 return SLJIT_SUCCESS; 733 } 734 735 dst_r = FAST_IS_REG(dst) ? dst : TMP_REG1; 736 737 if ((dst & SLJIT_MEM) && FAST_IS_REG(src)) 738 dst_r = src; 739 else { 740 if (sign) { 741 inst = emit_x86_instruction(compiler, 1, dst_r, 0, src, srcw); 742 FAIL_IF(!inst); 743 *inst++ = MOVSXD_r_rm; 744 } else { 745 compiler->mode32 = 1; 746 FAIL_IF(emit_mov(compiler, dst_r, 0, src, srcw)); 747 compiler->mode32 = 0; 748 } 749 } 750 751 if (dst & SLJIT_MEM) { 752 compiler->mode32 = 1; 753 inst = emit_x86_instruction(compiler, 1, dst_r, 0, dst, dstw); 754 FAIL_IF(!inst); 755 *inst = MOV_rm_r; 756 compiler->mode32 = 0; 757 } 758 759 return SLJIT_SUCCESS; 760} 761