/* * Copyright (C) 2012 The Android Open Source Project * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include "codegen_x86.h" #include "base/bit_utils.h" #include "base/logging.h" #include "dex/compiler_ir.h" #include "dex/quick/mir_to_lir.h" #include "oat.h" #include "utils.h" #include "x86_lir.h" namespace art { #define MAX_ASSEMBLER_RETRIES 50 const X86EncodingMap X86Mir2Lir::EncodingMap[kX86Last] = { { kX8632BitData, kData, IS_UNARY_OP, { 0, 0, 0x00, 0, 0, 0, 0, 4, false }, "data", "0x!0d" }, { kX86Bkpt, kNullary, NO_OPERAND | IS_BRANCH, { 0, 0, 0xCC, 0, 0, 0, 0, 0, false }, "int 3", "" }, { kX86Nop, kNop, NO_OPERAND, { 0, 0, 0x90, 0, 0, 0, 0, 0, false }, "nop", "" }, #define ENCODING_MAP(opname, mem_use, reg_def, uses_ccodes, \ rm8_r8, rm32_r32, \ r8_rm8, r32_rm32, \ ax8_i8, ax32_i32, \ rm8_i8, rm8_i8_modrm, \ rm32_i32, rm32_i32_modrm, \ rm32_i8, rm32_i8_modrm) \ { kX86 ## opname ## 8MR, kMemReg, mem_use | IS_TERTIARY_OP | REG_USE02 | SETS_CCODES | uses_ccodes, { 0, 0, rm8_r8, 0, 0, 0, 0, 0, true }, #opname "8MR", "[!0r+!1d],!2r" }, \ { kX86 ## opname ## 8AR, kArrayReg, mem_use | IS_QUIN_OP | REG_USE014 | SETS_CCODES | uses_ccodes, { 0, 0, rm8_r8, 0, 0, 0, 0, 0, true }, #opname "8AR", "[!0r+!1r< 7; } static uint8_t LowRegisterBits(int32_t raw_reg) { uint8_t low_reg = RegStorage::RegNum(raw_reg) & kRegNumMask32; // 3 bits DCHECK_LT(low_reg, 8); return low_reg; } static bool HasModrm(const X86EncodingMap* entry) { switch (entry->kind) { case kNullary: return false; case kRegOpcode: return false; default: return true; } } static bool HasSib(const X86EncodingMap* entry) { switch (entry->kind) { case kArray: return true; case kArrayReg: return true; case kRegArray: return true; case kArrayImm: return true; case kRegArrayImm: return true; case kShiftArrayImm: return true; case kShiftArrayCl: return true; case kArrayCond: return true; case kCall: switch (entry->opcode) { case kX86CallA: return true; default: return false; } case kPcRel: switch (entry->opcode) { case kX86PcRelLoadRA: return true; default: return false; } default: return false; } } static bool ModrmIsRegReg(const X86EncodingMap* entry) { switch (entry->kind) { // There is no modrm for this kind of instruction, therefore the reg doesn't form part of the // modrm: case kNullary: return true; case kRegOpcode: return true; case kMovRegImm: return true; // Regular modrm value of 3 cases, when there is one register the other register holds an // opcode so the base register is special. case kReg: return true; case kRegReg: return true; case kRegRegStore: return true; case kRegImm: return true; case kRegRegImm: return true; case kRegRegImmStore: return true; case kShiftRegImm: return true; case kShiftRegCl: return true; case kRegCond: return true; case kRegRegCond: return true; case kShiftRegRegCl: return true; case kJmp: switch (entry->opcode) { case kX86JmpR: return true; default: return false; } case kCall: switch (entry->opcode) { case kX86CallR: return true; default: return false; } default: return false; } } static bool IsByteSecondOperand(const X86EncodingMap* entry) { return StartsWith(entry->name, "Movzx8") || StartsWith(entry->name, "Movsx8"); } size_t X86Mir2Lir::ComputeSize(const X86EncodingMap* entry, int32_t raw_reg, int32_t raw_index, int32_t raw_base, int32_t displacement) { bool has_modrm = HasModrm(entry); bool has_sib = HasSib(entry); bool r8_form = entry->skeleton.r8_form; bool modrm_is_reg_reg = ModrmIsRegReg(entry); if (has_sib) { DCHECK(!modrm_is_reg_reg); } size_t size = 0; if (entry->skeleton.prefix1 > 0) { ++size; if (entry->skeleton.prefix2 > 0) { ++size; } } if (cu_->target64 || kIsDebugBuild) { bool registers_need_rex_prefix = NeedsRex(raw_reg) || NeedsRex(raw_index) || NeedsRex(raw_base); if (r8_form) { // Do we need an empty REX prefix to normalize byte registers? registers_need_rex_prefix = registers_need_rex_prefix || (RegStorage::RegNum(raw_reg) >= 4 && !IsByteSecondOperand(entry)); registers_need_rex_prefix = registers_need_rex_prefix || (modrm_is_reg_reg && (RegStorage::RegNum(raw_base) >= 4)); } if (registers_need_rex_prefix) { DCHECK(cu_->target64) << "Attempt to use a 64-bit only addressable register " << RegStorage::RegNum(raw_reg) << " with instruction " << entry->name; if (entry->skeleton.prefix1 != REX_W && entry->skeleton.prefix2 != REX_W && entry->skeleton.prefix1 != REX && entry->skeleton.prefix2 != REX) { ++size; // rex } } } ++size; // opcode if (entry->skeleton.opcode == 0x0F) { ++size; if (entry->skeleton.extra_opcode1 == 0x38 || entry->skeleton.extra_opcode1 == 0x3A) { ++size; } } if (has_modrm) { ++size; // modrm } if (!modrm_is_reg_reg) { if (has_sib || (LowRegisterBits(raw_base) == rs_rX86_SP_32.GetRegNum()) || (cu_->target64 && entry->skeleton.prefix1 == THREAD_PREFIX)) { // SP requires a SIB byte. // GS access also needs a SIB byte for absolute adressing in 64-bit mode. ++size; } if (displacement != 0 || LowRegisterBits(raw_base) == rs_rBP.GetRegNum()) { // BP requires an explicit displacement, even when it's 0. if (entry->opcode != kX86Lea32RA && entry->opcode != kX86Lea64RA && entry->opcode != kX86Lea32RM && entry->opcode != kX86Lea64RM) { DCHECK_NE(entry->flags & (IS_LOAD | IS_STORE), UINT64_C(0)) << entry->name; } if (raw_base == kRIPReg) { DCHECK(cu_->target64) << "Attempt to use a 64-bit RIP adressing with instruction " << entry->name; size += 4; } else { size += IS_SIMM8(displacement) ? 1 : 4; } } } size += entry->skeleton.immediate_bytes; return size; } size_t X86Mir2Lir::GetInsnSize(LIR* lir) { DCHECK(!IsPseudoLirOp(lir->opcode)); const X86EncodingMap* entry = &X86Mir2Lir::EncodingMap[lir->opcode]; DCHECK_EQ(entry->opcode, lir->opcode) << entry->name; switch (entry->kind) { case kData: return 4; // 4 bytes of data. case kNop: return lir->operands[0]; // Length of nop is sole operand. case kNullary: return ComputeSize(entry, NO_REG, NO_REG, NO_REG, 0); case kRegOpcode: // lir operands - 0: reg return ComputeSize(entry, NO_REG, NO_REG, lir->operands[0], 0); case kReg: // lir operands - 0: reg return ComputeSize(entry, NO_REG, NO_REG, lir->operands[0], 0); case kMem: // lir operands - 0: base, 1: disp return ComputeSize(entry, NO_REG, NO_REG, lir->operands[0], lir->operands[1]); case kArray: // lir operands - 0: base, 1: index, 2: scale, 3: disp return ComputeSize(entry, NO_REG, lir->operands[1], lir->operands[0], lir->operands[3]); case kMemReg: // lir operands - 0: base, 1: disp, 2: reg return ComputeSize(entry, lir->operands[2], NO_REG, lir->operands[0], lir->operands[1]); case kMemRegImm: // lir operands - 0: base, 1: disp, 2: reg 3: immediate return ComputeSize(entry, lir->operands[2], NO_REG, lir->operands[0], lir->operands[1]); case kArrayReg: // lir operands - 0: base, 1: index, 2: scale, 3: disp, 4: reg return ComputeSize(entry, lir->operands[4], lir->operands[1], lir->operands[0], lir->operands[3]); case kThreadReg: // lir operands - 0: disp, 1: reg // Thread displacement size is always 32bit. return ComputeSize(entry, lir->operands[1], NO_REG, NO_REG, 0x12345678); case kRegReg: // lir operands - 0: reg1, 1: reg2 return ComputeSize(entry, lir->operands[0], NO_REG, lir->operands[1], 0); case kRegRegStore: // lir operands - 0: reg2, 1: reg1 return ComputeSize(entry, lir->operands[1], NO_REG, lir->operands[0], 0); case kRegMem: // lir operands - 0: reg, 1: base, 2: disp return ComputeSize(entry, lir->operands[0], NO_REG, lir->operands[1], lir->operands[2]); case kRegArray: // lir operands - 0: reg, 1: base, 2: index, 3: scale, 4: disp return ComputeSize(entry, lir->operands[0], lir->operands[2], lir->operands[1], lir->operands[4]); case kRegThread: // lir operands - 0: reg, 1: disp // Thread displacement size is always 32bit. return ComputeSize(entry, lir->operands[0], NO_REG, NO_REG, 0x12345678); case kRegImm: { // lir operands - 0: reg, 1: immediate size_t size = ComputeSize(entry, lir->operands[0], NO_REG, NO_REG, 0); // AX opcodes don't require the modrm byte. if (entry->skeleton.ax_opcode == 0) { return size; } else { return size - (RegStorage::RegNum(lir->operands[0]) == rs_rAX.GetRegNum() ? 1 : 0); } } case kMemImm: // lir operands - 0: base, 1: disp, 2: immediate return ComputeSize(entry, NO_REG, NO_REG, lir->operands[0], lir->operands[1]); case kArrayImm: // lir operands - 0: base, 1: index, 2: scale, 3: disp 4: immediate return ComputeSize(entry, NO_REG, lir->operands[1], lir->operands[0], lir->operands[3]); case kThreadImm: // lir operands - 0: disp, 1: imm // Thread displacement size is always 32bit. return ComputeSize(entry, NO_REG, NO_REG, NO_REG, 0x12345678); case kRegRegImm: // lir operands - 0: reg1, 1: reg2, 2: imm // Note: RegRegImm form passes reg2 as index but encodes it using base. return ComputeSize(entry, lir->operands[0], lir->operands[1], NO_REG, 0); case kRegRegImmStore: // lir operands - 0: reg2, 1: reg1, 2: imm // Note: RegRegImmStore form passes reg1 as index but encodes it using base. return ComputeSize(entry, lir->operands[1], lir->operands[0], NO_REG, 0); case kRegMemImm: // lir operands - 0: reg, 1: base, 2: disp, 3: imm return ComputeSize(entry, lir->operands[0], NO_REG, lir->operands[1], lir->operands[2]); case kRegArrayImm: // lir operands - 0: reg, 1: base, 2: index, 3: scale, 4: disp, 5: imm return ComputeSize(entry, lir->operands[0], lir->operands[2], lir->operands[1], lir->operands[4]); case kMovRegImm: // lir operands - 0: reg, 1: immediate case kMovRegQuadImm: return ((entry->skeleton.prefix1 != 0 || NeedsRex(lir->operands[0])) ? 1 : 0) + 1 + entry->skeleton.immediate_bytes; case kShiftRegImm: // lir operands - 0: reg, 1: immediate // Shift by immediate one has a shorter opcode. return ComputeSize(entry, lir->operands[0], NO_REG, NO_REG, 0) - (lir->operands[1] == 1 ? 1 : 0); case kShiftMemImm: // lir operands - 0: base, 1: disp, 2: immediate // Shift by immediate one has a shorter opcode. return ComputeSize(entry, NO_REG, NO_REG, lir->operands[0], lir->operands[1]) - (lir->operands[2] == 1 ? 1 : 0); case kShiftArrayImm: // lir operands - 0: base, 1: index, 2: scale, 3: disp 4: immediate // Shift by immediate one has a shorter opcode. return ComputeSize(entry, NO_REG, lir->operands[1], lir->operands[0], lir->operands[3]) - (lir->operands[4] == 1 ? 1 : 0); case kShiftRegCl: // lir operands - 0: reg, 1: cl DCHECK_EQ(rs_rCX.GetRegNum(), RegStorage::RegNum(lir->operands[1])); // Note: ShiftRegCl form passes reg as reg but encodes it using base. return ComputeSize(entry, lir->operands[0], NO_REG, NO_REG, 0); case kShiftMemCl: // lir operands - 0: base, 1: disp, 2: cl DCHECK_EQ(rs_rCX.GetRegNum(), RegStorage::RegNum(lir->operands[2])); return ComputeSize(entry, NO_REG, NO_REG, lir->operands[0], lir->operands[1]); case kShiftArrayCl: // lir operands - 0: base, 1: index, 2: scale, 3: disp, 4: cl DCHECK_EQ(rs_rCX.GetRegNum(), RegStorage::RegNum(lir->operands[4])); return ComputeSize(entry, lir->operands[4], lir->operands[1], lir->operands[0], lir->operands[3]); case kShiftRegRegCl: // lir operands - 0: reg1, 1: reg2, 2: cl DCHECK_EQ(rs_rCX.GetRegNum(), RegStorage::RegNum(lir->operands[2])); return ComputeSize(entry, lir->operands[0], NO_REG, lir->operands[1], 0); case kRegCond: // lir operands - 0: reg, 1: cond return ComputeSize(entry, NO_REG, NO_REG, lir->operands[0], 0); case kMemCond: // lir operands - 0: base, 1: disp, 2: cond return ComputeSize(entry, NO_REG, NO_REG, lir->operands[0], lir->operands[1]); case kArrayCond: // lir operands - 0: base, 1: index, 2: scale, 3: disp, 4: cond DCHECK_EQ(false, entry->skeleton.r8_form); return ComputeSize(entry, NO_REG, lir->operands[1], lir->operands[0], lir->operands[3]); case kRegRegCond: // lir operands - 0: reg1, 1: reg2, 2: cond DCHECK_EQ(false, entry->skeleton.r8_form); return ComputeSize(entry, lir->operands[0], NO_REG, lir->operands[1], 0); case kRegMemCond: // lir operands - 0: reg, 1: base, 2: disp, 3:cond DCHECK_EQ(false, entry->skeleton.r8_form); return ComputeSize(entry, lir->operands[0], NO_REG, lir->operands[1], lir->operands[2]); case kJcc: if (lir->opcode == kX86Jcc8) { return 2; // opcode + rel8 } else { DCHECK(lir->opcode == kX86Jcc32); return 6; // 2 byte opcode + rel32 } case kJmp: if (lir->opcode == kX86Jmp8 || lir->opcode == kX86Jecxz8) { return 2; // opcode + rel8 } else if (lir->opcode == kX86Jmp32) { return 5; // opcode + rel32 } else if (lir->opcode == kX86JmpT) { // Thread displacement size is always 32bit. return ComputeSize(entry, NO_REG, NO_REG, NO_REG, 0x12345678); } else { DCHECK(lir->opcode == kX86JmpR); if (NeedsRex(lir->operands[0])) { return 3; // REX.B + opcode + modrm } else { return 2; // opcode + modrm } } case kCall: switch (lir->opcode) { case kX86CallI: return 5; // opcode 0:disp case kX86CallR: return 2; // opcode modrm case kX86CallM: // lir operands - 0: base, 1: disp return ComputeSize(entry, NO_REG, NO_REG, lir->operands[0], lir->operands[1]); case kX86CallA: // lir operands - 0: base, 1: index, 2: scale, 3: disp return ComputeSize(entry, NO_REG, lir->operands[1], lir->operands[0], lir->operands[3]); case kX86CallT: // lir operands - 0: disp // Thread displacement size is always 32bit. return ComputeSize(entry, NO_REG, NO_REG, NO_REG, 0x12345678); default: break; } break; case kPcRel: if (entry->opcode == kX86PcRelLoadRA) { // lir operands - 0: reg, 1: base, 2: index, 3: scale, 4: table // Force the displacement size to 32bit, it will hold a computed offset later. return ComputeSize(entry, lir->operands[0], lir->operands[2], lir->operands[1], 0x12345678); } else { DCHECK_EQ(entry->opcode, kX86PcRelAdr); return 5; // opcode with reg + 4 byte immediate } case kUnimplemented: break; } UNIMPLEMENTED(FATAL) << "Unimplemented size encoding for: " << entry->name; return 0; } static uint8_t ModrmForDisp(int base, int disp) { // BP requires an explicit disp, so do not omit it in the 0 case if (disp == 0 && RegStorage::RegNum(base) != rs_rBP.GetRegNum()) { return 0; } else if (IS_SIMM8(disp)) { return 1; } else { return 2; } } void X86Mir2Lir::CheckValidByteRegister(const X86EncodingMap* entry, int32_t raw_reg) { if (kIsDebugBuild) { // Sanity check r8_form is correctly specified. if (entry->skeleton.r8_form) { CHECK(strchr(entry->name, '8') != nullptr) << entry->name; } else { if (entry->skeleton.immediate_bytes != 1) { // Ignore ...I8 instructions. if (!StartsWith(entry->name, "Movzx8") && !StartsWith(entry->name, "Movsx8") && !StartsWith(entry->name, "Movzx8q") && !StartsWith(entry->name, "Movsx8q")) { CHECK(strchr(entry->name, '8') == nullptr) << entry->name; } } } if (RegStorage::RegNum(raw_reg) >= 4) { // ah, bh, ch and dh are not valid registers in 32-bit. CHECK(cu_->target64 || !entry->skeleton.r8_form) << "Invalid register " << static_cast(RegStorage::RegNum(raw_reg)) << " for instruction " << entry->name << " in " << PrettyMethod(cu_->method_idx, *cu_->dex_file); } } } void X86Mir2Lir::EmitPrefix(const X86EncodingMap* entry, int32_t raw_reg_r, int32_t raw_reg_x, int32_t raw_reg_b) { // REX.WRXB // W - 64-bit operand // R - MODRM.reg // X - SIB.index // B - MODRM.rm/SIB.base bool w = (entry->skeleton.prefix1 == REX_W) || (entry->skeleton.prefix2 == REX_W); bool r = NeedsRex(raw_reg_r); bool x = NeedsRex(raw_reg_x); bool b = NeedsRex(raw_reg_b); bool r8_form = entry->skeleton.r8_form; bool modrm_is_reg_reg = ModrmIsRegReg(entry); uint8_t rex = 0; if (r8_form) { // Do we need an empty REX prefix to normalize byte register addressing? if (RegStorage::RegNum(raw_reg_r) >= 4 && !IsByteSecondOperand(entry)) { rex |= REX; // REX.0000 } else if (modrm_is_reg_reg && RegStorage::RegNum(raw_reg_b) >= 4) { rex |= REX; // REX.0000 } } if (w) { rex |= REX_W; // REX.W000 } if (r) { rex |= REX_R; // REX.0R00 } if (x) { rex |= REX_X; // REX.00X0 } if (b) { rex |= REX_B; // REX.000B } if (entry->skeleton.prefix1 != 0) { if (cu_->target64 && entry->skeleton.prefix1 == THREAD_PREFIX) { // 64 bit addresses by GS, not FS. code_buffer_.push_back(THREAD_PREFIX_GS); } else { if (entry->skeleton.prefix1 == REX_W || entry->skeleton.prefix1 == REX) { DCHECK(cu_->target64); rex |= entry->skeleton.prefix1; code_buffer_.push_back(rex); rex = 0; } else { code_buffer_.push_back(entry->skeleton.prefix1); } } if (entry->skeleton.prefix2 != 0) { if (entry->skeleton.prefix2 == REX_W || entry->skeleton.prefix1 == REX) { DCHECK(cu_->target64); rex |= entry->skeleton.prefix2; code_buffer_.push_back(rex); rex = 0; } else { code_buffer_.push_back(entry->skeleton.prefix2); } } } else { DCHECK_EQ(0, entry->skeleton.prefix2); } if (rex != 0) { DCHECK(cu_->target64); code_buffer_.push_back(rex); } } void X86Mir2Lir::EmitOpcode(const X86EncodingMap* entry) { code_buffer_.push_back(entry->skeleton.opcode); if (entry->skeleton.opcode == 0x0F) { code_buffer_.push_back(entry->skeleton.extra_opcode1); if (entry->skeleton.extra_opcode1 == 0x38 || entry->skeleton.extra_opcode1 == 0x3A) { code_buffer_.push_back(entry->skeleton.extra_opcode2); } else { DCHECK_EQ(0, entry->skeleton.extra_opcode2); } } else { DCHECK_EQ(0, entry->skeleton.extra_opcode1); DCHECK_EQ(0, entry->skeleton.extra_opcode2); } } void X86Mir2Lir::EmitPrefixAndOpcode(const X86EncodingMap* entry, int32_t raw_reg_r, int32_t raw_reg_x, int32_t raw_reg_b) { EmitPrefix(entry, raw_reg_r, raw_reg_x, raw_reg_b); EmitOpcode(entry); } void X86Mir2Lir::EmitDisp(uint8_t base, int32_t disp) { // BP requires an explicit disp, so do not omit it in the 0 case if (disp == 0 && RegStorage::RegNum(base) != rs_rBP.GetRegNum()) { return; } else if (IS_SIMM8(disp)) { code_buffer_.push_back(disp & 0xFF); } else { code_buffer_.push_back(disp & 0xFF); code_buffer_.push_back((disp >> 8) & 0xFF); code_buffer_.push_back((disp >> 16) & 0xFF); code_buffer_.push_back((disp >> 24) & 0xFF); } } void X86Mir2Lir::EmitModrmThread(uint8_t reg_or_opcode) { if (cu_->target64) { // Absolute adressing for GS access. uint8_t modrm = (0 << 6) | (reg_or_opcode << 3) | rs_rX86_SP_32.GetRegNum(); code_buffer_.push_back(modrm); uint8_t sib = (0/*TIMES_1*/ << 6) | (rs_rX86_SP_32.GetRegNum() << 3) | rs_rBP.GetRegNum(); code_buffer_.push_back(sib); } else { uint8_t modrm = (0 << 6) | (reg_or_opcode << 3) | rs_rBP.GetRegNum(); code_buffer_.push_back(modrm); } } void X86Mir2Lir::EmitModrmDisp(uint8_t reg_or_opcode, uint8_t base, int32_t disp) { DCHECK_LT(reg_or_opcode, 8); if (base == kRIPReg) { // x86_64 RIP handling: always 32 bit displacement. uint8_t modrm = (0x0 << 6) | (reg_or_opcode << 3) | 0x5; code_buffer_.push_back(modrm); code_buffer_.push_back(disp & 0xFF); code_buffer_.push_back((disp >> 8) & 0xFF); code_buffer_.push_back((disp >> 16) & 0xFF); code_buffer_.push_back((disp >> 24) & 0xFF); } else { DCHECK_LT(base, 8); uint8_t modrm = (ModrmForDisp(base, disp) << 6) | (reg_or_opcode << 3) | base; code_buffer_.push_back(modrm); if (base == rs_rX86_SP_32.GetRegNum()) { // Special SIB for SP base code_buffer_.push_back(0 << 6 | rs_rX86_SP_32.GetRegNum() << 3 | rs_rX86_SP_32.GetRegNum()); } EmitDisp(base, disp); } } void X86Mir2Lir::EmitModrmSibDisp(uint8_t reg_or_opcode, uint8_t base, uint8_t index, int scale, int32_t disp) { DCHECK_LT(RegStorage::RegNum(reg_or_opcode), 8); uint8_t modrm = (ModrmForDisp(base, disp) << 6) | RegStorage::RegNum(reg_or_opcode) << 3 | rs_rX86_SP_32.GetRegNum(); code_buffer_.push_back(modrm); DCHECK_LT(scale, 4); DCHECK_LT(RegStorage::RegNum(index), 8); DCHECK_LT(RegStorage::RegNum(base), 8); uint8_t sib = (scale << 6) | (RegStorage::RegNum(index) << 3) | RegStorage::RegNum(base); code_buffer_.push_back(sib); EmitDisp(base, disp); } void X86Mir2Lir::EmitImm(const X86EncodingMap* entry, int64_t imm) { switch (entry->skeleton.immediate_bytes) { case 1: DCHECK(IS_SIMM8(imm)); code_buffer_.push_back(imm & 0xFF); break; case 2: DCHECK(IS_SIMM16(imm)); code_buffer_.push_back(imm & 0xFF); code_buffer_.push_back((imm >> 8) & 0xFF); break; case 4: DCHECK(IS_SIMM32(imm)); code_buffer_.push_back(imm & 0xFF); code_buffer_.push_back((imm >> 8) & 0xFF); code_buffer_.push_back((imm >> 16) & 0xFF); code_buffer_.push_back((imm >> 24) & 0xFF); break; case 8: code_buffer_.push_back(imm & 0xFF); code_buffer_.push_back((imm >> 8) & 0xFF); code_buffer_.push_back((imm >> 16) & 0xFF); code_buffer_.push_back((imm >> 24) & 0xFF); code_buffer_.push_back((imm >> 32) & 0xFF); code_buffer_.push_back((imm >> 40) & 0xFF); code_buffer_.push_back((imm >> 48) & 0xFF); code_buffer_.push_back((imm >> 56) & 0xFF); break; default: LOG(FATAL) << "Unexpected immediate bytes (" << entry->skeleton.immediate_bytes << ") for instruction: " << entry->name; break; } } void X86Mir2Lir::EmitNullary(const X86EncodingMap* entry) { DCHECK_EQ(false, entry->skeleton.r8_form); EmitPrefixAndOpcode(entry, NO_REG, NO_REG, NO_REG); DCHECK_EQ(0, entry->skeleton.modrm_opcode); DCHECK_EQ(0, entry->skeleton.ax_opcode); DCHECK_EQ(0, entry->skeleton.immediate_bytes); } void X86Mir2Lir::EmitOpRegOpcode(const X86EncodingMap* entry, int32_t raw_reg) { DCHECK_EQ(false, entry->skeleton.r8_form); EmitPrefixAndOpcode(entry, NO_REG, NO_REG, raw_reg); // There's no 3-byte instruction with +rd DCHECK(entry->skeleton.opcode != 0x0F || (entry->skeleton.extra_opcode1 != 0x38 && entry->skeleton.extra_opcode1 != 0x3A)); DCHECK(!RegStorage::IsFloat(raw_reg)); uint8_t low_reg = LowRegisterBits(raw_reg); code_buffer_.back() += low_reg; DCHECK_EQ(0, entry->skeleton.ax_opcode); DCHECK_EQ(0, entry->skeleton.immediate_bytes); } void X86Mir2Lir::EmitOpReg(const X86EncodingMap* entry, int32_t raw_reg) { CheckValidByteRegister(entry, raw_reg); EmitPrefixAndOpcode(entry, NO_REG, NO_REG, raw_reg); uint8_t low_reg = LowRegisterBits(raw_reg); uint8_t modrm = (3 << 6) | (entry->skeleton.modrm_opcode << 3) | low_reg; code_buffer_.push_back(modrm); DCHECK_EQ(0, entry->skeleton.ax_opcode); DCHECK_EQ(0, entry->skeleton.immediate_bytes); } void X86Mir2Lir::EmitOpMem(const X86EncodingMap* entry, int32_t raw_base, int32_t disp) { DCHECK_EQ(false, entry->skeleton.r8_form); EmitPrefix(entry, NO_REG, NO_REG, raw_base); code_buffer_.push_back(entry->skeleton.opcode); DCHECK_NE(0x0F, entry->skeleton.opcode); DCHECK_EQ(0, entry->skeleton.extra_opcode1); DCHECK_EQ(0, entry->skeleton.extra_opcode2); uint8_t low_base = LowRegisterBits(raw_base); EmitModrmDisp(entry->skeleton.modrm_opcode, low_base, disp); DCHECK_EQ(0, entry->skeleton.ax_opcode); DCHECK_EQ(0, entry->skeleton.immediate_bytes); } void X86Mir2Lir::EmitOpArray(const X86EncodingMap* entry, int32_t raw_base, int32_t raw_index, int scale, int32_t disp) { DCHECK_EQ(false, entry->skeleton.r8_form); EmitPrefixAndOpcode(entry, NO_REG, raw_index, raw_base); uint8_t low_index = LowRegisterBits(raw_index); uint8_t low_base = LowRegisterBits(raw_base); EmitModrmSibDisp(entry->skeleton.modrm_opcode, low_base, low_index, scale, disp); DCHECK_EQ(0, entry->skeleton.ax_opcode); DCHECK_EQ(0, entry->skeleton.immediate_bytes); } void X86Mir2Lir::EmitMemReg(const X86EncodingMap* entry, int32_t raw_base, int32_t disp, int32_t raw_reg) { CheckValidByteRegister(entry, raw_reg); EmitPrefixAndOpcode(entry, raw_reg, NO_REG, raw_base); uint8_t low_reg = LowRegisterBits(raw_reg); uint8_t low_base = (raw_base == kRIPReg) ? raw_base : LowRegisterBits(raw_base); EmitModrmDisp(low_reg, low_base, disp); DCHECK_EQ(0, entry->skeleton.modrm_opcode); DCHECK_EQ(0, entry->skeleton.ax_opcode); DCHECK_EQ(0, entry->skeleton.immediate_bytes); } void X86Mir2Lir::EmitRegMem(const X86EncodingMap* entry, int32_t raw_reg, int32_t raw_base, int32_t disp) { // Opcode will flip operands. EmitMemReg(entry, raw_base, disp, raw_reg); } void X86Mir2Lir::EmitRegArray(const X86EncodingMap* entry, int32_t raw_reg, int32_t raw_base, int32_t raw_index, int scale, int32_t disp) { CheckValidByteRegister(entry, raw_reg); EmitPrefixAndOpcode(entry, raw_reg, raw_index, raw_base); uint8_t low_reg = LowRegisterBits(raw_reg); uint8_t low_index = LowRegisterBits(raw_index); uint8_t low_base = LowRegisterBits(raw_base); EmitModrmSibDisp(low_reg, low_base, low_index, scale, disp); DCHECK_EQ(0, entry->skeleton.modrm_opcode); DCHECK_EQ(0, entry->skeleton.ax_opcode); DCHECK_EQ(0, entry->skeleton.immediate_bytes); } void X86Mir2Lir::EmitArrayReg(const X86EncodingMap* entry, int32_t raw_base, int32_t raw_index, int scale, int32_t disp, int32_t raw_reg) { // Opcode will flip operands. EmitRegArray(entry, raw_reg, raw_base, raw_index, scale, disp); } void X86Mir2Lir::EmitMemImm(const X86EncodingMap* entry, int32_t raw_base, int32_t disp, int32_t imm) { DCHECK_EQ(false, entry->skeleton.r8_form); EmitPrefixAndOpcode(entry, NO_REG, NO_REG, raw_base); uint8_t low_base = LowRegisterBits(raw_base); EmitModrmDisp(entry->skeleton.modrm_opcode, low_base, disp); DCHECK_EQ(0, entry->skeleton.ax_opcode); EmitImm(entry, imm); } void X86Mir2Lir::EmitArrayImm(const X86EncodingMap* entry, int32_t raw_base, int32_t raw_index, int scale, int32_t disp, int32_t imm) { DCHECK_EQ(false, entry->skeleton.r8_form); EmitPrefixAndOpcode(entry, NO_REG, raw_index, raw_base); uint8_t low_index = LowRegisterBits(raw_index); uint8_t low_base = LowRegisterBits(raw_base); EmitModrmSibDisp(entry->skeleton.modrm_opcode, low_base, low_index, scale, disp); DCHECK_EQ(0, entry->skeleton.ax_opcode); EmitImm(entry, imm); } void X86Mir2Lir::EmitRegThread(const X86EncodingMap* entry, int32_t raw_reg, int32_t disp) { DCHECK_EQ(false, entry->skeleton.r8_form); DCHECK_NE(entry->skeleton.prefix1, 0); EmitPrefixAndOpcode(entry, raw_reg, NO_REG, NO_REG); uint8_t low_reg = LowRegisterBits(raw_reg); EmitModrmThread(low_reg); code_buffer_.push_back(disp & 0xFF); code_buffer_.push_back((disp >> 8) & 0xFF); code_buffer_.push_back((disp >> 16) & 0xFF); code_buffer_.push_back((disp >> 24) & 0xFF); DCHECK_EQ(0, entry->skeleton.modrm_opcode); DCHECK_EQ(0, entry->skeleton.ax_opcode); DCHECK_EQ(0, entry->skeleton.immediate_bytes); } void X86Mir2Lir::EmitRegReg(const X86EncodingMap* entry, int32_t raw_reg1, int32_t raw_reg2) { if (!IsByteSecondOperand(entry)) { CheckValidByteRegister(entry, raw_reg1); } CheckValidByteRegister(entry, raw_reg2); EmitPrefixAndOpcode(entry, raw_reg1, NO_REG, raw_reg2); uint8_t low_reg1 = LowRegisterBits(raw_reg1); uint8_t low_reg2 = LowRegisterBits(raw_reg2); uint8_t modrm = (3 << 6) | (low_reg1 << 3) | low_reg2; code_buffer_.push_back(modrm); DCHECK_EQ(0, entry->skeleton.modrm_opcode); DCHECK_EQ(0, entry->skeleton.ax_opcode); DCHECK_EQ(0, entry->skeleton.immediate_bytes); } void X86Mir2Lir::EmitRegRegImm(const X86EncodingMap* entry, int32_t raw_reg1, int32_t raw_reg2, int32_t imm) { DCHECK_EQ(false, entry->skeleton.r8_form); EmitPrefixAndOpcode(entry, raw_reg1, NO_REG, raw_reg2); uint8_t low_reg1 = LowRegisterBits(raw_reg1); uint8_t low_reg2 = LowRegisterBits(raw_reg2); uint8_t modrm = (3 << 6) | (low_reg1 << 3) | low_reg2; code_buffer_.push_back(modrm); DCHECK_EQ(0, entry->skeleton.modrm_opcode); DCHECK_EQ(0, entry->skeleton.ax_opcode); EmitImm(entry, imm); } void X86Mir2Lir::EmitRegMemImm(const X86EncodingMap* entry, int32_t raw_reg, int32_t raw_base, int disp, int32_t imm) { DCHECK(!RegStorage::IsFloat(raw_reg)); CheckValidByteRegister(entry, raw_reg); EmitPrefixAndOpcode(entry, raw_reg, NO_REG, raw_base); uint8_t low_reg = LowRegisterBits(raw_reg); uint8_t low_base = LowRegisterBits(raw_base); EmitModrmDisp(low_reg, low_base, disp); DCHECK_EQ(0, entry->skeleton.modrm_opcode); DCHECK_EQ(0, entry->skeleton.ax_opcode); EmitImm(entry, imm); } void X86Mir2Lir::EmitMemRegImm(const X86EncodingMap* entry, int32_t raw_base, int32_t disp, int32_t raw_reg, int32_t imm) { // Opcode will flip operands. EmitRegMemImm(entry, raw_reg, raw_base, disp, imm); } void X86Mir2Lir::EmitRegImm(const X86EncodingMap* entry, int32_t raw_reg, int32_t imm) { CheckValidByteRegister(entry, raw_reg); EmitPrefix(entry, NO_REG, NO_REG, raw_reg); if (RegStorage::RegNum(raw_reg) == rs_rAX.GetRegNum() && entry->skeleton.ax_opcode != 0) { code_buffer_.push_back(entry->skeleton.ax_opcode); } else { uint8_t low_reg = LowRegisterBits(raw_reg); EmitOpcode(entry); uint8_t modrm = (3 << 6) | (entry->skeleton.modrm_opcode << 3) | low_reg; code_buffer_.push_back(modrm); } EmitImm(entry, imm); } void X86Mir2Lir::EmitThreadImm(const X86EncodingMap* entry, int32_t disp, int32_t imm) { DCHECK_EQ(false, entry->skeleton.r8_form); EmitPrefixAndOpcode(entry, NO_REG, NO_REG, NO_REG); EmitModrmThread(entry->skeleton.modrm_opcode); code_buffer_.push_back(disp & 0xFF); code_buffer_.push_back((disp >> 8) & 0xFF); code_buffer_.push_back((disp >> 16) & 0xFF); code_buffer_.push_back((disp >> 24) & 0xFF); EmitImm(entry, imm); DCHECK_EQ(entry->skeleton.ax_opcode, 0); } void X86Mir2Lir::EmitMovRegImm(const X86EncodingMap* entry, int32_t raw_reg, int64_t imm) { DCHECK_EQ(false, entry->skeleton.r8_form); EmitPrefix(entry, NO_REG, NO_REG, raw_reg); uint8_t low_reg = LowRegisterBits(raw_reg); code_buffer_.push_back(0xB8 + low_reg); switch (entry->skeleton.immediate_bytes) { case 4: code_buffer_.push_back(imm & 0xFF); code_buffer_.push_back((imm >> 8) & 0xFF); code_buffer_.push_back((imm >> 16) & 0xFF); code_buffer_.push_back((imm >> 24) & 0xFF); break; case 8: code_buffer_.push_back(imm & 0xFF); code_buffer_.push_back((imm >> 8) & 0xFF); code_buffer_.push_back((imm >> 16) & 0xFF); code_buffer_.push_back((imm >> 24) & 0xFF); code_buffer_.push_back((imm >> 32) & 0xFF); code_buffer_.push_back((imm >> 40) & 0xFF); code_buffer_.push_back((imm >> 48) & 0xFF); code_buffer_.push_back((imm >> 56) & 0xFF); break; default: LOG(FATAL) << "Unsupported immediate size for EmitMovRegImm: " << static_cast(entry->skeleton.immediate_bytes); } } void X86Mir2Lir::EmitShiftRegImm(const X86EncodingMap* entry, int32_t raw_reg, int32_t imm) { CheckValidByteRegister(entry, raw_reg); EmitPrefix(entry, NO_REG, NO_REG, raw_reg); if (imm != 1) { code_buffer_.push_back(entry->skeleton.opcode); } else { // Shorter encoding for 1 bit shift code_buffer_.push_back(entry->skeleton.ax_opcode); } DCHECK_NE(0x0F, entry->skeleton.opcode); DCHECK_EQ(0, entry->skeleton.extra_opcode1); DCHECK_EQ(0, entry->skeleton.extra_opcode2); uint8_t low_reg = LowRegisterBits(raw_reg); uint8_t modrm = (3 << 6) | (entry->skeleton.modrm_opcode << 3) | low_reg; code_buffer_.push_back(modrm); if (imm != 1) { DCHECK_EQ(entry->skeleton.immediate_bytes, 1); DCHECK(IS_SIMM8(imm)); code_buffer_.push_back(imm & 0xFF); } } void X86Mir2Lir::EmitShiftRegCl(const X86EncodingMap* entry, int32_t raw_reg, int32_t raw_cl) { CheckValidByteRegister(entry, raw_reg); DCHECK_EQ(rs_rCX.GetRegNum(), RegStorage::RegNum(raw_cl)); EmitPrefix(entry, NO_REG, NO_REG, raw_reg); code_buffer_.push_back(entry->skeleton.opcode); DCHECK_NE(0x0F, entry->skeleton.opcode); DCHECK_EQ(0, entry->skeleton.extra_opcode1); DCHECK_EQ(0, entry->skeleton.extra_opcode2); uint8_t low_reg = LowRegisterBits(raw_reg); uint8_t modrm = (3 << 6) | (entry->skeleton.modrm_opcode << 3) | low_reg; code_buffer_.push_back(modrm); DCHECK_EQ(0, entry->skeleton.ax_opcode); DCHECK_EQ(0, entry->skeleton.immediate_bytes); } void X86Mir2Lir::EmitShiftMemCl(const X86EncodingMap* entry, int32_t raw_base, int32_t displacement, int32_t raw_cl) { DCHECK_EQ(false, entry->skeleton.r8_form); DCHECK_EQ(rs_rCX.GetRegNum(), RegStorage::RegNum(raw_cl)); EmitPrefix(entry, NO_REG, NO_REG, raw_base); code_buffer_.push_back(entry->skeleton.opcode); DCHECK_NE(0x0F, entry->skeleton.opcode); DCHECK_EQ(0, entry->skeleton.extra_opcode1); DCHECK_EQ(0, entry->skeleton.extra_opcode2); uint8_t low_base = LowRegisterBits(raw_base); EmitModrmDisp(entry->skeleton.modrm_opcode, low_base, displacement); DCHECK_EQ(0, entry->skeleton.ax_opcode); DCHECK_EQ(0, entry->skeleton.immediate_bytes); } void X86Mir2Lir::EmitShiftRegRegCl(const X86EncodingMap* entry, int32_t raw_reg1, int32_t raw_reg2, int32_t raw_cl) { DCHECK_EQ(false, entry->skeleton.r8_form); DCHECK_EQ(rs_rCX.GetRegNum(), RegStorage::RegNum(raw_cl)); EmitPrefixAndOpcode(entry, raw_reg1, NO_REG, raw_reg2); uint8_t low_reg1 = LowRegisterBits(raw_reg1); uint8_t low_reg2 = LowRegisterBits(raw_reg2); uint8_t modrm = (3 << 6) | (low_reg1 << 3) | low_reg2; code_buffer_.push_back(modrm); DCHECK_EQ(0, entry->skeleton.modrm_opcode); DCHECK_EQ(0, entry->skeleton.ax_opcode); DCHECK_EQ(0, entry->skeleton.immediate_bytes); } void X86Mir2Lir::EmitShiftMemImm(const X86EncodingMap* entry, int32_t raw_base, int32_t disp, int32_t imm) { DCHECK_EQ(false, entry->skeleton.r8_form); EmitPrefix(entry, NO_REG, NO_REG, raw_base); if (imm != 1) { code_buffer_.push_back(entry->skeleton.opcode); } else { // Shorter encoding for 1 bit shift code_buffer_.push_back(entry->skeleton.ax_opcode); } DCHECK_NE(0x0F, entry->skeleton.opcode); DCHECK_EQ(0, entry->skeleton.extra_opcode1); DCHECK_EQ(0, entry->skeleton.extra_opcode2); uint8_t low_base = LowRegisterBits(raw_base); EmitModrmDisp(entry->skeleton.modrm_opcode, low_base, disp); if (imm != 1) { DCHECK_EQ(entry->skeleton.immediate_bytes, 1); DCHECK(IS_SIMM8(imm)); code_buffer_.push_back(imm & 0xFF); } } void X86Mir2Lir::EmitRegCond(const X86EncodingMap* entry, int32_t raw_reg, int32_t cc) { CheckValidByteRegister(entry, raw_reg); EmitPrefix(entry, NO_REG, NO_REG, raw_reg); DCHECK_EQ(0, entry->skeleton.ax_opcode); DCHECK_EQ(0x0F, entry->skeleton.opcode); code_buffer_.push_back(0x0F); DCHECK_EQ(0x90, entry->skeleton.extra_opcode1); DCHECK_GE(cc, 0); DCHECK_LT(cc, 16); code_buffer_.push_back(0x90 | cc); DCHECK_EQ(0, entry->skeleton.extra_opcode2); uint8_t low_reg = LowRegisterBits(raw_reg); uint8_t modrm = (3 << 6) | (entry->skeleton.modrm_opcode << 3) | low_reg; code_buffer_.push_back(modrm); DCHECK_EQ(entry->skeleton.immediate_bytes, 0); } void X86Mir2Lir::EmitMemCond(const X86EncodingMap* entry, int32_t raw_base, int32_t disp, int32_t cc) { DCHECK_EQ(false, entry->skeleton.r8_form); if (entry->skeleton.prefix1 != 0) { code_buffer_.push_back(entry->skeleton.prefix1); if (entry->skeleton.prefix2 != 0) { code_buffer_.push_back(entry->skeleton.prefix2); } } else { DCHECK_EQ(0, entry->skeleton.prefix2); } DCHECK_EQ(0, entry->skeleton.ax_opcode); DCHECK_EQ(0x0F, entry->skeleton.opcode); code_buffer_.push_back(0x0F); DCHECK_EQ(0x90, entry->skeleton.extra_opcode1); DCHECK_GE(cc, 0); DCHECK_LT(cc, 16); code_buffer_.push_back(0x90 | cc); DCHECK_EQ(0, entry->skeleton.extra_opcode2); uint8_t low_base = LowRegisterBits(raw_base); EmitModrmDisp(entry->skeleton.modrm_opcode, low_base, disp); DCHECK_EQ(entry->skeleton.immediate_bytes, 0); } void X86Mir2Lir::EmitRegRegCond(const X86EncodingMap* entry, int32_t raw_reg1, int32_t raw_reg2, int32_t cc) { // Generate prefix and opcode without the condition. DCHECK_EQ(false, entry->skeleton.r8_form); EmitPrefixAndOpcode(entry, raw_reg1, NO_REG, raw_reg2); // Now add the condition. The last byte of opcode is the one that receives it. DCHECK_GE(cc, 0); DCHECK_LT(cc, 16); code_buffer_.back() += cc; // Not expecting to have to encode immediate or do anything special for ModR/M since there are // two registers. DCHECK_EQ(0, entry->skeleton.immediate_bytes); DCHECK_EQ(0, entry->skeleton.modrm_opcode); // For register to register encoding, the mod is 3. const uint8_t mod = (3 << 6); // Encode the ModR/M byte now. uint8_t low_reg1 = LowRegisterBits(raw_reg1); uint8_t low_reg2 = LowRegisterBits(raw_reg2); const uint8_t modrm = mod | (low_reg1 << 3) | low_reg2; code_buffer_.push_back(modrm); } void X86Mir2Lir::EmitRegMemCond(const X86EncodingMap* entry, int32_t raw_reg1, int32_t raw_base, int32_t disp, int32_t cc) { // Generate prefix and opcode without the condition. DCHECK_EQ(false, entry->skeleton.r8_form); EmitPrefixAndOpcode(entry, raw_reg1, NO_REG, raw_base); // Now add the condition. The last byte of opcode is the one that receives it. DCHECK_GE(cc, 0); DCHECK_LT(cc, 16); code_buffer_.back() += cc; // Not expecting to have to encode immediate or do anything special for ModR/M since there are // two registers. DCHECK_EQ(0, entry->skeleton.immediate_bytes); DCHECK_EQ(0, entry->skeleton.modrm_opcode); uint8_t low_reg1 = LowRegisterBits(raw_reg1); uint8_t low_base = LowRegisterBits(raw_base); EmitModrmDisp(low_reg1, low_base, disp); } void X86Mir2Lir::EmitJmp(const X86EncodingMap* entry, int32_t rel) { if (entry->opcode == kX86Jmp8) { DCHECK(IS_SIMM8(rel)); code_buffer_.push_back(0xEB); code_buffer_.push_back(rel & 0xFF); } else if (entry->opcode == kX86Jmp32) { code_buffer_.push_back(0xE9); code_buffer_.push_back(rel & 0xFF); code_buffer_.push_back((rel >> 8) & 0xFF); code_buffer_.push_back((rel >> 16) & 0xFF); code_buffer_.push_back((rel >> 24) & 0xFF); } else if (entry->opcode == kX86Jecxz8) { DCHECK(IS_SIMM8(rel)); code_buffer_.push_back(0xE3); code_buffer_.push_back(rel & 0xFF); } else { DCHECK(entry->opcode == kX86JmpR); DCHECK_EQ(false, entry->skeleton.r8_form); EmitPrefix(entry, NO_REG, NO_REG, rel); code_buffer_.push_back(entry->skeleton.opcode); uint8_t low_reg = LowRegisterBits(rel); uint8_t modrm = (3 << 6) | (entry->skeleton.modrm_opcode << 3) | low_reg; code_buffer_.push_back(modrm); } } void X86Mir2Lir::EmitJcc(const X86EncodingMap* entry, int32_t rel, int32_t cc) { DCHECK_GE(cc, 0); DCHECK_LT(cc, 16); if (entry->opcode == kX86Jcc8) { DCHECK(IS_SIMM8(rel)); code_buffer_.push_back(0x70 | cc); code_buffer_.push_back(rel & 0xFF); } else { DCHECK(entry->opcode == kX86Jcc32); code_buffer_.push_back(0x0F); code_buffer_.push_back(0x80 | cc); code_buffer_.push_back(rel & 0xFF); code_buffer_.push_back((rel >> 8) & 0xFF); code_buffer_.push_back((rel >> 16) & 0xFF); code_buffer_.push_back((rel >> 24) & 0xFF); } } void X86Mir2Lir::EmitCallMem(const X86EncodingMap* entry, int32_t raw_base, int32_t disp) { DCHECK_EQ(false, entry->skeleton.r8_form); EmitPrefixAndOpcode(entry, NO_REG, NO_REG, raw_base); uint8_t low_base = LowRegisterBits(raw_base); EmitModrmDisp(entry->skeleton.modrm_opcode, low_base, disp); DCHECK_EQ(0, entry->skeleton.ax_opcode); DCHECK_EQ(0, entry->skeleton.immediate_bytes); } void X86Mir2Lir::EmitCallImmediate(const X86EncodingMap* entry, int32_t disp) { DCHECK_EQ(false, entry->skeleton.r8_form); EmitPrefixAndOpcode(entry, NO_REG, NO_REG, NO_REG); DCHECK_EQ(4, entry->skeleton.immediate_bytes); code_buffer_.push_back(disp & 0xFF); code_buffer_.push_back((disp >> 8) & 0xFF); code_buffer_.push_back((disp >> 16) & 0xFF); code_buffer_.push_back((disp >> 24) & 0xFF); DCHECK_EQ(0, entry->skeleton.ax_opcode); } void X86Mir2Lir::EmitCallThread(const X86EncodingMap* entry, int32_t disp) { DCHECK_EQ(false, entry->skeleton.r8_form); DCHECK_NE(entry->skeleton.prefix1, 0); EmitPrefixAndOpcode(entry, NO_REG, NO_REG, NO_REG); EmitModrmThread(entry->skeleton.modrm_opcode); code_buffer_.push_back(disp & 0xFF); code_buffer_.push_back((disp >> 8) & 0xFF); code_buffer_.push_back((disp >> 16) & 0xFF); code_buffer_.push_back((disp >> 24) & 0xFF); DCHECK_EQ(0, entry->skeleton.ax_opcode); DCHECK_EQ(0, entry->skeleton.immediate_bytes); } void X86Mir2Lir::EmitPcRel(const X86EncodingMap* entry, int32_t raw_reg, int32_t raw_base_or_table, int32_t raw_index, int scale, int32_t table_or_disp) { int disp; if (entry->opcode == kX86PcRelLoadRA) { const SwitchTable* tab_rec = UnwrapPointer(table_or_disp); disp = tab_rec->offset - tab_rec->anchor->offset; } else { DCHECK(entry->opcode == kX86PcRelAdr); const EmbeddedData* tab_rec = UnwrapPointer(raw_base_or_table); disp = tab_rec->offset; } if (entry->opcode == kX86PcRelLoadRA) { DCHECK_EQ(false, entry->skeleton.r8_form); EmitPrefix(entry, raw_reg, raw_index, raw_base_or_table); code_buffer_.push_back(entry->skeleton.opcode); DCHECK_NE(0x0F, entry->skeleton.opcode); DCHECK_EQ(0, entry->skeleton.extra_opcode1); DCHECK_EQ(0, entry->skeleton.extra_opcode2); uint8_t low_reg = LowRegisterBits(raw_reg); uint8_t modrm = (2 << 6) | (low_reg << 3) | rs_rX86_SP_32.GetRegNum(); code_buffer_.push_back(modrm); DCHECK_LT(scale, 4); uint8_t low_base_or_table = LowRegisterBits(raw_base_or_table); uint8_t low_index = LowRegisterBits(raw_index); uint8_t sib = (scale << 6) | (low_index << 3) | low_base_or_table; code_buffer_.push_back(sib); DCHECK_EQ(0, entry->skeleton.immediate_bytes); } else { uint8_t low_reg = LowRegisterBits(raw_reg); code_buffer_.push_back(entry->skeleton.opcode + low_reg); } code_buffer_.push_back(disp & 0xFF); code_buffer_.push_back((disp >> 8) & 0xFF); code_buffer_.push_back((disp >> 16) & 0xFF); code_buffer_.push_back((disp >> 24) & 0xFF); DCHECK_EQ(0, entry->skeleton.modrm_opcode); DCHECK_EQ(0, entry->skeleton.ax_opcode); } void X86Mir2Lir::EmitUnimplemented(const X86EncodingMap* entry, LIR* lir) { UNIMPLEMENTED(WARNING) << "encoding kind for " << entry->name << " " << BuildInsnString(entry->fmt, lir, 0); for (size_t i = 0; i < GetInsnSize(lir); ++i) { code_buffer_.push_back(0xCC); // push breakpoint instruction - int 3 } } /* * Assemble the LIR into binary instruction format. Note that we may * discover that pc-relative displacements may not fit the selected * instruction. In those cases we will try to substitute a new code * sequence or request that the trace be shortened and retried. */ AssemblerStatus X86Mir2Lir::AssembleInstructions(LIR* first_lir_insn, CodeOffset start_addr) { UNUSED(start_addr); LIR *lir; AssemblerStatus res = kSuccess; // Assume success const bool kVerbosePcFixup = false; for (lir = first_lir_insn; lir != nullptr; lir = NEXT_LIR(lir)) { if (IsPseudoLirOp(lir->opcode)) { continue; } if (lir->flags.is_nop) { continue; } if (lir->flags.fixup != kFixupNone) { switch (lir->opcode) { case kX86Jcc8: { LIR *target_lir = lir->target; DCHECK(target_lir != nullptr); int delta = 0; CodeOffset pc; if (IS_SIMM8(lir->operands[0])) { pc = lir->offset + 2 /* opcode + rel8 */; } else { pc = lir->offset + 6 /* 2 byte opcode + rel32 */; } CodeOffset target = target_lir->offset; delta = target - pc; if (IS_SIMM8(delta) != IS_SIMM8(lir->operands[0])) { if (kVerbosePcFixup) { LOG(INFO) << "Retry for JCC growth at " << lir->offset << " delta: " << delta << " old delta: " << lir->operands[0]; } lir->opcode = kX86Jcc32; lir->flags.size = GetInsnSize(lir); DCHECK(lir->u.m.def_mask->Equals(kEncodeAll)); DCHECK(lir->u.m.use_mask->Equals(kEncodeAll)); res = kRetryAll; } if (kVerbosePcFixup) { LOG(INFO) << "Source:"; DumpLIRInsn(lir, 0); LOG(INFO) << "Target:"; DumpLIRInsn(target_lir, 0); LOG(INFO) << "Delta " << delta; } lir->operands[0] = delta; break; } case kX86Jcc32: { LIR *target_lir = lir->target; DCHECK(target_lir != nullptr); CodeOffset pc = lir->offset + 6 /* 2 byte opcode + rel32 */; CodeOffset target = target_lir->offset; int delta = target - pc; if (kVerbosePcFixup) { LOG(INFO) << "Source:"; DumpLIRInsn(lir, 0); LOG(INFO) << "Target:"; DumpLIRInsn(target_lir, 0); LOG(INFO) << "Delta " << delta; } lir->operands[0] = delta; break; } case kX86Jecxz8: { LIR *target_lir = lir->target; DCHECK(target_lir != nullptr); CodeOffset pc; pc = lir->offset + 2; // opcode + rel8 CodeOffset target = target_lir->offset; int delta = target - pc; lir->operands[0] = delta; DCHECK(IS_SIMM8(delta)); break; } case kX86Jmp8: { LIR *target_lir = lir->target; DCHECK(target_lir != nullptr); int delta = 0; CodeOffset pc; if (IS_SIMM8(lir->operands[0])) { pc = lir->offset + 2 /* opcode + rel8 */; } else { pc = lir->offset + 5 /* opcode + rel32 */; } CodeOffset target = target_lir->offset; delta = target - pc; if (!(cu_->disable_opt & (1 << kSafeOptimizations)) && delta == 0) { // Useless branch NopLIR(lir); if (kVerbosePcFixup) { LOG(INFO) << "Retry for useless branch at " << lir->offset; } res = kRetryAll; } else if (IS_SIMM8(delta) != IS_SIMM8(lir->operands[0])) { if (kVerbosePcFixup) { LOG(INFO) << "Retry for JMP growth at " << lir->offset; } lir->opcode = kX86Jmp32; lir->flags.size = GetInsnSize(lir); DCHECK(lir->u.m.def_mask->Equals(kEncodeAll)); DCHECK(lir->u.m.use_mask->Equals(kEncodeAll)); res = kRetryAll; } lir->operands[0] = delta; break; } case kX86Jmp32: { LIR *target_lir = lir->target; DCHECK(target_lir != nullptr); CodeOffset pc = lir->offset + 5 /* opcode + rel32 */; CodeOffset target = target_lir->offset; int delta = target - pc; lir->operands[0] = delta; break; } default: if (lir->flags.fixup == kFixupLoad) { LIR *target_lir = lir->target; DCHECK(target_lir != nullptr); CodeOffset target = target_lir->offset; // Handle 64 bit RIP addressing. if (lir->operands[1] == kRIPReg) { // Offset is relative to next instruction. lir->operands[2] = target - (lir->offset + lir->flags.size); } else { const LIR* anchor = UnwrapPointer(lir->operands[4]); lir->operands[2] = target - anchor->offset; int newSize = GetInsnSize(lir); if (newSize != lir->flags.size) { lir->flags.size = newSize; res = kRetryAll; } } } else if (lir->flags.fixup == kFixupSwitchTable) { DCHECK(cu_->target64); DCHECK_EQ(lir->opcode, kX86Lea64RM) << "Unknown instruction: " << X86Mir2Lir::EncodingMap[lir->opcode].name; DCHECK_EQ(lir->operands[1], static_cast(kRIPReg)); // Grab the target offset from the saved data. const EmbeddedData* tab_rec = UnwrapPointer(lir->operands[4]); CodeOffset target = tab_rec->offset; // Handle 64 bit RIP addressing. // Offset is relative to next instruction. lir->operands[2] = target - (lir->offset + lir->flags.size); } break; } } /* * If one of the pc-relative instructions expanded we'll have * to make another pass. Don't bother to fully assemble the * instruction. */ if (res != kSuccess) { continue; } CHECK_EQ(static_cast(lir->offset), code_buffer_.size()); const X86EncodingMap *entry = &X86Mir2Lir::EncodingMap[lir->opcode]; size_t starting_cbuf_size = code_buffer_.size(); switch (entry->kind) { case kData: // 4 bytes of data code_buffer_.push_back(lir->operands[0]); break; case kNullary: // 1 byte of opcode and possible prefixes. EmitNullary(entry); break; case kRegOpcode: // lir operands - 0: reg EmitOpRegOpcode(entry, lir->operands[0]); break; case kReg: // lir operands - 0: reg EmitOpReg(entry, lir->operands[0]); break; case kMem: // lir operands - 0: base, 1: disp EmitOpMem(entry, lir->operands[0], lir->operands[1]); break; case kArray: // lir operands - 0: base, 1: index, 2: scale, 3: disp EmitOpArray(entry, lir->operands[0], lir->operands[1], lir->operands[2], lir->operands[3]); break; case kMemReg: // lir operands - 0: base, 1: disp, 2: reg EmitMemReg(entry, lir->operands[0], lir->operands[1], lir->operands[2]); break; case kMemImm: // lir operands - 0: base, 1: disp, 2: immediate EmitMemImm(entry, lir->operands[0], lir->operands[1], lir->operands[2]); break; case kArrayImm: // lir operands - 0: base, 1: index, 2: disp, 3:scale, 4:immediate EmitArrayImm(entry, lir->operands[0], lir->operands[1], lir->operands[2], lir->operands[3], lir->operands[4]); break; case kArrayReg: // lir operands - 0: base, 1: index, 2: scale, 3: disp, 4: reg EmitArrayReg(entry, lir->operands[0], lir->operands[1], lir->operands[2], lir->operands[3], lir->operands[4]); break; case kRegMem: // lir operands - 0: reg, 1: base, 2: disp EmitRegMem(entry, lir->operands[0], lir->operands[1], lir->operands[2]); break; case kRegArray: // lir operands - 0: reg, 1: base, 2: index, 3: scale, 4: disp EmitRegArray(entry, lir->operands[0], lir->operands[1], lir->operands[2], lir->operands[3], lir->operands[4]); break; case kRegThread: // lir operands - 0: reg, 1: disp EmitRegThread(entry, lir->operands[0], lir->operands[1]); break; case kRegReg: // lir operands - 0: reg1, 1: reg2 EmitRegReg(entry, lir->operands[0], lir->operands[1]); break; case kRegRegStore: // lir operands - 0: reg2, 1: reg1 EmitRegReg(entry, lir->operands[1], lir->operands[0]); break; case kMemRegImm: // lir operands - 0: base, 1: disp, 2: reg 3: immediate EmitMemRegImm(entry, lir->operands[0], lir->operands[1], lir->operands[2], lir->operands[3]); break; case kRegRegImm: // lir operands - 0: reg1, 1: reg2, 2: imm EmitRegRegImm(entry, lir->operands[0], lir->operands[1], lir->operands[2]); break; case kRegRegImmStore: // lir operands - 0: reg2, 1: reg1, 2: imm EmitRegRegImm(entry, lir->operands[1], lir->operands[0], lir->operands[2]); break; case kRegMemImm: // lir operands - 0: reg, 1: base, 2: disp, 3: imm EmitRegMemImm(entry, lir->operands[0], lir->operands[1], lir->operands[2], lir->operands[3]); break; case kRegImm: // lir operands - 0: reg, 1: immediate EmitRegImm(entry, lir->operands[0], lir->operands[1]); break; case kThreadImm: // lir operands - 0: disp, 1: immediate EmitThreadImm(entry, lir->operands[0], lir->operands[1]); break; case kMovRegImm: // lir operands - 0: reg, 1: immediate EmitMovRegImm(entry, lir->operands[0], lir->operands[1]); break; case kMovRegQuadImm: { int64_t value = static_cast(static_cast(lir->operands[1]) << 32 | static_cast(lir->operands[2])); EmitMovRegImm(entry, lir->operands[0], value); } break; case kShiftRegImm: // lir operands - 0: reg, 1: immediate EmitShiftRegImm(entry, lir->operands[0], lir->operands[1]); break; case kShiftMemImm: // lir operands - 0: base, 1: disp, 2:immediate EmitShiftMemImm(entry, lir->operands[0], lir->operands[1], lir->operands[2]); break; case kShiftRegCl: // lir operands - 0: reg, 1: cl EmitShiftRegCl(entry, lir->operands[0], lir->operands[1]); break; case kShiftMemCl: // lir operands - 0: base, 1:displacement, 2: cl EmitShiftMemCl(entry, lir->operands[0], lir->operands[1], lir->operands[2]); break; case kShiftRegRegCl: // lir operands - 0: reg1, 1: reg2, 2: cl EmitShiftRegRegCl(entry, lir->operands[1], lir->operands[0], lir->operands[2]); break; case kRegCond: // lir operands - 0: reg, 1: condition EmitRegCond(entry, lir->operands[0], lir->operands[1]); break; case kMemCond: // lir operands - 0: base, 1: displacement, 2: condition EmitMemCond(entry, lir->operands[0], lir->operands[1], lir->operands[2]); break; case kRegRegCond: // lir operands - 0: reg, 1: reg, 2: condition EmitRegRegCond(entry, lir->operands[0], lir->operands[1], lir->operands[2]); break; case kRegMemCond: // lir operands - 0: reg, 1: reg, displacement, 3: condition EmitRegMemCond(entry, lir->operands[0], lir->operands[1], lir->operands[2], lir->operands[3]); break; case kJmp: // lir operands - 0: rel if (entry->opcode == kX86JmpT) { // This works since the instruction format for jmp and call is basically the same and // EmitCallThread loads opcode info. EmitCallThread(entry, lir->operands[0]); } else { EmitJmp(entry, lir->operands[0]); } break; case kJcc: // lir operands - 0: rel, 1: CC, target assigned EmitJcc(entry, lir->operands[0], lir->operands[1]); break; case kCall: switch (entry->opcode) { case kX86CallI: // lir operands - 0: disp EmitCallImmediate(entry, lir->operands[0]); break; case kX86CallM: // lir operands - 0: base, 1: disp EmitCallMem(entry, lir->operands[0], lir->operands[1]); break; case kX86CallT: // lir operands - 0: disp EmitCallThread(entry, lir->operands[0]); break; default: EmitUnimplemented(entry, lir); break; } break; case kPcRel: // lir operands - 0: reg, 1: base, 2: index, 3: scale, 4: table EmitPcRel(entry, lir->operands[0], lir->operands[1], lir->operands[2], lir->operands[3], lir->operands[4]); break; case kNop: // TODO: these instruction kinds are missing implementations. case kThreadReg: case kRegArrayImm: case kShiftArrayImm: case kShiftArrayCl: case kArrayCond: case kUnimplemented: EmitUnimplemented(entry, lir); break; } DCHECK_EQ(lir->flags.size, GetInsnSize(lir)); CHECK_EQ(lir->flags.size, code_buffer_.size() - starting_cbuf_size) << "Instruction size mismatch for entry: " << X86Mir2Lir::EncodingMap[lir->opcode].name; } return res; } // LIR offset assignment. // TODO: consolidate w/ Arm assembly mechanism. int X86Mir2Lir::AssignInsnOffsets() { LIR* lir; int offset = 0; for (lir = first_lir_insn_; lir != nullptr; lir = NEXT_LIR(lir)) { lir->offset = offset; if (LIKELY(!IsPseudoLirOp(lir->opcode))) { if (!lir->flags.is_nop) { offset += lir->flags.size; } } else if (UNLIKELY(lir->opcode == kPseudoPseudoAlign4)) { if (offset & 0x2) { offset += 2; lir->operands[0] = 1; } else { lir->operands[0] = 0; } } /* Pseudo opcodes don't consume space */ } return offset; } /* * Walk the compilation unit and assign offsets to instructions * and literals and compute the total size of the compiled unit. * TODO: consolidate w/ Arm assembly mechanism. */ void X86Mir2Lir::AssignOffsets() { int offset = AssignInsnOffsets(); if (const_vectors_ != nullptr) { // Vector literals must be 16-byte aligned. The header that is placed // in the code section causes misalignment so we take it into account. // Otherwise, we are sure that for x86 method is aligned to 16. DCHECK_EQ(GetInstructionSetAlignment(cu_->instruction_set), 16u); uint32_t bytes_to_fill = (0x10 - ((offset + sizeof(OatQuickMethodHeader)) & 0xF)) & 0xF; offset += bytes_to_fill; // Now assign each literal the right offset. for (LIR *p = const_vectors_; p != nullptr; p = p->next) { p->offset = offset; offset += 16; } } /* Const values have to be word aligned */ offset = RoundUp(offset, 4); /* Set up offsets for literals */ data_offset_ = offset; offset = AssignLiteralOffset(offset); offset = AssignSwitchTablesOffset(offset); offset = AssignFillArrayDataOffset(offset); total_size_ = offset; } /* * Go over each instruction in the list and calculate the offset from the top * before sending them off to the assembler. If out-of-range branch distance is * seen rearrange the instructions a bit to correct it. * TODO: consolidate w/ Arm assembly mechanism. */ void X86Mir2Lir::AssembleLIR() { cu_->NewTimingSplit("Assemble"); // We will remove the method address if we never ended up using it if (pc_rel_base_reg_.Valid() && !pc_rel_base_reg_used_) { if (kIsDebugBuild) { LOG(WARNING) << "PC-relative addressing base promoted but unused in " << PrettyMethod(cu_->method_idx, *cu_->dex_file); } setup_pc_rel_base_reg_->flags.is_nop = true; NEXT_LIR(setup_pc_rel_base_reg_)->flags.is_nop = true; } AssignOffsets(); int assembler_retries = 0; /* * Assemble here. Note that we generate code with optimistic assumptions * and if found now to work, we'll have to redo the sequence and retry. */ while (true) { AssemblerStatus res = AssembleInstructions(first_lir_insn_, 0); if (res == kSuccess) { break; } else { assembler_retries++; if (assembler_retries > MAX_ASSEMBLER_RETRIES) { CodegenDump(); LOG(FATAL) << "Assembler error - too many retries"; } // Redo offsets and try again AssignOffsets(); code_buffer_.clear(); } } // Install literals InstallLiteralPools(); // Install switch tables InstallSwitchTables(); // Install fill array data InstallFillArrayData(); // Create the mapping table and native offset to reference map. cu_->NewTimingSplit("PcMappingTable"); CreateMappingTables(); cu_->NewTimingSplit("GcMap"); CreateNativeGcMap(); } } // namespace art