// Copyright 2013 the V8 project authors. All rights reserved. // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. #ifndef V8_ARM64_ASSEMBLER_ARM64_H_ #define V8_ARM64_ASSEMBLER_ARM64_H_ #include #include #include #include #include "src/arm64/instructions-arm64.h" #include "src/assembler.h" #include "src/globals.h" #include "src/utils.h" namespace v8 { namespace internal { // ----------------------------------------------------------------------------- // Registers. // clang-format off #define GENERAL_REGISTER_CODE_LIST(R) \ R(0) R(1) R(2) R(3) R(4) R(5) R(6) R(7) \ R(8) R(9) R(10) R(11) R(12) R(13) R(14) R(15) \ R(16) R(17) R(18) R(19) R(20) R(21) R(22) R(23) \ R(24) R(25) R(26) R(27) R(28) R(29) R(30) R(31) #define GENERAL_REGISTERS(R) \ R(x0) R(x1) R(x2) R(x3) R(x4) R(x5) R(x6) R(x7) \ R(x8) R(x9) R(x10) R(x11) R(x12) R(x13) R(x14) R(x15) \ R(x16) R(x17) R(x18) R(x19) R(x20) R(x21) R(x22) R(x23) \ R(x24) R(x25) R(x26) R(x27) R(x28) R(x29) R(x30) R(x31) #define ALLOCATABLE_GENERAL_REGISTERS(R) \ R(x0) R(x1) R(x2) R(x3) R(x4) R(x5) R(x6) R(x7) \ R(x8) R(x9) R(x10) R(x11) R(x12) R(x13) R(x14) R(x15) \ R(x18) R(x19) R(x20) R(x21) R(x22) R(x23) R(x24) R(x27) #define DOUBLE_REGISTERS(R) \ R(d0) R(d1) R(d2) R(d3) R(d4) R(d5) R(d6) R(d7) \ R(d8) R(d9) R(d10) R(d11) R(d12) R(d13) R(d14) R(d15) \ R(d16) R(d17) R(d18) R(d19) R(d20) R(d21) R(d22) R(d23) \ R(d24) R(d25) R(d26) R(d27) R(d28) R(d29) R(d30) R(d31) #define ALLOCATABLE_DOUBLE_REGISTERS(R) \ R(d0) R(d1) R(d2) R(d3) R(d4) R(d5) R(d6) R(d7) \ R(d8) R(d9) R(d10) R(d11) R(d12) R(d13) R(d14) R(d16) \ R(d17) R(d18) R(d19) R(d20) R(d21) R(d22) R(d23) R(d24) \ R(d25) R(d26) R(d27) R(d28) // clang-format on static const int kRegListSizeInBits = sizeof(RegList) * kBitsPerByte; // Some CPURegister methods can return Register and FPRegister types, so we // need to declare them in advance. struct Register; struct FPRegister; struct CPURegister { enum Code { #define REGISTER_CODE(R) kCode_##R, GENERAL_REGISTERS(REGISTER_CODE) #undef REGISTER_CODE kAfterLast, kCode_no_reg = -1 }; enum RegisterType { // The kInvalid value is used to detect uninitialized static instances, // which are always zero-initialized before any constructors are called. kInvalid = 0, kRegister, kFPRegister, kNoRegister }; static CPURegister Create(int code, int size, RegisterType type) { CPURegister r = {code, size, type}; return r; } int code() const; RegisterType type() const; RegList Bit() const; int SizeInBits() const; int SizeInBytes() const; bool Is32Bits() const; bool Is64Bits() const; bool IsValid() const; bool IsValidOrNone() const; bool IsValidRegister() const; bool IsValidFPRegister() const; bool IsNone() const; bool Is(const CPURegister& other) const; bool Aliases(const CPURegister& other) const; bool IsZero() const; bool IsSP() const; bool IsRegister() const; bool IsFPRegister() const; Register X() const; Register W() const; FPRegister D() const; FPRegister S() const; bool IsSameSizeAndType(const CPURegister& other) const; // V8 compatibility. bool is(const CPURegister& other) const { return Is(other); } bool is_valid() const { return IsValid(); } int reg_code; int reg_size; RegisterType reg_type; }; struct Register : public CPURegister { static Register Create(int code, int size) { return Register(CPURegister::Create(code, size, CPURegister::kRegister)); } Register() { reg_code = 0; reg_size = 0; reg_type = CPURegister::kNoRegister; } explicit Register(const CPURegister& r) { reg_code = r.reg_code; reg_size = r.reg_size; reg_type = r.reg_type; DCHECK(IsValidOrNone()); } Register(const Register& r) { // NOLINT(runtime/explicit) reg_code = r.reg_code; reg_size = r.reg_size; reg_type = r.reg_type; DCHECK(IsValidOrNone()); } const char* ToString(); bool IsAllocatable() const; bool IsValid() const { DCHECK(IsRegister() || IsNone()); return IsValidRegister(); } static Register XRegFromCode(unsigned code); static Register WRegFromCode(unsigned code); // Start of V8 compatibility section --------------------- // These memebers are necessary for compilation. // A few of them may be unused for now. static const int kNumRegisters = kNumberOfRegisters; STATIC_ASSERT(kNumRegisters == Code::kAfterLast); static int NumRegisters() { return kNumRegisters; } // We allow crankshaft to use the following registers: // - x0 to x15 // - x18 to x24 // - x27 (also context) // // TODO(all): Register x25 is currently free and could be available for // crankshaft, but we don't use it as we might use it as a per function // literal pool pointer in the future. // // TODO(all): Consider storing cp in x25 to have only two ranges. // We split allocatable registers in three ranges called // - "low range" // - "high range" // - "context" static Register from_code(int code) { // Always return an X register. return Register::Create(code, kXRegSizeInBits); } // End of V8 compatibility section ----------------------- }; struct FPRegister : public CPURegister { enum Code { #define REGISTER_CODE(R) kCode_##R, DOUBLE_REGISTERS(REGISTER_CODE) #undef REGISTER_CODE kAfterLast, kCode_no_reg = -1 }; static FPRegister Create(int code, int size) { return FPRegister( CPURegister::Create(code, size, CPURegister::kFPRegister)); } FPRegister() { reg_code = 0; reg_size = 0; reg_type = CPURegister::kNoRegister; } explicit FPRegister(const CPURegister& r) { reg_code = r.reg_code; reg_size = r.reg_size; reg_type = r.reg_type; DCHECK(IsValidOrNone()); } FPRegister(const FPRegister& r) { // NOLINT(runtime/explicit) reg_code = r.reg_code; reg_size = r.reg_size; reg_type = r.reg_type; DCHECK(IsValidOrNone()); } const char* ToString(); bool IsAllocatable() const; bool IsValid() const { DCHECK(IsFPRegister() || IsNone()); return IsValidFPRegister(); } static FPRegister SRegFromCode(unsigned code); static FPRegister DRegFromCode(unsigned code); // Start of V8 compatibility section --------------------- static const int kMaxNumRegisters = kNumberOfFPRegisters; STATIC_ASSERT(kMaxNumRegisters == Code::kAfterLast); // Crankshaft can use all the FP registers except: // - d15 which is used to keep the 0 double value // - d30 which is used in crankshaft as a double scratch register // - d31 which is used in the MacroAssembler as a double scratch register static FPRegister from_code(int code) { // Always return a D register. return FPRegister::Create(code, kDRegSizeInBits); } // End of V8 compatibility section ----------------------- }; STATIC_ASSERT(sizeof(CPURegister) == sizeof(Register)); STATIC_ASSERT(sizeof(CPURegister) == sizeof(FPRegister)); #if defined(ARM64_DEFINE_REG_STATICS) #define INITIALIZE_REGISTER(register_class, name, code, size, type) \ const CPURegister init_##register_class##_##name = {code, size, type}; \ const register_class& name = *reinterpret_cast( \ &init_##register_class##_##name) #define ALIAS_REGISTER(register_class, alias, name) \ const register_class& alias = *reinterpret_cast( \ &init_##register_class##_##name) #else #define INITIALIZE_REGISTER(register_class, name, code, size, type) \ extern const register_class& name #define ALIAS_REGISTER(register_class, alias, name) \ extern const register_class& alias #endif // defined(ARM64_DEFINE_REG_STATICS) // No*Reg is used to indicate an unused argument, or an error case. Note that // these all compare equal (using the Is() method). The Register and FPRegister // variants are provided for convenience. INITIALIZE_REGISTER(Register, NoReg, 0, 0, CPURegister::kNoRegister); INITIALIZE_REGISTER(FPRegister, NoFPReg, 0, 0, CPURegister::kNoRegister); INITIALIZE_REGISTER(CPURegister, NoCPUReg, 0, 0, CPURegister::kNoRegister); // v8 compatibility. INITIALIZE_REGISTER(Register, no_reg, 0, 0, CPURegister::kNoRegister); #define DEFINE_REGISTERS(N) \ INITIALIZE_REGISTER(Register, w##N, N, \ kWRegSizeInBits, CPURegister::kRegister); \ INITIALIZE_REGISTER(Register, x##N, N, \ kXRegSizeInBits, CPURegister::kRegister); GENERAL_REGISTER_CODE_LIST(DEFINE_REGISTERS) #undef DEFINE_REGISTERS INITIALIZE_REGISTER(Register, wcsp, kSPRegInternalCode, kWRegSizeInBits, CPURegister::kRegister); INITIALIZE_REGISTER(Register, csp, kSPRegInternalCode, kXRegSizeInBits, CPURegister::kRegister); #define DEFINE_FPREGISTERS(N) \ INITIALIZE_REGISTER(FPRegister, s##N, N, \ kSRegSizeInBits, CPURegister::kFPRegister); \ INITIALIZE_REGISTER(FPRegister, d##N, N, \ kDRegSizeInBits, CPURegister::kFPRegister); GENERAL_REGISTER_CODE_LIST(DEFINE_FPREGISTERS) #undef DEFINE_FPREGISTERS #undef INITIALIZE_REGISTER // Registers aliases. ALIAS_REGISTER(Register, ip0, x16); ALIAS_REGISTER(Register, ip1, x17); ALIAS_REGISTER(Register, wip0, w16); ALIAS_REGISTER(Register, wip1, w17); // Root register. ALIAS_REGISTER(Register, root, x26); ALIAS_REGISTER(Register, rr, x26); // Context pointer register. ALIAS_REGISTER(Register, cp, x27); // We use a register as a JS stack pointer to overcome the restriction on the // architectural SP alignment. // We chose x28 because it is contiguous with the other specific purpose // registers. STATIC_ASSERT(kJSSPCode == 28); ALIAS_REGISTER(Register, jssp, x28); ALIAS_REGISTER(Register, wjssp, w28); ALIAS_REGISTER(Register, fp, x29); ALIAS_REGISTER(Register, lr, x30); ALIAS_REGISTER(Register, xzr, x31); ALIAS_REGISTER(Register, wzr, w31); // Keeps the 0 double value. ALIAS_REGISTER(FPRegister, fp_zero, d15); // Crankshaft double scratch register. ALIAS_REGISTER(FPRegister, crankshaft_fp_scratch, d29); // MacroAssembler double scratch registers. ALIAS_REGISTER(FPRegister, fp_scratch, d30); ALIAS_REGISTER(FPRegister, fp_scratch1, d30); ALIAS_REGISTER(FPRegister, fp_scratch2, d31); #undef ALIAS_REGISTER Register GetAllocatableRegisterThatIsNotOneOf(Register reg1, Register reg2 = NoReg, Register reg3 = NoReg, Register reg4 = NoReg); // AreAliased returns true if any of the named registers overlap. Arguments set // to NoReg are ignored. The system stack pointer may be specified. bool AreAliased(const CPURegister& reg1, const CPURegister& reg2, const CPURegister& reg3 = NoReg, const CPURegister& reg4 = NoReg, const CPURegister& reg5 = NoReg, const CPURegister& reg6 = NoReg, const CPURegister& reg7 = NoReg, const CPURegister& reg8 = NoReg); // AreSameSizeAndType returns true if all of the specified registers have the // same size, and are of the same type. The system stack pointer may be // specified. Arguments set to NoReg are ignored, as are any subsequent // arguments. At least one argument (reg1) must be valid (not NoCPUReg). bool AreSameSizeAndType(const CPURegister& reg1, const CPURegister& reg2, const CPURegister& reg3 = NoCPUReg, const CPURegister& reg4 = NoCPUReg, const CPURegister& reg5 = NoCPUReg, const CPURegister& reg6 = NoCPUReg, const CPURegister& reg7 = NoCPUReg, const CPURegister& reg8 = NoCPUReg); typedef FPRegister DoubleRegister; // ----------------------------------------------------------------------------- // Lists of registers. class CPURegList { public: explicit CPURegList(CPURegister reg1, CPURegister reg2 = NoCPUReg, CPURegister reg3 = NoCPUReg, CPURegister reg4 = NoCPUReg) : list_(reg1.Bit() | reg2.Bit() | reg3.Bit() | reg4.Bit()), size_(reg1.SizeInBits()), type_(reg1.type()) { DCHECK(AreSameSizeAndType(reg1, reg2, reg3, reg4)); DCHECK(IsValid()); } CPURegList(CPURegister::RegisterType type, int size, RegList list) : list_(list), size_(size), type_(type) { DCHECK(IsValid()); } CPURegList(CPURegister::RegisterType type, int size, int first_reg, int last_reg) : size_(size), type_(type) { DCHECK(((type == CPURegister::kRegister) && (last_reg < kNumberOfRegisters)) || ((type == CPURegister::kFPRegister) && (last_reg < kNumberOfFPRegisters))); DCHECK(last_reg >= first_reg); list_ = (1UL << (last_reg + 1)) - 1; list_ &= ~((1UL << first_reg) - 1); DCHECK(IsValid()); } CPURegister::RegisterType type() const { DCHECK(IsValid()); return type_; } RegList list() const { DCHECK(IsValid()); return list_; } inline void set_list(RegList new_list) { DCHECK(IsValid()); list_ = new_list; } // Combine another CPURegList into this one. Registers that already exist in // this list are left unchanged. The type and size of the registers in the // 'other' list must match those in this list. void Combine(const CPURegList& other); // Remove every register in the other CPURegList from this one. Registers that // do not exist in this list are ignored. The type of the registers in the // 'other' list must match those in this list. void Remove(const CPURegList& other); // Variants of Combine and Remove which take CPURegisters. void Combine(const CPURegister& other); void Remove(const CPURegister& other1, const CPURegister& other2 = NoCPUReg, const CPURegister& other3 = NoCPUReg, const CPURegister& other4 = NoCPUReg); // Variants of Combine and Remove which take a single register by its code; // the type and size of the register is inferred from this list. void Combine(int code); void Remove(int code); // Remove all callee-saved registers from the list. This can be useful when // preparing registers for an AAPCS64 function call, for example. void RemoveCalleeSaved(); CPURegister PopLowestIndex(); CPURegister PopHighestIndex(); // AAPCS64 callee-saved registers. static CPURegList GetCalleeSaved(int size = kXRegSizeInBits); static CPURegList GetCalleeSavedFP(int size = kDRegSizeInBits); // AAPCS64 caller-saved registers. Note that this includes lr. static CPURegList GetCallerSaved(int size = kXRegSizeInBits); static CPURegList GetCallerSavedFP(int size = kDRegSizeInBits); // Registers saved as safepoints. static CPURegList GetSafepointSavedRegisters(); bool IsEmpty() const { DCHECK(IsValid()); return list_ == 0; } bool IncludesAliasOf(const CPURegister& other1, const CPURegister& other2 = NoCPUReg, const CPURegister& other3 = NoCPUReg, const CPURegister& other4 = NoCPUReg) const { DCHECK(IsValid()); RegList list = 0; if (!other1.IsNone() && (other1.type() == type_)) list |= other1.Bit(); if (!other2.IsNone() && (other2.type() == type_)) list |= other2.Bit(); if (!other3.IsNone() && (other3.type() == type_)) list |= other3.Bit(); if (!other4.IsNone() && (other4.type() == type_)) list |= other4.Bit(); return (list_ & list) != 0; } int Count() const { DCHECK(IsValid()); return CountSetBits(list_, kRegListSizeInBits); } int RegisterSizeInBits() const { DCHECK(IsValid()); return size_; } int RegisterSizeInBytes() const { int size_in_bits = RegisterSizeInBits(); DCHECK((size_in_bits % kBitsPerByte) == 0); return size_in_bits / kBitsPerByte; } int TotalSizeInBytes() const { DCHECK(IsValid()); return RegisterSizeInBytes() * Count(); } private: RegList list_; int size_; CPURegister::RegisterType type_; bool IsValid() const { const RegList kValidRegisters = 0x8000000ffffffff; const RegList kValidFPRegisters = 0x0000000ffffffff; switch (type_) { case CPURegister::kRegister: return (list_ & kValidRegisters) == list_; case CPURegister::kFPRegister: return (list_ & kValidFPRegisters) == list_; case CPURegister::kNoRegister: return list_ == 0; default: UNREACHABLE(); return false; } } }; // AAPCS64 callee-saved registers. #define kCalleeSaved CPURegList::GetCalleeSaved() #define kCalleeSavedFP CPURegList::GetCalleeSavedFP() // AAPCS64 caller-saved registers. Note that this includes lr. #define kCallerSaved CPURegList::GetCallerSaved() #define kCallerSavedFP CPURegList::GetCallerSavedFP() // ----------------------------------------------------------------------------- // Immediates. class Immediate { public: template inline explicit Immediate(Handle handle); // This is allowed to be an implicit constructor because Immediate is // a wrapper class that doesn't normally perform any type conversion. template inline Immediate(T value); // NOLINT(runtime/explicit) template inline Immediate(T value, RelocInfo::Mode rmode); int64_t value() const { return value_; } RelocInfo::Mode rmode() const { return rmode_; } private: void InitializeHandle(Handle value); int64_t value_; RelocInfo::Mode rmode_; }; // ----------------------------------------------------------------------------- // Operands. const int kSmiShift = kSmiTagSize + kSmiShiftSize; const uint64_t kSmiShiftMask = (1UL << kSmiShift) - 1; // Represents an operand in a machine instruction. class Operand { // TODO(all): If necessary, study more in details which methods // TODO(all): should be inlined or not. public: // rm, { {#}} // where is one of {LSL, LSR, ASR, ROR}. // is uint6_t. // This is allowed to be an implicit constructor because Operand is // a wrapper class that doesn't normally perform any type conversion. inline Operand(Register reg, Shift shift = LSL, unsigned shift_amount = 0); // NOLINT(runtime/explicit) // rm, {#} // where is one of {UXTB, UXTH, UXTW, UXTX, SXTB, SXTH, SXTW, SXTX}. // is uint2_t. inline Operand(Register reg, Extend extend, unsigned shift_amount = 0); template inline explicit Operand(Handle handle); // Implicit constructor for all int types, ExternalReference, and Smi. template inline Operand(T t); // NOLINT(runtime/explicit) // Implicit constructor for int types. template inline Operand(T t, RelocInfo::Mode rmode); inline bool IsImmediate() const; inline bool IsShiftedRegister() const; inline bool IsExtendedRegister() const; inline bool IsZero() const; // This returns an LSL shift (<= 4) operand as an equivalent extend operand, // which helps in the encoding of instructions that use the stack pointer. inline Operand ToExtendedRegister() const; inline Immediate immediate() const; inline int64_t ImmediateValue() const; inline Register reg() const; inline Shift shift() const; inline Extend extend() const; inline unsigned shift_amount() const; // Relocation information. bool NeedsRelocation(const Assembler* assembler) const; // Helpers inline static Operand UntagSmi(Register smi); inline static Operand UntagSmiAndScale(Register smi, int scale); private: Immediate immediate_; Register reg_; Shift shift_; Extend extend_; unsigned shift_amount_; }; // MemOperand represents a memory operand in a load or store instruction. class MemOperand { public: inline MemOperand(); inline explicit MemOperand(Register base, int64_t offset = 0, AddrMode addrmode = Offset); inline explicit MemOperand(Register base, Register regoffset, Shift shift = LSL, unsigned shift_amount = 0); inline explicit MemOperand(Register base, Register regoffset, Extend extend, unsigned shift_amount = 0); inline explicit MemOperand(Register base, const Operand& offset, AddrMode addrmode = Offset); const Register& base() const { return base_; } const Register& regoffset() const { return regoffset_; } int64_t offset() const { return offset_; } AddrMode addrmode() const { return addrmode_; } Shift shift() const { return shift_; } Extend extend() const { return extend_; } unsigned shift_amount() const { return shift_amount_; } inline bool IsImmediateOffset() const; inline bool IsRegisterOffset() const; inline bool IsPreIndex() const; inline bool IsPostIndex() const; // For offset modes, return the offset as an Operand. This helper cannot // handle indexed modes. inline Operand OffsetAsOperand() const; enum PairResult { kNotPair, // Can't use a pair instruction. kPairAB, // Can use a pair instruction (operandA has lower address). kPairBA // Can use a pair instruction (operandB has lower address). }; // Check if two MemOperand are consistent for stp/ldp use. static PairResult AreConsistentForPair(const MemOperand& operandA, const MemOperand& operandB, int access_size_log2 = kXRegSizeLog2); private: Register base_; Register regoffset_; int64_t offset_; AddrMode addrmode_; Shift shift_; Extend extend_; unsigned shift_amount_; }; class ConstPool { public: explicit ConstPool(Assembler* assm) : assm_(assm), first_use_(-1), shared_entries_count(0) {} void RecordEntry(intptr_t data, RelocInfo::Mode mode); int EntryCount() const { return shared_entries_count + static_cast(unique_entries_.size()); } bool IsEmpty() const { return shared_entries_.empty() && unique_entries_.empty(); } // Distance in bytes between the current pc and the first instruction // using the pool. If there are no pending entries return kMaxInt. int DistanceToFirstUse(); // Offset after which instructions using the pool will be out of range. int MaxPcOffset(); // Maximum size the constant pool can be with current entries. It always // includes alignment padding and branch over. int WorstCaseSize(); // Size in bytes of the literal pool *if* it is emitted at the current // pc. The size will include the branch over the pool if it was requested. int SizeIfEmittedAtCurrentPc(bool require_jump); // Emit the literal pool at the current pc with a branch over the pool if // requested. void Emit(bool require_jump); // Discard any pending pool entries. void Clear(); private: bool CanBeShared(RelocInfo::Mode mode); void EmitMarker(); void EmitGuard(); void EmitEntries(); Assembler* assm_; // Keep track of the first instruction requiring a constant pool entry // since the previous constant pool was emitted. int first_use_; // values, pc offset(s) of entries which can be shared. std::multimap shared_entries_; // Number of distinct literal in shared entries. int shared_entries_count; // values, pc offset of entries which cannot be shared. std::vector > unique_entries_; }; // ----------------------------------------------------------------------------- // Assembler. class Assembler : public AssemblerBase { public: // Create an assembler. Instructions and relocation information are emitted // into a buffer, with the instructions starting from the beginning and the // relocation information starting from the end of the buffer. See CodeDesc // for a detailed comment on the layout (globals.h). // // If the provided buffer is NULL, the assembler allocates and grows its own // buffer, and buffer_size determines the initial buffer size. The buffer is // owned by the assembler and deallocated upon destruction of the assembler. // // If the provided buffer is not NULL, the assembler uses the provided buffer // for code generation and assumes its size to be buffer_size. If the buffer // is too small, a fatal error occurs. No deallocation of the buffer is done // upon destruction of the assembler. Assembler(Isolate* arg_isolate, void* buffer, int buffer_size); virtual ~Assembler(); virtual void AbortedCodeGeneration() { constpool_.Clear(); } // System functions --------------------------------------------------------- // Start generating code from the beginning of the buffer, discarding any code // and data that has already been emitted into the buffer. // // In order to avoid any accidental transfer of state, Reset DCHECKs that the // constant pool is not blocked. void Reset(); // GetCode emits any pending (non-emitted) code and fills the descriptor // desc. GetCode() is idempotent; it returns the same result if no other // Assembler functions are invoked in between GetCode() calls. // // The descriptor (desc) can be NULL. In that case, the code is finalized as // usual, but the descriptor is not populated. void GetCode(CodeDesc* desc); // Insert the smallest number of nop instructions // possible to align the pc offset to a multiple // of m. m must be a power of 2 (>= 4). void Align(int m); // Insert the smallest number of zero bytes possible to align the pc offset // to a mulitple of m. m must be a power of 2 (>= 2). void DataAlign(int m); inline void Unreachable(); // Label -------------------------------------------------------------------- // Bind a label to the current pc. Note that labels can only be bound once, // and if labels are linked to other instructions, they _must_ be bound // before they go out of scope. void bind(Label* label); // RelocInfo and pools ------------------------------------------------------ // Record relocation information for current pc_. void RecordRelocInfo(RelocInfo::Mode rmode, intptr_t data = 0); // Return the address in the constant pool of the code target address used by // the branch/call instruction at pc. inline static Address target_pointer_address_at(Address pc); // Read/Modify the code target address in the branch/call instruction at pc. inline static Address target_address_at(Address pc, Address constant_pool); inline static void set_target_address_at( Isolate* isolate, Address pc, Address constant_pool, Address target, ICacheFlushMode icache_flush_mode = FLUSH_ICACHE_IF_NEEDED); static inline Address target_address_at(Address pc, Code* code); static inline void set_target_address_at( Isolate* isolate, Address pc, Code* code, Address target, ICacheFlushMode icache_flush_mode = FLUSH_ICACHE_IF_NEEDED); // Return the code target address at a call site from the return address of // that call in the instruction stream. inline static Address target_address_from_return_address(Address pc); // Given the address of the beginning of a call, return the address in the // instruction stream that call will return from. inline static Address return_address_from_call_start(Address pc); // This sets the branch destination (which is in the constant pool on ARM). // This is for calls and branches within generated code. inline static void deserialization_set_special_target_at( Isolate* isolate, Address constant_pool_entry, Code* code, Address target); // This sets the internal reference at the pc. inline static void deserialization_set_target_internal_reference_at( Isolate* isolate, Address pc, Address target, RelocInfo::Mode mode = RelocInfo::INTERNAL_REFERENCE); // All addresses in the constant pool are the same size as pointers. static const int kSpecialTargetSize = kPointerSize; // The sizes of the call sequences emitted by MacroAssembler::Call. // Wherever possible, use MacroAssembler::CallSize instead of these constants, // as it will choose the correct value for a given relocation mode. // // Without relocation: // movz temp, #(target & 0x000000000000ffff) // movk temp, #(target & 0x00000000ffff0000) // movk temp, #(target & 0x0000ffff00000000) // blr temp // // With relocation: // ldr temp, =target // blr temp static const int kCallSizeWithoutRelocation = 4 * kInstructionSize; static const int kCallSizeWithRelocation = 2 * kInstructionSize; // Size of the generated code in bytes uint64_t SizeOfGeneratedCode() const { DCHECK((pc_ >= buffer_) && (pc_ < (buffer_ + buffer_size_))); return pc_ - buffer_; } // Return the code size generated from label to the current position. uint64_t SizeOfCodeGeneratedSince(const Label* label) { DCHECK(label->is_bound()); DCHECK(pc_offset() >= label->pos()); DCHECK(pc_offset() < buffer_size_); return pc_offset() - label->pos(); } // Check the size of the code generated since the given label. This function // is used primarily to work around comparisons between signed and unsigned // quantities, since V8 uses both. // TODO(jbramley): Work out what sign to use for these things and if possible, // change things to be consistent. void AssertSizeOfCodeGeneratedSince(const Label* label, ptrdiff_t size) { DCHECK(size >= 0); DCHECK(static_cast(size) == SizeOfCodeGeneratedSince(label)); } // Return the number of instructions generated from label to the // current position. uint64_t InstructionsGeneratedSince(const Label* label) { return SizeOfCodeGeneratedSince(label) / kInstructionSize; } static const int kPatchDebugBreakSlotAddressOffset = 0; // Number of instructions necessary to be able to later patch it to a call. static const int kDebugBreakSlotInstructions = 5; static const int kDebugBreakSlotLength = kDebugBreakSlotInstructions * kInstructionSize; // Prevent contant pool emission until EndBlockConstPool is called. // Call to this function can be nested but must be followed by an equal // number of call to EndBlockConstpool. void StartBlockConstPool(); // Resume constant pool emission. Need to be called as many time as // StartBlockConstPool to have an effect. void EndBlockConstPool(); bool is_const_pool_blocked() const; static bool IsConstantPoolAt(Instruction* instr); static int ConstantPoolSizeAt(Instruction* instr); // See Assembler::CheckConstPool for more info. void EmitPoolGuard(); // Prevent veneer pool emission until EndBlockVeneerPool is called. // Call to this function can be nested but must be followed by an equal // number of call to EndBlockConstpool. void StartBlockVeneerPool(); // Resume constant pool emission. Need to be called as many time as // StartBlockVeneerPool to have an effect. void EndBlockVeneerPool(); bool is_veneer_pool_blocked() const { return veneer_pool_blocked_nesting_ > 0; } // Block/resume emission of constant pools and veneer pools. void StartBlockPools() { StartBlockConstPool(); StartBlockVeneerPool(); } void EndBlockPools() { EndBlockConstPool(); EndBlockVeneerPool(); } // Debugging ---------------------------------------------------------------- PositionsRecorder* positions_recorder() { return &positions_recorder_; } void RecordComment(const char* msg); // Record a deoptimization reason that can be used by a log or cpu profiler. // Use --trace-deopt to enable. void RecordDeoptReason(const int reason, const SourcePosition position); int buffer_space() const; // Mark generator continuation. void RecordGeneratorContinuation(); // Mark address of a debug break slot. void RecordDebugBreakSlot(RelocInfo::Mode mode); // Record the emission of a constant pool. // // The emission of constant and veneer pools depends on the size of the code // generated and the number of RelocInfo recorded. // The Debug mechanism needs to map code offsets between two versions of a // function, compiled with and without debugger support (see for example // Debug::PrepareForBreakPoints()). // Compiling functions with debugger support generates additional code // (DebugCodegen::GenerateSlot()). This may affect the emission of the pools // and cause the version of the code with debugger support to have pools // generated in different places. // Recording the position and size of emitted pools allows to correctly // compute the offset mappings between the different versions of a function in // all situations. // // The parameter indicates the size of the pool (in bytes), including // the marker and branch over the data. void RecordConstPool(int size); // Instruction set functions ------------------------------------------------ // Branch / Jump instructions. // For branches offsets are scaled, i.e. they in instrcutions not in bytes. // Branch to register. void br(const Register& xn); // Branch-link to register. void blr(const Register& xn); // Branch to register with return hint. void ret(const Register& xn = lr); // Unconditional branch to label. void b(Label* label); // Conditional branch to label. void b(Label* label, Condition cond); // Unconditional branch to PC offset. void b(int imm26); // Conditional branch to PC offset. void b(int imm19, Condition cond); // Branch-link to label / pc offset. void bl(Label* label); void bl(int imm26); // Compare and branch to label / pc offset if zero. void cbz(const Register& rt, Label* label); void cbz(const Register& rt, int imm19); // Compare and branch to label / pc offset if not zero. void cbnz(const Register& rt, Label* label); void cbnz(const Register& rt, int imm19); // Test bit and branch to label / pc offset if zero. void tbz(const Register& rt, unsigned bit_pos, Label* label); void tbz(const Register& rt, unsigned bit_pos, int imm14); // Test bit and branch to label / pc offset if not zero. void tbnz(const Register& rt, unsigned bit_pos, Label* label); void tbnz(const Register& rt, unsigned bit_pos, int imm14); // Address calculation instructions. // Calculate a PC-relative address. Unlike for branches the offset in adr is // unscaled (i.e. the result can be unaligned). void adr(const Register& rd, Label* label); void adr(const Register& rd, int imm21); // Data Processing instructions. // Add. void add(const Register& rd, const Register& rn, const Operand& operand); // Add and update status flags. void adds(const Register& rd, const Register& rn, const Operand& operand); // Compare negative. void cmn(const Register& rn, const Operand& operand); // Subtract. void sub(const Register& rd, const Register& rn, const Operand& operand); // Subtract and update status flags. void subs(const Register& rd, const Register& rn, const Operand& operand); // Compare. void cmp(const Register& rn, const Operand& operand); // Negate. void neg(const Register& rd, const Operand& operand); // Negate and update status flags. void negs(const Register& rd, const Operand& operand); // Add with carry bit. void adc(const Register& rd, const Register& rn, const Operand& operand); // Add with carry bit and update status flags. void adcs(const Register& rd, const Register& rn, const Operand& operand); // Subtract with carry bit. void sbc(const Register& rd, const Register& rn, const Operand& operand); // Subtract with carry bit and update status flags. void sbcs(const Register& rd, const Register& rn, const Operand& operand); // Negate with carry bit. void ngc(const Register& rd, const Operand& operand); // Negate with carry bit and update status flags. void ngcs(const Register& rd, const Operand& operand); // Logical instructions. // Bitwise and (A & B). void and_(const Register& rd, const Register& rn, const Operand& operand); // Bitwise and (A & B) and update status flags. void ands(const Register& rd, const Register& rn, const Operand& operand); // Bit test, and set flags. void tst(const Register& rn, const Operand& operand); // Bit clear (A & ~B). void bic(const Register& rd, const Register& rn, const Operand& operand); // Bit clear (A & ~B) and update status flags. void bics(const Register& rd, const Register& rn, const Operand& operand); // Bitwise or (A | B). void orr(const Register& rd, const Register& rn, const Operand& operand); // Bitwise nor (A | ~B). void orn(const Register& rd, const Register& rn, const Operand& operand); // Bitwise eor/xor (A ^ B). void eor(const Register& rd, const Register& rn, const Operand& operand); // Bitwise enor/xnor (A ^ ~B). void eon(const Register& rd, const Register& rn, const Operand& operand); // Logical shift left variable. void lslv(const Register& rd, const Register& rn, const Register& rm); // Logical shift right variable. void lsrv(const Register& rd, const Register& rn, const Register& rm); // Arithmetic shift right variable. void asrv(const Register& rd, const Register& rn, const Register& rm); // Rotate right variable. void rorv(const Register& rd, const Register& rn, const Register& rm); // Bitfield instructions. // Bitfield move. void bfm(const Register& rd, const Register& rn, int immr, int imms); // Signed bitfield move. void sbfm(const Register& rd, const Register& rn, int immr, int imms); // Unsigned bitfield move. void ubfm(const Register& rd, const Register& rn, int immr, int imms); // Bfm aliases. // Bitfield insert. void bfi(const Register& rd, const Register& rn, int lsb, int width) { DCHECK(width >= 1); DCHECK(lsb + width <= rn.SizeInBits()); bfm(rd, rn, (rd.SizeInBits() - lsb) & (rd.SizeInBits() - 1), width - 1); } // Bitfield extract and insert low. void bfxil(const Register& rd, const Register& rn, int lsb, int width) { DCHECK(width >= 1); DCHECK(lsb + width <= rn.SizeInBits()); bfm(rd, rn, lsb, lsb + width - 1); } // Sbfm aliases. // Arithmetic shift right. void asr(const Register& rd, const Register& rn, int shift) { DCHECK(shift < rd.SizeInBits()); sbfm(rd, rn, shift, rd.SizeInBits() - 1); } // Signed bitfield insert in zero. void sbfiz(const Register& rd, const Register& rn, int lsb, int width) { DCHECK(width >= 1); DCHECK(lsb + width <= rn.SizeInBits()); sbfm(rd, rn, (rd.SizeInBits() - lsb) & (rd.SizeInBits() - 1), width - 1); } // Signed bitfield extract. void sbfx(const Register& rd, const Register& rn, int lsb, int width) { DCHECK(width >= 1); DCHECK(lsb + width <= rn.SizeInBits()); sbfm(rd, rn, lsb, lsb + width - 1); } // Signed extend byte. void sxtb(const Register& rd, const Register& rn) { sbfm(rd, rn, 0, 7); } // Signed extend halfword. void sxth(const Register& rd, const Register& rn) { sbfm(rd, rn, 0, 15); } // Signed extend word. void sxtw(const Register& rd, const Register& rn) { sbfm(rd, rn, 0, 31); } // Ubfm aliases. // Logical shift left. void lsl(const Register& rd, const Register& rn, int shift) { int reg_size = rd.SizeInBits(); DCHECK(shift < reg_size); ubfm(rd, rn, (reg_size - shift) % reg_size, reg_size - shift - 1); } // Logical shift right. void lsr(const Register& rd, const Register& rn, int shift) { DCHECK(shift < rd.SizeInBits()); ubfm(rd, rn, shift, rd.SizeInBits() - 1); } // Unsigned bitfield insert in zero. void ubfiz(const Register& rd, const Register& rn, int lsb, int width) { DCHECK(width >= 1); DCHECK(lsb + width <= rn.SizeInBits()); ubfm(rd, rn, (rd.SizeInBits() - lsb) & (rd.SizeInBits() - 1), width - 1); } // Unsigned bitfield extract. void ubfx(const Register& rd, const Register& rn, int lsb, int width) { DCHECK(width >= 1); DCHECK(lsb + width <= rn.SizeInBits()); ubfm(rd, rn, lsb, lsb + width - 1); } // Unsigned extend byte. void uxtb(const Register& rd, const Register& rn) { ubfm(rd, rn, 0, 7); } // Unsigned extend halfword. void uxth(const Register& rd, const Register& rn) { ubfm(rd, rn, 0, 15); } // Unsigned extend word. void uxtw(const Register& rd, const Register& rn) { ubfm(rd, rn, 0, 31); } // Extract. void extr(const Register& rd, const Register& rn, const Register& rm, int lsb); // Conditional select: rd = cond ? rn : rm. void csel(const Register& rd, const Register& rn, const Register& rm, Condition cond); // Conditional select increment: rd = cond ? rn : rm + 1. void csinc(const Register& rd, const Register& rn, const Register& rm, Condition cond); // Conditional select inversion: rd = cond ? rn : ~rm. void csinv(const Register& rd, const Register& rn, const Register& rm, Condition cond); // Conditional select negation: rd = cond ? rn : -rm. void csneg(const Register& rd, const Register& rn, const Register& rm, Condition cond); // Conditional set: rd = cond ? 1 : 0. void cset(const Register& rd, Condition cond); // Conditional set minus: rd = cond ? -1 : 0. void csetm(const Register& rd, Condition cond); // Conditional increment: rd = cond ? rn + 1 : rn. void cinc(const Register& rd, const Register& rn, Condition cond); // Conditional invert: rd = cond ? ~rn : rn. void cinv(const Register& rd, const Register& rn, Condition cond); // Conditional negate: rd = cond ? -rn : rn. void cneg(const Register& rd, const Register& rn, Condition cond); // Extr aliases. void ror(const Register& rd, const Register& rs, unsigned shift) { extr(rd, rs, rs, shift); } // Conditional comparison. // Conditional compare negative. void ccmn(const Register& rn, const Operand& operand, StatusFlags nzcv, Condition cond); // Conditional compare. void ccmp(const Register& rn, const Operand& operand, StatusFlags nzcv, Condition cond); // Multiplication. // 32 x 32 -> 32-bit and 64 x 64 -> 64-bit multiply. void mul(const Register& rd, const Register& rn, const Register& rm); // 32 + 32 x 32 -> 32-bit and 64 + 64 x 64 -> 64-bit multiply accumulate. void madd(const Register& rd, const Register& rn, const Register& rm, const Register& ra); // -(32 x 32) -> 32-bit and -(64 x 64) -> 64-bit multiply. void mneg(const Register& rd, const Register& rn, const Register& rm); // 32 - 32 x 32 -> 32-bit and 64 - 64 x 64 -> 64-bit multiply subtract. void msub(const Register& rd, const Register& rn, const Register& rm, const Register& ra); // 32 x 32 -> 64-bit multiply. void smull(const Register& rd, const Register& rn, const Register& rm); // Xd = bits<127:64> of Xn * Xm. void smulh(const Register& rd, const Register& rn, const Register& rm); // Signed 32 x 32 -> 64-bit multiply and accumulate. void smaddl(const Register& rd, const Register& rn, const Register& rm, const Register& ra); // Unsigned 32 x 32 -> 64-bit multiply and accumulate. void umaddl(const Register& rd, const Register& rn, const Register& rm, const Register& ra); // Signed 32 x 32 -> 64-bit multiply and subtract. void smsubl(const Register& rd, const Register& rn, const Register& rm, const Register& ra); // Unsigned 32 x 32 -> 64-bit multiply and subtract. void umsubl(const Register& rd, const Register& rn, const Register& rm, const Register& ra); // Signed integer divide. void sdiv(const Register& rd, const Register& rn, const Register& rm); // Unsigned integer divide. void udiv(const Register& rd, const Register& rn, const Register& rm); // Bit count, bit reverse and endian reverse. void rbit(const Register& rd, const Register& rn); void rev16(const Register& rd, const Register& rn); void rev32(const Register& rd, const Register& rn); void rev(const Register& rd, const Register& rn); void clz(const Register& rd, const Register& rn); void cls(const Register& rd, const Register& rn); // Memory instructions. // Load integer or FP register. void ldr(const CPURegister& rt, const MemOperand& src); // Store integer or FP register. void str(const CPURegister& rt, const MemOperand& dst); // Load word with sign extension. void ldrsw(const Register& rt, const MemOperand& src); // Load byte. void ldrb(const Register& rt, const MemOperand& src); // Store byte. void strb(const Register& rt, const MemOperand& dst); // Load byte with sign extension. void ldrsb(const Register& rt, const MemOperand& src); // Load half-word. void ldrh(const Register& rt, const MemOperand& src); // Store half-word. void strh(const Register& rt, const MemOperand& dst); // Load half-word with sign extension. void ldrsh(const Register& rt, const MemOperand& src); // Load integer or FP register pair. void ldp(const CPURegister& rt, const CPURegister& rt2, const MemOperand& src); // Store integer or FP register pair. void stp(const CPURegister& rt, const CPURegister& rt2, const MemOperand& dst); // Load word pair with sign extension. void ldpsw(const Register& rt, const Register& rt2, const MemOperand& src); // Load literal to register from a pc relative address. void ldr_pcrel(const CPURegister& rt, int imm19); // Load literal to register. void ldr(const CPURegister& rt, const Immediate& imm); // Move instructions. The default shift of -1 indicates that the move // instruction will calculate an appropriate 16-bit immediate and left shift // that is equal to the 64-bit immediate argument. If an explicit left shift // is specified (0, 16, 32 or 48), the immediate must be a 16-bit value. // // For movk, an explicit shift can be used to indicate which half word should // be overwritten, eg. movk(x0, 0, 0) will overwrite the least-significant // half word with zero, whereas movk(x0, 0, 48) will overwrite the // most-significant. // Move and keep. void movk(const Register& rd, uint64_t imm, int shift = -1) { MoveWide(rd, imm, shift, MOVK); } // Move with non-zero. void movn(const Register& rd, uint64_t imm, int shift = -1) { MoveWide(rd, imm, shift, MOVN); } // Move with zero. void movz(const Register& rd, uint64_t imm, int shift = -1) { MoveWide(rd, imm, shift, MOVZ); } // Misc instructions. // Monitor debug-mode breakpoint. void brk(int code); // Halting debug-mode breakpoint. void hlt(int code); // Move register to register. void mov(const Register& rd, const Register& rn); // Move NOT(operand) to register. void mvn(const Register& rd, const Operand& operand); // System instructions. // Move to register from system register. void mrs(const Register& rt, SystemRegister sysreg); // Move from register to system register. void msr(SystemRegister sysreg, const Register& rt); // System hint. void hint(SystemHint code); // Data memory barrier void dmb(BarrierDomain domain, BarrierType type); // Data synchronization barrier void dsb(BarrierDomain domain, BarrierType type); // Instruction synchronization barrier void isb(); // Alias for system instructions. void nop() { hint(NOP); } // Different nop operations are used by the code generator to detect certain // states of the generated code. enum NopMarkerTypes { DEBUG_BREAK_NOP, INTERRUPT_CODE_NOP, ADR_FAR_NOP, FIRST_NOP_MARKER = DEBUG_BREAK_NOP, LAST_NOP_MARKER = ADR_FAR_NOP }; void nop(NopMarkerTypes n) { DCHECK((FIRST_NOP_MARKER <= n) && (n <= LAST_NOP_MARKER)); mov(Register::XRegFromCode(n), Register::XRegFromCode(n)); } // FP instructions. // Move immediate to FP register. void fmov(FPRegister fd, double imm); void fmov(FPRegister fd, float imm); // Move FP register to register. void fmov(Register rd, FPRegister fn); // Move register to FP register. void fmov(FPRegister fd, Register rn); // Move FP register to FP register. void fmov(FPRegister fd, FPRegister fn); // FP add. void fadd(const FPRegister& fd, const FPRegister& fn, const FPRegister& fm); // FP subtract. void fsub(const FPRegister& fd, const FPRegister& fn, const FPRegister& fm); // FP multiply. void fmul(const FPRegister& fd, const FPRegister& fn, const FPRegister& fm); // FP fused multiply and add. void fmadd(const FPRegister& fd, const FPRegister& fn, const FPRegister& fm, const FPRegister& fa); // FP fused multiply and subtract. void fmsub(const FPRegister& fd, const FPRegister& fn, const FPRegister& fm, const FPRegister& fa); // FP fused multiply, add and negate. void fnmadd(const FPRegister& fd, const FPRegister& fn, const FPRegister& fm, const FPRegister& fa); // FP fused multiply, subtract and negate. void fnmsub(const FPRegister& fd, const FPRegister& fn, const FPRegister& fm, const FPRegister& fa); // FP divide. void fdiv(const FPRegister& fd, const FPRegister& fn, const FPRegister& fm); // FP maximum. void fmax(const FPRegister& fd, const FPRegister& fn, const FPRegister& fm); // FP minimum. void fmin(const FPRegister& fd, const FPRegister& fn, const FPRegister& fm); // FP maximum. void fmaxnm(const FPRegister& fd, const FPRegister& fn, const FPRegister& fm); // FP minimum. void fminnm(const FPRegister& fd, const FPRegister& fn, const FPRegister& fm); // FP absolute. void fabs(const FPRegister& fd, const FPRegister& fn); // FP negate. void fneg(const FPRegister& fd, const FPRegister& fn); // FP square root. void fsqrt(const FPRegister& fd, const FPRegister& fn); // FP round to integer (nearest with ties to away). void frinta(const FPRegister& fd, const FPRegister& fn); // FP round to integer (toward minus infinity). void frintm(const FPRegister& fd, const FPRegister& fn); // FP round to integer (nearest with ties to even). void frintn(const FPRegister& fd, const FPRegister& fn); // FP round to integer (towards plus infinity). void frintp(const FPRegister& fd, const FPRegister& fn); // FP round to integer (towards zero.) void frintz(const FPRegister& fd, const FPRegister& fn); // FP compare registers. void fcmp(const FPRegister& fn, const FPRegister& fm); // FP compare immediate. void fcmp(const FPRegister& fn, double value); // FP conditional compare. void fccmp(const FPRegister& fn, const FPRegister& fm, StatusFlags nzcv, Condition cond); // FP conditional select. void fcsel(const FPRegister& fd, const FPRegister& fn, const FPRegister& fm, Condition cond); // Common FP Convert function void FPConvertToInt(const Register& rd, const FPRegister& fn, FPIntegerConvertOp op); // FP convert between single and double precision. void fcvt(const FPRegister& fd, const FPRegister& fn); // Convert FP to unsigned integer (nearest with ties to away). void fcvtau(const Register& rd, const FPRegister& fn); // Convert FP to signed integer (nearest with ties to away). void fcvtas(const Register& rd, const FPRegister& fn); // Convert FP to unsigned integer (round towards -infinity). void fcvtmu(const Register& rd, const FPRegister& fn); // Convert FP to signed integer (round towards -infinity). void fcvtms(const Register& rd, const FPRegister& fn); // Convert FP to unsigned integer (nearest with ties to even). void fcvtnu(const Register& rd, const FPRegister& fn); // Convert FP to signed integer (nearest with ties to even). void fcvtns(const Register& rd, const FPRegister& fn); // Convert FP to unsigned integer (round towards zero). void fcvtzu(const Register& rd, const FPRegister& fn); // Convert FP to signed integer (rounf towards zero). void fcvtzs(const Register& rd, const FPRegister& fn); // Convert signed integer or fixed point to FP. void scvtf(const FPRegister& fd, const Register& rn, unsigned fbits = 0); // Convert unsigned integer or fixed point to FP. void ucvtf(const FPRegister& fd, const Register& rn, unsigned fbits = 0); // Instruction functions used only for test, debug, and patching. // Emit raw instructions in the instruction stream. void dci(Instr raw_inst) { Emit(raw_inst); } // Emit 8 bits of data in the instruction stream. void dc8(uint8_t data) { EmitData(&data, sizeof(data)); } // Emit 32 bits of data in the instruction stream. void dc32(uint32_t data) { EmitData(&data, sizeof(data)); } // Emit 64 bits of data in the instruction stream. void dc64(uint64_t data) { EmitData(&data, sizeof(data)); } // Emit an address in the instruction stream. void dcptr(Label* label); // Copy a string into the instruction stream, including the terminating NULL // character. The instruction pointer (pc_) is then aligned correctly for // subsequent instructions. void EmitStringData(const char* string); // Pseudo-instructions ------------------------------------------------------ // Parameters are described in arm64/instructions-arm64.h. void debug(const char* message, uint32_t code, Instr params = BREAK); // Required by V8. void dd(uint32_t data) { dc32(data); } void db(uint8_t data) { dc8(data); } void dq(uint64_t data) { dc64(data); } void dp(uintptr_t data) { dc64(data); } // Code generation helpers -------------------------------------------------- bool IsConstPoolEmpty() const { return constpool_.IsEmpty(); } Instruction* pc() const { return Instruction::Cast(pc_); } Instruction* InstructionAt(ptrdiff_t offset) const { return reinterpret_cast(buffer_ + offset); } ptrdiff_t InstructionOffset(Instruction* instr) const { return reinterpret_cast(instr) - buffer_; } // Register encoding. static Instr Rd(CPURegister rd) { DCHECK(rd.code() != kSPRegInternalCode); return rd.code() << Rd_offset; } static Instr Rn(CPURegister rn) { DCHECK(rn.code() != kSPRegInternalCode); return rn.code() << Rn_offset; } static Instr Rm(CPURegister rm) { DCHECK(rm.code() != kSPRegInternalCode); return rm.code() << Rm_offset; } static Instr Ra(CPURegister ra) { DCHECK(ra.code() != kSPRegInternalCode); return ra.code() << Ra_offset; } static Instr Rt(CPURegister rt) { DCHECK(rt.code() != kSPRegInternalCode); return rt.code() << Rt_offset; } static Instr Rt2(CPURegister rt2) { DCHECK(rt2.code() != kSPRegInternalCode); return rt2.code() << Rt2_offset; } // These encoding functions allow the stack pointer to be encoded, and // disallow the zero register. static Instr RdSP(Register rd) { DCHECK(!rd.IsZero()); return (rd.code() & kRegCodeMask) << Rd_offset; } static Instr RnSP(Register rn) { DCHECK(!rn.IsZero()); return (rn.code() & kRegCodeMask) << Rn_offset; } // Flags encoding. inline static Instr Flags(FlagsUpdate S); inline static Instr Cond(Condition cond); // PC-relative address encoding. inline static Instr ImmPCRelAddress(int imm21); // Branch encoding. inline static Instr ImmUncondBranch(int imm26); inline static Instr ImmCondBranch(int imm19); inline static Instr ImmCmpBranch(int imm19); inline static Instr ImmTestBranch(int imm14); inline static Instr ImmTestBranchBit(unsigned bit_pos); // Data Processing encoding. inline static Instr SF(Register rd); inline static Instr ImmAddSub(int imm); inline static Instr ImmS(unsigned imms, unsigned reg_size); inline static Instr ImmR(unsigned immr, unsigned reg_size); inline static Instr ImmSetBits(unsigned imms, unsigned reg_size); inline static Instr ImmRotate(unsigned immr, unsigned reg_size); inline static Instr ImmLLiteral(int imm19); inline static Instr BitN(unsigned bitn, unsigned reg_size); inline static Instr ShiftDP(Shift shift); inline static Instr ImmDPShift(unsigned amount); inline static Instr ExtendMode(Extend extend); inline static Instr ImmExtendShift(unsigned left_shift); inline static Instr ImmCondCmp(unsigned imm); inline static Instr Nzcv(StatusFlags nzcv); static bool IsImmAddSub(int64_t immediate); static bool IsImmLogical(uint64_t value, unsigned width, unsigned* n, unsigned* imm_s, unsigned* imm_r); // MemOperand offset encoding. inline static Instr ImmLSUnsigned(int imm12); inline static Instr ImmLS(int imm9); inline static Instr ImmLSPair(int imm7, LSDataSize size); inline static Instr ImmShiftLS(unsigned shift_amount); inline static Instr ImmException(int imm16); inline static Instr ImmSystemRegister(int imm15); inline static Instr ImmHint(int imm7); inline static Instr ImmBarrierDomain(int imm2); inline static Instr ImmBarrierType(int imm2); inline static LSDataSize CalcLSDataSize(LoadStoreOp op); static bool IsImmLSUnscaled(int64_t offset); static bool IsImmLSScaled(int64_t offset, LSDataSize size); static bool IsImmLLiteral(int64_t offset); // Move immediates encoding. inline static Instr ImmMoveWide(int imm); inline static Instr ShiftMoveWide(int shift); // FP Immediates. static Instr ImmFP32(float imm); static Instr ImmFP64(double imm); inline static Instr FPScale(unsigned scale); // FP register type. inline static Instr FPType(FPRegister fd); // Class for scoping postponing the constant pool generation. class BlockConstPoolScope { public: explicit BlockConstPoolScope(Assembler* assem) : assem_(assem) { assem_->StartBlockConstPool(); } ~BlockConstPoolScope() { assem_->EndBlockConstPool(); } private: Assembler* assem_; DISALLOW_IMPLICIT_CONSTRUCTORS(BlockConstPoolScope); }; // Check if is time to emit a constant pool. void CheckConstPool(bool force_emit, bool require_jump); void PatchConstantPoolAccessInstruction(int pc_offset, int offset, ConstantPoolEntry::Access access, ConstantPoolEntry::Type type) { // No embedded constant pool support. UNREACHABLE(); } // Returns true if we should emit a veneer as soon as possible for a branch // which can at most reach to specified pc. bool ShouldEmitVeneer(int max_reachable_pc, int margin = kVeneerDistanceMargin); bool ShouldEmitVeneers(int margin = kVeneerDistanceMargin) { return ShouldEmitVeneer(unresolved_branches_first_limit(), margin); } // The maximum code size generated for a veneer. Currently one branch // instruction. This is for code size checking purposes, and can be extended // in the future for example if we decide to add nops between the veneers. static const int kMaxVeneerCodeSize = 1 * kInstructionSize; void RecordVeneerPool(int location_offset, int size); // Emits veneers for branches that are approaching their maximum range. // If need_protection is true, the veneers are protected by a branch jumping // over the code. void EmitVeneers(bool force_emit, bool need_protection, int margin = kVeneerDistanceMargin); void EmitVeneersGuard() { EmitPoolGuard(); } // Checks whether veneers need to be emitted at this point. // If force_emit is set, a veneer is generated for *all* unresolved branches. void CheckVeneerPool(bool force_emit, bool require_jump, int margin = kVeneerDistanceMargin); class BlockPoolsScope { public: explicit BlockPoolsScope(Assembler* assem) : assem_(assem) { assem_->StartBlockPools(); } ~BlockPoolsScope() { assem_->EndBlockPools(); } private: Assembler* assem_; DISALLOW_IMPLICIT_CONSTRUCTORS(BlockPoolsScope); }; protected: inline const Register& AppropriateZeroRegFor(const CPURegister& reg) const; void LoadStore(const CPURegister& rt, const MemOperand& addr, LoadStoreOp op); void LoadStorePair(const CPURegister& rt, const CPURegister& rt2, const MemOperand& addr, LoadStorePairOp op); static bool IsImmLSPair(int64_t offset, LSDataSize size); void Logical(const Register& rd, const Register& rn, const Operand& operand, LogicalOp op); void LogicalImmediate(const Register& rd, const Register& rn, unsigned n, unsigned imm_s, unsigned imm_r, LogicalOp op); void ConditionalCompare(const Register& rn, const Operand& operand, StatusFlags nzcv, Condition cond, ConditionalCompareOp op); static bool IsImmConditionalCompare(int64_t immediate); void AddSubWithCarry(const Register& rd, const Register& rn, const Operand& operand, FlagsUpdate S, AddSubWithCarryOp op); // Functions for emulating operands not directly supported by the instruction // set. void EmitShift(const Register& rd, const Register& rn, Shift shift, unsigned amount); void EmitExtendShift(const Register& rd, const Register& rn, Extend extend, unsigned left_shift); void AddSub(const Register& rd, const Register& rn, const Operand& operand, FlagsUpdate S, AddSubOp op); static bool IsImmFP32(float imm); static bool IsImmFP64(double imm); // Find an appropriate LoadStoreOp or LoadStorePairOp for the specified // registers. Only simple loads are supported; sign- and zero-extension (such // as in LDPSW_x or LDRB_w) are not supported. static inline LoadStoreOp LoadOpFor(const CPURegister& rt); static inline LoadStorePairOp LoadPairOpFor(const CPURegister& rt, const CPURegister& rt2); static inline LoadStoreOp StoreOpFor(const CPURegister& rt); static inline LoadStorePairOp StorePairOpFor(const CPURegister& rt, const CPURegister& rt2); static inline LoadLiteralOp LoadLiteralOpFor(const CPURegister& rt); // Remove the specified branch from the unbound label link chain. // If available, a veneer for this label can be used for other branches in the // chain if the link chain cannot be fixed up without this branch. void RemoveBranchFromLabelLinkChain(Instruction* branch, Label* label, Instruction* label_veneer = NULL); private: // Instruction helpers. void MoveWide(const Register& rd, uint64_t imm, int shift, MoveWideImmediateOp mov_op); void DataProcShiftedRegister(const Register& rd, const Register& rn, const Operand& operand, FlagsUpdate S, Instr op); void DataProcExtendedRegister(const Register& rd, const Register& rn, const Operand& operand, FlagsUpdate S, Instr op); void ConditionalSelect(const Register& rd, const Register& rn, const Register& rm, Condition cond, ConditionalSelectOp op); void DataProcessing1Source(const Register& rd, const Register& rn, DataProcessing1SourceOp op); void DataProcessing3Source(const Register& rd, const Register& rn, const Register& rm, const Register& ra, DataProcessing3SourceOp op); void FPDataProcessing1Source(const FPRegister& fd, const FPRegister& fn, FPDataProcessing1SourceOp op); void FPDataProcessing2Source(const FPRegister& fd, const FPRegister& fn, const FPRegister& fm, FPDataProcessing2SourceOp op); void FPDataProcessing3Source(const FPRegister& fd, const FPRegister& fn, const FPRegister& fm, const FPRegister& fa, FPDataProcessing3SourceOp op); // Label helpers. // Return an offset for a label-referencing instruction, typically a branch. int LinkAndGetByteOffsetTo(Label* label); // This is the same as LinkAndGetByteOffsetTo, but return an offset // suitable for fields that take instruction offsets. inline int LinkAndGetInstructionOffsetTo(Label* label); static const int kStartOfLabelLinkChain = 0; // Verify that a label's link chain is intact. void CheckLabelLinkChain(Label const * label); void RecordLiteral(int64_t imm, unsigned size); // Postpone the generation of the constant pool for the specified number of // instructions. void BlockConstPoolFor(int instructions); // Set how far from current pc the next constant pool check will be. void SetNextConstPoolCheckIn(int instructions) { next_constant_pool_check_ = pc_offset() + instructions * kInstructionSize; } // Emit the instruction at pc_. void Emit(Instr instruction) { STATIC_ASSERT(sizeof(*pc_) == 1); STATIC_ASSERT(sizeof(instruction) == kInstructionSize); DCHECK((pc_ + sizeof(instruction)) <= (buffer_ + buffer_size_)); memcpy(pc_, &instruction, sizeof(instruction)); pc_ += sizeof(instruction); CheckBuffer(); } // Emit data inline in the instruction stream. void EmitData(void const * data, unsigned size) { DCHECK(sizeof(*pc_) == 1); DCHECK((pc_ + size) <= (buffer_ + buffer_size_)); // TODO(all): Somehow register we have some data here. Then we can // disassemble it correctly. memcpy(pc_, data, size); pc_ += size; CheckBuffer(); } void GrowBuffer(); void CheckBufferSpace(); void CheckBuffer(); // Pc offset of the next constant pool check. int next_constant_pool_check_; // Constant pool generation // Pools are emitted in the instruction stream. They are emitted when: // * the distance to the first use is above a pre-defined distance or // * the numbers of entries in the pool is above a pre-defined size or // * code generation is finished // If a pool needs to be emitted before code generation is finished a branch // over the emitted pool will be inserted. // Constants in the pool may be addresses of functions that gets relocated; // if so, a relocation info entry is associated to the constant pool entry. // Repeated checking whether the constant pool should be emitted is rather // expensive. By default we only check again once a number of instructions // has been generated. That also means that the sizing of the buffers is not // an exact science, and that we rely on some slop to not overrun buffers. static const int kCheckConstPoolInterval = 128; // Distance to first use after a which a pool will be emitted. Pool entries // are accessed with pc relative load therefore this cannot be more than // 1 * MB. Since constant pool emission checks are interval based this value // is an approximation. static const int kApproxMaxDistToConstPool = 64 * KB; // Number of pool entries after which a pool will be emitted. Since constant // pool emission checks are interval based this value is an approximation. static const int kApproxMaxPoolEntryCount = 512; // Emission of the constant pool may be blocked in some code sequences. int const_pool_blocked_nesting_; // Block emission if this is not zero. int no_const_pool_before_; // Block emission before this pc offset. // Emission of the veneer pools may be blocked in some code sequences. int veneer_pool_blocked_nesting_; // Block emission if this is not zero. // Relocation info generation // Each relocation is encoded as a variable size value static const int kMaxRelocSize = RelocInfoWriter::kMaxSize; RelocInfoWriter reloc_info_writer; // Internal reference positions, required for (potential) patching in // GrowBuffer(); contains only those internal references whose labels // are already bound. std::deque internal_reference_positions_; // Relocation info records are also used during code generation as temporary // containers for constants and code target addresses until they are emitted // to the constant pool. These pending relocation info records are temporarily // stored in a separate buffer until a constant pool is emitted. // If every instruction in a long sequence is accessing the pool, we need one // pending relocation entry per instruction. // The pending constant pool. ConstPool constpool_; // Relocation for a type-recording IC has the AST id added to it. This // member variable is a way to pass the information from the call site to // the relocation info. TypeFeedbackId recorded_ast_id_; inline TypeFeedbackId RecordedAstId(); inline void ClearRecordedAstId(); protected: // Record the AST id of the CallIC being compiled, so that it can be placed // in the relocation information. void SetRecordedAstId(TypeFeedbackId ast_id) { DCHECK(recorded_ast_id_.IsNone()); recorded_ast_id_ = ast_id; } // Code generation // The relocation writer's position is at least kGap bytes below the end of // the generated instructions. This is so that multi-instruction sequences do // not have to check for overflow. The same is true for writes of large // relocation info entries, and debug strings encoded in the instruction // stream. static const int kGap = 128; public: class FarBranchInfo { public: FarBranchInfo(int offset, Label* label) : pc_offset_(offset), label_(label) {} // Offset of the branch in the code generation buffer. int pc_offset_; // The label branched to. Label* label_; }; protected: // Information about unresolved (forward) branches. // The Assembler is only allowed to delete out-of-date information from here // after a label is bound. The MacroAssembler uses this information to // generate veneers. // // The second member gives information about the unresolved branch. The first // member of the pair is the maximum offset that the branch can reach in the // buffer. The map is sorted according to this reachable offset, allowing to // easily check when veneers need to be emitted. // Note that the maximum reachable offset (first member of the pairs) should // always be positive but has the same type as the return value for // pc_offset() for convenience. std::multimap unresolved_branches_; // We generate a veneer for a branch if we reach within this distance of the // limit of the range. static const int kVeneerDistanceMargin = 1 * KB; // The factor of 2 is a finger in the air guess. With a default margin of // 1KB, that leaves us an addional 256 instructions to avoid generating a // protective branch. static const int kVeneerNoProtectionFactor = 2; static const int kVeneerDistanceCheckMargin = kVeneerNoProtectionFactor * kVeneerDistanceMargin; int unresolved_branches_first_limit() const { DCHECK(!unresolved_branches_.empty()); return unresolved_branches_.begin()->first; } // This is similar to next_constant_pool_check_ and helps reduce the overhead // of checking for veneer pools. // It is maintained to the closest unresolved branch limit minus the maximum // veneer margin (or kMaxInt if there are no unresolved branches). int next_veneer_pool_check_; private: // If a veneer is emitted for a branch instruction, that instruction must be // removed from the associated label's link chain so that the assembler does // not later attempt (likely unsuccessfully) to patch it to branch directly to // the label. void DeleteUnresolvedBranchInfoForLabel(Label* label); // This function deletes the information related to the label by traversing // the label chain, and for each PC-relative instruction in the chain checking // if pending unresolved information exists. Its complexity is proportional to // the length of the label chain. void DeleteUnresolvedBranchInfoForLabelTraverse(Label* label); private: PositionsRecorder positions_recorder_; friend class PositionsRecorder; friend class EnsureSpace; friend class ConstPool; }; class PatchingAssembler : public Assembler { public: // Create an Assembler with a buffer starting at 'start'. // The buffer size is // size of instructions to patch + kGap // Where kGap is the distance from which the Assembler tries to grow the // buffer. // If more or fewer instructions than expected are generated or if some // relocation information takes space in the buffer, the PatchingAssembler // will crash trying to grow the buffer. PatchingAssembler(Isolate* isolate, Instruction* start, unsigned count) : Assembler(isolate, reinterpret_cast(start), count * kInstructionSize + kGap) { StartBlockPools(); } PatchingAssembler(Isolate* isolate, byte* start, unsigned count) : Assembler(isolate, start, count * kInstructionSize + kGap) { // Block constant pool emission. StartBlockPools(); } ~PatchingAssembler() { // Const pool should still be blocked. DCHECK(is_const_pool_blocked()); EndBlockPools(); // Verify we have generated the number of instruction we expected. DCHECK((pc_offset() + kGap) == buffer_size_); // Verify no relocation information has been emitted. DCHECK(IsConstPoolEmpty()); // Flush the Instruction cache. size_t length = buffer_size_ - kGap; Assembler::FlushICache(isolate(), buffer_, length); } // See definition of PatchAdrFar() for details. static const int kAdrFarPatchableNNops = 2; static const int kAdrFarPatchableNInstrs = kAdrFarPatchableNNops + 2; void PatchAdrFar(int64_t target_offset); }; class EnsureSpace BASE_EMBEDDED { public: explicit EnsureSpace(Assembler* assembler) { assembler->CheckBufferSpace(); } }; } // namespace internal } // namespace v8 #endif // V8_ARM64_ASSEMBLER_ARM64_H_