1 /*
2 * Copyright (C) 2011 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 #include "codegen_arm64.h"
18
19 #include "arch/arm64/instruction_set_features_arm64.h"
20 #include "arm64_lir.h"
21 #include "base/logging.h"
22 #include "dex/compiler_ir.h"
23 #include "dex/quick/mir_to_lir-inl.h"
24 #include "driver/compiler_driver.h"
25
26 namespace art {
27
28 // The macros below are exclusively used in the encoding map.
29
30 // Most generic way of providing two variants for one instructions.
31 #define CUSTOM_VARIANTS(variant1, variant2) variant1, variant2
32
33 // Used for instructions which do not have a wide variant.
34 #define NO_VARIANTS(variant) \
35 CUSTOM_VARIANTS(variant, 0)
36
37 // Used for instructions which have a wide variant with the sf bit set to 1.
38 #define SF_VARIANTS(sf0_skeleton) \
39 CUSTOM_VARIANTS(sf0_skeleton, (sf0_skeleton | 0x80000000))
40
41 // Used for instructions which have a wide variant with the size bits set to either x0 or x1.
42 #define SIZE_VARIANTS(sizex0_skeleton) \
43 CUSTOM_VARIANTS(sizex0_skeleton, (sizex0_skeleton | 0x40000000))
44
45 // Used for instructions which have a wide variant with the sf and n bits set to 1.
46 #define SF_N_VARIANTS(sf0_n0_skeleton) \
47 CUSTOM_VARIANTS(sf0_n0_skeleton, (sf0_n0_skeleton | 0x80400000))
48
49 // Used for FP instructions which have a single and double precision variants, with he type bits set
50 // to either 00 or 01.
51 #define FLOAT_VARIANTS(type00_skeleton) \
52 CUSTOM_VARIANTS(type00_skeleton, (type00_skeleton | 0x00400000))
53
54 /*
55 * opcode: A64Opcode enum
56 * variants: instruction skeletons supplied via CUSTOM_VARIANTS or derived macros.
57 * a{n}k: key to applying argument {n} \
58 * a{n}s: argument {n} start bit position | n = 0, 1, 2, 3
59 * a{n}e: argument {n} end bit position /
60 * flags: instruction attributes (used in optimization)
61 * name: mnemonic name
62 * fmt: for pretty-printing
63 * fixup: used for second-pass fixes (e.g. adresses fixups in branch instructions).
64 */
65 #define ENCODING_MAP(opcode, variants, a0k, a0s, a0e, a1k, a1s, a1e, a2k, a2s, a2e, \
66 a3k, a3s, a3e, flags, name, fmt, fixup) \
67 {variants, {{a0k, a0s, a0e}, {a1k, a1s, a1e}, {a2k, a2s, a2e}, \
68 {a3k, a3s, a3e}}, opcode, flags, name, fmt, 4, fixup}
69
70 /* Instruction dump string format keys: !pf, where "!" is the start
71 * of the key, "p" is which numeric operand to use and "f" is the
72 * print format.
73 *
74 * [p]ositions:
75 * 0 -> operands[0] (dest)
76 * 1 -> operands[1] (src1)
77 * 2 -> operands[2] (src2)
78 * 3 -> operands[3] (extra)
79 *
80 * [f]ormats:
81 * d -> decimal
82 * D -> decimal*4 or decimal*8 depending on the instruction width
83 * E -> decimal*4
84 * F -> decimal*2
85 * G -> ", lsl #2" or ", lsl #3" depending on the instruction width
86 * c -> branch condition (eq, ne, etc.)
87 * t -> pc-relative target
88 * p -> pc-relative address
89 * s -> single precision floating point register
90 * S -> double precision floating point register
91 * f -> single or double precision register (depending on instruction width)
92 * I -> 8-bit immediate floating point number
93 * l -> logical immediate
94 * M -> 16-bit shift expression ("" or ", lsl #16" or ", lsl #32"...)
95 * B -> dmb option string (sy, st, ish, ishst, nsh, hshst)
96 * H -> operand shift
97 * h -> 6-bit shift immediate
98 * T -> register shift (either ", lsl #0" or ", lsl #12")
99 * e -> register extend (e.g. uxtb #1)
100 * o -> register shift (e.g. lsl #1) for Word registers
101 * w -> word (32-bit) register wn, or wzr
102 * W -> word (32-bit) register wn, or wsp
103 * x -> extended (64-bit) register xn, or xzr
104 * X -> extended (64-bit) register xn, or sp
105 * r -> register with same width as instruction, r31 -> wzr, xzr
106 * R -> register with same width as instruction, r31 -> wsp, sp
107 *
108 * [!] escape. To insert "!", use "!!"
109 */
110 /* NOTE: must be kept in sync with enum A64Opcode from arm64_lir.h */
111 const A64EncodingMap Arm64Mir2Lir::EncodingMap[kA64Last] = {
112 ENCODING_MAP(WIDE(kA64Adc3rrr), SF_VARIANTS(0x1a000000),
113 kFmtRegR, 4, 0, kFmtRegR, 9, 5, kFmtRegR, 20, 16,
114 kFmtUnused, -1, -1, IS_TERTIARY_OP | REG_DEF0_USE12 | USES_CCODES,
115 "adc", "!0r, !1r, !2r", kFixupNone),
116 ENCODING_MAP(WIDE(kA64Add4RRdT), SF_VARIANTS(0x11000000),
117 kFmtRegROrSp, 4, 0, kFmtRegROrSp, 9, 5, kFmtBitBlt, 21, 10,
118 kFmtBitBlt, 23, 22, IS_QUAD_OP | REG_DEF0_USE1,
119 "add", "!0R, !1R, #!2d!3T", kFixupNone),
120 ENCODING_MAP(WIDE(kA64Add4rrro), SF_VARIANTS(0x0b000000),
121 kFmtRegR, 4, 0, kFmtRegR, 9, 5, kFmtRegR, 20, 16,
122 kFmtShift, -1, -1, IS_QUAD_OP | REG_DEF0_USE12,
123 "add", "!0r, !1r, !2r!3o", kFixupNone),
124 ENCODING_MAP(WIDE(kA64Add4RRre), SF_VARIANTS(0x0b200000),
125 kFmtRegROrSp, 4, 0, kFmtRegROrSp, 9, 5, kFmtRegR, 20, 16,
126 kFmtExtend, -1, -1, IS_QUAD_OP | REG_DEF0_USE12,
127 "add", "!0r, !1r, !2r!3e", kFixupNone),
128 // Note: adr is binary, but declared as tertiary. The third argument is used while doing the
129 // fixups and contains information to identify the adr label.
130 ENCODING_MAP(kA64Adr2xd, NO_VARIANTS(0x10000000),
131 kFmtRegX, 4, 0, kFmtImm21, -1, -1, kFmtUnused, -1, -1,
132 kFmtUnused, -1, -1, IS_TERTIARY_OP | REG_DEF0 | NEEDS_FIXUP,
133 "adr", "!0x, #!1d", kFixupAdr),
134 ENCODING_MAP(kA64Adrp2xd, NO_VARIANTS(0x90000000),
135 kFmtRegX, 4, 0, kFmtImm21, -1, -1, kFmtUnused, -1, -1,
136 kFmtUnused, -1, -1, IS_BINARY_OP | REG_DEF0 | NEEDS_FIXUP,
137 "adrp", "!0x, #!1d", kFixupLabel),
138 ENCODING_MAP(WIDE(kA64And3Rrl), SF_VARIANTS(0x12000000),
139 kFmtRegROrSp, 4, 0, kFmtRegR, 9, 5, kFmtBitBlt, 22, 10,
140 kFmtUnused, -1, -1, IS_TERTIARY_OP | REG_DEF0_USE1,
141 "and", "!0R, !1r, #!2l", kFixupNone),
142 ENCODING_MAP(WIDE(kA64And4rrro), SF_VARIANTS(0x0a000000),
143 kFmtRegR, 4, 0, kFmtRegR, 9, 5, kFmtRegR, 20, 16,
144 kFmtShift, -1, -1, IS_QUAD_OP | REG_DEF0_USE12,
145 "and", "!0r, !1r, !2r!3o", kFixupNone),
146 ENCODING_MAP(WIDE(kA64Asr3rrd), CUSTOM_VARIANTS(0x13007c00, 0x9340fc00),
147 kFmtRegR, 4, 0, kFmtRegR, 9, 5, kFmtBitBlt, 21, 16,
148 kFmtUnused, -1, -1, IS_TERTIARY_OP | REG_DEF0_USE1,
149 "asr", "!0r, !1r, #!2d", kFixupNone),
150 ENCODING_MAP(WIDE(kA64Asr3rrr), SF_VARIANTS(0x1ac02800),
151 kFmtRegR, 4, 0, kFmtRegR, 9, 5, kFmtRegR, 20, 16,
152 kFmtUnused, -1, -1, IS_TERTIARY_OP | REG_DEF0_USE12,
153 "asr", "!0r, !1r, !2r", kFixupNone),
154 ENCODING_MAP(kA64B2ct, NO_VARIANTS(0x54000000),
155 kFmtBitBlt, 3, 0, kFmtBitBlt, 23, 5, kFmtUnused, -1, -1,
156 kFmtUnused, -1, -1, IS_BINARY_OP | IS_BRANCH | USES_CCODES |
157 NEEDS_FIXUP, "b.!0c", "!1t", kFixupCondBranch),
158 ENCODING_MAP(kA64Blr1x, NO_VARIANTS(0xd63f0000),
159 kFmtRegX, 9, 5, kFmtUnused, -1, -1, kFmtUnused, -1, -1,
160 kFmtUnused, -1, -1,
161 IS_UNARY_OP | REG_USE0 | IS_BRANCH | REG_DEF_LR,
162 "blr", "!0x", kFixupNone),
163 ENCODING_MAP(kA64Br1x, NO_VARIANTS(0xd61f0000),
164 kFmtRegX, 9, 5, kFmtUnused, -1, -1, kFmtUnused, -1, -1,
165 kFmtUnused, -1, -1, IS_UNARY_OP | REG_USE0 | IS_BRANCH,
166 "br", "!0x", kFixupNone),
167 ENCODING_MAP(kA64Bl1t, NO_VARIANTS(0x94000000),
168 kFmtBitBlt, 25, 0, kFmtUnused, -1, -1, kFmtUnused, -1, -1,
169 kFmtUnused, -1, -1, IS_UNARY_OP | IS_BRANCH | REG_DEF_LR | NEEDS_FIXUP,
170 "bl", "!0T", kFixupLabel),
171 ENCODING_MAP(kA64Brk1d, NO_VARIANTS(0xd4200000),
172 kFmtBitBlt, 20, 5, kFmtUnused, -1, -1, kFmtUnused, -1, -1,
173 kFmtUnused, -1, -1, IS_UNARY_OP | IS_BRANCH,
174 "brk", "!0d", kFixupNone),
175 ENCODING_MAP(kA64B1t, NO_VARIANTS(0x14000000),
176 kFmtBitBlt, 25, 0, kFmtUnused, -1, -1, kFmtUnused, -1, -1,
177 kFmtUnused, -1, -1, IS_UNARY_OP | IS_BRANCH | NEEDS_FIXUP,
178 "b", "!0t", kFixupT1Branch),
179 ENCODING_MAP(WIDE(kA64Cbnz2rt), SF_VARIANTS(0x35000000),
180 kFmtRegR, 4, 0, kFmtBitBlt, 23, 5, kFmtUnused, -1, -1,
181 kFmtUnused, -1, -1,
182 IS_BINARY_OP | REG_USE0 | IS_BRANCH | NEEDS_FIXUP,
183 "cbnz", "!0r, !1t", kFixupCBxZ),
184 ENCODING_MAP(WIDE(kA64Cbz2rt), SF_VARIANTS(0x34000000),
185 kFmtRegR, 4, 0, kFmtBitBlt, 23, 5, kFmtUnused, -1, -1,
186 kFmtUnused, -1, -1,
187 IS_BINARY_OP | REG_USE0 | IS_BRANCH | NEEDS_FIXUP,
188 "cbz", "!0r, !1t", kFixupCBxZ),
189 ENCODING_MAP(WIDE(kA64Cmn3rro), SF_VARIANTS(0x2b00001f),
190 kFmtRegR, 9, 5, kFmtRegR, 20, 16, kFmtShift, -1, -1,
191 kFmtUnused, -1, -1, IS_TERTIARY_OP | REG_USE01 | SETS_CCODES,
192 "cmn", "!0r, !1r!2o", kFixupNone),
193 ENCODING_MAP(WIDE(kA64Cmn3Rre), SF_VARIANTS(0x2b20001f),
194 kFmtRegROrSp, 9, 5, kFmtRegR, 20, 16, kFmtExtend, -1, -1,
195 kFmtUnused, -1, -1, IS_TERTIARY_OP | REG_USE01 | SETS_CCODES,
196 "cmn", "!0R, !1r!2e", kFixupNone),
197 ENCODING_MAP(WIDE(kA64Cmn3RdT), SF_VARIANTS(0x3100001f),
198 kFmtRegROrSp, 9, 5, kFmtBitBlt, 21, 10, kFmtBitBlt, 23, 22,
199 kFmtUnused, -1, -1, IS_TERTIARY_OP | REG_USE0 | SETS_CCODES,
200 "cmn", "!0R, #!1d!2T", kFixupNone),
201 ENCODING_MAP(WIDE(kA64Cmp3rro), SF_VARIANTS(0x6b00001f),
202 kFmtRegR, 9, 5, kFmtRegR, 20, 16, kFmtShift, -1, -1,
203 kFmtUnused, -1, -1, IS_TERTIARY_OP | REG_USE01 | SETS_CCODES,
204 "cmp", "!0r, !1r!2o", kFixupNone),
205 ENCODING_MAP(WIDE(kA64Cmp3Rre), SF_VARIANTS(0x6b20001f),
206 kFmtRegROrSp, 9, 5, kFmtRegR, 20, 16, kFmtExtend, -1, -1,
207 kFmtUnused, -1, -1, IS_TERTIARY_OP | REG_USE01 | SETS_CCODES,
208 "cmp", "!0R, !1r!2e", kFixupNone),
209 ENCODING_MAP(WIDE(kA64Cmp3RdT), SF_VARIANTS(0x7100001f),
210 kFmtRegROrSp, 9, 5, kFmtBitBlt, 21, 10, kFmtBitBlt, 23, 22,
211 kFmtUnused, -1, -1, IS_TERTIARY_OP | REG_USE0 | SETS_CCODES,
212 "cmp", "!0R, #!1d!2T", kFixupNone),
213 ENCODING_MAP(WIDE(kA64Csel4rrrc), SF_VARIANTS(0x1a800000),
214 kFmtRegR, 4, 0, kFmtRegR, 9, 5, kFmtRegR, 20, 16,
215 kFmtBitBlt, 15, 12, IS_QUAD_OP | REG_DEF0_USE12 | USES_CCODES,
216 "csel", "!0r, !1r, !2r, !3c", kFixupNone),
217 ENCODING_MAP(WIDE(kA64Csinc4rrrc), SF_VARIANTS(0x1a800400),
218 kFmtRegR, 4, 0, kFmtRegR, 9, 5, kFmtRegR, 20, 16,
219 kFmtBitBlt, 15, 12, IS_QUAD_OP | REG_DEF0_USE12 | USES_CCODES,
220 "csinc", "!0r, !1r, !2r, !3c", kFixupNone),
221 ENCODING_MAP(WIDE(kA64Csinv4rrrc), SF_VARIANTS(0x5a800000),
222 kFmtRegR, 4, 0, kFmtRegR, 9, 5, kFmtRegR, 20, 16,
223 kFmtBitBlt, 15, 12, IS_QUAD_OP | REG_DEF0_USE12 | USES_CCODES,
224 "csinv", "!0r, !1r, !2r, !3c", kFixupNone),
225 ENCODING_MAP(WIDE(kA64Csneg4rrrc), SF_VARIANTS(0x5a800400),
226 kFmtRegR, 4, 0, kFmtRegR, 9, 5, kFmtRegR, 20, 16,
227 kFmtBitBlt, 15, 12, IS_QUAD_OP | REG_DEF0_USE12 | USES_CCODES,
228 "csneg", "!0r, !1r, !2r, !3c", kFixupNone),
229 ENCODING_MAP(kA64Dmb1B, NO_VARIANTS(0xd50330bf),
230 kFmtBitBlt, 11, 8, kFmtUnused, -1, -1, kFmtUnused, -1, -1,
231 kFmtUnused, -1, -1, IS_UNARY_OP | IS_VOLATILE,
232 "dmb", "#!0B", kFixupNone),
233 ENCODING_MAP(WIDE(kA64Eor3Rrl), SF_VARIANTS(0x52000000),
234 kFmtRegROrSp, 4, 0, kFmtRegR, 9, 5, kFmtBitBlt, 22, 10,
235 kFmtUnused, -1, -1, IS_TERTIARY_OP | REG_DEF0_USE1,
236 "eor", "!0R, !1r, #!2l", kFixupNone),
237 ENCODING_MAP(WIDE(kA64Eor4rrro), SF_VARIANTS(0x4a000000),
238 kFmtRegR, 4, 0, kFmtRegR, 9, 5, kFmtRegR, 20, 16,
239 kFmtShift, -1, -1, IS_QUAD_OP | REG_DEF0_USE12,
240 "eor", "!0r, !1r, !2r!3o", kFixupNone),
241 ENCODING_MAP(WIDE(kA64Extr4rrrd), SF_N_VARIANTS(0x13800000),
242 kFmtRegR, 4, 0, kFmtRegR, 9, 5, kFmtRegR, 20, 16,
243 kFmtBitBlt, 15, 10, IS_QUAD_OP | REG_DEF0_USE12,
244 "extr", "!0r, !1r, !2r, #!3d", kFixupNone),
245 ENCODING_MAP(WIDE(kA64Fabs2ff), FLOAT_VARIANTS(0x1e20c000),
246 kFmtRegF, 4, 0, kFmtRegF, 9, 5, kFmtUnused, -1, -1,
247 kFmtUnused, -1, -1, IS_BINARY_OP| REG_DEF0_USE1,
248 "fabs", "!0f, !1f", kFixupNone),
249 ENCODING_MAP(WIDE(kA64Fadd3fff), FLOAT_VARIANTS(0x1e202800),
250 kFmtRegF, 4, 0, kFmtRegF, 9, 5, kFmtRegF, 20, 16,
251 kFmtUnused, -1, -1, IS_TERTIARY_OP | REG_DEF0_USE12,
252 "fadd", "!0f, !1f, !2f", kFixupNone),
253 ENCODING_MAP(WIDE(kA64Fcmp1f), FLOAT_VARIANTS(0x1e202008),
254 kFmtRegF, 9, 5, kFmtUnused, -1, -1, kFmtUnused, -1, -1,
255 kFmtUnused, -1, -1, IS_UNARY_OP | REG_USE0 | SETS_CCODES,
256 "fcmp", "!0f, #0", kFixupNone),
257 ENCODING_MAP(WIDE(kA64Fcmp2ff), FLOAT_VARIANTS(0x1e202000),
258 kFmtRegF, 9, 5, kFmtRegF, 20, 16, kFmtUnused, -1, -1,
259 kFmtUnused, -1, -1, IS_BINARY_OP | REG_USE01 | SETS_CCODES,
260 "fcmp", "!0f, !1f", kFixupNone),
261 ENCODING_MAP(WIDE(kA64Fcvtzs2wf), FLOAT_VARIANTS(0x1e380000),
262 kFmtRegW, 4, 0, kFmtRegF, 9, 5, kFmtUnused, -1, -1,
263 kFmtUnused, -1, -1, IS_BINARY_OP | REG_DEF0_USE1,
264 "fcvtzs", "!0w, !1f", kFixupNone),
265 ENCODING_MAP(WIDE(kA64Fcvtzs2xf), FLOAT_VARIANTS(0x9e380000),
266 kFmtRegX, 4, 0, kFmtRegF, 9, 5, kFmtUnused, -1, -1,
267 kFmtUnused, -1, -1, IS_BINARY_OP | REG_DEF0_USE1,
268 "fcvtzs", "!0x, !1f", kFixupNone),
269 ENCODING_MAP(kA64Fcvt2Ss, NO_VARIANTS(0x1e22C000),
270 kFmtRegD, 4, 0, kFmtRegS, 9, 5, kFmtUnused, -1, -1,
271 kFmtUnused, -1, -1, IS_BINARY_OP | REG_DEF0_USE1,
272 "fcvt", "!0S, !1s", kFixupNone),
273 ENCODING_MAP(kA64Fcvt2sS, NO_VARIANTS(0x1e624000),
274 kFmtRegS, 4, 0, kFmtRegD, 9, 5, kFmtUnused, -1, -1,
275 kFmtUnused, -1, -1, IS_BINARY_OP | REG_DEF0_USE1,
276 "fcvt", "!0s, !1S", kFixupNone),
277 ENCODING_MAP(kA64Fcvtms2ws, NO_VARIANTS(0x1e300000),
278 kFmtRegW, 4, 0, kFmtRegS, 9, 5, kFmtUnused, -1, -1,
279 kFmtUnused, -1, -1, IS_BINARY_OP | REG_DEF0_USE1,
280 "fcvtms", "!0w, !1s", kFixupNone),
281 ENCODING_MAP(kA64Fcvtms2xS, NO_VARIANTS(0x9e700000),
282 kFmtRegX, 4, 0, kFmtRegD, 9, 5, kFmtUnused, -1, -1,
283 kFmtUnused, -1, -1, IS_BINARY_OP | REG_DEF0_USE1,
284 "fcvtms", "!0x, !1S", kFixupNone),
285 ENCODING_MAP(WIDE(kA64Fdiv3fff), FLOAT_VARIANTS(0x1e201800),
286 kFmtRegF, 4, 0, kFmtRegF, 9, 5, kFmtRegF, 20, 16,
287 kFmtUnused, -1, -1, IS_TERTIARY_OP | REG_DEF0_USE12,
288 "fdiv", "!0f, !1f, !2f", kFixupNone),
289 ENCODING_MAP(WIDE(kA64Fmax3fff), FLOAT_VARIANTS(0x1e204800),
290 kFmtRegF, 4, 0, kFmtRegF, 9, 5, kFmtRegF, 20, 16,
291 kFmtUnused, -1, -1, IS_TERTIARY_OP | REG_DEF0_USE12,
292 "fmax", "!0f, !1f, !2f", kFixupNone),
293 ENCODING_MAP(WIDE(kA64Fmin3fff), FLOAT_VARIANTS(0x1e205800),
294 kFmtRegF, 4, 0, kFmtRegF, 9, 5, kFmtRegF, 20, 16,
295 kFmtUnused, -1, -1, IS_TERTIARY_OP | REG_DEF0_USE12,
296 "fmin", "!0f, !1f, !2f", kFixupNone),
297 ENCODING_MAP(WIDE(kA64Fmov2ff), FLOAT_VARIANTS(0x1e204000),
298 kFmtRegF, 4, 0, kFmtRegF, 9, 5, kFmtUnused, -1, -1,
299 kFmtUnused, -1, -1, IS_BINARY_OP | REG_DEF0_USE1 | IS_MOVE,
300 "fmov", "!0f, !1f", kFixupNone),
301 ENCODING_MAP(WIDE(kA64Fmov2fI), FLOAT_VARIANTS(0x1e201000),
302 kFmtRegF, 4, 0, kFmtBitBlt, 20, 13, kFmtUnused, -1, -1,
303 kFmtUnused, -1, -1, IS_BINARY_OP | REG_DEF0,
304 "fmov", "!0f, #!1I", kFixupNone),
305 ENCODING_MAP(kA64Fmov2sw, NO_VARIANTS(0x1e270000),
306 kFmtRegS, 4, 0, kFmtRegW, 9, 5, kFmtUnused, -1, -1,
307 kFmtUnused, -1, -1, IS_BINARY_OP | REG_DEF0_USE1,
308 "fmov", "!0s, !1w", kFixupNone),
309 ENCODING_MAP(kA64Fmov2Sx, NO_VARIANTS(0x9e670000),
310 kFmtRegD, 4, 0, kFmtRegX, 9, 5, kFmtUnused, -1, -1,
311 kFmtUnused, -1, -1, IS_BINARY_OP | REG_DEF0_USE1,
312 "fmov", "!0S, !1x", kFixupNone),
313 ENCODING_MAP(kA64Fmov2ws, NO_VARIANTS(0x1e260000),
314 kFmtRegW, 4, 0, kFmtRegS, 9, 5, kFmtUnused, -1, -1,
315 kFmtUnused, -1, -1, IS_BINARY_OP | REG_DEF0_USE1,
316 "fmov", "!0w, !1s", kFixupNone),
317 ENCODING_MAP(kA64Fmov2xS, NO_VARIANTS(0x9e660000),
318 kFmtRegX, 4, 0, kFmtRegD, 9, 5, kFmtUnused, -1, -1,
319 kFmtUnused, -1, -1, IS_BINARY_OP | REG_DEF0_USE1,
320 "fmov", "!0x, !1S", kFixupNone),
321 ENCODING_MAP(WIDE(kA64Fmul3fff), FLOAT_VARIANTS(0x1e200800),
322 kFmtRegF, 4, 0, kFmtRegF, 9, 5, kFmtRegF, 20, 16,
323 kFmtUnused, -1, -1, IS_TERTIARY_OP | REG_DEF0_USE12,
324 "fmul", "!0f, !1f, !2f", kFixupNone),
325 ENCODING_MAP(WIDE(kA64Fneg2ff), FLOAT_VARIANTS(0x1e214000),
326 kFmtRegF, 4, 0, kFmtRegF, 9, 5, kFmtUnused, -1, -1,
327 kFmtUnused, -1, -1, IS_BINARY_OP | REG_DEF0_USE1,
328 "fneg", "!0f, !1f", kFixupNone),
329 ENCODING_MAP(WIDE(kA64Frintp2ff), FLOAT_VARIANTS(0x1e24c000),
330 kFmtRegF, 4, 0, kFmtRegF, 9, 5, kFmtUnused, -1, -1,
331 kFmtUnused, -1, -1, IS_BINARY_OP | REG_DEF0_USE1,
332 "frintp", "!0f, !1f", kFixupNone),
333 ENCODING_MAP(WIDE(kA64Frintm2ff), FLOAT_VARIANTS(0x1e254000),
334 kFmtRegF, 4, 0, kFmtRegF, 9, 5, kFmtUnused, -1, -1,
335 kFmtUnused, -1, -1, IS_BINARY_OP | REG_DEF0_USE1,
336 "frintm", "!0f, !1f", kFixupNone),
337 ENCODING_MAP(WIDE(kA64Frintn2ff), FLOAT_VARIANTS(0x1e244000),
338 kFmtRegF, 4, 0, kFmtRegF, 9, 5, kFmtUnused, -1, -1,
339 kFmtUnused, -1, -1, IS_BINARY_OP | REG_DEF0_USE1,
340 "frintn", "!0f, !1f", kFixupNone),
341 ENCODING_MAP(WIDE(kA64Frintz2ff), FLOAT_VARIANTS(0x1e25c000),
342 kFmtRegF, 4, 0, kFmtRegF, 9, 5, kFmtUnused, -1, -1,
343 kFmtUnused, -1, -1, IS_BINARY_OP | REG_DEF0_USE1,
344 "frintz", "!0f, !1f", kFixupNone),
345 ENCODING_MAP(WIDE(kA64Fsqrt2ff), FLOAT_VARIANTS(0x1e61c000),
346 kFmtRegF, 4, 0, kFmtRegF, 9, 5, kFmtUnused, -1, -1,
347 kFmtUnused, -1, -1, IS_BINARY_OP | REG_DEF0_USE1,
348 "fsqrt", "!0f, !1f", kFixupNone),
349 ENCODING_MAP(WIDE(kA64Fsub3fff), FLOAT_VARIANTS(0x1e203800),
350 kFmtRegF, 4, 0, kFmtRegF, 9, 5, kFmtRegF, 20, 16,
351 kFmtUnused, -1, -1, IS_TERTIARY_OP | REG_DEF0_USE12,
352 "fsub", "!0f, !1f, !2f", kFixupNone),
353 ENCODING_MAP(kA64Ldrb3wXd, NO_VARIANTS(0x39400000),
354 kFmtRegW, 4, 0, kFmtRegXOrSp, 9, 5, kFmtBitBlt, 21, 10,
355 kFmtUnused, -1, -1, IS_TERTIARY_OP | REG_DEF0_USE1 | IS_LOAD_OFF,
356 "ldrb", "!0w, [!1X, #!2d]", kFixupNone),
357 ENCODING_MAP(kA64Ldrb3wXx, NO_VARIANTS(0x38606800),
358 kFmtRegW, 4, 0, kFmtRegXOrSp, 9, 5, kFmtRegX, 20, 16,
359 kFmtUnused, -1, -1, IS_TERTIARY_OP | REG_DEF0_USE12 | IS_LOAD,
360 "ldrb", "!0w, [!1X, !2x]", kFixupNone),
361 ENCODING_MAP(WIDE(kA64Ldrsb3rXd), CUSTOM_VARIANTS(0x39c00000, 0x39800000),
362 kFmtRegR, 4, 0, kFmtRegXOrSp, 9, 5, kFmtBitBlt, 21, 10,
363 kFmtUnused, -1, -1, IS_TERTIARY_OP | REG_DEF0_USE1 | IS_LOAD_OFF,
364 "ldrsb", "!0r, [!1X, #!2d]", kFixupNone),
365 ENCODING_MAP(WIDE(kA64Ldrsb3rXx), CUSTOM_VARIANTS(0x38e06800, 0x38a06800),
366 kFmtRegR, 4, 0, kFmtRegXOrSp, 9, 5, kFmtRegX, 20, 16,
367 kFmtUnused, -1, -1, IS_TERTIARY_OP | REG_DEF0_USE12 | IS_LOAD,
368 "ldrsb", "!0r, [!1X, !2x]", kFixupNone),
369 ENCODING_MAP(kA64Ldrh3wXF, NO_VARIANTS(0x79400000),
370 kFmtRegW, 4, 0, kFmtRegXOrSp, 9, 5, kFmtBitBlt, 21, 10,
371 kFmtUnused, -1, -1, IS_TERTIARY_OP | REG_DEF0_USE1 | IS_LOAD_OFF,
372 "ldrh", "!0w, [!1X, #!2F]", kFixupNone),
373 ENCODING_MAP(kA64Ldrh4wXxd, NO_VARIANTS(0x78606800),
374 kFmtRegW, 4, 0, kFmtRegXOrSp, 9, 5, kFmtRegX, 20, 16,
375 kFmtBitBlt, 12, 12, IS_QUAD_OP | REG_DEF0_USE12 | IS_LOAD_OFF,
376 "ldrh", "!0w, [!1X, !2x, lsl #!3d]", kFixupNone),
377 ENCODING_MAP(WIDE(kA64Ldrsh3rXF), CUSTOM_VARIANTS(0x79c00000, 0x79800000),
378 kFmtRegR, 4, 0, kFmtRegXOrSp, 9, 5, kFmtBitBlt, 21, 10,
379 kFmtUnused, -1, -1, IS_TERTIARY_OP | REG_DEF0_USE1 | IS_LOAD_OFF,
380 "ldrsh", "!0r, [!1X, #!2F]", kFixupNone),
381 ENCODING_MAP(WIDE(kA64Ldrsh4rXxd), CUSTOM_VARIANTS(0x78e06800, 0x78906800),
382 kFmtRegR, 4, 0, kFmtRegXOrSp, 9, 5, kFmtRegX, 20, 16,
383 kFmtBitBlt, 12, 12, IS_QUAD_OP | REG_DEF0_USE12 | IS_LOAD_OFF,
384 "ldrsh", "!0r, [!1X, !2x, lsl #!3d]", kFixupNone),
385 ENCODING_MAP(WIDE(kA64Ldr2fp), SIZE_VARIANTS(0x1c000000),
386 kFmtRegF, 4, 0, kFmtBitBlt, 23, 5, kFmtUnused, -1, -1,
387 kFmtUnused, -1, -1,
388 IS_BINARY_OP | REG_DEF0 | REG_USE_PC | IS_LOAD | NEEDS_FIXUP,
389 "ldr", "!0f, !1p", kFixupLoad),
390 ENCODING_MAP(WIDE(kA64Ldr2rp), SIZE_VARIANTS(0x18000000),
391 kFmtRegR, 4, 0, kFmtBitBlt, 23, 5, kFmtUnused, -1, -1,
392 kFmtUnused, -1, -1,
393 IS_BINARY_OP | REG_DEF0 | REG_USE_PC | IS_LOAD | NEEDS_FIXUP,
394 "ldr", "!0r, !1p", kFixupLoad),
395 ENCODING_MAP(WIDE(kA64Ldr3fXD), SIZE_VARIANTS(0xbd400000),
396 kFmtRegF, 4, 0, kFmtRegXOrSp, 9, 5, kFmtBitBlt, 21, 10,
397 kFmtUnused, -1, -1, IS_TERTIARY_OP | REG_DEF0_USE1 | IS_LOAD_OFF,
398 "ldr", "!0f, [!1X, #!2D]", kFixupNone),
399 ENCODING_MAP(WIDE(kA64Ldr3rXD), SIZE_VARIANTS(0xb9400000),
400 kFmtRegR, 4, 0, kFmtRegXOrSp, 9, 5, kFmtBitBlt, 21, 10,
401 kFmtUnused, -1, -1, IS_TERTIARY_OP | REG_DEF0_USE1 | IS_LOAD_OFF,
402 "ldr", "!0r, [!1X, #!2D]", kFixupNone),
403 ENCODING_MAP(WIDE(kA64Ldr4fXxG), SIZE_VARIANTS(0xbc606800),
404 kFmtRegF, 4, 0, kFmtRegXOrSp, 9, 5, kFmtRegX, 20, 16,
405 kFmtBitBlt, 12, 12, IS_QUAD_OP | REG_DEF0_USE12 | IS_LOAD,
406 "ldr", "!0f, [!1X, !2x!3G]", kFixupNone),
407 ENCODING_MAP(WIDE(kA64Ldr4rXxG), SIZE_VARIANTS(0xb8606800),
408 kFmtRegR, 4, 0, kFmtRegXOrSp, 9, 5, kFmtRegX, 20, 16,
409 kFmtBitBlt, 12, 12, IS_QUAD_OP | REG_DEF0_USE12 | IS_LOAD,
410 "ldr", "!0r, [!1X, !2x!3G]", kFixupNone),
411 ENCODING_MAP(WIDE(kA64LdrPost3rXd), SIZE_VARIANTS(0xb8400400),
412 kFmtRegR, 4, 0, kFmtRegXOrSp, 9, 5, kFmtBitBlt, 20, 12,
413 kFmtUnused, -1, -1, IS_TERTIARY_OP | REG_DEF01 | REG_USE1 | IS_LOAD,
414 "ldr", "!0r, [!1X], #!2d", kFixupNone),
415 ENCODING_MAP(WIDE(kA64Ldp4ffXD), CUSTOM_VARIANTS(0x2d400000, 0x6d400000),
416 kFmtRegF, 4, 0, kFmtRegF, 14, 10, kFmtRegXOrSp, 9, 5,
417 kFmtBitBlt, 21, 15, IS_QUAD_OP | REG_USE2 | REG_DEF01 | IS_LOAD_OFF,
418 "ldp", "!0f, !1f, [!2X, #!3D]", kFixupNone),
419 ENCODING_MAP(WIDE(kA64Ldp4rrXD), SF_VARIANTS(0x29400000),
420 kFmtRegR, 4, 0, kFmtRegR, 14, 10, kFmtRegXOrSp, 9, 5,
421 kFmtBitBlt, 21, 15, IS_QUAD_OP | REG_USE2 | REG_DEF01 | IS_LOAD_OFF,
422 "ldp", "!0r, !1r, [!2X, #!3D]", kFixupNone),
423 ENCODING_MAP(WIDE(kA64LdpPost4rrXD), CUSTOM_VARIANTS(0x28c00000, 0xa8c00000),
424 kFmtRegR, 4, 0, kFmtRegR, 14, 10, kFmtRegXOrSp, 9, 5,
425 kFmtBitBlt, 21, 15, IS_QUAD_OP | REG_USE2 | REG_DEF012 | IS_LOAD,
426 "ldp", "!0r, !1r, [!2X], #!3D", kFixupNone),
427 ENCODING_MAP(WIDE(kA64Ldur3fXd), CUSTOM_VARIANTS(0xbc400000, 0xfc400000),
428 kFmtRegF, 4, 0, kFmtRegXOrSp, 9, 5, kFmtBitBlt, 20, 12,
429 kFmtUnused, -1, -1, IS_TERTIARY_OP | REG_DEF0_USE1 | IS_LOAD,
430 "ldur", "!0f, [!1X, #!2d]", kFixupNone),
431 ENCODING_MAP(WIDE(kA64Ldur3rXd), SIZE_VARIANTS(0xb8400000),
432 kFmtRegR, 4, 0, kFmtRegXOrSp, 9, 5, kFmtBitBlt, 20, 12,
433 kFmtUnused, -1, -1, IS_TERTIARY_OP | REG_DEF0_USE1 | IS_LOAD,
434 "ldur", "!0r, [!1X, #!2d]", kFixupNone),
435 ENCODING_MAP(WIDE(kA64Ldxr2rX), SIZE_VARIANTS(0x885f7c00),
436 kFmtRegR, 4, 0, kFmtRegXOrSp, 9, 5, kFmtUnused, -1, -1,
437 kFmtUnused, -1, -1, IS_BINARY_OP | REG_DEF0_USE1 | IS_LOADX,
438 "ldxr", "!0r, [!1X]", kFixupNone),
439 ENCODING_MAP(WIDE(kA64Ldaxr2rX), SIZE_VARIANTS(0x885ffc00),
440 kFmtRegR, 4, 0, kFmtRegXOrSp, 9, 5, kFmtUnused, -1, -1,
441 kFmtUnused, -1, -1, IS_BINARY_OP | REG_DEF0_USE1 | IS_LOADX,
442 "ldaxr", "!0r, [!1X]", kFixupNone),
443 ENCODING_MAP(WIDE(kA64Lsl3rrr), SF_VARIANTS(0x1ac02000),
444 kFmtRegR, 4, 0, kFmtRegR, 9, 5, kFmtRegR, 20, 16,
445 kFmtUnused, -1, -1, IS_TERTIARY_OP | REG_DEF0_USE12,
446 "lsl", "!0r, !1r, !2r", kFixupNone),
447 ENCODING_MAP(WIDE(kA64Lsr3rrd), CUSTOM_VARIANTS(0x53007c00, 0xd340fc00),
448 kFmtRegR, 4, 0, kFmtRegR, 9, 5, kFmtBitBlt, 21, 16,
449 kFmtUnused, -1, -1, IS_TERTIARY_OP | REG_DEF0_USE1,
450 "lsr", "!0r, !1r, #!2d", kFixupNone),
451 ENCODING_MAP(WIDE(kA64Lsr3rrr), SF_VARIANTS(0x1ac02400),
452 kFmtRegR, 4, 0, kFmtRegR, 9, 5, kFmtRegR, 20, 16,
453 kFmtUnused, -1, -1, IS_TERTIARY_OP | REG_DEF0_USE12,
454 "lsr", "!0r, !1r, !2r", kFixupNone),
455 ENCODING_MAP(WIDE(kA64Madd4rrrr), SF_VARIANTS(0x1b000000),
456 kFmtRegR, 4, 0, kFmtRegR, 9, 5, kFmtRegR, 20, 16,
457 kFmtRegR, 14, 10, IS_QUAD_OP | REG_DEF0_USE123 | NEEDS_FIXUP,
458 "madd", "!0r, !1r, !2r, !3r", kFixupA53Erratum835769),
459 ENCODING_MAP(WIDE(kA64Movk3rdM), SF_VARIANTS(0x72800000),
460 kFmtRegR, 4, 0, kFmtBitBlt, 20, 5, kFmtBitBlt, 22, 21,
461 kFmtUnused, -1, -1, IS_TERTIARY_OP | REG_DEF0_USE0,
462 "movk", "!0r, #!1d!2M", kFixupNone),
463 ENCODING_MAP(WIDE(kA64Movn3rdM), SF_VARIANTS(0x12800000),
464 kFmtRegR, 4, 0, kFmtBitBlt, 20, 5, kFmtBitBlt, 22, 21,
465 kFmtUnused, -1, -1, IS_TERTIARY_OP | REG_DEF0,
466 "movn", "!0r, #!1d!2M", kFixupNone),
467 ENCODING_MAP(WIDE(kA64Movz3rdM), SF_VARIANTS(0x52800000),
468 kFmtRegR, 4, 0, kFmtBitBlt, 20, 5, kFmtBitBlt, 22, 21,
469 kFmtUnused, -1, -1, IS_TERTIARY_OP | REG_DEF0,
470 "movz", "!0r, #!1d!2M", kFixupNone),
471 ENCODING_MAP(WIDE(kA64Mov2rr), SF_VARIANTS(0x2a0003e0),
472 kFmtRegR, 4, 0, kFmtRegR, 20, 16, kFmtUnused, -1, -1,
473 kFmtUnused, -1, -1, IS_BINARY_OP | REG_DEF0_USE1 | IS_MOVE,
474 "mov", "!0r, !1r", kFixupNone),
475 ENCODING_MAP(WIDE(kA64Mvn2rr), SF_VARIANTS(0x2a2003e0),
476 kFmtRegR, 4, 0, kFmtRegR, 20, 16, kFmtUnused, -1, -1,
477 kFmtUnused, -1, -1, IS_BINARY_OP | REG_DEF0_USE1,
478 "mvn", "!0r, !1r", kFixupNone),
479 ENCODING_MAP(WIDE(kA64Mul3rrr), SF_VARIANTS(0x1b007c00),
480 kFmtRegR, 4, 0, kFmtRegR, 9, 5, kFmtRegR, 20, 16,
481 kFmtUnused, -1, -1, IS_TERTIARY_OP | REG_DEF0_USE12,
482 "mul", "!0r, !1r, !2r", kFixupNone),
483 ENCODING_MAP(WIDE(kA64Msub4rrrr), SF_VARIANTS(0x1b008000),
484 kFmtRegR, 4, 0, kFmtRegR, 9, 5, kFmtRegR, 20, 16,
485 kFmtRegR, 14, 10, IS_QUAD_OP | REG_DEF0_USE123 | NEEDS_FIXUP,
486 "msub", "!0r, !1r, !2r, !3r", kFixupA53Erratum835769),
487 ENCODING_MAP(WIDE(kA64Neg3rro), SF_VARIANTS(0x4b0003e0),
488 kFmtRegR, 4, 0, kFmtRegR, 20, 16, kFmtShift, -1, -1,
489 kFmtUnused, -1, -1, IS_TERTIARY_OP | REG_DEF0_USE1,
490 "neg", "!0r, !1r!2o", kFixupNone),
491 ENCODING_MAP(kA64Nop0, NO_VARIANTS(0xd503201f),
492 kFmtUnused, -1, -1, kFmtUnused, -1, -1, kFmtUnused, -1, -1,
493 kFmtUnused, -1, -1, NO_OPERAND,
494 "nop", "", kFixupNone),
495 ENCODING_MAP(WIDE(kA64Orr3Rrl), SF_VARIANTS(0x32000000),
496 kFmtRegROrSp, 4, 0, kFmtRegR, 9, 5, kFmtBitBlt, 22, 10,
497 kFmtUnused, -1, -1, IS_TERTIARY_OP | REG_DEF0_USE1,
498 "orr", "!0R, !1r, #!2l", kFixupNone),
499 ENCODING_MAP(WIDE(kA64Orr4rrro), SF_VARIANTS(0x2a000000),
500 kFmtRegR, 4, 0, kFmtRegR, 9, 5, kFmtRegR, 20, 16,
501 kFmtShift, -1, -1, IS_QUAD_OP | REG_DEF0_USE12,
502 "orr", "!0r, !1r, !2r!3o", kFixupNone),
503 ENCODING_MAP(kA64Ret, NO_VARIANTS(0xd65f03c0),
504 kFmtUnused, -1, -1, kFmtUnused, -1, -1, kFmtUnused, -1, -1,
505 kFmtUnused, -1, -1, NO_OPERAND | IS_BRANCH,
506 "ret", "", kFixupNone),
507 ENCODING_MAP(WIDE(kA64Rbit2rr), SF_VARIANTS(0x5ac00000),
508 kFmtRegR, 4, 0, kFmtRegR, 9, 5, kFmtUnused, -1, -1,
509 kFmtUnused, -1, -1, IS_BINARY_OP | REG_DEF0_USE1,
510 "rbit", "!0r, !1r", kFixupNone),
511 ENCODING_MAP(WIDE(kA64Rev2rr), CUSTOM_VARIANTS(0x5ac00800, 0xdac00c00),
512 kFmtRegR, 4, 0, kFmtRegR, 9, 5, kFmtUnused, -1, -1,
513 kFmtUnused, -1, -1, IS_BINARY_OP | REG_DEF0_USE1,
514 "rev", "!0r, !1r", kFixupNone),
515 ENCODING_MAP(WIDE(kA64Rev162rr), SF_VARIANTS(0x5ac00400),
516 kFmtRegR, 4, 0, kFmtRegR, 9, 5, kFmtUnused, -1, -1,
517 kFmtUnused, -1, -1, IS_BINARY_OP | REG_DEF0_USE1,
518 "rev16", "!0r, !1r", kFixupNone),
519 ENCODING_MAP(WIDE(kA64Ror3rrr), SF_VARIANTS(0x1ac02c00),
520 kFmtRegR, 4, 0, kFmtRegR, 9, 5, kFmtRegR, 20, 16,
521 kFmtUnused, -1, -1, IS_TERTIARY_OP | REG_DEF0_USE12,
522 "ror", "!0r, !1r, !2r", kFixupNone),
523 ENCODING_MAP(WIDE(kA64Sbc3rrr), SF_VARIANTS(0x5a000000),
524 kFmtRegR, 4, 0, kFmtRegR, 9, 5, kFmtRegR, 20, 16,
525 kFmtUnused, -1, -1, IS_TERTIARY_OP | REG_DEF0_USE12 | USES_CCODES,
526 "sbc", "!0r, !1r, !2r", kFixupNone),
527 ENCODING_MAP(WIDE(kA64Sbfm4rrdd), SF_N_VARIANTS(0x13000000),
528 kFmtRegR, 4, 0, kFmtRegR, 9, 5, kFmtBitBlt, 21, 16,
529 kFmtBitBlt, 15, 10, IS_QUAD_OP | REG_DEF0_USE1,
530 "sbfm", "!0r, !1r, #!2d, #!3d", kFixupNone),
531 ENCODING_MAP(WIDE(kA64Scvtf2fw), FLOAT_VARIANTS(0x1e220000),
532 kFmtRegF, 4, 0, kFmtRegW, 9, 5, kFmtUnused, -1, -1,
533 kFmtUnused, -1, -1, IS_BINARY_OP | REG_DEF0_USE1,
534 "scvtf", "!0f, !1w", kFixupNone),
535 ENCODING_MAP(WIDE(kA64Scvtf2fx), FLOAT_VARIANTS(0x9e220000),
536 kFmtRegF, 4, 0, kFmtRegX, 9, 5, kFmtUnused, -1, -1,
537 kFmtUnused, -1, -1, IS_BINARY_OP | REG_DEF0_USE1,
538 "scvtf", "!0f, !1x", kFixupNone),
539 ENCODING_MAP(WIDE(kA64Sdiv3rrr), SF_VARIANTS(0x1ac00c00),
540 kFmtRegR, 4, 0, kFmtRegR, 9, 5, kFmtRegR, 20, 16,
541 kFmtUnused, -1, -1, IS_TERTIARY_OP | REG_DEF0_USE12,
542 "sdiv", "!0r, !1r, !2r", kFixupNone),
543 ENCODING_MAP(kA64Smull3xww, NO_VARIANTS(0x9b207c00),
544 kFmtRegX, 4, 0, kFmtRegW, 9, 5, kFmtRegW, 20, 16,
545 kFmtUnused, -1, -1, IS_TERTIARY_OP | REG_DEF0_USE12,
546 "smull", "!0x, !1w, !2w", kFixupNone),
547 ENCODING_MAP(kA64Smulh3xxx, NO_VARIANTS(0x9b407c00),
548 kFmtRegX, 4, 0, kFmtRegX, 9, 5, kFmtRegX, 20, 16,
549 kFmtUnused, -1, -1, IS_TERTIARY_OP | REG_DEF0_USE12,
550 "smulh", "!0x, !1x, !2x", kFixupNone),
551 ENCODING_MAP(WIDE(kA64Stp4ffXD), CUSTOM_VARIANTS(0x2d000000, 0x6d000000),
552 kFmtRegF, 4, 0, kFmtRegF, 14, 10, kFmtRegXOrSp, 9, 5,
553 kFmtBitBlt, 21, 15, IS_QUAD_OP | REG_USE012 | IS_STORE_OFF,
554 "stp", "!0f, !1f, [!2X, #!3D]", kFixupNone),
555 ENCODING_MAP(WIDE(kA64Stp4rrXD), SF_VARIANTS(0x29000000),
556 kFmtRegR, 4, 0, kFmtRegR, 14, 10, kFmtRegXOrSp, 9, 5,
557 kFmtBitBlt, 21, 15, IS_QUAD_OP | REG_USE012 | IS_STORE_OFF,
558 "stp", "!0r, !1r, [!2X, #!3D]", kFixupNone),
559 ENCODING_MAP(WIDE(kA64StpPost4rrXD), CUSTOM_VARIANTS(0x28800000, 0xa8800000),
560 kFmtRegR, 4, 0, kFmtRegR, 14, 10, kFmtRegXOrSp, 9, 5,
561 kFmtBitBlt, 21, 15, IS_QUAD_OP | REG_DEF2 | REG_USE012 | IS_STORE,
562 "stp", "!0r, !1r, [!2X], #!3D", kFixupNone),
563 ENCODING_MAP(WIDE(kA64StpPre4ffXD), CUSTOM_VARIANTS(0x2d800000, 0x6d800000),
564 kFmtRegF, 4, 0, kFmtRegF, 14, 10, kFmtRegXOrSp, 9, 5,
565 kFmtBitBlt, 21, 15, IS_QUAD_OP | REG_DEF2 | REG_USE012 | IS_STORE,
566 "stp", "!0f, !1f, [!2X, #!3D]!!", kFixupNone),
567 ENCODING_MAP(WIDE(kA64StpPre4rrXD), CUSTOM_VARIANTS(0x29800000, 0xa9800000),
568 kFmtRegR, 4, 0, kFmtRegR, 14, 10, kFmtRegXOrSp, 9, 5,
569 kFmtBitBlt, 21, 15, IS_QUAD_OP | REG_DEF2 | REG_USE012 | IS_STORE,
570 "stp", "!0r, !1r, [!2X, #!3D]!!", kFixupNone),
571 ENCODING_MAP(WIDE(kA64Str3fXD), CUSTOM_VARIANTS(0xbd000000, 0xfd000000),
572 kFmtRegF, 4, 0, kFmtRegXOrSp, 9, 5, kFmtBitBlt, 21, 10,
573 kFmtUnused, -1, -1, IS_TERTIARY_OP | REG_USE01 | IS_STORE_OFF,
574 "str", "!0f, [!1X, #!2D]", kFixupNone),
575 ENCODING_MAP(WIDE(kA64Str4fXxG), CUSTOM_VARIANTS(0xbc206800, 0xfc206800),
576 kFmtRegF, 4, 0, kFmtRegXOrSp, 9, 5, kFmtRegX, 20, 16,
577 kFmtBitBlt, 12, 12, IS_QUAD_OP | REG_USE012 | IS_STORE,
578 "str", "!0f, [!1X, !2x!3G]", kFixupNone),
579 ENCODING_MAP(WIDE(kA64Str3rXD), SIZE_VARIANTS(0xb9000000),
580 kFmtRegR, 4, 0, kFmtRegXOrSp, 9, 5, kFmtBitBlt, 21, 10,
581 kFmtUnused, -1, -1, IS_TERTIARY_OP | REG_USE01 | IS_STORE_OFF,
582 "str", "!0r, [!1X, #!2D]", kFixupNone),
583 ENCODING_MAP(WIDE(kA64Str4rXxG), SIZE_VARIANTS(0xb8206800),
584 kFmtRegR, 4, 0, kFmtRegXOrSp, 9, 5, kFmtRegX, 20, 16,
585 kFmtBitBlt, 12, 12, IS_QUAD_OP | REG_USE012 | IS_STORE,
586 "str", "!0r, [!1X, !2x!3G]", kFixupNone),
587 ENCODING_MAP(kA64Strb3wXd, NO_VARIANTS(0x39000000),
588 kFmtRegW, 4, 0, kFmtRegXOrSp, 9, 5, kFmtBitBlt, 21, 10,
589 kFmtUnused, -1, -1, IS_TERTIARY_OP | REG_USE01 | IS_STORE_OFF,
590 "strb", "!0w, [!1X, #!2d]", kFixupNone),
591 ENCODING_MAP(kA64Strb3wXx, NO_VARIANTS(0x38206800),
592 kFmtRegW, 4, 0, kFmtRegXOrSp, 9, 5, kFmtRegX, 20, 16,
593 kFmtUnused, -1, -1, IS_TERTIARY_OP | REG_USE012 | IS_STORE,
594 "strb", "!0w, [!1X, !2x]", kFixupNone),
595 ENCODING_MAP(kA64Strh3wXF, NO_VARIANTS(0x79000000),
596 kFmtRegW, 4, 0, kFmtRegXOrSp, 9, 5, kFmtBitBlt, 21, 10,
597 kFmtUnused, -1, -1, IS_TERTIARY_OP | REG_USE01 | IS_STORE_OFF,
598 "strh", "!0w, [!1X, #!2F]", kFixupNone),
599 ENCODING_MAP(kA64Strh4wXxd, NO_VARIANTS(0x78206800),
600 kFmtRegW, 4, 0, kFmtRegXOrSp, 9, 5, kFmtRegX, 20, 16,
601 kFmtBitBlt, 12, 12, IS_QUAD_OP | REG_USE012 | IS_STORE,
602 "strh", "!0w, [!1X, !2x, lsl #!3d]", kFixupNone),
603 ENCODING_MAP(WIDE(kA64StrPost3rXd), SIZE_VARIANTS(0xb8000400),
604 kFmtRegR, 4, 0, kFmtRegXOrSp, 9, 5, kFmtBitBlt, 20, 12,
605 kFmtUnused, -1, -1, IS_TERTIARY_OP | REG_USE01 | REG_DEF1 | IS_STORE,
606 "str", "!0r, [!1X], #!2d", kFixupNone),
607 ENCODING_MAP(WIDE(kA64Stur3fXd), CUSTOM_VARIANTS(0xbc000000, 0xfc000000),
608 kFmtRegF, 4, 0, kFmtRegXOrSp, 9, 5, kFmtBitBlt, 20, 12,
609 kFmtUnused, -1, -1, IS_TERTIARY_OP | REG_USE01 | IS_STORE,
610 "stur", "!0f, [!1X, #!2d]", kFixupNone),
611 ENCODING_MAP(WIDE(kA64Stur3rXd), SIZE_VARIANTS(0xb8000000),
612 kFmtRegR, 4, 0, kFmtRegXOrSp, 9, 5, kFmtBitBlt, 20, 12,
613 kFmtUnused, -1, -1, IS_TERTIARY_OP | REG_USE01 | IS_STORE,
614 "stur", "!0r, [!1X, #!2d]", kFixupNone),
615 ENCODING_MAP(WIDE(kA64Stxr3wrX), SIZE_VARIANTS(0x88007c00),
616 kFmtRegW, 20, 16, kFmtRegR, 4, 0, kFmtRegXOrSp, 9, 5,
617 kFmtUnused, -1, -1, IS_TERTIARY_OP | REG_DEF0_USE12 | IS_STOREX,
618 "stxr", "!0w, !1r, [!2X]", kFixupNone),
619 ENCODING_MAP(WIDE(kA64Stlxr3wrX), SIZE_VARIANTS(0x8800fc00),
620 kFmtRegW, 20, 16, kFmtRegR, 4, 0, kFmtRegXOrSp, 9, 5,
621 kFmtUnused, -1, -1, IS_TERTIARY_OP | REG_DEF0_USE12 | IS_STOREX,
622 "stlxr", "!0w, !1r, [!2X]", kFixupNone),
623 ENCODING_MAP(WIDE(kA64Sub4RRdT), SF_VARIANTS(0x51000000),
624 kFmtRegROrSp, 4, 0, kFmtRegROrSp, 9, 5, kFmtBitBlt, 21, 10,
625 kFmtBitBlt, 23, 22, IS_QUAD_OP | REG_DEF0_USE1,
626 "sub", "!0R, !1R, #!2d!3T", kFixupNone),
627 ENCODING_MAP(WIDE(kA64Sub4rrro), SF_VARIANTS(0x4b000000),
628 kFmtRegR, 4, 0, kFmtRegR, 9, 5, kFmtRegR, 20, 16,
629 kFmtShift, -1, -1, IS_QUAD_OP | REG_DEF0_USE12,
630 "sub", "!0r, !1r, !2r!3o", kFixupNone),
631 ENCODING_MAP(WIDE(kA64Sub4RRre), SF_VARIANTS(0x4b200000),
632 kFmtRegROrSp, 4, 0, kFmtRegROrSp, 9, 5, kFmtRegR, 20, 16,
633 kFmtExtend, -1, -1, IS_QUAD_OP | REG_DEF0_USE12,
634 "sub", "!0r, !1r, !2r!3e", kFixupNone),
635 ENCODING_MAP(WIDE(kA64Subs3rRd), SF_VARIANTS(0x71000000),
636 kFmtRegR, 4, 0, kFmtRegROrSp, 9, 5, kFmtBitBlt, 21, 10,
637 kFmtUnused, -1, -1, IS_TERTIARY_OP | REG_DEF0_USE1 | SETS_CCODES,
638 "subs", "!0r, !1R, #!2d", kFixupNone),
639 ENCODING_MAP(WIDE(kA64Tst2rl), SF_VARIANTS(0x7200001f),
640 kFmtRegR, 9, 5, kFmtBitBlt, 22, 10, kFmtUnused, -1, -1,
641 kFmtUnused, -1, -1, IS_TERTIARY_OP | REG_USE0 | SETS_CCODES,
642 "tst", "!0r, !1l", kFixupNone),
643 ENCODING_MAP(WIDE(kA64Tst3rro), SF_VARIANTS(0x6a00001f),
644 kFmtRegR, 9, 5, kFmtRegR, 20, 16, kFmtShift, -1, -1,
645 kFmtUnused, -1, -1, IS_TERTIARY_OP | REG_USE01 | SETS_CCODES,
646 "tst", "!0r, !1r!2o", kFixupNone),
647 // NOTE: Tbz/Tbnz does not require SETS_CCODES, but it may be replaced by some other LIRs
648 // which require SETS_CCODES in the fix-up stage.
649 ENCODING_MAP(WIDE(kA64Tbnz3rht), CUSTOM_VARIANTS(0x37000000, 0x37000000),
650 kFmtRegR, 4, 0, kFmtImm6Shift, -1, -1, kFmtBitBlt, 18, 5, kFmtUnused, -1, -1,
651 IS_TERTIARY_OP | REG_USE0 | IS_BRANCH | NEEDS_FIXUP | SETS_CCODES,
652 "tbnz", "!0r, #!1h, !2t", kFixupTBxZ),
653 ENCODING_MAP(WIDE(kA64Tbz3rht), CUSTOM_VARIANTS(0x36000000, 0x36000000),
654 kFmtRegR, 4, 0, kFmtImm6Shift, -1, -1, kFmtBitBlt, 18, 5, kFmtUnused, -1, -1,
655 IS_TERTIARY_OP | REG_USE0 | IS_BRANCH | NEEDS_FIXUP | SETS_CCODES,
656 "tbz", "!0r, #!1h, !2t", kFixupTBxZ),
657 ENCODING_MAP(WIDE(kA64Ubfm4rrdd), SF_N_VARIANTS(0x53000000),
658 kFmtRegR, 4, 0, kFmtRegR, 9, 5, kFmtBitBlt, 21, 16,
659 kFmtBitBlt, 15, 10, IS_QUAD_OP | REG_DEF0_USE1,
660 "ubfm", "!0r, !1r, !2d, !3d", kFixupNone),
661 };
662
663 // new_lir replaces orig_lir in the pcrel_fixup list.
ReplaceFixup(LIR * prev_lir,LIR * orig_lir,LIR * new_lir)664 void Arm64Mir2Lir::ReplaceFixup(LIR* prev_lir, LIR* orig_lir, LIR* new_lir) {
665 new_lir->u.a.pcrel_next = orig_lir->u.a.pcrel_next;
666 if (UNLIKELY(prev_lir == nullptr)) {
667 first_fixup_ = new_lir;
668 } else {
669 prev_lir->u.a.pcrel_next = new_lir;
670 }
671 orig_lir->flags.fixup = kFixupNone;
672 }
673
674 // new_lir is inserted before orig_lir in the pcrel_fixup list.
InsertFixupBefore(LIR * prev_lir,LIR * orig_lir,LIR * new_lir)675 void Arm64Mir2Lir::InsertFixupBefore(LIR* prev_lir, LIR* orig_lir, LIR* new_lir) {
676 new_lir->u.a.pcrel_next = orig_lir;
677 if (UNLIKELY(prev_lir == nullptr)) {
678 first_fixup_ = new_lir;
679 } else {
680 DCHECK(prev_lir->u.a.pcrel_next == orig_lir);
681 prev_lir->u.a.pcrel_next = new_lir;
682 }
683 }
684
685 /* Nop, used for aligning code. Nop is an alias for hint #0. */
686 #define PADDING_NOP (UINT32_C(0xd503201f))
687
EncodeLIRs(uint8_t * write_pos,LIR * lir)688 uint8_t* Arm64Mir2Lir::EncodeLIRs(uint8_t* write_pos, LIR* lir) {
689 uint8_t* const write_buffer = write_pos;
690 for (; lir != nullptr; lir = NEXT_LIR(lir)) {
691 lir->offset = (write_pos - write_buffer);
692 bool opcode_is_wide = IS_WIDE(lir->opcode);
693 A64Opcode opcode = UNWIDE(lir->opcode);
694
695 if (UNLIKELY(IsPseudoLirOp(opcode))) {
696 continue;
697 }
698
699 if (LIKELY(!lir->flags.is_nop)) {
700 const A64EncodingMap *encoder = &EncodingMap[opcode];
701
702 // Select the right variant of the skeleton.
703 uint32_t bits = opcode_is_wide ? encoder->xskeleton : encoder->wskeleton;
704 DCHECK(!opcode_is_wide || IS_WIDE(encoder->opcode));
705
706 for (int i = 0; i < 4; i++) {
707 A64EncodingKind kind = encoder->field_loc[i].kind;
708 uint32_t operand = lir->operands[i];
709 uint32_t value;
710
711 if (LIKELY(static_cast<unsigned>(kind) <= kFmtBitBlt)) {
712 // Note: this will handle kFmtReg* and kFmtBitBlt.
713
714 if (static_cast<unsigned>(kind) < kFmtBitBlt) {
715 bool is_zero = A64_REG_IS_ZR(operand);
716
717 if (kIsDebugBuild && (kFailOnSizeError || kReportSizeError)) {
718 // Register usage checks: First establish register usage requirements based on the
719 // format in `kind'.
720 bool want_float = false; // Want a float (rather than core) register.
721 bool want_64_bit = false; // Want a 64-bit (rather than 32-bit) register.
722 bool want_var_size = true; // Want register with variable size (kFmtReg{R,F}).
723 bool want_zero = false; // Want the zero (rather than sp) register.
724 switch (kind) {
725 case kFmtRegX:
726 want_64_bit = true;
727 FALLTHROUGH_INTENDED;
728 case kFmtRegW:
729 want_var_size = false;
730 FALLTHROUGH_INTENDED;
731 case kFmtRegR:
732 want_zero = true;
733 break;
734 case kFmtRegXOrSp:
735 want_64_bit = true;
736 FALLTHROUGH_INTENDED;
737 case kFmtRegWOrSp:
738 want_var_size = false;
739 break;
740 case kFmtRegROrSp:
741 break;
742 case kFmtRegD:
743 want_64_bit = true;
744 FALLTHROUGH_INTENDED;
745 case kFmtRegS:
746 want_var_size = false;
747 FALLTHROUGH_INTENDED;
748 case kFmtRegF:
749 want_float = true;
750 break;
751 default:
752 LOG(FATAL) << "Bad fmt for arg n. " << i << " of " << encoder->name
753 << " (" << kind << ")";
754 break;
755 }
756
757 // want_var_size == true means kind == kFmtReg{R,F}. In these two cases, we want
758 // the register size to be coherent with the instruction width.
759 if (want_var_size) {
760 want_64_bit = opcode_is_wide;
761 }
762
763 // Now check that the requirements are satisfied.
764 RegStorage reg(operand | RegStorage::kValid);
765 const char *expected = nullptr;
766 if (want_float) {
767 if (!reg.IsFloat()) {
768 expected = "float register";
769 } else if (reg.IsDouble() != want_64_bit) {
770 expected = (want_64_bit) ? "double register" : "single register";
771 }
772 } else {
773 if (reg.IsFloat()) {
774 expected = "core register";
775 } else if (reg.Is64Bit() != want_64_bit) {
776 expected = (want_64_bit) ? "x-register" : "w-register";
777 } else if (A64_REGSTORAGE_IS_SP_OR_ZR(reg) && is_zero != want_zero) {
778 expected = (want_zero) ? "zero-register" : "sp-register";
779 }
780 }
781
782 // Fail, if `expected' contains an unsatisfied requirement.
783 if (expected != nullptr) {
784 LOG(WARNING) << "Method: " << PrettyMethod(cu_->method_idx, *cu_->dex_file)
785 << " @ 0x" << std::hex << lir->dalvik_offset;
786 if (kFailOnSizeError) {
787 LOG(FATAL) << "Bad argument n. " << i << " of " << encoder->name
788 << "(" << UNWIDE(encoder->opcode) << ", " << encoder->fmt << ")"
789 << ". Expected " << expected << ", got 0x" << std::hex << operand;
790 } else {
791 LOG(WARNING) << "Bad argument n. " << i << " of " << encoder->name
792 << ". Expected " << expected << ", got 0x" << std::hex << operand;
793 }
794 }
795 }
796
797 // In the lines below, we rely on (operand & 0x1f) == 31 to be true for register sp
798 // and zr. This means that these two registers do not need any special treatment, as
799 // their bottom 5 bits are correctly set to 31 == 0b11111, which is the right
800 // value for encoding both sp and zr.
801 static_assert((rxzr & 0x1f) == 0x1f, "rzr register number must be 31");
802 static_assert((rsp & 0x1f) == 0x1f, "rsp register number must be 31");
803 }
804
805 value = (operand << encoder->field_loc[i].start) &
806 ((1 << (encoder->field_loc[i].end + 1)) - 1);
807 bits |= value;
808 } else {
809 switch (kind) {
810 case kFmtSkip:
811 break; // Nothing to do, but continue to next.
812 case kFmtUnused:
813 i = 4; // Done, break out of the enclosing loop.
814 break;
815 case kFmtShift:
816 // Intentional fallthrough.
817 case kFmtExtend:
818 DCHECK_EQ((operand & (1 << 6)) == 0, kind == kFmtShift);
819 value = (operand & 0x3f) << 10;
820 value |= ((operand & 0x1c0) >> 6) << 21;
821 bits |= value;
822 break;
823 case kFmtImm21:
824 value = (operand & 0x3) << 29;
825 value |= ((operand & 0x1ffffc) >> 2) << 5;
826 bits |= value;
827 break;
828 case kFmtImm6Shift:
829 value = (operand & 0x1f) << 19;
830 value |= ((operand & 0x20) >> 5) << 31;
831 bits |= value;
832 break;
833 default:
834 LOG(FATAL) << "Bad fmt for arg. " << i << " in " << encoder->name
835 << " (" << kind << ")";
836 }
837 }
838 }
839
840 DCHECK_EQ(encoder->size, 4);
841 write_pos[0] = (bits & 0xff);
842 write_pos[1] = ((bits >> 8) & 0xff);
843 write_pos[2] = ((bits >> 16) & 0xff);
844 write_pos[3] = ((bits >> 24) & 0xff);
845 write_pos += 4;
846 }
847 }
848
849 return write_pos;
850 }
851
852 // Align data offset on 8 byte boundary: it will only contain double-word items, as word immediates
853 // are better set directly from the code (they will require no more than 2 instructions).
854 #define ALIGNED_DATA_OFFSET(offset) (((offset) + 0x7) & ~0x7)
855
856 /*
857 * Get the LIR which emits the instruction preceding the given LIR.
858 * Returns nullptr, if no previous emitting insn found.
859 */
GetPrevEmittingLIR(LIR * lir)860 static LIR* GetPrevEmittingLIR(LIR* lir) {
861 DCHECK(lir != nullptr);
862 LIR* prev_lir = lir->prev;
863 while ((prev_lir != nullptr) &&
864 (prev_lir->flags.is_nop || Mir2Lir::IsPseudoLirOp(prev_lir->opcode))) {
865 prev_lir = prev_lir->prev;
866 }
867 return prev_lir;
868 }
869
870 // Assemble the LIR into binary instruction format.
AssembleLIR()871 void Arm64Mir2Lir::AssembleLIR() {
872 LIR* lir;
873 LIR* prev_lir;
874 cu_->NewTimingSplit("Assemble");
875 int assembler_retries = 0;
876 CodeOffset starting_offset = LinkFixupInsns(first_lir_insn_, last_lir_insn_, 0);
877 data_offset_ = ALIGNED_DATA_OFFSET(starting_offset);
878 int32_t offset_adjustment;
879 AssignDataOffsets();
880
881 /*
882 * Note: generation must be 1 on first pass (to distinguish from initialized state of 0
883 * for non-visited nodes). Start at zero here, and bit will be flipped to 1 on entry to the loop.
884 */
885 int generation = 0;
886 while (true) {
887 offset_adjustment = 0;
888 AssemblerStatus res = kSuccess; // Assume success
889 generation ^= 1;
890 // Note: nodes requiring possible fixup linked in ascending order.
891 lir = first_fixup_;
892 prev_lir = nullptr;
893 while (lir != nullptr) {
894 // NOTE: Any new non-pc_rel instructions inserted due to retry must be explicitly encoded at
895 // the time of insertion. Note that inserted instructions don't need use/def flags, but do
896 // need size and pc-rel status properly updated.
897 lir->offset += offset_adjustment;
898 // During pass, allows us to tell whether a node has been updated with offset_adjustment yet.
899 lir->flags.generation = generation;
900 switch (static_cast<FixupKind>(lir->flags.fixup)) {
901 case kFixupLabel:
902 case kFixupNone:
903 case kFixupVLoad:
904 break;
905 case kFixupT1Branch: {
906 LIR *target_lir = lir->target;
907 DCHECK(target_lir);
908 CodeOffset pc = lir->offset;
909 CodeOffset target = target_lir->offset +
910 ((target_lir->flags.generation == lir->flags.generation) ? 0 : offset_adjustment);
911 int32_t delta = target - pc;
912 DCHECK_EQ(delta & 0x3, 0);
913 if (!IS_SIGNED_IMM26(delta >> 2)) {
914 LOG(FATAL) << "Invalid jump range in kFixupT1Branch";
915 }
916 lir->operands[0] = delta >> 2;
917 if (!(cu_->disable_opt & (1 << kSafeOptimizations)) && lir->operands[0] == 1) {
918 // Useless branch.
919 offset_adjustment -= lir->flags.size;
920 lir->flags.is_nop = true;
921 // Don't unlink - just set to do-nothing.
922 lir->flags.fixup = kFixupNone;
923 res = kRetryAll;
924 }
925 break;
926 }
927 case kFixupLoad:
928 case kFixupCBxZ:
929 case kFixupCondBranch: {
930 LIR *target_lir = lir->target;
931 DCHECK(target_lir);
932 CodeOffset pc = lir->offset;
933 CodeOffset target = target_lir->offset +
934 ((target_lir->flags.generation == lir->flags.generation) ? 0 : offset_adjustment);
935 int32_t delta = target - pc;
936 DCHECK_EQ(delta & 0x3, 0);
937 if (!IS_SIGNED_IMM19(delta >> 2)) {
938 LOG(FATAL) << "Invalid jump range in kFixupLoad";
939 }
940 lir->operands[1] = delta >> 2;
941 break;
942 }
943 case kFixupTBxZ: {
944 int16_t opcode = lir->opcode;
945 RegStorage reg(lir->operands[0] | RegStorage::kValid);
946 int32_t imm = lir->operands[1];
947 DCHECK_EQ(IS_WIDE(opcode), reg.Is64Bit());
948 DCHECK_LT(imm, 64);
949 if (imm >= 32) {
950 DCHECK(IS_WIDE(opcode));
951 } else if (kIsDebugBuild && IS_WIDE(opcode)) {
952 // "tbz/tbnz x0, #imm(<32)" is the same with "tbz/tbnz w0, #imm(<32)", but GCC/oatdump
953 // will disassemble it as "tbz/tbnz w0, #imm(<32)". So unwide the LIR to make the
954 // compiler log behave the same with those disassembler in debug build.
955 // This will also affect tst instruction if it need to be replaced, but there is no
956 // performance difference between "tst Xt" and "tst Wt".
957 lir->opcode = UNWIDE(opcode);
958 lir->operands[0] = As32BitReg(reg).GetReg();
959 }
960
961 // Fix-up branch offset.
962 LIR *target_lir = lir->target;
963 DCHECK(target_lir);
964 CodeOffset pc = lir->offset;
965 CodeOffset target = target_lir->offset +
966 ((target_lir->flags.generation == lir->flags.generation) ? 0 : offset_adjustment);
967 int32_t delta = target - pc;
968 DCHECK_EQ(delta & 0x3, 0);
969 // Check if branch offset can be encoded in tbz/tbnz.
970 if (!IS_SIGNED_IMM14(delta >> 2)) {
971 DexOffset dalvik_offset = lir->dalvik_offset;
972 LIR* targetLIR = lir->target;
973 // "tbz/tbnz Rt, #imm, label" -> "tst Rt, #(1<<imm)".
974 offset_adjustment -= lir->flags.size;
975 int32_t encodedImm = EncodeLogicalImmediate(IS_WIDE(opcode), 1 << lir->operands[1]);
976 DCHECK_NE(encodedImm, -1);
977 lir->opcode = IS_WIDE(opcode) ? WIDE(kA64Tst2rl) : kA64Tst2rl;
978 lir->operands[1] = encodedImm;
979 lir->target = nullptr;
980 lir->flags.fixup = EncodingMap[kA64Tst2rl].fixup;
981 lir->flags.size = EncodingMap[kA64Tst2rl].size;
982 offset_adjustment += lir->flags.size;
983 // Insert "beq/bneq label".
984 opcode = UNWIDE(opcode);
985 DCHECK(opcode == kA64Tbz3rht || opcode == kA64Tbnz3rht);
986 LIR* new_lir = RawLIR(dalvik_offset, kA64B2ct,
987 opcode == kA64Tbz3rht ? kArmCondEq : kArmCondNe, 0, 0, 0, 0, targetLIR);
988 InsertLIRAfter(lir, new_lir);
989 new_lir->offset = lir->offset + lir->flags.size;
990 new_lir->flags.generation = generation;
991 new_lir->flags.fixup = EncodingMap[kA64B2ct].fixup;
992 new_lir->flags.size = EncodingMap[kA64B2ct].size;
993 offset_adjustment += new_lir->flags.size;
994 // lir no longer pcrel, unlink and link in new_lir.
995 ReplaceFixup(prev_lir, lir, new_lir);
996 prev_lir = new_lir; // Continue with the new instruction.
997 lir = new_lir->u.a.pcrel_next;
998 res = kRetryAll;
999 continue;
1000 }
1001 lir->operands[2] = delta >> 2;
1002 break;
1003 }
1004 case kFixupAdr: {
1005 LIR* target_lir = lir->target;
1006 int32_t delta;
1007 if (target_lir) {
1008 CodeOffset target_offs = ((target_lir->flags.generation == lir->flags.generation) ?
1009 0 : offset_adjustment) + target_lir->offset;
1010 delta = target_offs - lir->offset;
1011 } else if (lir->operands[2] >= 0) {
1012 const EmbeddedData* tab = UnwrapPointer<EmbeddedData>(lir->operands[2]);
1013 delta = tab->offset + offset_adjustment - lir->offset;
1014 } else {
1015 // No fixup: this usage allows to retrieve the current PC.
1016 delta = lir->operands[1];
1017 }
1018 if (!IS_SIGNED_IMM21(delta)) {
1019 LOG(FATAL) << "Jump range above 1MB in kFixupAdr";
1020 }
1021 lir->operands[1] = delta;
1022 break;
1023 }
1024 case kFixupA53Erratum835769:
1025 // Avoid emitting code that could trigger Cortex A53's erratum 835769.
1026 // This fixup should be carried out for all multiply-accumulate instructions: madd, msub,
1027 // smaddl, smsubl, umaddl and umsubl.
1028 if (cu_->compiler_driver->GetInstructionSetFeatures()->AsArm64InstructionSetFeatures()
1029 ->NeedFixCortexA53_835769()) {
1030 // Check that this is a 64-bit multiply-accumulate.
1031 if (IS_WIDE(lir->opcode)) {
1032 LIR* prev_insn = GetPrevEmittingLIR(lir);
1033 if (prev_insn == nullptr) {
1034 break;
1035 }
1036 uint64_t prev_insn_flags = EncodingMap[UNWIDE(prev_insn->opcode)].flags;
1037 // Check that the instruction preceding the multiply-accumulate is a load or store.
1038 if ((prev_insn_flags & IS_LOAD) != 0 || (prev_insn_flags & IS_STORE) != 0) {
1039 // insert a NOP between the load/store and the multiply-accumulate.
1040 LIR* new_lir = RawLIR(lir->dalvik_offset, kA64Nop0, 0, 0, 0, 0, 0, nullptr);
1041 new_lir->offset = lir->offset;
1042 new_lir->flags.fixup = kFixupNone;
1043 new_lir->flags.size = EncodingMap[kA64Nop0].size;
1044 InsertLIRBefore(lir, new_lir);
1045 lir->offset += new_lir->flags.size;
1046 offset_adjustment += new_lir->flags.size;
1047 res = kRetryAll;
1048 }
1049 }
1050 }
1051 break;
1052 default:
1053 LOG(FATAL) << "Unexpected case " << lir->flags.fixup;
1054 }
1055 prev_lir = lir;
1056 lir = lir->u.a.pcrel_next;
1057 }
1058
1059 if (res == kSuccess) {
1060 DCHECK_EQ(offset_adjustment, 0);
1061 break;
1062 } else {
1063 assembler_retries++;
1064 if (assembler_retries > MAX_ASSEMBLER_RETRIES) {
1065 CodegenDump();
1066 LOG(FATAL) << "Assembler error - too many retries";
1067 }
1068 starting_offset += offset_adjustment;
1069 data_offset_ = ALIGNED_DATA_OFFSET(starting_offset);
1070 AssignDataOffsets();
1071 }
1072 }
1073
1074 // Build the CodeBuffer.
1075 DCHECK_LE(data_offset_, total_size_);
1076 code_buffer_.reserve(total_size_);
1077 code_buffer_.resize(starting_offset);
1078 uint8_t* write_pos = &code_buffer_[0];
1079 write_pos = EncodeLIRs(write_pos, first_lir_insn_);
1080 DCHECK_EQ(static_cast<CodeOffset>(write_pos - &code_buffer_[0]), starting_offset);
1081
1082 DCHECK_EQ(data_offset_, ALIGNED_DATA_OFFSET(code_buffer_.size()));
1083
1084 // Install literals
1085 InstallLiteralPools();
1086
1087 // Install switch tables
1088 InstallSwitchTables();
1089
1090 // Install fill array data
1091 InstallFillArrayData();
1092
1093 // Create the mapping table and native offset to reference map.
1094 cu_->NewTimingSplit("PcMappingTable");
1095 CreateMappingTables();
1096
1097 cu_->NewTimingSplit("GcMap");
1098 CreateNativeGcMap();
1099 }
1100
GetInsnSize(LIR * lir)1101 size_t Arm64Mir2Lir::GetInsnSize(LIR* lir) {
1102 A64Opcode opcode = UNWIDE(lir->opcode);
1103 DCHECK(!IsPseudoLirOp(opcode));
1104 return EncodingMap[opcode].size;
1105 }
1106
1107 // Encode instruction bit pattern and assign offsets.
LinkFixupInsns(LIR * head_lir,LIR * tail_lir,uint32_t offset)1108 uint32_t Arm64Mir2Lir::LinkFixupInsns(LIR* head_lir, LIR* tail_lir, uint32_t offset) {
1109 LIR* end_lir = tail_lir->next;
1110
1111 LIR* last_fixup = nullptr;
1112 for (LIR* lir = head_lir; lir != end_lir; lir = NEXT_LIR(lir)) {
1113 A64Opcode opcode = UNWIDE(lir->opcode);
1114 if (!lir->flags.is_nop) {
1115 if (lir->flags.fixup != kFixupNone) {
1116 if (!IsPseudoLirOp(opcode)) {
1117 lir->flags.size = EncodingMap[opcode].size;
1118 lir->flags.fixup = EncodingMap[opcode].fixup;
1119 } else {
1120 DCHECK_NE(static_cast<int>(opcode), kPseudoPseudoAlign4);
1121 lir->flags.size = 0;
1122 lir->flags.fixup = kFixupLabel;
1123 }
1124 // Link into the fixup chain.
1125 lir->flags.use_def_invalid = true;
1126 lir->u.a.pcrel_next = nullptr;
1127 if (first_fixup_ == nullptr) {
1128 first_fixup_ = lir;
1129 } else {
1130 last_fixup->u.a.pcrel_next = lir;
1131 }
1132 last_fixup = lir;
1133 lir->offset = offset;
1134 }
1135 offset += lir->flags.size;
1136 }
1137 }
1138 return offset;
1139 }
1140
AssignDataOffsets()1141 void Arm64Mir2Lir::AssignDataOffsets() {
1142 /* Set up offsets for literals */
1143 CodeOffset offset = data_offset_;
1144
1145 offset = AssignLiteralOffset(offset);
1146
1147 offset = AssignSwitchTablesOffset(offset);
1148
1149 total_size_ = AssignFillArrayDataOffset(offset);
1150 }
1151
1152 } // namespace art
1153