1 /*
2  * Copyright (C) 2011 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 #include "codegen_arm64.h"
18 
19 #include <inttypes.h>
20 
21 #include <string>
22 #include <sstream>
23 
24 #include "backend_arm64.h"
25 #include "base/logging.h"
26 #include "dex/mir_graph.h"
27 #include "dex/quick/mir_to_lir-inl.h"
28 #include "dex/reg_storage_eq.h"
29 
30 namespace art {
31 
32 static constexpr RegStorage core_regs_arr[] =
33     {rs_w0, rs_w1, rs_w2, rs_w3, rs_w4, rs_w5, rs_w6, rs_w7,
34      rs_w8, rs_w9, rs_w10, rs_w11, rs_w12, rs_w13, rs_w14, rs_w15,
35      rs_w16, rs_w17, rs_w18, rs_w19, rs_w20, rs_w21, rs_w22, rs_w23,
36      rs_w24, rs_w25, rs_w26, rs_w27, rs_w28, rs_w29, rs_w30, rs_w31,
37      rs_wzr};
38 static constexpr RegStorage core64_regs_arr[] =
39     {rs_x0, rs_x1, rs_x2, rs_x3, rs_x4, rs_x5, rs_x6, rs_x7,
40      rs_x8, rs_x9, rs_x10, rs_x11, rs_x12, rs_x13, rs_x14, rs_x15,
41      rs_x16, rs_x17, rs_x18, rs_x19, rs_x20, rs_x21, rs_x22, rs_x23,
42      rs_x24, rs_x25, rs_x26, rs_x27, rs_x28, rs_x29, rs_x30, rs_x31,
43      rs_xzr};
44 static constexpr RegStorage sp_regs_arr[] =
45     {rs_f0, rs_f1, rs_f2, rs_f3, rs_f4, rs_f5, rs_f6, rs_f7,
46      rs_f8, rs_f9, rs_f10, rs_f11, rs_f12, rs_f13, rs_f14, rs_f15,
47      rs_f16, rs_f17, rs_f18, rs_f19, rs_f20, rs_f21, rs_f22, rs_f23,
48      rs_f24, rs_f25, rs_f26, rs_f27, rs_f28, rs_f29, rs_f30, rs_f31};
49 static constexpr RegStorage dp_regs_arr[] =
50     {rs_d0, rs_d1, rs_d2, rs_d3, rs_d4, rs_d5, rs_d6, rs_d7,
51      rs_d8, rs_d9, rs_d10, rs_d11, rs_d12, rs_d13, rs_d14, rs_d15,
52      rs_d16, rs_d17, rs_d18, rs_d19, rs_d20, rs_d21, rs_d22, rs_d23,
53      rs_d24, rs_d25, rs_d26, rs_d27, rs_d28, rs_d29, rs_d30, rs_d31};
54 // Note: we are not able to call to C function since rs_xSELF is a special register need to be
55 // preserved but would be scratched by native functions follow aapcs64.
56 static constexpr RegStorage reserved_regs_arr[] = {rs_wSELF, rs_wsp, rs_wLR, rs_wzr};
57 static constexpr RegStorage reserved64_regs_arr[] = {rs_xSELF, rs_sp, rs_xLR, rs_xzr};
58 
59 static constexpr RegStorage core_temps_arr[] =
60     {rs_w0, rs_w1, rs_w2, rs_w3, rs_w4, rs_w5, rs_w6, rs_w7,
61      rs_w8, rs_w9, rs_w10, rs_w11, rs_w12, rs_w13, rs_w14, rs_w15, rs_w16,
62      rs_w17};
63 static constexpr RegStorage core64_temps_arr[] =
64     {rs_x0, rs_x1, rs_x2, rs_x3, rs_x4, rs_x5, rs_x6, rs_x7,
65      rs_x8, rs_x9, rs_x10, rs_x11, rs_x12, rs_x13, rs_x14, rs_x15, rs_x16,
66      rs_x17};
67 static constexpr RegStorage sp_temps_arr[] =
68     {rs_f0, rs_f1, rs_f2, rs_f3, rs_f4, rs_f5, rs_f6, rs_f7,
69      rs_f16, rs_f17, rs_f18, rs_f19, rs_f20, rs_f21, rs_f22, rs_f23,
70      rs_f24, rs_f25, rs_f26, rs_f27, rs_f28, rs_f29, rs_f30, rs_f31};
71 static constexpr RegStorage dp_temps_arr[] =
72     {rs_d0, rs_d1, rs_d2, rs_d3, rs_d4, rs_d5, rs_d6, rs_d7,
73      rs_d16, rs_d17, rs_d18, rs_d19, rs_d20, rs_d21, rs_d22, rs_d23,
74      rs_d24, rs_d25, rs_d26, rs_d27, rs_d28, rs_d29, rs_d30, rs_d31};
75 
76 static constexpr ArrayRef<const RegStorage> core_regs(core_regs_arr);
77 static constexpr ArrayRef<const RegStorage> core64_regs(core64_regs_arr);
78 static constexpr ArrayRef<const RegStorage> sp_regs(sp_regs_arr);
79 static constexpr ArrayRef<const RegStorage> dp_regs(dp_regs_arr);
80 static constexpr ArrayRef<const RegStorage> reserved_regs(reserved_regs_arr);
81 static constexpr ArrayRef<const RegStorage> reserved64_regs(reserved64_regs_arr);
82 static constexpr ArrayRef<const RegStorage> core_temps(core_temps_arr);
83 static constexpr ArrayRef<const RegStorage> core64_temps(core64_temps_arr);
84 static constexpr ArrayRef<const RegStorage> sp_temps(sp_temps_arr);
85 static constexpr ArrayRef<const RegStorage> dp_temps(dp_temps_arr);
86 
LocCReturn()87 RegLocation Arm64Mir2Lir::LocCReturn() {
88   return a64_loc_c_return;
89 }
90 
LocCReturnRef()91 RegLocation Arm64Mir2Lir::LocCReturnRef() {
92   return a64_loc_c_return_ref;
93 }
94 
LocCReturnWide()95 RegLocation Arm64Mir2Lir::LocCReturnWide() {
96   return a64_loc_c_return_wide;
97 }
98 
LocCReturnFloat()99 RegLocation Arm64Mir2Lir::LocCReturnFloat() {
100   return a64_loc_c_return_float;
101 }
102 
LocCReturnDouble()103 RegLocation Arm64Mir2Lir::LocCReturnDouble() {
104   return a64_loc_c_return_double;
105 }
106 
107 // Return a target-dependent special register.
TargetReg(SpecialTargetRegister reg)108 RegStorage Arm64Mir2Lir::TargetReg(SpecialTargetRegister reg) {
109   RegStorage res_reg = RegStorage::InvalidReg();
110   switch (reg) {
111     case kSelf: res_reg = rs_wSELF; break;
112     case kSuspend: res_reg = RegStorage::InvalidReg(); break;
113     case kLr: res_reg =  rs_wLR; break;
114     case kPc: res_reg = RegStorage::InvalidReg(); break;
115     case kSp: res_reg =  rs_wsp; break;
116     case kArg0: res_reg = rs_w0; break;
117     case kArg1: res_reg = rs_w1; break;
118     case kArg2: res_reg = rs_w2; break;
119     case kArg3: res_reg = rs_w3; break;
120     case kArg4: res_reg = rs_w4; break;
121     case kArg5: res_reg = rs_w5; break;
122     case kArg6: res_reg = rs_w6; break;
123     case kArg7: res_reg = rs_w7; break;
124     case kFArg0: res_reg = rs_f0; break;
125     case kFArg1: res_reg = rs_f1; break;
126     case kFArg2: res_reg = rs_f2; break;
127     case kFArg3: res_reg = rs_f3; break;
128     case kFArg4: res_reg = rs_f4; break;
129     case kFArg5: res_reg = rs_f5; break;
130     case kFArg6: res_reg = rs_f6; break;
131     case kFArg7: res_reg = rs_f7; break;
132     case kRet0: res_reg = rs_w0; break;
133     case kRet1: res_reg = rs_w1; break;
134     case kInvokeTgt: res_reg = rs_wLR; break;
135     case kHiddenArg: res_reg = rs_wIP1; break;
136     case kHiddenFpArg: res_reg = RegStorage::InvalidReg(); break;
137     case kCount: res_reg = RegStorage::InvalidReg(); break;
138     default: res_reg = RegStorage::InvalidReg();
139   }
140   return res_reg;
141 }
142 
143 /*
144  * Decode the register id. This routine makes assumptions on the encoding made by RegStorage.
145  */
GetRegMaskCommon(const RegStorage & reg) const146 ResourceMask Arm64Mir2Lir::GetRegMaskCommon(const RegStorage& reg) const {
147   // TODO(Arm64): this function depends too much on the internal RegStorage encoding. Refactor.
148 
149   // Check if the shape mask is zero (i.e. invalid).
150   if (UNLIKELY(reg == rs_wzr || reg == rs_xzr)) {
151     // The zero register is not a true register. It is just an immediate zero.
152     return kEncodeNone;
153   }
154 
155   return ResourceMask::Bit(
156       // FP register starts at bit position 32.
157       (reg.IsFloat() ? kA64FPReg0 : 0) + reg.GetRegNum());
158 }
159 
GetPCUseDefEncoding() const160 ResourceMask Arm64Mir2Lir::GetPCUseDefEncoding() const {
161   // Note: On arm64, we are not able to set pc except branch instructions, which is regarded as a
162   //       kind of barrier. All other instructions only use pc, which has no dependency between any
163   //       of them. So it is fine to just return kEncodeNone here.
164   return kEncodeNone;
165 }
166 
167 // Arm64 specific setup.  TODO: inline?:
SetupTargetResourceMasks(LIR * lir,uint64_t flags,ResourceMask * use_mask,ResourceMask * def_mask)168 void Arm64Mir2Lir::SetupTargetResourceMasks(LIR* lir, uint64_t flags,
169                                             ResourceMask* use_mask, ResourceMask* def_mask) {
170   DCHECK_EQ(cu_->instruction_set, kArm64);
171   DCHECK(!lir->flags.use_def_invalid);
172 
173   // Note: REG_USE_PC is ignored, the reason is the same with what we do in GetPCUseDefEncoding().
174   // These flags are somewhat uncommon - bypass if we can.
175   if ((flags & (REG_DEF_SP | REG_USE_SP | REG_DEF_LR)) != 0) {
176     if (flags & REG_DEF_SP) {
177       def_mask->SetBit(kA64RegSP);
178     }
179 
180     if (flags & REG_USE_SP) {
181       use_mask->SetBit(kA64RegSP);
182     }
183 
184     if (flags & REG_DEF_LR) {
185       def_mask->SetBit(kA64RegLR);
186     }
187   }
188 }
189 
ArmConditionEncoding(ConditionCode ccode)190 ArmConditionCode Arm64Mir2Lir::ArmConditionEncoding(ConditionCode ccode) {
191   ArmConditionCode res;
192   switch (ccode) {
193     case kCondEq: res = kArmCondEq; break;
194     case kCondNe: res = kArmCondNe; break;
195     case kCondCs: res = kArmCondCs; break;
196     case kCondCc: res = kArmCondCc; break;
197     case kCondUlt: res = kArmCondCc; break;
198     case kCondUge: res = kArmCondCs; break;
199     case kCondMi: res = kArmCondMi; break;
200     case kCondPl: res = kArmCondPl; break;
201     case kCondVs: res = kArmCondVs; break;
202     case kCondVc: res = kArmCondVc; break;
203     case kCondHi: res = kArmCondHi; break;
204     case kCondLs: res = kArmCondLs; break;
205     case kCondGe: res = kArmCondGe; break;
206     case kCondLt: res = kArmCondLt; break;
207     case kCondGt: res = kArmCondGt; break;
208     case kCondLe: res = kArmCondLe; break;
209     case kCondAl: res = kArmCondAl; break;
210     case kCondNv: res = kArmCondNv; break;
211     default:
212       LOG(FATAL) << "Bad condition code " << ccode;
213       res = static_cast<ArmConditionCode>(0);  // Quiet gcc
214   }
215   return res;
216 }
217 
218 static const char *shift_names[4] = {
219   "lsl",
220   "lsr",
221   "asr",
222   "ror"
223 };
224 
225 static const char* extend_names[8] = {
226   "uxtb",
227   "uxth",
228   "uxtw",
229   "uxtx",
230   "sxtb",
231   "sxth",
232   "sxtw",
233   "sxtx",
234 };
235 
236 /* Decode and print a register extension (e.g. ", uxtb #1") */
DecodeRegExtendOrShift(int operand,char * buf,size_t buf_size)237 static void DecodeRegExtendOrShift(int operand, char *buf, size_t buf_size) {
238   if ((operand & (1 << 6)) == 0) {
239     const char *shift_name = shift_names[(operand >> 7) & 0x3];
240     int amount = operand & 0x3f;
241     snprintf(buf, buf_size, ", %s #%d", shift_name, amount);
242   } else {
243     const char *extend_name = extend_names[(operand >> 3) & 0x7];
244     int amount = operand & 0x7;
245     if (amount == 0) {
246       snprintf(buf, buf_size, ", %s", extend_name);
247     } else {
248       snprintf(buf, buf_size, ", %s #%d", extend_name, amount);
249     }
250   }
251 }
252 
bit_mask(unsigned width)253 static uint64_t bit_mask(unsigned width) {
254   DCHECK_LE(width, 64U);
255   return (width == 64) ? static_cast<uint64_t>(-1) : ((UINT64_C(1) << (width)) - UINT64_C(1));
256 }
257 
RotateRight(uint64_t value,unsigned rotate,unsigned width)258 static uint64_t RotateRight(uint64_t value, unsigned rotate, unsigned width) {
259   DCHECK_LE(width, 64U);
260   rotate &= 63;
261   value = value & bit_mask(width);
262   return ((value & bit_mask(rotate)) << (width - rotate)) | (value >> rotate);
263 }
264 
RepeatBitsAcrossReg(bool is_wide,uint64_t value,unsigned width)265 static uint64_t RepeatBitsAcrossReg(bool is_wide, uint64_t value, unsigned width) {
266   unsigned i;
267   unsigned reg_size = (is_wide) ? 64 : 32;
268   uint64_t result = value & bit_mask(width);
269   for (i = width; i < reg_size; i *= 2) {
270     result |= (result << i);
271   }
272   DCHECK_EQ(i, reg_size);
273   return result;
274 }
275 
276 /**
277  * @brief Decode an immediate in the form required by logical instructions.
278  *
279  * @param is_wide Whether @p value encodes a 64-bit (as opposed to 32-bit) immediate.
280  * @param value The encoded logical immediates that is to be decoded.
281  * @return The decoded logical immediate.
282  * @note This is the inverse of Arm64Mir2Lir::EncodeLogicalImmediate().
283  */
DecodeLogicalImmediate(bool is_wide,int value)284 uint64_t Arm64Mir2Lir::DecodeLogicalImmediate(bool is_wide, int value) {
285   unsigned n     = (value >> 12) & 0x01;
286   unsigned imm_r = (value >>  6) & 0x3f;
287   unsigned imm_s = (value >>  0) & 0x3f;
288 
289   // An integer is constructed from the n, imm_s and imm_r bits according to
290   // the following table:
291   //
292   // N   imms immr  size S             R
293   // 1 ssssss rrrrrr 64  UInt(ssssss) UInt(rrrrrr)
294   // 0 0sssss xrrrrr 32  UInt(sssss)  UInt(rrrrr)
295   // 0 10ssss xxrrrr 16  UInt(ssss)   UInt(rrrr)
296   // 0 110sss xxxrrr 8   UInt(sss)    UInt(rrr)
297   // 0 1110ss xxxxrr 4   UInt(ss)     UInt(rr)
298   // 0 11110s xxxxxr 2   UInt(s)      UInt(r)
299   // (s bits must not be all set)
300   //
301   // A pattern is constructed of size bits, where the least significant S+1
302   // bits are set. The pattern is rotated right by R, and repeated across a
303   // 32 or 64-bit value, depending on destination register width.
304 
305   if (n == 1) {
306     DCHECK_NE(imm_s, 0x3fU);
307     uint64_t bits = bit_mask(imm_s + 1);
308     return RotateRight(bits, imm_r, 64);
309   } else {
310     DCHECK_NE((imm_s >> 1), 0x1fU);
311     for (unsigned width = 0x20; width >= 0x2; width >>= 1) {
312       if ((imm_s & width) == 0) {
313         unsigned mask = (unsigned)(width - 1);
314         DCHECK_NE((imm_s & mask), mask);
315         uint64_t bits = bit_mask((imm_s & mask) + 1);
316         return RepeatBitsAcrossReg(is_wide, RotateRight(bits, imm_r & mask, width), width);
317       }
318     }
319   }
320   return 0;
321 }
322 
323 /**
324  * @brief Decode an 8-bit single point number encoded with EncodeImmSingle().
325  */
DecodeImmSingle(uint8_t small_float)326 static float DecodeImmSingle(uint8_t small_float) {
327   int mantissa = (small_float & 0x0f) + 0x10;
328   int sign = ((small_float & 0x80) == 0) ? 1 : -1;
329   float signed_mantissa = static_cast<float>(sign*mantissa);
330   int exponent = (((small_float >> 4) & 0x7) + 4) & 0x7;
331   return signed_mantissa*static_cast<float>(1 << exponent)*0.0078125f;
332 }
333 
334 static const char* cc_names[] = {"eq", "ne", "cs", "cc", "mi", "pl", "vs", "vc",
335                                  "hi", "ls", "ge", "lt", "gt", "le", "al", "nv"};
336 /*
337  * Interpret a format string and build a string no longer than size
338  * See format key in assemble_arm64.cc.
339  */
BuildInsnString(const char * fmt,LIR * lir,unsigned char * base_addr)340 std::string Arm64Mir2Lir::BuildInsnString(const char* fmt, LIR* lir, unsigned char* base_addr) {
341   std::string buf;
342   const char* fmt_end = &fmt[strlen(fmt)];
343   char tbuf[256];
344   const char* name;
345   char nc;
346   while (fmt < fmt_end) {
347     int operand;
348     if (*fmt == '!') {
349       fmt++;
350       DCHECK_LT(fmt, fmt_end);
351       nc = *fmt++;
352       if (nc == '!') {
353         strcpy(tbuf, "!");
354       } else {
355          DCHECK_LT(fmt, fmt_end);
356          DCHECK_LT(static_cast<unsigned>(nc-'0'), 4U);
357          operand = lir->operands[nc-'0'];
358          switch (*fmt++) {
359            case 'e':  {
360                // Omit ", uxtw #0" in strings like "add w0, w1, w3, uxtw #0" and
361                // ", uxtx #0" in strings like "add x0, x1, x3, uxtx #0"
362                int omittable = ((IS_WIDE(lir->opcode)) ? EncodeExtend(kA64Uxtw, 0) :
363                                 EncodeExtend(kA64Uxtw, 0));
364                if (LIKELY(operand == omittable)) {
365                  strcpy(tbuf, "");
366                } else {
367                  DecodeRegExtendOrShift(operand, tbuf, arraysize(tbuf));
368                }
369              }
370              break;
371            case 'o':
372              // Omit ", lsl #0"
373              if (LIKELY(operand == EncodeShift(kA64Lsl, 0))) {
374                strcpy(tbuf, "");
375              } else {
376                DecodeRegExtendOrShift(operand, tbuf, arraysize(tbuf));
377              }
378              break;
379            case 'B':
380              switch (operand) {
381                case kSY:
382                  name = "sy";
383                  break;
384                case kST:
385                  name = "st";
386                  break;
387                case kISH:
388                  name = "ish";
389                  break;
390                case kISHST:
391                  name = "ishst";
392                  break;
393                case kNSH:
394                  name = "nsh";
395                  break;
396                case kNSHST:
397                  name = "shst";
398                  break;
399                default:
400                  name = "DecodeError2";
401                  break;
402              }
403              strcpy(tbuf, name);
404              break;
405            case 's':
406              snprintf(tbuf, arraysize(tbuf), "s%d", operand & RegStorage::kRegNumMask);
407              break;
408            case 'S':
409              snprintf(tbuf, arraysize(tbuf), "d%d", operand & RegStorage::kRegNumMask);
410              break;
411            case 'f':
412              snprintf(tbuf, arraysize(tbuf), "%c%d", (IS_WIDE(lir->opcode)) ? 'd' : 's',
413                       operand & RegStorage::kRegNumMask);
414              break;
415            case 'l': {
416                bool is_wide = IS_WIDE(lir->opcode);
417                uint64_t imm = DecodeLogicalImmediate(is_wide, operand);
418                snprintf(tbuf, arraysize(tbuf), "%" PRId64 " (%#" PRIx64 ")", imm, imm);
419              }
420              break;
421            case 'I':
422              snprintf(tbuf, arraysize(tbuf), "%f", DecodeImmSingle(operand));
423              break;
424            case 'M':
425              if (LIKELY(operand == 0))
426                strcpy(tbuf, "");
427              else
428                snprintf(tbuf, arraysize(tbuf), ", lsl #%d", 16*operand);
429              break;
430            case 'd':
431              snprintf(tbuf, arraysize(tbuf), "%d", operand);
432              break;
433            case 'w':
434              if (LIKELY(operand != rwzr))
435                snprintf(tbuf, arraysize(tbuf), "w%d", operand & RegStorage::kRegNumMask);
436              else
437                strcpy(tbuf, "wzr");
438              break;
439            case 'W':
440              if (LIKELY(operand != rwsp))
441                snprintf(tbuf, arraysize(tbuf), "w%d", operand & RegStorage::kRegNumMask);
442              else
443                strcpy(tbuf, "wsp");
444              break;
445            case 'x':
446              if (LIKELY(operand != rxzr))
447                snprintf(tbuf, arraysize(tbuf), "x%d", operand & RegStorage::kRegNumMask);
448              else
449                strcpy(tbuf, "xzr");
450              break;
451            case 'X':
452              if (LIKELY(operand != rsp))
453                snprintf(tbuf, arraysize(tbuf), "x%d", operand & RegStorage::kRegNumMask);
454              else
455                strcpy(tbuf, "sp");
456              break;
457            case 'D':
458              snprintf(tbuf, arraysize(tbuf), "%d", operand*((IS_WIDE(lir->opcode)) ? 8 : 4));
459              break;
460            case 'E':
461              snprintf(tbuf, arraysize(tbuf), "%d", operand*4);
462              break;
463            case 'F':
464              snprintf(tbuf, arraysize(tbuf), "%d", operand*2);
465              break;
466            case 'G':
467              if (LIKELY(operand == 0))
468                strcpy(tbuf, "");
469              else
470                strcpy(tbuf, (IS_WIDE(lir->opcode)) ? ", lsl #3" : ", lsl #2");
471              break;
472            case 'c':
473              strcpy(tbuf, cc_names[operand]);
474              break;
475            case 't':
476              snprintf(tbuf, arraysize(tbuf), "0x%08" PRIxPTR " (L%p)",
477                  reinterpret_cast<uintptr_t>(base_addr) + lir->offset + (operand << 2),
478                  lir->target);
479              break;
480            case 'r': {
481                bool is_wide = IS_WIDE(lir->opcode);
482                if (LIKELY(operand != rwzr && operand != rxzr)) {
483                  snprintf(tbuf, arraysize(tbuf), "%c%d", (is_wide) ? 'x' : 'w',
484                           operand & RegStorage::kRegNumMask);
485                } else {
486                  strcpy(tbuf, (is_wide) ? "xzr" : "wzr");
487                }
488              }
489              break;
490            case 'R': {
491                bool is_wide = IS_WIDE(lir->opcode);
492                if (LIKELY(operand != rwsp && operand != rsp)) {
493                  snprintf(tbuf, arraysize(tbuf), "%c%d", (is_wide) ? 'x' : 'w',
494                           operand & RegStorage::kRegNumMask);
495                } else {
496                  strcpy(tbuf, (is_wide) ? "sp" : "wsp");
497                }
498              }
499              break;
500            case 'p':
501              snprintf(tbuf, arraysize(tbuf), ".+%d (addr %#" PRIxPTR ")", 4*operand,
502                       reinterpret_cast<uintptr_t>(base_addr) + lir->offset + 4*operand);
503              break;
504            case 'T':
505              if (LIKELY(operand == 0))
506                strcpy(tbuf, "");
507              else if (operand == 1)
508                strcpy(tbuf, ", lsl #12");
509              else
510                strcpy(tbuf, ", DecodeError3");
511              break;
512            case 'h':
513              snprintf(tbuf, arraysize(tbuf), "%d", operand);
514              break;
515            default:
516              strcpy(tbuf, "DecodeError1");
517              break;
518         }
519         buf += tbuf;
520       }
521     } else {
522        buf += *fmt++;
523     }
524   }
525   // Dump thread offset.
526   std::string fmt_str = GetTargetInstFmt(lir->opcode);
527   if (std::string::npos != fmt_str.find(", [!1X, #!2") && rxSELF == lir->operands[1] &&
528       std::string::npos != buf.find(", [")) {
529     int offset = lir->operands[2];
530     if (std::string::npos != fmt_str.find("#!2d")) {
531     } else if (std::string::npos != fmt_str.find("#!2D")) {
532       offset *= (IS_WIDE(lir->opcode)) ? 8 : 4;
533     } else if (std::string::npos != fmt_str.find("#!2F")) {
534       offset *= 2;
535     } else {
536       LOG(FATAL) << "Should not reach here";
537     }
538     std::ostringstream tmp_stream;
539     Thread::DumpThreadOffset<8>(tmp_stream, offset);
540     buf += "  ; ";
541     buf += tmp_stream.str();
542   }
543   return buf;
544 }
545 
DumpResourceMask(LIR * arm_lir,const ResourceMask & mask,const char * prefix)546 void Arm64Mir2Lir::DumpResourceMask(LIR* arm_lir, const ResourceMask& mask, const char* prefix) {
547   char buf[256];
548   buf[0] = 0;
549 
550   if (mask.Equals(kEncodeAll)) {
551     strcpy(buf, "all");
552   } else {
553     char num[8];
554     int i;
555 
556     for (i = 0; i < kA64RegEnd; i++) {
557       if (mask.HasBit(i)) {
558         snprintf(num, arraysize(num), "%d ", i);
559         strcat(buf, num);
560       }
561     }
562 
563     if (mask.HasBit(ResourceMask::kCCode)) {
564       strcat(buf, "cc ");
565     }
566     if (mask.HasBit(ResourceMask::kFPStatus)) {
567       strcat(buf, "fpcc ");
568     }
569 
570     /* Memory bits */
571     if (arm_lir && (mask.HasBit(ResourceMask::kDalvikReg))) {
572       snprintf(buf + strlen(buf), arraysize(buf) - strlen(buf), "dr%d%s",
573                DECODE_ALIAS_INFO_REG(arm_lir->flags.alias_info),
574                DECODE_ALIAS_INFO_WIDE(arm_lir->flags.alias_info) ? "(+1)" : "");
575     }
576     if (mask.HasBit(ResourceMask::kLiteral)) {
577       strcat(buf, "lit ");
578     }
579 
580     if (mask.HasBit(ResourceMask::kHeapRef)) {
581       strcat(buf, "heap ");
582     }
583     if (mask.HasBit(ResourceMask::kMustNotAlias)) {
584       strcat(buf, "noalias ");
585     }
586   }
587   if (buf[0]) {
588     LOG(INFO) << prefix << ": " << buf;
589   }
590 }
591 
IsUnconditionalBranch(LIR * lir)592 bool Arm64Mir2Lir::IsUnconditionalBranch(LIR* lir) {
593   return (lir->opcode == kA64B1t);
594 }
595 
RegClassForFieldLoadStore(OpSize size,bool is_volatile)596 RegisterClass Arm64Mir2Lir::RegClassForFieldLoadStore(OpSize size, bool is_volatile) {
597   if (UNLIKELY(is_volatile)) {
598     // On arm64, fp register load/store is atomic only for single bytes.
599     if (size != kSignedByte && size != kUnsignedByte) {
600       return (size == kReference) ? kRefReg : kCoreReg;
601     }
602   }
603   return RegClassBySize(size);
604 }
605 
Arm64Mir2Lir(CompilationUnit * cu,MIRGraph * mir_graph,ArenaAllocator * arena)606 Arm64Mir2Lir::Arm64Mir2Lir(CompilationUnit* cu, MIRGraph* mir_graph, ArenaAllocator* arena)
607     : Mir2Lir(cu, mir_graph, arena),
608       call_method_insns_(arena->Adapter()),
609       dex_cache_access_insns_(arena->Adapter()) {
610   // Sanity check - make sure encoding map lines up.
611   for (int i = 0; i < kA64Last; i++) {
612     DCHECK_EQ(UNWIDE(Arm64Mir2Lir::EncodingMap[i].opcode), i)
613         << "Encoding order for " << Arm64Mir2Lir::EncodingMap[i].name
614         << " is wrong: expecting " << i << ", seeing "
615         << static_cast<int>(Arm64Mir2Lir::EncodingMap[i].opcode);
616   }
617 }
618 
Arm64CodeGenerator(CompilationUnit * const cu,MIRGraph * const mir_graph,ArenaAllocator * const arena)619 Mir2Lir* Arm64CodeGenerator(CompilationUnit* const cu, MIRGraph* const mir_graph,
620                             ArenaAllocator* const arena) {
621   return new Arm64Mir2Lir(cu, mir_graph, arena);
622 }
623 
CompilerInitializeRegAlloc()624 void Arm64Mir2Lir::CompilerInitializeRegAlloc() {
625   reg_pool_.reset(new (arena_) RegisterPool(this, arena_, core_regs, core64_regs, sp_regs, dp_regs,
626                                             reserved_regs, reserved64_regs,
627                                             core_temps, core64_temps, sp_temps, dp_temps));
628 
629   // Target-specific adjustments.
630   // Alias single precision float registers to corresponding double registers.
631   for (RegisterInfo* info : reg_pool_->sp_regs_) {
632     int fp_reg_num = info->GetReg().GetRegNum();
633     RegStorage dp_reg = RegStorage::FloatSolo64(fp_reg_num);
634     RegisterInfo* dp_reg_info = GetRegInfo(dp_reg);
635     // Double precision register's master storage should refer to itself.
636     DCHECK_EQ(dp_reg_info, dp_reg_info->Master());
637     // Redirect single precision's master storage to master.
638     info->SetMaster(dp_reg_info);
639     // Singles should show a single 32-bit mask bit, at first referring to the low half.
640     DCHECK_EQ(info->StorageMask(), 0x1U);
641   }
642 
643   // Alias 32bit W registers to corresponding 64bit X registers.
644   for (RegisterInfo* info : reg_pool_->core_regs_) {
645     int x_reg_num = info->GetReg().GetRegNum();
646     RegStorage x_reg = RegStorage::Solo64(x_reg_num);
647     RegisterInfo* x_reg_info = GetRegInfo(x_reg);
648     // 64bit X register's master storage should refer to itself.
649     DCHECK_EQ(x_reg_info, x_reg_info->Master());
650     // Redirect 32bit W master storage to 64bit X.
651     info->SetMaster(x_reg_info);
652     // 32bit W should show a single 32-bit mask bit, at first referring to the low half.
653     DCHECK_EQ(info->StorageMask(), 0x1U);
654   }
655 
656   // Don't start allocating temps at r0/s0/d0 or you may clobber return regs in early-exit methods.
657   // TODO: adjust when we roll to hard float calling convention.
658   reg_pool_->next_core_reg_ = 2;
659   reg_pool_->next_sp_reg_ = 0;
660   reg_pool_->next_dp_reg_ = 0;
661 }
662 
663 /*
664  * TUNING: is true leaf?  Can't just use METHOD_IS_LEAF to determine as some
665  * instructions might call out to C/assembly helper functions.  Until
666  * machinery is in place, always spill lr.
667  */
668 
AdjustSpillMask()669 void Arm64Mir2Lir::AdjustSpillMask() {
670   core_spill_mask_ |= (1 << rs_xLR.GetRegNum());
671   num_core_spills_++;
672 }
673 
674 /* Clobber all regs that might be used by an external C call */
ClobberCallerSave()675 void Arm64Mir2Lir::ClobberCallerSave() {
676   Clobber(rs_x0);
677   Clobber(rs_x1);
678   Clobber(rs_x2);
679   Clobber(rs_x3);
680   Clobber(rs_x4);
681   Clobber(rs_x5);
682   Clobber(rs_x6);
683   Clobber(rs_x7);
684   Clobber(rs_x8);
685   Clobber(rs_x9);
686   Clobber(rs_x10);
687   Clobber(rs_x11);
688   Clobber(rs_x12);
689   Clobber(rs_x13);
690   Clobber(rs_x14);
691   Clobber(rs_x15);
692   Clobber(rs_x16);
693   Clobber(rs_x17);
694   Clobber(rs_x30);
695 
696   Clobber(rs_f0);
697   Clobber(rs_f1);
698   Clobber(rs_f2);
699   Clobber(rs_f3);
700   Clobber(rs_f4);
701   Clobber(rs_f5);
702   Clobber(rs_f6);
703   Clobber(rs_f7);
704   Clobber(rs_f16);
705   Clobber(rs_f17);
706   Clobber(rs_f18);
707   Clobber(rs_f19);
708   Clobber(rs_f20);
709   Clobber(rs_f21);
710   Clobber(rs_f22);
711   Clobber(rs_f23);
712   Clobber(rs_f24);
713   Clobber(rs_f25);
714   Clobber(rs_f26);
715   Clobber(rs_f27);
716   Clobber(rs_f28);
717   Clobber(rs_f29);
718   Clobber(rs_f30);
719   Clobber(rs_f31);
720 }
721 
GetReturnWideAlt()722 RegLocation Arm64Mir2Lir::GetReturnWideAlt() {
723   RegLocation res = LocCReturnWide();
724   res.reg.SetReg(rx2);
725   res.reg.SetHighReg(rx3);
726   Clobber(rs_x2);
727   Clobber(rs_x3);
728   MarkInUse(rs_x2);
729   MarkInUse(rs_x3);
730   MarkWide(res.reg);
731   return res;
732 }
733 
GetReturnAlt()734 RegLocation Arm64Mir2Lir::GetReturnAlt() {
735   RegLocation res = LocCReturn();
736   res.reg.SetReg(rx1);
737   Clobber(rs_x1);
738   MarkInUse(rs_x1);
739   return res;
740 }
741 
742 /* To be used when explicitly managing register use */
LockCallTemps()743 void Arm64Mir2Lir::LockCallTemps() {
744   // TODO: needs cleanup.
745   LockTemp(rs_x0);
746   LockTemp(rs_x1);
747   LockTemp(rs_x2);
748   LockTemp(rs_x3);
749   LockTemp(rs_x4);
750   LockTemp(rs_x5);
751   LockTemp(rs_x6);
752   LockTemp(rs_x7);
753   LockTemp(rs_f0);
754   LockTemp(rs_f1);
755   LockTemp(rs_f2);
756   LockTemp(rs_f3);
757   LockTemp(rs_f4);
758   LockTemp(rs_f5);
759   LockTemp(rs_f6);
760   LockTemp(rs_f7);
761 }
762 
763 /* To be used when explicitly managing register use */
FreeCallTemps()764 void Arm64Mir2Lir::FreeCallTemps() {
765   // TODO: needs cleanup.
766   FreeTemp(rs_x0);
767   FreeTemp(rs_x1);
768   FreeTemp(rs_x2);
769   FreeTemp(rs_x3);
770   FreeTemp(rs_x4);
771   FreeTemp(rs_x5);
772   FreeTemp(rs_x6);
773   FreeTemp(rs_x7);
774   FreeTemp(rs_f0);
775   FreeTemp(rs_f1);
776   FreeTemp(rs_f2);
777   FreeTemp(rs_f3);
778   FreeTemp(rs_f4);
779   FreeTemp(rs_f5);
780   FreeTemp(rs_f6);
781   FreeTemp(rs_f7);
782   FreeTemp(TargetReg(kHiddenArg));
783 }
784 
LoadHelper(QuickEntrypointEnum trampoline)785 RegStorage Arm64Mir2Lir::LoadHelper(QuickEntrypointEnum trampoline) {
786   // TODO(Arm64): use LoadWordDisp instead.
787   //   e.g. LoadWordDisp(rs_rA64_SELF, offset.Int32Value(), rs_rA64_LR);
788   LoadBaseDisp(rs_xSELF, GetThreadOffset<8>(trampoline).Int32Value(), rs_xLR, k64, kNotVolatile);
789   return rs_xLR;
790 }
791 
CheckSuspendUsingLoad()792 LIR* Arm64Mir2Lir::CheckSuspendUsingLoad() {
793   RegStorage tmp = rs_x0;
794   LoadWordDisp(rs_xSELF, Thread::ThreadSuspendTriggerOffset<8>().Int32Value(), tmp);
795   LIR* load2 = LoadWordDisp(tmp, 0, tmp);
796   return load2;
797 }
798 
GetTargetInstFlags(int opcode)799 uint64_t Arm64Mir2Lir::GetTargetInstFlags(int opcode) {
800   DCHECK(!IsPseudoLirOp(opcode));
801   return Arm64Mir2Lir::EncodingMap[UNWIDE(opcode)].flags;
802 }
803 
GetTargetInstName(int opcode)804 const char* Arm64Mir2Lir::GetTargetInstName(int opcode) {
805   DCHECK(!IsPseudoLirOp(opcode));
806   return Arm64Mir2Lir::EncodingMap[UNWIDE(opcode)].name;
807 }
808 
GetTargetInstFmt(int opcode)809 const char* Arm64Mir2Lir::GetTargetInstFmt(int opcode) {
810   DCHECK(!IsPseudoLirOp(opcode));
811   return Arm64Mir2Lir::EncodingMap[UNWIDE(opcode)].fmt;
812 }
813 
GetNextReg(ShortyArg arg)814 RegStorage Arm64Mir2Lir::InToRegStorageArm64Mapper::GetNextReg(ShortyArg arg) {
815   const RegStorage coreArgMappingToPhysicalReg[] =
816       {rs_x1, rs_x2, rs_x3, rs_x4, rs_x5, rs_x6, rs_x7};
817   const size_t coreArgMappingToPhysicalRegSize = arraysize(coreArgMappingToPhysicalReg);
818   const RegStorage fpArgMappingToPhysicalReg[] =
819       {rs_f0, rs_f1, rs_f2, rs_f3, rs_f4, rs_f5, rs_f6, rs_f7};
820   const size_t fpArgMappingToPhysicalRegSize = arraysize(fpArgMappingToPhysicalReg);
821 
822   RegStorage result = RegStorage::InvalidReg();
823   if (arg.IsFP()) {
824     if (cur_fp_reg_ < fpArgMappingToPhysicalRegSize) {
825       DCHECK(!arg.IsRef());
826       result = fpArgMappingToPhysicalReg[cur_fp_reg_++];
827       if (result.Valid()) {
828         // TODO: switching between widths remains a bit ugly.  Better way?
829         int res_reg = result.GetReg();
830         result = arg.IsWide() ? RegStorage::FloatSolo64(res_reg) : RegStorage::FloatSolo32(res_reg);
831       }
832     }
833   } else {
834     if (cur_core_reg_ < coreArgMappingToPhysicalRegSize) {
835       result = coreArgMappingToPhysicalReg[cur_core_reg_++];
836       if (result.Valid()) {
837         // TODO: switching between widths remains a bit ugly.  Better way?
838         int res_reg = result.GetReg();
839         DCHECK(!(arg.IsWide() && arg.IsRef()));
840         result = (arg.IsWide() || arg.IsRef()) ?
841                  RegStorage::Solo64(res_reg) : RegStorage::Solo32(res_reg);
842       }
843     }
844   }
845   return result;
846 }
847 
InstallLiteralPools()848 void Arm64Mir2Lir::InstallLiteralPools() {
849   patches_.reserve(call_method_insns_.size() + dex_cache_access_insns_.size());
850 
851   // PC-relative calls to methods.
852   for (LIR* p : call_method_insns_) {
853       DCHECK_EQ(p->opcode, kA64Bl1t);
854       uint32_t target_method_idx = p->operands[1];
855       const DexFile* target_dex_file = UnwrapPointer<DexFile>(p->operands[2]);
856       patches_.push_back(LinkerPatch::RelativeCodePatch(p->offset,
857                                                         target_dex_file, target_method_idx));
858   }
859 
860   // PC-relative references to dex cache arrays.
861   for (LIR* p : dex_cache_access_insns_) {
862     auto non_wide = UNWIDE(p->opcode);  // May be a wide load for ArtMethod*.
863     DCHECK(non_wide == kA64Adrp2xd || non_wide == kA64Ldr3rXD) << p->opcode << " " << non_wide;
864     const LIR* adrp = UnwrapPointer<LIR>(p->operands[4]);
865     DCHECK_EQ(adrp->opcode, kA64Adrp2xd);
866     const DexFile* dex_file = UnwrapPointer<DexFile>(adrp->operands[2]);
867     uint32_t offset = adrp->operands[3];
868     DCHECK(!p->flags.is_nop);
869     DCHECK(!adrp->flags.is_nop);
870     patches_.push_back(LinkerPatch::DexCacheArrayPatch(p->offset, dex_file, adrp->offset, offset));
871   }
872 
873   // And do the normal processing.
874   Mir2Lir::InstallLiteralPools();
875 }
876 
GenDalvikArgsBulkCopy(CallInfo *,int,int count)877 int Arm64Mir2Lir::GenDalvikArgsBulkCopy(CallInfo* /*info*/, int /*first*/, int count) {
878   /*
879    * TODO: Improve by adding block copy for large number of arguments.  For now, just
880    * copy a Dalvik vreg at a time.
881    */
882   return count;
883 }
884 
GenMachineSpecificExtendedMethodMIR(BasicBlock * bb,MIR * mir)885 void Arm64Mir2Lir::GenMachineSpecificExtendedMethodMIR(BasicBlock* bb, MIR* mir) {
886   UNUSED(bb);
887   DCHECK(MIR::DecodedInstruction::IsPseudoMirOp(mir->dalvikInsn.opcode));
888   RegLocation rl_src[3];
889   RegLocation rl_dest = mir_graph_->GetBadLoc();
890   rl_src[0] = rl_src[1] = rl_src[2] = mir_graph_->GetBadLoc();
891   ExtendedMIROpcode opcode = static_cast<ExtendedMIROpcode>(mir->dalvikInsn.opcode);
892   switch (opcode) {
893     case kMirOpMaddInt:
894     case kMirOpMsubInt:
895       rl_dest = mir_graph_->GetDest(mir);
896       rl_src[0] = mir_graph_->GetSrc(mir, 0);
897       rl_src[1] = mir_graph_->GetSrc(mir, 1);
898       rl_src[2]= mir_graph_->GetSrc(mir, 2);
899       GenMaddMsubInt(rl_dest, rl_src[0], rl_src[1], rl_src[2], opcode == kMirOpMsubInt);
900       break;
901     case kMirOpMaddLong:
902     case kMirOpMsubLong:
903       rl_dest = mir_graph_->GetDestWide(mir);
904       rl_src[0] = mir_graph_->GetSrcWide(mir, 0);
905       rl_src[1] = mir_graph_->GetSrcWide(mir, 2);
906       rl_src[2] = mir_graph_->GetSrcWide(mir, 4);
907       GenMaddMsubLong(rl_dest, rl_src[0], rl_src[1], rl_src[2], opcode == kMirOpMsubLong);
908       break;
909     default:
910       LOG(FATAL) << "Unexpected opcode: " << static_cast<int>(opcode);
911   }
912 }
913 
914 }  // namespace art
915