target_arm64.cc revision 0955f7e470fb733aef07096536e9fba7c99250aa 
1/*
2 * Copyright (C) 2011 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *      http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17#include "codegen_arm64.h"
18
19#include <inttypes.h>
20
21#include <string>
22
23#include "dex/compiler_internals.h"
24#include "dex/quick/mir_to_lir-inl.h"
25
26namespace art {
27
28// TODO: rework this when c++11 support allows.
29static const RegStorage core_regs_arr[] =
30    {rs_w0, rs_w1, rs_w2, rs_w3, rs_w4, rs_w5, rs_w6, rs_w7,
31     rs_w8, rs_w9, rs_w10, rs_w11, rs_w12, rs_w13, rs_w14, rs_w15,
32     rs_w16, rs_w17, rs_w18, rs_w19, rs_w20, rs_w21, rs_w22, rs_w23,
33     rs_w24, rs_w25, rs_w26, rs_w27, rs_w28, rs_w29, rs_w30, rs_w31,
34     rs_wzr};
35static const RegStorage core64_regs_arr[] =
36    {rs_x0, rs_x1, rs_x2, rs_x3, rs_x4, rs_x5, rs_x6, rs_x7,
37     rs_x8, rs_x9, rs_x10, rs_x11, rs_x12, rs_x13, rs_x14, rs_x15,
38     rs_x16, rs_x17, rs_x18, rs_x19, rs_x20, rs_x21, rs_x22, rs_x23,
39     rs_x24, rs_x25, rs_x26, rs_x27, rs_x28, rs_x29, rs_x30, rs_x31,
40     rs_xzr};
41static const RegStorage sp_regs_arr[] =
42    {rs_f0, rs_f1, rs_f2, rs_f3, rs_f4, rs_f5, rs_f6, rs_f7,
43     rs_f8, rs_f9, rs_f10, rs_f11, rs_f12, rs_f13, rs_f14, rs_f15,
44     rs_f16, rs_f17, rs_f18, rs_f19, rs_f20, rs_f21, rs_f22, rs_f23,
45     rs_f24, rs_f25, rs_f26, rs_f27, rs_f28, rs_f29, rs_f30, rs_f31};
46static const RegStorage dp_regs_arr[] =
47    {rs_d0, rs_d1, rs_d2, rs_d3, rs_d4, rs_d5, rs_d6, rs_d7,
48     rs_d8, rs_d9, rs_d10, rs_d11, rs_d12, rs_d13, rs_d14, rs_d15,
49     rs_d16, rs_d17, rs_d18, rs_d19, rs_d20, rs_d21, rs_d22, rs_d23,
50     rs_d24, rs_d25, rs_d26, rs_d27, rs_d28, rs_d29, rs_d30, rs_d31};
51static const RegStorage reserved_regs_arr[] =
52    {rs_rA32_SUSPEND, rs_rA32_SELF, rs_rA32_SP, rs_rA32_LR, rs_wzr};
53static const RegStorage reserved64_regs_arr[] =
54    {rs_rA64_SUSPEND, rs_rA64_SELF, rs_rA64_SP, rs_rA64_LR, rs_xzr};
55// TUNING: Are there too many temp registers and too less promote target?
56// This definition need to be matched with runtime.cc, quick entry assembly and JNI compiler
57// Note: we are not able to call to C function directly if it un-match C ABI.
58// Currently, rs_rA64_SELF is not a callee save register which does not match C ABI.
59static const RegStorage core_temps_arr[] =
60    {rs_w0, rs_w1, rs_w2, rs_w3, rs_w4, rs_w5, rs_w6, rs_w7,
61     rs_w8, rs_w9, rs_w10, rs_w11, rs_w12, rs_w13, rs_w14, rs_w15, rs_w16,
62     rs_w17};
63static const RegStorage core64_temps_arr[] =
64    {rs_x0, rs_x1, rs_x2, rs_x3, rs_x4, rs_x5, rs_x6, rs_x7,
65     rs_x8, rs_x9, rs_x10, rs_x11, rs_x12, rs_x13, rs_x14, rs_x15, rs_x16,
66     rs_x17};
67static const RegStorage sp_temps_arr[] =
68    {rs_f0, rs_f1, rs_f2, rs_f3, rs_f4, rs_f5, rs_f6, rs_f7,
69     rs_f16, rs_f17, rs_f18, rs_f19, rs_f20, rs_f21, rs_f22, rs_f23,
70     rs_f24, rs_f25, rs_f26, rs_f27, rs_f28, rs_f29, rs_f30, rs_f31};
71static const RegStorage dp_temps_arr[] =
72    {rs_d0, rs_d1, rs_d2, rs_d3, rs_d4, rs_d5, rs_d6, rs_d7,
73     rs_d16, rs_d17, rs_d18, rs_d19, rs_d20, rs_d21, rs_d22, rs_d23,
74     rs_d24, rs_d25, rs_d26, rs_d27, rs_d28, rs_d29, rs_d30, rs_d31};
75
76static const std::vector<RegStorage> core_regs(core_regs_arr,
77    core_regs_arr + arraysize(core_regs_arr));
78static const std::vector<RegStorage> core64_regs(core64_regs_arr,
79    core64_regs_arr + arraysize(core64_regs_arr));
80static const std::vector<RegStorage> sp_regs(sp_regs_arr,
81    sp_regs_arr + arraysize(sp_regs_arr));
82static const std::vector<RegStorage> dp_regs(dp_regs_arr,
83    dp_regs_arr + arraysize(dp_regs_arr));
84static const std::vector<RegStorage> reserved_regs(reserved_regs_arr,
85    reserved_regs_arr + arraysize(reserved_regs_arr));
86static const std::vector<RegStorage> reserved64_regs(reserved64_regs_arr,
87    reserved64_regs_arr + arraysize(reserved64_regs_arr));
88static const std::vector<RegStorage> core_temps(core_temps_arr,
89    core_temps_arr + arraysize(core_temps_arr));
90static const std::vector<RegStorage> core64_temps(core64_temps_arr,
91    core64_temps_arr + arraysize(core64_temps_arr));
92static const std::vector<RegStorage> sp_temps(sp_temps_arr, sp_temps_arr + arraysize(sp_temps_arr));
93static const std::vector<RegStorage> dp_temps(dp_temps_arr, dp_temps_arr + arraysize(dp_temps_arr));
94
95RegLocation Arm64Mir2Lir::LocCReturn() {
96  return arm_loc_c_return;
97}
98
99RegLocation Arm64Mir2Lir::LocCReturnWide() {
100  return arm_loc_c_return_wide;
101}
102
103RegLocation Arm64Mir2Lir::LocCReturnFloat() {
104  return arm_loc_c_return_float;
105}
106
107RegLocation Arm64Mir2Lir::LocCReturnDouble() {
108  return arm_loc_c_return_double;
109}
110
111// Return a target-dependent special register.
112RegStorage Arm64Mir2Lir::TargetReg(SpecialTargetRegister reg) {
113  // TODO(Arm64): this function doesn't work for hard-float ABI.
114  RegStorage res_reg = RegStorage::InvalidReg();
115  switch (reg) {
116    case kSelf: res_reg = rs_rA64_SELF; break;
117    case kSuspend: res_reg = rs_rA64_SUSPEND; break;
118    case kLr: res_reg =  rs_rA64_LR; break;
119    case kPc: res_reg = RegStorage::InvalidReg(); break;
120    case kSp: res_reg =  rs_rA64_SP; break;
121    case kArg0: res_reg = rs_x0; break;
122    case kArg1: res_reg = rs_x1; break;
123    case kArg2: res_reg = rs_x2; break;
124    case kArg3: res_reg = rs_x3; break;
125    case kFArg0: res_reg = rs_f0; break;
126    case kFArg1: res_reg = rs_f1; break;
127    case kFArg2: res_reg = rs_f2; break;
128    case kFArg3: res_reg = rs_f3; break;
129    case kRet0: res_reg = rs_x0; break;
130    case kRet1: res_reg = rs_x0; break;
131    case kInvokeTgt: res_reg = rs_rA64_LR; break;
132    case kHiddenArg: res_reg = rs_x12; break;
133    case kHiddenFpArg: res_reg = RegStorage::InvalidReg(); break;
134    case kCount: res_reg = RegStorage::InvalidReg(); break;
135  }
136  return res_reg;
137}
138
139RegStorage Arm64Mir2Lir::GetArgMappingToPhysicalReg(int arg_num) {
140  return RegStorage::InvalidReg();
141}
142
143/*
144 * Decode the register id. This routine makes assumptions on the encoding made by RegStorage.
145 */
146uint64_t Arm64Mir2Lir::GetRegMaskCommon(RegStorage reg) {
147  // TODO(Arm64): this function depends too much on the internal RegStorage encoding. Refactor.
148
149  int reg_raw = reg.GetRawBits();
150  // Check if the shape mask is zero (i.e. invalid).
151  if (UNLIKELY(reg == rs_wzr || reg == rs_xzr)) {
152    // The zero register is not a true register. It is just an immediate zero.
153    return 0;
154  }
155
156  return UINT64_C(1) << (reg_raw & RegStorage::kRegTypeMask);
157}
158
159uint64_t Arm64Mir2Lir::GetPCUseDefEncoding() {
160  LOG(FATAL) << "Unexpected call to GetPCUseDefEncoding for Arm64";
161  return 0ULL;
162}
163
164// Arm64 specific setup.  TODO: inline?:
165void Arm64Mir2Lir::SetupTargetResourceMasks(LIR* lir, uint64_t flags) {
166  DCHECK_EQ(cu_->instruction_set, kArm64);
167  DCHECK(!lir->flags.use_def_invalid);
168
169  // These flags are somewhat uncommon - bypass if we can.
170  if ((flags & (REG_DEF_SP | REG_USE_SP | REG_DEF_LR)) != 0) {
171    if (flags & REG_DEF_SP) {
172      lir->u.m.def_mask |= ENCODE_ARM_REG_SP;
173    }
174
175    if (flags & REG_USE_SP) {
176      lir->u.m.use_mask |= ENCODE_ARM_REG_SP;
177    }
178
179    if (flags & REG_DEF_LR) {
180      lir->u.m.def_mask |= ENCODE_ARM_REG_LR;
181    }
182  }
183}
184
185ArmConditionCode Arm64Mir2Lir::ArmConditionEncoding(ConditionCode ccode) {
186  ArmConditionCode res;
187  switch (ccode) {
188    case kCondEq: res = kArmCondEq; break;
189    case kCondNe: res = kArmCondNe; break;
190    case kCondCs: res = kArmCondCs; break;
191    case kCondCc: res = kArmCondCc; break;
192    case kCondUlt: res = kArmCondCc; break;
193    case kCondUge: res = kArmCondCs; break;
194    case kCondMi: res = kArmCondMi; break;
195    case kCondPl: res = kArmCondPl; break;
196    case kCondVs: res = kArmCondVs; break;
197    case kCondVc: res = kArmCondVc; break;
198    case kCondHi: res = kArmCondHi; break;
199    case kCondLs: res = kArmCondLs; break;
200    case kCondGe: res = kArmCondGe; break;
201    case kCondLt: res = kArmCondLt; break;
202    case kCondGt: res = kArmCondGt; break;
203    case kCondLe: res = kArmCondLe; break;
204    case kCondAl: res = kArmCondAl; break;
205    case kCondNv: res = kArmCondNv; break;
206    default:
207      LOG(FATAL) << "Bad condition code " << ccode;
208      res = static_cast<ArmConditionCode>(0);  // Quiet gcc
209  }
210  return res;
211}
212
213static const char *shift_names[4] = {
214  "lsl",
215  "lsr",
216  "asr",
217  "ror"
218};
219
220static const char* extend_names[8] = {
221  "uxtb",
222  "uxth",
223  "uxtw",
224  "uxtx",
225  "sxtb",
226  "sxth",
227  "sxtw",
228  "sxtx",
229};
230
231/* Decode and print a register extension (e.g. ", uxtb #1") */
232static void DecodeRegExtendOrShift(int operand, char *buf, size_t buf_size) {
233  if ((operand & (1 << 6)) == 0) {
234    const char *shift_name = shift_names[(operand >> 7) & 0x3];
235    int amount = operand & 0x3f;
236    snprintf(buf, buf_size, ", %s #%d", shift_name, amount);
237  } else {
238    const char *extend_name = extend_names[(operand >> 3) & 0x7];
239    int amount = operand & 0x7;
240    if (amount == 0) {
241      snprintf(buf, buf_size, ", %s", extend_name);
242    } else {
243      snprintf(buf, buf_size, ", %s #%d", extend_name, amount);
244    }
245  }
246}
247
248#define BIT_MASK(w) ((UINT64_C(1) << (w)) - UINT64_C(1))
249
250static uint64_t RotateRight(uint64_t value, unsigned rotate, unsigned width) {
251  DCHECK_LE(width, 64U);
252  rotate &= 63;
253  value = value & BIT_MASK(width);
254  return ((value & BIT_MASK(rotate)) << (width - rotate)) | (value >> rotate);
255}
256
257static uint64_t RepeatBitsAcrossReg(bool is_wide, uint64_t value, unsigned width) {
258  unsigned i;
259  unsigned reg_size = (is_wide) ? 64 : 32;
260  uint64_t result = value & BIT_MASK(width);
261  for (i = width; i < reg_size; i *= 2) {
262    result |= (result << i);
263  }
264  DCHECK_EQ(i, reg_size);
265  return result;
266}
267
268/**
269 * @brief Decode an immediate in the form required by logical instructions.
270 *
271 * @param is_wide Whether @p value encodes a 64-bit (as opposed to 32-bit) immediate.
272 * @param value The encoded logical immediates that is to be decoded.
273 * @return The decoded logical immediate.
274 * @note This is the inverse of Arm64Mir2Lir::EncodeLogicalImmediate().
275 */
276uint64_t Arm64Mir2Lir::DecodeLogicalImmediate(bool is_wide, int value) {
277  unsigned n     = (value >> 12) & 0x01;
278  unsigned imm_r = (value >>  6) & 0x3f;
279  unsigned imm_s = (value >>  0) & 0x3f;
280
281  // An integer is constructed from the n, imm_s and imm_r bits according to
282  // the following table:
283  //
284  // N   imms immr  size S             R
285  // 1 ssssss rrrrrr 64  UInt(ssssss) UInt(rrrrrr)
286  // 0 0sssss xrrrrr 32  UInt(sssss)  UInt(rrrrr)
287  // 0 10ssss xxrrrr 16  UInt(ssss)   UInt(rrrr)
288  // 0 110sss xxxrrr 8   UInt(sss)    UInt(rrr)
289  // 0 1110ss xxxxrr 4   UInt(ss)     UInt(rr)
290  // 0 11110s xxxxxr 2   UInt(s)      UInt(r)
291  // (s bits must not be all set)
292  //
293  // A pattern is constructed of size bits, where the least significant S+1
294  // bits are set. The pattern is rotated right by R, and repeated across a
295  // 32 or 64-bit value, depending on destination register width.
296
297  if (n == 1) {
298    DCHECK_NE(imm_s, 0x3fU);
299    uint64_t bits = BIT_MASK(imm_s + 1);
300    return RotateRight(bits, imm_r, 64);
301  } else {
302    DCHECK_NE((imm_s >> 1), 0x1fU);
303    for (unsigned width = 0x20; width >= 0x2; width >>= 1) {
304      if ((imm_s & width) == 0) {
305        unsigned mask = (unsigned)(width - 1);
306        DCHECK_NE((imm_s & mask), mask);
307        uint64_t bits = BIT_MASK((imm_s & mask) + 1);
308        return RepeatBitsAcrossReg(is_wide, RotateRight(bits, imm_r & mask, width), width);
309      }
310    }
311  }
312  return 0;
313}
314
315/**
316 * @brief Decode an 8-bit single point number encoded with EncodeImmSingle().
317 */
318static float DecodeImmSingle(uint8_t small_float) {
319  int mantissa = (small_float & 0x0f) + 0x10;
320  int sign = ((small_float & 0x80) == 0) ? 1 : -1;
321  float signed_mantissa = static_cast<float>(sign*mantissa);
322  int exponent = (((small_float >> 4) & 0x7) + 4) & 0x7;
323  return signed_mantissa*static_cast<float>(1 << exponent)*0.0078125f;
324}
325
326static const char* cc_names[] = {"eq", "ne", "cs", "cc", "mi", "pl", "vs", "vc",
327                                 "hi", "ls", "ge", "lt", "gt", "le", "al", "nv"};
328/*
329 * Interpret a format string and build a string no longer than size
330 * See format key in assemble_arm64.cc.
331 */
332std::string Arm64Mir2Lir::BuildInsnString(const char* fmt, LIR* lir, unsigned char* base_addr) {
333  std::string buf;
334  const char* fmt_end = &fmt[strlen(fmt)];
335  char tbuf[256];
336  const char* name;
337  char nc;
338  while (fmt < fmt_end) {
339    int operand;
340    if (*fmt == '!') {
341      fmt++;
342      DCHECK_LT(fmt, fmt_end);
343      nc = *fmt++;
344      if (nc == '!') {
345        strcpy(tbuf, "!");
346      } else {
347         DCHECK_LT(fmt, fmt_end);
348         DCHECK_LT(static_cast<unsigned>(nc-'0'), 4U);
349         operand = lir->operands[nc-'0'];
350         switch (*fmt++) {
351           case 'e':  {
352               // Omit ", uxtw #0" in strings like "add w0, w1, w3, uxtw #0" and
353               // ", uxtx #0" in strings like "add x0, x1, x3, uxtx #0"
354               int omittable = ((IS_WIDE(lir->opcode)) ? EncodeExtend(kA64Uxtw, 0) :
355                                EncodeExtend(kA64Uxtw, 0));
356               if (LIKELY(operand == omittable)) {
357                 strcpy(tbuf, "");
358               } else {
359                 DecodeRegExtendOrShift(operand, tbuf, arraysize(tbuf));
360               }
361             }
362             break;
363           case 'o':
364             // Omit ", lsl #0"
365             if (LIKELY(operand == EncodeShift(kA64Lsl, 0))) {
366               strcpy(tbuf, "");
367             } else {
368               DecodeRegExtendOrShift(operand, tbuf, arraysize(tbuf));
369             }
370             break;
371           case 'B':
372             switch (operand) {
373               case kSY:
374                 name = "sy";
375                 break;
376               case kST:
377                 name = "st";
378                 break;
379               case kISH:
380                 name = "ish";
381                 break;
382               case kISHST:
383                 name = "ishst";
384                 break;
385               case kNSH:
386                 name = "nsh";
387                 break;
388               case kNSHST:
389                 name = "shst";
390                 break;
391               default:
392                 name = "DecodeError2";
393                 break;
394             }
395             strcpy(tbuf, name);
396             break;
397           case 's':
398             snprintf(tbuf, arraysize(tbuf), "s%d", operand & RegStorage::kRegNumMask);
399             break;
400           case 'S':
401             snprintf(tbuf, arraysize(tbuf), "d%d", operand & RegStorage::kRegNumMask);
402             break;
403           case 'f':
404             snprintf(tbuf, arraysize(tbuf), "%c%d", (IS_FWIDE(lir->opcode)) ? 'd' : 's',
405                      operand & RegStorage::kRegNumMask);
406             break;
407           case 'l': {
408               bool is_wide = IS_WIDE(lir->opcode);
409               uint64_t imm = DecodeLogicalImmediate(is_wide, operand);
410               snprintf(tbuf, arraysize(tbuf), "%" PRId64 " (%#" PRIx64 ")", imm, imm);
411             }
412             break;
413           case 'I':
414             snprintf(tbuf, arraysize(tbuf), "%f", DecodeImmSingle(operand));
415             break;
416           case 'M':
417             if (LIKELY(operand == 0))
418               strcpy(tbuf, "");
419             else
420               snprintf(tbuf, arraysize(tbuf), ", lsl #%d", 16*operand);
421             break;
422           case 'd':
423             snprintf(tbuf, arraysize(tbuf), "%d", operand);
424             break;
425           case 'w':
426             if (LIKELY(operand != rwzr))
427               snprintf(tbuf, arraysize(tbuf), "w%d", operand & RegStorage::kRegNumMask);
428             else
429               strcpy(tbuf, "wzr");
430             break;
431           case 'W':
432             if (LIKELY(operand != rwsp))
433               snprintf(tbuf, arraysize(tbuf), "w%d", operand & RegStorage::kRegNumMask);
434             else
435               strcpy(tbuf, "wsp");
436             break;
437           case 'x':
438             if (LIKELY(operand != rxzr))
439               snprintf(tbuf, arraysize(tbuf), "x%d", operand & RegStorage::kRegNumMask);
440             else
441               strcpy(tbuf, "xzr");
442             break;
443           case 'X':
444             if (LIKELY(operand != rsp))
445               snprintf(tbuf, arraysize(tbuf), "x%d", operand & RegStorage::kRegNumMask);
446             else
447               strcpy(tbuf, "sp");
448             break;
449           case 'D':
450             snprintf(tbuf, arraysize(tbuf), "%d", operand*((IS_WIDE(lir->opcode)) ? 8 : 4));
451             break;
452           case 'E':
453             snprintf(tbuf, arraysize(tbuf), "%d", operand*4);
454             break;
455           case 'F':
456             snprintf(tbuf, arraysize(tbuf), "%d", operand*2);
457             break;
458           case 'G':
459             if (LIKELY(operand == 0))
460               strcpy(tbuf, "");
461             else
462               strcpy(tbuf, (IS_WIDE(lir->opcode)) ? ", lsl #3" : ", lsl #2");
463             break;
464           case 'c':
465             strcpy(tbuf, cc_names[operand]);
466             break;
467           case 't':
468             snprintf(tbuf, arraysize(tbuf), "0x%08" PRIxPTR " (L%p)",
469                 reinterpret_cast<uintptr_t>(base_addr) + lir->offset + (operand << 2),
470                 lir->target);
471             break;
472           case 'r': {
473               bool is_wide = IS_WIDE(lir->opcode);
474               if (LIKELY(operand != rwzr && operand != rxzr)) {
475                 snprintf(tbuf, arraysize(tbuf), "%c%d", (is_wide) ? 'x' : 'w',
476                          operand & RegStorage::kRegNumMask);
477               } else {
478                 strcpy(tbuf, (is_wide) ? "xzr" : "wzr");
479               }
480             }
481             break;
482           case 'R': {
483               bool is_wide = IS_WIDE(lir->opcode);
484               if (LIKELY(operand != rwsp && operand != rsp)) {
485                 snprintf(tbuf, arraysize(tbuf), "%c%d", (is_wide) ? 'x' : 'w',
486                          operand & RegStorage::kRegNumMask);
487               } else {
488                 strcpy(tbuf, (is_wide) ? "sp" : "wsp");
489               }
490             }
491             break;
492           case 'p':
493             snprintf(tbuf, arraysize(tbuf), ".+%d (addr %#" PRIxPTR ")", 4*operand,
494                      reinterpret_cast<uintptr_t>(base_addr) + lir->offset + 4*operand);
495             break;
496           case 'T':
497             if (LIKELY(operand == 0))
498               strcpy(tbuf, "");
499             else if (operand == 1)
500               strcpy(tbuf, ", lsl #12");
501             else
502               strcpy(tbuf, ", DecodeError3");
503             break;
504           default:
505             strcpy(tbuf, "DecodeError1");
506             break;
507        }
508        buf += tbuf;
509      }
510    } else {
511       buf += *fmt++;
512    }
513  }
514  return buf;
515}
516
517void Arm64Mir2Lir::DumpResourceMask(LIR* arm_lir, uint64_t mask, const char* prefix) {
518  char buf[256];
519  buf[0] = 0;
520
521  if (mask == ENCODE_ALL) {
522    strcpy(buf, "all");
523  } else {
524    char num[8];
525    int i;
526
527    for (i = 0; i < kArmRegEnd; i++) {
528      if (mask & (1ULL << i)) {
529        snprintf(num, arraysize(num), "%d ", i);
530        strcat(buf, num);
531      }
532    }
533
534    if (mask & ENCODE_CCODE) {
535      strcat(buf, "cc ");
536    }
537    if (mask & ENCODE_FP_STATUS) {
538      strcat(buf, "fpcc ");
539    }
540
541    /* Memory bits */
542    if (arm_lir && (mask & ENCODE_DALVIK_REG)) {
543      snprintf(buf + strlen(buf), arraysize(buf) - strlen(buf), "dr%d%s",
544               DECODE_ALIAS_INFO_REG(arm_lir->flags.alias_info),
545               DECODE_ALIAS_INFO_WIDE(arm_lir->flags.alias_info) ? "(+1)" : "");
546    }
547    if (mask & ENCODE_LITERAL) {
548      strcat(buf, "lit ");
549    }
550
551    if (mask & ENCODE_HEAP_REF) {
552      strcat(buf, "heap ");
553    }
554    if (mask & ENCODE_MUST_NOT_ALIAS) {
555      strcat(buf, "noalias ");
556    }
557  }
558  if (buf[0]) {
559    LOG(INFO) << prefix << ": " << buf;
560  }
561}
562
563bool Arm64Mir2Lir::IsUnconditionalBranch(LIR* lir) {
564  return (lir->opcode == kA64B1t);
565}
566
567bool Arm64Mir2Lir::SupportsVolatileLoadStore(OpSize size) {
568  return true;
569}
570
571RegisterClass Arm64Mir2Lir::RegClassForFieldLoadStore(OpSize size, bool is_volatile) {
572  if (UNLIKELY(is_volatile)) {
573    // On arm64, fp register load/store is atomic only for single bytes.
574    if (size != kSignedByte && size != kUnsignedByte) {
575      return kCoreReg;
576    }
577  }
578  return RegClassBySize(size);
579}
580
581Arm64Mir2Lir::Arm64Mir2Lir(CompilationUnit* cu, MIRGraph* mir_graph, ArenaAllocator* arena)
582    : Mir2Lir(cu, mir_graph, arena) {
583  // Sanity check - make sure encoding map lines up.
584  for (int i = 0; i < kA64Last; i++) {
585    if (UNWIDE(Arm64Mir2Lir::EncodingMap[i].opcode) != i) {
586      LOG(FATAL) << "Encoding order for " << Arm64Mir2Lir::EncodingMap[i].name
587                 << " is wrong: expecting " << i << ", seeing "
588                 << static_cast<int>(Arm64Mir2Lir::EncodingMap[i].opcode);
589    }
590  }
591}
592
593Mir2Lir* Arm64CodeGenerator(CompilationUnit* const cu, MIRGraph* const mir_graph,
594                            ArenaAllocator* const arena) {
595  return new Arm64Mir2Lir(cu, mir_graph, arena);
596}
597
598void Arm64Mir2Lir::CompilerInitializeRegAlloc() {
599  reg_pool_ = new (arena_) RegisterPool(this, arena_, core_regs, core64_regs, sp_regs, dp_regs,
600                                        reserved_regs, reserved64_regs, core_temps, core64_temps,
601                                        sp_temps, dp_temps);
602
603  // Target-specific adjustments.
604  // Alias single precision float registers to corresponding double registers.
605  GrowableArray<RegisterInfo*>::Iterator fp_it(&reg_pool_->sp_regs_);
606  for (RegisterInfo* info = fp_it.Next(); info != nullptr; info = fp_it.Next()) {
607    int fp_reg_num = info->GetReg().GetRegNum();
608    RegStorage dp_reg = RegStorage::FloatSolo64(fp_reg_num);
609    RegisterInfo* dp_reg_info = GetRegInfo(dp_reg);
610    // Double precision register's master storage should refer to itself.
611    DCHECK_EQ(dp_reg_info, dp_reg_info->Master());
612    // Redirect single precision's master storage to master.
613    info->SetMaster(dp_reg_info);
614    // Singles should show a single 32-bit mask bit, at first referring to the low half.
615    DCHECK_EQ(info->StorageMask(), 0x1U);
616  }
617
618  // Alias 32bit W registers to corresponding 64bit X registers.
619  GrowableArray<RegisterInfo*>::Iterator w_it(&reg_pool_->core_regs_);
620  for (RegisterInfo* info = w_it.Next(); info != nullptr; info = w_it.Next()) {
621    int x_reg_num = info->GetReg().GetRegNum();
622    RegStorage x_reg = RegStorage::Solo64(x_reg_num);
623    RegisterInfo* x_reg_info = GetRegInfo(x_reg);
624    // 64bit X register's master storage should refer to itself.
625    DCHECK_EQ(x_reg_info, x_reg_info->Master());
626    // Redirect 32bit W master storage to 64bit X.
627    info->SetMaster(x_reg_info);
628    // 32bit W should show a single 32-bit mask bit, at first referring to the low half.
629    DCHECK_EQ(info->StorageMask(), 0x1U);
630  }
631
632  // Don't start allocating temps at r0/s0/d0 or you may clobber return regs in early-exit methods.
633  // TODO: adjust when we roll to hard float calling convention.
634  reg_pool_->next_core_reg_ = 2;
635  reg_pool_->next_sp_reg_ = 0;
636  reg_pool_->next_dp_reg_ = 0;
637}
638
639/*
640 * TUNING: is true leaf?  Can't just use METHOD_IS_LEAF to determine as some
641 * instructions might call out to C/assembly helper functions.  Until
642 * machinery is in place, always spill lr.
643 */
644
645void Arm64Mir2Lir::AdjustSpillMask() {
646  core_spill_mask_ |= (1 << rs_rA64_LR.GetRegNum());
647  num_core_spills_++;
648}
649
650/*
651 * Mark a callee-save fp register as promoted.
652 */
653void Arm64Mir2Lir::MarkPreservedSingle(int v_reg, RegStorage reg) {
654  DCHECK(reg.IsFloat());
655  int adjusted_reg_num = reg.GetRegNum() - A64_FP_CALLEE_SAVE_BASE;
656  // Ensure fp_vmap_table is large enough
657  int table_size = fp_vmap_table_.size();
658  for (int i = table_size; i < (adjusted_reg_num + 1); i++) {
659    fp_vmap_table_.push_back(INVALID_VREG);
660  }
661  // Add the current mapping
662  fp_vmap_table_[adjusted_reg_num] = v_reg;
663  // Size of fp_vmap_table is high-water mark, use to set mask
664  num_fp_spills_ = fp_vmap_table_.size();
665  fp_spill_mask_ = ((1 << num_fp_spills_) - 1) << A64_FP_CALLEE_SAVE_BASE;
666}
667
668void Arm64Mir2Lir::MarkPreservedDouble(int v_reg, RegStorage reg) {
669  DCHECK(reg.IsDouble());
670  MarkPreservedSingle(v_reg, reg);
671}
672
673/* Clobber all regs that might be used by an external C call */
674void Arm64Mir2Lir::ClobberCallerSave() {
675  Clobber(rs_x0);
676  Clobber(rs_x1);
677  Clobber(rs_x2);
678  Clobber(rs_x3);
679  Clobber(rs_x4);
680  Clobber(rs_x5);
681  Clobber(rs_x6);
682  Clobber(rs_x7);
683  Clobber(rs_x8);
684  Clobber(rs_x9);
685  Clobber(rs_x10);
686  Clobber(rs_x11);
687  Clobber(rs_x12);
688  Clobber(rs_x13);
689  Clobber(rs_x14);
690  Clobber(rs_x15);
691  Clobber(rs_x16);
692  Clobber(rs_x17);
693  Clobber(rs_x30);
694
695  Clobber(rs_f0);
696  Clobber(rs_f1);
697  Clobber(rs_f2);
698  Clobber(rs_f3);
699  Clobber(rs_f4);
700  Clobber(rs_f5);
701  Clobber(rs_f6);
702  Clobber(rs_f7);
703  Clobber(rs_f16);
704  Clobber(rs_f17);
705  Clobber(rs_f18);
706  Clobber(rs_f19);
707  Clobber(rs_f20);
708  Clobber(rs_f21);
709  Clobber(rs_f22);
710  Clobber(rs_f23);
711  Clobber(rs_f24);
712  Clobber(rs_f25);
713  Clobber(rs_f26);
714  Clobber(rs_f27);
715  Clobber(rs_f28);
716  Clobber(rs_f29);
717  Clobber(rs_f30);
718  Clobber(rs_f31);
719}
720
721RegLocation Arm64Mir2Lir::GetReturnWideAlt() {
722  RegLocation res = LocCReturnWide();
723  res.reg.SetReg(rx2);
724  res.reg.SetHighReg(rx3);
725  Clobber(rs_x2);
726  Clobber(rs_x3);
727  MarkInUse(rs_x2);
728  MarkInUse(rs_x3);
729  MarkWide(res.reg);
730  return res;
731}
732
733RegLocation Arm64Mir2Lir::GetReturnAlt() {
734  RegLocation res = LocCReturn();
735  res.reg.SetReg(rx1);
736  Clobber(rs_x1);
737  MarkInUse(rs_x1);
738  return res;
739}
740
741/* To be used when explicitly managing register use */
742void Arm64Mir2Lir::LockCallTemps() {
743  LockTemp(rs_x0);
744  LockTemp(rs_x1);
745  LockTemp(rs_x2);
746  LockTemp(rs_x3);
747}
748
749/* To be used when explicitly managing register use */
750void Arm64Mir2Lir::FreeCallTemps() {
751  FreeTemp(rs_x0);
752  FreeTemp(rs_x1);
753  FreeTemp(rs_x2);
754  FreeTemp(rs_x3);
755}
756
757RegStorage Arm64Mir2Lir::LoadHelper(ThreadOffset<4> offset) {
758  UNIMPLEMENTED(FATAL) << "Should not be called.";
759  return RegStorage::InvalidReg();
760}
761
762RegStorage Arm64Mir2Lir::LoadHelper(ThreadOffset<8> offset) {
763  // TODO(Arm64): use LoadWordDisp instead.
764  //   e.g. LoadWordDisp(rs_rA64_SELF, offset.Int32Value(), rs_rA64_LR);
765  LoadBaseDisp(rs_rA64_SELF, offset.Int32Value(), rs_rA64_LR, k64);
766  return rs_rA64_LR;
767}
768
769LIR* Arm64Mir2Lir::CheckSuspendUsingLoad() {
770  RegStorage tmp = rs_x0;
771  LoadWordDisp(rs_rA64_SELF, Thread::ThreadSuspendTriggerOffset<8>().Int32Value(), tmp);
772  LIR* load2 = LoadWordDisp(tmp, 0, tmp);
773  return load2;
774}
775
776uint64_t Arm64Mir2Lir::GetTargetInstFlags(int opcode) {
777  DCHECK(!IsPseudoLirOp(opcode));
778  return Arm64Mir2Lir::EncodingMap[UNWIDE(opcode)].flags;
779}
780
781const char* Arm64Mir2Lir::GetTargetInstName(int opcode) {
782  DCHECK(!IsPseudoLirOp(opcode));
783  return Arm64Mir2Lir::EncodingMap[UNWIDE(opcode)].name;
784}
785
786const char* Arm64Mir2Lir::GetTargetInstFmt(int opcode) {
787  DCHECK(!IsPseudoLirOp(opcode));
788  return Arm64Mir2Lir::EncodingMap[UNWIDE(opcode)].fmt;
789}
790
791// TODO(Arm64): reuse info in QuickArgumentVisitor?
792static RegStorage GetArgPhysicalReg(RegLocation* loc, int* num_gpr_used, int* num_fpr_used,
793                                    OpSize* op_size) {
794  if (loc->fp) {
795    int n = *num_fpr_used;
796    if (n < 8) {
797      *num_fpr_used = n + 1;
798      RegStorage::RegStorageKind reg_kind;
799      if (loc->wide) {
800        *op_size = kDouble;
801        reg_kind = RegStorage::k64BitSolo;
802      } else {
803        *op_size = kSingle;
804        reg_kind = RegStorage::k32BitSolo;
805      }
806      return RegStorage(RegStorage::kValid | reg_kind | RegStorage::kFloatingPoint | n);
807    }
808  } else {
809    int n = *num_gpr_used;
810    if (n < 7) {
811      *num_gpr_used = n + 1;
812      if (loc->wide) {
813        *op_size = k64;
814        return RegStorage::Solo64(n);
815      } else {
816        *op_size = k32;
817        return RegStorage::Solo32(n);
818      }
819    }
820  }
821
822  return RegStorage::InvalidReg();
823}
824
825/*
826 * If there are any ins passed in registers that have not been promoted
827 * to a callee-save register, flush them to the frame.  Perform initial
828 * assignment of promoted arguments.
829 *
830 * ArgLocs is an array of location records describing the incoming arguments
831 * with one location record per word of argument.
832 */
833void Arm64Mir2Lir::FlushIns(RegLocation* ArgLocs, RegLocation rl_method) {
834  int num_gpr_used = 1;
835  int num_fpr_used = 0;
836
837  /*
838   * Dummy up a RegLocation for the incoming Method*
839   * It will attempt to keep kArg0 live (or copy it to home location
840   * if promoted).
841   */
842  RegLocation rl_src = rl_method;
843  rl_src.location = kLocPhysReg;
844  rl_src.reg = TargetReg(kArg0);
845  rl_src.home = false;
846  MarkLive(rl_src);
847
848  // rl_method might be 32-bit, but ArtMethod* on stack is 64-bit, so always flush it.
849  StoreWordDisp(TargetReg(kSp), 0, TargetReg(kArg0));
850
851  // If Method* has been promoted, load it,
852  // otherwise, rl_method is the 32-bit value on [sp], and has already been loaded.
853  if (rl_method.location == kLocPhysReg) {
854    StoreValue(rl_method, rl_src);
855  }
856
857  if (cu_->num_ins == 0) {
858    return;
859  }
860
861  int start_vreg = cu_->num_dalvik_registers - cu_->num_ins;
862  for (int i = 0; i < cu_->num_ins; i++) {
863    PromotionMap* v_map = &promotion_map_[start_vreg + i];
864    RegLocation* t_loc = &ArgLocs[i];
865    OpSize op_size;
866    RegStorage reg = GetArgPhysicalReg(t_loc, &num_gpr_used, &num_fpr_used, &op_size);
867
868    if (reg.Valid()) {
869      if ((v_map->core_location == kLocPhysReg) && !t_loc->fp) {
870        OpRegCopy(RegStorage::Solo32(v_map->core_reg), reg);
871      } else if ((v_map->fp_location == kLocPhysReg) && t_loc->fp) {
872        OpRegCopy(RegStorage::Solo32(v_map->FpReg), reg);
873      } else {
874        StoreBaseDisp(TargetReg(kSp), SRegOffset(start_vreg + i), reg, op_size);
875        if (reg.Is64Bit()) {
876          if (SRegOffset(start_vreg + i) + 4 != SRegOffset(start_vreg + i + 1)) {
877            LOG(FATAL) << "64 bit value stored in non-consecutive 4 bytes slots";
878          }
879          i += 1;
880        }
881      }
882    } else {
883      // If arriving in frame & promoted
884      if (v_map->core_location == kLocPhysReg) {
885        LoadWordDisp(TargetReg(kSp), SRegOffset(start_vreg + i),
886                     RegStorage::Solo32(v_map->core_reg));
887      }
888      if (v_map->fp_location == kLocPhysReg) {
889        LoadWordDisp(TargetReg(kSp), SRegOffset(start_vreg + i), RegStorage::Solo32(v_map->FpReg));
890      }
891    }
892  }
893}
894
895int Arm64Mir2Lir::LoadArgRegs(CallInfo* info, int call_state,
896                              NextCallInsn next_call_insn,
897                              const MethodReference& target_method,
898                              uint32_t vtable_idx, uintptr_t direct_code,
899                              uintptr_t direct_method, InvokeType type, bool skip_this) {
900  int last_arg_reg = TargetReg(kArg3).GetReg();
901  int next_reg = TargetReg(kArg1).GetReg();
902  int next_arg = 0;
903  if (skip_this) {
904    next_reg++;
905    next_arg++;
906  }
907  for (; (next_reg <= last_arg_reg) && (next_arg < info->num_arg_words); next_reg++) {
908    RegLocation rl_arg = info->args[next_arg++];
909    rl_arg = UpdateRawLoc(rl_arg);
910    if (rl_arg.wide && (next_reg <= TargetReg(kArg2).GetReg())) {
911      RegStorage r_tmp(RegStorage::k64BitPair, next_reg, next_reg + 1);
912      LoadValueDirectWideFixed(rl_arg, r_tmp);
913      next_reg++;
914      next_arg++;
915    } else {
916      if (rl_arg.wide) {
917        rl_arg = NarrowRegLoc(rl_arg);
918        rl_arg.is_const = false;
919      }
920      LoadValueDirectFixed(rl_arg, RegStorage::Solo32(next_reg));
921    }
922    call_state = next_call_insn(cu_, info, call_state, target_method, vtable_idx,
923                                direct_code, direct_method, type);
924  }
925  return call_state;
926}
927
928}  // namespace art
929