target_arm64.cc revision 5655e84e8d71697d8ef3ea901a0b853af42c559e 
1/*
2 * Copyright (C) 2011 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *      http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17#include "codegen_arm64.h"
18
19#include <inttypes.h>
20
21#include <string>
22
23#include "dex/compiler_internals.h"
24#include "dex/quick/mir_to_lir-inl.h"
25
26namespace art {
27
28static constexpr RegStorage core_regs_arr[] =
29    {rs_w0, rs_w1, rs_w2, rs_w3, rs_w4, rs_w5, rs_w6, rs_w7,
30     rs_w8, rs_w9, rs_w10, rs_w11, rs_w12, rs_w13, rs_w14, rs_w15,
31     rs_w16, rs_w17, rs_w18, rs_w19, rs_w20, rs_w21, rs_w22, rs_w23,
32     rs_w24, rs_w25, rs_w26, rs_w27, rs_w28, rs_w29, rs_w30, rs_w31,
33     rs_wzr};
34static constexpr RegStorage core64_regs_arr[] =
35    {rs_x0, rs_x1, rs_x2, rs_x3, rs_x4, rs_x5, rs_x6, rs_x7,
36     rs_x8, rs_x9, rs_x10, rs_x11, rs_x12, rs_x13, rs_x14, rs_x15,
37     rs_x16, rs_x17, rs_x18, rs_x19, rs_x20, rs_x21, rs_x22, rs_x23,
38     rs_x24, rs_x25, rs_x26, rs_x27, rs_x28, rs_x29, rs_x30, rs_x31,
39     rs_xzr};
40static constexpr RegStorage sp_regs_arr[] =
41    {rs_f0, rs_f1, rs_f2, rs_f3, rs_f4, rs_f5, rs_f6, rs_f7,
42     rs_f8, rs_f9, rs_f10, rs_f11, rs_f12, rs_f13, rs_f14, rs_f15,
43     rs_f16, rs_f17, rs_f18, rs_f19, rs_f20, rs_f21, rs_f22, rs_f23,
44     rs_f24, rs_f25, rs_f26, rs_f27, rs_f28, rs_f29, rs_f30, rs_f31};
45static constexpr RegStorage dp_regs_arr[] =
46    {rs_d0, rs_d1, rs_d2, rs_d3, rs_d4, rs_d5, rs_d6, rs_d7,
47     rs_d8, rs_d9, rs_d10, rs_d11, rs_d12, rs_d13, rs_d14, rs_d15,
48     rs_d16, rs_d17, rs_d18, rs_d19, rs_d20, rs_d21, rs_d22, rs_d23,
49     rs_d24, rs_d25, rs_d26, rs_d27, rs_d28, rs_d29, rs_d30, rs_d31};
50static constexpr RegStorage reserved_regs_arr[] =
51    {rs_rA32_SUSPEND, rs_rA32_SELF, rs_rA32_SP, rs_rA32_LR, rs_wzr};
52static constexpr RegStorage reserved64_regs_arr[] =
53    {rs_rA64_SUSPEND, rs_rA64_SELF, rs_rA64_SP, rs_rA64_LR, rs_xzr};
54// TUNING: Are there too many temp registers and too less promote target?
55// This definition need to be matched with runtime.cc, quick entry assembly and JNI compiler
56// Note: we are not able to call to C function directly if it un-match C ABI.
57// Currently, rs_rA64_SELF is not a callee save register which does not match C ABI.
58static constexpr RegStorage core_temps_arr[] =
59    {rs_w0, rs_w1, rs_w2, rs_w3, rs_w4, rs_w5, rs_w6, rs_w7,
60     rs_w8, rs_w9, rs_w10, rs_w11, rs_w12, rs_w13, rs_w14, rs_w15, rs_w16,
61     rs_w17};
62static constexpr RegStorage core64_temps_arr[] =
63    {rs_x0, rs_x1, rs_x2, rs_x3, rs_x4, rs_x5, rs_x6, rs_x7,
64     rs_x8, rs_x9, rs_x10, rs_x11, rs_x12, rs_x13, rs_x14, rs_x15, rs_x16,
65     rs_x17};
66static constexpr RegStorage sp_temps_arr[] =
67    {rs_f0, rs_f1, rs_f2, rs_f3, rs_f4, rs_f5, rs_f6, rs_f7,
68     rs_f16, rs_f17, rs_f18, rs_f19, rs_f20, rs_f21, rs_f22, rs_f23,
69     rs_f24, rs_f25, rs_f26, rs_f27, rs_f28, rs_f29, rs_f30, rs_f31};
70static constexpr RegStorage dp_temps_arr[] =
71    {rs_d0, rs_d1, rs_d2, rs_d3, rs_d4, rs_d5, rs_d6, rs_d7,
72     rs_d16, rs_d17, rs_d18, rs_d19, rs_d20, rs_d21, rs_d22, rs_d23,
73     rs_d24, rs_d25, rs_d26, rs_d27, rs_d28, rs_d29, rs_d30, rs_d31};
74
75static constexpr ArrayRef<const RegStorage> core_regs(core_regs_arr);
76static constexpr ArrayRef<const RegStorage> core64_regs(core64_regs_arr);
77static constexpr ArrayRef<const RegStorage> sp_regs(sp_regs_arr);
78static constexpr ArrayRef<const RegStorage> dp_regs(dp_regs_arr);
79static constexpr ArrayRef<const RegStorage> reserved_regs(reserved_regs_arr);
80static constexpr ArrayRef<const RegStorage> reserved64_regs(reserved64_regs_arr);
81static constexpr ArrayRef<const RegStorage> core_temps(core_temps_arr);
82static constexpr ArrayRef<const RegStorage> core64_temps(core64_temps_arr);
83static constexpr ArrayRef<const RegStorage> sp_temps(sp_temps_arr);
84static constexpr ArrayRef<const RegStorage> dp_temps(dp_temps_arr);
85
86RegLocation Arm64Mir2Lir::LocCReturn() {
87  return arm_loc_c_return;
88}
89
90RegLocation Arm64Mir2Lir::LocCReturnRef() {
91  return arm_loc_c_return;
92}
93
94RegLocation Arm64Mir2Lir::LocCReturnWide() {
95  return arm_loc_c_return_wide;
96}
97
98RegLocation Arm64Mir2Lir::LocCReturnFloat() {
99  return arm_loc_c_return_float;
100}
101
102RegLocation Arm64Mir2Lir::LocCReturnDouble() {
103  return arm_loc_c_return_double;
104}
105
106// Return a target-dependent special register.
107RegStorage Arm64Mir2Lir::TargetReg(SpecialTargetRegister reg) {
108  RegStorage res_reg = RegStorage::InvalidReg();
109  switch (reg) {
110    case kSelf: res_reg = rs_rA64_SELF; break;
111    case kSuspend: res_reg = rs_rA64_SUSPEND; break;
112    case kLr: res_reg =  rs_rA64_LR; break;
113    case kPc: res_reg = RegStorage::InvalidReg(); break;
114    case kSp: res_reg =  rs_rA64_SP; break;
115    case kArg0: res_reg = rs_x0; break;
116    case kArg1: res_reg = rs_x1; break;
117    case kArg2: res_reg = rs_x2; break;
118    case kArg3: res_reg = rs_x3; break;
119    case kArg4: res_reg = rs_x4; break;
120    case kArg5: res_reg = rs_x5; break;
121    case kArg6: res_reg = rs_x6; break;
122    case kArg7: res_reg = rs_x7; break;
123    case kFArg0: res_reg = rs_f0; break;
124    case kFArg1: res_reg = rs_f1; break;
125    case kFArg2: res_reg = rs_f2; break;
126    case kFArg3: res_reg = rs_f3; break;
127    case kFArg4: res_reg = rs_f4; break;
128    case kFArg5: res_reg = rs_f5; break;
129    case kFArg6: res_reg = rs_f6; break;
130    case kFArg7: res_reg = rs_f7; break;
131    case kRet0: res_reg = rs_x0; break;
132    case kRet1: res_reg = rs_x1; break;
133    case kInvokeTgt: res_reg = rs_rA64_LR; break;
134    case kHiddenArg: res_reg = rs_x12; break;
135    case kHiddenFpArg: res_reg = RegStorage::InvalidReg(); break;
136    case kCount: res_reg = RegStorage::InvalidReg(); break;
137    default: res_reg = RegStorage::InvalidReg();
138  }
139  return res_reg;
140}
141
142/*
143 * Decode the register id. This routine makes assumptions on the encoding made by RegStorage.
144 */
145ResourceMask Arm64Mir2Lir::GetRegMaskCommon(const RegStorage& reg) const {
146  // TODO(Arm64): this function depends too much on the internal RegStorage encoding. Refactor.
147
148  // Check if the shape mask is zero (i.e. invalid).
149  if (UNLIKELY(reg == rs_wzr || reg == rs_xzr)) {
150    // The zero register is not a true register. It is just an immediate zero.
151    return kEncodeNone;
152  }
153
154  return ResourceMask::Bit(
155      // FP register starts at bit position 32.
156      (reg.IsFloat() ? kArm64FPReg0 : 0) + reg.GetRegNum());
157}
158
159ResourceMask Arm64Mir2Lir::GetPCUseDefEncoding() const {
160  LOG(FATAL) << "Unexpected call to GetPCUseDefEncoding for Arm64";
161  return kEncodeNone;
162}
163
164// Arm64 specific setup.  TODO: inline?:
165void Arm64Mir2Lir::SetupTargetResourceMasks(LIR* lir, uint64_t flags,
166                                            ResourceMask* use_mask, ResourceMask* def_mask) {
167  DCHECK_EQ(cu_->instruction_set, kArm64);
168  DCHECK(!lir->flags.use_def_invalid);
169
170  // These flags are somewhat uncommon - bypass if we can.
171  if ((flags & (REG_DEF_SP | REG_USE_SP | REG_DEF_LR)) != 0) {
172    if (flags & REG_DEF_SP) {
173      def_mask->SetBit(kArm64RegSP);
174    }
175
176    if (flags & REG_USE_SP) {
177      use_mask->SetBit(kArm64RegSP);
178    }
179
180    if (flags & REG_DEF_LR) {
181      def_mask->SetBit(kArm64RegLR);
182    }
183  }
184}
185
186ArmConditionCode Arm64Mir2Lir::ArmConditionEncoding(ConditionCode ccode) {
187  ArmConditionCode res;
188  switch (ccode) {
189    case kCondEq: res = kArmCondEq; break;
190    case kCondNe: res = kArmCondNe; break;
191    case kCondCs: res = kArmCondCs; break;
192    case kCondCc: res = kArmCondCc; break;
193    case kCondUlt: res = kArmCondCc; break;
194    case kCondUge: res = kArmCondCs; break;
195    case kCondMi: res = kArmCondMi; break;
196    case kCondPl: res = kArmCondPl; break;
197    case kCondVs: res = kArmCondVs; break;
198    case kCondVc: res = kArmCondVc; break;
199    case kCondHi: res = kArmCondHi; break;
200    case kCondLs: res = kArmCondLs; break;
201    case kCondGe: res = kArmCondGe; break;
202    case kCondLt: res = kArmCondLt; break;
203    case kCondGt: res = kArmCondGt; break;
204    case kCondLe: res = kArmCondLe; break;
205    case kCondAl: res = kArmCondAl; break;
206    case kCondNv: res = kArmCondNv; break;
207    default:
208      LOG(FATAL) << "Bad condition code " << ccode;
209      res = static_cast<ArmConditionCode>(0);  // Quiet gcc
210  }
211  return res;
212}
213
214static const char *shift_names[4] = {
215  "lsl",
216  "lsr",
217  "asr",
218  "ror"
219};
220
221static const char* extend_names[8] = {
222  "uxtb",
223  "uxth",
224  "uxtw",
225  "uxtx",
226  "sxtb",
227  "sxth",
228  "sxtw",
229  "sxtx",
230};
231
232/* Decode and print a register extension (e.g. ", uxtb #1") */
233static void DecodeRegExtendOrShift(int operand, char *buf, size_t buf_size) {
234  if ((operand & (1 << 6)) == 0) {
235    const char *shift_name = shift_names[(operand >> 7) & 0x3];
236    int amount = operand & 0x3f;
237    snprintf(buf, buf_size, ", %s #%d", shift_name, amount);
238  } else {
239    const char *extend_name = extend_names[(operand >> 3) & 0x7];
240    int amount = operand & 0x7;
241    if (amount == 0) {
242      snprintf(buf, buf_size, ", %s", extend_name);
243    } else {
244      snprintf(buf, buf_size, ", %s #%d", extend_name, amount);
245    }
246  }
247}
248
249#define BIT_MASK(w) ((UINT64_C(1) << (w)) - UINT64_C(1))
250
251static uint64_t RotateRight(uint64_t value, unsigned rotate, unsigned width) {
252  DCHECK_LE(width, 64U);
253  rotate &= 63;
254  value = value & BIT_MASK(width);
255  return ((value & BIT_MASK(rotate)) << (width - rotate)) | (value >> rotate);
256}
257
258static uint64_t RepeatBitsAcrossReg(bool is_wide, uint64_t value, unsigned width) {
259  unsigned i;
260  unsigned reg_size = (is_wide) ? 64 : 32;
261  uint64_t result = value & BIT_MASK(width);
262  for (i = width; i < reg_size; i *= 2) {
263    result |= (result << i);
264  }
265  DCHECK_EQ(i, reg_size);
266  return result;
267}
268
269/**
270 * @brief Decode an immediate in the form required by logical instructions.
271 *
272 * @param is_wide Whether @p value encodes a 64-bit (as opposed to 32-bit) immediate.
273 * @param value The encoded logical immediates that is to be decoded.
274 * @return The decoded logical immediate.
275 * @note This is the inverse of Arm64Mir2Lir::EncodeLogicalImmediate().
276 */
277uint64_t Arm64Mir2Lir::DecodeLogicalImmediate(bool is_wide, int value) {
278  unsigned n     = (value >> 12) & 0x01;
279  unsigned imm_r = (value >>  6) & 0x3f;
280  unsigned imm_s = (value >>  0) & 0x3f;
281
282  // An integer is constructed from the n, imm_s and imm_r bits according to
283  // the following table:
284  //
285  // N   imms immr  size S             R
286  // 1 ssssss rrrrrr 64  UInt(ssssss) UInt(rrrrrr)
287  // 0 0sssss xrrrrr 32  UInt(sssss)  UInt(rrrrr)
288  // 0 10ssss xxrrrr 16  UInt(ssss)   UInt(rrrr)
289  // 0 110sss xxxrrr 8   UInt(sss)    UInt(rrr)
290  // 0 1110ss xxxxrr 4   UInt(ss)     UInt(rr)
291  // 0 11110s xxxxxr 2   UInt(s)      UInt(r)
292  // (s bits must not be all set)
293  //
294  // A pattern is constructed of size bits, where the least significant S+1
295  // bits are set. The pattern is rotated right by R, and repeated across a
296  // 32 or 64-bit value, depending on destination register width.
297
298  if (n == 1) {
299    DCHECK_NE(imm_s, 0x3fU);
300    uint64_t bits = BIT_MASK(imm_s + 1);
301    return RotateRight(bits, imm_r, 64);
302  } else {
303    DCHECK_NE((imm_s >> 1), 0x1fU);
304    for (unsigned width = 0x20; width >= 0x2; width >>= 1) {
305      if ((imm_s & width) == 0) {
306        unsigned mask = (unsigned)(width - 1);
307        DCHECK_NE((imm_s & mask), mask);
308        uint64_t bits = BIT_MASK((imm_s & mask) + 1);
309        return RepeatBitsAcrossReg(is_wide, RotateRight(bits, imm_r & mask, width), width);
310      }
311    }
312  }
313  return 0;
314}
315
316/**
317 * @brief Decode an 8-bit single point number encoded with EncodeImmSingle().
318 */
319static float DecodeImmSingle(uint8_t small_float) {
320  int mantissa = (small_float & 0x0f) + 0x10;
321  int sign = ((small_float & 0x80) == 0) ? 1 : -1;
322  float signed_mantissa = static_cast<float>(sign*mantissa);
323  int exponent = (((small_float >> 4) & 0x7) + 4) & 0x7;
324  return signed_mantissa*static_cast<float>(1 << exponent)*0.0078125f;
325}
326
327static const char* cc_names[] = {"eq", "ne", "cs", "cc", "mi", "pl", "vs", "vc",
328                                 "hi", "ls", "ge", "lt", "gt", "le", "al", "nv"};
329/*
330 * Interpret a format string and build a string no longer than size
331 * See format key in assemble_arm64.cc.
332 */
333std::string Arm64Mir2Lir::BuildInsnString(const char* fmt, LIR* lir, unsigned char* base_addr) {
334  std::string buf;
335  const char* fmt_end = &fmt[strlen(fmt)];
336  char tbuf[256];
337  const char* name;
338  char nc;
339  while (fmt < fmt_end) {
340    int operand;
341    if (*fmt == '!') {
342      fmt++;
343      DCHECK_LT(fmt, fmt_end);
344      nc = *fmt++;
345      if (nc == '!') {
346        strcpy(tbuf, "!");
347      } else {
348         DCHECK_LT(fmt, fmt_end);
349         DCHECK_LT(static_cast<unsigned>(nc-'0'), 4U);
350         operand = lir->operands[nc-'0'];
351         switch (*fmt++) {
352           case 'e':  {
353               // Omit ", uxtw #0" in strings like "add w0, w1, w3, uxtw #0" and
354               // ", uxtx #0" in strings like "add x0, x1, x3, uxtx #0"
355               int omittable = ((IS_WIDE(lir->opcode)) ? EncodeExtend(kA64Uxtw, 0) :
356                                EncodeExtend(kA64Uxtw, 0));
357               if (LIKELY(operand == omittable)) {
358                 strcpy(tbuf, "");
359               } else {
360                 DecodeRegExtendOrShift(operand, tbuf, arraysize(tbuf));
361               }
362             }
363             break;
364           case 'o':
365             // Omit ", lsl #0"
366             if (LIKELY(operand == EncodeShift(kA64Lsl, 0))) {
367               strcpy(tbuf, "");
368             } else {
369               DecodeRegExtendOrShift(operand, tbuf, arraysize(tbuf));
370             }
371             break;
372           case 'B':
373             switch (operand) {
374               case kSY:
375                 name = "sy";
376                 break;
377               case kST:
378                 name = "st";
379                 break;
380               case kISH:
381                 name = "ish";
382                 break;
383               case kISHST:
384                 name = "ishst";
385                 break;
386               case kNSH:
387                 name = "nsh";
388                 break;
389               case kNSHST:
390                 name = "shst";
391                 break;
392               default:
393                 name = "DecodeError2";
394                 break;
395             }
396             strcpy(tbuf, name);
397             break;
398           case 's':
399             snprintf(tbuf, arraysize(tbuf), "s%d", operand & RegStorage::kRegNumMask);
400             break;
401           case 'S':
402             snprintf(tbuf, arraysize(tbuf), "d%d", operand & RegStorage::kRegNumMask);
403             break;
404           case 'f':
405             snprintf(tbuf, arraysize(tbuf), "%c%d", (IS_FWIDE(lir->opcode)) ? 'd' : 's',
406                      operand & RegStorage::kRegNumMask);
407             break;
408           case 'l': {
409               bool is_wide = IS_WIDE(lir->opcode);
410               uint64_t imm = DecodeLogicalImmediate(is_wide, operand);
411               snprintf(tbuf, arraysize(tbuf), "%" PRId64 " (%#" PRIx64 ")", imm, imm);
412             }
413             break;
414           case 'I':
415             snprintf(tbuf, arraysize(tbuf), "%f", DecodeImmSingle(operand));
416             break;
417           case 'M':
418             if (LIKELY(operand == 0))
419               strcpy(tbuf, "");
420             else
421               snprintf(tbuf, arraysize(tbuf), ", lsl #%d", 16*operand);
422             break;
423           case 'd':
424             snprintf(tbuf, arraysize(tbuf), "%d", operand);
425             break;
426           case 'w':
427             if (LIKELY(operand != rwzr))
428               snprintf(tbuf, arraysize(tbuf), "w%d", operand & RegStorage::kRegNumMask);
429             else
430               strcpy(tbuf, "wzr");
431             break;
432           case 'W':
433             if (LIKELY(operand != rwsp))
434               snprintf(tbuf, arraysize(tbuf), "w%d", operand & RegStorage::kRegNumMask);
435             else
436               strcpy(tbuf, "wsp");
437             break;
438           case 'x':
439             if (LIKELY(operand != rxzr))
440               snprintf(tbuf, arraysize(tbuf), "x%d", operand & RegStorage::kRegNumMask);
441             else
442               strcpy(tbuf, "xzr");
443             break;
444           case 'X':
445             if (LIKELY(operand != rsp))
446               snprintf(tbuf, arraysize(tbuf), "x%d", operand & RegStorage::kRegNumMask);
447             else
448               strcpy(tbuf, "sp");
449             break;
450           case 'D':
451             snprintf(tbuf, arraysize(tbuf), "%d", operand*((IS_WIDE(lir->opcode)) ? 8 : 4));
452             break;
453           case 'E':
454             snprintf(tbuf, arraysize(tbuf), "%d", operand*4);
455             break;
456           case 'F':
457             snprintf(tbuf, arraysize(tbuf), "%d", operand*2);
458             break;
459           case 'G':
460             if (LIKELY(operand == 0))
461               strcpy(tbuf, "");
462             else
463               strcpy(tbuf, (IS_WIDE(lir->opcode)) ? ", lsl #3" : ", lsl #2");
464             break;
465           case 'c':
466             strcpy(tbuf, cc_names[operand]);
467             break;
468           case 't':
469             snprintf(tbuf, arraysize(tbuf), "0x%08" PRIxPTR " (L%p)",
470                 reinterpret_cast<uintptr_t>(base_addr) + lir->offset + (operand << 2),
471                 lir->target);
472             break;
473           case 'r': {
474               bool is_wide = IS_WIDE(lir->opcode);
475               if (LIKELY(operand != rwzr && operand != rxzr)) {
476                 snprintf(tbuf, arraysize(tbuf), "%c%d", (is_wide) ? 'x' : 'w',
477                          operand & RegStorage::kRegNumMask);
478               } else {
479                 strcpy(tbuf, (is_wide) ? "xzr" : "wzr");
480               }
481             }
482             break;
483           case 'R': {
484               bool is_wide = IS_WIDE(lir->opcode);
485               if (LIKELY(operand != rwsp && operand != rsp)) {
486                 snprintf(tbuf, arraysize(tbuf), "%c%d", (is_wide) ? 'x' : 'w',
487                          operand & RegStorage::kRegNumMask);
488               } else {
489                 strcpy(tbuf, (is_wide) ? "sp" : "wsp");
490               }
491             }
492             break;
493           case 'p':
494             snprintf(tbuf, arraysize(tbuf), ".+%d (addr %#" PRIxPTR ")", 4*operand,
495                      reinterpret_cast<uintptr_t>(base_addr) + lir->offset + 4*operand);
496             break;
497           case 'T':
498             if (LIKELY(operand == 0))
499               strcpy(tbuf, "");
500             else if (operand == 1)
501               strcpy(tbuf, ", lsl #12");
502             else
503               strcpy(tbuf, ", DecodeError3");
504             break;
505           default:
506             strcpy(tbuf, "DecodeError1");
507             break;
508        }
509        buf += tbuf;
510      }
511    } else {
512       buf += *fmt++;
513    }
514  }
515  return buf;
516}
517
518void Arm64Mir2Lir::DumpResourceMask(LIR* arm_lir, const ResourceMask& mask, const char* prefix) {
519  char buf[256];
520  buf[0] = 0;
521
522  if (mask.Equals(kEncodeAll)) {
523    strcpy(buf, "all");
524  } else {
525    char num[8];
526    int i;
527
528    for (i = 0; i < kArm64RegEnd; i++) {
529      if (mask.HasBit(i)) {
530        snprintf(num, arraysize(num), "%d ", i);
531        strcat(buf, num);
532      }
533    }
534
535    if (mask.HasBit(ResourceMask::kCCode)) {
536      strcat(buf, "cc ");
537    }
538    if (mask.HasBit(ResourceMask::kFPStatus)) {
539      strcat(buf, "fpcc ");
540    }
541
542    /* Memory bits */
543    if (arm_lir && (mask.HasBit(ResourceMask::kDalvikReg))) {
544      snprintf(buf + strlen(buf), arraysize(buf) - strlen(buf), "dr%d%s",
545               DECODE_ALIAS_INFO_REG(arm_lir->flags.alias_info),
546               DECODE_ALIAS_INFO_WIDE(arm_lir->flags.alias_info) ? "(+1)" : "");
547    }
548    if (mask.HasBit(ResourceMask::kLiteral)) {
549      strcat(buf, "lit ");
550    }
551
552    if (mask.HasBit(ResourceMask::kHeapRef)) {
553      strcat(buf, "heap ");
554    }
555    if (mask.HasBit(ResourceMask::kMustNotAlias)) {
556      strcat(buf, "noalias ");
557    }
558  }
559  if (buf[0]) {
560    LOG(INFO) << prefix << ": " << buf;
561  }
562}
563
564bool Arm64Mir2Lir::IsUnconditionalBranch(LIR* lir) {
565  return (lir->opcode == kA64B1t);
566}
567
568bool Arm64Mir2Lir::SupportsVolatileLoadStore(OpSize size) {
569  return true;
570}
571
572RegisterClass Arm64Mir2Lir::RegClassForFieldLoadStore(OpSize size, bool is_volatile) {
573  if (UNLIKELY(is_volatile)) {
574    // On arm64, fp register load/store is atomic only for single bytes.
575    if (size != kSignedByte && size != kUnsignedByte) {
576      return (size == kReference) ? kRefReg : kCoreReg;
577    }
578  }
579  return RegClassBySize(size);
580}
581
582Arm64Mir2Lir::Arm64Mir2Lir(CompilationUnit* cu, MIRGraph* mir_graph, ArenaAllocator* arena)
583    : Mir2Lir(cu, mir_graph, arena) {
584  // Sanity check - make sure encoding map lines up.
585  for (int i = 0; i < kA64Last; i++) {
586    if (UNWIDE(Arm64Mir2Lir::EncodingMap[i].opcode) != i) {
587      LOG(FATAL) << "Encoding order for " << Arm64Mir2Lir::EncodingMap[i].name
588                 << " is wrong: expecting " << i << ", seeing "
589                 << static_cast<int>(Arm64Mir2Lir::EncodingMap[i].opcode);
590    }
591  }
592}
593
594Mir2Lir* Arm64CodeGenerator(CompilationUnit* const cu, MIRGraph* const mir_graph,
595                            ArenaAllocator* const arena) {
596  return new Arm64Mir2Lir(cu, mir_graph, arena);
597}
598
599void Arm64Mir2Lir::CompilerInitializeRegAlloc() {
600  reg_pool_ = new (arena_) RegisterPool(this, arena_, core_regs, core64_regs, sp_regs, dp_regs,
601                                        reserved_regs, reserved64_regs, core_temps, core64_temps,
602                                        sp_temps, dp_temps);
603
604  // Target-specific adjustments.
605  // Alias single precision float registers to corresponding double registers.
606  GrowableArray<RegisterInfo*>::Iterator fp_it(&reg_pool_->sp_regs_);
607  for (RegisterInfo* info = fp_it.Next(); info != nullptr; info = fp_it.Next()) {
608    int fp_reg_num = info->GetReg().GetRegNum();
609    RegStorage dp_reg = RegStorage::FloatSolo64(fp_reg_num);
610    RegisterInfo* dp_reg_info = GetRegInfo(dp_reg);
611    // Double precision register's master storage should refer to itself.
612    DCHECK_EQ(dp_reg_info, dp_reg_info->Master());
613    // Redirect single precision's master storage to master.
614    info->SetMaster(dp_reg_info);
615    // Singles should show a single 32-bit mask bit, at first referring to the low half.
616    DCHECK_EQ(info->StorageMask(), 0x1U);
617  }
618
619  // Alias 32bit W registers to corresponding 64bit X registers.
620  GrowableArray<RegisterInfo*>::Iterator w_it(&reg_pool_->core_regs_);
621  for (RegisterInfo* info = w_it.Next(); info != nullptr; info = w_it.Next()) {
622    int x_reg_num = info->GetReg().GetRegNum();
623    RegStorage x_reg = RegStorage::Solo64(x_reg_num);
624    RegisterInfo* x_reg_info = GetRegInfo(x_reg);
625    // 64bit X register's master storage should refer to itself.
626    DCHECK_EQ(x_reg_info, x_reg_info->Master());
627    // Redirect 32bit W master storage to 64bit X.
628    info->SetMaster(x_reg_info);
629    // 32bit W should show a single 32-bit mask bit, at first referring to the low half.
630    DCHECK_EQ(info->StorageMask(), 0x1U);
631  }
632
633  // Don't start allocating temps at r0/s0/d0 or you may clobber return regs in early-exit methods.
634  // TODO: adjust when we roll to hard float calling convention.
635  reg_pool_->next_core_reg_ = 2;
636  reg_pool_->next_sp_reg_ = 0;
637  reg_pool_->next_dp_reg_ = 0;
638}
639
640/*
641 * TUNING: is true leaf?  Can't just use METHOD_IS_LEAF to determine as some
642 * instructions might call out to C/assembly helper functions.  Until
643 * machinery is in place, always spill lr.
644 */
645
646void Arm64Mir2Lir::AdjustSpillMask() {
647  core_spill_mask_ |= (1 << rs_rA64_LR.GetRegNum());
648  num_core_spills_++;
649}
650
651/*
652 * Mark a callee-save fp register as promoted.
653 */
654void Arm64Mir2Lir::MarkPreservedSingle(int v_reg, RegStorage reg) {
655  DCHECK(reg.IsFloat());
656  int adjusted_reg_num = reg.GetRegNum() - A64_FP_CALLEE_SAVE_BASE;
657  // Ensure fp_vmap_table is large enough
658  int table_size = fp_vmap_table_.size();
659  for (int i = table_size; i < (adjusted_reg_num + 1); i++) {
660    fp_vmap_table_.push_back(INVALID_VREG);
661  }
662  // Add the current mapping
663  fp_vmap_table_[adjusted_reg_num] = v_reg;
664  // Size of fp_vmap_table is high-water mark, use to set mask
665  num_fp_spills_ = fp_vmap_table_.size();
666  fp_spill_mask_ = ((1 << num_fp_spills_) - 1) << A64_FP_CALLEE_SAVE_BASE;
667}
668
669void Arm64Mir2Lir::MarkPreservedDouble(int v_reg, RegStorage reg) {
670  DCHECK(reg.IsDouble());
671  MarkPreservedSingle(v_reg, reg);
672}
673
674/* Clobber all regs that might be used by an external C call */
675void Arm64Mir2Lir::ClobberCallerSave() {
676  Clobber(rs_x0);
677  Clobber(rs_x1);
678  Clobber(rs_x2);
679  Clobber(rs_x3);
680  Clobber(rs_x4);
681  Clobber(rs_x5);
682  Clobber(rs_x6);
683  Clobber(rs_x7);
684  Clobber(rs_x8);
685  Clobber(rs_x9);
686  Clobber(rs_x10);
687  Clobber(rs_x11);
688  Clobber(rs_x12);
689  Clobber(rs_x13);
690  Clobber(rs_x14);
691  Clobber(rs_x15);
692  Clobber(rs_x16);
693  Clobber(rs_x17);
694  Clobber(rs_x30);
695
696  Clobber(rs_f0);
697  Clobber(rs_f1);
698  Clobber(rs_f2);
699  Clobber(rs_f3);
700  Clobber(rs_f4);
701  Clobber(rs_f5);
702  Clobber(rs_f6);
703  Clobber(rs_f7);
704  Clobber(rs_f16);
705  Clobber(rs_f17);
706  Clobber(rs_f18);
707  Clobber(rs_f19);
708  Clobber(rs_f20);
709  Clobber(rs_f21);
710  Clobber(rs_f22);
711  Clobber(rs_f23);
712  Clobber(rs_f24);
713  Clobber(rs_f25);
714  Clobber(rs_f26);
715  Clobber(rs_f27);
716  Clobber(rs_f28);
717  Clobber(rs_f29);
718  Clobber(rs_f30);
719  Clobber(rs_f31);
720}
721
722RegLocation Arm64Mir2Lir::GetReturnWideAlt() {
723  RegLocation res = LocCReturnWide();
724  res.reg.SetReg(rx2);
725  res.reg.SetHighReg(rx3);
726  Clobber(rs_x2);
727  Clobber(rs_x3);
728  MarkInUse(rs_x2);
729  MarkInUse(rs_x3);
730  MarkWide(res.reg);
731  return res;
732}
733
734RegLocation Arm64Mir2Lir::GetReturnAlt() {
735  RegLocation res = LocCReturn();
736  res.reg.SetReg(rx1);
737  Clobber(rs_x1);
738  MarkInUse(rs_x1);
739  return res;
740}
741
742/* To be used when explicitly managing register use */
743void Arm64Mir2Lir::LockCallTemps() {
744  // TODO: needs cleanup.
745  LockTemp(rs_x0);
746  LockTemp(rs_x1);
747  LockTemp(rs_x2);
748  LockTemp(rs_x3);
749  LockTemp(rs_x4);
750  LockTemp(rs_x5);
751  LockTemp(rs_x6);
752  LockTemp(rs_x7);
753  LockTemp(rs_f0);
754  LockTemp(rs_f1);
755  LockTemp(rs_f2);
756  LockTemp(rs_f3);
757  LockTemp(rs_f4);
758  LockTemp(rs_f5);
759  LockTemp(rs_f6);
760  LockTemp(rs_f7);
761}
762
763/* To be used when explicitly managing register use */
764void Arm64Mir2Lir::FreeCallTemps() {
765  // TODO: needs cleanup.
766  FreeTemp(rs_x0);
767  FreeTemp(rs_x1);
768  FreeTemp(rs_x2);
769  FreeTemp(rs_x3);
770  FreeTemp(rs_x4);
771  FreeTemp(rs_x5);
772  FreeTemp(rs_x6);
773  FreeTemp(rs_x7);
774  FreeTemp(rs_f0);
775  FreeTemp(rs_f1);
776  FreeTemp(rs_f2);
777  FreeTemp(rs_f3);
778  FreeTemp(rs_f4);
779  FreeTemp(rs_f5);
780  FreeTemp(rs_f6);
781  FreeTemp(rs_f7);
782}
783
784RegStorage Arm64Mir2Lir::LoadHelper(ThreadOffset<4> offset) {
785  UNIMPLEMENTED(FATAL) << "Should not be called.";
786  return RegStorage::InvalidReg();
787}
788
789RegStorage Arm64Mir2Lir::LoadHelper(ThreadOffset<8> offset) {
790  // TODO(Arm64): use LoadWordDisp instead.
791  //   e.g. LoadWordDisp(rs_rA64_SELF, offset.Int32Value(), rs_rA64_LR);
792  LoadBaseDisp(rs_rA64_SELF, offset.Int32Value(), rs_rA64_LR, k64);
793  return rs_rA64_LR;
794}
795
796LIR* Arm64Mir2Lir::CheckSuspendUsingLoad() {
797  RegStorage tmp = rs_x0;
798  LoadWordDisp(rs_rA64_SELF, Thread::ThreadSuspendTriggerOffset<8>().Int32Value(), tmp);
799  LIR* load2 = LoadWordDisp(tmp, 0, tmp);
800  return load2;
801}
802
803uint64_t Arm64Mir2Lir::GetTargetInstFlags(int opcode) {
804  DCHECK(!IsPseudoLirOp(opcode));
805  return Arm64Mir2Lir::EncodingMap[UNWIDE(opcode)].flags;
806}
807
808const char* Arm64Mir2Lir::GetTargetInstName(int opcode) {
809  DCHECK(!IsPseudoLirOp(opcode));
810  return Arm64Mir2Lir::EncodingMap[UNWIDE(opcode)].name;
811}
812
813const char* Arm64Mir2Lir::GetTargetInstFmt(int opcode) {
814  DCHECK(!IsPseudoLirOp(opcode));
815  return Arm64Mir2Lir::EncodingMap[UNWIDE(opcode)].fmt;
816}
817
818RegStorage Arm64Mir2Lir::InToRegStorageArm64Mapper::GetNextReg(bool is_double_or_float,
819                                                               bool is_wide) {
820  const RegStorage coreArgMappingToPhysicalReg[] =
821      {rs_x1, rs_x2, rs_x3, rs_x4, rs_x5, rs_x6, rs_x7};
822  const int coreArgMappingToPhysicalRegSize =
823      sizeof(coreArgMappingToPhysicalReg) / sizeof(RegStorage);
824  const RegStorage fpArgMappingToPhysicalReg[] =
825      {rs_f0, rs_f1, rs_f2, rs_f3, rs_f4, rs_f5, rs_f6, rs_f7};
826  const int fpArgMappingToPhysicalRegSize =
827      sizeof(fpArgMappingToPhysicalReg) / sizeof(RegStorage);
828
829  RegStorage result = RegStorage::InvalidReg();
830  if (is_double_or_float) {
831    if (cur_fp_reg_ < fpArgMappingToPhysicalRegSize) {
832      result = fpArgMappingToPhysicalReg[cur_fp_reg_++];
833      if (result.Valid()) {
834        // TODO: switching between widths remains a bit ugly.  Better way?
835        int res_reg = result.GetReg();
836        result = is_wide ? RegStorage::FloatSolo64(res_reg) : RegStorage::FloatSolo32(res_reg);
837      }
838    }
839  } else {
840    if (cur_core_reg_ < coreArgMappingToPhysicalRegSize) {
841      result = coreArgMappingToPhysicalReg[cur_core_reg_++];
842      if (result.Valid()) {
843        // TODO: switching between widths remains a bit ugly.  Better way?
844        int res_reg = result.GetReg();
845        result = is_wide ? RegStorage::Solo64(res_reg) : RegStorage::Solo32(res_reg);
846      }
847    }
848  }
849  return result;
850}
851
852RegStorage Arm64Mir2Lir::InToRegStorageMapping::Get(int in_position) {
853  DCHECK(IsInitialized());
854  auto res = mapping_.find(in_position);
855  return res != mapping_.end() ? res->second : RegStorage::InvalidReg();
856}
857
858void Arm64Mir2Lir::InToRegStorageMapping::Initialize(RegLocation* arg_locs, int count,
859                                                     InToRegStorageMapper* mapper) {
860  DCHECK(mapper != nullptr);
861  max_mapped_in_ = -1;
862  is_there_stack_mapped_ = false;
863  for (int in_position = 0; in_position < count; in_position++) {
864     RegStorage reg = mapper->GetNextReg(arg_locs[in_position].fp, arg_locs[in_position].wide);
865     if (reg.Valid()) {
866       mapping_[in_position] = reg;
867       max_mapped_in_ = std::max(max_mapped_in_, in_position);
868       if (reg.Is64BitSolo()) {
869         // We covered 2 args, so skip the next one
870         in_position++;
871       }
872     } else {
873       is_there_stack_mapped_ = true;
874     }
875  }
876  initialized_ = true;
877}
878
879
880// Deprecate.  Use the new mechanism.
881// TODO(Arm64): reuse info in QuickArgumentVisitor?
882static RegStorage GetArgPhysicalReg(RegLocation* loc, int* num_gpr_used, int* num_fpr_used,
883                                    OpSize* op_size) {
884  if (loc->fp) {
885    int n = *num_fpr_used;
886    if (n < 8) {
887      *num_fpr_used = n + 1;
888      RegStorage::RegStorageKind reg_kind;
889      if (loc->wide) {
890        *op_size = kDouble;
891        reg_kind = RegStorage::k64BitSolo;
892      } else {
893        *op_size = kSingle;
894        reg_kind = RegStorage::k32BitSolo;
895      }
896      return RegStorage(RegStorage::kValid | reg_kind | RegStorage::kFloatingPoint | n);
897    }
898  } else {
899    int n = *num_gpr_used;
900    if (n < 8) {
901      *num_gpr_used = n + 1;
902      if (loc->wide) {
903        *op_size = k64;
904        return RegStorage::Solo64(n);
905      } else {
906        *op_size = k32;
907        return RegStorage::Solo32(n);
908      }
909    }
910  }
911  *op_size = kWord;
912  return RegStorage::InvalidReg();
913}
914
915RegStorage Arm64Mir2Lir::GetArgMappingToPhysicalReg(int arg_num) {
916  if (!in_to_reg_storage_mapping_.IsInitialized()) {
917    int start_vreg = cu_->num_dalvik_registers - cu_->num_ins;
918    RegLocation* arg_locs = &mir_graph_->reg_location_[start_vreg];
919
920    InToRegStorageArm64Mapper mapper;
921    in_to_reg_storage_mapping_.Initialize(arg_locs, cu_->num_ins, &mapper);
922  }
923  return in_to_reg_storage_mapping_.Get(arg_num);
924}
925
926
927/*
928 * If there are any ins passed in registers that have not been promoted
929 * to a callee-save register, flush them to the frame.  Perform initial
930 * assignment of promoted arguments.
931 *
932 * ArgLocs is an array of location records describing the incoming arguments
933 * with one location record per word of argument.
934 */
935void Arm64Mir2Lir::FlushIns(RegLocation* ArgLocs, RegLocation rl_method) {
936  int num_gpr_used = 1;
937  int num_fpr_used = 0;
938
939  /*
940   * Dummy up a RegLocation for the incoming StackReference<mirror::ArtMethod>
941   * It will attempt to keep kArg0 live (or copy it to home location
942   * if promoted).
943   */
944  RegLocation rl_src = rl_method;
945  rl_src.location = kLocPhysReg;
946  rl_src.reg = TargetReg(kArg0);
947  rl_src.home = false;
948  MarkLive(rl_src);
949  StoreValue(rl_method, rl_src);
950  // If Method* has been promoted, explicitly flush
951  if (rl_method.location == kLocPhysReg) {
952    StoreRefDisp(TargetReg(kSp), 0, TargetReg(kArg0));
953  }
954
955  if (cu_->num_ins == 0) {
956    return;
957  }
958
959  // Handle dalvik registers.
960  ScopedMemRefType mem_ref_type(this, ResourceMask::kDalvikReg);
961  int start_vreg = cu_->num_dalvik_registers - cu_->num_ins;
962  for (int i = 0; i < cu_->num_ins; i++) {
963    PromotionMap* v_map = &promotion_map_[start_vreg + i];
964    RegLocation* t_loc = &ArgLocs[i];
965    OpSize op_size;
966    RegStorage reg = GetArgPhysicalReg(t_loc, &num_gpr_used, &num_fpr_used, &op_size);
967
968    if (reg.Valid()) {
969      if ((v_map->core_location == kLocPhysReg) && !t_loc->fp) {
970        OpRegCopy(RegStorage::Solo32(v_map->core_reg), reg);
971      } else if ((v_map->fp_location == kLocPhysReg) && t_loc->fp) {
972        OpRegCopy(RegStorage::Solo32(v_map->FpReg), reg);
973      } else {
974        StoreBaseDisp(TargetReg(kSp), SRegOffset(start_vreg + i), reg, op_size);
975        if (reg.Is64Bit()) {
976          if (SRegOffset(start_vreg + i) + 4 != SRegOffset(start_vreg + i + 1)) {
977            LOG(FATAL) << "64 bit value stored in non-consecutive 4 bytes slots";
978          }
979          i += 1;
980        }
981      }
982    } else {
983      // If arriving in frame & promoted
984      if (v_map->core_location == kLocPhysReg) {
985        LoadWordDisp(TargetReg(kSp), SRegOffset(start_vreg + i),
986                     RegStorage::Solo32(v_map->core_reg));
987      }
988      if (v_map->fp_location == kLocPhysReg) {
989        LoadWordDisp(TargetReg(kSp), SRegOffset(start_vreg + i), RegStorage::Solo32(v_map->FpReg));
990      }
991    }
992  }
993}
994
995/*
996 * Load up to 5 arguments, the first three of which will be in
997 * kArg1 .. kArg3.  On entry kArg0 contains the current method pointer,
998 * and as part of the load sequence, it must be replaced with
999 * the target method pointer.
1000 */
1001int Arm64Mir2Lir::GenDalvikArgsNoRange(CallInfo* info,
1002                                       int call_state, LIR** pcrLabel, NextCallInsn next_call_insn,
1003                                       const MethodReference& target_method,
1004                                       uint32_t vtable_idx, uintptr_t direct_code,
1005                                       uintptr_t direct_method, InvokeType type, bool skip_this) {
1006  return GenDalvikArgsRange(info,
1007                       call_state, pcrLabel, next_call_insn,
1008                       target_method,
1009                       vtable_idx, direct_code,
1010                       direct_method, type, skip_this);
1011}
1012
1013/*
1014 * May have 0+ arguments (also used for jumbo).  Note that
1015 * source virtual registers may be in physical registers, so may
1016 * need to be flushed to home location before copying.  This
1017 * applies to arg3 and above (see below).
1018 *
1019 * FIXME: update comments.
1020 *
1021 * Two general strategies:
1022 *    If < 20 arguments
1023 *       Pass args 3-18 using vldm/vstm block copy
1024 *       Pass arg0, arg1 & arg2 in kArg1-kArg3
1025 *    If 20+ arguments
1026 *       Pass args arg19+ using memcpy block copy
1027 *       Pass arg0, arg1 & arg2 in kArg1-kArg3
1028 *
1029 */
1030int Arm64Mir2Lir::GenDalvikArgsRange(CallInfo* info, int call_state,
1031                                     LIR** pcrLabel, NextCallInsn next_call_insn,
1032                                     const MethodReference& target_method,
1033                                     uint32_t vtable_idx, uintptr_t direct_code,
1034                                     uintptr_t direct_method, InvokeType type, bool skip_this) {
1035  /* If no arguments, just return */
1036  if (info->num_arg_words == 0)
1037    return call_state;
1038
1039  const int start_index = skip_this ? 1 : 0;
1040
1041  InToRegStorageArm64Mapper mapper;
1042  InToRegStorageMapping in_to_reg_storage_mapping;
1043  in_to_reg_storage_mapping.Initialize(info->args, info->num_arg_words, &mapper);
1044  const int last_mapped_in = in_to_reg_storage_mapping.GetMaxMappedIn();
1045  const int size_of_the_last_mapped = last_mapped_in == -1 ? 1 :
1046          in_to_reg_storage_mapping.Get(last_mapped_in).Is64BitSolo() ? 2 : 1;
1047  int regs_left_to_pass_via_stack = info->num_arg_words - (last_mapped_in + size_of_the_last_mapped);
1048
1049  // Fisrt of all, check whether it make sense to use bulk copying
1050  // Optimization is aplicable only for range case
1051  // TODO: make a constant instead of 2
1052  if (info->is_range && regs_left_to_pass_via_stack >= 2) {
1053    // Scan the rest of the args - if in phys_reg flush to memory
1054    for (int next_arg = last_mapped_in + size_of_the_last_mapped; next_arg < info->num_arg_words;) {
1055      RegLocation loc = info->args[next_arg];
1056      if (loc.wide) {
1057        loc = UpdateLocWide(loc);
1058        if (loc.location == kLocPhysReg) {
1059          ScopedMemRefType mem_ref_type(this, ResourceMask::kDalvikReg);
1060          StoreBaseDisp(TargetReg(kSp), SRegOffset(loc.s_reg_low), loc.reg, k64);
1061        }
1062        next_arg += 2;
1063      } else {
1064        loc = UpdateLoc(loc);
1065        if (loc.location == kLocPhysReg) {
1066          ScopedMemRefType mem_ref_type(this, ResourceMask::kDalvikReg);
1067          StoreBaseDisp(TargetReg(kSp), SRegOffset(loc.s_reg_low), loc.reg, k32);
1068        }
1069        next_arg++;
1070      }
1071    }
1072
1073    // Logic below assumes that Method pointer is at offset zero from SP.
1074    DCHECK_EQ(VRegOffset(static_cast<int>(kVRegMethodPtrBaseReg)), 0);
1075
1076    // The rest can be copied together
1077    int start_offset = SRegOffset(info->args[last_mapped_in + size_of_the_last_mapped].s_reg_low);
1078    int outs_offset = StackVisitor::GetOutVROffset(last_mapped_in + size_of_the_last_mapped,
1079                                                   cu_->instruction_set);
1080
1081    int current_src_offset = start_offset;
1082    int current_dest_offset = outs_offset;
1083
1084    // Only davik regs are accessed in this loop; no next_call_insn() calls.
1085    ScopedMemRefType mem_ref_type(this, ResourceMask::kDalvikReg);
1086    while (regs_left_to_pass_via_stack > 0) {
1087      /*
1088       * TODO: Improve by adding block copy for large number of arguments.  This
1089       * should be done, if possible, as a target-depending helper.  For now, just
1090       * copy a Dalvik vreg at a time.
1091       */
1092      // Moving 32-bits via general purpose register.
1093      size_t bytes_to_move = sizeof(uint32_t);
1094
1095      // Instead of allocating a new temp, simply reuse one of the registers being used
1096      // for argument passing.
1097      RegStorage temp = TargetReg(kArg3);
1098
1099      // Now load the argument VR and store to the outs.
1100      Load32Disp(TargetReg(kSp), current_src_offset, temp);
1101      Store32Disp(TargetReg(kSp), current_dest_offset, temp);
1102
1103      current_src_offset += bytes_to_move;
1104      current_dest_offset += bytes_to_move;
1105      regs_left_to_pass_via_stack -= (bytes_to_move >> 2);
1106    }
1107    DCHECK_EQ(regs_left_to_pass_via_stack, 0);
1108  }
1109
1110  // Now handle rest not registers if they are
1111  if (in_to_reg_storage_mapping.IsThereStackMapped()) {
1112    RegStorage regSingle = TargetReg(kArg2);
1113    RegStorage regWide = RegStorage::Solo64(TargetReg(kArg3).GetReg());
1114    for (int i = start_index; i <= last_mapped_in + regs_left_to_pass_via_stack; i++) {
1115      RegLocation rl_arg = info->args[i];
1116      rl_arg = UpdateRawLoc(rl_arg);
1117      RegStorage reg = in_to_reg_storage_mapping.Get(i);
1118      if (!reg.Valid()) {
1119        int out_offset = StackVisitor::GetOutVROffset(i, cu_->instruction_set);
1120
1121        {
1122          ScopedMemRefType mem_ref_type(this, ResourceMask::kDalvikReg);
1123          if (rl_arg.wide) {
1124            if (rl_arg.location == kLocPhysReg) {
1125              StoreBaseDisp(TargetReg(kSp), out_offset, rl_arg.reg, k64);
1126            } else {
1127              LoadValueDirectWideFixed(rl_arg, regWide);
1128              StoreBaseDisp(TargetReg(kSp), out_offset, regWide, k64);
1129            }
1130            i++;
1131          } else {
1132            if (rl_arg.location == kLocPhysReg) {
1133              StoreBaseDisp(TargetReg(kSp), out_offset, rl_arg.reg, k32);
1134            } else {
1135              LoadValueDirectFixed(rl_arg, regSingle);
1136              StoreBaseDisp(TargetReg(kSp), out_offset, regSingle, k32);
1137            }
1138          }
1139        }
1140        call_state = next_call_insn(cu_, info, call_state, target_method,
1141                                    vtable_idx, direct_code, direct_method, type);
1142      }
1143    }
1144  }
1145
1146  // Finish with mapped registers
1147  for (int i = start_index; i <= last_mapped_in; i++) {
1148    RegLocation rl_arg = info->args[i];
1149    rl_arg = UpdateRawLoc(rl_arg);
1150    RegStorage reg = in_to_reg_storage_mapping.Get(i);
1151    if (reg.Valid()) {
1152      if (rl_arg.wide) {
1153        LoadValueDirectWideFixed(rl_arg, reg);
1154        i++;
1155      } else {
1156        LoadValueDirectFixed(rl_arg, reg);
1157      }
1158      call_state = next_call_insn(cu_, info, call_state, target_method, vtable_idx,
1159                               direct_code, direct_method, type);
1160    }
1161  }
1162
1163  call_state = next_call_insn(cu_, info, call_state, target_method, vtable_idx,
1164                           direct_code, direct_method, type);
1165  if (pcrLabel) {
1166    if (cu_->compiler_driver->GetCompilerOptions().GetExplicitNullChecks()) {
1167      *pcrLabel = GenExplicitNullCheck(TargetReg(kArg1), info->opt_flags);
1168    } else {
1169      *pcrLabel = nullptr;
1170      // In lieu of generating a check for kArg1 being null, we need to
1171      // perform a load when doing implicit checks.
1172      RegStorage tmp = AllocTemp();
1173      Load32Disp(TargetReg(kArg1), 0, tmp);
1174      MarkPossibleNullPointerException(info->opt_flags);
1175      FreeTemp(tmp);
1176    }
1177  }
1178  return call_state;
1179}
1180
1181}  // namespace art
1182