utility_arm64.cc revision 2689fbad6b5ec1ae8f8c8791a80c6fd3cf24144d 
1/*
2 * Copyright (C) 2011 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *      http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17#include "arm64_lir.h"
18#include "codegen_arm64.h"
19#include "dex/quick/mir_to_lir-inl.h"
20
21namespace art {
22
23/* This file contains codegen for the A64 ISA. */
24
25static int32_t EncodeImmSingle(uint32_t bits) {
26  /*
27   * Valid values will have the form:
28   *
29   *   aBbb.bbbc.defg.h000.0000.0000.0000.0000
30   *
31   * where B = not(b). In other words, if b == 1, then B == 0 and viceversa.
32   */
33
34  // bits[19..0] are cleared.
35  if ((bits & 0x0007ffff) != 0)
36    return -1;
37
38  // bits[29..25] are all set or all cleared.
39  uint32_t b_pattern = (bits >> 16) & 0x3e00;
40  if (b_pattern != 0 && b_pattern != 0x3e00)
41    return -1;
42
43  // bit[30] and bit[29] are opposite.
44  if (((bits ^ (bits << 1)) & 0x40000000) == 0)
45    return -1;
46
47  // bits: aBbb.bbbc.defg.h000.0000.0000.0000.0000
48  // bit7: a000.0000
49  uint32_t bit7 = ((bits >> 31) & 0x1) << 7;
50  // bit6: 0b00.0000
51  uint32_t bit6 = ((bits >> 29) & 0x1) << 6;
52  // bit5_to_0: 00cd.efgh
53  uint32_t bit5_to_0 = (bits >> 19) & 0x3f;
54  return (bit7 | bit6 | bit5_to_0);
55}
56
57static int32_t EncodeImmDouble(uint64_t bits) {
58  /*
59   * Valid values will have the form:
60   *
61   *   aBbb.bbbb.bbcd.efgh.0000.0000.0000.0000
62   *   0000.0000.0000.0000.0000.0000.0000.0000
63   *
64   * where B = not(b).
65   */
66
67  // bits[47..0] are cleared.
68  if ((bits & UINT64_C(0xffffffffffff)) != 0)
69    return -1;
70
71  // bits[61..54] are all set or all cleared.
72  uint32_t b_pattern = (bits >> 48) & 0x3fc0;
73  if (b_pattern != 0 && b_pattern != 0x3fc0)
74    return -1;
75
76  // bit[62] and bit[61] are opposite.
77  if (((bits ^ (bits << 1)) & UINT64_C(0x4000000000000000)) == 0)
78    return -1;
79
80  // bit7: a000.0000
81  uint32_t bit7 = ((bits >> 63) & 0x1) << 7;
82  // bit6: 0b00.0000
83  uint32_t bit6 = ((bits >> 61) & 0x1) << 6;
84  // bit5_to_0: 00cd.efgh
85  uint32_t bit5_to_0 = (bits >> 48) & 0x3f;
86  return (bit7 | bit6 | bit5_to_0);
87}
88
89LIR* Arm64Mir2Lir::LoadFPConstantValue(RegStorage r_dest, int32_t value) {
90  DCHECK(r_dest.IsSingle());
91  if (value == 0) {
92    return NewLIR2(kA64Fmov2sw, r_dest.GetReg(), rwzr);
93  } else {
94    int32_t encoded_imm = EncodeImmSingle((uint32_t)value);
95    if (encoded_imm >= 0) {
96      return NewLIR2(kA64Fmov2fI, r_dest.GetReg(), encoded_imm);
97    }
98  }
99
100  LIR* data_target = ScanLiteralPool(literal_list_, value, 0);
101  if (data_target == NULL) {
102    // Wide, as we need 8B alignment.
103    data_target = AddWideData(&literal_list_, value, 0);
104  }
105
106  ScopedMemRefType mem_ref_type(this, ResourceMask::kLiteral);
107  LIR* load_pc_rel = RawLIR(current_dalvik_offset_, kA64Ldr2fp,
108                            r_dest.GetReg(), 0, 0, 0, 0, data_target);
109  AppendLIR(load_pc_rel);
110  return load_pc_rel;
111}
112
113LIR* Arm64Mir2Lir::LoadFPConstantValueWide(RegStorage r_dest, int64_t value) {
114  DCHECK(r_dest.IsDouble());
115  if (value == 0) {
116    return NewLIR2(kA64Fmov2Sx, r_dest.GetReg(), rxzr);
117  } else {
118    int32_t encoded_imm = EncodeImmDouble(value);
119    if (encoded_imm >= 0) {
120      return NewLIR2(FWIDE(kA64Fmov2fI), r_dest.GetReg(), encoded_imm);
121    }
122  }
123
124  // No short form - load from the literal pool.
125  int32_t val_lo = Low32Bits(value);
126  int32_t val_hi = High32Bits(value);
127  LIR* data_target = ScanLiteralPoolWide(literal_list_, val_lo, val_hi);
128  if (data_target == NULL) {
129    data_target = AddWideData(&literal_list_, val_lo, val_hi);
130  }
131
132  ScopedMemRefType mem_ref_type(this, ResourceMask::kLiteral);
133  LIR* load_pc_rel = RawLIR(current_dalvik_offset_, FWIDE(kA64Ldr2fp),
134                            r_dest.GetReg(), 0, 0, 0, 0, data_target);
135  AppendLIR(load_pc_rel);
136  return load_pc_rel;
137}
138
139static int CountLeadingZeros(bool is_wide, uint64_t value) {
140  return (is_wide) ? __builtin_clzll(value) : __builtin_clz((uint32_t)value);
141}
142
143static int CountTrailingZeros(bool is_wide, uint64_t value) {
144  return (is_wide) ? __builtin_ctzll(value) : __builtin_ctz((uint32_t)value);
145}
146
147static int CountSetBits(bool is_wide, uint64_t value) {
148  return ((is_wide) ?
149          __builtin_popcountll(value) : __builtin_popcount((uint32_t)value));
150}
151
152/**
153 * @brief Try encoding an immediate in the form required by logical instructions.
154 *
155 * @param is_wide Whether @p value is a 64-bit (as opposed to 32-bit) value.
156 * @param value An integer to be encoded. This is interpreted as 64-bit if @p is_wide is true and as
157 *   32-bit if @p is_wide is false.
158 * @return A non-negative integer containing the encoded immediate or -1 if the encoding failed.
159 * @note This is the inverse of Arm64Mir2Lir::DecodeLogicalImmediate().
160 */
161int Arm64Mir2Lir::EncodeLogicalImmediate(bool is_wide, uint64_t value) {
162  unsigned n, imm_s, imm_r;
163
164  // Logical immediates are encoded using parameters n, imm_s and imm_r using
165  // the following table:
166  //
167  //  N   imms    immr    size        S             R
168  //  1  ssssss  rrrrrr    64    UInt(ssssss)  UInt(rrrrrr)
169  //  0  0sssss  xrrrrr    32    UInt(sssss)   UInt(rrrrr)
170  //  0  10ssss  xxrrrr    16    UInt(ssss)    UInt(rrrr)
171  //  0  110sss  xxxrrr     8    UInt(sss)     UInt(rrr)
172  //  0  1110ss  xxxxrr     4    UInt(ss)      UInt(rr)
173  //  0  11110s  xxxxxr     2    UInt(s)       UInt(r)
174  // (s bits must not be all set)
175  //
176  // A pattern is constructed of size bits, where the least significant S+1
177  // bits are set. The pattern is rotated right by R, and repeated across a
178  // 32 or 64-bit value, depending on destination register width.
179  //
180  // To test if an arbitary immediate can be encoded using this scheme, an
181  // iterative algorithm is used.
182  //
183
184  // 1. If the value has all set or all clear bits, it can't be encoded.
185  if (value == 0 || value == ~UINT64_C(0) ||
186      (!is_wide && (uint32_t)value == ~UINT32_C(0))) {
187    return -1;
188  }
189
190  unsigned lead_zero  = CountLeadingZeros(is_wide, value);
191  unsigned lead_one   = CountLeadingZeros(is_wide, ~value);
192  unsigned trail_zero = CountTrailingZeros(is_wide, value);
193  unsigned trail_one  = CountTrailingZeros(is_wide, ~value);
194  unsigned set_bits   = CountSetBits(is_wide, value);
195
196  // The fixed bits in the immediate s field.
197  // If width == 64 (X reg), start at 0xFFFFFF80.
198  // If width == 32 (W reg), start at 0xFFFFFFC0, as the iteration for 64-bit
199  // widths won't be executed.
200  unsigned width = (is_wide) ? 64 : 32;
201  int imm_s_fixed = (is_wide) ? -128 : -64;
202  int imm_s_mask = 0x3f;
203
204  for (;;) {
205    // 2. If the value is two bits wide, it can be encoded.
206    if (width == 2) {
207      n = 0;
208      imm_s = 0x3C;
209      imm_r = (value & 3) - 1;
210      break;
211    }
212
213    n = (width == 64) ? 1 : 0;
214    imm_s = ((imm_s_fixed | (set_bits - 1)) & imm_s_mask);
215    if ((lead_zero + set_bits) == width) {
216      imm_r = 0;
217    } else {
218      imm_r = (lead_zero > 0) ? (width - trail_zero) : lead_one;
219    }
220
221    // 3. If the sum of leading zeros, trailing zeros and set bits is
222    //    equal to the bit width of the value, it can be encoded.
223    if (lead_zero + trail_zero + set_bits == width) {
224      break;
225    }
226
227    // 4. If the sum of leading ones, trailing ones and unset bits in the
228    //    value is equal to the bit width of the value, it can be encoded.
229    if (lead_one + trail_one + (width - set_bits) == width) {
230      break;
231    }
232
233    // 5. If the most-significant half of the bitwise value is equal to
234    //    the least-significant half, return to step 2 using the
235    //    least-significant half of the value.
236    uint64_t mask = (UINT64_C(1) << (width >> 1)) - 1;
237    if ((value & mask) == ((value >> (width >> 1)) & mask)) {
238      width >>= 1;
239      set_bits >>= 1;
240      imm_s_fixed >>= 1;
241      continue;
242    }
243
244    // 6. Otherwise, the value can't be encoded.
245    return -1;
246  }
247
248  return (n << 12 | imm_r << 6 | imm_s);
249}
250
251bool Arm64Mir2Lir::InexpensiveConstantInt(int32_t value) {
252  return false;  // (ModifiedImmediate(value) >= 0) || (ModifiedImmediate(~value) >= 0);
253}
254
255bool Arm64Mir2Lir::InexpensiveConstantFloat(int32_t value) {
256  return EncodeImmSingle(value) >= 0;
257}
258
259bool Arm64Mir2Lir::InexpensiveConstantLong(int64_t value) {
260  return InexpensiveConstantInt(High32Bits(value)) && InexpensiveConstantInt(Low32Bits(value));
261}
262
263bool Arm64Mir2Lir::InexpensiveConstantDouble(int64_t value) {
264  return EncodeImmDouble(value) >= 0;
265}
266
267/*
268 * Load a immediate using one single instruction when possible; otherwise
269 * use a pair of movz and movk instructions.
270 *
271 * No additional register clobbering operation performed. Use this version when
272 * 1) r_dest is freshly returned from AllocTemp or
273 * 2) The codegen is under fixed register usage
274 */
275LIR* Arm64Mir2Lir::LoadConstantNoClobber(RegStorage r_dest, int value) {
276  LIR* res;
277
278  if (r_dest.IsFloat()) {
279    return LoadFPConstantValue(r_dest, value);
280  }
281
282  if (r_dest.Is64Bit()) {
283    return LoadConstantWide(r_dest, value);
284  }
285
286  // Loading SP/ZR with an immediate is not supported.
287  DCHECK(!A64_REG_IS_SP(r_dest.GetReg()));
288  DCHECK(!A64_REG_IS_ZR(r_dest.GetReg()));
289
290  // Compute how many movk, movz instructions are needed to load the value.
291  uint16_t high_bits = High16Bits(value);
292  uint16_t low_bits = Low16Bits(value);
293
294  bool low_fast = ((uint16_t)(low_bits + 1) <= 1);
295  bool high_fast = ((uint16_t)(high_bits + 1) <= 1);
296
297  if (LIKELY(low_fast || high_fast)) {
298    // 1 instruction is enough to load the immediate.
299    if (LIKELY(low_bits == high_bits)) {
300      // Value is either 0 or -1: we can just use wzr.
301      ArmOpcode opcode = LIKELY(low_bits == 0) ? kA64Mov2rr : kA64Mvn2rr;
302      res = NewLIR2(opcode, r_dest.GetReg(), rwzr);
303    } else {
304      uint16_t uniform_bits, useful_bits;
305      int shift;
306
307      if (LIKELY(high_fast)) {
308        shift = 0;
309        uniform_bits = high_bits;
310        useful_bits = low_bits;
311      } else {
312        shift = 1;
313        uniform_bits = low_bits;
314        useful_bits = high_bits;
315      }
316
317      if (UNLIKELY(uniform_bits != 0)) {
318        res = NewLIR3(kA64Movn3rdM, r_dest.GetReg(), ~useful_bits, shift);
319      } else {
320        res = NewLIR3(kA64Movz3rdM, r_dest.GetReg(), useful_bits, shift);
321      }
322    }
323  } else {
324    // movk, movz require 2 instructions. Try detecting logical immediates.
325    int log_imm = EncodeLogicalImmediate(/*is_wide=*/false, value);
326    if (log_imm >= 0) {
327      res = NewLIR3(kA64Orr3Rrl, r_dest.GetReg(), rwzr, log_imm);
328    } else {
329      // Use 2 instructions.
330      res = NewLIR3(kA64Movz3rdM, r_dest.GetReg(), low_bits, 0);
331      NewLIR3(kA64Movk3rdM, r_dest.GetReg(), high_bits, 1);
332    }
333  }
334
335  return res;
336}
337
338// TODO: clean up the names. LoadConstantWide() should really be LoadConstantNoClobberWide().
339LIR* Arm64Mir2Lir::LoadConstantWide(RegStorage r_dest, int64_t value) {
340  // Maximum number of instructions to use for encoding the immediate.
341  const int max_num_ops = 2;
342
343  if (r_dest.IsFloat()) {
344    return LoadFPConstantValueWide(r_dest, value);
345  }
346
347  DCHECK(r_dest.Is64Bit());
348
349  // Loading SP/ZR with an immediate is not supported.
350  DCHECK(!A64_REG_IS_SP(r_dest.GetReg()));
351  DCHECK(!A64_REG_IS_ZR(r_dest.GetReg()));
352
353  if (LIKELY(value == INT64_C(0) || value == INT64_C(-1))) {
354    // value is either 0 or -1: we can just use xzr.
355    ArmOpcode opcode = LIKELY(value == 0) ? WIDE(kA64Mov2rr) : WIDE(kA64Mvn2rr);
356    return NewLIR2(opcode, r_dest.GetReg(), rxzr);
357  }
358
359  // At least one in value's halfwords is not 0x0, nor 0xffff: find out how many.
360  int num_0000_halfwords = 0;
361  int num_ffff_halfwords = 0;
362  uint64_t uvalue = static_cast<uint64_t>(value);
363  for (int shift = 0; shift < 64; shift += 16) {
364    uint16_t halfword = static_cast<uint16_t>(uvalue >> shift);
365    if (halfword == 0)
366      num_0000_halfwords++;
367    else if (halfword == UINT16_C(0xffff))
368      num_ffff_halfwords++;
369  }
370  int num_fast_halfwords = std::max(num_0000_halfwords, num_ffff_halfwords);
371
372  if (num_fast_halfwords < 3) {
373    // A single movz/movn is not enough. Try the logical immediate route.
374    int log_imm = EncodeLogicalImmediate(/*is_wide=*/true, value);
375    if (log_imm >= 0) {
376      return NewLIR3(WIDE(kA64Orr3Rrl), r_dest.GetReg(), rxzr, log_imm);
377    }
378  }
379
380  if (num_fast_halfwords >= 4 - max_num_ops) {
381    // We can encode the number using a movz/movn followed by one or more movk.
382    ArmOpcode op;
383    uint16_t background;
384    LIR* res = nullptr;
385
386    // Decide whether to use a movz or a movn.
387    if (num_0000_halfwords >= num_ffff_halfwords) {
388      op = WIDE(kA64Movz3rdM);
389      background = 0;
390    } else {
391      op = WIDE(kA64Movn3rdM);
392      background = 0xffff;
393    }
394
395    // Emit the first instruction (movz, movn).
396    int shift;
397    for (shift = 0; shift < 4; shift++) {
398      uint16_t halfword = static_cast<uint16_t>(uvalue >> (shift << 4));
399      if (halfword != background) {
400        res = NewLIR3(op, r_dest.GetReg(), halfword ^ background, shift);
401        break;
402      }
403    }
404
405    // Emit the movk instructions.
406    for (shift++; shift < 4; shift++) {
407      uint16_t halfword = static_cast<uint16_t>(uvalue >> (shift << 4));
408      if (halfword != background) {
409        NewLIR3(WIDE(kA64Movk3rdM), r_dest.GetReg(), halfword, shift);
410      }
411    }
412    return res;
413  }
414
415  // Use the literal pool.
416  int32_t val_lo = Low32Bits(value);
417  int32_t val_hi = High32Bits(value);
418  LIR* data_target = ScanLiteralPoolWide(literal_list_, val_lo, val_hi);
419  if (data_target == NULL) {
420    data_target = AddWideData(&literal_list_, val_lo, val_hi);
421  }
422
423  ScopedMemRefType mem_ref_type(this, ResourceMask::kLiteral);
424  LIR *res = RawLIR(current_dalvik_offset_, WIDE(kA64Ldr2rp),
425                    r_dest.GetReg(), 0, 0, 0, 0, data_target);
426  AppendLIR(res);
427  return res;
428}
429
430LIR* Arm64Mir2Lir::OpUnconditionalBranch(LIR* target) {
431  LIR* res = NewLIR1(kA64B1t, 0 /* offset to be patched  during assembly */);
432  res->target = target;
433  return res;
434}
435
436LIR* Arm64Mir2Lir::OpCondBranch(ConditionCode cc, LIR* target) {
437  LIR* branch = NewLIR2(kA64B2ct, ArmConditionEncoding(cc),
438                        0 /* offset to be patched */);
439  branch->target = target;
440  return branch;
441}
442
443LIR* Arm64Mir2Lir::OpReg(OpKind op, RegStorage r_dest_src) {
444  ArmOpcode opcode = kA64Brk1d;
445  switch (op) {
446    case kOpBlx:
447      opcode = kA64Blr1x;
448      break;
449    // TODO(Arm64): port kThumbBx.
450    // case kOpBx:
451    //   opcode = kThumbBx;
452    //   break;
453    default:
454      LOG(FATAL) << "Bad opcode " << op;
455  }
456  return NewLIR1(opcode, r_dest_src.GetReg());
457}
458
459LIR* Arm64Mir2Lir::OpRegRegShift(OpKind op, RegStorage r_dest_src1, RegStorage r_src2, int shift) {
460  ArmOpcode wide = (r_dest_src1.Is64Bit()) ? WIDE(0) : UNWIDE(0);
461  CHECK_EQ(r_dest_src1.Is64Bit(), r_src2.Is64Bit());
462  ArmOpcode opcode = kA64Brk1d;
463
464  switch (op) {
465    case kOpCmn:
466      opcode = kA64Cmn3rro;
467      break;
468    case kOpCmp:
469      opcode = kA64Cmp3rro;
470      break;
471    case kOpMov:
472      opcode = kA64Mov2rr;
473      break;
474    case kOpMvn:
475      opcode = kA64Mvn2rr;
476      break;
477    case kOpNeg:
478      opcode = kA64Neg3rro;
479      break;
480    case kOpTst:
481      opcode = kA64Tst3rro;
482      break;
483    case kOpRev:
484      DCHECK_EQ(shift, 0);
485      // Binary, but rm is encoded twice.
486      return NewLIR2(kA64Rev2rr | wide, r_dest_src1.GetReg(), r_src2.GetReg());
487      break;
488    case kOpRevsh:
489      // Binary, but rm is encoded twice.
490      return NewLIR2(kA64Rev162rr | wide, r_dest_src1.GetReg(), r_src2.GetReg());
491      break;
492    case kOp2Byte:
493      DCHECK_EQ(shift, ENCODE_NO_SHIFT);
494      // "sbfx r1, r2, #imm1, #imm2" is "sbfm r1, r2, #imm1, #(imm1 + imm2 - 1)".
495      // For now we use sbfm directly.
496      return NewLIR4(kA64Sbfm4rrdd | wide, r_dest_src1.GetReg(), r_src2.GetReg(), 0, 7);
497    case kOp2Short:
498      DCHECK_EQ(shift, ENCODE_NO_SHIFT);
499      // For now we use sbfm rather than its alias, sbfx.
500      return NewLIR4(kA64Sbfm4rrdd | wide, r_dest_src1.GetReg(), r_src2.GetReg(), 0, 15);
501    case kOp2Char:
502      // "ubfx r1, r2, #imm1, #imm2" is "ubfm r1, r2, #imm1, #(imm1 + imm2 - 1)".
503      // For now we use ubfm directly.
504      DCHECK_EQ(shift, ENCODE_NO_SHIFT);
505      return NewLIR4(kA64Ubfm4rrdd | wide, r_dest_src1.GetReg(), r_src2.GetReg(), 0, 15);
506    default:
507      return OpRegRegRegShift(op, r_dest_src1, r_dest_src1, r_src2, shift);
508  }
509
510  DCHECK(!IsPseudoLirOp(opcode));
511  if (EncodingMap[opcode].flags & IS_BINARY_OP) {
512    DCHECK_EQ(shift, ENCODE_NO_SHIFT);
513    return NewLIR2(opcode | wide, r_dest_src1.GetReg(), r_src2.GetReg());
514  } else if (EncodingMap[opcode].flags & IS_TERTIARY_OP) {
515    ArmEncodingKind kind = EncodingMap[opcode].field_loc[2].kind;
516    if (kind == kFmtShift) {
517      return NewLIR3(opcode | wide, r_dest_src1.GetReg(), r_src2.GetReg(), shift);
518    }
519  }
520
521  LOG(FATAL) << "Unexpected encoding operand count";
522  return NULL;
523}
524
525LIR* Arm64Mir2Lir::OpRegRegExtend(OpKind op, RegStorage r_dest_src1, RegStorage r_src2, int extend) {
526  ArmOpcode wide = (r_dest_src1.Is64Bit()) ? WIDE(0) : UNWIDE(0);
527  ArmOpcode opcode = kA64Brk1d;
528
529  switch (op) {
530    case kOpCmn:
531      opcode = kA64Cmn3Rre;
532      break;
533    case kOpCmp:
534      opcode = kA64Cmp3Rre;
535      break;
536    default:
537      LOG(FATAL) << "Bad Opcode: " << opcode;
538      break;
539  }
540
541  DCHECK(!IsPseudoLirOp(opcode));
542  if (EncodingMap[opcode].flags & IS_TERTIARY_OP) {
543    ArmEncodingKind kind = EncodingMap[opcode].field_loc[2].kind;
544    if (kind == kFmtExtend) {
545      return NewLIR3(opcode | wide, r_dest_src1.GetReg(), r_src2.GetReg(), extend);
546    }
547  }
548
549  LOG(FATAL) << "Unexpected encoding operand count";
550  return NULL;
551}
552
553LIR* Arm64Mir2Lir::OpRegReg(OpKind op, RegStorage r_dest_src1, RegStorage r_src2) {
554  /* RegReg operations with SP in first parameter need extended register instruction form.
555   * Only CMN and CMP instructions are implemented.
556   */
557  if (r_dest_src1 == rs_rA64_SP) {
558    return OpRegRegExtend(op, r_dest_src1, r_src2, ENCODE_NO_EXTEND);
559  } else {
560    return OpRegRegShift(op, r_dest_src1, r_src2, ENCODE_NO_SHIFT);
561  }
562}
563
564LIR* Arm64Mir2Lir::OpMovRegMem(RegStorage r_dest, RegStorage r_base, int offset, MoveType move_type) {
565  UNIMPLEMENTED(FATAL);
566  return nullptr;
567}
568
569LIR* Arm64Mir2Lir::OpMovMemReg(RegStorage r_base, int offset, RegStorage r_src, MoveType move_type) {
570  UNIMPLEMENTED(FATAL);
571  return nullptr;
572}
573
574LIR* Arm64Mir2Lir::OpCondRegReg(OpKind op, ConditionCode cc, RegStorage r_dest, RegStorage r_src) {
575  LOG(FATAL) << "Unexpected use of OpCondRegReg for Arm64";
576  return NULL;
577}
578
579LIR* Arm64Mir2Lir::OpRegRegRegShift(OpKind op, RegStorage r_dest, RegStorage r_src1,
580                                    RegStorage r_src2, int shift) {
581  ArmOpcode opcode = kA64Brk1d;
582
583  switch (op) {
584    case kOpAdd:
585      opcode = kA64Add4rrro;
586      break;
587    case kOpSub:
588      opcode = kA64Sub4rrro;
589      break;
590    // case kOpRsub:
591    //   opcode = kA64RsubWWW;
592    //   break;
593    case kOpAdc:
594      opcode = kA64Adc3rrr;
595      break;
596    case kOpAnd:
597      opcode = kA64And4rrro;
598      break;
599    case kOpXor:
600      opcode = kA64Eor4rrro;
601      break;
602    case kOpMul:
603      opcode = kA64Mul3rrr;
604      break;
605    case kOpDiv:
606      opcode = kA64Sdiv3rrr;
607      break;
608    case kOpOr:
609      opcode = kA64Orr4rrro;
610      break;
611    case kOpSbc:
612      opcode = kA64Sbc3rrr;
613      break;
614    case kOpLsl:
615      opcode = kA64Lsl3rrr;
616      break;
617    case kOpLsr:
618      opcode = kA64Lsr3rrr;
619      break;
620    case kOpAsr:
621      opcode = kA64Asr3rrr;
622      break;
623    case kOpRor:
624      opcode = kA64Ror3rrr;
625      break;
626    default:
627      LOG(FATAL) << "Bad opcode: " << op;
628      break;
629  }
630
631  // The instructions above belong to two kinds:
632  // - 4-operands instructions, where the last operand is a shift/extend immediate,
633  // - 3-operands instructions with no shift/extend.
634  ArmOpcode widened_opcode = r_dest.Is64Bit() ? WIDE(opcode) : opcode;
635  CHECK_EQ(r_dest.Is64Bit(), r_src1.Is64Bit());
636  CHECK_EQ(r_dest.Is64Bit(), r_src2.Is64Bit());
637  if (EncodingMap[opcode].flags & IS_QUAD_OP) {
638    DCHECK(!IsExtendEncoding(shift));
639    return NewLIR4(widened_opcode, r_dest.GetReg(), r_src1.GetReg(), r_src2.GetReg(), shift);
640  } else {
641    DCHECK(EncodingMap[opcode].flags & IS_TERTIARY_OP);
642    DCHECK_EQ(shift, ENCODE_NO_SHIFT);
643    return NewLIR3(widened_opcode, r_dest.GetReg(), r_src1.GetReg(), r_src2.GetReg());
644  }
645}
646
647LIR* Arm64Mir2Lir::OpRegRegRegExtend(OpKind op, RegStorage r_dest, RegStorage r_src1,
648                                     RegStorage r_src2, A64RegExtEncodings ext, uint8_t amount) {
649  ArmOpcode opcode = kA64Brk1d;
650
651  switch (op) {
652    case kOpAdd:
653      opcode = kA64Add4RRre;
654      break;
655    case kOpSub:
656      opcode = kA64Sub4RRre;
657      break;
658    default:
659      LOG(FATAL) << "Unimplemented opcode: " << op;
660      break;
661  }
662  ArmOpcode widened_opcode = r_dest.Is64Bit() ? WIDE(opcode) : opcode;
663
664  if (r_dest.Is64Bit()) {
665    CHECK(r_src1.Is64Bit());
666
667    // dest determines whether the op is wide or not. Up-convert src2 when necessary.
668    // Note: this is not according to aarch64 specifications, but our encoding.
669    if (!r_src2.Is64Bit()) {
670      r_src2 = As64BitReg(r_src2);
671    }
672  } else {
673    CHECK(!r_src1.Is64Bit());
674    CHECK(!r_src2.Is64Bit());
675  }
676
677  // Sanity checks.
678  //    1) Amount is in the range 0..4
679  CHECK_LE(amount, 4);
680
681  return NewLIR4(widened_opcode, r_dest.GetReg(), r_src1.GetReg(), r_src2.GetReg(),
682                 EncodeExtend(ext, amount));
683}
684
685LIR* Arm64Mir2Lir::OpRegRegReg(OpKind op, RegStorage r_dest, RegStorage r_src1, RegStorage r_src2) {
686  return OpRegRegRegShift(op, r_dest, r_src1, r_src2, ENCODE_NO_SHIFT);
687}
688
689LIR* Arm64Mir2Lir::OpRegRegImm(OpKind op, RegStorage r_dest, RegStorage r_src1, int value) {
690  return OpRegRegImm64(op, r_dest, r_src1, static_cast<int64_t>(value));
691}
692
693LIR* Arm64Mir2Lir::OpRegRegImm64(OpKind op, RegStorage r_dest, RegStorage r_src1, int64_t value) {
694  LIR* res;
695  bool neg = (value < 0);
696  int64_t abs_value = (neg) ? -value : value;
697  ArmOpcode opcode = kA64Brk1d;
698  ArmOpcode alt_opcode = kA64Brk1d;
699  int32_t log_imm = -1;
700  bool is_wide = r_dest.Is64Bit();
701  ArmOpcode wide = (is_wide) ? WIDE(0) : UNWIDE(0);
702  int info = 0;
703
704  switch (op) {
705    case kOpLsl: {
706      // "lsl w1, w2, #imm" is an alias of "ubfm w1, w2, #(-imm MOD 32), #(31-imm)"
707      // and "lsl x1, x2, #imm" of "ubfm x1, x2, #(-imm MOD 64), #(63-imm)".
708      // For now, we just use ubfm directly.
709      int max_value = (is_wide) ? 63 : 31;
710      return NewLIR4(kA64Ubfm4rrdd | wide, r_dest.GetReg(), r_src1.GetReg(),
711                     (-value) & max_value, max_value - value);
712    }
713    case kOpLsr:
714      return NewLIR3(kA64Lsr3rrd | wide, r_dest.GetReg(), r_src1.GetReg(), value);
715    case kOpAsr:
716      return NewLIR3(kA64Asr3rrd | wide, r_dest.GetReg(), r_src1.GetReg(), value);
717    case kOpRor:
718      // "ror r1, r2, #imm" is an alias of "extr r1, r2, r2, #imm".
719      // For now, we just use extr directly.
720      return NewLIR4(kA64Extr4rrrd | wide, r_dest.GetReg(), r_src1.GetReg(), r_src1.GetReg(),
721                     value);
722    case kOpAdd:
723      neg = !neg;
724      // Note: intentional fallthrough
725    case kOpSub:
726      // Add and sub below read/write sp rather than xzr.
727      if (abs_value < 0x1000) {
728        opcode = (neg) ? kA64Add4RRdT : kA64Sub4RRdT;
729        return NewLIR4(opcode | wide, r_dest.GetReg(), r_src1.GetReg(), abs_value, 0);
730      } else if ((abs_value & UINT64_C(0xfff)) == 0 && ((abs_value >> 12) < 0x1000)) {
731        opcode = (neg) ? kA64Add4RRdT : kA64Sub4RRdT;
732        return NewLIR4(opcode | wide, r_dest.GetReg(), r_src1.GetReg(), abs_value >> 12, 1);
733      } else {
734        log_imm = -1;
735        alt_opcode = (neg) ? kA64Add4RRre : kA64Sub4RRre;
736        info = EncodeExtend(is_wide ? kA64Uxtx : kA64Uxtw, 0);
737      }
738      break;
739    // case kOpRsub:
740    //   opcode = kThumb2RsubRRI8M;
741    //   alt_opcode = kThumb2RsubRRR;
742    //   break;
743    case kOpAdc:
744      log_imm = -1;
745      alt_opcode = kA64Adc3rrr;
746      break;
747    case kOpSbc:
748      log_imm = -1;
749      alt_opcode = kA64Sbc3rrr;
750      break;
751    case kOpOr:
752      log_imm = EncodeLogicalImmediate(is_wide, value);
753      opcode = kA64Orr3Rrl;
754      alt_opcode = kA64Orr4rrro;
755      break;
756    case kOpAnd:
757      log_imm = EncodeLogicalImmediate(is_wide, value);
758      opcode = kA64And3Rrl;
759      alt_opcode = kA64And4rrro;
760      break;
761    case kOpXor:
762      log_imm = EncodeLogicalImmediate(is_wide, value);
763      opcode = kA64Eor3Rrl;
764      alt_opcode = kA64Eor4rrro;
765      break;
766    case kOpMul:
767      // TUNING: power of 2, shift & add
768      log_imm = -1;
769      alt_opcode = kA64Mul3rrr;
770      break;
771    default:
772      LOG(FATAL) << "Bad opcode: " << op;
773  }
774
775  if (log_imm >= 0) {
776    return NewLIR3(opcode | wide, r_dest.GetReg(), r_src1.GetReg(), log_imm);
777  } else {
778    RegStorage r_scratch;
779    if (is_wide) {
780      r_scratch = AllocTempWide();
781      LoadConstantWide(r_scratch, value);
782    } else {
783      r_scratch = AllocTemp();
784      LoadConstant(r_scratch, value);
785    }
786    if (EncodingMap[alt_opcode].flags & IS_QUAD_OP)
787      res = NewLIR4(alt_opcode | wide, r_dest.GetReg(), r_src1.GetReg(), r_scratch.GetReg(), info);
788    else
789      res = NewLIR3(alt_opcode | wide, r_dest.GetReg(), r_src1.GetReg(), r_scratch.GetReg());
790    FreeTemp(r_scratch);
791    return res;
792  }
793}
794
795LIR* Arm64Mir2Lir::OpRegImm(OpKind op, RegStorage r_dest_src1, int value) {
796  return OpRegImm64(op, r_dest_src1, static_cast<int64_t>(value));
797}
798
799LIR* Arm64Mir2Lir::OpRegImm64(OpKind op, RegStorage r_dest_src1, int64_t value) {
800  ArmOpcode wide = (r_dest_src1.Is64Bit()) ? WIDE(0) : UNWIDE(0);
801  ArmOpcode opcode = kA64Brk1d;
802  ArmOpcode neg_opcode = kA64Brk1d;
803  bool shift;
804  bool neg = (value < 0);
805  uint64_t abs_value = (neg) ? -value : value;
806
807  if (LIKELY(abs_value < 0x1000)) {
808    // abs_value is a 12-bit immediate.
809    shift = false;
810  } else if ((abs_value & UINT64_C(0xfff)) == 0 && ((abs_value >> 12) < 0x1000)) {
811    // abs_value is a shifted 12-bit immediate.
812    shift = true;
813    abs_value >>= 12;
814  } else if (LIKELY(abs_value < 0x1000000 && (op == kOpAdd || op == kOpSub))) {
815    // Note: It is better to use two ADD/SUB instead of loading a number to a temp register.
816    // This works for both normal registers and SP.
817    // For a frame size == 0x2468, it will be encoded as:
818    //   sub sp, #0x2000
819    //   sub sp, #0x468
820    if (neg) {
821      op = (op == kOpAdd) ? kOpSub : kOpAdd;
822    }
823    OpRegImm64(op, r_dest_src1, abs_value & (~INT64_C(0xfff)));
824    return OpRegImm64(op, r_dest_src1, abs_value & 0xfff);
825  } else if (LIKELY(A64_REG_IS_SP(r_dest_src1.GetReg()) && (op == kOpAdd || op == kOpSub))) {
826    // Note: "sub sp, sp, Xm" is not correct on arm64.
827    // We need special instructions for SP.
828    // Also operation on 32-bit SP should be avoided.
829    DCHECK(IS_WIDE(wide));
830    RegStorage r_tmp = AllocTempWide();
831    OpRegRegImm(kOpAdd, r_tmp, r_dest_src1, 0);
832    OpRegImm64(op, r_tmp, value);
833    return OpRegRegImm(kOpAdd, r_dest_src1, r_tmp, 0);
834  } else {
835    RegStorage r_tmp;
836    LIR* res;
837    if (IS_WIDE(wide)) {
838      r_tmp = AllocTempWide();
839      res = LoadConstantWide(r_tmp, value);
840    } else {
841      r_tmp = AllocTemp();
842      res = LoadConstant(r_tmp, value);
843    }
844    OpRegReg(op, r_dest_src1, r_tmp);
845    FreeTemp(r_tmp);
846    return res;
847  }
848
849  switch (op) {
850    case kOpAdd:
851      neg_opcode = kA64Sub4RRdT;
852      opcode = kA64Add4RRdT;
853      break;
854    case kOpSub:
855      neg_opcode = kA64Add4RRdT;
856      opcode = kA64Sub4RRdT;
857      break;
858    case kOpCmp:
859      neg_opcode = kA64Cmn3RdT;
860      opcode = kA64Cmp3RdT;
861      break;
862    default:
863      LOG(FATAL) << "Bad op-kind in OpRegImm: " << op;
864      break;
865  }
866
867  if (UNLIKELY(neg))
868    opcode = neg_opcode;
869
870  if (EncodingMap[opcode].flags & IS_QUAD_OP)
871    return NewLIR4(opcode | wide, r_dest_src1.GetReg(), r_dest_src1.GetReg(), abs_value,
872                   (shift) ? 1 : 0);
873  else
874    return NewLIR3(opcode | wide, r_dest_src1.GetReg(), abs_value, (shift) ? 1 : 0);
875}
876
877int Arm64Mir2Lir::EncodeShift(int shift_type, int amount) {
878  return ((shift_type & 0x3) << 7) | (amount & 0x1f);
879}
880
881int Arm64Mir2Lir::EncodeExtend(int extend_type, int amount) {
882  return  (1 << 6) | ((extend_type & 0x7) << 3) | (amount & 0x7);
883}
884
885bool Arm64Mir2Lir::IsExtendEncoding(int encoded_value) {
886  return ((1 << 6) & encoded_value) != 0;
887}
888
889LIR* Arm64Mir2Lir::LoadBaseIndexed(RegStorage r_base, RegStorage r_index, RegStorage r_dest,
890                                   int scale, OpSize size) {
891  LIR* load;
892  int expected_scale = 0;
893  ArmOpcode opcode = kA64Brk1d;
894  DCHECK(r_base.Is64Bit());
895  // TODO: need a cleaner handling of index registers here and throughout.
896  r_index = Check32BitReg(r_index);
897
898  if (r_dest.IsFloat()) {
899    if (r_dest.IsDouble()) {
900      DCHECK(size == k64 || size == kDouble);
901      expected_scale = 3;
902      opcode = FWIDE(kA64Ldr4fXxG);
903    } else {
904      DCHECK(r_dest.IsSingle());
905      DCHECK(size == k32 || size == kSingle);
906      expected_scale = 2;
907      opcode = kA64Ldr4fXxG;
908    }
909
910    DCHECK(scale == 0 || scale == expected_scale);
911    return NewLIR4(opcode, r_dest.GetReg(), r_base.GetReg(), r_index.GetReg(),
912                   (scale != 0) ? 1 : 0);
913  }
914
915  switch (size) {
916    case kDouble:
917    case kWord:
918    case k64:
919      r_dest = Check64BitReg(r_dest);
920      opcode = WIDE(kA64Ldr4rXxG);
921      expected_scale = 3;
922      break;
923    case kSingle:
924    case k32:
925    case kReference:
926      r_dest = Check32BitReg(r_dest);
927      opcode = kA64Ldr4rXxG;
928      expected_scale = 2;
929      break;
930    case kUnsignedHalf:
931      opcode = kA64Ldrh4wXxd;
932      expected_scale = 1;
933      break;
934    case kSignedHalf:
935      opcode = kA64Ldrsh4rXxd;
936      expected_scale = 1;
937      break;
938    case kUnsignedByte:
939      opcode = kA64Ldrb3wXx;
940      break;
941    case kSignedByte:
942      opcode = kA64Ldrsb3rXx;
943      break;
944    default:
945      LOG(FATAL) << "Bad size: " << size;
946  }
947
948  if (UNLIKELY(expected_scale == 0)) {
949    // This is a tertiary op (e.g. ldrb, ldrsb), it does not not support scale.
950    DCHECK_NE(EncodingMap[UNWIDE(opcode)].flags & IS_TERTIARY_OP, 0U);
951    DCHECK_EQ(scale, 0);
952    load = NewLIR3(opcode, r_dest.GetReg(), r_base.GetReg(), r_index.GetReg());
953  } else {
954    DCHECK(scale == 0 || scale == expected_scale);
955    load = NewLIR4(opcode, r_dest.GetReg(), r_base.GetReg(), r_index.GetReg(),
956                   (scale != 0) ? 1 : 0);
957  }
958
959  return load;
960}
961
962LIR* Arm64Mir2Lir::LoadRefIndexed(RegStorage r_base, RegStorage r_index, RegStorage r_dest) {
963  return LoadBaseIndexed(r_base, r_index, As32BitReg(r_dest), 2, kReference);
964}
965
966LIR* Arm64Mir2Lir::StoreBaseIndexed(RegStorage r_base, RegStorage r_index, RegStorage r_src,
967                                    int scale, OpSize size) {
968  LIR* store;
969  int expected_scale = 0;
970  ArmOpcode opcode = kA64Brk1d;
971  DCHECK(r_base.Is64Bit());
972  // TODO: need a cleaner handling of index registers here and throughout.
973  r_index = Check32BitReg(r_index);
974
975  if (r_src.IsFloat()) {
976    if (r_src.IsDouble()) {
977      DCHECK(size == k64 || size == kDouble);
978      expected_scale = 3;
979      opcode = FWIDE(kA64Str4fXxG);
980    } else {
981      DCHECK(r_src.IsSingle());
982      DCHECK(size == k32 || size == kSingle);
983      expected_scale = 2;
984      opcode = kA64Str4fXxG;
985    }
986
987    DCHECK(scale == 0 || scale == expected_scale);
988    return NewLIR4(opcode, r_src.GetReg(), r_base.GetReg(), r_index.GetReg(),
989                   (scale != 0) ? 1 : 0);
990  }
991
992  switch (size) {
993    case kDouble:     // Intentional fall-trough.
994    case kWord:       // Intentional fall-trough.
995    case k64:
996      r_src = Check64BitReg(r_src);
997      opcode = WIDE(kA64Str4rXxG);
998      expected_scale = 3;
999      break;
1000    case kSingle:     // Intentional fall-trough.
1001    case k32:         // Intentional fall-trough.
1002    case kReference:
1003      r_src = Check32BitReg(r_src);
1004      opcode = kA64Str4rXxG;
1005      expected_scale = 2;
1006      break;
1007    case kUnsignedHalf:
1008    case kSignedHalf:
1009      opcode = kA64Strh4wXxd;
1010      expected_scale = 1;
1011      break;
1012    case kUnsignedByte:
1013    case kSignedByte:
1014      opcode = kA64Strb3wXx;
1015      break;
1016    default:
1017      LOG(FATAL) << "Bad size: " << size;
1018  }
1019
1020  if (UNLIKELY(expected_scale == 0)) {
1021    // This is a tertiary op (e.g. strb), it does not not support scale.
1022    DCHECK_NE(EncodingMap[UNWIDE(opcode)].flags & IS_TERTIARY_OP, 0U);
1023    DCHECK_EQ(scale, 0);
1024    store = NewLIR3(opcode, r_src.GetReg(), r_base.GetReg(), r_index.GetReg());
1025  } else {
1026    store = NewLIR4(opcode, r_src.GetReg(), r_base.GetReg(), r_index.GetReg(),
1027                    (scale != 0) ? 1 : 0);
1028  }
1029
1030  return store;
1031}
1032
1033LIR* Arm64Mir2Lir::StoreRefIndexed(RegStorage r_base, RegStorage r_index, RegStorage r_src) {
1034  return StoreBaseIndexed(r_base, r_index, As32BitReg(r_src), 2, kReference);
1035}
1036
1037/*
1038 * Load value from base + displacement.  Optionally perform null check
1039 * on base (which must have an associated s_reg and MIR).  If not
1040 * performing null check, incoming MIR can be null.
1041 */
1042LIR* Arm64Mir2Lir::LoadBaseDispBody(RegStorage r_base, int displacement, RegStorage r_dest,
1043                                    OpSize size) {
1044  LIR* load = NULL;
1045  ArmOpcode opcode = kA64Brk1d;
1046  ArmOpcode alt_opcode = kA64Brk1d;
1047  int scale = 0;
1048
1049  switch (size) {
1050    case kDouble:     // Intentional fall-through.
1051    case kWord:       // Intentional fall-through.
1052    case k64:
1053      r_dest = Check64BitReg(r_dest);
1054      scale = 3;
1055      if (r_dest.IsFloat()) {
1056        DCHECK(r_dest.IsDouble());
1057        opcode = FWIDE(kA64Ldr3fXD);
1058        alt_opcode = FWIDE(kA64Ldur3fXd);
1059      } else {
1060        opcode = WIDE(kA64Ldr3rXD);
1061        alt_opcode = WIDE(kA64Ldur3rXd);
1062      }
1063      break;
1064    case kSingle:     // Intentional fall-through.
1065    case k32:         // Intentional fall-trough.
1066    case kReference:
1067      r_dest = Check32BitReg(r_dest);
1068      scale = 2;
1069      if (r_dest.IsFloat()) {
1070        DCHECK(r_dest.IsSingle());
1071        opcode = kA64Ldr3fXD;
1072      } else {
1073        opcode = kA64Ldr3rXD;
1074      }
1075      break;
1076    case kUnsignedHalf:
1077      scale = 1;
1078      opcode = kA64Ldrh3wXF;
1079      break;
1080    case kSignedHalf:
1081      scale = 1;
1082      opcode = kA64Ldrsh3rXF;
1083      break;
1084    case kUnsignedByte:
1085      opcode = kA64Ldrb3wXd;
1086      break;
1087    case kSignedByte:
1088      opcode = kA64Ldrsb3rXd;
1089      break;
1090    default:
1091      LOG(FATAL) << "Bad size: " << size;
1092  }
1093
1094  bool displacement_is_aligned = (displacement & ((1 << scale) - 1)) == 0;
1095  int scaled_disp = displacement >> scale;
1096  if (displacement_is_aligned && scaled_disp >= 0 && scaled_disp < 4096) {
1097    // Can use scaled load.
1098    load = NewLIR3(opcode, r_dest.GetReg(), r_base.GetReg(), scaled_disp);
1099  } else if (alt_opcode != kA64Brk1d && IS_SIGNED_IMM9(displacement)) {
1100    // Can use unscaled load.
1101    load = NewLIR3(alt_opcode, r_dest.GetReg(), r_base.GetReg(), displacement);
1102  } else {
1103    // Use long sequence.
1104    // TODO: cleaner support for index/displacement registers?  Not a reference, but must match width.
1105    RegStorage r_scratch = AllocTempWide();
1106    LoadConstantWide(r_scratch, displacement);
1107    load = LoadBaseIndexed(r_base, r_scratch, r_dest, 0, size);
1108    FreeTemp(r_scratch);
1109  }
1110
1111  // TODO: in future may need to differentiate Dalvik accesses w/ spills
1112  if (mem_ref_type_ == ResourceMask::kDalvikReg) {
1113    DCHECK(r_base == rs_rA64_SP);
1114    AnnotateDalvikRegAccess(load, displacement >> 2, true /* is_load */, r_dest.Is64Bit());
1115  }
1116  return load;
1117}
1118
1119LIR* Arm64Mir2Lir::LoadBaseDisp(RegStorage r_base, int displacement, RegStorage r_dest,
1120                                OpSize size, VolatileKind is_volatile) {
1121  // LoadBaseDisp() will emit correct insn for atomic load on arm64
1122  // assuming r_dest is correctly prepared using RegClassForFieldLoadStore().
1123
1124  LIR* load = LoadBaseDispBody(r_base, displacement, r_dest, size);
1125
1126  if (UNLIKELY(is_volatile == kVolatile)) {
1127    // Without context sensitive analysis, we must issue the most conservative barriers.
1128    // In this case, either a load or store may follow so we issue both barriers.
1129    GenMemBarrier(kLoadLoad);
1130    GenMemBarrier(kLoadStore);
1131  }
1132
1133  return load;
1134}
1135
1136LIR* Arm64Mir2Lir::LoadRefDisp(RegStorage r_base, int displacement, RegStorage r_dest,
1137                               VolatileKind is_volatile) {
1138  return LoadBaseDisp(r_base, displacement, As32BitReg(r_dest), kReference, is_volatile);
1139}
1140
1141LIR* Arm64Mir2Lir::StoreBaseDispBody(RegStorage r_base, int displacement, RegStorage r_src,
1142                                     OpSize size) {
1143  LIR* store = NULL;
1144  ArmOpcode opcode = kA64Brk1d;
1145  ArmOpcode alt_opcode = kA64Brk1d;
1146  int scale = 0;
1147
1148  switch (size) {
1149    case kDouble:     // Intentional fall-through.
1150    case kWord:       // Intentional fall-through.
1151    case k64:
1152      r_src = Check64BitReg(r_src);
1153      scale = 3;
1154      if (r_src.IsFloat()) {
1155        DCHECK(r_src.IsDouble());
1156        opcode = FWIDE(kA64Str3fXD);
1157        alt_opcode = FWIDE(kA64Stur3fXd);
1158      } else {
1159        opcode = FWIDE(kA64Str3rXD);
1160        alt_opcode = FWIDE(kA64Stur3rXd);
1161      }
1162      break;
1163    case kSingle:     // Intentional fall-through.
1164    case k32:         // Intentional fall-trough.
1165    case kReference:
1166      r_src = Check32BitReg(r_src);
1167      scale = 2;
1168      if (r_src.IsFloat()) {
1169        DCHECK(r_src.IsSingle());
1170        opcode = kA64Str3fXD;
1171      } else {
1172        opcode = kA64Str3rXD;
1173      }
1174      break;
1175    case kUnsignedHalf:
1176    case kSignedHalf:
1177      scale = 1;
1178      opcode = kA64Strh3wXF;
1179      break;
1180    case kUnsignedByte:
1181    case kSignedByte:
1182      opcode = kA64Strb3wXd;
1183      break;
1184    default:
1185      LOG(FATAL) << "Bad size: " << size;
1186  }
1187
1188  bool displacement_is_aligned = (displacement & ((1 << scale) - 1)) == 0;
1189  int scaled_disp = displacement >> scale;
1190  if (displacement_is_aligned && scaled_disp >= 0 && scaled_disp < 4096) {
1191    // Can use scaled store.
1192    store = NewLIR3(opcode, r_src.GetReg(), r_base.GetReg(), scaled_disp);
1193  } else if (alt_opcode != kA64Brk1d && IS_SIGNED_IMM9(displacement)) {
1194    // Can use unscaled store.
1195    store = NewLIR3(alt_opcode, r_src.GetReg(), r_base.GetReg(), displacement);
1196  } else {
1197    // Use long sequence.
1198    RegStorage r_scratch = AllocTempWide();
1199    LoadConstantWide(r_scratch, displacement);
1200    store = StoreBaseIndexed(r_base, r_scratch, r_src, 0, size);
1201    FreeTemp(r_scratch);
1202  }
1203
1204  // TODO: In future, may need to differentiate Dalvik & spill accesses.
1205  if (mem_ref_type_ == ResourceMask::kDalvikReg) {
1206    DCHECK(r_base == rs_rA64_SP);
1207    AnnotateDalvikRegAccess(store, displacement >> 2, false /* is_load */, r_src.Is64Bit());
1208  }
1209  return store;
1210}
1211
1212LIR* Arm64Mir2Lir::StoreBaseDisp(RegStorage r_base, int displacement, RegStorage r_src,
1213                                 OpSize size, VolatileKind is_volatile) {
1214  if (UNLIKELY(is_volatile == kVolatile)) {
1215    // There might have been a store before this volatile one so insert StoreStore barrier.
1216    GenMemBarrier(kStoreStore);
1217  }
1218
1219  // StoreBaseDisp() will emit correct insn for atomic store on arm64
1220  // assuming r_dest is correctly prepared using RegClassForFieldLoadStore().
1221
1222  LIR* store = StoreBaseDispBody(r_base, displacement, r_src, size);
1223
1224  if (UNLIKELY(is_volatile == kVolatile)) {
1225    // A load might follow the volatile store so insert a StoreLoad barrier.
1226    GenMemBarrier(kStoreLoad);
1227  }
1228
1229  return store;
1230}
1231
1232LIR* Arm64Mir2Lir::StoreRefDisp(RegStorage r_base, int displacement, RegStorage r_src,
1233                                VolatileKind is_volatile) {
1234  return StoreBaseDisp(r_base, displacement, As32BitReg(r_src), kReference, is_volatile);
1235}
1236
1237LIR* Arm64Mir2Lir::OpFpRegCopy(RegStorage r_dest, RegStorage r_src) {
1238  LOG(FATAL) << "Unexpected use of OpFpRegCopy for Arm64";
1239  return NULL;
1240}
1241
1242LIR* Arm64Mir2Lir::OpThreadMem(OpKind op, ThreadOffset<4> thread_offset) {
1243  UNIMPLEMENTED(FATAL) << "Should not be used.";
1244  return nullptr;
1245}
1246
1247LIR* Arm64Mir2Lir::OpThreadMem(OpKind op, ThreadOffset<8> thread_offset) {
1248  LOG(FATAL) << "Unexpected use of OpThreadMem for Arm64";
1249  return NULL;
1250}
1251
1252LIR* Arm64Mir2Lir::OpMem(OpKind op, RegStorage r_base, int disp) {
1253  LOG(FATAL) << "Unexpected use of OpMem for Arm64";
1254  return NULL;
1255}
1256
1257LIR* Arm64Mir2Lir::StoreBaseIndexedDisp(RegStorage r_base, RegStorage r_index, int scale,
1258                                        int displacement, RegStorage r_src, OpSize size) {
1259  LOG(FATAL) << "Unexpected use of StoreBaseIndexedDisp for Arm64";
1260  return NULL;
1261}
1262
1263LIR* Arm64Mir2Lir::OpRegMem(OpKind op, RegStorage r_dest, RegStorage r_base, int offset) {
1264  LOG(FATAL) << "Unexpected use of OpRegMem for Arm64";
1265  return NULL;
1266}
1267
1268LIR* Arm64Mir2Lir::LoadBaseIndexedDisp(RegStorage r_base, RegStorage r_index, int scale,
1269                                       int displacement, RegStorage r_dest, OpSize size) {
1270  LOG(FATAL) << "Unexpected use of LoadBaseIndexedDisp for Arm64";
1271  return NULL;
1272}
1273
1274}  // namespace art
1275