utility_arm64.cc revision 2eba1fa7e9e5f91e18ae3778d529520bd2c78d55
1/* 2 * Copyright (C) 2011 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17#include "arm64_lir.h" 18#include "codegen_arm64.h" 19#include "dex/quick/mir_to_lir-inl.h" 20#include "dex/reg_storage_eq.h" 21 22namespace art { 23 24/* This file contains codegen for the A64 ISA. */ 25 26int32_t Arm64Mir2Lir::EncodeImmSingle(uint32_t bits) { 27 /* 28 * Valid values will have the form: 29 * 30 * aBbb.bbbc.defg.h000.0000.0000.0000.0000 31 * 32 * where B = not(b). In other words, if b == 1, then B == 0 and viceversa. 33 */ 34 35 // bits[19..0] are cleared. 36 if ((bits & 0x0007ffff) != 0) 37 return -1; 38 39 // bits[29..25] are all set or all cleared. 40 uint32_t b_pattern = (bits >> 16) & 0x3e00; 41 if (b_pattern != 0 && b_pattern != 0x3e00) 42 return -1; 43 44 // bit[30] and bit[29] are opposite. 45 if (((bits ^ (bits << 1)) & 0x40000000) == 0) 46 return -1; 47 48 // bits: aBbb.bbbc.defg.h000.0000.0000.0000.0000 49 // bit7: a000.0000 50 uint32_t bit7 = ((bits >> 31) & 0x1) << 7; 51 // bit6: 0b00.0000 52 uint32_t bit6 = ((bits >> 29) & 0x1) << 6; 53 // bit5_to_0: 00cd.efgh 54 uint32_t bit5_to_0 = (bits >> 19) & 0x3f; 55 return (bit7 | bit6 | bit5_to_0); 56} 57 58int32_t Arm64Mir2Lir::EncodeImmDouble(uint64_t bits) { 59 /* 60 * Valid values will have the form: 61 * 62 * aBbb.bbbb.bbcd.efgh.0000.0000.0000.0000 63 * 0000.0000.0000.0000.0000.0000.0000.0000 64 * 65 * where B = not(b). 66 */ 67 68 // bits[47..0] are cleared. 69 if ((bits & UINT64_C(0xffffffffffff)) != 0) 70 return -1; 71 72 // bits[61..54] are all set or all cleared. 73 uint32_t b_pattern = (bits >> 48) & 0x3fc0; 74 if (b_pattern != 0 && b_pattern != 0x3fc0) 75 return -1; 76 77 // bit[62] and bit[61] are opposite. 78 if (((bits ^ (bits << 1)) & UINT64_C(0x4000000000000000)) == 0) 79 return -1; 80 81 // bit7: a000.0000 82 uint32_t bit7 = ((bits >> 63) & 0x1) << 7; 83 // bit6: 0b00.0000 84 uint32_t bit6 = ((bits >> 61) & 0x1) << 6; 85 // bit5_to_0: 00cd.efgh 86 uint32_t bit5_to_0 = (bits >> 48) & 0x3f; 87 return (bit7 | bit6 | bit5_to_0); 88} 89 90size_t Arm64Mir2Lir::GetLoadStoreSize(LIR* lir) { 91 bool opcode_is_wide = IS_WIDE(lir->opcode); 92 ArmOpcode opcode = UNWIDE(lir->opcode); 93 DCHECK(!IsPseudoLirOp(opcode)); 94 const ArmEncodingMap *encoder = &EncodingMap[opcode]; 95 uint32_t bits = opcode_is_wide ? encoder->xskeleton : encoder->wskeleton; 96 return (bits >> 30); 97} 98 99size_t Arm64Mir2Lir::GetInstructionOffset(LIR* lir) { 100 size_t offset = lir->operands[2]; 101 uint64_t check_flags = GetTargetInstFlags(lir->opcode); 102 DCHECK((check_flags & IS_LOAD) || (check_flags & IS_STORE)); 103 if (check_flags & SCALED_OFFSET_X0) { 104 DCHECK(check_flags & IS_TERTIARY_OP); 105 offset = offset * (1 << GetLoadStoreSize(lir)); 106 } 107 return offset; 108} 109 110LIR* Arm64Mir2Lir::LoadFPConstantValue(RegStorage r_dest, int32_t value) { 111 DCHECK(r_dest.IsSingle()); 112 if (value == 0) { 113 return NewLIR2(kA64Fmov2sw, r_dest.GetReg(), rwzr); 114 } else { 115 int32_t encoded_imm = EncodeImmSingle((uint32_t)value); 116 if (encoded_imm >= 0) { 117 return NewLIR2(kA64Fmov2fI, r_dest.GetReg(), encoded_imm); 118 } 119 } 120 121 LIR* data_target = ScanLiteralPool(literal_list_, value, 0); 122 if (data_target == NULL) { 123 // Wide, as we need 8B alignment. 124 data_target = AddWideData(&literal_list_, value, 0); 125 } 126 127 ScopedMemRefType mem_ref_type(this, ResourceMask::kLiteral); 128 LIR* load_pc_rel = RawLIR(current_dalvik_offset_, kA64Ldr2fp, 129 r_dest.GetReg(), 0, 0, 0, 0, data_target); 130 AppendLIR(load_pc_rel); 131 return load_pc_rel; 132} 133 134LIR* Arm64Mir2Lir::LoadFPConstantValueWide(RegStorage r_dest, int64_t value) { 135 DCHECK(r_dest.IsDouble()); 136 if (value == 0) { 137 return NewLIR2(kA64Fmov2Sx, r_dest.GetReg(), rxzr); 138 } else { 139 int32_t encoded_imm = EncodeImmDouble(value); 140 if (encoded_imm >= 0) { 141 return NewLIR2(FWIDE(kA64Fmov2fI), r_dest.GetReg(), encoded_imm); 142 } 143 } 144 145 // No short form - load from the literal pool. 146 int32_t val_lo = Low32Bits(value); 147 int32_t val_hi = High32Bits(value); 148 LIR* data_target = ScanLiteralPoolWide(literal_list_, val_lo, val_hi); 149 if (data_target == NULL) { 150 data_target = AddWideData(&literal_list_, val_lo, val_hi); 151 } 152 153 ScopedMemRefType mem_ref_type(this, ResourceMask::kLiteral); 154 LIR* load_pc_rel = RawLIR(current_dalvik_offset_, FWIDE(kA64Ldr2fp), 155 r_dest.GetReg(), 0, 0, 0, 0, data_target); 156 AppendLIR(load_pc_rel); 157 return load_pc_rel; 158} 159 160static int CountLeadingZeros(bool is_wide, uint64_t value) { 161 return (is_wide) ? __builtin_clzll(value) : __builtin_clz((uint32_t)value); 162} 163 164static int CountTrailingZeros(bool is_wide, uint64_t value) { 165 return (is_wide) ? __builtin_ctzll(value) : __builtin_ctz((uint32_t)value); 166} 167 168static int CountSetBits(bool is_wide, uint64_t value) { 169 return ((is_wide) ? 170 __builtin_popcountll(value) : __builtin_popcount((uint32_t)value)); 171} 172 173/** 174 * @brief Try encoding an immediate in the form required by logical instructions. 175 * 176 * @param is_wide Whether @p value is a 64-bit (as opposed to 32-bit) value. 177 * @param value An integer to be encoded. This is interpreted as 64-bit if @p is_wide is true and as 178 * 32-bit if @p is_wide is false. 179 * @return A non-negative integer containing the encoded immediate or -1 if the encoding failed. 180 * @note This is the inverse of Arm64Mir2Lir::DecodeLogicalImmediate(). 181 */ 182int Arm64Mir2Lir::EncodeLogicalImmediate(bool is_wide, uint64_t value) { 183 unsigned n, imm_s, imm_r; 184 185 // Logical immediates are encoded using parameters n, imm_s and imm_r using 186 // the following table: 187 // 188 // N imms immr size S R 189 // 1 ssssss rrrrrr 64 UInt(ssssss) UInt(rrrrrr) 190 // 0 0sssss xrrrrr 32 UInt(sssss) UInt(rrrrr) 191 // 0 10ssss xxrrrr 16 UInt(ssss) UInt(rrrr) 192 // 0 110sss xxxrrr 8 UInt(sss) UInt(rrr) 193 // 0 1110ss xxxxrr 4 UInt(ss) UInt(rr) 194 // 0 11110s xxxxxr 2 UInt(s) UInt(r) 195 // (s bits must not be all set) 196 // 197 // A pattern is constructed of size bits, where the least significant S+1 198 // bits are set. The pattern is rotated right by R, and repeated across a 199 // 32 or 64-bit value, depending on destination register width. 200 // 201 // To test if an arbitary immediate can be encoded using this scheme, an 202 // iterative algorithm is used. 203 // 204 205 // 1. If the value has all set or all clear bits, it can't be encoded. 206 if (value == 0 || value == ~UINT64_C(0) || 207 (!is_wide && (uint32_t)value == ~UINT32_C(0))) { 208 return -1; 209 } 210 211 unsigned lead_zero = CountLeadingZeros(is_wide, value); 212 unsigned lead_one = CountLeadingZeros(is_wide, ~value); 213 unsigned trail_zero = CountTrailingZeros(is_wide, value); 214 unsigned trail_one = CountTrailingZeros(is_wide, ~value); 215 unsigned set_bits = CountSetBits(is_wide, value); 216 217 // The fixed bits in the immediate s field. 218 // If width == 64 (X reg), start at 0xFFFFFF80. 219 // If width == 32 (W reg), start at 0xFFFFFFC0, as the iteration for 64-bit 220 // widths won't be executed. 221 unsigned width = (is_wide) ? 64 : 32; 222 int imm_s_fixed = (is_wide) ? -128 : -64; 223 int imm_s_mask = 0x3f; 224 225 for (;;) { 226 // 2. If the value is two bits wide, it can be encoded. 227 if (width == 2) { 228 n = 0; 229 imm_s = 0x3C; 230 imm_r = (value & 3) - 1; 231 break; 232 } 233 234 n = (width == 64) ? 1 : 0; 235 imm_s = ((imm_s_fixed | (set_bits - 1)) & imm_s_mask); 236 if ((lead_zero + set_bits) == width) { 237 imm_r = 0; 238 } else { 239 imm_r = (lead_zero > 0) ? (width - trail_zero) : lead_one; 240 } 241 242 // 3. If the sum of leading zeros, trailing zeros and set bits is 243 // equal to the bit width of the value, it can be encoded. 244 if (lead_zero + trail_zero + set_bits == width) { 245 break; 246 } 247 248 // 4. If the sum of leading ones, trailing ones and unset bits in the 249 // value is equal to the bit width of the value, it can be encoded. 250 if (lead_one + trail_one + (width - set_bits) == width) { 251 break; 252 } 253 254 // 5. If the most-significant half of the bitwise value is equal to 255 // the least-significant half, return to step 2 using the 256 // least-significant half of the value. 257 uint64_t mask = (UINT64_C(1) << (width >> 1)) - 1; 258 if ((value & mask) == ((value >> (width >> 1)) & mask)) { 259 width >>= 1; 260 set_bits >>= 1; 261 imm_s_fixed >>= 1; 262 continue; 263 } 264 265 // 6. Otherwise, the value can't be encoded. 266 return -1; 267 } 268 269 return (n << 12 | imm_r << 6 | imm_s); 270} 271 272bool Arm64Mir2Lir::InexpensiveConstantInt(int32_t value) { 273 return false; // (ModifiedImmediate(value) >= 0) || (ModifiedImmediate(~value) >= 0); 274} 275 276bool Arm64Mir2Lir::InexpensiveConstantFloat(int32_t value) { 277 return EncodeImmSingle(value) >= 0; 278} 279 280bool Arm64Mir2Lir::InexpensiveConstantLong(int64_t value) { 281 return InexpensiveConstantInt(High32Bits(value)) && InexpensiveConstantInt(Low32Bits(value)); 282} 283 284bool Arm64Mir2Lir::InexpensiveConstantDouble(int64_t value) { 285 return EncodeImmDouble(value) >= 0; 286} 287 288/* 289 * Load a immediate using one single instruction when possible; otherwise 290 * use a pair of movz and movk instructions. 291 * 292 * No additional register clobbering operation performed. Use this version when 293 * 1) r_dest is freshly returned from AllocTemp or 294 * 2) The codegen is under fixed register usage 295 */ 296LIR* Arm64Mir2Lir::LoadConstantNoClobber(RegStorage r_dest, int value) { 297 LIR* res; 298 299 if (r_dest.IsFloat()) { 300 return LoadFPConstantValue(r_dest, value); 301 } 302 303 if (r_dest.Is64Bit()) { 304 return LoadConstantWide(r_dest, value); 305 } 306 307 // Loading SP/ZR with an immediate is not supported. 308 DCHECK(!A64_REG_IS_SP(r_dest.GetReg())); 309 DCHECK(!A64_REG_IS_ZR(r_dest.GetReg())); 310 311 // Compute how many movk, movz instructions are needed to load the value. 312 uint16_t high_bits = High16Bits(value); 313 uint16_t low_bits = Low16Bits(value); 314 315 bool low_fast = ((uint16_t)(low_bits + 1) <= 1); 316 bool high_fast = ((uint16_t)(high_bits + 1) <= 1); 317 318 if (LIKELY(low_fast || high_fast)) { 319 // 1 instruction is enough to load the immediate. 320 if (LIKELY(low_bits == high_bits)) { 321 // Value is either 0 or -1: we can just use wzr. 322 ArmOpcode opcode = LIKELY(low_bits == 0) ? kA64Mov2rr : kA64Mvn2rr; 323 res = NewLIR2(opcode, r_dest.GetReg(), rwzr); 324 } else { 325 uint16_t uniform_bits, useful_bits; 326 int shift; 327 328 if (LIKELY(high_fast)) { 329 shift = 0; 330 uniform_bits = high_bits; 331 useful_bits = low_bits; 332 } else { 333 shift = 1; 334 uniform_bits = low_bits; 335 useful_bits = high_bits; 336 } 337 338 if (UNLIKELY(uniform_bits != 0)) { 339 res = NewLIR3(kA64Movn3rdM, r_dest.GetReg(), ~useful_bits, shift); 340 } else { 341 res = NewLIR3(kA64Movz3rdM, r_dest.GetReg(), useful_bits, shift); 342 } 343 } 344 } else { 345 // movk, movz require 2 instructions. Try detecting logical immediates. 346 int log_imm = EncodeLogicalImmediate(/*is_wide=*/false, value); 347 if (log_imm >= 0) { 348 res = NewLIR3(kA64Orr3Rrl, r_dest.GetReg(), rwzr, log_imm); 349 } else { 350 // Use 2 instructions. 351 res = NewLIR3(kA64Movz3rdM, r_dest.GetReg(), low_bits, 0); 352 NewLIR3(kA64Movk3rdM, r_dest.GetReg(), high_bits, 1); 353 } 354 } 355 356 return res; 357} 358 359// TODO: clean up the names. LoadConstantWide() should really be LoadConstantNoClobberWide(). 360LIR* Arm64Mir2Lir::LoadConstantWide(RegStorage r_dest, int64_t value) { 361 // Maximum number of instructions to use for encoding the immediate. 362 const int max_num_ops = 2; 363 364 if (r_dest.IsFloat()) { 365 return LoadFPConstantValueWide(r_dest, value); 366 } 367 368 DCHECK(r_dest.Is64Bit()); 369 370 // Loading SP/ZR with an immediate is not supported. 371 DCHECK(!A64_REG_IS_SP(r_dest.GetReg())); 372 DCHECK(!A64_REG_IS_ZR(r_dest.GetReg())); 373 374 if (LIKELY(value == INT64_C(0) || value == INT64_C(-1))) { 375 // value is either 0 or -1: we can just use xzr. 376 ArmOpcode opcode = LIKELY(value == 0) ? WIDE(kA64Mov2rr) : WIDE(kA64Mvn2rr); 377 return NewLIR2(opcode, r_dest.GetReg(), rxzr); 378 } 379 380 // At least one in value's halfwords is not 0x0, nor 0xffff: find out how many. 381 int num_0000_halfwords = 0; 382 int num_ffff_halfwords = 0; 383 uint64_t uvalue = static_cast<uint64_t>(value); 384 for (int shift = 0; shift < 64; shift += 16) { 385 uint16_t halfword = static_cast<uint16_t>(uvalue >> shift); 386 if (halfword == 0) 387 num_0000_halfwords++; 388 else if (halfword == UINT16_C(0xffff)) 389 num_ffff_halfwords++; 390 } 391 int num_fast_halfwords = std::max(num_0000_halfwords, num_ffff_halfwords); 392 393 if (num_fast_halfwords < 3) { 394 // A single movz/movn is not enough. Try the logical immediate route. 395 int log_imm = EncodeLogicalImmediate(/*is_wide=*/true, value); 396 if (log_imm >= 0) { 397 return NewLIR3(WIDE(kA64Orr3Rrl), r_dest.GetReg(), rxzr, log_imm); 398 } 399 } 400 401 if (num_fast_halfwords >= 4 - max_num_ops) { 402 // We can encode the number using a movz/movn followed by one or more movk. 403 ArmOpcode op; 404 uint16_t background; 405 LIR* res = nullptr; 406 407 // Decide whether to use a movz or a movn. 408 if (num_0000_halfwords >= num_ffff_halfwords) { 409 op = WIDE(kA64Movz3rdM); 410 background = 0; 411 } else { 412 op = WIDE(kA64Movn3rdM); 413 background = 0xffff; 414 } 415 416 // Emit the first instruction (movz, movn). 417 int shift; 418 for (shift = 0; shift < 4; shift++) { 419 uint16_t halfword = static_cast<uint16_t>(uvalue >> (shift << 4)); 420 if (halfword != background) { 421 res = NewLIR3(op, r_dest.GetReg(), halfword ^ background, shift); 422 break; 423 } 424 } 425 426 // Emit the movk instructions. 427 for (shift++; shift < 4; shift++) { 428 uint16_t halfword = static_cast<uint16_t>(uvalue >> (shift << 4)); 429 if (halfword != background) { 430 NewLIR3(WIDE(kA64Movk3rdM), r_dest.GetReg(), halfword, shift); 431 } 432 } 433 return res; 434 } 435 436 // Use the literal pool. 437 int32_t val_lo = Low32Bits(value); 438 int32_t val_hi = High32Bits(value); 439 LIR* data_target = ScanLiteralPoolWide(literal_list_, val_lo, val_hi); 440 if (data_target == NULL) { 441 data_target = AddWideData(&literal_list_, val_lo, val_hi); 442 } 443 444 ScopedMemRefType mem_ref_type(this, ResourceMask::kLiteral); 445 LIR *res = RawLIR(current_dalvik_offset_, WIDE(kA64Ldr2rp), 446 r_dest.GetReg(), 0, 0, 0, 0, data_target); 447 AppendLIR(res); 448 return res; 449} 450 451LIR* Arm64Mir2Lir::OpUnconditionalBranch(LIR* target) { 452 LIR* res = NewLIR1(kA64B1t, 0 /* offset to be patched during assembly */); 453 res->target = target; 454 return res; 455} 456 457LIR* Arm64Mir2Lir::OpCondBranch(ConditionCode cc, LIR* target) { 458 LIR* branch = NewLIR2(kA64B2ct, ArmConditionEncoding(cc), 459 0 /* offset to be patched */); 460 branch->target = target; 461 return branch; 462} 463 464LIR* Arm64Mir2Lir::OpReg(OpKind op, RegStorage r_dest_src) { 465 ArmOpcode opcode = kA64Brk1d; 466 switch (op) { 467 case kOpBlx: 468 opcode = kA64Blr1x; 469 break; 470 // TODO(Arm64): port kThumbBx. 471 // case kOpBx: 472 // opcode = kThumbBx; 473 // break; 474 default: 475 LOG(FATAL) << "Bad opcode " << op; 476 } 477 return NewLIR1(opcode, r_dest_src.GetReg()); 478} 479 480LIR* Arm64Mir2Lir::OpRegRegShift(OpKind op, RegStorage r_dest_src1, RegStorage r_src2, int shift) { 481 ArmOpcode wide = (r_dest_src1.Is64Bit()) ? WIDE(0) : UNWIDE(0); 482 CHECK_EQ(r_dest_src1.Is64Bit(), r_src2.Is64Bit()); 483 ArmOpcode opcode = kA64Brk1d; 484 485 switch (op) { 486 case kOpCmn: 487 opcode = kA64Cmn3rro; 488 break; 489 case kOpCmp: 490 opcode = kA64Cmp3rro; 491 break; 492 case kOpMov: 493 opcode = kA64Mov2rr; 494 break; 495 case kOpMvn: 496 opcode = kA64Mvn2rr; 497 break; 498 case kOpNeg: 499 opcode = kA64Neg3rro; 500 break; 501 case kOpTst: 502 opcode = kA64Tst3rro; 503 break; 504 case kOpRev: 505 DCHECK_EQ(shift, 0); 506 // Binary, but rm is encoded twice. 507 return NewLIR2(kA64Rev2rr | wide, r_dest_src1.GetReg(), r_src2.GetReg()); 508 break; 509 case kOpRevsh: 510 // Binary, but rm is encoded twice. 511 NewLIR2(kA64Rev162rr | wide, r_dest_src1.GetReg(), r_src2.GetReg()); 512 // "sxth r1, r2" is "sbfm r1, r2, #0, #15" 513 return NewLIR4(kA64Sbfm4rrdd | wide, r_dest_src1.GetReg(), r_dest_src1.GetReg(), 0, 15); 514 break; 515 case kOp2Byte: 516 DCHECK_EQ(shift, ENCODE_NO_SHIFT); 517 // "sbfx r1, r2, #imm1, #imm2" is "sbfm r1, r2, #imm1, #(imm1 + imm2 - 1)". 518 // For now we use sbfm directly. 519 return NewLIR4(kA64Sbfm4rrdd | wide, r_dest_src1.GetReg(), r_src2.GetReg(), 0, 7); 520 case kOp2Short: 521 DCHECK_EQ(shift, ENCODE_NO_SHIFT); 522 // For now we use sbfm rather than its alias, sbfx. 523 return NewLIR4(kA64Sbfm4rrdd | wide, r_dest_src1.GetReg(), r_src2.GetReg(), 0, 15); 524 case kOp2Char: 525 // "ubfx r1, r2, #imm1, #imm2" is "ubfm r1, r2, #imm1, #(imm1 + imm2 - 1)". 526 // For now we use ubfm directly. 527 DCHECK_EQ(shift, ENCODE_NO_SHIFT); 528 return NewLIR4(kA64Ubfm4rrdd | wide, r_dest_src1.GetReg(), r_src2.GetReg(), 0, 15); 529 default: 530 return OpRegRegRegShift(op, r_dest_src1, r_dest_src1, r_src2, shift); 531 } 532 533 DCHECK(!IsPseudoLirOp(opcode)); 534 if (EncodingMap[opcode].flags & IS_BINARY_OP) { 535 DCHECK_EQ(shift, ENCODE_NO_SHIFT); 536 return NewLIR2(opcode | wide, r_dest_src1.GetReg(), r_src2.GetReg()); 537 } else if (EncodingMap[opcode].flags & IS_TERTIARY_OP) { 538 ArmEncodingKind kind = EncodingMap[opcode].field_loc[2].kind; 539 if (kind == kFmtShift) { 540 return NewLIR3(opcode | wide, r_dest_src1.GetReg(), r_src2.GetReg(), shift); 541 } 542 } 543 544 LOG(FATAL) << "Unexpected encoding operand count"; 545 return NULL; 546} 547 548LIR* Arm64Mir2Lir::OpRegRegExtend(OpKind op, RegStorage r_dest_src1, RegStorage r_src2, 549 A64RegExtEncodings ext, uint8_t amount) { 550 ArmOpcode wide = (r_dest_src1.Is64Bit()) ? WIDE(0) : UNWIDE(0); 551 ArmOpcode opcode = kA64Brk1d; 552 553 switch (op) { 554 case kOpCmn: 555 opcode = kA64Cmn3Rre; 556 break; 557 case kOpCmp: 558 opcode = kA64Cmp3Rre; 559 break; 560 case kOpAdd: 561 // Note: intentional fallthrough 562 case kOpSub: 563 return OpRegRegRegExtend(op, r_dest_src1, r_dest_src1, r_src2, ext, amount); 564 break; 565 default: 566 LOG(FATAL) << "Bad Opcode: " << opcode; 567 break; 568 } 569 570 DCHECK(!IsPseudoLirOp(opcode)); 571 if (EncodingMap[opcode].flags & IS_TERTIARY_OP) { 572 ArmEncodingKind kind = EncodingMap[opcode].field_loc[2].kind; 573 if (kind == kFmtExtend) { 574 return NewLIR3(opcode | wide, r_dest_src1.GetReg(), r_src2.GetReg(), 575 EncodeExtend(ext, amount)); 576 } 577 } 578 579 LOG(FATAL) << "Unexpected encoding operand count"; 580 return NULL; 581} 582 583LIR* Arm64Mir2Lir::OpRegReg(OpKind op, RegStorage r_dest_src1, RegStorage r_src2) { 584 /* RegReg operations with SP in first parameter need extended register instruction form. 585 * Only CMN, CMP, ADD & SUB instructions are implemented. 586 */ 587 if (r_dest_src1 == rs_sp) { 588 return OpRegRegExtend(op, r_dest_src1, r_src2, kA64Uxtx, 0); 589 } else { 590 return OpRegRegShift(op, r_dest_src1, r_src2, ENCODE_NO_SHIFT); 591 } 592} 593 594LIR* Arm64Mir2Lir::OpMovRegMem(RegStorage r_dest, RegStorage r_base, int offset, MoveType move_type) { 595 UNIMPLEMENTED(FATAL); 596 return nullptr; 597} 598 599LIR* Arm64Mir2Lir::OpMovMemReg(RegStorage r_base, int offset, RegStorage r_src, MoveType move_type) { 600 UNIMPLEMENTED(FATAL); 601 return nullptr; 602} 603 604LIR* Arm64Mir2Lir::OpCondRegReg(OpKind op, ConditionCode cc, RegStorage r_dest, RegStorage r_src) { 605 LOG(FATAL) << "Unexpected use of OpCondRegReg for Arm64"; 606 return NULL; 607} 608 609LIR* Arm64Mir2Lir::OpRegRegRegShift(OpKind op, RegStorage r_dest, RegStorage r_src1, 610 RegStorage r_src2, int shift) { 611 ArmOpcode opcode = kA64Brk1d; 612 613 switch (op) { 614 case kOpAdd: 615 opcode = kA64Add4rrro; 616 break; 617 case kOpSub: 618 opcode = kA64Sub4rrro; 619 break; 620 // case kOpRsub: 621 // opcode = kA64RsubWWW; 622 // break; 623 case kOpAdc: 624 opcode = kA64Adc3rrr; 625 break; 626 case kOpAnd: 627 opcode = kA64And4rrro; 628 break; 629 case kOpXor: 630 opcode = kA64Eor4rrro; 631 break; 632 case kOpMul: 633 opcode = kA64Mul3rrr; 634 break; 635 case kOpDiv: 636 opcode = kA64Sdiv3rrr; 637 break; 638 case kOpOr: 639 opcode = kA64Orr4rrro; 640 break; 641 case kOpSbc: 642 opcode = kA64Sbc3rrr; 643 break; 644 case kOpLsl: 645 opcode = kA64Lsl3rrr; 646 break; 647 case kOpLsr: 648 opcode = kA64Lsr3rrr; 649 break; 650 case kOpAsr: 651 opcode = kA64Asr3rrr; 652 break; 653 case kOpRor: 654 opcode = kA64Ror3rrr; 655 break; 656 default: 657 LOG(FATAL) << "Bad opcode: " << op; 658 break; 659 } 660 661 // The instructions above belong to two kinds: 662 // - 4-operands instructions, where the last operand is a shift/extend immediate, 663 // - 3-operands instructions with no shift/extend. 664 ArmOpcode widened_opcode = r_dest.Is64Bit() ? WIDE(opcode) : opcode; 665 CHECK_EQ(r_dest.Is64Bit(), r_src1.Is64Bit()); 666 CHECK_EQ(r_dest.Is64Bit(), r_src2.Is64Bit()); 667 if (EncodingMap[opcode].flags & IS_QUAD_OP) { 668 DCHECK(!IsExtendEncoding(shift)); 669 return NewLIR4(widened_opcode, r_dest.GetReg(), r_src1.GetReg(), r_src2.GetReg(), shift); 670 } else { 671 DCHECK(EncodingMap[opcode].flags & IS_TERTIARY_OP); 672 DCHECK_EQ(shift, ENCODE_NO_SHIFT); 673 return NewLIR3(widened_opcode, r_dest.GetReg(), r_src1.GetReg(), r_src2.GetReg()); 674 } 675} 676 677LIR* Arm64Mir2Lir::OpRegRegRegExtend(OpKind op, RegStorage r_dest, RegStorage r_src1, 678 RegStorage r_src2, A64RegExtEncodings ext, uint8_t amount) { 679 ArmOpcode opcode = kA64Brk1d; 680 681 switch (op) { 682 case kOpAdd: 683 opcode = kA64Add4RRre; 684 break; 685 case kOpSub: 686 opcode = kA64Sub4RRre; 687 break; 688 default: 689 LOG(FATAL) << "Unimplemented opcode: " << op; 690 break; 691 } 692 ArmOpcode widened_opcode = r_dest.Is64Bit() ? WIDE(opcode) : opcode; 693 694 if (r_dest.Is64Bit()) { 695 CHECK(r_src1.Is64Bit()); 696 697 // dest determines whether the op is wide or not. Up-convert src2 when necessary. 698 // Note: this is not according to aarch64 specifications, but our encoding. 699 if (!r_src2.Is64Bit()) { 700 r_src2 = As64BitReg(r_src2); 701 } 702 } else { 703 CHECK(!r_src1.Is64Bit()); 704 CHECK(!r_src2.Is64Bit()); 705 } 706 707 // Sanity checks. 708 // 1) Amount is in the range 0..4 709 CHECK_LE(amount, 4); 710 711 return NewLIR4(widened_opcode, r_dest.GetReg(), r_src1.GetReg(), r_src2.GetReg(), 712 EncodeExtend(ext, amount)); 713} 714 715LIR* Arm64Mir2Lir::OpRegRegReg(OpKind op, RegStorage r_dest, RegStorage r_src1, RegStorage r_src2) { 716 return OpRegRegRegShift(op, r_dest, r_src1, r_src2, ENCODE_NO_SHIFT); 717} 718 719LIR* Arm64Mir2Lir::OpRegRegImm(OpKind op, RegStorage r_dest, RegStorage r_src1, int value) { 720 return OpRegRegImm64(op, r_dest, r_src1, static_cast<int64_t>(value)); 721} 722 723LIR* Arm64Mir2Lir::OpRegRegImm64(OpKind op, RegStorage r_dest, RegStorage r_src1, int64_t value) { 724 LIR* res; 725 bool neg = (value < 0); 726 int64_t abs_value = (neg) ? -value : value; 727 ArmOpcode opcode = kA64Brk1d; 728 ArmOpcode alt_opcode = kA64Brk1d; 729 int32_t log_imm = -1; 730 bool is_wide = r_dest.Is64Bit(); 731 ArmOpcode wide = (is_wide) ? WIDE(0) : UNWIDE(0); 732 int info = 0; 733 734 switch (op) { 735 case kOpLsl: { 736 // "lsl w1, w2, #imm" is an alias of "ubfm w1, w2, #(-imm MOD 32), #(31-imm)" 737 // and "lsl x1, x2, #imm" of "ubfm x1, x2, #(-imm MOD 64), #(63-imm)". 738 // For now, we just use ubfm directly. 739 int max_value = (is_wide) ? 63 : 31; 740 return NewLIR4(kA64Ubfm4rrdd | wide, r_dest.GetReg(), r_src1.GetReg(), 741 (-value) & max_value, max_value - value); 742 } 743 case kOpLsr: 744 return NewLIR3(kA64Lsr3rrd | wide, r_dest.GetReg(), r_src1.GetReg(), value); 745 case kOpAsr: 746 return NewLIR3(kA64Asr3rrd | wide, r_dest.GetReg(), r_src1.GetReg(), value); 747 case kOpRor: 748 // "ror r1, r2, #imm" is an alias of "extr r1, r2, r2, #imm". 749 // For now, we just use extr directly. 750 return NewLIR4(kA64Extr4rrrd | wide, r_dest.GetReg(), r_src1.GetReg(), r_src1.GetReg(), 751 value); 752 case kOpAdd: 753 neg = !neg; 754 // Note: intentional fallthrough 755 case kOpSub: 756 // Add and sub below read/write sp rather than xzr. 757 if (abs_value < 0x1000) { 758 opcode = (neg) ? kA64Add4RRdT : kA64Sub4RRdT; 759 return NewLIR4(opcode | wide, r_dest.GetReg(), r_src1.GetReg(), abs_value, 0); 760 } else if ((abs_value & UINT64_C(0xfff)) == 0 && ((abs_value >> 12) < 0x1000)) { 761 opcode = (neg) ? kA64Add4RRdT : kA64Sub4RRdT; 762 return NewLIR4(opcode | wide, r_dest.GetReg(), r_src1.GetReg(), abs_value >> 12, 1); 763 } else { 764 log_imm = -1; 765 alt_opcode = (op == kOpAdd) ? kA64Add4RRre : kA64Sub4RRre; 766 info = EncodeExtend(is_wide ? kA64Uxtx : kA64Uxtw, 0); 767 } 768 break; 769 // case kOpRsub: 770 // opcode = kThumb2RsubRRI8M; 771 // alt_opcode = kThumb2RsubRRR; 772 // break; 773 case kOpAdc: 774 log_imm = -1; 775 alt_opcode = kA64Adc3rrr; 776 break; 777 case kOpSbc: 778 log_imm = -1; 779 alt_opcode = kA64Sbc3rrr; 780 break; 781 case kOpOr: 782 log_imm = EncodeLogicalImmediate(is_wide, value); 783 opcode = kA64Orr3Rrl; 784 alt_opcode = kA64Orr4rrro; 785 break; 786 case kOpAnd: 787 log_imm = EncodeLogicalImmediate(is_wide, value); 788 opcode = kA64And3Rrl; 789 alt_opcode = kA64And4rrro; 790 break; 791 case kOpXor: 792 log_imm = EncodeLogicalImmediate(is_wide, value); 793 opcode = kA64Eor3Rrl; 794 alt_opcode = kA64Eor4rrro; 795 break; 796 case kOpMul: 797 // TUNING: power of 2, shift & add 798 log_imm = -1; 799 alt_opcode = kA64Mul3rrr; 800 break; 801 default: 802 LOG(FATAL) << "Bad opcode: " << op; 803 } 804 805 if (log_imm >= 0) { 806 return NewLIR3(opcode | wide, r_dest.GetReg(), r_src1.GetReg(), log_imm); 807 } else { 808 RegStorage r_scratch; 809 if (is_wide) { 810 r_scratch = AllocTempWide(); 811 LoadConstantWide(r_scratch, value); 812 } else { 813 r_scratch = AllocTemp(); 814 LoadConstant(r_scratch, value); 815 } 816 if (EncodingMap[alt_opcode].flags & IS_QUAD_OP) 817 res = NewLIR4(alt_opcode | wide, r_dest.GetReg(), r_src1.GetReg(), r_scratch.GetReg(), info); 818 else 819 res = NewLIR3(alt_opcode | wide, r_dest.GetReg(), r_src1.GetReg(), r_scratch.GetReg()); 820 FreeTemp(r_scratch); 821 return res; 822 } 823} 824 825LIR* Arm64Mir2Lir::OpRegImm(OpKind op, RegStorage r_dest_src1, int value) { 826 return OpRegImm64(op, r_dest_src1, static_cast<int64_t>(value)); 827} 828 829LIR* Arm64Mir2Lir::OpRegImm64(OpKind op, RegStorage r_dest_src1, int64_t value) { 830 ArmOpcode wide = (r_dest_src1.Is64Bit()) ? WIDE(0) : UNWIDE(0); 831 ArmOpcode opcode = kA64Brk1d; 832 ArmOpcode neg_opcode = kA64Brk1d; 833 bool shift; 834 bool neg = (value < 0); 835 uint64_t abs_value = (neg) ? -value : value; 836 837 if (LIKELY(abs_value < 0x1000)) { 838 // abs_value is a 12-bit immediate. 839 shift = false; 840 } else if ((abs_value & UINT64_C(0xfff)) == 0 && ((abs_value >> 12) < 0x1000)) { 841 // abs_value is a shifted 12-bit immediate. 842 shift = true; 843 abs_value >>= 12; 844 } else if (LIKELY(abs_value < 0x1000000 && (op == kOpAdd || op == kOpSub))) { 845 // Note: It is better to use two ADD/SUB instead of loading a number to a temp register. 846 // This works for both normal registers and SP. 847 // For a frame size == 0x2468, it will be encoded as: 848 // sub sp, #0x2000 849 // sub sp, #0x468 850 if (neg) { 851 op = (op == kOpAdd) ? kOpSub : kOpAdd; 852 } 853 OpRegImm64(op, r_dest_src1, abs_value & (~INT64_C(0xfff))); 854 return OpRegImm64(op, r_dest_src1, abs_value & 0xfff); 855 } else { 856 RegStorage r_tmp; 857 LIR* res; 858 if (IS_WIDE(wide)) { 859 r_tmp = AllocTempWide(); 860 res = LoadConstantWide(r_tmp, value); 861 } else { 862 r_tmp = AllocTemp(); 863 res = LoadConstant(r_tmp, value); 864 } 865 OpRegReg(op, r_dest_src1, r_tmp); 866 FreeTemp(r_tmp); 867 return res; 868 } 869 870 switch (op) { 871 case kOpAdd: 872 neg_opcode = kA64Sub4RRdT; 873 opcode = kA64Add4RRdT; 874 break; 875 case kOpSub: 876 neg_opcode = kA64Add4RRdT; 877 opcode = kA64Sub4RRdT; 878 break; 879 case kOpCmp: 880 neg_opcode = kA64Cmn3RdT; 881 opcode = kA64Cmp3RdT; 882 break; 883 default: 884 LOG(FATAL) << "Bad op-kind in OpRegImm: " << op; 885 break; 886 } 887 888 if (UNLIKELY(neg)) 889 opcode = neg_opcode; 890 891 if (EncodingMap[opcode].flags & IS_QUAD_OP) 892 return NewLIR4(opcode | wide, r_dest_src1.GetReg(), r_dest_src1.GetReg(), abs_value, 893 (shift) ? 1 : 0); 894 else 895 return NewLIR3(opcode | wide, r_dest_src1.GetReg(), abs_value, (shift) ? 1 : 0); 896} 897 898int Arm64Mir2Lir::EncodeShift(int shift_type, int amount) { 899 DCHECK_EQ(shift_type & 0x3, shift_type); 900 DCHECK_EQ(amount & 0x3f, amount); 901 return ((shift_type & 0x3) << 7) | (amount & 0x3f); 902} 903 904int Arm64Mir2Lir::EncodeExtend(int extend_type, int amount) { 905 DCHECK_EQ(extend_type & 0x7, extend_type); 906 DCHECK_EQ(amount & 0x7, amount); 907 return (1 << 6) | ((extend_type & 0x7) << 3) | (amount & 0x7); 908} 909 910bool Arm64Mir2Lir::IsExtendEncoding(int encoded_value) { 911 return ((1 << 6) & encoded_value) != 0; 912} 913 914LIR* Arm64Mir2Lir::LoadBaseIndexed(RegStorage r_base, RegStorage r_index, RegStorage r_dest, 915 int scale, OpSize size) { 916 LIR* load; 917 int expected_scale = 0; 918 ArmOpcode opcode = kA64Brk1d; 919 r_base = Check64BitReg(r_base); 920 921 // TODO(Arm64): The sign extension of r_index should be carried out by using an extended 922 // register offset load (rather than doing the sign extension in a separate instruction). 923 if (r_index.Is32Bit()) { 924 // Assemble: ``sxtw xN, wN''. 925 r_index = As64BitReg(r_index); 926 NewLIR4(WIDE(kA64Sbfm4rrdd), r_index.GetReg(), r_index.GetReg(), 0, 31); 927 } 928 929 if (r_dest.IsFloat()) { 930 if (r_dest.IsDouble()) { 931 DCHECK(size == k64 || size == kDouble); 932 expected_scale = 3; 933 opcode = FWIDE(kA64Ldr4fXxG); 934 } else { 935 DCHECK(r_dest.IsSingle()); 936 DCHECK(size == k32 || size == kSingle); 937 expected_scale = 2; 938 opcode = kA64Ldr4fXxG; 939 } 940 941 DCHECK(scale == 0 || scale == expected_scale); 942 return NewLIR4(opcode, r_dest.GetReg(), r_base.GetReg(), r_index.GetReg(), 943 (scale != 0) ? 1 : 0); 944 } 945 946 switch (size) { 947 case kDouble: 948 case kWord: 949 case k64: 950 r_dest = Check64BitReg(r_dest); 951 opcode = WIDE(kA64Ldr4rXxG); 952 expected_scale = 3; 953 break; 954 case kSingle: // Intentional fall-through. 955 case k32: // Intentional fall-through. 956 case kReference: 957 r_dest = Check32BitReg(r_dest); 958 opcode = kA64Ldr4rXxG; 959 expected_scale = 2; 960 break; 961 case kUnsignedHalf: 962 r_dest = Check32BitReg(r_dest); 963 opcode = kA64Ldrh4wXxd; 964 expected_scale = 1; 965 break; 966 case kSignedHalf: 967 r_dest = Check32BitReg(r_dest); 968 opcode = kA64Ldrsh4rXxd; 969 expected_scale = 1; 970 break; 971 case kUnsignedByte: 972 r_dest = Check32BitReg(r_dest); 973 opcode = kA64Ldrb3wXx; 974 break; 975 case kSignedByte: 976 r_dest = Check32BitReg(r_dest); 977 opcode = kA64Ldrsb3rXx; 978 break; 979 default: 980 LOG(FATAL) << "Bad size: " << size; 981 } 982 983 if (UNLIKELY(expected_scale == 0)) { 984 // This is a tertiary op (e.g. ldrb, ldrsb), it does not not support scale. 985 DCHECK_NE(EncodingMap[UNWIDE(opcode)].flags & IS_TERTIARY_OP, 0U); 986 DCHECK_EQ(scale, 0); 987 load = NewLIR3(opcode, r_dest.GetReg(), r_base.GetReg(), r_index.GetReg()); 988 } else { 989 DCHECK(scale == 0 || scale == expected_scale); 990 load = NewLIR4(opcode, r_dest.GetReg(), r_base.GetReg(), r_index.GetReg(), 991 (scale != 0) ? 1 : 0); 992 } 993 994 return load; 995} 996 997LIR* Arm64Mir2Lir::LoadRefIndexed(RegStorage r_base, RegStorage r_index, RegStorage r_dest, 998 int scale) { 999 return LoadBaseIndexed(r_base, r_index, As32BitReg(r_dest), scale, kReference); 1000} 1001 1002LIR* Arm64Mir2Lir::StoreBaseIndexed(RegStorage r_base, RegStorage r_index, RegStorage r_src, 1003 int scale, OpSize size) { 1004 LIR* store; 1005 int expected_scale = 0; 1006 ArmOpcode opcode = kA64Brk1d; 1007 r_base = Check64BitReg(r_base); 1008 1009 // TODO(Arm64): The sign extension of r_index should be carried out by using an extended 1010 // register offset store (rather than doing the sign extension in a separate instruction). 1011 if (r_index.Is32Bit()) { 1012 // Assemble: ``sxtw xN, wN''. 1013 r_index = As64BitReg(r_index); 1014 NewLIR4(WIDE(kA64Sbfm4rrdd), r_index.GetReg(), r_index.GetReg(), 0, 31); 1015 } 1016 1017 if (r_src.IsFloat()) { 1018 if (r_src.IsDouble()) { 1019 DCHECK(size == k64 || size == kDouble); 1020 expected_scale = 3; 1021 opcode = FWIDE(kA64Str4fXxG); 1022 } else { 1023 DCHECK(r_src.IsSingle()); 1024 DCHECK(size == k32 || size == kSingle); 1025 expected_scale = 2; 1026 opcode = kA64Str4fXxG; 1027 } 1028 1029 DCHECK(scale == 0 || scale == expected_scale); 1030 return NewLIR4(opcode, r_src.GetReg(), r_base.GetReg(), r_index.GetReg(), 1031 (scale != 0) ? 1 : 0); 1032 } 1033 1034 switch (size) { 1035 case kDouble: // Intentional fall-trough. 1036 case kWord: // Intentional fall-trough. 1037 case k64: 1038 r_src = Check64BitReg(r_src); 1039 opcode = WIDE(kA64Str4rXxG); 1040 expected_scale = 3; 1041 break; 1042 case kSingle: // Intentional fall-trough. 1043 case k32: // Intentional fall-trough. 1044 case kReference: 1045 r_src = Check32BitReg(r_src); 1046 opcode = kA64Str4rXxG; 1047 expected_scale = 2; 1048 break; 1049 case kUnsignedHalf: 1050 case kSignedHalf: 1051 r_src = Check32BitReg(r_src); 1052 opcode = kA64Strh4wXxd; 1053 expected_scale = 1; 1054 break; 1055 case kUnsignedByte: 1056 case kSignedByte: 1057 r_src = Check32BitReg(r_src); 1058 opcode = kA64Strb3wXx; 1059 break; 1060 default: 1061 LOG(FATAL) << "Bad size: " << size; 1062 } 1063 1064 if (UNLIKELY(expected_scale == 0)) { 1065 // This is a tertiary op (e.g. strb), it does not not support scale. 1066 DCHECK_NE(EncodingMap[UNWIDE(opcode)].flags & IS_TERTIARY_OP, 0U); 1067 DCHECK_EQ(scale, 0); 1068 store = NewLIR3(opcode, r_src.GetReg(), r_base.GetReg(), r_index.GetReg()); 1069 } else { 1070 store = NewLIR4(opcode, r_src.GetReg(), r_base.GetReg(), r_index.GetReg(), 1071 (scale != 0) ? 1 : 0); 1072 } 1073 1074 return store; 1075} 1076 1077LIR* Arm64Mir2Lir::StoreRefIndexed(RegStorage r_base, RegStorage r_index, RegStorage r_src, 1078 int scale) { 1079 return StoreBaseIndexed(r_base, r_index, As32BitReg(r_src), scale, kReference); 1080} 1081 1082/* 1083 * Load value from base + displacement. Optionally perform null check 1084 * on base (which must have an associated s_reg and MIR). If not 1085 * performing null check, incoming MIR can be null. 1086 */ 1087LIR* Arm64Mir2Lir::LoadBaseDispBody(RegStorage r_base, int displacement, RegStorage r_dest, 1088 OpSize size) { 1089 LIR* load = NULL; 1090 ArmOpcode opcode = kA64Brk1d; 1091 ArmOpcode alt_opcode = kA64Brk1d; 1092 int scale = 0; 1093 1094 switch (size) { 1095 case kDouble: // Intentional fall-through. 1096 case kWord: // Intentional fall-through. 1097 case k64: 1098 r_dest = Check64BitReg(r_dest); 1099 scale = 3; 1100 if (r_dest.IsFloat()) { 1101 DCHECK(r_dest.IsDouble()); 1102 opcode = FWIDE(kA64Ldr3fXD); 1103 alt_opcode = FWIDE(kA64Ldur3fXd); 1104 } else { 1105 opcode = WIDE(kA64Ldr3rXD); 1106 alt_opcode = WIDE(kA64Ldur3rXd); 1107 } 1108 break; 1109 case kSingle: // Intentional fall-through. 1110 case k32: // Intentional fall-trough. 1111 case kReference: 1112 r_dest = Check32BitReg(r_dest); 1113 scale = 2; 1114 if (r_dest.IsFloat()) { 1115 DCHECK(r_dest.IsSingle()); 1116 opcode = kA64Ldr3fXD; 1117 } else { 1118 opcode = kA64Ldr3rXD; 1119 } 1120 break; 1121 case kUnsignedHalf: 1122 scale = 1; 1123 opcode = kA64Ldrh3wXF; 1124 break; 1125 case kSignedHalf: 1126 scale = 1; 1127 opcode = kA64Ldrsh3rXF; 1128 break; 1129 case kUnsignedByte: 1130 opcode = kA64Ldrb3wXd; 1131 break; 1132 case kSignedByte: 1133 opcode = kA64Ldrsb3rXd; 1134 break; 1135 default: 1136 LOG(FATAL) << "Bad size: " << size; 1137 } 1138 1139 bool displacement_is_aligned = (displacement & ((1 << scale) - 1)) == 0; 1140 int scaled_disp = displacement >> scale; 1141 if (displacement_is_aligned && scaled_disp >= 0 && scaled_disp < 4096) { 1142 // Can use scaled load. 1143 load = NewLIR3(opcode, r_dest.GetReg(), r_base.GetReg(), scaled_disp); 1144 } else if (alt_opcode != kA64Brk1d && IS_SIGNED_IMM9(displacement)) { 1145 // Can use unscaled load. 1146 load = NewLIR3(alt_opcode, r_dest.GetReg(), r_base.GetReg(), displacement); 1147 } else { 1148 // Use long sequence. 1149 // TODO: cleaner support for index/displacement registers? Not a reference, but must match width. 1150 RegStorage r_scratch = AllocTempWide(); 1151 LoadConstantWide(r_scratch, displacement); 1152 load = LoadBaseIndexed(r_base, r_scratch, r_dest, 0, size); 1153 FreeTemp(r_scratch); 1154 } 1155 1156 // TODO: in future may need to differentiate Dalvik accesses w/ spills 1157 if (mem_ref_type_ == ResourceMask::kDalvikReg) { 1158 DCHECK(r_base == rs_sp); 1159 AnnotateDalvikRegAccess(load, displacement >> 2, true /* is_load */, r_dest.Is64Bit()); 1160 } 1161 return load; 1162} 1163 1164LIR* Arm64Mir2Lir::LoadBaseDisp(RegStorage r_base, int displacement, RegStorage r_dest, 1165 OpSize size, VolatileKind is_volatile) { 1166 // LoadBaseDisp() will emit correct insn for atomic load on arm64 1167 // assuming r_dest is correctly prepared using RegClassForFieldLoadStore(). 1168 1169 LIR* load = LoadBaseDispBody(r_base, displacement, r_dest, size); 1170 1171 if (UNLIKELY(is_volatile == kVolatile)) { 1172 // TODO: This should generate an acquire load instead of the barrier. 1173 GenMemBarrier(kLoadAny); 1174 } 1175 1176 return load; 1177} 1178 1179LIR* Arm64Mir2Lir::LoadRefDisp(RegStorage r_base, int displacement, RegStorage r_dest, 1180 VolatileKind is_volatile) { 1181 return LoadBaseDisp(r_base, displacement, As32BitReg(r_dest), kReference, is_volatile); 1182} 1183 1184LIR* Arm64Mir2Lir::StoreBaseDispBody(RegStorage r_base, int displacement, RegStorage r_src, 1185 OpSize size) { 1186 LIR* store = NULL; 1187 ArmOpcode opcode = kA64Brk1d; 1188 ArmOpcode alt_opcode = kA64Brk1d; 1189 int scale = 0; 1190 1191 switch (size) { 1192 case kDouble: // Intentional fall-through. 1193 case kWord: // Intentional fall-through. 1194 case k64: 1195 r_src = Check64BitReg(r_src); 1196 scale = 3; 1197 if (r_src.IsFloat()) { 1198 DCHECK(r_src.IsDouble()); 1199 opcode = FWIDE(kA64Str3fXD); 1200 alt_opcode = FWIDE(kA64Stur3fXd); 1201 } else { 1202 opcode = FWIDE(kA64Str3rXD); 1203 alt_opcode = FWIDE(kA64Stur3rXd); 1204 } 1205 break; 1206 case kSingle: // Intentional fall-through. 1207 case k32: // Intentional fall-trough. 1208 case kReference: 1209 r_src = Check32BitReg(r_src); 1210 scale = 2; 1211 if (r_src.IsFloat()) { 1212 DCHECK(r_src.IsSingle()); 1213 opcode = kA64Str3fXD; 1214 } else { 1215 opcode = kA64Str3rXD; 1216 } 1217 break; 1218 case kUnsignedHalf: 1219 case kSignedHalf: 1220 scale = 1; 1221 opcode = kA64Strh3wXF; 1222 break; 1223 case kUnsignedByte: 1224 case kSignedByte: 1225 opcode = kA64Strb3wXd; 1226 break; 1227 default: 1228 LOG(FATAL) << "Bad size: " << size; 1229 } 1230 1231 bool displacement_is_aligned = (displacement & ((1 << scale) - 1)) == 0; 1232 int scaled_disp = displacement >> scale; 1233 if (displacement_is_aligned && scaled_disp >= 0 && scaled_disp < 4096) { 1234 // Can use scaled store. 1235 store = NewLIR3(opcode, r_src.GetReg(), r_base.GetReg(), scaled_disp); 1236 } else if (alt_opcode != kA64Brk1d && IS_SIGNED_IMM9(displacement)) { 1237 // Can use unscaled store. 1238 store = NewLIR3(alt_opcode, r_src.GetReg(), r_base.GetReg(), displacement); 1239 } else { 1240 // Use long sequence. 1241 RegStorage r_scratch = AllocTempWide(); 1242 LoadConstantWide(r_scratch, displacement); 1243 store = StoreBaseIndexed(r_base, r_scratch, r_src, 0, size); 1244 FreeTemp(r_scratch); 1245 } 1246 1247 // TODO: In future, may need to differentiate Dalvik & spill accesses. 1248 if (mem_ref_type_ == ResourceMask::kDalvikReg) { 1249 DCHECK(r_base == rs_sp); 1250 AnnotateDalvikRegAccess(store, displacement >> 2, false /* is_load */, r_src.Is64Bit()); 1251 } 1252 return store; 1253} 1254 1255LIR* Arm64Mir2Lir::StoreBaseDisp(RegStorage r_base, int displacement, RegStorage r_src, 1256 OpSize size, VolatileKind is_volatile) { 1257 // TODO: This should generate a release store and no barriers. 1258 if (UNLIKELY(is_volatile == kVolatile)) { 1259 // Ensure that prior accesses become visible to other threads first. 1260 GenMemBarrier(kAnyStore); 1261 } 1262 1263 // StoreBaseDisp() will emit correct insn for atomic store on arm64 1264 // assuming r_dest is correctly prepared using RegClassForFieldLoadStore(). 1265 1266 LIR* store = StoreBaseDispBody(r_base, displacement, r_src, size); 1267 1268 if (UNLIKELY(is_volatile == kVolatile)) { 1269 // Preserve order with respect to any subsequent volatile loads. 1270 // We need StoreLoad, but that generally requires the most expensive barrier. 1271 GenMemBarrier(kAnyAny); 1272 } 1273 1274 return store; 1275} 1276 1277LIR* Arm64Mir2Lir::StoreRefDisp(RegStorage r_base, int displacement, RegStorage r_src, 1278 VolatileKind is_volatile) { 1279 return StoreBaseDisp(r_base, displacement, As32BitReg(r_src), kReference, is_volatile); 1280} 1281 1282LIR* Arm64Mir2Lir::OpFpRegCopy(RegStorage r_dest, RegStorage r_src) { 1283 LOG(FATAL) << "Unexpected use of OpFpRegCopy for Arm64"; 1284 return NULL; 1285} 1286 1287LIR* Arm64Mir2Lir::OpMem(OpKind op, RegStorage r_base, int disp) { 1288 LOG(FATAL) << "Unexpected use of OpMem for Arm64"; 1289 return NULL; 1290} 1291 1292LIR* Arm64Mir2Lir::InvokeTrampoline(OpKind op, RegStorage r_tgt, QuickEntrypointEnum trampoline) { 1293 return OpReg(op, r_tgt); 1294} 1295 1296} // namespace art 1297