int_arm64.cc revision 3c12c512faf6837844d5465b23b9410889e5eb11
1/* 2 * Copyright (C) 2011 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17/* This file contains codegen for the Thumb2 ISA. */ 18 19#include "arm64_lir.h" 20#include "codegen_arm64.h" 21#include "dex/quick/mir_to_lir-inl.h" 22#include "entrypoints/quick/quick_entrypoints.h" 23#include "mirror/array.h" 24 25namespace art { 26 27LIR* Arm64Mir2Lir::OpCmpBranch(ConditionCode cond, RegStorage src1, RegStorage src2, LIR* target) { 28 OpRegReg(kOpCmp, src1, src2); 29 return OpCondBranch(cond, target); 30} 31 32LIR* Arm64Mir2Lir::OpIT(ConditionCode ccode, const char* guide) { 33 LOG(FATAL) << "Unexpected use of OpIT for Arm64"; 34 return NULL; 35} 36 37void Arm64Mir2Lir::OpEndIT(LIR* it) { 38 LOG(FATAL) << "Unexpected use of OpEndIT for Arm64"; 39} 40 41/* 42 * 64-bit 3way compare function. 43 * cmp xA, xB 44 * csinc wC, wzr, wzr, eq // wC = (xA == xB) ? 0 : 1 45 * csneg wC, wC, wC, ge // wC = (xA >= xB) ? wC : -wC 46 */ 47void Arm64Mir2Lir::GenCmpLong(RegLocation rl_dest, RegLocation rl_src1, 48 RegLocation rl_src2) { 49 RegLocation rl_result; 50 rl_src1 = LoadValueWide(rl_src1, kCoreReg); 51 rl_src2 = LoadValueWide(rl_src2, kCoreReg); 52 rl_result = EvalLoc(rl_dest, kCoreReg, true); 53 54 OpRegReg(kOpCmp, rl_src1.reg, rl_src2.reg); 55 NewLIR4(kA64Csinc4rrrc, rl_result.reg.GetReg(), rwzr, rwzr, kArmCondEq); 56 NewLIR4(kA64Csneg4rrrc, rl_result.reg.GetReg(), rl_result.reg.GetReg(), 57 rl_result.reg.GetReg(), kArmCondGe); 58 StoreValue(rl_dest, rl_result); 59} 60 61void Arm64Mir2Lir::GenShiftOpLong(Instruction::Code opcode, RegLocation rl_dest, 62 RegLocation rl_src1, RegLocation rl_shift) { 63 OpKind op = kOpBkpt; 64 switch (opcode) { 65 case Instruction::SHL_LONG: 66 case Instruction::SHL_LONG_2ADDR: 67 op = kOpLsl; 68 break; 69 case Instruction::SHR_LONG: 70 case Instruction::SHR_LONG_2ADDR: 71 op = kOpAsr; 72 break; 73 case Instruction::USHR_LONG: 74 case Instruction::USHR_LONG_2ADDR: 75 op = kOpLsr; 76 break; 77 default: 78 LOG(FATAL) << "Unexpected case: " << opcode; 79 } 80 rl_shift = LoadValue(rl_shift, kCoreReg); 81 rl_src1 = LoadValueWide(rl_src1, kCoreReg); 82 RegLocation rl_result = EvalLocWide(rl_dest, kCoreReg, true); 83 OpRegRegReg(op, rl_result.reg, rl_src1.reg, As64BitReg(rl_shift.reg)); 84 StoreValueWide(rl_dest, rl_result); 85} 86 87void Arm64Mir2Lir::GenSelect(BasicBlock* bb, MIR* mir) { 88 RegLocation rl_result; 89 RegLocation rl_src = mir_graph_->GetSrc(mir, 0); 90 RegLocation rl_dest = mir_graph_->GetDest(mir); 91 RegisterClass src_reg_class = rl_src.ref ? kRefReg : kCoreReg; 92 RegisterClass result_reg_class = rl_dest.ref ? kRefReg : kCoreReg; 93 rl_src = LoadValue(rl_src, src_reg_class); 94 ArmConditionCode code = ArmConditionEncoding(mir->meta.ccode); 95 96 RegLocation rl_true = mir_graph_->reg_location_[mir->ssa_rep->uses[1]]; 97 RegLocation rl_false = mir_graph_->reg_location_[mir->ssa_rep->uses[2]]; 98 rl_true = LoadValue(rl_true, result_reg_class); 99 rl_false = LoadValue(rl_false, result_reg_class); 100 rl_result = EvalLoc(rl_dest, result_reg_class, true); 101 OpRegImm(kOpCmp, rl_src.reg, 0); 102 NewLIR4(kA64Csel4rrrc, rl_result.reg.GetReg(), rl_true.reg.GetReg(), 103 rl_false.reg.GetReg(), code); 104 StoreValue(rl_dest, rl_result); 105} 106 107void Arm64Mir2Lir::GenFusedLongCmpBranch(BasicBlock* bb, MIR* mir) { 108 RegLocation rl_src1 = mir_graph_->GetSrcWide(mir, 0); 109 RegLocation rl_src2 = mir_graph_->GetSrcWide(mir, 2); 110 LIR* taken = &block_label_list_[bb->taken]; 111 LIR* not_taken = &block_label_list_[bb->fall_through]; 112 rl_src1 = LoadValueWide(rl_src1, kCoreReg); 113 // Normalize such that if either operand is constant, src2 will be constant. 114 ConditionCode ccode = mir->meta.ccode; 115 if (rl_src1.is_const) { 116 std::swap(rl_src1, rl_src2); 117 ccode = FlipComparisonOrder(ccode); 118 } 119 120 if (rl_src2.is_const) { 121 rl_src2 = UpdateLocWide(rl_src2); 122 int64_t val = mir_graph_->ConstantValueWide(rl_src2); 123 // Special handling using cbz & cbnz. 124 if (val == 0 && (ccode == kCondEq || ccode == kCondNe)) { 125 OpCmpImmBranch(ccode, rl_src1.reg, 0, taken); 126 OpCmpImmBranch(NegateComparison(ccode), rl_src1.reg, 0, not_taken); 127 return; 128 // Only handle Imm if src2 is not already in a register. 129 } else if (rl_src2.location != kLocPhysReg) { 130 OpRegImm64(kOpCmp, rl_src1.reg, val); 131 OpCondBranch(ccode, taken); 132 OpCondBranch(NegateComparison(ccode), not_taken); 133 return; 134 } 135 } 136 137 rl_src2 = LoadValueWide(rl_src2, kCoreReg); 138 OpRegReg(kOpCmp, rl_src1.reg, rl_src2.reg); 139 OpCondBranch(ccode, taken); 140 OpCondBranch(NegateComparison(ccode), not_taken); 141} 142 143/* 144 * Generate a register comparison to an immediate and branch. Caller 145 * is responsible for setting branch target field. 146 */ 147LIR* Arm64Mir2Lir::OpCmpImmBranch(ConditionCode cond, RegStorage reg, int check_value, 148 LIR* target) { 149 LIR* branch; 150 ArmConditionCode arm_cond = ArmConditionEncoding(cond); 151 if (check_value == 0 && (arm_cond == kArmCondEq || arm_cond == kArmCondNe)) { 152 ArmOpcode opcode = (arm_cond == kArmCondEq) ? kA64Cbz2rt : kA64Cbnz2rt; 153 ArmOpcode wide = reg.Is64Bit() ? WIDE(0) : UNWIDE(0); 154 branch = NewLIR2(opcode | wide, reg.GetReg(), 0); 155 } else { 156 OpRegImm(kOpCmp, reg, check_value); 157 branch = NewLIR2(kA64B2ct, arm_cond, 0); 158 } 159 branch->target = target; 160 return branch; 161} 162 163LIR* Arm64Mir2Lir::OpCmpMemImmBranch(ConditionCode cond, RegStorage temp_reg, 164 RegStorage base_reg, int offset, int check_value, 165 LIR* target) { 166 // It is possible that temp register is 64-bit. (ArgReg or RefReg) 167 // Always compare 32-bit value no matter what temp_reg is. 168 if (temp_reg.Is64Bit()) { 169 temp_reg = As32BitReg(temp_reg); 170 } 171 Load32Disp(base_reg, offset, temp_reg); 172 LIR* branch = OpCmpImmBranch(cond, temp_reg, check_value, target); 173 return branch; 174} 175 176LIR* Arm64Mir2Lir::OpRegCopyNoInsert(RegStorage r_dest, RegStorage r_src) { 177 bool dest_is_fp = r_dest.IsFloat(); 178 bool src_is_fp = r_src.IsFloat(); 179 ArmOpcode opcode = kA64Brk1d; 180 LIR* res; 181 182 if (LIKELY(dest_is_fp == src_is_fp)) { 183 if (LIKELY(!dest_is_fp)) { 184 // Core/core copy. 185 // Copies involving the sp register require a different instruction. 186 opcode = UNLIKELY(A64_REG_IS_SP(r_dest.GetReg())) ? kA64Add4RRdT : kA64Mov2rr; 187 188 // TODO(Arm64): kA64Add4RRdT formally has 4 args, but is used as a 2 args instruction. 189 // This currently works because the other arguments are set to 0 by default. We should 190 // rather introduce an alias kA64Mov2RR. 191 192 // core/core copy. Do a x/x copy only if both registers are x. 193 if (r_dest.Is64Bit() && r_src.Is64Bit()) { 194 opcode = WIDE(opcode); 195 } 196 } else { 197 // Float/float copy. 198 bool dest_is_double = r_dest.IsDouble(); 199 bool src_is_double = r_src.IsDouble(); 200 201 // We do not do float/double or double/float casts here. 202 DCHECK_EQ(dest_is_double, src_is_double); 203 204 // Homogeneous float/float copy. 205 opcode = (dest_is_double) ? FWIDE(kA64Fmov2ff) : kA64Fmov2ff; 206 } 207 } else { 208 // Inhomogeneous register copy. 209 if (dest_is_fp) { 210 if (r_dest.IsDouble()) { 211 opcode = kA64Fmov2Sx; 212 } else { 213 DCHECK(r_src.IsSingle()); 214 opcode = kA64Fmov2sw; 215 } 216 } else { 217 if (r_src.IsDouble()) { 218 opcode = kA64Fmov2xS; 219 } else { 220 DCHECK(r_dest.Is32Bit()); 221 opcode = kA64Fmov2ws; 222 } 223 } 224 } 225 226 res = RawLIR(current_dalvik_offset_, opcode, r_dest.GetReg(), r_src.GetReg()); 227 228 if (!(cu_->disable_opt & (1 << kSafeOptimizations)) && r_dest == r_src) { 229 res->flags.is_nop = true; 230 } 231 232 return res; 233} 234 235void Arm64Mir2Lir::OpRegCopy(RegStorage r_dest, RegStorage r_src) { 236 if (r_dest != r_src) { 237 LIR* res = OpRegCopyNoInsert(r_dest, r_src); 238 AppendLIR(res); 239 } 240} 241 242void Arm64Mir2Lir::OpRegCopyWide(RegStorage r_dest, RegStorage r_src) { 243 OpRegCopy(r_dest, r_src); 244} 245 246// Table of magic divisors 247struct MagicTable { 248 uint32_t magic; 249 uint32_t shift; 250 DividePattern pattern; 251}; 252 253static const MagicTable magic_table[] = { 254 {0, 0, DivideNone}, // 0 255 {0, 0, DivideNone}, // 1 256 {0, 0, DivideNone}, // 2 257 {0x55555556, 0, Divide3}, // 3 258 {0, 0, DivideNone}, // 4 259 {0x66666667, 1, Divide5}, // 5 260 {0x2AAAAAAB, 0, Divide3}, // 6 261 {0x92492493, 2, Divide7}, // 7 262 {0, 0, DivideNone}, // 8 263 {0x38E38E39, 1, Divide5}, // 9 264 {0x66666667, 2, Divide5}, // 10 265 {0x2E8BA2E9, 1, Divide5}, // 11 266 {0x2AAAAAAB, 1, Divide5}, // 12 267 {0x4EC4EC4F, 2, Divide5}, // 13 268 {0x92492493, 3, Divide7}, // 14 269 {0x88888889, 3, Divide7}, // 15 270}; 271 272// Integer division by constant via reciprocal multiply (Hacker's Delight, 10-4) 273bool Arm64Mir2Lir::SmallLiteralDivRem(Instruction::Code dalvik_opcode, bool is_div, 274 RegLocation rl_src, RegLocation rl_dest, int lit) { 275 // TODO(Arm64): fix this for Arm64. Note: may be worth revisiting the magic table. 276 // It should be possible subtracting one from all its entries, and using smaddl 277 // to counteract this. The advantage is that integers should then be easier to 278 // encode as logical immediates (0x55555555 rather than 0x55555556). 279 UNIMPLEMENTED(FATAL); 280 281 if ((lit < 0) || (lit >= static_cast<int>(sizeof(magic_table)/sizeof(magic_table[0])))) { 282 return false; 283 } 284 DividePattern pattern = magic_table[lit].pattern; 285 if (pattern == DivideNone) { 286 return false; 287 } 288 // Tuning: add rem patterns 289 if (!is_div) { 290 return false; 291 } 292 293 RegStorage r_magic = AllocTemp(); 294 LoadConstant(r_magic, magic_table[lit].magic); 295 rl_src = LoadValue(rl_src, kCoreReg); 296 RegLocation rl_result = EvalLoc(rl_dest, kCoreReg, true); 297 RegStorage r_hi = AllocTemp(); 298 RegStorage r_lo = AllocTemp(); 299 NewLIR4(kA64Smaddl4xwwx, r_lo.GetReg(), r_magic.GetReg(), rl_src.reg.GetReg(), rxzr); 300 switch (pattern) { 301 case Divide3: 302 OpRegRegRegShift(kOpSub, rl_result.reg, r_hi, rl_src.reg, EncodeShift(kA64Asr, 31)); 303 break; 304 case Divide5: 305 OpRegRegImm(kOpAsr, r_lo, rl_src.reg, 31); 306 OpRegRegRegShift(kOpRsub, rl_result.reg, r_lo, r_hi, EncodeShift(kA64Asr, magic_table[lit].shift)); 307 break; 308 case Divide7: 309 OpRegReg(kOpAdd, r_hi, rl_src.reg); 310 OpRegRegImm(kOpAsr, r_lo, rl_src.reg, 31); 311 OpRegRegRegShift(kOpRsub, rl_result.reg, r_lo, r_hi, EncodeShift(kA64Asr, magic_table[lit].shift)); 312 break; 313 default: 314 LOG(FATAL) << "Unexpected pattern: " << pattern; 315 } 316 StoreValue(rl_dest, rl_result); 317 return true; 318} 319 320bool Arm64Mir2Lir::EasyMultiply(RegLocation rl_src, RegLocation rl_dest, int lit) { 321 LOG(FATAL) << "Unexpected use of EasyMultiply for Arm64"; 322 return false; 323} 324 325RegLocation Arm64Mir2Lir::GenDivRem(RegLocation rl_dest, RegLocation rl_src1, 326 RegLocation rl_src2, bool is_div, bool check_zero) { 327 LOG(FATAL) << "Unexpected use of GenDivRem for Arm64"; 328 return rl_dest; 329} 330 331RegLocation Arm64Mir2Lir::GenDivRemLit(RegLocation rl_dest, RegLocation rl_src1, int lit, bool is_div) { 332 LOG(FATAL) << "Unexpected use of GenDivRemLit for Arm64"; 333 return rl_dest; 334} 335 336RegLocation Arm64Mir2Lir::GenDivRemLit(RegLocation rl_dest, RegStorage reg1, int lit, bool is_div) { 337 RegLocation rl_result = EvalLoc(rl_dest, kCoreReg, true); 338 339 // Put the literal in a temp. 340 RegStorage lit_temp = AllocTemp(); 341 LoadConstant(lit_temp, lit); 342 // Use the generic case for div/rem with arg2 in a register. 343 // TODO: The literal temp can be freed earlier during a modulus to reduce reg pressure. 344 rl_result = GenDivRem(rl_result, reg1, lit_temp, is_div); 345 FreeTemp(lit_temp); 346 347 return rl_result; 348} 349 350RegLocation Arm64Mir2Lir::GenDivRem(RegLocation rl_dest, RegStorage r_src1, RegStorage r_src2, 351 bool is_div) { 352 CHECK_EQ(r_src1.Is64Bit(), r_src2.Is64Bit()); 353 354 RegLocation rl_result = EvalLoc(rl_dest, kCoreReg, true); 355 if (is_div) { 356 OpRegRegReg(kOpDiv, rl_result.reg, r_src1, r_src2); 357 } else { 358 // temp = r_src1 / r_src2 359 // dest = r_src1 - temp * r_src2 360 RegStorage temp; 361 ArmOpcode wide; 362 if (rl_result.reg.Is64Bit()) { 363 temp = AllocTempWide(); 364 wide = WIDE(0); 365 } else { 366 temp = AllocTemp(); 367 wide = UNWIDE(0); 368 } 369 OpRegRegReg(kOpDiv, temp, r_src1, r_src2); 370 NewLIR4(kA64Msub4rrrr | wide, rl_result.reg.GetReg(), temp.GetReg(), 371 r_src1.GetReg(), r_src2.GetReg()); 372 FreeTemp(temp); 373 } 374 return rl_result; 375} 376 377bool Arm64Mir2Lir::GenInlinedAbsLong(CallInfo* info) { 378 RegLocation rl_src = info->args[0]; 379 rl_src = LoadValueWide(rl_src, kCoreReg); 380 RegLocation rl_dest = InlineTargetWide(info); 381 RegLocation rl_result = EvalLoc(rl_dest, kCoreReg, true); 382 RegStorage sign_reg = AllocTempWide(); 383 // abs(x) = y<=x>>63, (x+y)^y. 384 OpRegRegImm(kOpAsr, sign_reg, rl_src.reg, 63); 385 OpRegRegReg(kOpAdd, rl_result.reg, rl_src.reg, sign_reg); 386 OpRegReg(kOpXor, rl_result.reg, sign_reg); 387 StoreValueWide(rl_dest, rl_result); 388 return true; 389} 390 391bool Arm64Mir2Lir::GenInlinedMinMaxInt(CallInfo* info, bool is_min) { 392 DCHECK_EQ(cu_->instruction_set, kArm64); 393 RegLocation rl_src1 = info->args[0]; 394 RegLocation rl_src2 = info->args[1]; 395 rl_src1 = LoadValue(rl_src1, kCoreReg); 396 rl_src2 = LoadValue(rl_src2, kCoreReg); 397 RegLocation rl_dest = InlineTarget(info); 398 RegLocation rl_result = EvalLoc(rl_dest, kCoreReg, true); 399 OpRegReg(kOpCmp, rl_src1.reg, rl_src2.reg); 400 NewLIR4(kA64Csel4rrrc, rl_result.reg.GetReg(), rl_src1.reg.GetReg(), 401 rl_src2.reg.GetReg(), (is_min) ? kArmCondLt : kArmCondGt); 402 StoreValue(rl_dest, rl_result); 403 return true; 404} 405 406bool Arm64Mir2Lir::GenInlinedPeek(CallInfo* info, OpSize size) { 407 RegLocation rl_src_address = info->args[0]; // long address 408 rl_src_address = NarrowRegLoc(rl_src_address); // ignore high half in info->args[1] ? 409 RegLocation rl_dest = InlineTarget(info); 410 RegLocation rl_address = LoadValue(rl_src_address, kCoreReg); // kRefReg 411 RegLocation rl_result = EvalLoc(rl_dest, kCoreReg, true); 412 413 LoadBaseDisp(rl_address.reg, 0, rl_result.reg, size, kNotVolatile); 414 if (size == k64) { 415 StoreValueWide(rl_dest, rl_result); 416 } else { 417 DCHECK(size == kSignedByte || size == kSignedHalf || size == k32); 418 StoreValue(rl_dest, rl_result); 419 } 420 return true; 421} 422 423bool Arm64Mir2Lir::GenInlinedPoke(CallInfo* info, OpSize size) { 424 RegLocation rl_src_address = info->args[0]; // long address 425 rl_src_address = NarrowRegLoc(rl_src_address); // ignore high half in info->args[1] 426 RegLocation rl_src_value = info->args[2]; // [size] value 427 RegLocation rl_address = LoadValue(rl_src_address, kCoreReg); // kRefReg 428 429 RegLocation rl_value; 430 if (size == k64) { 431 rl_value = LoadValueWide(rl_src_value, kCoreReg); 432 } else { 433 DCHECK(size == kSignedByte || size == kSignedHalf || size == k32); 434 rl_value = LoadValue(rl_src_value, kCoreReg); 435 } 436 StoreBaseDisp(rl_address.reg, 0, rl_value.reg, size, kNotVolatile); 437 return true; 438} 439 440void Arm64Mir2Lir::OpLea(RegStorage r_base, RegStorage reg1, RegStorage reg2, int scale, int offset) { 441 LOG(FATAL) << "Unexpected use of OpLea for Arm64"; 442} 443 444void Arm64Mir2Lir::OpTlsCmp(ThreadOffset<4> offset, int val) { 445 UNIMPLEMENTED(FATAL) << "Should not be used."; 446} 447 448void Arm64Mir2Lir::OpTlsCmp(ThreadOffset<8> offset, int val) { 449 LOG(FATAL) << "Unexpected use of OpTlsCmp for Arm64"; 450} 451 452bool Arm64Mir2Lir::GenInlinedCas(CallInfo* info, bool is_long, bool is_object) { 453 DCHECK_EQ(cu_->instruction_set, kArm64); 454 ArmOpcode wide = is_long ? WIDE(0) : UNWIDE(0); 455 // Unused - RegLocation rl_src_unsafe = info->args[0]; 456 RegLocation rl_src_obj = info->args[1]; // Object - known non-null 457 RegLocation rl_src_offset = info->args[2]; // long low 458 rl_src_offset = NarrowRegLoc(rl_src_offset); // ignore high half in info->args[3] //TODO: do we really need this 459 RegLocation rl_src_expected = info->args[4]; // int, long or Object 460 // If is_long, high half is in info->args[5] 461 RegLocation rl_src_new_value = info->args[is_long ? 6 : 5]; // int, long or Object 462 // If is_long, high half is in info->args[7] 463 RegLocation rl_dest = InlineTarget(info); // boolean place for result 464 465 // Load Object and offset 466 RegLocation rl_object = LoadValue(rl_src_obj, kRefReg); 467 RegLocation rl_offset = LoadValue(rl_src_offset, kRefReg); 468 469 RegLocation rl_new_value; 470 RegLocation rl_expected; 471 if (is_long) { 472 rl_new_value = LoadValueWide(rl_src_new_value, kCoreReg); 473 rl_expected = LoadValueWide(rl_src_expected, kCoreReg); 474 } else { 475 rl_new_value = LoadValue(rl_src_new_value, is_object ? kRefReg : kCoreReg); 476 rl_expected = LoadValue(rl_src_expected, is_object ? kRefReg : kCoreReg); 477 } 478 479 if (is_object && !mir_graph_->IsConstantNullRef(rl_new_value)) { 480 // Mark card for object assuming new value is stored. 481 MarkGCCard(rl_new_value.reg, rl_object.reg); 482 } 483 484 RegStorage r_ptr = AllocTempRef(); 485 OpRegRegReg(kOpAdd, r_ptr, rl_object.reg, rl_offset.reg); 486 487 // Free now unneeded rl_object and rl_offset to give more temps. 488 ClobberSReg(rl_object.s_reg_low); 489 FreeTemp(rl_object.reg); 490 ClobberSReg(rl_offset.s_reg_low); 491 FreeTemp(rl_offset.reg); 492 493 // do { 494 // tmp = [r_ptr] - expected; 495 // } while (tmp == 0 && failure([r_ptr] <- r_new_value)); 496 // result = tmp != 0; 497 498 RegStorage r_tmp; 499 if (is_long) { 500 r_tmp = AllocTempWide(); 501 } else if (is_object) { 502 r_tmp = AllocTempRef(); 503 } else { 504 r_tmp = AllocTemp(); 505 } 506 507 LIR* loop = NewLIR0(kPseudoTargetLabel); 508 NewLIR2(kA64Ldaxr2rX | wide, r_tmp.GetReg(), r_ptr.GetReg()); 509 OpRegReg(kOpCmp, r_tmp, rl_expected.reg); 510 DCHECK(last_lir_insn_->u.m.def_mask->HasBit(ResourceMask::kCCode)); 511 LIR* early_exit = OpCondBranch(kCondNe, NULL); 512 513 NewLIR3(kA64Stlxr3wrX | wide, As32BitReg(r_tmp).GetReg(), rl_new_value.reg.GetReg(), r_ptr.GetReg()); 514 NewLIR3(kA64Cmp3RdT, As32BitReg(r_tmp).GetReg(), 0, ENCODE_NO_SHIFT); 515 DCHECK(last_lir_insn_->u.m.def_mask->HasBit(ResourceMask::kCCode)); 516 OpCondBranch(kCondNe, loop); 517 518 RegLocation rl_result = EvalLoc(rl_dest, kCoreReg, true); 519 LIR* exit = NewLIR4(kA64Csinc4rrrc, rl_result.reg.GetReg(), rwzr, rwzr, kArmCondNe); 520 early_exit->target = exit; 521 522 FreeTemp(r_tmp); // Now unneeded. 523 FreeTemp(r_ptr); // Now unneeded. 524 525 StoreValue(rl_dest, rl_result); 526 527 return true; 528} 529 530LIR* Arm64Mir2Lir::OpPcRelLoad(RegStorage reg, LIR* target) { 531 return RawLIR(current_dalvik_offset_, WIDE(kA64Ldr2rp), reg.GetReg(), 0, 0, 0, 0, target); 532} 533 534LIR* Arm64Mir2Lir::OpVldm(RegStorage r_base, int count) { 535 LOG(FATAL) << "Unexpected use of OpVldm for Arm64"; 536 return NULL; 537} 538 539LIR* Arm64Mir2Lir::OpVstm(RegStorage r_base, int count) { 540 LOG(FATAL) << "Unexpected use of OpVstm for Arm64"; 541 return NULL; 542} 543 544void Arm64Mir2Lir::GenMultiplyByTwoBitMultiplier(RegLocation rl_src, 545 RegLocation rl_result, int lit, 546 int first_bit, int second_bit) { 547 OpRegRegRegShift(kOpAdd, rl_result.reg, rl_src.reg, rl_src.reg, EncodeShift(kA64Lsl, second_bit - first_bit)); 548 if (first_bit != 0) { 549 OpRegRegImm(kOpLsl, rl_result.reg, rl_result.reg, first_bit); 550 } 551} 552 553void Arm64Mir2Lir::GenDivZeroCheckWide(RegStorage reg) { 554 LOG(FATAL) << "Unexpected use of GenDivZero for Arm64"; 555} 556 557// Test suspend flag, return target of taken suspend branch 558LIR* Arm64Mir2Lir::OpTestSuspend(LIR* target) { 559 // FIXME: Define rA64_SUSPEND as w19, when we do not need two copies of reserved register. 560 // Note: The opcode is not set as wide, so actually we are using the 32-bit version register. 561 NewLIR3(kA64Subs3rRd, rA64_SUSPEND, rA64_SUSPEND, 1); 562 return OpCondBranch((target == NULL) ? kCondEq : kCondNe, target); 563} 564 565// Decrement register and branch on condition 566LIR* Arm64Mir2Lir::OpDecAndBranch(ConditionCode c_code, RegStorage reg, LIR* target) { 567 // Combine sub & test using sub setflags encoding here. We need to make sure a 568 // subtract form that sets carry is used, so generate explicitly. 569 // TODO: might be best to add a new op, kOpSubs, and handle it generically. 570 ArmOpcode opcode = reg.Is64Bit() ? WIDE(kA64Subs3rRd) : UNWIDE(kA64Subs3rRd); 571 NewLIR3(opcode, reg.GetReg(), reg.GetReg(), 1); // For value == 1, this should set flags. 572 DCHECK(last_lir_insn_->u.m.def_mask->HasBit(ResourceMask::kCCode)); 573 return OpCondBranch(c_code, target); 574} 575 576bool Arm64Mir2Lir::GenMemBarrier(MemBarrierKind barrier_kind) { 577#if ANDROID_SMP != 0 578 // Start off with using the last LIR as the barrier. If it is not enough, then we will generate one. 579 LIR* barrier = last_lir_insn_; 580 581 int dmb_flavor; 582 // TODO: revisit Arm barrier kinds 583 switch (barrier_kind) { 584 case kLoadStore: dmb_flavor = kISH; break; 585 case kLoadLoad: dmb_flavor = kISH; break; 586 case kStoreStore: dmb_flavor = kISHST; break; 587 case kStoreLoad: dmb_flavor = kISH; break; 588 default: 589 LOG(FATAL) << "Unexpected MemBarrierKind: " << barrier_kind; 590 dmb_flavor = kSY; // quiet gcc. 591 break; 592 } 593 594 bool ret = false; 595 596 // If the same barrier already exists, don't generate another. 597 if (barrier == nullptr 598 || (barrier->opcode != kA64Dmb1B || barrier->operands[0] != dmb_flavor)) { 599 barrier = NewLIR1(kA64Dmb1B, dmb_flavor); 600 ret = true; 601 } 602 603 // At this point we must have a memory barrier. Mark it as a scheduling barrier as well. 604 DCHECK(!barrier->flags.use_def_invalid); 605 barrier->u.m.def_mask = &kEncodeAll; 606 return ret; 607#else 608 return false; 609#endif 610} 611 612void Arm64Mir2Lir::GenIntToLong(RegLocation rl_dest, RegLocation rl_src) { 613 RegLocation rl_result; 614 615 rl_src = LoadValue(rl_src, kCoreReg); 616 rl_result = EvalLocWide(rl_dest, kCoreReg, true); 617 NewLIR4(WIDE(kA64Sbfm4rrdd), rl_result.reg.GetReg(), rl_src.reg.GetReg(), 0, 31); 618 StoreValueWide(rl_dest, rl_result); 619} 620 621void Arm64Mir2Lir::GenDivRemLong(Instruction::Code opcode, RegLocation rl_dest, 622 RegLocation rl_src1, RegLocation rl_src2, bool is_div) { 623 RegLocation rl_result; 624 rl_src1 = LoadValueWide(rl_src1, kCoreReg); 625 rl_src2 = LoadValueWide(rl_src2, kCoreReg); 626 GenDivZeroCheck(rl_src2.reg); 627 rl_result = GenDivRem(rl_dest, rl_src1.reg, rl_src2.reg, is_div); 628 StoreValueWide(rl_dest, rl_result); 629} 630 631void Arm64Mir2Lir::GenLongOp(OpKind op, RegLocation rl_dest, RegLocation rl_src1, 632 RegLocation rl_src2) { 633 RegLocation rl_result; 634 635 rl_src1 = LoadValueWide(rl_src1, kCoreReg); 636 rl_src2 = LoadValueWide(rl_src2, kCoreReg); 637 rl_result = EvalLocWide(rl_dest, kCoreReg, true); 638 OpRegRegRegShift(op, rl_result.reg, rl_src1.reg, rl_src2.reg, ENCODE_NO_SHIFT); 639 StoreValueWide(rl_dest, rl_result); 640} 641 642void Arm64Mir2Lir::GenNegLong(RegLocation rl_dest, RegLocation rl_src) { 643 RegLocation rl_result; 644 645 rl_src = LoadValueWide(rl_src, kCoreReg); 646 rl_result = EvalLocWide(rl_dest, kCoreReg, true); 647 OpRegRegShift(kOpNeg, rl_result.reg, rl_src.reg, ENCODE_NO_SHIFT); 648 StoreValueWide(rl_dest, rl_result); 649} 650 651void Arm64Mir2Lir::GenNotLong(RegLocation rl_dest, RegLocation rl_src) { 652 RegLocation rl_result; 653 654 rl_src = LoadValueWide(rl_src, kCoreReg); 655 rl_result = EvalLocWide(rl_dest, kCoreReg, true); 656 OpRegRegShift(kOpMvn, rl_result.reg, rl_src.reg, ENCODE_NO_SHIFT); 657 StoreValueWide(rl_dest, rl_result); 658} 659 660void Arm64Mir2Lir::GenMulLong(Instruction::Code opcode, RegLocation rl_dest, 661 RegLocation rl_src1, RegLocation rl_src2) { 662 GenLongOp(kOpMul, rl_dest, rl_src1, rl_src2); 663} 664 665void Arm64Mir2Lir::GenAddLong(Instruction::Code opcode, RegLocation rl_dest, RegLocation rl_src1, 666 RegLocation rl_src2) { 667 GenLongOp(kOpAdd, rl_dest, rl_src1, rl_src2); 668} 669 670void Arm64Mir2Lir::GenSubLong(Instruction::Code opcode, RegLocation rl_dest, RegLocation rl_src1, 671 RegLocation rl_src2) { 672 GenLongOp(kOpSub, rl_dest, rl_src1, rl_src2); 673} 674 675void Arm64Mir2Lir::GenAndLong(Instruction::Code opcode, RegLocation rl_dest, RegLocation rl_src1, 676 RegLocation rl_src2) { 677 GenLongOp(kOpAnd, rl_dest, rl_src1, rl_src2); 678} 679 680void Arm64Mir2Lir::GenOrLong(Instruction::Code opcode, RegLocation rl_dest, RegLocation rl_src1, 681 RegLocation rl_src2) { 682 GenLongOp(kOpOr, rl_dest, rl_src1, rl_src2); 683} 684 685void Arm64Mir2Lir::GenXorLong(Instruction::Code opcode, RegLocation rl_dest, RegLocation rl_src1, 686 RegLocation rl_src2) { 687 GenLongOp(kOpXor, rl_dest, rl_src1, rl_src2); 688} 689 690/* 691 * Generate array load 692 */ 693void Arm64Mir2Lir::GenArrayGet(int opt_flags, OpSize size, RegLocation rl_array, 694 RegLocation rl_index, RegLocation rl_dest, int scale) { 695 RegisterClass reg_class = RegClassBySize(size); 696 int len_offset = mirror::Array::LengthOffset().Int32Value(); 697 int data_offset; 698 RegLocation rl_result; 699 bool constant_index = rl_index.is_const; 700 rl_array = LoadValue(rl_array, kRefReg); 701 if (!constant_index) { 702 rl_index = LoadValue(rl_index, kCoreReg); 703 } 704 705 if (rl_dest.wide) { 706 data_offset = mirror::Array::DataOffset(sizeof(int64_t)).Int32Value(); 707 } else { 708 data_offset = mirror::Array::DataOffset(sizeof(int32_t)).Int32Value(); 709 } 710 711 // If index is constant, just fold it into the data offset 712 if (constant_index) { 713 data_offset += mir_graph_->ConstantValue(rl_index) << scale; 714 } 715 716 /* null object? */ 717 GenNullCheck(rl_array.reg, opt_flags); 718 719 bool needs_range_check = (!(opt_flags & MIR_IGNORE_RANGE_CHECK)); 720 RegStorage reg_len; 721 if (needs_range_check) { 722 reg_len = AllocTemp(); 723 /* Get len */ 724 Load32Disp(rl_array.reg, len_offset, reg_len); 725 MarkPossibleNullPointerException(opt_flags); 726 } else { 727 ForceImplicitNullCheck(rl_array.reg, opt_flags); 728 } 729 if (rl_dest.wide || rl_dest.fp || constant_index) { 730 RegStorage reg_ptr; 731 if (constant_index) { 732 reg_ptr = rl_array.reg; // NOTE: must not alter reg_ptr in constant case. 733 } else { 734 // No special indexed operation, lea + load w/ displacement 735 reg_ptr = AllocTempRef(); 736 OpRegRegRegShift(kOpAdd, reg_ptr, rl_array.reg, As64BitReg(rl_index.reg), 737 EncodeShift(kA64Lsl, scale)); 738 FreeTemp(rl_index.reg); 739 } 740 rl_result = EvalLoc(rl_dest, reg_class, true); 741 742 if (needs_range_check) { 743 if (constant_index) { 744 GenArrayBoundsCheck(mir_graph_->ConstantValue(rl_index), reg_len); 745 } else { 746 GenArrayBoundsCheck(rl_index.reg, reg_len); 747 } 748 FreeTemp(reg_len); 749 } 750 if (rl_result.ref) { 751 LoadRefDisp(reg_ptr, data_offset, rl_result.reg, kNotVolatile); 752 } else { 753 LoadBaseDisp(reg_ptr, data_offset, rl_result.reg, size, kNotVolatile); 754 } 755 MarkPossibleNullPointerException(opt_flags); 756 if (!constant_index) { 757 FreeTemp(reg_ptr); 758 } 759 if (rl_dest.wide) { 760 StoreValueWide(rl_dest, rl_result); 761 } else { 762 StoreValue(rl_dest, rl_result); 763 } 764 } else { 765 // Offset base, then use indexed load 766 RegStorage reg_ptr = AllocTempRef(); 767 OpRegRegImm(kOpAdd, reg_ptr, rl_array.reg, data_offset); 768 FreeTemp(rl_array.reg); 769 rl_result = EvalLoc(rl_dest, reg_class, true); 770 771 if (needs_range_check) { 772 GenArrayBoundsCheck(rl_index.reg, reg_len); 773 FreeTemp(reg_len); 774 } 775 if (rl_result.ref) { 776 LoadRefIndexed(reg_ptr, As64BitReg(rl_index.reg), rl_result.reg); 777 } else { 778 LoadBaseIndexed(reg_ptr, As64BitReg(rl_index.reg), rl_result.reg, scale, size); 779 } 780 MarkPossibleNullPointerException(opt_flags); 781 FreeTemp(reg_ptr); 782 StoreValue(rl_dest, rl_result); 783 } 784} 785 786/* 787 * Generate array store 788 * 789 */ 790void Arm64Mir2Lir::GenArrayPut(int opt_flags, OpSize size, RegLocation rl_array, 791 RegLocation rl_index, RegLocation rl_src, int scale, bool card_mark) { 792 RegisterClass reg_class = RegClassBySize(size); 793 int len_offset = mirror::Array::LengthOffset().Int32Value(); 794 bool constant_index = rl_index.is_const; 795 796 int data_offset; 797 if (size == k64 || size == kDouble) { 798 data_offset = mirror::Array::DataOffset(sizeof(int64_t)).Int32Value(); 799 } else { 800 data_offset = mirror::Array::DataOffset(sizeof(int32_t)).Int32Value(); 801 } 802 803 // If index is constant, just fold it into the data offset. 804 if (constant_index) { 805 data_offset += mir_graph_->ConstantValue(rl_index) << scale; 806 } 807 808 rl_array = LoadValue(rl_array, kRefReg); 809 if (!constant_index) { 810 rl_index = LoadValue(rl_index, kCoreReg); 811 } 812 813 RegStorage reg_ptr; 814 bool allocated_reg_ptr_temp = false; 815 if (constant_index) { 816 reg_ptr = rl_array.reg; 817 } else if (IsTemp(rl_array.reg) && !card_mark) { 818 Clobber(rl_array.reg); 819 reg_ptr = rl_array.reg; 820 } else { 821 allocated_reg_ptr_temp = true; 822 reg_ptr = AllocTempRef(); 823 } 824 825 /* null object? */ 826 GenNullCheck(rl_array.reg, opt_flags); 827 828 bool needs_range_check = (!(opt_flags & MIR_IGNORE_RANGE_CHECK)); 829 RegStorage reg_len; 830 if (needs_range_check) { 831 reg_len = AllocTemp(); 832 // NOTE: max live temps(4) here. 833 /* Get len */ 834 Load32Disp(rl_array.reg, len_offset, reg_len); 835 MarkPossibleNullPointerException(opt_flags); 836 } else { 837 ForceImplicitNullCheck(rl_array.reg, opt_flags); 838 } 839 /* at this point, reg_ptr points to array, 2 live temps */ 840 if (rl_src.wide || rl_src.fp || constant_index) { 841 if (rl_src.wide) { 842 rl_src = LoadValueWide(rl_src, reg_class); 843 } else { 844 rl_src = LoadValue(rl_src, reg_class); 845 } 846 if (!constant_index) { 847 OpRegRegRegShift(kOpAdd, reg_ptr, rl_array.reg, As64BitReg(rl_index.reg), 848 EncodeShift(kA64Lsl, scale)); 849 } 850 if (needs_range_check) { 851 if (constant_index) { 852 GenArrayBoundsCheck(mir_graph_->ConstantValue(rl_index), reg_len); 853 } else { 854 GenArrayBoundsCheck(rl_index.reg, reg_len); 855 } 856 FreeTemp(reg_len); 857 } 858 if (rl_src.ref) { 859 StoreRefDisp(reg_ptr, data_offset, rl_src.reg, kNotVolatile); 860 } else { 861 StoreBaseDisp(reg_ptr, data_offset, rl_src.reg, size, kNotVolatile); 862 } 863 MarkPossibleNullPointerException(opt_flags); 864 } else { 865 /* reg_ptr -> array data */ 866 OpRegRegImm(kOpAdd, reg_ptr, rl_array.reg, data_offset); 867 rl_src = LoadValue(rl_src, reg_class); 868 if (needs_range_check) { 869 GenArrayBoundsCheck(rl_index.reg, reg_len); 870 FreeTemp(reg_len); 871 } 872 if (rl_src.ref) { 873 StoreRefIndexed(reg_ptr, As64BitReg(rl_index.reg), rl_src.reg); 874 } else { 875 StoreBaseIndexed(reg_ptr, As64BitReg(rl_index.reg), rl_src.reg, scale, size); 876 } 877 MarkPossibleNullPointerException(opt_flags); 878 } 879 if (allocated_reg_ptr_temp) { 880 FreeTemp(reg_ptr); 881 } 882 if (card_mark) { 883 MarkGCCard(rl_src.reg, rl_array.reg); 884 } 885} 886 887void Arm64Mir2Lir::GenShiftImmOpLong(Instruction::Code opcode, 888 RegLocation rl_dest, RegLocation rl_src, RegLocation rl_shift) { 889 OpKind op = kOpBkpt; 890 // Per spec, we only care about low 6 bits of shift amount. 891 int shift_amount = mir_graph_->ConstantValue(rl_shift) & 0x3f; 892 rl_src = LoadValueWide(rl_src, kCoreReg); 893 if (shift_amount == 0) { 894 StoreValueWide(rl_dest, rl_src); 895 return; 896 } 897 898 RegLocation rl_result = EvalLocWide(rl_dest, kCoreReg, true); 899 switch (opcode) { 900 case Instruction::SHL_LONG: 901 case Instruction::SHL_LONG_2ADDR: 902 op = kOpLsl; 903 break; 904 case Instruction::SHR_LONG: 905 case Instruction::SHR_LONG_2ADDR: 906 op = kOpAsr; 907 break; 908 case Instruction::USHR_LONG: 909 case Instruction::USHR_LONG_2ADDR: 910 op = kOpLsr; 911 break; 912 default: 913 LOG(FATAL) << "Unexpected case"; 914 } 915 OpRegRegImm(op, rl_result.reg, rl_src.reg, shift_amount); 916 StoreValueWide(rl_dest, rl_result); 917} 918 919void Arm64Mir2Lir::GenArithImmOpLong(Instruction::Code opcode, RegLocation rl_dest, 920 RegLocation rl_src1, RegLocation rl_src2) { 921 if ((opcode == Instruction::SUB_LONG) || (opcode == Instruction::SUB_LONG_2ADDR)) { 922 if (!rl_src2.is_const) { 923 return GenArithOpLong(opcode, rl_dest, rl_src1, rl_src2); 924 } 925 } else { 926 // Associativity. 927 if (!rl_src2.is_const) { 928 DCHECK(rl_src1.is_const); 929 std::swap(rl_src1, rl_src2); 930 } 931 } 932 DCHECK(rl_src2.is_const); 933 934 OpKind op = kOpBkpt; 935 int64_t val = mir_graph_->ConstantValueWide(rl_src2); 936 937 switch (opcode) { 938 case Instruction::ADD_LONG: 939 case Instruction::ADD_LONG_2ADDR: 940 op = kOpAdd; 941 break; 942 case Instruction::SUB_LONG: 943 case Instruction::SUB_LONG_2ADDR: 944 op = kOpSub; 945 break; 946 case Instruction::AND_LONG: 947 case Instruction::AND_LONG_2ADDR: 948 op = kOpAnd; 949 break; 950 case Instruction::OR_LONG: 951 case Instruction::OR_LONG_2ADDR: 952 op = kOpOr; 953 break; 954 case Instruction::XOR_LONG: 955 case Instruction::XOR_LONG_2ADDR: 956 op = kOpXor; 957 break; 958 default: 959 LOG(FATAL) << "Unexpected opcode"; 960 } 961 962 rl_src1 = LoadValueWide(rl_src1, kCoreReg); 963 RegLocation rl_result = EvalLocWide(rl_dest, kCoreReg, true); 964 OpRegRegImm64(op, rl_result.reg, rl_src1.reg, val); 965 StoreValueWide(rl_dest, rl_result); 966} 967 968/** 969 * @brief Split a register list in pairs or registers. 970 * 971 * Given a list of registers in @p reg_mask, split the list in pairs. Use as follows: 972 * @code 973 * int reg1 = -1, reg2 = -1; 974 * while (reg_mask) { 975 * reg_mask = GenPairWise(reg_mask, & reg1, & reg2); 976 * if (UNLIKELY(reg2 < 0)) { 977 * // Single register in reg1. 978 * } else { 979 * // Pair in reg1, reg2. 980 * } 981 * } 982 * @endcode 983 */ 984uint32_t Arm64Mir2Lir::GenPairWise(uint32_t reg_mask, int* reg1, int* reg2) { 985 // Find first register. 986 int first_bit_set = __builtin_ctz(reg_mask) + 1; 987 int reg = *reg1 + first_bit_set; 988 reg_mask >>= first_bit_set; 989 990 if (LIKELY(reg_mask)) { 991 // Save the first register, find the second and use the pair opcode. 992 int second_bit_set = __builtin_ctz(reg_mask) + 1; 993 *reg2 = reg; 994 reg_mask >>= second_bit_set; 995 *reg1 = reg + second_bit_set; 996 return reg_mask; 997 } 998 999 // Use the single opcode, as we just have one register. 1000 *reg1 = reg; 1001 *reg2 = -1; 1002 return reg_mask; 1003} 1004 1005void Arm64Mir2Lir::UnSpillCoreRegs(RegStorage base, int offset, uint32_t reg_mask) { 1006 int reg1 = -1, reg2 = -1; 1007 const int reg_log2_size = 3; 1008 1009 for (offset = (offset >> reg_log2_size); reg_mask; offset += 2) { 1010 reg_mask = GenPairWise(reg_mask, & reg1, & reg2); 1011 if (UNLIKELY(reg2 < 0)) { 1012 NewLIR3(WIDE(kA64Ldr3rXD), RegStorage::Solo64(reg1).GetReg(), base.GetReg(), offset); 1013 } else { 1014 NewLIR4(WIDE(kA64Ldp4rrXD), RegStorage::Solo64(reg2).GetReg(), 1015 RegStorage::Solo64(reg1).GetReg(), base.GetReg(), offset); 1016 } 1017 } 1018} 1019 1020void Arm64Mir2Lir::SpillCoreRegs(RegStorage base, int offset, uint32_t reg_mask) { 1021 int reg1 = -1, reg2 = -1; 1022 const int reg_log2_size = 3; 1023 1024 for (offset = (offset >> reg_log2_size); reg_mask; offset += 2) { 1025 reg_mask = GenPairWise(reg_mask, & reg1, & reg2); 1026 if (UNLIKELY(reg2 < 0)) { 1027 NewLIR3(WIDE(kA64Str3rXD), RegStorage::Solo64(reg1).GetReg(), base.GetReg(), offset); 1028 } else { 1029 NewLIR4(WIDE(kA64Stp4rrXD), RegStorage::Solo64(reg2).GetReg(), 1030 RegStorage::Solo64(reg1).GetReg(), base.GetReg(), offset); 1031 } 1032 } 1033} 1034 1035void Arm64Mir2Lir::UnSpillFPRegs(RegStorage base, int offset, uint32_t reg_mask) { 1036 int reg1 = -1, reg2 = -1; 1037 const int reg_log2_size = 3; 1038 1039 for (offset = (offset >> reg_log2_size); reg_mask; offset += 2) { 1040 reg_mask = GenPairWise(reg_mask, & reg1, & reg2); 1041 if (UNLIKELY(reg2 < 0)) { 1042 NewLIR3(FWIDE(kA64Ldr3fXD), RegStorage::FloatSolo64(reg1).GetReg(), base.GetReg(), offset); 1043 } else { 1044 NewLIR4(WIDE(kA64Ldp4ffXD), RegStorage::FloatSolo64(reg2).GetReg(), 1045 RegStorage::FloatSolo64(reg1).GetReg(), base.GetReg(), offset); 1046 } 1047 } 1048} 1049 1050// TODO(Arm64): consider using ld1 and st1? 1051void Arm64Mir2Lir::SpillFPRegs(RegStorage base, int offset, uint32_t reg_mask) { 1052 int reg1 = -1, reg2 = -1; 1053 const int reg_log2_size = 3; 1054 1055 for (offset = (offset >> reg_log2_size); reg_mask; offset += 2) { 1056 reg_mask = GenPairWise(reg_mask, & reg1, & reg2); 1057 if (UNLIKELY(reg2 < 0)) { 1058 NewLIR3(FWIDE(kA64Str3fXD), RegStorage::FloatSolo64(reg1).GetReg(), base.GetReg(), offset); 1059 } else { 1060 NewLIR4(WIDE(kA64Stp4ffXD), RegStorage::FloatSolo64(reg2).GetReg(), 1061 RegStorage::FloatSolo64(reg1).GetReg(), base.GetReg(), offset); 1062 } 1063 } 1064} 1065 1066} // namespace art 1067