int_arm64.cc revision 0025a86411145eb7cd4971f9234fc21c7b4aced1
1/* 2 * Copyright (C) 2011 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17/* This file contains codegen for the Thumb2 ISA. */ 18 19#include "arm64_lir.h" 20#include "codegen_arm64.h" 21#include "dex/quick/mir_to_lir-inl.h" 22#include "dex/reg_storage_eq.h" 23#include "entrypoints/quick/quick_entrypoints.h" 24#include "mirror/array.h" 25 26namespace art { 27 28LIR* Arm64Mir2Lir::OpCmpBranch(ConditionCode cond, RegStorage src1, RegStorage src2, LIR* target) { 29 OpRegReg(kOpCmp, src1, src2); 30 return OpCondBranch(cond, target); 31} 32 33LIR* Arm64Mir2Lir::OpIT(ConditionCode ccode, const char* guide) { 34 LOG(FATAL) << "Unexpected use of OpIT for Arm64"; 35 return NULL; 36} 37 38void Arm64Mir2Lir::OpEndIT(LIR* it) { 39 LOG(FATAL) << "Unexpected use of OpEndIT for Arm64"; 40} 41 42/* 43 * 64-bit 3way compare function. 44 * cmp xA, xB 45 * csinc wC, wzr, wzr, eq // wC = (xA == xB) ? 0 : 1 46 * csneg wC, wC, wC, ge // wC = (xA >= xB) ? wC : -wC 47 */ 48void Arm64Mir2Lir::GenCmpLong(RegLocation rl_dest, RegLocation rl_src1, 49 RegLocation rl_src2) { 50 RegLocation rl_result; 51 rl_src1 = LoadValueWide(rl_src1, kCoreReg); 52 rl_src2 = LoadValueWide(rl_src2, kCoreReg); 53 rl_result = EvalLoc(rl_dest, kCoreReg, true); 54 55 OpRegReg(kOpCmp, rl_src1.reg, rl_src2.reg); 56 NewLIR4(kA64Csinc4rrrc, rl_result.reg.GetReg(), rwzr, rwzr, kArmCondEq); 57 NewLIR4(kA64Csneg4rrrc, rl_result.reg.GetReg(), rl_result.reg.GetReg(), 58 rl_result.reg.GetReg(), kArmCondGe); 59 StoreValue(rl_dest, rl_result); 60} 61 62void Arm64Mir2Lir::GenShiftOpLong(Instruction::Code opcode, RegLocation rl_dest, 63 RegLocation rl_src1, RegLocation rl_shift) { 64 OpKind op = kOpBkpt; 65 switch (opcode) { 66 case Instruction::SHL_LONG: 67 case Instruction::SHL_LONG_2ADDR: 68 op = kOpLsl; 69 break; 70 case Instruction::SHR_LONG: 71 case Instruction::SHR_LONG_2ADDR: 72 op = kOpAsr; 73 break; 74 case Instruction::USHR_LONG: 75 case Instruction::USHR_LONG_2ADDR: 76 op = kOpLsr; 77 break; 78 default: 79 LOG(FATAL) << "Unexpected case: " << opcode; 80 } 81 rl_shift = LoadValue(rl_shift, kCoreReg); 82 rl_src1 = LoadValueWide(rl_src1, kCoreReg); 83 RegLocation rl_result = EvalLocWide(rl_dest, kCoreReg, true); 84 OpRegRegReg(op, rl_result.reg, rl_src1.reg, As64BitReg(rl_shift.reg)); 85 StoreValueWide(rl_dest, rl_result); 86} 87 88void Arm64Mir2Lir::GenSelect(BasicBlock* bb, MIR* mir) { 89 RegLocation rl_result; 90 RegLocation rl_src = mir_graph_->GetSrc(mir, 0); 91 RegLocation rl_dest = mir_graph_->GetDest(mir); 92 RegisterClass src_reg_class = rl_src.ref ? kRefReg : kCoreReg; 93 RegisterClass result_reg_class = rl_dest.ref ? kRefReg : kCoreReg; 94 rl_src = LoadValue(rl_src, src_reg_class); 95 ArmConditionCode code = ArmConditionEncoding(mir->meta.ccode); 96 97 RegLocation rl_true = mir_graph_->reg_location_[mir->ssa_rep->uses[1]]; 98 RegLocation rl_false = mir_graph_->reg_location_[mir->ssa_rep->uses[2]]; 99 rl_true = LoadValue(rl_true, result_reg_class); 100 rl_false = LoadValue(rl_false, result_reg_class); 101 rl_result = EvalLoc(rl_dest, result_reg_class, true); 102 OpRegImm(kOpCmp, rl_src.reg, 0); 103 NewLIR4(kA64Csel4rrrc, rl_result.reg.GetReg(), rl_true.reg.GetReg(), 104 rl_false.reg.GetReg(), code); 105 StoreValue(rl_dest, rl_result); 106} 107 108void Arm64Mir2Lir::GenFusedLongCmpBranch(BasicBlock* bb, MIR* mir) { 109 RegLocation rl_src1 = mir_graph_->GetSrcWide(mir, 0); 110 RegLocation rl_src2 = mir_graph_->GetSrcWide(mir, 2); 111 LIR* taken = &block_label_list_[bb->taken]; 112 LIR* not_taken = &block_label_list_[bb->fall_through]; 113 rl_src1 = LoadValueWide(rl_src1, kCoreReg); 114 // Normalize such that if either operand is constant, src2 will be constant. 115 ConditionCode ccode = mir->meta.ccode; 116 if (rl_src1.is_const) { 117 std::swap(rl_src1, rl_src2); 118 ccode = FlipComparisonOrder(ccode); 119 } 120 121 if (rl_src2.is_const) { 122 rl_src2 = UpdateLocWide(rl_src2); 123 int64_t val = mir_graph_->ConstantValueWide(rl_src2); 124 // Special handling using cbz & cbnz. 125 if (val == 0 && (ccode == kCondEq || ccode == kCondNe)) { 126 OpCmpImmBranch(ccode, rl_src1.reg, 0, taken); 127 OpCmpImmBranch(NegateComparison(ccode), rl_src1.reg, 0, not_taken); 128 return; 129 // Only handle Imm if src2 is not already in a register. 130 } else if (rl_src2.location != kLocPhysReg) { 131 OpRegImm64(kOpCmp, rl_src1.reg, val); 132 OpCondBranch(ccode, taken); 133 OpCondBranch(NegateComparison(ccode), not_taken); 134 return; 135 } 136 } 137 138 rl_src2 = LoadValueWide(rl_src2, kCoreReg); 139 OpRegReg(kOpCmp, rl_src1.reg, rl_src2.reg); 140 OpCondBranch(ccode, taken); 141 OpCondBranch(NegateComparison(ccode), not_taken); 142} 143 144/* 145 * Generate a register comparison to an immediate and branch. Caller 146 * is responsible for setting branch target field. 147 */ 148LIR* Arm64Mir2Lir::OpCmpImmBranch(ConditionCode cond, RegStorage reg, int check_value, 149 LIR* target) { 150 LIR* branch; 151 ArmConditionCode arm_cond = ArmConditionEncoding(cond); 152 if (check_value == 0 && (arm_cond == kArmCondEq || arm_cond == kArmCondNe)) { 153 ArmOpcode opcode = (arm_cond == kArmCondEq) ? kA64Cbz2rt : kA64Cbnz2rt; 154 ArmOpcode wide = reg.Is64Bit() ? WIDE(0) : UNWIDE(0); 155 branch = NewLIR2(opcode | wide, reg.GetReg(), 0); 156 } else { 157 OpRegImm(kOpCmp, reg, check_value); 158 branch = NewLIR2(kA64B2ct, arm_cond, 0); 159 } 160 branch->target = target; 161 return branch; 162} 163 164LIR* Arm64Mir2Lir::OpCmpMemImmBranch(ConditionCode cond, RegStorage temp_reg, 165 RegStorage base_reg, int offset, int check_value, 166 LIR* target) { 167 // It is possible that temp register is 64-bit. (ArgReg or RefReg) 168 // Always compare 32-bit value no matter what temp_reg is. 169 if (temp_reg.Is64Bit()) { 170 temp_reg = As32BitReg(temp_reg); 171 } 172 Load32Disp(base_reg, offset, temp_reg); 173 LIR* branch = OpCmpImmBranch(cond, temp_reg, check_value, target); 174 return branch; 175} 176 177LIR* Arm64Mir2Lir::OpRegCopyNoInsert(RegStorage r_dest, RegStorage r_src) { 178 bool dest_is_fp = r_dest.IsFloat(); 179 bool src_is_fp = r_src.IsFloat(); 180 ArmOpcode opcode = kA64Brk1d; 181 LIR* res; 182 183 if (LIKELY(dest_is_fp == src_is_fp)) { 184 if (LIKELY(!dest_is_fp)) { 185 DCHECK_EQ(r_dest.Is64Bit(), r_src.Is64Bit()); 186 187 // Core/core copy. 188 // Copies involving the sp register require a different instruction. 189 opcode = UNLIKELY(A64_REG_IS_SP(r_dest.GetReg())) ? kA64Add4RRdT : kA64Mov2rr; 190 191 // TODO(Arm64): kA64Add4RRdT formally has 4 args, but is used as a 2 args instruction. 192 // This currently works because the other arguments are set to 0 by default. We should 193 // rather introduce an alias kA64Mov2RR. 194 195 // core/core copy. Do a x/x copy only if both registers are x. 196 if (r_dest.Is64Bit() && r_src.Is64Bit()) { 197 opcode = WIDE(opcode); 198 } 199 } else { 200 // Float/float copy. 201 bool dest_is_double = r_dest.IsDouble(); 202 bool src_is_double = r_src.IsDouble(); 203 204 // We do not do float/double or double/float casts here. 205 DCHECK_EQ(dest_is_double, src_is_double); 206 207 // Homogeneous float/float copy. 208 opcode = (dest_is_double) ? FWIDE(kA64Fmov2ff) : kA64Fmov2ff; 209 } 210 } else { 211 // Inhomogeneous register copy. 212 if (dest_is_fp) { 213 if (r_dest.IsDouble()) { 214 opcode = kA64Fmov2Sx; 215 } else { 216 r_src = Check32BitReg(r_src); 217 opcode = kA64Fmov2sw; 218 } 219 } else { 220 if (r_src.IsDouble()) { 221 opcode = kA64Fmov2xS; 222 } else { 223 r_dest = Check32BitReg(r_dest); 224 opcode = kA64Fmov2ws; 225 } 226 } 227 } 228 229 res = RawLIR(current_dalvik_offset_, opcode, r_dest.GetReg(), r_src.GetReg()); 230 231 if (!(cu_->disable_opt & (1 << kSafeOptimizations)) && r_dest == r_src) { 232 res->flags.is_nop = true; 233 } 234 235 return res; 236} 237 238void Arm64Mir2Lir::OpRegCopy(RegStorage r_dest, RegStorage r_src) { 239 if (r_dest != r_src) { 240 LIR* res = OpRegCopyNoInsert(r_dest, r_src); 241 AppendLIR(res); 242 } 243} 244 245void Arm64Mir2Lir::OpRegCopyWide(RegStorage r_dest, RegStorage r_src) { 246 OpRegCopy(r_dest, r_src); 247} 248 249// Table of magic divisors 250struct MagicTable { 251 uint32_t magic; 252 uint32_t shift; 253 DividePattern pattern; 254}; 255 256static const MagicTable magic_table[] = { 257 {0, 0, DivideNone}, // 0 258 {0, 0, DivideNone}, // 1 259 {0, 0, DivideNone}, // 2 260 {0x55555556, 0, Divide3}, // 3 261 {0, 0, DivideNone}, // 4 262 {0x66666667, 1, Divide5}, // 5 263 {0x2AAAAAAB, 0, Divide3}, // 6 264 {0x92492493, 2, Divide7}, // 7 265 {0, 0, DivideNone}, // 8 266 {0x38E38E39, 1, Divide5}, // 9 267 {0x66666667, 2, Divide5}, // 10 268 {0x2E8BA2E9, 1, Divide5}, // 11 269 {0x2AAAAAAB, 1, Divide5}, // 12 270 {0x4EC4EC4F, 2, Divide5}, // 13 271 {0x92492493, 3, Divide7}, // 14 272 {0x88888889, 3, Divide7}, // 15 273}; 274 275// Integer division by constant via reciprocal multiply (Hacker's Delight, 10-4) 276bool Arm64Mir2Lir::SmallLiteralDivRem(Instruction::Code dalvik_opcode, bool is_div, 277 RegLocation rl_src, RegLocation rl_dest, int lit) { 278 if ((lit < 0) || (lit >= static_cast<int>(arraysize(magic_table)))) { 279 return false; 280 } 281 DividePattern pattern = magic_table[lit].pattern; 282 if (pattern == DivideNone) { 283 return false; 284 } 285 // Tuning: add rem patterns 286 if (!is_div) { 287 return false; 288 } 289 290 RegStorage r_magic = AllocTemp(); 291 LoadConstant(r_magic, magic_table[lit].magic); 292 rl_src = LoadValue(rl_src, kCoreReg); 293 RegLocation rl_result = EvalLoc(rl_dest, kCoreReg, true); 294 RegStorage r_long_mul = AllocTemp(); 295 NewLIR4(kA64Smaddl4xwwx, As64BitReg(r_long_mul).GetReg(), 296 r_magic.GetReg(), rl_src.reg.GetReg(), rxzr); 297 switch (pattern) { 298 case Divide3: 299 OpRegRegImm(kOpLsr, As64BitReg(r_long_mul), As64BitReg(r_long_mul), 32); 300 OpRegRegRegShift(kOpSub, rl_result.reg, r_long_mul, rl_src.reg, EncodeShift(kA64Asr, 31)); 301 break; 302 case Divide5: 303 OpRegRegImm(kOpAsr, As64BitReg(r_long_mul), As64BitReg(r_long_mul), 304 32 + magic_table[lit].shift); 305 OpRegRegRegShift(kOpSub, rl_result.reg, r_long_mul, rl_src.reg, EncodeShift(kA64Asr, 31)); 306 break; 307 case Divide7: 308 OpRegRegRegShift(kOpAdd, As64BitReg(r_long_mul), As64BitReg(rl_src.reg), 309 As64BitReg(r_long_mul), EncodeShift(kA64Lsr, 32)); 310 OpRegRegImm(kOpAsr, r_long_mul, r_long_mul, magic_table[lit].shift); 311 OpRegRegRegShift(kOpSub, rl_result.reg, r_long_mul, rl_src.reg, EncodeShift(kA64Asr, 31)); 312 break; 313 default: 314 LOG(FATAL) << "Unexpected pattern: " << pattern; 315 } 316 StoreValue(rl_dest, rl_result); 317 return true; 318} 319 320// Returns true if it added instructions to 'cu' to divide 'rl_src' by 'lit' 321// and store the result in 'rl_dest'. 322bool Arm64Mir2Lir::HandleEasyDivRem(Instruction::Code dalvik_opcode, bool is_div, 323 RegLocation rl_src, RegLocation rl_dest, int lit) { 324 if (lit < 2) { 325 return false; 326 } 327 if (!IsPowerOfTwo(lit)) { 328 return SmallLiteralDivRem(dalvik_opcode, is_div, rl_src, rl_dest, lit); 329 } 330 int k = LowestSetBit(lit); 331 if (k >= 30) { 332 // Avoid special cases. 333 return false; 334 } 335 rl_src = LoadValue(rl_src, kCoreReg); 336 RegLocation rl_result = EvalLoc(rl_dest, kCoreReg, true); 337 if (is_div) { 338 RegStorage t_reg = AllocTemp(); 339 if (lit == 2) { 340 // Division by 2 is by far the most common division by constant. 341 OpRegRegRegShift(kOpAdd, t_reg, rl_src.reg, rl_src.reg, EncodeShift(kA64Lsr, 32 - k)); 342 OpRegRegImm(kOpAsr, rl_result.reg, t_reg, k); 343 } else { 344 OpRegRegImm(kOpAsr, t_reg, rl_src.reg, 31); 345 OpRegRegRegShift(kOpAdd, t_reg, rl_src.reg, t_reg, EncodeShift(kA64Lsr, 32 - k)); 346 OpRegRegImm(kOpAsr, rl_result.reg, t_reg, k); 347 } 348 } else { 349 RegStorage t_reg = AllocTemp(); 350 if (lit == 2) { 351 OpRegRegRegShift(kOpAdd, t_reg, rl_src.reg, rl_src.reg, EncodeShift(kA64Lsr, 32 - k)); 352 OpRegRegImm(kOpAnd, t_reg, t_reg, lit - 1); 353 OpRegRegRegShift(kOpSub, rl_result.reg, t_reg, rl_src.reg, EncodeShift(kA64Lsr, 32 - k)); 354 } else { 355 RegStorage t_reg2 = AllocTemp(); 356 OpRegRegImm(kOpAsr, t_reg, rl_src.reg, 31); 357 OpRegRegRegShift(kOpAdd, t_reg2, rl_src.reg, t_reg, EncodeShift(kA64Lsr, 32 - k)); 358 OpRegRegImm(kOpAnd, t_reg2, t_reg2, lit - 1); 359 OpRegRegRegShift(kOpSub, rl_result.reg, t_reg2, t_reg, EncodeShift(kA64Lsr, 32 - k)); 360 } 361 } 362 StoreValue(rl_dest, rl_result); 363 return true; 364} 365 366bool Arm64Mir2Lir::EasyMultiply(RegLocation rl_src, RegLocation rl_dest, int lit) { 367 LOG(FATAL) << "Unexpected use of EasyMultiply for Arm64"; 368 return false; 369} 370 371RegLocation Arm64Mir2Lir::GenDivRem(RegLocation rl_dest, RegLocation rl_src1, 372 RegLocation rl_src2, bool is_div, bool check_zero) { 373 LOG(FATAL) << "Unexpected use of GenDivRem for Arm64"; 374 return rl_dest; 375} 376 377RegLocation Arm64Mir2Lir::GenDivRemLit(RegLocation rl_dest, RegLocation rl_src1, int lit, bool is_div) { 378 LOG(FATAL) << "Unexpected use of GenDivRemLit for Arm64"; 379 return rl_dest; 380} 381 382RegLocation Arm64Mir2Lir::GenDivRemLit(RegLocation rl_dest, RegStorage reg1, int lit, bool is_div) { 383 RegLocation rl_result = EvalLoc(rl_dest, kCoreReg, true); 384 385 // Put the literal in a temp. 386 RegStorage lit_temp = AllocTemp(); 387 LoadConstant(lit_temp, lit); 388 // Use the generic case for div/rem with arg2 in a register. 389 // TODO: The literal temp can be freed earlier during a modulus to reduce reg pressure. 390 rl_result = GenDivRem(rl_result, reg1, lit_temp, is_div); 391 FreeTemp(lit_temp); 392 393 return rl_result; 394} 395 396RegLocation Arm64Mir2Lir::GenDivRem(RegLocation rl_dest, RegStorage r_src1, RegStorage r_src2, 397 bool is_div) { 398 CHECK_EQ(r_src1.Is64Bit(), r_src2.Is64Bit()); 399 400 RegLocation rl_result = EvalLoc(rl_dest, kCoreReg, true); 401 if (is_div) { 402 OpRegRegReg(kOpDiv, rl_result.reg, r_src1, r_src2); 403 } else { 404 // temp = r_src1 / r_src2 405 // dest = r_src1 - temp * r_src2 406 RegStorage temp; 407 ArmOpcode wide; 408 if (rl_result.reg.Is64Bit()) { 409 temp = AllocTempWide(); 410 wide = WIDE(0); 411 } else { 412 temp = AllocTemp(); 413 wide = UNWIDE(0); 414 } 415 OpRegRegReg(kOpDiv, temp, r_src1, r_src2); 416 NewLIR4(kA64Msub4rrrr | wide, rl_result.reg.GetReg(), temp.GetReg(), 417 r_src1.GetReg(), r_src2.GetReg()); 418 FreeTemp(temp); 419 } 420 return rl_result; 421} 422 423bool Arm64Mir2Lir::GenInlinedAbsLong(CallInfo* info) { 424 RegLocation rl_src = info->args[0]; 425 rl_src = LoadValueWide(rl_src, kCoreReg); 426 RegLocation rl_dest = InlineTargetWide(info); 427 RegLocation rl_result = EvalLoc(rl_dest, kCoreReg, true); 428 RegStorage sign_reg = AllocTempWide(); 429 // abs(x) = y<=x>>63, (x+y)^y. 430 OpRegRegImm(kOpAsr, sign_reg, rl_src.reg, 63); 431 OpRegRegReg(kOpAdd, rl_result.reg, rl_src.reg, sign_reg); 432 OpRegReg(kOpXor, rl_result.reg, sign_reg); 433 StoreValueWide(rl_dest, rl_result); 434 return true; 435} 436 437bool Arm64Mir2Lir::GenInlinedMinMax(CallInfo* info, bool is_min, bool is_long) { 438 DCHECK_EQ(cu_->instruction_set, kArm64); 439 RegLocation rl_src1 = info->args[0]; 440 RegLocation rl_src2 = (is_long) ? info->args[2] : info->args[1]; 441 rl_src1 = (is_long) ? LoadValueWide(rl_src1, kCoreReg) : LoadValue(rl_src1, kCoreReg); 442 rl_src2 = (is_long) ? LoadValueWide(rl_src2, kCoreReg) : LoadValue(rl_src2, kCoreReg); 443 RegLocation rl_dest = (is_long) ? InlineTargetWide(info) : InlineTarget(info); 444 RegLocation rl_result = EvalLoc(rl_dest, kCoreReg, true); 445 OpRegReg(kOpCmp, rl_src1.reg, rl_src2.reg); 446 NewLIR4((is_long) ? WIDE(kA64Csel4rrrc) : kA64Csel4rrrc, rl_result.reg.GetReg(), 447 rl_src1.reg.GetReg(), rl_src2.reg.GetReg(), (is_min) ? kArmCondLt : kArmCondGt); 448 (is_long) ? StoreValueWide(rl_dest, rl_result) :StoreValue(rl_dest, rl_result); 449 return true; 450} 451 452bool Arm64Mir2Lir::GenInlinedPeek(CallInfo* info, OpSize size) { 453 RegLocation rl_src_address = info->args[0]; // long address 454 RegLocation rl_dest = (size == k64) ? InlineTargetWide(info) : InlineTarget(info); 455 RegLocation rl_address = LoadValueWide(rl_src_address, kCoreReg); 456 RegLocation rl_result = EvalLoc(rl_dest, kCoreReg, true); 457 458 LoadBaseDisp(rl_address.reg, 0, rl_result.reg, size, kNotVolatile); 459 if (size == k64) { 460 StoreValueWide(rl_dest, rl_result); 461 } else { 462 DCHECK(size == kSignedByte || size == kSignedHalf || size == k32); 463 StoreValue(rl_dest, rl_result); 464 } 465 return true; 466} 467 468bool Arm64Mir2Lir::GenInlinedPoke(CallInfo* info, OpSize size) { 469 RegLocation rl_src_address = info->args[0]; // long address 470 RegLocation rl_src_value = info->args[2]; // [size] value 471 RegLocation rl_address = LoadValueWide(rl_src_address, kCoreReg); 472 473 RegLocation rl_value; 474 if (size == k64) { 475 rl_value = LoadValueWide(rl_src_value, kCoreReg); 476 } else { 477 DCHECK(size == kSignedByte || size == kSignedHalf || size == k32); 478 rl_value = LoadValue(rl_src_value, kCoreReg); 479 } 480 StoreBaseDisp(rl_address.reg, 0, rl_value.reg, size, kNotVolatile); 481 return true; 482} 483 484void Arm64Mir2Lir::OpLea(RegStorage r_base, RegStorage reg1, RegStorage reg2, int scale, int offset) { 485 LOG(FATAL) << "Unexpected use of OpLea for Arm64"; 486} 487 488void Arm64Mir2Lir::OpTlsCmp(ThreadOffset<4> offset, int val) { 489 UNIMPLEMENTED(FATAL) << "Should not be used."; 490} 491 492void Arm64Mir2Lir::OpTlsCmp(ThreadOffset<8> offset, int val) { 493 LOG(FATAL) << "Unexpected use of OpTlsCmp for Arm64"; 494} 495 496bool Arm64Mir2Lir::GenInlinedCas(CallInfo* info, bool is_long, bool is_object) { 497 DCHECK_EQ(cu_->instruction_set, kArm64); 498 // Unused - RegLocation rl_src_unsafe = info->args[0]; 499 RegLocation rl_src_obj = info->args[1]; // Object - known non-null 500 RegLocation rl_src_offset = info->args[2]; // long low 501 RegLocation rl_src_expected = info->args[4]; // int, long or Object 502 // If is_long, high half is in info->args[5] 503 RegLocation rl_src_new_value = info->args[is_long ? 6 : 5]; // int, long or Object 504 // If is_long, high half is in info->args[7] 505 RegLocation rl_dest = InlineTarget(info); // boolean place for result 506 507 // Load Object and offset 508 RegLocation rl_object = LoadValue(rl_src_obj, kRefReg); 509 RegLocation rl_offset = LoadValueWide(rl_src_offset, kCoreReg); 510 511 RegLocation rl_new_value; 512 RegLocation rl_expected; 513 if (is_long) { 514 rl_new_value = LoadValueWide(rl_src_new_value, kCoreReg); 515 rl_expected = LoadValueWide(rl_src_expected, kCoreReg); 516 } else { 517 rl_new_value = LoadValue(rl_src_new_value, is_object ? kRefReg : kCoreReg); 518 rl_expected = LoadValue(rl_src_expected, is_object ? kRefReg : kCoreReg); 519 } 520 521 if (is_object && !mir_graph_->IsConstantNullRef(rl_new_value)) { 522 // Mark card for object assuming new value is stored. 523 MarkGCCard(rl_new_value.reg, rl_object.reg); 524 } 525 526 RegStorage r_ptr = AllocTempRef(); 527 OpRegRegReg(kOpAdd, r_ptr, rl_object.reg, rl_offset.reg); 528 529 // Free now unneeded rl_object and rl_offset to give more temps. 530 ClobberSReg(rl_object.s_reg_low); 531 FreeTemp(rl_object.reg); 532 ClobberSReg(rl_offset.s_reg_low); 533 FreeTemp(rl_offset.reg); 534 535 // do { 536 // tmp = [r_ptr] - expected; 537 // } while (tmp == 0 && failure([r_ptr] <- r_new_value)); 538 // result = tmp != 0; 539 540 RegStorage r_tmp; 541 RegStorage r_tmp_stored; 542 RegStorage rl_new_value_stored = rl_new_value.reg; 543 ArmOpcode wide = UNWIDE(0); 544 if (is_long) { 545 r_tmp_stored = r_tmp = AllocTempWide(); 546 wide = WIDE(0); 547 } else if (is_object) { 548 // References use 64-bit registers, but are stored as compressed 32-bit values. 549 // This means r_tmp_stored != r_tmp. 550 r_tmp = AllocTempRef(); 551 r_tmp_stored = As32BitReg(r_tmp); 552 rl_new_value_stored = As32BitReg(rl_new_value_stored); 553 } else { 554 r_tmp_stored = r_tmp = AllocTemp(); 555 } 556 557 RegStorage r_tmp32 = (r_tmp.Is32Bit()) ? r_tmp : As32BitReg(r_tmp); 558 LIR* loop = NewLIR0(kPseudoTargetLabel); 559 NewLIR2(kA64Ldaxr2rX | wide, r_tmp_stored.GetReg(), r_ptr.GetReg()); 560 OpRegReg(kOpCmp, r_tmp, rl_expected.reg); 561 DCHECK(last_lir_insn_->u.m.def_mask->HasBit(ResourceMask::kCCode)); 562 LIR* early_exit = OpCondBranch(kCondNe, NULL); 563 NewLIR3(kA64Stlxr3wrX | wide, r_tmp32.GetReg(), rl_new_value_stored.GetReg(), r_ptr.GetReg()); 564 NewLIR3(kA64Cmp3RdT, r_tmp32.GetReg(), 0, ENCODE_NO_SHIFT); 565 DCHECK(last_lir_insn_->u.m.def_mask->HasBit(ResourceMask::kCCode)); 566 OpCondBranch(kCondNe, loop); 567 568 LIR* exit_loop = NewLIR0(kPseudoTargetLabel); 569 early_exit->target = exit_loop; 570 571 RegLocation rl_result = EvalLoc(rl_dest, kCoreReg, true); 572 NewLIR4(kA64Csinc4rrrc, rl_result.reg.GetReg(), rwzr, rwzr, kArmCondNe); 573 574 FreeTemp(r_tmp); // Now unneeded. 575 FreeTemp(r_ptr); // Now unneeded. 576 577 StoreValue(rl_dest, rl_result); 578 579 return true; 580} 581 582LIR* Arm64Mir2Lir::OpPcRelLoad(RegStorage reg, LIR* target) { 583 return RawLIR(current_dalvik_offset_, WIDE(kA64Ldr2rp), reg.GetReg(), 0, 0, 0, 0, target); 584} 585 586LIR* Arm64Mir2Lir::OpVldm(RegStorage r_base, int count) { 587 LOG(FATAL) << "Unexpected use of OpVldm for Arm64"; 588 return NULL; 589} 590 591LIR* Arm64Mir2Lir::OpVstm(RegStorage r_base, int count) { 592 LOG(FATAL) << "Unexpected use of OpVstm for Arm64"; 593 return NULL; 594} 595 596void Arm64Mir2Lir::GenMultiplyByTwoBitMultiplier(RegLocation rl_src, 597 RegLocation rl_result, int lit, 598 int first_bit, int second_bit) { 599 OpRegRegRegShift(kOpAdd, rl_result.reg, rl_src.reg, rl_src.reg, EncodeShift(kA64Lsl, second_bit - first_bit)); 600 if (first_bit != 0) { 601 OpRegRegImm(kOpLsl, rl_result.reg, rl_result.reg, first_bit); 602 } 603} 604 605void Arm64Mir2Lir::GenDivZeroCheckWide(RegStorage reg) { 606 LOG(FATAL) << "Unexpected use of GenDivZero for Arm64"; 607} 608 609// Test suspend flag, return target of taken suspend branch 610LIR* Arm64Mir2Lir::OpTestSuspend(LIR* target) { 611 NewLIR3(kA64Subs3rRd, rwSUSPEND, rwSUSPEND, 1); 612 return OpCondBranch((target == NULL) ? kCondEq : kCondNe, target); 613} 614 615// Decrement register and branch on condition 616LIR* Arm64Mir2Lir::OpDecAndBranch(ConditionCode c_code, RegStorage reg, LIR* target) { 617 // Combine sub & test using sub setflags encoding here. We need to make sure a 618 // subtract form that sets carry is used, so generate explicitly. 619 // TODO: might be best to add a new op, kOpSubs, and handle it generically. 620 ArmOpcode opcode = reg.Is64Bit() ? WIDE(kA64Subs3rRd) : UNWIDE(kA64Subs3rRd); 621 NewLIR3(opcode, reg.GetReg(), reg.GetReg(), 1); // For value == 1, this should set flags. 622 DCHECK(last_lir_insn_->u.m.def_mask->HasBit(ResourceMask::kCCode)); 623 return OpCondBranch(c_code, target); 624} 625 626bool Arm64Mir2Lir::GenMemBarrier(MemBarrierKind barrier_kind) { 627#if ANDROID_SMP != 0 628 // Start off with using the last LIR as the barrier. If it is not enough, then we will generate one. 629 LIR* barrier = last_lir_insn_; 630 631 int dmb_flavor; 632 // TODO: revisit Arm barrier kinds 633 switch (barrier_kind) { 634 case kLoadStore: dmb_flavor = kISH; break; 635 case kLoadLoad: dmb_flavor = kISH; break; 636 case kStoreStore: dmb_flavor = kISHST; break; 637 case kStoreLoad: dmb_flavor = kISH; break; 638 default: 639 LOG(FATAL) << "Unexpected MemBarrierKind: " << barrier_kind; 640 dmb_flavor = kSY; // quiet gcc. 641 break; 642 } 643 644 bool ret = false; 645 646 // If the same barrier already exists, don't generate another. 647 if (barrier == nullptr 648 || (barrier->opcode != kA64Dmb1B || barrier->operands[0] != dmb_flavor)) { 649 barrier = NewLIR1(kA64Dmb1B, dmb_flavor); 650 ret = true; 651 } 652 653 // At this point we must have a memory barrier. Mark it as a scheduling barrier as well. 654 DCHECK(!barrier->flags.use_def_invalid); 655 barrier->u.m.def_mask = &kEncodeAll; 656 return ret; 657#else 658 return false; 659#endif 660} 661 662void Arm64Mir2Lir::GenIntToLong(RegLocation rl_dest, RegLocation rl_src) { 663 RegLocation rl_result; 664 665 rl_src = LoadValue(rl_src, kCoreReg); 666 rl_result = EvalLocWide(rl_dest, kCoreReg, true); 667 NewLIR4(WIDE(kA64Sbfm4rrdd), rl_result.reg.GetReg(), As64BitReg(rl_src.reg).GetReg(), 0, 31); 668 StoreValueWide(rl_dest, rl_result); 669} 670 671void Arm64Mir2Lir::GenDivRemLong(Instruction::Code opcode, RegLocation rl_dest, 672 RegLocation rl_src1, RegLocation rl_src2, bool is_div) { 673 RegLocation rl_result; 674 rl_src1 = LoadValueWide(rl_src1, kCoreReg); 675 rl_src2 = LoadValueWide(rl_src2, kCoreReg); 676 GenDivZeroCheck(rl_src2.reg); 677 rl_result = GenDivRem(rl_dest, rl_src1.reg, rl_src2.reg, is_div); 678 StoreValueWide(rl_dest, rl_result); 679} 680 681void Arm64Mir2Lir::GenLongOp(OpKind op, RegLocation rl_dest, RegLocation rl_src1, 682 RegLocation rl_src2) { 683 RegLocation rl_result; 684 685 rl_src1 = LoadValueWide(rl_src1, kCoreReg); 686 rl_src2 = LoadValueWide(rl_src2, kCoreReg); 687 rl_result = EvalLocWide(rl_dest, kCoreReg, true); 688 OpRegRegRegShift(op, rl_result.reg, rl_src1.reg, rl_src2.reg, ENCODE_NO_SHIFT); 689 StoreValueWide(rl_dest, rl_result); 690} 691 692void Arm64Mir2Lir::GenNegLong(RegLocation rl_dest, RegLocation rl_src) { 693 RegLocation rl_result; 694 695 rl_src = LoadValueWide(rl_src, kCoreReg); 696 rl_result = EvalLocWide(rl_dest, kCoreReg, true); 697 OpRegRegShift(kOpNeg, rl_result.reg, rl_src.reg, ENCODE_NO_SHIFT); 698 StoreValueWide(rl_dest, rl_result); 699} 700 701void Arm64Mir2Lir::GenNotLong(RegLocation rl_dest, RegLocation rl_src) { 702 RegLocation rl_result; 703 704 rl_src = LoadValueWide(rl_src, kCoreReg); 705 rl_result = EvalLocWide(rl_dest, kCoreReg, true); 706 OpRegRegShift(kOpMvn, rl_result.reg, rl_src.reg, ENCODE_NO_SHIFT); 707 StoreValueWide(rl_dest, rl_result); 708} 709 710void Arm64Mir2Lir::GenMulLong(Instruction::Code opcode, RegLocation rl_dest, 711 RegLocation rl_src1, RegLocation rl_src2) { 712 GenLongOp(kOpMul, rl_dest, rl_src1, rl_src2); 713} 714 715void Arm64Mir2Lir::GenAddLong(Instruction::Code opcode, RegLocation rl_dest, RegLocation rl_src1, 716 RegLocation rl_src2) { 717 GenLongOp(kOpAdd, rl_dest, rl_src1, rl_src2); 718} 719 720void Arm64Mir2Lir::GenSubLong(Instruction::Code opcode, RegLocation rl_dest, RegLocation rl_src1, 721 RegLocation rl_src2) { 722 GenLongOp(kOpSub, rl_dest, rl_src1, rl_src2); 723} 724 725void Arm64Mir2Lir::GenAndLong(Instruction::Code opcode, RegLocation rl_dest, RegLocation rl_src1, 726 RegLocation rl_src2) { 727 GenLongOp(kOpAnd, rl_dest, rl_src1, rl_src2); 728} 729 730void Arm64Mir2Lir::GenOrLong(Instruction::Code opcode, RegLocation rl_dest, RegLocation rl_src1, 731 RegLocation rl_src2) { 732 GenLongOp(kOpOr, rl_dest, rl_src1, rl_src2); 733} 734 735void Arm64Mir2Lir::GenXorLong(Instruction::Code opcode, RegLocation rl_dest, RegLocation rl_src1, 736 RegLocation rl_src2) { 737 GenLongOp(kOpXor, rl_dest, rl_src1, rl_src2); 738} 739 740/* 741 * Generate array load 742 */ 743void Arm64Mir2Lir::GenArrayGet(int opt_flags, OpSize size, RegLocation rl_array, 744 RegLocation rl_index, RegLocation rl_dest, int scale) { 745 RegisterClass reg_class = RegClassBySize(size); 746 int len_offset = mirror::Array::LengthOffset().Int32Value(); 747 int data_offset; 748 RegLocation rl_result; 749 bool constant_index = rl_index.is_const; 750 rl_array = LoadValue(rl_array, kRefReg); 751 if (!constant_index) { 752 rl_index = LoadValue(rl_index, kCoreReg); 753 } 754 755 if (rl_dest.wide) { 756 data_offset = mirror::Array::DataOffset(sizeof(int64_t)).Int32Value(); 757 } else { 758 data_offset = mirror::Array::DataOffset(sizeof(int32_t)).Int32Value(); 759 } 760 761 // If index is constant, just fold it into the data offset 762 if (constant_index) { 763 data_offset += mir_graph_->ConstantValue(rl_index) << scale; 764 } 765 766 /* null object? */ 767 GenNullCheck(rl_array.reg, opt_flags); 768 769 bool needs_range_check = (!(opt_flags & MIR_IGNORE_RANGE_CHECK)); 770 RegStorage reg_len; 771 if (needs_range_check) { 772 reg_len = AllocTemp(); 773 /* Get len */ 774 Load32Disp(rl_array.reg, len_offset, reg_len); 775 MarkPossibleNullPointerException(opt_flags); 776 } else { 777 ForceImplicitNullCheck(rl_array.reg, opt_flags); 778 } 779 if (rl_dest.wide || rl_dest.fp || constant_index) { 780 RegStorage reg_ptr; 781 if (constant_index) { 782 reg_ptr = rl_array.reg; // NOTE: must not alter reg_ptr in constant case. 783 } else { 784 // No special indexed operation, lea + load w/ displacement 785 reg_ptr = AllocTempRef(); 786 OpRegRegRegShift(kOpAdd, reg_ptr, rl_array.reg, As64BitReg(rl_index.reg), 787 EncodeShift(kA64Lsl, scale)); 788 FreeTemp(rl_index.reg); 789 } 790 rl_result = EvalLoc(rl_dest, reg_class, true); 791 792 if (needs_range_check) { 793 if (constant_index) { 794 GenArrayBoundsCheck(mir_graph_->ConstantValue(rl_index), reg_len); 795 } else { 796 GenArrayBoundsCheck(rl_index.reg, reg_len); 797 } 798 FreeTemp(reg_len); 799 } 800 if (rl_result.ref) { 801 LoadRefDisp(reg_ptr, data_offset, rl_result.reg, kNotVolatile); 802 } else { 803 LoadBaseDisp(reg_ptr, data_offset, rl_result.reg, size, kNotVolatile); 804 } 805 MarkPossibleNullPointerException(opt_flags); 806 if (!constant_index) { 807 FreeTemp(reg_ptr); 808 } 809 if (rl_dest.wide) { 810 StoreValueWide(rl_dest, rl_result); 811 } else { 812 StoreValue(rl_dest, rl_result); 813 } 814 } else { 815 // Offset base, then use indexed load 816 RegStorage reg_ptr = AllocTempRef(); 817 OpRegRegImm(kOpAdd, reg_ptr, rl_array.reg, data_offset); 818 FreeTemp(rl_array.reg); 819 rl_result = EvalLoc(rl_dest, reg_class, true); 820 821 if (needs_range_check) { 822 GenArrayBoundsCheck(rl_index.reg, reg_len); 823 FreeTemp(reg_len); 824 } 825 if (rl_result.ref) { 826 LoadRefIndexed(reg_ptr, As64BitReg(rl_index.reg), rl_result.reg, scale); 827 } else { 828 LoadBaseIndexed(reg_ptr, As64BitReg(rl_index.reg), rl_result.reg, scale, size); 829 } 830 MarkPossibleNullPointerException(opt_flags); 831 FreeTemp(reg_ptr); 832 StoreValue(rl_dest, rl_result); 833 } 834} 835 836/* 837 * Generate array store 838 * 839 */ 840void Arm64Mir2Lir::GenArrayPut(int opt_flags, OpSize size, RegLocation rl_array, 841 RegLocation rl_index, RegLocation rl_src, int scale, bool card_mark) { 842 RegisterClass reg_class = RegClassBySize(size); 843 int len_offset = mirror::Array::LengthOffset().Int32Value(); 844 bool constant_index = rl_index.is_const; 845 846 int data_offset; 847 if (size == k64 || size == kDouble) { 848 data_offset = mirror::Array::DataOffset(sizeof(int64_t)).Int32Value(); 849 } else { 850 data_offset = mirror::Array::DataOffset(sizeof(int32_t)).Int32Value(); 851 } 852 853 // If index is constant, just fold it into the data offset. 854 if (constant_index) { 855 data_offset += mir_graph_->ConstantValue(rl_index) << scale; 856 } 857 858 rl_array = LoadValue(rl_array, kRefReg); 859 if (!constant_index) { 860 rl_index = LoadValue(rl_index, kCoreReg); 861 } 862 863 RegStorage reg_ptr; 864 bool allocated_reg_ptr_temp = false; 865 if (constant_index) { 866 reg_ptr = rl_array.reg; 867 } else if (IsTemp(rl_array.reg) && !card_mark) { 868 Clobber(rl_array.reg); 869 reg_ptr = rl_array.reg; 870 } else { 871 allocated_reg_ptr_temp = true; 872 reg_ptr = AllocTempRef(); 873 } 874 875 /* null object? */ 876 GenNullCheck(rl_array.reg, opt_flags); 877 878 bool needs_range_check = (!(opt_flags & MIR_IGNORE_RANGE_CHECK)); 879 RegStorage reg_len; 880 if (needs_range_check) { 881 reg_len = AllocTemp(); 882 // NOTE: max live temps(4) here. 883 /* Get len */ 884 Load32Disp(rl_array.reg, len_offset, reg_len); 885 MarkPossibleNullPointerException(opt_flags); 886 } else { 887 ForceImplicitNullCheck(rl_array.reg, opt_flags); 888 } 889 /* at this point, reg_ptr points to array, 2 live temps */ 890 if (rl_src.wide || rl_src.fp || constant_index) { 891 if (rl_src.wide) { 892 rl_src = LoadValueWide(rl_src, reg_class); 893 } else { 894 rl_src = LoadValue(rl_src, reg_class); 895 } 896 if (!constant_index) { 897 OpRegRegRegShift(kOpAdd, reg_ptr, rl_array.reg, As64BitReg(rl_index.reg), 898 EncodeShift(kA64Lsl, scale)); 899 } 900 if (needs_range_check) { 901 if (constant_index) { 902 GenArrayBoundsCheck(mir_graph_->ConstantValue(rl_index), reg_len); 903 } else { 904 GenArrayBoundsCheck(rl_index.reg, reg_len); 905 } 906 FreeTemp(reg_len); 907 } 908 if (rl_src.ref) { 909 StoreRefDisp(reg_ptr, data_offset, rl_src.reg, kNotVolatile); 910 } else { 911 StoreBaseDisp(reg_ptr, data_offset, rl_src.reg, size, kNotVolatile); 912 } 913 MarkPossibleNullPointerException(opt_flags); 914 } else { 915 /* reg_ptr -> array data */ 916 OpRegRegImm(kOpAdd, reg_ptr, rl_array.reg, data_offset); 917 rl_src = LoadValue(rl_src, reg_class); 918 if (needs_range_check) { 919 GenArrayBoundsCheck(rl_index.reg, reg_len); 920 FreeTemp(reg_len); 921 } 922 if (rl_src.ref) { 923 StoreRefIndexed(reg_ptr, As64BitReg(rl_index.reg), rl_src.reg, scale); 924 } else { 925 StoreBaseIndexed(reg_ptr, As64BitReg(rl_index.reg), rl_src.reg, scale, size); 926 } 927 MarkPossibleNullPointerException(opt_flags); 928 } 929 if (allocated_reg_ptr_temp) { 930 FreeTemp(reg_ptr); 931 } 932 if (card_mark) { 933 MarkGCCard(rl_src.reg, rl_array.reg); 934 } 935} 936 937void Arm64Mir2Lir::GenShiftImmOpLong(Instruction::Code opcode, 938 RegLocation rl_dest, RegLocation rl_src, RegLocation rl_shift) { 939 OpKind op = kOpBkpt; 940 // Per spec, we only care about low 6 bits of shift amount. 941 int shift_amount = mir_graph_->ConstantValue(rl_shift) & 0x3f; 942 rl_src = LoadValueWide(rl_src, kCoreReg); 943 if (shift_amount == 0) { 944 StoreValueWide(rl_dest, rl_src); 945 return; 946 } 947 948 RegLocation rl_result = EvalLocWide(rl_dest, kCoreReg, true); 949 switch (opcode) { 950 case Instruction::SHL_LONG: 951 case Instruction::SHL_LONG_2ADDR: 952 op = kOpLsl; 953 break; 954 case Instruction::SHR_LONG: 955 case Instruction::SHR_LONG_2ADDR: 956 op = kOpAsr; 957 break; 958 case Instruction::USHR_LONG: 959 case Instruction::USHR_LONG_2ADDR: 960 op = kOpLsr; 961 break; 962 default: 963 LOG(FATAL) << "Unexpected case"; 964 } 965 OpRegRegImm(op, rl_result.reg, rl_src.reg, shift_amount); 966 StoreValueWide(rl_dest, rl_result); 967} 968 969void Arm64Mir2Lir::GenArithImmOpLong(Instruction::Code opcode, RegLocation rl_dest, 970 RegLocation rl_src1, RegLocation rl_src2) { 971 if ((opcode == Instruction::SUB_LONG) || (opcode == Instruction::SUB_LONG_2ADDR)) { 972 if (!rl_src2.is_const) { 973 return GenArithOpLong(opcode, rl_dest, rl_src1, rl_src2); 974 } 975 } else { 976 // Associativity. 977 if (!rl_src2.is_const) { 978 DCHECK(rl_src1.is_const); 979 std::swap(rl_src1, rl_src2); 980 } 981 } 982 DCHECK(rl_src2.is_const); 983 984 OpKind op = kOpBkpt; 985 int64_t val = mir_graph_->ConstantValueWide(rl_src2); 986 987 switch (opcode) { 988 case Instruction::ADD_LONG: 989 case Instruction::ADD_LONG_2ADDR: 990 op = kOpAdd; 991 break; 992 case Instruction::SUB_LONG: 993 case Instruction::SUB_LONG_2ADDR: 994 op = kOpSub; 995 break; 996 case Instruction::AND_LONG: 997 case Instruction::AND_LONG_2ADDR: 998 op = kOpAnd; 999 break; 1000 case Instruction::OR_LONG: 1001 case Instruction::OR_LONG_2ADDR: 1002 op = kOpOr; 1003 break; 1004 case Instruction::XOR_LONG: 1005 case Instruction::XOR_LONG_2ADDR: 1006 op = kOpXor; 1007 break; 1008 default: 1009 LOG(FATAL) << "Unexpected opcode"; 1010 } 1011 1012 rl_src1 = LoadValueWide(rl_src1, kCoreReg); 1013 RegLocation rl_result = EvalLocWide(rl_dest, kCoreReg, true); 1014 OpRegRegImm64(op, rl_result.reg, rl_src1.reg, val); 1015 StoreValueWide(rl_dest, rl_result); 1016} 1017 1018/** 1019 * @brief Split a register list in pairs or registers. 1020 * 1021 * Given a list of registers in @p reg_mask, split the list in pairs. Use as follows: 1022 * @code 1023 * int reg1 = -1, reg2 = -1; 1024 * while (reg_mask) { 1025 * reg_mask = GenPairWise(reg_mask, & reg1, & reg2); 1026 * if (UNLIKELY(reg2 < 0)) { 1027 * // Single register in reg1. 1028 * } else { 1029 * // Pair in reg1, reg2. 1030 * } 1031 * } 1032 * @endcode 1033 */ 1034uint32_t Arm64Mir2Lir::GenPairWise(uint32_t reg_mask, int* reg1, int* reg2) { 1035 // Find first register. 1036 int first_bit_set = __builtin_ctz(reg_mask) + 1; 1037 int reg = *reg1 + first_bit_set; 1038 reg_mask >>= first_bit_set; 1039 1040 if (LIKELY(reg_mask)) { 1041 // Save the first register, find the second and use the pair opcode. 1042 int second_bit_set = __builtin_ctz(reg_mask) + 1; 1043 *reg2 = reg; 1044 reg_mask >>= second_bit_set; 1045 *reg1 = reg + second_bit_set; 1046 return reg_mask; 1047 } 1048 1049 // Use the single opcode, as we just have one register. 1050 *reg1 = reg; 1051 *reg2 = -1; 1052 return reg_mask; 1053} 1054 1055void Arm64Mir2Lir::UnSpillCoreRegs(RegStorage base, int offset, uint32_t reg_mask) { 1056 int reg1 = -1, reg2 = -1; 1057 const int reg_log2_size = 3; 1058 1059 for (offset = (offset >> reg_log2_size); reg_mask; offset += 2) { 1060 reg_mask = GenPairWise(reg_mask, & reg1, & reg2); 1061 if (UNLIKELY(reg2 < 0)) { 1062 NewLIR3(WIDE(kA64Ldr3rXD), RegStorage::Solo64(reg1).GetReg(), base.GetReg(), offset); 1063 } else { 1064 DCHECK_LE(offset, 63); 1065 NewLIR4(WIDE(kA64Ldp4rrXD), RegStorage::Solo64(reg2).GetReg(), 1066 RegStorage::Solo64(reg1).GetReg(), base.GetReg(), offset); 1067 } 1068 } 1069} 1070 1071void Arm64Mir2Lir::SpillCoreRegs(RegStorage base, int offset, uint32_t reg_mask) { 1072 int reg1 = -1, reg2 = -1; 1073 const int reg_log2_size = 3; 1074 1075 for (offset = (offset >> reg_log2_size); reg_mask; offset += 2) { 1076 reg_mask = GenPairWise(reg_mask, & reg1, & reg2); 1077 if (UNLIKELY(reg2 < 0)) { 1078 NewLIR3(WIDE(kA64Str3rXD), RegStorage::Solo64(reg1).GetReg(), base.GetReg(), offset); 1079 } else { 1080 NewLIR4(WIDE(kA64Stp4rrXD), RegStorage::Solo64(reg2).GetReg(), 1081 RegStorage::Solo64(reg1).GetReg(), base.GetReg(), offset); 1082 } 1083 } 1084} 1085 1086void Arm64Mir2Lir::UnSpillFPRegs(RegStorage base, int offset, uint32_t reg_mask) { 1087 int reg1 = -1, reg2 = -1; 1088 const int reg_log2_size = 3; 1089 1090 for (offset = (offset >> reg_log2_size); reg_mask; offset += 2) { 1091 reg_mask = GenPairWise(reg_mask, & reg1, & reg2); 1092 if (UNLIKELY(reg2 < 0)) { 1093 NewLIR3(FWIDE(kA64Ldr3fXD), RegStorage::FloatSolo64(reg1).GetReg(), base.GetReg(), offset); 1094 } else { 1095 NewLIR4(WIDE(kA64Ldp4ffXD), RegStorage::FloatSolo64(reg2).GetReg(), 1096 RegStorage::FloatSolo64(reg1).GetReg(), base.GetReg(), offset); 1097 } 1098 } 1099} 1100 1101// TODO(Arm64): consider using ld1 and st1? 1102void Arm64Mir2Lir::SpillFPRegs(RegStorage base, int offset, uint32_t reg_mask) { 1103 int reg1 = -1, reg2 = -1; 1104 const int reg_log2_size = 3; 1105 1106 for (offset = (offset >> reg_log2_size); reg_mask; offset += 2) { 1107 reg_mask = GenPairWise(reg_mask, & reg1, & reg2); 1108 if (UNLIKELY(reg2 < 0)) { 1109 NewLIR3(FWIDE(kA64Str3fXD), RegStorage::FloatSolo64(reg1).GetReg(), base.GetReg(), offset); 1110 } else { 1111 NewLIR4(WIDE(kA64Stp4ffXD), RegStorage::FloatSolo64(reg2).GetReg(), 1112 RegStorage::FloatSolo64(reg1).GetReg(), base.GetReg(), offset); 1113 } 1114 } 1115} 1116 1117bool Arm64Mir2Lir::GenInlinedReverseBits(CallInfo* info, OpSize size) { 1118 ArmOpcode wide = (size == k64) ? WIDE(0) : UNWIDE(0); 1119 RegLocation rl_src_i = info->args[0]; 1120 RegLocation rl_dest = (size == k64) ? InlineTargetWide(info) : InlineTarget(info); // result reg 1121 RegLocation rl_result = EvalLoc(rl_dest, kCoreReg, true); 1122 RegLocation rl_i = (size == k64) ? LoadValueWide(rl_src_i, kCoreReg) : LoadValue(rl_src_i, kCoreReg); 1123 NewLIR2(kA64Rbit2rr | wide, rl_result.reg.GetReg(), rl_i.reg.GetReg()); 1124 (size == k64) ? StoreValueWide(rl_dest, rl_result) : StoreValue(rl_dest, rl_result); 1125 return true; 1126} 1127 1128} // namespace art 1129