code_generator_arm.cc revision 9fd8c60cdff7b28a89bb97fd90ae9d0f37cf8f8b
1/* 2 * Copyright (C) 2014 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17#include "code_generator_arm.h" 18 19#include "arch/arm/instruction_set_features_arm.h" 20#include "art_method.h" 21#include "code_generator_utils.h" 22#include "compiled_method.h" 23#include "entrypoints/quick/quick_entrypoints.h" 24#include "gc/accounting/card_table.h" 25#include "intrinsics.h" 26#include "intrinsics_arm.h" 27#include "mirror/array-inl.h" 28#include "mirror/class-inl.h" 29#include "thread.h" 30#include "utils/arm/assembler_arm.h" 31#include "utils/arm/managed_register_arm.h" 32#include "utils/assembler.h" 33#include "utils/stack_checks.h" 34 35namespace art { 36 37template<class MirrorType> 38class GcRoot; 39 40namespace arm { 41 42static bool ExpectedPairLayout(Location location) { 43 // We expected this for both core and fpu register pairs. 44 return ((location.low() & 1) == 0) && (location.low() + 1 == location.high()); 45} 46 47static constexpr int kCurrentMethodStackOffset = 0; 48static constexpr Register kMethodRegisterArgument = R0; 49 50static constexpr Register kCoreAlwaysSpillRegister = R5; 51static constexpr Register kCoreCalleeSaves[] = 52 { R5, R6, R7, R8, R10, R11, LR }; 53static constexpr SRegister kFpuCalleeSaves[] = 54 { S16, S17, S18, S19, S20, S21, S22, S23, S24, S25, S26, S27, S28, S29, S30, S31 }; 55 56// D31 cannot be split into two S registers, and the register allocator only works on 57// S registers. Therefore there is no need to block it. 58static constexpr DRegister DTMP = D31; 59 60static constexpr uint32_t kPackedSwitchCompareJumpThreshold = 7; 61 62// NOLINT on __ macro to suppress wrong warning/fix (misc-macro-parentheses) from clang-tidy. 63#define __ down_cast<ArmAssembler*>(codegen->GetAssembler())-> // NOLINT 64#define QUICK_ENTRY_POINT(x) QUICK_ENTRYPOINT_OFFSET(kArmPointerSize, x).Int32Value() 65 66static constexpr int kRegListThreshold = 4; 67 68// SaveLiveRegisters and RestoreLiveRegisters from SlowPathCodeARM operate on sets of S registers, 69// for each live D registers they treat two corresponding S registers as live ones. 70// 71// Two following functions (SaveContiguousSRegisterList, RestoreContiguousSRegisterList) build 72// from a list of contiguous S registers a list of contiguous D registers (processing first/last 73// S registers corner cases) and save/restore this new list treating them as D registers. 74// - decreasing code size 75// - avoiding hazards on Cortex-A57, when a pair of S registers for an actual live D register is 76// restored and then used in regular non SlowPath code as D register. 77// 78// For the following example (v means the S register is live): 79// D names: | D0 | D1 | D2 | D4 | ... 80// S names: | S0 | S1 | S2 | S3 | S4 | S5 | S6 | S7 | ... 81// Live? | | v | v | v | v | v | v | | ... 82// 83// S1 and S6 will be saved/restored independently; D registers list (D1, D2) will be processed 84// as D registers. 85static size_t SaveContiguousSRegisterList(size_t first, 86 size_t last, 87 CodeGenerator* codegen, 88 size_t stack_offset) { 89 DCHECK_LE(first, last); 90 if ((first == last) && (first == 0)) { 91 stack_offset += codegen->SaveFloatingPointRegister(stack_offset, first); 92 return stack_offset; 93 } 94 if (first % 2 == 1) { 95 stack_offset += codegen->SaveFloatingPointRegister(stack_offset, first++); 96 } 97 98 bool save_last = false; 99 if (last % 2 == 0) { 100 save_last = true; 101 --last; 102 } 103 104 if (first < last) { 105 DRegister d_reg = static_cast<DRegister>(first / 2); 106 DCHECK_EQ((last - first + 1) % 2, 0u); 107 size_t number_of_d_regs = (last - first + 1) / 2; 108 109 if (number_of_d_regs == 1) { 110 __ StoreDToOffset(d_reg, SP, stack_offset); 111 } else if (number_of_d_regs > 1) { 112 __ add(IP, SP, ShifterOperand(stack_offset)); 113 __ vstmiad(IP, d_reg, number_of_d_regs); 114 } 115 stack_offset += number_of_d_regs * kArmWordSize * 2; 116 } 117 118 if (save_last) { 119 stack_offset += codegen->SaveFloatingPointRegister(stack_offset, last + 1); 120 } 121 122 return stack_offset; 123} 124 125static size_t RestoreContiguousSRegisterList(size_t first, 126 size_t last, 127 CodeGenerator* codegen, 128 size_t stack_offset) { 129 DCHECK_LE(first, last); 130 if ((first == last) && (first == 0)) { 131 stack_offset += codegen->RestoreFloatingPointRegister(stack_offset, first); 132 return stack_offset; 133 } 134 if (first % 2 == 1) { 135 stack_offset += codegen->RestoreFloatingPointRegister(stack_offset, first++); 136 } 137 138 bool restore_last = false; 139 if (last % 2 == 0) { 140 restore_last = true; 141 --last; 142 } 143 144 if (first < last) { 145 DRegister d_reg = static_cast<DRegister>(first / 2); 146 DCHECK_EQ((last - first + 1) % 2, 0u); 147 size_t number_of_d_regs = (last - first + 1) / 2; 148 if (number_of_d_regs == 1) { 149 __ LoadDFromOffset(d_reg, SP, stack_offset); 150 } else if (number_of_d_regs > 1) { 151 __ add(IP, SP, ShifterOperand(stack_offset)); 152 __ vldmiad(IP, d_reg, number_of_d_regs); 153 } 154 stack_offset += number_of_d_regs * kArmWordSize * 2; 155 } 156 157 if (restore_last) { 158 stack_offset += codegen->RestoreFloatingPointRegister(stack_offset, last + 1); 159 } 160 161 return stack_offset; 162} 163 164void SlowPathCodeARM::SaveLiveRegisters(CodeGenerator* codegen, LocationSummary* locations) { 165 size_t stack_offset = codegen->GetFirstRegisterSlotInSlowPath(); 166 size_t orig_offset = stack_offset; 167 168 const uint32_t core_spills = codegen->GetSlowPathSpills(locations, /* core_registers */ true); 169 for (uint32_t i : LowToHighBits(core_spills)) { 170 // If the register holds an object, update the stack mask. 171 if (locations->RegisterContainsObject(i)) { 172 locations->SetStackBit(stack_offset / kVRegSize); 173 } 174 DCHECK_LT(stack_offset, codegen->GetFrameSize() - codegen->FrameEntrySpillSize()); 175 DCHECK_LT(i, kMaximumNumberOfExpectedRegisters); 176 saved_core_stack_offsets_[i] = stack_offset; 177 stack_offset += kArmWordSize; 178 } 179 180 int reg_num = POPCOUNT(core_spills); 181 if (reg_num != 0) { 182 if (reg_num > kRegListThreshold) { 183 __ StoreList(RegList(core_spills), orig_offset); 184 } else { 185 stack_offset = orig_offset; 186 for (uint32_t i : LowToHighBits(core_spills)) { 187 stack_offset += codegen->SaveCoreRegister(stack_offset, i); 188 } 189 } 190 } 191 192 uint32_t fp_spills = codegen->GetSlowPathSpills(locations, /* core_registers */ false); 193 orig_offset = stack_offset; 194 for (uint32_t i : LowToHighBits(fp_spills)) { 195 DCHECK_LT(i, kMaximumNumberOfExpectedRegisters); 196 saved_fpu_stack_offsets_[i] = stack_offset; 197 stack_offset += kArmWordSize; 198 } 199 200 stack_offset = orig_offset; 201 while (fp_spills != 0u) { 202 uint32_t begin = CTZ(fp_spills); 203 uint32_t tmp = fp_spills + (1u << begin); 204 fp_spills &= tmp; // Clear the contiguous range of 1s. 205 uint32_t end = (tmp == 0u) ? 32u : CTZ(tmp); // CTZ(0) is undefined. 206 stack_offset = SaveContiguousSRegisterList(begin, end - 1, codegen, stack_offset); 207 } 208 DCHECK_LE(stack_offset, codegen->GetFrameSize() - codegen->FrameEntrySpillSize()); 209} 210 211void SlowPathCodeARM::RestoreLiveRegisters(CodeGenerator* codegen, LocationSummary* locations) { 212 size_t stack_offset = codegen->GetFirstRegisterSlotInSlowPath(); 213 size_t orig_offset = stack_offset; 214 215 const uint32_t core_spills = codegen->GetSlowPathSpills(locations, /* core_registers */ true); 216 for (uint32_t i : LowToHighBits(core_spills)) { 217 DCHECK_LT(stack_offset, codegen->GetFrameSize() - codegen->FrameEntrySpillSize()); 218 DCHECK_LT(i, kMaximumNumberOfExpectedRegisters); 219 stack_offset += kArmWordSize; 220 } 221 222 int reg_num = POPCOUNT(core_spills); 223 if (reg_num != 0) { 224 if (reg_num > kRegListThreshold) { 225 __ LoadList(RegList(core_spills), orig_offset); 226 } else { 227 stack_offset = orig_offset; 228 for (uint32_t i : LowToHighBits(core_spills)) { 229 stack_offset += codegen->RestoreCoreRegister(stack_offset, i); 230 } 231 } 232 } 233 234 uint32_t fp_spills = codegen->GetSlowPathSpills(locations, /* core_registers */ false); 235 while (fp_spills != 0u) { 236 uint32_t begin = CTZ(fp_spills); 237 uint32_t tmp = fp_spills + (1u << begin); 238 fp_spills &= tmp; // Clear the contiguous range of 1s. 239 uint32_t end = (tmp == 0u) ? 32u : CTZ(tmp); // CTZ(0) is undefined. 240 stack_offset = RestoreContiguousSRegisterList(begin, end - 1, codegen, stack_offset); 241 } 242 DCHECK_LE(stack_offset, codegen->GetFrameSize() - codegen->FrameEntrySpillSize()); 243} 244 245class NullCheckSlowPathARM : public SlowPathCodeARM { 246 public: 247 explicit NullCheckSlowPathARM(HNullCheck* instruction) : SlowPathCodeARM(instruction) {} 248 249 void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { 250 CodeGeneratorARM* arm_codegen = down_cast<CodeGeneratorARM*>(codegen); 251 __ Bind(GetEntryLabel()); 252 if (instruction_->CanThrowIntoCatchBlock()) { 253 // Live registers will be restored in the catch block if caught. 254 SaveLiveRegisters(codegen, instruction_->GetLocations()); 255 } 256 arm_codegen->InvokeRuntime(kQuickThrowNullPointer, 257 instruction_, 258 instruction_->GetDexPc(), 259 this); 260 CheckEntrypointTypes<kQuickThrowNullPointer, void, void>(); 261 } 262 263 bool IsFatal() const OVERRIDE { return true; } 264 265 const char* GetDescription() const OVERRIDE { return "NullCheckSlowPathARM"; } 266 267 private: 268 DISALLOW_COPY_AND_ASSIGN(NullCheckSlowPathARM); 269}; 270 271class DivZeroCheckSlowPathARM : public SlowPathCodeARM { 272 public: 273 explicit DivZeroCheckSlowPathARM(HDivZeroCheck* instruction) : SlowPathCodeARM(instruction) {} 274 275 void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { 276 CodeGeneratorARM* arm_codegen = down_cast<CodeGeneratorARM*>(codegen); 277 __ Bind(GetEntryLabel()); 278 arm_codegen->InvokeRuntime(kQuickThrowDivZero, instruction_, instruction_->GetDexPc(), this); 279 CheckEntrypointTypes<kQuickThrowDivZero, void, void>(); 280 } 281 282 bool IsFatal() const OVERRIDE { return true; } 283 284 const char* GetDescription() const OVERRIDE { return "DivZeroCheckSlowPathARM"; } 285 286 private: 287 DISALLOW_COPY_AND_ASSIGN(DivZeroCheckSlowPathARM); 288}; 289 290class SuspendCheckSlowPathARM : public SlowPathCodeARM { 291 public: 292 SuspendCheckSlowPathARM(HSuspendCheck* instruction, HBasicBlock* successor) 293 : SlowPathCodeARM(instruction), successor_(successor) {} 294 295 void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { 296 CodeGeneratorARM* arm_codegen = down_cast<CodeGeneratorARM*>(codegen); 297 __ Bind(GetEntryLabel()); 298 arm_codegen->InvokeRuntime(kQuickTestSuspend, instruction_, instruction_->GetDexPc(), this); 299 CheckEntrypointTypes<kQuickTestSuspend, void, void>(); 300 if (successor_ == nullptr) { 301 __ b(GetReturnLabel()); 302 } else { 303 __ b(arm_codegen->GetLabelOf(successor_)); 304 } 305 } 306 307 Label* GetReturnLabel() { 308 DCHECK(successor_ == nullptr); 309 return &return_label_; 310 } 311 312 HBasicBlock* GetSuccessor() const { 313 return successor_; 314 } 315 316 const char* GetDescription() const OVERRIDE { return "SuspendCheckSlowPathARM"; } 317 318 private: 319 // If not null, the block to branch to after the suspend check. 320 HBasicBlock* const successor_; 321 322 // If `successor_` is null, the label to branch to after the suspend check. 323 Label return_label_; 324 325 DISALLOW_COPY_AND_ASSIGN(SuspendCheckSlowPathARM); 326}; 327 328class BoundsCheckSlowPathARM : public SlowPathCodeARM { 329 public: 330 explicit BoundsCheckSlowPathARM(HBoundsCheck* instruction) 331 : SlowPathCodeARM(instruction) {} 332 333 void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { 334 CodeGeneratorARM* arm_codegen = down_cast<CodeGeneratorARM*>(codegen); 335 LocationSummary* locations = instruction_->GetLocations(); 336 337 __ Bind(GetEntryLabel()); 338 if (instruction_->CanThrowIntoCatchBlock()) { 339 // Live registers will be restored in the catch block if caught. 340 SaveLiveRegisters(codegen, instruction_->GetLocations()); 341 } 342 // We're moving two locations to locations that could overlap, so we need a parallel 343 // move resolver. 344 InvokeRuntimeCallingConvention calling_convention; 345 codegen->EmitParallelMoves( 346 locations->InAt(0), 347 Location::RegisterLocation(calling_convention.GetRegisterAt(0)), 348 Primitive::kPrimInt, 349 locations->InAt(1), 350 Location::RegisterLocation(calling_convention.GetRegisterAt(1)), 351 Primitive::kPrimInt); 352 QuickEntrypointEnum entrypoint = instruction_->AsBoundsCheck()->IsStringCharAt() 353 ? kQuickThrowStringBounds 354 : kQuickThrowArrayBounds; 355 arm_codegen->InvokeRuntime(entrypoint, instruction_, instruction_->GetDexPc(), this); 356 CheckEntrypointTypes<kQuickThrowStringBounds, void, int32_t, int32_t>(); 357 CheckEntrypointTypes<kQuickThrowArrayBounds, void, int32_t, int32_t>(); 358 } 359 360 bool IsFatal() const OVERRIDE { return true; } 361 362 const char* GetDescription() const OVERRIDE { return "BoundsCheckSlowPathARM"; } 363 364 private: 365 DISALLOW_COPY_AND_ASSIGN(BoundsCheckSlowPathARM); 366}; 367 368class LoadClassSlowPathARM : public SlowPathCodeARM { 369 public: 370 LoadClassSlowPathARM(HLoadClass* cls, 371 HInstruction* at, 372 uint32_t dex_pc, 373 bool do_clinit) 374 : SlowPathCodeARM(at), cls_(cls), at_(at), dex_pc_(dex_pc), do_clinit_(do_clinit) { 375 DCHECK(at->IsLoadClass() || at->IsClinitCheck()); 376 } 377 378 void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { 379 LocationSummary* locations = at_->GetLocations(); 380 381 CodeGeneratorARM* arm_codegen = down_cast<CodeGeneratorARM*>(codegen); 382 __ Bind(GetEntryLabel()); 383 SaveLiveRegisters(codegen, locations); 384 385 InvokeRuntimeCallingConvention calling_convention; 386 __ LoadImmediate(calling_convention.GetRegisterAt(0), cls_->GetTypeIndex()); 387 QuickEntrypointEnum entrypoint = do_clinit_ ? kQuickInitializeStaticStorage 388 : kQuickInitializeType; 389 arm_codegen->InvokeRuntime(entrypoint, at_, dex_pc_, this); 390 if (do_clinit_) { 391 CheckEntrypointTypes<kQuickInitializeStaticStorage, void*, uint32_t>(); 392 } else { 393 CheckEntrypointTypes<kQuickInitializeType, void*, uint32_t>(); 394 } 395 396 // Move the class to the desired location. 397 Location out = locations->Out(); 398 if (out.IsValid()) { 399 DCHECK(out.IsRegister() && !locations->GetLiveRegisters()->ContainsCoreRegister(out.reg())); 400 arm_codegen->Move32(locations->Out(), Location::RegisterLocation(R0)); 401 } 402 RestoreLiveRegisters(codegen, locations); 403 __ b(GetExitLabel()); 404 } 405 406 const char* GetDescription() const OVERRIDE { return "LoadClassSlowPathARM"; } 407 408 private: 409 // The class this slow path will load. 410 HLoadClass* const cls_; 411 412 // The instruction where this slow path is happening. 413 // (Might be the load class or an initialization check). 414 HInstruction* const at_; 415 416 // The dex PC of `at_`. 417 const uint32_t dex_pc_; 418 419 // Whether to initialize the class. 420 const bool do_clinit_; 421 422 DISALLOW_COPY_AND_ASSIGN(LoadClassSlowPathARM); 423}; 424 425class LoadStringSlowPathARM : public SlowPathCodeARM { 426 public: 427 explicit LoadStringSlowPathARM(HLoadString* instruction) : SlowPathCodeARM(instruction) {} 428 429 void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { 430 LocationSummary* locations = instruction_->GetLocations(); 431 DCHECK(!locations->GetLiveRegisters()->ContainsCoreRegister(locations->Out().reg())); 432 HLoadString* load = instruction_->AsLoadString(); 433 const uint32_t string_index = load->GetStringIndex(); 434 Register out = locations->Out().AsRegister<Register>(); 435 Register temp = locations->GetTemp(0).AsRegister<Register>(); 436 constexpr bool call_saves_everything_except_r0 = (!kUseReadBarrier || kUseBakerReadBarrier); 437 438 CodeGeneratorARM* arm_codegen = down_cast<CodeGeneratorARM*>(codegen); 439 __ Bind(GetEntryLabel()); 440 SaveLiveRegisters(codegen, locations); 441 442 InvokeRuntimeCallingConvention calling_convention; 443 // In the unlucky case that the `temp` is R0, we preserve the address in `out` across 444 // the kSaveEverything call (or use `out` for the address after non-kSaveEverything call). 445 bool temp_is_r0 = (temp == calling_convention.GetRegisterAt(0)); 446 Register entry_address = temp_is_r0 ? out : temp; 447 DCHECK_NE(entry_address, calling_convention.GetRegisterAt(0)); 448 if (call_saves_everything_except_r0 && temp_is_r0) { 449 __ mov(entry_address, ShifterOperand(temp)); 450 } 451 452 __ LoadImmediate(calling_convention.GetRegisterAt(0), string_index); 453 arm_codegen->InvokeRuntime(kQuickResolveString, instruction_, instruction_->GetDexPc(), this); 454 CheckEntrypointTypes<kQuickResolveString, void*, uint32_t>(); 455 456 // Store the resolved String to the .bss entry. 457 if (call_saves_everything_except_r0) { 458 // The string entry address was preserved in `entry_address` thanks to kSaveEverything. 459 __ str(R0, Address(entry_address)); 460 } else { 461 // For non-Baker read barrier, we need to re-calculate the address of the string entry. 462 CodeGeneratorARM::PcRelativePatchInfo* labels = 463 arm_codegen->NewPcRelativeStringPatch(load->GetDexFile(), string_index); 464 __ BindTrackedLabel(&labels->movw_label); 465 __ movw(entry_address, /* placeholder */ 0u); 466 __ BindTrackedLabel(&labels->movt_label); 467 __ movt(entry_address, /* placeholder */ 0u); 468 __ BindTrackedLabel(&labels->add_pc_label); 469 __ add(entry_address, entry_address, ShifterOperand(PC)); 470 __ str(R0, Address(entry_address)); 471 } 472 473 arm_codegen->Move32(locations->Out(), Location::RegisterLocation(R0)); 474 RestoreLiveRegisters(codegen, locations); 475 476 __ b(GetExitLabel()); 477 } 478 479 const char* GetDescription() const OVERRIDE { return "LoadStringSlowPathARM"; } 480 481 private: 482 DISALLOW_COPY_AND_ASSIGN(LoadStringSlowPathARM); 483}; 484 485class TypeCheckSlowPathARM : public SlowPathCodeARM { 486 public: 487 TypeCheckSlowPathARM(HInstruction* instruction, bool is_fatal) 488 : SlowPathCodeARM(instruction), is_fatal_(is_fatal) {} 489 490 void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { 491 LocationSummary* locations = instruction_->GetLocations(); 492 DCHECK(instruction_->IsCheckCast() 493 || !locations->GetLiveRegisters()->ContainsCoreRegister(locations->Out().reg())); 494 495 CodeGeneratorARM* arm_codegen = down_cast<CodeGeneratorARM*>(codegen); 496 __ Bind(GetEntryLabel()); 497 498 if (!is_fatal_) { 499 SaveLiveRegisters(codegen, locations); 500 } 501 502 // We're moving two locations to locations that could overlap, so we need a parallel 503 // move resolver. 504 InvokeRuntimeCallingConvention calling_convention; 505 codegen->EmitParallelMoves(locations->InAt(0), 506 Location::RegisterLocation(calling_convention.GetRegisterAt(0)), 507 Primitive::kPrimNot, 508 locations->InAt(1), 509 Location::RegisterLocation(calling_convention.GetRegisterAt(1)), 510 Primitive::kPrimNot); 511 if (instruction_->IsInstanceOf()) { 512 arm_codegen->InvokeRuntime(kQuickInstanceofNonTrivial, 513 instruction_, 514 instruction_->GetDexPc(), 515 this); 516 CheckEntrypointTypes<kQuickInstanceofNonTrivial, size_t, mirror::Object*, mirror::Class*>(); 517 arm_codegen->Move32(locations->Out(), Location::RegisterLocation(R0)); 518 } else { 519 DCHECK(instruction_->IsCheckCast()); 520 arm_codegen->InvokeRuntime(kQuickCheckInstanceOf, 521 instruction_, 522 instruction_->GetDexPc(), 523 this); 524 CheckEntrypointTypes<kQuickCheckInstanceOf, void, mirror::Object*, mirror::Class*>(); 525 } 526 527 if (!is_fatal_) { 528 RestoreLiveRegisters(codegen, locations); 529 __ b(GetExitLabel()); 530 } 531 } 532 533 const char* GetDescription() const OVERRIDE { return "TypeCheckSlowPathARM"; } 534 535 bool IsFatal() const OVERRIDE { return is_fatal_; } 536 537 private: 538 const bool is_fatal_; 539 540 DISALLOW_COPY_AND_ASSIGN(TypeCheckSlowPathARM); 541}; 542 543class DeoptimizationSlowPathARM : public SlowPathCodeARM { 544 public: 545 explicit DeoptimizationSlowPathARM(HDeoptimize* instruction) 546 : SlowPathCodeARM(instruction) {} 547 548 void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { 549 CodeGeneratorARM* arm_codegen = down_cast<CodeGeneratorARM*>(codegen); 550 __ Bind(GetEntryLabel()); 551 arm_codegen->InvokeRuntime(kQuickDeoptimize, instruction_, instruction_->GetDexPc(), this); 552 CheckEntrypointTypes<kQuickDeoptimize, void, void>(); 553 } 554 555 const char* GetDescription() const OVERRIDE { return "DeoptimizationSlowPathARM"; } 556 557 private: 558 DISALLOW_COPY_AND_ASSIGN(DeoptimizationSlowPathARM); 559}; 560 561class ArraySetSlowPathARM : public SlowPathCodeARM { 562 public: 563 explicit ArraySetSlowPathARM(HInstruction* instruction) : SlowPathCodeARM(instruction) {} 564 565 void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { 566 LocationSummary* locations = instruction_->GetLocations(); 567 __ Bind(GetEntryLabel()); 568 SaveLiveRegisters(codegen, locations); 569 570 InvokeRuntimeCallingConvention calling_convention; 571 HParallelMove parallel_move(codegen->GetGraph()->GetArena()); 572 parallel_move.AddMove( 573 locations->InAt(0), 574 Location::RegisterLocation(calling_convention.GetRegisterAt(0)), 575 Primitive::kPrimNot, 576 nullptr); 577 parallel_move.AddMove( 578 locations->InAt(1), 579 Location::RegisterLocation(calling_convention.GetRegisterAt(1)), 580 Primitive::kPrimInt, 581 nullptr); 582 parallel_move.AddMove( 583 locations->InAt(2), 584 Location::RegisterLocation(calling_convention.GetRegisterAt(2)), 585 Primitive::kPrimNot, 586 nullptr); 587 codegen->GetMoveResolver()->EmitNativeCode(¶llel_move); 588 589 CodeGeneratorARM* arm_codegen = down_cast<CodeGeneratorARM*>(codegen); 590 arm_codegen->InvokeRuntime(kQuickAputObject, instruction_, instruction_->GetDexPc(), this); 591 CheckEntrypointTypes<kQuickAputObject, void, mirror::Array*, int32_t, mirror::Object*>(); 592 RestoreLiveRegisters(codegen, locations); 593 __ b(GetExitLabel()); 594 } 595 596 const char* GetDescription() const OVERRIDE { return "ArraySetSlowPathARM"; } 597 598 private: 599 DISALLOW_COPY_AND_ASSIGN(ArraySetSlowPathARM); 600}; 601 602// Slow path marking an object reference `ref` during a read 603// barrier. The field `obj.field` in the object `obj` holding this 604// reference does not get updated by this slow path after marking (see 605// ReadBarrierMarkAndUpdateFieldSlowPathARM below for that). 606// 607// This means that after the execution of this slow path, `ref` will 608// always be up-to-date, but `obj.field` may not; i.e., after the 609// flip, `ref` will be a to-space reference, but `obj.field` will 610// probably still be a from-space reference (unless it gets updated by 611// another thread, or if another thread installed another object 612// reference (different from `ref`) in `obj.field`). 613class ReadBarrierMarkSlowPathARM : public SlowPathCodeARM { 614 public: 615 ReadBarrierMarkSlowPathARM(HInstruction* instruction, 616 Location ref, 617 Location entrypoint = Location::NoLocation()) 618 : SlowPathCodeARM(instruction), ref_(ref), entrypoint_(entrypoint) { 619 DCHECK(kEmitCompilerReadBarrier); 620 } 621 622 const char* GetDescription() const OVERRIDE { return "ReadBarrierMarkSlowPathARM"; } 623 624 void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { 625 LocationSummary* locations = instruction_->GetLocations(); 626 Register ref_reg = ref_.AsRegister<Register>(); 627 DCHECK(locations->CanCall()); 628 DCHECK(!locations->GetLiveRegisters()->ContainsCoreRegister(ref_reg)) << ref_reg; 629 DCHECK(instruction_->IsInstanceFieldGet() || 630 instruction_->IsStaticFieldGet() || 631 instruction_->IsArrayGet() || 632 instruction_->IsArraySet() || 633 instruction_->IsLoadClass() || 634 instruction_->IsLoadString() || 635 instruction_->IsInstanceOf() || 636 instruction_->IsCheckCast() || 637 (instruction_->IsInvokeVirtual() && instruction_->GetLocations()->Intrinsified()) || 638 (instruction_->IsInvokeStaticOrDirect() && instruction_->GetLocations()->Intrinsified())) 639 << "Unexpected instruction in read barrier marking slow path: " 640 << instruction_->DebugName(); 641 // The read barrier instrumentation of object ArrayGet 642 // instructions does not support the HIntermediateAddress 643 // instruction. 644 DCHECK(!(instruction_->IsArrayGet() && 645 instruction_->AsArrayGet()->GetArray()->IsIntermediateAddress())); 646 647 __ Bind(GetEntryLabel()); 648 // No need to save live registers; it's taken care of by the 649 // entrypoint. Also, there is no need to update the stack mask, 650 // as this runtime call will not trigger a garbage collection. 651 CodeGeneratorARM* arm_codegen = down_cast<CodeGeneratorARM*>(codegen); 652 DCHECK_NE(ref_reg, SP); 653 DCHECK_NE(ref_reg, LR); 654 DCHECK_NE(ref_reg, PC); 655 // IP is used internally by the ReadBarrierMarkRegX entry point 656 // as a temporary, it cannot be the entry point's input/output. 657 DCHECK_NE(ref_reg, IP); 658 DCHECK(0 <= ref_reg && ref_reg < kNumberOfCoreRegisters) << ref_reg; 659 // "Compact" slow path, saving two moves. 660 // 661 // Instead of using the standard runtime calling convention (input 662 // and output in R0): 663 // 664 // R0 <- ref 665 // R0 <- ReadBarrierMark(R0) 666 // ref <- R0 667 // 668 // we just use rX (the register containing `ref`) as input and output 669 // of a dedicated entrypoint: 670 // 671 // rX <- ReadBarrierMarkRegX(rX) 672 // 673 if (entrypoint_.IsValid()) { 674 arm_codegen->ValidateInvokeRuntimeWithoutRecordingPcInfo(instruction_, this); 675 __ blx(entrypoint_.AsRegister<Register>()); 676 } else { 677 int32_t entry_point_offset = 678 CodeGenerator::GetReadBarrierMarkEntryPointsOffset<kArmPointerSize>(ref_reg); 679 // This runtime call does not require a stack map. 680 arm_codegen->InvokeRuntimeWithoutRecordingPcInfo(entry_point_offset, instruction_, this); 681 } 682 __ b(GetExitLabel()); 683 } 684 685 private: 686 // The location (register) of the marked object reference. 687 const Location ref_; 688 689 // The location of the entrypoint if already loaded. 690 const Location entrypoint_; 691 692 DISALLOW_COPY_AND_ASSIGN(ReadBarrierMarkSlowPathARM); 693}; 694 695// Slow path marking an object reference `ref` during a read barrier, 696// and if needed, atomically updating the field `obj.field` in the 697// object `obj` holding this reference after marking (contrary to 698// ReadBarrierMarkSlowPathARM above, which never tries to update 699// `obj.field`). 700// 701// This means that after the execution of this slow path, both `ref` 702// and `obj.field` will be up-to-date; i.e., after the flip, both will 703// hold the same to-space reference (unless another thread installed 704// another object reference (different from `ref`) in `obj.field`). 705class ReadBarrierMarkAndUpdateFieldSlowPathARM : public SlowPathCodeARM { 706 public: 707 ReadBarrierMarkAndUpdateFieldSlowPathARM(HInstruction* instruction, 708 Location ref, 709 Register obj, 710 Location field_offset, 711 Register temp1, 712 Register temp2) 713 : SlowPathCodeARM(instruction), 714 ref_(ref), 715 obj_(obj), 716 field_offset_(field_offset), 717 temp1_(temp1), 718 temp2_(temp2) { 719 DCHECK(kEmitCompilerReadBarrier); 720 } 721 722 const char* GetDescription() const OVERRIDE { return "ReadBarrierMarkAndUpdateFieldSlowPathARM"; } 723 724 void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { 725 LocationSummary* locations = instruction_->GetLocations(); 726 Register ref_reg = ref_.AsRegister<Register>(); 727 DCHECK(locations->CanCall()); 728 DCHECK(!locations->GetLiveRegisters()->ContainsCoreRegister(ref_reg)) << ref_reg; 729 // This slow path is only used by the UnsafeCASObject intrinsic. 730 DCHECK((instruction_->IsInvokeVirtual() && instruction_->GetLocations()->Intrinsified())) 731 << "Unexpected instruction in read barrier marking and field updating slow path: " 732 << instruction_->DebugName(); 733 DCHECK(instruction_->GetLocations()->Intrinsified()); 734 DCHECK_EQ(instruction_->AsInvoke()->GetIntrinsic(), Intrinsics::kUnsafeCASObject); 735 DCHECK(field_offset_.IsRegisterPair()) << field_offset_; 736 737 __ Bind(GetEntryLabel()); 738 739 // Save the old reference. 740 // Note that we cannot use IP to save the old reference, as IP is 741 // used internally by the ReadBarrierMarkRegX entry point, and we 742 // need the old reference after the call to that entry point. 743 DCHECK_NE(temp1_, IP); 744 __ Mov(temp1_, ref_reg); 745 746 // No need to save live registers; it's taken care of by the 747 // entrypoint. Also, there is no need to update the stack mask, 748 // as this runtime call will not trigger a garbage collection. 749 CodeGeneratorARM* arm_codegen = down_cast<CodeGeneratorARM*>(codegen); 750 DCHECK_NE(ref_reg, SP); 751 DCHECK_NE(ref_reg, LR); 752 DCHECK_NE(ref_reg, PC); 753 // IP is used internally by the ReadBarrierMarkRegX entry point 754 // as a temporary, it cannot be the entry point's input/output. 755 DCHECK_NE(ref_reg, IP); 756 DCHECK(0 <= ref_reg && ref_reg < kNumberOfCoreRegisters) << ref_reg; 757 // "Compact" slow path, saving two moves. 758 // 759 // Instead of using the standard runtime calling convention (input 760 // and output in R0): 761 // 762 // R0 <- ref 763 // R0 <- ReadBarrierMark(R0) 764 // ref <- R0 765 // 766 // we just use rX (the register containing `ref`) as input and output 767 // of a dedicated entrypoint: 768 // 769 // rX <- ReadBarrierMarkRegX(rX) 770 // 771 int32_t entry_point_offset = 772 CodeGenerator::GetReadBarrierMarkEntryPointsOffset<kArmPointerSize>(ref_reg); 773 // This runtime call does not require a stack map. 774 arm_codegen->InvokeRuntimeWithoutRecordingPcInfo(entry_point_offset, instruction_, this); 775 776 // If the new reference is different from the old reference, 777 // update the field in the holder (`*(obj_ + field_offset_)`). 778 // 779 // Note that this field could also hold a different object, if 780 // another thread had concurrently changed it. In that case, the 781 // LDREX/SUBS/ITNE sequence of instructions in the compare-and-set 782 // (CAS) operation below would abort the CAS, leaving the field 783 // as-is. 784 Label done; 785 __ cmp(temp1_, ShifterOperand(ref_reg)); 786 __ b(&done, EQ); 787 788 // Update the the holder's field atomically. This may fail if 789 // mutator updates before us, but it's OK. This is achieved 790 // using a strong compare-and-set (CAS) operation with relaxed 791 // memory synchronization ordering, where the expected value is 792 // the old reference and the desired value is the new reference. 793 794 // Convenience aliases. 795 Register base = obj_; 796 // The UnsafeCASObject intrinsic uses a register pair as field 797 // offset ("long offset"), of which only the low part contains 798 // data. 799 Register offset = field_offset_.AsRegisterPairLow<Register>(); 800 Register expected = temp1_; 801 Register value = ref_reg; 802 Register tmp_ptr = IP; // Pointer to actual memory. 803 Register tmp = temp2_; // Value in memory. 804 805 __ add(tmp_ptr, base, ShifterOperand(offset)); 806 807 if (kPoisonHeapReferences) { 808 __ PoisonHeapReference(expected); 809 if (value == expected) { 810 // Do not poison `value`, as it is the same register as 811 // `expected`, which has just been poisoned. 812 } else { 813 __ PoisonHeapReference(value); 814 } 815 } 816 817 // do { 818 // tmp = [r_ptr] - expected; 819 // } while (tmp == 0 && failure([r_ptr] <- r_new_value)); 820 821 Label loop_head, exit_loop; 822 __ Bind(&loop_head); 823 824 __ ldrex(tmp, tmp_ptr); 825 826 __ subs(tmp, tmp, ShifterOperand(expected)); 827 828 __ it(NE); 829 __ clrex(NE); 830 831 __ b(&exit_loop, NE); 832 833 __ strex(tmp, value, tmp_ptr); 834 __ cmp(tmp, ShifterOperand(1)); 835 __ b(&loop_head, EQ); 836 837 __ Bind(&exit_loop); 838 839 if (kPoisonHeapReferences) { 840 __ UnpoisonHeapReference(expected); 841 if (value == expected) { 842 // Do not unpoison `value`, as it is the same register as 843 // `expected`, which has just been unpoisoned. 844 } else { 845 __ UnpoisonHeapReference(value); 846 } 847 } 848 849 __ Bind(&done); 850 __ b(GetExitLabel()); 851 } 852 853 private: 854 // The location (register) of the marked object reference. 855 const Location ref_; 856 // The register containing the object holding the marked object reference field. 857 const Register obj_; 858 // The location of the offset of the marked reference field within `obj_`. 859 Location field_offset_; 860 861 const Register temp1_; 862 const Register temp2_; 863 864 DISALLOW_COPY_AND_ASSIGN(ReadBarrierMarkAndUpdateFieldSlowPathARM); 865}; 866 867// Slow path generating a read barrier for a heap reference. 868class ReadBarrierForHeapReferenceSlowPathARM : public SlowPathCodeARM { 869 public: 870 ReadBarrierForHeapReferenceSlowPathARM(HInstruction* instruction, 871 Location out, 872 Location ref, 873 Location obj, 874 uint32_t offset, 875 Location index) 876 : SlowPathCodeARM(instruction), 877 out_(out), 878 ref_(ref), 879 obj_(obj), 880 offset_(offset), 881 index_(index) { 882 DCHECK(kEmitCompilerReadBarrier); 883 // If `obj` is equal to `out` or `ref`, it means the initial object 884 // has been overwritten by (or after) the heap object reference load 885 // to be instrumented, e.g.: 886 // 887 // __ LoadFromOffset(kLoadWord, out, out, offset); 888 // codegen_->GenerateReadBarrierSlow(instruction, out_loc, out_loc, out_loc, offset); 889 // 890 // In that case, we have lost the information about the original 891 // object, and the emitted read barrier cannot work properly. 892 DCHECK(!obj.Equals(out)) << "obj=" << obj << " out=" << out; 893 DCHECK(!obj.Equals(ref)) << "obj=" << obj << " ref=" << ref; 894 } 895 896 void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { 897 CodeGeneratorARM* arm_codegen = down_cast<CodeGeneratorARM*>(codegen); 898 LocationSummary* locations = instruction_->GetLocations(); 899 Register reg_out = out_.AsRegister<Register>(); 900 DCHECK(locations->CanCall()); 901 DCHECK(!locations->GetLiveRegisters()->ContainsCoreRegister(reg_out)); 902 DCHECK(instruction_->IsInstanceFieldGet() || 903 instruction_->IsStaticFieldGet() || 904 instruction_->IsArrayGet() || 905 instruction_->IsInstanceOf() || 906 instruction_->IsCheckCast() || 907 (instruction_->IsInvokeVirtual()) && instruction_->GetLocations()->Intrinsified()) 908 << "Unexpected instruction in read barrier for heap reference slow path: " 909 << instruction_->DebugName(); 910 // The read barrier instrumentation of object ArrayGet 911 // instructions does not support the HIntermediateAddress 912 // instruction. 913 DCHECK(!(instruction_->IsArrayGet() && 914 instruction_->AsArrayGet()->GetArray()->IsIntermediateAddress())); 915 916 __ Bind(GetEntryLabel()); 917 SaveLiveRegisters(codegen, locations); 918 919 // We may have to change the index's value, but as `index_` is a 920 // constant member (like other "inputs" of this slow path), 921 // introduce a copy of it, `index`. 922 Location index = index_; 923 if (index_.IsValid()) { 924 // Handle `index_` for HArrayGet and UnsafeGetObject/UnsafeGetObjectVolatile intrinsics. 925 if (instruction_->IsArrayGet()) { 926 // Compute the actual memory offset and store it in `index`. 927 Register index_reg = index_.AsRegister<Register>(); 928 DCHECK(locations->GetLiveRegisters()->ContainsCoreRegister(index_reg)); 929 if (codegen->IsCoreCalleeSaveRegister(index_reg)) { 930 // We are about to change the value of `index_reg` (see the 931 // calls to art::arm::Thumb2Assembler::Lsl and 932 // art::arm::Thumb2Assembler::AddConstant below), but it has 933 // not been saved by the previous call to 934 // art::SlowPathCode::SaveLiveRegisters, as it is a 935 // callee-save register -- 936 // art::SlowPathCode::SaveLiveRegisters does not consider 937 // callee-save registers, as it has been designed with the 938 // assumption that callee-save registers are supposed to be 939 // handled by the called function. So, as a callee-save 940 // register, `index_reg` _would_ eventually be saved onto 941 // the stack, but it would be too late: we would have 942 // changed its value earlier. Therefore, we manually save 943 // it here into another freely available register, 944 // `free_reg`, chosen of course among the caller-save 945 // registers (as a callee-save `free_reg` register would 946 // exhibit the same problem). 947 // 948 // Note we could have requested a temporary register from 949 // the register allocator instead; but we prefer not to, as 950 // this is a slow path, and we know we can find a 951 // caller-save register that is available. 952 Register free_reg = FindAvailableCallerSaveRegister(codegen); 953 __ Mov(free_reg, index_reg); 954 index_reg = free_reg; 955 index = Location::RegisterLocation(index_reg); 956 } else { 957 // The initial register stored in `index_` has already been 958 // saved in the call to art::SlowPathCode::SaveLiveRegisters 959 // (as it is not a callee-save register), so we can freely 960 // use it. 961 } 962 // Shifting the index value contained in `index_reg` by the scale 963 // factor (2) cannot overflow in practice, as the runtime is 964 // unable to allocate object arrays with a size larger than 965 // 2^26 - 1 (that is, 2^28 - 4 bytes). 966 __ Lsl(index_reg, index_reg, TIMES_4); 967 static_assert( 968 sizeof(mirror::HeapReference<mirror::Object>) == sizeof(int32_t), 969 "art::mirror::HeapReference<art::mirror::Object> and int32_t have different sizes."); 970 __ AddConstant(index_reg, index_reg, offset_); 971 } else { 972 // In the case of the UnsafeGetObject/UnsafeGetObjectVolatile 973 // intrinsics, `index_` is not shifted by a scale factor of 2 974 // (as in the case of ArrayGet), as it is actually an offset 975 // to an object field within an object. 976 DCHECK(instruction_->IsInvoke()) << instruction_->DebugName(); 977 DCHECK(instruction_->GetLocations()->Intrinsified()); 978 DCHECK((instruction_->AsInvoke()->GetIntrinsic() == Intrinsics::kUnsafeGetObject) || 979 (instruction_->AsInvoke()->GetIntrinsic() == Intrinsics::kUnsafeGetObjectVolatile)) 980 << instruction_->AsInvoke()->GetIntrinsic(); 981 DCHECK_EQ(offset_, 0U); 982 DCHECK(index_.IsRegisterPair()); 983 // UnsafeGet's offset location is a register pair, the low 984 // part contains the correct offset. 985 index = index_.ToLow(); 986 } 987 } 988 989 // We're moving two or three locations to locations that could 990 // overlap, so we need a parallel move resolver. 991 InvokeRuntimeCallingConvention calling_convention; 992 HParallelMove parallel_move(codegen->GetGraph()->GetArena()); 993 parallel_move.AddMove(ref_, 994 Location::RegisterLocation(calling_convention.GetRegisterAt(0)), 995 Primitive::kPrimNot, 996 nullptr); 997 parallel_move.AddMove(obj_, 998 Location::RegisterLocation(calling_convention.GetRegisterAt(1)), 999 Primitive::kPrimNot, 1000 nullptr); 1001 if (index.IsValid()) { 1002 parallel_move.AddMove(index, 1003 Location::RegisterLocation(calling_convention.GetRegisterAt(2)), 1004 Primitive::kPrimInt, 1005 nullptr); 1006 codegen->GetMoveResolver()->EmitNativeCode(¶llel_move); 1007 } else { 1008 codegen->GetMoveResolver()->EmitNativeCode(¶llel_move); 1009 __ LoadImmediate(calling_convention.GetRegisterAt(2), offset_); 1010 } 1011 arm_codegen->InvokeRuntime(kQuickReadBarrierSlow, instruction_, instruction_->GetDexPc(), this); 1012 CheckEntrypointTypes< 1013 kQuickReadBarrierSlow, mirror::Object*, mirror::Object*, mirror::Object*, uint32_t>(); 1014 arm_codegen->Move32(out_, Location::RegisterLocation(R0)); 1015 1016 RestoreLiveRegisters(codegen, locations); 1017 __ b(GetExitLabel()); 1018 } 1019 1020 const char* GetDescription() const OVERRIDE { return "ReadBarrierForHeapReferenceSlowPathARM"; } 1021 1022 private: 1023 Register FindAvailableCallerSaveRegister(CodeGenerator* codegen) { 1024 size_t ref = static_cast<int>(ref_.AsRegister<Register>()); 1025 size_t obj = static_cast<int>(obj_.AsRegister<Register>()); 1026 for (size_t i = 0, e = codegen->GetNumberOfCoreRegisters(); i < e; ++i) { 1027 if (i != ref && i != obj && !codegen->IsCoreCalleeSaveRegister(i)) { 1028 return static_cast<Register>(i); 1029 } 1030 } 1031 // We shall never fail to find a free caller-save register, as 1032 // there are more than two core caller-save registers on ARM 1033 // (meaning it is possible to find one which is different from 1034 // `ref` and `obj`). 1035 DCHECK_GT(codegen->GetNumberOfCoreCallerSaveRegisters(), 2u); 1036 LOG(FATAL) << "Could not find a free caller-save register"; 1037 UNREACHABLE(); 1038 } 1039 1040 const Location out_; 1041 const Location ref_; 1042 const Location obj_; 1043 const uint32_t offset_; 1044 // An additional location containing an index to an array. 1045 // Only used for HArrayGet and the UnsafeGetObject & 1046 // UnsafeGetObjectVolatile intrinsics. 1047 const Location index_; 1048 1049 DISALLOW_COPY_AND_ASSIGN(ReadBarrierForHeapReferenceSlowPathARM); 1050}; 1051 1052// Slow path generating a read barrier for a GC root. 1053class ReadBarrierForRootSlowPathARM : public SlowPathCodeARM { 1054 public: 1055 ReadBarrierForRootSlowPathARM(HInstruction* instruction, Location out, Location root) 1056 : SlowPathCodeARM(instruction), out_(out), root_(root) { 1057 DCHECK(kEmitCompilerReadBarrier); 1058 } 1059 1060 void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { 1061 LocationSummary* locations = instruction_->GetLocations(); 1062 Register reg_out = out_.AsRegister<Register>(); 1063 DCHECK(locations->CanCall()); 1064 DCHECK(!locations->GetLiveRegisters()->ContainsCoreRegister(reg_out)); 1065 DCHECK(instruction_->IsLoadClass() || instruction_->IsLoadString()) 1066 << "Unexpected instruction in read barrier for GC root slow path: " 1067 << instruction_->DebugName(); 1068 1069 __ Bind(GetEntryLabel()); 1070 SaveLiveRegisters(codegen, locations); 1071 1072 InvokeRuntimeCallingConvention calling_convention; 1073 CodeGeneratorARM* arm_codegen = down_cast<CodeGeneratorARM*>(codegen); 1074 arm_codegen->Move32(Location::RegisterLocation(calling_convention.GetRegisterAt(0)), root_); 1075 arm_codegen->InvokeRuntime(kQuickReadBarrierForRootSlow, 1076 instruction_, 1077 instruction_->GetDexPc(), 1078 this); 1079 CheckEntrypointTypes<kQuickReadBarrierForRootSlow, mirror::Object*, GcRoot<mirror::Object>*>(); 1080 arm_codegen->Move32(out_, Location::RegisterLocation(R0)); 1081 1082 RestoreLiveRegisters(codegen, locations); 1083 __ b(GetExitLabel()); 1084 } 1085 1086 const char* GetDescription() const OVERRIDE { return "ReadBarrierForRootSlowPathARM"; } 1087 1088 private: 1089 const Location out_; 1090 const Location root_; 1091 1092 DISALLOW_COPY_AND_ASSIGN(ReadBarrierForRootSlowPathARM); 1093}; 1094 1095#undef __ 1096// NOLINT on __ macro to suppress wrong warning/fix (misc-macro-parentheses) from clang-tidy. 1097#define __ down_cast<ArmAssembler*>(GetAssembler())-> // NOLINT 1098 1099inline Condition ARMCondition(IfCondition cond) { 1100 switch (cond) { 1101 case kCondEQ: return EQ; 1102 case kCondNE: return NE; 1103 case kCondLT: return LT; 1104 case kCondLE: return LE; 1105 case kCondGT: return GT; 1106 case kCondGE: return GE; 1107 case kCondB: return LO; 1108 case kCondBE: return LS; 1109 case kCondA: return HI; 1110 case kCondAE: return HS; 1111 } 1112 LOG(FATAL) << "Unreachable"; 1113 UNREACHABLE(); 1114} 1115 1116// Maps signed condition to unsigned condition. 1117inline Condition ARMUnsignedCondition(IfCondition cond) { 1118 switch (cond) { 1119 case kCondEQ: return EQ; 1120 case kCondNE: return NE; 1121 // Signed to unsigned. 1122 case kCondLT: return LO; 1123 case kCondLE: return LS; 1124 case kCondGT: return HI; 1125 case kCondGE: return HS; 1126 // Unsigned remain unchanged. 1127 case kCondB: return LO; 1128 case kCondBE: return LS; 1129 case kCondA: return HI; 1130 case kCondAE: return HS; 1131 } 1132 LOG(FATAL) << "Unreachable"; 1133 UNREACHABLE(); 1134} 1135 1136inline Condition ARMFPCondition(IfCondition cond, bool gt_bias) { 1137 // The ARM condition codes can express all the necessary branches, see the 1138 // "Meaning (floating-point)" column in the table A8-1 of the ARMv7 reference manual. 1139 // There is no dex instruction or HIR that would need the missing conditions 1140 // "equal or unordered" or "not equal". 1141 switch (cond) { 1142 case kCondEQ: return EQ; 1143 case kCondNE: return NE /* unordered */; 1144 case kCondLT: return gt_bias ? CC : LT /* unordered */; 1145 case kCondLE: return gt_bias ? LS : LE /* unordered */; 1146 case kCondGT: return gt_bias ? HI /* unordered */ : GT; 1147 case kCondGE: return gt_bias ? CS /* unordered */ : GE; 1148 default: 1149 LOG(FATAL) << "UNREACHABLE"; 1150 UNREACHABLE(); 1151 } 1152} 1153 1154void CodeGeneratorARM::DumpCoreRegister(std::ostream& stream, int reg) const { 1155 stream << Register(reg); 1156} 1157 1158void CodeGeneratorARM::DumpFloatingPointRegister(std::ostream& stream, int reg) const { 1159 stream << SRegister(reg); 1160} 1161 1162size_t CodeGeneratorARM::SaveCoreRegister(size_t stack_index, uint32_t reg_id) { 1163 __ StoreToOffset(kStoreWord, static_cast<Register>(reg_id), SP, stack_index); 1164 return kArmWordSize; 1165} 1166 1167size_t CodeGeneratorARM::RestoreCoreRegister(size_t stack_index, uint32_t reg_id) { 1168 __ LoadFromOffset(kLoadWord, static_cast<Register>(reg_id), SP, stack_index); 1169 return kArmWordSize; 1170} 1171 1172size_t CodeGeneratorARM::SaveFloatingPointRegister(size_t stack_index, uint32_t reg_id) { 1173 __ StoreSToOffset(static_cast<SRegister>(reg_id), SP, stack_index); 1174 return kArmWordSize; 1175} 1176 1177size_t CodeGeneratorARM::RestoreFloatingPointRegister(size_t stack_index, uint32_t reg_id) { 1178 __ LoadSFromOffset(static_cast<SRegister>(reg_id), SP, stack_index); 1179 return kArmWordSize; 1180} 1181 1182CodeGeneratorARM::CodeGeneratorARM(HGraph* graph, 1183 const ArmInstructionSetFeatures& isa_features, 1184 const CompilerOptions& compiler_options, 1185 OptimizingCompilerStats* stats) 1186 : CodeGenerator(graph, 1187 kNumberOfCoreRegisters, 1188 kNumberOfSRegisters, 1189 kNumberOfRegisterPairs, 1190 ComputeRegisterMask(reinterpret_cast<const int*>(kCoreCalleeSaves), 1191 arraysize(kCoreCalleeSaves)), 1192 ComputeRegisterMask(reinterpret_cast<const int*>(kFpuCalleeSaves), 1193 arraysize(kFpuCalleeSaves)), 1194 compiler_options, 1195 stats), 1196 block_labels_(nullptr), 1197 location_builder_(graph, this), 1198 instruction_visitor_(graph, this), 1199 move_resolver_(graph->GetArena(), this), 1200 assembler_(graph->GetArena()), 1201 isa_features_(isa_features), 1202 uint32_literals_(std::less<uint32_t>(), 1203 graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), 1204 method_patches_(MethodReferenceComparator(), 1205 graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), 1206 call_patches_(MethodReferenceComparator(), 1207 graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), 1208 relative_call_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), 1209 pc_relative_dex_cache_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), 1210 boot_image_string_patches_(StringReferenceValueComparator(), 1211 graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), 1212 pc_relative_string_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), 1213 boot_image_type_patches_(TypeReferenceValueComparator(), 1214 graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), 1215 pc_relative_type_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), 1216 boot_image_address_patches_(std::less<uint32_t>(), 1217 graph->GetArena()->Adapter(kArenaAllocCodeGenerator)) { 1218 // Always save the LR register to mimic Quick. 1219 AddAllocatedRegister(Location::RegisterLocation(LR)); 1220} 1221 1222void CodeGeneratorARM::Finalize(CodeAllocator* allocator) { 1223 // Ensure that we fix up branches and literal loads and emit the literal pool. 1224 __ FinalizeCode(); 1225 1226 // Adjust native pc offsets in stack maps. 1227 for (size_t i = 0, num = stack_map_stream_.GetNumberOfStackMaps(); i != num; ++i) { 1228 uint32_t old_position = stack_map_stream_.GetStackMap(i).native_pc_offset; 1229 uint32_t new_position = __ GetAdjustedPosition(old_position); 1230 stack_map_stream_.SetStackMapNativePcOffset(i, new_position); 1231 } 1232 // Adjust pc offsets for the disassembly information. 1233 if (disasm_info_ != nullptr) { 1234 GeneratedCodeInterval* frame_entry_interval = disasm_info_->GetFrameEntryInterval(); 1235 frame_entry_interval->start = __ GetAdjustedPosition(frame_entry_interval->start); 1236 frame_entry_interval->end = __ GetAdjustedPosition(frame_entry_interval->end); 1237 for (auto& it : *disasm_info_->GetInstructionIntervals()) { 1238 it.second.start = __ GetAdjustedPosition(it.second.start); 1239 it.second.end = __ GetAdjustedPosition(it.second.end); 1240 } 1241 for (auto& it : *disasm_info_->GetSlowPathIntervals()) { 1242 it.code_interval.start = __ GetAdjustedPosition(it.code_interval.start); 1243 it.code_interval.end = __ GetAdjustedPosition(it.code_interval.end); 1244 } 1245 } 1246 1247 CodeGenerator::Finalize(allocator); 1248} 1249 1250void CodeGeneratorARM::SetupBlockedRegisters() const { 1251 // Stack register, LR and PC are always reserved. 1252 blocked_core_registers_[SP] = true; 1253 blocked_core_registers_[LR] = true; 1254 blocked_core_registers_[PC] = true; 1255 1256 // Reserve thread register. 1257 blocked_core_registers_[TR] = true; 1258 1259 // Reserve temp register. 1260 blocked_core_registers_[IP] = true; 1261 1262 if (GetGraph()->IsDebuggable()) { 1263 // Stubs do not save callee-save floating point registers. If the graph 1264 // is debuggable, we need to deal with these registers differently. For 1265 // now, just block them. 1266 for (size_t i = 0; i < arraysize(kFpuCalleeSaves); ++i) { 1267 blocked_fpu_registers_[kFpuCalleeSaves[i]] = true; 1268 } 1269 } 1270} 1271 1272InstructionCodeGeneratorARM::InstructionCodeGeneratorARM(HGraph* graph, CodeGeneratorARM* codegen) 1273 : InstructionCodeGenerator(graph, codegen), 1274 assembler_(codegen->GetAssembler()), 1275 codegen_(codegen) {} 1276 1277void CodeGeneratorARM::ComputeSpillMask() { 1278 core_spill_mask_ = allocated_registers_.GetCoreRegisters() & core_callee_save_mask_; 1279 DCHECK_NE(core_spill_mask_, 0u) << "At least the return address register must be saved"; 1280 // There is no easy instruction to restore just the PC on thumb2. We spill and 1281 // restore another arbitrary register. 1282 core_spill_mask_ |= (1 << kCoreAlwaysSpillRegister); 1283 fpu_spill_mask_ = allocated_registers_.GetFloatingPointRegisters() & fpu_callee_save_mask_; 1284 // We use vpush and vpop for saving and restoring floating point registers, which take 1285 // a SRegister and the number of registers to save/restore after that SRegister. We 1286 // therefore update the `fpu_spill_mask_` to also contain those registers not allocated, 1287 // but in the range. 1288 if (fpu_spill_mask_ != 0) { 1289 uint32_t least_significant_bit = LeastSignificantBit(fpu_spill_mask_); 1290 uint32_t most_significant_bit = MostSignificantBit(fpu_spill_mask_); 1291 for (uint32_t i = least_significant_bit + 1 ; i < most_significant_bit; ++i) { 1292 fpu_spill_mask_ |= (1 << i); 1293 } 1294 } 1295} 1296 1297static dwarf::Reg DWARFReg(Register reg) { 1298 return dwarf::Reg::ArmCore(static_cast<int>(reg)); 1299} 1300 1301static dwarf::Reg DWARFReg(SRegister reg) { 1302 return dwarf::Reg::ArmFp(static_cast<int>(reg)); 1303} 1304 1305void CodeGeneratorARM::GenerateFrameEntry() { 1306 bool skip_overflow_check = 1307 IsLeafMethod() && !FrameNeedsStackCheck(GetFrameSize(), InstructionSet::kArm); 1308 DCHECK(GetCompilerOptions().GetImplicitStackOverflowChecks()); 1309 __ Bind(&frame_entry_label_); 1310 1311 if (HasEmptyFrame()) { 1312 return; 1313 } 1314 1315 if (!skip_overflow_check) { 1316 __ AddConstant(IP, SP, -static_cast<int32_t>(GetStackOverflowReservedBytes(kArm))); 1317 __ LoadFromOffset(kLoadWord, IP, IP, 0); 1318 RecordPcInfo(nullptr, 0); 1319 } 1320 1321 __ PushList(core_spill_mask_); 1322 __ cfi().AdjustCFAOffset(kArmWordSize * POPCOUNT(core_spill_mask_)); 1323 __ cfi().RelOffsetForMany(DWARFReg(kMethodRegisterArgument), 0, core_spill_mask_, kArmWordSize); 1324 if (fpu_spill_mask_ != 0) { 1325 SRegister start_register = SRegister(LeastSignificantBit(fpu_spill_mask_)); 1326 __ vpushs(start_register, POPCOUNT(fpu_spill_mask_)); 1327 __ cfi().AdjustCFAOffset(kArmWordSize * POPCOUNT(fpu_spill_mask_)); 1328 __ cfi().RelOffsetForMany(DWARFReg(S0), 0, fpu_spill_mask_, kArmWordSize); 1329 } 1330 int adjust = GetFrameSize() - FrameEntrySpillSize(); 1331 __ AddConstant(SP, -adjust); 1332 __ cfi().AdjustCFAOffset(adjust); 1333 1334 // Save the current method if we need it. Note that we do not 1335 // do this in HCurrentMethod, as the instruction might have been removed 1336 // in the SSA graph. 1337 if (RequiresCurrentMethod()) { 1338 __ StoreToOffset(kStoreWord, kMethodRegisterArgument, SP, 0); 1339 } 1340} 1341 1342void CodeGeneratorARM::GenerateFrameExit() { 1343 if (HasEmptyFrame()) { 1344 __ bx(LR); 1345 return; 1346 } 1347 __ cfi().RememberState(); 1348 int adjust = GetFrameSize() - FrameEntrySpillSize(); 1349 __ AddConstant(SP, adjust); 1350 __ cfi().AdjustCFAOffset(-adjust); 1351 if (fpu_spill_mask_ != 0) { 1352 SRegister start_register = SRegister(LeastSignificantBit(fpu_spill_mask_)); 1353 __ vpops(start_register, POPCOUNT(fpu_spill_mask_)); 1354 __ cfi().AdjustCFAOffset(-static_cast<int>(kArmPointerSize) * POPCOUNT(fpu_spill_mask_)); 1355 __ cfi().RestoreMany(DWARFReg(SRegister(0)), fpu_spill_mask_); 1356 } 1357 // Pop LR into PC to return. 1358 DCHECK_NE(core_spill_mask_ & (1 << LR), 0U); 1359 uint32_t pop_mask = (core_spill_mask_ & (~(1 << LR))) | 1 << PC; 1360 __ PopList(pop_mask); 1361 __ cfi().RestoreState(); 1362 __ cfi().DefCFAOffset(GetFrameSize()); 1363} 1364 1365void CodeGeneratorARM::Bind(HBasicBlock* block) { 1366 Label* label = GetLabelOf(block); 1367 __ BindTrackedLabel(label); 1368} 1369 1370Location InvokeDexCallingConventionVisitorARM::GetNextLocation(Primitive::Type type) { 1371 switch (type) { 1372 case Primitive::kPrimBoolean: 1373 case Primitive::kPrimByte: 1374 case Primitive::kPrimChar: 1375 case Primitive::kPrimShort: 1376 case Primitive::kPrimInt: 1377 case Primitive::kPrimNot: { 1378 uint32_t index = gp_index_++; 1379 uint32_t stack_index = stack_index_++; 1380 if (index < calling_convention.GetNumberOfRegisters()) { 1381 return Location::RegisterLocation(calling_convention.GetRegisterAt(index)); 1382 } else { 1383 return Location::StackSlot(calling_convention.GetStackOffsetOf(stack_index)); 1384 } 1385 } 1386 1387 case Primitive::kPrimLong: { 1388 uint32_t index = gp_index_; 1389 uint32_t stack_index = stack_index_; 1390 gp_index_ += 2; 1391 stack_index_ += 2; 1392 if (index + 1 < calling_convention.GetNumberOfRegisters()) { 1393 if (calling_convention.GetRegisterAt(index) == R1) { 1394 // Skip R1, and use R2_R3 instead. 1395 gp_index_++; 1396 index++; 1397 } 1398 } 1399 if (index + 1 < calling_convention.GetNumberOfRegisters()) { 1400 DCHECK_EQ(calling_convention.GetRegisterAt(index) + 1, 1401 calling_convention.GetRegisterAt(index + 1)); 1402 1403 return Location::RegisterPairLocation(calling_convention.GetRegisterAt(index), 1404 calling_convention.GetRegisterAt(index + 1)); 1405 } else { 1406 return Location::DoubleStackSlot(calling_convention.GetStackOffsetOf(stack_index)); 1407 } 1408 } 1409 1410 case Primitive::kPrimFloat: { 1411 uint32_t stack_index = stack_index_++; 1412 if (float_index_ % 2 == 0) { 1413 float_index_ = std::max(double_index_, float_index_); 1414 } 1415 if (float_index_ < calling_convention.GetNumberOfFpuRegisters()) { 1416 return Location::FpuRegisterLocation(calling_convention.GetFpuRegisterAt(float_index_++)); 1417 } else { 1418 return Location::StackSlot(calling_convention.GetStackOffsetOf(stack_index)); 1419 } 1420 } 1421 1422 case Primitive::kPrimDouble: { 1423 double_index_ = std::max(double_index_, RoundUp(float_index_, 2)); 1424 uint32_t stack_index = stack_index_; 1425 stack_index_ += 2; 1426 if (double_index_ + 1 < calling_convention.GetNumberOfFpuRegisters()) { 1427 uint32_t index = double_index_; 1428 double_index_ += 2; 1429 Location result = Location::FpuRegisterPairLocation( 1430 calling_convention.GetFpuRegisterAt(index), 1431 calling_convention.GetFpuRegisterAt(index + 1)); 1432 DCHECK(ExpectedPairLayout(result)); 1433 return result; 1434 } else { 1435 return Location::DoubleStackSlot(calling_convention.GetStackOffsetOf(stack_index)); 1436 } 1437 } 1438 1439 case Primitive::kPrimVoid: 1440 LOG(FATAL) << "Unexpected parameter type " << type; 1441 break; 1442 } 1443 return Location::NoLocation(); 1444} 1445 1446Location InvokeDexCallingConventionVisitorARM::GetReturnLocation(Primitive::Type type) const { 1447 switch (type) { 1448 case Primitive::kPrimBoolean: 1449 case Primitive::kPrimByte: 1450 case Primitive::kPrimChar: 1451 case Primitive::kPrimShort: 1452 case Primitive::kPrimInt: 1453 case Primitive::kPrimNot: { 1454 return Location::RegisterLocation(R0); 1455 } 1456 1457 case Primitive::kPrimFloat: { 1458 return Location::FpuRegisterLocation(S0); 1459 } 1460 1461 case Primitive::kPrimLong: { 1462 return Location::RegisterPairLocation(R0, R1); 1463 } 1464 1465 case Primitive::kPrimDouble: { 1466 return Location::FpuRegisterPairLocation(S0, S1); 1467 } 1468 1469 case Primitive::kPrimVoid: 1470 return Location::NoLocation(); 1471 } 1472 1473 UNREACHABLE(); 1474} 1475 1476Location InvokeDexCallingConventionVisitorARM::GetMethodLocation() const { 1477 return Location::RegisterLocation(kMethodRegisterArgument); 1478} 1479 1480void CodeGeneratorARM::Move32(Location destination, Location source) { 1481 if (source.Equals(destination)) { 1482 return; 1483 } 1484 if (destination.IsRegister()) { 1485 if (source.IsRegister()) { 1486 __ Mov(destination.AsRegister<Register>(), source.AsRegister<Register>()); 1487 } else if (source.IsFpuRegister()) { 1488 __ vmovrs(destination.AsRegister<Register>(), source.AsFpuRegister<SRegister>()); 1489 } else { 1490 __ LoadFromOffset(kLoadWord, destination.AsRegister<Register>(), SP, source.GetStackIndex()); 1491 } 1492 } else if (destination.IsFpuRegister()) { 1493 if (source.IsRegister()) { 1494 __ vmovsr(destination.AsFpuRegister<SRegister>(), source.AsRegister<Register>()); 1495 } else if (source.IsFpuRegister()) { 1496 __ vmovs(destination.AsFpuRegister<SRegister>(), source.AsFpuRegister<SRegister>()); 1497 } else { 1498 __ LoadSFromOffset(destination.AsFpuRegister<SRegister>(), SP, source.GetStackIndex()); 1499 } 1500 } else { 1501 DCHECK(destination.IsStackSlot()) << destination; 1502 if (source.IsRegister()) { 1503 __ StoreToOffset(kStoreWord, source.AsRegister<Register>(), SP, destination.GetStackIndex()); 1504 } else if (source.IsFpuRegister()) { 1505 __ StoreSToOffset(source.AsFpuRegister<SRegister>(), SP, destination.GetStackIndex()); 1506 } else { 1507 DCHECK(source.IsStackSlot()) << source; 1508 __ LoadFromOffset(kLoadWord, IP, SP, source.GetStackIndex()); 1509 __ StoreToOffset(kStoreWord, IP, SP, destination.GetStackIndex()); 1510 } 1511 } 1512} 1513 1514void CodeGeneratorARM::Move64(Location destination, Location source) { 1515 if (source.Equals(destination)) { 1516 return; 1517 } 1518 if (destination.IsRegisterPair()) { 1519 if (source.IsRegisterPair()) { 1520 EmitParallelMoves( 1521 Location::RegisterLocation(source.AsRegisterPairHigh<Register>()), 1522 Location::RegisterLocation(destination.AsRegisterPairHigh<Register>()), 1523 Primitive::kPrimInt, 1524 Location::RegisterLocation(source.AsRegisterPairLow<Register>()), 1525 Location::RegisterLocation(destination.AsRegisterPairLow<Register>()), 1526 Primitive::kPrimInt); 1527 } else if (source.IsFpuRegister()) { 1528 UNIMPLEMENTED(FATAL); 1529 } else if (source.IsFpuRegisterPair()) { 1530 __ vmovrrd(destination.AsRegisterPairLow<Register>(), 1531 destination.AsRegisterPairHigh<Register>(), 1532 FromLowSToD(source.AsFpuRegisterPairLow<SRegister>())); 1533 } else { 1534 DCHECK(source.IsDoubleStackSlot()); 1535 DCHECK(ExpectedPairLayout(destination)); 1536 __ LoadFromOffset(kLoadWordPair, destination.AsRegisterPairLow<Register>(), 1537 SP, source.GetStackIndex()); 1538 } 1539 } else if (destination.IsFpuRegisterPair()) { 1540 if (source.IsDoubleStackSlot()) { 1541 __ LoadDFromOffset(FromLowSToD(destination.AsFpuRegisterPairLow<SRegister>()), 1542 SP, 1543 source.GetStackIndex()); 1544 } else if (source.IsRegisterPair()) { 1545 __ vmovdrr(FromLowSToD(destination.AsFpuRegisterPairLow<SRegister>()), 1546 source.AsRegisterPairLow<Register>(), 1547 source.AsRegisterPairHigh<Register>()); 1548 } else { 1549 UNIMPLEMENTED(FATAL); 1550 } 1551 } else { 1552 DCHECK(destination.IsDoubleStackSlot()); 1553 if (source.IsRegisterPair()) { 1554 // No conflict possible, so just do the moves. 1555 if (source.AsRegisterPairLow<Register>() == R1) { 1556 DCHECK_EQ(source.AsRegisterPairHigh<Register>(), R2); 1557 __ StoreToOffset(kStoreWord, R1, SP, destination.GetStackIndex()); 1558 __ StoreToOffset(kStoreWord, R2, SP, destination.GetHighStackIndex(kArmWordSize)); 1559 } else { 1560 __ StoreToOffset(kStoreWordPair, source.AsRegisterPairLow<Register>(), 1561 SP, destination.GetStackIndex()); 1562 } 1563 } else if (source.IsFpuRegisterPair()) { 1564 __ StoreDToOffset(FromLowSToD(source.AsFpuRegisterPairLow<SRegister>()), 1565 SP, 1566 destination.GetStackIndex()); 1567 } else { 1568 DCHECK(source.IsDoubleStackSlot()); 1569 EmitParallelMoves( 1570 Location::StackSlot(source.GetStackIndex()), 1571 Location::StackSlot(destination.GetStackIndex()), 1572 Primitive::kPrimInt, 1573 Location::StackSlot(source.GetHighStackIndex(kArmWordSize)), 1574 Location::StackSlot(destination.GetHighStackIndex(kArmWordSize)), 1575 Primitive::kPrimInt); 1576 } 1577 } 1578} 1579 1580void CodeGeneratorARM::MoveConstant(Location location, int32_t value) { 1581 DCHECK(location.IsRegister()); 1582 __ LoadImmediate(location.AsRegister<Register>(), value); 1583} 1584 1585void CodeGeneratorARM::MoveLocation(Location dst, Location src, Primitive::Type dst_type) { 1586 HParallelMove move(GetGraph()->GetArena()); 1587 move.AddMove(src, dst, dst_type, nullptr); 1588 GetMoveResolver()->EmitNativeCode(&move); 1589} 1590 1591void CodeGeneratorARM::AddLocationAsTemp(Location location, LocationSummary* locations) { 1592 if (location.IsRegister()) { 1593 locations->AddTemp(location); 1594 } else if (location.IsRegisterPair()) { 1595 locations->AddTemp(Location::RegisterLocation(location.AsRegisterPairLow<Register>())); 1596 locations->AddTemp(Location::RegisterLocation(location.AsRegisterPairHigh<Register>())); 1597 } else { 1598 UNIMPLEMENTED(FATAL) << "AddLocationAsTemp not implemented for location " << location; 1599 } 1600} 1601 1602void CodeGeneratorARM::InvokeRuntime(QuickEntrypointEnum entrypoint, 1603 HInstruction* instruction, 1604 uint32_t dex_pc, 1605 SlowPathCode* slow_path) { 1606 ValidateInvokeRuntime(entrypoint, instruction, slow_path); 1607 GenerateInvokeRuntime(GetThreadOffset<kArmPointerSize>(entrypoint).Int32Value()); 1608 if (EntrypointRequiresStackMap(entrypoint)) { 1609 RecordPcInfo(instruction, dex_pc, slow_path); 1610 } 1611} 1612 1613void CodeGeneratorARM::InvokeRuntimeWithoutRecordingPcInfo(int32_t entry_point_offset, 1614 HInstruction* instruction, 1615 SlowPathCode* slow_path) { 1616 ValidateInvokeRuntimeWithoutRecordingPcInfo(instruction, slow_path); 1617 GenerateInvokeRuntime(entry_point_offset); 1618} 1619 1620void CodeGeneratorARM::GenerateInvokeRuntime(int32_t entry_point_offset) { 1621 __ LoadFromOffset(kLoadWord, LR, TR, entry_point_offset); 1622 __ blx(LR); 1623} 1624 1625void InstructionCodeGeneratorARM::HandleGoto(HInstruction* got, HBasicBlock* successor) { 1626 DCHECK(!successor->IsExitBlock()); 1627 1628 HBasicBlock* block = got->GetBlock(); 1629 HInstruction* previous = got->GetPrevious(); 1630 1631 HLoopInformation* info = block->GetLoopInformation(); 1632 if (info != nullptr && info->IsBackEdge(*block) && info->HasSuspendCheck()) { 1633 codegen_->ClearSpillSlotsFromLoopPhisInStackMap(info->GetSuspendCheck()); 1634 GenerateSuspendCheck(info->GetSuspendCheck(), successor); 1635 return; 1636 } 1637 1638 if (block->IsEntryBlock() && (previous != nullptr) && previous->IsSuspendCheck()) { 1639 GenerateSuspendCheck(previous->AsSuspendCheck(), nullptr); 1640 } 1641 if (!codegen_->GoesToNextBlock(got->GetBlock(), successor)) { 1642 __ b(codegen_->GetLabelOf(successor)); 1643 } 1644} 1645 1646void LocationsBuilderARM::VisitGoto(HGoto* got) { 1647 got->SetLocations(nullptr); 1648} 1649 1650void InstructionCodeGeneratorARM::VisitGoto(HGoto* got) { 1651 HandleGoto(got, got->GetSuccessor()); 1652} 1653 1654void LocationsBuilderARM::VisitTryBoundary(HTryBoundary* try_boundary) { 1655 try_boundary->SetLocations(nullptr); 1656} 1657 1658void InstructionCodeGeneratorARM::VisitTryBoundary(HTryBoundary* try_boundary) { 1659 HBasicBlock* successor = try_boundary->GetNormalFlowSuccessor(); 1660 if (!successor->IsExitBlock()) { 1661 HandleGoto(try_boundary, successor); 1662 } 1663} 1664 1665void LocationsBuilderARM::VisitExit(HExit* exit) { 1666 exit->SetLocations(nullptr); 1667} 1668 1669void InstructionCodeGeneratorARM::VisitExit(HExit* exit ATTRIBUTE_UNUSED) { 1670} 1671 1672void InstructionCodeGeneratorARM::GenerateVcmp(HInstruction* instruction) { 1673 Primitive::Type type = instruction->InputAt(0)->GetType(); 1674 Location lhs_loc = instruction->GetLocations()->InAt(0); 1675 Location rhs_loc = instruction->GetLocations()->InAt(1); 1676 if (rhs_loc.IsConstant()) { 1677 // 0.0 is the only immediate that can be encoded directly in 1678 // a VCMP instruction. 1679 // 1680 // Both the JLS (section 15.20.1) and the JVMS (section 6.5) 1681 // specify that in a floating-point comparison, positive zero 1682 // and negative zero are considered equal, so we can use the 1683 // literal 0.0 for both cases here. 1684 // 1685 // Note however that some methods (Float.equal, Float.compare, 1686 // Float.compareTo, Double.equal, Double.compare, 1687 // Double.compareTo, Math.max, Math.min, StrictMath.max, 1688 // StrictMath.min) consider 0.0 to be (strictly) greater than 1689 // -0.0. So if we ever translate calls to these methods into a 1690 // HCompare instruction, we must handle the -0.0 case with 1691 // care here. 1692 DCHECK(rhs_loc.GetConstant()->IsArithmeticZero()); 1693 if (type == Primitive::kPrimFloat) { 1694 __ vcmpsz(lhs_loc.AsFpuRegister<SRegister>()); 1695 } else { 1696 DCHECK_EQ(type, Primitive::kPrimDouble); 1697 __ vcmpdz(FromLowSToD(lhs_loc.AsFpuRegisterPairLow<SRegister>())); 1698 } 1699 } else { 1700 if (type == Primitive::kPrimFloat) { 1701 __ vcmps(lhs_loc.AsFpuRegister<SRegister>(), rhs_loc.AsFpuRegister<SRegister>()); 1702 } else { 1703 DCHECK_EQ(type, Primitive::kPrimDouble); 1704 __ vcmpd(FromLowSToD(lhs_loc.AsFpuRegisterPairLow<SRegister>()), 1705 FromLowSToD(rhs_loc.AsFpuRegisterPairLow<SRegister>())); 1706 } 1707 } 1708} 1709 1710void InstructionCodeGeneratorARM::GenerateFPJumps(HCondition* cond, 1711 Label* true_label, 1712 Label* false_label ATTRIBUTE_UNUSED) { 1713 __ vmstat(); // transfer FP status register to ARM APSR. 1714 __ b(true_label, ARMFPCondition(cond->GetCondition(), cond->IsGtBias())); 1715} 1716 1717void InstructionCodeGeneratorARM::GenerateLongComparesAndJumps(HCondition* cond, 1718 Label* true_label, 1719 Label* false_label) { 1720 LocationSummary* locations = cond->GetLocations(); 1721 Location left = locations->InAt(0); 1722 Location right = locations->InAt(1); 1723 IfCondition if_cond = cond->GetCondition(); 1724 1725 Register left_high = left.AsRegisterPairHigh<Register>(); 1726 Register left_low = left.AsRegisterPairLow<Register>(); 1727 IfCondition true_high_cond = if_cond; 1728 IfCondition false_high_cond = cond->GetOppositeCondition(); 1729 Condition final_condition = ARMUnsignedCondition(if_cond); // unsigned on lower part 1730 1731 // Set the conditions for the test, remembering that == needs to be 1732 // decided using the low words. 1733 // TODO: consider avoiding jumps with temporary and CMP low+SBC high 1734 switch (if_cond) { 1735 case kCondEQ: 1736 case kCondNE: 1737 // Nothing to do. 1738 break; 1739 case kCondLT: 1740 false_high_cond = kCondGT; 1741 break; 1742 case kCondLE: 1743 true_high_cond = kCondLT; 1744 break; 1745 case kCondGT: 1746 false_high_cond = kCondLT; 1747 break; 1748 case kCondGE: 1749 true_high_cond = kCondGT; 1750 break; 1751 case kCondB: 1752 false_high_cond = kCondA; 1753 break; 1754 case kCondBE: 1755 true_high_cond = kCondB; 1756 break; 1757 case kCondA: 1758 false_high_cond = kCondB; 1759 break; 1760 case kCondAE: 1761 true_high_cond = kCondA; 1762 break; 1763 } 1764 if (right.IsConstant()) { 1765 int64_t value = right.GetConstant()->AsLongConstant()->GetValue(); 1766 int32_t val_low = Low32Bits(value); 1767 int32_t val_high = High32Bits(value); 1768 1769 __ CmpConstant(left_high, val_high); 1770 if (if_cond == kCondNE) { 1771 __ b(true_label, ARMCondition(true_high_cond)); 1772 } else if (if_cond == kCondEQ) { 1773 __ b(false_label, ARMCondition(false_high_cond)); 1774 } else { 1775 __ b(true_label, ARMCondition(true_high_cond)); 1776 __ b(false_label, ARMCondition(false_high_cond)); 1777 } 1778 // Must be equal high, so compare the lows. 1779 __ CmpConstant(left_low, val_low); 1780 } else { 1781 Register right_high = right.AsRegisterPairHigh<Register>(); 1782 Register right_low = right.AsRegisterPairLow<Register>(); 1783 1784 __ cmp(left_high, ShifterOperand(right_high)); 1785 if (if_cond == kCondNE) { 1786 __ b(true_label, ARMCondition(true_high_cond)); 1787 } else if (if_cond == kCondEQ) { 1788 __ b(false_label, ARMCondition(false_high_cond)); 1789 } else { 1790 __ b(true_label, ARMCondition(true_high_cond)); 1791 __ b(false_label, ARMCondition(false_high_cond)); 1792 } 1793 // Must be equal high, so compare the lows. 1794 __ cmp(left_low, ShifterOperand(right_low)); 1795 } 1796 // The last comparison might be unsigned. 1797 // TODO: optimize cases where this is always true/false 1798 __ b(true_label, final_condition); 1799} 1800 1801void InstructionCodeGeneratorARM::GenerateCompareTestAndBranch(HCondition* condition, 1802 Label* true_target_in, 1803 Label* false_target_in) { 1804 // Generated branching requires both targets to be explicit. If either of the 1805 // targets is nullptr (fallthrough) use and bind `fallthrough_target` instead. 1806 Label fallthrough_target; 1807 Label* true_target = true_target_in == nullptr ? &fallthrough_target : true_target_in; 1808 Label* false_target = false_target_in == nullptr ? &fallthrough_target : false_target_in; 1809 1810 Primitive::Type type = condition->InputAt(0)->GetType(); 1811 switch (type) { 1812 case Primitive::kPrimLong: 1813 GenerateLongComparesAndJumps(condition, true_target, false_target); 1814 break; 1815 case Primitive::kPrimFloat: 1816 case Primitive::kPrimDouble: 1817 GenerateVcmp(condition); 1818 GenerateFPJumps(condition, true_target, false_target); 1819 break; 1820 default: 1821 LOG(FATAL) << "Unexpected compare type " << type; 1822 } 1823 1824 if (false_target != &fallthrough_target) { 1825 __ b(false_target); 1826 } 1827 1828 if (fallthrough_target.IsLinked()) { 1829 __ Bind(&fallthrough_target); 1830 } 1831} 1832 1833void InstructionCodeGeneratorARM::GenerateTestAndBranch(HInstruction* instruction, 1834 size_t condition_input_index, 1835 Label* true_target, 1836 Label* false_target) { 1837 HInstruction* cond = instruction->InputAt(condition_input_index); 1838 1839 if (true_target == nullptr && false_target == nullptr) { 1840 // Nothing to do. The code always falls through. 1841 return; 1842 } else if (cond->IsIntConstant()) { 1843 // Constant condition, statically compared against "true" (integer value 1). 1844 if (cond->AsIntConstant()->IsTrue()) { 1845 if (true_target != nullptr) { 1846 __ b(true_target); 1847 } 1848 } else { 1849 DCHECK(cond->AsIntConstant()->IsFalse()) << cond->AsIntConstant()->GetValue(); 1850 if (false_target != nullptr) { 1851 __ b(false_target); 1852 } 1853 } 1854 return; 1855 } 1856 1857 // The following code generates these patterns: 1858 // (1) true_target == nullptr && false_target != nullptr 1859 // - opposite condition true => branch to false_target 1860 // (2) true_target != nullptr && false_target == nullptr 1861 // - condition true => branch to true_target 1862 // (3) true_target != nullptr && false_target != nullptr 1863 // - condition true => branch to true_target 1864 // - branch to false_target 1865 if (IsBooleanValueOrMaterializedCondition(cond)) { 1866 // Condition has been materialized, compare the output to 0. 1867 Location cond_val = instruction->GetLocations()->InAt(condition_input_index); 1868 DCHECK(cond_val.IsRegister()); 1869 if (true_target == nullptr) { 1870 __ CompareAndBranchIfZero(cond_val.AsRegister<Register>(), false_target); 1871 } else { 1872 __ CompareAndBranchIfNonZero(cond_val.AsRegister<Register>(), true_target); 1873 } 1874 } else { 1875 // Condition has not been materialized. Use its inputs as the comparison and 1876 // its condition as the branch condition. 1877 HCondition* condition = cond->AsCondition(); 1878 1879 // If this is a long or FP comparison that has been folded into 1880 // the HCondition, generate the comparison directly. 1881 Primitive::Type type = condition->InputAt(0)->GetType(); 1882 if (type == Primitive::kPrimLong || Primitive::IsFloatingPointType(type)) { 1883 GenerateCompareTestAndBranch(condition, true_target, false_target); 1884 return; 1885 } 1886 1887 LocationSummary* locations = cond->GetLocations(); 1888 DCHECK(locations->InAt(0).IsRegister()); 1889 Register left = locations->InAt(0).AsRegister<Register>(); 1890 Location right = locations->InAt(1); 1891 if (right.IsRegister()) { 1892 __ cmp(left, ShifterOperand(right.AsRegister<Register>())); 1893 } else { 1894 DCHECK(right.IsConstant()); 1895 __ CmpConstant(left, CodeGenerator::GetInt32ValueOf(right.GetConstant())); 1896 } 1897 if (true_target == nullptr) { 1898 __ b(false_target, ARMCondition(condition->GetOppositeCondition())); 1899 } else { 1900 __ b(true_target, ARMCondition(condition->GetCondition())); 1901 } 1902 } 1903 1904 // If neither branch falls through (case 3), the conditional branch to `true_target` 1905 // was already emitted (case 2) and we need to emit a jump to `false_target`. 1906 if (true_target != nullptr && false_target != nullptr) { 1907 __ b(false_target); 1908 } 1909} 1910 1911void LocationsBuilderARM::VisitIf(HIf* if_instr) { 1912 LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(if_instr); 1913 if (IsBooleanValueOrMaterializedCondition(if_instr->InputAt(0))) { 1914 locations->SetInAt(0, Location::RequiresRegister()); 1915 } 1916} 1917 1918void InstructionCodeGeneratorARM::VisitIf(HIf* if_instr) { 1919 HBasicBlock* true_successor = if_instr->IfTrueSuccessor(); 1920 HBasicBlock* false_successor = if_instr->IfFalseSuccessor(); 1921 Label* true_target = codegen_->GoesToNextBlock(if_instr->GetBlock(), true_successor) ? 1922 nullptr : codegen_->GetLabelOf(true_successor); 1923 Label* false_target = codegen_->GoesToNextBlock(if_instr->GetBlock(), false_successor) ? 1924 nullptr : codegen_->GetLabelOf(false_successor); 1925 GenerateTestAndBranch(if_instr, /* condition_input_index */ 0, true_target, false_target); 1926} 1927 1928void LocationsBuilderARM::VisitDeoptimize(HDeoptimize* deoptimize) { 1929 LocationSummary* locations = new (GetGraph()->GetArena()) 1930 LocationSummary(deoptimize, LocationSummary::kCallOnSlowPath); 1931 locations->SetCustomSlowPathCallerSaves(RegisterSet::Empty()); // No caller-save registers. 1932 if (IsBooleanValueOrMaterializedCondition(deoptimize->InputAt(0))) { 1933 locations->SetInAt(0, Location::RequiresRegister()); 1934 } 1935} 1936 1937void InstructionCodeGeneratorARM::VisitDeoptimize(HDeoptimize* deoptimize) { 1938 SlowPathCodeARM* slow_path = deopt_slow_paths_.NewSlowPath<DeoptimizationSlowPathARM>(deoptimize); 1939 GenerateTestAndBranch(deoptimize, 1940 /* condition_input_index */ 0, 1941 slow_path->GetEntryLabel(), 1942 /* false_target */ nullptr); 1943} 1944 1945void LocationsBuilderARM::VisitSelect(HSelect* select) { 1946 LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(select); 1947 if (Primitive::IsFloatingPointType(select->GetType())) { 1948 locations->SetInAt(0, Location::RequiresFpuRegister()); 1949 locations->SetInAt(1, Location::RequiresFpuRegister()); 1950 } else { 1951 locations->SetInAt(0, Location::RequiresRegister()); 1952 locations->SetInAt(1, Location::RequiresRegister()); 1953 } 1954 if (IsBooleanValueOrMaterializedCondition(select->GetCondition())) { 1955 locations->SetInAt(2, Location::RequiresRegister()); 1956 } 1957 locations->SetOut(Location::SameAsFirstInput()); 1958} 1959 1960void InstructionCodeGeneratorARM::VisitSelect(HSelect* select) { 1961 LocationSummary* locations = select->GetLocations(); 1962 Label false_target; 1963 GenerateTestAndBranch(select, 1964 /* condition_input_index */ 2, 1965 /* true_target */ nullptr, 1966 &false_target); 1967 codegen_->MoveLocation(locations->Out(), locations->InAt(1), select->GetType()); 1968 __ Bind(&false_target); 1969} 1970 1971void LocationsBuilderARM::VisitNativeDebugInfo(HNativeDebugInfo* info) { 1972 new (GetGraph()->GetArena()) LocationSummary(info); 1973} 1974 1975void InstructionCodeGeneratorARM::VisitNativeDebugInfo(HNativeDebugInfo*) { 1976 // MaybeRecordNativeDebugInfo is already called implicitly in CodeGenerator::Compile. 1977} 1978 1979void CodeGeneratorARM::GenerateNop() { 1980 __ nop(); 1981} 1982 1983void LocationsBuilderARM::HandleCondition(HCondition* cond) { 1984 LocationSummary* locations = 1985 new (GetGraph()->GetArena()) LocationSummary(cond, LocationSummary::kNoCall); 1986 // Handle the long/FP comparisons made in instruction simplification. 1987 switch (cond->InputAt(0)->GetType()) { 1988 case Primitive::kPrimLong: 1989 locations->SetInAt(0, Location::RequiresRegister()); 1990 locations->SetInAt(1, Location::RegisterOrConstant(cond->InputAt(1))); 1991 if (!cond->IsEmittedAtUseSite()) { 1992 locations->SetOut(Location::RequiresRegister(), Location::kOutputOverlap); 1993 } 1994 break; 1995 1996 case Primitive::kPrimFloat: 1997 case Primitive::kPrimDouble: 1998 locations->SetInAt(0, Location::RequiresFpuRegister()); 1999 locations->SetInAt(1, ArithmeticZeroOrFpuRegister(cond->InputAt(1))); 2000 if (!cond->IsEmittedAtUseSite()) { 2001 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 2002 } 2003 break; 2004 2005 default: 2006 locations->SetInAt(0, Location::RequiresRegister()); 2007 locations->SetInAt(1, Location::RegisterOrConstant(cond->InputAt(1))); 2008 if (!cond->IsEmittedAtUseSite()) { 2009 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 2010 } 2011 } 2012} 2013 2014void InstructionCodeGeneratorARM::HandleCondition(HCondition* cond) { 2015 if (cond->IsEmittedAtUseSite()) { 2016 return; 2017 } 2018 2019 LocationSummary* locations = cond->GetLocations(); 2020 Location left = locations->InAt(0); 2021 Location right = locations->InAt(1); 2022 Register out = locations->Out().AsRegister<Register>(); 2023 Label true_label, false_label; 2024 2025 switch (cond->InputAt(0)->GetType()) { 2026 default: { 2027 // Integer case. 2028 if (right.IsRegister()) { 2029 __ cmp(left.AsRegister<Register>(), ShifterOperand(right.AsRegister<Register>())); 2030 } else { 2031 DCHECK(right.IsConstant()); 2032 __ CmpConstant(left.AsRegister<Register>(), 2033 CodeGenerator::GetInt32ValueOf(right.GetConstant())); 2034 } 2035 __ it(ARMCondition(cond->GetCondition()), kItElse); 2036 __ mov(locations->Out().AsRegister<Register>(), ShifterOperand(1), 2037 ARMCondition(cond->GetCondition())); 2038 __ mov(locations->Out().AsRegister<Register>(), ShifterOperand(0), 2039 ARMCondition(cond->GetOppositeCondition())); 2040 return; 2041 } 2042 case Primitive::kPrimLong: 2043 GenerateLongComparesAndJumps(cond, &true_label, &false_label); 2044 break; 2045 case Primitive::kPrimFloat: 2046 case Primitive::kPrimDouble: 2047 GenerateVcmp(cond); 2048 GenerateFPJumps(cond, &true_label, &false_label); 2049 break; 2050 } 2051 2052 // Convert the jumps into the result. 2053 Label done_label; 2054 2055 // False case: result = 0. 2056 __ Bind(&false_label); 2057 __ LoadImmediate(out, 0); 2058 __ b(&done_label); 2059 2060 // True case: result = 1. 2061 __ Bind(&true_label); 2062 __ LoadImmediate(out, 1); 2063 __ Bind(&done_label); 2064} 2065 2066void LocationsBuilderARM::VisitEqual(HEqual* comp) { 2067 HandleCondition(comp); 2068} 2069 2070void InstructionCodeGeneratorARM::VisitEqual(HEqual* comp) { 2071 HandleCondition(comp); 2072} 2073 2074void LocationsBuilderARM::VisitNotEqual(HNotEqual* comp) { 2075 HandleCondition(comp); 2076} 2077 2078void InstructionCodeGeneratorARM::VisitNotEqual(HNotEqual* comp) { 2079 HandleCondition(comp); 2080} 2081 2082void LocationsBuilderARM::VisitLessThan(HLessThan* comp) { 2083 HandleCondition(comp); 2084} 2085 2086void InstructionCodeGeneratorARM::VisitLessThan(HLessThan* comp) { 2087 HandleCondition(comp); 2088} 2089 2090void LocationsBuilderARM::VisitLessThanOrEqual(HLessThanOrEqual* comp) { 2091 HandleCondition(comp); 2092} 2093 2094void InstructionCodeGeneratorARM::VisitLessThanOrEqual(HLessThanOrEqual* comp) { 2095 HandleCondition(comp); 2096} 2097 2098void LocationsBuilderARM::VisitGreaterThan(HGreaterThan* comp) { 2099 HandleCondition(comp); 2100} 2101 2102void InstructionCodeGeneratorARM::VisitGreaterThan(HGreaterThan* comp) { 2103 HandleCondition(comp); 2104} 2105 2106void LocationsBuilderARM::VisitGreaterThanOrEqual(HGreaterThanOrEqual* comp) { 2107 HandleCondition(comp); 2108} 2109 2110void InstructionCodeGeneratorARM::VisitGreaterThanOrEqual(HGreaterThanOrEqual* comp) { 2111 HandleCondition(comp); 2112} 2113 2114void LocationsBuilderARM::VisitBelow(HBelow* comp) { 2115 HandleCondition(comp); 2116} 2117 2118void InstructionCodeGeneratorARM::VisitBelow(HBelow* comp) { 2119 HandleCondition(comp); 2120} 2121 2122void LocationsBuilderARM::VisitBelowOrEqual(HBelowOrEqual* comp) { 2123 HandleCondition(comp); 2124} 2125 2126void InstructionCodeGeneratorARM::VisitBelowOrEqual(HBelowOrEqual* comp) { 2127 HandleCondition(comp); 2128} 2129 2130void LocationsBuilderARM::VisitAbove(HAbove* comp) { 2131 HandleCondition(comp); 2132} 2133 2134void InstructionCodeGeneratorARM::VisitAbove(HAbove* comp) { 2135 HandleCondition(comp); 2136} 2137 2138void LocationsBuilderARM::VisitAboveOrEqual(HAboveOrEqual* comp) { 2139 HandleCondition(comp); 2140} 2141 2142void InstructionCodeGeneratorARM::VisitAboveOrEqual(HAboveOrEqual* comp) { 2143 HandleCondition(comp); 2144} 2145 2146void LocationsBuilderARM::VisitIntConstant(HIntConstant* constant) { 2147 LocationSummary* locations = 2148 new (GetGraph()->GetArena()) LocationSummary(constant, LocationSummary::kNoCall); 2149 locations->SetOut(Location::ConstantLocation(constant)); 2150} 2151 2152void InstructionCodeGeneratorARM::VisitIntConstant(HIntConstant* constant ATTRIBUTE_UNUSED) { 2153 // Will be generated at use site. 2154} 2155 2156void LocationsBuilderARM::VisitNullConstant(HNullConstant* constant) { 2157 LocationSummary* locations = 2158 new (GetGraph()->GetArena()) LocationSummary(constant, LocationSummary::kNoCall); 2159 locations->SetOut(Location::ConstantLocation(constant)); 2160} 2161 2162void InstructionCodeGeneratorARM::VisitNullConstant(HNullConstant* constant ATTRIBUTE_UNUSED) { 2163 // Will be generated at use site. 2164} 2165 2166void LocationsBuilderARM::VisitLongConstant(HLongConstant* constant) { 2167 LocationSummary* locations = 2168 new (GetGraph()->GetArena()) LocationSummary(constant, LocationSummary::kNoCall); 2169 locations->SetOut(Location::ConstantLocation(constant)); 2170} 2171 2172void InstructionCodeGeneratorARM::VisitLongConstant(HLongConstant* constant ATTRIBUTE_UNUSED) { 2173 // Will be generated at use site. 2174} 2175 2176void LocationsBuilderARM::VisitFloatConstant(HFloatConstant* constant) { 2177 LocationSummary* locations = 2178 new (GetGraph()->GetArena()) LocationSummary(constant, LocationSummary::kNoCall); 2179 locations->SetOut(Location::ConstantLocation(constant)); 2180} 2181 2182void InstructionCodeGeneratorARM::VisitFloatConstant(HFloatConstant* constant ATTRIBUTE_UNUSED) { 2183 // Will be generated at use site. 2184} 2185 2186void LocationsBuilderARM::VisitDoubleConstant(HDoubleConstant* constant) { 2187 LocationSummary* locations = 2188 new (GetGraph()->GetArena()) LocationSummary(constant, LocationSummary::kNoCall); 2189 locations->SetOut(Location::ConstantLocation(constant)); 2190} 2191 2192void InstructionCodeGeneratorARM::VisitDoubleConstant(HDoubleConstant* constant ATTRIBUTE_UNUSED) { 2193 // Will be generated at use site. 2194} 2195 2196void LocationsBuilderARM::VisitMemoryBarrier(HMemoryBarrier* memory_barrier) { 2197 memory_barrier->SetLocations(nullptr); 2198} 2199 2200void InstructionCodeGeneratorARM::VisitMemoryBarrier(HMemoryBarrier* memory_barrier) { 2201 codegen_->GenerateMemoryBarrier(memory_barrier->GetBarrierKind()); 2202} 2203 2204void LocationsBuilderARM::VisitReturnVoid(HReturnVoid* ret) { 2205 ret->SetLocations(nullptr); 2206} 2207 2208void InstructionCodeGeneratorARM::VisitReturnVoid(HReturnVoid* ret ATTRIBUTE_UNUSED) { 2209 codegen_->GenerateFrameExit(); 2210} 2211 2212void LocationsBuilderARM::VisitReturn(HReturn* ret) { 2213 LocationSummary* locations = 2214 new (GetGraph()->GetArena()) LocationSummary(ret, LocationSummary::kNoCall); 2215 locations->SetInAt(0, parameter_visitor_.GetReturnLocation(ret->InputAt(0)->GetType())); 2216} 2217 2218void InstructionCodeGeneratorARM::VisitReturn(HReturn* ret ATTRIBUTE_UNUSED) { 2219 codegen_->GenerateFrameExit(); 2220} 2221 2222void LocationsBuilderARM::VisitInvokeUnresolved(HInvokeUnresolved* invoke) { 2223 // The trampoline uses the same calling convention as dex calling conventions, 2224 // except instead of loading arg0/r0 with the target Method*, arg0/r0 will contain 2225 // the method_idx. 2226 HandleInvoke(invoke); 2227} 2228 2229void InstructionCodeGeneratorARM::VisitInvokeUnresolved(HInvokeUnresolved* invoke) { 2230 codegen_->GenerateInvokeUnresolvedRuntimeCall(invoke); 2231} 2232 2233void LocationsBuilderARM::VisitInvokeStaticOrDirect(HInvokeStaticOrDirect* invoke) { 2234 // Explicit clinit checks triggered by static invokes must have been pruned by 2235 // art::PrepareForRegisterAllocation. 2236 DCHECK(!invoke->IsStaticWithExplicitClinitCheck()); 2237 2238 IntrinsicLocationsBuilderARM intrinsic(codegen_); 2239 if (intrinsic.TryDispatch(invoke)) { 2240 if (invoke->GetLocations()->CanCall() && invoke->HasPcRelativeDexCache()) { 2241 invoke->GetLocations()->SetInAt(invoke->GetSpecialInputIndex(), Location::Any()); 2242 } 2243 return; 2244 } 2245 2246 HandleInvoke(invoke); 2247 2248 // For PC-relative dex cache the invoke has an extra input, the PC-relative address base. 2249 if (invoke->HasPcRelativeDexCache()) { 2250 invoke->GetLocations()->SetInAt(invoke->GetSpecialInputIndex(), Location::RequiresRegister()); 2251 } 2252} 2253 2254static bool TryGenerateIntrinsicCode(HInvoke* invoke, CodeGeneratorARM* codegen) { 2255 if (invoke->GetLocations()->Intrinsified()) { 2256 IntrinsicCodeGeneratorARM intrinsic(codegen); 2257 intrinsic.Dispatch(invoke); 2258 return true; 2259 } 2260 return false; 2261} 2262 2263void InstructionCodeGeneratorARM::VisitInvokeStaticOrDirect(HInvokeStaticOrDirect* invoke) { 2264 // Explicit clinit checks triggered by static invokes must have been pruned by 2265 // art::PrepareForRegisterAllocation. 2266 DCHECK(!invoke->IsStaticWithExplicitClinitCheck()); 2267 2268 if (TryGenerateIntrinsicCode(invoke, codegen_)) { 2269 return; 2270 } 2271 2272 LocationSummary* locations = invoke->GetLocations(); 2273 codegen_->GenerateStaticOrDirectCall( 2274 invoke, locations->HasTemps() ? locations->GetTemp(0) : Location::NoLocation()); 2275 codegen_->RecordPcInfo(invoke, invoke->GetDexPc()); 2276} 2277 2278void LocationsBuilderARM::HandleInvoke(HInvoke* invoke) { 2279 InvokeDexCallingConventionVisitorARM calling_convention_visitor; 2280 CodeGenerator::CreateCommonInvokeLocationSummary(invoke, &calling_convention_visitor); 2281} 2282 2283void LocationsBuilderARM::VisitInvokeVirtual(HInvokeVirtual* invoke) { 2284 IntrinsicLocationsBuilderARM intrinsic(codegen_); 2285 if (intrinsic.TryDispatch(invoke)) { 2286 return; 2287 } 2288 2289 HandleInvoke(invoke); 2290} 2291 2292void InstructionCodeGeneratorARM::VisitInvokeVirtual(HInvokeVirtual* invoke) { 2293 if (TryGenerateIntrinsicCode(invoke, codegen_)) { 2294 return; 2295 } 2296 2297 codegen_->GenerateVirtualCall(invoke, invoke->GetLocations()->GetTemp(0)); 2298 DCHECK(!codegen_->IsLeafMethod()); 2299 codegen_->RecordPcInfo(invoke, invoke->GetDexPc()); 2300} 2301 2302void LocationsBuilderARM::VisitInvokeInterface(HInvokeInterface* invoke) { 2303 HandleInvoke(invoke); 2304 // Add the hidden argument. 2305 invoke->GetLocations()->AddTemp(Location::RegisterLocation(R12)); 2306} 2307 2308void InstructionCodeGeneratorARM::VisitInvokeInterface(HInvokeInterface* invoke) { 2309 // TODO: b/18116999, our IMTs can miss an IncompatibleClassChangeError. 2310 LocationSummary* locations = invoke->GetLocations(); 2311 Register temp = locations->GetTemp(0).AsRegister<Register>(); 2312 Register hidden_reg = locations->GetTemp(1).AsRegister<Register>(); 2313 Location receiver = locations->InAt(0); 2314 uint32_t class_offset = mirror::Object::ClassOffset().Int32Value(); 2315 2316 // Set the hidden argument. This is safe to do this here, as R12 2317 // won't be modified thereafter, before the `blx` (call) instruction. 2318 DCHECK_EQ(R12, hidden_reg); 2319 __ LoadImmediate(hidden_reg, invoke->GetDexMethodIndex()); 2320 2321 if (receiver.IsStackSlot()) { 2322 __ LoadFromOffset(kLoadWord, temp, SP, receiver.GetStackIndex()); 2323 // /* HeapReference<Class> */ temp = temp->klass_ 2324 __ LoadFromOffset(kLoadWord, temp, temp, class_offset); 2325 } else { 2326 // /* HeapReference<Class> */ temp = receiver->klass_ 2327 __ LoadFromOffset(kLoadWord, temp, receiver.AsRegister<Register>(), class_offset); 2328 } 2329 codegen_->MaybeRecordImplicitNullCheck(invoke); 2330 // Instead of simply (possibly) unpoisoning `temp` here, we should 2331 // emit a read barrier for the previous class reference load. 2332 // However this is not required in practice, as this is an 2333 // intermediate/temporary reference and because the current 2334 // concurrent copying collector keeps the from-space memory 2335 // intact/accessible until the end of the marking phase (the 2336 // concurrent copying collector may not in the future). 2337 __ MaybeUnpoisonHeapReference(temp); 2338 __ LoadFromOffset(kLoadWord, temp, temp, 2339 mirror::Class::ImtPtrOffset(kArmPointerSize).Uint32Value()); 2340 uint32_t method_offset = static_cast<uint32_t>(ImTable::OffsetOfElement( 2341 invoke->GetImtIndex(), kArmPointerSize)); 2342 // temp = temp->GetImtEntryAt(method_offset); 2343 __ LoadFromOffset(kLoadWord, temp, temp, method_offset); 2344 uint32_t entry_point = 2345 ArtMethod::EntryPointFromQuickCompiledCodeOffset(kArmPointerSize).Int32Value(); 2346 // LR = temp->GetEntryPoint(); 2347 __ LoadFromOffset(kLoadWord, LR, temp, entry_point); 2348 // LR(); 2349 __ blx(LR); 2350 DCHECK(!codegen_->IsLeafMethod()); 2351 codegen_->RecordPcInfo(invoke, invoke->GetDexPc()); 2352} 2353 2354void LocationsBuilderARM::VisitNeg(HNeg* neg) { 2355 LocationSummary* locations = 2356 new (GetGraph()->GetArena()) LocationSummary(neg, LocationSummary::kNoCall); 2357 switch (neg->GetResultType()) { 2358 case Primitive::kPrimInt: { 2359 locations->SetInAt(0, Location::RequiresRegister()); 2360 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 2361 break; 2362 } 2363 case Primitive::kPrimLong: { 2364 locations->SetInAt(0, Location::RequiresRegister()); 2365 locations->SetOut(Location::RequiresRegister(), Location::kOutputOverlap); 2366 break; 2367 } 2368 2369 case Primitive::kPrimFloat: 2370 case Primitive::kPrimDouble: 2371 locations->SetInAt(0, Location::RequiresFpuRegister()); 2372 locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap); 2373 break; 2374 2375 default: 2376 LOG(FATAL) << "Unexpected neg type " << neg->GetResultType(); 2377 } 2378} 2379 2380void InstructionCodeGeneratorARM::VisitNeg(HNeg* neg) { 2381 LocationSummary* locations = neg->GetLocations(); 2382 Location out = locations->Out(); 2383 Location in = locations->InAt(0); 2384 switch (neg->GetResultType()) { 2385 case Primitive::kPrimInt: 2386 DCHECK(in.IsRegister()); 2387 __ rsb(out.AsRegister<Register>(), in.AsRegister<Register>(), ShifterOperand(0)); 2388 break; 2389 2390 case Primitive::kPrimLong: 2391 DCHECK(in.IsRegisterPair()); 2392 // out.lo = 0 - in.lo (and update the carry/borrow (C) flag) 2393 __ rsbs(out.AsRegisterPairLow<Register>(), 2394 in.AsRegisterPairLow<Register>(), 2395 ShifterOperand(0)); 2396 // We cannot emit an RSC (Reverse Subtract with Carry) 2397 // instruction here, as it does not exist in the Thumb-2 2398 // instruction set. We use the following approach 2399 // using SBC and SUB instead. 2400 // 2401 // out.hi = -C 2402 __ sbc(out.AsRegisterPairHigh<Register>(), 2403 out.AsRegisterPairHigh<Register>(), 2404 ShifterOperand(out.AsRegisterPairHigh<Register>())); 2405 // out.hi = out.hi - in.hi 2406 __ sub(out.AsRegisterPairHigh<Register>(), 2407 out.AsRegisterPairHigh<Register>(), 2408 ShifterOperand(in.AsRegisterPairHigh<Register>())); 2409 break; 2410 2411 case Primitive::kPrimFloat: 2412 DCHECK(in.IsFpuRegister()); 2413 __ vnegs(out.AsFpuRegister<SRegister>(), in.AsFpuRegister<SRegister>()); 2414 break; 2415 2416 case Primitive::kPrimDouble: 2417 DCHECK(in.IsFpuRegisterPair()); 2418 __ vnegd(FromLowSToD(out.AsFpuRegisterPairLow<SRegister>()), 2419 FromLowSToD(in.AsFpuRegisterPairLow<SRegister>())); 2420 break; 2421 2422 default: 2423 LOG(FATAL) << "Unexpected neg type " << neg->GetResultType(); 2424 } 2425} 2426 2427void LocationsBuilderARM::VisitTypeConversion(HTypeConversion* conversion) { 2428 Primitive::Type result_type = conversion->GetResultType(); 2429 Primitive::Type input_type = conversion->GetInputType(); 2430 DCHECK_NE(result_type, input_type); 2431 2432 // The float-to-long, double-to-long and long-to-float type conversions 2433 // rely on a call to the runtime. 2434 LocationSummary::CallKind call_kind = 2435 (((input_type == Primitive::kPrimFloat || input_type == Primitive::kPrimDouble) 2436 && result_type == Primitive::kPrimLong) 2437 || (input_type == Primitive::kPrimLong && result_type == Primitive::kPrimFloat)) 2438 ? LocationSummary::kCallOnMainOnly 2439 : LocationSummary::kNoCall; 2440 LocationSummary* locations = 2441 new (GetGraph()->GetArena()) LocationSummary(conversion, call_kind); 2442 2443 // The Java language does not allow treating boolean as an integral type but 2444 // our bit representation makes it safe. 2445 2446 switch (result_type) { 2447 case Primitive::kPrimByte: 2448 switch (input_type) { 2449 case Primitive::kPrimLong: 2450 // Type conversion from long to byte is a result of code transformations. 2451 case Primitive::kPrimBoolean: 2452 // Boolean input is a result of code transformations. 2453 case Primitive::kPrimShort: 2454 case Primitive::kPrimInt: 2455 case Primitive::kPrimChar: 2456 // Processing a Dex `int-to-byte' instruction. 2457 locations->SetInAt(0, Location::RequiresRegister()); 2458 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 2459 break; 2460 2461 default: 2462 LOG(FATAL) << "Unexpected type conversion from " << input_type 2463 << " to " << result_type; 2464 } 2465 break; 2466 2467 case Primitive::kPrimShort: 2468 switch (input_type) { 2469 case Primitive::kPrimLong: 2470 // Type conversion from long to short is a result of code transformations. 2471 case Primitive::kPrimBoolean: 2472 // Boolean input is a result of code transformations. 2473 case Primitive::kPrimByte: 2474 case Primitive::kPrimInt: 2475 case Primitive::kPrimChar: 2476 // Processing a Dex `int-to-short' instruction. 2477 locations->SetInAt(0, Location::RequiresRegister()); 2478 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 2479 break; 2480 2481 default: 2482 LOG(FATAL) << "Unexpected type conversion from " << input_type 2483 << " to " << result_type; 2484 } 2485 break; 2486 2487 case Primitive::kPrimInt: 2488 switch (input_type) { 2489 case Primitive::kPrimLong: 2490 // Processing a Dex `long-to-int' instruction. 2491 locations->SetInAt(0, Location::Any()); 2492 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 2493 break; 2494 2495 case Primitive::kPrimFloat: 2496 // Processing a Dex `float-to-int' instruction. 2497 locations->SetInAt(0, Location::RequiresFpuRegister()); 2498 locations->SetOut(Location::RequiresRegister()); 2499 locations->AddTemp(Location::RequiresFpuRegister()); 2500 break; 2501 2502 case Primitive::kPrimDouble: 2503 // Processing a Dex `double-to-int' instruction. 2504 locations->SetInAt(0, Location::RequiresFpuRegister()); 2505 locations->SetOut(Location::RequiresRegister()); 2506 locations->AddTemp(Location::RequiresFpuRegister()); 2507 break; 2508 2509 default: 2510 LOG(FATAL) << "Unexpected type conversion from " << input_type 2511 << " to " << result_type; 2512 } 2513 break; 2514 2515 case Primitive::kPrimLong: 2516 switch (input_type) { 2517 case Primitive::kPrimBoolean: 2518 // Boolean input is a result of code transformations. 2519 case Primitive::kPrimByte: 2520 case Primitive::kPrimShort: 2521 case Primitive::kPrimInt: 2522 case Primitive::kPrimChar: 2523 // Processing a Dex `int-to-long' instruction. 2524 locations->SetInAt(0, Location::RequiresRegister()); 2525 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 2526 break; 2527 2528 case Primitive::kPrimFloat: { 2529 // Processing a Dex `float-to-long' instruction. 2530 InvokeRuntimeCallingConvention calling_convention; 2531 locations->SetInAt(0, Location::FpuRegisterLocation( 2532 calling_convention.GetFpuRegisterAt(0))); 2533 locations->SetOut(Location::RegisterPairLocation(R0, R1)); 2534 break; 2535 } 2536 2537 case Primitive::kPrimDouble: { 2538 // Processing a Dex `double-to-long' instruction. 2539 InvokeRuntimeCallingConvention calling_convention; 2540 locations->SetInAt(0, Location::FpuRegisterPairLocation( 2541 calling_convention.GetFpuRegisterAt(0), 2542 calling_convention.GetFpuRegisterAt(1))); 2543 locations->SetOut(Location::RegisterPairLocation(R0, R1)); 2544 break; 2545 } 2546 2547 default: 2548 LOG(FATAL) << "Unexpected type conversion from " << input_type 2549 << " to " << result_type; 2550 } 2551 break; 2552 2553 case Primitive::kPrimChar: 2554 switch (input_type) { 2555 case Primitive::kPrimLong: 2556 // Type conversion from long to char is a result of code transformations. 2557 case Primitive::kPrimBoolean: 2558 // Boolean input is a result of code transformations. 2559 case Primitive::kPrimByte: 2560 case Primitive::kPrimShort: 2561 case Primitive::kPrimInt: 2562 // Processing a Dex `int-to-char' instruction. 2563 locations->SetInAt(0, Location::RequiresRegister()); 2564 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 2565 break; 2566 2567 default: 2568 LOG(FATAL) << "Unexpected type conversion from " << input_type 2569 << " to " << result_type; 2570 } 2571 break; 2572 2573 case Primitive::kPrimFloat: 2574 switch (input_type) { 2575 case Primitive::kPrimBoolean: 2576 // Boolean input is a result of code transformations. 2577 case Primitive::kPrimByte: 2578 case Primitive::kPrimShort: 2579 case Primitive::kPrimInt: 2580 case Primitive::kPrimChar: 2581 // Processing a Dex `int-to-float' instruction. 2582 locations->SetInAt(0, Location::RequiresRegister()); 2583 locations->SetOut(Location::RequiresFpuRegister()); 2584 break; 2585 2586 case Primitive::kPrimLong: { 2587 // Processing a Dex `long-to-float' instruction. 2588 InvokeRuntimeCallingConvention calling_convention; 2589 locations->SetInAt(0, Location::RegisterPairLocation( 2590 calling_convention.GetRegisterAt(0), calling_convention.GetRegisterAt(1))); 2591 locations->SetOut(Location::FpuRegisterLocation(calling_convention.GetFpuRegisterAt(0))); 2592 break; 2593 } 2594 2595 case Primitive::kPrimDouble: 2596 // Processing a Dex `double-to-float' instruction. 2597 locations->SetInAt(0, Location::RequiresFpuRegister()); 2598 locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap); 2599 break; 2600 2601 default: 2602 LOG(FATAL) << "Unexpected type conversion from " << input_type 2603 << " to " << result_type; 2604 }; 2605 break; 2606 2607 case Primitive::kPrimDouble: 2608 switch (input_type) { 2609 case Primitive::kPrimBoolean: 2610 // Boolean input is a result of code transformations. 2611 case Primitive::kPrimByte: 2612 case Primitive::kPrimShort: 2613 case Primitive::kPrimInt: 2614 case Primitive::kPrimChar: 2615 // Processing a Dex `int-to-double' instruction. 2616 locations->SetInAt(0, Location::RequiresRegister()); 2617 locations->SetOut(Location::RequiresFpuRegister()); 2618 break; 2619 2620 case Primitive::kPrimLong: 2621 // Processing a Dex `long-to-double' instruction. 2622 locations->SetInAt(0, Location::RequiresRegister()); 2623 locations->SetOut(Location::RequiresFpuRegister()); 2624 locations->AddTemp(Location::RequiresFpuRegister()); 2625 locations->AddTemp(Location::RequiresFpuRegister()); 2626 break; 2627 2628 case Primitive::kPrimFloat: 2629 // Processing a Dex `float-to-double' instruction. 2630 locations->SetInAt(0, Location::RequiresFpuRegister()); 2631 locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap); 2632 break; 2633 2634 default: 2635 LOG(FATAL) << "Unexpected type conversion from " << input_type 2636 << " to " << result_type; 2637 }; 2638 break; 2639 2640 default: 2641 LOG(FATAL) << "Unexpected type conversion from " << input_type 2642 << " to " << result_type; 2643 } 2644} 2645 2646void InstructionCodeGeneratorARM::VisitTypeConversion(HTypeConversion* conversion) { 2647 LocationSummary* locations = conversion->GetLocations(); 2648 Location out = locations->Out(); 2649 Location in = locations->InAt(0); 2650 Primitive::Type result_type = conversion->GetResultType(); 2651 Primitive::Type input_type = conversion->GetInputType(); 2652 DCHECK_NE(result_type, input_type); 2653 switch (result_type) { 2654 case Primitive::kPrimByte: 2655 switch (input_type) { 2656 case Primitive::kPrimLong: 2657 // Type conversion from long to byte is a result of code transformations. 2658 __ sbfx(out.AsRegister<Register>(), in.AsRegisterPairLow<Register>(), 0, 8); 2659 break; 2660 case Primitive::kPrimBoolean: 2661 // Boolean input is a result of code transformations. 2662 case Primitive::kPrimShort: 2663 case Primitive::kPrimInt: 2664 case Primitive::kPrimChar: 2665 // Processing a Dex `int-to-byte' instruction. 2666 __ sbfx(out.AsRegister<Register>(), in.AsRegister<Register>(), 0, 8); 2667 break; 2668 2669 default: 2670 LOG(FATAL) << "Unexpected type conversion from " << input_type 2671 << " to " << result_type; 2672 } 2673 break; 2674 2675 case Primitive::kPrimShort: 2676 switch (input_type) { 2677 case Primitive::kPrimLong: 2678 // Type conversion from long to short is a result of code transformations. 2679 __ sbfx(out.AsRegister<Register>(), in.AsRegisterPairLow<Register>(), 0, 16); 2680 break; 2681 case Primitive::kPrimBoolean: 2682 // Boolean input is a result of code transformations. 2683 case Primitive::kPrimByte: 2684 case Primitive::kPrimInt: 2685 case Primitive::kPrimChar: 2686 // Processing a Dex `int-to-short' instruction. 2687 __ sbfx(out.AsRegister<Register>(), in.AsRegister<Register>(), 0, 16); 2688 break; 2689 2690 default: 2691 LOG(FATAL) << "Unexpected type conversion from " << input_type 2692 << " to " << result_type; 2693 } 2694 break; 2695 2696 case Primitive::kPrimInt: 2697 switch (input_type) { 2698 case Primitive::kPrimLong: 2699 // Processing a Dex `long-to-int' instruction. 2700 DCHECK(out.IsRegister()); 2701 if (in.IsRegisterPair()) { 2702 __ Mov(out.AsRegister<Register>(), in.AsRegisterPairLow<Register>()); 2703 } else if (in.IsDoubleStackSlot()) { 2704 __ LoadFromOffset(kLoadWord, out.AsRegister<Register>(), SP, in.GetStackIndex()); 2705 } else { 2706 DCHECK(in.IsConstant()); 2707 DCHECK(in.GetConstant()->IsLongConstant()); 2708 int64_t value = in.GetConstant()->AsLongConstant()->GetValue(); 2709 __ LoadImmediate(out.AsRegister<Register>(), static_cast<int32_t>(value)); 2710 } 2711 break; 2712 2713 case Primitive::kPrimFloat: { 2714 // Processing a Dex `float-to-int' instruction. 2715 SRegister temp = locations->GetTemp(0).AsFpuRegisterPairLow<SRegister>(); 2716 __ vcvtis(temp, in.AsFpuRegister<SRegister>()); 2717 __ vmovrs(out.AsRegister<Register>(), temp); 2718 break; 2719 } 2720 2721 case Primitive::kPrimDouble: { 2722 // Processing a Dex `double-to-int' instruction. 2723 SRegister temp_s = locations->GetTemp(0).AsFpuRegisterPairLow<SRegister>(); 2724 __ vcvtid(temp_s, FromLowSToD(in.AsFpuRegisterPairLow<SRegister>())); 2725 __ vmovrs(out.AsRegister<Register>(), temp_s); 2726 break; 2727 } 2728 2729 default: 2730 LOG(FATAL) << "Unexpected type conversion from " << input_type 2731 << " to " << result_type; 2732 } 2733 break; 2734 2735 case Primitive::kPrimLong: 2736 switch (input_type) { 2737 case Primitive::kPrimBoolean: 2738 // Boolean input is a result of code transformations. 2739 case Primitive::kPrimByte: 2740 case Primitive::kPrimShort: 2741 case Primitive::kPrimInt: 2742 case Primitive::kPrimChar: 2743 // Processing a Dex `int-to-long' instruction. 2744 DCHECK(out.IsRegisterPair()); 2745 DCHECK(in.IsRegister()); 2746 __ Mov(out.AsRegisterPairLow<Register>(), in.AsRegister<Register>()); 2747 // Sign extension. 2748 __ Asr(out.AsRegisterPairHigh<Register>(), 2749 out.AsRegisterPairLow<Register>(), 2750 31); 2751 break; 2752 2753 case Primitive::kPrimFloat: 2754 // Processing a Dex `float-to-long' instruction. 2755 codegen_->InvokeRuntime(kQuickF2l, conversion, conversion->GetDexPc()); 2756 CheckEntrypointTypes<kQuickF2l, int64_t, float>(); 2757 break; 2758 2759 case Primitive::kPrimDouble: 2760 // Processing a Dex `double-to-long' instruction. 2761 codegen_->InvokeRuntime(kQuickD2l, conversion, conversion->GetDexPc()); 2762 CheckEntrypointTypes<kQuickD2l, int64_t, double>(); 2763 break; 2764 2765 default: 2766 LOG(FATAL) << "Unexpected type conversion from " << input_type 2767 << " to " << result_type; 2768 } 2769 break; 2770 2771 case Primitive::kPrimChar: 2772 switch (input_type) { 2773 case Primitive::kPrimLong: 2774 // Type conversion from long to char is a result of code transformations. 2775 __ ubfx(out.AsRegister<Register>(), in.AsRegisterPairLow<Register>(), 0, 16); 2776 break; 2777 case Primitive::kPrimBoolean: 2778 // Boolean input is a result of code transformations. 2779 case Primitive::kPrimByte: 2780 case Primitive::kPrimShort: 2781 case Primitive::kPrimInt: 2782 // Processing a Dex `int-to-char' instruction. 2783 __ ubfx(out.AsRegister<Register>(), in.AsRegister<Register>(), 0, 16); 2784 break; 2785 2786 default: 2787 LOG(FATAL) << "Unexpected type conversion from " << input_type 2788 << " to " << result_type; 2789 } 2790 break; 2791 2792 case Primitive::kPrimFloat: 2793 switch (input_type) { 2794 case Primitive::kPrimBoolean: 2795 // Boolean input is a result of code transformations. 2796 case Primitive::kPrimByte: 2797 case Primitive::kPrimShort: 2798 case Primitive::kPrimInt: 2799 case Primitive::kPrimChar: { 2800 // Processing a Dex `int-to-float' instruction. 2801 __ vmovsr(out.AsFpuRegister<SRegister>(), in.AsRegister<Register>()); 2802 __ vcvtsi(out.AsFpuRegister<SRegister>(), out.AsFpuRegister<SRegister>()); 2803 break; 2804 } 2805 2806 case Primitive::kPrimLong: 2807 // Processing a Dex `long-to-float' instruction. 2808 codegen_->InvokeRuntime(kQuickL2f, conversion, conversion->GetDexPc()); 2809 CheckEntrypointTypes<kQuickL2f, float, int64_t>(); 2810 break; 2811 2812 case Primitive::kPrimDouble: 2813 // Processing a Dex `double-to-float' instruction. 2814 __ vcvtsd(out.AsFpuRegister<SRegister>(), 2815 FromLowSToD(in.AsFpuRegisterPairLow<SRegister>())); 2816 break; 2817 2818 default: 2819 LOG(FATAL) << "Unexpected type conversion from " << input_type 2820 << " to " << result_type; 2821 }; 2822 break; 2823 2824 case Primitive::kPrimDouble: 2825 switch (input_type) { 2826 case Primitive::kPrimBoolean: 2827 // Boolean input is a result of code transformations. 2828 case Primitive::kPrimByte: 2829 case Primitive::kPrimShort: 2830 case Primitive::kPrimInt: 2831 case Primitive::kPrimChar: { 2832 // Processing a Dex `int-to-double' instruction. 2833 __ vmovsr(out.AsFpuRegisterPairLow<SRegister>(), in.AsRegister<Register>()); 2834 __ vcvtdi(FromLowSToD(out.AsFpuRegisterPairLow<SRegister>()), 2835 out.AsFpuRegisterPairLow<SRegister>()); 2836 break; 2837 } 2838 2839 case Primitive::kPrimLong: { 2840 // Processing a Dex `long-to-double' instruction. 2841 Register low = in.AsRegisterPairLow<Register>(); 2842 Register high = in.AsRegisterPairHigh<Register>(); 2843 SRegister out_s = out.AsFpuRegisterPairLow<SRegister>(); 2844 DRegister out_d = FromLowSToD(out_s); 2845 SRegister temp_s = locations->GetTemp(0).AsFpuRegisterPairLow<SRegister>(); 2846 DRegister temp_d = FromLowSToD(temp_s); 2847 SRegister constant_s = locations->GetTemp(1).AsFpuRegisterPairLow<SRegister>(); 2848 DRegister constant_d = FromLowSToD(constant_s); 2849 2850 // temp_d = int-to-double(high) 2851 __ vmovsr(temp_s, high); 2852 __ vcvtdi(temp_d, temp_s); 2853 // constant_d = k2Pow32EncodingForDouble 2854 __ LoadDImmediate(constant_d, bit_cast<double, int64_t>(k2Pow32EncodingForDouble)); 2855 // out_d = unsigned-to-double(low) 2856 __ vmovsr(out_s, low); 2857 __ vcvtdu(out_d, out_s); 2858 // out_d += temp_d * constant_d 2859 __ vmlad(out_d, temp_d, constant_d); 2860 break; 2861 } 2862 2863 case Primitive::kPrimFloat: 2864 // Processing a Dex `float-to-double' instruction. 2865 __ vcvtds(FromLowSToD(out.AsFpuRegisterPairLow<SRegister>()), 2866 in.AsFpuRegister<SRegister>()); 2867 break; 2868 2869 default: 2870 LOG(FATAL) << "Unexpected type conversion from " << input_type 2871 << " to " << result_type; 2872 }; 2873 break; 2874 2875 default: 2876 LOG(FATAL) << "Unexpected type conversion from " << input_type 2877 << " to " << result_type; 2878 } 2879} 2880 2881void LocationsBuilderARM::VisitAdd(HAdd* add) { 2882 LocationSummary* locations = 2883 new (GetGraph()->GetArena()) LocationSummary(add, LocationSummary::kNoCall); 2884 switch (add->GetResultType()) { 2885 case Primitive::kPrimInt: { 2886 locations->SetInAt(0, Location::RequiresRegister()); 2887 locations->SetInAt(1, Location::RegisterOrConstant(add->InputAt(1))); 2888 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 2889 break; 2890 } 2891 2892 case Primitive::kPrimLong: { 2893 locations->SetInAt(0, Location::RequiresRegister()); 2894 locations->SetInAt(1, ArmEncodableConstantOrRegister(add->InputAt(1), ADD)); 2895 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 2896 break; 2897 } 2898 2899 case Primitive::kPrimFloat: 2900 case Primitive::kPrimDouble: { 2901 locations->SetInAt(0, Location::RequiresFpuRegister()); 2902 locations->SetInAt(1, Location::RequiresFpuRegister()); 2903 locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap); 2904 break; 2905 } 2906 2907 default: 2908 LOG(FATAL) << "Unexpected add type " << add->GetResultType(); 2909 } 2910} 2911 2912void InstructionCodeGeneratorARM::VisitAdd(HAdd* add) { 2913 LocationSummary* locations = add->GetLocations(); 2914 Location out = locations->Out(); 2915 Location first = locations->InAt(0); 2916 Location second = locations->InAt(1); 2917 switch (add->GetResultType()) { 2918 case Primitive::kPrimInt: 2919 if (second.IsRegister()) { 2920 __ add(out.AsRegister<Register>(), 2921 first.AsRegister<Register>(), 2922 ShifterOperand(second.AsRegister<Register>())); 2923 } else { 2924 __ AddConstant(out.AsRegister<Register>(), 2925 first.AsRegister<Register>(), 2926 second.GetConstant()->AsIntConstant()->GetValue()); 2927 } 2928 break; 2929 2930 case Primitive::kPrimLong: { 2931 if (second.IsConstant()) { 2932 uint64_t value = static_cast<uint64_t>(Int64FromConstant(second.GetConstant())); 2933 GenerateAddLongConst(out, first, value); 2934 } else { 2935 DCHECK(second.IsRegisterPair()); 2936 __ adds(out.AsRegisterPairLow<Register>(), 2937 first.AsRegisterPairLow<Register>(), 2938 ShifterOperand(second.AsRegisterPairLow<Register>())); 2939 __ adc(out.AsRegisterPairHigh<Register>(), 2940 first.AsRegisterPairHigh<Register>(), 2941 ShifterOperand(second.AsRegisterPairHigh<Register>())); 2942 } 2943 break; 2944 } 2945 2946 case Primitive::kPrimFloat: 2947 __ vadds(out.AsFpuRegister<SRegister>(), 2948 first.AsFpuRegister<SRegister>(), 2949 second.AsFpuRegister<SRegister>()); 2950 break; 2951 2952 case Primitive::kPrimDouble: 2953 __ vaddd(FromLowSToD(out.AsFpuRegisterPairLow<SRegister>()), 2954 FromLowSToD(first.AsFpuRegisterPairLow<SRegister>()), 2955 FromLowSToD(second.AsFpuRegisterPairLow<SRegister>())); 2956 break; 2957 2958 default: 2959 LOG(FATAL) << "Unexpected add type " << add->GetResultType(); 2960 } 2961} 2962 2963void LocationsBuilderARM::VisitSub(HSub* sub) { 2964 LocationSummary* locations = 2965 new (GetGraph()->GetArena()) LocationSummary(sub, LocationSummary::kNoCall); 2966 switch (sub->GetResultType()) { 2967 case Primitive::kPrimInt: { 2968 locations->SetInAt(0, Location::RequiresRegister()); 2969 locations->SetInAt(1, Location::RegisterOrConstant(sub->InputAt(1))); 2970 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 2971 break; 2972 } 2973 2974 case Primitive::kPrimLong: { 2975 locations->SetInAt(0, Location::RequiresRegister()); 2976 locations->SetInAt(1, ArmEncodableConstantOrRegister(sub->InputAt(1), SUB)); 2977 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 2978 break; 2979 } 2980 case Primitive::kPrimFloat: 2981 case Primitive::kPrimDouble: { 2982 locations->SetInAt(0, Location::RequiresFpuRegister()); 2983 locations->SetInAt(1, Location::RequiresFpuRegister()); 2984 locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap); 2985 break; 2986 } 2987 default: 2988 LOG(FATAL) << "Unexpected sub type " << sub->GetResultType(); 2989 } 2990} 2991 2992void InstructionCodeGeneratorARM::VisitSub(HSub* sub) { 2993 LocationSummary* locations = sub->GetLocations(); 2994 Location out = locations->Out(); 2995 Location first = locations->InAt(0); 2996 Location second = locations->InAt(1); 2997 switch (sub->GetResultType()) { 2998 case Primitive::kPrimInt: { 2999 if (second.IsRegister()) { 3000 __ sub(out.AsRegister<Register>(), 3001 first.AsRegister<Register>(), 3002 ShifterOperand(second.AsRegister<Register>())); 3003 } else { 3004 __ AddConstant(out.AsRegister<Register>(), 3005 first.AsRegister<Register>(), 3006 -second.GetConstant()->AsIntConstant()->GetValue()); 3007 } 3008 break; 3009 } 3010 3011 case Primitive::kPrimLong: { 3012 if (second.IsConstant()) { 3013 uint64_t value = static_cast<uint64_t>(Int64FromConstant(second.GetConstant())); 3014 GenerateAddLongConst(out, first, -value); 3015 } else { 3016 DCHECK(second.IsRegisterPair()); 3017 __ subs(out.AsRegisterPairLow<Register>(), 3018 first.AsRegisterPairLow<Register>(), 3019 ShifterOperand(second.AsRegisterPairLow<Register>())); 3020 __ sbc(out.AsRegisterPairHigh<Register>(), 3021 first.AsRegisterPairHigh<Register>(), 3022 ShifterOperand(second.AsRegisterPairHigh<Register>())); 3023 } 3024 break; 3025 } 3026 3027 case Primitive::kPrimFloat: { 3028 __ vsubs(out.AsFpuRegister<SRegister>(), 3029 first.AsFpuRegister<SRegister>(), 3030 second.AsFpuRegister<SRegister>()); 3031 break; 3032 } 3033 3034 case Primitive::kPrimDouble: { 3035 __ vsubd(FromLowSToD(out.AsFpuRegisterPairLow<SRegister>()), 3036 FromLowSToD(first.AsFpuRegisterPairLow<SRegister>()), 3037 FromLowSToD(second.AsFpuRegisterPairLow<SRegister>())); 3038 break; 3039 } 3040 3041 3042 default: 3043 LOG(FATAL) << "Unexpected sub type " << sub->GetResultType(); 3044 } 3045} 3046 3047void LocationsBuilderARM::VisitMul(HMul* mul) { 3048 LocationSummary* locations = 3049 new (GetGraph()->GetArena()) LocationSummary(mul, LocationSummary::kNoCall); 3050 switch (mul->GetResultType()) { 3051 case Primitive::kPrimInt: 3052 case Primitive::kPrimLong: { 3053 locations->SetInAt(0, Location::RequiresRegister()); 3054 locations->SetInAt(1, Location::RequiresRegister()); 3055 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 3056 break; 3057 } 3058 3059 case Primitive::kPrimFloat: 3060 case Primitive::kPrimDouble: { 3061 locations->SetInAt(0, Location::RequiresFpuRegister()); 3062 locations->SetInAt(1, Location::RequiresFpuRegister()); 3063 locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap); 3064 break; 3065 } 3066 3067 default: 3068 LOG(FATAL) << "Unexpected mul type " << mul->GetResultType(); 3069 } 3070} 3071 3072void InstructionCodeGeneratorARM::VisitMul(HMul* mul) { 3073 LocationSummary* locations = mul->GetLocations(); 3074 Location out = locations->Out(); 3075 Location first = locations->InAt(0); 3076 Location second = locations->InAt(1); 3077 switch (mul->GetResultType()) { 3078 case Primitive::kPrimInt: { 3079 __ mul(out.AsRegister<Register>(), 3080 first.AsRegister<Register>(), 3081 second.AsRegister<Register>()); 3082 break; 3083 } 3084 case Primitive::kPrimLong: { 3085 Register out_hi = out.AsRegisterPairHigh<Register>(); 3086 Register out_lo = out.AsRegisterPairLow<Register>(); 3087 Register in1_hi = first.AsRegisterPairHigh<Register>(); 3088 Register in1_lo = first.AsRegisterPairLow<Register>(); 3089 Register in2_hi = second.AsRegisterPairHigh<Register>(); 3090 Register in2_lo = second.AsRegisterPairLow<Register>(); 3091 3092 // Extra checks to protect caused by the existence of R1_R2. 3093 // The algorithm is wrong if out.hi is either in1.lo or in2.lo: 3094 // (e.g. in1=r0_r1, in2=r2_r3 and out=r1_r2); 3095 DCHECK_NE(out_hi, in1_lo); 3096 DCHECK_NE(out_hi, in2_lo); 3097 3098 // input: in1 - 64 bits, in2 - 64 bits 3099 // output: out 3100 // formula: out.hi : out.lo = (in1.lo * in2.hi + in1.hi * in2.lo)* 2^32 + in1.lo * in2.lo 3101 // parts: out.hi = in1.lo * in2.hi + in1.hi * in2.lo + (in1.lo * in2.lo)[63:32] 3102 // parts: out.lo = (in1.lo * in2.lo)[31:0] 3103 3104 // IP <- in1.lo * in2.hi 3105 __ mul(IP, in1_lo, in2_hi); 3106 // out.hi <- in1.lo * in2.hi + in1.hi * in2.lo 3107 __ mla(out_hi, in1_hi, in2_lo, IP); 3108 // out.lo <- (in1.lo * in2.lo)[31:0]; 3109 __ umull(out_lo, IP, in1_lo, in2_lo); 3110 // out.hi <- in2.hi * in1.lo + in2.lo * in1.hi + (in1.lo * in2.lo)[63:32] 3111 __ add(out_hi, out_hi, ShifterOperand(IP)); 3112 break; 3113 } 3114 3115 case Primitive::kPrimFloat: { 3116 __ vmuls(out.AsFpuRegister<SRegister>(), 3117 first.AsFpuRegister<SRegister>(), 3118 second.AsFpuRegister<SRegister>()); 3119 break; 3120 } 3121 3122 case Primitive::kPrimDouble: { 3123 __ vmuld(FromLowSToD(out.AsFpuRegisterPairLow<SRegister>()), 3124 FromLowSToD(first.AsFpuRegisterPairLow<SRegister>()), 3125 FromLowSToD(second.AsFpuRegisterPairLow<SRegister>())); 3126 break; 3127 } 3128 3129 default: 3130 LOG(FATAL) << "Unexpected mul type " << mul->GetResultType(); 3131 } 3132} 3133 3134void InstructionCodeGeneratorARM::DivRemOneOrMinusOne(HBinaryOperation* instruction) { 3135 DCHECK(instruction->IsDiv() || instruction->IsRem()); 3136 DCHECK(instruction->GetResultType() == Primitive::kPrimInt); 3137 3138 LocationSummary* locations = instruction->GetLocations(); 3139 Location second = locations->InAt(1); 3140 DCHECK(second.IsConstant()); 3141 3142 Register out = locations->Out().AsRegister<Register>(); 3143 Register dividend = locations->InAt(0).AsRegister<Register>(); 3144 int32_t imm = second.GetConstant()->AsIntConstant()->GetValue(); 3145 DCHECK(imm == 1 || imm == -1); 3146 3147 if (instruction->IsRem()) { 3148 __ LoadImmediate(out, 0); 3149 } else { 3150 if (imm == 1) { 3151 __ Mov(out, dividend); 3152 } else { 3153 __ rsb(out, dividend, ShifterOperand(0)); 3154 } 3155 } 3156} 3157 3158void InstructionCodeGeneratorARM::DivRemByPowerOfTwo(HBinaryOperation* instruction) { 3159 DCHECK(instruction->IsDiv() || instruction->IsRem()); 3160 DCHECK(instruction->GetResultType() == Primitive::kPrimInt); 3161 3162 LocationSummary* locations = instruction->GetLocations(); 3163 Location second = locations->InAt(1); 3164 DCHECK(second.IsConstant()); 3165 3166 Register out = locations->Out().AsRegister<Register>(); 3167 Register dividend = locations->InAt(0).AsRegister<Register>(); 3168 Register temp = locations->GetTemp(0).AsRegister<Register>(); 3169 int32_t imm = second.GetConstant()->AsIntConstant()->GetValue(); 3170 uint32_t abs_imm = static_cast<uint32_t>(AbsOrMin(imm)); 3171 int ctz_imm = CTZ(abs_imm); 3172 3173 if (ctz_imm == 1) { 3174 __ Lsr(temp, dividend, 32 - ctz_imm); 3175 } else { 3176 __ Asr(temp, dividend, 31); 3177 __ Lsr(temp, temp, 32 - ctz_imm); 3178 } 3179 __ add(out, temp, ShifterOperand(dividend)); 3180 3181 if (instruction->IsDiv()) { 3182 __ Asr(out, out, ctz_imm); 3183 if (imm < 0) { 3184 __ rsb(out, out, ShifterOperand(0)); 3185 } 3186 } else { 3187 __ ubfx(out, out, 0, ctz_imm); 3188 __ sub(out, out, ShifterOperand(temp)); 3189 } 3190} 3191 3192void InstructionCodeGeneratorARM::GenerateDivRemWithAnyConstant(HBinaryOperation* instruction) { 3193 DCHECK(instruction->IsDiv() || instruction->IsRem()); 3194 DCHECK(instruction->GetResultType() == Primitive::kPrimInt); 3195 3196 LocationSummary* locations = instruction->GetLocations(); 3197 Location second = locations->InAt(1); 3198 DCHECK(second.IsConstant()); 3199 3200 Register out = locations->Out().AsRegister<Register>(); 3201 Register dividend = locations->InAt(0).AsRegister<Register>(); 3202 Register temp1 = locations->GetTemp(0).AsRegister<Register>(); 3203 Register temp2 = locations->GetTemp(1).AsRegister<Register>(); 3204 int64_t imm = second.GetConstant()->AsIntConstant()->GetValue(); 3205 3206 int64_t magic; 3207 int shift; 3208 CalculateMagicAndShiftForDivRem(imm, false /* is_long */, &magic, &shift); 3209 3210 __ LoadImmediate(temp1, magic); 3211 __ smull(temp2, temp1, dividend, temp1); 3212 3213 if (imm > 0 && magic < 0) { 3214 __ add(temp1, temp1, ShifterOperand(dividend)); 3215 } else if (imm < 0 && magic > 0) { 3216 __ sub(temp1, temp1, ShifterOperand(dividend)); 3217 } 3218 3219 if (shift != 0) { 3220 __ Asr(temp1, temp1, shift); 3221 } 3222 3223 if (instruction->IsDiv()) { 3224 __ sub(out, temp1, ShifterOperand(temp1, ASR, 31)); 3225 } else { 3226 __ sub(temp1, temp1, ShifterOperand(temp1, ASR, 31)); 3227 // TODO: Strength reduction for mls. 3228 __ LoadImmediate(temp2, imm); 3229 __ mls(out, temp1, temp2, dividend); 3230 } 3231} 3232 3233void InstructionCodeGeneratorARM::GenerateDivRemConstantIntegral(HBinaryOperation* instruction) { 3234 DCHECK(instruction->IsDiv() || instruction->IsRem()); 3235 DCHECK(instruction->GetResultType() == Primitive::kPrimInt); 3236 3237 LocationSummary* locations = instruction->GetLocations(); 3238 Location second = locations->InAt(1); 3239 DCHECK(second.IsConstant()); 3240 3241 int32_t imm = second.GetConstant()->AsIntConstant()->GetValue(); 3242 if (imm == 0) { 3243 // Do not generate anything. DivZeroCheck would prevent any code to be executed. 3244 } else if (imm == 1 || imm == -1) { 3245 DivRemOneOrMinusOne(instruction); 3246 } else if (IsPowerOfTwo(AbsOrMin(imm))) { 3247 DivRemByPowerOfTwo(instruction); 3248 } else { 3249 DCHECK(imm <= -2 || imm >= 2); 3250 GenerateDivRemWithAnyConstant(instruction); 3251 } 3252} 3253 3254void LocationsBuilderARM::VisitDiv(HDiv* div) { 3255 LocationSummary::CallKind call_kind = LocationSummary::kNoCall; 3256 if (div->GetResultType() == Primitive::kPrimLong) { 3257 // pLdiv runtime call. 3258 call_kind = LocationSummary::kCallOnMainOnly; 3259 } else if (div->GetResultType() == Primitive::kPrimInt && div->InputAt(1)->IsConstant()) { 3260 // sdiv will be replaced by other instruction sequence. 3261 } else if (div->GetResultType() == Primitive::kPrimInt && 3262 !codegen_->GetInstructionSetFeatures().HasDivideInstruction()) { 3263 // pIdivmod runtime call. 3264 call_kind = LocationSummary::kCallOnMainOnly; 3265 } 3266 3267 LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(div, call_kind); 3268 3269 switch (div->GetResultType()) { 3270 case Primitive::kPrimInt: { 3271 if (div->InputAt(1)->IsConstant()) { 3272 locations->SetInAt(0, Location::RequiresRegister()); 3273 locations->SetInAt(1, Location::ConstantLocation(div->InputAt(1)->AsConstant())); 3274 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 3275 int32_t value = div->InputAt(1)->AsIntConstant()->GetValue(); 3276 if (value == 1 || value == 0 || value == -1) { 3277 // No temp register required. 3278 } else { 3279 locations->AddTemp(Location::RequiresRegister()); 3280 if (!IsPowerOfTwo(AbsOrMin(value))) { 3281 locations->AddTemp(Location::RequiresRegister()); 3282 } 3283 } 3284 } else if (codegen_->GetInstructionSetFeatures().HasDivideInstruction()) { 3285 locations->SetInAt(0, Location::RequiresRegister()); 3286 locations->SetInAt(1, Location::RequiresRegister()); 3287 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 3288 } else { 3289 InvokeRuntimeCallingConvention calling_convention; 3290 locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(0))); 3291 locations->SetInAt(1, Location::RegisterLocation(calling_convention.GetRegisterAt(1))); 3292 // Note: divrem will compute both the quotient and the remainder as the pair R0 and R1, but 3293 // we only need the former. 3294 locations->SetOut(Location::RegisterLocation(R0)); 3295 } 3296 break; 3297 } 3298 case Primitive::kPrimLong: { 3299 InvokeRuntimeCallingConvention calling_convention; 3300 locations->SetInAt(0, Location::RegisterPairLocation( 3301 calling_convention.GetRegisterAt(0), calling_convention.GetRegisterAt(1))); 3302 locations->SetInAt(1, Location::RegisterPairLocation( 3303 calling_convention.GetRegisterAt(2), calling_convention.GetRegisterAt(3))); 3304 locations->SetOut(Location::RegisterPairLocation(R0, R1)); 3305 break; 3306 } 3307 case Primitive::kPrimFloat: 3308 case Primitive::kPrimDouble: { 3309 locations->SetInAt(0, Location::RequiresFpuRegister()); 3310 locations->SetInAt(1, Location::RequiresFpuRegister()); 3311 locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap); 3312 break; 3313 } 3314 3315 default: 3316 LOG(FATAL) << "Unexpected div type " << div->GetResultType(); 3317 } 3318} 3319 3320void InstructionCodeGeneratorARM::VisitDiv(HDiv* div) { 3321 LocationSummary* locations = div->GetLocations(); 3322 Location out = locations->Out(); 3323 Location first = locations->InAt(0); 3324 Location second = locations->InAt(1); 3325 3326 switch (div->GetResultType()) { 3327 case Primitive::kPrimInt: { 3328 if (second.IsConstant()) { 3329 GenerateDivRemConstantIntegral(div); 3330 } else if (codegen_->GetInstructionSetFeatures().HasDivideInstruction()) { 3331 __ sdiv(out.AsRegister<Register>(), 3332 first.AsRegister<Register>(), 3333 second.AsRegister<Register>()); 3334 } else { 3335 InvokeRuntimeCallingConvention calling_convention; 3336 DCHECK_EQ(calling_convention.GetRegisterAt(0), first.AsRegister<Register>()); 3337 DCHECK_EQ(calling_convention.GetRegisterAt(1), second.AsRegister<Register>()); 3338 DCHECK_EQ(R0, out.AsRegister<Register>()); 3339 3340 codegen_->InvokeRuntime(kQuickIdivmod, div, div->GetDexPc()); 3341 CheckEntrypointTypes<kQuickIdivmod, int32_t, int32_t, int32_t>(); 3342 } 3343 break; 3344 } 3345 3346 case Primitive::kPrimLong: { 3347 InvokeRuntimeCallingConvention calling_convention; 3348 DCHECK_EQ(calling_convention.GetRegisterAt(0), first.AsRegisterPairLow<Register>()); 3349 DCHECK_EQ(calling_convention.GetRegisterAt(1), first.AsRegisterPairHigh<Register>()); 3350 DCHECK_EQ(calling_convention.GetRegisterAt(2), second.AsRegisterPairLow<Register>()); 3351 DCHECK_EQ(calling_convention.GetRegisterAt(3), second.AsRegisterPairHigh<Register>()); 3352 DCHECK_EQ(R0, out.AsRegisterPairLow<Register>()); 3353 DCHECK_EQ(R1, out.AsRegisterPairHigh<Register>()); 3354 3355 codegen_->InvokeRuntime(kQuickLdiv, div, div->GetDexPc()); 3356 CheckEntrypointTypes<kQuickLdiv, int64_t, int64_t, int64_t>(); 3357 break; 3358 } 3359 3360 case Primitive::kPrimFloat: { 3361 __ vdivs(out.AsFpuRegister<SRegister>(), 3362 first.AsFpuRegister<SRegister>(), 3363 second.AsFpuRegister<SRegister>()); 3364 break; 3365 } 3366 3367 case Primitive::kPrimDouble: { 3368 __ vdivd(FromLowSToD(out.AsFpuRegisterPairLow<SRegister>()), 3369 FromLowSToD(first.AsFpuRegisterPairLow<SRegister>()), 3370 FromLowSToD(second.AsFpuRegisterPairLow<SRegister>())); 3371 break; 3372 } 3373 3374 default: 3375 LOG(FATAL) << "Unexpected div type " << div->GetResultType(); 3376 } 3377} 3378 3379void LocationsBuilderARM::VisitRem(HRem* rem) { 3380 Primitive::Type type = rem->GetResultType(); 3381 3382 // Most remainders are implemented in the runtime. 3383 LocationSummary::CallKind call_kind = LocationSummary::kCallOnMainOnly; 3384 if (rem->GetResultType() == Primitive::kPrimInt && rem->InputAt(1)->IsConstant()) { 3385 // sdiv will be replaced by other instruction sequence. 3386 call_kind = LocationSummary::kNoCall; 3387 } else if ((rem->GetResultType() == Primitive::kPrimInt) 3388 && codegen_->GetInstructionSetFeatures().HasDivideInstruction()) { 3389 // Have hardware divide instruction for int, do it with three instructions. 3390 call_kind = LocationSummary::kNoCall; 3391 } 3392 3393 LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(rem, call_kind); 3394 3395 switch (type) { 3396 case Primitive::kPrimInt: { 3397 if (rem->InputAt(1)->IsConstant()) { 3398 locations->SetInAt(0, Location::RequiresRegister()); 3399 locations->SetInAt(1, Location::ConstantLocation(rem->InputAt(1)->AsConstant())); 3400 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 3401 int32_t value = rem->InputAt(1)->AsIntConstant()->GetValue(); 3402 if (value == 1 || value == 0 || value == -1) { 3403 // No temp register required. 3404 } else { 3405 locations->AddTemp(Location::RequiresRegister()); 3406 if (!IsPowerOfTwo(AbsOrMin(value))) { 3407 locations->AddTemp(Location::RequiresRegister()); 3408 } 3409 } 3410 } else if (codegen_->GetInstructionSetFeatures().HasDivideInstruction()) { 3411 locations->SetInAt(0, Location::RequiresRegister()); 3412 locations->SetInAt(1, Location::RequiresRegister()); 3413 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 3414 locations->AddTemp(Location::RequiresRegister()); 3415 } else { 3416 InvokeRuntimeCallingConvention calling_convention; 3417 locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(0))); 3418 locations->SetInAt(1, Location::RegisterLocation(calling_convention.GetRegisterAt(1))); 3419 // Note: divrem will compute both the quotient and the remainder as the pair R0 and R1, but 3420 // we only need the latter. 3421 locations->SetOut(Location::RegisterLocation(R1)); 3422 } 3423 break; 3424 } 3425 case Primitive::kPrimLong: { 3426 InvokeRuntimeCallingConvention calling_convention; 3427 locations->SetInAt(0, Location::RegisterPairLocation( 3428 calling_convention.GetRegisterAt(0), calling_convention.GetRegisterAt(1))); 3429 locations->SetInAt(1, Location::RegisterPairLocation( 3430 calling_convention.GetRegisterAt(2), calling_convention.GetRegisterAt(3))); 3431 // The runtime helper puts the output in R2,R3. 3432 locations->SetOut(Location::RegisterPairLocation(R2, R3)); 3433 break; 3434 } 3435 case Primitive::kPrimFloat: { 3436 InvokeRuntimeCallingConvention calling_convention; 3437 locations->SetInAt(0, Location::FpuRegisterLocation(calling_convention.GetFpuRegisterAt(0))); 3438 locations->SetInAt(1, Location::FpuRegisterLocation(calling_convention.GetFpuRegisterAt(1))); 3439 locations->SetOut(Location::FpuRegisterLocation(S0)); 3440 break; 3441 } 3442 3443 case Primitive::kPrimDouble: { 3444 InvokeRuntimeCallingConvention calling_convention; 3445 locations->SetInAt(0, Location::FpuRegisterPairLocation( 3446 calling_convention.GetFpuRegisterAt(0), calling_convention.GetFpuRegisterAt(1))); 3447 locations->SetInAt(1, Location::FpuRegisterPairLocation( 3448 calling_convention.GetFpuRegisterAt(2), calling_convention.GetFpuRegisterAt(3))); 3449 locations->SetOut(Location::Location::FpuRegisterPairLocation(S0, S1)); 3450 break; 3451 } 3452 3453 default: 3454 LOG(FATAL) << "Unexpected rem type " << type; 3455 } 3456} 3457 3458void InstructionCodeGeneratorARM::VisitRem(HRem* rem) { 3459 LocationSummary* locations = rem->GetLocations(); 3460 Location out = locations->Out(); 3461 Location first = locations->InAt(0); 3462 Location second = locations->InAt(1); 3463 3464 Primitive::Type type = rem->GetResultType(); 3465 switch (type) { 3466 case Primitive::kPrimInt: { 3467 if (second.IsConstant()) { 3468 GenerateDivRemConstantIntegral(rem); 3469 } else if (codegen_->GetInstructionSetFeatures().HasDivideInstruction()) { 3470 Register reg1 = first.AsRegister<Register>(); 3471 Register reg2 = second.AsRegister<Register>(); 3472 Register temp = locations->GetTemp(0).AsRegister<Register>(); 3473 3474 // temp = reg1 / reg2 (integer division) 3475 // dest = reg1 - temp * reg2 3476 __ sdiv(temp, reg1, reg2); 3477 __ mls(out.AsRegister<Register>(), temp, reg2, reg1); 3478 } else { 3479 InvokeRuntimeCallingConvention calling_convention; 3480 DCHECK_EQ(calling_convention.GetRegisterAt(0), first.AsRegister<Register>()); 3481 DCHECK_EQ(calling_convention.GetRegisterAt(1), second.AsRegister<Register>()); 3482 DCHECK_EQ(R1, out.AsRegister<Register>()); 3483 3484 codegen_->InvokeRuntime(kQuickIdivmod, rem, rem->GetDexPc()); 3485 CheckEntrypointTypes<kQuickIdivmod, int32_t, int32_t, int32_t>(); 3486 } 3487 break; 3488 } 3489 3490 case Primitive::kPrimLong: { 3491 codegen_->InvokeRuntime(kQuickLmod, rem, rem->GetDexPc()); 3492 CheckEntrypointTypes<kQuickLmod, int64_t, int64_t, int64_t>(); 3493 break; 3494 } 3495 3496 case Primitive::kPrimFloat: { 3497 codegen_->InvokeRuntime(kQuickFmodf, rem, rem->GetDexPc()); 3498 CheckEntrypointTypes<kQuickFmodf, float, float, float>(); 3499 break; 3500 } 3501 3502 case Primitive::kPrimDouble: { 3503 codegen_->InvokeRuntime(kQuickFmod, rem, rem->GetDexPc()); 3504 CheckEntrypointTypes<kQuickFmod, double, double, double>(); 3505 break; 3506 } 3507 3508 default: 3509 LOG(FATAL) << "Unexpected rem type " << type; 3510 } 3511} 3512 3513void LocationsBuilderARM::VisitDivZeroCheck(HDivZeroCheck* instruction) { 3514 LocationSummary* locations = codegen_->CreateThrowingSlowPathLocations(instruction); 3515 locations->SetInAt(0, Location::RegisterOrConstant(instruction->InputAt(0))); 3516} 3517 3518void InstructionCodeGeneratorARM::VisitDivZeroCheck(HDivZeroCheck* instruction) { 3519 SlowPathCodeARM* slow_path = new (GetGraph()->GetArena()) DivZeroCheckSlowPathARM(instruction); 3520 codegen_->AddSlowPath(slow_path); 3521 3522 LocationSummary* locations = instruction->GetLocations(); 3523 Location value = locations->InAt(0); 3524 3525 switch (instruction->GetType()) { 3526 case Primitive::kPrimBoolean: 3527 case Primitive::kPrimByte: 3528 case Primitive::kPrimChar: 3529 case Primitive::kPrimShort: 3530 case Primitive::kPrimInt: { 3531 if (value.IsRegister()) { 3532 __ CompareAndBranchIfZero(value.AsRegister<Register>(), slow_path->GetEntryLabel()); 3533 } else { 3534 DCHECK(value.IsConstant()) << value; 3535 if (value.GetConstant()->AsIntConstant()->GetValue() == 0) { 3536 __ b(slow_path->GetEntryLabel()); 3537 } 3538 } 3539 break; 3540 } 3541 case Primitive::kPrimLong: { 3542 if (value.IsRegisterPair()) { 3543 __ orrs(IP, 3544 value.AsRegisterPairLow<Register>(), 3545 ShifterOperand(value.AsRegisterPairHigh<Register>())); 3546 __ b(slow_path->GetEntryLabel(), EQ); 3547 } else { 3548 DCHECK(value.IsConstant()) << value; 3549 if (value.GetConstant()->AsLongConstant()->GetValue() == 0) { 3550 __ b(slow_path->GetEntryLabel()); 3551 } 3552 } 3553 break; 3554 default: 3555 LOG(FATAL) << "Unexpected type for HDivZeroCheck " << instruction->GetType(); 3556 } 3557 } 3558} 3559 3560void InstructionCodeGeneratorARM::HandleIntegerRotate(LocationSummary* locations) { 3561 Register in = locations->InAt(0).AsRegister<Register>(); 3562 Location rhs = locations->InAt(1); 3563 Register out = locations->Out().AsRegister<Register>(); 3564 3565 if (rhs.IsConstant()) { 3566 // Arm32 and Thumb2 assemblers require a rotation on the interval [1,31], 3567 // so map all rotations to a +ve. equivalent in that range. 3568 // (e.g. left *or* right by -2 bits == 30 bits in the same direction.) 3569 uint32_t rot = CodeGenerator::GetInt32ValueOf(rhs.GetConstant()) & 0x1F; 3570 if (rot) { 3571 // Rotate, mapping left rotations to right equivalents if necessary. 3572 // (e.g. left by 2 bits == right by 30.) 3573 __ Ror(out, in, rot); 3574 } else if (out != in) { 3575 __ Mov(out, in); 3576 } 3577 } else { 3578 __ Ror(out, in, rhs.AsRegister<Register>()); 3579 } 3580} 3581 3582// Gain some speed by mapping all Long rotates onto equivalent pairs of Integer 3583// rotates by swapping input regs (effectively rotating by the first 32-bits of 3584// a larger rotation) or flipping direction (thus treating larger right/left 3585// rotations as sub-word sized rotations in the other direction) as appropriate. 3586void InstructionCodeGeneratorARM::HandleLongRotate(LocationSummary* locations) { 3587 Register in_reg_lo = locations->InAt(0).AsRegisterPairLow<Register>(); 3588 Register in_reg_hi = locations->InAt(0).AsRegisterPairHigh<Register>(); 3589 Location rhs = locations->InAt(1); 3590 Register out_reg_lo = locations->Out().AsRegisterPairLow<Register>(); 3591 Register out_reg_hi = locations->Out().AsRegisterPairHigh<Register>(); 3592 3593 if (rhs.IsConstant()) { 3594 uint64_t rot = CodeGenerator::GetInt64ValueOf(rhs.GetConstant()); 3595 // Map all rotations to +ve. equivalents on the interval [0,63]. 3596 rot &= kMaxLongShiftDistance; 3597 // For rotates over a word in size, 'pre-rotate' by 32-bits to keep rotate 3598 // logic below to a simple pair of binary orr. 3599 // (e.g. 34 bits == in_reg swap + 2 bits right.) 3600 if (rot >= kArmBitsPerWord) { 3601 rot -= kArmBitsPerWord; 3602 std::swap(in_reg_hi, in_reg_lo); 3603 } 3604 // Rotate, or mov to out for zero or word size rotations. 3605 if (rot != 0u) { 3606 __ Lsr(out_reg_hi, in_reg_hi, rot); 3607 __ orr(out_reg_hi, out_reg_hi, ShifterOperand(in_reg_lo, arm::LSL, kArmBitsPerWord - rot)); 3608 __ Lsr(out_reg_lo, in_reg_lo, rot); 3609 __ orr(out_reg_lo, out_reg_lo, ShifterOperand(in_reg_hi, arm::LSL, kArmBitsPerWord - rot)); 3610 } else { 3611 __ Mov(out_reg_lo, in_reg_lo); 3612 __ Mov(out_reg_hi, in_reg_hi); 3613 } 3614 } else { 3615 Register shift_right = locations->GetTemp(0).AsRegister<Register>(); 3616 Register shift_left = locations->GetTemp(1).AsRegister<Register>(); 3617 Label end; 3618 Label shift_by_32_plus_shift_right; 3619 3620 __ and_(shift_right, rhs.AsRegister<Register>(), ShifterOperand(0x1F)); 3621 __ Lsrs(shift_left, rhs.AsRegister<Register>(), 6); 3622 __ rsb(shift_left, shift_right, ShifterOperand(kArmBitsPerWord), AL, kCcKeep); 3623 __ b(&shift_by_32_plus_shift_right, CC); 3624 3625 // out_reg_hi = (reg_hi << shift_left) | (reg_lo >> shift_right). 3626 // out_reg_lo = (reg_lo << shift_left) | (reg_hi >> shift_right). 3627 __ Lsl(out_reg_hi, in_reg_hi, shift_left); 3628 __ Lsr(out_reg_lo, in_reg_lo, shift_right); 3629 __ add(out_reg_hi, out_reg_hi, ShifterOperand(out_reg_lo)); 3630 __ Lsl(out_reg_lo, in_reg_lo, shift_left); 3631 __ Lsr(shift_left, in_reg_hi, shift_right); 3632 __ add(out_reg_lo, out_reg_lo, ShifterOperand(shift_left)); 3633 __ b(&end); 3634 3635 __ Bind(&shift_by_32_plus_shift_right); // Shift by 32+shift_right. 3636 // out_reg_hi = (reg_hi >> shift_right) | (reg_lo << shift_left). 3637 // out_reg_lo = (reg_lo >> shift_right) | (reg_hi << shift_left). 3638 __ Lsr(out_reg_hi, in_reg_hi, shift_right); 3639 __ Lsl(out_reg_lo, in_reg_lo, shift_left); 3640 __ add(out_reg_hi, out_reg_hi, ShifterOperand(out_reg_lo)); 3641 __ Lsr(out_reg_lo, in_reg_lo, shift_right); 3642 __ Lsl(shift_right, in_reg_hi, shift_left); 3643 __ add(out_reg_lo, out_reg_lo, ShifterOperand(shift_right)); 3644 3645 __ Bind(&end); 3646 } 3647} 3648 3649void LocationsBuilderARM::VisitRor(HRor* ror) { 3650 LocationSummary* locations = 3651 new (GetGraph()->GetArena()) LocationSummary(ror, LocationSummary::kNoCall); 3652 switch (ror->GetResultType()) { 3653 case Primitive::kPrimInt: { 3654 locations->SetInAt(0, Location::RequiresRegister()); 3655 locations->SetInAt(1, Location::RegisterOrConstant(ror->InputAt(1))); 3656 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 3657 break; 3658 } 3659 case Primitive::kPrimLong: { 3660 locations->SetInAt(0, Location::RequiresRegister()); 3661 if (ror->InputAt(1)->IsConstant()) { 3662 locations->SetInAt(1, Location::ConstantLocation(ror->InputAt(1)->AsConstant())); 3663 } else { 3664 locations->SetInAt(1, Location::RequiresRegister()); 3665 locations->AddTemp(Location::RequiresRegister()); 3666 locations->AddTemp(Location::RequiresRegister()); 3667 } 3668 locations->SetOut(Location::RequiresRegister(), Location::kOutputOverlap); 3669 break; 3670 } 3671 default: 3672 LOG(FATAL) << "Unexpected operation type " << ror->GetResultType(); 3673 } 3674} 3675 3676void InstructionCodeGeneratorARM::VisitRor(HRor* ror) { 3677 LocationSummary* locations = ror->GetLocations(); 3678 Primitive::Type type = ror->GetResultType(); 3679 switch (type) { 3680 case Primitive::kPrimInt: { 3681 HandleIntegerRotate(locations); 3682 break; 3683 } 3684 case Primitive::kPrimLong: { 3685 HandleLongRotate(locations); 3686 break; 3687 } 3688 default: 3689 LOG(FATAL) << "Unexpected operation type " << type; 3690 UNREACHABLE(); 3691 } 3692} 3693 3694void LocationsBuilderARM::HandleShift(HBinaryOperation* op) { 3695 DCHECK(op->IsShl() || op->IsShr() || op->IsUShr()); 3696 3697 LocationSummary* locations = 3698 new (GetGraph()->GetArena()) LocationSummary(op, LocationSummary::kNoCall); 3699 3700 switch (op->GetResultType()) { 3701 case Primitive::kPrimInt: { 3702 locations->SetInAt(0, Location::RequiresRegister()); 3703 if (op->InputAt(1)->IsConstant()) { 3704 locations->SetInAt(1, Location::ConstantLocation(op->InputAt(1)->AsConstant())); 3705 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 3706 } else { 3707 locations->SetInAt(1, Location::RequiresRegister()); 3708 // Make the output overlap, as it will be used to hold the masked 3709 // second input. 3710 locations->SetOut(Location::RequiresRegister(), Location::kOutputOverlap); 3711 } 3712 break; 3713 } 3714 case Primitive::kPrimLong: { 3715 locations->SetInAt(0, Location::RequiresRegister()); 3716 if (op->InputAt(1)->IsConstant()) { 3717 locations->SetInAt(1, Location::ConstantLocation(op->InputAt(1)->AsConstant())); 3718 // For simplicity, use kOutputOverlap even though we only require that low registers 3719 // don't clash with high registers which the register allocator currently guarantees. 3720 locations->SetOut(Location::RequiresRegister(), Location::kOutputOverlap); 3721 } else { 3722 locations->SetInAt(1, Location::RequiresRegister()); 3723 locations->AddTemp(Location::RequiresRegister()); 3724 locations->SetOut(Location::RequiresRegister(), Location::kOutputOverlap); 3725 } 3726 break; 3727 } 3728 default: 3729 LOG(FATAL) << "Unexpected operation type " << op->GetResultType(); 3730 } 3731} 3732 3733void InstructionCodeGeneratorARM::HandleShift(HBinaryOperation* op) { 3734 DCHECK(op->IsShl() || op->IsShr() || op->IsUShr()); 3735 3736 LocationSummary* locations = op->GetLocations(); 3737 Location out = locations->Out(); 3738 Location first = locations->InAt(0); 3739 Location second = locations->InAt(1); 3740 3741 Primitive::Type type = op->GetResultType(); 3742 switch (type) { 3743 case Primitive::kPrimInt: { 3744 Register out_reg = out.AsRegister<Register>(); 3745 Register first_reg = first.AsRegister<Register>(); 3746 if (second.IsRegister()) { 3747 Register second_reg = second.AsRegister<Register>(); 3748 // ARM doesn't mask the shift count so we need to do it ourselves. 3749 __ and_(out_reg, second_reg, ShifterOperand(kMaxIntShiftDistance)); 3750 if (op->IsShl()) { 3751 __ Lsl(out_reg, first_reg, out_reg); 3752 } else if (op->IsShr()) { 3753 __ Asr(out_reg, first_reg, out_reg); 3754 } else { 3755 __ Lsr(out_reg, first_reg, out_reg); 3756 } 3757 } else { 3758 int32_t cst = second.GetConstant()->AsIntConstant()->GetValue(); 3759 uint32_t shift_value = cst & kMaxIntShiftDistance; 3760 if (shift_value == 0) { // ARM does not support shifting with 0 immediate. 3761 __ Mov(out_reg, first_reg); 3762 } else if (op->IsShl()) { 3763 __ Lsl(out_reg, first_reg, shift_value); 3764 } else if (op->IsShr()) { 3765 __ Asr(out_reg, first_reg, shift_value); 3766 } else { 3767 __ Lsr(out_reg, first_reg, shift_value); 3768 } 3769 } 3770 break; 3771 } 3772 case Primitive::kPrimLong: { 3773 Register o_h = out.AsRegisterPairHigh<Register>(); 3774 Register o_l = out.AsRegisterPairLow<Register>(); 3775 3776 Register high = first.AsRegisterPairHigh<Register>(); 3777 Register low = first.AsRegisterPairLow<Register>(); 3778 3779 if (second.IsRegister()) { 3780 Register temp = locations->GetTemp(0).AsRegister<Register>(); 3781 3782 Register second_reg = second.AsRegister<Register>(); 3783 3784 if (op->IsShl()) { 3785 __ and_(o_l, second_reg, ShifterOperand(kMaxLongShiftDistance)); 3786 // Shift the high part 3787 __ Lsl(o_h, high, o_l); 3788 // Shift the low part and `or` what overflew on the high part 3789 __ rsb(temp, o_l, ShifterOperand(kArmBitsPerWord)); 3790 __ Lsr(temp, low, temp); 3791 __ orr(o_h, o_h, ShifterOperand(temp)); 3792 // If the shift is > 32 bits, override the high part 3793 __ subs(temp, o_l, ShifterOperand(kArmBitsPerWord)); 3794 __ it(PL); 3795 __ Lsl(o_h, low, temp, PL); 3796 // Shift the low part 3797 __ Lsl(o_l, low, o_l); 3798 } else if (op->IsShr()) { 3799 __ and_(o_h, second_reg, ShifterOperand(kMaxLongShiftDistance)); 3800 // Shift the low part 3801 __ Lsr(o_l, low, o_h); 3802 // Shift the high part and `or` what underflew on the low part 3803 __ rsb(temp, o_h, ShifterOperand(kArmBitsPerWord)); 3804 __ Lsl(temp, high, temp); 3805 __ orr(o_l, o_l, ShifterOperand(temp)); 3806 // If the shift is > 32 bits, override the low part 3807 __ subs(temp, o_h, ShifterOperand(kArmBitsPerWord)); 3808 __ it(PL); 3809 __ Asr(o_l, high, temp, PL); 3810 // Shift the high part 3811 __ Asr(o_h, high, o_h); 3812 } else { 3813 __ and_(o_h, second_reg, ShifterOperand(kMaxLongShiftDistance)); 3814 // same as Shr except we use `Lsr`s and not `Asr`s 3815 __ Lsr(o_l, low, o_h); 3816 __ rsb(temp, o_h, ShifterOperand(kArmBitsPerWord)); 3817 __ Lsl(temp, high, temp); 3818 __ orr(o_l, o_l, ShifterOperand(temp)); 3819 __ subs(temp, o_h, ShifterOperand(kArmBitsPerWord)); 3820 __ it(PL); 3821 __ Lsr(o_l, high, temp, PL); 3822 __ Lsr(o_h, high, o_h); 3823 } 3824 } else { 3825 // Register allocator doesn't create partial overlap. 3826 DCHECK_NE(o_l, high); 3827 DCHECK_NE(o_h, low); 3828 int32_t cst = second.GetConstant()->AsIntConstant()->GetValue(); 3829 uint32_t shift_value = cst & kMaxLongShiftDistance; 3830 if (shift_value > 32) { 3831 if (op->IsShl()) { 3832 __ Lsl(o_h, low, shift_value - 32); 3833 __ LoadImmediate(o_l, 0); 3834 } else if (op->IsShr()) { 3835 __ Asr(o_l, high, shift_value - 32); 3836 __ Asr(o_h, high, 31); 3837 } else { 3838 __ Lsr(o_l, high, shift_value - 32); 3839 __ LoadImmediate(o_h, 0); 3840 } 3841 } else if (shift_value == 32) { 3842 if (op->IsShl()) { 3843 __ mov(o_h, ShifterOperand(low)); 3844 __ LoadImmediate(o_l, 0); 3845 } else if (op->IsShr()) { 3846 __ mov(o_l, ShifterOperand(high)); 3847 __ Asr(o_h, high, 31); 3848 } else { 3849 __ mov(o_l, ShifterOperand(high)); 3850 __ LoadImmediate(o_h, 0); 3851 } 3852 } else if (shift_value == 1) { 3853 if (op->IsShl()) { 3854 __ Lsls(o_l, low, 1); 3855 __ adc(o_h, high, ShifterOperand(high)); 3856 } else if (op->IsShr()) { 3857 __ Asrs(o_h, high, 1); 3858 __ Rrx(o_l, low); 3859 } else { 3860 __ Lsrs(o_h, high, 1); 3861 __ Rrx(o_l, low); 3862 } 3863 } else { 3864 DCHECK(2 <= shift_value && shift_value < 32) << shift_value; 3865 if (op->IsShl()) { 3866 __ Lsl(o_h, high, shift_value); 3867 __ orr(o_h, o_h, ShifterOperand(low, LSR, 32 - shift_value)); 3868 __ Lsl(o_l, low, shift_value); 3869 } else if (op->IsShr()) { 3870 __ Lsr(o_l, low, shift_value); 3871 __ orr(o_l, o_l, ShifterOperand(high, LSL, 32 - shift_value)); 3872 __ Asr(o_h, high, shift_value); 3873 } else { 3874 __ Lsr(o_l, low, shift_value); 3875 __ orr(o_l, o_l, ShifterOperand(high, LSL, 32 - shift_value)); 3876 __ Lsr(o_h, high, shift_value); 3877 } 3878 } 3879 } 3880 break; 3881 } 3882 default: 3883 LOG(FATAL) << "Unexpected operation type " << type; 3884 UNREACHABLE(); 3885 } 3886} 3887 3888void LocationsBuilderARM::VisitShl(HShl* shl) { 3889 HandleShift(shl); 3890} 3891 3892void InstructionCodeGeneratorARM::VisitShl(HShl* shl) { 3893 HandleShift(shl); 3894} 3895 3896void LocationsBuilderARM::VisitShr(HShr* shr) { 3897 HandleShift(shr); 3898} 3899 3900void InstructionCodeGeneratorARM::VisitShr(HShr* shr) { 3901 HandleShift(shr); 3902} 3903 3904void LocationsBuilderARM::VisitUShr(HUShr* ushr) { 3905 HandleShift(ushr); 3906} 3907 3908void InstructionCodeGeneratorARM::VisitUShr(HUShr* ushr) { 3909 HandleShift(ushr); 3910} 3911 3912void LocationsBuilderARM::VisitNewInstance(HNewInstance* instruction) { 3913 LocationSummary* locations = 3914 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCallOnMainOnly); 3915 if (instruction->IsStringAlloc()) { 3916 locations->AddTemp(Location::RegisterLocation(kMethodRegisterArgument)); 3917 } else { 3918 InvokeRuntimeCallingConvention calling_convention; 3919 locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(0))); 3920 locations->SetInAt(1, Location::RegisterLocation(calling_convention.GetRegisterAt(1))); 3921 } 3922 locations->SetOut(Location::RegisterLocation(R0)); 3923} 3924 3925void InstructionCodeGeneratorARM::VisitNewInstance(HNewInstance* instruction) { 3926 // Note: if heap poisoning is enabled, the entry point takes cares 3927 // of poisoning the reference. 3928 if (instruction->IsStringAlloc()) { 3929 // String is allocated through StringFactory. Call NewEmptyString entry point. 3930 Register temp = instruction->GetLocations()->GetTemp(0).AsRegister<Register>(); 3931 MemberOffset code_offset = ArtMethod::EntryPointFromQuickCompiledCodeOffset(kArmPointerSize); 3932 __ LoadFromOffset(kLoadWord, temp, TR, QUICK_ENTRY_POINT(pNewEmptyString)); 3933 __ LoadFromOffset(kLoadWord, LR, temp, code_offset.Int32Value()); 3934 __ blx(LR); 3935 codegen_->RecordPcInfo(instruction, instruction->GetDexPc()); 3936 } else { 3937 codegen_->InvokeRuntime(instruction->GetEntrypoint(), instruction, instruction->GetDexPc()); 3938 CheckEntrypointTypes<kQuickAllocObjectWithAccessCheck, void*, uint32_t, ArtMethod*>(); 3939 } 3940} 3941 3942void LocationsBuilderARM::VisitNewArray(HNewArray* instruction) { 3943 LocationSummary* locations = 3944 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCallOnMainOnly); 3945 InvokeRuntimeCallingConvention calling_convention; 3946 locations->AddTemp(Location::RegisterLocation(calling_convention.GetRegisterAt(0))); 3947 locations->SetOut(Location::RegisterLocation(R0)); 3948 locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(1))); 3949 locations->SetInAt(1, Location::RegisterLocation(calling_convention.GetRegisterAt(2))); 3950} 3951 3952void InstructionCodeGeneratorARM::VisitNewArray(HNewArray* instruction) { 3953 InvokeRuntimeCallingConvention calling_convention; 3954 __ LoadImmediate(calling_convention.GetRegisterAt(0), instruction->GetTypeIndex()); 3955 // Note: if heap poisoning is enabled, the entry point takes cares 3956 // of poisoning the reference. 3957 codegen_->InvokeRuntime(instruction->GetEntrypoint(), instruction, instruction->GetDexPc()); 3958 CheckEntrypointTypes<kQuickAllocArrayWithAccessCheck, void*, uint32_t, int32_t, ArtMethod*>(); 3959} 3960 3961void LocationsBuilderARM::VisitParameterValue(HParameterValue* instruction) { 3962 LocationSummary* locations = 3963 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall); 3964 Location location = parameter_visitor_.GetNextLocation(instruction->GetType()); 3965 if (location.IsStackSlot()) { 3966 location = Location::StackSlot(location.GetStackIndex() + codegen_->GetFrameSize()); 3967 } else if (location.IsDoubleStackSlot()) { 3968 location = Location::DoubleStackSlot(location.GetStackIndex() + codegen_->GetFrameSize()); 3969 } 3970 locations->SetOut(location); 3971} 3972 3973void InstructionCodeGeneratorARM::VisitParameterValue( 3974 HParameterValue* instruction ATTRIBUTE_UNUSED) { 3975 // Nothing to do, the parameter is already at its location. 3976} 3977 3978void LocationsBuilderARM::VisitCurrentMethod(HCurrentMethod* instruction) { 3979 LocationSummary* locations = 3980 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall); 3981 locations->SetOut(Location::RegisterLocation(kMethodRegisterArgument)); 3982} 3983 3984void InstructionCodeGeneratorARM::VisitCurrentMethod(HCurrentMethod* instruction ATTRIBUTE_UNUSED) { 3985 // Nothing to do, the method is already at its location. 3986} 3987 3988void LocationsBuilderARM::VisitNot(HNot* not_) { 3989 LocationSummary* locations = 3990 new (GetGraph()->GetArena()) LocationSummary(not_, LocationSummary::kNoCall); 3991 locations->SetInAt(0, Location::RequiresRegister()); 3992 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 3993} 3994 3995void InstructionCodeGeneratorARM::VisitNot(HNot* not_) { 3996 LocationSummary* locations = not_->GetLocations(); 3997 Location out = locations->Out(); 3998 Location in = locations->InAt(0); 3999 switch (not_->GetResultType()) { 4000 case Primitive::kPrimInt: 4001 __ mvn(out.AsRegister<Register>(), ShifterOperand(in.AsRegister<Register>())); 4002 break; 4003 4004 case Primitive::kPrimLong: 4005 __ mvn(out.AsRegisterPairLow<Register>(), 4006 ShifterOperand(in.AsRegisterPairLow<Register>())); 4007 __ mvn(out.AsRegisterPairHigh<Register>(), 4008 ShifterOperand(in.AsRegisterPairHigh<Register>())); 4009 break; 4010 4011 default: 4012 LOG(FATAL) << "Unimplemented type for not operation " << not_->GetResultType(); 4013 } 4014} 4015 4016void LocationsBuilderARM::VisitBooleanNot(HBooleanNot* bool_not) { 4017 LocationSummary* locations = 4018 new (GetGraph()->GetArena()) LocationSummary(bool_not, LocationSummary::kNoCall); 4019 locations->SetInAt(0, Location::RequiresRegister()); 4020 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 4021} 4022 4023void InstructionCodeGeneratorARM::VisitBooleanNot(HBooleanNot* bool_not) { 4024 LocationSummary* locations = bool_not->GetLocations(); 4025 Location out = locations->Out(); 4026 Location in = locations->InAt(0); 4027 __ eor(out.AsRegister<Register>(), in.AsRegister<Register>(), ShifterOperand(1)); 4028} 4029 4030void LocationsBuilderARM::VisitCompare(HCompare* compare) { 4031 LocationSummary* locations = 4032 new (GetGraph()->GetArena()) LocationSummary(compare, LocationSummary::kNoCall); 4033 switch (compare->InputAt(0)->GetType()) { 4034 case Primitive::kPrimBoolean: 4035 case Primitive::kPrimByte: 4036 case Primitive::kPrimShort: 4037 case Primitive::kPrimChar: 4038 case Primitive::kPrimInt: 4039 case Primitive::kPrimLong: { 4040 locations->SetInAt(0, Location::RequiresRegister()); 4041 locations->SetInAt(1, Location::RequiresRegister()); 4042 // Output overlaps because it is written before doing the low comparison. 4043 locations->SetOut(Location::RequiresRegister(), Location::kOutputOverlap); 4044 break; 4045 } 4046 case Primitive::kPrimFloat: 4047 case Primitive::kPrimDouble: { 4048 locations->SetInAt(0, Location::RequiresFpuRegister()); 4049 locations->SetInAt(1, ArithmeticZeroOrFpuRegister(compare->InputAt(1))); 4050 locations->SetOut(Location::RequiresRegister()); 4051 break; 4052 } 4053 default: 4054 LOG(FATAL) << "Unexpected type for compare operation " << compare->InputAt(0)->GetType(); 4055 } 4056} 4057 4058void InstructionCodeGeneratorARM::VisitCompare(HCompare* compare) { 4059 LocationSummary* locations = compare->GetLocations(); 4060 Register out = locations->Out().AsRegister<Register>(); 4061 Location left = locations->InAt(0); 4062 Location right = locations->InAt(1); 4063 4064 Label less, greater, done; 4065 Primitive::Type type = compare->InputAt(0)->GetType(); 4066 Condition less_cond; 4067 switch (type) { 4068 case Primitive::kPrimBoolean: 4069 case Primitive::kPrimByte: 4070 case Primitive::kPrimShort: 4071 case Primitive::kPrimChar: 4072 case Primitive::kPrimInt: { 4073 __ LoadImmediate(out, 0); 4074 __ cmp(left.AsRegister<Register>(), 4075 ShifterOperand(right.AsRegister<Register>())); // Signed compare. 4076 less_cond = LT; 4077 break; 4078 } 4079 case Primitive::kPrimLong: { 4080 __ cmp(left.AsRegisterPairHigh<Register>(), 4081 ShifterOperand(right.AsRegisterPairHigh<Register>())); // Signed compare. 4082 __ b(&less, LT); 4083 __ b(&greater, GT); 4084 // Do LoadImmediate before the last `cmp`, as LoadImmediate might affect the status flags. 4085 __ LoadImmediate(out, 0); 4086 __ cmp(left.AsRegisterPairLow<Register>(), 4087 ShifterOperand(right.AsRegisterPairLow<Register>())); // Unsigned compare. 4088 less_cond = LO; 4089 break; 4090 } 4091 case Primitive::kPrimFloat: 4092 case Primitive::kPrimDouble: { 4093 __ LoadImmediate(out, 0); 4094 GenerateVcmp(compare); 4095 __ vmstat(); // transfer FP status register to ARM APSR. 4096 less_cond = ARMFPCondition(kCondLT, compare->IsGtBias()); 4097 break; 4098 } 4099 default: 4100 LOG(FATAL) << "Unexpected compare type " << type; 4101 UNREACHABLE(); 4102 } 4103 4104 __ b(&done, EQ); 4105 __ b(&less, less_cond); 4106 4107 __ Bind(&greater); 4108 __ LoadImmediate(out, 1); 4109 __ b(&done); 4110 4111 __ Bind(&less); 4112 __ LoadImmediate(out, -1); 4113 4114 __ Bind(&done); 4115} 4116 4117void LocationsBuilderARM::VisitPhi(HPhi* instruction) { 4118 LocationSummary* locations = 4119 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall); 4120 for (size_t i = 0, e = locations->GetInputCount(); i < e; ++i) { 4121 locations->SetInAt(i, Location::Any()); 4122 } 4123 locations->SetOut(Location::Any()); 4124} 4125 4126void InstructionCodeGeneratorARM::VisitPhi(HPhi* instruction ATTRIBUTE_UNUSED) { 4127 LOG(FATAL) << "Unreachable"; 4128} 4129 4130void CodeGeneratorARM::GenerateMemoryBarrier(MemBarrierKind kind) { 4131 // TODO (ported from quick): revisit ARM barrier kinds. 4132 DmbOptions flavor = DmbOptions::ISH; // Quiet C++ warnings. 4133 switch (kind) { 4134 case MemBarrierKind::kAnyStore: 4135 case MemBarrierKind::kLoadAny: 4136 case MemBarrierKind::kAnyAny: { 4137 flavor = DmbOptions::ISH; 4138 break; 4139 } 4140 case MemBarrierKind::kStoreStore: { 4141 flavor = DmbOptions::ISHST; 4142 break; 4143 } 4144 default: 4145 LOG(FATAL) << "Unexpected memory barrier " << kind; 4146 } 4147 __ dmb(flavor); 4148} 4149 4150void InstructionCodeGeneratorARM::GenerateWideAtomicLoad(Register addr, 4151 uint32_t offset, 4152 Register out_lo, 4153 Register out_hi) { 4154 if (offset != 0) { 4155 // Ensure `out_lo` is different from `addr`, so that loading 4156 // `offset` into `out_lo` does not clutter `addr`. 4157 DCHECK_NE(out_lo, addr); 4158 __ LoadImmediate(out_lo, offset); 4159 __ add(IP, addr, ShifterOperand(out_lo)); 4160 addr = IP; 4161 } 4162 __ ldrexd(out_lo, out_hi, addr); 4163} 4164 4165void InstructionCodeGeneratorARM::GenerateWideAtomicStore(Register addr, 4166 uint32_t offset, 4167 Register value_lo, 4168 Register value_hi, 4169 Register temp1, 4170 Register temp2, 4171 HInstruction* instruction) { 4172 Label fail; 4173 if (offset != 0) { 4174 __ LoadImmediate(temp1, offset); 4175 __ add(IP, addr, ShifterOperand(temp1)); 4176 addr = IP; 4177 } 4178 __ Bind(&fail); 4179 // We need a load followed by store. (The address used in a STREX instruction must 4180 // be the same as the address in the most recently executed LDREX instruction.) 4181 __ ldrexd(temp1, temp2, addr); 4182 codegen_->MaybeRecordImplicitNullCheck(instruction); 4183 __ strexd(temp1, value_lo, value_hi, addr); 4184 __ CompareAndBranchIfNonZero(temp1, &fail); 4185} 4186 4187void LocationsBuilderARM::HandleFieldSet(HInstruction* instruction, const FieldInfo& field_info) { 4188 DCHECK(instruction->IsInstanceFieldSet() || instruction->IsStaticFieldSet()); 4189 4190 LocationSummary* locations = 4191 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall); 4192 locations->SetInAt(0, Location::RequiresRegister()); 4193 4194 Primitive::Type field_type = field_info.GetFieldType(); 4195 if (Primitive::IsFloatingPointType(field_type)) { 4196 locations->SetInAt(1, Location::RequiresFpuRegister()); 4197 } else { 4198 locations->SetInAt(1, Location::RequiresRegister()); 4199 } 4200 4201 bool is_wide = field_type == Primitive::kPrimLong || field_type == Primitive::kPrimDouble; 4202 bool generate_volatile = field_info.IsVolatile() 4203 && is_wide 4204 && !codegen_->GetInstructionSetFeatures().HasAtomicLdrdAndStrd(); 4205 bool needs_write_barrier = 4206 CodeGenerator::StoreNeedsWriteBarrier(field_type, instruction->InputAt(1)); 4207 // Temporary registers for the write barrier. 4208 // TODO: consider renaming StoreNeedsWriteBarrier to StoreNeedsGCMark. 4209 if (needs_write_barrier) { 4210 locations->AddTemp(Location::RequiresRegister()); // Possibly used for reference poisoning too. 4211 locations->AddTemp(Location::RequiresRegister()); 4212 } else if (generate_volatile) { 4213 // ARM encoding have some additional constraints for ldrexd/strexd: 4214 // - registers need to be consecutive 4215 // - the first register should be even but not R14. 4216 // We don't test for ARM yet, and the assertion makes sure that we 4217 // revisit this if we ever enable ARM encoding. 4218 DCHECK_EQ(InstructionSet::kThumb2, codegen_->GetInstructionSet()); 4219 4220 locations->AddTemp(Location::RequiresRegister()); 4221 locations->AddTemp(Location::RequiresRegister()); 4222 if (field_type == Primitive::kPrimDouble) { 4223 // For doubles we need two more registers to copy the value. 4224 locations->AddTemp(Location::RegisterLocation(R2)); 4225 locations->AddTemp(Location::RegisterLocation(R3)); 4226 } 4227 } 4228} 4229 4230void InstructionCodeGeneratorARM::HandleFieldSet(HInstruction* instruction, 4231 const FieldInfo& field_info, 4232 bool value_can_be_null) { 4233 DCHECK(instruction->IsInstanceFieldSet() || instruction->IsStaticFieldSet()); 4234 4235 LocationSummary* locations = instruction->GetLocations(); 4236 Register base = locations->InAt(0).AsRegister<Register>(); 4237 Location value = locations->InAt(1); 4238 4239 bool is_volatile = field_info.IsVolatile(); 4240 bool atomic_ldrd_strd = codegen_->GetInstructionSetFeatures().HasAtomicLdrdAndStrd(); 4241 Primitive::Type field_type = field_info.GetFieldType(); 4242 uint32_t offset = field_info.GetFieldOffset().Uint32Value(); 4243 bool needs_write_barrier = 4244 CodeGenerator::StoreNeedsWriteBarrier(field_type, instruction->InputAt(1)); 4245 4246 if (is_volatile) { 4247 codegen_->GenerateMemoryBarrier(MemBarrierKind::kAnyStore); 4248 } 4249 4250 switch (field_type) { 4251 case Primitive::kPrimBoolean: 4252 case Primitive::kPrimByte: { 4253 __ StoreToOffset(kStoreByte, value.AsRegister<Register>(), base, offset); 4254 break; 4255 } 4256 4257 case Primitive::kPrimShort: 4258 case Primitive::kPrimChar: { 4259 __ StoreToOffset(kStoreHalfword, value.AsRegister<Register>(), base, offset); 4260 break; 4261 } 4262 4263 case Primitive::kPrimInt: 4264 case Primitive::kPrimNot: { 4265 if (kPoisonHeapReferences && needs_write_barrier) { 4266 // Note that in the case where `value` is a null reference, 4267 // we do not enter this block, as a null reference does not 4268 // need poisoning. 4269 DCHECK_EQ(field_type, Primitive::kPrimNot); 4270 Register temp = locations->GetTemp(0).AsRegister<Register>(); 4271 __ Mov(temp, value.AsRegister<Register>()); 4272 __ PoisonHeapReference(temp); 4273 __ StoreToOffset(kStoreWord, temp, base, offset); 4274 } else { 4275 __ StoreToOffset(kStoreWord, value.AsRegister<Register>(), base, offset); 4276 } 4277 break; 4278 } 4279 4280 case Primitive::kPrimLong: { 4281 if (is_volatile && !atomic_ldrd_strd) { 4282 GenerateWideAtomicStore(base, offset, 4283 value.AsRegisterPairLow<Register>(), 4284 value.AsRegisterPairHigh<Register>(), 4285 locations->GetTemp(0).AsRegister<Register>(), 4286 locations->GetTemp(1).AsRegister<Register>(), 4287 instruction); 4288 } else { 4289 __ StoreToOffset(kStoreWordPair, value.AsRegisterPairLow<Register>(), base, offset); 4290 codegen_->MaybeRecordImplicitNullCheck(instruction); 4291 } 4292 break; 4293 } 4294 4295 case Primitive::kPrimFloat: { 4296 __ StoreSToOffset(value.AsFpuRegister<SRegister>(), base, offset); 4297 break; 4298 } 4299 4300 case Primitive::kPrimDouble: { 4301 DRegister value_reg = FromLowSToD(value.AsFpuRegisterPairLow<SRegister>()); 4302 if (is_volatile && !atomic_ldrd_strd) { 4303 Register value_reg_lo = locations->GetTemp(0).AsRegister<Register>(); 4304 Register value_reg_hi = locations->GetTemp(1).AsRegister<Register>(); 4305 4306 __ vmovrrd(value_reg_lo, value_reg_hi, value_reg); 4307 4308 GenerateWideAtomicStore(base, offset, 4309 value_reg_lo, 4310 value_reg_hi, 4311 locations->GetTemp(2).AsRegister<Register>(), 4312 locations->GetTemp(3).AsRegister<Register>(), 4313 instruction); 4314 } else { 4315 __ StoreDToOffset(value_reg, base, offset); 4316 codegen_->MaybeRecordImplicitNullCheck(instruction); 4317 } 4318 break; 4319 } 4320 4321 case Primitive::kPrimVoid: 4322 LOG(FATAL) << "Unreachable type " << field_type; 4323 UNREACHABLE(); 4324 } 4325 4326 // Longs and doubles are handled in the switch. 4327 if (field_type != Primitive::kPrimLong && field_type != Primitive::kPrimDouble) { 4328 codegen_->MaybeRecordImplicitNullCheck(instruction); 4329 } 4330 4331 if (CodeGenerator::StoreNeedsWriteBarrier(field_type, instruction->InputAt(1))) { 4332 Register temp = locations->GetTemp(0).AsRegister<Register>(); 4333 Register card = locations->GetTemp(1).AsRegister<Register>(); 4334 codegen_->MarkGCCard( 4335 temp, card, base, value.AsRegister<Register>(), value_can_be_null); 4336 } 4337 4338 if (is_volatile) { 4339 codegen_->GenerateMemoryBarrier(MemBarrierKind::kAnyAny); 4340 } 4341} 4342 4343void LocationsBuilderARM::HandleFieldGet(HInstruction* instruction, const FieldInfo& field_info) { 4344 DCHECK(instruction->IsInstanceFieldGet() || instruction->IsStaticFieldGet()); 4345 4346 bool object_field_get_with_read_barrier = 4347 kEmitCompilerReadBarrier && (field_info.GetFieldType() == Primitive::kPrimNot); 4348 LocationSummary* locations = 4349 new (GetGraph()->GetArena()) LocationSummary(instruction, 4350 object_field_get_with_read_barrier ? 4351 LocationSummary::kCallOnSlowPath : 4352 LocationSummary::kNoCall); 4353 if (object_field_get_with_read_barrier && kUseBakerReadBarrier) { 4354 locations->SetCustomSlowPathCallerSaves(RegisterSet::Empty()); // No caller-save registers. 4355 } 4356 locations->SetInAt(0, Location::RequiresRegister()); 4357 4358 bool volatile_for_double = field_info.IsVolatile() 4359 && (field_info.GetFieldType() == Primitive::kPrimDouble) 4360 && !codegen_->GetInstructionSetFeatures().HasAtomicLdrdAndStrd(); 4361 // The output overlaps in case of volatile long: we don't want the 4362 // code generated by GenerateWideAtomicLoad to overwrite the 4363 // object's location. Likewise, in the case of an object field get 4364 // with read barriers enabled, we do not want the load to overwrite 4365 // the object's location, as we need it to emit the read barrier. 4366 bool overlap = (field_info.IsVolatile() && (field_info.GetFieldType() == Primitive::kPrimLong)) || 4367 object_field_get_with_read_barrier; 4368 4369 if (Primitive::IsFloatingPointType(instruction->GetType())) { 4370 locations->SetOut(Location::RequiresFpuRegister()); 4371 } else { 4372 locations->SetOut(Location::RequiresRegister(), 4373 (overlap ? Location::kOutputOverlap : Location::kNoOutputOverlap)); 4374 } 4375 if (volatile_for_double) { 4376 // ARM encoding have some additional constraints for ldrexd/strexd: 4377 // - registers need to be consecutive 4378 // - the first register should be even but not R14. 4379 // We don't test for ARM yet, and the assertion makes sure that we 4380 // revisit this if we ever enable ARM encoding. 4381 DCHECK_EQ(InstructionSet::kThumb2, codegen_->GetInstructionSet()); 4382 locations->AddTemp(Location::RequiresRegister()); 4383 locations->AddTemp(Location::RequiresRegister()); 4384 } else if (object_field_get_with_read_barrier && kUseBakerReadBarrier) { 4385 // We need a temporary register for the read barrier marking slow 4386 // path in CodeGeneratorARM::GenerateFieldLoadWithBakerReadBarrier. 4387 locations->AddTemp(Location::RequiresRegister()); 4388 } 4389} 4390 4391Location LocationsBuilderARM::ArithmeticZeroOrFpuRegister(HInstruction* input) { 4392 DCHECK(input->GetType() == Primitive::kPrimDouble || input->GetType() == Primitive::kPrimFloat) 4393 << input->GetType(); 4394 if ((input->IsFloatConstant() && (input->AsFloatConstant()->IsArithmeticZero())) || 4395 (input->IsDoubleConstant() && (input->AsDoubleConstant()->IsArithmeticZero()))) { 4396 return Location::ConstantLocation(input->AsConstant()); 4397 } else { 4398 return Location::RequiresFpuRegister(); 4399 } 4400} 4401 4402Location LocationsBuilderARM::ArmEncodableConstantOrRegister(HInstruction* constant, 4403 Opcode opcode) { 4404 DCHECK(!Primitive::IsFloatingPointType(constant->GetType())); 4405 if (constant->IsConstant() && 4406 CanEncodeConstantAsImmediate(constant->AsConstant(), opcode)) { 4407 return Location::ConstantLocation(constant->AsConstant()); 4408 } 4409 return Location::RequiresRegister(); 4410} 4411 4412bool LocationsBuilderARM::CanEncodeConstantAsImmediate(HConstant* input_cst, 4413 Opcode opcode) { 4414 uint64_t value = static_cast<uint64_t>(Int64FromConstant(input_cst)); 4415 if (Primitive::Is64BitType(input_cst->GetType())) { 4416 Opcode high_opcode = opcode; 4417 SetCc low_set_cc = kCcDontCare; 4418 switch (opcode) { 4419 case SUB: 4420 // Flip the operation to an ADD. 4421 value = -value; 4422 opcode = ADD; 4423 FALLTHROUGH_INTENDED; 4424 case ADD: 4425 if (Low32Bits(value) == 0u) { 4426 return CanEncodeConstantAsImmediate(High32Bits(value), opcode, kCcDontCare); 4427 } 4428 high_opcode = ADC; 4429 low_set_cc = kCcSet; 4430 break; 4431 default: 4432 break; 4433 } 4434 return CanEncodeConstantAsImmediate(Low32Bits(value), opcode, low_set_cc) && 4435 CanEncodeConstantAsImmediate(High32Bits(value), high_opcode, kCcDontCare); 4436 } else { 4437 return CanEncodeConstantAsImmediate(Low32Bits(value), opcode); 4438 } 4439} 4440 4441bool LocationsBuilderARM::CanEncodeConstantAsImmediate(uint32_t value, 4442 Opcode opcode, 4443 SetCc set_cc) { 4444 ShifterOperand so; 4445 ArmAssembler* assembler = codegen_->GetAssembler(); 4446 if (assembler->ShifterOperandCanHold(kNoRegister, kNoRegister, opcode, value, set_cc, &so)) { 4447 return true; 4448 } 4449 Opcode neg_opcode = kNoOperand; 4450 switch (opcode) { 4451 case AND: neg_opcode = BIC; value = ~value; break; 4452 case ORR: neg_opcode = ORN; value = ~value; break; 4453 case ADD: neg_opcode = SUB; value = -value; break; 4454 case ADC: neg_opcode = SBC; value = ~value; break; 4455 case SUB: neg_opcode = ADD; value = -value; break; 4456 case SBC: neg_opcode = ADC; value = ~value; break; 4457 default: 4458 return false; 4459 } 4460 return assembler->ShifterOperandCanHold(kNoRegister, kNoRegister, neg_opcode, value, set_cc, &so); 4461} 4462 4463void InstructionCodeGeneratorARM::HandleFieldGet(HInstruction* instruction, 4464 const FieldInfo& field_info) { 4465 DCHECK(instruction->IsInstanceFieldGet() || instruction->IsStaticFieldGet()); 4466 4467 LocationSummary* locations = instruction->GetLocations(); 4468 Location base_loc = locations->InAt(0); 4469 Register base = base_loc.AsRegister<Register>(); 4470 Location out = locations->Out(); 4471 bool is_volatile = field_info.IsVolatile(); 4472 bool atomic_ldrd_strd = codegen_->GetInstructionSetFeatures().HasAtomicLdrdAndStrd(); 4473 Primitive::Type field_type = field_info.GetFieldType(); 4474 uint32_t offset = field_info.GetFieldOffset().Uint32Value(); 4475 4476 switch (field_type) { 4477 case Primitive::kPrimBoolean: 4478 __ LoadFromOffset(kLoadUnsignedByte, out.AsRegister<Register>(), base, offset); 4479 break; 4480 4481 case Primitive::kPrimByte: 4482 __ LoadFromOffset(kLoadSignedByte, out.AsRegister<Register>(), base, offset); 4483 break; 4484 4485 case Primitive::kPrimShort: 4486 __ LoadFromOffset(kLoadSignedHalfword, out.AsRegister<Register>(), base, offset); 4487 break; 4488 4489 case Primitive::kPrimChar: 4490 __ LoadFromOffset(kLoadUnsignedHalfword, out.AsRegister<Register>(), base, offset); 4491 break; 4492 4493 case Primitive::kPrimInt: 4494 __ LoadFromOffset(kLoadWord, out.AsRegister<Register>(), base, offset); 4495 break; 4496 4497 case Primitive::kPrimNot: { 4498 // /* HeapReference<Object> */ out = *(base + offset) 4499 if (kEmitCompilerReadBarrier && kUseBakerReadBarrier) { 4500 Location temp_loc = locations->GetTemp(0); 4501 // Note that a potential implicit null check is handled in this 4502 // CodeGeneratorARM::GenerateFieldLoadWithBakerReadBarrier call. 4503 codegen_->GenerateFieldLoadWithBakerReadBarrier( 4504 instruction, out, base, offset, temp_loc, /* needs_null_check */ true); 4505 if (is_volatile) { 4506 codegen_->GenerateMemoryBarrier(MemBarrierKind::kLoadAny); 4507 } 4508 } else { 4509 __ LoadFromOffset(kLoadWord, out.AsRegister<Register>(), base, offset); 4510 codegen_->MaybeRecordImplicitNullCheck(instruction); 4511 if (is_volatile) { 4512 codegen_->GenerateMemoryBarrier(MemBarrierKind::kLoadAny); 4513 } 4514 // If read barriers are enabled, emit read barriers other than 4515 // Baker's using a slow path (and also unpoison the loaded 4516 // reference, if heap poisoning is enabled). 4517 codegen_->MaybeGenerateReadBarrierSlow(instruction, out, out, base_loc, offset); 4518 } 4519 break; 4520 } 4521 4522 case Primitive::kPrimLong: 4523 if (is_volatile && !atomic_ldrd_strd) { 4524 GenerateWideAtomicLoad(base, offset, 4525 out.AsRegisterPairLow<Register>(), 4526 out.AsRegisterPairHigh<Register>()); 4527 } else { 4528 __ LoadFromOffset(kLoadWordPair, out.AsRegisterPairLow<Register>(), base, offset); 4529 } 4530 break; 4531 4532 case Primitive::kPrimFloat: 4533 __ LoadSFromOffset(out.AsFpuRegister<SRegister>(), base, offset); 4534 break; 4535 4536 case Primitive::kPrimDouble: { 4537 DRegister out_reg = FromLowSToD(out.AsFpuRegisterPairLow<SRegister>()); 4538 if (is_volatile && !atomic_ldrd_strd) { 4539 Register lo = locations->GetTemp(0).AsRegister<Register>(); 4540 Register hi = locations->GetTemp(1).AsRegister<Register>(); 4541 GenerateWideAtomicLoad(base, offset, lo, hi); 4542 codegen_->MaybeRecordImplicitNullCheck(instruction); 4543 __ vmovdrr(out_reg, lo, hi); 4544 } else { 4545 __ LoadDFromOffset(out_reg, base, offset); 4546 codegen_->MaybeRecordImplicitNullCheck(instruction); 4547 } 4548 break; 4549 } 4550 4551 case Primitive::kPrimVoid: 4552 LOG(FATAL) << "Unreachable type " << field_type; 4553 UNREACHABLE(); 4554 } 4555 4556 if (field_type == Primitive::kPrimNot || field_type == Primitive::kPrimDouble) { 4557 // Potential implicit null checks, in the case of reference or 4558 // double fields, are handled in the previous switch statement. 4559 } else { 4560 codegen_->MaybeRecordImplicitNullCheck(instruction); 4561 } 4562 4563 if (is_volatile) { 4564 if (field_type == Primitive::kPrimNot) { 4565 // Memory barriers, in the case of references, are also handled 4566 // in the previous switch statement. 4567 } else { 4568 codegen_->GenerateMemoryBarrier(MemBarrierKind::kLoadAny); 4569 } 4570 } 4571} 4572 4573void LocationsBuilderARM::VisitInstanceFieldSet(HInstanceFieldSet* instruction) { 4574 HandleFieldSet(instruction, instruction->GetFieldInfo()); 4575} 4576 4577void InstructionCodeGeneratorARM::VisitInstanceFieldSet(HInstanceFieldSet* instruction) { 4578 HandleFieldSet(instruction, instruction->GetFieldInfo(), instruction->GetValueCanBeNull()); 4579} 4580 4581void LocationsBuilderARM::VisitInstanceFieldGet(HInstanceFieldGet* instruction) { 4582 HandleFieldGet(instruction, instruction->GetFieldInfo()); 4583} 4584 4585void InstructionCodeGeneratorARM::VisitInstanceFieldGet(HInstanceFieldGet* instruction) { 4586 HandleFieldGet(instruction, instruction->GetFieldInfo()); 4587} 4588 4589void LocationsBuilderARM::VisitStaticFieldGet(HStaticFieldGet* instruction) { 4590 HandleFieldGet(instruction, instruction->GetFieldInfo()); 4591} 4592 4593void InstructionCodeGeneratorARM::VisitStaticFieldGet(HStaticFieldGet* instruction) { 4594 HandleFieldGet(instruction, instruction->GetFieldInfo()); 4595} 4596 4597void LocationsBuilderARM::VisitStaticFieldSet(HStaticFieldSet* instruction) { 4598 HandleFieldSet(instruction, instruction->GetFieldInfo()); 4599} 4600 4601void InstructionCodeGeneratorARM::VisitStaticFieldSet(HStaticFieldSet* instruction) { 4602 HandleFieldSet(instruction, instruction->GetFieldInfo(), instruction->GetValueCanBeNull()); 4603} 4604 4605void LocationsBuilderARM::VisitUnresolvedInstanceFieldGet( 4606 HUnresolvedInstanceFieldGet* instruction) { 4607 FieldAccessCallingConventionARM calling_convention; 4608 codegen_->CreateUnresolvedFieldLocationSummary( 4609 instruction, instruction->GetFieldType(), calling_convention); 4610} 4611 4612void InstructionCodeGeneratorARM::VisitUnresolvedInstanceFieldGet( 4613 HUnresolvedInstanceFieldGet* instruction) { 4614 FieldAccessCallingConventionARM calling_convention; 4615 codegen_->GenerateUnresolvedFieldAccess(instruction, 4616 instruction->GetFieldType(), 4617 instruction->GetFieldIndex(), 4618 instruction->GetDexPc(), 4619 calling_convention); 4620} 4621 4622void LocationsBuilderARM::VisitUnresolvedInstanceFieldSet( 4623 HUnresolvedInstanceFieldSet* instruction) { 4624 FieldAccessCallingConventionARM calling_convention; 4625 codegen_->CreateUnresolvedFieldLocationSummary( 4626 instruction, instruction->GetFieldType(), calling_convention); 4627} 4628 4629void InstructionCodeGeneratorARM::VisitUnresolvedInstanceFieldSet( 4630 HUnresolvedInstanceFieldSet* instruction) { 4631 FieldAccessCallingConventionARM calling_convention; 4632 codegen_->GenerateUnresolvedFieldAccess(instruction, 4633 instruction->GetFieldType(), 4634 instruction->GetFieldIndex(), 4635 instruction->GetDexPc(), 4636 calling_convention); 4637} 4638 4639void LocationsBuilderARM::VisitUnresolvedStaticFieldGet( 4640 HUnresolvedStaticFieldGet* instruction) { 4641 FieldAccessCallingConventionARM calling_convention; 4642 codegen_->CreateUnresolvedFieldLocationSummary( 4643 instruction, instruction->GetFieldType(), calling_convention); 4644} 4645 4646void InstructionCodeGeneratorARM::VisitUnresolvedStaticFieldGet( 4647 HUnresolvedStaticFieldGet* instruction) { 4648 FieldAccessCallingConventionARM calling_convention; 4649 codegen_->GenerateUnresolvedFieldAccess(instruction, 4650 instruction->GetFieldType(), 4651 instruction->GetFieldIndex(), 4652 instruction->GetDexPc(), 4653 calling_convention); 4654} 4655 4656void LocationsBuilderARM::VisitUnresolvedStaticFieldSet( 4657 HUnresolvedStaticFieldSet* instruction) { 4658 FieldAccessCallingConventionARM calling_convention; 4659 codegen_->CreateUnresolvedFieldLocationSummary( 4660 instruction, instruction->GetFieldType(), calling_convention); 4661} 4662 4663void InstructionCodeGeneratorARM::VisitUnresolvedStaticFieldSet( 4664 HUnresolvedStaticFieldSet* instruction) { 4665 FieldAccessCallingConventionARM calling_convention; 4666 codegen_->GenerateUnresolvedFieldAccess(instruction, 4667 instruction->GetFieldType(), 4668 instruction->GetFieldIndex(), 4669 instruction->GetDexPc(), 4670 calling_convention); 4671} 4672 4673void LocationsBuilderARM::VisitNullCheck(HNullCheck* instruction) { 4674 LocationSummary* locations = codegen_->CreateThrowingSlowPathLocations(instruction); 4675 locations->SetInAt(0, Location::RequiresRegister()); 4676} 4677 4678void CodeGeneratorARM::GenerateImplicitNullCheck(HNullCheck* instruction) { 4679 if (CanMoveNullCheckToUser(instruction)) { 4680 return; 4681 } 4682 Location obj = instruction->GetLocations()->InAt(0); 4683 4684 __ LoadFromOffset(kLoadWord, IP, obj.AsRegister<Register>(), 0); 4685 RecordPcInfo(instruction, instruction->GetDexPc()); 4686} 4687 4688void CodeGeneratorARM::GenerateExplicitNullCheck(HNullCheck* instruction) { 4689 SlowPathCodeARM* slow_path = new (GetGraph()->GetArena()) NullCheckSlowPathARM(instruction); 4690 AddSlowPath(slow_path); 4691 4692 LocationSummary* locations = instruction->GetLocations(); 4693 Location obj = locations->InAt(0); 4694 4695 __ CompareAndBranchIfZero(obj.AsRegister<Register>(), slow_path->GetEntryLabel()); 4696} 4697 4698void InstructionCodeGeneratorARM::VisitNullCheck(HNullCheck* instruction) { 4699 codegen_->GenerateNullCheck(instruction); 4700} 4701 4702static LoadOperandType GetLoadOperandType(Primitive::Type type) { 4703 switch (type) { 4704 case Primitive::kPrimNot: 4705 return kLoadWord; 4706 case Primitive::kPrimBoolean: 4707 return kLoadUnsignedByte; 4708 case Primitive::kPrimByte: 4709 return kLoadSignedByte; 4710 case Primitive::kPrimChar: 4711 return kLoadUnsignedHalfword; 4712 case Primitive::kPrimShort: 4713 return kLoadSignedHalfword; 4714 case Primitive::kPrimInt: 4715 return kLoadWord; 4716 case Primitive::kPrimLong: 4717 return kLoadWordPair; 4718 case Primitive::kPrimFloat: 4719 return kLoadSWord; 4720 case Primitive::kPrimDouble: 4721 return kLoadDWord; 4722 default: 4723 LOG(FATAL) << "Unreachable type " << type; 4724 UNREACHABLE(); 4725 } 4726} 4727 4728static StoreOperandType GetStoreOperandType(Primitive::Type type) { 4729 switch (type) { 4730 case Primitive::kPrimNot: 4731 return kStoreWord; 4732 case Primitive::kPrimBoolean: 4733 case Primitive::kPrimByte: 4734 return kStoreByte; 4735 case Primitive::kPrimChar: 4736 case Primitive::kPrimShort: 4737 return kStoreHalfword; 4738 case Primitive::kPrimInt: 4739 return kStoreWord; 4740 case Primitive::kPrimLong: 4741 return kStoreWordPair; 4742 case Primitive::kPrimFloat: 4743 return kStoreSWord; 4744 case Primitive::kPrimDouble: 4745 return kStoreDWord; 4746 default: 4747 LOG(FATAL) << "Unreachable type " << type; 4748 UNREACHABLE(); 4749 } 4750} 4751 4752void CodeGeneratorARM::LoadFromShiftedRegOffset(Primitive::Type type, 4753 Location out_loc, 4754 Register base, 4755 Register reg_offset, 4756 Condition cond) { 4757 uint32_t shift_count = Primitive::ComponentSizeShift(type); 4758 Address mem_address(base, reg_offset, Shift::LSL, shift_count); 4759 4760 switch (type) { 4761 case Primitive::kPrimByte: 4762 __ ldrsb(out_loc.AsRegister<Register>(), mem_address, cond); 4763 break; 4764 case Primitive::kPrimBoolean: 4765 __ ldrb(out_loc.AsRegister<Register>(), mem_address, cond); 4766 break; 4767 case Primitive::kPrimShort: 4768 __ ldrsh(out_loc.AsRegister<Register>(), mem_address, cond); 4769 break; 4770 case Primitive::kPrimChar: 4771 __ ldrh(out_loc.AsRegister<Register>(), mem_address, cond); 4772 break; 4773 case Primitive::kPrimNot: 4774 case Primitive::kPrimInt: 4775 __ ldr(out_loc.AsRegister<Register>(), mem_address, cond); 4776 break; 4777 // T32 doesn't support LoadFromShiftedRegOffset mem address mode for these types. 4778 case Primitive::kPrimLong: 4779 case Primitive::kPrimFloat: 4780 case Primitive::kPrimDouble: 4781 default: 4782 LOG(FATAL) << "Unreachable type " << type; 4783 UNREACHABLE(); 4784 } 4785} 4786 4787void CodeGeneratorARM::StoreToShiftedRegOffset(Primitive::Type type, 4788 Location loc, 4789 Register base, 4790 Register reg_offset, 4791 Condition cond) { 4792 uint32_t shift_count = Primitive::ComponentSizeShift(type); 4793 Address mem_address(base, reg_offset, Shift::LSL, shift_count); 4794 4795 switch (type) { 4796 case Primitive::kPrimByte: 4797 case Primitive::kPrimBoolean: 4798 __ strb(loc.AsRegister<Register>(), mem_address, cond); 4799 break; 4800 case Primitive::kPrimShort: 4801 case Primitive::kPrimChar: 4802 __ strh(loc.AsRegister<Register>(), mem_address, cond); 4803 break; 4804 case Primitive::kPrimNot: 4805 case Primitive::kPrimInt: 4806 __ str(loc.AsRegister<Register>(), mem_address, cond); 4807 break; 4808 // T32 doesn't support StoreToShiftedRegOffset mem address mode for these types. 4809 case Primitive::kPrimLong: 4810 case Primitive::kPrimFloat: 4811 case Primitive::kPrimDouble: 4812 default: 4813 LOG(FATAL) << "Unreachable type " << type; 4814 UNREACHABLE(); 4815 } 4816} 4817 4818void LocationsBuilderARM::VisitArrayGet(HArrayGet* instruction) { 4819 bool object_array_get_with_read_barrier = 4820 kEmitCompilerReadBarrier && (instruction->GetType() == Primitive::kPrimNot); 4821 LocationSummary* locations = 4822 new (GetGraph()->GetArena()) LocationSummary(instruction, 4823 object_array_get_with_read_barrier ? 4824 LocationSummary::kCallOnSlowPath : 4825 LocationSummary::kNoCall); 4826 if (object_array_get_with_read_barrier && kUseBakerReadBarrier) { 4827 locations->SetCustomSlowPathCallerSaves(RegisterSet::Empty()); // No caller-save registers. 4828 } 4829 locations->SetInAt(0, Location::RequiresRegister()); 4830 locations->SetInAt(1, Location::RegisterOrConstant(instruction->InputAt(1))); 4831 if (Primitive::IsFloatingPointType(instruction->GetType())) { 4832 locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap); 4833 } else { 4834 // The output overlaps in the case of an object array get with 4835 // read barriers enabled: we do not want the move to overwrite the 4836 // array's location, as we need it to emit the read barrier. 4837 locations->SetOut( 4838 Location::RequiresRegister(), 4839 object_array_get_with_read_barrier ? Location::kOutputOverlap : Location::kNoOutputOverlap); 4840 } 4841 // We need a temporary register for the read barrier marking slow 4842 // path in CodeGeneratorARM::GenerateArrayLoadWithBakerReadBarrier. 4843 // Also need for String compression feature. 4844 if ((object_array_get_with_read_barrier && kUseBakerReadBarrier) 4845 || (mirror::kUseStringCompression && instruction->IsStringCharAt())) { 4846 locations->AddTemp(Location::RequiresRegister()); 4847 } 4848} 4849 4850void InstructionCodeGeneratorARM::VisitArrayGet(HArrayGet* instruction) { 4851 LocationSummary* locations = instruction->GetLocations(); 4852 Location obj_loc = locations->InAt(0); 4853 Register obj = obj_loc.AsRegister<Register>(); 4854 Location index = locations->InAt(1); 4855 Location out_loc = locations->Out(); 4856 uint32_t data_offset = CodeGenerator::GetArrayDataOffset(instruction); 4857 Primitive::Type type = instruction->GetType(); 4858 const bool maybe_compressed_char_at = mirror::kUseStringCompression && 4859 instruction->IsStringCharAt(); 4860 HInstruction* array_instr = instruction->GetArray(); 4861 bool has_intermediate_address = array_instr->IsIntermediateAddress(); 4862 4863 switch (type) { 4864 case Primitive::kPrimBoolean: 4865 case Primitive::kPrimByte: 4866 case Primitive::kPrimShort: 4867 case Primitive::kPrimChar: 4868 case Primitive::kPrimInt: { 4869 Register length; 4870 if (maybe_compressed_char_at) { 4871 length = locations->GetTemp(0).AsRegister<Register>(); 4872 uint32_t count_offset = mirror::String::CountOffset().Uint32Value(); 4873 __ LoadFromOffset(kLoadWord, length, obj, count_offset); 4874 codegen_->MaybeRecordImplicitNullCheck(instruction); 4875 } 4876 if (index.IsConstant()) { 4877 int32_t const_index = index.GetConstant()->AsIntConstant()->GetValue(); 4878 if (maybe_compressed_char_at) { 4879 Label uncompressed_load, done; 4880 __ Lsrs(length, length, 1u); // LSRS has a 16-bit encoding, TST (immediate) does not. 4881 static_assert(static_cast<uint32_t>(mirror::StringCompressionFlag::kCompressed) == 0u, 4882 "Expecting 0=compressed, 1=uncompressed"); 4883 __ b(&uncompressed_load, CS); 4884 __ LoadFromOffset(kLoadUnsignedByte, 4885 out_loc.AsRegister<Register>(), 4886 obj, 4887 data_offset + const_index); 4888 __ b(&done); 4889 __ Bind(&uncompressed_load); 4890 __ LoadFromOffset(GetLoadOperandType(Primitive::kPrimChar), 4891 out_loc.AsRegister<Register>(), 4892 obj, 4893 data_offset + (const_index << 1)); 4894 __ Bind(&done); 4895 } else { 4896 uint32_t full_offset = data_offset + (const_index << Primitive::ComponentSizeShift(type)); 4897 4898 LoadOperandType load_type = GetLoadOperandType(type); 4899 __ LoadFromOffset(load_type, out_loc.AsRegister<Register>(), obj, full_offset); 4900 } 4901 } else { 4902 Register temp = IP; 4903 4904 if (has_intermediate_address) { 4905 // We do not need to compute the intermediate address from the array: the 4906 // input instruction has done it already. See the comment in 4907 // `TryExtractArrayAccessAddress()`. 4908 if (kIsDebugBuild) { 4909 HIntermediateAddress* tmp = array_instr->AsIntermediateAddress(); 4910 DCHECK_EQ(tmp->GetOffset()->AsIntConstant()->GetValueAsUint64(), data_offset); 4911 } 4912 temp = obj; 4913 } else { 4914 __ add(temp, obj, ShifterOperand(data_offset)); 4915 } 4916 if (maybe_compressed_char_at) { 4917 Label uncompressed_load, done; 4918 __ Lsrs(length, length, 1u); // LSRS has a 16-bit encoding, TST (immediate) does not. 4919 static_assert(static_cast<uint32_t>(mirror::StringCompressionFlag::kCompressed) == 0u, 4920 "Expecting 0=compressed, 1=uncompressed"); 4921 __ b(&uncompressed_load, CS); 4922 __ ldrb(out_loc.AsRegister<Register>(), 4923 Address(temp, index.AsRegister<Register>(), Shift::LSL, 0)); 4924 __ b(&done); 4925 __ Bind(&uncompressed_load); 4926 __ ldrh(out_loc.AsRegister<Register>(), 4927 Address(temp, index.AsRegister<Register>(), Shift::LSL, 1)); 4928 __ Bind(&done); 4929 } else { 4930 codegen_->LoadFromShiftedRegOffset(type, out_loc, temp, index.AsRegister<Register>()); 4931 } 4932 } 4933 break; 4934 } 4935 4936 case Primitive::kPrimNot: { 4937 // The read barrier instrumentation of object ArrayGet 4938 // instructions does not support the HIntermediateAddress 4939 // instruction. 4940 DCHECK(!(has_intermediate_address && kEmitCompilerReadBarrier)); 4941 4942 static_assert( 4943 sizeof(mirror::HeapReference<mirror::Object>) == sizeof(int32_t), 4944 "art::mirror::HeapReference<art::mirror::Object> and int32_t have different sizes."); 4945 // /* HeapReference<Object> */ out = 4946 // *(obj + data_offset + index * sizeof(HeapReference<Object>)) 4947 if (kEmitCompilerReadBarrier && kUseBakerReadBarrier) { 4948 Location temp = locations->GetTemp(0); 4949 // Note that a potential implicit null check is handled in this 4950 // CodeGeneratorARM::GenerateArrayLoadWithBakerReadBarrier call. 4951 codegen_->GenerateArrayLoadWithBakerReadBarrier( 4952 instruction, out_loc, obj, data_offset, index, temp, /* needs_null_check */ true); 4953 } else { 4954 Register out = out_loc.AsRegister<Register>(); 4955 if (index.IsConstant()) { 4956 size_t offset = 4957 (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4) + data_offset; 4958 __ LoadFromOffset(kLoadWord, out, obj, offset); 4959 codegen_->MaybeRecordImplicitNullCheck(instruction); 4960 // If read barriers are enabled, emit read barriers other than 4961 // Baker's using a slow path (and also unpoison the loaded 4962 // reference, if heap poisoning is enabled). 4963 codegen_->MaybeGenerateReadBarrierSlow(instruction, out_loc, out_loc, obj_loc, offset); 4964 } else { 4965 Register temp = IP; 4966 4967 if (has_intermediate_address) { 4968 // We do not need to compute the intermediate address from the array: the 4969 // input instruction has done it already. See the comment in 4970 // `TryExtractArrayAccessAddress()`. 4971 if (kIsDebugBuild) { 4972 HIntermediateAddress* tmp = array_instr->AsIntermediateAddress(); 4973 DCHECK_EQ(tmp->GetOffset()->AsIntConstant()->GetValueAsUint64(), data_offset); 4974 } 4975 temp = obj; 4976 } else { 4977 __ add(temp, obj, ShifterOperand(data_offset)); 4978 } 4979 codegen_->LoadFromShiftedRegOffset(type, out_loc, temp, index.AsRegister<Register>()); 4980 4981 codegen_->MaybeRecordImplicitNullCheck(instruction); 4982 // If read barriers are enabled, emit read barriers other than 4983 // Baker's using a slow path (and also unpoison the loaded 4984 // reference, if heap poisoning is enabled). 4985 codegen_->MaybeGenerateReadBarrierSlow( 4986 instruction, out_loc, out_loc, obj_loc, data_offset, index); 4987 } 4988 } 4989 break; 4990 } 4991 4992 case Primitive::kPrimLong: { 4993 if (index.IsConstant()) { 4994 size_t offset = 4995 (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_8) + data_offset; 4996 __ LoadFromOffset(kLoadWordPair, out_loc.AsRegisterPairLow<Register>(), obj, offset); 4997 } else { 4998 __ add(IP, obj, ShifterOperand(index.AsRegister<Register>(), LSL, TIMES_8)); 4999 __ LoadFromOffset(kLoadWordPair, out_loc.AsRegisterPairLow<Register>(), IP, data_offset); 5000 } 5001 break; 5002 } 5003 5004 case Primitive::kPrimFloat: { 5005 SRegister out = out_loc.AsFpuRegister<SRegister>(); 5006 if (index.IsConstant()) { 5007 size_t offset = (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4) + data_offset; 5008 __ LoadSFromOffset(out, obj, offset); 5009 } else { 5010 __ add(IP, obj, ShifterOperand(index.AsRegister<Register>(), LSL, TIMES_4)); 5011 __ LoadSFromOffset(out, IP, data_offset); 5012 } 5013 break; 5014 } 5015 5016 case Primitive::kPrimDouble: { 5017 SRegister out = out_loc.AsFpuRegisterPairLow<SRegister>(); 5018 if (index.IsConstant()) { 5019 size_t offset = (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_8) + data_offset; 5020 __ LoadDFromOffset(FromLowSToD(out), obj, offset); 5021 } else { 5022 __ add(IP, obj, ShifterOperand(index.AsRegister<Register>(), LSL, TIMES_8)); 5023 __ LoadDFromOffset(FromLowSToD(out), IP, data_offset); 5024 } 5025 break; 5026 } 5027 5028 case Primitive::kPrimVoid: 5029 LOG(FATAL) << "Unreachable type " << type; 5030 UNREACHABLE(); 5031 } 5032 5033 if (type == Primitive::kPrimNot) { 5034 // Potential implicit null checks, in the case of reference 5035 // arrays, are handled in the previous switch statement. 5036 } else if (!maybe_compressed_char_at) { 5037 codegen_->MaybeRecordImplicitNullCheck(instruction); 5038 } 5039} 5040 5041void LocationsBuilderARM::VisitArraySet(HArraySet* instruction) { 5042 Primitive::Type value_type = instruction->GetComponentType(); 5043 5044 bool needs_write_barrier = 5045 CodeGenerator::StoreNeedsWriteBarrier(value_type, instruction->GetValue()); 5046 bool may_need_runtime_call_for_type_check = instruction->NeedsTypeCheck(); 5047 5048 LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary( 5049 instruction, 5050 may_need_runtime_call_for_type_check ? 5051 LocationSummary::kCallOnSlowPath : 5052 LocationSummary::kNoCall); 5053 5054 locations->SetInAt(0, Location::RequiresRegister()); 5055 locations->SetInAt(1, Location::RegisterOrConstant(instruction->InputAt(1))); 5056 if (Primitive::IsFloatingPointType(value_type)) { 5057 locations->SetInAt(2, Location::RequiresFpuRegister()); 5058 } else { 5059 locations->SetInAt(2, Location::RequiresRegister()); 5060 } 5061 if (needs_write_barrier) { 5062 // Temporary registers for the write barrier. 5063 locations->AddTemp(Location::RequiresRegister()); // Possibly used for ref. poisoning too. 5064 locations->AddTemp(Location::RequiresRegister()); 5065 } 5066} 5067 5068void InstructionCodeGeneratorARM::VisitArraySet(HArraySet* instruction) { 5069 LocationSummary* locations = instruction->GetLocations(); 5070 Location array_loc = locations->InAt(0); 5071 Register array = array_loc.AsRegister<Register>(); 5072 Location index = locations->InAt(1); 5073 Primitive::Type value_type = instruction->GetComponentType(); 5074 bool may_need_runtime_call_for_type_check = instruction->NeedsTypeCheck(); 5075 bool needs_write_barrier = 5076 CodeGenerator::StoreNeedsWriteBarrier(value_type, instruction->GetValue()); 5077 uint32_t data_offset = 5078 mirror::Array::DataOffset(Primitive::ComponentSize(value_type)).Uint32Value(); 5079 Location value_loc = locations->InAt(2); 5080 HInstruction* array_instr = instruction->GetArray(); 5081 bool has_intermediate_address = array_instr->IsIntermediateAddress(); 5082 5083 switch (value_type) { 5084 case Primitive::kPrimBoolean: 5085 case Primitive::kPrimByte: 5086 case Primitive::kPrimShort: 5087 case Primitive::kPrimChar: 5088 case Primitive::kPrimInt: { 5089 if (index.IsConstant()) { 5090 int32_t const_index = index.GetConstant()->AsIntConstant()->GetValue(); 5091 uint32_t full_offset = 5092 data_offset + (const_index << Primitive::ComponentSizeShift(value_type)); 5093 StoreOperandType store_type = GetStoreOperandType(value_type); 5094 __ StoreToOffset(store_type, value_loc.AsRegister<Register>(), array, full_offset); 5095 } else { 5096 Register temp = IP; 5097 5098 if (has_intermediate_address) { 5099 // We do not need to compute the intermediate address from the array: the 5100 // input instruction has done it already. See the comment in 5101 // `TryExtractArrayAccessAddress()`. 5102 if (kIsDebugBuild) { 5103 HIntermediateAddress* tmp = array_instr->AsIntermediateAddress(); 5104 DCHECK(tmp->GetOffset()->AsIntConstant()->GetValueAsUint64() == data_offset); 5105 } 5106 temp = array; 5107 } else { 5108 __ add(temp, array, ShifterOperand(data_offset)); 5109 } 5110 codegen_->StoreToShiftedRegOffset(value_type, 5111 value_loc, 5112 temp, 5113 index.AsRegister<Register>()); 5114 } 5115 break; 5116 } 5117 5118 case Primitive::kPrimNot: { 5119 Register value = value_loc.AsRegister<Register>(); 5120 // TryExtractArrayAccessAddress optimization is never applied for non-primitive ArraySet. 5121 // See the comment in instruction_simplifier_shared.cc. 5122 DCHECK(!has_intermediate_address); 5123 5124 if (instruction->InputAt(2)->IsNullConstant()) { 5125 // Just setting null. 5126 if (index.IsConstant()) { 5127 size_t offset = 5128 (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4) + data_offset; 5129 __ StoreToOffset(kStoreWord, value, array, offset); 5130 } else { 5131 DCHECK(index.IsRegister()) << index; 5132 __ add(IP, array, ShifterOperand(data_offset)); 5133 codegen_->StoreToShiftedRegOffset(value_type, 5134 value_loc, 5135 IP, 5136 index.AsRegister<Register>()); 5137 } 5138 codegen_->MaybeRecordImplicitNullCheck(instruction); 5139 DCHECK(!needs_write_barrier); 5140 DCHECK(!may_need_runtime_call_for_type_check); 5141 break; 5142 } 5143 5144 DCHECK(needs_write_barrier); 5145 Location temp1_loc = locations->GetTemp(0); 5146 Register temp1 = temp1_loc.AsRegister<Register>(); 5147 Location temp2_loc = locations->GetTemp(1); 5148 Register temp2 = temp2_loc.AsRegister<Register>(); 5149 uint32_t class_offset = mirror::Object::ClassOffset().Int32Value(); 5150 uint32_t super_offset = mirror::Class::SuperClassOffset().Int32Value(); 5151 uint32_t component_offset = mirror::Class::ComponentTypeOffset().Int32Value(); 5152 Label done; 5153 SlowPathCodeARM* slow_path = nullptr; 5154 5155 if (may_need_runtime_call_for_type_check) { 5156 slow_path = new (GetGraph()->GetArena()) ArraySetSlowPathARM(instruction); 5157 codegen_->AddSlowPath(slow_path); 5158 if (instruction->GetValueCanBeNull()) { 5159 Label non_zero; 5160 __ CompareAndBranchIfNonZero(value, &non_zero); 5161 if (index.IsConstant()) { 5162 size_t offset = 5163 (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4) + data_offset; 5164 __ StoreToOffset(kStoreWord, value, array, offset); 5165 } else { 5166 DCHECK(index.IsRegister()) << index; 5167 __ add(IP, array, ShifterOperand(data_offset)); 5168 codegen_->StoreToShiftedRegOffset(value_type, 5169 value_loc, 5170 IP, 5171 index.AsRegister<Register>()); 5172 } 5173 codegen_->MaybeRecordImplicitNullCheck(instruction); 5174 __ b(&done); 5175 __ Bind(&non_zero); 5176 } 5177 5178 // Note that when read barriers are enabled, the type checks 5179 // are performed without read barriers. This is fine, even in 5180 // the case where a class object is in the from-space after 5181 // the flip, as a comparison involving such a type would not 5182 // produce a false positive; it may of course produce a false 5183 // negative, in which case we would take the ArraySet slow 5184 // path. 5185 5186 // /* HeapReference<Class> */ temp1 = array->klass_ 5187 __ LoadFromOffset(kLoadWord, temp1, array, class_offset); 5188 codegen_->MaybeRecordImplicitNullCheck(instruction); 5189 __ MaybeUnpoisonHeapReference(temp1); 5190 5191 // /* HeapReference<Class> */ temp1 = temp1->component_type_ 5192 __ LoadFromOffset(kLoadWord, temp1, temp1, component_offset); 5193 // /* HeapReference<Class> */ temp2 = value->klass_ 5194 __ LoadFromOffset(kLoadWord, temp2, value, class_offset); 5195 // If heap poisoning is enabled, no need to unpoison `temp1` 5196 // nor `temp2`, as we are comparing two poisoned references. 5197 __ cmp(temp1, ShifterOperand(temp2)); 5198 5199 if (instruction->StaticTypeOfArrayIsObjectArray()) { 5200 Label do_put; 5201 __ b(&do_put, EQ); 5202 // If heap poisoning is enabled, the `temp1` reference has 5203 // not been unpoisoned yet; unpoison it now. 5204 __ MaybeUnpoisonHeapReference(temp1); 5205 5206 // /* HeapReference<Class> */ temp1 = temp1->super_class_ 5207 __ LoadFromOffset(kLoadWord, temp1, temp1, super_offset); 5208 // If heap poisoning is enabled, no need to unpoison 5209 // `temp1`, as we are comparing against null below. 5210 __ CompareAndBranchIfNonZero(temp1, slow_path->GetEntryLabel()); 5211 __ Bind(&do_put); 5212 } else { 5213 __ b(slow_path->GetEntryLabel(), NE); 5214 } 5215 } 5216 5217 Register source = value; 5218 if (kPoisonHeapReferences) { 5219 // Note that in the case where `value` is a null reference, 5220 // we do not enter this block, as a null reference does not 5221 // need poisoning. 5222 DCHECK_EQ(value_type, Primitive::kPrimNot); 5223 __ Mov(temp1, value); 5224 __ PoisonHeapReference(temp1); 5225 source = temp1; 5226 } 5227 5228 if (index.IsConstant()) { 5229 size_t offset = 5230 (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4) + data_offset; 5231 __ StoreToOffset(kStoreWord, source, array, offset); 5232 } else { 5233 DCHECK(index.IsRegister()) << index; 5234 5235 __ add(IP, array, ShifterOperand(data_offset)); 5236 codegen_->StoreToShiftedRegOffset(value_type, 5237 Location::RegisterLocation(source), 5238 IP, 5239 index.AsRegister<Register>()); 5240 } 5241 5242 if (!may_need_runtime_call_for_type_check) { 5243 codegen_->MaybeRecordImplicitNullCheck(instruction); 5244 } 5245 5246 codegen_->MarkGCCard(temp1, temp2, array, value, instruction->GetValueCanBeNull()); 5247 5248 if (done.IsLinked()) { 5249 __ Bind(&done); 5250 } 5251 5252 if (slow_path != nullptr) { 5253 __ Bind(slow_path->GetExitLabel()); 5254 } 5255 5256 break; 5257 } 5258 5259 case Primitive::kPrimLong: { 5260 Location value = locations->InAt(2); 5261 if (index.IsConstant()) { 5262 size_t offset = 5263 (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_8) + data_offset; 5264 __ StoreToOffset(kStoreWordPair, value.AsRegisterPairLow<Register>(), array, offset); 5265 } else { 5266 __ add(IP, array, ShifterOperand(index.AsRegister<Register>(), LSL, TIMES_8)); 5267 __ StoreToOffset(kStoreWordPair, value.AsRegisterPairLow<Register>(), IP, data_offset); 5268 } 5269 break; 5270 } 5271 5272 case Primitive::kPrimFloat: { 5273 Location value = locations->InAt(2); 5274 DCHECK(value.IsFpuRegister()); 5275 if (index.IsConstant()) { 5276 size_t offset = (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4) + data_offset; 5277 __ StoreSToOffset(value.AsFpuRegister<SRegister>(), array, offset); 5278 } else { 5279 __ add(IP, array, ShifterOperand(index.AsRegister<Register>(), LSL, TIMES_4)); 5280 __ StoreSToOffset(value.AsFpuRegister<SRegister>(), IP, data_offset); 5281 } 5282 break; 5283 } 5284 5285 case Primitive::kPrimDouble: { 5286 Location value = locations->InAt(2); 5287 DCHECK(value.IsFpuRegisterPair()); 5288 if (index.IsConstant()) { 5289 size_t offset = (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_8) + data_offset; 5290 __ StoreDToOffset(FromLowSToD(value.AsFpuRegisterPairLow<SRegister>()), array, offset); 5291 } else { 5292 __ add(IP, array, ShifterOperand(index.AsRegister<Register>(), LSL, TIMES_8)); 5293 __ StoreDToOffset(FromLowSToD(value.AsFpuRegisterPairLow<SRegister>()), IP, data_offset); 5294 } 5295 5296 break; 5297 } 5298 5299 case Primitive::kPrimVoid: 5300 LOG(FATAL) << "Unreachable type " << value_type; 5301 UNREACHABLE(); 5302 } 5303 5304 // Objects are handled in the switch. 5305 if (value_type != Primitive::kPrimNot) { 5306 codegen_->MaybeRecordImplicitNullCheck(instruction); 5307 } 5308} 5309 5310void LocationsBuilderARM::VisitArrayLength(HArrayLength* instruction) { 5311 LocationSummary* locations = 5312 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall); 5313 locations->SetInAt(0, Location::RequiresRegister()); 5314 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 5315} 5316 5317void InstructionCodeGeneratorARM::VisitArrayLength(HArrayLength* instruction) { 5318 LocationSummary* locations = instruction->GetLocations(); 5319 uint32_t offset = CodeGenerator::GetArrayLengthOffset(instruction); 5320 Register obj = locations->InAt(0).AsRegister<Register>(); 5321 Register out = locations->Out().AsRegister<Register>(); 5322 __ LoadFromOffset(kLoadWord, out, obj, offset); 5323 codegen_->MaybeRecordImplicitNullCheck(instruction); 5324 // Mask out compression flag from String's array length. 5325 if (mirror::kUseStringCompression && instruction->IsStringLength()) { 5326 __ Lsr(out, out, 1u); 5327 } 5328} 5329 5330void LocationsBuilderARM::VisitIntermediateAddress(HIntermediateAddress* instruction) { 5331 LocationSummary* locations = 5332 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall); 5333 5334 locations->SetInAt(0, Location::RequiresRegister()); 5335 locations->SetInAt(1, Location::RegisterOrConstant(instruction->GetOffset())); 5336 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 5337} 5338 5339void InstructionCodeGeneratorARM::VisitIntermediateAddress(HIntermediateAddress* instruction) { 5340 LocationSummary* locations = instruction->GetLocations(); 5341 Location out = locations->Out(); 5342 Location first = locations->InAt(0); 5343 Location second = locations->InAt(1); 5344 5345 if (second.IsRegister()) { 5346 __ add(out.AsRegister<Register>(), 5347 first.AsRegister<Register>(), 5348 ShifterOperand(second.AsRegister<Register>())); 5349 } else { 5350 __ AddConstant(out.AsRegister<Register>(), 5351 first.AsRegister<Register>(), 5352 second.GetConstant()->AsIntConstant()->GetValue()); 5353 } 5354} 5355 5356void LocationsBuilderARM::VisitBoundsCheck(HBoundsCheck* instruction) { 5357 RegisterSet caller_saves = RegisterSet::Empty(); 5358 InvokeRuntimeCallingConvention calling_convention; 5359 caller_saves.Add(Location::RegisterLocation(calling_convention.GetRegisterAt(0))); 5360 caller_saves.Add(Location::RegisterLocation(calling_convention.GetRegisterAt(1))); 5361 LocationSummary* locations = codegen_->CreateThrowingSlowPathLocations(instruction, caller_saves); 5362 locations->SetInAt(0, Location::RequiresRegister()); 5363 locations->SetInAt(1, Location::RequiresRegister()); 5364} 5365 5366void InstructionCodeGeneratorARM::VisitBoundsCheck(HBoundsCheck* instruction) { 5367 LocationSummary* locations = instruction->GetLocations(); 5368 SlowPathCodeARM* slow_path = 5369 new (GetGraph()->GetArena()) BoundsCheckSlowPathARM(instruction); 5370 codegen_->AddSlowPath(slow_path); 5371 5372 Register index = locations->InAt(0).AsRegister<Register>(); 5373 Register length = locations->InAt(1).AsRegister<Register>(); 5374 5375 __ cmp(index, ShifterOperand(length)); 5376 __ b(slow_path->GetEntryLabel(), HS); 5377} 5378 5379void CodeGeneratorARM::MarkGCCard(Register temp, 5380 Register card, 5381 Register object, 5382 Register value, 5383 bool can_be_null) { 5384 Label is_null; 5385 if (can_be_null) { 5386 __ CompareAndBranchIfZero(value, &is_null); 5387 } 5388 __ LoadFromOffset(kLoadWord, card, TR, Thread::CardTableOffset<kArmPointerSize>().Int32Value()); 5389 __ Lsr(temp, object, gc::accounting::CardTable::kCardShift); 5390 __ strb(card, Address(card, temp)); 5391 if (can_be_null) { 5392 __ Bind(&is_null); 5393 } 5394} 5395 5396void LocationsBuilderARM::VisitParallelMove(HParallelMove* instruction ATTRIBUTE_UNUSED) { 5397 LOG(FATAL) << "Unreachable"; 5398} 5399 5400void InstructionCodeGeneratorARM::VisitParallelMove(HParallelMove* instruction) { 5401 codegen_->GetMoveResolver()->EmitNativeCode(instruction); 5402} 5403 5404void LocationsBuilderARM::VisitSuspendCheck(HSuspendCheck* instruction) { 5405 LocationSummary* locations = 5406 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCallOnSlowPath); 5407 locations->SetCustomSlowPathCallerSaves(RegisterSet::Empty()); // No caller-save registers. 5408} 5409 5410void InstructionCodeGeneratorARM::VisitSuspendCheck(HSuspendCheck* instruction) { 5411 HBasicBlock* block = instruction->GetBlock(); 5412 if (block->GetLoopInformation() != nullptr) { 5413 DCHECK(block->GetLoopInformation()->GetSuspendCheck() == instruction); 5414 // The back edge will generate the suspend check. 5415 return; 5416 } 5417 if (block->IsEntryBlock() && instruction->GetNext()->IsGoto()) { 5418 // The goto will generate the suspend check. 5419 return; 5420 } 5421 GenerateSuspendCheck(instruction, nullptr); 5422} 5423 5424void InstructionCodeGeneratorARM::GenerateSuspendCheck(HSuspendCheck* instruction, 5425 HBasicBlock* successor) { 5426 SuspendCheckSlowPathARM* slow_path = 5427 down_cast<SuspendCheckSlowPathARM*>(instruction->GetSlowPath()); 5428 if (slow_path == nullptr) { 5429 slow_path = new (GetGraph()->GetArena()) SuspendCheckSlowPathARM(instruction, successor); 5430 instruction->SetSlowPath(slow_path); 5431 codegen_->AddSlowPath(slow_path); 5432 if (successor != nullptr) { 5433 DCHECK(successor->IsLoopHeader()); 5434 codegen_->ClearSpillSlotsFromLoopPhisInStackMap(instruction); 5435 } 5436 } else { 5437 DCHECK_EQ(slow_path->GetSuccessor(), successor); 5438 } 5439 5440 __ LoadFromOffset( 5441 kLoadUnsignedHalfword, IP, TR, Thread::ThreadFlagsOffset<kArmPointerSize>().Int32Value()); 5442 if (successor == nullptr) { 5443 __ CompareAndBranchIfNonZero(IP, slow_path->GetEntryLabel()); 5444 __ Bind(slow_path->GetReturnLabel()); 5445 } else { 5446 __ CompareAndBranchIfZero(IP, codegen_->GetLabelOf(successor)); 5447 __ b(slow_path->GetEntryLabel()); 5448 } 5449} 5450 5451ArmAssembler* ParallelMoveResolverARM::GetAssembler() const { 5452 return codegen_->GetAssembler(); 5453} 5454 5455void ParallelMoveResolverARM::EmitMove(size_t index) { 5456 MoveOperands* move = moves_[index]; 5457 Location source = move->GetSource(); 5458 Location destination = move->GetDestination(); 5459 5460 if (source.IsRegister()) { 5461 if (destination.IsRegister()) { 5462 __ Mov(destination.AsRegister<Register>(), source.AsRegister<Register>()); 5463 } else if (destination.IsFpuRegister()) { 5464 __ vmovsr(destination.AsFpuRegister<SRegister>(), source.AsRegister<Register>()); 5465 } else { 5466 DCHECK(destination.IsStackSlot()); 5467 __ StoreToOffset(kStoreWord, source.AsRegister<Register>(), 5468 SP, destination.GetStackIndex()); 5469 } 5470 } else if (source.IsStackSlot()) { 5471 if (destination.IsRegister()) { 5472 __ LoadFromOffset(kLoadWord, destination.AsRegister<Register>(), 5473 SP, source.GetStackIndex()); 5474 } else if (destination.IsFpuRegister()) { 5475 __ LoadSFromOffset(destination.AsFpuRegister<SRegister>(), SP, source.GetStackIndex()); 5476 } else { 5477 DCHECK(destination.IsStackSlot()); 5478 __ LoadFromOffset(kLoadWord, IP, SP, source.GetStackIndex()); 5479 __ StoreToOffset(kStoreWord, IP, SP, destination.GetStackIndex()); 5480 } 5481 } else if (source.IsFpuRegister()) { 5482 if (destination.IsRegister()) { 5483 __ vmovrs(destination.AsRegister<Register>(), source.AsFpuRegister<SRegister>()); 5484 } else if (destination.IsFpuRegister()) { 5485 __ vmovs(destination.AsFpuRegister<SRegister>(), source.AsFpuRegister<SRegister>()); 5486 } else { 5487 DCHECK(destination.IsStackSlot()); 5488 __ StoreSToOffset(source.AsFpuRegister<SRegister>(), SP, destination.GetStackIndex()); 5489 } 5490 } else if (source.IsDoubleStackSlot()) { 5491 if (destination.IsDoubleStackSlot()) { 5492 __ LoadDFromOffset(DTMP, SP, source.GetStackIndex()); 5493 __ StoreDToOffset(DTMP, SP, destination.GetStackIndex()); 5494 } else if (destination.IsRegisterPair()) { 5495 DCHECK(ExpectedPairLayout(destination)); 5496 __ LoadFromOffset( 5497 kLoadWordPair, destination.AsRegisterPairLow<Register>(), SP, source.GetStackIndex()); 5498 } else { 5499 DCHECK(destination.IsFpuRegisterPair()) << destination; 5500 __ LoadDFromOffset(FromLowSToD(destination.AsFpuRegisterPairLow<SRegister>()), 5501 SP, 5502 source.GetStackIndex()); 5503 } 5504 } else if (source.IsRegisterPair()) { 5505 if (destination.IsRegisterPair()) { 5506 __ Mov(destination.AsRegisterPairLow<Register>(), source.AsRegisterPairLow<Register>()); 5507 __ Mov(destination.AsRegisterPairHigh<Register>(), source.AsRegisterPairHigh<Register>()); 5508 } else if (destination.IsFpuRegisterPair()) { 5509 __ vmovdrr(FromLowSToD(destination.AsFpuRegisterPairLow<SRegister>()), 5510 source.AsRegisterPairLow<Register>(), 5511 source.AsRegisterPairHigh<Register>()); 5512 } else { 5513 DCHECK(destination.IsDoubleStackSlot()) << destination; 5514 DCHECK(ExpectedPairLayout(source)); 5515 __ StoreToOffset( 5516 kStoreWordPair, source.AsRegisterPairLow<Register>(), SP, destination.GetStackIndex()); 5517 } 5518 } else if (source.IsFpuRegisterPair()) { 5519 if (destination.IsRegisterPair()) { 5520 __ vmovrrd(destination.AsRegisterPairLow<Register>(), 5521 destination.AsRegisterPairHigh<Register>(), 5522 FromLowSToD(source.AsFpuRegisterPairLow<SRegister>())); 5523 } else if (destination.IsFpuRegisterPair()) { 5524 __ vmovd(FromLowSToD(destination.AsFpuRegisterPairLow<SRegister>()), 5525 FromLowSToD(source.AsFpuRegisterPairLow<SRegister>())); 5526 } else { 5527 DCHECK(destination.IsDoubleStackSlot()) << destination; 5528 __ StoreDToOffset(FromLowSToD(source.AsFpuRegisterPairLow<SRegister>()), 5529 SP, 5530 destination.GetStackIndex()); 5531 } 5532 } else { 5533 DCHECK(source.IsConstant()) << source; 5534 HConstant* constant = source.GetConstant(); 5535 if (constant->IsIntConstant() || constant->IsNullConstant()) { 5536 int32_t value = CodeGenerator::GetInt32ValueOf(constant); 5537 if (destination.IsRegister()) { 5538 __ LoadImmediate(destination.AsRegister<Register>(), value); 5539 } else { 5540 DCHECK(destination.IsStackSlot()); 5541 __ LoadImmediate(IP, value); 5542 __ StoreToOffset(kStoreWord, IP, SP, destination.GetStackIndex()); 5543 } 5544 } else if (constant->IsLongConstant()) { 5545 int64_t value = constant->AsLongConstant()->GetValue(); 5546 if (destination.IsRegisterPair()) { 5547 __ LoadImmediate(destination.AsRegisterPairLow<Register>(), Low32Bits(value)); 5548 __ LoadImmediate(destination.AsRegisterPairHigh<Register>(), High32Bits(value)); 5549 } else { 5550 DCHECK(destination.IsDoubleStackSlot()) << destination; 5551 __ LoadImmediate(IP, Low32Bits(value)); 5552 __ StoreToOffset(kStoreWord, IP, SP, destination.GetStackIndex()); 5553 __ LoadImmediate(IP, High32Bits(value)); 5554 __ StoreToOffset(kStoreWord, IP, SP, destination.GetHighStackIndex(kArmWordSize)); 5555 } 5556 } else if (constant->IsDoubleConstant()) { 5557 double value = constant->AsDoubleConstant()->GetValue(); 5558 if (destination.IsFpuRegisterPair()) { 5559 __ LoadDImmediate(FromLowSToD(destination.AsFpuRegisterPairLow<SRegister>()), value); 5560 } else { 5561 DCHECK(destination.IsDoubleStackSlot()) << destination; 5562 uint64_t int_value = bit_cast<uint64_t, double>(value); 5563 __ LoadImmediate(IP, Low32Bits(int_value)); 5564 __ StoreToOffset(kStoreWord, IP, SP, destination.GetStackIndex()); 5565 __ LoadImmediate(IP, High32Bits(int_value)); 5566 __ StoreToOffset(kStoreWord, IP, SP, destination.GetHighStackIndex(kArmWordSize)); 5567 } 5568 } else { 5569 DCHECK(constant->IsFloatConstant()) << constant->DebugName(); 5570 float value = constant->AsFloatConstant()->GetValue(); 5571 if (destination.IsFpuRegister()) { 5572 __ LoadSImmediate(destination.AsFpuRegister<SRegister>(), value); 5573 } else { 5574 DCHECK(destination.IsStackSlot()); 5575 __ LoadImmediate(IP, bit_cast<int32_t, float>(value)); 5576 __ StoreToOffset(kStoreWord, IP, SP, destination.GetStackIndex()); 5577 } 5578 } 5579 } 5580} 5581 5582void ParallelMoveResolverARM::Exchange(Register reg, int mem) { 5583 __ Mov(IP, reg); 5584 __ LoadFromOffset(kLoadWord, reg, SP, mem); 5585 __ StoreToOffset(kStoreWord, IP, SP, mem); 5586} 5587 5588void ParallelMoveResolverARM::Exchange(int mem1, int mem2) { 5589 ScratchRegisterScope ensure_scratch(this, IP, R0, codegen_->GetNumberOfCoreRegisters()); 5590 int stack_offset = ensure_scratch.IsSpilled() ? kArmWordSize : 0; 5591 __ LoadFromOffset(kLoadWord, static_cast<Register>(ensure_scratch.GetRegister()), 5592 SP, mem1 + stack_offset); 5593 __ LoadFromOffset(kLoadWord, IP, SP, mem2 + stack_offset); 5594 __ StoreToOffset(kStoreWord, static_cast<Register>(ensure_scratch.GetRegister()), 5595 SP, mem2 + stack_offset); 5596 __ StoreToOffset(kStoreWord, IP, SP, mem1 + stack_offset); 5597} 5598 5599void ParallelMoveResolverARM::EmitSwap(size_t index) { 5600 MoveOperands* move = moves_[index]; 5601 Location source = move->GetSource(); 5602 Location destination = move->GetDestination(); 5603 5604 if (source.IsRegister() && destination.IsRegister()) { 5605 DCHECK_NE(source.AsRegister<Register>(), IP); 5606 DCHECK_NE(destination.AsRegister<Register>(), IP); 5607 __ Mov(IP, source.AsRegister<Register>()); 5608 __ Mov(source.AsRegister<Register>(), destination.AsRegister<Register>()); 5609 __ Mov(destination.AsRegister<Register>(), IP); 5610 } else if (source.IsRegister() && destination.IsStackSlot()) { 5611 Exchange(source.AsRegister<Register>(), destination.GetStackIndex()); 5612 } else if (source.IsStackSlot() && destination.IsRegister()) { 5613 Exchange(destination.AsRegister<Register>(), source.GetStackIndex()); 5614 } else if (source.IsStackSlot() && destination.IsStackSlot()) { 5615 Exchange(source.GetStackIndex(), destination.GetStackIndex()); 5616 } else if (source.IsFpuRegister() && destination.IsFpuRegister()) { 5617 __ vmovrs(IP, source.AsFpuRegister<SRegister>()); 5618 __ vmovs(source.AsFpuRegister<SRegister>(), destination.AsFpuRegister<SRegister>()); 5619 __ vmovsr(destination.AsFpuRegister<SRegister>(), IP); 5620 } else if (source.IsRegisterPair() && destination.IsRegisterPair()) { 5621 __ vmovdrr(DTMP, source.AsRegisterPairLow<Register>(), source.AsRegisterPairHigh<Register>()); 5622 __ Mov(source.AsRegisterPairLow<Register>(), destination.AsRegisterPairLow<Register>()); 5623 __ Mov(source.AsRegisterPairHigh<Register>(), destination.AsRegisterPairHigh<Register>()); 5624 __ vmovrrd(destination.AsRegisterPairLow<Register>(), 5625 destination.AsRegisterPairHigh<Register>(), 5626 DTMP); 5627 } else if (source.IsRegisterPair() || destination.IsRegisterPair()) { 5628 Register low_reg = source.IsRegisterPair() 5629 ? source.AsRegisterPairLow<Register>() 5630 : destination.AsRegisterPairLow<Register>(); 5631 int mem = source.IsRegisterPair() 5632 ? destination.GetStackIndex() 5633 : source.GetStackIndex(); 5634 DCHECK(ExpectedPairLayout(source.IsRegisterPair() ? source : destination)); 5635 __ vmovdrr(DTMP, low_reg, static_cast<Register>(low_reg + 1)); 5636 __ LoadFromOffset(kLoadWordPair, low_reg, SP, mem); 5637 __ StoreDToOffset(DTMP, SP, mem); 5638 } else if (source.IsFpuRegisterPair() && destination.IsFpuRegisterPair()) { 5639 DRegister first = FromLowSToD(source.AsFpuRegisterPairLow<SRegister>()); 5640 DRegister second = FromLowSToD(destination.AsFpuRegisterPairLow<SRegister>()); 5641 __ vmovd(DTMP, first); 5642 __ vmovd(first, second); 5643 __ vmovd(second, DTMP); 5644 } else if (source.IsFpuRegisterPair() || destination.IsFpuRegisterPair()) { 5645 DRegister reg = source.IsFpuRegisterPair() 5646 ? FromLowSToD(source.AsFpuRegisterPairLow<SRegister>()) 5647 : FromLowSToD(destination.AsFpuRegisterPairLow<SRegister>()); 5648 int mem = source.IsFpuRegisterPair() 5649 ? destination.GetStackIndex() 5650 : source.GetStackIndex(); 5651 __ vmovd(DTMP, reg); 5652 __ LoadDFromOffset(reg, SP, mem); 5653 __ StoreDToOffset(DTMP, SP, mem); 5654 } else if (source.IsFpuRegister() || destination.IsFpuRegister()) { 5655 SRegister reg = source.IsFpuRegister() ? source.AsFpuRegister<SRegister>() 5656 : destination.AsFpuRegister<SRegister>(); 5657 int mem = source.IsFpuRegister() 5658 ? destination.GetStackIndex() 5659 : source.GetStackIndex(); 5660 5661 __ vmovrs(IP, reg); 5662 __ LoadSFromOffset(reg, SP, mem); 5663 __ StoreToOffset(kStoreWord, IP, SP, mem); 5664 } else if (source.IsDoubleStackSlot() && destination.IsDoubleStackSlot()) { 5665 Exchange(source.GetStackIndex(), destination.GetStackIndex()); 5666 Exchange(source.GetHighStackIndex(kArmWordSize), destination.GetHighStackIndex(kArmWordSize)); 5667 } else { 5668 LOG(FATAL) << "Unimplemented" << source << " <-> " << destination; 5669 } 5670} 5671 5672void ParallelMoveResolverARM::SpillScratch(int reg) { 5673 __ Push(static_cast<Register>(reg)); 5674} 5675 5676void ParallelMoveResolverARM::RestoreScratch(int reg) { 5677 __ Pop(static_cast<Register>(reg)); 5678} 5679 5680HLoadClass::LoadKind CodeGeneratorARM::GetSupportedLoadClassKind( 5681 HLoadClass::LoadKind desired_class_load_kind) { 5682 switch (desired_class_load_kind) { 5683 case HLoadClass::LoadKind::kReferrersClass: 5684 break; 5685 case HLoadClass::LoadKind::kBootImageLinkTimeAddress: 5686 DCHECK(!GetCompilerOptions().GetCompilePic()); 5687 break; 5688 case HLoadClass::LoadKind::kBootImageLinkTimePcRelative: 5689 DCHECK(GetCompilerOptions().GetCompilePic()); 5690 break; 5691 case HLoadClass::LoadKind::kBootImageAddress: 5692 break; 5693 case HLoadClass::LoadKind::kDexCacheAddress: 5694 DCHECK(Runtime::Current()->UseJitCompilation()); 5695 break; 5696 case HLoadClass::LoadKind::kDexCachePcRelative: 5697 DCHECK(!Runtime::Current()->UseJitCompilation()); 5698 // We disable pc-relative load when there is an irreducible loop, as the optimization 5699 // is incompatible with it. 5700 // TODO: Create as many ArmDexCacheArraysBase instructions as needed for methods 5701 // with irreducible loops. 5702 if (GetGraph()->HasIrreducibleLoops()) { 5703 return HLoadClass::LoadKind::kDexCacheViaMethod; 5704 } 5705 break; 5706 case HLoadClass::LoadKind::kDexCacheViaMethod: 5707 break; 5708 } 5709 return desired_class_load_kind; 5710} 5711 5712void LocationsBuilderARM::VisitLoadClass(HLoadClass* cls) { 5713 if (cls->NeedsAccessCheck()) { 5714 InvokeRuntimeCallingConvention calling_convention; 5715 CodeGenerator::CreateLoadClassLocationSummary( 5716 cls, 5717 Location::RegisterLocation(calling_convention.GetRegisterAt(0)), 5718 Location::RegisterLocation(R0), 5719 /* code_generator_supports_read_barrier */ true); 5720 return; 5721 } 5722 5723 const bool requires_read_barrier = kEmitCompilerReadBarrier && !cls->IsInBootImage(); 5724 LocationSummary::CallKind call_kind = (cls->NeedsEnvironment() || requires_read_barrier) 5725 ? LocationSummary::kCallOnSlowPath 5726 : LocationSummary::kNoCall; 5727 LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(cls, call_kind); 5728 if (kUseBakerReadBarrier && requires_read_barrier && !cls->NeedsEnvironment()) { 5729 locations->SetCustomSlowPathCallerSaves(RegisterSet::Empty()); // No caller-save registers. 5730 } 5731 5732 HLoadClass::LoadKind load_kind = cls->GetLoadKind(); 5733 if (load_kind == HLoadClass::LoadKind::kReferrersClass || 5734 load_kind == HLoadClass::LoadKind::kDexCacheViaMethod || 5735 load_kind == HLoadClass::LoadKind::kDexCachePcRelative) { 5736 locations->SetInAt(0, Location::RequiresRegister()); 5737 } 5738 locations->SetOut(Location::RequiresRegister()); 5739} 5740 5741void InstructionCodeGeneratorARM::VisitLoadClass(HLoadClass* cls) { 5742 LocationSummary* locations = cls->GetLocations(); 5743 if (cls->NeedsAccessCheck()) { 5744 codegen_->MoveConstant(locations->GetTemp(0), cls->GetTypeIndex()); 5745 codegen_->InvokeRuntime(kQuickInitializeTypeAndVerifyAccess, cls, cls->GetDexPc()); 5746 CheckEntrypointTypes<kQuickInitializeTypeAndVerifyAccess, void*, uint32_t>(); 5747 return; 5748 } 5749 5750 Location out_loc = locations->Out(); 5751 Register out = out_loc.AsRegister<Register>(); 5752 5753 const ReadBarrierOption read_barrier_option = cls->IsInBootImage() 5754 ? kWithoutReadBarrier 5755 : kCompilerReadBarrierOption; 5756 bool generate_null_check = false; 5757 switch (cls->GetLoadKind()) { 5758 case HLoadClass::LoadKind::kReferrersClass: { 5759 DCHECK(!cls->CanCallRuntime()); 5760 DCHECK(!cls->MustGenerateClinitCheck()); 5761 // /* GcRoot<mirror::Class> */ out = current_method->declaring_class_ 5762 Register current_method = locations->InAt(0).AsRegister<Register>(); 5763 GenerateGcRootFieldLoad(cls, 5764 out_loc, 5765 current_method, 5766 ArtMethod::DeclaringClassOffset().Int32Value(), 5767 read_barrier_option); 5768 break; 5769 } 5770 case HLoadClass::LoadKind::kBootImageLinkTimeAddress: { 5771 DCHECK_EQ(read_barrier_option, kWithoutReadBarrier); 5772 __ LoadLiteral(out, codegen_->DeduplicateBootImageTypeLiteral(cls->GetDexFile(), 5773 cls->GetTypeIndex())); 5774 break; 5775 } 5776 case HLoadClass::LoadKind::kBootImageLinkTimePcRelative: { 5777 DCHECK_EQ(read_barrier_option, kWithoutReadBarrier); 5778 CodeGeneratorARM::PcRelativePatchInfo* labels = 5779 codegen_->NewPcRelativeTypePatch(cls->GetDexFile(), cls->GetTypeIndex()); 5780 __ BindTrackedLabel(&labels->movw_label); 5781 __ movw(out, /* placeholder */ 0u); 5782 __ BindTrackedLabel(&labels->movt_label); 5783 __ movt(out, /* placeholder */ 0u); 5784 __ BindTrackedLabel(&labels->add_pc_label); 5785 __ add(out, out, ShifterOperand(PC)); 5786 break; 5787 } 5788 case HLoadClass::LoadKind::kBootImageAddress: { 5789 DCHECK_EQ(read_barrier_option, kWithoutReadBarrier); 5790 DCHECK_NE(cls->GetAddress(), 0u); 5791 uint32_t address = dchecked_integral_cast<uint32_t>(cls->GetAddress()); 5792 __ LoadLiteral(out, codegen_->DeduplicateBootImageAddressLiteral(address)); 5793 break; 5794 } 5795 case HLoadClass::LoadKind::kDexCacheAddress: { 5796 DCHECK_NE(cls->GetAddress(), 0u); 5797 uint32_t address = dchecked_integral_cast<uint32_t>(cls->GetAddress()); 5798 // 16-bit LDR immediate has a 5-bit offset multiplied by the size and that gives 5799 // a 128B range. To try and reduce the number of literals if we load multiple types, 5800 // simply split the dex cache address to a 128B aligned base loaded from a literal 5801 // and the remaining offset embedded in the load. 5802 static_assert(sizeof(GcRoot<mirror::Class>) == 4u, "Expected GC root to be 4 bytes."); 5803 DCHECK_ALIGNED(cls->GetAddress(), 4u); 5804 constexpr size_t offset_bits = /* encoded bits */ 5 + /* scale */ 2; 5805 uint32_t base_address = address & ~MaxInt<uint32_t>(offset_bits); 5806 uint32_t offset = address & MaxInt<uint32_t>(offset_bits); 5807 __ LoadLiteral(out, codegen_->DeduplicateDexCacheAddressLiteral(base_address)); 5808 // /* GcRoot<mirror::Class> */ out = *(base_address + offset) 5809 GenerateGcRootFieldLoad(cls, out_loc, out, offset, read_barrier_option); 5810 generate_null_check = !cls->IsInDexCache(); 5811 break; 5812 } 5813 case HLoadClass::LoadKind::kDexCachePcRelative: { 5814 Register base_reg = locations->InAt(0).AsRegister<Register>(); 5815 HArmDexCacheArraysBase* base = cls->InputAt(0)->AsArmDexCacheArraysBase(); 5816 int32_t offset = cls->GetDexCacheElementOffset() - base->GetElementOffset(); 5817 // /* GcRoot<mirror::Class> */ out = *(dex_cache_arrays_base + offset) 5818 GenerateGcRootFieldLoad(cls, out_loc, base_reg, offset, read_barrier_option); 5819 generate_null_check = !cls->IsInDexCache(); 5820 break; 5821 } 5822 case HLoadClass::LoadKind::kDexCacheViaMethod: { 5823 // /* GcRoot<mirror::Class>[] */ out = 5824 // current_method.ptr_sized_fields_->dex_cache_resolved_types_ 5825 Register current_method = locations->InAt(0).AsRegister<Register>(); 5826 __ LoadFromOffset(kLoadWord, 5827 out, 5828 current_method, 5829 ArtMethod::DexCacheResolvedTypesOffset(kArmPointerSize).Int32Value()); 5830 // /* GcRoot<mirror::Class> */ out = out[type_index] 5831 size_t offset = CodeGenerator::GetCacheOffset(cls->GetTypeIndex()); 5832 GenerateGcRootFieldLoad(cls, out_loc, out, offset, read_barrier_option); 5833 generate_null_check = !cls->IsInDexCache(); 5834 } 5835 } 5836 5837 if (generate_null_check || cls->MustGenerateClinitCheck()) { 5838 DCHECK(cls->CanCallRuntime()); 5839 SlowPathCodeARM* slow_path = new (GetGraph()->GetArena()) LoadClassSlowPathARM( 5840 cls, cls, cls->GetDexPc(), cls->MustGenerateClinitCheck()); 5841 codegen_->AddSlowPath(slow_path); 5842 if (generate_null_check) { 5843 __ CompareAndBranchIfZero(out, slow_path->GetEntryLabel()); 5844 } 5845 if (cls->MustGenerateClinitCheck()) { 5846 GenerateClassInitializationCheck(slow_path, out); 5847 } else { 5848 __ Bind(slow_path->GetExitLabel()); 5849 } 5850 } 5851} 5852 5853void LocationsBuilderARM::VisitClinitCheck(HClinitCheck* check) { 5854 LocationSummary* locations = 5855 new (GetGraph()->GetArena()) LocationSummary(check, LocationSummary::kCallOnSlowPath); 5856 locations->SetInAt(0, Location::RequiresRegister()); 5857 if (check->HasUses()) { 5858 locations->SetOut(Location::SameAsFirstInput()); 5859 } 5860} 5861 5862void InstructionCodeGeneratorARM::VisitClinitCheck(HClinitCheck* check) { 5863 // We assume the class is not null. 5864 SlowPathCodeARM* slow_path = new (GetGraph()->GetArena()) LoadClassSlowPathARM( 5865 check->GetLoadClass(), check, check->GetDexPc(), true); 5866 codegen_->AddSlowPath(slow_path); 5867 GenerateClassInitializationCheck(slow_path, 5868 check->GetLocations()->InAt(0).AsRegister<Register>()); 5869} 5870 5871void InstructionCodeGeneratorARM::GenerateClassInitializationCheck( 5872 SlowPathCodeARM* slow_path, Register class_reg) { 5873 __ LoadFromOffset(kLoadWord, IP, class_reg, mirror::Class::StatusOffset().Int32Value()); 5874 __ cmp(IP, ShifterOperand(mirror::Class::kStatusInitialized)); 5875 __ b(slow_path->GetEntryLabel(), LT); 5876 // Even if the initialized flag is set, we may be in a situation where caches are not synced 5877 // properly. Therefore, we do a memory fence. 5878 __ dmb(ISH); 5879 __ Bind(slow_path->GetExitLabel()); 5880} 5881 5882HLoadString::LoadKind CodeGeneratorARM::GetSupportedLoadStringKind( 5883 HLoadString::LoadKind desired_string_load_kind) { 5884 switch (desired_string_load_kind) { 5885 case HLoadString::LoadKind::kBootImageLinkTimeAddress: 5886 DCHECK(!GetCompilerOptions().GetCompilePic()); 5887 break; 5888 case HLoadString::LoadKind::kBootImageLinkTimePcRelative: 5889 DCHECK(GetCompilerOptions().GetCompilePic()); 5890 break; 5891 case HLoadString::LoadKind::kBootImageAddress: 5892 break; 5893 case HLoadString::LoadKind::kBssEntry: 5894 DCHECK(!Runtime::Current()->UseJitCompilation()); 5895 break; 5896 case HLoadString::LoadKind::kDexCacheViaMethod: 5897 break; 5898 } 5899 return desired_string_load_kind; 5900} 5901 5902void LocationsBuilderARM::VisitLoadString(HLoadString* load) { 5903 LocationSummary::CallKind call_kind = load->NeedsEnvironment() 5904 ? ((load->GetLoadKind() == HLoadString::LoadKind::kDexCacheViaMethod) 5905 ? LocationSummary::kCallOnMainOnly 5906 : LocationSummary::kCallOnSlowPath) 5907 : LocationSummary::kNoCall; 5908 LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(load, call_kind); 5909 5910 HLoadString::LoadKind load_kind = load->GetLoadKind(); 5911 if (load_kind == HLoadString::LoadKind::kDexCacheViaMethod) { 5912 locations->SetOut(Location::RegisterLocation(R0)); 5913 } else { 5914 locations->SetOut(Location::RequiresRegister()); 5915 if (load_kind == HLoadString::LoadKind::kBssEntry) { 5916 if (!kUseReadBarrier || kUseBakerReadBarrier) { 5917 // Rely on the pResolveString and/or marking to save everything, including temps. 5918 // Note that IP may theoretically be clobbered by saving/restoring the live register 5919 // (only one thanks to the custom calling convention), so we request a different temp. 5920 locations->AddTemp(Location::RequiresRegister()); 5921 RegisterSet caller_saves = RegisterSet::Empty(); 5922 InvokeRuntimeCallingConvention calling_convention; 5923 caller_saves.Add(Location::RegisterLocation(calling_convention.GetRegisterAt(0))); 5924 // TODO: Add GetReturnLocation() to the calling convention so that we can DCHECK() 5925 // that the the kPrimNot result register is the same as the first argument register. 5926 locations->SetCustomSlowPathCallerSaves(caller_saves); 5927 } else { 5928 // For non-Baker read barrier we have a temp-clobbering call. 5929 } 5930 } 5931 } 5932} 5933 5934void InstructionCodeGeneratorARM::VisitLoadString(HLoadString* load) { 5935 LocationSummary* locations = load->GetLocations(); 5936 Location out_loc = locations->Out(); 5937 Register out = out_loc.AsRegister<Register>(); 5938 HLoadString::LoadKind load_kind = load->GetLoadKind(); 5939 5940 switch (load_kind) { 5941 case HLoadString::LoadKind::kBootImageLinkTimeAddress: { 5942 __ LoadLiteral(out, codegen_->DeduplicateBootImageStringLiteral(load->GetDexFile(), 5943 load->GetStringIndex())); 5944 return; // No dex cache slow path. 5945 } 5946 case HLoadString::LoadKind::kBootImageLinkTimePcRelative: { 5947 DCHECK(codegen_->GetCompilerOptions().IsBootImage()); 5948 CodeGeneratorARM::PcRelativePatchInfo* labels = 5949 codegen_->NewPcRelativeStringPatch(load->GetDexFile(), load->GetStringIndex()); 5950 __ BindTrackedLabel(&labels->movw_label); 5951 __ movw(out, /* placeholder */ 0u); 5952 __ BindTrackedLabel(&labels->movt_label); 5953 __ movt(out, /* placeholder */ 0u); 5954 __ BindTrackedLabel(&labels->add_pc_label); 5955 __ add(out, out, ShifterOperand(PC)); 5956 return; // No dex cache slow path. 5957 } 5958 case HLoadString::LoadKind::kBootImageAddress: { 5959 DCHECK_NE(load->GetAddress(), 0u); 5960 uint32_t address = dchecked_integral_cast<uint32_t>(load->GetAddress()); 5961 __ LoadLiteral(out, codegen_->DeduplicateBootImageAddressLiteral(address)); 5962 return; // No dex cache slow path. 5963 } 5964 case HLoadString::LoadKind::kBssEntry: { 5965 DCHECK(!codegen_->GetCompilerOptions().IsBootImage()); 5966 Register temp = locations->GetTemp(0).AsRegister<Register>(); 5967 CodeGeneratorARM::PcRelativePatchInfo* labels = 5968 codegen_->NewPcRelativeStringPatch(load->GetDexFile(), load->GetStringIndex()); 5969 __ BindTrackedLabel(&labels->movw_label); 5970 __ movw(temp, /* placeholder */ 0u); 5971 __ BindTrackedLabel(&labels->movt_label); 5972 __ movt(temp, /* placeholder */ 0u); 5973 __ BindTrackedLabel(&labels->add_pc_label); 5974 __ add(temp, temp, ShifterOperand(PC)); 5975 GenerateGcRootFieldLoad(load, out_loc, temp, /* offset */ 0, kCompilerReadBarrierOption); 5976 SlowPathCode* slow_path = new (GetGraph()->GetArena()) LoadStringSlowPathARM(load); 5977 codegen_->AddSlowPath(slow_path); 5978 __ CompareAndBranchIfZero(out, slow_path->GetEntryLabel()); 5979 __ Bind(slow_path->GetExitLabel()); 5980 return; 5981 } 5982 default: 5983 break; 5984 } 5985 5986 // TODO: Consider re-adding the compiler code to do string dex cache lookup again. 5987 DCHECK(load_kind == HLoadString::LoadKind::kDexCacheViaMethod); 5988 InvokeRuntimeCallingConvention calling_convention; 5989 DCHECK_EQ(calling_convention.GetRegisterAt(0), out); 5990 __ LoadImmediate(calling_convention.GetRegisterAt(0), load->GetStringIndex()); 5991 codegen_->InvokeRuntime(kQuickResolveString, load, load->GetDexPc()); 5992 CheckEntrypointTypes<kQuickResolveString, void*, uint32_t>(); 5993} 5994 5995static int32_t GetExceptionTlsOffset() { 5996 return Thread::ExceptionOffset<kArmPointerSize>().Int32Value(); 5997} 5998 5999void LocationsBuilderARM::VisitLoadException(HLoadException* load) { 6000 LocationSummary* locations = 6001 new (GetGraph()->GetArena()) LocationSummary(load, LocationSummary::kNoCall); 6002 locations->SetOut(Location::RequiresRegister()); 6003} 6004 6005void InstructionCodeGeneratorARM::VisitLoadException(HLoadException* load) { 6006 Register out = load->GetLocations()->Out().AsRegister<Register>(); 6007 __ LoadFromOffset(kLoadWord, out, TR, GetExceptionTlsOffset()); 6008} 6009 6010void LocationsBuilderARM::VisitClearException(HClearException* clear) { 6011 new (GetGraph()->GetArena()) LocationSummary(clear, LocationSummary::kNoCall); 6012} 6013 6014void InstructionCodeGeneratorARM::VisitClearException(HClearException* clear ATTRIBUTE_UNUSED) { 6015 __ LoadImmediate(IP, 0); 6016 __ StoreToOffset(kStoreWord, IP, TR, GetExceptionTlsOffset()); 6017} 6018 6019void LocationsBuilderARM::VisitThrow(HThrow* instruction) { 6020 LocationSummary* locations = 6021 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCallOnMainOnly); 6022 InvokeRuntimeCallingConvention calling_convention; 6023 locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(0))); 6024} 6025 6026void InstructionCodeGeneratorARM::VisitThrow(HThrow* instruction) { 6027 codegen_->InvokeRuntime(kQuickDeliverException, instruction, instruction->GetDexPc()); 6028 CheckEntrypointTypes<kQuickDeliverException, void, mirror::Object*>(); 6029} 6030 6031// Temp is used for read barrier. 6032static size_t NumberOfInstanceOfTemps(TypeCheckKind type_check_kind) { 6033 if (kEmitCompilerReadBarrier && 6034 (kUseBakerReadBarrier || 6035 type_check_kind == TypeCheckKind::kAbstractClassCheck || 6036 type_check_kind == TypeCheckKind::kClassHierarchyCheck || 6037 type_check_kind == TypeCheckKind::kArrayObjectCheck)) { 6038 return 1; 6039 } 6040 return 0; 6041} 6042 6043// Interface case has 3 temps, one for holding the number of interfaces, one for the current 6044// interface pointer, one for loading the current interface. 6045// The other checks have one temp for loading the object's class. 6046static size_t NumberOfCheckCastTemps(TypeCheckKind type_check_kind) { 6047 if (type_check_kind == TypeCheckKind::kInterfaceCheck) { 6048 return 3; 6049 } 6050 return 1 + NumberOfInstanceOfTemps(type_check_kind); 6051} 6052 6053void LocationsBuilderARM::VisitInstanceOf(HInstanceOf* instruction) { 6054 LocationSummary::CallKind call_kind = LocationSummary::kNoCall; 6055 TypeCheckKind type_check_kind = instruction->GetTypeCheckKind(); 6056 bool baker_read_barrier_slow_path = false; 6057 switch (type_check_kind) { 6058 case TypeCheckKind::kExactCheck: 6059 case TypeCheckKind::kAbstractClassCheck: 6060 case TypeCheckKind::kClassHierarchyCheck: 6061 case TypeCheckKind::kArrayObjectCheck: 6062 call_kind = 6063 kEmitCompilerReadBarrier ? LocationSummary::kCallOnSlowPath : LocationSummary::kNoCall; 6064 baker_read_barrier_slow_path = kUseBakerReadBarrier; 6065 break; 6066 case TypeCheckKind::kArrayCheck: 6067 case TypeCheckKind::kUnresolvedCheck: 6068 case TypeCheckKind::kInterfaceCheck: 6069 call_kind = LocationSummary::kCallOnSlowPath; 6070 break; 6071 } 6072 6073 LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction, call_kind); 6074 if (baker_read_barrier_slow_path) { 6075 locations->SetCustomSlowPathCallerSaves(RegisterSet::Empty()); // No caller-save registers. 6076 } 6077 locations->SetInAt(0, Location::RequiresRegister()); 6078 locations->SetInAt(1, Location::RequiresRegister()); 6079 // The "out" register is used as a temporary, so it overlaps with the inputs. 6080 // Note that TypeCheckSlowPathARM uses this register too. 6081 locations->SetOut(Location::RequiresRegister(), Location::kOutputOverlap); 6082 locations->AddRegisterTemps(NumberOfInstanceOfTemps(type_check_kind)); 6083} 6084 6085void InstructionCodeGeneratorARM::VisitInstanceOf(HInstanceOf* instruction) { 6086 TypeCheckKind type_check_kind = instruction->GetTypeCheckKind(); 6087 LocationSummary* locations = instruction->GetLocations(); 6088 Location obj_loc = locations->InAt(0); 6089 Register obj = obj_loc.AsRegister<Register>(); 6090 Register cls = locations->InAt(1).AsRegister<Register>(); 6091 Location out_loc = locations->Out(); 6092 Register out = out_loc.AsRegister<Register>(); 6093 const size_t num_temps = NumberOfInstanceOfTemps(type_check_kind); 6094 DCHECK_LE(num_temps, 1u); 6095 Location maybe_temp_loc = (num_temps >= 1) ? locations->GetTemp(0) : Location::NoLocation(); 6096 uint32_t class_offset = mirror::Object::ClassOffset().Int32Value(); 6097 uint32_t super_offset = mirror::Class::SuperClassOffset().Int32Value(); 6098 uint32_t component_offset = mirror::Class::ComponentTypeOffset().Int32Value(); 6099 uint32_t primitive_offset = mirror::Class::PrimitiveTypeOffset().Int32Value(); 6100 Label done, zero; 6101 SlowPathCodeARM* slow_path = nullptr; 6102 6103 // Return 0 if `obj` is null. 6104 // avoid null check if we know obj is not null. 6105 if (instruction->MustDoNullCheck()) { 6106 __ CompareAndBranchIfZero(obj, &zero); 6107 } 6108 6109 switch (type_check_kind) { 6110 case TypeCheckKind::kExactCheck: { 6111 // /* HeapReference<Class> */ out = obj->klass_ 6112 GenerateReferenceLoadTwoRegisters(instruction, 6113 out_loc, 6114 obj_loc, 6115 class_offset, 6116 maybe_temp_loc, 6117 kCompilerReadBarrierOption); 6118 __ cmp(out, ShifterOperand(cls)); 6119 // Classes must be equal for the instanceof to succeed. 6120 __ b(&zero, NE); 6121 __ LoadImmediate(out, 1); 6122 __ b(&done); 6123 break; 6124 } 6125 6126 case TypeCheckKind::kAbstractClassCheck: { 6127 // /* HeapReference<Class> */ out = obj->klass_ 6128 GenerateReferenceLoadTwoRegisters(instruction, 6129 out_loc, 6130 obj_loc, 6131 class_offset, 6132 maybe_temp_loc, 6133 kCompilerReadBarrierOption); 6134 // If the class is abstract, we eagerly fetch the super class of the 6135 // object to avoid doing a comparison we know will fail. 6136 Label loop; 6137 __ Bind(&loop); 6138 // /* HeapReference<Class> */ out = out->super_class_ 6139 GenerateReferenceLoadOneRegister(instruction, 6140 out_loc, 6141 super_offset, 6142 maybe_temp_loc, 6143 kCompilerReadBarrierOption); 6144 // If `out` is null, we use it for the result, and jump to `done`. 6145 __ CompareAndBranchIfZero(out, &done); 6146 __ cmp(out, ShifterOperand(cls)); 6147 __ b(&loop, NE); 6148 __ LoadImmediate(out, 1); 6149 if (zero.IsLinked()) { 6150 __ b(&done); 6151 } 6152 break; 6153 } 6154 6155 case TypeCheckKind::kClassHierarchyCheck: { 6156 // /* HeapReference<Class> */ out = obj->klass_ 6157 GenerateReferenceLoadTwoRegisters(instruction, 6158 out_loc, 6159 obj_loc, 6160 class_offset, 6161 maybe_temp_loc, 6162 kCompilerReadBarrierOption); 6163 // Walk over the class hierarchy to find a match. 6164 Label loop, success; 6165 __ Bind(&loop); 6166 __ cmp(out, ShifterOperand(cls)); 6167 __ b(&success, EQ); 6168 // /* HeapReference<Class> */ out = out->super_class_ 6169 GenerateReferenceLoadOneRegister(instruction, 6170 out_loc, 6171 super_offset, 6172 maybe_temp_loc, 6173 kCompilerReadBarrierOption); 6174 __ CompareAndBranchIfNonZero(out, &loop); 6175 // If `out` is null, we use it for the result, and jump to `done`. 6176 __ b(&done); 6177 __ Bind(&success); 6178 __ LoadImmediate(out, 1); 6179 if (zero.IsLinked()) { 6180 __ b(&done); 6181 } 6182 break; 6183 } 6184 6185 case TypeCheckKind::kArrayObjectCheck: { 6186 // /* HeapReference<Class> */ out = obj->klass_ 6187 GenerateReferenceLoadTwoRegisters(instruction, 6188 out_loc, 6189 obj_loc, 6190 class_offset, 6191 maybe_temp_loc, 6192 kCompilerReadBarrierOption); 6193 // Do an exact check. 6194 Label exact_check; 6195 __ cmp(out, ShifterOperand(cls)); 6196 __ b(&exact_check, EQ); 6197 // Otherwise, we need to check that the object's class is a non-primitive array. 6198 // /* HeapReference<Class> */ out = out->component_type_ 6199 GenerateReferenceLoadOneRegister(instruction, 6200 out_loc, 6201 component_offset, 6202 maybe_temp_loc, 6203 kCompilerReadBarrierOption); 6204 // If `out` is null, we use it for the result, and jump to `done`. 6205 __ CompareAndBranchIfZero(out, &done); 6206 __ LoadFromOffset(kLoadUnsignedHalfword, out, out, primitive_offset); 6207 static_assert(Primitive::kPrimNot == 0, "Expected 0 for kPrimNot"); 6208 __ CompareAndBranchIfNonZero(out, &zero); 6209 __ Bind(&exact_check); 6210 __ LoadImmediate(out, 1); 6211 __ b(&done); 6212 break; 6213 } 6214 6215 case TypeCheckKind::kArrayCheck: { 6216 // No read barrier since the slow path will retry upon failure. 6217 // /* HeapReference<Class> */ out = obj->klass_ 6218 GenerateReferenceLoadTwoRegisters(instruction, 6219 out_loc, 6220 obj_loc, 6221 class_offset, 6222 maybe_temp_loc, 6223 kWithoutReadBarrier); 6224 __ cmp(out, ShifterOperand(cls)); 6225 DCHECK(locations->OnlyCallsOnSlowPath()); 6226 slow_path = new (GetGraph()->GetArena()) TypeCheckSlowPathARM(instruction, 6227 /* is_fatal */ false); 6228 codegen_->AddSlowPath(slow_path); 6229 __ b(slow_path->GetEntryLabel(), NE); 6230 __ LoadImmediate(out, 1); 6231 if (zero.IsLinked()) { 6232 __ b(&done); 6233 } 6234 break; 6235 } 6236 6237 case TypeCheckKind::kUnresolvedCheck: 6238 case TypeCheckKind::kInterfaceCheck: { 6239 // Note that we indeed only call on slow path, but we always go 6240 // into the slow path for the unresolved and interface check 6241 // cases. 6242 // 6243 // We cannot directly call the InstanceofNonTrivial runtime 6244 // entry point without resorting to a type checking slow path 6245 // here (i.e. by calling InvokeRuntime directly), as it would 6246 // require to assign fixed registers for the inputs of this 6247 // HInstanceOf instruction (following the runtime calling 6248 // convention), which might be cluttered by the potential first 6249 // read barrier emission at the beginning of this method. 6250 // 6251 // TODO: Introduce a new runtime entry point taking the object 6252 // to test (instead of its class) as argument, and let it deal 6253 // with the read barrier issues. This will let us refactor this 6254 // case of the `switch` code as it was previously (with a direct 6255 // call to the runtime not using a type checking slow path). 6256 // This should also be beneficial for the other cases above. 6257 DCHECK(locations->OnlyCallsOnSlowPath()); 6258 slow_path = new (GetGraph()->GetArena()) TypeCheckSlowPathARM(instruction, 6259 /* is_fatal */ false); 6260 codegen_->AddSlowPath(slow_path); 6261 __ b(slow_path->GetEntryLabel()); 6262 if (zero.IsLinked()) { 6263 __ b(&done); 6264 } 6265 break; 6266 } 6267 } 6268 6269 if (zero.IsLinked()) { 6270 __ Bind(&zero); 6271 __ LoadImmediate(out, 0); 6272 } 6273 6274 if (done.IsLinked()) { 6275 __ Bind(&done); 6276 } 6277 6278 if (slow_path != nullptr) { 6279 __ Bind(slow_path->GetExitLabel()); 6280 } 6281} 6282 6283void LocationsBuilderARM::VisitCheckCast(HCheckCast* instruction) { 6284 LocationSummary::CallKind call_kind = LocationSummary::kNoCall; 6285 bool throws_into_catch = instruction->CanThrowIntoCatchBlock(); 6286 6287 TypeCheckKind type_check_kind = instruction->GetTypeCheckKind(); 6288 switch (type_check_kind) { 6289 case TypeCheckKind::kExactCheck: 6290 case TypeCheckKind::kAbstractClassCheck: 6291 case TypeCheckKind::kClassHierarchyCheck: 6292 case TypeCheckKind::kArrayObjectCheck: 6293 call_kind = (throws_into_catch || kEmitCompilerReadBarrier) ? 6294 LocationSummary::kCallOnSlowPath : 6295 LocationSummary::kNoCall; // In fact, call on a fatal (non-returning) slow path. 6296 break; 6297 case TypeCheckKind::kArrayCheck: 6298 case TypeCheckKind::kUnresolvedCheck: 6299 case TypeCheckKind::kInterfaceCheck: 6300 call_kind = LocationSummary::kCallOnSlowPath; 6301 break; 6302 } 6303 6304 LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction, call_kind); 6305 locations->SetInAt(0, Location::RequiresRegister()); 6306 locations->SetInAt(1, Location::RequiresRegister()); 6307 locations->AddRegisterTemps(NumberOfCheckCastTemps(type_check_kind)); 6308} 6309 6310void InstructionCodeGeneratorARM::VisitCheckCast(HCheckCast* instruction) { 6311 TypeCheckKind type_check_kind = instruction->GetTypeCheckKind(); 6312 LocationSummary* locations = instruction->GetLocations(); 6313 Location obj_loc = locations->InAt(0); 6314 Register obj = obj_loc.AsRegister<Register>(); 6315 Register cls = locations->InAt(1).AsRegister<Register>(); 6316 Location temp_loc = locations->GetTemp(0); 6317 Register temp = temp_loc.AsRegister<Register>(); 6318 const size_t num_temps = NumberOfCheckCastTemps(type_check_kind); 6319 DCHECK_LE(num_temps, 3u); 6320 Location maybe_temp2_loc = (num_temps >= 2) ? locations->GetTemp(1) : Location::NoLocation(); 6321 Location maybe_temp3_loc = (num_temps >= 3) ? locations->GetTemp(2) : Location::NoLocation(); 6322 const uint32_t class_offset = mirror::Object::ClassOffset().Int32Value(); 6323 const uint32_t super_offset = mirror::Class::SuperClassOffset().Int32Value(); 6324 const uint32_t component_offset = mirror::Class::ComponentTypeOffset().Int32Value(); 6325 const uint32_t primitive_offset = mirror::Class::PrimitiveTypeOffset().Int32Value(); 6326 const uint32_t iftable_offset = mirror::Class::IfTableOffset().Uint32Value(); 6327 const uint32_t array_length_offset = mirror::Array::LengthOffset().Uint32Value(); 6328 const uint32_t object_array_data_offset = 6329 mirror::Array::DataOffset(kHeapReferenceSize).Uint32Value(); 6330 6331 // Always false for read barriers since we may need to go to the entrypoint for non-fatal cases 6332 // from false negatives. The false negatives may come from avoiding read barriers below. Avoiding 6333 // read barriers is done for performance and code size reasons. 6334 bool is_type_check_slow_path_fatal = false; 6335 if (!kEmitCompilerReadBarrier) { 6336 is_type_check_slow_path_fatal = 6337 (type_check_kind == TypeCheckKind::kExactCheck || 6338 type_check_kind == TypeCheckKind::kAbstractClassCheck || 6339 type_check_kind == TypeCheckKind::kClassHierarchyCheck || 6340 type_check_kind == TypeCheckKind::kArrayObjectCheck) && 6341 !instruction->CanThrowIntoCatchBlock(); 6342 } 6343 SlowPathCodeARM* type_check_slow_path = 6344 new (GetGraph()->GetArena()) TypeCheckSlowPathARM(instruction, 6345 is_type_check_slow_path_fatal); 6346 codegen_->AddSlowPath(type_check_slow_path); 6347 6348 Label done; 6349 // Avoid null check if we know obj is not null. 6350 if (instruction->MustDoNullCheck()) { 6351 __ CompareAndBranchIfZero(obj, &done); 6352 } 6353 6354 switch (type_check_kind) { 6355 case TypeCheckKind::kExactCheck: 6356 case TypeCheckKind::kArrayCheck: { 6357 // /* HeapReference<Class> */ temp = obj->klass_ 6358 GenerateReferenceLoadTwoRegisters(instruction, 6359 temp_loc, 6360 obj_loc, 6361 class_offset, 6362 maybe_temp2_loc, 6363 kWithoutReadBarrier); 6364 6365 __ cmp(temp, ShifterOperand(cls)); 6366 // Jump to slow path for throwing the exception or doing a 6367 // more involved array check. 6368 __ b(type_check_slow_path->GetEntryLabel(), NE); 6369 break; 6370 } 6371 6372 case TypeCheckKind::kAbstractClassCheck: { 6373 // /* HeapReference<Class> */ temp = obj->klass_ 6374 GenerateReferenceLoadTwoRegisters(instruction, 6375 temp_loc, 6376 obj_loc, 6377 class_offset, 6378 maybe_temp2_loc, 6379 kWithoutReadBarrier); 6380 6381 // If the class is abstract, we eagerly fetch the super class of the 6382 // object to avoid doing a comparison we know will fail. 6383 Label loop; 6384 __ Bind(&loop); 6385 // /* HeapReference<Class> */ temp = temp->super_class_ 6386 GenerateReferenceLoadOneRegister(instruction, 6387 temp_loc, 6388 super_offset, 6389 maybe_temp2_loc, 6390 kWithoutReadBarrier); 6391 6392 // If the class reference currently in `temp` is null, jump to the slow path to throw the 6393 // exception. 6394 __ CompareAndBranchIfZero(temp, type_check_slow_path->GetEntryLabel()); 6395 6396 // Otherwise, compare the classes. 6397 __ cmp(temp, ShifterOperand(cls)); 6398 __ b(&loop, NE); 6399 break; 6400 } 6401 6402 case TypeCheckKind::kClassHierarchyCheck: { 6403 // /* HeapReference<Class> */ temp = obj->klass_ 6404 GenerateReferenceLoadTwoRegisters(instruction, 6405 temp_loc, 6406 obj_loc, 6407 class_offset, 6408 maybe_temp2_loc, 6409 kWithoutReadBarrier); 6410 6411 // Walk over the class hierarchy to find a match. 6412 Label loop; 6413 __ Bind(&loop); 6414 __ cmp(temp, ShifterOperand(cls)); 6415 __ b(&done, EQ); 6416 6417 // /* HeapReference<Class> */ temp = temp->super_class_ 6418 GenerateReferenceLoadOneRegister(instruction, 6419 temp_loc, 6420 super_offset, 6421 maybe_temp2_loc, 6422 kWithoutReadBarrier); 6423 6424 // If the class reference currently in `temp` is null, jump to the slow path to throw the 6425 // exception. 6426 __ CompareAndBranchIfZero(temp, type_check_slow_path->GetEntryLabel()); 6427 // Otherwise, jump to the beginning of the loop. 6428 __ b(&loop); 6429 break; 6430 } 6431 6432 case TypeCheckKind::kArrayObjectCheck: { 6433 // /* HeapReference<Class> */ temp = obj->klass_ 6434 GenerateReferenceLoadTwoRegisters(instruction, 6435 temp_loc, 6436 obj_loc, 6437 class_offset, 6438 maybe_temp2_loc, 6439 kWithoutReadBarrier); 6440 6441 // Do an exact check. 6442 __ cmp(temp, ShifterOperand(cls)); 6443 __ b(&done, EQ); 6444 6445 // Otherwise, we need to check that the object's class is a non-primitive array. 6446 // /* HeapReference<Class> */ temp = temp->component_type_ 6447 GenerateReferenceLoadOneRegister(instruction, 6448 temp_loc, 6449 component_offset, 6450 maybe_temp2_loc, 6451 kWithoutReadBarrier); 6452 // If the component type is null, jump to the slow path to throw the exception. 6453 __ CompareAndBranchIfZero(temp, type_check_slow_path->GetEntryLabel()); 6454 // Otherwise,the object is indeed an array, jump to label `check_non_primitive_component_type` 6455 // to further check that this component type is not a primitive type. 6456 __ LoadFromOffset(kLoadUnsignedHalfword, temp, temp, primitive_offset); 6457 static_assert(Primitive::kPrimNot == 0, "Expected 0 for art::Primitive::kPrimNot"); 6458 __ CompareAndBranchIfNonZero(temp, type_check_slow_path->GetEntryLabel()); 6459 break; 6460 } 6461 6462 case TypeCheckKind::kUnresolvedCheck: 6463 // We always go into the type check slow path for the unresolved check case. 6464 // We cannot directly call the CheckCast runtime entry point 6465 // without resorting to a type checking slow path here (i.e. by 6466 // calling InvokeRuntime directly), as it would require to 6467 // assign fixed registers for the inputs of this HInstanceOf 6468 // instruction (following the runtime calling convention), which 6469 // might be cluttered by the potential first read barrier 6470 // emission at the beginning of this method. 6471 6472 __ b(type_check_slow_path->GetEntryLabel()); 6473 break; 6474 6475 case TypeCheckKind::kInterfaceCheck: { 6476 // Avoid read barriers to improve performance of the fast path. We can not get false 6477 // positives by doing this. 6478 // /* HeapReference<Class> */ temp = obj->klass_ 6479 GenerateReferenceLoadTwoRegisters(instruction, 6480 temp_loc, 6481 obj_loc, 6482 class_offset, 6483 maybe_temp2_loc, 6484 kWithoutReadBarrier); 6485 6486 // /* HeapReference<Class> */ temp = temp->iftable_ 6487 GenerateReferenceLoadTwoRegisters(instruction, 6488 temp_loc, 6489 temp_loc, 6490 iftable_offset, 6491 maybe_temp2_loc, 6492 kWithoutReadBarrier); 6493 // Null iftable means it is empty and will always fail the check. 6494 __ CompareAndBranchIfZero(temp, type_check_slow_path->GetEntryLabel()); 6495 6496 // Loop through the iftable and check if any class matches. 6497 __ ldr(maybe_temp2_loc.AsRegister<Register>(), Address(temp, array_length_offset)); 6498 6499 Label start_loop; 6500 __ Bind(&start_loop); 6501 __ CompareAndBranchIfZero(maybe_temp2_loc.AsRegister<Register>(), 6502 type_check_slow_path->GetEntryLabel()); 6503 __ ldr(maybe_temp3_loc.AsRegister<Register>(), Address(temp, object_array_data_offset)); 6504 __ MaybeUnpoisonHeapReference(maybe_temp3_loc.AsRegister<Register>()); 6505 // Go to next interface. 6506 __ add(temp, temp, ShifterOperand(2 * kHeapReferenceSize)); 6507 __ sub(maybe_temp2_loc.AsRegister<Register>(), 6508 maybe_temp2_loc.AsRegister<Register>(), 6509 ShifterOperand(2)); 6510 // Compare the classes and continue the loop if they do not match. 6511 __ cmp(cls, ShifterOperand(maybe_temp3_loc.AsRegister<Register>())); 6512 __ b(&start_loop, NE); 6513 break; 6514 } 6515 } 6516 __ Bind(&done); 6517 6518 __ Bind(type_check_slow_path->GetExitLabel()); 6519} 6520 6521void LocationsBuilderARM::VisitMonitorOperation(HMonitorOperation* instruction) { 6522 LocationSummary* locations = 6523 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCallOnMainOnly); 6524 InvokeRuntimeCallingConvention calling_convention; 6525 locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(0))); 6526} 6527 6528void InstructionCodeGeneratorARM::VisitMonitorOperation(HMonitorOperation* instruction) { 6529 codegen_->InvokeRuntime(instruction->IsEnter() ? kQuickLockObject : kQuickUnlockObject, 6530 instruction, 6531 instruction->GetDexPc()); 6532 if (instruction->IsEnter()) { 6533 CheckEntrypointTypes<kQuickLockObject, void, mirror::Object*>(); 6534 } else { 6535 CheckEntrypointTypes<kQuickUnlockObject, void, mirror::Object*>(); 6536 } 6537} 6538 6539void LocationsBuilderARM::VisitAnd(HAnd* instruction) { HandleBitwiseOperation(instruction, AND); } 6540void LocationsBuilderARM::VisitOr(HOr* instruction) { HandleBitwiseOperation(instruction, ORR); } 6541void LocationsBuilderARM::VisitXor(HXor* instruction) { HandleBitwiseOperation(instruction, EOR); } 6542 6543void LocationsBuilderARM::HandleBitwiseOperation(HBinaryOperation* instruction, Opcode opcode) { 6544 LocationSummary* locations = 6545 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall); 6546 DCHECK(instruction->GetResultType() == Primitive::kPrimInt 6547 || instruction->GetResultType() == Primitive::kPrimLong); 6548 // Note: GVN reorders commutative operations to have the constant on the right hand side. 6549 locations->SetInAt(0, Location::RequiresRegister()); 6550 locations->SetInAt(1, ArmEncodableConstantOrRegister(instruction->InputAt(1), opcode)); 6551 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 6552} 6553 6554void InstructionCodeGeneratorARM::VisitAnd(HAnd* instruction) { 6555 HandleBitwiseOperation(instruction); 6556} 6557 6558void InstructionCodeGeneratorARM::VisitOr(HOr* instruction) { 6559 HandleBitwiseOperation(instruction); 6560} 6561 6562void InstructionCodeGeneratorARM::VisitXor(HXor* instruction) { 6563 HandleBitwiseOperation(instruction); 6564} 6565 6566 6567void LocationsBuilderARM::VisitBitwiseNegatedRight(HBitwiseNegatedRight* instruction) { 6568 LocationSummary* locations = 6569 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall); 6570 DCHECK(instruction->GetResultType() == Primitive::kPrimInt 6571 || instruction->GetResultType() == Primitive::kPrimLong); 6572 6573 locations->SetInAt(0, Location::RequiresRegister()); 6574 locations->SetInAt(1, Location::RequiresRegister()); 6575 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 6576} 6577 6578void InstructionCodeGeneratorARM::VisitBitwiseNegatedRight(HBitwiseNegatedRight* instruction) { 6579 LocationSummary* locations = instruction->GetLocations(); 6580 Location first = locations->InAt(0); 6581 Location second = locations->InAt(1); 6582 Location out = locations->Out(); 6583 6584 if (instruction->GetResultType() == Primitive::kPrimInt) { 6585 Register first_reg = first.AsRegister<Register>(); 6586 ShifterOperand second_reg(second.AsRegister<Register>()); 6587 Register out_reg = out.AsRegister<Register>(); 6588 6589 switch (instruction->GetOpKind()) { 6590 case HInstruction::kAnd: 6591 __ bic(out_reg, first_reg, second_reg); 6592 break; 6593 case HInstruction::kOr: 6594 __ orn(out_reg, first_reg, second_reg); 6595 break; 6596 // There is no EON on arm. 6597 case HInstruction::kXor: 6598 default: 6599 LOG(FATAL) << "Unexpected instruction " << instruction->DebugName(); 6600 UNREACHABLE(); 6601 } 6602 return; 6603 6604 } else { 6605 DCHECK_EQ(instruction->GetResultType(), Primitive::kPrimLong); 6606 Register first_low = first.AsRegisterPairLow<Register>(); 6607 Register first_high = first.AsRegisterPairHigh<Register>(); 6608 ShifterOperand second_low(second.AsRegisterPairLow<Register>()); 6609 ShifterOperand second_high(second.AsRegisterPairHigh<Register>()); 6610 Register out_low = out.AsRegisterPairLow<Register>(); 6611 Register out_high = out.AsRegisterPairHigh<Register>(); 6612 6613 switch (instruction->GetOpKind()) { 6614 case HInstruction::kAnd: 6615 __ bic(out_low, first_low, second_low); 6616 __ bic(out_high, first_high, second_high); 6617 break; 6618 case HInstruction::kOr: 6619 __ orn(out_low, first_low, second_low); 6620 __ orn(out_high, first_high, second_high); 6621 break; 6622 // There is no EON on arm. 6623 case HInstruction::kXor: 6624 default: 6625 LOG(FATAL) << "Unexpected instruction " << instruction->DebugName(); 6626 UNREACHABLE(); 6627 } 6628 } 6629} 6630 6631void InstructionCodeGeneratorARM::GenerateAndConst(Register out, Register first, uint32_t value) { 6632 // Optimize special cases for individual halfs of `and-long` (`and` is simplified earlier). 6633 if (value == 0xffffffffu) { 6634 if (out != first) { 6635 __ mov(out, ShifterOperand(first)); 6636 } 6637 return; 6638 } 6639 if (value == 0u) { 6640 __ mov(out, ShifterOperand(0)); 6641 return; 6642 } 6643 ShifterOperand so; 6644 if (__ ShifterOperandCanHold(kNoRegister, kNoRegister, AND, value, &so)) { 6645 __ and_(out, first, so); 6646 } else { 6647 DCHECK(__ ShifterOperandCanHold(kNoRegister, kNoRegister, BIC, ~value, &so)); 6648 __ bic(out, first, ShifterOperand(~value)); 6649 } 6650} 6651 6652void InstructionCodeGeneratorARM::GenerateOrrConst(Register out, Register first, uint32_t value) { 6653 // Optimize special cases for individual halfs of `or-long` (`or` is simplified earlier). 6654 if (value == 0u) { 6655 if (out != first) { 6656 __ mov(out, ShifterOperand(first)); 6657 } 6658 return; 6659 } 6660 if (value == 0xffffffffu) { 6661 __ mvn(out, ShifterOperand(0)); 6662 return; 6663 } 6664 ShifterOperand so; 6665 if (__ ShifterOperandCanHold(kNoRegister, kNoRegister, ORR, value, &so)) { 6666 __ orr(out, first, so); 6667 } else { 6668 DCHECK(__ ShifterOperandCanHold(kNoRegister, kNoRegister, ORN, ~value, &so)); 6669 __ orn(out, first, ShifterOperand(~value)); 6670 } 6671} 6672 6673void InstructionCodeGeneratorARM::GenerateEorConst(Register out, Register first, uint32_t value) { 6674 // Optimize special case for individual halfs of `xor-long` (`xor` is simplified earlier). 6675 if (value == 0u) { 6676 if (out != first) { 6677 __ mov(out, ShifterOperand(first)); 6678 } 6679 return; 6680 } 6681 __ eor(out, first, ShifterOperand(value)); 6682} 6683 6684void InstructionCodeGeneratorARM::GenerateAddLongConst(Location out, 6685 Location first, 6686 uint64_t value) { 6687 Register out_low = out.AsRegisterPairLow<Register>(); 6688 Register out_high = out.AsRegisterPairHigh<Register>(); 6689 Register first_low = first.AsRegisterPairLow<Register>(); 6690 Register first_high = first.AsRegisterPairHigh<Register>(); 6691 uint32_t value_low = Low32Bits(value); 6692 uint32_t value_high = High32Bits(value); 6693 if (value_low == 0u) { 6694 if (out_low != first_low) { 6695 __ mov(out_low, ShifterOperand(first_low)); 6696 } 6697 __ AddConstant(out_high, first_high, value_high); 6698 return; 6699 } 6700 __ AddConstantSetFlags(out_low, first_low, value_low); 6701 ShifterOperand so; 6702 if (__ ShifterOperandCanHold(out_high, first_high, ADC, value_high, kCcDontCare, &so)) { 6703 __ adc(out_high, first_high, so); 6704 } else if (__ ShifterOperandCanHold(out_low, first_low, SBC, ~value_high, kCcDontCare, &so)) { 6705 __ sbc(out_high, first_high, so); 6706 } else { 6707 LOG(FATAL) << "Unexpected constant " << value_high; 6708 UNREACHABLE(); 6709 } 6710} 6711 6712void InstructionCodeGeneratorARM::HandleBitwiseOperation(HBinaryOperation* instruction) { 6713 LocationSummary* locations = instruction->GetLocations(); 6714 Location first = locations->InAt(0); 6715 Location second = locations->InAt(1); 6716 Location out = locations->Out(); 6717 6718 if (second.IsConstant()) { 6719 uint64_t value = static_cast<uint64_t>(Int64FromConstant(second.GetConstant())); 6720 uint32_t value_low = Low32Bits(value); 6721 if (instruction->GetResultType() == Primitive::kPrimInt) { 6722 Register first_reg = first.AsRegister<Register>(); 6723 Register out_reg = out.AsRegister<Register>(); 6724 if (instruction->IsAnd()) { 6725 GenerateAndConst(out_reg, first_reg, value_low); 6726 } else if (instruction->IsOr()) { 6727 GenerateOrrConst(out_reg, first_reg, value_low); 6728 } else { 6729 DCHECK(instruction->IsXor()); 6730 GenerateEorConst(out_reg, first_reg, value_low); 6731 } 6732 } else { 6733 DCHECK_EQ(instruction->GetResultType(), Primitive::kPrimLong); 6734 uint32_t value_high = High32Bits(value); 6735 Register first_low = first.AsRegisterPairLow<Register>(); 6736 Register first_high = first.AsRegisterPairHigh<Register>(); 6737 Register out_low = out.AsRegisterPairLow<Register>(); 6738 Register out_high = out.AsRegisterPairHigh<Register>(); 6739 if (instruction->IsAnd()) { 6740 GenerateAndConst(out_low, first_low, value_low); 6741 GenerateAndConst(out_high, first_high, value_high); 6742 } else if (instruction->IsOr()) { 6743 GenerateOrrConst(out_low, first_low, value_low); 6744 GenerateOrrConst(out_high, first_high, value_high); 6745 } else { 6746 DCHECK(instruction->IsXor()); 6747 GenerateEorConst(out_low, first_low, value_low); 6748 GenerateEorConst(out_high, first_high, value_high); 6749 } 6750 } 6751 return; 6752 } 6753 6754 if (instruction->GetResultType() == Primitive::kPrimInt) { 6755 Register first_reg = first.AsRegister<Register>(); 6756 ShifterOperand second_reg(second.AsRegister<Register>()); 6757 Register out_reg = out.AsRegister<Register>(); 6758 if (instruction->IsAnd()) { 6759 __ and_(out_reg, first_reg, second_reg); 6760 } else if (instruction->IsOr()) { 6761 __ orr(out_reg, first_reg, second_reg); 6762 } else { 6763 DCHECK(instruction->IsXor()); 6764 __ eor(out_reg, first_reg, second_reg); 6765 } 6766 } else { 6767 DCHECK_EQ(instruction->GetResultType(), Primitive::kPrimLong); 6768 Register first_low = first.AsRegisterPairLow<Register>(); 6769 Register first_high = first.AsRegisterPairHigh<Register>(); 6770 ShifterOperand second_low(second.AsRegisterPairLow<Register>()); 6771 ShifterOperand second_high(second.AsRegisterPairHigh<Register>()); 6772 Register out_low = out.AsRegisterPairLow<Register>(); 6773 Register out_high = out.AsRegisterPairHigh<Register>(); 6774 if (instruction->IsAnd()) { 6775 __ and_(out_low, first_low, second_low); 6776 __ and_(out_high, first_high, second_high); 6777 } else if (instruction->IsOr()) { 6778 __ orr(out_low, first_low, second_low); 6779 __ orr(out_high, first_high, second_high); 6780 } else { 6781 DCHECK(instruction->IsXor()); 6782 __ eor(out_low, first_low, second_low); 6783 __ eor(out_high, first_high, second_high); 6784 } 6785 } 6786} 6787 6788void InstructionCodeGeneratorARM::GenerateReferenceLoadOneRegister( 6789 HInstruction* instruction, 6790 Location out, 6791 uint32_t offset, 6792 Location maybe_temp, 6793 ReadBarrierOption read_barrier_option) { 6794 Register out_reg = out.AsRegister<Register>(); 6795 if (read_barrier_option == kWithReadBarrier) { 6796 CHECK(kEmitCompilerReadBarrier); 6797 DCHECK(maybe_temp.IsRegister()) << maybe_temp; 6798 if (kUseBakerReadBarrier) { 6799 // Load with fast path based Baker's read barrier. 6800 // /* HeapReference<Object> */ out = *(out + offset) 6801 codegen_->GenerateFieldLoadWithBakerReadBarrier( 6802 instruction, out, out_reg, offset, maybe_temp, /* needs_null_check */ false); 6803 } else { 6804 // Load with slow path based read barrier. 6805 // Save the value of `out` into `maybe_temp` before overwriting it 6806 // in the following move operation, as we will need it for the 6807 // read barrier below. 6808 __ Mov(maybe_temp.AsRegister<Register>(), out_reg); 6809 // /* HeapReference<Object> */ out = *(out + offset) 6810 __ LoadFromOffset(kLoadWord, out_reg, out_reg, offset); 6811 codegen_->GenerateReadBarrierSlow(instruction, out, out, maybe_temp, offset); 6812 } 6813 } else { 6814 // Plain load with no read barrier. 6815 // /* HeapReference<Object> */ out = *(out + offset) 6816 __ LoadFromOffset(kLoadWord, out_reg, out_reg, offset); 6817 __ MaybeUnpoisonHeapReference(out_reg); 6818 } 6819} 6820 6821void InstructionCodeGeneratorARM::GenerateReferenceLoadTwoRegisters( 6822 HInstruction* instruction, 6823 Location out, 6824 Location obj, 6825 uint32_t offset, 6826 Location maybe_temp, 6827 ReadBarrierOption read_barrier_option) { 6828 Register out_reg = out.AsRegister<Register>(); 6829 Register obj_reg = obj.AsRegister<Register>(); 6830 if (read_barrier_option == kWithReadBarrier) { 6831 CHECK(kEmitCompilerReadBarrier); 6832 if (kUseBakerReadBarrier) { 6833 DCHECK(maybe_temp.IsRegister()) << maybe_temp; 6834 // Load with fast path based Baker's read barrier. 6835 // /* HeapReference<Object> */ out = *(obj + offset) 6836 codegen_->GenerateFieldLoadWithBakerReadBarrier( 6837 instruction, out, obj_reg, offset, maybe_temp, /* needs_null_check */ false); 6838 } else { 6839 // Load with slow path based read barrier. 6840 // /* HeapReference<Object> */ out = *(obj + offset) 6841 __ LoadFromOffset(kLoadWord, out_reg, obj_reg, offset); 6842 codegen_->GenerateReadBarrierSlow(instruction, out, out, obj, offset); 6843 } 6844 } else { 6845 // Plain load with no read barrier. 6846 // /* HeapReference<Object> */ out = *(obj + offset) 6847 __ LoadFromOffset(kLoadWord, out_reg, obj_reg, offset); 6848 __ MaybeUnpoisonHeapReference(out_reg); 6849 } 6850} 6851 6852void InstructionCodeGeneratorARM::GenerateGcRootFieldLoad(HInstruction* instruction, 6853 Location root, 6854 Register obj, 6855 uint32_t offset, 6856 ReadBarrierOption read_barrier_option) { 6857 Register root_reg = root.AsRegister<Register>(); 6858 if (read_barrier_option == kWithReadBarrier) { 6859 DCHECK(kEmitCompilerReadBarrier); 6860 if (kUseBakerReadBarrier) { 6861 // Fast path implementation of art::ReadBarrier::BarrierForRoot when 6862 // Baker's read barrier are used: 6863 // 6864 // root = obj.field; 6865 // temp = Thread::Current()->pReadBarrierMarkReg ## root.reg() 6866 // if (temp != null) { 6867 // root = temp(root) 6868 // } 6869 6870 // /* GcRoot<mirror::Object> */ root = *(obj + offset) 6871 __ LoadFromOffset(kLoadWord, root_reg, obj, offset); 6872 static_assert( 6873 sizeof(mirror::CompressedReference<mirror::Object>) == sizeof(GcRoot<mirror::Object>), 6874 "art::mirror::CompressedReference<mirror::Object> and art::GcRoot<mirror::Object> " 6875 "have different sizes."); 6876 static_assert(sizeof(mirror::CompressedReference<mirror::Object>) == sizeof(int32_t), 6877 "art::mirror::CompressedReference<mirror::Object> and int32_t " 6878 "have different sizes."); 6879 6880 // Slow path marking the GC root `root`. 6881 Location temp = Location::RegisterLocation(LR); 6882 SlowPathCodeARM* slow_path = 6883 new (GetGraph()->GetArena()) ReadBarrierMarkSlowPathARM( 6884 instruction, 6885 root, 6886 /*entrypoint*/ temp); 6887 codegen_->AddSlowPath(slow_path); 6888 6889 // temp = Thread::Current()->pReadBarrierMarkReg ## root.reg() 6890 const int32_t entry_point_offset = 6891 CodeGenerator::GetReadBarrierMarkEntryPointsOffset<kArmPointerSize>(root.reg()); 6892 // Loading the entrypoint does not require a load acquire since it is only changed when 6893 // threads are suspended or running a checkpoint. 6894 __ LoadFromOffset(kLoadWord, temp.AsRegister<Register>(), TR, entry_point_offset); 6895 // The entrypoint is null when the GC is not marking, this prevents one load compared to 6896 // checking GetIsGcMarking. 6897 __ CompareAndBranchIfNonZero(temp.AsRegister<Register>(), slow_path->GetEntryLabel()); 6898 __ Bind(slow_path->GetExitLabel()); 6899 } else { 6900 // GC root loaded through a slow path for read barriers other 6901 // than Baker's. 6902 // /* GcRoot<mirror::Object>* */ root = obj + offset 6903 __ AddConstant(root_reg, obj, offset); 6904 // /* mirror::Object* */ root = root->Read() 6905 codegen_->GenerateReadBarrierForRootSlow(instruction, root, root); 6906 } 6907 } else { 6908 // Plain GC root load with no read barrier. 6909 // /* GcRoot<mirror::Object> */ root = *(obj + offset) 6910 __ LoadFromOffset(kLoadWord, root_reg, obj, offset); 6911 // Note that GC roots are not affected by heap poisoning, thus we 6912 // do not have to unpoison `root_reg` here. 6913 } 6914} 6915 6916void CodeGeneratorARM::GenerateFieldLoadWithBakerReadBarrier(HInstruction* instruction, 6917 Location ref, 6918 Register obj, 6919 uint32_t offset, 6920 Location temp, 6921 bool needs_null_check) { 6922 DCHECK(kEmitCompilerReadBarrier); 6923 DCHECK(kUseBakerReadBarrier); 6924 6925 // /* HeapReference<Object> */ ref = *(obj + offset) 6926 Location no_index = Location::NoLocation(); 6927 ScaleFactor no_scale_factor = TIMES_1; 6928 GenerateReferenceLoadWithBakerReadBarrier( 6929 instruction, ref, obj, offset, no_index, no_scale_factor, temp, needs_null_check); 6930} 6931 6932void CodeGeneratorARM::GenerateArrayLoadWithBakerReadBarrier(HInstruction* instruction, 6933 Location ref, 6934 Register obj, 6935 uint32_t data_offset, 6936 Location index, 6937 Location temp, 6938 bool needs_null_check) { 6939 DCHECK(kEmitCompilerReadBarrier); 6940 DCHECK(kUseBakerReadBarrier); 6941 6942 static_assert( 6943 sizeof(mirror::HeapReference<mirror::Object>) == sizeof(int32_t), 6944 "art::mirror::HeapReference<art::mirror::Object> and int32_t have different sizes."); 6945 // /* HeapReference<Object> */ ref = 6946 // *(obj + data_offset + index * sizeof(HeapReference<Object>)) 6947 ScaleFactor scale_factor = TIMES_4; 6948 GenerateReferenceLoadWithBakerReadBarrier( 6949 instruction, ref, obj, data_offset, index, scale_factor, temp, needs_null_check); 6950} 6951 6952void CodeGeneratorARM::GenerateReferenceLoadWithBakerReadBarrier(HInstruction* instruction, 6953 Location ref, 6954 Register obj, 6955 uint32_t offset, 6956 Location index, 6957 ScaleFactor scale_factor, 6958 Location temp, 6959 bool needs_null_check, 6960 bool always_update_field, 6961 Register* temp2) { 6962 DCHECK(kEmitCompilerReadBarrier); 6963 DCHECK(kUseBakerReadBarrier); 6964 6965 // In slow path based read barriers, the read barrier call is 6966 // inserted after the original load. However, in fast path based 6967 // Baker's read barriers, we need to perform the load of 6968 // mirror::Object::monitor_ *before* the original reference load. 6969 // This load-load ordering is required by the read barrier. 6970 // The fast path/slow path (for Baker's algorithm) should look like: 6971 // 6972 // uint32_t rb_state = Lockword(obj->monitor_).ReadBarrierState(); 6973 // lfence; // Load fence or artificial data dependency to prevent load-load reordering 6974 // HeapReference<Object> ref = *src; // Original reference load. 6975 // bool is_gray = (rb_state == ReadBarrier::GrayState()); 6976 // if (is_gray) { 6977 // ref = ReadBarrier::Mark(ref); // Performed by runtime entrypoint slow path. 6978 // } 6979 // 6980 // Note: the original implementation in ReadBarrier::Barrier is 6981 // slightly more complex as it performs additional checks that we do 6982 // not do here for performance reasons. 6983 6984 Register ref_reg = ref.AsRegister<Register>(); 6985 Register temp_reg = temp.AsRegister<Register>(); 6986 uint32_t monitor_offset = mirror::Object::MonitorOffset().Int32Value(); 6987 6988 // /* int32_t */ monitor = obj->monitor_ 6989 __ LoadFromOffset(kLoadWord, temp_reg, obj, monitor_offset); 6990 if (needs_null_check) { 6991 MaybeRecordImplicitNullCheck(instruction); 6992 } 6993 // /* LockWord */ lock_word = LockWord(monitor) 6994 static_assert(sizeof(LockWord) == sizeof(int32_t), 6995 "art::LockWord and int32_t have different sizes."); 6996 6997 // Introduce a dependency on the lock_word including the rb_state, 6998 // which shall prevent load-load reordering without using 6999 // a memory barrier (which would be more expensive). 7000 // `obj` is unchanged by this operation, but its value now depends 7001 // on `temp_reg`. 7002 __ add(obj, obj, ShifterOperand(temp_reg, LSR, 32)); 7003 7004 // The actual reference load. 7005 if (index.IsValid()) { 7006 // Load types involving an "index": ArrayGet, 7007 // UnsafeGetObject/UnsafeGetObjectVolatile and UnsafeCASObject 7008 // intrinsics. 7009 // /* HeapReference<Object> */ ref = *(obj + offset + (index << scale_factor)) 7010 if (index.IsConstant()) { 7011 size_t computed_offset = 7012 (index.GetConstant()->AsIntConstant()->GetValue() << scale_factor) + offset; 7013 __ LoadFromOffset(kLoadWord, ref_reg, obj, computed_offset); 7014 } else { 7015 // Handle the special case of the 7016 // UnsafeGetObject/UnsafeGetObjectVolatile and UnsafeCASObject 7017 // intrinsics, which use a register pair as index ("long 7018 // offset"), of which only the low part contains data. 7019 Register index_reg = index.IsRegisterPair() 7020 ? index.AsRegisterPairLow<Register>() 7021 : index.AsRegister<Register>(); 7022 __ add(IP, obj, ShifterOperand(index_reg, LSL, scale_factor)); 7023 __ LoadFromOffset(kLoadWord, ref_reg, IP, offset); 7024 } 7025 } else { 7026 // /* HeapReference<Object> */ ref = *(obj + offset) 7027 __ LoadFromOffset(kLoadWord, ref_reg, obj, offset); 7028 } 7029 7030 // Object* ref = ref_addr->AsMirrorPtr() 7031 __ MaybeUnpoisonHeapReference(ref_reg); 7032 7033 // Slow path marking the object `ref` when it is gray. 7034 SlowPathCodeARM* slow_path; 7035 if (always_update_field) { 7036 DCHECK(temp2 != nullptr); 7037 // ReadBarrierMarkAndUpdateFieldSlowPathARM only supports address 7038 // of the form `obj + field_offset`, where `obj` is a register and 7039 // `field_offset` is a register pair (of which only the lower half 7040 // is used). Thus `offset` and `scale_factor` above are expected 7041 // to be null in this code path. 7042 DCHECK_EQ(offset, 0u); 7043 DCHECK_EQ(scale_factor, ScaleFactor::TIMES_1); 7044 slow_path = new (GetGraph()->GetArena()) ReadBarrierMarkAndUpdateFieldSlowPathARM( 7045 instruction, ref, obj, /* field_offset */ index, temp_reg, *temp2); 7046 } else { 7047 slow_path = new (GetGraph()->GetArena()) ReadBarrierMarkSlowPathARM(instruction, ref); 7048 } 7049 AddSlowPath(slow_path); 7050 7051 // if (rb_state == ReadBarrier::GrayState()) 7052 // ref = ReadBarrier::Mark(ref); 7053 // Given the numeric representation, it's enough to check the low bit of the 7054 // rb_state. We do that by shifting the bit out of the lock word with LSRS 7055 // which can be a 16-bit instruction unlike the TST immediate. 7056 static_assert(ReadBarrier::WhiteState() == 0, "Expecting white to have value 0"); 7057 static_assert(ReadBarrier::GrayState() == 1, "Expecting gray to have value 1"); 7058 __ Lsrs(temp_reg, temp_reg, LockWord::kReadBarrierStateShift + 1); 7059 __ b(slow_path->GetEntryLabel(), CS); // Carry flag is the last bit shifted out by LSRS. 7060 __ Bind(slow_path->GetExitLabel()); 7061} 7062 7063void CodeGeneratorARM::GenerateReadBarrierSlow(HInstruction* instruction, 7064 Location out, 7065 Location ref, 7066 Location obj, 7067 uint32_t offset, 7068 Location index) { 7069 DCHECK(kEmitCompilerReadBarrier); 7070 7071 // Insert a slow path based read barrier *after* the reference load. 7072 // 7073 // If heap poisoning is enabled, the unpoisoning of the loaded 7074 // reference will be carried out by the runtime within the slow 7075 // path. 7076 // 7077 // Note that `ref` currently does not get unpoisoned (when heap 7078 // poisoning is enabled), which is alright as the `ref` argument is 7079 // not used by the artReadBarrierSlow entry point. 7080 // 7081 // TODO: Unpoison `ref` when it is used by artReadBarrierSlow. 7082 SlowPathCodeARM* slow_path = new (GetGraph()->GetArena()) 7083 ReadBarrierForHeapReferenceSlowPathARM(instruction, out, ref, obj, offset, index); 7084 AddSlowPath(slow_path); 7085 7086 __ b(slow_path->GetEntryLabel()); 7087 __ Bind(slow_path->GetExitLabel()); 7088} 7089 7090void CodeGeneratorARM::MaybeGenerateReadBarrierSlow(HInstruction* instruction, 7091 Location out, 7092 Location ref, 7093 Location obj, 7094 uint32_t offset, 7095 Location index) { 7096 if (kEmitCompilerReadBarrier) { 7097 // Baker's read barriers shall be handled by the fast path 7098 // (CodeGeneratorARM::GenerateReferenceLoadWithBakerReadBarrier). 7099 DCHECK(!kUseBakerReadBarrier); 7100 // If heap poisoning is enabled, unpoisoning will be taken care of 7101 // by the runtime within the slow path. 7102 GenerateReadBarrierSlow(instruction, out, ref, obj, offset, index); 7103 } else if (kPoisonHeapReferences) { 7104 __ UnpoisonHeapReference(out.AsRegister<Register>()); 7105 } 7106} 7107 7108void CodeGeneratorARM::GenerateReadBarrierForRootSlow(HInstruction* instruction, 7109 Location out, 7110 Location root) { 7111 DCHECK(kEmitCompilerReadBarrier); 7112 7113 // Insert a slow path based read barrier *after* the GC root load. 7114 // 7115 // Note that GC roots are not affected by heap poisoning, so we do 7116 // not need to do anything special for this here. 7117 SlowPathCodeARM* slow_path = 7118 new (GetGraph()->GetArena()) ReadBarrierForRootSlowPathARM(instruction, out, root); 7119 AddSlowPath(slow_path); 7120 7121 __ b(slow_path->GetEntryLabel()); 7122 __ Bind(slow_path->GetExitLabel()); 7123} 7124 7125HInvokeStaticOrDirect::DispatchInfo CodeGeneratorARM::GetSupportedInvokeStaticOrDirectDispatch( 7126 const HInvokeStaticOrDirect::DispatchInfo& desired_dispatch_info, 7127 HInvokeStaticOrDirect* invoke) { 7128 HInvokeStaticOrDirect::DispatchInfo dispatch_info = desired_dispatch_info; 7129 // We disable pc-relative load when there is an irreducible loop, as the optimization 7130 // is incompatible with it. 7131 // TODO: Create as many ArmDexCacheArraysBase instructions as needed for methods 7132 // with irreducible loops. 7133 if (GetGraph()->HasIrreducibleLoops() && 7134 (dispatch_info.method_load_kind == 7135 HInvokeStaticOrDirect::MethodLoadKind::kDexCachePcRelative)) { 7136 dispatch_info.method_load_kind = HInvokeStaticOrDirect::MethodLoadKind::kDexCacheViaMethod; 7137 } 7138 7139 if (dispatch_info.code_ptr_location == HInvokeStaticOrDirect::CodePtrLocation::kCallPCRelative) { 7140 const DexFile& outer_dex_file = GetGraph()->GetDexFile(); 7141 if (&outer_dex_file != invoke->GetTargetMethod().dex_file) { 7142 // Calls across dex files are more likely to exceed the available BL range, 7143 // so use absolute patch with fixup if available and kCallArtMethod otherwise. 7144 HInvokeStaticOrDirect::CodePtrLocation code_ptr_location = 7145 (desired_dispatch_info.method_load_kind == 7146 HInvokeStaticOrDirect::MethodLoadKind::kDirectAddressWithFixup) 7147 ? HInvokeStaticOrDirect::CodePtrLocation::kCallDirectWithFixup 7148 : HInvokeStaticOrDirect::CodePtrLocation::kCallArtMethod; 7149 return HInvokeStaticOrDirect::DispatchInfo { 7150 dispatch_info.method_load_kind, 7151 code_ptr_location, 7152 dispatch_info.method_load_data, 7153 0u 7154 }; 7155 } 7156 } 7157 return dispatch_info; 7158} 7159 7160Register CodeGeneratorARM::GetInvokeStaticOrDirectExtraParameter(HInvokeStaticOrDirect* invoke, 7161 Register temp) { 7162 DCHECK_EQ(invoke->InputCount(), invoke->GetNumberOfArguments() + 1u); 7163 Location location = invoke->GetLocations()->InAt(invoke->GetSpecialInputIndex()); 7164 if (!invoke->GetLocations()->Intrinsified()) { 7165 return location.AsRegister<Register>(); 7166 } 7167 // For intrinsics we allow any location, so it may be on the stack. 7168 if (!location.IsRegister()) { 7169 __ LoadFromOffset(kLoadWord, temp, SP, location.GetStackIndex()); 7170 return temp; 7171 } 7172 // For register locations, check if the register was saved. If so, get it from the stack. 7173 // Note: There is a chance that the register was saved but not overwritten, so we could 7174 // save one load. However, since this is just an intrinsic slow path we prefer this 7175 // simple and more robust approach rather that trying to determine if that's the case. 7176 SlowPathCode* slow_path = GetCurrentSlowPath(); 7177 DCHECK(slow_path != nullptr); // For intrinsified invokes the call is emitted on the slow path. 7178 if (slow_path->IsCoreRegisterSaved(location.AsRegister<Register>())) { 7179 int stack_offset = slow_path->GetStackOffsetOfCoreRegister(location.AsRegister<Register>()); 7180 __ LoadFromOffset(kLoadWord, temp, SP, stack_offset); 7181 return temp; 7182 } 7183 return location.AsRegister<Register>(); 7184} 7185 7186void CodeGeneratorARM::GenerateStaticOrDirectCall(HInvokeStaticOrDirect* invoke, Location temp) { 7187 // For better instruction scheduling we load the direct code pointer before the method pointer. 7188 switch (invoke->GetCodePtrLocation()) { 7189 case HInvokeStaticOrDirect::CodePtrLocation::kCallDirectWithFixup: 7190 // LR = code address from literal pool with link-time patch. 7191 __ LoadLiteral(LR, DeduplicateMethodCodeLiteral(invoke->GetTargetMethod())); 7192 break; 7193 case HInvokeStaticOrDirect::CodePtrLocation::kCallDirect: 7194 // LR = invoke->GetDirectCodePtr(); 7195 __ LoadImmediate(LR, invoke->GetDirectCodePtr()); 7196 break; 7197 default: 7198 break; 7199 } 7200 7201 Location callee_method = temp; // For all kinds except kRecursive, callee will be in temp. 7202 switch (invoke->GetMethodLoadKind()) { 7203 case HInvokeStaticOrDirect::MethodLoadKind::kStringInit: { 7204 uint32_t offset = 7205 GetThreadOffset<kArmPointerSize>(invoke->GetStringInitEntryPoint()).Int32Value(); 7206 // temp = thread->string_init_entrypoint 7207 __ LoadFromOffset(kLoadWord, temp.AsRegister<Register>(), TR, offset); 7208 break; 7209 } 7210 case HInvokeStaticOrDirect::MethodLoadKind::kRecursive: 7211 callee_method = invoke->GetLocations()->InAt(invoke->GetSpecialInputIndex()); 7212 break; 7213 case HInvokeStaticOrDirect::MethodLoadKind::kDirectAddress: 7214 __ LoadImmediate(temp.AsRegister<Register>(), invoke->GetMethodAddress()); 7215 break; 7216 case HInvokeStaticOrDirect::MethodLoadKind::kDirectAddressWithFixup: 7217 __ LoadLiteral(temp.AsRegister<Register>(), 7218 DeduplicateMethodAddressLiteral(invoke->GetTargetMethod())); 7219 break; 7220 case HInvokeStaticOrDirect::MethodLoadKind::kDexCachePcRelative: { 7221 HArmDexCacheArraysBase* base = 7222 invoke->InputAt(invoke->GetSpecialInputIndex())->AsArmDexCacheArraysBase(); 7223 Register base_reg = GetInvokeStaticOrDirectExtraParameter(invoke, 7224 temp.AsRegister<Register>()); 7225 int32_t offset = invoke->GetDexCacheArrayOffset() - base->GetElementOffset(); 7226 __ LoadFromOffset(kLoadWord, temp.AsRegister<Register>(), base_reg, offset); 7227 break; 7228 } 7229 case HInvokeStaticOrDirect::MethodLoadKind::kDexCacheViaMethod: { 7230 Location current_method = invoke->GetLocations()->InAt(invoke->GetSpecialInputIndex()); 7231 Register method_reg; 7232 Register reg = temp.AsRegister<Register>(); 7233 if (current_method.IsRegister()) { 7234 method_reg = current_method.AsRegister<Register>(); 7235 } else { 7236 DCHECK(invoke->GetLocations()->Intrinsified()); 7237 DCHECK(!current_method.IsValid()); 7238 method_reg = reg; 7239 __ LoadFromOffset(kLoadWord, reg, SP, kCurrentMethodStackOffset); 7240 } 7241 // /* ArtMethod*[] */ temp = temp.ptr_sized_fields_->dex_cache_resolved_methods_; 7242 __ LoadFromOffset(kLoadWord, 7243 reg, 7244 method_reg, 7245 ArtMethod::DexCacheResolvedMethodsOffset(kArmPointerSize).Int32Value()); 7246 // temp = temp[index_in_cache]; 7247 // Note: Don't use invoke->GetTargetMethod() as it may point to a different dex file. 7248 uint32_t index_in_cache = invoke->GetDexMethodIndex(); 7249 __ LoadFromOffset(kLoadWord, reg, reg, CodeGenerator::GetCachePointerOffset(index_in_cache)); 7250 break; 7251 } 7252 } 7253 7254 switch (invoke->GetCodePtrLocation()) { 7255 case HInvokeStaticOrDirect::CodePtrLocation::kCallSelf: 7256 __ bl(GetFrameEntryLabel()); 7257 break; 7258 case HInvokeStaticOrDirect::CodePtrLocation::kCallPCRelative: 7259 relative_call_patches_.emplace_back(*invoke->GetTargetMethod().dex_file, 7260 invoke->GetTargetMethod().dex_method_index); 7261 __ BindTrackedLabel(&relative_call_patches_.back().label); 7262 // Arbitrarily branch to the BL itself, override at link time. 7263 __ bl(&relative_call_patches_.back().label); 7264 break; 7265 case HInvokeStaticOrDirect::CodePtrLocation::kCallDirectWithFixup: 7266 case HInvokeStaticOrDirect::CodePtrLocation::kCallDirect: 7267 // LR prepared above for better instruction scheduling. 7268 // LR() 7269 __ blx(LR); 7270 break; 7271 case HInvokeStaticOrDirect::CodePtrLocation::kCallArtMethod: 7272 // LR = callee_method->entry_point_from_quick_compiled_code_ 7273 __ LoadFromOffset( 7274 kLoadWord, LR, callee_method.AsRegister<Register>(), 7275 ArtMethod::EntryPointFromQuickCompiledCodeOffset(kArmPointerSize).Int32Value()); 7276 // LR() 7277 __ blx(LR); 7278 break; 7279 } 7280 7281 DCHECK(!IsLeafMethod()); 7282} 7283 7284void CodeGeneratorARM::GenerateVirtualCall(HInvokeVirtual* invoke, Location temp_location) { 7285 Register temp = temp_location.AsRegister<Register>(); 7286 uint32_t method_offset = mirror::Class::EmbeddedVTableEntryOffset( 7287 invoke->GetVTableIndex(), kArmPointerSize).Uint32Value(); 7288 7289 // Use the calling convention instead of the location of the receiver, as 7290 // intrinsics may have put the receiver in a different register. In the intrinsics 7291 // slow path, the arguments have been moved to the right place, so here we are 7292 // guaranteed that the receiver is the first register of the calling convention. 7293 InvokeDexCallingConvention calling_convention; 7294 Register receiver = calling_convention.GetRegisterAt(0); 7295 uint32_t class_offset = mirror::Object::ClassOffset().Int32Value(); 7296 // /* HeapReference<Class> */ temp = receiver->klass_ 7297 __ LoadFromOffset(kLoadWord, temp, receiver, class_offset); 7298 MaybeRecordImplicitNullCheck(invoke); 7299 // Instead of simply (possibly) unpoisoning `temp` here, we should 7300 // emit a read barrier for the previous class reference load. 7301 // However this is not required in practice, as this is an 7302 // intermediate/temporary reference and because the current 7303 // concurrent copying collector keeps the from-space memory 7304 // intact/accessible until the end of the marking phase (the 7305 // concurrent copying collector may not in the future). 7306 __ MaybeUnpoisonHeapReference(temp); 7307 // temp = temp->GetMethodAt(method_offset); 7308 uint32_t entry_point = ArtMethod::EntryPointFromQuickCompiledCodeOffset( 7309 kArmPointerSize).Int32Value(); 7310 __ LoadFromOffset(kLoadWord, temp, temp, method_offset); 7311 // LR = temp->GetEntryPoint(); 7312 __ LoadFromOffset(kLoadWord, LR, temp, entry_point); 7313 // LR(); 7314 __ blx(LR); 7315} 7316 7317CodeGeneratorARM::PcRelativePatchInfo* CodeGeneratorARM::NewPcRelativeStringPatch( 7318 const DexFile& dex_file, uint32_t string_index) { 7319 return NewPcRelativePatch(dex_file, string_index, &pc_relative_string_patches_); 7320} 7321 7322CodeGeneratorARM::PcRelativePatchInfo* CodeGeneratorARM::NewPcRelativeTypePatch( 7323 const DexFile& dex_file, uint32_t type_index) { 7324 return NewPcRelativePatch(dex_file, type_index, &pc_relative_type_patches_); 7325} 7326 7327CodeGeneratorARM::PcRelativePatchInfo* CodeGeneratorARM::NewPcRelativeDexCacheArrayPatch( 7328 const DexFile& dex_file, uint32_t element_offset) { 7329 return NewPcRelativePatch(dex_file, element_offset, &pc_relative_dex_cache_patches_); 7330} 7331 7332CodeGeneratorARM::PcRelativePatchInfo* CodeGeneratorARM::NewPcRelativePatch( 7333 const DexFile& dex_file, uint32_t offset_or_index, ArenaDeque<PcRelativePatchInfo>* patches) { 7334 patches->emplace_back(dex_file, offset_or_index); 7335 return &patches->back(); 7336} 7337 7338Literal* CodeGeneratorARM::DeduplicateBootImageStringLiteral(const DexFile& dex_file, 7339 uint32_t string_index) { 7340 return boot_image_string_patches_.GetOrCreate( 7341 StringReference(&dex_file, string_index), 7342 [this]() { return __ NewLiteral<uint32_t>(/* placeholder */ 0u); }); 7343} 7344 7345Literal* CodeGeneratorARM::DeduplicateBootImageTypeLiteral(const DexFile& dex_file, 7346 uint32_t type_index) { 7347 return boot_image_type_patches_.GetOrCreate( 7348 TypeReference(&dex_file, type_index), 7349 [this]() { return __ NewLiteral<uint32_t>(/* placeholder */ 0u); }); 7350} 7351 7352Literal* CodeGeneratorARM::DeduplicateBootImageAddressLiteral(uint32_t address) { 7353 bool needs_patch = GetCompilerOptions().GetIncludePatchInformation(); 7354 Uint32ToLiteralMap* map = needs_patch ? &boot_image_address_patches_ : &uint32_literals_; 7355 return DeduplicateUint32Literal(dchecked_integral_cast<uint32_t>(address), map); 7356} 7357 7358Literal* CodeGeneratorARM::DeduplicateDexCacheAddressLiteral(uint32_t address) { 7359 return DeduplicateUint32Literal(address, &uint32_literals_); 7360} 7361 7362template <LinkerPatch (*Factory)(size_t, const DexFile*, uint32_t, uint32_t)> 7363inline void CodeGeneratorARM::EmitPcRelativeLinkerPatches( 7364 const ArenaDeque<PcRelativePatchInfo>& infos, 7365 ArenaVector<LinkerPatch>* linker_patches) { 7366 for (const PcRelativePatchInfo& info : infos) { 7367 const DexFile& dex_file = info.target_dex_file; 7368 size_t offset_or_index = info.offset_or_index; 7369 DCHECK(info.add_pc_label.IsBound()); 7370 uint32_t add_pc_offset = dchecked_integral_cast<uint32_t>(info.add_pc_label.Position()); 7371 // Add MOVW patch. 7372 DCHECK(info.movw_label.IsBound()); 7373 uint32_t movw_offset = dchecked_integral_cast<uint32_t>(info.movw_label.Position()); 7374 linker_patches->push_back(Factory(movw_offset, &dex_file, add_pc_offset, offset_or_index)); 7375 // Add MOVT patch. 7376 DCHECK(info.movt_label.IsBound()); 7377 uint32_t movt_offset = dchecked_integral_cast<uint32_t>(info.movt_label.Position()); 7378 linker_patches->push_back(Factory(movt_offset, &dex_file, add_pc_offset, offset_or_index)); 7379 } 7380} 7381 7382void CodeGeneratorARM::EmitLinkerPatches(ArenaVector<LinkerPatch>* linker_patches) { 7383 DCHECK(linker_patches->empty()); 7384 size_t size = 7385 method_patches_.size() + 7386 call_patches_.size() + 7387 relative_call_patches_.size() + 7388 /* MOVW+MOVT for each entry */ 2u * pc_relative_dex_cache_patches_.size() + 7389 boot_image_string_patches_.size() + 7390 /* MOVW+MOVT for each entry */ 2u * pc_relative_string_patches_.size() + 7391 boot_image_type_patches_.size() + 7392 /* MOVW+MOVT for each entry */ 2u * pc_relative_type_patches_.size() + 7393 boot_image_address_patches_.size(); 7394 linker_patches->reserve(size); 7395 for (const auto& entry : method_patches_) { 7396 const MethodReference& target_method = entry.first; 7397 Literal* literal = entry.second; 7398 DCHECK(literal->GetLabel()->IsBound()); 7399 uint32_t literal_offset = literal->GetLabel()->Position(); 7400 linker_patches->push_back(LinkerPatch::MethodPatch(literal_offset, 7401 target_method.dex_file, 7402 target_method.dex_method_index)); 7403 } 7404 for (const auto& entry : call_patches_) { 7405 const MethodReference& target_method = entry.first; 7406 Literal* literal = entry.second; 7407 DCHECK(literal->GetLabel()->IsBound()); 7408 uint32_t literal_offset = literal->GetLabel()->Position(); 7409 linker_patches->push_back(LinkerPatch::CodePatch(literal_offset, 7410 target_method.dex_file, 7411 target_method.dex_method_index)); 7412 } 7413 for (const PatchInfo<Label>& info : relative_call_patches_) { 7414 uint32_t literal_offset = info.label.Position(); 7415 linker_patches->push_back( 7416 LinkerPatch::RelativeCodePatch(literal_offset, &info.dex_file, info.index)); 7417 } 7418 EmitPcRelativeLinkerPatches<LinkerPatch::DexCacheArrayPatch>(pc_relative_dex_cache_patches_, 7419 linker_patches); 7420 for (const auto& entry : boot_image_string_patches_) { 7421 const StringReference& target_string = entry.first; 7422 Literal* literal = entry.second; 7423 DCHECK(literal->GetLabel()->IsBound()); 7424 uint32_t literal_offset = literal->GetLabel()->Position(); 7425 linker_patches->push_back(LinkerPatch::StringPatch(literal_offset, 7426 target_string.dex_file, 7427 target_string.string_index)); 7428 } 7429 if (!GetCompilerOptions().IsBootImage()) { 7430 EmitPcRelativeLinkerPatches<LinkerPatch::StringBssEntryPatch>(pc_relative_string_patches_, 7431 linker_patches); 7432 } else { 7433 EmitPcRelativeLinkerPatches<LinkerPatch::RelativeStringPatch>(pc_relative_string_patches_, 7434 linker_patches); 7435 } 7436 for (const auto& entry : boot_image_type_patches_) { 7437 const TypeReference& target_type = entry.first; 7438 Literal* literal = entry.second; 7439 DCHECK(literal->GetLabel()->IsBound()); 7440 uint32_t literal_offset = literal->GetLabel()->Position(); 7441 linker_patches->push_back(LinkerPatch::TypePatch(literal_offset, 7442 target_type.dex_file, 7443 target_type.type_index)); 7444 } 7445 EmitPcRelativeLinkerPatches<LinkerPatch::RelativeTypePatch>(pc_relative_type_patches_, 7446 linker_patches); 7447 for (const auto& entry : boot_image_address_patches_) { 7448 DCHECK(GetCompilerOptions().GetIncludePatchInformation()); 7449 Literal* literal = entry.second; 7450 DCHECK(literal->GetLabel()->IsBound()); 7451 uint32_t literal_offset = literal->GetLabel()->Position(); 7452 linker_patches->push_back(LinkerPatch::RecordPosition(literal_offset)); 7453 } 7454} 7455 7456Literal* CodeGeneratorARM::DeduplicateUint32Literal(uint32_t value, Uint32ToLiteralMap* map) { 7457 return map->GetOrCreate( 7458 value, 7459 [this, value]() { return __ NewLiteral<uint32_t>(value); }); 7460} 7461 7462Literal* CodeGeneratorARM::DeduplicateMethodLiteral(MethodReference target_method, 7463 MethodToLiteralMap* map) { 7464 return map->GetOrCreate( 7465 target_method, 7466 [this]() { return __ NewLiteral<uint32_t>(/* placeholder */ 0u); }); 7467} 7468 7469Literal* CodeGeneratorARM::DeduplicateMethodAddressLiteral(MethodReference target_method) { 7470 return DeduplicateMethodLiteral(target_method, &method_patches_); 7471} 7472 7473Literal* CodeGeneratorARM::DeduplicateMethodCodeLiteral(MethodReference target_method) { 7474 return DeduplicateMethodLiteral(target_method, &call_patches_); 7475} 7476 7477void LocationsBuilderARM::VisitMultiplyAccumulate(HMultiplyAccumulate* instr) { 7478 LocationSummary* locations = 7479 new (GetGraph()->GetArena()) LocationSummary(instr, LocationSummary::kNoCall); 7480 locations->SetInAt(HMultiplyAccumulate::kInputAccumulatorIndex, 7481 Location::RequiresRegister()); 7482 locations->SetInAt(HMultiplyAccumulate::kInputMulLeftIndex, Location::RequiresRegister()); 7483 locations->SetInAt(HMultiplyAccumulate::kInputMulRightIndex, Location::RequiresRegister()); 7484 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 7485} 7486 7487void InstructionCodeGeneratorARM::VisitMultiplyAccumulate(HMultiplyAccumulate* instr) { 7488 LocationSummary* locations = instr->GetLocations(); 7489 Register res = locations->Out().AsRegister<Register>(); 7490 Register accumulator = 7491 locations->InAt(HMultiplyAccumulate::kInputAccumulatorIndex).AsRegister<Register>(); 7492 Register mul_left = 7493 locations->InAt(HMultiplyAccumulate::kInputMulLeftIndex).AsRegister<Register>(); 7494 Register mul_right = 7495 locations->InAt(HMultiplyAccumulate::kInputMulRightIndex).AsRegister<Register>(); 7496 7497 if (instr->GetOpKind() == HInstruction::kAdd) { 7498 __ mla(res, mul_left, mul_right, accumulator); 7499 } else { 7500 __ mls(res, mul_left, mul_right, accumulator); 7501 } 7502} 7503 7504void LocationsBuilderARM::VisitBoundType(HBoundType* instruction ATTRIBUTE_UNUSED) { 7505 // Nothing to do, this should be removed during prepare for register allocator. 7506 LOG(FATAL) << "Unreachable"; 7507} 7508 7509void InstructionCodeGeneratorARM::VisitBoundType(HBoundType* instruction ATTRIBUTE_UNUSED) { 7510 // Nothing to do, this should be removed during prepare for register allocator. 7511 LOG(FATAL) << "Unreachable"; 7512} 7513 7514// Simple implementation of packed switch - generate cascaded compare/jumps. 7515void LocationsBuilderARM::VisitPackedSwitch(HPackedSwitch* switch_instr) { 7516 LocationSummary* locations = 7517 new (GetGraph()->GetArena()) LocationSummary(switch_instr, LocationSummary::kNoCall); 7518 locations->SetInAt(0, Location::RequiresRegister()); 7519 if (switch_instr->GetNumEntries() > kPackedSwitchCompareJumpThreshold && 7520 codegen_->GetAssembler()->IsThumb()) { 7521 locations->AddTemp(Location::RequiresRegister()); // We need a temp for the table base. 7522 if (switch_instr->GetStartValue() != 0) { 7523 locations->AddTemp(Location::RequiresRegister()); // We need a temp for the bias. 7524 } 7525 } 7526} 7527 7528void InstructionCodeGeneratorARM::VisitPackedSwitch(HPackedSwitch* switch_instr) { 7529 int32_t lower_bound = switch_instr->GetStartValue(); 7530 uint32_t num_entries = switch_instr->GetNumEntries(); 7531 LocationSummary* locations = switch_instr->GetLocations(); 7532 Register value_reg = locations->InAt(0).AsRegister<Register>(); 7533 HBasicBlock* default_block = switch_instr->GetDefaultBlock(); 7534 7535 if (num_entries <= kPackedSwitchCompareJumpThreshold || !codegen_->GetAssembler()->IsThumb()) { 7536 // Create a series of compare/jumps. 7537 Register temp_reg = IP; 7538 // Note: It is fine for the below AddConstantSetFlags() using IP register to temporarily store 7539 // the immediate, because IP is used as the destination register. For the other 7540 // AddConstantSetFlags() and GenerateCompareWithImmediate(), the immediate values are constant, 7541 // and they can be encoded in the instruction without making use of IP register. 7542 __ AddConstantSetFlags(temp_reg, value_reg, -lower_bound); 7543 7544 const ArenaVector<HBasicBlock*>& successors = switch_instr->GetBlock()->GetSuccessors(); 7545 // Jump to successors[0] if value == lower_bound. 7546 __ b(codegen_->GetLabelOf(successors[0]), EQ); 7547 int32_t last_index = 0; 7548 for (; num_entries - last_index > 2; last_index += 2) { 7549 __ AddConstantSetFlags(temp_reg, temp_reg, -2); 7550 // Jump to successors[last_index + 1] if value < case_value[last_index + 2]. 7551 __ b(codegen_->GetLabelOf(successors[last_index + 1]), LO); 7552 // Jump to successors[last_index + 2] if value == case_value[last_index + 2]. 7553 __ b(codegen_->GetLabelOf(successors[last_index + 2]), EQ); 7554 } 7555 if (num_entries - last_index == 2) { 7556 // The last missing case_value. 7557 __ CmpConstant(temp_reg, 1); 7558 __ b(codegen_->GetLabelOf(successors[last_index + 1]), EQ); 7559 } 7560 7561 // And the default for any other value. 7562 if (!codegen_->GoesToNextBlock(switch_instr->GetBlock(), default_block)) { 7563 __ b(codegen_->GetLabelOf(default_block)); 7564 } 7565 } else { 7566 // Create a table lookup. 7567 Register temp_reg = locations->GetTemp(0).AsRegister<Register>(); 7568 7569 // Materialize a pointer to the switch table 7570 std::vector<Label*> labels(num_entries); 7571 const ArenaVector<HBasicBlock*>& successors = switch_instr->GetBlock()->GetSuccessors(); 7572 for (uint32_t i = 0; i < num_entries; i++) { 7573 labels[i] = codegen_->GetLabelOf(successors[i]); 7574 } 7575 JumpTable* table = __ CreateJumpTable(std::move(labels), temp_reg); 7576 7577 // Remove the bias. 7578 Register key_reg; 7579 if (lower_bound != 0) { 7580 key_reg = locations->GetTemp(1).AsRegister<Register>(); 7581 __ AddConstant(key_reg, value_reg, -lower_bound); 7582 } else { 7583 key_reg = value_reg; 7584 } 7585 7586 // Check whether the value is in the table, jump to default block if not. 7587 __ CmpConstant(key_reg, num_entries - 1); 7588 __ b(codegen_->GetLabelOf(default_block), Condition::HI); 7589 7590 // Load the displacement from the table. 7591 __ ldr(temp_reg, Address(temp_reg, key_reg, Shift::LSL, 2)); 7592 7593 // Dispatch is a direct add to the PC (for Thumb2). 7594 __ EmitJumpTableDispatch(table, temp_reg); 7595 } 7596} 7597 7598void LocationsBuilderARM::VisitArmDexCacheArraysBase(HArmDexCacheArraysBase* base) { 7599 LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(base); 7600 locations->SetOut(Location::RequiresRegister()); 7601} 7602 7603void InstructionCodeGeneratorARM::VisitArmDexCacheArraysBase(HArmDexCacheArraysBase* base) { 7604 Register base_reg = base->GetLocations()->Out().AsRegister<Register>(); 7605 CodeGeneratorARM::PcRelativePatchInfo* labels = 7606 codegen_->NewPcRelativeDexCacheArrayPatch(base->GetDexFile(), base->GetElementOffset()); 7607 __ BindTrackedLabel(&labels->movw_label); 7608 __ movw(base_reg, /* placeholder */ 0u); 7609 __ BindTrackedLabel(&labels->movt_label); 7610 __ movt(base_reg, /* placeholder */ 0u); 7611 __ BindTrackedLabel(&labels->add_pc_label); 7612 __ add(base_reg, base_reg, ShifterOperand(PC)); 7613} 7614 7615void CodeGeneratorARM::MoveFromReturnRegister(Location trg, Primitive::Type type) { 7616 if (!trg.IsValid()) { 7617 DCHECK_EQ(type, Primitive::kPrimVoid); 7618 return; 7619 } 7620 7621 DCHECK_NE(type, Primitive::kPrimVoid); 7622 7623 Location return_loc = InvokeDexCallingConventionVisitorARM().GetReturnLocation(type); 7624 if (return_loc.Equals(trg)) { 7625 return; 7626 } 7627 7628 // TODO: Consider pairs in the parallel move resolver, then this could be nicely merged 7629 // with the last branch. 7630 if (type == Primitive::kPrimLong) { 7631 HParallelMove parallel_move(GetGraph()->GetArena()); 7632 parallel_move.AddMove(return_loc.ToLow(), trg.ToLow(), Primitive::kPrimInt, nullptr); 7633 parallel_move.AddMove(return_loc.ToHigh(), trg.ToHigh(), Primitive::kPrimInt, nullptr); 7634 GetMoveResolver()->EmitNativeCode(¶llel_move); 7635 } else if (type == Primitive::kPrimDouble) { 7636 HParallelMove parallel_move(GetGraph()->GetArena()); 7637 parallel_move.AddMove(return_loc.ToLow(), trg.ToLow(), Primitive::kPrimFloat, nullptr); 7638 parallel_move.AddMove(return_loc.ToHigh(), trg.ToHigh(), Primitive::kPrimFloat, nullptr); 7639 GetMoveResolver()->EmitNativeCode(¶llel_move); 7640 } else { 7641 // Let the parallel move resolver take care of all of this. 7642 HParallelMove parallel_move(GetGraph()->GetArena()); 7643 parallel_move.AddMove(return_loc, trg, type, nullptr); 7644 GetMoveResolver()->EmitNativeCode(¶llel_move); 7645 } 7646} 7647 7648void LocationsBuilderARM::VisitClassTableGet(HClassTableGet* instruction) { 7649 LocationSummary* locations = 7650 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall); 7651 locations->SetInAt(0, Location::RequiresRegister()); 7652 locations->SetOut(Location::RequiresRegister()); 7653} 7654 7655void InstructionCodeGeneratorARM::VisitClassTableGet(HClassTableGet* instruction) { 7656 LocationSummary* locations = instruction->GetLocations(); 7657 if (instruction->GetTableKind() == HClassTableGet::TableKind::kVTable) { 7658 uint32_t method_offset = mirror::Class::EmbeddedVTableEntryOffset( 7659 instruction->GetIndex(), kArmPointerSize).SizeValue(); 7660 __ LoadFromOffset(kLoadWord, 7661 locations->Out().AsRegister<Register>(), 7662 locations->InAt(0).AsRegister<Register>(), 7663 method_offset); 7664 } else { 7665 uint32_t method_offset = static_cast<uint32_t>(ImTable::OffsetOfElement( 7666 instruction->GetIndex(), kArmPointerSize)); 7667 __ LoadFromOffset(kLoadWord, 7668 locations->Out().AsRegister<Register>(), 7669 locations->InAt(0).AsRegister<Register>(), 7670 mirror::Class::ImtPtrOffset(kArmPointerSize).Uint32Value()); 7671 __ LoadFromOffset(kLoadWord, 7672 locations->Out().AsRegister<Register>(), 7673 locations->Out().AsRegister<Register>(), 7674 method_offset); 7675 } 7676} 7677 7678#undef __ 7679#undef QUICK_ENTRY_POINT 7680 7681} // namespace arm 7682} // namespace art 7683