code_generator_arm.cc revision 3395fbc20bcd20948bec8958db91b304c17cacd8
1/* 2 * Copyright (C) 2014 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17#include "code_generator_arm.h" 18 19#include "arch/arm/instruction_set_features_arm.h" 20#include "art_method.h" 21#include "code_generator_utils.h" 22#include "compiled_method.h" 23#include "entrypoints/quick/quick_entrypoints.h" 24#include "gc/accounting/card_table.h" 25#include "intrinsics.h" 26#include "intrinsics_arm.h" 27#include "mirror/array-inl.h" 28#include "mirror/class-inl.h" 29#include "thread.h" 30#include "utils/arm/assembler_arm.h" 31#include "utils/arm/managed_register_arm.h" 32#include "utils/assembler.h" 33#include "utils/stack_checks.h" 34 35namespace art { 36 37template<class MirrorType> 38class GcRoot; 39 40namespace arm { 41 42static bool ExpectedPairLayout(Location location) { 43 // We expected this for both core and fpu register pairs. 44 return ((location.low() & 1) == 0) && (location.low() + 1 == location.high()); 45} 46 47static constexpr int kCurrentMethodStackOffset = 0; 48static constexpr Register kMethodRegisterArgument = R0; 49 50static constexpr Register kCoreAlwaysSpillRegister = R5; 51static constexpr Register kCoreCalleeSaves[] = 52 { R5, R6, R7, R8, R10, R11, LR }; 53static constexpr SRegister kFpuCalleeSaves[] = 54 { S16, S17, S18, S19, S20, S21, S22, S23, S24, S25, S26, S27, S28, S29, S30, S31 }; 55 56// D31 cannot be split into two S registers, and the register allocator only works on 57// S registers. Therefore there is no need to block it. 58static constexpr DRegister DTMP = D31; 59 60static constexpr uint32_t kPackedSwitchCompareJumpThreshold = 7; 61 62// NOLINT on __ macro to suppress wrong warning/fix (misc-macro-parentheses) from clang-tidy. 63#define __ down_cast<ArmAssembler*>(codegen->GetAssembler())-> // NOLINT 64#define QUICK_ENTRY_POINT(x) QUICK_ENTRYPOINT_OFFSET(kArmPointerSize, x).Int32Value() 65 66static constexpr int kRegListThreshold = 4; 67 68// SaveLiveRegisters and RestoreLiveRegisters from SlowPathCodeARM operate on sets of S registers, 69// for each live D registers they treat two corresponding S registers as live ones. 70// 71// Two following functions (SaveContiguousSRegisterList, RestoreContiguousSRegisterList) build 72// from a list of contiguous S registers a list of contiguous D registers (processing first/last 73// S registers corner cases) and save/restore this new list treating them as D registers. 74// - decreasing code size 75// - avoiding hazards on Cortex-A57, when a pair of S registers for an actual live D register is 76// restored and then used in regular non SlowPath code as D register. 77// 78// For the following example (v means the S register is live): 79// D names: | D0 | D1 | D2 | D4 | ... 80// S names: | S0 | S1 | S2 | S3 | S4 | S5 | S6 | S7 | ... 81// Live? | | v | v | v | v | v | v | | ... 82// 83// S1 and S6 will be saved/restored independently; D registers list (D1, D2) will be processed 84// as D registers. 85static size_t SaveContiguousSRegisterList(size_t first, 86 size_t last, 87 CodeGenerator* codegen, 88 size_t stack_offset) { 89 DCHECK_LE(first, last); 90 if ((first == last) && (first == 0)) { 91 stack_offset += codegen->SaveFloatingPointRegister(stack_offset, first); 92 return stack_offset; 93 } 94 if (first % 2 == 1) { 95 stack_offset += codegen->SaveFloatingPointRegister(stack_offset, first++); 96 } 97 98 bool save_last = false; 99 if (last % 2 == 0) { 100 save_last = true; 101 --last; 102 } 103 104 if (first < last) { 105 DRegister d_reg = static_cast<DRegister>(first / 2); 106 DCHECK_EQ((last - first + 1) % 2, 0u); 107 size_t number_of_d_regs = (last - first + 1) / 2; 108 109 if (number_of_d_regs == 1) { 110 __ StoreDToOffset(d_reg, SP, stack_offset); 111 } else if (number_of_d_regs > 1) { 112 __ add(IP, SP, ShifterOperand(stack_offset)); 113 __ vstmiad(IP, d_reg, number_of_d_regs); 114 } 115 stack_offset += number_of_d_regs * kArmWordSize * 2; 116 } 117 118 if (save_last) { 119 stack_offset += codegen->SaveFloatingPointRegister(stack_offset, last + 1); 120 } 121 122 return stack_offset; 123} 124 125static size_t RestoreContiguousSRegisterList(size_t first, 126 size_t last, 127 CodeGenerator* codegen, 128 size_t stack_offset) { 129 DCHECK_LE(first, last); 130 if ((first == last) && (first == 0)) { 131 stack_offset += codegen->RestoreFloatingPointRegister(stack_offset, first); 132 return stack_offset; 133 } 134 if (first % 2 == 1) { 135 stack_offset += codegen->RestoreFloatingPointRegister(stack_offset, first++); 136 } 137 138 bool restore_last = false; 139 if (last % 2 == 0) { 140 restore_last = true; 141 --last; 142 } 143 144 if (first < last) { 145 DRegister d_reg = static_cast<DRegister>(first / 2); 146 DCHECK_EQ((last - first + 1) % 2, 0u); 147 size_t number_of_d_regs = (last - first + 1) / 2; 148 if (number_of_d_regs == 1) { 149 __ LoadDFromOffset(d_reg, SP, stack_offset); 150 } else if (number_of_d_regs > 1) { 151 __ add(IP, SP, ShifterOperand(stack_offset)); 152 __ vldmiad(IP, d_reg, number_of_d_regs); 153 } 154 stack_offset += number_of_d_regs * kArmWordSize * 2; 155 } 156 157 if (restore_last) { 158 stack_offset += codegen->RestoreFloatingPointRegister(stack_offset, last + 1); 159 } 160 161 return stack_offset; 162} 163 164void SlowPathCodeARM::SaveLiveRegisters(CodeGenerator* codegen, LocationSummary* locations) { 165 size_t stack_offset = codegen->GetFirstRegisterSlotInSlowPath(); 166 size_t orig_offset = stack_offset; 167 168 const uint32_t core_spills = codegen->GetSlowPathSpills(locations, /* core_registers */ true); 169 for (uint32_t i : LowToHighBits(core_spills)) { 170 // If the register holds an object, update the stack mask. 171 if (locations->RegisterContainsObject(i)) { 172 locations->SetStackBit(stack_offset / kVRegSize); 173 } 174 DCHECK_LT(stack_offset, codegen->GetFrameSize() - codegen->FrameEntrySpillSize()); 175 DCHECK_LT(i, kMaximumNumberOfExpectedRegisters); 176 saved_core_stack_offsets_[i] = stack_offset; 177 stack_offset += kArmWordSize; 178 } 179 180 int reg_num = POPCOUNT(core_spills); 181 if (reg_num != 0) { 182 if (reg_num > kRegListThreshold) { 183 __ StoreList(RegList(core_spills), orig_offset); 184 } else { 185 stack_offset = orig_offset; 186 for (uint32_t i : LowToHighBits(core_spills)) { 187 stack_offset += codegen->SaveCoreRegister(stack_offset, i); 188 } 189 } 190 } 191 192 uint32_t fp_spills = codegen->GetSlowPathSpills(locations, /* core_registers */ false); 193 orig_offset = stack_offset; 194 for (uint32_t i : LowToHighBits(fp_spills)) { 195 DCHECK_LT(i, kMaximumNumberOfExpectedRegisters); 196 saved_fpu_stack_offsets_[i] = stack_offset; 197 stack_offset += kArmWordSize; 198 } 199 200 stack_offset = orig_offset; 201 while (fp_spills != 0u) { 202 uint32_t begin = CTZ(fp_spills); 203 uint32_t tmp = fp_spills + (1u << begin); 204 fp_spills &= tmp; // Clear the contiguous range of 1s. 205 uint32_t end = (tmp == 0u) ? 32u : CTZ(tmp); // CTZ(0) is undefined. 206 stack_offset = SaveContiguousSRegisterList(begin, end - 1, codegen, stack_offset); 207 } 208 DCHECK_LE(stack_offset, codegen->GetFrameSize() - codegen->FrameEntrySpillSize()); 209} 210 211void SlowPathCodeARM::RestoreLiveRegisters(CodeGenerator* codegen, LocationSummary* locations) { 212 size_t stack_offset = codegen->GetFirstRegisterSlotInSlowPath(); 213 size_t orig_offset = stack_offset; 214 215 const uint32_t core_spills = codegen->GetSlowPathSpills(locations, /* core_registers */ true); 216 for (uint32_t i : LowToHighBits(core_spills)) { 217 DCHECK_LT(stack_offset, codegen->GetFrameSize() - codegen->FrameEntrySpillSize()); 218 DCHECK_LT(i, kMaximumNumberOfExpectedRegisters); 219 stack_offset += kArmWordSize; 220 } 221 222 int reg_num = POPCOUNT(core_spills); 223 if (reg_num != 0) { 224 if (reg_num > kRegListThreshold) { 225 __ LoadList(RegList(core_spills), orig_offset); 226 } else { 227 stack_offset = orig_offset; 228 for (uint32_t i : LowToHighBits(core_spills)) { 229 stack_offset += codegen->RestoreCoreRegister(stack_offset, i); 230 } 231 } 232 } 233 234 uint32_t fp_spills = codegen->GetSlowPathSpills(locations, /* core_registers */ false); 235 while (fp_spills != 0u) { 236 uint32_t begin = CTZ(fp_spills); 237 uint32_t tmp = fp_spills + (1u << begin); 238 fp_spills &= tmp; // Clear the contiguous range of 1s. 239 uint32_t end = (tmp == 0u) ? 32u : CTZ(tmp); // CTZ(0) is undefined. 240 stack_offset = RestoreContiguousSRegisterList(begin, end - 1, codegen, stack_offset); 241 } 242 DCHECK_LE(stack_offset, codegen->GetFrameSize() - codegen->FrameEntrySpillSize()); 243} 244 245class NullCheckSlowPathARM : public SlowPathCodeARM { 246 public: 247 explicit NullCheckSlowPathARM(HNullCheck* instruction) : SlowPathCodeARM(instruction) {} 248 249 void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { 250 CodeGeneratorARM* arm_codegen = down_cast<CodeGeneratorARM*>(codegen); 251 __ Bind(GetEntryLabel()); 252 if (instruction_->CanThrowIntoCatchBlock()) { 253 // Live registers will be restored in the catch block if caught. 254 SaveLiveRegisters(codegen, instruction_->GetLocations()); 255 } 256 arm_codegen->InvokeRuntime(kQuickThrowNullPointer, 257 instruction_, 258 instruction_->GetDexPc(), 259 this); 260 CheckEntrypointTypes<kQuickThrowNullPointer, void, void>(); 261 } 262 263 bool IsFatal() const OVERRIDE { return true; } 264 265 const char* GetDescription() const OVERRIDE { return "NullCheckSlowPathARM"; } 266 267 private: 268 DISALLOW_COPY_AND_ASSIGN(NullCheckSlowPathARM); 269}; 270 271class DivZeroCheckSlowPathARM : public SlowPathCodeARM { 272 public: 273 explicit DivZeroCheckSlowPathARM(HDivZeroCheck* instruction) : SlowPathCodeARM(instruction) {} 274 275 void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { 276 CodeGeneratorARM* arm_codegen = down_cast<CodeGeneratorARM*>(codegen); 277 __ Bind(GetEntryLabel()); 278 arm_codegen->InvokeRuntime(kQuickThrowDivZero, instruction_, instruction_->GetDexPc(), this); 279 CheckEntrypointTypes<kQuickThrowDivZero, void, void>(); 280 } 281 282 bool IsFatal() const OVERRIDE { return true; } 283 284 const char* GetDescription() const OVERRIDE { return "DivZeroCheckSlowPathARM"; } 285 286 private: 287 DISALLOW_COPY_AND_ASSIGN(DivZeroCheckSlowPathARM); 288}; 289 290class SuspendCheckSlowPathARM : public SlowPathCodeARM { 291 public: 292 SuspendCheckSlowPathARM(HSuspendCheck* instruction, HBasicBlock* successor) 293 : SlowPathCodeARM(instruction), successor_(successor) {} 294 295 void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { 296 CodeGeneratorARM* arm_codegen = down_cast<CodeGeneratorARM*>(codegen); 297 __ Bind(GetEntryLabel()); 298 arm_codegen->InvokeRuntime(kQuickTestSuspend, instruction_, instruction_->GetDexPc(), this); 299 CheckEntrypointTypes<kQuickTestSuspend, void, void>(); 300 if (successor_ == nullptr) { 301 __ b(GetReturnLabel()); 302 } else { 303 __ b(arm_codegen->GetLabelOf(successor_)); 304 } 305 } 306 307 Label* GetReturnLabel() { 308 DCHECK(successor_ == nullptr); 309 return &return_label_; 310 } 311 312 HBasicBlock* GetSuccessor() const { 313 return successor_; 314 } 315 316 const char* GetDescription() const OVERRIDE { return "SuspendCheckSlowPathARM"; } 317 318 private: 319 // If not null, the block to branch to after the suspend check. 320 HBasicBlock* const successor_; 321 322 // If `successor_` is null, the label to branch to after the suspend check. 323 Label return_label_; 324 325 DISALLOW_COPY_AND_ASSIGN(SuspendCheckSlowPathARM); 326}; 327 328class BoundsCheckSlowPathARM : public SlowPathCodeARM { 329 public: 330 explicit BoundsCheckSlowPathARM(HBoundsCheck* instruction) 331 : SlowPathCodeARM(instruction) {} 332 333 void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { 334 CodeGeneratorARM* arm_codegen = down_cast<CodeGeneratorARM*>(codegen); 335 LocationSummary* locations = instruction_->GetLocations(); 336 337 __ Bind(GetEntryLabel()); 338 if (instruction_->CanThrowIntoCatchBlock()) { 339 // Live registers will be restored in the catch block if caught. 340 SaveLiveRegisters(codegen, instruction_->GetLocations()); 341 } 342 // We're moving two locations to locations that could overlap, so we need a parallel 343 // move resolver. 344 InvokeRuntimeCallingConvention calling_convention; 345 codegen->EmitParallelMoves( 346 locations->InAt(0), 347 Location::RegisterLocation(calling_convention.GetRegisterAt(0)), 348 Primitive::kPrimInt, 349 locations->InAt(1), 350 Location::RegisterLocation(calling_convention.GetRegisterAt(1)), 351 Primitive::kPrimInt); 352 QuickEntrypointEnum entrypoint = instruction_->AsBoundsCheck()->IsStringCharAt() 353 ? kQuickThrowStringBounds 354 : kQuickThrowArrayBounds; 355 arm_codegen->InvokeRuntime(entrypoint, instruction_, instruction_->GetDexPc(), this); 356 CheckEntrypointTypes<kQuickThrowStringBounds, void, int32_t, int32_t>(); 357 CheckEntrypointTypes<kQuickThrowArrayBounds, void, int32_t, int32_t>(); 358 } 359 360 bool IsFatal() const OVERRIDE { return true; } 361 362 const char* GetDescription() const OVERRIDE { return "BoundsCheckSlowPathARM"; } 363 364 private: 365 DISALLOW_COPY_AND_ASSIGN(BoundsCheckSlowPathARM); 366}; 367 368class LoadClassSlowPathARM : public SlowPathCodeARM { 369 public: 370 LoadClassSlowPathARM(HLoadClass* cls, 371 HInstruction* at, 372 uint32_t dex_pc, 373 bool do_clinit) 374 : SlowPathCodeARM(at), cls_(cls), at_(at), dex_pc_(dex_pc), do_clinit_(do_clinit) { 375 DCHECK(at->IsLoadClass() || at->IsClinitCheck()); 376 } 377 378 void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { 379 LocationSummary* locations = at_->GetLocations(); 380 381 CodeGeneratorARM* arm_codegen = down_cast<CodeGeneratorARM*>(codegen); 382 __ Bind(GetEntryLabel()); 383 SaveLiveRegisters(codegen, locations); 384 385 InvokeRuntimeCallingConvention calling_convention; 386 __ LoadImmediate(calling_convention.GetRegisterAt(0), cls_->GetTypeIndex()); 387 QuickEntrypointEnum entrypoint = do_clinit_ ? kQuickInitializeStaticStorage 388 : kQuickInitializeType; 389 arm_codegen->InvokeRuntime(entrypoint, at_, dex_pc_, this); 390 if (do_clinit_) { 391 CheckEntrypointTypes<kQuickInitializeStaticStorage, void*, uint32_t>(); 392 } else { 393 CheckEntrypointTypes<kQuickInitializeType, void*, uint32_t>(); 394 } 395 396 // Move the class to the desired location. 397 Location out = locations->Out(); 398 if (out.IsValid()) { 399 DCHECK(out.IsRegister() && !locations->GetLiveRegisters()->ContainsCoreRegister(out.reg())); 400 arm_codegen->Move32(locations->Out(), Location::RegisterLocation(R0)); 401 } 402 RestoreLiveRegisters(codegen, locations); 403 __ b(GetExitLabel()); 404 } 405 406 const char* GetDescription() const OVERRIDE { return "LoadClassSlowPathARM"; } 407 408 private: 409 // The class this slow path will load. 410 HLoadClass* const cls_; 411 412 // The instruction where this slow path is happening. 413 // (Might be the load class or an initialization check). 414 HInstruction* const at_; 415 416 // The dex PC of `at_`. 417 const uint32_t dex_pc_; 418 419 // Whether to initialize the class. 420 const bool do_clinit_; 421 422 DISALLOW_COPY_AND_ASSIGN(LoadClassSlowPathARM); 423}; 424 425class LoadStringSlowPathARM : public SlowPathCodeARM { 426 public: 427 explicit LoadStringSlowPathARM(HLoadString* instruction) : SlowPathCodeARM(instruction) {} 428 429 void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { 430 LocationSummary* locations = instruction_->GetLocations(); 431 DCHECK(!locations->GetLiveRegisters()->ContainsCoreRegister(locations->Out().reg())); 432 HLoadString* load = instruction_->AsLoadString(); 433 const uint32_t string_index = load->GetStringIndex(); 434 Register out = locations->Out().AsRegister<Register>(); 435 Register temp = locations->GetTemp(0).AsRegister<Register>(); 436 constexpr bool call_saves_everything_except_r0 = (!kUseReadBarrier || kUseBakerReadBarrier); 437 438 CodeGeneratorARM* arm_codegen = down_cast<CodeGeneratorARM*>(codegen); 439 __ Bind(GetEntryLabel()); 440 SaveLiveRegisters(codegen, locations); 441 442 InvokeRuntimeCallingConvention calling_convention; 443 // In the unlucky case that the `temp` is R0, we preserve the address in `out` across 444 // the kSaveEverything call (or use `out` for the address after non-kSaveEverything call). 445 bool temp_is_r0 = (temp == calling_convention.GetRegisterAt(0)); 446 Register entry_address = temp_is_r0 ? out : temp; 447 DCHECK_NE(entry_address, calling_convention.GetRegisterAt(0)); 448 if (call_saves_everything_except_r0 && temp_is_r0) { 449 __ mov(entry_address, ShifterOperand(temp)); 450 } 451 452 __ LoadImmediate(calling_convention.GetRegisterAt(0), string_index); 453 arm_codegen->InvokeRuntime(kQuickResolveString, instruction_, instruction_->GetDexPc(), this); 454 CheckEntrypointTypes<kQuickResolveString, void*, uint32_t>(); 455 456 // Store the resolved String to the .bss entry. 457 if (call_saves_everything_except_r0) { 458 // The string entry address was preserved in `entry_address` thanks to kSaveEverything. 459 __ str(R0, Address(entry_address)); 460 } else { 461 // For non-Baker read barrier, we need to re-calculate the address of the string entry. 462 CodeGeneratorARM::PcRelativePatchInfo* labels = 463 arm_codegen->NewPcRelativeStringPatch(load->GetDexFile(), string_index); 464 __ BindTrackedLabel(&labels->movw_label); 465 __ movw(entry_address, /* placeholder */ 0u); 466 __ BindTrackedLabel(&labels->movt_label); 467 __ movt(entry_address, /* placeholder */ 0u); 468 __ BindTrackedLabel(&labels->add_pc_label); 469 __ add(entry_address, entry_address, ShifterOperand(PC)); 470 __ str(R0, Address(entry_address)); 471 } 472 473 arm_codegen->Move32(locations->Out(), Location::RegisterLocation(R0)); 474 RestoreLiveRegisters(codegen, locations); 475 476 __ b(GetExitLabel()); 477 } 478 479 const char* GetDescription() const OVERRIDE { return "LoadStringSlowPathARM"; } 480 481 private: 482 DISALLOW_COPY_AND_ASSIGN(LoadStringSlowPathARM); 483}; 484 485class TypeCheckSlowPathARM : public SlowPathCodeARM { 486 public: 487 TypeCheckSlowPathARM(HInstruction* instruction, bool is_fatal) 488 : SlowPathCodeARM(instruction), is_fatal_(is_fatal) {} 489 490 void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { 491 LocationSummary* locations = instruction_->GetLocations(); 492 Location arg0, arg1; 493 if (instruction_->IsInstanceOf()) { 494 arg0 = locations->InAt(1); 495 arg1 = locations->Out(); 496 } else { 497 arg0 = locations->InAt(0); 498 arg1 = locations->InAt(1); 499 } 500 DCHECK(instruction_->IsCheckCast() 501 || !locations->GetLiveRegisters()->ContainsCoreRegister(locations->Out().reg())); 502 503 CodeGeneratorARM* arm_codegen = down_cast<CodeGeneratorARM*>(codegen); 504 __ Bind(GetEntryLabel()); 505 506 if (!is_fatal_) { 507 SaveLiveRegisters(codegen, locations); 508 } 509 510 // We're moving two locations to locations that could overlap, so we need a parallel 511 // move resolver. 512 InvokeRuntimeCallingConvention calling_convention; 513 codegen->EmitParallelMoves(arg0, 514 Location::RegisterLocation(calling_convention.GetRegisterAt(0)), 515 Primitive::kPrimNot, 516 arg1, 517 Location::RegisterLocation(calling_convention.GetRegisterAt(1)), 518 Primitive::kPrimNot); 519 if (instruction_->IsInstanceOf()) { 520 arm_codegen->InvokeRuntime(kQuickInstanceofNonTrivial, 521 instruction_, 522 instruction_->GetDexPc(), 523 this); 524 CheckEntrypointTypes<kQuickInstanceofNonTrivial, size_t, mirror::Class*, mirror::Class*>(); 525 arm_codegen->Move32(locations->Out(), Location::RegisterLocation(R0)); 526 } else { 527 DCHECK(instruction_->IsCheckCast()); 528 arm_codegen->InvokeRuntime(kQuickCheckInstanceOf, 529 instruction_, 530 instruction_->GetDexPc(), 531 this); 532 CheckEntrypointTypes<kQuickCheckInstanceOf, void, mirror::Object*, mirror::Class*>(); 533 } 534 535 if (!is_fatal_) { 536 RestoreLiveRegisters(codegen, locations); 537 __ b(GetExitLabel()); 538 } 539 } 540 541 const char* GetDescription() const OVERRIDE { return "TypeCheckSlowPathARM"; } 542 543 bool IsFatal() const OVERRIDE { return is_fatal_; } 544 545 private: 546 const bool is_fatal_; 547 548 DISALLOW_COPY_AND_ASSIGN(TypeCheckSlowPathARM); 549}; 550 551class DeoptimizationSlowPathARM : public SlowPathCodeARM { 552 public: 553 explicit DeoptimizationSlowPathARM(HDeoptimize* instruction) 554 : SlowPathCodeARM(instruction) {} 555 556 void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { 557 CodeGeneratorARM* arm_codegen = down_cast<CodeGeneratorARM*>(codegen); 558 __ Bind(GetEntryLabel()); 559 arm_codegen->InvokeRuntime(kQuickDeoptimize, instruction_, instruction_->GetDexPc(), this); 560 CheckEntrypointTypes<kQuickDeoptimize, void, void>(); 561 } 562 563 const char* GetDescription() const OVERRIDE { return "DeoptimizationSlowPathARM"; } 564 565 private: 566 DISALLOW_COPY_AND_ASSIGN(DeoptimizationSlowPathARM); 567}; 568 569class ArraySetSlowPathARM : public SlowPathCodeARM { 570 public: 571 explicit ArraySetSlowPathARM(HInstruction* instruction) : SlowPathCodeARM(instruction) {} 572 573 void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { 574 LocationSummary* locations = instruction_->GetLocations(); 575 __ Bind(GetEntryLabel()); 576 SaveLiveRegisters(codegen, locations); 577 578 InvokeRuntimeCallingConvention calling_convention; 579 HParallelMove parallel_move(codegen->GetGraph()->GetArena()); 580 parallel_move.AddMove( 581 locations->InAt(0), 582 Location::RegisterLocation(calling_convention.GetRegisterAt(0)), 583 Primitive::kPrimNot, 584 nullptr); 585 parallel_move.AddMove( 586 locations->InAt(1), 587 Location::RegisterLocation(calling_convention.GetRegisterAt(1)), 588 Primitive::kPrimInt, 589 nullptr); 590 parallel_move.AddMove( 591 locations->InAt(2), 592 Location::RegisterLocation(calling_convention.GetRegisterAt(2)), 593 Primitive::kPrimNot, 594 nullptr); 595 codegen->GetMoveResolver()->EmitNativeCode(¶llel_move); 596 597 CodeGeneratorARM* arm_codegen = down_cast<CodeGeneratorARM*>(codegen); 598 arm_codegen->InvokeRuntime(kQuickAputObject, instruction_, instruction_->GetDexPc(), this); 599 CheckEntrypointTypes<kQuickAputObject, void, mirror::Array*, int32_t, mirror::Object*>(); 600 RestoreLiveRegisters(codegen, locations); 601 __ b(GetExitLabel()); 602 } 603 604 const char* GetDescription() const OVERRIDE { return "ArraySetSlowPathARM"; } 605 606 private: 607 DISALLOW_COPY_AND_ASSIGN(ArraySetSlowPathARM); 608}; 609 610// Slow path marking an object reference `ref` during a read 611// barrier. The field `obj.field` in the object `obj` holding this 612// reference does not get updated by this slow path after marking (see 613// ReadBarrierMarkAndUpdateFieldSlowPathARM below for that). 614// 615// This means that after the execution of this slow path, `ref` will 616// always be up-to-date, but `obj.field` may not; i.e., after the 617// flip, `ref` will be a to-space reference, but `obj.field` will 618// probably still be a from-space reference (unless it gets updated by 619// another thread, or if another thread installed another object 620// reference (different from `ref`) in `obj.field`). 621class ReadBarrierMarkSlowPathARM : public SlowPathCodeARM { 622 public: 623 ReadBarrierMarkSlowPathARM(HInstruction* instruction, 624 Location ref, 625 Location entrypoint = Location::NoLocation()) 626 : SlowPathCodeARM(instruction), ref_(ref), entrypoint_(entrypoint) { 627 DCHECK(kEmitCompilerReadBarrier); 628 } 629 630 const char* GetDescription() const OVERRIDE { return "ReadBarrierMarkSlowPathARM"; } 631 632 void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { 633 LocationSummary* locations = instruction_->GetLocations(); 634 Register ref_reg = ref_.AsRegister<Register>(); 635 DCHECK(locations->CanCall()); 636 DCHECK(!locations->GetLiveRegisters()->ContainsCoreRegister(ref_reg)) << ref_reg; 637 DCHECK(instruction_->IsInstanceFieldGet() || 638 instruction_->IsStaticFieldGet() || 639 instruction_->IsArrayGet() || 640 instruction_->IsArraySet() || 641 instruction_->IsLoadClass() || 642 instruction_->IsLoadString() || 643 instruction_->IsInstanceOf() || 644 instruction_->IsCheckCast() || 645 (instruction_->IsInvokeVirtual() && instruction_->GetLocations()->Intrinsified()) || 646 (instruction_->IsInvokeStaticOrDirect() && instruction_->GetLocations()->Intrinsified())) 647 << "Unexpected instruction in read barrier marking slow path: " 648 << instruction_->DebugName(); 649 // The read barrier instrumentation of object ArrayGet 650 // instructions does not support the HIntermediateAddress 651 // instruction. 652 DCHECK(!(instruction_->IsArrayGet() && 653 instruction_->AsArrayGet()->GetArray()->IsIntermediateAddress())); 654 655 __ Bind(GetEntryLabel()); 656 // No need to save live registers; it's taken care of by the 657 // entrypoint. Also, there is no need to update the stack mask, 658 // as this runtime call will not trigger a garbage collection. 659 CodeGeneratorARM* arm_codegen = down_cast<CodeGeneratorARM*>(codegen); 660 DCHECK_NE(ref_reg, SP); 661 DCHECK_NE(ref_reg, LR); 662 DCHECK_NE(ref_reg, PC); 663 // IP is used internally by the ReadBarrierMarkRegX entry point 664 // as a temporary, it cannot be the entry point's input/output. 665 DCHECK_NE(ref_reg, IP); 666 DCHECK(0 <= ref_reg && ref_reg < kNumberOfCoreRegisters) << ref_reg; 667 // "Compact" slow path, saving two moves. 668 // 669 // Instead of using the standard runtime calling convention (input 670 // and output in R0): 671 // 672 // R0 <- ref 673 // R0 <- ReadBarrierMark(R0) 674 // ref <- R0 675 // 676 // we just use rX (the register containing `ref`) as input and output 677 // of a dedicated entrypoint: 678 // 679 // rX <- ReadBarrierMarkRegX(rX) 680 // 681 if (entrypoint_.IsValid()) { 682 arm_codegen->ValidateInvokeRuntimeWithoutRecordingPcInfo(instruction_, this); 683 __ blx(entrypoint_.AsRegister<Register>()); 684 } else { 685 int32_t entry_point_offset = 686 CodeGenerator::GetReadBarrierMarkEntryPointsOffset<kArmPointerSize>(ref_reg); 687 // This runtime call does not require a stack map. 688 arm_codegen->InvokeRuntimeWithoutRecordingPcInfo(entry_point_offset, instruction_, this); 689 } 690 __ b(GetExitLabel()); 691 } 692 693 private: 694 // The location (register) of the marked object reference. 695 const Location ref_; 696 697 // The location of the entrypoint if already loaded. 698 const Location entrypoint_; 699 700 DISALLOW_COPY_AND_ASSIGN(ReadBarrierMarkSlowPathARM); 701}; 702 703// Slow path marking an object reference `ref` during a read barrier, 704// and if needed, atomically updating the field `obj.field` in the 705// object `obj` holding this reference after marking (contrary to 706// ReadBarrierMarkSlowPathARM above, which never tries to update 707// `obj.field`). 708// 709// This means that after the execution of this slow path, both `ref` 710// and `obj.field` will be up-to-date; i.e., after the flip, both will 711// hold the same to-space reference (unless another thread installed 712// another object reference (different from `ref`) in `obj.field`). 713class ReadBarrierMarkAndUpdateFieldSlowPathARM : public SlowPathCodeARM { 714 public: 715 ReadBarrierMarkAndUpdateFieldSlowPathARM(HInstruction* instruction, 716 Location ref, 717 Register obj, 718 Location field_offset, 719 Register temp1, 720 Register temp2) 721 : SlowPathCodeARM(instruction), 722 ref_(ref), 723 obj_(obj), 724 field_offset_(field_offset), 725 temp1_(temp1), 726 temp2_(temp2) { 727 DCHECK(kEmitCompilerReadBarrier); 728 } 729 730 const char* GetDescription() const OVERRIDE { return "ReadBarrierMarkAndUpdateFieldSlowPathARM"; } 731 732 void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { 733 LocationSummary* locations = instruction_->GetLocations(); 734 Register ref_reg = ref_.AsRegister<Register>(); 735 DCHECK(locations->CanCall()); 736 DCHECK(!locations->GetLiveRegisters()->ContainsCoreRegister(ref_reg)) << ref_reg; 737 // This slow path is only used by the UnsafeCASObject intrinsic. 738 DCHECK((instruction_->IsInvokeVirtual() && instruction_->GetLocations()->Intrinsified())) 739 << "Unexpected instruction in read barrier marking and field updating slow path: " 740 << instruction_->DebugName(); 741 DCHECK(instruction_->GetLocations()->Intrinsified()); 742 DCHECK_EQ(instruction_->AsInvoke()->GetIntrinsic(), Intrinsics::kUnsafeCASObject); 743 DCHECK(field_offset_.IsRegisterPair()) << field_offset_; 744 745 __ Bind(GetEntryLabel()); 746 747 // Save the old reference. 748 // Note that we cannot use IP to save the old reference, as IP is 749 // used internally by the ReadBarrierMarkRegX entry point, and we 750 // need the old reference after the call to that entry point. 751 DCHECK_NE(temp1_, IP); 752 __ Mov(temp1_, ref_reg); 753 754 // No need to save live registers; it's taken care of by the 755 // entrypoint. Also, there is no need to update the stack mask, 756 // as this runtime call will not trigger a garbage collection. 757 CodeGeneratorARM* arm_codegen = down_cast<CodeGeneratorARM*>(codegen); 758 DCHECK_NE(ref_reg, SP); 759 DCHECK_NE(ref_reg, LR); 760 DCHECK_NE(ref_reg, PC); 761 // IP is used internally by the ReadBarrierMarkRegX entry point 762 // as a temporary, it cannot be the entry point's input/output. 763 DCHECK_NE(ref_reg, IP); 764 DCHECK(0 <= ref_reg && ref_reg < kNumberOfCoreRegisters) << ref_reg; 765 // "Compact" slow path, saving two moves. 766 // 767 // Instead of using the standard runtime calling convention (input 768 // and output in R0): 769 // 770 // R0 <- ref 771 // R0 <- ReadBarrierMark(R0) 772 // ref <- R0 773 // 774 // we just use rX (the register containing `ref`) as input and output 775 // of a dedicated entrypoint: 776 // 777 // rX <- ReadBarrierMarkRegX(rX) 778 // 779 int32_t entry_point_offset = 780 CodeGenerator::GetReadBarrierMarkEntryPointsOffset<kArmPointerSize>(ref_reg); 781 // This runtime call does not require a stack map. 782 arm_codegen->InvokeRuntimeWithoutRecordingPcInfo(entry_point_offset, instruction_, this); 783 784 // If the new reference is different from the old reference, 785 // update the field in the holder (`*(obj_ + field_offset_)`). 786 // 787 // Note that this field could also hold a different object, if 788 // another thread had concurrently changed it. In that case, the 789 // LDREX/SUBS/ITNE sequence of instructions in the compare-and-set 790 // (CAS) operation below would abort the CAS, leaving the field 791 // as-is. 792 Label done; 793 __ cmp(temp1_, ShifterOperand(ref_reg)); 794 __ b(&done, EQ); 795 796 // Update the the holder's field atomically. This may fail if 797 // mutator updates before us, but it's OK. This is achieved 798 // using a strong compare-and-set (CAS) operation with relaxed 799 // memory synchronization ordering, where the expected value is 800 // the old reference and the desired value is the new reference. 801 802 // Convenience aliases. 803 Register base = obj_; 804 // The UnsafeCASObject intrinsic uses a register pair as field 805 // offset ("long offset"), of which only the low part contains 806 // data. 807 Register offset = field_offset_.AsRegisterPairLow<Register>(); 808 Register expected = temp1_; 809 Register value = ref_reg; 810 Register tmp_ptr = IP; // Pointer to actual memory. 811 Register tmp = temp2_; // Value in memory. 812 813 __ add(tmp_ptr, base, ShifterOperand(offset)); 814 815 if (kPoisonHeapReferences) { 816 __ PoisonHeapReference(expected); 817 if (value == expected) { 818 // Do not poison `value`, as it is the same register as 819 // `expected`, which has just been poisoned. 820 } else { 821 __ PoisonHeapReference(value); 822 } 823 } 824 825 // do { 826 // tmp = [r_ptr] - expected; 827 // } while (tmp == 0 && failure([r_ptr] <- r_new_value)); 828 829 Label loop_head, exit_loop; 830 __ Bind(&loop_head); 831 832 __ ldrex(tmp, tmp_ptr); 833 834 __ subs(tmp, tmp, ShifterOperand(expected)); 835 836 __ it(NE); 837 __ clrex(NE); 838 839 __ b(&exit_loop, NE); 840 841 __ strex(tmp, value, tmp_ptr); 842 __ cmp(tmp, ShifterOperand(1)); 843 __ b(&loop_head, EQ); 844 845 __ Bind(&exit_loop); 846 847 if (kPoisonHeapReferences) { 848 __ UnpoisonHeapReference(expected); 849 if (value == expected) { 850 // Do not unpoison `value`, as it is the same register as 851 // `expected`, which has just been unpoisoned. 852 } else { 853 __ UnpoisonHeapReference(value); 854 } 855 } 856 857 __ Bind(&done); 858 __ b(GetExitLabel()); 859 } 860 861 private: 862 // The location (register) of the marked object reference. 863 const Location ref_; 864 // The register containing the object holding the marked object reference field. 865 const Register obj_; 866 // The location of the offset of the marked reference field within `obj_`. 867 Location field_offset_; 868 869 const Register temp1_; 870 const Register temp2_; 871 872 DISALLOW_COPY_AND_ASSIGN(ReadBarrierMarkAndUpdateFieldSlowPathARM); 873}; 874 875// Slow path generating a read barrier for a heap reference. 876class ReadBarrierForHeapReferenceSlowPathARM : public SlowPathCodeARM { 877 public: 878 ReadBarrierForHeapReferenceSlowPathARM(HInstruction* instruction, 879 Location out, 880 Location ref, 881 Location obj, 882 uint32_t offset, 883 Location index) 884 : SlowPathCodeARM(instruction), 885 out_(out), 886 ref_(ref), 887 obj_(obj), 888 offset_(offset), 889 index_(index) { 890 DCHECK(kEmitCompilerReadBarrier); 891 // If `obj` is equal to `out` or `ref`, it means the initial object 892 // has been overwritten by (or after) the heap object reference load 893 // to be instrumented, e.g.: 894 // 895 // __ LoadFromOffset(kLoadWord, out, out, offset); 896 // codegen_->GenerateReadBarrierSlow(instruction, out_loc, out_loc, out_loc, offset); 897 // 898 // In that case, we have lost the information about the original 899 // object, and the emitted read barrier cannot work properly. 900 DCHECK(!obj.Equals(out)) << "obj=" << obj << " out=" << out; 901 DCHECK(!obj.Equals(ref)) << "obj=" << obj << " ref=" << ref; 902 } 903 904 void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { 905 CodeGeneratorARM* arm_codegen = down_cast<CodeGeneratorARM*>(codegen); 906 LocationSummary* locations = instruction_->GetLocations(); 907 Register reg_out = out_.AsRegister<Register>(); 908 DCHECK(locations->CanCall()); 909 DCHECK(!locations->GetLiveRegisters()->ContainsCoreRegister(reg_out)); 910 DCHECK(instruction_->IsInstanceFieldGet() || 911 instruction_->IsStaticFieldGet() || 912 instruction_->IsArrayGet() || 913 instruction_->IsInstanceOf() || 914 instruction_->IsCheckCast() || 915 (instruction_->IsInvokeVirtual()) && instruction_->GetLocations()->Intrinsified()) 916 << "Unexpected instruction in read barrier for heap reference slow path: " 917 << instruction_->DebugName(); 918 // The read barrier instrumentation of object ArrayGet 919 // instructions does not support the HIntermediateAddress 920 // instruction. 921 DCHECK(!(instruction_->IsArrayGet() && 922 instruction_->AsArrayGet()->GetArray()->IsIntermediateAddress())); 923 924 __ Bind(GetEntryLabel()); 925 SaveLiveRegisters(codegen, locations); 926 927 // We may have to change the index's value, but as `index_` is a 928 // constant member (like other "inputs" of this slow path), 929 // introduce a copy of it, `index`. 930 Location index = index_; 931 if (index_.IsValid()) { 932 // Handle `index_` for HArrayGet and UnsafeGetObject/UnsafeGetObjectVolatile intrinsics. 933 if (instruction_->IsArrayGet()) { 934 // Compute the actual memory offset and store it in `index`. 935 Register index_reg = index_.AsRegister<Register>(); 936 DCHECK(locations->GetLiveRegisters()->ContainsCoreRegister(index_reg)); 937 if (codegen->IsCoreCalleeSaveRegister(index_reg)) { 938 // We are about to change the value of `index_reg` (see the 939 // calls to art::arm::Thumb2Assembler::Lsl and 940 // art::arm::Thumb2Assembler::AddConstant below), but it has 941 // not been saved by the previous call to 942 // art::SlowPathCode::SaveLiveRegisters, as it is a 943 // callee-save register -- 944 // art::SlowPathCode::SaveLiveRegisters does not consider 945 // callee-save registers, as it has been designed with the 946 // assumption that callee-save registers are supposed to be 947 // handled by the called function. So, as a callee-save 948 // register, `index_reg` _would_ eventually be saved onto 949 // the stack, but it would be too late: we would have 950 // changed its value earlier. Therefore, we manually save 951 // it here into another freely available register, 952 // `free_reg`, chosen of course among the caller-save 953 // registers (as a callee-save `free_reg` register would 954 // exhibit the same problem). 955 // 956 // Note we could have requested a temporary register from 957 // the register allocator instead; but we prefer not to, as 958 // this is a slow path, and we know we can find a 959 // caller-save register that is available. 960 Register free_reg = FindAvailableCallerSaveRegister(codegen); 961 __ Mov(free_reg, index_reg); 962 index_reg = free_reg; 963 index = Location::RegisterLocation(index_reg); 964 } else { 965 // The initial register stored in `index_` has already been 966 // saved in the call to art::SlowPathCode::SaveLiveRegisters 967 // (as it is not a callee-save register), so we can freely 968 // use it. 969 } 970 // Shifting the index value contained in `index_reg` by the scale 971 // factor (2) cannot overflow in practice, as the runtime is 972 // unable to allocate object arrays with a size larger than 973 // 2^26 - 1 (that is, 2^28 - 4 bytes). 974 __ Lsl(index_reg, index_reg, TIMES_4); 975 static_assert( 976 sizeof(mirror::HeapReference<mirror::Object>) == sizeof(int32_t), 977 "art::mirror::HeapReference<art::mirror::Object> and int32_t have different sizes."); 978 __ AddConstant(index_reg, index_reg, offset_); 979 } else { 980 // In the case of the UnsafeGetObject/UnsafeGetObjectVolatile 981 // intrinsics, `index_` is not shifted by a scale factor of 2 982 // (as in the case of ArrayGet), as it is actually an offset 983 // to an object field within an object. 984 DCHECK(instruction_->IsInvoke()) << instruction_->DebugName(); 985 DCHECK(instruction_->GetLocations()->Intrinsified()); 986 DCHECK((instruction_->AsInvoke()->GetIntrinsic() == Intrinsics::kUnsafeGetObject) || 987 (instruction_->AsInvoke()->GetIntrinsic() == Intrinsics::kUnsafeGetObjectVolatile)) 988 << instruction_->AsInvoke()->GetIntrinsic(); 989 DCHECK_EQ(offset_, 0U); 990 DCHECK(index_.IsRegisterPair()); 991 // UnsafeGet's offset location is a register pair, the low 992 // part contains the correct offset. 993 index = index_.ToLow(); 994 } 995 } 996 997 // We're moving two or three locations to locations that could 998 // overlap, so we need a parallel move resolver. 999 InvokeRuntimeCallingConvention calling_convention; 1000 HParallelMove parallel_move(codegen->GetGraph()->GetArena()); 1001 parallel_move.AddMove(ref_, 1002 Location::RegisterLocation(calling_convention.GetRegisterAt(0)), 1003 Primitive::kPrimNot, 1004 nullptr); 1005 parallel_move.AddMove(obj_, 1006 Location::RegisterLocation(calling_convention.GetRegisterAt(1)), 1007 Primitive::kPrimNot, 1008 nullptr); 1009 if (index.IsValid()) { 1010 parallel_move.AddMove(index, 1011 Location::RegisterLocation(calling_convention.GetRegisterAt(2)), 1012 Primitive::kPrimInt, 1013 nullptr); 1014 codegen->GetMoveResolver()->EmitNativeCode(¶llel_move); 1015 } else { 1016 codegen->GetMoveResolver()->EmitNativeCode(¶llel_move); 1017 __ LoadImmediate(calling_convention.GetRegisterAt(2), offset_); 1018 } 1019 arm_codegen->InvokeRuntime(kQuickReadBarrierSlow, instruction_, instruction_->GetDexPc(), this); 1020 CheckEntrypointTypes< 1021 kQuickReadBarrierSlow, mirror::Object*, mirror::Object*, mirror::Object*, uint32_t>(); 1022 arm_codegen->Move32(out_, Location::RegisterLocation(R0)); 1023 1024 RestoreLiveRegisters(codegen, locations); 1025 __ b(GetExitLabel()); 1026 } 1027 1028 const char* GetDescription() const OVERRIDE { return "ReadBarrierForHeapReferenceSlowPathARM"; } 1029 1030 private: 1031 Register FindAvailableCallerSaveRegister(CodeGenerator* codegen) { 1032 size_t ref = static_cast<int>(ref_.AsRegister<Register>()); 1033 size_t obj = static_cast<int>(obj_.AsRegister<Register>()); 1034 for (size_t i = 0, e = codegen->GetNumberOfCoreRegisters(); i < e; ++i) { 1035 if (i != ref && i != obj && !codegen->IsCoreCalleeSaveRegister(i)) { 1036 return static_cast<Register>(i); 1037 } 1038 } 1039 // We shall never fail to find a free caller-save register, as 1040 // there are more than two core caller-save registers on ARM 1041 // (meaning it is possible to find one which is different from 1042 // `ref` and `obj`). 1043 DCHECK_GT(codegen->GetNumberOfCoreCallerSaveRegisters(), 2u); 1044 LOG(FATAL) << "Could not find a free caller-save register"; 1045 UNREACHABLE(); 1046 } 1047 1048 const Location out_; 1049 const Location ref_; 1050 const Location obj_; 1051 const uint32_t offset_; 1052 // An additional location containing an index to an array. 1053 // Only used for HArrayGet and the UnsafeGetObject & 1054 // UnsafeGetObjectVolatile intrinsics. 1055 const Location index_; 1056 1057 DISALLOW_COPY_AND_ASSIGN(ReadBarrierForHeapReferenceSlowPathARM); 1058}; 1059 1060// Slow path generating a read barrier for a GC root. 1061class ReadBarrierForRootSlowPathARM : public SlowPathCodeARM { 1062 public: 1063 ReadBarrierForRootSlowPathARM(HInstruction* instruction, Location out, Location root) 1064 : SlowPathCodeARM(instruction), out_(out), root_(root) { 1065 DCHECK(kEmitCompilerReadBarrier); 1066 } 1067 1068 void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { 1069 LocationSummary* locations = instruction_->GetLocations(); 1070 Register reg_out = out_.AsRegister<Register>(); 1071 DCHECK(locations->CanCall()); 1072 DCHECK(!locations->GetLiveRegisters()->ContainsCoreRegister(reg_out)); 1073 DCHECK(instruction_->IsLoadClass() || instruction_->IsLoadString()) 1074 << "Unexpected instruction in read barrier for GC root slow path: " 1075 << instruction_->DebugName(); 1076 1077 __ Bind(GetEntryLabel()); 1078 SaveLiveRegisters(codegen, locations); 1079 1080 InvokeRuntimeCallingConvention calling_convention; 1081 CodeGeneratorARM* arm_codegen = down_cast<CodeGeneratorARM*>(codegen); 1082 arm_codegen->Move32(Location::RegisterLocation(calling_convention.GetRegisterAt(0)), root_); 1083 arm_codegen->InvokeRuntime(kQuickReadBarrierForRootSlow, 1084 instruction_, 1085 instruction_->GetDexPc(), 1086 this); 1087 CheckEntrypointTypes<kQuickReadBarrierForRootSlow, mirror::Object*, GcRoot<mirror::Object>*>(); 1088 arm_codegen->Move32(out_, Location::RegisterLocation(R0)); 1089 1090 RestoreLiveRegisters(codegen, locations); 1091 __ b(GetExitLabel()); 1092 } 1093 1094 const char* GetDescription() const OVERRIDE { return "ReadBarrierForRootSlowPathARM"; } 1095 1096 private: 1097 const Location out_; 1098 const Location root_; 1099 1100 DISALLOW_COPY_AND_ASSIGN(ReadBarrierForRootSlowPathARM); 1101}; 1102 1103#undef __ 1104// NOLINT on __ macro to suppress wrong warning/fix (misc-macro-parentheses) from clang-tidy. 1105#define __ down_cast<ArmAssembler*>(GetAssembler())-> // NOLINT 1106 1107inline Condition ARMCondition(IfCondition cond) { 1108 switch (cond) { 1109 case kCondEQ: return EQ; 1110 case kCondNE: return NE; 1111 case kCondLT: return LT; 1112 case kCondLE: return LE; 1113 case kCondGT: return GT; 1114 case kCondGE: return GE; 1115 case kCondB: return LO; 1116 case kCondBE: return LS; 1117 case kCondA: return HI; 1118 case kCondAE: return HS; 1119 } 1120 LOG(FATAL) << "Unreachable"; 1121 UNREACHABLE(); 1122} 1123 1124// Maps signed condition to unsigned condition. 1125inline Condition ARMUnsignedCondition(IfCondition cond) { 1126 switch (cond) { 1127 case kCondEQ: return EQ; 1128 case kCondNE: return NE; 1129 // Signed to unsigned. 1130 case kCondLT: return LO; 1131 case kCondLE: return LS; 1132 case kCondGT: return HI; 1133 case kCondGE: return HS; 1134 // Unsigned remain unchanged. 1135 case kCondB: return LO; 1136 case kCondBE: return LS; 1137 case kCondA: return HI; 1138 case kCondAE: return HS; 1139 } 1140 LOG(FATAL) << "Unreachable"; 1141 UNREACHABLE(); 1142} 1143 1144inline Condition ARMFPCondition(IfCondition cond, bool gt_bias) { 1145 // The ARM condition codes can express all the necessary branches, see the 1146 // "Meaning (floating-point)" column in the table A8-1 of the ARMv7 reference manual. 1147 // There is no dex instruction or HIR that would need the missing conditions 1148 // "equal or unordered" or "not equal". 1149 switch (cond) { 1150 case kCondEQ: return EQ; 1151 case kCondNE: return NE /* unordered */; 1152 case kCondLT: return gt_bias ? CC : LT /* unordered */; 1153 case kCondLE: return gt_bias ? LS : LE /* unordered */; 1154 case kCondGT: return gt_bias ? HI /* unordered */ : GT; 1155 case kCondGE: return gt_bias ? CS /* unordered */ : GE; 1156 default: 1157 LOG(FATAL) << "UNREACHABLE"; 1158 UNREACHABLE(); 1159 } 1160} 1161 1162void CodeGeneratorARM::DumpCoreRegister(std::ostream& stream, int reg) const { 1163 stream << Register(reg); 1164} 1165 1166void CodeGeneratorARM::DumpFloatingPointRegister(std::ostream& stream, int reg) const { 1167 stream << SRegister(reg); 1168} 1169 1170size_t CodeGeneratorARM::SaveCoreRegister(size_t stack_index, uint32_t reg_id) { 1171 __ StoreToOffset(kStoreWord, static_cast<Register>(reg_id), SP, stack_index); 1172 return kArmWordSize; 1173} 1174 1175size_t CodeGeneratorARM::RestoreCoreRegister(size_t stack_index, uint32_t reg_id) { 1176 __ LoadFromOffset(kLoadWord, static_cast<Register>(reg_id), SP, stack_index); 1177 return kArmWordSize; 1178} 1179 1180size_t CodeGeneratorARM::SaveFloatingPointRegister(size_t stack_index, uint32_t reg_id) { 1181 __ StoreSToOffset(static_cast<SRegister>(reg_id), SP, stack_index); 1182 return kArmWordSize; 1183} 1184 1185size_t CodeGeneratorARM::RestoreFloatingPointRegister(size_t stack_index, uint32_t reg_id) { 1186 __ LoadSFromOffset(static_cast<SRegister>(reg_id), SP, stack_index); 1187 return kArmWordSize; 1188} 1189 1190CodeGeneratorARM::CodeGeneratorARM(HGraph* graph, 1191 const ArmInstructionSetFeatures& isa_features, 1192 const CompilerOptions& compiler_options, 1193 OptimizingCompilerStats* stats) 1194 : CodeGenerator(graph, 1195 kNumberOfCoreRegisters, 1196 kNumberOfSRegisters, 1197 kNumberOfRegisterPairs, 1198 ComputeRegisterMask(reinterpret_cast<const int*>(kCoreCalleeSaves), 1199 arraysize(kCoreCalleeSaves)), 1200 ComputeRegisterMask(reinterpret_cast<const int*>(kFpuCalleeSaves), 1201 arraysize(kFpuCalleeSaves)), 1202 compiler_options, 1203 stats), 1204 block_labels_(nullptr), 1205 location_builder_(graph, this), 1206 instruction_visitor_(graph, this), 1207 move_resolver_(graph->GetArena(), this), 1208 assembler_(graph->GetArena()), 1209 isa_features_(isa_features), 1210 uint32_literals_(std::less<uint32_t>(), 1211 graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), 1212 method_patches_(MethodReferenceComparator(), 1213 graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), 1214 call_patches_(MethodReferenceComparator(), 1215 graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), 1216 relative_call_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), 1217 pc_relative_dex_cache_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), 1218 boot_image_string_patches_(StringReferenceValueComparator(), 1219 graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), 1220 pc_relative_string_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), 1221 boot_image_type_patches_(TypeReferenceValueComparator(), 1222 graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), 1223 pc_relative_type_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), 1224 boot_image_address_patches_(std::less<uint32_t>(), 1225 graph->GetArena()->Adapter(kArenaAllocCodeGenerator)) { 1226 // Always save the LR register to mimic Quick. 1227 AddAllocatedRegister(Location::RegisterLocation(LR)); 1228} 1229 1230void CodeGeneratorARM::Finalize(CodeAllocator* allocator) { 1231 // Ensure that we fix up branches and literal loads and emit the literal pool. 1232 __ FinalizeCode(); 1233 1234 // Adjust native pc offsets in stack maps. 1235 for (size_t i = 0, num = stack_map_stream_.GetNumberOfStackMaps(); i != num; ++i) { 1236 uint32_t old_position = stack_map_stream_.GetStackMap(i).native_pc_offset; 1237 uint32_t new_position = __ GetAdjustedPosition(old_position); 1238 stack_map_stream_.SetStackMapNativePcOffset(i, new_position); 1239 } 1240 // Adjust pc offsets for the disassembly information. 1241 if (disasm_info_ != nullptr) { 1242 GeneratedCodeInterval* frame_entry_interval = disasm_info_->GetFrameEntryInterval(); 1243 frame_entry_interval->start = __ GetAdjustedPosition(frame_entry_interval->start); 1244 frame_entry_interval->end = __ GetAdjustedPosition(frame_entry_interval->end); 1245 for (auto& it : *disasm_info_->GetInstructionIntervals()) { 1246 it.second.start = __ GetAdjustedPosition(it.second.start); 1247 it.second.end = __ GetAdjustedPosition(it.second.end); 1248 } 1249 for (auto& it : *disasm_info_->GetSlowPathIntervals()) { 1250 it.code_interval.start = __ GetAdjustedPosition(it.code_interval.start); 1251 it.code_interval.end = __ GetAdjustedPosition(it.code_interval.end); 1252 } 1253 } 1254 1255 CodeGenerator::Finalize(allocator); 1256} 1257 1258void CodeGeneratorARM::SetupBlockedRegisters() const { 1259 // Stack register, LR and PC are always reserved. 1260 blocked_core_registers_[SP] = true; 1261 blocked_core_registers_[LR] = true; 1262 blocked_core_registers_[PC] = true; 1263 1264 // Reserve thread register. 1265 blocked_core_registers_[TR] = true; 1266 1267 // Reserve temp register. 1268 blocked_core_registers_[IP] = true; 1269 1270 if (GetGraph()->IsDebuggable()) { 1271 // Stubs do not save callee-save floating point registers. If the graph 1272 // is debuggable, we need to deal with these registers differently. For 1273 // now, just block them. 1274 for (size_t i = 0; i < arraysize(kFpuCalleeSaves); ++i) { 1275 blocked_fpu_registers_[kFpuCalleeSaves[i]] = true; 1276 } 1277 } 1278} 1279 1280InstructionCodeGeneratorARM::InstructionCodeGeneratorARM(HGraph* graph, CodeGeneratorARM* codegen) 1281 : InstructionCodeGenerator(graph, codegen), 1282 assembler_(codegen->GetAssembler()), 1283 codegen_(codegen) {} 1284 1285void CodeGeneratorARM::ComputeSpillMask() { 1286 core_spill_mask_ = allocated_registers_.GetCoreRegisters() & core_callee_save_mask_; 1287 DCHECK_NE(core_spill_mask_, 0u) << "At least the return address register must be saved"; 1288 // There is no easy instruction to restore just the PC on thumb2. We spill and 1289 // restore another arbitrary register. 1290 core_spill_mask_ |= (1 << kCoreAlwaysSpillRegister); 1291 fpu_spill_mask_ = allocated_registers_.GetFloatingPointRegisters() & fpu_callee_save_mask_; 1292 // We use vpush and vpop for saving and restoring floating point registers, which take 1293 // a SRegister and the number of registers to save/restore after that SRegister. We 1294 // therefore update the `fpu_spill_mask_` to also contain those registers not allocated, 1295 // but in the range. 1296 if (fpu_spill_mask_ != 0) { 1297 uint32_t least_significant_bit = LeastSignificantBit(fpu_spill_mask_); 1298 uint32_t most_significant_bit = MostSignificantBit(fpu_spill_mask_); 1299 for (uint32_t i = least_significant_bit + 1 ; i < most_significant_bit; ++i) { 1300 fpu_spill_mask_ |= (1 << i); 1301 } 1302 } 1303} 1304 1305static dwarf::Reg DWARFReg(Register reg) { 1306 return dwarf::Reg::ArmCore(static_cast<int>(reg)); 1307} 1308 1309static dwarf::Reg DWARFReg(SRegister reg) { 1310 return dwarf::Reg::ArmFp(static_cast<int>(reg)); 1311} 1312 1313void CodeGeneratorARM::GenerateFrameEntry() { 1314 bool skip_overflow_check = 1315 IsLeafMethod() && !FrameNeedsStackCheck(GetFrameSize(), InstructionSet::kArm); 1316 DCHECK(GetCompilerOptions().GetImplicitStackOverflowChecks()); 1317 __ Bind(&frame_entry_label_); 1318 1319 if (HasEmptyFrame()) { 1320 return; 1321 } 1322 1323 if (!skip_overflow_check) { 1324 __ AddConstant(IP, SP, -static_cast<int32_t>(GetStackOverflowReservedBytes(kArm))); 1325 __ LoadFromOffset(kLoadWord, IP, IP, 0); 1326 RecordPcInfo(nullptr, 0); 1327 } 1328 1329 __ PushList(core_spill_mask_); 1330 __ cfi().AdjustCFAOffset(kArmWordSize * POPCOUNT(core_spill_mask_)); 1331 __ cfi().RelOffsetForMany(DWARFReg(kMethodRegisterArgument), 0, core_spill_mask_, kArmWordSize); 1332 if (fpu_spill_mask_ != 0) { 1333 SRegister start_register = SRegister(LeastSignificantBit(fpu_spill_mask_)); 1334 __ vpushs(start_register, POPCOUNT(fpu_spill_mask_)); 1335 __ cfi().AdjustCFAOffset(kArmWordSize * POPCOUNT(fpu_spill_mask_)); 1336 __ cfi().RelOffsetForMany(DWARFReg(S0), 0, fpu_spill_mask_, kArmWordSize); 1337 } 1338 int adjust = GetFrameSize() - FrameEntrySpillSize(); 1339 __ AddConstant(SP, -adjust); 1340 __ cfi().AdjustCFAOffset(adjust); 1341 1342 // Save the current method if we need it. Note that we do not 1343 // do this in HCurrentMethod, as the instruction might have been removed 1344 // in the SSA graph. 1345 if (RequiresCurrentMethod()) { 1346 __ StoreToOffset(kStoreWord, kMethodRegisterArgument, SP, 0); 1347 } 1348} 1349 1350void CodeGeneratorARM::GenerateFrameExit() { 1351 if (HasEmptyFrame()) { 1352 __ bx(LR); 1353 return; 1354 } 1355 __ cfi().RememberState(); 1356 int adjust = GetFrameSize() - FrameEntrySpillSize(); 1357 __ AddConstant(SP, adjust); 1358 __ cfi().AdjustCFAOffset(-adjust); 1359 if (fpu_spill_mask_ != 0) { 1360 SRegister start_register = SRegister(LeastSignificantBit(fpu_spill_mask_)); 1361 __ vpops(start_register, POPCOUNT(fpu_spill_mask_)); 1362 __ cfi().AdjustCFAOffset(-static_cast<int>(kArmPointerSize) * POPCOUNT(fpu_spill_mask_)); 1363 __ cfi().RestoreMany(DWARFReg(SRegister(0)), fpu_spill_mask_); 1364 } 1365 // Pop LR into PC to return. 1366 DCHECK_NE(core_spill_mask_ & (1 << LR), 0U); 1367 uint32_t pop_mask = (core_spill_mask_ & (~(1 << LR))) | 1 << PC; 1368 __ PopList(pop_mask); 1369 __ cfi().RestoreState(); 1370 __ cfi().DefCFAOffset(GetFrameSize()); 1371} 1372 1373void CodeGeneratorARM::Bind(HBasicBlock* block) { 1374 Label* label = GetLabelOf(block); 1375 __ BindTrackedLabel(label); 1376} 1377 1378Location InvokeDexCallingConventionVisitorARM::GetNextLocation(Primitive::Type type) { 1379 switch (type) { 1380 case Primitive::kPrimBoolean: 1381 case Primitive::kPrimByte: 1382 case Primitive::kPrimChar: 1383 case Primitive::kPrimShort: 1384 case Primitive::kPrimInt: 1385 case Primitive::kPrimNot: { 1386 uint32_t index = gp_index_++; 1387 uint32_t stack_index = stack_index_++; 1388 if (index < calling_convention.GetNumberOfRegisters()) { 1389 return Location::RegisterLocation(calling_convention.GetRegisterAt(index)); 1390 } else { 1391 return Location::StackSlot(calling_convention.GetStackOffsetOf(stack_index)); 1392 } 1393 } 1394 1395 case Primitive::kPrimLong: { 1396 uint32_t index = gp_index_; 1397 uint32_t stack_index = stack_index_; 1398 gp_index_ += 2; 1399 stack_index_ += 2; 1400 if (index + 1 < calling_convention.GetNumberOfRegisters()) { 1401 if (calling_convention.GetRegisterAt(index) == R1) { 1402 // Skip R1, and use R2_R3 instead. 1403 gp_index_++; 1404 index++; 1405 } 1406 } 1407 if (index + 1 < calling_convention.GetNumberOfRegisters()) { 1408 DCHECK_EQ(calling_convention.GetRegisterAt(index) + 1, 1409 calling_convention.GetRegisterAt(index + 1)); 1410 1411 return Location::RegisterPairLocation(calling_convention.GetRegisterAt(index), 1412 calling_convention.GetRegisterAt(index + 1)); 1413 } else { 1414 return Location::DoubleStackSlot(calling_convention.GetStackOffsetOf(stack_index)); 1415 } 1416 } 1417 1418 case Primitive::kPrimFloat: { 1419 uint32_t stack_index = stack_index_++; 1420 if (float_index_ % 2 == 0) { 1421 float_index_ = std::max(double_index_, float_index_); 1422 } 1423 if (float_index_ < calling_convention.GetNumberOfFpuRegisters()) { 1424 return Location::FpuRegisterLocation(calling_convention.GetFpuRegisterAt(float_index_++)); 1425 } else { 1426 return Location::StackSlot(calling_convention.GetStackOffsetOf(stack_index)); 1427 } 1428 } 1429 1430 case Primitive::kPrimDouble: { 1431 double_index_ = std::max(double_index_, RoundUp(float_index_, 2)); 1432 uint32_t stack_index = stack_index_; 1433 stack_index_ += 2; 1434 if (double_index_ + 1 < calling_convention.GetNumberOfFpuRegisters()) { 1435 uint32_t index = double_index_; 1436 double_index_ += 2; 1437 Location result = Location::FpuRegisterPairLocation( 1438 calling_convention.GetFpuRegisterAt(index), 1439 calling_convention.GetFpuRegisterAt(index + 1)); 1440 DCHECK(ExpectedPairLayout(result)); 1441 return result; 1442 } else { 1443 return Location::DoubleStackSlot(calling_convention.GetStackOffsetOf(stack_index)); 1444 } 1445 } 1446 1447 case Primitive::kPrimVoid: 1448 LOG(FATAL) << "Unexpected parameter type " << type; 1449 break; 1450 } 1451 return Location::NoLocation(); 1452} 1453 1454Location InvokeDexCallingConventionVisitorARM::GetReturnLocation(Primitive::Type type) const { 1455 switch (type) { 1456 case Primitive::kPrimBoolean: 1457 case Primitive::kPrimByte: 1458 case Primitive::kPrimChar: 1459 case Primitive::kPrimShort: 1460 case Primitive::kPrimInt: 1461 case Primitive::kPrimNot: { 1462 return Location::RegisterLocation(R0); 1463 } 1464 1465 case Primitive::kPrimFloat: { 1466 return Location::FpuRegisterLocation(S0); 1467 } 1468 1469 case Primitive::kPrimLong: { 1470 return Location::RegisterPairLocation(R0, R1); 1471 } 1472 1473 case Primitive::kPrimDouble: { 1474 return Location::FpuRegisterPairLocation(S0, S1); 1475 } 1476 1477 case Primitive::kPrimVoid: 1478 return Location::NoLocation(); 1479 } 1480 1481 UNREACHABLE(); 1482} 1483 1484Location InvokeDexCallingConventionVisitorARM::GetMethodLocation() const { 1485 return Location::RegisterLocation(kMethodRegisterArgument); 1486} 1487 1488void CodeGeneratorARM::Move32(Location destination, Location source) { 1489 if (source.Equals(destination)) { 1490 return; 1491 } 1492 if (destination.IsRegister()) { 1493 if (source.IsRegister()) { 1494 __ Mov(destination.AsRegister<Register>(), source.AsRegister<Register>()); 1495 } else if (source.IsFpuRegister()) { 1496 __ vmovrs(destination.AsRegister<Register>(), source.AsFpuRegister<SRegister>()); 1497 } else { 1498 __ LoadFromOffset(kLoadWord, destination.AsRegister<Register>(), SP, source.GetStackIndex()); 1499 } 1500 } else if (destination.IsFpuRegister()) { 1501 if (source.IsRegister()) { 1502 __ vmovsr(destination.AsFpuRegister<SRegister>(), source.AsRegister<Register>()); 1503 } else if (source.IsFpuRegister()) { 1504 __ vmovs(destination.AsFpuRegister<SRegister>(), source.AsFpuRegister<SRegister>()); 1505 } else { 1506 __ LoadSFromOffset(destination.AsFpuRegister<SRegister>(), SP, source.GetStackIndex()); 1507 } 1508 } else { 1509 DCHECK(destination.IsStackSlot()) << destination; 1510 if (source.IsRegister()) { 1511 __ StoreToOffset(kStoreWord, source.AsRegister<Register>(), SP, destination.GetStackIndex()); 1512 } else if (source.IsFpuRegister()) { 1513 __ StoreSToOffset(source.AsFpuRegister<SRegister>(), SP, destination.GetStackIndex()); 1514 } else { 1515 DCHECK(source.IsStackSlot()) << source; 1516 __ LoadFromOffset(kLoadWord, IP, SP, source.GetStackIndex()); 1517 __ StoreToOffset(kStoreWord, IP, SP, destination.GetStackIndex()); 1518 } 1519 } 1520} 1521 1522void CodeGeneratorARM::Move64(Location destination, Location source) { 1523 if (source.Equals(destination)) { 1524 return; 1525 } 1526 if (destination.IsRegisterPair()) { 1527 if (source.IsRegisterPair()) { 1528 EmitParallelMoves( 1529 Location::RegisterLocation(source.AsRegisterPairHigh<Register>()), 1530 Location::RegisterLocation(destination.AsRegisterPairHigh<Register>()), 1531 Primitive::kPrimInt, 1532 Location::RegisterLocation(source.AsRegisterPairLow<Register>()), 1533 Location::RegisterLocation(destination.AsRegisterPairLow<Register>()), 1534 Primitive::kPrimInt); 1535 } else if (source.IsFpuRegister()) { 1536 UNIMPLEMENTED(FATAL); 1537 } else if (source.IsFpuRegisterPair()) { 1538 __ vmovrrd(destination.AsRegisterPairLow<Register>(), 1539 destination.AsRegisterPairHigh<Register>(), 1540 FromLowSToD(source.AsFpuRegisterPairLow<SRegister>())); 1541 } else { 1542 DCHECK(source.IsDoubleStackSlot()); 1543 DCHECK(ExpectedPairLayout(destination)); 1544 __ LoadFromOffset(kLoadWordPair, destination.AsRegisterPairLow<Register>(), 1545 SP, source.GetStackIndex()); 1546 } 1547 } else if (destination.IsFpuRegisterPair()) { 1548 if (source.IsDoubleStackSlot()) { 1549 __ LoadDFromOffset(FromLowSToD(destination.AsFpuRegisterPairLow<SRegister>()), 1550 SP, 1551 source.GetStackIndex()); 1552 } else if (source.IsRegisterPair()) { 1553 __ vmovdrr(FromLowSToD(destination.AsFpuRegisterPairLow<SRegister>()), 1554 source.AsRegisterPairLow<Register>(), 1555 source.AsRegisterPairHigh<Register>()); 1556 } else { 1557 UNIMPLEMENTED(FATAL); 1558 } 1559 } else { 1560 DCHECK(destination.IsDoubleStackSlot()); 1561 if (source.IsRegisterPair()) { 1562 // No conflict possible, so just do the moves. 1563 if (source.AsRegisterPairLow<Register>() == R1) { 1564 DCHECK_EQ(source.AsRegisterPairHigh<Register>(), R2); 1565 __ StoreToOffset(kStoreWord, R1, SP, destination.GetStackIndex()); 1566 __ StoreToOffset(kStoreWord, R2, SP, destination.GetHighStackIndex(kArmWordSize)); 1567 } else { 1568 __ StoreToOffset(kStoreWordPair, source.AsRegisterPairLow<Register>(), 1569 SP, destination.GetStackIndex()); 1570 } 1571 } else if (source.IsFpuRegisterPair()) { 1572 __ StoreDToOffset(FromLowSToD(source.AsFpuRegisterPairLow<SRegister>()), 1573 SP, 1574 destination.GetStackIndex()); 1575 } else { 1576 DCHECK(source.IsDoubleStackSlot()); 1577 EmitParallelMoves( 1578 Location::StackSlot(source.GetStackIndex()), 1579 Location::StackSlot(destination.GetStackIndex()), 1580 Primitive::kPrimInt, 1581 Location::StackSlot(source.GetHighStackIndex(kArmWordSize)), 1582 Location::StackSlot(destination.GetHighStackIndex(kArmWordSize)), 1583 Primitive::kPrimInt); 1584 } 1585 } 1586} 1587 1588void CodeGeneratorARM::MoveConstant(Location location, int32_t value) { 1589 DCHECK(location.IsRegister()); 1590 __ LoadImmediate(location.AsRegister<Register>(), value); 1591} 1592 1593void CodeGeneratorARM::MoveLocation(Location dst, Location src, Primitive::Type dst_type) { 1594 HParallelMove move(GetGraph()->GetArena()); 1595 move.AddMove(src, dst, dst_type, nullptr); 1596 GetMoveResolver()->EmitNativeCode(&move); 1597} 1598 1599void CodeGeneratorARM::AddLocationAsTemp(Location location, LocationSummary* locations) { 1600 if (location.IsRegister()) { 1601 locations->AddTemp(location); 1602 } else if (location.IsRegisterPair()) { 1603 locations->AddTemp(Location::RegisterLocation(location.AsRegisterPairLow<Register>())); 1604 locations->AddTemp(Location::RegisterLocation(location.AsRegisterPairHigh<Register>())); 1605 } else { 1606 UNIMPLEMENTED(FATAL) << "AddLocationAsTemp not implemented for location " << location; 1607 } 1608} 1609 1610void CodeGeneratorARM::InvokeRuntime(QuickEntrypointEnum entrypoint, 1611 HInstruction* instruction, 1612 uint32_t dex_pc, 1613 SlowPathCode* slow_path) { 1614 ValidateInvokeRuntime(entrypoint, instruction, slow_path); 1615 GenerateInvokeRuntime(GetThreadOffset<kArmPointerSize>(entrypoint).Int32Value()); 1616 if (EntrypointRequiresStackMap(entrypoint)) { 1617 RecordPcInfo(instruction, dex_pc, slow_path); 1618 } 1619} 1620 1621void CodeGeneratorARM::InvokeRuntimeWithoutRecordingPcInfo(int32_t entry_point_offset, 1622 HInstruction* instruction, 1623 SlowPathCode* slow_path) { 1624 ValidateInvokeRuntimeWithoutRecordingPcInfo(instruction, slow_path); 1625 GenerateInvokeRuntime(entry_point_offset); 1626} 1627 1628void CodeGeneratorARM::GenerateInvokeRuntime(int32_t entry_point_offset) { 1629 __ LoadFromOffset(kLoadWord, LR, TR, entry_point_offset); 1630 __ blx(LR); 1631} 1632 1633void InstructionCodeGeneratorARM::HandleGoto(HInstruction* got, HBasicBlock* successor) { 1634 DCHECK(!successor->IsExitBlock()); 1635 1636 HBasicBlock* block = got->GetBlock(); 1637 HInstruction* previous = got->GetPrevious(); 1638 1639 HLoopInformation* info = block->GetLoopInformation(); 1640 if (info != nullptr && info->IsBackEdge(*block) && info->HasSuspendCheck()) { 1641 codegen_->ClearSpillSlotsFromLoopPhisInStackMap(info->GetSuspendCheck()); 1642 GenerateSuspendCheck(info->GetSuspendCheck(), successor); 1643 return; 1644 } 1645 1646 if (block->IsEntryBlock() && (previous != nullptr) && previous->IsSuspendCheck()) { 1647 GenerateSuspendCheck(previous->AsSuspendCheck(), nullptr); 1648 } 1649 if (!codegen_->GoesToNextBlock(got->GetBlock(), successor)) { 1650 __ b(codegen_->GetLabelOf(successor)); 1651 } 1652} 1653 1654void LocationsBuilderARM::VisitGoto(HGoto* got) { 1655 got->SetLocations(nullptr); 1656} 1657 1658void InstructionCodeGeneratorARM::VisitGoto(HGoto* got) { 1659 HandleGoto(got, got->GetSuccessor()); 1660} 1661 1662void LocationsBuilderARM::VisitTryBoundary(HTryBoundary* try_boundary) { 1663 try_boundary->SetLocations(nullptr); 1664} 1665 1666void InstructionCodeGeneratorARM::VisitTryBoundary(HTryBoundary* try_boundary) { 1667 HBasicBlock* successor = try_boundary->GetNormalFlowSuccessor(); 1668 if (!successor->IsExitBlock()) { 1669 HandleGoto(try_boundary, successor); 1670 } 1671} 1672 1673void LocationsBuilderARM::VisitExit(HExit* exit) { 1674 exit->SetLocations(nullptr); 1675} 1676 1677void InstructionCodeGeneratorARM::VisitExit(HExit* exit ATTRIBUTE_UNUSED) { 1678} 1679 1680void InstructionCodeGeneratorARM::GenerateVcmp(HInstruction* instruction) { 1681 Primitive::Type type = instruction->InputAt(0)->GetType(); 1682 Location lhs_loc = instruction->GetLocations()->InAt(0); 1683 Location rhs_loc = instruction->GetLocations()->InAt(1); 1684 if (rhs_loc.IsConstant()) { 1685 // 0.0 is the only immediate that can be encoded directly in 1686 // a VCMP instruction. 1687 // 1688 // Both the JLS (section 15.20.1) and the JVMS (section 6.5) 1689 // specify that in a floating-point comparison, positive zero 1690 // and negative zero are considered equal, so we can use the 1691 // literal 0.0 for both cases here. 1692 // 1693 // Note however that some methods (Float.equal, Float.compare, 1694 // Float.compareTo, Double.equal, Double.compare, 1695 // Double.compareTo, Math.max, Math.min, StrictMath.max, 1696 // StrictMath.min) consider 0.0 to be (strictly) greater than 1697 // -0.0. So if we ever translate calls to these methods into a 1698 // HCompare instruction, we must handle the -0.0 case with 1699 // care here. 1700 DCHECK(rhs_loc.GetConstant()->IsArithmeticZero()); 1701 if (type == Primitive::kPrimFloat) { 1702 __ vcmpsz(lhs_loc.AsFpuRegister<SRegister>()); 1703 } else { 1704 DCHECK_EQ(type, Primitive::kPrimDouble); 1705 __ vcmpdz(FromLowSToD(lhs_loc.AsFpuRegisterPairLow<SRegister>())); 1706 } 1707 } else { 1708 if (type == Primitive::kPrimFloat) { 1709 __ vcmps(lhs_loc.AsFpuRegister<SRegister>(), rhs_loc.AsFpuRegister<SRegister>()); 1710 } else { 1711 DCHECK_EQ(type, Primitive::kPrimDouble); 1712 __ vcmpd(FromLowSToD(lhs_loc.AsFpuRegisterPairLow<SRegister>()), 1713 FromLowSToD(rhs_loc.AsFpuRegisterPairLow<SRegister>())); 1714 } 1715 } 1716} 1717 1718void InstructionCodeGeneratorARM::GenerateFPJumps(HCondition* cond, 1719 Label* true_label, 1720 Label* false_label ATTRIBUTE_UNUSED) { 1721 __ vmstat(); // transfer FP status register to ARM APSR. 1722 __ b(true_label, ARMFPCondition(cond->GetCondition(), cond->IsGtBias())); 1723} 1724 1725void InstructionCodeGeneratorARM::GenerateLongComparesAndJumps(HCondition* cond, 1726 Label* true_label, 1727 Label* false_label) { 1728 LocationSummary* locations = cond->GetLocations(); 1729 Location left = locations->InAt(0); 1730 Location right = locations->InAt(1); 1731 IfCondition if_cond = cond->GetCondition(); 1732 1733 Register left_high = left.AsRegisterPairHigh<Register>(); 1734 Register left_low = left.AsRegisterPairLow<Register>(); 1735 IfCondition true_high_cond = if_cond; 1736 IfCondition false_high_cond = cond->GetOppositeCondition(); 1737 Condition final_condition = ARMUnsignedCondition(if_cond); // unsigned on lower part 1738 1739 // Set the conditions for the test, remembering that == needs to be 1740 // decided using the low words. 1741 // TODO: consider avoiding jumps with temporary and CMP low+SBC high 1742 switch (if_cond) { 1743 case kCondEQ: 1744 case kCondNE: 1745 // Nothing to do. 1746 break; 1747 case kCondLT: 1748 false_high_cond = kCondGT; 1749 break; 1750 case kCondLE: 1751 true_high_cond = kCondLT; 1752 break; 1753 case kCondGT: 1754 false_high_cond = kCondLT; 1755 break; 1756 case kCondGE: 1757 true_high_cond = kCondGT; 1758 break; 1759 case kCondB: 1760 false_high_cond = kCondA; 1761 break; 1762 case kCondBE: 1763 true_high_cond = kCondB; 1764 break; 1765 case kCondA: 1766 false_high_cond = kCondB; 1767 break; 1768 case kCondAE: 1769 true_high_cond = kCondA; 1770 break; 1771 } 1772 if (right.IsConstant()) { 1773 int64_t value = right.GetConstant()->AsLongConstant()->GetValue(); 1774 int32_t val_low = Low32Bits(value); 1775 int32_t val_high = High32Bits(value); 1776 1777 __ CmpConstant(left_high, val_high); 1778 if (if_cond == kCondNE) { 1779 __ b(true_label, ARMCondition(true_high_cond)); 1780 } else if (if_cond == kCondEQ) { 1781 __ b(false_label, ARMCondition(false_high_cond)); 1782 } else { 1783 __ b(true_label, ARMCondition(true_high_cond)); 1784 __ b(false_label, ARMCondition(false_high_cond)); 1785 } 1786 // Must be equal high, so compare the lows. 1787 __ CmpConstant(left_low, val_low); 1788 } else { 1789 Register right_high = right.AsRegisterPairHigh<Register>(); 1790 Register right_low = right.AsRegisterPairLow<Register>(); 1791 1792 __ cmp(left_high, ShifterOperand(right_high)); 1793 if (if_cond == kCondNE) { 1794 __ b(true_label, ARMCondition(true_high_cond)); 1795 } else if (if_cond == kCondEQ) { 1796 __ b(false_label, ARMCondition(false_high_cond)); 1797 } else { 1798 __ b(true_label, ARMCondition(true_high_cond)); 1799 __ b(false_label, ARMCondition(false_high_cond)); 1800 } 1801 // Must be equal high, so compare the lows. 1802 __ cmp(left_low, ShifterOperand(right_low)); 1803 } 1804 // The last comparison might be unsigned. 1805 // TODO: optimize cases where this is always true/false 1806 __ b(true_label, final_condition); 1807} 1808 1809void InstructionCodeGeneratorARM::GenerateCompareTestAndBranch(HCondition* condition, 1810 Label* true_target_in, 1811 Label* false_target_in) { 1812 // Generated branching requires both targets to be explicit. If either of the 1813 // targets is nullptr (fallthrough) use and bind `fallthrough_target` instead. 1814 Label fallthrough_target; 1815 Label* true_target = true_target_in == nullptr ? &fallthrough_target : true_target_in; 1816 Label* false_target = false_target_in == nullptr ? &fallthrough_target : false_target_in; 1817 1818 Primitive::Type type = condition->InputAt(0)->GetType(); 1819 switch (type) { 1820 case Primitive::kPrimLong: 1821 GenerateLongComparesAndJumps(condition, true_target, false_target); 1822 break; 1823 case Primitive::kPrimFloat: 1824 case Primitive::kPrimDouble: 1825 GenerateVcmp(condition); 1826 GenerateFPJumps(condition, true_target, false_target); 1827 break; 1828 default: 1829 LOG(FATAL) << "Unexpected compare type " << type; 1830 } 1831 1832 if (false_target != &fallthrough_target) { 1833 __ b(false_target); 1834 } 1835 1836 if (fallthrough_target.IsLinked()) { 1837 __ Bind(&fallthrough_target); 1838 } 1839} 1840 1841void InstructionCodeGeneratorARM::GenerateTestAndBranch(HInstruction* instruction, 1842 size_t condition_input_index, 1843 Label* true_target, 1844 Label* false_target) { 1845 HInstruction* cond = instruction->InputAt(condition_input_index); 1846 1847 if (true_target == nullptr && false_target == nullptr) { 1848 // Nothing to do. The code always falls through. 1849 return; 1850 } else if (cond->IsIntConstant()) { 1851 // Constant condition, statically compared against "true" (integer value 1). 1852 if (cond->AsIntConstant()->IsTrue()) { 1853 if (true_target != nullptr) { 1854 __ b(true_target); 1855 } 1856 } else { 1857 DCHECK(cond->AsIntConstant()->IsFalse()) << cond->AsIntConstant()->GetValue(); 1858 if (false_target != nullptr) { 1859 __ b(false_target); 1860 } 1861 } 1862 return; 1863 } 1864 1865 // The following code generates these patterns: 1866 // (1) true_target == nullptr && false_target != nullptr 1867 // - opposite condition true => branch to false_target 1868 // (2) true_target != nullptr && false_target == nullptr 1869 // - condition true => branch to true_target 1870 // (3) true_target != nullptr && false_target != nullptr 1871 // - condition true => branch to true_target 1872 // - branch to false_target 1873 if (IsBooleanValueOrMaterializedCondition(cond)) { 1874 // Condition has been materialized, compare the output to 0. 1875 Location cond_val = instruction->GetLocations()->InAt(condition_input_index); 1876 DCHECK(cond_val.IsRegister()); 1877 if (true_target == nullptr) { 1878 __ CompareAndBranchIfZero(cond_val.AsRegister<Register>(), false_target); 1879 } else { 1880 __ CompareAndBranchIfNonZero(cond_val.AsRegister<Register>(), true_target); 1881 } 1882 } else { 1883 // Condition has not been materialized. Use its inputs as the comparison and 1884 // its condition as the branch condition. 1885 HCondition* condition = cond->AsCondition(); 1886 1887 // If this is a long or FP comparison that has been folded into 1888 // the HCondition, generate the comparison directly. 1889 Primitive::Type type = condition->InputAt(0)->GetType(); 1890 if (type == Primitive::kPrimLong || Primitive::IsFloatingPointType(type)) { 1891 GenerateCompareTestAndBranch(condition, true_target, false_target); 1892 return; 1893 } 1894 1895 LocationSummary* locations = cond->GetLocations(); 1896 DCHECK(locations->InAt(0).IsRegister()); 1897 Register left = locations->InAt(0).AsRegister<Register>(); 1898 Location right = locations->InAt(1); 1899 if (right.IsRegister()) { 1900 __ cmp(left, ShifterOperand(right.AsRegister<Register>())); 1901 } else { 1902 DCHECK(right.IsConstant()); 1903 __ CmpConstant(left, CodeGenerator::GetInt32ValueOf(right.GetConstant())); 1904 } 1905 if (true_target == nullptr) { 1906 __ b(false_target, ARMCondition(condition->GetOppositeCondition())); 1907 } else { 1908 __ b(true_target, ARMCondition(condition->GetCondition())); 1909 } 1910 } 1911 1912 // If neither branch falls through (case 3), the conditional branch to `true_target` 1913 // was already emitted (case 2) and we need to emit a jump to `false_target`. 1914 if (true_target != nullptr && false_target != nullptr) { 1915 __ b(false_target); 1916 } 1917} 1918 1919void LocationsBuilderARM::VisitIf(HIf* if_instr) { 1920 LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(if_instr); 1921 if (IsBooleanValueOrMaterializedCondition(if_instr->InputAt(0))) { 1922 locations->SetInAt(0, Location::RequiresRegister()); 1923 } 1924} 1925 1926void InstructionCodeGeneratorARM::VisitIf(HIf* if_instr) { 1927 HBasicBlock* true_successor = if_instr->IfTrueSuccessor(); 1928 HBasicBlock* false_successor = if_instr->IfFalseSuccessor(); 1929 Label* true_target = codegen_->GoesToNextBlock(if_instr->GetBlock(), true_successor) ? 1930 nullptr : codegen_->GetLabelOf(true_successor); 1931 Label* false_target = codegen_->GoesToNextBlock(if_instr->GetBlock(), false_successor) ? 1932 nullptr : codegen_->GetLabelOf(false_successor); 1933 GenerateTestAndBranch(if_instr, /* condition_input_index */ 0, true_target, false_target); 1934} 1935 1936void LocationsBuilderARM::VisitDeoptimize(HDeoptimize* deoptimize) { 1937 LocationSummary* locations = new (GetGraph()->GetArena()) 1938 LocationSummary(deoptimize, LocationSummary::kCallOnSlowPath); 1939 locations->SetCustomSlowPathCallerSaves(RegisterSet::Empty()); // No caller-save registers. 1940 if (IsBooleanValueOrMaterializedCondition(deoptimize->InputAt(0))) { 1941 locations->SetInAt(0, Location::RequiresRegister()); 1942 } 1943} 1944 1945void InstructionCodeGeneratorARM::VisitDeoptimize(HDeoptimize* deoptimize) { 1946 SlowPathCodeARM* slow_path = deopt_slow_paths_.NewSlowPath<DeoptimizationSlowPathARM>(deoptimize); 1947 GenerateTestAndBranch(deoptimize, 1948 /* condition_input_index */ 0, 1949 slow_path->GetEntryLabel(), 1950 /* false_target */ nullptr); 1951} 1952 1953void LocationsBuilderARM::VisitSelect(HSelect* select) { 1954 LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(select); 1955 if (Primitive::IsFloatingPointType(select->GetType())) { 1956 locations->SetInAt(0, Location::RequiresFpuRegister()); 1957 locations->SetInAt(1, Location::RequiresFpuRegister()); 1958 } else { 1959 locations->SetInAt(0, Location::RequiresRegister()); 1960 locations->SetInAt(1, Location::RequiresRegister()); 1961 } 1962 if (IsBooleanValueOrMaterializedCondition(select->GetCondition())) { 1963 locations->SetInAt(2, Location::RequiresRegister()); 1964 } 1965 locations->SetOut(Location::SameAsFirstInput()); 1966} 1967 1968void InstructionCodeGeneratorARM::VisitSelect(HSelect* select) { 1969 LocationSummary* locations = select->GetLocations(); 1970 Label false_target; 1971 GenerateTestAndBranch(select, 1972 /* condition_input_index */ 2, 1973 /* true_target */ nullptr, 1974 &false_target); 1975 codegen_->MoveLocation(locations->Out(), locations->InAt(1), select->GetType()); 1976 __ Bind(&false_target); 1977} 1978 1979void LocationsBuilderARM::VisitNativeDebugInfo(HNativeDebugInfo* info) { 1980 new (GetGraph()->GetArena()) LocationSummary(info); 1981} 1982 1983void InstructionCodeGeneratorARM::VisitNativeDebugInfo(HNativeDebugInfo*) { 1984 // MaybeRecordNativeDebugInfo is already called implicitly in CodeGenerator::Compile. 1985} 1986 1987void CodeGeneratorARM::GenerateNop() { 1988 __ nop(); 1989} 1990 1991void LocationsBuilderARM::HandleCondition(HCondition* cond) { 1992 LocationSummary* locations = 1993 new (GetGraph()->GetArena()) LocationSummary(cond, LocationSummary::kNoCall); 1994 // Handle the long/FP comparisons made in instruction simplification. 1995 switch (cond->InputAt(0)->GetType()) { 1996 case Primitive::kPrimLong: 1997 locations->SetInAt(0, Location::RequiresRegister()); 1998 locations->SetInAt(1, Location::RegisterOrConstant(cond->InputAt(1))); 1999 if (!cond->IsEmittedAtUseSite()) { 2000 locations->SetOut(Location::RequiresRegister(), Location::kOutputOverlap); 2001 } 2002 break; 2003 2004 case Primitive::kPrimFloat: 2005 case Primitive::kPrimDouble: 2006 locations->SetInAt(0, Location::RequiresFpuRegister()); 2007 locations->SetInAt(1, ArithmeticZeroOrFpuRegister(cond->InputAt(1))); 2008 if (!cond->IsEmittedAtUseSite()) { 2009 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 2010 } 2011 break; 2012 2013 default: 2014 locations->SetInAt(0, Location::RequiresRegister()); 2015 locations->SetInAt(1, Location::RegisterOrConstant(cond->InputAt(1))); 2016 if (!cond->IsEmittedAtUseSite()) { 2017 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 2018 } 2019 } 2020} 2021 2022void InstructionCodeGeneratorARM::HandleCondition(HCondition* cond) { 2023 if (cond->IsEmittedAtUseSite()) { 2024 return; 2025 } 2026 2027 LocationSummary* locations = cond->GetLocations(); 2028 Location left = locations->InAt(0); 2029 Location right = locations->InAt(1); 2030 Register out = locations->Out().AsRegister<Register>(); 2031 Label true_label, false_label; 2032 2033 switch (cond->InputAt(0)->GetType()) { 2034 default: { 2035 // Integer case. 2036 if (right.IsRegister()) { 2037 __ cmp(left.AsRegister<Register>(), ShifterOperand(right.AsRegister<Register>())); 2038 } else { 2039 DCHECK(right.IsConstant()); 2040 __ CmpConstant(left.AsRegister<Register>(), 2041 CodeGenerator::GetInt32ValueOf(right.GetConstant())); 2042 } 2043 __ it(ARMCondition(cond->GetCondition()), kItElse); 2044 __ mov(locations->Out().AsRegister<Register>(), ShifterOperand(1), 2045 ARMCondition(cond->GetCondition())); 2046 __ mov(locations->Out().AsRegister<Register>(), ShifterOperand(0), 2047 ARMCondition(cond->GetOppositeCondition())); 2048 return; 2049 } 2050 case Primitive::kPrimLong: 2051 GenerateLongComparesAndJumps(cond, &true_label, &false_label); 2052 break; 2053 case Primitive::kPrimFloat: 2054 case Primitive::kPrimDouble: 2055 GenerateVcmp(cond); 2056 GenerateFPJumps(cond, &true_label, &false_label); 2057 break; 2058 } 2059 2060 // Convert the jumps into the result. 2061 Label done_label; 2062 2063 // False case: result = 0. 2064 __ Bind(&false_label); 2065 __ LoadImmediate(out, 0); 2066 __ b(&done_label); 2067 2068 // True case: result = 1. 2069 __ Bind(&true_label); 2070 __ LoadImmediate(out, 1); 2071 __ Bind(&done_label); 2072} 2073 2074void LocationsBuilderARM::VisitEqual(HEqual* comp) { 2075 HandleCondition(comp); 2076} 2077 2078void InstructionCodeGeneratorARM::VisitEqual(HEqual* comp) { 2079 HandleCondition(comp); 2080} 2081 2082void LocationsBuilderARM::VisitNotEqual(HNotEqual* comp) { 2083 HandleCondition(comp); 2084} 2085 2086void InstructionCodeGeneratorARM::VisitNotEqual(HNotEqual* comp) { 2087 HandleCondition(comp); 2088} 2089 2090void LocationsBuilderARM::VisitLessThan(HLessThan* comp) { 2091 HandleCondition(comp); 2092} 2093 2094void InstructionCodeGeneratorARM::VisitLessThan(HLessThan* comp) { 2095 HandleCondition(comp); 2096} 2097 2098void LocationsBuilderARM::VisitLessThanOrEqual(HLessThanOrEqual* comp) { 2099 HandleCondition(comp); 2100} 2101 2102void InstructionCodeGeneratorARM::VisitLessThanOrEqual(HLessThanOrEqual* comp) { 2103 HandleCondition(comp); 2104} 2105 2106void LocationsBuilderARM::VisitGreaterThan(HGreaterThan* comp) { 2107 HandleCondition(comp); 2108} 2109 2110void InstructionCodeGeneratorARM::VisitGreaterThan(HGreaterThan* comp) { 2111 HandleCondition(comp); 2112} 2113 2114void LocationsBuilderARM::VisitGreaterThanOrEqual(HGreaterThanOrEqual* comp) { 2115 HandleCondition(comp); 2116} 2117 2118void InstructionCodeGeneratorARM::VisitGreaterThanOrEqual(HGreaterThanOrEqual* comp) { 2119 HandleCondition(comp); 2120} 2121 2122void LocationsBuilderARM::VisitBelow(HBelow* comp) { 2123 HandleCondition(comp); 2124} 2125 2126void InstructionCodeGeneratorARM::VisitBelow(HBelow* comp) { 2127 HandleCondition(comp); 2128} 2129 2130void LocationsBuilderARM::VisitBelowOrEqual(HBelowOrEqual* comp) { 2131 HandleCondition(comp); 2132} 2133 2134void InstructionCodeGeneratorARM::VisitBelowOrEqual(HBelowOrEqual* comp) { 2135 HandleCondition(comp); 2136} 2137 2138void LocationsBuilderARM::VisitAbove(HAbove* comp) { 2139 HandleCondition(comp); 2140} 2141 2142void InstructionCodeGeneratorARM::VisitAbove(HAbove* comp) { 2143 HandleCondition(comp); 2144} 2145 2146void LocationsBuilderARM::VisitAboveOrEqual(HAboveOrEqual* comp) { 2147 HandleCondition(comp); 2148} 2149 2150void InstructionCodeGeneratorARM::VisitAboveOrEqual(HAboveOrEqual* comp) { 2151 HandleCondition(comp); 2152} 2153 2154void LocationsBuilderARM::VisitIntConstant(HIntConstant* constant) { 2155 LocationSummary* locations = 2156 new (GetGraph()->GetArena()) LocationSummary(constant, LocationSummary::kNoCall); 2157 locations->SetOut(Location::ConstantLocation(constant)); 2158} 2159 2160void InstructionCodeGeneratorARM::VisitIntConstant(HIntConstant* constant ATTRIBUTE_UNUSED) { 2161 // Will be generated at use site. 2162} 2163 2164void LocationsBuilderARM::VisitNullConstant(HNullConstant* constant) { 2165 LocationSummary* locations = 2166 new (GetGraph()->GetArena()) LocationSummary(constant, LocationSummary::kNoCall); 2167 locations->SetOut(Location::ConstantLocation(constant)); 2168} 2169 2170void InstructionCodeGeneratorARM::VisitNullConstant(HNullConstant* constant ATTRIBUTE_UNUSED) { 2171 // Will be generated at use site. 2172} 2173 2174void LocationsBuilderARM::VisitLongConstant(HLongConstant* constant) { 2175 LocationSummary* locations = 2176 new (GetGraph()->GetArena()) LocationSummary(constant, LocationSummary::kNoCall); 2177 locations->SetOut(Location::ConstantLocation(constant)); 2178} 2179 2180void InstructionCodeGeneratorARM::VisitLongConstant(HLongConstant* constant ATTRIBUTE_UNUSED) { 2181 // Will be generated at use site. 2182} 2183 2184void LocationsBuilderARM::VisitFloatConstant(HFloatConstant* constant) { 2185 LocationSummary* locations = 2186 new (GetGraph()->GetArena()) LocationSummary(constant, LocationSummary::kNoCall); 2187 locations->SetOut(Location::ConstantLocation(constant)); 2188} 2189 2190void InstructionCodeGeneratorARM::VisitFloatConstant(HFloatConstant* constant ATTRIBUTE_UNUSED) { 2191 // Will be generated at use site. 2192} 2193 2194void LocationsBuilderARM::VisitDoubleConstant(HDoubleConstant* constant) { 2195 LocationSummary* locations = 2196 new (GetGraph()->GetArena()) LocationSummary(constant, LocationSummary::kNoCall); 2197 locations->SetOut(Location::ConstantLocation(constant)); 2198} 2199 2200void InstructionCodeGeneratorARM::VisitDoubleConstant(HDoubleConstant* constant ATTRIBUTE_UNUSED) { 2201 // Will be generated at use site. 2202} 2203 2204void LocationsBuilderARM::VisitMemoryBarrier(HMemoryBarrier* memory_barrier) { 2205 memory_barrier->SetLocations(nullptr); 2206} 2207 2208void InstructionCodeGeneratorARM::VisitMemoryBarrier(HMemoryBarrier* memory_barrier) { 2209 codegen_->GenerateMemoryBarrier(memory_barrier->GetBarrierKind()); 2210} 2211 2212void LocationsBuilderARM::VisitReturnVoid(HReturnVoid* ret) { 2213 ret->SetLocations(nullptr); 2214} 2215 2216void InstructionCodeGeneratorARM::VisitReturnVoid(HReturnVoid* ret ATTRIBUTE_UNUSED) { 2217 codegen_->GenerateFrameExit(); 2218} 2219 2220void LocationsBuilderARM::VisitReturn(HReturn* ret) { 2221 LocationSummary* locations = 2222 new (GetGraph()->GetArena()) LocationSummary(ret, LocationSummary::kNoCall); 2223 locations->SetInAt(0, parameter_visitor_.GetReturnLocation(ret->InputAt(0)->GetType())); 2224} 2225 2226void InstructionCodeGeneratorARM::VisitReturn(HReturn* ret ATTRIBUTE_UNUSED) { 2227 codegen_->GenerateFrameExit(); 2228} 2229 2230void LocationsBuilderARM::VisitInvokeUnresolved(HInvokeUnresolved* invoke) { 2231 // The trampoline uses the same calling convention as dex calling conventions, 2232 // except instead of loading arg0/r0 with the target Method*, arg0/r0 will contain 2233 // the method_idx. 2234 HandleInvoke(invoke); 2235} 2236 2237void InstructionCodeGeneratorARM::VisitInvokeUnresolved(HInvokeUnresolved* invoke) { 2238 codegen_->GenerateInvokeUnresolvedRuntimeCall(invoke); 2239} 2240 2241void LocationsBuilderARM::VisitInvokeStaticOrDirect(HInvokeStaticOrDirect* invoke) { 2242 // Explicit clinit checks triggered by static invokes must have been pruned by 2243 // art::PrepareForRegisterAllocation. 2244 DCHECK(!invoke->IsStaticWithExplicitClinitCheck()); 2245 2246 IntrinsicLocationsBuilderARM intrinsic(codegen_); 2247 if (intrinsic.TryDispatch(invoke)) { 2248 if (invoke->GetLocations()->CanCall() && invoke->HasPcRelativeDexCache()) { 2249 invoke->GetLocations()->SetInAt(invoke->GetSpecialInputIndex(), Location::Any()); 2250 } 2251 return; 2252 } 2253 2254 HandleInvoke(invoke); 2255 2256 // For PC-relative dex cache the invoke has an extra input, the PC-relative address base. 2257 if (invoke->HasPcRelativeDexCache()) { 2258 invoke->GetLocations()->SetInAt(invoke->GetSpecialInputIndex(), Location::RequiresRegister()); 2259 } 2260} 2261 2262static bool TryGenerateIntrinsicCode(HInvoke* invoke, CodeGeneratorARM* codegen) { 2263 if (invoke->GetLocations()->Intrinsified()) { 2264 IntrinsicCodeGeneratorARM intrinsic(codegen); 2265 intrinsic.Dispatch(invoke); 2266 return true; 2267 } 2268 return false; 2269} 2270 2271void InstructionCodeGeneratorARM::VisitInvokeStaticOrDirect(HInvokeStaticOrDirect* invoke) { 2272 // Explicit clinit checks triggered by static invokes must have been pruned by 2273 // art::PrepareForRegisterAllocation. 2274 DCHECK(!invoke->IsStaticWithExplicitClinitCheck()); 2275 2276 if (TryGenerateIntrinsicCode(invoke, codegen_)) { 2277 return; 2278 } 2279 2280 LocationSummary* locations = invoke->GetLocations(); 2281 codegen_->GenerateStaticOrDirectCall( 2282 invoke, locations->HasTemps() ? locations->GetTemp(0) : Location::NoLocation()); 2283 codegen_->RecordPcInfo(invoke, invoke->GetDexPc()); 2284} 2285 2286void LocationsBuilderARM::HandleInvoke(HInvoke* invoke) { 2287 InvokeDexCallingConventionVisitorARM calling_convention_visitor; 2288 CodeGenerator::CreateCommonInvokeLocationSummary(invoke, &calling_convention_visitor); 2289} 2290 2291void LocationsBuilderARM::VisitInvokeVirtual(HInvokeVirtual* invoke) { 2292 IntrinsicLocationsBuilderARM intrinsic(codegen_); 2293 if (intrinsic.TryDispatch(invoke)) { 2294 return; 2295 } 2296 2297 HandleInvoke(invoke); 2298} 2299 2300void InstructionCodeGeneratorARM::VisitInvokeVirtual(HInvokeVirtual* invoke) { 2301 if (TryGenerateIntrinsicCode(invoke, codegen_)) { 2302 return; 2303 } 2304 2305 codegen_->GenerateVirtualCall(invoke, invoke->GetLocations()->GetTemp(0)); 2306 DCHECK(!codegen_->IsLeafMethod()); 2307 codegen_->RecordPcInfo(invoke, invoke->GetDexPc()); 2308} 2309 2310void LocationsBuilderARM::VisitInvokeInterface(HInvokeInterface* invoke) { 2311 HandleInvoke(invoke); 2312 // Add the hidden argument. 2313 invoke->GetLocations()->AddTemp(Location::RegisterLocation(R12)); 2314} 2315 2316void InstructionCodeGeneratorARM::VisitInvokeInterface(HInvokeInterface* invoke) { 2317 // TODO: b/18116999, our IMTs can miss an IncompatibleClassChangeError. 2318 LocationSummary* locations = invoke->GetLocations(); 2319 Register temp = locations->GetTemp(0).AsRegister<Register>(); 2320 Register hidden_reg = locations->GetTemp(1).AsRegister<Register>(); 2321 Location receiver = locations->InAt(0); 2322 uint32_t class_offset = mirror::Object::ClassOffset().Int32Value(); 2323 2324 // Set the hidden argument. This is safe to do this here, as R12 2325 // won't be modified thereafter, before the `blx` (call) instruction. 2326 DCHECK_EQ(R12, hidden_reg); 2327 __ LoadImmediate(hidden_reg, invoke->GetDexMethodIndex()); 2328 2329 if (receiver.IsStackSlot()) { 2330 __ LoadFromOffset(kLoadWord, temp, SP, receiver.GetStackIndex()); 2331 // /* HeapReference<Class> */ temp = temp->klass_ 2332 __ LoadFromOffset(kLoadWord, temp, temp, class_offset); 2333 } else { 2334 // /* HeapReference<Class> */ temp = receiver->klass_ 2335 __ LoadFromOffset(kLoadWord, temp, receiver.AsRegister<Register>(), class_offset); 2336 } 2337 codegen_->MaybeRecordImplicitNullCheck(invoke); 2338 // Instead of simply (possibly) unpoisoning `temp` here, we should 2339 // emit a read barrier for the previous class reference load. 2340 // However this is not required in practice, as this is an 2341 // intermediate/temporary reference and because the current 2342 // concurrent copying collector keeps the from-space memory 2343 // intact/accessible until the end of the marking phase (the 2344 // concurrent copying collector may not in the future). 2345 __ MaybeUnpoisonHeapReference(temp); 2346 __ LoadFromOffset(kLoadWord, temp, temp, 2347 mirror::Class::ImtPtrOffset(kArmPointerSize).Uint32Value()); 2348 uint32_t method_offset = static_cast<uint32_t>(ImTable::OffsetOfElement( 2349 invoke->GetImtIndex(), kArmPointerSize)); 2350 // temp = temp->GetImtEntryAt(method_offset); 2351 __ LoadFromOffset(kLoadWord, temp, temp, method_offset); 2352 uint32_t entry_point = 2353 ArtMethod::EntryPointFromQuickCompiledCodeOffset(kArmPointerSize).Int32Value(); 2354 // LR = temp->GetEntryPoint(); 2355 __ LoadFromOffset(kLoadWord, LR, temp, entry_point); 2356 // LR(); 2357 __ blx(LR); 2358 DCHECK(!codegen_->IsLeafMethod()); 2359 codegen_->RecordPcInfo(invoke, invoke->GetDexPc()); 2360} 2361 2362void LocationsBuilderARM::VisitNeg(HNeg* neg) { 2363 LocationSummary* locations = 2364 new (GetGraph()->GetArena()) LocationSummary(neg, LocationSummary::kNoCall); 2365 switch (neg->GetResultType()) { 2366 case Primitive::kPrimInt: { 2367 locations->SetInAt(0, Location::RequiresRegister()); 2368 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 2369 break; 2370 } 2371 case Primitive::kPrimLong: { 2372 locations->SetInAt(0, Location::RequiresRegister()); 2373 locations->SetOut(Location::RequiresRegister(), Location::kOutputOverlap); 2374 break; 2375 } 2376 2377 case Primitive::kPrimFloat: 2378 case Primitive::kPrimDouble: 2379 locations->SetInAt(0, Location::RequiresFpuRegister()); 2380 locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap); 2381 break; 2382 2383 default: 2384 LOG(FATAL) << "Unexpected neg type " << neg->GetResultType(); 2385 } 2386} 2387 2388void InstructionCodeGeneratorARM::VisitNeg(HNeg* neg) { 2389 LocationSummary* locations = neg->GetLocations(); 2390 Location out = locations->Out(); 2391 Location in = locations->InAt(0); 2392 switch (neg->GetResultType()) { 2393 case Primitive::kPrimInt: 2394 DCHECK(in.IsRegister()); 2395 __ rsb(out.AsRegister<Register>(), in.AsRegister<Register>(), ShifterOperand(0)); 2396 break; 2397 2398 case Primitive::kPrimLong: 2399 DCHECK(in.IsRegisterPair()); 2400 // out.lo = 0 - in.lo (and update the carry/borrow (C) flag) 2401 __ rsbs(out.AsRegisterPairLow<Register>(), 2402 in.AsRegisterPairLow<Register>(), 2403 ShifterOperand(0)); 2404 // We cannot emit an RSC (Reverse Subtract with Carry) 2405 // instruction here, as it does not exist in the Thumb-2 2406 // instruction set. We use the following approach 2407 // using SBC and SUB instead. 2408 // 2409 // out.hi = -C 2410 __ sbc(out.AsRegisterPairHigh<Register>(), 2411 out.AsRegisterPairHigh<Register>(), 2412 ShifterOperand(out.AsRegisterPairHigh<Register>())); 2413 // out.hi = out.hi - in.hi 2414 __ sub(out.AsRegisterPairHigh<Register>(), 2415 out.AsRegisterPairHigh<Register>(), 2416 ShifterOperand(in.AsRegisterPairHigh<Register>())); 2417 break; 2418 2419 case Primitive::kPrimFloat: 2420 DCHECK(in.IsFpuRegister()); 2421 __ vnegs(out.AsFpuRegister<SRegister>(), in.AsFpuRegister<SRegister>()); 2422 break; 2423 2424 case Primitive::kPrimDouble: 2425 DCHECK(in.IsFpuRegisterPair()); 2426 __ vnegd(FromLowSToD(out.AsFpuRegisterPairLow<SRegister>()), 2427 FromLowSToD(in.AsFpuRegisterPairLow<SRegister>())); 2428 break; 2429 2430 default: 2431 LOG(FATAL) << "Unexpected neg type " << neg->GetResultType(); 2432 } 2433} 2434 2435void LocationsBuilderARM::VisitTypeConversion(HTypeConversion* conversion) { 2436 Primitive::Type result_type = conversion->GetResultType(); 2437 Primitive::Type input_type = conversion->GetInputType(); 2438 DCHECK_NE(result_type, input_type); 2439 2440 // The float-to-long, double-to-long and long-to-float type conversions 2441 // rely on a call to the runtime. 2442 LocationSummary::CallKind call_kind = 2443 (((input_type == Primitive::kPrimFloat || input_type == Primitive::kPrimDouble) 2444 && result_type == Primitive::kPrimLong) 2445 || (input_type == Primitive::kPrimLong && result_type == Primitive::kPrimFloat)) 2446 ? LocationSummary::kCallOnMainOnly 2447 : LocationSummary::kNoCall; 2448 LocationSummary* locations = 2449 new (GetGraph()->GetArena()) LocationSummary(conversion, call_kind); 2450 2451 // The Java language does not allow treating boolean as an integral type but 2452 // our bit representation makes it safe. 2453 2454 switch (result_type) { 2455 case Primitive::kPrimByte: 2456 switch (input_type) { 2457 case Primitive::kPrimLong: 2458 // Type conversion from long to byte is a result of code transformations. 2459 case Primitive::kPrimBoolean: 2460 // Boolean input is a result of code transformations. 2461 case Primitive::kPrimShort: 2462 case Primitive::kPrimInt: 2463 case Primitive::kPrimChar: 2464 // Processing a Dex `int-to-byte' instruction. 2465 locations->SetInAt(0, Location::RequiresRegister()); 2466 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 2467 break; 2468 2469 default: 2470 LOG(FATAL) << "Unexpected type conversion from " << input_type 2471 << " to " << result_type; 2472 } 2473 break; 2474 2475 case Primitive::kPrimShort: 2476 switch (input_type) { 2477 case Primitive::kPrimLong: 2478 // Type conversion from long to short is a result of code transformations. 2479 case Primitive::kPrimBoolean: 2480 // Boolean input is a result of code transformations. 2481 case Primitive::kPrimByte: 2482 case Primitive::kPrimInt: 2483 case Primitive::kPrimChar: 2484 // Processing a Dex `int-to-short' instruction. 2485 locations->SetInAt(0, Location::RequiresRegister()); 2486 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 2487 break; 2488 2489 default: 2490 LOG(FATAL) << "Unexpected type conversion from " << input_type 2491 << " to " << result_type; 2492 } 2493 break; 2494 2495 case Primitive::kPrimInt: 2496 switch (input_type) { 2497 case Primitive::kPrimLong: 2498 // Processing a Dex `long-to-int' instruction. 2499 locations->SetInAt(0, Location::Any()); 2500 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 2501 break; 2502 2503 case Primitive::kPrimFloat: 2504 // Processing a Dex `float-to-int' instruction. 2505 locations->SetInAt(0, Location::RequiresFpuRegister()); 2506 locations->SetOut(Location::RequiresRegister()); 2507 locations->AddTemp(Location::RequiresFpuRegister()); 2508 break; 2509 2510 case Primitive::kPrimDouble: 2511 // Processing a Dex `double-to-int' instruction. 2512 locations->SetInAt(0, Location::RequiresFpuRegister()); 2513 locations->SetOut(Location::RequiresRegister()); 2514 locations->AddTemp(Location::RequiresFpuRegister()); 2515 break; 2516 2517 default: 2518 LOG(FATAL) << "Unexpected type conversion from " << input_type 2519 << " to " << result_type; 2520 } 2521 break; 2522 2523 case Primitive::kPrimLong: 2524 switch (input_type) { 2525 case Primitive::kPrimBoolean: 2526 // Boolean input is a result of code transformations. 2527 case Primitive::kPrimByte: 2528 case Primitive::kPrimShort: 2529 case Primitive::kPrimInt: 2530 case Primitive::kPrimChar: 2531 // Processing a Dex `int-to-long' instruction. 2532 locations->SetInAt(0, Location::RequiresRegister()); 2533 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 2534 break; 2535 2536 case Primitive::kPrimFloat: { 2537 // Processing a Dex `float-to-long' instruction. 2538 InvokeRuntimeCallingConvention calling_convention; 2539 locations->SetInAt(0, Location::FpuRegisterLocation( 2540 calling_convention.GetFpuRegisterAt(0))); 2541 locations->SetOut(Location::RegisterPairLocation(R0, R1)); 2542 break; 2543 } 2544 2545 case Primitive::kPrimDouble: { 2546 // Processing a Dex `double-to-long' instruction. 2547 InvokeRuntimeCallingConvention calling_convention; 2548 locations->SetInAt(0, Location::FpuRegisterPairLocation( 2549 calling_convention.GetFpuRegisterAt(0), 2550 calling_convention.GetFpuRegisterAt(1))); 2551 locations->SetOut(Location::RegisterPairLocation(R0, R1)); 2552 break; 2553 } 2554 2555 default: 2556 LOG(FATAL) << "Unexpected type conversion from " << input_type 2557 << " to " << result_type; 2558 } 2559 break; 2560 2561 case Primitive::kPrimChar: 2562 switch (input_type) { 2563 case Primitive::kPrimLong: 2564 // Type conversion from long to char is a result of code transformations. 2565 case Primitive::kPrimBoolean: 2566 // Boolean input is a result of code transformations. 2567 case Primitive::kPrimByte: 2568 case Primitive::kPrimShort: 2569 case Primitive::kPrimInt: 2570 // Processing a Dex `int-to-char' instruction. 2571 locations->SetInAt(0, Location::RequiresRegister()); 2572 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 2573 break; 2574 2575 default: 2576 LOG(FATAL) << "Unexpected type conversion from " << input_type 2577 << " to " << result_type; 2578 } 2579 break; 2580 2581 case Primitive::kPrimFloat: 2582 switch (input_type) { 2583 case Primitive::kPrimBoolean: 2584 // Boolean input is a result of code transformations. 2585 case Primitive::kPrimByte: 2586 case Primitive::kPrimShort: 2587 case Primitive::kPrimInt: 2588 case Primitive::kPrimChar: 2589 // Processing a Dex `int-to-float' instruction. 2590 locations->SetInAt(0, Location::RequiresRegister()); 2591 locations->SetOut(Location::RequiresFpuRegister()); 2592 break; 2593 2594 case Primitive::kPrimLong: { 2595 // Processing a Dex `long-to-float' instruction. 2596 InvokeRuntimeCallingConvention calling_convention; 2597 locations->SetInAt(0, Location::RegisterPairLocation( 2598 calling_convention.GetRegisterAt(0), calling_convention.GetRegisterAt(1))); 2599 locations->SetOut(Location::FpuRegisterLocation(calling_convention.GetFpuRegisterAt(0))); 2600 break; 2601 } 2602 2603 case Primitive::kPrimDouble: 2604 // Processing a Dex `double-to-float' instruction. 2605 locations->SetInAt(0, Location::RequiresFpuRegister()); 2606 locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap); 2607 break; 2608 2609 default: 2610 LOG(FATAL) << "Unexpected type conversion from " << input_type 2611 << " to " << result_type; 2612 }; 2613 break; 2614 2615 case Primitive::kPrimDouble: 2616 switch (input_type) { 2617 case Primitive::kPrimBoolean: 2618 // Boolean input is a result of code transformations. 2619 case Primitive::kPrimByte: 2620 case Primitive::kPrimShort: 2621 case Primitive::kPrimInt: 2622 case Primitive::kPrimChar: 2623 // Processing a Dex `int-to-double' instruction. 2624 locations->SetInAt(0, Location::RequiresRegister()); 2625 locations->SetOut(Location::RequiresFpuRegister()); 2626 break; 2627 2628 case Primitive::kPrimLong: 2629 // Processing a Dex `long-to-double' instruction. 2630 locations->SetInAt(0, Location::RequiresRegister()); 2631 locations->SetOut(Location::RequiresFpuRegister()); 2632 locations->AddTemp(Location::RequiresFpuRegister()); 2633 locations->AddTemp(Location::RequiresFpuRegister()); 2634 break; 2635 2636 case Primitive::kPrimFloat: 2637 // Processing a Dex `float-to-double' instruction. 2638 locations->SetInAt(0, Location::RequiresFpuRegister()); 2639 locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap); 2640 break; 2641 2642 default: 2643 LOG(FATAL) << "Unexpected type conversion from " << input_type 2644 << " to " << result_type; 2645 }; 2646 break; 2647 2648 default: 2649 LOG(FATAL) << "Unexpected type conversion from " << input_type 2650 << " to " << result_type; 2651 } 2652} 2653 2654void InstructionCodeGeneratorARM::VisitTypeConversion(HTypeConversion* conversion) { 2655 LocationSummary* locations = conversion->GetLocations(); 2656 Location out = locations->Out(); 2657 Location in = locations->InAt(0); 2658 Primitive::Type result_type = conversion->GetResultType(); 2659 Primitive::Type input_type = conversion->GetInputType(); 2660 DCHECK_NE(result_type, input_type); 2661 switch (result_type) { 2662 case Primitive::kPrimByte: 2663 switch (input_type) { 2664 case Primitive::kPrimLong: 2665 // Type conversion from long to byte is a result of code transformations. 2666 __ sbfx(out.AsRegister<Register>(), in.AsRegisterPairLow<Register>(), 0, 8); 2667 break; 2668 case Primitive::kPrimBoolean: 2669 // Boolean input is a result of code transformations. 2670 case Primitive::kPrimShort: 2671 case Primitive::kPrimInt: 2672 case Primitive::kPrimChar: 2673 // Processing a Dex `int-to-byte' instruction. 2674 __ sbfx(out.AsRegister<Register>(), in.AsRegister<Register>(), 0, 8); 2675 break; 2676 2677 default: 2678 LOG(FATAL) << "Unexpected type conversion from " << input_type 2679 << " to " << result_type; 2680 } 2681 break; 2682 2683 case Primitive::kPrimShort: 2684 switch (input_type) { 2685 case Primitive::kPrimLong: 2686 // Type conversion from long to short is a result of code transformations. 2687 __ sbfx(out.AsRegister<Register>(), in.AsRegisterPairLow<Register>(), 0, 16); 2688 break; 2689 case Primitive::kPrimBoolean: 2690 // Boolean input is a result of code transformations. 2691 case Primitive::kPrimByte: 2692 case Primitive::kPrimInt: 2693 case Primitive::kPrimChar: 2694 // Processing a Dex `int-to-short' instruction. 2695 __ sbfx(out.AsRegister<Register>(), in.AsRegister<Register>(), 0, 16); 2696 break; 2697 2698 default: 2699 LOG(FATAL) << "Unexpected type conversion from " << input_type 2700 << " to " << result_type; 2701 } 2702 break; 2703 2704 case Primitive::kPrimInt: 2705 switch (input_type) { 2706 case Primitive::kPrimLong: 2707 // Processing a Dex `long-to-int' instruction. 2708 DCHECK(out.IsRegister()); 2709 if (in.IsRegisterPair()) { 2710 __ Mov(out.AsRegister<Register>(), in.AsRegisterPairLow<Register>()); 2711 } else if (in.IsDoubleStackSlot()) { 2712 __ LoadFromOffset(kLoadWord, out.AsRegister<Register>(), SP, in.GetStackIndex()); 2713 } else { 2714 DCHECK(in.IsConstant()); 2715 DCHECK(in.GetConstant()->IsLongConstant()); 2716 int64_t value = in.GetConstant()->AsLongConstant()->GetValue(); 2717 __ LoadImmediate(out.AsRegister<Register>(), static_cast<int32_t>(value)); 2718 } 2719 break; 2720 2721 case Primitive::kPrimFloat: { 2722 // Processing a Dex `float-to-int' instruction. 2723 SRegister temp = locations->GetTemp(0).AsFpuRegisterPairLow<SRegister>(); 2724 __ vcvtis(temp, in.AsFpuRegister<SRegister>()); 2725 __ vmovrs(out.AsRegister<Register>(), temp); 2726 break; 2727 } 2728 2729 case Primitive::kPrimDouble: { 2730 // Processing a Dex `double-to-int' instruction. 2731 SRegister temp_s = locations->GetTemp(0).AsFpuRegisterPairLow<SRegister>(); 2732 __ vcvtid(temp_s, FromLowSToD(in.AsFpuRegisterPairLow<SRegister>())); 2733 __ vmovrs(out.AsRegister<Register>(), temp_s); 2734 break; 2735 } 2736 2737 default: 2738 LOG(FATAL) << "Unexpected type conversion from " << input_type 2739 << " to " << result_type; 2740 } 2741 break; 2742 2743 case Primitive::kPrimLong: 2744 switch (input_type) { 2745 case Primitive::kPrimBoolean: 2746 // Boolean input is a result of code transformations. 2747 case Primitive::kPrimByte: 2748 case Primitive::kPrimShort: 2749 case Primitive::kPrimInt: 2750 case Primitive::kPrimChar: 2751 // Processing a Dex `int-to-long' instruction. 2752 DCHECK(out.IsRegisterPair()); 2753 DCHECK(in.IsRegister()); 2754 __ Mov(out.AsRegisterPairLow<Register>(), in.AsRegister<Register>()); 2755 // Sign extension. 2756 __ Asr(out.AsRegisterPairHigh<Register>(), 2757 out.AsRegisterPairLow<Register>(), 2758 31); 2759 break; 2760 2761 case Primitive::kPrimFloat: 2762 // Processing a Dex `float-to-long' instruction. 2763 codegen_->InvokeRuntime(kQuickF2l, conversion, conversion->GetDexPc()); 2764 CheckEntrypointTypes<kQuickF2l, int64_t, float>(); 2765 break; 2766 2767 case Primitive::kPrimDouble: 2768 // Processing a Dex `double-to-long' instruction. 2769 codegen_->InvokeRuntime(kQuickD2l, conversion, conversion->GetDexPc()); 2770 CheckEntrypointTypes<kQuickD2l, int64_t, double>(); 2771 break; 2772 2773 default: 2774 LOG(FATAL) << "Unexpected type conversion from " << input_type 2775 << " to " << result_type; 2776 } 2777 break; 2778 2779 case Primitive::kPrimChar: 2780 switch (input_type) { 2781 case Primitive::kPrimLong: 2782 // Type conversion from long to char is a result of code transformations. 2783 __ ubfx(out.AsRegister<Register>(), in.AsRegisterPairLow<Register>(), 0, 16); 2784 break; 2785 case Primitive::kPrimBoolean: 2786 // Boolean input is a result of code transformations. 2787 case Primitive::kPrimByte: 2788 case Primitive::kPrimShort: 2789 case Primitive::kPrimInt: 2790 // Processing a Dex `int-to-char' instruction. 2791 __ ubfx(out.AsRegister<Register>(), in.AsRegister<Register>(), 0, 16); 2792 break; 2793 2794 default: 2795 LOG(FATAL) << "Unexpected type conversion from " << input_type 2796 << " to " << result_type; 2797 } 2798 break; 2799 2800 case Primitive::kPrimFloat: 2801 switch (input_type) { 2802 case Primitive::kPrimBoolean: 2803 // Boolean input is a result of code transformations. 2804 case Primitive::kPrimByte: 2805 case Primitive::kPrimShort: 2806 case Primitive::kPrimInt: 2807 case Primitive::kPrimChar: { 2808 // Processing a Dex `int-to-float' instruction. 2809 __ vmovsr(out.AsFpuRegister<SRegister>(), in.AsRegister<Register>()); 2810 __ vcvtsi(out.AsFpuRegister<SRegister>(), out.AsFpuRegister<SRegister>()); 2811 break; 2812 } 2813 2814 case Primitive::kPrimLong: 2815 // Processing a Dex `long-to-float' instruction. 2816 codegen_->InvokeRuntime(kQuickL2f, conversion, conversion->GetDexPc()); 2817 CheckEntrypointTypes<kQuickL2f, float, int64_t>(); 2818 break; 2819 2820 case Primitive::kPrimDouble: 2821 // Processing a Dex `double-to-float' instruction. 2822 __ vcvtsd(out.AsFpuRegister<SRegister>(), 2823 FromLowSToD(in.AsFpuRegisterPairLow<SRegister>())); 2824 break; 2825 2826 default: 2827 LOG(FATAL) << "Unexpected type conversion from " << input_type 2828 << " to " << result_type; 2829 }; 2830 break; 2831 2832 case Primitive::kPrimDouble: 2833 switch (input_type) { 2834 case Primitive::kPrimBoolean: 2835 // Boolean input is a result of code transformations. 2836 case Primitive::kPrimByte: 2837 case Primitive::kPrimShort: 2838 case Primitive::kPrimInt: 2839 case Primitive::kPrimChar: { 2840 // Processing a Dex `int-to-double' instruction. 2841 __ vmovsr(out.AsFpuRegisterPairLow<SRegister>(), in.AsRegister<Register>()); 2842 __ vcvtdi(FromLowSToD(out.AsFpuRegisterPairLow<SRegister>()), 2843 out.AsFpuRegisterPairLow<SRegister>()); 2844 break; 2845 } 2846 2847 case Primitive::kPrimLong: { 2848 // Processing a Dex `long-to-double' instruction. 2849 Register low = in.AsRegisterPairLow<Register>(); 2850 Register high = in.AsRegisterPairHigh<Register>(); 2851 SRegister out_s = out.AsFpuRegisterPairLow<SRegister>(); 2852 DRegister out_d = FromLowSToD(out_s); 2853 SRegister temp_s = locations->GetTemp(0).AsFpuRegisterPairLow<SRegister>(); 2854 DRegister temp_d = FromLowSToD(temp_s); 2855 SRegister constant_s = locations->GetTemp(1).AsFpuRegisterPairLow<SRegister>(); 2856 DRegister constant_d = FromLowSToD(constant_s); 2857 2858 // temp_d = int-to-double(high) 2859 __ vmovsr(temp_s, high); 2860 __ vcvtdi(temp_d, temp_s); 2861 // constant_d = k2Pow32EncodingForDouble 2862 __ LoadDImmediate(constant_d, bit_cast<double, int64_t>(k2Pow32EncodingForDouble)); 2863 // out_d = unsigned-to-double(low) 2864 __ vmovsr(out_s, low); 2865 __ vcvtdu(out_d, out_s); 2866 // out_d += temp_d * constant_d 2867 __ vmlad(out_d, temp_d, constant_d); 2868 break; 2869 } 2870 2871 case Primitive::kPrimFloat: 2872 // Processing a Dex `float-to-double' instruction. 2873 __ vcvtds(FromLowSToD(out.AsFpuRegisterPairLow<SRegister>()), 2874 in.AsFpuRegister<SRegister>()); 2875 break; 2876 2877 default: 2878 LOG(FATAL) << "Unexpected type conversion from " << input_type 2879 << " to " << result_type; 2880 }; 2881 break; 2882 2883 default: 2884 LOG(FATAL) << "Unexpected type conversion from " << input_type 2885 << " to " << result_type; 2886 } 2887} 2888 2889void LocationsBuilderARM::VisitAdd(HAdd* add) { 2890 LocationSummary* locations = 2891 new (GetGraph()->GetArena()) LocationSummary(add, LocationSummary::kNoCall); 2892 switch (add->GetResultType()) { 2893 case Primitive::kPrimInt: { 2894 locations->SetInAt(0, Location::RequiresRegister()); 2895 locations->SetInAt(1, Location::RegisterOrConstant(add->InputAt(1))); 2896 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 2897 break; 2898 } 2899 2900 case Primitive::kPrimLong: { 2901 locations->SetInAt(0, Location::RequiresRegister()); 2902 locations->SetInAt(1, ArmEncodableConstantOrRegister(add->InputAt(1), ADD)); 2903 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 2904 break; 2905 } 2906 2907 case Primitive::kPrimFloat: 2908 case Primitive::kPrimDouble: { 2909 locations->SetInAt(0, Location::RequiresFpuRegister()); 2910 locations->SetInAt(1, Location::RequiresFpuRegister()); 2911 locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap); 2912 break; 2913 } 2914 2915 default: 2916 LOG(FATAL) << "Unexpected add type " << add->GetResultType(); 2917 } 2918} 2919 2920void InstructionCodeGeneratorARM::VisitAdd(HAdd* add) { 2921 LocationSummary* locations = add->GetLocations(); 2922 Location out = locations->Out(); 2923 Location first = locations->InAt(0); 2924 Location second = locations->InAt(1); 2925 switch (add->GetResultType()) { 2926 case Primitive::kPrimInt: 2927 if (second.IsRegister()) { 2928 __ add(out.AsRegister<Register>(), 2929 first.AsRegister<Register>(), 2930 ShifterOperand(second.AsRegister<Register>())); 2931 } else { 2932 __ AddConstant(out.AsRegister<Register>(), 2933 first.AsRegister<Register>(), 2934 second.GetConstant()->AsIntConstant()->GetValue()); 2935 } 2936 break; 2937 2938 case Primitive::kPrimLong: { 2939 if (second.IsConstant()) { 2940 uint64_t value = static_cast<uint64_t>(Int64FromConstant(second.GetConstant())); 2941 GenerateAddLongConst(out, first, value); 2942 } else { 2943 DCHECK(second.IsRegisterPair()); 2944 __ adds(out.AsRegisterPairLow<Register>(), 2945 first.AsRegisterPairLow<Register>(), 2946 ShifterOperand(second.AsRegisterPairLow<Register>())); 2947 __ adc(out.AsRegisterPairHigh<Register>(), 2948 first.AsRegisterPairHigh<Register>(), 2949 ShifterOperand(second.AsRegisterPairHigh<Register>())); 2950 } 2951 break; 2952 } 2953 2954 case Primitive::kPrimFloat: 2955 __ vadds(out.AsFpuRegister<SRegister>(), 2956 first.AsFpuRegister<SRegister>(), 2957 second.AsFpuRegister<SRegister>()); 2958 break; 2959 2960 case Primitive::kPrimDouble: 2961 __ vaddd(FromLowSToD(out.AsFpuRegisterPairLow<SRegister>()), 2962 FromLowSToD(first.AsFpuRegisterPairLow<SRegister>()), 2963 FromLowSToD(second.AsFpuRegisterPairLow<SRegister>())); 2964 break; 2965 2966 default: 2967 LOG(FATAL) << "Unexpected add type " << add->GetResultType(); 2968 } 2969} 2970 2971void LocationsBuilderARM::VisitSub(HSub* sub) { 2972 LocationSummary* locations = 2973 new (GetGraph()->GetArena()) LocationSummary(sub, LocationSummary::kNoCall); 2974 switch (sub->GetResultType()) { 2975 case Primitive::kPrimInt: { 2976 locations->SetInAt(0, Location::RequiresRegister()); 2977 locations->SetInAt(1, Location::RegisterOrConstant(sub->InputAt(1))); 2978 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 2979 break; 2980 } 2981 2982 case Primitive::kPrimLong: { 2983 locations->SetInAt(0, Location::RequiresRegister()); 2984 locations->SetInAt(1, ArmEncodableConstantOrRegister(sub->InputAt(1), SUB)); 2985 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 2986 break; 2987 } 2988 case Primitive::kPrimFloat: 2989 case Primitive::kPrimDouble: { 2990 locations->SetInAt(0, Location::RequiresFpuRegister()); 2991 locations->SetInAt(1, Location::RequiresFpuRegister()); 2992 locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap); 2993 break; 2994 } 2995 default: 2996 LOG(FATAL) << "Unexpected sub type " << sub->GetResultType(); 2997 } 2998} 2999 3000void InstructionCodeGeneratorARM::VisitSub(HSub* sub) { 3001 LocationSummary* locations = sub->GetLocations(); 3002 Location out = locations->Out(); 3003 Location first = locations->InAt(0); 3004 Location second = locations->InAt(1); 3005 switch (sub->GetResultType()) { 3006 case Primitive::kPrimInt: { 3007 if (second.IsRegister()) { 3008 __ sub(out.AsRegister<Register>(), 3009 first.AsRegister<Register>(), 3010 ShifterOperand(second.AsRegister<Register>())); 3011 } else { 3012 __ AddConstant(out.AsRegister<Register>(), 3013 first.AsRegister<Register>(), 3014 -second.GetConstant()->AsIntConstant()->GetValue()); 3015 } 3016 break; 3017 } 3018 3019 case Primitive::kPrimLong: { 3020 if (second.IsConstant()) { 3021 uint64_t value = static_cast<uint64_t>(Int64FromConstant(second.GetConstant())); 3022 GenerateAddLongConst(out, first, -value); 3023 } else { 3024 DCHECK(second.IsRegisterPair()); 3025 __ subs(out.AsRegisterPairLow<Register>(), 3026 first.AsRegisterPairLow<Register>(), 3027 ShifterOperand(second.AsRegisterPairLow<Register>())); 3028 __ sbc(out.AsRegisterPairHigh<Register>(), 3029 first.AsRegisterPairHigh<Register>(), 3030 ShifterOperand(second.AsRegisterPairHigh<Register>())); 3031 } 3032 break; 3033 } 3034 3035 case Primitive::kPrimFloat: { 3036 __ vsubs(out.AsFpuRegister<SRegister>(), 3037 first.AsFpuRegister<SRegister>(), 3038 second.AsFpuRegister<SRegister>()); 3039 break; 3040 } 3041 3042 case Primitive::kPrimDouble: { 3043 __ vsubd(FromLowSToD(out.AsFpuRegisterPairLow<SRegister>()), 3044 FromLowSToD(first.AsFpuRegisterPairLow<SRegister>()), 3045 FromLowSToD(second.AsFpuRegisterPairLow<SRegister>())); 3046 break; 3047 } 3048 3049 3050 default: 3051 LOG(FATAL) << "Unexpected sub type " << sub->GetResultType(); 3052 } 3053} 3054 3055void LocationsBuilderARM::VisitMul(HMul* mul) { 3056 LocationSummary* locations = 3057 new (GetGraph()->GetArena()) LocationSummary(mul, LocationSummary::kNoCall); 3058 switch (mul->GetResultType()) { 3059 case Primitive::kPrimInt: 3060 case Primitive::kPrimLong: { 3061 locations->SetInAt(0, Location::RequiresRegister()); 3062 locations->SetInAt(1, Location::RequiresRegister()); 3063 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 3064 break; 3065 } 3066 3067 case Primitive::kPrimFloat: 3068 case Primitive::kPrimDouble: { 3069 locations->SetInAt(0, Location::RequiresFpuRegister()); 3070 locations->SetInAt(1, Location::RequiresFpuRegister()); 3071 locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap); 3072 break; 3073 } 3074 3075 default: 3076 LOG(FATAL) << "Unexpected mul type " << mul->GetResultType(); 3077 } 3078} 3079 3080void InstructionCodeGeneratorARM::VisitMul(HMul* mul) { 3081 LocationSummary* locations = mul->GetLocations(); 3082 Location out = locations->Out(); 3083 Location first = locations->InAt(0); 3084 Location second = locations->InAt(1); 3085 switch (mul->GetResultType()) { 3086 case Primitive::kPrimInt: { 3087 __ mul(out.AsRegister<Register>(), 3088 first.AsRegister<Register>(), 3089 second.AsRegister<Register>()); 3090 break; 3091 } 3092 case Primitive::kPrimLong: { 3093 Register out_hi = out.AsRegisterPairHigh<Register>(); 3094 Register out_lo = out.AsRegisterPairLow<Register>(); 3095 Register in1_hi = first.AsRegisterPairHigh<Register>(); 3096 Register in1_lo = first.AsRegisterPairLow<Register>(); 3097 Register in2_hi = second.AsRegisterPairHigh<Register>(); 3098 Register in2_lo = second.AsRegisterPairLow<Register>(); 3099 3100 // Extra checks to protect caused by the existence of R1_R2. 3101 // The algorithm is wrong if out.hi is either in1.lo or in2.lo: 3102 // (e.g. in1=r0_r1, in2=r2_r3 and out=r1_r2); 3103 DCHECK_NE(out_hi, in1_lo); 3104 DCHECK_NE(out_hi, in2_lo); 3105 3106 // input: in1 - 64 bits, in2 - 64 bits 3107 // output: out 3108 // formula: out.hi : out.lo = (in1.lo * in2.hi + in1.hi * in2.lo)* 2^32 + in1.lo * in2.lo 3109 // parts: out.hi = in1.lo * in2.hi + in1.hi * in2.lo + (in1.lo * in2.lo)[63:32] 3110 // parts: out.lo = (in1.lo * in2.lo)[31:0] 3111 3112 // IP <- in1.lo * in2.hi 3113 __ mul(IP, in1_lo, in2_hi); 3114 // out.hi <- in1.lo * in2.hi + in1.hi * in2.lo 3115 __ mla(out_hi, in1_hi, in2_lo, IP); 3116 // out.lo <- (in1.lo * in2.lo)[31:0]; 3117 __ umull(out_lo, IP, in1_lo, in2_lo); 3118 // out.hi <- in2.hi * in1.lo + in2.lo * in1.hi + (in1.lo * in2.lo)[63:32] 3119 __ add(out_hi, out_hi, ShifterOperand(IP)); 3120 break; 3121 } 3122 3123 case Primitive::kPrimFloat: { 3124 __ vmuls(out.AsFpuRegister<SRegister>(), 3125 first.AsFpuRegister<SRegister>(), 3126 second.AsFpuRegister<SRegister>()); 3127 break; 3128 } 3129 3130 case Primitive::kPrimDouble: { 3131 __ vmuld(FromLowSToD(out.AsFpuRegisterPairLow<SRegister>()), 3132 FromLowSToD(first.AsFpuRegisterPairLow<SRegister>()), 3133 FromLowSToD(second.AsFpuRegisterPairLow<SRegister>())); 3134 break; 3135 } 3136 3137 default: 3138 LOG(FATAL) << "Unexpected mul type " << mul->GetResultType(); 3139 } 3140} 3141 3142void InstructionCodeGeneratorARM::DivRemOneOrMinusOne(HBinaryOperation* instruction) { 3143 DCHECK(instruction->IsDiv() || instruction->IsRem()); 3144 DCHECK(instruction->GetResultType() == Primitive::kPrimInt); 3145 3146 LocationSummary* locations = instruction->GetLocations(); 3147 Location second = locations->InAt(1); 3148 DCHECK(second.IsConstant()); 3149 3150 Register out = locations->Out().AsRegister<Register>(); 3151 Register dividend = locations->InAt(0).AsRegister<Register>(); 3152 int32_t imm = second.GetConstant()->AsIntConstant()->GetValue(); 3153 DCHECK(imm == 1 || imm == -1); 3154 3155 if (instruction->IsRem()) { 3156 __ LoadImmediate(out, 0); 3157 } else { 3158 if (imm == 1) { 3159 __ Mov(out, dividend); 3160 } else { 3161 __ rsb(out, dividend, ShifterOperand(0)); 3162 } 3163 } 3164} 3165 3166void InstructionCodeGeneratorARM::DivRemByPowerOfTwo(HBinaryOperation* instruction) { 3167 DCHECK(instruction->IsDiv() || instruction->IsRem()); 3168 DCHECK(instruction->GetResultType() == Primitive::kPrimInt); 3169 3170 LocationSummary* locations = instruction->GetLocations(); 3171 Location second = locations->InAt(1); 3172 DCHECK(second.IsConstant()); 3173 3174 Register out = locations->Out().AsRegister<Register>(); 3175 Register dividend = locations->InAt(0).AsRegister<Register>(); 3176 Register temp = locations->GetTemp(0).AsRegister<Register>(); 3177 int32_t imm = second.GetConstant()->AsIntConstant()->GetValue(); 3178 uint32_t abs_imm = static_cast<uint32_t>(AbsOrMin(imm)); 3179 int ctz_imm = CTZ(abs_imm); 3180 3181 if (ctz_imm == 1) { 3182 __ Lsr(temp, dividend, 32 - ctz_imm); 3183 } else { 3184 __ Asr(temp, dividend, 31); 3185 __ Lsr(temp, temp, 32 - ctz_imm); 3186 } 3187 __ add(out, temp, ShifterOperand(dividend)); 3188 3189 if (instruction->IsDiv()) { 3190 __ Asr(out, out, ctz_imm); 3191 if (imm < 0) { 3192 __ rsb(out, out, ShifterOperand(0)); 3193 } 3194 } else { 3195 __ ubfx(out, out, 0, ctz_imm); 3196 __ sub(out, out, ShifterOperand(temp)); 3197 } 3198} 3199 3200void InstructionCodeGeneratorARM::GenerateDivRemWithAnyConstant(HBinaryOperation* instruction) { 3201 DCHECK(instruction->IsDiv() || instruction->IsRem()); 3202 DCHECK(instruction->GetResultType() == Primitive::kPrimInt); 3203 3204 LocationSummary* locations = instruction->GetLocations(); 3205 Location second = locations->InAt(1); 3206 DCHECK(second.IsConstant()); 3207 3208 Register out = locations->Out().AsRegister<Register>(); 3209 Register dividend = locations->InAt(0).AsRegister<Register>(); 3210 Register temp1 = locations->GetTemp(0).AsRegister<Register>(); 3211 Register temp2 = locations->GetTemp(1).AsRegister<Register>(); 3212 int64_t imm = second.GetConstant()->AsIntConstant()->GetValue(); 3213 3214 int64_t magic; 3215 int shift; 3216 CalculateMagicAndShiftForDivRem(imm, false /* is_long */, &magic, &shift); 3217 3218 __ LoadImmediate(temp1, magic); 3219 __ smull(temp2, temp1, dividend, temp1); 3220 3221 if (imm > 0 && magic < 0) { 3222 __ add(temp1, temp1, ShifterOperand(dividend)); 3223 } else if (imm < 0 && magic > 0) { 3224 __ sub(temp1, temp1, ShifterOperand(dividend)); 3225 } 3226 3227 if (shift != 0) { 3228 __ Asr(temp1, temp1, shift); 3229 } 3230 3231 if (instruction->IsDiv()) { 3232 __ sub(out, temp1, ShifterOperand(temp1, ASR, 31)); 3233 } else { 3234 __ sub(temp1, temp1, ShifterOperand(temp1, ASR, 31)); 3235 // TODO: Strength reduction for mls. 3236 __ LoadImmediate(temp2, imm); 3237 __ mls(out, temp1, temp2, dividend); 3238 } 3239} 3240 3241void InstructionCodeGeneratorARM::GenerateDivRemConstantIntegral(HBinaryOperation* instruction) { 3242 DCHECK(instruction->IsDiv() || instruction->IsRem()); 3243 DCHECK(instruction->GetResultType() == Primitive::kPrimInt); 3244 3245 LocationSummary* locations = instruction->GetLocations(); 3246 Location second = locations->InAt(1); 3247 DCHECK(second.IsConstant()); 3248 3249 int32_t imm = second.GetConstant()->AsIntConstant()->GetValue(); 3250 if (imm == 0) { 3251 // Do not generate anything. DivZeroCheck would prevent any code to be executed. 3252 } else if (imm == 1 || imm == -1) { 3253 DivRemOneOrMinusOne(instruction); 3254 } else if (IsPowerOfTwo(AbsOrMin(imm))) { 3255 DivRemByPowerOfTwo(instruction); 3256 } else { 3257 DCHECK(imm <= -2 || imm >= 2); 3258 GenerateDivRemWithAnyConstant(instruction); 3259 } 3260} 3261 3262void LocationsBuilderARM::VisitDiv(HDiv* div) { 3263 LocationSummary::CallKind call_kind = LocationSummary::kNoCall; 3264 if (div->GetResultType() == Primitive::kPrimLong) { 3265 // pLdiv runtime call. 3266 call_kind = LocationSummary::kCallOnMainOnly; 3267 } else if (div->GetResultType() == Primitive::kPrimInt && div->InputAt(1)->IsConstant()) { 3268 // sdiv will be replaced by other instruction sequence. 3269 } else if (div->GetResultType() == Primitive::kPrimInt && 3270 !codegen_->GetInstructionSetFeatures().HasDivideInstruction()) { 3271 // pIdivmod runtime call. 3272 call_kind = LocationSummary::kCallOnMainOnly; 3273 } 3274 3275 LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(div, call_kind); 3276 3277 switch (div->GetResultType()) { 3278 case Primitive::kPrimInt: { 3279 if (div->InputAt(1)->IsConstant()) { 3280 locations->SetInAt(0, Location::RequiresRegister()); 3281 locations->SetInAt(1, Location::ConstantLocation(div->InputAt(1)->AsConstant())); 3282 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 3283 int32_t value = div->InputAt(1)->AsIntConstant()->GetValue(); 3284 if (value == 1 || value == 0 || value == -1) { 3285 // No temp register required. 3286 } else { 3287 locations->AddTemp(Location::RequiresRegister()); 3288 if (!IsPowerOfTwo(AbsOrMin(value))) { 3289 locations->AddTemp(Location::RequiresRegister()); 3290 } 3291 } 3292 } else if (codegen_->GetInstructionSetFeatures().HasDivideInstruction()) { 3293 locations->SetInAt(0, Location::RequiresRegister()); 3294 locations->SetInAt(1, Location::RequiresRegister()); 3295 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 3296 } else { 3297 InvokeRuntimeCallingConvention calling_convention; 3298 locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(0))); 3299 locations->SetInAt(1, Location::RegisterLocation(calling_convention.GetRegisterAt(1))); 3300 // Note: divrem will compute both the quotient and the remainder as the pair R0 and R1, but 3301 // we only need the former. 3302 locations->SetOut(Location::RegisterLocation(R0)); 3303 } 3304 break; 3305 } 3306 case Primitive::kPrimLong: { 3307 InvokeRuntimeCallingConvention calling_convention; 3308 locations->SetInAt(0, Location::RegisterPairLocation( 3309 calling_convention.GetRegisterAt(0), calling_convention.GetRegisterAt(1))); 3310 locations->SetInAt(1, Location::RegisterPairLocation( 3311 calling_convention.GetRegisterAt(2), calling_convention.GetRegisterAt(3))); 3312 locations->SetOut(Location::RegisterPairLocation(R0, R1)); 3313 break; 3314 } 3315 case Primitive::kPrimFloat: 3316 case Primitive::kPrimDouble: { 3317 locations->SetInAt(0, Location::RequiresFpuRegister()); 3318 locations->SetInAt(1, Location::RequiresFpuRegister()); 3319 locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap); 3320 break; 3321 } 3322 3323 default: 3324 LOG(FATAL) << "Unexpected div type " << div->GetResultType(); 3325 } 3326} 3327 3328void InstructionCodeGeneratorARM::VisitDiv(HDiv* div) { 3329 LocationSummary* locations = div->GetLocations(); 3330 Location out = locations->Out(); 3331 Location first = locations->InAt(0); 3332 Location second = locations->InAt(1); 3333 3334 switch (div->GetResultType()) { 3335 case Primitive::kPrimInt: { 3336 if (second.IsConstant()) { 3337 GenerateDivRemConstantIntegral(div); 3338 } else if (codegen_->GetInstructionSetFeatures().HasDivideInstruction()) { 3339 __ sdiv(out.AsRegister<Register>(), 3340 first.AsRegister<Register>(), 3341 second.AsRegister<Register>()); 3342 } else { 3343 InvokeRuntimeCallingConvention calling_convention; 3344 DCHECK_EQ(calling_convention.GetRegisterAt(0), first.AsRegister<Register>()); 3345 DCHECK_EQ(calling_convention.GetRegisterAt(1), second.AsRegister<Register>()); 3346 DCHECK_EQ(R0, out.AsRegister<Register>()); 3347 3348 codegen_->InvokeRuntime(kQuickIdivmod, div, div->GetDexPc()); 3349 CheckEntrypointTypes<kQuickIdivmod, int32_t, int32_t, int32_t>(); 3350 } 3351 break; 3352 } 3353 3354 case Primitive::kPrimLong: { 3355 InvokeRuntimeCallingConvention calling_convention; 3356 DCHECK_EQ(calling_convention.GetRegisterAt(0), first.AsRegisterPairLow<Register>()); 3357 DCHECK_EQ(calling_convention.GetRegisterAt(1), first.AsRegisterPairHigh<Register>()); 3358 DCHECK_EQ(calling_convention.GetRegisterAt(2), second.AsRegisterPairLow<Register>()); 3359 DCHECK_EQ(calling_convention.GetRegisterAt(3), second.AsRegisterPairHigh<Register>()); 3360 DCHECK_EQ(R0, out.AsRegisterPairLow<Register>()); 3361 DCHECK_EQ(R1, out.AsRegisterPairHigh<Register>()); 3362 3363 codegen_->InvokeRuntime(kQuickLdiv, div, div->GetDexPc()); 3364 CheckEntrypointTypes<kQuickLdiv, int64_t, int64_t, int64_t>(); 3365 break; 3366 } 3367 3368 case Primitive::kPrimFloat: { 3369 __ vdivs(out.AsFpuRegister<SRegister>(), 3370 first.AsFpuRegister<SRegister>(), 3371 second.AsFpuRegister<SRegister>()); 3372 break; 3373 } 3374 3375 case Primitive::kPrimDouble: { 3376 __ vdivd(FromLowSToD(out.AsFpuRegisterPairLow<SRegister>()), 3377 FromLowSToD(first.AsFpuRegisterPairLow<SRegister>()), 3378 FromLowSToD(second.AsFpuRegisterPairLow<SRegister>())); 3379 break; 3380 } 3381 3382 default: 3383 LOG(FATAL) << "Unexpected div type " << div->GetResultType(); 3384 } 3385} 3386 3387void LocationsBuilderARM::VisitRem(HRem* rem) { 3388 Primitive::Type type = rem->GetResultType(); 3389 3390 // Most remainders are implemented in the runtime. 3391 LocationSummary::CallKind call_kind = LocationSummary::kCallOnMainOnly; 3392 if (rem->GetResultType() == Primitive::kPrimInt && rem->InputAt(1)->IsConstant()) { 3393 // sdiv will be replaced by other instruction sequence. 3394 call_kind = LocationSummary::kNoCall; 3395 } else if ((rem->GetResultType() == Primitive::kPrimInt) 3396 && codegen_->GetInstructionSetFeatures().HasDivideInstruction()) { 3397 // Have hardware divide instruction for int, do it with three instructions. 3398 call_kind = LocationSummary::kNoCall; 3399 } 3400 3401 LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(rem, call_kind); 3402 3403 switch (type) { 3404 case Primitive::kPrimInt: { 3405 if (rem->InputAt(1)->IsConstant()) { 3406 locations->SetInAt(0, Location::RequiresRegister()); 3407 locations->SetInAt(1, Location::ConstantLocation(rem->InputAt(1)->AsConstant())); 3408 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 3409 int32_t value = rem->InputAt(1)->AsIntConstant()->GetValue(); 3410 if (value == 1 || value == 0 || value == -1) { 3411 // No temp register required. 3412 } else { 3413 locations->AddTemp(Location::RequiresRegister()); 3414 if (!IsPowerOfTwo(AbsOrMin(value))) { 3415 locations->AddTemp(Location::RequiresRegister()); 3416 } 3417 } 3418 } else if (codegen_->GetInstructionSetFeatures().HasDivideInstruction()) { 3419 locations->SetInAt(0, Location::RequiresRegister()); 3420 locations->SetInAt(1, Location::RequiresRegister()); 3421 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 3422 locations->AddTemp(Location::RequiresRegister()); 3423 } else { 3424 InvokeRuntimeCallingConvention calling_convention; 3425 locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(0))); 3426 locations->SetInAt(1, Location::RegisterLocation(calling_convention.GetRegisterAt(1))); 3427 // Note: divrem will compute both the quotient and the remainder as the pair R0 and R1, but 3428 // we only need the latter. 3429 locations->SetOut(Location::RegisterLocation(R1)); 3430 } 3431 break; 3432 } 3433 case Primitive::kPrimLong: { 3434 InvokeRuntimeCallingConvention calling_convention; 3435 locations->SetInAt(0, Location::RegisterPairLocation( 3436 calling_convention.GetRegisterAt(0), calling_convention.GetRegisterAt(1))); 3437 locations->SetInAt(1, Location::RegisterPairLocation( 3438 calling_convention.GetRegisterAt(2), calling_convention.GetRegisterAt(3))); 3439 // The runtime helper puts the output in R2,R3. 3440 locations->SetOut(Location::RegisterPairLocation(R2, R3)); 3441 break; 3442 } 3443 case Primitive::kPrimFloat: { 3444 InvokeRuntimeCallingConvention calling_convention; 3445 locations->SetInAt(0, Location::FpuRegisterLocation(calling_convention.GetFpuRegisterAt(0))); 3446 locations->SetInAt(1, Location::FpuRegisterLocation(calling_convention.GetFpuRegisterAt(1))); 3447 locations->SetOut(Location::FpuRegisterLocation(S0)); 3448 break; 3449 } 3450 3451 case Primitive::kPrimDouble: { 3452 InvokeRuntimeCallingConvention calling_convention; 3453 locations->SetInAt(0, Location::FpuRegisterPairLocation( 3454 calling_convention.GetFpuRegisterAt(0), calling_convention.GetFpuRegisterAt(1))); 3455 locations->SetInAt(1, Location::FpuRegisterPairLocation( 3456 calling_convention.GetFpuRegisterAt(2), calling_convention.GetFpuRegisterAt(3))); 3457 locations->SetOut(Location::Location::FpuRegisterPairLocation(S0, S1)); 3458 break; 3459 } 3460 3461 default: 3462 LOG(FATAL) << "Unexpected rem type " << type; 3463 } 3464} 3465 3466void InstructionCodeGeneratorARM::VisitRem(HRem* rem) { 3467 LocationSummary* locations = rem->GetLocations(); 3468 Location out = locations->Out(); 3469 Location first = locations->InAt(0); 3470 Location second = locations->InAt(1); 3471 3472 Primitive::Type type = rem->GetResultType(); 3473 switch (type) { 3474 case Primitive::kPrimInt: { 3475 if (second.IsConstant()) { 3476 GenerateDivRemConstantIntegral(rem); 3477 } else if (codegen_->GetInstructionSetFeatures().HasDivideInstruction()) { 3478 Register reg1 = first.AsRegister<Register>(); 3479 Register reg2 = second.AsRegister<Register>(); 3480 Register temp = locations->GetTemp(0).AsRegister<Register>(); 3481 3482 // temp = reg1 / reg2 (integer division) 3483 // dest = reg1 - temp * reg2 3484 __ sdiv(temp, reg1, reg2); 3485 __ mls(out.AsRegister<Register>(), temp, reg2, reg1); 3486 } else { 3487 InvokeRuntimeCallingConvention calling_convention; 3488 DCHECK_EQ(calling_convention.GetRegisterAt(0), first.AsRegister<Register>()); 3489 DCHECK_EQ(calling_convention.GetRegisterAt(1), second.AsRegister<Register>()); 3490 DCHECK_EQ(R1, out.AsRegister<Register>()); 3491 3492 codegen_->InvokeRuntime(kQuickIdivmod, rem, rem->GetDexPc()); 3493 CheckEntrypointTypes<kQuickIdivmod, int32_t, int32_t, int32_t>(); 3494 } 3495 break; 3496 } 3497 3498 case Primitive::kPrimLong: { 3499 codegen_->InvokeRuntime(kQuickLmod, rem, rem->GetDexPc()); 3500 CheckEntrypointTypes<kQuickLmod, int64_t, int64_t, int64_t>(); 3501 break; 3502 } 3503 3504 case Primitive::kPrimFloat: { 3505 codegen_->InvokeRuntime(kQuickFmodf, rem, rem->GetDexPc()); 3506 CheckEntrypointTypes<kQuickFmodf, float, float, float>(); 3507 break; 3508 } 3509 3510 case Primitive::kPrimDouble: { 3511 codegen_->InvokeRuntime(kQuickFmod, rem, rem->GetDexPc()); 3512 CheckEntrypointTypes<kQuickFmod, double, double, double>(); 3513 break; 3514 } 3515 3516 default: 3517 LOG(FATAL) << "Unexpected rem type " << type; 3518 } 3519} 3520 3521void LocationsBuilderARM::VisitDivZeroCheck(HDivZeroCheck* instruction) { 3522 LocationSummary* locations = codegen_->CreateThrowingSlowPathLocations(instruction); 3523 locations->SetInAt(0, Location::RegisterOrConstant(instruction->InputAt(0))); 3524} 3525 3526void InstructionCodeGeneratorARM::VisitDivZeroCheck(HDivZeroCheck* instruction) { 3527 SlowPathCodeARM* slow_path = new (GetGraph()->GetArena()) DivZeroCheckSlowPathARM(instruction); 3528 codegen_->AddSlowPath(slow_path); 3529 3530 LocationSummary* locations = instruction->GetLocations(); 3531 Location value = locations->InAt(0); 3532 3533 switch (instruction->GetType()) { 3534 case Primitive::kPrimBoolean: 3535 case Primitive::kPrimByte: 3536 case Primitive::kPrimChar: 3537 case Primitive::kPrimShort: 3538 case Primitive::kPrimInt: { 3539 if (value.IsRegister()) { 3540 __ CompareAndBranchIfZero(value.AsRegister<Register>(), slow_path->GetEntryLabel()); 3541 } else { 3542 DCHECK(value.IsConstant()) << value; 3543 if (value.GetConstant()->AsIntConstant()->GetValue() == 0) { 3544 __ b(slow_path->GetEntryLabel()); 3545 } 3546 } 3547 break; 3548 } 3549 case Primitive::kPrimLong: { 3550 if (value.IsRegisterPair()) { 3551 __ orrs(IP, 3552 value.AsRegisterPairLow<Register>(), 3553 ShifterOperand(value.AsRegisterPairHigh<Register>())); 3554 __ b(slow_path->GetEntryLabel(), EQ); 3555 } else { 3556 DCHECK(value.IsConstant()) << value; 3557 if (value.GetConstant()->AsLongConstant()->GetValue() == 0) { 3558 __ b(slow_path->GetEntryLabel()); 3559 } 3560 } 3561 break; 3562 default: 3563 LOG(FATAL) << "Unexpected type for HDivZeroCheck " << instruction->GetType(); 3564 } 3565 } 3566} 3567 3568void InstructionCodeGeneratorARM::HandleIntegerRotate(LocationSummary* locations) { 3569 Register in = locations->InAt(0).AsRegister<Register>(); 3570 Location rhs = locations->InAt(1); 3571 Register out = locations->Out().AsRegister<Register>(); 3572 3573 if (rhs.IsConstant()) { 3574 // Arm32 and Thumb2 assemblers require a rotation on the interval [1,31], 3575 // so map all rotations to a +ve. equivalent in that range. 3576 // (e.g. left *or* right by -2 bits == 30 bits in the same direction.) 3577 uint32_t rot = CodeGenerator::GetInt32ValueOf(rhs.GetConstant()) & 0x1F; 3578 if (rot) { 3579 // Rotate, mapping left rotations to right equivalents if necessary. 3580 // (e.g. left by 2 bits == right by 30.) 3581 __ Ror(out, in, rot); 3582 } else if (out != in) { 3583 __ Mov(out, in); 3584 } 3585 } else { 3586 __ Ror(out, in, rhs.AsRegister<Register>()); 3587 } 3588} 3589 3590// Gain some speed by mapping all Long rotates onto equivalent pairs of Integer 3591// rotates by swapping input regs (effectively rotating by the first 32-bits of 3592// a larger rotation) or flipping direction (thus treating larger right/left 3593// rotations as sub-word sized rotations in the other direction) as appropriate. 3594void InstructionCodeGeneratorARM::HandleLongRotate(LocationSummary* locations) { 3595 Register in_reg_lo = locations->InAt(0).AsRegisterPairLow<Register>(); 3596 Register in_reg_hi = locations->InAt(0).AsRegisterPairHigh<Register>(); 3597 Location rhs = locations->InAt(1); 3598 Register out_reg_lo = locations->Out().AsRegisterPairLow<Register>(); 3599 Register out_reg_hi = locations->Out().AsRegisterPairHigh<Register>(); 3600 3601 if (rhs.IsConstant()) { 3602 uint64_t rot = CodeGenerator::GetInt64ValueOf(rhs.GetConstant()); 3603 // Map all rotations to +ve. equivalents on the interval [0,63]. 3604 rot &= kMaxLongShiftDistance; 3605 // For rotates over a word in size, 'pre-rotate' by 32-bits to keep rotate 3606 // logic below to a simple pair of binary orr. 3607 // (e.g. 34 bits == in_reg swap + 2 bits right.) 3608 if (rot >= kArmBitsPerWord) { 3609 rot -= kArmBitsPerWord; 3610 std::swap(in_reg_hi, in_reg_lo); 3611 } 3612 // Rotate, or mov to out for zero or word size rotations. 3613 if (rot != 0u) { 3614 __ Lsr(out_reg_hi, in_reg_hi, rot); 3615 __ orr(out_reg_hi, out_reg_hi, ShifterOperand(in_reg_lo, arm::LSL, kArmBitsPerWord - rot)); 3616 __ Lsr(out_reg_lo, in_reg_lo, rot); 3617 __ orr(out_reg_lo, out_reg_lo, ShifterOperand(in_reg_hi, arm::LSL, kArmBitsPerWord - rot)); 3618 } else { 3619 __ Mov(out_reg_lo, in_reg_lo); 3620 __ Mov(out_reg_hi, in_reg_hi); 3621 } 3622 } else { 3623 Register shift_right = locations->GetTemp(0).AsRegister<Register>(); 3624 Register shift_left = locations->GetTemp(1).AsRegister<Register>(); 3625 Label end; 3626 Label shift_by_32_plus_shift_right; 3627 3628 __ and_(shift_right, rhs.AsRegister<Register>(), ShifterOperand(0x1F)); 3629 __ Lsrs(shift_left, rhs.AsRegister<Register>(), 6); 3630 __ rsb(shift_left, shift_right, ShifterOperand(kArmBitsPerWord), AL, kCcKeep); 3631 __ b(&shift_by_32_plus_shift_right, CC); 3632 3633 // out_reg_hi = (reg_hi << shift_left) | (reg_lo >> shift_right). 3634 // out_reg_lo = (reg_lo << shift_left) | (reg_hi >> shift_right). 3635 __ Lsl(out_reg_hi, in_reg_hi, shift_left); 3636 __ Lsr(out_reg_lo, in_reg_lo, shift_right); 3637 __ add(out_reg_hi, out_reg_hi, ShifterOperand(out_reg_lo)); 3638 __ Lsl(out_reg_lo, in_reg_lo, shift_left); 3639 __ Lsr(shift_left, in_reg_hi, shift_right); 3640 __ add(out_reg_lo, out_reg_lo, ShifterOperand(shift_left)); 3641 __ b(&end); 3642 3643 __ Bind(&shift_by_32_plus_shift_right); // Shift by 32+shift_right. 3644 // out_reg_hi = (reg_hi >> shift_right) | (reg_lo << shift_left). 3645 // out_reg_lo = (reg_lo >> shift_right) | (reg_hi << shift_left). 3646 __ Lsr(out_reg_hi, in_reg_hi, shift_right); 3647 __ Lsl(out_reg_lo, in_reg_lo, shift_left); 3648 __ add(out_reg_hi, out_reg_hi, ShifterOperand(out_reg_lo)); 3649 __ Lsr(out_reg_lo, in_reg_lo, shift_right); 3650 __ Lsl(shift_right, in_reg_hi, shift_left); 3651 __ add(out_reg_lo, out_reg_lo, ShifterOperand(shift_right)); 3652 3653 __ Bind(&end); 3654 } 3655} 3656 3657void LocationsBuilderARM::VisitRor(HRor* ror) { 3658 LocationSummary* locations = 3659 new (GetGraph()->GetArena()) LocationSummary(ror, LocationSummary::kNoCall); 3660 switch (ror->GetResultType()) { 3661 case Primitive::kPrimInt: { 3662 locations->SetInAt(0, Location::RequiresRegister()); 3663 locations->SetInAt(1, Location::RegisterOrConstant(ror->InputAt(1))); 3664 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 3665 break; 3666 } 3667 case Primitive::kPrimLong: { 3668 locations->SetInAt(0, Location::RequiresRegister()); 3669 if (ror->InputAt(1)->IsConstant()) { 3670 locations->SetInAt(1, Location::ConstantLocation(ror->InputAt(1)->AsConstant())); 3671 } else { 3672 locations->SetInAt(1, Location::RequiresRegister()); 3673 locations->AddTemp(Location::RequiresRegister()); 3674 locations->AddTemp(Location::RequiresRegister()); 3675 } 3676 locations->SetOut(Location::RequiresRegister(), Location::kOutputOverlap); 3677 break; 3678 } 3679 default: 3680 LOG(FATAL) << "Unexpected operation type " << ror->GetResultType(); 3681 } 3682} 3683 3684void InstructionCodeGeneratorARM::VisitRor(HRor* ror) { 3685 LocationSummary* locations = ror->GetLocations(); 3686 Primitive::Type type = ror->GetResultType(); 3687 switch (type) { 3688 case Primitive::kPrimInt: { 3689 HandleIntegerRotate(locations); 3690 break; 3691 } 3692 case Primitive::kPrimLong: { 3693 HandleLongRotate(locations); 3694 break; 3695 } 3696 default: 3697 LOG(FATAL) << "Unexpected operation type " << type; 3698 UNREACHABLE(); 3699 } 3700} 3701 3702void LocationsBuilderARM::HandleShift(HBinaryOperation* op) { 3703 DCHECK(op->IsShl() || op->IsShr() || op->IsUShr()); 3704 3705 LocationSummary* locations = 3706 new (GetGraph()->GetArena()) LocationSummary(op, LocationSummary::kNoCall); 3707 3708 switch (op->GetResultType()) { 3709 case Primitive::kPrimInt: { 3710 locations->SetInAt(0, Location::RequiresRegister()); 3711 if (op->InputAt(1)->IsConstant()) { 3712 locations->SetInAt(1, Location::ConstantLocation(op->InputAt(1)->AsConstant())); 3713 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 3714 } else { 3715 locations->SetInAt(1, Location::RequiresRegister()); 3716 // Make the output overlap, as it will be used to hold the masked 3717 // second input. 3718 locations->SetOut(Location::RequiresRegister(), Location::kOutputOverlap); 3719 } 3720 break; 3721 } 3722 case Primitive::kPrimLong: { 3723 locations->SetInAt(0, Location::RequiresRegister()); 3724 if (op->InputAt(1)->IsConstant()) { 3725 locations->SetInAt(1, Location::ConstantLocation(op->InputAt(1)->AsConstant())); 3726 // For simplicity, use kOutputOverlap even though we only require that low registers 3727 // don't clash with high registers which the register allocator currently guarantees. 3728 locations->SetOut(Location::RequiresRegister(), Location::kOutputOverlap); 3729 } else { 3730 locations->SetInAt(1, Location::RequiresRegister()); 3731 locations->AddTemp(Location::RequiresRegister()); 3732 locations->SetOut(Location::RequiresRegister(), Location::kOutputOverlap); 3733 } 3734 break; 3735 } 3736 default: 3737 LOG(FATAL) << "Unexpected operation type " << op->GetResultType(); 3738 } 3739} 3740 3741void InstructionCodeGeneratorARM::HandleShift(HBinaryOperation* op) { 3742 DCHECK(op->IsShl() || op->IsShr() || op->IsUShr()); 3743 3744 LocationSummary* locations = op->GetLocations(); 3745 Location out = locations->Out(); 3746 Location first = locations->InAt(0); 3747 Location second = locations->InAt(1); 3748 3749 Primitive::Type type = op->GetResultType(); 3750 switch (type) { 3751 case Primitive::kPrimInt: { 3752 Register out_reg = out.AsRegister<Register>(); 3753 Register first_reg = first.AsRegister<Register>(); 3754 if (second.IsRegister()) { 3755 Register second_reg = second.AsRegister<Register>(); 3756 // ARM doesn't mask the shift count so we need to do it ourselves. 3757 __ and_(out_reg, second_reg, ShifterOperand(kMaxIntShiftDistance)); 3758 if (op->IsShl()) { 3759 __ Lsl(out_reg, first_reg, out_reg); 3760 } else if (op->IsShr()) { 3761 __ Asr(out_reg, first_reg, out_reg); 3762 } else { 3763 __ Lsr(out_reg, first_reg, out_reg); 3764 } 3765 } else { 3766 int32_t cst = second.GetConstant()->AsIntConstant()->GetValue(); 3767 uint32_t shift_value = cst & kMaxIntShiftDistance; 3768 if (shift_value == 0) { // ARM does not support shifting with 0 immediate. 3769 __ Mov(out_reg, first_reg); 3770 } else if (op->IsShl()) { 3771 __ Lsl(out_reg, first_reg, shift_value); 3772 } else if (op->IsShr()) { 3773 __ Asr(out_reg, first_reg, shift_value); 3774 } else { 3775 __ Lsr(out_reg, first_reg, shift_value); 3776 } 3777 } 3778 break; 3779 } 3780 case Primitive::kPrimLong: { 3781 Register o_h = out.AsRegisterPairHigh<Register>(); 3782 Register o_l = out.AsRegisterPairLow<Register>(); 3783 3784 Register high = first.AsRegisterPairHigh<Register>(); 3785 Register low = first.AsRegisterPairLow<Register>(); 3786 3787 if (second.IsRegister()) { 3788 Register temp = locations->GetTemp(0).AsRegister<Register>(); 3789 3790 Register second_reg = second.AsRegister<Register>(); 3791 3792 if (op->IsShl()) { 3793 __ and_(o_l, second_reg, ShifterOperand(kMaxLongShiftDistance)); 3794 // Shift the high part 3795 __ Lsl(o_h, high, o_l); 3796 // Shift the low part and `or` what overflew on the high part 3797 __ rsb(temp, o_l, ShifterOperand(kArmBitsPerWord)); 3798 __ Lsr(temp, low, temp); 3799 __ orr(o_h, o_h, ShifterOperand(temp)); 3800 // If the shift is > 32 bits, override the high part 3801 __ subs(temp, o_l, ShifterOperand(kArmBitsPerWord)); 3802 __ it(PL); 3803 __ Lsl(o_h, low, temp, PL); 3804 // Shift the low part 3805 __ Lsl(o_l, low, o_l); 3806 } else if (op->IsShr()) { 3807 __ and_(o_h, second_reg, ShifterOperand(kMaxLongShiftDistance)); 3808 // Shift the low part 3809 __ Lsr(o_l, low, o_h); 3810 // Shift the high part and `or` what underflew on the low part 3811 __ rsb(temp, o_h, ShifterOperand(kArmBitsPerWord)); 3812 __ Lsl(temp, high, temp); 3813 __ orr(o_l, o_l, ShifterOperand(temp)); 3814 // If the shift is > 32 bits, override the low part 3815 __ subs(temp, o_h, ShifterOperand(kArmBitsPerWord)); 3816 __ it(PL); 3817 __ Asr(o_l, high, temp, PL); 3818 // Shift the high part 3819 __ Asr(o_h, high, o_h); 3820 } else { 3821 __ and_(o_h, second_reg, ShifterOperand(kMaxLongShiftDistance)); 3822 // same as Shr except we use `Lsr`s and not `Asr`s 3823 __ Lsr(o_l, low, o_h); 3824 __ rsb(temp, o_h, ShifterOperand(kArmBitsPerWord)); 3825 __ Lsl(temp, high, temp); 3826 __ orr(o_l, o_l, ShifterOperand(temp)); 3827 __ subs(temp, o_h, ShifterOperand(kArmBitsPerWord)); 3828 __ it(PL); 3829 __ Lsr(o_l, high, temp, PL); 3830 __ Lsr(o_h, high, o_h); 3831 } 3832 } else { 3833 // Register allocator doesn't create partial overlap. 3834 DCHECK_NE(o_l, high); 3835 DCHECK_NE(o_h, low); 3836 int32_t cst = second.GetConstant()->AsIntConstant()->GetValue(); 3837 uint32_t shift_value = cst & kMaxLongShiftDistance; 3838 if (shift_value > 32) { 3839 if (op->IsShl()) { 3840 __ Lsl(o_h, low, shift_value - 32); 3841 __ LoadImmediate(o_l, 0); 3842 } else if (op->IsShr()) { 3843 __ Asr(o_l, high, shift_value - 32); 3844 __ Asr(o_h, high, 31); 3845 } else { 3846 __ Lsr(o_l, high, shift_value - 32); 3847 __ LoadImmediate(o_h, 0); 3848 } 3849 } else if (shift_value == 32) { 3850 if (op->IsShl()) { 3851 __ mov(o_h, ShifterOperand(low)); 3852 __ LoadImmediate(o_l, 0); 3853 } else if (op->IsShr()) { 3854 __ mov(o_l, ShifterOperand(high)); 3855 __ Asr(o_h, high, 31); 3856 } else { 3857 __ mov(o_l, ShifterOperand(high)); 3858 __ LoadImmediate(o_h, 0); 3859 } 3860 } else if (shift_value == 1) { 3861 if (op->IsShl()) { 3862 __ Lsls(o_l, low, 1); 3863 __ adc(o_h, high, ShifterOperand(high)); 3864 } else if (op->IsShr()) { 3865 __ Asrs(o_h, high, 1); 3866 __ Rrx(o_l, low); 3867 } else { 3868 __ Lsrs(o_h, high, 1); 3869 __ Rrx(o_l, low); 3870 } 3871 } else { 3872 DCHECK(2 <= shift_value && shift_value < 32) << shift_value; 3873 if (op->IsShl()) { 3874 __ Lsl(o_h, high, shift_value); 3875 __ orr(o_h, o_h, ShifterOperand(low, LSR, 32 - shift_value)); 3876 __ Lsl(o_l, low, shift_value); 3877 } else if (op->IsShr()) { 3878 __ Lsr(o_l, low, shift_value); 3879 __ orr(o_l, o_l, ShifterOperand(high, LSL, 32 - shift_value)); 3880 __ Asr(o_h, high, shift_value); 3881 } else { 3882 __ Lsr(o_l, low, shift_value); 3883 __ orr(o_l, o_l, ShifterOperand(high, LSL, 32 - shift_value)); 3884 __ Lsr(o_h, high, shift_value); 3885 } 3886 } 3887 } 3888 break; 3889 } 3890 default: 3891 LOG(FATAL) << "Unexpected operation type " << type; 3892 UNREACHABLE(); 3893 } 3894} 3895 3896void LocationsBuilderARM::VisitShl(HShl* shl) { 3897 HandleShift(shl); 3898} 3899 3900void InstructionCodeGeneratorARM::VisitShl(HShl* shl) { 3901 HandleShift(shl); 3902} 3903 3904void LocationsBuilderARM::VisitShr(HShr* shr) { 3905 HandleShift(shr); 3906} 3907 3908void InstructionCodeGeneratorARM::VisitShr(HShr* shr) { 3909 HandleShift(shr); 3910} 3911 3912void LocationsBuilderARM::VisitUShr(HUShr* ushr) { 3913 HandleShift(ushr); 3914} 3915 3916void InstructionCodeGeneratorARM::VisitUShr(HUShr* ushr) { 3917 HandleShift(ushr); 3918} 3919 3920void LocationsBuilderARM::VisitNewInstance(HNewInstance* instruction) { 3921 LocationSummary* locations = 3922 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCallOnMainOnly); 3923 if (instruction->IsStringAlloc()) { 3924 locations->AddTemp(Location::RegisterLocation(kMethodRegisterArgument)); 3925 } else { 3926 InvokeRuntimeCallingConvention calling_convention; 3927 locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(0))); 3928 locations->SetInAt(1, Location::RegisterLocation(calling_convention.GetRegisterAt(1))); 3929 } 3930 locations->SetOut(Location::RegisterLocation(R0)); 3931} 3932 3933void InstructionCodeGeneratorARM::VisitNewInstance(HNewInstance* instruction) { 3934 // Note: if heap poisoning is enabled, the entry point takes cares 3935 // of poisoning the reference. 3936 if (instruction->IsStringAlloc()) { 3937 // String is allocated through StringFactory. Call NewEmptyString entry point. 3938 Register temp = instruction->GetLocations()->GetTemp(0).AsRegister<Register>(); 3939 MemberOffset code_offset = ArtMethod::EntryPointFromQuickCompiledCodeOffset(kArmPointerSize); 3940 __ LoadFromOffset(kLoadWord, temp, TR, QUICK_ENTRY_POINT(pNewEmptyString)); 3941 __ LoadFromOffset(kLoadWord, LR, temp, code_offset.Int32Value()); 3942 __ blx(LR); 3943 codegen_->RecordPcInfo(instruction, instruction->GetDexPc()); 3944 } else { 3945 codegen_->InvokeRuntime(instruction->GetEntrypoint(), instruction, instruction->GetDexPc()); 3946 CheckEntrypointTypes<kQuickAllocObjectWithAccessCheck, void*, uint32_t, ArtMethod*>(); 3947 } 3948} 3949 3950void LocationsBuilderARM::VisitNewArray(HNewArray* instruction) { 3951 LocationSummary* locations = 3952 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCallOnMainOnly); 3953 InvokeRuntimeCallingConvention calling_convention; 3954 locations->AddTemp(Location::RegisterLocation(calling_convention.GetRegisterAt(0))); 3955 locations->SetOut(Location::RegisterLocation(R0)); 3956 locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(1))); 3957 locations->SetInAt(1, Location::RegisterLocation(calling_convention.GetRegisterAt(2))); 3958} 3959 3960void InstructionCodeGeneratorARM::VisitNewArray(HNewArray* instruction) { 3961 InvokeRuntimeCallingConvention calling_convention; 3962 __ LoadImmediate(calling_convention.GetRegisterAt(0), instruction->GetTypeIndex()); 3963 // Note: if heap poisoning is enabled, the entry point takes cares 3964 // of poisoning the reference. 3965 codegen_->InvokeRuntime(instruction->GetEntrypoint(), instruction, instruction->GetDexPc()); 3966 CheckEntrypointTypes<kQuickAllocArrayWithAccessCheck, void*, uint32_t, int32_t, ArtMethod*>(); 3967} 3968 3969void LocationsBuilderARM::VisitParameterValue(HParameterValue* instruction) { 3970 LocationSummary* locations = 3971 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall); 3972 Location location = parameter_visitor_.GetNextLocation(instruction->GetType()); 3973 if (location.IsStackSlot()) { 3974 location = Location::StackSlot(location.GetStackIndex() + codegen_->GetFrameSize()); 3975 } else if (location.IsDoubleStackSlot()) { 3976 location = Location::DoubleStackSlot(location.GetStackIndex() + codegen_->GetFrameSize()); 3977 } 3978 locations->SetOut(location); 3979} 3980 3981void InstructionCodeGeneratorARM::VisitParameterValue( 3982 HParameterValue* instruction ATTRIBUTE_UNUSED) { 3983 // Nothing to do, the parameter is already at its location. 3984} 3985 3986void LocationsBuilderARM::VisitCurrentMethod(HCurrentMethod* instruction) { 3987 LocationSummary* locations = 3988 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall); 3989 locations->SetOut(Location::RegisterLocation(kMethodRegisterArgument)); 3990} 3991 3992void InstructionCodeGeneratorARM::VisitCurrentMethod(HCurrentMethod* instruction ATTRIBUTE_UNUSED) { 3993 // Nothing to do, the method is already at its location. 3994} 3995 3996void LocationsBuilderARM::VisitNot(HNot* not_) { 3997 LocationSummary* locations = 3998 new (GetGraph()->GetArena()) LocationSummary(not_, LocationSummary::kNoCall); 3999 locations->SetInAt(0, Location::RequiresRegister()); 4000 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 4001} 4002 4003void InstructionCodeGeneratorARM::VisitNot(HNot* not_) { 4004 LocationSummary* locations = not_->GetLocations(); 4005 Location out = locations->Out(); 4006 Location in = locations->InAt(0); 4007 switch (not_->GetResultType()) { 4008 case Primitive::kPrimInt: 4009 __ mvn(out.AsRegister<Register>(), ShifterOperand(in.AsRegister<Register>())); 4010 break; 4011 4012 case Primitive::kPrimLong: 4013 __ mvn(out.AsRegisterPairLow<Register>(), 4014 ShifterOperand(in.AsRegisterPairLow<Register>())); 4015 __ mvn(out.AsRegisterPairHigh<Register>(), 4016 ShifterOperand(in.AsRegisterPairHigh<Register>())); 4017 break; 4018 4019 default: 4020 LOG(FATAL) << "Unimplemented type for not operation " << not_->GetResultType(); 4021 } 4022} 4023 4024void LocationsBuilderARM::VisitBooleanNot(HBooleanNot* bool_not) { 4025 LocationSummary* locations = 4026 new (GetGraph()->GetArena()) LocationSummary(bool_not, LocationSummary::kNoCall); 4027 locations->SetInAt(0, Location::RequiresRegister()); 4028 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 4029} 4030 4031void InstructionCodeGeneratorARM::VisitBooleanNot(HBooleanNot* bool_not) { 4032 LocationSummary* locations = bool_not->GetLocations(); 4033 Location out = locations->Out(); 4034 Location in = locations->InAt(0); 4035 __ eor(out.AsRegister<Register>(), in.AsRegister<Register>(), ShifterOperand(1)); 4036} 4037 4038void LocationsBuilderARM::VisitCompare(HCompare* compare) { 4039 LocationSummary* locations = 4040 new (GetGraph()->GetArena()) LocationSummary(compare, LocationSummary::kNoCall); 4041 switch (compare->InputAt(0)->GetType()) { 4042 case Primitive::kPrimBoolean: 4043 case Primitive::kPrimByte: 4044 case Primitive::kPrimShort: 4045 case Primitive::kPrimChar: 4046 case Primitive::kPrimInt: 4047 case Primitive::kPrimLong: { 4048 locations->SetInAt(0, Location::RequiresRegister()); 4049 locations->SetInAt(1, Location::RequiresRegister()); 4050 // Output overlaps because it is written before doing the low comparison. 4051 locations->SetOut(Location::RequiresRegister(), Location::kOutputOverlap); 4052 break; 4053 } 4054 case Primitive::kPrimFloat: 4055 case Primitive::kPrimDouble: { 4056 locations->SetInAt(0, Location::RequiresFpuRegister()); 4057 locations->SetInAt(1, ArithmeticZeroOrFpuRegister(compare->InputAt(1))); 4058 locations->SetOut(Location::RequiresRegister()); 4059 break; 4060 } 4061 default: 4062 LOG(FATAL) << "Unexpected type for compare operation " << compare->InputAt(0)->GetType(); 4063 } 4064} 4065 4066void InstructionCodeGeneratorARM::VisitCompare(HCompare* compare) { 4067 LocationSummary* locations = compare->GetLocations(); 4068 Register out = locations->Out().AsRegister<Register>(); 4069 Location left = locations->InAt(0); 4070 Location right = locations->InAt(1); 4071 4072 Label less, greater, done; 4073 Primitive::Type type = compare->InputAt(0)->GetType(); 4074 Condition less_cond; 4075 switch (type) { 4076 case Primitive::kPrimBoolean: 4077 case Primitive::kPrimByte: 4078 case Primitive::kPrimShort: 4079 case Primitive::kPrimChar: 4080 case Primitive::kPrimInt: { 4081 __ LoadImmediate(out, 0); 4082 __ cmp(left.AsRegister<Register>(), 4083 ShifterOperand(right.AsRegister<Register>())); // Signed compare. 4084 less_cond = LT; 4085 break; 4086 } 4087 case Primitive::kPrimLong: { 4088 __ cmp(left.AsRegisterPairHigh<Register>(), 4089 ShifterOperand(right.AsRegisterPairHigh<Register>())); // Signed compare. 4090 __ b(&less, LT); 4091 __ b(&greater, GT); 4092 // Do LoadImmediate before the last `cmp`, as LoadImmediate might affect the status flags. 4093 __ LoadImmediate(out, 0); 4094 __ cmp(left.AsRegisterPairLow<Register>(), 4095 ShifterOperand(right.AsRegisterPairLow<Register>())); // Unsigned compare. 4096 less_cond = LO; 4097 break; 4098 } 4099 case Primitive::kPrimFloat: 4100 case Primitive::kPrimDouble: { 4101 __ LoadImmediate(out, 0); 4102 GenerateVcmp(compare); 4103 __ vmstat(); // transfer FP status register to ARM APSR. 4104 less_cond = ARMFPCondition(kCondLT, compare->IsGtBias()); 4105 break; 4106 } 4107 default: 4108 LOG(FATAL) << "Unexpected compare type " << type; 4109 UNREACHABLE(); 4110 } 4111 4112 __ b(&done, EQ); 4113 __ b(&less, less_cond); 4114 4115 __ Bind(&greater); 4116 __ LoadImmediate(out, 1); 4117 __ b(&done); 4118 4119 __ Bind(&less); 4120 __ LoadImmediate(out, -1); 4121 4122 __ Bind(&done); 4123} 4124 4125void LocationsBuilderARM::VisitPhi(HPhi* instruction) { 4126 LocationSummary* locations = 4127 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall); 4128 for (size_t i = 0, e = locations->GetInputCount(); i < e; ++i) { 4129 locations->SetInAt(i, Location::Any()); 4130 } 4131 locations->SetOut(Location::Any()); 4132} 4133 4134void InstructionCodeGeneratorARM::VisitPhi(HPhi* instruction ATTRIBUTE_UNUSED) { 4135 LOG(FATAL) << "Unreachable"; 4136} 4137 4138void CodeGeneratorARM::GenerateMemoryBarrier(MemBarrierKind kind) { 4139 // TODO (ported from quick): revisit ARM barrier kinds. 4140 DmbOptions flavor = DmbOptions::ISH; // Quiet C++ warnings. 4141 switch (kind) { 4142 case MemBarrierKind::kAnyStore: 4143 case MemBarrierKind::kLoadAny: 4144 case MemBarrierKind::kAnyAny: { 4145 flavor = DmbOptions::ISH; 4146 break; 4147 } 4148 case MemBarrierKind::kStoreStore: { 4149 flavor = DmbOptions::ISHST; 4150 break; 4151 } 4152 default: 4153 LOG(FATAL) << "Unexpected memory barrier " << kind; 4154 } 4155 __ dmb(flavor); 4156} 4157 4158void InstructionCodeGeneratorARM::GenerateWideAtomicLoad(Register addr, 4159 uint32_t offset, 4160 Register out_lo, 4161 Register out_hi) { 4162 if (offset != 0) { 4163 // Ensure `out_lo` is different from `addr`, so that loading 4164 // `offset` into `out_lo` does not clutter `addr`. 4165 DCHECK_NE(out_lo, addr); 4166 __ LoadImmediate(out_lo, offset); 4167 __ add(IP, addr, ShifterOperand(out_lo)); 4168 addr = IP; 4169 } 4170 __ ldrexd(out_lo, out_hi, addr); 4171} 4172 4173void InstructionCodeGeneratorARM::GenerateWideAtomicStore(Register addr, 4174 uint32_t offset, 4175 Register value_lo, 4176 Register value_hi, 4177 Register temp1, 4178 Register temp2, 4179 HInstruction* instruction) { 4180 Label fail; 4181 if (offset != 0) { 4182 __ LoadImmediate(temp1, offset); 4183 __ add(IP, addr, ShifterOperand(temp1)); 4184 addr = IP; 4185 } 4186 __ Bind(&fail); 4187 // We need a load followed by store. (The address used in a STREX instruction must 4188 // be the same as the address in the most recently executed LDREX instruction.) 4189 __ ldrexd(temp1, temp2, addr); 4190 codegen_->MaybeRecordImplicitNullCheck(instruction); 4191 __ strexd(temp1, value_lo, value_hi, addr); 4192 __ CompareAndBranchIfNonZero(temp1, &fail); 4193} 4194 4195void LocationsBuilderARM::HandleFieldSet(HInstruction* instruction, const FieldInfo& field_info) { 4196 DCHECK(instruction->IsInstanceFieldSet() || instruction->IsStaticFieldSet()); 4197 4198 LocationSummary* locations = 4199 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall); 4200 locations->SetInAt(0, Location::RequiresRegister()); 4201 4202 Primitive::Type field_type = field_info.GetFieldType(); 4203 if (Primitive::IsFloatingPointType(field_type)) { 4204 locations->SetInAt(1, Location::RequiresFpuRegister()); 4205 } else { 4206 locations->SetInAt(1, Location::RequiresRegister()); 4207 } 4208 4209 bool is_wide = field_type == Primitive::kPrimLong || field_type == Primitive::kPrimDouble; 4210 bool generate_volatile = field_info.IsVolatile() 4211 && is_wide 4212 && !codegen_->GetInstructionSetFeatures().HasAtomicLdrdAndStrd(); 4213 bool needs_write_barrier = 4214 CodeGenerator::StoreNeedsWriteBarrier(field_type, instruction->InputAt(1)); 4215 // Temporary registers for the write barrier. 4216 // TODO: consider renaming StoreNeedsWriteBarrier to StoreNeedsGCMark. 4217 if (needs_write_barrier) { 4218 locations->AddTemp(Location::RequiresRegister()); // Possibly used for reference poisoning too. 4219 locations->AddTemp(Location::RequiresRegister()); 4220 } else if (generate_volatile) { 4221 // ARM encoding have some additional constraints for ldrexd/strexd: 4222 // - registers need to be consecutive 4223 // - the first register should be even but not R14. 4224 // We don't test for ARM yet, and the assertion makes sure that we 4225 // revisit this if we ever enable ARM encoding. 4226 DCHECK_EQ(InstructionSet::kThumb2, codegen_->GetInstructionSet()); 4227 4228 locations->AddTemp(Location::RequiresRegister()); 4229 locations->AddTemp(Location::RequiresRegister()); 4230 if (field_type == Primitive::kPrimDouble) { 4231 // For doubles we need two more registers to copy the value. 4232 locations->AddTemp(Location::RegisterLocation(R2)); 4233 locations->AddTemp(Location::RegisterLocation(R3)); 4234 } 4235 } 4236} 4237 4238void InstructionCodeGeneratorARM::HandleFieldSet(HInstruction* instruction, 4239 const FieldInfo& field_info, 4240 bool value_can_be_null) { 4241 DCHECK(instruction->IsInstanceFieldSet() || instruction->IsStaticFieldSet()); 4242 4243 LocationSummary* locations = instruction->GetLocations(); 4244 Register base = locations->InAt(0).AsRegister<Register>(); 4245 Location value = locations->InAt(1); 4246 4247 bool is_volatile = field_info.IsVolatile(); 4248 bool atomic_ldrd_strd = codegen_->GetInstructionSetFeatures().HasAtomicLdrdAndStrd(); 4249 Primitive::Type field_type = field_info.GetFieldType(); 4250 uint32_t offset = field_info.GetFieldOffset().Uint32Value(); 4251 bool needs_write_barrier = 4252 CodeGenerator::StoreNeedsWriteBarrier(field_type, instruction->InputAt(1)); 4253 4254 if (is_volatile) { 4255 codegen_->GenerateMemoryBarrier(MemBarrierKind::kAnyStore); 4256 } 4257 4258 switch (field_type) { 4259 case Primitive::kPrimBoolean: 4260 case Primitive::kPrimByte: { 4261 __ StoreToOffset(kStoreByte, value.AsRegister<Register>(), base, offset); 4262 break; 4263 } 4264 4265 case Primitive::kPrimShort: 4266 case Primitive::kPrimChar: { 4267 __ StoreToOffset(kStoreHalfword, value.AsRegister<Register>(), base, offset); 4268 break; 4269 } 4270 4271 case Primitive::kPrimInt: 4272 case Primitive::kPrimNot: { 4273 if (kPoisonHeapReferences && needs_write_barrier) { 4274 // Note that in the case where `value` is a null reference, 4275 // we do not enter this block, as a null reference does not 4276 // need poisoning. 4277 DCHECK_EQ(field_type, Primitive::kPrimNot); 4278 Register temp = locations->GetTemp(0).AsRegister<Register>(); 4279 __ Mov(temp, value.AsRegister<Register>()); 4280 __ PoisonHeapReference(temp); 4281 __ StoreToOffset(kStoreWord, temp, base, offset); 4282 } else { 4283 __ StoreToOffset(kStoreWord, value.AsRegister<Register>(), base, offset); 4284 } 4285 break; 4286 } 4287 4288 case Primitive::kPrimLong: { 4289 if (is_volatile && !atomic_ldrd_strd) { 4290 GenerateWideAtomicStore(base, offset, 4291 value.AsRegisterPairLow<Register>(), 4292 value.AsRegisterPairHigh<Register>(), 4293 locations->GetTemp(0).AsRegister<Register>(), 4294 locations->GetTemp(1).AsRegister<Register>(), 4295 instruction); 4296 } else { 4297 __ StoreToOffset(kStoreWordPair, value.AsRegisterPairLow<Register>(), base, offset); 4298 codegen_->MaybeRecordImplicitNullCheck(instruction); 4299 } 4300 break; 4301 } 4302 4303 case Primitive::kPrimFloat: { 4304 __ StoreSToOffset(value.AsFpuRegister<SRegister>(), base, offset); 4305 break; 4306 } 4307 4308 case Primitive::kPrimDouble: { 4309 DRegister value_reg = FromLowSToD(value.AsFpuRegisterPairLow<SRegister>()); 4310 if (is_volatile && !atomic_ldrd_strd) { 4311 Register value_reg_lo = locations->GetTemp(0).AsRegister<Register>(); 4312 Register value_reg_hi = locations->GetTemp(1).AsRegister<Register>(); 4313 4314 __ vmovrrd(value_reg_lo, value_reg_hi, value_reg); 4315 4316 GenerateWideAtomicStore(base, offset, 4317 value_reg_lo, 4318 value_reg_hi, 4319 locations->GetTemp(2).AsRegister<Register>(), 4320 locations->GetTemp(3).AsRegister<Register>(), 4321 instruction); 4322 } else { 4323 __ StoreDToOffset(value_reg, base, offset); 4324 codegen_->MaybeRecordImplicitNullCheck(instruction); 4325 } 4326 break; 4327 } 4328 4329 case Primitive::kPrimVoid: 4330 LOG(FATAL) << "Unreachable type " << field_type; 4331 UNREACHABLE(); 4332 } 4333 4334 // Longs and doubles are handled in the switch. 4335 if (field_type != Primitive::kPrimLong && field_type != Primitive::kPrimDouble) { 4336 codegen_->MaybeRecordImplicitNullCheck(instruction); 4337 } 4338 4339 if (CodeGenerator::StoreNeedsWriteBarrier(field_type, instruction->InputAt(1))) { 4340 Register temp = locations->GetTemp(0).AsRegister<Register>(); 4341 Register card = locations->GetTemp(1).AsRegister<Register>(); 4342 codegen_->MarkGCCard( 4343 temp, card, base, value.AsRegister<Register>(), value_can_be_null); 4344 } 4345 4346 if (is_volatile) { 4347 codegen_->GenerateMemoryBarrier(MemBarrierKind::kAnyAny); 4348 } 4349} 4350 4351void LocationsBuilderARM::HandleFieldGet(HInstruction* instruction, const FieldInfo& field_info) { 4352 DCHECK(instruction->IsInstanceFieldGet() || instruction->IsStaticFieldGet()); 4353 4354 bool object_field_get_with_read_barrier = 4355 kEmitCompilerReadBarrier && (field_info.GetFieldType() == Primitive::kPrimNot); 4356 LocationSummary* locations = 4357 new (GetGraph()->GetArena()) LocationSummary(instruction, 4358 object_field_get_with_read_barrier ? 4359 LocationSummary::kCallOnSlowPath : 4360 LocationSummary::kNoCall); 4361 if (object_field_get_with_read_barrier && kUseBakerReadBarrier) { 4362 locations->SetCustomSlowPathCallerSaves(RegisterSet::Empty()); // No caller-save registers. 4363 } 4364 locations->SetInAt(0, Location::RequiresRegister()); 4365 4366 bool volatile_for_double = field_info.IsVolatile() 4367 && (field_info.GetFieldType() == Primitive::kPrimDouble) 4368 && !codegen_->GetInstructionSetFeatures().HasAtomicLdrdAndStrd(); 4369 // The output overlaps in case of volatile long: we don't want the 4370 // code generated by GenerateWideAtomicLoad to overwrite the 4371 // object's location. Likewise, in the case of an object field get 4372 // with read barriers enabled, we do not want the load to overwrite 4373 // the object's location, as we need it to emit the read barrier. 4374 bool overlap = (field_info.IsVolatile() && (field_info.GetFieldType() == Primitive::kPrimLong)) || 4375 object_field_get_with_read_barrier; 4376 4377 if (Primitive::IsFloatingPointType(instruction->GetType())) { 4378 locations->SetOut(Location::RequiresFpuRegister()); 4379 } else { 4380 locations->SetOut(Location::RequiresRegister(), 4381 (overlap ? Location::kOutputOverlap : Location::kNoOutputOverlap)); 4382 } 4383 if (volatile_for_double) { 4384 // ARM encoding have some additional constraints for ldrexd/strexd: 4385 // - registers need to be consecutive 4386 // - the first register should be even but not R14. 4387 // We don't test for ARM yet, and the assertion makes sure that we 4388 // revisit this if we ever enable ARM encoding. 4389 DCHECK_EQ(InstructionSet::kThumb2, codegen_->GetInstructionSet()); 4390 locations->AddTemp(Location::RequiresRegister()); 4391 locations->AddTemp(Location::RequiresRegister()); 4392 } else if (object_field_get_with_read_barrier && kUseBakerReadBarrier) { 4393 // We need a temporary register for the read barrier marking slow 4394 // path in CodeGeneratorARM::GenerateFieldLoadWithBakerReadBarrier. 4395 locations->AddTemp(Location::RequiresRegister()); 4396 } 4397} 4398 4399Location LocationsBuilderARM::ArithmeticZeroOrFpuRegister(HInstruction* input) { 4400 DCHECK(input->GetType() == Primitive::kPrimDouble || input->GetType() == Primitive::kPrimFloat) 4401 << input->GetType(); 4402 if ((input->IsFloatConstant() && (input->AsFloatConstant()->IsArithmeticZero())) || 4403 (input->IsDoubleConstant() && (input->AsDoubleConstant()->IsArithmeticZero()))) { 4404 return Location::ConstantLocation(input->AsConstant()); 4405 } else { 4406 return Location::RequiresFpuRegister(); 4407 } 4408} 4409 4410Location LocationsBuilderARM::ArmEncodableConstantOrRegister(HInstruction* constant, 4411 Opcode opcode) { 4412 DCHECK(!Primitive::IsFloatingPointType(constant->GetType())); 4413 if (constant->IsConstant() && 4414 CanEncodeConstantAsImmediate(constant->AsConstant(), opcode)) { 4415 return Location::ConstantLocation(constant->AsConstant()); 4416 } 4417 return Location::RequiresRegister(); 4418} 4419 4420bool LocationsBuilderARM::CanEncodeConstantAsImmediate(HConstant* input_cst, 4421 Opcode opcode) { 4422 uint64_t value = static_cast<uint64_t>(Int64FromConstant(input_cst)); 4423 if (Primitive::Is64BitType(input_cst->GetType())) { 4424 Opcode high_opcode = opcode; 4425 SetCc low_set_cc = kCcDontCare; 4426 switch (opcode) { 4427 case SUB: 4428 // Flip the operation to an ADD. 4429 value = -value; 4430 opcode = ADD; 4431 FALLTHROUGH_INTENDED; 4432 case ADD: 4433 if (Low32Bits(value) == 0u) { 4434 return CanEncodeConstantAsImmediate(High32Bits(value), opcode, kCcDontCare); 4435 } 4436 high_opcode = ADC; 4437 low_set_cc = kCcSet; 4438 break; 4439 default: 4440 break; 4441 } 4442 return CanEncodeConstantAsImmediate(Low32Bits(value), opcode, low_set_cc) && 4443 CanEncodeConstantAsImmediate(High32Bits(value), high_opcode, kCcDontCare); 4444 } else { 4445 return CanEncodeConstantAsImmediate(Low32Bits(value), opcode); 4446 } 4447} 4448 4449bool LocationsBuilderARM::CanEncodeConstantAsImmediate(uint32_t value, 4450 Opcode opcode, 4451 SetCc set_cc) { 4452 ShifterOperand so; 4453 ArmAssembler* assembler = codegen_->GetAssembler(); 4454 if (assembler->ShifterOperandCanHold(kNoRegister, kNoRegister, opcode, value, set_cc, &so)) { 4455 return true; 4456 } 4457 Opcode neg_opcode = kNoOperand; 4458 switch (opcode) { 4459 case AND: neg_opcode = BIC; value = ~value; break; 4460 case ORR: neg_opcode = ORN; value = ~value; break; 4461 case ADD: neg_opcode = SUB; value = -value; break; 4462 case ADC: neg_opcode = SBC; value = ~value; break; 4463 case SUB: neg_opcode = ADD; value = -value; break; 4464 case SBC: neg_opcode = ADC; value = ~value; break; 4465 default: 4466 return false; 4467 } 4468 return assembler->ShifterOperandCanHold(kNoRegister, kNoRegister, neg_opcode, value, set_cc, &so); 4469} 4470 4471void InstructionCodeGeneratorARM::HandleFieldGet(HInstruction* instruction, 4472 const FieldInfo& field_info) { 4473 DCHECK(instruction->IsInstanceFieldGet() || instruction->IsStaticFieldGet()); 4474 4475 LocationSummary* locations = instruction->GetLocations(); 4476 Location base_loc = locations->InAt(0); 4477 Register base = base_loc.AsRegister<Register>(); 4478 Location out = locations->Out(); 4479 bool is_volatile = field_info.IsVolatile(); 4480 bool atomic_ldrd_strd = codegen_->GetInstructionSetFeatures().HasAtomicLdrdAndStrd(); 4481 Primitive::Type field_type = field_info.GetFieldType(); 4482 uint32_t offset = field_info.GetFieldOffset().Uint32Value(); 4483 4484 switch (field_type) { 4485 case Primitive::kPrimBoolean: 4486 __ LoadFromOffset(kLoadUnsignedByte, out.AsRegister<Register>(), base, offset); 4487 break; 4488 4489 case Primitive::kPrimByte: 4490 __ LoadFromOffset(kLoadSignedByte, out.AsRegister<Register>(), base, offset); 4491 break; 4492 4493 case Primitive::kPrimShort: 4494 __ LoadFromOffset(kLoadSignedHalfword, out.AsRegister<Register>(), base, offset); 4495 break; 4496 4497 case Primitive::kPrimChar: 4498 __ LoadFromOffset(kLoadUnsignedHalfword, out.AsRegister<Register>(), base, offset); 4499 break; 4500 4501 case Primitive::kPrimInt: 4502 __ LoadFromOffset(kLoadWord, out.AsRegister<Register>(), base, offset); 4503 break; 4504 4505 case Primitive::kPrimNot: { 4506 // /* HeapReference<Object> */ out = *(base + offset) 4507 if (kEmitCompilerReadBarrier && kUseBakerReadBarrier) { 4508 Location temp_loc = locations->GetTemp(0); 4509 // Note that a potential implicit null check is handled in this 4510 // CodeGeneratorARM::GenerateFieldLoadWithBakerReadBarrier call. 4511 codegen_->GenerateFieldLoadWithBakerReadBarrier( 4512 instruction, out, base, offset, temp_loc, /* needs_null_check */ true); 4513 if (is_volatile) { 4514 codegen_->GenerateMemoryBarrier(MemBarrierKind::kLoadAny); 4515 } 4516 } else { 4517 __ LoadFromOffset(kLoadWord, out.AsRegister<Register>(), base, offset); 4518 codegen_->MaybeRecordImplicitNullCheck(instruction); 4519 if (is_volatile) { 4520 codegen_->GenerateMemoryBarrier(MemBarrierKind::kLoadAny); 4521 } 4522 // If read barriers are enabled, emit read barriers other than 4523 // Baker's using a slow path (and also unpoison the loaded 4524 // reference, if heap poisoning is enabled). 4525 codegen_->MaybeGenerateReadBarrierSlow(instruction, out, out, base_loc, offset); 4526 } 4527 break; 4528 } 4529 4530 case Primitive::kPrimLong: 4531 if (is_volatile && !atomic_ldrd_strd) { 4532 GenerateWideAtomicLoad(base, offset, 4533 out.AsRegisterPairLow<Register>(), 4534 out.AsRegisterPairHigh<Register>()); 4535 } else { 4536 __ LoadFromOffset(kLoadWordPair, out.AsRegisterPairLow<Register>(), base, offset); 4537 } 4538 break; 4539 4540 case Primitive::kPrimFloat: 4541 __ LoadSFromOffset(out.AsFpuRegister<SRegister>(), base, offset); 4542 break; 4543 4544 case Primitive::kPrimDouble: { 4545 DRegister out_reg = FromLowSToD(out.AsFpuRegisterPairLow<SRegister>()); 4546 if (is_volatile && !atomic_ldrd_strd) { 4547 Register lo = locations->GetTemp(0).AsRegister<Register>(); 4548 Register hi = locations->GetTemp(1).AsRegister<Register>(); 4549 GenerateWideAtomicLoad(base, offset, lo, hi); 4550 codegen_->MaybeRecordImplicitNullCheck(instruction); 4551 __ vmovdrr(out_reg, lo, hi); 4552 } else { 4553 __ LoadDFromOffset(out_reg, base, offset); 4554 codegen_->MaybeRecordImplicitNullCheck(instruction); 4555 } 4556 break; 4557 } 4558 4559 case Primitive::kPrimVoid: 4560 LOG(FATAL) << "Unreachable type " << field_type; 4561 UNREACHABLE(); 4562 } 4563 4564 if (field_type == Primitive::kPrimNot || field_type == Primitive::kPrimDouble) { 4565 // Potential implicit null checks, in the case of reference or 4566 // double fields, are handled in the previous switch statement. 4567 } else { 4568 codegen_->MaybeRecordImplicitNullCheck(instruction); 4569 } 4570 4571 if (is_volatile) { 4572 if (field_type == Primitive::kPrimNot) { 4573 // Memory barriers, in the case of references, are also handled 4574 // in the previous switch statement. 4575 } else { 4576 codegen_->GenerateMemoryBarrier(MemBarrierKind::kLoadAny); 4577 } 4578 } 4579} 4580 4581void LocationsBuilderARM::VisitInstanceFieldSet(HInstanceFieldSet* instruction) { 4582 HandleFieldSet(instruction, instruction->GetFieldInfo()); 4583} 4584 4585void InstructionCodeGeneratorARM::VisitInstanceFieldSet(HInstanceFieldSet* instruction) { 4586 HandleFieldSet(instruction, instruction->GetFieldInfo(), instruction->GetValueCanBeNull()); 4587} 4588 4589void LocationsBuilderARM::VisitInstanceFieldGet(HInstanceFieldGet* instruction) { 4590 HandleFieldGet(instruction, instruction->GetFieldInfo()); 4591} 4592 4593void InstructionCodeGeneratorARM::VisitInstanceFieldGet(HInstanceFieldGet* instruction) { 4594 HandleFieldGet(instruction, instruction->GetFieldInfo()); 4595} 4596 4597void LocationsBuilderARM::VisitStaticFieldGet(HStaticFieldGet* instruction) { 4598 HandleFieldGet(instruction, instruction->GetFieldInfo()); 4599} 4600 4601void InstructionCodeGeneratorARM::VisitStaticFieldGet(HStaticFieldGet* instruction) { 4602 HandleFieldGet(instruction, instruction->GetFieldInfo()); 4603} 4604 4605void LocationsBuilderARM::VisitStaticFieldSet(HStaticFieldSet* instruction) { 4606 HandleFieldSet(instruction, instruction->GetFieldInfo()); 4607} 4608 4609void InstructionCodeGeneratorARM::VisitStaticFieldSet(HStaticFieldSet* instruction) { 4610 HandleFieldSet(instruction, instruction->GetFieldInfo(), instruction->GetValueCanBeNull()); 4611} 4612 4613void LocationsBuilderARM::VisitUnresolvedInstanceFieldGet( 4614 HUnresolvedInstanceFieldGet* instruction) { 4615 FieldAccessCallingConventionARM calling_convention; 4616 codegen_->CreateUnresolvedFieldLocationSummary( 4617 instruction, instruction->GetFieldType(), calling_convention); 4618} 4619 4620void InstructionCodeGeneratorARM::VisitUnresolvedInstanceFieldGet( 4621 HUnresolvedInstanceFieldGet* instruction) { 4622 FieldAccessCallingConventionARM calling_convention; 4623 codegen_->GenerateUnresolvedFieldAccess(instruction, 4624 instruction->GetFieldType(), 4625 instruction->GetFieldIndex(), 4626 instruction->GetDexPc(), 4627 calling_convention); 4628} 4629 4630void LocationsBuilderARM::VisitUnresolvedInstanceFieldSet( 4631 HUnresolvedInstanceFieldSet* instruction) { 4632 FieldAccessCallingConventionARM calling_convention; 4633 codegen_->CreateUnresolvedFieldLocationSummary( 4634 instruction, instruction->GetFieldType(), calling_convention); 4635} 4636 4637void InstructionCodeGeneratorARM::VisitUnresolvedInstanceFieldSet( 4638 HUnresolvedInstanceFieldSet* instruction) { 4639 FieldAccessCallingConventionARM calling_convention; 4640 codegen_->GenerateUnresolvedFieldAccess(instruction, 4641 instruction->GetFieldType(), 4642 instruction->GetFieldIndex(), 4643 instruction->GetDexPc(), 4644 calling_convention); 4645} 4646 4647void LocationsBuilderARM::VisitUnresolvedStaticFieldGet( 4648 HUnresolvedStaticFieldGet* instruction) { 4649 FieldAccessCallingConventionARM calling_convention; 4650 codegen_->CreateUnresolvedFieldLocationSummary( 4651 instruction, instruction->GetFieldType(), calling_convention); 4652} 4653 4654void InstructionCodeGeneratorARM::VisitUnresolvedStaticFieldGet( 4655 HUnresolvedStaticFieldGet* instruction) { 4656 FieldAccessCallingConventionARM calling_convention; 4657 codegen_->GenerateUnresolvedFieldAccess(instruction, 4658 instruction->GetFieldType(), 4659 instruction->GetFieldIndex(), 4660 instruction->GetDexPc(), 4661 calling_convention); 4662} 4663 4664void LocationsBuilderARM::VisitUnresolvedStaticFieldSet( 4665 HUnresolvedStaticFieldSet* instruction) { 4666 FieldAccessCallingConventionARM calling_convention; 4667 codegen_->CreateUnresolvedFieldLocationSummary( 4668 instruction, instruction->GetFieldType(), calling_convention); 4669} 4670 4671void InstructionCodeGeneratorARM::VisitUnresolvedStaticFieldSet( 4672 HUnresolvedStaticFieldSet* instruction) { 4673 FieldAccessCallingConventionARM calling_convention; 4674 codegen_->GenerateUnresolvedFieldAccess(instruction, 4675 instruction->GetFieldType(), 4676 instruction->GetFieldIndex(), 4677 instruction->GetDexPc(), 4678 calling_convention); 4679} 4680 4681void LocationsBuilderARM::VisitNullCheck(HNullCheck* instruction) { 4682 LocationSummary* locations = codegen_->CreateThrowingSlowPathLocations(instruction); 4683 locations->SetInAt(0, Location::RequiresRegister()); 4684} 4685 4686void CodeGeneratorARM::GenerateImplicitNullCheck(HNullCheck* instruction) { 4687 if (CanMoveNullCheckToUser(instruction)) { 4688 return; 4689 } 4690 Location obj = instruction->GetLocations()->InAt(0); 4691 4692 __ LoadFromOffset(kLoadWord, IP, obj.AsRegister<Register>(), 0); 4693 RecordPcInfo(instruction, instruction->GetDexPc()); 4694} 4695 4696void CodeGeneratorARM::GenerateExplicitNullCheck(HNullCheck* instruction) { 4697 SlowPathCodeARM* slow_path = new (GetGraph()->GetArena()) NullCheckSlowPathARM(instruction); 4698 AddSlowPath(slow_path); 4699 4700 LocationSummary* locations = instruction->GetLocations(); 4701 Location obj = locations->InAt(0); 4702 4703 __ CompareAndBranchIfZero(obj.AsRegister<Register>(), slow_path->GetEntryLabel()); 4704} 4705 4706void InstructionCodeGeneratorARM::VisitNullCheck(HNullCheck* instruction) { 4707 codegen_->GenerateNullCheck(instruction); 4708} 4709 4710static LoadOperandType GetLoadOperandType(Primitive::Type type) { 4711 switch (type) { 4712 case Primitive::kPrimNot: 4713 return kLoadWord; 4714 case Primitive::kPrimBoolean: 4715 return kLoadUnsignedByte; 4716 case Primitive::kPrimByte: 4717 return kLoadSignedByte; 4718 case Primitive::kPrimChar: 4719 return kLoadUnsignedHalfword; 4720 case Primitive::kPrimShort: 4721 return kLoadSignedHalfword; 4722 case Primitive::kPrimInt: 4723 return kLoadWord; 4724 case Primitive::kPrimLong: 4725 return kLoadWordPair; 4726 case Primitive::kPrimFloat: 4727 return kLoadSWord; 4728 case Primitive::kPrimDouble: 4729 return kLoadDWord; 4730 default: 4731 LOG(FATAL) << "Unreachable type " << type; 4732 UNREACHABLE(); 4733 } 4734} 4735 4736static StoreOperandType GetStoreOperandType(Primitive::Type type) { 4737 switch (type) { 4738 case Primitive::kPrimNot: 4739 return kStoreWord; 4740 case Primitive::kPrimBoolean: 4741 case Primitive::kPrimByte: 4742 return kStoreByte; 4743 case Primitive::kPrimChar: 4744 case Primitive::kPrimShort: 4745 return kStoreHalfword; 4746 case Primitive::kPrimInt: 4747 return kStoreWord; 4748 case Primitive::kPrimLong: 4749 return kStoreWordPair; 4750 case Primitive::kPrimFloat: 4751 return kStoreSWord; 4752 case Primitive::kPrimDouble: 4753 return kStoreDWord; 4754 default: 4755 LOG(FATAL) << "Unreachable type " << type; 4756 UNREACHABLE(); 4757 } 4758} 4759 4760void CodeGeneratorARM::LoadFromShiftedRegOffset(Primitive::Type type, 4761 Location out_loc, 4762 Register base, 4763 Register reg_offset, 4764 Condition cond) { 4765 uint32_t shift_count = Primitive::ComponentSizeShift(type); 4766 Address mem_address(base, reg_offset, Shift::LSL, shift_count); 4767 4768 switch (type) { 4769 case Primitive::kPrimByte: 4770 __ ldrsb(out_loc.AsRegister<Register>(), mem_address, cond); 4771 break; 4772 case Primitive::kPrimBoolean: 4773 __ ldrb(out_loc.AsRegister<Register>(), mem_address, cond); 4774 break; 4775 case Primitive::kPrimShort: 4776 __ ldrsh(out_loc.AsRegister<Register>(), mem_address, cond); 4777 break; 4778 case Primitive::kPrimChar: 4779 __ ldrh(out_loc.AsRegister<Register>(), mem_address, cond); 4780 break; 4781 case Primitive::kPrimNot: 4782 case Primitive::kPrimInt: 4783 __ ldr(out_loc.AsRegister<Register>(), mem_address, cond); 4784 break; 4785 // T32 doesn't support LoadFromShiftedRegOffset mem address mode for these types. 4786 case Primitive::kPrimLong: 4787 case Primitive::kPrimFloat: 4788 case Primitive::kPrimDouble: 4789 default: 4790 LOG(FATAL) << "Unreachable type " << type; 4791 UNREACHABLE(); 4792 } 4793} 4794 4795void CodeGeneratorARM::StoreToShiftedRegOffset(Primitive::Type type, 4796 Location loc, 4797 Register base, 4798 Register reg_offset, 4799 Condition cond) { 4800 uint32_t shift_count = Primitive::ComponentSizeShift(type); 4801 Address mem_address(base, reg_offset, Shift::LSL, shift_count); 4802 4803 switch (type) { 4804 case Primitive::kPrimByte: 4805 case Primitive::kPrimBoolean: 4806 __ strb(loc.AsRegister<Register>(), mem_address, cond); 4807 break; 4808 case Primitive::kPrimShort: 4809 case Primitive::kPrimChar: 4810 __ strh(loc.AsRegister<Register>(), mem_address, cond); 4811 break; 4812 case Primitive::kPrimNot: 4813 case Primitive::kPrimInt: 4814 __ str(loc.AsRegister<Register>(), mem_address, cond); 4815 break; 4816 // T32 doesn't support StoreToShiftedRegOffset mem address mode for these types. 4817 case Primitive::kPrimLong: 4818 case Primitive::kPrimFloat: 4819 case Primitive::kPrimDouble: 4820 default: 4821 LOG(FATAL) << "Unreachable type " << type; 4822 UNREACHABLE(); 4823 } 4824} 4825 4826void LocationsBuilderARM::VisitArrayGet(HArrayGet* instruction) { 4827 bool object_array_get_with_read_barrier = 4828 kEmitCompilerReadBarrier && (instruction->GetType() == Primitive::kPrimNot); 4829 LocationSummary* locations = 4830 new (GetGraph()->GetArena()) LocationSummary(instruction, 4831 object_array_get_with_read_barrier ? 4832 LocationSummary::kCallOnSlowPath : 4833 LocationSummary::kNoCall); 4834 if (object_array_get_with_read_barrier && kUseBakerReadBarrier) { 4835 locations->SetCustomSlowPathCallerSaves(RegisterSet::Empty()); // No caller-save registers. 4836 } 4837 locations->SetInAt(0, Location::RequiresRegister()); 4838 locations->SetInAt(1, Location::RegisterOrConstant(instruction->InputAt(1))); 4839 if (Primitive::IsFloatingPointType(instruction->GetType())) { 4840 locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap); 4841 } else { 4842 // The output overlaps in the case of an object array get with 4843 // read barriers enabled: we do not want the move to overwrite the 4844 // array's location, as we need it to emit the read barrier. 4845 locations->SetOut( 4846 Location::RequiresRegister(), 4847 object_array_get_with_read_barrier ? Location::kOutputOverlap : Location::kNoOutputOverlap); 4848 } 4849 // We need a temporary register for the read barrier marking slow 4850 // path in CodeGeneratorARM::GenerateArrayLoadWithBakerReadBarrier. 4851 // Also need for String compression feature. 4852 if ((object_array_get_with_read_barrier && kUseBakerReadBarrier) 4853 || (mirror::kUseStringCompression && instruction->IsStringCharAt())) { 4854 locations->AddTemp(Location::RequiresRegister()); 4855 } 4856} 4857 4858void InstructionCodeGeneratorARM::VisitArrayGet(HArrayGet* instruction) { 4859 LocationSummary* locations = instruction->GetLocations(); 4860 Location obj_loc = locations->InAt(0); 4861 Register obj = obj_loc.AsRegister<Register>(); 4862 Location index = locations->InAt(1); 4863 Location out_loc = locations->Out(); 4864 uint32_t data_offset = CodeGenerator::GetArrayDataOffset(instruction); 4865 Primitive::Type type = instruction->GetType(); 4866 const bool maybe_compressed_char_at = mirror::kUseStringCompression && 4867 instruction->IsStringCharAt(); 4868 HInstruction* array_instr = instruction->GetArray(); 4869 bool has_intermediate_address = array_instr->IsIntermediateAddress(); 4870 4871 switch (type) { 4872 case Primitive::kPrimBoolean: 4873 case Primitive::kPrimByte: 4874 case Primitive::kPrimShort: 4875 case Primitive::kPrimChar: 4876 case Primitive::kPrimInt: { 4877 Register length; 4878 if (maybe_compressed_char_at) { 4879 length = locations->GetTemp(0).AsRegister<Register>(); 4880 uint32_t count_offset = mirror::String::CountOffset().Uint32Value(); 4881 __ LoadFromOffset(kLoadWord, length, obj, count_offset); 4882 codegen_->MaybeRecordImplicitNullCheck(instruction); 4883 } 4884 if (index.IsConstant()) { 4885 int32_t const_index = index.GetConstant()->AsIntConstant()->GetValue(); 4886 if (maybe_compressed_char_at) { 4887 Label uncompressed_load, done; 4888 __ Lsrs(length, length, 1u); // LSRS has a 16-bit encoding, TST (immediate) does not. 4889 static_assert(static_cast<uint32_t>(mirror::StringCompressionFlag::kCompressed) == 0u, 4890 "Expecting 0=compressed, 1=uncompressed"); 4891 __ b(&uncompressed_load, CS); 4892 __ LoadFromOffset(kLoadUnsignedByte, 4893 out_loc.AsRegister<Register>(), 4894 obj, 4895 data_offset + const_index); 4896 __ b(&done); 4897 __ Bind(&uncompressed_load); 4898 __ LoadFromOffset(GetLoadOperandType(Primitive::kPrimChar), 4899 out_loc.AsRegister<Register>(), 4900 obj, 4901 data_offset + (const_index << 1)); 4902 __ Bind(&done); 4903 } else { 4904 uint32_t full_offset = data_offset + (const_index << Primitive::ComponentSizeShift(type)); 4905 4906 LoadOperandType load_type = GetLoadOperandType(type); 4907 __ LoadFromOffset(load_type, out_loc.AsRegister<Register>(), obj, full_offset); 4908 } 4909 } else { 4910 Register temp = IP; 4911 4912 if (has_intermediate_address) { 4913 // We do not need to compute the intermediate address from the array: the 4914 // input instruction has done it already. See the comment in 4915 // `TryExtractArrayAccessAddress()`. 4916 if (kIsDebugBuild) { 4917 HIntermediateAddress* tmp = array_instr->AsIntermediateAddress(); 4918 DCHECK_EQ(tmp->GetOffset()->AsIntConstant()->GetValueAsUint64(), data_offset); 4919 } 4920 temp = obj; 4921 } else { 4922 __ add(temp, obj, ShifterOperand(data_offset)); 4923 } 4924 if (maybe_compressed_char_at) { 4925 Label uncompressed_load, done; 4926 __ Lsrs(length, length, 1u); // LSRS has a 16-bit encoding, TST (immediate) does not. 4927 static_assert(static_cast<uint32_t>(mirror::StringCompressionFlag::kCompressed) == 0u, 4928 "Expecting 0=compressed, 1=uncompressed"); 4929 __ b(&uncompressed_load, CS); 4930 __ ldrb(out_loc.AsRegister<Register>(), 4931 Address(temp, index.AsRegister<Register>(), Shift::LSL, 0)); 4932 __ b(&done); 4933 __ Bind(&uncompressed_load); 4934 __ ldrh(out_loc.AsRegister<Register>(), 4935 Address(temp, index.AsRegister<Register>(), Shift::LSL, 1)); 4936 __ Bind(&done); 4937 } else { 4938 codegen_->LoadFromShiftedRegOffset(type, out_loc, temp, index.AsRegister<Register>()); 4939 } 4940 } 4941 break; 4942 } 4943 4944 case Primitive::kPrimNot: { 4945 // The read barrier instrumentation of object ArrayGet 4946 // instructions does not support the HIntermediateAddress 4947 // instruction. 4948 DCHECK(!(has_intermediate_address && kEmitCompilerReadBarrier)); 4949 4950 static_assert( 4951 sizeof(mirror::HeapReference<mirror::Object>) == sizeof(int32_t), 4952 "art::mirror::HeapReference<art::mirror::Object> and int32_t have different sizes."); 4953 // /* HeapReference<Object> */ out = 4954 // *(obj + data_offset + index * sizeof(HeapReference<Object>)) 4955 if (kEmitCompilerReadBarrier && kUseBakerReadBarrier) { 4956 Location temp = locations->GetTemp(0); 4957 // Note that a potential implicit null check is handled in this 4958 // CodeGeneratorARM::GenerateArrayLoadWithBakerReadBarrier call. 4959 codegen_->GenerateArrayLoadWithBakerReadBarrier( 4960 instruction, out_loc, obj, data_offset, index, temp, /* needs_null_check */ true); 4961 } else { 4962 Register out = out_loc.AsRegister<Register>(); 4963 if (index.IsConstant()) { 4964 size_t offset = 4965 (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4) + data_offset; 4966 __ LoadFromOffset(kLoadWord, out, obj, offset); 4967 codegen_->MaybeRecordImplicitNullCheck(instruction); 4968 // If read barriers are enabled, emit read barriers other than 4969 // Baker's using a slow path (and also unpoison the loaded 4970 // reference, if heap poisoning is enabled). 4971 codegen_->MaybeGenerateReadBarrierSlow(instruction, out_loc, out_loc, obj_loc, offset); 4972 } else { 4973 Register temp = IP; 4974 4975 if (has_intermediate_address) { 4976 // We do not need to compute the intermediate address from the array: the 4977 // input instruction has done it already. See the comment in 4978 // `TryExtractArrayAccessAddress()`. 4979 if (kIsDebugBuild) { 4980 HIntermediateAddress* tmp = array_instr->AsIntermediateAddress(); 4981 DCHECK_EQ(tmp->GetOffset()->AsIntConstant()->GetValueAsUint64(), data_offset); 4982 } 4983 temp = obj; 4984 } else { 4985 __ add(temp, obj, ShifterOperand(data_offset)); 4986 } 4987 codegen_->LoadFromShiftedRegOffset(type, out_loc, temp, index.AsRegister<Register>()); 4988 4989 codegen_->MaybeRecordImplicitNullCheck(instruction); 4990 // If read barriers are enabled, emit read barriers other than 4991 // Baker's using a slow path (and also unpoison the loaded 4992 // reference, if heap poisoning is enabled). 4993 codegen_->MaybeGenerateReadBarrierSlow( 4994 instruction, out_loc, out_loc, obj_loc, data_offset, index); 4995 } 4996 } 4997 break; 4998 } 4999 5000 case Primitive::kPrimLong: { 5001 if (index.IsConstant()) { 5002 size_t offset = 5003 (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_8) + data_offset; 5004 __ LoadFromOffset(kLoadWordPair, out_loc.AsRegisterPairLow<Register>(), obj, offset); 5005 } else { 5006 __ add(IP, obj, ShifterOperand(index.AsRegister<Register>(), LSL, TIMES_8)); 5007 __ LoadFromOffset(kLoadWordPair, out_loc.AsRegisterPairLow<Register>(), IP, data_offset); 5008 } 5009 break; 5010 } 5011 5012 case Primitive::kPrimFloat: { 5013 SRegister out = out_loc.AsFpuRegister<SRegister>(); 5014 if (index.IsConstant()) { 5015 size_t offset = (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4) + data_offset; 5016 __ LoadSFromOffset(out, obj, offset); 5017 } else { 5018 __ add(IP, obj, ShifterOperand(index.AsRegister<Register>(), LSL, TIMES_4)); 5019 __ LoadSFromOffset(out, IP, data_offset); 5020 } 5021 break; 5022 } 5023 5024 case Primitive::kPrimDouble: { 5025 SRegister out = out_loc.AsFpuRegisterPairLow<SRegister>(); 5026 if (index.IsConstant()) { 5027 size_t offset = (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_8) + data_offset; 5028 __ LoadDFromOffset(FromLowSToD(out), obj, offset); 5029 } else { 5030 __ add(IP, obj, ShifterOperand(index.AsRegister<Register>(), LSL, TIMES_8)); 5031 __ LoadDFromOffset(FromLowSToD(out), IP, data_offset); 5032 } 5033 break; 5034 } 5035 5036 case Primitive::kPrimVoid: 5037 LOG(FATAL) << "Unreachable type " << type; 5038 UNREACHABLE(); 5039 } 5040 5041 if (type == Primitive::kPrimNot) { 5042 // Potential implicit null checks, in the case of reference 5043 // arrays, are handled in the previous switch statement. 5044 } else if (!maybe_compressed_char_at) { 5045 codegen_->MaybeRecordImplicitNullCheck(instruction); 5046 } 5047} 5048 5049void LocationsBuilderARM::VisitArraySet(HArraySet* instruction) { 5050 Primitive::Type value_type = instruction->GetComponentType(); 5051 5052 bool needs_write_barrier = 5053 CodeGenerator::StoreNeedsWriteBarrier(value_type, instruction->GetValue()); 5054 bool may_need_runtime_call_for_type_check = instruction->NeedsTypeCheck(); 5055 5056 LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary( 5057 instruction, 5058 may_need_runtime_call_for_type_check ? 5059 LocationSummary::kCallOnSlowPath : 5060 LocationSummary::kNoCall); 5061 5062 locations->SetInAt(0, Location::RequiresRegister()); 5063 locations->SetInAt(1, Location::RegisterOrConstant(instruction->InputAt(1))); 5064 if (Primitive::IsFloatingPointType(value_type)) { 5065 locations->SetInAt(2, Location::RequiresFpuRegister()); 5066 } else { 5067 locations->SetInAt(2, Location::RequiresRegister()); 5068 } 5069 if (needs_write_barrier) { 5070 // Temporary registers for the write barrier. 5071 locations->AddTemp(Location::RequiresRegister()); // Possibly used for ref. poisoning too. 5072 locations->AddTemp(Location::RequiresRegister()); 5073 } 5074} 5075 5076void InstructionCodeGeneratorARM::VisitArraySet(HArraySet* instruction) { 5077 LocationSummary* locations = instruction->GetLocations(); 5078 Location array_loc = locations->InAt(0); 5079 Register array = array_loc.AsRegister<Register>(); 5080 Location index = locations->InAt(1); 5081 Primitive::Type value_type = instruction->GetComponentType(); 5082 bool may_need_runtime_call_for_type_check = instruction->NeedsTypeCheck(); 5083 bool needs_write_barrier = 5084 CodeGenerator::StoreNeedsWriteBarrier(value_type, instruction->GetValue()); 5085 uint32_t data_offset = 5086 mirror::Array::DataOffset(Primitive::ComponentSize(value_type)).Uint32Value(); 5087 Location value_loc = locations->InAt(2); 5088 HInstruction* array_instr = instruction->GetArray(); 5089 bool has_intermediate_address = array_instr->IsIntermediateAddress(); 5090 5091 switch (value_type) { 5092 case Primitive::kPrimBoolean: 5093 case Primitive::kPrimByte: 5094 case Primitive::kPrimShort: 5095 case Primitive::kPrimChar: 5096 case Primitive::kPrimInt: { 5097 if (index.IsConstant()) { 5098 int32_t const_index = index.GetConstant()->AsIntConstant()->GetValue(); 5099 uint32_t full_offset = 5100 data_offset + (const_index << Primitive::ComponentSizeShift(value_type)); 5101 StoreOperandType store_type = GetStoreOperandType(value_type); 5102 __ StoreToOffset(store_type, value_loc.AsRegister<Register>(), array, full_offset); 5103 } else { 5104 Register temp = IP; 5105 5106 if (has_intermediate_address) { 5107 // We do not need to compute the intermediate address from the array: the 5108 // input instruction has done it already. See the comment in 5109 // `TryExtractArrayAccessAddress()`. 5110 if (kIsDebugBuild) { 5111 HIntermediateAddress* tmp = array_instr->AsIntermediateAddress(); 5112 DCHECK(tmp->GetOffset()->AsIntConstant()->GetValueAsUint64() == data_offset); 5113 } 5114 temp = array; 5115 } else { 5116 __ add(temp, array, ShifterOperand(data_offset)); 5117 } 5118 codegen_->StoreToShiftedRegOffset(value_type, 5119 value_loc, 5120 temp, 5121 index.AsRegister<Register>()); 5122 } 5123 break; 5124 } 5125 5126 case Primitive::kPrimNot: { 5127 Register value = value_loc.AsRegister<Register>(); 5128 // TryExtractArrayAccessAddress optimization is never applied for non-primitive ArraySet. 5129 // See the comment in instruction_simplifier_shared.cc. 5130 DCHECK(!has_intermediate_address); 5131 5132 if (instruction->InputAt(2)->IsNullConstant()) { 5133 // Just setting null. 5134 if (index.IsConstant()) { 5135 size_t offset = 5136 (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4) + data_offset; 5137 __ StoreToOffset(kStoreWord, value, array, offset); 5138 } else { 5139 DCHECK(index.IsRegister()) << index; 5140 __ add(IP, array, ShifterOperand(data_offset)); 5141 codegen_->StoreToShiftedRegOffset(value_type, 5142 value_loc, 5143 IP, 5144 index.AsRegister<Register>()); 5145 } 5146 codegen_->MaybeRecordImplicitNullCheck(instruction); 5147 DCHECK(!needs_write_barrier); 5148 DCHECK(!may_need_runtime_call_for_type_check); 5149 break; 5150 } 5151 5152 DCHECK(needs_write_barrier); 5153 Location temp1_loc = locations->GetTemp(0); 5154 Register temp1 = temp1_loc.AsRegister<Register>(); 5155 Location temp2_loc = locations->GetTemp(1); 5156 Register temp2 = temp2_loc.AsRegister<Register>(); 5157 uint32_t class_offset = mirror::Object::ClassOffset().Int32Value(); 5158 uint32_t super_offset = mirror::Class::SuperClassOffset().Int32Value(); 5159 uint32_t component_offset = mirror::Class::ComponentTypeOffset().Int32Value(); 5160 Label done; 5161 SlowPathCodeARM* slow_path = nullptr; 5162 5163 if (may_need_runtime_call_for_type_check) { 5164 slow_path = new (GetGraph()->GetArena()) ArraySetSlowPathARM(instruction); 5165 codegen_->AddSlowPath(slow_path); 5166 if (instruction->GetValueCanBeNull()) { 5167 Label non_zero; 5168 __ CompareAndBranchIfNonZero(value, &non_zero); 5169 if (index.IsConstant()) { 5170 size_t offset = 5171 (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4) + data_offset; 5172 __ StoreToOffset(kStoreWord, value, array, offset); 5173 } else { 5174 DCHECK(index.IsRegister()) << index; 5175 __ add(IP, array, ShifterOperand(data_offset)); 5176 codegen_->StoreToShiftedRegOffset(value_type, 5177 value_loc, 5178 IP, 5179 index.AsRegister<Register>()); 5180 } 5181 codegen_->MaybeRecordImplicitNullCheck(instruction); 5182 __ b(&done); 5183 __ Bind(&non_zero); 5184 } 5185 5186 // Note that when read barriers are enabled, the type checks 5187 // are performed without read barriers. This is fine, even in 5188 // the case where a class object is in the from-space after 5189 // the flip, as a comparison involving such a type would not 5190 // produce a false positive; it may of course produce a false 5191 // negative, in which case we would take the ArraySet slow 5192 // path. 5193 5194 // /* HeapReference<Class> */ temp1 = array->klass_ 5195 __ LoadFromOffset(kLoadWord, temp1, array, class_offset); 5196 codegen_->MaybeRecordImplicitNullCheck(instruction); 5197 __ MaybeUnpoisonHeapReference(temp1); 5198 5199 // /* HeapReference<Class> */ temp1 = temp1->component_type_ 5200 __ LoadFromOffset(kLoadWord, temp1, temp1, component_offset); 5201 // /* HeapReference<Class> */ temp2 = value->klass_ 5202 __ LoadFromOffset(kLoadWord, temp2, value, class_offset); 5203 // If heap poisoning is enabled, no need to unpoison `temp1` 5204 // nor `temp2`, as we are comparing two poisoned references. 5205 __ cmp(temp1, ShifterOperand(temp2)); 5206 5207 if (instruction->StaticTypeOfArrayIsObjectArray()) { 5208 Label do_put; 5209 __ b(&do_put, EQ); 5210 // If heap poisoning is enabled, the `temp1` reference has 5211 // not been unpoisoned yet; unpoison it now. 5212 __ MaybeUnpoisonHeapReference(temp1); 5213 5214 // /* HeapReference<Class> */ temp1 = temp1->super_class_ 5215 __ LoadFromOffset(kLoadWord, temp1, temp1, super_offset); 5216 // If heap poisoning is enabled, no need to unpoison 5217 // `temp1`, as we are comparing against null below. 5218 __ CompareAndBranchIfNonZero(temp1, slow_path->GetEntryLabel()); 5219 __ Bind(&do_put); 5220 } else { 5221 __ b(slow_path->GetEntryLabel(), NE); 5222 } 5223 } 5224 5225 Register source = value; 5226 if (kPoisonHeapReferences) { 5227 // Note that in the case where `value` is a null reference, 5228 // we do not enter this block, as a null reference does not 5229 // need poisoning. 5230 DCHECK_EQ(value_type, Primitive::kPrimNot); 5231 __ Mov(temp1, value); 5232 __ PoisonHeapReference(temp1); 5233 source = temp1; 5234 } 5235 5236 if (index.IsConstant()) { 5237 size_t offset = 5238 (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4) + data_offset; 5239 __ StoreToOffset(kStoreWord, source, array, offset); 5240 } else { 5241 DCHECK(index.IsRegister()) << index; 5242 5243 __ add(IP, array, ShifterOperand(data_offset)); 5244 codegen_->StoreToShiftedRegOffset(value_type, 5245 Location::RegisterLocation(source), 5246 IP, 5247 index.AsRegister<Register>()); 5248 } 5249 5250 if (!may_need_runtime_call_for_type_check) { 5251 codegen_->MaybeRecordImplicitNullCheck(instruction); 5252 } 5253 5254 codegen_->MarkGCCard(temp1, temp2, array, value, instruction->GetValueCanBeNull()); 5255 5256 if (done.IsLinked()) { 5257 __ Bind(&done); 5258 } 5259 5260 if (slow_path != nullptr) { 5261 __ Bind(slow_path->GetExitLabel()); 5262 } 5263 5264 break; 5265 } 5266 5267 case Primitive::kPrimLong: { 5268 Location value = locations->InAt(2); 5269 if (index.IsConstant()) { 5270 size_t offset = 5271 (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_8) + data_offset; 5272 __ StoreToOffset(kStoreWordPair, value.AsRegisterPairLow<Register>(), array, offset); 5273 } else { 5274 __ add(IP, array, ShifterOperand(index.AsRegister<Register>(), LSL, TIMES_8)); 5275 __ StoreToOffset(kStoreWordPair, value.AsRegisterPairLow<Register>(), IP, data_offset); 5276 } 5277 break; 5278 } 5279 5280 case Primitive::kPrimFloat: { 5281 Location value = locations->InAt(2); 5282 DCHECK(value.IsFpuRegister()); 5283 if (index.IsConstant()) { 5284 size_t offset = (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4) + data_offset; 5285 __ StoreSToOffset(value.AsFpuRegister<SRegister>(), array, offset); 5286 } else { 5287 __ add(IP, array, ShifterOperand(index.AsRegister<Register>(), LSL, TIMES_4)); 5288 __ StoreSToOffset(value.AsFpuRegister<SRegister>(), IP, data_offset); 5289 } 5290 break; 5291 } 5292 5293 case Primitive::kPrimDouble: { 5294 Location value = locations->InAt(2); 5295 DCHECK(value.IsFpuRegisterPair()); 5296 if (index.IsConstant()) { 5297 size_t offset = (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_8) + data_offset; 5298 __ StoreDToOffset(FromLowSToD(value.AsFpuRegisterPairLow<SRegister>()), array, offset); 5299 } else { 5300 __ add(IP, array, ShifterOperand(index.AsRegister<Register>(), LSL, TIMES_8)); 5301 __ StoreDToOffset(FromLowSToD(value.AsFpuRegisterPairLow<SRegister>()), IP, data_offset); 5302 } 5303 5304 break; 5305 } 5306 5307 case Primitive::kPrimVoid: 5308 LOG(FATAL) << "Unreachable type " << value_type; 5309 UNREACHABLE(); 5310 } 5311 5312 // Objects are handled in the switch. 5313 if (value_type != Primitive::kPrimNot) { 5314 codegen_->MaybeRecordImplicitNullCheck(instruction); 5315 } 5316} 5317 5318void LocationsBuilderARM::VisitArrayLength(HArrayLength* instruction) { 5319 LocationSummary* locations = 5320 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall); 5321 locations->SetInAt(0, Location::RequiresRegister()); 5322 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 5323} 5324 5325void InstructionCodeGeneratorARM::VisitArrayLength(HArrayLength* instruction) { 5326 LocationSummary* locations = instruction->GetLocations(); 5327 uint32_t offset = CodeGenerator::GetArrayLengthOffset(instruction); 5328 Register obj = locations->InAt(0).AsRegister<Register>(); 5329 Register out = locations->Out().AsRegister<Register>(); 5330 __ LoadFromOffset(kLoadWord, out, obj, offset); 5331 codegen_->MaybeRecordImplicitNullCheck(instruction); 5332 // Mask out compression flag from String's array length. 5333 if (mirror::kUseStringCompression && instruction->IsStringLength()) { 5334 __ Lsr(out, out, 1u); 5335 } 5336} 5337 5338void LocationsBuilderARM::VisitIntermediateAddress(HIntermediateAddress* instruction) { 5339 LocationSummary* locations = 5340 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall); 5341 5342 locations->SetInAt(0, Location::RequiresRegister()); 5343 locations->SetInAt(1, Location::RegisterOrConstant(instruction->GetOffset())); 5344 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 5345} 5346 5347void InstructionCodeGeneratorARM::VisitIntermediateAddress(HIntermediateAddress* instruction) { 5348 LocationSummary* locations = instruction->GetLocations(); 5349 Location out = locations->Out(); 5350 Location first = locations->InAt(0); 5351 Location second = locations->InAt(1); 5352 5353 if (second.IsRegister()) { 5354 __ add(out.AsRegister<Register>(), 5355 first.AsRegister<Register>(), 5356 ShifterOperand(second.AsRegister<Register>())); 5357 } else { 5358 __ AddConstant(out.AsRegister<Register>(), 5359 first.AsRegister<Register>(), 5360 second.GetConstant()->AsIntConstant()->GetValue()); 5361 } 5362} 5363 5364void LocationsBuilderARM::VisitBoundsCheck(HBoundsCheck* instruction) { 5365 RegisterSet caller_saves = RegisterSet::Empty(); 5366 InvokeRuntimeCallingConvention calling_convention; 5367 caller_saves.Add(Location::RegisterLocation(calling_convention.GetRegisterAt(0))); 5368 caller_saves.Add(Location::RegisterLocation(calling_convention.GetRegisterAt(1))); 5369 LocationSummary* locations = codegen_->CreateThrowingSlowPathLocations(instruction, caller_saves); 5370 locations->SetInAt(0, Location::RequiresRegister()); 5371 locations->SetInAt(1, Location::RequiresRegister()); 5372} 5373 5374void InstructionCodeGeneratorARM::VisitBoundsCheck(HBoundsCheck* instruction) { 5375 LocationSummary* locations = instruction->GetLocations(); 5376 SlowPathCodeARM* slow_path = 5377 new (GetGraph()->GetArena()) BoundsCheckSlowPathARM(instruction); 5378 codegen_->AddSlowPath(slow_path); 5379 5380 Register index = locations->InAt(0).AsRegister<Register>(); 5381 Register length = locations->InAt(1).AsRegister<Register>(); 5382 5383 __ cmp(index, ShifterOperand(length)); 5384 __ b(slow_path->GetEntryLabel(), HS); 5385} 5386 5387void CodeGeneratorARM::MarkGCCard(Register temp, 5388 Register card, 5389 Register object, 5390 Register value, 5391 bool can_be_null) { 5392 Label is_null; 5393 if (can_be_null) { 5394 __ CompareAndBranchIfZero(value, &is_null); 5395 } 5396 __ LoadFromOffset(kLoadWord, card, TR, Thread::CardTableOffset<kArmPointerSize>().Int32Value()); 5397 __ Lsr(temp, object, gc::accounting::CardTable::kCardShift); 5398 __ strb(card, Address(card, temp)); 5399 if (can_be_null) { 5400 __ Bind(&is_null); 5401 } 5402} 5403 5404void LocationsBuilderARM::VisitParallelMove(HParallelMove* instruction ATTRIBUTE_UNUSED) { 5405 LOG(FATAL) << "Unreachable"; 5406} 5407 5408void InstructionCodeGeneratorARM::VisitParallelMove(HParallelMove* instruction) { 5409 codegen_->GetMoveResolver()->EmitNativeCode(instruction); 5410} 5411 5412void LocationsBuilderARM::VisitSuspendCheck(HSuspendCheck* instruction) { 5413 LocationSummary* locations = 5414 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCallOnSlowPath); 5415 locations->SetCustomSlowPathCallerSaves(RegisterSet::Empty()); // No caller-save registers. 5416} 5417 5418void InstructionCodeGeneratorARM::VisitSuspendCheck(HSuspendCheck* instruction) { 5419 HBasicBlock* block = instruction->GetBlock(); 5420 if (block->GetLoopInformation() != nullptr) { 5421 DCHECK(block->GetLoopInformation()->GetSuspendCheck() == instruction); 5422 // The back edge will generate the suspend check. 5423 return; 5424 } 5425 if (block->IsEntryBlock() && instruction->GetNext()->IsGoto()) { 5426 // The goto will generate the suspend check. 5427 return; 5428 } 5429 GenerateSuspendCheck(instruction, nullptr); 5430} 5431 5432void InstructionCodeGeneratorARM::GenerateSuspendCheck(HSuspendCheck* instruction, 5433 HBasicBlock* successor) { 5434 SuspendCheckSlowPathARM* slow_path = 5435 down_cast<SuspendCheckSlowPathARM*>(instruction->GetSlowPath()); 5436 if (slow_path == nullptr) { 5437 slow_path = new (GetGraph()->GetArena()) SuspendCheckSlowPathARM(instruction, successor); 5438 instruction->SetSlowPath(slow_path); 5439 codegen_->AddSlowPath(slow_path); 5440 if (successor != nullptr) { 5441 DCHECK(successor->IsLoopHeader()); 5442 codegen_->ClearSpillSlotsFromLoopPhisInStackMap(instruction); 5443 } 5444 } else { 5445 DCHECK_EQ(slow_path->GetSuccessor(), successor); 5446 } 5447 5448 __ LoadFromOffset( 5449 kLoadUnsignedHalfword, IP, TR, Thread::ThreadFlagsOffset<kArmPointerSize>().Int32Value()); 5450 if (successor == nullptr) { 5451 __ CompareAndBranchIfNonZero(IP, slow_path->GetEntryLabel()); 5452 __ Bind(slow_path->GetReturnLabel()); 5453 } else { 5454 __ CompareAndBranchIfZero(IP, codegen_->GetLabelOf(successor)); 5455 __ b(slow_path->GetEntryLabel()); 5456 } 5457} 5458 5459ArmAssembler* ParallelMoveResolverARM::GetAssembler() const { 5460 return codegen_->GetAssembler(); 5461} 5462 5463void ParallelMoveResolverARM::EmitMove(size_t index) { 5464 MoveOperands* move = moves_[index]; 5465 Location source = move->GetSource(); 5466 Location destination = move->GetDestination(); 5467 5468 if (source.IsRegister()) { 5469 if (destination.IsRegister()) { 5470 __ Mov(destination.AsRegister<Register>(), source.AsRegister<Register>()); 5471 } else if (destination.IsFpuRegister()) { 5472 __ vmovsr(destination.AsFpuRegister<SRegister>(), source.AsRegister<Register>()); 5473 } else { 5474 DCHECK(destination.IsStackSlot()); 5475 __ StoreToOffset(kStoreWord, source.AsRegister<Register>(), 5476 SP, destination.GetStackIndex()); 5477 } 5478 } else if (source.IsStackSlot()) { 5479 if (destination.IsRegister()) { 5480 __ LoadFromOffset(kLoadWord, destination.AsRegister<Register>(), 5481 SP, source.GetStackIndex()); 5482 } else if (destination.IsFpuRegister()) { 5483 __ LoadSFromOffset(destination.AsFpuRegister<SRegister>(), SP, source.GetStackIndex()); 5484 } else { 5485 DCHECK(destination.IsStackSlot()); 5486 __ LoadFromOffset(kLoadWord, IP, SP, source.GetStackIndex()); 5487 __ StoreToOffset(kStoreWord, IP, SP, destination.GetStackIndex()); 5488 } 5489 } else if (source.IsFpuRegister()) { 5490 if (destination.IsRegister()) { 5491 __ vmovrs(destination.AsRegister<Register>(), source.AsFpuRegister<SRegister>()); 5492 } else if (destination.IsFpuRegister()) { 5493 __ vmovs(destination.AsFpuRegister<SRegister>(), source.AsFpuRegister<SRegister>()); 5494 } else { 5495 DCHECK(destination.IsStackSlot()); 5496 __ StoreSToOffset(source.AsFpuRegister<SRegister>(), SP, destination.GetStackIndex()); 5497 } 5498 } else if (source.IsDoubleStackSlot()) { 5499 if (destination.IsDoubleStackSlot()) { 5500 __ LoadDFromOffset(DTMP, SP, source.GetStackIndex()); 5501 __ StoreDToOffset(DTMP, SP, destination.GetStackIndex()); 5502 } else if (destination.IsRegisterPair()) { 5503 DCHECK(ExpectedPairLayout(destination)); 5504 __ LoadFromOffset( 5505 kLoadWordPair, destination.AsRegisterPairLow<Register>(), SP, source.GetStackIndex()); 5506 } else { 5507 DCHECK(destination.IsFpuRegisterPair()) << destination; 5508 __ LoadDFromOffset(FromLowSToD(destination.AsFpuRegisterPairLow<SRegister>()), 5509 SP, 5510 source.GetStackIndex()); 5511 } 5512 } else if (source.IsRegisterPair()) { 5513 if (destination.IsRegisterPair()) { 5514 __ Mov(destination.AsRegisterPairLow<Register>(), source.AsRegisterPairLow<Register>()); 5515 __ Mov(destination.AsRegisterPairHigh<Register>(), source.AsRegisterPairHigh<Register>()); 5516 } else if (destination.IsFpuRegisterPair()) { 5517 __ vmovdrr(FromLowSToD(destination.AsFpuRegisterPairLow<SRegister>()), 5518 source.AsRegisterPairLow<Register>(), 5519 source.AsRegisterPairHigh<Register>()); 5520 } else { 5521 DCHECK(destination.IsDoubleStackSlot()) << destination; 5522 DCHECK(ExpectedPairLayout(source)); 5523 __ StoreToOffset( 5524 kStoreWordPair, source.AsRegisterPairLow<Register>(), SP, destination.GetStackIndex()); 5525 } 5526 } else if (source.IsFpuRegisterPair()) { 5527 if (destination.IsRegisterPair()) { 5528 __ vmovrrd(destination.AsRegisterPairLow<Register>(), 5529 destination.AsRegisterPairHigh<Register>(), 5530 FromLowSToD(source.AsFpuRegisterPairLow<SRegister>())); 5531 } else if (destination.IsFpuRegisterPair()) { 5532 __ vmovd(FromLowSToD(destination.AsFpuRegisterPairLow<SRegister>()), 5533 FromLowSToD(source.AsFpuRegisterPairLow<SRegister>())); 5534 } else { 5535 DCHECK(destination.IsDoubleStackSlot()) << destination; 5536 __ StoreDToOffset(FromLowSToD(source.AsFpuRegisterPairLow<SRegister>()), 5537 SP, 5538 destination.GetStackIndex()); 5539 } 5540 } else { 5541 DCHECK(source.IsConstant()) << source; 5542 HConstant* constant = source.GetConstant(); 5543 if (constant->IsIntConstant() || constant->IsNullConstant()) { 5544 int32_t value = CodeGenerator::GetInt32ValueOf(constant); 5545 if (destination.IsRegister()) { 5546 __ LoadImmediate(destination.AsRegister<Register>(), value); 5547 } else { 5548 DCHECK(destination.IsStackSlot()); 5549 __ LoadImmediate(IP, value); 5550 __ StoreToOffset(kStoreWord, IP, SP, destination.GetStackIndex()); 5551 } 5552 } else if (constant->IsLongConstant()) { 5553 int64_t value = constant->AsLongConstant()->GetValue(); 5554 if (destination.IsRegisterPair()) { 5555 __ LoadImmediate(destination.AsRegisterPairLow<Register>(), Low32Bits(value)); 5556 __ LoadImmediate(destination.AsRegisterPairHigh<Register>(), High32Bits(value)); 5557 } else { 5558 DCHECK(destination.IsDoubleStackSlot()) << destination; 5559 __ LoadImmediate(IP, Low32Bits(value)); 5560 __ StoreToOffset(kStoreWord, IP, SP, destination.GetStackIndex()); 5561 __ LoadImmediate(IP, High32Bits(value)); 5562 __ StoreToOffset(kStoreWord, IP, SP, destination.GetHighStackIndex(kArmWordSize)); 5563 } 5564 } else if (constant->IsDoubleConstant()) { 5565 double value = constant->AsDoubleConstant()->GetValue(); 5566 if (destination.IsFpuRegisterPair()) { 5567 __ LoadDImmediate(FromLowSToD(destination.AsFpuRegisterPairLow<SRegister>()), value); 5568 } else { 5569 DCHECK(destination.IsDoubleStackSlot()) << destination; 5570 uint64_t int_value = bit_cast<uint64_t, double>(value); 5571 __ LoadImmediate(IP, Low32Bits(int_value)); 5572 __ StoreToOffset(kStoreWord, IP, SP, destination.GetStackIndex()); 5573 __ LoadImmediate(IP, High32Bits(int_value)); 5574 __ StoreToOffset(kStoreWord, IP, SP, destination.GetHighStackIndex(kArmWordSize)); 5575 } 5576 } else { 5577 DCHECK(constant->IsFloatConstant()) << constant->DebugName(); 5578 float value = constant->AsFloatConstant()->GetValue(); 5579 if (destination.IsFpuRegister()) { 5580 __ LoadSImmediate(destination.AsFpuRegister<SRegister>(), value); 5581 } else { 5582 DCHECK(destination.IsStackSlot()); 5583 __ LoadImmediate(IP, bit_cast<int32_t, float>(value)); 5584 __ StoreToOffset(kStoreWord, IP, SP, destination.GetStackIndex()); 5585 } 5586 } 5587 } 5588} 5589 5590void ParallelMoveResolverARM::Exchange(Register reg, int mem) { 5591 __ Mov(IP, reg); 5592 __ LoadFromOffset(kLoadWord, reg, SP, mem); 5593 __ StoreToOffset(kStoreWord, IP, SP, mem); 5594} 5595 5596void ParallelMoveResolverARM::Exchange(int mem1, int mem2) { 5597 ScratchRegisterScope ensure_scratch(this, IP, R0, codegen_->GetNumberOfCoreRegisters()); 5598 int stack_offset = ensure_scratch.IsSpilled() ? kArmWordSize : 0; 5599 __ LoadFromOffset(kLoadWord, static_cast<Register>(ensure_scratch.GetRegister()), 5600 SP, mem1 + stack_offset); 5601 __ LoadFromOffset(kLoadWord, IP, SP, mem2 + stack_offset); 5602 __ StoreToOffset(kStoreWord, static_cast<Register>(ensure_scratch.GetRegister()), 5603 SP, mem2 + stack_offset); 5604 __ StoreToOffset(kStoreWord, IP, SP, mem1 + stack_offset); 5605} 5606 5607void ParallelMoveResolverARM::EmitSwap(size_t index) { 5608 MoveOperands* move = moves_[index]; 5609 Location source = move->GetSource(); 5610 Location destination = move->GetDestination(); 5611 5612 if (source.IsRegister() && destination.IsRegister()) { 5613 DCHECK_NE(source.AsRegister<Register>(), IP); 5614 DCHECK_NE(destination.AsRegister<Register>(), IP); 5615 __ Mov(IP, source.AsRegister<Register>()); 5616 __ Mov(source.AsRegister<Register>(), destination.AsRegister<Register>()); 5617 __ Mov(destination.AsRegister<Register>(), IP); 5618 } else if (source.IsRegister() && destination.IsStackSlot()) { 5619 Exchange(source.AsRegister<Register>(), destination.GetStackIndex()); 5620 } else if (source.IsStackSlot() && destination.IsRegister()) { 5621 Exchange(destination.AsRegister<Register>(), source.GetStackIndex()); 5622 } else if (source.IsStackSlot() && destination.IsStackSlot()) { 5623 Exchange(source.GetStackIndex(), destination.GetStackIndex()); 5624 } else if (source.IsFpuRegister() && destination.IsFpuRegister()) { 5625 __ vmovrs(IP, source.AsFpuRegister<SRegister>()); 5626 __ vmovs(source.AsFpuRegister<SRegister>(), destination.AsFpuRegister<SRegister>()); 5627 __ vmovsr(destination.AsFpuRegister<SRegister>(), IP); 5628 } else if (source.IsRegisterPair() && destination.IsRegisterPair()) { 5629 __ vmovdrr(DTMP, source.AsRegisterPairLow<Register>(), source.AsRegisterPairHigh<Register>()); 5630 __ Mov(source.AsRegisterPairLow<Register>(), destination.AsRegisterPairLow<Register>()); 5631 __ Mov(source.AsRegisterPairHigh<Register>(), destination.AsRegisterPairHigh<Register>()); 5632 __ vmovrrd(destination.AsRegisterPairLow<Register>(), 5633 destination.AsRegisterPairHigh<Register>(), 5634 DTMP); 5635 } else if (source.IsRegisterPair() || destination.IsRegisterPair()) { 5636 Register low_reg = source.IsRegisterPair() 5637 ? source.AsRegisterPairLow<Register>() 5638 : destination.AsRegisterPairLow<Register>(); 5639 int mem = source.IsRegisterPair() 5640 ? destination.GetStackIndex() 5641 : source.GetStackIndex(); 5642 DCHECK(ExpectedPairLayout(source.IsRegisterPair() ? source : destination)); 5643 __ vmovdrr(DTMP, low_reg, static_cast<Register>(low_reg + 1)); 5644 __ LoadFromOffset(kLoadWordPair, low_reg, SP, mem); 5645 __ StoreDToOffset(DTMP, SP, mem); 5646 } else if (source.IsFpuRegisterPair() && destination.IsFpuRegisterPair()) { 5647 DRegister first = FromLowSToD(source.AsFpuRegisterPairLow<SRegister>()); 5648 DRegister second = FromLowSToD(destination.AsFpuRegisterPairLow<SRegister>()); 5649 __ vmovd(DTMP, first); 5650 __ vmovd(first, second); 5651 __ vmovd(second, DTMP); 5652 } else if (source.IsFpuRegisterPair() || destination.IsFpuRegisterPair()) { 5653 DRegister reg = source.IsFpuRegisterPair() 5654 ? FromLowSToD(source.AsFpuRegisterPairLow<SRegister>()) 5655 : FromLowSToD(destination.AsFpuRegisterPairLow<SRegister>()); 5656 int mem = source.IsFpuRegisterPair() 5657 ? destination.GetStackIndex() 5658 : source.GetStackIndex(); 5659 __ vmovd(DTMP, reg); 5660 __ LoadDFromOffset(reg, SP, mem); 5661 __ StoreDToOffset(DTMP, SP, mem); 5662 } else if (source.IsFpuRegister() || destination.IsFpuRegister()) { 5663 SRegister reg = source.IsFpuRegister() ? source.AsFpuRegister<SRegister>() 5664 : destination.AsFpuRegister<SRegister>(); 5665 int mem = source.IsFpuRegister() 5666 ? destination.GetStackIndex() 5667 : source.GetStackIndex(); 5668 5669 __ vmovrs(IP, reg); 5670 __ LoadSFromOffset(reg, SP, mem); 5671 __ StoreToOffset(kStoreWord, IP, SP, mem); 5672 } else if (source.IsDoubleStackSlot() && destination.IsDoubleStackSlot()) { 5673 Exchange(source.GetStackIndex(), destination.GetStackIndex()); 5674 Exchange(source.GetHighStackIndex(kArmWordSize), destination.GetHighStackIndex(kArmWordSize)); 5675 } else { 5676 LOG(FATAL) << "Unimplemented" << source << " <-> " << destination; 5677 } 5678} 5679 5680void ParallelMoveResolverARM::SpillScratch(int reg) { 5681 __ Push(static_cast<Register>(reg)); 5682} 5683 5684void ParallelMoveResolverARM::RestoreScratch(int reg) { 5685 __ Pop(static_cast<Register>(reg)); 5686} 5687 5688HLoadClass::LoadKind CodeGeneratorARM::GetSupportedLoadClassKind( 5689 HLoadClass::LoadKind desired_class_load_kind) { 5690 switch (desired_class_load_kind) { 5691 case HLoadClass::LoadKind::kReferrersClass: 5692 break; 5693 case HLoadClass::LoadKind::kBootImageLinkTimeAddress: 5694 DCHECK(!GetCompilerOptions().GetCompilePic()); 5695 break; 5696 case HLoadClass::LoadKind::kBootImageLinkTimePcRelative: 5697 DCHECK(GetCompilerOptions().GetCompilePic()); 5698 break; 5699 case HLoadClass::LoadKind::kBootImageAddress: 5700 break; 5701 case HLoadClass::LoadKind::kDexCacheAddress: 5702 DCHECK(Runtime::Current()->UseJitCompilation()); 5703 break; 5704 case HLoadClass::LoadKind::kDexCachePcRelative: 5705 DCHECK(!Runtime::Current()->UseJitCompilation()); 5706 // We disable pc-relative load when there is an irreducible loop, as the optimization 5707 // is incompatible with it. 5708 // TODO: Create as many ArmDexCacheArraysBase instructions as needed for methods 5709 // with irreducible loops. 5710 if (GetGraph()->HasIrreducibleLoops()) { 5711 return HLoadClass::LoadKind::kDexCacheViaMethod; 5712 } 5713 break; 5714 case HLoadClass::LoadKind::kDexCacheViaMethod: 5715 break; 5716 } 5717 return desired_class_load_kind; 5718} 5719 5720void LocationsBuilderARM::VisitLoadClass(HLoadClass* cls) { 5721 if (cls->NeedsAccessCheck()) { 5722 InvokeRuntimeCallingConvention calling_convention; 5723 CodeGenerator::CreateLoadClassLocationSummary( 5724 cls, 5725 Location::RegisterLocation(calling_convention.GetRegisterAt(0)), 5726 Location::RegisterLocation(R0), 5727 /* code_generator_supports_read_barrier */ true); 5728 return; 5729 } 5730 5731 const bool requires_read_barrier = kEmitCompilerReadBarrier && !cls->IsInBootImage(); 5732 LocationSummary::CallKind call_kind = (cls->NeedsEnvironment() || requires_read_barrier) 5733 ? LocationSummary::kCallOnSlowPath 5734 : LocationSummary::kNoCall; 5735 LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(cls, call_kind); 5736 if (kUseBakerReadBarrier && requires_read_barrier && !cls->NeedsEnvironment()) { 5737 locations->SetCustomSlowPathCallerSaves(RegisterSet::Empty()); // No caller-save registers. 5738 } 5739 5740 HLoadClass::LoadKind load_kind = cls->GetLoadKind(); 5741 if (load_kind == HLoadClass::LoadKind::kReferrersClass || 5742 load_kind == HLoadClass::LoadKind::kDexCacheViaMethod || 5743 load_kind == HLoadClass::LoadKind::kDexCachePcRelative) { 5744 locations->SetInAt(0, Location::RequiresRegister()); 5745 } 5746 locations->SetOut(Location::RequiresRegister()); 5747} 5748 5749void InstructionCodeGeneratorARM::VisitLoadClass(HLoadClass* cls) { 5750 LocationSummary* locations = cls->GetLocations(); 5751 if (cls->NeedsAccessCheck()) { 5752 codegen_->MoveConstant(locations->GetTemp(0), cls->GetTypeIndex()); 5753 codegen_->InvokeRuntime(kQuickInitializeTypeAndVerifyAccess, cls, cls->GetDexPc()); 5754 CheckEntrypointTypes<kQuickInitializeTypeAndVerifyAccess, void*, uint32_t>(); 5755 return; 5756 } 5757 5758 Location out_loc = locations->Out(); 5759 Register out = out_loc.AsRegister<Register>(); 5760 5761 const ReadBarrierOption read_barrier_option = cls->IsInBootImage() 5762 ? kWithoutReadBarrier 5763 : kCompilerReadBarrierOption; 5764 bool generate_null_check = false; 5765 switch (cls->GetLoadKind()) { 5766 case HLoadClass::LoadKind::kReferrersClass: { 5767 DCHECK(!cls->CanCallRuntime()); 5768 DCHECK(!cls->MustGenerateClinitCheck()); 5769 // /* GcRoot<mirror::Class> */ out = current_method->declaring_class_ 5770 Register current_method = locations->InAt(0).AsRegister<Register>(); 5771 GenerateGcRootFieldLoad(cls, 5772 out_loc, 5773 current_method, 5774 ArtMethod::DeclaringClassOffset().Int32Value(), 5775 read_barrier_option); 5776 break; 5777 } 5778 case HLoadClass::LoadKind::kBootImageLinkTimeAddress: { 5779 DCHECK_EQ(read_barrier_option, kWithoutReadBarrier); 5780 __ LoadLiteral(out, codegen_->DeduplicateBootImageTypeLiteral(cls->GetDexFile(), 5781 cls->GetTypeIndex())); 5782 break; 5783 } 5784 case HLoadClass::LoadKind::kBootImageLinkTimePcRelative: { 5785 DCHECK_EQ(read_barrier_option, kWithoutReadBarrier); 5786 CodeGeneratorARM::PcRelativePatchInfo* labels = 5787 codegen_->NewPcRelativeTypePatch(cls->GetDexFile(), cls->GetTypeIndex()); 5788 __ BindTrackedLabel(&labels->movw_label); 5789 __ movw(out, /* placeholder */ 0u); 5790 __ BindTrackedLabel(&labels->movt_label); 5791 __ movt(out, /* placeholder */ 0u); 5792 __ BindTrackedLabel(&labels->add_pc_label); 5793 __ add(out, out, ShifterOperand(PC)); 5794 break; 5795 } 5796 case HLoadClass::LoadKind::kBootImageAddress: { 5797 DCHECK_EQ(read_barrier_option, kWithoutReadBarrier); 5798 DCHECK_NE(cls->GetAddress(), 0u); 5799 uint32_t address = dchecked_integral_cast<uint32_t>(cls->GetAddress()); 5800 __ LoadLiteral(out, codegen_->DeduplicateBootImageAddressLiteral(address)); 5801 break; 5802 } 5803 case HLoadClass::LoadKind::kDexCacheAddress: { 5804 DCHECK_NE(cls->GetAddress(), 0u); 5805 uint32_t address = dchecked_integral_cast<uint32_t>(cls->GetAddress()); 5806 // 16-bit LDR immediate has a 5-bit offset multiplied by the size and that gives 5807 // a 128B range. To try and reduce the number of literals if we load multiple types, 5808 // simply split the dex cache address to a 128B aligned base loaded from a literal 5809 // and the remaining offset embedded in the load. 5810 static_assert(sizeof(GcRoot<mirror::Class>) == 4u, "Expected GC root to be 4 bytes."); 5811 DCHECK_ALIGNED(cls->GetAddress(), 4u); 5812 constexpr size_t offset_bits = /* encoded bits */ 5 + /* scale */ 2; 5813 uint32_t base_address = address & ~MaxInt<uint32_t>(offset_bits); 5814 uint32_t offset = address & MaxInt<uint32_t>(offset_bits); 5815 __ LoadLiteral(out, codegen_->DeduplicateDexCacheAddressLiteral(base_address)); 5816 // /* GcRoot<mirror::Class> */ out = *(base_address + offset) 5817 GenerateGcRootFieldLoad(cls, out_loc, out, offset, read_barrier_option); 5818 generate_null_check = !cls->IsInDexCache(); 5819 break; 5820 } 5821 case HLoadClass::LoadKind::kDexCachePcRelative: { 5822 Register base_reg = locations->InAt(0).AsRegister<Register>(); 5823 HArmDexCacheArraysBase* base = cls->InputAt(0)->AsArmDexCacheArraysBase(); 5824 int32_t offset = cls->GetDexCacheElementOffset() - base->GetElementOffset(); 5825 // /* GcRoot<mirror::Class> */ out = *(dex_cache_arrays_base + offset) 5826 GenerateGcRootFieldLoad(cls, out_loc, base_reg, offset, read_barrier_option); 5827 generate_null_check = !cls->IsInDexCache(); 5828 break; 5829 } 5830 case HLoadClass::LoadKind::kDexCacheViaMethod: { 5831 // /* GcRoot<mirror::Class>[] */ out = 5832 // current_method.ptr_sized_fields_->dex_cache_resolved_types_ 5833 Register current_method = locations->InAt(0).AsRegister<Register>(); 5834 __ LoadFromOffset(kLoadWord, 5835 out, 5836 current_method, 5837 ArtMethod::DexCacheResolvedTypesOffset(kArmPointerSize).Int32Value()); 5838 // /* GcRoot<mirror::Class> */ out = out[type_index] 5839 size_t offset = CodeGenerator::GetCacheOffset(cls->GetTypeIndex()); 5840 GenerateGcRootFieldLoad(cls, out_loc, out, offset, read_barrier_option); 5841 generate_null_check = !cls->IsInDexCache(); 5842 } 5843 } 5844 5845 if (generate_null_check || cls->MustGenerateClinitCheck()) { 5846 DCHECK(cls->CanCallRuntime()); 5847 SlowPathCodeARM* slow_path = new (GetGraph()->GetArena()) LoadClassSlowPathARM( 5848 cls, cls, cls->GetDexPc(), cls->MustGenerateClinitCheck()); 5849 codegen_->AddSlowPath(slow_path); 5850 if (generate_null_check) { 5851 __ CompareAndBranchIfZero(out, slow_path->GetEntryLabel()); 5852 } 5853 if (cls->MustGenerateClinitCheck()) { 5854 GenerateClassInitializationCheck(slow_path, out); 5855 } else { 5856 __ Bind(slow_path->GetExitLabel()); 5857 } 5858 } 5859} 5860 5861void LocationsBuilderARM::VisitClinitCheck(HClinitCheck* check) { 5862 LocationSummary* locations = 5863 new (GetGraph()->GetArena()) LocationSummary(check, LocationSummary::kCallOnSlowPath); 5864 locations->SetInAt(0, Location::RequiresRegister()); 5865 if (check->HasUses()) { 5866 locations->SetOut(Location::SameAsFirstInput()); 5867 } 5868} 5869 5870void InstructionCodeGeneratorARM::VisitClinitCheck(HClinitCheck* check) { 5871 // We assume the class is not null. 5872 SlowPathCodeARM* slow_path = new (GetGraph()->GetArena()) LoadClassSlowPathARM( 5873 check->GetLoadClass(), check, check->GetDexPc(), true); 5874 codegen_->AddSlowPath(slow_path); 5875 GenerateClassInitializationCheck(slow_path, 5876 check->GetLocations()->InAt(0).AsRegister<Register>()); 5877} 5878 5879void InstructionCodeGeneratorARM::GenerateClassInitializationCheck( 5880 SlowPathCodeARM* slow_path, Register class_reg) { 5881 __ LoadFromOffset(kLoadWord, IP, class_reg, mirror::Class::StatusOffset().Int32Value()); 5882 __ cmp(IP, ShifterOperand(mirror::Class::kStatusInitialized)); 5883 __ b(slow_path->GetEntryLabel(), LT); 5884 // Even if the initialized flag is set, we may be in a situation where caches are not synced 5885 // properly. Therefore, we do a memory fence. 5886 __ dmb(ISH); 5887 __ Bind(slow_path->GetExitLabel()); 5888} 5889 5890HLoadString::LoadKind CodeGeneratorARM::GetSupportedLoadStringKind( 5891 HLoadString::LoadKind desired_string_load_kind) { 5892 switch (desired_string_load_kind) { 5893 case HLoadString::LoadKind::kBootImageLinkTimeAddress: 5894 DCHECK(!GetCompilerOptions().GetCompilePic()); 5895 break; 5896 case HLoadString::LoadKind::kBootImageLinkTimePcRelative: 5897 DCHECK(GetCompilerOptions().GetCompilePic()); 5898 break; 5899 case HLoadString::LoadKind::kBootImageAddress: 5900 break; 5901 case HLoadString::LoadKind::kBssEntry: 5902 DCHECK(!Runtime::Current()->UseJitCompilation()); 5903 break; 5904 case HLoadString::LoadKind::kDexCacheViaMethod: 5905 break; 5906 } 5907 return desired_string_load_kind; 5908} 5909 5910void LocationsBuilderARM::VisitLoadString(HLoadString* load) { 5911 LocationSummary::CallKind call_kind = load->NeedsEnvironment() 5912 ? ((load->GetLoadKind() == HLoadString::LoadKind::kDexCacheViaMethod) 5913 ? LocationSummary::kCallOnMainOnly 5914 : LocationSummary::kCallOnSlowPath) 5915 : LocationSummary::kNoCall; 5916 LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(load, call_kind); 5917 5918 HLoadString::LoadKind load_kind = load->GetLoadKind(); 5919 if (load_kind == HLoadString::LoadKind::kDexCacheViaMethod) { 5920 locations->SetOut(Location::RegisterLocation(R0)); 5921 } else { 5922 locations->SetOut(Location::RequiresRegister()); 5923 if (load_kind == HLoadString::LoadKind::kBssEntry) { 5924 if (!kUseReadBarrier || kUseBakerReadBarrier) { 5925 // Rely on the pResolveString and/or marking to save everything, including temps. 5926 // Note that IP may theoretically be clobbered by saving/restoring the live register 5927 // (only one thanks to the custom calling convention), so we request a different temp. 5928 locations->AddTemp(Location::RequiresRegister()); 5929 RegisterSet caller_saves = RegisterSet::Empty(); 5930 InvokeRuntimeCallingConvention calling_convention; 5931 caller_saves.Add(Location::RegisterLocation(calling_convention.GetRegisterAt(0))); 5932 // TODO: Add GetReturnLocation() to the calling convention so that we can DCHECK() 5933 // that the the kPrimNot result register is the same as the first argument register. 5934 locations->SetCustomSlowPathCallerSaves(caller_saves); 5935 } else { 5936 // For non-Baker read barrier we have a temp-clobbering call. 5937 } 5938 } 5939 } 5940} 5941 5942void InstructionCodeGeneratorARM::VisitLoadString(HLoadString* load) { 5943 LocationSummary* locations = load->GetLocations(); 5944 Location out_loc = locations->Out(); 5945 Register out = out_loc.AsRegister<Register>(); 5946 HLoadString::LoadKind load_kind = load->GetLoadKind(); 5947 5948 switch (load_kind) { 5949 case HLoadString::LoadKind::kBootImageLinkTimeAddress: { 5950 __ LoadLiteral(out, codegen_->DeduplicateBootImageStringLiteral(load->GetDexFile(), 5951 load->GetStringIndex())); 5952 return; // No dex cache slow path. 5953 } 5954 case HLoadString::LoadKind::kBootImageLinkTimePcRelative: { 5955 DCHECK(codegen_->GetCompilerOptions().IsBootImage()); 5956 CodeGeneratorARM::PcRelativePatchInfo* labels = 5957 codegen_->NewPcRelativeStringPatch(load->GetDexFile(), load->GetStringIndex()); 5958 __ BindTrackedLabel(&labels->movw_label); 5959 __ movw(out, /* placeholder */ 0u); 5960 __ BindTrackedLabel(&labels->movt_label); 5961 __ movt(out, /* placeholder */ 0u); 5962 __ BindTrackedLabel(&labels->add_pc_label); 5963 __ add(out, out, ShifterOperand(PC)); 5964 return; // No dex cache slow path. 5965 } 5966 case HLoadString::LoadKind::kBootImageAddress: { 5967 DCHECK_NE(load->GetAddress(), 0u); 5968 uint32_t address = dchecked_integral_cast<uint32_t>(load->GetAddress()); 5969 __ LoadLiteral(out, codegen_->DeduplicateBootImageAddressLiteral(address)); 5970 return; // No dex cache slow path. 5971 } 5972 case HLoadString::LoadKind::kBssEntry: { 5973 DCHECK(!codegen_->GetCompilerOptions().IsBootImage()); 5974 Register temp = locations->GetTemp(0).AsRegister<Register>(); 5975 CodeGeneratorARM::PcRelativePatchInfo* labels = 5976 codegen_->NewPcRelativeStringPatch(load->GetDexFile(), load->GetStringIndex()); 5977 __ BindTrackedLabel(&labels->movw_label); 5978 __ movw(temp, /* placeholder */ 0u); 5979 __ BindTrackedLabel(&labels->movt_label); 5980 __ movt(temp, /* placeholder */ 0u); 5981 __ BindTrackedLabel(&labels->add_pc_label); 5982 __ add(temp, temp, ShifterOperand(PC)); 5983 GenerateGcRootFieldLoad(load, out_loc, temp, /* offset */ 0, kCompilerReadBarrierOption); 5984 SlowPathCode* slow_path = new (GetGraph()->GetArena()) LoadStringSlowPathARM(load); 5985 codegen_->AddSlowPath(slow_path); 5986 __ CompareAndBranchIfZero(out, slow_path->GetEntryLabel()); 5987 __ Bind(slow_path->GetExitLabel()); 5988 return; 5989 } 5990 default: 5991 break; 5992 } 5993 5994 // TODO: Consider re-adding the compiler code to do string dex cache lookup again. 5995 DCHECK(load_kind == HLoadString::LoadKind::kDexCacheViaMethod); 5996 InvokeRuntimeCallingConvention calling_convention; 5997 DCHECK_EQ(calling_convention.GetRegisterAt(0), out); 5998 __ LoadImmediate(calling_convention.GetRegisterAt(0), load->GetStringIndex()); 5999 codegen_->InvokeRuntime(kQuickResolveString, load, load->GetDexPc()); 6000 CheckEntrypointTypes<kQuickResolveString, void*, uint32_t>(); 6001} 6002 6003static int32_t GetExceptionTlsOffset() { 6004 return Thread::ExceptionOffset<kArmPointerSize>().Int32Value(); 6005} 6006 6007void LocationsBuilderARM::VisitLoadException(HLoadException* load) { 6008 LocationSummary* locations = 6009 new (GetGraph()->GetArena()) LocationSummary(load, LocationSummary::kNoCall); 6010 locations->SetOut(Location::RequiresRegister()); 6011} 6012 6013void InstructionCodeGeneratorARM::VisitLoadException(HLoadException* load) { 6014 Register out = load->GetLocations()->Out().AsRegister<Register>(); 6015 __ LoadFromOffset(kLoadWord, out, TR, GetExceptionTlsOffset()); 6016} 6017 6018void LocationsBuilderARM::VisitClearException(HClearException* clear) { 6019 new (GetGraph()->GetArena()) LocationSummary(clear, LocationSummary::kNoCall); 6020} 6021 6022void InstructionCodeGeneratorARM::VisitClearException(HClearException* clear ATTRIBUTE_UNUSED) { 6023 __ LoadImmediate(IP, 0); 6024 __ StoreToOffset(kStoreWord, IP, TR, GetExceptionTlsOffset()); 6025} 6026 6027void LocationsBuilderARM::VisitThrow(HThrow* instruction) { 6028 LocationSummary* locations = 6029 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCallOnMainOnly); 6030 InvokeRuntimeCallingConvention calling_convention; 6031 locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(0))); 6032} 6033 6034void InstructionCodeGeneratorARM::VisitThrow(HThrow* instruction) { 6035 codegen_->InvokeRuntime(kQuickDeliverException, instruction, instruction->GetDexPc()); 6036 CheckEntrypointTypes<kQuickDeliverException, void, mirror::Object*>(); 6037} 6038 6039// Temp is used for read barrier. 6040static size_t NumberOfInstanceOfTemps(TypeCheckKind type_check_kind) { 6041 if (kEmitCompilerReadBarrier && 6042 (kUseBakerReadBarrier || 6043 type_check_kind == TypeCheckKind::kAbstractClassCheck || 6044 type_check_kind == TypeCheckKind::kClassHierarchyCheck || 6045 type_check_kind == TypeCheckKind::kArrayObjectCheck)) { 6046 return 1; 6047 } 6048 return 0; 6049} 6050 6051// Interface case has 3 temps, one for holding the number of interfaces, one for the current 6052// interface pointer, one for loading the current interface. 6053// The other checks have one temp for loading the object's class. 6054static size_t NumberOfCheckCastTemps(TypeCheckKind type_check_kind) { 6055 if (type_check_kind == TypeCheckKind::kInterfaceCheck) { 6056 return 3; 6057 } 6058 return 1 + NumberOfInstanceOfTemps(type_check_kind); 6059} 6060 6061void LocationsBuilderARM::VisitInstanceOf(HInstanceOf* instruction) { 6062 LocationSummary::CallKind call_kind = LocationSummary::kNoCall; 6063 TypeCheckKind type_check_kind = instruction->GetTypeCheckKind(); 6064 bool baker_read_barrier_slow_path = false; 6065 switch (type_check_kind) { 6066 case TypeCheckKind::kExactCheck: 6067 case TypeCheckKind::kAbstractClassCheck: 6068 case TypeCheckKind::kClassHierarchyCheck: 6069 case TypeCheckKind::kArrayObjectCheck: 6070 call_kind = 6071 kEmitCompilerReadBarrier ? LocationSummary::kCallOnSlowPath : LocationSummary::kNoCall; 6072 baker_read_barrier_slow_path = kUseBakerReadBarrier; 6073 break; 6074 case TypeCheckKind::kArrayCheck: 6075 case TypeCheckKind::kUnresolvedCheck: 6076 case TypeCheckKind::kInterfaceCheck: 6077 call_kind = LocationSummary::kCallOnSlowPath; 6078 break; 6079 } 6080 6081 LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction, call_kind); 6082 if (baker_read_barrier_slow_path) { 6083 locations->SetCustomSlowPathCallerSaves(RegisterSet::Empty()); // No caller-save registers. 6084 } 6085 locations->SetInAt(0, Location::RequiresRegister()); 6086 locations->SetInAt(1, Location::RequiresRegister()); 6087 // The "out" register is used as a temporary, so it overlaps with the inputs. 6088 // Note that TypeCheckSlowPathARM uses this register too. 6089 locations->SetOut(Location::RequiresRegister(), Location::kOutputOverlap); 6090 locations->AddRegisterTemps(NumberOfInstanceOfTemps(type_check_kind)); 6091} 6092 6093void InstructionCodeGeneratorARM::VisitInstanceOf(HInstanceOf* instruction) { 6094 TypeCheckKind type_check_kind = instruction->GetTypeCheckKind(); 6095 LocationSummary* locations = instruction->GetLocations(); 6096 Location obj_loc = locations->InAt(0); 6097 Register obj = obj_loc.AsRegister<Register>(); 6098 Register cls = locations->InAt(1).AsRegister<Register>(); 6099 Location out_loc = locations->Out(); 6100 Register out = out_loc.AsRegister<Register>(); 6101 const size_t num_temps = NumberOfInstanceOfTemps(type_check_kind); 6102 DCHECK_LE(num_temps, 1u); 6103 Location maybe_temp_loc = (num_temps >= 1) ? locations->GetTemp(0) : Location::NoLocation(); 6104 uint32_t class_offset = mirror::Object::ClassOffset().Int32Value(); 6105 uint32_t super_offset = mirror::Class::SuperClassOffset().Int32Value(); 6106 uint32_t component_offset = mirror::Class::ComponentTypeOffset().Int32Value(); 6107 uint32_t primitive_offset = mirror::Class::PrimitiveTypeOffset().Int32Value(); 6108 Label done, zero; 6109 SlowPathCodeARM* slow_path = nullptr; 6110 6111 // Return 0 if `obj` is null. 6112 // avoid null check if we know obj is not null. 6113 if (instruction->MustDoNullCheck()) { 6114 __ CompareAndBranchIfZero(obj, &zero); 6115 } 6116 6117 // /* HeapReference<Class> */ out = obj->klass_ 6118 GenerateReferenceLoadTwoRegisters(instruction, 6119 out_loc, 6120 obj_loc, 6121 class_offset, 6122 maybe_temp_loc, 6123 kCompilerReadBarrierOption); 6124 6125 switch (type_check_kind) { 6126 case TypeCheckKind::kExactCheck: { 6127 __ cmp(out, ShifterOperand(cls)); 6128 // Classes must be equal for the instanceof to succeed. 6129 __ b(&zero, NE); 6130 __ LoadImmediate(out, 1); 6131 __ b(&done); 6132 break; 6133 } 6134 6135 case TypeCheckKind::kAbstractClassCheck: { 6136 // If the class is abstract, we eagerly fetch the super class of the 6137 // object to avoid doing a comparison we know will fail. 6138 Label loop; 6139 __ Bind(&loop); 6140 // /* HeapReference<Class> */ out = out->super_class_ 6141 GenerateReferenceLoadOneRegister(instruction, 6142 out_loc, 6143 super_offset, 6144 maybe_temp_loc, 6145 kCompilerReadBarrierOption); 6146 // If `out` is null, we use it for the result, and jump to `done`. 6147 __ CompareAndBranchIfZero(out, &done); 6148 __ cmp(out, ShifterOperand(cls)); 6149 __ b(&loop, NE); 6150 __ LoadImmediate(out, 1); 6151 if (zero.IsLinked()) { 6152 __ b(&done); 6153 } 6154 break; 6155 } 6156 6157 case TypeCheckKind::kClassHierarchyCheck: { 6158 // Walk over the class hierarchy to find a match. 6159 Label loop, success; 6160 __ Bind(&loop); 6161 __ cmp(out, ShifterOperand(cls)); 6162 __ b(&success, EQ); 6163 // /* HeapReference<Class> */ out = out->super_class_ 6164 GenerateReferenceLoadOneRegister(instruction, 6165 out_loc, 6166 super_offset, 6167 maybe_temp_loc, 6168 kCompilerReadBarrierOption); 6169 __ CompareAndBranchIfNonZero(out, &loop); 6170 // If `out` is null, we use it for the result, and jump to `done`. 6171 __ b(&done); 6172 __ Bind(&success); 6173 __ LoadImmediate(out, 1); 6174 if (zero.IsLinked()) { 6175 __ b(&done); 6176 } 6177 break; 6178 } 6179 6180 case TypeCheckKind::kArrayObjectCheck: { 6181 // Do an exact check. 6182 Label exact_check; 6183 __ cmp(out, ShifterOperand(cls)); 6184 __ b(&exact_check, EQ); 6185 // Otherwise, we need to check that the object's class is a non-primitive array. 6186 // /* HeapReference<Class> */ out = out->component_type_ 6187 GenerateReferenceLoadOneRegister(instruction, 6188 out_loc, 6189 component_offset, 6190 maybe_temp_loc, 6191 kCompilerReadBarrierOption); 6192 // If `out` is null, we use it for the result, and jump to `done`. 6193 __ CompareAndBranchIfZero(out, &done); 6194 __ LoadFromOffset(kLoadUnsignedHalfword, out, out, primitive_offset); 6195 static_assert(Primitive::kPrimNot == 0, "Expected 0 for kPrimNot"); 6196 __ CompareAndBranchIfNonZero(out, &zero); 6197 __ Bind(&exact_check); 6198 __ LoadImmediate(out, 1); 6199 __ b(&done); 6200 break; 6201 } 6202 6203 case TypeCheckKind::kArrayCheck: { 6204 __ cmp(out, ShifterOperand(cls)); 6205 DCHECK(locations->OnlyCallsOnSlowPath()); 6206 slow_path = new (GetGraph()->GetArena()) TypeCheckSlowPathARM(instruction, 6207 /* is_fatal */ false); 6208 codegen_->AddSlowPath(slow_path); 6209 __ b(slow_path->GetEntryLabel(), NE); 6210 __ LoadImmediate(out, 1); 6211 if (zero.IsLinked()) { 6212 __ b(&done); 6213 } 6214 break; 6215 } 6216 6217 case TypeCheckKind::kUnresolvedCheck: 6218 case TypeCheckKind::kInterfaceCheck: { 6219 // Note that we indeed only call on slow path, but we always go 6220 // into the slow path for the unresolved and interface check 6221 // cases. 6222 // 6223 // We cannot directly call the InstanceofNonTrivial runtime 6224 // entry point without resorting to a type checking slow path 6225 // here (i.e. by calling InvokeRuntime directly), as it would 6226 // require to assign fixed registers for the inputs of this 6227 // HInstanceOf instruction (following the runtime calling 6228 // convention), which might be cluttered by the potential first 6229 // read barrier emission at the beginning of this method. 6230 // 6231 // TODO: Introduce a new runtime entry point taking the object 6232 // to test (instead of its class) as argument, and let it deal 6233 // with the read barrier issues. This will let us refactor this 6234 // case of the `switch` code as it was previously (with a direct 6235 // call to the runtime not using a type checking slow path). 6236 // This should also be beneficial for the other cases above. 6237 DCHECK(locations->OnlyCallsOnSlowPath()); 6238 slow_path = new (GetGraph()->GetArena()) TypeCheckSlowPathARM(instruction, 6239 /* is_fatal */ false); 6240 codegen_->AddSlowPath(slow_path); 6241 __ b(slow_path->GetEntryLabel()); 6242 if (zero.IsLinked()) { 6243 __ b(&done); 6244 } 6245 break; 6246 } 6247 } 6248 6249 if (zero.IsLinked()) { 6250 __ Bind(&zero); 6251 __ LoadImmediate(out, 0); 6252 } 6253 6254 if (done.IsLinked()) { 6255 __ Bind(&done); 6256 } 6257 6258 if (slow_path != nullptr) { 6259 __ Bind(slow_path->GetExitLabel()); 6260 } 6261} 6262 6263void LocationsBuilderARM::VisitCheckCast(HCheckCast* instruction) { 6264 LocationSummary::CallKind call_kind = LocationSummary::kNoCall; 6265 bool throws_into_catch = instruction->CanThrowIntoCatchBlock(); 6266 6267 TypeCheckKind type_check_kind = instruction->GetTypeCheckKind(); 6268 switch (type_check_kind) { 6269 case TypeCheckKind::kExactCheck: 6270 case TypeCheckKind::kAbstractClassCheck: 6271 case TypeCheckKind::kClassHierarchyCheck: 6272 case TypeCheckKind::kArrayObjectCheck: 6273 call_kind = (throws_into_catch || kEmitCompilerReadBarrier) ? 6274 LocationSummary::kCallOnSlowPath : 6275 LocationSummary::kNoCall; // In fact, call on a fatal (non-returning) slow path. 6276 break; 6277 case TypeCheckKind::kArrayCheck: 6278 case TypeCheckKind::kUnresolvedCheck: 6279 case TypeCheckKind::kInterfaceCheck: 6280 call_kind = LocationSummary::kCallOnSlowPath; 6281 break; 6282 } 6283 6284 LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction, call_kind); 6285 locations->SetInAt(0, Location::RequiresRegister()); 6286 locations->SetInAt(1, Location::RequiresRegister()); 6287 locations->AddRegisterTemps(NumberOfCheckCastTemps(type_check_kind)); 6288} 6289 6290void InstructionCodeGeneratorARM::VisitCheckCast(HCheckCast* instruction) { 6291 TypeCheckKind type_check_kind = instruction->GetTypeCheckKind(); 6292 LocationSummary* locations = instruction->GetLocations(); 6293 Location obj_loc = locations->InAt(0); 6294 Register obj = obj_loc.AsRegister<Register>(); 6295 Register cls = locations->InAt(1).AsRegister<Register>(); 6296 Location temp_loc = locations->GetTemp(0); 6297 Register temp = temp_loc.AsRegister<Register>(); 6298 const size_t num_temps = NumberOfCheckCastTemps(type_check_kind); 6299 DCHECK_LE(num_temps, 3u); 6300 Location maybe_temp2_loc = (num_temps >= 2) ? locations->GetTemp(1) : Location::NoLocation(); 6301 Location maybe_temp3_loc = (num_temps >= 3) ? locations->GetTemp(2) : Location::NoLocation(); 6302 const uint32_t class_offset = mirror::Object::ClassOffset().Int32Value(); 6303 const uint32_t super_offset = mirror::Class::SuperClassOffset().Int32Value(); 6304 const uint32_t component_offset = mirror::Class::ComponentTypeOffset().Int32Value(); 6305 const uint32_t primitive_offset = mirror::Class::PrimitiveTypeOffset().Int32Value(); 6306 const uint32_t iftable_offset = mirror::Class::IfTableOffset().Uint32Value(); 6307 const uint32_t array_length_offset = mirror::Array::LengthOffset().Uint32Value(); 6308 const uint32_t object_array_data_offset = 6309 mirror::Array::DataOffset(kHeapReferenceSize).Uint32Value(); 6310 6311 // Always false for read barriers since we may need to go to the entrypoint for non-fatal cases 6312 // from false negatives. The false negatives may come from avoiding read barriers below. Avoiding 6313 // read barriers is done for performance and code size reasons. 6314 bool is_type_check_slow_path_fatal = false; 6315 if (!kEmitCompilerReadBarrier) { 6316 is_type_check_slow_path_fatal = 6317 (type_check_kind == TypeCheckKind::kExactCheck || 6318 type_check_kind == TypeCheckKind::kAbstractClassCheck || 6319 type_check_kind == TypeCheckKind::kClassHierarchyCheck || 6320 type_check_kind == TypeCheckKind::kArrayObjectCheck) && 6321 !instruction->CanThrowIntoCatchBlock(); 6322 } 6323 SlowPathCodeARM* type_check_slow_path = 6324 new (GetGraph()->GetArena()) TypeCheckSlowPathARM(instruction, 6325 is_type_check_slow_path_fatal); 6326 codegen_->AddSlowPath(type_check_slow_path); 6327 6328 Label done; 6329 // Avoid null check if we know obj is not null. 6330 if (instruction->MustDoNullCheck()) { 6331 __ CompareAndBranchIfZero(obj, &done); 6332 } 6333 6334 switch (type_check_kind) { 6335 case TypeCheckKind::kExactCheck: 6336 case TypeCheckKind::kArrayCheck: { 6337 // /* HeapReference<Class> */ temp = obj->klass_ 6338 GenerateReferenceLoadTwoRegisters(instruction, 6339 temp_loc, 6340 obj_loc, 6341 class_offset, 6342 maybe_temp2_loc, 6343 kWithoutReadBarrier); 6344 6345 __ cmp(temp, ShifterOperand(cls)); 6346 // Jump to slow path for throwing the exception or doing a 6347 // more involved array check. 6348 __ b(type_check_slow_path->GetEntryLabel(), NE); 6349 break; 6350 } 6351 6352 case TypeCheckKind::kAbstractClassCheck: { 6353 // /* HeapReference<Class> */ temp = obj->klass_ 6354 GenerateReferenceLoadTwoRegisters(instruction, 6355 temp_loc, 6356 obj_loc, 6357 class_offset, 6358 maybe_temp2_loc, 6359 kWithoutReadBarrier); 6360 6361 // If the class is abstract, we eagerly fetch the super class of the 6362 // object to avoid doing a comparison we know will fail. 6363 Label loop; 6364 __ Bind(&loop); 6365 // /* HeapReference<Class> */ temp = temp->super_class_ 6366 GenerateReferenceLoadOneRegister(instruction, 6367 temp_loc, 6368 super_offset, 6369 maybe_temp2_loc, 6370 kWithoutReadBarrier); 6371 6372 // If the class reference currently in `temp` is null, jump to the slow path to throw the 6373 // exception. 6374 __ CompareAndBranchIfZero(temp, type_check_slow_path->GetEntryLabel()); 6375 6376 // Otherwise, compare the classes. 6377 __ cmp(temp, ShifterOperand(cls)); 6378 __ b(&loop, NE); 6379 break; 6380 } 6381 6382 case TypeCheckKind::kClassHierarchyCheck: { 6383 // /* HeapReference<Class> */ temp = obj->klass_ 6384 GenerateReferenceLoadTwoRegisters(instruction, 6385 temp_loc, 6386 obj_loc, 6387 class_offset, 6388 maybe_temp2_loc, 6389 kWithoutReadBarrier); 6390 6391 // Walk over the class hierarchy to find a match. 6392 Label loop; 6393 __ Bind(&loop); 6394 __ cmp(temp, ShifterOperand(cls)); 6395 __ b(&done, EQ); 6396 6397 // /* HeapReference<Class> */ temp = temp->super_class_ 6398 GenerateReferenceLoadOneRegister(instruction, 6399 temp_loc, 6400 super_offset, 6401 maybe_temp2_loc, 6402 kWithoutReadBarrier); 6403 6404 // If the class reference currently in `temp` is null, jump to the slow path to throw the 6405 // exception. 6406 __ CompareAndBranchIfZero(temp, type_check_slow_path->GetEntryLabel()); 6407 // Otherwise, jump to the beginning of the loop. 6408 __ b(&loop); 6409 break; 6410 } 6411 6412 case TypeCheckKind::kArrayObjectCheck: { 6413 // /* HeapReference<Class> */ temp = obj->klass_ 6414 GenerateReferenceLoadTwoRegisters(instruction, 6415 temp_loc, 6416 obj_loc, 6417 class_offset, 6418 maybe_temp2_loc, 6419 kWithoutReadBarrier); 6420 6421 // Do an exact check. 6422 __ cmp(temp, ShifterOperand(cls)); 6423 __ b(&done, EQ); 6424 6425 // Otherwise, we need to check that the object's class is a non-primitive array. 6426 // /* HeapReference<Class> */ temp = temp->component_type_ 6427 GenerateReferenceLoadOneRegister(instruction, 6428 temp_loc, 6429 component_offset, 6430 maybe_temp2_loc, 6431 kWithoutReadBarrier); 6432 // If the component type is null, jump to the slow path to throw the exception. 6433 __ CompareAndBranchIfZero(temp, type_check_slow_path->GetEntryLabel()); 6434 // Otherwise,the object is indeed an array, jump to label `check_non_primitive_component_type` 6435 // to further check that this component type is not a primitive type. 6436 __ LoadFromOffset(kLoadUnsignedHalfword, temp, temp, primitive_offset); 6437 static_assert(Primitive::kPrimNot == 0, "Expected 0 for art::Primitive::kPrimNot"); 6438 __ CompareAndBranchIfNonZero(temp, type_check_slow_path->GetEntryLabel()); 6439 break; 6440 } 6441 6442 case TypeCheckKind::kUnresolvedCheck: 6443 // We always go into the type check slow path for the unresolved check case. 6444 // We cannot directly call the CheckCast runtime entry point 6445 // without resorting to a type checking slow path here (i.e. by 6446 // calling InvokeRuntime directly), as it would require to 6447 // assign fixed registers for the inputs of this HInstanceOf 6448 // instruction (following the runtime calling convention), which 6449 // might be cluttered by the potential first read barrier 6450 // emission at the beginning of this method. 6451 6452 __ b(type_check_slow_path->GetEntryLabel()); 6453 break; 6454 6455 case TypeCheckKind::kInterfaceCheck: { 6456 // Avoid read barriers to improve performance of the fast path. We can not get false 6457 // positives by doing this. 6458 // /* HeapReference<Class> */ temp = obj->klass_ 6459 GenerateReferenceLoadTwoRegisters(instruction, 6460 temp_loc, 6461 obj_loc, 6462 class_offset, 6463 maybe_temp2_loc, 6464 kWithoutReadBarrier); 6465 6466 // /* HeapReference<Class> */ temp = temp->iftable_ 6467 GenerateReferenceLoadTwoRegisters(instruction, 6468 temp_loc, 6469 temp_loc, 6470 iftable_offset, 6471 maybe_temp2_loc, 6472 kWithoutReadBarrier); 6473 Label is_null; 6474 // Null iftable means it is empty and will always fail the check. 6475 // Not cbz since the temp may not be a low register. 6476 __ CompareAndBranchIfZero(temp, &is_null); 6477 6478 // Loop through the iftable and check if any class matches. 6479 __ ldr(maybe_temp2_loc.AsRegister<Register>(), Address(temp, array_length_offset)); 6480 6481 Label start_loop; 6482 __ Bind(&start_loop); 6483 __ ldr(maybe_temp3_loc.AsRegister<Register>(), Address(temp, object_array_data_offset)); 6484 __ MaybeUnpoisonHeapReference(maybe_temp3_loc.AsRegister<Register>()); 6485 __ cmp(cls, ShifterOperand(maybe_temp3_loc.AsRegister<Register>())); 6486 __ b(&done, EQ); // Return if same class. 6487 // Go to next interface. 6488 __ add(temp, temp, ShifterOperand(2 * kHeapReferenceSize)); 6489 __ sub(maybe_temp2_loc.AsRegister<Register>(), 6490 maybe_temp2_loc.AsRegister<Register>(), 6491 ShifterOperand(2)); 6492 // Not cbnz since the temp may not be a low register. 6493 __ CompareAndBranchIfNonZero(maybe_temp2_loc.AsRegister<Register>(), &start_loop); 6494 __ Bind(&is_null); 6495 6496 __ b(type_check_slow_path->GetEntryLabel()); 6497 break; 6498 } 6499 } 6500 __ Bind(&done); 6501 6502 __ Bind(type_check_slow_path->GetExitLabel()); 6503} 6504 6505void LocationsBuilderARM::VisitMonitorOperation(HMonitorOperation* instruction) { 6506 LocationSummary* locations = 6507 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCallOnMainOnly); 6508 InvokeRuntimeCallingConvention calling_convention; 6509 locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(0))); 6510} 6511 6512void InstructionCodeGeneratorARM::VisitMonitorOperation(HMonitorOperation* instruction) { 6513 codegen_->InvokeRuntime(instruction->IsEnter() ? kQuickLockObject : kQuickUnlockObject, 6514 instruction, 6515 instruction->GetDexPc()); 6516 if (instruction->IsEnter()) { 6517 CheckEntrypointTypes<kQuickLockObject, void, mirror::Object*>(); 6518 } else { 6519 CheckEntrypointTypes<kQuickUnlockObject, void, mirror::Object*>(); 6520 } 6521} 6522 6523void LocationsBuilderARM::VisitAnd(HAnd* instruction) { HandleBitwiseOperation(instruction, AND); } 6524void LocationsBuilderARM::VisitOr(HOr* instruction) { HandleBitwiseOperation(instruction, ORR); } 6525void LocationsBuilderARM::VisitXor(HXor* instruction) { HandleBitwiseOperation(instruction, EOR); } 6526 6527void LocationsBuilderARM::HandleBitwiseOperation(HBinaryOperation* instruction, Opcode opcode) { 6528 LocationSummary* locations = 6529 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall); 6530 DCHECK(instruction->GetResultType() == Primitive::kPrimInt 6531 || instruction->GetResultType() == Primitive::kPrimLong); 6532 // Note: GVN reorders commutative operations to have the constant on the right hand side. 6533 locations->SetInAt(0, Location::RequiresRegister()); 6534 locations->SetInAt(1, ArmEncodableConstantOrRegister(instruction->InputAt(1), opcode)); 6535 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 6536} 6537 6538void InstructionCodeGeneratorARM::VisitAnd(HAnd* instruction) { 6539 HandleBitwiseOperation(instruction); 6540} 6541 6542void InstructionCodeGeneratorARM::VisitOr(HOr* instruction) { 6543 HandleBitwiseOperation(instruction); 6544} 6545 6546void InstructionCodeGeneratorARM::VisitXor(HXor* instruction) { 6547 HandleBitwiseOperation(instruction); 6548} 6549 6550 6551void LocationsBuilderARM::VisitBitwiseNegatedRight(HBitwiseNegatedRight* instruction) { 6552 LocationSummary* locations = 6553 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall); 6554 DCHECK(instruction->GetResultType() == Primitive::kPrimInt 6555 || instruction->GetResultType() == Primitive::kPrimLong); 6556 6557 locations->SetInAt(0, Location::RequiresRegister()); 6558 locations->SetInAt(1, Location::RequiresRegister()); 6559 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 6560} 6561 6562void InstructionCodeGeneratorARM::VisitBitwiseNegatedRight(HBitwiseNegatedRight* instruction) { 6563 LocationSummary* locations = instruction->GetLocations(); 6564 Location first = locations->InAt(0); 6565 Location second = locations->InAt(1); 6566 Location out = locations->Out(); 6567 6568 if (instruction->GetResultType() == Primitive::kPrimInt) { 6569 Register first_reg = first.AsRegister<Register>(); 6570 ShifterOperand second_reg(second.AsRegister<Register>()); 6571 Register out_reg = out.AsRegister<Register>(); 6572 6573 switch (instruction->GetOpKind()) { 6574 case HInstruction::kAnd: 6575 __ bic(out_reg, first_reg, second_reg); 6576 break; 6577 case HInstruction::kOr: 6578 __ orn(out_reg, first_reg, second_reg); 6579 break; 6580 // There is no EON on arm. 6581 case HInstruction::kXor: 6582 default: 6583 LOG(FATAL) << "Unexpected instruction " << instruction->DebugName(); 6584 UNREACHABLE(); 6585 } 6586 return; 6587 6588 } else { 6589 DCHECK_EQ(instruction->GetResultType(), Primitive::kPrimLong); 6590 Register first_low = first.AsRegisterPairLow<Register>(); 6591 Register first_high = first.AsRegisterPairHigh<Register>(); 6592 ShifterOperand second_low(second.AsRegisterPairLow<Register>()); 6593 ShifterOperand second_high(second.AsRegisterPairHigh<Register>()); 6594 Register out_low = out.AsRegisterPairLow<Register>(); 6595 Register out_high = out.AsRegisterPairHigh<Register>(); 6596 6597 switch (instruction->GetOpKind()) { 6598 case HInstruction::kAnd: 6599 __ bic(out_low, first_low, second_low); 6600 __ bic(out_high, first_high, second_high); 6601 break; 6602 case HInstruction::kOr: 6603 __ orn(out_low, first_low, second_low); 6604 __ orn(out_high, first_high, second_high); 6605 break; 6606 // There is no EON on arm. 6607 case HInstruction::kXor: 6608 default: 6609 LOG(FATAL) << "Unexpected instruction " << instruction->DebugName(); 6610 UNREACHABLE(); 6611 } 6612 } 6613} 6614 6615void InstructionCodeGeneratorARM::GenerateAndConst(Register out, Register first, uint32_t value) { 6616 // Optimize special cases for individual halfs of `and-long` (`and` is simplified earlier). 6617 if (value == 0xffffffffu) { 6618 if (out != first) { 6619 __ mov(out, ShifterOperand(first)); 6620 } 6621 return; 6622 } 6623 if (value == 0u) { 6624 __ mov(out, ShifterOperand(0)); 6625 return; 6626 } 6627 ShifterOperand so; 6628 if (__ ShifterOperandCanHold(kNoRegister, kNoRegister, AND, value, &so)) { 6629 __ and_(out, first, so); 6630 } else { 6631 DCHECK(__ ShifterOperandCanHold(kNoRegister, kNoRegister, BIC, ~value, &so)); 6632 __ bic(out, first, ShifterOperand(~value)); 6633 } 6634} 6635 6636void InstructionCodeGeneratorARM::GenerateOrrConst(Register out, Register first, uint32_t value) { 6637 // Optimize special cases for individual halfs of `or-long` (`or` is simplified earlier). 6638 if (value == 0u) { 6639 if (out != first) { 6640 __ mov(out, ShifterOperand(first)); 6641 } 6642 return; 6643 } 6644 if (value == 0xffffffffu) { 6645 __ mvn(out, ShifterOperand(0)); 6646 return; 6647 } 6648 ShifterOperand so; 6649 if (__ ShifterOperandCanHold(kNoRegister, kNoRegister, ORR, value, &so)) { 6650 __ orr(out, first, so); 6651 } else { 6652 DCHECK(__ ShifterOperandCanHold(kNoRegister, kNoRegister, ORN, ~value, &so)); 6653 __ orn(out, first, ShifterOperand(~value)); 6654 } 6655} 6656 6657void InstructionCodeGeneratorARM::GenerateEorConst(Register out, Register first, uint32_t value) { 6658 // Optimize special case for individual halfs of `xor-long` (`xor` is simplified earlier). 6659 if (value == 0u) { 6660 if (out != first) { 6661 __ mov(out, ShifterOperand(first)); 6662 } 6663 return; 6664 } 6665 __ eor(out, first, ShifterOperand(value)); 6666} 6667 6668void InstructionCodeGeneratorARM::GenerateAddLongConst(Location out, 6669 Location first, 6670 uint64_t value) { 6671 Register out_low = out.AsRegisterPairLow<Register>(); 6672 Register out_high = out.AsRegisterPairHigh<Register>(); 6673 Register first_low = first.AsRegisterPairLow<Register>(); 6674 Register first_high = first.AsRegisterPairHigh<Register>(); 6675 uint32_t value_low = Low32Bits(value); 6676 uint32_t value_high = High32Bits(value); 6677 if (value_low == 0u) { 6678 if (out_low != first_low) { 6679 __ mov(out_low, ShifterOperand(first_low)); 6680 } 6681 __ AddConstant(out_high, first_high, value_high); 6682 return; 6683 } 6684 __ AddConstantSetFlags(out_low, first_low, value_low); 6685 ShifterOperand so; 6686 if (__ ShifterOperandCanHold(out_high, first_high, ADC, value_high, kCcDontCare, &so)) { 6687 __ adc(out_high, first_high, so); 6688 } else if (__ ShifterOperandCanHold(out_low, first_low, SBC, ~value_high, kCcDontCare, &so)) { 6689 __ sbc(out_high, first_high, so); 6690 } else { 6691 LOG(FATAL) << "Unexpected constant " << value_high; 6692 UNREACHABLE(); 6693 } 6694} 6695 6696void InstructionCodeGeneratorARM::HandleBitwiseOperation(HBinaryOperation* instruction) { 6697 LocationSummary* locations = instruction->GetLocations(); 6698 Location first = locations->InAt(0); 6699 Location second = locations->InAt(1); 6700 Location out = locations->Out(); 6701 6702 if (second.IsConstant()) { 6703 uint64_t value = static_cast<uint64_t>(Int64FromConstant(second.GetConstant())); 6704 uint32_t value_low = Low32Bits(value); 6705 if (instruction->GetResultType() == Primitive::kPrimInt) { 6706 Register first_reg = first.AsRegister<Register>(); 6707 Register out_reg = out.AsRegister<Register>(); 6708 if (instruction->IsAnd()) { 6709 GenerateAndConst(out_reg, first_reg, value_low); 6710 } else if (instruction->IsOr()) { 6711 GenerateOrrConst(out_reg, first_reg, value_low); 6712 } else { 6713 DCHECK(instruction->IsXor()); 6714 GenerateEorConst(out_reg, first_reg, value_low); 6715 } 6716 } else { 6717 DCHECK_EQ(instruction->GetResultType(), Primitive::kPrimLong); 6718 uint32_t value_high = High32Bits(value); 6719 Register first_low = first.AsRegisterPairLow<Register>(); 6720 Register first_high = first.AsRegisterPairHigh<Register>(); 6721 Register out_low = out.AsRegisterPairLow<Register>(); 6722 Register out_high = out.AsRegisterPairHigh<Register>(); 6723 if (instruction->IsAnd()) { 6724 GenerateAndConst(out_low, first_low, value_low); 6725 GenerateAndConst(out_high, first_high, value_high); 6726 } else if (instruction->IsOr()) { 6727 GenerateOrrConst(out_low, first_low, value_low); 6728 GenerateOrrConst(out_high, first_high, value_high); 6729 } else { 6730 DCHECK(instruction->IsXor()); 6731 GenerateEorConst(out_low, first_low, value_low); 6732 GenerateEorConst(out_high, first_high, value_high); 6733 } 6734 } 6735 return; 6736 } 6737 6738 if (instruction->GetResultType() == Primitive::kPrimInt) { 6739 Register first_reg = first.AsRegister<Register>(); 6740 ShifterOperand second_reg(second.AsRegister<Register>()); 6741 Register out_reg = out.AsRegister<Register>(); 6742 if (instruction->IsAnd()) { 6743 __ and_(out_reg, first_reg, second_reg); 6744 } else if (instruction->IsOr()) { 6745 __ orr(out_reg, first_reg, second_reg); 6746 } else { 6747 DCHECK(instruction->IsXor()); 6748 __ eor(out_reg, first_reg, second_reg); 6749 } 6750 } else { 6751 DCHECK_EQ(instruction->GetResultType(), Primitive::kPrimLong); 6752 Register first_low = first.AsRegisterPairLow<Register>(); 6753 Register first_high = first.AsRegisterPairHigh<Register>(); 6754 ShifterOperand second_low(second.AsRegisterPairLow<Register>()); 6755 ShifterOperand second_high(second.AsRegisterPairHigh<Register>()); 6756 Register out_low = out.AsRegisterPairLow<Register>(); 6757 Register out_high = out.AsRegisterPairHigh<Register>(); 6758 if (instruction->IsAnd()) { 6759 __ and_(out_low, first_low, second_low); 6760 __ and_(out_high, first_high, second_high); 6761 } else if (instruction->IsOr()) { 6762 __ orr(out_low, first_low, second_low); 6763 __ orr(out_high, first_high, second_high); 6764 } else { 6765 DCHECK(instruction->IsXor()); 6766 __ eor(out_low, first_low, second_low); 6767 __ eor(out_high, first_high, second_high); 6768 } 6769 } 6770} 6771 6772void InstructionCodeGeneratorARM::GenerateReferenceLoadOneRegister( 6773 HInstruction* instruction, 6774 Location out, 6775 uint32_t offset, 6776 Location maybe_temp, 6777 ReadBarrierOption read_barrier_option) { 6778 Register out_reg = out.AsRegister<Register>(); 6779 if (read_barrier_option == kWithReadBarrier) { 6780 CHECK(kEmitCompilerReadBarrier); 6781 DCHECK(maybe_temp.IsRegister()) << maybe_temp; 6782 if (kUseBakerReadBarrier) { 6783 // Load with fast path based Baker's read barrier. 6784 // /* HeapReference<Object> */ out = *(out + offset) 6785 codegen_->GenerateFieldLoadWithBakerReadBarrier( 6786 instruction, out, out_reg, offset, maybe_temp, /* needs_null_check */ false); 6787 } else { 6788 // Load with slow path based read barrier. 6789 // Save the value of `out` into `maybe_temp` before overwriting it 6790 // in the following move operation, as we will need it for the 6791 // read barrier below. 6792 __ Mov(maybe_temp.AsRegister<Register>(), out_reg); 6793 // /* HeapReference<Object> */ out = *(out + offset) 6794 __ LoadFromOffset(kLoadWord, out_reg, out_reg, offset); 6795 codegen_->GenerateReadBarrierSlow(instruction, out, out, maybe_temp, offset); 6796 } 6797 } else { 6798 // Plain load with no read barrier. 6799 // /* HeapReference<Object> */ out = *(out + offset) 6800 __ LoadFromOffset(kLoadWord, out_reg, out_reg, offset); 6801 __ MaybeUnpoisonHeapReference(out_reg); 6802 } 6803} 6804 6805void InstructionCodeGeneratorARM::GenerateReferenceLoadTwoRegisters( 6806 HInstruction* instruction, 6807 Location out, 6808 Location obj, 6809 uint32_t offset, 6810 Location maybe_temp, 6811 ReadBarrierOption read_barrier_option) { 6812 Register out_reg = out.AsRegister<Register>(); 6813 Register obj_reg = obj.AsRegister<Register>(); 6814 if (read_barrier_option == kWithReadBarrier) { 6815 CHECK(kEmitCompilerReadBarrier); 6816 if (kUseBakerReadBarrier) { 6817 DCHECK(maybe_temp.IsRegister()) << maybe_temp; 6818 // Load with fast path based Baker's read barrier. 6819 // /* HeapReference<Object> */ out = *(obj + offset) 6820 codegen_->GenerateFieldLoadWithBakerReadBarrier( 6821 instruction, out, obj_reg, offset, maybe_temp, /* needs_null_check */ false); 6822 } else { 6823 // Load with slow path based read barrier. 6824 // /* HeapReference<Object> */ out = *(obj + offset) 6825 __ LoadFromOffset(kLoadWord, out_reg, obj_reg, offset); 6826 codegen_->GenerateReadBarrierSlow(instruction, out, out, obj, offset); 6827 } 6828 } else { 6829 // Plain load with no read barrier. 6830 // /* HeapReference<Object> */ out = *(obj + offset) 6831 __ LoadFromOffset(kLoadWord, out_reg, obj_reg, offset); 6832 __ MaybeUnpoisonHeapReference(out_reg); 6833 } 6834} 6835 6836void InstructionCodeGeneratorARM::GenerateGcRootFieldLoad(HInstruction* instruction, 6837 Location root, 6838 Register obj, 6839 uint32_t offset, 6840 ReadBarrierOption read_barrier_option) { 6841 Register root_reg = root.AsRegister<Register>(); 6842 if (read_barrier_option == kWithReadBarrier) { 6843 DCHECK(kEmitCompilerReadBarrier); 6844 if (kUseBakerReadBarrier) { 6845 // Fast path implementation of art::ReadBarrier::BarrierForRoot when 6846 // Baker's read barrier are used: 6847 // 6848 // root = obj.field; 6849 // temp = Thread::Current()->pReadBarrierMarkReg ## root.reg() 6850 // if (temp != null) { 6851 // root = temp(root) 6852 // } 6853 6854 // /* GcRoot<mirror::Object> */ root = *(obj + offset) 6855 __ LoadFromOffset(kLoadWord, root_reg, obj, offset); 6856 static_assert( 6857 sizeof(mirror::CompressedReference<mirror::Object>) == sizeof(GcRoot<mirror::Object>), 6858 "art::mirror::CompressedReference<mirror::Object> and art::GcRoot<mirror::Object> " 6859 "have different sizes."); 6860 static_assert(sizeof(mirror::CompressedReference<mirror::Object>) == sizeof(int32_t), 6861 "art::mirror::CompressedReference<mirror::Object> and int32_t " 6862 "have different sizes."); 6863 6864 // Slow path marking the GC root `root`. 6865 Location temp = Location::RegisterLocation(LR); 6866 SlowPathCodeARM* slow_path = 6867 new (GetGraph()->GetArena()) ReadBarrierMarkSlowPathARM( 6868 instruction, 6869 root, 6870 /*entrypoint*/ temp); 6871 codegen_->AddSlowPath(slow_path); 6872 6873 // temp = Thread::Current()->pReadBarrierMarkReg ## root.reg() 6874 const int32_t entry_point_offset = 6875 CodeGenerator::GetReadBarrierMarkEntryPointsOffset<kArmPointerSize>(root.reg()); 6876 // Loading the entrypoint does not require a load acquire since it is only changed when 6877 // threads are suspended or running a checkpoint. 6878 __ LoadFromOffset(kLoadWord, temp.AsRegister<Register>(), TR, entry_point_offset); 6879 // The entrypoint is null when the GC is not marking, this prevents one load compared to 6880 // checking GetIsGcMarking. 6881 __ CompareAndBranchIfNonZero(temp.AsRegister<Register>(), slow_path->GetEntryLabel()); 6882 __ Bind(slow_path->GetExitLabel()); 6883 } else { 6884 // GC root loaded through a slow path for read barriers other 6885 // than Baker's. 6886 // /* GcRoot<mirror::Object>* */ root = obj + offset 6887 __ AddConstant(root_reg, obj, offset); 6888 // /* mirror::Object* */ root = root->Read() 6889 codegen_->GenerateReadBarrierForRootSlow(instruction, root, root); 6890 } 6891 } else { 6892 // Plain GC root load with no read barrier. 6893 // /* GcRoot<mirror::Object> */ root = *(obj + offset) 6894 __ LoadFromOffset(kLoadWord, root_reg, obj, offset); 6895 // Note that GC roots are not affected by heap poisoning, thus we 6896 // do not have to unpoison `root_reg` here. 6897 } 6898} 6899 6900void CodeGeneratorARM::GenerateFieldLoadWithBakerReadBarrier(HInstruction* instruction, 6901 Location ref, 6902 Register obj, 6903 uint32_t offset, 6904 Location temp, 6905 bool needs_null_check) { 6906 DCHECK(kEmitCompilerReadBarrier); 6907 DCHECK(kUseBakerReadBarrier); 6908 6909 // /* HeapReference<Object> */ ref = *(obj + offset) 6910 Location no_index = Location::NoLocation(); 6911 ScaleFactor no_scale_factor = TIMES_1; 6912 GenerateReferenceLoadWithBakerReadBarrier( 6913 instruction, ref, obj, offset, no_index, no_scale_factor, temp, needs_null_check); 6914} 6915 6916void CodeGeneratorARM::GenerateArrayLoadWithBakerReadBarrier(HInstruction* instruction, 6917 Location ref, 6918 Register obj, 6919 uint32_t data_offset, 6920 Location index, 6921 Location temp, 6922 bool needs_null_check) { 6923 DCHECK(kEmitCompilerReadBarrier); 6924 DCHECK(kUseBakerReadBarrier); 6925 6926 static_assert( 6927 sizeof(mirror::HeapReference<mirror::Object>) == sizeof(int32_t), 6928 "art::mirror::HeapReference<art::mirror::Object> and int32_t have different sizes."); 6929 // /* HeapReference<Object> */ ref = 6930 // *(obj + data_offset + index * sizeof(HeapReference<Object>)) 6931 ScaleFactor scale_factor = TIMES_4; 6932 GenerateReferenceLoadWithBakerReadBarrier( 6933 instruction, ref, obj, data_offset, index, scale_factor, temp, needs_null_check); 6934} 6935 6936void CodeGeneratorARM::GenerateReferenceLoadWithBakerReadBarrier(HInstruction* instruction, 6937 Location ref, 6938 Register obj, 6939 uint32_t offset, 6940 Location index, 6941 ScaleFactor scale_factor, 6942 Location temp, 6943 bool needs_null_check, 6944 bool always_update_field, 6945 Register* temp2) { 6946 DCHECK(kEmitCompilerReadBarrier); 6947 DCHECK(kUseBakerReadBarrier); 6948 6949 // In slow path based read barriers, the read barrier call is 6950 // inserted after the original load. However, in fast path based 6951 // Baker's read barriers, we need to perform the load of 6952 // mirror::Object::monitor_ *before* the original reference load. 6953 // This load-load ordering is required by the read barrier. 6954 // The fast path/slow path (for Baker's algorithm) should look like: 6955 // 6956 // uint32_t rb_state = Lockword(obj->monitor_).ReadBarrierState(); 6957 // lfence; // Load fence or artificial data dependency to prevent load-load reordering 6958 // HeapReference<Object> ref = *src; // Original reference load. 6959 // bool is_gray = (rb_state == ReadBarrier::GrayState()); 6960 // if (is_gray) { 6961 // ref = ReadBarrier::Mark(ref); // Performed by runtime entrypoint slow path. 6962 // } 6963 // 6964 // Note: the original implementation in ReadBarrier::Barrier is 6965 // slightly more complex as it performs additional checks that we do 6966 // not do here for performance reasons. 6967 6968 Register ref_reg = ref.AsRegister<Register>(); 6969 Register temp_reg = temp.AsRegister<Register>(); 6970 uint32_t monitor_offset = mirror::Object::MonitorOffset().Int32Value(); 6971 6972 // /* int32_t */ monitor = obj->monitor_ 6973 __ LoadFromOffset(kLoadWord, temp_reg, obj, monitor_offset); 6974 if (needs_null_check) { 6975 MaybeRecordImplicitNullCheck(instruction); 6976 } 6977 // /* LockWord */ lock_word = LockWord(monitor) 6978 static_assert(sizeof(LockWord) == sizeof(int32_t), 6979 "art::LockWord and int32_t have different sizes."); 6980 6981 // Introduce a dependency on the lock_word including the rb_state, 6982 // which shall prevent load-load reordering without using 6983 // a memory barrier (which would be more expensive). 6984 // `obj` is unchanged by this operation, but its value now depends 6985 // on `temp_reg`. 6986 __ add(obj, obj, ShifterOperand(temp_reg, LSR, 32)); 6987 6988 // The actual reference load. 6989 if (index.IsValid()) { 6990 // Load types involving an "index": ArrayGet, 6991 // UnsafeGetObject/UnsafeGetObjectVolatile and UnsafeCASObject 6992 // intrinsics. 6993 // /* HeapReference<Object> */ ref = *(obj + offset + (index << scale_factor)) 6994 if (index.IsConstant()) { 6995 size_t computed_offset = 6996 (index.GetConstant()->AsIntConstant()->GetValue() << scale_factor) + offset; 6997 __ LoadFromOffset(kLoadWord, ref_reg, obj, computed_offset); 6998 } else { 6999 // Handle the special case of the 7000 // UnsafeGetObject/UnsafeGetObjectVolatile and UnsafeCASObject 7001 // intrinsics, which use a register pair as index ("long 7002 // offset"), of which only the low part contains data. 7003 Register index_reg = index.IsRegisterPair() 7004 ? index.AsRegisterPairLow<Register>() 7005 : index.AsRegister<Register>(); 7006 __ add(IP, obj, ShifterOperand(index_reg, LSL, scale_factor)); 7007 __ LoadFromOffset(kLoadWord, ref_reg, IP, offset); 7008 } 7009 } else { 7010 // /* HeapReference<Object> */ ref = *(obj + offset) 7011 __ LoadFromOffset(kLoadWord, ref_reg, obj, offset); 7012 } 7013 7014 // Object* ref = ref_addr->AsMirrorPtr() 7015 __ MaybeUnpoisonHeapReference(ref_reg); 7016 7017 // Slow path marking the object `ref` when it is gray. 7018 SlowPathCodeARM* slow_path; 7019 if (always_update_field) { 7020 DCHECK(temp2 != nullptr); 7021 // ReadBarrierMarkAndUpdateFieldSlowPathARM only supports address 7022 // of the form `obj + field_offset`, where `obj` is a register and 7023 // `field_offset` is a register pair (of which only the lower half 7024 // is used). Thus `offset` and `scale_factor` above are expected 7025 // to be null in this code path. 7026 DCHECK_EQ(offset, 0u); 7027 DCHECK_EQ(scale_factor, ScaleFactor::TIMES_1); 7028 slow_path = new (GetGraph()->GetArena()) ReadBarrierMarkAndUpdateFieldSlowPathARM( 7029 instruction, ref, obj, /* field_offset */ index, temp_reg, *temp2); 7030 } else { 7031 slow_path = new (GetGraph()->GetArena()) ReadBarrierMarkSlowPathARM(instruction, ref); 7032 } 7033 AddSlowPath(slow_path); 7034 7035 // if (rb_state == ReadBarrier::GrayState()) 7036 // ref = ReadBarrier::Mark(ref); 7037 // Given the numeric representation, it's enough to check the low bit of the 7038 // rb_state. We do that by shifting the bit out of the lock word with LSRS 7039 // which can be a 16-bit instruction unlike the TST immediate. 7040 static_assert(ReadBarrier::WhiteState() == 0, "Expecting white to have value 0"); 7041 static_assert(ReadBarrier::GrayState() == 1, "Expecting gray to have value 1"); 7042 __ Lsrs(temp_reg, temp_reg, LockWord::kReadBarrierStateShift + 1); 7043 __ b(slow_path->GetEntryLabel(), CS); // Carry flag is the last bit shifted out by LSRS. 7044 __ Bind(slow_path->GetExitLabel()); 7045} 7046 7047void CodeGeneratorARM::GenerateReadBarrierSlow(HInstruction* instruction, 7048 Location out, 7049 Location ref, 7050 Location obj, 7051 uint32_t offset, 7052 Location index) { 7053 DCHECK(kEmitCompilerReadBarrier); 7054 7055 // Insert a slow path based read barrier *after* the reference load. 7056 // 7057 // If heap poisoning is enabled, the unpoisoning of the loaded 7058 // reference will be carried out by the runtime within the slow 7059 // path. 7060 // 7061 // Note that `ref` currently does not get unpoisoned (when heap 7062 // poisoning is enabled), which is alright as the `ref` argument is 7063 // not used by the artReadBarrierSlow entry point. 7064 // 7065 // TODO: Unpoison `ref` when it is used by artReadBarrierSlow. 7066 SlowPathCodeARM* slow_path = new (GetGraph()->GetArena()) 7067 ReadBarrierForHeapReferenceSlowPathARM(instruction, out, ref, obj, offset, index); 7068 AddSlowPath(slow_path); 7069 7070 __ b(slow_path->GetEntryLabel()); 7071 __ Bind(slow_path->GetExitLabel()); 7072} 7073 7074void CodeGeneratorARM::MaybeGenerateReadBarrierSlow(HInstruction* instruction, 7075 Location out, 7076 Location ref, 7077 Location obj, 7078 uint32_t offset, 7079 Location index) { 7080 if (kEmitCompilerReadBarrier) { 7081 // Baker's read barriers shall be handled by the fast path 7082 // (CodeGeneratorARM::GenerateReferenceLoadWithBakerReadBarrier). 7083 DCHECK(!kUseBakerReadBarrier); 7084 // If heap poisoning is enabled, unpoisoning will be taken care of 7085 // by the runtime within the slow path. 7086 GenerateReadBarrierSlow(instruction, out, ref, obj, offset, index); 7087 } else if (kPoisonHeapReferences) { 7088 __ UnpoisonHeapReference(out.AsRegister<Register>()); 7089 } 7090} 7091 7092void CodeGeneratorARM::GenerateReadBarrierForRootSlow(HInstruction* instruction, 7093 Location out, 7094 Location root) { 7095 DCHECK(kEmitCompilerReadBarrier); 7096 7097 // Insert a slow path based read barrier *after* the GC root load. 7098 // 7099 // Note that GC roots are not affected by heap poisoning, so we do 7100 // not need to do anything special for this here. 7101 SlowPathCodeARM* slow_path = 7102 new (GetGraph()->GetArena()) ReadBarrierForRootSlowPathARM(instruction, out, root); 7103 AddSlowPath(slow_path); 7104 7105 __ b(slow_path->GetEntryLabel()); 7106 __ Bind(slow_path->GetExitLabel()); 7107} 7108 7109HInvokeStaticOrDirect::DispatchInfo CodeGeneratorARM::GetSupportedInvokeStaticOrDirectDispatch( 7110 const HInvokeStaticOrDirect::DispatchInfo& desired_dispatch_info, 7111 HInvokeStaticOrDirect* invoke) { 7112 HInvokeStaticOrDirect::DispatchInfo dispatch_info = desired_dispatch_info; 7113 // We disable pc-relative load when there is an irreducible loop, as the optimization 7114 // is incompatible with it. 7115 // TODO: Create as many ArmDexCacheArraysBase instructions as needed for methods 7116 // with irreducible loops. 7117 if (GetGraph()->HasIrreducibleLoops() && 7118 (dispatch_info.method_load_kind == 7119 HInvokeStaticOrDirect::MethodLoadKind::kDexCachePcRelative)) { 7120 dispatch_info.method_load_kind = HInvokeStaticOrDirect::MethodLoadKind::kDexCacheViaMethod; 7121 } 7122 7123 if (dispatch_info.code_ptr_location == HInvokeStaticOrDirect::CodePtrLocation::kCallPCRelative) { 7124 const DexFile& outer_dex_file = GetGraph()->GetDexFile(); 7125 if (&outer_dex_file != invoke->GetTargetMethod().dex_file) { 7126 // Calls across dex files are more likely to exceed the available BL range, 7127 // so use absolute patch with fixup if available and kCallArtMethod otherwise. 7128 HInvokeStaticOrDirect::CodePtrLocation code_ptr_location = 7129 (desired_dispatch_info.method_load_kind == 7130 HInvokeStaticOrDirect::MethodLoadKind::kDirectAddressWithFixup) 7131 ? HInvokeStaticOrDirect::CodePtrLocation::kCallDirectWithFixup 7132 : HInvokeStaticOrDirect::CodePtrLocation::kCallArtMethod; 7133 return HInvokeStaticOrDirect::DispatchInfo { 7134 dispatch_info.method_load_kind, 7135 code_ptr_location, 7136 dispatch_info.method_load_data, 7137 0u 7138 }; 7139 } 7140 } 7141 return dispatch_info; 7142} 7143 7144Register CodeGeneratorARM::GetInvokeStaticOrDirectExtraParameter(HInvokeStaticOrDirect* invoke, 7145 Register temp) { 7146 DCHECK_EQ(invoke->InputCount(), invoke->GetNumberOfArguments() + 1u); 7147 Location location = invoke->GetLocations()->InAt(invoke->GetSpecialInputIndex()); 7148 if (!invoke->GetLocations()->Intrinsified()) { 7149 return location.AsRegister<Register>(); 7150 } 7151 // For intrinsics we allow any location, so it may be on the stack. 7152 if (!location.IsRegister()) { 7153 __ LoadFromOffset(kLoadWord, temp, SP, location.GetStackIndex()); 7154 return temp; 7155 } 7156 // For register locations, check if the register was saved. If so, get it from the stack. 7157 // Note: There is a chance that the register was saved but not overwritten, so we could 7158 // save one load. However, since this is just an intrinsic slow path we prefer this 7159 // simple and more robust approach rather that trying to determine if that's the case. 7160 SlowPathCode* slow_path = GetCurrentSlowPath(); 7161 DCHECK(slow_path != nullptr); // For intrinsified invokes the call is emitted on the slow path. 7162 if (slow_path->IsCoreRegisterSaved(location.AsRegister<Register>())) { 7163 int stack_offset = slow_path->GetStackOffsetOfCoreRegister(location.AsRegister<Register>()); 7164 __ LoadFromOffset(kLoadWord, temp, SP, stack_offset); 7165 return temp; 7166 } 7167 return location.AsRegister<Register>(); 7168} 7169 7170void CodeGeneratorARM::GenerateStaticOrDirectCall(HInvokeStaticOrDirect* invoke, Location temp) { 7171 // For better instruction scheduling we load the direct code pointer before the method pointer. 7172 switch (invoke->GetCodePtrLocation()) { 7173 case HInvokeStaticOrDirect::CodePtrLocation::kCallDirectWithFixup: 7174 // LR = code address from literal pool with link-time patch. 7175 __ LoadLiteral(LR, DeduplicateMethodCodeLiteral(invoke->GetTargetMethod())); 7176 break; 7177 case HInvokeStaticOrDirect::CodePtrLocation::kCallDirect: 7178 // LR = invoke->GetDirectCodePtr(); 7179 __ LoadImmediate(LR, invoke->GetDirectCodePtr()); 7180 break; 7181 default: 7182 break; 7183 } 7184 7185 Location callee_method = temp; // For all kinds except kRecursive, callee will be in temp. 7186 switch (invoke->GetMethodLoadKind()) { 7187 case HInvokeStaticOrDirect::MethodLoadKind::kStringInit: { 7188 uint32_t offset = 7189 GetThreadOffset<kArmPointerSize>(invoke->GetStringInitEntryPoint()).Int32Value(); 7190 // temp = thread->string_init_entrypoint 7191 __ LoadFromOffset(kLoadWord, temp.AsRegister<Register>(), TR, offset); 7192 break; 7193 } 7194 case HInvokeStaticOrDirect::MethodLoadKind::kRecursive: 7195 callee_method = invoke->GetLocations()->InAt(invoke->GetSpecialInputIndex()); 7196 break; 7197 case HInvokeStaticOrDirect::MethodLoadKind::kDirectAddress: 7198 __ LoadImmediate(temp.AsRegister<Register>(), invoke->GetMethodAddress()); 7199 break; 7200 case HInvokeStaticOrDirect::MethodLoadKind::kDirectAddressWithFixup: 7201 __ LoadLiteral(temp.AsRegister<Register>(), 7202 DeduplicateMethodAddressLiteral(invoke->GetTargetMethod())); 7203 break; 7204 case HInvokeStaticOrDirect::MethodLoadKind::kDexCachePcRelative: { 7205 HArmDexCacheArraysBase* base = 7206 invoke->InputAt(invoke->GetSpecialInputIndex())->AsArmDexCacheArraysBase(); 7207 Register base_reg = GetInvokeStaticOrDirectExtraParameter(invoke, 7208 temp.AsRegister<Register>()); 7209 int32_t offset = invoke->GetDexCacheArrayOffset() - base->GetElementOffset(); 7210 __ LoadFromOffset(kLoadWord, temp.AsRegister<Register>(), base_reg, offset); 7211 break; 7212 } 7213 case HInvokeStaticOrDirect::MethodLoadKind::kDexCacheViaMethod: { 7214 Location current_method = invoke->GetLocations()->InAt(invoke->GetSpecialInputIndex()); 7215 Register method_reg; 7216 Register reg = temp.AsRegister<Register>(); 7217 if (current_method.IsRegister()) { 7218 method_reg = current_method.AsRegister<Register>(); 7219 } else { 7220 DCHECK(invoke->GetLocations()->Intrinsified()); 7221 DCHECK(!current_method.IsValid()); 7222 method_reg = reg; 7223 __ LoadFromOffset(kLoadWord, reg, SP, kCurrentMethodStackOffset); 7224 } 7225 // /* ArtMethod*[] */ temp = temp.ptr_sized_fields_->dex_cache_resolved_methods_; 7226 __ LoadFromOffset(kLoadWord, 7227 reg, 7228 method_reg, 7229 ArtMethod::DexCacheResolvedMethodsOffset(kArmPointerSize).Int32Value()); 7230 // temp = temp[index_in_cache]; 7231 // Note: Don't use invoke->GetTargetMethod() as it may point to a different dex file. 7232 uint32_t index_in_cache = invoke->GetDexMethodIndex(); 7233 __ LoadFromOffset(kLoadWord, reg, reg, CodeGenerator::GetCachePointerOffset(index_in_cache)); 7234 break; 7235 } 7236 } 7237 7238 switch (invoke->GetCodePtrLocation()) { 7239 case HInvokeStaticOrDirect::CodePtrLocation::kCallSelf: 7240 __ bl(GetFrameEntryLabel()); 7241 break; 7242 case HInvokeStaticOrDirect::CodePtrLocation::kCallPCRelative: 7243 relative_call_patches_.emplace_back(*invoke->GetTargetMethod().dex_file, 7244 invoke->GetTargetMethod().dex_method_index); 7245 __ BindTrackedLabel(&relative_call_patches_.back().label); 7246 // Arbitrarily branch to the BL itself, override at link time. 7247 __ bl(&relative_call_patches_.back().label); 7248 break; 7249 case HInvokeStaticOrDirect::CodePtrLocation::kCallDirectWithFixup: 7250 case HInvokeStaticOrDirect::CodePtrLocation::kCallDirect: 7251 // LR prepared above for better instruction scheduling. 7252 // LR() 7253 __ blx(LR); 7254 break; 7255 case HInvokeStaticOrDirect::CodePtrLocation::kCallArtMethod: 7256 // LR = callee_method->entry_point_from_quick_compiled_code_ 7257 __ LoadFromOffset( 7258 kLoadWord, LR, callee_method.AsRegister<Register>(), 7259 ArtMethod::EntryPointFromQuickCompiledCodeOffset(kArmPointerSize).Int32Value()); 7260 // LR() 7261 __ blx(LR); 7262 break; 7263 } 7264 7265 DCHECK(!IsLeafMethod()); 7266} 7267 7268void CodeGeneratorARM::GenerateVirtualCall(HInvokeVirtual* invoke, Location temp_location) { 7269 Register temp = temp_location.AsRegister<Register>(); 7270 uint32_t method_offset = mirror::Class::EmbeddedVTableEntryOffset( 7271 invoke->GetVTableIndex(), kArmPointerSize).Uint32Value(); 7272 7273 // Use the calling convention instead of the location of the receiver, as 7274 // intrinsics may have put the receiver in a different register. In the intrinsics 7275 // slow path, the arguments have been moved to the right place, so here we are 7276 // guaranteed that the receiver is the first register of the calling convention. 7277 InvokeDexCallingConvention calling_convention; 7278 Register receiver = calling_convention.GetRegisterAt(0); 7279 uint32_t class_offset = mirror::Object::ClassOffset().Int32Value(); 7280 // /* HeapReference<Class> */ temp = receiver->klass_ 7281 __ LoadFromOffset(kLoadWord, temp, receiver, class_offset); 7282 MaybeRecordImplicitNullCheck(invoke); 7283 // Instead of simply (possibly) unpoisoning `temp` here, we should 7284 // emit a read barrier for the previous class reference load. 7285 // However this is not required in practice, as this is an 7286 // intermediate/temporary reference and because the current 7287 // concurrent copying collector keeps the from-space memory 7288 // intact/accessible until the end of the marking phase (the 7289 // concurrent copying collector may not in the future). 7290 __ MaybeUnpoisonHeapReference(temp); 7291 // temp = temp->GetMethodAt(method_offset); 7292 uint32_t entry_point = ArtMethod::EntryPointFromQuickCompiledCodeOffset( 7293 kArmPointerSize).Int32Value(); 7294 __ LoadFromOffset(kLoadWord, temp, temp, method_offset); 7295 // LR = temp->GetEntryPoint(); 7296 __ LoadFromOffset(kLoadWord, LR, temp, entry_point); 7297 // LR(); 7298 __ blx(LR); 7299} 7300 7301CodeGeneratorARM::PcRelativePatchInfo* CodeGeneratorARM::NewPcRelativeStringPatch( 7302 const DexFile& dex_file, uint32_t string_index) { 7303 return NewPcRelativePatch(dex_file, string_index, &pc_relative_string_patches_); 7304} 7305 7306CodeGeneratorARM::PcRelativePatchInfo* CodeGeneratorARM::NewPcRelativeTypePatch( 7307 const DexFile& dex_file, uint32_t type_index) { 7308 return NewPcRelativePatch(dex_file, type_index, &pc_relative_type_patches_); 7309} 7310 7311CodeGeneratorARM::PcRelativePatchInfo* CodeGeneratorARM::NewPcRelativeDexCacheArrayPatch( 7312 const DexFile& dex_file, uint32_t element_offset) { 7313 return NewPcRelativePatch(dex_file, element_offset, &pc_relative_dex_cache_patches_); 7314} 7315 7316CodeGeneratorARM::PcRelativePatchInfo* CodeGeneratorARM::NewPcRelativePatch( 7317 const DexFile& dex_file, uint32_t offset_or_index, ArenaDeque<PcRelativePatchInfo>* patches) { 7318 patches->emplace_back(dex_file, offset_or_index); 7319 return &patches->back(); 7320} 7321 7322Literal* CodeGeneratorARM::DeduplicateBootImageStringLiteral(const DexFile& dex_file, 7323 uint32_t string_index) { 7324 return boot_image_string_patches_.GetOrCreate( 7325 StringReference(&dex_file, string_index), 7326 [this]() { return __ NewLiteral<uint32_t>(/* placeholder */ 0u); }); 7327} 7328 7329Literal* CodeGeneratorARM::DeduplicateBootImageTypeLiteral(const DexFile& dex_file, 7330 uint32_t type_index) { 7331 return boot_image_type_patches_.GetOrCreate( 7332 TypeReference(&dex_file, type_index), 7333 [this]() { return __ NewLiteral<uint32_t>(/* placeholder */ 0u); }); 7334} 7335 7336Literal* CodeGeneratorARM::DeduplicateBootImageAddressLiteral(uint32_t address) { 7337 bool needs_patch = GetCompilerOptions().GetIncludePatchInformation(); 7338 Uint32ToLiteralMap* map = needs_patch ? &boot_image_address_patches_ : &uint32_literals_; 7339 return DeduplicateUint32Literal(dchecked_integral_cast<uint32_t>(address), map); 7340} 7341 7342Literal* CodeGeneratorARM::DeduplicateDexCacheAddressLiteral(uint32_t address) { 7343 return DeduplicateUint32Literal(address, &uint32_literals_); 7344} 7345 7346template <LinkerPatch (*Factory)(size_t, const DexFile*, uint32_t, uint32_t)> 7347inline void CodeGeneratorARM::EmitPcRelativeLinkerPatches( 7348 const ArenaDeque<PcRelativePatchInfo>& infos, 7349 ArenaVector<LinkerPatch>* linker_patches) { 7350 for (const PcRelativePatchInfo& info : infos) { 7351 const DexFile& dex_file = info.target_dex_file; 7352 size_t offset_or_index = info.offset_or_index; 7353 DCHECK(info.add_pc_label.IsBound()); 7354 uint32_t add_pc_offset = dchecked_integral_cast<uint32_t>(info.add_pc_label.Position()); 7355 // Add MOVW patch. 7356 DCHECK(info.movw_label.IsBound()); 7357 uint32_t movw_offset = dchecked_integral_cast<uint32_t>(info.movw_label.Position()); 7358 linker_patches->push_back(Factory(movw_offset, &dex_file, add_pc_offset, offset_or_index)); 7359 // Add MOVT patch. 7360 DCHECK(info.movt_label.IsBound()); 7361 uint32_t movt_offset = dchecked_integral_cast<uint32_t>(info.movt_label.Position()); 7362 linker_patches->push_back(Factory(movt_offset, &dex_file, add_pc_offset, offset_or_index)); 7363 } 7364} 7365 7366void CodeGeneratorARM::EmitLinkerPatches(ArenaVector<LinkerPatch>* linker_patches) { 7367 DCHECK(linker_patches->empty()); 7368 size_t size = 7369 method_patches_.size() + 7370 call_patches_.size() + 7371 relative_call_patches_.size() + 7372 /* MOVW+MOVT for each entry */ 2u * pc_relative_dex_cache_patches_.size() + 7373 boot_image_string_patches_.size() + 7374 /* MOVW+MOVT for each entry */ 2u * pc_relative_string_patches_.size() + 7375 boot_image_type_patches_.size() + 7376 /* MOVW+MOVT for each entry */ 2u * pc_relative_type_patches_.size() + 7377 boot_image_address_patches_.size(); 7378 linker_patches->reserve(size); 7379 for (const auto& entry : method_patches_) { 7380 const MethodReference& target_method = entry.first; 7381 Literal* literal = entry.second; 7382 DCHECK(literal->GetLabel()->IsBound()); 7383 uint32_t literal_offset = literal->GetLabel()->Position(); 7384 linker_patches->push_back(LinkerPatch::MethodPatch(literal_offset, 7385 target_method.dex_file, 7386 target_method.dex_method_index)); 7387 } 7388 for (const auto& entry : call_patches_) { 7389 const MethodReference& target_method = entry.first; 7390 Literal* literal = entry.second; 7391 DCHECK(literal->GetLabel()->IsBound()); 7392 uint32_t literal_offset = literal->GetLabel()->Position(); 7393 linker_patches->push_back(LinkerPatch::CodePatch(literal_offset, 7394 target_method.dex_file, 7395 target_method.dex_method_index)); 7396 } 7397 for (const PatchInfo<Label>& info : relative_call_patches_) { 7398 uint32_t literal_offset = info.label.Position(); 7399 linker_patches->push_back( 7400 LinkerPatch::RelativeCodePatch(literal_offset, &info.dex_file, info.index)); 7401 } 7402 EmitPcRelativeLinkerPatches<LinkerPatch::DexCacheArrayPatch>(pc_relative_dex_cache_patches_, 7403 linker_patches); 7404 for (const auto& entry : boot_image_string_patches_) { 7405 const StringReference& target_string = entry.first; 7406 Literal* literal = entry.second; 7407 DCHECK(literal->GetLabel()->IsBound()); 7408 uint32_t literal_offset = literal->GetLabel()->Position(); 7409 linker_patches->push_back(LinkerPatch::StringPatch(literal_offset, 7410 target_string.dex_file, 7411 target_string.string_index)); 7412 } 7413 if (!GetCompilerOptions().IsBootImage()) { 7414 EmitPcRelativeLinkerPatches<LinkerPatch::StringBssEntryPatch>(pc_relative_string_patches_, 7415 linker_patches); 7416 } else { 7417 EmitPcRelativeLinkerPatches<LinkerPatch::RelativeStringPatch>(pc_relative_string_patches_, 7418 linker_patches); 7419 } 7420 for (const auto& entry : boot_image_type_patches_) { 7421 const TypeReference& target_type = entry.first; 7422 Literal* literal = entry.second; 7423 DCHECK(literal->GetLabel()->IsBound()); 7424 uint32_t literal_offset = literal->GetLabel()->Position(); 7425 linker_patches->push_back(LinkerPatch::TypePatch(literal_offset, 7426 target_type.dex_file, 7427 target_type.type_index)); 7428 } 7429 EmitPcRelativeLinkerPatches<LinkerPatch::RelativeTypePatch>(pc_relative_type_patches_, 7430 linker_patches); 7431 for (const auto& entry : boot_image_address_patches_) { 7432 DCHECK(GetCompilerOptions().GetIncludePatchInformation()); 7433 Literal* literal = entry.second; 7434 DCHECK(literal->GetLabel()->IsBound()); 7435 uint32_t literal_offset = literal->GetLabel()->Position(); 7436 linker_patches->push_back(LinkerPatch::RecordPosition(literal_offset)); 7437 } 7438} 7439 7440Literal* CodeGeneratorARM::DeduplicateUint32Literal(uint32_t value, Uint32ToLiteralMap* map) { 7441 return map->GetOrCreate( 7442 value, 7443 [this, value]() { return __ NewLiteral<uint32_t>(value); }); 7444} 7445 7446Literal* CodeGeneratorARM::DeduplicateMethodLiteral(MethodReference target_method, 7447 MethodToLiteralMap* map) { 7448 return map->GetOrCreate( 7449 target_method, 7450 [this]() { return __ NewLiteral<uint32_t>(/* placeholder */ 0u); }); 7451} 7452 7453Literal* CodeGeneratorARM::DeduplicateMethodAddressLiteral(MethodReference target_method) { 7454 return DeduplicateMethodLiteral(target_method, &method_patches_); 7455} 7456 7457Literal* CodeGeneratorARM::DeduplicateMethodCodeLiteral(MethodReference target_method) { 7458 return DeduplicateMethodLiteral(target_method, &call_patches_); 7459} 7460 7461void LocationsBuilderARM::VisitMultiplyAccumulate(HMultiplyAccumulate* instr) { 7462 LocationSummary* locations = 7463 new (GetGraph()->GetArena()) LocationSummary(instr, LocationSummary::kNoCall); 7464 locations->SetInAt(HMultiplyAccumulate::kInputAccumulatorIndex, 7465 Location::RequiresRegister()); 7466 locations->SetInAt(HMultiplyAccumulate::kInputMulLeftIndex, Location::RequiresRegister()); 7467 locations->SetInAt(HMultiplyAccumulate::kInputMulRightIndex, Location::RequiresRegister()); 7468 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 7469} 7470 7471void InstructionCodeGeneratorARM::VisitMultiplyAccumulate(HMultiplyAccumulate* instr) { 7472 LocationSummary* locations = instr->GetLocations(); 7473 Register res = locations->Out().AsRegister<Register>(); 7474 Register accumulator = 7475 locations->InAt(HMultiplyAccumulate::kInputAccumulatorIndex).AsRegister<Register>(); 7476 Register mul_left = 7477 locations->InAt(HMultiplyAccumulate::kInputMulLeftIndex).AsRegister<Register>(); 7478 Register mul_right = 7479 locations->InAt(HMultiplyAccumulate::kInputMulRightIndex).AsRegister<Register>(); 7480 7481 if (instr->GetOpKind() == HInstruction::kAdd) { 7482 __ mla(res, mul_left, mul_right, accumulator); 7483 } else { 7484 __ mls(res, mul_left, mul_right, accumulator); 7485 } 7486} 7487 7488void LocationsBuilderARM::VisitBoundType(HBoundType* instruction ATTRIBUTE_UNUSED) { 7489 // Nothing to do, this should be removed during prepare for register allocator. 7490 LOG(FATAL) << "Unreachable"; 7491} 7492 7493void InstructionCodeGeneratorARM::VisitBoundType(HBoundType* instruction ATTRIBUTE_UNUSED) { 7494 // Nothing to do, this should be removed during prepare for register allocator. 7495 LOG(FATAL) << "Unreachable"; 7496} 7497 7498// Simple implementation of packed switch - generate cascaded compare/jumps. 7499void LocationsBuilderARM::VisitPackedSwitch(HPackedSwitch* switch_instr) { 7500 LocationSummary* locations = 7501 new (GetGraph()->GetArena()) LocationSummary(switch_instr, LocationSummary::kNoCall); 7502 locations->SetInAt(0, Location::RequiresRegister()); 7503 if (switch_instr->GetNumEntries() > kPackedSwitchCompareJumpThreshold && 7504 codegen_->GetAssembler()->IsThumb()) { 7505 locations->AddTemp(Location::RequiresRegister()); // We need a temp for the table base. 7506 if (switch_instr->GetStartValue() != 0) { 7507 locations->AddTemp(Location::RequiresRegister()); // We need a temp for the bias. 7508 } 7509 } 7510} 7511 7512void InstructionCodeGeneratorARM::VisitPackedSwitch(HPackedSwitch* switch_instr) { 7513 int32_t lower_bound = switch_instr->GetStartValue(); 7514 uint32_t num_entries = switch_instr->GetNumEntries(); 7515 LocationSummary* locations = switch_instr->GetLocations(); 7516 Register value_reg = locations->InAt(0).AsRegister<Register>(); 7517 HBasicBlock* default_block = switch_instr->GetDefaultBlock(); 7518 7519 if (num_entries <= kPackedSwitchCompareJumpThreshold || !codegen_->GetAssembler()->IsThumb()) { 7520 // Create a series of compare/jumps. 7521 Register temp_reg = IP; 7522 // Note: It is fine for the below AddConstantSetFlags() using IP register to temporarily store 7523 // the immediate, because IP is used as the destination register. For the other 7524 // AddConstantSetFlags() and GenerateCompareWithImmediate(), the immediate values are constant, 7525 // and they can be encoded in the instruction without making use of IP register. 7526 __ AddConstantSetFlags(temp_reg, value_reg, -lower_bound); 7527 7528 const ArenaVector<HBasicBlock*>& successors = switch_instr->GetBlock()->GetSuccessors(); 7529 // Jump to successors[0] if value == lower_bound. 7530 __ b(codegen_->GetLabelOf(successors[0]), EQ); 7531 int32_t last_index = 0; 7532 for (; num_entries - last_index > 2; last_index += 2) { 7533 __ AddConstantSetFlags(temp_reg, temp_reg, -2); 7534 // Jump to successors[last_index + 1] if value < case_value[last_index + 2]. 7535 __ b(codegen_->GetLabelOf(successors[last_index + 1]), LO); 7536 // Jump to successors[last_index + 2] if value == case_value[last_index + 2]. 7537 __ b(codegen_->GetLabelOf(successors[last_index + 2]), EQ); 7538 } 7539 if (num_entries - last_index == 2) { 7540 // The last missing case_value. 7541 __ CmpConstant(temp_reg, 1); 7542 __ b(codegen_->GetLabelOf(successors[last_index + 1]), EQ); 7543 } 7544 7545 // And the default for any other value. 7546 if (!codegen_->GoesToNextBlock(switch_instr->GetBlock(), default_block)) { 7547 __ b(codegen_->GetLabelOf(default_block)); 7548 } 7549 } else { 7550 // Create a table lookup. 7551 Register temp_reg = locations->GetTemp(0).AsRegister<Register>(); 7552 7553 // Materialize a pointer to the switch table 7554 std::vector<Label*> labels(num_entries); 7555 const ArenaVector<HBasicBlock*>& successors = switch_instr->GetBlock()->GetSuccessors(); 7556 for (uint32_t i = 0; i < num_entries; i++) { 7557 labels[i] = codegen_->GetLabelOf(successors[i]); 7558 } 7559 JumpTable* table = __ CreateJumpTable(std::move(labels), temp_reg); 7560 7561 // Remove the bias. 7562 Register key_reg; 7563 if (lower_bound != 0) { 7564 key_reg = locations->GetTemp(1).AsRegister<Register>(); 7565 __ AddConstant(key_reg, value_reg, -lower_bound); 7566 } else { 7567 key_reg = value_reg; 7568 } 7569 7570 // Check whether the value is in the table, jump to default block if not. 7571 __ CmpConstant(key_reg, num_entries - 1); 7572 __ b(codegen_->GetLabelOf(default_block), Condition::HI); 7573 7574 // Load the displacement from the table. 7575 __ ldr(temp_reg, Address(temp_reg, key_reg, Shift::LSL, 2)); 7576 7577 // Dispatch is a direct add to the PC (for Thumb2). 7578 __ EmitJumpTableDispatch(table, temp_reg); 7579 } 7580} 7581 7582void LocationsBuilderARM::VisitArmDexCacheArraysBase(HArmDexCacheArraysBase* base) { 7583 LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(base); 7584 locations->SetOut(Location::RequiresRegister()); 7585} 7586 7587void InstructionCodeGeneratorARM::VisitArmDexCacheArraysBase(HArmDexCacheArraysBase* base) { 7588 Register base_reg = base->GetLocations()->Out().AsRegister<Register>(); 7589 CodeGeneratorARM::PcRelativePatchInfo* labels = 7590 codegen_->NewPcRelativeDexCacheArrayPatch(base->GetDexFile(), base->GetElementOffset()); 7591 __ BindTrackedLabel(&labels->movw_label); 7592 __ movw(base_reg, /* placeholder */ 0u); 7593 __ BindTrackedLabel(&labels->movt_label); 7594 __ movt(base_reg, /* placeholder */ 0u); 7595 __ BindTrackedLabel(&labels->add_pc_label); 7596 __ add(base_reg, base_reg, ShifterOperand(PC)); 7597} 7598 7599void CodeGeneratorARM::MoveFromReturnRegister(Location trg, Primitive::Type type) { 7600 if (!trg.IsValid()) { 7601 DCHECK_EQ(type, Primitive::kPrimVoid); 7602 return; 7603 } 7604 7605 DCHECK_NE(type, Primitive::kPrimVoid); 7606 7607 Location return_loc = InvokeDexCallingConventionVisitorARM().GetReturnLocation(type); 7608 if (return_loc.Equals(trg)) { 7609 return; 7610 } 7611 7612 // TODO: Consider pairs in the parallel move resolver, then this could be nicely merged 7613 // with the last branch. 7614 if (type == Primitive::kPrimLong) { 7615 HParallelMove parallel_move(GetGraph()->GetArena()); 7616 parallel_move.AddMove(return_loc.ToLow(), trg.ToLow(), Primitive::kPrimInt, nullptr); 7617 parallel_move.AddMove(return_loc.ToHigh(), trg.ToHigh(), Primitive::kPrimInt, nullptr); 7618 GetMoveResolver()->EmitNativeCode(¶llel_move); 7619 } else if (type == Primitive::kPrimDouble) { 7620 HParallelMove parallel_move(GetGraph()->GetArena()); 7621 parallel_move.AddMove(return_loc.ToLow(), trg.ToLow(), Primitive::kPrimFloat, nullptr); 7622 parallel_move.AddMove(return_loc.ToHigh(), trg.ToHigh(), Primitive::kPrimFloat, nullptr); 7623 GetMoveResolver()->EmitNativeCode(¶llel_move); 7624 } else { 7625 // Let the parallel move resolver take care of all of this. 7626 HParallelMove parallel_move(GetGraph()->GetArena()); 7627 parallel_move.AddMove(return_loc, trg, type, nullptr); 7628 GetMoveResolver()->EmitNativeCode(¶llel_move); 7629 } 7630} 7631 7632void LocationsBuilderARM::VisitClassTableGet(HClassTableGet* instruction) { 7633 LocationSummary* locations = 7634 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall); 7635 locations->SetInAt(0, Location::RequiresRegister()); 7636 locations->SetOut(Location::RequiresRegister()); 7637} 7638 7639void InstructionCodeGeneratorARM::VisitClassTableGet(HClassTableGet* instruction) { 7640 LocationSummary* locations = instruction->GetLocations(); 7641 if (instruction->GetTableKind() == HClassTableGet::TableKind::kVTable) { 7642 uint32_t method_offset = mirror::Class::EmbeddedVTableEntryOffset( 7643 instruction->GetIndex(), kArmPointerSize).SizeValue(); 7644 __ LoadFromOffset(kLoadWord, 7645 locations->Out().AsRegister<Register>(), 7646 locations->InAt(0).AsRegister<Register>(), 7647 method_offset); 7648 } else { 7649 uint32_t method_offset = static_cast<uint32_t>(ImTable::OffsetOfElement( 7650 instruction->GetIndex(), kArmPointerSize)); 7651 __ LoadFromOffset(kLoadWord, 7652 locations->Out().AsRegister<Register>(), 7653 locations->InAt(0).AsRegister<Register>(), 7654 mirror::Class::ImtPtrOffset(kArmPointerSize).Uint32Value()); 7655 __ LoadFromOffset(kLoadWord, 7656 locations->Out().AsRegister<Register>(), 7657 locations->Out().AsRegister<Register>(), 7658 method_offset); 7659 } 7660} 7661 7662#undef __ 7663#undef QUICK_ENTRY_POINT 7664 7665} // namespace arm 7666} // namespace art 7667