code_generator_arm.cc revision fe814e89965ddf9a8b603863bd28259f8dd7be35
1/* 2 * Copyright (C) 2014 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17#include "code_generator_arm.h" 18 19#include "arch/arm/instruction_set_features_arm.h" 20#include "art_method.h" 21#include "code_generator_utils.h" 22#include "compiled_method.h" 23#include "entrypoints/quick/quick_entrypoints.h" 24#include "gc/accounting/card_table.h" 25#include "intrinsics.h" 26#include "intrinsics_arm.h" 27#include "mirror/array-inl.h" 28#include "mirror/class-inl.h" 29#include "thread.h" 30#include "utils/arm/assembler_arm.h" 31#include "utils/arm/managed_register_arm.h" 32#include "utils/assembler.h" 33#include "utils/stack_checks.h" 34 35namespace art { 36 37template<class MirrorType> 38class GcRoot; 39 40namespace arm { 41 42static bool ExpectedPairLayout(Location location) { 43 // We expected this for both core and fpu register pairs. 44 return ((location.low() & 1) == 0) && (location.low() + 1 == location.high()); 45} 46 47static constexpr int kCurrentMethodStackOffset = 0; 48static constexpr Register kMethodRegisterArgument = R0; 49 50static constexpr Register kCoreAlwaysSpillRegister = R5; 51static constexpr Register kCoreCalleeSaves[] = 52 { R5, R6, R7, R8, R10, R11, LR }; 53static constexpr SRegister kFpuCalleeSaves[] = 54 { S16, S17, S18, S19, S20, S21, S22, S23, S24, S25, S26, S27, S28, S29, S30, S31 }; 55 56// D31 cannot be split into two S registers, and the register allocator only works on 57// S registers. Therefore there is no need to block it. 58static constexpr DRegister DTMP = D31; 59 60static constexpr uint32_t kPackedSwitchCompareJumpThreshold = 7; 61 62// NOLINT on __ macro to suppress wrong warning/fix (misc-macro-parentheses) from clang-tidy. 63#define __ down_cast<ArmAssembler*>(codegen->GetAssembler())-> // NOLINT 64#define QUICK_ENTRY_POINT(x) QUICK_ENTRYPOINT_OFFSET(kArmPointerSize, x).Int32Value() 65 66static constexpr int kRegListThreshold = 4; 67 68// SaveLiveRegisters and RestoreLiveRegisters from SlowPathCodeARM operate on sets of S registers, 69// for each live D registers they treat two corresponding S registers as live ones. 70// 71// Two following functions (SaveContiguousSRegisterList, RestoreContiguousSRegisterList) build 72// from a list of contiguous S registers a list of contiguous D registers (processing first/last 73// S registers corner cases) and save/restore this new list treating them as D registers. 74// - decreasing code size 75// - avoiding hazards on Cortex-A57, when a pair of S registers for an actual live D register is 76// restored and then used in regular non SlowPath code as D register. 77// 78// For the following example (v means the S register is live): 79// D names: | D0 | D1 | D2 | D4 | ... 80// S names: | S0 | S1 | S2 | S3 | S4 | S5 | S6 | S7 | ... 81// Live? | | v | v | v | v | v | v | | ... 82// 83// S1 and S6 will be saved/restored independently; D registers list (D1, D2) will be processed 84// as D registers. 85static size_t SaveContiguousSRegisterList(size_t first, 86 size_t last, 87 CodeGenerator* codegen, 88 size_t stack_offset) { 89 DCHECK_LE(first, last); 90 if ((first == last) && (first == 0)) { 91 stack_offset += codegen->SaveFloatingPointRegister(stack_offset, first); 92 return stack_offset; 93 } 94 if (first % 2 == 1) { 95 stack_offset += codegen->SaveFloatingPointRegister(stack_offset, first++); 96 } 97 98 bool save_last = false; 99 if (last % 2 == 0) { 100 save_last = true; 101 --last; 102 } 103 104 if (first < last) { 105 DRegister d_reg = static_cast<DRegister>(first / 2); 106 DCHECK_EQ((last - first + 1) % 2, 0u); 107 size_t number_of_d_regs = (last - first + 1) / 2; 108 109 if (number_of_d_regs == 1) { 110 __ StoreDToOffset(d_reg, SP, stack_offset); 111 } else if (number_of_d_regs > 1) { 112 __ add(IP, SP, ShifterOperand(stack_offset)); 113 __ vstmiad(IP, d_reg, number_of_d_regs); 114 } 115 stack_offset += number_of_d_regs * kArmWordSize * 2; 116 } 117 118 if (save_last) { 119 stack_offset += codegen->SaveFloatingPointRegister(stack_offset, last + 1); 120 } 121 122 return stack_offset; 123} 124 125static size_t RestoreContiguousSRegisterList(size_t first, 126 size_t last, 127 CodeGenerator* codegen, 128 size_t stack_offset) { 129 DCHECK_LE(first, last); 130 if ((first == last) && (first == 0)) { 131 stack_offset += codegen->RestoreFloatingPointRegister(stack_offset, first); 132 return stack_offset; 133 } 134 if (first % 2 == 1) { 135 stack_offset += codegen->RestoreFloatingPointRegister(stack_offset, first++); 136 } 137 138 bool restore_last = false; 139 if (last % 2 == 0) { 140 restore_last = true; 141 --last; 142 } 143 144 if (first < last) { 145 DRegister d_reg = static_cast<DRegister>(first / 2); 146 DCHECK_EQ((last - first + 1) % 2, 0u); 147 size_t number_of_d_regs = (last - first + 1) / 2; 148 if (number_of_d_regs == 1) { 149 __ LoadDFromOffset(d_reg, SP, stack_offset); 150 } else if (number_of_d_regs > 1) { 151 __ add(IP, SP, ShifterOperand(stack_offset)); 152 __ vldmiad(IP, d_reg, number_of_d_regs); 153 } 154 stack_offset += number_of_d_regs * kArmWordSize * 2; 155 } 156 157 if (restore_last) { 158 stack_offset += codegen->RestoreFloatingPointRegister(stack_offset, last + 1); 159 } 160 161 return stack_offset; 162} 163 164void SlowPathCodeARM::SaveLiveRegisters(CodeGenerator* codegen, LocationSummary* locations) { 165 size_t stack_offset = codegen->GetFirstRegisterSlotInSlowPath(); 166 size_t orig_offset = stack_offset; 167 168 const uint32_t core_spills = codegen->GetSlowPathSpills(locations, /* core_registers */ true); 169 for (uint32_t i : LowToHighBits(core_spills)) { 170 // If the register holds an object, update the stack mask. 171 if (locations->RegisterContainsObject(i)) { 172 locations->SetStackBit(stack_offset / kVRegSize); 173 } 174 DCHECK_LT(stack_offset, codegen->GetFrameSize() - codegen->FrameEntrySpillSize()); 175 DCHECK_LT(i, kMaximumNumberOfExpectedRegisters); 176 saved_core_stack_offsets_[i] = stack_offset; 177 stack_offset += kArmWordSize; 178 } 179 180 int reg_num = POPCOUNT(core_spills); 181 if (reg_num != 0) { 182 if (reg_num > kRegListThreshold) { 183 __ StoreList(RegList(core_spills), orig_offset); 184 } else { 185 stack_offset = orig_offset; 186 for (uint32_t i : LowToHighBits(core_spills)) { 187 stack_offset += codegen->SaveCoreRegister(stack_offset, i); 188 } 189 } 190 } 191 192 uint32_t fp_spills = codegen->GetSlowPathSpills(locations, /* core_registers */ false); 193 orig_offset = stack_offset; 194 for (uint32_t i : LowToHighBits(fp_spills)) { 195 DCHECK_LT(i, kMaximumNumberOfExpectedRegisters); 196 saved_fpu_stack_offsets_[i] = stack_offset; 197 stack_offset += kArmWordSize; 198 } 199 200 stack_offset = orig_offset; 201 while (fp_spills != 0u) { 202 uint32_t begin = CTZ(fp_spills); 203 uint32_t tmp = fp_spills + (1u << begin); 204 fp_spills &= tmp; // Clear the contiguous range of 1s. 205 uint32_t end = (tmp == 0u) ? 32u : CTZ(tmp); // CTZ(0) is undefined. 206 stack_offset = SaveContiguousSRegisterList(begin, end - 1, codegen, stack_offset); 207 } 208 DCHECK_LE(stack_offset, codegen->GetFrameSize() - codegen->FrameEntrySpillSize()); 209} 210 211void SlowPathCodeARM::RestoreLiveRegisters(CodeGenerator* codegen, LocationSummary* locations) { 212 size_t stack_offset = codegen->GetFirstRegisterSlotInSlowPath(); 213 size_t orig_offset = stack_offset; 214 215 const uint32_t core_spills = codegen->GetSlowPathSpills(locations, /* core_registers */ true); 216 for (uint32_t i : LowToHighBits(core_spills)) { 217 DCHECK_LT(stack_offset, codegen->GetFrameSize() - codegen->FrameEntrySpillSize()); 218 DCHECK_LT(i, kMaximumNumberOfExpectedRegisters); 219 stack_offset += kArmWordSize; 220 } 221 222 int reg_num = POPCOUNT(core_spills); 223 if (reg_num != 0) { 224 if (reg_num > kRegListThreshold) { 225 __ LoadList(RegList(core_spills), orig_offset); 226 } else { 227 stack_offset = orig_offset; 228 for (uint32_t i : LowToHighBits(core_spills)) { 229 stack_offset += codegen->RestoreCoreRegister(stack_offset, i); 230 } 231 } 232 } 233 234 uint32_t fp_spills = codegen->GetSlowPathSpills(locations, /* core_registers */ false); 235 while (fp_spills != 0u) { 236 uint32_t begin = CTZ(fp_spills); 237 uint32_t tmp = fp_spills + (1u << begin); 238 fp_spills &= tmp; // Clear the contiguous range of 1s. 239 uint32_t end = (tmp == 0u) ? 32u : CTZ(tmp); // CTZ(0) is undefined. 240 stack_offset = RestoreContiguousSRegisterList(begin, end - 1, codegen, stack_offset); 241 } 242 DCHECK_LE(stack_offset, codegen->GetFrameSize() - codegen->FrameEntrySpillSize()); 243} 244 245class NullCheckSlowPathARM : public SlowPathCodeARM { 246 public: 247 explicit NullCheckSlowPathARM(HNullCheck* instruction) : SlowPathCodeARM(instruction) {} 248 249 void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { 250 CodeGeneratorARM* arm_codegen = down_cast<CodeGeneratorARM*>(codegen); 251 __ Bind(GetEntryLabel()); 252 if (instruction_->CanThrowIntoCatchBlock()) { 253 // Live registers will be restored in the catch block if caught. 254 SaveLiveRegisters(codegen, instruction_->GetLocations()); 255 } 256 arm_codegen->InvokeRuntime(kQuickThrowNullPointer, 257 instruction_, 258 instruction_->GetDexPc(), 259 this); 260 CheckEntrypointTypes<kQuickThrowNullPointer, void, void>(); 261 } 262 263 bool IsFatal() const OVERRIDE { return true; } 264 265 const char* GetDescription() const OVERRIDE { return "NullCheckSlowPathARM"; } 266 267 private: 268 DISALLOW_COPY_AND_ASSIGN(NullCheckSlowPathARM); 269}; 270 271class DivZeroCheckSlowPathARM : public SlowPathCodeARM { 272 public: 273 explicit DivZeroCheckSlowPathARM(HDivZeroCheck* instruction) : SlowPathCodeARM(instruction) {} 274 275 void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { 276 CodeGeneratorARM* arm_codegen = down_cast<CodeGeneratorARM*>(codegen); 277 __ Bind(GetEntryLabel()); 278 arm_codegen->InvokeRuntime(kQuickThrowDivZero, instruction_, instruction_->GetDexPc(), this); 279 CheckEntrypointTypes<kQuickThrowDivZero, void, void>(); 280 } 281 282 bool IsFatal() const OVERRIDE { return true; } 283 284 const char* GetDescription() const OVERRIDE { return "DivZeroCheckSlowPathARM"; } 285 286 private: 287 DISALLOW_COPY_AND_ASSIGN(DivZeroCheckSlowPathARM); 288}; 289 290class SuspendCheckSlowPathARM : public SlowPathCodeARM { 291 public: 292 SuspendCheckSlowPathARM(HSuspendCheck* instruction, HBasicBlock* successor) 293 : SlowPathCodeARM(instruction), successor_(successor) {} 294 295 void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { 296 CodeGeneratorARM* arm_codegen = down_cast<CodeGeneratorARM*>(codegen); 297 __ Bind(GetEntryLabel()); 298 arm_codegen->InvokeRuntime(kQuickTestSuspend, instruction_, instruction_->GetDexPc(), this); 299 CheckEntrypointTypes<kQuickTestSuspend, void, void>(); 300 if (successor_ == nullptr) { 301 __ b(GetReturnLabel()); 302 } else { 303 __ b(arm_codegen->GetLabelOf(successor_)); 304 } 305 } 306 307 Label* GetReturnLabel() { 308 DCHECK(successor_ == nullptr); 309 return &return_label_; 310 } 311 312 HBasicBlock* GetSuccessor() const { 313 return successor_; 314 } 315 316 const char* GetDescription() const OVERRIDE { return "SuspendCheckSlowPathARM"; } 317 318 private: 319 // If not null, the block to branch to after the suspend check. 320 HBasicBlock* const successor_; 321 322 // If `successor_` is null, the label to branch to after the suspend check. 323 Label return_label_; 324 325 DISALLOW_COPY_AND_ASSIGN(SuspendCheckSlowPathARM); 326}; 327 328class BoundsCheckSlowPathARM : public SlowPathCodeARM { 329 public: 330 explicit BoundsCheckSlowPathARM(HBoundsCheck* instruction) 331 : SlowPathCodeARM(instruction) {} 332 333 void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { 334 CodeGeneratorARM* arm_codegen = down_cast<CodeGeneratorARM*>(codegen); 335 LocationSummary* locations = instruction_->GetLocations(); 336 337 __ Bind(GetEntryLabel()); 338 if (instruction_->CanThrowIntoCatchBlock()) { 339 // Live registers will be restored in the catch block if caught. 340 SaveLiveRegisters(codegen, instruction_->GetLocations()); 341 } 342 // We're moving two locations to locations that could overlap, so we need a parallel 343 // move resolver. 344 InvokeRuntimeCallingConvention calling_convention; 345 codegen->EmitParallelMoves( 346 locations->InAt(0), 347 Location::RegisterLocation(calling_convention.GetRegisterAt(0)), 348 Primitive::kPrimInt, 349 locations->InAt(1), 350 Location::RegisterLocation(calling_convention.GetRegisterAt(1)), 351 Primitive::kPrimInt); 352 QuickEntrypointEnum entrypoint = instruction_->AsBoundsCheck()->IsStringCharAt() 353 ? kQuickThrowStringBounds 354 : kQuickThrowArrayBounds; 355 arm_codegen->InvokeRuntime(entrypoint, instruction_, instruction_->GetDexPc(), this); 356 CheckEntrypointTypes<kQuickThrowStringBounds, void, int32_t, int32_t>(); 357 CheckEntrypointTypes<kQuickThrowArrayBounds, void, int32_t, int32_t>(); 358 } 359 360 bool IsFatal() const OVERRIDE { return true; } 361 362 const char* GetDescription() const OVERRIDE { return "BoundsCheckSlowPathARM"; } 363 364 private: 365 DISALLOW_COPY_AND_ASSIGN(BoundsCheckSlowPathARM); 366}; 367 368class LoadClassSlowPathARM : public SlowPathCodeARM { 369 public: 370 LoadClassSlowPathARM(HLoadClass* cls, 371 HInstruction* at, 372 uint32_t dex_pc, 373 bool do_clinit) 374 : SlowPathCodeARM(at), cls_(cls), at_(at), dex_pc_(dex_pc), do_clinit_(do_clinit) { 375 DCHECK(at->IsLoadClass() || at->IsClinitCheck()); 376 } 377 378 void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { 379 LocationSummary* locations = at_->GetLocations(); 380 381 CodeGeneratorARM* arm_codegen = down_cast<CodeGeneratorARM*>(codegen); 382 __ Bind(GetEntryLabel()); 383 SaveLiveRegisters(codegen, locations); 384 385 InvokeRuntimeCallingConvention calling_convention; 386 __ LoadImmediate(calling_convention.GetRegisterAt(0), cls_->GetTypeIndex()); 387 QuickEntrypointEnum entrypoint = do_clinit_ ? kQuickInitializeStaticStorage 388 : kQuickInitializeType; 389 arm_codegen->InvokeRuntime(entrypoint, at_, dex_pc_, this); 390 if (do_clinit_) { 391 CheckEntrypointTypes<kQuickInitializeStaticStorage, void*, uint32_t>(); 392 } else { 393 CheckEntrypointTypes<kQuickInitializeType, void*, uint32_t>(); 394 } 395 396 // Move the class to the desired location. 397 Location out = locations->Out(); 398 if (out.IsValid()) { 399 DCHECK(out.IsRegister() && !locations->GetLiveRegisters()->ContainsCoreRegister(out.reg())); 400 arm_codegen->Move32(locations->Out(), Location::RegisterLocation(R0)); 401 } 402 RestoreLiveRegisters(codegen, locations); 403 __ b(GetExitLabel()); 404 } 405 406 const char* GetDescription() const OVERRIDE { return "LoadClassSlowPathARM"; } 407 408 private: 409 // The class this slow path will load. 410 HLoadClass* const cls_; 411 412 // The instruction where this slow path is happening. 413 // (Might be the load class or an initialization check). 414 HInstruction* const at_; 415 416 // The dex PC of `at_`. 417 const uint32_t dex_pc_; 418 419 // Whether to initialize the class. 420 const bool do_clinit_; 421 422 DISALLOW_COPY_AND_ASSIGN(LoadClassSlowPathARM); 423}; 424 425class LoadStringSlowPathARM : public SlowPathCodeARM { 426 public: 427 explicit LoadStringSlowPathARM(HLoadString* instruction) : SlowPathCodeARM(instruction) {} 428 429 void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { 430 LocationSummary* locations = instruction_->GetLocations(); 431 DCHECK(!locations->GetLiveRegisters()->ContainsCoreRegister(locations->Out().reg())); 432 HLoadString* load = instruction_->AsLoadString(); 433 const uint32_t string_index = load->GetStringIndex(); 434 Register out = locations->Out().AsRegister<Register>(); 435 Register temp = locations->GetTemp(0).AsRegister<Register>(); 436 constexpr bool call_saves_everything_except_r0 = (!kUseReadBarrier || kUseBakerReadBarrier); 437 438 CodeGeneratorARM* arm_codegen = down_cast<CodeGeneratorARM*>(codegen); 439 __ Bind(GetEntryLabel()); 440 SaveLiveRegisters(codegen, locations); 441 442 InvokeRuntimeCallingConvention calling_convention; 443 // In the unlucky case that the `temp` is R0, we preserve the address in `out` across 444 // the kSaveEverything call (or use `out` for the address after non-kSaveEverything call). 445 bool temp_is_r0 = (temp == calling_convention.GetRegisterAt(0)); 446 Register entry_address = temp_is_r0 ? out : temp; 447 DCHECK_NE(entry_address, calling_convention.GetRegisterAt(0)); 448 if (call_saves_everything_except_r0 && temp_is_r0) { 449 __ mov(entry_address, ShifterOperand(temp)); 450 } 451 452 __ LoadImmediate(calling_convention.GetRegisterAt(0), string_index); 453 arm_codegen->InvokeRuntime(kQuickResolveString, instruction_, instruction_->GetDexPc(), this); 454 CheckEntrypointTypes<kQuickResolveString, void*, uint32_t>(); 455 456 // Store the resolved String to the .bss entry. 457 if (call_saves_everything_except_r0) { 458 // The string entry address was preserved in `entry_address` thanks to kSaveEverything. 459 __ str(R0, Address(entry_address)); 460 } else { 461 // For non-Baker read barrier, we need to re-calculate the address of the string entry. 462 CodeGeneratorARM::PcRelativePatchInfo* labels = 463 arm_codegen->NewPcRelativeStringPatch(load->GetDexFile(), string_index); 464 __ BindTrackedLabel(&labels->movw_label); 465 __ movw(entry_address, /* placeholder */ 0u); 466 __ BindTrackedLabel(&labels->movt_label); 467 __ movt(entry_address, /* placeholder */ 0u); 468 __ BindTrackedLabel(&labels->add_pc_label); 469 __ add(entry_address, entry_address, ShifterOperand(PC)); 470 __ str(R0, Address(entry_address)); 471 } 472 473 arm_codegen->Move32(locations->Out(), Location::RegisterLocation(R0)); 474 RestoreLiveRegisters(codegen, locations); 475 476 __ b(GetExitLabel()); 477 } 478 479 const char* GetDescription() const OVERRIDE { return "LoadStringSlowPathARM"; } 480 481 private: 482 DISALLOW_COPY_AND_ASSIGN(LoadStringSlowPathARM); 483}; 484 485class TypeCheckSlowPathARM : public SlowPathCodeARM { 486 public: 487 TypeCheckSlowPathARM(HInstruction* instruction, bool is_fatal) 488 : SlowPathCodeARM(instruction), is_fatal_(is_fatal) {} 489 490 void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { 491 LocationSummary* locations = instruction_->GetLocations(); 492 Location arg0, arg1; 493 if (instruction_->IsInstanceOf()) { 494 arg0 = locations->InAt(1); 495 arg1 = locations->Out(); 496 } else { 497 arg0 = locations->InAt(0); 498 arg1 = locations->InAt(1); 499 } 500 DCHECK(instruction_->IsCheckCast() 501 || !locations->GetLiveRegisters()->ContainsCoreRegister(locations->Out().reg())); 502 503 CodeGeneratorARM* arm_codegen = down_cast<CodeGeneratorARM*>(codegen); 504 __ Bind(GetEntryLabel()); 505 506 if (!is_fatal_) { 507 SaveLiveRegisters(codegen, locations); 508 } 509 510 // We're moving two locations to locations that could overlap, so we need a parallel 511 // move resolver. 512 InvokeRuntimeCallingConvention calling_convention; 513 codegen->EmitParallelMoves(arg0, 514 Location::RegisterLocation(calling_convention.GetRegisterAt(0)), 515 Primitive::kPrimNot, 516 arg1, 517 Location::RegisterLocation(calling_convention.GetRegisterAt(1)), 518 Primitive::kPrimNot); 519 if (instruction_->IsInstanceOf()) { 520 arm_codegen->InvokeRuntime(kQuickInstanceofNonTrivial, 521 instruction_, 522 instruction_->GetDexPc(), 523 this); 524 CheckEntrypointTypes<kQuickInstanceofNonTrivial, size_t, mirror::Class*, mirror::Class*>(); 525 arm_codegen->Move32(locations->Out(), Location::RegisterLocation(R0)); 526 } else { 527 DCHECK(instruction_->IsCheckCast()); 528 arm_codegen->InvokeRuntime(kQuickCheckInstanceOf, 529 instruction_, 530 instruction_->GetDexPc(), 531 this); 532 CheckEntrypointTypes<kQuickCheckInstanceOf, void, mirror::Object*, mirror::Class*>(); 533 } 534 535 if (!is_fatal_) { 536 RestoreLiveRegisters(codegen, locations); 537 __ b(GetExitLabel()); 538 } 539 } 540 541 const char* GetDescription() const OVERRIDE { return "TypeCheckSlowPathARM"; } 542 543 bool IsFatal() const OVERRIDE { return is_fatal_; } 544 545 private: 546 const bool is_fatal_; 547 548 DISALLOW_COPY_AND_ASSIGN(TypeCheckSlowPathARM); 549}; 550 551class DeoptimizationSlowPathARM : public SlowPathCodeARM { 552 public: 553 explicit DeoptimizationSlowPathARM(HDeoptimize* instruction) 554 : SlowPathCodeARM(instruction) {} 555 556 void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { 557 CodeGeneratorARM* arm_codegen = down_cast<CodeGeneratorARM*>(codegen); 558 __ Bind(GetEntryLabel()); 559 arm_codegen->InvokeRuntime(kQuickDeoptimize, instruction_, instruction_->GetDexPc(), this); 560 CheckEntrypointTypes<kQuickDeoptimize, void, void>(); 561 } 562 563 const char* GetDescription() const OVERRIDE { return "DeoptimizationSlowPathARM"; } 564 565 private: 566 DISALLOW_COPY_AND_ASSIGN(DeoptimizationSlowPathARM); 567}; 568 569class ArraySetSlowPathARM : public SlowPathCodeARM { 570 public: 571 explicit ArraySetSlowPathARM(HInstruction* instruction) : SlowPathCodeARM(instruction) {} 572 573 void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { 574 LocationSummary* locations = instruction_->GetLocations(); 575 __ Bind(GetEntryLabel()); 576 SaveLiveRegisters(codegen, locations); 577 578 InvokeRuntimeCallingConvention calling_convention; 579 HParallelMove parallel_move(codegen->GetGraph()->GetArena()); 580 parallel_move.AddMove( 581 locations->InAt(0), 582 Location::RegisterLocation(calling_convention.GetRegisterAt(0)), 583 Primitive::kPrimNot, 584 nullptr); 585 parallel_move.AddMove( 586 locations->InAt(1), 587 Location::RegisterLocation(calling_convention.GetRegisterAt(1)), 588 Primitive::kPrimInt, 589 nullptr); 590 parallel_move.AddMove( 591 locations->InAt(2), 592 Location::RegisterLocation(calling_convention.GetRegisterAt(2)), 593 Primitive::kPrimNot, 594 nullptr); 595 codegen->GetMoveResolver()->EmitNativeCode(¶llel_move); 596 597 CodeGeneratorARM* arm_codegen = down_cast<CodeGeneratorARM*>(codegen); 598 arm_codegen->InvokeRuntime(kQuickAputObject, instruction_, instruction_->GetDexPc(), this); 599 CheckEntrypointTypes<kQuickAputObject, void, mirror::Array*, int32_t, mirror::Object*>(); 600 RestoreLiveRegisters(codegen, locations); 601 __ b(GetExitLabel()); 602 } 603 604 const char* GetDescription() const OVERRIDE { return "ArraySetSlowPathARM"; } 605 606 private: 607 DISALLOW_COPY_AND_ASSIGN(ArraySetSlowPathARM); 608}; 609 610// Slow path marking an object reference `ref` during a read 611// barrier. The field `obj.field` in the object `obj` holding this 612// reference does not get updated by this slow path after marking (see 613// ReadBarrierMarkAndUpdateFieldSlowPathARM below for that). 614// 615// This means that after the execution of this slow path, `ref` will 616// always be up-to-date, but `obj.field` may not; i.e., after the 617// flip, `ref` will be a to-space reference, but `obj.field` will 618// probably still be a from-space reference (unless it gets updated by 619// another thread, or if another thread installed another object 620// reference (different from `ref`) in `obj.field`). 621class ReadBarrierMarkSlowPathARM : public SlowPathCodeARM { 622 public: 623 ReadBarrierMarkSlowPathARM(HInstruction* instruction, 624 Location ref, 625 Location entrypoint = Location::NoLocation()) 626 : SlowPathCodeARM(instruction), ref_(ref), entrypoint_(entrypoint) { 627 DCHECK(kEmitCompilerReadBarrier); 628 } 629 630 const char* GetDescription() const OVERRIDE { return "ReadBarrierMarkSlowPathARM"; } 631 632 void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { 633 LocationSummary* locations = instruction_->GetLocations(); 634 Register ref_reg = ref_.AsRegister<Register>(); 635 DCHECK(locations->CanCall()); 636 DCHECK(!locations->GetLiveRegisters()->ContainsCoreRegister(ref_reg)) << ref_reg; 637 DCHECK(instruction_->IsInstanceFieldGet() || 638 instruction_->IsStaticFieldGet() || 639 instruction_->IsArrayGet() || 640 instruction_->IsArraySet() || 641 instruction_->IsLoadClass() || 642 instruction_->IsLoadString() || 643 instruction_->IsInstanceOf() || 644 instruction_->IsCheckCast() || 645 (instruction_->IsInvokeVirtual() && instruction_->GetLocations()->Intrinsified()) || 646 (instruction_->IsInvokeStaticOrDirect() && instruction_->GetLocations()->Intrinsified())) 647 << "Unexpected instruction in read barrier marking slow path: " 648 << instruction_->DebugName(); 649 // The read barrier instrumentation of object ArrayGet 650 // instructions does not support the HIntermediateAddress 651 // instruction. 652 DCHECK(!(instruction_->IsArrayGet() && 653 instruction_->AsArrayGet()->GetArray()->IsIntermediateAddress())); 654 655 __ Bind(GetEntryLabel()); 656 // No need to save live registers; it's taken care of by the 657 // entrypoint. Also, there is no need to update the stack mask, 658 // as this runtime call will not trigger a garbage collection. 659 CodeGeneratorARM* arm_codegen = down_cast<CodeGeneratorARM*>(codegen); 660 DCHECK_NE(ref_reg, SP); 661 DCHECK_NE(ref_reg, LR); 662 DCHECK_NE(ref_reg, PC); 663 // IP is used internally by the ReadBarrierMarkRegX entry point 664 // as a temporary, it cannot be the entry point's input/output. 665 DCHECK_NE(ref_reg, IP); 666 DCHECK(0 <= ref_reg && ref_reg < kNumberOfCoreRegisters) << ref_reg; 667 // "Compact" slow path, saving two moves. 668 // 669 // Instead of using the standard runtime calling convention (input 670 // and output in R0): 671 // 672 // R0 <- ref 673 // R0 <- ReadBarrierMark(R0) 674 // ref <- R0 675 // 676 // we just use rX (the register containing `ref`) as input and output 677 // of a dedicated entrypoint: 678 // 679 // rX <- ReadBarrierMarkRegX(rX) 680 // 681 if (entrypoint_.IsValid()) { 682 arm_codegen->ValidateInvokeRuntimeWithoutRecordingPcInfo(instruction_, this); 683 __ blx(entrypoint_.AsRegister<Register>()); 684 } else { 685 int32_t entry_point_offset = 686 CodeGenerator::GetReadBarrierMarkEntryPointsOffset<kArmPointerSize>(ref_reg); 687 // This runtime call does not require a stack map. 688 arm_codegen->InvokeRuntimeWithoutRecordingPcInfo(entry_point_offset, instruction_, this); 689 } 690 __ b(GetExitLabel()); 691 } 692 693 private: 694 // The location (register) of the marked object reference. 695 const Location ref_; 696 697 // The location of the entrypoint if already loaded. 698 const Location entrypoint_; 699 700 DISALLOW_COPY_AND_ASSIGN(ReadBarrierMarkSlowPathARM); 701}; 702 703// Slow path marking an object reference `ref` during a read barrier, 704// and if needed, atomically updating the field `obj.field` in the 705// object `obj` holding this reference after marking (contrary to 706// ReadBarrierMarkSlowPathARM above, which never tries to update 707// `obj.field`). 708// 709// This means that after the execution of this slow path, both `ref` 710// and `obj.field` will be up-to-date; i.e., after the flip, both will 711// hold the same to-space reference (unless another thread installed 712// another object reference (different from `ref`) in `obj.field`). 713class ReadBarrierMarkAndUpdateFieldSlowPathARM : public SlowPathCodeARM { 714 public: 715 ReadBarrierMarkAndUpdateFieldSlowPathARM(HInstruction* instruction, 716 Location ref, 717 Register obj, 718 Location field_offset, 719 Register temp1, 720 Register temp2) 721 : SlowPathCodeARM(instruction), 722 ref_(ref), 723 obj_(obj), 724 field_offset_(field_offset), 725 temp1_(temp1), 726 temp2_(temp2) { 727 DCHECK(kEmitCompilerReadBarrier); 728 } 729 730 const char* GetDescription() const OVERRIDE { return "ReadBarrierMarkAndUpdateFieldSlowPathARM"; } 731 732 void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { 733 LocationSummary* locations = instruction_->GetLocations(); 734 Register ref_reg = ref_.AsRegister<Register>(); 735 DCHECK(locations->CanCall()); 736 DCHECK(!locations->GetLiveRegisters()->ContainsCoreRegister(ref_reg)) << ref_reg; 737 // This slow path is only used by the UnsafeCASObject intrinsic. 738 DCHECK((instruction_->IsInvokeVirtual() && instruction_->GetLocations()->Intrinsified())) 739 << "Unexpected instruction in read barrier marking and field updating slow path: " 740 << instruction_->DebugName(); 741 DCHECK(instruction_->GetLocations()->Intrinsified()); 742 DCHECK_EQ(instruction_->AsInvoke()->GetIntrinsic(), Intrinsics::kUnsafeCASObject); 743 DCHECK(field_offset_.IsRegisterPair()) << field_offset_; 744 745 __ Bind(GetEntryLabel()); 746 747 // Save the old reference. 748 // Note that we cannot use IP to save the old reference, as IP is 749 // used internally by the ReadBarrierMarkRegX entry point, and we 750 // need the old reference after the call to that entry point. 751 DCHECK_NE(temp1_, IP); 752 __ Mov(temp1_, ref_reg); 753 754 // No need to save live registers; it's taken care of by the 755 // entrypoint. Also, there is no need to update the stack mask, 756 // as this runtime call will not trigger a garbage collection. 757 CodeGeneratorARM* arm_codegen = down_cast<CodeGeneratorARM*>(codegen); 758 DCHECK_NE(ref_reg, SP); 759 DCHECK_NE(ref_reg, LR); 760 DCHECK_NE(ref_reg, PC); 761 // IP is used internally by the ReadBarrierMarkRegX entry point 762 // as a temporary, it cannot be the entry point's input/output. 763 DCHECK_NE(ref_reg, IP); 764 DCHECK(0 <= ref_reg && ref_reg < kNumberOfCoreRegisters) << ref_reg; 765 // "Compact" slow path, saving two moves. 766 // 767 // Instead of using the standard runtime calling convention (input 768 // and output in R0): 769 // 770 // R0 <- ref 771 // R0 <- ReadBarrierMark(R0) 772 // ref <- R0 773 // 774 // we just use rX (the register containing `ref`) as input and output 775 // of a dedicated entrypoint: 776 // 777 // rX <- ReadBarrierMarkRegX(rX) 778 // 779 int32_t entry_point_offset = 780 CodeGenerator::GetReadBarrierMarkEntryPointsOffset<kArmPointerSize>(ref_reg); 781 // This runtime call does not require a stack map. 782 arm_codegen->InvokeRuntimeWithoutRecordingPcInfo(entry_point_offset, instruction_, this); 783 784 // If the new reference is different from the old reference, 785 // update the field in the holder (`*(obj_ + field_offset_)`). 786 // 787 // Note that this field could also hold a different object, if 788 // another thread had concurrently changed it. In that case, the 789 // LDREX/SUBS/ITNE sequence of instructions in the compare-and-set 790 // (CAS) operation below would abort the CAS, leaving the field 791 // as-is. 792 Label done; 793 __ cmp(temp1_, ShifterOperand(ref_reg)); 794 __ b(&done, EQ); 795 796 // Update the the holder's field atomically. This may fail if 797 // mutator updates before us, but it's OK. This is achieved 798 // using a strong compare-and-set (CAS) operation with relaxed 799 // memory synchronization ordering, where the expected value is 800 // the old reference and the desired value is the new reference. 801 802 // Convenience aliases. 803 Register base = obj_; 804 // The UnsafeCASObject intrinsic uses a register pair as field 805 // offset ("long offset"), of which only the low part contains 806 // data. 807 Register offset = field_offset_.AsRegisterPairLow<Register>(); 808 Register expected = temp1_; 809 Register value = ref_reg; 810 Register tmp_ptr = IP; // Pointer to actual memory. 811 Register tmp = temp2_; // Value in memory. 812 813 __ add(tmp_ptr, base, ShifterOperand(offset)); 814 815 if (kPoisonHeapReferences) { 816 __ PoisonHeapReference(expected); 817 if (value == expected) { 818 // Do not poison `value`, as it is the same register as 819 // `expected`, which has just been poisoned. 820 } else { 821 __ PoisonHeapReference(value); 822 } 823 } 824 825 // do { 826 // tmp = [r_ptr] - expected; 827 // } while (tmp == 0 && failure([r_ptr] <- r_new_value)); 828 829 Label loop_head, exit_loop; 830 __ Bind(&loop_head); 831 832 __ ldrex(tmp, tmp_ptr); 833 834 __ subs(tmp, tmp, ShifterOperand(expected)); 835 836 __ it(NE); 837 __ clrex(NE); 838 839 __ b(&exit_loop, NE); 840 841 __ strex(tmp, value, tmp_ptr); 842 __ cmp(tmp, ShifterOperand(1)); 843 __ b(&loop_head, EQ); 844 845 __ Bind(&exit_loop); 846 847 if (kPoisonHeapReferences) { 848 __ UnpoisonHeapReference(expected); 849 if (value == expected) { 850 // Do not unpoison `value`, as it is the same register as 851 // `expected`, which has just been unpoisoned. 852 } else { 853 __ UnpoisonHeapReference(value); 854 } 855 } 856 857 __ Bind(&done); 858 __ b(GetExitLabel()); 859 } 860 861 private: 862 // The location (register) of the marked object reference. 863 const Location ref_; 864 // The register containing the object holding the marked object reference field. 865 const Register obj_; 866 // The location of the offset of the marked reference field within `obj_`. 867 Location field_offset_; 868 869 const Register temp1_; 870 const Register temp2_; 871 872 DISALLOW_COPY_AND_ASSIGN(ReadBarrierMarkAndUpdateFieldSlowPathARM); 873}; 874 875// Slow path generating a read barrier for a heap reference. 876class ReadBarrierForHeapReferenceSlowPathARM : public SlowPathCodeARM { 877 public: 878 ReadBarrierForHeapReferenceSlowPathARM(HInstruction* instruction, 879 Location out, 880 Location ref, 881 Location obj, 882 uint32_t offset, 883 Location index) 884 : SlowPathCodeARM(instruction), 885 out_(out), 886 ref_(ref), 887 obj_(obj), 888 offset_(offset), 889 index_(index) { 890 DCHECK(kEmitCompilerReadBarrier); 891 // If `obj` is equal to `out` or `ref`, it means the initial object 892 // has been overwritten by (or after) the heap object reference load 893 // to be instrumented, e.g.: 894 // 895 // __ LoadFromOffset(kLoadWord, out, out, offset); 896 // codegen_->GenerateReadBarrierSlow(instruction, out_loc, out_loc, out_loc, offset); 897 // 898 // In that case, we have lost the information about the original 899 // object, and the emitted read barrier cannot work properly. 900 DCHECK(!obj.Equals(out)) << "obj=" << obj << " out=" << out; 901 DCHECK(!obj.Equals(ref)) << "obj=" << obj << " ref=" << ref; 902 } 903 904 void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { 905 CodeGeneratorARM* arm_codegen = down_cast<CodeGeneratorARM*>(codegen); 906 LocationSummary* locations = instruction_->GetLocations(); 907 Register reg_out = out_.AsRegister<Register>(); 908 DCHECK(locations->CanCall()); 909 DCHECK(!locations->GetLiveRegisters()->ContainsCoreRegister(reg_out)); 910 DCHECK(instruction_->IsInstanceFieldGet() || 911 instruction_->IsStaticFieldGet() || 912 instruction_->IsArrayGet() || 913 instruction_->IsInstanceOf() || 914 instruction_->IsCheckCast() || 915 (instruction_->IsInvokeVirtual()) && instruction_->GetLocations()->Intrinsified()) 916 << "Unexpected instruction in read barrier for heap reference slow path: " 917 << instruction_->DebugName(); 918 // The read barrier instrumentation of object ArrayGet 919 // instructions does not support the HIntermediateAddress 920 // instruction. 921 DCHECK(!(instruction_->IsArrayGet() && 922 instruction_->AsArrayGet()->GetArray()->IsIntermediateAddress())); 923 924 __ Bind(GetEntryLabel()); 925 SaveLiveRegisters(codegen, locations); 926 927 // We may have to change the index's value, but as `index_` is a 928 // constant member (like other "inputs" of this slow path), 929 // introduce a copy of it, `index`. 930 Location index = index_; 931 if (index_.IsValid()) { 932 // Handle `index_` for HArrayGet and UnsafeGetObject/UnsafeGetObjectVolatile intrinsics. 933 if (instruction_->IsArrayGet()) { 934 // Compute the actual memory offset and store it in `index`. 935 Register index_reg = index_.AsRegister<Register>(); 936 DCHECK(locations->GetLiveRegisters()->ContainsCoreRegister(index_reg)); 937 if (codegen->IsCoreCalleeSaveRegister(index_reg)) { 938 // We are about to change the value of `index_reg` (see the 939 // calls to art::arm::Thumb2Assembler::Lsl and 940 // art::arm::Thumb2Assembler::AddConstant below), but it has 941 // not been saved by the previous call to 942 // art::SlowPathCode::SaveLiveRegisters, as it is a 943 // callee-save register -- 944 // art::SlowPathCode::SaveLiveRegisters does not consider 945 // callee-save registers, as it has been designed with the 946 // assumption that callee-save registers are supposed to be 947 // handled by the called function. So, as a callee-save 948 // register, `index_reg` _would_ eventually be saved onto 949 // the stack, but it would be too late: we would have 950 // changed its value earlier. Therefore, we manually save 951 // it here into another freely available register, 952 // `free_reg`, chosen of course among the caller-save 953 // registers (as a callee-save `free_reg` register would 954 // exhibit the same problem). 955 // 956 // Note we could have requested a temporary register from 957 // the register allocator instead; but we prefer not to, as 958 // this is a slow path, and we know we can find a 959 // caller-save register that is available. 960 Register free_reg = FindAvailableCallerSaveRegister(codegen); 961 __ Mov(free_reg, index_reg); 962 index_reg = free_reg; 963 index = Location::RegisterLocation(index_reg); 964 } else { 965 // The initial register stored in `index_` has already been 966 // saved in the call to art::SlowPathCode::SaveLiveRegisters 967 // (as it is not a callee-save register), so we can freely 968 // use it. 969 } 970 // Shifting the index value contained in `index_reg` by the scale 971 // factor (2) cannot overflow in practice, as the runtime is 972 // unable to allocate object arrays with a size larger than 973 // 2^26 - 1 (that is, 2^28 - 4 bytes). 974 __ Lsl(index_reg, index_reg, TIMES_4); 975 static_assert( 976 sizeof(mirror::HeapReference<mirror::Object>) == sizeof(int32_t), 977 "art::mirror::HeapReference<art::mirror::Object> and int32_t have different sizes."); 978 __ AddConstant(index_reg, index_reg, offset_); 979 } else { 980 // In the case of the UnsafeGetObject/UnsafeGetObjectVolatile 981 // intrinsics, `index_` is not shifted by a scale factor of 2 982 // (as in the case of ArrayGet), as it is actually an offset 983 // to an object field within an object. 984 DCHECK(instruction_->IsInvoke()) << instruction_->DebugName(); 985 DCHECK(instruction_->GetLocations()->Intrinsified()); 986 DCHECK((instruction_->AsInvoke()->GetIntrinsic() == Intrinsics::kUnsafeGetObject) || 987 (instruction_->AsInvoke()->GetIntrinsic() == Intrinsics::kUnsafeGetObjectVolatile)) 988 << instruction_->AsInvoke()->GetIntrinsic(); 989 DCHECK_EQ(offset_, 0U); 990 DCHECK(index_.IsRegisterPair()); 991 // UnsafeGet's offset location is a register pair, the low 992 // part contains the correct offset. 993 index = index_.ToLow(); 994 } 995 } 996 997 // We're moving two or three locations to locations that could 998 // overlap, so we need a parallel move resolver. 999 InvokeRuntimeCallingConvention calling_convention; 1000 HParallelMove parallel_move(codegen->GetGraph()->GetArena()); 1001 parallel_move.AddMove(ref_, 1002 Location::RegisterLocation(calling_convention.GetRegisterAt(0)), 1003 Primitive::kPrimNot, 1004 nullptr); 1005 parallel_move.AddMove(obj_, 1006 Location::RegisterLocation(calling_convention.GetRegisterAt(1)), 1007 Primitive::kPrimNot, 1008 nullptr); 1009 if (index.IsValid()) { 1010 parallel_move.AddMove(index, 1011 Location::RegisterLocation(calling_convention.GetRegisterAt(2)), 1012 Primitive::kPrimInt, 1013 nullptr); 1014 codegen->GetMoveResolver()->EmitNativeCode(¶llel_move); 1015 } else { 1016 codegen->GetMoveResolver()->EmitNativeCode(¶llel_move); 1017 __ LoadImmediate(calling_convention.GetRegisterAt(2), offset_); 1018 } 1019 arm_codegen->InvokeRuntime(kQuickReadBarrierSlow, instruction_, instruction_->GetDexPc(), this); 1020 CheckEntrypointTypes< 1021 kQuickReadBarrierSlow, mirror::Object*, mirror::Object*, mirror::Object*, uint32_t>(); 1022 arm_codegen->Move32(out_, Location::RegisterLocation(R0)); 1023 1024 RestoreLiveRegisters(codegen, locations); 1025 __ b(GetExitLabel()); 1026 } 1027 1028 const char* GetDescription() const OVERRIDE { return "ReadBarrierForHeapReferenceSlowPathARM"; } 1029 1030 private: 1031 Register FindAvailableCallerSaveRegister(CodeGenerator* codegen) { 1032 size_t ref = static_cast<int>(ref_.AsRegister<Register>()); 1033 size_t obj = static_cast<int>(obj_.AsRegister<Register>()); 1034 for (size_t i = 0, e = codegen->GetNumberOfCoreRegisters(); i < e; ++i) { 1035 if (i != ref && i != obj && !codegen->IsCoreCalleeSaveRegister(i)) { 1036 return static_cast<Register>(i); 1037 } 1038 } 1039 // We shall never fail to find a free caller-save register, as 1040 // there are more than two core caller-save registers on ARM 1041 // (meaning it is possible to find one which is different from 1042 // `ref` and `obj`). 1043 DCHECK_GT(codegen->GetNumberOfCoreCallerSaveRegisters(), 2u); 1044 LOG(FATAL) << "Could not find a free caller-save register"; 1045 UNREACHABLE(); 1046 } 1047 1048 const Location out_; 1049 const Location ref_; 1050 const Location obj_; 1051 const uint32_t offset_; 1052 // An additional location containing an index to an array. 1053 // Only used for HArrayGet and the UnsafeGetObject & 1054 // UnsafeGetObjectVolatile intrinsics. 1055 const Location index_; 1056 1057 DISALLOW_COPY_AND_ASSIGN(ReadBarrierForHeapReferenceSlowPathARM); 1058}; 1059 1060// Slow path generating a read barrier for a GC root. 1061class ReadBarrierForRootSlowPathARM : public SlowPathCodeARM { 1062 public: 1063 ReadBarrierForRootSlowPathARM(HInstruction* instruction, Location out, Location root) 1064 : SlowPathCodeARM(instruction), out_(out), root_(root) { 1065 DCHECK(kEmitCompilerReadBarrier); 1066 } 1067 1068 void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { 1069 LocationSummary* locations = instruction_->GetLocations(); 1070 Register reg_out = out_.AsRegister<Register>(); 1071 DCHECK(locations->CanCall()); 1072 DCHECK(!locations->GetLiveRegisters()->ContainsCoreRegister(reg_out)); 1073 DCHECK(instruction_->IsLoadClass() || instruction_->IsLoadString()) 1074 << "Unexpected instruction in read barrier for GC root slow path: " 1075 << instruction_->DebugName(); 1076 1077 __ Bind(GetEntryLabel()); 1078 SaveLiveRegisters(codegen, locations); 1079 1080 InvokeRuntimeCallingConvention calling_convention; 1081 CodeGeneratorARM* arm_codegen = down_cast<CodeGeneratorARM*>(codegen); 1082 arm_codegen->Move32(Location::RegisterLocation(calling_convention.GetRegisterAt(0)), root_); 1083 arm_codegen->InvokeRuntime(kQuickReadBarrierForRootSlow, 1084 instruction_, 1085 instruction_->GetDexPc(), 1086 this); 1087 CheckEntrypointTypes<kQuickReadBarrierForRootSlow, mirror::Object*, GcRoot<mirror::Object>*>(); 1088 arm_codegen->Move32(out_, Location::RegisterLocation(R0)); 1089 1090 RestoreLiveRegisters(codegen, locations); 1091 __ b(GetExitLabel()); 1092 } 1093 1094 const char* GetDescription() const OVERRIDE { return "ReadBarrierForRootSlowPathARM"; } 1095 1096 private: 1097 const Location out_; 1098 const Location root_; 1099 1100 DISALLOW_COPY_AND_ASSIGN(ReadBarrierForRootSlowPathARM); 1101}; 1102 1103#undef __ 1104// NOLINT on __ macro to suppress wrong warning/fix (misc-macro-parentheses) from clang-tidy. 1105#define __ down_cast<ArmAssembler*>(GetAssembler())-> // NOLINT 1106 1107inline Condition ARMCondition(IfCondition cond) { 1108 switch (cond) { 1109 case kCondEQ: return EQ; 1110 case kCondNE: return NE; 1111 case kCondLT: return LT; 1112 case kCondLE: return LE; 1113 case kCondGT: return GT; 1114 case kCondGE: return GE; 1115 case kCondB: return LO; 1116 case kCondBE: return LS; 1117 case kCondA: return HI; 1118 case kCondAE: return HS; 1119 } 1120 LOG(FATAL) << "Unreachable"; 1121 UNREACHABLE(); 1122} 1123 1124// Maps signed condition to unsigned condition. 1125inline Condition ARMUnsignedCondition(IfCondition cond) { 1126 switch (cond) { 1127 case kCondEQ: return EQ; 1128 case kCondNE: return NE; 1129 // Signed to unsigned. 1130 case kCondLT: return LO; 1131 case kCondLE: return LS; 1132 case kCondGT: return HI; 1133 case kCondGE: return HS; 1134 // Unsigned remain unchanged. 1135 case kCondB: return LO; 1136 case kCondBE: return LS; 1137 case kCondA: return HI; 1138 case kCondAE: return HS; 1139 } 1140 LOG(FATAL) << "Unreachable"; 1141 UNREACHABLE(); 1142} 1143 1144inline Condition ARMFPCondition(IfCondition cond, bool gt_bias) { 1145 // The ARM condition codes can express all the necessary branches, see the 1146 // "Meaning (floating-point)" column in the table A8-1 of the ARMv7 reference manual. 1147 // There is no dex instruction or HIR that would need the missing conditions 1148 // "equal or unordered" or "not equal". 1149 switch (cond) { 1150 case kCondEQ: return EQ; 1151 case kCondNE: return NE /* unordered */; 1152 case kCondLT: return gt_bias ? CC : LT /* unordered */; 1153 case kCondLE: return gt_bias ? LS : LE /* unordered */; 1154 case kCondGT: return gt_bias ? HI /* unordered */ : GT; 1155 case kCondGE: return gt_bias ? CS /* unordered */ : GE; 1156 default: 1157 LOG(FATAL) << "UNREACHABLE"; 1158 UNREACHABLE(); 1159 } 1160} 1161 1162void CodeGeneratorARM::DumpCoreRegister(std::ostream& stream, int reg) const { 1163 stream << Register(reg); 1164} 1165 1166void CodeGeneratorARM::DumpFloatingPointRegister(std::ostream& stream, int reg) const { 1167 stream << SRegister(reg); 1168} 1169 1170size_t CodeGeneratorARM::SaveCoreRegister(size_t stack_index, uint32_t reg_id) { 1171 __ StoreToOffset(kStoreWord, static_cast<Register>(reg_id), SP, stack_index); 1172 return kArmWordSize; 1173} 1174 1175size_t CodeGeneratorARM::RestoreCoreRegister(size_t stack_index, uint32_t reg_id) { 1176 __ LoadFromOffset(kLoadWord, static_cast<Register>(reg_id), SP, stack_index); 1177 return kArmWordSize; 1178} 1179 1180size_t CodeGeneratorARM::SaveFloatingPointRegister(size_t stack_index, uint32_t reg_id) { 1181 __ StoreSToOffset(static_cast<SRegister>(reg_id), SP, stack_index); 1182 return kArmWordSize; 1183} 1184 1185size_t CodeGeneratorARM::RestoreFloatingPointRegister(size_t stack_index, uint32_t reg_id) { 1186 __ LoadSFromOffset(static_cast<SRegister>(reg_id), SP, stack_index); 1187 return kArmWordSize; 1188} 1189 1190CodeGeneratorARM::CodeGeneratorARM(HGraph* graph, 1191 const ArmInstructionSetFeatures& isa_features, 1192 const CompilerOptions& compiler_options, 1193 OptimizingCompilerStats* stats) 1194 : CodeGenerator(graph, 1195 kNumberOfCoreRegisters, 1196 kNumberOfSRegisters, 1197 kNumberOfRegisterPairs, 1198 ComputeRegisterMask(reinterpret_cast<const int*>(kCoreCalleeSaves), 1199 arraysize(kCoreCalleeSaves)), 1200 ComputeRegisterMask(reinterpret_cast<const int*>(kFpuCalleeSaves), 1201 arraysize(kFpuCalleeSaves)), 1202 compiler_options, 1203 stats), 1204 block_labels_(nullptr), 1205 location_builder_(graph, this), 1206 instruction_visitor_(graph, this), 1207 move_resolver_(graph->GetArena(), this), 1208 assembler_(graph->GetArena()), 1209 isa_features_(isa_features), 1210 uint32_literals_(std::less<uint32_t>(), 1211 graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), 1212 method_patches_(MethodReferenceComparator(), 1213 graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), 1214 call_patches_(MethodReferenceComparator(), 1215 graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), 1216 relative_call_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), 1217 pc_relative_dex_cache_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), 1218 boot_image_string_patches_(StringReferenceValueComparator(), 1219 graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), 1220 pc_relative_string_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), 1221 boot_image_type_patches_(TypeReferenceValueComparator(), 1222 graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), 1223 pc_relative_type_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), 1224 boot_image_address_patches_(std::less<uint32_t>(), 1225 graph->GetArena()->Adapter(kArenaAllocCodeGenerator)) { 1226 // Always save the LR register to mimic Quick. 1227 AddAllocatedRegister(Location::RegisterLocation(LR)); 1228} 1229 1230void CodeGeneratorARM::Finalize(CodeAllocator* allocator) { 1231 // Ensure that we fix up branches and literal loads and emit the literal pool. 1232 __ FinalizeCode(); 1233 1234 // Adjust native pc offsets in stack maps. 1235 for (size_t i = 0, num = stack_map_stream_.GetNumberOfStackMaps(); i != num; ++i) { 1236 uint32_t old_position = stack_map_stream_.GetStackMap(i).native_pc_offset; 1237 uint32_t new_position = __ GetAdjustedPosition(old_position); 1238 stack_map_stream_.SetStackMapNativePcOffset(i, new_position); 1239 } 1240 // Adjust pc offsets for the disassembly information. 1241 if (disasm_info_ != nullptr) { 1242 GeneratedCodeInterval* frame_entry_interval = disasm_info_->GetFrameEntryInterval(); 1243 frame_entry_interval->start = __ GetAdjustedPosition(frame_entry_interval->start); 1244 frame_entry_interval->end = __ GetAdjustedPosition(frame_entry_interval->end); 1245 for (auto& it : *disasm_info_->GetInstructionIntervals()) { 1246 it.second.start = __ GetAdjustedPosition(it.second.start); 1247 it.second.end = __ GetAdjustedPosition(it.second.end); 1248 } 1249 for (auto& it : *disasm_info_->GetSlowPathIntervals()) { 1250 it.code_interval.start = __ GetAdjustedPosition(it.code_interval.start); 1251 it.code_interval.end = __ GetAdjustedPosition(it.code_interval.end); 1252 } 1253 } 1254 1255 CodeGenerator::Finalize(allocator); 1256} 1257 1258void CodeGeneratorARM::SetupBlockedRegisters() const { 1259 // Stack register, LR and PC are always reserved. 1260 blocked_core_registers_[SP] = true; 1261 blocked_core_registers_[LR] = true; 1262 blocked_core_registers_[PC] = true; 1263 1264 // Reserve thread register. 1265 blocked_core_registers_[TR] = true; 1266 1267 // Reserve temp register. 1268 blocked_core_registers_[IP] = true; 1269 1270 if (GetGraph()->IsDebuggable()) { 1271 // Stubs do not save callee-save floating point registers. If the graph 1272 // is debuggable, we need to deal with these registers differently. For 1273 // now, just block them. 1274 for (size_t i = 0; i < arraysize(kFpuCalleeSaves); ++i) { 1275 blocked_fpu_registers_[kFpuCalleeSaves[i]] = true; 1276 } 1277 } 1278} 1279 1280InstructionCodeGeneratorARM::InstructionCodeGeneratorARM(HGraph* graph, CodeGeneratorARM* codegen) 1281 : InstructionCodeGenerator(graph, codegen), 1282 assembler_(codegen->GetAssembler()), 1283 codegen_(codegen) {} 1284 1285void CodeGeneratorARM::ComputeSpillMask() { 1286 core_spill_mask_ = allocated_registers_.GetCoreRegisters() & core_callee_save_mask_; 1287 DCHECK_NE(core_spill_mask_, 0u) << "At least the return address register must be saved"; 1288 // There is no easy instruction to restore just the PC on thumb2. We spill and 1289 // restore another arbitrary register. 1290 core_spill_mask_ |= (1 << kCoreAlwaysSpillRegister); 1291 fpu_spill_mask_ = allocated_registers_.GetFloatingPointRegisters() & fpu_callee_save_mask_; 1292 // We use vpush and vpop for saving and restoring floating point registers, which take 1293 // a SRegister and the number of registers to save/restore after that SRegister. We 1294 // therefore update the `fpu_spill_mask_` to also contain those registers not allocated, 1295 // but in the range. 1296 if (fpu_spill_mask_ != 0) { 1297 uint32_t least_significant_bit = LeastSignificantBit(fpu_spill_mask_); 1298 uint32_t most_significant_bit = MostSignificantBit(fpu_spill_mask_); 1299 for (uint32_t i = least_significant_bit + 1 ; i < most_significant_bit; ++i) { 1300 fpu_spill_mask_ |= (1 << i); 1301 } 1302 } 1303} 1304 1305static dwarf::Reg DWARFReg(Register reg) { 1306 return dwarf::Reg::ArmCore(static_cast<int>(reg)); 1307} 1308 1309static dwarf::Reg DWARFReg(SRegister reg) { 1310 return dwarf::Reg::ArmFp(static_cast<int>(reg)); 1311} 1312 1313void CodeGeneratorARM::GenerateFrameEntry() { 1314 bool skip_overflow_check = 1315 IsLeafMethod() && !FrameNeedsStackCheck(GetFrameSize(), InstructionSet::kArm); 1316 DCHECK(GetCompilerOptions().GetImplicitStackOverflowChecks()); 1317 __ Bind(&frame_entry_label_); 1318 1319 if (HasEmptyFrame()) { 1320 return; 1321 } 1322 1323 if (!skip_overflow_check) { 1324 __ AddConstant(IP, SP, -static_cast<int32_t>(GetStackOverflowReservedBytes(kArm))); 1325 __ LoadFromOffset(kLoadWord, IP, IP, 0); 1326 RecordPcInfo(nullptr, 0); 1327 } 1328 1329 __ PushList(core_spill_mask_); 1330 __ cfi().AdjustCFAOffset(kArmWordSize * POPCOUNT(core_spill_mask_)); 1331 __ cfi().RelOffsetForMany(DWARFReg(kMethodRegisterArgument), 0, core_spill_mask_, kArmWordSize); 1332 if (fpu_spill_mask_ != 0) { 1333 SRegister start_register = SRegister(LeastSignificantBit(fpu_spill_mask_)); 1334 __ vpushs(start_register, POPCOUNT(fpu_spill_mask_)); 1335 __ cfi().AdjustCFAOffset(kArmWordSize * POPCOUNT(fpu_spill_mask_)); 1336 __ cfi().RelOffsetForMany(DWARFReg(S0), 0, fpu_spill_mask_, kArmWordSize); 1337 } 1338 int adjust = GetFrameSize() - FrameEntrySpillSize(); 1339 __ AddConstant(SP, -adjust); 1340 __ cfi().AdjustCFAOffset(adjust); 1341 1342 // Save the current method if we need it. Note that we do not 1343 // do this in HCurrentMethod, as the instruction might have been removed 1344 // in the SSA graph. 1345 if (RequiresCurrentMethod()) { 1346 __ StoreToOffset(kStoreWord, kMethodRegisterArgument, SP, 0); 1347 } 1348} 1349 1350void CodeGeneratorARM::GenerateFrameExit() { 1351 if (HasEmptyFrame()) { 1352 __ bx(LR); 1353 return; 1354 } 1355 __ cfi().RememberState(); 1356 int adjust = GetFrameSize() - FrameEntrySpillSize(); 1357 __ AddConstant(SP, adjust); 1358 __ cfi().AdjustCFAOffset(-adjust); 1359 if (fpu_spill_mask_ != 0) { 1360 SRegister start_register = SRegister(LeastSignificantBit(fpu_spill_mask_)); 1361 __ vpops(start_register, POPCOUNT(fpu_spill_mask_)); 1362 __ cfi().AdjustCFAOffset(-static_cast<int>(kArmPointerSize) * POPCOUNT(fpu_spill_mask_)); 1363 __ cfi().RestoreMany(DWARFReg(SRegister(0)), fpu_spill_mask_); 1364 } 1365 // Pop LR into PC to return. 1366 DCHECK_NE(core_spill_mask_ & (1 << LR), 0U); 1367 uint32_t pop_mask = (core_spill_mask_ & (~(1 << LR))) | 1 << PC; 1368 __ PopList(pop_mask); 1369 __ cfi().RestoreState(); 1370 __ cfi().DefCFAOffset(GetFrameSize()); 1371} 1372 1373void CodeGeneratorARM::Bind(HBasicBlock* block) { 1374 Label* label = GetLabelOf(block); 1375 __ BindTrackedLabel(label); 1376} 1377 1378Location InvokeDexCallingConventionVisitorARM::GetNextLocation(Primitive::Type type) { 1379 switch (type) { 1380 case Primitive::kPrimBoolean: 1381 case Primitive::kPrimByte: 1382 case Primitive::kPrimChar: 1383 case Primitive::kPrimShort: 1384 case Primitive::kPrimInt: 1385 case Primitive::kPrimNot: { 1386 uint32_t index = gp_index_++; 1387 uint32_t stack_index = stack_index_++; 1388 if (index < calling_convention.GetNumberOfRegisters()) { 1389 return Location::RegisterLocation(calling_convention.GetRegisterAt(index)); 1390 } else { 1391 return Location::StackSlot(calling_convention.GetStackOffsetOf(stack_index)); 1392 } 1393 } 1394 1395 case Primitive::kPrimLong: { 1396 uint32_t index = gp_index_; 1397 uint32_t stack_index = stack_index_; 1398 gp_index_ += 2; 1399 stack_index_ += 2; 1400 if (index + 1 < calling_convention.GetNumberOfRegisters()) { 1401 if (calling_convention.GetRegisterAt(index) == R1) { 1402 // Skip R1, and use R2_R3 instead. 1403 gp_index_++; 1404 index++; 1405 } 1406 } 1407 if (index + 1 < calling_convention.GetNumberOfRegisters()) { 1408 DCHECK_EQ(calling_convention.GetRegisterAt(index) + 1, 1409 calling_convention.GetRegisterAt(index + 1)); 1410 1411 return Location::RegisterPairLocation(calling_convention.GetRegisterAt(index), 1412 calling_convention.GetRegisterAt(index + 1)); 1413 } else { 1414 return Location::DoubleStackSlot(calling_convention.GetStackOffsetOf(stack_index)); 1415 } 1416 } 1417 1418 case Primitive::kPrimFloat: { 1419 uint32_t stack_index = stack_index_++; 1420 if (float_index_ % 2 == 0) { 1421 float_index_ = std::max(double_index_, float_index_); 1422 } 1423 if (float_index_ < calling_convention.GetNumberOfFpuRegisters()) { 1424 return Location::FpuRegisterLocation(calling_convention.GetFpuRegisterAt(float_index_++)); 1425 } else { 1426 return Location::StackSlot(calling_convention.GetStackOffsetOf(stack_index)); 1427 } 1428 } 1429 1430 case Primitive::kPrimDouble: { 1431 double_index_ = std::max(double_index_, RoundUp(float_index_, 2)); 1432 uint32_t stack_index = stack_index_; 1433 stack_index_ += 2; 1434 if (double_index_ + 1 < calling_convention.GetNumberOfFpuRegisters()) { 1435 uint32_t index = double_index_; 1436 double_index_ += 2; 1437 Location result = Location::FpuRegisterPairLocation( 1438 calling_convention.GetFpuRegisterAt(index), 1439 calling_convention.GetFpuRegisterAt(index + 1)); 1440 DCHECK(ExpectedPairLayout(result)); 1441 return result; 1442 } else { 1443 return Location::DoubleStackSlot(calling_convention.GetStackOffsetOf(stack_index)); 1444 } 1445 } 1446 1447 case Primitive::kPrimVoid: 1448 LOG(FATAL) << "Unexpected parameter type " << type; 1449 break; 1450 } 1451 return Location::NoLocation(); 1452} 1453 1454Location InvokeDexCallingConventionVisitorARM::GetReturnLocation(Primitive::Type type) const { 1455 switch (type) { 1456 case Primitive::kPrimBoolean: 1457 case Primitive::kPrimByte: 1458 case Primitive::kPrimChar: 1459 case Primitive::kPrimShort: 1460 case Primitive::kPrimInt: 1461 case Primitive::kPrimNot: { 1462 return Location::RegisterLocation(R0); 1463 } 1464 1465 case Primitive::kPrimFloat: { 1466 return Location::FpuRegisterLocation(S0); 1467 } 1468 1469 case Primitive::kPrimLong: { 1470 return Location::RegisterPairLocation(R0, R1); 1471 } 1472 1473 case Primitive::kPrimDouble: { 1474 return Location::FpuRegisterPairLocation(S0, S1); 1475 } 1476 1477 case Primitive::kPrimVoid: 1478 return Location::NoLocation(); 1479 } 1480 1481 UNREACHABLE(); 1482} 1483 1484Location InvokeDexCallingConventionVisitorARM::GetMethodLocation() const { 1485 return Location::RegisterLocation(kMethodRegisterArgument); 1486} 1487 1488void CodeGeneratorARM::Move32(Location destination, Location source) { 1489 if (source.Equals(destination)) { 1490 return; 1491 } 1492 if (destination.IsRegister()) { 1493 if (source.IsRegister()) { 1494 __ Mov(destination.AsRegister<Register>(), source.AsRegister<Register>()); 1495 } else if (source.IsFpuRegister()) { 1496 __ vmovrs(destination.AsRegister<Register>(), source.AsFpuRegister<SRegister>()); 1497 } else { 1498 __ LoadFromOffset(kLoadWord, destination.AsRegister<Register>(), SP, source.GetStackIndex()); 1499 } 1500 } else if (destination.IsFpuRegister()) { 1501 if (source.IsRegister()) { 1502 __ vmovsr(destination.AsFpuRegister<SRegister>(), source.AsRegister<Register>()); 1503 } else if (source.IsFpuRegister()) { 1504 __ vmovs(destination.AsFpuRegister<SRegister>(), source.AsFpuRegister<SRegister>()); 1505 } else { 1506 __ LoadSFromOffset(destination.AsFpuRegister<SRegister>(), SP, source.GetStackIndex()); 1507 } 1508 } else { 1509 DCHECK(destination.IsStackSlot()) << destination; 1510 if (source.IsRegister()) { 1511 __ StoreToOffset(kStoreWord, source.AsRegister<Register>(), SP, destination.GetStackIndex()); 1512 } else if (source.IsFpuRegister()) { 1513 __ StoreSToOffset(source.AsFpuRegister<SRegister>(), SP, destination.GetStackIndex()); 1514 } else { 1515 DCHECK(source.IsStackSlot()) << source; 1516 __ LoadFromOffset(kLoadWord, IP, SP, source.GetStackIndex()); 1517 __ StoreToOffset(kStoreWord, IP, SP, destination.GetStackIndex()); 1518 } 1519 } 1520} 1521 1522void CodeGeneratorARM::Move64(Location destination, Location source) { 1523 if (source.Equals(destination)) { 1524 return; 1525 } 1526 if (destination.IsRegisterPair()) { 1527 if (source.IsRegisterPair()) { 1528 EmitParallelMoves( 1529 Location::RegisterLocation(source.AsRegisterPairHigh<Register>()), 1530 Location::RegisterLocation(destination.AsRegisterPairHigh<Register>()), 1531 Primitive::kPrimInt, 1532 Location::RegisterLocation(source.AsRegisterPairLow<Register>()), 1533 Location::RegisterLocation(destination.AsRegisterPairLow<Register>()), 1534 Primitive::kPrimInt); 1535 } else if (source.IsFpuRegister()) { 1536 UNIMPLEMENTED(FATAL); 1537 } else if (source.IsFpuRegisterPair()) { 1538 __ vmovrrd(destination.AsRegisterPairLow<Register>(), 1539 destination.AsRegisterPairHigh<Register>(), 1540 FromLowSToD(source.AsFpuRegisterPairLow<SRegister>())); 1541 } else { 1542 DCHECK(source.IsDoubleStackSlot()); 1543 DCHECK(ExpectedPairLayout(destination)); 1544 __ LoadFromOffset(kLoadWordPair, destination.AsRegisterPairLow<Register>(), 1545 SP, source.GetStackIndex()); 1546 } 1547 } else if (destination.IsFpuRegisterPair()) { 1548 if (source.IsDoubleStackSlot()) { 1549 __ LoadDFromOffset(FromLowSToD(destination.AsFpuRegisterPairLow<SRegister>()), 1550 SP, 1551 source.GetStackIndex()); 1552 } else if (source.IsRegisterPair()) { 1553 __ vmovdrr(FromLowSToD(destination.AsFpuRegisterPairLow<SRegister>()), 1554 source.AsRegisterPairLow<Register>(), 1555 source.AsRegisterPairHigh<Register>()); 1556 } else { 1557 UNIMPLEMENTED(FATAL); 1558 } 1559 } else { 1560 DCHECK(destination.IsDoubleStackSlot()); 1561 if (source.IsRegisterPair()) { 1562 // No conflict possible, so just do the moves. 1563 if (source.AsRegisterPairLow<Register>() == R1) { 1564 DCHECK_EQ(source.AsRegisterPairHigh<Register>(), R2); 1565 __ StoreToOffset(kStoreWord, R1, SP, destination.GetStackIndex()); 1566 __ StoreToOffset(kStoreWord, R2, SP, destination.GetHighStackIndex(kArmWordSize)); 1567 } else { 1568 __ StoreToOffset(kStoreWordPair, source.AsRegisterPairLow<Register>(), 1569 SP, destination.GetStackIndex()); 1570 } 1571 } else if (source.IsFpuRegisterPair()) { 1572 __ StoreDToOffset(FromLowSToD(source.AsFpuRegisterPairLow<SRegister>()), 1573 SP, 1574 destination.GetStackIndex()); 1575 } else { 1576 DCHECK(source.IsDoubleStackSlot()); 1577 EmitParallelMoves( 1578 Location::StackSlot(source.GetStackIndex()), 1579 Location::StackSlot(destination.GetStackIndex()), 1580 Primitive::kPrimInt, 1581 Location::StackSlot(source.GetHighStackIndex(kArmWordSize)), 1582 Location::StackSlot(destination.GetHighStackIndex(kArmWordSize)), 1583 Primitive::kPrimInt); 1584 } 1585 } 1586} 1587 1588void CodeGeneratorARM::MoveConstant(Location location, int32_t value) { 1589 DCHECK(location.IsRegister()); 1590 __ LoadImmediate(location.AsRegister<Register>(), value); 1591} 1592 1593void CodeGeneratorARM::MoveLocation(Location dst, Location src, Primitive::Type dst_type) { 1594 HParallelMove move(GetGraph()->GetArena()); 1595 move.AddMove(src, dst, dst_type, nullptr); 1596 GetMoveResolver()->EmitNativeCode(&move); 1597} 1598 1599void CodeGeneratorARM::AddLocationAsTemp(Location location, LocationSummary* locations) { 1600 if (location.IsRegister()) { 1601 locations->AddTemp(location); 1602 } else if (location.IsRegisterPair()) { 1603 locations->AddTemp(Location::RegisterLocation(location.AsRegisterPairLow<Register>())); 1604 locations->AddTemp(Location::RegisterLocation(location.AsRegisterPairHigh<Register>())); 1605 } else { 1606 UNIMPLEMENTED(FATAL) << "AddLocationAsTemp not implemented for location " << location; 1607 } 1608} 1609 1610void CodeGeneratorARM::InvokeRuntime(QuickEntrypointEnum entrypoint, 1611 HInstruction* instruction, 1612 uint32_t dex_pc, 1613 SlowPathCode* slow_path) { 1614 ValidateInvokeRuntime(entrypoint, instruction, slow_path); 1615 GenerateInvokeRuntime(GetThreadOffset<kArmPointerSize>(entrypoint).Int32Value()); 1616 if (EntrypointRequiresStackMap(entrypoint)) { 1617 RecordPcInfo(instruction, dex_pc, slow_path); 1618 } 1619} 1620 1621void CodeGeneratorARM::InvokeRuntimeWithoutRecordingPcInfo(int32_t entry_point_offset, 1622 HInstruction* instruction, 1623 SlowPathCode* slow_path) { 1624 ValidateInvokeRuntimeWithoutRecordingPcInfo(instruction, slow_path); 1625 GenerateInvokeRuntime(entry_point_offset); 1626} 1627 1628void CodeGeneratorARM::GenerateInvokeRuntime(int32_t entry_point_offset) { 1629 __ LoadFromOffset(kLoadWord, LR, TR, entry_point_offset); 1630 __ blx(LR); 1631} 1632 1633void InstructionCodeGeneratorARM::HandleGoto(HInstruction* got, HBasicBlock* successor) { 1634 DCHECK(!successor->IsExitBlock()); 1635 1636 HBasicBlock* block = got->GetBlock(); 1637 HInstruction* previous = got->GetPrevious(); 1638 1639 HLoopInformation* info = block->GetLoopInformation(); 1640 if (info != nullptr && info->IsBackEdge(*block) && info->HasSuspendCheck()) { 1641 codegen_->ClearSpillSlotsFromLoopPhisInStackMap(info->GetSuspendCheck()); 1642 GenerateSuspendCheck(info->GetSuspendCheck(), successor); 1643 return; 1644 } 1645 1646 if (block->IsEntryBlock() && (previous != nullptr) && previous->IsSuspendCheck()) { 1647 GenerateSuspendCheck(previous->AsSuspendCheck(), nullptr); 1648 } 1649 if (!codegen_->GoesToNextBlock(got->GetBlock(), successor)) { 1650 __ b(codegen_->GetLabelOf(successor)); 1651 } 1652} 1653 1654void LocationsBuilderARM::VisitGoto(HGoto* got) { 1655 got->SetLocations(nullptr); 1656} 1657 1658void InstructionCodeGeneratorARM::VisitGoto(HGoto* got) { 1659 HandleGoto(got, got->GetSuccessor()); 1660} 1661 1662void LocationsBuilderARM::VisitTryBoundary(HTryBoundary* try_boundary) { 1663 try_boundary->SetLocations(nullptr); 1664} 1665 1666void InstructionCodeGeneratorARM::VisitTryBoundary(HTryBoundary* try_boundary) { 1667 HBasicBlock* successor = try_boundary->GetNormalFlowSuccessor(); 1668 if (!successor->IsExitBlock()) { 1669 HandleGoto(try_boundary, successor); 1670 } 1671} 1672 1673void LocationsBuilderARM::VisitExit(HExit* exit) { 1674 exit->SetLocations(nullptr); 1675} 1676 1677void InstructionCodeGeneratorARM::VisitExit(HExit* exit ATTRIBUTE_UNUSED) { 1678} 1679 1680void InstructionCodeGeneratorARM::GenerateVcmp(HInstruction* instruction) { 1681 Primitive::Type type = instruction->InputAt(0)->GetType(); 1682 Location lhs_loc = instruction->GetLocations()->InAt(0); 1683 Location rhs_loc = instruction->GetLocations()->InAt(1); 1684 if (rhs_loc.IsConstant()) { 1685 // 0.0 is the only immediate that can be encoded directly in 1686 // a VCMP instruction. 1687 // 1688 // Both the JLS (section 15.20.1) and the JVMS (section 6.5) 1689 // specify that in a floating-point comparison, positive zero 1690 // and negative zero are considered equal, so we can use the 1691 // literal 0.0 for both cases here. 1692 // 1693 // Note however that some methods (Float.equal, Float.compare, 1694 // Float.compareTo, Double.equal, Double.compare, 1695 // Double.compareTo, Math.max, Math.min, StrictMath.max, 1696 // StrictMath.min) consider 0.0 to be (strictly) greater than 1697 // -0.0. So if we ever translate calls to these methods into a 1698 // HCompare instruction, we must handle the -0.0 case with 1699 // care here. 1700 DCHECK(rhs_loc.GetConstant()->IsArithmeticZero()); 1701 if (type == Primitive::kPrimFloat) { 1702 __ vcmpsz(lhs_loc.AsFpuRegister<SRegister>()); 1703 } else { 1704 DCHECK_EQ(type, Primitive::kPrimDouble); 1705 __ vcmpdz(FromLowSToD(lhs_loc.AsFpuRegisterPairLow<SRegister>())); 1706 } 1707 } else { 1708 if (type == Primitive::kPrimFloat) { 1709 __ vcmps(lhs_loc.AsFpuRegister<SRegister>(), rhs_loc.AsFpuRegister<SRegister>()); 1710 } else { 1711 DCHECK_EQ(type, Primitive::kPrimDouble); 1712 __ vcmpd(FromLowSToD(lhs_loc.AsFpuRegisterPairLow<SRegister>()), 1713 FromLowSToD(rhs_loc.AsFpuRegisterPairLow<SRegister>())); 1714 } 1715 } 1716} 1717 1718void InstructionCodeGeneratorARM::GenerateFPJumps(HCondition* cond, 1719 Label* true_label, 1720 Label* false_label ATTRIBUTE_UNUSED) { 1721 __ vmstat(); // transfer FP status register to ARM APSR. 1722 __ b(true_label, ARMFPCondition(cond->GetCondition(), cond->IsGtBias())); 1723} 1724 1725void InstructionCodeGeneratorARM::GenerateLongComparesAndJumps(HCondition* cond, 1726 Label* true_label, 1727 Label* false_label) { 1728 LocationSummary* locations = cond->GetLocations(); 1729 Location left = locations->InAt(0); 1730 Location right = locations->InAt(1); 1731 IfCondition if_cond = cond->GetCondition(); 1732 1733 Register left_high = left.AsRegisterPairHigh<Register>(); 1734 Register left_low = left.AsRegisterPairLow<Register>(); 1735 IfCondition true_high_cond = if_cond; 1736 IfCondition false_high_cond = cond->GetOppositeCondition(); 1737 Condition final_condition = ARMUnsignedCondition(if_cond); // unsigned on lower part 1738 1739 // Set the conditions for the test, remembering that == needs to be 1740 // decided using the low words. 1741 // TODO: consider avoiding jumps with temporary and CMP low+SBC high 1742 switch (if_cond) { 1743 case kCondEQ: 1744 case kCondNE: 1745 // Nothing to do. 1746 break; 1747 case kCondLT: 1748 false_high_cond = kCondGT; 1749 break; 1750 case kCondLE: 1751 true_high_cond = kCondLT; 1752 break; 1753 case kCondGT: 1754 false_high_cond = kCondLT; 1755 break; 1756 case kCondGE: 1757 true_high_cond = kCondGT; 1758 break; 1759 case kCondB: 1760 false_high_cond = kCondA; 1761 break; 1762 case kCondBE: 1763 true_high_cond = kCondB; 1764 break; 1765 case kCondA: 1766 false_high_cond = kCondB; 1767 break; 1768 case kCondAE: 1769 true_high_cond = kCondA; 1770 break; 1771 } 1772 if (right.IsConstant()) { 1773 int64_t value = right.GetConstant()->AsLongConstant()->GetValue(); 1774 int32_t val_low = Low32Bits(value); 1775 int32_t val_high = High32Bits(value); 1776 1777 __ CmpConstant(left_high, val_high); 1778 if (if_cond == kCondNE) { 1779 __ b(true_label, ARMCondition(true_high_cond)); 1780 } else if (if_cond == kCondEQ) { 1781 __ b(false_label, ARMCondition(false_high_cond)); 1782 } else { 1783 __ b(true_label, ARMCondition(true_high_cond)); 1784 __ b(false_label, ARMCondition(false_high_cond)); 1785 } 1786 // Must be equal high, so compare the lows. 1787 __ CmpConstant(left_low, val_low); 1788 } else { 1789 Register right_high = right.AsRegisterPairHigh<Register>(); 1790 Register right_low = right.AsRegisterPairLow<Register>(); 1791 1792 __ cmp(left_high, ShifterOperand(right_high)); 1793 if (if_cond == kCondNE) { 1794 __ b(true_label, ARMCondition(true_high_cond)); 1795 } else if (if_cond == kCondEQ) { 1796 __ b(false_label, ARMCondition(false_high_cond)); 1797 } else { 1798 __ b(true_label, ARMCondition(true_high_cond)); 1799 __ b(false_label, ARMCondition(false_high_cond)); 1800 } 1801 // Must be equal high, so compare the lows. 1802 __ cmp(left_low, ShifterOperand(right_low)); 1803 } 1804 // The last comparison might be unsigned. 1805 // TODO: optimize cases where this is always true/false 1806 __ b(true_label, final_condition); 1807} 1808 1809void InstructionCodeGeneratorARM::GenerateCompareTestAndBranch(HCondition* condition, 1810 Label* true_target_in, 1811 Label* false_target_in) { 1812 // Generated branching requires both targets to be explicit. If either of the 1813 // targets is nullptr (fallthrough) use and bind `fallthrough_target` instead. 1814 Label fallthrough_target; 1815 Label* true_target = true_target_in == nullptr ? &fallthrough_target : true_target_in; 1816 Label* false_target = false_target_in == nullptr ? &fallthrough_target : false_target_in; 1817 1818 Primitive::Type type = condition->InputAt(0)->GetType(); 1819 switch (type) { 1820 case Primitive::kPrimLong: 1821 GenerateLongComparesAndJumps(condition, true_target, false_target); 1822 break; 1823 case Primitive::kPrimFloat: 1824 case Primitive::kPrimDouble: 1825 GenerateVcmp(condition); 1826 GenerateFPJumps(condition, true_target, false_target); 1827 break; 1828 default: 1829 LOG(FATAL) << "Unexpected compare type " << type; 1830 } 1831 1832 if (false_target != &fallthrough_target) { 1833 __ b(false_target); 1834 } 1835 1836 if (fallthrough_target.IsLinked()) { 1837 __ Bind(&fallthrough_target); 1838 } 1839} 1840 1841void InstructionCodeGeneratorARM::GenerateTestAndBranch(HInstruction* instruction, 1842 size_t condition_input_index, 1843 Label* true_target, 1844 Label* false_target) { 1845 HInstruction* cond = instruction->InputAt(condition_input_index); 1846 1847 if (true_target == nullptr && false_target == nullptr) { 1848 // Nothing to do. The code always falls through. 1849 return; 1850 } else if (cond->IsIntConstant()) { 1851 // Constant condition, statically compared against "true" (integer value 1). 1852 if (cond->AsIntConstant()->IsTrue()) { 1853 if (true_target != nullptr) { 1854 __ b(true_target); 1855 } 1856 } else { 1857 DCHECK(cond->AsIntConstant()->IsFalse()) << cond->AsIntConstant()->GetValue(); 1858 if (false_target != nullptr) { 1859 __ b(false_target); 1860 } 1861 } 1862 return; 1863 } 1864 1865 // The following code generates these patterns: 1866 // (1) true_target == nullptr && false_target != nullptr 1867 // - opposite condition true => branch to false_target 1868 // (2) true_target != nullptr && false_target == nullptr 1869 // - condition true => branch to true_target 1870 // (3) true_target != nullptr && false_target != nullptr 1871 // - condition true => branch to true_target 1872 // - branch to false_target 1873 if (IsBooleanValueOrMaterializedCondition(cond)) { 1874 // Condition has been materialized, compare the output to 0. 1875 Location cond_val = instruction->GetLocations()->InAt(condition_input_index); 1876 DCHECK(cond_val.IsRegister()); 1877 if (true_target == nullptr) { 1878 __ CompareAndBranchIfZero(cond_val.AsRegister<Register>(), false_target); 1879 } else { 1880 __ CompareAndBranchIfNonZero(cond_val.AsRegister<Register>(), true_target); 1881 } 1882 } else { 1883 // Condition has not been materialized. Use its inputs as the comparison and 1884 // its condition as the branch condition. 1885 HCondition* condition = cond->AsCondition(); 1886 1887 // If this is a long or FP comparison that has been folded into 1888 // the HCondition, generate the comparison directly. 1889 Primitive::Type type = condition->InputAt(0)->GetType(); 1890 if (type == Primitive::kPrimLong || Primitive::IsFloatingPointType(type)) { 1891 GenerateCompareTestAndBranch(condition, true_target, false_target); 1892 return; 1893 } 1894 1895 LocationSummary* locations = cond->GetLocations(); 1896 DCHECK(locations->InAt(0).IsRegister()); 1897 Register left = locations->InAt(0).AsRegister<Register>(); 1898 Location right = locations->InAt(1); 1899 if (right.IsRegister()) { 1900 __ cmp(left, ShifterOperand(right.AsRegister<Register>())); 1901 } else { 1902 DCHECK(right.IsConstant()); 1903 __ CmpConstant(left, CodeGenerator::GetInt32ValueOf(right.GetConstant())); 1904 } 1905 if (true_target == nullptr) { 1906 __ b(false_target, ARMCondition(condition->GetOppositeCondition())); 1907 } else { 1908 __ b(true_target, ARMCondition(condition->GetCondition())); 1909 } 1910 } 1911 1912 // If neither branch falls through (case 3), the conditional branch to `true_target` 1913 // was already emitted (case 2) and we need to emit a jump to `false_target`. 1914 if (true_target != nullptr && false_target != nullptr) { 1915 __ b(false_target); 1916 } 1917} 1918 1919void LocationsBuilderARM::VisitIf(HIf* if_instr) { 1920 LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(if_instr); 1921 if (IsBooleanValueOrMaterializedCondition(if_instr->InputAt(0))) { 1922 locations->SetInAt(0, Location::RequiresRegister()); 1923 } 1924} 1925 1926void InstructionCodeGeneratorARM::VisitIf(HIf* if_instr) { 1927 HBasicBlock* true_successor = if_instr->IfTrueSuccessor(); 1928 HBasicBlock* false_successor = if_instr->IfFalseSuccessor(); 1929 Label* true_target = codegen_->GoesToNextBlock(if_instr->GetBlock(), true_successor) ? 1930 nullptr : codegen_->GetLabelOf(true_successor); 1931 Label* false_target = codegen_->GoesToNextBlock(if_instr->GetBlock(), false_successor) ? 1932 nullptr : codegen_->GetLabelOf(false_successor); 1933 GenerateTestAndBranch(if_instr, /* condition_input_index */ 0, true_target, false_target); 1934} 1935 1936void LocationsBuilderARM::VisitDeoptimize(HDeoptimize* deoptimize) { 1937 LocationSummary* locations = new (GetGraph()->GetArena()) 1938 LocationSummary(deoptimize, LocationSummary::kCallOnSlowPath); 1939 locations->SetCustomSlowPathCallerSaves(RegisterSet::Empty()); // No caller-save registers. 1940 if (IsBooleanValueOrMaterializedCondition(deoptimize->InputAt(0))) { 1941 locations->SetInAt(0, Location::RequiresRegister()); 1942 } 1943} 1944 1945void InstructionCodeGeneratorARM::VisitDeoptimize(HDeoptimize* deoptimize) { 1946 SlowPathCodeARM* slow_path = deopt_slow_paths_.NewSlowPath<DeoptimizationSlowPathARM>(deoptimize); 1947 GenerateTestAndBranch(deoptimize, 1948 /* condition_input_index */ 0, 1949 slow_path->GetEntryLabel(), 1950 /* false_target */ nullptr); 1951} 1952 1953void LocationsBuilderARM::VisitSelect(HSelect* select) { 1954 LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(select); 1955 if (Primitive::IsFloatingPointType(select->GetType())) { 1956 locations->SetInAt(0, Location::RequiresFpuRegister()); 1957 locations->SetInAt(1, Location::RequiresFpuRegister()); 1958 } else { 1959 locations->SetInAt(0, Location::RequiresRegister()); 1960 locations->SetInAt(1, Location::RequiresRegister()); 1961 } 1962 if (IsBooleanValueOrMaterializedCondition(select->GetCondition())) { 1963 locations->SetInAt(2, Location::RequiresRegister()); 1964 } 1965 locations->SetOut(Location::SameAsFirstInput()); 1966} 1967 1968void InstructionCodeGeneratorARM::VisitSelect(HSelect* select) { 1969 LocationSummary* locations = select->GetLocations(); 1970 Label false_target; 1971 GenerateTestAndBranch(select, 1972 /* condition_input_index */ 2, 1973 /* true_target */ nullptr, 1974 &false_target); 1975 codegen_->MoveLocation(locations->Out(), locations->InAt(1), select->GetType()); 1976 __ Bind(&false_target); 1977} 1978 1979void LocationsBuilderARM::VisitNativeDebugInfo(HNativeDebugInfo* info) { 1980 new (GetGraph()->GetArena()) LocationSummary(info); 1981} 1982 1983void InstructionCodeGeneratorARM::VisitNativeDebugInfo(HNativeDebugInfo*) { 1984 // MaybeRecordNativeDebugInfo is already called implicitly in CodeGenerator::Compile. 1985} 1986 1987void CodeGeneratorARM::GenerateNop() { 1988 __ nop(); 1989} 1990 1991void LocationsBuilderARM::HandleCondition(HCondition* cond) { 1992 LocationSummary* locations = 1993 new (GetGraph()->GetArena()) LocationSummary(cond, LocationSummary::kNoCall); 1994 // Handle the long/FP comparisons made in instruction simplification. 1995 switch (cond->InputAt(0)->GetType()) { 1996 case Primitive::kPrimLong: 1997 locations->SetInAt(0, Location::RequiresRegister()); 1998 locations->SetInAt(1, Location::RegisterOrConstant(cond->InputAt(1))); 1999 if (!cond->IsEmittedAtUseSite()) { 2000 locations->SetOut(Location::RequiresRegister(), Location::kOutputOverlap); 2001 } 2002 break; 2003 2004 case Primitive::kPrimFloat: 2005 case Primitive::kPrimDouble: 2006 locations->SetInAt(0, Location::RequiresFpuRegister()); 2007 locations->SetInAt(1, ArithmeticZeroOrFpuRegister(cond->InputAt(1))); 2008 if (!cond->IsEmittedAtUseSite()) { 2009 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 2010 } 2011 break; 2012 2013 default: 2014 locations->SetInAt(0, Location::RequiresRegister()); 2015 locations->SetInAt(1, Location::RegisterOrConstant(cond->InputAt(1))); 2016 if (!cond->IsEmittedAtUseSite()) { 2017 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 2018 } 2019 } 2020} 2021 2022void InstructionCodeGeneratorARM::HandleCondition(HCondition* cond) { 2023 if (cond->IsEmittedAtUseSite()) { 2024 return; 2025 } 2026 2027 LocationSummary* locations = cond->GetLocations(); 2028 Location left = locations->InAt(0); 2029 Location right = locations->InAt(1); 2030 Register out = locations->Out().AsRegister<Register>(); 2031 Label true_label, false_label; 2032 2033 switch (cond->InputAt(0)->GetType()) { 2034 default: { 2035 // Integer case. 2036 if (right.IsRegister()) { 2037 __ cmp(left.AsRegister<Register>(), ShifterOperand(right.AsRegister<Register>())); 2038 } else { 2039 DCHECK(right.IsConstant()); 2040 __ CmpConstant(left.AsRegister<Register>(), 2041 CodeGenerator::GetInt32ValueOf(right.GetConstant())); 2042 } 2043 __ it(ARMCondition(cond->GetCondition()), kItElse); 2044 __ mov(locations->Out().AsRegister<Register>(), ShifterOperand(1), 2045 ARMCondition(cond->GetCondition())); 2046 __ mov(locations->Out().AsRegister<Register>(), ShifterOperand(0), 2047 ARMCondition(cond->GetOppositeCondition())); 2048 return; 2049 } 2050 case Primitive::kPrimLong: 2051 GenerateLongComparesAndJumps(cond, &true_label, &false_label); 2052 break; 2053 case Primitive::kPrimFloat: 2054 case Primitive::kPrimDouble: 2055 GenerateVcmp(cond); 2056 GenerateFPJumps(cond, &true_label, &false_label); 2057 break; 2058 } 2059 2060 // Convert the jumps into the result. 2061 Label done_label; 2062 2063 // False case: result = 0. 2064 __ Bind(&false_label); 2065 __ LoadImmediate(out, 0); 2066 __ b(&done_label); 2067 2068 // True case: result = 1. 2069 __ Bind(&true_label); 2070 __ LoadImmediate(out, 1); 2071 __ Bind(&done_label); 2072} 2073 2074void LocationsBuilderARM::VisitEqual(HEqual* comp) { 2075 HandleCondition(comp); 2076} 2077 2078void InstructionCodeGeneratorARM::VisitEqual(HEqual* comp) { 2079 HandleCondition(comp); 2080} 2081 2082void LocationsBuilderARM::VisitNotEqual(HNotEqual* comp) { 2083 HandleCondition(comp); 2084} 2085 2086void InstructionCodeGeneratorARM::VisitNotEqual(HNotEqual* comp) { 2087 HandleCondition(comp); 2088} 2089 2090void LocationsBuilderARM::VisitLessThan(HLessThan* comp) { 2091 HandleCondition(comp); 2092} 2093 2094void InstructionCodeGeneratorARM::VisitLessThan(HLessThan* comp) { 2095 HandleCondition(comp); 2096} 2097 2098void LocationsBuilderARM::VisitLessThanOrEqual(HLessThanOrEqual* comp) { 2099 HandleCondition(comp); 2100} 2101 2102void InstructionCodeGeneratorARM::VisitLessThanOrEqual(HLessThanOrEqual* comp) { 2103 HandleCondition(comp); 2104} 2105 2106void LocationsBuilderARM::VisitGreaterThan(HGreaterThan* comp) { 2107 HandleCondition(comp); 2108} 2109 2110void InstructionCodeGeneratorARM::VisitGreaterThan(HGreaterThan* comp) { 2111 HandleCondition(comp); 2112} 2113 2114void LocationsBuilderARM::VisitGreaterThanOrEqual(HGreaterThanOrEqual* comp) { 2115 HandleCondition(comp); 2116} 2117 2118void InstructionCodeGeneratorARM::VisitGreaterThanOrEqual(HGreaterThanOrEqual* comp) { 2119 HandleCondition(comp); 2120} 2121 2122void LocationsBuilderARM::VisitBelow(HBelow* comp) { 2123 HandleCondition(comp); 2124} 2125 2126void InstructionCodeGeneratorARM::VisitBelow(HBelow* comp) { 2127 HandleCondition(comp); 2128} 2129 2130void LocationsBuilderARM::VisitBelowOrEqual(HBelowOrEqual* comp) { 2131 HandleCondition(comp); 2132} 2133 2134void InstructionCodeGeneratorARM::VisitBelowOrEqual(HBelowOrEqual* comp) { 2135 HandleCondition(comp); 2136} 2137 2138void LocationsBuilderARM::VisitAbove(HAbove* comp) { 2139 HandleCondition(comp); 2140} 2141 2142void InstructionCodeGeneratorARM::VisitAbove(HAbove* comp) { 2143 HandleCondition(comp); 2144} 2145 2146void LocationsBuilderARM::VisitAboveOrEqual(HAboveOrEqual* comp) { 2147 HandleCondition(comp); 2148} 2149 2150void InstructionCodeGeneratorARM::VisitAboveOrEqual(HAboveOrEqual* comp) { 2151 HandleCondition(comp); 2152} 2153 2154void LocationsBuilderARM::VisitIntConstant(HIntConstant* constant) { 2155 LocationSummary* locations = 2156 new (GetGraph()->GetArena()) LocationSummary(constant, LocationSummary::kNoCall); 2157 locations->SetOut(Location::ConstantLocation(constant)); 2158} 2159 2160void InstructionCodeGeneratorARM::VisitIntConstant(HIntConstant* constant ATTRIBUTE_UNUSED) { 2161 // Will be generated at use site. 2162} 2163 2164void LocationsBuilderARM::VisitNullConstant(HNullConstant* constant) { 2165 LocationSummary* locations = 2166 new (GetGraph()->GetArena()) LocationSummary(constant, LocationSummary::kNoCall); 2167 locations->SetOut(Location::ConstantLocation(constant)); 2168} 2169 2170void InstructionCodeGeneratorARM::VisitNullConstant(HNullConstant* constant ATTRIBUTE_UNUSED) { 2171 // Will be generated at use site. 2172} 2173 2174void LocationsBuilderARM::VisitLongConstant(HLongConstant* constant) { 2175 LocationSummary* locations = 2176 new (GetGraph()->GetArena()) LocationSummary(constant, LocationSummary::kNoCall); 2177 locations->SetOut(Location::ConstantLocation(constant)); 2178} 2179 2180void InstructionCodeGeneratorARM::VisitLongConstant(HLongConstant* constant ATTRIBUTE_UNUSED) { 2181 // Will be generated at use site. 2182} 2183 2184void LocationsBuilderARM::VisitFloatConstant(HFloatConstant* constant) { 2185 LocationSummary* locations = 2186 new (GetGraph()->GetArena()) LocationSummary(constant, LocationSummary::kNoCall); 2187 locations->SetOut(Location::ConstantLocation(constant)); 2188} 2189 2190void InstructionCodeGeneratorARM::VisitFloatConstant(HFloatConstant* constant ATTRIBUTE_UNUSED) { 2191 // Will be generated at use site. 2192} 2193 2194void LocationsBuilderARM::VisitDoubleConstant(HDoubleConstant* constant) { 2195 LocationSummary* locations = 2196 new (GetGraph()->GetArena()) LocationSummary(constant, LocationSummary::kNoCall); 2197 locations->SetOut(Location::ConstantLocation(constant)); 2198} 2199 2200void InstructionCodeGeneratorARM::VisitDoubleConstant(HDoubleConstant* constant ATTRIBUTE_UNUSED) { 2201 // Will be generated at use site. 2202} 2203 2204void LocationsBuilderARM::VisitMemoryBarrier(HMemoryBarrier* memory_barrier) { 2205 memory_barrier->SetLocations(nullptr); 2206} 2207 2208void InstructionCodeGeneratorARM::VisitMemoryBarrier(HMemoryBarrier* memory_barrier) { 2209 codegen_->GenerateMemoryBarrier(memory_barrier->GetBarrierKind()); 2210} 2211 2212void LocationsBuilderARM::VisitReturnVoid(HReturnVoid* ret) { 2213 ret->SetLocations(nullptr); 2214} 2215 2216void InstructionCodeGeneratorARM::VisitReturnVoid(HReturnVoid* ret ATTRIBUTE_UNUSED) { 2217 codegen_->GenerateFrameExit(); 2218} 2219 2220void LocationsBuilderARM::VisitReturn(HReturn* ret) { 2221 LocationSummary* locations = 2222 new (GetGraph()->GetArena()) LocationSummary(ret, LocationSummary::kNoCall); 2223 locations->SetInAt(0, parameter_visitor_.GetReturnLocation(ret->InputAt(0)->GetType())); 2224} 2225 2226void InstructionCodeGeneratorARM::VisitReturn(HReturn* ret ATTRIBUTE_UNUSED) { 2227 codegen_->GenerateFrameExit(); 2228} 2229 2230void LocationsBuilderARM::VisitInvokeUnresolved(HInvokeUnresolved* invoke) { 2231 // The trampoline uses the same calling convention as dex calling conventions, 2232 // except instead of loading arg0/r0 with the target Method*, arg0/r0 will contain 2233 // the method_idx. 2234 HandleInvoke(invoke); 2235} 2236 2237void InstructionCodeGeneratorARM::VisitInvokeUnresolved(HInvokeUnresolved* invoke) { 2238 codegen_->GenerateInvokeUnresolvedRuntimeCall(invoke); 2239} 2240 2241void LocationsBuilderARM::VisitInvokeStaticOrDirect(HInvokeStaticOrDirect* invoke) { 2242 // Explicit clinit checks triggered by static invokes must have been pruned by 2243 // art::PrepareForRegisterAllocation. 2244 DCHECK(!invoke->IsStaticWithExplicitClinitCheck()); 2245 2246 IntrinsicLocationsBuilderARM intrinsic(codegen_); 2247 if (intrinsic.TryDispatch(invoke)) { 2248 if (invoke->GetLocations()->CanCall() && invoke->HasPcRelativeDexCache()) { 2249 invoke->GetLocations()->SetInAt(invoke->GetSpecialInputIndex(), Location::Any()); 2250 } 2251 return; 2252 } 2253 2254 HandleInvoke(invoke); 2255 2256 // For PC-relative dex cache the invoke has an extra input, the PC-relative address base. 2257 if (invoke->HasPcRelativeDexCache()) { 2258 invoke->GetLocations()->SetInAt(invoke->GetSpecialInputIndex(), Location::RequiresRegister()); 2259 } 2260} 2261 2262static bool TryGenerateIntrinsicCode(HInvoke* invoke, CodeGeneratorARM* codegen) { 2263 if (invoke->GetLocations()->Intrinsified()) { 2264 IntrinsicCodeGeneratorARM intrinsic(codegen); 2265 intrinsic.Dispatch(invoke); 2266 return true; 2267 } 2268 return false; 2269} 2270 2271void InstructionCodeGeneratorARM::VisitInvokeStaticOrDirect(HInvokeStaticOrDirect* invoke) { 2272 // Explicit clinit checks triggered by static invokes must have been pruned by 2273 // art::PrepareForRegisterAllocation. 2274 DCHECK(!invoke->IsStaticWithExplicitClinitCheck()); 2275 2276 if (TryGenerateIntrinsicCode(invoke, codegen_)) { 2277 return; 2278 } 2279 2280 LocationSummary* locations = invoke->GetLocations(); 2281 codegen_->GenerateStaticOrDirectCall( 2282 invoke, locations->HasTemps() ? locations->GetTemp(0) : Location::NoLocation()); 2283 codegen_->RecordPcInfo(invoke, invoke->GetDexPc()); 2284} 2285 2286void LocationsBuilderARM::HandleInvoke(HInvoke* invoke) { 2287 InvokeDexCallingConventionVisitorARM calling_convention_visitor; 2288 CodeGenerator::CreateCommonInvokeLocationSummary(invoke, &calling_convention_visitor); 2289} 2290 2291void LocationsBuilderARM::VisitInvokeVirtual(HInvokeVirtual* invoke) { 2292 IntrinsicLocationsBuilderARM intrinsic(codegen_); 2293 if (intrinsic.TryDispatch(invoke)) { 2294 return; 2295 } 2296 2297 HandleInvoke(invoke); 2298} 2299 2300void InstructionCodeGeneratorARM::VisitInvokeVirtual(HInvokeVirtual* invoke) { 2301 if (TryGenerateIntrinsicCode(invoke, codegen_)) { 2302 return; 2303 } 2304 2305 codegen_->GenerateVirtualCall(invoke, invoke->GetLocations()->GetTemp(0)); 2306 DCHECK(!codegen_->IsLeafMethod()); 2307 codegen_->RecordPcInfo(invoke, invoke->GetDexPc()); 2308} 2309 2310void LocationsBuilderARM::VisitInvokeInterface(HInvokeInterface* invoke) { 2311 HandleInvoke(invoke); 2312 // Add the hidden argument. 2313 invoke->GetLocations()->AddTemp(Location::RegisterLocation(R12)); 2314} 2315 2316void InstructionCodeGeneratorARM::VisitInvokeInterface(HInvokeInterface* invoke) { 2317 // TODO: b/18116999, our IMTs can miss an IncompatibleClassChangeError. 2318 LocationSummary* locations = invoke->GetLocations(); 2319 Register temp = locations->GetTemp(0).AsRegister<Register>(); 2320 Register hidden_reg = locations->GetTemp(1).AsRegister<Register>(); 2321 Location receiver = locations->InAt(0); 2322 uint32_t class_offset = mirror::Object::ClassOffset().Int32Value(); 2323 2324 // Set the hidden argument. This is safe to do this here, as R12 2325 // won't be modified thereafter, before the `blx` (call) instruction. 2326 DCHECK_EQ(R12, hidden_reg); 2327 __ LoadImmediate(hidden_reg, invoke->GetDexMethodIndex()); 2328 2329 if (receiver.IsStackSlot()) { 2330 __ LoadFromOffset(kLoadWord, temp, SP, receiver.GetStackIndex()); 2331 // /* HeapReference<Class> */ temp = temp->klass_ 2332 __ LoadFromOffset(kLoadWord, temp, temp, class_offset); 2333 } else { 2334 // /* HeapReference<Class> */ temp = receiver->klass_ 2335 __ LoadFromOffset(kLoadWord, temp, receiver.AsRegister<Register>(), class_offset); 2336 } 2337 codegen_->MaybeRecordImplicitNullCheck(invoke); 2338 // Instead of simply (possibly) unpoisoning `temp` here, we should 2339 // emit a read barrier for the previous class reference load. 2340 // However this is not required in practice, as this is an 2341 // intermediate/temporary reference and because the current 2342 // concurrent copying collector keeps the from-space memory 2343 // intact/accessible until the end of the marking phase (the 2344 // concurrent copying collector may not in the future). 2345 __ MaybeUnpoisonHeapReference(temp); 2346 __ LoadFromOffset(kLoadWord, temp, temp, 2347 mirror::Class::ImtPtrOffset(kArmPointerSize).Uint32Value()); 2348 uint32_t method_offset = static_cast<uint32_t>(ImTable::OffsetOfElement( 2349 invoke->GetImtIndex(), kArmPointerSize)); 2350 // temp = temp->GetImtEntryAt(method_offset); 2351 __ LoadFromOffset(kLoadWord, temp, temp, method_offset); 2352 uint32_t entry_point = 2353 ArtMethod::EntryPointFromQuickCompiledCodeOffset(kArmPointerSize).Int32Value(); 2354 // LR = temp->GetEntryPoint(); 2355 __ LoadFromOffset(kLoadWord, LR, temp, entry_point); 2356 // LR(); 2357 __ blx(LR); 2358 DCHECK(!codegen_->IsLeafMethod()); 2359 codegen_->RecordPcInfo(invoke, invoke->GetDexPc()); 2360} 2361 2362void LocationsBuilderARM::VisitNeg(HNeg* neg) { 2363 LocationSummary* locations = 2364 new (GetGraph()->GetArena()) LocationSummary(neg, LocationSummary::kNoCall); 2365 switch (neg->GetResultType()) { 2366 case Primitive::kPrimInt: { 2367 locations->SetInAt(0, Location::RequiresRegister()); 2368 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 2369 break; 2370 } 2371 case Primitive::kPrimLong: { 2372 locations->SetInAt(0, Location::RequiresRegister()); 2373 locations->SetOut(Location::RequiresRegister(), Location::kOutputOverlap); 2374 break; 2375 } 2376 2377 case Primitive::kPrimFloat: 2378 case Primitive::kPrimDouble: 2379 locations->SetInAt(0, Location::RequiresFpuRegister()); 2380 locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap); 2381 break; 2382 2383 default: 2384 LOG(FATAL) << "Unexpected neg type " << neg->GetResultType(); 2385 } 2386} 2387 2388void InstructionCodeGeneratorARM::VisitNeg(HNeg* neg) { 2389 LocationSummary* locations = neg->GetLocations(); 2390 Location out = locations->Out(); 2391 Location in = locations->InAt(0); 2392 switch (neg->GetResultType()) { 2393 case Primitive::kPrimInt: 2394 DCHECK(in.IsRegister()); 2395 __ rsb(out.AsRegister<Register>(), in.AsRegister<Register>(), ShifterOperand(0)); 2396 break; 2397 2398 case Primitive::kPrimLong: 2399 DCHECK(in.IsRegisterPair()); 2400 // out.lo = 0 - in.lo (and update the carry/borrow (C) flag) 2401 __ rsbs(out.AsRegisterPairLow<Register>(), 2402 in.AsRegisterPairLow<Register>(), 2403 ShifterOperand(0)); 2404 // We cannot emit an RSC (Reverse Subtract with Carry) 2405 // instruction here, as it does not exist in the Thumb-2 2406 // instruction set. We use the following approach 2407 // using SBC and SUB instead. 2408 // 2409 // out.hi = -C 2410 __ sbc(out.AsRegisterPairHigh<Register>(), 2411 out.AsRegisterPairHigh<Register>(), 2412 ShifterOperand(out.AsRegisterPairHigh<Register>())); 2413 // out.hi = out.hi - in.hi 2414 __ sub(out.AsRegisterPairHigh<Register>(), 2415 out.AsRegisterPairHigh<Register>(), 2416 ShifterOperand(in.AsRegisterPairHigh<Register>())); 2417 break; 2418 2419 case Primitive::kPrimFloat: 2420 DCHECK(in.IsFpuRegister()); 2421 __ vnegs(out.AsFpuRegister<SRegister>(), in.AsFpuRegister<SRegister>()); 2422 break; 2423 2424 case Primitive::kPrimDouble: 2425 DCHECK(in.IsFpuRegisterPair()); 2426 __ vnegd(FromLowSToD(out.AsFpuRegisterPairLow<SRegister>()), 2427 FromLowSToD(in.AsFpuRegisterPairLow<SRegister>())); 2428 break; 2429 2430 default: 2431 LOG(FATAL) << "Unexpected neg type " << neg->GetResultType(); 2432 } 2433} 2434 2435void LocationsBuilderARM::VisitTypeConversion(HTypeConversion* conversion) { 2436 Primitive::Type result_type = conversion->GetResultType(); 2437 Primitive::Type input_type = conversion->GetInputType(); 2438 DCHECK_NE(result_type, input_type); 2439 2440 // The float-to-long, double-to-long and long-to-float type conversions 2441 // rely on a call to the runtime. 2442 LocationSummary::CallKind call_kind = 2443 (((input_type == Primitive::kPrimFloat || input_type == Primitive::kPrimDouble) 2444 && result_type == Primitive::kPrimLong) 2445 || (input_type == Primitive::kPrimLong && result_type == Primitive::kPrimFloat)) 2446 ? LocationSummary::kCallOnMainOnly 2447 : LocationSummary::kNoCall; 2448 LocationSummary* locations = 2449 new (GetGraph()->GetArena()) LocationSummary(conversion, call_kind); 2450 2451 // The Java language does not allow treating boolean as an integral type but 2452 // our bit representation makes it safe. 2453 2454 switch (result_type) { 2455 case Primitive::kPrimByte: 2456 switch (input_type) { 2457 case Primitive::kPrimLong: 2458 // Type conversion from long to byte is a result of code transformations. 2459 case Primitive::kPrimBoolean: 2460 // Boolean input is a result of code transformations. 2461 case Primitive::kPrimShort: 2462 case Primitive::kPrimInt: 2463 case Primitive::kPrimChar: 2464 // Processing a Dex `int-to-byte' instruction. 2465 locations->SetInAt(0, Location::RequiresRegister()); 2466 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 2467 break; 2468 2469 default: 2470 LOG(FATAL) << "Unexpected type conversion from " << input_type 2471 << " to " << result_type; 2472 } 2473 break; 2474 2475 case Primitive::kPrimShort: 2476 switch (input_type) { 2477 case Primitive::kPrimLong: 2478 // Type conversion from long to short is a result of code transformations. 2479 case Primitive::kPrimBoolean: 2480 // Boolean input is a result of code transformations. 2481 case Primitive::kPrimByte: 2482 case Primitive::kPrimInt: 2483 case Primitive::kPrimChar: 2484 // Processing a Dex `int-to-short' instruction. 2485 locations->SetInAt(0, Location::RequiresRegister()); 2486 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 2487 break; 2488 2489 default: 2490 LOG(FATAL) << "Unexpected type conversion from " << input_type 2491 << " to " << result_type; 2492 } 2493 break; 2494 2495 case Primitive::kPrimInt: 2496 switch (input_type) { 2497 case Primitive::kPrimLong: 2498 // Processing a Dex `long-to-int' instruction. 2499 locations->SetInAt(0, Location::Any()); 2500 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 2501 break; 2502 2503 case Primitive::kPrimFloat: 2504 // Processing a Dex `float-to-int' instruction. 2505 locations->SetInAt(0, Location::RequiresFpuRegister()); 2506 locations->SetOut(Location::RequiresRegister()); 2507 locations->AddTemp(Location::RequiresFpuRegister()); 2508 break; 2509 2510 case Primitive::kPrimDouble: 2511 // Processing a Dex `double-to-int' instruction. 2512 locations->SetInAt(0, Location::RequiresFpuRegister()); 2513 locations->SetOut(Location::RequiresRegister()); 2514 locations->AddTemp(Location::RequiresFpuRegister()); 2515 break; 2516 2517 default: 2518 LOG(FATAL) << "Unexpected type conversion from " << input_type 2519 << " to " << result_type; 2520 } 2521 break; 2522 2523 case Primitive::kPrimLong: 2524 switch (input_type) { 2525 case Primitive::kPrimBoolean: 2526 // Boolean input is a result of code transformations. 2527 case Primitive::kPrimByte: 2528 case Primitive::kPrimShort: 2529 case Primitive::kPrimInt: 2530 case Primitive::kPrimChar: 2531 // Processing a Dex `int-to-long' instruction. 2532 locations->SetInAt(0, Location::RequiresRegister()); 2533 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 2534 break; 2535 2536 case Primitive::kPrimFloat: { 2537 // Processing a Dex `float-to-long' instruction. 2538 InvokeRuntimeCallingConvention calling_convention; 2539 locations->SetInAt(0, Location::FpuRegisterLocation( 2540 calling_convention.GetFpuRegisterAt(0))); 2541 locations->SetOut(Location::RegisterPairLocation(R0, R1)); 2542 break; 2543 } 2544 2545 case Primitive::kPrimDouble: { 2546 // Processing a Dex `double-to-long' instruction. 2547 InvokeRuntimeCallingConvention calling_convention; 2548 locations->SetInAt(0, Location::FpuRegisterPairLocation( 2549 calling_convention.GetFpuRegisterAt(0), 2550 calling_convention.GetFpuRegisterAt(1))); 2551 locations->SetOut(Location::RegisterPairLocation(R0, R1)); 2552 break; 2553 } 2554 2555 default: 2556 LOG(FATAL) << "Unexpected type conversion from " << input_type 2557 << " to " << result_type; 2558 } 2559 break; 2560 2561 case Primitive::kPrimChar: 2562 switch (input_type) { 2563 case Primitive::kPrimLong: 2564 // Type conversion from long to char is a result of code transformations. 2565 case Primitive::kPrimBoolean: 2566 // Boolean input is a result of code transformations. 2567 case Primitive::kPrimByte: 2568 case Primitive::kPrimShort: 2569 case Primitive::kPrimInt: 2570 // Processing a Dex `int-to-char' instruction. 2571 locations->SetInAt(0, Location::RequiresRegister()); 2572 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 2573 break; 2574 2575 default: 2576 LOG(FATAL) << "Unexpected type conversion from " << input_type 2577 << " to " << result_type; 2578 } 2579 break; 2580 2581 case Primitive::kPrimFloat: 2582 switch (input_type) { 2583 case Primitive::kPrimBoolean: 2584 // Boolean input is a result of code transformations. 2585 case Primitive::kPrimByte: 2586 case Primitive::kPrimShort: 2587 case Primitive::kPrimInt: 2588 case Primitive::kPrimChar: 2589 // Processing a Dex `int-to-float' instruction. 2590 locations->SetInAt(0, Location::RequiresRegister()); 2591 locations->SetOut(Location::RequiresFpuRegister()); 2592 break; 2593 2594 case Primitive::kPrimLong: { 2595 // Processing a Dex `long-to-float' instruction. 2596 InvokeRuntimeCallingConvention calling_convention; 2597 locations->SetInAt(0, Location::RegisterPairLocation( 2598 calling_convention.GetRegisterAt(0), calling_convention.GetRegisterAt(1))); 2599 locations->SetOut(Location::FpuRegisterLocation(calling_convention.GetFpuRegisterAt(0))); 2600 break; 2601 } 2602 2603 case Primitive::kPrimDouble: 2604 // Processing a Dex `double-to-float' instruction. 2605 locations->SetInAt(0, Location::RequiresFpuRegister()); 2606 locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap); 2607 break; 2608 2609 default: 2610 LOG(FATAL) << "Unexpected type conversion from " << input_type 2611 << " to " << result_type; 2612 }; 2613 break; 2614 2615 case Primitive::kPrimDouble: 2616 switch (input_type) { 2617 case Primitive::kPrimBoolean: 2618 // Boolean input is a result of code transformations. 2619 case Primitive::kPrimByte: 2620 case Primitive::kPrimShort: 2621 case Primitive::kPrimInt: 2622 case Primitive::kPrimChar: 2623 // Processing a Dex `int-to-double' instruction. 2624 locations->SetInAt(0, Location::RequiresRegister()); 2625 locations->SetOut(Location::RequiresFpuRegister()); 2626 break; 2627 2628 case Primitive::kPrimLong: 2629 // Processing a Dex `long-to-double' instruction. 2630 locations->SetInAt(0, Location::RequiresRegister()); 2631 locations->SetOut(Location::RequiresFpuRegister()); 2632 locations->AddTemp(Location::RequiresFpuRegister()); 2633 locations->AddTemp(Location::RequiresFpuRegister()); 2634 break; 2635 2636 case Primitive::kPrimFloat: 2637 // Processing a Dex `float-to-double' instruction. 2638 locations->SetInAt(0, Location::RequiresFpuRegister()); 2639 locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap); 2640 break; 2641 2642 default: 2643 LOG(FATAL) << "Unexpected type conversion from " << input_type 2644 << " to " << result_type; 2645 }; 2646 break; 2647 2648 default: 2649 LOG(FATAL) << "Unexpected type conversion from " << input_type 2650 << " to " << result_type; 2651 } 2652} 2653 2654void InstructionCodeGeneratorARM::VisitTypeConversion(HTypeConversion* conversion) { 2655 LocationSummary* locations = conversion->GetLocations(); 2656 Location out = locations->Out(); 2657 Location in = locations->InAt(0); 2658 Primitive::Type result_type = conversion->GetResultType(); 2659 Primitive::Type input_type = conversion->GetInputType(); 2660 DCHECK_NE(result_type, input_type); 2661 switch (result_type) { 2662 case Primitive::kPrimByte: 2663 switch (input_type) { 2664 case Primitive::kPrimLong: 2665 // Type conversion from long to byte is a result of code transformations. 2666 __ sbfx(out.AsRegister<Register>(), in.AsRegisterPairLow<Register>(), 0, 8); 2667 break; 2668 case Primitive::kPrimBoolean: 2669 // Boolean input is a result of code transformations. 2670 case Primitive::kPrimShort: 2671 case Primitive::kPrimInt: 2672 case Primitive::kPrimChar: 2673 // Processing a Dex `int-to-byte' instruction. 2674 __ sbfx(out.AsRegister<Register>(), in.AsRegister<Register>(), 0, 8); 2675 break; 2676 2677 default: 2678 LOG(FATAL) << "Unexpected type conversion from " << input_type 2679 << " to " << result_type; 2680 } 2681 break; 2682 2683 case Primitive::kPrimShort: 2684 switch (input_type) { 2685 case Primitive::kPrimLong: 2686 // Type conversion from long to short is a result of code transformations. 2687 __ sbfx(out.AsRegister<Register>(), in.AsRegisterPairLow<Register>(), 0, 16); 2688 break; 2689 case Primitive::kPrimBoolean: 2690 // Boolean input is a result of code transformations. 2691 case Primitive::kPrimByte: 2692 case Primitive::kPrimInt: 2693 case Primitive::kPrimChar: 2694 // Processing a Dex `int-to-short' instruction. 2695 __ sbfx(out.AsRegister<Register>(), in.AsRegister<Register>(), 0, 16); 2696 break; 2697 2698 default: 2699 LOG(FATAL) << "Unexpected type conversion from " << input_type 2700 << " to " << result_type; 2701 } 2702 break; 2703 2704 case Primitive::kPrimInt: 2705 switch (input_type) { 2706 case Primitive::kPrimLong: 2707 // Processing a Dex `long-to-int' instruction. 2708 DCHECK(out.IsRegister()); 2709 if (in.IsRegisterPair()) { 2710 __ Mov(out.AsRegister<Register>(), in.AsRegisterPairLow<Register>()); 2711 } else if (in.IsDoubleStackSlot()) { 2712 __ LoadFromOffset(kLoadWord, out.AsRegister<Register>(), SP, in.GetStackIndex()); 2713 } else { 2714 DCHECK(in.IsConstant()); 2715 DCHECK(in.GetConstant()->IsLongConstant()); 2716 int64_t value = in.GetConstant()->AsLongConstant()->GetValue(); 2717 __ LoadImmediate(out.AsRegister<Register>(), static_cast<int32_t>(value)); 2718 } 2719 break; 2720 2721 case Primitive::kPrimFloat: { 2722 // Processing a Dex `float-to-int' instruction. 2723 SRegister temp = locations->GetTemp(0).AsFpuRegisterPairLow<SRegister>(); 2724 __ vcvtis(temp, in.AsFpuRegister<SRegister>()); 2725 __ vmovrs(out.AsRegister<Register>(), temp); 2726 break; 2727 } 2728 2729 case Primitive::kPrimDouble: { 2730 // Processing a Dex `double-to-int' instruction. 2731 SRegister temp_s = locations->GetTemp(0).AsFpuRegisterPairLow<SRegister>(); 2732 __ vcvtid(temp_s, FromLowSToD(in.AsFpuRegisterPairLow<SRegister>())); 2733 __ vmovrs(out.AsRegister<Register>(), temp_s); 2734 break; 2735 } 2736 2737 default: 2738 LOG(FATAL) << "Unexpected type conversion from " << input_type 2739 << " to " << result_type; 2740 } 2741 break; 2742 2743 case Primitive::kPrimLong: 2744 switch (input_type) { 2745 case Primitive::kPrimBoolean: 2746 // Boolean input is a result of code transformations. 2747 case Primitive::kPrimByte: 2748 case Primitive::kPrimShort: 2749 case Primitive::kPrimInt: 2750 case Primitive::kPrimChar: 2751 // Processing a Dex `int-to-long' instruction. 2752 DCHECK(out.IsRegisterPair()); 2753 DCHECK(in.IsRegister()); 2754 __ Mov(out.AsRegisterPairLow<Register>(), in.AsRegister<Register>()); 2755 // Sign extension. 2756 __ Asr(out.AsRegisterPairHigh<Register>(), 2757 out.AsRegisterPairLow<Register>(), 2758 31); 2759 break; 2760 2761 case Primitive::kPrimFloat: 2762 // Processing a Dex `float-to-long' instruction. 2763 codegen_->InvokeRuntime(kQuickF2l, conversion, conversion->GetDexPc()); 2764 CheckEntrypointTypes<kQuickF2l, int64_t, float>(); 2765 break; 2766 2767 case Primitive::kPrimDouble: 2768 // Processing a Dex `double-to-long' instruction. 2769 codegen_->InvokeRuntime(kQuickD2l, conversion, conversion->GetDexPc()); 2770 CheckEntrypointTypes<kQuickD2l, int64_t, double>(); 2771 break; 2772 2773 default: 2774 LOG(FATAL) << "Unexpected type conversion from " << input_type 2775 << " to " << result_type; 2776 } 2777 break; 2778 2779 case Primitive::kPrimChar: 2780 switch (input_type) { 2781 case Primitive::kPrimLong: 2782 // Type conversion from long to char is a result of code transformations. 2783 __ ubfx(out.AsRegister<Register>(), in.AsRegisterPairLow<Register>(), 0, 16); 2784 break; 2785 case Primitive::kPrimBoolean: 2786 // Boolean input is a result of code transformations. 2787 case Primitive::kPrimByte: 2788 case Primitive::kPrimShort: 2789 case Primitive::kPrimInt: 2790 // Processing a Dex `int-to-char' instruction. 2791 __ ubfx(out.AsRegister<Register>(), in.AsRegister<Register>(), 0, 16); 2792 break; 2793 2794 default: 2795 LOG(FATAL) << "Unexpected type conversion from " << input_type 2796 << " to " << result_type; 2797 } 2798 break; 2799 2800 case Primitive::kPrimFloat: 2801 switch (input_type) { 2802 case Primitive::kPrimBoolean: 2803 // Boolean input is a result of code transformations. 2804 case Primitive::kPrimByte: 2805 case Primitive::kPrimShort: 2806 case Primitive::kPrimInt: 2807 case Primitive::kPrimChar: { 2808 // Processing a Dex `int-to-float' instruction. 2809 __ vmovsr(out.AsFpuRegister<SRegister>(), in.AsRegister<Register>()); 2810 __ vcvtsi(out.AsFpuRegister<SRegister>(), out.AsFpuRegister<SRegister>()); 2811 break; 2812 } 2813 2814 case Primitive::kPrimLong: 2815 // Processing a Dex `long-to-float' instruction. 2816 codegen_->InvokeRuntime(kQuickL2f, conversion, conversion->GetDexPc()); 2817 CheckEntrypointTypes<kQuickL2f, float, int64_t>(); 2818 break; 2819 2820 case Primitive::kPrimDouble: 2821 // Processing a Dex `double-to-float' instruction. 2822 __ vcvtsd(out.AsFpuRegister<SRegister>(), 2823 FromLowSToD(in.AsFpuRegisterPairLow<SRegister>())); 2824 break; 2825 2826 default: 2827 LOG(FATAL) << "Unexpected type conversion from " << input_type 2828 << " to " << result_type; 2829 }; 2830 break; 2831 2832 case Primitive::kPrimDouble: 2833 switch (input_type) { 2834 case Primitive::kPrimBoolean: 2835 // Boolean input is a result of code transformations. 2836 case Primitive::kPrimByte: 2837 case Primitive::kPrimShort: 2838 case Primitive::kPrimInt: 2839 case Primitive::kPrimChar: { 2840 // Processing a Dex `int-to-double' instruction. 2841 __ vmovsr(out.AsFpuRegisterPairLow<SRegister>(), in.AsRegister<Register>()); 2842 __ vcvtdi(FromLowSToD(out.AsFpuRegisterPairLow<SRegister>()), 2843 out.AsFpuRegisterPairLow<SRegister>()); 2844 break; 2845 } 2846 2847 case Primitive::kPrimLong: { 2848 // Processing a Dex `long-to-double' instruction. 2849 Register low = in.AsRegisterPairLow<Register>(); 2850 Register high = in.AsRegisterPairHigh<Register>(); 2851 SRegister out_s = out.AsFpuRegisterPairLow<SRegister>(); 2852 DRegister out_d = FromLowSToD(out_s); 2853 SRegister temp_s = locations->GetTemp(0).AsFpuRegisterPairLow<SRegister>(); 2854 DRegister temp_d = FromLowSToD(temp_s); 2855 SRegister constant_s = locations->GetTemp(1).AsFpuRegisterPairLow<SRegister>(); 2856 DRegister constant_d = FromLowSToD(constant_s); 2857 2858 // temp_d = int-to-double(high) 2859 __ vmovsr(temp_s, high); 2860 __ vcvtdi(temp_d, temp_s); 2861 // constant_d = k2Pow32EncodingForDouble 2862 __ LoadDImmediate(constant_d, bit_cast<double, int64_t>(k2Pow32EncodingForDouble)); 2863 // out_d = unsigned-to-double(low) 2864 __ vmovsr(out_s, low); 2865 __ vcvtdu(out_d, out_s); 2866 // out_d += temp_d * constant_d 2867 __ vmlad(out_d, temp_d, constant_d); 2868 break; 2869 } 2870 2871 case Primitive::kPrimFloat: 2872 // Processing a Dex `float-to-double' instruction. 2873 __ vcvtds(FromLowSToD(out.AsFpuRegisterPairLow<SRegister>()), 2874 in.AsFpuRegister<SRegister>()); 2875 break; 2876 2877 default: 2878 LOG(FATAL) << "Unexpected type conversion from " << input_type 2879 << " to " << result_type; 2880 }; 2881 break; 2882 2883 default: 2884 LOG(FATAL) << "Unexpected type conversion from " << input_type 2885 << " to " << result_type; 2886 } 2887} 2888 2889void LocationsBuilderARM::VisitAdd(HAdd* add) { 2890 LocationSummary* locations = 2891 new (GetGraph()->GetArena()) LocationSummary(add, LocationSummary::kNoCall); 2892 switch (add->GetResultType()) { 2893 case Primitive::kPrimInt: { 2894 locations->SetInAt(0, Location::RequiresRegister()); 2895 locations->SetInAt(1, Location::RegisterOrConstant(add->InputAt(1))); 2896 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 2897 break; 2898 } 2899 2900 case Primitive::kPrimLong: { 2901 locations->SetInAt(0, Location::RequiresRegister()); 2902 locations->SetInAt(1, ArmEncodableConstantOrRegister(add->InputAt(1), ADD)); 2903 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 2904 break; 2905 } 2906 2907 case Primitive::kPrimFloat: 2908 case Primitive::kPrimDouble: { 2909 locations->SetInAt(0, Location::RequiresFpuRegister()); 2910 locations->SetInAt(1, Location::RequiresFpuRegister()); 2911 locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap); 2912 break; 2913 } 2914 2915 default: 2916 LOG(FATAL) << "Unexpected add type " << add->GetResultType(); 2917 } 2918} 2919 2920void InstructionCodeGeneratorARM::VisitAdd(HAdd* add) { 2921 LocationSummary* locations = add->GetLocations(); 2922 Location out = locations->Out(); 2923 Location first = locations->InAt(0); 2924 Location second = locations->InAt(1); 2925 switch (add->GetResultType()) { 2926 case Primitive::kPrimInt: 2927 if (second.IsRegister()) { 2928 __ add(out.AsRegister<Register>(), 2929 first.AsRegister<Register>(), 2930 ShifterOperand(second.AsRegister<Register>())); 2931 } else { 2932 __ AddConstant(out.AsRegister<Register>(), 2933 first.AsRegister<Register>(), 2934 second.GetConstant()->AsIntConstant()->GetValue()); 2935 } 2936 break; 2937 2938 case Primitive::kPrimLong: { 2939 if (second.IsConstant()) { 2940 uint64_t value = static_cast<uint64_t>(Int64FromConstant(second.GetConstant())); 2941 GenerateAddLongConst(out, first, value); 2942 } else { 2943 DCHECK(second.IsRegisterPair()); 2944 __ adds(out.AsRegisterPairLow<Register>(), 2945 first.AsRegisterPairLow<Register>(), 2946 ShifterOperand(second.AsRegisterPairLow<Register>())); 2947 __ adc(out.AsRegisterPairHigh<Register>(), 2948 first.AsRegisterPairHigh<Register>(), 2949 ShifterOperand(second.AsRegisterPairHigh<Register>())); 2950 } 2951 break; 2952 } 2953 2954 case Primitive::kPrimFloat: 2955 __ vadds(out.AsFpuRegister<SRegister>(), 2956 first.AsFpuRegister<SRegister>(), 2957 second.AsFpuRegister<SRegister>()); 2958 break; 2959 2960 case Primitive::kPrimDouble: 2961 __ vaddd(FromLowSToD(out.AsFpuRegisterPairLow<SRegister>()), 2962 FromLowSToD(first.AsFpuRegisterPairLow<SRegister>()), 2963 FromLowSToD(second.AsFpuRegisterPairLow<SRegister>())); 2964 break; 2965 2966 default: 2967 LOG(FATAL) << "Unexpected add type " << add->GetResultType(); 2968 } 2969} 2970 2971void LocationsBuilderARM::VisitSub(HSub* sub) { 2972 LocationSummary* locations = 2973 new (GetGraph()->GetArena()) LocationSummary(sub, LocationSummary::kNoCall); 2974 switch (sub->GetResultType()) { 2975 case Primitive::kPrimInt: { 2976 locations->SetInAt(0, Location::RequiresRegister()); 2977 locations->SetInAt(1, Location::RegisterOrConstant(sub->InputAt(1))); 2978 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 2979 break; 2980 } 2981 2982 case Primitive::kPrimLong: { 2983 locations->SetInAt(0, Location::RequiresRegister()); 2984 locations->SetInAt(1, ArmEncodableConstantOrRegister(sub->InputAt(1), SUB)); 2985 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 2986 break; 2987 } 2988 case Primitive::kPrimFloat: 2989 case Primitive::kPrimDouble: { 2990 locations->SetInAt(0, Location::RequiresFpuRegister()); 2991 locations->SetInAt(1, Location::RequiresFpuRegister()); 2992 locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap); 2993 break; 2994 } 2995 default: 2996 LOG(FATAL) << "Unexpected sub type " << sub->GetResultType(); 2997 } 2998} 2999 3000void InstructionCodeGeneratorARM::VisitSub(HSub* sub) { 3001 LocationSummary* locations = sub->GetLocations(); 3002 Location out = locations->Out(); 3003 Location first = locations->InAt(0); 3004 Location second = locations->InAt(1); 3005 switch (sub->GetResultType()) { 3006 case Primitive::kPrimInt: { 3007 if (second.IsRegister()) { 3008 __ sub(out.AsRegister<Register>(), 3009 first.AsRegister<Register>(), 3010 ShifterOperand(second.AsRegister<Register>())); 3011 } else { 3012 __ AddConstant(out.AsRegister<Register>(), 3013 first.AsRegister<Register>(), 3014 -second.GetConstant()->AsIntConstant()->GetValue()); 3015 } 3016 break; 3017 } 3018 3019 case Primitive::kPrimLong: { 3020 if (second.IsConstant()) { 3021 uint64_t value = static_cast<uint64_t>(Int64FromConstant(second.GetConstant())); 3022 GenerateAddLongConst(out, first, -value); 3023 } else { 3024 DCHECK(second.IsRegisterPair()); 3025 __ subs(out.AsRegisterPairLow<Register>(), 3026 first.AsRegisterPairLow<Register>(), 3027 ShifterOperand(second.AsRegisterPairLow<Register>())); 3028 __ sbc(out.AsRegisterPairHigh<Register>(), 3029 first.AsRegisterPairHigh<Register>(), 3030 ShifterOperand(second.AsRegisterPairHigh<Register>())); 3031 } 3032 break; 3033 } 3034 3035 case Primitive::kPrimFloat: { 3036 __ vsubs(out.AsFpuRegister<SRegister>(), 3037 first.AsFpuRegister<SRegister>(), 3038 second.AsFpuRegister<SRegister>()); 3039 break; 3040 } 3041 3042 case Primitive::kPrimDouble: { 3043 __ vsubd(FromLowSToD(out.AsFpuRegisterPairLow<SRegister>()), 3044 FromLowSToD(first.AsFpuRegisterPairLow<SRegister>()), 3045 FromLowSToD(second.AsFpuRegisterPairLow<SRegister>())); 3046 break; 3047 } 3048 3049 3050 default: 3051 LOG(FATAL) << "Unexpected sub type " << sub->GetResultType(); 3052 } 3053} 3054 3055void LocationsBuilderARM::VisitMul(HMul* mul) { 3056 LocationSummary* locations = 3057 new (GetGraph()->GetArena()) LocationSummary(mul, LocationSummary::kNoCall); 3058 switch (mul->GetResultType()) { 3059 case Primitive::kPrimInt: 3060 case Primitive::kPrimLong: { 3061 locations->SetInAt(0, Location::RequiresRegister()); 3062 locations->SetInAt(1, Location::RequiresRegister()); 3063 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 3064 break; 3065 } 3066 3067 case Primitive::kPrimFloat: 3068 case Primitive::kPrimDouble: { 3069 locations->SetInAt(0, Location::RequiresFpuRegister()); 3070 locations->SetInAt(1, Location::RequiresFpuRegister()); 3071 locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap); 3072 break; 3073 } 3074 3075 default: 3076 LOG(FATAL) << "Unexpected mul type " << mul->GetResultType(); 3077 } 3078} 3079 3080void InstructionCodeGeneratorARM::VisitMul(HMul* mul) { 3081 LocationSummary* locations = mul->GetLocations(); 3082 Location out = locations->Out(); 3083 Location first = locations->InAt(0); 3084 Location second = locations->InAt(1); 3085 switch (mul->GetResultType()) { 3086 case Primitive::kPrimInt: { 3087 __ mul(out.AsRegister<Register>(), 3088 first.AsRegister<Register>(), 3089 second.AsRegister<Register>()); 3090 break; 3091 } 3092 case Primitive::kPrimLong: { 3093 Register out_hi = out.AsRegisterPairHigh<Register>(); 3094 Register out_lo = out.AsRegisterPairLow<Register>(); 3095 Register in1_hi = first.AsRegisterPairHigh<Register>(); 3096 Register in1_lo = first.AsRegisterPairLow<Register>(); 3097 Register in2_hi = second.AsRegisterPairHigh<Register>(); 3098 Register in2_lo = second.AsRegisterPairLow<Register>(); 3099 3100 // Extra checks to protect caused by the existence of R1_R2. 3101 // The algorithm is wrong if out.hi is either in1.lo or in2.lo: 3102 // (e.g. in1=r0_r1, in2=r2_r3 and out=r1_r2); 3103 DCHECK_NE(out_hi, in1_lo); 3104 DCHECK_NE(out_hi, in2_lo); 3105 3106 // input: in1 - 64 bits, in2 - 64 bits 3107 // output: out 3108 // formula: out.hi : out.lo = (in1.lo * in2.hi + in1.hi * in2.lo)* 2^32 + in1.lo * in2.lo 3109 // parts: out.hi = in1.lo * in2.hi + in1.hi * in2.lo + (in1.lo * in2.lo)[63:32] 3110 // parts: out.lo = (in1.lo * in2.lo)[31:0] 3111 3112 // IP <- in1.lo * in2.hi 3113 __ mul(IP, in1_lo, in2_hi); 3114 // out.hi <- in1.lo * in2.hi + in1.hi * in2.lo 3115 __ mla(out_hi, in1_hi, in2_lo, IP); 3116 // out.lo <- (in1.lo * in2.lo)[31:0]; 3117 __ umull(out_lo, IP, in1_lo, in2_lo); 3118 // out.hi <- in2.hi * in1.lo + in2.lo * in1.hi + (in1.lo * in2.lo)[63:32] 3119 __ add(out_hi, out_hi, ShifterOperand(IP)); 3120 break; 3121 } 3122 3123 case Primitive::kPrimFloat: { 3124 __ vmuls(out.AsFpuRegister<SRegister>(), 3125 first.AsFpuRegister<SRegister>(), 3126 second.AsFpuRegister<SRegister>()); 3127 break; 3128 } 3129 3130 case Primitive::kPrimDouble: { 3131 __ vmuld(FromLowSToD(out.AsFpuRegisterPairLow<SRegister>()), 3132 FromLowSToD(first.AsFpuRegisterPairLow<SRegister>()), 3133 FromLowSToD(second.AsFpuRegisterPairLow<SRegister>())); 3134 break; 3135 } 3136 3137 default: 3138 LOG(FATAL) << "Unexpected mul type " << mul->GetResultType(); 3139 } 3140} 3141 3142void InstructionCodeGeneratorARM::DivRemOneOrMinusOne(HBinaryOperation* instruction) { 3143 DCHECK(instruction->IsDiv() || instruction->IsRem()); 3144 DCHECK(instruction->GetResultType() == Primitive::kPrimInt); 3145 3146 LocationSummary* locations = instruction->GetLocations(); 3147 Location second = locations->InAt(1); 3148 DCHECK(second.IsConstant()); 3149 3150 Register out = locations->Out().AsRegister<Register>(); 3151 Register dividend = locations->InAt(0).AsRegister<Register>(); 3152 int32_t imm = second.GetConstant()->AsIntConstant()->GetValue(); 3153 DCHECK(imm == 1 || imm == -1); 3154 3155 if (instruction->IsRem()) { 3156 __ LoadImmediate(out, 0); 3157 } else { 3158 if (imm == 1) { 3159 __ Mov(out, dividend); 3160 } else { 3161 __ rsb(out, dividend, ShifterOperand(0)); 3162 } 3163 } 3164} 3165 3166void InstructionCodeGeneratorARM::DivRemByPowerOfTwo(HBinaryOperation* instruction) { 3167 DCHECK(instruction->IsDiv() || instruction->IsRem()); 3168 DCHECK(instruction->GetResultType() == Primitive::kPrimInt); 3169 3170 LocationSummary* locations = instruction->GetLocations(); 3171 Location second = locations->InAt(1); 3172 DCHECK(second.IsConstant()); 3173 3174 Register out = locations->Out().AsRegister<Register>(); 3175 Register dividend = locations->InAt(0).AsRegister<Register>(); 3176 Register temp = locations->GetTemp(0).AsRegister<Register>(); 3177 int32_t imm = second.GetConstant()->AsIntConstant()->GetValue(); 3178 uint32_t abs_imm = static_cast<uint32_t>(AbsOrMin(imm)); 3179 int ctz_imm = CTZ(abs_imm); 3180 3181 if (ctz_imm == 1) { 3182 __ Lsr(temp, dividend, 32 - ctz_imm); 3183 } else { 3184 __ Asr(temp, dividend, 31); 3185 __ Lsr(temp, temp, 32 - ctz_imm); 3186 } 3187 __ add(out, temp, ShifterOperand(dividend)); 3188 3189 if (instruction->IsDiv()) { 3190 __ Asr(out, out, ctz_imm); 3191 if (imm < 0) { 3192 __ rsb(out, out, ShifterOperand(0)); 3193 } 3194 } else { 3195 __ ubfx(out, out, 0, ctz_imm); 3196 __ sub(out, out, ShifterOperand(temp)); 3197 } 3198} 3199 3200void InstructionCodeGeneratorARM::GenerateDivRemWithAnyConstant(HBinaryOperation* instruction) { 3201 DCHECK(instruction->IsDiv() || instruction->IsRem()); 3202 DCHECK(instruction->GetResultType() == Primitive::kPrimInt); 3203 3204 LocationSummary* locations = instruction->GetLocations(); 3205 Location second = locations->InAt(1); 3206 DCHECK(second.IsConstant()); 3207 3208 Register out = locations->Out().AsRegister<Register>(); 3209 Register dividend = locations->InAt(0).AsRegister<Register>(); 3210 Register temp1 = locations->GetTemp(0).AsRegister<Register>(); 3211 Register temp2 = locations->GetTemp(1).AsRegister<Register>(); 3212 int64_t imm = second.GetConstant()->AsIntConstant()->GetValue(); 3213 3214 int64_t magic; 3215 int shift; 3216 CalculateMagicAndShiftForDivRem(imm, false /* is_long */, &magic, &shift); 3217 3218 __ LoadImmediate(temp1, magic); 3219 __ smull(temp2, temp1, dividend, temp1); 3220 3221 if (imm > 0 && magic < 0) { 3222 __ add(temp1, temp1, ShifterOperand(dividend)); 3223 } else if (imm < 0 && magic > 0) { 3224 __ sub(temp1, temp1, ShifterOperand(dividend)); 3225 } 3226 3227 if (shift != 0) { 3228 __ Asr(temp1, temp1, shift); 3229 } 3230 3231 if (instruction->IsDiv()) { 3232 __ sub(out, temp1, ShifterOperand(temp1, ASR, 31)); 3233 } else { 3234 __ sub(temp1, temp1, ShifterOperand(temp1, ASR, 31)); 3235 // TODO: Strength reduction for mls. 3236 __ LoadImmediate(temp2, imm); 3237 __ mls(out, temp1, temp2, dividend); 3238 } 3239} 3240 3241void InstructionCodeGeneratorARM::GenerateDivRemConstantIntegral(HBinaryOperation* instruction) { 3242 DCHECK(instruction->IsDiv() || instruction->IsRem()); 3243 DCHECK(instruction->GetResultType() == Primitive::kPrimInt); 3244 3245 LocationSummary* locations = instruction->GetLocations(); 3246 Location second = locations->InAt(1); 3247 DCHECK(second.IsConstant()); 3248 3249 int32_t imm = second.GetConstant()->AsIntConstant()->GetValue(); 3250 if (imm == 0) { 3251 // Do not generate anything. DivZeroCheck would prevent any code to be executed. 3252 } else if (imm == 1 || imm == -1) { 3253 DivRemOneOrMinusOne(instruction); 3254 } else if (IsPowerOfTwo(AbsOrMin(imm))) { 3255 DivRemByPowerOfTwo(instruction); 3256 } else { 3257 DCHECK(imm <= -2 || imm >= 2); 3258 GenerateDivRemWithAnyConstant(instruction); 3259 } 3260} 3261 3262void LocationsBuilderARM::VisitDiv(HDiv* div) { 3263 LocationSummary::CallKind call_kind = LocationSummary::kNoCall; 3264 if (div->GetResultType() == Primitive::kPrimLong) { 3265 // pLdiv runtime call. 3266 call_kind = LocationSummary::kCallOnMainOnly; 3267 } else if (div->GetResultType() == Primitive::kPrimInt && div->InputAt(1)->IsConstant()) { 3268 // sdiv will be replaced by other instruction sequence. 3269 } else if (div->GetResultType() == Primitive::kPrimInt && 3270 !codegen_->GetInstructionSetFeatures().HasDivideInstruction()) { 3271 // pIdivmod runtime call. 3272 call_kind = LocationSummary::kCallOnMainOnly; 3273 } 3274 3275 LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(div, call_kind); 3276 3277 switch (div->GetResultType()) { 3278 case Primitive::kPrimInt: { 3279 if (div->InputAt(1)->IsConstant()) { 3280 locations->SetInAt(0, Location::RequiresRegister()); 3281 locations->SetInAt(1, Location::ConstantLocation(div->InputAt(1)->AsConstant())); 3282 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 3283 int32_t value = div->InputAt(1)->AsIntConstant()->GetValue(); 3284 if (value == 1 || value == 0 || value == -1) { 3285 // No temp register required. 3286 } else { 3287 locations->AddTemp(Location::RequiresRegister()); 3288 if (!IsPowerOfTwo(AbsOrMin(value))) { 3289 locations->AddTemp(Location::RequiresRegister()); 3290 } 3291 } 3292 } else if (codegen_->GetInstructionSetFeatures().HasDivideInstruction()) { 3293 locations->SetInAt(0, Location::RequiresRegister()); 3294 locations->SetInAt(1, Location::RequiresRegister()); 3295 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 3296 } else { 3297 InvokeRuntimeCallingConvention calling_convention; 3298 locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(0))); 3299 locations->SetInAt(1, Location::RegisterLocation(calling_convention.GetRegisterAt(1))); 3300 // Note: divrem will compute both the quotient and the remainder as the pair R0 and R1, but 3301 // we only need the former. 3302 locations->SetOut(Location::RegisterLocation(R0)); 3303 } 3304 break; 3305 } 3306 case Primitive::kPrimLong: { 3307 InvokeRuntimeCallingConvention calling_convention; 3308 locations->SetInAt(0, Location::RegisterPairLocation( 3309 calling_convention.GetRegisterAt(0), calling_convention.GetRegisterAt(1))); 3310 locations->SetInAt(1, Location::RegisterPairLocation( 3311 calling_convention.GetRegisterAt(2), calling_convention.GetRegisterAt(3))); 3312 locations->SetOut(Location::RegisterPairLocation(R0, R1)); 3313 break; 3314 } 3315 case Primitive::kPrimFloat: 3316 case Primitive::kPrimDouble: { 3317 locations->SetInAt(0, Location::RequiresFpuRegister()); 3318 locations->SetInAt(1, Location::RequiresFpuRegister()); 3319 locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap); 3320 break; 3321 } 3322 3323 default: 3324 LOG(FATAL) << "Unexpected div type " << div->GetResultType(); 3325 } 3326} 3327 3328void InstructionCodeGeneratorARM::VisitDiv(HDiv* div) { 3329 LocationSummary* locations = div->GetLocations(); 3330 Location out = locations->Out(); 3331 Location first = locations->InAt(0); 3332 Location second = locations->InAt(1); 3333 3334 switch (div->GetResultType()) { 3335 case Primitive::kPrimInt: { 3336 if (second.IsConstant()) { 3337 GenerateDivRemConstantIntegral(div); 3338 } else if (codegen_->GetInstructionSetFeatures().HasDivideInstruction()) { 3339 __ sdiv(out.AsRegister<Register>(), 3340 first.AsRegister<Register>(), 3341 second.AsRegister<Register>()); 3342 } else { 3343 InvokeRuntimeCallingConvention calling_convention; 3344 DCHECK_EQ(calling_convention.GetRegisterAt(0), first.AsRegister<Register>()); 3345 DCHECK_EQ(calling_convention.GetRegisterAt(1), second.AsRegister<Register>()); 3346 DCHECK_EQ(R0, out.AsRegister<Register>()); 3347 3348 codegen_->InvokeRuntime(kQuickIdivmod, div, div->GetDexPc()); 3349 CheckEntrypointTypes<kQuickIdivmod, int32_t, int32_t, int32_t>(); 3350 } 3351 break; 3352 } 3353 3354 case Primitive::kPrimLong: { 3355 InvokeRuntimeCallingConvention calling_convention; 3356 DCHECK_EQ(calling_convention.GetRegisterAt(0), first.AsRegisterPairLow<Register>()); 3357 DCHECK_EQ(calling_convention.GetRegisterAt(1), first.AsRegisterPairHigh<Register>()); 3358 DCHECK_EQ(calling_convention.GetRegisterAt(2), second.AsRegisterPairLow<Register>()); 3359 DCHECK_EQ(calling_convention.GetRegisterAt(3), second.AsRegisterPairHigh<Register>()); 3360 DCHECK_EQ(R0, out.AsRegisterPairLow<Register>()); 3361 DCHECK_EQ(R1, out.AsRegisterPairHigh<Register>()); 3362 3363 codegen_->InvokeRuntime(kQuickLdiv, div, div->GetDexPc()); 3364 CheckEntrypointTypes<kQuickLdiv, int64_t, int64_t, int64_t>(); 3365 break; 3366 } 3367 3368 case Primitive::kPrimFloat: { 3369 __ vdivs(out.AsFpuRegister<SRegister>(), 3370 first.AsFpuRegister<SRegister>(), 3371 second.AsFpuRegister<SRegister>()); 3372 break; 3373 } 3374 3375 case Primitive::kPrimDouble: { 3376 __ vdivd(FromLowSToD(out.AsFpuRegisterPairLow<SRegister>()), 3377 FromLowSToD(first.AsFpuRegisterPairLow<SRegister>()), 3378 FromLowSToD(second.AsFpuRegisterPairLow<SRegister>())); 3379 break; 3380 } 3381 3382 default: 3383 LOG(FATAL) << "Unexpected div type " << div->GetResultType(); 3384 } 3385} 3386 3387void LocationsBuilderARM::VisitRem(HRem* rem) { 3388 Primitive::Type type = rem->GetResultType(); 3389 3390 // Most remainders are implemented in the runtime. 3391 LocationSummary::CallKind call_kind = LocationSummary::kCallOnMainOnly; 3392 if (rem->GetResultType() == Primitive::kPrimInt && rem->InputAt(1)->IsConstant()) { 3393 // sdiv will be replaced by other instruction sequence. 3394 call_kind = LocationSummary::kNoCall; 3395 } else if ((rem->GetResultType() == Primitive::kPrimInt) 3396 && codegen_->GetInstructionSetFeatures().HasDivideInstruction()) { 3397 // Have hardware divide instruction for int, do it with three instructions. 3398 call_kind = LocationSummary::kNoCall; 3399 } 3400 3401 LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(rem, call_kind); 3402 3403 switch (type) { 3404 case Primitive::kPrimInt: { 3405 if (rem->InputAt(1)->IsConstant()) { 3406 locations->SetInAt(0, Location::RequiresRegister()); 3407 locations->SetInAt(1, Location::ConstantLocation(rem->InputAt(1)->AsConstant())); 3408 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 3409 int32_t value = rem->InputAt(1)->AsIntConstant()->GetValue(); 3410 if (value == 1 || value == 0 || value == -1) { 3411 // No temp register required. 3412 } else { 3413 locations->AddTemp(Location::RequiresRegister()); 3414 if (!IsPowerOfTwo(AbsOrMin(value))) { 3415 locations->AddTemp(Location::RequiresRegister()); 3416 } 3417 } 3418 } else if (codegen_->GetInstructionSetFeatures().HasDivideInstruction()) { 3419 locations->SetInAt(0, Location::RequiresRegister()); 3420 locations->SetInAt(1, Location::RequiresRegister()); 3421 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 3422 locations->AddTemp(Location::RequiresRegister()); 3423 } else { 3424 InvokeRuntimeCallingConvention calling_convention; 3425 locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(0))); 3426 locations->SetInAt(1, Location::RegisterLocation(calling_convention.GetRegisterAt(1))); 3427 // Note: divrem will compute both the quotient and the remainder as the pair R0 and R1, but 3428 // we only need the latter. 3429 locations->SetOut(Location::RegisterLocation(R1)); 3430 } 3431 break; 3432 } 3433 case Primitive::kPrimLong: { 3434 InvokeRuntimeCallingConvention calling_convention; 3435 locations->SetInAt(0, Location::RegisterPairLocation( 3436 calling_convention.GetRegisterAt(0), calling_convention.GetRegisterAt(1))); 3437 locations->SetInAt(1, Location::RegisterPairLocation( 3438 calling_convention.GetRegisterAt(2), calling_convention.GetRegisterAt(3))); 3439 // The runtime helper puts the output in R2,R3. 3440 locations->SetOut(Location::RegisterPairLocation(R2, R3)); 3441 break; 3442 } 3443 case Primitive::kPrimFloat: { 3444 InvokeRuntimeCallingConvention calling_convention; 3445 locations->SetInAt(0, Location::FpuRegisterLocation(calling_convention.GetFpuRegisterAt(0))); 3446 locations->SetInAt(1, Location::FpuRegisterLocation(calling_convention.GetFpuRegisterAt(1))); 3447 locations->SetOut(Location::FpuRegisterLocation(S0)); 3448 break; 3449 } 3450 3451 case Primitive::kPrimDouble: { 3452 InvokeRuntimeCallingConvention calling_convention; 3453 locations->SetInAt(0, Location::FpuRegisterPairLocation( 3454 calling_convention.GetFpuRegisterAt(0), calling_convention.GetFpuRegisterAt(1))); 3455 locations->SetInAt(1, Location::FpuRegisterPairLocation( 3456 calling_convention.GetFpuRegisterAt(2), calling_convention.GetFpuRegisterAt(3))); 3457 locations->SetOut(Location::Location::FpuRegisterPairLocation(S0, S1)); 3458 break; 3459 } 3460 3461 default: 3462 LOG(FATAL) << "Unexpected rem type " << type; 3463 } 3464} 3465 3466void InstructionCodeGeneratorARM::VisitRem(HRem* rem) { 3467 LocationSummary* locations = rem->GetLocations(); 3468 Location out = locations->Out(); 3469 Location first = locations->InAt(0); 3470 Location second = locations->InAt(1); 3471 3472 Primitive::Type type = rem->GetResultType(); 3473 switch (type) { 3474 case Primitive::kPrimInt: { 3475 if (second.IsConstant()) { 3476 GenerateDivRemConstantIntegral(rem); 3477 } else if (codegen_->GetInstructionSetFeatures().HasDivideInstruction()) { 3478 Register reg1 = first.AsRegister<Register>(); 3479 Register reg2 = second.AsRegister<Register>(); 3480 Register temp = locations->GetTemp(0).AsRegister<Register>(); 3481 3482 // temp = reg1 / reg2 (integer division) 3483 // dest = reg1 - temp * reg2 3484 __ sdiv(temp, reg1, reg2); 3485 __ mls(out.AsRegister<Register>(), temp, reg2, reg1); 3486 } else { 3487 InvokeRuntimeCallingConvention calling_convention; 3488 DCHECK_EQ(calling_convention.GetRegisterAt(0), first.AsRegister<Register>()); 3489 DCHECK_EQ(calling_convention.GetRegisterAt(1), second.AsRegister<Register>()); 3490 DCHECK_EQ(R1, out.AsRegister<Register>()); 3491 3492 codegen_->InvokeRuntime(kQuickIdivmod, rem, rem->GetDexPc()); 3493 CheckEntrypointTypes<kQuickIdivmod, int32_t, int32_t, int32_t>(); 3494 } 3495 break; 3496 } 3497 3498 case Primitive::kPrimLong: { 3499 codegen_->InvokeRuntime(kQuickLmod, rem, rem->GetDexPc()); 3500 CheckEntrypointTypes<kQuickLmod, int64_t, int64_t, int64_t>(); 3501 break; 3502 } 3503 3504 case Primitive::kPrimFloat: { 3505 codegen_->InvokeRuntime(kQuickFmodf, rem, rem->GetDexPc()); 3506 CheckEntrypointTypes<kQuickFmodf, float, float, float>(); 3507 break; 3508 } 3509 3510 case Primitive::kPrimDouble: { 3511 codegen_->InvokeRuntime(kQuickFmod, rem, rem->GetDexPc()); 3512 CheckEntrypointTypes<kQuickFmod, double, double, double>(); 3513 break; 3514 } 3515 3516 default: 3517 LOG(FATAL) << "Unexpected rem type " << type; 3518 } 3519} 3520 3521void LocationsBuilderARM::VisitDivZeroCheck(HDivZeroCheck* instruction) { 3522 LocationSummary* locations = codegen_->CreateThrowingSlowPathLocations(instruction); 3523 locations->SetInAt(0, Location::RegisterOrConstant(instruction->InputAt(0))); 3524} 3525 3526void InstructionCodeGeneratorARM::VisitDivZeroCheck(HDivZeroCheck* instruction) { 3527 SlowPathCodeARM* slow_path = new (GetGraph()->GetArena()) DivZeroCheckSlowPathARM(instruction); 3528 codegen_->AddSlowPath(slow_path); 3529 3530 LocationSummary* locations = instruction->GetLocations(); 3531 Location value = locations->InAt(0); 3532 3533 switch (instruction->GetType()) { 3534 case Primitive::kPrimBoolean: 3535 case Primitive::kPrimByte: 3536 case Primitive::kPrimChar: 3537 case Primitive::kPrimShort: 3538 case Primitive::kPrimInt: { 3539 if (value.IsRegister()) { 3540 __ CompareAndBranchIfZero(value.AsRegister<Register>(), slow_path->GetEntryLabel()); 3541 } else { 3542 DCHECK(value.IsConstant()) << value; 3543 if (value.GetConstant()->AsIntConstant()->GetValue() == 0) { 3544 __ b(slow_path->GetEntryLabel()); 3545 } 3546 } 3547 break; 3548 } 3549 case Primitive::kPrimLong: { 3550 if (value.IsRegisterPair()) { 3551 __ orrs(IP, 3552 value.AsRegisterPairLow<Register>(), 3553 ShifterOperand(value.AsRegisterPairHigh<Register>())); 3554 __ b(slow_path->GetEntryLabel(), EQ); 3555 } else { 3556 DCHECK(value.IsConstant()) << value; 3557 if (value.GetConstant()->AsLongConstant()->GetValue() == 0) { 3558 __ b(slow_path->GetEntryLabel()); 3559 } 3560 } 3561 break; 3562 default: 3563 LOG(FATAL) << "Unexpected type for HDivZeroCheck " << instruction->GetType(); 3564 } 3565 } 3566} 3567 3568void InstructionCodeGeneratorARM::HandleIntegerRotate(LocationSummary* locations) { 3569 Register in = locations->InAt(0).AsRegister<Register>(); 3570 Location rhs = locations->InAt(1); 3571 Register out = locations->Out().AsRegister<Register>(); 3572 3573 if (rhs.IsConstant()) { 3574 // Arm32 and Thumb2 assemblers require a rotation on the interval [1,31], 3575 // so map all rotations to a +ve. equivalent in that range. 3576 // (e.g. left *or* right by -2 bits == 30 bits in the same direction.) 3577 uint32_t rot = CodeGenerator::GetInt32ValueOf(rhs.GetConstant()) & 0x1F; 3578 if (rot) { 3579 // Rotate, mapping left rotations to right equivalents if necessary. 3580 // (e.g. left by 2 bits == right by 30.) 3581 __ Ror(out, in, rot); 3582 } else if (out != in) { 3583 __ Mov(out, in); 3584 } 3585 } else { 3586 __ Ror(out, in, rhs.AsRegister<Register>()); 3587 } 3588} 3589 3590// Gain some speed by mapping all Long rotates onto equivalent pairs of Integer 3591// rotates by swapping input regs (effectively rotating by the first 32-bits of 3592// a larger rotation) or flipping direction (thus treating larger right/left 3593// rotations as sub-word sized rotations in the other direction) as appropriate. 3594void InstructionCodeGeneratorARM::HandleLongRotate(LocationSummary* locations) { 3595 Register in_reg_lo = locations->InAt(0).AsRegisterPairLow<Register>(); 3596 Register in_reg_hi = locations->InAt(0).AsRegisterPairHigh<Register>(); 3597 Location rhs = locations->InAt(1); 3598 Register out_reg_lo = locations->Out().AsRegisterPairLow<Register>(); 3599 Register out_reg_hi = locations->Out().AsRegisterPairHigh<Register>(); 3600 3601 if (rhs.IsConstant()) { 3602 uint64_t rot = CodeGenerator::GetInt64ValueOf(rhs.GetConstant()); 3603 // Map all rotations to +ve. equivalents on the interval [0,63]. 3604 rot &= kMaxLongShiftDistance; 3605 // For rotates over a word in size, 'pre-rotate' by 32-bits to keep rotate 3606 // logic below to a simple pair of binary orr. 3607 // (e.g. 34 bits == in_reg swap + 2 bits right.) 3608 if (rot >= kArmBitsPerWord) { 3609 rot -= kArmBitsPerWord; 3610 std::swap(in_reg_hi, in_reg_lo); 3611 } 3612 // Rotate, or mov to out for zero or word size rotations. 3613 if (rot != 0u) { 3614 __ Lsr(out_reg_hi, in_reg_hi, rot); 3615 __ orr(out_reg_hi, out_reg_hi, ShifterOperand(in_reg_lo, arm::LSL, kArmBitsPerWord - rot)); 3616 __ Lsr(out_reg_lo, in_reg_lo, rot); 3617 __ orr(out_reg_lo, out_reg_lo, ShifterOperand(in_reg_hi, arm::LSL, kArmBitsPerWord - rot)); 3618 } else { 3619 __ Mov(out_reg_lo, in_reg_lo); 3620 __ Mov(out_reg_hi, in_reg_hi); 3621 } 3622 } else { 3623 Register shift_right = locations->GetTemp(0).AsRegister<Register>(); 3624 Register shift_left = locations->GetTemp(1).AsRegister<Register>(); 3625 Label end; 3626 Label shift_by_32_plus_shift_right; 3627 3628 __ and_(shift_right, rhs.AsRegister<Register>(), ShifterOperand(0x1F)); 3629 __ Lsrs(shift_left, rhs.AsRegister<Register>(), 6); 3630 __ rsb(shift_left, shift_right, ShifterOperand(kArmBitsPerWord), AL, kCcKeep); 3631 __ b(&shift_by_32_plus_shift_right, CC); 3632 3633 // out_reg_hi = (reg_hi << shift_left) | (reg_lo >> shift_right). 3634 // out_reg_lo = (reg_lo << shift_left) | (reg_hi >> shift_right). 3635 __ Lsl(out_reg_hi, in_reg_hi, shift_left); 3636 __ Lsr(out_reg_lo, in_reg_lo, shift_right); 3637 __ add(out_reg_hi, out_reg_hi, ShifterOperand(out_reg_lo)); 3638 __ Lsl(out_reg_lo, in_reg_lo, shift_left); 3639 __ Lsr(shift_left, in_reg_hi, shift_right); 3640 __ add(out_reg_lo, out_reg_lo, ShifterOperand(shift_left)); 3641 __ b(&end); 3642 3643 __ Bind(&shift_by_32_plus_shift_right); // Shift by 32+shift_right. 3644 // out_reg_hi = (reg_hi >> shift_right) | (reg_lo << shift_left). 3645 // out_reg_lo = (reg_lo >> shift_right) | (reg_hi << shift_left). 3646 __ Lsr(out_reg_hi, in_reg_hi, shift_right); 3647 __ Lsl(out_reg_lo, in_reg_lo, shift_left); 3648 __ add(out_reg_hi, out_reg_hi, ShifterOperand(out_reg_lo)); 3649 __ Lsr(out_reg_lo, in_reg_lo, shift_right); 3650 __ Lsl(shift_right, in_reg_hi, shift_left); 3651 __ add(out_reg_lo, out_reg_lo, ShifterOperand(shift_right)); 3652 3653 __ Bind(&end); 3654 } 3655} 3656 3657void LocationsBuilderARM::VisitRor(HRor* ror) { 3658 LocationSummary* locations = 3659 new (GetGraph()->GetArena()) LocationSummary(ror, LocationSummary::kNoCall); 3660 switch (ror->GetResultType()) { 3661 case Primitive::kPrimInt: { 3662 locations->SetInAt(0, Location::RequiresRegister()); 3663 locations->SetInAt(1, Location::RegisterOrConstant(ror->InputAt(1))); 3664 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 3665 break; 3666 } 3667 case Primitive::kPrimLong: { 3668 locations->SetInAt(0, Location::RequiresRegister()); 3669 if (ror->InputAt(1)->IsConstant()) { 3670 locations->SetInAt(1, Location::ConstantLocation(ror->InputAt(1)->AsConstant())); 3671 } else { 3672 locations->SetInAt(1, Location::RequiresRegister()); 3673 locations->AddTemp(Location::RequiresRegister()); 3674 locations->AddTemp(Location::RequiresRegister()); 3675 } 3676 locations->SetOut(Location::RequiresRegister(), Location::kOutputOverlap); 3677 break; 3678 } 3679 default: 3680 LOG(FATAL) << "Unexpected operation type " << ror->GetResultType(); 3681 } 3682} 3683 3684void InstructionCodeGeneratorARM::VisitRor(HRor* ror) { 3685 LocationSummary* locations = ror->GetLocations(); 3686 Primitive::Type type = ror->GetResultType(); 3687 switch (type) { 3688 case Primitive::kPrimInt: { 3689 HandleIntegerRotate(locations); 3690 break; 3691 } 3692 case Primitive::kPrimLong: { 3693 HandleLongRotate(locations); 3694 break; 3695 } 3696 default: 3697 LOG(FATAL) << "Unexpected operation type " << type; 3698 UNREACHABLE(); 3699 } 3700} 3701 3702void LocationsBuilderARM::HandleShift(HBinaryOperation* op) { 3703 DCHECK(op->IsShl() || op->IsShr() || op->IsUShr()); 3704 3705 LocationSummary* locations = 3706 new (GetGraph()->GetArena()) LocationSummary(op, LocationSummary::kNoCall); 3707 3708 switch (op->GetResultType()) { 3709 case Primitive::kPrimInt: { 3710 locations->SetInAt(0, Location::RequiresRegister()); 3711 if (op->InputAt(1)->IsConstant()) { 3712 locations->SetInAt(1, Location::ConstantLocation(op->InputAt(1)->AsConstant())); 3713 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 3714 } else { 3715 locations->SetInAt(1, Location::RequiresRegister()); 3716 // Make the output overlap, as it will be used to hold the masked 3717 // second input. 3718 locations->SetOut(Location::RequiresRegister(), Location::kOutputOverlap); 3719 } 3720 break; 3721 } 3722 case Primitive::kPrimLong: { 3723 locations->SetInAt(0, Location::RequiresRegister()); 3724 if (op->InputAt(1)->IsConstant()) { 3725 locations->SetInAt(1, Location::ConstantLocation(op->InputAt(1)->AsConstant())); 3726 // For simplicity, use kOutputOverlap even though we only require that low registers 3727 // don't clash with high registers which the register allocator currently guarantees. 3728 locations->SetOut(Location::RequiresRegister(), Location::kOutputOverlap); 3729 } else { 3730 locations->SetInAt(1, Location::RequiresRegister()); 3731 locations->AddTemp(Location::RequiresRegister()); 3732 locations->SetOut(Location::RequiresRegister(), Location::kOutputOverlap); 3733 } 3734 break; 3735 } 3736 default: 3737 LOG(FATAL) << "Unexpected operation type " << op->GetResultType(); 3738 } 3739} 3740 3741void InstructionCodeGeneratorARM::HandleShift(HBinaryOperation* op) { 3742 DCHECK(op->IsShl() || op->IsShr() || op->IsUShr()); 3743 3744 LocationSummary* locations = op->GetLocations(); 3745 Location out = locations->Out(); 3746 Location first = locations->InAt(0); 3747 Location second = locations->InAt(1); 3748 3749 Primitive::Type type = op->GetResultType(); 3750 switch (type) { 3751 case Primitive::kPrimInt: { 3752 Register out_reg = out.AsRegister<Register>(); 3753 Register first_reg = first.AsRegister<Register>(); 3754 if (second.IsRegister()) { 3755 Register second_reg = second.AsRegister<Register>(); 3756 // ARM doesn't mask the shift count so we need to do it ourselves. 3757 __ and_(out_reg, second_reg, ShifterOperand(kMaxIntShiftDistance)); 3758 if (op->IsShl()) { 3759 __ Lsl(out_reg, first_reg, out_reg); 3760 } else if (op->IsShr()) { 3761 __ Asr(out_reg, first_reg, out_reg); 3762 } else { 3763 __ Lsr(out_reg, first_reg, out_reg); 3764 } 3765 } else { 3766 int32_t cst = second.GetConstant()->AsIntConstant()->GetValue(); 3767 uint32_t shift_value = cst & kMaxIntShiftDistance; 3768 if (shift_value == 0) { // ARM does not support shifting with 0 immediate. 3769 __ Mov(out_reg, first_reg); 3770 } else if (op->IsShl()) { 3771 __ Lsl(out_reg, first_reg, shift_value); 3772 } else if (op->IsShr()) { 3773 __ Asr(out_reg, first_reg, shift_value); 3774 } else { 3775 __ Lsr(out_reg, first_reg, shift_value); 3776 } 3777 } 3778 break; 3779 } 3780 case Primitive::kPrimLong: { 3781 Register o_h = out.AsRegisterPairHigh<Register>(); 3782 Register o_l = out.AsRegisterPairLow<Register>(); 3783 3784 Register high = first.AsRegisterPairHigh<Register>(); 3785 Register low = first.AsRegisterPairLow<Register>(); 3786 3787 if (second.IsRegister()) { 3788 Register temp = locations->GetTemp(0).AsRegister<Register>(); 3789 3790 Register second_reg = second.AsRegister<Register>(); 3791 3792 if (op->IsShl()) { 3793 __ and_(o_l, second_reg, ShifterOperand(kMaxLongShiftDistance)); 3794 // Shift the high part 3795 __ Lsl(o_h, high, o_l); 3796 // Shift the low part and `or` what overflew on the high part 3797 __ rsb(temp, o_l, ShifterOperand(kArmBitsPerWord)); 3798 __ Lsr(temp, low, temp); 3799 __ orr(o_h, o_h, ShifterOperand(temp)); 3800 // If the shift is > 32 bits, override the high part 3801 __ subs(temp, o_l, ShifterOperand(kArmBitsPerWord)); 3802 __ it(PL); 3803 __ Lsl(o_h, low, temp, PL); 3804 // Shift the low part 3805 __ Lsl(o_l, low, o_l); 3806 } else if (op->IsShr()) { 3807 __ and_(o_h, second_reg, ShifterOperand(kMaxLongShiftDistance)); 3808 // Shift the low part 3809 __ Lsr(o_l, low, o_h); 3810 // Shift the high part and `or` what underflew on the low part 3811 __ rsb(temp, o_h, ShifterOperand(kArmBitsPerWord)); 3812 __ Lsl(temp, high, temp); 3813 __ orr(o_l, o_l, ShifterOperand(temp)); 3814 // If the shift is > 32 bits, override the low part 3815 __ subs(temp, o_h, ShifterOperand(kArmBitsPerWord)); 3816 __ it(PL); 3817 __ Asr(o_l, high, temp, PL); 3818 // Shift the high part 3819 __ Asr(o_h, high, o_h); 3820 } else { 3821 __ and_(o_h, second_reg, ShifterOperand(kMaxLongShiftDistance)); 3822 // same as Shr except we use `Lsr`s and not `Asr`s 3823 __ Lsr(o_l, low, o_h); 3824 __ rsb(temp, o_h, ShifterOperand(kArmBitsPerWord)); 3825 __ Lsl(temp, high, temp); 3826 __ orr(o_l, o_l, ShifterOperand(temp)); 3827 __ subs(temp, o_h, ShifterOperand(kArmBitsPerWord)); 3828 __ it(PL); 3829 __ Lsr(o_l, high, temp, PL); 3830 __ Lsr(o_h, high, o_h); 3831 } 3832 } else { 3833 // Register allocator doesn't create partial overlap. 3834 DCHECK_NE(o_l, high); 3835 DCHECK_NE(o_h, low); 3836 int32_t cst = second.GetConstant()->AsIntConstant()->GetValue(); 3837 uint32_t shift_value = cst & kMaxLongShiftDistance; 3838 if (shift_value > 32) { 3839 if (op->IsShl()) { 3840 __ Lsl(o_h, low, shift_value - 32); 3841 __ LoadImmediate(o_l, 0); 3842 } else if (op->IsShr()) { 3843 __ Asr(o_l, high, shift_value - 32); 3844 __ Asr(o_h, high, 31); 3845 } else { 3846 __ Lsr(o_l, high, shift_value - 32); 3847 __ LoadImmediate(o_h, 0); 3848 } 3849 } else if (shift_value == 32) { 3850 if (op->IsShl()) { 3851 __ mov(o_h, ShifterOperand(low)); 3852 __ LoadImmediate(o_l, 0); 3853 } else if (op->IsShr()) { 3854 __ mov(o_l, ShifterOperand(high)); 3855 __ Asr(o_h, high, 31); 3856 } else { 3857 __ mov(o_l, ShifterOperand(high)); 3858 __ LoadImmediate(o_h, 0); 3859 } 3860 } else if (shift_value == 1) { 3861 if (op->IsShl()) { 3862 __ Lsls(o_l, low, 1); 3863 __ adc(o_h, high, ShifterOperand(high)); 3864 } else if (op->IsShr()) { 3865 __ Asrs(o_h, high, 1); 3866 __ Rrx(o_l, low); 3867 } else { 3868 __ Lsrs(o_h, high, 1); 3869 __ Rrx(o_l, low); 3870 } 3871 } else { 3872 DCHECK(2 <= shift_value && shift_value < 32) << shift_value; 3873 if (op->IsShl()) { 3874 __ Lsl(o_h, high, shift_value); 3875 __ orr(o_h, o_h, ShifterOperand(low, LSR, 32 - shift_value)); 3876 __ Lsl(o_l, low, shift_value); 3877 } else if (op->IsShr()) { 3878 __ Lsr(o_l, low, shift_value); 3879 __ orr(o_l, o_l, ShifterOperand(high, LSL, 32 - shift_value)); 3880 __ Asr(o_h, high, shift_value); 3881 } else { 3882 __ Lsr(o_l, low, shift_value); 3883 __ orr(o_l, o_l, ShifterOperand(high, LSL, 32 - shift_value)); 3884 __ Lsr(o_h, high, shift_value); 3885 } 3886 } 3887 } 3888 break; 3889 } 3890 default: 3891 LOG(FATAL) << "Unexpected operation type " << type; 3892 UNREACHABLE(); 3893 } 3894} 3895 3896void LocationsBuilderARM::VisitShl(HShl* shl) { 3897 HandleShift(shl); 3898} 3899 3900void InstructionCodeGeneratorARM::VisitShl(HShl* shl) { 3901 HandleShift(shl); 3902} 3903 3904void LocationsBuilderARM::VisitShr(HShr* shr) { 3905 HandleShift(shr); 3906} 3907 3908void InstructionCodeGeneratorARM::VisitShr(HShr* shr) { 3909 HandleShift(shr); 3910} 3911 3912void LocationsBuilderARM::VisitUShr(HUShr* ushr) { 3913 HandleShift(ushr); 3914} 3915 3916void InstructionCodeGeneratorARM::VisitUShr(HUShr* ushr) { 3917 HandleShift(ushr); 3918} 3919 3920void LocationsBuilderARM::VisitNewInstance(HNewInstance* instruction) { 3921 LocationSummary* locations = 3922 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCallOnMainOnly); 3923 if (instruction->IsStringAlloc()) { 3924 locations->AddTemp(Location::RegisterLocation(kMethodRegisterArgument)); 3925 } else { 3926 InvokeRuntimeCallingConvention calling_convention; 3927 locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(0))); 3928 locations->SetInAt(1, Location::RegisterLocation(calling_convention.GetRegisterAt(1))); 3929 } 3930 locations->SetOut(Location::RegisterLocation(R0)); 3931} 3932 3933void InstructionCodeGeneratorARM::VisitNewInstance(HNewInstance* instruction) { 3934 // Note: if heap poisoning is enabled, the entry point takes cares 3935 // of poisoning the reference. 3936 if (instruction->IsStringAlloc()) { 3937 // String is allocated through StringFactory. Call NewEmptyString entry point. 3938 Register temp = instruction->GetLocations()->GetTemp(0).AsRegister<Register>(); 3939 MemberOffset code_offset = ArtMethod::EntryPointFromQuickCompiledCodeOffset(kArmPointerSize); 3940 __ LoadFromOffset(kLoadWord, temp, TR, QUICK_ENTRY_POINT(pNewEmptyString)); 3941 __ LoadFromOffset(kLoadWord, LR, temp, code_offset.Int32Value()); 3942 __ blx(LR); 3943 codegen_->RecordPcInfo(instruction, instruction->GetDexPc()); 3944 } else { 3945 codegen_->InvokeRuntime(instruction->GetEntrypoint(), instruction, instruction->GetDexPc()); 3946 CheckEntrypointTypes<kQuickAllocObjectWithAccessCheck, void*, uint32_t, ArtMethod*>(); 3947 } 3948} 3949 3950void LocationsBuilderARM::VisitNewArray(HNewArray* instruction) { 3951 LocationSummary* locations = 3952 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCallOnMainOnly); 3953 InvokeRuntimeCallingConvention calling_convention; 3954 locations->AddTemp(Location::RegisterLocation(calling_convention.GetRegisterAt(0))); 3955 locations->SetOut(Location::RegisterLocation(R0)); 3956 locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(1))); 3957 locations->SetInAt(1, Location::RegisterLocation(calling_convention.GetRegisterAt(2))); 3958} 3959 3960void InstructionCodeGeneratorARM::VisitNewArray(HNewArray* instruction) { 3961 InvokeRuntimeCallingConvention calling_convention; 3962 __ LoadImmediate(calling_convention.GetRegisterAt(0), instruction->GetTypeIndex()); 3963 // Note: if heap poisoning is enabled, the entry point takes cares 3964 // of poisoning the reference. 3965 codegen_->InvokeRuntime(instruction->GetEntrypoint(), instruction, instruction->GetDexPc()); 3966 CheckEntrypointTypes<kQuickAllocArrayWithAccessCheck, void*, uint32_t, int32_t, ArtMethod*>(); 3967} 3968 3969void LocationsBuilderARM::VisitParameterValue(HParameterValue* instruction) { 3970 LocationSummary* locations = 3971 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall); 3972 Location location = parameter_visitor_.GetNextLocation(instruction->GetType()); 3973 if (location.IsStackSlot()) { 3974 location = Location::StackSlot(location.GetStackIndex() + codegen_->GetFrameSize()); 3975 } else if (location.IsDoubleStackSlot()) { 3976 location = Location::DoubleStackSlot(location.GetStackIndex() + codegen_->GetFrameSize()); 3977 } 3978 locations->SetOut(location); 3979} 3980 3981void InstructionCodeGeneratorARM::VisitParameterValue( 3982 HParameterValue* instruction ATTRIBUTE_UNUSED) { 3983 // Nothing to do, the parameter is already at its location. 3984} 3985 3986void LocationsBuilderARM::VisitCurrentMethod(HCurrentMethod* instruction) { 3987 LocationSummary* locations = 3988 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall); 3989 locations->SetOut(Location::RegisterLocation(kMethodRegisterArgument)); 3990} 3991 3992void InstructionCodeGeneratorARM::VisitCurrentMethod(HCurrentMethod* instruction ATTRIBUTE_UNUSED) { 3993 // Nothing to do, the method is already at its location. 3994} 3995 3996void LocationsBuilderARM::VisitNot(HNot* not_) { 3997 LocationSummary* locations = 3998 new (GetGraph()->GetArena()) LocationSummary(not_, LocationSummary::kNoCall); 3999 locations->SetInAt(0, Location::RequiresRegister()); 4000 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 4001} 4002 4003void InstructionCodeGeneratorARM::VisitNot(HNot* not_) { 4004 LocationSummary* locations = not_->GetLocations(); 4005 Location out = locations->Out(); 4006 Location in = locations->InAt(0); 4007 switch (not_->GetResultType()) { 4008 case Primitive::kPrimInt: 4009 __ mvn(out.AsRegister<Register>(), ShifterOperand(in.AsRegister<Register>())); 4010 break; 4011 4012 case Primitive::kPrimLong: 4013 __ mvn(out.AsRegisterPairLow<Register>(), 4014 ShifterOperand(in.AsRegisterPairLow<Register>())); 4015 __ mvn(out.AsRegisterPairHigh<Register>(), 4016 ShifterOperand(in.AsRegisterPairHigh<Register>())); 4017 break; 4018 4019 default: 4020 LOG(FATAL) << "Unimplemented type for not operation " << not_->GetResultType(); 4021 } 4022} 4023 4024void LocationsBuilderARM::VisitBooleanNot(HBooleanNot* bool_not) { 4025 LocationSummary* locations = 4026 new (GetGraph()->GetArena()) LocationSummary(bool_not, LocationSummary::kNoCall); 4027 locations->SetInAt(0, Location::RequiresRegister()); 4028 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 4029} 4030 4031void InstructionCodeGeneratorARM::VisitBooleanNot(HBooleanNot* bool_not) { 4032 LocationSummary* locations = bool_not->GetLocations(); 4033 Location out = locations->Out(); 4034 Location in = locations->InAt(0); 4035 __ eor(out.AsRegister<Register>(), in.AsRegister<Register>(), ShifterOperand(1)); 4036} 4037 4038void LocationsBuilderARM::VisitCompare(HCompare* compare) { 4039 LocationSummary* locations = 4040 new (GetGraph()->GetArena()) LocationSummary(compare, LocationSummary::kNoCall); 4041 switch (compare->InputAt(0)->GetType()) { 4042 case Primitive::kPrimBoolean: 4043 case Primitive::kPrimByte: 4044 case Primitive::kPrimShort: 4045 case Primitive::kPrimChar: 4046 case Primitive::kPrimInt: 4047 case Primitive::kPrimLong: { 4048 locations->SetInAt(0, Location::RequiresRegister()); 4049 locations->SetInAt(1, Location::RequiresRegister()); 4050 // Output overlaps because it is written before doing the low comparison. 4051 locations->SetOut(Location::RequiresRegister(), Location::kOutputOverlap); 4052 break; 4053 } 4054 case Primitive::kPrimFloat: 4055 case Primitive::kPrimDouble: { 4056 locations->SetInAt(0, Location::RequiresFpuRegister()); 4057 locations->SetInAt(1, ArithmeticZeroOrFpuRegister(compare->InputAt(1))); 4058 locations->SetOut(Location::RequiresRegister()); 4059 break; 4060 } 4061 default: 4062 LOG(FATAL) << "Unexpected type for compare operation " << compare->InputAt(0)->GetType(); 4063 } 4064} 4065 4066void InstructionCodeGeneratorARM::VisitCompare(HCompare* compare) { 4067 LocationSummary* locations = compare->GetLocations(); 4068 Register out = locations->Out().AsRegister<Register>(); 4069 Location left = locations->InAt(0); 4070 Location right = locations->InAt(1); 4071 4072 Label less, greater, done; 4073 Primitive::Type type = compare->InputAt(0)->GetType(); 4074 Condition less_cond; 4075 switch (type) { 4076 case Primitive::kPrimBoolean: 4077 case Primitive::kPrimByte: 4078 case Primitive::kPrimShort: 4079 case Primitive::kPrimChar: 4080 case Primitive::kPrimInt: { 4081 __ LoadImmediate(out, 0); 4082 __ cmp(left.AsRegister<Register>(), 4083 ShifterOperand(right.AsRegister<Register>())); // Signed compare. 4084 less_cond = LT; 4085 break; 4086 } 4087 case Primitive::kPrimLong: { 4088 __ cmp(left.AsRegisterPairHigh<Register>(), 4089 ShifterOperand(right.AsRegisterPairHigh<Register>())); // Signed compare. 4090 __ b(&less, LT); 4091 __ b(&greater, GT); 4092 // Do LoadImmediate before the last `cmp`, as LoadImmediate might affect the status flags. 4093 __ LoadImmediate(out, 0); 4094 __ cmp(left.AsRegisterPairLow<Register>(), 4095 ShifterOperand(right.AsRegisterPairLow<Register>())); // Unsigned compare. 4096 less_cond = LO; 4097 break; 4098 } 4099 case Primitive::kPrimFloat: 4100 case Primitive::kPrimDouble: { 4101 __ LoadImmediate(out, 0); 4102 GenerateVcmp(compare); 4103 __ vmstat(); // transfer FP status register to ARM APSR. 4104 less_cond = ARMFPCondition(kCondLT, compare->IsGtBias()); 4105 break; 4106 } 4107 default: 4108 LOG(FATAL) << "Unexpected compare type " << type; 4109 UNREACHABLE(); 4110 } 4111 4112 __ b(&done, EQ); 4113 __ b(&less, less_cond); 4114 4115 __ Bind(&greater); 4116 __ LoadImmediate(out, 1); 4117 __ b(&done); 4118 4119 __ Bind(&less); 4120 __ LoadImmediate(out, -1); 4121 4122 __ Bind(&done); 4123} 4124 4125void LocationsBuilderARM::VisitPhi(HPhi* instruction) { 4126 LocationSummary* locations = 4127 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall); 4128 for (size_t i = 0, e = locations->GetInputCount(); i < e; ++i) { 4129 locations->SetInAt(i, Location::Any()); 4130 } 4131 locations->SetOut(Location::Any()); 4132} 4133 4134void InstructionCodeGeneratorARM::VisitPhi(HPhi* instruction ATTRIBUTE_UNUSED) { 4135 LOG(FATAL) << "Unreachable"; 4136} 4137 4138void CodeGeneratorARM::GenerateMemoryBarrier(MemBarrierKind kind) { 4139 // TODO (ported from quick): revisit ARM barrier kinds. 4140 DmbOptions flavor = DmbOptions::ISH; // Quiet C++ warnings. 4141 switch (kind) { 4142 case MemBarrierKind::kAnyStore: 4143 case MemBarrierKind::kLoadAny: 4144 case MemBarrierKind::kAnyAny: { 4145 flavor = DmbOptions::ISH; 4146 break; 4147 } 4148 case MemBarrierKind::kStoreStore: { 4149 flavor = DmbOptions::ISHST; 4150 break; 4151 } 4152 default: 4153 LOG(FATAL) << "Unexpected memory barrier " << kind; 4154 } 4155 __ dmb(flavor); 4156} 4157 4158void InstructionCodeGeneratorARM::GenerateWideAtomicLoad(Register addr, 4159 uint32_t offset, 4160 Register out_lo, 4161 Register out_hi) { 4162 if (offset != 0) { 4163 // Ensure `out_lo` is different from `addr`, so that loading 4164 // `offset` into `out_lo` does not clutter `addr`. 4165 DCHECK_NE(out_lo, addr); 4166 __ LoadImmediate(out_lo, offset); 4167 __ add(IP, addr, ShifterOperand(out_lo)); 4168 addr = IP; 4169 } 4170 __ ldrexd(out_lo, out_hi, addr); 4171} 4172 4173void InstructionCodeGeneratorARM::GenerateWideAtomicStore(Register addr, 4174 uint32_t offset, 4175 Register value_lo, 4176 Register value_hi, 4177 Register temp1, 4178 Register temp2, 4179 HInstruction* instruction) { 4180 Label fail; 4181 if (offset != 0) { 4182 __ LoadImmediate(temp1, offset); 4183 __ add(IP, addr, ShifterOperand(temp1)); 4184 addr = IP; 4185 } 4186 __ Bind(&fail); 4187 // We need a load followed by store. (The address used in a STREX instruction must 4188 // be the same as the address in the most recently executed LDREX instruction.) 4189 __ ldrexd(temp1, temp2, addr); 4190 codegen_->MaybeRecordImplicitNullCheck(instruction); 4191 __ strexd(temp1, value_lo, value_hi, addr); 4192 __ CompareAndBranchIfNonZero(temp1, &fail); 4193} 4194 4195void LocationsBuilderARM::HandleFieldSet(HInstruction* instruction, const FieldInfo& field_info) { 4196 DCHECK(instruction->IsInstanceFieldSet() || instruction->IsStaticFieldSet()); 4197 4198 LocationSummary* locations = 4199 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall); 4200 locations->SetInAt(0, Location::RequiresRegister()); 4201 4202 Primitive::Type field_type = field_info.GetFieldType(); 4203 if (Primitive::IsFloatingPointType(field_type)) { 4204 locations->SetInAt(1, Location::RequiresFpuRegister()); 4205 } else { 4206 locations->SetInAt(1, Location::RequiresRegister()); 4207 } 4208 4209 bool is_wide = field_type == Primitive::kPrimLong || field_type == Primitive::kPrimDouble; 4210 bool generate_volatile = field_info.IsVolatile() 4211 && is_wide 4212 && !codegen_->GetInstructionSetFeatures().HasAtomicLdrdAndStrd(); 4213 bool needs_write_barrier = 4214 CodeGenerator::StoreNeedsWriteBarrier(field_type, instruction->InputAt(1)); 4215 // Temporary registers for the write barrier. 4216 // TODO: consider renaming StoreNeedsWriteBarrier to StoreNeedsGCMark. 4217 if (needs_write_barrier) { 4218 locations->AddTemp(Location::RequiresRegister()); // Possibly used for reference poisoning too. 4219 locations->AddTemp(Location::RequiresRegister()); 4220 } else if (generate_volatile) { 4221 // ARM encoding have some additional constraints for ldrexd/strexd: 4222 // - registers need to be consecutive 4223 // - the first register should be even but not R14. 4224 // We don't test for ARM yet, and the assertion makes sure that we 4225 // revisit this if we ever enable ARM encoding. 4226 DCHECK_EQ(InstructionSet::kThumb2, codegen_->GetInstructionSet()); 4227 4228 locations->AddTemp(Location::RequiresRegister()); 4229 locations->AddTemp(Location::RequiresRegister()); 4230 if (field_type == Primitive::kPrimDouble) { 4231 // For doubles we need two more registers to copy the value. 4232 locations->AddTemp(Location::RegisterLocation(R2)); 4233 locations->AddTemp(Location::RegisterLocation(R3)); 4234 } 4235 } 4236} 4237 4238void InstructionCodeGeneratorARM::HandleFieldSet(HInstruction* instruction, 4239 const FieldInfo& field_info, 4240 bool value_can_be_null) { 4241 DCHECK(instruction->IsInstanceFieldSet() || instruction->IsStaticFieldSet()); 4242 4243 LocationSummary* locations = instruction->GetLocations(); 4244 Register base = locations->InAt(0).AsRegister<Register>(); 4245 Location value = locations->InAt(1); 4246 4247 bool is_volatile = field_info.IsVolatile(); 4248 bool atomic_ldrd_strd = codegen_->GetInstructionSetFeatures().HasAtomicLdrdAndStrd(); 4249 Primitive::Type field_type = field_info.GetFieldType(); 4250 uint32_t offset = field_info.GetFieldOffset().Uint32Value(); 4251 bool needs_write_barrier = 4252 CodeGenerator::StoreNeedsWriteBarrier(field_type, instruction->InputAt(1)); 4253 4254 if (is_volatile) { 4255 codegen_->GenerateMemoryBarrier(MemBarrierKind::kAnyStore); 4256 } 4257 4258 switch (field_type) { 4259 case Primitive::kPrimBoolean: 4260 case Primitive::kPrimByte: { 4261 __ StoreToOffset(kStoreByte, value.AsRegister<Register>(), base, offset); 4262 break; 4263 } 4264 4265 case Primitive::kPrimShort: 4266 case Primitive::kPrimChar: { 4267 __ StoreToOffset(kStoreHalfword, value.AsRegister<Register>(), base, offset); 4268 break; 4269 } 4270 4271 case Primitive::kPrimInt: 4272 case Primitive::kPrimNot: { 4273 if (kPoisonHeapReferences && needs_write_barrier) { 4274 // Note that in the case where `value` is a null reference, 4275 // we do not enter this block, as a null reference does not 4276 // need poisoning. 4277 DCHECK_EQ(field_type, Primitive::kPrimNot); 4278 Register temp = locations->GetTemp(0).AsRegister<Register>(); 4279 __ Mov(temp, value.AsRegister<Register>()); 4280 __ PoisonHeapReference(temp); 4281 __ StoreToOffset(kStoreWord, temp, base, offset); 4282 } else { 4283 __ StoreToOffset(kStoreWord, value.AsRegister<Register>(), base, offset); 4284 } 4285 break; 4286 } 4287 4288 case Primitive::kPrimLong: { 4289 if (is_volatile && !atomic_ldrd_strd) { 4290 GenerateWideAtomicStore(base, offset, 4291 value.AsRegisterPairLow<Register>(), 4292 value.AsRegisterPairHigh<Register>(), 4293 locations->GetTemp(0).AsRegister<Register>(), 4294 locations->GetTemp(1).AsRegister<Register>(), 4295 instruction); 4296 } else { 4297 __ StoreToOffset(kStoreWordPair, value.AsRegisterPairLow<Register>(), base, offset); 4298 codegen_->MaybeRecordImplicitNullCheck(instruction); 4299 } 4300 break; 4301 } 4302 4303 case Primitive::kPrimFloat: { 4304 __ StoreSToOffset(value.AsFpuRegister<SRegister>(), base, offset); 4305 break; 4306 } 4307 4308 case Primitive::kPrimDouble: { 4309 DRegister value_reg = FromLowSToD(value.AsFpuRegisterPairLow<SRegister>()); 4310 if (is_volatile && !atomic_ldrd_strd) { 4311 Register value_reg_lo = locations->GetTemp(0).AsRegister<Register>(); 4312 Register value_reg_hi = locations->GetTemp(1).AsRegister<Register>(); 4313 4314 __ vmovrrd(value_reg_lo, value_reg_hi, value_reg); 4315 4316 GenerateWideAtomicStore(base, offset, 4317 value_reg_lo, 4318 value_reg_hi, 4319 locations->GetTemp(2).AsRegister<Register>(), 4320 locations->GetTemp(3).AsRegister<Register>(), 4321 instruction); 4322 } else { 4323 __ StoreDToOffset(value_reg, base, offset); 4324 codegen_->MaybeRecordImplicitNullCheck(instruction); 4325 } 4326 break; 4327 } 4328 4329 case Primitive::kPrimVoid: 4330 LOG(FATAL) << "Unreachable type " << field_type; 4331 UNREACHABLE(); 4332 } 4333 4334 // Longs and doubles are handled in the switch. 4335 if (field_type != Primitive::kPrimLong && field_type != Primitive::kPrimDouble) { 4336 codegen_->MaybeRecordImplicitNullCheck(instruction); 4337 } 4338 4339 if (CodeGenerator::StoreNeedsWriteBarrier(field_type, instruction->InputAt(1))) { 4340 Register temp = locations->GetTemp(0).AsRegister<Register>(); 4341 Register card = locations->GetTemp(1).AsRegister<Register>(); 4342 codegen_->MarkGCCard( 4343 temp, card, base, value.AsRegister<Register>(), value_can_be_null); 4344 } 4345 4346 if (is_volatile) { 4347 codegen_->GenerateMemoryBarrier(MemBarrierKind::kAnyAny); 4348 } 4349} 4350 4351void LocationsBuilderARM::HandleFieldGet(HInstruction* instruction, const FieldInfo& field_info) { 4352 DCHECK(instruction->IsInstanceFieldGet() || instruction->IsStaticFieldGet()); 4353 4354 bool object_field_get_with_read_barrier = 4355 kEmitCompilerReadBarrier && (field_info.GetFieldType() == Primitive::kPrimNot); 4356 LocationSummary* locations = 4357 new (GetGraph()->GetArena()) LocationSummary(instruction, 4358 object_field_get_with_read_barrier ? 4359 LocationSummary::kCallOnSlowPath : 4360 LocationSummary::kNoCall); 4361 if (object_field_get_with_read_barrier && kUseBakerReadBarrier) { 4362 locations->SetCustomSlowPathCallerSaves(RegisterSet::Empty()); // No caller-save registers. 4363 } 4364 locations->SetInAt(0, Location::RequiresRegister()); 4365 4366 bool volatile_for_double = field_info.IsVolatile() 4367 && (field_info.GetFieldType() == Primitive::kPrimDouble) 4368 && !codegen_->GetInstructionSetFeatures().HasAtomicLdrdAndStrd(); 4369 // The output overlaps in case of volatile long: we don't want the 4370 // code generated by GenerateWideAtomicLoad to overwrite the 4371 // object's location. Likewise, in the case of an object field get 4372 // with read barriers enabled, we do not want the load to overwrite 4373 // the object's location, as we need it to emit the read barrier. 4374 bool overlap = (field_info.IsVolatile() && (field_info.GetFieldType() == Primitive::kPrimLong)) || 4375 object_field_get_with_read_barrier; 4376 4377 if (Primitive::IsFloatingPointType(instruction->GetType())) { 4378 locations->SetOut(Location::RequiresFpuRegister()); 4379 } else { 4380 locations->SetOut(Location::RequiresRegister(), 4381 (overlap ? Location::kOutputOverlap : Location::kNoOutputOverlap)); 4382 } 4383 if (volatile_for_double) { 4384 // ARM encoding have some additional constraints for ldrexd/strexd: 4385 // - registers need to be consecutive 4386 // - the first register should be even but not R14. 4387 // We don't test for ARM yet, and the assertion makes sure that we 4388 // revisit this if we ever enable ARM encoding. 4389 DCHECK_EQ(InstructionSet::kThumb2, codegen_->GetInstructionSet()); 4390 locations->AddTemp(Location::RequiresRegister()); 4391 locations->AddTemp(Location::RequiresRegister()); 4392 } else if (object_field_get_with_read_barrier && kUseBakerReadBarrier) { 4393 // We need a temporary register for the read barrier marking slow 4394 // path in CodeGeneratorARM::GenerateFieldLoadWithBakerReadBarrier. 4395 locations->AddTemp(Location::RequiresRegister()); 4396 } 4397} 4398 4399Location LocationsBuilderARM::ArithmeticZeroOrFpuRegister(HInstruction* input) { 4400 DCHECK(input->GetType() == Primitive::kPrimDouble || input->GetType() == Primitive::kPrimFloat) 4401 << input->GetType(); 4402 if ((input->IsFloatConstant() && (input->AsFloatConstant()->IsArithmeticZero())) || 4403 (input->IsDoubleConstant() && (input->AsDoubleConstant()->IsArithmeticZero()))) { 4404 return Location::ConstantLocation(input->AsConstant()); 4405 } else { 4406 return Location::RequiresFpuRegister(); 4407 } 4408} 4409 4410Location LocationsBuilderARM::ArmEncodableConstantOrRegister(HInstruction* constant, 4411 Opcode opcode) { 4412 DCHECK(!Primitive::IsFloatingPointType(constant->GetType())); 4413 if (constant->IsConstant() && 4414 CanEncodeConstantAsImmediate(constant->AsConstant(), opcode)) { 4415 return Location::ConstantLocation(constant->AsConstant()); 4416 } 4417 return Location::RequiresRegister(); 4418} 4419 4420bool LocationsBuilderARM::CanEncodeConstantAsImmediate(HConstant* input_cst, 4421 Opcode opcode) { 4422 uint64_t value = static_cast<uint64_t>(Int64FromConstant(input_cst)); 4423 if (Primitive::Is64BitType(input_cst->GetType())) { 4424 Opcode high_opcode = opcode; 4425 SetCc low_set_cc = kCcDontCare; 4426 switch (opcode) { 4427 case SUB: 4428 // Flip the operation to an ADD. 4429 value = -value; 4430 opcode = ADD; 4431 FALLTHROUGH_INTENDED; 4432 case ADD: 4433 if (Low32Bits(value) == 0u) { 4434 return CanEncodeConstantAsImmediate(High32Bits(value), opcode, kCcDontCare); 4435 } 4436 high_opcode = ADC; 4437 low_set_cc = kCcSet; 4438 break; 4439 default: 4440 break; 4441 } 4442 return CanEncodeConstantAsImmediate(Low32Bits(value), opcode, low_set_cc) && 4443 CanEncodeConstantAsImmediate(High32Bits(value), high_opcode, kCcDontCare); 4444 } else { 4445 return CanEncodeConstantAsImmediate(Low32Bits(value), opcode); 4446 } 4447} 4448 4449bool LocationsBuilderARM::CanEncodeConstantAsImmediate(uint32_t value, 4450 Opcode opcode, 4451 SetCc set_cc) { 4452 ShifterOperand so; 4453 ArmAssembler* assembler = codegen_->GetAssembler(); 4454 if (assembler->ShifterOperandCanHold(kNoRegister, kNoRegister, opcode, value, set_cc, &so)) { 4455 return true; 4456 } 4457 Opcode neg_opcode = kNoOperand; 4458 switch (opcode) { 4459 case AND: neg_opcode = BIC; value = ~value; break; 4460 case ORR: neg_opcode = ORN; value = ~value; break; 4461 case ADD: neg_opcode = SUB; value = -value; break; 4462 case ADC: neg_opcode = SBC; value = ~value; break; 4463 case SUB: neg_opcode = ADD; value = -value; break; 4464 case SBC: neg_opcode = ADC; value = ~value; break; 4465 default: 4466 return false; 4467 } 4468 return assembler->ShifterOperandCanHold(kNoRegister, kNoRegister, neg_opcode, value, set_cc, &so); 4469} 4470 4471void InstructionCodeGeneratorARM::HandleFieldGet(HInstruction* instruction, 4472 const FieldInfo& field_info) { 4473 DCHECK(instruction->IsInstanceFieldGet() || instruction->IsStaticFieldGet()); 4474 4475 LocationSummary* locations = instruction->GetLocations(); 4476 Location base_loc = locations->InAt(0); 4477 Register base = base_loc.AsRegister<Register>(); 4478 Location out = locations->Out(); 4479 bool is_volatile = field_info.IsVolatile(); 4480 bool atomic_ldrd_strd = codegen_->GetInstructionSetFeatures().HasAtomicLdrdAndStrd(); 4481 Primitive::Type field_type = field_info.GetFieldType(); 4482 uint32_t offset = field_info.GetFieldOffset().Uint32Value(); 4483 4484 switch (field_type) { 4485 case Primitive::kPrimBoolean: 4486 __ LoadFromOffset(kLoadUnsignedByte, out.AsRegister<Register>(), base, offset); 4487 break; 4488 4489 case Primitive::kPrimByte: 4490 __ LoadFromOffset(kLoadSignedByte, out.AsRegister<Register>(), base, offset); 4491 break; 4492 4493 case Primitive::kPrimShort: 4494 __ LoadFromOffset(kLoadSignedHalfword, out.AsRegister<Register>(), base, offset); 4495 break; 4496 4497 case Primitive::kPrimChar: 4498 __ LoadFromOffset(kLoadUnsignedHalfword, out.AsRegister<Register>(), base, offset); 4499 break; 4500 4501 case Primitive::kPrimInt: 4502 __ LoadFromOffset(kLoadWord, out.AsRegister<Register>(), base, offset); 4503 break; 4504 4505 case Primitive::kPrimNot: { 4506 // /* HeapReference<Object> */ out = *(base + offset) 4507 if (kEmitCompilerReadBarrier && kUseBakerReadBarrier) { 4508 Location temp_loc = locations->GetTemp(0); 4509 // Note that a potential implicit null check is handled in this 4510 // CodeGeneratorARM::GenerateFieldLoadWithBakerReadBarrier call. 4511 codegen_->GenerateFieldLoadWithBakerReadBarrier( 4512 instruction, out, base, offset, temp_loc, /* needs_null_check */ true); 4513 if (is_volatile) { 4514 codegen_->GenerateMemoryBarrier(MemBarrierKind::kLoadAny); 4515 } 4516 } else { 4517 __ LoadFromOffset(kLoadWord, out.AsRegister<Register>(), base, offset); 4518 codegen_->MaybeRecordImplicitNullCheck(instruction); 4519 if (is_volatile) { 4520 codegen_->GenerateMemoryBarrier(MemBarrierKind::kLoadAny); 4521 } 4522 // If read barriers are enabled, emit read barriers other than 4523 // Baker's using a slow path (and also unpoison the loaded 4524 // reference, if heap poisoning is enabled). 4525 codegen_->MaybeGenerateReadBarrierSlow(instruction, out, out, base_loc, offset); 4526 } 4527 break; 4528 } 4529 4530 case Primitive::kPrimLong: 4531 if (is_volatile && !atomic_ldrd_strd) { 4532 GenerateWideAtomicLoad(base, offset, 4533 out.AsRegisterPairLow<Register>(), 4534 out.AsRegisterPairHigh<Register>()); 4535 } else { 4536 __ LoadFromOffset(kLoadWordPair, out.AsRegisterPairLow<Register>(), base, offset); 4537 } 4538 break; 4539 4540 case Primitive::kPrimFloat: 4541 __ LoadSFromOffset(out.AsFpuRegister<SRegister>(), base, offset); 4542 break; 4543 4544 case Primitive::kPrimDouble: { 4545 DRegister out_reg = FromLowSToD(out.AsFpuRegisterPairLow<SRegister>()); 4546 if (is_volatile && !atomic_ldrd_strd) { 4547 Register lo = locations->GetTemp(0).AsRegister<Register>(); 4548 Register hi = locations->GetTemp(1).AsRegister<Register>(); 4549 GenerateWideAtomicLoad(base, offset, lo, hi); 4550 codegen_->MaybeRecordImplicitNullCheck(instruction); 4551 __ vmovdrr(out_reg, lo, hi); 4552 } else { 4553 __ LoadDFromOffset(out_reg, base, offset); 4554 codegen_->MaybeRecordImplicitNullCheck(instruction); 4555 } 4556 break; 4557 } 4558 4559 case Primitive::kPrimVoid: 4560 LOG(FATAL) << "Unreachable type " << field_type; 4561 UNREACHABLE(); 4562 } 4563 4564 if (field_type == Primitive::kPrimNot || field_type == Primitive::kPrimDouble) { 4565 // Potential implicit null checks, in the case of reference or 4566 // double fields, are handled in the previous switch statement. 4567 } else { 4568 codegen_->MaybeRecordImplicitNullCheck(instruction); 4569 } 4570 4571 if (is_volatile) { 4572 if (field_type == Primitive::kPrimNot) { 4573 // Memory barriers, in the case of references, are also handled 4574 // in the previous switch statement. 4575 } else { 4576 codegen_->GenerateMemoryBarrier(MemBarrierKind::kLoadAny); 4577 } 4578 } 4579} 4580 4581void LocationsBuilderARM::VisitInstanceFieldSet(HInstanceFieldSet* instruction) { 4582 HandleFieldSet(instruction, instruction->GetFieldInfo()); 4583} 4584 4585void InstructionCodeGeneratorARM::VisitInstanceFieldSet(HInstanceFieldSet* instruction) { 4586 HandleFieldSet(instruction, instruction->GetFieldInfo(), instruction->GetValueCanBeNull()); 4587} 4588 4589void LocationsBuilderARM::VisitInstanceFieldGet(HInstanceFieldGet* instruction) { 4590 HandleFieldGet(instruction, instruction->GetFieldInfo()); 4591} 4592 4593void InstructionCodeGeneratorARM::VisitInstanceFieldGet(HInstanceFieldGet* instruction) { 4594 HandleFieldGet(instruction, instruction->GetFieldInfo()); 4595} 4596 4597void LocationsBuilderARM::VisitStaticFieldGet(HStaticFieldGet* instruction) { 4598 HandleFieldGet(instruction, instruction->GetFieldInfo()); 4599} 4600 4601void InstructionCodeGeneratorARM::VisitStaticFieldGet(HStaticFieldGet* instruction) { 4602 HandleFieldGet(instruction, instruction->GetFieldInfo()); 4603} 4604 4605void LocationsBuilderARM::VisitStaticFieldSet(HStaticFieldSet* instruction) { 4606 HandleFieldSet(instruction, instruction->GetFieldInfo()); 4607} 4608 4609void InstructionCodeGeneratorARM::VisitStaticFieldSet(HStaticFieldSet* instruction) { 4610 HandleFieldSet(instruction, instruction->GetFieldInfo(), instruction->GetValueCanBeNull()); 4611} 4612 4613void LocationsBuilderARM::VisitUnresolvedInstanceFieldGet( 4614 HUnresolvedInstanceFieldGet* instruction) { 4615 FieldAccessCallingConventionARM calling_convention; 4616 codegen_->CreateUnresolvedFieldLocationSummary( 4617 instruction, instruction->GetFieldType(), calling_convention); 4618} 4619 4620void InstructionCodeGeneratorARM::VisitUnresolvedInstanceFieldGet( 4621 HUnresolvedInstanceFieldGet* instruction) { 4622 FieldAccessCallingConventionARM calling_convention; 4623 codegen_->GenerateUnresolvedFieldAccess(instruction, 4624 instruction->GetFieldType(), 4625 instruction->GetFieldIndex(), 4626 instruction->GetDexPc(), 4627 calling_convention); 4628} 4629 4630void LocationsBuilderARM::VisitUnresolvedInstanceFieldSet( 4631 HUnresolvedInstanceFieldSet* instruction) { 4632 FieldAccessCallingConventionARM calling_convention; 4633 codegen_->CreateUnresolvedFieldLocationSummary( 4634 instruction, instruction->GetFieldType(), calling_convention); 4635} 4636 4637void InstructionCodeGeneratorARM::VisitUnresolvedInstanceFieldSet( 4638 HUnresolvedInstanceFieldSet* instruction) { 4639 FieldAccessCallingConventionARM calling_convention; 4640 codegen_->GenerateUnresolvedFieldAccess(instruction, 4641 instruction->GetFieldType(), 4642 instruction->GetFieldIndex(), 4643 instruction->GetDexPc(), 4644 calling_convention); 4645} 4646 4647void LocationsBuilderARM::VisitUnresolvedStaticFieldGet( 4648 HUnresolvedStaticFieldGet* instruction) { 4649 FieldAccessCallingConventionARM calling_convention; 4650 codegen_->CreateUnresolvedFieldLocationSummary( 4651 instruction, instruction->GetFieldType(), calling_convention); 4652} 4653 4654void InstructionCodeGeneratorARM::VisitUnresolvedStaticFieldGet( 4655 HUnresolvedStaticFieldGet* instruction) { 4656 FieldAccessCallingConventionARM calling_convention; 4657 codegen_->GenerateUnresolvedFieldAccess(instruction, 4658 instruction->GetFieldType(), 4659 instruction->GetFieldIndex(), 4660 instruction->GetDexPc(), 4661 calling_convention); 4662} 4663 4664void LocationsBuilderARM::VisitUnresolvedStaticFieldSet( 4665 HUnresolvedStaticFieldSet* instruction) { 4666 FieldAccessCallingConventionARM calling_convention; 4667 codegen_->CreateUnresolvedFieldLocationSummary( 4668 instruction, instruction->GetFieldType(), calling_convention); 4669} 4670 4671void InstructionCodeGeneratorARM::VisitUnresolvedStaticFieldSet( 4672 HUnresolvedStaticFieldSet* instruction) { 4673 FieldAccessCallingConventionARM calling_convention; 4674 codegen_->GenerateUnresolvedFieldAccess(instruction, 4675 instruction->GetFieldType(), 4676 instruction->GetFieldIndex(), 4677 instruction->GetDexPc(), 4678 calling_convention); 4679} 4680 4681void LocationsBuilderARM::VisitNullCheck(HNullCheck* instruction) { 4682 LocationSummary* locations = codegen_->CreateThrowingSlowPathLocations(instruction); 4683 locations->SetInAt(0, Location::RequiresRegister()); 4684} 4685 4686void CodeGeneratorARM::GenerateImplicitNullCheck(HNullCheck* instruction) { 4687 if (CanMoveNullCheckToUser(instruction)) { 4688 return; 4689 } 4690 Location obj = instruction->GetLocations()->InAt(0); 4691 4692 __ LoadFromOffset(kLoadWord, IP, obj.AsRegister<Register>(), 0); 4693 RecordPcInfo(instruction, instruction->GetDexPc()); 4694} 4695 4696void CodeGeneratorARM::GenerateExplicitNullCheck(HNullCheck* instruction) { 4697 SlowPathCodeARM* slow_path = new (GetGraph()->GetArena()) NullCheckSlowPathARM(instruction); 4698 AddSlowPath(slow_path); 4699 4700 LocationSummary* locations = instruction->GetLocations(); 4701 Location obj = locations->InAt(0); 4702 4703 __ CompareAndBranchIfZero(obj.AsRegister<Register>(), slow_path->GetEntryLabel()); 4704} 4705 4706void InstructionCodeGeneratorARM::VisitNullCheck(HNullCheck* instruction) { 4707 codegen_->GenerateNullCheck(instruction); 4708} 4709 4710static LoadOperandType GetLoadOperandType(Primitive::Type type) { 4711 switch (type) { 4712 case Primitive::kPrimNot: 4713 return kLoadWord; 4714 case Primitive::kPrimBoolean: 4715 return kLoadUnsignedByte; 4716 case Primitive::kPrimByte: 4717 return kLoadSignedByte; 4718 case Primitive::kPrimChar: 4719 return kLoadUnsignedHalfword; 4720 case Primitive::kPrimShort: 4721 return kLoadSignedHalfword; 4722 case Primitive::kPrimInt: 4723 return kLoadWord; 4724 case Primitive::kPrimLong: 4725 return kLoadWordPair; 4726 case Primitive::kPrimFloat: 4727 return kLoadSWord; 4728 case Primitive::kPrimDouble: 4729 return kLoadDWord; 4730 default: 4731 LOG(FATAL) << "Unreachable type " << type; 4732 UNREACHABLE(); 4733 } 4734} 4735 4736static StoreOperandType GetStoreOperandType(Primitive::Type type) { 4737 switch (type) { 4738 case Primitive::kPrimNot: 4739 return kStoreWord; 4740 case Primitive::kPrimBoolean: 4741 case Primitive::kPrimByte: 4742 return kStoreByte; 4743 case Primitive::kPrimChar: 4744 case Primitive::kPrimShort: 4745 return kStoreHalfword; 4746 case Primitive::kPrimInt: 4747 return kStoreWord; 4748 case Primitive::kPrimLong: 4749 return kStoreWordPair; 4750 case Primitive::kPrimFloat: 4751 return kStoreSWord; 4752 case Primitive::kPrimDouble: 4753 return kStoreDWord; 4754 default: 4755 LOG(FATAL) << "Unreachable type " << type; 4756 UNREACHABLE(); 4757 } 4758} 4759 4760void CodeGeneratorARM::LoadFromShiftedRegOffset(Primitive::Type type, 4761 Location out_loc, 4762 Register base, 4763 Register reg_offset, 4764 Condition cond) { 4765 uint32_t shift_count = Primitive::ComponentSizeShift(type); 4766 Address mem_address(base, reg_offset, Shift::LSL, shift_count); 4767 4768 switch (type) { 4769 case Primitive::kPrimByte: 4770 __ ldrsb(out_loc.AsRegister<Register>(), mem_address, cond); 4771 break; 4772 case Primitive::kPrimBoolean: 4773 __ ldrb(out_loc.AsRegister<Register>(), mem_address, cond); 4774 break; 4775 case Primitive::kPrimShort: 4776 __ ldrsh(out_loc.AsRegister<Register>(), mem_address, cond); 4777 break; 4778 case Primitive::kPrimChar: 4779 __ ldrh(out_loc.AsRegister<Register>(), mem_address, cond); 4780 break; 4781 case Primitive::kPrimNot: 4782 case Primitive::kPrimInt: 4783 __ ldr(out_loc.AsRegister<Register>(), mem_address, cond); 4784 break; 4785 // T32 doesn't support LoadFromShiftedRegOffset mem address mode for these types. 4786 case Primitive::kPrimLong: 4787 case Primitive::kPrimFloat: 4788 case Primitive::kPrimDouble: 4789 default: 4790 LOG(FATAL) << "Unreachable type " << type; 4791 UNREACHABLE(); 4792 } 4793} 4794 4795void CodeGeneratorARM::StoreToShiftedRegOffset(Primitive::Type type, 4796 Location loc, 4797 Register base, 4798 Register reg_offset, 4799 Condition cond) { 4800 uint32_t shift_count = Primitive::ComponentSizeShift(type); 4801 Address mem_address(base, reg_offset, Shift::LSL, shift_count); 4802 4803 switch (type) { 4804 case Primitive::kPrimByte: 4805 case Primitive::kPrimBoolean: 4806 __ strb(loc.AsRegister<Register>(), mem_address, cond); 4807 break; 4808 case Primitive::kPrimShort: 4809 case Primitive::kPrimChar: 4810 __ strh(loc.AsRegister<Register>(), mem_address, cond); 4811 break; 4812 case Primitive::kPrimNot: 4813 case Primitive::kPrimInt: 4814 __ str(loc.AsRegister<Register>(), mem_address, cond); 4815 break; 4816 // T32 doesn't support StoreToShiftedRegOffset mem address mode for these types. 4817 case Primitive::kPrimLong: 4818 case Primitive::kPrimFloat: 4819 case Primitive::kPrimDouble: 4820 default: 4821 LOG(FATAL) << "Unreachable type " << type; 4822 UNREACHABLE(); 4823 } 4824} 4825 4826void LocationsBuilderARM::VisitArrayGet(HArrayGet* instruction) { 4827 bool object_array_get_with_read_barrier = 4828 kEmitCompilerReadBarrier && (instruction->GetType() == Primitive::kPrimNot); 4829 LocationSummary* locations = 4830 new (GetGraph()->GetArena()) LocationSummary(instruction, 4831 object_array_get_with_read_barrier ? 4832 LocationSummary::kCallOnSlowPath : 4833 LocationSummary::kNoCall); 4834 if (object_array_get_with_read_barrier && kUseBakerReadBarrier) { 4835 locations->SetCustomSlowPathCallerSaves(RegisterSet::Empty()); // No caller-save registers. 4836 } 4837 locations->SetInAt(0, Location::RequiresRegister()); 4838 locations->SetInAt(1, Location::RegisterOrConstant(instruction->InputAt(1))); 4839 if (Primitive::IsFloatingPointType(instruction->GetType())) { 4840 locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap); 4841 } else { 4842 // The output overlaps in the case of an object array get with 4843 // read barriers enabled: we do not want the move to overwrite the 4844 // array's location, as we need it to emit the read barrier. 4845 locations->SetOut( 4846 Location::RequiresRegister(), 4847 object_array_get_with_read_barrier ? Location::kOutputOverlap : Location::kNoOutputOverlap); 4848 } 4849 // We need a temporary register for the read barrier marking slow 4850 // path in CodeGeneratorARM::GenerateArrayLoadWithBakerReadBarrier. 4851 // Also need for String compression feature. 4852 if ((object_array_get_with_read_barrier && kUseBakerReadBarrier) 4853 || (mirror::kUseStringCompression && instruction->IsStringCharAt())) { 4854 locations->AddTemp(Location::RequiresRegister()); 4855 } 4856} 4857 4858void InstructionCodeGeneratorARM::VisitArrayGet(HArrayGet* instruction) { 4859 LocationSummary* locations = instruction->GetLocations(); 4860 Location obj_loc = locations->InAt(0); 4861 Register obj = obj_loc.AsRegister<Register>(); 4862 Location index = locations->InAt(1); 4863 Location out_loc = locations->Out(); 4864 uint32_t data_offset = CodeGenerator::GetArrayDataOffset(instruction); 4865 Primitive::Type type = instruction->GetType(); 4866 const bool maybe_compressed_char_at = mirror::kUseStringCompression && 4867 instruction->IsStringCharAt(); 4868 HInstruction* array_instr = instruction->GetArray(); 4869 bool has_intermediate_address = array_instr->IsIntermediateAddress(); 4870 4871 switch (type) { 4872 case Primitive::kPrimBoolean: 4873 case Primitive::kPrimByte: 4874 case Primitive::kPrimShort: 4875 case Primitive::kPrimChar: 4876 case Primitive::kPrimInt: { 4877 Register length; 4878 if (maybe_compressed_char_at) { 4879 length = locations->GetTemp(0).AsRegister<Register>(); 4880 uint32_t count_offset = mirror::String::CountOffset().Uint32Value(); 4881 __ LoadFromOffset(kLoadWord, length, obj, count_offset); 4882 codegen_->MaybeRecordImplicitNullCheck(instruction); 4883 } 4884 if (index.IsConstant()) { 4885 int32_t const_index = index.GetConstant()->AsIntConstant()->GetValue(); 4886 if (maybe_compressed_char_at) { 4887 Label uncompressed_load, done; 4888 __ Lsrs(length, length, 1u); // LSRS has a 16-bit encoding, TST (immediate) does not. 4889 static_assert(static_cast<uint32_t>(mirror::StringCompressionFlag::kCompressed) == 0u, 4890 "Expecting 0=compressed, 1=uncompressed"); 4891 __ b(&uncompressed_load, CS); 4892 __ LoadFromOffset(kLoadUnsignedByte, 4893 out_loc.AsRegister<Register>(), 4894 obj, 4895 data_offset + const_index); 4896 __ b(&done); 4897 __ Bind(&uncompressed_load); 4898 __ LoadFromOffset(GetLoadOperandType(Primitive::kPrimChar), 4899 out_loc.AsRegister<Register>(), 4900 obj, 4901 data_offset + (const_index << 1)); 4902 __ Bind(&done); 4903 } else { 4904 uint32_t full_offset = data_offset + (const_index << Primitive::ComponentSizeShift(type)); 4905 4906 LoadOperandType load_type = GetLoadOperandType(type); 4907 __ LoadFromOffset(load_type, out_loc.AsRegister<Register>(), obj, full_offset); 4908 } 4909 } else { 4910 Register temp = IP; 4911 4912 if (has_intermediate_address) { 4913 // We do not need to compute the intermediate address from the array: the 4914 // input instruction has done it already. See the comment in 4915 // `TryExtractArrayAccessAddress()`. 4916 if (kIsDebugBuild) { 4917 HIntermediateAddress* tmp = array_instr->AsIntermediateAddress(); 4918 DCHECK_EQ(tmp->GetOffset()->AsIntConstant()->GetValueAsUint64(), data_offset); 4919 } 4920 temp = obj; 4921 } else { 4922 __ add(temp, obj, ShifterOperand(data_offset)); 4923 } 4924 if (maybe_compressed_char_at) { 4925 Label uncompressed_load, done; 4926 __ Lsrs(length, length, 1u); // LSRS has a 16-bit encoding, TST (immediate) does not. 4927 static_assert(static_cast<uint32_t>(mirror::StringCompressionFlag::kCompressed) == 0u, 4928 "Expecting 0=compressed, 1=uncompressed"); 4929 __ b(&uncompressed_load, CS); 4930 __ ldrb(out_loc.AsRegister<Register>(), 4931 Address(temp, index.AsRegister<Register>(), Shift::LSL, 0)); 4932 __ b(&done); 4933 __ Bind(&uncompressed_load); 4934 __ ldrh(out_loc.AsRegister<Register>(), 4935 Address(temp, index.AsRegister<Register>(), Shift::LSL, 1)); 4936 __ Bind(&done); 4937 } else { 4938 codegen_->LoadFromShiftedRegOffset(type, out_loc, temp, index.AsRegister<Register>()); 4939 } 4940 } 4941 break; 4942 } 4943 4944 case Primitive::kPrimNot: { 4945 // The read barrier instrumentation of object ArrayGet 4946 // instructions does not support the HIntermediateAddress 4947 // instruction. 4948 DCHECK(!(has_intermediate_address && kEmitCompilerReadBarrier)); 4949 4950 static_assert( 4951 sizeof(mirror::HeapReference<mirror::Object>) == sizeof(int32_t), 4952 "art::mirror::HeapReference<art::mirror::Object> and int32_t have different sizes."); 4953 // /* HeapReference<Object> */ out = 4954 // *(obj + data_offset + index * sizeof(HeapReference<Object>)) 4955 if (kEmitCompilerReadBarrier && kUseBakerReadBarrier) { 4956 Location temp = locations->GetTemp(0); 4957 // Note that a potential implicit null check is handled in this 4958 // CodeGeneratorARM::GenerateArrayLoadWithBakerReadBarrier call. 4959 codegen_->GenerateArrayLoadWithBakerReadBarrier( 4960 instruction, out_loc, obj, data_offset, index, temp, /* needs_null_check */ true); 4961 } else { 4962 Register out = out_loc.AsRegister<Register>(); 4963 if (index.IsConstant()) { 4964 size_t offset = 4965 (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4) + data_offset; 4966 __ LoadFromOffset(kLoadWord, out, obj, offset); 4967 codegen_->MaybeRecordImplicitNullCheck(instruction); 4968 // If read barriers are enabled, emit read barriers other than 4969 // Baker's using a slow path (and also unpoison the loaded 4970 // reference, if heap poisoning is enabled). 4971 codegen_->MaybeGenerateReadBarrierSlow(instruction, out_loc, out_loc, obj_loc, offset); 4972 } else { 4973 Register temp = IP; 4974 4975 if (has_intermediate_address) { 4976 // We do not need to compute the intermediate address from the array: the 4977 // input instruction has done it already. See the comment in 4978 // `TryExtractArrayAccessAddress()`. 4979 if (kIsDebugBuild) { 4980 HIntermediateAddress* tmp = array_instr->AsIntermediateAddress(); 4981 DCHECK_EQ(tmp->GetOffset()->AsIntConstant()->GetValueAsUint64(), data_offset); 4982 } 4983 temp = obj; 4984 } else { 4985 __ add(temp, obj, ShifterOperand(data_offset)); 4986 } 4987 codegen_->LoadFromShiftedRegOffset(type, out_loc, temp, index.AsRegister<Register>()); 4988 4989 codegen_->MaybeRecordImplicitNullCheck(instruction); 4990 // If read barriers are enabled, emit read barriers other than 4991 // Baker's using a slow path (and also unpoison the loaded 4992 // reference, if heap poisoning is enabled). 4993 codegen_->MaybeGenerateReadBarrierSlow( 4994 instruction, out_loc, out_loc, obj_loc, data_offset, index); 4995 } 4996 } 4997 break; 4998 } 4999 5000 case Primitive::kPrimLong: { 5001 if (index.IsConstant()) { 5002 size_t offset = 5003 (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_8) + data_offset; 5004 __ LoadFromOffset(kLoadWordPair, out_loc.AsRegisterPairLow<Register>(), obj, offset); 5005 } else { 5006 __ add(IP, obj, ShifterOperand(index.AsRegister<Register>(), LSL, TIMES_8)); 5007 __ LoadFromOffset(kLoadWordPair, out_loc.AsRegisterPairLow<Register>(), IP, data_offset); 5008 } 5009 break; 5010 } 5011 5012 case Primitive::kPrimFloat: { 5013 SRegister out = out_loc.AsFpuRegister<SRegister>(); 5014 if (index.IsConstant()) { 5015 size_t offset = (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4) + data_offset; 5016 __ LoadSFromOffset(out, obj, offset); 5017 } else { 5018 __ add(IP, obj, ShifterOperand(index.AsRegister<Register>(), LSL, TIMES_4)); 5019 __ LoadSFromOffset(out, IP, data_offset); 5020 } 5021 break; 5022 } 5023 5024 case Primitive::kPrimDouble: { 5025 SRegister out = out_loc.AsFpuRegisterPairLow<SRegister>(); 5026 if (index.IsConstant()) { 5027 size_t offset = (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_8) + data_offset; 5028 __ LoadDFromOffset(FromLowSToD(out), obj, offset); 5029 } else { 5030 __ add(IP, obj, ShifterOperand(index.AsRegister<Register>(), LSL, TIMES_8)); 5031 __ LoadDFromOffset(FromLowSToD(out), IP, data_offset); 5032 } 5033 break; 5034 } 5035 5036 case Primitive::kPrimVoid: 5037 LOG(FATAL) << "Unreachable type " << type; 5038 UNREACHABLE(); 5039 } 5040 5041 if (type == Primitive::kPrimNot) { 5042 // Potential implicit null checks, in the case of reference 5043 // arrays, are handled in the previous switch statement. 5044 } else if (!maybe_compressed_char_at) { 5045 codegen_->MaybeRecordImplicitNullCheck(instruction); 5046 } 5047} 5048 5049void LocationsBuilderARM::VisitArraySet(HArraySet* instruction) { 5050 Primitive::Type value_type = instruction->GetComponentType(); 5051 5052 bool needs_write_barrier = 5053 CodeGenerator::StoreNeedsWriteBarrier(value_type, instruction->GetValue()); 5054 bool may_need_runtime_call_for_type_check = instruction->NeedsTypeCheck(); 5055 5056 LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary( 5057 instruction, 5058 may_need_runtime_call_for_type_check ? 5059 LocationSummary::kCallOnSlowPath : 5060 LocationSummary::kNoCall); 5061 5062 locations->SetInAt(0, Location::RequiresRegister()); 5063 locations->SetInAt(1, Location::RegisterOrConstant(instruction->InputAt(1))); 5064 if (Primitive::IsFloatingPointType(value_type)) { 5065 locations->SetInAt(2, Location::RequiresFpuRegister()); 5066 } else { 5067 locations->SetInAt(2, Location::RequiresRegister()); 5068 } 5069 if (needs_write_barrier) { 5070 // Temporary registers for the write barrier. 5071 locations->AddTemp(Location::RequiresRegister()); // Possibly used for ref. poisoning too. 5072 locations->AddTemp(Location::RequiresRegister()); 5073 } 5074} 5075 5076void InstructionCodeGeneratorARM::VisitArraySet(HArraySet* instruction) { 5077 LocationSummary* locations = instruction->GetLocations(); 5078 Location array_loc = locations->InAt(0); 5079 Register array = array_loc.AsRegister<Register>(); 5080 Location index = locations->InAt(1); 5081 Primitive::Type value_type = instruction->GetComponentType(); 5082 bool may_need_runtime_call_for_type_check = instruction->NeedsTypeCheck(); 5083 bool needs_write_barrier = 5084 CodeGenerator::StoreNeedsWriteBarrier(value_type, instruction->GetValue()); 5085 uint32_t data_offset = 5086 mirror::Array::DataOffset(Primitive::ComponentSize(value_type)).Uint32Value(); 5087 Location value_loc = locations->InAt(2); 5088 HInstruction* array_instr = instruction->GetArray(); 5089 bool has_intermediate_address = array_instr->IsIntermediateAddress(); 5090 5091 switch (value_type) { 5092 case Primitive::kPrimBoolean: 5093 case Primitive::kPrimByte: 5094 case Primitive::kPrimShort: 5095 case Primitive::kPrimChar: 5096 case Primitive::kPrimInt: { 5097 if (index.IsConstant()) { 5098 int32_t const_index = index.GetConstant()->AsIntConstant()->GetValue(); 5099 uint32_t full_offset = 5100 data_offset + (const_index << Primitive::ComponentSizeShift(value_type)); 5101 StoreOperandType store_type = GetStoreOperandType(value_type); 5102 __ StoreToOffset(store_type, value_loc.AsRegister<Register>(), array, full_offset); 5103 } else { 5104 Register temp = IP; 5105 5106 if (has_intermediate_address) { 5107 // We do not need to compute the intermediate address from the array: the 5108 // input instruction has done it already. See the comment in 5109 // `TryExtractArrayAccessAddress()`. 5110 if (kIsDebugBuild) { 5111 HIntermediateAddress* tmp = array_instr->AsIntermediateAddress(); 5112 DCHECK(tmp->GetOffset()->AsIntConstant()->GetValueAsUint64() == data_offset); 5113 } 5114 temp = array; 5115 } else { 5116 __ add(temp, array, ShifterOperand(data_offset)); 5117 } 5118 codegen_->StoreToShiftedRegOffset(value_type, 5119 value_loc, 5120 temp, 5121 index.AsRegister<Register>()); 5122 } 5123 break; 5124 } 5125 5126 case Primitive::kPrimNot: { 5127 Register value = value_loc.AsRegister<Register>(); 5128 // TryExtractArrayAccessAddress optimization is never applied for non-primitive ArraySet. 5129 // See the comment in instruction_simplifier_shared.cc. 5130 DCHECK(!has_intermediate_address); 5131 5132 if (instruction->InputAt(2)->IsNullConstant()) { 5133 // Just setting null. 5134 if (index.IsConstant()) { 5135 size_t offset = 5136 (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4) + data_offset; 5137 __ StoreToOffset(kStoreWord, value, array, offset); 5138 } else { 5139 DCHECK(index.IsRegister()) << index; 5140 __ add(IP, array, ShifterOperand(data_offset)); 5141 codegen_->StoreToShiftedRegOffset(value_type, 5142 value_loc, 5143 IP, 5144 index.AsRegister<Register>()); 5145 } 5146 codegen_->MaybeRecordImplicitNullCheck(instruction); 5147 DCHECK(!needs_write_barrier); 5148 DCHECK(!may_need_runtime_call_for_type_check); 5149 break; 5150 } 5151 5152 DCHECK(needs_write_barrier); 5153 Location temp1_loc = locations->GetTemp(0); 5154 Register temp1 = temp1_loc.AsRegister<Register>(); 5155 Location temp2_loc = locations->GetTemp(1); 5156 Register temp2 = temp2_loc.AsRegister<Register>(); 5157 uint32_t class_offset = mirror::Object::ClassOffset().Int32Value(); 5158 uint32_t super_offset = mirror::Class::SuperClassOffset().Int32Value(); 5159 uint32_t component_offset = mirror::Class::ComponentTypeOffset().Int32Value(); 5160 Label done; 5161 SlowPathCodeARM* slow_path = nullptr; 5162 5163 if (may_need_runtime_call_for_type_check) { 5164 slow_path = new (GetGraph()->GetArena()) ArraySetSlowPathARM(instruction); 5165 codegen_->AddSlowPath(slow_path); 5166 if (instruction->GetValueCanBeNull()) { 5167 Label non_zero; 5168 __ CompareAndBranchIfNonZero(value, &non_zero); 5169 if (index.IsConstant()) { 5170 size_t offset = 5171 (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4) + data_offset; 5172 __ StoreToOffset(kStoreWord, value, array, offset); 5173 } else { 5174 DCHECK(index.IsRegister()) << index; 5175 __ add(IP, array, ShifterOperand(data_offset)); 5176 codegen_->StoreToShiftedRegOffset(value_type, 5177 value_loc, 5178 IP, 5179 index.AsRegister<Register>()); 5180 } 5181 codegen_->MaybeRecordImplicitNullCheck(instruction); 5182 __ b(&done); 5183 __ Bind(&non_zero); 5184 } 5185 5186 // Note that when read barriers are enabled, the type checks 5187 // are performed without read barriers. This is fine, even in 5188 // the case where a class object is in the from-space after 5189 // the flip, as a comparison involving such a type would not 5190 // produce a false positive; it may of course produce a false 5191 // negative, in which case we would take the ArraySet slow 5192 // path. 5193 5194 // /* HeapReference<Class> */ temp1 = array->klass_ 5195 __ LoadFromOffset(kLoadWord, temp1, array, class_offset); 5196 codegen_->MaybeRecordImplicitNullCheck(instruction); 5197 __ MaybeUnpoisonHeapReference(temp1); 5198 5199 // /* HeapReference<Class> */ temp1 = temp1->component_type_ 5200 __ LoadFromOffset(kLoadWord, temp1, temp1, component_offset); 5201 // /* HeapReference<Class> */ temp2 = value->klass_ 5202 __ LoadFromOffset(kLoadWord, temp2, value, class_offset); 5203 // If heap poisoning is enabled, no need to unpoison `temp1` 5204 // nor `temp2`, as we are comparing two poisoned references. 5205 __ cmp(temp1, ShifterOperand(temp2)); 5206 5207 if (instruction->StaticTypeOfArrayIsObjectArray()) { 5208 Label do_put; 5209 __ b(&do_put, EQ); 5210 // If heap poisoning is enabled, the `temp1` reference has 5211 // not been unpoisoned yet; unpoison it now. 5212 __ MaybeUnpoisonHeapReference(temp1); 5213 5214 // /* HeapReference<Class> */ temp1 = temp1->super_class_ 5215 __ LoadFromOffset(kLoadWord, temp1, temp1, super_offset); 5216 // If heap poisoning is enabled, no need to unpoison 5217 // `temp1`, as we are comparing against null below. 5218 __ CompareAndBranchIfNonZero(temp1, slow_path->GetEntryLabel()); 5219 __ Bind(&do_put); 5220 } else { 5221 __ b(slow_path->GetEntryLabel(), NE); 5222 } 5223 } 5224 5225 Register source = value; 5226 if (kPoisonHeapReferences) { 5227 // Note that in the case where `value` is a null reference, 5228 // we do not enter this block, as a null reference does not 5229 // need poisoning. 5230 DCHECK_EQ(value_type, Primitive::kPrimNot); 5231 __ Mov(temp1, value); 5232 __ PoisonHeapReference(temp1); 5233 source = temp1; 5234 } 5235 5236 if (index.IsConstant()) { 5237 size_t offset = 5238 (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4) + data_offset; 5239 __ StoreToOffset(kStoreWord, source, array, offset); 5240 } else { 5241 DCHECK(index.IsRegister()) << index; 5242 5243 __ add(IP, array, ShifterOperand(data_offset)); 5244 codegen_->StoreToShiftedRegOffset(value_type, 5245 Location::RegisterLocation(source), 5246 IP, 5247 index.AsRegister<Register>()); 5248 } 5249 5250 if (!may_need_runtime_call_for_type_check) { 5251 codegen_->MaybeRecordImplicitNullCheck(instruction); 5252 } 5253 5254 codegen_->MarkGCCard(temp1, temp2, array, value, instruction->GetValueCanBeNull()); 5255 5256 if (done.IsLinked()) { 5257 __ Bind(&done); 5258 } 5259 5260 if (slow_path != nullptr) { 5261 __ Bind(slow_path->GetExitLabel()); 5262 } 5263 5264 break; 5265 } 5266 5267 case Primitive::kPrimLong: { 5268 Location value = locations->InAt(2); 5269 if (index.IsConstant()) { 5270 size_t offset = 5271 (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_8) + data_offset; 5272 __ StoreToOffset(kStoreWordPair, value.AsRegisterPairLow<Register>(), array, offset); 5273 } else { 5274 __ add(IP, array, ShifterOperand(index.AsRegister<Register>(), LSL, TIMES_8)); 5275 __ StoreToOffset(kStoreWordPair, value.AsRegisterPairLow<Register>(), IP, data_offset); 5276 } 5277 break; 5278 } 5279 5280 case Primitive::kPrimFloat: { 5281 Location value = locations->InAt(2); 5282 DCHECK(value.IsFpuRegister()); 5283 if (index.IsConstant()) { 5284 size_t offset = (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4) + data_offset; 5285 __ StoreSToOffset(value.AsFpuRegister<SRegister>(), array, offset); 5286 } else { 5287 __ add(IP, array, ShifterOperand(index.AsRegister<Register>(), LSL, TIMES_4)); 5288 __ StoreSToOffset(value.AsFpuRegister<SRegister>(), IP, data_offset); 5289 } 5290 break; 5291 } 5292 5293 case Primitive::kPrimDouble: { 5294 Location value = locations->InAt(2); 5295 DCHECK(value.IsFpuRegisterPair()); 5296 if (index.IsConstant()) { 5297 size_t offset = (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_8) + data_offset; 5298 __ StoreDToOffset(FromLowSToD(value.AsFpuRegisterPairLow<SRegister>()), array, offset); 5299 } else { 5300 __ add(IP, array, ShifterOperand(index.AsRegister<Register>(), LSL, TIMES_8)); 5301 __ StoreDToOffset(FromLowSToD(value.AsFpuRegisterPairLow<SRegister>()), IP, data_offset); 5302 } 5303 5304 break; 5305 } 5306 5307 case Primitive::kPrimVoid: 5308 LOG(FATAL) << "Unreachable type " << value_type; 5309 UNREACHABLE(); 5310 } 5311 5312 // Objects are handled in the switch. 5313 if (value_type != Primitive::kPrimNot) { 5314 codegen_->MaybeRecordImplicitNullCheck(instruction); 5315 } 5316} 5317 5318void LocationsBuilderARM::VisitArrayLength(HArrayLength* instruction) { 5319 LocationSummary* locations = 5320 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall); 5321 locations->SetInAt(0, Location::RequiresRegister()); 5322 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 5323} 5324 5325void InstructionCodeGeneratorARM::VisitArrayLength(HArrayLength* instruction) { 5326 LocationSummary* locations = instruction->GetLocations(); 5327 uint32_t offset = CodeGenerator::GetArrayLengthOffset(instruction); 5328 Register obj = locations->InAt(0).AsRegister<Register>(); 5329 Register out = locations->Out().AsRegister<Register>(); 5330 __ LoadFromOffset(kLoadWord, out, obj, offset); 5331 codegen_->MaybeRecordImplicitNullCheck(instruction); 5332 // Mask out compression flag from String's array length. 5333 if (mirror::kUseStringCompression && instruction->IsStringLength()) { 5334 __ Lsr(out, out, 1u); 5335 } 5336} 5337 5338void LocationsBuilderARM::VisitIntermediateAddress(HIntermediateAddress* instruction) { 5339 LocationSummary* locations = 5340 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall); 5341 5342 locations->SetInAt(0, Location::RequiresRegister()); 5343 locations->SetInAt(1, Location::RegisterOrConstant(instruction->GetOffset())); 5344 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 5345} 5346 5347void InstructionCodeGeneratorARM::VisitIntermediateAddress(HIntermediateAddress* instruction) { 5348 LocationSummary* locations = instruction->GetLocations(); 5349 Location out = locations->Out(); 5350 Location first = locations->InAt(0); 5351 Location second = locations->InAt(1); 5352 5353 if (second.IsRegister()) { 5354 __ add(out.AsRegister<Register>(), 5355 first.AsRegister<Register>(), 5356 ShifterOperand(second.AsRegister<Register>())); 5357 } else { 5358 __ AddConstant(out.AsRegister<Register>(), 5359 first.AsRegister<Register>(), 5360 second.GetConstant()->AsIntConstant()->GetValue()); 5361 } 5362} 5363 5364void LocationsBuilderARM::VisitBoundsCheck(HBoundsCheck* instruction) { 5365 RegisterSet caller_saves = RegisterSet::Empty(); 5366 InvokeRuntimeCallingConvention calling_convention; 5367 caller_saves.Add(Location::RegisterLocation(calling_convention.GetRegisterAt(0))); 5368 caller_saves.Add(Location::RegisterLocation(calling_convention.GetRegisterAt(1))); 5369 LocationSummary* locations = codegen_->CreateThrowingSlowPathLocations(instruction, caller_saves); 5370 locations->SetInAt(0, Location::RequiresRegister()); 5371 locations->SetInAt(1, Location::RequiresRegister()); 5372} 5373 5374void InstructionCodeGeneratorARM::VisitBoundsCheck(HBoundsCheck* instruction) { 5375 LocationSummary* locations = instruction->GetLocations(); 5376 SlowPathCodeARM* slow_path = 5377 new (GetGraph()->GetArena()) BoundsCheckSlowPathARM(instruction); 5378 codegen_->AddSlowPath(slow_path); 5379 5380 Register index = locations->InAt(0).AsRegister<Register>(); 5381 Register length = locations->InAt(1).AsRegister<Register>(); 5382 5383 __ cmp(index, ShifterOperand(length)); 5384 __ b(slow_path->GetEntryLabel(), HS); 5385} 5386 5387void CodeGeneratorARM::MarkGCCard(Register temp, 5388 Register card, 5389 Register object, 5390 Register value, 5391 bool can_be_null) { 5392 Label is_null; 5393 if (can_be_null) { 5394 __ CompareAndBranchIfZero(value, &is_null); 5395 } 5396 __ LoadFromOffset(kLoadWord, card, TR, Thread::CardTableOffset<kArmPointerSize>().Int32Value()); 5397 __ Lsr(temp, object, gc::accounting::CardTable::kCardShift); 5398 __ strb(card, Address(card, temp)); 5399 if (can_be_null) { 5400 __ Bind(&is_null); 5401 } 5402} 5403 5404void LocationsBuilderARM::VisitParallelMove(HParallelMove* instruction ATTRIBUTE_UNUSED) { 5405 LOG(FATAL) << "Unreachable"; 5406} 5407 5408void InstructionCodeGeneratorARM::VisitParallelMove(HParallelMove* instruction) { 5409 codegen_->GetMoveResolver()->EmitNativeCode(instruction); 5410} 5411 5412void LocationsBuilderARM::VisitSuspendCheck(HSuspendCheck* instruction) { 5413 LocationSummary* locations = 5414 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCallOnSlowPath); 5415 locations->SetCustomSlowPathCallerSaves(RegisterSet::Empty()); // No caller-save registers. 5416} 5417 5418void InstructionCodeGeneratorARM::VisitSuspendCheck(HSuspendCheck* instruction) { 5419 HBasicBlock* block = instruction->GetBlock(); 5420 if (block->GetLoopInformation() != nullptr) { 5421 DCHECK(block->GetLoopInformation()->GetSuspendCheck() == instruction); 5422 // The back edge will generate the suspend check. 5423 return; 5424 } 5425 if (block->IsEntryBlock() && instruction->GetNext()->IsGoto()) { 5426 // The goto will generate the suspend check. 5427 return; 5428 } 5429 GenerateSuspendCheck(instruction, nullptr); 5430} 5431 5432void InstructionCodeGeneratorARM::GenerateSuspendCheck(HSuspendCheck* instruction, 5433 HBasicBlock* successor) { 5434 SuspendCheckSlowPathARM* slow_path = 5435 down_cast<SuspendCheckSlowPathARM*>(instruction->GetSlowPath()); 5436 if (slow_path == nullptr) { 5437 slow_path = new (GetGraph()->GetArena()) SuspendCheckSlowPathARM(instruction, successor); 5438 instruction->SetSlowPath(slow_path); 5439 codegen_->AddSlowPath(slow_path); 5440 if (successor != nullptr) { 5441 DCHECK(successor->IsLoopHeader()); 5442 codegen_->ClearSpillSlotsFromLoopPhisInStackMap(instruction); 5443 } 5444 } else { 5445 DCHECK_EQ(slow_path->GetSuccessor(), successor); 5446 } 5447 5448 __ LoadFromOffset( 5449 kLoadUnsignedHalfword, IP, TR, Thread::ThreadFlagsOffset<kArmPointerSize>().Int32Value()); 5450 if (successor == nullptr) { 5451 __ CompareAndBranchIfNonZero(IP, slow_path->GetEntryLabel()); 5452 __ Bind(slow_path->GetReturnLabel()); 5453 } else { 5454 __ CompareAndBranchIfZero(IP, codegen_->GetLabelOf(successor)); 5455 __ b(slow_path->GetEntryLabel()); 5456 } 5457} 5458 5459ArmAssembler* ParallelMoveResolverARM::GetAssembler() const { 5460 return codegen_->GetAssembler(); 5461} 5462 5463void ParallelMoveResolverARM::EmitMove(size_t index) { 5464 MoveOperands* move = moves_[index]; 5465 Location source = move->GetSource(); 5466 Location destination = move->GetDestination(); 5467 5468 if (source.IsRegister()) { 5469 if (destination.IsRegister()) { 5470 __ Mov(destination.AsRegister<Register>(), source.AsRegister<Register>()); 5471 } else if (destination.IsFpuRegister()) { 5472 __ vmovsr(destination.AsFpuRegister<SRegister>(), source.AsRegister<Register>()); 5473 } else { 5474 DCHECK(destination.IsStackSlot()); 5475 __ StoreToOffset(kStoreWord, source.AsRegister<Register>(), 5476 SP, destination.GetStackIndex()); 5477 } 5478 } else if (source.IsStackSlot()) { 5479 if (destination.IsRegister()) { 5480 __ LoadFromOffset(kLoadWord, destination.AsRegister<Register>(), 5481 SP, source.GetStackIndex()); 5482 } else if (destination.IsFpuRegister()) { 5483 __ LoadSFromOffset(destination.AsFpuRegister<SRegister>(), SP, source.GetStackIndex()); 5484 } else { 5485 DCHECK(destination.IsStackSlot()); 5486 __ LoadFromOffset(kLoadWord, IP, SP, source.GetStackIndex()); 5487 __ StoreToOffset(kStoreWord, IP, SP, destination.GetStackIndex()); 5488 } 5489 } else if (source.IsFpuRegister()) { 5490 if (destination.IsRegister()) { 5491 __ vmovrs(destination.AsRegister<Register>(), source.AsFpuRegister<SRegister>()); 5492 } else if (destination.IsFpuRegister()) { 5493 __ vmovs(destination.AsFpuRegister<SRegister>(), source.AsFpuRegister<SRegister>()); 5494 } else { 5495 DCHECK(destination.IsStackSlot()); 5496 __ StoreSToOffset(source.AsFpuRegister<SRegister>(), SP, destination.GetStackIndex()); 5497 } 5498 } else if (source.IsDoubleStackSlot()) { 5499 if (destination.IsDoubleStackSlot()) { 5500 __ LoadDFromOffset(DTMP, SP, source.GetStackIndex()); 5501 __ StoreDToOffset(DTMP, SP, destination.GetStackIndex()); 5502 } else if (destination.IsRegisterPair()) { 5503 DCHECK(ExpectedPairLayout(destination)); 5504 __ LoadFromOffset( 5505 kLoadWordPair, destination.AsRegisterPairLow<Register>(), SP, source.GetStackIndex()); 5506 } else { 5507 DCHECK(destination.IsFpuRegisterPair()) << destination; 5508 __ LoadDFromOffset(FromLowSToD(destination.AsFpuRegisterPairLow<SRegister>()), 5509 SP, 5510 source.GetStackIndex()); 5511 } 5512 } else if (source.IsRegisterPair()) { 5513 if (destination.IsRegisterPair()) { 5514 __ Mov(destination.AsRegisterPairLow<Register>(), source.AsRegisterPairLow<Register>()); 5515 __ Mov(destination.AsRegisterPairHigh<Register>(), source.AsRegisterPairHigh<Register>()); 5516 } else if (destination.IsFpuRegisterPair()) { 5517 __ vmovdrr(FromLowSToD(destination.AsFpuRegisterPairLow<SRegister>()), 5518 source.AsRegisterPairLow<Register>(), 5519 source.AsRegisterPairHigh<Register>()); 5520 } else { 5521 DCHECK(destination.IsDoubleStackSlot()) << destination; 5522 DCHECK(ExpectedPairLayout(source)); 5523 __ StoreToOffset( 5524 kStoreWordPair, source.AsRegisterPairLow<Register>(), SP, destination.GetStackIndex()); 5525 } 5526 } else if (source.IsFpuRegisterPair()) { 5527 if (destination.IsRegisterPair()) { 5528 __ vmovrrd(destination.AsRegisterPairLow<Register>(), 5529 destination.AsRegisterPairHigh<Register>(), 5530 FromLowSToD(source.AsFpuRegisterPairLow<SRegister>())); 5531 } else if (destination.IsFpuRegisterPair()) { 5532 __ vmovd(FromLowSToD(destination.AsFpuRegisterPairLow<SRegister>()), 5533 FromLowSToD(source.AsFpuRegisterPairLow<SRegister>())); 5534 } else { 5535 DCHECK(destination.IsDoubleStackSlot()) << destination; 5536 __ StoreDToOffset(FromLowSToD(source.AsFpuRegisterPairLow<SRegister>()), 5537 SP, 5538 destination.GetStackIndex()); 5539 } 5540 } else { 5541 DCHECK(source.IsConstant()) << source; 5542 HConstant* constant = source.GetConstant(); 5543 if (constant->IsIntConstant() || constant->IsNullConstant()) { 5544 int32_t value = CodeGenerator::GetInt32ValueOf(constant); 5545 if (destination.IsRegister()) { 5546 __ LoadImmediate(destination.AsRegister<Register>(), value); 5547 } else { 5548 DCHECK(destination.IsStackSlot()); 5549 __ LoadImmediate(IP, value); 5550 __ StoreToOffset(kStoreWord, IP, SP, destination.GetStackIndex()); 5551 } 5552 } else if (constant->IsLongConstant()) { 5553 int64_t value = constant->AsLongConstant()->GetValue(); 5554 if (destination.IsRegisterPair()) { 5555 __ LoadImmediate(destination.AsRegisterPairLow<Register>(), Low32Bits(value)); 5556 __ LoadImmediate(destination.AsRegisterPairHigh<Register>(), High32Bits(value)); 5557 } else { 5558 DCHECK(destination.IsDoubleStackSlot()) << destination; 5559 __ LoadImmediate(IP, Low32Bits(value)); 5560 __ StoreToOffset(kStoreWord, IP, SP, destination.GetStackIndex()); 5561 __ LoadImmediate(IP, High32Bits(value)); 5562 __ StoreToOffset(kStoreWord, IP, SP, destination.GetHighStackIndex(kArmWordSize)); 5563 } 5564 } else if (constant->IsDoubleConstant()) { 5565 double value = constant->AsDoubleConstant()->GetValue(); 5566 if (destination.IsFpuRegisterPair()) { 5567 __ LoadDImmediate(FromLowSToD(destination.AsFpuRegisterPairLow<SRegister>()), value); 5568 } else { 5569 DCHECK(destination.IsDoubleStackSlot()) << destination; 5570 uint64_t int_value = bit_cast<uint64_t, double>(value); 5571 __ LoadImmediate(IP, Low32Bits(int_value)); 5572 __ StoreToOffset(kStoreWord, IP, SP, destination.GetStackIndex()); 5573 __ LoadImmediate(IP, High32Bits(int_value)); 5574 __ StoreToOffset(kStoreWord, IP, SP, destination.GetHighStackIndex(kArmWordSize)); 5575 } 5576 } else { 5577 DCHECK(constant->IsFloatConstant()) << constant->DebugName(); 5578 float value = constant->AsFloatConstant()->GetValue(); 5579 if (destination.IsFpuRegister()) { 5580 __ LoadSImmediate(destination.AsFpuRegister<SRegister>(), value); 5581 } else { 5582 DCHECK(destination.IsStackSlot()); 5583 __ LoadImmediate(IP, bit_cast<int32_t, float>(value)); 5584 __ StoreToOffset(kStoreWord, IP, SP, destination.GetStackIndex()); 5585 } 5586 } 5587 } 5588} 5589 5590void ParallelMoveResolverARM::Exchange(Register reg, int mem) { 5591 __ Mov(IP, reg); 5592 __ LoadFromOffset(kLoadWord, reg, SP, mem); 5593 __ StoreToOffset(kStoreWord, IP, SP, mem); 5594} 5595 5596void ParallelMoveResolverARM::Exchange(int mem1, int mem2) { 5597 ScratchRegisterScope ensure_scratch(this, IP, R0, codegen_->GetNumberOfCoreRegisters()); 5598 int stack_offset = ensure_scratch.IsSpilled() ? kArmWordSize : 0; 5599 __ LoadFromOffset(kLoadWord, static_cast<Register>(ensure_scratch.GetRegister()), 5600 SP, mem1 + stack_offset); 5601 __ LoadFromOffset(kLoadWord, IP, SP, mem2 + stack_offset); 5602 __ StoreToOffset(kStoreWord, static_cast<Register>(ensure_scratch.GetRegister()), 5603 SP, mem2 + stack_offset); 5604 __ StoreToOffset(kStoreWord, IP, SP, mem1 + stack_offset); 5605} 5606 5607void ParallelMoveResolverARM::EmitSwap(size_t index) { 5608 MoveOperands* move = moves_[index]; 5609 Location source = move->GetSource(); 5610 Location destination = move->GetDestination(); 5611 5612 if (source.IsRegister() && destination.IsRegister()) { 5613 DCHECK_NE(source.AsRegister<Register>(), IP); 5614 DCHECK_NE(destination.AsRegister<Register>(), IP); 5615 __ Mov(IP, source.AsRegister<Register>()); 5616 __ Mov(source.AsRegister<Register>(), destination.AsRegister<Register>()); 5617 __ Mov(destination.AsRegister<Register>(), IP); 5618 } else if (source.IsRegister() && destination.IsStackSlot()) { 5619 Exchange(source.AsRegister<Register>(), destination.GetStackIndex()); 5620 } else if (source.IsStackSlot() && destination.IsRegister()) { 5621 Exchange(destination.AsRegister<Register>(), source.GetStackIndex()); 5622 } else if (source.IsStackSlot() && destination.IsStackSlot()) { 5623 Exchange(source.GetStackIndex(), destination.GetStackIndex()); 5624 } else if (source.IsFpuRegister() && destination.IsFpuRegister()) { 5625 __ vmovrs(IP, source.AsFpuRegister<SRegister>()); 5626 __ vmovs(source.AsFpuRegister<SRegister>(), destination.AsFpuRegister<SRegister>()); 5627 __ vmovsr(destination.AsFpuRegister<SRegister>(), IP); 5628 } else if (source.IsRegisterPair() && destination.IsRegisterPair()) { 5629 __ vmovdrr(DTMP, source.AsRegisterPairLow<Register>(), source.AsRegisterPairHigh<Register>()); 5630 __ Mov(source.AsRegisterPairLow<Register>(), destination.AsRegisterPairLow<Register>()); 5631 __ Mov(source.AsRegisterPairHigh<Register>(), destination.AsRegisterPairHigh<Register>()); 5632 __ vmovrrd(destination.AsRegisterPairLow<Register>(), 5633 destination.AsRegisterPairHigh<Register>(), 5634 DTMP); 5635 } else if (source.IsRegisterPair() || destination.IsRegisterPair()) { 5636 Register low_reg = source.IsRegisterPair() 5637 ? source.AsRegisterPairLow<Register>() 5638 : destination.AsRegisterPairLow<Register>(); 5639 int mem = source.IsRegisterPair() 5640 ? destination.GetStackIndex() 5641 : source.GetStackIndex(); 5642 DCHECK(ExpectedPairLayout(source.IsRegisterPair() ? source : destination)); 5643 __ vmovdrr(DTMP, low_reg, static_cast<Register>(low_reg + 1)); 5644 __ LoadFromOffset(kLoadWordPair, low_reg, SP, mem); 5645 __ StoreDToOffset(DTMP, SP, mem); 5646 } else if (source.IsFpuRegisterPair() && destination.IsFpuRegisterPair()) { 5647 DRegister first = FromLowSToD(source.AsFpuRegisterPairLow<SRegister>()); 5648 DRegister second = FromLowSToD(destination.AsFpuRegisterPairLow<SRegister>()); 5649 __ vmovd(DTMP, first); 5650 __ vmovd(first, second); 5651 __ vmovd(second, DTMP); 5652 } else if (source.IsFpuRegisterPair() || destination.IsFpuRegisterPair()) { 5653 DRegister reg = source.IsFpuRegisterPair() 5654 ? FromLowSToD(source.AsFpuRegisterPairLow<SRegister>()) 5655 : FromLowSToD(destination.AsFpuRegisterPairLow<SRegister>()); 5656 int mem = source.IsFpuRegisterPair() 5657 ? destination.GetStackIndex() 5658 : source.GetStackIndex(); 5659 __ vmovd(DTMP, reg); 5660 __ LoadDFromOffset(reg, SP, mem); 5661 __ StoreDToOffset(DTMP, SP, mem); 5662 } else if (source.IsFpuRegister() || destination.IsFpuRegister()) { 5663 SRegister reg = source.IsFpuRegister() ? source.AsFpuRegister<SRegister>() 5664 : destination.AsFpuRegister<SRegister>(); 5665 int mem = source.IsFpuRegister() 5666 ? destination.GetStackIndex() 5667 : source.GetStackIndex(); 5668 5669 __ vmovrs(IP, reg); 5670 __ LoadSFromOffset(reg, SP, mem); 5671 __ StoreToOffset(kStoreWord, IP, SP, mem); 5672 } else if (source.IsDoubleStackSlot() && destination.IsDoubleStackSlot()) { 5673 Exchange(source.GetStackIndex(), destination.GetStackIndex()); 5674 Exchange(source.GetHighStackIndex(kArmWordSize), destination.GetHighStackIndex(kArmWordSize)); 5675 } else { 5676 LOG(FATAL) << "Unimplemented" << source << " <-> " << destination; 5677 } 5678} 5679 5680void ParallelMoveResolverARM::SpillScratch(int reg) { 5681 __ Push(static_cast<Register>(reg)); 5682} 5683 5684void ParallelMoveResolverARM::RestoreScratch(int reg) { 5685 __ Pop(static_cast<Register>(reg)); 5686} 5687 5688HLoadClass::LoadKind CodeGeneratorARM::GetSupportedLoadClassKind( 5689 HLoadClass::LoadKind desired_class_load_kind) { 5690 switch (desired_class_load_kind) { 5691 case HLoadClass::LoadKind::kReferrersClass: 5692 break; 5693 case HLoadClass::LoadKind::kBootImageLinkTimeAddress: 5694 DCHECK(!GetCompilerOptions().GetCompilePic()); 5695 break; 5696 case HLoadClass::LoadKind::kBootImageLinkTimePcRelative: 5697 DCHECK(GetCompilerOptions().GetCompilePic()); 5698 break; 5699 case HLoadClass::LoadKind::kBootImageAddress: 5700 break; 5701 case HLoadClass::LoadKind::kDexCacheAddress: 5702 DCHECK(Runtime::Current()->UseJitCompilation()); 5703 break; 5704 case HLoadClass::LoadKind::kDexCachePcRelative: 5705 DCHECK(!Runtime::Current()->UseJitCompilation()); 5706 // We disable pc-relative load when there is an irreducible loop, as the optimization 5707 // is incompatible with it. 5708 // TODO: Create as many ArmDexCacheArraysBase instructions as needed for methods 5709 // with irreducible loops. 5710 if (GetGraph()->HasIrreducibleLoops()) { 5711 return HLoadClass::LoadKind::kDexCacheViaMethod; 5712 } 5713 break; 5714 case HLoadClass::LoadKind::kDexCacheViaMethod: 5715 break; 5716 } 5717 return desired_class_load_kind; 5718} 5719 5720void LocationsBuilderARM::VisitLoadClass(HLoadClass* cls) { 5721 if (cls->NeedsAccessCheck()) { 5722 InvokeRuntimeCallingConvention calling_convention; 5723 CodeGenerator::CreateLoadClassLocationSummary( 5724 cls, 5725 Location::RegisterLocation(calling_convention.GetRegisterAt(0)), 5726 Location::RegisterLocation(R0), 5727 /* code_generator_supports_read_barrier */ true); 5728 return; 5729 } 5730 5731 const bool requires_read_barrier = kEmitCompilerReadBarrier && !cls->IsInBootImage(); 5732 LocationSummary::CallKind call_kind = (cls->NeedsEnvironment() || requires_read_barrier) 5733 ? LocationSummary::kCallOnSlowPath 5734 : LocationSummary::kNoCall; 5735 LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(cls, call_kind); 5736 if (kUseBakerReadBarrier && requires_read_barrier && !cls->NeedsEnvironment()) { 5737 locations->SetCustomSlowPathCallerSaves(RegisterSet::Empty()); // No caller-save registers. 5738 } 5739 5740 HLoadClass::LoadKind load_kind = cls->GetLoadKind(); 5741 if (load_kind == HLoadClass::LoadKind::kReferrersClass || 5742 load_kind == HLoadClass::LoadKind::kDexCacheViaMethod || 5743 load_kind == HLoadClass::LoadKind::kDexCachePcRelative) { 5744 locations->SetInAt(0, Location::RequiresRegister()); 5745 } 5746 locations->SetOut(Location::RequiresRegister()); 5747} 5748 5749void InstructionCodeGeneratorARM::VisitLoadClass(HLoadClass* cls) { 5750 LocationSummary* locations = cls->GetLocations(); 5751 if (cls->NeedsAccessCheck()) { 5752 codegen_->MoveConstant(locations->GetTemp(0), cls->GetTypeIndex()); 5753 codegen_->InvokeRuntime(kQuickInitializeTypeAndVerifyAccess, cls, cls->GetDexPc()); 5754 CheckEntrypointTypes<kQuickInitializeTypeAndVerifyAccess, void*, uint32_t>(); 5755 return; 5756 } 5757 5758 Location out_loc = locations->Out(); 5759 Register out = out_loc.AsRegister<Register>(); 5760 5761 const bool requires_read_barrier = kEmitCompilerReadBarrier && !cls->IsInBootImage(); 5762 bool generate_null_check = false; 5763 switch (cls->GetLoadKind()) { 5764 case HLoadClass::LoadKind::kReferrersClass: { 5765 DCHECK(!cls->CanCallRuntime()); 5766 DCHECK(!cls->MustGenerateClinitCheck()); 5767 // /* GcRoot<mirror::Class> */ out = current_method->declaring_class_ 5768 Register current_method = locations->InAt(0).AsRegister<Register>(); 5769 GenerateGcRootFieldLoad(cls, 5770 out_loc, 5771 current_method, 5772 ArtMethod::DeclaringClassOffset().Int32Value(), 5773 requires_read_barrier); 5774 break; 5775 } 5776 case HLoadClass::LoadKind::kBootImageLinkTimeAddress: { 5777 DCHECK(!requires_read_barrier); 5778 __ LoadLiteral(out, codegen_->DeduplicateBootImageTypeLiteral(cls->GetDexFile(), 5779 cls->GetTypeIndex())); 5780 break; 5781 } 5782 case HLoadClass::LoadKind::kBootImageLinkTimePcRelative: { 5783 DCHECK(!requires_read_barrier); 5784 CodeGeneratorARM::PcRelativePatchInfo* labels = 5785 codegen_->NewPcRelativeTypePatch(cls->GetDexFile(), cls->GetTypeIndex()); 5786 __ BindTrackedLabel(&labels->movw_label); 5787 __ movw(out, /* placeholder */ 0u); 5788 __ BindTrackedLabel(&labels->movt_label); 5789 __ movt(out, /* placeholder */ 0u); 5790 __ BindTrackedLabel(&labels->add_pc_label); 5791 __ add(out, out, ShifterOperand(PC)); 5792 break; 5793 } 5794 case HLoadClass::LoadKind::kBootImageAddress: { 5795 DCHECK(!requires_read_barrier); 5796 DCHECK_NE(cls->GetAddress(), 0u); 5797 uint32_t address = dchecked_integral_cast<uint32_t>(cls->GetAddress()); 5798 __ LoadLiteral(out, codegen_->DeduplicateBootImageAddressLiteral(address)); 5799 break; 5800 } 5801 case HLoadClass::LoadKind::kDexCacheAddress: { 5802 DCHECK_NE(cls->GetAddress(), 0u); 5803 uint32_t address = dchecked_integral_cast<uint32_t>(cls->GetAddress()); 5804 // 16-bit LDR immediate has a 5-bit offset multiplied by the size and that gives 5805 // a 128B range. To try and reduce the number of literals if we load multiple types, 5806 // simply split the dex cache address to a 128B aligned base loaded from a literal 5807 // and the remaining offset embedded in the load. 5808 static_assert(sizeof(GcRoot<mirror::Class>) == 4u, "Expected GC root to be 4 bytes."); 5809 DCHECK_ALIGNED(cls->GetAddress(), 4u); 5810 constexpr size_t offset_bits = /* encoded bits */ 5 + /* scale */ 2; 5811 uint32_t base_address = address & ~MaxInt<uint32_t>(offset_bits); 5812 uint32_t offset = address & MaxInt<uint32_t>(offset_bits); 5813 __ LoadLiteral(out, codegen_->DeduplicateDexCacheAddressLiteral(base_address)); 5814 // /* GcRoot<mirror::Class> */ out = *(base_address + offset) 5815 GenerateGcRootFieldLoad(cls, out_loc, out, offset, requires_read_barrier); 5816 generate_null_check = !cls->IsInDexCache(); 5817 break; 5818 } 5819 case HLoadClass::LoadKind::kDexCachePcRelative: { 5820 Register base_reg = locations->InAt(0).AsRegister<Register>(); 5821 HArmDexCacheArraysBase* base = cls->InputAt(0)->AsArmDexCacheArraysBase(); 5822 int32_t offset = cls->GetDexCacheElementOffset() - base->GetElementOffset(); 5823 // /* GcRoot<mirror::Class> */ out = *(dex_cache_arrays_base + offset) 5824 GenerateGcRootFieldLoad(cls, out_loc, base_reg, offset, requires_read_barrier); 5825 generate_null_check = !cls->IsInDexCache(); 5826 break; 5827 } 5828 case HLoadClass::LoadKind::kDexCacheViaMethod: { 5829 // /* GcRoot<mirror::Class>[] */ out = 5830 // current_method.ptr_sized_fields_->dex_cache_resolved_types_ 5831 Register current_method = locations->InAt(0).AsRegister<Register>(); 5832 __ LoadFromOffset(kLoadWord, 5833 out, 5834 current_method, 5835 ArtMethod::DexCacheResolvedTypesOffset(kArmPointerSize).Int32Value()); 5836 // /* GcRoot<mirror::Class> */ out = out[type_index] 5837 size_t offset = CodeGenerator::GetCacheOffset(cls->GetTypeIndex()); 5838 GenerateGcRootFieldLoad(cls, out_loc, out, offset, requires_read_barrier); 5839 generate_null_check = !cls->IsInDexCache(); 5840 } 5841 } 5842 5843 if (generate_null_check || cls->MustGenerateClinitCheck()) { 5844 DCHECK(cls->CanCallRuntime()); 5845 SlowPathCodeARM* slow_path = new (GetGraph()->GetArena()) LoadClassSlowPathARM( 5846 cls, cls, cls->GetDexPc(), cls->MustGenerateClinitCheck()); 5847 codegen_->AddSlowPath(slow_path); 5848 if (generate_null_check) { 5849 __ CompareAndBranchIfZero(out, slow_path->GetEntryLabel()); 5850 } 5851 if (cls->MustGenerateClinitCheck()) { 5852 GenerateClassInitializationCheck(slow_path, out); 5853 } else { 5854 __ Bind(slow_path->GetExitLabel()); 5855 } 5856 } 5857} 5858 5859void LocationsBuilderARM::VisitClinitCheck(HClinitCheck* check) { 5860 LocationSummary* locations = 5861 new (GetGraph()->GetArena()) LocationSummary(check, LocationSummary::kCallOnSlowPath); 5862 locations->SetInAt(0, Location::RequiresRegister()); 5863 if (check->HasUses()) { 5864 locations->SetOut(Location::SameAsFirstInput()); 5865 } 5866} 5867 5868void InstructionCodeGeneratorARM::VisitClinitCheck(HClinitCheck* check) { 5869 // We assume the class is not null. 5870 SlowPathCodeARM* slow_path = new (GetGraph()->GetArena()) LoadClassSlowPathARM( 5871 check->GetLoadClass(), check, check->GetDexPc(), true); 5872 codegen_->AddSlowPath(slow_path); 5873 GenerateClassInitializationCheck(slow_path, 5874 check->GetLocations()->InAt(0).AsRegister<Register>()); 5875} 5876 5877void InstructionCodeGeneratorARM::GenerateClassInitializationCheck( 5878 SlowPathCodeARM* slow_path, Register class_reg) { 5879 __ LoadFromOffset(kLoadWord, IP, class_reg, mirror::Class::StatusOffset().Int32Value()); 5880 __ cmp(IP, ShifterOperand(mirror::Class::kStatusInitialized)); 5881 __ b(slow_path->GetEntryLabel(), LT); 5882 // Even if the initialized flag is set, we may be in a situation where caches are not synced 5883 // properly. Therefore, we do a memory fence. 5884 __ dmb(ISH); 5885 __ Bind(slow_path->GetExitLabel()); 5886} 5887 5888HLoadString::LoadKind CodeGeneratorARM::GetSupportedLoadStringKind( 5889 HLoadString::LoadKind desired_string_load_kind) { 5890 switch (desired_string_load_kind) { 5891 case HLoadString::LoadKind::kBootImageLinkTimeAddress: 5892 DCHECK(!GetCompilerOptions().GetCompilePic()); 5893 break; 5894 case HLoadString::LoadKind::kBootImageLinkTimePcRelative: 5895 DCHECK(GetCompilerOptions().GetCompilePic()); 5896 break; 5897 case HLoadString::LoadKind::kBootImageAddress: 5898 break; 5899 case HLoadString::LoadKind::kBssEntry: 5900 DCHECK(!Runtime::Current()->UseJitCompilation()); 5901 break; 5902 case HLoadString::LoadKind::kDexCacheViaMethod: 5903 break; 5904 } 5905 return desired_string_load_kind; 5906} 5907 5908void LocationsBuilderARM::VisitLoadString(HLoadString* load) { 5909 LocationSummary::CallKind call_kind = load->NeedsEnvironment() 5910 ? ((load->GetLoadKind() == HLoadString::LoadKind::kDexCacheViaMethod) 5911 ? LocationSummary::kCallOnMainOnly 5912 : LocationSummary::kCallOnSlowPath) 5913 : LocationSummary::kNoCall; 5914 LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(load, call_kind); 5915 5916 HLoadString::LoadKind load_kind = load->GetLoadKind(); 5917 if (load_kind == HLoadString::LoadKind::kDexCacheViaMethod) { 5918 locations->SetOut(Location::RegisterLocation(R0)); 5919 } else { 5920 locations->SetOut(Location::RequiresRegister()); 5921 if (load_kind == HLoadString::LoadKind::kBssEntry) { 5922 if (!kUseReadBarrier || kUseBakerReadBarrier) { 5923 // Rely on the pResolveString and/or marking to save everything, including temps. 5924 // Note that IP may theoretically be clobbered by saving/restoring the live register 5925 // (only one thanks to the custom calling convention), so we request a different temp. 5926 locations->AddTemp(Location::RequiresRegister()); 5927 RegisterSet caller_saves = RegisterSet::Empty(); 5928 InvokeRuntimeCallingConvention calling_convention; 5929 caller_saves.Add(Location::RegisterLocation(calling_convention.GetRegisterAt(0))); 5930 // TODO: Add GetReturnLocation() to the calling convention so that we can DCHECK() 5931 // that the the kPrimNot result register is the same as the first argument register. 5932 locations->SetCustomSlowPathCallerSaves(caller_saves); 5933 } else { 5934 // For non-Baker read barrier we have a temp-clobbering call. 5935 } 5936 } 5937 } 5938} 5939 5940void InstructionCodeGeneratorARM::VisitLoadString(HLoadString* load) { 5941 LocationSummary* locations = load->GetLocations(); 5942 Location out_loc = locations->Out(); 5943 Register out = out_loc.AsRegister<Register>(); 5944 HLoadString::LoadKind load_kind = load->GetLoadKind(); 5945 5946 switch (load_kind) { 5947 case HLoadString::LoadKind::kBootImageLinkTimeAddress: { 5948 __ LoadLiteral(out, codegen_->DeduplicateBootImageStringLiteral(load->GetDexFile(), 5949 load->GetStringIndex())); 5950 return; // No dex cache slow path. 5951 } 5952 case HLoadString::LoadKind::kBootImageLinkTimePcRelative: { 5953 DCHECK(codegen_->GetCompilerOptions().IsBootImage()); 5954 CodeGeneratorARM::PcRelativePatchInfo* labels = 5955 codegen_->NewPcRelativeStringPatch(load->GetDexFile(), load->GetStringIndex()); 5956 __ BindTrackedLabel(&labels->movw_label); 5957 __ movw(out, /* placeholder */ 0u); 5958 __ BindTrackedLabel(&labels->movt_label); 5959 __ movt(out, /* placeholder */ 0u); 5960 __ BindTrackedLabel(&labels->add_pc_label); 5961 __ add(out, out, ShifterOperand(PC)); 5962 return; // No dex cache slow path. 5963 } 5964 case HLoadString::LoadKind::kBootImageAddress: { 5965 DCHECK_NE(load->GetAddress(), 0u); 5966 uint32_t address = dchecked_integral_cast<uint32_t>(load->GetAddress()); 5967 __ LoadLiteral(out, codegen_->DeduplicateBootImageAddressLiteral(address)); 5968 return; // No dex cache slow path. 5969 } 5970 case HLoadString::LoadKind::kBssEntry: { 5971 DCHECK(!codegen_->GetCompilerOptions().IsBootImage()); 5972 Register temp = locations->GetTemp(0).AsRegister<Register>(); 5973 CodeGeneratorARM::PcRelativePatchInfo* labels = 5974 codegen_->NewPcRelativeStringPatch(load->GetDexFile(), load->GetStringIndex()); 5975 __ BindTrackedLabel(&labels->movw_label); 5976 __ movw(temp, /* placeholder */ 0u); 5977 __ BindTrackedLabel(&labels->movt_label); 5978 __ movt(temp, /* placeholder */ 0u); 5979 __ BindTrackedLabel(&labels->add_pc_label); 5980 __ add(temp, temp, ShifterOperand(PC)); 5981 GenerateGcRootFieldLoad(load, out_loc, temp, /* offset */ 0, kEmitCompilerReadBarrier); 5982 SlowPathCode* slow_path = new (GetGraph()->GetArena()) LoadStringSlowPathARM(load); 5983 codegen_->AddSlowPath(slow_path); 5984 __ CompareAndBranchIfZero(out, slow_path->GetEntryLabel()); 5985 __ Bind(slow_path->GetExitLabel()); 5986 return; 5987 } 5988 default: 5989 break; 5990 } 5991 5992 // TODO: Consider re-adding the compiler code to do string dex cache lookup again. 5993 DCHECK(load_kind == HLoadString::LoadKind::kDexCacheViaMethod); 5994 InvokeRuntimeCallingConvention calling_convention; 5995 DCHECK_EQ(calling_convention.GetRegisterAt(0), out); 5996 __ LoadImmediate(calling_convention.GetRegisterAt(0), load->GetStringIndex()); 5997 codegen_->InvokeRuntime(kQuickResolveString, load, load->GetDexPc()); 5998 CheckEntrypointTypes<kQuickResolveString, void*, uint32_t>(); 5999} 6000 6001static int32_t GetExceptionTlsOffset() { 6002 return Thread::ExceptionOffset<kArmPointerSize>().Int32Value(); 6003} 6004 6005void LocationsBuilderARM::VisitLoadException(HLoadException* load) { 6006 LocationSummary* locations = 6007 new (GetGraph()->GetArena()) LocationSummary(load, LocationSummary::kNoCall); 6008 locations->SetOut(Location::RequiresRegister()); 6009} 6010 6011void InstructionCodeGeneratorARM::VisitLoadException(HLoadException* load) { 6012 Register out = load->GetLocations()->Out().AsRegister<Register>(); 6013 __ LoadFromOffset(kLoadWord, out, TR, GetExceptionTlsOffset()); 6014} 6015 6016void LocationsBuilderARM::VisitClearException(HClearException* clear) { 6017 new (GetGraph()->GetArena()) LocationSummary(clear, LocationSummary::kNoCall); 6018} 6019 6020void InstructionCodeGeneratorARM::VisitClearException(HClearException* clear ATTRIBUTE_UNUSED) { 6021 __ LoadImmediate(IP, 0); 6022 __ StoreToOffset(kStoreWord, IP, TR, GetExceptionTlsOffset()); 6023} 6024 6025void LocationsBuilderARM::VisitThrow(HThrow* instruction) { 6026 LocationSummary* locations = 6027 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCallOnMainOnly); 6028 InvokeRuntimeCallingConvention calling_convention; 6029 locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(0))); 6030} 6031 6032void InstructionCodeGeneratorARM::VisitThrow(HThrow* instruction) { 6033 codegen_->InvokeRuntime(kQuickDeliverException, instruction, instruction->GetDexPc()); 6034 CheckEntrypointTypes<kQuickDeliverException, void, mirror::Object*>(); 6035} 6036 6037// Temp is used for read barrier. 6038static size_t NumberOfInstanceOfTemps(TypeCheckKind type_check_kind) { 6039 if (kEmitCompilerReadBarrier && 6040 (kUseBakerReadBarrier || 6041 type_check_kind == TypeCheckKind::kAbstractClassCheck || 6042 type_check_kind == TypeCheckKind::kClassHierarchyCheck || 6043 type_check_kind == TypeCheckKind::kArrayObjectCheck)) { 6044 return 1; 6045 } 6046 return 0; 6047} 6048 6049// Interface case has 3 temps, one for holding the number of interfaces, one for the current 6050// interface pointer, one for loading the current interface. 6051// The other checks have one temp for loading the object's class. 6052static size_t NumberOfCheckCastTemps(TypeCheckKind type_check_kind) { 6053 if (type_check_kind == TypeCheckKind::kInterfaceCheck) { 6054 return 3; 6055 } 6056 return 1 + NumberOfInstanceOfTemps(type_check_kind); 6057} 6058 6059void LocationsBuilderARM::VisitInstanceOf(HInstanceOf* instruction) { 6060 LocationSummary::CallKind call_kind = LocationSummary::kNoCall; 6061 TypeCheckKind type_check_kind = instruction->GetTypeCheckKind(); 6062 bool baker_read_barrier_slow_path = false; 6063 switch (type_check_kind) { 6064 case TypeCheckKind::kExactCheck: 6065 case TypeCheckKind::kAbstractClassCheck: 6066 case TypeCheckKind::kClassHierarchyCheck: 6067 case TypeCheckKind::kArrayObjectCheck: 6068 call_kind = 6069 kEmitCompilerReadBarrier ? LocationSummary::kCallOnSlowPath : LocationSummary::kNoCall; 6070 baker_read_barrier_slow_path = kUseBakerReadBarrier; 6071 break; 6072 case TypeCheckKind::kArrayCheck: 6073 case TypeCheckKind::kUnresolvedCheck: 6074 case TypeCheckKind::kInterfaceCheck: 6075 call_kind = LocationSummary::kCallOnSlowPath; 6076 break; 6077 } 6078 6079 LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction, call_kind); 6080 if (baker_read_barrier_slow_path) { 6081 locations->SetCustomSlowPathCallerSaves(RegisterSet::Empty()); // No caller-save registers. 6082 } 6083 locations->SetInAt(0, Location::RequiresRegister()); 6084 locations->SetInAt(1, Location::RequiresRegister()); 6085 // The "out" register is used as a temporary, so it overlaps with the inputs. 6086 // Note that TypeCheckSlowPathARM uses this register too. 6087 locations->SetOut(Location::RequiresRegister(), Location::kOutputOverlap); 6088 locations->AddRegisterTemps(NumberOfInstanceOfTemps(type_check_kind)); 6089} 6090 6091void InstructionCodeGeneratorARM::VisitInstanceOf(HInstanceOf* instruction) { 6092 TypeCheckKind type_check_kind = instruction->GetTypeCheckKind(); 6093 LocationSummary* locations = instruction->GetLocations(); 6094 Location obj_loc = locations->InAt(0); 6095 Register obj = obj_loc.AsRegister<Register>(); 6096 Register cls = locations->InAt(1).AsRegister<Register>(); 6097 Location out_loc = locations->Out(); 6098 Register out = out_loc.AsRegister<Register>(); 6099 const size_t num_temps = NumberOfInstanceOfTemps(type_check_kind); 6100 DCHECK_LE(num_temps, 1u); 6101 Location maybe_temp_loc = (num_temps >= 1) ? locations->GetTemp(0) : Location::NoLocation(); 6102 uint32_t class_offset = mirror::Object::ClassOffset().Int32Value(); 6103 uint32_t super_offset = mirror::Class::SuperClassOffset().Int32Value(); 6104 uint32_t component_offset = mirror::Class::ComponentTypeOffset().Int32Value(); 6105 uint32_t primitive_offset = mirror::Class::PrimitiveTypeOffset().Int32Value(); 6106 Label done, zero; 6107 SlowPathCodeARM* slow_path = nullptr; 6108 6109 // Return 0 if `obj` is null. 6110 // avoid null check if we know obj is not null. 6111 if (instruction->MustDoNullCheck()) { 6112 __ CompareAndBranchIfZero(obj, &zero); 6113 } 6114 6115 // /* HeapReference<Class> */ out = obj->klass_ 6116 GenerateReferenceLoadTwoRegisters(instruction, 6117 out_loc, 6118 obj_loc, 6119 class_offset, 6120 maybe_temp_loc, 6121 kEmitCompilerReadBarrier); 6122 6123 switch (type_check_kind) { 6124 case TypeCheckKind::kExactCheck: { 6125 __ cmp(out, ShifterOperand(cls)); 6126 // Classes must be equal for the instanceof to succeed. 6127 __ b(&zero, NE); 6128 __ LoadImmediate(out, 1); 6129 __ b(&done); 6130 break; 6131 } 6132 6133 case TypeCheckKind::kAbstractClassCheck: { 6134 // If the class is abstract, we eagerly fetch the super class of the 6135 // object to avoid doing a comparison we know will fail. 6136 Label loop; 6137 __ Bind(&loop); 6138 // /* HeapReference<Class> */ out = out->super_class_ 6139 GenerateReferenceLoadOneRegister(instruction, 6140 out_loc, 6141 super_offset, 6142 maybe_temp_loc, 6143 kEmitCompilerReadBarrier); 6144 // If `out` is null, we use it for the result, and jump to `done`. 6145 __ CompareAndBranchIfZero(out, &done); 6146 __ cmp(out, ShifterOperand(cls)); 6147 __ b(&loop, NE); 6148 __ LoadImmediate(out, 1); 6149 if (zero.IsLinked()) { 6150 __ b(&done); 6151 } 6152 break; 6153 } 6154 6155 case TypeCheckKind::kClassHierarchyCheck: { 6156 // Walk over the class hierarchy to find a match. 6157 Label loop, success; 6158 __ Bind(&loop); 6159 __ cmp(out, ShifterOperand(cls)); 6160 __ b(&success, EQ); 6161 // /* HeapReference<Class> */ out = out->super_class_ 6162 GenerateReferenceLoadOneRegister(instruction, 6163 out_loc, 6164 super_offset, 6165 maybe_temp_loc, 6166 kEmitCompilerReadBarrier); 6167 __ CompareAndBranchIfNonZero(out, &loop); 6168 // If `out` is null, we use it for the result, and jump to `done`. 6169 __ b(&done); 6170 __ Bind(&success); 6171 __ LoadImmediate(out, 1); 6172 if (zero.IsLinked()) { 6173 __ b(&done); 6174 } 6175 break; 6176 } 6177 6178 case TypeCheckKind::kArrayObjectCheck: { 6179 // Do an exact check. 6180 Label exact_check; 6181 __ cmp(out, ShifterOperand(cls)); 6182 __ b(&exact_check, EQ); 6183 // Otherwise, we need to check that the object's class is a non-primitive array. 6184 // /* HeapReference<Class> */ out = out->component_type_ 6185 GenerateReferenceLoadOneRegister(instruction, 6186 out_loc, 6187 component_offset, 6188 maybe_temp_loc, 6189 kEmitCompilerReadBarrier); 6190 // If `out` is null, we use it for the result, and jump to `done`. 6191 __ CompareAndBranchIfZero(out, &done); 6192 __ LoadFromOffset(kLoadUnsignedHalfword, out, out, primitive_offset); 6193 static_assert(Primitive::kPrimNot == 0, "Expected 0 for kPrimNot"); 6194 __ CompareAndBranchIfNonZero(out, &zero); 6195 __ Bind(&exact_check); 6196 __ LoadImmediate(out, 1); 6197 __ b(&done); 6198 break; 6199 } 6200 6201 case TypeCheckKind::kArrayCheck: { 6202 __ cmp(out, ShifterOperand(cls)); 6203 DCHECK(locations->OnlyCallsOnSlowPath()); 6204 slow_path = new (GetGraph()->GetArena()) TypeCheckSlowPathARM(instruction, 6205 /* is_fatal */ false); 6206 codegen_->AddSlowPath(slow_path); 6207 __ b(slow_path->GetEntryLabel(), NE); 6208 __ LoadImmediate(out, 1); 6209 if (zero.IsLinked()) { 6210 __ b(&done); 6211 } 6212 break; 6213 } 6214 6215 case TypeCheckKind::kUnresolvedCheck: 6216 case TypeCheckKind::kInterfaceCheck: { 6217 // Note that we indeed only call on slow path, but we always go 6218 // into the slow path for the unresolved and interface check 6219 // cases. 6220 // 6221 // We cannot directly call the InstanceofNonTrivial runtime 6222 // entry point without resorting to a type checking slow path 6223 // here (i.e. by calling InvokeRuntime directly), as it would 6224 // require to assign fixed registers for the inputs of this 6225 // HInstanceOf instruction (following the runtime calling 6226 // convention), which might be cluttered by the potential first 6227 // read barrier emission at the beginning of this method. 6228 // 6229 // TODO: Introduce a new runtime entry point taking the object 6230 // to test (instead of its class) as argument, and let it deal 6231 // with the read barrier issues. This will let us refactor this 6232 // case of the `switch` code as it was previously (with a direct 6233 // call to the runtime not using a type checking slow path). 6234 // This should also be beneficial for the other cases above. 6235 DCHECK(locations->OnlyCallsOnSlowPath()); 6236 slow_path = new (GetGraph()->GetArena()) TypeCheckSlowPathARM(instruction, 6237 /* is_fatal */ false); 6238 codegen_->AddSlowPath(slow_path); 6239 __ b(slow_path->GetEntryLabel()); 6240 if (zero.IsLinked()) { 6241 __ b(&done); 6242 } 6243 break; 6244 } 6245 } 6246 6247 if (zero.IsLinked()) { 6248 __ Bind(&zero); 6249 __ LoadImmediate(out, 0); 6250 } 6251 6252 if (done.IsLinked()) { 6253 __ Bind(&done); 6254 } 6255 6256 if (slow_path != nullptr) { 6257 __ Bind(slow_path->GetExitLabel()); 6258 } 6259} 6260 6261void LocationsBuilderARM::VisitCheckCast(HCheckCast* instruction) { 6262 LocationSummary::CallKind call_kind = LocationSummary::kNoCall; 6263 bool throws_into_catch = instruction->CanThrowIntoCatchBlock(); 6264 6265 TypeCheckKind type_check_kind = instruction->GetTypeCheckKind(); 6266 switch (type_check_kind) { 6267 case TypeCheckKind::kExactCheck: 6268 case TypeCheckKind::kAbstractClassCheck: 6269 case TypeCheckKind::kClassHierarchyCheck: 6270 case TypeCheckKind::kArrayObjectCheck: 6271 call_kind = (throws_into_catch || kEmitCompilerReadBarrier) ? 6272 LocationSummary::kCallOnSlowPath : 6273 LocationSummary::kNoCall; // In fact, call on a fatal (non-returning) slow path. 6274 break; 6275 case TypeCheckKind::kArrayCheck: 6276 case TypeCheckKind::kUnresolvedCheck: 6277 case TypeCheckKind::kInterfaceCheck: 6278 call_kind = LocationSummary::kCallOnSlowPath; 6279 break; 6280 } 6281 6282 LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction, call_kind); 6283 locations->SetInAt(0, Location::RequiresRegister()); 6284 locations->SetInAt(1, Location::RequiresRegister()); 6285 locations->AddRegisterTemps(NumberOfCheckCastTemps(type_check_kind)); 6286} 6287 6288void InstructionCodeGeneratorARM::VisitCheckCast(HCheckCast* instruction) { 6289 TypeCheckKind type_check_kind = instruction->GetTypeCheckKind(); 6290 LocationSummary* locations = instruction->GetLocations(); 6291 Location obj_loc = locations->InAt(0); 6292 Register obj = obj_loc.AsRegister<Register>(); 6293 Register cls = locations->InAt(1).AsRegister<Register>(); 6294 Location temp_loc = locations->GetTemp(0); 6295 Register temp = temp_loc.AsRegister<Register>(); 6296 const size_t num_temps = NumberOfCheckCastTemps(type_check_kind); 6297 DCHECK_LE(num_temps, 3u); 6298 Location maybe_temp2_loc = (num_temps >= 2) ? locations->GetTemp(1) : Location::NoLocation(); 6299 Location maybe_temp3_loc = (num_temps >= 3) ? locations->GetTemp(2) : Location::NoLocation(); 6300 const uint32_t class_offset = mirror::Object::ClassOffset().Int32Value(); 6301 const uint32_t super_offset = mirror::Class::SuperClassOffset().Int32Value(); 6302 const uint32_t component_offset = mirror::Class::ComponentTypeOffset().Int32Value(); 6303 const uint32_t primitive_offset = mirror::Class::PrimitiveTypeOffset().Int32Value(); 6304 const uint32_t iftable_offset = mirror::Class::IfTableOffset().Uint32Value(); 6305 const uint32_t array_length_offset = mirror::Array::LengthOffset().Uint32Value(); 6306 const uint32_t object_array_data_offset = 6307 mirror::Array::DataOffset(kHeapReferenceSize).Uint32Value(); 6308 6309 bool is_type_check_slow_path_fatal = false; 6310 if (!kEmitCompilerReadBarrier) { 6311 is_type_check_slow_path_fatal = 6312 (type_check_kind == TypeCheckKind::kExactCheck || 6313 type_check_kind == TypeCheckKind::kAbstractClassCheck || 6314 type_check_kind == TypeCheckKind::kClassHierarchyCheck || 6315 type_check_kind == TypeCheckKind::kArrayObjectCheck) && 6316 !instruction->CanThrowIntoCatchBlock(); 6317 } 6318 SlowPathCodeARM* type_check_slow_path = 6319 new (GetGraph()->GetArena()) TypeCheckSlowPathARM(instruction, 6320 is_type_check_slow_path_fatal); 6321 codegen_->AddSlowPath(type_check_slow_path); 6322 6323 Label done; 6324 // Avoid null check if we know obj is not null. 6325 if (instruction->MustDoNullCheck()) { 6326 __ CompareAndBranchIfZero(obj, &done); 6327 } 6328 6329 switch (type_check_kind) { 6330 case TypeCheckKind::kExactCheck: 6331 case TypeCheckKind::kArrayCheck: { 6332 // /* HeapReference<Class> */ temp = obj->klass_ 6333 GenerateReferenceLoadTwoRegisters(instruction, 6334 temp_loc, 6335 obj_loc, 6336 class_offset, 6337 maybe_temp2_loc, 6338 /*emit_read_barrier*/ false); 6339 6340 __ cmp(temp, ShifterOperand(cls)); 6341 // Jump to slow path for throwing the exception or doing a 6342 // more involved array check. 6343 __ b(type_check_slow_path->GetEntryLabel(), NE); 6344 break; 6345 } 6346 6347 case TypeCheckKind::kAbstractClassCheck: { 6348 // /* HeapReference<Class> */ temp = obj->klass_ 6349 GenerateReferenceLoadTwoRegisters(instruction, 6350 temp_loc, 6351 obj_loc, 6352 class_offset, 6353 maybe_temp2_loc, 6354 /*emit_read_barrier*/ false); 6355 6356 // If the class is abstract, we eagerly fetch the super class of the 6357 // object to avoid doing a comparison we know will fail. 6358 Label loop; 6359 __ Bind(&loop); 6360 // /* HeapReference<Class> */ temp = temp->super_class_ 6361 GenerateReferenceLoadOneRegister(instruction, 6362 temp_loc, 6363 super_offset, 6364 maybe_temp2_loc, 6365 /*emit_read_barrier*/ false); 6366 6367 // If the class reference currently in `temp` is null, jump to the slow path to throw the 6368 // exception. 6369 __ CompareAndBranchIfZero(temp, type_check_slow_path->GetEntryLabel()); 6370 6371 // Otherwise, compare the classes. 6372 __ cmp(temp, ShifterOperand(cls)); 6373 __ b(&loop, NE); 6374 break; 6375 } 6376 6377 case TypeCheckKind::kClassHierarchyCheck: { 6378 // /* HeapReference<Class> */ temp = obj->klass_ 6379 GenerateReferenceLoadTwoRegisters(instruction, 6380 temp_loc, 6381 obj_loc, 6382 class_offset, 6383 maybe_temp2_loc, 6384 /*emit_read_barrier*/ false); 6385 6386 // Walk over the class hierarchy to find a match. 6387 Label loop; 6388 __ Bind(&loop); 6389 __ cmp(temp, ShifterOperand(cls)); 6390 __ b(&done, EQ); 6391 6392 // /* HeapReference<Class> */ temp = temp->super_class_ 6393 GenerateReferenceLoadOneRegister(instruction, 6394 temp_loc, 6395 super_offset, 6396 maybe_temp2_loc, 6397 /*emit_read_barrier*/ false); 6398 6399 // If the class reference currently in `temp` is null, jump to the slow path to throw the 6400 // exception. 6401 __ CompareAndBranchIfZero(temp, type_check_slow_path->GetEntryLabel()); 6402 // Otherwise, jump to the beginning of the loop. 6403 __ b(&loop); 6404 break; 6405 } 6406 6407 case TypeCheckKind::kArrayObjectCheck: { 6408 // /* HeapReference<Class> */ temp = obj->klass_ 6409 GenerateReferenceLoadTwoRegisters(instruction, 6410 temp_loc, 6411 obj_loc, 6412 class_offset, 6413 maybe_temp2_loc, 6414 /*emit_read_barrier*/ false); 6415 6416 // Do an exact check. 6417 __ cmp(temp, ShifterOperand(cls)); 6418 __ b(&done, EQ); 6419 6420 // Otherwise, we need to check that the object's class is a non-primitive array. 6421 // /* HeapReference<Class> */ temp = temp->component_type_ 6422 GenerateReferenceLoadOneRegister(instruction, 6423 temp_loc, 6424 component_offset, 6425 maybe_temp2_loc, 6426 /*emit_read_barrier*/ false); 6427 // If the component type is null, jump to the slow path to throw the exception. 6428 __ CompareAndBranchIfZero(temp, type_check_slow_path->GetEntryLabel()); 6429 // Otherwise,the object is indeed an array, jump to label `check_non_primitive_component_type` 6430 // to further check that this component type is not a primitive type. 6431 __ LoadFromOffset(kLoadUnsignedHalfword, temp, temp, primitive_offset); 6432 static_assert(Primitive::kPrimNot == 0, "Expected 0 for art::Primitive::kPrimNot"); 6433 __ CompareAndBranchIfNonZero(temp, type_check_slow_path->GetEntryLabel()); 6434 break; 6435 } 6436 6437 case TypeCheckKind::kUnresolvedCheck: 6438 // We always go into the type check slow path for the unresolved check case. 6439 // We cannot directly call the CheckCast runtime entry point 6440 // without resorting to a type checking slow path here (i.e. by 6441 // calling InvokeRuntime directly), as it would require to 6442 // assign fixed registers for the inputs of this HInstanceOf 6443 // instruction (following the runtime calling convention), which 6444 // might be cluttered by the potential first read barrier 6445 // emission at the beginning of this method. 6446 6447 __ b(type_check_slow_path->GetEntryLabel()); 6448 break; 6449 6450 case TypeCheckKind::kInterfaceCheck: { 6451 // Avoid read barriers to improve performance of the fast path. We can not get false 6452 // positives by doing this. 6453 // /* HeapReference<Class> */ temp = obj->klass_ 6454 GenerateReferenceLoadTwoRegisters(instruction, 6455 temp_loc, 6456 obj_loc, 6457 class_offset, 6458 maybe_temp2_loc, 6459 /*emit_read_barrier*/ false); 6460 6461 // /* HeapReference<Class> */ temp = temp->iftable_ 6462 GenerateReferenceLoadTwoRegisters(instruction, 6463 temp_loc, 6464 temp_loc, 6465 iftable_offset, 6466 maybe_temp2_loc, 6467 /*emit_read_barrier*/ false); 6468 Label is_null; 6469 // Null iftable means it is empty and will always fail the check. 6470 // Not cbz since the temp may not be a low register. 6471 __ CompareAndBranchIfZero(temp, &is_null); 6472 6473 // Loop through the iftable and check if any class matches. 6474 __ ldr(maybe_temp2_loc.AsRegister<Register>(), Address(temp, array_length_offset)); 6475 6476 Label start_loop; 6477 __ Bind(&start_loop); 6478 __ ldr(maybe_temp3_loc.AsRegister<Register>(), Address(temp, object_array_data_offset)); 6479 __ MaybeUnpoisonHeapReference(maybe_temp3_loc.AsRegister<Register>()); 6480 __ cmp(cls, ShifterOperand(maybe_temp3_loc.AsRegister<Register>())); 6481 __ b(&done, EQ); // Return if same class. 6482 // Go to next interface. 6483 __ add(temp, temp, ShifterOperand(2 * kHeapReferenceSize)); 6484 __ sub(maybe_temp2_loc.AsRegister<Register>(), 6485 maybe_temp2_loc.AsRegister<Register>(), 6486 ShifterOperand(2)); 6487 // Not cbnz since the temp may not be a low register. 6488 __ CompareAndBranchIfNonZero(maybe_temp2_loc.AsRegister<Register>(), &start_loop); 6489 __ Bind(&is_null); 6490 6491 __ b(type_check_slow_path->GetEntryLabel()); 6492 break; 6493 } 6494 } 6495 __ Bind(&done); 6496 6497 __ Bind(type_check_slow_path->GetExitLabel()); 6498} 6499 6500void LocationsBuilderARM::VisitMonitorOperation(HMonitorOperation* instruction) { 6501 LocationSummary* locations = 6502 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCallOnMainOnly); 6503 InvokeRuntimeCallingConvention calling_convention; 6504 locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(0))); 6505} 6506 6507void InstructionCodeGeneratorARM::VisitMonitorOperation(HMonitorOperation* instruction) { 6508 codegen_->InvokeRuntime(instruction->IsEnter() ? kQuickLockObject : kQuickUnlockObject, 6509 instruction, 6510 instruction->GetDexPc()); 6511 if (instruction->IsEnter()) { 6512 CheckEntrypointTypes<kQuickLockObject, void, mirror::Object*>(); 6513 } else { 6514 CheckEntrypointTypes<kQuickUnlockObject, void, mirror::Object*>(); 6515 } 6516} 6517 6518void LocationsBuilderARM::VisitAnd(HAnd* instruction) { HandleBitwiseOperation(instruction, AND); } 6519void LocationsBuilderARM::VisitOr(HOr* instruction) { HandleBitwiseOperation(instruction, ORR); } 6520void LocationsBuilderARM::VisitXor(HXor* instruction) { HandleBitwiseOperation(instruction, EOR); } 6521 6522void LocationsBuilderARM::HandleBitwiseOperation(HBinaryOperation* instruction, Opcode opcode) { 6523 LocationSummary* locations = 6524 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall); 6525 DCHECK(instruction->GetResultType() == Primitive::kPrimInt 6526 || instruction->GetResultType() == Primitive::kPrimLong); 6527 // Note: GVN reorders commutative operations to have the constant on the right hand side. 6528 locations->SetInAt(0, Location::RequiresRegister()); 6529 locations->SetInAt(1, ArmEncodableConstantOrRegister(instruction->InputAt(1), opcode)); 6530 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 6531} 6532 6533void InstructionCodeGeneratorARM::VisitAnd(HAnd* instruction) { 6534 HandleBitwiseOperation(instruction); 6535} 6536 6537void InstructionCodeGeneratorARM::VisitOr(HOr* instruction) { 6538 HandleBitwiseOperation(instruction); 6539} 6540 6541void InstructionCodeGeneratorARM::VisitXor(HXor* instruction) { 6542 HandleBitwiseOperation(instruction); 6543} 6544 6545 6546void LocationsBuilderARM::VisitBitwiseNegatedRight(HBitwiseNegatedRight* instruction) { 6547 LocationSummary* locations = 6548 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall); 6549 DCHECK(instruction->GetResultType() == Primitive::kPrimInt 6550 || instruction->GetResultType() == Primitive::kPrimLong); 6551 6552 locations->SetInAt(0, Location::RequiresRegister()); 6553 locations->SetInAt(1, Location::RequiresRegister()); 6554 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 6555} 6556 6557void InstructionCodeGeneratorARM::VisitBitwiseNegatedRight(HBitwiseNegatedRight* instruction) { 6558 LocationSummary* locations = instruction->GetLocations(); 6559 Location first = locations->InAt(0); 6560 Location second = locations->InAt(1); 6561 Location out = locations->Out(); 6562 6563 if (instruction->GetResultType() == Primitive::kPrimInt) { 6564 Register first_reg = first.AsRegister<Register>(); 6565 ShifterOperand second_reg(second.AsRegister<Register>()); 6566 Register out_reg = out.AsRegister<Register>(); 6567 6568 switch (instruction->GetOpKind()) { 6569 case HInstruction::kAnd: 6570 __ bic(out_reg, first_reg, second_reg); 6571 break; 6572 case HInstruction::kOr: 6573 __ orn(out_reg, first_reg, second_reg); 6574 break; 6575 // There is no EON on arm. 6576 case HInstruction::kXor: 6577 default: 6578 LOG(FATAL) << "Unexpected instruction " << instruction->DebugName(); 6579 UNREACHABLE(); 6580 } 6581 return; 6582 6583 } else { 6584 DCHECK_EQ(instruction->GetResultType(), Primitive::kPrimLong); 6585 Register first_low = first.AsRegisterPairLow<Register>(); 6586 Register first_high = first.AsRegisterPairHigh<Register>(); 6587 ShifterOperand second_low(second.AsRegisterPairLow<Register>()); 6588 ShifterOperand second_high(second.AsRegisterPairHigh<Register>()); 6589 Register out_low = out.AsRegisterPairLow<Register>(); 6590 Register out_high = out.AsRegisterPairHigh<Register>(); 6591 6592 switch (instruction->GetOpKind()) { 6593 case HInstruction::kAnd: 6594 __ bic(out_low, first_low, second_low); 6595 __ bic(out_high, first_high, second_high); 6596 break; 6597 case HInstruction::kOr: 6598 __ orn(out_low, first_low, second_low); 6599 __ orn(out_high, first_high, second_high); 6600 break; 6601 // There is no EON on arm. 6602 case HInstruction::kXor: 6603 default: 6604 LOG(FATAL) << "Unexpected instruction " << instruction->DebugName(); 6605 UNREACHABLE(); 6606 } 6607 } 6608} 6609 6610void InstructionCodeGeneratorARM::GenerateAndConst(Register out, Register first, uint32_t value) { 6611 // Optimize special cases for individual halfs of `and-long` (`and` is simplified earlier). 6612 if (value == 0xffffffffu) { 6613 if (out != first) { 6614 __ mov(out, ShifterOperand(first)); 6615 } 6616 return; 6617 } 6618 if (value == 0u) { 6619 __ mov(out, ShifterOperand(0)); 6620 return; 6621 } 6622 ShifterOperand so; 6623 if (__ ShifterOperandCanHold(kNoRegister, kNoRegister, AND, value, &so)) { 6624 __ and_(out, first, so); 6625 } else { 6626 DCHECK(__ ShifterOperandCanHold(kNoRegister, kNoRegister, BIC, ~value, &so)); 6627 __ bic(out, first, ShifterOperand(~value)); 6628 } 6629} 6630 6631void InstructionCodeGeneratorARM::GenerateOrrConst(Register out, Register first, uint32_t value) { 6632 // Optimize special cases for individual halfs of `or-long` (`or` is simplified earlier). 6633 if (value == 0u) { 6634 if (out != first) { 6635 __ mov(out, ShifterOperand(first)); 6636 } 6637 return; 6638 } 6639 if (value == 0xffffffffu) { 6640 __ mvn(out, ShifterOperand(0)); 6641 return; 6642 } 6643 ShifterOperand so; 6644 if (__ ShifterOperandCanHold(kNoRegister, kNoRegister, ORR, value, &so)) { 6645 __ orr(out, first, so); 6646 } else { 6647 DCHECK(__ ShifterOperandCanHold(kNoRegister, kNoRegister, ORN, ~value, &so)); 6648 __ orn(out, first, ShifterOperand(~value)); 6649 } 6650} 6651 6652void InstructionCodeGeneratorARM::GenerateEorConst(Register out, Register first, uint32_t value) { 6653 // Optimize special case for individual halfs of `xor-long` (`xor` is simplified earlier). 6654 if (value == 0u) { 6655 if (out != first) { 6656 __ mov(out, ShifterOperand(first)); 6657 } 6658 return; 6659 } 6660 __ eor(out, first, ShifterOperand(value)); 6661} 6662 6663void InstructionCodeGeneratorARM::GenerateAddLongConst(Location out, 6664 Location first, 6665 uint64_t value) { 6666 Register out_low = out.AsRegisterPairLow<Register>(); 6667 Register out_high = out.AsRegisterPairHigh<Register>(); 6668 Register first_low = first.AsRegisterPairLow<Register>(); 6669 Register first_high = first.AsRegisterPairHigh<Register>(); 6670 uint32_t value_low = Low32Bits(value); 6671 uint32_t value_high = High32Bits(value); 6672 if (value_low == 0u) { 6673 if (out_low != first_low) { 6674 __ mov(out_low, ShifterOperand(first_low)); 6675 } 6676 __ AddConstant(out_high, first_high, value_high); 6677 return; 6678 } 6679 __ AddConstantSetFlags(out_low, first_low, value_low); 6680 ShifterOperand so; 6681 if (__ ShifterOperandCanHold(out_high, first_high, ADC, value_high, kCcDontCare, &so)) { 6682 __ adc(out_high, first_high, so); 6683 } else if (__ ShifterOperandCanHold(out_low, first_low, SBC, ~value_high, kCcDontCare, &so)) { 6684 __ sbc(out_high, first_high, so); 6685 } else { 6686 LOG(FATAL) << "Unexpected constant " << value_high; 6687 UNREACHABLE(); 6688 } 6689} 6690 6691void InstructionCodeGeneratorARM::HandleBitwiseOperation(HBinaryOperation* instruction) { 6692 LocationSummary* locations = instruction->GetLocations(); 6693 Location first = locations->InAt(0); 6694 Location second = locations->InAt(1); 6695 Location out = locations->Out(); 6696 6697 if (second.IsConstant()) { 6698 uint64_t value = static_cast<uint64_t>(Int64FromConstant(second.GetConstant())); 6699 uint32_t value_low = Low32Bits(value); 6700 if (instruction->GetResultType() == Primitive::kPrimInt) { 6701 Register first_reg = first.AsRegister<Register>(); 6702 Register out_reg = out.AsRegister<Register>(); 6703 if (instruction->IsAnd()) { 6704 GenerateAndConst(out_reg, first_reg, value_low); 6705 } else if (instruction->IsOr()) { 6706 GenerateOrrConst(out_reg, first_reg, value_low); 6707 } else { 6708 DCHECK(instruction->IsXor()); 6709 GenerateEorConst(out_reg, first_reg, value_low); 6710 } 6711 } else { 6712 DCHECK_EQ(instruction->GetResultType(), Primitive::kPrimLong); 6713 uint32_t value_high = High32Bits(value); 6714 Register first_low = first.AsRegisterPairLow<Register>(); 6715 Register first_high = first.AsRegisterPairHigh<Register>(); 6716 Register out_low = out.AsRegisterPairLow<Register>(); 6717 Register out_high = out.AsRegisterPairHigh<Register>(); 6718 if (instruction->IsAnd()) { 6719 GenerateAndConst(out_low, first_low, value_low); 6720 GenerateAndConst(out_high, first_high, value_high); 6721 } else if (instruction->IsOr()) { 6722 GenerateOrrConst(out_low, first_low, value_low); 6723 GenerateOrrConst(out_high, first_high, value_high); 6724 } else { 6725 DCHECK(instruction->IsXor()); 6726 GenerateEorConst(out_low, first_low, value_low); 6727 GenerateEorConst(out_high, first_high, value_high); 6728 } 6729 } 6730 return; 6731 } 6732 6733 if (instruction->GetResultType() == Primitive::kPrimInt) { 6734 Register first_reg = first.AsRegister<Register>(); 6735 ShifterOperand second_reg(second.AsRegister<Register>()); 6736 Register out_reg = out.AsRegister<Register>(); 6737 if (instruction->IsAnd()) { 6738 __ and_(out_reg, first_reg, second_reg); 6739 } else if (instruction->IsOr()) { 6740 __ orr(out_reg, first_reg, second_reg); 6741 } else { 6742 DCHECK(instruction->IsXor()); 6743 __ eor(out_reg, first_reg, second_reg); 6744 } 6745 } else { 6746 DCHECK_EQ(instruction->GetResultType(), Primitive::kPrimLong); 6747 Register first_low = first.AsRegisterPairLow<Register>(); 6748 Register first_high = first.AsRegisterPairHigh<Register>(); 6749 ShifterOperand second_low(second.AsRegisterPairLow<Register>()); 6750 ShifterOperand second_high(second.AsRegisterPairHigh<Register>()); 6751 Register out_low = out.AsRegisterPairLow<Register>(); 6752 Register out_high = out.AsRegisterPairHigh<Register>(); 6753 if (instruction->IsAnd()) { 6754 __ and_(out_low, first_low, second_low); 6755 __ and_(out_high, first_high, second_high); 6756 } else if (instruction->IsOr()) { 6757 __ orr(out_low, first_low, second_low); 6758 __ orr(out_high, first_high, second_high); 6759 } else { 6760 DCHECK(instruction->IsXor()); 6761 __ eor(out_low, first_low, second_low); 6762 __ eor(out_high, first_high, second_high); 6763 } 6764 } 6765} 6766 6767void InstructionCodeGeneratorARM::GenerateReferenceLoadOneRegister(HInstruction* instruction, 6768 Location out, 6769 uint32_t offset, 6770 Location maybe_temp, 6771 bool emit_read_barrier) { 6772 Register out_reg = out.AsRegister<Register>(); 6773 if (emit_read_barrier) { 6774 CHECK(kEmitCompilerReadBarrier); 6775 DCHECK(maybe_temp.IsRegister()) << maybe_temp; 6776 if (kUseBakerReadBarrier) { 6777 // Load with fast path based Baker's read barrier. 6778 // /* HeapReference<Object> */ out = *(out + offset) 6779 codegen_->GenerateFieldLoadWithBakerReadBarrier( 6780 instruction, out, out_reg, offset, maybe_temp, /* needs_null_check */ false); 6781 } else { 6782 // Load with slow path based read barrier. 6783 // Save the value of `out` into `maybe_temp` before overwriting it 6784 // in the following move operation, as we will need it for the 6785 // read barrier below. 6786 __ Mov(maybe_temp.AsRegister<Register>(), out_reg); 6787 // /* HeapReference<Object> */ out = *(out + offset) 6788 __ LoadFromOffset(kLoadWord, out_reg, out_reg, offset); 6789 codegen_->GenerateReadBarrierSlow(instruction, out, out, maybe_temp, offset); 6790 } 6791 } else { 6792 // Plain load with no read barrier. 6793 // /* HeapReference<Object> */ out = *(out + offset) 6794 __ LoadFromOffset(kLoadWord, out_reg, out_reg, offset); 6795 __ MaybeUnpoisonHeapReference(out_reg); 6796 } 6797} 6798 6799void InstructionCodeGeneratorARM::GenerateReferenceLoadTwoRegisters(HInstruction* instruction, 6800 Location out, 6801 Location obj, 6802 uint32_t offset, 6803 Location maybe_temp, 6804 bool emit_read_barrier) { 6805 Register out_reg = out.AsRegister<Register>(); 6806 Register obj_reg = obj.AsRegister<Register>(); 6807 if (emit_read_barrier) { 6808 CHECK(kEmitCompilerReadBarrier); 6809 if (kUseBakerReadBarrier) { 6810 DCHECK(maybe_temp.IsRegister()) << maybe_temp; 6811 // Load with fast path based Baker's read barrier. 6812 // /* HeapReference<Object> */ out = *(obj + offset) 6813 codegen_->GenerateFieldLoadWithBakerReadBarrier( 6814 instruction, out, obj_reg, offset, maybe_temp, /* needs_null_check */ false); 6815 } else { 6816 // Load with slow path based read barrier. 6817 // /* HeapReference<Object> */ out = *(obj + offset) 6818 __ LoadFromOffset(kLoadWord, out_reg, obj_reg, offset); 6819 codegen_->GenerateReadBarrierSlow(instruction, out, out, obj, offset); 6820 } 6821 } else { 6822 // Plain load with no read barrier. 6823 // /* HeapReference<Object> */ out = *(obj + offset) 6824 __ LoadFromOffset(kLoadWord, out_reg, obj_reg, offset); 6825 __ MaybeUnpoisonHeapReference(out_reg); 6826 } 6827} 6828 6829void InstructionCodeGeneratorARM::GenerateGcRootFieldLoad(HInstruction* instruction, 6830 Location root, 6831 Register obj, 6832 uint32_t offset, 6833 bool requires_read_barrier) { 6834 Register root_reg = root.AsRegister<Register>(); 6835 if (requires_read_barrier) { 6836 DCHECK(kEmitCompilerReadBarrier); 6837 if (kUseBakerReadBarrier) { 6838 // Fast path implementation of art::ReadBarrier::BarrierForRoot when 6839 // Baker's read barrier are used: 6840 // 6841 // root = obj.field; 6842 // temp = Thread::Current()->pReadBarrierMarkReg ## root.reg() 6843 // if (temp != null) { 6844 // root = temp(root) 6845 // } 6846 6847 // /* GcRoot<mirror::Object> */ root = *(obj + offset) 6848 __ LoadFromOffset(kLoadWord, root_reg, obj, offset); 6849 static_assert( 6850 sizeof(mirror::CompressedReference<mirror::Object>) == sizeof(GcRoot<mirror::Object>), 6851 "art::mirror::CompressedReference<mirror::Object> and art::GcRoot<mirror::Object> " 6852 "have different sizes."); 6853 static_assert(sizeof(mirror::CompressedReference<mirror::Object>) == sizeof(int32_t), 6854 "art::mirror::CompressedReference<mirror::Object> and int32_t " 6855 "have different sizes."); 6856 6857 // Slow path marking the GC root `root`. 6858 Location temp = Location::RegisterLocation(LR); 6859 SlowPathCodeARM* slow_path = 6860 new (GetGraph()->GetArena()) ReadBarrierMarkSlowPathARM( 6861 instruction, 6862 root, 6863 /*entrypoint*/ temp); 6864 codegen_->AddSlowPath(slow_path); 6865 6866 // temp = Thread::Current()->pReadBarrierMarkReg ## root.reg() 6867 const int32_t entry_point_offset = 6868 CodeGenerator::GetReadBarrierMarkEntryPointsOffset<kArmPointerSize>(root.reg()); 6869 // Loading the entrypoint does not require a load acquire since it is only changed when 6870 // threads are suspended or running a checkpoint. 6871 __ LoadFromOffset(kLoadWord, temp.AsRegister<Register>(), TR, entry_point_offset); 6872 // The entrypoint is null when the GC is not marking, this prevents one load compared to 6873 // checking GetIsGcMarking. 6874 __ CompareAndBranchIfNonZero(temp.AsRegister<Register>(), slow_path->GetEntryLabel()); 6875 __ Bind(slow_path->GetExitLabel()); 6876 } else { 6877 // GC root loaded through a slow path for read barriers other 6878 // than Baker's. 6879 // /* GcRoot<mirror::Object>* */ root = obj + offset 6880 __ AddConstant(root_reg, obj, offset); 6881 // /* mirror::Object* */ root = root->Read() 6882 codegen_->GenerateReadBarrierForRootSlow(instruction, root, root); 6883 } 6884 } else { 6885 // Plain GC root load with no read barrier. 6886 // /* GcRoot<mirror::Object> */ root = *(obj + offset) 6887 __ LoadFromOffset(kLoadWord, root_reg, obj, offset); 6888 // Note that GC roots are not affected by heap poisoning, thus we 6889 // do not have to unpoison `root_reg` here. 6890 } 6891} 6892 6893void CodeGeneratorARM::GenerateFieldLoadWithBakerReadBarrier(HInstruction* instruction, 6894 Location ref, 6895 Register obj, 6896 uint32_t offset, 6897 Location temp, 6898 bool needs_null_check) { 6899 DCHECK(kEmitCompilerReadBarrier); 6900 DCHECK(kUseBakerReadBarrier); 6901 6902 // /* HeapReference<Object> */ ref = *(obj + offset) 6903 Location no_index = Location::NoLocation(); 6904 ScaleFactor no_scale_factor = TIMES_1; 6905 GenerateReferenceLoadWithBakerReadBarrier( 6906 instruction, ref, obj, offset, no_index, no_scale_factor, temp, needs_null_check); 6907} 6908 6909void CodeGeneratorARM::GenerateArrayLoadWithBakerReadBarrier(HInstruction* instruction, 6910 Location ref, 6911 Register obj, 6912 uint32_t data_offset, 6913 Location index, 6914 Location temp, 6915 bool needs_null_check) { 6916 DCHECK(kEmitCompilerReadBarrier); 6917 DCHECK(kUseBakerReadBarrier); 6918 6919 static_assert( 6920 sizeof(mirror::HeapReference<mirror::Object>) == sizeof(int32_t), 6921 "art::mirror::HeapReference<art::mirror::Object> and int32_t have different sizes."); 6922 // /* HeapReference<Object> */ ref = 6923 // *(obj + data_offset + index * sizeof(HeapReference<Object>)) 6924 ScaleFactor scale_factor = TIMES_4; 6925 GenerateReferenceLoadWithBakerReadBarrier( 6926 instruction, ref, obj, data_offset, index, scale_factor, temp, needs_null_check); 6927} 6928 6929void CodeGeneratorARM::GenerateReferenceLoadWithBakerReadBarrier(HInstruction* instruction, 6930 Location ref, 6931 Register obj, 6932 uint32_t offset, 6933 Location index, 6934 ScaleFactor scale_factor, 6935 Location temp, 6936 bool needs_null_check, 6937 bool always_update_field, 6938 Register* temp2) { 6939 DCHECK(kEmitCompilerReadBarrier); 6940 DCHECK(kUseBakerReadBarrier); 6941 6942 // In slow path based read barriers, the read barrier call is 6943 // inserted after the original load. However, in fast path based 6944 // Baker's read barriers, we need to perform the load of 6945 // mirror::Object::monitor_ *before* the original reference load. 6946 // This load-load ordering is required by the read barrier. 6947 // The fast path/slow path (for Baker's algorithm) should look like: 6948 // 6949 // uint32_t rb_state = Lockword(obj->monitor_).ReadBarrierState(); 6950 // lfence; // Load fence or artificial data dependency to prevent load-load reordering 6951 // HeapReference<Object> ref = *src; // Original reference load. 6952 // bool is_gray = (rb_state == ReadBarrier::GrayState()); 6953 // if (is_gray) { 6954 // ref = ReadBarrier::Mark(ref); // Performed by runtime entrypoint slow path. 6955 // } 6956 // 6957 // Note: the original implementation in ReadBarrier::Barrier is 6958 // slightly more complex as it performs additional checks that we do 6959 // not do here for performance reasons. 6960 6961 Register ref_reg = ref.AsRegister<Register>(); 6962 Register temp_reg = temp.AsRegister<Register>(); 6963 uint32_t monitor_offset = mirror::Object::MonitorOffset().Int32Value(); 6964 6965 // /* int32_t */ monitor = obj->monitor_ 6966 __ LoadFromOffset(kLoadWord, temp_reg, obj, monitor_offset); 6967 if (needs_null_check) { 6968 MaybeRecordImplicitNullCheck(instruction); 6969 } 6970 // /* LockWord */ lock_word = LockWord(monitor) 6971 static_assert(sizeof(LockWord) == sizeof(int32_t), 6972 "art::LockWord and int32_t have different sizes."); 6973 6974 // Introduce a dependency on the lock_word including the rb_state, 6975 // which shall prevent load-load reordering without using 6976 // a memory barrier (which would be more expensive). 6977 // `obj` is unchanged by this operation, but its value now depends 6978 // on `temp_reg`. 6979 __ add(obj, obj, ShifterOperand(temp_reg, LSR, 32)); 6980 6981 // The actual reference load. 6982 if (index.IsValid()) { 6983 // Load types involving an "index": ArrayGet, 6984 // UnsafeGetObject/UnsafeGetObjectVolatile and UnsafeCASObject 6985 // intrinsics. 6986 // /* HeapReference<Object> */ ref = *(obj + offset + (index << scale_factor)) 6987 if (index.IsConstant()) { 6988 size_t computed_offset = 6989 (index.GetConstant()->AsIntConstant()->GetValue() << scale_factor) + offset; 6990 __ LoadFromOffset(kLoadWord, ref_reg, obj, computed_offset); 6991 } else { 6992 // Handle the special case of the 6993 // UnsafeGetObject/UnsafeGetObjectVolatile and UnsafeCASObject 6994 // intrinsics, which use a register pair as index ("long 6995 // offset"), of which only the low part contains data. 6996 Register index_reg = index.IsRegisterPair() 6997 ? index.AsRegisterPairLow<Register>() 6998 : index.AsRegister<Register>(); 6999 __ add(IP, obj, ShifterOperand(index_reg, LSL, scale_factor)); 7000 __ LoadFromOffset(kLoadWord, ref_reg, IP, offset); 7001 } 7002 } else { 7003 // /* HeapReference<Object> */ ref = *(obj + offset) 7004 __ LoadFromOffset(kLoadWord, ref_reg, obj, offset); 7005 } 7006 7007 // Object* ref = ref_addr->AsMirrorPtr() 7008 __ MaybeUnpoisonHeapReference(ref_reg); 7009 7010 // Slow path marking the object `ref` when it is gray. 7011 SlowPathCodeARM* slow_path; 7012 if (always_update_field) { 7013 DCHECK(temp2 != nullptr); 7014 // ReadBarrierMarkAndUpdateFieldSlowPathARM only supports address 7015 // of the form `obj + field_offset`, where `obj` is a register and 7016 // `field_offset` is a register pair (of which only the lower half 7017 // is used). Thus `offset` and `scale_factor` above are expected 7018 // to be null in this code path. 7019 DCHECK_EQ(offset, 0u); 7020 DCHECK_EQ(scale_factor, ScaleFactor::TIMES_1); 7021 slow_path = new (GetGraph()->GetArena()) ReadBarrierMarkAndUpdateFieldSlowPathARM( 7022 instruction, ref, obj, /* field_offset */ index, temp_reg, *temp2); 7023 } else { 7024 slow_path = new (GetGraph()->GetArena()) ReadBarrierMarkSlowPathARM(instruction, ref); 7025 } 7026 AddSlowPath(slow_path); 7027 7028 // if (rb_state == ReadBarrier::GrayState()) 7029 // ref = ReadBarrier::Mark(ref); 7030 // Given the numeric representation, it's enough to check the low bit of the 7031 // rb_state. We do that by shifting the bit out of the lock word with LSRS 7032 // which can be a 16-bit instruction unlike the TST immediate. 7033 static_assert(ReadBarrier::WhiteState() == 0, "Expecting white to have value 0"); 7034 static_assert(ReadBarrier::GrayState() == 1, "Expecting gray to have value 1"); 7035 __ Lsrs(temp_reg, temp_reg, LockWord::kReadBarrierStateShift + 1); 7036 __ b(slow_path->GetEntryLabel(), CS); // Carry flag is the last bit shifted out by LSRS. 7037 __ Bind(slow_path->GetExitLabel()); 7038} 7039 7040void CodeGeneratorARM::GenerateReadBarrierSlow(HInstruction* instruction, 7041 Location out, 7042 Location ref, 7043 Location obj, 7044 uint32_t offset, 7045 Location index) { 7046 DCHECK(kEmitCompilerReadBarrier); 7047 7048 // Insert a slow path based read barrier *after* the reference load. 7049 // 7050 // If heap poisoning is enabled, the unpoisoning of the loaded 7051 // reference will be carried out by the runtime within the slow 7052 // path. 7053 // 7054 // Note that `ref` currently does not get unpoisoned (when heap 7055 // poisoning is enabled), which is alright as the `ref` argument is 7056 // not used by the artReadBarrierSlow entry point. 7057 // 7058 // TODO: Unpoison `ref` when it is used by artReadBarrierSlow. 7059 SlowPathCodeARM* slow_path = new (GetGraph()->GetArena()) 7060 ReadBarrierForHeapReferenceSlowPathARM(instruction, out, ref, obj, offset, index); 7061 AddSlowPath(slow_path); 7062 7063 __ b(slow_path->GetEntryLabel()); 7064 __ Bind(slow_path->GetExitLabel()); 7065} 7066 7067void CodeGeneratorARM::MaybeGenerateReadBarrierSlow(HInstruction* instruction, 7068 Location out, 7069 Location ref, 7070 Location obj, 7071 uint32_t offset, 7072 Location index) { 7073 if (kEmitCompilerReadBarrier) { 7074 // Baker's read barriers shall be handled by the fast path 7075 // (CodeGeneratorARM::GenerateReferenceLoadWithBakerReadBarrier). 7076 DCHECK(!kUseBakerReadBarrier); 7077 // If heap poisoning is enabled, unpoisoning will be taken care of 7078 // by the runtime within the slow path. 7079 GenerateReadBarrierSlow(instruction, out, ref, obj, offset, index); 7080 } else if (kPoisonHeapReferences) { 7081 __ UnpoisonHeapReference(out.AsRegister<Register>()); 7082 } 7083} 7084 7085void CodeGeneratorARM::GenerateReadBarrierForRootSlow(HInstruction* instruction, 7086 Location out, 7087 Location root) { 7088 DCHECK(kEmitCompilerReadBarrier); 7089 7090 // Insert a slow path based read barrier *after* the GC root load. 7091 // 7092 // Note that GC roots are not affected by heap poisoning, so we do 7093 // not need to do anything special for this here. 7094 SlowPathCodeARM* slow_path = 7095 new (GetGraph()->GetArena()) ReadBarrierForRootSlowPathARM(instruction, out, root); 7096 AddSlowPath(slow_path); 7097 7098 __ b(slow_path->GetEntryLabel()); 7099 __ Bind(slow_path->GetExitLabel()); 7100} 7101 7102HInvokeStaticOrDirect::DispatchInfo CodeGeneratorARM::GetSupportedInvokeStaticOrDirectDispatch( 7103 const HInvokeStaticOrDirect::DispatchInfo& desired_dispatch_info, 7104 HInvokeStaticOrDirect* invoke) { 7105 HInvokeStaticOrDirect::DispatchInfo dispatch_info = desired_dispatch_info; 7106 // We disable pc-relative load when there is an irreducible loop, as the optimization 7107 // is incompatible with it. 7108 // TODO: Create as many ArmDexCacheArraysBase instructions as needed for methods 7109 // with irreducible loops. 7110 if (GetGraph()->HasIrreducibleLoops() && 7111 (dispatch_info.method_load_kind == 7112 HInvokeStaticOrDirect::MethodLoadKind::kDexCachePcRelative)) { 7113 dispatch_info.method_load_kind = HInvokeStaticOrDirect::MethodLoadKind::kDexCacheViaMethod; 7114 } 7115 7116 if (dispatch_info.code_ptr_location == HInvokeStaticOrDirect::CodePtrLocation::kCallPCRelative) { 7117 const DexFile& outer_dex_file = GetGraph()->GetDexFile(); 7118 if (&outer_dex_file != invoke->GetTargetMethod().dex_file) { 7119 // Calls across dex files are more likely to exceed the available BL range, 7120 // so use absolute patch with fixup if available and kCallArtMethod otherwise. 7121 HInvokeStaticOrDirect::CodePtrLocation code_ptr_location = 7122 (desired_dispatch_info.method_load_kind == 7123 HInvokeStaticOrDirect::MethodLoadKind::kDirectAddressWithFixup) 7124 ? HInvokeStaticOrDirect::CodePtrLocation::kCallDirectWithFixup 7125 : HInvokeStaticOrDirect::CodePtrLocation::kCallArtMethod; 7126 return HInvokeStaticOrDirect::DispatchInfo { 7127 dispatch_info.method_load_kind, 7128 code_ptr_location, 7129 dispatch_info.method_load_data, 7130 0u 7131 }; 7132 } 7133 } 7134 return dispatch_info; 7135} 7136 7137Register CodeGeneratorARM::GetInvokeStaticOrDirectExtraParameter(HInvokeStaticOrDirect* invoke, 7138 Register temp) { 7139 DCHECK_EQ(invoke->InputCount(), invoke->GetNumberOfArguments() + 1u); 7140 Location location = invoke->GetLocations()->InAt(invoke->GetSpecialInputIndex()); 7141 if (!invoke->GetLocations()->Intrinsified()) { 7142 return location.AsRegister<Register>(); 7143 } 7144 // For intrinsics we allow any location, so it may be on the stack. 7145 if (!location.IsRegister()) { 7146 __ LoadFromOffset(kLoadWord, temp, SP, location.GetStackIndex()); 7147 return temp; 7148 } 7149 // For register locations, check if the register was saved. If so, get it from the stack. 7150 // Note: There is a chance that the register was saved but not overwritten, so we could 7151 // save one load. However, since this is just an intrinsic slow path we prefer this 7152 // simple and more robust approach rather that trying to determine if that's the case. 7153 SlowPathCode* slow_path = GetCurrentSlowPath(); 7154 DCHECK(slow_path != nullptr); // For intrinsified invokes the call is emitted on the slow path. 7155 if (slow_path->IsCoreRegisterSaved(location.AsRegister<Register>())) { 7156 int stack_offset = slow_path->GetStackOffsetOfCoreRegister(location.AsRegister<Register>()); 7157 __ LoadFromOffset(kLoadWord, temp, SP, stack_offset); 7158 return temp; 7159 } 7160 return location.AsRegister<Register>(); 7161} 7162 7163void CodeGeneratorARM::GenerateStaticOrDirectCall(HInvokeStaticOrDirect* invoke, Location temp) { 7164 // For better instruction scheduling we load the direct code pointer before the method pointer. 7165 switch (invoke->GetCodePtrLocation()) { 7166 case HInvokeStaticOrDirect::CodePtrLocation::kCallDirectWithFixup: 7167 // LR = code address from literal pool with link-time patch. 7168 __ LoadLiteral(LR, DeduplicateMethodCodeLiteral(invoke->GetTargetMethod())); 7169 break; 7170 case HInvokeStaticOrDirect::CodePtrLocation::kCallDirect: 7171 // LR = invoke->GetDirectCodePtr(); 7172 __ LoadImmediate(LR, invoke->GetDirectCodePtr()); 7173 break; 7174 default: 7175 break; 7176 } 7177 7178 Location callee_method = temp; // For all kinds except kRecursive, callee will be in temp. 7179 switch (invoke->GetMethodLoadKind()) { 7180 case HInvokeStaticOrDirect::MethodLoadKind::kStringInit: { 7181 uint32_t offset = 7182 GetThreadOffset<kArmPointerSize>(invoke->GetStringInitEntryPoint()).Int32Value(); 7183 // temp = thread->string_init_entrypoint 7184 __ LoadFromOffset(kLoadWord, temp.AsRegister<Register>(), TR, offset); 7185 break; 7186 } 7187 case HInvokeStaticOrDirect::MethodLoadKind::kRecursive: 7188 callee_method = invoke->GetLocations()->InAt(invoke->GetSpecialInputIndex()); 7189 break; 7190 case HInvokeStaticOrDirect::MethodLoadKind::kDirectAddress: 7191 __ LoadImmediate(temp.AsRegister<Register>(), invoke->GetMethodAddress()); 7192 break; 7193 case HInvokeStaticOrDirect::MethodLoadKind::kDirectAddressWithFixup: 7194 __ LoadLiteral(temp.AsRegister<Register>(), 7195 DeduplicateMethodAddressLiteral(invoke->GetTargetMethod())); 7196 break; 7197 case HInvokeStaticOrDirect::MethodLoadKind::kDexCachePcRelative: { 7198 HArmDexCacheArraysBase* base = 7199 invoke->InputAt(invoke->GetSpecialInputIndex())->AsArmDexCacheArraysBase(); 7200 Register base_reg = GetInvokeStaticOrDirectExtraParameter(invoke, 7201 temp.AsRegister<Register>()); 7202 int32_t offset = invoke->GetDexCacheArrayOffset() - base->GetElementOffset(); 7203 __ LoadFromOffset(kLoadWord, temp.AsRegister<Register>(), base_reg, offset); 7204 break; 7205 } 7206 case HInvokeStaticOrDirect::MethodLoadKind::kDexCacheViaMethod: { 7207 Location current_method = invoke->GetLocations()->InAt(invoke->GetSpecialInputIndex()); 7208 Register method_reg; 7209 Register reg = temp.AsRegister<Register>(); 7210 if (current_method.IsRegister()) { 7211 method_reg = current_method.AsRegister<Register>(); 7212 } else { 7213 DCHECK(invoke->GetLocations()->Intrinsified()); 7214 DCHECK(!current_method.IsValid()); 7215 method_reg = reg; 7216 __ LoadFromOffset(kLoadWord, reg, SP, kCurrentMethodStackOffset); 7217 } 7218 // /* ArtMethod*[] */ temp = temp.ptr_sized_fields_->dex_cache_resolved_methods_; 7219 __ LoadFromOffset(kLoadWord, 7220 reg, 7221 method_reg, 7222 ArtMethod::DexCacheResolvedMethodsOffset(kArmPointerSize).Int32Value()); 7223 // temp = temp[index_in_cache]; 7224 // Note: Don't use invoke->GetTargetMethod() as it may point to a different dex file. 7225 uint32_t index_in_cache = invoke->GetDexMethodIndex(); 7226 __ LoadFromOffset(kLoadWord, reg, reg, CodeGenerator::GetCachePointerOffset(index_in_cache)); 7227 break; 7228 } 7229 } 7230 7231 switch (invoke->GetCodePtrLocation()) { 7232 case HInvokeStaticOrDirect::CodePtrLocation::kCallSelf: 7233 __ bl(GetFrameEntryLabel()); 7234 break; 7235 case HInvokeStaticOrDirect::CodePtrLocation::kCallPCRelative: 7236 relative_call_patches_.emplace_back(*invoke->GetTargetMethod().dex_file, 7237 invoke->GetTargetMethod().dex_method_index); 7238 __ BindTrackedLabel(&relative_call_patches_.back().label); 7239 // Arbitrarily branch to the BL itself, override at link time. 7240 __ bl(&relative_call_patches_.back().label); 7241 break; 7242 case HInvokeStaticOrDirect::CodePtrLocation::kCallDirectWithFixup: 7243 case HInvokeStaticOrDirect::CodePtrLocation::kCallDirect: 7244 // LR prepared above for better instruction scheduling. 7245 // LR() 7246 __ blx(LR); 7247 break; 7248 case HInvokeStaticOrDirect::CodePtrLocation::kCallArtMethod: 7249 // LR = callee_method->entry_point_from_quick_compiled_code_ 7250 __ LoadFromOffset( 7251 kLoadWord, LR, callee_method.AsRegister<Register>(), 7252 ArtMethod::EntryPointFromQuickCompiledCodeOffset(kArmPointerSize).Int32Value()); 7253 // LR() 7254 __ blx(LR); 7255 break; 7256 } 7257 7258 DCHECK(!IsLeafMethod()); 7259} 7260 7261void CodeGeneratorARM::GenerateVirtualCall(HInvokeVirtual* invoke, Location temp_location) { 7262 Register temp = temp_location.AsRegister<Register>(); 7263 uint32_t method_offset = mirror::Class::EmbeddedVTableEntryOffset( 7264 invoke->GetVTableIndex(), kArmPointerSize).Uint32Value(); 7265 7266 // Use the calling convention instead of the location of the receiver, as 7267 // intrinsics may have put the receiver in a different register. In the intrinsics 7268 // slow path, the arguments have been moved to the right place, so here we are 7269 // guaranteed that the receiver is the first register of the calling convention. 7270 InvokeDexCallingConvention calling_convention; 7271 Register receiver = calling_convention.GetRegisterAt(0); 7272 uint32_t class_offset = mirror::Object::ClassOffset().Int32Value(); 7273 // /* HeapReference<Class> */ temp = receiver->klass_ 7274 __ LoadFromOffset(kLoadWord, temp, receiver, class_offset); 7275 MaybeRecordImplicitNullCheck(invoke); 7276 // Instead of simply (possibly) unpoisoning `temp` here, we should 7277 // emit a read barrier for the previous class reference load. 7278 // However this is not required in practice, as this is an 7279 // intermediate/temporary reference and because the current 7280 // concurrent copying collector keeps the from-space memory 7281 // intact/accessible until the end of the marking phase (the 7282 // concurrent copying collector may not in the future). 7283 __ MaybeUnpoisonHeapReference(temp); 7284 // temp = temp->GetMethodAt(method_offset); 7285 uint32_t entry_point = ArtMethod::EntryPointFromQuickCompiledCodeOffset( 7286 kArmPointerSize).Int32Value(); 7287 __ LoadFromOffset(kLoadWord, temp, temp, method_offset); 7288 // LR = temp->GetEntryPoint(); 7289 __ LoadFromOffset(kLoadWord, LR, temp, entry_point); 7290 // LR(); 7291 __ blx(LR); 7292} 7293 7294CodeGeneratorARM::PcRelativePatchInfo* CodeGeneratorARM::NewPcRelativeStringPatch( 7295 const DexFile& dex_file, uint32_t string_index) { 7296 return NewPcRelativePatch(dex_file, string_index, &pc_relative_string_patches_); 7297} 7298 7299CodeGeneratorARM::PcRelativePatchInfo* CodeGeneratorARM::NewPcRelativeTypePatch( 7300 const DexFile& dex_file, uint32_t type_index) { 7301 return NewPcRelativePatch(dex_file, type_index, &pc_relative_type_patches_); 7302} 7303 7304CodeGeneratorARM::PcRelativePatchInfo* CodeGeneratorARM::NewPcRelativeDexCacheArrayPatch( 7305 const DexFile& dex_file, uint32_t element_offset) { 7306 return NewPcRelativePatch(dex_file, element_offset, &pc_relative_dex_cache_patches_); 7307} 7308 7309CodeGeneratorARM::PcRelativePatchInfo* CodeGeneratorARM::NewPcRelativePatch( 7310 const DexFile& dex_file, uint32_t offset_or_index, ArenaDeque<PcRelativePatchInfo>* patches) { 7311 patches->emplace_back(dex_file, offset_or_index); 7312 return &patches->back(); 7313} 7314 7315Literal* CodeGeneratorARM::DeduplicateBootImageStringLiteral(const DexFile& dex_file, 7316 uint32_t string_index) { 7317 return boot_image_string_patches_.GetOrCreate( 7318 StringReference(&dex_file, string_index), 7319 [this]() { return __ NewLiteral<uint32_t>(/* placeholder */ 0u); }); 7320} 7321 7322Literal* CodeGeneratorARM::DeduplicateBootImageTypeLiteral(const DexFile& dex_file, 7323 uint32_t type_index) { 7324 return boot_image_type_patches_.GetOrCreate( 7325 TypeReference(&dex_file, type_index), 7326 [this]() { return __ NewLiteral<uint32_t>(/* placeholder */ 0u); }); 7327} 7328 7329Literal* CodeGeneratorARM::DeduplicateBootImageAddressLiteral(uint32_t address) { 7330 bool needs_patch = GetCompilerOptions().GetIncludePatchInformation(); 7331 Uint32ToLiteralMap* map = needs_patch ? &boot_image_address_patches_ : &uint32_literals_; 7332 return DeduplicateUint32Literal(dchecked_integral_cast<uint32_t>(address), map); 7333} 7334 7335Literal* CodeGeneratorARM::DeduplicateDexCacheAddressLiteral(uint32_t address) { 7336 return DeduplicateUint32Literal(address, &uint32_literals_); 7337} 7338 7339template <LinkerPatch (*Factory)(size_t, const DexFile*, uint32_t, uint32_t)> 7340inline void CodeGeneratorARM::EmitPcRelativeLinkerPatches( 7341 const ArenaDeque<PcRelativePatchInfo>& infos, 7342 ArenaVector<LinkerPatch>* linker_patches) { 7343 for (const PcRelativePatchInfo& info : infos) { 7344 const DexFile& dex_file = info.target_dex_file; 7345 size_t offset_or_index = info.offset_or_index; 7346 DCHECK(info.add_pc_label.IsBound()); 7347 uint32_t add_pc_offset = dchecked_integral_cast<uint32_t>(info.add_pc_label.Position()); 7348 // Add MOVW patch. 7349 DCHECK(info.movw_label.IsBound()); 7350 uint32_t movw_offset = dchecked_integral_cast<uint32_t>(info.movw_label.Position()); 7351 linker_patches->push_back(Factory(movw_offset, &dex_file, add_pc_offset, offset_or_index)); 7352 // Add MOVT patch. 7353 DCHECK(info.movt_label.IsBound()); 7354 uint32_t movt_offset = dchecked_integral_cast<uint32_t>(info.movt_label.Position()); 7355 linker_patches->push_back(Factory(movt_offset, &dex_file, add_pc_offset, offset_or_index)); 7356 } 7357} 7358 7359void CodeGeneratorARM::EmitLinkerPatches(ArenaVector<LinkerPatch>* linker_patches) { 7360 DCHECK(linker_patches->empty()); 7361 size_t size = 7362 method_patches_.size() + 7363 call_patches_.size() + 7364 relative_call_patches_.size() + 7365 /* MOVW+MOVT for each entry */ 2u * pc_relative_dex_cache_patches_.size() + 7366 boot_image_string_patches_.size() + 7367 /* MOVW+MOVT for each entry */ 2u * pc_relative_string_patches_.size() + 7368 boot_image_type_patches_.size() + 7369 /* MOVW+MOVT for each entry */ 2u * pc_relative_type_patches_.size() + 7370 boot_image_address_patches_.size(); 7371 linker_patches->reserve(size); 7372 for (const auto& entry : method_patches_) { 7373 const MethodReference& target_method = entry.first; 7374 Literal* literal = entry.second; 7375 DCHECK(literal->GetLabel()->IsBound()); 7376 uint32_t literal_offset = literal->GetLabel()->Position(); 7377 linker_patches->push_back(LinkerPatch::MethodPatch(literal_offset, 7378 target_method.dex_file, 7379 target_method.dex_method_index)); 7380 } 7381 for (const auto& entry : call_patches_) { 7382 const MethodReference& target_method = entry.first; 7383 Literal* literal = entry.second; 7384 DCHECK(literal->GetLabel()->IsBound()); 7385 uint32_t literal_offset = literal->GetLabel()->Position(); 7386 linker_patches->push_back(LinkerPatch::CodePatch(literal_offset, 7387 target_method.dex_file, 7388 target_method.dex_method_index)); 7389 } 7390 for (const PatchInfo<Label>& info : relative_call_patches_) { 7391 uint32_t literal_offset = info.label.Position(); 7392 linker_patches->push_back( 7393 LinkerPatch::RelativeCodePatch(literal_offset, &info.dex_file, info.index)); 7394 } 7395 EmitPcRelativeLinkerPatches<LinkerPatch::DexCacheArrayPatch>(pc_relative_dex_cache_patches_, 7396 linker_patches); 7397 for (const auto& entry : boot_image_string_patches_) { 7398 const StringReference& target_string = entry.first; 7399 Literal* literal = entry.second; 7400 DCHECK(literal->GetLabel()->IsBound()); 7401 uint32_t literal_offset = literal->GetLabel()->Position(); 7402 linker_patches->push_back(LinkerPatch::StringPatch(literal_offset, 7403 target_string.dex_file, 7404 target_string.string_index)); 7405 } 7406 if (!GetCompilerOptions().IsBootImage()) { 7407 EmitPcRelativeLinkerPatches<LinkerPatch::StringBssEntryPatch>(pc_relative_string_patches_, 7408 linker_patches); 7409 } else { 7410 EmitPcRelativeLinkerPatches<LinkerPatch::RelativeStringPatch>(pc_relative_string_patches_, 7411 linker_patches); 7412 } 7413 for (const auto& entry : boot_image_type_patches_) { 7414 const TypeReference& target_type = entry.first; 7415 Literal* literal = entry.second; 7416 DCHECK(literal->GetLabel()->IsBound()); 7417 uint32_t literal_offset = literal->GetLabel()->Position(); 7418 linker_patches->push_back(LinkerPatch::TypePatch(literal_offset, 7419 target_type.dex_file, 7420 target_type.type_index)); 7421 } 7422 EmitPcRelativeLinkerPatches<LinkerPatch::RelativeTypePatch>(pc_relative_type_patches_, 7423 linker_patches); 7424 for (const auto& entry : boot_image_address_patches_) { 7425 DCHECK(GetCompilerOptions().GetIncludePatchInformation()); 7426 Literal* literal = entry.second; 7427 DCHECK(literal->GetLabel()->IsBound()); 7428 uint32_t literal_offset = literal->GetLabel()->Position(); 7429 linker_patches->push_back(LinkerPatch::RecordPosition(literal_offset)); 7430 } 7431} 7432 7433Literal* CodeGeneratorARM::DeduplicateUint32Literal(uint32_t value, Uint32ToLiteralMap* map) { 7434 return map->GetOrCreate( 7435 value, 7436 [this, value]() { return __ NewLiteral<uint32_t>(value); }); 7437} 7438 7439Literal* CodeGeneratorARM::DeduplicateMethodLiteral(MethodReference target_method, 7440 MethodToLiteralMap* map) { 7441 return map->GetOrCreate( 7442 target_method, 7443 [this]() { return __ NewLiteral<uint32_t>(/* placeholder */ 0u); }); 7444} 7445 7446Literal* CodeGeneratorARM::DeduplicateMethodAddressLiteral(MethodReference target_method) { 7447 return DeduplicateMethodLiteral(target_method, &method_patches_); 7448} 7449 7450Literal* CodeGeneratorARM::DeduplicateMethodCodeLiteral(MethodReference target_method) { 7451 return DeduplicateMethodLiteral(target_method, &call_patches_); 7452} 7453 7454void LocationsBuilderARM::VisitMultiplyAccumulate(HMultiplyAccumulate* instr) { 7455 LocationSummary* locations = 7456 new (GetGraph()->GetArena()) LocationSummary(instr, LocationSummary::kNoCall); 7457 locations->SetInAt(HMultiplyAccumulate::kInputAccumulatorIndex, 7458 Location::RequiresRegister()); 7459 locations->SetInAt(HMultiplyAccumulate::kInputMulLeftIndex, Location::RequiresRegister()); 7460 locations->SetInAt(HMultiplyAccumulate::kInputMulRightIndex, Location::RequiresRegister()); 7461 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 7462} 7463 7464void InstructionCodeGeneratorARM::VisitMultiplyAccumulate(HMultiplyAccumulate* instr) { 7465 LocationSummary* locations = instr->GetLocations(); 7466 Register res = locations->Out().AsRegister<Register>(); 7467 Register accumulator = 7468 locations->InAt(HMultiplyAccumulate::kInputAccumulatorIndex).AsRegister<Register>(); 7469 Register mul_left = 7470 locations->InAt(HMultiplyAccumulate::kInputMulLeftIndex).AsRegister<Register>(); 7471 Register mul_right = 7472 locations->InAt(HMultiplyAccumulate::kInputMulRightIndex).AsRegister<Register>(); 7473 7474 if (instr->GetOpKind() == HInstruction::kAdd) { 7475 __ mla(res, mul_left, mul_right, accumulator); 7476 } else { 7477 __ mls(res, mul_left, mul_right, accumulator); 7478 } 7479} 7480 7481void LocationsBuilderARM::VisitBoundType(HBoundType* instruction ATTRIBUTE_UNUSED) { 7482 // Nothing to do, this should be removed during prepare for register allocator. 7483 LOG(FATAL) << "Unreachable"; 7484} 7485 7486void InstructionCodeGeneratorARM::VisitBoundType(HBoundType* instruction ATTRIBUTE_UNUSED) { 7487 // Nothing to do, this should be removed during prepare for register allocator. 7488 LOG(FATAL) << "Unreachable"; 7489} 7490 7491// Simple implementation of packed switch - generate cascaded compare/jumps. 7492void LocationsBuilderARM::VisitPackedSwitch(HPackedSwitch* switch_instr) { 7493 LocationSummary* locations = 7494 new (GetGraph()->GetArena()) LocationSummary(switch_instr, LocationSummary::kNoCall); 7495 locations->SetInAt(0, Location::RequiresRegister()); 7496 if (switch_instr->GetNumEntries() > kPackedSwitchCompareJumpThreshold && 7497 codegen_->GetAssembler()->IsThumb()) { 7498 locations->AddTemp(Location::RequiresRegister()); // We need a temp for the table base. 7499 if (switch_instr->GetStartValue() != 0) { 7500 locations->AddTemp(Location::RequiresRegister()); // We need a temp for the bias. 7501 } 7502 } 7503} 7504 7505void InstructionCodeGeneratorARM::VisitPackedSwitch(HPackedSwitch* switch_instr) { 7506 int32_t lower_bound = switch_instr->GetStartValue(); 7507 uint32_t num_entries = switch_instr->GetNumEntries(); 7508 LocationSummary* locations = switch_instr->GetLocations(); 7509 Register value_reg = locations->InAt(0).AsRegister<Register>(); 7510 HBasicBlock* default_block = switch_instr->GetDefaultBlock(); 7511 7512 if (num_entries <= kPackedSwitchCompareJumpThreshold || !codegen_->GetAssembler()->IsThumb()) { 7513 // Create a series of compare/jumps. 7514 Register temp_reg = IP; 7515 // Note: It is fine for the below AddConstantSetFlags() using IP register to temporarily store 7516 // the immediate, because IP is used as the destination register. For the other 7517 // AddConstantSetFlags() and GenerateCompareWithImmediate(), the immediate values are constant, 7518 // and they can be encoded in the instruction without making use of IP register. 7519 __ AddConstantSetFlags(temp_reg, value_reg, -lower_bound); 7520 7521 const ArenaVector<HBasicBlock*>& successors = switch_instr->GetBlock()->GetSuccessors(); 7522 // Jump to successors[0] if value == lower_bound. 7523 __ b(codegen_->GetLabelOf(successors[0]), EQ); 7524 int32_t last_index = 0; 7525 for (; num_entries - last_index > 2; last_index += 2) { 7526 __ AddConstantSetFlags(temp_reg, temp_reg, -2); 7527 // Jump to successors[last_index + 1] if value < case_value[last_index + 2]. 7528 __ b(codegen_->GetLabelOf(successors[last_index + 1]), LO); 7529 // Jump to successors[last_index + 2] if value == case_value[last_index + 2]. 7530 __ b(codegen_->GetLabelOf(successors[last_index + 2]), EQ); 7531 } 7532 if (num_entries - last_index == 2) { 7533 // The last missing case_value. 7534 __ CmpConstant(temp_reg, 1); 7535 __ b(codegen_->GetLabelOf(successors[last_index + 1]), EQ); 7536 } 7537 7538 // And the default for any other value. 7539 if (!codegen_->GoesToNextBlock(switch_instr->GetBlock(), default_block)) { 7540 __ b(codegen_->GetLabelOf(default_block)); 7541 } 7542 } else { 7543 // Create a table lookup. 7544 Register temp_reg = locations->GetTemp(0).AsRegister<Register>(); 7545 7546 // Materialize a pointer to the switch table 7547 std::vector<Label*> labels(num_entries); 7548 const ArenaVector<HBasicBlock*>& successors = switch_instr->GetBlock()->GetSuccessors(); 7549 for (uint32_t i = 0; i < num_entries; i++) { 7550 labels[i] = codegen_->GetLabelOf(successors[i]); 7551 } 7552 JumpTable* table = __ CreateJumpTable(std::move(labels), temp_reg); 7553 7554 // Remove the bias. 7555 Register key_reg; 7556 if (lower_bound != 0) { 7557 key_reg = locations->GetTemp(1).AsRegister<Register>(); 7558 __ AddConstant(key_reg, value_reg, -lower_bound); 7559 } else { 7560 key_reg = value_reg; 7561 } 7562 7563 // Check whether the value is in the table, jump to default block if not. 7564 __ CmpConstant(key_reg, num_entries - 1); 7565 __ b(codegen_->GetLabelOf(default_block), Condition::HI); 7566 7567 // Load the displacement from the table. 7568 __ ldr(temp_reg, Address(temp_reg, key_reg, Shift::LSL, 2)); 7569 7570 // Dispatch is a direct add to the PC (for Thumb2). 7571 __ EmitJumpTableDispatch(table, temp_reg); 7572 } 7573} 7574 7575void LocationsBuilderARM::VisitArmDexCacheArraysBase(HArmDexCacheArraysBase* base) { 7576 LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(base); 7577 locations->SetOut(Location::RequiresRegister()); 7578} 7579 7580void InstructionCodeGeneratorARM::VisitArmDexCacheArraysBase(HArmDexCacheArraysBase* base) { 7581 Register base_reg = base->GetLocations()->Out().AsRegister<Register>(); 7582 CodeGeneratorARM::PcRelativePatchInfo* labels = 7583 codegen_->NewPcRelativeDexCacheArrayPatch(base->GetDexFile(), base->GetElementOffset()); 7584 __ BindTrackedLabel(&labels->movw_label); 7585 __ movw(base_reg, /* placeholder */ 0u); 7586 __ BindTrackedLabel(&labels->movt_label); 7587 __ movt(base_reg, /* placeholder */ 0u); 7588 __ BindTrackedLabel(&labels->add_pc_label); 7589 __ add(base_reg, base_reg, ShifterOperand(PC)); 7590} 7591 7592void CodeGeneratorARM::MoveFromReturnRegister(Location trg, Primitive::Type type) { 7593 if (!trg.IsValid()) { 7594 DCHECK_EQ(type, Primitive::kPrimVoid); 7595 return; 7596 } 7597 7598 DCHECK_NE(type, Primitive::kPrimVoid); 7599 7600 Location return_loc = InvokeDexCallingConventionVisitorARM().GetReturnLocation(type); 7601 if (return_loc.Equals(trg)) { 7602 return; 7603 } 7604 7605 // TODO: Consider pairs in the parallel move resolver, then this could be nicely merged 7606 // with the last branch. 7607 if (type == Primitive::kPrimLong) { 7608 HParallelMove parallel_move(GetGraph()->GetArena()); 7609 parallel_move.AddMove(return_loc.ToLow(), trg.ToLow(), Primitive::kPrimInt, nullptr); 7610 parallel_move.AddMove(return_loc.ToHigh(), trg.ToHigh(), Primitive::kPrimInt, nullptr); 7611 GetMoveResolver()->EmitNativeCode(¶llel_move); 7612 } else if (type == Primitive::kPrimDouble) { 7613 HParallelMove parallel_move(GetGraph()->GetArena()); 7614 parallel_move.AddMove(return_loc.ToLow(), trg.ToLow(), Primitive::kPrimFloat, nullptr); 7615 parallel_move.AddMove(return_loc.ToHigh(), trg.ToHigh(), Primitive::kPrimFloat, nullptr); 7616 GetMoveResolver()->EmitNativeCode(¶llel_move); 7617 } else { 7618 // Let the parallel move resolver take care of all of this. 7619 HParallelMove parallel_move(GetGraph()->GetArena()); 7620 parallel_move.AddMove(return_loc, trg, type, nullptr); 7621 GetMoveResolver()->EmitNativeCode(¶llel_move); 7622 } 7623} 7624 7625void LocationsBuilderARM::VisitClassTableGet(HClassTableGet* instruction) { 7626 LocationSummary* locations = 7627 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall); 7628 locations->SetInAt(0, Location::RequiresRegister()); 7629 locations->SetOut(Location::RequiresRegister()); 7630} 7631 7632void InstructionCodeGeneratorARM::VisitClassTableGet(HClassTableGet* instruction) { 7633 LocationSummary* locations = instruction->GetLocations(); 7634 if (instruction->GetTableKind() == HClassTableGet::TableKind::kVTable) { 7635 uint32_t method_offset = mirror::Class::EmbeddedVTableEntryOffset( 7636 instruction->GetIndex(), kArmPointerSize).SizeValue(); 7637 __ LoadFromOffset(kLoadWord, 7638 locations->Out().AsRegister<Register>(), 7639 locations->InAt(0).AsRegister<Register>(), 7640 method_offset); 7641 } else { 7642 uint32_t method_offset = static_cast<uint32_t>(ImTable::OffsetOfElement( 7643 instruction->GetIndex(), kArmPointerSize)); 7644 __ LoadFromOffset(kLoadWord, 7645 locations->Out().AsRegister<Register>(), 7646 locations->InAt(0).AsRegister<Register>(), 7647 mirror::Class::ImtPtrOffset(kArmPointerSize).Uint32Value()); 7648 __ LoadFromOffset(kLoadWord, 7649 locations->Out().AsRegister<Register>(), 7650 locations->Out().AsRegister<Register>(), 7651 method_offset); 7652 } 7653} 7654 7655#undef __ 7656#undef QUICK_ENTRY_POINT 7657 7658} // namespace arm 7659} // namespace art 7660