code_generator_arm.cc revision 19c5419d21376dd69404736b998fbbb9da54af56
1/* 2 * Copyright (C) 2014 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17#include "code_generator_arm.h" 18 19#include "arch/arm/instruction_set_features_arm.h" 20#include "art_method.h" 21#include "code_generator_utils.h" 22#include "compiled_method.h" 23#include "entrypoints/quick/quick_entrypoints.h" 24#include "gc/accounting/card_table.h" 25#include "intrinsics.h" 26#include "intrinsics_arm.h" 27#include "mirror/array-inl.h" 28#include "mirror/class-inl.h" 29#include "thread.h" 30#include "utils/arm/assembler_arm.h" 31#include "utils/arm/managed_register_arm.h" 32#include "utils/assembler.h" 33#include "utils/stack_checks.h" 34 35namespace art { 36 37template<class MirrorType> 38class GcRoot; 39 40namespace arm { 41 42static bool ExpectedPairLayout(Location location) { 43 // We expected this for both core and fpu register pairs. 44 return ((location.low() & 1) == 0) && (location.low() + 1 == location.high()); 45} 46 47static constexpr int kCurrentMethodStackOffset = 0; 48static constexpr Register kMethodRegisterArgument = R0; 49 50static constexpr Register kCoreAlwaysSpillRegister = R5; 51static constexpr Register kCoreCalleeSaves[] = 52 { R5, R6, R7, R8, R10, R11, LR }; 53static constexpr SRegister kFpuCalleeSaves[] = 54 { S16, S17, S18, S19, S20, S21, S22, S23, S24, S25, S26, S27, S28, S29, S30, S31 }; 55 56// D31 cannot be split into two S registers, and the register allocator only works on 57// S registers. Therefore there is no need to block it. 58static constexpr DRegister DTMP = D31; 59 60static constexpr uint32_t kPackedSwitchCompareJumpThreshold = 7; 61 62// NOLINT on __ macro to suppress wrong warning/fix (misc-macro-parentheses) from clang-tidy. 63#define __ down_cast<ArmAssembler*>(codegen->GetAssembler())-> // NOLINT 64#define QUICK_ENTRY_POINT(x) QUICK_ENTRYPOINT_OFFSET(kArmPointerSize, x).Int32Value() 65 66static constexpr int kRegListThreshold = 4; 67 68// SaveLiveRegisters and RestoreLiveRegisters from SlowPathCodeARM operate on sets of S registers, 69// for each live D registers they treat two corresponding S registers as live ones. 70// 71// Two following functions (SaveContiguousSRegisterList, RestoreContiguousSRegisterList) build 72// from a list of contiguous S registers a list of contiguous D registers (processing first/last 73// S registers corner cases) and save/restore this new list treating them as D registers. 74// - decreasing code size 75// - avoiding hazards on Cortex-A57, when a pair of S registers for an actual live D register is 76// restored and then used in regular non SlowPath code as D register. 77// 78// For the following example (v means the S register is live): 79// D names: | D0 | D1 | D2 | D4 | ... 80// S names: | S0 | S1 | S2 | S3 | S4 | S5 | S6 | S7 | ... 81// Live? | | v | v | v | v | v | v | | ... 82// 83// S1 and S6 will be saved/restored independently; D registers list (D1, D2) will be processed 84// as D registers. 85static size_t SaveContiguousSRegisterList(size_t first, 86 size_t last, 87 CodeGenerator* codegen, 88 size_t stack_offset) { 89 DCHECK_LE(first, last); 90 if ((first == last) && (first == 0)) { 91 stack_offset += codegen->SaveFloatingPointRegister(stack_offset, first); 92 return stack_offset; 93 } 94 if (first % 2 == 1) { 95 stack_offset += codegen->SaveFloatingPointRegister(stack_offset, first++); 96 } 97 98 bool save_last = false; 99 if (last % 2 == 0) { 100 save_last = true; 101 --last; 102 } 103 104 if (first < last) { 105 DRegister d_reg = static_cast<DRegister>(first / 2); 106 DCHECK_EQ((last - first + 1) % 2, 0u); 107 size_t number_of_d_regs = (last - first + 1) / 2; 108 109 if (number_of_d_regs == 1) { 110 __ StoreDToOffset(d_reg, SP, stack_offset); 111 } else if (number_of_d_regs > 1) { 112 __ add(IP, SP, ShifterOperand(stack_offset)); 113 __ vstmiad(IP, d_reg, number_of_d_regs); 114 } 115 stack_offset += number_of_d_regs * kArmWordSize * 2; 116 } 117 118 if (save_last) { 119 stack_offset += codegen->SaveFloatingPointRegister(stack_offset, last + 1); 120 } 121 122 return stack_offset; 123} 124 125static size_t RestoreContiguousSRegisterList(size_t first, 126 size_t last, 127 CodeGenerator* codegen, 128 size_t stack_offset) { 129 DCHECK_LE(first, last); 130 if ((first == last) && (first == 0)) { 131 stack_offset += codegen->RestoreFloatingPointRegister(stack_offset, first); 132 return stack_offset; 133 } 134 if (first % 2 == 1) { 135 stack_offset += codegen->RestoreFloatingPointRegister(stack_offset, first++); 136 } 137 138 bool restore_last = false; 139 if (last % 2 == 0) { 140 restore_last = true; 141 --last; 142 } 143 144 if (first < last) { 145 DRegister d_reg = static_cast<DRegister>(first / 2); 146 DCHECK_EQ((last - first + 1) % 2, 0u); 147 size_t number_of_d_regs = (last - first + 1) / 2; 148 if (number_of_d_regs == 1) { 149 __ LoadDFromOffset(d_reg, SP, stack_offset); 150 } else if (number_of_d_regs > 1) { 151 __ add(IP, SP, ShifterOperand(stack_offset)); 152 __ vldmiad(IP, d_reg, number_of_d_regs); 153 } 154 stack_offset += number_of_d_regs * kArmWordSize * 2; 155 } 156 157 if (restore_last) { 158 stack_offset += codegen->RestoreFloatingPointRegister(stack_offset, last + 1); 159 } 160 161 return stack_offset; 162} 163 164void SlowPathCodeARM::SaveLiveRegisters(CodeGenerator* codegen, LocationSummary* locations) { 165 size_t stack_offset = codegen->GetFirstRegisterSlotInSlowPath(); 166 size_t orig_offset = stack_offset; 167 168 const uint32_t core_spills = codegen->GetSlowPathSpills(locations, /* core_registers */ true); 169 for (uint32_t i : LowToHighBits(core_spills)) { 170 // If the register holds an object, update the stack mask. 171 if (locations->RegisterContainsObject(i)) { 172 locations->SetStackBit(stack_offset / kVRegSize); 173 } 174 DCHECK_LT(stack_offset, codegen->GetFrameSize() - codegen->FrameEntrySpillSize()); 175 DCHECK_LT(i, kMaximumNumberOfExpectedRegisters); 176 saved_core_stack_offsets_[i] = stack_offset; 177 stack_offset += kArmWordSize; 178 } 179 180 int reg_num = POPCOUNT(core_spills); 181 if (reg_num != 0) { 182 if (reg_num > kRegListThreshold) { 183 __ StoreList(RegList(core_spills), orig_offset); 184 } else { 185 stack_offset = orig_offset; 186 for (uint32_t i : LowToHighBits(core_spills)) { 187 stack_offset += codegen->SaveCoreRegister(stack_offset, i); 188 } 189 } 190 } 191 192 uint32_t fp_spills = codegen->GetSlowPathSpills(locations, /* core_registers */ false); 193 orig_offset = stack_offset; 194 for (uint32_t i : LowToHighBits(fp_spills)) { 195 DCHECK_LT(i, kMaximumNumberOfExpectedRegisters); 196 saved_fpu_stack_offsets_[i] = stack_offset; 197 stack_offset += kArmWordSize; 198 } 199 200 stack_offset = orig_offset; 201 while (fp_spills != 0u) { 202 uint32_t begin = CTZ(fp_spills); 203 uint32_t tmp = fp_spills + (1u << begin); 204 fp_spills &= tmp; // Clear the contiguous range of 1s. 205 uint32_t end = (tmp == 0u) ? 32u : CTZ(tmp); // CTZ(0) is undefined. 206 stack_offset = SaveContiguousSRegisterList(begin, end - 1, codegen, stack_offset); 207 } 208 DCHECK_LE(stack_offset, codegen->GetFrameSize() - codegen->FrameEntrySpillSize()); 209} 210 211void SlowPathCodeARM::RestoreLiveRegisters(CodeGenerator* codegen, LocationSummary* locations) { 212 size_t stack_offset = codegen->GetFirstRegisterSlotInSlowPath(); 213 size_t orig_offset = stack_offset; 214 215 const uint32_t core_spills = codegen->GetSlowPathSpills(locations, /* core_registers */ true); 216 for (uint32_t i : LowToHighBits(core_spills)) { 217 DCHECK_LT(stack_offset, codegen->GetFrameSize() - codegen->FrameEntrySpillSize()); 218 DCHECK_LT(i, kMaximumNumberOfExpectedRegisters); 219 stack_offset += kArmWordSize; 220 } 221 222 int reg_num = POPCOUNT(core_spills); 223 if (reg_num != 0) { 224 if (reg_num > kRegListThreshold) { 225 __ LoadList(RegList(core_spills), orig_offset); 226 } else { 227 stack_offset = orig_offset; 228 for (uint32_t i : LowToHighBits(core_spills)) { 229 stack_offset += codegen->RestoreCoreRegister(stack_offset, i); 230 } 231 } 232 } 233 234 uint32_t fp_spills = codegen->GetSlowPathSpills(locations, /* core_registers */ false); 235 while (fp_spills != 0u) { 236 uint32_t begin = CTZ(fp_spills); 237 uint32_t tmp = fp_spills + (1u << begin); 238 fp_spills &= tmp; // Clear the contiguous range of 1s. 239 uint32_t end = (tmp == 0u) ? 32u : CTZ(tmp); // CTZ(0) is undefined. 240 stack_offset = RestoreContiguousSRegisterList(begin, end - 1, codegen, stack_offset); 241 } 242 DCHECK_LE(stack_offset, codegen->GetFrameSize() - codegen->FrameEntrySpillSize()); 243} 244 245class NullCheckSlowPathARM : public SlowPathCodeARM { 246 public: 247 explicit NullCheckSlowPathARM(HNullCheck* instruction) : SlowPathCodeARM(instruction) {} 248 249 void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { 250 CodeGeneratorARM* arm_codegen = down_cast<CodeGeneratorARM*>(codegen); 251 __ Bind(GetEntryLabel()); 252 if (instruction_->CanThrowIntoCatchBlock()) { 253 // Live registers will be restored in the catch block if caught. 254 SaveLiveRegisters(codegen, instruction_->GetLocations()); 255 } 256 arm_codegen->InvokeRuntime(kQuickThrowNullPointer, 257 instruction_, 258 instruction_->GetDexPc(), 259 this); 260 CheckEntrypointTypes<kQuickThrowNullPointer, void, void>(); 261 } 262 263 bool IsFatal() const OVERRIDE { return true; } 264 265 const char* GetDescription() const OVERRIDE { return "NullCheckSlowPathARM"; } 266 267 private: 268 DISALLOW_COPY_AND_ASSIGN(NullCheckSlowPathARM); 269}; 270 271class DivZeroCheckSlowPathARM : public SlowPathCodeARM { 272 public: 273 explicit DivZeroCheckSlowPathARM(HDivZeroCheck* instruction) : SlowPathCodeARM(instruction) {} 274 275 void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { 276 CodeGeneratorARM* arm_codegen = down_cast<CodeGeneratorARM*>(codegen); 277 __ Bind(GetEntryLabel()); 278 arm_codegen->InvokeRuntime(kQuickThrowDivZero, instruction_, instruction_->GetDexPc(), this); 279 CheckEntrypointTypes<kQuickThrowDivZero, void, void>(); 280 } 281 282 bool IsFatal() const OVERRIDE { return true; } 283 284 const char* GetDescription() const OVERRIDE { return "DivZeroCheckSlowPathARM"; } 285 286 private: 287 DISALLOW_COPY_AND_ASSIGN(DivZeroCheckSlowPathARM); 288}; 289 290class SuspendCheckSlowPathARM : public SlowPathCodeARM { 291 public: 292 SuspendCheckSlowPathARM(HSuspendCheck* instruction, HBasicBlock* successor) 293 : SlowPathCodeARM(instruction), successor_(successor) {} 294 295 void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { 296 CodeGeneratorARM* arm_codegen = down_cast<CodeGeneratorARM*>(codegen); 297 __ Bind(GetEntryLabel()); 298 arm_codegen->InvokeRuntime(kQuickTestSuspend, instruction_, instruction_->GetDexPc(), this); 299 CheckEntrypointTypes<kQuickTestSuspend, void, void>(); 300 if (successor_ == nullptr) { 301 __ b(GetReturnLabel()); 302 } else { 303 __ b(arm_codegen->GetLabelOf(successor_)); 304 } 305 } 306 307 Label* GetReturnLabel() { 308 DCHECK(successor_ == nullptr); 309 return &return_label_; 310 } 311 312 HBasicBlock* GetSuccessor() const { 313 return successor_; 314 } 315 316 const char* GetDescription() const OVERRIDE { return "SuspendCheckSlowPathARM"; } 317 318 private: 319 // If not null, the block to branch to after the suspend check. 320 HBasicBlock* const successor_; 321 322 // If `successor_` is null, the label to branch to after the suspend check. 323 Label return_label_; 324 325 DISALLOW_COPY_AND_ASSIGN(SuspendCheckSlowPathARM); 326}; 327 328class BoundsCheckSlowPathARM : public SlowPathCodeARM { 329 public: 330 explicit BoundsCheckSlowPathARM(HBoundsCheck* instruction) 331 : SlowPathCodeARM(instruction) {} 332 333 void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { 334 CodeGeneratorARM* arm_codegen = down_cast<CodeGeneratorARM*>(codegen); 335 LocationSummary* locations = instruction_->GetLocations(); 336 337 __ Bind(GetEntryLabel()); 338 if (instruction_->CanThrowIntoCatchBlock()) { 339 // Live registers will be restored in the catch block if caught. 340 SaveLiveRegisters(codegen, instruction_->GetLocations()); 341 } 342 // We're moving two locations to locations that could overlap, so we need a parallel 343 // move resolver. 344 InvokeRuntimeCallingConvention calling_convention; 345 codegen->EmitParallelMoves( 346 locations->InAt(0), 347 Location::RegisterLocation(calling_convention.GetRegisterAt(0)), 348 Primitive::kPrimInt, 349 locations->InAt(1), 350 Location::RegisterLocation(calling_convention.GetRegisterAt(1)), 351 Primitive::kPrimInt); 352 QuickEntrypointEnum entrypoint = instruction_->AsBoundsCheck()->IsStringCharAt() 353 ? kQuickThrowStringBounds 354 : kQuickThrowArrayBounds; 355 arm_codegen->InvokeRuntime(entrypoint, instruction_, instruction_->GetDexPc(), this); 356 CheckEntrypointTypes<kQuickThrowStringBounds, void, int32_t, int32_t>(); 357 CheckEntrypointTypes<kQuickThrowArrayBounds, void, int32_t, int32_t>(); 358 } 359 360 bool IsFatal() const OVERRIDE { return true; } 361 362 const char* GetDescription() const OVERRIDE { return "BoundsCheckSlowPathARM"; } 363 364 private: 365 DISALLOW_COPY_AND_ASSIGN(BoundsCheckSlowPathARM); 366}; 367 368class LoadClassSlowPathARM : public SlowPathCodeARM { 369 public: 370 LoadClassSlowPathARM(HLoadClass* cls, 371 HInstruction* at, 372 uint32_t dex_pc, 373 bool do_clinit) 374 : SlowPathCodeARM(at), cls_(cls), at_(at), dex_pc_(dex_pc), do_clinit_(do_clinit) { 375 DCHECK(at->IsLoadClass() || at->IsClinitCheck()); 376 } 377 378 void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { 379 LocationSummary* locations = at_->GetLocations(); 380 381 CodeGeneratorARM* arm_codegen = down_cast<CodeGeneratorARM*>(codegen); 382 __ Bind(GetEntryLabel()); 383 SaveLiveRegisters(codegen, locations); 384 385 InvokeRuntimeCallingConvention calling_convention; 386 __ LoadImmediate(calling_convention.GetRegisterAt(0), cls_->GetTypeIndex()); 387 QuickEntrypointEnum entrypoint = do_clinit_ ? kQuickInitializeStaticStorage 388 : kQuickInitializeType; 389 arm_codegen->InvokeRuntime(entrypoint, at_, dex_pc_, this); 390 if (do_clinit_) { 391 CheckEntrypointTypes<kQuickInitializeStaticStorage, void*, uint32_t>(); 392 } else { 393 CheckEntrypointTypes<kQuickInitializeType, void*, uint32_t>(); 394 } 395 396 // Move the class to the desired location. 397 Location out = locations->Out(); 398 if (out.IsValid()) { 399 DCHECK(out.IsRegister() && !locations->GetLiveRegisters()->ContainsCoreRegister(out.reg())); 400 arm_codegen->Move32(locations->Out(), Location::RegisterLocation(R0)); 401 } 402 RestoreLiveRegisters(codegen, locations); 403 __ b(GetExitLabel()); 404 } 405 406 const char* GetDescription() const OVERRIDE { return "LoadClassSlowPathARM"; } 407 408 private: 409 // The class this slow path will load. 410 HLoadClass* const cls_; 411 412 // The instruction where this slow path is happening. 413 // (Might be the load class or an initialization check). 414 HInstruction* const at_; 415 416 // The dex PC of `at_`. 417 const uint32_t dex_pc_; 418 419 // Whether to initialize the class. 420 const bool do_clinit_; 421 422 DISALLOW_COPY_AND_ASSIGN(LoadClassSlowPathARM); 423}; 424 425class LoadStringSlowPathARM : public SlowPathCodeARM { 426 public: 427 explicit LoadStringSlowPathARM(HLoadString* instruction) : SlowPathCodeARM(instruction) {} 428 429 void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { 430 LocationSummary* locations = instruction_->GetLocations(); 431 DCHECK(!locations->GetLiveRegisters()->ContainsCoreRegister(locations->Out().reg())); 432 HLoadString* load = instruction_->AsLoadString(); 433 const uint32_t string_index = load->GetStringIndex(); 434 Register out = locations->Out().AsRegister<Register>(); 435 Register temp = locations->GetTemp(0).AsRegister<Register>(); 436 constexpr bool call_saves_everything_except_r0 = (!kUseReadBarrier || kUseBakerReadBarrier); 437 438 CodeGeneratorARM* arm_codegen = down_cast<CodeGeneratorARM*>(codegen); 439 __ Bind(GetEntryLabel()); 440 SaveLiveRegisters(codegen, locations); 441 442 InvokeRuntimeCallingConvention calling_convention; 443 // In the unlucky case that the `temp` is R0, we preserve the address in `out` across 444 // the kSaveEverything call (or use `out` for the address after non-kSaveEverything call). 445 bool temp_is_r0 = (temp == calling_convention.GetRegisterAt(0)); 446 Register entry_address = temp_is_r0 ? out : temp; 447 DCHECK_NE(entry_address, calling_convention.GetRegisterAt(0)); 448 if (call_saves_everything_except_r0 && temp_is_r0) { 449 __ mov(entry_address, ShifterOperand(temp)); 450 } 451 452 __ LoadImmediate(calling_convention.GetRegisterAt(0), string_index); 453 arm_codegen->InvokeRuntime(kQuickResolveString, instruction_, instruction_->GetDexPc(), this); 454 CheckEntrypointTypes<kQuickResolveString, void*, uint32_t>(); 455 456 // Store the resolved String to the .bss entry. 457 if (call_saves_everything_except_r0) { 458 // The string entry address was preserved in `entry_address` thanks to kSaveEverything. 459 __ str(R0, Address(entry_address)); 460 } else { 461 // For non-Baker read barrier, we need to re-calculate the address of the string entry. 462 CodeGeneratorARM::PcRelativePatchInfo* labels = 463 arm_codegen->NewPcRelativeStringPatch(load->GetDexFile(), string_index); 464 __ BindTrackedLabel(&labels->movw_label); 465 __ movw(entry_address, /* placeholder */ 0u); 466 __ BindTrackedLabel(&labels->movt_label); 467 __ movt(entry_address, /* placeholder */ 0u); 468 __ BindTrackedLabel(&labels->add_pc_label); 469 __ add(entry_address, entry_address, ShifterOperand(PC)); 470 __ str(R0, Address(entry_address)); 471 } 472 473 arm_codegen->Move32(locations->Out(), Location::RegisterLocation(R0)); 474 RestoreLiveRegisters(codegen, locations); 475 476 __ b(GetExitLabel()); 477 } 478 479 const char* GetDescription() const OVERRIDE { return "LoadStringSlowPathARM"; } 480 481 private: 482 DISALLOW_COPY_AND_ASSIGN(LoadStringSlowPathARM); 483}; 484 485class TypeCheckSlowPathARM : public SlowPathCodeARM { 486 public: 487 TypeCheckSlowPathARM(HInstruction* instruction, bool is_fatal) 488 : SlowPathCodeARM(instruction), is_fatal_(is_fatal) {} 489 490 void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { 491 LocationSummary* locations = instruction_->GetLocations(); 492 Location object_class = instruction_->IsCheckCast() ? locations->GetTemp(0) 493 : locations->Out(); 494 DCHECK(instruction_->IsCheckCast() 495 || !locations->GetLiveRegisters()->ContainsCoreRegister(locations->Out().reg())); 496 497 CodeGeneratorARM* arm_codegen = down_cast<CodeGeneratorARM*>(codegen); 498 __ Bind(GetEntryLabel()); 499 500 if (!is_fatal_) { 501 SaveLiveRegisters(codegen, locations); 502 } 503 504 // We're moving two locations to locations that could overlap, so we need a parallel 505 // move resolver. 506 InvokeRuntimeCallingConvention calling_convention; 507 codegen->EmitParallelMoves( 508 locations->InAt(1), 509 Location::RegisterLocation(calling_convention.GetRegisterAt(0)), 510 Primitive::kPrimNot, 511 object_class, 512 Location::RegisterLocation(calling_convention.GetRegisterAt(1)), 513 Primitive::kPrimNot); 514 515 if (instruction_->IsInstanceOf()) { 516 arm_codegen->InvokeRuntime(kQuickInstanceofNonTrivial, 517 instruction_, 518 instruction_->GetDexPc(), 519 this); 520 CheckEntrypointTypes< 521 kQuickInstanceofNonTrivial, size_t, const mirror::Class*, const mirror::Class*>(); 522 arm_codegen->Move32(locations->Out(), Location::RegisterLocation(R0)); 523 } else { 524 DCHECK(instruction_->IsCheckCast()); 525 arm_codegen->InvokeRuntime(kQuickCheckCast, instruction_, instruction_->GetDexPc(), this); 526 CheckEntrypointTypes<kQuickCheckCast, void, const mirror::Class*, const mirror::Class*>(); 527 } 528 529 if (!is_fatal_) { 530 RestoreLiveRegisters(codegen, locations); 531 __ b(GetExitLabel()); 532 } 533 } 534 535 const char* GetDescription() const OVERRIDE { return "TypeCheckSlowPathARM"; } 536 537 bool IsFatal() const OVERRIDE { return is_fatal_; } 538 539 private: 540 const bool is_fatal_; 541 542 DISALLOW_COPY_AND_ASSIGN(TypeCheckSlowPathARM); 543}; 544 545class DeoptimizationSlowPathARM : public SlowPathCodeARM { 546 public: 547 explicit DeoptimizationSlowPathARM(HDeoptimize* instruction) 548 : SlowPathCodeARM(instruction) {} 549 550 void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { 551 CodeGeneratorARM* arm_codegen = down_cast<CodeGeneratorARM*>(codegen); 552 __ Bind(GetEntryLabel()); 553 arm_codegen->InvokeRuntime(kQuickDeoptimize, instruction_, instruction_->GetDexPc(), this); 554 CheckEntrypointTypes<kQuickDeoptimize, void, void>(); 555 } 556 557 const char* GetDescription() const OVERRIDE { return "DeoptimizationSlowPathARM"; } 558 559 private: 560 DISALLOW_COPY_AND_ASSIGN(DeoptimizationSlowPathARM); 561}; 562 563class ArraySetSlowPathARM : public SlowPathCodeARM { 564 public: 565 explicit ArraySetSlowPathARM(HInstruction* instruction) : SlowPathCodeARM(instruction) {} 566 567 void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { 568 LocationSummary* locations = instruction_->GetLocations(); 569 __ Bind(GetEntryLabel()); 570 SaveLiveRegisters(codegen, locations); 571 572 InvokeRuntimeCallingConvention calling_convention; 573 HParallelMove parallel_move(codegen->GetGraph()->GetArena()); 574 parallel_move.AddMove( 575 locations->InAt(0), 576 Location::RegisterLocation(calling_convention.GetRegisterAt(0)), 577 Primitive::kPrimNot, 578 nullptr); 579 parallel_move.AddMove( 580 locations->InAt(1), 581 Location::RegisterLocation(calling_convention.GetRegisterAt(1)), 582 Primitive::kPrimInt, 583 nullptr); 584 parallel_move.AddMove( 585 locations->InAt(2), 586 Location::RegisterLocation(calling_convention.GetRegisterAt(2)), 587 Primitive::kPrimNot, 588 nullptr); 589 codegen->GetMoveResolver()->EmitNativeCode(¶llel_move); 590 591 CodeGeneratorARM* arm_codegen = down_cast<CodeGeneratorARM*>(codegen); 592 arm_codegen->InvokeRuntime(kQuickAputObject, instruction_, instruction_->GetDexPc(), this); 593 CheckEntrypointTypes<kQuickAputObject, void, mirror::Array*, int32_t, mirror::Object*>(); 594 RestoreLiveRegisters(codegen, locations); 595 __ b(GetExitLabel()); 596 } 597 598 const char* GetDescription() const OVERRIDE { return "ArraySetSlowPathARM"; } 599 600 private: 601 DISALLOW_COPY_AND_ASSIGN(ArraySetSlowPathARM); 602}; 603 604// Slow path marking an object reference `ref` during a read 605// barrier. The field `obj.field` in the object `obj` holding this 606// reference does not get updated by this slow path after marking (see 607// ReadBarrierMarkAndUpdateFieldSlowPathARM below for that). 608// 609// This means that after the execution of this slow path, `ref` will 610// always be up-to-date, but `obj.field` may not; i.e., after the 611// flip, `ref` will be a to-space reference, but `obj.field` will 612// probably still be a from-space reference (unless it gets updated by 613// another thread, or if another thread installed another object 614// reference (different from `ref`) in `obj.field`). 615class ReadBarrierMarkSlowPathARM : public SlowPathCodeARM { 616 public: 617 ReadBarrierMarkSlowPathARM(HInstruction* instruction, Location ref) 618 : SlowPathCodeARM(instruction), ref_(ref) { 619 DCHECK(kEmitCompilerReadBarrier); 620 } 621 622 const char* GetDescription() const OVERRIDE { return "ReadBarrierMarkSlowPathARM"; } 623 624 void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { 625 LocationSummary* locations = instruction_->GetLocations(); 626 Register ref_reg = ref_.AsRegister<Register>(); 627 DCHECK(locations->CanCall()); 628 DCHECK(!locations->GetLiveRegisters()->ContainsCoreRegister(ref_reg)) << ref_reg; 629 DCHECK(instruction_->IsInstanceFieldGet() || 630 instruction_->IsStaticFieldGet() || 631 instruction_->IsArrayGet() || 632 instruction_->IsArraySet() || 633 instruction_->IsLoadClass() || 634 instruction_->IsLoadString() || 635 instruction_->IsInstanceOf() || 636 instruction_->IsCheckCast() || 637 (instruction_->IsInvokeVirtual() && instruction_->GetLocations()->Intrinsified()) || 638 (instruction_->IsInvokeStaticOrDirect() && instruction_->GetLocations()->Intrinsified())) 639 << "Unexpected instruction in read barrier marking slow path: " 640 << instruction_->DebugName(); 641 // The read barrier instrumentation of object ArrayGet 642 // instructions does not support the HIntermediateAddress 643 // instruction. 644 DCHECK(!(instruction_->IsArrayGet() && 645 instruction_->AsArrayGet()->GetArray()->IsIntermediateAddress())); 646 647 __ Bind(GetEntryLabel()); 648 // No need to save live registers; it's taken care of by the 649 // entrypoint. Also, there is no need to update the stack mask, 650 // as this runtime call will not trigger a garbage collection. 651 CodeGeneratorARM* arm_codegen = down_cast<CodeGeneratorARM*>(codegen); 652 DCHECK_NE(ref_reg, SP); 653 DCHECK_NE(ref_reg, LR); 654 DCHECK_NE(ref_reg, PC); 655 // IP is used internally by the ReadBarrierMarkRegX entry point 656 // as a temporary, it cannot be the entry point's input/output. 657 DCHECK_NE(ref_reg, IP); 658 DCHECK(0 <= ref_reg && ref_reg < kNumberOfCoreRegisters) << ref_reg; 659 // "Compact" slow path, saving two moves. 660 // 661 // Instead of using the standard runtime calling convention (input 662 // and output in R0): 663 // 664 // R0 <- ref 665 // R0 <- ReadBarrierMark(R0) 666 // ref <- R0 667 // 668 // we just use rX (the register containing `ref`) as input and output 669 // of a dedicated entrypoint: 670 // 671 // rX <- ReadBarrierMarkRegX(rX) 672 // 673 int32_t entry_point_offset = 674 CodeGenerator::GetReadBarrierMarkEntryPointsOffset<kArmPointerSize>(ref_reg); 675 // This runtime call does not require a stack map. 676 arm_codegen->InvokeRuntimeWithoutRecordingPcInfo(entry_point_offset, instruction_, this); 677 __ b(GetExitLabel()); 678 } 679 680 private: 681 // The location (register) of the marked object reference. 682 const Location ref_; 683 684 DISALLOW_COPY_AND_ASSIGN(ReadBarrierMarkSlowPathARM); 685}; 686 687// Slow path marking an object reference `ref` during a read barrier, 688// and if needed, atomically updating the field `obj.field` in the 689// object `obj` holding this reference after marking (contrary to 690// ReadBarrierMarkSlowPathARM above, which never tries to update 691// `obj.field`). 692// 693// This means that after the execution of this slow path, both `ref` 694// and `obj.field` will be up-to-date; i.e., after the flip, both will 695// hold the same to-space reference (unless another thread installed 696// another object reference (different from `ref`) in `obj.field`). 697class ReadBarrierMarkAndUpdateFieldSlowPathARM : public SlowPathCodeARM { 698 public: 699 ReadBarrierMarkAndUpdateFieldSlowPathARM(HInstruction* instruction, 700 Location ref, 701 Register obj, 702 Location field_offset, 703 Register temp1, 704 Register temp2) 705 : SlowPathCodeARM(instruction), 706 ref_(ref), 707 obj_(obj), 708 field_offset_(field_offset), 709 temp1_(temp1), 710 temp2_(temp2) { 711 DCHECK(kEmitCompilerReadBarrier); 712 } 713 714 const char* GetDescription() const OVERRIDE { return "ReadBarrierMarkAndUpdateFieldSlowPathARM"; } 715 716 void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { 717 LocationSummary* locations = instruction_->GetLocations(); 718 Register ref_reg = ref_.AsRegister<Register>(); 719 DCHECK(locations->CanCall()); 720 DCHECK(!locations->GetLiveRegisters()->ContainsCoreRegister(ref_reg)) << ref_reg; 721 // This slow path is only used by the UnsafeCASObject intrinsic. 722 DCHECK((instruction_->IsInvokeVirtual() && instruction_->GetLocations()->Intrinsified())) 723 << "Unexpected instruction in read barrier marking and field updating slow path: " 724 << instruction_->DebugName(); 725 DCHECK(instruction_->GetLocations()->Intrinsified()); 726 DCHECK_EQ(instruction_->AsInvoke()->GetIntrinsic(), Intrinsics::kUnsafeCASObject); 727 DCHECK(field_offset_.IsRegisterPair()) << field_offset_; 728 729 __ Bind(GetEntryLabel()); 730 731 // Save the old reference. 732 // Note that we cannot use IP to save the old reference, as IP is 733 // used internally by the ReadBarrierMarkRegX entry point, and we 734 // need the old reference after the call to that entry point. 735 DCHECK_NE(temp1_, IP); 736 __ Mov(temp1_, ref_reg); 737 738 // No need to save live registers; it's taken care of by the 739 // entrypoint. Also, there is no need to update the stack mask, 740 // as this runtime call will not trigger a garbage collection. 741 CodeGeneratorARM* arm_codegen = down_cast<CodeGeneratorARM*>(codegen); 742 DCHECK_NE(ref_reg, SP); 743 DCHECK_NE(ref_reg, LR); 744 DCHECK_NE(ref_reg, PC); 745 // IP is used internally by the ReadBarrierMarkRegX entry point 746 // as a temporary, it cannot be the entry point's input/output. 747 DCHECK_NE(ref_reg, IP); 748 DCHECK(0 <= ref_reg && ref_reg < kNumberOfCoreRegisters) << ref_reg; 749 // "Compact" slow path, saving two moves. 750 // 751 // Instead of using the standard runtime calling convention (input 752 // and output in R0): 753 // 754 // R0 <- ref 755 // R0 <- ReadBarrierMark(R0) 756 // ref <- R0 757 // 758 // we just use rX (the register containing `ref`) as input and output 759 // of a dedicated entrypoint: 760 // 761 // rX <- ReadBarrierMarkRegX(rX) 762 // 763 int32_t entry_point_offset = 764 CodeGenerator::GetReadBarrierMarkEntryPointsOffset<kArmPointerSize>(ref_reg); 765 // This runtime call does not require a stack map. 766 arm_codegen->InvokeRuntimeWithoutRecordingPcInfo(entry_point_offset, instruction_, this); 767 768 // If the new reference is different from the old reference, 769 // update the field in the holder (`*(obj_ + field_offset_)`). 770 // 771 // Note that this field could also hold a different object, if 772 // another thread had concurrently changed it. In that case, the 773 // LDREX/SUBS/ITNE sequence of instructions in the compare-and-set 774 // (CAS) operation below would abort the CAS, leaving the field 775 // as-is. 776 Label done; 777 __ cmp(temp1_, ShifterOperand(ref_reg)); 778 __ b(&done, EQ); 779 780 // Update the the holder's field atomically. This may fail if 781 // mutator updates before us, but it's OK. This is achieved 782 // using a strong compare-and-set (CAS) operation with relaxed 783 // memory synchronization ordering, where the expected value is 784 // the old reference and the desired value is the new reference. 785 786 // Convenience aliases. 787 Register base = obj_; 788 // The UnsafeCASObject intrinsic uses a register pair as field 789 // offset ("long offset"), of which only the low part contains 790 // data. 791 Register offset = field_offset_.AsRegisterPairLow<Register>(); 792 Register expected = temp1_; 793 Register value = ref_reg; 794 Register tmp_ptr = IP; // Pointer to actual memory. 795 Register tmp = temp2_; // Value in memory. 796 797 __ add(tmp_ptr, base, ShifterOperand(offset)); 798 799 if (kPoisonHeapReferences) { 800 __ PoisonHeapReference(expected); 801 if (value == expected) { 802 // Do not poison `value`, as it is the same register as 803 // `expected`, which has just been poisoned. 804 } else { 805 __ PoisonHeapReference(value); 806 } 807 } 808 809 // do { 810 // tmp = [r_ptr] - expected; 811 // } while (tmp == 0 && failure([r_ptr] <- r_new_value)); 812 813 Label loop_head, exit_loop; 814 __ Bind(&loop_head); 815 816 __ ldrex(tmp, tmp_ptr); 817 818 __ subs(tmp, tmp, ShifterOperand(expected)); 819 820 __ it(NE); 821 __ clrex(NE); 822 823 __ b(&exit_loop, NE); 824 825 __ strex(tmp, value, tmp_ptr); 826 __ cmp(tmp, ShifterOperand(1)); 827 __ b(&loop_head, EQ); 828 829 __ Bind(&exit_loop); 830 831 if (kPoisonHeapReferences) { 832 __ UnpoisonHeapReference(expected); 833 if (value == expected) { 834 // Do not unpoison `value`, as it is the same register as 835 // `expected`, which has just been unpoisoned. 836 } else { 837 __ UnpoisonHeapReference(value); 838 } 839 } 840 841 __ Bind(&done); 842 __ b(GetExitLabel()); 843 } 844 845 private: 846 // The location (register) of the marked object reference. 847 const Location ref_; 848 // The register containing the object holding the marked object reference field. 849 const Register obj_; 850 // The location of the offset of the marked reference field within `obj_`. 851 Location field_offset_; 852 853 const Register temp1_; 854 const Register temp2_; 855 856 DISALLOW_COPY_AND_ASSIGN(ReadBarrierMarkAndUpdateFieldSlowPathARM); 857}; 858 859// Slow path generating a read barrier for a heap reference. 860class ReadBarrierForHeapReferenceSlowPathARM : public SlowPathCodeARM { 861 public: 862 ReadBarrierForHeapReferenceSlowPathARM(HInstruction* instruction, 863 Location out, 864 Location ref, 865 Location obj, 866 uint32_t offset, 867 Location index) 868 : SlowPathCodeARM(instruction), 869 out_(out), 870 ref_(ref), 871 obj_(obj), 872 offset_(offset), 873 index_(index) { 874 DCHECK(kEmitCompilerReadBarrier); 875 // If `obj` is equal to `out` or `ref`, it means the initial object 876 // has been overwritten by (or after) the heap object reference load 877 // to be instrumented, e.g.: 878 // 879 // __ LoadFromOffset(kLoadWord, out, out, offset); 880 // codegen_->GenerateReadBarrierSlow(instruction, out_loc, out_loc, out_loc, offset); 881 // 882 // In that case, we have lost the information about the original 883 // object, and the emitted read barrier cannot work properly. 884 DCHECK(!obj.Equals(out)) << "obj=" << obj << " out=" << out; 885 DCHECK(!obj.Equals(ref)) << "obj=" << obj << " ref=" << ref; 886 } 887 888 void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { 889 CodeGeneratorARM* arm_codegen = down_cast<CodeGeneratorARM*>(codegen); 890 LocationSummary* locations = instruction_->GetLocations(); 891 Register reg_out = out_.AsRegister<Register>(); 892 DCHECK(locations->CanCall()); 893 DCHECK(!locations->GetLiveRegisters()->ContainsCoreRegister(reg_out)); 894 DCHECK(instruction_->IsInstanceFieldGet() || 895 instruction_->IsStaticFieldGet() || 896 instruction_->IsArrayGet() || 897 instruction_->IsInstanceOf() || 898 instruction_->IsCheckCast() || 899 (instruction_->IsInvokeVirtual()) && instruction_->GetLocations()->Intrinsified()) 900 << "Unexpected instruction in read barrier for heap reference slow path: " 901 << instruction_->DebugName(); 902 // The read barrier instrumentation of object ArrayGet 903 // instructions does not support the HIntermediateAddress 904 // instruction. 905 DCHECK(!(instruction_->IsArrayGet() && 906 instruction_->AsArrayGet()->GetArray()->IsIntermediateAddress())); 907 908 __ Bind(GetEntryLabel()); 909 SaveLiveRegisters(codegen, locations); 910 911 // We may have to change the index's value, but as `index_` is a 912 // constant member (like other "inputs" of this slow path), 913 // introduce a copy of it, `index`. 914 Location index = index_; 915 if (index_.IsValid()) { 916 // Handle `index_` for HArrayGet and UnsafeGetObject/UnsafeGetObjectVolatile intrinsics. 917 if (instruction_->IsArrayGet()) { 918 // Compute the actual memory offset and store it in `index`. 919 Register index_reg = index_.AsRegister<Register>(); 920 DCHECK(locations->GetLiveRegisters()->ContainsCoreRegister(index_reg)); 921 if (codegen->IsCoreCalleeSaveRegister(index_reg)) { 922 // We are about to change the value of `index_reg` (see the 923 // calls to art::arm::Thumb2Assembler::Lsl and 924 // art::arm::Thumb2Assembler::AddConstant below), but it has 925 // not been saved by the previous call to 926 // art::SlowPathCode::SaveLiveRegisters, as it is a 927 // callee-save register -- 928 // art::SlowPathCode::SaveLiveRegisters does not consider 929 // callee-save registers, as it has been designed with the 930 // assumption that callee-save registers are supposed to be 931 // handled by the called function. So, as a callee-save 932 // register, `index_reg` _would_ eventually be saved onto 933 // the stack, but it would be too late: we would have 934 // changed its value earlier. Therefore, we manually save 935 // it here into another freely available register, 936 // `free_reg`, chosen of course among the caller-save 937 // registers (as a callee-save `free_reg` register would 938 // exhibit the same problem). 939 // 940 // Note we could have requested a temporary register from 941 // the register allocator instead; but we prefer not to, as 942 // this is a slow path, and we know we can find a 943 // caller-save register that is available. 944 Register free_reg = FindAvailableCallerSaveRegister(codegen); 945 __ Mov(free_reg, index_reg); 946 index_reg = free_reg; 947 index = Location::RegisterLocation(index_reg); 948 } else { 949 // The initial register stored in `index_` has already been 950 // saved in the call to art::SlowPathCode::SaveLiveRegisters 951 // (as it is not a callee-save register), so we can freely 952 // use it. 953 } 954 // Shifting the index value contained in `index_reg` by the scale 955 // factor (2) cannot overflow in practice, as the runtime is 956 // unable to allocate object arrays with a size larger than 957 // 2^26 - 1 (that is, 2^28 - 4 bytes). 958 __ Lsl(index_reg, index_reg, TIMES_4); 959 static_assert( 960 sizeof(mirror::HeapReference<mirror::Object>) == sizeof(int32_t), 961 "art::mirror::HeapReference<art::mirror::Object> and int32_t have different sizes."); 962 __ AddConstant(index_reg, index_reg, offset_); 963 } else { 964 // In the case of the UnsafeGetObject/UnsafeGetObjectVolatile 965 // intrinsics, `index_` is not shifted by a scale factor of 2 966 // (as in the case of ArrayGet), as it is actually an offset 967 // to an object field within an object. 968 DCHECK(instruction_->IsInvoke()) << instruction_->DebugName(); 969 DCHECK(instruction_->GetLocations()->Intrinsified()); 970 DCHECK((instruction_->AsInvoke()->GetIntrinsic() == Intrinsics::kUnsafeGetObject) || 971 (instruction_->AsInvoke()->GetIntrinsic() == Intrinsics::kUnsafeGetObjectVolatile)) 972 << instruction_->AsInvoke()->GetIntrinsic(); 973 DCHECK_EQ(offset_, 0U); 974 DCHECK(index_.IsRegisterPair()); 975 // UnsafeGet's offset location is a register pair, the low 976 // part contains the correct offset. 977 index = index_.ToLow(); 978 } 979 } 980 981 // We're moving two or three locations to locations that could 982 // overlap, so we need a parallel move resolver. 983 InvokeRuntimeCallingConvention calling_convention; 984 HParallelMove parallel_move(codegen->GetGraph()->GetArena()); 985 parallel_move.AddMove(ref_, 986 Location::RegisterLocation(calling_convention.GetRegisterAt(0)), 987 Primitive::kPrimNot, 988 nullptr); 989 parallel_move.AddMove(obj_, 990 Location::RegisterLocation(calling_convention.GetRegisterAt(1)), 991 Primitive::kPrimNot, 992 nullptr); 993 if (index.IsValid()) { 994 parallel_move.AddMove(index, 995 Location::RegisterLocation(calling_convention.GetRegisterAt(2)), 996 Primitive::kPrimInt, 997 nullptr); 998 codegen->GetMoveResolver()->EmitNativeCode(¶llel_move); 999 } else { 1000 codegen->GetMoveResolver()->EmitNativeCode(¶llel_move); 1001 __ LoadImmediate(calling_convention.GetRegisterAt(2), offset_); 1002 } 1003 arm_codegen->InvokeRuntime(kQuickReadBarrierSlow, instruction_, instruction_->GetDexPc(), this); 1004 CheckEntrypointTypes< 1005 kQuickReadBarrierSlow, mirror::Object*, mirror::Object*, mirror::Object*, uint32_t>(); 1006 arm_codegen->Move32(out_, Location::RegisterLocation(R0)); 1007 1008 RestoreLiveRegisters(codegen, locations); 1009 __ b(GetExitLabel()); 1010 } 1011 1012 const char* GetDescription() const OVERRIDE { return "ReadBarrierForHeapReferenceSlowPathARM"; } 1013 1014 private: 1015 Register FindAvailableCallerSaveRegister(CodeGenerator* codegen) { 1016 size_t ref = static_cast<int>(ref_.AsRegister<Register>()); 1017 size_t obj = static_cast<int>(obj_.AsRegister<Register>()); 1018 for (size_t i = 0, e = codegen->GetNumberOfCoreRegisters(); i < e; ++i) { 1019 if (i != ref && i != obj && !codegen->IsCoreCalleeSaveRegister(i)) { 1020 return static_cast<Register>(i); 1021 } 1022 } 1023 // We shall never fail to find a free caller-save register, as 1024 // there are more than two core caller-save registers on ARM 1025 // (meaning it is possible to find one which is different from 1026 // `ref` and `obj`). 1027 DCHECK_GT(codegen->GetNumberOfCoreCallerSaveRegisters(), 2u); 1028 LOG(FATAL) << "Could not find a free caller-save register"; 1029 UNREACHABLE(); 1030 } 1031 1032 const Location out_; 1033 const Location ref_; 1034 const Location obj_; 1035 const uint32_t offset_; 1036 // An additional location containing an index to an array. 1037 // Only used for HArrayGet and the UnsafeGetObject & 1038 // UnsafeGetObjectVolatile intrinsics. 1039 const Location index_; 1040 1041 DISALLOW_COPY_AND_ASSIGN(ReadBarrierForHeapReferenceSlowPathARM); 1042}; 1043 1044// Slow path generating a read barrier for a GC root. 1045class ReadBarrierForRootSlowPathARM : public SlowPathCodeARM { 1046 public: 1047 ReadBarrierForRootSlowPathARM(HInstruction* instruction, Location out, Location root) 1048 : SlowPathCodeARM(instruction), out_(out), root_(root) { 1049 DCHECK(kEmitCompilerReadBarrier); 1050 } 1051 1052 void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { 1053 LocationSummary* locations = instruction_->GetLocations(); 1054 Register reg_out = out_.AsRegister<Register>(); 1055 DCHECK(locations->CanCall()); 1056 DCHECK(!locations->GetLiveRegisters()->ContainsCoreRegister(reg_out)); 1057 DCHECK(instruction_->IsLoadClass() || instruction_->IsLoadString()) 1058 << "Unexpected instruction in read barrier for GC root slow path: " 1059 << instruction_->DebugName(); 1060 1061 __ Bind(GetEntryLabel()); 1062 SaveLiveRegisters(codegen, locations); 1063 1064 InvokeRuntimeCallingConvention calling_convention; 1065 CodeGeneratorARM* arm_codegen = down_cast<CodeGeneratorARM*>(codegen); 1066 arm_codegen->Move32(Location::RegisterLocation(calling_convention.GetRegisterAt(0)), root_); 1067 arm_codegen->InvokeRuntime(kQuickReadBarrierForRootSlow, 1068 instruction_, 1069 instruction_->GetDexPc(), 1070 this); 1071 CheckEntrypointTypes<kQuickReadBarrierForRootSlow, mirror::Object*, GcRoot<mirror::Object>*>(); 1072 arm_codegen->Move32(out_, Location::RegisterLocation(R0)); 1073 1074 RestoreLiveRegisters(codegen, locations); 1075 __ b(GetExitLabel()); 1076 } 1077 1078 const char* GetDescription() const OVERRIDE { return "ReadBarrierForRootSlowPathARM"; } 1079 1080 private: 1081 const Location out_; 1082 const Location root_; 1083 1084 DISALLOW_COPY_AND_ASSIGN(ReadBarrierForRootSlowPathARM); 1085}; 1086 1087#undef __ 1088// NOLINT on __ macro to suppress wrong warning/fix (misc-macro-parentheses) from clang-tidy. 1089#define __ down_cast<ArmAssembler*>(GetAssembler())-> // NOLINT 1090 1091inline Condition ARMCondition(IfCondition cond) { 1092 switch (cond) { 1093 case kCondEQ: return EQ; 1094 case kCondNE: return NE; 1095 case kCondLT: return LT; 1096 case kCondLE: return LE; 1097 case kCondGT: return GT; 1098 case kCondGE: return GE; 1099 case kCondB: return LO; 1100 case kCondBE: return LS; 1101 case kCondA: return HI; 1102 case kCondAE: return HS; 1103 } 1104 LOG(FATAL) << "Unreachable"; 1105 UNREACHABLE(); 1106} 1107 1108// Maps signed condition to unsigned condition. 1109inline Condition ARMUnsignedCondition(IfCondition cond) { 1110 switch (cond) { 1111 case kCondEQ: return EQ; 1112 case kCondNE: return NE; 1113 // Signed to unsigned. 1114 case kCondLT: return LO; 1115 case kCondLE: return LS; 1116 case kCondGT: return HI; 1117 case kCondGE: return HS; 1118 // Unsigned remain unchanged. 1119 case kCondB: return LO; 1120 case kCondBE: return LS; 1121 case kCondA: return HI; 1122 case kCondAE: return HS; 1123 } 1124 LOG(FATAL) << "Unreachable"; 1125 UNREACHABLE(); 1126} 1127 1128inline Condition ARMFPCondition(IfCondition cond, bool gt_bias) { 1129 // The ARM condition codes can express all the necessary branches, see the 1130 // "Meaning (floating-point)" column in the table A8-1 of the ARMv7 reference manual. 1131 // There is no dex instruction or HIR that would need the missing conditions 1132 // "equal or unordered" or "not equal". 1133 switch (cond) { 1134 case kCondEQ: return EQ; 1135 case kCondNE: return NE /* unordered */; 1136 case kCondLT: return gt_bias ? CC : LT /* unordered */; 1137 case kCondLE: return gt_bias ? LS : LE /* unordered */; 1138 case kCondGT: return gt_bias ? HI /* unordered */ : GT; 1139 case kCondGE: return gt_bias ? CS /* unordered */ : GE; 1140 default: 1141 LOG(FATAL) << "UNREACHABLE"; 1142 UNREACHABLE(); 1143 } 1144} 1145 1146void CodeGeneratorARM::DumpCoreRegister(std::ostream& stream, int reg) const { 1147 stream << Register(reg); 1148} 1149 1150void CodeGeneratorARM::DumpFloatingPointRegister(std::ostream& stream, int reg) const { 1151 stream << SRegister(reg); 1152} 1153 1154size_t CodeGeneratorARM::SaveCoreRegister(size_t stack_index, uint32_t reg_id) { 1155 __ StoreToOffset(kStoreWord, static_cast<Register>(reg_id), SP, stack_index); 1156 return kArmWordSize; 1157} 1158 1159size_t CodeGeneratorARM::RestoreCoreRegister(size_t stack_index, uint32_t reg_id) { 1160 __ LoadFromOffset(kLoadWord, static_cast<Register>(reg_id), SP, stack_index); 1161 return kArmWordSize; 1162} 1163 1164size_t CodeGeneratorARM::SaveFloatingPointRegister(size_t stack_index, uint32_t reg_id) { 1165 __ StoreSToOffset(static_cast<SRegister>(reg_id), SP, stack_index); 1166 return kArmWordSize; 1167} 1168 1169size_t CodeGeneratorARM::RestoreFloatingPointRegister(size_t stack_index, uint32_t reg_id) { 1170 __ LoadSFromOffset(static_cast<SRegister>(reg_id), SP, stack_index); 1171 return kArmWordSize; 1172} 1173 1174CodeGeneratorARM::CodeGeneratorARM(HGraph* graph, 1175 const ArmInstructionSetFeatures& isa_features, 1176 const CompilerOptions& compiler_options, 1177 OptimizingCompilerStats* stats) 1178 : CodeGenerator(graph, 1179 kNumberOfCoreRegisters, 1180 kNumberOfSRegisters, 1181 kNumberOfRegisterPairs, 1182 ComputeRegisterMask(reinterpret_cast<const int*>(kCoreCalleeSaves), 1183 arraysize(kCoreCalleeSaves)), 1184 ComputeRegisterMask(reinterpret_cast<const int*>(kFpuCalleeSaves), 1185 arraysize(kFpuCalleeSaves)), 1186 compiler_options, 1187 stats), 1188 block_labels_(nullptr), 1189 location_builder_(graph, this), 1190 instruction_visitor_(graph, this), 1191 move_resolver_(graph->GetArena(), this), 1192 assembler_(graph->GetArena()), 1193 isa_features_(isa_features), 1194 uint32_literals_(std::less<uint32_t>(), 1195 graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), 1196 method_patches_(MethodReferenceComparator(), 1197 graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), 1198 call_patches_(MethodReferenceComparator(), 1199 graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), 1200 relative_call_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), 1201 pc_relative_dex_cache_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), 1202 boot_image_string_patches_(StringReferenceValueComparator(), 1203 graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), 1204 pc_relative_string_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), 1205 boot_image_type_patches_(TypeReferenceValueComparator(), 1206 graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), 1207 pc_relative_type_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), 1208 boot_image_address_patches_(std::less<uint32_t>(), 1209 graph->GetArena()->Adapter(kArenaAllocCodeGenerator)) { 1210 // Always save the LR register to mimic Quick. 1211 AddAllocatedRegister(Location::RegisterLocation(LR)); 1212} 1213 1214void CodeGeneratorARM::Finalize(CodeAllocator* allocator) { 1215 // Ensure that we fix up branches and literal loads and emit the literal pool. 1216 __ FinalizeCode(); 1217 1218 // Adjust native pc offsets in stack maps. 1219 for (size_t i = 0, num = stack_map_stream_.GetNumberOfStackMaps(); i != num; ++i) { 1220 uint32_t old_position = stack_map_stream_.GetStackMap(i).native_pc_offset; 1221 uint32_t new_position = __ GetAdjustedPosition(old_position); 1222 stack_map_stream_.SetStackMapNativePcOffset(i, new_position); 1223 } 1224 // Adjust pc offsets for the disassembly information. 1225 if (disasm_info_ != nullptr) { 1226 GeneratedCodeInterval* frame_entry_interval = disasm_info_->GetFrameEntryInterval(); 1227 frame_entry_interval->start = __ GetAdjustedPosition(frame_entry_interval->start); 1228 frame_entry_interval->end = __ GetAdjustedPosition(frame_entry_interval->end); 1229 for (auto& it : *disasm_info_->GetInstructionIntervals()) { 1230 it.second.start = __ GetAdjustedPosition(it.second.start); 1231 it.second.end = __ GetAdjustedPosition(it.second.end); 1232 } 1233 for (auto& it : *disasm_info_->GetSlowPathIntervals()) { 1234 it.code_interval.start = __ GetAdjustedPosition(it.code_interval.start); 1235 it.code_interval.end = __ GetAdjustedPosition(it.code_interval.end); 1236 } 1237 } 1238 1239 CodeGenerator::Finalize(allocator); 1240} 1241 1242void CodeGeneratorARM::SetupBlockedRegisters() const { 1243 // Stack register, LR and PC are always reserved. 1244 blocked_core_registers_[SP] = true; 1245 blocked_core_registers_[LR] = true; 1246 blocked_core_registers_[PC] = true; 1247 1248 // Reserve thread register. 1249 blocked_core_registers_[TR] = true; 1250 1251 // Reserve temp register. 1252 blocked_core_registers_[IP] = true; 1253 1254 if (GetGraph()->IsDebuggable()) { 1255 // Stubs do not save callee-save floating point registers. If the graph 1256 // is debuggable, we need to deal with these registers differently. For 1257 // now, just block them. 1258 for (size_t i = 0; i < arraysize(kFpuCalleeSaves); ++i) { 1259 blocked_fpu_registers_[kFpuCalleeSaves[i]] = true; 1260 } 1261 } 1262} 1263 1264InstructionCodeGeneratorARM::InstructionCodeGeneratorARM(HGraph* graph, CodeGeneratorARM* codegen) 1265 : InstructionCodeGenerator(graph, codegen), 1266 assembler_(codegen->GetAssembler()), 1267 codegen_(codegen) {} 1268 1269void CodeGeneratorARM::ComputeSpillMask() { 1270 core_spill_mask_ = allocated_registers_.GetCoreRegisters() & core_callee_save_mask_; 1271 DCHECK_NE(core_spill_mask_, 0u) << "At least the return address register must be saved"; 1272 // There is no easy instruction to restore just the PC on thumb2. We spill and 1273 // restore another arbitrary register. 1274 core_spill_mask_ |= (1 << kCoreAlwaysSpillRegister); 1275 fpu_spill_mask_ = allocated_registers_.GetFloatingPointRegisters() & fpu_callee_save_mask_; 1276 // We use vpush and vpop for saving and restoring floating point registers, which take 1277 // a SRegister and the number of registers to save/restore after that SRegister. We 1278 // therefore update the `fpu_spill_mask_` to also contain those registers not allocated, 1279 // but in the range. 1280 if (fpu_spill_mask_ != 0) { 1281 uint32_t least_significant_bit = LeastSignificantBit(fpu_spill_mask_); 1282 uint32_t most_significant_bit = MostSignificantBit(fpu_spill_mask_); 1283 for (uint32_t i = least_significant_bit + 1 ; i < most_significant_bit; ++i) { 1284 fpu_spill_mask_ |= (1 << i); 1285 } 1286 } 1287} 1288 1289static dwarf::Reg DWARFReg(Register reg) { 1290 return dwarf::Reg::ArmCore(static_cast<int>(reg)); 1291} 1292 1293static dwarf::Reg DWARFReg(SRegister reg) { 1294 return dwarf::Reg::ArmFp(static_cast<int>(reg)); 1295} 1296 1297void CodeGeneratorARM::GenerateFrameEntry() { 1298 bool skip_overflow_check = 1299 IsLeafMethod() && !FrameNeedsStackCheck(GetFrameSize(), InstructionSet::kArm); 1300 DCHECK(GetCompilerOptions().GetImplicitStackOverflowChecks()); 1301 __ Bind(&frame_entry_label_); 1302 1303 if (HasEmptyFrame()) { 1304 return; 1305 } 1306 1307 if (!skip_overflow_check) { 1308 __ AddConstant(IP, SP, -static_cast<int32_t>(GetStackOverflowReservedBytes(kArm))); 1309 __ LoadFromOffset(kLoadWord, IP, IP, 0); 1310 RecordPcInfo(nullptr, 0); 1311 } 1312 1313 __ PushList(core_spill_mask_); 1314 __ cfi().AdjustCFAOffset(kArmWordSize * POPCOUNT(core_spill_mask_)); 1315 __ cfi().RelOffsetForMany(DWARFReg(kMethodRegisterArgument), 0, core_spill_mask_, kArmWordSize); 1316 if (fpu_spill_mask_ != 0) { 1317 SRegister start_register = SRegister(LeastSignificantBit(fpu_spill_mask_)); 1318 __ vpushs(start_register, POPCOUNT(fpu_spill_mask_)); 1319 __ cfi().AdjustCFAOffset(kArmWordSize * POPCOUNT(fpu_spill_mask_)); 1320 __ cfi().RelOffsetForMany(DWARFReg(S0), 0, fpu_spill_mask_, kArmWordSize); 1321 } 1322 int adjust = GetFrameSize() - FrameEntrySpillSize(); 1323 __ AddConstant(SP, -adjust); 1324 __ cfi().AdjustCFAOffset(adjust); 1325 1326 // Save the current method if we need it. Note that we do not 1327 // do this in HCurrentMethod, as the instruction might have been removed 1328 // in the SSA graph. 1329 if (RequiresCurrentMethod()) { 1330 __ StoreToOffset(kStoreWord, kMethodRegisterArgument, SP, 0); 1331 } 1332} 1333 1334void CodeGeneratorARM::GenerateFrameExit() { 1335 if (HasEmptyFrame()) { 1336 __ bx(LR); 1337 return; 1338 } 1339 __ cfi().RememberState(); 1340 int adjust = GetFrameSize() - FrameEntrySpillSize(); 1341 __ AddConstant(SP, adjust); 1342 __ cfi().AdjustCFAOffset(-adjust); 1343 if (fpu_spill_mask_ != 0) { 1344 SRegister start_register = SRegister(LeastSignificantBit(fpu_spill_mask_)); 1345 __ vpops(start_register, POPCOUNT(fpu_spill_mask_)); 1346 __ cfi().AdjustCFAOffset(-static_cast<int>(kArmPointerSize) * POPCOUNT(fpu_spill_mask_)); 1347 __ cfi().RestoreMany(DWARFReg(SRegister(0)), fpu_spill_mask_); 1348 } 1349 // Pop LR into PC to return. 1350 DCHECK_NE(core_spill_mask_ & (1 << LR), 0U); 1351 uint32_t pop_mask = (core_spill_mask_ & (~(1 << LR))) | 1 << PC; 1352 __ PopList(pop_mask); 1353 __ cfi().RestoreState(); 1354 __ cfi().DefCFAOffset(GetFrameSize()); 1355} 1356 1357void CodeGeneratorARM::Bind(HBasicBlock* block) { 1358 Label* label = GetLabelOf(block); 1359 __ BindTrackedLabel(label); 1360} 1361 1362Location InvokeDexCallingConventionVisitorARM::GetNextLocation(Primitive::Type type) { 1363 switch (type) { 1364 case Primitive::kPrimBoolean: 1365 case Primitive::kPrimByte: 1366 case Primitive::kPrimChar: 1367 case Primitive::kPrimShort: 1368 case Primitive::kPrimInt: 1369 case Primitive::kPrimNot: { 1370 uint32_t index = gp_index_++; 1371 uint32_t stack_index = stack_index_++; 1372 if (index < calling_convention.GetNumberOfRegisters()) { 1373 return Location::RegisterLocation(calling_convention.GetRegisterAt(index)); 1374 } else { 1375 return Location::StackSlot(calling_convention.GetStackOffsetOf(stack_index)); 1376 } 1377 } 1378 1379 case Primitive::kPrimLong: { 1380 uint32_t index = gp_index_; 1381 uint32_t stack_index = stack_index_; 1382 gp_index_ += 2; 1383 stack_index_ += 2; 1384 if (index + 1 < calling_convention.GetNumberOfRegisters()) { 1385 if (calling_convention.GetRegisterAt(index) == R1) { 1386 // Skip R1, and use R2_R3 instead. 1387 gp_index_++; 1388 index++; 1389 } 1390 } 1391 if (index + 1 < calling_convention.GetNumberOfRegisters()) { 1392 DCHECK_EQ(calling_convention.GetRegisterAt(index) + 1, 1393 calling_convention.GetRegisterAt(index + 1)); 1394 1395 return Location::RegisterPairLocation(calling_convention.GetRegisterAt(index), 1396 calling_convention.GetRegisterAt(index + 1)); 1397 } else { 1398 return Location::DoubleStackSlot(calling_convention.GetStackOffsetOf(stack_index)); 1399 } 1400 } 1401 1402 case Primitive::kPrimFloat: { 1403 uint32_t stack_index = stack_index_++; 1404 if (float_index_ % 2 == 0) { 1405 float_index_ = std::max(double_index_, float_index_); 1406 } 1407 if (float_index_ < calling_convention.GetNumberOfFpuRegisters()) { 1408 return Location::FpuRegisterLocation(calling_convention.GetFpuRegisterAt(float_index_++)); 1409 } else { 1410 return Location::StackSlot(calling_convention.GetStackOffsetOf(stack_index)); 1411 } 1412 } 1413 1414 case Primitive::kPrimDouble: { 1415 double_index_ = std::max(double_index_, RoundUp(float_index_, 2)); 1416 uint32_t stack_index = stack_index_; 1417 stack_index_ += 2; 1418 if (double_index_ + 1 < calling_convention.GetNumberOfFpuRegisters()) { 1419 uint32_t index = double_index_; 1420 double_index_ += 2; 1421 Location result = Location::FpuRegisterPairLocation( 1422 calling_convention.GetFpuRegisterAt(index), 1423 calling_convention.GetFpuRegisterAt(index + 1)); 1424 DCHECK(ExpectedPairLayout(result)); 1425 return result; 1426 } else { 1427 return Location::DoubleStackSlot(calling_convention.GetStackOffsetOf(stack_index)); 1428 } 1429 } 1430 1431 case Primitive::kPrimVoid: 1432 LOG(FATAL) << "Unexpected parameter type " << type; 1433 break; 1434 } 1435 return Location::NoLocation(); 1436} 1437 1438Location InvokeDexCallingConventionVisitorARM::GetReturnLocation(Primitive::Type type) const { 1439 switch (type) { 1440 case Primitive::kPrimBoolean: 1441 case Primitive::kPrimByte: 1442 case Primitive::kPrimChar: 1443 case Primitive::kPrimShort: 1444 case Primitive::kPrimInt: 1445 case Primitive::kPrimNot: { 1446 return Location::RegisterLocation(R0); 1447 } 1448 1449 case Primitive::kPrimFloat: { 1450 return Location::FpuRegisterLocation(S0); 1451 } 1452 1453 case Primitive::kPrimLong: { 1454 return Location::RegisterPairLocation(R0, R1); 1455 } 1456 1457 case Primitive::kPrimDouble: { 1458 return Location::FpuRegisterPairLocation(S0, S1); 1459 } 1460 1461 case Primitive::kPrimVoid: 1462 return Location::NoLocation(); 1463 } 1464 1465 UNREACHABLE(); 1466} 1467 1468Location InvokeDexCallingConventionVisitorARM::GetMethodLocation() const { 1469 return Location::RegisterLocation(kMethodRegisterArgument); 1470} 1471 1472void CodeGeneratorARM::Move32(Location destination, Location source) { 1473 if (source.Equals(destination)) { 1474 return; 1475 } 1476 if (destination.IsRegister()) { 1477 if (source.IsRegister()) { 1478 __ Mov(destination.AsRegister<Register>(), source.AsRegister<Register>()); 1479 } else if (source.IsFpuRegister()) { 1480 __ vmovrs(destination.AsRegister<Register>(), source.AsFpuRegister<SRegister>()); 1481 } else { 1482 __ LoadFromOffset(kLoadWord, destination.AsRegister<Register>(), SP, source.GetStackIndex()); 1483 } 1484 } else if (destination.IsFpuRegister()) { 1485 if (source.IsRegister()) { 1486 __ vmovsr(destination.AsFpuRegister<SRegister>(), source.AsRegister<Register>()); 1487 } else if (source.IsFpuRegister()) { 1488 __ vmovs(destination.AsFpuRegister<SRegister>(), source.AsFpuRegister<SRegister>()); 1489 } else { 1490 __ LoadSFromOffset(destination.AsFpuRegister<SRegister>(), SP, source.GetStackIndex()); 1491 } 1492 } else { 1493 DCHECK(destination.IsStackSlot()) << destination; 1494 if (source.IsRegister()) { 1495 __ StoreToOffset(kStoreWord, source.AsRegister<Register>(), SP, destination.GetStackIndex()); 1496 } else if (source.IsFpuRegister()) { 1497 __ StoreSToOffset(source.AsFpuRegister<SRegister>(), SP, destination.GetStackIndex()); 1498 } else { 1499 DCHECK(source.IsStackSlot()) << source; 1500 __ LoadFromOffset(kLoadWord, IP, SP, source.GetStackIndex()); 1501 __ StoreToOffset(kStoreWord, IP, SP, destination.GetStackIndex()); 1502 } 1503 } 1504} 1505 1506void CodeGeneratorARM::Move64(Location destination, Location source) { 1507 if (source.Equals(destination)) { 1508 return; 1509 } 1510 if (destination.IsRegisterPair()) { 1511 if (source.IsRegisterPair()) { 1512 EmitParallelMoves( 1513 Location::RegisterLocation(source.AsRegisterPairHigh<Register>()), 1514 Location::RegisterLocation(destination.AsRegisterPairHigh<Register>()), 1515 Primitive::kPrimInt, 1516 Location::RegisterLocation(source.AsRegisterPairLow<Register>()), 1517 Location::RegisterLocation(destination.AsRegisterPairLow<Register>()), 1518 Primitive::kPrimInt); 1519 } else if (source.IsFpuRegister()) { 1520 UNIMPLEMENTED(FATAL); 1521 } else if (source.IsFpuRegisterPair()) { 1522 __ vmovrrd(destination.AsRegisterPairLow<Register>(), 1523 destination.AsRegisterPairHigh<Register>(), 1524 FromLowSToD(source.AsFpuRegisterPairLow<SRegister>())); 1525 } else { 1526 DCHECK(source.IsDoubleStackSlot()); 1527 DCHECK(ExpectedPairLayout(destination)); 1528 __ LoadFromOffset(kLoadWordPair, destination.AsRegisterPairLow<Register>(), 1529 SP, source.GetStackIndex()); 1530 } 1531 } else if (destination.IsFpuRegisterPair()) { 1532 if (source.IsDoubleStackSlot()) { 1533 __ LoadDFromOffset(FromLowSToD(destination.AsFpuRegisterPairLow<SRegister>()), 1534 SP, 1535 source.GetStackIndex()); 1536 } else if (source.IsRegisterPair()) { 1537 __ vmovdrr(FromLowSToD(destination.AsFpuRegisterPairLow<SRegister>()), 1538 source.AsRegisterPairLow<Register>(), 1539 source.AsRegisterPairHigh<Register>()); 1540 } else { 1541 UNIMPLEMENTED(FATAL); 1542 } 1543 } else { 1544 DCHECK(destination.IsDoubleStackSlot()); 1545 if (source.IsRegisterPair()) { 1546 // No conflict possible, so just do the moves. 1547 if (source.AsRegisterPairLow<Register>() == R1) { 1548 DCHECK_EQ(source.AsRegisterPairHigh<Register>(), R2); 1549 __ StoreToOffset(kStoreWord, R1, SP, destination.GetStackIndex()); 1550 __ StoreToOffset(kStoreWord, R2, SP, destination.GetHighStackIndex(kArmWordSize)); 1551 } else { 1552 __ StoreToOffset(kStoreWordPair, source.AsRegisterPairLow<Register>(), 1553 SP, destination.GetStackIndex()); 1554 } 1555 } else if (source.IsFpuRegisterPair()) { 1556 __ StoreDToOffset(FromLowSToD(source.AsFpuRegisterPairLow<SRegister>()), 1557 SP, 1558 destination.GetStackIndex()); 1559 } else { 1560 DCHECK(source.IsDoubleStackSlot()); 1561 EmitParallelMoves( 1562 Location::StackSlot(source.GetStackIndex()), 1563 Location::StackSlot(destination.GetStackIndex()), 1564 Primitive::kPrimInt, 1565 Location::StackSlot(source.GetHighStackIndex(kArmWordSize)), 1566 Location::StackSlot(destination.GetHighStackIndex(kArmWordSize)), 1567 Primitive::kPrimInt); 1568 } 1569 } 1570} 1571 1572void CodeGeneratorARM::MoveConstant(Location location, int32_t value) { 1573 DCHECK(location.IsRegister()); 1574 __ LoadImmediate(location.AsRegister<Register>(), value); 1575} 1576 1577void CodeGeneratorARM::MoveLocation(Location dst, Location src, Primitive::Type dst_type) { 1578 HParallelMove move(GetGraph()->GetArena()); 1579 move.AddMove(src, dst, dst_type, nullptr); 1580 GetMoveResolver()->EmitNativeCode(&move); 1581} 1582 1583void CodeGeneratorARM::AddLocationAsTemp(Location location, LocationSummary* locations) { 1584 if (location.IsRegister()) { 1585 locations->AddTemp(location); 1586 } else if (location.IsRegisterPair()) { 1587 locations->AddTemp(Location::RegisterLocation(location.AsRegisterPairLow<Register>())); 1588 locations->AddTemp(Location::RegisterLocation(location.AsRegisterPairHigh<Register>())); 1589 } else { 1590 UNIMPLEMENTED(FATAL) << "AddLocationAsTemp not implemented for location " << location; 1591 } 1592} 1593 1594void CodeGeneratorARM::InvokeRuntime(QuickEntrypointEnum entrypoint, 1595 HInstruction* instruction, 1596 uint32_t dex_pc, 1597 SlowPathCode* slow_path) { 1598 ValidateInvokeRuntime(entrypoint, instruction, slow_path); 1599 GenerateInvokeRuntime(GetThreadOffset<kArmPointerSize>(entrypoint).Int32Value()); 1600 if (EntrypointRequiresStackMap(entrypoint)) { 1601 RecordPcInfo(instruction, dex_pc, slow_path); 1602 } 1603} 1604 1605void CodeGeneratorARM::InvokeRuntimeWithoutRecordingPcInfo(int32_t entry_point_offset, 1606 HInstruction* instruction, 1607 SlowPathCode* slow_path) { 1608 ValidateInvokeRuntimeWithoutRecordingPcInfo(instruction, slow_path); 1609 GenerateInvokeRuntime(entry_point_offset); 1610} 1611 1612void CodeGeneratorARM::GenerateInvokeRuntime(int32_t entry_point_offset) { 1613 __ LoadFromOffset(kLoadWord, LR, TR, entry_point_offset); 1614 __ blx(LR); 1615} 1616 1617void InstructionCodeGeneratorARM::HandleGoto(HInstruction* got, HBasicBlock* successor) { 1618 DCHECK(!successor->IsExitBlock()); 1619 1620 HBasicBlock* block = got->GetBlock(); 1621 HInstruction* previous = got->GetPrevious(); 1622 1623 HLoopInformation* info = block->GetLoopInformation(); 1624 if (info != nullptr && info->IsBackEdge(*block) && info->HasSuspendCheck()) { 1625 codegen_->ClearSpillSlotsFromLoopPhisInStackMap(info->GetSuspendCheck()); 1626 GenerateSuspendCheck(info->GetSuspendCheck(), successor); 1627 return; 1628 } 1629 1630 if (block->IsEntryBlock() && (previous != nullptr) && previous->IsSuspendCheck()) { 1631 GenerateSuspendCheck(previous->AsSuspendCheck(), nullptr); 1632 } 1633 if (!codegen_->GoesToNextBlock(got->GetBlock(), successor)) { 1634 __ b(codegen_->GetLabelOf(successor)); 1635 } 1636} 1637 1638void LocationsBuilderARM::VisitGoto(HGoto* got) { 1639 got->SetLocations(nullptr); 1640} 1641 1642void InstructionCodeGeneratorARM::VisitGoto(HGoto* got) { 1643 HandleGoto(got, got->GetSuccessor()); 1644} 1645 1646void LocationsBuilderARM::VisitTryBoundary(HTryBoundary* try_boundary) { 1647 try_boundary->SetLocations(nullptr); 1648} 1649 1650void InstructionCodeGeneratorARM::VisitTryBoundary(HTryBoundary* try_boundary) { 1651 HBasicBlock* successor = try_boundary->GetNormalFlowSuccessor(); 1652 if (!successor->IsExitBlock()) { 1653 HandleGoto(try_boundary, successor); 1654 } 1655} 1656 1657void LocationsBuilderARM::VisitExit(HExit* exit) { 1658 exit->SetLocations(nullptr); 1659} 1660 1661void InstructionCodeGeneratorARM::VisitExit(HExit* exit ATTRIBUTE_UNUSED) { 1662} 1663 1664void InstructionCodeGeneratorARM::GenerateVcmp(HInstruction* instruction) { 1665 Primitive::Type type = instruction->InputAt(0)->GetType(); 1666 Location lhs_loc = instruction->GetLocations()->InAt(0); 1667 Location rhs_loc = instruction->GetLocations()->InAt(1); 1668 if (rhs_loc.IsConstant()) { 1669 // 0.0 is the only immediate that can be encoded directly in 1670 // a VCMP instruction. 1671 // 1672 // Both the JLS (section 15.20.1) and the JVMS (section 6.5) 1673 // specify that in a floating-point comparison, positive zero 1674 // and negative zero are considered equal, so we can use the 1675 // literal 0.0 for both cases here. 1676 // 1677 // Note however that some methods (Float.equal, Float.compare, 1678 // Float.compareTo, Double.equal, Double.compare, 1679 // Double.compareTo, Math.max, Math.min, StrictMath.max, 1680 // StrictMath.min) consider 0.0 to be (strictly) greater than 1681 // -0.0. So if we ever translate calls to these methods into a 1682 // HCompare instruction, we must handle the -0.0 case with 1683 // care here. 1684 DCHECK(rhs_loc.GetConstant()->IsArithmeticZero()); 1685 if (type == Primitive::kPrimFloat) { 1686 __ vcmpsz(lhs_loc.AsFpuRegister<SRegister>()); 1687 } else { 1688 DCHECK_EQ(type, Primitive::kPrimDouble); 1689 __ vcmpdz(FromLowSToD(lhs_loc.AsFpuRegisterPairLow<SRegister>())); 1690 } 1691 } else { 1692 if (type == Primitive::kPrimFloat) { 1693 __ vcmps(lhs_loc.AsFpuRegister<SRegister>(), rhs_loc.AsFpuRegister<SRegister>()); 1694 } else { 1695 DCHECK_EQ(type, Primitive::kPrimDouble); 1696 __ vcmpd(FromLowSToD(lhs_loc.AsFpuRegisterPairLow<SRegister>()), 1697 FromLowSToD(rhs_loc.AsFpuRegisterPairLow<SRegister>())); 1698 } 1699 } 1700} 1701 1702void InstructionCodeGeneratorARM::GenerateFPJumps(HCondition* cond, 1703 Label* true_label, 1704 Label* false_label ATTRIBUTE_UNUSED) { 1705 __ vmstat(); // transfer FP status register to ARM APSR. 1706 __ b(true_label, ARMFPCondition(cond->GetCondition(), cond->IsGtBias())); 1707} 1708 1709void InstructionCodeGeneratorARM::GenerateLongComparesAndJumps(HCondition* cond, 1710 Label* true_label, 1711 Label* false_label) { 1712 LocationSummary* locations = cond->GetLocations(); 1713 Location left = locations->InAt(0); 1714 Location right = locations->InAt(1); 1715 IfCondition if_cond = cond->GetCondition(); 1716 1717 Register left_high = left.AsRegisterPairHigh<Register>(); 1718 Register left_low = left.AsRegisterPairLow<Register>(); 1719 IfCondition true_high_cond = if_cond; 1720 IfCondition false_high_cond = cond->GetOppositeCondition(); 1721 Condition final_condition = ARMUnsignedCondition(if_cond); // unsigned on lower part 1722 1723 // Set the conditions for the test, remembering that == needs to be 1724 // decided using the low words. 1725 // TODO: consider avoiding jumps with temporary and CMP low+SBC high 1726 switch (if_cond) { 1727 case kCondEQ: 1728 case kCondNE: 1729 // Nothing to do. 1730 break; 1731 case kCondLT: 1732 false_high_cond = kCondGT; 1733 break; 1734 case kCondLE: 1735 true_high_cond = kCondLT; 1736 break; 1737 case kCondGT: 1738 false_high_cond = kCondLT; 1739 break; 1740 case kCondGE: 1741 true_high_cond = kCondGT; 1742 break; 1743 case kCondB: 1744 false_high_cond = kCondA; 1745 break; 1746 case kCondBE: 1747 true_high_cond = kCondB; 1748 break; 1749 case kCondA: 1750 false_high_cond = kCondB; 1751 break; 1752 case kCondAE: 1753 true_high_cond = kCondA; 1754 break; 1755 } 1756 if (right.IsConstant()) { 1757 int64_t value = right.GetConstant()->AsLongConstant()->GetValue(); 1758 int32_t val_low = Low32Bits(value); 1759 int32_t val_high = High32Bits(value); 1760 1761 __ CmpConstant(left_high, val_high); 1762 if (if_cond == kCondNE) { 1763 __ b(true_label, ARMCondition(true_high_cond)); 1764 } else if (if_cond == kCondEQ) { 1765 __ b(false_label, ARMCondition(false_high_cond)); 1766 } else { 1767 __ b(true_label, ARMCondition(true_high_cond)); 1768 __ b(false_label, ARMCondition(false_high_cond)); 1769 } 1770 // Must be equal high, so compare the lows. 1771 __ CmpConstant(left_low, val_low); 1772 } else { 1773 Register right_high = right.AsRegisterPairHigh<Register>(); 1774 Register right_low = right.AsRegisterPairLow<Register>(); 1775 1776 __ cmp(left_high, ShifterOperand(right_high)); 1777 if (if_cond == kCondNE) { 1778 __ b(true_label, ARMCondition(true_high_cond)); 1779 } else if (if_cond == kCondEQ) { 1780 __ b(false_label, ARMCondition(false_high_cond)); 1781 } else { 1782 __ b(true_label, ARMCondition(true_high_cond)); 1783 __ b(false_label, ARMCondition(false_high_cond)); 1784 } 1785 // Must be equal high, so compare the lows. 1786 __ cmp(left_low, ShifterOperand(right_low)); 1787 } 1788 // The last comparison might be unsigned. 1789 // TODO: optimize cases where this is always true/false 1790 __ b(true_label, final_condition); 1791} 1792 1793void InstructionCodeGeneratorARM::GenerateCompareTestAndBranch(HCondition* condition, 1794 Label* true_target_in, 1795 Label* false_target_in) { 1796 // Generated branching requires both targets to be explicit. If either of the 1797 // targets is nullptr (fallthrough) use and bind `fallthrough_target` instead. 1798 Label fallthrough_target; 1799 Label* true_target = true_target_in == nullptr ? &fallthrough_target : true_target_in; 1800 Label* false_target = false_target_in == nullptr ? &fallthrough_target : false_target_in; 1801 1802 Primitive::Type type = condition->InputAt(0)->GetType(); 1803 switch (type) { 1804 case Primitive::kPrimLong: 1805 GenerateLongComparesAndJumps(condition, true_target, false_target); 1806 break; 1807 case Primitive::kPrimFloat: 1808 case Primitive::kPrimDouble: 1809 GenerateVcmp(condition); 1810 GenerateFPJumps(condition, true_target, false_target); 1811 break; 1812 default: 1813 LOG(FATAL) << "Unexpected compare type " << type; 1814 } 1815 1816 if (false_target != &fallthrough_target) { 1817 __ b(false_target); 1818 } 1819 1820 if (fallthrough_target.IsLinked()) { 1821 __ Bind(&fallthrough_target); 1822 } 1823} 1824 1825void InstructionCodeGeneratorARM::GenerateTestAndBranch(HInstruction* instruction, 1826 size_t condition_input_index, 1827 Label* true_target, 1828 Label* false_target) { 1829 HInstruction* cond = instruction->InputAt(condition_input_index); 1830 1831 if (true_target == nullptr && false_target == nullptr) { 1832 // Nothing to do. The code always falls through. 1833 return; 1834 } else if (cond->IsIntConstant()) { 1835 // Constant condition, statically compared against "true" (integer value 1). 1836 if (cond->AsIntConstant()->IsTrue()) { 1837 if (true_target != nullptr) { 1838 __ b(true_target); 1839 } 1840 } else { 1841 DCHECK(cond->AsIntConstant()->IsFalse()) << cond->AsIntConstant()->GetValue(); 1842 if (false_target != nullptr) { 1843 __ b(false_target); 1844 } 1845 } 1846 return; 1847 } 1848 1849 // The following code generates these patterns: 1850 // (1) true_target == nullptr && false_target != nullptr 1851 // - opposite condition true => branch to false_target 1852 // (2) true_target != nullptr && false_target == nullptr 1853 // - condition true => branch to true_target 1854 // (3) true_target != nullptr && false_target != nullptr 1855 // - condition true => branch to true_target 1856 // - branch to false_target 1857 if (IsBooleanValueOrMaterializedCondition(cond)) { 1858 // Condition has been materialized, compare the output to 0. 1859 Location cond_val = instruction->GetLocations()->InAt(condition_input_index); 1860 DCHECK(cond_val.IsRegister()); 1861 if (true_target == nullptr) { 1862 __ CompareAndBranchIfZero(cond_val.AsRegister<Register>(), false_target); 1863 } else { 1864 __ CompareAndBranchIfNonZero(cond_val.AsRegister<Register>(), true_target); 1865 } 1866 } else { 1867 // Condition has not been materialized. Use its inputs as the comparison and 1868 // its condition as the branch condition. 1869 HCondition* condition = cond->AsCondition(); 1870 1871 // If this is a long or FP comparison that has been folded into 1872 // the HCondition, generate the comparison directly. 1873 Primitive::Type type = condition->InputAt(0)->GetType(); 1874 if (type == Primitive::kPrimLong || Primitive::IsFloatingPointType(type)) { 1875 GenerateCompareTestAndBranch(condition, true_target, false_target); 1876 return; 1877 } 1878 1879 LocationSummary* locations = cond->GetLocations(); 1880 DCHECK(locations->InAt(0).IsRegister()); 1881 Register left = locations->InAt(0).AsRegister<Register>(); 1882 Location right = locations->InAt(1); 1883 if (right.IsRegister()) { 1884 __ cmp(left, ShifterOperand(right.AsRegister<Register>())); 1885 } else { 1886 DCHECK(right.IsConstant()); 1887 __ CmpConstant(left, CodeGenerator::GetInt32ValueOf(right.GetConstant())); 1888 } 1889 if (true_target == nullptr) { 1890 __ b(false_target, ARMCondition(condition->GetOppositeCondition())); 1891 } else { 1892 __ b(true_target, ARMCondition(condition->GetCondition())); 1893 } 1894 } 1895 1896 // If neither branch falls through (case 3), the conditional branch to `true_target` 1897 // was already emitted (case 2) and we need to emit a jump to `false_target`. 1898 if (true_target != nullptr && false_target != nullptr) { 1899 __ b(false_target); 1900 } 1901} 1902 1903void LocationsBuilderARM::VisitIf(HIf* if_instr) { 1904 LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(if_instr); 1905 if (IsBooleanValueOrMaterializedCondition(if_instr->InputAt(0))) { 1906 locations->SetInAt(0, Location::RequiresRegister()); 1907 } 1908} 1909 1910void InstructionCodeGeneratorARM::VisitIf(HIf* if_instr) { 1911 HBasicBlock* true_successor = if_instr->IfTrueSuccessor(); 1912 HBasicBlock* false_successor = if_instr->IfFalseSuccessor(); 1913 Label* true_target = codegen_->GoesToNextBlock(if_instr->GetBlock(), true_successor) ? 1914 nullptr : codegen_->GetLabelOf(true_successor); 1915 Label* false_target = codegen_->GoesToNextBlock(if_instr->GetBlock(), false_successor) ? 1916 nullptr : codegen_->GetLabelOf(false_successor); 1917 GenerateTestAndBranch(if_instr, /* condition_input_index */ 0, true_target, false_target); 1918} 1919 1920void LocationsBuilderARM::VisitDeoptimize(HDeoptimize* deoptimize) { 1921 LocationSummary* locations = new (GetGraph()->GetArena()) 1922 LocationSummary(deoptimize, LocationSummary::kCallOnSlowPath); 1923 locations->SetCustomSlowPathCallerSaves(RegisterSet::Empty()); // No caller-save registers. 1924 if (IsBooleanValueOrMaterializedCondition(deoptimize->InputAt(0))) { 1925 locations->SetInAt(0, Location::RequiresRegister()); 1926 } 1927} 1928 1929void InstructionCodeGeneratorARM::VisitDeoptimize(HDeoptimize* deoptimize) { 1930 SlowPathCodeARM* slow_path = deopt_slow_paths_.NewSlowPath<DeoptimizationSlowPathARM>(deoptimize); 1931 GenerateTestAndBranch(deoptimize, 1932 /* condition_input_index */ 0, 1933 slow_path->GetEntryLabel(), 1934 /* false_target */ nullptr); 1935} 1936 1937void LocationsBuilderARM::VisitSelect(HSelect* select) { 1938 LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(select); 1939 if (Primitive::IsFloatingPointType(select->GetType())) { 1940 locations->SetInAt(0, Location::RequiresFpuRegister()); 1941 locations->SetInAt(1, Location::RequiresFpuRegister()); 1942 } else { 1943 locations->SetInAt(0, Location::RequiresRegister()); 1944 locations->SetInAt(1, Location::RequiresRegister()); 1945 } 1946 if (IsBooleanValueOrMaterializedCondition(select->GetCondition())) { 1947 locations->SetInAt(2, Location::RequiresRegister()); 1948 } 1949 locations->SetOut(Location::SameAsFirstInput()); 1950} 1951 1952void InstructionCodeGeneratorARM::VisitSelect(HSelect* select) { 1953 LocationSummary* locations = select->GetLocations(); 1954 Label false_target; 1955 GenerateTestAndBranch(select, 1956 /* condition_input_index */ 2, 1957 /* true_target */ nullptr, 1958 &false_target); 1959 codegen_->MoveLocation(locations->Out(), locations->InAt(1), select->GetType()); 1960 __ Bind(&false_target); 1961} 1962 1963void LocationsBuilderARM::VisitNativeDebugInfo(HNativeDebugInfo* info) { 1964 new (GetGraph()->GetArena()) LocationSummary(info); 1965} 1966 1967void InstructionCodeGeneratorARM::VisitNativeDebugInfo(HNativeDebugInfo*) { 1968 // MaybeRecordNativeDebugInfo is already called implicitly in CodeGenerator::Compile. 1969} 1970 1971void CodeGeneratorARM::GenerateNop() { 1972 __ nop(); 1973} 1974 1975void LocationsBuilderARM::HandleCondition(HCondition* cond) { 1976 LocationSummary* locations = 1977 new (GetGraph()->GetArena()) LocationSummary(cond, LocationSummary::kNoCall); 1978 // Handle the long/FP comparisons made in instruction simplification. 1979 switch (cond->InputAt(0)->GetType()) { 1980 case Primitive::kPrimLong: 1981 locations->SetInAt(0, Location::RequiresRegister()); 1982 locations->SetInAt(1, Location::RegisterOrConstant(cond->InputAt(1))); 1983 if (!cond->IsEmittedAtUseSite()) { 1984 locations->SetOut(Location::RequiresRegister(), Location::kOutputOverlap); 1985 } 1986 break; 1987 1988 case Primitive::kPrimFloat: 1989 case Primitive::kPrimDouble: 1990 locations->SetInAt(0, Location::RequiresFpuRegister()); 1991 locations->SetInAt(1, ArithmeticZeroOrFpuRegister(cond->InputAt(1))); 1992 if (!cond->IsEmittedAtUseSite()) { 1993 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 1994 } 1995 break; 1996 1997 default: 1998 locations->SetInAt(0, Location::RequiresRegister()); 1999 locations->SetInAt(1, Location::RegisterOrConstant(cond->InputAt(1))); 2000 if (!cond->IsEmittedAtUseSite()) { 2001 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 2002 } 2003 } 2004} 2005 2006void InstructionCodeGeneratorARM::HandleCondition(HCondition* cond) { 2007 if (cond->IsEmittedAtUseSite()) { 2008 return; 2009 } 2010 2011 LocationSummary* locations = cond->GetLocations(); 2012 Location left = locations->InAt(0); 2013 Location right = locations->InAt(1); 2014 Register out = locations->Out().AsRegister<Register>(); 2015 Label true_label, false_label; 2016 2017 switch (cond->InputAt(0)->GetType()) { 2018 default: { 2019 // Integer case. 2020 if (right.IsRegister()) { 2021 __ cmp(left.AsRegister<Register>(), ShifterOperand(right.AsRegister<Register>())); 2022 } else { 2023 DCHECK(right.IsConstant()); 2024 __ CmpConstant(left.AsRegister<Register>(), 2025 CodeGenerator::GetInt32ValueOf(right.GetConstant())); 2026 } 2027 __ it(ARMCondition(cond->GetCondition()), kItElse); 2028 __ mov(locations->Out().AsRegister<Register>(), ShifterOperand(1), 2029 ARMCondition(cond->GetCondition())); 2030 __ mov(locations->Out().AsRegister<Register>(), ShifterOperand(0), 2031 ARMCondition(cond->GetOppositeCondition())); 2032 return; 2033 } 2034 case Primitive::kPrimLong: 2035 GenerateLongComparesAndJumps(cond, &true_label, &false_label); 2036 break; 2037 case Primitive::kPrimFloat: 2038 case Primitive::kPrimDouble: 2039 GenerateVcmp(cond); 2040 GenerateFPJumps(cond, &true_label, &false_label); 2041 break; 2042 } 2043 2044 // Convert the jumps into the result. 2045 Label done_label; 2046 2047 // False case: result = 0. 2048 __ Bind(&false_label); 2049 __ LoadImmediate(out, 0); 2050 __ b(&done_label); 2051 2052 // True case: result = 1. 2053 __ Bind(&true_label); 2054 __ LoadImmediate(out, 1); 2055 __ Bind(&done_label); 2056} 2057 2058void LocationsBuilderARM::VisitEqual(HEqual* comp) { 2059 HandleCondition(comp); 2060} 2061 2062void InstructionCodeGeneratorARM::VisitEqual(HEqual* comp) { 2063 HandleCondition(comp); 2064} 2065 2066void LocationsBuilderARM::VisitNotEqual(HNotEqual* comp) { 2067 HandleCondition(comp); 2068} 2069 2070void InstructionCodeGeneratorARM::VisitNotEqual(HNotEqual* comp) { 2071 HandleCondition(comp); 2072} 2073 2074void LocationsBuilderARM::VisitLessThan(HLessThan* comp) { 2075 HandleCondition(comp); 2076} 2077 2078void InstructionCodeGeneratorARM::VisitLessThan(HLessThan* comp) { 2079 HandleCondition(comp); 2080} 2081 2082void LocationsBuilderARM::VisitLessThanOrEqual(HLessThanOrEqual* comp) { 2083 HandleCondition(comp); 2084} 2085 2086void InstructionCodeGeneratorARM::VisitLessThanOrEqual(HLessThanOrEqual* comp) { 2087 HandleCondition(comp); 2088} 2089 2090void LocationsBuilderARM::VisitGreaterThan(HGreaterThan* comp) { 2091 HandleCondition(comp); 2092} 2093 2094void InstructionCodeGeneratorARM::VisitGreaterThan(HGreaterThan* comp) { 2095 HandleCondition(comp); 2096} 2097 2098void LocationsBuilderARM::VisitGreaterThanOrEqual(HGreaterThanOrEqual* comp) { 2099 HandleCondition(comp); 2100} 2101 2102void InstructionCodeGeneratorARM::VisitGreaterThanOrEqual(HGreaterThanOrEqual* comp) { 2103 HandleCondition(comp); 2104} 2105 2106void LocationsBuilderARM::VisitBelow(HBelow* comp) { 2107 HandleCondition(comp); 2108} 2109 2110void InstructionCodeGeneratorARM::VisitBelow(HBelow* comp) { 2111 HandleCondition(comp); 2112} 2113 2114void LocationsBuilderARM::VisitBelowOrEqual(HBelowOrEqual* comp) { 2115 HandleCondition(comp); 2116} 2117 2118void InstructionCodeGeneratorARM::VisitBelowOrEqual(HBelowOrEqual* comp) { 2119 HandleCondition(comp); 2120} 2121 2122void LocationsBuilderARM::VisitAbove(HAbove* comp) { 2123 HandleCondition(comp); 2124} 2125 2126void InstructionCodeGeneratorARM::VisitAbove(HAbove* comp) { 2127 HandleCondition(comp); 2128} 2129 2130void LocationsBuilderARM::VisitAboveOrEqual(HAboveOrEqual* comp) { 2131 HandleCondition(comp); 2132} 2133 2134void InstructionCodeGeneratorARM::VisitAboveOrEqual(HAboveOrEqual* comp) { 2135 HandleCondition(comp); 2136} 2137 2138void LocationsBuilderARM::VisitIntConstant(HIntConstant* constant) { 2139 LocationSummary* locations = 2140 new (GetGraph()->GetArena()) LocationSummary(constant, LocationSummary::kNoCall); 2141 locations->SetOut(Location::ConstantLocation(constant)); 2142} 2143 2144void InstructionCodeGeneratorARM::VisitIntConstant(HIntConstant* constant ATTRIBUTE_UNUSED) { 2145 // Will be generated at use site. 2146} 2147 2148void LocationsBuilderARM::VisitNullConstant(HNullConstant* constant) { 2149 LocationSummary* locations = 2150 new (GetGraph()->GetArena()) LocationSummary(constant, LocationSummary::kNoCall); 2151 locations->SetOut(Location::ConstantLocation(constant)); 2152} 2153 2154void InstructionCodeGeneratorARM::VisitNullConstant(HNullConstant* constant ATTRIBUTE_UNUSED) { 2155 // Will be generated at use site. 2156} 2157 2158void LocationsBuilderARM::VisitLongConstant(HLongConstant* constant) { 2159 LocationSummary* locations = 2160 new (GetGraph()->GetArena()) LocationSummary(constant, LocationSummary::kNoCall); 2161 locations->SetOut(Location::ConstantLocation(constant)); 2162} 2163 2164void InstructionCodeGeneratorARM::VisitLongConstant(HLongConstant* constant ATTRIBUTE_UNUSED) { 2165 // Will be generated at use site. 2166} 2167 2168void LocationsBuilderARM::VisitFloatConstant(HFloatConstant* constant) { 2169 LocationSummary* locations = 2170 new (GetGraph()->GetArena()) LocationSummary(constant, LocationSummary::kNoCall); 2171 locations->SetOut(Location::ConstantLocation(constant)); 2172} 2173 2174void InstructionCodeGeneratorARM::VisitFloatConstant(HFloatConstant* constant ATTRIBUTE_UNUSED) { 2175 // Will be generated at use site. 2176} 2177 2178void LocationsBuilderARM::VisitDoubleConstant(HDoubleConstant* constant) { 2179 LocationSummary* locations = 2180 new (GetGraph()->GetArena()) LocationSummary(constant, LocationSummary::kNoCall); 2181 locations->SetOut(Location::ConstantLocation(constant)); 2182} 2183 2184void InstructionCodeGeneratorARM::VisitDoubleConstant(HDoubleConstant* constant ATTRIBUTE_UNUSED) { 2185 // Will be generated at use site. 2186} 2187 2188void LocationsBuilderARM::VisitMemoryBarrier(HMemoryBarrier* memory_barrier) { 2189 memory_barrier->SetLocations(nullptr); 2190} 2191 2192void InstructionCodeGeneratorARM::VisitMemoryBarrier(HMemoryBarrier* memory_barrier) { 2193 codegen_->GenerateMemoryBarrier(memory_barrier->GetBarrierKind()); 2194} 2195 2196void LocationsBuilderARM::VisitReturnVoid(HReturnVoid* ret) { 2197 ret->SetLocations(nullptr); 2198} 2199 2200void InstructionCodeGeneratorARM::VisitReturnVoid(HReturnVoid* ret ATTRIBUTE_UNUSED) { 2201 codegen_->GenerateFrameExit(); 2202} 2203 2204void LocationsBuilderARM::VisitReturn(HReturn* ret) { 2205 LocationSummary* locations = 2206 new (GetGraph()->GetArena()) LocationSummary(ret, LocationSummary::kNoCall); 2207 locations->SetInAt(0, parameter_visitor_.GetReturnLocation(ret->InputAt(0)->GetType())); 2208} 2209 2210void InstructionCodeGeneratorARM::VisitReturn(HReturn* ret ATTRIBUTE_UNUSED) { 2211 codegen_->GenerateFrameExit(); 2212} 2213 2214void LocationsBuilderARM::VisitInvokeUnresolved(HInvokeUnresolved* invoke) { 2215 // The trampoline uses the same calling convention as dex calling conventions, 2216 // except instead of loading arg0/r0 with the target Method*, arg0/r0 will contain 2217 // the method_idx. 2218 HandleInvoke(invoke); 2219} 2220 2221void InstructionCodeGeneratorARM::VisitInvokeUnresolved(HInvokeUnresolved* invoke) { 2222 codegen_->GenerateInvokeUnresolvedRuntimeCall(invoke); 2223} 2224 2225void LocationsBuilderARM::VisitInvokeStaticOrDirect(HInvokeStaticOrDirect* invoke) { 2226 // Explicit clinit checks triggered by static invokes must have been pruned by 2227 // art::PrepareForRegisterAllocation. 2228 DCHECK(!invoke->IsStaticWithExplicitClinitCheck()); 2229 2230 IntrinsicLocationsBuilderARM intrinsic(codegen_); 2231 if (intrinsic.TryDispatch(invoke)) { 2232 if (invoke->GetLocations()->CanCall() && invoke->HasPcRelativeDexCache()) { 2233 invoke->GetLocations()->SetInAt(invoke->GetSpecialInputIndex(), Location::Any()); 2234 } 2235 return; 2236 } 2237 2238 HandleInvoke(invoke); 2239 2240 // For PC-relative dex cache the invoke has an extra input, the PC-relative address base. 2241 if (invoke->HasPcRelativeDexCache()) { 2242 invoke->GetLocations()->SetInAt(invoke->GetSpecialInputIndex(), Location::RequiresRegister()); 2243 } 2244} 2245 2246static bool TryGenerateIntrinsicCode(HInvoke* invoke, CodeGeneratorARM* codegen) { 2247 if (invoke->GetLocations()->Intrinsified()) { 2248 IntrinsicCodeGeneratorARM intrinsic(codegen); 2249 intrinsic.Dispatch(invoke); 2250 return true; 2251 } 2252 return false; 2253} 2254 2255void InstructionCodeGeneratorARM::VisitInvokeStaticOrDirect(HInvokeStaticOrDirect* invoke) { 2256 // Explicit clinit checks triggered by static invokes must have been pruned by 2257 // art::PrepareForRegisterAllocation. 2258 DCHECK(!invoke->IsStaticWithExplicitClinitCheck()); 2259 2260 if (TryGenerateIntrinsicCode(invoke, codegen_)) { 2261 return; 2262 } 2263 2264 LocationSummary* locations = invoke->GetLocations(); 2265 codegen_->GenerateStaticOrDirectCall( 2266 invoke, locations->HasTemps() ? locations->GetTemp(0) : Location::NoLocation()); 2267 codegen_->RecordPcInfo(invoke, invoke->GetDexPc()); 2268} 2269 2270void LocationsBuilderARM::HandleInvoke(HInvoke* invoke) { 2271 InvokeDexCallingConventionVisitorARM calling_convention_visitor; 2272 CodeGenerator::CreateCommonInvokeLocationSummary(invoke, &calling_convention_visitor); 2273} 2274 2275void LocationsBuilderARM::VisitInvokeVirtual(HInvokeVirtual* invoke) { 2276 IntrinsicLocationsBuilderARM intrinsic(codegen_); 2277 if (intrinsic.TryDispatch(invoke)) { 2278 return; 2279 } 2280 2281 HandleInvoke(invoke); 2282} 2283 2284void InstructionCodeGeneratorARM::VisitInvokeVirtual(HInvokeVirtual* invoke) { 2285 if (TryGenerateIntrinsicCode(invoke, codegen_)) { 2286 return; 2287 } 2288 2289 codegen_->GenerateVirtualCall(invoke, invoke->GetLocations()->GetTemp(0)); 2290 DCHECK(!codegen_->IsLeafMethod()); 2291 codegen_->RecordPcInfo(invoke, invoke->GetDexPc()); 2292} 2293 2294void LocationsBuilderARM::VisitInvokeInterface(HInvokeInterface* invoke) { 2295 HandleInvoke(invoke); 2296 // Add the hidden argument. 2297 invoke->GetLocations()->AddTemp(Location::RegisterLocation(R12)); 2298} 2299 2300void InstructionCodeGeneratorARM::VisitInvokeInterface(HInvokeInterface* invoke) { 2301 // TODO: b/18116999, our IMTs can miss an IncompatibleClassChangeError. 2302 LocationSummary* locations = invoke->GetLocations(); 2303 Register temp = locations->GetTemp(0).AsRegister<Register>(); 2304 Register hidden_reg = locations->GetTemp(1).AsRegister<Register>(); 2305 Location receiver = locations->InAt(0); 2306 uint32_t class_offset = mirror::Object::ClassOffset().Int32Value(); 2307 2308 // Set the hidden argument. This is safe to do this here, as R12 2309 // won't be modified thereafter, before the `blx` (call) instruction. 2310 DCHECK_EQ(R12, hidden_reg); 2311 __ LoadImmediate(hidden_reg, invoke->GetDexMethodIndex()); 2312 2313 if (receiver.IsStackSlot()) { 2314 __ LoadFromOffset(kLoadWord, temp, SP, receiver.GetStackIndex()); 2315 // /* HeapReference<Class> */ temp = temp->klass_ 2316 __ LoadFromOffset(kLoadWord, temp, temp, class_offset); 2317 } else { 2318 // /* HeapReference<Class> */ temp = receiver->klass_ 2319 __ LoadFromOffset(kLoadWord, temp, receiver.AsRegister<Register>(), class_offset); 2320 } 2321 codegen_->MaybeRecordImplicitNullCheck(invoke); 2322 // Instead of simply (possibly) unpoisoning `temp` here, we should 2323 // emit a read barrier for the previous class reference load. 2324 // However this is not required in practice, as this is an 2325 // intermediate/temporary reference and because the current 2326 // concurrent copying collector keeps the from-space memory 2327 // intact/accessible until the end of the marking phase (the 2328 // concurrent copying collector may not in the future). 2329 __ MaybeUnpoisonHeapReference(temp); 2330 __ LoadFromOffset(kLoadWord, temp, temp, 2331 mirror::Class::ImtPtrOffset(kArmPointerSize).Uint32Value()); 2332 uint32_t method_offset = static_cast<uint32_t>(ImTable::OffsetOfElement( 2333 invoke->GetImtIndex(), kArmPointerSize)); 2334 // temp = temp->GetImtEntryAt(method_offset); 2335 __ LoadFromOffset(kLoadWord, temp, temp, method_offset); 2336 uint32_t entry_point = 2337 ArtMethod::EntryPointFromQuickCompiledCodeOffset(kArmPointerSize).Int32Value(); 2338 // LR = temp->GetEntryPoint(); 2339 __ LoadFromOffset(kLoadWord, LR, temp, entry_point); 2340 // LR(); 2341 __ blx(LR); 2342 DCHECK(!codegen_->IsLeafMethod()); 2343 codegen_->RecordPcInfo(invoke, invoke->GetDexPc()); 2344} 2345 2346void LocationsBuilderARM::VisitNeg(HNeg* neg) { 2347 LocationSummary* locations = 2348 new (GetGraph()->GetArena()) LocationSummary(neg, LocationSummary::kNoCall); 2349 switch (neg->GetResultType()) { 2350 case Primitive::kPrimInt: { 2351 locations->SetInAt(0, Location::RequiresRegister()); 2352 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 2353 break; 2354 } 2355 case Primitive::kPrimLong: { 2356 locations->SetInAt(0, Location::RequiresRegister()); 2357 locations->SetOut(Location::RequiresRegister(), Location::kOutputOverlap); 2358 break; 2359 } 2360 2361 case Primitive::kPrimFloat: 2362 case Primitive::kPrimDouble: 2363 locations->SetInAt(0, Location::RequiresFpuRegister()); 2364 locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap); 2365 break; 2366 2367 default: 2368 LOG(FATAL) << "Unexpected neg type " << neg->GetResultType(); 2369 } 2370} 2371 2372void InstructionCodeGeneratorARM::VisitNeg(HNeg* neg) { 2373 LocationSummary* locations = neg->GetLocations(); 2374 Location out = locations->Out(); 2375 Location in = locations->InAt(0); 2376 switch (neg->GetResultType()) { 2377 case Primitive::kPrimInt: 2378 DCHECK(in.IsRegister()); 2379 __ rsb(out.AsRegister<Register>(), in.AsRegister<Register>(), ShifterOperand(0)); 2380 break; 2381 2382 case Primitive::kPrimLong: 2383 DCHECK(in.IsRegisterPair()); 2384 // out.lo = 0 - in.lo (and update the carry/borrow (C) flag) 2385 __ rsbs(out.AsRegisterPairLow<Register>(), 2386 in.AsRegisterPairLow<Register>(), 2387 ShifterOperand(0)); 2388 // We cannot emit an RSC (Reverse Subtract with Carry) 2389 // instruction here, as it does not exist in the Thumb-2 2390 // instruction set. We use the following approach 2391 // using SBC and SUB instead. 2392 // 2393 // out.hi = -C 2394 __ sbc(out.AsRegisterPairHigh<Register>(), 2395 out.AsRegisterPairHigh<Register>(), 2396 ShifterOperand(out.AsRegisterPairHigh<Register>())); 2397 // out.hi = out.hi - in.hi 2398 __ sub(out.AsRegisterPairHigh<Register>(), 2399 out.AsRegisterPairHigh<Register>(), 2400 ShifterOperand(in.AsRegisterPairHigh<Register>())); 2401 break; 2402 2403 case Primitive::kPrimFloat: 2404 DCHECK(in.IsFpuRegister()); 2405 __ vnegs(out.AsFpuRegister<SRegister>(), in.AsFpuRegister<SRegister>()); 2406 break; 2407 2408 case Primitive::kPrimDouble: 2409 DCHECK(in.IsFpuRegisterPair()); 2410 __ vnegd(FromLowSToD(out.AsFpuRegisterPairLow<SRegister>()), 2411 FromLowSToD(in.AsFpuRegisterPairLow<SRegister>())); 2412 break; 2413 2414 default: 2415 LOG(FATAL) << "Unexpected neg type " << neg->GetResultType(); 2416 } 2417} 2418 2419void LocationsBuilderARM::VisitTypeConversion(HTypeConversion* conversion) { 2420 Primitive::Type result_type = conversion->GetResultType(); 2421 Primitive::Type input_type = conversion->GetInputType(); 2422 DCHECK_NE(result_type, input_type); 2423 2424 // The float-to-long, double-to-long and long-to-float type conversions 2425 // rely on a call to the runtime. 2426 LocationSummary::CallKind call_kind = 2427 (((input_type == Primitive::kPrimFloat || input_type == Primitive::kPrimDouble) 2428 && result_type == Primitive::kPrimLong) 2429 || (input_type == Primitive::kPrimLong && result_type == Primitive::kPrimFloat)) 2430 ? LocationSummary::kCallOnMainOnly 2431 : LocationSummary::kNoCall; 2432 LocationSummary* locations = 2433 new (GetGraph()->GetArena()) LocationSummary(conversion, call_kind); 2434 2435 // The Java language does not allow treating boolean as an integral type but 2436 // our bit representation makes it safe. 2437 2438 switch (result_type) { 2439 case Primitive::kPrimByte: 2440 switch (input_type) { 2441 case Primitive::kPrimLong: 2442 // Type conversion from long to byte is a result of code transformations. 2443 case Primitive::kPrimBoolean: 2444 // Boolean input is a result of code transformations. 2445 case Primitive::kPrimShort: 2446 case Primitive::kPrimInt: 2447 case Primitive::kPrimChar: 2448 // Processing a Dex `int-to-byte' instruction. 2449 locations->SetInAt(0, Location::RequiresRegister()); 2450 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 2451 break; 2452 2453 default: 2454 LOG(FATAL) << "Unexpected type conversion from " << input_type 2455 << " to " << result_type; 2456 } 2457 break; 2458 2459 case Primitive::kPrimShort: 2460 switch (input_type) { 2461 case Primitive::kPrimLong: 2462 // Type conversion from long to short is a result of code transformations. 2463 case Primitive::kPrimBoolean: 2464 // Boolean input is a result of code transformations. 2465 case Primitive::kPrimByte: 2466 case Primitive::kPrimInt: 2467 case Primitive::kPrimChar: 2468 // Processing a Dex `int-to-short' instruction. 2469 locations->SetInAt(0, Location::RequiresRegister()); 2470 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 2471 break; 2472 2473 default: 2474 LOG(FATAL) << "Unexpected type conversion from " << input_type 2475 << " to " << result_type; 2476 } 2477 break; 2478 2479 case Primitive::kPrimInt: 2480 switch (input_type) { 2481 case Primitive::kPrimLong: 2482 // Processing a Dex `long-to-int' instruction. 2483 locations->SetInAt(0, Location::Any()); 2484 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 2485 break; 2486 2487 case Primitive::kPrimFloat: 2488 // Processing a Dex `float-to-int' instruction. 2489 locations->SetInAt(0, Location::RequiresFpuRegister()); 2490 locations->SetOut(Location::RequiresRegister()); 2491 locations->AddTemp(Location::RequiresFpuRegister()); 2492 break; 2493 2494 case Primitive::kPrimDouble: 2495 // Processing a Dex `double-to-int' instruction. 2496 locations->SetInAt(0, Location::RequiresFpuRegister()); 2497 locations->SetOut(Location::RequiresRegister()); 2498 locations->AddTemp(Location::RequiresFpuRegister()); 2499 break; 2500 2501 default: 2502 LOG(FATAL) << "Unexpected type conversion from " << input_type 2503 << " to " << result_type; 2504 } 2505 break; 2506 2507 case Primitive::kPrimLong: 2508 switch (input_type) { 2509 case Primitive::kPrimBoolean: 2510 // Boolean input is a result of code transformations. 2511 case Primitive::kPrimByte: 2512 case Primitive::kPrimShort: 2513 case Primitive::kPrimInt: 2514 case Primitive::kPrimChar: 2515 // Processing a Dex `int-to-long' instruction. 2516 locations->SetInAt(0, Location::RequiresRegister()); 2517 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 2518 break; 2519 2520 case Primitive::kPrimFloat: { 2521 // Processing a Dex `float-to-long' instruction. 2522 InvokeRuntimeCallingConvention calling_convention; 2523 locations->SetInAt(0, Location::FpuRegisterLocation( 2524 calling_convention.GetFpuRegisterAt(0))); 2525 locations->SetOut(Location::RegisterPairLocation(R0, R1)); 2526 break; 2527 } 2528 2529 case Primitive::kPrimDouble: { 2530 // Processing a Dex `double-to-long' instruction. 2531 InvokeRuntimeCallingConvention calling_convention; 2532 locations->SetInAt(0, Location::FpuRegisterPairLocation( 2533 calling_convention.GetFpuRegisterAt(0), 2534 calling_convention.GetFpuRegisterAt(1))); 2535 locations->SetOut(Location::RegisterPairLocation(R0, R1)); 2536 break; 2537 } 2538 2539 default: 2540 LOG(FATAL) << "Unexpected type conversion from " << input_type 2541 << " to " << result_type; 2542 } 2543 break; 2544 2545 case Primitive::kPrimChar: 2546 switch (input_type) { 2547 case Primitive::kPrimLong: 2548 // Type conversion from long to char is a result of code transformations. 2549 case Primitive::kPrimBoolean: 2550 // Boolean input is a result of code transformations. 2551 case Primitive::kPrimByte: 2552 case Primitive::kPrimShort: 2553 case Primitive::kPrimInt: 2554 // Processing a Dex `int-to-char' instruction. 2555 locations->SetInAt(0, Location::RequiresRegister()); 2556 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 2557 break; 2558 2559 default: 2560 LOG(FATAL) << "Unexpected type conversion from " << input_type 2561 << " to " << result_type; 2562 } 2563 break; 2564 2565 case Primitive::kPrimFloat: 2566 switch (input_type) { 2567 case Primitive::kPrimBoolean: 2568 // Boolean input is a result of code transformations. 2569 case Primitive::kPrimByte: 2570 case Primitive::kPrimShort: 2571 case Primitive::kPrimInt: 2572 case Primitive::kPrimChar: 2573 // Processing a Dex `int-to-float' instruction. 2574 locations->SetInAt(0, Location::RequiresRegister()); 2575 locations->SetOut(Location::RequiresFpuRegister()); 2576 break; 2577 2578 case Primitive::kPrimLong: { 2579 // Processing a Dex `long-to-float' instruction. 2580 InvokeRuntimeCallingConvention calling_convention; 2581 locations->SetInAt(0, Location::RegisterPairLocation( 2582 calling_convention.GetRegisterAt(0), calling_convention.GetRegisterAt(1))); 2583 locations->SetOut(Location::FpuRegisterLocation(calling_convention.GetFpuRegisterAt(0))); 2584 break; 2585 } 2586 2587 case Primitive::kPrimDouble: 2588 // Processing a Dex `double-to-float' instruction. 2589 locations->SetInAt(0, Location::RequiresFpuRegister()); 2590 locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap); 2591 break; 2592 2593 default: 2594 LOG(FATAL) << "Unexpected type conversion from " << input_type 2595 << " to " << result_type; 2596 }; 2597 break; 2598 2599 case Primitive::kPrimDouble: 2600 switch (input_type) { 2601 case Primitive::kPrimBoolean: 2602 // Boolean input is a result of code transformations. 2603 case Primitive::kPrimByte: 2604 case Primitive::kPrimShort: 2605 case Primitive::kPrimInt: 2606 case Primitive::kPrimChar: 2607 // Processing a Dex `int-to-double' instruction. 2608 locations->SetInAt(0, Location::RequiresRegister()); 2609 locations->SetOut(Location::RequiresFpuRegister()); 2610 break; 2611 2612 case Primitive::kPrimLong: 2613 // Processing a Dex `long-to-double' instruction. 2614 locations->SetInAt(0, Location::RequiresRegister()); 2615 locations->SetOut(Location::RequiresFpuRegister()); 2616 locations->AddTemp(Location::RequiresFpuRegister()); 2617 locations->AddTemp(Location::RequiresFpuRegister()); 2618 break; 2619 2620 case Primitive::kPrimFloat: 2621 // Processing a Dex `float-to-double' instruction. 2622 locations->SetInAt(0, Location::RequiresFpuRegister()); 2623 locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap); 2624 break; 2625 2626 default: 2627 LOG(FATAL) << "Unexpected type conversion from " << input_type 2628 << " to " << result_type; 2629 }; 2630 break; 2631 2632 default: 2633 LOG(FATAL) << "Unexpected type conversion from " << input_type 2634 << " to " << result_type; 2635 } 2636} 2637 2638void InstructionCodeGeneratorARM::VisitTypeConversion(HTypeConversion* conversion) { 2639 LocationSummary* locations = conversion->GetLocations(); 2640 Location out = locations->Out(); 2641 Location in = locations->InAt(0); 2642 Primitive::Type result_type = conversion->GetResultType(); 2643 Primitive::Type input_type = conversion->GetInputType(); 2644 DCHECK_NE(result_type, input_type); 2645 switch (result_type) { 2646 case Primitive::kPrimByte: 2647 switch (input_type) { 2648 case Primitive::kPrimLong: 2649 // Type conversion from long to byte is a result of code transformations. 2650 __ sbfx(out.AsRegister<Register>(), in.AsRegisterPairLow<Register>(), 0, 8); 2651 break; 2652 case Primitive::kPrimBoolean: 2653 // Boolean input is a result of code transformations. 2654 case Primitive::kPrimShort: 2655 case Primitive::kPrimInt: 2656 case Primitive::kPrimChar: 2657 // Processing a Dex `int-to-byte' instruction. 2658 __ sbfx(out.AsRegister<Register>(), in.AsRegister<Register>(), 0, 8); 2659 break; 2660 2661 default: 2662 LOG(FATAL) << "Unexpected type conversion from " << input_type 2663 << " to " << result_type; 2664 } 2665 break; 2666 2667 case Primitive::kPrimShort: 2668 switch (input_type) { 2669 case Primitive::kPrimLong: 2670 // Type conversion from long to short is a result of code transformations. 2671 __ sbfx(out.AsRegister<Register>(), in.AsRegisterPairLow<Register>(), 0, 16); 2672 break; 2673 case Primitive::kPrimBoolean: 2674 // Boolean input is a result of code transformations. 2675 case Primitive::kPrimByte: 2676 case Primitive::kPrimInt: 2677 case Primitive::kPrimChar: 2678 // Processing a Dex `int-to-short' instruction. 2679 __ sbfx(out.AsRegister<Register>(), in.AsRegister<Register>(), 0, 16); 2680 break; 2681 2682 default: 2683 LOG(FATAL) << "Unexpected type conversion from " << input_type 2684 << " to " << result_type; 2685 } 2686 break; 2687 2688 case Primitive::kPrimInt: 2689 switch (input_type) { 2690 case Primitive::kPrimLong: 2691 // Processing a Dex `long-to-int' instruction. 2692 DCHECK(out.IsRegister()); 2693 if (in.IsRegisterPair()) { 2694 __ Mov(out.AsRegister<Register>(), in.AsRegisterPairLow<Register>()); 2695 } else if (in.IsDoubleStackSlot()) { 2696 __ LoadFromOffset(kLoadWord, out.AsRegister<Register>(), SP, in.GetStackIndex()); 2697 } else { 2698 DCHECK(in.IsConstant()); 2699 DCHECK(in.GetConstant()->IsLongConstant()); 2700 int64_t value = in.GetConstant()->AsLongConstant()->GetValue(); 2701 __ LoadImmediate(out.AsRegister<Register>(), static_cast<int32_t>(value)); 2702 } 2703 break; 2704 2705 case Primitive::kPrimFloat: { 2706 // Processing a Dex `float-to-int' instruction. 2707 SRegister temp = locations->GetTemp(0).AsFpuRegisterPairLow<SRegister>(); 2708 __ vcvtis(temp, in.AsFpuRegister<SRegister>()); 2709 __ vmovrs(out.AsRegister<Register>(), temp); 2710 break; 2711 } 2712 2713 case Primitive::kPrimDouble: { 2714 // Processing a Dex `double-to-int' instruction. 2715 SRegister temp_s = locations->GetTemp(0).AsFpuRegisterPairLow<SRegister>(); 2716 __ vcvtid(temp_s, FromLowSToD(in.AsFpuRegisterPairLow<SRegister>())); 2717 __ vmovrs(out.AsRegister<Register>(), temp_s); 2718 break; 2719 } 2720 2721 default: 2722 LOG(FATAL) << "Unexpected type conversion from " << input_type 2723 << " to " << result_type; 2724 } 2725 break; 2726 2727 case Primitive::kPrimLong: 2728 switch (input_type) { 2729 case Primitive::kPrimBoolean: 2730 // Boolean input is a result of code transformations. 2731 case Primitive::kPrimByte: 2732 case Primitive::kPrimShort: 2733 case Primitive::kPrimInt: 2734 case Primitive::kPrimChar: 2735 // Processing a Dex `int-to-long' instruction. 2736 DCHECK(out.IsRegisterPair()); 2737 DCHECK(in.IsRegister()); 2738 __ Mov(out.AsRegisterPairLow<Register>(), in.AsRegister<Register>()); 2739 // Sign extension. 2740 __ Asr(out.AsRegisterPairHigh<Register>(), 2741 out.AsRegisterPairLow<Register>(), 2742 31); 2743 break; 2744 2745 case Primitive::kPrimFloat: 2746 // Processing a Dex `float-to-long' instruction. 2747 codegen_->InvokeRuntime(kQuickF2l, conversion, conversion->GetDexPc()); 2748 CheckEntrypointTypes<kQuickF2l, int64_t, float>(); 2749 break; 2750 2751 case Primitive::kPrimDouble: 2752 // Processing a Dex `double-to-long' instruction. 2753 codegen_->InvokeRuntime(kQuickD2l, conversion, conversion->GetDexPc()); 2754 CheckEntrypointTypes<kQuickD2l, int64_t, double>(); 2755 break; 2756 2757 default: 2758 LOG(FATAL) << "Unexpected type conversion from " << input_type 2759 << " to " << result_type; 2760 } 2761 break; 2762 2763 case Primitive::kPrimChar: 2764 switch (input_type) { 2765 case Primitive::kPrimLong: 2766 // Type conversion from long to char is a result of code transformations. 2767 __ ubfx(out.AsRegister<Register>(), in.AsRegisterPairLow<Register>(), 0, 16); 2768 break; 2769 case Primitive::kPrimBoolean: 2770 // Boolean input is a result of code transformations. 2771 case Primitive::kPrimByte: 2772 case Primitive::kPrimShort: 2773 case Primitive::kPrimInt: 2774 // Processing a Dex `int-to-char' instruction. 2775 __ ubfx(out.AsRegister<Register>(), in.AsRegister<Register>(), 0, 16); 2776 break; 2777 2778 default: 2779 LOG(FATAL) << "Unexpected type conversion from " << input_type 2780 << " to " << result_type; 2781 } 2782 break; 2783 2784 case Primitive::kPrimFloat: 2785 switch (input_type) { 2786 case Primitive::kPrimBoolean: 2787 // Boolean input is a result of code transformations. 2788 case Primitive::kPrimByte: 2789 case Primitive::kPrimShort: 2790 case Primitive::kPrimInt: 2791 case Primitive::kPrimChar: { 2792 // Processing a Dex `int-to-float' instruction. 2793 __ vmovsr(out.AsFpuRegister<SRegister>(), in.AsRegister<Register>()); 2794 __ vcvtsi(out.AsFpuRegister<SRegister>(), out.AsFpuRegister<SRegister>()); 2795 break; 2796 } 2797 2798 case Primitive::kPrimLong: 2799 // Processing a Dex `long-to-float' instruction. 2800 codegen_->InvokeRuntime(kQuickL2f, conversion, conversion->GetDexPc()); 2801 CheckEntrypointTypes<kQuickL2f, float, int64_t>(); 2802 break; 2803 2804 case Primitive::kPrimDouble: 2805 // Processing a Dex `double-to-float' instruction. 2806 __ vcvtsd(out.AsFpuRegister<SRegister>(), 2807 FromLowSToD(in.AsFpuRegisterPairLow<SRegister>())); 2808 break; 2809 2810 default: 2811 LOG(FATAL) << "Unexpected type conversion from " << input_type 2812 << " to " << result_type; 2813 }; 2814 break; 2815 2816 case Primitive::kPrimDouble: 2817 switch (input_type) { 2818 case Primitive::kPrimBoolean: 2819 // Boolean input is a result of code transformations. 2820 case Primitive::kPrimByte: 2821 case Primitive::kPrimShort: 2822 case Primitive::kPrimInt: 2823 case Primitive::kPrimChar: { 2824 // Processing a Dex `int-to-double' instruction. 2825 __ vmovsr(out.AsFpuRegisterPairLow<SRegister>(), in.AsRegister<Register>()); 2826 __ vcvtdi(FromLowSToD(out.AsFpuRegisterPairLow<SRegister>()), 2827 out.AsFpuRegisterPairLow<SRegister>()); 2828 break; 2829 } 2830 2831 case Primitive::kPrimLong: { 2832 // Processing a Dex `long-to-double' instruction. 2833 Register low = in.AsRegisterPairLow<Register>(); 2834 Register high = in.AsRegisterPairHigh<Register>(); 2835 SRegister out_s = out.AsFpuRegisterPairLow<SRegister>(); 2836 DRegister out_d = FromLowSToD(out_s); 2837 SRegister temp_s = locations->GetTemp(0).AsFpuRegisterPairLow<SRegister>(); 2838 DRegister temp_d = FromLowSToD(temp_s); 2839 SRegister constant_s = locations->GetTemp(1).AsFpuRegisterPairLow<SRegister>(); 2840 DRegister constant_d = FromLowSToD(constant_s); 2841 2842 // temp_d = int-to-double(high) 2843 __ vmovsr(temp_s, high); 2844 __ vcvtdi(temp_d, temp_s); 2845 // constant_d = k2Pow32EncodingForDouble 2846 __ LoadDImmediate(constant_d, bit_cast<double, int64_t>(k2Pow32EncodingForDouble)); 2847 // out_d = unsigned-to-double(low) 2848 __ vmovsr(out_s, low); 2849 __ vcvtdu(out_d, out_s); 2850 // out_d += temp_d * constant_d 2851 __ vmlad(out_d, temp_d, constant_d); 2852 break; 2853 } 2854 2855 case Primitive::kPrimFloat: 2856 // Processing a Dex `float-to-double' instruction. 2857 __ vcvtds(FromLowSToD(out.AsFpuRegisterPairLow<SRegister>()), 2858 in.AsFpuRegister<SRegister>()); 2859 break; 2860 2861 default: 2862 LOG(FATAL) << "Unexpected type conversion from " << input_type 2863 << " to " << result_type; 2864 }; 2865 break; 2866 2867 default: 2868 LOG(FATAL) << "Unexpected type conversion from " << input_type 2869 << " to " << result_type; 2870 } 2871} 2872 2873void LocationsBuilderARM::VisitAdd(HAdd* add) { 2874 LocationSummary* locations = 2875 new (GetGraph()->GetArena()) LocationSummary(add, LocationSummary::kNoCall); 2876 switch (add->GetResultType()) { 2877 case Primitive::kPrimInt: { 2878 locations->SetInAt(0, Location::RequiresRegister()); 2879 locations->SetInAt(1, Location::RegisterOrConstant(add->InputAt(1))); 2880 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 2881 break; 2882 } 2883 2884 case Primitive::kPrimLong: { 2885 locations->SetInAt(0, Location::RequiresRegister()); 2886 locations->SetInAt(1, ArmEncodableConstantOrRegister(add->InputAt(1), ADD)); 2887 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 2888 break; 2889 } 2890 2891 case Primitive::kPrimFloat: 2892 case Primitive::kPrimDouble: { 2893 locations->SetInAt(0, Location::RequiresFpuRegister()); 2894 locations->SetInAt(1, Location::RequiresFpuRegister()); 2895 locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap); 2896 break; 2897 } 2898 2899 default: 2900 LOG(FATAL) << "Unexpected add type " << add->GetResultType(); 2901 } 2902} 2903 2904void InstructionCodeGeneratorARM::VisitAdd(HAdd* add) { 2905 LocationSummary* locations = add->GetLocations(); 2906 Location out = locations->Out(); 2907 Location first = locations->InAt(0); 2908 Location second = locations->InAt(1); 2909 switch (add->GetResultType()) { 2910 case Primitive::kPrimInt: 2911 if (second.IsRegister()) { 2912 __ add(out.AsRegister<Register>(), 2913 first.AsRegister<Register>(), 2914 ShifterOperand(second.AsRegister<Register>())); 2915 } else { 2916 __ AddConstant(out.AsRegister<Register>(), 2917 first.AsRegister<Register>(), 2918 second.GetConstant()->AsIntConstant()->GetValue()); 2919 } 2920 break; 2921 2922 case Primitive::kPrimLong: { 2923 if (second.IsConstant()) { 2924 uint64_t value = static_cast<uint64_t>(Int64FromConstant(second.GetConstant())); 2925 GenerateAddLongConst(out, first, value); 2926 } else { 2927 DCHECK(second.IsRegisterPair()); 2928 __ adds(out.AsRegisterPairLow<Register>(), 2929 first.AsRegisterPairLow<Register>(), 2930 ShifterOperand(second.AsRegisterPairLow<Register>())); 2931 __ adc(out.AsRegisterPairHigh<Register>(), 2932 first.AsRegisterPairHigh<Register>(), 2933 ShifterOperand(second.AsRegisterPairHigh<Register>())); 2934 } 2935 break; 2936 } 2937 2938 case Primitive::kPrimFloat: 2939 __ vadds(out.AsFpuRegister<SRegister>(), 2940 first.AsFpuRegister<SRegister>(), 2941 second.AsFpuRegister<SRegister>()); 2942 break; 2943 2944 case Primitive::kPrimDouble: 2945 __ vaddd(FromLowSToD(out.AsFpuRegisterPairLow<SRegister>()), 2946 FromLowSToD(first.AsFpuRegisterPairLow<SRegister>()), 2947 FromLowSToD(second.AsFpuRegisterPairLow<SRegister>())); 2948 break; 2949 2950 default: 2951 LOG(FATAL) << "Unexpected add type " << add->GetResultType(); 2952 } 2953} 2954 2955void LocationsBuilderARM::VisitSub(HSub* sub) { 2956 LocationSummary* locations = 2957 new (GetGraph()->GetArena()) LocationSummary(sub, LocationSummary::kNoCall); 2958 switch (sub->GetResultType()) { 2959 case Primitive::kPrimInt: { 2960 locations->SetInAt(0, Location::RequiresRegister()); 2961 locations->SetInAt(1, Location::RegisterOrConstant(sub->InputAt(1))); 2962 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 2963 break; 2964 } 2965 2966 case Primitive::kPrimLong: { 2967 locations->SetInAt(0, Location::RequiresRegister()); 2968 locations->SetInAt(1, ArmEncodableConstantOrRegister(sub->InputAt(1), SUB)); 2969 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 2970 break; 2971 } 2972 case Primitive::kPrimFloat: 2973 case Primitive::kPrimDouble: { 2974 locations->SetInAt(0, Location::RequiresFpuRegister()); 2975 locations->SetInAt(1, Location::RequiresFpuRegister()); 2976 locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap); 2977 break; 2978 } 2979 default: 2980 LOG(FATAL) << "Unexpected sub type " << sub->GetResultType(); 2981 } 2982} 2983 2984void InstructionCodeGeneratorARM::VisitSub(HSub* sub) { 2985 LocationSummary* locations = sub->GetLocations(); 2986 Location out = locations->Out(); 2987 Location first = locations->InAt(0); 2988 Location second = locations->InAt(1); 2989 switch (sub->GetResultType()) { 2990 case Primitive::kPrimInt: { 2991 if (second.IsRegister()) { 2992 __ sub(out.AsRegister<Register>(), 2993 first.AsRegister<Register>(), 2994 ShifterOperand(second.AsRegister<Register>())); 2995 } else { 2996 __ AddConstant(out.AsRegister<Register>(), 2997 first.AsRegister<Register>(), 2998 -second.GetConstant()->AsIntConstant()->GetValue()); 2999 } 3000 break; 3001 } 3002 3003 case Primitive::kPrimLong: { 3004 if (second.IsConstant()) { 3005 uint64_t value = static_cast<uint64_t>(Int64FromConstant(second.GetConstant())); 3006 GenerateAddLongConst(out, first, -value); 3007 } else { 3008 DCHECK(second.IsRegisterPair()); 3009 __ subs(out.AsRegisterPairLow<Register>(), 3010 first.AsRegisterPairLow<Register>(), 3011 ShifterOperand(second.AsRegisterPairLow<Register>())); 3012 __ sbc(out.AsRegisterPairHigh<Register>(), 3013 first.AsRegisterPairHigh<Register>(), 3014 ShifterOperand(second.AsRegisterPairHigh<Register>())); 3015 } 3016 break; 3017 } 3018 3019 case Primitive::kPrimFloat: { 3020 __ vsubs(out.AsFpuRegister<SRegister>(), 3021 first.AsFpuRegister<SRegister>(), 3022 second.AsFpuRegister<SRegister>()); 3023 break; 3024 } 3025 3026 case Primitive::kPrimDouble: { 3027 __ vsubd(FromLowSToD(out.AsFpuRegisterPairLow<SRegister>()), 3028 FromLowSToD(first.AsFpuRegisterPairLow<SRegister>()), 3029 FromLowSToD(second.AsFpuRegisterPairLow<SRegister>())); 3030 break; 3031 } 3032 3033 3034 default: 3035 LOG(FATAL) << "Unexpected sub type " << sub->GetResultType(); 3036 } 3037} 3038 3039void LocationsBuilderARM::VisitMul(HMul* mul) { 3040 LocationSummary* locations = 3041 new (GetGraph()->GetArena()) LocationSummary(mul, LocationSummary::kNoCall); 3042 switch (mul->GetResultType()) { 3043 case Primitive::kPrimInt: 3044 case Primitive::kPrimLong: { 3045 locations->SetInAt(0, Location::RequiresRegister()); 3046 locations->SetInAt(1, Location::RequiresRegister()); 3047 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 3048 break; 3049 } 3050 3051 case Primitive::kPrimFloat: 3052 case Primitive::kPrimDouble: { 3053 locations->SetInAt(0, Location::RequiresFpuRegister()); 3054 locations->SetInAt(1, Location::RequiresFpuRegister()); 3055 locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap); 3056 break; 3057 } 3058 3059 default: 3060 LOG(FATAL) << "Unexpected mul type " << mul->GetResultType(); 3061 } 3062} 3063 3064void InstructionCodeGeneratorARM::VisitMul(HMul* mul) { 3065 LocationSummary* locations = mul->GetLocations(); 3066 Location out = locations->Out(); 3067 Location first = locations->InAt(0); 3068 Location second = locations->InAt(1); 3069 switch (mul->GetResultType()) { 3070 case Primitive::kPrimInt: { 3071 __ mul(out.AsRegister<Register>(), 3072 first.AsRegister<Register>(), 3073 second.AsRegister<Register>()); 3074 break; 3075 } 3076 case Primitive::kPrimLong: { 3077 Register out_hi = out.AsRegisterPairHigh<Register>(); 3078 Register out_lo = out.AsRegisterPairLow<Register>(); 3079 Register in1_hi = first.AsRegisterPairHigh<Register>(); 3080 Register in1_lo = first.AsRegisterPairLow<Register>(); 3081 Register in2_hi = second.AsRegisterPairHigh<Register>(); 3082 Register in2_lo = second.AsRegisterPairLow<Register>(); 3083 3084 // Extra checks to protect caused by the existence of R1_R2. 3085 // The algorithm is wrong if out.hi is either in1.lo or in2.lo: 3086 // (e.g. in1=r0_r1, in2=r2_r3 and out=r1_r2); 3087 DCHECK_NE(out_hi, in1_lo); 3088 DCHECK_NE(out_hi, in2_lo); 3089 3090 // input: in1 - 64 bits, in2 - 64 bits 3091 // output: out 3092 // formula: out.hi : out.lo = (in1.lo * in2.hi + in1.hi * in2.lo)* 2^32 + in1.lo * in2.lo 3093 // parts: out.hi = in1.lo * in2.hi + in1.hi * in2.lo + (in1.lo * in2.lo)[63:32] 3094 // parts: out.lo = (in1.lo * in2.lo)[31:0] 3095 3096 // IP <- in1.lo * in2.hi 3097 __ mul(IP, in1_lo, in2_hi); 3098 // out.hi <- in1.lo * in2.hi + in1.hi * in2.lo 3099 __ mla(out_hi, in1_hi, in2_lo, IP); 3100 // out.lo <- (in1.lo * in2.lo)[31:0]; 3101 __ umull(out_lo, IP, in1_lo, in2_lo); 3102 // out.hi <- in2.hi * in1.lo + in2.lo * in1.hi + (in1.lo * in2.lo)[63:32] 3103 __ add(out_hi, out_hi, ShifterOperand(IP)); 3104 break; 3105 } 3106 3107 case Primitive::kPrimFloat: { 3108 __ vmuls(out.AsFpuRegister<SRegister>(), 3109 first.AsFpuRegister<SRegister>(), 3110 second.AsFpuRegister<SRegister>()); 3111 break; 3112 } 3113 3114 case Primitive::kPrimDouble: { 3115 __ vmuld(FromLowSToD(out.AsFpuRegisterPairLow<SRegister>()), 3116 FromLowSToD(first.AsFpuRegisterPairLow<SRegister>()), 3117 FromLowSToD(second.AsFpuRegisterPairLow<SRegister>())); 3118 break; 3119 } 3120 3121 default: 3122 LOG(FATAL) << "Unexpected mul type " << mul->GetResultType(); 3123 } 3124} 3125 3126void InstructionCodeGeneratorARM::DivRemOneOrMinusOne(HBinaryOperation* instruction) { 3127 DCHECK(instruction->IsDiv() || instruction->IsRem()); 3128 DCHECK(instruction->GetResultType() == Primitive::kPrimInt); 3129 3130 LocationSummary* locations = instruction->GetLocations(); 3131 Location second = locations->InAt(1); 3132 DCHECK(second.IsConstant()); 3133 3134 Register out = locations->Out().AsRegister<Register>(); 3135 Register dividend = locations->InAt(0).AsRegister<Register>(); 3136 int32_t imm = second.GetConstant()->AsIntConstant()->GetValue(); 3137 DCHECK(imm == 1 || imm == -1); 3138 3139 if (instruction->IsRem()) { 3140 __ LoadImmediate(out, 0); 3141 } else { 3142 if (imm == 1) { 3143 __ Mov(out, dividend); 3144 } else { 3145 __ rsb(out, dividend, ShifterOperand(0)); 3146 } 3147 } 3148} 3149 3150void InstructionCodeGeneratorARM::DivRemByPowerOfTwo(HBinaryOperation* instruction) { 3151 DCHECK(instruction->IsDiv() || instruction->IsRem()); 3152 DCHECK(instruction->GetResultType() == Primitive::kPrimInt); 3153 3154 LocationSummary* locations = instruction->GetLocations(); 3155 Location second = locations->InAt(1); 3156 DCHECK(second.IsConstant()); 3157 3158 Register out = locations->Out().AsRegister<Register>(); 3159 Register dividend = locations->InAt(0).AsRegister<Register>(); 3160 Register temp = locations->GetTemp(0).AsRegister<Register>(); 3161 int32_t imm = second.GetConstant()->AsIntConstant()->GetValue(); 3162 uint32_t abs_imm = static_cast<uint32_t>(AbsOrMin(imm)); 3163 int ctz_imm = CTZ(abs_imm); 3164 3165 if (ctz_imm == 1) { 3166 __ Lsr(temp, dividend, 32 - ctz_imm); 3167 } else { 3168 __ Asr(temp, dividend, 31); 3169 __ Lsr(temp, temp, 32 - ctz_imm); 3170 } 3171 __ add(out, temp, ShifterOperand(dividend)); 3172 3173 if (instruction->IsDiv()) { 3174 __ Asr(out, out, ctz_imm); 3175 if (imm < 0) { 3176 __ rsb(out, out, ShifterOperand(0)); 3177 } 3178 } else { 3179 __ ubfx(out, out, 0, ctz_imm); 3180 __ sub(out, out, ShifterOperand(temp)); 3181 } 3182} 3183 3184void InstructionCodeGeneratorARM::GenerateDivRemWithAnyConstant(HBinaryOperation* instruction) { 3185 DCHECK(instruction->IsDiv() || instruction->IsRem()); 3186 DCHECK(instruction->GetResultType() == Primitive::kPrimInt); 3187 3188 LocationSummary* locations = instruction->GetLocations(); 3189 Location second = locations->InAt(1); 3190 DCHECK(second.IsConstant()); 3191 3192 Register out = locations->Out().AsRegister<Register>(); 3193 Register dividend = locations->InAt(0).AsRegister<Register>(); 3194 Register temp1 = locations->GetTemp(0).AsRegister<Register>(); 3195 Register temp2 = locations->GetTemp(1).AsRegister<Register>(); 3196 int64_t imm = second.GetConstant()->AsIntConstant()->GetValue(); 3197 3198 int64_t magic; 3199 int shift; 3200 CalculateMagicAndShiftForDivRem(imm, false /* is_long */, &magic, &shift); 3201 3202 __ LoadImmediate(temp1, magic); 3203 __ smull(temp2, temp1, dividend, temp1); 3204 3205 if (imm > 0 && magic < 0) { 3206 __ add(temp1, temp1, ShifterOperand(dividend)); 3207 } else if (imm < 0 && magic > 0) { 3208 __ sub(temp1, temp1, ShifterOperand(dividend)); 3209 } 3210 3211 if (shift != 0) { 3212 __ Asr(temp1, temp1, shift); 3213 } 3214 3215 if (instruction->IsDiv()) { 3216 __ sub(out, temp1, ShifterOperand(temp1, ASR, 31)); 3217 } else { 3218 __ sub(temp1, temp1, ShifterOperand(temp1, ASR, 31)); 3219 // TODO: Strength reduction for mls. 3220 __ LoadImmediate(temp2, imm); 3221 __ mls(out, temp1, temp2, dividend); 3222 } 3223} 3224 3225void InstructionCodeGeneratorARM::GenerateDivRemConstantIntegral(HBinaryOperation* instruction) { 3226 DCHECK(instruction->IsDiv() || instruction->IsRem()); 3227 DCHECK(instruction->GetResultType() == Primitive::kPrimInt); 3228 3229 LocationSummary* locations = instruction->GetLocations(); 3230 Location second = locations->InAt(1); 3231 DCHECK(second.IsConstant()); 3232 3233 int32_t imm = second.GetConstant()->AsIntConstant()->GetValue(); 3234 if (imm == 0) { 3235 // Do not generate anything. DivZeroCheck would prevent any code to be executed. 3236 } else if (imm == 1 || imm == -1) { 3237 DivRemOneOrMinusOne(instruction); 3238 } else if (IsPowerOfTwo(AbsOrMin(imm))) { 3239 DivRemByPowerOfTwo(instruction); 3240 } else { 3241 DCHECK(imm <= -2 || imm >= 2); 3242 GenerateDivRemWithAnyConstant(instruction); 3243 } 3244} 3245 3246void LocationsBuilderARM::VisitDiv(HDiv* div) { 3247 LocationSummary::CallKind call_kind = LocationSummary::kNoCall; 3248 if (div->GetResultType() == Primitive::kPrimLong) { 3249 // pLdiv runtime call. 3250 call_kind = LocationSummary::kCallOnMainOnly; 3251 } else if (div->GetResultType() == Primitive::kPrimInt && div->InputAt(1)->IsConstant()) { 3252 // sdiv will be replaced by other instruction sequence. 3253 } else if (div->GetResultType() == Primitive::kPrimInt && 3254 !codegen_->GetInstructionSetFeatures().HasDivideInstruction()) { 3255 // pIdivmod runtime call. 3256 call_kind = LocationSummary::kCallOnMainOnly; 3257 } 3258 3259 LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(div, call_kind); 3260 3261 switch (div->GetResultType()) { 3262 case Primitive::kPrimInt: { 3263 if (div->InputAt(1)->IsConstant()) { 3264 locations->SetInAt(0, Location::RequiresRegister()); 3265 locations->SetInAt(1, Location::ConstantLocation(div->InputAt(1)->AsConstant())); 3266 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 3267 int32_t value = div->InputAt(1)->AsIntConstant()->GetValue(); 3268 if (value == 1 || value == 0 || value == -1) { 3269 // No temp register required. 3270 } else { 3271 locations->AddTemp(Location::RequiresRegister()); 3272 if (!IsPowerOfTwo(AbsOrMin(value))) { 3273 locations->AddTemp(Location::RequiresRegister()); 3274 } 3275 } 3276 } else if (codegen_->GetInstructionSetFeatures().HasDivideInstruction()) { 3277 locations->SetInAt(0, Location::RequiresRegister()); 3278 locations->SetInAt(1, Location::RequiresRegister()); 3279 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 3280 } else { 3281 InvokeRuntimeCallingConvention calling_convention; 3282 locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(0))); 3283 locations->SetInAt(1, Location::RegisterLocation(calling_convention.GetRegisterAt(1))); 3284 // Note: divrem will compute both the quotient and the remainder as the pair R0 and R1, but 3285 // we only need the former. 3286 locations->SetOut(Location::RegisterLocation(R0)); 3287 } 3288 break; 3289 } 3290 case Primitive::kPrimLong: { 3291 InvokeRuntimeCallingConvention calling_convention; 3292 locations->SetInAt(0, Location::RegisterPairLocation( 3293 calling_convention.GetRegisterAt(0), calling_convention.GetRegisterAt(1))); 3294 locations->SetInAt(1, Location::RegisterPairLocation( 3295 calling_convention.GetRegisterAt(2), calling_convention.GetRegisterAt(3))); 3296 locations->SetOut(Location::RegisterPairLocation(R0, R1)); 3297 break; 3298 } 3299 case Primitive::kPrimFloat: 3300 case Primitive::kPrimDouble: { 3301 locations->SetInAt(0, Location::RequiresFpuRegister()); 3302 locations->SetInAt(1, Location::RequiresFpuRegister()); 3303 locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap); 3304 break; 3305 } 3306 3307 default: 3308 LOG(FATAL) << "Unexpected div type " << div->GetResultType(); 3309 } 3310} 3311 3312void InstructionCodeGeneratorARM::VisitDiv(HDiv* div) { 3313 LocationSummary* locations = div->GetLocations(); 3314 Location out = locations->Out(); 3315 Location first = locations->InAt(0); 3316 Location second = locations->InAt(1); 3317 3318 switch (div->GetResultType()) { 3319 case Primitive::kPrimInt: { 3320 if (second.IsConstant()) { 3321 GenerateDivRemConstantIntegral(div); 3322 } else if (codegen_->GetInstructionSetFeatures().HasDivideInstruction()) { 3323 __ sdiv(out.AsRegister<Register>(), 3324 first.AsRegister<Register>(), 3325 second.AsRegister<Register>()); 3326 } else { 3327 InvokeRuntimeCallingConvention calling_convention; 3328 DCHECK_EQ(calling_convention.GetRegisterAt(0), first.AsRegister<Register>()); 3329 DCHECK_EQ(calling_convention.GetRegisterAt(1), second.AsRegister<Register>()); 3330 DCHECK_EQ(R0, out.AsRegister<Register>()); 3331 3332 codegen_->InvokeRuntime(kQuickIdivmod, div, div->GetDexPc()); 3333 CheckEntrypointTypes<kQuickIdivmod, int32_t, int32_t, int32_t>(); 3334 } 3335 break; 3336 } 3337 3338 case Primitive::kPrimLong: { 3339 InvokeRuntimeCallingConvention calling_convention; 3340 DCHECK_EQ(calling_convention.GetRegisterAt(0), first.AsRegisterPairLow<Register>()); 3341 DCHECK_EQ(calling_convention.GetRegisterAt(1), first.AsRegisterPairHigh<Register>()); 3342 DCHECK_EQ(calling_convention.GetRegisterAt(2), second.AsRegisterPairLow<Register>()); 3343 DCHECK_EQ(calling_convention.GetRegisterAt(3), second.AsRegisterPairHigh<Register>()); 3344 DCHECK_EQ(R0, out.AsRegisterPairLow<Register>()); 3345 DCHECK_EQ(R1, out.AsRegisterPairHigh<Register>()); 3346 3347 codegen_->InvokeRuntime(kQuickLdiv, div, div->GetDexPc()); 3348 CheckEntrypointTypes<kQuickLdiv, int64_t, int64_t, int64_t>(); 3349 break; 3350 } 3351 3352 case Primitive::kPrimFloat: { 3353 __ vdivs(out.AsFpuRegister<SRegister>(), 3354 first.AsFpuRegister<SRegister>(), 3355 second.AsFpuRegister<SRegister>()); 3356 break; 3357 } 3358 3359 case Primitive::kPrimDouble: { 3360 __ vdivd(FromLowSToD(out.AsFpuRegisterPairLow<SRegister>()), 3361 FromLowSToD(first.AsFpuRegisterPairLow<SRegister>()), 3362 FromLowSToD(second.AsFpuRegisterPairLow<SRegister>())); 3363 break; 3364 } 3365 3366 default: 3367 LOG(FATAL) << "Unexpected div type " << div->GetResultType(); 3368 } 3369} 3370 3371void LocationsBuilderARM::VisitRem(HRem* rem) { 3372 Primitive::Type type = rem->GetResultType(); 3373 3374 // Most remainders are implemented in the runtime. 3375 LocationSummary::CallKind call_kind = LocationSummary::kCallOnMainOnly; 3376 if (rem->GetResultType() == Primitive::kPrimInt && rem->InputAt(1)->IsConstant()) { 3377 // sdiv will be replaced by other instruction sequence. 3378 call_kind = LocationSummary::kNoCall; 3379 } else if ((rem->GetResultType() == Primitive::kPrimInt) 3380 && codegen_->GetInstructionSetFeatures().HasDivideInstruction()) { 3381 // Have hardware divide instruction for int, do it with three instructions. 3382 call_kind = LocationSummary::kNoCall; 3383 } 3384 3385 LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(rem, call_kind); 3386 3387 switch (type) { 3388 case Primitive::kPrimInt: { 3389 if (rem->InputAt(1)->IsConstant()) { 3390 locations->SetInAt(0, Location::RequiresRegister()); 3391 locations->SetInAt(1, Location::ConstantLocation(rem->InputAt(1)->AsConstant())); 3392 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 3393 int32_t value = rem->InputAt(1)->AsIntConstant()->GetValue(); 3394 if (value == 1 || value == 0 || value == -1) { 3395 // No temp register required. 3396 } else { 3397 locations->AddTemp(Location::RequiresRegister()); 3398 if (!IsPowerOfTwo(AbsOrMin(value))) { 3399 locations->AddTemp(Location::RequiresRegister()); 3400 } 3401 } 3402 } else if (codegen_->GetInstructionSetFeatures().HasDivideInstruction()) { 3403 locations->SetInAt(0, Location::RequiresRegister()); 3404 locations->SetInAt(1, Location::RequiresRegister()); 3405 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 3406 locations->AddTemp(Location::RequiresRegister()); 3407 } else { 3408 InvokeRuntimeCallingConvention calling_convention; 3409 locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(0))); 3410 locations->SetInAt(1, Location::RegisterLocation(calling_convention.GetRegisterAt(1))); 3411 // Note: divrem will compute both the quotient and the remainder as the pair R0 and R1, but 3412 // we only need the latter. 3413 locations->SetOut(Location::RegisterLocation(R1)); 3414 } 3415 break; 3416 } 3417 case Primitive::kPrimLong: { 3418 InvokeRuntimeCallingConvention calling_convention; 3419 locations->SetInAt(0, Location::RegisterPairLocation( 3420 calling_convention.GetRegisterAt(0), calling_convention.GetRegisterAt(1))); 3421 locations->SetInAt(1, Location::RegisterPairLocation( 3422 calling_convention.GetRegisterAt(2), calling_convention.GetRegisterAt(3))); 3423 // The runtime helper puts the output in R2,R3. 3424 locations->SetOut(Location::RegisterPairLocation(R2, R3)); 3425 break; 3426 } 3427 case Primitive::kPrimFloat: { 3428 InvokeRuntimeCallingConvention calling_convention; 3429 locations->SetInAt(0, Location::FpuRegisterLocation(calling_convention.GetFpuRegisterAt(0))); 3430 locations->SetInAt(1, Location::FpuRegisterLocation(calling_convention.GetFpuRegisterAt(1))); 3431 locations->SetOut(Location::FpuRegisterLocation(S0)); 3432 break; 3433 } 3434 3435 case Primitive::kPrimDouble: { 3436 InvokeRuntimeCallingConvention calling_convention; 3437 locations->SetInAt(0, Location::FpuRegisterPairLocation( 3438 calling_convention.GetFpuRegisterAt(0), calling_convention.GetFpuRegisterAt(1))); 3439 locations->SetInAt(1, Location::FpuRegisterPairLocation( 3440 calling_convention.GetFpuRegisterAt(2), calling_convention.GetFpuRegisterAt(3))); 3441 locations->SetOut(Location::Location::FpuRegisterPairLocation(S0, S1)); 3442 break; 3443 } 3444 3445 default: 3446 LOG(FATAL) << "Unexpected rem type " << type; 3447 } 3448} 3449 3450void InstructionCodeGeneratorARM::VisitRem(HRem* rem) { 3451 LocationSummary* locations = rem->GetLocations(); 3452 Location out = locations->Out(); 3453 Location first = locations->InAt(0); 3454 Location second = locations->InAt(1); 3455 3456 Primitive::Type type = rem->GetResultType(); 3457 switch (type) { 3458 case Primitive::kPrimInt: { 3459 if (second.IsConstant()) { 3460 GenerateDivRemConstantIntegral(rem); 3461 } else if (codegen_->GetInstructionSetFeatures().HasDivideInstruction()) { 3462 Register reg1 = first.AsRegister<Register>(); 3463 Register reg2 = second.AsRegister<Register>(); 3464 Register temp = locations->GetTemp(0).AsRegister<Register>(); 3465 3466 // temp = reg1 / reg2 (integer division) 3467 // dest = reg1 - temp * reg2 3468 __ sdiv(temp, reg1, reg2); 3469 __ mls(out.AsRegister<Register>(), temp, reg2, reg1); 3470 } else { 3471 InvokeRuntimeCallingConvention calling_convention; 3472 DCHECK_EQ(calling_convention.GetRegisterAt(0), first.AsRegister<Register>()); 3473 DCHECK_EQ(calling_convention.GetRegisterAt(1), second.AsRegister<Register>()); 3474 DCHECK_EQ(R1, out.AsRegister<Register>()); 3475 3476 codegen_->InvokeRuntime(kQuickIdivmod, rem, rem->GetDexPc()); 3477 CheckEntrypointTypes<kQuickIdivmod, int32_t, int32_t, int32_t>(); 3478 } 3479 break; 3480 } 3481 3482 case Primitive::kPrimLong: { 3483 codegen_->InvokeRuntime(kQuickLmod, rem, rem->GetDexPc()); 3484 CheckEntrypointTypes<kQuickLmod, int64_t, int64_t, int64_t>(); 3485 break; 3486 } 3487 3488 case Primitive::kPrimFloat: { 3489 codegen_->InvokeRuntime(kQuickFmodf, rem, rem->GetDexPc()); 3490 CheckEntrypointTypes<kQuickFmodf, float, float, float>(); 3491 break; 3492 } 3493 3494 case Primitive::kPrimDouble: { 3495 codegen_->InvokeRuntime(kQuickFmod, rem, rem->GetDexPc()); 3496 CheckEntrypointTypes<kQuickFmod, double, double, double>(); 3497 break; 3498 } 3499 3500 default: 3501 LOG(FATAL) << "Unexpected rem type " << type; 3502 } 3503} 3504 3505void LocationsBuilderARM::VisitDivZeroCheck(HDivZeroCheck* instruction) { 3506 LocationSummary* locations = codegen_->CreateThrowingSlowPathLocations(instruction); 3507 locations->SetInAt(0, Location::RegisterOrConstant(instruction->InputAt(0))); 3508} 3509 3510void InstructionCodeGeneratorARM::VisitDivZeroCheck(HDivZeroCheck* instruction) { 3511 SlowPathCodeARM* slow_path = new (GetGraph()->GetArena()) DivZeroCheckSlowPathARM(instruction); 3512 codegen_->AddSlowPath(slow_path); 3513 3514 LocationSummary* locations = instruction->GetLocations(); 3515 Location value = locations->InAt(0); 3516 3517 switch (instruction->GetType()) { 3518 case Primitive::kPrimBoolean: 3519 case Primitive::kPrimByte: 3520 case Primitive::kPrimChar: 3521 case Primitive::kPrimShort: 3522 case Primitive::kPrimInt: { 3523 if (value.IsRegister()) { 3524 __ CompareAndBranchIfZero(value.AsRegister<Register>(), slow_path->GetEntryLabel()); 3525 } else { 3526 DCHECK(value.IsConstant()) << value; 3527 if (value.GetConstant()->AsIntConstant()->GetValue() == 0) { 3528 __ b(slow_path->GetEntryLabel()); 3529 } 3530 } 3531 break; 3532 } 3533 case Primitive::kPrimLong: { 3534 if (value.IsRegisterPair()) { 3535 __ orrs(IP, 3536 value.AsRegisterPairLow<Register>(), 3537 ShifterOperand(value.AsRegisterPairHigh<Register>())); 3538 __ b(slow_path->GetEntryLabel(), EQ); 3539 } else { 3540 DCHECK(value.IsConstant()) << value; 3541 if (value.GetConstant()->AsLongConstant()->GetValue() == 0) { 3542 __ b(slow_path->GetEntryLabel()); 3543 } 3544 } 3545 break; 3546 default: 3547 LOG(FATAL) << "Unexpected type for HDivZeroCheck " << instruction->GetType(); 3548 } 3549 } 3550} 3551 3552void InstructionCodeGeneratorARM::HandleIntegerRotate(LocationSummary* locations) { 3553 Register in = locations->InAt(0).AsRegister<Register>(); 3554 Location rhs = locations->InAt(1); 3555 Register out = locations->Out().AsRegister<Register>(); 3556 3557 if (rhs.IsConstant()) { 3558 // Arm32 and Thumb2 assemblers require a rotation on the interval [1,31], 3559 // so map all rotations to a +ve. equivalent in that range. 3560 // (e.g. left *or* right by -2 bits == 30 bits in the same direction.) 3561 uint32_t rot = CodeGenerator::GetInt32ValueOf(rhs.GetConstant()) & 0x1F; 3562 if (rot) { 3563 // Rotate, mapping left rotations to right equivalents if necessary. 3564 // (e.g. left by 2 bits == right by 30.) 3565 __ Ror(out, in, rot); 3566 } else if (out != in) { 3567 __ Mov(out, in); 3568 } 3569 } else { 3570 __ Ror(out, in, rhs.AsRegister<Register>()); 3571 } 3572} 3573 3574// Gain some speed by mapping all Long rotates onto equivalent pairs of Integer 3575// rotates by swapping input regs (effectively rotating by the first 32-bits of 3576// a larger rotation) or flipping direction (thus treating larger right/left 3577// rotations as sub-word sized rotations in the other direction) as appropriate. 3578void InstructionCodeGeneratorARM::HandleLongRotate(LocationSummary* locations) { 3579 Register in_reg_lo = locations->InAt(0).AsRegisterPairLow<Register>(); 3580 Register in_reg_hi = locations->InAt(0).AsRegisterPairHigh<Register>(); 3581 Location rhs = locations->InAt(1); 3582 Register out_reg_lo = locations->Out().AsRegisterPairLow<Register>(); 3583 Register out_reg_hi = locations->Out().AsRegisterPairHigh<Register>(); 3584 3585 if (rhs.IsConstant()) { 3586 uint64_t rot = CodeGenerator::GetInt64ValueOf(rhs.GetConstant()); 3587 // Map all rotations to +ve. equivalents on the interval [0,63]. 3588 rot &= kMaxLongShiftDistance; 3589 // For rotates over a word in size, 'pre-rotate' by 32-bits to keep rotate 3590 // logic below to a simple pair of binary orr. 3591 // (e.g. 34 bits == in_reg swap + 2 bits right.) 3592 if (rot >= kArmBitsPerWord) { 3593 rot -= kArmBitsPerWord; 3594 std::swap(in_reg_hi, in_reg_lo); 3595 } 3596 // Rotate, or mov to out for zero or word size rotations. 3597 if (rot != 0u) { 3598 __ Lsr(out_reg_hi, in_reg_hi, rot); 3599 __ orr(out_reg_hi, out_reg_hi, ShifterOperand(in_reg_lo, arm::LSL, kArmBitsPerWord - rot)); 3600 __ Lsr(out_reg_lo, in_reg_lo, rot); 3601 __ orr(out_reg_lo, out_reg_lo, ShifterOperand(in_reg_hi, arm::LSL, kArmBitsPerWord - rot)); 3602 } else { 3603 __ Mov(out_reg_lo, in_reg_lo); 3604 __ Mov(out_reg_hi, in_reg_hi); 3605 } 3606 } else { 3607 Register shift_right = locations->GetTemp(0).AsRegister<Register>(); 3608 Register shift_left = locations->GetTemp(1).AsRegister<Register>(); 3609 Label end; 3610 Label shift_by_32_plus_shift_right; 3611 3612 __ and_(shift_right, rhs.AsRegister<Register>(), ShifterOperand(0x1F)); 3613 __ Lsrs(shift_left, rhs.AsRegister<Register>(), 6); 3614 __ rsb(shift_left, shift_right, ShifterOperand(kArmBitsPerWord), AL, kCcKeep); 3615 __ b(&shift_by_32_plus_shift_right, CC); 3616 3617 // out_reg_hi = (reg_hi << shift_left) | (reg_lo >> shift_right). 3618 // out_reg_lo = (reg_lo << shift_left) | (reg_hi >> shift_right). 3619 __ Lsl(out_reg_hi, in_reg_hi, shift_left); 3620 __ Lsr(out_reg_lo, in_reg_lo, shift_right); 3621 __ add(out_reg_hi, out_reg_hi, ShifterOperand(out_reg_lo)); 3622 __ Lsl(out_reg_lo, in_reg_lo, shift_left); 3623 __ Lsr(shift_left, in_reg_hi, shift_right); 3624 __ add(out_reg_lo, out_reg_lo, ShifterOperand(shift_left)); 3625 __ b(&end); 3626 3627 __ Bind(&shift_by_32_plus_shift_right); // Shift by 32+shift_right. 3628 // out_reg_hi = (reg_hi >> shift_right) | (reg_lo << shift_left). 3629 // out_reg_lo = (reg_lo >> shift_right) | (reg_hi << shift_left). 3630 __ Lsr(out_reg_hi, in_reg_hi, shift_right); 3631 __ Lsl(out_reg_lo, in_reg_lo, shift_left); 3632 __ add(out_reg_hi, out_reg_hi, ShifterOperand(out_reg_lo)); 3633 __ Lsr(out_reg_lo, in_reg_lo, shift_right); 3634 __ Lsl(shift_right, in_reg_hi, shift_left); 3635 __ add(out_reg_lo, out_reg_lo, ShifterOperand(shift_right)); 3636 3637 __ Bind(&end); 3638 } 3639} 3640 3641void LocationsBuilderARM::VisitRor(HRor* ror) { 3642 LocationSummary* locations = 3643 new (GetGraph()->GetArena()) LocationSummary(ror, LocationSummary::kNoCall); 3644 switch (ror->GetResultType()) { 3645 case Primitive::kPrimInt: { 3646 locations->SetInAt(0, Location::RequiresRegister()); 3647 locations->SetInAt(1, Location::RegisterOrConstant(ror->InputAt(1))); 3648 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 3649 break; 3650 } 3651 case Primitive::kPrimLong: { 3652 locations->SetInAt(0, Location::RequiresRegister()); 3653 if (ror->InputAt(1)->IsConstant()) { 3654 locations->SetInAt(1, Location::ConstantLocation(ror->InputAt(1)->AsConstant())); 3655 } else { 3656 locations->SetInAt(1, Location::RequiresRegister()); 3657 locations->AddTemp(Location::RequiresRegister()); 3658 locations->AddTemp(Location::RequiresRegister()); 3659 } 3660 locations->SetOut(Location::RequiresRegister(), Location::kOutputOverlap); 3661 break; 3662 } 3663 default: 3664 LOG(FATAL) << "Unexpected operation type " << ror->GetResultType(); 3665 } 3666} 3667 3668void InstructionCodeGeneratorARM::VisitRor(HRor* ror) { 3669 LocationSummary* locations = ror->GetLocations(); 3670 Primitive::Type type = ror->GetResultType(); 3671 switch (type) { 3672 case Primitive::kPrimInt: { 3673 HandleIntegerRotate(locations); 3674 break; 3675 } 3676 case Primitive::kPrimLong: { 3677 HandleLongRotate(locations); 3678 break; 3679 } 3680 default: 3681 LOG(FATAL) << "Unexpected operation type " << type; 3682 UNREACHABLE(); 3683 } 3684} 3685 3686void LocationsBuilderARM::HandleShift(HBinaryOperation* op) { 3687 DCHECK(op->IsShl() || op->IsShr() || op->IsUShr()); 3688 3689 LocationSummary* locations = 3690 new (GetGraph()->GetArena()) LocationSummary(op, LocationSummary::kNoCall); 3691 3692 switch (op->GetResultType()) { 3693 case Primitive::kPrimInt: { 3694 locations->SetInAt(0, Location::RequiresRegister()); 3695 if (op->InputAt(1)->IsConstant()) { 3696 locations->SetInAt(1, Location::ConstantLocation(op->InputAt(1)->AsConstant())); 3697 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 3698 } else { 3699 locations->SetInAt(1, Location::RequiresRegister()); 3700 // Make the output overlap, as it will be used to hold the masked 3701 // second input. 3702 locations->SetOut(Location::RequiresRegister(), Location::kOutputOverlap); 3703 } 3704 break; 3705 } 3706 case Primitive::kPrimLong: { 3707 locations->SetInAt(0, Location::RequiresRegister()); 3708 if (op->InputAt(1)->IsConstant()) { 3709 locations->SetInAt(1, Location::ConstantLocation(op->InputAt(1)->AsConstant())); 3710 // For simplicity, use kOutputOverlap even though we only require that low registers 3711 // don't clash with high registers which the register allocator currently guarantees. 3712 locations->SetOut(Location::RequiresRegister(), Location::kOutputOverlap); 3713 } else { 3714 locations->SetInAt(1, Location::RequiresRegister()); 3715 locations->AddTemp(Location::RequiresRegister()); 3716 locations->SetOut(Location::RequiresRegister(), Location::kOutputOverlap); 3717 } 3718 break; 3719 } 3720 default: 3721 LOG(FATAL) << "Unexpected operation type " << op->GetResultType(); 3722 } 3723} 3724 3725void InstructionCodeGeneratorARM::HandleShift(HBinaryOperation* op) { 3726 DCHECK(op->IsShl() || op->IsShr() || op->IsUShr()); 3727 3728 LocationSummary* locations = op->GetLocations(); 3729 Location out = locations->Out(); 3730 Location first = locations->InAt(0); 3731 Location second = locations->InAt(1); 3732 3733 Primitive::Type type = op->GetResultType(); 3734 switch (type) { 3735 case Primitive::kPrimInt: { 3736 Register out_reg = out.AsRegister<Register>(); 3737 Register first_reg = first.AsRegister<Register>(); 3738 if (second.IsRegister()) { 3739 Register second_reg = second.AsRegister<Register>(); 3740 // ARM doesn't mask the shift count so we need to do it ourselves. 3741 __ and_(out_reg, second_reg, ShifterOperand(kMaxIntShiftDistance)); 3742 if (op->IsShl()) { 3743 __ Lsl(out_reg, first_reg, out_reg); 3744 } else if (op->IsShr()) { 3745 __ Asr(out_reg, first_reg, out_reg); 3746 } else { 3747 __ Lsr(out_reg, first_reg, out_reg); 3748 } 3749 } else { 3750 int32_t cst = second.GetConstant()->AsIntConstant()->GetValue(); 3751 uint32_t shift_value = cst & kMaxIntShiftDistance; 3752 if (shift_value == 0) { // ARM does not support shifting with 0 immediate. 3753 __ Mov(out_reg, first_reg); 3754 } else if (op->IsShl()) { 3755 __ Lsl(out_reg, first_reg, shift_value); 3756 } else if (op->IsShr()) { 3757 __ Asr(out_reg, first_reg, shift_value); 3758 } else { 3759 __ Lsr(out_reg, first_reg, shift_value); 3760 } 3761 } 3762 break; 3763 } 3764 case Primitive::kPrimLong: { 3765 Register o_h = out.AsRegisterPairHigh<Register>(); 3766 Register o_l = out.AsRegisterPairLow<Register>(); 3767 3768 Register high = first.AsRegisterPairHigh<Register>(); 3769 Register low = first.AsRegisterPairLow<Register>(); 3770 3771 if (second.IsRegister()) { 3772 Register temp = locations->GetTemp(0).AsRegister<Register>(); 3773 3774 Register second_reg = second.AsRegister<Register>(); 3775 3776 if (op->IsShl()) { 3777 __ and_(o_l, second_reg, ShifterOperand(kMaxLongShiftDistance)); 3778 // Shift the high part 3779 __ Lsl(o_h, high, o_l); 3780 // Shift the low part and `or` what overflew on the high part 3781 __ rsb(temp, o_l, ShifterOperand(kArmBitsPerWord)); 3782 __ Lsr(temp, low, temp); 3783 __ orr(o_h, o_h, ShifterOperand(temp)); 3784 // If the shift is > 32 bits, override the high part 3785 __ subs(temp, o_l, ShifterOperand(kArmBitsPerWord)); 3786 __ it(PL); 3787 __ Lsl(o_h, low, temp, PL); 3788 // Shift the low part 3789 __ Lsl(o_l, low, o_l); 3790 } else if (op->IsShr()) { 3791 __ and_(o_h, second_reg, ShifterOperand(kMaxLongShiftDistance)); 3792 // Shift the low part 3793 __ Lsr(o_l, low, o_h); 3794 // Shift the high part and `or` what underflew on the low part 3795 __ rsb(temp, o_h, ShifterOperand(kArmBitsPerWord)); 3796 __ Lsl(temp, high, temp); 3797 __ orr(o_l, o_l, ShifterOperand(temp)); 3798 // If the shift is > 32 bits, override the low part 3799 __ subs(temp, o_h, ShifterOperand(kArmBitsPerWord)); 3800 __ it(PL); 3801 __ Asr(o_l, high, temp, PL); 3802 // Shift the high part 3803 __ Asr(o_h, high, o_h); 3804 } else { 3805 __ and_(o_h, second_reg, ShifterOperand(kMaxLongShiftDistance)); 3806 // same as Shr except we use `Lsr`s and not `Asr`s 3807 __ Lsr(o_l, low, o_h); 3808 __ rsb(temp, o_h, ShifterOperand(kArmBitsPerWord)); 3809 __ Lsl(temp, high, temp); 3810 __ orr(o_l, o_l, ShifterOperand(temp)); 3811 __ subs(temp, o_h, ShifterOperand(kArmBitsPerWord)); 3812 __ it(PL); 3813 __ Lsr(o_l, high, temp, PL); 3814 __ Lsr(o_h, high, o_h); 3815 } 3816 } else { 3817 // Register allocator doesn't create partial overlap. 3818 DCHECK_NE(o_l, high); 3819 DCHECK_NE(o_h, low); 3820 int32_t cst = second.GetConstant()->AsIntConstant()->GetValue(); 3821 uint32_t shift_value = cst & kMaxLongShiftDistance; 3822 if (shift_value > 32) { 3823 if (op->IsShl()) { 3824 __ Lsl(o_h, low, shift_value - 32); 3825 __ LoadImmediate(o_l, 0); 3826 } else if (op->IsShr()) { 3827 __ Asr(o_l, high, shift_value - 32); 3828 __ Asr(o_h, high, 31); 3829 } else { 3830 __ Lsr(o_l, high, shift_value - 32); 3831 __ LoadImmediate(o_h, 0); 3832 } 3833 } else if (shift_value == 32) { 3834 if (op->IsShl()) { 3835 __ mov(o_h, ShifterOperand(low)); 3836 __ LoadImmediate(o_l, 0); 3837 } else if (op->IsShr()) { 3838 __ mov(o_l, ShifterOperand(high)); 3839 __ Asr(o_h, high, 31); 3840 } else { 3841 __ mov(o_l, ShifterOperand(high)); 3842 __ LoadImmediate(o_h, 0); 3843 } 3844 } else if (shift_value == 1) { 3845 if (op->IsShl()) { 3846 __ Lsls(o_l, low, 1); 3847 __ adc(o_h, high, ShifterOperand(high)); 3848 } else if (op->IsShr()) { 3849 __ Asrs(o_h, high, 1); 3850 __ Rrx(o_l, low); 3851 } else { 3852 __ Lsrs(o_h, high, 1); 3853 __ Rrx(o_l, low); 3854 } 3855 } else { 3856 DCHECK(2 <= shift_value && shift_value < 32) << shift_value; 3857 if (op->IsShl()) { 3858 __ Lsl(o_h, high, shift_value); 3859 __ orr(o_h, o_h, ShifterOperand(low, LSR, 32 - shift_value)); 3860 __ Lsl(o_l, low, shift_value); 3861 } else if (op->IsShr()) { 3862 __ Lsr(o_l, low, shift_value); 3863 __ orr(o_l, o_l, ShifterOperand(high, LSL, 32 - shift_value)); 3864 __ Asr(o_h, high, shift_value); 3865 } else { 3866 __ Lsr(o_l, low, shift_value); 3867 __ orr(o_l, o_l, ShifterOperand(high, LSL, 32 - shift_value)); 3868 __ Lsr(o_h, high, shift_value); 3869 } 3870 } 3871 } 3872 break; 3873 } 3874 default: 3875 LOG(FATAL) << "Unexpected operation type " << type; 3876 UNREACHABLE(); 3877 } 3878} 3879 3880void LocationsBuilderARM::VisitShl(HShl* shl) { 3881 HandleShift(shl); 3882} 3883 3884void InstructionCodeGeneratorARM::VisitShl(HShl* shl) { 3885 HandleShift(shl); 3886} 3887 3888void LocationsBuilderARM::VisitShr(HShr* shr) { 3889 HandleShift(shr); 3890} 3891 3892void InstructionCodeGeneratorARM::VisitShr(HShr* shr) { 3893 HandleShift(shr); 3894} 3895 3896void LocationsBuilderARM::VisitUShr(HUShr* ushr) { 3897 HandleShift(ushr); 3898} 3899 3900void InstructionCodeGeneratorARM::VisitUShr(HUShr* ushr) { 3901 HandleShift(ushr); 3902} 3903 3904void LocationsBuilderARM::VisitNewInstance(HNewInstance* instruction) { 3905 LocationSummary* locations = 3906 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCallOnMainOnly); 3907 if (instruction->IsStringAlloc()) { 3908 locations->AddTemp(Location::RegisterLocation(kMethodRegisterArgument)); 3909 } else { 3910 InvokeRuntimeCallingConvention calling_convention; 3911 locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(0))); 3912 locations->SetInAt(1, Location::RegisterLocation(calling_convention.GetRegisterAt(1))); 3913 } 3914 locations->SetOut(Location::RegisterLocation(R0)); 3915} 3916 3917void InstructionCodeGeneratorARM::VisitNewInstance(HNewInstance* instruction) { 3918 // Note: if heap poisoning is enabled, the entry point takes cares 3919 // of poisoning the reference. 3920 if (instruction->IsStringAlloc()) { 3921 // String is allocated through StringFactory. Call NewEmptyString entry point. 3922 Register temp = instruction->GetLocations()->GetTemp(0).AsRegister<Register>(); 3923 MemberOffset code_offset = ArtMethod::EntryPointFromQuickCompiledCodeOffset(kArmPointerSize); 3924 __ LoadFromOffset(kLoadWord, temp, TR, QUICK_ENTRY_POINT(pNewEmptyString)); 3925 __ LoadFromOffset(kLoadWord, LR, temp, code_offset.Int32Value()); 3926 __ blx(LR); 3927 codegen_->RecordPcInfo(instruction, instruction->GetDexPc()); 3928 } else { 3929 codegen_->InvokeRuntime(instruction->GetEntrypoint(), instruction, instruction->GetDexPc()); 3930 CheckEntrypointTypes<kQuickAllocObjectWithAccessCheck, void*, uint32_t, ArtMethod*>(); 3931 } 3932} 3933 3934void LocationsBuilderARM::VisitNewArray(HNewArray* instruction) { 3935 LocationSummary* locations = 3936 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCallOnMainOnly); 3937 InvokeRuntimeCallingConvention calling_convention; 3938 locations->AddTemp(Location::RegisterLocation(calling_convention.GetRegisterAt(0))); 3939 locations->SetOut(Location::RegisterLocation(R0)); 3940 locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(1))); 3941 locations->SetInAt(1, Location::RegisterLocation(calling_convention.GetRegisterAt(2))); 3942} 3943 3944void InstructionCodeGeneratorARM::VisitNewArray(HNewArray* instruction) { 3945 InvokeRuntimeCallingConvention calling_convention; 3946 __ LoadImmediate(calling_convention.GetRegisterAt(0), instruction->GetTypeIndex()); 3947 // Note: if heap poisoning is enabled, the entry point takes cares 3948 // of poisoning the reference. 3949 codegen_->InvokeRuntime(instruction->GetEntrypoint(), instruction, instruction->GetDexPc()); 3950 CheckEntrypointTypes<kQuickAllocArrayWithAccessCheck, void*, uint32_t, int32_t, ArtMethod*>(); 3951} 3952 3953void LocationsBuilderARM::VisitParameterValue(HParameterValue* instruction) { 3954 LocationSummary* locations = 3955 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall); 3956 Location location = parameter_visitor_.GetNextLocation(instruction->GetType()); 3957 if (location.IsStackSlot()) { 3958 location = Location::StackSlot(location.GetStackIndex() + codegen_->GetFrameSize()); 3959 } else if (location.IsDoubleStackSlot()) { 3960 location = Location::DoubleStackSlot(location.GetStackIndex() + codegen_->GetFrameSize()); 3961 } 3962 locations->SetOut(location); 3963} 3964 3965void InstructionCodeGeneratorARM::VisitParameterValue( 3966 HParameterValue* instruction ATTRIBUTE_UNUSED) { 3967 // Nothing to do, the parameter is already at its location. 3968} 3969 3970void LocationsBuilderARM::VisitCurrentMethod(HCurrentMethod* instruction) { 3971 LocationSummary* locations = 3972 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall); 3973 locations->SetOut(Location::RegisterLocation(kMethodRegisterArgument)); 3974} 3975 3976void InstructionCodeGeneratorARM::VisitCurrentMethod(HCurrentMethod* instruction ATTRIBUTE_UNUSED) { 3977 // Nothing to do, the method is already at its location. 3978} 3979 3980void LocationsBuilderARM::VisitNot(HNot* not_) { 3981 LocationSummary* locations = 3982 new (GetGraph()->GetArena()) LocationSummary(not_, LocationSummary::kNoCall); 3983 locations->SetInAt(0, Location::RequiresRegister()); 3984 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 3985} 3986 3987void InstructionCodeGeneratorARM::VisitNot(HNot* not_) { 3988 LocationSummary* locations = not_->GetLocations(); 3989 Location out = locations->Out(); 3990 Location in = locations->InAt(0); 3991 switch (not_->GetResultType()) { 3992 case Primitive::kPrimInt: 3993 __ mvn(out.AsRegister<Register>(), ShifterOperand(in.AsRegister<Register>())); 3994 break; 3995 3996 case Primitive::kPrimLong: 3997 __ mvn(out.AsRegisterPairLow<Register>(), 3998 ShifterOperand(in.AsRegisterPairLow<Register>())); 3999 __ mvn(out.AsRegisterPairHigh<Register>(), 4000 ShifterOperand(in.AsRegisterPairHigh<Register>())); 4001 break; 4002 4003 default: 4004 LOG(FATAL) << "Unimplemented type for not operation " << not_->GetResultType(); 4005 } 4006} 4007 4008void LocationsBuilderARM::VisitBooleanNot(HBooleanNot* bool_not) { 4009 LocationSummary* locations = 4010 new (GetGraph()->GetArena()) LocationSummary(bool_not, LocationSummary::kNoCall); 4011 locations->SetInAt(0, Location::RequiresRegister()); 4012 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 4013} 4014 4015void InstructionCodeGeneratorARM::VisitBooleanNot(HBooleanNot* bool_not) { 4016 LocationSummary* locations = bool_not->GetLocations(); 4017 Location out = locations->Out(); 4018 Location in = locations->InAt(0); 4019 __ eor(out.AsRegister<Register>(), in.AsRegister<Register>(), ShifterOperand(1)); 4020} 4021 4022void LocationsBuilderARM::VisitCompare(HCompare* compare) { 4023 LocationSummary* locations = 4024 new (GetGraph()->GetArena()) LocationSummary(compare, LocationSummary::kNoCall); 4025 switch (compare->InputAt(0)->GetType()) { 4026 case Primitive::kPrimBoolean: 4027 case Primitive::kPrimByte: 4028 case Primitive::kPrimShort: 4029 case Primitive::kPrimChar: 4030 case Primitive::kPrimInt: 4031 case Primitive::kPrimLong: { 4032 locations->SetInAt(0, Location::RequiresRegister()); 4033 locations->SetInAt(1, Location::RequiresRegister()); 4034 // Output overlaps because it is written before doing the low comparison. 4035 locations->SetOut(Location::RequiresRegister(), Location::kOutputOverlap); 4036 break; 4037 } 4038 case Primitive::kPrimFloat: 4039 case Primitive::kPrimDouble: { 4040 locations->SetInAt(0, Location::RequiresFpuRegister()); 4041 locations->SetInAt(1, ArithmeticZeroOrFpuRegister(compare->InputAt(1))); 4042 locations->SetOut(Location::RequiresRegister()); 4043 break; 4044 } 4045 default: 4046 LOG(FATAL) << "Unexpected type for compare operation " << compare->InputAt(0)->GetType(); 4047 } 4048} 4049 4050void InstructionCodeGeneratorARM::VisitCompare(HCompare* compare) { 4051 LocationSummary* locations = compare->GetLocations(); 4052 Register out = locations->Out().AsRegister<Register>(); 4053 Location left = locations->InAt(0); 4054 Location right = locations->InAt(1); 4055 4056 Label less, greater, done; 4057 Primitive::Type type = compare->InputAt(0)->GetType(); 4058 Condition less_cond; 4059 switch (type) { 4060 case Primitive::kPrimBoolean: 4061 case Primitive::kPrimByte: 4062 case Primitive::kPrimShort: 4063 case Primitive::kPrimChar: 4064 case Primitive::kPrimInt: { 4065 __ LoadImmediate(out, 0); 4066 __ cmp(left.AsRegister<Register>(), 4067 ShifterOperand(right.AsRegister<Register>())); // Signed compare. 4068 less_cond = LT; 4069 break; 4070 } 4071 case Primitive::kPrimLong: { 4072 __ cmp(left.AsRegisterPairHigh<Register>(), 4073 ShifterOperand(right.AsRegisterPairHigh<Register>())); // Signed compare. 4074 __ b(&less, LT); 4075 __ b(&greater, GT); 4076 // Do LoadImmediate before the last `cmp`, as LoadImmediate might affect the status flags. 4077 __ LoadImmediate(out, 0); 4078 __ cmp(left.AsRegisterPairLow<Register>(), 4079 ShifterOperand(right.AsRegisterPairLow<Register>())); // Unsigned compare. 4080 less_cond = LO; 4081 break; 4082 } 4083 case Primitive::kPrimFloat: 4084 case Primitive::kPrimDouble: { 4085 __ LoadImmediate(out, 0); 4086 GenerateVcmp(compare); 4087 __ vmstat(); // transfer FP status register to ARM APSR. 4088 less_cond = ARMFPCondition(kCondLT, compare->IsGtBias()); 4089 break; 4090 } 4091 default: 4092 LOG(FATAL) << "Unexpected compare type " << type; 4093 UNREACHABLE(); 4094 } 4095 4096 __ b(&done, EQ); 4097 __ b(&less, less_cond); 4098 4099 __ Bind(&greater); 4100 __ LoadImmediate(out, 1); 4101 __ b(&done); 4102 4103 __ Bind(&less); 4104 __ LoadImmediate(out, -1); 4105 4106 __ Bind(&done); 4107} 4108 4109void LocationsBuilderARM::VisitPhi(HPhi* instruction) { 4110 LocationSummary* locations = 4111 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall); 4112 for (size_t i = 0, e = locations->GetInputCount(); i < e; ++i) { 4113 locations->SetInAt(i, Location::Any()); 4114 } 4115 locations->SetOut(Location::Any()); 4116} 4117 4118void InstructionCodeGeneratorARM::VisitPhi(HPhi* instruction ATTRIBUTE_UNUSED) { 4119 LOG(FATAL) << "Unreachable"; 4120} 4121 4122void CodeGeneratorARM::GenerateMemoryBarrier(MemBarrierKind kind) { 4123 // TODO (ported from quick): revisit ARM barrier kinds. 4124 DmbOptions flavor = DmbOptions::ISH; // Quiet C++ warnings. 4125 switch (kind) { 4126 case MemBarrierKind::kAnyStore: 4127 case MemBarrierKind::kLoadAny: 4128 case MemBarrierKind::kAnyAny: { 4129 flavor = DmbOptions::ISH; 4130 break; 4131 } 4132 case MemBarrierKind::kStoreStore: { 4133 flavor = DmbOptions::ISHST; 4134 break; 4135 } 4136 default: 4137 LOG(FATAL) << "Unexpected memory barrier " << kind; 4138 } 4139 __ dmb(flavor); 4140} 4141 4142void InstructionCodeGeneratorARM::GenerateWideAtomicLoad(Register addr, 4143 uint32_t offset, 4144 Register out_lo, 4145 Register out_hi) { 4146 if (offset != 0) { 4147 // Ensure `out_lo` is different from `addr`, so that loading 4148 // `offset` into `out_lo` does not clutter `addr`. 4149 DCHECK_NE(out_lo, addr); 4150 __ LoadImmediate(out_lo, offset); 4151 __ add(IP, addr, ShifterOperand(out_lo)); 4152 addr = IP; 4153 } 4154 __ ldrexd(out_lo, out_hi, addr); 4155} 4156 4157void InstructionCodeGeneratorARM::GenerateWideAtomicStore(Register addr, 4158 uint32_t offset, 4159 Register value_lo, 4160 Register value_hi, 4161 Register temp1, 4162 Register temp2, 4163 HInstruction* instruction) { 4164 Label fail; 4165 if (offset != 0) { 4166 __ LoadImmediate(temp1, offset); 4167 __ add(IP, addr, ShifterOperand(temp1)); 4168 addr = IP; 4169 } 4170 __ Bind(&fail); 4171 // We need a load followed by store. (The address used in a STREX instruction must 4172 // be the same as the address in the most recently executed LDREX instruction.) 4173 __ ldrexd(temp1, temp2, addr); 4174 codegen_->MaybeRecordImplicitNullCheck(instruction); 4175 __ strexd(temp1, value_lo, value_hi, addr); 4176 __ CompareAndBranchIfNonZero(temp1, &fail); 4177} 4178 4179void LocationsBuilderARM::HandleFieldSet(HInstruction* instruction, const FieldInfo& field_info) { 4180 DCHECK(instruction->IsInstanceFieldSet() || instruction->IsStaticFieldSet()); 4181 4182 LocationSummary* locations = 4183 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall); 4184 locations->SetInAt(0, Location::RequiresRegister()); 4185 4186 Primitive::Type field_type = field_info.GetFieldType(); 4187 if (Primitive::IsFloatingPointType(field_type)) { 4188 locations->SetInAt(1, Location::RequiresFpuRegister()); 4189 } else { 4190 locations->SetInAt(1, Location::RequiresRegister()); 4191 } 4192 4193 bool is_wide = field_type == Primitive::kPrimLong || field_type == Primitive::kPrimDouble; 4194 bool generate_volatile = field_info.IsVolatile() 4195 && is_wide 4196 && !codegen_->GetInstructionSetFeatures().HasAtomicLdrdAndStrd(); 4197 bool needs_write_barrier = 4198 CodeGenerator::StoreNeedsWriteBarrier(field_type, instruction->InputAt(1)); 4199 // Temporary registers for the write barrier. 4200 // TODO: consider renaming StoreNeedsWriteBarrier to StoreNeedsGCMark. 4201 if (needs_write_barrier) { 4202 locations->AddTemp(Location::RequiresRegister()); // Possibly used for reference poisoning too. 4203 locations->AddTemp(Location::RequiresRegister()); 4204 } else if (generate_volatile) { 4205 // ARM encoding have some additional constraints for ldrexd/strexd: 4206 // - registers need to be consecutive 4207 // - the first register should be even but not R14. 4208 // We don't test for ARM yet, and the assertion makes sure that we 4209 // revisit this if we ever enable ARM encoding. 4210 DCHECK_EQ(InstructionSet::kThumb2, codegen_->GetInstructionSet()); 4211 4212 locations->AddTemp(Location::RequiresRegister()); 4213 locations->AddTemp(Location::RequiresRegister()); 4214 if (field_type == Primitive::kPrimDouble) { 4215 // For doubles we need two more registers to copy the value. 4216 locations->AddTemp(Location::RegisterLocation(R2)); 4217 locations->AddTemp(Location::RegisterLocation(R3)); 4218 } 4219 } 4220} 4221 4222void InstructionCodeGeneratorARM::HandleFieldSet(HInstruction* instruction, 4223 const FieldInfo& field_info, 4224 bool value_can_be_null) { 4225 DCHECK(instruction->IsInstanceFieldSet() || instruction->IsStaticFieldSet()); 4226 4227 LocationSummary* locations = instruction->GetLocations(); 4228 Register base = locations->InAt(0).AsRegister<Register>(); 4229 Location value = locations->InAt(1); 4230 4231 bool is_volatile = field_info.IsVolatile(); 4232 bool atomic_ldrd_strd = codegen_->GetInstructionSetFeatures().HasAtomicLdrdAndStrd(); 4233 Primitive::Type field_type = field_info.GetFieldType(); 4234 uint32_t offset = field_info.GetFieldOffset().Uint32Value(); 4235 bool needs_write_barrier = 4236 CodeGenerator::StoreNeedsWriteBarrier(field_type, instruction->InputAt(1)); 4237 4238 if (is_volatile) { 4239 codegen_->GenerateMemoryBarrier(MemBarrierKind::kAnyStore); 4240 } 4241 4242 switch (field_type) { 4243 case Primitive::kPrimBoolean: 4244 case Primitive::kPrimByte: { 4245 __ StoreToOffset(kStoreByte, value.AsRegister<Register>(), base, offset); 4246 break; 4247 } 4248 4249 case Primitive::kPrimShort: 4250 case Primitive::kPrimChar: { 4251 __ StoreToOffset(kStoreHalfword, value.AsRegister<Register>(), base, offset); 4252 break; 4253 } 4254 4255 case Primitive::kPrimInt: 4256 case Primitive::kPrimNot: { 4257 if (kPoisonHeapReferences && needs_write_barrier) { 4258 // Note that in the case where `value` is a null reference, 4259 // we do not enter this block, as a null reference does not 4260 // need poisoning. 4261 DCHECK_EQ(field_type, Primitive::kPrimNot); 4262 Register temp = locations->GetTemp(0).AsRegister<Register>(); 4263 __ Mov(temp, value.AsRegister<Register>()); 4264 __ PoisonHeapReference(temp); 4265 __ StoreToOffset(kStoreWord, temp, base, offset); 4266 } else { 4267 __ StoreToOffset(kStoreWord, value.AsRegister<Register>(), base, offset); 4268 } 4269 break; 4270 } 4271 4272 case Primitive::kPrimLong: { 4273 if (is_volatile && !atomic_ldrd_strd) { 4274 GenerateWideAtomicStore(base, offset, 4275 value.AsRegisterPairLow<Register>(), 4276 value.AsRegisterPairHigh<Register>(), 4277 locations->GetTemp(0).AsRegister<Register>(), 4278 locations->GetTemp(1).AsRegister<Register>(), 4279 instruction); 4280 } else { 4281 __ StoreToOffset(kStoreWordPair, value.AsRegisterPairLow<Register>(), base, offset); 4282 codegen_->MaybeRecordImplicitNullCheck(instruction); 4283 } 4284 break; 4285 } 4286 4287 case Primitive::kPrimFloat: { 4288 __ StoreSToOffset(value.AsFpuRegister<SRegister>(), base, offset); 4289 break; 4290 } 4291 4292 case Primitive::kPrimDouble: { 4293 DRegister value_reg = FromLowSToD(value.AsFpuRegisterPairLow<SRegister>()); 4294 if (is_volatile && !atomic_ldrd_strd) { 4295 Register value_reg_lo = locations->GetTemp(0).AsRegister<Register>(); 4296 Register value_reg_hi = locations->GetTemp(1).AsRegister<Register>(); 4297 4298 __ vmovrrd(value_reg_lo, value_reg_hi, value_reg); 4299 4300 GenerateWideAtomicStore(base, offset, 4301 value_reg_lo, 4302 value_reg_hi, 4303 locations->GetTemp(2).AsRegister<Register>(), 4304 locations->GetTemp(3).AsRegister<Register>(), 4305 instruction); 4306 } else { 4307 __ StoreDToOffset(value_reg, base, offset); 4308 codegen_->MaybeRecordImplicitNullCheck(instruction); 4309 } 4310 break; 4311 } 4312 4313 case Primitive::kPrimVoid: 4314 LOG(FATAL) << "Unreachable type " << field_type; 4315 UNREACHABLE(); 4316 } 4317 4318 // Longs and doubles are handled in the switch. 4319 if (field_type != Primitive::kPrimLong && field_type != Primitive::kPrimDouble) { 4320 codegen_->MaybeRecordImplicitNullCheck(instruction); 4321 } 4322 4323 if (CodeGenerator::StoreNeedsWriteBarrier(field_type, instruction->InputAt(1))) { 4324 Register temp = locations->GetTemp(0).AsRegister<Register>(); 4325 Register card = locations->GetTemp(1).AsRegister<Register>(); 4326 codegen_->MarkGCCard( 4327 temp, card, base, value.AsRegister<Register>(), value_can_be_null); 4328 } 4329 4330 if (is_volatile) { 4331 codegen_->GenerateMemoryBarrier(MemBarrierKind::kAnyAny); 4332 } 4333} 4334 4335void LocationsBuilderARM::HandleFieldGet(HInstruction* instruction, const FieldInfo& field_info) { 4336 DCHECK(instruction->IsInstanceFieldGet() || instruction->IsStaticFieldGet()); 4337 4338 bool object_field_get_with_read_barrier = 4339 kEmitCompilerReadBarrier && (field_info.GetFieldType() == Primitive::kPrimNot); 4340 LocationSummary* locations = 4341 new (GetGraph()->GetArena()) LocationSummary(instruction, 4342 object_field_get_with_read_barrier ? 4343 LocationSummary::kCallOnSlowPath : 4344 LocationSummary::kNoCall); 4345 if (object_field_get_with_read_barrier && kUseBakerReadBarrier) { 4346 locations->SetCustomSlowPathCallerSaves(RegisterSet::Empty()); // No caller-save registers. 4347 } 4348 locations->SetInAt(0, Location::RequiresRegister()); 4349 4350 bool volatile_for_double = field_info.IsVolatile() 4351 && (field_info.GetFieldType() == Primitive::kPrimDouble) 4352 && !codegen_->GetInstructionSetFeatures().HasAtomicLdrdAndStrd(); 4353 // The output overlaps in case of volatile long: we don't want the 4354 // code generated by GenerateWideAtomicLoad to overwrite the 4355 // object's location. Likewise, in the case of an object field get 4356 // with read barriers enabled, we do not want the load to overwrite 4357 // the object's location, as we need it to emit the read barrier. 4358 bool overlap = (field_info.IsVolatile() && (field_info.GetFieldType() == Primitive::kPrimLong)) || 4359 object_field_get_with_read_barrier; 4360 4361 if (Primitive::IsFloatingPointType(instruction->GetType())) { 4362 locations->SetOut(Location::RequiresFpuRegister()); 4363 } else { 4364 locations->SetOut(Location::RequiresRegister(), 4365 (overlap ? Location::kOutputOverlap : Location::kNoOutputOverlap)); 4366 } 4367 if (volatile_for_double) { 4368 // ARM encoding have some additional constraints for ldrexd/strexd: 4369 // - registers need to be consecutive 4370 // - the first register should be even but not R14. 4371 // We don't test for ARM yet, and the assertion makes sure that we 4372 // revisit this if we ever enable ARM encoding. 4373 DCHECK_EQ(InstructionSet::kThumb2, codegen_->GetInstructionSet()); 4374 locations->AddTemp(Location::RequiresRegister()); 4375 locations->AddTemp(Location::RequiresRegister()); 4376 } else if (object_field_get_with_read_barrier && kUseBakerReadBarrier) { 4377 // We need a temporary register for the read barrier marking slow 4378 // path in CodeGeneratorARM::GenerateFieldLoadWithBakerReadBarrier. 4379 locations->AddTemp(Location::RequiresRegister()); 4380 } 4381} 4382 4383Location LocationsBuilderARM::ArithmeticZeroOrFpuRegister(HInstruction* input) { 4384 DCHECK(input->GetType() == Primitive::kPrimDouble || input->GetType() == Primitive::kPrimFloat) 4385 << input->GetType(); 4386 if ((input->IsFloatConstant() && (input->AsFloatConstant()->IsArithmeticZero())) || 4387 (input->IsDoubleConstant() && (input->AsDoubleConstant()->IsArithmeticZero()))) { 4388 return Location::ConstantLocation(input->AsConstant()); 4389 } else { 4390 return Location::RequiresFpuRegister(); 4391 } 4392} 4393 4394Location LocationsBuilderARM::ArmEncodableConstantOrRegister(HInstruction* constant, 4395 Opcode opcode) { 4396 DCHECK(!Primitive::IsFloatingPointType(constant->GetType())); 4397 if (constant->IsConstant() && 4398 CanEncodeConstantAsImmediate(constant->AsConstant(), opcode)) { 4399 return Location::ConstantLocation(constant->AsConstant()); 4400 } 4401 return Location::RequiresRegister(); 4402} 4403 4404bool LocationsBuilderARM::CanEncodeConstantAsImmediate(HConstant* input_cst, 4405 Opcode opcode) { 4406 uint64_t value = static_cast<uint64_t>(Int64FromConstant(input_cst)); 4407 if (Primitive::Is64BitType(input_cst->GetType())) { 4408 Opcode high_opcode = opcode; 4409 SetCc low_set_cc = kCcDontCare; 4410 switch (opcode) { 4411 case SUB: 4412 // Flip the operation to an ADD. 4413 value = -value; 4414 opcode = ADD; 4415 FALLTHROUGH_INTENDED; 4416 case ADD: 4417 if (Low32Bits(value) == 0u) { 4418 return CanEncodeConstantAsImmediate(High32Bits(value), opcode, kCcDontCare); 4419 } 4420 high_opcode = ADC; 4421 low_set_cc = kCcSet; 4422 break; 4423 default: 4424 break; 4425 } 4426 return CanEncodeConstantAsImmediate(Low32Bits(value), opcode, low_set_cc) && 4427 CanEncodeConstantAsImmediate(High32Bits(value), high_opcode, kCcDontCare); 4428 } else { 4429 return CanEncodeConstantAsImmediate(Low32Bits(value), opcode); 4430 } 4431} 4432 4433bool LocationsBuilderARM::CanEncodeConstantAsImmediate(uint32_t value, 4434 Opcode opcode, 4435 SetCc set_cc) { 4436 ShifterOperand so; 4437 ArmAssembler* assembler = codegen_->GetAssembler(); 4438 if (assembler->ShifterOperandCanHold(kNoRegister, kNoRegister, opcode, value, set_cc, &so)) { 4439 return true; 4440 } 4441 Opcode neg_opcode = kNoOperand; 4442 switch (opcode) { 4443 case AND: neg_opcode = BIC; value = ~value; break; 4444 case ORR: neg_opcode = ORN; value = ~value; break; 4445 case ADD: neg_opcode = SUB; value = -value; break; 4446 case ADC: neg_opcode = SBC; value = ~value; break; 4447 case SUB: neg_opcode = ADD; value = -value; break; 4448 case SBC: neg_opcode = ADC; value = ~value; break; 4449 default: 4450 return false; 4451 } 4452 return assembler->ShifterOperandCanHold(kNoRegister, kNoRegister, neg_opcode, value, set_cc, &so); 4453} 4454 4455void InstructionCodeGeneratorARM::HandleFieldGet(HInstruction* instruction, 4456 const FieldInfo& field_info) { 4457 DCHECK(instruction->IsInstanceFieldGet() || instruction->IsStaticFieldGet()); 4458 4459 LocationSummary* locations = instruction->GetLocations(); 4460 Location base_loc = locations->InAt(0); 4461 Register base = base_loc.AsRegister<Register>(); 4462 Location out = locations->Out(); 4463 bool is_volatile = field_info.IsVolatile(); 4464 bool atomic_ldrd_strd = codegen_->GetInstructionSetFeatures().HasAtomicLdrdAndStrd(); 4465 Primitive::Type field_type = field_info.GetFieldType(); 4466 uint32_t offset = field_info.GetFieldOffset().Uint32Value(); 4467 4468 switch (field_type) { 4469 case Primitive::kPrimBoolean: 4470 __ LoadFromOffset(kLoadUnsignedByte, out.AsRegister<Register>(), base, offset); 4471 break; 4472 4473 case Primitive::kPrimByte: 4474 __ LoadFromOffset(kLoadSignedByte, out.AsRegister<Register>(), base, offset); 4475 break; 4476 4477 case Primitive::kPrimShort: 4478 __ LoadFromOffset(kLoadSignedHalfword, out.AsRegister<Register>(), base, offset); 4479 break; 4480 4481 case Primitive::kPrimChar: 4482 __ LoadFromOffset(kLoadUnsignedHalfword, out.AsRegister<Register>(), base, offset); 4483 break; 4484 4485 case Primitive::kPrimInt: 4486 __ LoadFromOffset(kLoadWord, out.AsRegister<Register>(), base, offset); 4487 break; 4488 4489 case Primitive::kPrimNot: { 4490 // /* HeapReference<Object> */ out = *(base + offset) 4491 if (kEmitCompilerReadBarrier && kUseBakerReadBarrier) { 4492 Location temp_loc = locations->GetTemp(0); 4493 // Note that a potential implicit null check is handled in this 4494 // CodeGeneratorARM::GenerateFieldLoadWithBakerReadBarrier call. 4495 codegen_->GenerateFieldLoadWithBakerReadBarrier( 4496 instruction, out, base, offset, temp_loc, /* needs_null_check */ true); 4497 if (is_volatile) { 4498 codegen_->GenerateMemoryBarrier(MemBarrierKind::kLoadAny); 4499 } 4500 } else { 4501 __ LoadFromOffset(kLoadWord, out.AsRegister<Register>(), base, offset); 4502 codegen_->MaybeRecordImplicitNullCheck(instruction); 4503 if (is_volatile) { 4504 codegen_->GenerateMemoryBarrier(MemBarrierKind::kLoadAny); 4505 } 4506 // If read barriers are enabled, emit read barriers other than 4507 // Baker's using a slow path (and also unpoison the loaded 4508 // reference, if heap poisoning is enabled). 4509 codegen_->MaybeGenerateReadBarrierSlow(instruction, out, out, base_loc, offset); 4510 } 4511 break; 4512 } 4513 4514 case Primitive::kPrimLong: 4515 if (is_volatile && !atomic_ldrd_strd) { 4516 GenerateWideAtomicLoad(base, offset, 4517 out.AsRegisterPairLow<Register>(), 4518 out.AsRegisterPairHigh<Register>()); 4519 } else { 4520 __ LoadFromOffset(kLoadWordPair, out.AsRegisterPairLow<Register>(), base, offset); 4521 } 4522 break; 4523 4524 case Primitive::kPrimFloat: 4525 __ LoadSFromOffset(out.AsFpuRegister<SRegister>(), base, offset); 4526 break; 4527 4528 case Primitive::kPrimDouble: { 4529 DRegister out_reg = FromLowSToD(out.AsFpuRegisterPairLow<SRegister>()); 4530 if (is_volatile && !atomic_ldrd_strd) { 4531 Register lo = locations->GetTemp(0).AsRegister<Register>(); 4532 Register hi = locations->GetTemp(1).AsRegister<Register>(); 4533 GenerateWideAtomicLoad(base, offset, lo, hi); 4534 codegen_->MaybeRecordImplicitNullCheck(instruction); 4535 __ vmovdrr(out_reg, lo, hi); 4536 } else { 4537 __ LoadDFromOffset(out_reg, base, offset); 4538 codegen_->MaybeRecordImplicitNullCheck(instruction); 4539 } 4540 break; 4541 } 4542 4543 case Primitive::kPrimVoid: 4544 LOG(FATAL) << "Unreachable type " << field_type; 4545 UNREACHABLE(); 4546 } 4547 4548 if (field_type == Primitive::kPrimNot || field_type == Primitive::kPrimDouble) { 4549 // Potential implicit null checks, in the case of reference or 4550 // double fields, are handled in the previous switch statement. 4551 } else { 4552 codegen_->MaybeRecordImplicitNullCheck(instruction); 4553 } 4554 4555 if (is_volatile) { 4556 if (field_type == Primitive::kPrimNot) { 4557 // Memory barriers, in the case of references, are also handled 4558 // in the previous switch statement. 4559 } else { 4560 codegen_->GenerateMemoryBarrier(MemBarrierKind::kLoadAny); 4561 } 4562 } 4563} 4564 4565void LocationsBuilderARM::VisitInstanceFieldSet(HInstanceFieldSet* instruction) { 4566 HandleFieldSet(instruction, instruction->GetFieldInfo()); 4567} 4568 4569void InstructionCodeGeneratorARM::VisitInstanceFieldSet(HInstanceFieldSet* instruction) { 4570 HandleFieldSet(instruction, instruction->GetFieldInfo(), instruction->GetValueCanBeNull()); 4571} 4572 4573void LocationsBuilderARM::VisitInstanceFieldGet(HInstanceFieldGet* instruction) { 4574 HandleFieldGet(instruction, instruction->GetFieldInfo()); 4575} 4576 4577void InstructionCodeGeneratorARM::VisitInstanceFieldGet(HInstanceFieldGet* instruction) { 4578 HandleFieldGet(instruction, instruction->GetFieldInfo()); 4579} 4580 4581void LocationsBuilderARM::VisitStaticFieldGet(HStaticFieldGet* instruction) { 4582 HandleFieldGet(instruction, instruction->GetFieldInfo()); 4583} 4584 4585void InstructionCodeGeneratorARM::VisitStaticFieldGet(HStaticFieldGet* instruction) { 4586 HandleFieldGet(instruction, instruction->GetFieldInfo()); 4587} 4588 4589void LocationsBuilderARM::VisitStaticFieldSet(HStaticFieldSet* instruction) { 4590 HandleFieldSet(instruction, instruction->GetFieldInfo()); 4591} 4592 4593void InstructionCodeGeneratorARM::VisitStaticFieldSet(HStaticFieldSet* instruction) { 4594 HandleFieldSet(instruction, instruction->GetFieldInfo(), instruction->GetValueCanBeNull()); 4595} 4596 4597void LocationsBuilderARM::VisitUnresolvedInstanceFieldGet( 4598 HUnresolvedInstanceFieldGet* instruction) { 4599 FieldAccessCallingConventionARM calling_convention; 4600 codegen_->CreateUnresolvedFieldLocationSummary( 4601 instruction, instruction->GetFieldType(), calling_convention); 4602} 4603 4604void InstructionCodeGeneratorARM::VisitUnresolvedInstanceFieldGet( 4605 HUnresolvedInstanceFieldGet* instruction) { 4606 FieldAccessCallingConventionARM calling_convention; 4607 codegen_->GenerateUnresolvedFieldAccess(instruction, 4608 instruction->GetFieldType(), 4609 instruction->GetFieldIndex(), 4610 instruction->GetDexPc(), 4611 calling_convention); 4612} 4613 4614void LocationsBuilderARM::VisitUnresolvedInstanceFieldSet( 4615 HUnresolvedInstanceFieldSet* instruction) { 4616 FieldAccessCallingConventionARM calling_convention; 4617 codegen_->CreateUnresolvedFieldLocationSummary( 4618 instruction, instruction->GetFieldType(), calling_convention); 4619} 4620 4621void InstructionCodeGeneratorARM::VisitUnresolvedInstanceFieldSet( 4622 HUnresolvedInstanceFieldSet* instruction) { 4623 FieldAccessCallingConventionARM calling_convention; 4624 codegen_->GenerateUnresolvedFieldAccess(instruction, 4625 instruction->GetFieldType(), 4626 instruction->GetFieldIndex(), 4627 instruction->GetDexPc(), 4628 calling_convention); 4629} 4630 4631void LocationsBuilderARM::VisitUnresolvedStaticFieldGet( 4632 HUnresolvedStaticFieldGet* instruction) { 4633 FieldAccessCallingConventionARM calling_convention; 4634 codegen_->CreateUnresolvedFieldLocationSummary( 4635 instruction, instruction->GetFieldType(), calling_convention); 4636} 4637 4638void InstructionCodeGeneratorARM::VisitUnresolvedStaticFieldGet( 4639 HUnresolvedStaticFieldGet* instruction) { 4640 FieldAccessCallingConventionARM calling_convention; 4641 codegen_->GenerateUnresolvedFieldAccess(instruction, 4642 instruction->GetFieldType(), 4643 instruction->GetFieldIndex(), 4644 instruction->GetDexPc(), 4645 calling_convention); 4646} 4647 4648void LocationsBuilderARM::VisitUnresolvedStaticFieldSet( 4649 HUnresolvedStaticFieldSet* instruction) { 4650 FieldAccessCallingConventionARM calling_convention; 4651 codegen_->CreateUnresolvedFieldLocationSummary( 4652 instruction, instruction->GetFieldType(), calling_convention); 4653} 4654 4655void InstructionCodeGeneratorARM::VisitUnresolvedStaticFieldSet( 4656 HUnresolvedStaticFieldSet* instruction) { 4657 FieldAccessCallingConventionARM calling_convention; 4658 codegen_->GenerateUnresolvedFieldAccess(instruction, 4659 instruction->GetFieldType(), 4660 instruction->GetFieldIndex(), 4661 instruction->GetDexPc(), 4662 calling_convention); 4663} 4664 4665void LocationsBuilderARM::VisitNullCheck(HNullCheck* instruction) { 4666 LocationSummary* locations = codegen_->CreateThrowingSlowPathLocations(instruction); 4667 locations->SetInAt(0, Location::RequiresRegister()); 4668} 4669 4670void CodeGeneratorARM::GenerateImplicitNullCheck(HNullCheck* instruction) { 4671 if (CanMoveNullCheckToUser(instruction)) { 4672 return; 4673 } 4674 Location obj = instruction->GetLocations()->InAt(0); 4675 4676 __ LoadFromOffset(kLoadWord, IP, obj.AsRegister<Register>(), 0); 4677 RecordPcInfo(instruction, instruction->GetDexPc()); 4678} 4679 4680void CodeGeneratorARM::GenerateExplicitNullCheck(HNullCheck* instruction) { 4681 SlowPathCodeARM* slow_path = new (GetGraph()->GetArena()) NullCheckSlowPathARM(instruction); 4682 AddSlowPath(slow_path); 4683 4684 LocationSummary* locations = instruction->GetLocations(); 4685 Location obj = locations->InAt(0); 4686 4687 __ CompareAndBranchIfZero(obj.AsRegister<Register>(), slow_path->GetEntryLabel()); 4688} 4689 4690void InstructionCodeGeneratorARM::VisitNullCheck(HNullCheck* instruction) { 4691 codegen_->GenerateNullCheck(instruction); 4692} 4693 4694static LoadOperandType GetLoadOperandType(Primitive::Type type) { 4695 switch (type) { 4696 case Primitive::kPrimNot: 4697 return kLoadWord; 4698 case Primitive::kPrimBoolean: 4699 return kLoadUnsignedByte; 4700 case Primitive::kPrimByte: 4701 return kLoadSignedByte; 4702 case Primitive::kPrimChar: 4703 return kLoadUnsignedHalfword; 4704 case Primitive::kPrimShort: 4705 return kLoadSignedHalfword; 4706 case Primitive::kPrimInt: 4707 return kLoadWord; 4708 case Primitive::kPrimLong: 4709 return kLoadWordPair; 4710 case Primitive::kPrimFloat: 4711 return kLoadSWord; 4712 case Primitive::kPrimDouble: 4713 return kLoadDWord; 4714 default: 4715 LOG(FATAL) << "Unreachable type " << type; 4716 UNREACHABLE(); 4717 } 4718} 4719 4720static StoreOperandType GetStoreOperandType(Primitive::Type type) { 4721 switch (type) { 4722 case Primitive::kPrimNot: 4723 return kStoreWord; 4724 case Primitive::kPrimBoolean: 4725 case Primitive::kPrimByte: 4726 return kStoreByte; 4727 case Primitive::kPrimChar: 4728 case Primitive::kPrimShort: 4729 return kStoreHalfword; 4730 case Primitive::kPrimInt: 4731 return kStoreWord; 4732 case Primitive::kPrimLong: 4733 return kStoreWordPair; 4734 case Primitive::kPrimFloat: 4735 return kStoreSWord; 4736 case Primitive::kPrimDouble: 4737 return kStoreDWord; 4738 default: 4739 LOG(FATAL) << "Unreachable type " << type; 4740 UNREACHABLE(); 4741 } 4742} 4743 4744void CodeGeneratorARM::LoadFromShiftedRegOffset(Primitive::Type type, 4745 Location out_loc, 4746 Register base, 4747 Register reg_offset, 4748 Condition cond) { 4749 uint32_t shift_count = Primitive::ComponentSizeShift(type); 4750 Address mem_address(base, reg_offset, Shift::LSL, shift_count); 4751 4752 switch (type) { 4753 case Primitive::kPrimByte: 4754 __ ldrsb(out_loc.AsRegister<Register>(), mem_address, cond); 4755 break; 4756 case Primitive::kPrimBoolean: 4757 __ ldrb(out_loc.AsRegister<Register>(), mem_address, cond); 4758 break; 4759 case Primitive::kPrimShort: 4760 __ ldrsh(out_loc.AsRegister<Register>(), mem_address, cond); 4761 break; 4762 case Primitive::kPrimChar: 4763 __ ldrh(out_loc.AsRegister<Register>(), mem_address, cond); 4764 break; 4765 case Primitive::kPrimNot: 4766 case Primitive::kPrimInt: 4767 __ ldr(out_loc.AsRegister<Register>(), mem_address, cond); 4768 break; 4769 // T32 doesn't support LoadFromShiftedRegOffset mem address mode for these types. 4770 case Primitive::kPrimLong: 4771 case Primitive::kPrimFloat: 4772 case Primitive::kPrimDouble: 4773 default: 4774 LOG(FATAL) << "Unreachable type " << type; 4775 UNREACHABLE(); 4776 } 4777} 4778 4779void CodeGeneratorARM::StoreToShiftedRegOffset(Primitive::Type type, 4780 Location loc, 4781 Register base, 4782 Register reg_offset, 4783 Condition cond) { 4784 uint32_t shift_count = Primitive::ComponentSizeShift(type); 4785 Address mem_address(base, reg_offset, Shift::LSL, shift_count); 4786 4787 switch (type) { 4788 case Primitive::kPrimByte: 4789 case Primitive::kPrimBoolean: 4790 __ strb(loc.AsRegister<Register>(), mem_address, cond); 4791 break; 4792 case Primitive::kPrimShort: 4793 case Primitive::kPrimChar: 4794 __ strh(loc.AsRegister<Register>(), mem_address, cond); 4795 break; 4796 case Primitive::kPrimNot: 4797 case Primitive::kPrimInt: 4798 __ str(loc.AsRegister<Register>(), mem_address, cond); 4799 break; 4800 // T32 doesn't support StoreToShiftedRegOffset mem address mode for these types. 4801 case Primitive::kPrimLong: 4802 case Primitive::kPrimFloat: 4803 case Primitive::kPrimDouble: 4804 default: 4805 LOG(FATAL) << "Unreachable type " << type; 4806 UNREACHABLE(); 4807 } 4808} 4809 4810void LocationsBuilderARM::VisitArrayGet(HArrayGet* instruction) { 4811 bool object_array_get_with_read_barrier = 4812 kEmitCompilerReadBarrier && (instruction->GetType() == Primitive::kPrimNot); 4813 LocationSummary* locations = 4814 new (GetGraph()->GetArena()) LocationSummary(instruction, 4815 object_array_get_with_read_barrier ? 4816 LocationSummary::kCallOnSlowPath : 4817 LocationSummary::kNoCall); 4818 if (object_array_get_with_read_barrier && kUseBakerReadBarrier) { 4819 locations->SetCustomSlowPathCallerSaves(RegisterSet::Empty()); // No caller-save registers. 4820 } 4821 locations->SetInAt(0, Location::RequiresRegister()); 4822 locations->SetInAt(1, Location::RegisterOrConstant(instruction->InputAt(1))); 4823 if (Primitive::IsFloatingPointType(instruction->GetType())) { 4824 locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap); 4825 } else { 4826 // The output overlaps in the case of an object array get with 4827 // read barriers enabled: we do not want the move to overwrite the 4828 // array's location, as we need it to emit the read barrier. 4829 locations->SetOut( 4830 Location::RequiresRegister(), 4831 object_array_get_with_read_barrier ? Location::kOutputOverlap : Location::kNoOutputOverlap); 4832 } 4833 // We need a temporary register for the read barrier marking slow 4834 // path in CodeGeneratorARM::GenerateArrayLoadWithBakerReadBarrier. 4835 // Also need for String compression feature. 4836 if ((object_array_get_with_read_barrier && kUseBakerReadBarrier) 4837 || (mirror::kUseStringCompression && instruction->IsStringCharAt())) { 4838 locations->AddTemp(Location::RequiresRegister()); 4839 } 4840} 4841 4842void InstructionCodeGeneratorARM::VisitArrayGet(HArrayGet* instruction) { 4843 LocationSummary* locations = instruction->GetLocations(); 4844 Location obj_loc = locations->InAt(0); 4845 Register obj = obj_loc.AsRegister<Register>(); 4846 Location index = locations->InAt(1); 4847 Location out_loc = locations->Out(); 4848 uint32_t data_offset = CodeGenerator::GetArrayDataOffset(instruction); 4849 Primitive::Type type = instruction->GetType(); 4850 const bool maybe_compressed_char_at = mirror::kUseStringCompression && 4851 instruction->IsStringCharAt(); 4852 HInstruction* array_instr = instruction->GetArray(); 4853 bool has_intermediate_address = array_instr->IsIntermediateAddress(); 4854 4855 switch (type) { 4856 case Primitive::kPrimBoolean: 4857 case Primitive::kPrimByte: 4858 case Primitive::kPrimShort: 4859 case Primitive::kPrimChar: 4860 case Primitive::kPrimInt: { 4861 if (index.IsConstant()) { 4862 int32_t const_index = index.GetConstant()->AsIntConstant()->GetValue(); 4863 if (maybe_compressed_char_at) { 4864 Register length = IP; 4865 Label uncompressed_load, done; 4866 uint32_t count_offset = mirror::String::CountOffset().Uint32Value(); 4867 __ LoadFromOffset(kLoadWord, length, obj, count_offset); 4868 codegen_->MaybeRecordImplicitNullCheck(instruction); 4869 __ cmp(length, ShifterOperand(0)); 4870 __ b(&uncompressed_load, GE); 4871 __ LoadFromOffset(kLoadUnsignedByte, 4872 out_loc.AsRegister<Register>(), 4873 obj, 4874 data_offset + const_index); 4875 __ b(&done); 4876 __ Bind(&uncompressed_load); 4877 __ LoadFromOffset(GetLoadOperandType(Primitive::kPrimChar), 4878 out_loc.AsRegister<Register>(), 4879 obj, 4880 data_offset + (const_index << 1)); 4881 __ Bind(&done); 4882 } else { 4883 uint32_t full_offset = data_offset + (const_index << Primitive::ComponentSizeShift(type)); 4884 4885 LoadOperandType load_type = GetLoadOperandType(type); 4886 __ LoadFromOffset(load_type, out_loc.AsRegister<Register>(), obj, full_offset); 4887 } 4888 } else { 4889 Register temp = IP; 4890 4891 if (has_intermediate_address) { 4892 // We do not need to compute the intermediate address from the array: the 4893 // input instruction has done it already. See the comment in 4894 // `TryExtractArrayAccessAddress()`. 4895 if (kIsDebugBuild) { 4896 HIntermediateAddress* tmp = array_instr->AsIntermediateAddress(); 4897 DCHECK_EQ(tmp->GetOffset()->AsIntConstant()->GetValueAsUint64(), data_offset); 4898 } 4899 temp = obj; 4900 } else { 4901 __ add(temp, obj, ShifterOperand(data_offset)); 4902 } 4903 if (maybe_compressed_char_at) { 4904 Label uncompressed_load, done; 4905 uint32_t count_offset = mirror::String::CountOffset().Uint32Value(); 4906 Register length = locations->GetTemp(0).AsRegister<Register>(); 4907 __ LoadFromOffset(kLoadWord, length, obj, count_offset); 4908 codegen_->MaybeRecordImplicitNullCheck(instruction); 4909 __ cmp(length, ShifterOperand(0)); 4910 __ b(&uncompressed_load, GE); 4911 __ ldrb(out_loc.AsRegister<Register>(), 4912 Address(temp, index.AsRegister<Register>(), Shift::LSL, 0)); 4913 __ b(&done); 4914 __ Bind(&uncompressed_load); 4915 __ ldrh(out_loc.AsRegister<Register>(), 4916 Address(temp, index.AsRegister<Register>(), Shift::LSL, 1)); 4917 __ Bind(&done); 4918 } else { 4919 codegen_->LoadFromShiftedRegOffset(type, out_loc, temp, index.AsRegister<Register>()); 4920 } 4921 } 4922 break; 4923 } 4924 4925 case Primitive::kPrimNot: { 4926 // The read barrier instrumentation of object ArrayGet 4927 // instructions does not support the HIntermediateAddress 4928 // instruction. 4929 DCHECK(!(has_intermediate_address && kEmitCompilerReadBarrier)); 4930 4931 static_assert( 4932 sizeof(mirror::HeapReference<mirror::Object>) == sizeof(int32_t), 4933 "art::mirror::HeapReference<art::mirror::Object> and int32_t have different sizes."); 4934 // /* HeapReference<Object> */ out = 4935 // *(obj + data_offset + index * sizeof(HeapReference<Object>)) 4936 if (kEmitCompilerReadBarrier && kUseBakerReadBarrier) { 4937 Location temp = locations->GetTemp(0); 4938 // Note that a potential implicit null check is handled in this 4939 // CodeGeneratorARM::GenerateArrayLoadWithBakerReadBarrier call. 4940 codegen_->GenerateArrayLoadWithBakerReadBarrier( 4941 instruction, out_loc, obj, data_offset, index, temp, /* needs_null_check */ true); 4942 } else { 4943 Register out = out_loc.AsRegister<Register>(); 4944 if (index.IsConstant()) { 4945 size_t offset = 4946 (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4) + data_offset; 4947 __ LoadFromOffset(kLoadWord, out, obj, offset); 4948 codegen_->MaybeRecordImplicitNullCheck(instruction); 4949 // If read barriers are enabled, emit read barriers other than 4950 // Baker's using a slow path (and also unpoison the loaded 4951 // reference, if heap poisoning is enabled). 4952 codegen_->MaybeGenerateReadBarrierSlow(instruction, out_loc, out_loc, obj_loc, offset); 4953 } else { 4954 Register temp = IP; 4955 4956 if (has_intermediate_address) { 4957 // We do not need to compute the intermediate address from the array: the 4958 // input instruction has done it already. See the comment in 4959 // `TryExtractArrayAccessAddress()`. 4960 if (kIsDebugBuild) { 4961 HIntermediateAddress* tmp = array_instr->AsIntermediateAddress(); 4962 DCHECK_EQ(tmp->GetOffset()->AsIntConstant()->GetValueAsUint64(), data_offset); 4963 } 4964 temp = obj; 4965 } else { 4966 __ add(temp, obj, ShifterOperand(data_offset)); 4967 } 4968 codegen_->LoadFromShiftedRegOffset(type, out_loc, temp, index.AsRegister<Register>()); 4969 4970 codegen_->MaybeRecordImplicitNullCheck(instruction); 4971 // If read barriers are enabled, emit read barriers other than 4972 // Baker's using a slow path (and also unpoison the loaded 4973 // reference, if heap poisoning is enabled). 4974 codegen_->MaybeGenerateReadBarrierSlow( 4975 instruction, out_loc, out_loc, obj_loc, data_offset, index); 4976 } 4977 } 4978 break; 4979 } 4980 4981 case Primitive::kPrimLong: { 4982 if (index.IsConstant()) { 4983 size_t offset = 4984 (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_8) + data_offset; 4985 __ LoadFromOffset(kLoadWordPair, out_loc.AsRegisterPairLow<Register>(), obj, offset); 4986 } else { 4987 __ add(IP, obj, ShifterOperand(index.AsRegister<Register>(), LSL, TIMES_8)); 4988 __ LoadFromOffset(kLoadWordPair, out_loc.AsRegisterPairLow<Register>(), IP, data_offset); 4989 } 4990 break; 4991 } 4992 4993 case Primitive::kPrimFloat: { 4994 SRegister out = out_loc.AsFpuRegister<SRegister>(); 4995 if (index.IsConstant()) { 4996 size_t offset = (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4) + data_offset; 4997 __ LoadSFromOffset(out, obj, offset); 4998 } else { 4999 __ add(IP, obj, ShifterOperand(index.AsRegister<Register>(), LSL, TIMES_4)); 5000 __ LoadSFromOffset(out, IP, data_offset); 5001 } 5002 break; 5003 } 5004 5005 case Primitive::kPrimDouble: { 5006 SRegister out = out_loc.AsFpuRegisterPairLow<SRegister>(); 5007 if (index.IsConstant()) { 5008 size_t offset = (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_8) + data_offset; 5009 __ LoadDFromOffset(FromLowSToD(out), obj, offset); 5010 } else { 5011 __ add(IP, obj, ShifterOperand(index.AsRegister<Register>(), LSL, TIMES_8)); 5012 __ LoadDFromOffset(FromLowSToD(out), IP, data_offset); 5013 } 5014 break; 5015 } 5016 5017 case Primitive::kPrimVoid: 5018 LOG(FATAL) << "Unreachable type " << type; 5019 UNREACHABLE(); 5020 } 5021 5022 if (type == Primitive::kPrimNot) { 5023 // Potential implicit null checks, in the case of reference 5024 // arrays, are handled in the previous switch statement. 5025 } else if (!maybe_compressed_char_at) { 5026 codegen_->MaybeRecordImplicitNullCheck(instruction); 5027 } 5028} 5029 5030void LocationsBuilderARM::VisitArraySet(HArraySet* instruction) { 5031 Primitive::Type value_type = instruction->GetComponentType(); 5032 5033 bool needs_write_barrier = 5034 CodeGenerator::StoreNeedsWriteBarrier(value_type, instruction->GetValue()); 5035 bool may_need_runtime_call_for_type_check = instruction->NeedsTypeCheck(); 5036 5037 LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary( 5038 instruction, 5039 may_need_runtime_call_for_type_check ? 5040 LocationSummary::kCallOnSlowPath : 5041 LocationSummary::kNoCall); 5042 5043 locations->SetInAt(0, Location::RequiresRegister()); 5044 locations->SetInAt(1, Location::RegisterOrConstant(instruction->InputAt(1))); 5045 if (Primitive::IsFloatingPointType(value_type)) { 5046 locations->SetInAt(2, Location::RequiresFpuRegister()); 5047 } else { 5048 locations->SetInAt(2, Location::RequiresRegister()); 5049 } 5050 if (needs_write_barrier) { 5051 // Temporary registers for the write barrier. 5052 locations->AddTemp(Location::RequiresRegister()); // Possibly used for ref. poisoning too. 5053 locations->AddTemp(Location::RequiresRegister()); 5054 } 5055} 5056 5057void InstructionCodeGeneratorARM::VisitArraySet(HArraySet* instruction) { 5058 LocationSummary* locations = instruction->GetLocations(); 5059 Location array_loc = locations->InAt(0); 5060 Register array = array_loc.AsRegister<Register>(); 5061 Location index = locations->InAt(1); 5062 Primitive::Type value_type = instruction->GetComponentType(); 5063 bool may_need_runtime_call_for_type_check = instruction->NeedsTypeCheck(); 5064 bool needs_write_barrier = 5065 CodeGenerator::StoreNeedsWriteBarrier(value_type, instruction->GetValue()); 5066 uint32_t data_offset = 5067 mirror::Array::DataOffset(Primitive::ComponentSize(value_type)).Uint32Value(); 5068 Location value_loc = locations->InAt(2); 5069 HInstruction* array_instr = instruction->GetArray(); 5070 bool has_intermediate_address = array_instr->IsIntermediateAddress(); 5071 5072 switch (value_type) { 5073 case Primitive::kPrimBoolean: 5074 case Primitive::kPrimByte: 5075 case Primitive::kPrimShort: 5076 case Primitive::kPrimChar: 5077 case Primitive::kPrimInt: { 5078 if (index.IsConstant()) { 5079 int32_t const_index = index.GetConstant()->AsIntConstant()->GetValue(); 5080 uint32_t full_offset = 5081 data_offset + (const_index << Primitive::ComponentSizeShift(value_type)); 5082 StoreOperandType store_type = GetStoreOperandType(value_type); 5083 __ StoreToOffset(store_type, value_loc.AsRegister<Register>(), array, full_offset); 5084 } else { 5085 Register temp = IP; 5086 5087 if (has_intermediate_address) { 5088 // We do not need to compute the intermediate address from the array: the 5089 // input instruction has done it already. See the comment in 5090 // `TryExtractArrayAccessAddress()`. 5091 if (kIsDebugBuild) { 5092 HIntermediateAddress* tmp = array_instr->AsIntermediateAddress(); 5093 DCHECK(tmp->GetOffset()->AsIntConstant()->GetValueAsUint64() == data_offset); 5094 } 5095 temp = array; 5096 } else { 5097 __ add(temp, array, ShifterOperand(data_offset)); 5098 } 5099 codegen_->StoreToShiftedRegOffset(value_type, 5100 value_loc, 5101 temp, 5102 index.AsRegister<Register>()); 5103 } 5104 break; 5105 } 5106 5107 case Primitive::kPrimNot: { 5108 Register value = value_loc.AsRegister<Register>(); 5109 // TryExtractArrayAccessAddress optimization is never applied for non-primitive ArraySet. 5110 // See the comment in instruction_simplifier_shared.cc. 5111 DCHECK(!has_intermediate_address); 5112 5113 if (instruction->InputAt(2)->IsNullConstant()) { 5114 // Just setting null. 5115 if (index.IsConstant()) { 5116 size_t offset = 5117 (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4) + data_offset; 5118 __ StoreToOffset(kStoreWord, value, array, offset); 5119 } else { 5120 DCHECK(index.IsRegister()) << index; 5121 __ add(IP, array, ShifterOperand(data_offset)); 5122 codegen_->StoreToShiftedRegOffset(value_type, 5123 value_loc, 5124 IP, 5125 index.AsRegister<Register>()); 5126 } 5127 codegen_->MaybeRecordImplicitNullCheck(instruction); 5128 DCHECK(!needs_write_barrier); 5129 DCHECK(!may_need_runtime_call_for_type_check); 5130 break; 5131 } 5132 5133 DCHECK(needs_write_barrier); 5134 Location temp1_loc = locations->GetTemp(0); 5135 Register temp1 = temp1_loc.AsRegister<Register>(); 5136 Location temp2_loc = locations->GetTemp(1); 5137 Register temp2 = temp2_loc.AsRegister<Register>(); 5138 uint32_t class_offset = mirror::Object::ClassOffset().Int32Value(); 5139 uint32_t super_offset = mirror::Class::SuperClassOffset().Int32Value(); 5140 uint32_t component_offset = mirror::Class::ComponentTypeOffset().Int32Value(); 5141 Label done; 5142 SlowPathCodeARM* slow_path = nullptr; 5143 5144 if (may_need_runtime_call_for_type_check) { 5145 slow_path = new (GetGraph()->GetArena()) ArraySetSlowPathARM(instruction); 5146 codegen_->AddSlowPath(slow_path); 5147 if (instruction->GetValueCanBeNull()) { 5148 Label non_zero; 5149 __ CompareAndBranchIfNonZero(value, &non_zero); 5150 if (index.IsConstant()) { 5151 size_t offset = 5152 (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4) + data_offset; 5153 __ StoreToOffset(kStoreWord, value, array, offset); 5154 } else { 5155 DCHECK(index.IsRegister()) << index; 5156 __ add(IP, array, ShifterOperand(data_offset)); 5157 codegen_->StoreToShiftedRegOffset(value_type, 5158 value_loc, 5159 IP, 5160 index.AsRegister<Register>()); 5161 } 5162 codegen_->MaybeRecordImplicitNullCheck(instruction); 5163 __ b(&done); 5164 __ Bind(&non_zero); 5165 } 5166 5167 // Note that when read barriers are enabled, the type checks 5168 // are performed without read barriers. This is fine, even in 5169 // the case where a class object is in the from-space after 5170 // the flip, as a comparison involving such a type would not 5171 // produce a false positive; it may of course produce a false 5172 // negative, in which case we would take the ArraySet slow 5173 // path. 5174 5175 // /* HeapReference<Class> */ temp1 = array->klass_ 5176 __ LoadFromOffset(kLoadWord, temp1, array, class_offset); 5177 codegen_->MaybeRecordImplicitNullCheck(instruction); 5178 __ MaybeUnpoisonHeapReference(temp1); 5179 5180 // /* HeapReference<Class> */ temp1 = temp1->component_type_ 5181 __ LoadFromOffset(kLoadWord, temp1, temp1, component_offset); 5182 // /* HeapReference<Class> */ temp2 = value->klass_ 5183 __ LoadFromOffset(kLoadWord, temp2, value, class_offset); 5184 // If heap poisoning is enabled, no need to unpoison `temp1` 5185 // nor `temp2`, as we are comparing two poisoned references. 5186 __ cmp(temp1, ShifterOperand(temp2)); 5187 5188 if (instruction->StaticTypeOfArrayIsObjectArray()) { 5189 Label do_put; 5190 __ b(&do_put, EQ); 5191 // If heap poisoning is enabled, the `temp1` reference has 5192 // not been unpoisoned yet; unpoison it now. 5193 __ MaybeUnpoisonHeapReference(temp1); 5194 5195 // /* HeapReference<Class> */ temp1 = temp1->super_class_ 5196 __ LoadFromOffset(kLoadWord, temp1, temp1, super_offset); 5197 // If heap poisoning is enabled, no need to unpoison 5198 // `temp1`, as we are comparing against null below. 5199 __ CompareAndBranchIfNonZero(temp1, slow_path->GetEntryLabel()); 5200 __ Bind(&do_put); 5201 } else { 5202 __ b(slow_path->GetEntryLabel(), NE); 5203 } 5204 } 5205 5206 Register source = value; 5207 if (kPoisonHeapReferences) { 5208 // Note that in the case where `value` is a null reference, 5209 // we do not enter this block, as a null reference does not 5210 // need poisoning. 5211 DCHECK_EQ(value_type, Primitive::kPrimNot); 5212 __ Mov(temp1, value); 5213 __ PoisonHeapReference(temp1); 5214 source = temp1; 5215 } 5216 5217 if (index.IsConstant()) { 5218 size_t offset = 5219 (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4) + data_offset; 5220 __ StoreToOffset(kStoreWord, source, array, offset); 5221 } else { 5222 DCHECK(index.IsRegister()) << index; 5223 5224 __ add(IP, array, ShifterOperand(data_offset)); 5225 codegen_->StoreToShiftedRegOffset(value_type, 5226 Location::RegisterLocation(source), 5227 IP, 5228 index.AsRegister<Register>()); 5229 } 5230 5231 if (!may_need_runtime_call_for_type_check) { 5232 codegen_->MaybeRecordImplicitNullCheck(instruction); 5233 } 5234 5235 codegen_->MarkGCCard(temp1, temp2, array, value, instruction->GetValueCanBeNull()); 5236 5237 if (done.IsLinked()) { 5238 __ Bind(&done); 5239 } 5240 5241 if (slow_path != nullptr) { 5242 __ Bind(slow_path->GetExitLabel()); 5243 } 5244 5245 break; 5246 } 5247 5248 case Primitive::kPrimLong: { 5249 Location value = locations->InAt(2); 5250 if (index.IsConstant()) { 5251 size_t offset = 5252 (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_8) + data_offset; 5253 __ StoreToOffset(kStoreWordPair, value.AsRegisterPairLow<Register>(), array, offset); 5254 } else { 5255 __ add(IP, array, ShifterOperand(index.AsRegister<Register>(), LSL, TIMES_8)); 5256 __ StoreToOffset(kStoreWordPair, value.AsRegisterPairLow<Register>(), IP, data_offset); 5257 } 5258 break; 5259 } 5260 5261 case Primitive::kPrimFloat: { 5262 Location value = locations->InAt(2); 5263 DCHECK(value.IsFpuRegister()); 5264 if (index.IsConstant()) { 5265 size_t offset = (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4) + data_offset; 5266 __ StoreSToOffset(value.AsFpuRegister<SRegister>(), array, offset); 5267 } else { 5268 __ add(IP, array, ShifterOperand(index.AsRegister<Register>(), LSL, TIMES_4)); 5269 __ StoreSToOffset(value.AsFpuRegister<SRegister>(), IP, data_offset); 5270 } 5271 break; 5272 } 5273 5274 case Primitive::kPrimDouble: { 5275 Location value = locations->InAt(2); 5276 DCHECK(value.IsFpuRegisterPair()); 5277 if (index.IsConstant()) { 5278 size_t offset = (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_8) + data_offset; 5279 __ StoreDToOffset(FromLowSToD(value.AsFpuRegisterPairLow<SRegister>()), array, offset); 5280 } else { 5281 __ add(IP, array, ShifterOperand(index.AsRegister<Register>(), LSL, TIMES_8)); 5282 __ StoreDToOffset(FromLowSToD(value.AsFpuRegisterPairLow<SRegister>()), IP, data_offset); 5283 } 5284 5285 break; 5286 } 5287 5288 case Primitive::kPrimVoid: 5289 LOG(FATAL) << "Unreachable type " << value_type; 5290 UNREACHABLE(); 5291 } 5292 5293 // Objects are handled in the switch. 5294 if (value_type != Primitive::kPrimNot) { 5295 codegen_->MaybeRecordImplicitNullCheck(instruction); 5296 } 5297} 5298 5299void LocationsBuilderARM::VisitArrayLength(HArrayLength* instruction) { 5300 LocationSummary* locations = 5301 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall); 5302 locations->SetInAt(0, Location::RequiresRegister()); 5303 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 5304} 5305 5306void InstructionCodeGeneratorARM::VisitArrayLength(HArrayLength* instruction) { 5307 LocationSummary* locations = instruction->GetLocations(); 5308 uint32_t offset = CodeGenerator::GetArrayLengthOffset(instruction); 5309 Register obj = locations->InAt(0).AsRegister<Register>(); 5310 Register out = locations->Out().AsRegister<Register>(); 5311 __ LoadFromOffset(kLoadWord, out, obj, offset); 5312 codegen_->MaybeRecordImplicitNullCheck(instruction); 5313 // Mask out compression flag from String's array length. 5314 if (mirror::kUseStringCompression && instruction->IsStringLength()) { 5315 __ bic(out, out, ShifterOperand(1u << 31)); 5316 } 5317} 5318 5319void LocationsBuilderARM::VisitIntermediateAddress(HIntermediateAddress* instruction) { 5320 LocationSummary* locations = 5321 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall); 5322 5323 locations->SetInAt(0, Location::RequiresRegister()); 5324 locations->SetInAt(1, Location::RegisterOrConstant(instruction->GetOffset())); 5325 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 5326} 5327 5328void InstructionCodeGeneratorARM::VisitIntermediateAddress(HIntermediateAddress* instruction) { 5329 LocationSummary* locations = instruction->GetLocations(); 5330 Location out = locations->Out(); 5331 Location first = locations->InAt(0); 5332 Location second = locations->InAt(1); 5333 5334 if (second.IsRegister()) { 5335 __ add(out.AsRegister<Register>(), 5336 first.AsRegister<Register>(), 5337 ShifterOperand(second.AsRegister<Register>())); 5338 } else { 5339 __ AddConstant(out.AsRegister<Register>(), 5340 first.AsRegister<Register>(), 5341 second.GetConstant()->AsIntConstant()->GetValue()); 5342 } 5343} 5344 5345void LocationsBuilderARM::VisitBoundsCheck(HBoundsCheck* instruction) { 5346 RegisterSet caller_saves = RegisterSet::Empty(); 5347 InvokeRuntimeCallingConvention calling_convention; 5348 caller_saves.Add(Location::RegisterLocation(calling_convention.GetRegisterAt(0))); 5349 caller_saves.Add(Location::RegisterLocation(calling_convention.GetRegisterAt(1))); 5350 LocationSummary* locations = codegen_->CreateThrowingSlowPathLocations(instruction, caller_saves); 5351 locations->SetInAt(0, Location::RequiresRegister()); 5352 locations->SetInAt(1, Location::RequiresRegister()); 5353} 5354 5355void InstructionCodeGeneratorARM::VisitBoundsCheck(HBoundsCheck* instruction) { 5356 LocationSummary* locations = instruction->GetLocations(); 5357 SlowPathCodeARM* slow_path = 5358 new (GetGraph()->GetArena()) BoundsCheckSlowPathARM(instruction); 5359 codegen_->AddSlowPath(slow_path); 5360 5361 Register index = locations->InAt(0).AsRegister<Register>(); 5362 Register length = locations->InAt(1).AsRegister<Register>(); 5363 5364 __ cmp(index, ShifterOperand(length)); 5365 __ b(slow_path->GetEntryLabel(), HS); 5366} 5367 5368void CodeGeneratorARM::MarkGCCard(Register temp, 5369 Register card, 5370 Register object, 5371 Register value, 5372 bool can_be_null) { 5373 Label is_null; 5374 if (can_be_null) { 5375 __ CompareAndBranchIfZero(value, &is_null); 5376 } 5377 __ LoadFromOffset(kLoadWord, card, TR, Thread::CardTableOffset<kArmPointerSize>().Int32Value()); 5378 __ Lsr(temp, object, gc::accounting::CardTable::kCardShift); 5379 __ strb(card, Address(card, temp)); 5380 if (can_be_null) { 5381 __ Bind(&is_null); 5382 } 5383} 5384 5385void LocationsBuilderARM::VisitParallelMove(HParallelMove* instruction ATTRIBUTE_UNUSED) { 5386 LOG(FATAL) << "Unreachable"; 5387} 5388 5389void InstructionCodeGeneratorARM::VisitParallelMove(HParallelMove* instruction) { 5390 codegen_->GetMoveResolver()->EmitNativeCode(instruction); 5391} 5392 5393void LocationsBuilderARM::VisitSuspendCheck(HSuspendCheck* instruction) { 5394 LocationSummary* locations = 5395 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCallOnSlowPath); 5396 locations->SetCustomSlowPathCallerSaves(RegisterSet::Empty()); // No caller-save registers. 5397} 5398 5399void InstructionCodeGeneratorARM::VisitSuspendCheck(HSuspendCheck* instruction) { 5400 HBasicBlock* block = instruction->GetBlock(); 5401 if (block->GetLoopInformation() != nullptr) { 5402 DCHECK(block->GetLoopInformation()->GetSuspendCheck() == instruction); 5403 // The back edge will generate the suspend check. 5404 return; 5405 } 5406 if (block->IsEntryBlock() && instruction->GetNext()->IsGoto()) { 5407 // The goto will generate the suspend check. 5408 return; 5409 } 5410 GenerateSuspendCheck(instruction, nullptr); 5411} 5412 5413void InstructionCodeGeneratorARM::GenerateSuspendCheck(HSuspendCheck* instruction, 5414 HBasicBlock* successor) { 5415 SuspendCheckSlowPathARM* slow_path = 5416 down_cast<SuspendCheckSlowPathARM*>(instruction->GetSlowPath()); 5417 if (slow_path == nullptr) { 5418 slow_path = new (GetGraph()->GetArena()) SuspendCheckSlowPathARM(instruction, successor); 5419 instruction->SetSlowPath(slow_path); 5420 codegen_->AddSlowPath(slow_path); 5421 if (successor != nullptr) { 5422 DCHECK(successor->IsLoopHeader()); 5423 codegen_->ClearSpillSlotsFromLoopPhisInStackMap(instruction); 5424 } 5425 } else { 5426 DCHECK_EQ(slow_path->GetSuccessor(), successor); 5427 } 5428 5429 __ LoadFromOffset( 5430 kLoadUnsignedHalfword, IP, TR, Thread::ThreadFlagsOffset<kArmPointerSize>().Int32Value()); 5431 if (successor == nullptr) { 5432 __ CompareAndBranchIfNonZero(IP, slow_path->GetEntryLabel()); 5433 __ Bind(slow_path->GetReturnLabel()); 5434 } else { 5435 __ CompareAndBranchIfZero(IP, codegen_->GetLabelOf(successor)); 5436 __ b(slow_path->GetEntryLabel()); 5437 } 5438} 5439 5440ArmAssembler* ParallelMoveResolverARM::GetAssembler() const { 5441 return codegen_->GetAssembler(); 5442} 5443 5444void ParallelMoveResolverARM::EmitMove(size_t index) { 5445 MoveOperands* move = moves_[index]; 5446 Location source = move->GetSource(); 5447 Location destination = move->GetDestination(); 5448 5449 if (source.IsRegister()) { 5450 if (destination.IsRegister()) { 5451 __ Mov(destination.AsRegister<Register>(), source.AsRegister<Register>()); 5452 } else if (destination.IsFpuRegister()) { 5453 __ vmovsr(destination.AsFpuRegister<SRegister>(), source.AsRegister<Register>()); 5454 } else { 5455 DCHECK(destination.IsStackSlot()); 5456 __ StoreToOffset(kStoreWord, source.AsRegister<Register>(), 5457 SP, destination.GetStackIndex()); 5458 } 5459 } else if (source.IsStackSlot()) { 5460 if (destination.IsRegister()) { 5461 __ LoadFromOffset(kLoadWord, destination.AsRegister<Register>(), 5462 SP, source.GetStackIndex()); 5463 } else if (destination.IsFpuRegister()) { 5464 __ LoadSFromOffset(destination.AsFpuRegister<SRegister>(), SP, source.GetStackIndex()); 5465 } else { 5466 DCHECK(destination.IsStackSlot()); 5467 __ LoadFromOffset(kLoadWord, IP, SP, source.GetStackIndex()); 5468 __ StoreToOffset(kStoreWord, IP, SP, destination.GetStackIndex()); 5469 } 5470 } else if (source.IsFpuRegister()) { 5471 if (destination.IsRegister()) { 5472 __ vmovrs(destination.AsRegister<Register>(), source.AsFpuRegister<SRegister>()); 5473 } else if (destination.IsFpuRegister()) { 5474 __ vmovs(destination.AsFpuRegister<SRegister>(), source.AsFpuRegister<SRegister>()); 5475 } else { 5476 DCHECK(destination.IsStackSlot()); 5477 __ StoreSToOffset(source.AsFpuRegister<SRegister>(), SP, destination.GetStackIndex()); 5478 } 5479 } else if (source.IsDoubleStackSlot()) { 5480 if (destination.IsDoubleStackSlot()) { 5481 __ LoadDFromOffset(DTMP, SP, source.GetStackIndex()); 5482 __ StoreDToOffset(DTMP, SP, destination.GetStackIndex()); 5483 } else if (destination.IsRegisterPair()) { 5484 DCHECK(ExpectedPairLayout(destination)); 5485 __ LoadFromOffset( 5486 kLoadWordPair, destination.AsRegisterPairLow<Register>(), SP, source.GetStackIndex()); 5487 } else { 5488 DCHECK(destination.IsFpuRegisterPair()) << destination; 5489 __ LoadDFromOffset(FromLowSToD(destination.AsFpuRegisterPairLow<SRegister>()), 5490 SP, 5491 source.GetStackIndex()); 5492 } 5493 } else if (source.IsRegisterPair()) { 5494 if (destination.IsRegisterPair()) { 5495 __ Mov(destination.AsRegisterPairLow<Register>(), source.AsRegisterPairLow<Register>()); 5496 __ Mov(destination.AsRegisterPairHigh<Register>(), source.AsRegisterPairHigh<Register>()); 5497 } else if (destination.IsFpuRegisterPair()) { 5498 __ vmovdrr(FromLowSToD(destination.AsFpuRegisterPairLow<SRegister>()), 5499 source.AsRegisterPairLow<Register>(), 5500 source.AsRegisterPairHigh<Register>()); 5501 } else { 5502 DCHECK(destination.IsDoubleStackSlot()) << destination; 5503 DCHECK(ExpectedPairLayout(source)); 5504 __ StoreToOffset( 5505 kStoreWordPair, source.AsRegisterPairLow<Register>(), SP, destination.GetStackIndex()); 5506 } 5507 } else if (source.IsFpuRegisterPair()) { 5508 if (destination.IsRegisterPair()) { 5509 __ vmovrrd(destination.AsRegisterPairLow<Register>(), 5510 destination.AsRegisterPairHigh<Register>(), 5511 FromLowSToD(source.AsFpuRegisterPairLow<SRegister>())); 5512 } else if (destination.IsFpuRegisterPair()) { 5513 __ vmovd(FromLowSToD(destination.AsFpuRegisterPairLow<SRegister>()), 5514 FromLowSToD(source.AsFpuRegisterPairLow<SRegister>())); 5515 } else { 5516 DCHECK(destination.IsDoubleStackSlot()) << destination; 5517 __ StoreDToOffset(FromLowSToD(source.AsFpuRegisterPairLow<SRegister>()), 5518 SP, 5519 destination.GetStackIndex()); 5520 } 5521 } else { 5522 DCHECK(source.IsConstant()) << source; 5523 HConstant* constant = source.GetConstant(); 5524 if (constant->IsIntConstant() || constant->IsNullConstant()) { 5525 int32_t value = CodeGenerator::GetInt32ValueOf(constant); 5526 if (destination.IsRegister()) { 5527 __ LoadImmediate(destination.AsRegister<Register>(), value); 5528 } else { 5529 DCHECK(destination.IsStackSlot()); 5530 __ LoadImmediate(IP, value); 5531 __ StoreToOffset(kStoreWord, IP, SP, destination.GetStackIndex()); 5532 } 5533 } else if (constant->IsLongConstant()) { 5534 int64_t value = constant->AsLongConstant()->GetValue(); 5535 if (destination.IsRegisterPair()) { 5536 __ LoadImmediate(destination.AsRegisterPairLow<Register>(), Low32Bits(value)); 5537 __ LoadImmediate(destination.AsRegisterPairHigh<Register>(), High32Bits(value)); 5538 } else { 5539 DCHECK(destination.IsDoubleStackSlot()) << destination; 5540 __ LoadImmediate(IP, Low32Bits(value)); 5541 __ StoreToOffset(kStoreWord, IP, SP, destination.GetStackIndex()); 5542 __ LoadImmediate(IP, High32Bits(value)); 5543 __ StoreToOffset(kStoreWord, IP, SP, destination.GetHighStackIndex(kArmWordSize)); 5544 } 5545 } else if (constant->IsDoubleConstant()) { 5546 double value = constant->AsDoubleConstant()->GetValue(); 5547 if (destination.IsFpuRegisterPair()) { 5548 __ LoadDImmediate(FromLowSToD(destination.AsFpuRegisterPairLow<SRegister>()), value); 5549 } else { 5550 DCHECK(destination.IsDoubleStackSlot()) << destination; 5551 uint64_t int_value = bit_cast<uint64_t, double>(value); 5552 __ LoadImmediate(IP, Low32Bits(int_value)); 5553 __ StoreToOffset(kStoreWord, IP, SP, destination.GetStackIndex()); 5554 __ LoadImmediate(IP, High32Bits(int_value)); 5555 __ StoreToOffset(kStoreWord, IP, SP, destination.GetHighStackIndex(kArmWordSize)); 5556 } 5557 } else { 5558 DCHECK(constant->IsFloatConstant()) << constant->DebugName(); 5559 float value = constant->AsFloatConstant()->GetValue(); 5560 if (destination.IsFpuRegister()) { 5561 __ LoadSImmediate(destination.AsFpuRegister<SRegister>(), value); 5562 } else { 5563 DCHECK(destination.IsStackSlot()); 5564 __ LoadImmediate(IP, bit_cast<int32_t, float>(value)); 5565 __ StoreToOffset(kStoreWord, IP, SP, destination.GetStackIndex()); 5566 } 5567 } 5568 } 5569} 5570 5571void ParallelMoveResolverARM::Exchange(Register reg, int mem) { 5572 __ Mov(IP, reg); 5573 __ LoadFromOffset(kLoadWord, reg, SP, mem); 5574 __ StoreToOffset(kStoreWord, IP, SP, mem); 5575} 5576 5577void ParallelMoveResolverARM::Exchange(int mem1, int mem2) { 5578 ScratchRegisterScope ensure_scratch(this, IP, R0, codegen_->GetNumberOfCoreRegisters()); 5579 int stack_offset = ensure_scratch.IsSpilled() ? kArmWordSize : 0; 5580 __ LoadFromOffset(kLoadWord, static_cast<Register>(ensure_scratch.GetRegister()), 5581 SP, mem1 + stack_offset); 5582 __ LoadFromOffset(kLoadWord, IP, SP, mem2 + stack_offset); 5583 __ StoreToOffset(kStoreWord, static_cast<Register>(ensure_scratch.GetRegister()), 5584 SP, mem2 + stack_offset); 5585 __ StoreToOffset(kStoreWord, IP, SP, mem1 + stack_offset); 5586} 5587 5588void ParallelMoveResolverARM::EmitSwap(size_t index) { 5589 MoveOperands* move = moves_[index]; 5590 Location source = move->GetSource(); 5591 Location destination = move->GetDestination(); 5592 5593 if (source.IsRegister() && destination.IsRegister()) { 5594 DCHECK_NE(source.AsRegister<Register>(), IP); 5595 DCHECK_NE(destination.AsRegister<Register>(), IP); 5596 __ Mov(IP, source.AsRegister<Register>()); 5597 __ Mov(source.AsRegister<Register>(), destination.AsRegister<Register>()); 5598 __ Mov(destination.AsRegister<Register>(), IP); 5599 } else if (source.IsRegister() && destination.IsStackSlot()) { 5600 Exchange(source.AsRegister<Register>(), destination.GetStackIndex()); 5601 } else if (source.IsStackSlot() && destination.IsRegister()) { 5602 Exchange(destination.AsRegister<Register>(), source.GetStackIndex()); 5603 } else if (source.IsStackSlot() && destination.IsStackSlot()) { 5604 Exchange(source.GetStackIndex(), destination.GetStackIndex()); 5605 } else if (source.IsFpuRegister() && destination.IsFpuRegister()) { 5606 __ vmovrs(IP, source.AsFpuRegister<SRegister>()); 5607 __ vmovs(source.AsFpuRegister<SRegister>(), destination.AsFpuRegister<SRegister>()); 5608 __ vmovsr(destination.AsFpuRegister<SRegister>(), IP); 5609 } else if (source.IsRegisterPair() && destination.IsRegisterPair()) { 5610 __ vmovdrr(DTMP, source.AsRegisterPairLow<Register>(), source.AsRegisterPairHigh<Register>()); 5611 __ Mov(source.AsRegisterPairLow<Register>(), destination.AsRegisterPairLow<Register>()); 5612 __ Mov(source.AsRegisterPairHigh<Register>(), destination.AsRegisterPairHigh<Register>()); 5613 __ vmovrrd(destination.AsRegisterPairLow<Register>(), 5614 destination.AsRegisterPairHigh<Register>(), 5615 DTMP); 5616 } else if (source.IsRegisterPair() || destination.IsRegisterPair()) { 5617 Register low_reg = source.IsRegisterPair() 5618 ? source.AsRegisterPairLow<Register>() 5619 : destination.AsRegisterPairLow<Register>(); 5620 int mem = source.IsRegisterPair() 5621 ? destination.GetStackIndex() 5622 : source.GetStackIndex(); 5623 DCHECK(ExpectedPairLayout(source.IsRegisterPair() ? source : destination)); 5624 __ vmovdrr(DTMP, low_reg, static_cast<Register>(low_reg + 1)); 5625 __ LoadFromOffset(kLoadWordPair, low_reg, SP, mem); 5626 __ StoreDToOffset(DTMP, SP, mem); 5627 } else if (source.IsFpuRegisterPair() && destination.IsFpuRegisterPair()) { 5628 DRegister first = FromLowSToD(source.AsFpuRegisterPairLow<SRegister>()); 5629 DRegister second = FromLowSToD(destination.AsFpuRegisterPairLow<SRegister>()); 5630 __ vmovd(DTMP, first); 5631 __ vmovd(first, second); 5632 __ vmovd(second, DTMP); 5633 } else if (source.IsFpuRegisterPair() || destination.IsFpuRegisterPair()) { 5634 DRegister reg = source.IsFpuRegisterPair() 5635 ? FromLowSToD(source.AsFpuRegisterPairLow<SRegister>()) 5636 : FromLowSToD(destination.AsFpuRegisterPairLow<SRegister>()); 5637 int mem = source.IsFpuRegisterPair() 5638 ? destination.GetStackIndex() 5639 : source.GetStackIndex(); 5640 __ vmovd(DTMP, reg); 5641 __ LoadDFromOffset(reg, SP, mem); 5642 __ StoreDToOffset(DTMP, SP, mem); 5643 } else if (source.IsFpuRegister() || destination.IsFpuRegister()) { 5644 SRegister reg = source.IsFpuRegister() ? source.AsFpuRegister<SRegister>() 5645 : destination.AsFpuRegister<SRegister>(); 5646 int mem = source.IsFpuRegister() 5647 ? destination.GetStackIndex() 5648 : source.GetStackIndex(); 5649 5650 __ vmovrs(IP, reg); 5651 __ LoadSFromOffset(reg, SP, mem); 5652 __ StoreToOffset(kStoreWord, IP, SP, mem); 5653 } else if (source.IsDoubleStackSlot() && destination.IsDoubleStackSlot()) { 5654 Exchange(source.GetStackIndex(), destination.GetStackIndex()); 5655 Exchange(source.GetHighStackIndex(kArmWordSize), destination.GetHighStackIndex(kArmWordSize)); 5656 } else { 5657 LOG(FATAL) << "Unimplemented" << source << " <-> " << destination; 5658 } 5659} 5660 5661void ParallelMoveResolverARM::SpillScratch(int reg) { 5662 __ Push(static_cast<Register>(reg)); 5663} 5664 5665void ParallelMoveResolverARM::RestoreScratch(int reg) { 5666 __ Pop(static_cast<Register>(reg)); 5667} 5668 5669HLoadClass::LoadKind CodeGeneratorARM::GetSupportedLoadClassKind( 5670 HLoadClass::LoadKind desired_class_load_kind) { 5671 switch (desired_class_load_kind) { 5672 case HLoadClass::LoadKind::kReferrersClass: 5673 break; 5674 case HLoadClass::LoadKind::kBootImageLinkTimeAddress: 5675 DCHECK(!GetCompilerOptions().GetCompilePic()); 5676 break; 5677 case HLoadClass::LoadKind::kBootImageLinkTimePcRelative: 5678 DCHECK(GetCompilerOptions().GetCompilePic()); 5679 break; 5680 case HLoadClass::LoadKind::kBootImageAddress: 5681 break; 5682 case HLoadClass::LoadKind::kDexCacheAddress: 5683 DCHECK(Runtime::Current()->UseJitCompilation()); 5684 break; 5685 case HLoadClass::LoadKind::kDexCachePcRelative: 5686 DCHECK(!Runtime::Current()->UseJitCompilation()); 5687 // We disable pc-relative load when there is an irreducible loop, as the optimization 5688 // is incompatible with it. 5689 // TODO: Create as many ArmDexCacheArraysBase instructions as needed for methods 5690 // with irreducible loops. 5691 if (GetGraph()->HasIrreducibleLoops()) { 5692 return HLoadClass::LoadKind::kDexCacheViaMethod; 5693 } 5694 break; 5695 case HLoadClass::LoadKind::kDexCacheViaMethod: 5696 break; 5697 } 5698 return desired_class_load_kind; 5699} 5700 5701void LocationsBuilderARM::VisitLoadClass(HLoadClass* cls) { 5702 if (cls->NeedsAccessCheck()) { 5703 InvokeRuntimeCallingConvention calling_convention; 5704 CodeGenerator::CreateLoadClassLocationSummary( 5705 cls, 5706 Location::RegisterLocation(calling_convention.GetRegisterAt(0)), 5707 Location::RegisterLocation(R0), 5708 /* code_generator_supports_read_barrier */ true); 5709 return; 5710 } 5711 5712 const bool requires_read_barrier = kEmitCompilerReadBarrier && !cls->IsInBootImage(); 5713 LocationSummary::CallKind call_kind = (cls->NeedsEnvironment() || requires_read_barrier) 5714 ? LocationSummary::kCallOnSlowPath 5715 : LocationSummary::kNoCall; 5716 LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(cls, call_kind); 5717 if (kUseBakerReadBarrier && requires_read_barrier && !cls->NeedsEnvironment()) { 5718 locations->SetCustomSlowPathCallerSaves(RegisterSet::Empty()); // No caller-save registers. 5719 } 5720 5721 HLoadClass::LoadKind load_kind = cls->GetLoadKind(); 5722 if (load_kind == HLoadClass::LoadKind::kReferrersClass || 5723 load_kind == HLoadClass::LoadKind::kDexCacheViaMethod || 5724 load_kind == HLoadClass::LoadKind::kDexCachePcRelative) { 5725 locations->SetInAt(0, Location::RequiresRegister()); 5726 } 5727 locations->SetOut(Location::RequiresRegister()); 5728} 5729 5730void InstructionCodeGeneratorARM::VisitLoadClass(HLoadClass* cls) { 5731 LocationSummary* locations = cls->GetLocations(); 5732 if (cls->NeedsAccessCheck()) { 5733 codegen_->MoveConstant(locations->GetTemp(0), cls->GetTypeIndex()); 5734 codegen_->InvokeRuntime(kQuickInitializeTypeAndVerifyAccess, cls, cls->GetDexPc()); 5735 CheckEntrypointTypes<kQuickInitializeTypeAndVerifyAccess, void*, uint32_t>(); 5736 return; 5737 } 5738 5739 Location out_loc = locations->Out(); 5740 Register out = out_loc.AsRegister<Register>(); 5741 5742 const bool requires_read_barrier = kEmitCompilerReadBarrier && !cls->IsInBootImage(); 5743 bool generate_null_check = false; 5744 switch (cls->GetLoadKind()) { 5745 case HLoadClass::LoadKind::kReferrersClass: { 5746 DCHECK(!cls->CanCallRuntime()); 5747 DCHECK(!cls->MustGenerateClinitCheck()); 5748 // /* GcRoot<mirror::Class> */ out = current_method->declaring_class_ 5749 Register current_method = locations->InAt(0).AsRegister<Register>(); 5750 GenerateGcRootFieldLoad(cls, 5751 out_loc, 5752 current_method, 5753 ArtMethod::DeclaringClassOffset().Int32Value(), 5754 requires_read_barrier); 5755 break; 5756 } 5757 case HLoadClass::LoadKind::kBootImageLinkTimeAddress: { 5758 DCHECK(!requires_read_barrier); 5759 __ LoadLiteral(out, codegen_->DeduplicateBootImageTypeLiteral(cls->GetDexFile(), 5760 cls->GetTypeIndex())); 5761 break; 5762 } 5763 case HLoadClass::LoadKind::kBootImageLinkTimePcRelative: { 5764 DCHECK(!requires_read_barrier); 5765 CodeGeneratorARM::PcRelativePatchInfo* labels = 5766 codegen_->NewPcRelativeTypePatch(cls->GetDexFile(), cls->GetTypeIndex()); 5767 __ BindTrackedLabel(&labels->movw_label); 5768 __ movw(out, /* placeholder */ 0u); 5769 __ BindTrackedLabel(&labels->movt_label); 5770 __ movt(out, /* placeholder */ 0u); 5771 __ BindTrackedLabel(&labels->add_pc_label); 5772 __ add(out, out, ShifterOperand(PC)); 5773 break; 5774 } 5775 case HLoadClass::LoadKind::kBootImageAddress: { 5776 DCHECK(!requires_read_barrier); 5777 DCHECK_NE(cls->GetAddress(), 0u); 5778 uint32_t address = dchecked_integral_cast<uint32_t>(cls->GetAddress()); 5779 __ LoadLiteral(out, codegen_->DeduplicateBootImageAddressLiteral(address)); 5780 break; 5781 } 5782 case HLoadClass::LoadKind::kDexCacheAddress: { 5783 DCHECK_NE(cls->GetAddress(), 0u); 5784 uint32_t address = dchecked_integral_cast<uint32_t>(cls->GetAddress()); 5785 // 16-bit LDR immediate has a 5-bit offset multiplied by the size and that gives 5786 // a 128B range. To try and reduce the number of literals if we load multiple types, 5787 // simply split the dex cache address to a 128B aligned base loaded from a literal 5788 // and the remaining offset embedded in the load. 5789 static_assert(sizeof(GcRoot<mirror::Class>) == 4u, "Expected GC root to be 4 bytes."); 5790 DCHECK_ALIGNED(cls->GetAddress(), 4u); 5791 constexpr size_t offset_bits = /* encoded bits */ 5 + /* scale */ 2; 5792 uint32_t base_address = address & ~MaxInt<uint32_t>(offset_bits); 5793 uint32_t offset = address & MaxInt<uint32_t>(offset_bits); 5794 __ LoadLiteral(out, codegen_->DeduplicateDexCacheAddressLiteral(base_address)); 5795 // /* GcRoot<mirror::Class> */ out = *(base_address + offset) 5796 GenerateGcRootFieldLoad(cls, out_loc, out, offset, requires_read_barrier); 5797 generate_null_check = !cls->IsInDexCache(); 5798 break; 5799 } 5800 case HLoadClass::LoadKind::kDexCachePcRelative: { 5801 Register base_reg = locations->InAt(0).AsRegister<Register>(); 5802 HArmDexCacheArraysBase* base = cls->InputAt(0)->AsArmDexCacheArraysBase(); 5803 int32_t offset = cls->GetDexCacheElementOffset() - base->GetElementOffset(); 5804 // /* GcRoot<mirror::Class> */ out = *(dex_cache_arrays_base + offset) 5805 GenerateGcRootFieldLoad(cls, out_loc, base_reg, offset, requires_read_barrier); 5806 generate_null_check = !cls->IsInDexCache(); 5807 break; 5808 } 5809 case HLoadClass::LoadKind::kDexCacheViaMethod: { 5810 // /* GcRoot<mirror::Class>[] */ out = 5811 // current_method.ptr_sized_fields_->dex_cache_resolved_types_ 5812 Register current_method = locations->InAt(0).AsRegister<Register>(); 5813 __ LoadFromOffset(kLoadWord, 5814 out, 5815 current_method, 5816 ArtMethod::DexCacheResolvedTypesOffset(kArmPointerSize).Int32Value()); 5817 // /* GcRoot<mirror::Class> */ out = out[type_index] 5818 size_t offset = CodeGenerator::GetCacheOffset(cls->GetTypeIndex()); 5819 GenerateGcRootFieldLoad(cls, out_loc, out, offset, requires_read_barrier); 5820 generate_null_check = !cls->IsInDexCache(); 5821 } 5822 } 5823 5824 if (generate_null_check || cls->MustGenerateClinitCheck()) { 5825 DCHECK(cls->CanCallRuntime()); 5826 SlowPathCodeARM* slow_path = new (GetGraph()->GetArena()) LoadClassSlowPathARM( 5827 cls, cls, cls->GetDexPc(), cls->MustGenerateClinitCheck()); 5828 codegen_->AddSlowPath(slow_path); 5829 if (generate_null_check) { 5830 __ CompareAndBranchIfZero(out, slow_path->GetEntryLabel()); 5831 } 5832 if (cls->MustGenerateClinitCheck()) { 5833 GenerateClassInitializationCheck(slow_path, out); 5834 } else { 5835 __ Bind(slow_path->GetExitLabel()); 5836 } 5837 } 5838} 5839 5840void LocationsBuilderARM::VisitClinitCheck(HClinitCheck* check) { 5841 LocationSummary* locations = 5842 new (GetGraph()->GetArena()) LocationSummary(check, LocationSummary::kCallOnSlowPath); 5843 locations->SetInAt(0, Location::RequiresRegister()); 5844 if (check->HasUses()) { 5845 locations->SetOut(Location::SameAsFirstInput()); 5846 } 5847} 5848 5849void InstructionCodeGeneratorARM::VisitClinitCheck(HClinitCheck* check) { 5850 // We assume the class is not null. 5851 SlowPathCodeARM* slow_path = new (GetGraph()->GetArena()) LoadClassSlowPathARM( 5852 check->GetLoadClass(), check, check->GetDexPc(), true); 5853 codegen_->AddSlowPath(slow_path); 5854 GenerateClassInitializationCheck(slow_path, 5855 check->GetLocations()->InAt(0).AsRegister<Register>()); 5856} 5857 5858void InstructionCodeGeneratorARM::GenerateClassInitializationCheck( 5859 SlowPathCodeARM* slow_path, Register class_reg) { 5860 __ LoadFromOffset(kLoadWord, IP, class_reg, mirror::Class::StatusOffset().Int32Value()); 5861 __ cmp(IP, ShifterOperand(mirror::Class::kStatusInitialized)); 5862 __ b(slow_path->GetEntryLabel(), LT); 5863 // Even if the initialized flag is set, we may be in a situation where caches are not synced 5864 // properly. Therefore, we do a memory fence. 5865 __ dmb(ISH); 5866 __ Bind(slow_path->GetExitLabel()); 5867} 5868 5869HLoadString::LoadKind CodeGeneratorARM::GetSupportedLoadStringKind( 5870 HLoadString::LoadKind desired_string_load_kind) { 5871 switch (desired_string_load_kind) { 5872 case HLoadString::LoadKind::kBootImageLinkTimeAddress: 5873 DCHECK(!GetCompilerOptions().GetCompilePic()); 5874 break; 5875 case HLoadString::LoadKind::kBootImageLinkTimePcRelative: 5876 DCHECK(GetCompilerOptions().GetCompilePic()); 5877 break; 5878 case HLoadString::LoadKind::kBootImageAddress: 5879 break; 5880 case HLoadString::LoadKind::kDexCacheAddress: 5881 DCHECK(Runtime::Current()->UseJitCompilation()); 5882 break; 5883 case HLoadString::LoadKind::kBssEntry: 5884 DCHECK(!Runtime::Current()->UseJitCompilation()); 5885 break; 5886 case HLoadString::LoadKind::kDexCacheViaMethod: 5887 break; 5888 } 5889 return desired_string_load_kind; 5890} 5891 5892void LocationsBuilderARM::VisitLoadString(HLoadString* load) { 5893 LocationSummary::CallKind call_kind = load->NeedsEnvironment() 5894 ? ((load->GetLoadKind() == HLoadString::LoadKind::kDexCacheViaMethod) 5895 ? LocationSummary::kCallOnMainOnly 5896 : LocationSummary::kCallOnSlowPath) 5897 : LocationSummary::kNoCall; 5898 LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(load, call_kind); 5899 5900 HLoadString::LoadKind load_kind = load->GetLoadKind(); 5901 if (load_kind == HLoadString::LoadKind::kDexCacheViaMethod) { 5902 locations->SetOut(Location::RegisterLocation(R0)); 5903 } else { 5904 locations->SetOut(Location::RequiresRegister()); 5905 if (load_kind == HLoadString::LoadKind::kBssEntry) { 5906 if (!kUseReadBarrier || kUseBakerReadBarrier) { 5907 // Rely on the pResolveString and/or marking to save everything, including temps. 5908 // Note that IP may theoretically be clobbered by saving/restoring the live register 5909 // (only one thanks to the custom calling convention), so we request a different temp. 5910 locations->AddTemp(Location::RequiresRegister()); 5911 RegisterSet caller_saves = RegisterSet::Empty(); 5912 InvokeRuntimeCallingConvention calling_convention; 5913 caller_saves.Add(Location::RegisterLocation(calling_convention.GetRegisterAt(0))); 5914 // TODO: Add GetReturnLocation() to the calling convention so that we can DCHECK() 5915 // that the the kPrimNot result register is the same as the first argument register. 5916 locations->SetCustomSlowPathCallerSaves(caller_saves); 5917 } else { 5918 // For non-Baker read barrier we have a temp-clobbering call. 5919 } 5920 } 5921 } 5922} 5923 5924void InstructionCodeGeneratorARM::VisitLoadString(HLoadString* load) { 5925 LocationSummary* locations = load->GetLocations(); 5926 Location out_loc = locations->Out(); 5927 Register out = out_loc.AsRegister<Register>(); 5928 HLoadString::LoadKind load_kind = load->GetLoadKind(); 5929 5930 switch (load_kind) { 5931 case HLoadString::LoadKind::kBootImageLinkTimeAddress: { 5932 __ LoadLiteral(out, codegen_->DeduplicateBootImageStringLiteral(load->GetDexFile(), 5933 load->GetStringIndex())); 5934 return; // No dex cache slow path. 5935 } 5936 case HLoadString::LoadKind::kBootImageLinkTimePcRelative: { 5937 DCHECK(codegen_->GetCompilerOptions().IsBootImage()); 5938 CodeGeneratorARM::PcRelativePatchInfo* labels = 5939 codegen_->NewPcRelativeStringPatch(load->GetDexFile(), load->GetStringIndex()); 5940 __ BindTrackedLabel(&labels->movw_label); 5941 __ movw(out, /* placeholder */ 0u); 5942 __ BindTrackedLabel(&labels->movt_label); 5943 __ movt(out, /* placeholder */ 0u); 5944 __ BindTrackedLabel(&labels->add_pc_label); 5945 __ add(out, out, ShifterOperand(PC)); 5946 return; // No dex cache slow path. 5947 } 5948 case HLoadString::LoadKind::kBootImageAddress: { 5949 DCHECK_NE(load->GetAddress(), 0u); 5950 uint32_t address = dchecked_integral_cast<uint32_t>(load->GetAddress()); 5951 __ LoadLiteral(out, codegen_->DeduplicateBootImageAddressLiteral(address)); 5952 return; // No dex cache slow path. 5953 } 5954 case HLoadString::LoadKind::kBssEntry: { 5955 DCHECK(!codegen_->GetCompilerOptions().IsBootImage()); 5956 Register temp = locations->GetTemp(0).AsRegister<Register>(); 5957 CodeGeneratorARM::PcRelativePatchInfo* labels = 5958 codegen_->NewPcRelativeStringPatch(load->GetDexFile(), load->GetStringIndex()); 5959 __ BindTrackedLabel(&labels->movw_label); 5960 __ movw(temp, /* placeholder */ 0u); 5961 __ BindTrackedLabel(&labels->movt_label); 5962 __ movt(temp, /* placeholder */ 0u); 5963 __ BindTrackedLabel(&labels->add_pc_label); 5964 __ add(temp, temp, ShifterOperand(PC)); 5965 GenerateGcRootFieldLoad(load, out_loc, temp, /* offset */ 0, kEmitCompilerReadBarrier); 5966 SlowPathCode* slow_path = new (GetGraph()->GetArena()) LoadStringSlowPathARM(load); 5967 codegen_->AddSlowPath(slow_path); 5968 __ CompareAndBranchIfZero(out, slow_path->GetEntryLabel()); 5969 __ Bind(slow_path->GetExitLabel()); 5970 return; 5971 } 5972 default: 5973 break; 5974 } 5975 5976 // TODO: Consider re-adding the compiler code to do string dex cache lookup again. 5977 DCHECK(load_kind == HLoadString::LoadKind::kDexCacheViaMethod); 5978 InvokeRuntimeCallingConvention calling_convention; 5979 DCHECK_EQ(calling_convention.GetRegisterAt(0), out); 5980 __ LoadImmediate(calling_convention.GetRegisterAt(0), load->GetStringIndex()); 5981 codegen_->InvokeRuntime(kQuickResolveString, load, load->GetDexPc()); 5982 CheckEntrypointTypes<kQuickResolveString, void*, uint32_t>(); 5983} 5984 5985static int32_t GetExceptionTlsOffset() { 5986 return Thread::ExceptionOffset<kArmPointerSize>().Int32Value(); 5987} 5988 5989void LocationsBuilderARM::VisitLoadException(HLoadException* load) { 5990 LocationSummary* locations = 5991 new (GetGraph()->GetArena()) LocationSummary(load, LocationSummary::kNoCall); 5992 locations->SetOut(Location::RequiresRegister()); 5993} 5994 5995void InstructionCodeGeneratorARM::VisitLoadException(HLoadException* load) { 5996 Register out = load->GetLocations()->Out().AsRegister<Register>(); 5997 __ LoadFromOffset(kLoadWord, out, TR, GetExceptionTlsOffset()); 5998} 5999 6000void LocationsBuilderARM::VisitClearException(HClearException* clear) { 6001 new (GetGraph()->GetArena()) LocationSummary(clear, LocationSummary::kNoCall); 6002} 6003 6004void InstructionCodeGeneratorARM::VisitClearException(HClearException* clear ATTRIBUTE_UNUSED) { 6005 __ LoadImmediate(IP, 0); 6006 __ StoreToOffset(kStoreWord, IP, TR, GetExceptionTlsOffset()); 6007} 6008 6009void LocationsBuilderARM::VisitThrow(HThrow* instruction) { 6010 LocationSummary* locations = 6011 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCallOnMainOnly); 6012 InvokeRuntimeCallingConvention calling_convention; 6013 locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(0))); 6014} 6015 6016void InstructionCodeGeneratorARM::VisitThrow(HThrow* instruction) { 6017 codegen_->InvokeRuntime(kQuickDeliverException, instruction, instruction->GetDexPc()); 6018 CheckEntrypointTypes<kQuickDeliverException, void, mirror::Object*>(); 6019} 6020 6021static bool TypeCheckNeedsATemporary(TypeCheckKind type_check_kind) { 6022 return kEmitCompilerReadBarrier && 6023 (kUseBakerReadBarrier || 6024 type_check_kind == TypeCheckKind::kAbstractClassCheck || 6025 type_check_kind == TypeCheckKind::kClassHierarchyCheck || 6026 type_check_kind == TypeCheckKind::kArrayObjectCheck); 6027} 6028 6029void LocationsBuilderARM::VisitInstanceOf(HInstanceOf* instruction) { 6030 LocationSummary::CallKind call_kind = LocationSummary::kNoCall; 6031 TypeCheckKind type_check_kind = instruction->GetTypeCheckKind(); 6032 bool baker_read_barrier_slow_path = false; 6033 switch (type_check_kind) { 6034 case TypeCheckKind::kExactCheck: 6035 case TypeCheckKind::kAbstractClassCheck: 6036 case TypeCheckKind::kClassHierarchyCheck: 6037 case TypeCheckKind::kArrayObjectCheck: 6038 call_kind = 6039 kEmitCompilerReadBarrier ? LocationSummary::kCallOnSlowPath : LocationSummary::kNoCall; 6040 baker_read_barrier_slow_path = kUseBakerReadBarrier; 6041 break; 6042 case TypeCheckKind::kArrayCheck: 6043 case TypeCheckKind::kUnresolvedCheck: 6044 case TypeCheckKind::kInterfaceCheck: 6045 call_kind = LocationSummary::kCallOnSlowPath; 6046 break; 6047 } 6048 6049 LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction, call_kind); 6050 if (baker_read_barrier_slow_path) { 6051 locations->SetCustomSlowPathCallerSaves(RegisterSet::Empty()); // No caller-save registers. 6052 } 6053 locations->SetInAt(0, Location::RequiresRegister()); 6054 locations->SetInAt(1, Location::RequiresRegister()); 6055 // The "out" register is used as a temporary, so it overlaps with the inputs. 6056 // Note that TypeCheckSlowPathARM uses this register too. 6057 locations->SetOut(Location::RequiresRegister(), Location::kOutputOverlap); 6058 // When read barriers are enabled, we need a temporary register for 6059 // some cases. 6060 if (TypeCheckNeedsATemporary(type_check_kind)) { 6061 locations->AddTemp(Location::RequiresRegister()); 6062 } 6063} 6064 6065void InstructionCodeGeneratorARM::VisitInstanceOf(HInstanceOf* instruction) { 6066 TypeCheckKind type_check_kind = instruction->GetTypeCheckKind(); 6067 LocationSummary* locations = instruction->GetLocations(); 6068 Location obj_loc = locations->InAt(0); 6069 Register obj = obj_loc.AsRegister<Register>(); 6070 Register cls = locations->InAt(1).AsRegister<Register>(); 6071 Location out_loc = locations->Out(); 6072 Register out = out_loc.AsRegister<Register>(); 6073 Location maybe_temp_loc = TypeCheckNeedsATemporary(type_check_kind) ? 6074 locations->GetTemp(0) : 6075 Location::NoLocation(); 6076 uint32_t class_offset = mirror::Object::ClassOffset().Int32Value(); 6077 uint32_t super_offset = mirror::Class::SuperClassOffset().Int32Value(); 6078 uint32_t component_offset = mirror::Class::ComponentTypeOffset().Int32Value(); 6079 uint32_t primitive_offset = mirror::Class::PrimitiveTypeOffset().Int32Value(); 6080 Label done, zero; 6081 SlowPathCodeARM* slow_path = nullptr; 6082 6083 // Return 0 if `obj` is null. 6084 // avoid null check if we know obj is not null. 6085 if (instruction->MustDoNullCheck()) { 6086 __ CompareAndBranchIfZero(obj, &zero); 6087 } 6088 6089 // /* HeapReference<Class> */ out = obj->klass_ 6090 GenerateReferenceLoadTwoRegisters(instruction, out_loc, obj_loc, class_offset, maybe_temp_loc); 6091 6092 switch (type_check_kind) { 6093 case TypeCheckKind::kExactCheck: { 6094 __ cmp(out, ShifterOperand(cls)); 6095 // Classes must be equal for the instanceof to succeed. 6096 __ b(&zero, NE); 6097 __ LoadImmediate(out, 1); 6098 __ b(&done); 6099 break; 6100 } 6101 6102 case TypeCheckKind::kAbstractClassCheck: { 6103 // If the class is abstract, we eagerly fetch the super class of the 6104 // object to avoid doing a comparison we know will fail. 6105 Label loop; 6106 __ Bind(&loop); 6107 // /* HeapReference<Class> */ out = out->super_class_ 6108 GenerateReferenceLoadOneRegister(instruction, out_loc, super_offset, maybe_temp_loc); 6109 // If `out` is null, we use it for the result, and jump to `done`. 6110 __ CompareAndBranchIfZero(out, &done); 6111 __ cmp(out, ShifterOperand(cls)); 6112 __ b(&loop, NE); 6113 __ LoadImmediate(out, 1); 6114 if (zero.IsLinked()) { 6115 __ b(&done); 6116 } 6117 break; 6118 } 6119 6120 case TypeCheckKind::kClassHierarchyCheck: { 6121 // Walk over the class hierarchy to find a match. 6122 Label loop, success; 6123 __ Bind(&loop); 6124 __ cmp(out, ShifterOperand(cls)); 6125 __ b(&success, EQ); 6126 // /* HeapReference<Class> */ out = out->super_class_ 6127 GenerateReferenceLoadOneRegister(instruction, out_loc, super_offset, maybe_temp_loc); 6128 __ CompareAndBranchIfNonZero(out, &loop); 6129 // If `out` is null, we use it for the result, and jump to `done`. 6130 __ b(&done); 6131 __ Bind(&success); 6132 __ LoadImmediate(out, 1); 6133 if (zero.IsLinked()) { 6134 __ b(&done); 6135 } 6136 break; 6137 } 6138 6139 case TypeCheckKind::kArrayObjectCheck: { 6140 // Do an exact check. 6141 Label exact_check; 6142 __ cmp(out, ShifterOperand(cls)); 6143 __ b(&exact_check, EQ); 6144 // Otherwise, we need to check that the object's class is a non-primitive array. 6145 // /* HeapReference<Class> */ out = out->component_type_ 6146 GenerateReferenceLoadOneRegister(instruction, out_loc, component_offset, maybe_temp_loc); 6147 // If `out` is null, we use it for the result, and jump to `done`. 6148 __ CompareAndBranchIfZero(out, &done); 6149 __ LoadFromOffset(kLoadUnsignedHalfword, out, out, primitive_offset); 6150 static_assert(Primitive::kPrimNot == 0, "Expected 0 for kPrimNot"); 6151 __ CompareAndBranchIfNonZero(out, &zero); 6152 __ Bind(&exact_check); 6153 __ LoadImmediate(out, 1); 6154 __ b(&done); 6155 break; 6156 } 6157 6158 case TypeCheckKind::kArrayCheck: { 6159 __ cmp(out, ShifterOperand(cls)); 6160 DCHECK(locations->OnlyCallsOnSlowPath()); 6161 slow_path = new (GetGraph()->GetArena()) TypeCheckSlowPathARM(instruction, 6162 /* is_fatal */ false); 6163 codegen_->AddSlowPath(slow_path); 6164 __ b(slow_path->GetEntryLabel(), NE); 6165 __ LoadImmediate(out, 1); 6166 if (zero.IsLinked()) { 6167 __ b(&done); 6168 } 6169 break; 6170 } 6171 6172 case TypeCheckKind::kUnresolvedCheck: 6173 case TypeCheckKind::kInterfaceCheck: { 6174 // Note that we indeed only call on slow path, but we always go 6175 // into the slow path for the unresolved and interface check 6176 // cases. 6177 // 6178 // We cannot directly call the InstanceofNonTrivial runtime 6179 // entry point without resorting to a type checking slow path 6180 // here (i.e. by calling InvokeRuntime directly), as it would 6181 // require to assign fixed registers for the inputs of this 6182 // HInstanceOf instruction (following the runtime calling 6183 // convention), which might be cluttered by the potential first 6184 // read barrier emission at the beginning of this method. 6185 // 6186 // TODO: Introduce a new runtime entry point taking the object 6187 // to test (instead of its class) as argument, and let it deal 6188 // with the read barrier issues. This will let us refactor this 6189 // case of the `switch` code as it was previously (with a direct 6190 // call to the runtime not using a type checking slow path). 6191 // This should also be beneficial for the other cases above. 6192 DCHECK(locations->OnlyCallsOnSlowPath()); 6193 slow_path = new (GetGraph()->GetArena()) TypeCheckSlowPathARM(instruction, 6194 /* is_fatal */ false); 6195 codegen_->AddSlowPath(slow_path); 6196 __ b(slow_path->GetEntryLabel()); 6197 if (zero.IsLinked()) { 6198 __ b(&done); 6199 } 6200 break; 6201 } 6202 } 6203 6204 if (zero.IsLinked()) { 6205 __ Bind(&zero); 6206 __ LoadImmediate(out, 0); 6207 } 6208 6209 if (done.IsLinked()) { 6210 __ Bind(&done); 6211 } 6212 6213 if (slow_path != nullptr) { 6214 __ Bind(slow_path->GetExitLabel()); 6215 } 6216} 6217 6218void LocationsBuilderARM::VisitCheckCast(HCheckCast* instruction) { 6219 LocationSummary::CallKind call_kind = LocationSummary::kNoCall; 6220 bool throws_into_catch = instruction->CanThrowIntoCatchBlock(); 6221 6222 TypeCheckKind type_check_kind = instruction->GetTypeCheckKind(); 6223 switch (type_check_kind) { 6224 case TypeCheckKind::kExactCheck: 6225 case TypeCheckKind::kAbstractClassCheck: 6226 case TypeCheckKind::kClassHierarchyCheck: 6227 case TypeCheckKind::kArrayObjectCheck: 6228 call_kind = (throws_into_catch || kEmitCompilerReadBarrier) ? 6229 LocationSummary::kCallOnSlowPath : 6230 LocationSummary::kNoCall; // In fact, call on a fatal (non-returning) slow path. 6231 break; 6232 case TypeCheckKind::kArrayCheck: 6233 case TypeCheckKind::kUnresolvedCheck: 6234 case TypeCheckKind::kInterfaceCheck: 6235 call_kind = LocationSummary::kCallOnSlowPath; 6236 break; 6237 } 6238 6239 LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction, call_kind); 6240 locations->SetInAt(0, Location::RequiresRegister()); 6241 locations->SetInAt(1, Location::RequiresRegister()); 6242 // Note that TypeCheckSlowPathARM uses this "temp" register too. 6243 locations->AddTemp(Location::RequiresRegister()); 6244 // When read barriers are enabled, we need an additional temporary 6245 // register for some cases. 6246 if (TypeCheckNeedsATemporary(type_check_kind)) { 6247 locations->AddTemp(Location::RequiresRegister()); 6248 } 6249} 6250 6251void InstructionCodeGeneratorARM::VisitCheckCast(HCheckCast* instruction) { 6252 TypeCheckKind type_check_kind = instruction->GetTypeCheckKind(); 6253 LocationSummary* locations = instruction->GetLocations(); 6254 Location obj_loc = locations->InAt(0); 6255 Register obj = obj_loc.AsRegister<Register>(); 6256 Register cls = locations->InAt(1).AsRegister<Register>(); 6257 Location temp_loc = locations->GetTemp(0); 6258 Register temp = temp_loc.AsRegister<Register>(); 6259 Location maybe_temp2_loc = TypeCheckNeedsATemporary(type_check_kind) ? 6260 locations->GetTemp(1) : 6261 Location::NoLocation(); 6262 uint32_t class_offset = mirror::Object::ClassOffset().Int32Value(); 6263 uint32_t super_offset = mirror::Class::SuperClassOffset().Int32Value(); 6264 uint32_t component_offset = mirror::Class::ComponentTypeOffset().Int32Value(); 6265 uint32_t primitive_offset = mirror::Class::PrimitiveTypeOffset().Int32Value(); 6266 6267 bool is_type_check_slow_path_fatal = 6268 (type_check_kind == TypeCheckKind::kExactCheck || 6269 type_check_kind == TypeCheckKind::kAbstractClassCheck || 6270 type_check_kind == TypeCheckKind::kClassHierarchyCheck || 6271 type_check_kind == TypeCheckKind::kArrayObjectCheck) && 6272 !instruction->CanThrowIntoCatchBlock(); 6273 SlowPathCodeARM* type_check_slow_path = 6274 new (GetGraph()->GetArena()) TypeCheckSlowPathARM(instruction, 6275 is_type_check_slow_path_fatal); 6276 codegen_->AddSlowPath(type_check_slow_path); 6277 6278 Label done; 6279 // Avoid null check if we know obj is not null. 6280 if (instruction->MustDoNullCheck()) { 6281 __ CompareAndBranchIfZero(obj, &done); 6282 } 6283 6284 // /* HeapReference<Class> */ temp = obj->klass_ 6285 GenerateReferenceLoadTwoRegisters(instruction, temp_loc, obj_loc, class_offset, maybe_temp2_loc); 6286 6287 switch (type_check_kind) { 6288 case TypeCheckKind::kExactCheck: 6289 case TypeCheckKind::kArrayCheck: { 6290 __ cmp(temp, ShifterOperand(cls)); 6291 // Jump to slow path for throwing the exception or doing a 6292 // more involved array check. 6293 __ b(type_check_slow_path->GetEntryLabel(), NE); 6294 break; 6295 } 6296 6297 case TypeCheckKind::kAbstractClassCheck: { 6298 // If the class is abstract, we eagerly fetch the super class of the 6299 // object to avoid doing a comparison we know will fail. 6300 Label loop, compare_classes; 6301 __ Bind(&loop); 6302 // /* HeapReference<Class> */ temp = temp->super_class_ 6303 GenerateReferenceLoadOneRegister(instruction, temp_loc, super_offset, maybe_temp2_loc); 6304 6305 // If the class reference currently in `temp` is not null, jump 6306 // to the `compare_classes` label to compare it with the checked 6307 // class. 6308 __ CompareAndBranchIfNonZero(temp, &compare_classes); 6309 // Otherwise, jump to the slow path to throw the exception. 6310 // 6311 // But before, move back the object's class into `temp` before 6312 // going into the slow path, as it has been overwritten in the 6313 // meantime. 6314 // /* HeapReference<Class> */ temp = obj->klass_ 6315 GenerateReferenceLoadTwoRegisters( 6316 instruction, temp_loc, obj_loc, class_offset, maybe_temp2_loc); 6317 __ b(type_check_slow_path->GetEntryLabel()); 6318 6319 __ Bind(&compare_classes); 6320 __ cmp(temp, ShifterOperand(cls)); 6321 __ b(&loop, NE); 6322 break; 6323 } 6324 6325 case TypeCheckKind::kClassHierarchyCheck: { 6326 // Walk over the class hierarchy to find a match. 6327 Label loop; 6328 __ Bind(&loop); 6329 __ cmp(temp, ShifterOperand(cls)); 6330 __ b(&done, EQ); 6331 6332 // /* HeapReference<Class> */ temp = temp->super_class_ 6333 GenerateReferenceLoadOneRegister(instruction, temp_loc, super_offset, maybe_temp2_loc); 6334 6335 // If the class reference currently in `temp` is not null, jump 6336 // back at the beginning of the loop. 6337 __ CompareAndBranchIfNonZero(temp, &loop); 6338 // Otherwise, jump to the slow path to throw the exception. 6339 // 6340 // But before, move back the object's class into `temp` before 6341 // going into the slow path, as it has been overwritten in the 6342 // meantime. 6343 // /* HeapReference<Class> */ temp = obj->klass_ 6344 GenerateReferenceLoadTwoRegisters( 6345 instruction, temp_loc, obj_loc, class_offset, maybe_temp2_loc); 6346 __ b(type_check_slow_path->GetEntryLabel()); 6347 break; 6348 } 6349 6350 case TypeCheckKind::kArrayObjectCheck: { 6351 // Do an exact check. 6352 Label check_non_primitive_component_type; 6353 __ cmp(temp, ShifterOperand(cls)); 6354 __ b(&done, EQ); 6355 6356 // Otherwise, we need to check that the object's class is a non-primitive array. 6357 // /* HeapReference<Class> */ temp = temp->component_type_ 6358 GenerateReferenceLoadOneRegister(instruction, temp_loc, component_offset, maybe_temp2_loc); 6359 6360 // If the component type is not null (i.e. the object is indeed 6361 // an array), jump to label `check_non_primitive_component_type` 6362 // to further check that this component type is not a primitive 6363 // type. 6364 __ CompareAndBranchIfNonZero(temp, &check_non_primitive_component_type); 6365 // Otherwise, jump to the slow path to throw the exception. 6366 // 6367 // But before, move back the object's class into `temp` before 6368 // going into the slow path, as it has been overwritten in the 6369 // meantime. 6370 // /* HeapReference<Class> */ temp = obj->klass_ 6371 GenerateReferenceLoadTwoRegisters( 6372 instruction, temp_loc, obj_loc, class_offset, maybe_temp2_loc); 6373 __ b(type_check_slow_path->GetEntryLabel()); 6374 6375 __ Bind(&check_non_primitive_component_type); 6376 __ LoadFromOffset(kLoadUnsignedHalfword, temp, temp, primitive_offset); 6377 static_assert(Primitive::kPrimNot == 0, "Expected 0 for art::Primitive::kPrimNot"); 6378 __ CompareAndBranchIfZero(temp, &done); 6379 // Same comment as above regarding `temp` and the slow path. 6380 // /* HeapReference<Class> */ temp = obj->klass_ 6381 GenerateReferenceLoadTwoRegisters( 6382 instruction, temp_loc, obj_loc, class_offset, maybe_temp2_loc); 6383 __ b(type_check_slow_path->GetEntryLabel()); 6384 break; 6385 } 6386 6387 case TypeCheckKind::kUnresolvedCheck: 6388 case TypeCheckKind::kInterfaceCheck: 6389 // We always go into the type check slow path for the unresolved 6390 // and interface check cases. 6391 // 6392 // We cannot directly call the CheckCast runtime entry point 6393 // without resorting to a type checking slow path here (i.e. by 6394 // calling InvokeRuntime directly), as it would require to 6395 // assign fixed registers for the inputs of this HInstanceOf 6396 // instruction (following the runtime calling convention), which 6397 // might be cluttered by the potential first read barrier 6398 // emission at the beginning of this method. 6399 // 6400 // TODO: Introduce a new runtime entry point taking the object 6401 // to test (instead of its class) as argument, and let it deal 6402 // with the read barrier issues. This will let us refactor this 6403 // case of the `switch` code as it was previously (with a direct 6404 // call to the runtime not using a type checking slow path). 6405 // This should also be beneficial for the other cases above. 6406 __ b(type_check_slow_path->GetEntryLabel()); 6407 break; 6408 } 6409 __ Bind(&done); 6410 6411 __ Bind(type_check_slow_path->GetExitLabel()); 6412} 6413 6414void LocationsBuilderARM::VisitMonitorOperation(HMonitorOperation* instruction) { 6415 LocationSummary* locations = 6416 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCallOnMainOnly); 6417 InvokeRuntimeCallingConvention calling_convention; 6418 locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(0))); 6419} 6420 6421void InstructionCodeGeneratorARM::VisitMonitorOperation(HMonitorOperation* instruction) { 6422 codegen_->InvokeRuntime(instruction->IsEnter() ? kQuickLockObject : kQuickUnlockObject, 6423 instruction, 6424 instruction->GetDexPc()); 6425 if (instruction->IsEnter()) { 6426 CheckEntrypointTypes<kQuickLockObject, void, mirror::Object*>(); 6427 } else { 6428 CheckEntrypointTypes<kQuickUnlockObject, void, mirror::Object*>(); 6429 } 6430} 6431 6432void LocationsBuilderARM::VisitAnd(HAnd* instruction) { HandleBitwiseOperation(instruction, AND); } 6433void LocationsBuilderARM::VisitOr(HOr* instruction) { HandleBitwiseOperation(instruction, ORR); } 6434void LocationsBuilderARM::VisitXor(HXor* instruction) { HandleBitwiseOperation(instruction, EOR); } 6435 6436void LocationsBuilderARM::HandleBitwiseOperation(HBinaryOperation* instruction, Opcode opcode) { 6437 LocationSummary* locations = 6438 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall); 6439 DCHECK(instruction->GetResultType() == Primitive::kPrimInt 6440 || instruction->GetResultType() == Primitive::kPrimLong); 6441 // Note: GVN reorders commutative operations to have the constant on the right hand side. 6442 locations->SetInAt(0, Location::RequiresRegister()); 6443 locations->SetInAt(1, ArmEncodableConstantOrRegister(instruction->InputAt(1), opcode)); 6444 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 6445} 6446 6447void InstructionCodeGeneratorARM::VisitAnd(HAnd* instruction) { 6448 HandleBitwiseOperation(instruction); 6449} 6450 6451void InstructionCodeGeneratorARM::VisitOr(HOr* instruction) { 6452 HandleBitwiseOperation(instruction); 6453} 6454 6455void InstructionCodeGeneratorARM::VisitXor(HXor* instruction) { 6456 HandleBitwiseOperation(instruction); 6457} 6458 6459 6460void LocationsBuilderARM::VisitBitwiseNegatedRight(HBitwiseNegatedRight* instruction) { 6461 LocationSummary* locations = 6462 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall); 6463 DCHECK(instruction->GetResultType() == Primitive::kPrimInt 6464 || instruction->GetResultType() == Primitive::kPrimLong); 6465 6466 locations->SetInAt(0, Location::RequiresRegister()); 6467 locations->SetInAt(1, Location::RequiresRegister()); 6468 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 6469} 6470 6471void InstructionCodeGeneratorARM::VisitBitwiseNegatedRight(HBitwiseNegatedRight* instruction) { 6472 LocationSummary* locations = instruction->GetLocations(); 6473 Location first = locations->InAt(0); 6474 Location second = locations->InAt(1); 6475 Location out = locations->Out(); 6476 6477 if (instruction->GetResultType() == Primitive::kPrimInt) { 6478 Register first_reg = first.AsRegister<Register>(); 6479 ShifterOperand second_reg(second.AsRegister<Register>()); 6480 Register out_reg = out.AsRegister<Register>(); 6481 6482 switch (instruction->GetOpKind()) { 6483 case HInstruction::kAnd: 6484 __ bic(out_reg, first_reg, second_reg); 6485 break; 6486 case HInstruction::kOr: 6487 __ orn(out_reg, first_reg, second_reg); 6488 break; 6489 // There is no EON on arm. 6490 case HInstruction::kXor: 6491 default: 6492 LOG(FATAL) << "Unexpected instruction " << instruction->DebugName(); 6493 UNREACHABLE(); 6494 } 6495 return; 6496 6497 } else { 6498 DCHECK_EQ(instruction->GetResultType(), Primitive::kPrimLong); 6499 Register first_low = first.AsRegisterPairLow<Register>(); 6500 Register first_high = first.AsRegisterPairHigh<Register>(); 6501 ShifterOperand second_low(second.AsRegisterPairLow<Register>()); 6502 ShifterOperand second_high(second.AsRegisterPairHigh<Register>()); 6503 Register out_low = out.AsRegisterPairLow<Register>(); 6504 Register out_high = out.AsRegisterPairHigh<Register>(); 6505 6506 switch (instruction->GetOpKind()) { 6507 case HInstruction::kAnd: 6508 __ bic(out_low, first_low, second_low); 6509 __ bic(out_high, first_high, second_high); 6510 break; 6511 case HInstruction::kOr: 6512 __ orn(out_low, first_low, second_low); 6513 __ orn(out_high, first_high, second_high); 6514 break; 6515 // There is no EON on arm. 6516 case HInstruction::kXor: 6517 default: 6518 LOG(FATAL) << "Unexpected instruction " << instruction->DebugName(); 6519 UNREACHABLE(); 6520 } 6521 } 6522} 6523 6524void InstructionCodeGeneratorARM::GenerateAndConst(Register out, Register first, uint32_t value) { 6525 // Optimize special cases for individual halfs of `and-long` (`and` is simplified earlier). 6526 if (value == 0xffffffffu) { 6527 if (out != first) { 6528 __ mov(out, ShifterOperand(first)); 6529 } 6530 return; 6531 } 6532 if (value == 0u) { 6533 __ mov(out, ShifterOperand(0)); 6534 return; 6535 } 6536 ShifterOperand so; 6537 if (__ ShifterOperandCanHold(kNoRegister, kNoRegister, AND, value, &so)) { 6538 __ and_(out, first, so); 6539 } else { 6540 DCHECK(__ ShifterOperandCanHold(kNoRegister, kNoRegister, BIC, ~value, &so)); 6541 __ bic(out, first, ShifterOperand(~value)); 6542 } 6543} 6544 6545void InstructionCodeGeneratorARM::GenerateOrrConst(Register out, Register first, uint32_t value) { 6546 // Optimize special cases for individual halfs of `or-long` (`or` is simplified earlier). 6547 if (value == 0u) { 6548 if (out != first) { 6549 __ mov(out, ShifterOperand(first)); 6550 } 6551 return; 6552 } 6553 if (value == 0xffffffffu) { 6554 __ mvn(out, ShifterOperand(0)); 6555 return; 6556 } 6557 ShifterOperand so; 6558 if (__ ShifterOperandCanHold(kNoRegister, kNoRegister, ORR, value, &so)) { 6559 __ orr(out, first, so); 6560 } else { 6561 DCHECK(__ ShifterOperandCanHold(kNoRegister, kNoRegister, ORN, ~value, &so)); 6562 __ orn(out, first, ShifterOperand(~value)); 6563 } 6564} 6565 6566void InstructionCodeGeneratorARM::GenerateEorConst(Register out, Register first, uint32_t value) { 6567 // Optimize special case for individual halfs of `xor-long` (`xor` is simplified earlier). 6568 if (value == 0u) { 6569 if (out != first) { 6570 __ mov(out, ShifterOperand(first)); 6571 } 6572 return; 6573 } 6574 __ eor(out, first, ShifterOperand(value)); 6575} 6576 6577void InstructionCodeGeneratorARM::GenerateAddLongConst(Location out, 6578 Location first, 6579 uint64_t value) { 6580 Register out_low = out.AsRegisterPairLow<Register>(); 6581 Register out_high = out.AsRegisterPairHigh<Register>(); 6582 Register first_low = first.AsRegisterPairLow<Register>(); 6583 Register first_high = first.AsRegisterPairHigh<Register>(); 6584 uint32_t value_low = Low32Bits(value); 6585 uint32_t value_high = High32Bits(value); 6586 if (value_low == 0u) { 6587 if (out_low != first_low) { 6588 __ mov(out_low, ShifterOperand(first_low)); 6589 } 6590 __ AddConstant(out_high, first_high, value_high); 6591 return; 6592 } 6593 __ AddConstantSetFlags(out_low, first_low, value_low); 6594 ShifterOperand so; 6595 if (__ ShifterOperandCanHold(out_high, first_high, ADC, value_high, kCcDontCare, &so)) { 6596 __ adc(out_high, first_high, so); 6597 } else if (__ ShifterOperandCanHold(out_low, first_low, SBC, ~value_high, kCcDontCare, &so)) { 6598 __ sbc(out_high, first_high, so); 6599 } else { 6600 LOG(FATAL) << "Unexpected constant " << value_high; 6601 UNREACHABLE(); 6602 } 6603} 6604 6605void InstructionCodeGeneratorARM::HandleBitwiseOperation(HBinaryOperation* instruction) { 6606 LocationSummary* locations = instruction->GetLocations(); 6607 Location first = locations->InAt(0); 6608 Location second = locations->InAt(1); 6609 Location out = locations->Out(); 6610 6611 if (second.IsConstant()) { 6612 uint64_t value = static_cast<uint64_t>(Int64FromConstant(second.GetConstant())); 6613 uint32_t value_low = Low32Bits(value); 6614 if (instruction->GetResultType() == Primitive::kPrimInt) { 6615 Register first_reg = first.AsRegister<Register>(); 6616 Register out_reg = out.AsRegister<Register>(); 6617 if (instruction->IsAnd()) { 6618 GenerateAndConst(out_reg, first_reg, value_low); 6619 } else if (instruction->IsOr()) { 6620 GenerateOrrConst(out_reg, first_reg, value_low); 6621 } else { 6622 DCHECK(instruction->IsXor()); 6623 GenerateEorConst(out_reg, first_reg, value_low); 6624 } 6625 } else { 6626 DCHECK_EQ(instruction->GetResultType(), Primitive::kPrimLong); 6627 uint32_t value_high = High32Bits(value); 6628 Register first_low = first.AsRegisterPairLow<Register>(); 6629 Register first_high = first.AsRegisterPairHigh<Register>(); 6630 Register out_low = out.AsRegisterPairLow<Register>(); 6631 Register out_high = out.AsRegisterPairHigh<Register>(); 6632 if (instruction->IsAnd()) { 6633 GenerateAndConst(out_low, first_low, value_low); 6634 GenerateAndConst(out_high, first_high, value_high); 6635 } else if (instruction->IsOr()) { 6636 GenerateOrrConst(out_low, first_low, value_low); 6637 GenerateOrrConst(out_high, first_high, value_high); 6638 } else { 6639 DCHECK(instruction->IsXor()); 6640 GenerateEorConst(out_low, first_low, value_low); 6641 GenerateEorConst(out_high, first_high, value_high); 6642 } 6643 } 6644 return; 6645 } 6646 6647 if (instruction->GetResultType() == Primitive::kPrimInt) { 6648 Register first_reg = first.AsRegister<Register>(); 6649 ShifterOperand second_reg(second.AsRegister<Register>()); 6650 Register out_reg = out.AsRegister<Register>(); 6651 if (instruction->IsAnd()) { 6652 __ and_(out_reg, first_reg, second_reg); 6653 } else if (instruction->IsOr()) { 6654 __ orr(out_reg, first_reg, second_reg); 6655 } else { 6656 DCHECK(instruction->IsXor()); 6657 __ eor(out_reg, first_reg, second_reg); 6658 } 6659 } else { 6660 DCHECK_EQ(instruction->GetResultType(), Primitive::kPrimLong); 6661 Register first_low = first.AsRegisterPairLow<Register>(); 6662 Register first_high = first.AsRegisterPairHigh<Register>(); 6663 ShifterOperand second_low(second.AsRegisterPairLow<Register>()); 6664 ShifterOperand second_high(second.AsRegisterPairHigh<Register>()); 6665 Register out_low = out.AsRegisterPairLow<Register>(); 6666 Register out_high = out.AsRegisterPairHigh<Register>(); 6667 if (instruction->IsAnd()) { 6668 __ and_(out_low, first_low, second_low); 6669 __ and_(out_high, first_high, second_high); 6670 } else if (instruction->IsOr()) { 6671 __ orr(out_low, first_low, second_low); 6672 __ orr(out_high, first_high, second_high); 6673 } else { 6674 DCHECK(instruction->IsXor()); 6675 __ eor(out_low, first_low, second_low); 6676 __ eor(out_high, first_high, second_high); 6677 } 6678 } 6679} 6680 6681void InstructionCodeGeneratorARM::GenerateReferenceLoadOneRegister(HInstruction* instruction, 6682 Location out, 6683 uint32_t offset, 6684 Location maybe_temp) { 6685 Register out_reg = out.AsRegister<Register>(); 6686 if (kEmitCompilerReadBarrier) { 6687 DCHECK(maybe_temp.IsRegister()) << maybe_temp; 6688 if (kUseBakerReadBarrier) { 6689 // Load with fast path based Baker's read barrier. 6690 // /* HeapReference<Object> */ out = *(out + offset) 6691 codegen_->GenerateFieldLoadWithBakerReadBarrier( 6692 instruction, out, out_reg, offset, maybe_temp, /* needs_null_check */ false); 6693 } else { 6694 // Load with slow path based read barrier. 6695 // Save the value of `out` into `maybe_temp` before overwriting it 6696 // in the following move operation, as we will need it for the 6697 // read barrier below. 6698 __ Mov(maybe_temp.AsRegister<Register>(), out_reg); 6699 // /* HeapReference<Object> */ out = *(out + offset) 6700 __ LoadFromOffset(kLoadWord, out_reg, out_reg, offset); 6701 codegen_->GenerateReadBarrierSlow(instruction, out, out, maybe_temp, offset); 6702 } 6703 } else { 6704 // Plain load with no read barrier. 6705 // /* HeapReference<Object> */ out = *(out + offset) 6706 __ LoadFromOffset(kLoadWord, out_reg, out_reg, offset); 6707 __ MaybeUnpoisonHeapReference(out_reg); 6708 } 6709} 6710 6711void InstructionCodeGeneratorARM::GenerateReferenceLoadTwoRegisters(HInstruction* instruction, 6712 Location out, 6713 Location obj, 6714 uint32_t offset, 6715 Location maybe_temp) { 6716 Register out_reg = out.AsRegister<Register>(); 6717 Register obj_reg = obj.AsRegister<Register>(); 6718 if (kEmitCompilerReadBarrier) { 6719 if (kUseBakerReadBarrier) { 6720 DCHECK(maybe_temp.IsRegister()) << maybe_temp; 6721 // Load with fast path based Baker's read barrier. 6722 // /* HeapReference<Object> */ out = *(obj + offset) 6723 codegen_->GenerateFieldLoadWithBakerReadBarrier( 6724 instruction, out, obj_reg, offset, maybe_temp, /* needs_null_check */ false); 6725 } else { 6726 // Load with slow path based read barrier. 6727 // /* HeapReference<Object> */ out = *(obj + offset) 6728 __ LoadFromOffset(kLoadWord, out_reg, obj_reg, offset); 6729 codegen_->GenerateReadBarrierSlow(instruction, out, out, obj, offset); 6730 } 6731 } else { 6732 // Plain load with no read barrier. 6733 // /* HeapReference<Object> */ out = *(obj + offset) 6734 __ LoadFromOffset(kLoadWord, out_reg, obj_reg, offset); 6735 __ MaybeUnpoisonHeapReference(out_reg); 6736 } 6737} 6738 6739void InstructionCodeGeneratorARM::GenerateGcRootFieldLoad(HInstruction* instruction, 6740 Location root, 6741 Register obj, 6742 uint32_t offset, 6743 bool requires_read_barrier) { 6744 Register root_reg = root.AsRegister<Register>(); 6745 if (requires_read_barrier) { 6746 DCHECK(kEmitCompilerReadBarrier); 6747 if (kUseBakerReadBarrier) { 6748 // Fast path implementation of art::ReadBarrier::BarrierForRoot when 6749 // Baker's read barrier are used: 6750 // 6751 // root = obj.field; 6752 // if (Thread::Current()->GetIsGcMarking()) { 6753 // root = ReadBarrier::Mark(root) 6754 // } 6755 6756 // /* GcRoot<mirror::Object> */ root = *(obj + offset) 6757 __ LoadFromOffset(kLoadWord, root_reg, obj, offset); 6758 static_assert( 6759 sizeof(mirror::CompressedReference<mirror::Object>) == sizeof(GcRoot<mirror::Object>), 6760 "art::mirror::CompressedReference<mirror::Object> and art::GcRoot<mirror::Object> " 6761 "have different sizes."); 6762 static_assert(sizeof(mirror::CompressedReference<mirror::Object>) == sizeof(int32_t), 6763 "art::mirror::CompressedReference<mirror::Object> and int32_t " 6764 "have different sizes."); 6765 6766 // Slow path marking the GC root `root`. 6767 SlowPathCodeARM* slow_path = 6768 new (GetGraph()->GetArena()) ReadBarrierMarkSlowPathARM(instruction, root); 6769 codegen_->AddSlowPath(slow_path); 6770 6771 // IP = Thread::Current()->GetIsGcMarking() 6772 __ LoadFromOffset( 6773 kLoadWord, IP, TR, Thread::IsGcMarkingOffset<kArmPointerSize>().Int32Value()); 6774 __ CompareAndBranchIfNonZero(IP, slow_path->GetEntryLabel()); 6775 __ Bind(slow_path->GetExitLabel()); 6776 } else { 6777 // GC root loaded through a slow path for read barriers other 6778 // than Baker's. 6779 // /* GcRoot<mirror::Object>* */ root = obj + offset 6780 __ AddConstant(root_reg, obj, offset); 6781 // /* mirror::Object* */ root = root->Read() 6782 codegen_->GenerateReadBarrierForRootSlow(instruction, root, root); 6783 } 6784 } else { 6785 // Plain GC root load with no read barrier. 6786 // /* GcRoot<mirror::Object> */ root = *(obj + offset) 6787 __ LoadFromOffset(kLoadWord, root_reg, obj, offset); 6788 // Note that GC roots are not affected by heap poisoning, thus we 6789 // do not have to unpoison `root_reg` here. 6790 } 6791} 6792 6793void CodeGeneratorARM::GenerateFieldLoadWithBakerReadBarrier(HInstruction* instruction, 6794 Location ref, 6795 Register obj, 6796 uint32_t offset, 6797 Location temp, 6798 bool needs_null_check) { 6799 DCHECK(kEmitCompilerReadBarrier); 6800 DCHECK(kUseBakerReadBarrier); 6801 6802 // /* HeapReference<Object> */ ref = *(obj + offset) 6803 Location no_index = Location::NoLocation(); 6804 ScaleFactor no_scale_factor = TIMES_1; 6805 GenerateReferenceLoadWithBakerReadBarrier( 6806 instruction, ref, obj, offset, no_index, no_scale_factor, temp, needs_null_check); 6807} 6808 6809void CodeGeneratorARM::GenerateArrayLoadWithBakerReadBarrier(HInstruction* instruction, 6810 Location ref, 6811 Register obj, 6812 uint32_t data_offset, 6813 Location index, 6814 Location temp, 6815 bool needs_null_check) { 6816 DCHECK(kEmitCompilerReadBarrier); 6817 DCHECK(kUseBakerReadBarrier); 6818 6819 static_assert( 6820 sizeof(mirror::HeapReference<mirror::Object>) == sizeof(int32_t), 6821 "art::mirror::HeapReference<art::mirror::Object> and int32_t have different sizes."); 6822 // /* HeapReference<Object> */ ref = 6823 // *(obj + data_offset + index * sizeof(HeapReference<Object>)) 6824 ScaleFactor scale_factor = TIMES_4; 6825 GenerateReferenceLoadWithBakerReadBarrier( 6826 instruction, ref, obj, data_offset, index, scale_factor, temp, needs_null_check); 6827} 6828 6829void CodeGeneratorARM::GenerateReferenceLoadWithBakerReadBarrier(HInstruction* instruction, 6830 Location ref, 6831 Register obj, 6832 uint32_t offset, 6833 Location index, 6834 ScaleFactor scale_factor, 6835 Location temp, 6836 bool needs_null_check, 6837 bool always_update_field, 6838 Register* temp2) { 6839 DCHECK(kEmitCompilerReadBarrier); 6840 DCHECK(kUseBakerReadBarrier); 6841 6842 // In slow path based read barriers, the read barrier call is 6843 // inserted after the original load. However, in fast path based 6844 // Baker's read barriers, we need to perform the load of 6845 // mirror::Object::monitor_ *before* the original reference load. 6846 // This load-load ordering is required by the read barrier. 6847 // The fast path/slow path (for Baker's algorithm) should look like: 6848 // 6849 // uint32_t rb_state = Lockword(obj->monitor_).ReadBarrierState(); 6850 // lfence; // Load fence or artificial data dependency to prevent load-load reordering 6851 // HeapReference<Object> ref = *src; // Original reference load. 6852 // bool is_gray = (rb_state == ReadBarrier::GrayState()); 6853 // if (is_gray) { 6854 // ref = ReadBarrier::Mark(ref); // Performed by runtime entrypoint slow path. 6855 // } 6856 // 6857 // Note: the original implementation in ReadBarrier::Barrier is 6858 // slightly more complex as it performs additional checks that we do 6859 // not do here for performance reasons. 6860 6861 Register ref_reg = ref.AsRegister<Register>(); 6862 Register temp_reg = temp.AsRegister<Register>(); 6863 uint32_t monitor_offset = mirror::Object::MonitorOffset().Int32Value(); 6864 6865 // /* int32_t */ monitor = obj->monitor_ 6866 __ LoadFromOffset(kLoadWord, temp_reg, obj, monitor_offset); 6867 if (needs_null_check) { 6868 MaybeRecordImplicitNullCheck(instruction); 6869 } 6870 // /* LockWord */ lock_word = LockWord(monitor) 6871 static_assert(sizeof(LockWord) == sizeof(int32_t), 6872 "art::LockWord and int32_t have different sizes."); 6873 6874 // Introduce a dependency on the lock_word including the rb_state, 6875 // which shall prevent load-load reordering without using 6876 // a memory barrier (which would be more expensive). 6877 // `obj` is unchanged by this operation, but its value now depends 6878 // on `temp_reg`. 6879 __ add(obj, obj, ShifterOperand(temp_reg, LSR, 32)); 6880 6881 // The actual reference load. 6882 if (index.IsValid()) { 6883 // Load types involving an "index": ArrayGet, 6884 // UnsafeGetObject/UnsafeGetObjectVolatile and UnsafeCASObject 6885 // intrinsics. 6886 // /* HeapReference<Object> */ ref = *(obj + offset + (index << scale_factor)) 6887 if (index.IsConstant()) { 6888 size_t computed_offset = 6889 (index.GetConstant()->AsIntConstant()->GetValue() << scale_factor) + offset; 6890 __ LoadFromOffset(kLoadWord, ref_reg, obj, computed_offset); 6891 } else { 6892 // Handle the special case of the 6893 // UnsafeGetObject/UnsafeGetObjectVolatile and UnsafeCASObject 6894 // intrinsics, which use a register pair as index ("long 6895 // offset"), of which only the low part contains data. 6896 Register index_reg = index.IsRegisterPair() 6897 ? index.AsRegisterPairLow<Register>() 6898 : index.AsRegister<Register>(); 6899 __ add(IP, obj, ShifterOperand(index_reg, LSL, scale_factor)); 6900 __ LoadFromOffset(kLoadWord, ref_reg, IP, offset); 6901 } 6902 } else { 6903 // /* HeapReference<Object> */ ref = *(obj + offset) 6904 __ LoadFromOffset(kLoadWord, ref_reg, obj, offset); 6905 } 6906 6907 // Object* ref = ref_addr->AsMirrorPtr() 6908 __ MaybeUnpoisonHeapReference(ref_reg); 6909 6910 // Slow path marking the object `ref` when it is gray. 6911 SlowPathCodeARM* slow_path; 6912 if (always_update_field) { 6913 DCHECK(temp2 != nullptr); 6914 // ReadBarrierMarkAndUpdateFieldSlowPathARM only supports address 6915 // of the form `obj + field_offset`, where `obj` is a register and 6916 // `field_offset` is a register pair (of which only the lower half 6917 // is used). Thus `offset` and `scale_factor` above are expected 6918 // to be null in this code path. 6919 DCHECK_EQ(offset, 0u); 6920 DCHECK_EQ(scale_factor, ScaleFactor::TIMES_1); 6921 slow_path = new (GetGraph()->GetArena()) ReadBarrierMarkAndUpdateFieldSlowPathARM( 6922 instruction, ref, obj, /* field_offset */ index, temp_reg, *temp2); 6923 } else { 6924 slow_path = new (GetGraph()->GetArena()) ReadBarrierMarkSlowPathARM(instruction, ref); 6925 } 6926 AddSlowPath(slow_path); 6927 6928 // if (rb_state == ReadBarrier::GrayState()) 6929 // ref = ReadBarrier::Mark(ref); 6930 // Given the numeric representation, it's enough to check the low bit of the 6931 // rb_state. We do that by shifting the bit out of the lock word with LSRS 6932 // which can be a 16-bit instruction unlike the TST immediate. 6933 static_assert(ReadBarrier::WhiteState() == 0, "Expecting white to have value 0"); 6934 static_assert(ReadBarrier::GrayState() == 1, "Expecting gray to have value 1"); 6935 __ Lsrs(temp_reg, temp_reg, LockWord::kReadBarrierStateShift + 1); 6936 __ b(slow_path->GetEntryLabel(), CS); // Carry flag is the last bit shifted out by LSRS. 6937 __ Bind(slow_path->GetExitLabel()); 6938} 6939 6940void CodeGeneratorARM::GenerateReadBarrierSlow(HInstruction* instruction, 6941 Location out, 6942 Location ref, 6943 Location obj, 6944 uint32_t offset, 6945 Location index) { 6946 DCHECK(kEmitCompilerReadBarrier); 6947 6948 // Insert a slow path based read barrier *after* the reference load. 6949 // 6950 // If heap poisoning is enabled, the unpoisoning of the loaded 6951 // reference will be carried out by the runtime within the slow 6952 // path. 6953 // 6954 // Note that `ref` currently does not get unpoisoned (when heap 6955 // poisoning is enabled), which is alright as the `ref` argument is 6956 // not used by the artReadBarrierSlow entry point. 6957 // 6958 // TODO: Unpoison `ref` when it is used by artReadBarrierSlow. 6959 SlowPathCodeARM* slow_path = new (GetGraph()->GetArena()) 6960 ReadBarrierForHeapReferenceSlowPathARM(instruction, out, ref, obj, offset, index); 6961 AddSlowPath(slow_path); 6962 6963 __ b(slow_path->GetEntryLabel()); 6964 __ Bind(slow_path->GetExitLabel()); 6965} 6966 6967void CodeGeneratorARM::MaybeGenerateReadBarrierSlow(HInstruction* instruction, 6968 Location out, 6969 Location ref, 6970 Location obj, 6971 uint32_t offset, 6972 Location index) { 6973 if (kEmitCompilerReadBarrier) { 6974 // Baker's read barriers shall be handled by the fast path 6975 // (CodeGeneratorARM::GenerateReferenceLoadWithBakerReadBarrier). 6976 DCHECK(!kUseBakerReadBarrier); 6977 // If heap poisoning is enabled, unpoisoning will be taken care of 6978 // by the runtime within the slow path. 6979 GenerateReadBarrierSlow(instruction, out, ref, obj, offset, index); 6980 } else if (kPoisonHeapReferences) { 6981 __ UnpoisonHeapReference(out.AsRegister<Register>()); 6982 } 6983} 6984 6985void CodeGeneratorARM::GenerateReadBarrierForRootSlow(HInstruction* instruction, 6986 Location out, 6987 Location root) { 6988 DCHECK(kEmitCompilerReadBarrier); 6989 6990 // Insert a slow path based read barrier *after* the GC root load. 6991 // 6992 // Note that GC roots are not affected by heap poisoning, so we do 6993 // not need to do anything special for this here. 6994 SlowPathCodeARM* slow_path = 6995 new (GetGraph()->GetArena()) ReadBarrierForRootSlowPathARM(instruction, out, root); 6996 AddSlowPath(slow_path); 6997 6998 __ b(slow_path->GetEntryLabel()); 6999 __ Bind(slow_path->GetExitLabel()); 7000} 7001 7002HInvokeStaticOrDirect::DispatchInfo CodeGeneratorARM::GetSupportedInvokeStaticOrDirectDispatch( 7003 const HInvokeStaticOrDirect::DispatchInfo& desired_dispatch_info, 7004 HInvokeStaticOrDirect* invoke) { 7005 HInvokeStaticOrDirect::DispatchInfo dispatch_info = desired_dispatch_info; 7006 // We disable pc-relative load when there is an irreducible loop, as the optimization 7007 // is incompatible with it. 7008 // TODO: Create as many ArmDexCacheArraysBase instructions as needed for methods 7009 // with irreducible loops. 7010 if (GetGraph()->HasIrreducibleLoops() && 7011 (dispatch_info.method_load_kind == 7012 HInvokeStaticOrDirect::MethodLoadKind::kDexCachePcRelative)) { 7013 dispatch_info.method_load_kind = HInvokeStaticOrDirect::MethodLoadKind::kDexCacheViaMethod; 7014 } 7015 7016 if (dispatch_info.code_ptr_location == HInvokeStaticOrDirect::CodePtrLocation::kCallPCRelative) { 7017 const DexFile& outer_dex_file = GetGraph()->GetDexFile(); 7018 if (&outer_dex_file != invoke->GetTargetMethod().dex_file) { 7019 // Calls across dex files are more likely to exceed the available BL range, 7020 // so use absolute patch with fixup if available and kCallArtMethod otherwise. 7021 HInvokeStaticOrDirect::CodePtrLocation code_ptr_location = 7022 (desired_dispatch_info.method_load_kind == 7023 HInvokeStaticOrDirect::MethodLoadKind::kDirectAddressWithFixup) 7024 ? HInvokeStaticOrDirect::CodePtrLocation::kCallDirectWithFixup 7025 : HInvokeStaticOrDirect::CodePtrLocation::kCallArtMethod; 7026 return HInvokeStaticOrDirect::DispatchInfo { 7027 dispatch_info.method_load_kind, 7028 code_ptr_location, 7029 dispatch_info.method_load_data, 7030 0u 7031 }; 7032 } 7033 } 7034 return dispatch_info; 7035} 7036 7037Register CodeGeneratorARM::GetInvokeStaticOrDirectExtraParameter(HInvokeStaticOrDirect* invoke, 7038 Register temp) { 7039 DCHECK_EQ(invoke->InputCount(), invoke->GetNumberOfArguments() + 1u); 7040 Location location = invoke->GetLocations()->InAt(invoke->GetSpecialInputIndex()); 7041 if (!invoke->GetLocations()->Intrinsified()) { 7042 return location.AsRegister<Register>(); 7043 } 7044 // For intrinsics we allow any location, so it may be on the stack. 7045 if (!location.IsRegister()) { 7046 __ LoadFromOffset(kLoadWord, temp, SP, location.GetStackIndex()); 7047 return temp; 7048 } 7049 // For register locations, check if the register was saved. If so, get it from the stack. 7050 // Note: There is a chance that the register was saved but not overwritten, so we could 7051 // save one load. However, since this is just an intrinsic slow path we prefer this 7052 // simple and more robust approach rather that trying to determine if that's the case. 7053 SlowPathCode* slow_path = GetCurrentSlowPath(); 7054 DCHECK(slow_path != nullptr); // For intrinsified invokes the call is emitted on the slow path. 7055 if (slow_path->IsCoreRegisterSaved(location.AsRegister<Register>())) { 7056 int stack_offset = slow_path->GetStackOffsetOfCoreRegister(location.AsRegister<Register>()); 7057 __ LoadFromOffset(kLoadWord, temp, SP, stack_offset); 7058 return temp; 7059 } 7060 return location.AsRegister<Register>(); 7061} 7062 7063void CodeGeneratorARM::GenerateStaticOrDirectCall(HInvokeStaticOrDirect* invoke, Location temp) { 7064 // For better instruction scheduling we load the direct code pointer before the method pointer. 7065 switch (invoke->GetCodePtrLocation()) { 7066 case HInvokeStaticOrDirect::CodePtrLocation::kCallDirectWithFixup: 7067 // LR = code address from literal pool with link-time patch. 7068 __ LoadLiteral(LR, DeduplicateMethodCodeLiteral(invoke->GetTargetMethod())); 7069 break; 7070 case HInvokeStaticOrDirect::CodePtrLocation::kCallDirect: 7071 // LR = invoke->GetDirectCodePtr(); 7072 __ LoadImmediate(LR, invoke->GetDirectCodePtr()); 7073 break; 7074 default: 7075 break; 7076 } 7077 7078 Location callee_method = temp; // For all kinds except kRecursive, callee will be in temp. 7079 switch (invoke->GetMethodLoadKind()) { 7080 case HInvokeStaticOrDirect::MethodLoadKind::kStringInit: { 7081 uint32_t offset = 7082 GetThreadOffset<kArmPointerSize>(invoke->GetStringInitEntryPoint()).Int32Value(); 7083 // temp = thread->string_init_entrypoint 7084 __ LoadFromOffset(kLoadWord, temp.AsRegister<Register>(), TR, offset); 7085 break; 7086 } 7087 case HInvokeStaticOrDirect::MethodLoadKind::kRecursive: 7088 callee_method = invoke->GetLocations()->InAt(invoke->GetSpecialInputIndex()); 7089 break; 7090 case HInvokeStaticOrDirect::MethodLoadKind::kDirectAddress: 7091 __ LoadImmediate(temp.AsRegister<Register>(), invoke->GetMethodAddress()); 7092 break; 7093 case HInvokeStaticOrDirect::MethodLoadKind::kDirectAddressWithFixup: 7094 __ LoadLiteral(temp.AsRegister<Register>(), 7095 DeduplicateMethodAddressLiteral(invoke->GetTargetMethod())); 7096 break; 7097 case HInvokeStaticOrDirect::MethodLoadKind::kDexCachePcRelative: { 7098 HArmDexCacheArraysBase* base = 7099 invoke->InputAt(invoke->GetSpecialInputIndex())->AsArmDexCacheArraysBase(); 7100 Register base_reg = GetInvokeStaticOrDirectExtraParameter(invoke, 7101 temp.AsRegister<Register>()); 7102 int32_t offset = invoke->GetDexCacheArrayOffset() - base->GetElementOffset(); 7103 __ LoadFromOffset(kLoadWord, temp.AsRegister<Register>(), base_reg, offset); 7104 break; 7105 } 7106 case HInvokeStaticOrDirect::MethodLoadKind::kDexCacheViaMethod: { 7107 Location current_method = invoke->GetLocations()->InAt(invoke->GetSpecialInputIndex()); 7108 Register method_reg; 7109 Register reg = temp.AsRegister<Register>(); 7110 if (current_method.IsRegister()) { 7111 method_reg = current_method.AsRegister<Register>(); 7112 } else { 7113 DCHECK(invoke->GetLocations()->Intrinsified()); 7114 DCHECK(!current_method.IsValid()); 7115 method_reg = reg; 7116 __ LoadFromOffset(kLoadWord, reg, SP, kCurrentMethodStackOffset); 7117 } 7118 // /* ArtMethod*[] */ temp = temp.ptr_sized_fields_->dex_cache_resolved_methods_; 7119 __ LoadFromOffset(kLoadWord, 7120 reg, 7121 method_reg, 7122 ArtMethod::DexCacheResolvedMethodsOffset(kArmPointerSize).Int32Value()); 7123 // temp = temp[index_in_cache]; 7124 // Note: Don't use invoke->GetTargetMethod() as it may point to a different dex file. 7125 uint32_t index_in_cache = invoke->GetDexMethodIndex(); 7126 __ LoadFromOffset(kLoadWord, reg, reg, CodeGenerator::GetCachePointerOffset(index_in_cache)); 7127 break; 7128 } 7129 } 7130 7131 switch (invoke->GetCodePtrLocation()) { 7132 case HInvokeStaticOrDirect::CodePtrLocation::kCallSelf: 7133 __ bl(GetFrameEntryLabel()); 7134 break; 7135 case HInvokeStaticOrDirect::CodePtrLocation::kCallPCRelative: 7136 relative_call_patches_.emplace_back(*invoke->GetTargetMethod().dex_file, 7137 invoke->GetTargetMethod().dex_method_index); 7138 __ BindTrackedLabel(&relative_call_patches_.back().label); 7139 // Arbitrarily branch to the BL itself, override at link time. 7140 __ bl(&relative_call_patches_.back().label); 7141 break; 7142 case HInvokeStaticOrDirect::CodePtrLocation::kCallDirectWithFixup: 7143 case HInvokeStaticOrDirect::CodePtrLocation::kCallDirect: 7144 // LR prepared above for better instruction scheduling. 7145 // LR() 7146 __ blx(LR); 7147 break; 7148 case HInvokeStaticOrDirect::CodePtrLocation::kCallArtMethod: 7149 // LR = callee_method->entry_point_from_quick_compiled_code_ 7150 __ LoadFromOffset( 7151 kLoadWord, LR, callee_method.AsRegister<Register>(), 7152 ArtMethod::EntryPointFromQuickCompiledCodeOffset(kArmPointerSize).Int32Value()); 7153 // LR() 7154 __ blx(LR); 7155 break; 7156 } 7157 7158 DCHECK(!IsLeafMethod()); 7159} 7160 7161void CodeGeneratorARM::GenerateVirtualCall(HInvokeVirtual* invoke, Location temp_location) { 7162 Register temp = temp_location.AsRegister<Register>(); 7163 uint32_t method_offset = mirror::Class::EmbeddedVTableEntryOffset( 7164 invoke->GetVTableIndex(), kArmPointerSize).Uint32Value(); 7165 7166 // Use the calling convention instead of the location of the receiver, as 7167 // intrinsics may have put the receiver in a different register. In the intrinsics 7168 // slow path, the arguments have been moved to the right place, so here we are 7169 // guaranteed that the receiver is the first register of the calling convention. 7170 InvokeDexCallingConvention calling_convention; 7171 Register receiver = calling_convention.GetRegisterAt(0); 7172 uint32_t class_offset = mirror::Object::ClassOffset().Int32Value(); 7173 // /* HeapReference<Class> */ temp = receiver->klass_ 7174 __ LoadFromOffset(kLoadWord, temp, receiver, class_offset); 7175 MaybeRecordImplicitNullCheck(invoke); 7176 // Instead of simply (possibly) unpoisoning `temp` here, we should 7177 // emit a read barrier for the previous class reference load. 7178 // However this is not required in practice, as this is an 7179 // intermediate/temporary reference and because the current 7180 // concurrent copying collector keeps the from-space memory 7181 // intact/accessible until the end of the marking phase (the 7182 // concurrent copying collector may not in the future). 7183 __ MaybeUnpoisonHeapReference(temp); 7184 // temp = temp->GetMethodAt(method_offset); 7185 uint32_t entry_point = ArtMethod::EntryPointFromQuickCompiledCodeOffset( 7186 kArmPointerSize).Int32Value(); 7187 __ LoadFromOffset(kLoadWord, temp, temp, method_offset); 7188 // LR = temp->GetEntryPoint(); 7189 __ LoadFromOffset(kLoadWord, LR, temp, entry_point); 7190 // LR(); 7191 __ blx(LR); 7192} 7193 7194CodeGeneratorARM::PcRelativePatchInfo* CodeGeneratorARM::NewPcRelativeStringPatch( 7195 const DexFile& dex_file, uint32_t string_index) { 7196 return NewPcRelativePatch(dex_file, string_index, &pc_relative_string_patches_); 7197} 7198 7199CodeGeneratorARM::PcRelativePatchInfo* CodeGeneratorARM::NewPcRelativeTypePatch( 7200 const DexFile& dex_file, uint32_t type_index) { 7201 return NewPcRelativePatch(dex_file, type_index, &pc_relative_type_patches_); 7202} 7203 7204CodeGeneratorARM::PcRelativePatchInfo* CodeGeneratorARM::NewPcRelativeDexCacheArrayPatch( 7205 const DexFile& dex_file, uint32_t element_offset) { 7206 return NewPcRelativePatch(dex_file, element_offset, &pc_relative_dex_cache_patches_); 7207} 7208 7209CodeGeneratorARM::PcRelativePatchInfo* CodeGeneratorARM::NewPcRelativePatch( 7210 const DexFile& dex_file, uint32_t offset_or_index, ArenaDeque<PcRelativePatchInfo>* patches) { 7211 patches->emplace_back(dex_file, offset_or_index); 7212 return &patches->back(); 7213} 7214 7215Literal* CodeGeneratorARM::DeduplicateBootImageStringLiteral(const DexFile& dex_file, 7216 uint32_t string_index) { 7217 return boot_image_string_patches_.GetOrCreate( 7218 StringReference(&dex_file, string_index), 7219 [this]() { return __ NewLiteral<uint32_t>(/* placeholder */ 0u); }); 7220} 7221 7222Literal* CodeGeneratorARM::DeduplicateBootImageTypeLiteral(const DexFile& dex_file, 7223 uint32_t type_index) { 7224 return boot_image_type_patches_.GetOrCreate( 7225 TypeReference(&dex_file, type_index), 7226 [this]() { return __ NewLiteral<uint32_t>(/* placeholder */ 0u); }); 7227} 7228 7229Literal* CodeGeneratorARM::DeduplicateBootImageAddressLiteral(uint32_t address) { 7230 bool needs_patch = GetCompilerOptions().GetIncludePatchInformation(); 7231 Uint32ToLiteralMap* map = needs_patch ? &boot_image_address_patches_ : &uint32_literals_; 7232 return DeduplicateUint32Literal(dchecked_integral_cast<uint32_t>(address), map); 7233} 7234 7235Literal* CodeGeneratorARM::DeduplicateDexCacheAddressLiteral(uint32_t address) { 7236 return DeduplicateUint32Literal(address, &uint32_literals_); 7237} 7238 7239template <LinkerPatch (*Factory)(size_t, const DexFile*, uint32_t, uint32_t)> 7240inline void CodeGeneratorARM::EmitPcRelativeLinkerPatches( 7241 const ArenaDeque<PcRelativePatchInfo>& infos, 7242 ArenaVector<LinkerPatch>* linker_patches) { 7243 for (const PcRelativePatchInfo& info : infos) { 7244 const DexFile& dex_file = info.target_dex_file; 7245 size_t offset_or_index = info.offset_or_index; 7246 DCHECK(info.add_pc_label.IsBound()); 7247 uint32_t add_pc_offset = dchecked_integral_cast<uint32_t>(info.add_pc_label.Position()); 7248 // Add MOVW patch. 7249 DCHECK(info.movw_label.IsBound()); 7250 uint32_t movw_offset = dchecked_integral_cast<uint32_t>(info.movw_label.Position()); 7251 linker_patches->push_back(Factory(movw_offset, &dex_file, add_pc_offset, offset_or_index)); 7252 // Add MOVT patch. 7253 DCHECK(info.movt_label.IsBound()); 7254 uint32_t movt_offset = dchecked_integral_cast<uint32_t>(info.movt_label.Position()); 7255 linker_patches->push_back(Factory(movt_offset, &dex_file, add_pc_offset, offset_or_index)); 7256 } 7257} 7258 7259void CodeGeneratorARM::EmitLinkerPatches(ArenaVector<LinkerPatch>* linker_patches) { 7260 DCHECK(linker_patches->empty()); 7261 size_t size = 7262 method_patches_.size() + 7263 call_patches_.size() + 7264 relative_call_patches_.size() + 7265 /* MOVW+MOVT for each entry */ 2u * pc_relative_dex_cache_patches_.size() + 7266 boot_image_string_patches_.size() + 7267 /* MOVW+MOVT for each entry */ 2u * pc_relative_string_patches_.size() + 7268 boot_image_type_patches_.size() + 7269 /* MOVW+MOVT for each entry */ 2u * pc_relative_type_patches_.size() + 7270 boot_image_address_patches_.size(); 7271 linker_patches->reserve(size); 7272 for (const auto& entry : method_patches_) { 7273 const MethodReference& target_method = entry.first; 7274 Literal* literal = entry.second; 7275 DCHECK(literal->GetLabel()->IsBound()); 7276 uint32_t literal_offset = literal->GetLabel()->Position(); 7277 linker_patches->push_back(LinkerPatch::MethodPatch(literal_offset, 7278 target_method.dex_file, 7279 target_method.dex_method_index)); 7280 } 7281 for (const auto& entry : call_patches_) { 7282 const MethodReference& target_method = entry.first; 7283 Literal* literal = entry.second; 7284 DCHECK(literal->GetLabel()->IsBound()); 7285 uint32_t literal_offset = literal->GetLabel()->Position(); 7286 linker_patches->push_back(LinkerPatch::CodePatch(literal_offset, 7287 target_method.dex_file, 7288 target_method.dex_method_index)); 7289 } 7290 for (const PatchInfo<Label>& info : relative_call_patches_) { 7291 uint32_t literal_offset = info.label.Position(); 7292 linker_patches->push_back( 7293 LinkerPatch::RelativeCodePatch(literal_offset, &info.dex_file, info.index)); 7294 } 7295 EmitPcRelativeLinkerPatches<LinkerPatch::DexCacheArrayPatch>(pc_relative_dex_cache_patches_, 7296 linker_patches); 7297 for (const auto& entry : boot_image_string_patches_) { 7298 const StringReference& target_string = entry.first; 7299 Literal* literal = entry.second; 7300 DCHECK(literal->GetLabel()->IsBound()); 7301 uint32_t literal_offset = literal->GetLabel()->Position(); 7302 linker_patches->push_back(LinkerPatch::StringPatch(literal_offset, 7303 target_string.dex_file, 7304 target_string.string_index)); 7305 } 7306 if (!GetCompilerOptions().IsBootImage()) { 7307 EmitPcRelativeLinkerPatches<LinkerPatch::StringBssEntryPatch>(pc_relative_string_patches_, 7308 linker_patches); 7309 } else { 7310 EmitPcRelativeLinkerPatches<LinkerPatch::RelativeStringPatch>(pc_relative_string_patches_, 7311 linker_patches); 7312 } 7313 for (const auto& entry : boot_image_type_patches_) { 7314 const TypeReference& target_type = entry.first; 7315 Literal* literal = entry.second; 7316 DCHECK(literal->GetLabel()->IsBound()); 7317 uint32_t literal_offset = literal->GetLabel()->Position(); 7318 linker_patches->push_back(LinkerPatch::TypePatch(literal_offset, 7319 target_type.dex_file, 7320 target_type.type_index)); 7321 } 7322 EmitPcRelativeLinkerPatches<LinkerPatch::RelativeTypePatch>(pc_relative_type_patches_, 7323 linker_patches); 7324 for (const auto& entry : boot_image_address_patches_) { 7325 DCHECK(GetCompilerOptions().GetIncludePatchInformation()); 7326 Literal* literal = entry.second; 7327 DCHECK(literal->GetLabel()->IsBound()); 7328 uint32_t literal_offset = literal->GetLabel()->Position(); 7329 linker_patches->push_back(LinkerPatch::RecordPosition(literal_offset)); 7330 } 7331} 7332 7333Literal* CodeGeneratorARM::DeduplicateUint32Literal(uint32_t value, Uint32ToLiteralMap* map) { 7334 return map->GetOrCreate( 7335 value, 7336 [this, value]() { return __ NewLiteral<uint32_t>(value); }); 7337} 7338 7339Literal* CodeGeneratorARM::DeduplicateMethodLiteral(MethodReference target_method, 7340 MethodToLiteralMap* map) { 7341 return map->GetOrCreate( 7342 target_method, 7343 [this]() { return __ NewLiteral<uint32_t>(/* placeholder */ 0u); }); 7344} 7345 7346Literal* CodeGeneratorARM::DeduplicateMethodAddressLiteral(MethodReference target_method) { 7347 return DeduplicateMethodLiteral(target_method, &method_patches_); 7348} 7349 7350Literal* CodeGeneratorARM::DeduplicateMethodCodeLiteral(MethodReference target_method) { 7351 return DeduplicateMethodLiteral(target_method, &call_patches_); 7352} 7353 7354void LocationsBuilderARM::VisitMultiplyAccumulate(HMultiplyAccumulate* instr) { 7355 LocationSummary* locations = 7356 new (GetGraph()->GetArena()) LocationSummary(instr, LocationSummary::kNoCall); 7357 locations->SetInAt(HMultiplyAccumulate::kInputAccumulatorIndex, 7358 Location::RequiresRegister()); 7359 locations->SetInAt(HMultiplyAccumulate::kInputMulLeftIndex, Location::RequiresRegister()); 7360 locations->SetInAt(HMultiplyAccumulate::kInputMulRightIndex, Location::RequiresRegister()); 7361 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 7362} 7363 7364void InstructionCodeGeneratorARM::VisitMultiplyAccumulate(HMultiplyAccumulate* instr) { 7365 LocationSummary* locations = instr->GetLocations(); 7366 Register res = locations->Out().AsRegister<Register>(); 7367 Register accumulator = 7368 locations->InAt(HMultiplyAccumulate::kInputAccumulatorIndex).AsRegister<Register>(); 7369 Register mul_left = 7370 locations->InAt(HMultiplyAccumulate::kInputMulLeftIndex).AsRegister<Register>(); 7371 Register mul_right = 7372 locations->InAt(HMultiplyAccumulate::kInputMulRightIndex).AsRegister<Register>(); 7373 7374 if (instr->GetOpKind() == HInstruction::kAdd) { 7375 __ mla(res, mul_left, mul_right, accumulator); 7376 } else { 7377 __ mls(res, mul_left, mul_right, accumulator); 7378 } 7379} 7380 7381void LocationsBuilderARM::VisitBoundType(HBoundType* instruction ATTRIBUTE_UNUSED) { 7382 // Nothing to do, this should be removed during prepare for register allocator. 7383 LOG(FATAL) << "Unreachable"; 7384} 7385 7386void InstructionCodeGeneratorARM::VisitBoundType(HBoundType* instruction ATTRIBUTE_UNUSED) { 7387 // Nothing to do, this should be removed during prepare for register allocator. 7388 LOG(FATAL) << "Unreachable"; 7389} 7390 7391// Simple implementation of packed switch - generate cascaded compare/jumps. 7392void LocationsBuilderARM::VisitPackedSwitch(HPackedSwitch* switch_instr) { 7393 LocationSummary* locations = 7394 new (GetGraph()->GetArena()) LocationSummary(switch_instr, LocationSummary::kNoCall); 7395 locations->SetInAt(0, Location::RequiresRegister()); 7396 if (switch_instr->GetNumEntries() > kPackedSwitchCompareJumpThreshold && 7397 codegen_->GetAssembler()->IsThumb()) { 7398 locations->AddTemp(Location::RequiresRegister()); // We need a temp for the table base. 7399 if (switch_instr->GetStartValue() != 0) { 7400 locations->AddTemp(Location::RequiresRegister()); // We need a temp for the bias. 7401 } 7402 } 7403} 7404 7405void InstructionCodeGeneratorARM::VisitPackedSwitch(HPackedSwitch* switch_instr) { 7406 int32_t lower_bound = switch_instr->GetStartValue(); 7407 uint32_t num_entries = switch_instr->GetNumEntries(); 7408 LocationSummary* locations = switch_instr->GetLocations(); 7409 Register value_reg = locations->InAt(0).AsRegister<Register>(); 7410 HBasicBlock* default_block = switch_instr->GetDefaultBlock(); 7411 7412 if (num_entries <= kPackedSwitchCompareJumpThreshold || !codegen_->GetAssembler()->IsThumb()) { 7413 // Create a series of compare/jumps. 7414 Register temp_reg = IP; 7415 // Note: It is fine for the below AddConstantSetFlags() using IP register to temporarily store 7416 // the immediate, because IP is used as the destination register. For the other 7417 // AddConstantSetFlags() and GenerateCompareWithImmediate(), the immediate values are constant, 7418 // and they can be encoded in the instruction without making use of IP register. 7419 __ AddConstantSetFlags(temp_reg, value_reg, -lower_bound); 7420 7421 const ArenaVector<HBasicBlock*>& successors = switch_instr->GetBlock()->GetSuccessors(); 7422 // Jump to successors[0] if value == lower_bound. 7423 __ b(codegen_->GetLabelOf(successors[0]), EQ); 7424 int32_t last_index = 0; 7425 for (; num_entries - last_index > 2; last_index += 2) { 7426 __ AddConstantSetFlags(temp_reg, temp_reg, -2); 7427 // Jump to successors[last_index + 1] if value < case_value[last_index + 2]. 7428 __ b(codegen_->GetLabelOf(successors[last_index + 1]), LO); 7429 // Jump to successors[last_index + 2] if value == case_value[last_index + 2]. 7430 __ b(codegen_->GetLabelOf(successors[last_index + 2]), EQ); 7431 } 7432 if (num_entries - last_index == 2) { 7433 // The last missing case_value. 7434 __ CmpConstant(temp_reg, 1); 7435 __ b(codegen_->GetLabelOf(successors[last_index + 1]), EQ); 7436 } 7437 7438 // And the default for any other value. 7439 if (!codegen_->GoesToNextBlock(switch_instr->GetBlock(), default_block)) { 7440 __ b(codegen_->GetLabelOf(default_block)); 7441 } 7442 } else { 7443 // Create a table lookup. 7444 Register temp_reg = locations->GetTemp(0).AsRegister<Register>(); 7445 7446 // Materialize a pointer to the switch table 7447 std::vector<Label*> labels(num_entries); 7448 const ArenaVector<HBasicBlock*>& successors = switch_instr->GetBlock()->GetSuccessors(); 7449 for (uint32_t i = 0; i < num_entries; i++) { 7450 labels[i] = codegen_->GetLabelOf(successors[i]); 7451 } 7452 JumpTable* table = __ CreateJumpTable(std::move(labels), temp_reg); 7453 7454 // Remove the bias. 7455 Register key_reg; 7456 if (lower_bound != 0) { 7457 key_reg = locations->GetTemp(1).AsRegister<Register>(); 7458 __ AddConstant(key_reg, value_reg, -lower_bound); 7459 } else { 7460 key_reg = value_reg; 7461 } 7462 7463 // Check whether the value is in the table, jump to default block if not. 7464 __ CmpConstant(key_reg, num_entries - 1); 7465 __ b(codegen_->GetLabelOf(default_block), Condition::HI); 7466 7467 // Load the displacement from the table. 7468 __ ldr(temp_reg, Address(temp_reg, key_reg, Shift::LSL, 2)); 7469 7470 // Dispatch is a direct add to the PC (for Thumb2). 7471 __ EmitJumpTableDispatch(table, temp_reg); 7472 } 7473} 7474 7475void LocationsBuilderARM::VisitArmDexCacheArraysBase(HArmDexCacheArraysBase* base) { 7476 LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(base); 7477 locations->SetOut(Location::RequiresRegister()); 7478} 7479 7480void InstructionCodeGeneratorARM::VisitArmDexCacheArraysBase(HArmDexCacheArraysBase* base) { 7481 Register base_reg = base->GetLocations()->Out().AsRegister<Register>(); 7482 CodeGeneratorARM::PcRelativePatchInfo* labels = 7483 codegen_->NewPcRelativeDexCacheArrayPatch(base->GetDexFile(), base->GetElementOffset()); 7484 __ BindTrackedLabel(&labels->movw_label); 7485 __ movw(base_reg, /* placeholder */ 0u); 7486 __ BindTrackedLabel(&labels->movt_label); 7487 __ movt(base_reg, /* placeholder */ 0u); 7488 __ BindTrackedLabel(&labels->add_pc_label); 7489 __ add(base_reg, base_reg, ShifterOperand(PC)); 7490} 7491 7492void CodeGeneratorARM::MoveFromReturnRegister(Location trg, Primitive::Type type) { 7493 if (!trg.IsValid()) { 7494 DCHECK_EQ(type, Primitive::kPrimVoid); 7495 return; 7496 } 7497 7498 DCHECK_NE(type, Primitive::kPrimVoid); 7499 7500 Location return_loc = InvokeDexCallingConventionVisitorARM().GetReturnLocation(type); 7501 if (return_loc.Equals(trg)) { 7502 return; 7503 } 7504 7505 // TODO: Consider pairs in the parallel move resolver, then this could be nicely merged 7506 // with the last branch. 7507 if (type == Primitive::kPrimLong) { 7508 HParallelMove parallel_move(GetGraph()->GetArena()); 7509 parallel_move.AddMove(return_loc.ToLow(), trg.ToLow(), Primitive::kPrimInt, nullptr); 7510 parallel_move.AddMove(return_loc.ToHigh(), trg.ToHigh(), Primitive::kPrimInt, nullptr); 7511 GetMoveResolver()->EmitNativeCode(¶llel_move); 7512 } else if (type == Primitive::kPrimDouble) { 7513 HParallelMove parallel_move(GetGraph()->GetArena()); 7514 parallel_move.AddMove(return_loc.ToLow(), trg.ToLow(), Primitive::kPrimFloat, nullptr); 7515 parallel_move.AddMove(return_loc.ToHigh(), trg.ToHigh(), Primitive::kPrimFloat, nullptr); 7516 GetMoveResolver()->EmitNativeCode(¶llel_move); 7517 } else { 7518 // Let the parallel move resolver take care of all of this. 7519 HParallelMove parallel_move(GetGraph()->GetArena()); 7520 parallel_move.AddMove(return_loc, trg, type, nullptr); 7521 GetMoveResolver()->EmitNativeCode(¶llel_move); 7522 } 7523} 7524 7525void LocationsBuilderARM::VisitClassTableGet(HClassTableGet* instruction) { 7526 LocationSummary* locations = 7527 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall); 7528 locations->SetInAt(0, Location::RequiresRegister()); 7529 locations->SetOut(Location::RequiresRegister()); 7530} 7531 7532void InstructionCodeGeneratorARM::VisitClassTableGet(HClassTableGet* instruction) { 7533 LocationSummary* locations = instruction->GetLocations(); 7534 if (instruction->GetTableKind() == HClassTableGet::TableKind::kVTable) { 7535 uint32_t method_offset = mirror::Class::EmbeddedVTableEntryOffset( 7536 instruction->GetIndex(), kArmPointerSize).SizeValue(); 7537 __ LoadFromOffset(kLoadWord, 7538 locations->Out().AsRegister<Register>(), 7539 locations->InAt(0).AsRegister<Register>(), 7540 method_offset); 7541 } else { 7542 uint32_t method_offset = static_cast<uint32_t>(ImTable::OffsetOfElement( 7543 instruction->GetIndex(), kArmPointerSize)); 7544 __ LoadFromOffset(kLoadWord, 7545 locations->Out().AsRegister<Register>(), 7546 locations->InAt(0).AsRegister<Register>(), 7547 mirror::Class::ImtPtrOffset(kArmPointerSize).Uint32Value()); 7548 __ LoadFromOffset(kLoadWord, 7549 locations->Out().AsRegister<Register>(), 7550 locations->Out().AsRegister<Register>(), 7551 method_offset); 7552 } 7553} 7554 7555#undef __ 7556#undef QUICK_ENTRY_POINT 7557 7558} // namespace arm 7559} // namespace art 7560