code_generator_arm.cc revision c1a42cf3873be202c8c0ca3c4e67500b470ab075
1/* 2 * Copyright (C) 2014 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17#include "code_generator_arm.h" 18 19#include "arch/arm/instruction_set_features_arm.h" 20#include "art_method.h" 21#include "code_generator_utils.h" 22#include "compiled_method.h" 23#include "entrypoints/quick/quick_entrypoints.h" 24#include "gc/accounting/card_table.h" 25#include "intrinsics.h" 26#include "intrinsics_arm.h" 27#include "mirror/array-inl.h" 28#include "mirror/class-inl.h" 29#include "thread.h" 30#include "utils/arm/assembler_arm.h" 31#include "utils/arm/managed_register_arm.h" 32#include "utils/assembler.h" 33#include "utils/stack_checks.h" 34 35namespace art { 36 37template<class MirrorType> 38class GcRoot; 39 40namespace arm { 41 42static bool ExpectedPairLayout(Location location) { 43 // We expected this for both core and fpu register pairs. 44 return ((location.low() & 1) == 0) && (location.low() + 1 == location.high()); 45} 46 47static constexpr int kCurrentMethodStackOffset = 0; 48static constexpr Register kMethodRegisterArgument = R0; 49 50static constexpr Register kCoreAlwaysSpillRegister = R5; 51static constexpr Register kCoreCalleeSaves[] = 52 { R5, R6, R7, R8, R10, R11, LR }; 53static constexpr SRegister kFpuCalleeSaves[] = 54 { S16, S17, S18, S19, S20, S21, S22, S23, S24, S25, S26, S27, S28, S29, S30, S31 }; 55 56// D31 cannot be split into two S registers, and the register allocator only works on 57// S registers. Therefore there is no need to block it. 58static constexpr DRegister DTMP = D31; 59 60static constexpr uint32_t kPackedSwitchCompareJumpThreshold = 7; 61 62// NOLINT on __ macro to suppress wrong warning/fix (misc-macro-parentheses) from clang-tidy. 63#define __ down_cast<ArmAssembler*>(codegen->GetAssembler())-> // NOLINT 64#define QUICK_ENTRY_POINT(x) QUICK_ENTRYPOINT_OFFSET(kArmPointerSize, x).Int32Value() 65 66static constexpr int kRegListThreshold = 4; 67 68// SaveLiveRegisters and RestoreLiveRegisters from SlowPathCodeARM operate on sets of S registers, 69// for each live D registers they treat two corresponding S registers as live ones. 70// 71// Two following functions (SaveContiguousSRegisterList, RestoreContiguousSRegisterList) build 72// from a list of contiguous S registers a list of contiguous D registers (processing first/last 73// S registers corner cases) and save/restore this new list treating them as D registers. 74// - decreasing code size 75// - avoiding hazards on Cortex-A57, when a pair of S registers for an actual live D register is 76// restored and then used in regular non SlowPath code as D register. 77// 78// For the following example (v means the S register is live): 79// D names: | D0 | D1 | D2 | D4 | ... 80// S names: | S0 | S1 | S2 | S3 | S4 | S5 | S6 | S7 | ... 81// Live? | | v | v | v | v | v | v | | ... 82// 83// S1 and S6 will be saved/restored independently; D registers list (D1, D2) will be processed 84// as D registers. 85static size_t SaveContiguousSRegisterList(size_t first, 86 size_t last, 87 CodeGenerator* codegen, 88 size_t stack_offset) { 89 DCHECK_LE(first, last); 90 if ((first == last) && (first == 0)) { 91 stack_offset += codegen->SaveFloatingPointRegister(stack_offset, first); 92 return stack_offset; 93 } 94 if (first % 2 == 1) { 95 stack_offset += codegen->SaveFloatingPointRegister(stack_offset, first++); 96 } 97 98 bool save_last = false; 99 if (last % 2 == 0) { 100 save_last = true; 101 --last; 102 } 103 104 if (first < last) { 105 DRegister d_reg = static_cast<DRegister>(first / 2); 106 DCHECK_EQ((last - first + 1) % 2, 0u); 107 size_t number_of_d_regs = (last - first + 1) / 2; 108 109 if (number_of_d_regs == 1) { 110 __ StoreDToOffset(d_reg, SP, stack_offset); 111 } else if (number_of_d_regs > 1) { 112 __ add(IP, SP, ShifterOperand(stack_offset)); 113 __ vstmiad(IP, d_reg, number_of_d_regs); 114 } 115 stack_offset += number_of_d_regs * kArmWordSize * 2; 116 } 117 118 if (save_last) { 119 stack_offset += codegen->SaveFloatingPointRegister(stack_offset, last + 1); 120 } 121 122 return stack_offset; 123} 124 125static size_t RestoreContiguousSRegisterList(size_t first, 126 size_t last, 127 CodeGenerator* codegen, 128 size_t stack_offset) { 129 DCHECK_LE(first, last); 130 if ((first == last) && (first == 0)) { 131 stack_offset += codegen->RestoreFloatingPointRegister(stack_offset, first); 132 return stack_offset; 133 } 134 if (first % 2 == 1) { 135 stack_offset += codegen->RestoreFloatingPointRegister(stack_offset, first++); 136 } 137 138 bool restore_last = false; 139 if (last % 2 == 0) { 140 restore_last = true; 141 --last; 142 } 143 144 if (first < last) { 145 DRegister d_reg = static_cast<DRegister>(first / 2); 146 DCHECK_EQ((last - first + 1) % 2, 0u); 147 size_t number_of_d_regs = (last - first + 1) / 2; 148 if (number_of_d_regs == 1) { 149 __ LoadDFromOffset(d_reg, SP, stack_offset); 150 } else if (number_of_d_regs > 1) { 151 __ add(IP, SP, ShifterOperand(stack_offset)); 152 __ vldmiad(IP, d_reg, number_of_d_regs); 153 } 154 stack_offset += number_of_d_regs * kArmWordSize * 2; 155 } 156 157 if (restore_last) { 158 stack_offset += codegen->RestoreFloatingPointRegister(stack_offset, last + 1); 159 } 160 161 return stack_offset; 162} 163 164void SlowPathCodeARM::SaveLiveRegisters(CodeGenerator* codegen, LocationSummary* locations) { 165 size_t stack_offset = codegen->GetFirstRegisterSlotInSlowPath(); 166 size_t orig_offset = stack_offset; 167 168 const uint32_t core_spills = codegen->GetSlowPathSpills(locations, /* core_registers */ true); 169 for (uint32_t i : LowToHighBits(core_spills)) { 170 // If the register holds an object, update the stack mask. 171 if (locations->RegisterContainsObject(i)) { 172 locations->SetStackBit(stack_offset / kVRegSize); 173 } 174 DCHECK_LT(stack_offset, codegen->GetFrameSize() - codegen->FrameEntrySpillSize()); 175 DCHECK_LT(i, kMaximumNumberOfExpectedRegisters); 176 saved_core_stack_offsets_[i] = stack_offset; 177 stack_offset += kArmWordSize; 178 } 179 180 int reg_num = POPCOUNT(core_spills); 181 if (reg_num != 0) { 182 if (reg_num > kRegListThreshold) { 183 __ StoreList(RegList(core_spills), orig_offset); 184 } else { 185 stack_offset = orig_offset; 186 for (uint32_t i : LowToHighBits(core_spills)) { 187 stack_offset += codegen->SaveCoreRegister(stack_offset, i); 188 } 189 } 190 } 191 192 uint32_t fp_spills = codegen->GetSlowPathSpills(locations, /* core_registers */ false); 193 orig_offset = stack_offset; 194 for (uint32_t i : LowToHighBits(fp_spills)) { 195 DCHECK_LT(i, kMaximumNumberOfExpectedRegisters); 196 saved_fpu_stack_offsets_[i] = stack_offset; 197 stack_offset += kArmWordSize; 198 } 199 200 stack_offset = orig_offset; 201 while (fp_spills != 0u) { 202 uint32_t begin = CTZ(fp_spills); 203 uint32_t tmp = fp_spills + (1u << begin); 204 fp_spills &= tmp; // Clear the contiguous range of 1s. 205 uint32_t end = (tmp == 0u) ? 32u : CTZ(tmp); // CTZ(0) is undefined. 206 stack_offset = SaveContiguousSRegisterList(begin, end - 1, codegen, stack_offset); 207 } 208 DCHECK_LE(stack_offset, codegen->GetFrameSize() - codegen->FrameEntrySpillSize()); 209} 210 211void SlowPathCodeARM::RestoreLiveRegisters(CodeGenerator* codegen, LocationSummary* locations) { 212 size_t stack_offset = codegen->GetFirstRegisterSlotInSlowPath(); 213 size_t orig_offset = stack_offset; 214 215 const uint32_t core_spills = codegen->GetSlowPathSpills(locations, /* core_registers */ true); 216 for (uint32_t i : LowToHighBits(core_spills)) { 217 DCHECK_LT(stack_offset, codegen->GetFrameSize() - codegen->FrameEntrySpillSize()); 218 DCHECK_LT(i, kMaximumNumberOfExpectedRegisters); 219 stack_offset += kArmWordSize; 220 } 221 222 int reg_num = POPCOUNT(core_spills); 223 if (reg_num != 0) { 224 if (reg_num > kRegListThreshold) { 225 __ LoadList(RegList(core_spills), orig_offset); 226 } else { 227 stack_offset = orig_offset; 228 for (uint32_t i : LowToHighBits(core_spills)) { 229 stack_offset += codegen->RestoreCoreRegister(stack_offset, i); 230 } 231 } 232 } 233 234 uint32_t fp_spills = codegen->GetSlowPathSpills(locations, /* core_registers */ false); 235 while (fp_spills != 0u) { 236 uint32_t begin = CTZ(fp_spills); 237 uint32_t tmp = fp_spills + (1u << begin); 238 fp_spills &= tmp; // Clear the contiguous range of 1s. 239 uint32_t end = (tmp == 0u) ? 32u : CTZ(tmp); // CTZ(0) is undefined. 240 stack_offset = RestoreContiguousSRegisterList(begin, end - 1, codegen, stack_offset); 241 } 242 DCHECK_LE(stack_offset, codegen->GetFrameSize() - codegen->FrameEntrySpillSize()); 243} 244 245class NullCheckSlowPathARM : public SlowPathCodeARM { 246 public: 247 explicit NullCheckSlowPathARM(HNullCheck* instruction) : SlowPathCodeARM(instruction) {} 248 249 void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { 250 CodeGeneratorARM* arm_codegen = down_cast<CodeGeneratorARM*>(codegen); 251 __ Bind(GetEntryLabel()); 252 if (instruction_->CanThrowIntoCatchBlock()) { 253 // Live registers will be restored in the catch block if caught. 254 SaveLiveRegisters(codegen, instruction_->GetLocations()); 255 } 256 arm_codegen->InvokeRuntime(kQuickThrowNullPointer, 257 instruction_, 258 instruction_->GetDexPc(), 259 this); 260 CheckEntrypointTypes<kQuickThrowNullPointer, void, void>(); 261 } 262 263 bool IsFatal() const OVERRIDE { return true; } 264 265 const char* GetDescription() const OVERRIDE { return "NullCheckSlowPathARM"; } 266 267 private: 268 DISALLOW_COPY_AND_ASSIGN(NullCheckSlowPathARM); 269}; 270 271class DivZeroCheckSlowPathARM : public SlowPathCodeARM { 272 public: 273 explicit DivZeroCheckSlowPathARM(HDivZeroCheck* instruction) : SlowPathCodeARM(instruction) {} 274 275 void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { 276 CodeGeneratorARM* arm_codegen = down_cast<CodeGeneratorARM*>(codegen); 277 __ Bind(GetEntryLabel()); 278 arm_codegen->InvokeRuntime(kQuickThrowDivZero, instruction_, instruction_->GetDexPc(), this); 279 CheckEntrypointTypes<kQuickThrowDivZero, void, void>(); 280 } 281 282 bool IsFatal() const OVERRIDE { return true; } 283 284 const char* GetDescription() const OVERRIDE { return "DivZeroCheckSlowPathARM"; } 285 286 private: 287 DISALLOW_COPY_AND_ASSIGN(DivZeroCheckSlowPathARM); 288}; 289 290class SuspendCheckSlowPathARM : public SlowPathCodeARM { 291 public: 292 SuspendCheckSlowPathARM(HSuspendCheck* instruction, HBasicBlock* successor) 293 : SlowPathCodeARM(instruction), successor_(successor) {} 294 295 void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { 296 CodeGeneratorARM* arm_codegen = down_cast<CodeGeneratorARM*>(codegen); 297 __ Bind(GetEntryLabel()); 298 arm_codegen->InvokeRuntime(kQuickTestSuspend, instruction_, instruction_->GetDexPc(), this); 299 CheckEntrypointTypes<kQuickTestSuspend, void, void>(); 300 if (successor_ == nullptr) { 301 __ b(GetReturnLabel()); 302 } else { 303 __ b(arm_codegen->GetLabelOf(successor_)); 304 } 305 } 306 307 Label* GetReturnLabel() { 308 DCHECK(successor_ == nullptr); 309 return &return_label_; 310 } 311 312 HBasicBlock* GetSuccessor() const { 313 return successor_; 314 } 315 316 const char* GetDescription() const OVERRIDE { return "SuspendCheckSlowPathARM"; } 317 318 private: 319 // If not null, the block to branch to after the suspend check. 320 HBasicBlock* const successor_; 321 322 // If `successor_` is null, the label to branch to after the suspend check. 323 Label return_label_; 324 325 DISALLOW_COPY_AND_ASSIGN(SuspendCheckSlowPathARM); 326}; 327 328class BoundsCheckSlowPathARM : public SlowPathCodeARM { 329 public: 330 explicit BoundsCheckSlowPathARM(HBoundsCheck* instruction) 331 : SlowPathCodeARM(instruction) {} 332 333 void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { 334 CodeGeneratorARM* arm_codegen = down_cast<CodeGeneratorARM*>(codegen); 335 LocationSummary* locations = instruction_->GetLocations(); 336 337 __ Bind(GetEntryLabel()); 338 if (instruction_->CanThrowIntoCatchBlock()) { 339 // Live registers will be restored in the catch block if caught. 340 SaveLiveRegisters(codegen, instruction_->GetLocations()); 341 } 342 // We're moving two locations to locations that could overlap, so we need a parallel 343 // move resolver. 344 InvokeRuntimeCallingConvention calling_convention; 345 codegen->EmitParallelMoves( 346 locations->InAt(0), 347 Location::RegisterLocation(calling_convention.GetRegisterAt(0)), 348 Primitive::kPrimInt, 349 locations->InAt(1), 350 Location::RegisterLocation(calling_convention.GetRegisterAt(1)), 351 Primitive::kPrimInt); 352 QuickEntrypointEnum entrypoint = instruction_->AsBoundsCheck()->IsStringCharAt() 353 ? kQuickThrowStringBounds 354 : kQuickThrowArrayBounds; 355 arm_codegen->InvokeRuntime(entrypoint, instruction_, instruction_->GetDexPc(), this); 356 CheckEntrypointTypes<kQuickThrowStringBounds, void, int32_t, int32_t>(); 357 CheckEntrypointTypes<kQuickThrowArrayBounds, void, int32_t, int32_t>(); 358 } 359 360 bool IsFatal() const OVERRIDE { return true; } 361 362 const char* GetDescription() const OVERRIDE { return "BoundsCheckSlowPathARM"; } 363 364 private: 365 DISALLOW_COPY_AND_ASSIGN(BoundsCheckSlowPathARM); 366}; 367 368class LoadClassSlowPathARM : public SlowPathCodeARM { 369 public: 370 LoadClassSlowPathARM(HLoadClass* cls, 371 HInstruction* at, 372 uint32_t dex_pc, 373 bool do_clinit) 374 : SlowPathCodeARM(at), cls_(cls), at_(at), dex_pc_(dex_pc), do_clinit_(do_clinit) { 375 DCHECK(at->IsLoadClass() || at->IsClinitCheck()); 376 } 377 378 void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { 379 LocationSummary* locations = at_->GetLocations(); 380 381 CodeGeneratorARM* arm_codegen = down_cast<CodeGeneratorARM*>(codegen); 382 __ Bind(GetEntryLabel()); 383 SaveLiveRegisters(codegen, locations); 384 385 InvokeRuntimeCallingConvention calling_convention; 386 __ LoadImmediate(calling_convention.GetRegisterAt(0), cls_->GetTypeIndex().index_); 387 QuickEntrypointEnum entrypoint = do_clinit_ ? kQuickInitializeStaticStorage 388 : kQuickInitializeType; 389 arm_codegen->InvokeRuntime(entrypoint, at_, dex_pc_, this); 390 if (do_clinit_) { 391 CheckEntrypointTypes<kQuickInitializeStaticStorage, void*, uint32_t>(); 392 } else { 393 CheckEntrypointTypes<kQuickInitializeType, void*, uint32_t>(); 394 } 395 396 // Move the class to the desired location. 397 Location out = locations->Out(); 398 if (out.IsValid()) { 399 DCHECK(out.IsRegister() && !locations->GetLiveRegisters()->ContainsCoreRegister(out.reg())); 400 arm_codegen->Move32(locations->Out(), Location::RegisterLocation(R0)); 401 } 402 RestoreLiveRegisters(codegen, locations); 403 __ b(GetExitLabel()); 404 } 405 406 const char* GetDescription() const OVERRIDE { return "LoadClassSlowPathARM"; } 407 408 private: 409 // The class this slow path will load. 410 HLoadClass* const cls_; 411 412 // The instruction where this slow path is happening. 413 // (Might be the load class or an initialization check). 414 HInstruction* const at_; 415 416 // The dex PC of `at_`. 417 const uint32_t dex_pc_; 418 419 // Whether to initialize the class. 420 const bool do_clinit_; 421 422 DISALLOW_COPY_AND_ASSIGN(LoadClassSlowPathARM); 423}; 424 425class LoadStringSlowPathARM : public SlowPathCodeARM { 426 public: 427 explicit LoadStringSlowPathARM(HLoadString* instruction) : SlowPathCodeARM(instruction) {} 428 429 void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { 430 LocationSummary* locations = instruction_->GetLocations(); 431 DCHECK(!locations->GetLiveRegisters()->ContainsCoreRegister(locations->Out().reg())); 432 HLoadString* load = instruction_->AsLoadString(); 433 const uint32_t string_index = load->GetStringIndex().index_; 434 Register out = locations->Out().AsRegister<Register>(); 435 Register temp = locations->GetTemp(0).AsRegister<Register>(); 436 constexpr bool call_saves_everything_except_r0 = (!kUseReadBarrier || kUseBakerReadBarrier); 437 438 CodeGeneratorARM* arm_codegen = down_cast<CodeGeneratorARM*>(codegen); 439 __ Bind(GetEntryLabel()); 440 SaveLiveRegisters(codegen, locations); 441 442 InvokeRuntimeCallingConvention calling_convention; 443 // In the unlucky case that the `temp` is R0, we preserve the address in `out` across 444 // the kSaveEverything call (or use `out` for the address after non-kSaveEverything call). 445 bool temp_is_r0 = (temp == calling_convention.GetRegisterAt(0)); 446 Register entry_address = temp_is_r0 ? out : temp; 447 DCHECK_NE(entry_address, calling_convention.GetRegisterAt(0)); 448 if (call_saves_everything_except_r0 && temp_is_r0) { 449 __ mov(entry_address, ShifterOperand(temp)); 450 } 451 452 __ LoadImmediate(calling_convention.GetRegisterAt(0), string_index); 453 arm_codegen->InvokeRuntime(kQuickResolveString, instruction_, instruction_->GetDexPc(), this); 454 CheckEntrypointTypes<kQuickResolveString, void*, uint32_t>(); 455 456 // Store the resolved String to the .bss entry. 457 if (call_saves_everything_except_r0) { 458 // The string entry address was preserved in `entry_address` thanks to kSaveEverything. 459 __ str(R0, Address(entry_address)); 460 } else { 461 // For non-Baker read barrier, we need to re-calculate the address of the string entry. 462 CodeGeneratorARM::PcRelativePatchInfo* labels = 463 arm_codegen->NewPcRelativeStringPatch(load->GetDexFile(), string_index); 464 __ BindTrackedLabel(&labels->movw_label); 465 __ movw(entry_address, /* placeholder */ 0u); 466 __ BindTrackedLabel(&labels->movt_label); 467 __ movt(entry_address, /* placeholder */ 0u); 468 __ BindTrackedLabel(&labels->add_pc_label); 469 __ add(entry_address, entry_address, ShifterOperand(PC)); 470 __ str(R0, Address(entry_address)); 471 } 472 473 arm_codegen->Move32(locations->Out(), Location::RegisterLocation(R0)); 474 RestoreLiveRegisters(codegen, locations); 475 476 __ b(GetExitLabel()); 477 } 478 479 const char* GetDescription() const OVERRIDE { return "LoadStringSlowPathARM"; } 480 481 private: 482 DISALLOW_COPY_AND_ASSIGN(LoadStringSlowPathARM); 483}; 484 485class TypeCheckSlowPathARM : public SlowPathCodeARM { 486 public: 487 TypeCheckSlowPathARM(HInstruction* instruction, bool is_fatal) 488 : SlowPathCodeARM(instruction), is_fatal_(is_fatal) {} 489 490 void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { 491 LocationSummary* locations = instruction_->GetLocations(); 492 DCHECK(instruction_->IsCheckCast() 493 || !locations->GetLiveRegisters()->ContainsCoreRegister(locations->Out().reg())); 494 495 CodeGeneratorARM* arm_codegen = down_cast<CodeGeneratorARM*>(codegen); 496 __ Bind(GetEntryLabel()); 497 498 if (!is_fatal_) { 499 SaveLiveRegisters(codegen, locations); 500 } 501 502 // We're moving two locations to locations that could overlap, so we need a parallel 503 // move resolver. 504 InvokeRuntimeCallingConvention calling_convention; 505 codegen->EmitParallelMoves(locations->InAt(0), 506 Location::RegisterLocation(calling_convention.GetRegisterAt(0)), 507 Primitive::kPrimNot, 508 locations->InAt(1), 509 Location::RegisterLocation(calling_convention.GetRegisterAt(1)), 510 Primitive::kPrimNot); 511 if (instruction_->IsInstanceOf()) { 512 arm_codegen->InvokeRuntime(kQuickInstanceofNonTrivial, 513 instruction_, 514 instruction_->GetDexPc(), 515 this); 516 CheckEntrypointTypes<kQuickInstanceofNonTrivial, size_t, mirror::Object*, mirror::Class*>(); 517 arm_codegen->Move32(locations->Out(), Location::RegisterLocation(R0)); 518 } else { 519 DCHECK(instruction_->IsCheckCast()); 520 arm_codegen->InvokeRuntime(kQuickCheckInstanceOf, 521 instruction_, 522 instruction_->GetDexPc(), 523 this); 524 CheckEntrypointTypes<kQuickCheckInstanceOf, void, mirror::Object*, mirror::Class*>(); 525 } 526 527 if (!is_fatal_) { 528 RestoreLiveRegisters(codegen, locations); 529 __ b(GetExitLabel()); 530 } 531 } 532 533 const char* GetDescription() const OVERRIDE { return "TypeCheckSlowPathARM"; } 534 535 bool IsFatal() const OVERRIDE { return is_fatal_; } 536 537 private: 538 const bool is_fatal_; 539 540 DISALLOW_COPY_AND_ASSIGN(TypeCheckSlowPathARM); 541}; 542 543class DeoptimizationSlowPathARM : public SlowPathCodeARM { 544 public: 545 explicit DeoptimizationSlowPathARM(HDeoptimize* instruction) 546 : SlowPathCodeARM(instruction) {} 547 548 void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { 549 CodeGeneratorARM* arm_codegen = down_cast<CodeGeneratorARM*>(codegen); 550 __ Bind(GetEntryLabel()); 551 arm_codegen->InvokeRuntime(kQuickDeoptimize, instruction_, instruction_->GetDexPc(), this); 552 CheckEntrypointTypes<kQuickDeoptimize, void, void>(); 553 } 554 555 const char* GetDescription() const OVERRIDE { return "DeoptimizationSlowPathARM"; } 556 557 private: 558 DISALLOW_COPY_AND_ASSIGN(DeoptimizationSlowPathARM); 559}; 560 561class ArraySetSlowPathARM : public SlowPathCodeARM { 562 public: 563 explicit ArraySetSlowPathARM(HInstruction* instruction) : SlowPathCodeARM(instruction) {} 564 565 void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { 566 LocationSummary* locations = instruction_->GetLocations(); 567 __ Bind(GetEntryLabel()); 568 SaveLiveRegisters(codegen, locations); 569 570 InvokeRuntimeCallingConvention calling_convention; 571 HParallelMove parallel_move(codegen->GetGraph()->GetArena()); 572 parallel_move.AddMove( 573 locations->InAt(0), 574 Location::RegisterLocation(calling_convention.GetRegisterAt(0)), 575 Primitive::kPrimNot, 576 nullptr); 577 parallel_move.AddMove( 578 locations->InAt(1), 579 Location::RegisterLocation(calling_convention.GetRegisterAt(1)), 580 Primitive::kPrimInt, 581 nullptr); 582 parallel_move.AddMove( 583 locations->InAt(2), 584 Location::RegisterLocation(calling_convention.GetRegisterAt(2)), 585 Primitive::kPrimNot, 586 nullptr); 587 codegen->GetMoveResolver()->EmitNativeCode(¶llel_move); 588 589 CodeGeneratorARM* arm_codegen = down_cast<CodeGeneratorARM*>(codegen); 590 arm_codegen->InvokeRuntime(kQuickAputObject, instruction_, instruction_->GetDexPc(), this); 591 CheckEntrypointTypes<kQuickAputObject, void, mirror::Array*, int32_t, mirror::Object*>(); 592 RestoreLiveRegisters(codegen, locations); 593 __ b(GetExitLabel()); 594 } 595 596 const char* GetDescription() const OVERRIDE { return "ArraySetSlowPathARM"; } 597 598 private: 599 DISALLOW_COPY_AND_ASSIGN(ArraySetSlowPathARM); 600}; 601 602// Slow path marking an object reference `ref` during a read 603// barrier. The field `obj.field` in the object `obj` holding this 604// reference does not get updated by this slow path after marking (see 605// ReadBarrierMarkAndUpdateFieldSlowPathARM below for that). 606// 607// This means that after the execution of this slow path, `ref` will 608// always be up-to-date, but `obj.field` may not; i.e., after the 609// flip, `ref` will be a to-space reference, but `obj.field` will 610// probably still be a from-space reference (unless it gets updated by 611// another thread, or if another thread installed another object 612// reference (different from `ref`) in `obj.field`). 613class ReadBarrierMarkSlowPathARM : public SlowPathCodeARM { 614 public: 615 ReadBarrierMarkSlowPathARM(HInstruction* instruction, 616 Location ref, 617 Location entrypoint = Location::NoLocation()) 618 : SlowPathCodeARM(instruction), ref_(ref), entrypoint_(entrypoint) { 619 DCHECK(kEmitCompilerReadBarrier); 620 } 621 622 const char* GetDescription() const OVERRIDE { return "ReadBarrierMarkSlowPathARM"; } 623 624 void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { 625 LocationSummary* locations = instruction_->GetLocations(); 626 Register ref_reg = ref_.AsRegister<Register>(); 627 DCHECK(locations->CanCall()); 628 DCHECK(!locations->GetLiveRegisters()->ContainsCoreRegister(ref_reg)) << ref_reg; 629 DCHECK(instruction_->IsInstanceFieldGet() || 630 instruction_->IsStaticFieldGet() || 631 instruction_->IsArrayGet() || 632 instruction_->IsArraySet() || 633 instruction_->IsLoadClass() || 634 instruction_->IsLoadString() || 635 instruction_->IsInstanceOf() || 636 instruction_->IsCheckCast() || 637 (instruction_->IsInvokeVirtual() && instruction_->GetLocations()->Intrinsified()) || 638 (instruction_->IsInvokeStaticOrDirect() && instruction_->GetLocations()->Intrinsified())) 639 << "Unexpected instruction in read barrier marking slow path: " 640 << instruction_->DebugName(); 641 // The read barrier instrumentation of object ArrayGet 642 // instructions does not support the HIntermediateAddress 643 // instruction. 644 DCHECK(!(instruction_->IsArrayGet() && 645 instruction_->AsArrayGet()->GetArray()->IsIntermediateAddress())); 646 647 __ Bind(GetEntryLabel()); 648 // No need to save live registers; it's taken care of by the 649 // entrypoint. Also, there is no need to update the stack mask, 650 // as this runtime call will not trigger a garbage collection. 651 CodeGeneratorARM* arm_codegen = down_cast<CodeGeneratorARM*>(codegen); 652 DCHECK_NE(ref_reg, SP); 653 DCHECK_NE(ref_reg, LR); 654 DCHECK_NE(ref_reg, PC); 655 // IP is used internally by the ReadBarrierMarkRegX entry point 656 // as a temporary, it cannot be the entry point's input/output. 657 DCHECK_NE(ref_reg, IP); 658 DCHECK(0 <= ref_reg && ref_reg < kNumberOfCoreRegisters) << ref_reg; 659 // "Compact" slow path, saving two moves. 660 // 661 // Instead of using the standard runtime calling convention (input 662 // and output in R0): 663 // 664 // R0 <- ref 665 // R0 <- ReadBarrierMark(R0) 666 // ref <- R0 667 // 668 // we just use rX (the register containing `ref`) as input and output 669 // of a dedicated entrypoint: 670 // 671 // rX <- ReadBarrierMarkRegX(rX) 672 // 673 if (entrypoint_.IsValid()) { 674 arm_codegen->ValidateInvokeRuntimeWithoutRecordingPcInfo(instruction_, this); 675 __ blx(entrypoint_.AsRegister<Register>()); 676 } else { 677 int32_t entry_point_offset = 678 CodeGenerator::GetReadBarrierMarkEntryPointsOffset<kArmPointerSize>(ref_reg); 679 // This runtime call does not require a stack map. 680 arm_codegen->InvokeRuntimeWithoutRecordingPcInfo(entry_point_offset, instruction_, this); 681 } 682 __ b(GetExitLabel()); 683 } 684 685 private: 686 // The location (register) of the marked object reference. 687 const Location ref_; 688 689 // The location of the entrypoint if already loaded. 690 const Location entrypoint_; 691 692 DISALLOW_COPY_AND_ASSIGN(ReadBarrierMarkSlowPathARM); 693}; 694 695// Slow path marking an object reference `ref` during a read barrier, 696// and if needed, atomically updating the field `obj.field` in the 697// object `obj` holding this reference after marking (contrary to 698// ReadBarrierMarkSlowPathARM above, which never tries to update 699// `obj.field`). 700// 701// This means that after the execution of this slow path, both `ref` 702// and `obj.field` will be up-to-date; i.e., after the flip, both will 703// hold the same to-space reference (unless another thread installed 704// another object reference (different from `ref`) in `obj.field`). 705class ReadBarrierMarkAndUpdateFieldSlowPathARM : public SlowPathCodeARM { 706 public: 707 ReadBarrierMarkAndUpdateFieldSlowPathARM(HInstruction* instruction, 708 Location ref, 709 Register obj, 710 Location field_offset, 711 Register temp1, 712 Register temp2) 713 : SlowPathCodeARM(instruction), 714 ref_(ref), 715 obj_(obj), 716 field_offset_(field_offset), 717 temp1_(temp1), 718 temp2_(temp2) { 719 DCHECK(kEmitCompilerReadBarrier); 720 } 721 722 const char* GetDescription() const OVERRIDE { return "ReadBarrierMarkAndUpdateFieldSlowPathARM"; } 723 724 void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { 725 LocationSummary* locations = instruction_->GetLocations(); 726 Register ref_reg = ref_.AsRegister<Register>(); 727 DCHECK(locations->CanCall()); 728 DCHECK(!locations->GetLiveRegisters()->ContainsCoreRegister(ref_reg)) << ref_reg; 729 // This slow path is only used by the UnsafeCASObject intrinsic. 730 DCHECK((instruction_->IsInvokeVirtual() && instruction_->GetLocations()->Intrinsified())) 731 << "Unexpected instruction in read barrier marking and field updating slow path: " 732 << instruction_->DebugName(); 733 DCHECK(instruction_->GetLocations()->Intrinsified()); 734 DCHECK_EQ(instruction_->AsInvoke()->GetIntrinsic(), Intrinsics::kUnsafeCASObject); 735 DCHECK(field_offset_.IsRegisterPair()) << field_offset_; 736 737 __ Bind(GetEntryLabel()); 738 739 // Save the old reference. 740 // Note that we cannot use IP to save the old reference, as IP is 741 // used internally by the ReadBarrierMarkRegX entry point, and we 742 // need the old reference after the call to that entry point. 743 DCHECK_NE(temp1_, IP); 744 __ Mov(temp1_, ref_reg); 745 746 // No need to save live registers; it's taken care of by the 747 // entrypoint. Also, there is no need to update the stack mask, 748 // as this runtime call will not trigger a garbage collection. 749 CodeGeneratorARM* arm_codegen = down_cast<CodeGeneratorARM*>(codegen); 750 DCHECK_NE(ref_reg, SP); 751 DCHECK_NE(ref_reg, LR); 752 DCHECK_NE(ref_reg, PC); 753 // IP is used internally by the ReadBarrierMarkRegX entry point 754 // as a temporary, it cannot be the entry point's input/output. 755 DCHECK_NE(ref_reg, IP); 756 DCHECK(0 <= ref_reg && ref_reg < kNumberOfCoreRegisters) << ref_reg; 757 // "Compact" slow path, saving two moves. 758 // 759 // Instead of using the standard runtime calling convention (input 760 // and output in R0): 761 // 762 // R0 <- ref 763 // R0 <- ReadBarrierMark(R0) 764 // ref <- R0 765 // 766 // we just use rX (the register containing `ref`) as input and output 767 // of a dedicated entrypoint: 768 // 769 // rX <- ReadBarrierMarkRegX(rX) 770 // 771 int32_t entry_point_offset = 772 CodeGenerator::GetReadBarrierMarkEntryPointsOffset<kArmPointerSize>(ref_reg); 773 // This runtime call does not require a stack map. 774 arm_codegen->InvokeRuntimeWithoutRecordingPcInfo(entry_point_offset, instruction_, this); 775 776 // If the new reference is different from the old reference, 777 // update the field in the holder (`*(obj_ + field_offset_)`). 778 // 779 // Note that this field could also hold a different object, if 780 // another thread had concurrently changed it. In that case, the 781 // LDREX/SUBS/ITNE sequence of instructions in the compare-and-set 782 // (CAS) operation below would abort the CAS, leaving the field 783 // as-is. 784 Label done; 785 __ cmp(temp1_, ShifterOperand(ref_reg)); 786 __ b(&done, EQ); 787 788 // Update the the holder's field atomically. This may fail if 789 // mutator updates before us, but it's OK. This is achieved 790 // using a strong compare-and-set (CAS) operation with relaxed 791 // memory synchronization ordering, where the expected value is 792 // the old reference and the desired value is the new reference. 793 794 // Convenience aliases. 795 Register base = obj_; 796 // The UnsafeCASObject intrinsic uses a register pair as field 797 // offset ("long offset"), of which only the low part contains 798 // data. 799 Register offset = field_offset_.AsRegisterPairLow<Register>(); 800 Register expected = temp1_; 801 Register value = ref_reg; 802 Register tmp_ptr = IP; // Pointer to actual memory. 803 Register tmp = temp2_; // Value in memory. 804 805 __ add(tmp_ptr, base, ShifterOperand(offset)); 806 807 if (kPoisonHeapReferences) { 808 __ PoisonHeapReference(expected); 809 if (value == expected) { 810 // Do not poison `value`, as it is the same register as 811 // `expected`, which has just been poisoned. 812 } else { 813 __ PoisonHeapReference(value); 814 } 815 } 816 817 // do { 818 // tmp = [r_ptr] - expected; 819 // } while (tmp == 0 && failure([r_ptr] <- r_new_value)); 820 821 Label loop_head, exit_loop; 822 __ Bind(&loop_head); 823 824 __ ldrex(tmp, tmp_ptr); 825 826 __ subs(tmp, tmp, ShifterOperand(expected)); 827 828 __ it(NE); 829 __ clrex(NE); 830 831 __ b(&exit_loop, NE); 832 833 __ strex(tmp, value, tmp_ptr); 834 __ cmp(tmp, ShifterOperand(1)); 835 __ b(&loop_head, EQ); 836 837 __ Bind(&exit_loop); 838 839 if (kPoisonHeapReferences) { 840 __ UnpoisonHeapReference(expected); 841 if (value == expected) { 842 // Do not unpoison `value`, as it is the same register as 843 // `expected`, which has just been unpoisoned. 844 } else { 845 __ UnpoisonHeapReference(value); 846 } 847 } 848 849 __ Bind(&done); 850 __ b(GetExitLabel()); 851 } 852 853 private: 854 // The location (register) of the marked object reference. 855 const Location ref_; 856 // The register containing the object holding the marked object reference field. 857 const Register obj_; 858 // The location of the offset of the marked reference field within `obj_`. 859 Location field_offset_; 860 861 const Register temp1_; 862 const Register temp2_; 863 864 DISALLOW_COPY_AND_ASSIGN(ReadBarrierMarkAndUpdateFieldSlowPathARM); 865}; 866 867// Slow path generating a read barrier for a heap reference. 868class ReadBarrierForHeapReferenceSlowPathARM : public SlowPathCodeARM { 869 public: 870 ReadBarrierForHeapReferenceSlowPathARM(HInstruction* instruction, 871 Location out, 872 Location ref, 873 Location obj, 874 uint32_t offset, 875 Location index) 876 : SlowPathCodeARM(instruction), 877 out_(out), 878 ref_(ref), 879 obj_(obj), 880 offset_(offset), 881 index_(index) { 882 DCHECK(kEmitCompilerReadBarrier); 883 // If `obj` is equal to `out` or `ref`, it means the initial object 884 // has been overwritten by (or after) the heap object reference load 885 // to be instrumented, e.g.: 886 // 887 // __ LoadFromOffset(kLoadWord, out, out, offset); 888 // codegen_->GenerateReadBarrierSlow(instruction, out_loc, out_loc, out_loc, offset); 889 // 890 // In that case, we have lost the information about the original 891 // object, and the emitted read barrier cannot work properly. 892 DCHECK(!obj.Equals(out)) << "obj=" << obj << " out=" << out; 893 DCHECK(!obj.Equals(ref)) << "obj=" << obj << " ref=" << ref; 894 } 895 896 void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { 897 CodeGeneratorARM* arm_codegen = down_cast<CodeGeneratorARM*>(codegen); 898 LocationSummary* locations = instruction_->GetLocations(); 899 Register reg_out = out_.AsRegister<Register>(); 900 DCHECK(locations->CanCall()); 901 DCHECK(!locations->GetLiveRegisters()->ContainsCoreRegister(reg_out)); 902 DCHECK(instruction_->IsInstanceFieldGet() || 903 instruction_->IsStaticFieldGet() || 904 instruction_->IsArrayGet() || 905 instruction_->IsInstanceOf() || 906 instruction_->IsCheckCast() || 907 (instruction_->IsInvokeVirtual()) && instruction_->GetLocations()->Intrinsified()) 908 << "Unexpected instruction in read barrier for heap reference slow path: " 909 << instruction_->DebugName(); 910 // The read barrier instrumentation of object ArrayGet 911 // instructions does not support the HIntermediateAddress 912 // instruction. 913 DCHECK(!(instruction_->IsArrayGet() && 914 instruction_->AsArrayGet()->GetArray()->IsIntermediateAddress())); 915 916 __ Bind(GetEntryLabel()); 917 SaveLiveRegisters(codegen, locations); 918 919 // We may have to change the index's value, but as `index_` is a 920 // constant member (like other "inputs" of this slow path), 921 // introduce a copy of it, `index`. 922 Location index = index_; 923 if (index_.IsValid()) { 924 // Handle `index_` for HArrayGet and UnsafeGetObject/UnsafeGetObjectVolatile intrinsics. 925 if (instruction_->IsArrayGet()) { 926 // Compute the actual memory offset and store it in `index`. 927 Register index_reg = index_.AsRegister<Register>(); 928 DCHECK(locations->GetLiveRegisters()->ContainsCoreRegister(index_reg)); 929 if (codegen->IsCoreCalleeSaveRegister(index_reg)) { 930 // We are about to change the value of `index_reg` (see the 931 // calls to art::arm::Thumb2Assembler::Lsl and 932 // art::arm::Thumb2Assembler::AddConstant below), but it has 933 // not been saved by the previous call to 934 // art::SlowPathCode::SaveLiveRegisters, as it is a 935 // callee-save register -- 936 // art::SlowPathCode::SaveLiveRegisters does not consider 937 // callee-save registers, as it has been designed with the 938 // assumption that callee-save registers are supposed to be 939 // handled by the called function. So, as a callee-save 940 // register, `index_reg` _would_ eventually be saved onto 941 // the stack, but it would be too late: we would have 942 // changed its value earlier. Therefore, we manually save 943 // it here into another freely available register, 944 // `free_reg`, chosen of course among the caller-save 945 // registers (as a callee-save `free_reg` register would 946 // exhibit the same problem). 947 // 948 // Note we could have requested a temporary register from 949 // the register allocator instead; but we prefer not to, as 950 // this is a slow path, and we know we can find a 951 // caller-save register that is available. 952 Register free_reg = FindAvailableCallerSaveRegister(codegen); 953 __ Mov(free_reg, index_reg); 954 index_reg = free_reg; 955 index = Location::RegisterLocation(index_reg); 956 } else { 957 // The initial register stored in `index_` has already been 958 // saved in the call to art::SlowPathCode::SaveLiveRegisters 959 // (as it is not a callee-save register), so we can freely 960 // use it. 961 } 962 // Shifting the index value contained in `index_reg` by the scale 963 // factor (2) cannot overflow in practice, as the runtime is 964 // unable to allocate object arrays with a size larger than 965 // 2^26 - 1 (that is, 2^28 - 4 bytes). 966 __ Lsl(index_reg, index_reg, TIMES_4); 967 static_assert( 968 sizeof(mirror::HeapReference<mirror::Object>) == sizeof(int32_t), 969 "art::mirror::HeapReference<art::mirror::Object> and int32_t have different sizes."); 970 __ AddConstant(index_reg, index_reg, offset_); 971 } else { 972 // In the case of the UnsafeGetObject/UnsafeGetObjectVolatile 973 // intrinsics, `index_` is not shifted by a scale factor of 2 974 // (as in the case of ArrayGet), as it is actually an offset 975 // to an object field within an object. 976 DCHECK(instruction_->IsInvoke()) << instruction_->DebugName(); 977 DCHECK(instruction_->GetLocations()->Intrinsified()); 978 DCHECK((instruction_->AsInvoke()->GetIntrinsic() == Intrinsics::kUnsafeGetObject) || 979 (instruction_->AsInvoke()->GetIntrinsic() == Intrinsics::kUnsafeGetObjectVolatile)) 980 << instruction_->AsInvoke()->GetIntrinsic(); 981 DCHECK_EQ(offset_, 0U); 982 DCHECK(index_.IsRegisterPair()); 983 // UnsafeGet's offset location is a register pair, the low 984 // part contains the correct offset. 985 index = index_.ToLow(); 986 } 987 } 988 989 // We're moving two or three locations to locations that could 990 // overlap, so we need a parallel move resolver. 991 InvokeRuntimeCallingConvention calling_convention; 992 HParallelMove parallel_move(codegen->GetGraph()->GetArena()); 993 parallel_move.AddMove(ref_, 994 Location::RegisterLocation(calling_convention.GetRegisterAt(0)), 995 Primitive::kPrimNot, 996 nullptr); 997 parallel_move.AddMove(obj_, 998 Location::RegisterLocation(calling_convention.GetRegisterAt(1)), 999 Primitive::kPrimNot, 1000 nullptr); 1001 if (index.IsValid()) { 1002 parallel_move.AddMove(index, 1003 Location::RegisterLocation(calling_convention.GetRegisterAt(2)), 1004 Primitive::kPrimInt, 1005 nullptr); 1006 codegen->GetMoveResolver()->EmitNativeCode(¶llel_move); 1007 } else { 1008 codegen->GetMoveResolver()->EmitNativeCode(¶llel_move); 1009 __ LoadImmediate(calling_convention.GetRegisterAt(2), offset_); 1010 } 1011 arm_codegen->InvokeRuntime(kQuickReadBarrierSlow, instruction_, instruction_->GetDexPc(), this); 1012 CheckEntrypointTypes< 1013 kQuickReadBarrierSlow, mirror::Object*, mirror::Object*, mirror::Object*, uint32_t>(); 1014 arm_codegen->Move32(out_, Location::RegisterLocation(R0)); 1015 1016 RestoreLiveRegisters(codegen, locations); 1017 __ b(GetExitLabel()); 1018 } 1019 1020 const char* GetDescription() const OVERRIDE { return "ReadBarrierForHeapReferenceSlowPathARM"; } 1021 1022 private: 1023 Register FindAvailableCallerSaveRegister(CodeGenerator* codegen) { 1024 size_t ref = static_cast<int>(ref_.AsRegister<Register>()); 1025 size_t obj = static_cast<int>(obj_.AsRegister<Register>()); 1026 for (size_t i = 0, e = codegen->GetNumberOfCoreRegisters(); i < e; ++i) { 1027 if (i != ref && i != obj && !codegen->IsCoreCalleeSaveRegister(i)) { 1028 return static_cast<Register>(i); 1029 } 1030 } 1031 // We shall never fail to find a free caller-save register, as 1032 // there are more than two core caller-save registers on ARM 1033 // (meaning it is possible to find one which is different from 1034 // `ref` and `obj`). 1035 DCHECK_GT(codegen->GetNumberOfCoreCallerSaveRegisters(), 2u); 1036 LOG(FATAL) << "Could not find a free caller-save register"; 1037 UNREACHABLE(); 1038 } 1039 1040 const Location out_; 1041 const Location ref_; 1042 const Location obj_; 1043 const uint32_t offset_; 1044 // An additional location containing an index to an array. 1045 // Only used for HArrayGet and the UnsafeGetObject & 1046 // UnsafeGetObjectVolatile intrinsics. 1047 const Location index_; 1048 1049 DISALLOW_COPY_AND_ASSIGN(ReadBarrierForHeapReferenceSlowPathARM); 1050}; 1051 1052// Slow path generating a read barrier for a GC root. 1053class ReadBarrierForRootSlowPathARM : public SlowPathCodeARM { 1054 public: 1055 ReadBarrierForRootSlowPathARM(HInstruction* instruction, Location out, Location root) 1056 : SlowPathCodeARM(instruction), out_(out), root_(root) { 1057 DCHECK(kEmitCompilerReadBarrier); 1058 } 1059 1060 void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { 1061 LocationSummary* locations = instruction_->GetLocations(); 1062 Register reg_out = out_.AsRegister<Register>(); 1063 DCHECK(locations->CanCall()); 1064 DCHECK(!locations->GetLiveRegisters()->ContainsCoreRegister(reg_out)); 1065 DCHECK(instruction_->IsLoadClass() || instruction_->IsLoadString()) 1066 << "Unexpected instruction in read barrier for GC root slow path: " 1067 << instruction_->DebugName(); 1068 1069 __ Bind(GetEntryLabel()); 1070 SaveLiveRegisters(codegen, locations); 1071 1072 InvokeRuntimeCallingConvention calling_convention; 1073 CodeGeneratorARM* arm_codegen = down_cast<CodeGeneratorARM*>(codegen); 1074 arm_codegen->Move32(Location::RegisterLocation(calling_convention.GetRegisterAt(0)), root_); 1075 arm_codegen->InvokeRuntime(kQuickReadBarrierForRootSlow, 1076 instruction_, 1077 instruction_->GetDexPc(), 1078 this); 1079 CheckEntrypointTypes<kQuickReadBarrierForRootSlow, mirror::Object*, GcRoot<mirror::Object>*>(); 1080 arm_codegen->Move32(out_, Location::RegisterLocation(R0)); 1081 1082 RestoreLiveRegisters(codegen, locations); 1083 __ b(GetExitLabel()); 1084 } 1085 1086 const char* GetDescription() const OVERRIDE { return "ReadBarrierForRootSlowPathARM"; } 1087 1088 private: 1089 const Location out_; 1090 const Location root_; 1091 1092 DISALLOW_COPY_AND_ASSIGN(ReadBarrierForRootSlowPathARM); 1093}; 1094 1095#undef __ 1096// NOLINT on __ macro to suppress wrong warning/fix (misc-macro-parentheses) from clang-tidy. 1097#define __ down_cast<ArmAssembler*>(GetAssembler())-> // NOLINT 1098 1099inline Condition ARMCondition(IfCondition cond) { 1100 switch (cond) { 1101 case kCondEQ: return EQ; 1102 case kCondNE: return NE; 1103 case kCondLT: return LT; 1104 case kCondLE: return LE; 1105 case kCondGT: return GT; 1106 case kCondGE: return GE; 1107 case kCondB: return LO; 1108 case kCondBE: return LS; 1109 case kCondA: return HI; 1110 case kCondAE: return HS; 1111 } 1112 LOG(FATAL) << "Unreachable"; 1113 UNREACHABLE(); 1114} 1115 1116// Maps signed condition to unsigned condition. 1117inline Condition ARMUnsignedCondition(IfCondition cond) { 1118 switch (cond) { 1119 case kCondEQ: return EQ; 1120 case kCondNE: return NE; 1121 // Signed to unsigned. 1122 case kCondLT: return LO; 1123 case kCondLE: return LS; 1124 case kCondGT: return HI; 1125 case kCondGE: return HS; 1126 // Unsigned remain unchanged. 1127 case kCondB: return LO; 1128 case kCondBE: return LS; 1129 case kCondA: return HI; 1130 case kCondAE: return HS; 1131 } 1132 LOG(FATAL) << "Unreachable"; 1133 UNREACHABLE(); 1134} 1135 1136inline Condition ARMFPCondition(IfCondition cond, bool gt_bias) { 1137 // The ARM condition codes can express all the necessary branches, see the 1138 // "Meaning (floating-point)" column in the table A8-1 of the ARMv7 reference manual. 1139 // There is no dex instruction or HIR that would need the missing conditions 1140 // "equal or unordered" or "not equal". 1141 switch (cond) { 1142 case kCondEQ: return EQ; 1143 case kCondNE: return NE /* unordered */; 1144 case kCondLT: return gt_bias ? CC : LT /* unordered */; 1145 case kCondLE: return gt_bias ? LS : LE /* unordered */; 1146 case kCondGT: return gt_bias ? HI /* unordered */ : GT; 1147 case kCondGE: return gt_bias ? CS /* unordered */ : GE; 1148 default: 1149 LOG(FATAL) << "UNREACHABLE"; 1150 UNREACHABLE(); 1151 } 1152} 1153 1154void CodeGeneratorARM::DumpCoreRegister(std::ostream& stream, int reg) const { 1155 stream << Register(reg); 1156} 1157 1158void CodeGeneratorARM::DumpFloatingPointRegister(std::ostream& stream, int reg) const { 1159 stream << SRegister(reg); 1160} 1161 1162size_t CodeGeneratorARM::SaveCoreRegister(size_t stack_index, uint32_t reg_id) { 1163 __ StoreToOffset(kStoreWord, static_cast<Register>(reg_id), SP, stack_index); 1164 return kArmWordSize; 1165} 1166 1167size_t CodeGeneratorARM::RestoreCoreRegister(size_t stack_index, uint32_t reg_id) { 1168 __ LoadFromOffset(kLoadWord, static_cast<Register>(reg_id), SP, stack_index); 1169 return kArmWordSize; 1170} 1171 1172size_t CodeGeneratorARM::SaveFloatingPointRegister(size_t stack_index, uint32_t reg_id) { 1173 __ StoreSToOffset(static_cast<SRegister>(reg_id), SP, stack_index); 1174 return kArmWordSize; 1175} 1176 1177size_t CodeGeneratorARM::RestoreFloatingPointRegister(size_t stack_index, uint32_t reg_id) { 1178 __ LoadSFromOffset(static_cast<SRegister>(reg_id), SP, stack_index); 1179 return kArmWordSize; 1180} 1181 1182CodeGeneratorARM::CodeGeneratorARM(HGraph* graph, 1183 const ArmInstructionSetFeatures& isa_features, 1184 const CompilerOptions& compiler_options, 1185 OptimizingCompilerStats* stats) 1186 : CodeGenerator(graph, 1187 kNumberOfCoreRegisters, 1188 kNumberOfSRegisters, 1189 kNumberOfRegisterPairs, 1190 ComputeRegisterMask(reinterpret_cast<const int*>(kCoreCalleeSaves), 1191 arraysize(kCoreCalleeSaves)), 1192 ComputeRegisterMask(reinterpret_cast<const int*>(kFpuCalleeSaves), 1193 arraysize(kFpuCalleeSaves)), 1194 compiler_options, 1195 stats), 1196 block_labels_(nullptr), 1197 location_builder_(graph, this), 1198 instruction_visitor_(graph, this), 1199 move_resolver_(graph->GetArena(), this), 1200 assembler_(graph->GetArena()), 1201 isa_features_(isa_features), 1202 uint32_literals_(std::less<uint32_t>(), 1203 graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), 1204 pc_relative_dex_cache_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), 1205 boot_image_string_patches_(StringReferenceValueComparator(), 1206 graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), 1207 pc_relative_string_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), 1208 boot_image_type_patches_(TypeReferenceValueComparator(), 1209 graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), 1210 pc_relative_type_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), 1211 boot_image_address_patches_(std::less<uint32_t>(), 1212 graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), 1213 jit_string_patches_(StringReferenceValueComparator(), 1214 graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), 1215 jit_class_patches_(TypeReferenceValueComparator(), 1216 graph->GetArena()->Adapter(kArenaAllocCodeGenerator)) { 1217 // Always save the LR register to mimic Quick. 1218 AddAllocatedRegister(Location::RegisterLocation(LR)); 1219} 1220 1221void CodeGeneratorARM::Finalize(CodeAllocator* allocator) { 1222 // Ensure that we fix up branches and literal loads and emit the literal pool. 1223 __ FinalizeCode(); 1224 1225 // Adjust native pc offsets in stack maps. 1226 for (size_t i = 0, num = stack_map_stream_.GetNumberOfStackMaps(); i != num; ++i) { 1227 uint32_t old_position = stack_map_stream_.GetStackMap(i).native_pc_offset; 1228 uint32_t new_position = __ GetAdjustedPosition(old_position); 1229 stack_map_stream_.SetStackMapNativePcOffset(i, new_position); 1230 } 1231 // Adjust pc offsets for the disassembly information. 1232 if (disasm_info_ != nullptr) { 1233 GeneratedCodeInterval* frame_entry_interval = disasm_info_->GetFrameEntryInterval(); 1234 frame_entry_interval->start = __ GetAdjustedPosition(frame_entry_interval->start); 1235 frame_entry_interval->end = __ GetAdjustedPosition(frame_entry_interval->end); 1236 for (auto& it : *disasm_info_->GetInstructionIntervals()) { 1237 it.second.start = __ GetAdjustedPosition(it.second.start); 1238 it.second.end = __ GetAdjustedPosition(it.second.end); 1239 } 1240 for (auto& it : *disasm_info_->GetSlowPathIntervals()) { 1241 it.code_interval.start = __ GetAdjustedPosition(it.code_interval.start); 1242 it.code_interval.end = __ GetAdjustedPosition(it.code_interval.end); 1243 } 1244 } 1245 1246 CodeGenerator::Finalize(allocator); 1247} 1248 1249void CodeGeneratorARM::SetupBlockedRegisters() const { 1250 // Stack register, LR and PC are always reserved. 1251 blocked_core_registers_[SP] = true; 1252 blocked_core_registers_[LR] = true; 1253 blocked_core_registers_[PC] = true; 1254 1255 // Reserve thread register. 1256 blocked_core_registers_[TR] = true; 1257 1258 // Reserve temp register. 1259 blocked_core_registers_[IP] = true; 1260 1261 if (GetGraph()->IsDebuggable()) { 1262 // Stubs do not save callee-save floating point registers. If the graph 1263 // is debuggable, we need to deal with these registers differently. For 1264 // now, just block them. 1265 for (size_t i = 0; i < arraysize(kFpuCalleeSaves); ++i) { 1266 blocked_fpu_registers_[kFpuCalleeSaves[i]] = true; 1267 } 1268 } 1269} 1270 1271InstructionCodeGeneratorARM::InstructionCodeGeneratorARM(HGraph* graph, CodeGeneratorARM* codegen) 1272 : InstructionCodeGenerator(graph, codegen), 1273 assembler_(codegen->GetAssembler()), 1274 codegen_(codegen) {} 1275 1276void CodeGeneratorARM::ComputeSpillMask() { 1277 core_spill_mask_ = allocated_registers_.GetCoreRegisters() & core_callee_save_mask_; 1278 DCHECK_NE(core_spill_mask_, 0u) << "At least the return address register must be saved"; 1279 // There is no easy instruction to restore just the PC on thumb2. We spill and 1280 // restore another arbitrary register. 1281 core_spill_mask_ |= (1 << kCoreAlwaysSpillRegister); 1282 fpu_spill_mask_ = allocated_registers_.GetFloatingPointRegisters() & fpu_callee_save_mask_; 1283 // We use vpush and vpop for saving and restoring floating point registers, which take 1284 // a SRegister and the number of registers to save/restore after that SRegister. We 1285 // therefore update the `fpu_spill_mask_` to also contain those registers not allocated, 1286 // but in the range. 1287 if (fpu_spill_mask_ != 0) { 1288 uint32_t least_significant_bit = LeastSignificantBit(fpu_spill_mask_); 1289 uint32_t most_significant_bit = MostSignificantBit(fpu_spill_mask_); 1290 for (uint32_t i = least_significant_bit + 1 ; i < most_significant_bit; ++i) { 1291 fpu_spill_mask_ |= (1 << i); 1292 } 1293 } 1294} 1295 1296static dwarf::Reg DWARFReg(Register reg) { 1297 return dwarf::Reg::ArmCore(static_cast<int>(reg)); 1298} 1299 1300static dwarf::Reg DWARFReg(SRegister reg) { 1301 return dwarf::Reg::ArmFp(static_cast<int>(reg)); 1302} 1303 1304void CodeGeneratorARM::GenerateFrameEntry() { 1305 bool skip_overflow_check = 1306 IsLeafMethod() && !FrameNeedsStackCheck(GetFrameSize(), InstructionSet::kArm); 1307 DCHECK(GetCompilerOptions().GetImplicitStackOverflowChecks()); 1308 __ Bind(&frame_entry_label_); 1309 1310 if (HasEmptyFrame()) { 1311 return; 1312 } 1313 1314 if (!skip_overflow_check) { 1315 __ AddConstant(IP, SP, -static_cast<int32_t>(GetStackOverflowReservedBytes(kArm))); 1316 __ LoadFromOffset(kLoadWord, IP, IP, 0); 1317 RecordPcInfo(nullptr, 0); 1318 } 1319 1320 __ PushList(core_spill_mask_); 1321 __ cfi().AdjustCFAOffset(kArmWordSize * POPCOUNT(core_spill_mask_)); 1322 __ cfi().RelOffsetForMany(DWARFReg(kMethodRegisterArgument), 0, core_spill_mask_, kArmWordSize); 1323 if (fpu_spill_mask_ != 0) { 1324 SRegister start_register = SRegister(LeastSignificantBit(fpu_spill_mask_)); 1325 __ vpushs(start_register, POPCOUNT(fpu_spill_mask_)); 1326 __ cfi().AdjustCFAOffset(kArmWordSize * POPCOUNT(fpu_spill_mask_)); 1327 __ cfi().RelOffsetForMany(DWARFReg(S0), 0, fpu_spill_mask_, kArmWordSize); 1328 } 1329 1330 if (GetGraph()->HasShouldDeoptimizeFlag()) { 1331 // Initialize should_deoptimize flag to 0. 1332 __ mov(IP, ShifterOperand(0)); 1333 __ StoreToOffset(kStoreWord, IP, SP, -kShouldDeoptimizeFlagSize); 1334 } 1335 1336 int adjust = GetFrameSize() - FrameEntrySpillSize(); 1337 __ AddConstant(SP, -adjust); 1338 __ cfi().AdjustCFAOffset(adjust); 1339 1340 // Save the current method if we need it. Note that we do not 1341 // do this in HCurrentMethod, as the instruction might have been removed 1342 // in the SSA graph. 1343 if (RequiresCurrentMethod()) { 1344 __ StoreToOffset(kStoreWord, kMethodRegisterArgument, SP, 0); 1345 } 1346} 1347 1348void CodeGeneratorARM::GenerateFrameExit() { 1349 if (HasEmptyFrame()) { 1350 __ bx(LR); 1351 return; 1352 } 1353 __ cfi().RememberState(); 1354 int adjust = GetFrameSize() - FrameEntrySpillSize(); 1355 __ AddConstant(SP, adjust); 1356 __ cfi().AdjustCFAOffset(-adjust); 1357 if (fpu_spill_mask_ != 0) { 1358 SRegister start_register = SRegister(LeastSignificantBit(fpu_spill_mask_)); 1359 __ vpops(start_register, POPCOUNT(fpu_spill_mask_)); 1360 __ cfi().AdjustCFAOffset(-static_cast<int>(kArmPointerSize) * POPCOUNT(fpu_spill_mask_)); 1361 __ cfi().RestoreMany(DWARFReg(SRegister(0)), fpu_spill_mask_); 1362 } 1363 // Pop LR into PC to return. 1364 DCHECK_NE(core_spill_mask_ & (1 << LR), 0U); 1365 uint32_t pop_mask = (core_spill_mask_ & (~(1 << LR))) | 1 << PC; 1366 __ PopList(pop_mask); 1367 __ cfi().RestoreState(); 1368 __ cfi().DefCFAOffset(GetFrameSize()); 1369} 1370 1371void CodeGeneratorARM::Bind(HBasicBlock* block) { 1372 Label* label = GetLabelOf(block); 1373 __ BindTrackedLabel(label); 1374} 1375 1376Location InvokeDexCallingConventionVisitorARM::GetNextLocation(Primitive::Type type) { 1377 switch (type) { 1378 case Primitive::kPrimBoolean: 1379 case Primitive::kPrimByte: 1380 case Primitive::kPrimChar: 1381 case Primitive::kPrimShort: 1382 case Primitive::kPrimInt: 1383 case Primitive::kPrimNot: { 1384 uint32_t index = gp_index_++; 1385 uint32_t stack_index = stack_index_++; 1386 if (index < calling_convention.GetNumberOfRegisters()) { 1387 return Location::RegisterLocation(calling_convention.GetRegisterAt(index)); 1388 } else { 1389 return Location::StackSlot(calling_convention.GetStackOffsetOf(stack_index)); 1390 } 1391 } 1392 1393 case Primitive::kPrimLong: { 1394 uint32_t index = gp_index_; 1395 uint32_t stack_index = stack_index_; 1396 gp_index_ += 2; 1397 stack_index_ += 2; 1398 if (index + 1 < calling_convention.GetNumberOfRegisters()) { 1399 if (calling_convention.GetRegisterAt(index) == R1) { 1400 // Skip R1, and use R2_R3 instead. 1401 gp_index_++; 1402 index++; 1403 } 1404 } 1405 if (index + 1 < calling_convention.GetNumberOfRegisters()) { 1406 DCHECK_EQ(calling_convention.GetRegisterAt(index) + 1, 1407 calling_convention.GetRegisterAt(index + 1)); 1408 1409 return Location::RegisterPairLocation(calling_convention.GetRegisterAt(index), 1410 calling_convention.GetRegisterAt(index + 1)); 1411 } else { 1412 return Location::DoubleStackSlot(calling_convention.GetStackOffsetOf(stack_index)); 1413 } 1414 } 1415 1416 case Primitive::kPrimFloat: { 1417 uint32_t stack_index = stack_index_++; 1418 if (float_index_ % 2 == 0) { 1419 float_index_ = std::max(double_index_, float_index_); 1420 } 1421 if (float_index_ < calling_convention.GetNumberOfFpuRegisters()) { 1422 return Location::FpuRegisterLocation(calling_convention.GetFpuRegisterAt(float_index_++)); 1423 } else { 1424 return Location::StackSlot(calling_convention.GetStackOffsetOf(stack_index)); 1425 } 1426 } 1427 1428 case Primitive::kPrimDouble: { 1429 double_index_ = std::max(double_index_, RoundUp(float_index_, 2)); 1430 uint32_t stack_index = stack_index_; 1431 stack_index_ += 2; 1432 if (double_index_ + 1 < calling_convention.GetNumberOfFpuRegisters()) { 1433 uint32_t index = double_index_; 1434 double_index_ += 2; 1435 Location result = Location::FpuRegisterPairLocation( 1436 calling_convention.GetFpuRegisterAt(index), 1437 calling_convention.GetFpuRegisterAt(index + 1)); 1438 DCHECK(ExpectedPairLayout(result)); 1439 return result; 1440 } else { 1441 return Location::DoubleStackSlot(calling_convention.GetStackOffsetOf(stack_index)); 1442 } 1443 } 1444 1445 case Primitive::kPrimVoid: 1446 LOG(FATAL) << "Unexpected parameter type " << type; 1447 break; 1448 } 1449 return Location::NoLocation(); 1450} 1451 1452Location InvokeDexCallingConventionVisitorARM::GetReturnLocation(Primitive::Type type) const { 1453 switch (type) { 1454 case Primitive::kPrimBoolean: 1455 case Primitive::kPrimByte: 1456 case Primitive::kPrimChar: 1457 case Primitive::kPrimShort: 1458 case Primitive::kPrimInt: 1459 case Primitive::kPrimNot: { 1460 return Location::RegisterLocation(R0); 1461 } 1462 1463 case Primitive::kPrimFloat: { 1464 return Location::FpuRegisterLocation(S0); 1465 } 1466 1467 case Primitive::kPrimLong: { 1468 return Location::RegisterPairLocation(R0, R1); 1469 } 1470 1471 case Primitive::kPrimDouble: { 1472 return Location::FpuRegisterPairLocation(S0, S1); 1473 } 1474 1475 case Primitive::kPrimVoid: 1476 return Location::NoLocation(); 1477 } 1478 1479 UNREACHABLE(); 1480} 1481 1482Location InvokeDexCallingConventionVisitorARM::GetMethodLocation() const { 1483 return Location::RegisterLocation(kMethodRegisterArgument); 1484} 1485 1486void CodeGeneratorARM::Move32(Location destination, Location source) { 1487 if (source.Equals(destination)) { 1488 return; 1489 } 1490 if (destination.IsRegister()) { 1491 if (source.IsRegister()) { 1492 __ Mov(destination.AsRegister<Register>(), source.AsRegister<Register>()); 1493 } else if (source.IsFpuRegister()) { 1494 __ vmovrs(destination.AsRegister<Register>(), source.AsFpuRegister<SRegister>()); 1495 } else { 1496 __ LoadFromOffset(kLoadWord, destination.AsRegister<Register>(), SP, source.GetStackIndex()); 1497 } 1498 } else if (destination.IsFpuRegister()) { 1499 if (source.IsRegister()) { 1500 __ vmovsr(destination.AsFpuRegister<SRegister>(), source.AsRegister<Register>()); 1501 } else if (source.IsFpuRegister()) { 1502 __ vmovs(destination.AsFpuRegister<SRegister>(), source.AsFpuRegister<SRegister>()); 1503 } else { 1504 __ LoadSFromOffset(destination.AsFpuRegister<SRegister>(), SP, source.GetStackIndex()); 1505 } 1506 } else { 1507 DCHECK(destination.IsStackSlot()) << destination; 1508 if (source.IsRegister()) { 1509 __ StoreToOffset(kStoreWord, source.AsRegister<Register>(), SP, destination.GetStackIndex()); 1510 } else if (source.IsFpuRegister()) { 1511 __ StoreSToOffset(source.AsFpuRegister<SRegister>(), SP, destination.GetStackIndex()); 1512 } else { 1513 DCHECK(source.IsStackSlot()) << source; 1514 __ LoadFromOffset(kLoadWord, IP, SP, source.GetStackIndex()); 1515 __ StoreToOffset(kStoreWord, IP, SP, destination.GetStackIndex()); 1516 } 1517 } 1518} 1519 1520void CodeGeneratorARM::Move64(Location destination, Location source) { 1521 if (source.Equals(destination)) { 1522 return; 1523 } 1524 if (destination.IsRegisterPair()) { 1525 if (source.IsRegisterPair()) { 1526 EmitParallelMoves( 1527 Location::RegisterLocation(source.AsRegisterPairHigh<Register>()), 1528 Location::RegisterLocation(destination.AsRegisterPairHigh<Register>()), 1529 Primitive::kPrimInt, 1530 Location::RegisterLocation(source.AsRegisterPairLow<Register>()), 1531 Location::RegisterLocation(destination.AsRegisterPairLow<Register>()), 1532 Primitive::kPrimInt); 1533 } else if (source.IsFpuRegister()) { 1534 UNIMPLEMENTED(FATAL); 1535 } else if (source.IsFpuRegisterPair()) { 1536 __ vmovrrd(destination.AsRegisterPairLow<Register>(), 1537 destination.AsRegisterPairHigh<Register>(), 1538 FromLowSToD(source.AsFpuRegisterPairLow<SRegister>())); 1539 } else { 1540 DCHECK(source.IsDoubleStackSlot()); 1541 DCHECK(ExpectedPairLayout(destination)); 1542 __ LoadFromOffset(kLoadWordPair, destination.AsRegisterPairLow<Register>(), 1543 SP, source.GetStackIndex()); 1544 } 1545 } else if (destination.IsFpuRegisterPair()) { 1546 if (source.IsDoubleStackSlot()) { 1547 __ LoadDFromOffset(FromLowSToD(destination.AsFpuRegisterPairLow<SRegister>()), 1548 SP, 1549 source.GetStackIndex()); 1550 } else if (source.IsRegisterPair()) { 1551 __ vmovdrr(FromLowSToD(destination.AsFpuRegisterPairLow<SRegister>()), 1552 source.AsRegisterPairLow<Register>(), 1553 source.AsRegisterPairHigh<Register>()); 1554 } else { 1555 UNIMPLEMENTED(FATAL); 1556 } 1557 } else { 1558 DCHECK(destination.IsDoubleStackSlot()); 1559 if (source.IsRegisterPair()) { 1560 // No conflict possible, so just do the moves. 1561 if (source.AsRegisterPairLow<Register>() == R1) { 1562 DCHECK_EQ(source.AsRegisterPairHigh<Register>(), R2); 1563 __ StoreToOffset(kStoreWord, R1, SP, destination.GetStackIndex()); 1564 __ StoreToOffset(kStoreWord, R2, SP, destination.GetHighStackIndex(kArmWordSize)); 1565 } else { 1566 __ StoreToOffset(kStoreWordPair, source.AsRegisterPairLow<Register>(), 1567 SP, destination.GetStackIndex()); 1568 } 1569 } else if (source.IsFpuRegisterPair()) { 1570 __ StoreDToOffset(FromLowSToD(source.AsFpuRegisterPairLow<SRegister>()), 1571 SP, 1572 destination.GetStackIndex()); 1573 } else { 1574 DCHECK(source.IsDoubleStackSlot()); 1575 EmitParallelMoves( 1576 Location::StackSlot(source.GetStackIndex()), 1577 Location::StackSlot(destination.GetStackIndex()), 1578 Primitive::kPrimInt, 1579 Location::StackSlot(source.GetHighStackIndex(kArmWordSize)), 1580 Location::StackSlot(destination.GetHighStackIndex(kArmWordSize)), 1581 Primitive::kPrimInt); 1582 } 1583 } 1584} 1585 1586void CodeGeneratorARM::MoveConstant(Location location, int32_t value) { 1587 DCHECK(location.IsRegister()); 1588 __ LoadImmediate(location.AsRegister<Register>(), value); 1589} 1590 1591void CodeGeneratorARM::MoveLocation(Location dst, Location src, Primitive::Type dst_type) { 1592 HParallelMove move(GetGraph()->GetArena()); 1593 move.AddMove(src, dst, dst_type, nullptr); 1594 GetMoveResolver()->EmitNativeCode(&move); 1595} 1596 1597void CodeGeneratorARM::AddLocationAsTemp(Location location, LocationSummary* locations) { 1598 if (location.IsRegister()) { 1599 locations->AddTemp(location); 1600 } else if (location.IsRegisterPair()) { 1601 locations->AddTemp(Location::RegisterLocation(location.AsRegisterPairLow<Register>())); 1602 locations->AddTemp(Location::RegisterLocation(location.AsRegisterPairHigh<Register>())); 1603 } else { 1604 UNIMPLEMENTED(FATAL) << "AddLocationAsTemp not implemented for location " << location; 1605 } 1606} 1607 1608void CodeGeneratorARM::InvokeRuntime(QuickEntrypointEnum entrypoint, 1609 HInstruction* instruction, 1610 uint32_t dex_pc, 1611 SlowPathCode* slow_path) { 1612 ValidateInvokeRuntime(entrypoint, instruction, slow_path); 1613 GenerateInvokeRuntime(GetThreadOffset<kArmPointerSize>(entrypoint).Int32Value()); 1614 if (EntrypointRequiresStackMap(entrypoint)) { 1615 RecordPcInfo(instruction, dex_pc, slow_path); 1616 } 1617} 1618 1619void CodeGeneratorARM::InvokeRuntimeWithoutRecordingPcInfo(int32_t entry_point_offset, 1620 HInstruction* instruction, 1621 SlowPathCode* slow_path) { 1622 ValidateInvokeRuntimeWithoutRecordingPcInfo(instruction, slow_path); 1623 GenerateInvokeRuntime(entry_point_offset); 1624} 1625 1626void CodeGeneratorARM::GenerateInvokeRuntime(int32_t entry_point_offset) { 1627 __ LoadFromOffset(kLoadWord, LR, TR, entry_point_offset); 1628 __ blx(LR); 1629} 1630 1631void InstructionCodeGeneratorARM::HandleGoto(HInstruction* got, HBasicBlock* successor) { 1632 DCHECK(!successor->IsExitBlock()); 1633 1634 HBasicBlock* block = got->GetBlock(); 1635 HInstruction* previous = got->GetPrevious(); 1636 1637 HLoopInformation* info = block->GetLoopInformation(); 1638 if (info != nullptr && info->IsBackEdge(*block) && info->HasSuspendCheck()) { 1639 codegen_->ClearSpillSlotsFromLoopPhisInStackMap(info->GetSuspendCheck()); 1640 GenerateSuspendCheck(info->GetSuspendCheck(), successor); 1641 return; 1642 } 1643 1644 if (block->IsEntryBlock() && (previous != nullptr) && previous->IsSuspendCheck()) { 1645 GenerateSuspendCheck(previous->AsSuspendCheck(), nullptr); 1646 } 1647 if (!codegen_->GoesToNextBlock(got->GetBlock(), successor)) { 1648 __ b(codegen_->GetLabelOf(successor)); 1649 } 1650} 1651 1652void LocationsBuilderARM::VisitGoto(HGoto* got) { 1653 got->SetLocations(nullptr); 1654} 1655 1656void InstructionCodeGeneratorARM::VisitGoto(HGoto* got) { 1657 HandleGoto(got, got->GetSuccessor()); 1658} 1659 1660void LocationsBuilderARM::VisitTryBoundary(HTryBoundary* try_boundary) { 1661 try_boundary->SetLocations(nullptr); 1662} 1663 1664void InstructionCodeGeneratorARM::VisitTryBoundary(HTryBoundary* try_boundary) { 1665 HBasicBlock* successor = try_boundary->GetNormalFlowSuccessor(); 1666 if (!successor->IsExitBlock()) { 1667 HandleGoto(try_boundary, successor); 1668 } 1669} 1670 1671void LocationsBuilderARM::VisitExit(HExit* exit) { 1672 exit->SetLocations(nullptr); 1673} 1674 1675void InstructionCodeGeneratorARM::VisitExit(HExit* exit ATTRIBUTE_UNUSED) { 1676} 1677 1678void InstructionCodeGeneratorARM::GenerateVcmp(HInstruction* instruction) { 1679 Primitive::Type type = instruction->InputAt(0)->GetType(); 1680 Location lhs_loc = instruction->GetLocations()->InAt(0); 1681 Location rhs_loc = instruction->GetLocations()->InAt(1); 1682 if (rhs_loc.IsConstant()) { 1683 // 0.0 is the only immediate that can be encoded directly in 1684 // a VCMP instruction. 1685 // 1686 // Both the JLS (section 15.20.1) and the JVMS (section 6.5) 1687 // specify that in a floating-point comparison, positive zero 1688 // and negative zero are considered equal, so we can use the 1689 // literal 0.0 for both cases here. 1690 // 1691 // Note however that some methods (Float.equal, Float.compare, 1692 // Float.compareTo, Double.equal, Double.compare, 1693 // Double.compareTo, Math.max, Math.min, StrictMath.max, 1694 // StrictMath.min) consider 0.0 to be (strictly) greater than 1695 // -0.0. So if we ever translate calls to these methods into a 1696 // HCompare instruction, we must handle the -0.0 case with 1697 // care here. 1698 DCHECK(rhs_loc.GetConstant()->IsArithmeticZero()); 1699 if (type == Primitive::kPrimFloat) { 1700 __ vcmpsz(lhs_loc.AsFpuRegister<SRegister>()); 1701 } else { 1702 DCHECK_EQ(type, Primitive::kPrimDouble); 1703 __ vcmpdz(FromLowSToD(lhs_loc.AsFpuRegisterPairLow<SRegister>())); 1704 } 1705 } else { 1706 if (type == Primitive::kPrimFloat) { 1707 __ vcmps(lhs_loc.AsFpuRegister<SRegister>(), rhs_loc.AsFpuRegister<SRegister>()); 1708 } else { 1709 DCHECK_EQ(type, Primitive::kPrimDouble); 1710 __ vcmpd(FromLowSToD(lhs_loc.AsFpuRegisterPairLow<SRegister>()), 1711 FromLowSToD(rhs_loc.AsFpuRegisterPairLow<SRegister>())); 1712 } 1713 } 1714} 1715 1716void InstructionCodeGeneratorARM::GenerateFPJumps(HCondition* cond, 1717 Label* true_label, 1718 Label* false_label ATTRIBUTE_UNUSED) { 1719 __ vmstat(); // transfer FP status register to ARM APSR. 1720 __ b(true_label, ARMFPCondition(cond->GetCondition(), cond->IsGtBias())); 1721} 1722 1723void InstructionCodeGeneratorARM::GenerateLongComparesAndJumps(HCondition* cond, 1724 Label* true_label, 1725 Label* false_label) { 1726 LocationSummary* locations = cond->GetLocations(); 1727 Location left = locations->InAt(0); 1728 Location right = locations->InAt(1); 1729 IfCondition if_cond = cond->GetCondition(); 1730 1731 Register left_high = left.AsRegisterPairHigh<Register>(); 1732 Register left_low = left.AsRegisterPairLow<Register>(); 1733 IfCondition true_high_cond = if_cond; 1734 IfCondition false_high_cond = cond->GetOppositeCondition(); 1735 Condition final_condition = ARMUnsignedCondition(if_cond); // unsigned on lower part 1736 1737 // Set the conditions for the test, remembering that == needs to be 1738 // decided using the low words. 1739 // TODO: consider avoiding jumps with temporary and CMP low+SBC high 1740 switch (if_cond) { 1741 case kCondEQ: 1742 case kCondNE: 1743 // Nothing to do. 1744 break; 1745 case kCondLT: 1746 false_high_cond = kCondGT; 1747 break; 1748 case kCondLE: 1749 true_high_cond = kCondLT; 1750 break; 1751 case kCondGT: 1752 false_high_cond = kCondLT; 1753 break; 1754 case kCondGE: 1755 true_high_cond = kCondGT; 1756 break; 1757 case kCondB: 1758 false_high_cond = kCondA; 1759 break; 1760 case kCondBE: 1761 true_high_cond = kCondB; 1762 break; 1763 case kCondA: 1764 false_high_cond = kCondB; 1765 break; 1766 case kCondAE: 1767 true_high_cond = kCondA; 1768 break; 1769 } 1770 if (right.IsConstant()) { 1771 int64_t value = right.GetConstant()->AsLongConstant()->GetValue(); 1772 int32_t val_low = Low32Bits(value); 1773 int32_t val_high = High32Bits(value); 1774 1775 __ CmpConstant(left_high, val_high); 1776 if (if_cond == kCondNE) { 1777 __ b(true_label, ARMCondition(true_high_cond)); 1778 } else if (if_cond == kCondEQ) { 1779 __ b(false_label, ARMCondition(false_high_cond)); 1780 } else { 1781 __ b(true_label, ARMCondition(true_high_cond)); 1782 __ b(false_label, ARMCondition(false_high_cond)); 1783 } 1784 // Must be equal high, so compare the lows. 1785 __ CmpConstant(left_low, val_low); 1786 } else { 1787 Register right_high = right.AsRegisterPairHigh<Register>(); 1788 Register right_low = right.AsRegisterPairLow<Register>(); 1789 1790 __ cmp(left_high, ShifterOperand(right_high)); 1791 if (if_cond == kCondNE) { 1792 __ b(true_label, ARMCondition(true_high_cond)); 1793 } else if (if_cond == kCondEQ) { 1794 __ b(false_label, ARMCondition(false_high_cond)); 1795 } else { 1796 __ b(true_label, ARMCondition(true_high_cond)); 1797 __ b(false_label, ARMCondition(false_high_cond)); 1798 } 1799 // Must be equal high, so compare the lows. 1800 __ cmp(left_low, ShifterOperand(right_low)); 1801 } 1802 // The last comparison might be unsigned. 1803 // TODO: optimize cases where this is always true/false 1804 __ b(true_label, final_condition); 1805} 1806 1807void InstructionCodeGeneratorARM::GenerateCompareTestAndBranch(HCondition* condition, 1808 Label* true_target_in, 1809 Label* false_target_in) { 1810 // Generated branching requires both targets to be explicit. If either of the 1811 // targets is nullptr (fallthrough) use and bind `fallthrough_target` instead. 1812 Label fallthrough_target; 1813 Label* true_target = true_target_in == nullptr ? &fallthrough_target : true_target_in; 1814 Label* false_target = false_target_in == nullptr ? &fallthrough_target : false_target_in; 1815 1816 Primitive::Type type = condition->InputAt(0)->GetType(); 1817 switch (type) { 1818 case Primitive::kPrimLong: 1819 GenerateLongComparesAndJumps(condition, true_target, false_target); 1820 break; 1821 case Primitive::kPrimFloat: 1822 case Primitive::kPrimDouble: 1823 GenerateVcmp(condition); 1824 GenerateFPJumps(condition, true_target, false_target); 1825 break; 1826 default: 1827 LOG(FATAL) << "Unexpected compare type " << type; 1828 } 1829 1830 if (false_target != &fallthrough_target) { 1831 __ b(false_target); 1832 } 1833 1834 if (fallthrough_target.IsLinked()) { 1835 __ Bind(&fallthrough_target); 1836 } 1837} 1838 1839void InstructionCodeGeneratorARM::GenerateTestAndBranch(HInstruction* instruction, 1840 size_t condition_input_index, 1841 Label* true_target, 1842 Label* false_target) { 1843 HInstruction* cond = instruction->InputAt(condition_input_index); 1844 1845 if (true_target == nullptr && false_target == nullptr) { 1846 // Nothing to do. The code always falls through. 1847 return; 1848 } else if (cond->IsIntConstant()) { 1849 // Constant condition, statically compared against "true" (integer value 1). 1850 if (cond->AsIntConstant()->IsTrue()) { 1851 if (true_target != nullptr) { 1852 __ b(true_target); 1853 } 1854 } else { 1855 DCHECK(cond->AsIntConstant()->IsFalse()) << cond->AsIntConstant()->GetValue(); 1856 if (false_target != nullptr) { 1857 __ b(false_target); 1858 } 1859 } 1860 return; 1861 } 1862 1863 // The following code generates these patterns: 1864 // (1) true_target == nullptr && false_target != nullptr 1865 // - opposite condition true => branch to false_target 1866 // (2) true_target != nullptr && false_target == nullptr 1867 // - condition true => branch to true_target 1868 // (3) true_target != nullptr && false_target != nullptr 1869 // - condition true => branch to true_target 1870 // - branch to false_target 1871 if (IsBooleanValueOrMaterializedCondition(cond)) { 1872 // Condition has been materialized, compare the output to 0. 1873 Location cond_val = instruction->GetLocations()->InAt(condition_input_index); 1874 DCHECK(cond_val.IsRegister()); 1875 if (true_target == nullptr) { 1876 __ CompareAndBranchIfZero(cond_val.AsRegister<Register>(), false_target); 1877 } else { 1878 __ CompareAndBranchIfNonZero(cond_val.AsRegister<Register>(), true_target); 1879 } 1880 } else { 1881 // Condition has not been materialized. Use its inputs as the comparison and 1882 // its condition as the branch condition. 1883 HCondition* condition = cond->AsCondition(); 1884 1885 // If this is a long or FP comparison that has been folded into 1886 // the HCondition, generate the comparison directly. 1887 Primitive::Type type = condition->InputAt(0)->GetType(); 1888 if (type == Primitive::kPrimLong || Primitive::IsFloatingPointType(type)) { 1889 GenerateCompareTestAndBranch(condition, true_target, false_target); 1890 return; 1891 } 1892 1893 LocationSummary* locations = cond->GetLocations(); 1894 DCHECK(locations->InAt(0).IsRegister()); 1895 Register left = locations->InAt(0).AsRegister<Register>(); 1896 Location right = locations->InAt(1); 1897 if (right.IsRegister()) { 1898 __ cmp(left, ShifterOperand(right.AsRegister<Register>())); 1899 } else { 1900 DCHECK(right.IsConstant()); 1901 __ CmpConstant(left, CodeGenerator::GetInt32ValueOf(right.GetConstant())); 1902 } 1903 if (true_target == nullptr) { 1904 __ b(false_target, ARMCondition(condition->GetOppositeCondition())); 1905 } else { 1906 __ b(true_target, ARMCondition(condition->GetCondition())); 1907 } 1908 } 1909 1910 // If neither branch falls through (case 3), the conditional branch to `true_target` 1911 // was already emitted (case 2) and we need to emit a jump to `false_target`. 1912 if (true_target != nullptr && false_target != nullptr) { 1913 __ b(false_target); 1914 } 1915} 1916 1917void LocationsBuilderARM::VisitIf(HIf* if_instr) { 1918 LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(if_instr); 1919 if (IsBooleanValueOrMaterializedCondition(if_instr->InputAt(0))) { 1920 locations->SetInAt(0, Location::RequiresRegister()); 1921 } 1922} 1923 1924void InstructionCodeGeneratorARM::VisitIf(HIf* if_instr) { 1925 HBasicBlock* true_successor = if_instr->IfTrueSuccessor(); 1926 HBasicBlock* false_successor = if_instr->IfFalseSuccessor(); 1927 Label* true_target = codegen_->GoesToNextBlock(if_instr->GetBlock(), true_successor) ? 1928 nullptr : codegen_->GetLabelOf(true_successor); 1929 Label* false_target = codegen_->GoesToNextBlock(if_instr->GetBlock(), false_successor) ? 1930 nullptr : codegen_->GetLabelOf(false_successor); 1931 GenerateTestAndBranch(if_instr, /* condition_input_index */ 0, true_target, false_target); 1932} 1933 1934void LocationsBuilderARM::VisitDeoptimize(HDeoptimize* deoptimize) { 1935 LocationSummary* locations = new (GetGraph()->GetArena()) 1936 LocationSummary(deoptimize, LocationSummary::kCallOnSlowPath); 1937 locations->SetCustomSlowPathCallerSaves(RegisterSet::Empty()); // No caller-save registers. 1938 if (IsBooleanValueOrMaterializedCondition(deoptimize->InputAt(0))) { 1939 locations->SetInAt(0, Location::RequiresRegister()); 1940 } 1941} 1942 1943void InstructionCodeGeneratorARM::VisitDeoptimize(HDeoptimize* deoptimize) { 1944 SlowPathCodeARM* slow_path = deopt_slow_paths_.NewSlowPath<DeoptimizationSlowPathARM>(deoptimize); 1945 GenerateTestAndBranch(deoptimize, 1946 /* condition_input_index */ 0, 1947 slow_path->GetEntryLabel(), 1948 /* false_target */ nullptr); 1949} 1950 1951void LocationsBuilderARM::VisitShouldDeoptimizeFlag(HShouldDeoptimizeFlag* flag) { 1952 LocationSummary* locations = new (GetGraph()->GetArena()) 1953 LocationSummary(flag, LocationSummary::kNoCall); 1954 locations->SetOut(Location::RequiresRegister()); 1955} 1956 1957void InstructionCodeGeneratorARM::VisitShouldDeoptimizeFlag(HShouldDeoptimizeFlag* flag) { 1958 __ LoadFromOffset(kLoadWord, 1959 flag->GetLocations()->Out().AsRegister<Register>(), 1960 SP, 1961 codegen_->GetStackOffsetOfShouldDeoptimizeFlag()); 1962} 1963 1964void LocationsBuilderARM::VisitSelect(HSelect* select) { 1965 LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(select); 1966 if (Primitive::IsFloatingPointType(select->GetType())) { 1967 locations->SetInAt(0, Location::RequiresFpuRegister()); 1968 locations->SetInAt(1, Location::RequiresFpuRegister()); 1969 } else { 1970 locations->SetInAt(0, Location::RequiresRegister()); 1971 locations->SetInAt(1, Location::RequiresRegister()); 1972 } 1973 if (IsBooleanValueOrMaterializedCondition(select->GetCondition())) { 1974 locations->SetInAt(2, Location::RequiresRegister()); 1975 } 1976 locations->SetOut(Location::SameAsFirstInput()); 1977} 1978 1979void InstructionCodeGeneratorARM::VisitSelect(HSelect* select) { 1980 LocationSummary* locations = select->GetLocations(); 1981 Label false_target; 1982 GenerateTestAndBranch(select, 1983 /* condition_input_index */ 2, 1984 /* true_target */ nullptr, 1985 &false_target); 1986 codegen_->MoveLocation(locations->Out(), locations->InAt(1), select->GetType()); 1987 __ Bind(&false_target); 1988} 1989 1990void LocationsBuilderARM::VisitNativeDebugInfo(HNativeDebugInfo* info) { 1991 new (GetGraph()->GetArena()) LocationSummary(info); 1992} 1993 1994void InstructionCodeGeneratorARM::VisitNativeDebugInfo(HNativeDebugInfo*) { 1995 // MaybeRecordNativeDebugInfo is already called implicitly in CodeGenerator::Compile. 1996} 1997 1998void CodeGeneratorARM::GenerateNop() { 1999 __ nop(); 2000} 2001 2002void LocationsBuilderARM::HandleCondition(HCondition* cond) { 2003 LocationSummary* locations = 2004 new (GetGraph()->GetArena()) LocationSummary(cond, LocationSummary::kNoCall); 2005 // Handle the long/FP comparisons made in instruction simplification. 2006 switch (cond->InputAt(0)->GetType()) { 2007 case Primitive::kPrimLong: 2008 locations->SetInAt(0, Location::RequiresRegister()); 2009 locations->SetInAt(1, Location::RegisterOrConstant(cond->InputAt(1))); 2010 if (!cond->IsEmittedAtUseSite()) { 2011 locations->SetOut(Location::RequiresRegister(), Location::kOutputOverlap); 2012 } 2013 break; 2014 2015 case Primitive::kPrimFloat: 2016 case Primitive::kPrimDouble: 2017 locations->SetInAt(0, Location::RequiresFpuRegister()); 2018 locations->SetInAt(1, ArithmeticZeroOrFpuRegister(cond->InputAt(1))); 2019 if (!cond->IsEmittedAtUseSite()) { 2020 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 2021 } 2022 break; 2023 2024 default: 2025 locations->SetInAt(0, Location::RequiresRegister()); 2026 locations->SetInAt(1, Location::RegisterOrConstant(cond->InputAt(1))); 2027 if (!cond->IsEmittedAtUseSite()) { 2028 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 2029 } 2030 } 2031} 2032 2033void InstructionCodeGeneratorARM::HandleCondition(HCondition* cond) { 2034 if (cond->IsEmittedAtUseSite()) { 2035 return; 2036 } 2037 2038 LocationSummary* locations = cond->GetLocations(); 2039 Location left = locations->InAt(0); 2040 Location right = locations->InAt(1); 2041 Register out = locations->Out().AsRegister<Register>(); 2042 Label true_label, false_label; 2043 2044 switch (cond->InputAt(0)->GetType()) { 2045 default: { 2046 // Integer case. 2047 if (right.IsRegister()) { 2048 __ cmp(left.AsRegister<Register>(), ShifterOperand(right.AsRegister<Register>())); 2049 } else { 2050 DCHECK(right.IsConstant()); 2051 __ CmpConstant(left.AsRegister<Register>(), 2052 CodeGenerator::GetInt32ValueOf(right.GetConstant())); 2053 } 2054 __ it(ARMCondition(cond->GetCondition()), kItElse); 2055 __ mov(locations->Out().AsRegister<Register>(), ShifterOperand(1), 2056 ARMCondition(cond->GetCondition())); 2057 __ mov(locations->Out().AsRegister<Register>(), ShifterOperand(0), 2058 ARMCondition(cond->GetOppositeCondition())); 2059 return; 2060 } 2061 case Primitive::kPrimLong: 2062 GenerateLongComparesAndJumps(cond, &true_label, &false_label); 2063 break; 2064 case Primitive::kPrimFloat: 2065 case Primitive::kPrimDouble: 2066 GenerateVcmp(cond); 2067 GenerateFPJumps(cond, &true_label, &false_label); 2068 break; 2069 } 2070 2071 // Convert the jumps into the result. 2072 Label done_label; 2073 2074 // False case: result = 0. 2075 __ Bind(&false_label); 2076 __ LoadImmediate(out, 0); 2077 __ b(&done_label); 2078 2079 // True case: result = 1. 2080 __ Bind(&true_label); 2081 __ LoadImmediate(out, 1); 2082 __ Bind(&done_label); 2083} 2084 2085void LocationsBuilderARM::VisitEqual(HEqual* comp) { 2086 HandleCondition(comp); 2087} 2088 2089void InstructionCodeGeneratorARM::VisitEqual(HEqual* comp) { 2090 HandleCondition(comp); 2091} 2092 2093void LocationsBuilderARM::VisitNotEqual(HNotEqual* comp) { 2094 HandleCondition(comp); 2095} 2096 2097void InstructionCodeGeneratorARM::VisitNotEqual(HNotEqual* comp) { 2098 HandleCondition(comp); 2099} 2100 2101void LocationsBuilderARM::VisitLessThan(HLessThan* comp) { 2102 HandleCondition(comp); 2103} 2104 2105void InstructionCodeGeneratorARM::VisitLessThan(HLessThan* comp) { 2106 HandleCondition(comp); 2107} 2108 2109void LocationsBuilderARM::VisitLessThanOrEqual(HLessThanOrEqual* comp) { 2110 HandleCondition(comp); 2111} 2112 2113void InstructionCodeGeneratorARM::VisitLessThanOrEqual(HLessThanOrEqual* comp) { 2114 HandleCondition(comp); 2115} 2116 2117void LocationsBuilderARM::VisitGreaterThan(HGreaterThan* comp) { 2118 HandleCondition(comp); 2119} 2120 2121void InstructionCodeGeneratorARM::VisitGreaterThan(HGreaterThan* comp) { 2122 HandleCondition(comp); 2123} 2124 2125void LocationsBuilderARM::VisitGreaterThanOrEqual(HGreaterThanOrEqual* comp) { 2126 HandleCondition(comp); 2127} 2128 2129void InstructionCodeGeneratorARM::VisitGreaterThanOrEqual(HGreaterThanOrEqual* comp) { 2130 HandleCondition(comp); 2131} 2132 2133void LocationsBuilderARM::VisitBelow(HBelow* comp) { 2134 HandleCondition(comp); 2135} 2136 2137void InstructionCodeGeneratorARM::VisitBelow(HBelow* comp) { 2138 HandleCondition(comp); 2139} 2140 2141void LocationsBuilderARM::VisitBelowOrEqual(HBelowOrEqual* comp) { 2142 HandleCondition(comp); 2143} 2144 2145void InstructionCodeGeneratorARM::VisitBelowOrEqual(HBelowOrEqual* comp) { 2146 HandleCondition(comp); 2147} 2148 2149void LocationsBuilderARM::VisitAbove(HAbove* comp) { 2150 HandleCondition(comp); 2151} 2152 2153void InstructionCodeGeneratorARM::VisitAbove(HAbove* comp) { 2154 HandleCondition(comp); 2155} 2156 2157void LocationsBuilderARM::VisitAboveOrEqual(HAboveOrEqual* comp) { 2158 HandleCondition(comp); 2159} 2160 2161void InstructionCodeGeneratorARM::VisitAboveOrEqual(HAboveOrEqual* comp) { 2162 HandleCondition(comp); 2163} 2164 2165void LocationsBuilderARM::VisitIntConstant(HIntConstant* constant) { 2166 LocationSummary* locations = 2167 new (GetGraph()->GetArena()) LocationSummary(constant, LocationSummary::kNoCall); 2168 locations->SetOut(Location::ConstantLocation(constant)); 2169} 2170 2171void InstructionCodeGeneratorARM::VisitIntConstant(HIntConstant* constant ATTRIBUTE_UNUSED) { 2172 // Will be generated at use site. 2173} 2174 2175void LocationsBuilderARM::VisitNullConstant(HNullConstant* constant) { 2176 LocationSummary* locations = 2177 new (GetGraph()->GetArena()) LocationSummary(constant, LocationSummary::kNoCall); 2178 locations->SetOut(Location::ConstantLocation(constant)); 2179} 2180 2181void InstructionCodeGeneratorARM::VisitNullConstant(HNullConstant* constant ATTRIBUTE_UNUSED) { 2182 // Will be generated at use site. 2183} 2184 2185void LocationsBuilderARM::VisitLongConstant(HLongConstant* constant) { 2186 LocationSummary* locations = 2187 new (GetGraph()->GetArena()) LocationSummary(constant, LocationSummary::kNoCall); 2188 locations->SetOut(Location::ConstantLocation(constant)); 2189} 2190 2191void InstructionCodeGeneratorARM::VisitLongConstant(HLongConstant* constant ATTRIBUTE_UNUSED) { 2192 // Will be generated at use site. 2193} 2194 2195void LocationsBuilderARM::VisitFloatConstant(HFloatConstant* constant) { 2196 LocationSummary* locations = 2197 new (GetGraph()->GetArena()) LocationSummary(constant, LocationSummary::kNoCall); 2198 locations->SetOut(Location::ConstantLocation(constant)); 2199} 2200 2201void InstructionCodeGeneratorARM::VisitFloatConstant(HFloatConstant* constant ATTRIBUTE_UNUSED) { 2202 // Will be generated at use site. 2203} 2204 2205void LocationsBuilderARM::VisitDoubleConstant(HDoubleConstant* constant) { 2206 LocationSummary* locations = 2207 new (GetGraph()->GetArena()) LocationSummary(constant, LocationSummary::kNoCall); 2208 locations->SetOut(Location::ConstantLocation(constant)); 2209} 2210 2211void InstructionCodeGeneratorARM::VisitDoubleConstant(HDoubleConstant* constant ATTRIBUTE_UNUSED) { 2212 // Will be generated at use site. 2213} 2214 2215void LocationsBuilderARM::VisitMemoryBarrier(HMemoryBarrier* memory_barrier) { 2216 memory_barrier->SetLocations(nullptr); 2217} 2218 2219void InstructionCodeGeneratorARM::VisitMemoryBarrier(HMemoryBarrier* memory_barrier) { 2220 codegen_->GenerateMemoryBarrier(memory_barrier->GetBarrierKind()); 2221} 2222 2223void LocationsBuilderARM::VisitReturnVoid(HReturnVoid* ret) { 2224 ret->SetLocations(nullptr); 2225} 2226 2227void InstructionCodeGeneratorARM::VisitReturnVoid(HReturnVoid* ret ATTRIBUTE_UNUSED) { 2228 codegen_->GenerateFrameExit(); 2229} 2230 2231void LocationsBuilderARM::VisitReturn(HReturn* ret) { 2232 LocationSummary* locations = 2233 new (GetGraph()->GetArena()) LocationSummary(ret, LocationSummary::kNoCall); 2234 locations->SetInAt(0, parameter_visitor_.GetReturnLocation(ret->InputAt(0)->GetType())); 2235} 2236 2237void InstructionCodeGeneratorARM::VisitReturn(HReturn* ret ATTRIBUTE_UNUSED) { 2238 codegen_->GenerateFrameExit(); 2239} 2240 2241void LocationsBuilderARM::VisitInvokeUnresolved(HInvokeUnresolved* invoke) { 2242 // The trampoline uses the same calling convention as dex calling conventions, 2243 // except instead of loading arg0/r0 with the target Method*, arg0/r0 will contain 2244 // the method_idx. 2245 HandleInvoke(invoke); 2246} 2247 2248void InstructionCodeGeneratorARM::VisitInvokeUnresolved(HInvokeUnresolved* invoke) { 2249 codegen_->GenerateInvokeUnresolvedRuntimeCall(invoke); 2250} 2251 2252void LocationsBuilderARM::VisitInvokeStaticOrDirect(HInvokeStaticOrDirect* invoke) { 2253 // Explicit clinit checks triggered by static invokes must have been pruned by 2254 // art::PrepareForRegisterAllocation. 2255 DCHECK(!invoke->IsStaticWithExplicitClinitCheck()); 2256 2257 IntrinsicLocationsBuilderARM intrinsic(codegen_); 2258 if (intrinsic.TryDispatch(invoke)) { 2259 if (invoke->GetLocations()->CanCall() && invoke->HasPcRelativeDexCache()) { 2260 invoke->GetLocations()->SetInAt(invoke->GetSpecialInputIndex(), Location::Any()); 2261 } 2262 return; 2263 } 2264 2265 HandleInvoke(invoke); 2266 2267 // For PC-relative dex cache the invoke has an extra input, the PC-relative address base. 2268 if (invoke->HasPcRelativeDexCache()) { 2269 invoke->GetLocations()->SetInAt(invoke->GetSpecialInputIndex(), Location::RequiresRegister()); 2270 } 2271} 2272 2273static bool TryGenerateIntrinsicCode(HInvoke* invoke, CodeGeneratorARM* codegen) { 2274 if (invoke->GetLocations()->Intrinsified()) { 2275 IntrinsicCodeGeneratorARM intrinsic(codegen); 2276 intrinsic.Dispatch(invoke); 2277 return true; 2278 } 2279 return false; 2280} 2281 2282void InstructionCodeGeneratorARM::VisitInvokeStaticOrDirect(HInvokeStaticOrDirect* invoke) { 2283 // Explicit clinit checks triggered by static invokes must have been pruned by 2284 // art::PrepareForRegisterAllocation. 2285 DCHECK(!invoke->IsStaticWithExplicitClinitCheck()); 2286 2287 if (TryGenerateIntrinsicCode(invoke, codegen_)) { 2288 return; 2289 } 2290 2291 LocationSummary* locations = invoke->GetLocations(); 2292 codegen_->GenerateStaticOrDirectCall( 2293 invoke, locations->HasTemps() ? locations->GetTemp(0) : Location::NoLocation()); 2294 codegen_->RecordPcInfo(invoke, invoke->GetDexPc()); 2295} 2296 2297void LocationsBuilderARM::HandleInvoke(HInvoke* invoke) { 2298 InvokeDexCallingConventionVisitorARM calling_convention_visitor; 2299 CodeGenerator::CreateCommonInvokeLocationSummary(invoke, &calling_convention_visitor); 2300} 2301 2302void LocationsBuilderARM::VisitInvokeVirtual(HInvokeVirtual* invoke) { 2303 IntrinsicLocationsBuilderARM intrinsic(codegen_); 2304 if (intrinsic.TryDispatch(invoke)) { 2305 return; 2306 } 2307 2308 HandleInvoke(invoke); 2309} 2310 2311void InstructionCodeGeneratorARM::VisitInvokeVirtual(HInvokeVirtual* invoke) { 2312 if (TryGenerateIntrinsicCode(invoke, codegen_)) { 2313 return; 2314 } 2315 2316 codegen_->GenerateVirtualCall(invoke, invoke->GetLocations()->GetTemp(0)); 2317 DCHECK(!codegen_->IsLeafMethod()); 2318 codegen_->RecordPcInfo(invoke, invoke->GetDexPc()); 2319} 2320 2321void LocationsBuilderARM::VisitInvokeInterface(HInvokeInterface* invoke) { 2322 HandleInvoke(invoke); 2323 // Add the hidden argument. 2324 invoke->GetLocations()->AddTemp(Location::RegisterLocation(R12)); 2325} 2326 2327void InstructionCodeGeneratorARM::VisitInvokeInterface(HInvokeInterface* invoke) { 2328 // TODO: b/18116999, our IMTs can miss an IncompatibleClassChangeError. 2329 LocationSummary* locations = invoke->GetLocations(); 2330 Register temp = locations->GetTemp(0).AsRegister<Register>(); 2331 Register hidden_reg = locations->GetTemp(1).AsRegister<Register>(); 2332 Location receiver = locations->InAt(0); 2333 uint32_t class_offset = mirror::Object::ClassOffset().Int32Value(); 2334 2335 // Set the hidden argument. This is safe to do this here, as R12 2336 // won't be modified thereafter, before the `blx` (call) instruction. 2337 DCHECK_EQ(R12, hidden_reg); 2338 __ LoadImmediate(hidden_reg, invoke->GetDexMethodIndex()); 2339 2340 if (receiver.IsStackSlot()) { 2341 __ LoadFromOffset(kLoadWord, temp, SP, receiver.GetStackIndex()); 2342 // /* HeapReference<Class> */ temp = temp->klass_ 2343 __ LoadFromOffset(kLoadWord, temp, temp, class_offset); 2344 } else { 2345 // /* HeapReference<Class> */ temp = receiver->klass_ 2346 __ LoadFromOffset(kLoadWord, temp, receiver.AsRegister<Register>(), class_offset); 2347 } 2348 codegen_->MaybeRecordImplicitNullCheck(invoke); 2349 // Instead of simply (possibly) unpoisoning `temp` here, we should 2350 // emit a read barrier for the previous class reference load. 2351 // However this is not required in practice, as this is an 2352 // intermediate/temporary reference and because the current 2353 // concurrent copying collector keeps the from-space memory 2354 // intact/accessible until the end of the marking phase (the 2355 // concurrent copying collector may not in the future). 2356 __ MaybeUnpoisonHeapReference(temp); 2357 __ LoadFromOffset(kLoadWord, temp, temp, 2358 mirror::Class::ImtPtrOffset(kArmPointerSize).Uint32Value()); 2359 uint32_t method_offset = static_cast<uint32_t>(ImTable::OffsetOfElement( 2360 invoke->GetImtIndex(), kArmPointerSize)); 2361 // temp = temp->GetImtEntryAt(method_offset); 2362 __ LoadFromOffset(kLoadWord, temp, temp, method_offset); 2363 uint32_t entry_point = 2364 ArtMethod::EntryPointFromQuickCompiledCodeOffset(kArmPointerSize).Int32Value(); 2365 // LR = temp->GetEntryPoint(); 2366 __ LoadFromOffset(kLoadWord, LR, temp, entry_point); 2367 // LR(); 2368 __ blx(LR); 2369 DCHECK(!codegen_->IsLeafMethod()); 2370 codegen_->RecordPcInfo(invoke, invoke->GetDexPc()); 2371} 2372 2373void LocationsBuilderARM::VisitNeg(HNeg* neg) { 2374 LocationSummary* locations = 2375 new (GetGraph()->GetArena()) LocationSummary(neg, LocationSummary::kNoCall); 2376 switch (neg->GetResultType()) { 2377 case Primitive::kPrimInt: { 2378 locations->SetInAt(0, Location::RequiresRegister()); 2379 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 2380 break; 2381 } 2382 case Primitive::kPrimLong: { 2383 locations->SetInAt(0, Location::RequiresRegister()); 2384 locations->SetOut(Location::RequiresRegister(), Location::kOutputOverlap); 2385 break; 2386 } 2387 2388 case Primitive::kPrimFloat: 2389 case Primitive::kPrimDouble: 2390 locations->SetInAt(0, Location::RequiresFpuRegister()); 2391 locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap); 2392 break; 2393 2394 default: 2395 LOG(FATAL) << "Unexpected neg type " << neg->GetResultType(); 2396 } 2397} 2398 2399void InstructionCodeGeneratorARM::VisitNeg(HNeg* neg) { 2400 LocationSummary* locations = neg->GetLocations(); 2401 Location out = locations->Out(); 2402 Location in = locations->InAt(0); 2403 switch (neg->GetResultType()) { 2404 case Primitive::kPrimInt: 2405 DCHECK(in.IsRegister()); 2406 __ rsb(out.AsRegister<Register>(), in.AsRegister<Register>(), ShifterOperand(0)); 2407 break; 2408 2409 case Primitive::kPrimLong: 2410 DCHECK(in.IsRegisterPair()); 2411 // out.lo = 0 - in.lo (and update the carry/borrow (C) flag) 2412 __ rsbs(out.AsRegisterPairLow<Register>(), 2413 in.AsRegisterPairLow<Register>(), 2414 ShifterOperand(0)); 2415 // We cannot emit an RSC (Reverse Subtract with Carry) 2416 // instruction here, as it does not exist in the Thumb-2 2417 // instruction set. We use the following approach 2418 // using SBC and SUB instead. 2419 // 2420 // out.hi = -C 2421 __ sbc(out.AsRegisterPairHigh<Register>(), 2422 out.AsRegisterPairHigh<Register>(), 2423 ShifterOperand(out.AsRegisterPairHigh<Register>())); 2424 // out.hi = out.hi - in.hi 2425 __ sub(out.AsRegisterPairHigh<Register>(), 2426 out.AsRegisterPairHigh<Register>(), 2427 ShifterOperand(in.AsRegisterPairHigh<Register>())); 2428 break; 2429 2430 case Primitive::kPrimFloat: 2431 DCHECK(in.IsFpuRegister()); 2432 __ vnegs(out.AsFpuRegister<SRegister>(), in.AsFpuRegister<SRegister>()); 2433 break; 2434 2435 case Primitive::kPrimDouble: 2436 DCHECK(in.IsFpuRegisterPair()); 2437 __ vnegd(FromLowSToD(out.AsFpuRegisterPairLow<SRegister>()), 2438 FromLowSToD(in.AsFpuRegisterPairLow<SRegister>())); 2439 break; 2440 2441 default: 2442 LOG(FATAL) << "Unexpected neg type " << neg->GetResultType(); 2443 } 2444} 2445 2446void LocationsBuilderARM::VisitTypeConversion(HTypeConversion* conversion) { 2447 Primitive::Type result_type = conversion->GetResultType(); 2448 Primitive::Type input_type = conversion->GetInputType(); 2449 DCHECK_NE(result_type, input_type); 2450 2451 // The float-to-long, double-to-long and long-to-float type conversions 2452 // rely on a call to the runtime. 2453 LocationSummary::CallKind call_kind = 2454 (((input_type == Primitive::kPrimFloat || input_type == Primitive::kPrimDouble) 2455 && result_type == Primitive::kPrimLong) 2456 || (input_type == Primitive::kPrimLong && result_type == Primitive::kPrimFloat)) 2457 ? LocationSummary::kCallOnMainOnly 2458 : LocationSummary::kNoCall; 2459 LocationSummary* locations = 2460 new (GetGraph()->GetArena()) LocationSummary(conversion, call_kind); 2461 2462 // The Java language does not allow treating boolean as an integral type but 2463 // our bit representation makes it safe. 2464 2465 switch (result_type) { 2466 case Primitive::kPrimByte: 2467 switch (input_type) { 2468 case Primitive::kPrimLong: 2469 // Type conversion from long to byte is a result of code transformations. 2470 case Primitive::kPrimBoolean: 2471 // Boolean input is a result of code transformations. 2472 case Primitive::kPrimShort: 2473 case Primitive::kPrimInt: 2474 case Primitive::kPrimChar: 2475 // Processing a Dex `int-to-byte' instruction. 2476 locations->SetInAt(0, Location::RequiresRegister()); 2477 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 2478 break; 2479 2480 default: 2481 LOG(FATAL) << "Unexpected type conversion from " << input_type 2482 << " to " << result_type; 2483 } 2484 break; 2485 2486 case Primitive::kPrimShort: 2487 switch (input_type) { 2488 case Primitive::kPrimLong: 2489 // Type conversion from long to short is a result of code transformations. 2490 case Primitive::kPrimBoolean: 2491 // Boolean input is a result of code transformations. 2492 case Primitive::kPrimByte: 2493 case Primitive::kPrimInt: 2494 case Primitive::kPrimChar: 2495 // Processing a Dex `int-to-short' instruction. 2496 locations->SetInAt(0, Location::RequiresRegister()); 2497 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 2498 break; 2499 2500 default: 2501 LOG(FATAL) << "Unexpected type conversion from " << input_type 2502 << " to " << result_type; 2503 } 2504 break; 2505 2506 case Primitive::kPrimInt: 2507 switch (input_type) { 2508 case Primitive::kPrimLong: 2509 // Processing a Dex `long-to-int' instruction. 2510 locations->SetInAt(0, Location::Any()); 2511 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 2512 break; 2513 2514 case Primitive::kPrimFloat: 2515 // Processing a Dex `float-to-int' instruction. 2516 locations->SetInAt(0, Location::RequiresFpuRegister()); 2517 locations->SetOut(Location::RequiresRegister()); 2518 locations->AddTemp(Location::RequiresFpuRegister()); 2519 break; 2520 2521 case Primitive::kPrimDouble: 2522 // Processing a Dex `double-to-int' instruction. 2523 locations->SetInAt(0, Location::RequiresFpuRegister()); 2524 locations->SetOut(Location::RequiresRegister()); 2525 locations->AddTemp(Location::RequiresFpuRegister()); 2526 break; 2527 2528 default: 2529 LOG(FATAL) << "Unexpected type conversion from " << input_type 2530 << " to " << result_type; 2531 } 2532 break; 2533 2534 case Primitive::kPrimLong: 2535 switch (input_type) { 2536 case Primitive::kPrimBoolean: 2537 // Boolean input is a result of code transformations. 2538 case Primitive::kPrimByte: 2539 case Primitive::kPrimShort: 2540 case Primitive::kPrimInt: 2541 case Primitive::kPrimChar: 2542 // Processing a Dex `int-to-long' instruction. 2543 locations->SetInAt(0, Location::RequiresRegister()); 2544 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 2545 break; 2546 2547 case Primitive::kPrimFloat: { 2548 // Processing a Dex `float-to-long' instruction. 2549 InvokeRuntimeCallingConvention calling_convention; 2550 locations->SetInAt(0, Location::FpuRegisterLocation( 2551 calling_convention.GetFpuRegisterAt(0))); 2552 locations->SetOut(Location::RegisterPairLocation(R0, R1)); 2553 break; 2554 } 2555 2556 case Primitive::kPrimDouble: { 2557 // Processing a Dex `double-to-long' instruction. 2558 InvokeRuntimeCallingConvention calling_convention; 2559 locations->SetInAt(0, Location::FpuRegisterPairLocation( 2560 calling_convention.GetFpuRegisterAt(0), 2561 calling_convention.GetFpuRegisterAt(1))); 2562 locations->SetOut(Location::RegisterPairLocation(R0, R1)); 2563 break; 2564 } 2565 2566 default: 2567 LOG(FATAL) << "Unexpected type conversion from " << input_type 2568 << " to " << result_type; 2569 } 2570 break; 2571 2572 case Primitive::kPrimChar: 2573 switch (input_type) { 2574 case Primitive::kPrimLong: 2575 // Type conversion from long to char is a result of code transformations. 2576 case Primitive::kPrimBoolean: 2577 // Boolean input is a result of code transformations. 2578 case Primitive::kPrimByte: 2579 case Primitive::kPrimShort: 2580 case Primitive::kPrimInt: 2581 // Processing a Dex `int-to-char' instruction. 2582 locations->SetInAt(0, Location::RequiresRegister()); 2583 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 2584 break; 2585 2586 default: 2587 LOG(FATAL) << "Unexpected type conversion from " << input_type 2588 << " to " << result_type; 2589 } 2590 break; 2591 2592 case Primitive::kPrimFloat: 2593 switch (input_type) { 2594 case Primitive::kPrimBoolean: 2595 // Boolean input is a result of code transformations. 2596 case Primitive::kPrimByte: 2597 case Primitive::kPrimShort: 2598 case Primitive::kPrimInt: 2599 case Primitive::kPrimChar: 2600 // Processing a Dex `int-to-float' instruction. 2601 locations->SetInAt(0, Location::RequiresRegister()); 2602 locations->SetOut(Location::RequiresFpuRegister()); 2603 break; 2604 2605 case Primitive::kPrimLong: { 2606 // Processing a Dex `long-to-float' instruction. 2607 InvokeRuntimeCallingConvention calling_convention; 2608 locations->SetInAt(0, Location::RegisterPairLocation( 2609 calling_convention.GetRegisterAt(0), calling_convention.GetRegisterAt(1))); 2610 locations->SetOut(Location::FpuRegisterLocation(calling_convention.GetFpuRegisterAt(0))); 2611 break; 2612 } 2613 2614 case Primitive::kPrimDouble: 2615 // Processing a Dex `double-to-float' instruction. 2616 locations->SetInAt(0, Location::RequiresFpuRegister()); 2617 locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap); 2618 break; 2619 2620 default: 2621 LOG(FATAL) << "Unexpected type conversion from " << input_type 2622 << " to " << result_type; 2623 }; 2624 break; 2625 2626 case Primitive::kPrimDouble: 2627 switch (input_type) { 2628 case Primitive::kPrimBoolean: 2629 // Boolean input is a result of code transformations. 2630 case Primitive::kPrimByte: 2631 case Primitive::kPrimShort: 2632 case Primitive::kPrimInt: 2633 case Primitive::kPrimChar: 2634 // Processing a Dex `int-to-double' instruction. 2635 locations->SetInAt(0, Location::RequiresRegister()); 2636 locations->SetOut(Location::RequiresFpuRegister()); 2637 break; 2638 2639 case Primitive::kPrimLong: 2640 // Processing a Dex `long-to-double' instruction. 2641 locations->SetInAt(0, Location::RequiresRegister()); 2642 locations->SetOut(Location::RequiresFpuRegister()); 2643 locations->AddTemp(Location::RequiresFpuRegister()); 2644 locations->AddTemp(Location::RequiresFpuRegister()); 2645 break; 2646 2647 case Primitive::kPrimFloat: 2648 // Processing a Dex `float-to-double' instruction. 2649 locations->SetInAt(0, Location::RequiresFpuRegister()); 2650 locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap); 2651 break; 2652 2653 default: 2654 LOG(FATAL) << "Unexpected type conversion from " << input_type 2655 << " to " << result_type; 2656 }; 2657 break; 2658 2659 default: 2660 LOG(FATAL) << "Unexpected type conversion from " << input_type 2661 << " to " << result_type; 2662 } 2663} 2664 2665void InstructionCodeGeneratorARM::VisitTypeConversion(HTypeConversion* conversion) { 2666 LocationSummary* locations = conversion->GetLocations(); 2667 Location out = locations->Out(); 2668 Location in = locations->InAt(0); 2669 Primitive::Type result_type = conversion->GetResultType(); 2670 Primitive::Type input_type = conversion->GetInputType(); 2671 DCHECK_NE(result_type, input_type); 2672 switch (result_type) { 2673 case Primitive::kPrimByte: 2674 switch (input_type) { 2675 case Primitive::kPrimLong: 2676 // Type conversion from long to byte is a result of code transformations. 2677 __ sbfx(out.AsRegister<Register>(), in.AsRegisterPairLow<Register>(), 0, 8); 2678 break; 2679 case Primitive::kPrimBoolean: 2680 // Boolean input is a result of code transformations. 2681 case Primitive::kPrimShort: 2682 case Primitive::kPrimInt: 2683 case Primitive::kPrimChar: 2684 // Processing a Dex `int-to-byte' instruction. 2685 __ sbfx(out.AsRegister<Register>(), in.AsRegister<Register>(), 0, 8); 2686 break; 2687 2688 default: 2689 LOG(FATAL) << "Unexpected type conversion from " << input_type 2690 << " to " << result_type; 2691 } 2692 break; 2693 2694 case Primitive::kPrimShort: 2695 switch (input_type) { 2696 case Primitive::kPrimLong: 2697 // Type conversion from long to short is a result of code transformations. 2698 __ sbfx(out.AsRegister<Register>(), in.AsRegisterPairLow<Register>(), 0, 16); 2699 break; 2700 case Primitive::kPrimBoolean: 2701 // Boolean input is a result of code transformations. 2702 case Primitive::kPrimByte: 2703 case Primitive::kPrimInt: 2704 case Primitive::kPrimChar: 2705 // Processing a Dex `int-to-short' instruction. 2706 __ sbfx(out.AsRegister<Register>(), in.AsRegister<Register>(), 0, 16); 2707 break; 2708 2709 default: 2710 LOG(FATAL) << "Unexpected type conversion from " << input_type 2711 << " to " << result_type; 2712 } 2713 break; 2714 2715 case Primitive::kPrimInt: 2716 switch (input_type) { 2717 case Primitive::kPrimLong: 2718 // Processing a Dex `long-to-int' instruction. 2719 DCHECK(out.IsRegister()); 2720 if (in.IsRegisterPair()) { 2721 __ Mov(out.AsRegister<Register>(), in.AsRegisterPairLow<Register>()); 2722 } else if (in.IsDoubleStackSlot()) { 2723 __ LoadFromOffset(kLoadWord, out.AsRegister<Register>(), SP, in.GetStackIndex()); 2724 } else { 2725 DCHECK(in.IsConstant()); 2726 DCHECK(in.GetConstant()->IsLongConstant()); 2727 int64_t value = in.GetConstant()->AsLongConstant()->GetValue(); 2728 __ LoadImmediate(out.AsRegister<Register>(), static_cast<int32_t>(value)); 2729 } 2730 break; 2731 2732 case Primitive::kPrimFloat: { 2733 // Processing a Dex `float-to-int' instruction. 2734 SRegister temp = locations->GetTemp(0).AsFpuRegisterPairLow<SRegister>(); 2735 __ vcvtis(temp, in.AsFpuRegister<SRegister>()); 2736 __ vmovrs(out.AsRegister<Register>(), temp); 2737 break; 2738 } 2739 2740 case Primitive::kPrimDouble: { 2741 // Processing a Dex `double-to-int' instruction. 2742 SRegister temp_s = locations->GetTemp(0).AsFpuRegisterPairLow<SRegister>(); 2743 __ vcvtid(temp_s, FromLowSToD(in.AsFpuRegisterPairLow<SRegister>())); 2744 __ vmovrs(out.AsRegister<Register>(), temp_s); 2745 break; 2746 } 2747 2748 default: 2749 LOG(FATAL) << "Unexpected type conversion from " << input_type 2750 << " to " << result_type; 2751 } 2752 break; 2753 2754 case Primitive::kPrimLong: 2755 switch (input_type) { 2756 case Primitive::kPrimBoolean: 2757 // Boolean input is a result of code transformations. 2758 case Primitive::kPrimByte: 2759 case Primitive::kPrimShort: 2760 case Primitive::kPrimInt: 2761 case Primitive::kPrimChar: 2762 // Processing a Dex `int-to-long' instruction. 2763 DCHECK(out.IsRegisterPair()); 2764 DCHECK(in.IsRegister()); 2765 __ Mov(out.AsRegisterPairLow<Register>(), in.AsRegister<Register>()); 2766 // Sign extension. 2767 __ Asr(out.AsRegisterPairHigh<Register>(), 2768 out.AsRegisterPairLow<Register>(), 2769 31); 2770 break; 2771 2772 case Primitive::kPrimFloat: 2773 // Processing a Dex `float-to-long' instruction. 2774 codegen_->InvokeRuntime(kQuickF2l, conversion, conversion->GetDexPc()); 2775 CheckEntrypointTypes<kQuickF2l, int64_t, float>(); 2776 break; 2777 2778 case Primitive::kPrimDouble: 2779 // Processing a Dex `double-to-long' instruction. 2780 codegen_->InvokeRuntime(kQuickD2l, conversion, conversion->GetDexPc()); 2781 CheckEntrypointTypes<kQuickD2l, int64_t, double>(); 2782 break; 2783 2784 default: 2785 LOG(FATAL) << "Unexpected type conversion from " << input_type 2786 << " to " << result_type; 2787 } 2788 break; 2789 2790 case Primitive::kPrimChar: 2791 switch (input_type) { 2792 case Primitive::kPrimLong: 2793 // Type conversion from long to char is a result of code transformations. 2794 __ ubfx(out.AsRegister<Register>(), in.AsRegisterPairLow<Register>(), 0, 16); 2795 break; 2796 case Primitive::kPrimBoolean: 2797 // Boolean input is a result of code transformations. 2798 case Primitive::kPrimByte: 2799 case Primitive::kPrimShort: 2800 case Primitive::kPrimInt: 2801 // Processing a Dex `int-to-char' instruction. 2802 __ ubfx(out.AsRegister<Register>(), in.AsRegister<Register>(), 0, 16); 2803 break; 2804 2805 default: 2806 LOG(FATAL) << "Unexpected type conversion from " << input_type 2807 << " to " << result_type; 2808 } 2809 break; 2810 2811 case Primitive::kPrimFloat: 2812 switch (input_type) { 2813 case Primitive::kPrimBoolean: 2814 // Boolean input is a result of code transformations. 2815 case Primitive::kPrimByte: 2816 case Primitive::kPrimShort: 2817 case Primitive::kPrimInt: 2818 case Primitive::kPrimChar: { 2819 // Processing a Dex `int-to-float' instruction. 2820 __ vmovsr(out.AsFpuRegister<SRegister>(), in.AsRegister<Register>()); 2821 __ vcvtsi(out.AsFpuRegister<SRegister>(), out.AsFpuRegister<SRegister>()); 2822 break; 2823 } 2824 2825 case Primitive::kPrimLong: 2826 // Processing a Dex `long-to-float' instruction. 2827 codegen_->InvokeRuntime(kQuickL2f, conversion, conversion->GetDexPc()); 2828 CheckEntrypointTypes<kQuickL2f, float, int64_t>(); 2829 break; 2830 2831 case Primitive::kPrimDouble: 2832 // Processing a Dex `double-to-float' instruction. 2833 __ vcvtsd(out.AsFpuRegister<SRegister>(), 2834 FromLowSToD(in.AsFpuRegisterPairLow<SRegister>())); 2835 break; 2836 2837 default: 2838 LOG(FATAL) << "Unexpected type conversion from " << input_type 2839 << " to " << result_type; 2840 }; 2841 break; 2842 2843 case Primitive::kPrimDouble: 2844 switch (input_type) { 2845 case Primitive::kPrimBoolean: 2846 // Boolean input is a result of code transformations. 2847 case Primitive::kPrimByte: 2848 case Primitive::kPrimShort: 2849 case Primitive::kPrimInt: 2850 case Primitive::kPrimChar: { 2851 // Processing a Dex `int-to-double' instruction. 2852 __ vmovsr(out.AsFpuRegisterPairLow<SRegister>(), in.AsRegister<Register>()); 2853 __ vcvtdi(FromLowSToD(out.AsFpuRegisterPairLow<SRegister>()), 2854 out.AsFpuRegisterPairLow<SRegister>()); 2855 break; 2856 } 2857 2858 case Primitive::kPrimLong: { 2859 // Processing a Dex `long-to-double' instruction. 2860 Register low = in.AsRegisterPairLow<Register>(); 2861 Register high = in.AsRegisterPairHigh<Register>(); 2862 SRegister out_s = out.AsFpuRegisterPairLow<SRegister>(); 2863 DRegister out_d = FromLowSToD(out_s); 2864 SRegister temp_s = locations->GetTemp(0).AsFpuRegisterPairLow<SRegister>(); 2865 DRegister temp_d = FromLowSToD(temp_s); 2866 SRegister constant_s = locations->GetTemp(1).AsFpuRegisterPairLow<SRegister>(); 2867 DRegister constant_d = FromLowSToD(constant_s); 2868 2869 // temp_d = int-to-double(high) 2870 __ vmovsr(temp_s, high); 2871 __ vcvtdi(temp_d, temp_s); 2872 // constant_d = k2Pow32EncodingForDouble 2873 __ LoadDImmediate(constant_d, bit_cast<double, int64_t>(k2Pow32EncodingForDouble)); 2874 // out_d = unsigned-to-double(low) 2875 __ vmovsr(out_s, low); 2876 __ vcvtdu(out_d, out_s); 2877 // out_d += temp_d * constant_d 2878 __ vmlad(out_d, temp_d, constant_d); 2879 break; 2880 } 2881 2882 case Primitive::kPrimFloat: 2883 // Processing a Dex `float-to-double' instruction. 2884 __ vcvtds(FromLowSToD(out.AsFpuRegisterPairLow<SRegister>()), 2885 in.AsFpuRegister<SRegister>()); 2886 break; 2887 2888 default: 2889 LOG(FATAL) << "Unexpected type conversion from " << input_type 2890 << " to " << result_type; 2891 }; 2892 break; 2893 2894 default: 2895 LOG(FATAL) << "Unexpected type conversion from " << input_type 2896 << " to " << result_type; 2897 } 2898} 2899 2900void LocationsBuilderARM::VisitAdd(HAdd* add) { 2901 LocationSummary* locations = 2902 new (GetGraph()->GetArena()) LocationSummary(add, LocationSummary::kNoCall); 2903 switch (add->GetResultType()) { 2904 case Primitive::kPrimInt: { 2905 locations->SetInAt(0, Location::RequiresRegister()); 2906 locations->SetInAt(1, Location::RegisterOrConstant(add->InputAt(1))); 2907 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 2908 break; 2909 } 2910 2911 case Primitive::kPrimLong: { 2912 locations->SetInAt(0, Location::RequiresRegister()); 2913 locations->SetInAt(1, ArmEncodableConstantOrRegister(add->InputAt(1), ADD)); 2914 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 2915 break; 2916 } 2917 2918 case Primitive::kPrimFloat: 2919 case Primitive::kPrimDouble: { 2920 locations->SetInAt(0, Location::RequiresFpuRegister()); 2921 locations->SetInAt(1, Location::RequiresFpuRegister()); 2922 locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap); 2923 break; 2924 } 2925 2926 default: 2927 LOG(FATAL) << "Unexpected add type " << add->GetResultType(); 2928 } 2929} 2930 2931void InstructionCodeGeneratorARM::VisitAdd(HAdd* add) { 2932 LocationSummary* locations = add->GetLocations(); 2933 Location out = locations->Out(); 2934 Location first = locations->InAt(0); 2935 Location second = locations->InAt(1); 2936 switch (add->GetResultType()) { 2937 case Primitive::kPrimInt: 2938 if (second.IsRegister()) { 2939 __ add(out.AsRegister<Register>(), 2940 first.AsRegister<Register>(), 2941 ShifterOperand(second.AsRegister<Register>())); 2942 } else { 2943 __ AddConstant(out.AsRegister<Register>(), 2944 first.AsRegister<Register>(), 2945 second.GetConstant()->AsIntConstant()->GetValue()); 2946 } 2947 break; 2948 2949 case Primitive::kPrimLong: { 2950 if (second.IsConstant()) { 2951 uint64_t value = static_cast<uint64_t>(Int64FromConstant(second.GetConstant())); 2952 GenerateAddLongConst(out, first, value); 2953 } else { 2954 DCHECK(second.IsRegisterPair()); 2955 __ adds(out.AsRegisterPairLow<Register>(), 2956 first.AsRegisterPairLow<Register>(), 2957 ShifterOperand(second.AsRegisterPairLow<Register>())); 2958 __ adc(out.AsRegisterPairHigh<Register>(), 2959 first.AsRegisterPairHigh<Register>(), 2960 ShifterOperand(second.AsRegisterPairHigh<Register>())); 2961 } 2962 break; 2963 } 2964 2965 case Primitive::kPrimFloat: 2966 __ vadds(out.AsFpuRegister<SRegister>(), 2967 first.AsFpuRegister<SRegister>(), 2968 second.AsFpuRegister<SRegister>()); 2969 break; 2970 2971 case Primitive::kPrimDouble: 2972 __ vaddd(FromLowSToD(out.AsFpuRegisterPairLow<SRegister>()), 2973 FromLowSToD(first.AsFpuRegisterPairLow<SRegister>()), 2974 FromLowSToD(second.AsFpuRegisterPairLow<SRegister>())); 2975 break; 2976 2977 default: 2978 LOG(FATAL) << "Unexpected add type " << add->GetResultType(); 2979 } 2980} 2981 2982void LocationsBuilderARM::VisitSub(HSub* sub) { 2983 LocationSummary* locations = 2984 new (GetGraph()->GetArena()) LocationSummary(sub, LocationSummary::kNoCall); 2985 switch (sub->GetResultType()) { 2986 case Primitive::kPrimInt: { 2987 locations->SetInAt(0, Location::RequiresRegister()); 2988 locations->SetInAt(1, Location::RegisterOrConstant(sub->InputAt(1))); 2989 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 2990 break; 2991 } 2992 2993 case Primitive::kPrimLong: { 2994 locations->SetInAt(0, Location::RequiresRegister()); 2995 locations->SetInAt(1, ArmEncodableConstantOrRegister(sub->InputAt(1), SUB)); 2996 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 2997 break; 2998 } 2999 case Primitive::kPrimFloat: 3000 case Primitive::kPrimDouble: { 3001 locations->SetInAt(0, Location::RequiresFpuRegister()); 3002 locations->SetInAt(1, Location::RequiresFpuRegister()); 3003 locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap); 3004 break; 3005 } 3006 default: 3007 LOG(FATAL) << "Unexpected sub type " << sub->GetResultType(); 3008 } 3009} 3010 3011void InstructionCodeGeneratorARM::VisitSub(HSub* sub) { 3012 LocationSummary* locations = sub->GetLocations(); 3013 Location out = locations->Out(); 3014 Location first = locations->InAt(0); 3015 Location second = locations->InAt(1); 3016 switch (sub->GetResultType()) { 3017 case Primitive::kPrimInt: { 3018 if (second.IsRegister()) { 3019 __ sub(out.AsRegister<Register>(), 3020 first.AsRegister<Register>(), 3021 ShifterOperand(second.AsRegister<Register>())); 3022 } else { 3023 __ AddConstant(out.AsRegister<Register>(), 3024 first.AsRegister<Register>(), 3025 -second.GetConstant()->AsIntConstant()->GetValue()); 3026 } 3027 break; 3028 } 3029 3030 case Primitive::kPrimLong: { 3031 if (second.IsConstant()) { 3032 uint64_t value = static_cast<uint64_t>(Int64FromConstant(second.GetConstant())); 3033 GenerateAddLongConst(out, first, -value); 3034 } else { 3035 DCHECK(second.IsRegisterPair()); 3036 __ subs(out.AsRegisterPairLow<Register>(), 3037 first.AsRegisterPairLow<Register>(), 3038 ShifterOperand(second.AsRegisterPairLow<Register>())); 3039 __ sbc(out.AsRegisterPairHigh<Register>(), 3040 first.AsRegisterPairHigh<Register>(), 3041 ShifterOperand(second.AsRegisterPairHigh<Register>())); 3042 } 3043 break; 3044 } 3045 3046 case Primitive::kPrimFloat: { 3047 __ vsubs(out.AsFpuRegister<SRegister>(), 3048 first.AsFpuRegister<SRegister>(), 3049 second.AsFpuRegister<SRegister>()); 3050 break; 3051 } 3052 3053 case Primitive::kPrimDouble: { 3054 __ vsubd(FromLowSToD(out.AsFpuRegisterPairLow<SRegister>()), 3055 FromLowSToD(first.AsFpuRegisterPairLow<SRegister>()), 3056 FromLowSToD(second.AsFpuRegisterPairLow<SRegister>())); 3057 break; 3058 } 3059 3060 3061 default: 3062 LOG(FATAL) << "Unexpected sub type " << sub->GetResultType(); 3063 } 3064} 3065 3066void LocationsBuilderARM::VisitMul(HMul* mul) { 3067 LocationSummary* locations = 3068 new (GetGraph()->GetArena()) LocationSummary(mul, LocationSummary::kNoCall); 3069 switch (mul->GetResultType()) { 3070 case Primitive::kPrimInt: 3071 case Primitive::kPrimLong: { 3072 locations->SetInAt(0, Location::RequiresRegister()); 3073 locations->SetInAt(1, Location::RequiresRegister()); 3074 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 3075 break; 3076 } 3077 3078 case Primitive::kPrimFloat: 3079 case Primitive::kPrimDouble: { 3080 locations->SetInAt(0, Location::RequiresFpuRegister()); 3081 locations->SetInAt(1, Location::RequiresFpuRegister()); 3082 locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap); 3083 break; 3084 } 3085 3086 default: 3087 LOG(FATAL) << "Unexpected mul type " << mul->GetResultType(); 3088 } 3089} 3090 3091void InstructionCodeGeneratorARM::VisitMul(HMul* mul) { 3092 LocationSummary* locations = mul->GetLocations(); 3093 Location out = locations->Out(); 3094 Location first = locations->InAt(0); 3095 Location second = locations->InAt(1); 3096 switch (mul->GetResultType()) { 3097 case Primitive::kPrimInt: { 3098 __ mul(out.AsRegister<Register>(), 3099 first.AsRegister<Register>(), 3100 second.AsRegister<Register>()); 3101 break; 3102 } 3103 case Primitive::kPrimLong: { 3104 Register out_hi = out.AsRegisterPairHigh<Register>(); 3105 Register out_lo = out.AsRegisterPairLow<Register>(); 3106 Register in1_hi = first.AsRegisterPairHigh<Register>(); 3107 Register in1_lo = first.AsRegisterPairLow<Register>(); 3108 Register in2_hi = second.AsRegisterPairHigh<Register>(); 3109 Register in2_lo = second.AsRegisterPairLow<Register>(); 3110 3111 // Extra checks to protect caused by the existence of R1_R2. 3112 // The algorithm is wrong if out.hi is either in1.lo or in2.lo: 3113 // (e.g. in1=r0_r1, in2=r2_r3 and out=r1_r2); 3114 DCHECK_NE(out_hi, in1_lo); 3115 DCHECK_NE(out_hi, in2_lo); 3116 3117 // input: in1 - 64 bits, in2 - 64 bits 3118 // output: out 3119 // formula: out.hi : out.lo = (in1.lo * in2.hi + in1.hi * in2.lo)* 2^32 + in1.lo * in2.lo 3120 // parts: out.hi = in1.lo * in2.hi + in1.hi * in2.lo + (in1.lo * in2.lo)[63:32] 3121 // parts: out.lo = (in1.lo * in2.lo)[31:0] 3122 3123 // IP <- in1.lo * in2.hi 3124 __ mul(IP, in1_lo, in2_hi); 3125 // out.hi <- in1.lo * in2.hi + in1.hi * in2.lo 3126 __ mla(out_hi, in1_hi, in2_lo, IP); 3127 // out.lo <- (in1.lo * in2.lo)[31:0]; 3128 __ umull(out_lo, IP, in1_lo, in2_lo); 3129 // out.hi <- in2.hi * in1.lo + in2.lo * in1.hi + (in1.lo * in2.lo)[63:32] 3130 __ add(out_hi, out_hi, ShifterOperand(IP)); 3131 break; 3132 } 3133 3134 case Primitive::kPrimFloat: { 3135 __ vmuls(out.AsFpuRegister<SRegister>(), 3136 first.AsFpuRegister<SRegister>(), 3137 second.AsFpuRegister<SRegister>()); 3138 break; 3139 } 3140 3141 case Primitive::kPrimDouble: { 3142 __ vmuld(FromLowSToD(out.AsFpuRegisterPairLow<SRegister>()), 3143 FromLowSToD(first.AsFpuRegisterPairLow<SRegister>()), 3144 FromLowSToD(second.AsFpuRegisterPairLow<SRegister>())); 3145 break; 3146 } 3147 3148 default: 3149 LOG(FATAL) << "Unexpected mul type " << mul->GetResultType(); 3150 } 3151} 3152 3153void InstructionCodeGeneratorARM::DivRemOneOrMinusOne(HBinaryOperation* instruction) { 3154 DCHECK(instruction->IsDiv() || instruction->IsRem()); 3155 DCHECK(instruction->GetResultType() == Primitive::kPrimInt); 3156 3157 LocationSummary* locations = instruction->GetLocations(); 3158 Location second = locations->InAt(1); 3159 DCHECK(second.IsConstant()); 3160 3161 Register out = locations->Out().AsRegister<Register>(); 3162 Register dividend = locations->InAt(0).AsRegister<Register>(); 3163 int32_t imm = second.GetConstant()->AsIntConstant()->GetValue(); 3164 DCHECK(imm == 1 || imm == -1); 3165 3166 if (instruction->IsRem()) { 3167 __ LoadImmediate(out, 0); 3168 } else { 3169 if (imm == 1) { 3170 __ Mov(out, dividend); 3171 } else { 3172 __ rsb(out, dividend, ShifterOperand(0)); 3173 } 3174 } 3175} 3176 3177void InstructionCodeGeneratorARM::DivRemByPowerOfTwo(HBinaryOperation* instruction) { 3178 DCHECK(instruction->IsDiv() || instruction->IsRem()); 3179 DCHECK(instruction->GetResultType() == Primitive::kPrimInt); 3180 3181 LocationSummary* locations = instruction->GetLocations(); 3182 Location second = locations->InAt(1); 3183 DCHECK(second.IsConstant()); 3184 3185 Register out = locations->Out().AsRegister<Register>(); 3186 Register dividend = locations->InAt(0).AsRegister<Register>(); 3187 Register temp = locations->GetTemp(0).AsRegister<Register>(); 3188 int32_t imm = second.GetConstant()->AsIntConstant()->GetValue(); 3189 uint32_t abs_imm = static_cast<uint32_t>(AbsOrMin(imm)); 3190 int ctz_imm = CTZ(abs_imm); 3191 3192 if (ctz_imm == 1) { 3193 __ Lsr(temp, dividend, 32 - ctz_imm); 3194 } else { 3195 __ Asr(temp, dividend, 31); 3196 __ Lsr(temp, temp, 32 - ctz_imm); 3197 } 3198 __ add(out, temp, ShifterOperand(dividend)); 3199 3200 if (instruction->IsDiv()) { 3201 __ Asr(out, out, ctz_imm); 3202 if (imm < 0) { 3203 __ rsb(out, out, ShifterOperand(0)); 3204 } 3205 } else { 3206 __ ubfx(out, out, 0, ctz_imm); 3207 __ sub(out, out, ShifterOperand(temp)); 3208 } 3209} 3210 3211void InstructionCodeGeneratorARM::GenerateDivRemWithAnyConstant(HBinaryOperation* instruction) { 3212 DCHECK(instruction->IsDiv() || instruction->IsRem()); 3213 DCHECK(instruction->GetResultType() == Primitive::kPrimInt); 3214 3215 LocationSummary* locations = instruction->GetLocations(); 3216 Location second = locations->InAt(1); 3217 DCHECK(second.IsConstant()); 3218 3219 Register out = locations->Out().AsRegister<Register>(); 3220 Register dividend = locations->InAt(0).AsRegister<Register>(); 3221 Register temp1 = locations->GetTemp(0).AsRegister<Register>(); 3222 Register temp2 = locations->GetTemp(1).AsRegister<Register>(); 3223 int64_t imm = second.GetConstant()->AsIntConstant()->GetValue(); 3224 3225 int64_t magic; 3226 int shift; 3227 CalculateMagicAndShiftForDivRem(imm, false /* is_long */, &magic, &shift); 3228 3229 __ LoadImmediate(temp1, magic); 3230 __ smull(temp2, temp1, dividend, temp1); 3231 3232 if (imm > 0 && magic < 0) { 3233 __ add(temp1, temp1, ShifterOperand(dividend)); 3234 } else if (imm < 0 && magic > 0) { 3235 __ sub(temp1, temp1, ShifterOperand(dividend)); 3236 } 3237 3238 if (shift != 0) { 3239 __ Asr(temp1, temp1, shift); 3240 } 3241 3242 if (instruction->IsDiv()) { 3243 __ sub(out, temp1, ShifterOperand(temp1, ASR, 31)); 3244 } else { 3245 __ sub(temp1, temp1, ShifterOperand(temp1, ASR, 31)); 3246 // TODO: Strength reduction for mls. 3247 __ LoadImmediate(temp2, imm); 3248 __ mls(out, temp1, temp2, dividend); 3249 } 3250} 3251 3252void InstructionCodeGeneratorARM::GenerateDivRemConstantIntegral(HBinaryOperation* instruction) { 3253 DCHECK(instruction->IsDiv() || instruction->IsRem()); 3254 DCHECK(instruction->GetResultType() == Primitive::kPrimInt); 3255 3256 LocationSummary* locations = instruction->GetLocations(); 3257 Location second = locations->InAt(1); 3258 DCHECK(second.IsConstant()); 3259 3260 int32_t imm = second.GetConstant()->AsIntConstant()->GetValue(); 3261 if (imm == 0) { 3262 // Do not generate anything. DivZeroCheck would prevent any code to be executed. 3263 } else if (imm == 1 || imm == -1) { 3264 DivRemOneOrMinusOne(instruction); 3265 } else if (IsPowerOfTwo(AbsOrMin(imm))) { 3266 DivRemByPowerOfTwo(instruction); 3267 } else { 3268 DCHECK(imm <= -2 || imm >= 2); 3269 GenerateDivRemWithAnyConstant(instruction); 3270 } 3271} 3272 3273void LocationsBuilderARM::VisitDiv(HDiv* div) { 3274 LocationSummary::CallKind call_kind = LocationSummary::kNoCall; 3275 if (div->GetResultType() == Primitive::kPrimLong) { 3276 // pLdiv runtime call. 3277 call_kind = LocationSummary::kCallOnMainOnly; 3278 } else if (div->GetResultType() == Primitive::kPrimInt && div->InputAt(1)->IsConstant()) { 3279 // sdiv will be replaced by other instruction sequence. 3280 } else if (div->GetResultType() == Primitive::kPrimInt && 3281 !codegen_->GetInstructionSetFeatures().HasDivideInstruction()) { 3282 // pIdivmod runtime call. 3283 call_kind = LocationSummary::kCallOnMainOnly; 3284 } 3285 3286 LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(div, call_kind); 3287 3288 switch (div->GetResultType()) { 3289 case Primitive::kPrimInt: { 3290 if (div->InputAt(1)->IsConstant()) { 3291 locations->SetInAt(0, Location::RequiresRegister()); 3292 locations->SetInAt(1, Location::ConstantLocation(div->InputAt(1)->AsConstant())); 3293 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 3294 int32_t value = div->InputAt(1)->AsIntConstant()->GetValue(); 3295 if (value == 1 || value == 0 || value == -1) { 3296 // No temp register required. 3297 } else { 3298 locations->AddTemp(Location::RequiresRegister()); 3299 if (!IsPowerOfTwo(AbsOrMin(value))) { 3300 locations->AddTemp(Location::RequiresRegister()); 3301 } 3302 } 3303 } else if (codegen_->GetInstructionSetFeatures().HasDivideInstruction()) { 3304 locations->SetInAt(0, Location::RequiresRegister()); 3305 locations->SetInAt(1, Location::RequiresRegister()); 3306 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 3307 } else { 3308 InvokeRuntimeCallingConvention calling_convention; 3309 locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(0))); 3310 locations->SetInAt(1, Location::RegisterLocation(calling_convention.GetRegisterAt(1))); 3311 // Note: divrem will compute both the quotient and the remainder as the pair R0 and R1, but 3312 // we only need the former. 3313 locations->SetOut(Location::RegisterLocation(R0)); 3314 } 3315 break; 3316 } 3317 case Primitive::kPrimLong: { 3318 InvokeRuntimeCallingConvention calling_convention; 3319 locations->SetInAt(0, Location::RegisterPairLocation( 3320 calling_convention.GetRegisterAt(0), calling_convention.GetRegisterAt(1))); 3321 locations->SetInAt(1, Location::RegisterPairLocation( 3322 calling_convention.GetRegisterAt(2), calling_convention.GetRegisterAt(3))); 3323 locations->SetOut(Location::RegisterPairLocation(R0, R1)); 3324 break; 3325 } 3326 case Primitive::kPrimFloat: 3327 case Primitive::kPrimDouble: { 3328 locations->SetInAt(0, Location::RequiresFpuRegister()); 3329 locations->SetInAt(1, Location::RequiresFpuRegister()); 3330 locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap); 3331 break; 3332 } 3333 3334 default: 3335 LOG(FATAL) << "Unexpected div type " << div->GetResultType(); 3336 } 3337} 3338 3339void InstructionCodeGeneratorARM::VisitDiv(HDiv* div) { 3340 LocationSummary* locations = div->GetLocations(); 3341 Location out = locations->Out(); 3342 Location first = locations->InAt(0); 3343 Location second = locations->InAt(1); 3344 3345 switch (div->GetResultType()) { 3346 case Primitive::kPrimInt: { 3347 if (second.IsConstant()) { 3348 GenerateDivRemConstantIntegral(div); 3349 } else if (codegen_->GetInstructionSetFeatures().HasDivideInstruction()) { 3350 __ sdiv(out.AsRegister<Register>(), 3351 first.AsRegister<Register>(), 3352 second.AsRegister<Register>()); 3353 } else { 3354 InvokeRuntimeCallingConvention calling_convention; 3355 DCHECK_EQ(calling_convention.GetRegisterAt(0), first.AsRegister<Register>()); 3356 DCHECK_EQ(calling_convention.GetRegisterAt(1), second.AsRegister<Register>()); 3357 DCHECK_EQ(R0, out.AsRegister<Register>()); 3358 3359 codegen_->InvokeRuntime(kQuickIdivmod, div, div->GetDexPc()); 3360 CheckEntrypointTypes<kQuickIdivmod, int32_t, int32_t, int32_t>(); 3361 } 3362 break; 3363 } 3364 3365 case Primitive::kPrimLong: { 3366 InvokeRuntimeCallingConvention calling_convention; 3367 DCHECK_EQ(calling_convention.GetRegisterAt(0), first.AsRegisterPairLow<Register>()); 3368 DCHECK_EQ(calling_convention.GetRegisterAt(1), first.AsRegisterPairHigh<Register>()); 3369 DCHECK_EQ(calling_convention.GetRegisterAt(2), second.AsRegisterPairLow<Register>()); 3370 DCHECK_EQ(calling_convention.GetRegisterAt(3), second.AsRegisterPairHigh<Register>()); 3371 DCHECK_EQ(R0, out.AsRegisterPairLow<Register>()); 3372 DCHECK_EQ(R1, out.AsRegisterPairHigh<Register>()); 3373 3374 codegen_->InvokeRuntime(kQuickLdiv, div, div->GetDexPc()); 3375 CheckEntrypointTypes<kQuickLdiv, int64_t, int64_t, int64_t>(); 3376 break; 3377 } 3378 3379 case Primitive::kPrimFloat: { 3380 __ vdivs(out.AsFpuRegister<SRegister>(), 3381 first.AsFpuRegister<SRegister>(), 3382 second.AsFpuRegister<SRegister>()); 3383 break; 3384 } 3385 3386 case Primitive::kPrimDouble: { 3387 __ vdivd(FromLowSToD(out.AsFpuRegisterPairLow<SRegister>()), 3388 FromLowSToD(first.AsFpuRegisterPairLow<SRegister>()), 3389 FromLowSToD(second.AsFpuRegisterPairLow<SRegister>())); 3390 break; 3391 } 3392 3393 default: 3394 LOG(FATAL) << "Unexpected div type " << div->GetResultType(); 3395 } 3396} 3397 3398void LocationsBuilderARM::VisitRem(HRem* rem) { 3399 Primitive::Type type = rem->GetResultType(); 3400 3401 // Most remainders are implemented in the runtime. 3402 LocationSummary::CallKind call_kind = LocationSummary::kCallOnMainOnly; 3403 if (rem->GetResultType() == Primitive::kPrimInt && rem->InputAt(1)->IsConstant()) { 3404 // sdiv will be replaced by other instruction sequence. 3405 call_kind = LocationSummary::kNoCall; 3406 } else if ((rem->GetResultType() == Primitive::kPrimInt) 3407 && codegen_->GetInstructionSetFeatures().HasDivideInstruction()) { 3408 // Have hardware divide instruction for int, do it with three instructions. 3409 call_kind = LocationSummary::kNoCall; 3410 } 3411 3412 LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(rem, call_kind); 3413 3414 switch (type) { 3415 case Primitive::kPrimInt: { 3416 if (rem->InputAt(1)->IsConstant()) { 3417 locations->SetInAt(0, Location::RequiresRegister()); 3418 locations->SetInAt(1, Location::ConstantLocation(rem->InputAt(1)->AsConstant())); 3419 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 3420 int32_t value = rem->InputAt(1)->AsIntConstant()->GetValue(); 3421 if (value == 1 || value == 0 || value == -1) { 3422 // No temp register required. 3423 } else { 3424 locations->AddTemp(Location::RequiresRegister()); 3425 if (!IsPowerOfTwo(AbsOrMin(value))) { 3426 locations->AddTemp(Location::RequiresRegister()); 3427 } 3428 } 3429 } else if (codegen_->GetInstructionSetFeatures().HasDivideInstruction()) { 3430 locations->SetInAt(0, Location::RequiresRegister()); 3431 locations->SetInAt(1, Location::RequiresRegister()); 3432 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 3433 locations->AddTemp(Location::RequiresRegister()); 3434 } else { 3435 InvokeRuntimeCallingConvention calling_convention; 3436 locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(0))); 3437 locations->SetInAt(1, Location::RegisterLocation(calling_convention.GetRegisterAt(1))); 3438 // Note: divrem will compute both the quotient and the remainder as the pair R0 and R1, but 3439 // we only need the latter. 3440 locations->SetOut(Location::RegisterLocation(R1)); 3441 } 3442 break; 3443 } 3444 case Primitive::kPrimLong: { 3445 InvokeRuntimeCallingConvention calling_convention; 3446 locations->SetInAt(0, Location::RegisterPairLocation( 3447 calling_convention.GetRegisterAt(0), calling_convention.GetRegisterAt(1))); 3448 locations->SetInAt(1, Location::RegisterPairLocation( 3449 calling_convention.GetRegisterAt(2), calling_convention.GetRegisterAt(3))); 3450 // The runtime helper puts the output in R2,R3. 3451 locations->SetOut(Location::RegisterPairLocation(R2, R3)); 3452 break; 3453 } 3454 case Primitive::kPrimFloat: { 3455 InvokeRuntimeCallingConvention calling_convention; 3456 locations->SetInAt(0, Location::FpuRegisterLocation(calling_convention.GetFpuRegisterAt(0))); 3457 locations->SetInAt(1, Location::FpuRegisterLocation(calling_convention.GetFpuRegisterAt(1))); 3458 locations->SetOut(Location::FpuRegisterLocation(S0)); 3459 break; 3460 } 3461 3462 case Primitive::kPrimDouble: { 3463 InvokeRuntimeCallingConvention calling_convention; 3464 locations->SetInAt(0, Location::FpuRegisterPairLocation( 3465 calling_convention.GetFpuRegisterAt(0), calling_convention.GetFpuRegisterAt(1))); 3466 locations->SetInAt(1, Location::FpuRegisterPairLocation( 3467 calling_convention.GetFpuRegisterAt(2), calling_convention.GetFpuRegisterAt(3))); 3468 locations->SetOut(Location::Location::FpuRegisterPairLocation(S0, S1)); 3469 break; 3470 } 3471 3472 default: 3473 LOG(FATAL) << "Unexpected rem type " << type; 3474 } 3475} 3476 3477void InstructionCodeGeneratorARM::VisitRem(HRem* rem) { 3478 LocationSummary* locations = rem->GetLocations(); 3479 Location out = locations->Out(); 3480 Location first = locations->InAt(0); 3481 Location second = locations->InAt(1); 3482 3483 Primitive::Type type = rem->GetResultType(); 3484 switch (type) { 3485 case Primitive::kPrimInt: { 3486 if (second.IsConstant()) { 3487 GenerateDivRemConstantIntegral(rem); 3488 } else if (codegen_->GetInstructionSetFeatures().HasDivideInstruction()) { 3489 Register reg1 = first.AsRegister<Register>(); 3490 Register reg2 = second.AsRegister<Register>(); 3491 Register temp = locations->GetTemp(0).AsRegister<Register>(); 3492 3493 // temp = reg1 / reg2 (integer division) 3494 // dest = reg1 - temp * reg2 3495 __ sdiv(temp, reg1, reg2); 3496 __ mls(out.AsRegister<Register>(), temp, reg2, reg1); 3497 } else { 3498 InvokeRuntimeCallingConvention calling_convention; 3499 DCHECK_EQ(calling_convention.GetRegisterAt(0), first.AsRegister<Register>()); 3500 DCHECK_EQ(calling_convention.GetRegisterAt(1), second.AsRegister<Register>()); 3501 DCHECK_EQ(R1, out.AsRegister<Register>()); 3502 3503 codegen_->InvokeRuntime(kQuickIdivmod, rem, rem->GetDexPc()); 3504 CheckEntrypointTypes<kQuickIdivmod, int32_t, int32_t, int32_t>(); 3505 } 3506 break; 3507 } 3508 3509 case Primitive::kPrimLong: { 3510 codegen_->InvokeRuntime(kQuickLmod, rem, rem->GetDexPc()); 3511 CheckEntrypointTypes<kQuickLmod, int64_t, int64_t, int64_t>(); 3512 break; 3513 } 3514 3515 case Primitive::kPrimFloat: { 3516 codegen_->InvokeRuntime(kQuickFmodf, rem, rem->GetDexPc()); 3517 CheckEntrypointTypes<kQuickFmodf, float, float, float>(); 3518 break; 3519 } 3520 3521 case Primitive::kPrimDouble: { 3522 codegen_->InvokeRuntime(kQuickFmod, rem, rem->GetDexPc()); 3523 CheckEntrypointTypes<kQuickFmod, double, double, double>(); 3524 break; 3525 } 3526 3527 default: 3528 LOG(FATAL) << "Unexpected rem type " << type; 3529 } 3530} 3531 3532void LocationsBuilderARM::VisitDivZeroCheck(HDivZeroCheck* instruction) { 3533 LocationSummary* locations = codegen_->CreateThrowingSlowPathLocations(instruction); 3534 locations->SetInAt(0, Location::RegisterOrConstant(instruction->InputAt(0))); 3535} 3536 3537void InstructionCodeGeneratorARM::VisitDivZeroCheck(HDivZeroCheck* instruction) { 3538 SlowPathCodeARM* slow_path = new (GetGraph()->GetArena()) DivZeroCheckSlowPathARM(instruction); 3539 codegen_->AddSlowPath(slow_path); 3540 3541 LocationSummary* locations = instruction->GetLocations(); 3542 Location value = locations->InAt(0); 3543 3544 switch (instruction->GetType()) { 3545 case Primitive::kPrimBoolean: 3546 case Primitive::kPrimByte: 3547 case Primitive::kPrimChar: 3548 case Primitive::kPrimShort: 3549 case Primitive::kPrimInt: { 3550 if (value.IsRegister()) { 3551 __ CompareAndBranchIfZero(value.AsRegister<Register>(), slow_path->GetEntryLabel()); 3552 } else { 3553 DCHECK(value.IsConstant()) << value; 3554 if (value.GetConstant()->AsIntConstant()->GetValue() == 0) { 3555 __ b(slow_path->GetEntryLabel()); 3556 } 3557 } 3558 break; 3559 } 3560 case Primitive::kPrimLong: { 3561 if (value.IsRegisterPair()) { 3562 __ orrs(IP, 3563 value.AsRegisterPairLow<Register>(), 3564 ShifterOperand(value.AsRegisterPairHigh<Register>())); 3565 __ b(slow_path->GetEntryLabel(), EQ); 3566 } else { 3567 DCHECK(value.IsConstant()) << value; 3568 if (value.GetConstant()->AsLongConstant()->GetValue() == 0) { 3569 __ b(slow_path->GetEntryLabel()); 3570 } 3571 } 3572 break; 3573 default: 3574 LOG(FATAL) << "Unexpected type for HDivZeroCheck " << instruction->GetType(); 3575 } 3576 } 3577} 3578 3579void InstructionCodeGeneratorARM::HandleIntegerRotate(LocationSummary* locations) { 3580 Register in = locations->InAt(0).AsRegister<Register>(); 3581 Location rhs = locations->InAt(1); 3582 Register out = locations->Out().AsRegister<Register>(); 3583 3584 if (rhs.IsConstant()) { 3585 // Arm32 and Thumb2 assemblers require a rotation on the interval [1,31], 3586 // so map all rotations to a +ve. equivalent in that range. 3587 // (e.g. left *or* right by -2 bits == 30 bits in the same direction.) 3588 uint32_t rot = CodeGenerator::GetInt32ValueOf(rhs.GetConstant()) & 0x1F; 3589 if (rot) { 3590 // Rotate, mapping left rotations to right equivalents if necessary. 3591 // (e.g. left by 2 bits == right by 30.) 3592 __ Ror(out, in, rot); 3593 } else if (out != in) { 3594 __ Mov(out, in); 3595 } 3596 } else { 3597 __ Ror(out, in, rhs.AsRegister<Register>()); 3598 } 3599} 3600 3601// Gain some speed by mapping all Long rotates onto equivalent pairs of Integer 3602// rotates by swapping input regs (effectively rotating by the first 32-bits of 3603// a larger rotation) or flipping direction (thus treating larger right/left 3604// rotations as sub-word sized rotations in the other direction) as appropriate. 3605void InstructionCodeGeneratorARM::HandleLongRotate(LocationSummary* locations) { 3606 Register in_reg_lo = locations->InAt(0).AsRegisterPairLow<Register>(); 3607 Register in_reg_hi = locations->InAt(0).AsRegisterPairHigh<Register>(); 3608 Location rhs = locations->InAt(1); 3609 Register out_reg_lo = locations->Out().AsRegisterPairLow<Register>(); 3610 Register out_reg_hi = locations->Out().AsRegisterPairHigh<Register>(); 3611 3612 if (rhs.IsConstant()) { 3613 uint64_t rot = CodeGenerator::GetInt64ValueOf(rhs.GetConstant()); 3614 // Map all rotations to +ve. equivalents on the interval [0,63]. 3615 rot &= kMaxLongShiftDistance; 3616 // For rotates over a word in size, 'pre-rotate' by 32-bits to keep rotate 3617 // logic below to a simple pair of binary orr. 3618 // (e.g. 34 bits == in_reg swap + 2 bits right.) 3619 if (rot >= kArmBitsPerWord) { 3620 rot -= kArmBitsPerWord; 3621 std::swap(in_reg_hi, in_reg_lo); 3622 } 3623 // Rotate, or mov to out for zero or word size rotations. 3624 if (rot != 0u) { 3625 __ Lsr(out_reg_hi, in_reg_hi, rot); 3626 __ orr(out_reg_hi, out_reg_hi, ShifterOperand(in_reg_lo, arm::LSL, kArmBitsPerWord - rot)); 3627 __ Lsr(out_reg_lo, in_reg_lo, rot); 3628 __ orr(out_reg_lo, out_reg_lo, ShifterOperand(in_reg_hi, arm::LSL, kArmBitsPerWord - rot)); 3629 } else { 3630 __ Mov(out_reg_lo, in_reg_lo); 3631 __ Mov(out_reg_hi, in_reg_hi); 3632 } 3633 } else { 3634 Register shift_right = locations->GetTemp(0).AsRegister<Register>(); 3635 Register shift_left = locations->GetTemp(1).AsRegister<Register>(); 3636 Label end; 3637 Label shift_by_32_plus_shift_right; 3638 3639 __ and_(shift_right, rhs.AsRegister<Register>(), ShifterOperand(0x1F)); 3640 __ Lsrs(shift_left, rhs.AsRegister<Register>(), 6); 3641 __ rsb(shift_left, shift_right, ShifterOperand(kArmBitsPerWord), AL, kCcKeep); 3642 __ b(&shift_by_32_plus_shift_right, CC); 3643 3644 // out_reg_hi = (reg_hi << shift_left) | (reg_lo >> shift_right). 3645 // out_reg_lo = (reg_lo << shift_left) | (reg_hi >> shift_right). 3646 __ Lsl(out_reg_hi, in_reg_hi, shift_left); 3647 __ Lsr(out_reg_lo, in_reg_lo, shift_right); 3648 __ add(out_reg_hi, out_reg_hi, ShifterOperand(out_reg_lo)); 3649 __ Lsl(out_reg_lo, in_reg_lo, shift_left); 3650 __ Lsr(shift_left, in_reg_hi, shift_right); 3651 __ add(out_reg_lo, out_reg_lo, ShifterOperand(shift_left)); 3652 __ b(&end); 3653 3654 __ Bind(&shift_by_32_plus_shift_right); // Shift by 32+shift_right. 3655 // out_reg_hi = (reg_hi >> shift_right) | (reg_lo << shift_left). 3656 // out_reg_lo = (reg_lo >> shift_right) | (reg_hi << shift_left). 3657 __ Lsr(out_reg_hi, in_reg_hi, shift_right); 3658 __ Lsl(out_reg_lo, in_reg_lo, shift_left); 3659 __ add(out_reg_hi, out_reg_hi, ShifterOperand(out_reg_lo)); 3660 __ Lsr(out_reg_lo, in_reg_lo, shift_right); 3661 __ Lsl(shift_right, in_reg_hi, shift_left); 3662 __ add(out_reg_lo, out_reg_lo, ShifterOperand(shift_right)); 3663 3664 __ Bind(&end); 3665 } 3666} 3667 3668void LocationsBuilderARM::VisitRor(HRor* ror) { 3669 LocationSummary* locations = 3670 new (GetGraph()->GetArena()) LocationSummary(ror, LocationSummary::kNoCall); 3671 switch (ror->GetResultType()) { 3672 case Primitive::kPrimInt: { 3673 locations->SetInAt(0, Location::RequiresRegister()); 3674 locations->SetInAt(1, Location::RegisterOrConstant(ror->InputAt(1))); 3675 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 3676 break; 3677 } 3678 case Primitive::kPrimLong: { 3679 locations->SetInAt(0, Location::RequiresRegister()); 3680 if (ror->InputAt(1)->IsConstant()) { 3681 locations->SetInAt(1, Location::ConstantLocation(ror->InputAt(1)->AsConstant())); 3682 } else { 3683 locations->SetInAt(1, Location::RequiresRegister()); 3684 locations->AddTemp(Location::RequiresRegister()); 3685 locations->AddTemp(Location::RequiresRegister()); 3686 } 3687 locations->SetOut(Location::RequiresRegister(), Location::kOutputOverlap); 3688 break; 3689 } 3690 default: 3691 LOG(FATAL) << "Unexpected operation type " << ror->GetResultType(); 3692 } 3693} 3694 3695void InstructionCodeGeneratorARM::VisitRor(HRor* ror) { 3696 LocationSummary* locations = ror->GetLocations(); 3697 Primitive::Type type = ror->GetResultType(); 3698 switch (type) { 3699 case Primitive::kPrimInt: { 3700 HandleIntegerRotate(locations); 3701 break; 3702 } 3703 case Primitive::kPrimLong: { 3704 HandleLongRotate(locations); 3705 break; 3706 } 3707 default: 3708 LOG(FATAL) << "Unexpected operation type " << type; 3709 UNREACHABLE(); 3710 } 3711} 3712 3713void LocationsBuilderARM::HandleShift(HBinaryOperation* op) { 3714 DCHECK(op->IsShl() || op->IsShr() || op->IsUShr()); 3715 3716 LocationSummary* locations = 3717 new (GetGraph()->GetArena()) LocationSummary(op, LocationSummary::kNoCall); 3718 3719 switch (op->GetResultType()) { 3720 case Primitive::kPrimInt: { 3721 locations->SetInAt(0, Location::RequiresRegister()); 3722 if (op->InputAt(1)->IsConstant()) { 3723 locations->SetInAt(1, Location::ConstantLocation(op->InputAt(1)->AsConstant())); 3724 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 3725 } else { 3726 locations->SetInAt(1, Location::RequiresRegister()); 3727 // Make the output overlap, as it will be used to hold the masked 3728 // second input. 3729 locations->SetOut(Location::RequiresRegister(), Location::kOutputOverlap); 3730 } 3731 break; 3732 } 3733 case Primitive::kPrimLong: { 3734 locations->SetInAt(0, Location::RequiresRegister()); 3735 if (op->InputAt(1)->IsConstant()) { 3736 locations->SetInAt(1, Location::ConstantLocation(op->InputAt(1)->AsConstant())); 3737 // For simplicity, use kOutputOverlap even though we only require that low registers 3738 // don't clash with high registers which the register allocator currently guarantees. 3739 locations->SetOut(Location::RequiresRegister(), Location::kOutputOverlap); 3740 } else { 3741 locations->SetInAt(1, Location::RequiresRegister()); 3742 locations->AddTemp(Location::RequiresRegister()); 3743 locations->SetOut(Location::RequiresRegister(), Location::kOutputOverlap); 3744 } 3745 break; 3746 } 3747 default: 3748 LOG(FATAL) << "Unexpected operation type " << op->GetResultType(); 3749 } 3750} 3751 3752void InstructionCodeGeneratorARM::HandleShift(HBinaryOperation* op) { 3753 DCHECK(op->IsShl() || op->IsShr() || op->IsUShr()); 3754 3755 LocationSummary* locations = op->GetLocations(); 3756 Location out = locations->Out(); 3757 Location first = locations->InAt(0); 3758 Location second = locations->InAt(1); 3759 3760 Primitive::Type type = op->GetResultType(); 3761 switch (type) { 3762 case Primitive::kPrimInt: { 3763 Register out_reg = out.AsRegister<Register>(); 3764 Register first_reg = first.AsRegister<Register>(); 3765 if (second.IsRegister()) { 3766 Register second_reg = second.AsRegister<Register>(); 3767 // ARM doesn't mask the shift count so we need to do it ourselves. 3768 __ and_(out_reg, second_reg, ShifterOperand(kMaxIntShiftDistance)); 3769 if (op->IsShl()) { 3770 __ Lsl(out_reg, first_reg, out_reg); 3771 } else if (op->IsShr()) { 3772 __ Asr(out_reg, first_reg, out_reg); 3773 } else { 3774 __ Lsr(out_reg, first_reg, out_reg); 3775 } 3776 } else { 3777 int32_t cst = second.GetConstant()->AsIntConstant()->GetValue(); 3778 uint32_t shift_value = cst & kMaxIntShiftDistance; 3779 if (shift_value == 0) { // ARM does not support shifting with 0 immediate. 3780 __ Mov(out_reg, first_reg); 3781 } else if (op->IsShl()) { 3782 __ Lsl(out_reg, first_reg, shift_value); 3783 } else if (op->IsShr()) { 3784 __ Asr(out_reg, first_reg, shift_value); 3785 } else { 3786 __ Lsr(out_reg, first_reg, shift_value); 3787 } 3788 } 3789 break; 3790 } 3791 case Primitive::kPrimLong: { 3792 Register o_h = out.AsRegisterPairHigh<Register>(); 3793 Register o_l = out.AsRegisterPairLow<Register>(); 3794 3795 Register high = first.AsRegisterPairHigh<Register>(); 3796 Register low = first.AsRegisterPairLow<Register>(); 3797 3798 if (second.IsRegister()) { 3799 Register temp = locations->GetTemp(0).AsRegister<Register>(); 3800 3801 Register second_reg = second.AsRegister<Register>(); 3802 3803 if (op->IsShl()) { 3804 __ and_(o_l, second_reg, ShifterOperand(kMaxLongShiftDistance)); 3805 // Shift the high part 3806 __ Lsl(o_h, high, o_l); 3807 // Shift the low part and `or` what overflew on the high part 3808 __ rsb(temp, o_l, ShifterOperand(kArmBitsPerWord)); 3809 __ Lsr(temp, low, temp); 3810 __ orr(o_h, o_h, ShifterOperand(temp)); 3811 // If the shift is > 32 bits, override the high part 3812 __ subs(temp, o_l, ShifterOperand(kArmBitsPerWord)); 3813 __ it(PL); 3814 __ Lsl(o_h, low, temp, PL); 3815 // Shift the low part 3816 __ Lsl(o_l, low, o_l); 3817 } else if (op->IsShr()) { 3818 __ and_(o_h, second_reg, ShifterOperand(kMaxLongShiftDistance)); 3819 // Shift the low part 3820 __ Lsr(o_l, low, o_h); 3821 // Shift the high part and `or` what underflew on the low part 3822 __ rsb(temp, o_h, ShifterOperand(kArmBitsPerWord)); 3823 __ Lsl(temp, high, temp); 3824 __ orr(o_l, o_l, ShifterOperand(temp)); 3825 // If the shift is > 32 bits, override the low part 3826 __ subs(temp, o_h, ShifterOperand(kArmBitsPerWord)); 3827 __ it(PL); 3828 __ Asr(o_l, high, temp, PL); 3829 // Shift the high part 3830 __ Asr(o_h, high, o_h); 3831 } else { 3832 __ and_(o_h, second_reg, ShifterOperand(kMaxLongShiftDistance)); 3833 // same as Shr except we use `Lsr`s and not `Asr`s 3834 __ Lsr(o_l, low, o_h); 3835 __ rsb(temp, o_h, ShifterOperand(kArmBitsPerWord)); 3836 __ Lsl(temp, high, temp); 3837 __ orr(o_l, o_l, ShifterOperand(temp)); 3838 __ subs(temp, o_h, ShifterOperand(kArmBitsPerWord)); 3839 __ it(PL); 3840 __ Lsr(o_l, high, temp, PL); 3841 __ Lsr(o_h, high, o_h); 3842 } 3843 } else { 3844 // Register allocator doesn't create partial overlap. 3845 DCHECK_NE(o_l, high); 3846 DCHECK_NE(o_h, low); 3847 int32_t cst = second.GetConstant()->AsIntConstant()->GetValue(); 3848 uint32_t shift_value = cst & kMaxLongShiftDistance; 3849 if (shift_value > 32) { 3850 if (op->IsShl()) { 3851 __ Lsl(o_h, low, shift_value - 32); 3852 __ LoadImmediate(o_l, 0); 3853 } else if (op->IsShr()) { 3854 __ Asr(o_l, high, shift_value - 32); 3855 __ Asr(o_h, high, 31); 3856 } else { 3857 __ Lsr(o_l, high, shift_value - 32); 3858 __ LoadImmediate(o_h, 0); 3859 } 3860 } else if (shift_value == 32) { 3861 if (op->IsShl()) { 3862 __ mov(o_h, ShifterOperand(low)); 3863 __ LoadImmediate(o_l, 0); 3864 } else if (op->IsShr()) { 3865 __ mov(o_l, ShifterOperand(high)); 3866 __ Asr(o_h, high, 31); 3867 } else { 3868 __ mov(o_l, ShifterOperand(high)); 3869 __ LoadImmediate(o_h, 0); 3870 } 3871 } else if (shift_value == 1) { 3872 if (op->IsShl()) { 3873 __ Lsls(o_l, low, 1); 3874 __ adc(o_h, high, ShifterOperand(high)); 3875 } else if (op->IsShr()) { 3876 __ Asrs(o_h, high, 1); 3877 __ Rrx(o_l, low); 3878 } else { 3879 __ Lsrs(o_h, high, 1); 3880 __ Rrx(o_l, low); 3881 } 3882 } else { 3883 DCHECK(2 <= shift_value && shift_value < 32) << shift_value; 3884 if (op->IsShl()) { 3885 __ Lsl(o_h, high, shift_value); 3886 __ orr(o_h, o_h, ShifterOperand(low, LSR, 32 - shift_value)); 3887 __ Lsl(o_l, low, shift_value); 3888 } else if (op->IsShr()) { 3889 __ Lsr(o_l, low, shift_value); 3890 __ orr(o_l, o_l, ShifterOperand(high, LSL, 32 - shift_value)); 3891 __ Asr(o_h, high, shift_value); 3892 } else { 3893 __ Lsr(o_l, low, shift_value); 3894 __ orr(o_l, o_l, ShifterOperand(high, LSL, 32 - shift_value)); 3895 __ Lsr(o_h, high, shift_value); 3896 } 3897 } 3898 } 3899 break; 3900 } 3901 default: 3902 LOG(FATAL) << "Unexpected operation type " << type; 3903 UNREACHABLE(); 3904 } 3905} 3906 3907void LocationsBuilderARM::VisitShl(HShl* shl) { 3908 HandleShift(shl); 3909} 3910 3911void InstructionCodeGeneratorARM::VisitShl(HShl* shl) { 3912 HandleShift(shl); 3913} 3914 3915void LocationsBuilderARM::VisitShr(HShr* shr) { 3916 HandleShift(shr); 3917} 3918 3919void InstructionCodeGeneratorARM::VisitShr(HShr* shr) { 3920 HandleShift(shr); 3921} 3922 3923void LocationsBuilderARM::VisitUShr(HUShr* ushr) { 3924 HandleShift(ushr); 3925} 3926 3927void InstructionCodeGeneratorARM::VisitUShr(HUShr* ushr) { 3928 HandleShift(ushr); 3929} 3930 3931void LocationsBuilderARM::VisitNewInstance(HNewInstance* instruction) { 3932 LocationSummary* locations = 3933 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCallOnMainOnly); 3934 if (instruction->IsStringAlloc()) { 3935 locations->AddTemp(Location::RegisterLocation(kMethodRegisterArgument)); 3936 } else { 3937 InvokeRuntimeCallingConvention calling_convention; 3938 locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(0))); 3939 locations->SetInAt(1, Location::RegisterLocation(calling_convention.GetRegisterAt(1))); 3940 } 3941 locations->SetOut(Location::RegisterLocation(R0)); 3942} 3943 3944void InstructionCodeGeneratorARM::VisitNewInstance(HNewInstance* instruction) { 3945 // Note: if heap poisoning is enabled, the entry point takes cares 3946 // of poisoning the reference. 3947 if (instruction->IsStringAlloc()) { 3948 // String is allocated through StringFactory. Call NewEmptyString entry point. 3949 Register temp = instruction->GetLocations()->GetTemp(0).AsRegister<Register>(); 3950 MemberOffset code_offset = ArtMethod::EntryPointFromQuickCompiledCodeOffset(kArmPointerSize); 3951 __ LoadFromOffset(kLoadWord, temp, TR, QUICK_ENTRY_POINT(pNewEmptyString)); 3952 __ LoadFromOffset(kLoadWord, LR, temp, code_offset.Int32Value()); 3953 __ blx(LR); 3954 codegen_->RecordPcInfo(instruction, instruction->GetDexPc()); 3955 } else { 3956 codegen_->InvokeRuntime(instruction->GetEntrypoint(), instruction, instruction->GetDexPc()); 3957 CheckEntrypointTypes<kQuickAllocObjectWithAccessCheck, void*, uint32_t, ArtMethod*>(); 3958 } 3959} 3960 3961void LocationsBuilderARM::VisitNewArray(HNewArray* instruction) { 3962 LocationSummary* locations = 3963 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCallOnMainOnly); 3964 InvokeRuntimeCallingConvention calling_convention; 3965 locations->AddTemp(Location::RegisterLocation(calling_convention.GetRegisterAt(0))); 3966 locations->SetOut(Location::RegisterLocation(R0)); 3967 locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(1))); 3968 locations->SetInAt(1, Location::RegisterLocation(calling_convention.GetRegisterAt(2))); 3969} 3970 3971void InstructionCodeGeneratorARM::VisitNewArray(HNewArray* instruction) { 3972 InvokeRuntimeCallingConvention calling_convention; 3973 __ LoadImmediate(calling_convention.GetRegisterAt(0), instruction->GetTypeIndex().index_); 3974 // Note: if heap poisoning is enabled, the entry point takes cares 3975 // of poisoning the reference. 3976 codegen_->InvokeRuntime(instruction->GetEntrypoint(), instruction, instruction->GetDexPc()); 3977 CheckEntrypointTypes<kQuickAllocArrayWithAccessCheck, void*, uint32_t, int32_t, ArtMethod*>(); 3978} 3979 3980void LocationsBuilderARM::VisitParameterValue(HParameterValue* instruction) { 3981 LocationSummary* locations = 3982 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall); 3983 Location location = parameter_visitor_.GetNextLocation(instruction->GetType()); 3984 if (location.IsStackSlot()) { 3985 location = Location::StackSlot(location.GetStackIndex() + codegen_->GetFrameSize()); 3986 } else if (location.IsDoubleStackSlot()) { 3987 location = Location::DoubleStackSlot(location.GetStackIndex() + codegen_->GetFrameSize()); 3988 } 3989 locations->SetOut(location); 3990} 3991 3992void InstructionCodeGeneratorARM::VisitParameterValue( 3993 HParameterValue* instruction ATTRIBUTE_UNUSED) { 3994 // Nothing to do, the parameter is already at its location. 3995} 3996 3997void LocationsBuilderARM::VisitCurrentMethod(HCurrentMethod* instruction) { 3998 LocationSummary* locations = 3999 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall); 4000 locations->SetOut(Location::RegisterLocation(kMethodRegisterArgument)); 4001} 4002 4003void InstructionCodeGeneratorARM::VisitCurrentMethod(HCurrentMethod* instruction ATTRIBUTE_UNUSED) { 4004 // Nothing to do, the method is already at its location. 4005} 4006 4007void LocationsBuilderARM::VisitNot(HNot* not_) { 4008 LocationSummary* locations = 4009 new (GetGraph()->GetArena()) LocationSummary(not_, LocationSummary::kNoCall); 4010 locations->SetInAt(0, Location::RequiresRegister()); 4011 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 4012} 4013 4014void InstructionCodeGeneratorARM::VisitNot(HNot* not_) { 4015 LocationSummary* locations = not_->GetLocations(); 4016 Location out = locations->Out(); 4017 Location in = locations->InAt(0); 4018 switch (not_->GetResultType()) { 4019 case Primitive::kPrimInt: 4020 __ mvn(out.AsRegister<Register>(), ShifterOperand(in.AsRegister<Register>())); 4021 break; 4022 4023 case Primitive::kPrimLong: 4024 __ mvn(out.AsRegisterPairLow<Register>(), 4025 ShifterOperand(in.AsRegisterPairLow<Register>())); 4026 __ mvn(out.AsRegisterPairHigh<Register>(), 4027 ShifterOperand(in.AsRegisterPairHigh<Register>())); 4028 break; 4029 4030 default: 4031 LOG(FATAL) << "Unimplemented type for not operation " << not_->GetResultType(); 4032 } 4033} 4034 4035void LocationsBuilderARM::VisitBooleanNot(HBooleanNot* bool_not) { 4036 LocationSummary* locations = 4037 new (GetGraph()->GetArena()) LocationSummary(bool_not, LocationSummary::kNoCall); 4038 locations->SetInAt(0, Location::RequiresRegister()); 4039 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 4040} 4041 4042void InstructionCodeGeneratorARM::VisitBooleanNot(HBooleanNot* bool_not) { 4043 LocationSummary* locations = bool_not->GetLocations(); 4044 Location out = locations->Out(); 4045 Location in = locations->InAt(0); 4046 __ eor(out.AsRegister<Register>(), in.AsRegister<Register>(), ShifterOperand(1)); 4047} 4048 4049void LocationsBuilderARM::VisitCompare(HCompare* compare) { 4050 LocationSummary* locations = 4051 new (GetGraph()->GetArena()) LocationSummary(compare, LocationSummary::kNoCall); 4052 switch (compare->InputAt(0)->GetType()) { 4053 case Primitive::kPrimBoolean: 4054 case Primitive::kPrimByte: 4055 case Primitive::kPrimShort: 4056 case Primitive::kPrimChar: 4057 case Primitive::kPrimInt: 4058 case Primitive::kPrimLong: { 4059 locations->SetInAt(0, Location::RequiresRegister()); 4060 locations->SetInAt(1, Location::RequiresRegister()); 4061 // Output overlaps because it is written before doing the low comparison. 4062 locations->SetOut(Location::RequiresRegister(), Location::kOutputOverlap); 4063 break; 4064 } 4065 case Primitive::kPrimFloat: 4066 case Primitive::kPrimDouble: { 4067 locations->SetInAt(0, Location::RequiresFpuRegister()); 4068 locations->SetInAt(1, ArithmeticZeroOrFpuRegister(compare->InputAt(1))); 4069 locations->SetOut(Location::RequiresRegister()); 4070 break; 4071 } 4072 default: 4073 LOG(FATAL) << "Unexpected type for compare operation " << compare->InputAt(0)->GetType(); 4074 } 4075} 4076 4077void InstructionCodeGeneratorARM::VisitCompare(HCompare* compare) { 4078 LocationSummary* locations = compare->GetLocations(); 4079 Register out = locations->Out().AsRegister<Register>(); 4080 Location left = locations->InAt(0); 4081 Location right = locations->InAt(1); 4082 4083 Label less, greater, done; 4084 Primitive::Type type = compare->InputAt(0)->GetType(); 4085 Condition less_cond; 4086 switch (type) { 4087 case Primitive::kPrimBoolean: 4088 case Primitive::kPrimByte: 4089 case Primitive::kPrimShort: 4090 case Primitive::kPrimChar: 4091 case Primitive::kPrimInt: { 4092 __ LoadImmediate(out, 0); 4093 __ cmp(left.AsRegister<Register>(), 4094 ShifterOperand(right.AsRegister<Register>())); // Signed compare. 4095 less_cond = LT; 4096 break; 4097 } 4098 case Primitive::kPrimLong: { 4099 __ cmp(left.AsRegisterPairHigh<Register>(), 4100 ShifterOperand(right.AsRegisterPairHigh<Register>())); // Signed compare. 4101 __ b(&less, LT); 4102 __ b(&greater, GT); 4103 // Do LoadImmediate before the last `cmp`, as LoadImmediate might affect the status flags. 4104 __ LoadImmediate(out, 0); 4105 __ cmp(left.AsRegisterPairLow<Register>(), 4106 ShifterOperand(right.AsRegisterPairLow<Register>())); // Unsigned compare. 4107 less_cond = LO; 4108 break; 4109 } 4110 case Primitive::kPrimFloat: 4111 case Primitive::kPrimDouble: { 4112 __ LoadImmediate(out, 0); 4113 GenerateVcmp(compare); 4114 __ vmstat(); // transfer FP status register to ARM APSR. 4115 less_cond = ARMFPCondition(kCondLT, compare->IsGtBias()); 4116 break; 4117 } 4118 default: 4119 LOG(FATAL) << "Unexpected compare type " << type; 4120 UNREACHABLE(); 4121 } 4122 4123 __ b(&done, EQ); 4124 __ b(&less, less_cond); 4125 4126 __ Bind(&greater); 4127 __ LoadImmediate(out, 1); 4128 __ b(&done); 4129 4130 __ Bind(&less); 4131 __ LoadImmediate(out, -1); 4132 4133 __ Bind(&done); 4134} 4135 4136void LocationsBuilderARM::VisitPhi(HPhi* instruction) { 4137 LocationSummary* locations = 4138 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall); 4139 for (size_t i = 0, e = locations->GetInputCount(); i < e; ++i) { 4140 locations->SetInAt(i, Location::Any()); 4141 } 4142 locations->SetOut(Location::Any()); 4143} 4144 4145void InstructionCodeGeneratorARM::VisitPhi(HPhi* instruction ATTRIBUTE_UNUSED) { 4146 LOG(FATAL) << "Unreachable"; 4147} 4148 4149void CodeGeneratorARM::GenerateMemoryBarrier(MemBarrierKind kind) { 4150 // TODO (ported from quick): revisit ARM barrier kinds. 4151 DmbOptions flavor = DmbOptions::ISH; // Quiet C++ warnings. 4152 switch (kind) { 4153 case MemBarrierKind::kAnyStore: 4154 case MemBarrierKind::kLoadAny: 4155 case MemBarrierKind::kAnyAny: { 4156 flavor = DmbOptions::ISH; 4157 break; 4158 } 4159 case MemBarrierKind::kStoreStore: { 4160 flavor = DmbOptions::ISHST; 4161 break; 4162 } 4163 default: 4164 LOG(FATAL) << "Unexpected memory barrier " << kind; 4165 } 4166 __ dmb(flavor); 4167} 4168 4169void InstructionCodeGeneratorARM::GenerateWideAtomicLoad(Register addr, 4170 uint32_t offset, 4171 Register out_lo, 4172 Register out_hi) { 4173 if (offset != 0) { 4174 // Ensure `out_lo` is different from `addr`, so that loading 4175 // `offset` into `out_lo` does not clutter `addr`. 4176 DCHECK_NE(out_lo, addr); 4177 __ LoadImmediate(out_lo, offset); 4178 __ add(IP, addr, ShifterOperand(out_lo)); 4179 addr = IP; 4180 } 4181 __ ldrexd(out_lo, out_hi, addr); 4182} 4183 4184void InstructionCodeGeneratorARM::GenerateWideAtomicStore(Register addr, 4185 uint32_t offset, 4186 Register value_lo, 4187 Register value_hi, 4188 Register temp1, 4189 Register temp2, 4190 HInstruction* instruction) { 4191 Label fail; 4192 if (offset != 0) { 4193 __ LoadImmediate(temp1, offset); 4194 __ add(IP, addr, ShifterOperand(temp1)); 4195 addr = IP; 4196 } 4197 __ Bind(&fail); 4198 // We need a load followed by store. (The address used in a STREX instruction must 4199 // be the same as the address in the most recently executed LDREX instruction.) 4200 __ ldrexd(temp1, temp2, addr); 4201 codegen_->MaybeRecordImplicitNullCheck(instruction); 4202 __ strexd(temp1, value_lo, value_hi, addr); 4203 __ CompareAndBranchIfNonZero(temp1, &fail); 4204} 4205 4206void LocationsBuilderARM::HandleFieldSet(HInstruction* instruction, const FieldInfo& field_info) { 4207 DCHECK(instruction->IsInstanceFieldSet() || instruction->IsStaticFieldSet()); 4208 4209 LocationSummary* locations = 4210 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall); 4211 locations->SetInAt(0, Location::RequiresRegister()); 4212 4213 Primitive::Type field_type = field_info.GetFieldType(); 4214 if (Primitive::IsFloatingPointType(field_type)) { 4215 locations->SetInAt(1, Location::RequiresFpuRegister()); 4216 } else { 4217 locations->SetInAt(1, Location::RequiresRegister()); 4218 } 4219 4220 bool is_wide = field_type == Primitive::kPrimLong || field_type == Primitive::kPrimDouble; 4221 bool generate_volatile = field_info.IsVolatile() 4222 && is_wide 4223 && !codegen_->GetInstructionSetFeatures().HasAtomicLdrdAndStrd(); 4224 bool needs_write_barrier = 4225 CodeGenerator::StoreNeedsWriteBarrier(field_type, instruction->InputAt(1)); 4226 // Temporary registers for the write barrier. 4227 // TODO: consider renaming StoreNeedsWriteBarrier to StoreNeedsGCMark. 4228 if (needs_write_barrier) { 4229 locations->AddTemp(Location::RequiresRegister()); // Possibly used for reference poisoning too. 4230 locations->AddTemp(Location::RequiresRegister()); 4231 } else if (generate_volatile) { 4232 // ARM encoding have some additional constraints for ldrexd/strexd: 4233 // - registers need to be consecutive 4234 // - the first register should be even but not R14. 4235 // We don't test for ARM yet, and the assertion makes sure that we 4236 // revisit this if we ever enable ARM encoding. 4237 DCHECK_EQ(InstructionSet::kThumb2, codegen_->GetInstructionSet()); 4238 4239 locations->AddTemp(Location::RequiresRegister()); 4240 locations->AddTemp(Location::RequiresRegister()); 4241 if (field_type == Primitive::kPrimDouble) { 4242 // For doubles we need two more registers to copy the value. 4243 locations->AddTemp(Location::RegisterLocation(R2)); 4244 locations->AddTemp(Location::RegisterLocation(R3)); 4245 } 4246 } 4247} 4248 4249void InstructionCodeGeneratorARM::HandleFieldSet(HInstruction* instruction, 4250 const FieldInfo& field_info, 4251 bool value_can_be_null) { 4252 DCHECK(instruction->IsInstanceFieldSet() || instruction->IsStaticFieldSet()); 4253 4254 LocationSummary* locations = instruction->GetLocations(); 4255 Register base = locations->InAt(0).AsRegister<Register>(); 4256 Location value = locations->InAt(1); 4257 4258 bool is_volatile = field_info.IsVolatile(); 4259 bool atomic_ldrd_strd = codegen_->GetInstructionSetFeatures().HasAtomicLdrdAndStrd(); 4260 Primitive::Type field_type = field_info.GetFieldType(); 4261 uint32_t offset = field_info.GetFieldOffset().Uint32Value(); 4262 bool needs_write_barrier = 4263 CodeGenerator::StoreNeedsWriteBarrier(field_type, instruction->InputAt(1)); 4264 4265 if (is_volatile) { 4266 codegen_->GenerateMemoryBarrier(MemBarrierKind::kAnyStore); 4267 } 4268 4269 switch (field_type) { 4270 case Primitive::kPrimBoolean: 4271 case Primitive::kPrimByte: { 4272 __ StoreToOffset(kStoreByte, value.AsRegister<Register>(), base, offset); 4273 break; 4274 } 4275 4276 case Primitive::kPrimShort: 4277 case Primitive::kPrimChar: { 4278 __ StoreToOffset(kStoreHalfword, value.AsRegister<Register>(), base, offset); 4279 break; 4280 } 4281 4282 case Primitive::kPrimInt: 4283 case Primitive::kPrimNot: { 4284 if (kPoisonHeapReferences && needs_write_barrier) { 4285 // Note that in the case where `value` is a null reference, 4286 // we do not enter this block, as a null reference does not 4287 // need poisoning. 4288 DCHECK_EQ(field_type, Primitive::kPrimNot); 4289 Register temp = locations->GetTemp(0).AsRegister<Register>(); 4290 __ Mov(temp, value.AsRegister<Register>()); 4291 __ PoisonHeapReference(temp); 4292 __ StoreToOffset(kStoreWord, temp, base, offset); 4293 } else { 4294 __ StoreToOffset(kStoreWord, value.AsRegister<Register>(), base, offset); 4295 } 4296 break; 4297 } 4298 4299 case Primitive::kPrimLong: { 4300 if (is_volatile && !atomic_ldrd_strd) { 4301 GenerateWideAtomicStore(base, offset, 4302 value.AsRegisterPairLow<Register>(), 4303 value.AsRegisterPairHigh<Register>(), 4304 locations->GetTemp(0).AsRegister<Register>(), 4305 locations->GetTemp(1).AsRegister<Register>(), 4306 instruction); 4307 } else { 4308 __ StoreToOffset(kStoreWordPair, value.AsRegisterPairLow<Register>(), base, offset); 4309 codegen_->MaybeRecordImplicitNullCheck(instruction); 4310 } 4311 break; 4312 } 4313 4314 case Primitive::kPrimFloat: { 4315 __ StoreSToOffset(value.AsFpuRegister<SRegister>(), base, offset); 4316 break; 4317 } 4318 4319 case Primitive::kPrimDouble: { 4320 DRegister value_reg = FromLowSToD(value.AsFpuRegisterPairLow<SRegister>()); 4321 if (is_volatile && !atomic_ldrd_strd) { 4322 Register value_reg_lo = locations->GetTemp(0).AsRegister<Register>(); 4323 Register value_reg_hi = locations->GetTemp(1).AsRegister<Register>(); 4324 4325 __ vmovrrd(value_reg_lo, value_reg_hi, value_reg); 4326 4327 GenerateWideAtomicStore(base, offset, 4328 value_reg_lo, 4329 value_reg_hi, 4330 locations->GetTemp(2).AsRegister<Register>(), 4331 locations->GetTemp(3).AsRegister<Register>(), 4332 instruction); 4333 } else { 4334 __ StoreDToOffset(value_reg, base, offset); 4335 codegen_->MaybeRecordImplicitNullCheck(instruction); 4336 } 4337 break; 4338 } 4339 4340 case Primitive::kPrimVoid: 4341 LOG(FATAL) << "Unreachable type " << field_type; 4342 UNREACHABLE(); 4343 } 4344 4345 // Longs and doubles are handled in the switch. 4346 if (field_type != Primitive::kPrimLong && field_type != Primitive::kPrimDouble) { 4347 codegen_->MaybeRecordImplicitNullCheck(instruction); 4348 } 4349 4350 if (CodeGenerator::StoreNeedsWriteBarrier(field_type, instruction->InputAt(1))) { 4351 Register temp = locations->GetTemp(0).AsRegister<Register>(); 4352 Register card = locations->GetTemp(1).AsRegister<Register>(); 4353 codegen_->MarkGCCard( 4354 temp, card, base, value.AsRegister<Register>(), value_can_be_null); 4355 } 4356 4357 if (is_volatile) { 4358 codegen_->GenerateMemoryBarrier(MemBarrierKind::kAnyAny); 4359 } 4360} 4361 4362void LocationsBuilderARM::HandleFieldGet(HInstruction* instruction, const FieldInfo& field_info) { 4363 DCHECK(instruction->IsInstanceFieldGet() || instruction->IsStaticFieldGet()); 4364 4365 bool object_field_get_with_read_barrier = 4366 kEmitCompilerReadBarrier && (field_info.GetFieldType() == Primitive::kPrimNot); 4367 LocationSummary* locations = 4368 new (GetGraph()->GetArena()) LocationSummary(instruction, 4369 object_field_get_with_read_barrier ? 4370 LocationSummary::kCallOnSlowPath : 4371 LocationSummary::kNoCall); 4372 if (object_field_get_with_read_barrier && kUseBakerReadBarrier) { 4373 locations->SetCustomSlowPathCallerSaves(RegisterSet::Empty()); // No caller-save registers. 4374 } 4375 locations->SetInAt(0, Location::RequiresRegister()); 4376 4377 bool volatile_for_double = field_info.IsVolatile() 4378 && (field_info.GetFieldType() == Primitive::kPrimDouble) 4379 && !codegen_->GetInstructionSetFeatures().HasAtomicLdrdAndStrd(); 4380 // The output overlaps in case of volatile long: we don't want the 4381 // code generated by GenerateWideAtomicLoad to overwrite the 4382 // object's location. Likewise, in the case of an object field get 4383 // with read barriers enabled, we do not want the load to overwrite 4384 // the object's location, as we need it to emit the read barrier. 4385 bool overlap = (field_info.IsVolatile() && (field_info.GetFieldType() == Primitive::kPrimLong)) || 4386 object_field_get_with_read_barrier; 4387 4388 if (Primitive::IsFloatingPointType(instruction->GetType())) { 4389 locations->SetOut(Location::RequiresFpuRegister()); 4390 } else { 4391 locations->SetOut(Location::RequiresRegister(), 4392 (overlap ? Location::kOutputOverlap : Location::kNoOutputOverlap)); 4393 } 4394 if (volatile_for_double) { 4395 // ARM encoding have some additional constraints for ldrexd/strexd: 4396 // - registers need to be consecutive 4397 // - the first register should be even but not R14. 4398 // We don't test for ARM yet, and the assertion makes sure that we 4399 // revisit this if we ever enable ARM encoding. 4400 DCHECK_EQ(InstructionSet::kThumb2, codegen_->GetInstructionSet()); 4401 locations->AddTemp(Location::RequiresRegister()); 4402 locations->AddTemp(Location::RequiresRegister()); 4403 } else if (object_field_get_with_read_barrier && kUseBakerReadBarrier) { 4404 // We need a temporary register for the read barrier marking slow 4405 // path in CodeGeneratorARM::GenerateFieldLoadWithBakerReadBarrier. 4406 locations->AddTemp(Location::RequiresRegister()); 4407 } 4408} 4409 4410Location LocationsBuilderARM::ArithmeticZeroOrFpuRegister(HInstruction* input) { 4411 DCHECK(input->GetType() == Primitive::kPrimDouble || input->GetType() == Primitive::kPrimFloat) 4412 << input->GetType(); 4413 if ((input->IsFloatConstant() && (input->AsFloatConstant()->IsArithmeticZero())) || 4414 (input->IsDoubleConstant() && (input->AsDoubleConstant()->IsArithmeticZero()))) { 4415 return Location::ConstantLocation(input->AsConstant()); 4416 } else { 4417 return Location::RequiresFpuRegister(); 4418 } 4419} 4420 4421Location LocationsBuilderARM::ArmEncodableConstantOrRegister(HInstruction* constant, 4422 Opcode opcode) { 4423 DCHECK(!Primitive::IsFloatingPointType(constant->GetType())); 4424 if (constant->IsConstant() && 4425 CanEncodeConstantAsImmediate(constant->AsConstant(), opcode)) { 4426 return Location::ConstantLocation(constant->AsConstant()); 4427 } 4428 return Location::RequiresRegister(); 4429} 4430 4431bool LocationsBuilderARM::CanEncodeConstantAsImmediate(HConstant* input_cst, 4432 Opcode opcode) { 4433 uint64_t value = static_cast<uint64_t>(Int64FromConstant(input_cst)); 4434 if (Primitive::Is64BitType(input_cst->GetType())) { 4435 Opcode high_opcode = opcode; 4436 SetCc low_set_cc = kCcDontCare; 4437 switch (opcode) { 4438 case SUB: 4439 // Flip the operation to an ADD. 4440 value = -value; 4441 opcode = ADD; 4442 FALLTHROUGH_INTENDED; 4443 case ADD: 4444 if (Low32Bits(value) == 0u) { 4445 return CanEncodeConstantAsImmediate(High32Bits(value), opcode, kCcDontCare); 4446 } 4447 high_opcode = ADC; 4448 low_set_cc = kCcSet; 4449 break; 4450 default: 4451 break; 4452 } 4453 return CanEncodeConstantAsImmediate(Low32Bits(value), opcode, low_set_cc) && 4454 CanEncodeConstantAsImmediate(High32Bits(value), high_opcode, kCcDontCare); 4455 } else { 4456 return CanEncodeConstantAsImmediate(Low32Bits(value), opcode); 4457 } 4458} 4459 4460bool LocationsBuilderARM::CanEncodeConstantAsImmediate(uint32_t value, 4461 Opcode opcode, 4462 SetCc set_cc) { 4463 ShifterOperand so; 4464 ArmAssembler* assembler = codegen_->GetAssembler(); 4465 if (assembler->ShifterOperandCanHold(kNoRegister, kNoRegister, opcode, value, set_cc, &so)) { 4466 return true; 4467 } 4468 Opcode neg_opcode = kNoOperand; 4469 switch (opcode) { 4470 case AND: neg_opcode = BIC; value = ~value; break; 4471 case ORR: neg_opcode = ORN; value = ~value; break; 4472 case ADD: neg_opcode = SUB; value = -value; break; 4473 case ADC: neg_opcode = SBC; value = ~value; break; 4474 case SUB: neg_opcode = ADD; value = -value; break; 4475 case SBC: neg_opcode = ADC; value = ~value; break; 4476 default: 4477 return false; 4478 } 4479 return assembler->ShifterOperandCanHold(kNoRegister, kNoRegister, neg_opcode, value, set_cc, &so); 4480} 4481 4482void InstructionCodeGeneratorARM::HandleFieldGet(HInstruction* instruction, 4483 const FieldInfo& field_info) { 4484 DCHECK(instruction->IsInstanceFieldGet() || instruction->IsStaticFieldGet()); 4485 4486 LocationSummary* locations = instruction->GetLocations(); 4487 Location base_loc = locations->InAt(0); 4488 Register base = base_loc.AsRegister<Register>(); 4489 Location out = locations->Out(); 4490 bool is_volatile = field_info.IsVolatile(); 4491 bool atomic_ldrd_strd = codegen_->GetInstructionSetFeatures().HasAtomicLdrdAndStrd(); 4492 Primitive::Type field_type = field_info.GetFieldType(); 4493 uint32_t offset = field_info.GetFieldOffset().Uint32Value(); 4494 4495 switch (field_type) { 4496 case Primitive::kPrimBoolean: 4497 __ LoadFromOffset(kLoadUnsignedByte, out.AsRegister<Register>(), base, offset); 4498 break; 4499 4500 case Primitive::kPrimByte: 4501 __ LoadFromOffset(kLoadSignedByte, out.AsRegister<Register>(), base, offset); 4502 break; 4503 4504 case Primitive::kPrimShort: 4505 __ LoadFromOffset(kLoadSignedHalfword, out.AsRegister<Register>(), base, offset); 4506 break; 4507 4508 case Primitive::kPrimChar: 4509 __ LoadFromOffset(kLoadUnsignedHalfword, out.AsRegister<Register>(), base, offset); 4510 break; 4511 4512 case Primitive::kPrimInt: 4513 __ LoadFromOffset(kLoadWord, out.AsRegister<Register>(), base, offset); 4514 break; 4515 4516 case Primitive::kPrimNot: { 4517 // /* HeapReference<Object> */ out = *(base + offset) 4518 if (kEmitCompilerReadBarrier && kUseBakerReadBarrier) { 4519 Location temp_loc = locations->GetTemp(0); 4520 // Note that a potential implicit null check is handled in this 4521 // CodeGeneratorARM::GenerateFieldLoadWithBakerReadBarrier call. 4522 codegen_->GenerateFieldLoadWithBakerReadBarrier( 4523 instruction, out, base, offset, temp_loc, /* needs_null_check */ true); 4524 if (is_volatile) { 4525 codegen_->GenerateMemoryBarrier(MemBarrierKind::kLoadAny); 4526 } 4527 } else { 4528 __ LoadFromOffset(kLoadWord, out.AsRegister<Register>(), base, offset); 4529 codegen_->MaybeRecordImplicitNullCheck(instruction); 4530 if (is_volatile) { 4531 codegen_->GenerateMemoryBarrier(MemBarrierKind::kLoadAny); 4532 } 4533 // If read barriers are enabled, emit read barriers other than 4534 // Baker's using a slow path (and also unpoison the loaded 4535 // reference, if heap poisoning is enabled). 4536 codegen_->MaybeGenerateReadBarrierSlow(instruction, out, out, base_loc, offset); 4537 } 4538 break; 4539 } 4540 4541 case Primitive::kPrimLong: 4542 if (is_volatile && !atomic_ldrd_strd) { 4543 GenerateWideAtomicLoad(base, offset, 4544 out.AsRegisterPairLow<Register>(), 4545 out.AsRegisterPairHigh<Register>()); 4546 } else { 4547 __ LoadFromOffset(kLoadWordPair, out.AsRegisterPairLow<Register>(), base, offset); 4548 } 4549 break; 4550 4551 case Primitive::kPrimFloat: 4552 __ LoadSFromOffset(out.AsFpuRegister<SRegister>(), base, offset); 4553 break; 4554 4555 case Primitive::kPrimDouble: { 4556 DRegister out_reg = FromLowSToD(out.AsFpuRegisterPairLow<SRegister>()); 4557 if (is_volatile && !atomic_ldrd_strd) { 4558 Register lo = locations->GetTemp(0).AsRegister<Register>(); 4559 Register hi = locations->GetTemp(1).AsRegister<Register>(); 4560 GenerateWideAtomicLoad(base, offset, lo, hi); 4561 codegen_->MaybeRecordImplicitNullCheck(instruction); 4562 __ vmovdrr(out_reg, lo, hi); 4563 } else { 4564 __ LoadDFromOffset(out_reg, base, offset); 4565 codegen_->MaybeRecordImplicitNullCheck(instruction); 4566 } 4567 break; 4568 } 4569 4570 case Primitive::kPrimVoid: 4571 LOG(FATAL) << "Unreachable type " << field_type; 4572 UNREACHABLE(); 4573 } 4574 4575 if (field_type == Primitive::kPrimNot || field_type == Primitive::kPrimDouble) { 4576 // Potential implicit null checks, in the case of reference or 4577 // double fields, are handled in the previous switch statement. 4578 } else { 4579 codegen_->MaybeRecordImplicitNullCheck(instruction); 4580 } 4581 4582 if (is_volatile) { 4583 if (field_type == Primitive::kPrimNot) { 4584 // Memory barriers, in the case of references, are also handled 4585 // in the previous switch statement. 4586 } else { 4587 codegen_->GenerateMemoryBarrier(MemBarrierKind::kLoadAny); 4588 } 4589 } 4590} 4591 4592void LocationsBuilderARM::VisitInstanceFieldSet(HInstanceFieldSet* instruction) { 4593 HandleFieldSet(instruction, instruction->GetFieldInfo()); 4594} 4595 4596void InstructionCodeGeneratorARM::VisitInstanceFieldSet(HInstanceFieldSet* instruction) { 4597 HandleFieldSet(instruction, instruction->GetFieldInfo(), instruction->GetValueCanBeNull()); 4598} 4599 4600void LocationsBuilderARM::VisitInstanceFieldGet(HInstanceFieldGet* instruction) { 4601 HandleFieldGet(instruction, instruction->GetFieldInfo()); 4602} 4603 4604void InstructionCodeGeneratorARM::VisitInstanceFieldGet(HInstanceFieldGet* instruction) { 4605 HandleFieldGet(instruction, instruction->GetFieldInfo()); 4606} 4607 4608void LocationsBuilderARM::VisitStaticFieldGet(HStaticFieldGet* instruction) { 4609 HandleFieldGet(instruction, instruction->GetFieldInfo()); 4610} 4611 4612void InstructionCodeGeneratorARM::VisitStaticFieldGet(HStaticFieldGet* instruction) { 4613 HandleFieldGet(instruction, instruction->GetFieldInfo()); 4614} 4615 4616void LocationsBuilderARM::VisitStaticFieldSet(HStaticFieldSet* instruction) { 4617 HandleFieldSet(instruction, instruction->GetFieldInfo()); 4618} 4619 4620void InstructionCodeGeneratorARM::VisitStaticFieldSet(HStaticFieldSet* instruction) { 4621 HandleFieldSet(instruction, instruction->GetFieldInfo(), instruction->GetValueCanBeNull()); 4622} 4623 4624void LocationsBuilderARM::VisitUnresolvedInstanceFieldGet( 4625 HUnresolvedInstanceFieldGet* instruction) { 4626 FieldAccessCallingConventionARM calling_convention; 4627 codegen_->CreateUnresolvedFieldLocationSummary( 4628 instruction, instruction->GetFieldType(), calling_convention); 4629} 4630 4631void InstructionCodeGeneratorARM::VisitUnresolvedInstanceFieldGet( 4632 HUnresolvedInstanceFieldGet* instruction) { 4633 FieldAccessCallingConventionARM calling_convention; 4634 codegen_->GenerateUnresolvedFieldAccess(instruction, 4635 instruction->GetFieldType(), 4636 instruction->GetFieldIndex(), 4637 instruction->GetDexPc(), 4638 calling_convention); 4639} 4640 4641void LocationsBuilderARM::VisitUnresolvedInstanceFieldSet( 4642 HUnresolvedInstanceFieldSet* instruction) { 4643 FieldAccessCallingConventionARM calling_convention; 4644 codegen_->CreateUnresolvedFieldLocationSummary( 4645 instruction, instruction->GetFieldType(), calling_convention); 4646} 4647 4648void InstructionCodeGeneratorARM::VisitUnresolvedInstanceFieldSet( 4649 HUnresolvedInstanceFieldSet* instruction) { 4650 FieldAccessCallingConventionARM calling_convention; 4651 codegen_->GenerateUnresolvedFieldAccess(instruction, 4652 instruction->GetFieldType(), 4653 instruction->GetFieldIndex(), 4654 instruction->GetDexPc(), 4655 calling_convention); 4656} 4657 4658void LocationsBuilderARM::VisitUnresolvedStaticFieldGet( 4659 HUnresolvedStaticFieldGet* instruction) { 4660 FieldAccessCallingConventionARM calling_convention; 4661 codegen_->CreateUnresolvedFieldLocationSummary( 4662 instruction, instruction->GetFieldType(), calling_convention); 4663} 4664 4665void InstructionCodeGeneratorARM::VisitUnresolvedStaticFieldGet( 4666 HUnresolvedStaticFieldGet* instruction) { 4667 FieldAccessCallingConventionARM calling_convention; 4668 codegen_->GenerateUnresolvedFieldAccess(instruction, 4669 instruction->GetFieldType(), 4670 instruction->GetFieldIndex(), 4671 instruction->GetDexPc(), 4672 calling_convention); 4673} 4674 4675void LocationsBuilderARM::VisitUnresolvedStaticFieldSet( 4676 HUnresolvedStaticFieldSet* instruction) { 4677 FieldAccessCallingConventionARM calling_convention; 4678 codegen_->CreateUnresolvedFieldLocationSummary( 4679 instruction, instruction->GetFieldType(), calling_convention); 4680} 4681 4682void InstructionCodeGeneratorARM::VisitUnresolvedStaticFieldSet( 4683 HUnresolvedStaticFieldSet* instruction) { 4684 FieldAccessCallingConventionARM calling_convention; 4685 codegen_->GenerateUnresolvedFieldAccess(instruction, 4686 instruction->GetFieldType(), 4687 instruction->GetFieldIndex(), 4688 instruction->GetDexPc(), 4689 calling_convention); 4690} 4691 4692void LocationsBuilderARM::VisitNullCheck(HNullCheck* instruction) { 4693 LocationSummary* locations = codegen_->CreateThrowingSlowPathLocations(instruction); 4694 locations->SetInAt(0, Location::RequiresRegister()); 4695} 4696 4697void CodeGeneratorARM::GenerateImplicitNullCheck(HNullCheck* instruction) { 4698 if (CanMoveNullCheckToUser(instruction)) { 4699 return; 4700 } 4701 Location obj = instruction->GetLocations()->InAt(0); 4702 4703 __ LoadFromOffset(kLoadWord, IP, obj.AsRegister<Register>(), 0); 4704 RecordPcInfo(instruction, instruction->GetDexPc()); 4705} 4706 4707void CodeGeneratorARM::GenerateExplicitNullCheck(HNullCheck* instruction) { 4708 SlowPathCodeARM* slow_path = new (GetGraph()->GetArena()) NullCheckSlowPathARM(instruction); 4709 AddSlowPath(slow_path); 4710 4711 LocationSummary* locations = instruction->GetLocations(); 4712 Location obj = locations->InAt(0); 4713 4714 __ CompareAndBranchIfZero(obj.AsRegister<Register>(), slow_path->GetEntryLabel()); 4715} 4716 4717void InstructionCodeGeneratorARM::VisitNullCheck(HNullCheck* instruction) { 4718 codegen_->GenerateNullCheck(instruction); 4719} 4720 4721static LoadOperandType GetLoadOperandType(Primitive::Type type) { 4722 switch (type) { 4723 case Primitive::kPrimNot: 4724 return kLoadWord; 4725 case Primitive::kPrimBoolean: 4726 return kLoadUnsignedByte; 4727 case Primitive::kPrimByte: 4728 return kLoadSignedByte; 4729 case Primitive::kPrimChar: 4730 return kLoadUnsignedHalfword; 4731 case Primitive::kPrimShort: 4732 return kLoadSignedHalfword; 4733 case Primitive::kPrimInt: 4734 return kLoadWord; 4735 case Primitive::kPrimLong: 4736 return kLoadWordPair; 4737 case Primitive::kPrimFloat: 4738 return kLoadSWord; 4739 case Primitive::kPrimDouble: 4740 return kLoadDWord; 4741 default: 4742 LOG(FATAL) << "Unreachable type " << type; 4743 UNREACHABLE(); 4744 } 4745} 4746 4747static StoreOperandType GetStoreOperandType(Primitive::Type type) { 4748 switch (type) { 4749 case Primitive::kPrimNot: 4750 return kStoreWord; 4751 case Primitive::kPrimBoolean: 4752 case Primitive::kPrimByte: 4753 return kStoreByte; 4754 case Primitive::kPrimChar: 4755 case Primitive::kPrimShort: 4756 return kStoreHalfword; 4757 case Primitive::kPrimInt: 4758 return kStoreWord; 4759 case Primitive::kPrimLong: 4760 return kStoreWordPair; 4761 case Primitive::kPrimFloat: 4762 return kStoreSWord; 4763 case Primitive::kPrimDouble: 4764 return kStoreDWord; 4765 default: 4766 LOG(FATAL) << "Unreachable type " << type; 4767 UNREACHABLE(); 4768 } 4769} 4770 4771void CodeGeneratorARM::LoadFromShiftedRegOffset(Primitive::Type type, 4772 Location out_loc, 4773 Register base, 4774 Register reg_offset, 4775 Condition cond) { 4776 uint32_t shift_count = Primitive::ComponentSizeShift(type); 4777 Address mem_address(base, reg_offset, Shift::LSL, shift_count); 4778 4779 switch (type) { 4780 case Primitive::kPrimByte: 4781 __ ldrsb(out_loc.AsRegister<Register>(), mem_address, cond); 4782 break; 4783 case Primitive::kPrimBoolean: 4784 __ ldrb(out_loc.AsRegister<Register>(), mem_address, cond); 4785 break; 4786 case Primitive::kPrimShort: 4787 __ ldrsh(out_loc.AsRegister<Register>(), mem_address, cond); 4788 break; 4789 case Primitive::kPrimChar: 4790 __ ldrh(out_loc.AsRegister<Register>(), mem_address, cond); 4791 break; 4792 case Primitive::kPrimNot: 4793 case Primitive::kPrimInt: 4794 __ ldr(out_loc.AsRegister<Register>(), mem_address, cond); 4795 break; 4796 // T32 doesn't support LoadFromShiftedRegOffset mem address mode for these types. 4797 case Primitive::kPrimLong: 4798 case Primitive::kPrimFloat: 4799 case Primitive::kPrimDouble: 4800 default: 4801 LOG(FATAL) << "Unreachable type " << type; 4802 UNREACHABLE(); 4803 } 4804} 4805 4806void CodeGeneratorARM::StoreToShiftedRegOffset(Primitive::Type type, 4807 Location loc, 4808 Register base, 4809 Register reg_offset, 4810 Condition cond) { 4811 uint32_t shift_count = Primitive::ComponentSizeShift(type); 4812 Address mem_address(base, reg_offset, Shift::LSL, shift_count); 4813 4814 switch (type) { 4815 case Primitive::kPrimByte: 4816 case Primitive::kPrimBoolean: 4817 __ strb(loc.AsRegister<Register>(), mem_address, cond); 4818 break; 4819 case Primitive::kPrimShort: 4820 case Primitive::kPrimChar: 4821 __ strh(loc.AsRegister<Register>(), mem_address, cond); 4822 break; 4823 case Primitive::kPrimNot: 4824 case Primitive::kPrimInt: 4825 __ str(loc.AsRegister<Register>(), mem_address, cond); 4826 break; 4827 // T32 doesn't support StoreToShiftedRegOffset mem address mode for these types. 4828 case Primitive::kPrimLong: 4829 case Primitive::kPrimFloat: 4830 case Primitive::kPrimDouble: 4831 default: 4832 LOG(FATAL) << "Unreachable type " << type; 4833 UNREACHABLE(); 4834 } 4835} 4836 4837void LocationsBuilderARM::VisitArrayGet(HArrayGet* instruction) { 4838 bool object_array_get_with_read_barrier = 4839 kEmitCompilerReadBarrier && (instruction->GetType() == Primitive::kPrimNot); 4840 LocationSummary* locations = 4841 new (GetGraph()->GetArena()) LocationSummary(instruction, 4842 object_array_get_with_read_barrier ? 4843 LocationSummary::kCallOnSlowPath : 4844 LocationSummary::kNoCall); 4845 if (object_array_get_with_read_barrier && kUseBakerReadBarrier) { 4846 locations->SetCustomSlowPathCallerSaves(RegisterSet::Empty()); // No caller-save registers. 4847 } 4848 locations->SetInAt(0, Location::RequiresRegister()); 4849 locations->SetInAt(1, Location::RegisterOrConstant(instruction->InputAt(1))); 4850 if (Primitive::IsFloatingPointType(instruction->GetType())) { 4851 locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap); 4852 } else { 4853 // The output overlaps in the case of an object array get with 4854 // read barriers enabled: we do not want the move to overwrite the 4855 // array's location, as we need it to emit the read barrier. 4856 locations->SetOut( 4857 Location::RequiresRegister(), 4858 object_array_get_with_read_barrier ? Location::kOutputOverlap : Location::kNoOutputOverlap); 4859 } 4860 // We need a temporary register for the read barrier marking slow 4861 // path in CodeGeneratorARM::GenerateArrayLoadWithBakerReadBarrier. 4862 // Also need for String compression feature. 4863 if ((object_array_get_with_read_barrier && kUseBakerReadBarrier) 4864 || (mirror::kUseStringCompression && instruction->IsStringCharAt())) { 4865 locations->AddTemp(Location::RequiresRegister()); 4866 } 4867} 4868 4869void InstructionCodeGeneratorARM::VisitArrayGet(HArrayGet* instruction) { 4870 LocationSummary* locations = instruction->GetLocations(); 4871 Location obj_loc = locations->InAt(0); 4872 Register obj = obj_loc.AsRegister<Register>(); 4873 Location index = locations->InAt(1); 4874 Location out_loc = locations->Out(); 4875 uint32_t data_offset = CodeGenerator::GetArrayDataOffset(instruction); 4876 Primitive::Type type = instruction->GetType(); 4877 const bool maybe_compressed_char_at = mirror::kUseStringCompression && 4878 instruction->IsStringCharAt(); 4879 HInstruction* array_instr = instruction->GetArray(); 4880 bool has_intermediate_address = array_instr->IsIntermediateAddress(); 4881 4882 switch (type) { 4883 case Primitive::kPrimBoolean: 4884 case Primitive::kPrimByte: 4885 case Primitive::kPrimShort: 4886 case Primitive::kPrimChar: 4887 case Primitive::kPrimInt: { 4888 Register length; 4889 if (maybe_compressed_char_at) { 4890 length = locations->GetTemp(0).AsRegister<Register>(); 4891 uint32_t count_offset = mirror::String::CountOffset().Uint32Value(); 4892 __ LoadFromOffset(kLoadWord, length, obj, count_offset); 4893 codegen_->MaybeRecordImplicitNullCheck(instruction); 4894 } 4895 if (index.IsConstant()) { 4896 int32_t const_index = index.GetConstant()->AsIntConstant()->GetValue(); 4897 if (maybe_compressed_char_at) { 4898 Label uncompressed_load, done; 4899 __ Lsrs(length, length, 1u); // LSRS has a 16-bit encoding, TST (immediate) does not. 4900 static_assert(static_cast<uint32_t>(mirror::StringCompressionFlag::kCompressed) == 0u, 4901 "Expecting 0=compressed, 1=uncompressed"); 4902 __ b(&uncompressed_load, CS); 4903 __ LoadFromOffset(kLoadUnsignedByte, 4904 out_loc.AsRegister<Register>(), 4905 obj, 4906 data_offset + const_index); 4907 __ b(&done); 4908 __ Bind(&uncompressed_load); 4909 __ LoadFromOffset(GetLoadOperandType(Primitive::kPrimChar), 4910 out_loc.AsRegister<Register>(), 4911 obj, 4912 data_offset + (const_index << 1)); 4913 __ Bind(&done); 4914 } else { 4915 uint32_t full_offset = data_offset + (const_index << Primitive::ComponentSizeShift(type)); 4916 4917 LoadOperandType load_type = GetLoadOperandType(type); 4918 __ LoadFromOffset(load_type, out_loc.AsRegister<Register>(), obj, full_offset); 4919 } 4920 } else { 4921 Register temp = IP; 4922 4923 if (has_intermediate_address) { 4924 // We do not need to compute the intermediate address from the array: the 4925 // input instruction has done it already. See the comment in 4926 // `TryExtractArrayAccessAddress()`. 4927 if (kIsDebugBuild) { 4928 HIntermediateAddress* tmp = array_instr->AsIntermediateAddress(); 4929 DCHECK_EQ(tmp->GetOffset()->AsIntConstant()->GetValueAsUint64(), data_offset); 4930 } 4931 temp = obj; 4932 } else { 4933 __ add(temp, obj, ShifterOperand(data_offset)); 4934 } 4935 if (maybe_compressed_char_at) { 4936 Label uncompressed_load, done; 4937 __ Lsrs(length, length, 1u); // LSRS has a 16-bit encoding, TST (immediate) does not. 4938 static_assert(static_cast<uint32_t>(mirror::StringCompressionFlag::kCompressed) == 0u, 4939 "Expecting 0=compressed, 1=uncompressed"); 4940 __ b(&uncompressed_load, CS); 4941 __ ldrb(out_loc.AsRegister<Register>(), 4942 Address(temp, index.AsRegister<Register>(), Shift::LSL, 0)); 4943 __ b(&done); 4944 __ Bind(&uncompressed_load); 4945 __ ldrh(out_loc.AsRegister<Register>(), 4946 Address(temp, index.AsRegister<Register>(), Shift::LSL, 1)); 4947 __ Bind(&done); 4948 } else { 4949 codegen_->LoadFromShiftedRegOffset(type, out_loc, temp, index.AsRegister<Register>()); 4950 } 4951 } 4952 break; 4953 } 4954 4955 case Primitive::kPrimNot: { 4956 // The read barrier instrumentation of object ArrayGet 4957 // instructions does not support the HIntermediateAddress 4958 // instruction. 4959 DCHECK(!(has_intermediate_address && kEmitCompilerReadBarrier)); 4960 4961 static_assert( 4962 sizeof(mirror::HeapReference<mirror::Object>) == sizeof(int32_t), 4963 "art::mirror::HeapReference<art::mirror::Object> and int32_t have different sizes."); 4964 // /* HeapReference<Object> */ out = 4965 // *(obj + data_offset + index * sizeof(HeapReference<Object>)) 4966 if (kEmitCompilerReadBarrier && kUseBakerReadBarrier) { 4967 Location temp = locations->GetTemp(0); 4968 // Note that a potential implicit null check is handled in this 4969 // CodeGeneratorARM::GenerateArrayLoadWithBakerReadBarrier call. 4970 codegen_->GenerateArrayLoadWithBakerReadBarrier( 4971 instruction, out_loc, obj, data_offset, index, temp, /* needs_null_check */ true); 4972 } else { 4973 Register out = out_loc.AsRegister<Register>(); 4974 if (index.IsConstant()) { 4975 size_t offset = 4976 (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4) + data_offset; 4977 __ LoadFromOffset(kLoadWord, out, obj, offset); 4978 codegen_->MaybeRecordImplicitNullCheck(instruction); 4979 // If read barriers are enabled, emit read barriers other than 4980 // Baker's using a slow path (and also unpoison the loaded 4981 // reference, if heap poisoning is enabled). 4982 codegen_->MaybeGenerateReadBarrierSlow(instruction, out_loc, out_loc, obj_loc, offset); 4983 } else { 4984 Register temp = IP; 4985 4986 if (has_intermediate_address) { 4987 // We do not need to compute the intermediate address from the array: the 4988 // input instruction has done it already. See the comment in 4989 // `TryExtractArrayAccessAddress()`. 4990 if (kIsDebugBuild) { 4991 HIntermediateAddress* tmp = array_instr->AsIntermediateAddress(); 4992 DCHECK_EQ(tmp->GetOffset()->AsIntConstant()->GetValueAsUint64(), data_offset); 4993 } 4994 temp = obj; 4995 } else { 4996 __ add(temp, obj, ShifterOperand(data_offset)); 4997 } 4998 codegen_->LoadFromShiftedRegOffset(type, out_loc, temp, index.AsRegister<Register>()); 4999 5000 codegen_->MaybeRecordImplicitNullCheck(instruction); 5001 // If read barriers are enabled, emit read barriers other than 5002 // Baker's using a slow path (and also unpoison the loaded 5003 // reference, if heap poisoning is enabled). 5004 codegen_->MaybeGenerateReadBarrierSlow( 5005 instruction, out_loc, out_loc, obj_loc, data_offset, index); 5006 } 5007 } 5008 break; 5009 } 5010 5011 case Primitive::kPrimLong: { 5012 if (index.IsConstant()) { 5013 size_t offset = 5014 (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_8) + data_offset; 5015 __ LoadFromOffset(kLoadWordPair, out_loc.AsRegisterPairLow<Register>(), obj, offset); 5016 } else { 5017 __ add(IP, obj, ShifterOperand(index.AsRegister<Register>(), LSL, TIMES_8)); 5018 __ LoadFromOffset(kLoadWordPair, out_loc.AsRegisterPairLow<Register>(), IP, data_offset); 5019 } 5020 break; 5021 } 5022 5023 case Primitive::kPrimFloat: { 5024 SRegister out = out_loc.AsFpuRegister<SRegister>(); 5025 if (index.IsConstant()) { 5026 size_t offset = (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4) + data_offset; 5027 __ LoadSFromOffset(out, obj, offset); 5028 } else { 5029 __ add(IP, obj, ShifterOperand(index.AsRegister<Register>(), LSL, TIMES_4)); 5030 __ LoadSFromOffset(out, IP, data_offset); 5031 } 5032 break; 5033 } 5034 5035 case Primitive::kPrimDouble: { 5036 SRegister out = out_loc.AsFpuRegisterPairLow<SRegister>(); 5037 if (index.IsConstant()) { 5038 size_t offset = (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_8) + data_offset; 5039 __ LoadDFromOffset(FromLowSToD(out), obj, offset); 5040 } else { 5041 __ add(IP, obj, ShifterOperand(index.AsRegister<Register>(), LSL, TIMES_8)); 5042 __ LoadDFromOffset(FromLowSToD(out), IP, data_offset); 5043 } 5044 break; 5045 } 5046 5047 case Primitive::kPrimVoid: 5048 LOG(FATAL) << "Unreachable type " << type; 5049 UNREACHABLE(); 5050 } 5051 5052 if (type == Primitive::kPrimNot) { 5053 // Potential implicit null checks, in the case of reference 5054 // arrays, are handled in the previous switch statement. 5055 } else if (!maybe_compressed_char_at) { 5056 codegen_->MaybeRecordImplicitNullCheck(instruction); 5057 } 5058} 5059 5060void LocationsBuilderARM::VisitArraySet(HArraySet* instruction) { 5061 Primitive::Type value_type = instruction->GetComponentType(); 5062 5063 bool needs_write_barrier = 5064 CodeGenerator::StoreNeedsWriteBarrier(value_type, instruction->GetValue()); 5065 bool may_need_runtime_call_for_type_check = instruction->NeedsTypeCheck(); 5066 5067 LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary( 5068 instruction, 5069 may_need_runtime_call_for_type_check ? 5070 LocationSummary::kCallOnSlowPath : 5071 LocationSummary::kNoCall); 5072 5073 locations->SetInAt(0, Location::RequiresRegister()); 5074 locations->SetInAt(1, Location::RegisterOrConstant(instruction->InputAt(1))); 5075 if (Primitive::IsFloatingPointType(value_type)) { 5076 locations->SetInAt(2, Location::RequiresFpuRegister()); 5077 } else { 5078 locations->SetInAt(2, Location::RequiresRegister()); 5079 } 5080 if (needs_write_barrier) { 5081 // Temporary registers for the write barrier. 5082 locations->AddTemp(Location::RequiresRegister()); // Possibly used for ref. poisoning too. 5083 locations->AddTemp(Location::RequiresRegister()); 5084 } 5085} 5086 5087void InstructionCodeGeneratorARM::VisitArraySet(HArraySet* instruction) { 5088 LocationSummary* locations = instruction->GetLocations(); 5089 Location array_loc = locations->InAt(0); 5090 Register array = array_loc.AsRegister<Register>(); 5091 Location index = locations->InAt(1); 5092 Primitive::Type value_type = instruction->GetComponentType(); 5093 bool may_need_runtime_call_for_type_check = instruction->NeedsTypeCheck(); 5094 bool needs_write_barrier = 5095 CodeGenerator::StoreNeedsWriteBarrier(value_type, instruction->GetValue()); 5096 uint32_t data_offset = 5097 mirror::Array::DataOffset(Primitive::ComponentSize(value_type)).Uint32Value(); 5098 Location value_loc = locations->InAt(2); 5099 HInstruction* array_instr = instruction->GetArray(); 5100 bool has_intermediate_address = array_instr->IsIntermediateAddress(); 5101 5102 switch (value_type) { 5103 case Primitive::kPrimBoolean: 5104 case Primitive::kPrimByte: 5105 case Primitive::kPrimShort: 5106 case Primitive::kPrimChar: 5107 case Primitive::kPrimInt: { 5108 if (index.IsConstant()) { 5109 int32_t const_index = index.GetConstant()->AsIntConstant()->GetValue(); 5110 uint32_t full_offset = 5111 data_offset + (const_index << Primitive::ComponentSizeShift(value_type)); 5112 StoreOperandType store_type = GetStoreOperandType(value_type); 5113 __ StoreToOffset(store_type, value_loc.AsRegister<Register>(), array, full_offset); 5114 } else { 5115 Register temp = IP; 5116 5117 if (has_intermediate_address) { 5118 // We do not need to compute the intermediate address from the array: the 5119 // input instruction has done it already. See the comment in 5120 // `TryExtractArrayAccessAddress()`. 5121 if (kIsDebugBuild) { 5122 HIntermediateAddress* tmp = array_instr->AsIntermediateAddress(); 5123 DCHECK(tmp->GetOffset()->AsIntConstant()->GetValueAsUint64() == data_offset); 5124 } 5125 temp = array; 5126 } else { 5127 __ add(temp, array, ShifterOperand(data_offset)); 5128 } 5129 codegen_->StoreToShiftedRegOffset(value_type, 5130 value_loc, 5131 temp, 5132 index.AsRegister<Register>()); 5133 } 5134 break; 5135 } 5136 5137 case Primitive::kPrimNot: { 5138 Register value = value_loc.AsRegister<Register>(); 5139 // TryExtractArrayAccessAddress optimization is never applied for non-primitive ArraySet. 5140 // See the comment in instruction_simplifier_shared.cc. 5141 DCHECK(!has_intermediate_address); 5142 5143 if (instruction->InputAt(2)->IsNullConstant()) { 5144 // Just setting null. 5145 if (index.IsConstant()) { 5146 size_t offset = 5147 (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4) + data_offset; 5148 __ StoreToOffset(kStoreWord, value, array, offset); 5149 } else { 5150 DCHECK(index.IsRegister()) << index; 5151 __ add(IP, array, ShifterOperand(data_offset)); 5152 codegen_->StoreToShiftedRegOffset(value_type, 5153 value_loc, 5154 IP, 5155 index.AsRegister<Register>()); 5156 } 5157 codegen_->MaybeRecordImplicitNullCheck(instruction); 5158 DCHECK(!needs_write_barrier); 5159 DCHECK(!may_need_runtime_call_for_type_check); 5160 break; 5161 } 5162 5163 DCHECK(needs_write_barrier); 5164 Location temp1_loc = locations->GetTemp(0); 5165 Register temp1 = temp1_loc.AsRegister<Register>(); 5166 Location temp2_loc = locations->GetTemp(1); 5167 Register temp2 = temp2_loc.AsRegister<Register>(); 5168 uint32_t class_offset = mirror::Object::ClassOffset().Int32Value(); 5169 uint32_t super_offset = mirror::Class::SuperClassOffset().Int32Value(); 5170 uint32_t component_offset = mirror::Class::ComponentTypeOffset().Int32Value(); 5171 Label done; 5172 SlowPathCodeARM* slow_path = nullptr; 5173 5174 if (may_need_runtime_call_for_type_check) { 5175 slow_path = new (GetGraph()->GetArena()) ArraySetSlowPathARM(instruction); 5176 codegen_->AddSlowPath(slow_path); 5177 if (instruction->GetValueCanBeNull()) { 5178 Label non_zero; 5179 __ CompareAndBranchIfNonZero(value, &non_zero); 5180 if (index.IsConstant()) { 5181 size_t offset = 5182 (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4) + data_offset; 5183 __ StoreToOffset(kStoreWord, value, array, offset); 5184 } else { 5185 DCHECK(index.IsRegister()) << index; 5186 __ add(IP, array, ShifterOperand(data_offset)); 5187 codegen_->StoreToShiftedRegOffset(value_type, 5188 value_loc, 5189 IP, 5190 index.AsRegister<Register>()); 5191 } 5192 codegen_->MaybeRecordImplicitNullCheck(instruction); 5193 __ b(&done); 5194 __ Bind(&non_zero); 5195 } 5196 5197 // Note that when read barriers are enabled, the type checks 5198 // are performed without read barriers. This is fine, even in 5199 // the case where a class object is in the from-space after 5200 // the flip, as a comparison involving such a type would not 5201 // produce a false positive; it may of course produce a false 5202 // negative, in which case we would take the ArraySet slow 5203 // path. 5204 5205 // /* HeapReference<Class> */ temp1 = array->klass_ 5206 __ LoadFromOffset(kLoadWord, temp1, array, class_offset); 5207 codegen_->MaybeRecordImplicitNullCheck(instruction); 5208 __ MaybeUnpoisonHeapReference(temp1); 5209 5210 // /* HeapReference<Class> */ temp1 = temp1->component_type_ 5211 __ LoadFromOffset(kLoadWord, temp1, temp1, component_offset); 5212 // /* HeapReference<Class> */ temp2 = value->klass_ 5213 __ LoadFromOffset(kLoadWord, temp2, value, class_offset); 5214 // If heap poisoning is enabled, no need to unpoison `temp1` 5215 // nor `temp2`, as we are comparing two poisoned references. 5216 __ cmp(temp1, ShifterOperand(temp2)); 5217 5218 if (instruction->StaticTypeOfArrayIsObjectArray()) { 5219 Label do_put; 5220 __ b(&do_put, EQ); 5221 // If heap poisoning is enabled, the `temp1` reference has 5222 // not been unpoisoned yet; unpoison it now. 5223 __ MaybeUnpoisonHeapReference(temp1); 5224 5225 // /* HeapReference<Class> */ temp1 = temp1->super_class_ 5226 __ LoadFromOffset(kLoadWord, temp1, temp1, super_offset); 5227 // If heap poisoning is enabled, no need to unpoison 5228 // `temp1`, as we are comparing against null below. 5229 __ CompareAndBranchIfNonZero(temp1, slow_path->GetEntryLabel()); 5230 __ Bind(&do_put); 5231 } else { 5232 __ b(slow_path->GetEntryLabel(), NE); 5233 } 5234 } 5235 5236 Register source = value; 5237 if (kPoisonHeapReferences) { 5238 // Note that in the case where `value` is a null reference, 5239 // we do not enter this block, as a null reference does not 5240 // need poisoning. 5241 DCHECK_EQ(value_type, Primitive::kPrimNot); 5242 __ Mov(temp1, value); 5243 __ PoisonHeapReference(temp1); 5244 source = temp1; 5245 } 5246 5247 if (index.IsConstant()) { 5248 size_t offset = 5249 (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4) + data_offset; 5250 __ StoreToOffset(kStoreWord, source, array, offset); 5251 } else { 5252 DCHECK(index.IsRegister()) << index; 5253 5254 __ add(IP, array, ShifterOperand(data_offset)); 5255 codegen_->StoreToShiftedRegOffset(value_type, 5256 Location::RegisterLocation(source), 5257 IP, 5258 index.AsRegister<Register>()); 5259 } 5260 5261 if (!may_need_runtime_call_for_type_check) { 5262 codegen_->MaybeRecordImplicitNullCheck(instruction); 5263 } 5264 5265 codegen_->MarkGCCard(temp1, temp2, array, value, instruction->GetValueCanBeNull()); 5266 5267 if (done.IsLinked()) { 5268 __ Bind(&done); 5269 } 5270 5271 if (slow_path != nullptr) { 5272 __ Bind(slow_path->GetExitLabel()); 5273 } 5274 5275 break; 5276 } 5277 5278 case Primitive::kPrimLong: { 5279 Location value = locations->InAt(2); 5280 if (index.IsConstant()) { 5281 size_t offset = 5282 (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_8) + data_offset; 5283 __ StoreToOffset(kStoreWordPair, value.AsRegisterPairLow<Register>(), array, offset); 5284 } else { 5285 __ add(IP, array, ShifterOperand(index.AsRegister<Register>(), LSL, TIMES_8)); 5286 __ StoreToOffset(kStoreWordPair, value.AsRegisterPairLow<Register>(), IP, data_offset); 5287 } 5288 break; 5289 } 5290 5291 case Primitive::kPrimFloat: { 5292 Location value = locations->InAt(2); 5293 DCHECK(value.IsFpuRegister()); 5294 if (index.IsConstant()) { 5295 size_t offset = (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4) + data_offset; 5296 __ StoreSToOffset(value.AsFpuRegister<SRegister>(), array, offset); 5297 } else { 5298 __ add(IP, array, ShifterOperand(index.AsRegister<Register>(), LSL, TIMES_4)); 5299 __ StoreSToOffset(value.AsFpuRegister<SRegister>(), IP, data_offset); 5300 } 5301 break; 5302 } 5303 5304 case Primitive::kPrimDouble: { 5305 Location value = locations->InAt(2); 5306 DCHECK(value.IsFpuRegisterPair()); 5307 if (index.IsConstant()) { 5308 size_t offset = (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_8) + data_offset; 5309 __ StoreDToOffset(FromLowSToD(value.AsFpuRegisterPairLow<SRegister>()), array, offset); 5310 } else { 5311 __ add(IP, array, ShifterOperand(index.AsRegister<Register>(), LSL, TIMES_8)); 5312 __ StoreDToOffset(FromLowSToD(value.AsFpuRegisterPairLow<SRegister>()), IP, data_offset); 5313 } 5314 5315 break; 5316 } 5317 5318 case Primitive::kPrimVoid: 5319 LOG(FATAL) << "Unreachable type " << value_type; 5320 UNREACHABLE(); 5321 } 5322 5323 // Objects are handled in the switch. 5324 if (value_type != Primitive::kPrimNot) { 5325 codegen_->MaybeRecordImplicitNullCheck(instruction); 5326 } 5327} 5328 5329void LocationsBuilderARM::VisitArrayLength(HArrayLength* instruction) { 5330 LocationSummary* locations = 5331 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall); 5332 locations->SetInAt(0, Location::RequiresRegister()); 5333 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 5334} 5335 5336void InstructionCodeGeneratorARM::VisitArrayLength(HArrayLength* instruction) { 5337 LocationSummary* locations = instruction->GetLocations(); 5338 uint32_t offset = CodeGenerator::GetArrayLengthOffset(instruction); 5339 Register obj = locations->InAt(0).AsRegister<Register>(); 5340 Register out = locations->Out().AsRegister<Register>(); 5341 __ LoadFromOffset(kLoadWord, out, obj, offset); 5342 codegen_->MaybeRecordImplicitNullCheck(instruction); 5343 // Mask out compression flag from String's array length. 5344 if (mirror::kUseStringCompression && instruction->IsStringLength()) { 5345 __ Lsr(out, out, 1u); 5346 } 5347} 5348 5349void LocationsBuilderARM::VisitIntermediateAddress(HIntermediateAddress* instruction) { 5350 LocationSummary* locations = 5351 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall); 5352 5353 locations->SetInAt(0, Location::RequiresRegister()); 5354 locations->SetInAt(1, Location::RegisterOrConstant(instruction->GetOffset())); 5355 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 5356} 5357 5358void InstructionCodeGeneratorARM::VisitIntermediateAddress(HIntermediateAddress* instruction) { 5359 LocationSummary* locations = instruction->GetLocations(); 5360 Location out = locations->Out(); 5361 Location first = locations->InAt(0); 5362 Location second = locations->InAt(1); 5363 5364 if (second.IsRegister()) { 5365 __ add(out.AsRegister<Register>(), 5366 first.AsRegister<Register>(), 5367 ShifterOperand(second.AsRegister<Register>())); 5368 } else { 5369 __ AddConstant(out.AsRegister<Register>(), 5370 first.AsRegister<Register>(), 5371 second.GetConstant()->AsIntConstant()->GetValue()); 5372 } 5373} 5374 5375void LocationsBuilderARM::VisitBoundsCheck(HBoundsCheck* instruction) { 5376 RegisterSet caller_saves = RegisterSet::Empty(); 5377 InvokeRuntimeCallingConvention calling_convention; 5378 caller_saves.Add(Location::RegisterLocation(calling_convention.GetRegisterAt(0))); 5379 caller_saves.Add(Location::RegisterLocation(calling_convention.GetRegisterAt(1))); 5380 LocationSummary* locations = codegen_->CreateThrowingSlowPathLocations(instruction, caller_saves); 5381 locations->SetInAt(0, Location::RequiresRegister()); 5382 locations->SetInAt(1, Location::RequiresRegister()); 5383} 5384 5385void InstructionCodeGeneratorARM::VisitBoundsCheck(HBoundsCheck* instruction) { 5386 LocationSummary* locations = instruction->GetLocations(); 5387 SlowPathCodeARM* slow_path = 5388 new (GetGraph()->GetArena()) BoundsCheckSlowPathARM(instruction); 5389 codegen_->AddSlowPath(slow_path); 5390 5391 Register index = locations->InAt(0).AsRegister<Register>(); 5392 Register length = locations->InAt(1).AsRegister<Register>(); 5393 5394 __ cmp(index, ShifterOperand(length)); 5395 __ b(slow_path->GetEntryLabel(), HS); 5396} 5397 5398void CodeGeneratorARM::MarkGCCard(Register temp, 5399 Register card, 5400 Register object, 5401 Register value, 5402 bool can_be_null) { 5403 Label is_null; 5404 if (can_be_null) { 5405 __ CompareAndBranchIfZero(value, &is_null); 5406 } 5407 __ LoadFromOffset(kLoadWord, card, TR, Thread::CardTableOffset<kArmPointerSize>().Int32Value()); 5408 __ Lsr(temp, object, gc::accounting::CardTable::kCardShift); 5409 __ strb(card, Address(card, temp)); 5410 if (can_be_null) { 5411 __ Bind(&is_null); 5412 } 5413} 5414 5415void LocationsBuilderARM::VisitParallelMove(HParallelMove* instruction ATTRIBUTE_UNUSED) { 5416 LOG(FATAL) << "Unreachable"; 5417} 5418 5419void InstructionCodeGeneratorARM::VisitParallelMove(HParallelMove* instruction) { 5420 codegen_->GetMoveResolver()->EmitNativeCode(instruction); 5421} 5422 5423void LocationsBuilderARM::VisitSuspendCheck(HSuspendCheck* instruction) { 5424 LocationSummary* locations = 5425 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCallOnSlowPath); 5426 locations->SetCustomSlowPathCallerSaves(RegisterSet::Empty()); // No caller-save registers. 5427} 5428 5429void InstructionCodeGeneratorARM::VisitSuspendCheck(HSuspendCheck* instruction) { 5430 HBasicBlock* block = instruction->GetBlock(); 5431 if (block->GetLoopInformation() != nullptr) { 5432 DCHECK(block->GetLoopInformation()->GetSuspendCheck() == instruction); 5433 // The back edge will generate the suspend check. 5434 return; 5435 } 5436 if (block->IsEntryBlock() && instruction->GetNext()->IsGoto()) { 5437 // The goto will generate the suspend check. 5438 return; 5439 } 5440 GenerateSuspendCheck(instruction, nullptr); 5441} 5442 5443void InstructionCodeGeneratorARM::GenerateSuspendCheck(HSuspendCheck* instruction, 5444 HBasicBlock* successor) { 5445 SuspendCheckSlowPathARM* slow_path = 5446 down_cast<SuspendCheckSlowPathARM*>(instruction->GetSlowPath()); 5447 if (slow_path == nullptr) { 5448 slow_path = new (GetGraph()->GetArena()) SuspendCheckSlowPathARM(instruction, successor); 5449 instruction->SetSlowPath(slow_path); 5450 codegen_->AddSlowPath(slow_path); 5451 if (successor != nullptr) { 5452 DCHECK(successor->IsLoopHeader()); 5453 codegen_->ClearSpillSlotsFromLoopPhisInStackMap(instruction); 5454 } 5455 } else { 5456 DCHECK_EQ(slow_path->GetSuccessor(), successor); 5457 } 5458 5459 __ LoadFromOffset( 5460 kLoadUnsignedHalfword, IP, TR, Thread::ThreadFlagsOffset<kArmPointerSize>().Int32Value()); 5461 if (successor == nullptr) { 5462 __ CompareAndBranchIfNonZero(IP, slow_path->GetEntryLabel()); 5463 __ Bind(slow_path->GetReturnLabel()); 5464 } else { 5465 __ CompareAndBranchIfZero(IP, codegen_->GetLabelOf(successor)); 5466 __ b(slow_path->GetEntryLabel()); 5467 } 5468} 5469 5470ArmAssembler* ParallelMoveResolverARM::GetAssembler() const { 5471 return codegen_->GetAssembler(); 5472} 5473 5474void ParallelMoveResolverARM::EmitMove(size_t index) { 5475 MoveOperands* move = moves_[index]; 5476 Location source = move->GetSource(); 5477 Location destination = move->GetDestination(); 5478 5479 if (source.IsRegister()) { 5480 if (destination.IsRegister()) { 5481 __ Mov(destination.AsRegister<Register>(), source.AsRegister<Register>()); 5482 } else if (destination.IsFpuRegister()) { 5483 __ vmovsr(destination.AsFpuRegister<SRegister>(), source.AsRegister<Register>()); 5484 } else { 5485 DCHECK(destination.IsStackSlot()); 5486 __ StoreToOffset(kStoreWord, source.AsRegister<Register>(), 5487 SP, destination.GetStackIndex()); 5488 } 5489 } else if (source.IsStackSlot()) { 5490 if (destination.IsRegister()) { 5491 __ LoadFromOffset(kLoadWord, destination.AsRegister<Register>(), 5492 SP, source.GetStackIndex()); 5493 } else if (destination.IsFpuRegister()) { 5494 __ LoadSFromOffset(destination.AsFpuRegister<SRegister>(), SP, source.GetStackIndex()); 5495 } else { 5496 DCHECK(destination.IsStackSlot()); 5497 __ LoadFromOffset(kLoadWord, IP, SP, source.GetStackIndex()); 5498 __ StoreToOffset(kStoreWord, IP, SP, destination.GetStackIndex()); 5499 } 5500 } else if (source.IsFpuRegister()) { 5501 if (destination.IsRegister()) { 5502 __ vmovrs(destination.AsRegister<Register>(), source.AsFpuRegister<SRegister>()); 5503 } else if (destination.IsFpuRegister()) { 5504 __ vmovs(destination.AsFpuRegister<SRegister>(), source.AsFpuRegister<SRegister>()); 5505 } else { 5506 DCHECK(destination.IsStackSlot()); 5507 __ StoreSToOffset(source.AsFpuRegister<SRegister>(), SP, destination.GetStackIndex()); 5508 } 5509 } else if (source.IsDoubleStackSlot()) { 5510 if (destination.IsDoubleStackSlot()) { 5511 __ LoadDFromOffset(DTMP, SP, source.GetStackIndex()); 5512 __ StoreDToOffset(DTMP, SP, destination.GetStackIndex()); 5513 } else if (destination.IsRegisterPair()) { 5514 DCHECK(ExpectedPairLayout(destination)); 5515 __ LoadFromOffset( 5516 kLoadWordPair, destination.AsRegisterPairLow<Register>(), SP, source.GetStackIndex()); 5517 } else { 5518 DCHECK(destination.IsFpuRegisterPair()) << destination; 5519 __ LoadDFromOffset(FromLowSToD(destination.AsFpuRegisterPairLow<SRegister>()), 5520 SP, 5521 source.GetStackIndex()); 5522 } 5523 } else if (source.IsRegisterPair()) { 5524 if (destination.IsRegisterPair()) { 5525 __ Mov(destination.AsRegisterPairLow<Register>(), source.AsRegisterPairLow<Register>()); 5526 __ Mov(destination.AsRegisterPairHigh<Register>(), source.AsRegisterPairHigh<Register>()); 5527 } else if (destination.IsFpuRegisterPair()) { 5528 __ vmovdrr(FromLowSToD(destination.AsFpuRegisterPairLow<SRegister>()), 5529 source.AsRegisterPairLow<Register>(), 5530 source.AsRegisterPairHigh<Register>()); 5531 } else { 5532 DCHECK(destination.IsDoubleStackSlot()) << destination; 5533 DCHECK(ExpectedPairLayout(source)); 5534 __ StoreToOffset( 5535 kStoreWordPair, source.AsRegisterPairLow<Register>(), SP, destination.GetStackIndex()); 5536 } 5537 } else if (source.IsFpuRegisterPair()) { 5538 if (destination.IsRegisterPair()) { 5539 __ vmovrrd(destination.AsRegisterPairLow<Register>(), 5540 destination.AsRegisterPairHigh<Register>(), 5541 FromLowSToD(source.AsFpuRegisterPairLow<SRegister>())); 5542 } else if (destination.IsFpuRegisterPair()) { 5543 __ vmovd(FromLowSToD(destination.AsFpuRegisterPairLow<SRegister>()), 5544 FromLowSToD(source.AsFpuRegisterPairLow<SRegister>())); 5545 } else { 5546 DCHECK(destination.IsDoubleStackSlot()) << destination; 5547 __ StoreDToOffset(FromLowSToD(source.AsFpuRegisterPairLow<SRegister>()), 5548 SP, 5549 destination.GetStackIndex()); 5550 } 5551 } else { 5552 DCHECK(source.IsConstant()) << source; 5553 HConstant* constant = source.GetConstant(); 5554 if (constant->IsIntConstant() || constant->IsNullConstant()) { 5555 int32_t value = CodeGenerator::GetInt32ValueOf(constant); 5556 if (destination.IsRegister()) { 5557 __ LoadImmediate(destination.AsRegister<Register>(), value); 5558 } else { 5559 DCHECK(destination.IsStackSlot()); 5560 __ LoadImmediate(IP, value); 5561 __ StoreToOffset(kStoreWord, IP, SP, destination.GetStackIndex()); 5562 } 5563 } else if (constant->IsLongConstant()) { 5564 int64_t value = constant->AsLongConstant()->GetValue(); 5565 if (destination.IsRegisterPair()) { 5566 __ LoadImmediate(destination.AsRegisterPairLow<Register>(), Low32Bits(value)); 5567 __ LoadImmediate(destination.AsRegisterPairHigh<Register>(), High32Bits(value)); 5568 } else { 5569 DCHECK(destination.IsDoubleStackSlot()) << destination; 5570 __ LoadImmediate(IP, Low32Bits(value)); 5571 __ StoreToOffset(kStoreWord, IP, SP, destination.GetStackIndex()); 5572 __ LoadImmediate(IP, High32Bits(value)); 5573 __ StoreToOffset(kStoreWord, IP, SP, destination.GetHighStackIndex(kArmWordSize)); 5574 } 5575 } else if (constant->IsDoubleConstant()) { 5576 double value = constant->AsDoubleConstant()->GetValue(); 5577 if (destination.IsFpuRegisterPair()) { 5578 __ LoadDImmediate(FromLowSToD(destination.AsFpuRegisterPairLow<SRegister>()), value); 5579 } else { 5580 DCHECK(destination.IsDoubleStackSlot()) << destination; 5581 uint64_t int_value = bit_cast<uint64_t, double>(value); 5582 __ LoadImmediate(IP, Low32Bits(int_value)); 5583 __ StoreToOffset(kStoreWord, IP, SP, destination.GetStackIndex()); 5584 __ LoadImmediate(IP, High32Bits(int_value)); 5585 __ StoreToOffset(kStoreWord, IP, SP, destination.GetHighStackIndex(kArmWordSize)); 5586 } 5587 } else { 5588 DCHECK(constant->IsFloatConstant()) << constant->DebugName(); 5589 float value = constant->AsFloatConstant()->GetValue(); 5590 if (destination.IsFpuRegister()) { 5591 __ LoadSImmediate(destination.AsFpuRegister<SRegister>(), value); 5592 } else { 5593 DCHECK(destination.IsStackSlot()); 5594 __ LoadImmediate(IP, bit_cast<int32_t, float>(value)); 5595 __ StoreToOffset(kStoreWord, IP, SP, destination.GetStackIndex()); 5596 } 5597 } 5598 } 5599} 5600 5601void ParallelMoveResolverARM::Exchange(Register reg, int mem) { 5602 __ Mov(IP, reg); 5603 __ LoadFromOffset(kLoadWord, reg, SP, mem); 5604 __ StoreToOffset(kStoreWord, IP, SP, mem); 5605} 5606 5607void ParallelMoveResolverARM::Exchange(int mem1, int mem2) { 5608 ScratchRegisterScope ensure_scratch(this, IP, R0, codegen_->GetNumberOfCoreRegisters()); 5609 int stack_offset = ensure_scratch.IsSpilled() ? kArmWordSize : 0; 5610 __ LoadFromOffset(kLoadWord, static_cast<Register>(ensure_scratch.GetRegister()), 5611 SP, mem1 + stack_offset); 5612 __ LoadFromOffset(kLoadWord, IP, SP, mem2 + stack_offset); 5613 __ StoreToOffset(kStoreWord, static_cast<Register>(ensure_scratch.GetRegister()), 5614 SP, mem2 + stack_offset); 5615 __ StoreToOffset(kStoreWord, IP, SP, mem1 + stack_offset); 5616} 5617 5618void ParallelMoveResolverARM::EmitSwap(size_t index) { 5619 MoveOperands* move = moves_[index]; 5620 Location source = move->GetSource(); 5621 Location destination = move->GetDestination(); 5622 5623 if (source.IsRegister() && destination.IsRegister()) { 5624 DCHECK_NE(source.AsRegister<Register>(), IP); 5625 DCHECK_NE(destination.AsRegister<Register>(), IP); 5626 __ Mov(IP, source.AsRegister<Register>()); 5627 __ Mov(source.AsRegister<Register>(), destination.AsRegister<Register>()); 5628 __ Mov(destination.AsRegister<Register>(), IP); 5629 } else if (source.IsRegister() && destination.IsStackSlot()) { 5630 Exchange(source.AsRegister<Register>(), destination.GetStackIndex()); 5631 } else if (source.IsStackSlot() && destination.IsRegister()) { 5632 Exchange(destination.AsRegister<Register>(), source.GetStackIndex()); 5633 } else if (source.IsStackSlot() && destination.IsStackSlot()) { 5634 Exchange(source.GetStackIndex(), destination.GetStackIndex()); 5635 } else if (source.IsFpuRegister() && destination.IsFpuRegister()) { 5636 __ vmovrs(IP, source.AsFpuRegister<SRegister>()); 5637 __ vmovs(source.AsFpuRegister<SRegister>(), destination.AsFpuRegister<SRegister>()); 5638 __ vmovsr(destination.AsFpuRegister<SRegister>(), IP); 5639 } else if (source.IsRegisterPair() && destination.IsRegisterPair()) { 5640 __ vmovdrr(DTMP, source.AsRegisterPairLow<Register>(), source.AsRegisterPairHigh<Register>()); 5641 __ Mov(source.AsRegisterPairLow<Register>(), destination.AsRegisterPairLow<Register>()); 5642 __ Mov(source.AsRegisterPairHigh<Register>(), destination.AsRegisterPairHigh<Register>()); 5643 __ vmovrrd(destination.AsRegisterPairLow<Register>(), 5644 destination.AsRegisterPairHigh<Register>(), 5645 DTMP); 5646 } else if (source.IsRegisterPair() || destination.IsRegisterPair()) { 5647 Register low_reg = source.IsRegisterPair() 5648 ? source.AsRegisterPairLow<Register>() 5649 : destination.AsRegisterPairLow<Register>(); 5650 int mem = source.IsRegisterPair() 5651 ? destination.GetStackIndex() 5652 : source.GetStackIndex(); 5653 DCHECK(ExpectedPairLayout(source.IsRegisterPair() ? source : destination)); 5654 __ vmovdrr(DTMP, low_reg, static_cast<Register>(low_reg + 1)); 5655 __ LoadFromOffset(kLoadWordPair, low_reg, SP, mem); 5656 __ StoreDToOffset(DTMP, SP, mem); 5657 } else if (source.IsFpuRegisterPair() && destination.IsFpuRegisterPair()) { 5658 DRegister first = FromLowSToD(source.AsFpuRegisterPairLow<SRegister>()); 5659 DRegister second = FromLowSToD(destination.AsFpuRegisterPairLow<SRegister>()); 5660 __ vmovd(DTMP, first); 5661 __ vmovd(first, second); 5662 __ vmovd(second, DTMP); 5663 } else if (source.IsFpuRegisterPair() || destination.IsFpuRegisterPair()) { 5664 DRegister reg = source.IsFpuRegisterPair() 5665 ? FromLowSToD(source.AsFpuRegisterPairLow<SRegister>()) 5666 : FromLowSToD(destination.AsFpuRegisterPairLow<SRegister>()); 5667 int mem = source.IsFpuRegisterPair() 5668 ? destination.GetStackIndex() 5669 : source.GetStackIndex(); 5670 __ vmovd(DTMP, reg); 5671 __ LoadDFromOffset(reg, SP, mem); 5672 __ StoreDToOffset(DTMP, SP, mem); 5673 } else if (source.IsFpuRegister() || destination.IsFpuRegister()) { 5674 SRegister reg = source.IsFpuRegister() ? source.AsFpuRegister<SRegister>() 5675 : destination.AsFpuRegister<SRegister>(); 5676 int mem = source.IsFpuRegister() 5677 ? destination.GetStackIndex() 5678 : source.GetStackIndex(); 5679 5680 __ vmovrs(IP, reg); 5681 __ LoadSFromOffset(reg, SP, mem); 5682 __ StoreToOffset(kStoreWord, IP, SP, mem); 5683 } else if (source.IsDoubleStackSlot() && destination.IsDoubleStackSlot()) { 5684 Exchange(source.GetStackIndex(), destination.GetStackIndex()); 5685 Exchange(source.GetHighStackIndex(kArmWordSize), destination.GetHighStackIndex(kArmWordSize)); 5686 } else { 5687 LOG(FATAL) << "Unimplemented" << source << " <-> " << destination; 5688 } 5689} 5690 5691void ParallelMoveResolverARM::SpillScratch(int reg) { 5692 __ Push(static_cast<Register>(reg)); 5693} 5694 5695void ParallelMoveResolverARM::RestoreScratch(int reg) { 5696 __ Pop(static_cast<Register>(reg)); 5697} 5698 5699HLoadClass::LoadKind CodeGeneratorARM::GetSupportedLoadClassKind( 5700 HLoadClass::LoadKind desired_class_load_kind) { 5701 switch (desired_class_load_kind) { 5702 case HLoadClass::LoadKind::kReferrersClass: 5703 break; 5704 case HLoadClass::LoadKind::kBootImageLinkTimeAddress: 5705 DCHECK(!GetCompilerOptions().GetCompilePic()); 5706 break; 5707 case HLoadClass::LoadKind::kBootImageLinkTimePcRelative: 5708 DCHECK(GetCompilerOptions().GetCompilePic()); 5709 break; 5710 case HLoadClass::LoadKind::kBootImageAddress: 5711 break; 5712 case HLoadClass::LoadKind::kJitTableAddress: 5713 break; 5714 case HLoadClass::LoadKind::kDexCachePcRelative: 5715 DCHECK(!Runtime::Current()->UseJitCompilation()); 5716 // We disable pc-relative load when there is an irreducible loop, as the optimization 5717 // is incompatible with it. 5718 // TODO: Create as many ArmDexCacheArraysBase instructions as needed for methods 5719 // with irreducible loops. 5720 if (GetGraph()->HasIrreducibleLoops()) { 5721 return HLoadClass::LoadKind::kDexCacheViaMethod; 5722 } 5723 break; 5724 case HLoadClass::LoadKind::kDexCacheViaMethod: 5725 break; 5726 } 5727 return desired_class_load_kind; 5728} 5729 5730void LocationsBuilderARM::VisitLoadClass(HLoadClass* cls) { 5731 if (cls->NeedsAccessCheck()) { 5732 InvokeRuntimeCallingConvention calling_convention; 5733 CodeGenerator::CreateLoadClassLocationSummary( 5734 cls, 5735 Location::RegisterLocation(calling_convention.GetRegisterAt(0)), 5736 Location::RegisterLocation(R0), 5737 /* code_generator_supports_read_barrier */ true); 5738 return; 5739 } 5740 5741 const bool requires_read_barrier = kEmitCompilerReadBarrier && !cls->IsInBootImage(); 5742 LocationSummary::CallKind call_kind = (cls->NeedsEnvironment() || requires_read_barrier) 5743 ? LocationSummary::kCallOnSlowPath 5744 : LocationSummary::kNoCall; 5745 LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(cls, call_kind); 5746 if (kUseBakerReadBarrier && requires_read_barrier && !cls->NeedsEnvironment()) { 5747 locations->SetCustomSlowPathCallerSaves(RegisterSet::Empty()); // No caller-save registers. 5748 } 5749 5750 HLoadClass::LoadKind load_kind = cls->GetLoadKind(); 5751 if (load_kind == HLoadClass::LoadKind::kReferrersClass || 5752 load_kind == HLoadClass::LoadKind::kDexCacheViaMethod || 5753 load_kind == HLoadClass::LoadKind::kDexCachePcRelative) { 5754 locations->SetInAt(0, Location::RequiresRegister()); 5755 } 5756 locations->SetOut(Location::RequiresRegister()); 5757} 5758 5759void InstructionCodeGeneratorARM::VisitLoadClass(HLoadClass* cls) { 5760 LocationSummary* locations = cls->GetLocations(); 5761 if (cls->NeedsAccessCheck()) { 5762 codegen_->MoveConstant(locations->GetTemp(0), cls->GetTypeIndex().index_); 5763 codegen_->InvokeRuntime(kQuickInitializeTypeAndVerifyAccess, cls, cls->GetDexPc()); 5764 CheckEntrypointTypes<kQuickInitializeTypeAndVerifyAccess, void*, uint32_t>(); 5765 return; 5766 } 5767 5768 Location out_loc = locations->Out(); 5769 Register out = out_loc.AsRegister<Register>(); 5770 5771 const ReadBarrierOption read_barrier_option = cls->IsInBootImage() 5772 ? kWithoutReadBarrier 5773 : kCompilerReadBarrierOption; 5774 bool generate_null_check = false; 5775 switch (cls->GetLoadKind()) { 5776 case HLoadClass::LoadKind::kReferrersClass: { 5777 DCHECK(!cls->CanCallRuntime()); 5778 DCHECK(!cls->MustGenerateClinitCheck()); 5779 // /* GcRoot<mirror::Class> */ out = current_method->declaring_class_ 5780 Register current_method = locations->InAt(0).AsRegister<Register>(); 5781 GenerateGcRootFieldLoad(cls, 5782 out_loc, 5783 current_method, 5784 ArtMethod::DeclaringClassOffset().Int32Value(), 5785 read_barrier_option); 5786 break; 5787 } 5788 case HLoadClass::LoadKind::kBootImageLinkTimeAddress: { 5789 DCHECK_EQ(read_barrier_option, kWithoutReadBarrier); 5790 __ LoadLiteral(out, codegen_->DeduplicateBootImageTypeLiteral(cls->GetDexFile(), 5791 cls->GetTypeIndex())); 5792 break; 5793 } 5794 case HLoadClass::LoadKind::kBootImageLinkTimePcRelative: { 5795 DCHECK_EQ(read_barrier_option, kWithoutReadBarrier); 5796 CodeGeneratorARM::PcRelativePatchInfo* labels = 5797 codegen_->NewPcRelativeTypePatch(cls->GetDexFile(), cls->GetTypeIndex()); 5798 __ BindTrackedLabel(&labels->movw_label); 5799 __ movw(out, /* placeholder */ 0u); 5800 __ BindTrackedLabel(&labels->movt_label); 5801 __ movt(out, /* placeholder */ 0u); 5802 __ BindTrackedLabel(&labels->add_pc_label); 5803 __ add(out, out, ShifterOperand(PC)); 5804 break; 5805 } 5806 case HLoadClass::LoadKind::kBootImageAddress: { 5807 DCHECK_EQ(read_barrier_option, kWithoutReadBarrier); 5808 DCHECK_NE(cls->GetAddress(), 0u); 5809 uint32_t address = dchecked_integral_cast<uint32_t>(cls->GetAddress()); 5810 __ LoadLiteral(out, codegen_->DeduplicateBootImageAddressLiteral(address)); 5811 break; 5812 } 5813 case HLoadClass::LoadKind::kJitTableAddress: { 5814 __ LoadLiteral(out, codegen_->DeduplicateJitClassLiteral(cls->GetDexFile(), 5815 cls->GetTypeIndex(), 5816 cls->GetAddress())); 5817 // /* GcRoot<mirror::Class> */ out = *out 5818 GenerateGcRootFieldLoad(cls, out_loc, out, /* offset */ 0, kCompilerReadBarrierOption); 5819 break; 5820 } 5821 case HLoadClass::LoadKind::kDexCachePcRelative: { 5822 Register base_reg = locations->InAt(0).AsRegister<Register>(); 5823 HArmDexCacheArraysBase* base = cls->InputAt(0)->AsArmDexCacheArraysBase(); 5824 int32_t offset = cls->GetDexCacheElementOffset() - base->GetElementOffset(); 5825 // /* GcRoot<mirror::Class> */ out = *(dex_cache_arrays_base + offset) 5826 GenerateGcRootFieldLoad(cls, out_loc, base_reg, offset, read_barrier_option); 5827 generate_null_check = !cls->IsInDexCache(); 5828 break; 5829 } 5830 case HLoadClass::LoadKind::kDexCacheViaMethod: { 5831 // /* GcRoot<mirror::Class>[] */ out = 5832 // current_method.ptr_sized_fields_->dex_cache_resolved_types_ 5833 Register current_method = locations->InAt(0).AsRegister<Register>(); 5834 __ LoadFromOffset(kLoadWord, 5835 out, 5836 current_method, 5837 ArtMethod::DexCacheResolvedTypesOffset(kArmPointerSize).Int32Value()); 5838 // /* GcRoot<mirror::Class> */ out = out[type_index] 5839 size_t offset = CodeGenerator::GetCacheOffset(cls->GetTypeIndex().index_); 5840 GenerateGcRootFieldLoad(cls, out_loc, out, offset, read_barrier_option); 5841 generate_null_check = !cls->IsInDexCache(); 5842 } 5843 } 5844 5845 if (generate_null_check || cls->MustGenerateClinitCheck()) { 5846 DCHECK(cls->CanCallRuntime()); 5847 SlowPathCodeARM* slow_path = new (GetGraph()->GetArena()) LoadClassSlowPathARM( 5848 cls, cls, cls->GetDexPc(), cls->MustGenerateClinitCheck()); 5849 codegen_->AddSlowPath(slow_path); 5850 if (generate_null_check) { 5851 __ CompareAndBranchIfZero(out, slow_path->GetEntryLabel()); 5852 } 5853 if (cls->MustGenerateClinitCheck()) { 5854 GenerateClassInitializationCheck(slow_path, out); 5855 } else { 5856 __ Bind(slow_path->GetExitLabel()); 5857 } 5858 } 5859} 5860 5861void LocationsBuilderARM::VisitClinitCheck(HClinitCheck* check) { 5862 LocationSummary* locations = 5863 new (GetGraph()->GetArena()) LocationSummary(check, LocationSummary::kCallOnSlowPath); 5864 locations->SetInAt(0, Location::RequiresRegister()); 5865 if (check->HasUses()) { 5866 locations->SetOut(Location::SameAsFirstInput()); 5867 } 5868} 5869 5870void InstructionCodeGeneratorARM::VisitClinitCheck(HClinitCheck* check) { 5871 // We assume the class is not null. 5872 SlowPathCodeARM* slow_path = new (GetGraph()->GetArena()) LoadClassSlowPathARM( 5873 check->GetLoadClass(), check, check->GetDexPc(), true); 5874 codegen_->AddSlowPath(slow_path); 5875 GenerateClassInitializationCheck(slow_path, 5876 check->GetLocations()->InAt(0).AsRegister<Register>()); 5877} 5878 5879void InstructionCodeGeneratorARM::GenerateClassInitializationCheck( 5880 SlowPathCodeARM* slow_path, Register class_reg) { 5881 __ LoadFromOffset(kLoadWord, IP, class_reg, mirror::Class::StatusOffset().Int32Value()); 5882 __ cmp(IP, ShifterOperand(mirror::Class::kStatusInitialized)); 5883 __ b(slow_path->GetEntryLabel(), LT); 5884 // Even if the initialized flag is set, we may be in a situation where caches are not synced 5885 // properly. Therefore, we do a memory fence. 5886 __ dmb(ISH); 5887 __ Bind(slow_path->GetExitLabel()); 5888} 5889 5890HLoadString::LoadKind CodeGeneratorARM::GetSupportedLoadStringKind( 5891 HLoadString::LoadKind desired_string_load_kind) { 5892 switch (desired_string_load_kind) { 5893 case HLoadString::LoadKind::kBootImageLinkTimeAddress: 5894 DCHECK(!GetCompilerOptions().GetCompilePic()); 5895 break; 5896 case HLoadString::LoadKind::kBootImageLinkTimePcRelative: 5897 DCHECK(GetCompilerOptions().GetCompilePic()); 5898 break; 5899 case HLoadString::LoadKind::kBootImageAddress: 5900 break; 5901 case HLoadString::LoadKind::kBssEntry: 5902 DCHECK(!Runtime::Current()->UseJitCompilation()); 5903 break; 5904 case HLoadString::LoadKind::kJitTableAddress: 5905 DCHECK(Runtime::Current()->UseJitCompilation()); 5906 break; 5907 case HLoadString::LoadKind::kDexCacheViaMethod: 5908 break; 5909 } 5910 return desired_string_load_kind; 5911} 5912 5913void LocationsBuilderARM::VisitLoadString(HLoadString* load) { 5914 LocationSummary::CallKind call_kind = CodeGenerator::GetLoadStringCallKind(load); 5915 LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(load, call_kind); 5916 HLoadString::LoadKind load_kind = load->GetLoadKind(); 5917 if (load_kind == HLoadString::LoadKind::kDexCacheViaMethod) { 5918 locations->SetOut(Location::RegisterLocation(R0)); 5919 } else { 5920 locations->SetOut(Location::RequiresRegister()); 5921 if (load_kind == HLoadString::LoadKind::kBssEntry) { 5922 if (!kUseReadBarrier || kUseBakerReadBarrier) { 5923 // Rely on the pResolveString and/or marking to save everything, including temps. 5924 // Note that IP may theoretically be clobbered by saving/restoring the live register 5925 // (only one thanks to the custom calling convention), so we request a different temp. 5926 locations->AddTemp(Location::RequiresRegister()); 5927 RegisterSet caller_saves = RegisterSet::Empty(); 5928 InvokeRuntimeCallingConvention calling_convention; 5929 caller_saves.Add(Location::RegisterLocation(calling_convention.GetRegisterAt(0))); 5930 // TODO: Add GetReturnLocation() to the calling convention so that we can DCHECK() 5931 // that the the kPrimNot result register is the same as the first argument register. 5932 locations->SetCustomSlowPathCallerSaves(caller_saves); 5933 } else { 5934 // For non-Baker read barrier we have a temp-clobbering call. 5935 } 5936 } 5937 } 5938} 5939 5940void InstructionCodeGeneratorARM::VisitLoadString(HLoadString* load) { 5941 LocationSummary* locations = load->GetLocations(); 5942 Location out_loc = locations->Out(); 5943 Register out = out_loc.AsRegister<Register>(); 5944 HLoadString::LoadKind load_kind = load->GetLoadKind(); 5945 5946 switch (load_kind) { 5947 case HLoadString::LoadKind::kBootImageLinkTimeAddress: { 5948 __ LoadLiteral(out, codegen_->DeduplicateBootImageStringLiteral(load->GetDexFile(), 5949 load->GetStringIndex())); 5950 return; // No dex cache slow path. 5951 } 5952 case HLoadString::LoadKind::kBootImageLinkTimePcRelative: { 5953 DCHECK(codegen_->GetCompilerOptions().IsBootImage()); 5954 CodeGeneratorARM::PcRelativePatchInfo* labels = 5955 codegen_->NewPcRelativeStringPatch(load->GetDexFile(), load->GetStringIndex().index_); 5956 __ BindTrackedLabel(&labels->movw_label); 5957 __ movw(out, /* placeholder */ 0u); 5958 __ BindTrackedLabel(&labels->movt_label); 5959 __ movt(out, /* placeholder */ 0u); 5960 __ BindTrackedLabel(&labels->add_pc_label); 5961 __ add(out, out, ShifterOperand(PC)); 5962 return; // No dex cache slow path. 5963 } 5964 case HLoadString::LoadKind::kBootImageAddress: { 5965 DCHECK_NE(load->GetAddress(), 0u); 5966 uint32_t address = dchecked_integral_cast<uint32_t>(load->GetAddress()); 5967 __ LoadLiteral(out, codegen_->DeduplicateBootImageAddressLiteral(address)); 5968 return; // No dex cache slow path. 5969 } 5970 case HLoadString::LoadKind::kBssEntry: { 5971 DCHECK(!codegen_->GetCompilerOptions().IsBootImage()); 5972 Register temp = locations->GetTemp(0).AsRegister<Register>(); 5973 CodeGeneratorARM::PcRelativePatchInfo* labels = 5974 codegen_->NewPcRelativeStringPatch(load->GetDexFile(), load->GetStringIndex().index_); 5975 __ BindTrackedLabel(&labels->movw_label); 5976 __ movw(temp, /* placeholder */ 0u); 5977 __ BindTrackedLabel(&labels->movt_label); 5978 __ movt(temp, /* placeholder */ 0u); 5979 __ BindTrackedLabel(&labels->add_pc_label); 5980 __ add(temp, temp, ShifterOperand(PC)); 5981 GenerateGcRootFieldLoad(load, out_loc, temp, /* offset */ 0, kCompilerReadBarrierOption); 5982 SlowPathCode* slow_path = new (GetGraph()->GetArena()) LoadStringSlowPathARM(load); 5983 codegen_->AddSlowPath(slow_path); 5984 __ CompareAndBranchIfZero(out, slow_path->GetEntryLabel()); 5985 __ Bind(slow_path->GetExitLabel()); 5986 return; 5987 } 5988 case HLoadString::LoadKind::kJitTableAddress: { 5989 __ LoadLiteral(out, codegen_->DeduplicateJitStringLiteral(load->GetDexFile(), 5990 load->GetStringIndex())); 5991 // /* GcRoot<mirror::String> */ out = *out 5992 GenerateGcRootFieldLoad(load, out_loc, out, /* offset */ 0, kCompilerReadBarrierOption); 5993 return; 5994 } 5995 default: 5996 break; 5997 } 5998 5999 // TODO: Consider re-adding the compiler code to do string dex cache lookup again. 6000 DCHECK(load_kind == HLoadString::LoadKind::kDexCacheViaMethod); 6001 InvokeRuntimeCallingConvention calling_convention; 6002 DCHECK_EQ(calling_convention.GetRegisterAt(0), out); 6003 __ LoadImmediate(calling_convention.GetRegisterAt(0), load->GetStringIndex().index_); 6004 codegen_->InvokeRuntime(kQuickResolveString, load, load->GetDexPc()); 6005 CheckEntrypointTypes<kQuickResolveString, void*, uint32_t>(); 6006} 6007 6008static int32_t GetExceptionTlsOffset() { 6009 return Thread::ExceptionOffset<kArmPointerSize>().Int32Value(); 6010} 6011 6012void LocationsBuilderARM::VisitLoadException(HLoadException* load) { 6013 LocationSummary* locations = 6014 new (GetGraph()->GetArena()) LocationSummary(load, LocationSummary::kNoCall); 6015 locations->SetOut(Location::RequiresRegister()); 6016} 6017 6018void InstructionCodeGeneratorARM::VisitLoadException(HLoadException* load) { 6019 Register out = load->GetLocations()->Out().AsRegister<Register>(); 6020 __ LoadFromOffset(kLoadWord, out, TR, GetExceptionTlsOffset()); 6021} 6022 6023void LocationsBuilderARM::VisitClearException(HClearException* clear) { 6024 new (GetGraph()->GetArena()) LocationSummary(clear, LocationSummary::kNoCall); 6025} 6026 6027void InstructionCodeGeneratorARM::VisitClearException(HClearException* clear ATTRIBUTE_UNUSED) { 6028 __ LoadImmediate(IP, 0); 6029 __ StoreToOffset(kStoreWord, IP, TR, GetExceptionTlsOffset()); 6030} 6031 6032void LocationsBuilderARM::VisitThrow(HThrow* instruction) { 6033 LocationSummary* locations = 6034 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCallOnMainOnly); 6035 InvokeRuntimeCallingConvention calling_convention; 6036 locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(0))); 6037} 6038 6039void InstructionCodeGeneratorARM::VisitThrow(HThrow* instruction) { 6040 codegen_->InvokeRuntime(kQuickDeliverException, instruction, instruction->GetDexPc()); 6041 CheckEntrypointTypes<kQuickDeliverException, void, mirror::Object*>(); 6042} 6043 6044// Temp is used for read barrier. 6045static size_t NumberOfInstanceOfTemps(TypeCheckKind type_check_kind) { 6046 if (kEmitCompilerReadBarrier && 6047 (kUseBakerReadBarrier || 6048 type_check_kind == TypeCheckKind::kAbstractClassCheck || 6049 type_check_kind == TypeCheckKind::kClassHierarchyCheck || 6050 type_check_kind == TypeCheckKind::kArrayObjectCheck)) { 6051 return 1; 6052 } 6053 return 0; 6054} 6055 6056// Interface case has 3 temps, one for holding the number of interfaces, one for the current 6057// interface pointer, one for loading the current interface. 6058// The other checks have one temp for loading the object's class. 6059static size_t NumberOfCheckCastTemps(TypeCheckKind type_check_kind) { 6060 if (type_check_kind == TypeCheckKind::kInterfaceCheck) { 6061 return 3; 6062 } 6063 return 1 + NumberOfInstanceOfTemps(type_check_kind); 6064} 6065 6066void LocationsBuilderARM::VisitInstanceOf(HInstanceOf* instruction) { 6067 LocationSummary::CallKind call_kind = LocationSummary::kNoCall; 6068 TypeCheckKind type_check_kind = instruction->GetTypeCheckKind(); 6069 bool baker_read_barrier_slow_path = false; 6070 switch (type_check_kind) { 6071 case TypeCheckKind::kExactCheck: 6072 case TypeCheckKind::kAbstractClassCheck: 6073 case TypeCheckKind::kClassHierarchyCheck: 6074 case TypeCheckKind::kArrayObjectCheck: 6075 call_kind = 6076 kEmitCompilerReadBarrier ? LocationSummary::kCallOnSlowPath : LocationSummary::kNoCall; 6077 baker_read_barrier_slow_path = kUseBakerReadBarrier; 6078 break; 6079 case TypeCheckKind::kArrayCheck: 6080 case TypeCheckKind::kUnresolvedCheck: 6081 case TypeCheckKind::kInterfaceCheck: 6082 call_kind = LocationSummary::kCallOnSlowPath; 6083 break; 6084 } 6085 6086 LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction, call_kind); 6087 if (baker_read_barrier_slow_path) { 6088 locations->SetCustomSlowPathCallerSaves(RegisterSet::Empty()); // No caller-save registers. 6089 } 6090 locations->SetInAt(0, Location::RequiresRegister()); 6091 locations->SetInAt(1, Location::RequiresRegister()); 6092 // The "out" register is used as a temporary, so it overlaps with the inputs. 6093 // Note that TypeCheckSlowPathARM uses this register too. 6094 locations->SetOut(Location::RequiresRegister(), Location::kOutputOverlap); 6095 locations->AddRegisterTemps(NumberOfInstanceOfTemps(type_check_kind)); 6096} 6097 6098void InstructionCodeGeneratorARM::VisitInstanceOf(HInstanceOf* instruction) { 6099 TypeCheckKind type_check_kind = instruction->GetTypeCheckKind(); 6100 LocationSummary* locations = instruction->GetLocations(); 6101 Location obj_loc = locations->InAt(0); 6102 Register obj = obj_loc.AsRegister<Register>(); 6103 Register cls = locations->InAt(1).AsRegister<Register>(); 6104 Location out_loc = locations->Out(); 6105 Register out = out_loc.AsRegister<Register>(); 6106 const size_t num_temps = NumberOfInstanceOfTemps(type_check_kind); 6107 DCHECK_LE(num_temps, 1u); 6108 Location maybe_temp_loc = (num_temps >= 1) ? locations->GetTemp(0) : Location::NoLocation(); 6109 uint32_t class_offset = mirror::Object::ClassOffset().Int32Value(); 6110 uint32_t super_offset = mirror::Class::SuperClassOffset().Int32Value(); 6111 uint32_t component_offset = mirror::Class::ComponentTypeOffset().Int32Value(); 6112 uint32_t primitive_offset = mirror::Class::PrimitiveTypeOffset().Int32Value(); 6113 Label done, zero; 6114 SlowPathCodeARM* slow_path = nullptr; 6115 6116 // Return 0 if `obj` is null. 6117 // avoid null check if we know obj is not null. 6118 if (instruction->MustDoNullCheck()) { 6119 __ CompareAndBranchIfZero(obj, &zero); 6120 } 6121 6122 switch (type_check_kind) { 6123 case TypeCheckKind::kExactCheck: { 6124 // /* HeapReference<Class> */ out = obj->klass_ 6125 GenerateReferenceLoadTwoRegisters(instruction, 6126 out_loc, 6127 obj_loc, 6128 class_offset, 6129 maybe_temp_loc, 6130 kCompilerReadBarrierOption); 6131 __ cmp(out, ShifterOperand(cls)); 6132 // Classes must be equal for the instanceof to succeed. 6133 __ b(&zero, NE); 6134 __ LoadImmediate(out, 1); 6135 __ b(&done); 6136 break; 6137 } 6138 6139 case TypeCheckKind::kAbstractClassCheck: { 6140 // /* HeapReference<Class> */ out = obj->klass_ 6141 GenerateReferenceLoadTwoRegisters(instruction, 6142 out_loc, 6143 obj_loc, 6144 class_offset, 6145 maybe_temp_loc, 6146 kCompilerReadBarrierOption); 6147 // If the class is abstract, we eagerly fetch the super class of the 6148 // object to avoid doing a comparison we know will fail. 6149 Label loop; 6150 __ Bind(&loop); 6151 // /* HeapReference<Class> */ out = out->super_class_ 6152 GenerateReferenceLoadOneRegister(instruction, 6153 out_loc, 6154 super_offset, 6155 maybe_temp_loc, 6156 kCompilerReadBarrierOption); 6157 // If `out` is null, we use it for the result, and jump to `done`. 6158 __ CompareAndBranchIfZero(out, &done); 6159 __ cmp(out, ShifterOperand(cls)); 6160 __ b(&loop, NE); 6161 __ LoadImmediate(out, 1); 6162 if (zero.IsLinked()) { 6163 __ b(&done); 6164 } 6165 break; 6166 } 6167 6168 case TypeCheckKind::kClassHierarchyCheck: { 6169 // /* HeapReference<Class> */ out = obj->klass_ 6170 GenerateReferenceLoadTwoRegisters(instruction, 6171 out_loc, 6172 obj_loc, 6173 class_offset, 6174 maybe_temp_loc, 6175 kCompilerReadBarrierOption); 6176 // Walk over the class hierarchy to find a match. 6177 Label loop, success; 6178 __ Bind(&loop); 6179 __ cmp(out, ShifterOperand(cls)); 6180 __ b(&success, EQ); 6181 // /* HeapReference<Class> */ out = out->super_class_ 6182 GenerateReferenceLoadOneRegister(instruction, 6183 out_loc, 6184 super_offset, 6185 maybe_temp_loc, 6186 kCompilerReadBarrierOption); 6187 __ CompareAndBranchIfNonZero(out, &loop); 6188 // If `out` is null, we use it for the result, and jump to `done`. 6189 __ b(&done); 6190 __ Bind(&success); 6191 __ LoadImmediate(out, 1); 6192 if (zero.IsLinked()) { 6193 __ b(&done); 6194 } 6195 break; 6196 } 6197 6198 case TypeCheckKind::kArrayObjectCheck: { 6199 // /* HeapReference<Class> */ out = obj->klass_ 6200 GenerateReferenceLoadTwoRegisters(instruction, 6201 out_loc, 6202 obj_loc, 6203 class_offset, 6204 maybe_temp_loc, 6205 kCompilerReadBarrierOption); 6206 // Do an exact check. 6207 Label exact_check; 6208 __ cmp(out, ShifterOperand(cls)); 6209 __ b(&exact_check, EQ); 6210 // Otherwise, we need to check that the object's class is a non-primitive array. 6211 // /* HeapReference<Class> */ out = out->component_type_ 6212 GenerateReferenceLoadOneRegister(instruction, 6213 out_loc, 6214 component_offset, 6215 maybe_temp_loc, 6216 kCompilerReadBarrierOption); 6217 // If `out` is null, we use it for the result, and jump to `done`. 6218 __ CompareAndBranchIfZero(out, &done); 6219 __ LoadFromOffset(kLoadUnsignedHalfword, out, out, primitive_offset); 6220 static_assert(Primitive::kPrimNot == 0, "Expected 0 for kPrimNot"); 6221 __ CompareAndBranchIfNonZero(out, &zero); 6222 __ Bind(&exact_check); 6223 __ LoadImmediate(out, 1); 6224 __ b(&done); 6225 break; 6226 } 6227 6228 case TypeCheckKind::kArrayCheck: { 6229 // No read barrier since the slow path will retry upon failure. 6230 // /* HeapReference<Class> */ out = obj->klass_ 6231 GenerateReferenceLoadTwoRegisters(instruction, 6232 out_loc, 6233 obj_loc, 6234 class_offset, 6235 maybe_temp_loc, 6236 kWithoutReadBarrier); 6237 __ cmp(out, ShifterOperand(cls)); 6238 DCHECK(locations->OnlyCallsOnSlowPath()); 6239 slow_path = new (GetGraph()->GetArena()) TypeCheckSlowPathARM(instruction, 6240 /* is_fatal */ false); 6241 codegen_->AddSlowPath(slow_path); 6242 __ b(slow_path->GetEntryLabel(), NE); 6243 __ LoadImmediate(out, 1); 6244 if (zero.IsLinked()) { 6245 __ b(&done); 6246 } 6247 break; 6248 } 6249 6250 case TypeCheckKind::kUnresolvedCheck: 6251 case TypeCheckKind::kInterfaceCheck: { 6252 // Note that we indeed only call on slow path, but we always go 6253 // into the slow path for the unresolved and interface check 6254 // cases. 6255 // 6256 // We cannot directly call the InstanceofNonTrivial runtime 6257 // entry point without resorting to a type checking slow path 6258 // here (i.e. by calling InvokeRuntime directly), as it would 6259 // require to assign fixed registers for the inputs of this 6260 // HInstanceOf instruction (following the runtime calling 6261 // convention), which might be cluttered by the potential first 6262 // read barrier emission at the beginning of this method. 6263 // 6264 // TODO: Introduce a new runtime entry point taking the object 6265 // to test (instead of its class) as argument, and let it deal 6266 // with the read barrier issues. This will let us refactor this 6267 // case of the `switch` code as it was previously (with a direct 6268 // call to the runtime not using a type checking slow path). 6269 // This should also be beneficial for the other cases above. 6270 DCHECK(locations->OnlyCallsOnSlowPath()); 6271 slow_path = new (GetGraph()->GetArena()) TypeCheckSlowPathARM(instruction, 6272 /* is_fatal */ false); 6273 codegen_->AddSlowPath(slow_path); 6274 __ b(slow_path->GetEntryLabel()); 6275 if (zero.IsLinked()) { 6276 __ b(&done); 6277 } 6278 break; 6279 } 6280 } 6281 6282 if (zero.IsLinked()) { 6283 __ Bind(&zero); 6284 __ LoadImmediate(out, 0); 6285 } 6286 6287 if (done.IsLinked()) { 6288 __ Bind(&done); 6289 } 6290 6291 if (slow_path != nullptr) { 6292 __ Bind(slow_path->GetExitLabel()); 6293 } 6294} 6295 6296void LocationsBuilderARM::VisitCheckCast(HCheckCast* instruction) { 6297 LocationSummary::CallKind call_kind = LocationSummary::kNoCall; 6298 bool throws_into_catch = instruction->CanThrowIntoCatchBlock(); 6299 6300 TypeCheckKind type_check_kind = instruction->GetTypeCheckKind(); 6301 switch (type_check_kind) { 6302 case TypeCheckKind::kExactCheck: 6303 case TypeCheckKind::kAbstractClassCheck: 6304 case TypeCheckKind::kClassHierarchyCheck: 6305 case TypeCheckKind::kArrayObjectCheck: 6306 call_kind = (throws_into_catch || kEmitCompilerReadBarrier) ? 6307 LocationSummary::kCallOnSlowPath : 6308 LocationSummary::kNoCall; // In fact, call on a fatal (non-returning) slow path. 6309 break; 6310 case TypeCheckKind::kArrayCheck: 6311 case TypeCheckKind::kUnresolvedCheck: 6312 case TypeCheckKind::kInterfaceCheck: 6313 call_kind = LocationSummary::kCallOnSlowPath; 6314 break; 6315 } 6316 6317 LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction, call_kind); 6318 locations->SetInAt(0, Location::RequiresRegister()); 6319 locations->SetInAt(1, Location::RequiresRegister()); 6320 locations->AddRegisterTemps(NumberOfCheckCastTemps(type_check_kind)); 6321} 6322 6323void InstructionCodeGeneratorARM::VisitCheckCast(HCheckCast* instruction) { 6324 TypeCheckKind type_check_kind = instruction->GetTypeCheckKind(); 6325 LocationSummary* locations = instruction->GetLocations(); 6326 Location obj_loc = locations->InAt(0); 6327 Register obj = obj_loc.AsRegister<Register>(); 6328 Register cls = locations->InAt(1).AsRegister<Register>(); 6329 Location temp_loc = locations->GetTemp(0); 6330 Register temp = temp_loc.AsRegister<Register>(); 6331 const size_t num_temps = NumberOfCheckCastTemps(type_check_kind); 6332 DCHECK_LE(num_temps, 3u); 6333 Location maybe_temp2_loc = (num_temps >= 2) ? locations->GetTemp(1) : Location::NoLocation(); 6334 Location maybe_temp3_loc = (num_temps >= 3) ? locations->GetTemp(2) : Location::NoLocation(); 6335 const uint32_t class_offset = mirror::Object::ClassOffset().Int32Value(); 6336 const uint32_t super_offset = mirror::Class::SuperClassOffset().Int32Value(); 6337 const uint32_t component_offset = mirror::Class::ComponentTypeOffset().Int32Value(); 6338 const uint32_t primitive_offset = mirror::Class::PrimitiveTypeOffset().Int32Value(); 6339 const uint32_t iftable_offset = mirror::Class::IfTableOffset().Uint32Value(); 6340 const uint32_t array_length_offset = mirror::Array::LengthOffset().Uint32Value(); 6341 const uint32_t object_array_data_offset = 6342 mirror::Array::DataOffset(kHeapReferenceSize).Uint32Value(); 6343 6344 // Always false for read barriers since we may need to go to the entrypoint for non-fatal cases 6345 // from false negatives. The false negatives may come from avoiding read barriers below. Avoiding 6346 // read barriers is done for performance and code size reasons. 6347 bool is_type_check_slow_path_fatal = false; 6348 if (!kEmitCompilerReadBarrier) { 6349 is_type_check_slow_path_fatal = 6350 (type_check_kind == TypeCheckKind::kExactCheck || 6351 type_check_kind == TypeCheckKind::kAbstractClassCheck || 6352 type_check_kind == TypeCheckKind::kClassHierarchyCheck || 6353 type_check_kind == TypeCheckKind::kArrayObjectCheck) && 6354 !instruction->CanThrowIntoCatchBlock(); 6355 } 6356 SlowPathCodeARM* type_check_slow_path = 6357 new (GetGraph()->GetArena()) TypeCheckSlowPathARM(instruction, 6358 is_type_check_slow_path_fatal); 6359 codegen_->AddSlowPath(type_check_slow_path); 6360 6361 Label done; 6362 // Avoid null check if we know obj is not null. 6363 if (instruction->MustDoNullCheck()) { 6364 __ CompareAndBranchIfZero(obj, &done); 6365 } 6366 6367 switch (type_check_kind) { 6368 case TypeCheckKind::kExactCheck: 6369 case TypeCheckKind::kArrayCheck: { 6370 // /* HeapReference<Class> */ temp = obj->klass_ 6371 GenerateReferenceLoadTwoRegisters(instruction, 6372 temp_loc, 6373 obj_loc, 6374 class_offset, 6375 maybe_temp2_loc, 6376 kWithoutReadBarrier); 6377 6378 __ cmp(temp, ShifterOperand(cls)); 6379 // Jump to slow path for throwing the exception or doing a 6380 // more involved array check. 6381 __ b(type_check_slow_path->GetEntryLabel(), NE); 6382 break; 6383 } 6384 6385 case TypeCheckKind::kAbstractClassCheck: { 6386 // /* HeapReference<Class> */ temp = obj->klass_ 6387 GenerateReferenceLoadTwoRegisters(instruction, 6388 temp_loc, 6389 obj_loc, 6390 class_offset, 6391 maybe_temp2_loc, 6392 kWithoutReadBarrier); 6393 6394 // If the class is abstract, we eagerly fetch the super class of the 6395 // object to avoid doing a comparison we know will fail. 6396 Label loop; 6397 __ Bind(&loop); 6398 // /* HeapReference<Class> */ temp = temp->super_class_ 6399 GenerateReferenceLoadOneRegister(instruction, 6400 temp_loc, 6401 super_offset, 6402 maybe_temp2_loc, 6403 kWithoutReadBarrier); 6404 6405 // If the class reference currently in `temp` is null, jump to the slow path to throw the 6406 // exception. 6407 __ CompareAndBranchIfZero(temp, type_check_slow_path->GetEntryLabel()); 6408 6409 // Otherwise, compare the classes. 6410 __ cmp(temp, ShifterOperand(cls)); 6411 __ b(&loop, NE); 6412 break; 6413 } 6414 6415 case TypeCheckKind::kClassHierarchyCheck: { 6416 // /* HeapReference<Class> */ temp = obj->klass_ 6417 GenerateReferenceLoadTwoRegisters(instruction, 6418 temp_loc, 6419 obj_loc, 6420 class_offset, 6421 maybe_temp2_loc, 6422 kWithoutReadBarrier); 6423 6424 // Walk over the class hierarchy to find a match. 6425 Label loop; 6426 __ Bind(&loop); 6427 __ cmp(temp, ShifterOperand(cls)); 6428 __ b(&done, EQ); 6429 6430 // /* HeapReference<Class> */ temp = temp->super_class_ 6431 GenerateReferenceLoadOneRegister(instruction, 6432 temp_loc, 6433 super_offset, 6434 maybe_temp2_loc, 6435 kWithoutReadBarrier); 6436 6437 // If the class reference currently in `temp` is null, jump to the slow path to throw the 6438 // exception. 6439 __ CompareAndBranchIfZero(temp, type_check_slow_path->GetEntryLabel()); 6440 // Otherwise, jump to the beginning of the loop. 6441 __ b(&loop); 6442 break; 6443 } 6444 6445 case TypeCheckKind::kArrayObjectCheck: { 6446 // /* HeapReference<Class> */ temp = obj->klass_ 6447 GenerateReferenceLoadTwoRegisters(instruction, 6448 temp_loc, 6449 obj_loc, 6450 class_offset, 6451 maybe_temp2_loc, 6452 kWithoutReadBarrier); 6453 6454 // Do an exact check. 6455 __ cmp(temp, ShifterOperand(cls)); 6456 __ b(&done, EQ); 6457 6458 // Otherwise, we need to check that the object's class is a non-primitive array. 6459 // /* HeapReference<Class> */ temp = temp->component_type_ 6460 GenerateReferenceLoadOneRegister(instruction, 6461 temp_loc, 6462 component_offset, 6463 maybe_temp2_loc, 6464 kWithoutReadBarrier); 6465 // If the component type is null, jump to the slow path to throw the exception. 6466 __ CompareAndBranchIfZero(temp, type_check_slow_path->GetEntryLabel()); 6467 // Otherwise,the object is indeed an array, jump to label `check_non_primitive_component_type` 6468 // to further check that this component type is not a primitive type. 6469 __ LoadFromOffset(kLoadUnsignedHalfword, temp, temp, primitive_offset); 6470 static_assert(Primitive::kPrimNot == 0, "Expected 0 for art::Primitive::kPrimNot"); 6471 __ CompareAndBranchIfNonZero(temp, type_check_slow_path->GetEntryLabel()); 6472 break; 6473 } 6474 6475 case TypeCheckKind::kUnresolvedCheck: 6476 // We always go into the type check slow path for the unresolved check case. 6477 // We cannot directly call the CheckCast runtime entry point 6478 // without resorting to a type checking slow path here (i.e. by 6479 // calling InvokeRuntime directly), as it would require to 6480 // assign fixed registers for the inputs of this HInstanceOf 6481 // instruction (following the runtime calling convention), which 6482 // might be cluttered by the potential first read barrier 6483 // emission at the beginning of this method. 6484 6485 __ b(type_check_slow_path->GetEntryLabel()); 6486 break; 6487 6488 case TypeCheckKind::kInterfaceCheck: { 6489 // Avoid read barriers to improve performance of the fast path. We can not get false 6490 // positives by doing this. 6491 // /* HeapReference<Class> */ temp = obj->klass_ 6492 GenerateReferenceLoadTwoRegisters(instruction, 6493 temp_loc, 6494 obj_loc, 6495 class_offset, 6496 maybe_temp2_loc, 6497 kWithoutReadBarrier); 6498 6499 // /* HeapReference<Class> */ temp = temp->iftable_ 6500 GenerateReferenceLoadTwoRegisters(instruction, 6501 temp_loc, 6502 temp_loc, 6503 iftable_offset, 6504 maybe_temp2_loc, 6505 kWithoutReadBarrier); 6506 // Iftable is never null. 6507 __ ldr(maybe_temp2_loc.AsRegister<Register>(), Address(temp, array_length_offset)); 6508 // Loop through the iftable and check if any class matches. 6509 Label start_loop; 6510 __ Bind(&start_loop); 6511 __ CompareAndBranchIfZero(maybe_temp2_loc.AsRegister<Register>(), 6512 type_check_slow_path->GetEntryLabel()); 6513 __ ldr(maybe_temp3_loc.AsRegister<Register>(), Address(temp, object_array_data_offset)); 6514 __ MaybeUnpoisonHeapReference(maybe_temp3_loc.AsRegister<Register>()); 6515 // Go to next interface. 6516 __ add(temp, temp, ShifterOperand(2 * kHeapReferenceSize)); 6517 __ sub(maybe_temp2_loc.AsRegister<Register>(), 6518 maybe_temp2_loc.AsRegister<Register>(), 6519 ShifterOperand(2)); 6520 // Compare the classes and continue the loop if they do not match. 6521 __ cmp(cls, ShifterOperand(maybe_temp3_loc.AsRegister<Register>())); 6522 __ b(&start_loop, NE); 6523 break; 6524 } 6525 } 6526 __ Bind(&done); 6527 6528 __ Bind(type_check_slow_path->GetExitLabel()); 6529} 6530 6531void LocationsBuilderARM::VisitMonitorOperation(HMonitorOperation* instruction) { 6532 LocationSummary* locations = 6533 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCallOnMainOnly); 6534 InvokeRuntimeCallingConvention calling_convention; 6535 locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(0))); 6536} 6537 6538void InstructionCodeGeneratorARM::VisitMonitorOperation(HMonitorOperation* instruction) { 6539 codegen_->InvokeRuntime(instruction->IsEnter() ? kQuickLockObject : kQuickUnlockObject, 6540 instruction, 6541 instruction->GetDexPc()); 6542 if (instruction->IsEnter()) { 6543 CheckEntrypointTypes<kQuickLockObject, void, mirror::Object*>(); 6544 } else { 6545 CheckEntrypointTypes<kQuickUnlockObject, void, mirror::Object*>(); 6546 } 6547} 6548 6549void LocationsBuilderARM::VisitAnd(HAnd* instruction) { HandleBitwiseOperation(instruction, AND); } 6550void LocationsBuilderARM::VisitOr(HOr* instruction) { HandleBitwiseOperation(instruction, ORR); } 6551void LocationsBuilderARM::VisitXor(HXor* instruction) { HandleBitwiseOperation(instruction, EOR); } 6552 6553void LocationsBuilderARM::HandleBitwiseOperation(HBinaryOperation* instruction, Opcode opcode) { 6554 LocationSummary* locations = 6555 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall); 6556 DCHECK(instruction->GetResultType() == Primitive::kPrimInt 6557 || instruction->GetResultType() == Primitive::kPrimLong); 6558 // Note: GVN reorders commutative operations to have the constant on the right hand side. 6559 locations->SetInAt(0, Location::RequiresRegister()); 6560 locations->SetInAt(1, ArmEncodableConstantOrRegister(instruction->InputAt(1), opcode)); 6561 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 6562} 6563 6564void InstructionCodeGeneratorARM::VisitAnd(HAnd* instruction) { 6565 HandleBitwiseOperation(instruction); 6566} 6567 6568void InstructionCodeGeneratorARM::VisitOr(HOr* instruction) { 6569 HandleBitwiseOperation(instruction); 6570} 6571 6572void InstructionCodeGeneratorARM::VisitXor(HXor* instruction) { 6573 HandleBitwiseOperation(instruction); 6574} 6575 6576 6577void LocationsBuilderARM::VisitBitwiseNegatedRight(HBitwiseNegatedRight* instruction) { 6578 LocationSummary* locations = 6579 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall); 6580 DCHECK(instruction->GetResultType() == Primitive::kPrimInt 6581 || instruction->GetResultType() == Primitive::kPrimLong); 6582 6583 locations->SetInAt(0, Location::RequiresRegister()); 6584 locations->SetInAt(1, Location::RequiresRegister()); 6585 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 6586} 6587 6588void InstructionCodeGeneratorARM::VisitBitwiseNegatedRight(HBitwiseNegatedRight* instruction) { 6589 LocationSummary* locations = instruction->GetLocations(); 6590 Location first = locations->InAt(0); 6591 Location second = locations->InAt(1); 6592 Location out = locations->Out(); 6593 6594 if (instruction->GetResultType() == Primitive::kPrimInt) { 6595 Register first_reg = first.AsRegister<Register>(); 6596 ShifterOperand second_reg(second.AsRegister<Register>()); 6597 Register out_reg = out.AsRegister<Register>(); 6598 6599 switch (instruction->GetOpKind()) { 6600 case HInstruction::kAnd: 6601 __ bic(out_reg, first_reg, second_reg); 6602 break; 6603 case HInstruction::kOr: 6604 __ orn(out_reg, first_reg, second_reg); 6605 break; 6606 // There is no EON on arm. 6607 case HInstruction::kXor: 6608 default: 6609 LOG(FATAL) << "Unexpected instruction " << instruction->DebugName(); 6610 UNREACHABLE(); 6611 } 6612 return; 6613 6614 } else { 6615 DCHECK_EQ(instruction->GetResultType(), Primitive::kPrimLong); 6616 Register first_low = first.AsRegisterPairLow<Register>(); 6617 Register first_high = first.AsRegisterPairHigh<Register>(); 6618 ShifterOperand second_low(second.AsRegisterPairLow<Register>()); 6619 ShifterOperand second_high(second.AsRegisterPairHigh<Register>()); 6620 Register out_low = out.AsRegisterPairLow<Register>(); 6621 Register out_high = out.AsRegisterPairHigh<Register>(); 6622 6623 switch (instruction->GetOpKind()) { 6624 case HInstruction::kAnd: 6625 __ bic(out_low, first_low, second_low); 6626 __ bic(out_high, first_high, second_high); 6627 break; 6628 case HInstruction::kOr: 6629 __ orn(out_low, first_low, second_low); 6630 __ orn(out_high, first_high, second_high); 6631 break; 6632 // There is no EON on arm. 6633 case HInstruction::kXor: 6634 default: 6635 LOG(FATAL) << "Unexpected instruction " << instruction->DebugName(); 6636 UNREACHABLE(); 6637 } 6638 } 6639} 6640 6641void InstructionCodeGeneratorARM::GenerateAndConst(Register out, Register first, uint32_t value) { 6642 // Optimize special cases for individual halfs of `and-long` (`and` is simplified earlier). 6643 if (value == 0xffffffffu) { 6644 if (out != first) { 6645 __ mov(out, ShifterOperand(first)); 6646 } 6647 return; 6648 } 6649 if (value == 0u) { 6650 __ mov(out, ShifterOperand(0)); 6651 return; 6652 } 6653 ShifterOperand so; 6654 if (__ ShifterOperandCanHold(kNoRegister, kNoRegister, AND, value, &so)) { 6655 __ and_(out, first, so); 6656 } else { 6657 DCHECK(__ ShifterOperandCanHold(kNoRegister, kNoRegister, BIC, ~value, &so)); 6658 __ bic(out, first, ShifterOperand(~value)); 6659 } 6660} 6661 6662void InstructionCodeGeneratorARM::GenerateOrrConst(Register out, Register first, uint32_t value) { 6663 // Optimize special cases for individual halfs of `or-long` (`or` is simplified earlier). 6664 if (value == 0u) { 6665 if (out != first) { 6666 __ mov(out, ShifterOperand(first)); 6667 } 6668 return; 6669 } 6670 if (value == 0xffffffffu) { 6671 __ mvn(out, ShifterOperand(0)); 6672 return; 6673 } 6674 ShifterOperand so; 6675 if (__ ShifterOperandCanHold(kNoRegister, kNoRegister, ORR, value, &so)) { 6676 __ orr(out, first, so); 6677 } else { 6678 DCHECK(__ ShifterOperandCanHold(kNoRegister, kNoRegister, ORN, ~value, &so)); 6679 __ orn(out, first, ShifterOperand(~value)); 6680 } 6681} 6682 6683void InstructionCodeGeneratorARM::GenerateEorConst(Register out, Register first, uint32_t value) { 6684 // Optimize special case for individual halfs of `xor-long` (`xor` is simplified earlier). 6685 if (value == 0u) { 6686 if (out != first) { 6687 __ mov(out, ShifterOperand(first)); 6688 } 6689 return; 6690 } 6691 __ eor(out, first, ShifterOperand(value)); 6692} 6693 6694void InstructionCodeGeneratorARM::GenerateAddLongConst(Location out, 6695 Location first, 6696 uint64_t value) { 6697 Register out_low = out.AsRegisterPairLow<Register>(); 6698 Register out_high = out.AsRegisterPairHigh<Register>(); 6699 Register first_low = first.AsRegisterPairLow<Register>(); 6700 Register first_high = first.AsRegisterPairHigh<Register>(); 6701 uint32_t value_low = Low32Bits(value); 6702 uint32_t value_high = High32Bits(value); 6703 if (value_low == 0u) { 6704 if (out_low != first_low) { 6705 __ mov(out_low, ShifterOperand(first_low)); 6706 } 6707 __ AddConstant(out_high, first_high, value_high); 6708 return; 6709 } 6710 __ AddConstantSetFlags(out_low, first_low, value_low); 6711 ShifterOperand so; 6712 if (__ ShifterOperandCanHold(out_high, first_high, ADC, value_high, kCcDontCare, &so)) { 6713 __ adc(out_high, first_high, so); 6714 } else if (__ ShifterOperandCanHold(out_low, first_low, SBC, ~value_high, kCcDontCare, &so)) { 6715 __ sbc(out_high, first_high, so); 6716 } else { 6717 LOG(FATAL) << "Unexpected constant " << value_high; 6718 UNREACHABLE(); 6719 } 6720} 6721 6722void InstructionCodeGeneratorARM::HandleBitwiseOperation(HBinaryOperation* instruction) { 6723 LocationSummary* locations = instruction->GetLocations(); 6724 Location first = locations->InAt(0); 6725 Location second = locations->InAt(1); 6726 Location out = locations->Out(); 6727 6728 if (second.IsConstant()) { 6729 uint64_t value = static_cast<uint64_t>(Int64FromConstant(second.GetConstant())); 6730 uint32_t value_low = Low32Bits(value); 6731 if (instruction->GetResultType() == Primitive::kPrimInt) { 6732 Register first_reg = first.AsRegister<Register>(); 6733 Register out_reg = out.AsRegister<Register>(); 6734 if (instruction->IsAnd()) { 6735 GenerateAndConst(out_reg, first_reg, value_low); 6736 } else if (instruction->IsOr()) { 6737 GenerateOrrConst(out_reg, first_reg, value_low); 6738 } else { 6739 DCHECK(instruction->IsXor()); 6740 GenerateEorConst(out_reg, first_reg, value_low); 6741 } 6742 } else { 6743 DCHECK_EQ(instruction->GetResultType(), Primitive::kPrimLong); 6744 uint32_t value_high = High32Bits(value); 6745 Register first_low = first.AsRegisterPairLow<Register>(); 6746 Register first_high = first.AsRegisterPairHigh<Register>(); 6747 Register out_low = out.AsRegisterPairLow<Register>(); 6748 Register out_high = out.AsRegisterPairHigh<Register>(); 6749 if (instruction->IsAnd()) { 6750 GenerateAndConst(out_low, first_low, value_low); 6751 GenerateAndConst(out_high, first_high, value_high); 6752 } else if (instruction->IsOr()) { 6753 GenerateOrrConst(out_low, first_low, value_low); 6754 GenerateOrrConst(out_high, first_high, value_high); 6755 } else { 6756 DCHECK(instruction->IsXor()); 6757 GenerateEorConst(out_low, first_low, value_low); 6758 GenerateEorConst(out_high, first_high, value_high); 6759 } 6760 } 6761 return; 6762 } 6763 6764 if (instruction->GetResultType() == Primitive::kPrimInt) { 6765 Register first_reg = first.AsRegister<Register>(); 6766 ShifterOperand second_reg(second.AsRegister<Register>()); 6767 Register out_reg = out.AsRegister<Register>(); 6768 if (instruction->IsAnd()) { 6769 __ and_(out_reg, first_reg, second_reg); 6770 } else if (instruction->IsOr()) { 6771 __ orr(out_reg, first_reg, second_reg); 6772 } else { 6773 DCHECK(instruction->IsXor()); 6774 __ eor(out_reg, first_reg, second_reg); 6775 } 6776 } else { 6777 DCHECK_EQ(instruction->GetResultType(), Primitive::kPrimLong); 6778 Register first_low = first.AsRegisterPairLow<Register>(); 6779 Register first_high = first.AsRegisterPairHigh<Register>(); 6780 ShifterOperand second_low(second.AsRegisterPairLow<Register>()); 6781 ShifterOperand second_high(second.AsRegisterPairHigh<Register>()); 6782 Register out_low = out.AsRegisterPairLow<Register>(); 6783 Register out_high = out.AsRegisterPairHigh<Register>(); 6784 if (instruction->IsAnd()) { 6785 __ and_(out_low, first_low, second_low); 6786 __ and_(out_high, first_high, second_high); 6787 } else if (instruction->IsOr()) { 6788 __ orr(out_low, first_low, second_low); 6789 __ orr(out_high, first_high, second_high); 6790 } else { 6791 DCHECK(instruction->IsXor()); 6792 __ eor(out_low, first_low, second_low); 6793 __ eor(out_high, first_high, second_high); 6794 } 6795 } 6796} 6797 6798void InstructionCodeGeneratorARM::GenerateReferenceLoadOneRegister( 6799 HInstruction* instruction, 6800 Location out, 6801 uint32_t offset, 6802 Location maybe_temp, 6803 ReadBarrierOption read_barrier_option) { 6804 Register out_reg = out.AsRegister<Register>(); 6805 if (read_barrier_option == kWithReadBarrier) { 6806 CHECK(kEmitCompilerReadBarrier); 6807 DCHECK(maybe_temp.IsRegister()) << maybe_temp; 6808 if (kUseBakerReadBarrier) { 6809 // Load with fast path based Baker's read barrier. 6810 // /* HeapReference<Object> */ out = *(out + offset) 6811 codegen_->GenerateFieldLoadWithBakerReadBarrier( 6812 instruction, out, out_reg, offset, maybe_temp, /* needs_null_check */ false); 6813 } else { 6814 // Load with slow path based read barrier. 6815 // Save the value of `out` into `maybe_temp` before overwriting it 6816 // in the following move operation, as we will need it for the 6817 // read barrier below. 6818 __ Mov(maybe_temp.AsRegister<Register>(), out_reg); 6819 // /* HeapReference<Object> */ out = *(out + offset) 6820 __ LoadFromOffset(kLoadWord, out_reg, out_reg, offset); 6821 codegen_->GenerateReadBarrierSlow(instruction, out, out, maybe_temp, offset); 6822 } 6823 } else { 6824 // Plain load with no read barrier. 6825 // /* HeapReference<Object> */ out = *(out + offset) 6826 __ LoadFromOffset(kLoadWord, out_reg, out_reg, offset); 6827 __ MaybeUnpoisonHeapReference(out_reg); 6828 } 6829} 6830 6831void InstructionCodeGeneratorARM::GenerateReferenceLoadTwoRegisters( 6832 HInstruction* instruction, 6833 Location out, 6834 Location obj, 6835 uint32_t offset, 6836 Location maybe_temp, 6837 ReadBarrierOption read_barrier_option) { 6838 Register out_reg = out.AsRegister<Register>(); 6839 Register obj_reg = obj.AsRegister<Register>(); 6840 if (read_barrier_option == kWithReadBarrier) { 6841 CHECK(kEmitCompilerReadBarrier); 6842 if (kUseBakerReadBarrier) { 6843 DCHECK(maybe_temp.IsRegister()) << maybe_temp; 6844 // Load with fast path based Baker's read barrier. 6845 // /* HeapReference<Object> */ out = *(obj + offset) 6846 codegen_->GenerateFieldLoadWithBakerReadBarrier( 6847 instruction, out, obj_reg, offset, maybe_temp, /* needs_null_check */ false); 6848 } else { 6849 // Load with slow path based read barrier. 6850 // /* HeapReference<Object> */ out = *(obj + offset) 6851 __ LoadFromOffset(kLoadWord, out_reg, obj_reg, offset); 6852 codegen_->GenerateReadBarrierSlow(instruction, out, out, obj, offset); 6853 } 6854 } else { 6855 // Plain load with no read barrier. 6856 // /* HeapReference<Object> */ out = *(obj + offset) 6857 __ LoadFromOffset(kLoadWord, out_reg, obj_reg, offset); 6858 __ MaybeUnpoisonHeapReference(out_reg); 6859 } 6860} 6861 6862void InstructionCodeGeneratorARM::GenerateGcRootFieldLoad(HInstruction* instruction, 6863 Location root, 6864 Register obj, 6865 uint32_t offset, 6866 ReadBarrierOption read_barrier_option) { 6867 Register root_reg = root.AsRegister<Register>(); 6868 if (read_barrier_option == kWithReadBarrier) { 6869 DCHECK(kEmitCompilerReadBarrier); 6870 if (kUseBakerReadBarrier) { 6871 // Fast path implementation of art::ReadBarrier::BarrierForRoot when 6872 // Baker's read barrier are used: 6873 // 6874 // root = obj.field; 6875 // temp = Thread::Current()->pReadBarrierMarkReg ## root.reg() 6876 // if (temp != null) { 6877 // root = temp(root) 6878 // } 6879 6880 // /* GcRoot<mirror::Object> */ root = *(obj + offset) 6881 __ LoadFromOffset(kLoadWord, root_reg, obj, offset); 6882 static_assert( 6883 sizeof(mirror::CompressedReference<mirror::Object>) == sizeof(GcRoot<mirror::Object>), 6884 "art::mirror::CompressedReference<mirror::Object> and art::GcRoot<mirror::Object> " 6885 "have different sizes."); 6886 static_assert(sizeof(mirror::CompressedReference<mirror::Object>) == sizeof(int32_t), 6887 "art::mirror::CompressedReference<mirror::Object> and int32_t " 6888 "have different sizes."); 6889 6890 // Slow path marking the GC root `root`. 6891 Location temp = Location::RegisterLocation(LR); 6892 SlowPathCodeARM* slow_path = 6893 new (GetGraph()->GetArena()) ReadBarrierMarkSlowPathARM( 6894 instruction, 6895 root, 6896 /*entrypoint*/ temp); 6897 codegen_->AddSlowPath(slow_path); 6898 6899 // temp = Thread::Current()->pReadBarrierMarkReg ## root.reg() 6900 const int32_t entry_point_offset = 6901 CodeGenerator::GetReadBarrierMarkEntryPointsOffset<kArmPointerSize>(root.reg()); 6902 // Loading the entrypoint does not require a load acquire since it is only changed when 6903 // threads are suspended or running a checkpoint. 6904 __ LoadFromOffset(kLoadWord, temp.AsRegister<Register>(), TR, entry_point_offset); 6905 // The entrypoint is null when the GC is not marking, this prevents one load compared to 6906 // checking GetIsGcMarking. 6907 __ CompareAndBranchIfNonZero(temp.AsRegister<Register>(), slow_path->GetEntryLabel()); 6908 __ Bind(slow_path->GetExitLabel()); 6909 } else { 6910 // GC root loaded through a slow path for read barriers other 6911 // than Baker's. 6912 // /* GcRoot<mirror::Object>* */ root = obj + offset 6913 __ AddConstant(root_reg, obj, offset); 6914 // /* mirror::Object* */ root = root->Read() 6915 codegen_->GenerateReadBarrierForRootSlow(instruction, root, root); 6916 } 6917 } else { 6918 // Plain GC root load with no read barrier. 6919 // /* GcRoot<mirror::Object> */ root = *(obj + offset) 6920 __ LoadFromOffset(kLoadWord, root_reg, obj, offset); 6921 // Note that GC roots are not affected by heap poisoning, thus we 6922 // do not have to unpoison `root_reg` here. 6923 } 6924} 6925 6926void CodeGeneratorARM::GenerateFieldLoadWithBakerReadBarrier(HInstruction* instruction, 6927 Location ref, 6928 Register obj, 6929 uint32_t offset, 6930 Location temp, 6931 bool needs_null_check) { 6932 DCHECK(kEmitCompilerReadBarrier); 6933 DCHECK(kUseBakerReadBarrier); 6934 6935 // /* HeapReference<Object> */ ref = *(obj + offset) 6936 Location no_index = Location::NoLocation(); 6937 ScaleFactor no_scale_factor = TIMES_1; 6938 GenerateReferenceLoadWithBakerReadBarrier( 6939 instruction, ref, obj, offset, no_index, no_scale_factor, temp, needs_null_check); 6940} 6941 6942void CodeGeneratorARM::GenerateArrayLoadWithBakerReadBarrier(HInstruction* instruction, 6943 Location ref, 6944 Register obj, 6945 uint32_t data_offset, 6946 Location index, 6947 Location temp, 6948 bool needs_null_check) { 6949 DCHECK(kEmitCompilerReadBarrier); 6950 DCHECK(kUseBakerReadBarrier); 6951 6952 static_assert( 6953 sizeof(mirror::HeapReference<mirror::Object>) == sizeof(int32_t), 6954 "art::mirror::HeapReference<art::mirror::Object> and int32_t have different sizes."); 6955 // /* HeapReference<Object> */ ref = 6956 // *(obj + data_offset + index * sizeof(HeapReference<Object>)) 6957 ScaleFactor scale_factor = TIMES_4; 6958 GenerateReferenceLoadWithBakerReadBarrier( 6959 instruction, ref, obj, data_offset, index, scale_factor, temp, needs_null_check); 6960} 6961 6962void CodeGeneratorARM::GenerateReferenceLoadWithBakerReadBarrier(HInstruction* instruction, 6963 Location ref, 6964 Register obj, 6965 uint32_t offset, 6966 Location index, 6967 ScaleFactor scale_factor, 6968 Location temp, 6969 bool needs_null_check, 6970 bool always_update_field, 6971 Register* temp2) { 6972 DCHECK(kEmitCompilerReadBarrier); 6973 DCHECK(kUseBakerReadBarrier); 6974 6975 // In slow path based read barriers, the read barrier call is 6976 // inserted after the original load. However, in fast path based 6977 // Baker's read barriers, we need to perform the load of 6978 // mirror::Object::monitor_ *before* the original reference load. 6979 // This load-load ordering is required by the read barrier. 6980 // The fast path/slow path (for Baker's algorithm) should look like: 6981 // 6982 // uint32_t rb_state = Lockword(obj->monitor_).ReadBarrierState(); 6983 // lfence; // Load fence or artificial data dependency to prevent load-load reordering 6984 // HeapReference<Object> ref = *src; // Original reference load. 6985 // bool is_gray = (rb_state == ReadBarrier::GrayState()); 6986 // if (is_gray) { 6987 // ref = ReadBarrier::Mark(ref); // Performed by runtime entrypoint slow path. 6988 // } 6989 // 6990 // Note: the original implementation in ReadBarrier::Barrier is 6991 // slightly more complex as it performs additional checks that we do 6992 // not do here for performance reasons. 6993 6994 Register ref_reg = ref.AsRegister<Register>(); 6995 Register temp_reg = temp.AsRegister<Register>(); 6996 uint32_t monitor_offset = mirror::Object::MonitorOffset().Int32Value(); 6997 6998 // /* int32_t */ monitor = obj->monitor_ 6999 __ LoadFromOffset(kLoadWord, temp_reg, obj, monitor_offset); 7000 if (needs_null_check) { 7001 MaybeRecordImplicitNullCheck(instruction); 7002 } 7003 // /* LockWord */ lock_word = LockWord(monitor) 7004 static_assert(sizeof(LockWord) == sizeof(int32_t), 7005 "art::LockWord and int32_t have different sizes."); 7006 7007 // Introduce a dependency on the lock_word including the rb_state, 7008 // which shall prevent load-load reordering without using 7009 // a memory barrier (which would be more expensive). 7010 // `obj` is unchanged by this operation, but its value now depends 7011 // on `temp_reg`. 7012 __ add(obj, obj, ShifterOperand(temp_reg, LSR, 32)); 7013 7014 // The actual reference load. 7015 if (index.IsValid()) { 7016 // Load types involving an "index": ArrayGet, 7017 // UnsafeGetObject/UnsafeGetObjectVolatile and UnsafeCASObject 7018 // intrinsics. 7019 // /* HeapReference<Object> */ ref = *(obj + offset + (index << scale_factor)) 7020 if (index.IsConstant()) { 7021 size_t computed_offset = 7022 (index.GetConstant()->AsIntConstant()->GetValue() << scale_factor) + offset; 7023 __ LoadFromOffset(kLoadWord, ref_reg, obj, computed_offset); 7024 } else { 7025 // Handle the special case of the 7026 // UnsafeGetObject/UnsafeGetObjectVolatile and UnsafeCASObject 7027 // intrinsics, which use a register pair as index ("long 7028 // offset"), of which only the low part contains data. 7029 Register index_reg = index.IsRegisterPair() 7030 ? index.AsRegisterPairLow<Register>() 7031 : index.AsRegister<Register>(); 7032 __ add(IP, obj, ShifterOperand(index_reg, LSL, scale_factor)); 7033 __ LoadFromOffset(kLoadWord, ref_reg, IP, offset); 7034 } 7035 } else { 7036 // /* HeapReference<Object> */ ref = *(obj + offset) 7037 __ LoadFromOffset(kLoadWord, ref_reg, obj, offset); 7038 } 7039 7040 // Object* ref = ref_addr->AsMirrorPtr() 7041 __ MaybeUnpoisonHeapReference(ref_reg); 7042 7043 // Slow path marking the object `ref` when it is gray. 7044 SlowPathCodeARM* slow_path; 7045 if (always_update_field) { 7046 DCHECK(temp2 != nullptr); 7047 // ReadBarrierMarkAndUpdateFieldSlowPathARM only supports address 7048 // of the form `obj + field_offset`, where `obj` is a register and 7049 // `field_offset` is a register pair (of which only the lower half 7050 // is used). Thus `offset` and `scale_factor` above are expected 7051 // to be null in this code path. 7052 DCHECK_EQ(offset, 0u); 7053 DCHECK_EQ(scale_factor, ScaleFactor::TIMES_1); 7054 slow_path = new (GetGraph()->GetArena()) ReadBarrierMarkAndUpdateFieldSlowPathARM( 7055 instruction, ref, obj, /* field_offset */ index, temp_reg, *temp2); 7056 } else { 7057 slow_path = new (GetGraph()->GetArena()) ReadBarrierMarkSlowPathARM(instruction, ref); 7058 } 7059 AddSlowPath(slow_path); 7060 7061 // if (rb_state == ReadBarrier::GrayState()) 7062 // ref = ReadBarrier::Mark(ref); 7063 // Given the numeric representation, it's enough to check the low bit of the 7064 // rb_state. We do that by shifting the bit out of the lock word with LSRS 7065 // which can be a 16-bit instruction unlike the TST immediate. 7066 static_assert(ReadBarrier::WhiteState() == 0, "Expecting white to have value 0"); 7067 static_assert(ReadBarrier::GrayState() == 1, "Expecting gray to have value 1"); 7068 __ Lsrs(temp_reg, temp_reg, LockWord::kReadBarrierStateShift + 1); 7069 __ b(slow_path->GetEntryLabel(), CS); // Carry flag is the last bit shifted out by LSRS. 7070 __ Bind(slow_path->GetExitLabel()); 7071} 7072 7073void CodeGeneratorARM::GenerateReadBarrierSlow(HInstruction* instruction, 7074 Location out, 7075 Location ref, 7076 Location obj, 7077 uint32_t offset, 7078 Location index) { 7079 DCHECK(kEmitCompilerReadBarrier); 7080 7081 // Insert a slow path based read barrier *after* the reference load. 7082 // 7083 // If heap poisoning is enabled, the unpoisoning of the loaded 7084 // reference will be carried out by the runtime within the slow 7085 // path. 7086 // 7087 // Note that `ref` currently does not get unpoisoned (when heap 7088 // poisoning is enabled), which is alright as the `ref` argument is 7089 // not used by the artReadBarrierSlow entry point. 7090 // 7091 // TODO: Unpoison `ref` when it is used by artReadBarrierSlow. 7092 SlowPathCodeARM* slow_path = new (GetGraph()->GetArena()) 7093 ReadBarrierForHeapReferenceSlowPathARM(instruction, out, ref, obj, offset, index); 7094 AddSlowPath(slow_path); 7095 7096 __ b(slow_path->GetEntryLabel()); 7097 __ Bind(slow_path->GetExitLabel()); 7098} 7099 7100void CodeGeneratorARM::MaybeGenerateReadBarrierSlow(HInstruction* instruction, 7101 Location out, 7102 Location ref, 7103 Location obj, 7104 uint32_t offset, 7105 Location index) { 7106 if (kEmitCompilerReadBarrier) { 7107 // Baker's read barriers shall be handled by the fast path 7108 // (CodeGeneratorARM::GenerateReferenceLoadWithBakerReadBarrier). 7109 DCHECK(!kUseBakerReadBarrier); 7110 // If heap poisoning is enabled, unpoisoning will be taken care of 7111 // by the runtime within the slow path. 7112 GenerateReadBarrierSlow(instruction, out, ref, obj, offset, index); 7113 } else if (kPoisonHeapReferences) { 7114 __ UnpoisonHeapReference(out.AsRegister<Register>()); 7115 } 7116} 7117 7118void CodeGeneratorARM::GenerateReadBarrierForRootSlow(HInstruction* instruction, 7119 Location out, 7120 Location root) { 7121 DCHECK(kEmitCompilerReadBarrier); 7122 7123 // Insert a slow path based read barrier *after* the GC root load. 7124 // 7125 // Note that GC roots are not affected by heap poisoning, so we do 7126 // not need to do anything special for this here. 7127 SlowPathCodeARM* slow_path = 7128 new (GetGraph()->GetArena()) ReadBarrierForRootSlowPathARM(instruction, out, root); 7129 AddSlowPath(slow_path); 7130 7131 __ b(slow_path->GetEntryLabel()); 7132 __ Bind(slow_path->GetExitLabel()); 7133} 7134 7135HInvokeStaticOrDirect::DispatchInfo CodeGeneratorARM::GetSupportedInvokeStaticOrDirectDispatch( 7136 const HInvokeStaticOrDirect::DispatchInfo& desired_dispatch_info, 7137 HInvokeStaticOrDirect* invoke ATTRIBUTE_UNUSED) { 7138 HInvokeStaticOrDirect::DispatchInfo dispatch_info = desired_dispatch_info; 7139 // We disable pc-relative load when there is an irreducible loop, as the optimization 7140 // is incompatible with it. 7141 // TODO: Create as many ArmDexCacheArraysBase instructions as needed for methods 7142 // with irreducible loops. 7143 if (GetGraph()->HasIrreducibleLoops() && 7144 (dispatch_info.method_load_kind == 7145 HInvokeStaticOrDirect::MethodLoadKind::kDexCachePcRelative)) { 7146 dispatch_info.method_load_kind = HInvokeStaticOrDirect::MethodLoadKind::kDexCacheViaMethod; 7147 } 7148 7149 return dispatch_info; 7150} 7151 7152Register CodeGeneratorARM::GetInvokeStaticOrDirectExtraParameter(HInvokeStaticOrDirect* invoke, 7153 Register temp) { 7154 DCHECK_EQ(invoke->InputCount(), invoke->GetNumberOfArguments() + 1u); 7155 Location location = invoke->GetLocations()->InAt(invoke->GetSpecialInputIndex()); 7156 if (!invoke->GetLocations()->Intrinsified()) { 7157 return location.AsRegister<Register>(); 7158 } 7159 // For intrinsics we allow any location, so it may be on the stack. 7160 if (!location.IsRegister()) { 7161 __ LoadFromOffset(kLoadWord, temp, SP, location.GetStackIndex()); 7162 return temp; 7163 } 7164 // For register locations, check if the register was saved. If so, get it from the stack. 7165 // Note: There is a chance that the register was saved but not overwritten, so we could 7166 // save one load. However, since this is just an intrinsic slow path we prefer this 7167 // simple and more robust approach rather that trying to determine if that's the case. 7168 SlowPathCode* slow_path = GetCurrentSlowPath(); 7169 DCHECK(slow_path != nullptr); // For intrinsified invokes the call is emitted on the slow path. 7170 if (slow_path->IsCoreRegisterSaved(location.AsRegister<Register>())) { 7171 int stack_offset = slow_path->GetStackOffsetOfCoreRegister(location.AsRegister<Register>()); 7172 __ LoadFromOffset(kLoadWord, temp, SP, stack_offset); 7173 return temp; 7174 } 7175 return location.AsRegister<Register>(); 7176} 7177 7178void CodeGeneratorARM::GenerateStaticOrDirectCall(HInvokeStaticOrDirect* invoke, Location temp) { 7179 Location callee_method = temp; // For all kinds except kRecursive, callee will be in temp. 7180 switch (invoke->GetMethodLoadKind()) { 7181 case HInvokeStaticOrDirect::MethodLoadKind::kStringInit: { 7182 uint32_t offset = 7183 GetThreadOffset<kArmPointerSize>(invoke->GetStringInitEntryPoint()).Int32Value(); 7184 // temp = thread->string_init_entrypoint 7185 __ LoadFromOffset(kLoadWord, temp.AsRegister<Register>(), TR, offset); 7186 break; 7187 } 7188 case HInvokeStaticOrDirect::MethodLoadKind::kRecursive: 7189 callee_method = invoke->GetLocations()->InAt(invoke->GetSpecialInputIndex()); 7190 break; 7191 case HInvokeStaticOrDirect::MethodLoadKind::kDirectAddress: 7192 __ LoadImmediate(temp.AsRegister<Register>(), invoke->GetMethodAddress()); 7193 break; 7194 case HInvokeStaticOrDirect::MethodLoadKind::kDexCachePcRelative: { 7195 HArmDexCacheArraysBase* base = 7196 invoke->InputAt(invoke->GetSpecialInputIndex())->AsArmDexCacheArraysBase(); 7197 Register base_reg = GetInvokeStaticOrDirectExtraParameter(invoke, 7198 temp.AsRegister<Register>()); 7199 int32_t offset = invoke->GetDexCacheArrayOffset() - base->GetElementOffset(); 7200 __ LoadFromOffset(kLoadWord, temp.AsRegister<Register>(), base_reg, offset); 7201 break; 7202 } 7203 case HInvokeStaticOrDirect::MethodLoadKind::kDexCacheViaMethod: { 7204 Location current_method = invoke->GetLocations()->InAt(invoke->GetSpecialInputIndex()); 7205 Register method_reg; 7206 Register reg = temp.AsRegister<Register>(); 7207 if (current_method.IsRegister()) { 7208 method_reg = current_method.AsRegister<Register>(); 7209 } else { 7210 DCHECK(invoke->GetLocations()->Intrinsified()); 7211 DCHECK(!current_method.IsValid()); 7212 method_reg = reg; 7213 __ LoadFromOffset(kLoadWord, reg, SP, kCurrentMethodStackOffset); 7214 } 7215 // /* ArtMethod*[] */ temp = temp.ptr_sized_fields_->dex_cache_resolved_methods_; 7216 __ LoadFromOffset(kLoadWord, 7217 reg, 7218 method_reg, 7219 ArtMethod::DexCacheResolvedMethodsOffset(kArmPointerSize).Int32Value()); 7220 // temp = temp[index_in_cache]; 7221 // Note: Don't use invoke->GetTargetMethod() as it may point to a different dex file. 7222 uint32_t index_in_cache = invoke->GetDexMethodIndex(); 7223 __ LoadFromOffset(kLoadWord, reg, reg, CodeGenerator::GetCachePointerOffset(index_in_cache)); 7224 break; 7225 } 7226 } 7227 7228 switch (invoke->GetCodePtrLocation()) { 7229 case HInvokeStaticOrDirect::CodePtrLocation::kCallSelf: 7230 __ bl(GetFrameEntryLabel()); 7231 break; 7232 case HInvokeStaticOrDirect::CodePtrLocation::kCallArtMethod: 7233 // LR = callee_method->entry_point_from_quick_compiled_code_ 7234 __ LoadFromOffset( 7235 kLoadWord, LR, callee_method.AsRegister<Register>(), 7236 ArtMethod::EntryPointFromQuickCompiledCodeOffset(kArmPointerSize).Int32Value()); 7237 // LR() 7238 __ blx(LR); 7239 break; 7240 } 7241 7242 DCHECK(!IsLeafMethod()); 7243} 7244 7245void CodeGeneratorARM::GenerateVirtualCall(HInvokeVirtual* invoke, Location temp_location) { 7246 Register temp = temp_location.AsRegister<Register>(); 7247 uint32_t method_offset = mirror::Class::EmbeddedVTableEntryOffset( 7248 invoke->GetVTableIndex(), kArmPointerSize).Uint32Value(); 7249 7250 // Use the calling convention instead of the location of the receiver, as 7251 // intrinsics may have put the receiver in a different register. In the intrinsics 7252 // slow path, the arguments have been moved to the right place, so here we are 7253 // guaranteed that the receiver is the first register of the calling convention. 7254 InvokeDexCallingConvention calling_convention; 7255 Register receiver = calling_convention.GetRegisterAt(0); 7256 uint32_t class_offset = mirror::Object::ClassOffset().Int32Value(); 7257 // /* HeapReference<Class> */ temp = receiver->klass_ 7258 __ LoadFromOffset(kLoadWord, temp, receiver, class_offset); 7259 MaybeRecordImplicitNullCheck(invoke); 7260 // Instead of simply (possibly) unpoisoning `temp` here, we should 7261 // emit a read barrier for the previous class reference load. 7262 // However this is not required in practice, as this is an 7263 // intermediate/temporary reference and because the current 7264 // concurrent copying collector keeps the from-space memory 7265 // intact/accessible until the end of the marking phase (the 7266 // concurrent copying collector may not in the future). 7267 __ MaybeUnpoisonHeapReference(temp); 7268 // temp = temp->GetMethodAt(method_offset); 7269 uint32_t entry_point = ArtMethod::EntryPointFromQuickCompiledCodeOffset( 7270 kArmPointerSize).Int32Value(); 7271 __ LoadFromOffset(kLoadWord, temp, temp, method_offset); 7272 // LR = temp->GetEntryPoint(); 7273 __ LoadFromOffset(kLoadWord, LR, temp, entry_point); 7274 // LR(); 7275 __ blx(LR); 7276} 7277 7278CodeGeneratorARM::PcRelativePatchInfo* CodeGeneratorARM::NewPcRelativeStringPatch( 7279 const DexFile& dex_file, uint32_t string_index) { 7280 return NewPcRelativePatch(dex_file, string_index, &pc_relative_string_patches_); 7281} 7282 7283CodeGeneratorARM::PcRelativePatchInfo* CodeGeneratorARM::NewPcRelativeTypePatch( 7284 const DexFile& dex_file, dex::TypeIndex type_index) { 7285 return NewPcRelativePatch(dex_file, type_index.index_, &pc_relative_type_patches_); 7286} 7287 7288CodeGeneratorARM::PcRelativePatchInfo* CodeGeneratorARM::NewPcRelativeDexCacheArrayPatch( 7289 const DexFile& dex_file, uint32_t element_offset) { 7290 return NewPcRelativePatch(dex_file, element_offset, &pc_relative_dex_cache_patches_); 7291} 7292 7293CodeGeneratorARM::PcRelativePatchInfo* CodeGeneratorARM::NewPcRelativePatch( 7294 const DexFile& dex_file, uint32_t offset_or_index, ArenaDeque<PcRelativePatchInfo>* patches) { 7295 patches->emplace_back(dex_file, offset_or_index); 7296 return &patches->back(); 7297} 7298 7299Literal* CodeGeneratorARM::DeduplicateBootImageStringLiteral(const DexFile& dex_file, 7300 dex::StringIndex string_index) { 7301 return boot_image_string_patches_.GetOrCreate( 7302 StringReference(&dex_file, string_index), 7303 [this]() { return __ NewLiteral<uint32_t>(/* placeholder */ 0u); }); 7304} 7305 7306Literal* CodeGeneratorARM::DeduplicateBootImageTypeLiteral(const DexFile& dex_file, 7307 dex::TypeIndex type_index) { 7308 return boot_image_type_patches_.GetOrCreate( 7309 TypeReference(&dex_file, type_index), 7310 [this]() { return __ NewLiteral<uint32_t>(/* placeholder */ 0u); }); 7311} 7312 7313Literal* CodeGeneratorARM::DeduplicateBootImageAddressLiteral(uint32_t address) { 7314 bool needs_patch = GetCompilerOptions().GetIncludePatchInformation(); 7315 Uint32ToLiteralMap* map = needs_patch ? &boot_image_address_patches_ : &uint32_literals_; 7316 return DeduplicateUint32Literal(dchecked_integral_cast<uint32_t>(address), map); 7317} 7318 7319Literal* CodeGeneratorARM::DeduplicateJitStringLiteral(const DexFile& dex_file, 7320 dex::StringIndex string_index) { 7321 jit_string_roots_.Overwrite(StringReference(&dex_file, string_index), /* placeholder */ 0u); 7322 return jit_string_patches_.GetOrCreate( 7323 StringReference(&dex_file, string_index), 7324 [this]() { return __ NewLiteral<uint32_t>(/* placeholder */ 0u); }); 7325} 7326 7327Literal* CodeGeneratorARM::DeduplicateJitClassLiteral(const DexFile& dex_file, 7328 dex::TypeIndex type_index, 7329 uint64_t address) { 7330 jit_class_roots_.Overwrite(TypeReference(&dex_file, type_index), address); 7331 return jit_class_patches_.GetOrCreate( 7332 TypeReference(&dex_file, type_index), 7333 [this]() { return __ NewLiteral<uint32_t>(/* placeholder */ 0u); }); 7334} 7335 7336template <LinkerPatch (*Factory)(size_t, const DexFile*, uint32_t, uint32_t)> 7337inline void CodeGeneratorARM::EmitPcRelativeLinkerPatches( 7338 const ArenaDeque<PcRelativePatchInfo>& infos, 7339 ArenaVector<LinkerPatch>* linker_patches) { 7340 for (const PcRelativePatchInfo& info : infos) { 7341 const DexFile& dex_file = info.target_dex_file; 7342 size_t offset_or_index = info.offset_or_index; 7343 DCHECK(info.add_pc_label.IsBound()); 7344 uint32_t add_pc_offset = dchecked_integral_cast<uint32_t>(info.add_pc_label.Position()); 7345 // Add MOVW patch. 7346 DCHECK(info.movw_label.IsBound()); 7347 uint32_t movw_offset = dchecked_integral_cast<uint32_t>(info.movw_label.Position()); 7348 linker_patches->push_back(Factory(movw_offset, &dex_file, add_pc_offset, offset_or_index)); 7349 // Add MOVT patch. 7350 DCHECK(info.movt_label.IsBound()); 7351 uint32_t movt_offset = dchecked_integral_cast<uint32_t>(info.movt_label.Position()); 7352 linker_patches->push_back(Factory(movt_offset, &dex_file, add_pc_offset, offset_or_index)); 7353 } 7354} 7355 7356void CodeGeneratorARM::EmitLinkerPatches(ArenaVector<LinkerPatch>* linker_patches) { 7357 DCHECK(linker_patches->empty()); 7358 size_t size = 7359 /* MOVW+MOVT for each entry */ 2u * pc_relative_dex_cache_patches_.size() + 7360 boot_image_string_patches_.size() + 7361 /* MOVW+MOVT for each entry */ 2u * pc_relative_string_patches_.size() + 7362 boot_image_type_patches_.size() + 7363 /* MOVW+MOVT for each entry */ 2u * pc_relative_type_patches_.size() + 7364 boot_image_address_patches_.size(); 7365 linker_patches->reserve(size); 7366 EmitPcRelativeLinkerPatches<LinkerPatch::DexCacheArrayPatch>(pc_relative_dex_cache_patches_, 7367 linker_patches); 7368 for (const auto& entry : boot_image_string_patches_) { 7369 const StringReference& target_string = entry.first; 7370 Literal* literal = entry.second; 7371 DCHECK(literal->GetLabel()->IsBound()); 7372 uint32_t literal_offset = literal->GetLabel()->Position(); 7373 linker_patches->push_back(LinkerPatch::StringPatch(literal_offset, 7374 target_string.dex_file, 7375 target_string.string_index.index_)); 7376 } 7377 if (!GetCompilerOptions().IsBootImage()) { 7378 EmitPcRelativeLinkerPatches<LinkerPatch::StringBssEntryPatch>(pc_relative_string_patches_, 7379 linker_patches); 7380 } else { 7381 EmitPcRelativeLinkerPatches<LinkerPatch::RelativeStringPatch>(pc_relative_string_patches_, 7382 linker_patches); 7383 } 7384 for (const auto& entry : boot_image_type_patches_) { 7385 const TypeReference& target_type = entry.first; 7386 Literal* literal = entry.second; 7387 DCHECK(literal->GetLabel()->IsBound()); 7388 uint32_t literal_offset = literal->GetLabel()->Position(); 7389 linker_patches->push_back(LinkerPatch::TypePatch(literal_offset, 7390 target_type.dex_file, 7391 target_type.type_index.index_)); 7392 } 7393 EmitPcRelativeLinkerPatches<LinkerPatch::RelativeTypePatch>(pc_relative_type_patches_, 7394 linker_patches); 7395 for (const auto& entry : boot_image_address_patches_) { 7396 DCHECK(GetCompilerOptions().GetIncludePatchInformation()); 7397 Literal* literal = entry.second; 7398 DCHECK(literal->GetLabel()->IsBound()); 7399 uint32_t literal_offset = literal->GetLabel()->Position(); 7400 linker_patches->push_back(LinkerPatch::RecordPosition(literal_offset)); 7401 } 7402} 7403 7404Literal* CodeGeneratorARM::DeduplicateUint32Literal(uint32_t value, Uint32ToLiteralMap* map) { 7405 return map->GetOrCreate( 7406 value, 7407 [this, value]() { return __ NewLiteral<uint32_t>(value); }); 7408} 7409 7410Literal* CodeGeneratorARM::DeduplicateMethodLiteral(MethodReference target_method, 7411 MethodToLiteralMap* map) { 7412 return map->GetOrCreate( 7413 target_method, 7414 [this]() { return __ NewLiteral<uint32_t>(/* placeholder */ 0u); }); 7415} 7416 7417void LocationsBuilderARM::VisitMultiplyAccumulate(HMultiplyAccumulate* instr) { 7418 LocationSummary* locations = 7419 new (GetGraph()->GetArena()) LocationSummary(instr, LocationSummary::kNoCall); 7420 locations->SetInAt(HMultiplyAccumulate::kInputAccumulatorIndex, 7421 Location::RequiresRegister()); 7422 locations->SetInAt(HMultiplyAccumulate::kInputMulLeftIndex, Location::RequiresRegister()); 7423 locations->SetInAt(HMultiplyAccumulate::kInputMulRightIndex, Location::RequiresRegister()); 7424 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 7425} 7426 7427void InstructionCodeGeneratorARM::VisitMultiplyAccumulate(HMultiplyAccumulate* instr) { 7428 LocationSummary* locations = instr->GetLocations(); 7429 Register res = locations->Out().AsRegister<Register>(); 7430 Register accumulator = 7431 locations->InAt(HMultiplyAccumulate::kInputAccumulatorIndex).AsRegister<Register>(); 7432 Register mul_left = 7433 locations->InAt(HMultiplyAccumulate::kInputMulLeftIndex).AsRegister<Register>(); 7434 Register mul_right = 7435 locations->InAt(HMultiplyAccumulate::kInputMulRightIndex).AsRegister<Register>(); 7436 7437 if (instr->GetOpKind() == HInstruction::kAdd) { 7438 __ mla(res, mul_left, mul_right, accumulator); 7439 } else { 7440 __ mls(res, mul_left, mul_right, accumulator); 7441 } 7442} 7443 7444void LocationsBuilderARM::VisitBoundType(HBoundType* instruction ATTRIBUTE_UNUSED) { 7445 // Nothing to do, this should be removed during prepare for register allocator. 7446 LOG(FATAL) << "Unreachable"; 7447} 7448 7449void InstructionCodeGeneratorARM::VisitBoundType(HBoundType* instruction ATTRIBUTE_UNUSED) { 7450 // Nothing to do, this should be removed during prepare for register allocator. 7451 LOG(FATAL) << "Unreachable"; 7452} 7453 7454// Simple implementation of packed switch - generate cascaded compare/jumps. 7455void LocationsBuilderARM::VisitPackedSwitch(HPackedSwitch* switch_instr) { 7456 LocationSummary* locations = 7457 new (GetGraph()->GetArena()) LocationSummary(switch_instr, LocationSummary::kNoCall); 7458 locations->SetInAt(0, Location::RequiresRegister()); 7459 if (switch_instr->GetNumEntries() > kPackedSwitchCompareJumpThreshold && 7460 codegen_->GetAssembler()->IsThumb()) { 7461 locations->AddTemp(Location::RequiresRegister()); // We need a temp for the table base. 7462 if (switch_instr->GetStartValue() != 0) { 7463 locations->AddTemp(Location::RequiresRegister()); // We need a temp for the bias. 7464 } 7465 } 7466} 7467 7468void InstructionCodeGeneratorARM::VisitPackedSwitch(HPackedSwitch* switch_instr) { 7469 int32_t lower_bound = switch_instr->GetStartValue(); 7470 uint32_t num_entries = switch_instr->GetNumEntries(); 7471 LocationSummary* locations = switch_instr->GetLocations(); 7472 Register value_reg = locations->InAt(0).AsRegister<Register>(); 7473 HBasicBlock* default_block = switch_instr->GetDefaultBlock(); 7474 7475 if (num_entries <= kPackedSwitchCompareJumpThreshold || !codegen_->GetAssembler()->IsThumb()) { 7476 // Create a series of compare/jumps. 7477 Register temp_reg = IP; 7478 // Note: It is fine for the below AddConstantSetFlags() using IP register to temporarily store 7479 // the immediate, because IP is used as the destination register. For the other 7480 // AddConstantSetFlags() and GenerateCompareWithImmediate(), the immediate values are constant, 7481 // and they can be encoded in the instruction without making use of IP register. 7482 __ AddConstantSetFlags(temp_reg, value_reg, -lower_bound); 7483 7484 const ArenaVector<HBasicBlock*>& successors = switch_instr->GetBlock()->GetSuccessors(); 7485 // Jump to successors[0] if value == lower_bound. 7486 __ b(codegen_->GetLabelOf(successors[0]), EQ); 7487 int32_t last_index = 0; 7488 for (; num_entries - last_index > 2; last_index += 2) { 7489 __ AddConstantSetFlags(temp_reg, temp_reg, -2); 7490 // Jump to successors[last_index + 1] if value < case_value[last_index + 2]. 7491 __ b(codegen_->GetLabelOf(successors[last_index + 1]), LO); 7492 // Jump to successors[last_index + 2] if value == case_value[last_index + 2]. 7493 __ b(codegen_->GetLabelOf(successors[last_index + 2]), EQ); 7494 } 7495 if (num_entries - last_index == 2) { 7496 // The last missing case_value. 7497 __ CmpConstant(temp_reg, 1); 7498 __ b(codegen_->GetLabelOf(successors[last_index + 1]), EQ); 7499 } 7500 7501 // And the default for any other value. 7502 if (!codegen_->GoesToNextBlock(switch_instr->GetBlock(), default_block)) { 7503 __ b(codegen_->GetLabelOf(default_block)); 7504 } 7505 } else { 7506 // Create a table lookup. 7507 Register temp_reg = locations->GetTemp(0).AsRegister<Register>(); 7508 7509 // Materialize a pointer to the switch table 7510 std::vector<Label*> labels(num_entries); 7511 const ArenaVector<HBasicBlock*>& successors = switch_instr->GetBlock()->GetSuccessors(); 7512 for (uint32_t i = 0; i < num_entries; i++) { 7513 labels[i] = codegen_->GetLabelOf(successors[i]); 7514 } 7515 JumpTable* table = __ CreateJumpTable(std::move(labels), temp_reg); 7516 7517 // Remove the bias. 7518 Register key_reg; 7519 if (lower_bound != 0) { 7520 key_reg = locations->GetTemp(1).AsRegister<Register>(); 7521 __ AddConstant(key_reg, value_reg, -lower_bound); 7522 } else { 7523 key_reg = value_reg; 7524 } 7525 7526 // Check whether the value is in the table, jump to default block if not. 7527 __ CmpConstant(key_reg, num_entries - 1); 7528 __ b(codegen_->GetLabelOf(default_block), Condition::HI); 7529 7530 // Load the displacement from the table. 7531 __ ldr(temp_reg, Address(temp_reg, key_reg, Shift::LSL, 2)); 7532 7533 // Dispatch is a direct add to the PC (for Thumb2). 7534 __ EmitJumpTableDispatch(table, temp_reg); 7535 } 7536} 7537 7538void LocationsBuilderARM::VisitArmDexCacheArraysBase(HArmDexCacheArraysBase* base) { 7539 LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(base); 7540 locations->SetOut(Location::RequiresRegister()); 7541} 7542 7543void InstructionCodeGeneratorARM::VisitArmDexCacheArraysBase(HArmDexCacheArraysBase* base) { 7544 Register base_reg = base->GetLocations()->Out().AsRegister<Register>(); 7545 CodeGeneratorARM::PcRelativePatchInfo* labels = 7546 codegen_->NewPcRelativeDexCacheArrayPatch(base->GetDexFile(), base->GetElementOffset()); 7547 __ BindTrackedLabel(&labels->movw_label); 7548 __ movw(base_reg, /* placeholder */ 0u); 7549 __ BindTrackedLabel(&labels->movt_label); 7550 __ movt(base_reg, /* placeholder */ 0u); 7551 __ BindTrackedLabel(&labels->add_pc_label); 7552 __ add(base_reg, base_reg, ShifterOperand(PC)); 7553} 7554 7555void CodeGeneratorARM::MoveFromReturnRegister(Location trg, Primitive::Type type) { 7556 if (!trg.IsValid()) { 7557 DCHECK_EQ(type, Primitive::kPrimVoid); 7558 return; 7559 } 7560 7561 DCHECK_NE(type, Primitive::kPrimVoid); 7562 7563 Location return_loc = InvokeDexCallingConventionVisitorARM().GetReturnLocation(type); 7564 if (return_loc.Equals(trg)) { 7565 return; 7566 } 7567 7568 // TODO: Consider pairs in the parallel move resolver, then this could be nicely merged 7569 // with the last branch. 7570 if (type == Primitive::kPrimLong) { 7571 HParallelMove parallel_move(GetGraph()->GetArena()); 7572 parallel_move.AddMove(return_loc.ToLow(), trg.ToLow(), Primitive::kPrimInt, nullptr); 7573 parallel_move.AddMove(return_loc.ToHigh(), trg.ToHigh(), Primitive::kPrimInt, nullptr); 7574 GetMoveResolver()->EmitNativeCode(¶llel_move); 7575 } else if (type == Primitive::kPrimDouble) { 7576 HParallelMove parallel_move(GetGraph()->GetArena()); 7577 parallel_move.AddMove(return_loc.ToLow(), trg.ToLow(), Primitive::kPrimFloat, nullptr); 7578 parallel_move.AddMove(return_loc.ToHigh(), trg.ToHigh(), Primitive::kPrimFloat, nullptr); 7579 GetMoveResolver()->EmitNativeCode(¶llel_move); 7580 } else { 7581 // Let the parallel move resolver take care of all of this. 7582 HParallelMove parallel_move(GetGraph()->GetArena()); 7583 parallel_move.AddMove(return_loc, trg, type, nullptr); 7584 GetMoveResolver()->EmitNativeCode(¶llel_move); 7585 } 7586} 7587 7588void LocationsBuilderARM::VisitClassTableGet(HClassTableGet* instruction) { 7589 LocationSummary* locations = 7590 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall); 7591 locations->SetInAt(0, Location::RequiresRegister()); 7592 locations->SetOut(Location::RequiresRegister()); 7593} 7594 7595void InstructionCodeGeneratorARM::VisitClassTableGet(HClassTableGet* instruction) { 7596 LocationSummary* locations = instruction->GetLocations(); 7597 if (instruction->GetTableKind() == HClassTableGet::TableKind::kVTable) { 7598 uint32_t method_offset = mirror::Class::EmbeddedVTableEntryOffset( 7599 instruction->GetIndex(), kArmPointerSize).SizeValue(); 7600 __ LoadFromOffset(kLoadWord, 7601 locations->Out().AsRegister<Register>(), 7602 locations->InAt(0).AsRegister<Register>(), 7603 method_offset); 7604 } else { 7605 uint32_t method_offset = static_cast<uint32_t>(ImTable::OffsetOfElement( 7606 instruction->GetIndex(), kArmPointerSize)); 7607 __ LoadFromOffset(kLoadWord, 7608 locations->Out().AsRegister<Register>(), 7609 locations->InAt(0).AsRegister<Register>(), 7610 mirror::Class::ImtPtrOffset(kArmPointerSize).Uint32Value()); 7611 __ LoadFromOffset(kLoadWord, 7612 locations->Out().AsRegister<Register>(), 7613 locations->Out().AsRegister<Register>(), 7614 method_offset); 7615 } 7616} 7617 7618static void PatchJitRootUse(uint8_t* code, 7619 const uint8_t* roots_data, 7620 Literal* literal, 7621 uint64_t index_in_table) { 7622 DCHECK(literal->GetLabel()->IsBound()); 7623 uint32_t literal_offset = literal->GetLabel()->Position(); 7624 uintptr_t address = 7625 reinterpret_cast<uintptr_t>(roots_data) + index_in_table * sizeof(GcRoot<mirror::Object>); 7626 uint8_t* data = code + literal_offset; 7627 reinterpret_cast<uint32_t*>(data)[0] = dchecked_integral_cast<uint32_t>(address); 7628} 7629 7630void CodeGeneratorARM::EmitJitRootPatches(uint8_t* code, const uint8_t* roots_data) { 7631 for (const auto& entry : jit_string_patches_) { 7632 const auto& it = jit_string_roots_.find(entry.first); 7633 DCHECK(it != jit_string_roots_.end()); 7634 PatchJitRootUse(code, roots_data, entry.second, it->second); 7635 } 7636 for (const auto& entry : jit_class_patches_) { 7637 const auto& it = jit_class_roots_.find(entry.first); 7638 DCHECK(it != jit_class_roots_.end()); 7639 PatchJitRootUse(code, roots_data, entry.second, it->second); 7640 } 7641} 7642 7643#undef __ 7644#undef QUICK_ENTRY_POINT 7645 7646} // namespace arm 7647} // namespace art 7648