code_generator_arm.cc revision 02e3092f8d98f339588e48691db77f227b48ac1e
1/* 2 * Copyright (C) 2014 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17#include "code_generator_arm.h" 18 19#include "arch/arm/instruction_set_features_arm.h" 20#include "art_method.h" 21#include "code_generator_utils.h" 22#include "compiled_method.h" 23#include "entrypoints/quick/quick_entrypoints.h" 24#include "gc/accounting/card_table.h" 25#include "intrinsics.h" 26#include "intrinsics_arm.h" 27#include "mirror/array-inl.h" 28#include "mirror/class-inl.h" 29#include "thread.h" 30#include "utils/arm/assembler_arm.h" 31#include "utils/arm/managed_register_arm.h" 32#include "utils/assembler.h" 33#include "utils/stack_checks.h" 34 35namespace art { 36 37template<class MirrorType> 38class GcRoot; 39 40namespace arm { 41 42static bool ExpectedPairLayout(Location location) { 43 // We expected this for both core and fpu register pairs. 44 return ((location.low() & 1) == 0) && (location.low() + 1 == location.high()); 45} 46 47static constexpr int kCurrentMethodStackOffset = 0; 48static constexpr Register kMethodRegisterArgument = R0; 49 50static constexpr Register kCoreAlwaysSpillRegister = R5; 51static constexpr Register kCoreCalleeSaves[] = 52 { R5, R6, R7, R8, R10, R11, LR }; 53static constexpr SRegister kFpuCalleeSaves[] = 54 { S16, S17, S18, S19, S20, S21, S22, S23, S24, S25, S26, S27, S28, S29, S30, S31 }; 55 56// D31 cannot be split into two S registers, and the register allocator only works on 57// S registers. Therefore there is no need to block it. 58static constexpr DRegister DTMP = D31; 59 60static constexpr uint32_t kPackedSwitchCompareJumpThreshold = 7; 61 62// NOLINT on __ macro to suppress wrong warning/fix (misc-macro-parentheses) from clang-tidy. 63#define __ down_cast<ArmAssembler*>(codegen->GetAssembler())-> // NOLINT 64#define QUICK_ENTRY_POINT(x) QUICK_ENTRYPOINT_OFFSET(kArmPointerSize, x).Int32Value() 65 66static constexpr int kRegListThreshold = 4; 67 68// SaveLiveRegisters and RestoreLiveRegisters from SlowPathCodeARM operate on sets of S registers, 69// for each live D registers they treat two corresponding S registers as live ones. 70// 71// Two following functions (SaveContiguousSRegisterList, RestoreContiguousSRegisterList) build 72// from a list of contiguous S registers a list of contiguous D registers (processing first/last 73// S registers corner cases) and save/restore this new list treating them as D registers. 74// - decreasing code size 75// - avoiding hazards on Cortex-A57, when a pair of S registers for an actual live D register is 76// restored and then used in regular non SlowPath code as D register. 77// 78// For the following example (v means the S register is live): 79// D names: | D0 | D1 | D2 | D4 | ... 80// S names: | S0 | S1 | S2 | S3 | S4 | S5 | S6 | S7 | ... 81// Live? | | v | v | v | v | v | v | | ... 82// 83// S1 and S6 will be saved/restored independently; D registers list (D1, D2) will be processed 84// as D registers. 85static size_t SaveContiguousSRegisterList(size_t first, 86 size_t last, 87 CodeGenerator* codegen, 88 size_t stack_offset) { 89 DCHECK_LE(first, last); 90 if ((first == last) && (first == 0)) { 91 stack_offset += codegen->SaveFloatingPointRegister(stack_offset, first); 92 return stack_offset; 93 } 94 if (first % 2 == 1) { 95 stack_offset += codegen->SaveFloatingPointRegister(stack_offset, first++); 96 } 97 98 bool save_last = false; 99 if (last % 2 == 0) { 100 save_last = true; 101 --last; 102 } 103 104 if (first < last) { 105 DRegister d_reg = static_cast<DRegister>(first / 2); 106 DCHECK_EQ((last - first + 1) % 2, 0u); 107 size_t number_of_d_regs = (last - first + 1) / 2; 108 109 if (number_of_d_regs == 1) { 110 __ StoreDToOffset(d_reg, SP, stack_offset); 111 } else if (number_of_d_regs > 1) { 112 __ add(IP, SP, ShifterOperand(stack_offset)); 113 __ vstmiad(IP, d_reg, number_of_d_regs); 114 } 115 stack_offset += number_of_d_regs * kArmWordSize * 2; 116 } 117 118 if (save_last) { 119 stack_offset += codegen->SaveFloatingPointRegister(stack_offset, last + 1); 120 } 121 122 return stack_offset; 123} 124 125static size_t RestoreContiguousSRegisterList(size_t first, 126 size_t last, 127 CodeGenerator* codegen, 128 size_t stack_offset) { 129 DCHECK_LE(first, last); 130 if ((first == last) && (first == 0)) { 131 stack_offset += codegen->RestoreFloatingPointRegister(stack_offset, first); 132 return stack_offset; 133 } 134 if (first % 2 == 1) { 135 stack_offset += codegen->RestoreFloatingPointRegister(stack_offset, first++); 136 } 137 138 bool restore_last = false; 139 if (last % 2 == 0) { 140 restore_last = true; 141 --last; 142 } 143 144 if (first < last) { 145 DRegister d_reg = static_cast<DRegister>(first / 2); 146 DCHECK_EQ((last - first + 1) % 2, 0u); 147 size_t number_of_d_regs = (last - first + 1) / 2; 148 if (number_of_d_regs == 1) { 149 __ LoadDFromOffset(d_reg, SP, stack_offset); 150 } else if (number_of_d_regs > 1) { 151 __ add(IP, SP, ShifterOperand(stack_offset)); 152 __ vldmiad(IP, d_reg, number_of_d_regs); 153 } 154 stack_offset += number_of_d_regs * kArmWordSize * 2; 155 } 156 157 if (restore_last) { 158 stack_offset += codegen->RestoreFloatingPointRegister(stack_offset, last + 1); 159 } 160 161 return stack_offset; 162} 163 164void SlowPathCodeARM::SaveLiveRegisters(CodeGenerator* codegen, LocationSummary* locations) { 165 size_t stack_offset = codegen->GetFirstRegisterSlotInSlowPath(); 166 size_t orig_offset = stack_offset; 167 168 const uint32_t core_spills = codegen->GetSlowPathSpills(locations, /* core_registers */ true); 169 for (uint32_t i : LowToHighBits(core_spills)) { 170 // If the register holds an object, update the stack mask. 171 if (locations->RegisterContainsObject(i)) { 172 locations->SetStackBit(stack_offset / kVRegSize); 173 } 174 DCHECK_LT(stack_offset, codegen->GetFrameSize() - codegen->FrameEntrySpillSize()); 175 DCHECK_LT(i, kMaximumNumberOfExpectedRegisters); 176 saved_core_stack_offsets_[i] = stack_offset; 177 stack_offset += kArmWordSize; 178 } 179 180 int reg_num = POPCOUNT(core_spills); 181 if (reg_num != 0) { 182 if (reg_num > kRegListThreshold) { 183 __ StoreList(RegList(core_spills), orig_offset); 184 } else { 185 stack_offset = orig_offset; 186 for (uint32_t i : LowToHighBits(core_spills)) { 187 stack_offset += codegen->SaveCoreRegister(stack_offset, i); 188 } 189 } 190 } 191 192 uint32_t fp_spills = codegen->GetSlowPathSpills(locations, /* core_registers */ false); 193 orig_offset = stack_offset; 194 for (uint32_t i : LowToHighBits(fp_spills)) { 195 DCHECK_LT(i, kMaximumNumberOfExpectedRegisters); 196 saved_fpu_stack_offsets_[i] = stack_offset; 197 stack_offset += kArmWordSize; 198 } 199 200 stack_offset = orig_offset; 201 while (fp_spills != 0u) { 202 uint32_t begin = CTZ(fp_spills); 203 uint32_t tmp = fp_spills + (1u << begin); 204 fp_spills &= tmp; // Clear the contiguous range of 1s. 205 uint32_t end = (tmp == 0u) ? 32u : CTZ(tmp); // CTZ(0) is undefined. 206 stack_offset = SaveContiguousSRegisterList(begin, end - 1, codegen, stack_offset); 207 } 208 DCHECK_LE(stack_offset, codegen->GetFrameSize() - codegen->FrameEntrySpillSize()); 209} 210 211void SlowPathCodeARM::RestoreLiveRegisters(CodeGenerator* codegen, LocationSummary* locations) { 212 size_t stack_offset = codegen->GetFirstRegisterSlotInSlowPath(); 213 size_t orig_offset = stack_offset; 214 215 const uint32_t core_spills = codegen->GetSlowPathSpills(locations, /* core_registers */ true); 216 for (uint32_t i : LowToHighBits(core_spills)) { 217 DCHECK_LT(stack_offset, codegen->GetFrameSize() - codegen->FrameEntrySpillSize()); 218 DCHECK_LT(i, kMaximumNumberOfExpectedRegisters); 219 stack_offset += kArmWordSize; 220 } 221 222 int reg_num = POPCOUNT(core_spills); 223 if (reg_num != 0) { 224 if (reg_num > kRegListThreshold) { 225 __ LoadList(RegList(core_spills), orig_offset); 226 } else { 227 stack_offset = orig_offset; 228 for (uint32_t i : LowToHighBits(core_spills)) { 229 stack_offset += codegen->RestoreCoreRegister(stack_offset, i); 230 } 231 } 232 } 233 234 uint32_t fp_spills = codegen->GetSlowPathSpills(locations, /* core_registers */ false); 235 while (fp_spills != 0u) { 236 uint32_t begin = CTZ(fp_spills); 237 uint32_t tmp = fp_spills + (1u << begin); 238 fp_spills &= tmp; // Clear the contiguous range of 1s. 239 uint32_t end = (tmp == 0u) ? 32u : CTZ(tmp); // CTZ(0) is undefined. 240 stack_offset = RestoreContiguousSRegisterList(begin, end - 1, codegen, stack_offset); 241 } 242 DCHECK_LE(stack_offset, codegen->GetFrameSize() - codegen->FrameEntrySpillSize()); 243} 244 245class NullCheckSlowPathARM : public SlowPathCodeARM { 246 public: 247 explicit NullCheckSlowPathARM(HNullCheck* instruction) : SlowPathCodeARM(instruction) {} 248 249 void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { 250 CodeGeneratorARM* arm_codegen = down_cast<CodeGeneratorARM*>(codegen); 251 __ Bind(GetEntryLabel()); 252 if (instruction_->CanThrowIntoCatchBlock()) { 253 // Live registers will be restored in the catch block if caught. 254 SaveLiveRegisters(codegen, instruction_->GetLocations()); 255 } 256 arm_codegen->InvokeRuntime(kQuickThrowNullPointer, 257 instruction_, 258 instruction_->GetDexPc(), 259 this); 260 CheckEntrypointTypes<kQuickThrowNullPointer, void, void>(); 261 } 262 263 bool IsFatal() const OVERRIDE { return true; } 264 265 const char* GetDescription() const OVERRIDE { return "NullCheckSlowPathARM"; } 266 267 private: 268 DISALLOW_COPY_AND_ASSIGN(NullCheckSlowPathARM); 269}; 270 271class DivZeroCheckSlowPathARM : public SlowPathCodeARM { 272 public: 273 explicit DivZeroCheckSlowPathARM(HDivZeroCheck* instruction) : SlowPathCodeARM(instruction) {} 274 275 void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { 276 CodeGeneratorARM* arm_codegen = down_cast<CodeGeneratorARM*>(codegen); 277 __ Bind(GetEntryLabel()); 278 arm_codegen->InvokeRuntime(kQuickThrowDivZero, instruction_, instruction_->GetDexPc(), this); 279 CheckEntrypointTypes<kQuickThrowDivZero, void, void>(); 280 } 281 282 bool IsFatal() const OVERRIDE { return true; } 283 284 const char* GetDescription() const OVERRIDE { return "DivZeroCheckSlowPathARM"; } 285 286 private: 287 DISALLOW_COPY_AND_ASSIGN(DivZeroCheckSlowPathARM); 288}; 289 290class SuspendCheckSlowPathARM : public SlowPathCodeARM { 291 public: 292 SuspendCheckSlowPathARM(HSuspendCheck* instruction, HBasicBlock* successor) 293 : SlowPathCodeARM(instruction), successor_(successor) {} 294 295 void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { 296 CodeGeneratorARM* arm_codegen = down_cast<CodeGeneratorARM*>(codegen); 297 __ Bind(GetEntryLabel()); 298 arm_codegen->InvokeRuntime(kQuickTestSuspend, instruction_, instruction_->GetDexPc(), this); 299 CheckEntrypointTypes<kQuickTestSuspend, void, void>(); 300 if (successor_ == nullptr) { 301 __ b(GetReturnLabel()); 302 } else { 303 __ b(arm_codegen->GetLabelOf(successor_)); 304 } 305 } 306 307 Label* GetReturnLabel() { 308 DCHECK(successor_ == nullptr); 309 return &return_label_; 310 } 311 312 HBasicBlock* GetSuccessor() const { 313 return successor_; 314 } 315 316 const char* GetDescription() const OVERRIDE { return "SuspendCheckSlowPathARM"; } 317 318 private: 319 // If not null, the block to branch to after the suspend check. 320 HBasicBlock* const successor_; 321 322 // If `successor_` is null, the label to branch to after the suspend check. 323 Label return_label_; 324 325 DISALLOW_COPY_AND_ASSIGN(SuspendCheckSlowPathARM); 326}; 327 328class BoundsCheckSlowPathARM : public SlowPathCodeARM { 329 public: 330 explicit BoundsCheckSlowPathARM(HBoundsCheck* instruction) 331 : SlowPathCodeARM(instruction) {} 332 333 void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { 334 CodeGeneratorARM* arm_codegen = down_cast<CodeGeneratorARM*>(codegen); 335 LocationSummary* locations = instruction_->GetLocations(); 336 337 __ Bind(GetEntryLabel()); 338 if (instruction_->CanThrowIntoCatchBlock()) { 339 // Live registers will be restored in the catch block if caught. 340 SaveLiveRegisters(codegen, instruction_->GetLocations()); 341 } 342 // We're moving two locations to locations that could overlap, so we need a parallel 343 // move resolver. 344 InvokeRuntimeCallingConvention calling_convention; 345 codegen->EmitParallelMoves( 346 locations->InAt(0), 347 Location::RegisterLocation(calling_convention.GetRegisterAt(0)), 348 Primitive::kPrimInt, 349 locations->InAt(1), 350 Location::RegisterLocation(calling_convention.GetRegisterAt(1)), 351 Primitive::kPrimInt); 352 QuickEntrypointEnum entrypoint = instruction_->AsBoundsCheck()->IsStringCharAt() 353 ? kQuickThrowStringBounds 354 : kQuickThrowArrayBounds; 355 arm_codegen->InvokeRuntime(entrypoint, instruction_, instruction_->GetDexPc(), this); 356 CheckEntrypointTypes<kQuickThrowStringBounds, void, int32_t, int32_t>(); 357 CheckEntrypointTypes<kQuickThrowArrayBounds, void, int32_t, int32_t>(); 358 } 359 360 bool IsFatal() const OVERRIDE { return true; } 361 362 const char* GetDescription() const OVERRIDE { return "BoundsCheckSlowPathARM"; } 363 364 private: 365 DISALLOW_COPY_AND_ASSIGN(BoundsCheckSlowPathARM); 366}; 367 368class LoadClassSlowPathARM : public SlowPathCodeARM { 369 public: 370 LoadClassSlowPathARM(HLoadClass* cls, 371 HInstruction* at, 372 uint32_t dex_pc, 373 bool do_clinit) 374 : SlowPathCodeARM(at), cls_(cls), at_(at), dex_pc_(dex_pc), do_clinit_(do_clinit) { 375 DCHECK(at->IsLoadClass() || at->IsClinitCheck()); 376 } 377 378 void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { 379 LocationSummary* locations = at_->GetLocations(); 380 381 CodeGeneratorARM* arm_codegen = down_cast<CodeGeneratorARM*>(codegen); 382 __ Bind(GetEntryLabel()); 383 SaveLiveRegisters(codegen, locations); 384 385 InvokeRuntimeCallingConvention calling_convention; 386 __ LoadImmediate(calling_convention.GetRegisterAt(0), cls_->GetTypeIndex().index_); 387 QuickEntrypointEnum entrypoint = do_clinit_ ? kQuickInitializeStaticStorage 388 : kQuickInitializeType; 389 arm_codegen->InvokeRuntime(entrypoint, at_, dex_pc_, this); 390 if (do_clinit_) { 391 CheckEntrypointTypes<kQuickInitializeStaticStorage, void*, uint32_t>(); 392 } else { 393 CheckEntrypointTypes<kQuickInitializeType, void*, uint32_t>(); 394 } 395 396 // Move the class to the desired location. 397 Location out = locations->Out(); 398 if (out.IsValid()) { 399 DCHECK(out.IsRegister() && !locations->GetLiveRegisters()->ContainsCoreRegister(out.reg())); 400 arm_codegen->Move32(locations->Out(), Location::RegisterLocation(R0)); 401 } 402 RestoreLiveRegisters(codegen, locations); 403 __ b(GetExitLabel()); 404 } 405 406 const char* GetDescription() const OVERRIDE { return "LoadClassSlowPathARM"; } 407 408 private: 409 // The class this slow path will load. 410 HLoadClass* const cls_; 411 412 // The instruction where this slow path is happening. 413 // (Might be the load class or an initialization check). 414 HInstruction* const at_; 415 416 // The dex PC of `at_`. 417 const uint32_t dex_pc_; 418 419 // Whether to initialize the class. 420 const bool do_clinit_; 421 422 DISALLOW_COPY_AND_ASSIGN(LoadClassSlowPathARM); 423}; 424 425class LoadStringSlowPathARM : public SlowPathCodeARM { 426 public: 427 explicit LoadStringSlowPathARM(HLoadString* instruction) : SlowPathCodeARM(instruction) {} 428 429 void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { 430 LocationSummary* locations = instruction_->GetLocations(); 431 DCHECK(!locations->GetLiveRegisters()->ContainsCoreRegister(locations->Out().reg())); 432 HLoadString* load = instruction_->AsLoadString(); 433 const uint32_t string_index = load->GetStringIndex().index_; 434 Register out = locations->Out().AsRegister<Register>(); 435 Register temp = locations->GetTemp(0).AsRegister<Register>(); 436 constexpr bool call_saves_everything_except_r0 = (!kUseReadBarrier || kUseBakerReadBarrier); 437 438 CodeGeneratorARM* arm_codegen = down_cast<CodeGeneratorARM*>(codegen); 439 __ Bind(GetEntryLabel()); 440 SaveLiveRegisters(codegen, locations); 441 442 InvokeRuntimeCallingConvention calling_convention; 443 // In the unlucky case that the `temp` is R0, we preserve the address in `out` across 444 // the kSaveEverything call (or use `out` for the address after non-kSaveEverything call). 445 bool temp_is_r0 = (temp == calling_convention.GetRegisterAt(0)); 446 Register entry_address = temp_is_r0 ? out : temp; 447 DCHECK_NE(entry_address, calling_convention.GetRegisterAt(0)); 448 if (call_saves_everything_except_r0 && temp_is_r0) { 449 __ mov(entry_address, ShifterOperand(temp)); 450 } 451 452 __ LoadImmediate(calling_convention.GetRegisterAt(0), string_index); 453 arm_codegen->InvokeRuntime(kQuickResolveString, instruction_, instruction_->GetDexPc(), this); 454 CheckEntrypointTypes<kQuickResolveString, void*, uint32_t>(); 455 456 // Store the resolved String to the .bss entry. 457 if (call_saves_everything_except_r0) { 458 // The string entry address was preserved in `entry_address` thanks to kSaveEverything. 459 __ str(R0, Address(entry_address)); 460 } else { 461 // For non-Baker read barrier, we need to re-calculate the address of the string entry. 462 CodeGeneratorARM::PcRelativePatchInfo* labels = 463 arm_codegen->NewPcRelativeStringPatch(load->GetDexFile(), string_index); 464 __ BindTrackedLabel(&labels->movw_label); 465 __ movw(entry_address, /* placeholder */ 0u); 466 __ BindTrackedLabel(&labels->movt_label); 467 __ movt(entry_address, /* placeholder */ 0u); 468 __ BindTrackedLabel(&labels->add_pc_label); 469 __ add(entry_address, entry_address, ShifterOperand(PC)); 470 __ str(R0, Address(entry_address)); 471 } 472 473 arm_codegen->Move32(locations->Out(), Location::RegisterLocation(R0)); 474 RestoreLiveRegisters(codegen, locations); 475 476 __ b(GetExitLabel()); 477 } 478 479 const char* GetDescription() const OVERRIDE { return "LoadStringSlowPathARM"; } 480 481 private: 482 DISALLOW_COPY_AND_ASSIGN(LoadStringSlowPathARM); 483}; 484 485class TypeCheckSlowPathARM : public SlowPathCodeARM { 486 public: 487 TypeCheckSlowPathARM(HInstruction* instruction, bool is_fatal) 488 : SlowPathCodeARM(instruction), is_fatal_(is_fatal) {} 489 490 void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { 491 LocationSummary* locations = instruction_->GetLocations(); 492 DCHECK(instruction_->IsCheckCast() 493 || !locations->GetLiveRegisters()->ContainsCoreRegister(locations->Out().reg())); 494 495 CodeGeneratorARM* arm_codegen = down_cast<CodeGeneratorARM*>(codegen); 496 __ Bind(GetEntryLabel()); 497 498 if (!is_fatal_) { 499 SaveLiveRegisters(codegen, locations); 500 } 501 502 // We're moving two locations to locations that could overlap, so we need a parallel 503 // move resolver. 504 InvokeRuntimeCallingConvention calling_convention; 505 codegen->EmitParallelMoves(locations->InAt(0), 506 Location::RegisterLocation(calling_convention.GetRegisterAt(0)), 507 Primitive::kPrimNot, 508 locations->InAt(1), 509 Location::RegisterLocation(calling_convention.GetRegisterAt(1)), 510 Primitive::kPrimNot); 511 if (instruction_->IsInstanceOf()) { 512 arm_codegen->InvokeRuntime(kQuickInstanceofNonTrivial, 513 instruction_, 514 instruction_->GetDexPc(), 515 this); 516 CheckEntrypointTypes<kQuickInstanceofNonTrivial, size_t, mirror::Object*, mirror::Class*>(); 517 arm_codegen->Move32(locations->Out(), Location::RegisterLocation(R0)); 518 } else { 519 DCHECK(instruction_->IsCheckCast()); 520 arm_codegen->InvokeRuntime(kQuickCheckInstanceOf, 521 instruction_, 522 instruction_->GetDexPc(), 523 this); 524 CheckEntrypointTypes<kQuickCheckInstanceOf, void, mirror::Object*, mirror::Class*>(); 525 } 526 527 if (!is_fatal_) { 528 RestoreLiveRegisters(codegen, locations); 529 __ b(GetExitLabel()); 530 } 531 } 532 533 const char* GetDescription() const OVERRIDE { return "TypeCheckSlowPathARM"; } 534 535 bool IsFatal() const OVERRIDE { return is_fatal_; } 536 537 private: 538 const bool is_fatal_; 539 540 DISALLOW_COPY_AND_ASSIGN(TypeCheckSlowPathARM); 541}; 542 543class DeoptimizationSlowPathARM : public SlowPathCodeARM { 544 public: 545 explicit DeoptimizationSlowPathARM(HDeoptimize* instruction) 546 : SlowPathCodeARM(instruction) {} 547 548 void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { 549 CodeGeneratorARM* arm_codegen = down_cast<CodeGeneratorARM*>(codegen); 550 __ Bind(GetEntryLabel()); 551 arm_codegen->InvokeRuntime(kQuickDeoptimize, instruction_, instruction_->GetDexPc(), this); 552 CheckEntrypointTypes<kQuickDeoptimize, void, void>(); 553 } 554 555 const char* GetDescription() const OVERRIDE { return "DeoptimizationSlowPathARM"; } 556 557 private: 558 DISALLOW_COPY_AND_ASSIGN(DeoptimizationSlowPathARM); 559}; 560 561class ArraySetSlowPathARM : public SlowPathCodeARM { 562 public: 563 explicit ArraySetSlowPathARM(HInstruction* instruction) : SlowPathCodeARM(instruction) {} 564 565 void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { 566 LocationSummary* locations = instruction_->GetLocations(); 567 __ Bind(GetEntryLabel()); 568 SaveLiveRegisters(codegen, locations); 569 570 InvokeRuntimeCallingConvention calling_convention; 571 HParallelMove parallel_move(codegen->GetGraph()->GetArena()); 572 parallel_move.AddMove( 573 locations->InAt(0), 574 Location::RegisterLocation(calling_convention.GetRegisterAt(0)), 575 Primitive::kPrimNot, 576 nullptr); 577 parallel_move.AddMove( 578 locations->InAt(1), 579 Location::RegisterLocation(calling_convention.GetRegisterAt(1)), 580 Primitive::kPrimInt, 581 nullptr); 582 parallel_move.AddMove( 583 locations->InAt(2), 584 Location::RegisterLocation(calling_convention.GetRegisterAt(2)), 585 Primitive::kPrimNot, 586 nullptr); 587 codegen->GetMoveResolver()->EmitNativeCode(¶llel_move); 588 589 CodeGeneratorARM* arm_codegen = down_cast<CodeGeneratorARM*>(codegen); 590 arm_codegen->InvokeRuntime(kQuickAputObject, instruction_, instruction_->GetDexPc(), this); 591 CheckEntrypointTypes<kQuickAputObject, void, mirror::Array*, int32_t, mirror::Object*>(); 592 RestoreLiveRegisters(codegen, locations); 593 __ b(GetExitLabel()); 594 } 595 596 const char* GetDescription() const OVERRIDE { return "ArraySetSlowPathARM"; } 597 598 private: 599 DISALLOW_COPY_AND_ASSIGN(ArraySetSlowPathARM); 600}; 601 602// Slow path marking an object reference `ref` during a read 603// barrier. The field `obj.field` in the object `obj` holding this 604// reference does not get updated by this slow path after marking (see 605// ReadBarrierMarkAndUpdateFieldSlowPathARM below for that). 606// 607// This means that after the execution of this slow path, `ref` will 608// always be up-to-date, but `obj.field` may not; i.e., after the 609// flip, `ref` will be a to-space reference, but `obj.field` will 610// probably still be a from-space reference (unless it gets updated by 611// another thread, or if another thread installed another object 612// reference (different from `ref`) in `obj.field`). 613class ReadBarrierMarkSlowPathARM : public SlowPathCodeARM { 614 public: 615 ReadBarrierMarkSlowPathARM(HInstruction* instruction, 616 Location ref, 617 Location entrypoint = Location::NoLocation()) 618 : SlowPathCodeARM(instruction), ref_(ref), entrypoint_(entrypoint) { 619 DCHECK(kEmitCompilerReadBarrier); 620 } 621 622 const char* GetDescription() const OVERRIDE { return "ReadBarrierMarkSlowPathARM"; } 623 624 void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { 625 LocationSummary* locations = instruction_->GetLocations(); 626 Register ref_reg = ref_.AsRegister<Register>(); 627 DCHECK(locations->CanCall()); 628 DCHECK(!locations->GetLiveRegisters()->ContainsCoreRegister(ref_reg)) << ref_reg; 629 DCHECK(instruction_->IsInstanceFieldGet() || 630 instruction_->IsStaticFieldGet() || 631 instruction_->IsArrayGet() || 632 instruction_->IsArraySet() || 633 instruction_->IsLoadClass() || 634 instruction_->IsLoadString() || 635 instruction_->IsInstanceOf() || 636 instruction_->IsCheckCast() || 637 (instruction_->IsInvokeVirtual() && instruction_->GetLocations()->Intrinsified()) || 638 (instruction_->IsInvokeStaticOrDirect() && instruction_->GetLocations()->Intrinsified())) 639 << "Unexpected instruction in read barrier marking slow path: " 640 << instruction_->DebugName(); 641 // The read barrier instrumentation of object ArrayGet 642 // instructions does not support the HIntermediateAddress 643 // instruction. 644 DCHECK(!(instruction_->IsArrayGet() && 645 instruction_->AsArrayGet()->GetArray()->IsIntermediateAddress())); 646 647 __ Bind(GetEntryLabel()); 648 // No need to save live registers; it's taken care of by the 649 // entrypoint. Also, there is no need to update the stack mask, 650 // as this runtime call will not trigger a garbage collection. 651 CodeGeneratorARM* arm_codegen = down_cast<CodeGeneratorARM*>(codegen); 652 DCHECK_NE(ref_reg, SP); 653 DCHECK_NE(ref_reg, LR); 654 DCHECK_NE(ref_reg, PC); 655 // IP is used internally by the ReadBarrierMarkRegX entry point 656 // as a temporary, it cannot be the entry point's input/output. 657 DCHECK_NE(ref_reg, IP); 658 DCHECK(0 <= ref_reg && ref_reg < kNumberOfCoreRegisters) << ref_reg; 659 // "Compact" slow path, saving two moves. 660 // 661 // Instead of using the standard runtime calling convention (input 662 // and output in R0): 663 // 664 // R0 <- ref 665 // R0 <- ReadBarrierMark(R0) 666 // ref <- R0 667 // 668 // we just use rX (the register containing `ref`) as input and output 669 // of a dedicated entrypoint: 670 // 671 // rX <- ReadBarrierMarkRegX(rX) 672 // 673 if (entrypoint_.IsValid()) { 674 arm_codegen->ValidateInvokeRuntimeWithoutRecordingPcInfo(instruction_, this); 675 __ blx(entrypoint_.AsRegister<Register>()); 676 } else { 677 int32_t entry_point_offset = 678 CodeGenerator::GetReadBarrierMarkEntryPointsOffset<kArmPointerSize>(ref_reg); 679 // This runtime call does not require a stack map. 680 arm_codegen->InvokeRuntimeWithoutRecordingPcInfo(entry_point_offset, instruction_, this); 681 } 682 __ b(GetExitLabel()); 683 } 684 685 private: 686 // The location (register) of the marked object reference. 687 const Location ref_; 688 689 // The location of the entrypoint if already loaded. 690 const Location entrypoint_; 691 692 DISALLOW_COPY_AND_ASSIGN(ReadBarrierMarkSlowPathARM); 693}; 694 695// Slow path marking an object reference `ref` during a read barrier, 696// and if needed, atomically updating the field `obj.field` in the 697// object `obj` holding this reference after marking (contrary to 698// ReadBarrierMarkSlowPathARM above, which never tries to update 699// `obj.field`). 700// 701// This means that after the execution of this slow path, both `ref` 702// and `obj.field` will be up-to-date; i.e., after the flip, both will 703// hold the same to-space reference (unless another thread installed 704// another object reference (different from `ref`) in `obj.field`). 705class ReadBarrierMarkAndUpdateFieldSlowPathARM : public SlowPathCodeARM { 706 public: 707 ReadBarrierMarkAndUpdateFieldSlowPathARM(HInstruction* instruction, 708 Location ref, 709 Register obj, 710 Location field_offset, 711 Register temp1, 712 Register temp2) 713 : SlowPathCodeARM(instruction), 714 ref_(ref), 715 obj_(obj), 716 field_offset_(field_offset), 717 temp1_(temp1), 718 temp2_(temp2) { 719 DCHECK(kEmitCompilerReadBarrier); 720 } 721 722 const char* GetDescription() const OVERRIDE { return "ReadBarrierMarkAndUpdateFieldSlowPathARM"; } 723 724 void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { 725 LocationSummary* locations = instruction_->GetLocations(); 726 Register ref_reg = ref_.AsRegister<Register>(); 727 DCHECK(locations->CanCall()); 728 DCHECK(!locations->GetLiveRegisters()->ContainsCoreRegister(ref_reg)) << ref_reg; 729 // This slow path is only used by the UnsafeCASObject intrinsic. 730 DCHECK((instruction_->IsInvokeVirtual() && instruction_->GetLocations()->Intrinsified())) 731 << "Unexpected instruction in read barrier marking and field updating slow path: " 732 << instruction_->DebugName(); 733 DCHECK(instruction_->GetLocations()->Intrinsified()); 734 DCHECK_EQ(instruction_->AsInvoke()->GetIntrinsic(), Intrinsics::kUnsafeCASObject); 735 DCHECK(field_offset_.IsRegisterPair()) << field_offset_; 736 737 __ Bind(GetEntryLabel()); 738 739 // Save the old reference. 740 // Note that we cannot use IP to save the old reference, as IP is 741 // used internally by the ReadBarrierMarkRegX entry point, and we 742 // need the old reference after the call to that entry point. 743 DCHECK_NE(temp1_, IP); 744 __ Mov(temp1_, ref_reg); 745 746 // No need to save live registers; it's taken care of by the 747 // entrypoint. Also, there is no need to update the stack mask, 748 // as this runtime call will not trigger a garbage collection. 749 CodeGeneratorARM* arm_codegen = down_cast<CodeGeneratorARM*>(codegen); 750 DCHECK_NE(ref_reg, SP); 751 DCHECK_NE(ref_reg, LR); 752 DCHECK_NE(ref_reg, PC); 753 // IP is used internally by the ReadBarrierMarkRegX entry point 754 // as a temporary, it cannot be the entry point's input/output. 755 DCHECK_NE(ref_reg, IP); 756 DCHECK(0 <= ref_reg && ref_reg < kNumberOfCoreRegisters) << ref_reg; 757 // "Compact" slow path, saving two moves. 758 // 759 // Instead of using the standard runtime calling convention (input 760 // and output in R0): 761 // 762 // R0 <- ref 763 // R0 <- ReadBarrierMark(R0) 764 // ref <- R0 765 // 766 // we just use rX (the register containing `ref`) as input and output 767 // of a dedicated entrypoint: 768 // 769 // rX <- ReadBarrierMarkRegX(rX) 770 // 771 int32_t entry_point_offset = 772 CodeGenerator::GetReadBarrierMarkEntryPointsOffset<kArmPointerSize>(ref_reg); 773 // This runtime call does not require a stack map. 774 arm_codegen->InvokeRuntimeWithoutRecordingPcInfo(entry_point_offset, instruction_, this); 775 776 // If the new reference is different from the old reference, 777 // update the field in the holder (`*(obj_ + field_offset_)`). 778 // 779 // Note that this field could also hold a different object, if 780 // another thread had concurrently changed it. In that case, the 781 // LDREX/SUBS/ITNE sequence of instructions in the compare-and-set 782 // (CAS) operation below would abort the CAS, leaving the field 783 // as-is. 784 Label done; 785 __ cmp(temp1_, ShifterOperand(ref_reg)); 786 __ b(&done, EQ); 787 788 // Update the the holder's field atomically. This may fail if 789 // mutator updates before us, but it's OK. This is achieved 790 // using a strong compare-and-set (CAS) operation with relaxed 791 // memory synchronization ordering, where the expected value is 792 // the old reference and the desired value is the new reference. 793 794 // Convenience aliases. 795 Register base = obj_; 796 // The UnsafeCASObject intrinsic uses a register pair as field 797 // offset ("long offset"), of which only the low part contains 798 // data. 799 Register offset = field_offset_.AsRegisterPairLow<Register>(); 800 Register expected = temp1_; 801 Register value = ref_reg; 802 Register tmp_ptr = IP; // Pointer to actual memory. 803 Register tmp = temp2_; // Value in memory. 804 805 __ add(tmp_ptr, base, ShifterOperand(offset)); 806 807 if (kPoisonHeapReferences) { 808 __ PoisonHeapReference(expected); 809 if (value == expected) { 810 // Do not poison `value`, as it is the same register as 811 // `expected`, which has just been poisoned. 812 } else { 813 __ PoisonHeapReference(value); 814 } 815 } 816 817 // do { 818 // tmp = [r_ptr] - expected; 819 // } while (tmp == 0 && failure([r_ptr] <- r_new_value)); 820 821 Label loop_head, exit_loop; 822 __ Bind(&loop_head); 823 824 __ ldrex(tmp, tmp_ptr); 825 826 __ subs(tmp, tmp, ShifterOperand(expected)); 827 828 __ it(NE); 829 __ clrex(NE); 830 831 __ b(&exit_loop, NE); 832 833 __ strex(tmp, value, tmp_ptr); 834 __ cmp(tmp, ShifterOperand(1)); 835 __ b(&loop_head, EQ); 836 837 __ Bind(&exit_loop); 838 839 if (kPoisonHeapReferences) { 840 __ UnpoisonHeapReference(expected); 841 if (value == expected) { 842 // Do not unpoison `value`, as it is the same register as 843 // `expected`, which has just been unpoisoned. 844 } else { 845 __ UnpoisonHeapReference(value); 846 } 847 } 848 849 __ Bind(&done); 850 __ b(GetExitLabel()); 851 } 852 853 private: 854 // The location (register) of the marked object reference. 855 const Location ref_; 856 // The register containing the object holding the marked object reference field. 857 const Register obj_; 858 // The location of the offset of the marked reference field within `obj_`. 859 Location field_offset_; 860 861 const Register temp1_; 862 const Register temp2_; 863 864 DISALLOW_COPY_AND_ASSIGN(ReadBarrierMarkAndUpdateFieldSlowPathARM); 865}; 866 867// Slow path generating a read barrier for a heap reference. 868class ReadBarrierForHeapReferenceSlowPathARM : public SlowPathCodeARM { 869 public: 870 ReadBarrierForHeapReferenceSlowPathARM(HInstruction* instruction, 871 Location out, 872 Location ref, 873 Location obj, 874 uint32_t offset, 875 Location index) 876 : SlowPathCodeARM(instruction), 877 out_(out), 878 ref_(ref), 879 obj_(obj), 880 offset_(offset), 881 index_(index) { 882 DCHECK(kEmitCompilerReadBarrier); 883 // If `obj` is equal to `out` or `ref`, it means the initial object 884 // has been overwritten by (or after) the heap object reference load 885 // to be instrumented, e.g.: 886 // 887 // __ LoadFromOffset(kLoadWord, out, out, offset); 888 // codegen_->GenerateReadBarrierSlow(instruction, out_loc, out_loc, out_loc, offset); 889 // 890 // In that case, we have lost the information about the original 891 // object, and the emitted read barrier cannot work properly. 892 DCHECK(!obj.Equals(out)) << "obj=" << obj << " out=" << out; 893 DCHECK(!obj.Equals(ref)) << "obj=" << obj << " ref=" << ref; 894 } 895 896 void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { 897 CodeGeneratorARM* arm_codegen = down_cast<CodeGeneratorARM*>(codegen); 898 LocationSummary* locations = instruction_->GetLocations(); 899 Register reg_out = out_.AsRegister<Register>(); 900 DCHECK(locations->CanCall()); 901 DCHECK(!locations->GetLiveRegisters()->ContainsCoreRegister(reg_out)); 902 DCHECK(instruction_->IsInstanceFieldGet() || 903 instruction_->IsStaticFieldGet() || 904 instruction_->IsArrayGet() || 905 instruction_->IsInstanceOf() || 906 instruction_->IsCheckCast() || 907 (instruction_->IsInvokeVirtual()) && instruction_->GetLocations()->Intrinsified()) 908 << "Unexpected instruction in read barrier for heap reference slow path: " 909 << instruction_->DebugName(); 910 // The read barrier instrumentation of object ArrayGet 911 // instructions does not support the HIntermediateAddress 912 // instruction. 913 DCHECK(!(instruction_->IsArrayGet() && 914 instruction_->AsArrayGet()->GetArray()->IsIntermediateAddress())); 915 916 __ Bind(GetEntryLabel()); 917 SaveLiveRegisters(codegen, locations); 918 919 // We may have to change the index's value, but as `index_` is a 920 // constant member (like other "inputs" of this slow path), 921 // introduce a copy of it, `index`. 922 Location index = index_; 923 if (index_.IsValid()) { 924 // Handle `index_` for HArrayGet and UnsafeGetObject/UnsafeGetObjectVolatile intrinsics. 925 if (instruction_->IsArrayGet()) { 926 // Compute the actual memory offset and store it in `index`. 927 Register index_reg = index_.AsRegister<Register>(); 928 DCHECK(locations->GetLiveRegisters()->ContainsCoreRegister(index_reg)); 929 if (codegen->IsCoreCalleeSaveRegister(index_reg)) { 930 // We are about to change the value of `index_reg` (see the 931 // calls to art::arm::Thumb2Assembler::Lsl and 932 // art::arm::Thumb2Assembler::AddConstant below), but it has 933 // not been saved by the previous call to 934 // art::SlowPathCode::SaveLiveRegisters, as it is a 935 // callee-save register -- 936 // art::SlowPathCode::SaveLiveRegisters does not consider 937 // callee-save registers, as it has been designed with the 938 // assumption that callee-save registers are supposed to be 939 // handled by the called function. So, as a callee-save 940 // register, `index_reg` _would_ eventually be saved onto 941 // the stack, but it would be too late: we would have 942 // changed its value earlier. Therefore, we manually save 943 // it here into another freely available register, 944 // `free_reg`, chosen of course among the caller-save 945 // registers (as a callee-save `free_reg` register would 946 // exhibit the same problem). 947 // 948 // Note we could have requested a temporary register from 949 // the register allocator instead; but we prefer not to, as 950 // this is a slow path, and we know we can find a 951 // caller-save register that is available. 952 Register free_reg = FindAvailableCallerSaveRegister(codegen); 953 __ Mov(free_reg, index_reg); 954 index_reg = free_reg; 955 index = Location::RegisterLocation(index_reg); 956 } else { 957 // The initial register stored in `index_` has already been 958 // saved in the call to art::SlowPathCode::SaveLiveRegisters 959 // (as it is not a callee-save register), so we can freely 960 // use it. 961 } 962 // Shifting the index value contained in `index_reg` by the scale 963 // factor (2) cannot overflow in practice, as the runtime is 964 // unable to allocate object arrays with a size larger than 965 // 2^26 - 1 (that is, 2^28 - 4 bytes). 966 __ Lsl(index_reg, index_reg, TIMES_4); 967 static_assert( 968 sizeof(mirror::HeapReference<mirror::Object>) == sizeof(int32_t), 969 "art::mirror::HeapReference<art::mirror::Object> and int32_t have different sizes."); 970 __ AddConstant(index_reg, index_reg, offset_); 971 } else { 972 // In the case of the UnsafeGetObject/UnsafeGetObjectVolatile 973 // intrinsics, `index_` is not shifted by a scale factor of 2 974 // (as in the case of ArrayGet), as it is actually an offset 975 // to an object field within an object. 976 DCHECK(instruction_->IsInvoke()) << instruction_->DebugName(); 977 DCHECK(instruction_->GetLocations()->Intrinsified()); 978 DCHECK((instruction_->AsInvoke()->GetIntrinsic() == Intrinsics::kUnsafeGetObject) || 979 (instruction_->AsInvoke()->GetIntrinsic() == Intrinsics::kUnsafeGetObjectVolatile)) 980 << instruction_->AsInvoke()->GetIntrinsic(); 981 DCHECK_EQ(offset_, 0U); 982 DCHECK(index_.IsRegisterPair()); 983 // UnsafeGet's offset location is a register pair, the low 984 // part contains the correct offset. 985 index = index_.ToLow(); 986 } 987 } 988 989 // We're moving two or three locations to locations that could 990 // overlap, so we need a parallel move resolver. 991 InvokeRuntimeCallingConvention calling_convention; 992 HParallelMove parallel_move(codegen->GetGraph()->GetArena()); 993 parallel_move.AddMove(ref_, 994 Location::RegisterLocation(calling_convention.GetRegisterAt(0)), 995 Primitive::kPrimNot, 996 nullptr); 997 parallel_move.AddMove(obj_, 998 Location::RegisterLocation(calling_convention.GetRegisterAt(1)), 999 Primitive::kPrimNot, 1000 nullptr); 1001 if (index.IsValid()) { 1002 parallel_move.AddMove(index, 1003 Location::RegisterLocation(calling_convention.GetRegisterAt(2)), 1004 Primitive::kPrimInt, 1005 nullptr); 1006 codegen->GetMoveResolver()->EmitNativeCode(¶llel_move); 1007 } else { 1008 codegen->GetMoveResolver()->EmitNativeCode(¶llel_move); 1009 __ LoadImmediate(calling_convention.GetRegisterAt(2), offset_); 1010 } 1011 arm_codegen->InvokeRuntime(kQuickReadBarrierSlow, instruction_, instruction_->GetDexPc(), this); 1012 CheckEntrypointTypes< 1013 kQuickReadBarrierSlow, mirror::Object*, mirror::Object*, mirror::Object*, uint32_t>(); 1014 arm_codegen->Move32(out_, Location::RegisterLocation(R0)); 1015 1016 RestoreLiveRegisters(codegen, locations); 1017 __ b(GetExitLabel()); 1018 } 1019 1020 const char* GetDescription() const OVERRIDE { return "ReadBarrierForHeapReferenceSlowPathARM"; } 1021 1022 private: 1023 Register FindAvailableCallerSaveRegister(CodeGenerator* codegen) { 1024 size_t ref = static_cast<int>(ref_.AsRegister<Register>()); 1025 size_t obj = static_cast<int>(obj_.AsRegister<Register>()); 1026 for (size_t i = 0, e = codegen->GetNumberOfCoreRegisters(); i < e; ++i) { 1027 if (i != ref && i != obj && !codegen->IsCoreCalleeSaveRegister(i)) { 1028 return static_cast<Register>(i); 1029 } 1030 } 1031 // We shall never fail to find a free caller-save register, as 1032 // there are more than two core caller-save registers on ARM 1033 // (meaning it is possible to find one which is different from 1034 // `ref` and `obj`). 1035 DCHECK_GT(codegen->GetNumberOfCoreCallerSaveRegisters(), 2u); 1036 LOG(FATAL) << "Could not find a free caller-save register"; 1037 UNREACHABLE(); 1038 } 1039 1040 const Location out_; 1041 const Location ref_; 1042 const Location obj_; 1043 const uint32_t offset_; 1044 // An additional location containing an index to an array. 1045 // Only used for HArrayGet and the UnsafeGetObject & 1046 // UnsafeGetObjectVolatile intrinsics. 1047 const Location index_; 1048 1049 DISALLOW_COPY_AND_ASSIGN(ReadBarrierForHeapReferenceSlowPathARM); 1050}; 1051 1052// Slow path generating a read barrier for a GC root. 1053class ReadBarrierForRootSlowPathARM : public SlowPathCodeARM { 1054 public: 1055 ReadBarrierForRootSlowPathARM(HInstruction* instruction, Location out, Location root) 1056 : SlowPathCodeARM(instruction), out_(out), root_(root) { 1057 DCHECK(kEmitCompilerReadBarrier); 1058 } 1059 1060 void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { 1061 LocationSummary* locations = instruction_->GetLocations(); 1062 Register reg_out = out_.AsRegister<Register>(); 1063 DCHECK(locations->CanCall()); 1064 DCHECK(!locations->GetLiveRegisters()->ContainsCoreRegister(reg_out)); 1065 DCHECK(instruction_->IsLoadClass() || instruction_->IsLoadString()) 1066 << "Unexpected instruction in read barrier for GC root slow path: " 1067 << instruction_->DebugName(); 1068 1069 __ Bind(GetEntryLabel()); 1070 SaveLiveRegisters(codegen, locations); 1071 1072 InvokeRuntimeCallingConvention calling_convention; 1073 CodeGeneratorARM* arm_codegen = down_cast<CodeGeneratorARM*>(codegen); 1074 arm_codegen->Move32(Location::RegisterLocation(calling_convention.GetRegisterAt(0)), root_); 1075 arm_codegen->InvokeRuntime(kQuickReadBarrierForRootSlow, 1076 instruction_, 1077 instruction_->GetDexPc(), 1078 this); 1079 CheckEntrypointTypes<kQuickReadBarrierForRootSlow, mirror::Object*, GcRoot<mirror::Object>*>(); 1080 arm_codegen->Move32(out_, Location::RegisterLocation(R0)); 1081 1082 RestoreLiveRegisters(codegen, locations); 1083 __ b(GetExitLabel()); 1084 } 1085 1086 const char* GetDescription() const OVERRIDE { return "ReadBarrierForRootSlowPathARM"; } 1087 1088 private: 1089 const Location out_; 1090 const Location root_; 1091 1092 DISALLOW_COPY_AND_ASSIGN(ReadBarrierForRootSlowPathARM); 1093}; 1094 1095#undef __ 1096// NOLINT on __ macro to suppress wrong warning/fix (misc-macro-parentheses) from clang-tidy. 1097#define __ down_cast<ArmAssembler*>(GetAssembler())-> // NOLINT 1098 1099inline Condition ARMCondition(IfCondition cond) { 1100 switch (cond) { 1101 case kCondEQ: return EQ; 1102 case kCondNE: return NE; 1103 case kCondLT: return LT; 1104 case kCondLE: return LE; 1105 case kCondGT: return GT; 1106 case kCondGE: return GE; 1107 case kCondB: return LO; 1108 case kCondBE: return LS; 1109 case kCondA: return HI; 1110 case kCondAE: return HS; 1111 } 1112 LOG(FATAL) << "Unreachable"; 1113 UNREACHABLE(); 1114} 1115 1116// Maps signed condition to unsigned condition. 1117inline Condition ARMUnsignedCondition(IfCondition cond) { 1118 switch (cond) { 1119 case kCondEQ: return EQ; 1120 case kCondNE: return NE; 1121 // Signed to unsigned. 1122 case kCondLT: return LO; 1123 case kCondLE: return LS; 1124 case kCondGT: return HI; 1125 case kCondGE: return HS; 1126 // Unsigned remain unchanged. 1127 case kCondB: return LO; 1128 case kCondBE: return LS; 1129 case kCondA: return HI; 1130 case kCondAE: return HS; 1131 } 1132 LOG(FATAL) << "Unreachable"; 1133 UNREACHABLE(); 1134} 1135 1136inline Condition ARMFPCondition(IfCondition cond, bool gt_bias) { 1137 // The ARM condition codes can express all the necessary branches, see the 1138 // "Meaning (floating-point)" column in the table A8-1 of the ARMv7 reference manual. 1139 // There is no dex instruction or HIR that would need the missing conditions 1140 // "equal or unordered" or "not equal". 1141 switch (cond) { 1142 case kCondEQ: return EQ; 1143 case kCondNE: return NE /* unordered */; 1144 case kCondLT: return gt_bias ? CC : LT /* unordered */; 1145 case kCondLE: return gt_bias ? LS : LE /* unordered */; 1146 case kCondGT: return gt_bias ? HI /* unordered */ : GT; 1147 case kCondGE: return gt_bias ? CS /* unordered */ : GE; 1148 default: 1149 LOG(FATAL) << "UNREACHABLE"; 1150 UNREACHABLE(); 1151 } 1152} 1153 1154void CodeGeneratorARM::DumpCoreRegister(std::ostream& stream, int reg) const { 1155 stream << Register(reg); 1156} 1157 1158void CodeGeneratorARM::DumpFloatingPointRegister(std::ostream& stream, int reg) const { 1159 stream << SRegister(reg); 1160} 1161 1162size_t CodeGeneratorARM::SaveCoreRegister(size_t stack_index, uint32_t reg_id) { 1163 __ StoreToOffset(kStoreWord, static_cast<Register>(reg_id), SP, stack_index); 1164 return kArmWordSize; 1165} 1166 1167size_t CodeGeneratorARM::RestoreCoreRegister(size_t stack_index, uint32_t reg_id) { 1168 __ LoadFromOffset(kLoadWord, static_cast<Register>(reg_id), SP, stack_index); 1169 return kArmWordSize; 1170} 1171 1172size_t CodeGeneratorARM::SaveFloatingPointRegister(size_t stack_index, uint32_t reg_id) { 1173 __ StoreSToOffset(static_cast<SRegister>(reg_id), SP, stack_index); 1174 return kArmWordSize; 1175} 1176 1177size_t CodeGeneratorARM::RestoreFloatingPointRegister(size_t stack_index, uint32_t reg_id) { 1178 __ LoadSFromOffset(static_cast<SRegister>(reg_id), SP, stack_index); 1179 return kArmWordSize; 1180} 1181 1182CodeGeneratorARM::CodeGeneratorARM(HGraph* graph, 1183 const ArmInstructionSetFeatures& isa_features, 1184 const CompilerOptions& compiler_options, 1185 OptimizingCompilerStats* stats) 1186 : CodeGenerator(graph, 1187 kNumberOfCoreRegisters, 1188 kNumberOfSRegisters, 1189 kNumberOfRegisterPairs, 1190 ComputeRegisterMask(reinterpret_cast<const int*>(kCoreCalleeSaves), 1191 arraysize(kCoreCalleeSaves)), 1192 ComputeRegisterMask(reinterpret_cast<const int*>(kFpuCalleeSaves), 1193 arraysize(kFpuCalleeSaves)), 1194 compiler_options, 1195 stats), 1196 block_labels_(nullptr), 1197 location_builder_(graph, this), 1198 instruction_visitor_(graph, this), 1199 move_resolver_(graph->GetArena(), this), 1200 assembler_(graph->GetArena()), 1201 isa_features_(isa_features), 1202 uint32_literals_(std::less<uint32_t>(), 1203 graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), 1204 pc_relative_dex_cache_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), 1205 boot_image_string_patches_(StringReferenceValueComparator(), 1206 graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), 1207 pc_relative_string_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), 1208 boot_image_type_patches_(TypeReferenceValueComparator(), 1209 graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), 1210 pc_relative_type_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), 1211 boot_image_address_patches_(std::less<uint32_t>(), 1212 graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), 1213 jit_string_patches_(StringReferenceValueComparator(), 1214 graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), 1215 jit_class_patches_(TypeReferenceValueComparator(), 1216 graph->GetArena()->Adapter(kArenaAllocCodeGenerator)) { 1217 // Always save the LR register to mimic Quick. 1218 AddAllocatedRegister(Location::RegisterLocation(LR)); 1219} 1220 1221void CodeGeneratorARM::Finalize(CodeAllocator* allocator) { 1222 // Ensure that we fix up branches and literal loads and emit the literal pool. 1223 __ FinalizeCode(); 1224 1225 // Adjust native pc offsets in stack maps. 1226 for (size_t i = 0, num = stack_map_stream_.GetNumberOfStackMaps(); i != num; ++i) { 1227 uint32_t old_position = stack_map_stream_.GetStackMap(i).native_pc_offset; 1228 uint32_t new_position = __ GetAdjustedPosition(old_position); 1229 stack_map_stream_.SetStackMapNativePcOffset(i, new_position); 1230 } 1231 // Adjust pc offsets for the disassembly information. 1232 if (disasm_info_ != nullptr) { 1233 GeneratedCodeInterval* frame_entry_interval = disasm_info_->GetFrameEntryInterval(); 1234 frame_entry_interval->start = __ GetAdjustedPosition(frame_entry_interval->start); 1235 frame_entry_interval->end = __ GetAdjustedPosition(frame_entry_interval->end); 1236 for (auto& it : *disasm_info_->GetInstructionIntervals()) { 1237 it.second.start = __ GetAdjustedPosition(it.second.start); 1238 it.second.end = __ GetAdjustedPosition(it.second.end); 1239 } 1240 for (auto& it : *disasm_info_->GetSlowPathIntervals()) { 1241 it.code_interval.start = __ GetAdjustedPosition(it.code_interval.start); 1242 it.code_interval.end = __ GetAdjustedPosition(it.code_interval.end); 1243 } 1244 } 1245 1246 CodeGenerator::Finalize(allocator); 1247} 1248 1249void CodeGeneratorARM::SetupBlockedRegisters() const { 1250 // Stack register, LR and PC are always reserved. 1251 blocked_core_registers_[SP] = true; 1252 blocked_core_registers_[LR] = true; 1253 blocked_core_registers_[PC] = true; 1254 1255 // Reserve thread register. 1256 blocked_core_registers_[TR] = true; 1257 1258 // Reserve temp register. 1259 blocked_core_registers_[IP] = true; 1260 1261 if (GetGraph()->IsDebuggable()) { 1262 // Stubs do not save callee-save floating point registers. If the graph 1263 // is debuggable, we need to deal with these registers differently. For 1264 // now, just block them. 1265 for (size_t i = 0; i < arraysize(kFpuCalleeSaves); ++i) { 1266 blocked_fpu_registers_[kFpuCalleeSaves[i]] = true; 1267 } 1268 } 1269} 1270 1271InstructionCodeGeneratorARM::InstructionCodeGeneratorARM(HGraph* graph, CodeGeneratorARM* codegen) 1272 : InstructionCodeGenerator(graph, codegen), 1273 assembler_(codegen->GetAssembler()), 1274 codegen_(codegen) {} 1275 1276void CodeGeneratorARM::ComputeSpillMask() { 1277 core_spill_mask_ = allocated_registers_.GetCoreRegisters() & core_callee_save_mask_; 1278 DCHECK_NE(core_spill_mask_, 0u) << "At least the return address register must be saved"; 1279 // There is no easy instruction to restore just the PC on thumb2. We spill and 1280 // restore another arbitrary register. 1281 core_spill_mask_ |= (1 << kCoreAlwaysSpillRegister); 1282 fpu_spill_mask_ = allocated_registers_.GetFloatingPointRegisters() & fpu_callee_save_mask_; 1283 // We use vpush and vpop for saving and restoring floating point registers, which take 1284 // a SRegister and the number of registers to save/restore after that SRegister. We 1285 // therefore update the `fpu_spill_mask_` to also contain those registers not allocated, 1286 // but in the range. 1287 if (fpu_spill_mask_ != 0) { 1288 uint32_t least_significant_bit = LeastSignificantBit(fpu_spill_mask_); 1289 uint32_t most_significant_bit = MostSignificantBit(fpu_spill_mask_); 1290 for (uint32_t i = least_significant_bit + 1 ; i < most_significant_bit; ++i) { 1291 fpu_spill_mask_ |= (1 << i); 1292 } 1293 } 1294} 1295 1296static dwarf::Reg DWARFReg(Register reg) { 1297 return dwarf::Reg::ArmCore(static_cast<int>(reg)); 1298} 1299 1300static dwarf::Reg DWARFReg(SRegister reg) { 1301 return dwarf::Reg::ArmFp(static_cast<int>(reg)); 1302} 1303 1304void CodeGeneratorARM::GenerateFrameEntry() { 1305 bool skip_overflow_check = 1306 IsLeafMethod() && !FrameNeedsStackCheck(GetFrameSize(), InstructionSet::kArm); 1307 DCHECK(GetCompilerOptions().GetImplicitStackOverflowChecks()); 1308 __ Bind(&frame_entry_label_); 1309 1310 if (HasEmptyFrame()) { 1311 return; 1312 } 1313 1314 if (!skip_overflow_check) { 1315 __ AddConstant(IP, SP, -static_cast<int32_t>(GetStackOverflowReservedBytes(kArm))); 1316 __ LoadFromOffset(kLoadWord, IP, IP, 0); 1317 RecordPcInfo(nullptr, 0); 1318 } 1319 1320 __ PushList(core_spill_mask_); 1321 __ cfi().AdjustCFAOffset(kArmWordSize * POPCOUNT(core_spill_mask_)); 1322 __ cfi().RelOffsetForMany(DWARFReg(kMethodRegisterArgument), 0, core_spill_mask_, kArmWordSize); 1323 if (fpu_spill_mask_ != 0) { 1324 SRegister start_register = SRegister(LeastSignificantBit(fpu_spill_mask_)); 1325 __ vpushs(start_register, POPCOUNT(fpu_spill_mask_)); 1326 __ cfi().AdjustCFAOffset(kArmWordSize * POPCOUNT(fpu_spill_mask_)); 1327 __ cfi().RelOffsetForMany(DWARFReg(S0), 0, fpu_spill_mask_, kArmWordSize); 1328 } 1329 1330 if (GetGraph()->HasShouldDeoptimizeFlag()) { 1331 // Initialize should_deoptimize flag to 0. 1332 __ mov(IP, ShifterOperand(0)); 1333 __ StoreToOffset(kStoreWord, IP, SP, -kShouldDeoptimizeFlagSize); 1334 } 1335 1336 int adjust = GetFrameSize() - FrameEntrySpillSize(); 1337 __ AddConstant(SP, -adjust); 1338 __ cfi().AdjustCFAOffset(adjust); 1339 1340 // Save the current method if we need it. Note that we do not 1341 // do this in HCurrentMethod, as the instruction might have been removed 1342 // in the SSA graph. 1343 if (RequiresCurrentMethod()) { 1344 __ StoreToOffset(kStoreWord, kMethodRegisterArgument, SP, 0); 1345 } 1346} 1347 1348void CodeGeneratorARM::GenerateFrameExit() { 1349 if (HasEmptyFrame()) { 1350 __ bx(LR); 1351 return; 1352 } 1353 __ cfi().RememberState(); 1354 int adjust = GetFrameSize() - FrameEntrySpillSize(); 1355 __ AddConstant(SP, adjust); 1356 __ cfi().AdjustCFAOffset(-adjust); 1357 if (fpu_spill_mask_ != 0) { 1358 SRegister start_register = SRegister(LeastSignificantBit(fpu_spill_mask_)); 1359 __ vpops(start_register, POPCOUNT(fpu_spill_mask_)); 1360 __ cfi().AdjustCFAOffset(-static_cast<int>(kArmPointerSize) * POPCOUNT(fpu_spill_mask_)); 1361 __ cfi().RestoreMany(DWARFReg(SRegister(0)), fpu_spill_mask_); 1362 } 1363 // Pop LR into PC to return. 1364 DCHECK_NE(core_spill_mask_ & (1 << LR), 0U); 1365 uint32_t pop_mask = (core_spill_mask_ & (~(1 << LR))) | 1 << PC; 1366 __ PopList(pop_mask); 1367 __ cfi().RestoreState(); 1368 __ cfi().DefCFAOffset(GetFrameSize()); 1369} 1370 1371void CodeGeneratorARM::Bind(HBasicBlock* block) { 1372 Label* label = GetLabelOf(block); 1373 __ BindTrackedLabel(label); 1374} 1375 1376Location InvokeDexCallingConventionVisitorARM::GetNextLocation(Primitive::Type type) { 1377 switch (type) { 1378 case Primitive::kPrimBoolean: 1379 case Primitive::kPrimByte: 1380 case Primitive::kPrimChar: 1381 case Primitive::kPrimShort: 1382 case Primitive::kPrimInt: 1383 case Primitive::kPrimNot: { 1384 uint32_t index = gp_index_++; 1385 uint32_t stack_index = stack_index_++; 1386 if (index < calling_convention.GetNumberOfRegisters()) { 1387 return Location::RegisterLocation(calling_convention.GetRegisterAt(index)); 1388 } else { 1389 return Location::StackSlot(calling_convention.GetStackOffsetOf(stack_index)); 1390 } 1391 } 1392 1393 case Primitive::kPrimLong: { 1394 uint32_t index = gp_index_; 1395 uint32_t stack_index = stack_index_; 1396 gp_index_ += 2; 1397 stack_index_ += 2; 1398 if (index + 1 < calling_convention.GetNumberOfRegisters()) { 1399 if (calling_convention.GetRegisterAt(index) == R1) { 1400 // Skip R1, and use R2_R3 instead. 1401 gp_index_++; 1402 index++; 1403 } 1404 } 1405 if (index + 1 < calling_convention.GetNumberOfRegisters()) { 1406 DCHECK_EQ(calling_convention.GetRegisterAt(index) + 1, 1407 calling_convention.GetRegisterAt(index + 1)); 1408 1409 return Location::RegisterPairLocation(calling_convention.GetRegisterAt(index), 1410 calling_convention.GetRegisterAt(index + 1)); 1411 } else { 1412 return Location::DoubleStackSlot(calling_convention.GetStackOffsetOf(stack_index)); 1413 } 1414 } 1415 1416 case Primitive::kPrimFloat: { 1417 uint32_t stack_index = stack_index_++; 1418 if (float_index_ % 2 == 0) { 1419 float_index_ = std::max(double_index_, float_index_); 1420 } 1421 if (float_index_ < calling_convention.GetNumberOfFpuRegisters()) { 1422 return Location::FpuRegisterLocation(calling_convention.GetFpuRegisterAt(float_index_++)); 1423 } else { 1424 return Location::StackSlot(calling_convention.GetStackOffsetOf(stack_index)); 1425 } 1426 } 1427 1428 case Primitive::kPrimDouble: { 1429 double_index_ = std::max(double_index_, RoundUp(float_index_, 2)); 1430 uint32_t stack_index = stack_index_; 1431 stack_index_ += 2; 1432 if (double_index_ + 1 < calling_convention.GetNumberOfFpuRegisters()) { 1433 uint32_t index = double_index_; 1434 double_index_ += 2; 1435 Location result = Location::FpuRegisterPairLocation( 1436 calling_convention.GetFpuRegisterAt(index), 1437 calling_convention.GetFpuRegisterAt(index + 1)); 1438 DCHECK(ExpectedPairLayout(result)); 1439 return result; 1440 } else { 1441 return Location::DoubleStackSlot(calling_convention.GetStackOffsetOf(stack_index)); 1442 } 1443 } 1444 1445 case Primitive::kPrimVoid: 1446 LOG(FATAL) << "Unexpected parameter type " << type; 1447 break; 1448 } 1449 return Location::NoLocation(); 1450} 1451 1452Location InvokeDexCallingConventionVisitorARM::GetReturnLocation(Primitive::Type type) const { 1453 switch (type) { 1454 case Primitive::kPrimBoolean: 1455 case Primitive::kPrimByte: 1456 case Primitive::kPrimChar: 1457 case Primitive::kPrimShort: 1458 case Primitive::kPrimInt: 1459 case Primitive::kPrimNot: { 1460 return Location::RegisterLocation(R0); 1461 } 1462 1463 case Primitive::kPrimFloat: { 1464 return Location::FpuRegisterLocation(S0); 1465 } 1466 1467 case Primitive::kPrimLong: { 1468 return Location::RegisterPairLocation(R0, R1); 1469 } 1470 1471 case Primitive::kPrimDouble: { 1472 return Location::FpuRegisterPairLocation(S0, S1); 1473 } 1474 1475 case Primitive::kPrimVoid: 1476 return Location::NoLocation(); 1477 } 1478 1479 UNREACHABLE(); 1480} 1481 1482Location InvokeDexCallingConventionVisitorARM::GetMethodLocation() const { 1483 return Location::RegisterLocation(kMethodRegisterArgument); 1484} 1485 1486void CodeGeneratorARM::Move32(Location destination, Location source) { 1487 if (source.Equals(destination)) { 1488 return; 1489 } 1490 if (destination.IsRegister()) { 1491 if (source.IsRegister()) { 1492 __ Mov(destination.AsRegister<Register>(), source.AsRegister<Register>()); 1493 } else if (source.IsFpuRegister()) { 1494 __ vmovrs(destination.AsRegister<Register>(), source.AsFpuRegister<SRegister>()); 1495 } else { 1496 __ LoadFromOffset(kLoadWord, destination.AsRegister<Register>(), SP, source.GetStackIndex()); 1497 } 1498 } else if (destination.IsFpuRegister()) { 1499 if (source.IsRegister()) { 1500 __ vmovsr(destination.AsFpuRegister<SRegister>(), source.AsRegister<Register>()); 1501 } else if (source.IsFpuRegister()) { 1502 __ vmovs(destination.AsFpuRegister<SRegister>(), source.AsFpuRegister<SRegister>()); 1503 } else { 1504 __ LoadSFromOffset(destination.AsFpuRegister<SRegister>(), SP, source.GetStackIndex()); 1505 } 1506 } else { 1507 DCHECK(destination.IsStackSlot()) << destination; 1508 if (source.IsRegister()) { 1509 __ StoreToOffset(kStoreWord, source.AsRegister<Register>(), SP, destination.GetStackIndex()); 1510 } else if (source.IsFpuRegister()) { 1511 __ StoreSToOffset(source.AsFpuRegister<SRegister>(), SP, destination.GetStackIndex()); 1512 } else { 1513 DCHECK(source.IsStackSlot()) << source; 1514 __ LoadFromOffset(kLoadWord, IP, SP, source.GetStackIndex()); 1515 __ StoreToOffset(kStoreWord, IP, SP, destination.GetStackIndex()); 1516 } 1517 } 1518} 1519 1520void CodeGeneratorARM::Move64(Location destination, Location source) { 1521 if (source.Equals(destination)) { 1522 return; 1523 } 1524 if (destination.IsRegisterPair()) { 1525 if (source.IsRegisterPair()) { 1526 EmitParallelMoves( 1527 Location::RegisterLocation(source.AsRegisterPairHigh<Register>()), 1528 Location::RegisterLocation(destination.AsRegisterPairHigh<Register>()), 1529 Primitive::kPrimInt, 1530 Location::RegisterLocation(source.AsRegisterPairLow<Register>()), 1531 Location::RegisterLocation(destination.AsRegisterPairLow<Register>()), 1532 Primitive::kPrimInt); 1533 } else if (source.IsFpuRegister()) { 1534 UNIMPLEMENTED(FATAL); 1535 } else if (source.IsFpuRegisterPair()) { 1536 __ vmovrrd(destination.AsRegisterPairLow<Register>(), 1537 destination.AsRegisterPairHigh<Register>(), 1538 FromLowSToD(source.AsFpuRegisterPairLow<SRegister>())); 1539 } else { 1540 DCHECK(source.IsDoubleStackSlot()); 1541 DCHECK(ExpectedPairLayout(destination)); 1542 __ LoadFromOffset(kLoadWordPair, destination.AsRegisterPairLow<Register>(), 1543 SP, source.GetStackIndex()); 1544 } 1545 } else if (destination.IsFpuRegisterPair()) { 1546 if (source.IsDoubleStackSlot()) { 1547 __ LoadDFromOffset(FromLowSToD(destination.AsFpuRegisterPairLow<SRegister>()), 1548 SP, 1549 source.GetStackIndex()); 1550 } else if (source.IsRegisterPair()) { 1551 __ vmovdrr(FromLowSToD(destination.AsFpuRegisterPairLow<SRegister>()), 1552 source.AsRegisterPairLow<Register>(), 1553 source.AsRegisterPairHigh<Register>()); 1554 } else { 1555 UNIMPLEMENTED(FATAL); 1556 } 1557 } else { 1558 DCHECK(destination.IsDoubleStackSlot()); 1559 if (source.IsRegisterPair()) { 1560 // No conflict possible, so just do the moves. 1561 if (source.AsRegisterPairLow<Register>() == R1) { 1562 DCHECK_EQ(source.AsRegisterPairHigh<Register>(), R2); 1563 __ StoreToOffset(kStoreWord, R1, SP, destination.GetStackIndex()); 1564 __ StoreToOffset(kStoreWord, R2, SP, destination.GetHighStackIndex(kArmWordSize)); 1565 } else { 1566 __ StoreToOffset(kStoreWordPair, source.AsRegisterPairLow<Register>(), 1567 SP, destination.GetStackIndex()); 1568 } 1569 } else if (source.IsFpuRegisterPair()) { 1570 __ StoreDToOffset(FromLowSToD(source.AsFpuRegisterPairLow<SRegister>()), 1571 SP, 1572 destination.GetStackIndex()); 1573 } else { 1574 DCHECK(source.IsDoubleStackSlot()); 1575 EmitParallelMoves( 1576 Location::StackSlot(source.GetStackIndex()), 1577 Location::StackSlot(destination.GetStackIndex()), 1578 Primitive::kPrimInt, 1579 Location::StackSlot(source.GetHighStackIndex(kArmWordSize)), 1580 Location::StackSlot(destination.GetHighStackIndex(kArmWordSize)), 1581 Primitive::kPrimInt); 1582 } 1583 } 1584} 1585 1586void CodeGeneratorARM::MoveConstant(Location location, int32_t value) { 1587 DCHECK(location.IsRegister()); 1588 __ LoadImmediate(location.AsRegister<Register>(), value); 1589} 1590 1591void CodeGeneratorARM::MoveLocation(Location dst, Location src, Primitive::Type dst_type) { 1592 HParallelMove move(GetGraph()->GetArena()); 1593 move.AddMove(src, dst, dst_type, nullptr); 1594 GetMoveResolver()->EmitNativeCode(&move); 1595} 1596 1597void CodeGeneratorARM::AddLocationAsTemp(Location location, LocationSummary* locations) { 1598 if (location.IsRegister()) { 1599 locations->AddTemp(location); 1600 } else if (location.IsRegisterPair()) { 1601 locations->AddTemp(Location::RegisterLocation(location.AsRegisterPairLow<Register>())); 1602 locations->AddTemp(Location::RegisterLocation(location.AsRegisterPairHigh<Register>())); 1603 } else { 1604 UNIMPLEMENTED(FATAL) << "AddLocationAsTemp not implemented for location " << location; 1605 } 1606} 1607 1608void CodeGeneratorARM::InvokeRuntime(QuickEntrypointEnum entrypoint, 1609 HInstruction* instruction, 1610 uint32_t dex_pc, 1611 SlowPathCode* slow_path) { 1612 ValidateInvokeRuntime(entrypoint, instruction, slow_path); 1613 GenerateInvokeRuntime(GetThreadOffset<kArmPointerSize>(entrypoint).Int32Value()); 1614 if (EntrypointRequiresStackMap(entrypoint)) { 1615 RecordPcInfo(instruction, dex_pc, slow_path); 1616 } 1617} 1618 1619void CodeGeneratorARM::InvokeRuntimeWithoutRecordingPcInfo(int32_t entry_point_offset, 1620 HInstruction* instruction, 1621 SlowPathCode* slow_path) { 1622 ValidateInvokeRuntimeWithoutRecordingPcInfo(instruction, slow_path); 1623 GenerateInvokeRuntime(entry_point_offset); 1624} 1625 1626void CodeGeneratorARM::GenerateInvokeRuntime(int32_t entry_point_offset) { 1627 __ LoadFromOffset(kLoadWord, LR, TR, entry_point_offset); 1628 __ blx(LR); 1629} 1630 1631void InstructionCodeGeneratorARM::HandleGoto(HInstruction* got, HBasicBlock* successor) { 1632 DCHECK(!successor->IsExitBlock()); 1633 1634 HBasicBlock* block = got->GetBlock(); 1635 HInstruction* previous = got->GetPrevious(); 1636 1637 HLoopInformation* info = block->GetLoopInformation(); 1638 if (info != nullptr && info->IsBackEdge(*block) && info->HasSuspendCheck()) { 1639 codegen_->ClearSpillSlotsFromLoopPhisInStackMap(info->GetSuspendCheck()); 1640 GenerateSuspendCheck(info->GetSuspendCheck(), successor); 1641 return; 1642 } 1643 1644 if (block->IsEntryBlock() && (previous != nullptr) && previous->IsSuspendCheck()) { 1645 GenerateSuspendCheck(previous->AsSuspendCheck(), nullptr); 1646 } 1647 if (!codegen_->GoesToNextBlock(got->GetBlock(), successor)) { 1648 __ b(codegen_->GetLabelOf(successor)); 1649 } 1650} 1651 1652void LocationsBuilderARM::VisitGoto(HGoto* got) { 1653 got->SetLocations(nullptr); 1654} 1655 1656void InstructionCodeGeneratorARM::VisitGoto(HGoto* got) { 1657 HandleGoto(got, got->GetSuccessor()); 1658} 1659 1660void LocationsBuilderARM::VisitTryBoundary(HTryBoundary* try_boundary) { 1661 try_boundary->SetLocations(nullptr); 1662} 1663 1664void InstructionCodeGeneratorARM::VisitTryBoundary(HTryBoundary* try_boundary) { 1665 HBasicBlock* successor = try_boundary->GetNormalFlowSuccessor(); 1666 if (!successor->IsExitBlock()) { 1667 HandleGoto(try_boundary, successor); 1668 } 1669} 1670 1671void LocationsBuilderARM::VisitExit(HExit* exit) { 1672 exit->SetLocations(nullptr); 1673} 1674 1675void InstructionCodeGeneratorARM::VisitExit(HExit* exit ATTRIBUTE_UNUSED) { 1676} 1677 1678void InstructionCodeGeneratorARM::GenerateVcmp(HInstruction* instruction) { 1679 Primitive::Type type = instruction->InputAt(0)->GetType(); 1680 Location lhs_loc = instruction->GetLocations()->InAt(0); 1681 Location rhs_loc = instruction->GetLocations()->InAt(1); 1682 if (rhs_loc.IsConstant()) { 1683 // 0.0 is the only immediate that can be encoded directly in 1684 // a VCMP instruction. 1685 // 1686 // Both the JLS (section 15.20.1) and the JVMS (section 6.5) 1687 // specify that in a floating-point comparison, positive zero 1688 // and negative zero are considered equal, so we can use the 1689 // literal 0.0 for both cases here. 1690 // 1691 // Note however that some methods (Float.equal, Float.compare, 1692 // Float.compareTo, Double.equal, Double.compare, 1693 // Double.compareTo, Math.max, Math.min, StrictMath.max, 1694 // StrictMath.min) consider 0.0 to be (strictly) greater than 1695 // -0.0. So if we ever translate calls to these methods into a 1696 // HCompare instruction, we must handle the -0.0 case with 1697 // care here. 1698 DCHECK(rhs_loc.GetConstant()->IsArithmeticZero()); 1699 if (type == Primitive::kPrimFloat) { 1700 __ vcmpsz(lhs_loc.AsFpuRegister<SRegister>()); 1701 } else { 1702 DCHECK_EQ(type, Primitive::kPrimDouble); 1703 __ vcmpdz(FromLowSToD(lhs_loc.AsFpuRegisterPairLow<SRegister>())); 1704 } 1705 } else { 1706 if (type == Primitive::kPrimFloat) { 1707 __ vcmps(lhs_loc.AsFpuRegister<SRegister>(), rhs_loc.AsFpuRegister<SRegister>()); 1708 } else { 1709 DCHECK_EQ(type, Primitive::kPrimDouble); 1710 __ vcmpd(FromLowSToD(lhs_loc.AsFpuRegisterPairLow<SRegister>()), 1711 FromLowSToD(rhs_loc.AsFpuRegisterPairLow<SRegister>())); 1712 } 1713 } 1714} 1715 1716void InstructionCodeGeneratorARM::GenerateFPJumps(HCondition* cond, 1717 Label* true_label, 1718 Label* false_label ATTRIBUTE_UNUSED) { 1719 __ vmstat(); // transfer FP status register to ARM APSR. 1720 __ b(true_label, ARMFPCondition(cond->GetCondition(), cond->IsGtBias())); 1721} 1722 1723void InstructionCodeGeneratorARM::GenerateLongComparesAndJumps(HCondition* cond, 1724 Label* true_label, 1725 Label* false_label) { 1726 LocationSummary* locations = cond->GetLocations(); 1727 Location left = locations->InAt(0); 1728 Location right = locations->InAt(1); 1729 IfCondition if_cond = cond->GetCondition(); 1730 1731 Register left_high = left.AsRegisterPairHigh<Register>(); 1732 Register left_low = left.AsRegisterPairLow<Register>(); 1733 IfCondition true_high_cond = if_cond; 1734 IfCondition false_high_cond = cond->GetOppositeCondition(); 1735 Condition final_condition = ARMUnsignedCondition(if_cond); // unsigned on lower part 1736 1737 // Set the conditions for the test, remembering that == needs to be 1738 // decided using the low words. 1739 // TODO: consider avoiding jumps with temporary and CMP low+SBC high 1740 switch (if_cond) { 1741 case kCondEQ: 1742 case kCondNE: 1743 // Nothing to do. 1744 break; 1745 case kCondLT: 1746 false_high_cond = kCondGT; 1747 break; 1748 case kCondLE: 1749 true_high_cond = kCondLT; 1750 break; 1751 case kCondGT: 1752 false_high_cond = kCondLT; 1753 break; 1754 case kCondGE: 1755 true_high_cond = kCondGT; 1756 break; 1757 case kCondB: 1758 false_high_cond = kCondA; 1759 break; 1760 case kCondBE: 1761 true_high_cond = kCondB; 1762 break; 1763 case kCondA: 1764 false_high_cond = kCondB; 1765 break; 1766 case kCondAE: 1767 true_high_cond = kCondA; 1768 break; 1769 } 1770 if (right.IsConstant()) { 1771 int64_t value = right.GetConstant()->AsLongConstant()->GetValue(); 1772 int32_t val_low = Low32Bits(value); 1773 int32_t val_high = High32Bits(value); 1774 1775 __ CmpConstant(left_high, val_high); 1776 if (if_cond == kCondNE) { 1777 __ b(true_label, ARMCondition(true_high_cond)); 1778 } else if (if_cond == kCondEQ) { 1779 __ b(false_label, ARMCondition(false_high_cond)); 1780 } else { 1781 __ b(true_label, ARMCondition(true_high_cond)); 1782 __ b(false_label, ARMCondition(false_high_cond)); 1783 } 1784 // Must be equal high, so compare the lows. 1785 __ CmpConstant(left_low, val_low); 1786 } else { 1787 Register right_high = right.AsRegisterPairHigh<Register>(); 1788 Register right_low = right.AsRegisterPairLow<Register>(); 1789 1790 __ cmp(left_high, ShifterOperand(right_high)); 1791 if (if_cond == kCondNE) { 1792 __ b(true_label, ARMCondition(true_high_cond)); 1793 } else if (if_cond == kCondEQ) { 1794 __ b(false_label, ARMCondition(false_high_cond)); 1795 } else { 1796 __ b(true_label, ARMCondition(true_high_cond)); 1797 __ b(false_label, ARMCondition(false_high_cond)); 1798 } 1799 // Must be equal high, so compare the lows. 1800 __ cmp(left_low, ShifterOperand(right_low)); 1801 } 1802 // The last comparison might be unsigned. 1803 // TODO: optimize cases where this is always true/false 1804 __ b(true_label, final_condition); 1805} 1806 1807void InstructionCodeGeneratorARM::GenerateCompareTestAndBranch(HCondition* condition, 1808 Label* true_target_in, 1809 Label* false_target_in) { 1810 // Generated branching requires both targets to be explicit. If either of the 1811 // targets is nullptr (fallthrough) use and bind `fallthrough_target` instead. 1812 Label fallthrough_target; 1813 Label* true_target = true_target_in == nullptr ? &fallthrough_target : true_target_in; 1814 Label* false_target = false_target_in == nullptr ? &fallthrough_target : false_target_in; 1815 1816 Primitive::Type type = condition->InputAt(0)->GetType(); 1817 switch (type) { 1818 case Primitive::kPrimLong: 1819 GenerateLongComparesAndJumps(condition, true_target, false_target); 1820 break; 1821 case Primitive::kPrimFloat: 1822 case Primitive::kPrimDouble: 1823 GenerateVcmp(condition); 1824 GenerateFPJumps(condition, true_target, false_target); 1825 break; 1826 default: 1827 LOG(FATAL) << "Unexpected compare type " << type; 1828 } 1829 1830 if (false_target != &fallthrough_target) { 1831 __ b(false_target); 1832 } 1833 1834 if (fallthrough_target.IsLinked()) { 1835 __ Bind(&fallthrough_target); 1836 } 1837} 1838 1839void InstructionCodeGeneratorARM::GenerateTestAndBranch(HInstruction* instruction, 1840 size_t condition_input_index, 1841 Label* true_target, 1842 Label* false_target) { 1843 HInstruction* cond = instruction->InputAt(condition_input_index); 1844 1845 if (true_target == nullptr && false_target == nullptr) { 1846 // Nothing to do. The code always falls through. 1847 return; 1848 } else if (cond->IsIntConstant()) { 1849 // Constant condition, statically compared against "true" (integer value 1). 1850 if (cond->AsIntConstant()->IsTrue()) { 1851 if (true_target != nullptr) { 1852 __ b(true_target); 1853 } 1854 } else { 1855 DCHECK(cond->AsIntConstant()->IsFalse()) << cond->AsIntConstant()->GetValue(); 1856 if (false_target != nullptr) { 1857 __ b(false_target); 1858 } 1859 } 1860 return; 1861 } 1862 1863 // The following code generates these patterns: 1864 // (1) true_target == nullptr && false_target != nullptr 1865 // - opposite condition true => branch to false_target 1866 // (2) true_target != nullptr && false_target == nullptr 1867 // - condition true => branch to true_target 1868 // (3) true_target != nullptr && false_target != nullptr 1869 // - condition true => branch to true_target 1870 // - branch to false_target 1871 if (IsBooleanValueOrMaterializedCondition(cond)) { 1872 // Condition has been materialized, compare the output to 0. 1873 Location cond_val = instruction->GetLocations()->InAt(condition_input_index); 1874 DCHECK(cond_val.IsRegister()); 1875 if (true_target == nullptr) { 1876 __ CompareAndBranchIfZero(cond_val.AsRegister<Register>(), false_target); 1877 } else { 1878 __ CompareAndBranchIfNonZero(cond_val.AsRegister<Register>(), true_target); 1879 } 1880 } else { 1881 // Condition has not been materialized. Use its inputs as the comparison and 1882 // its condition as the branch condition. 1883 HCondition* condition = cond->AsCondition(); 1884 1885 // If this is a long or FP comparison that has been folded into 1886 // the HCondition, generate the comparison directly. 1887 Primitive::Type type = condition->InputAt(0)->GetType(); 1888 if (type == Primitive::kPrimLong || Primitive::IsFloatingPointType(type)) { 1889 GenerateCompareTestAndBranch(condition, true_target, false_target); 1890 return; 1891 } 1892 1893 LocationSummary* locations = cond->GetLocations(); 1894 DCHECK(locations->InAt(0).IsRegister()); 1895 Register left = locations->InAt(0).AsRegister<Register>(); 1896 Location right = locations->InAt(1); 1897 if (right.IsRegister()) { 1898 __ cmp(left, ShifterOperand(right.AsRegister<Register>())); 1899 } else { 1900 DCHECK(right.IsConstant()); 1901 __ CmpConstant(left, CodeGenerator::GetInt32ValueOf(right.GetConstant())); 1902 } 1903 if (true_target == nullptr) { 1904 __ b(false_target, ARMCondition(condition->GetOppositeCondition())); 1905 } else { 1906 __ b(true_target, ARMCondition(condition->GetCondition())); 1907 } 1908 } 1909 1910 // If neither branch falls through (case 3), the conditional branch to `true_target` 1911 // was already emitted (case 2) and we need to emit a jump to `false_target`. 1912 if (true_target != nullptr && false_target != nullptr) { 1913 __ b(false_target); 1914 } 1915} 1916 1917void LocationsBuilderARM::VisitIf(HIf* if_instr) { 1918 LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(if_instr); 1919 if (IsBooleanValueOrMaterializedCondition(if_instr->InputAt(0))) { 1920 locations->SetInAt(0, Location::RequiresRegister()); 1921 } 1922} 1923 1924void InstructionCodeGeneratorARM::VisitIf(HIf* if_instr) { 1925 HBasicBlock* true_successor = if_instr->IfTrueSuccessor(); 1926 HBasicBlock* false_successor = if_instr->IfFalseSuccessor(); 1927 Label* true_target = codegen_->GoesToNextBlock(if_instr->GetBlock(), true_successor) ? 1928 nullptr : codegen_->GetLabelOf(true_successor); 1929 Label* false_target = codegen_->GoesToNextBlock(if_instr->GetBlock(), false_successor) ? 1930 nullptr : codegen_->GetLabelOf(false_successor); 1931 GenerateTestAndBranch(if_instr, /* condition_input_index */ 0, true_target, false_target); 1932} 1933 1934void LocationsBuilderARM::VisitDeoptimize(HDeoptimize* deoptimize) { 1935 LocationSummary* locations = new (GetGraph()->GetArena()) 1936 LocationSummary(deoptimize, LocationSummary::kCallOnSlowPath); 1937 locations->SetCustomSlowPathCallerSaves(RegisterSet::Empty()); // No caller-save registers. 1938 if (IsBooleanValueOrMaterializedCondition(deoptimize->InputAt(0))) { 1939 locations->SetInAt(0, Location::RequiresRegister()); 1940 } 1941} 1942 1943void InstructionCodeGeneratorARM::VisitDeoptimize(HDeoptimize* deoptimize) { 1944 SlowPathCodeARM* slow_path = deopt_slow_paths_.NewSlowPath<DeoptimizationSlowPathARM>(deoptimize); 1945 GenerateTestAndBranch(deoptimize, 1946 /* condition_input_index */ 0, 1947 slow_path->GetEntryLabel(), 1948 /* false_target */ nullptr); 1949} 1950 1951void LocationsBuilderARM::VisitShouldDeoptimizeFlag(HShouldDeoptimizeFlag* flag) { 1952 LocationSummary* locations = new (GetGraph()->GetArena()) 1953 LocationSummary(flag, LocationSummary::kNoCall); 1954 locations->SetOut(Location::RequiresRegister()); 1955} 1956 1957void InstructionCodeGeneratorARM::VisitShouldDeoptimizeFlag(HShouldDeoptimizeFlag* flag) { 1958 __ LoadFromOffset(kLoadWord, 1959 flag->GetLocations()->Out().AsRegister<Register>(), 1960 SP, 1961 codegen_->GetStackOffsetOfShouldDeoptimizeFlag()); 1962} 1963 1964void LocationsBuilderARM::VisitSelect(HSelect* select) { 1965 LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(select); 1966 if (Primitive::IsFloatingPointType(select->GetType())) { 1967 locations->SetInAt(0, Location::RequiresFpuRegister()); 1968 locations->SetInAt(1, Location::RequiresFpuRegister()); 1969 } else { 1970 locations->SetInAt(0, Location::RequiresRegister()); 1971 locations->SetInAt(1, Location::RequiresRegister()); 1972 } 1973 if (IsBooleanValueOrMaterializedCondition(select->GetCondition())) { 1974 locations->SetInAt(2, Location::RequiresRegister()); 1975 } 1976 locations->SetOut(Location::SameAsFirstInput()); 1977} 1978 1979void InstructionCodeGeneratorARM::VisitSelect(HSelect* select) { 1980 LocationSummary* locations = select->GetLocations(); 1981 Label false_target; 1982 GenerateTestAndBranch(select, 1983 /* condition_input_index */ 2, 1984 /* true_target */ nullptr, 1985 &false_target); 1986 codegen_->MoveLocation(locations->Out(), locations->InAt(1), select->GetType()); 1987 __ Bind(&false_target); 1988} 1989 1990void LocationsBuilderARM::VisitNativeDebugInfo(HNativeDebugInfo* info) { 1991 new (GetGraph()->GetArena()) LocationSummary(info); 1992} 1993 1994void InstructionCodeGeneratorARM::VisitNativeDebugInfo(HNativeDebugInfo*) { 1995 // MaybeRecordNativeDebugInfo is already called implicitly in CodeGenerator::Compile. 1996} 1997 1998void CodeGeneratorARM::GenerateNop() { 1999 __ nop(); 2000} 2001 2002void LocationsBuilderARM::HandleCondition(HCondition* cond) { 2003 LocationSummary* locations = 2004 new (GetGraph()->GetArena()) LocationSummary(cond, LocationSummary::kNoCall); 2005 // Handle the long/FP comparisons made in instruction simplification. 2006 switch (cond->InputAt(0)->GetType()) { 2007 case Primitive::kPrimLong: 2008 locations->SetInAt(0, Location::RequiresRegister()); 2009 locations->SetInAt(1, Location::RegisterOrConstant(cond->InputAt(1))); 2010 if (!cond->IsEmittedAtUseSite()) { 2011 locations->SetOut(Location::RequiresRegister(), Location::kOutputOverlap); 2012 } 2013 break; 2014 2015 case Primitive::kPrimFloat: 2016 case Primitive::kPrimDouble: 2017 locations->SetInAt(0, Location::RequiresFpuRegister()); 2018 locations->SetInAt(1, ArithmeticZeroOrFpuRegister(cond->InputAt(1))); 2019 if (!cond->IsEmittedAtUseSite()) { 2020 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 2021 } 2022 break; 2023 2024 default: 2025 locations->SetInAt(0, Location::RequiresRegister()); 2026 locations->SetInAt(1, Location::RegisterOrConstant(cond->InputAt(1))); 2027 if (!cond->IsEmittedAtUseSite()) { 2028 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 2029 } 2030 } 2031} 2032 2033void InstructionCodeGeneratorARM::HandleCondition(HCondition* cond) { 2034 if (cond->IsEmittedAtUseSite()) { 2035 return; 2036 } 2037 2038 LocationSummary* locations = cond->GetLocations(); 2039 Location left = locations->InAt(0); 2040 Location right = locations->InAt(1); 2041 Register out = locations->Out().AsRegister<Register>(); 2042 Label true_label, false_label; 2043 2044 switch (cond->InputAt(0)->GetType()) { 2045 default: { 2046 // Integer case. 2047 if (right.IsRegister()) { 2048 __ cmp(left.AsRegister<Register>(), ShifterOperand(right.AsRegister<Register>())); 2049 } else { 2050 DCHECK(right.IsConstant()); 2051 __ CmpConstant(left.AsRegister<Register>(), 2052 CodeGenerator::GetInt32ValueOf(right.GetConstant())); 2053 } 2054 __ it(ARMCondition(cond->GetCondition()), kItElse); 2055 __ mov(locations->Out().AsRegister<Register>(), ShifterOperand(1), 2056 ARMCondition(cond->GetCondition())); 2057 __ mov(locations->Out().AsRegister<Register>(), ShifterOperand(0), 2058 ARMCondition(cond->GetOppositeCondition())); 2059 return; 2060 } 2061 case Primitive::kPrimLong: 2062 GenerateLongComparesAndJumps(cond, &true_label, &false_label); 2063 break; 2064 case Primitive::kPrimFloat: 2065 case Primitive::kPrimDouble: 2066 GenerateVcmp(cond); 2067 GenerateFPJumps(cond, &true_label, &false_label); 2068 break; 2069 } 2070 2071 // Convert the jumps into the result. 2072 Label done_label; 2073 2074 // False case: result = 0. 2075 __ Bind(&false_label); 2076 __ LoadImmediate(out, 0); 2077 __ b(&done_label); 2078 2079 // True case: result = 1. 2080 __ Bind(&true_label); 2081 __ LoadImmediate(out, 1); 2082 __ Bind(&done_label); 2083} 2084 2085void LocationsBuilderARM::VisitEqual(HEqual* comp) { 2086 HandleCondition(comp); 2087} 2088 2089void InstructionCodeGeneratorARM::VisitEqual(HEqual* comp) { 2090 HandleCondition(comp); 2091} 2092 2093void LocationsBuilderARM::VisitNotEqual(HNotEqual* comp) { 2094 HandleCondition(comp); 2095} 2096 2097void InstructionCodeGeneratorARM::VisitNotEqual(HNotEqual* comp) { 2098 HandleCondition(comp); 2099} 2100 2101void LocationsBuilderARM::VisitLessThan(HLessThan* comp) { 2102 HandleCondition(comp); 2103} 2104 2105void InstructionCodeGeneratorARM::VisitLessThan(HLessThan* comp) { 2106 HandleCondition(comp); 2107} 2108 2109void LocationsBuilderARM::VisitLessThanOrEqual(HLessThanOrEqual* comp) { 2110 HandleCondition(comp); 2111} 2112 2113void InstructionCodeGeneratorARM::VisitLessThanOrEqual(HLessThanOrEqual* comp) { 2114 HandleCondition(comp); 2115} 2116 2117void LocationsBuilderARM::VisitGreaterThan(HGreaterThan* comp) { 2118 HandleCondition(comp); 2119} 2120 2121void InstructionCodeGeneratorARM::VisitGreaterThan(HGreaterThan* comp) { 2122 HandleCondition(comp); 2123} 2124 2125void LocationsBuilderARM::VisitGreaterThanOrEqual(HGreaterThanOrEqual* comp) { 2126 HandleCondition(comp); 2127} 2128 2129void InstructionCodeGeneratorARM::VisitGreaterThanOrEqual(HGreaterThanOrEqual* comp) { 2130 HandleCondition(comp); 2131} 2132 2133void LocationsBuilderARM::VisitBelow(HBelow* comp) { 2134 HandleCondition(comp); 2135} 2136 2137void InstructionCodeGeneratorARM::VisitBelow(HBelow* comp) { 2138 HandleCondition(comp); 2139} 2140 2141void LocationsBuilderARM::VisitBelowOrEqual(HBelowOrEqual* comp) { 2142 HandleCondition(comp); 2143} 2144 2145void InstructionCodeGeneratorARM::VisitBelowOrEqual(HBelowOrEqual* comp) { 2146 HandleCondition(comp); 2147} 2148 2149void LocationsBuilderARM::VisitAbove(HAbove* comp) { 2150 HandleCondition(comp); 2151} 2152 2153void InstructionCodeGeneratorARM::VisitAbove(HAbove* comp) { 2154 HandleCondition(comp); 2155} 2156 2157void LocationsBuilderARM::VisitAboveOrEqual(HAboveOrEqual* comp) { 2158 HandleCondition(comp); 2159} 2160 2161void InstructionCodeGeneratorARM::VisitAboveOrEqual(HAboveOrEqual* comp) { 2162 HandleCondition(comp); 2163} 2164 2165void LocationsBuilderARM::VisitIntConstant(HIntConstant* constant) { 2166 LocationSummary* locations = 2167 new (GetGraph()->GetArena()) LocationSummary(constant, LocationSummary::kNoCall); 2168 locations->SetOut(Location::ConstantLocation(constant)); 2169} 2170 2171void InstructionCodeGeneratorARM::VisitIntConstant(HIntConstant* constant ATTRIBUTE_UNUSED) { 2172 // Will be generated at use site. 2173} 2174 2175void LocationsBuilderARM::VisitNullConstant(HNullConstant* constant) { 2176 LocationSummary* locations = 2177 new (GetGraph()->GetArena()) LocationSummary(constant, LocationSummary::kNoCall); 2178 locations->SetOut(Location::ConstantLocation(constant)); 2179} 2180 2181void InstructionCodeGeneratorARM::VisitNullConstant(HNullConstant* constant ATTRIBUTE_UNUSED) { 2182 // Will be generated at use site. 2183} 2184 2185void LocationsBuilderARM::VisitLongConstant(HLongConstant* constant) { 2186 LocationSummary* locations = 2187 new (GetGraph()->GetArena()) LocationSummary(constant, LocationSummary::kNoCall); 2188 locations->SetOut(Location::ConstantLocation(constant)); 2189} 2190 2191void InstructionCodeGeneratorARM::VisitLongConstant(HLongConstant* constant ATTRIBUTE_UNUSED) { 2192 // Will be generated at use site. 2193} 2194 2195void LocationsBuilderARM::VisitFloatConstant(HFloatConstant* constant) { 2196 LocationSummary* locations = 2197 new (GetGraph()->GetArena()) LocationSummary(constant, LocationSummary::kNoCall); 2198 locations->SetOut(Location::ConstantLocation(constant)); 2199} 2200 2201void InstructionCodeGeneratorARM::VisitFloatConstant(HFloatConstant* constant ATTRIBUTE_UNUSED) { 2202 // Will be generated at use site. 2203} 2204 2205void LocationsBuilderARM::VisitDoubleConstant(HDoubleConstant* constant) { 2206 LocationSummary* locations = 2207 new (GetGraph()->GetArena()) LocationSummary(constant, LocationSummary::kNoCall); 2208 locations->SetOut(Location::ConstantLocation(constant)); 2209} 2210 2211void InstructionCodeGeneratorARM::VisitDoubleConstant(HDoubleConstant* constant ATTRIBUTE_UNUSED) { 2212 // Will be generated at use site. 2213} 2214 2215void LocationsBuilderARM::VisitMemoryBarrier(HMemoryBarrier* memory_barrier) { 2216 memory_barrier->SetLocations(nullptr); 2217} 2218 2219void InstructionCodeGeneratorARM::VisitMemoryBarrier(HMemoryBarrier* memory_barrier) { 2220 codegen_->GenerateMemoryBarrier(memory_barrier->GetBarrierKind()); 2221} 2222 2223void LocationsBuilderARM::VisitReturnVoid(HReturnVoid* ret) { 2224 ret->SetLocations(nullptr); 2225} 2226 2227void InstructionCodeGeneratorARM::VisitReturnVoid(HReturnVoid* ret ATTRIBUTE_UNUSED) { 2228 codegen_->GenerateFrameExit(); 2229} 2230 2231void LocationsBuilderARM::VisitReturn(HReturn* ret) { 2232 LocationSummary* locations = 2233 new (GetGraph()->GetArena()) LocationSummary(ret, LocationSummary::kNoCall); 2234 locations->SetInAt(0, parameter_visitor_.GetReturnLocation(ret->InputAt(0)->GetType())); 2235} 2236 2237void InstructionCodeGeneratorARM::VisitReturn(HReturn* ret ATTRIBUTE_UNUSED) { 2238 codegen_->GenerateFrameExit(); 2239} 2240 2241void LocationsBuilderARM::VisitInvokeUnresolved(HInvokeUnresolved* invoke) { 2242 // The trampoline uses the same calling convention as dex calling conventions, 2243 // except instead of loading arg0/r0 with the target Method*, arg0/r0 will contain 2244 // the method_idx. 2245 HandleInvoke(invoke); 2246} 2247 2248void InstructionCodeGeneratorARM::VisitInvokeUnresolved(HInvokeUnresolved* invoke) { 2249 codegen_->GenerateInvokeUnresolvedRuntimeCall(invoke); 2250} 2251 2252void LocationsBuilderARM::VisitInvokeStaticOrDirect(HInvokeStaticOrDirect* invoke) { 2253 // Explicit clinit checks triggered by static invokes must have been pruned by 2254 // art::PrepareForRegisterAllocation. 2255 DCHECK(!invoke->IsStaticWithExplicitClinitCheck()); 2256 2257 IntrinsicLocationsBuilderARM intrinsic(codegen_); 2258 if (intrinsic.TryDispatch(invoke)) { 2259 if (invoke->GetLocations()->CanCall() && invoke->HasPcRelativeDexCache()) { 2260 invoke->GetLocations()->SetInAt(invoke->GetSpecialInputIndex(), Location::Any()); 2261 } 2262 return; 2263 } 2264 2265 HandleInvoke(invoke); 2266 2267 // For PC-relative dex cache the invoke has an extra input, the PC-relative address base. 2268 if (invoke->HasPcRelativeDexCache()) { 2269 invoke->GetLocations()->SetInAt(invoke->GetSpecialInputIndex(), Location::RequiresRegister()); 2270 } 2271} 2272 2273static bool TryGenerateIntrinsicCode(HInvoke* invoke, CodeGeneratorARM* codegen) { 2274 if (invoke->GetLocations()->Intrinsified()) { 2275 IntrinsicCodeGeneratorARM intrinsic(codegen); 2276 intrinsic.Dispatch(invoke); 2277 return true; 2278 } 2279 return false; 2280} 2281 2282void InstructionCodeGeneratorARM::VisitInvokeStaticOrDirect(HInvokeStaticOrDirect* invoke) { 2283 // Explicit clinit checks triggered by static invokes must have been pruned by 2284 // art::PrepareForRegisterAllocation. 2285 DCHECK(!invoke->IsStaticWithExplicitClinitCheck()); 2286 2287 if (TryGenerateIntrinsicCode(invoke, codegen_)) { 2288 return; 2289 } 2290 2291 LocationSummary* locations = invoke->GetLocations(); 2292 codegen_->GenerateStaticOrDirectCall( 2293 invoke, locations->HasTemps() ? locations->GetTemp(0) : Location::NoLocation()); 2294 codegen_->RecordPcInfo(invoke, invoke->GetDexPc()); 2295} 2296 2297void LocationsBuilderARM::HandleInvoke(HInvoke* invoke) { 2298 InvokeDexCallingConventionVisitorARM calling_convention_visitor; 2299 CodeGenerator::CreateCommonInvokeLocationSummary(invoke, &calling_convention_visitor); 2300} 2301 2302void LocationsBuilderARM::VisitInvokeVirtual(HInvokeVirtual* invoke) { 2303 IntrinsicLocationsBuilderARM intrinsic(codegen_); 2304 if (intrinsic.TryDispatch(invoke)) { 2305 return; 2306 } 2307 2308 HandleInvoke(invoke); 2309} 2310 2311void InstructionCodeGeneratorARM::VisitInvokeVirtual(HInvokeVirtual* invoke) { 2312 if (TryGenerateIntrinsicCode(invoke, codegen_)) { 2313 return; 2314 } 2315 2316 codegen_->GenerateVirtualCall(invoke, invoke->GetLocations()->GetTemp(0)); 2317 DCHECK(!codegen_->IsLeafMethod()); 2318 codegen_->RecordPcInfo(invoke, invoke->GetDexPc()); 2319} 2320 2321void LocationsBuilderARM::VisitInvokeInterface(HInvokeInterface* invoke) { 2322 HandleInvoke(invoke); 2323 // Add the hidden argument. 2324 invoke->GetLocations()->AddTemp(Location::RegisterLocation(R12)); 2325} 2326 2327void InstructionCodeGeneratorARM::VisitInvokeInterface(HInvokeInterface* invoke) { 2328 // TODO: b/18116999, our IMTs can miss an IncompatibleClassChangeError. 2329 LocationSummary* locations = invoke->GetLocations(); 2330 Register temp = locations->GetTemp(0).AsRegister<Register>(); 2331 Register hidden_reg = locations->GetTemp(1).AsRegister<Register>(); 2332 Location receiver = locations->InAt(0); 2333 uint32_t class_offset = mirror::Object::ClassOffset().Int32Value(); 2334 2335 // Set the hidden argument. This is safe to do this here, as R12 2336 // won't be modified thereafter, before the `blx` (call) instruction. 2337 DCHECK_EQ(R12, hidden_reg); 2338 __ LoadImmediate(hidden_reg, invoke->GetDexMethodIndex()); 2339 2340 if (receiver.IsStackSlot()) { 2341 __ LoadFromOffset(kLoadWord, temp, SP, receiver.GetStackIndex()); 2342 // /* HeapReference<Class> */ temp = temp->klass_ 2343 __ LoadFromOffset(kLoadWord, temp, temp, class_offset); 2344 } else { 2345 // /* HeapReference<Class> */ temp = receiver->klass_ 2346 __ LoadFromOffset(kLoadWord, temp, receiver.AsRegister<Register>(), class_offset); 2347 } 2348 codegen_->MaybeRecordImplicitNullCheck(invoke); 2349 // Instead of simply (possibly) unpoisoning `temp` here, we should 2350 // emit a read barrier for the previous class reference load. 2351 // However this is not required in practice, as this is an 2352 // intermediate/temporary reference and because the current 2353 // concurrent copying collector keeps the from-space memory 2354 // intact/accessible until the end of the marking phase (the 2355 // concurrent copying collector may not in the future). 2356 __ MaybeUnpoisonHeapReference(temp); 2357 __ LoadFromOffset(kLoadWord, temp, temp, 2358 mirror::Class::ImtPtrOffset(kArmPointerSize).Uint32Value()); 2359 uint32_t method_offset = static_cast<uint32_t>(ImTable::OffsetOfElement( 2360 invoke->GetImtIndex(), kArmPointerSize)); 2361 // temp = temp->GetImtEntryAt(method_offset); 2362 __ LoadFromOffset(kLoadWord, temp, temp, method_offset); 2363 uint32_t entry_point = 2364 ArtMethod::EntryPointFromQuickCompiledCodeOffset(kArmPointerSize).Int32Value(); 2365 // LR = temp->GetEntryPoint(); 2366 __ LoadFromOffset(kLoadWord, LR, temp, entry_point); 2367 // LR(); 2368 __ blx(LR); 2369 DCHECK(!codegen_->IsLeafMethod()); 2370 codegen_->RecordPcInfo(invoke, invoke->GetDexPc()); 2371} 2372 2373void LocationsBuilderARM::VisitInvokePolymorphic(HInvokePolymorphic* invoke) { 2374 HandleInvoke(invoke); 2375} 2376 2377void InstructionCodeGeneratorARM::VisitInvokePolymorphic(HInvokePolymorphic* invoke) { 2378 codegen_->GenerateInvokePolymorphicCall(invoke); 2379} 2380 2381void LocationsBuilderARM::VisitNeg(HNeg* neg) { 2382 LocationSummary* locations = 2383 new (GetGraph()->GetArena()) LocationSummary(neg, LocationSummary::kNoCall); 2384 switch (neg->GetResultType()) { 2385 case Primitive::kPrimInt: { 2386 locations->SetInAt(0, Location::RequiresRegister()); 2387 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 2388 break; 2389 } 2390 case Primitive::kPrimLong: { 2391 locations->SetInAt(0, Location::RequiresRegister()); 2392 locations->SetOut(Location::RequiresRegister(), Location::kOutputOverlap); 2393 break; 2394 } 2395 2396 case Primitive::kPrimFloat: 2397 case Primitive::kPrimDouble: 2398 locations->SetInAt(0, Location::RequiresFpuRegister()); 2399 locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap); 2400 break; 2401 2402 default: 2403 LOG(FATAL) << "Unexpected neg type " << neg->GetResultType(); 2404 } 2405} 2406 2407void InstructionCodeGeneratorARM::VisitNeg(HNeg* neg) { 2408 LocationSummary* locations = neg->GetLocations(); 2409 Location out = locations->Out(); 2410 Location in = locations->InAt(0); 2411 switch (neg->GetResultType()) { 2412 case Primitive::kPrimInt: 2413 DCHECK(in.IsRegister()); 2414 __ rsb(out.AsRegister<Register>(), in.AsRegister<Register>(), ShifterOperand(0)); 2415 break; 2416 2417 case Primitive::kPrimLong: 2418 DCHECK(in.IsRegisterPair()); 2419 // out.lo = 0 - in.lo (and update the carry/borrow (C) flag) 2420 __ rsbs(out.AsRegisterPairLow<Register>(), 2421 in.AsRegisterPairLow<Register>(), 2422 ShifterOperand(0)); 2423 // We cannot emit an RSC (Reverse Subtract with Carry) 2424 // instruction here, as it does not exist in the Thumb-2 2425 // instruction set. We use the following approach 2426 // using SBC and SUB instead. 2427 // 2428 // out.hi = -C 2429 __ sbc(out.AsRegisterPairHigh<Register>(), 2430 out.AsRegisterPairHigh<Register>(), 2431 ShifterOperand(out.AsRegisterPairHigh<Register>())); 2432 // out.hi = out.hi - in.hi 2433 __ sub(out.AsRegisterPairHigh<Register>(), 2434 out.AsRegisterPairHigh<Register>(), 2435 ShifterOperand(in.AsRegisterPairHigh<Register>())); 2436 break; 2437 2438 case Primitive::kPrimFloat: 2439 DCHECK(in.IsFpuRegister()); 2440 __ vnegs(out.AsFpuRegister<SRegister>(), in.AsFpuRegister<SRegister>()); 2441 break; 2442 2443 case Primitive::kPrimDouble: 2444 DCHECK(in.IsFpuRegisterPair()); 2445 __ vnegd(FromLowSToD(out.AsFpuRegisterPairLow<SRegister>()), 2446 FromLowSToD(in.AsFpuRegisterPairLow<SRegister>())); 2447 break; 2448 2449 default: 2450 LOG(FATAL) << "Unexpected neg type " << neg->GetResultType(); 2451 } 2452} 2453 2454void LocationsBuilderARM::VisitTypeConversion(HTypeConversion* conversion) { 2455 Primitive::Type result_type = conversion->GetResultType(); 2456 Primitive::Type input_type = conversion->GetInputType(); 2457 DCHECK_NE(result_type, input_type); 2458 2459 // The float-to-long, double-to-long and long-to-float type conversions 2460 // rely on a call to the runtime. 2461 LocationSummary::CallKind call_kind = 2462 (((input_type == Primitive::kPrimFloat || input_type == Primitive::kPrimDouble) 2463 && result_type == Primitive::kPrimLong) 2464 || (input_type == Primitive::kPrimLong && result_type == Primitive::kPrimFloat)) 2465 ? LocationSummary::kCallOnMainOnly 2466 : LocationSummary::kNoCall; 2467 LocationSummary* locations = 2468 new (GetGraph()->GetArena()) LocationSummary(conversion, call_kind); 2469 2470 // The Java language does not allow treating boolean as an integral type but 2471 // our bit representation makes it safe. 2472 2473 switch (result_type) { 2474 case Primitive::kPrimByte: 2475 switch (input_type) { 2476 case Primitive::kPrimLong: 2477 // Type conversion from long to byte is a result of code transformations. 2478 case Primitive::kPrimBoolean: 2479 // Boolean input is a result of code transformations. 2480 case Primitive::kPrimShort: 2481 case Primitive::kPrimInt: 2482 case Primitive::kPrimChar: 2483 // Processing a Dex `int-to-byte' instruction. 2484 locations->SetInAt(0, Location::RequiresRegister()); 2485 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 2486 break; 2487 2488 default: 2489 LOG(FATAL) << "Unexpected type conversion from " << input_type 2490 << " to " << result_type; 2491 } 2492 break; 2493 2494 case Primitive::kPrimShort: 2495 switch (input_type) { 2496 case Primitive::kPrimLong: 2497 // Type conversion from long to short is a result of code transformations. 2498 case Primitive::kPrimBoolean: 2499 // Boolean input is a result of code transformations. 2500 case Primitive::kPrimByte: 2501 case Primitive::kPrimInt: 2502 case Primitive::kPrimChar: 2503 // Processing a Dex `int-to-short' instruction. 2504 locations->SetInAt(0, Location::RequiresRegister()); 2505 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 2506 break; 2507 2508 default: 2509 LOG(FATAL) << "Unexpected type conversion from " << input_type 2510 << " to " << result_type; 2511 } 2512 break; 2513 2514 case Primitive::kPrimInt: 2515 switch (input_type) { 2516 case Primitive::kPrimLong: 2517 // Processing a Dex `long-to-int' instruction. 2518 locations->SetInAt(0, Location::Any()); 2519 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 2520 break; 2521 2522 case Primitive::kPrimFloat: 2523 // Processing a Dex `float-to-int' instruction. 2524 locations->SetInAt(0, Location::RequiresFpuRegister()); 2525 locations->SetOut(Location::RequiresRegister()); 2526 locations->AddTemp(Location::RequiresFpuRegister()); 2527 break; 2528 2529 case Primitive::kPrimDouble: 2530 // Processing a Dex `double-to-int' instruction. 2531 locations->SetInAt(0, Location::RequiresFpuRegister()); 2532 locations->SetOut(Location::RequiresRegister()); 2533 locations->AddTemp(Location::RequiresFpuRegister()); 2534 break; 2535 2536 default: 2537 LOG(FATAL) << "Unexpected type conversion from " << input_type 2538 << " to " << result_type; 2539 } 2540 break; 2541 2542 case Primitive::kPrimLong: 2543 switch (input_type) { 2544 case Primitive::kPrimBoolean: 2545 // Boolean input is a result of code transformations. 2546 case Primitive::kPrimByte: 2547 case Primitive::kPrimShort: 2548 case Primitive::kPrimInt: 2549 case Primitive::kPrimChar: 2550 // Processing a Dex `int-to-long' instruction. 2551 locations->SetInAt(0, Location::RequiresRegister()); 2552 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 2553 break; 2554 2555 case Primitive::kPrimFloat: { 2556 // Processing a Dex `float-to-long' instruction. 2557 InvokeRuntimeCallingConvention calling_convention; 2558 locations->SetInAt(0, Location::FpuRegisterLocation( 2559 calling_convention.GetFpuRegisterAt(0))); 2560 locations->SetOut(Location::RegisterPairLocation(R0, R1)); 2561 break; 2562 } 2563 2564 case Primitive::kPrimDouble: { 2565 // Processing a Dex `double-to-long' instruction. 2566 InvokeRuntimeCallingConvention calling_convention; 2567 locations->SetInAt(0, Location::FpuRegisterPairLocation( 2568 calling_convention.GetFpuRegisterAt(0), 2569 calling_convention.GetFpuRegisterAt(1))); 2570 locations->SetOut(Location::RegisterPairLocation(R0, R1)); 2571 break; 2572 } 2573 2574 default: 2575 LOG(FATAL) << "Unexpected type conversion from " << input_type 2576 << " to " << result_type; 2577 } 2578 break; 2579 2580 case Primitive::kPrimChar: 2581 switch (input_type) { 2582 case Primitive::kPrimLong: 2583 // Type conversion from long to char is a result of code transformations. 2584 case Primitive::kPrimBoolean: 2585 // Boolean input is a result of code transformations. 2586 case Primitive::kPrimByte: 2587 case Primitive::kPrimShort: 2588 case Primitive::kPrimInt: 2589 // Processing a Dex `int-to-char' instruction. 2590 locations->SetInAt(0, Location::RequiresRegister()); 2591 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 2592 break; 2593 2594 default: 2595 LOG(FATAL) << "Unexpected type conversion from " << input_type 2596 << " to " << result_type; 2597 } 2598 break; 2599 2600 case Primitive::kPrimFloat: 2601 switch (input_type) { 2602 case Primitive::kPrimBoolean: 2603 // Boolean input is a result of code transformations. 2604 case Primitive::kPrimByte: 2605 case Primitive::kPrimShort: 2606 case Primitive::kPrimInt: 2607 case Primitive::kPrimChar: 2608 // Processing a Dex `int-to-float' instruction. 2609 locations->SetInAt(0, Location::RequiresRegister()); 2610 locations->SetOut(Location::RequiresFpuRegister()); 2611 break; 2612 2613 case Primitive::kPrimLong: { 2614 // Processing a Dex `long-to-float' instruction. 2615 InvokeRuntimeCallingConvention calling_convention; 2616 locations->SetInAt(0, Location::RegisterPairLocation( 2617 calling_convention.GetRegisterAt(0), calling_convention.GetRegisterAt(1))); 2618 locations->SetOut(Location::FpuRegisterLocation(calling_convention.GetFpuRegisterAt(0))); 2619 break; 2620 } 2621 2622 case Primitive::kPrimDouble: 2623 // Processing a Dex `double-to-float' instruction. 2624 locations->SetInAt(0, Location::RequiresFpuRegister()); 2625 locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap); 2626 break; 2627 2628 default: 2629 LOG(FATAL) << "Unexpected type conversion from " << input_type 2630 << " to " << result_type; 2631 }; 2632 break; 2633 2634 case Primitive::kPrimDouble: 2635 switch (input_type) { 2636 case Primitive::kPrimBoolean: 2637 // Boolean input is a result of code transformations. 2638 case Primitive::kPrimByte: 2639 case Primitive::kPrimShort: 2640 case Primitive::kPrimInt: 2641 case Primitive::kPrimChar: 2642 // Processing a Dex `int-to-double' instruction. 2643 locations->SetInAt(0, Location::RequiresRegister()); 2644 locations->SetOut(Location::RequiresFpuRegister()); 2645 break; 2646 2647 case Primitive::kPrimLong: 2648 // Processing a Dex `long-to-double' instruction. 2649 locations->SetInAt(0, Location::RequiresRegister()); 2650 locations->SetOut(Location::RequiresFpuRegister()); 2651 locations->AddTemp(Location::RequiresFpuRegister()); 2652 locations->AddTemp(Location::RequiresFpuRegister()); 2653 break; 2654 2655 case Primitive::kPrimFloat: 2656 // Processing a Dex `float-to-double' instruction. 2657 locations->SetInAt(0, Location::RequiresFpuRegister()); 2658 locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap); 2659 break; 2660 2661 default: 2662 LOG(FATAL) << "Unexpected type conversion from " << input_type 2663 << " to " << result_type; 2664 }; 2665 break; 2666 2667 default: 2668 LOG(FATAL) << "Unexpected type conversion from " << input_type 2669 << " to " << result_type; 2670 } 2671} 2672 2673void InstructionCodeGeneratorARM::VisitTypeConversion(HTypeConversion* conversion) { 2674 LocationSummary* locations = conversion->GetLocations(); 2675 Location out = locations->Out(); 2676 Location in = locations->InAt(0); 2677 Primitive::Type result_type = conversion->GetResultType(); 2678 Primitive::Type input_type = conversion->GetInputType(); 2679 DCHECK_NE(result_type, input_type); 2680 switch (result_type) { 2681 case Primitive::kPrimByte: 2682 switch (input_type) { 2683 case Primitive::kPrimLong: 2684 // Type conversion from long to byte is a result of code transformations. 2685 __ sbfx(out.AsRegister<Register>(), in.AsRegisterPairLow<Register>(), 0, 8); 2686 break; 2687 case Primitive::kPrimBoolean: 2688 // Boolean input is a result of code transformations. 2689 case Primitive::kPrimShort: 2690 case Primitive::kPrimInt: 2691 case Primitive::kPrimChar: 2692 // Processing a Dex `int-to-byte' instruction. 2693 __ sbfx(out.AsRegister<Register>(), in.AsRegister<Register>(), 0, 8); 2694 break; 2695 2696 default: 2697 LOG(FATAL) << "Unexpected type conversion from " << input_type 2698 << " to " << result_type; 2699 } 2700 break; 2701 2702 case Primitive::kPrimShort: 2703 switch (input_type) { 2704 case Primitive::kPrimLong: 2705 // Type conversion from long to short is a result of code transformations. 2706 __ sbfx(out.AsRegister<Register>(), in.AsRegisterPairLow<Register>(), 0, 16); 2707 break; 2708 case Primitive::kPrimBoolean: 2709 // Boolean input is a result of code transformations. 2710 case Primitive::kPrimByte: 2711 case Primitive::kPrimInt: 2712 case Primitive::kPrimChar: 2713 // Processing a Dex `int-to-short' instruction. 2714 __ sbfx(out.AsRegister<Register>(), in.AsRegister<Register>(), 0, 16); 2715 break; 2716 2717 default: 2718 LOG(FATAL) << "Unexpected type conversion from " << input_type 2719 << " to " << result_type; 2720 } 2721 break; 2722 2723 case Primitive::kPrimInt: 2724 switch (input_type) { 2725 case Primitive::kPrimLong: 2726 // Processing a Dex `long-to-int' instruction. 2727 DCHECK(out.IsRegister()); 2728 if (in.IsRegisterPair()) { 2729 __ Mov(out.AsRegister<Register>(), in.AsRegisterPairLow<Register>()); 2730 } else if (in.IsDoubleStackSlot()) { 2731 __ LoadFromOffset(kLoadWord, out.AsRegister<Register>(), SP, in.GetStackIndex()); 2732 } else { 2733 DCHECK(in.IsConstant()); 2734 DCHECK(in.GetConstant()->IsLongConstant()); 2735 int64_t value = in.GetConstant()->AsLongConstant()->GetValue(); 2736 __ LoadImmediate(out.AsRegister<Register>(), static_cast<int32_t>(value)); 2737 } 2738 break; 2739 2740 case Primitive::kPrimFloat: { 2741 // Processing a Dex `float-to-int' instruction. 2742 SRegister temp = locations->GetTemp(0).AsFpuRegisterPairLow<SRegister>(); 2743 __ vcvtis(temp, in.AsFpuRegister<SRegister>()); 2744 __ vmovrs(out.AsRegister<Register>(), temp); 2745 break; 2746 } 2747 2748 case Primitive::kPrimDouble: { 2749 // Processing a Dex `double-to-int' instruction. 2750 SRegister temp_s = locations->GetTemp(0).AsFpuRegisterPairLow<SRegister>(); 2751 __ vcvtid(temp_s, FromLowSToD(in.AsFpuRegisterPairLow<SRegister>())); 2752 __ vmovrs(out.AsRegister<Register>(), temp_s); 2753 break; 2754 } 2755 2756 default: 2757 LOG(FATAL) << "Unexpected type conversion from " << input_type 2758 << " to " << result_type; 2759 } 2760 break; 2761 2762 case Primitive::kPrimLong: 2763 switch (input_type) { 2764 case Primitive::kPrimBoolean: 2765 // Boolean input is a result of code transformations. 2766 case Primitive::kPrimByte: 2767 case Primitive::kPrimShort: 2768 case Primitive::kPrimInt: 2769 case Primitive::kPrimChar: 2770 // Processing a Dex `int-to-long' instruction. 2771 DCHECK(out.IsRegisterPair()); 2772 DCHECK(in.IsRegister()); 2773 __ Mov(out.AsRegisterPairLow<Register>(), in.AsRegister<Register>()); 2774 // Sign extension. 2775 __ Asr(out.AsRegisterPairHigh<Register>(), 2776 out.AsRegisterPairLow<Register>(), 2777 31); 2778 break; 2779 2780 case Primitive::kPrimFloat: 2781 // Processing a Dex `float-to-long' instruction. 2782 codegen_->InvokeRuntime(kQuickF2l, conversion, conversion->GetDexPc()); 2783 CheckEntrypointTypes<kQuickF2l, int64_t, float>(); 2784 break; 2785 2786 case Primitive::kPrimDouble: 2787 // Processing a Dex `double-to-long' instruction. 2788 codegen_->InvokeRuntime(kQuickD2l, conversion, conversion->GetDexPc()); 2789 CheckEntrypointTypes<kQuickD2l, int64_t, double>(); 2790 break; 2791 2792 default: 2793 LOG(FATAL) << "Unexpected type conversion from " << input_type 2794 << " to " << result_type; 2795 } 2796 break; 2797 2798 case Primitive::kPrimChar: 2799 switch (input_type) { 2800 case Primitive::kPrimLong: 2801 // Type conversion from long to char is a result of code transformations. 2802 __ ubfx(out.AsRegister<Register>(), in.AsRegisterPairLow<Register>(), 0, 16); 2803 break; 2804 case Primitive::kPrimBoolean: 2805 // Boolean input is a result of code transformations. 2806 case Primitive::kPrimByte: 2807 case Primitive::kPrimShort: 2808 case Primitive::kPrimInt: 2809 // Processing a Dex `int-to-char' instruction. 2810 __ ubfx(out.AsRegister<Register>(), in.AsRegister<Register>(), 0, 16); 2811 break; 2812 2813 default: 2814 LOG(FATAL) << "Unexpected type conversion from " << input_type 2815 << " to " << result_type; 2816 } 2817 break; 2818 2819 case Primitive::kPrimFloat: 2820 switch (input_type) { 2821 case Primitive::kPrimBoolean: 2822 // Boolean input is a result of code transformations. 2823 case Primitive::kPrimByte: 2824 case Primitive::kPrimShort: 2825 case Primitive::kPrimInt: 2826 case Primitive::kPrimChar: { 2827 // Processing a Dex `int-to-float' instruction. 2828 __ vmovsr(out.AsFpuRegister<SRegister>(), in.AsRegister<Register>()); 2829 __ vcvtsi(out.AsFpuRegister<SRegister>(), out.AsFpuRegister<SRegister>()); 2830 break; 2831 } 2832 2833 case Primitive::kPrimLong: 2834 // Processing a Dex `long-to-float' instruction. 2835 codegen_->InvokeRuntime(kQuickL2f, conversion, conversion->GetDexPc()); 2836 CheckEntrypointTypes<kQuickL2f, float, int64_t>(); 2837 break; 2838 2839 case Primitive::kPrimDouble: 2840 // Processing a Dex `double-to-float' instruction. 2841 __ vcvtsd(out.AsFpuRegister<SRegister>(), 2842 FromLowSToD(in.AsFpuRegisterPairLow<SRegister>())); 2843 break; 2844 2845 default: 2846 LOG(FATAL) << "Unexpected type conversion from " << input_type 2847 << " to " << result_type; 2848 }; 2849 break; 2850 2851 case Primitive::kPrimDouble: 2852 switch (input_type) { 2853 case Primitive::kPrimBoolean: 2854 // Boolean input is a result of code transformations. 2855 case Primitive::kPrimByte: 2856 case Primitive::kPrimShort: 2857 case Primitive::kPrimInt: 2858 case Primitive::kPrimChar: { 2859 // Processing a Dex `int-to-double' instruction. 2860 __ vmovsr(out.AsFpuRegisterPairLow<SRegister>(), in.AsRegister<Register>()); 2861 __ vcvtdi(FromLowSToD(out.AsFpuRegisterPairLow<SRegister>()), 2862 out.AsFpuRegisterPairLow<SRegister>()); 2863 break; 2864 } 2865 2866 case Primitive::kPrimLong: { 2867 // Processing a Dex `long-to-double' instruction. 2868 Register low = in.AsRegisterPairLow<Register>(); 2869 Register high = in.AsRegisterPairHigh<Register>(); 2870 SRegister out_s = out.AsFpuRegisterPairLow<SRegister>(); 2871 DRegister out_d = FromLowSToD(out_s); 2872 SRegister temp_s = locations->GetTemp(0).AsFpuRegisterPairLow<SRegister>(); 2873 DRegister temp_d = FromLowSToD(temp_s); 2874 SRegister constant_s = locations->GetTemp(1).AsFpuRegisterPairLow<SRegister>(); 2875 DRegister constant_d = FromLowSToD(constant_s); 2876 2877 // temp_d = int-to-double(high) 2878 __ vmovsr(temp_s, high); 2879 __ vcvtdi(temp_d, temp_s); 2880 // constant_d = k2Pow32EncodingForDouble 2881 __ LoadDImmediate(constant_d, bit_cast<double, int64_t>(k2Pow32EncodingForDouble)); 2882 // out_d = unsigned-to-double(low) 2883 __ vmovsr(out_s, low); 2884 __ vcvtdu(out_d, out_s); 2885 // out_d += temp_d * constant_d 2886 __ vmlad(out_d, temp_d, constant_d); 2887 break; 2888 } 2889 2890 case Primitive::kPrimFloat: 2891 // Processing a Dex `float-to-double' instruction. 2892 __ vcvtds(FromLowSToD(out.AsFpuRegisterPairLow<SRegister>()), 2893 in.AsFpuRegister<SRegister>()); 2894 break; 2895 2896 default: 2897 LOG(FATAL) << "Unexpected type conversion from " << input_type 2898 << " to " << result_type; 2899 }; 2900 break; 2901 2902 default: 2903 LOG(FATAL) << "Unexpected type conversion from " << input_type 2904 << " to " << result_type; 2905 } 2906} 2907 2908void LocationsBuilderARM::VisitAdd(HAdd* add) { 2909 LocationSummary* locations = 2910 new (GetGraph()->GetArena()) LocationSummary(add, LocationSummary::kNoCall); 2911 switch (add->GetResultType()) { 2912 case Primitive::kPrimInt: { 2913 locations->SetInAt(0, Location::RequiresRegister()); 2914 locations->SetInAt(1, Location::RegisterOrConstant(add->InputAt(1))); 2915 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 2916 break; 2917 } 2918 2919 case Primitive::kPrimLong: { 2920 locations->SetInAt(0, Location::RequiresRegister()); 2921 locations->SetInAt(1, ArmEncodableConstantOrRegister(add->InputAt(1), ADD)); 2922 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 2923 break; 2924 } 2925 2926 case Primitive::kPrimFloat: 2927 case Primitive::kPrimDouble: { 2928 locations->SetInAt(0, Location::RequiresFpuRegister()); 2929 locations->SetInAt(1, Location::RequiresFpuRegister()); 2930 locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap); 2931 break; 2932 } 2933 2934 default: 2935 LOG(FATAL) << "Unexpected add type " << add->GetResultType(); 2936 } 2937} 2938 2939void InstructionCodeGeneratorARM::VisitAdd(HAdd* add) { 2940 LocationSummary* locations = add->GetLocations(); 2941 Location out = locations->Out(); 2942 Location first = locations->InAt(0); 2943 Location second = locations->InAt(1); 2944 switch (add->GetResultType()) { 2945 case Primitive::kPrimInt: 2946 if (second.IsRegister()) { 2947 __ add(out.AsRegister<Register>(), 2948 first.AsRegister<Register>(), 2949 ShifterOperand(second.AsRegister<Register>())); 2950 } else { 2951 __ AddConstant(out.AsRegister<Register>(), 2952 first.AsRegister<Register>(), 2953 second.GetConstant()->AsIntConstant()->GetValue()); 2954 } 2955 break; 2956 2957 case Primitive::kPrimLong: { 2958 if (second.IsConstant()) { 2959 uint64_t value = static_cast<uint64_t>(Int64FromConstant(second.GetConstant())); 2960 GenerateAddLongConst(out, first, value); 2961 } else { 2962 DCHECK(second.IsRegisterPair()); 2963 __ adds(out.AsRegisterPairLow<Register>(), 2964 first.AsRegisterPairLow<Register>(), 2965 ShifterOperand(second.AsRegisterPairLow<Register>())); 2966 __ adc(out.AsRegisterPairHigh<Register>(), 2967 first.AsRegisterPairHigh<Register>(), 2968 ShifterOperand(second.AsRegisterPairHigh<Register>())); 2969 } 2970 break; 2971 } 2972 2973 case Primitive::kPrimFloat: 2974 __ vadds(out.AsFpuRegister<SRegister>(), 2975 first.AsFpuRegister<SRegister>(), 2976 second.AsFpuRegister<SRegister>()); 2977 break; 2978 2979 case Primitive::kPrimDouble: 2980 __ vaddd(FromLowSToD(out.AsFpuRegisterPairLow<SRegister>()), 2981 FromLowSToD(first.AsFpuRegisterPairLow<SRegister>()), 2982 FromLowSToD(second.AsFpuRegisterPairLow<SRegister>())); 2983 break; 2984 2985 default: 2986 LOG(FATAL) << "Unexpected add type " << add->GetResultType(); 2987 } 2988} 2989 2990void LocationsBuilderARM::VisitSub(HSub* sub) { 2991 LocationSummary* locations = 2992 new (GetGraph()->GetArena()) LocationSummary(sub, LocationSummary::kNoCall); 2993 switch (sub->GetResultType()) { 2994 case Primitive::kPrimInt: { 2995 locations->SetInAt(0, Location::RequiresRegister()); 2996 locations->SetInAt(1, Location::RegisterOrConstant(sub->InputAt(1))); 2997 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 2998 break; 2999 } 3000 3001 case Primitive::kPrimLong: { 3002 locations->SetInAt(0, Location::RequiresRegister()); 3003 locations->SetInAt(1, ArmEncodableConstantOrRegister(sub->InputAt(1), SUB)); 3004 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 3005 break; 3006 } 3007 case Primitive::kPrimFloat: 3008 case Primitive::kPrimDouble: { 3009 locations->SetInAt(0, Location::RequiresFpuRegister()); 3010 locations->SetInAt(1, Location::RequiresFpuRegister()); 3011 locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap); 3012 break; 3013 } 3014 default: 3015 LOG(FATAL) << "Unexpected sub type " << sub->GetResultType(); 3016 } 3017} 3018 3019void InstructionCodeGeneratorARM::VisitSub(HSub* sub) { 3020 LocationSummary* locations = sub->GetLocations(); 3021 Location out = locations->Out(); 3022 Location first = locations->InAt(0); 3023 Location second = locations->InAt(1); 3024 switch (sub->GetResultType()) { 3025 case Primitive::kPrimInt: { 3026 if (second.IsRegister()) { 3027 __ sub(out.AsRegister<Register>(), 3028 first.AsRegister<Register>(), 3029 ShifterOperand(second.AsRegister<Register>())); 3030 } else { 3031 __ AddConstant(out.AsRegister<Register>(), 3032 first.AsRegister<Register>(), 3033 -second.GetConstant()->AsIntConstant()->GetValue()); 3034 } 3035 break; 3036 } 3037 3038 case Primitive::kPrimLong: { 3039 if (second.IsConstant()) { 3040 uint64_t value = static_cast<uint64_t>(Int64FromConstant(second.GetConstant())); 3041 GenerateAddLongConst(out, first, -value); 3042 } else { 3043 DCHECK(second.IsRegisterPair()); 3044 __ subs(out.AsRegisterPairLow<Register>(), 3045 first.AsRegisterPairLow<Register>(), 3046 ShifterOperand(second.AsRegisterPairLow<Register>())); 3047 __ sbc(out.AsRegisterPairHigh<Register>(), 3048 first.AsRegisterPairHigh<Register>(), 3049 ShifterOperand(second.AsRegisterPairHigh<Register>())); 3050 } 3051 break; 3052 } 3053 3054 case Primitive::kPrimFloat: { 3055 __ vsubs(out.AsFpuRegister<SRegister>(), 3056 first.AsFpuRegister<SRegister>(), 3057 second.AsFpuRegister<SRegister>()); 3058 break; 3059 } 3060 3061 case Primitive::kPrimDouble: { 3062 __ vsubd(FromLowSToD(out.AsFpuRegisterPairLow<SRegister>()), 3063 FromLowSToD(first.AsFpuRegisterPairLow<SRegister>()), 3064 FromLowSToD(second.AsFpuRegisterPairLow<SRegister>())); 3065 break; 3066 } 3067 3068 3069 default: 3070 LOG(FATAL) << "Unexpected sub type " << sub->GetResultType(); 3071 } 3072} 3073 3074void LocationsBuilderARM::VisitMul(HMul* mul) { 3075 LocationSummary* locations = 3076 new (GetGraph()->GetArena()) LocationSummary(mul, LocationSummary::kNoCall); 3077 switch (mul->GetResultType()) { 3078 case Primitive::kPrimInt: 3079 case Primitive::kPrimLong: { 3080 locations->SetInAt(0, Location::RequiresRegister()); 3081 locations->SetInAt(1, Location::RequiresRegister()); 3082 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 3083 break; 3084 } 3085 3086 case Primitive::kPrimFloat: 3087 case Primitive::kPrimDouble: { 3088 locations->SetInAt(0, Location::RequiresFpuRegister()); 3089 locations->SetInAt(1, Location::RequiresFpuRegister()); 3090 locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap); 3091 break; 3092 } 3093 3094 default: 3095 LOG(FATAL) << "Unexpected mul type " << mul->GetResultType(); 3096 } 3097} 3098 3099void InstructionCodeGeneratorARM::VisitMul(HMul* mul) { 3100 LocationSummary* locations = mul->GetLocations(); 3101 Location out = locations->Out(); 3102 Location first = locations->InAt(0); 3103 Location second = locations->InAt(1); 3104 switch (mul->GetResultType()) { 3105 case Primitive::kPrimInt: { 3106 __ mul(out.AsRegister<Register>(), 3107 first.AsRegister<Register>(), 3108 second.AsRegister<Register>()); 3109 break; 3110 } 3111 case Primitive::kPrimLong: { 3112 Register out_hi = out.AsRegisterPairHigh<Register>(); 3113 Register out_lo = out.AsRegisterPairLow<Register>(); 3114 Register in1_hi = first.AsRegisterPairHigh<Register>(); 3115 Register in1_lo = first.AsRegisterPairLow<Register>(); 3116 Register in2_hi = second.AsRegisterPairHigh<Register>(); 3117 Register in2_lo = second.AsRegisterPairLow<Register>(); 3118 3119 // Extra checks to protect caused by the existence of R1_R2. 3120 // The algorithm is wrong if out.hi is either in1.lo or in2.lo: 3121 // (e.g. in1=r0_r1, in2=r2_r3 and out=r1_r2); 3122 DCHECK_NE(out_hi, in1_lo); 3123 DCHECK_NE(out_hi, in2_lo); 3124 3125 // input: in1 - 64 bits, in2 - 64 bits 3126 // output: out 3127 // formula: out.hi : out.lo = (in1.lo * in2.hi + in1.hi * in2.lo)* 2^32 + in1.lo * in2.lo 3128 // parts: out.hi = in1.lo * in2.hi + in1.hi * in2.lo + (in1.lo * in2.lo)[63:32] 3129 // parts: out.lo = (in1.lo * in2.lo)[31:0] 3130 3131 // IP <- in1.lo * in2.hi 3132 __ mul(IP, in1_lo, in2_hi); 3133 // out.hi <- in1.lo * in2.hi + in1.hi * in2.lo 3134 __ mla(out_hi, in1_hi, in2_lo, IP); 3135 // out.lo <- (in1.lo * in2.lo)[31:0]; 3136 __ umull(out_lo, IP, in1_lo, in2_lo); 3137 // out.hi <- in2.hi * in1.lo + in2.lo * in1.hi + (in1.lo * in2.lo)[63:32] 3138 __ add(out_hi, out_hi, ShifterOperand(IP)); 3139 break; 3140 } 3141 3142 case Primitive::kPrimFloat: { 3143 __ vmuls(out.AsFpuRegister<SRegister>(), 3144 first.AsFpuRegister<SRegister>(), 3145 second.AsFpuRegister<SRegister>()); 3146 break; 3147 } 3148 3149 case Primitive::kPrimDouble: { 3150 __ vmuld(FromLowSToD(out.AsFpuRegisterPairLow<SRegister>()), 3151 FromLowSToD(first.AsFpuRegisterPairLow<SRegister>()), 3152 FromLowSToD(second.AsFpuRegisterPairLow<SRegister>())); 3153 break; 3154 } 3155 3156 default: 3157 LOG(FATAL) << "Unexpected mul type " << mul->GetResultType(); 3158 } 3159} 3160 3161void InstructionCodeGeneratorARM::DivRemOneOrMinusOne(HBinaryOperation* instruction) { 3162 DCHECK(instruction->IsDiv() || instruction->IsRem()); 3163 DCHECK(instruction->GetResultType() == Primitive::kPrimInt); 3164 3165 LocationSummary* locations = instruction->GetLocations(); 3166 Location second = locations->InAt(1); 3167 DCHECK(second.IsConstant()); 3168 3169 Register out = locations->Out().AsRegister<Register>(); 3170 Register dividend = locations->InAt(0).AsRegister<Register>(); 3171 int32_t imm = second.GetConstant()->AsIntConstant()->GetValue(); 3172 DCHECK(imm == 1 || imm == -1); 3173 3174 if (instruction->IsRem()) { 3175 __ LoadImmediate(out, 0); 3176 } else { 3177 if (imm == 1) { 3178 __ Mov(out, dividend); 3179 } else { 3180 __ rsb(out, dividend, ShifterOperand(0)); 3181 } 3182 } 3183} 3184 3185void InstructionCodeGeneratorARM::DivRemByPowerOfTwo(HBinaryOperation* instruction) { 3186 DCHECK(instruction->IsDiv() || instruction->IsRem()); 3187 DCHECK(instruction->GetResultType() == Primitive::kPrimInt); 3188 3189 LocationSummary* locations = instruction->GetLocations(); 3190 Location second = locations->InAt(1); 3191 DCHECK(second.IsConstant()); 3192 3193 Register out = locations->Out().AsRegister<Register>(); 3194 Register dividend = locations->InAt(0).AsRegister<Register>(); 3195 Register temp = locations->GetTemp(0).AsRegister<Register>(); 3196 int32_t imm = second.GetConstant()->AsIntConstant()->GetValue(); 3197 uint32_t abs_imm = static_cast<uint32_t>(AbsOrMin(imm)); 3198 int ctz_imm = CTZ(abs_imm); 3199 3200 if (ctz_imm == 1) { 3201 __ Lsr(temp, dividend, 32 - ctz_imm); 3202 } else { 3203 __ Asr(temp, dividend, 31); 3204 __ Lsr(temp, temp, 32 - ctz_imm); 3205 } 3206 __ add(out, temp, ShifterOperand(dividend)); 3207 3208 if (instruction->IsDiv()) { 3209 __ Asr(out, out, ctz_imm); 3210 if (imm < 0) { 3211 __ rsb(out, out, ShifterOperand(0)); 3212 } 3213 } else { 3214 __ ubfx(out, out, 0, ctz_imm); 3215 __ sub(out, out, ShifterOperand(temp)); 3216 } 3217} 3218 3219void InstructionCodeGeneratorARM::GenerateDivRemWithAnyConstant(HBinaryOperation* instruction) { 3220 DCHECK(instruction->IsDiv() || instruction->IsRem()); 3221 DCHECK(instruction->GetResultType() == Primitive::kPrimInt); 3222 3223 LocationSummary* locations = instruction->GetLocations(); 3224 Location second = locations->InAt(1); 3225 DCHECK(second.IsConstant()); 3226 3227 Register out = locations->Out().AsRegister<Register>(); 3228 Register dividend = locations->InAt(0).AsRegister<Register>(); 3229 Register temp1 = locations->GetTemp(0).AsRegister<Register>(); 3230 Register temp2 = locations->GetTemp(1).AsRegister<Register>(); 3231 int64_t imm = second.GetConstant()->AsIntConstant()->GetValue(); 3232 3233 int64_t magic; 3234 int shift; 3235 CalculateMagicAndShiftForDivRem(imm, false /* is_long */, &magic, &shift); 3236 3237 __ LoadImmediate(temp1, magic); 3238 __ smull(temp2, temp1, dividend, temp1); 3239 3240 if (imm > 0 && magic < 0) { 3241 __ add(temp1, temp1, ShifterOperand(dividend)); 3242 } else if (imm < 0 && magic > 0) { 3243 __ sub(temp1, temp1, ShifterOperand(dividend)); 3244 } 3245 3246 if (shift != 0) { 3247 __ Asr(temp1, temp1, shift); 3248 } 3249 3250 if (instruction->IsDiv()) { 3251 __ sub(out, temp1, ShifterOperand(temp1, ASR, 31)); 3252 } else { 3253 __ sub(temp1, temp1, ShifterOperand(temp1, ASR, 31)); 3254 // TODO: Strength reduction for mls. 3255 __ LoadImmediate(temp2, imm); 3256 __ mls(out, temp1, temp2, dividend); 3257 } 3258} 3259 3260void InstructionCodeGeneratorARM::GenerateDivRemConstantIntegral(HBinaryOperation* instruction) { 3261 DCHECK(instruction->IsDiv() || instruction->IsRem()); 3262 DCHECK(instruction->GetResultType() == Primitive::kPrimInt); 3263 3264 LocationSummary* locations = instruction->GetLocations(); 3265 Location second = locations->InAt(1); 3266 DCHECK(second.IsConstant()); 3267 3268 int32_t imm = second.GetConstant()->AsIntConstant()->GetValue(); 3269 if (imm == 0) { 3270 // Do not generate anything. DivZeroCheck would prevent any code to be executed. 3271 } else if (imm == 1 || imm == -1) { 3272 DivRemOneOrMinusOne(instruction); 3273 } else if (IsPowerOfTwo(AbsOrMin(imm))) { 3274 DivRemByPowerOfTwo(instruction); 3275 } else { 3276 DCHECK(imm <= -2 || imm >= 2); 3277 GenerateDivRemWithAnyConstant(instruction); 3278 } 3279} 3280 3281void LocationsBuilderARM::VisitDiv(HDiv* div) { 3282 LocationSummary::CallKind call_kind = LocationSummary::kNoCall; 3283 if (div->GetResultType() == Primitive::kPrimLong) { 3284 // pLdiv runtime call. 3285 call_kind = LocationSummary::kCallOnMainOnly; 3286 } else if (div->GetResultType() == Primitive::kPrimInt && div->InputAt(1)->IsConstant()) { 3287 // sdiv will be replaced by other instruction sequence. 3288 } else if (div->GetResultType() == Primitive::kPrimInt && 3289 !codegen_->GetInstructionSetFeatures().HasDivideInstruction()) { 3290 // pIdivmod runtime call. 3291 call_kind = LocationSummary::kCallOnMainOnly; 3292 } 3293 3294 LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(div, call_kind); 3295 3296 switch (div->GetResultType()) { 3297 case Primitive::kPrimInt: { 3298 if (div->InputAt(1)->IsConstant()) { 3299 locations->SetInAt(0, Location::RequiresRegister()); 3300 locations->SetInAt(1, Location::ConstantLocation(div->InputAt(1)->AsConstant())); 3301 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 3302 int32_t value = div->InputAt(1)->AsIntConstant()->GetValue(); 3303 if (value == 1 || value == 0 || value == -1) { 3304 // No temp register required. 3305 } else { 3306 locations->AddTemp(Location::RequiresRegister()); 3307 if (!IsPowerOfTwo(AbsOrMin(value))) { 3308 locations->AddTemp(Location::RequiresRegister()); 3309 } 3310 } 3311 } else if (codegen_->GetInstructionSetFeatures().HasDivideInstruction()) { 3312 locations->SetInAt(0, Location::RequiresRegister()); 3313 locations->SetInAt(1, Location::RequiresRegister()); 3314 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 3315 } else { 3316 InvokeRuntimeCallingConvention calling_convention; 3317 locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(0))); 3318 locations->SetInAt(1, Location::RegisterLocation(calling_convention.GetRegisterAt(1))); 3319 // Note: divrem will compute both the quotient and the remainder as the pair R0 and R1, but 3320 // we only need the former. 3321 locations->SetOut(Location::RegisterLocation(R0)); 3322 } 3323 break; 3324 } 3325 case Primitive::kPrimLong: { 3326 InvokeRuntimeCallingConvention calling_convention; 3327 locations->SetInAt(0, Location::RegisterPairLocation( 3328 calling_convention.GetRegisterAt(0), calling_convention.GetRegisterAt(1))); 3329 locations->SetInAt(1, Location::RegisterPairLocation( 3330 calling_convention.GetRegisterAt(2), calling_convention.GetRegisterAt(3))); 3331 locations->SetOut(Location::RegisterPairLocation(R0, R1)); 3332 break; 3333 } 3334 case Primitive::kPrimFloat: 3335 case Primitive::kPrimDouble: { 3336 locations->SetInAt(0, Location::RequiresFpuRegister()); 3337 locations->SetInAt(1, Location::RequiresFpuRegister()); 3338 locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap); 3339 break; 3340 } 3341 3342 default: 3343 LOG(FATAL) << "Unexpected div type " << div->GetResultType(); 3344 } 3345} 3346 3347void InstructionCodeGeneratorARM::VisitDiv(HDiv* div) { 3348 LocationSummary* locations = div->GetLocations(); 3349 Location out = locations->Out(); 3350 Location first = locations->InAt(0); 3351 Location second = locations->InAt(1); 3352 3353 switch (div->GetResultType()) { 3354 case Primitive::kPrimInt: { 3355 if (second.IsConstant()) { 3356 GenerateDivRemConstantIntegral(div); 3357 } else if (codegen_->GetInstructionSetFeatures().HasDivideInstruction()) { 3358 __ sdiv(out.AsRegister<Register>(), 3359 first.AsRegister<Register>(), 3360 second.AsRegister<Register>()); 3361 } else { 3362 InvokeRuntimeCallingConvention calling_convention; 3363 DCHECK_EQ(calling_convention.GetRegisterAt(0), first.AsRegister<Register>()); 3364 DCHECK_EQ(calling_convention.GetRegisterAt(1), second.AsRegister<Register>()); 3365 DCHECK_EQ(R0, out.AsRegister<Register>()); 3366 3367 codegen_->InvokeRuntime(kQuickIdivmod, div, div->GetDexPc()); 3368 CheckEntrypointTypes<kQuickIdivmod, int32_t, int32_t, int32_t>(); 3369 } 3370 break; 3371 } 3372 3373 case Primitive::kPrimLong: { 3374 InvokeRuntimeCallingConvention calling_convention; 3375 DCHECK_EQ(calling_convention.GetRegisterAt(0), first.AsRegisterPairLow<Register>()); 3376 DCHECK_EQ(calling_convention.GetRegisterAt(1), first.AsRegisterPairHigh<Register>()); 3377 DCHECK_EQ(calling_convention.GetRegisterAt(2), second.AsRegisterPairLow<Register>()); 3378 DCHECK_EQ(calling_convention.GetRegisterAt(3), second.AsRegisterPairHigh<Register>()); 3379 DCHECK_EQ(R0, out.AsRegisterPairLow<Register>()); 3380 DCHECK_EQ(R1, out.AsRegisterPairHigh<Register>()); 3381 3382 codegen_->InvokeRuntime(kQuickLdiv, div, div->GetDexPc()); 3383 CheckEntrypointTypes<kQuickLdiv, int64_t, int64_t, int64_t>(); 3384 break; 3385 } 3386 3387 case Primitive::kPrimFloat: { 3388 __ vdivs(out.AsFpuRegister<SRegister>(), 3389 first.AsFpuRegister<SRegister>(), 3390 second.AsFpuRegister<SRegister>()); 3391 break; 3392 } 3393 3394 case Primitive::kPrimDouble: { 3395 __ vdivd(FromLowSToD(out.AsFpuRegisterPairLow<SRegister>()), 3396 FromLowSToD(first.AsFpuRegisterPairLow<SRegister>()), 3397 FromLowSToD(second.AsFpuRegisterPairLow<SRegister>())); 3398 break; 3399 } 3400 3401 default: 3402 LOG(FATAL) << "Unexpected div type " << div->GetResultType(); 3403 } 3404} 3405 3406void LocationsBuilderARM::VisitRem(HRem* rem) { 3407 Primitive::Type type = rem->GetResultType(); 3408 3409 // Most remainders are implemented in the runtime. 3410 LocationSummary::CallKind call_kind = LocationSummary::kCallOnMainOnly; 3411 if (rem->GetResultType() == Primitive::kPrimInt && rem->InputAt(1)->IsConstant()) { 3412 // sdiv will be replaced by other instruction sequence. 3413 call_kind = LocationSummary::kNoCall; 3414 } else if ((rem->GetResultType() == Primitive::kPrimInt) 3415 && codegen_->GetInstructionSetFeatures().HasDivideInstruction()) { 3416 // Have hardware divide instruction for int, do it with three instructions. 3417 call_kind = LocationSummary::kNoCall; 3418 } 3419 3420 LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(rem, call_kind); 3421 3422 switch (type) { 3423 case Primitive::kPrimInt: { 3424 if (rem->InputAt(1)->IsConstant()) { 3425 locations->SetInAt(0, Location::RequiresRegister()); 3426 locations->SetInAt(1, Location::ConstantLocation(rem->InputAt(1)->AsConstant())); 3427 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 3428 int32_t value = rem->InputAt(1)->AsIntConstant()->GetValue(); 3429 if (value == 1 || value == 0 || value == -1) { 3430 // No temp register required. 3431 } else { 3432 locations->AddTemp(Location::RequiresRegister()); 3433 if (!IsPowerOfTwo(AbsOrMin(value))) { 3434 locations->AddTemp(Location::RequiresRegister()); 3435 } 3436 } 3437 } else if (codegen_->GetInstructionSetFeatures().HasDivideInstruction()) { 3438 locations->SetInAt(0, Location::RequiresRegister()); 3439 locations->SetInAt(1, Location::RequiresRegister()); 3440 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 3441 locations->AddTemp(Location::RequiresRegister()); 3442 } else { 3443 InvokeRuntimeCallingConvention calling_convention; 3444 locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(0))); 3445 locations->SetInAt(1, Location::RegisterLocation(calling_convention.GetRegisterAt(1))); 3446 // Note: divrem will compute both the quotient and the remainder as the pair R0 and R1, but 3447 // we only need the latter. 3448 locations->SetOut(Location::RegisterLocation(R1)); 3449 } 3450 break; 3451 } 3452 case Primitive::kPrimLong: { 3453 InvokeRuntimeCallingConvention calling_convention; 3454 locations->SetInAt(0, Location::RegisterPairLocation( 3455 calling_convention.GetRegisterAt(0), calling_convention.GetRegisterAt(1))); 3456 locations->SetInAt(1, Location::RegisterPairLocation( 3457 calling_convention.GetRegisterAt(2), calling_convention.GetRegisterAt(3))); 3458 // The runtime helper puts the output in R2,R3. 3459 locations->SetOut(Location::RegisterPairLocation(R2, R3)); 3460 break; 3461 } 3462 case Primitive::kPrimFloat: { 3463 InvokeRuntimeCallingConvention calling_convention; 3464 locations->SetInAt(0, Location::FpuRegisterLocation(calling_convention.GetFpuRegisterAt(0))); 3465 locations->SetInAt(1, Location::FpuRegisterLocation(calling_convention.GetFpuRegisterAt(1))); 3466 locations->SetOut(Location::FpuRegisterLocation(S0)); 3467 break; 3468 } 3469 3470 case Primitive::kPrimDouble: { 3471 InvokeRuntimeCallingConvention calling_convention; 3472 locations->SetInAt(0, Location::FpuRegisterPairLocation( 3473 calling_convention.GetFpuRegisterAt(0), calling_convention.GetFpuRegisterAt(1))); 3474 locations->SetInAt(1, Location::FpuRegisterPairLocation( 3475 calling_convention.GetFpuRegisterAt(2), calling_convention.GetFpuRegisterAt(3))); 3476 locations->SetOut(Location::Location::FpuRegisterPairLocation(S0, S1)); 3477 break; 3478 } 3479 3480 default: 3481 LOG(FATAL) << "Unexpected rem type " << type; 3482 } 3483} 3484 3485void InstructionCodeGeneratorARM::VisitRem(HRem* rem) { 3486 LocationSummary* locations = rem->GetLocations(); 3487 Location out = locations->Out(); 3488 Location first = locations->InAt(0); 3489 Location second = locations->InAt(1); 3490 3491 Primitive::Type type = rem->GetResultType(); 3492 switch (type) { 3493 case Primitive::kPrimInt: { 3494 if (second.IsConstant()) { 3495 GenerateDivRemConstantIntegral(rem); 3496 } else if (codegen_->GetInstructionSetFeatures().HasDivideInstruction()) { 3497 Register reg1 = first.AsRegister<Register>(); 3498 Register reg2 = second.AsRegister<Register>(); 3499 Register temp = locations->GetTemp(0).AsRegister<Register>(); 3500 3501 // temp = reg1 / reg2 (integer division) 3502 // dest = reg1 - temp * reg2 3503 __ sdiv(temp, reg1, reg2); 3504 __ mls(out.AsRegister<Register>(), temp, reg2, reg1); 3505 } else { 3506 InvokeRuntimeCallingConvention calling_convention; 3507 DCHECK_EQ(calling_convention.GetRegisterAt(0), first.AsRegister<Register>()); 3508 DCHECK_EQ(calling_convention.GetRegisterAt(1), second.AsRegister<Register>()); 3509 DCHECK_EQ(R1, out.AsRegister<Register>()); 3510 3511 codegen_->InvokeRuntime(kQuickIdivmod, rem, rem->GetDexPc()); 3512 CheckEntrypointTypes<kQuickIdivmod, int32_t, int32_t, int32_t>(); 3513 } 3514 break; 3515 } 3516 3517 case Primitive::kPrimLong: { 3518 codegen_->InvokeRuntime(kQuickLmod, rem, rem->GetDexPc()); 3519 CheckEntrypointTypes<kQuickLmod, int64_t, int64_t, int64_t>(); 3520 break; 3521 } 3522 3523 case Primitive::kPrimFloat: { 3524 codegen_->InvokeRuntime(kQuickFmodf, rem, rem->GetDexPc()); 3525 CheckEntrypointTypes<kQuickFmodf, float, float, float>(); 3526 break; 3527 } 3528 3529 case Primitive::kPrimDouble: { 3530 codegen_->InvokeRuntime(kQuickFmod, rem, rem->GetDexPc()); 3531 CheckEntrypointTypes<kQuickFmod, double, double, double>(); 3532 break; 3533 } 3534 3535 default: 3536 LOG(FATAL) << "Unexpected rem type " << type; 3537 } 3538} 3539 3540void LocationsBuilderARM::VisitDivZeroCheck(HDivZeroCheck* instruction) { 3541 LocationSummary* locations = codegen_->CreateThrowingSlowPathLocations(instruction); 3542 locations->SetInAt(0, Location::RegisterOrConstant(instruction->InputAt(0))); 3543} 3544 3545void InstructionCodeGeneratorARM::VisitDivZeroCheck(HDivZeroCheck* instruction) { 3546 SlowPathCodeARM* slow_path = new (GetGraph()->GetArena()) DivZeroCheckSlowPathARM(instruction); 3547 codegen_->AddSlowPath(slow_path); 3548 3549 LocationSummary* locations = instruction->GetLocations(); 3550 Location value = locations->InAt(0); 3551 3552 switch (instruction->GetType()) { 3553 case Primitive::kPrimBoolean: 3554 case Primitive::kPrimByte: 3555 case Primitive::kPrimChar: 3556 case Primitive::kPrimShort: 3557 case Primitive::kPrimInt: { 3558 if (value.IsRegister()) { 3559 __ CompareAndBranchIfZero(value.AsRegister<Register>(), slow_path->GetEntryLabel()); 3560 } else { 3561 DCHECK(value.IsConstant()) << value; 3562 if (value.GetConstant()->AsIntConstant()->GetValue() == 0) { 3563 __ b(slow_path->GetEntryLabel()); 3564 } 3565 } 3566 break; 3567 } 3568 case Primitive::kPrimLong: { 3569 if (value.IsRegisterPair()) { 3570 __ orrs(IP, 3571 value.AsRegisterPairLow<Register>(), 3572 ShifterOperand(value.AsRegisterPairHigh<Register>())); 3573 __ b(slow_path->GetEntryLabel(), EQ); 3574 } else { 3575 DCHECK(value.IsConstant()) << value; 3576 if (value.GetConstant()->AsLongConstant()->GetValue() == 0) { 3577 __ b(slow_path->GetEntryLabel()); 3578 } 3579 } 3580 break; 3581 default: 3582 LOG(FATAL) << "Unexpected type for HDivZeroCheck " << instruction->GetType(); 3583 } 3584 } 3585} 3586 3587void InstructionCodeGeneratorARM::HandleIntegerRotate(LocationSummary* locations) { 3588 Register in = locations->InAt(0).AsRegister<Register>(); 3589 Location rhs = locations->InAt(1); 3590 Register out = locations->Out().AsRegister<Register>(); 3591 3592 if (rhs.IsConstant()) { 3593 // Arm32 and Thumb2 assemblers require a rotation on the interval [1,31], 3594 // so map all rotations to a +ve. equivalent in that range. 3595 // (e.g. left *or* right by -2 bits == 30 bits in the same direction.) 3596 uint32_t rot = CodeGenerator::GetInt32ValueOf(rhs.GetConstant()) & 0x1F; 3597 if (rot) { 3598 // Rotate, mapping left rotations to right equivalents if necessary. 3599 // (e.g. left by 2 bits == right by 30.) 3600 __ Ror(out, in, rot); 3601 } else if (out != in) { 3602 __ Mov(out, in); 3603 } 3604 } else { 3605 __ Ror(out, in, rhs.AsRegister<Register>()); 3606 } 3607} 3608 3609// Gain some speed by mapping all Long rotates onto equivalent pairs of Integer 3610// rotates by swapping input regs (effectively rotating by the first 32-bits of 3611// a larger rotation) or flipping direction (thus treating larger right/left 3612// rotations as sub-word sized rotations in the other direction) as appropriate. 3613void InstructionCodeGeneratorARM::HandleLongRotate(LocationSummary* locations) { 3614 Register in_reg_lo = locations->InAt(0).AsRegisterPairLow<Register>(); 3615 Register in_reg_hi = locations->InAt(0).AsRegisterPairHigh<Register>(); 3616 Location rhs = locations->InAt(1); 3617 Register out_reg_lo = locations->Out().AsRegisterPairLow<Register>(); 3618 Register out_reg_hi = locations->Out().AsRegisterPairHigh<Register>(); 3619 3620 if (rhs.IsConstant()) { 3621 uint64_t rot = CodeGenerator::GetInt64ValueOf(rhs.GetConstant()); 3622 // Map all rotations to +ve. equivalents on the interval [0,63]. 3623 rot &= kMaxLongShiftDistance; 3624 // For rotates over a word in size, 'pre-rotate' by 32-bits to keep rotate 3625 // logic below to a simple pair of binary orr. 3626 // (e.g. 34 bits == in_reg swap + 2 bits right.) 3627 if (rot >= kArmBitsPerWord) { 3628 rot -= kArmBitsPerWord; 3629 std::swap(in_reg_hi, in_reg_lo); 3630 } 3631 // Rotate, or mov to out for zero or word size rotations. 3632 if (rot != 0u) { 3633 __ Lsr(out_reg_hi, in_reg_hi, rot); 3634 __ orr(out_reg_hi, out_reg_hi, ShifterOperand(in_reg_lo, arm::LSL, kArmBitsPerWord - rot)); 3635 __ Lsr(out_reg_lo, in_reg_lo, rot); 3636 __ orr(out_reg_lo, out_reg_lo, ShifterOperand(in_reg_hi, arm::LSL, kArmBitsPerWord - rot)); 3637 } else { 3638 __ Mov(out_reg_lo, in_reg_lo); 3639 __ Mov(out_reg_hi, in_reg_hi); 3640 } 3641 } else { 3642 Register shift_right = locations->GetTemp(0).AsRegister<Register>(); 3643 Register shift_left = locations->GetTemp(1).AsRegister<Register>(); 3644 Label end; 3645 Label shift_by_32_plus_shift_right; 3646 3647 __ and_(shift_right, rhs.AsRegister<Register>(), ShifterOperand(0x1F)); 3648 __ Lsrs(shift_left, rhs.AsRegister<Register>(), 6); 3649 __ rsb(shift_left, shift_right, ShifterOperand(kArmBitsPerWord), AL, kCcKeep); 3650 __ b(&shift_by_32_plus_shift_right, CC); 3651 3652 // out_reg_hi = (reg_hi << shift_left) | (reg_lo >> shift_right). 3653 // out_reg_lo = (reg_lo << shift_left) | (reg_hi >> shift_right). 3654 __ Lsl(out_reg_hi, in_reg_hi, shift_left); 3655 __ Lsr(out_reg_lo, in_reg_lo, shift_right); 3656 __ add(out_reg_hi, out_reg_hi, ShifterOperand(out_reg_lo)); 3657 __ Lsl(out_reg_lo, in_reg_lo, shift_left); 3658 __ Lsr(shift_left, in_reg_hi, shift_right); 3659 __ add(out_reg_lo, out_reg_lo, ShifterOperand(shift_left)); 3660 __ b(&end); 3661 3662 __ Bind(&shift_by_32_plus_shift_right); // Shift by 32+shift_right. 3663 // out_reg_hi = (reg_hi >> shift_right) | (reg_lo << shift_left). 3664 // out_reg_lo = (reg_lo >> shift_right) | (reg_hi << shift_left). 3665 __ Lsr(out_reg_hi, in_reg_hi, shift_right); 3666 __ Lsl(out_reg_lo, in_reg_lo, shift_left); 3667 __ add(out_reg_hi, out_reg_hi, ShifterOperand(out_reg_lo)); 3668 __ Lsr(out_reg_lo, in_reg_lo, shift_right); 3669 __ Lsl(shift_right, in_reg_hi, shift_left); 3670 __ add(out_reg_lo, out_reg_lo, ShifterOperand(shift_right)); 3671 3672 __ Bind(&end); 3673 } 3674} 3675 3676void LocationsBuilderARM::VisitRor(HRor* ror) { 3677 LocationSummary* locations = 3678 new (GetGraph()->GetArena()) LocationSummary(ror, LocationSummary::kNoCall); 3679 switch (ror->GetResultType()) { 3680 case Primitive::kPrimInt: { 3681 locations->SetInAt(0, Location::RequiresRegister()); 3682 locations->SetInAt(1, Location::RegisterOrConstant(ror->InputAt(1))); 3683 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 3684 break; 3685 } 3686 case Primitive::kPrimLong: { 3687 locations->SetInAt(0, Location::RequiresRegister()); 3688 if (ror->InputAt(1)->IsConstant()) { 3689 locations->SetInAt(1, Location::ConstantLocation(ror->InputAt(1)->AsConstant())); 3690 } else { 3691 locations->SetInAt(1, Location::RequiresRegister()); 3692 locations->AddTemp(Location::RequiresRegister()); 3693 locations->AddTemp(Location::RequiresRegister()); 3694 } 3695 locations->SetOut(Location::RequiresRegister(), Location::kOutputOverlap); 3696 break; 3697 } 3698 default: 3699 LOG(FATAL) << "Unexpected operation type " << ror->GetResultType(); 3700 } 3701} 3702 3703void InstructionCodeGeneratorARM::VisitRor(HRor* ror) { 3704 LocationSummary* locations = ror->GetLocations(); 3705 Primitive::Type type = ror->GetResultType(); 3706 switch (type) { 3707 case Primitive::kPrimInt: { 3708 HandleIntegerRotate(locations); 3709 break; 3710 } 3711 case Primitive::kPrimLong: { 3712 HandleLongRotate(locations); 3713 break; 3714 } 3715 default: 3716 LOG(FATAL) << "Unexpected operation type " << type; 3717 UNREACHABLE(); 3718 } 3719} 3720 3721void LocationsBuilderARM::HandleShift(HBinaryOperation* op) { 3722 DCHECK(op->IsShl() || op->IsShr() || op->IsUShr()); 3723 3724 LocationSummary* locations = 3725 new (GetGraph()->GetArena()) LocationSummary(op, LocationSummary::kNoCall); 3726 3727 switch (op->GetResultType()) { 3728 case Primitive::kPrimInt: { 3729 locations->SetInAt(0, Location::RequiresRegister()); 3730 if (op->InputAt(1)->IsConstant()) { 3731 locations->SetInAt(1, Location::ConstantLocation(op->InputAt(1)->AsConstant())); 3732 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 3733 } else { 3734 locations->SetInAt(1, Location::RequiresRegister()); 3735 // Make the output overlap, as it will be used to hold the masked 3736 // second input. 3737 locations->SetOut(Location::RequiresRegister(), Location::kOutputOverlap); 3738 } 3739 break; 3740 } 3741 case Primitive::kPrimLong: { 3742 locations->SetInAt(0, Location::RequiresRegister()); 3743 if (op->InputAt(1)->IsConstant()) { 3744 locations->SetInAt(1, Location::ConstantLocation(op->InputAt(1)->AsConstant())); 3745 // For simplicity, use kOutputOverlap even though we only require that low registers 3746 // don't clash with high registers which the register allocator currently guarantees. 3747 locations->SetOut(Location::RequiresRegister(), Location::kOutputOverlap); 3748 } else { 3749 locations->SetInAt(1, Location::RequiresRegister()); 3750 locations->AddTemp(Location::RequiresRegister()); 3751 locations->SetOut(Location::RequiresRegister(), Location::kOutputOverlap); 3752 } 3753 break; 3754 } 3755 default: 3756 LOG(FATAL) << "Unexpected operation type " << op->GetResultType(); 3757 } 3758} 3759 3760void InstructionCodeGeneratorARM::HandleShift(HBinaryOperation* op) { 3761 DCHECK(op->IsShl() || op->IsShr() || op->IsUShr()); 3762 3763 LocationSummary* locations = op->GetLocations(); 3764 Location out = locations->Out(); 3765 Location first = locations->InAt(0); 3766 Location second = locations->InAt(1); 3767 3768 Primitive::Type type = op->GetResultType(); 3769 switch (type) { 3770 case Primitive::kPrimInt: { 3771 Register out_reg = out.AsRegister<Register>(); 3772 Register first_reg = first.AsRegister<Register>(); 3773 if (second.IsRegister()) { 3774 Register second_reg = second.AsRegister<Register>(); 3775 // ARM doesn't mask the shift count so we need to do it ourselves. 3776 __ and_(out_reg, second_reg, ShifterOperand(kMaxIntShiftDistance)); 3777 if (op->IsShl()) { 3778 __ Lsl(out_reg, first_reg, out_reg); 3779 } else if (op->IsShr()) { 3780 __ Asr(out_reg, first_reg, out_reg); 3781 } else { 3782 __ Lsr(out_reg, first_reg, out_reg); 3783 } 3784 } else { 3785 int32_t cst = second.GetConstant()->AsIntConstant()->GetValue(); 3786 uint32_t shift_value = cst & kMaxIntShiftDistance; 3787 if (shift_value == 0) { // ARM does not support shifting with 0 immediate. 3788 __ Mov(out_reg, first_reg); 3789 } else if (op->IsShl()) { 3790 __ Lsl(out_reg, first_reg, shift_value); 3791 } else if (op->IsShr()) { 3792 __ Asr(out_reg, first_reg, shift_value); 3793 } else { 3794 __ Lsr(out_reg, first_reg, shift_value); 3795 } 3796 } 3797 break; 3798 } 3799 case Primitive::kPrimLong: { 3800 Register o_h = out.AsRegisterPairHigh<Register>(); 3801 Register o_l = out.AsRegisterPairLow<Register>(); 3802 3803 Register high = first.AsRegisterPairHigh<Register>(); 3804 Register low = first.AsRegisterPairLow<Register>(); 3805 3806 if (second.IsRegister()) { 3807 Register temp = locations->GetTemp(0).AsRegister<Register>(); 3808 3809 Register second_reg = second.AsRegister<Register>(); 3810 3811 if (op->IsShl()) { 3812 __ and_(o_l, second_reg, ShifterOperand(kMaxLongShiftDistance)); 3813 // Shift the high part 3814 __ Lsl(o_h, high, o_l); 3815 // Shift the low part and `or` what overflew on the high part 3816 __ rsb(temp, o_l, ShifterOperand(kArmBitsPerWord)); 3817 __ Lsr(temp, low, temp); 3818 __ orr(o_h, o_h, ShifterOperand(temp)); 3819 // If the shift is > 32 bits, override the high part 3820 __ subs(temp, o_l, ShifterOperand(kArmBitsPerWord)); 3821 __ it(PL); 3822 __ Lsl(o_h, low, temp, PL); 3823 // Shift the low part 3824 __ Lsl(o_l, low, o_l); 3825 } else if (op->IsShr()) { 3826 __ and_(o_h, second_reg, ShifterOperand(kMaxLongShiftDistance)); 3827 // Shift the low part 3828 __ Lsr(o_l, low, o_h); 3829 // Shift the high part and `or` what underflew on the low part 3830 __ rsb(temp, o_h, ShifterOperand(kArmBitsPerWord)); 3831 __ Lsl(temp, high, temp); 3832 __ orr(o_l, o_l, ShifterOperand(temp)); 3833 // If the shift is > 32 bits, override the low part 3834 __ subs(temp, o_h, ShifterOperand(kArmBitsPerWord)); 3835 __ it(PL); 3836 __ Asr(o_l, high, temp, PL); 3837 // Shift the high part 3838 __ Asr(o_h, high, o_h); 3839 } else { 3840 __ and_(o_h, second_reg, ShifterOperand(kMaxLongShiftDistance)); 3841 // same as Shr except we use `Lsr`s and not `Asr`s 3842 __ Lsr(o_l, low, o_h); 3843 __ rsb(temp, o_h, ShifterOperand(kArmBitsPerWord)); 3844 __ Lsl(temp, high, temp); 3845 __ orr(o_l, o_l, ShifterOperand(temp)); 3846 __ subs(temp, o_h, ShifterOperand(kArmBitsPerWord)); 3847 __ it(PL); 3848 __ Lsr(o_l, high, temp, PL); 3849 __ Lsr(o_h, high, o_h); 3850 } 3851 } else { 3852 // Register allocator doesn't create partial overlap. 3853 DCHECK_NE(o_l, high); 3854 DCHECK_NE(o_h, low); 3855 int32_t cst = second.GetConstant()->AsIntConstant()->GetValue(); 3856 uint32_t shift_value = cst & kMaxLongShiftDistance; 3857 if (shift_value > 32) { 3858 if (op->IsShl()) { 3859 __ Lsl(o_h, low, shift_value - 32); 3860 __ LoadImmediate(o_l, 0); 3861 } else if (op->IsShr()) { 3862 __ Asr(o_l, high, shift_value - 32); 3863 __ Asr(o_h, high, 31); 3864 } else { 3865 __ Lsr(o_l, high, shift_value - 32); 3866 __ LoadImmediate(o_h, 0); 3867 } 3868 } else if (shift_value == 32) { 3869 if (op->IsShl()) { 3870 __ mov(o_h, ShifterOperand(low)); 3871 __ LoadImmediate(o_l, 0); 3872 } else if (op->IsShr()) { 3873 __ mov(o_l, ShifterOperand(high)); 3874 __ Asr(o_h, high, 31); 3875 } else { 3876 __ mov(o_l, ShifterOperand(high)); 3877 __ LoadImmediate(o_h, 0); 3878 } 3879 } else if (shift_value == 1) { 3880 if (op->IsShl()) { 3881 __ Lsls(o_l, low, 1); 3882 __ adc(o_h, high, ShifterOperand(high)); 3883 } else if (op->IsShr()) { 3884 __ Asrs(o_h, high, 1); 3885 __ Rrx(o_l, low); 3886 } else { 3887 __ Lsrs(o_h, high, 1); 3888 __ Rrx(o_l, low); 3889 } 3890 } else { 3891 DCHECK(2 <= shift_value && shift_value < 32) << shift_value; 3892 if (op->IsShl()) { 3893 __ Lsl(o_h, high, shift_value); 3894 __ orr(o_h, o_h, ShifterOperand(low, LSR, 32 - shift_value)); 3895 __ Lsl(o_l, low, shift_value); 3896 } else if (op->IsShr()) { 3897 __ Lsr(o_l, low, shift_value); 3898 __ orr(o_l, o_l, ShifterOperand(high, LSL, 32 - shift_value)); 3899 __ Asr(o_h, high, shift_value); 3900 } else { 3901 __ Lsr(o_l, low, shift_value); 3902 __ orr(o_l, o_l, ShifterOperand(high, LSL, 32 - shift_value)); 3903 __ Lsr(o_h, high, shift_value); 3904 } 3905 } 3906 } 3907 break; 3908 } 3909 default: 3910 LOG(FATAL) << "Unexpected operation type " << type; 3911 UNREACHABLE(); 3912 } 3913} 3914 3915void LocationsBuilderARM::VisitShl(HShl* shl) { 3916 HandleShift(shl); 3917} 3918 3919void InstructionCodeGeneratorARM::VisitShl(HShl* shl) { 3920 HandleShift(shl); 3921} 3922 3923void LocationsBuilderARM::VisitShr(HShr* shr) { 3924 HandleShift(shr); 3925} 3926 3927void InstructionCodeGeneratorARM::VisitShr(HShr* shr) { 3928 HandleShift(shr); 3929} 3930 3931void LocationsBuilderARM::VisitUShr(HUShr* ushr) { 3932 HandleShift(ushr); 3933} 3934 3935void InstructionCodeGeneratorARM::VisitUShr(HUShr* ushr) { 3936 HandleShift(ushr); 3937} 3938 3939void LocationsBuilderARM::VisitNewInstance(HNewInstance* instruction) { 3940 LocationSummary* locations = 3941 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCallOnMainOnly); 3942 if (instruction->IsStringAlloc()) { 3943 locations->AddTemp(Location::RegisterLocation(kMethodRegisterArgument)); 3944 } else { 3945 InvokeRuntimeCallingConvention calling_convention; 3946 locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(0))); 3947 } 3948 locations->SetOut(Location::RegisterLocation(R0)); 3949} 3950 3951void InstructionCodeGeneratorARM::VisitNewInstance(HNewInstance* instruction) { 3952 // Note: if heap poisoning is enabled, the entry point takes cares 3953 // of poisoning the reference. 3954 if (instruction->IsStringAlloc()) { 3955 // String is allocated through StringFactory. Call NewEmptyString entry point. 3956 Register temp = instruction->GetLocations()->GetTemp(0).AsRegister<Register>(); 3957 MemberOffset code_offset = ArtMethod::EntryPointFromQuickCompiledCodeOffset(kArmPointerSize); 3958 __ LoadFromOffset(kLoadWord, temp, TR, QUICK_ENTRY_POINT(pNewEmptyString)); 3959 __ LoadFromOffset(kLoadWord, LR, temp, code_offset.Int32Value()); 3960 __ blx(LR); 3961 codegen_->RecordPcInfo(instruction, instruction->GetDexPc()); 3962 } else { 3963 codegen_->InvokeRuntime(instruction->GetEntrypoint(), instruction, instruction->GetDexPc()); 3964 CheckEntrypointTypes<kQuickAllocObjectWithChecks, void*, mirror::Class*>(); 3965 } 3966} 3967 3968void LocationsBuilderARM::VisitNewArray(HNewArray* instruction) { 3969 LocationSummary* locations = 3970 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCallOnMainOnly); 3971 InvokeRuntimeCallingConvention calling_convention; 3972 locations->AddTemp(Location::RegisterLocation(calling_convention.GetRegisterAt(0))); 3973 locations->SetOut(Location::RegisterLocation(R0)); 3974 locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(1))); 3975 locations->SetInAt(1, Location::RegisterLocation(calling_convention.GetRegisterAt(2))); 3976} 3977 3978void InstructionCodeGeneratorARM::VisitNewArray(HNewArray* instruction) { 3979 InvokeRuntimeCallingConvention calling_convention; 3980 __ LoadImmediate(calling_convention.GetRegisterAt(0), instruction->GetTypeIndex().index_); 3981 // Note: if heap poisoning is enabled, the entry point takes cares 3982 // of poisoning the reference. 3983 codegen_->InvokeRuntime(instruction->GetEntrypoint(), instruction, instruction->GetDexPc()); 3984 CheckEntrypointTypes<kQuickAllocArrayWithAccessCheck, void*, uint32_t, int32_t, ArtMethod*>(); 3985} 3986 3987void LocationsBuilderARM::VisitParameterValue(HParameterValue* instruction) { 3988 LocationSummary* locations = 3989 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall); 3990 Location location = parameter_visitor_.GetNextLocation(instruction->GetType()); 3991 if (location.IsStackSlot()) { 3992 location = Location::StackSlot(location.GetStackIndex() + codegen_->GetFrameSize()); 3993 } else if (location.IsDoubleStackSlot()) { 3994 location = Location::DoubleStackSlot(location.GetStackIndex() + codegen_->GetFrameSize()); 3995 } 3996 locations->SetOut(location); 3997} 3998 3999void InstructionCodeGeneratorARM::VisitParameterValue( 4000 HParameterValue* instruction ATTRIBUTE_UNUSED) { 4001 // Nothing to do, the parameter is already at its location. 4002} 4003 4004void LocationsBuilderARM::VisitCurrentMethod(HCurrentMethod* instruction) { 4005 LocationSummary* locations = 4006 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall); 4007 locations->SetOut(Location::RegisterLocation(kMethodRegisterArgument)); 4008} 4009 4010void InstructionCodeGeneratorARM::VisitCurrentMethod(HCurrentMethod* instruction ATTRIBUTE_UNUSED) { 4011 // Nothing to do, the method is already at its location. 4012} 4013 4014void LocationsBuilderARM::VisitNot(HNot* not_) { 4015 LocationSummary* locations = 4016 new (GetGraph()->GetArena()) LocationSummary(not_, LocationSummary::kNoCall); 4017 locations->SetInAt(0, Location::RequiresRegister()); 4018 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 4019} 4020 4021void InstructionCodeGeneratorARM::VisitNot(HNot* not_) { 4022 LocationSummary* locations = not_->GetLocations(); 4023 Location out = locations->Out(); 4024 Location in = locations->InAt(0); 4025 switch (not_->GetResultType()) { 4026 case Primitive::kPrimInt: 4027 __ mvn(out.AsRegister<Register>(), ShifterOperand(in.AsRegister<Register>())); 4028 break; 4029 4030 case Primitive::kPrimLong: 4031 __ mvn(out.AsRegisterPairLow<Register>(), 4032 ShifterOperand(in.AsRegisterPairLow<Register>())); 4033 __ mvn(out.AsRegisterPairHigh<Register>(), 4034 ShifterOperand(in.AsRegisterPairHigh<Register>())); 4035 break; 4036 4037 default: 4038 LOG(FATAL) << "Unimplemented type for not operation " << not_->GetResultType(); 4039 } 4040} 4041 4042void LocationsBuilderARM::VisitBooleanNot(HBooleanNot* bool_not) { 4043 LocationSummary* locations = 4044 new (GetGraph()->GetArena()) LocationSummary(bool_not, LocationSummary::kNoCall); 4045 locations->SetInAt(0, Location::RequiresRegister()); 4046 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 4047} 4048 4049void InstructionCodeGeneratorARM::VisitBooleanNot(HBooleanNot* bool_not) { 4050 LocationSummary* locations = bool_not->GetLocations(); 4051 Location out = locations->Out(); 4052 Location in = locations->InAt(0); 4053 __ eor(out.AsRegister<Register>(), in.AsRegister<Register>(), ShifterOperand(1)); 4054} 4055 4056void LocationsBuilderARM::VisitCompare(HCompare* compare) { 4057 LocationSummary* locations = 4058 new (GetGraph()->GetArena()) LocationSummary(compare, LocationSummary::kNoCall); 4059 switch (compare->InputAt(0)->GetType()) { 4060 case Primitive::kPrimBoolean: 4061 case Primitive::kPrimByte: 4062 case Primitive::kPrimShort: 4063 case Primitive::kPrimChar: 4064 case Primitive::kPrimInt: 4065 case Primitive::kPrimLong: { 4066 locations->SetInAt(0, Location::RequiresRegister()); 4067 locations->SetInAt(1, Location::RequiresRegister()); 4068 // Output overlaps because it is written before doing the low comparison. 4069 locations->SetOut(Location::RequiresRegister(), Location::kOutputOverlap); 4070 break; 4071 } 4072 case Primitive::kPrimFloat: 4073 case Primitive::kPrimDouble: { 4074 locations->SetInAt(0, Location::RequiresFpuRegister()); 4075 locations->SetInAt(1, ArithmeticZeroOrFpuRegister(compare->InputAt(1))); 4076 locations->SetOut(Location::RequiresRegister()); 4077 break; 4078 } 4079 default: 4080 LOG(FATAL) << "Unexpected type for compare operation " << compare->InputAt(0)->GetType(); 4081 } 4082} 4083 4084void InstructionCodeGeneratorARM::VisitCompare(HCompare* compare) { 4085 LocationSummary* locations = compare->GetLocations(); 4086 Register out = locations->Out().AsRegister<Register>(); 4087 Location left = locations->InAt(0); 4088 Location right = locations->InAt(1); 4089 4090 Label less, greater, done; 4091 Primitive::Type type = compare->InputAt(0)->GetType(); 4092 Condition less_cond; 4093 switch (type) { 4094 case Primitive::kPrimBoolean: 4095 case Primitive::kPrimByte: 4096 case Primitive::kPrimShort: 4097 case Primitive::kPrimChar: 4098 case Primitive::kPrimInt: { 4099 __ LoadImmediate(out, 0); 4100 __ cmp(left.AsRegister<Register>(), 4101 ShifterOperand(right.AsRegister<Register>())); // Signed compare. 4102 less_cond = LT; 4103 break; 4104 } 4105 case Primitive::kPrimLong: { 4106 __ cmp(left.AsRegisterPairHigh<Register>(), 4107 ShifterOperand(right.AsRegisterPairHigh<Register>())); // Signed compare. 4108 __ b(&less, LT); 4109 __ b(&greater, GT); 4110 // Do LoadImmediate before the last `cmp`, as LoadImmediate might affect the status flags. 4111 __ LoadImmediate(out, 0); 4112 __ cmp(left.AsRegisterPairLow<Register>(), 4113 ShifterOperand(right.AsRegisterPairLow<Register>())); // Unsigned compare. 4114 less_cond = LO; 4115 break; 4116 } 4117 case Primitive::kPrimFloat: 4118 case Primitive::kPrimDouble: { 4119 __ LoadImmediate(out, 0); 4120 GenerateVcmp(compare); 4121 __ vmstat(); // transfer FP status register to ARM APSR. 4122 less_cond = ARMFPCondition(kCondLT, compare->IsGtBias()); 4123 break; 4124 } 4125 default: 4126 LOG(FATAL) << "Unexpected compare type " << type; 4127 UNREACHABLE(); 4128 } 4129 4130 __ b(&done, EQ); 4131 __ b(&less, less_cond); 4132 4133 __ Bind(&greater); 4134 __ LoadImmediate(out, 1); 4135 __ b(&done); 4136 4137 __ Bind(&less); 4138 __ LoadImmediate(out, -1); 4139 4140 __ Bind(&done); 4141} 4142 4143void LocationsBuilderARM::VisitPhi(HPhi* instruction) { 4144 LocationSummary* locations = 4145 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall); 4146 for (size_t i = 0, e = locations->GetInputCount(); i < e; ++i) { 4147 locations->SetInAt(i, Location::Any()); 4148 } 4149 locations->SetOut(Location::Any()); 4150} 4151 4152void InstructionCodeGeneratorARM::VisitPhi(HPhi* instruction ATTRIBUTE_UNUSED) { 4153 LOG(FATAL) << "Unreachable"; 4154} 4155 4156void CodeGeneratorARM::GenerateMemoryBarrier(MemBarrierKind kind) { 4157 // TODO (ported from quick): revisit ARM barrier kinds. 4158 DmbOptions flavor = DmbOptions::ISH; // Quiet C++ warnings. 4159 switch (kind) { 4160 case MemBarrierKind::kAnyStore: 4161 case MemBarrierKind::kLoadAny: 4162 case MemBarrierKind::kAnyAny: { 4163 flavor = DmbOptions::ISH; 4164 break; 4165 } 4166 case MemBarrierKind::kStoreStore: { 4167 flavor = DmbOptions::ISHST; 4168 break; 4169 } 4170 default: 4171 LOG(FATAL) << "Unexpected memory barrier " << kind; 4172 } 4173 __ dmb(flavor); 4174} 4175 4176void InstructionCodeGeneratorARM::GenerateWideAtomicLoad(Register addr, 4177 uint32_t offset, 4178 Register out_lo, 4179 Register out_hi) { 4180 if (offset != 0) { 4181 // Ensure `out_lo` is different from `addr`, so that loading 4182 // `offset` into `out_lo` does not clutter `addr`. 4183 DCHECK_NE(out_lo, addr); 4184 __ LoadImmediate(out_lo, offset); 4185 __ add(IP, addr, ShifterOperand(out_lo)); 4186 addr = IP; 4187 } 4188 __ ldrexd(out_lo, out_hi, addr); 4189} 4190 4191void InstructionCodeGeneratorARM::GenerateWideAtomicStore(Register addr, 4192 uint32_t offset, 4193 Register value_lo, 4194 Register value_hi, 4195 Register temp1, 4196 Register temp2, 4197 HInstruction* instruction) { 4198 Label fail; 4199 if (offset != 0) { 4200 __ LoadImmediate(temp1, offset); 4201 __ add(IP, addr, ShifterOperand(temp1)); 4202 addr = IP; 4203 } 4204 __ Bind(&fail); 4205 // We need a load followed by store. (The address used in a STREX instruction must 4206 // be the same as the address in the most recently executed LDREX instruction.) 4207 __ ldrexd(temp1, temp2, addr); 4208 codegen_->MaybeRecordImplicitNullCheck(instruction); 4209 __ strexd(temp1, value_lo, value_hi, addr); 4210 __ CompareAndBranchIfNonZero(temp1, &fail); 4211} 4212 4213void LocationsBuilderARM::HandleFieldSet(HInstruction* instruction, const FieldInfo& field_info) { 4214 DCHECK(instruction->IsInstanceFieldSet() || instruction->IsStaticFieldSet()); 4215 4216 LocationSummary* locations = 4217 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall); 4218 locations->SetInAt(0, Location::RequiresRegister()); 4219 4220 Primitive::Type field_type = field_info.GetFieldType(); 4221 if (Primitive::IsFloatingPointType(field_type)) { 4222 locations->SetInAt(1, Location::RequiresFpuRegister()); 4223 } else { 4224 locations->SetInAt(1, Location::RequiresRegister()); 4225 } 4226 4227 bool is_wide = field_type == Primitive::kPrimLong || field_type == Primitive::kPrimDouble; 4228 bool generate_volatile = field_info.IsVolatile() 4229 && is_wide 4230 && !codegen_->GetInstructionSetFeatures().HasAtomicLdrdAndStrd(); 4231 bool needs_write_barrier = 4232 CodeGenerator::StoreNeedsWriteBarrier(field_type, instruction->InputAt(1)); 4233 // Temporary registers for the write barrier. 4234 // TODO: consider renaming StoreNeedsWriteBarrier to StoreNeedsGCMark. 4235 if (needs_write_barrier) { 4236 locations->AddTemp(Location::RequiresRegister()); // Possibly used for reference poisoning too. 4237 locations->AddTemp(Location::RequiresRegister()); 4238 } else if (generate_volatile) { 4239 // ARM encoding have some additional constraints for ldrexd/strexd: 4240 // - registers need to be consecutive 4241 // - the first register should be even but not R14. 4242 // We don't test for ARM yet, and the assertion makes sure that we 4243 // revisit this if we ever enable ARM encoding. 4244 DCHECK_EQ(InstructionSet::kThumb2, codegen_->GetInstructionSet()); 4245 4246 locations->AddTemp(Location::RequiresRegister()); 4247 locations->AddTemp(Location::RequiresRegister()); 4248 if (field_type == Primitive::kPrimDouble) { 4249 // For doubles we need two more registers to copy the value. 4250 locations->AddTemp(Location::RegisterLocation(R2)); 4251 locations->AddTemp(Location::RegisterLocation(R3)); 4252 } 4253 } 4254} 4255 4256void InstructionCodeGeneratorARM::HandleFieldSet(HInstruction* instruction, 4257 const FieldInfo& field_info, 4258 bool value_can_be_null) { 4259 DCHECK(instruction->IsInstanceFieldSet() || instruction->IsStaticFieldSet()); 4260 4261 LocationSummary* locations = instruction->GetLocations(); 4262 Register base = locations->InAt(0).AsRegister<Register>(); 4263 Location value = locations->InAt(1); 4264 4265 bool is_volatile = field_info.IsVolatile(); 4266 bool atomic_ldrd_strd = codegen_->GetInstructionSetFeatures().HasAtomicLdrdAndStrd(); 4267 Primitive::Type field_type = field_info.GetFieldType(); 4268 uint32_t offset = field_info.GetFieldOffset().Uint32Value(); 4269 bool needs_write_barrier = 4270 CodeGenerator::StoreNeedsWriteBarrier(field_type, instruction->InputAt(1)); 4271 4272 if (is_volatile) { 4273 codegen_->GenerateMemoryBarrier(MemBarrierKind::kAnyStore); 4274 } 4275 4276 switch (field_type) { 4277 case Primitive::kPrimBoolean: 4278 case Primitive::kPrimByte: { 4279 __ StoreToOffset(kStoreByte, value.AsRegister<Register>(), base, offset); 4280 break; 4281 } 4282 4283 case Primitive::kPrimShort: 4284 case Primitive::kPrimChar: { 4285 __ StoreToOffset(kStoreHalfword, value.AsRegister<Register>(), base, offset); 4286 break; 4287 } 4288 4289 case Primitive::kPrimInt: 4290 case Primitive::kPrimNot: { 4291 if (kPoisonHeapReferences && needs_write_barrier) { 4292 // Note that in the case where `value` is a null reference, 4293 // we do not enter this block, as a null reference does not 4294 // need poisoning. 4295 DCHECK_EQ(field_type, Primitive::kPrimNot); 4296 Register temp = locations->GetTemp(0).AsRegister<Register>(); 4297 __ Mov(temp, value.AsRegister<Register>()); 4298 __ PoisonHeapReference(temp); 4299 __ StoreToOffset(kStoreWord, temp, base, offset); 4300 } else { 4301 __ StoreToOffset(kStoreWord, value.AsRegister<Register>(), base, offset); 4302 } 4303 break; 4304 } 4305 4306 case Primitive::kPrimLong: { 4307 if (is_volatile && !atomic_ldrd_strd) { 4308 GenerateWideAtomicStore(base, offset, 4309 value.AsRegisterPairLow<Register>(), 4310 value.AsRegisterPairHigh<Register>(), 4311 locations->GetTemp(0).AsRegister<Register>(), 4312 locations->GetTemp(1).AsRegister<Register>(), 4313 instruction); 4314 } else { 4315 __ StoreToOffset(kStoreWordPair, value.AsRegisterPairLow<Register>(), base, offset); 4316 codegen_->MaybeRecordImplicitNullCheck(instruction); 4317 } 4318 break; 4319 } 4320 4321 case Primitive::kPrimFloat: { 4322 __ StoreSToOffset(value.AsFpuRegister<SRegister>(), base, offset); 4323 break; 4324 } 4325 4326 case Primitive::kPrimDouble: { 4327 DRegister value_reg = FromLowSToD(value.AsFpuRegisterPairLow<SRegister>()); 4328 if (is_volatile && !atomic_ldrd_strd) { 4329 Register value_reg_lo = locations->GetTemp(0).AsRegister<Register>(); 4330 Register value_reg_hi = locations->GetTemp(1).AsRegister<Register>(); 4331 4332 __ vmovrrd(value_reg_lo, value_reg_hi, value_reg); 4333 4334 GenerateWideAtomicStore(base, offset, 4335 value_reg_lo, 4336 value_reg_hi, 4337 locations->GetTemp(2).AsRegister<Register>(), 4338 locations->GetTemp(3).AsRegister<Register>(), 4339 instruction); 4340 } else { 4341 __ StoreDToOffset(value_reg, base, offset); 4342 codegen_->MaybeRecordImplicitNullCheck(instruction); 4343 } 4344 break; 4345 } 4346 4347 case Primitive::kPrimVoid: 4348 LOG(FATAL) << "Unreachable type " << field_type; 4349 UNREACHABLE(); 4350 } 4351 4352 // Longs and doubles are handled in the switch. 4353 if (field_type != Primitive::kPrimLong && field_type != Primitive::kPrimDouble) { 4354 codegen_->MaybeRecordImplicitNullCheck(instruction); 4355 } 4356 4357 if (CodeGenerator::StoreNeedsWriteBarrier(field_type, instruction->InputAt(1))) { 4358 Register temp = locations->GetTemp(0).AsRegister<Register>(); 4359 Register card = locations->GetTemp(1).AsRegister<Register>(); 4360 codegen_->MarkGCCard( 4361 temp, card, base, value.AsRegister<Register>(), value_can_be_null); 4362 } 4363 4364 if (is_volatile) { 4365 codegen_->GenerateMemoryBarrier(MemBarrierKind::kAnyAny); 4366 } 4367} 4368 4369void LocationsBuilderARM::HandleFieldGet(HInstruction* instruction, const FieldInfo& field_info) { 4370 DCHECK(instruction->IsInstanceFieldGet() || instruction->IsStaticFieldGet()); 4371 4372 bool object_field_get_with_read_barrier = 4373 kEmitCompilerReadBarrier && (field_info.GetFieldType() == Primitive::kPrimNot); 4374 LocationSummary* locations = 4375 new (GetGraph()->GetArena()) LocationSummary(instruction, 4376 object_field_get_with_read_barrier ? 4377 LocationSummary::kCallOnSlowPath : 4378 LocationSummary::kNoCall); 4379 if (object_field_get_with_read_barrier && kUseBakerReadBarrier) { 4380 locations->SetCustomSlowPathCallerSaves(RegisterSet::Empty()); // No caller-save registers. 4381 } 4382 locations->SetInAt(0, Location::RequiresRegister()); 4383 4384 bool volatile_for_double = field_info.IsVolatile() 4385 && (field_info.GetFieldType() == Primitive::kPrimDouble) 4386 && !codegen_->GetInstructionSetFeatures().HasAtomicLdrdAndStrd(); 4387 // The output overlaps in case of volatile long: we don't want the 4388 // code generated by GenerateWideAtomicLoad to overwrite the 4389 // object's location. Likewise, in the case of an object field get 4390 // with read barriers enabled, we do not want the load to overwrite 4391 // the object's location, as we need it to emit the read barrier. 4392 bool overlap = (field_info.IsVolatile() && (field_info.GetFieldType() == Primitive::kPrimLong)) || 4393 object_field_get_with_read_barrier; 4394 4395 if (Primitive::IsFloatingPointType(instruction->GetType())) { 4396 locations->SetOut(Location::RequiresFpuRegister()); 4397 } else { 4398 locations->SetOut(Location::RequiresRegister(), 4399 (overlap ? Location::kOutputOverlap : Location::kNoOutputOverlap)); 4400 } 4401 if (volatile_for_double) { 4402 // ARM encoding have some additional constraints for ldrexd/strexd: 4403 // - registers need to be consecutive 4404 // - the first register should be even but not R14. 4405 // We don't test for ARM yet, and the assertion makes sure that we 4406 // revisit this if we ever enable ARM encoding. 4407 DCHECK_EQ(InstructionSet::kThumb2, codegen_->GetInstructionSet()); 4408 locations->AddTemp(Location::RequiresRegister()); 4409 locations->AddTemp(Location::RequiresRegister()); 4410 } else if (object_field_get_with_read_barrier && kUseBakerReadBarrier) { 4411 // We need a temporary register for the read barrier marking slow 4412 // path in CodeGeneratorARM::GenerateFieldLoadWithBakerReadBarrier. 4413 locations->AddTemp(Location::RequiresRegister()); 4414 } 4415} 4416 4417Location LocationsBuilderARM::ArithmeticZeroOrFpuRegister(HInstruction* input) { 4418 DCHECK(input->GetType() == Primitive::kPrimDouble || input->GetType() == Primitive::kPrimFloat) 4419 << input->GetType(); 4420 if ((input->IsFloatConstant() && (input->AsFloatConstant()->IsArithmeticZero())) || 4421 (input->IsDoubleConstant() && (input->AsDoubleConstant()->IsArithmeticZero()))) { 4422 return Location::ConstantLocation(input->AsConstant()); 4423 } else { 4424 return Location::RequiresFpuRegister(); 4425 } 4426} 4427 4428Location LocationsBuilderARM::ArmEncodableConstantOrRegister(HInstruction* constant, 4429 Opcode opcode) { 4430 DCHECK(!Primitive::IsFloatingPointType(constant->GetType())); 4431 if (constant->IsConstant() && 4432 CanEncodeConstantAsImmediate(constant->AsConstant(), opcode)) { 4433 return Location::ConstantLocation(constant->AsConstant()); 4434 } 4435 return Location::RequiresRegister(); 4436} 4437 4438bool LocationsBuilderARM::CanEncodeConstantAsImmediate(HConstant* input_cst, 4439 Opcode opcode) { 4440 uint64_t value = static_cast<uint64_t>(Int64FromConstant(input_cst)); 4441 if (Primitive::Is64BitType(input_cst->GetType())) { 4442 Opcode high_opcode = opcode; 4443 SetCc low_set_cc = kCcDontCare; 4444 switch (opcode) { 4445 case SUB: 4446 // Flip the operation to an ADD. 4447 value = -value; 4448 opcode = ADD; 4449 FALLTHROUGH_INTENDED; 4450 case ADD: 4451 if (Low32Bits(value) == 0u) { 4452 return CanEncodeConstantAsImmediate(High32Bits(value), opcode, kCcDontCare); 4453 } 4454 high_opcode = ADC; 4455 low_set_cc = kCcSet; 4456 break; 4457 default: 4458 break; 4459 } 4460 return CanEncodeConstantAsImmediate(Low32Bits(value), opcode, low_set_cc) && 4461 CanEncodeConstantAsImmediate(High32Bits(value), high_opcode, kCcDontCare); 4462 } else { 4463 return CanEncodeConstantAsImmediate(Low32Bits(value), opcode); 4464 } 4465} 4466 4467bool LocationsBuilderARM::CanEncodeConstantAsImmediate(uint32_t value, 4468 Opcode opcode, 4469 SetCc set_cc) { 4470 ShifterOperand so; 4471 ArmAssembler* assembler = codegen_->GetAssembler(); 4472 if (assembler->ShifterOperandCanHold(kNoRegister, kNoRegister, opcode, value, set_cc, &so)) { 4473 return true; 4474 } 4475 Opcode neg_opcode = kNoOperand; 4476 switch (opcode) { 4477 case AND: neg_opcode = BIC; value = ~value; break; 4478 case ORR: neg_opcode = ORN; value = ~value; break; 4479 case ADD: neg_opcode = SUB; value = -value; break; 4480 case ADC: neg_opcode = SBC; value = ~value; break; 4481 case SUB: neg_opcode = ADD; value = -value; break; 4482 case SBC: neg_opcode = ADC; value = ~value; break; 4483 default: 4484 return false; 4485 } 4486 return assembler->ShifterOperandCanHold(kNoRegister, kNoRegister, neg_opcode, value, set_cc, &so); 4487} 4488 4489void InstructionCodeGeneratorARM::HandleFieldGet(HInstruction* instruction, 4490 const FieldInfo& field_info) { 4491 DCHECK(instruction->IsInstanceFieldGet() || instruction->IsStaticFieldGet()); 4492 4493 LocationSummary* locations = instruction->GetLocations(); 4494 Location base_loc = locations->InAt(0); 4495 Register base = base_loc.AsRegister<Register>(); 4496 Location out = locations->Out(); 4497 bool is_volatile = field_info.IsVolatile(); 4498 bool atomic_ldrd_strd = codegen_->GetInstructionSetFeatures().HasAtomicLdrdAndStrd(); 4499 Primitive::Type field_type = field_info.GetFieldType(); 4500 uint32_t offset = field_info.GetFieldOffset().Uint32Value(); 4501 4502 switch (field_type) { 4503 case Primitive::kPrimBoolean: 4504 __ LoadFromOffset(kLoadUnsignedByte, out.AsRegister<Register>(), base, offset); 4505 break; 4506 4507 case Primitive::kPrimByte: 4508 __ LoadFromOffset(kLoadSignedByte, out.AsRegister<Register>(), base, offset); 4509 break; 4510 4511 case Primitive::kPrimShort: 4512 __ LoadFromOffset(kLoadSignedHalfword, out.AsRegister<Register>(), base, offset); 4513 break; 4514 4515 case Primitive::kPrimChar: 4516 __ LoadFromOffset(kLoadUnsignedHalfword, out.AsRegister<Register>(), base, offset); 4517 break; 4518 4519 case Primitive::kPrimInt: 4520 __ LoadFromOffset(kLoadWord, out.AsRegister<Register>(), base, offset); 4521 break; 4522 4523 case Primitive::kPrimNot: { 4524 // /* HeapReference<Object> */ out = *(base + offset) 4525 if (kEmitCompilerReadBarrier && kUseBakerReadBarrier) { 4526 Location temp_loc = locations->GetTemp(0); 4527 // Note that a potential implicit null check is handled in this 4528 // CodeGeneratorARM::GenerateFieldLoadWithBakerReadBarrier call. 4529 codegen_->GenerateFieldLoadWithBakerReadBarrier( 4530 instruction, out, base, offset, temp_loc, /* needs_null_check */ true); 4531 if (is_volatile) { 4532 codegen_->GenerateMemoryBarrier(MemBarrierKind::kLoadAny); 4533 } 4534 } else { 4535 __ LoadFromOffset(kLoadWord, out.AsRegister<Register>(), base, offset); 4536 codegen_->MaybeRecordImplicitNullCheck(instruction); 4537 if (is_volatile) { 4538 codegen_->GenerateMemoryBarrier(MemBarrierKind::kLoadAny); 4539 } 4540 // If read barriers are enabled, emit read barriers other than 4541 // Baker's using a slow path (and also unpoison the loaded 4542 // reference, if heap poisoning is enabled). 4543 codegen_->MaybeGenerateReadBarrierSlow(instruction, out, out, base_loc, offset); 4544 } 4545 break; 4546 } 4547 4548 case Primitive::kPrimLong: 4549 if (is_volatile && !atomic_ldrd_strd) { 4550 GenerateWideAtomicLoad(base, offset, 4551 out.AsRegisterPairLow<Register>(), 4552 out.AsRegisterPairHigh<Register>()); 4553 } else { 4554 __ LoadFromOffset(kLoadWordPair, out.AsRegisterPairLow<Register>(), base, offset); 4555 } 4556 break; 4557 4558 case Primitive::kPrimFloat: 4559 __ LoadSFromOffset(out.AsFpuRegister<SRegister>(), base, offset); 4560 break; 4561 4562 case Primitive::kPrimDouble: { 4563 DRegister out_reg = FromLowSToD(out.AsFpuRegisterPairLow<SRegister>()); 4564 if (is_volatile && !atomic_ldrd_strd) { 4565 Register lo = locations->GetTemp(0).AsRegister<Register>(); 4566 Register hi = locations->GetTemp(1).AsRegister<Register>(); 4567 GenerateWideAtomicLoad(base, offset, lo, hi); 4568 codegen_->MaybeRecordImplicitNullCheck(instruction); 4569 __ vmovdrr(out_reg, lo, hi); 4570 } else { 4571 __ LoadDFromOffset(out_reg, base, offset); 4572 codegen_->MaybeRecordImplicitNullCheck(instruction); 4573 } 4574 break; 4575 } 4576 4577 case Primitive::kPrimVoid: 4578 LOG(FATAL) << "Unreachable type " << field_type; 4579 UNREACHABLE(); 4580 } 4581 4582 if (field_type == Primitive::kPrimNot || field_type == Primitive::kPrimDouble) { 4583 // Potential implicit null checks, in the case of reference or 4584 // double fields, are handled in the previous switch statement. 4585 } else { 4586 codegen_->MaybeRecordImplicitNullCheck(instruction); 4587 } 4588 4589 if (is_volatile) { 4590 if (field_type == Primitive::kPrimNot) { 4591 // Memory barriers, in the case of references, are also handled 4592 // in the previous switch statement. 4593 } else { 4594 codegen_->GenerateMemoryBarrier(MemBarrierKind::kLoadAny); 4595 } 4596 } 4597} 4598 4599void LocationsBuilderARM::VisitInstanceFieldSet(HInstanceFieldSet* instruction) { 4600 HandleFieldSet(instruction, instruction->GetFieldInfo()); 4601} 4602 4603void InstructionCodeGeneratorARM::VisitInstanceFieldSet(HInstanceFieldSet* instruction) { 4604 HandleFieldSet(instruction, instruction->GetFieldInfo(), instruction->GetValueCanBeNull()); 4605} 4606 4607void LocationsBuilderARM::VisitInstanceFieldGet(HInstanceFieldGet* instruction) { 4608 HandleFieldGet(instruction, instruction->GetFieldInfo()); 4609} 4610 4611void InstructionCodeGeneratorARM::VisitInstanceFieldGet(HInstanceFieldGet* instruction) { 4612 HandleFieldGet(instruction, instruction->GetFieldInfo()); 4613} 4614 4615void LocationsBuilderARM::VisitStaticFieldGet(HStaticFieldGet* instruction) { 4616 HandleFieldGet(instruction, instruction->GetFieldInfo()); 4617} 4618 4619void InstructionCodeGeneratorARM::VisitStaticFieldGet(HStaticFieldGet* instruction) { 4620 HandleFieldGet(instruction, instruction->GetFieldInfo()); 4621} 4622 4623void LocationsBuilderARM::VisitStaticFieldSet(HStaticFieldSet* instruction) { 4624 HandleFieldSet(instruction, instruction->GetFieldInfo()); 4625} 4626 4627void InstructionCodeGeneratorARM::VisitStaticFieldSet(HStaticFieldSet* instruction) { 4628 HandleFieldSet(instruction, instruction->GetFieldInfo(), instruction->GetValueCanBeNull()); 4629} 4630 4631void LocationsBuilderARM::VisitUnresolvedInstanceFieldGet( 4632 HUnresolvedInstanceFieldGet* instruction) { 4633 FieldAccessCallingConventionARM calling_convention; 4634 codegen_->CreateUnresolvedFieldLocationSummary( 4635 instruction, instruction->GetFieldType(), calling_convention); 4636} 4637 4638void InstructionCodeGeneratorARM::VisitUnresolvedInstanceFieldGet( 4639 HUnresolvedInstanceFieldGet* instruction) { 4640 FieldAccessCallingConventionARM calling_convention; 4641 codegen_->GenerateUnresolvedFieldAccess(instruction, 4642 instruction->GetFieldType(), 4643 instruction->GetFieldIndex(), 4644 instruction->GetDexPc(), 4645 calling_convention); 4646} 4647 4648void LocationsBuilderARM::VisitUnresolvedInstanceFieldSet( 4649 HUnresolvedInstanceFieldSet* instruction) { 4650 FieldAccessCallingConventionARM calling_convention; 4651 codegen_->CreateUnresolvedFieldLocationSummary( 4652 instruction, instruction->GetFieldType(), calling_convention); 4653} 4654 4655void InstructionCodeGeneratorARM::VisitUnresolvedInstanceFieldSet( 4656 HUnresolvedInstanceFieldSet* instruction) { 4657 FieldAccessCallingConventionARM calling_convention; 4658 codegen_->GenerateUnresolvedFieldAccess(instruction, 4659 instruction->GetFieldType(), 4660 instruction->GetFieldIndex(), 4661 instruction->GetDexPc(), 4662 calling_convention); 4663} 4664 4665void LocationsBuilderARM::VisitUnresolvedStaticFieldGet( 4666 HUnresolvedStaticFieldGet* instruction) { 4667 FieldAccessCallingConventionARM calling_convention; 4668 codegen_->CreateUnresolvedFieldLocationSummary( 4669 instruction, instruction->GetFieldType(), calling_convention); 4670} 4671 4672void InstructionCodeGeneratorARM::VisitUnresolvedStaticFieldGet( 4673 HUnresolvedStaticFieldGet* instruction) { 4674 FieldAccessCallingConventionARM calling_convention; 4675 codegen_->GenerateUnresolvedFieldAccess(instruction, 4676 instruction->GetFieldType(), 4677 instruction->GetFieldIndex(), 4678 instruction->GetDexPc(), 4679 calling_convention); 4680} 4681 4682void LocationsBuilderARM::VisitUnresolvedStaticFieldSet( 4683 HUnresolvedStaticFieldSet* instruction) { 4684 FieldAccessCallingConventionARM calling_convention; 4685 codegen_->CreateUnresolvedFieldLocationSummary( 4686 instruction, instruction->GetFieldType(), calling_convention); 4687} 4688 4689void InstructionCodeGeneratorARM::VisitUnresolvedStaticFieldSet( 4690 HUnresolvedStaticFieldSet* instruction) { 4691 FieldAccessCallingConventionARM calling_convention; 4692 codegen_->GenerateUnresolvedFieldAccess(instruction, 4693 instruction->GetFieldType(), 4694 instruction->GetFieldIndex(), 4695 instruction->GetDexPc(), 4696 calling_convention); 4697} 4698 4699void LocationsBuilderARM::VisitNullCheck(HNullCheck* instruction) { 4700 LocationSummary* locations = codegen_->CreateThrowingSlowPathLocations(instruction); 4701 locations->SetInAt(0, Location::RequiresRegister()); 4702} 4703 4704void CodeGeneratorARM::GenerateImplicitNullCheck(HNullCheck* instruction) { 4705 if (CanMoveNullCheckToUser(instruction)) { 4706 return; 4707 } 4708 Location obj = instruction->GetLocations()->InAt(0); 4709 4710 __ LoadFromOffset(kLoadWord, IP, obj.AsRegister<Register>(), 0); 4711 RecordPcInfo(instruction, instruction->GetDexPc()); 4712} 4713 4714void CodeGeneratorARM::GenerateExplicitNullCheck(HNullCheck* instruction) { 4715 SlowPathCodeARM* slow_path = new (GetGraph()->GetArena()) NullCheckSlowPathARM(instruction); 4716 AddSlowPath(slow_path); 4717 4718 LocationSummary* locations = instruction->GetLocations(); 4719 Location obj = locations->InAt(0); 4720 4721 __ CompareAndBranchIfZero(obj.AsRegister<Register>(), slow_path->GetEntryLabel()); 4722} 4723 4724void InstructionCodeGeneratorARM::VisitNullCheck(HNullCheck* instruction) { 4725 codegen_->GenerateNullCheck(instruction); 4726} 4727 4728static LoadOperandType GetLoadOperandType(Primitive::Type type) { 4729 switch (type) { 4730 case Primitive::kPrimNot: 4731 return kLoadWord; 4732 case Primitive::kPrimBoolean: 4733 return kLoadUnsignedByte; 4734 case Primitive::kPrimByte: 4735 return kLoadSignedByte; 4736 case Primitive::kPrimChar: 4737 return kLoadUnsignedHalfword; 4738 case Primitive::kPrimShort: 4739 return kLoadSignedHalfword; 4740 case Primitive::kPrimInt: 4741 return kLoadWord; 4742 case Primitive::kPrimLong: 4743 return kLoadWordPair; 4744 case Primitive::kPrimFloat: 4745 return kLoadSWord; 4746 case Primitive::kPrimDouble: 4747 return kLoadDWord; 4748 default: 4749 LOG(FATAL) << "Unreachable type " << type; 4750 UNREACHABLE(); 4751 } 4752} 4753 4754static StoreOperandType GetStoreOperandType(Primitive::Type type) { 4755 switch (type) { 4756 case Primitive::kPrimNot: 4757 return kStoreWord; 4758 case Primitive::kPrimBoolean: 4759 case Primitive::kPrimByte: 4760 return kStoreByte; 4761 case Primitive::kPrimChar: 4762 case Primitive::kPrimShort: 4763 return kStoreHalfword; 4764 case Primitive::kPrimInt: 4765 return kStoreWord; 4766 case Primitive::kPrimLong: 4767 return kStoreWordPair; 4768 case Primitive::kPrimFloat: 4769 return kStoreSWord; 4770 case Primitive::kPrimDouble: 4771 return kStoreDWord; 4772 default: 4773 LOG(FATAL) << "Unreachable type " << type; 4774 UNREACHABLE(); 4775 } 4776} 4777 4778void CodeGeneratorARM::LoadFromShiftedRegOffset(Primitive::Type type, 4779 Location out_loc, 4780 Register base, 4781 Register reg_offset, 4782 Condition cond) { 4783 uint32_t shift_count = Primitive::ComponentSizeShift(type); 4784 Address mem_address(base, reg_offset, Shift::LSL, shift_count); 4785 4786 switch (type) { 4787 case Primitive::kPrimByte: 4788 __ ldrsb(out_loc.AsRegister<Register>(), mem_address, cond); 4789 break; 4790 case Primitive::kPrimBoolean: 4791 __ ldrb(out_loc.AsRegister<Register>(), mem_address, cond); 4792 break; 4793 case Primitive::kPrimShort: 4794 __ ldrsh(out_loc.AsRegister<Register>(), mem_address, cond); 4795 break; 4796 case Primitive::kPrimChar: 4797 __ ldrh(out_loc.AsRegister<Register>(), mem_address, cond); 4798 break; 4799 case Primitive::kPrimNot: 4800 case Primitive::kPrimInt: 4801 __ ldr(out_loc.AsRegister<Register>(), mem_address, cond); 4802 break; 4803 // T32 doesn't support LoadFromShiftedRegOffset mem address mode for these types. 4804 case Primitive::kPrimLong: 4805 case Primitive::kPrimFloat: 4806 case Primitive::kPrimDouble: 4807 default: 4808 LOG(FATAL) << "Unreachable type " << type; 4809 UNREACHABLE(); 4810 } 4811} 4812 4813void CodeGeneratorARM::StoreToShiftedRegOffset(Primitive::Type type, 4814 Location loc, 4815 Register base, 4816 Register reg_offset, 4817 Condition cond) { 4818 uint32_t shift_count = Primitive::ComponentSizeShift(type); 4819 Address mem_address(base, reg_offset, Shift::LSL, shift_count); 4820 4821 switch (type) { 4822 case Primitive::kPrimByte: 4823 case Primitive::kPrimBoolean: 4824 __ strb(loc.AsRegister<Register>(), mem_address, cond); 4825 break; 4826 case Primitive::kPrimShort: 4827 case Primitive::kPrimChar: 4828 __ strh(loc.AsRegister<Register>(), mem_address, cond); 4829 break; 4830 case Primitive::kPrimNot: 4831 case Primitive::kPrimInt: 4832 __ str(loc.AsRegister<Register>(), mem_address, cond); 4833 break; 4834 // T32 doesn't support StoreToShiftedRegOffset mem address mode for these types. 4835 case Primitive::kPrimLong: 4836 case Primitive::kPrimFloat: 4837 case Primitive::kPrimDouble: 4838 default: 4839 LOG(FATAL) << "Unreachable type " << type; 4840 UNREACHABLE(); 4841 } 4842} 4843 4844void LocationsBuilderARM::VisitArrayGet(HArrayGet* instruction) { 4845 bool object_array_get_with_read_barrier = 4846 kEmitCompilerReadBarrier && (instruction->GetType() == Primitive::kPrimNot); 4847 LocationSummary* locations = 4848 new (GetGraph()->GetArena()) LocationSummary(instruction, 4849 object_array_get_with_read_barrier ? 4850 LocationSummary::kCallOnSlowPath : 4851 LocationSummary::kNoCall); 4852 if (object_array_get_with_read_barrier && kUseBakerReadBarrier) { 4853 locations->SetCustomSlowPathCallerSaves(RegisterSet::Empty()); // No caller-save registers. 4854 } 4855 locations->SetInAt(0, Location::RequiresRegister()); 4856 locations->SetInAt(1, Location::RegisterOrConstant(instruction->InputAt(1))); 4857 if (Primitive::IsFloatingPointType(instruction->GetType())) { 4858 locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap); 4859 } else { 4860 // The output overlaps in the case of an object array get with 4861 // read barriers enabled: we do not want the move to overwrite the 4862 // array's location, as we need it to emit the read barrier. 4863 locations->SetOut( 4864 Location::RequiresRegister(), 4865 object_array_get_with_read_barrier ? Location::kOutputOverlap : Location::kNoOutputOverlap); 4866 } 4867 // We need a temporary register for the read barrier marking slow 4868 // path in CodeGeneratorARM::GenerateArrayLoadWithBakerReadBarrier. 4869 // Also need for String compression feature. 4870 if ((object_array_get_with_read_barrier && kUseBakerReadBarrier) 4871 || (mirror::kUseStringCompression && instruction->IsStringCharAt())) { 4872 locations->AddTemp(Location::RequiresRegister()); 4873 } 4874} 4875 4876void InstructionCodeGeneratorARM::VisitArrayGet(HArrayGet* instruction) { 4877 LocationSummary* locations = instruction->GetLocations(); 4878 Location obj_loc = locations->InAt(0); 4879 Register obj = obj_loc.AsRegister<Register>(); 4880 Location index = locations->InAt(1); 4881 Location out_loc = locations->Out(); 4882 uint32_t data_offset = CodeGenerator::GetArrayDataOffset(instruction); 4883 Primitive::Type type = instruction->GetType(); 4884 const bool maybe_compressed_char_at = mirror::kUseStringCompression && 4885 instruction->IsStringCharAt(); 4886 HInstruction* array_instr = instruction->GetArray(); 4887 bool has_intermediate_address = array_instr->IsIntermediateAddress(); 4888 4889 switch (type) { 4890 case Primitive::kPrimBoolean: 4891 case Primitive::kPrimByte: 4892 case Primitive::kPrimShort: 4893 case Primitive::kPrimChar: 4894 case Primitive::kPrimInt: { 4895 Register length; 4896 if (maybe_compressed_char_at) { 4897 length = locations->GetTemp(0).AsRegister<Register>(); 4898 uint32_t count_offset = mirror::String::CountOffset().Uint32Value(); 4899 __ LoadFromOffset(kLoadWord, length, obj, count_offset); 4900 codegen_->MaybeRecordImplicitNullCheck(instruction); 4901 } 4902 if (index.IsConstant()) { 4903 int32_t const_index = index.GetConstant()->AsIntConstant()->GetValue(); 4904 if (maybe_compressed_char_at) { 4905 Label uncompressed_load, done; 4906 __ Lsrs(length, length, 1u); // LSRS has a 16-bit encoding, TST (immediate) does not. 4907 static_assert(static_cast<uint32_t>(mirror::StringCompressionFlag::kCompressed) == 0u, 4908 "Expecting 0=compressed, 1=uncompressed"); 4909 __ b(&uncompressed_load, CS); 4910 __ LoadFromOffset(kLoadUnsignedByte, 4911 out_loc.AsRegister<Register>(), 4912 obj, 4913 data_offset + const_index); 4914 __ b(&done); 4915 __ Bind(&uncompressed_load); 4916 __ LoadFromOffset(GetLoadOperandType(Primitive::kPrimChar), 4917 out_loc.AsRegister<Register>(), 4918 obj, 4919 data_offset + (const_index << 1)); 4920 __ Bind(&done); 4921 } else { 4922 uint32_t full_offset = data_offset + (const_index << Primitive::ComponentSizeShift(type)); 4923 4924 LoadOperandType load_type = GetLoadOperandType(type); 4925 __ LoadFromOffset(load_type, out_loc.AsRegister<Register>(), obj, full_offset); 4926 } 4927 } else { 4928 Register temp = IP; 4929 4930 if (has_intermediate_address) { 4931 // We do not need to compute the intermediate address from the array: the 4932 // input instruction has done it already. See the comment in 4933 // `TryExtractArrayAccessAddress()`. 4934 if (kIsDebugBuild) { 4935 HIntermediateAddress* tmp = array_instr->AsIntermediateAddress(); 4936 DCHECK_EQ(tmp->GetOffset()->AsIntConstant()->GetValueAsUint64(), data_offset); 4937 } 4938 temp = obj; 4939 } else { 4940 __ add(temp, obj, ShifterOperand(data_offset)); 4941 } 4942 if (maybe_compressed_char_at) { 4943 Label uncompressed_load, done; 4944 __ Lsrs(length, length, 1u); // LSRS has a 16-bit encoding, TST (immediate) does not. 4945 static_assert(static_cast<uint32_t>(mirror::StringCompressionFlag::kCompressed) == 0u, 4946 "Expecting 0=compressed, 1=uncompressed"); 4947 __ b(&uncompressed_load, CS); 4948 __ ldrb(out_loc.AsRegister<Register>(), 4949 Address(temp, index.AsRegister<Register>(), Shift::LSL, 0)); 4950 __ b(&done); 4951 __ Bind(&uncompressed_load); 4952 __ ldrh(out_loc.AsRegister<Register>(), 4953 Address(temp, index.AsRegister<Register>(), Shift::LSL, 1)); 4954 __ Bind(&done); 4955 } else { 4956 codegen_->LoadFromShiftedRegOffset(type, out_loc, temp, index.AsRegister<Register>()); 4957 } 4958 } 4959 break; 4960 } 4961 4962 case Primitive::kPrimNot: { 4963 // The read barrier instrumentation of object ArrayGet 4964 // instructions does not support the HIntermediateAddress 4965 // instruction. 4966 DCHECK(!(has_intermediate_address && kEmitCompilerReadBarrier)); 4967 4968 static_assert( 4969 sizeof(mirror::HeapReference<mirror::Object>) == sizeof(int32_t), 4970 "art::mirror::HeapReference<art::mirror::Object> and int32_t have different sizes."); 4971 // /* HeapReference<Object> */ out = 4972 // *(obj + data_offset + index * sizeof(HeapReference<Object>)) 4973 if (kEmitCompilerReadBarrier && kUseBakerReadBarrier) { 4974 Location temp = locations->GetTemp(0); 4975 // Note that a potential implicit null check is handled in this 4976 // CodeGeneratorARM::GenerateArrayLoadWithBakerReadBarrier call. 4977 codegen_->GenerateArrayLoadWithBakerReadBarrier( 4978 instruction, out_loc, obj, data_offset, index, temp, /* needs_null_check */ true); 4979 } else { 4980 Register out = out_loc.AsRegister<Register>(); 4981 if (index.IsConstant()) { 4982 size_t offset = 4983 (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4) + data_offset; 4984 __ LoadFromOffset(kLoadWord, out, obj, offset); 4985 codegen_->MaybeRecordImplicitNullCheck(instruction); 4986 // If read barriers are enabled, emit read barriers other than 4987 // Baker's using a slow path (and also unpoison the loaded 4988 // reference, if heap poisoning is enabled). 4989 codegen_->MaybeGenerateReadBarrierSlow(instruction, out_loc, out_loc, obj_loc, offset); 4990 } else { 4991 Register temp = IP; 4992 4993 if (has_intermediate_address) { 4994 // We do not need to compute the intermediate address from the array: the 4995 // input instruction has done it already. See the comment in 4996 // `TryExtractArrayAccessAddress()`. 4997 if (kIsDebugBuild) { 4998 HIntermediateAddress* tmp = array_instr->AsIntermediateAddress(); 4999 DCHECK_EQ(tmp->GetOffset()->AsIntConstant()->GetValueAsUint64(), data_offset); 5000 } 5001 temp = obj; 5002 } else { 5003 __ add(temp, obj, ShifterOperand(data_offset)); 5004 } 5005 codegen_->LoadFromShiftedRegOffset(type, out_loc, temp, index.AsRegister<Register>()); 5006 5007 codegen_->MaybeRecordImplicitNullCheck(instruction); 5008 // If read barriers are enabled, emit read barriers other than 5009 // Baker's using a slow path (and also unpoison the loaded 5010 // reference, if heap poisoning is enabled). 5011 codegen_->MaybeGenerateReadBarrierSlow( 5012 instruction, out_loc, out_loc, obj_loc, data_offset, index); 5013 } 5014 } 5015 break; 5016 } 5017 5018 case Primitive::kPrimLong: { 5019 if (index.IsConstant()) { 5020 size_t offset = 5021 (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_8) + data_offset; 5022 __ LoadFromOffset(kLoadWordPair, out_loc.AsRegisterPairLow<Register>(), obj, offset); 5023 } else { 5024 __ add(IP, obj, ShifterOperand(index.AsRegister<Register>(), LSL, TIMES_8)); 5025 __ LoadFromOffset(kLoadWordPair, out_loc.AsRegisterPairLow<Register>(), IP, data_offset); 5026 } 5027 break; 5028 } 5029 5030 case Primitive::kPrimFloat: { 5031 SRegister out = out_loc.AsFpuRegister<SRegister>(); 5032 if (index.IsConstant()) { 5033 size_t offset = (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4) + data_offset; 5034 __ LoadSFromOffset(out, obj, offset); 5035 } else { 5036 __ add(IP, obj, ShifterOperand(index.AsRegister<Register>(), LSL, TIMES_4)); 5037 __ LoadSFromOffset(out, IP, data_offset); 5038 } 5039 break; 5040 } 5041 5042 case Primitive::kPrimDouble: { 5043 SRegister out = out_loc.AsFpuRegisterPairLow<SRegister>(); 5044 if (index.IsConstant()) { 5045 size_t offset = (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_8) + data_offset; 5046 __ LoadDFromOffset(FromLowSToD(out), obj, offset); 5047 } else { 5048 __ add(IP, obj, ShifterOperand(index.AsRegister<Register>(), LSL, TIMES_8)); 5049 __ LoadDFromOffset(FromLowSToD(out), IP, data_offset); 5050 } 5051 break; 5052 } 5053 5054 case Primitive::kPrimVoid: 5055 LOG(FATAL) << "Unreachable type " << type; 5056 UNREACHABLE(); 5057 } 5058 5059 if (type == Primitive::kPrimNot) { 5060 // Potential implicit null checks, in the case of reference 5061 // arrays, are handled in the previous switch statement. 5062 } else if (!maybe_compressed_char_at) { 5063 codegen_->MaybeRecordImplicitNullCheck(instruction); 5064 } 5065} 5066 5067void LocationsBuilderARM::VisitArraySet(HArraySet* instruction) { 5068 Primitive::Type value_type = instruction->GetComponentType(); 5069 5070 bool needs_write_barrier = 5071 CodeGenerator::StoreNeedsWriteBarrier(value_type, instruction->GetValue()); 5072 bool may_need_runtime_call_for_type_check = instruction->NeedsTypeCheck(); 5073 5074 LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary( 5075 instruction, 5076 may_need_runtime_call_for_type_check ? 5077 LocationSummary::kCallOnSlowPath : 5078 LocationSummary::kNoCall); 5079 5080 locations->SetInAt(0, Location::RequiresRegister()); 5081 locations->SetInAt(1, Location::RegisterOrConstant(instruction->InputAt(1))); 5082 if (Primitive::IsFloatingPointType(value_type)) { 5083 locations->SetInAt(2, Location::RequiresFpuRegister()); 5084 } else { 5085 locations->SetInAt(2, Location::RequiresRegister()); 5086 } 5087 if (needs_write_barrier) { 5088 // Temporary registers for the write barrier. 5089 locations->AddTemp(Location::RequiresRegister()); // Possibly used for ref. poisoning too. 5090 locations->AddTemp(Location::RequiresRegister()); 5091 } 5092} 5093 5094void InstructionCodeGeneratorARM::VisitArraySet(HArraySet* instruction) { 5095 LocationSummary* locations = instruction->GetLocations(); 5096 Location array_loc = locations->InAt(0); 5097 Register array = array_loc.AsRegister<Register>(); 5098 Location index = locations->InAt(1); 5099 Primitive::Type value_type = instruction->GetComponentType(); 5100 bool may_need_runtime_call_for_type_check = instruction->NeedsTypeCheck(); 5101 bool needs_write_barrier = 5102 CodeGenerator::StoreNeedsWriteBarrier(value_type, instruction->GetValue()); 5103 uint32_t data_offset = 5104 mirror::Array::DataOffset(Primitive::ComponentSize(value_type)).Uint32Value(); 5105 Location value_loc = locations->InAt(2); 5106 HInstruction* array_instr = instruction->GetArray(); 5107 bool has_intermediate_address = array_instr->IsIntermediateAddress(); 5108 5109 switch (value_type) { 5110 case Primitive::kPrimBoolean: 5111 case Primitive::kPrimByte: 5112 case Primitive::kPrimShort: 5113 case Primitive::kPrimChar: 5114 case Primitive::kPrimInt: { 5115 if (index.IsConstant()) { 5116 int32_t const_index = index.GetConstant()->AsIntConstant()->GetValue(); 5117 uint32_t full_offset = 5118 data_offset + (const_index << Primitive::ComponentSizeShift(value_type)); 5119 StoreOperandType store_type = GetStoreOperandType(value_type); 5120 __ StoreToOffset(store_type, value_loc.AsRegister<Register>(), array, full_offset); 5121 } else { 5122 Register temp = IP; 5123 5124 if (has_intermediate_address) { 5125 // We do not need to compute the intermediate address from the array: the 5126 // input instruction has done it already. See the comment in 5127 // `TryExtractArrayAccessAddress()`. 5128 if (kIsDebugBuild) { 5129 HIntermediateAddress* tmp = array_instr->AsIntermediateAddress(); 5130 DCHECK(tmp->GetOffset()->AsIntConstant()->GetValueAsUint64() == data_offset); 5131 } 5132 temp = array; 5133 } else { 5134 __ add(temp, array, ShifterOperand(data_offset)); 5135 } 5136 codegen_->StoreToShiftedRegOffset(value_type, 5137 value_loc, 5138 temp, 5139 index.AsRegister<Register>()); 5140 } 5141 break; 5142 } 5143 5144 case Primitive::kPrimNot: { 5145 Register value = value_loc.AsRegister<Register>(); 5146 // TryExtractArrayAccessAddress optimization is never applied for non-primitive ArraySet. 5147 // See the comment in instruction_simplifier_shared.cc. 5148 DCHECK(!has_intermediate_address); 5149 5150 if (instruction->InputAt(2)->IsNullConstant()) { 5151 // Just setting null. 5152 if (index.IsConstant()) { 5153 size_t offset = 5154 (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4) + data_offset; 5155 __ StoreToOffset(kStoreWord, value, array, offset); 5156 } else { 5157 DCHECK(index.IsRegister()) << index; 5158 __ add(IP, array, ShifterOperand(data_offset)); 5159 codegen_->StoreToShiftedRegOffset(value_type, 5160 value_loc, 5161 IP, 5162 index.AsRegister<Register>()); 5163 } 5164 codegen_->MaybeRecordImplicitNullCheck(instruction); 5165 DCHECK(!needs_write_barrier); 5166 DCHECK(!may_need_runtime_call_for_type_check); 5167 break; 5168 } 5169 5170 DCHECK(needs_write_barrier); 5171 Location temp1_loc = locations->GetTemp(0); 5172 Register temp1 = temp1_loc.AsRegister<Register>(); 5173 Location temp2_loc = locations->GetTemp(1); 5174 Register temp2 = temp2_loc.AsRegister<Register>(); 5175 uint32_t class_offset = mirror::Object::ClassOffset().Int32Value(); 5176 uint32_t super_offset = mirror::Class::SuperClassOffset().Int32Value(); 5177 uint32_t component_offset = mirror::Class::ComponentTypeOffset().Int32Value(); 5178 Label done; 5179 SlowPathCodeARM* slow_path = nullptr; 5180 5181 if (may_need_runtime_call_for_type_check) { 5182 slow_path = new (GetGraph()->GetArena()) ArraySetSlowPathARM(instruction); 5183 codegen_->AddSlowPath(slow_path); 5184 if (instruction->GetValueCanBeNull()) { 5185 Label non_zero; 5186 __ CompareAndBranchIfNonZero(value, &non_zero); 5187 if (index.IsConstant()) { 5188 size_t offset = 5189 (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4) + data_offset; 5190 __ StoreToOffset(kStoreWord, value, array, offset); 5191 } else { 5192 DCHECK(index.IsRegister()) << index; 5193 __ add(IP, array, ShifterOperand(data_offset)); 5194 codegen_->StoreToShiftedRegOffset(value_type, 5195 value_loc, 5196 IP, 5197 index.AsRegister<Register>()); 5198 } 5199 codegen_->MaybeRecordImplicitNullCheck(instruction); 5200 __ b(&done); 5201 __ Bind(&non_zero); 5202 } 5203 5204 // Note that when read barriers are enabled, the type checks 5205 // are performed without read barriers. This is fine, even in 5206 // the case where a class object is in the from-space after 5207 // the flip, as a comparison involving such a type would not 5208 // produce a false positive; it may of course produce a false 5209 // negative, in which case we would take the ArraySet slow 5210 // path. 5211 5212 // /* HeapReference<Class> */ temp1 = array->klass_ 5213 __ LoadFromOffset(kLoadWord, temp1, array, class_offset); 5214 codegen_->MaybeRecordImplicitNullCheck(instruction); 5215 __ MaybeUnpoisonHeapReference(temp1); 5216 5217 // /* HeapReference<Class> */ temp1 = temp1->component_type_ 5218 __ LoadFromOffset(kLoadWord, temp1, temp1, component_offset); 5219 // /* HeapReference<Class> */ temp2 = value->klass_ 5220 __ LoadFromOffset(kLoadWord, temp2, value, class_offset); 5221 // If heap poisoning is enabled, no need to unpoison `temp1` 5222 // nor `temp2`, as we are comparing two poisoned references. 5223 __ cmp(temp1, ShifterOperand(temp2)); 5224 5225 if (instruction->StaticTypeOfArrayIsObjectArray()) { 5226 Label do_put; 5227 __ b(&do_put, EQ); 5228 // If heap poisoning is enabled, the `temp1` reference has 5229 // not been unpoisoned yet; unpoison it now. 5230 __ MaybeUnpoisonHeapReference(temp1); 5231 5232 // /* HeapReference<Class> */ temp1 = temp1->super_class_ 5233 __ LoadFromOffset(kLoadWord, temp1, temp1, super_offset); 5234 // If heap poisoning is enabled, no need to unpoison 5235 // `temp1`, as we are comparing against null below. 5236 __ CompareAndBranchIfNonZero(temp1, slow_path->GetEntryLabel()); 5237 __ Bind(&do_put); 5238 } else { 5239 __ b(slow_path->GetEntryLabel(), NE); 5240 } 5241 } 5242 5243 Register source = value; 5244 if (kPoisonHeapReferences) { 5245 // Note that in the case where `value` is a null reference, 5246 // we do not enter this block, as a null reference does not 5247 // need poisoning. 5248 DCHECK_EQ(value_type, Primitive::kPrimNot); 5249 __ Mov(temp1, value); 5250 __ PoisonHeapReference(temp1); 5251 source = temp1; 5252 } 5253 5254 if (index.IsConstant()) { 5255 size_t offset = 5256 (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4) + data_offset; 5257 __ StoreToOffset(kStoreWord, source, array, offset); 5258 } else { 5259 DCHECK(index.IsRegister()) << index; 5260 5261 __ add(IP, array, ShifterOperand(data_offset)); 5262 codegen_->StoreToShiftedRegOffset(value_type, 5263 Location::RegisterLocation(source), 5264 IP, 5265 index.AsRegister<Register>()); 5266 } 5267 5268 if (!may_need_runtime_call_for_type_check) { 5269 codegen_->MaybeRecordImplicitNullCheck(instruction); 5270 } 5271 5272 codegen_->MarkGCCard(temp1, temp2, array, value, instruction->GetValueCanBeNull()); 5273 5274 if (done.IsLinked()) { 5275 __ Bind(&done); 5276 } 5277 5278 if (slow_path != nullptr) { 5279 __ Bind(slow_path->GetExitLabel()); 5280 } 5281 5282 break; 5283 } 5284 5285 case Primitive::kPrimLong: { 5286 Location value = locations->InAt(2); 5287 if (index.IsConstant()) { 5288 size_t offset = 5289 (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_8) + data_offset; 5290 __ StoreToOffset(kStoreWordPair, value.AsRegisterPairLow<Register>(), array, offset); 5291 } else { 5292 __ add(IP, array, ShifterOperand(index.AsRegister<Register>(), LSL, TIMES_8)); 5293 __ StoreToOffset(kStoreWordPair, value.AsRegisterPairLow<Register>(), IP, data_offset); 5294 } 5295 break; 5296 } 5297 5298 case Primitive::kPrimFloat: { 5299 Location value = locations->InAt(2); 5300 DCHECK(value.IsFpuRegister()); 5301 if (index.IsConstant()) { 5302 size_t offset = (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4) + data_offset; 5303 __ StoreSToOffset(value.AsFpuRegister<SRegister>(), array, offset); 5304 } else { 5305 __ add(IP, array, ShifterOperand(index.AsRegister<Register>(), LSL, TIMES_4)); 5306 __ StoreSToOffset(value.AsFpuRegister<SRegister>(), IP, data_offset); 5307 } 5308 break; 5309 } 5310 5311 case Primitive::kPrimDouble: { 5312 Location value = locations->InAt(2); 5313 DCHECK(value.IsFpuRegisterPair()); 5314 if (index.IsConstant()) { 5315 size_t offset = (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_8) + data_offset; 5316 __ StoreDToOffset(FromLowSToD(value.AsFpuRegisterPairLow<SRegister>()), array, offset); 5317 } else { 5318 __ add(IP, array, ShifterOperand(index.AsRegister<Register>(), LSL, TIMES_8)); 5319 __ StoreDToOffset(FromLowSToD(value.AsFpuRegisterPairLow<SRegister>()), IP, data_offset); 5320 } 5321 5322 break; 5323 } 5324 5325 case Primitive::kPrimVoid: 5326 LOG(FATAL) << "Unreachable type " << value_type; 5327 UNREACHABLE(); 5328 } 5329 5330 // Objects are handled in the switch. 5331 if (value_type != Primitive::kPrimNot) { 5332 codegen_->MaybeRecordImplicitNullCheck(instruction); 5333 } 5334} 5335 5336void LocationsBuilderARM::VisitArrayLength(HArrayLength* instruction) { 5337 LocationSummary* locations = 5338 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall); 5339 locations->SetInAt(0, Location::RequiresRegister()); 5340 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 5341} 5342 5343void InstructionCodeGeneratorARM::VisitArrayLength(HArrayLength* instruction) { 5344 LocationSummary* locations = instruction->GetLocations(); 5345 uint32_t offset = CodeGenerator::GetArrayLengthOffset(instruction); 5346 Register obj = locations->InAt(0).AsRegister<Register>(); 5347 Register out = locations->Out().AsRegister<Register>(); 5348 __ LoadFromOffset(kLoadWord, out, obj, offset); 5349 codegen_->MaybeRecordImplicitNullCheck(instruction); 5350 // Mask out compression flag from String's array length. 5351 if (mirror::kUseStringCompression && instruction->IsStringLength()) { 5352 __ Lsr(out, out, 1u); 5353 } 5354} 5355 5356void LocationsBuilderARM::VisitIntermediateAddress(HIntermediateAddress* instruction) { 5357 LocationSummary* locations = 5358 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall); 5359 5360 locations->SetInAt(0, Location::RequiresRegister()); 5361 locations->SetInAt(1, Location::RegisterOrConstant(instruction->GetOffset())); 5362 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 5363} 5364 5365void InstructionCodeGeneratorARM::VisitIntermediateAddress(HIntermediateAddress* instruction) { 5366 LocationSummary* locations = instruction->GetLocations(); 5367 Location out = locations->Out(); 5368 Location first = locations->InAt(0); 5369 Location second = locations->InAt(1); 5370 5371 if (second.IsRegister()) { 5372 __ add(out.AsRegister<Register>(), 5373 first.AsRegister<Register>(), 5374 ShifterOperand(second.AsRegister<Register>())); 5375 } else { 5376 __ AddConstant(out.AsRegister<Register>(), 5377 first.AsRegister<Register>(), 5378 second.GetConstant()->AsIntConstant()->GetValue()); 5379 } 5380} 5381 5382void LocationsBuilderARM::VisitBoundsCheck(HBoundsCheck* instruction) { 5383 RegisterSet caller_saves = RegisterSet::Empty(); 5384 InvokeRuntimeCallingConvention calling_convention; 5385 caller_saves.Add(Location::RegisterLocation(calling_convention.GetRegisterAt(0))); 5386 caller_saves.Add(Location::RegisterLocation(calling_convention.GetRegisterAt(1))); 5387 LocationSummary* locations = codegen_->CreateThrowingSlowPathLocations(instruction, caller_saves); 5388 locations->SetInAt(0, Location::RequiresRegister()); 5389 locations->SetInAt(1, Location::RequiresRegister()); 5390} 5391 5392void InstructionCodeGeneratorARM::VisitBoundsCheck(HBoundsCheck* instruction) { 5393 LocationSummary* locations = instruction->GetLocations(); 5394 SlowPathCodeARM* slow_path = 5395 new (GetGraph()->GetArena()) BoundsCheckSlowPathARM(instruction); 5396 codegen_->AddSlowPath(slow_path); 5397 5398 Register index = locations->InAt(0).AsRegister<Register>(); 5399 Register length = locations->InAt(1).AsRegister<Register>(); 5400 5401 __ cmp(index, ShifterOperand(length)); 5402 __ b(slow_path->GetEntryLabel(), HS); 5403} 5404 5405void CodeGeneratorARM::MarkGCCard(Register temp, 5406 Register card, 5407 Register object, 5408 Register value, 5409 bool can_be_null) { 5410 Label is_null; 5411 if (can_be_null) { 5412 __ CompareAndBranchIfZero(value, &is_null); 5413 } 5414 __ LoadFromOffset(kLoadWord, card, TR, Thread::CardTableOffset<kArmPointerSize>().Int32Value()); 5415 __ Lsr(temp, object, gc::accounting::CardTable::kCardShift); 5416 __ strb(card, Address(card, temp)); 5417 if (can_be_null) { 5418 __ Bind(&is_null); 5419 } 5420} 5421 5422void LocationsBuilderARM::VisitParallelMove(HParallelMove* instruction ATTRIBUTE_UNUSED) { 5423 LOG(FATAL) << "Unreachable"; 5424} 5425 5426void InstructionCodeGeneratorARM::VisitParallelMove(HParallelMove* instruction) { 5427 codegen_->GetMoveResolver()->EmitNativeCode(instruction); 5428} 5429 5430void LocationsBuilderARM::VisitSuspendCheck(HSuspendCheck* instruction) { 5431 LocationSummary* locations = 5432 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCallOnSlowPath); 5433 locations->SetCustomSlowPathCallerSaves(RegisterSet::Empty()); // No caller-save registers. 5434} 5435 5436void InstructionCodeGeneratorARM::VisitSuspendCheck(HSuspendCheck* instruction) { 5437 HBasicBlock* block = instruction->GetBlock(); 5438 if (block->GetLoopInformation() != nullptr) { 5439 DCHECK(block->GetLoopInformation()->GetSuspendCheck() == instruction); 5440 // The back edge will generate the suspend check. 5441 return; 5442 } 5443 if (block->IsEntryBlock() && instruction->GetNext()->IsGoto()) { 5444 // The goto will generate the suspend check. 5445 return; 5446 } 5447 GenerateSuspendCheck(instruction, nullptr); 5448} 5449 5450void InstructionCodeGeneratorARM::GenerateSuspendCheck(HSuspendCheck* instruction, 5451 HBasicBlock* successor) { 5452 SuspendCheckSlowPathARM* slow_path = 5453 down_cast<SuspendCheckSlowPathARM*>(instruction->GetSlowPath()); 5454 if (slow_path == nullptr) { 5455 slow_path = new (GetGraph()->GetArena()) SuspendCheckSlowPathARM(instruction, successor); 5456 instruction->SetSlowPath(slow_path); 5457 codegen_->AddSlowPath(slow_path); 5458 if (successor != nullptr) { 5459 DCHECK(successor->IsLoopHeader()); 5460 codegen_->ClearSpillSlotsFromLoopPhisInStackMap(instruction); 5461 } 5462 } else { 5463 DCHECK_EQ(slow_path->GetSuccessor(), successor); 5464 } 5465 5466 __ LoadFromOffset( 5467 kLoadUnsignedHalfword, IP, TR, Thread::ThreadFlagsOffset<kArmPointerSize>().Int32Value()); 5468 if (successor == nullptr) { 5469 __ CompareAndBranchIfNonZero(IP, slow_path->GetEntryLabel()); 5470 __ Bind(slow_path->GetReturnLabel()); 5471 } else { 5472 __ CompareAndBranchIfZero(IP, codegen_->GetLabelOf(successor)); 5473 __ b(slow_path->GetEntryLabel()); 5474 } 5475} 5476 5477ArmAssembler* ParallelMoveResolverARM::GetAssembler() const { 5478 return codegen_->GetAssembler(); 5479} 5480 5481void ParallelMoveResolverARM::EmitMove(size_t index) { 5482 MoveOperands* move = moves_[index]; 5483 Location source = move->GetSource(); 5484 Location destination = move->GetDestination(); 5485 5486 if (source.IsRegister()) { 5487 if (destination.IsRegister()) { 5488 __ Mov(destination.AsRegister<Register>(), source.AsRegister<Register>()); 5489 } else if (destination.IsFpuRegister()) { 5490 __ vmovsr(destination.AsFpuRegister<SRegister>(), source.AsRegister<Register>()); 5491 } else { 5492 DCHECK(destination.IsStackSlot()); 5493 __ StoreToOffset(kStoreWord, source.AsRegister<Register>(), 5494 SP, destination.GetStackIndex()); 5495 } 5496 } else if (source.IsStackSlot()) { 5497 if (destination.IsRegister()) { 5498 __ LoadFromOffset(kLoadWord, destination.AsRegister<Register>(), 5499 SP, source.GetStackIndex()); 5500 } else if (destination.IsFpuRegister()) { 5501 __ LoadSFromOffset(destination.AsFpuRegister<SRegister>(), SP, source.GetStackIndex()); 5502 } else { 5503 DCHECK(destination.IsStackSlot()); 5504 __ LoadFromOffset(kLoadWord, IP, SP, source.GetStackIndex()); 5505 __ StoreToOffset(kStoreWord, IP, SP, destination.GetStackIndex()); 5506 } 5507 } else if (source.IsFpuRegister()) { 5508 if (destination.IsRegister()) { 5509 __ vmovrs(destination.AsRegister<Register>(), source.AsFpuRegister<SRegister>()); 5510 } else if (destination.IsFpuRegister()) { 5511 __ vmovs(destination.AsFpuRegister<SRegister>(), source.AsFpuRegister<SRegister>()); 5512 } else { 5513 DCHECK(destination.IsStackSlot()); 5514 __ StoreSToOffset(source.AsFpuRegister<SRegister>(), SP, destination.GetStackIndex()); 5515 } 5516 } else if (source.IsDoubleStackSlot()) { 5517 if (destination.IsDoubleStackSlot()) { 5518 __ LoadDFromOffset(DTMP, SP, source.GetStackIndex()); 5519 __ StoreDToOffset(DTMP, SP, destination.GetStackIndex()); 5520 } else if (destination.IsRegisterPair()) { 5521 DCHECK(ExpectedPairLayout(destination)); 5522 __ LoadFromOffset( 5523 kLoadWordPair, destination.AsRegisterPairLow<Register>(), SP, source.GetStackIndex()); 5524 } else { 5525 DCHECK(destination.IsFpuRegisterPair()) << destination; 5526 __ LoadDFromOffset(FromLowSToD(destination.AsFpuRegisterPairLow<SRegister>()), 5527 SP, 5528 source.GetStackIndex()); 5529 } 5530 } else if (source.IsRegisterPair()) { 5531 if (destination.IsRegisterPair()) { 5532 __ Mov(destination.AsRegisterPairLow<Register>(), source.AsRegisterPairLow<Register>()); 5533 __ Mov(destination.AsRegisterPairHigh<Register>(), source.AsRegisterPairHigh<Register>()); 5534 } else if (destination.IsFpuRegisterPair()) { 5535 __ vmovdrr(FromLowSToD(destination.AsFpuRegisterPairLow<SRegister>()), 5536 source.AsRegisterPairLow<Register>(), 5537 source.AsRegisterPairHigh<Register>()); 5538 } else { 5539 DCHECK(destination.IsDoubleStackSlot()) << destination; 5540 DCHECK(ExpectedPairLayout(source)); 5541 __ StoreToOffset( 5542 kStoreWordPair, source.AsRegisterPairLow<Register>(), SP, destination.GetStackIndex()); 5543 } 5544 } else if (source.IsFpuRegisterPair()) { 5545 if (destination.IsRegisterPair()) { 5546 __ vmovrrd(destination.AsRegisterPairLow<Register>(), 5547 destination.AsRegisterPairHigh<Register>(), 5548 FromLowSToD(source.AsFpuRegisterPairLow<SRegister>())); 5549 } else if (destination.IsFpuRegisterPair()) { 5550 __ vmovd(FromLowSToD(destination.AsFpuRegisterPairLow<SRegister>()), 5551 FromLowSToD(source.AsFpuRegisterPairLow<SRegister>())); 5552 } else { 5553 DCHECK(destination.IsDoubleStackSlot()) << destination; 5554 __ StoreDToOffset(FromLowSToD(source.AsFpuRegisterPairLow<SRegister>()), 5555 SP, 5556 destination.GetStackIndex()); 5557 } 5558 } else { 5559 DCHECK(source.IsConstant()) << source; 5560 HConstant* constant = source.GetConstant(); 5561 if (constant->IsIntConstant() || constant->IsNullConstant()) { 5562 int32_t value = CodeGenerator::GetInt32ValueOf(constant); 5563 if (destination.IsRegister()) { 5564 __ LoadImmediate(destination.AsRegister<Register>(), value); 5565 } else { 5566 DCHECK(destination.IsStackSlot()); 5567 __ LoadImmediate(IP, value); 5568 __ StoreToOffset(kStoreWord, IP, SP, destination.GetStackIndex()); 5569 } 5570 } else if (constant->IsLongConstant()) { 5571 int64_t value = constant->AsLongConstant()->GetValue(); 5572 if (destination.IsRegisterPair()) { 5573 __ LoadImmediate(destination.AsRegisterPairLow<Register>(), Low32Bits(value)); 5574 __ LoadImmediate(destination.AsRegisterPairHigh<Register>(), High32Bits(value)); 5575 } else { 5576 DCHECK(destination.IsDoubleStackSlot()) << destination; 5577 __ LoadImmediate(IP, Low32Bits(value)); 5578 __ StoreToOffset(kStoreWord, IP, SP, destination.GetStackIndex()); 5579 __ LoadImmediate(IP, High32Bits(value)); 5580 __ StoreToOffset(kStoreWord, IP, SP, destination.GetHighStackIndex(kArmWordSize)); 5581 } 5582 } else if (constant->IsDoubleConstant()) { 5583 double value = constant->AsDoubleConstant()->GetValue(); 5584 if (destination.IsFpuRegisterPair()) { 5585 __ LoadDImmediate(FromLowSToD(destination.AsFpuRegisterPairLow<SRegister>()), value); 5586 } else { 5587 DCHECK(destination.IsDoubleStackSlot()) << destination; 5588 uint64_t int_value = bit_cast<uint64_t, double>(value); 5589 __ LoadImmediate(IP, Low32Bits(int_value)); 5590 __ StoreToOffset(kStoreWord, IP, SP, destination.GetStackIndex()); 5591 __ LoadImmediate(IP, High32Bits(int_value)); 5592 __ StoreToOffset(kStoreWord, IP, SP, destination.GetHighStackIndex(kArmWordSize)); 5593 } 5594 } else { 5595 DCHECK(constant->IsFloatConstant()) << constant->DebugName(); 5596 float value = constant->AsFloatConstant()->GetValue(); 5597 if (destination.IsFpuRegister()) { 5598 __ LoadSImmediate(destination.AsFpuRegister<SRegister>(), value); 5599 } else { 5600 DCHECK(destination.IsStackSlot()); 5601 __ LoadImmediate(IP, bit_cast<int32_t, float>(value)); 5602 __ StoreToOffset(kStoreWord, IP, SP, destination.GetStackIndex()); 5603 } 5604 } 5605 } 5606} 5607 5608void ParallelMoveResolverARM::Exchange(Register reg, int mem) { 5609 __ Mov(IP, reg); 5610 __ LoadFromOffset(kLoadWord, reg, SP, mem); 5611 __ StoreToOffset(kStoreWord, IP, SP, mem); 5612} 5613 5614void ParallelMoveResolverARM::Exchange(int mem1, int mem2) { 5615 ScratchRegisterScope ensure_scratch(this, IP, R0, codegen_->GetNumberOfCoreRegisters()); 5616 int stack_offset = ensure_scratch.IsSpilled() ? kArmWordSize : 0; 5617 __ LoadFromOffset(kLoadWord, static_cast<Register>(ensure_scratch.GetRegister()), 5618 SP, mem1 + stack_offset); 5619 __ LoadFromOffset(kLoadWord, IP, SP, mem2 + stack_offset); 5620 __ StoreToOffset(kStoreWord, static_cast<Register>(ensure_scratch.GetRegister()), 5621 SP, mem2 + stack_offset); 5622 __ StoreToOffset(kStoreWord, IP, SP, mem1 + stack_offset); 5623} 5624 5625void ParallelMoveResolverARM::EmitSwap(size_t index) { 5626 MoveOperands* move = moves_[index]; 5627 Location source = move->GetSource(); 5628 Location destination = move->GetDestination(); 5629 5630 if (source.IsRegister() && destination.IsRegister()) { 5631 DCHECK_NE(source.AsRegister<Register>(), IP); 5632 DCHECK_NE(destination.AsRegister<Register>(), IP); 5633 __ Mov(IP, source.AsRegister<Register>()); 5634 __ Mov(source.AsRegister<Register>(), destination.AsRegister<Register>()); 5635 __ Mov(destination.AsRegister<Register>(), IP); 5636 } else if (source.IsRegister() && destination.IsStackSlot()) { 5637 Exchange(source.AsRegister<Register>(), destination.GetStackIndex()); 5638 } else if (source.IsStackSlot() && destination.IsRegister()) { 5639 Exchange(destination.AsRegister<Register>(), source.GetStackIndex()); 5640 } else if (source.IsStackSlot() && destination.IsStackSlot()) { 5641 Exchange(source.GetStackIndex(), destination.GetStackIndex()); 5642 } else if (source.IsFpuRegister() && destination.IsFpuRegister()) { 5643 __ vmovrs(IP, source.AsFpuRegister<SRegister>()); 5644 __ vmovs(source.AsFpuRegister<SRegister>(), destination.AsFpuRegister<SRegister>()); 5645 __ vmovsr(destination.AsFpuRegister<SRegister>(), IP); 5646 } else if (source.IsRegisterPair() && destination.IsRegisterPair()) { 5647 __ vmovdrr(DTMP, source.AsRegisterPairLow<Register>(), source.AsRegisterPairHigh<Register>()); 5648 __ Mov(source.AsRegisterPairLow<Register>(), destination.AsRegisterPairLow<Register>()); 5649 __ Mov(source.AsRegisterPairHigh<Register>(), destination.AsRegisterPairHigh<Register>()); 5650 __ vmovrrd(destination.AsRegisterPairLow<Register>(), 5651 destination.AsRegisterPairHigh<Register>(), 5652 DTMP); 5653 } else if (source.IsRegisterPair() || destination.IsRegisterPair()) { 5654 Register low_reg = source.IsRegisterPair() 5655 ? source.AsRegisterPairLow<Register>() 5656 : destination.AsRegisterPairLow<Register>(); 5657 int mem = source.IsRegisterPair() 5658 ? destination.GetStackIndex() 5659 : source.GetStackIndex(); 5660 DCHECK(ExpectedPairLayout(source.IsRegisterPair() ? source : destination)); 5661 __ vmovdrr(DTMP, low_reg, static_cast<Register>(low_reg + 1)); 5662 __ LoadFromOffset(kLoadWordPair, low_reg, SP, mem); 5663 __ StoreDToOffset(DTMP, SP, mem); 5664 } else if (source.IsFpuRegisterPair() && destination.IsFpuRegisterPair()) { 5665 DRegister first = FromLowSToD(source.AsFpuRegisterPairLow<SRegister>()); 5666 DRegister second = FromLowSToD(destination.AsFpuRegisterPairLow<SRegister>()); 5667 __ vmovd(DTMP, first); 5668 __ vmovd(first, second); 5669 __ vmovd(second, DTMP); 5670 } else if (source.IsFpuRegisterPair() || destination.IsFpuRegisterPair()) { 5671 DRegister reg = source.IsFpuRegisterPair() 5672 ? FromLowSToD(source.AsFpuRegisterPairLow<SRegister>()) 5673 : FromLowSToD(destination.AsFpuRegisterPairLow<SRegister>()); 5674 int mem = source.IsFpuRegisterPair() 5675 ? destination.GetStackIndex() 5676 : source.GetStackIndex(); 5677 __ vmovd(DTMP, reg); 5678 __ LoadDFromOffset(reg, SP, mem); 5679 __ StoreDToOffset(DTMP, SP, mem); 5680 } else if (source.IsFpuRegister() || destination.IsFpuRegister()) { 5681 SRegister reg = source.IsFpuRegister() ? source.AsFpuRegister<SRegister>() 5682 : destination.AsFpuRegister<SRegister>(); 5683 int mem = source.IsFpuRegister() 5684 ? destination.GetStackIndex() 5685 : source.GetStackIndex(); 5686 5687 __ vmovrs(IP, reg); 5688 __ LoadSFromOffset(reg, SP, mem); 5689 __ StoreToOffset(kStoreWord, IP, SP, mem); 5690 } else if (source.IsDoubleStackSlot() && destination.IsDoubleStackSlot()) { 5691 Exchange(source.GetStackIndex(), destination.GetStackIndex()); 5692 Exchange(source.GetHighStackIndex(kArmWordSize), destination.GetHighStackIndex(kArmWordSize)); 5693 } else { 5694 LOG(FATAL) << "Unimplemented" << source << " <-> " << destination; 5695 } 5696} 5697 5698void ParallelMoveResolverARM::SpillScratch(int reg) { 5699 __ Push(static_cast<Register>(reg)); 5700} 5701 5702void ParallelMoveResolverARM::RestoreScratch(int reg) { 5703 __ Pop(static_cast<Register>(reg)); 5704} 5705 5706HLoadClass::LoadKind CodeGeneratorARM::GetSupportedLoadClassKind( 5707 HLoadClass::LoadKind desired_class_load_kind) { 5708 switch (desired_class_load_kind) { 5709 case HLoadClass::LoadKind::kReferrersClass: 5710 break; 5711 case HLoadClass::LoadKind::kBootImageLinkTimeAddress: 5712 DCHECK(!GetCompilerOptions().GetCompilePic()); 5713 break; 5714 case HLoadClass::LoadKind::kBootImageLinkTimePcRelative: 5715 DCHECK(GetCompilerOptions().GetCompilePic()); 5716 break; 5717 case HLoadClass::LoadKind::kBootImageAddress: 5718 break; 5719 case HLoadClass::LoadKind::kJitTableAddress: 5720 break; 5721 case HLoadClass::LoadKind::kDexCachePcRelative: 5722 DCHECK(!Runtime::Current()->UseJitCompilation()); 5723 // We disable pc-relative load when there is an irreducible loop, as the optimization 5724 // is incompatible with it. 5725 // TODO: Create as many ArmDexCacheArraysBase instructions as needed for methods 5726 // with irreducible loops. 5727 if (GetGraph()->HasIrreducibleLoops()) { 5728 return HLoadClass::LoadKind::kDexCacheViaMethod; 5729 } 5730 break; 5731 case HLoadClass::LoadKind::kDexCacheViaMethod: 5732 break; 5733 } 5734 return desired_class_load_kind; 5735} 5736 5737void LocationsBuilderARM::VisitLoadClass(HLoadClass* cls) { 5738 if (cls->NeedsAccessCheck()) { 5739 InvokeRuntimeCallingConvention calling_convention; 5740 CodeGenerator::CreateLoadClassLocationSummary( 5741 cls, 5742 Location::RegisterLocation(calling_convention.GetRegisterAt(0)), 5743 Location::RegisterLocation(R0), 5744 /* code_generator_supports_read_barrier */ true); 5745 return; 5746 } 5747 5748 const bool requires_read_barrier = kEmitCompilerReadBarrier && !cls->IsInBootImage(); 5749 LocationSummary::CallKind call_kind = (cls->NeedsEnvironment() || requires_read_barrier) 5750 ? LocationSummary::kCallOnSlowPath 5751 : LocationSummary::kNoCall; 5752 LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(cls, call_kind); 5753 if (kUseBakerReadBarrier && requires_read_barrier && !cls->NeedsEnvironment()) { 5754 locations->SetCustomSlowPathCallerSaves(RegisterSet::Empty()); // No caller-save registers. 5755 } 5756 5757 HLoadClass::LoadKind load_kind = cls->GetLoadKind(); 5758 if (load_kind == HLoadClass::LoadKind::kReferrersClass || 5759 load_kind == HLoadClass::LoadKind::kDexCacheViaMethod || 5760 load_kind == HLoadClass::LoadKind::kDexCachePcRelative) { 5761 locations->SetInAt(0, Location::RequiresRegister()); 5762 } 5763 locations->SetOut(Location::RequiresRegister()); 5764} 5765 5766void InstructionCodeGeneratorARM::VisitLoadClass(HLoadClass* cls) { 5767 LocationSummary* locations = cls->GetLocations(); 5768 if (cls->NeedsAccessCheck()) { 5769 codegen_->MoveConstant(locations->GetTemp(0), cls->GetTypeIndex().index_); 5770 codegen_->InvokeRuntime(kQuickInitializeTypeAndVerifyAccess, cls, cls->GetDexPc()); 5771 CheckEntrypointTypes<kQuickInitializeTypeAndVerifyAccess, void*, uint32_t>(); 5772 return; 5773 } 5774 5775 Location out_loc = locations->Out(); 5776 Register out = out_loc.AsRegister<Register>(); 5777 5778 const ReadBarrierOption read_barrier_option = cls->IsInBootImage() 5779 ? kWithoutReadBarrier 5780 : kCompilerReadBarrierOption; 5781 bool generate_null_check = false; 5782 switch (cls->GetLoadKind()) { 5783 case HLoadClass::LoadKind::kReferrersClass: { 5784 DCHECK(!cls->CanCallRuntime()); 5785 DCHECK(!cls->MustGenerateClinitCheck()); 5786 // /* GcRoot<mirror::Class> */ out = current_method->declaring_class_ 5787 Register current_method = locations->InAt(0).AsRegister<Register>(); 5788 GenerateGcRootFieldLoad(cls, 5789 out_loc, 5790 current_method, 5791 ArtMethod::DeclaringClassOffset().Int32Value(), 5792 read_barrier_option); 5793 break; 5794 } 5795 case HLoadClass::LoadKind::kBootImageLinkTimeAddress: { 5796 DCHECK_EQ(read_barrier_option, kWithoutReadBarrier); 5797 __ LoadLiteral(out, codegen_->DeduplicateBootImageTypeLiteral(cls->GetDexFile(), 5798 cls->GetTypeIndex())); 5799 break; 5800 } 5801 case HLoadClass::LoadKind::kBootImageLinkTimePcRelative: { 5802 DCHECK_EQ(read_barrier_option, kWithoutReadBarrier); 5803 CodeGeneratorARM::PcRelativePatchInfo* labels = 5804 codegen_->NewPcRelativeTypePatch(cls->GetDexFile(), cls->GetTypeIndex()); 5805 __ BindTrackedLabel(&labels->movw_label); 5806 __ movw(out, /* placeholder */ 0u); 5807 __ BindTrackedLabel(&labels->movt_label); 5808 __ movt(out, /* placeholder */ 0u); 5809 __ BindTrackedLabel(&labels->add_pc_label); 5810 __ add(out, out, ShifterOperand(PC)); 5811 break; 5812 } 5813 case HLoadClass::LoadKind::kBootImageAddress: { 5814 DCHECK_EQ(read_barrier_option, kWithoutReadBarrier); 5815 DCHECK_NE(cls->GetAddress(), 0u); 5816 uint32_t address = dchecked_integral_cast<uint32_t>(cls->GetAddress()); 5817 __ LoadLiteral(out, codegen_->DeduplicateBootImageAddressLiteral(address)); 5818 break; 5819 } 5820 case HLoadClass::LoadKind::kJitTableAddress: { 5821 __ LoadLiteral(out, codegen_->DeduplicateJitClassLiteral(cls->GetDexFile(), 5822 cls->GetTypeIndex(), 5823 cls->GetAddress())); 5824 // /* GcRoot<mirror::Class> */ out = *out 5825 GenerateGcRootFieldLoad(cls, out_loc, out, /* offset */ 0, kCompilerReadBarrierOption); 5826 break; 5827 } 5828 case HLoadClass::LoadKind::kDexCachePcRelative: { 5829 Register base_reg = locations->InAt(0).AsRegister<Register>(); 5830 HArmDexCacheArraysBase* base = cls->InputAt(0)->AsArmDexCacheArraysBase(); 5831 int32_t offset = cls->GetDexCacheElementOffset() - base->GetElementOffset(); 5832 // /* GcRoot<mirror::Class> */ out = *(dex_cache_arrays_base + offset) 5833 GenerateGcRootFieldLoad(cls, out_loc, base_reg, offset, read_barrier_option); 5834 generate_null_check = !cls->IsInDexCache(); 5835 break; 5836 } 5837 case HLoadClass::LoadKind::kDexCacheViaMethod: { 5838 // /* GcRoot<mirror::Class>[] */ out = 5839 // current_method.ptr_sized_fields_->dex_cache_resolved_types_ 5840 Register current_method = locations->InAt(0).AsRegister<Register>(); 5841 __ LoadFromOffset(kLoadWord, 5842 out, 5843 current_method, 5844 ArtMethod::DexCacheResolvedTypesOffset(kArmPointerSize).Int32Value()); 5845 // /* GcRoot<mirror::Class> */ out = out[type_index] 5846 size_t offset = CodeGenerator::GetCacheOffset(cls->GetTypeIndex().index_); 5847 GenerateGcRootFieldLoad(cls, out_loc, out, offset, read_barrier_option); 5848 generate_null_check = !cls->IsInDexCache(); 5849 } 5850 } 5851 5852 if (generate_null_check || cls->MustGenerateClinitCheck()) { 5853 DCHECK(cls->CanCallRuntime()); 5854 SlowPathCodeARM* slow_path = new (GetGraph()->GetArena()) LoadClassSlowPathARM( 5855 cls, cls, cls->GetDexPc(), cls->MustGenerateClinitCheck()); 5856 codegen_->AddSlowPath(slow_path); 5857 if (generate_null_check) { 5858 __ CompareAndBranchIfZero(out, slow_path->GetEntryLabel()); 5859 } 5860 if (cls->MustGenerateClinitCheck()) { 5861 GenerateClassInitializationCheck(slow_path, out); 5862 } else { 5863 __ Bind(slow_path->GetExitLabel()); 5864 } 5865 } 5866} 5867 5868void LocationsBuilderARM::VisitClinitCheck(HClinitCheck* check) { 5869 LocationSummary* locations = 5870 new (GetGraph()->GetArena()) LocationSummary(check, LocationSummary::kCallOnSlowPath); 5871 locations->SetInAt(0, Location::RequiresRegister()); 5872 if (check->HasUses()) { 5873 locations->SetOut(Location::SameAsFirstInput()); 5874 } 5875} 5876 5877void InstructionCodeGeneratorARM::VisitClinitCheck(HClinitCheck* check) { 5878 // We assume the class is not null. 5879 SlowPathCodeARM* slow_path = new (GetGraph()->GetArena()) LoadClassSlowPathARM( 5880 check->GetLoadClass(), check, check->GetDexPc(), true); 5881 codegen_->AddSlowPath(slow_path); 5882 GenerateClassInitializationCheck(slow_path, 5883 check->GetLocations()->InAt(0).AsRegister<Register>()); 5884} 5885 5886void InstructionCodeGeneratorARM::GenerateClassInitializationCheck( 5887 SlowPathCodeARM* slow_path, Register class_reg) { 5888 __ LoadFromOffset(kLoadWord, IP, class_reg, mirror::Class::StatusOffset().Int32Value()); 5889 __ cmp(IP, ShifterOperand(mirror::Class::kStatusInitialized)); 5890 __ b(slow_path->GetEntryLabel(), LT); 5891 // Even if the initialized flag is set, we may be in a situation where caches are not synced 5892 // properly. Therefore, we do a memory fence. 5893 __ dmb(ISH); 5894 __ Bind(slow_path->GetExitLabel()); 5895} 5896 5897HLoadString::LoadKind CodeGeneratorARM::GetSupportedLoadStringKind( 5898 HLoadString::LoadKind desired_string_load_kind) { 5899 switch (desired_string_load_kind) { 5900 case HLoadString::LoadKind::kBootImageLinkTimeAddress: 5901 DCHECK(!GetCompilerOptions().GetCompilePic()); 5902 break; 5903 case HLoadString::LoadKind::kBootImageLinkTimePcRelative: 5904 DCHECK(GetCompilerOptions().GetCompilePic()); 5905 break; 5906 case HLoadString::LoadKind::kBootImageAddress: 5907 break; 5908 case HLoadString::LoadKind::kBssEntry: 5909 DCHECK(!Runtime::Current()->UseJitCompilation()); 5910 break; 5911 case HLoadString::LoadKind::kJitTableAddress: 5912 DCHECK(Runtime::Current()->UseJitCompilation()); 5913 break; 5914 case HLoadString::LoadKind::kDexCacheViaMethod: 5915 break; 5916 } 5917 return desired_string_load_kind; 5918} 5919 5920void LocationsBuilderARM::VisitLoadString(HLoadString* load) { 5921 LocationSummary::CallKind call_kind = CodeGenerator::GetLoadStringCallKind(load); 5922 LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(load, call_kind); 5923 HLoadString::LoadKind load_kind = load->GetLoadKind(); 5924 if (load_kind == HLoadString::LoadKind::kDexCacheViaMethod) { 5925 locations->SetOut(Location::RegisterLocation(R0)); 5926 } else { 5927 locations->SetOut(Location::RequiresRegister()); 5928 if (load_kind == HLoadString::LoadKind::kBssEntry) { 5929 if (!kUseReadBarrier || kUseBakerReadBarrier) { 5930 // Rely on the pResolveString and/or marking to save everything, including temps. 5931 // Note that IP may theoretically be clobbered by saving/restoring the live register 5932 // (only one thanks to the custom calling convention), so we request a different temp. 5933 locations->AddTemp(Location::RequiresRegister()); 5934 RegisterSet caller_saves = RegisterSet::Empty(); 5935 InvokeRuntimeCallingConvention calling_convention; 5936 caller_saves.Add(Location::RegisterLocation(calling_convention.GetRegisterAt(0))); 5937 // TODO: Add GetReturnLocation() to the calling convention so that we can DCHECK() 5938 // that the the kPrimNot result register is the same as the first argument register. 5939 locations->SetCustomSlowPathCallerSaves(caller_saves); 5940 } else { 5941 // For non-Baker read barrier we have a temp-clobbering call. 5942 } 5943 } 5944 } 5945} 5946 5947void InstructionCodeGeneratorARM::VisitLoadString(HLoadString* load) { 5948 LocationSummary* locations = load->GetLocations(); 5949 Location out_loc = locations->Out(); 5950 Register out = out_loc.AsRegister<Register>(); 5951 HLoadString::LoadKind load_kind = load->GetLoadKind(); 5952 5953 switch (load_kind) { 5954 case HLoadString::LoadKind::kBootImageLinkTimeAddress: { 5955 __ LoadLiteral(out, codegen_->DeduplicateBootImageStringLiteral(load->GetDexFile(), 5956 load->GetStringIndex())); 5957 return; // No dex cache slow path. 5958 } 5959 case HLoadString::LoadKind::kBootImageLinkTimePcRelative: { 5960 DCHECK(codegen_->GetCompilerOptions().IsBootImage()); 5961 CodeGeneratorARM::PcRelativePatchInfo* labels = 5962 codegen_->NewPcRelativeStringPatch(load->GetDexFile(), load->GetStringIndex().index_); 5963 __ BindTrackedLabel(&labels->movw_label); 5964 __ movw(out, /* placeholder */ 0u); 5965 __ BindTrackedLabel(&labels->movt_label); 5966 __ movt(out, /* placeholder */ 0u); 5967 __ BindTrackedLabel(&labels->add_pc_label); 5968 __ add(out, out, ShifterOperand(PC)); 5969 return; // No dex cache slow path. 5970 } 5971 case HLoadString::LoadKind::kBootImageAddress: { 5972 DCHECK_NE(load->GetAddress(), 0u); 5973 uint32_t address = dchecked_integral_cast<uint32_t>(load->GetAddress()); 5974 __ LoadLiteral(out, codegen_->DeduplicateBootImageAddressLiteral(address)); 5975 return; // No dex cache slow path. 5976 } 5977 case HLoadString::LoadKind::kBssEntry: { 5978 DCHECK(!codegen_->GetCompilerOptions().IsBootImage()); 5979 Register temp = locations->GetTemp(0).AsRegister<Register>(); 5980 CodeGeneratorARM::PcRelativePatchInfo* labels = 5981 codegen_->NewPcRelativeStringPatch(load->GetDexFile(), load->GetStringIndex().index_); 5982 __ BindTrackedLabel(&labels->movw_label); 5983 __ movw(temp, /* placeholder */ 0u); 5984 __ BindTrackedLabel(&labels->movt_label); 5985 __ movt(temp, /* placeholder */ 0u); 5986 __ BindTrackedLabel(&labels->add_pc_label); 5987 __ add(temp, temp, ShifterOperand(PC)); 5988 GenerateGcRootFieldLoad(load, out_loc, temp, /* offset */ 0, kCompilerReadBarrierOption); 5989 SlowPathCode* slow_path = new (GetGraph()->GetArena()) LoadStringSlowPathARM(load); 5990 codegen_->AddSlowPath(slow_path); 5991 __ CompareAndBranchIfZero(out, slow_path->GetEntryLabel()); 5992 __ Bind(slow_path->GetExitLabel()); 5993 return; 5994 } 5995 case HLoadString::LoadKind::kJitTableAddress: { 5996 __ LoadLiteral(out, codegen_->DeduplicateJitStringLiteral(load->GetDexFile(), 5997 load->GetStringIndex())); 5998 // /* GcRoot<mirror::String> */ out = *out 5999 GenerateGcRootFieldLoad(load, out_loc, out, /* offset */ 0, kCompilerReadBarrierOption); 6000 return; 6001 } 6002 default: 6003 break; 6004 } 6005 6006 // TODO: Consider re-adding the compiler code to do string dex cache lookup again. 6007 DCHECK(load_kind == HLoadString::LoadKind::kDexCacheViaMethod); 6008 InvokeRuntimeCallingConvention calling_convention; 6009 DCHECK_EQ(calling_convention.GetRegisterAt(0), out); 6010 __ LoadImmediate(calling_convention.GetRegisterAt(0), load->GetStringIndex().index_); 6011 codegen_->InvokeRuntime(kQuickResolveString, load, load->GetDexPc()); 6012 CheckEntrypointTypes<kQuickResolveString, void*, uint32_t>(); 6013} 6014 6015static int32_t GetExceptionTlsOffset() { 6016 return Thread::ExceptionOffset<kArmPointerSize>().Int32Value(); 6017} 6018 6019void LocationsBuilderARM::VisitLoadException(HLoadException* load) { 6020 LocationSummary* locations = 6021 new (GetGraph()->GetArena()) LocationSummary(load, LocationSummary::kNoCall); 6022 locations->SetOut(Location::RequiresRegister()); 6023} 6024 6025void InstructionCodeGeneratorARM::VisitLoadException(HLoadException* load) { 6026 Register out = load->GetLocations()->Out().AsRegister<Register>(); 6027 __ LoadFromOffset(kLoadWord, out, TR, GetExceptionTlsOffset()); 6028} 6029 6030void LocationsBuilderARM::VisitClearException(HClearException* clear) { 6031 new (GetGraph()->GetArena()) LocationSummary(clear, LocationSummary::kNoCall); 6032} 6033 6034void InstructionCodeGeneratorARM::VisitClearException(HClearException* clear ATTRIBUTE_UNUSED) { 6035 __ LoadImmediate(IP, 0); 6036 __ StoreToOffset(kStoreWord, IP, TR, GetExceptionTlsOffset()); 6037} 6038 6039void LocationsBuilderARM::VisitThrow(HThrow* instruction) { 6040 LocationSummary* locations = 6041 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCallOnMainOnly); 6042 InvokeRuntimeCallingConvention calling_convention; 6043 locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(0))); 6044} 6045 6046void InstructionCodeGeneratorARM::VisitThrow(HThrow* instruction) { 6047 codegen_->InvokeRuntime(kQuickDeliverException, instruction, instruction->GetDexPc()); 6048 CheckEntrypointTypes<kQuickDeliverException, void, mirror::Object*>(); 6049} 6050 6051// Temp is used for read barrier. 6052static size_t NumberOfInstanceOfTemps(TypeCheckKind type_check_kind) { 6053 if (kEmitCompilerReadBarrier && 6054 (kUseBakerReadBarrier || 6055 type_check_kind == TypeCheckKind::kAbstractClassCheck || 6056 type_check_kind == TypeCheckKind::kClassHierarchyCheck || 6057 type_check_kind == TypeCheckKind::kArrayObjectCheck)) { 6058 return 1; 6059 } 6060 return 0; 6061} 6062 6063// Interface case has 3 temps, one for holding the number of interfaces, one for the current 6064// interface pointer, one for loading the current interface. 6065// The other checks have one temp for loading the object's class. 6066static size_t NumberOfCheckCastTemps(TypeCheckKind type_check_kind) { 6067 if (type_check_kind == TypeCheckKind::kInterfaceCheck) { 6068 return 3; 6069 } 6070 return 1 + NumberOfInstanceOfTemps(type_check_kind); 6071} 6072 6073void LocationsBuilderARM::VisitInstanceOf(HInstanceOf* instruction) { 6074 LocationSummary::CallKind call_kind = LocationSummary::kNoCall; 6075 TypeCheckKind type_check_kind = instruction->GetTypeCheckKind(); 6076 bool baker_read_barrier_slow_path = false; 6077 switch (type_check_kind) { 6078 case TypeCheckKind::kExactCheck: 6079 case TypeCheckKind::kAbstractClassCheck: 6080 case TypeCheckKind::kClassHierarchyCheck: 6081 case TypeCheckKind::kArrayObjectCheck: 6082 call_kind = 6083 kEmitCompilerReadBarrier ? LocationSummary::kCallOnSlowPath : LocationSummary::kNoCall; 6084 baker_read_barrier_slow_path = kUseBakerReadBarrier; 6085 break; 6086 case TypeCheckKind::kArrayCheck: 6087 case TypeCheckKind::kUnresolvedCheck: 6088 case TypeCheckKind::kInterfaceCheck: 6089 call_kind = LocationSummary::kCallOnSlowPath; 6090 break; 6091 } 6092 6093 LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction, call_kind); 6094 if (baker_read_barrier_slow_path) { 6095 locations->SetCustomSlowPathCallerSaves(RegisterSet::Empty()); // No caller-save registers. 6096 } 6097 locations->SetInAt(0, Location::RequiresRegister()); 6098 locations->SetInAt(1, Location::RequiresRegister()); 6099 // The "out" register is used as a temporary, so it overlaps with the inputs. 6100 // Note that TypeCheckSlowPathARM uses this register too. 6101 locations->SetOut(Location::RequiresRegister(), Location::kOutputOverlap); 6102 locations->AddRegisterTemps(NumberOfInstanceOfTemps(type_check_kind)); 6103} 6104 6105void InstructionCodeGeneratorARM::VisitInstanceOf(HInstanceOf* instruction) { 6106 TypeCheckKind type_check_kind = instruction->GetTypeCheckKind(); 6107 LocationSummary* locations = instruction->GetLocations(); 6108 Location obj_loc = locations->InAt(0); 6109 Register obj = obj_loc.AsRegister<Register>(); 6110 Register cls = locations->InAt(1).AsRegister<Register>(); 6111 Location out_loc = locations->Out(); 6112 Register out = out_loc.AsRegister<Register>(); 6113 const size_t num_temps = NumberOfInstanceOfTemps(type_check_kind); 6114 DCHECK_LE(num_temps, 1u); 6115 Location maybe_temp_loc = (num_temps >= 1) ? locations->GetTemp(0) : Location::NoLocation(); 6116 uint32_t class_offset = mirror::Object::ClassOffset().Int32Value(); 6117 uint32_t super_offset = mirror::Class::SuperClassOffset().Int32Value(); 6118 uint32_t component_offset = mirror::Class::ComponentTypeOffset().Int32Value(); 6119 uint32_t primitive_offset = mirror::Class::PrimitiveTypeOffset().Int32Value(); 6120 Label done, zero; 6121 SlowPathCodeARM* slow_path = nullptr; 6122 6123 // Return 0 if `obj` is null. 6124 // avoid null check if we know obj is not null. 6125 if (instruction->MustDoNullCheck()) { 6126 __ CompareAndBranchIfZero(obj, &zero); 6127 } 6128 6129 switch (type_check_kind) { 6130 case TypeCheckKind::kExactCheck: { 6131 // /* HeapReference<Class> */ out = obj->klass_ 6132 GenerateReferenceLoadTwoRegisters(instruction, 6133 out_loc, 6134 obj_loc, 6135 class_offset, 6136 maybe_temp_loc, 6137 kCompilerReadBarrierOption); 6138 __ cmp(out, ShifterOperand(cls)); 6139 // Classes must be equal for the instanceof to succeed. 6140 __ b(&zero, NE); 6141 __ LoadImmediate(out, 1); 6142 __ b(&done); 6143 break; 6144 } 6145 6146 case TypeCheckKind::kAbstractClassCheck: { 6147 // /* HeapReference<Class> */ out = obj->klass_ 6148 GenerateReferenceLoadTwoRegisters(instruction, 6149 out_loc, 6150 obj_loc, 6151 class_offset, 6152 maybe_temp_loc, 6153 kCompilerReadBarrierOption); 6154 // If the class is abstract, we eagerly fetch the super class of the 6155 // object to avoid doing a comparison we know will fail. 6156 Label loop; 6157 __ Bind(&loop); 6158 // /* HeapReference<Class> */ out = out->super_class_ 6159 GenerateReferenceLoadOneRegister(instruction, 6160 out_loc, 6161 super_offset, 6162 maybe_temp_loc, 6163 kCompilerReadBarrierOption); 6164 // If `out` is null, we use it for the result, and jump to `done`. 6165 __ CompareAndBranchIfZero(out, &done); 6166 __ cmp(out, ShifterOperand(cls)); 6167 __ b(&loop, NE); 6168 __ LoadImmediate(out, 1); 6169 if (zero.IsLinked()) { 6170 __ b(&done); 6171 } 6172 break; 6173 } 6174 6175 case TypeCheckKind::kClassHierarchyCheck: { 6176 // /* HeapReference<Class> */ out = obj->klass_ 6177 GenerateReferenceLoadTwoRegisters(instruction, 6178 out_loc, 6179 obj_loc, 6180 class_offset, 6181 maybe_temp_loc, 6182 kCompilerReadBarrierOption); 6183 // Walk over the class hierarchy to find a match. 6184 Label loop, success; 6185 __ Bind(&loop); 6186 __ cmp(out, ShifterOperand(cls)); 6187 __ b(&success, EQ); 6188 // /* HeapReference<Class> */ out = out->super_class_ 6189 GenerateReferenceLoadOneRegister(instruction, 6190 out_loc, 6191 super_offset, 6192 maybe_temp_loc, 6193 kCompilerReadBarrierOption); 6194 __ CompareAndBranchIfNonZero(out, &loop); 6195 // If `out` is null, we use it for the result, and jump to `done`. 6196 __ b(&done); 6197 __ Bind(&success); 6198 __ LoadImmediate(out, 1); 6199 if (zero.IsLinked()) { 6200 __ b(&done); 6201 } 6202 break; 6203 } 6204 6205 case TypeCheckKind::kArrayObjectCheck: { 6206 // /* HeapReference<Class> */ out = obj->klass_ 6207 GenerateReferenceLoadTwoRegisters(instruction, 6208 out_loc, 6209 obj_loc, 6210 class_offset, 6211 maybe_temp_loc, 6212 kCompilerReadBarrierOption); 6213 // Do an exact check. 6214 Label exact_check; 6215 __ cmp(out, ShifterOperand(cls)); 6216 __ b(&exact_check, EQ); 6217 // Otherwise, we need to check that the object's class is a non-primitive array. 6218 // /* HeapReference<Class> */ out = out->component_type_ 6219 GenerateReferenceLoadOneRegister(instruction, 6220 out_loc, 6221 component_offset, 6222 maybe_temp_loc, 6223 kCompilerReadBarrierOption); 6224 // If `out` is null, we use it for the result, and jump to `done`. 6225 __ CompareAndBranchIfZero(out, &done); 6226 __ LoadFromOffset(kLoadUnsignedHalfword, out, out, primitive_offset); 6227 static_assert(Primitive::kPrimNot == 0, "Expected 0 for kPrimNot"); 6228 __ CompareAndBranchIfNonZero(out, &zero); 6229 __ Bind(&exact_check); 6230 __ LoadImmediate(out, 1); 6231 __ b(&done); 6232 break; 6233 } 6234 6235 case TypeCheckKind::kArrayCheck: { 6236 // No read barrier since the slow path will retry upon failure. 6237 // /* HeapReference<Class> */ out = obj->klass_ 6238 GenerateReferenceLoadTwoRegisters(instruction, 6239 out_loc, 6240 obj_loc, 6241 class_offset, 6242 maybe_temp_loc, 6243 kWithoutReadBarrier); 6244 __ cmp(out, ShifterOperand(cls)); 6245 DCHECK(locations->OnlyCallsOnSlowPath()); 6246 slow_path = new (GetGraph()->GetArena()) TypeCheckSlowPathARM(instruction, 6247 /* is_fatal */ false); 6248 codegen_->AddSlowPath(slow_path); 6249 __ b(slow_path->GetEntryLabel(), NE); 6250 __ LoadImmediate(out, 1); 6251 if (zero.IsLinked()) { 6252 __ b(&done); 6253 } 6254 break; 6255 } 6256 6257 case TypeCheckKind::kUnresolvedCheck: 6258 case TypeCheckKind::kInterfaceCheck: { 6259 // Note that we indeed only call on slow path, but we always go 6260 // into the slow path for the unresolved and interface check 6261 // cases. 6262 // 6263 // We cannot directly call the InstanceofNonTrivial runtime 6264 // entry point without resorting to a type checking slow path 6265 // here (i.e. by calling InvokeRuntime directly), as it would 6266 // require to assign fixed registers for the inputs of this 6267 // HInstanceOf instruction (following the runtime calling 6268 // convention), which might be cluttered by the potential first 6269 // read barrier emission at the beginning of this method. 6270 // 6271 // TODO: Introduce a new runtime entry point taking the object 6272 // to test (instead of its class) as argument, and let it deal 6273 // with the read barrier issues. This will let us refactor this 6274 // case of the `switch` code as it was previously (with a direct 6275 // call to the runtime not using a type checking slow path). 6276 // This should also be beneficial for the other cases above. 6277 DCHECK(locations->OnlyCallsOnSlowPath()); 6278 slow_path = new (GetGraph()->GetArena()) TypeCheckSlowPathARM(instruction, 6279 /* is_fatal */ false); 6280 codegen_->AddSlowPath(slow_path); 6281 __ b(slow_path->GetEntryLabel()); 6282 if (zero.IsLinked()) { 6283 __ b(&done); 6284 } 6285 break; 6286 } 6287 } 6288 6289 if (zero.IsLinked()) { 6290 __ Bind(&zero); 6291 __ LoadImmediate(out, 0); 6292 } 6293 6294 if (done.IsLinked()) { 6295 __ Bind(&done); 6296 } 6297 6298 if (slow_path != nullptr) { 6299 __ Bind(slow_path->GetExitLabel()); 6300 } 6301} 6302 6303void LocationsBuilderARM::VisitCheckCast(HCheckCast* instruction) { 6304 LocationSummary::CallKind call_kind = LocationSummary::kNoCall; 6305 bool throws_into_catch = instruction->CanThrowIntoCatchBlock(); 6306 6307 TypeCheckKind type_check_kind = instruction->GetTypeCheckKind(); 6308 switch (type_check_kind) { 6309 case TypeCheckKind::kExactCheck: 6310 case TypeCheckKind::kAbstractClassCheck: 6311 case TypeCheckKind::kClassHierarchyCheck: 6312 case TypeCheckKind::kArrayObjectCheck: 6313 call_kind = (throws_into_catch || kEmitCompilerReadBarrier) ? 6314 LocationSummary::kCallOnSlowPath : 6315 LocationSummary::kNoCall; // In fact, call on a fatal (non-returning) slow path. 6316 break; 6317 case TypeCheckKind::kArrayCheck: 6318 case TypeCheckKind::kUnresolvedCheck: 6319 case TypeCheckKind::kInterfaceCheck: 6320 call_kind = LocationSummary::kCallOnSlowPath; 6321 break; 6322 } 6323 6324 LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction, call_kind); 6325 locations->SetInAt(0, Location::RequiresRegister()); 6326 locations->SetInAt(1, Location::RequiresRegister()); 6327 locations->AddRegisterTemps(NumberOfCheckCastTemps(type_check_kind)); 6328} 6329 6330void InstructionCodeGeneratorARM::VisitCheckCast(HCheckCast* instruction) { 6331 TypeCheckKind type_check_kind = instruction->GetTypeCheckKind(); 6332 LocationSummary* locations = instruction->GetLocations(); 6333 Location obj_loc = locations->InAt(0); 6334 Register obj = obj_loc.AsRegister<Register>(); 6335 Register cls = locations->InAt(1).AsRegister<Register>(); 6336 Location temp_loc = locations->GetTemp(0); 6337 Register temp = temp_loc.AsRegister<Register>(); 6338 const size_t num_temps = NumberOfCheckCastTemps(type_check_kind); 6339 DCHECK_LE(num_temps, 3u); 6340 Location maybe_temp2_loc = (num_temps >= 2) ? locations->GetTemp(1) : Location::NoLocation(); 6341 Location maybe_temp3_loc = (num_temps >= 3) ? locations->GetTemp(2) : Location::NoLocation(); 6342 const uint32_t class_offset = mirror::Object::ClassOffset().Int32Value(); 6343 const uint32_t super_offset = mirror::Class::SuperClassOffset().Int32Value(); 6344 const uint32_t component_offset = mirror::Class::ComponentTypeOffset().Int32Value(); 6345 const uint32_t primitive_offset = mirror::Class::PrimitiveTypeOffset().Int32Value(); 6346 const uint32_t iftable_offset = mirror::Class::IfTableOffset().Uint32Value(); 6347 const uint32_t array_length_offset = mirror::Array::LengthOffset().Uint32Value(); 6348 const uint32_t object_array_data_offset = 6349 mirror::Array::DataOffset(kHeapReferenceSize).Uint32Value(); 6350 6351 // Always false for read barriers since we may need to go to the entrypoint for non-fatal cases 6352 // from false negatives. The false negatives may come from avoiding read barriers below. Avoiding 6353 // read barriers is done for performance and code size reasons. 6354 bool is_type_check_slow_path_fatal = false; 6355 if (!kEmitCompilerReadBarrier) { 6356 is_type_check_slow_path_fatal = 6357 (type_check_kind == TypeCheckKind::kExactCheck || 6358 type_check_kind == TypeCheckKind::kAbstractClassCheck || 6359 type_check_kind == TypeCheckKind::kClassHierarchyCheck || 6360 type_check_kind == TypeCheckKind::kArrayObjectCheck) && 6361 !instruction->CanThrowIntoCatchBlock(); 6362 } 6363 SlowPathCodeARM* type_check_slow_path = 6364 new (GetGraph()->GetArena()) TypeCheckSlowPathARM(instruction, 6365 is_type_check_slow_path_fatal); 6366 codegen_->AddSlowPath(type_check_slow_path); 6367 6368 Label done; 6369 // Avoid null check if we know obj is not null. 6370 if (instruction->MustDoNullCheck()) { 6371 __ CompareAndBranchIfZero(obj, &done); 6372 } 6373 6374 switch (type_check_kind) { 6375 case TypeCheckKind::kExactCheck: 6376 case TypeCheckKind::kArrayCheck: { 6377 // /* HeapReference<Class> */ temp = obj->klass_ 6378 GenerateReferenceLoadTwoRegisters(instruction, 6379 temp_loc, 6380 obj_loc, 6381 class_offset, 6382 maybe_temp2_loc, 6383 kWithoutReadBarrier); 6384 6385 __ cmp(temp, ShifterOperand(cls)); 6386 // Jump to slow path for throwing the exception or doing a 6387 // more involved array check. 6388 __ b(type_check_slow_path->GetEntryLabel(), NE); 6389 break; 6390 } 6391 6392 case TypeCheckKind::kAbstractClassCheck: { 6393 // /* HeapReference<Class> */ temp = obj->klass_ 6394 GenerateReferenceLoadTwoRegisters(instruction, 6395 temp_loc, 6396 obj_loc, 6397 class_offset, 6398 maybe_temp2_loc, 6399 kWithoutReadBarrier); 6400 6401 // If the class is abstract, we eagerly fetch the super class of the 6402 // object to avoid doing a comparison we know will fail. 6403 Label loop; 6404 __ Bind(&loop); 6405 // /* HeapReference<Class> */ temp = temp->super_class_ 6406 GenerateReferenceLoadOneRegister(instruction, 6407 temp_loc, 6408 super_offset, 6409 maybe_temp2_loc, 6410 kWithoutReadBarrier); 6411 6412 // If the class reference currently in `temp` is null, jump to the slow path to throw the 6413 // exception. 6414 __ CompareAndBranchIfZero(temp, type_check_slow_path->GetEntryLabel()); 6415 6416 // Otherwise, compare the classes. 6417 __ cmp(temp, ShifterOperand(cls)); 6418 __ b(&loop, NE); 6419 break; 6420 } 6421 6422 case TypeCheckKind::kClassHierarchyCheck: { 6423 // /* HeapReference<Class> */ temp = obj->klass_ 6424 GenerateReferenceLoadTwoRegisters(instruction, 6425 temp_loc, 6426 obj_loc, 6427 class_offset, 6428 maybe_temp2_loc, 6429 kWithoutReadBarrier); 6430 6431 // Walk over the class hierarchy to find a match. 6432 Label loop; 6433 __ Bind(&loop); 6434 __ cmp(temp, ShifterOperand(cls)); 6435 __ b(&done, EQ); 6436 6437 // /* HeapReference<Class> */ temp = temp->super_class_ 6438 GenerateReferenceLoadOneRegister(instruction, 6439 temp_loc, 6440 super_offset, 6441 maybe_temp2_loc, 6442 kWithoutReadBarrier); 6443 6444 // If the class reference currently in `temp` is null, jump to the slow path to throw the 6445 // exception. 6446 __ CompareAndBranchIfZero(temp, type_check_slow_path->GetEntryLabel()); 6447 // Otherwise, jump to the beginning of the loop. 6448 __ b(&loop); 6449 break; 6450 } 6451 6452 case TypeCheckKind::kArrayObjectCheck: { 6453 // /* HeapReference<Class> */ temp = obj->klass_ 6454 GenerateReferenceLoadTwoRegisters(instruction, 6455 temp_loc, 6456 obj_loc, 6457 class_offset, 6458 maybe_temp2_loc, 6459 kWithoutReadBarrier); 6460 6461 // Do an exact check. 6462 __ cmp(temp, ShifterOperand(cls)); 6463 __ b(&done, EQ); 6464 6465 // Otherwise, we need to check that the object's class is a non-primitive array. 6466 // /* HeapReference<Class> */ temp = temp->component_type_ 6467 GenerateReferenceLoadOneRegister(instruction, 6468 temp_loc, 6469 component_offset, 6470 maybe_temp2_loc, 6471 kWithoutReadBarrier); 6472 // If the component type is null, jump to the slow path to throw the exception. 6473 __ CompareAndBranchIfZero(temp, type_check_slow_path->GetEntryLabel()); 6474 // Otherwise,the object is indeed an array, jump to label `check_non_primitive_component_type` 6475 // to further check that this component type is not a primitive type. 6476 __ LoadFromOffset(kLoadUnsignedHalfword, temp, temp, primitive_offset); 6477 static_assert(Primitive::kPrimNot == 0, "Expected 0 for art::Primitive::kPrimNot"); 6478 __ CompareAndBranchIfNonZero(temp, type_check_slow_path->GetEntryLabel()); 6479 break; 6480 } 6481 6482 case TypeCheckKind::kUnresolvedCheck: 6483 // We always go into the type check slow path for the unresolved check case. 6484 // We cannot directly call the CheckCast runtime entry point 6485 // without resorting to a type checking slow path here (i.e. by 6486 // calling InvokeRuntime directly), as it would require to 6487 // assign fixed registers for the inputs of this HInstanceOf 6488 // instruction (following the runtime calling convention), which 6489 // might be cluttered by the potential first read barrier 6490 // emission at the beginning of this method. 6491 6492 __ b(type_check_slow_path->GetEntryLabel()); 6493 break; 6494 6495 case TypeCheckKind::kInterfaceCheck: { 6496 // Avoid read barriers to improve performance of the fast path. We can not get false 6497 // positives by doing this. 6498 // /* HeapReference<Class> */ temp = obj->klass_ 6499 GenerateReferenceLoadTwoRegisters(instruction, 6500 temp_loc, 6501 obj_loc, 6502 class_offset, 6503 maybe_temp2_loc, 6504 kWithoutReadBarrier); 6505 6506 // /* HeapReference<Class> */ temp = temp->iftable_ 6507 GenerateReferenceLoadTwoRegisters(instruction, 6508 temp_loc, 6509 temp_loc, 6510 iftable_offset, 6511 maybe_temp2_loc, 6512 kWithoutReadBarrier); 6513 // Iftable is never null. 6514 __ ldr(maybe_temp2_loc.AsRegister<Register>(), Address(temp, array_length_offset)); 6515 // Loop through the iftable and check if any class matches. 6516 Label start_loop; 6517 __ Bind(&start_loop); 6518 __ CompareAndBranchIfZero(maybe_temp2_loc.AsRegister<Register>(), 6519 type_check_slow_path->GetEntryLabel()); 6520 __ ldr(maybe_temp3_loc.AsRegister<Register>(), Address(temp, object_array_data_offset)); 6521 __ MaybeUnpoisonHeapReference(maybe_temp3_loc.AsRegister<Register>()); 6522 // Go to next interface. 6523 __ add(temp, temp, ShifterOperand(2 * kHeapReferenceSize)); 6524 __ sub(maybe_temp2_loc.AsRegister<Register>(), 6525 maybe_temp2_loc.AsRegister<Register>(), 6526 ShifterOperand(2)); 6527 // Compare the classes and continue the loop if they do not match. 6528 __ cmp(cls, ShifterOperand(maybe_temp3_loc.AsRegister<Register>())); 6529 __ b(&start_loop, NE); 6530 break; 6531 } 6532 } 6533 __ Bind(&done); 6534 6535 __ Bind(type_check_slow_path->GetExitLabel()); 6536} 6537 6538void LocationsBuilderARM::VisitMonitorOperation(HMonitorOperation* instruction) { 6539 LocationSummary* locations = 6540 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCallOnMainOnly); 6541 InvokeRuntimeCallingConvention calling_convention; 6542 locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(0))); 6543} 6544 6545void InstructionCodeGeneratorARM::VisitMonitorOperation(HMonitorOperation* instruction) { 6546 codegen_->InvokeRuntime(instruction->IsEnter() ? kQuickLockObject : kQuickUnlockObject, 6547 instruction, 6548 instruction->GetDexPc()); 6549 if (instruction->IsEnter()) { 6550 CheckEntrypointTypes<kQuickLockObject, void, mirror::Object*>(); 6551 } else { 6552 CheckEntrypointTypes<kQuickUnlockObject, void, mirror::Object*>(); 6553 } 6554} 6555 6556void LocationsBuilderARM::VisitAnd(HAnd* instruction) { HandleBitwiseOperation(instruction, AND); } 6557void LocationsBuilderARM::VisitOr(HOr* instruction) { HandleBitwiseOperation(instruction, ORR); } 6558void LocationsBuilderARM::VisitXor(HXor* instruction) { HandleBitwiseOperation(instruction, EOR); } 6559 6560void LocationsBuilderARM::HandleBitwiseOperation(HBinaryOperation* instruction, Opcode opcode) { 6561 LocationSummary* locations = 6562 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall); 6563 DCHECK(instruction->GetResultType() == Primitive::kPrimInt 6564 || instruction->GetResultType() == Primitive::kPrimLong); 6565 // Note: GVN reorders commutative operations to have the constant on the right hand side. 6566 locations->SetInAt(0, Location::RequiresRegister()); 6567 locations->SetInAt(1, ArmEncodableConstantOrRegister(instruction->InputAt(1), opcode)); 6568 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 6569} 6570 6571void InstructionCodeGeneratorARM::VisitAnd(HAnd* instruction) { 6572 HandleBitwiseOperation(instruction); 6573} 6574 6575void InstructionCodeGeneratorARM::VisitOr(HOr* instruction) { 6576 HandleBitwiseOperation(instruction); 6577} 6578 6579void InstructionCodeGeneratorARM::VisitXor(HXor* instruction) { 6580 HandleBitwiseOperation(instruction); 6581} 6582 6583 6584void LocationsBuilderARM::VisitBitwiseNegatedRight(HBitwiseNegatedRight* instruction) { 6585 LocationSummary* locations = 6586 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall); 6587 DCHECK(instruction->GetResultType() == Primitive::kPrimInt 6588 || instruction->GetResultType() == Primitive::kPrimLong); 6589 6590 locations->SetInAt(0, Location::RequiresRegister()); 6591 locations->SetInAt(1, Location::RequiresRegister()); 6592 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 6593} 6594 6595void InstructionCodeGeneratorARM::VisitBitwiseNegatedRight(HBitwiseNegatedRight* instruction) { 6596 LocationSummary* locations = instruction->GetLocations(); 6597 Location first = locations->InAt(0); 6598 Location second = locations->InAt(1); 6599 Location out = locations->Out(); 6600 6601 if (instruction->GetResultType() == Primitive::kPrimInt) { 6602 Register first_reg = first.AsRegister<Register>(); 6603 ShifterOperand second_reg(second.AsRegister<Register>()); 6604 Register out_reg = out.AsRegister<Register>(); 6605 6606 switch (instruction->GetOpKind()) { 6607 case HInstruction::kAnd: 6608 __ bic(out_reg, first_reg, second_reg); 6609 break; 6610 case HInstruction::kOr: 6611 __ orn(out_reg, first_reg, second_reg); 6612 break; 6613 // There is no EON on arm. 6614 case HInstruction::kXor: 6615 default: 6616 LOG(FATAL) << "Unexpected instruction " << instruction->DebugName(); 6617 UNREACHABLE(); 6618 } 6619 return; 6620 6621 } else { 6622 DCHECK_EQ(instruction->GetResultType(), Primitive::kPrimLong); 6623 Register first_low = first.AsRegisterPairLow<Register>(); 6624 Register first_high = first.AsRegisterPairHigh<Register>(); 6625 ShifterOperand second_low(second.AsRegisterPairLow<Register>()); 6626 ShifterOperand second_high(second.AsRegisterPairHigh<Register>()); 6627 Register out_low = out.AsRegisterPairLow<Register>(); 6628 Register out_high = out.AsRegisterPairHigh<Register>(); 6629 6630 switch (instruction->GetOpKind()) { 6631 case HInstruction::kAnd: 6632 __ bic(out_low, first_low, second_low); 6633 __ bic(out_high, first_high, second_high); 6634 break; 6635 case HInstruction::kOr: 6636 __ orn(out_low, first_low, second_low); 6637 __ orn(out_high, first_high, second_high); 6638 break; 6639 // There is no EON on arm. 6640 case HInstruction::kXor: 6641 default: 6642 LOG(FATAL) << "Unexpected instruction " << instruction->DebugName(); 6643 UNREACHABLE(); 6644 } 6645 } 6646} 6647 6648void InstructionCodeGeneratorARM::GenerateAndConst(Register out, Register first, uint32_t value) { 6649 // Optimize special cases for individual halfs of `and-long` (`and` is simplified earlier). 6650 if (value == 0xffffffffu) { 6651 if (out != first) { 6652 __ mov(out, ShifterOperand(first)); 6653 } 6654 return; 6655 } 6656 if (value == 0u) { 6657 __ mov(out, ShifterOperand(0)); 6658 return; 6659 } 6660 ShifterOperand so; 6661 if (__ ShifterOperandCanHold(kNoRegister, kNoRegister, AND, value, &so)) { 6662 __ and_(out, first, so); 6663 } else { 6664 DCHECK(__ ShifterOperandCanHold(kNoRegister, kNoRegister, BIC, ~value, &so)); 6665 __ bic(out, first, ShifterOperand(~value)); 6666 } 6667} 6668 6669void InstructionCodeGeneratorARM::GenerateOrrConst(Register out, Register first, uint32_t value) { 6670 // Optimize special cases for individual halfs of `or-long` (`or` is simplified earlier). 6671 if (value == 0u) { 6672 if (out != first) { 6673 __ mov(out, ShifterOperand(first)); 6674 } 6675 return; 6676 } 6677 if (value == 0xffffffffu) { 6678 __ mvn(out, ShifterOperand(0)); 6679 return; 6680 } 6681 ShifterOperand so; 6682 if (__ ShifterOperandCanHold(kNoRegister, kNoRegister, ORR, value, &so)) { 6683 __ orr(out, first, so); 6684 } else { 6685 DCHECK(__ ShifterOperandCanHold(kNoRegister, kNoRegister, ORN, ~value, &so)); 6686 __ orn(out, first, ShifterOperand(~value)); 6687 } 6688} 6689 6690void InstructionCodeGeneratorARM::GenerateEorConst(Register out, Register first, uint32_t value) { 6691 // Optimize special case for individual halfs of `xor-long` (`xor` is simplified earlier). 6692 if (value == 0u) { 6693 if (out != first) { 6694 __ mov(out, ShifterOperand(first)); 6695 } 6696 return; 6697 } 6698 __ eor(out, first, ShifterOperand(value)); 6699} 6700 6701void InstructionCodeGeneratorARM::GenerateAddLongConst(Location out, 6702 Location first, 6703 uint64_t value) { 6704 Register out_low = out.AsRegisterPairLow<Register>(); 6705 Register out_high = out.AsRegisterPairHigh<Register>(); 6706 Register first_low = first.AsRegisterPairLow<Register>(); 6707 Register first_high = first.AsRegisterPairHigh<Register>(); 6708 uint32_t value_low = Low32Bits(value); 6709 uint32_t value_high = High32Bits(value); 6710 if (value_low == 0u) { 6711 if (out_low != first_low) { 6712 __ mov(out_low, ShifterOperand(first_low)); 6713 } 6714 __ AddConstant(out_high, first_high, value_high); 6715 return; 6716 } 6717 __ AddConstantSetFlags(out_low, first_low, value_low); 6718 ShifterOperand so; 6719 if (__ ShifterOperandCanHold(out_high, first_high, ADC, value_high, kCcDontCare, &so)) { 6720 __ adc(out_high, first_high, so); 6721 } else if (__ ShifterOperandCanHold(out_low, first_low, SBC, ~value_high, kCcDontCare, &so)) { 6722 __ sbc(out_high, first_high, so); 6723 } else { 6724 LOG(FATAL) << "Unexpected constant " << value_high; 6725 UNREACHABLE(); 6726 } 6727} 6728 6729void InstructionCodeGeneratorARM::HandleBitwiseOperation(HBinaryOperation* instruction) { 6730 LocationSummary* locations = instruction->GetLocations(); 6731 Location first = locations->InAt(0); 6732 Location second = locations->InAt(1); 6733 Location out = locations->Out(); 6734 6735 if (second.IsConstant()) { 6736 uint64_t value = static_cast<uint64_t>(Int64FromConstant(second.GetConstant())); 6737 uint32_t value_low = Low32Bits(value); 6738 if (instruction->GetResultType() == Primitive::kPrimInt) { 6739 Register first_reg = first.AsRegister<Register>(); 6740 Register out_reg = out.AsRegister<Register>(); 6741 if (instruction->IsAnd()) { 6742 GenerateAndConst(out_reg, first_reg, value_low); 6743 } else if (instruction->IsOr()) { 6744 GenerateOrrConst(out_reg, first_reg, value_low); 6745 } else { 6746 DCHECK(instruction->IsXor()); 6747 GenerateEorConst(out_reg, first_reg, value_low); 6748 } 6749 } else { 6750 DCHECK_EQ(instruction->GetResultType(), Primitive::kPrimLong); 6751 uint32_t value_high = High32Bits(value); 6752 Register first_low = first.AsRegisterPairLow<Register>(); 6753 Register first_high = first.AsRegisterPairHigh<Register>(); 6754 Register out_low = out.AsRegisterPairLow<Register>(); 6755 Register out_high = out.AsRegisterPairHigh<Register>(); 6756 if (instruction->IsAnd()) { 6757 GenerateAndConst(out_low, first_low, value_low); 6758 GenerateAndConst(out_high, first_high, value_high); 6759 } else if (instruction->IsOr()) { 6760 GenerateOrrConst(out_low, first_low, value_low); 6761 GenerateOrrConst(out_high, first_high, value_high); 6762 } else { 6763 DCHECK(instruction->IsXor()); 6764 GenerateEorConst(out_low, first_low, value_low); 6765 GenerateEorConst(out_high, first_high, value_high); 6766 } 6767 } 6768 return; 6769 } 6770 6771 if (instruction->GetResultType() == Primitive::kPrimInt) { 6772 Register first_reg = first.AsRegister<Register>(); 6773 ShifterOperand second_reg(second.AsRegister<Register>()); 6774 Register out_reg = out.AsRegister<Register>(); 6775 if (instruction->IsAnd()) { 6776 __ and_(out_reg, first_reg, second_reg); 6777 } else if (instruction->IsOr()) { 6778 __ orr(out_reg, first_reg, second_reg); 6779 } else { 6780 DCHECK(instruction->IsXor()); 6781 __ eor(out_reg, first_reg, second_reg); 6782 } 6783 } else { 6784 DCHECK_EQ(instruction->GetResultType(), Primitive::kPrimLong); 6785 Register first_low = first.AsRegisterPairLow<Register>(); 6786 Register first_high = first.AsRegisterPairHigh<Register>(); 6787 ShifterOperand second_low(second.AsRegisterPairLow<Register>()); 6788 ShifterOperand second_high(second.AsRegisterPairHigh<Register>()); 6789 Register out_low = out.AsRegisterPairLow<Register>(); 6790 Register out_high = out.AsRegisterPairHigh<Register>(); 6791 if (instruction->IsAnd()) { 6792 __ and_(out_low, first_low, second_low); 6793 __ and_(out_high, first_high, second_high); 6794 } else if (instruction->IsOr()) { 6795 __ orr(out_low, first_low, second_low); 6796 __ orr(out_high, first_high, second_high); 6797 } else { 6798 DCHECK(instruction->IsXor()); 6799 __ eor(out_low, first_low, second_low); 6800 __ eor(out_high, first_high, second_high); 6801 } 6802 } 6803} 6804 6805void InstructionCodeGeneratorARM::GenerateReferenceLoadOneRegister( 6806 HInstruction* instruction, 6807 Location out, 6808 uint32_t offset, 6809 Location maybe_temp, 6810 ReadBarrierOption read_barrier_option) { 6811 Register out_reg = out.AsRegister<Register>(); 6812 if (read_barrier_option == kWithReadBarrier) { 6813 CHECK(kEmitCompilerReadBarrier); 6814 DCHECK(maybe_temp.IsRegister()) << maybe_temp; 6815 if (kUseBakerReadBarrier) { 6816 // Load with fast path based Baker's read barrier. 6817 // /* HeapReference<Object> */ out = *(out + offset) 6818 codegen_->GenerateFieldLoadWithBakerReadBarrier( 6819 instruction, out, out_reg, offset, maybe_temp, /* needs_null_check */ false); 6820 } else { 6821 // Load with slow path based read barrier. 6822 // Save the value of `out` into `maybe_temp` before overwriting it 6823 // in the following move operation, as we will need it for the 6824 // read barrier below. 6825 __ Mov(maybe_temp.AsRegister<Register>(), out_reg); 6826 // /* HeapReference<Object> */ out = *(out + offset) 6827 __ LoadFromOffset(kLoadWord, out_reg, out_reg, offset); 6828 codegen_->GenerateReadBarrierSlow(instruction, out, out, maybe_temp, offset); 6829 } 6830 } else { 6831 // Plain load with no read barrier. 6832 // /* HeapReference<Object> */ out = *(out + offset) 6833 __ LoadFromOffset(kLoadWord, out_reg, out_reg, offset); 6834 __ MaybeUnpoisonHeapReference(out_reg); 6835 } 6836} 6837 6838void InstructionCodeGeneratorARM::GenerateReferenceLoadTwoRegisters( 6839 HInstruction* instruction, 6840 Location out, 6841 Location obj, 6842 uint32_t offset, 6843 Location maybe_temp, 6844 ReadBarrierOption read_barrier_option) { 6845 Register out_reg = out.AsRegister<Register>(); 6846 Register obj_reg = obj.AsRegister<Register>(); 6847 if (read_barrier_option == kWithReadBarrier) { 6848 CHECK(kEmitCompilerReadBarrier); 6849 if (kUseBakerReadBarrier) { 6850 DCHECK(maybe_temp.IsRegister()) << maybe_temp; 6851 // Load with fast path based Baker's read barrier. 6852 // /* HeapReference<Object> */ out = *(obj + offset) 6853 codegen_->GenerateFieldLoadWithBakerReadBarrier( 6854 instruction, out, obj_reg, offset, maybe_temp, /* needs_null_check */ false); 6855 } else { 6856 // Load with slow path based read barrier. 6857 // /* HeapReference<Object> */ out = *(obj + offset) 6858 __ LoadFromOffset(kLoadWord, out_reg, obj_reg, offset); 6859 codegen_->GenerateReadBarrierSlow(instruction, out, out, obj, offset); 6860 } 6861 } else { 6862 // Plain load with no read barrier. 6863 // /* HeapReference<Object> */ out = *(obj + offset) 6864 __ LoadFromOffset(kLoadWord, out_reg, obj_reg, offset); 6865 __ MaybeUnpoisonHeapReference(out_reg); 6866 } 6867} 6868 6869void InstructionCodeGeneratorARM::GenerateGcRootFieldLoad(HInstruction* instruction, 6870 Location root, 6871 Register obj, 6872 uint32_t offset, 6873 ReadBarrierOption read_barrier_option) { 6874 Register root_reg = root.AsRegister<Register>(); 6875 if (read_barrier_option == kWithReadBarrier) { 6876 DCHECK(kEmitCompilerReadBarrier); 6877 if (kUseBakerReadBarrier) { 6878 // Fast path implementation of art::ReadBarrier::BarrierForRoot when 6879 // Baker's read barrier are used: 6880 // 6881 // root = obj.field; 6882 // temp = Thread::Current()->pReadBarrierMarkReg ## root.reg() 6883 // if (temp != null) { 6884 // root = temp(root) 6885 // } 6886 6887 // /* GcRoot<mirror::Object> */ root = *(obj + offset) 6888 __ LoadFromOffset(kLoadWord, root_reg, obj, offset); 6889 static_assert( 6890 sizeof(mirror::CompressedReference<mirror::Object>) == sizeof(GcRoot<mirror::Object>), 6891 "art::mirror::CompressedReference<mirror::Object> and art::GcRoot<mirror::Object> " 6892 "have different sizes."); 6893 static_assert(sizeof(mirror::CompressedReference<mirror::Object>) == sizeof(int32_t), 6894 "art::mirror::CompressedReference<mirror::Object> and int32_t " 6895 "have different sizes."); 6896 6897 // Slow path marking the GC root `root`. 6898 Location temp = Location::RegisterLocation(LR); 6899 SlowPathCodeARM* slow_path = 6900 new (GetGraph()->GetArena()) ReadBarrierMarkSlowPathARM( 6901 instruction, 6902 root, 6903 /*entrypoint*/ temp); 6904 codegen_->AddSlowPath(slow_path); 6905 6906 // temp = Thread::Current()->pReadBarrierMarkReg ## root.reg() 6907 const int32_t entry_point_offset = 6908 CodeGenerator::GetReadBarrierMarkEntryPointsOffset<kArmPointerSize>(root.reg()); 6909 // Loading the entrypoint does not require a load acquire since it is only changed when 6910 // threads are suspended or running a checkpoint. 6911 __ LoadFromOffset(kLoadWord, temp.AsRegister<Register>(), TR, entry_point_offset); 6912 // The entrypoint is null when the GC is not marking, this prevents one load compared to 6913 // checking GetIsGcMarking. 6914 __ CompareAndBranchIfNonZero(temp.AsRegister<Register>(), slow_path->GetEntryLabel()); 6915 __ Bind(slow_path->GetExitLabel()); 6916 } else { 6917 // GC root loaded through a slow path for read barriers other 6918 // than Baker's. 6919 // /* GcRoot<mirror::Object>* */ root = obj + offset 6920 __ AddConstant(root_reg, obj, offset); 6921 // /* mirror::Object* */ root = root->Read() 6922 codegen_->GenerateReadBarrierForRootSlow(instruction, root, root); 6923 } 6924 } else { 6925 // Plain GC root load with no read barrier. 6926 // /* GcRoot<mirror::Object> */ root = *(obj + offset) 6927 __ LoadFromOffset(kLoadWord, root_reg, obj, offset); 6928 // Note that GC roots are not affected by heap poisoning, thus we 6929 // do not have to unpoison `root_reg` here. 6930 } 6931} 6932 6933void CodeGeneratorARM::GenerateFieldLoadWithBakerReadBarrier(HInstruction* instruction, 6934 Location ref, 6935 Register obj, 6936 uint32_t offset, 6937 Location temp, 6938 bool needs_null_check) { 6939 DCHECK(kEmitCompilerReadBarrier); 6940 DCHECK(kUseBakerReadBarrier); 6941 6942 // /* HeapReference<Object> */ ref = *(obj + offset) 6943 Location no_index = Location::NoLocation(); 6944 ScaleFactor no_scale_factor = TIMES_1; 6945 GenerateReferenceLoadWithBakerReadBarrier( 6946 instruction, ref, obj, offset, no_index, no_scale_factor, temp, needs_null_check); 6947} 6948 6949void CodeGeneratorARM::GenerateArrayLoadWithBakerReadBarrier(HInstruction* instruction, 6950 Location ref, 6951 Register obj, 6952 uint32_t data_offset, 6953 Location index, 6954 Location temp, 6955 bool needs_null_check) { 6956 DCHECK(kEmitCompilerReadBarrier); 6957 DCHECK(kUseBakerReadBarrier); 6958 6959 static_assert( 6960 sizeof(mirror::HeapReference<mirror::Object>) == sizeof(int32_t), 6961 "art::mirror::HeapReference<art::mirror::Object> and int32_t have different sizes."); 6962 // /* HeapReference<Object> */ ref = 6963 // *(obj + data_offset + index * sizeof(HeapReference<Object>)) 6964 ScaleFactor scale_factor = TIMES_4; 6965 GenerateReferenceLoadWithBakerReadBarrier( 6966 instruction, ref, obj, data_offset, index, scale_factor, temp, needs_null_check); 6967} 6968 6969void CodeGeneratorARM::GenerateReferenceLoadWithBakerReadBarrier(HInstruction* instruction, 6970 Location ref, 6971 Register obj, 6972 uint32_t offset, 6973 Location index, 6974 ScaleFactor scale_factor, 6975 Location temp, 6976 bool needs_null_check, 6977 bool always_update_field, 6978 Register* temp2) { 6979 DCHECK(kEmitCompilerReadBarrier); 6980 DCHECK(kUseBakerReadBarrier); 6981 6982 // In slow path based read barriers, the read barrier call is 6983 // inserted after the original load. However, in fast path based 6984 // Baker's read barriers, we need to perform the load of 6985 // mirror::Object::monitor_ *before* the original reference load. 6986 // This load-load ordering is required by the read barrier. 6987 // The fast path/slow path (for Baker's algorithm) should look like: 6988 // 6989 // uint32_t rb_state = Lockword(obj->monitor_).ReadBarrierState(); 6990 // lfence; // Load fence or artificial data dependency to prevent load-load reordering 6991 // HeapReference<Object> ref = *src; // Original reference load. 6992 // bool is_gray = (rb_state == ReadBarrier::GrayState()); 6993 // if (is_gray) { 6994 // ref = ReadBarrier::Mark(ref); // Performed by runtime entrypoint slow path. 6995 // } 6996 // 6997 // Note: the original implementation in ReadBarrier::Barrier is 6998 // slightly more complex as it performs additional checks that we do 6999 // not do here for performance reasons. 7000 7001 Register ref_reg = ref.AsRegister<Register>(); 7002 Register temp_reg = temp.AsRegister<Register>(); 7003 uint32_t monitor_offset = mirror::Object::MonitorOffset().Int32Value(); 7004 7005 // /* int32_t */ monitor = obj->monitor_ 7006 __ LoadFromOffset(kLoadWord, temp_reg, obj, monitor_offset); 7007 if (needs_null_check) { 7008 MaybeRecordImplicitNullCheck(instruction); 7009 } 7010 // /* LockWord */ lock_word = LockWord(monitor) 7011 static_assert(sizeof(LockWord) == sizeof(int32_t), 7012 "art::LockWord and int32_t have different sizes."); 7013 7014 // Introduce a dependency on the lock_word including the rb_state, 7015 // which shall prevent load-load reordering without using 7016 // a memory barrier (which would be more expensive). 7017 // `obj` is unchanged by this operation, but its value now depends 7018 // on `temp_reg`. 7019 __ add(obj, obj, ShifterOperand(temp_reg, LSR, 32)); 7020 7021 // The actual reference load. 7022 if (index.IsValid()) { 7023 // Load types involving an "index": ArrayGet, 7024 // UnsafeGetObject/UnsafeGetObjectVolatile and UnsafeCASObject 7025 // intrinsics. 7026 // /* HeapReference<Object> */ ref = *(obj + offset + (index << scale_factor)) 7027 if (index.IsConstant()) { 7028 size_t computed_offset = 7029 (index.GetConstant()->AsIntConstant()->GetValue() << scale_factor) + offset; 7030 __ LoadFromOffset(kLoadWord, ref_reg, obj, computed_offset); 7031 } else { 7032 // Handle the special case of the 7033 // UnsafeGetObject/UnsafeGetObjectVolatile and UnsafeCASObject 7034 // intrinsics, which use a register pair as index ("long 7035 // offset"), of which only the low part contains data. 7036 Register index_reg = index.IsRegisterPair() 7037 ? index.AsRegisterPairLow<Register>() 7038 : index.AsRegister<Register>(); 7039 __ add(IP, obj, ShifterOperand(index_reg, LSL, scale_factor)); 7040 __ LoadFromOffset(kLoadWord, ref_reg, IP, offset); 7041 } 7042 } else { 7043 // /* HeapReference<Object> */ ref = *(obj + offset) 7044 __ LoadFromOffset(kLoadWord, ref_reg, obj, offset); 7045 } 7046 7047 // Object* ref = ref_addr->AsMirrorPtr() 7048 __ MaybeUnpoisonHeapReference(ref_reg); 7049 7050 // Slow path marking the object `ref` when it is gray. 7051 SlowPathCodeARM* slow_path; 7052 if (always_update_field) { 7053 DCHECK(temp2 != nullptr); 7054 // ReadBarrierMarkAndUpdateFieldSlowPathARM only supports address 7055 // of the form `obj + field_offset`, where `obj` is a register and 7056 // `field_offset` is a register pair (of which only the lower half 7057 // is used). Thus `offset` and `scale_factor` above are expected 7058 // to be null in this code path. 7059 DCHECK_EQ(offset, 0u); 7060 DCHECK_EQ(scale_factor, ScaleFactor::TIMES_1); 7061 slow_path = new (GetGraph()->GetArena()) ReadBarrierMarkAndUpdateFieldSlowPathARM( 7062 instruction, ref, obj, /* field_offset */ index, temp_reg, *temp2); 7063 } else { 7064 slow_path = new (GetGraph()->GetArena()) ReadBarrierMarkSlowPathARM(instruction, ref); 7065 } 7066 AddSlowPath(slow_path); 7067 7068 // if (rb_state == ReadBarrier::GrayState()) 7069 // ref = ReadBarrier::Mark(ref); 7070 // Given the numeric representation, it's enough to check the low bit of the 7071 // rb_state. We do that by shifting the bit out of the lock word with LSRS 7072 // which can be a 16-bit instruction unlike the TST immediate. 7073 static_assert(ReadBarrier::WhiteState() == 0, "Expecting white to have value 0"); 7074 static_assert(ReadBarrier::GrayState() == 1, "Expecting gray to have value 1"); 7075 __ Lsrs(temp_reg, temp_reg, LockWord::kReadBarrierStateShift + 1); 7076 __ b(slow_path->GetEntryLabel(), CS); // Carry flag is the last bit shifted out by LSRS. 7077 __ Bind(slow_path->GetExitLabel()); 7078} 7079 7080void CodeGeneratorARM::GenerateReadBarrierSlow(HInstruction* instruction, 7081 Location out, 7082 Location ref, 7083 Location obj, 7084 uint32_t offset, 7085 Location index) { 7086 DCHECK(kEmitCompilerReadBarrier); 7087 7088 // Insert a slow path based read barrier *after* the reference load. 7089 // 7090 // If heap poisoning is enabled, the unpoisoning of the loaded 7091 // reference will be carried out by the runtime within the slow 7092 // path. 7093 // 7094 // Note that `ref` currently does not get unpoisoned (when heap 7095 // poisoning is enabled), which is alright as the `ref` argument is 7096 // not used by the artReadBarrierSlow entry point. 7097 // 7098 // TODO: Unpoison `ref` when it is used by artReadBarrierSlow. 7099 SlowPathCodeARM* slow_path = new (GetGraph()->GetArena()) 7100 ReadBarrierForHeapReferenceSlowPathARM(instruction, out, ref, obj, offset, index); 7101 AddSlowPath(slow_path); 7102 7103 __ b(slow_path->GetEntryLabel()); 7104 __ Bind(slow_path->GetExitLabel()); 7105} 7106 7107void CodeGeneratorARM::MaybeGenerateReadBarrierSlow(HInstruction* instruction, 7108 Location out, 7109 Location ref, 7110 Location obj, 7111 uint32_t offset, 7112 Location index) { 7113 if (kEmitCompilerReadBarrier) { 7114 // Baker's read barriers shall be handled by the fast path 7115 // (CodeGeneratorARM::GenerateReferenceLoadWithBakerReadBarrier). 7116 DCHECK(!kUseBakerReadBarrier); 7117 // If heap poisoning is enabled, unpoisoning will be taken care of 7118 // by the runtime within the slow path. 7119 GenerateReadBarrierSlow(instruction, out, ref, obj, offset, index); 7120 } else if (kPoisonHeapReferences) { 7121 __ UnpoisonHeapReference(out.AsRegister<Register>()); 7122 } 7123} 7124 7125void CodeGeneratorARM::GenerateReadBarrierForRootSlow(HInstruction* instruction, 7126 Location out, 7127 Location root) { 7128 DCHECK(kEmitCompilerReadBarrier); 7129 7130 // Insert a slow path based read barrier *after* the GC root load. 7131 // 7132 // Note that GC roots are not affected by heap poisoning, so we do 7133 // not need to do anything special for this here. 7134 SlowPathCodeARM* slow_path = 7135 new (GetGraph()->GetArena()) ReadBarrierForRootSlowPathARM(instruction, out, root); 7136 AddSlowPath(slow_path); 7137 7138 __ b(slow_path->GetEntryLabel()); 7139 __ Bind(slow_path->GetExitLabel()); 7140} 7141 7142HInvokeStaticOrDirect::DispatchInfo CodeGeneratorARM::GetSupportedInvokeStaticOrDirectDispatch( 7143 const HInvokeStaticOrDirect::DispatchInfo& desired_dispatch_info, 7144 HInvokeStaticOrDirect* invoke ATTRIBUTE_UNUSED) { 7145 HInvokeStaticOrDirect::DispatchInfo dispatch_info = desired_dispatch_info; 7146 // We disable pc-relative load when there is an irreducible loop, as the optimization 7147 // is incompatible with it. 7148 // TODO: Create as many ArmDexCacheArraysBase instructions as needed for methods 7149 // with irreducible loops. 7150 if (GetGraph()->HasIrreducibleLoops() && 7151 (dispatch_info.method_load_kind == 7152 HInvokeStaticOrDirect::MethodLoadKind::kDexCachePcRelative)) { 7153 dispatch_info.method_load_kind = HInvokeStaticOrDirect::MethodLoadKind::kDexCacheViaMethod; 7154 } 7155 7156 return dispatch_info; 7157} 7158 7159Register CodeGeneratorARM::GetInvokeStaticOrDirectExtraParameter(HInvokeStaticOrDirect* invoke, 7160 Register temp) { 7161 DCHECK_EQ(invoke->InputCount(), invoke->GetNumberOfArguments() + 1u); 7162 Location location = invoke->GetLocations()->InAt(invoke->GetSpecialInputIndex()); 7163 if (!invoke->GetLocations()->Intrinsified()) { 7164 return location.AsRegister<Register>(); 7165 } 7166 // For intrinsics we allow any location, so it may be on the stack. 7167 if (!location.IsRegister()) { 7168 __ LoadFromOffset(kLoadWord, temp, SP, location.GetStackIndex()); 7169 return temp; 7170 } 7171 // For register locations, check if the register was saved. If so, get it from the stack. 7172 // Note: There is a chance that the register was saved but not overwritten, so we could 7173 // save one load. However, since this is just an intrinsic slow path we prefer this 7174 // simple and more robust approach rather that trying to determine if that's the case. 7175 SlowPathCode* slow_path = GetCurrentSlowPath(); 7176 DCHECK(slow_path != nullptr); // For intrinsified invokes the call is emitted on the slow path. 7177 if (slow_path->IsCoreRegisterSaved(location.AsRegister<Register>())) { 7178 int stack_offset = slow_path->GetStackOffsetOfCoreRegister(location.AsRegister<Register>()); 7179 __ LoadFromOffset(kLoadWord, temp, SP, stack_offset); 7180 return temp; 7181 } 7182 return location.AsRegister<Register>(); 7183} 7184 7185void CodeGeneratorARM::GenerateStaticOrDirectCall(HInvokeStaticOrDirect* invoke, Location temp) { 7186 Location callee_method = temp; // For all kinds except kRecursive, callee will be in temp. 7187 switch (invoke->GetMethodLoadKind()) { 7188 case HInvokeStaticOrDirect::MethodLoadKind::kStringInit: { 7189 uint32_t offset = 7190 GetThreadOffset<kArmPointerSize>(invoke->GetStringInitEntryPoint()).Int32Value(); 7191 // temp = thread->string_init_entrypoint 7192 __ LoadFromOffset(kLoadWord, temp.AsRegister<Register>(), TR, offset); 7193 break; 7194 } 7195 case HInvokeStaticOrDirect::MethodLoadKind::kRecursive: 7196 callee_method = invoke->GetLocations()->InAt(invoke->GetSpecialInputIndex()); 7197 break; 7198 case HInvokeStaticOrDirect::MethodLoadKind::kDirectAddress: 7199 __ LoadImmediate(temp.AsRegister<Register>(), invoke->GetMethodAddress()); 7200 break; 7201 case HInvokeStaticOrDirect::MethodLoadKind::kDexCachePcRelative: { 7202 HArmDexCacheArraysBase* base = 7203 invoke->InputAt(invoke->GetSpecialInputIndex())->AsArmDexCacheArraysBase(); 7204 Register base_reg = GetInvokeStaticOrDirectExtraParameter(invoke, 7205 temp.AsRegister<Register>()); 7206 int32_t offset = invoke->GetDexCacheArrayOffset() - base->GetElementOffset(); 7207 __ LoadFromOffset(kLoadWord, temp.AsRegister<Register>(), base_reg, offset); 7208 break; 7209 } 7210 case HInvokeStaticOrDirect::MethodLoadKind::kDexCacheViaMethod: { 7211 Location current_method = invoke->GetLocations()->InAt(invoke->GetSpecialInputIndex()); 7212 Register method_reg; 7213 Register reg = temp.AsRegister<Register>(); 7214 if (current_method.IsRegister()) { 7215 method_reg = current_method.AsRegister<Register>(); 7216 } else { 7217 DCHECK(invoke->GetLocations()->Intrinsified()); 7218 DCHECK(!current_method.IsValid()); 7219 method_reg = reg; 7220 __ LoadFromOffset(kLoadWord, reg, SP, kCurrentMethodStackOffset); 7221 } 7222 // /* ArtMethod*[] */ temp = temp.ptr_sized_fields_->dex_cache_resolved_methods_; 7223 __ LoadFromOffset(kLoadWord, 7224 reg, 7225 method_reg, 7226 ArtMethod::DexCacheResolvedMethodsOffset(kArmPointerSize).Int32Value()); 7227 // temp = temp[index_in_cache]; 7228 // Note: Don't use invoke->GetTargetMethod() as it may point to a different dex file. 7229 uint32_t index_in_cache = invoke->GetDexMethodIndex(); 7230 __ LoadFromOffset(kLoadWord, reg, reg, CodeGenerator::GetCachePointerOffset(index_in_cache)); 7231 break; 7232 } 7233 } 7234 7235 switch (invoke->GetCodePtrLocation()) { 7236 case HInvokeStaticOrDirect::CodePtrLocation::kCallSelf: 7237 __ bl(GetFrameEntryLabel()); 7238 break; 7239 case HInvokeStaticOrDirect::CodePtrLocation::kCallArtMethod: 7240 // LR = callee_method->entry_point_from_quick_compiled_code_ 7241 __ LoadFromOffset( 7242 kLoadWord, LR, callee_method.AsRegister<Register>(), 7243 ArtMethod::EntryPointFromQuickCompiledCodeOffset(kArmPointerSize).Int32Value()); 7244 // LR() 7245 __ blx(LR); 7246 break; 7247 } 7248 7249 DCHECK(!IsLeafMethod()); 7250} 7251 7252void CodeGeneratorARM::GenerateVirtualCall(HInvokeVirtual* invoke, Location temp_location) { 7253 Register temp = temp_location.AsRegister<Register>(); 7254 uint32_t method_offset = mirror::Class::EmbeddedVTableEntryOffset( 7255 invoke->GetVTableIndex(), kArmPointerSize).Uint32Value(); 7256 7257 // Use the calling convention instead of the location of the receiver, as 7258 // intrinsics may have put the receiver in a different register. In the intrinsics 7259 // slow path, the arguments have been moved to the right place, so here we are 7260 // guaranteed that the receiver is the first register of the calling convention. 7261 InvokeDexCallingConvention calling_convention; 7262 Register receiver = calling_convention.GetRegisterAt(0); 7263 uint32_t class_offset = mirror::Object::ClassOffset().Int32Value(); 7264 // /* HeapReference<Class> */ temp = receiver->klass_ 7265 __ LoadFromOffset(kLoadWord, temp, receiver, class_offset); 7266 MaybeRecordImplicitNullCheck(invoke); 7267 // Instead of simply (possibly) unpoisoning `temp` here, we should 7268 // emit a read barrier for the previous class reference load. 7269 // However this is not required in practice, as this is an 7270 // intermediate/temporary reference and because the current 7271 // concurrent copying collector keeps the from-space memory 7272 // intact/accessible until the end of the marking phase (the 7273 // concurrent copying collector may not in the future). 7274 __ MaybeUnpoisonHeapReference(temp); 7275 // temp = temp->GetMethodAt(method_offset); 7276 uint32_t entry_point = ArtMethod::EntryPointFromQuickCompiledCodeOffset( 7277 kArmPointerSize).Int32Value(); 7278 __ LoadFromOffset(kLoadWord, temp, temp, method_offset); 7279 // LR = temp->GetEntryPoint(); 7280 __ LoadFromOffset(kLoadWord, LR, temp, entry_point); 7281 // LR(); 7282 __ blx(LR); 7283} 7284 7285CodeGeneratorARM::PcRelativePatchInfo* CodeGeneratorARM::NewPcRelativeStringPatch( 7286 const DexFile& dex_file, uint32_t string_index) { 7287 return NewPcRelativePatch(dex_file, string_index, &pc_relative_string_patches_); 7288} 7289 7290CodeGeneratorARM::PcRelativePatchInfo* CodeGeneratorARM::NewPcRelativeTypePatch( 7291 const DexFile& dex_file, dex::TypeIndex type_index) { 7292 return NewPcRelativePatch(dex_file, type_index.index_, &pc_relative_type_patches_); 7293} 7294 7295CodeGeneratorARM::PcRelativePatchInfo* CodeGeneratorARM::NewPcRelativeDexCacheArrayPatch( 7296 const DexFile& dex_file, uint32_t element_offset) { 7297 return NewPcRelativePatch(dex_file, element_offset, &pc_relative_dex_cache_patches_); 7298} 7299 7300CodeGeneratorARM::PcRelativePatchInfo* CodeGeneratorARM::NewPcRelativePatch( 7301 const DexFile& dex_file, uint32_t offset_or_index, ArenaDeque<PcRelativePatchInfo>* patches) { 7302 patches->emplace_back(dex_file, offset_or_index); 7303 return &patches->back(); 7304} 7305 7306Literal* CodeGeneratorARM::DeduplicateBootImageStringLiteral(const DexFile& dex_file, 7307 dex::StringIndex string_index) { 7308 return boot_image_string_patches_.GetOrCreate( 7309 StringReference(&dex_file, string_index), 7310 [this]() { return __ NewLiteral<uint32_t>(/* placeholder */ 0u); }); 7311} 7312 7313Literal* CodeGeneratorARM::DeduplicateBootImageTypeLiteral(const DexFile& dex_file, 7314 dex::TypeIndex type_index) { 7315 return boot_image_type_patches_.GetOrCreate( 7316 TypeReference(&dex_file, type_index), 7317 [this]() { return __ NewLiteral<uint32_t>(/* placeholder */ 0u); }); 7318} 7319 7320Literal* CodeGeneratorARM::DeduplicateBootImageAddressLiteral(uint32_t address) { 7321 bool needs_patch = GetCompilerOptions().GetIncludePatchInformation(); 7322 Uint32ToLiteralMap* map = needs_patch ? &boot_image_address_patches_ : &uint32_literals_; 7323 return DeduplicateUint32Literal(dchecked_integral_cast<uint32_t>(address), map); 7324} 7325 7326Literal* CodeGeneratorARM::DeduplicateJitStringLiteral(const DexFile& dex_file, 7327 dex::StringIndex string_index) { 7328 jit_string_roots_.Overwrite(StringReference(&dex_file, string_index), /* placeholder */ 0u); 7329 return jit_string_patches_.GetOrCreate( 7330 StringReference(&dex_file, string_index), 7331 [this]() { return __ NewLiteral<uint32_t>(/* placeholder */ 0u); }); 7332} 7333 7334Literal* CodeGeneratorARM::DeduplicateJitClassLiteral(const DexFile& dex_file, 7335 dex::TypeIndex type_index, 7336 uint64_t address) { 7337 jit_class_roots_.Overwrite(TypeReference(&dex_file, type_index), address); 7338 return jit_class_patches_.GetOrCreate( 7339 TypeReference(&dex_file, type_index), 7340 [this]() { return __ NewLiteral<uint32_t>(/* placeholder */ 0u); }); 7341} 7342 7343template <LinkerPatch (*Factory)(size_t, const DexFile*, uint32_t, uint32_t)> 7344inline void CodeGeneratorARM::EmitPcRelativeLinkerPatches( 7345 const ArenaDeque<PcRelativePatchInfo>& infos, 7346 ArenaVector<LinkerPatch>* linker_patches) { 7347 for (const PcRelativePatchInfo& info : infos) { 7348 const DexFile& dex_file = info.target_dex_file; 7349 size_t offset_or_index = info.offset_or_index; 7350 DCHECK(info.add_pc_label.IsBound()); 7351 uint32_t add_pc_offset = dchecked_integral_cast<uint32_t>(info.add_pc_label.Position()); 7352 // Add MOVW patch. 7353 DCHECK(info.movw_label.IsBound()); 7354 uint32_t movw_offset = dchecked_integral_cast<uint32_t>(info.movw_label.Position()); 7355 linker_patches->push_back(Factory(movw_offset, &dex_file, add_pc_offset, offset_or_index)); 7356 // Add MOVT patch. 7357 DCHECK(info.movt_label.IsBound()); 7358 uint32_t movt_offset = dchecked_integral_cast<uint32_t>(info.movt_label.Position()); 7359 linker_patches->push_back(Factory(movt_offset, &dex_file, add_pc_offset, offset_or_index)); 7360 } 7361} 7362 7363void CodeGeneratorARM::EmitLinkerPatches(ArenaVector<LinkerPatch>* linker_patches) { 7364 DCHECK(linker_patches->empty()); 7365 size_t size = 7366 /* MOVW+MOVT for each entry */ 2u * pc_relative_dex_cache_patches_.size() + 7367 boot_image_string_patches_.size() + 7368 /* MOVW+MOVT for each entry */ 2u * pc_relative_string_patches_.size() + 7369 boot_image_type_patches_.size() + 7370 /* MOVW+MOVT for each entry */ 2u * pc_relative_type_patches_.size() + 7371 boot_image_address_patches_.size(); 7372 linker_patches->reserve(size); 7373 EmitPcRelativeLinkerPatches<LinkerPatch::DexCacheArrayPatch>(pc_relative_dex_cache_patches_, 7374 linker_patches); 7375 for (const auto& entry : boot_image_string_patches_) { 7376 const StringReference& target_string = entry.first; 7377 Literal* literal = entry.second; 7378 DCHECK(literal->GetLabel()->IsBound()); 7379 uint32_t literal_offset = literal->GetLabel()->Position(); 7380 linker_patches->push_back(LinkerPatch::StringPatch(literal_offset, 7381 target_string.dex_file, 7382 target_string.string_index.index_)); 7383 } 7384 if (!GetCompilerOptions().IsBootImage()) { 7385 EmitPcRelativeLinkerPatches<LinkerPatch::StringBssEntryPatch>(pc_relative_string_patches_, 7386 linker_patches); 7387 } else { 7388 EmitPcRelativeLinkerPatches<LinkerPatch::RelativeStringPatch>(pc_relative_string_patches_, 7389 linker_patches); 7390 } 7391 for (const auto& entry : boot_image_type_patches_) { 7392 const TypeReference& target_type = entry.first; 7393 Literal* literal = entry.second; 7394 DCHECK(literal->GetLabel()->IsBound()); 7395 uint32_t literal_offset = literal->GetLabel()->Position(); 7396 linker_patches->push_back(LinkerPatch::TypePatch(literal_offset, 7397 target_type.dex_file, 7398 target_type.type_index.index_)); 7399 } 7400 EmitPcRelativeLinkerPatches<LinkerPatch::RelativeTypePatch>(pc_relative_type_patches_, 7401 linker_patches); 7402 for (const auto& entry : boot_image_address_patches_) { 7403 DCHECK(GetCompilerOptions().GetIncludePatchInformation()); 7404 Literal* literal = entry.second; 7405 DCHECK(literal->GetLabel()->IsBound()); 7406 uint32_t literal_offset = literal->GetLabel()->Position(); 7407 linker_patches->push_back(LinkerPatch::RecordPosition(literal_offset)); 7408 } 7409} 7410 7411Literal* CodeGeneratorARM::DeduplicateUint32Literal(uint32_t value, Uint32ToLiteralMap* map) { 7412 return map->GetOrCreate( 7413 value, 7414 [this, value]() { return __ NewLiteral<uint32_t>(value); }); 7415} 7416 7417Literal* CodeGeneratorARM::DeduplicateMethodLiteral(MethodReference target_method, 7418 MethodToLiteralMap* map) { 7419 return map->GetOrCreate( 7420 target_method, 7421 [this]() { return __ NewLiteral<uint32_t>(/* placeholder */ 0u); }); 7422} 7423 7424void LocationsBuilderARM::VisitMultiplyAccumulate(HMultiplyAccumulate* instr) { 7425 LocationSummary* locations = 7426 new (GetGraph()->GetArena()) LocationSummary(instr, LocationSummary::kNoCall); 7427 locations->SetInAt(HMultiplyAccumulate::kInputAccumulatorIndex, 7428 Location::RequiresRegister()); 7429 locations->SetInAt(HMultiplyAccumulate::kInputMulLeftIndex, Location::RequiresRegister()); 7430 locations->SetInAt(HMultiplyAccumulate::kInputMulRightIndex, Location::RequiresRegister()); 7431 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 7432} 7433 7434void InstructionCodeGeneratorARM::VisitMultiplyAccumulate(HMultiplyAccumulate* instr) { 7435 LocationSummary* locations = instr->GetLocations(); 7436 Register res = locations->Out().AsRegister<Register>(); 7437 Register accumulator = 7438 locations->InAt(HMultiplyAccumulate::kInputAccumulatorIndex).AsRegister<Register>(); 7439 Register mul_left = 7440 locations->InAt(HMultiplyAccumulate::kInputMulLeftIndex).AsRegister<Register>(); 7441 Register mul_right = 7442 locations->InAt(HMultiplyAccumulate::kInputMulRightIndex).AsRegister<Register>(); 7443 7444 if (instr->GetOpKind() == HInstruction::kAdd) { 7445 __ mla(res, mul_left, mul_right, accumulator); 7446 } else { 7447 __ mls(res, mul_left, mul_right, accumulator); 7448 } 7449} 7450 7451void LocationsBuilderARM::VisitBoundType(HBoundType* instruction ATTRIBUTE_UNUSED) { 7452 // Nothing to do, this should be removed during prepare for register allocator. 7453 LOG(FATAL) << "Unreachable"; 7454} 7455 7456void InstructionCodeGeneratorARM::VisitBoundType(HBoundType* instruction ATTRIBUTE_UNUSED) { 7457 // Nothing to do, this should be removed during prepare for register allocator. 7458 LOG(FATAL) << "Unreachable"; 7459} 7460 7461// Simple implementation of packed switch - generate cascaded compare/jumps. 7462void LocationsBuilderARM::VisitPackedSwitch(HPackedSwitch* switch_instr) { 7463 LocationSummary* locations = 7464 new (GetGraph()->GetArena()) LocationSummary(switch_instr, LocationSummary::kNoCall); 7465 locations->SetInAt(0, Location::RequiresRegister()); 7466 if (switch_instr->GetNumEntries() > kPackedSwitchCompareJumpThreshold && 7467 codegen_->GetAssembler()->IsThumb()) { 7468 locations->AddTemp(Location::RequiresRegister()); // We need a temp for the table base. 7469 if (switch_instr->GetStartValue() != 0) { 7470 locations->AddTemp(Location::RequiresRegister()); // We need a temp for the bias. 7471 } 7472 } 7473} 7474 7475void InstructionCodeGeneratorARM::VisitPackedSwitch(HPackedSwitch* switch_instr) { 7476 int32_t lower_bound = switch_instr->GetStartValue(); 7477 uint32_t num_entries = switch_instr->GetNumEntries(); 7478 LocationSummary* locations = switch_instr->GetLocations(); 7479 Register value_reg = locations->InAt(0).AsRegister<Register>(); 7480 HBasicBlock* default_block = switch_instr->GetDefaultBlock(); 7481 7482 if (num_entries <= kPackedSwitchCompareJumpThreshold || !codegen_->GetAssembler()->IsThumb()) { 7483 // Create a series of compare/jumps. 7484 Register temp_reg = IP; 7485 // Note: It is fine for the below AddConstantSetFlags() using IP register to temporarily store 7486 // the immediate, because IP is used as the destination register. For the other 7487 // AddConstantSetFlags() and GenerateCompareWithImmediate(), the immediate values are constant, 7488 // and they can be encoded in the instruction without making use of IP register. 7489 __ AddConstantSetFlags(temp_reg, value_reg, -lower_bound); 7490 7491 const ArenaVector<HBasicBlock*>& successors = switch_instr->GetBlock()->GetSuccessors(); 7492 // Jump to successors[0] if value == lower_bound. 7493 __ b(codegen_->GetLabelOf(successors[0]), EQ); 7494 int32_t last_index = 0; 7495 for (; num_entries - last_index > 2; last_index += 2) { 7496 __ AddConstantSetFlags(temp_reg, temp_reg, -2); 7497 // Jump to successors[last_index + 1] if value < case_value[last_index + 2]. 7498 __ b(codegen_->GetLabelOf(successors[last_index + 1]), LO); 7499 // Jump to successors[last_index + 2] if value == case_value[last_index + 2]. 7500 __ b(codegen_->GetLabelOf(successors[last_index + 2]), EQ); 7501 } 7502 if (num_entries - last_index == 2) { 7503 // The last missing case_value. 7504 __ CmpConstant(temp_reg, 1); 7505 __ b(codegen_->GetLabelOf(successors[last_index + 1]), EQ); 7506 } 7507 7508 // And the default for any other value. 7509 if (!codegen_->GoesToNextBlock(switch_instr->GetBlock(), default_block)) { 7510 __ b(codegen_->GetLabelOf(default_block)); 7511 } 7512 } else { 7513 // Create a table lookup. 7514 Register temp_reg = locations->GetTemp(0).AsRegister<Register>(); 7515 7516 // Materialize a pointer to the switch table 7517 std::vector<Label*> labels(num_entries); 7518 const ArenaVector<HBasicBlock*>& successors = switch_instr->GetBlock()->GetSuccessors(); 7519 for (uint32_t i = 0; i < num_entries; i++) { 7520 labels[i] = codegen_->GetLabelOf(successors[i]); 7521 } 7522 JumpTable* table = __ CreateJumpTable(std::move(labels), temp_reg); 7523 7524 // Remove the bias. 7525 Register key_reg; 7526 if (lower_bound != 0) { 7527 key_reg = locations->GetTemp(1).AsRegister<Register>(); 7528 __ AddConstant(key_reg, value_reg, -lower_bound); 7529 } else { 7530 key_reg = value_reg; 7531 } 7532 7533 // Check whether the value is in the table, jump to default block if not. 7534 __ CmpConstant(key_reg, num_entries - 1); 7535 __ b(codegen_->GetLabelOf(default_block), Condition::HI); 7536 7537 // Load the displacement from the table. 7538 __ ldr(temp_reg, Address(temp_reg, key_reg, Shift::LSL, 2)); 7539 7540 // Dispatch is a direct add to the PC (for Thumb2). 7541 __ EmitJumpTableDispatch(table, temp_reg); 7542 } 7543} 7544 7545void LocationsBuilderARM::VisitArmDexCacheArraysBase(HArmDexCacheArraysBase* base) { 7546 LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(base); 7547 locations->SetOut(Location::RequiresRegister()); 7548} 7549 7550void InstructionCodeGeneratorARM::VisitArmDexCacheArraysBase(HArmDexCacheArraysBase* base) { 7551 Register base_reg = base->GetLocations()->Out().AsRegister<Register>(); 7552 CodeGeneratorARM::PcRelativePatchInfo* labels = 7553 codegen_->NewPcRelativeDexCacheArrayPatch(base->GetDexFile(), base->GetElementOffset()); 7554 __ BindTrackedLabel(&labels->movw_label); 7555 __ movw(base_reg, /* placeholder */ 0u); 7556 __ BindTrackedLabel(&labels->movt_label); 7557 __ movt(base_reg, /* placeholder */ 0u); 7558 __ BindTrackedLabel(&labels->add_pc_label); 7559 __ add(base_reg, base_reg, ShifterOperand(PC)); 7560} 7561 7562void CodeGeneratorARM::MoveFromReturnRegister(Location trg, Primitive::Type type) { 7563 if (!trg.IsValid()) { 7564 DCHECK_EQ(type, Primitive::kPrimVoid); 7565 return; 7566 } 7567 7568 DCHECK_NE(type, Primitive::kPrimVoid); 7569 7570 Location return_loc = InvokeDexCallingConventionVisitorARM().GetReturnLocation(type); 7571 if (return_loc.Equals(trg)) { 7572 return; 7573 } 7574 7575 // TODO: Consider pairs in the parallel move resolver, then this could be nicely merged 7576 // with the last branch. 7577 if (type == Primitive::kPrimLong) { 7578 HParallelMove parallel_move(GetGraph()->GetArena()); 7579 parallel_move.AddMove(return_loc.ToLow(), trg.ToLow(), Primitive::kPrimInt, nullptr); 7580 parallel_move.AddMove(return_loc.ToHigh(), trg.ToHigh(), Primitive::kPrimInt, nullptr); 7581 GetMoveResolver()->EmitNativeCode(¶llel_move); 7582 } else if (type == Primitive::kPrimDouble) { 7583 HParallelMove parallel_move(GetGraph()->GetArena()); 7584 parallel_move.AddMove(return_loc.ToLow(), trg.ToLow(), Primitive::kPrimFloat, nullptr); 7585 parallel_move.AddMove(return_loc.ToHigh(), trg.ToHigh(), Primitive::kPrimFloat, nullptr); 7586 GetMoveResolver()->EmitNativeCode(¶llel_move); 7587 } else { 7588 // Let the parallel move resolver take care of all of this. 7589 HParallelMove parallel_move(GetGraph()->GetArena()); 7590 parallel_move.AddMove(return_loc, trg, type, nullptr); 7591 GetMoveResolver()->EmitNativeCode(¶llel_move); 7592 } 7593} 7594 7595void LocationsBuilderARM::VisitClassTableGet(HClassTableGet* instruction) { 7596 LocationSummary* locations = 7597 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall); 7598 locations->SetInAt(0, Location::RequiresRegister()); 7599 locations->SetOut(Location::RequiresRegister()); 7600} 7601 7602void InstructionCodeGeneratorARM::VisitClassTableGet(HClassTableGet* instruction) { 7603 LocationSummary* locations = instruction->GetLocations(); 7604 if (instruction->GetTableKind() == HClassTableGet::TableKind::kVTable) { 7605 uint32_t method_offset = mirror::Class::EmbeddedVTableEntryOffset( 7606 instruction->GetIndex(), kArmPointerSize).SizeValue(); 7607 __ LoadFromOffset(kLoadWord, 7608 locations->Out().AsRegister<Register>(), 7609 locations->InAt(0).AsRegister<Register>(), 7610 method_offset); 7611 } else { 7612 uint32_t method_offset = static_cast<uint32_t>(ImTable::OffsetOfElement( 7613 instruction->GetIndex(), kArmPointerSize)); 7614 __ LoadFromOffset(kLoadWord, 7615 locations->Out().AsRegister<Register>(), 7616 locations->InAt(0).AsRegister<Register>(), 7617 mirror::Class::ImtPtrOffset(kArmPointerSize).Uint32Value()); 7618 __ LoadFromOffset(kLoadWord, 7619 locations->Out().AsRegister<Register>(), 7620 locations->Out().AsRegister<Register>(), 7621 method_offset); 7622 } 7623} 7624 7625static void PatchJitRootUse(uint8_t* code, 7626 const uint8_t* roots_data, 7627 Literal* literal, 7628 uint64_t index_in_table) { 7629 DCHECK(literal->GetLabel()->IsBound()); 7630 uint32_t literal_offset = literal->GetLabel()->Position(); 7631 uintptr_t address = 7632 reinterpret_cast<uintptr_t>(roots_data) + index_in_table * sizeof(GcRoot<mirror::Object>); 7633 uint8_t* data = code + literal_offset; 7634 reinterpret_cast<uint32_t*>(data)[0] = dchecked_integral_cast<uint32_t>(address); 7635} 7636 7637void CodeGeneratorARM::EmitJitRootPatches(uint8_t* code, const uint8_t* roots_data) { 7638 for (const auto& entry : jit_string_patches_) { 7639 const auto& it = jit_string_roots_.find(entry.first); 7640 DCHECK(it != jit_string_roots_.end()); 7641 PatchJitRootUse(code, roots_data, entry.second, it->second); 7642 } 7643 for (const auto& entry : jit_class_patches_) { 7644 const auto& it = jit_class_roots_.find(entry.first); 7645 DCHECK(it != jit_class_roots_.end()); 7646 PatchJitRootUse(code, roots_data, entry.second, it->second); 7647 } 7648} 7649 7650#undef __ 7651#undef QUICK_ENTRY_POINT 7652 7653} // namespace arm 7654} // namespace art 7655