code_generator_arm.cc revision d9911eeca13f609c885e0f6a5ce81af9b6340bfa
1/* 2 * Copyright (C) 2014 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17#include "code_generator_arm.h" 18 19#include "arch/arm/instruction_set_features_arm.h" 20#include "art_method.h" 21#include "code_generator_utils.h" 22#include "common_arm.h" 23#include "compiled_method.h" 24#include "entrypoints/quick/quick_entrypoints.h" 25#include "gc/accounting/card_table.h" 26#include "intrinsics.h" 27#include "intrinsics_arm.h" 28#include "mirror/array-inl.h" 29#include "mirror/class-inl.h" 30#include "thread.h" 31#include "utils/arm/assembler_arm.h" 32#include "utils/arm/managed_register_arm.h" 33#include "utils/assembler.h" 34#include "utils/stack_checks.h" 35 36namespace art { 37 38template<class MirrorType> 39class GcRoot; 40 41namespace arm { 42 43static bool ExpectedPairLayout(Location location) { 44 // We expected this for both core and fpu register pairs. 45 return ((location.low() & 1) == 0) && (location.low() + 1 == location.high()); 46} 47 48static constexpr int kCurrentMethodStackOffset = 0; 49static constexpr Register kMethodRegisterArgument = R0; 50 51static constexpr Register kCoreAlwaysSpillRegister = R5; 52static constexpr Register kCoreCalleeSaves[] = 53 { R5, R6, R7, R8, R10, R11, LR }; 54static constexpr SRegister kFpuCalleeSaves[] = 55 { S16, S17, S18, S19, S20, S21, S22, S23, S24, S25, S26, S27, S28, S29, S30, S31 }; 56 57// D31 cannot be split into two S registers, and the register allocator only works on 58// S registers. Therefore there is no need to block it. 59static constexpr DRegister DTMP = D31; 60 61static constexpr uint32_t kPackedSwitchCompareJumpThreshold = 7; 62 63// NOLINT on __ macro to suppress wrong warning/fix (misc-macro-parentheses) from clang-tidy. 64#define __ down_cast<ArmAssembler*>(codegen->GetAssembler())-> // NOLINT 65#define QUICK_ENTRY_POINT(x) QUICK_ENTRYPOINT_OFFSET(kArmPointerSize, x).Int32Value() 66 67static constexpr int kRegListThreshold = 4; 68 69// SaveLiveRegisters and RestoreLiveRegisters from SlowPathCodeARM operate on sets of S registers, 70// for each live D registers they treat two corresponding S registers as live ones. 71// 72// Two following functions (SaveContiguousSRegisterList, RestoreContiguousSRegisterList) build 73// from a list of contiguous S registers a list of contiguous D registers (processing first/last 74// S registers corner cases) and save/restore this new list treating them as D registers. 75// - decreasing code size 76// - avoiding hazards on Cortex-A57, when a pair of S registers for an actual live D register is 77// restored and then used in regular non SlowPath code as D register. 78// 79// For the following example (v means the S register is live): 80// D names: | D0 | D1 | D2 | D4 | ... 81// S names: | S0 | S1 | S2 | S3 | S4 | S5 | S6 | S7 | ... 82// Live? | | v | v | v | v | v | v | | ... 83// 84// S1 and S6 will be saved/restored independently; D registers list (D1, D2) will be processed 85// as D registers. 86static size_t SaveContiguousSRegisterList(size_t first, 87 size_t last, 88 CodeGenerator* codegen, 89 size_t stack_offset) { 90 DCHECK_LE(first, last); 91 if ((first == last) && (first == 0)) { 92 stack_offset += codegen->SaveFloatingPointRegister(stack_offset, first); 93 return stack_offset; 94 } 95 if (first % 2 == 1) { 96 stack_offset += codegen->SaveFloatingPointRegister(stack_offset, first++); 97 } 98 99 bool save_last = false; 100 if (last % 2 == 0) { 101 save_last = true; 102 --last; 103 } 104 105 if (first < last) { 106 DRegister d_reg = static_cast<DRegister>(first / 2); 107 DCHECK_EQ((last - first + 1) % 2, 0u); 108 size_t number_of_d_regs = (last - first + 1) / 2; 109 110 if (number_of_d_regs == 1) { 111 __ StoreDToOffset(d_reg, SP, stack_offset); 112 } else if (number_of_d_regs > 1) { 113 __ add(IP, SP, ShifterOperand(stack_offset)); 114 __ vstmiad(IP, d_reg, number_of_d_regs); 115 } 116 stack_offset += number_of_d_regs * kArmWordSize * 2; 117 } 118 119 if (save_last) { 120 stack_offset += codegen->SaveFloatingPointRegister(stack_offset, last + 1); 121 } 122 123 return stack_offset; 124} 125 126static size_t RestoreContiguousSRegisterList(size_t first, 127 size_t last, 128 CodeGenerator* codegen, 129 size_t stack_offset) { 130 DCHECK_LE(first, last); 131 if ((first == last) && (first == 0)) { 132 stack_offset += codegen->RestoreFloatingPointRegister(stack_offset, first); 133 return stack_offset; 134 } 135 if (first % 2 == 1) { 136 stack_offset += codegen->RestoreFloatingPointRegister(stack_offset, first++); 137 } 138 139 bool restore_last = false; 140 if (last % 2 == 0) { 141 restore_last = true; 142 --last; 143 } 144 145 if (first < last) { 146 DRegister d_reg = static_cast<DRegister>(first / 2); 147 DCHECK_EQ((last - first + 1) % 2, 0u); 148 size_t number_of_d_regs = (last - first + 1) / 2; 149 if (number_of_d_regs == 1) { 150 __ LoadDFromOffset(d_reg, SP, stack_offset); 151 } else if (number_of_d_regs > 1) { 152 __ add(IP, SP, ShifterOperand(stack_offset)); 153 __ vldmiad(IP, d_reg, number_of_d_regs); 154 } 155 stack_offset += number_of_d_regs * kArmWordSize * 2; 156 } 157 158 if (restore_last) { 159 stack_offset += codegen->RestoreFloatingPointRegister(stack_offset, last + 1); 160 } 161 162 return stack_offset; 163} 164 165void SlowPathCodeARM::SaveLiveRegisters(CodeGenerator* codegen, LocationSummary* locations) { 166 size_t stack_offset = codegen->GetFirstRegisterSlotInSlowPath(); 167 size_t orig_offset = stack_offset; 168 169 const uint32_t core_spills = codegen->GetSlowPathSpills(locations, /* core_registers */ true); 170 for (uint32_t i : LowToHighBits(core_spills)) { 171 // If the register holds an object, update the stack mask. 172 if (locations->RegisterContainsObject(i)) { 173 locations->SetStackBit(stack_offset / kVRegSize); 174 } 175 DCHECK_LT(stack_offset, codegen->GetFrameSize() - codegen->FrameEntrySpillSize()); 176 DCHECK_LT(i, kMaximumNumberOfExpectedRegisters); 177 saved_core_stack_offsets_[i] = stack_offset; 178 stack_offset += kArmWordSize; 179 } 180 181 int reg_num = POPCOUNT(core_spills); 182 if (reg_num != 0) { 183 if (reg_num > kRegListThreshold) { 184 __ StoreList(RegList(core_spills), orig_offset); 185 } else { 186 stack_offset = orig_offset; 187 for (uint32_t i : LowToHighBits(core_spills)) { 188 stack_offset += codegen->SaveCoreRegister(stack_offset, i); 189 } 190 } 191 } 192 193 uint32_t fp_spills = codegen->GetSlowPathSpills(locations, /* core_registers */ false); 194 orig_offset = stack_offset; 195 for (uint32_t i : LowToHighBits(fp_spills)) { 196 DCHECK_LT(i, kMaximumNumberOfExpectedRegisters); 197 saved_fpu_stack_offsets_[i] = stack_offset; 198 stack_offset += kArmWordSize; 199 } 200 201 stack_offset = orig_offset; 202 while (fp_spills != 0u) { 203 uint32_t begin = CTZ(fp_spills); 204 uint32_t tmp = fp_spills + (1u << begin); 205 fp_spills &= tmp; // Clear the contiguous range of 1s. 206 uint32_t end = (tmp == 0u) ? 32u : CTZ(tmp); // CTZ(0) is undefined. 207 stack_offset = SaveContiguousSRegisterList(begin, end - 1, codegen, stack_offset); 208 } 209 DCHECK_LE(stack_offset, codegen->GetFrameSize() - codegen->FrameEntrySpillSize()); 210} 211 212void SlowPathCodeARM::RestoreLiveRegisters(CodeGenerator* codegen, LocationSummary* locations) { 213 size_t stack_offset = codegen->GetFirstRegisterSlotInSlowPath(); 214 size_t orig_offset = stack_offset; 215 216 const uint32_t core_spills = codegen->GetSlowPathSpills(locations, /* core_registers */ true); 217 for (uint32_t i : LowToHighBits(core_spills)) { 218 DCHECK_LT(stack_offset, codegen->GetFrameSize() - codegen->FrameEntrySpillSize()); 219 DCHECK_LT(i, kMaximumNumberOfExpectedRegisters); 220 stack_offset += kArmWordSize; 221 } 222 223 int reg_num = POPCOUNT(core_spills); 224 if (reg_num != 0) { 225 if (reg_num > kRegListThreshold) { 226 __ LoadList(RegList(core_spills), orig_offset); 227 } else { 228 stack_offset = orig_offset; 229 for (uint32_t i : LowToHighBits(core_spills)) { 230 stack_offset += codegen->RestoreCoreRegister(stack_offset, i); 231 } 232 } 233 } 234 235 uint32_t fp_spills = codegen->GetSlowPathSpills(locations, /* core_registers */ false); 236 while (fp_spills != 0u) { 237 uint32_t begin = CTZ(fp_spills); 238 uint32_t tmp = fp_spills + (1u << begin); 239 fp_spills &= tmp; // Clear the contiguous range of 1s. 240 uint32_t end = (tmp == 0u) ? 32u : CTZ(tmp); // CTZ(0) is undefined. 241 stack_offset = RestoreContiguousSRegisterList(begin, end - 1, codegen, stack_offset); 242 } 243 DCHECK_LE(stack_offset, codegen->GetFrameSize() - codegen->FrameEntrySpillSize()); 244} 245 246class NullCheckSlowPathARM : public SlowPathCodeARM { 247 public: 248 explicit NullCheckSlowPathARM(HNullCheck* instruction) : SlowPathCodeARM(instruction) {} 249 250 void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { 251 CodeGeneratorARM* arm_codegen = down_cast<CodeGeneratorARM*>(codegen); 252 __ Bind(GetEntryLabel()); 253 if (instruction_->CanThrowIntoCatchBlock()) { 254 // Live registers will be restored in the catch block if caught. 255 SaveLiveRegisters(codegen, instruction_->GetLocations()); 256 } 257 arm_codegen->InvokeRuntime(kQuickThrowNullPointer, 258 instruction_, 259 instruction_->GetDexPc(), 260 this); 261 CheckEntrypointTypes<kQuickThrowNullPointer, void, void>(); 262 } 263 264 bool IsFatal() const OVERRIDE { return true; } 265 266 const char* GetDescription() const OVERRIDE { return "NullCheckSlowPathARM"; } 267 268 private: 269 DISALLOW_COPY_AND_ASSIGN(NullCheckSlowPathARM); 270}; 271 272class DivZeroCheckSlowPathARM : public SlowPathCodeARM { 273 public: 274 explicit DivZeroCheckSlowPathARM(HDivZeroCheck* instruction) : SlowPathCodeARM(instruction) {} 275 276 void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { 277 CodeGeneratorARM* arm_codegen = down_cast<CodeGeneratorARM*>(codegen); 278 __ Bind(GetEntryLabel()); 279 arm_codegen->InvokeRuntime(kQuickThrowDivZero, instruction_, instruction_->GetDexPc(), this); 280 CheckEntrypointTypes<kQuickThrowDivZero, void, void>(); 281 } 282 283 bool IsFatal() const OVERRIDE { return true; } 284 285 const char* GetDescription() const OVERRIDE { return "DivZeroCheckSlowPathARM"; } 286 287 private: 288 DISALLOW_COPY_AND_ASSIGN(DivZeroCheckSlowPathARM); 289}; 290 291class SuspendCheckSlowPathARM : public SlowPathCodeARM { 292 public: 293 SuspendCheckSlowPathARM(HSuspendCheck* instruction, HBasicBlock* successor) 294 : SlowPathCodeARM(instruction), successor_(successor) {} 295 296 void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { 297 CodeGeneratorARM* arm_codegen = down_cast<CodeGeneratorARM*>(codegen); 298 __ Bind(GetEntryLabel()); 299 arm_codegen->InvokeRuntime(kQuickTestSuspend, instruction_, instruction_->GetDexPc(), this); 300 CheckEntrypointTypes<kQuickTestSuspend, void, void>(); 301 if (successor_ == nullptr) { 302 __ b(GetReturnLabel()); 303 } else { 304 __ b(arm_codegen->GetLabelOf(successor_)); 305 } 306 } 307 308 Label* GetReturnLabel() { 309 DCHECK(successor_ == nullptr); 310 return &return_label_; 311 } 312 313 HBasicBlock* GetSuccessor() const { 314 return successor_; 315 } 316 317 const char* GetDescription() const OVERRIDE { return "SuspendCheckSlowPathARM"; } 318 319 private: 320 // If not null, the block to branch to after the suspend check. 321 HBasicBlock* const successor_; 322 323 // If `successor_` is null, the label to branch to after the suspend check. 324 Label return_label_; 325 326 DISALLOW_COPY_AND_ASSIGN(SuspendCheckSlowPathARM); 327}; 328 329class BoundsCheckSlowPathARM : public SlowPathCodeARM { 330 public: 331 explicit BoundsCheckSlowPathARM(HBoundsCheck* instruction) 332 : SlowPathCodeARM(instruction) {} 333 334 void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { 335 CodeGeneratorARM* arm_codegen = down_cast<CodeGeneratorARM*>(codegen); 336 LocationSummary* locations = instruction_->GetLocations(); 337 338 __ Bind(GetEntryLabel()); 339 if (instruction_->CanThrowIntoCatchBlock()) { 340 // Live registers will be restored in the catch block if caught. 341 SaveLiveRegisters(codegen, instruction_->GetLocations()); 342 } 343 // We're moving two locations to locations that could overlap, so we need a parallel 344 // move resolver. 345 InvokeRuntimeCallingConvention calling_convention; 346 codegen->EmitParallelMoves( 347 locations->InAt(0), 348 Location::RegisterLocation(calling_convention.GetRegisterAt(0)), 349 Primitive::kPrimInt, 350 locations->InAt(1), 351 Location::RegisterLocation(calling_convention.GetRegisterAt(1)), 352 Primitive::kPrimInt); 353 QuickEntrypointEnum entrypoint = instruction_->AsBoundsCheck()->IsStringCharAt() 354 ? kQuickThrowStringBounds 355 : kQuickThrowArrayBounds; 356 arm_codegen->InvokeRuntime(entrypoint, instruction_, instruction_->GetDexPc(), this); 357 CheckEntrypointTypes<kQuickThrowStringBounds, void, int32_t, int32_t>(); 358 CheckEntrypointTypes<kQuickThrowArrayBounds, void, int32_t, int32_t>(); 359 } 360 361 bool IsFatal() const OVERRIDE { return true; } 362 363 const char* GetDescription() const OVERRIDE { return "BoundsCheckSlowPathARM"; } 364 365 private: 366 DISALLOW_COPY_AND_ASSIGN(BoundsCheckSlowPathARM); 367}; 368 369class LoadClassSlowPathARM : public SlowPathCodeARM { 370 public: 371 LoadClassSlowPathARM(HLoadClass* cls, HInstruction* at, uint32_t dex_pc, bool do_clinit) 372 : SlowPathCodeARM(at), cls_(cls), dex_pc_(dex_pc), do_clinit_(do_clinit) { 373 DCHECK(at->IsLoadClass() || at->IsClinitCheck()); 374 } 375 376 void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { 377 LocationSummary* locations = instruction_->GetLocations(); 378 Location out = locations->Out(); 379 constexpr bool call_saves_everything_except_r0 = (!kUseReadBarrier || kUseBakerReadBarrier); 380 381 CodeGeneratorARM* arm_codegen = down_cast<CodeGeneratorARM*>(codegen); 382 __ Bind(GetEntryLabel()); 383 SaveLiveRegisters(codegen, locations); 384 385 InvokeRuntimeCallingConvention calling_convention; 386 // For HLoadClass/kBssEntry/kSaveEverything, make sure we preserve the address of the entry. 387 DCHECK_EQ(instruction_->IsLoadClass(), cls_ == instruction_); 388 bool is_load_class_bss_entry = 389 (cls_ == instruction_) && (cls_->GetLoadKind() == HLoadClass::LoadKind::kBssEntry); 390 Register entry_address = kNoRegister; 391 if (is_load_class_bss_entry && call_saves_everything_except_r0) { 392 Register temp = locations->GetTemp(0).AsRegister<Register>(); 393 // In the unlucky case that the `temp` is R0, we preserve the address in `out` across 394 // the kSaveEverything call. 395 bool temp_is_r0 = (temp == calling_convention.GetRegisterAt(0)); 396 entry_address = temp_is_r0 ? out.AsRegister<Register>() : temp; 397 DCHECK_NE(entry_address, calling_convention.GetRegisterAt(0)); 398 if (temp_is_r0) { 399 __ mov(entry_address, ShifterOperand(temp)); 400 } 401 } 402 dex::TypeIndex type_index = cls_->GetTypeIndex(); 403 __ LoadImmediate(calling_convention.GetRegisterAt(0), type_index.index_); 404 QuickEntrypointEnum entrypoint = do_clinit_ ? kQuickInitializeStaticStorage 405 : kQuickInitializeType; 406 arm_codegen->InvokeRuntime(entrypoint, instruction_, dex_pc_, this); 407 if (do_clinit_) { 408 CheckEntrypointTypes<kQuickInitializeStaticStorage, void*, uint32_t>(); 409 } else { 410 CheckEntrypointTypes<kQuickInitializeType, void*, uint32_t>(); 411 } 412 413 // For HLoadClass/kBssEntry, store the resolved Class to the BSS entry. 414 if (is_load_class_bss_entry) { 415 if (call_saves_everything_except_r0) { 416 // The class entry address was preserved in `entry_address` thanks to kSaveEverything. 417 __ str(R0, Address(entry_address)); 418 } else { 419 // For non-Baker read barrier, we need to re-calculate the address of the string entry. 420 Register temp = IP; 421 CodeGeneratorARM::PcRelativePatchInfo* labels = 422 arm_codegen->NewTypeBssEntryPatch(cls_->GetDexFile(), type_index); 423 __ BindTrackedLabel(&labels->movw_label); 424 __ movw(temp, /* placeholder */ 0u); 425 __ BindTrackedLabel(&labels->movt_label); 426 __ movt(temp, /* placeholder */ 0u); 427 __ BindTrackedLabel(&labels->add_pc_label); 428 __ add(temp, temp, ShifterOperand(PC)); 429 __ str(R0, Address(temp)); 430 } 431 } 432 // Move the class to the desired location. 433 if (out.IsValid()) { 434 DCHECK(out.IsRegister() && !locations->GetLiveRegisters()->ContainsCoreRegister(out.reg())); 435 arm_codegen->Move32(locations->Out(), Location::RegisterLocation(R0)); 436 } 437 RestoreLiveRegisters(codegen, locations); 438 __ b(GetExitLabel()); 439 } 440 441 const char* GetDescription() const OVERRIDE { return "LoadClassSlowPathARM"; } 442 443 private: 444 // The class this slow path will load. 445 HLoadClass* const cls_; 446 447 // The dex PC of `at_`. 448 const uint32_t dex_pc_; 449 450 // Whether to initialize the class. 451 const bool do_clinit_; 452 453 DISALLOW_COPY_AND_ASSIGN(LoadClassSlowPathARM); 454}; 455 456class LoadStringSlowPathARM : public SlowPathCodeARM { 457 public: 458 explicit LoadStringSlowPathARM(HLoadString* instruction) : SlowPathCodeARM(instruction) {} 459 460 void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { 461 DCHECK(instruction_->IsLoadString()); 462 DCHECK_EQ(instruction_->AsLoadString()->GetLoadKind(), HLoadString::LoadKind::kBssEntry); 463 LocationSummary* locations = instruction_->GetLocations(); 464 DCHECK(!locations->GetLiveRegisters()->ContainsCoreRegister(locations->Out().reg())); 465 HLoadString* load = instruction_->AsLoadString(); 466 const dex::StringIndex string_index = load->GetStringIndex(); 467 Register out = locations->Out().AsRegister<Register>(); 468 constexpr bool call_saves_everything_except_r0 = (!kUseReadBarrier || kUseBakerReadBarrier); 469 470 CodeGeneratorARM* arm_codegen = down_cast<CodeGeneratorARM*>(codegen); 471 __ Bind(GetEntryLabel()); 472 SaveLiveRegisters(codegen, locations); 473 474 InvokeRuntimeCallingConvention calling_convention; 475 // In the unlucky case that the `temp` is R0, we preserve the address in `out` across 476 // the kSaveEverything call. 477 Register entry_address = kNoRegister; 478 if (call_saves_everything_except_r0) { 479 Register temp = locations->GetTemp(0).AsRegister<Register>(); 480 bool temp_is_r0 = (temp == calling_convention.GetRegisterAt(0)); 481 entry_address = temp_is_r0 ? out : temp; 482 DCHECK_NE(entry_address, calling_convention.GetRegisterAt(0)); 483 if (temp_is_r0) { 484 __ mov(entry_address, ShifterOperand(temp)); 485 } 486 } 487 488 __ LoadImmediate(calling_convention.GetRegisterAt(0), string_index.index_); 489 arm_codegen->InvokeRuntime(kQuickResolveString, instruction_, instruction_->GetDexPc(), this); 490 CheckEntrypointTypes<kQuickResolveString, void*, uint32_t>(); 491 492 // Store the resolved String to the .bss entry. 493 if (call_saves_everything_except_r0) { 494 // The string entry address was preserved in `entry_address` thanks to kSaveEverything. 495 __ str(R0, Address(entry_address)); 496 } else { 497 // For non-Baker read barrier, we need to re-calculate the address of the string entry. 498 Register temp = IP; 499 CodeGeneratorARM::PcRelativePatchInfo* labels = 500 arm_codegen->NewPcRelativeStringPatch(load->GetDexFile(), string_index); 501 __ BindTrackedLabel(&labels->movw_label); 502 __ movw(temp, /* placeholder */ 0u); 503 __ BindTrackedLabel(&labels->movt_label); 504 __ movt(temp, /* placeholder */ 0u); 505 __ BindTrackedLabel(&labels->add_pc_label); 506 __ add(temp, temp, ShifterOperand(PC)); 507 __ str(R0, Address(temp)); 508 } 509 510 arm_codegen->Move32(locations->Out(), Location::RegisterLocation(R0)); 511 RestoreLiveRegisters(codegen, locations); 512 513 __ b(GetExitLabel()); 514 } 515 516 const char* GetDescription() const OVERRIDE { return "LoadStringSlowPathARM"; } 517 518 private: 519 DISALLOW_COPY_AND_ASSIGN(LoadStringSlowPathARM); 520}; 521 522class TypeCheckSlowPathARM : public SlowPathCodeARM { 523 public: 524 TypeCheckSlowPathARM(HInstruction* instruction, bool is_fatal) 525 : SlowPathCodeARM(instruction), is_fatal_(is_fatal) {} 526 527 void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { 528 LocationSummary* locations = instruction_->GetLocations(); 529 DCHECK(instruction_->IsCheckCast() 530 || !locations->GetLiveRegisters()->ContainsCoreRegister(locations->Out().reg())); 531 532 CodeGeneratorARM* arm_codegen = down_cast<CodeGeneratorARM*>(codegen); 533 __ Bind(GetEntryLabel()); 534 535 if (!is_fatal_) { 536 SaveLiveRegisters(codegen, locations); 537 } 538 539 // We're moving two locations to locations that could overlap, so we need a parallel 540 // move resolver. 541 InvokeRuntimeCallingConvention calling_convention; 542 codegen->EmitParallelMoves(locations->InAt(0), 543 Location::RegisterLocation(calling_convention.GetRegisterAt(0)), 544 Primitive::kPrimNot, 545 locations->InAt(1), 546 Location::RegisterLocation(calling_convention.GetRegisterAt(1)), 547 Primitive::kPrimNot); 548 if (instruction_->IsInstanceOf()) { 549 arm_codegen->InvokeRuntime(kQuickInstanceofNonTrivial, 550 instruction_, 551 instruction_->GetDexPc(), 552 this); 553 CheckEntrypointTypes<kQuickInstanceofNonTrivial, size_t, mirror::Object*, mirror::Class*>(); 554 arm_codegen->Move32(locations->Out(), Location::RegisterLocation(R0)); 555 } else { 556 DCHECK(instruction_->IsCheckCast()); 557 arm_codegen->InvokeRuntime(kQuickCheckInstanceOf, 558 instruction_, 559 instruction_->GetDexPc(), 560 this); 561 CheckEntrypointTypes<kQuickCheckInstanceOf, void, mirror::Object*, mirror::Class*>(); 562 } 563 564 if (!is_fatal_) { 565 RestoreLiveRegisters(codegen, locations); 566 __ b(GetExitLabel()); 567 } 568 } 569 570 const char* GetDescription() const OVERRIDE { return "TypeCheckSlowPathARM"; } 571 572 bool IsFatal() const OVERRIDE { return is_fatal_; } 573 574 private: 575 const bool is_fatal_; 576 577 DISALLOW_COPY_AND_ASSIGN(TypeCheckSlowPathARM); 578}; 579 580class DeoptimizationSlowPathARM : public SlowPathCodeARM { 581 public: 582 explicit DeoptimizationSlowPathARM(HDeoptimize* instruction) 583 : SlowPathCodeARM(instruction) {} 584 585 void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { 586 CodeGeneratorARM* arm_codegen = down_cast<CodeGeneratorARM*>(codegen); 587 __ Bind(GetEntryLabel()); 588 arm_codegen->InvokeRuntime(kQuickDeoptimize, instruction_, instruction_->GetDexPc(), this); 589 CheckEntrypointTypes<kQuickDeoptimize, void, void>(); 590 } 591 592 const char* GetDescription() const OVERRIDE { return "DeoptimizationSlowPathARM"; } 593 594 private: 595 DISALLOW_COPY_AND_ASSIGN(DeoptimizationSlowPathARM); 596}; 597 598class ArraySetSlowPathARM : public SlowPathCodeARM { 599 public: 600 explicit ArraySetSlowPathARM(HInstruction* instruction) : SlowPathCodeARM(instruction) {} 601 602 void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { 603 LocationSummary* locations = instruction_->GetLocations(); 604 __ Bind(GetEntryLabel()); 605 SaveLiveRegisters(codegen, locations); 606 607 InvokeRuntimeCallingConvention calling_convention; 608 HParallelMove parallel_move(codegen->GetGraph()->GetArena()); 609 parallel_move.AddMove( 610 locations->InAt(0), 611 Location::RegisterLocation(calling_convention.GetRegisterAt(0)), 612 Primitive::kPrimNot, 613 nullptr); 614 parallel_move.AddMove( 615 locations->InAt(1), 616 Location::RegisterLocation(calling_convention.GetRegisterAt(1)), 617 Primitive::kPrimInt, 618 nullptr); 619 parallel_move.AddMove( 620 locations->InAt(2), 621 Location::RegisterLocation(calling_convention.GetRegisterAt(2)), 622 Primitive::kPrimNot, 623 nullptr); 624 codegen->GetMoveResolver()->EmitNativeCode(¶llel_move); 625 626 CodeGeneratorARM* arm_codegen = down_cast<CodeGeneratorARM*>(codegen); 627 arm_codegen->InvokeRuntime(kQuickAputObject, instruction_, instruction_->GetDexPc(), this); 628 CheckEntrypointTypes<kQuickAputObject, void, mirror::Array*, int32_t, mirror::Object*>(); 629 RestoreLiveRegisters(codegen, locations); 630 __ b(GetExitLabel()); 631 } 632 633 const char* GetDescription() const OVERRIDE { return "ArraySetSlowPathARM"; } 634 635 private: 636 DISALLOW_COPY_AND_ASSIGN(ArraySetSlowPathARM); 637}; 638 639// Abstract base class for read barrier slow paths marking a reference 640// `ref`. 641// 642// Argument `entrypoint` must be a register location holding the read 643// barrier marking runtime entry point to be invoked. 644class ReadBarrierMarkSlowPathBaseARM : public SlowPathCodeARM { 645 protected: 646 ReadBarrierMarkSlowPathBaseARM(HInstruction* instruction, Location ref, Location entrypoint) 647 : SlowPathCodeARM(instruction), ref_(ref), entrypoint_(entrypoint) { 648 DCHECK(kEmitCompilerReadBarrier); 649 } 650 651 const char* GetDescription() const OVERRIDE { return "ReadBarrierMarkSlowPathBaseARM"; } 652 653 // Generate assembly code calling the read barrier marking runtime 654 // entry point (ReadBarrierMarkRegX). 655 void GenerateReadBarrierMarkRuntimeCall(CodeGenerator* codegen) { 656 Register ref_reg = ref_.AsRegister<Register>(); 657 658 // No need to save live registers; it's taken care of by the 659 // entrypoint. Also, there is no need to update the stack mask, 660 // as this runtime call will not trigger a garbage collection. 661 CodeGeneratorARM* arm_codegen = down_cast<CodeGeneratorARM*>(codegen); 662 DCHECK_NE(ref_reg, SP); 663 DCHECK_NE(ref_reg, LR); 664 DCHECK_NE(ref_reg, PC); 665 // IP is used internally by the ReadBarrierMarkRegX entry point 666 // as a temporary, it cannot be the entry point's input/output. 667 DCHECK_NE(ref_reg, IP); 668 DCHECK(0 <= ref_reg && ref_reg < kNumberOfCoreRegisters) << ref_reg; 669 // "Compact" slow path, saving two moves. 670 // 671 // Instead of using the standard runtime calling convention (input 672 // and output in R0): 673 // 674 // R0 <- ref 675 // R0 <- ReadBarrierMark(R0) 676 // ref <- R0 677 // 678 // we just use rX (the register containing `ref`) as input and output 679 // of a dedicated entrypoint: 680 // 681 // rX <- ReadBarrierMarkRegX(rX) 682 // 683 if (entrypoint_.IsValid()) { 684 arm_codegen->ValidateInvokeRuntimeWithoutRecordingPcInfo(instruction_, this); 685 __ blx(entrypoint_.AsRegister<Register>()); 686 } else { 687 // Entrypoint is not already loaded, load from the thread. 688 int32_t entry_point_offset = 689 CodeGenerator::GetReadBarrierMarkEntryPointsOffset<kArmPointerSize>(ref_reg); 690 // This runtime call does not require a stack map. 691 arm_codegen->InvokeRuntimeWithoutRecordingPcInfo(entry_point_offset, instruction_, this); 692 } 693 } 694 695 // The location (register) of the marked object reference. 696 const Location ref_; 697 698 // The location of the entrypoint if it is already loaded. 699 const Location entrypoint_; 700 701 private: 702 DISALLOW_COPY_AND_ASSIGN(ReadBarrierMarkSlowPathBaseARM); 703}; 704 705// Slow path marking an object reference `ref` during a read 706// barrier. The field `obj.field` in the object `obj` holding this 707// reference does not get updated by this slow path after marking. 708// 709// This means that after the execution of this slow path, `ref` will 710// always be up-to-date, but `obj.field` may not; i.e., after the 711// flip, `ref` will be a to-space reference, but `obj.field` will 712// probably still be a from-space reference (unless it gets updated by 713// another thread, or if another thread installed another object 714// reference (different from `ref`) in `obj.field`). 715// 716// If `entrypoint` is a valid location it is assumed to already be 717// holding the entrypoint. The case where the entrypoint is passed in 718// is when the decision to mark is based on whether the GC is marking. 719class ReadBarrierMarkSlowPathARM : public ReadBarrierMarkSlowPathBaseARM { 720 public: 721 ReadBarrierMarkSlowPathARM(HInstruction* instruction, 722 Location ref, 723 Location entrypoint = Location::NoLocation()) 724 : ReadBarrierMarkSlowPathBaseARM(instruction, ref, entrypoint) { 725 DCHECK(kEmitCompilerReadBarrier); 726 } 727 728 const char* GetDescription() const OVERRIDE { return "ReadBarrierMarkSlowPathARM"; } 729 730 void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { 731 LocationSummary* locations = instruction_->GetLocations(); 732 DCHECK(locations->CanCall()); 733 if (kIsDebugBuild) { 734 Register ref_reg = ref_.AsRegister<Register>(); 735 DCHECK(!locations->GetLiveRegisters()->ContainsCoreRegister(ref_reg)) << ref_reg; 736 } 737 DCHECK(instruction_->IsLoadClass() || instruction_->IsLoadString()) 738 << "Unexpected instruction in read barrier marking slow path: " 739 << instruction_->DebugName(); 740 741 __ Bind(GetEntryLabel()); 742 GenerateReadBarrierMarkRuntimeCall(codegen); 743 __ b(GetExitLabel()); 744 } 745 746 private: 747 DISALLOW_COPY_AND_ASSIGN(ReadBarrierMarkSlowPathARM); 748}; 749 750// Slow path loading `obj`'s lock word, loading a reference from 751// object `*(obj + offset + (index << scale_factor))` into `ref`, and 752// marking `ref` if `obj` is gray according to the lock word (Baker 753// read barrier). The field `obj.field` in the object `obj` holding 754// this reference does not get updated by this slow path after marking 755// (see LoadReferenceWithBakerReadBarrierAndUpdateFieldSlowPathARM 756// below for that). 757// 758// This means that after the execution of this slow path, `ref` will 759// always be up-to-date, but `obj.field` may not; i.e., after the 760// flip, `ref` will be a to-space reference, but `obj.field` will 761// probably still be a from-space reference (unless it gets updated by 762// another thread, or if another thread installed another object 763// reference (different from `ref`) in `obj.field`). 764// 765// Argument `entrypoint` must be a register location holding the read 766// barrier marking runtime entry point to be invoked. 767class LoadReferenceWithBakerReadBarrierSlowPathARM : public ReadBarrierMarkSlowPathBaseARM { 768 public: 769 LoadReferenceWithBakerReadBarrierSlowPathARM(HInstruction* instruction, 770 Location ref, 771 Register obj, 772 uint32_t offset, 773 Location index, 774 ScaleFactor scale_factor, 775 bool needs_null_check, 776 Register temp, 777 Location entrypoint) 778 : ReadBarrierMarkSlowPathBaseARM(instruction, ref, entrypoint), 779 obj_(obj), 780 offset_(offset), 781 index_(index), 782 scale_factor_(scale_factor), 783 needs_null_check_(needs_null_check), 784 temp_(temp) { 785 DCHECK(kEmitCompilerReadBarrier); 786 DCHECK(kUseBakerReadBarrier); 787 } 788 789 const char* GetDescription() const OVERRIDE { 790 return "LoadReferenceWithBakerReadBarrierSlowPathARM"; 791 } 792 793 void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { 794 LocationSummary* locations = instruction_->GetLocations(); 795 Register ref_reg = ref_.AsRegister<Register>(); 796 DCHECK(locations->CanCall()); 797 DCHECK(!locations->GetLiveRegisters()->ContainsCoreRegister(ref_reg)) << ref_reg; 798 DCHECK_NE(ref_reg, temp_); 799 DCHECK(instruction_->IsInstanceFieldGet() || 800 instruction_->IsStaticFieldGet() || 801 instruction_->IsArrayGet() || 802 instruction_->IsArraySet() || 803 instruction_->IsInstanceOf() || 804 instruction_->IsCheckCast() || 805 (instruction_->IsInvokeVirtual() && instruction_->GetLocations()->Intrinsified()) || 806 (instruction_->IsInvokeStaticOrDirect() && instruction_->GetLocations()->Intrinsified())) 807 << "Unexpected instruction in read barrier marking slow path: " 808 << instruction_->DebugName(); 809 // The read barrier instrumentation of object ArrayGet 810 // instructions does not support the HIntermediateAddress 811 // instruction. 812 DCHECK(!(instruction_->IsArrayGet() && 813 instruction_->AsArrayGet()->GetArray()->IsIntermediateAddress())); 814 815 __ Bind(GetEntryLabel()); 816 817 // When using MaybeGenerateReadBarrierSlow, the read barrier call is 818 // inserted after the original load. However, in fast path based 819 // Baker's read barriers, we need to perform the load of 820 // mirror::Object::monitor_ *before* the original reference load. 821 // This load-load ordering is required by the read barrier. 822 // The fast path/slow path (for Baker's algorithm) should look like: 823 // 824 // uint32_t rb_state = Lockword(obj->monitor_).ReadBarrierState(); 825 // lfence; // Load fence or artificial data dependency to prevent load-load reordering 826 // HeapReference<mirror::Object> ref = *src; // Original reference load. 827 // bool is_gray = (rb_state == ReadBarrier::GrayState()); 828 // if (is_gray) { 829 // ref = entrypoint(ref); // ref = ReadBarrier::Mark(ref); // Runtime entry point call. 830 // } 831 // 832 // Note: the original implementation in ReadBarrier::Barrier is 833 // slightly more complex as it performs additional checks that we do 834 // not do here for performance reasons. 835 836 // /* int32_t */ monitor = obj->monitor_ 837 uint32_t monitor_offset = mirror::Object::MonitorOffset().Int32Value(); 838 __ LoadFromOffset(kLoadWord, temp_, obj_, monitor_offset); 839 if (needs_null_check_) { 840 codegen->MaybeRecordImplicitNullCheck(instruction_); 841 } 842 // /* LockWord */ lock_word = LockWord(monitor) 843 static_assert(sizeof(LockWord) == sizeof(int32_t), 844 "art::LockWord and int32_t have different sizes."); 845 846 // Introduce a dependency on the lock_word including the rb_state, 847 // which shall prevent load-load reordering without using 848 // a memory barrier (which would be more expensive). 849 // `obj` is unchanged by this operation, but its value now depends 850 // on `temp`. 851 __ add(obj_, obj_, ShifterOperand(temp_, LSR, 32)); 852 853 // The actual reference load. 854 // A possible implicit null check has already been handled above. 855 CodeGeneratorARM* arm_codegen = down_cast<CodeGeneratorARM*>(codegen); 856 arm_codegen->GenerateRawReferenceLoad( 857 instruction_, ref_, obj_, offset_, index_, scale_factor_, /* needs_null_check */ false); 858 859 // Mark the object `ref` when `obj` is gray. 860 // 861 // if (rb_state == ReadBarrier::GrayState()) 862 // ref = ReadBarrier::Mark(ref); 863 // 864 // Given the numeric representation, it's enough to check the low bit of the 865 // rb_state. We do that by shifting the bit out of the lock word with LSRS 866 // which can be a 16-bit instruction unlike the TST immediate. 867 static_assert(ReadBarrier::WhiteState() == 0, "Expecting white to have value 0"); 868 static_assert(ReadBarrier::GrayState() == 1, "Expecting gray to have value 1"); 869 __ Lsrs(temp_, temp_, LockWord::kReadBarrierStateShift + 1); 870 __ b(GetExitLabel(), CC); // Carry flag is the last bit shifted out by LSRS. 871 GenerateReadBarrierMarkRuntimeCall(codegen); 872 873 __ b(GetExitLabel()); 874 } 875 876 private: 877 // The register containing the object holding the marked object reference field. 878 Register obj_; 879 // The offset, index and scale factor to access the reference in `obj_`. 880 uint32_t offset_; 881 Location index_; 882 ScaleFactor scale_factor_; 883 // Is a null check required? 884 bool needs_null_check_; 885 // A temporary register used to hold the lock word of `obj_`. 886 Register temp_; 887 888 DISALLOW_COPY_AND_ASSIGN(LoadReferenceWithBakerReadBarrierSlowPathARM); 889}; 890 891// Slow path loading `obj`'s lock word, loading a reference from 892// object `*(obj + offset + (index << scale_factor))` into `ref`, and 893// marking `ref` if `obj` is gray according to the lock word (Baker 894// read barrier). If needed, this slow path also atomically updates 895// the field `obj.field` in the object `obj` holding this reference 896// after marking (contrary to 897// LoadReferenceWithBakerReadBarrierSlowPathARM above, which never 898// tries to update `obj.field`). 899// 900// This means that after the execution of this slow path, both `ref` 901// and `obj.field` will be up-to-date; i.e., after the flip, both will 902// hold the same to-space reference (unless another thread installed 903// another object reference (different from `ref`) in `obj.field`). 904// 905// Argument `entrypoint` must be a register location holding the read 906// barrier marking runtime entry point to be invoked. 907class LoadReferenceWithBakerReadBarrierAndUpdateFieldSlowPathARM 908 : public ReadBarrierMarkSlowPathBaseARM { 909 public: 910 LoadReferenceWithBakerReadBarrierAndUpdateFieldSlowPathARM(HInstruction* instruction, 911 Location ref, 912 Register obj, 913 uint32_t offset, 914 Location index, 915 ScaleFactor scale_factor, 916 bool needs_null_check, 917 Register temp1, 918 Register temp2, 919 Location entrypoint) 920 : ReadBarrierMarkSlowPathBaseARM(instruction, ref, entrypoint), 921 obj_(obj), 922 offset_(offset), 923 index_(index), 924 scale_factor_(scale_factor), 925 needs_null_check_(needs_null_check), 926 temp1_(temp1), 927 temp2_(temp2) { 928 DCHECK(kEmitCompilerReadBarrier); 929 DCHECK(kUseBakerReadBarrier); 930 } 931 932 const char* GetDescription() const OVERRIDE { 933 return "LoadReferenceWithBakerReadBarrierAndUpdateFieldSlowPathARM"; 934 } 935 936 void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { 937 LocationSummary* locations = instruction_->GetLocations(); 938 Register ref_reg = ref_.AsRegister<Register>(); 939 DCHECK(locations->CanCall()); 940 DCHECK(!locations->GetLiveRegisters()->ContainsCoreRegister(ref_reg)) << ref_reg; 941 DCHECK_NE(ref_reg, temp1_); 942 943 // This slow path is only used by the UnsafeCASObject intrinsic at the moment. 944 DCHECK((instruction_->IsInvokeVirtual() && instruction_->GetLocations()->Intrinsified())) 945 << "Unexpected instruction in read barrier marking and field updating slow path: " 946 << instruction_->DebugName(); 947 DCHECK(instruction_->GetLocations()->Intrinsified()); 948 DCHECK_EQ(instruction_->AsInvoke()->GetIntrinsic(), Intrinsics::kUnsafeCASObject); 949 DCHECK_EQ(offset_, 0u); 950 DCHECK_EQ(scale_factor_, ScaleFactor::TIMES_1); 951 // The location of the offset of the marked reference field within `obj_`. 952 Location field_offset = index_; 953 DCHECK(field_offset.IsRegisterPair()) << field_offset; 954 955 __ Bind(GetEntryLabel()); 956 957 // /* int32_t */ monitor = obj->monitor_ 958 uint32_t monitor_offset = mirror::Object::MonitorOffset().Int32Value(); 959 __ LoadFromOffset(kLoadWord, temp1_, obj_, monitor_offset); 960 if (needs_null_check_) { 961 codegen->MaybeRecordImplicitNullCheck(instruction_); 962 } 963 // /* LockWord */ lock_word = LockWord(monitor) 964 static_assert(sizeof(LockWord) == sizeof(int32_t), 965 "art::LockWord and int32_t have different sizes."); 966 967 // Introduce a dependency on the lock_word including the rb_state, 968 // which shall prevent load-load reordering without using 969 // a memory barrier (which would be more expensive). 970 // `obj` is unchanged by this operation, but its value now depends 971 // on `temp1`. 972 __ add(obj_, obj_, ShifterOperand(temp1_, LSR, 32)); 973 974 // The actual reference load. 975 // A possible implicit null check has already been handled above. 976 CodeGeneratorARM* arm_codegen = down_cast<CodeGeneratorARM*>(codegen); 977 arm_codegen->GenerateRawReferenceLoad( 978 instruction_, ref_, obj_, offset_, index_, scale_factor_, /* needs_null_check */ false); 979 980 // Mark the object `ref` when `obj` is gray. 981 // 982 // if (rb_state == ReadBarrier::GrayState()) 983 // ref = ReadBarrier::Mark(ref); 984 // 985 // Given the numeric representation, it's enough to check the low bit of the 986 // rb_state. We do that by shifting the bit out of the lock word with LSRS 987 // which can be a 16-bit instruction unlike the TST immediate. 988 static_assert(ReadBarrier::WhiteState() == 0, "Expecting white to have value 0"); 989 static_assert(ReadBarrier::GrayState() == 1, "Expecting gray to have value 1"); 990 __ Lsrs(temp1_, temp1_, LockWord::kReadBarrierStateShift + 1); 991 __ b(GetExitLabel(), CC); // Carry flag is the last bit shifted out by LSRS. 992 993 // Save the old value of the reference before marking it. 994 // Note that we cannot use IP to save the old reference, as IP is 995 // used internally by the ReadBarrierMarkRegX entry point, and we 996 // need the old reference after the call to that entry point. 997 DCHECK_NE(temp1_, IP); 998 __ Mov(temp1_, ref_reg); 999 1000 GenerateReadBarrierMarkRuntimeCall(codegen); 1001 1002 // If the new reference is different from the old reference, 1003 // update the field in the holder (`*(obj_ + field_offset)`). 1004 // 1005 // Note that this field could also hold a different object, if 1006 // another thread had concurrently changed it. In that case, the 1007 // LDREX/SUBS/ITNE sequence of instructions in the compare-and-set 1008 // (CAS) operation below would abort the CAS, leaving the field 1009 // as-is. 1010 __ cmp(temp1_, ShifterOperand(ref_reg)); 1011 __ b(GetExitLabel(), EQ); 1012 1013 // Update the the holder's field atomically. This may fail if 1014 // mutator updates before us, but it's OK. This is achieved 1015 // using a strong compare-and-set (CAS) operation with relaxed 1016 // memory synchronization ordering, where the expected value is 1017 // the old reference and the desired value is the new reference. 1018 1019 // Convenience aliases. 1020 Register base = obj_; 1021 // The UnsafeCASObject intrinsic uses a register pair as field 1022 // offset ("long offset"), of which only the low part contains 1023 // data. 1024 Register offset = field_offset.AsRegisterPairLow<Register>(); 1025 Register expected = temp1_; 1026 Register value = ref_reg; 1027 Register tmp_ptr = IP; // Pointer to actual memory. 1028 Register tmp = temp2_; // Value in memory. 1029 1030 __ add(tmp_ptr, base, ShifterOperand(offset)); 1031 1032 if (kPoisonHeapReferences) { 1033 __ PoisonHeapReference(expected); 1034 if (value == expected) { 1035 // Do not poison `value`, as it is the same register as 1036 // `expected`, which has just been poisoned. 1037 } else { 1038 __ PoisonHeapReference(value); 1039 } 1040 } 1041 1042 // do { 1043 // tmp = [r_ptr] - expected; 1044 // } while (tmp == 0 && failure([r_ptr] <- r_new_value)); 1045 1046 Label loop_head, exit_loop; 1047 __ Bind(&loop_head); 1048 1049 __ ldrex(tmp, tmp_ptr); 1050 1051 __ subs(tmp, tmp, ShifterOperand(expected)); 1052 1053 __ it(NE); 1054 __ clrex(NE); 1055 1056 __ b(&exit_loop, NE); 1057 1058 __ strex(tmp, value, tmp_ptr); 1059 __ cmp(tmp, ShifterOperand(1)); 1060 __ b(&loop_head, EQ); 1061 1062 __ Bind(&exit_loop); 1063 1064 if (kPoisonHeapReferences) { 1065 __ UnpoisonHeapReference(expected); 1066 if (value == expected) { 1067 // Do not unpoison `value`, as it is the same register as 1068 // `expected`, which has just been unpoisoned. 1069 } else { 1070 __ UnpoisonHeapReference(value); 1071 } 1072 } 1073 1074 __ b(GetExitLabel()); 1075 } 1076 1077 private: 1078 // The register containing the object holding the marked object reference field. 1079 const Register obj_; 1080 // The offset, index and scale factor to access the reference in `obj_`. 1081 uint32_t offset_; 1082 Location index_; 1083 ScaleFactor scale_factor_; 1084 // Is a null check required? 1085 bool needs_null_check_; 1086 // A temporary register used to hold the lock word of `obj_`; and 1087 // also to hold the original reference value, when the reference is 1088 // marked. 1089 const Register temp1_; 1090 // A temporary register used in the implementation of the CAS, to 1091 // update the object's reference field. 1092 const Register temp2_; 1093 1094 DISALLOW_COPY_AND_ASSIGN(LoadReferenceWithBakerReadBarrierAndUpdateFieldSlowPathARM); 1095}; 1096 1097// Slow path generating a read barrier for a heap reference. 1098class ReadBarrierForHeapReferenceSlowPathARM : public SlowPathCodeARM { 1099 public: 1100 ReadBarrierForHeapReferenceSlowPathARM(HInstruction* instruction, 1101 Location out, 1102 Location ref, 1103 Location obj, 1104 uint32_t offset, 1105 Location index) 1106 : SlowPathCodeARM(instruction), 1107 out_(out), 1108 ref_(ref), 1109 obj_(obj), 1110 offset_(offset), 1111 index_(index) { 1112 DCHECK(kEmitCompilerReadBarrier); 1113 // If `obj` is equal to `out` or `ref`, it means the initial object 1114 // has been overwritten by (or after) the heap object reference load 1115 // to be instrumented, e.g.: 1116 // 1117 // __ LoadFromOffset(kLoadWord, out, out, offset); 1118 // codegen_->GenerateReadBarrierSlow(instruction, out_loc, out_loc, out_loc, offset); 1119 // 1120 // In that case, we have lost the information about the original 1121 // object, and the emitted read barrier cannot work properly. 1122 DCHECK(!obj.Equals(out)) << "obj=" << obj << " out=" << out; 1123 DCHECK(!obj.Equals(ref)) << "obj=" << obj << " ref=" << ref; 1124 } 1125 1126 void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { 1127 CodeGeneratorARM* arm_codegen = down_cast<CodeGeneratorARM*>(codegen); 1128 LocationSummary* locations = instruction_->GetLocations(); 1129 Register reg_out = out_.AsRegister<Register>(); 1130 DCHECK(locations->CanCall()); 1131 DCHECK(!locations->GetLiveRegisters()->ContainsCoreRegister(reg_out)); 1132 DCHECK(instruction_->IsInstanceFieldGet() || 1133 instruction_->IsStaticFieldGet() || 1134 instruction_->IsArrayGet() || 1135 instruction_->IsInstanceOf() || 1136 instruction_->IsCheckCast() || 1137 (instruction_->IsInvokeVirtual() && instruction_->GetLocations()->Intrinsified())) 1138 << "Unexpected instruction in read barrier for heap reference slow path: " 1139 << instruction_->DebugName(); 1140 // The read barrier instrumentation of object ArrayGet 1141 // instructions does not support the HIntermediateAddress 1142 // instruction. 1143 DCHECK(!(instruction_->IsArrayGet() && 1144 instruction_->AsArrayGet()->GetArray()->IsIntermediateAddress())); 1145 1146 __ Bind(GetEntryLabel()); 1147 SaveLiveRegisters(codegen, locations); 1148 1149 // We may have to change the index's value, but as `index_` is a 1150 // constant member (like other "inputs" of this slow path), 1151 // introduce a copy of it, `index`. 1152 Location index = index_; 1153 if (index_.IsValid()) { 1154 // Handle `index_` for HArrayGet and UnsafeGetObject/UnsafeGetObjectVolatile intrinsics. 1155 if (instruction_->IsArrayGet()) { 1156 // Compute the actual memory offset and store it in `index`. 1157 Register index_reg = index_.AsRegister<Register>(); 1158 DCHECK(locations->GetLiveRegisters()->ContainsCoreRegister(index_reg)); 1159 if (codegen->IsCoreCalleeSaveRegister(index_reg)) { 1160 // We are about to change the value of `index_reg` (see the 1161 // calls to art::arm::Thumb2Assembler::Lsl and 1162 // art::arm::Thumb2Assembler::AddConstant below), but it has 1163 // not been saved by the previous call to 1164 // art::SlowPathCode::SaveLiveRegisters, as it is a 1165 // callee-save register -- 1166 // art::SlowPathCode::SaveLiveRegisters does not consider 1167 // callee-save registers, as it has been designed with the 1168 // assumption that callee-save registers are supposed to be 1169 // handled by the called function. So, as a callee-save 1170 // register, `index_reg` _would_ eventually be saved onto 1171 // the stack, but it would be too late: we would have 1172 // changed its value earlier. Therefore, we manually save 1173 // it here into another freely available register, 1174 // `free_reg`, chosen of course among the caller-save 1175 // registers (as a callee-save `free_reg` register would 1176 // exhibit the same problem). 1177 // 1178 // Note we could have requested a temporary register from 1179 // the register allocator instead; but we prefer not to, as 1180 // this is a slow path, and we know we can find a 1181 // caller-save register that is available. 1182 Register free_reg = FindAvailableCallerSaveRegister(codegen); 1183 __ Mov(free_reg, index_reg); 1184 index_reg = free_reg; 1185 index = Location::RegisterLocation(index_reg); 1186 } else { 1187 // The initial register stored in `index_` has already been 1188 // saved in the call to art::SlowPathCode::SaveLiveRegisters 1189 // (as it is not a callee-save register), so we can freely 1190 // use it. 1191 } 1192 // Shifting the index value contained in `index_reg` by the scale 1193 // factor (2) cannot overflow in practice, as the runtime is 1194 // unable to allocate object arrays with a size larger than 1195 // 2^26 - 1 (that is, 2^28 - 4 bytes). 1196 __ Lsl(index_reg, index_reg, TIMES_4); 1197 static_assert( 1198 sizeof(mirror::HeapReference<mirror::Object>) == sizeof(int32_t), 1199 "art::mirror::HeapReference<art::mirror::Object> and int32_t have different sizes."); 1200 __ AddConstant(index_reg, index_reg, offset_); 1201 } else { 1202 // In the case of the UnsafeGetObject/UnsafeGetObjectVolatile 1203 // intrinsics, `index_` is not shifted by a scale factor of 2 1204 // (as in the case of ArrayGet), as it is actually an offset 1205 // to an object field within an object. 1206 DCHECK(instruction_->IsInvoke()) << instruction_->DebugName(); 1207 DCHECK(instruction_->GetLocations()->Intrinsified()); 1208 DCHECK((instruction_->AsInvoke()->GetIntrinsic() == Intrinsics::kUnsafeGetObject) || 1209 (instruction_->AsInvoke()->GetIntrinsic() == Intrinsics::kUnsafeGetObjectVolatile)) 1210 << instruction_->AsInvoke()->GetIntrinsic(); 1211 DCHECK_EQ(offset_, 0U); 1212 DCHECK(index_.IsRegisterPair()); 1213 // UnsafeGet's offset location is a register pair, the low 1214 // part contains the correct offset. 1215 index = index_.ToLow(); 1216 } 1217 } 1218 1219 // We're moving two or three locations to locations that could 1220 // overlap, so we need a parallel move resolver. 1221 InvokeRuntimeCallingConvention calling_convention; 1222 HParallelMove parallel_move(codegen->GetGraph()->GetArena()); 1223 parallel_move.AddMove(ref_, 1224 Location::RegisterLocation(calling_convention.GetRegisterAt(0)), 1225 Primitive::kPrimNot, 1226 nullptr); 1227 parallel_move.AddMove(obj_, 1228 Location::RegisterLocation(calling_convention.GetRegisterAt(1)), 1229 Primitive::kPrimNot, 1230 nullptr); 1231 if (index.IsValid()) { 1232 parallel_move.AddMove(index, 1233 Location::RegisterLocation(calling_convention.GetRegisterAt(2)), 1234 Primitive::kPrimInt, 1235 nullptr); 1236 codegen->GetMoveResolver()->EmitNativeCode(¶llel_move); 1237 } else { 1238 codegen->GetMoveResolver()->EmitNativeCode(¶llel_move); 1239 __ LoadImmediate(calling_convention.GetRegisterAt(2), offset_); 1240 } 1241 arm_codegen->InvokeRuntime(kQuickReadBarrierSlow, instruction_, instruction_->GetDexPc(), this); 1242 CheckEntrypointTypes< 1243 kQuickReadBarrierSlow, mirror::Object*, mirror::Object*, mirror::Object*, uint32_t>(); 1244 arm_codegen->Move32(out_, Location::RegisterLocation(R0)); 1245 1246 RestoreLiveRegisters(codegen, locations); 1247 __ b(GetExitLabel()); 1248 } 1249 1250 const char* GetDescription() const OVERRIDE { return "ReadBarrierForHeapReferenceSlowPathARM"; } 1251 1252 private: 1253 Register FindAvailableCallerSaveRegister(CodeGenerator* codegen) { 1254 size_t ref = static_cast<int>(ref_.AsRegister<Register>()); 1255 size_t obj = static_cast<int>(obj_.AsRegister<Register>()); 1256 for (size_t i = 0, e = codegen->GetNumberOfCoreRegisters(); i < e; ++i) { 1257 if (i != ref && i != obj && !codegen->IsCoreCalleeSaveRegister(i)) { 1258 return static_cast<Register>(i); 1259 } 1260 } 1261 // We shall never fail to find a free caller-save register, as 1262 // there are more than two core caller-save registers on ARM 1263 // (meaning it is possible to find one which is different from 1264 // `ref` and `obj`). 1265 DCHECK_GT(codegen->GetNumberOfCoreCallerSaveRegisters(), 2u); 1266 LOG(FATAL) << "Could not find a free caller-save register"; 1267 UNREACHABLE(); 1268 } 1269 1270 const Location out_; 1271 const Location ref_; 1272 const Location obj_; 1273 const uint32_t offset_; 1274 // An additional location containing an index to an array. 1275 // Only used for HArrayGet and the UnsafeGetObject & 1276 // UnsafeGetObjectVolatile intrinsics. 1277 const Location index_; 1278 1279 DISALLOW_COPY_AND_ASSIGN(ReadBarrierForHeapReferenceSlowPathARM); 1280}; 1281 1282// Slow path generating a read barrier for a GC root. 1283class ReadBarrierForRootSlowPathARM : public SlowPathCodeARM { 1284 public: 1285 ReadBarrierForRootSlowPathARM(HInstruction* instruction, Location out, Location root) 1286 : SlowPathCodeARM(instruction), out_(out), root_(root) { 1287 DCHECK(kEmitCompilerReadBarrier); 1288 } 1289 1290 void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { 1291 LocationSummary* locations = instruction_->GetLocations(); 1292 Register reg_out = out_.AsRegister<Register>(); 1293 DCHECK(locations->CanCall()); 1294 DCHECK(!locations->GetLiveRegisters()->ContainsCoreRegister(reg_out)); 1295 DCHECK(instruction_->IsLoadClass() || instruction_->IsLoadString()) 1296 << "Unexpected instruction in read barrier for GC root slow path: " 1297 << instruction_->DebugName(); 1298 1299 __ Bind(GetEntryLabel()); 1300 SaveLiveRegisters(codegen, locations); 1301 1302 InvokeRuntimeCallingConvention calling_convention; 1303 CodeGeneratorARM* arm_codegen = down_cast<CodeGeneratorARM*>(codegen); 1304 arm_codegen->Move32(Location::RegisterLocation(calling_convention.GetRegisterAt(0)), root_); 1305 arm_codegen->InvokeRuntime(kQuickReadBarrierForRootSlow, 1306 instruction_, 1307 instruction_->GetDexPc(), 1308 this); 1309 CheckEntrypointTypes<kQuickReadBarrierForRootSlow, mirror::Object*, GcRoot<mirror::Object>*>(); 1310 arm_codegen->Move32(out_, Location::RegisterLocation(R0)); 1311 1312 RestoreLiveRegisters(codegen, locations); 1313 __ b(GetExitLabel()); 1314 } 1315 1316 const char* GetDescription() const OVERRIDE { return "ReadBarrierForRootSlowPathARM"; } 1317 1318 private: 1319 const Location out_; 1320 const Location root_; 1321 1322 DISALLOW_COPY_AND_ASSIGN(ReadBarrierForRootSlowPathARM); 1323}; 1324 1325inline Condition ARMCondition(IfCondition cond) { 1326 switch (cond) { 1327 case kCondEQ: return EQ; 1328 case kCondNE: return NE; 1329 case kCondLT: return LT; 1330 case kCondLE: return LE; 1331 case kCondGT: return GT; 1332 case kCondGE: return GE; 1333 case kCondB: return LO; 1334 case kCondBE: return LS; 1335 case kCondA: return HI; 1336 case kCondAE: return HS; 1337 } 1338 LOG(FATAL) << "Unreachable"; 1339 UNREACHABLE(); 1340} 1341 1342// Maps signed condition to unsigned condition. 1343inline Condition ARMUnsignedCondition(IfCondition cond) { 1344 switch (cond) { 1345 case kCondEQ: return EQ; 1346 case kCondNE: return NE; 1347 // Signed to unsigned. 1348 case kCondLT: return LO; 1349 case kCondLE: return LS; 1350 case kCondGT: return HI; 1351 case kCondGE: return HS; 1352 // Unsigned remain unchanged. 1353 case kCondB: return LO; 1354 case kCondBE: return LS; 1355 case kCondA: return HI; 1356 case kCondAE: return HS; 1357 } 1358 LOG(FATAL) << "Unreachable"; 1359 UNREACHABLE(); 1360} 1361 1362inline Condition ARMFPCondition(IfCondition cond, bool gt_bias) { 1363 // The ARM condition codes can express all the necessary branches, see the 1364 // "Meaning (floating-point)" column in the table A8-1 of the ARMv7 reference manual. 1365 // There is no dex instruction or HIR that would need the missing conditions 1366 // "equal or unordered" or "not equal". 1367 switch (cond) { 1368 case kCondEQ: return EQ; 1369 case kCondNE: return NE /* unordered */; 1370 case kCondLT: return gt_bias ? CC : LT /* unordered */; 1371 case kCondLE: return gt_bias ? LS : LE /* unordered */; 1372 case kCondGT: return gt_bias ? HI /* unordered */ : GT; 1373 case kCondGE: return gt_bias ? CS /* unordered */ : GE; 1374 default: 1375 LOG(FATAL) << "UNREACHABLE"; 1376 UNREACHABLE(); 1377 } 1378} 1379 1380inline Shift ShiftFromOpKind(HDataProcWithShifterOp::OpKind op_kind) { 1381 switch (op_kind) { 1382 case HDataProcWithShifterOp::kASR: return ASR; 1383 case HDataProcWithShifterOp::kLSL: return LSL; 1384 case HDataProcWithShifterOp::kLSR: return LSR; 1385 default: 1386 LOG(FATAL) << "Unexpected op kind " << op_kind; 1387 UNREACHABLE(); 1388 } 1389} 1390 1391static void GenerateDataProcInstruction(HInstruction::InstructionKind kind, 1392 Register out, 1393 Register first, 1394 const ShifterOperand& second, 1395 CodeGeneratorARM* codegen) { 1396 if (second.IsImmediate() && second.GetImmediate() == 0) { 1397 const ShifterOperand in = kind == HInstruction::kAnd 1398 ? ShifterOperand(0) 1399 : ShifterOperand(first); 1400 1401 __ mov(out, in); 1402 } else { 1403 switch (kind) { 1404 case HInstruction::kAdd: 1405 __ add(out, first, second); 1406 break; 1407 case HInstruction::kAnd: 1408 __ and_(out, first, second); 1409 break; 1410 case HInstruction::kOr: 1411 __ orr(out, first, second); 1412 break; 1413 case HInstruction::kSub: 1414 __ sub(out, first, second); 1415 break; 1416 case HInstruction::kXor: 1417 __ eor(out, first, second); 1418 break; 1419 default: 1420 LOG(FATAL) << "Unexpected instruction kind: " << kind; 1421 UNREACHABLE(); 1422 } 1423 } 1424} 1425 1426static void GenerateDataProc(HInstruction::InstructionKind kind, 1427 const Location& out, 1428 const Location& first, 1429 const ShifterOperand& second_lo, 1430 const ShifterOperand& second_hi, 1431 CodeGeneratorARM* codegen) { 1432 const Register first_hi = first.AsRegisterPairHigh<Register>(); 1433 const Register first_lo = first.AsRegisterPairLow<Register>(); 1434 const Register out_hi = out.AsRegisterPairHigh<Register>(); 1435 const Register out_lo = out.AsRegisterPairLow<Register>(); 1436 1437 if (kind == HInstruction::kAdd) { 1438 __ adds(out_lo, first_lo, second_lo); 1439 __ adc(out_hi, first_hi, second_hi); 1440 } else if (kind == HInstruction::kSub) { 1441 __ subs(out_lo, first_lo, second_lo); 1442 __ sbc(out_hi, first_hi, second_hi); 1443 } else { 1444 GenerateDataProcInstruction(kind, out_lo, first_lo, second_lo, codegen); 1445 GenerateDataProcInstruction(kind, out_hi, first_hi, second_hi, codegen); 1446 } 1447} 1448 1449static ShifterOperand GetShifterOperand(Register rm, Shift shift, uint32_t shift_imm) { 1450 return shift_imm == 0 ? ShifterOperand(rm) : ShifterOperand(rm, shift, shift_imm); 1451} 1452 1453static void GenerateLongDataProc(HDataProcWithShifterOp* instruction, CodeGeneratorARM* codegen) { 1454 DCHECK_EQ(instruction->GetType(), Primitive::kPrimLong); 1455 DCHECK(HDataProcWithShifterOp::IsShiftOp(instruction->GetOpKind())); 1456 1457 const LocationSummary* const locations = instruction->GetLocations(); 1458 const uint32_t shift_value = instruction->GetShiftAmount(); 1459 const HInstruction::InstructionKind kind = instruction->GetInstrKind(); 1460 const Location first = locations->InAt(0); 1461 const Location second = locations->InAt(1); 1462 const Location out = locations->Out(); 1463 const Register first_hi = first.AsRegisterPairHigh<Register>(); 1464 const Register first_lo = first.AsRegisterPairLow<Register>(); 1465 const Register out_hi = out.AsRegisterPairHigh<Register>(); 1466 const Register out_lo = out.AsRegisterPairLow<Register>(); 1467 const Register second_hi = second.AsRegisterPairHigh<Register>(); 1468 const Register second_lo = second.AsRegisterPairLow<Register>(); 1469 const Shift shift = ShiftFromOpKind(instruction->GetOpKind()); 1470 1471 if (shift_value >= 32) { 1472 if (shift == LSL) { 1473 GenerateDataProcInstruction(kind, 1474 out_hi, 1475 first_hi, 1476 ShifterOperand(second_lo, LSL, shift_value - 32), 1477 codegen); 1478 GenerateDataProcInstruction(kind, 1479 out_lo, 1480 first_lo, 1481 ShifterOperand(0), 1482 codegen); 1483 } else if (shift == ASR) { 1484 GenerateDataProc(kind, 1485 out, 1486 first, 1487 GetShifterOperand(second_hi, ASR, shift_value - 32), 1488 ShifterOperand(second_hi, ASR, 31), 1489 codegen); 1490 } else { 1491 DCHECK_EQ(shift, LSR); 1492 GenerateDataProc(kind, 1493 out, 1494 first, 1495 GetShifterOperand(second_hi, LSR, shift_value - 32), 1496 ShifterOperand(0), 1497 codegen); 1498 } 1499 } else { 1500 DCHECK_GT(shift_value, 1U); 1501 DCHECK_LT(shift_value, 32U); 1502 1503 if (shift == LSL) { 1504 // We are not doing this for HInstruction::kAdd because the output will require 1505 // Location::kOutputOverlap; not applicable to other cases. 1506 if (kind == HInstruction::kOr || kind == HInstruction::kXor) { 1507 GenerateDataProcInstruction(kind, 1508 out_hi, 1509 first_hi, 1510 ShifterOperand(second_hi, LSL, shift_value), 1511 codegen); 1512 GenerateDataProcInstruction(kind, 1513 out_hi, 1514 out_hi, 1515 ShifterOperand(second_lo, LSR, 32 - shift_value), 1516 codegen); 1517 GenerateDataProcInstruction(kind, 1518 out_lo, 1519 first_lo, 1520 ShifterOperand(second_lo, LSL, shift_value), 1521 codegen); 1522 } else { 1523 __ Lsl(IP, second_hi, shift_value); 1524 __ orr(IP, IP, ShifterOperand(second_lo, LSR, 32 - shift_value)); 1525 GenerateDataProc(kind, 1526 out, 1527 first, 1528 ShifterOperand(second_lo, LSL, shift_value), 1529 ShifterOperand(IP), 1530 codegen); 1531 } 1532 } else { 1533 DCHECK(shift == ASR || shift == LSR); 1534 1535 // We are not doing this for HInstruction::kAdd because the output will require 1536 // Location::kOutputOverlap; not applicable to other cases. 1537 if (kind == HInstruction::kOr || kind == HInstruction::kXor) { 1538 GenerateDataProcInstruction(kind, 1539 out_lo, 1540 first_lo, 1541 ShifterOperand(second_lo, LSR, shift_value), 1542 codegen); 1543 GenerateDataProcInstruction(kind, 1544 out_lo, 1545 out_lo, 1546 ShifterOperand(second_hi, LSL, 32 - shift_value), 1547 codegen); 1548 GenerateDataProcInstruction(kind, 1549 out_hi, 1550 first_hi, 1551 ShifterOperand(second_hi, shift, shift_value), 1552 codegen); 1553 } else { 1554 __ Lsr(IP, second_lo, shift_value); 1555 __ orr(IP, IP, ShifterOperand(second_hi, LSL, 32 - shift_value)); 1556 GenerateDataProc(kind, 1557 out, 1558 first, 1559 ShifterOperand(IP), 1560 ShifterOperand(second_hi, shift, shift_value), 1561 codegen); 1562 } 1563 } 1564 } 1565} 1566 1567static void GenerateVcmp(HInstruction* instruction, CodeGeneratorARM* codegen) { 1568 Primitive::Type type = instruction->InputAt(0)->GetType(); 1569 Location lhs_loc = instruction->GetLocations()->InAt(0); 1570 Location rhs_loc = instruction->GetLocations()->InAt(1); 1571 if (rhs_loc.IsConstant()) { 1572 // 0.0 is the only immediate that can be encoded directly in 1573 // a VCMP instruction. 1574 // 1575 // Both the JLS (section 15.20.1) and the JVMS (section 6.5) 1576 // specify that in a floating-point comparison, positive zero 1577 // and negative zero are considered equal, so we can use the 1578 // literal 0.0 for both cases here. 1579 // 1580 // Note however that some methods (Float.equal, Float.compare, 1581 // Float.compareTo, Double.equal, Double.compare, 1582 // Double.compareTo, Math.max, Math.min, StrictMath.max, 1583 // StrictMath.min) consider 0.0 to be (strictly) greater than 1584 // -0.0. So if we ever translate calls to these methods into a 1585 // HCompare instruction, we must handle the -0.0 case with 1586 // care here. 1587 DCHECK(rhs_loc.GetConstant()->IsArithmeticZero()); 1588 if (type == Primitive::kPrimFloat) { 1589 __ vcmpsz(lhs_loc.AsFpuRegister<SRegister>()); 1590 } else { 1591 DCHECK_EQ(type, Primitive::kPrimDouble); 1592 __ vcmpdz(FromLowSToD(lhs_loc.AsFpuRegisterPairLow<SRegister>())); 1593 } 1594 } else { 1595 if (type == Primitive::kPrimFloat) { 1596 __ vcmps(lhs_loc.AsFpuRegister<SRegister>(), rhs_loc.AsFpuRegister<SRegister>()); 1597 } else { 1598 DCHECK_EQ(type, Primitive::kPrimDouble); 1599 __ vcmpd(FromLowSToD(lhs_loc.AsFpuRegisterPairLow<SRegister>()), 1600 FromLowSToD(rhs_loc.AsFpuRegisterPairLow<SRegister>())); 1601 } 1602 } 1603} 1604 1605static std::pair<Condition, Condition> GenerateLongTestConstant(HCondition* condition, 1606 bool invert, 1607 CodeGeneratorARM* codegen) { 1608 DCHECK_EQ(condition->GetLeft()->GetType(), Primitive::kPrimLong); 1609 1610 const LocationSummary* const locations = condition->GetLocations(); 1611 IfCondition cond = condition->GetCondition(); 1612 IfCondition opposite = condition->GetOppositeCondition(); 1613 1614 if (invert) { 1615 std::swap(cond, opposite); 1616 } 1617 1618 std::pair<Condition, Condition> ret; 1619 const Location left = locations->InAt(0); 1620 const Location right = locations->InAt(1); 1621 1622 DCHECK(right.IsConstant()); 1623 1624 const Register left_high = left.AsRegisterPairHigh<Register>(); 1625 const Register left_low = left.AsRegisterPairLow<Register>(); 1626 int64_t value = right.GetConstant()->AsLongConstant()->GetValue(); 1627 1628 switch (cond) { 1629 case kCondEQ: 1630 case kCondNE: 1631 case kCondB: 1632 case kCondBE: 1633 case kCondA: 1634 case kCondAE: 1635 __ CmpConstant(left_high, High32Bits(value)); 1636 __ it(EQ); 1637 __ cmp(left_low, ShifterOperand(Low32Bits(value)), EQ); 1638 ret = std::make_pair(ARMUnsignedCondition(cond), ARMUnsignedCondition(opposite)); 1639 break; 1640 case kCondLE: 1641 case kCondGT: 1642 // Trivially true or false. 1643 if (value == std::numeric_limits<int64_t>::max()) { 1644 __ cmp(left_low, ShifterOperand(left_low)); 1645 ret = cond == kCondLE ? std::make_pair(EQ, NE) : std::make_pair(NE, EQ); 1646 break; 1647 } 1648 1649 if (cond == kCondLE) { 1650 DCHECK_EQ(opposite, kCondGT); 1651 cond = kCondLT; 1652 opposite = kCondGE; 1653 } else { 1654 DCHECK_EQ(cond, kCondGT); 1655 DCHECK_EQ(opposite, kCondLE); 1656 cond = kCondGE; 1657 opposite = kCondLT; 1658 } 1659 1660 value++; 1661 FALLTHROUGH_INTENDED; 1662 case kCondGE: 1663 case kCondLT: 1664 __ CmpConstant(left_low, Low32Bits(value)); 1665 __ sbcs(IP, left_high, ShifterOperand(High32Bits(value))); 1666 ret = std::make_pair(ARMCondition(cond), ARMCondition(opposite)); 1667 break; 1668 default: 1669 LOG(FATAL) << "Unreachable"; 1670 UNREACHABLE(); 1671 } 1672 1673 return ret; 1674} 1675 1676static std::pair<Condition, Condition> GenerateLongTest(HCondition* condition, 1677 bool invert, 1678 CodeGeneratorARM* codegen) { 1679 DCHECK_EQ(condition->GetLeft()->GetType(), Primitive::kPrimLong); 1680 1681 const LocationSummary* const locations = condition->GetLocations(); 1682 IfCondition cond = condition->GetCondition(); 1683 IfCondition opposite = condition->GetOppositeCondition(); 1684 1685 if (invert) { 1686 std::swap(cond, opposite); 1687 } 1688 1689 std::pair<Condition, Condition> ret; 1690 Location left = locations->InAt(0); 1691 Location right = locations->InAt(1); 1692 1693 DCHECK(right.IsRegisterPair()); 1694 1695 switch (cond) { 1696 case kCondEQ: 1697 case kCondNE: 1698 case kCondB: 1699 case kCondBE: 1700 case kCondA: 1701 case kCondAE: 1702 __ cmp(left.AsRegisterPairHigh<Register>(), 1703 ShifterOperand(right.AsRegisterPairHigh<Register>())); 1704 __ it(EQ); 1705 __ cmp(left.AsRegisterPairLow<Register>(), 1706 ShifterOperand(right.AsRegisterPairLow<Register>()), 1707 EQ); 1708 ret = std::make_pair(ARMUnsignedCondition(cond), ARMUnsignedCondition(opposite)); 1709 break; 1710 case kCondLE: 1711 case kCondGT: 1712 if (cond == kCondLE) { 1713 DCHECK_EQ(opposite, kCondGT); 1714 cond = kCondGE; 1715 opposite = kCondLT; 1716 } else { 1717 DCHECK_EQ(cond, kCondGT); 1718 DCHECK_EQ(opposite, kCondLE); 1719 cond = kCondLT; 1720 opposite = kCondGE; 1721 } 1722 1723 std::swap(left, right); 1724 FALLTHROUGH_INTENDED; 1725 case kCondGE: 1726 case kCondLT: 1727 __ cmp(left.AsRegisterPairLow<Register>(), 1728 ShifterOperand(right.AsRegisterPairLow<Register>())); 1729 __ sbcs(IP, 1730 left.AsRegisterPairHigh<Register>(), 1731 ShifterOperand(right.AsRegisterPairHigh<Register>())); 1732 ret = std::make_pair(ARMCondition(cond), ARMCondition(opposite)); 1733 break; 1734 default: 1735 LOG(FATAL) << "Unreachable"; 1736 UNREACHABLE(); 1737 } 1738 1739 return ret; 1740} 1741 1742static std::pair<Condition, Condition> GenerateTest(HCondition* condition, 1743 bool invert, 1744 CodeGeneratorARM* codegen) { 1745 const LocationSummary* const locations = condition->GetLocations(); 1746 const Primitive::Type type = condition->GetLeft()->GetType(); 1747 IfCondition cond = condition->GetCondition(); 1748 IfCondition opposite = condition->GetOppositeCondition(); 1749 std::pair<Condition, Condition> ret; 1750 const Location right = locations->InAt(1); 1751 1752 if (invert) { 1753 std::swap(cond, opposite); 1754 } 1755 1756 if (type == Primitive::kPrimLong) { 1757 ret = locations->InAt(1).IsConstant() 1758 ? GenerateLongTestConstant(condition, invert, codegen) 1759 : GenerateLongTest(condition, invert, codegen); 1760 } else if (Primitive::IsFloatingPointType(type)) { 1761 GenerateVcmp(condition, codegen); 1762 __ vmstat(); 1763 ret = std::make_pair(ARMFPCondition(cond, condition->IsGtBias()), 1764 ARMFPCondition(opposite, condition->IsGtBias())); 1765 } else { 1766 DCHECK(Primitive::IsIntegralType(type) || type == Primitive::kPrimNot) << type; 1767 1768 const Register left = locations->InAt(0).AsRegister<Register>(); 1769 1770 if (right.IsRegister()) { 1771 __ cmp(left, ShifterOperand(right.AsRegister<Register>())); 1772 } else { 1773 DCHECK(right.IsConstant()); 1774 __ CmpConstant(left, CodeGenerator::GetInt32ValueOf(right.GetConstant())); 1775 } 1776 1777 ret = std::make_pair(ARMCondition(cond), ARMCondition(opposite)); 1778 } 1779 1780 return ret; 1781} 1782 1783static bool CanGenerateTest(HCondition* condition, ArmAssembler* assembler) { 1784 if (condition->GetLeft()->GetType() == Primitive::kPrimLong) { 1785 const LocationSummary* const locations = condition->GetLocations(); 1786 const IfCondition c = condition->GetCondition(); 1787 1788 if (locations->InAt(1).IsConstant()) { 1789 const int64_t value = locations->InAt(1).GetConstant()->AsLongConstant()->GetValue(); 1790 ShifterOperand so; 1791 1792 if (c < kCondLT || c > kCondGE) { 1793 // Since IT blocks longer than a 16-bit instruction are deprecated by ARMv8, 1794 // we check that the least significant half of the first input to be compared 1795 // is in a low register (the other half is read outside an IT block), and 1796 // the constant fits in an 8-bit unsigned integer, so that a 16-bit CMP 1797 // encoding can be used. 1798 if (!ArmAssembler::IsLowRegister(locations->InAt(0).AsRegisterPairLow<Register>()) || 1799 !IsUint<8>(Low32Bits(value))) { 1800 return false; 1801 } 1802 } else if (c == kCondLE || c == kCondGT) { 1803 if (value < std::numeric_limits<int64_t>::max() && 1804 !assembler->ShifterOperandCanHold(kNoRegister, 1805 kNoRegister, 1806 SBC, 1807 High32Bits(value + 1), 1808 kCcSet, 1809 &so)) { 1810 return false; 1811 } 1812 } else if (!assembler->ShifterOperandCanHold(kNoRegister, 1813 kNoRegister, 1814 SBC, 1815 High32Bits(value), 1816 kCcSet, 1817 &so)) { 1818 return false; 1819 } 1820 } 1821 } 1822 1823 return true; 1824} 1825 1826static bool CanEncodeConstantAs8BitImmediate(HConstant* constant) { 1827 const Primitive::Type type = constant->GetType(); 1828 bool ret = false; 1829 1830 DCHECK(Primitive::IsIntegralType(type) || type == Primitive::kPrimNot) << type; 1831 1832 if (type == Primitive::kPrimLong) { 1833 const uint64_t value = constant->AsLongConstant()->GetValueAsUint64(); 1834 1835 ret = IsUint<8>(Low32Bits(value)) && IsUint<8>(High32Bits(value)); 1836 } else { 1837 ret = IsUint<8>(CodeGenerator::GetInt32ValueOf(constant)); 1838 } 1839 1840 return ret; 1841} 1842 1843static Location Arm8BitEncodableConstantOrRegister(HInstruction* constant) { 1844 DCHECK(!Primitive::IsFloatingPointType(constant->GetType())); 1845 1846 if (constant->IsConstant() && CanEncodeConstantAs8BitImmediate(constant->AsConstant())) { 1847 return Location::ConstantLocation(constant->AsConstant()); 1848 } 1849 1850 return Location::RequiresRegister(); 1851} 1852 1853static bool CanGenerateConditionalMove(const Location& out, const Location& src) { 1854 // Since IT blocks longer than a 16-bit instruction are deprecated by ARMv8, 1855 // we check that we are not dealing with floating-point output (there is no 1856 // 16-bit VMOV encoding). 1857 if (!out.IsRegister() && !out.IsRegisterPair()) { 1858 return false; 1859 } 1860 1861 // For constants, we also check that the output is in one or two low registers, 1862 // and that the constants fit in an 8-bit unsigned integer, so that a 16-bit 1863 // MOV encoding can be used. 1864 if (src.IsConstant()) { 1865 if (!CanEncodeConstantAs8BitImmediate(src.GetConstant())) { 1866 return false; 1867 } 1868 1869 if (out.IsRegister()) { 1870 if (!ArmAssembler::IsLowRegister(out.AsRegister<Register>())) { 1871 return false; 1872 } 1873 } else { 1874 DCHECK(out.IsRegisterPair()); 1875 1876 if (!ArmAssembler::IsLowRegister(out.AsRegisterPairHigh<Register>())) { 1877 return false; 1878 } 1879 } 1880 } 1881 1882 return true; 1883} 1884 1885#undef __ 1886// NOLINT on __ macro to suppress wrong warning/fix (misc-macro-parentheses) from clang-tidy. 1887#define __ down_cast<ArmAssembler*>(GetAssembler())-> // NOLINT 1888 1889Label* CodeGeneratorARM::GetFinalLabel(HInstruction* instruction, Label* final_label) { 1890 DCHECK(!instruction->IsControlFlow() && !instruction->IsSuspendCheck()); 1891 DCHECK(!instruction->IsInvoke() || !instruction->GetLocations()->CanCall()); 1892 1893 const HBasicBlock* const block = instruction->GetBlock(); 1894 const HLoopInformation* const info = block->GetLoopInformation(); 1895 HInstruction* const next = instruction->GetNext(); 1896 1897 // Avoid a branch to a branch. 1898 if (next->IsGoto() && (info == nullptr || 1899 !info->IsBackEdge(*block) || 1900 !info->HasSuspendCheck())) { 1901 final_label = GetLabelOf(next->AsGoto()->GetSuccessor()); 1902 } 1903 1904 return final_label; 1905} 1906 1907void CodeGeneratorARM::DumpCoreRegister(std::ostream& stream, int reg) const { 1908 stream << Register(reg); 1909} 1910 1911void CodeGeneratorARM::DumpFloatingPointRegister(std::ostream& stream, int reg) const { 1912 stream << SRegister(reg); 1913} 1914 1915size_t CodeGeneratorARM::SaveCoreRegister(size_t stack_index, uint32_t reg_id) { 1916 __ StoreToOffset(kStoreWord, static_cast<Register>(reg_id), SP, stack_index); 1917 return kArmWordSize; 1918} 1919 1920size_t CodeGeneratorARM::RestoreCoreRegister(size_t stack_index, uint32_t reg_id) { 1921 __ LoadFromOffset(kLoadWord, static_cast<Register>(reg_id), SP, stack_index); 1922 return kArmWordSize; 1923} 1924 1925size_t CodeGeneratorARM::SaveFloatingPointRegister(size_t stack_index, uint32_t reg_id) { 1926 __ StoreSToOffset(static_cast<SRegister>(reg_id), SP, stack_index); 1927 return kArmWordSize; 1928} 1929 1930size_t CodeGeneratorARM::RestoreFloatingPointRegister(size_t stack_index, uint32_t reg_id) { 1931 __ LoadSFromOffset(static_cast<SRegister>(reg_id), SP, stack_index); 1932 return kArmWordSize; 1933} 1934 1935CodeGeneratorARM::CodeGeneratorARM(HGraph* graph, 1936 const ArmInstructionSetFeatures& isa_features, 1937 const CompilerOptions& compiler_options, 1938 OptimizingCompilerStats* stats) 1939 : CodeGenerator(graph, 1940 kNumberOfCoreRegisters, 1941 kNumberOfSRegisters, 1942 kNumberOfRegisterPairs, 1943 ComputeRegisterMask(reinterpret_cast<const int*>(kCoreCalleeSaves), 1944 arraysize(kCoreCalleeSaves)), 1945 ComputeRegisterMask(reinterpret_cast<const int*>(kFpuCalleeSaves), 1946 arraysize(kFpuCalleeSaves)), 1947 compiler_options, 1948 stats), 1949 block_labels_(nullptr), 1950 location_builder_(graph, this), 1951 instruction_visitor_(graph, this), 1952 move_resolver_(graph->GetArena(), this), 1953 assembler_(graph->GetArena()), 1954 isa_features_(isa_features), 1955 uint32_literals_(std::less<uint32_t>(), 1956 graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), 1957 pc_relative_dex_cache_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), 1958 boot_image_string_patches_(StringReferenceValueComparator(), 1959 graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), 1960 pc_relative_string_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), 1961 boot_image_type_patches_(TypeReferenceValueComparator(), 1962 graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), 1963 pc_relative_type_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), 1964 type_bss_entry_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), 1965 jit_string_patches_(StringReferenceValueComparator(), 1966 graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), 1967 jit_class_patches_(TypeReferenceValueComparator(), 1968 graph->GetArena()->Adapter(kArenaAllocCodeGenerator)) { 1969 // Always save the LR register to mimic Quick. 1970 AddAllocatedRegister(Location::RegisterLocation(LR)); 1971} 1972 1973void CodeGeneratorARM::Finalize(CodeAllocator* allocator) { 1974 // Ensure that we fix up branches and literal loads and emit the literal pool. 1975 __ FinalizeCode(); 1976 1977 // Adjust native pc offsets in stack maps. 1978 for (size_t i = 0, num = stack_map_stream_.GetNumberOfStackMaps(); i != num; ++i) { 1979 uint32_t old_position = 1980 stack_map_stream_.GetStackMap(i).native_pc_code_offset.Uint32Value(kThumb2); 1981 uint32_t new_position = __ GetAdjustedPosition(old_position); 1982 stack_map_stream_.SetStackMapNativePcOffset(i, new_position); 1983 } 1984 // Adjust pc offsets for the disassembly information. 1985 if (disasm_info_ != nullptr) { 1986 GeneratedCodeInterval* frame_entry_interval = disasm_info_->GetFrameEntryInterval(); 1987 frame_entry_interval->start = __ GetAdjustedPosition(frame_entry_interval->start); 1988 frame_entry_interval->end = __ GetAdjustedPosition(frame_entry_interval->end); 1989 for (auto& it : *disasm_info_->GetInstructionIntervals()) { 1990 it.second.start = __ GetAdjustedPosition(it.second.start); 1991 it.second.end = __ GetAdjustedPosition(it.second.end); 1992 } 1993 for (auto& it : *disasm_info_->GetSlowPathIntervals()) { 1994 it.code_interval.start = __ GetAdjustedPosition(it.code_interval.start); 1995 it.code_interval.end = __ GetAdjustedPosition(it.code_interval.end); 1996 } 1997 } 1998 1999 CodeGenerator::Finalize(allocator); 2000} 2001 2002void CodeGeneratorARM::SetupBlockedRegisters() const { 2003 // Stack register, LR and PC are always reserved. 2004 blocked_core_registers_[SP] = true; 2005 blocked_core_registers_[LR] = true; 2006 blocked_core_registers_[PC] = true; 2007 2008 // Reserve thread register. 2009 blocked_core_registers_[TR] = true; 2010 2011 // Reserve temp register. 2012 blocked_core_registers_[IP] = true; 2013 2014 if (GetGraph()->IsDebuggable()) { 2015 // Stubs do not save callee-save floating point registers. If the graph 2016 // is debuggable, we need to deal with these registers differently. For 2017 // now, just block them. 2018 for (size_t i = 0; i < arraysize(kFpuCalleeSaves); ++i) { 2019 blocked_fpu_registers_[kFpuCalleeSaves[i]] = true; 2020 } 2021 } 2022} 2023 2024InstructionCodeGeneratorARM::InstructionCodeGeneratorARM(HGraph* graph, CodeGeneratorARM* codegen) 2025 : InstructionCodeGenerator(graph, codegen), 2026 assembler_(codegen->GetAssembler()), 2027 codegen_(codegen) {} 2028 2029void CodeGeneratorARM::ComputeSpillMask() { 2030 core_spill_mask_ = allocated_registers_.GetCoreRegisters() & core_callee_save_mask_; 2031 DCHECK_NE(core_spill_mask_, 0u) << "At least the return address register must be saved"; 2032 // There is no easy instruction to restore just the PC on thumb2. We spill and 2033 // restore another arbitrary register. 2034 core_spill_mask_ |= (1 << kCoreAlwaysSpillRegister); 2035 fpu_spill_mask_ = allocated_registers_.GetFloatingPointRegisters() & fpu_callee_save_mask_; 2036 // We use vpush and vpop for saving and restoring floating point registers, which take 2037 // a SRegister and the number of registers to save/restore after that SRegister. We 2038 // therefore update the `fpu_spill_mask_` to also contain those registers not allocated, 2039 // but in the range. 2040 if (fpu_spill_mask_ != 0) { 2041 uint32_t least_significant_bit = LeastSignificantBit(fpu_spill_mask_); 2042 uint32_t most_significant_bit = MostSignificantBit(fpu_spill_mask_); 2043 for (uint32_t i = least_significant_bit + 1 ; i < most_significant_bit; ++i) { 2044 fpu_spill_mask_ |= (1 << i); 2045 } 2046 } 2047} 2048 2049static dwarf::Reg DWARFReg(Register reg) { 2050 return dwarf::Reg::ArmCore(static_cast<int>(reg)); 2051} 2052 2053static dwarf::Reg DWARFReg(SRegister reg) { 2054 return dwarf::Reg::ArmFp(static_cast<int>(reg)); 2055} 2056 2057void CodeGeneratorARM::GenerateFrameEntry() { 2058 bool skip_overflow_check = 2059 IsLeafMethod() && !FrameNeedsStackCheck(GetFrameSize(), InstructionSet::kArm); 2060 DCHECK(GetCompilerOptions().GetImplicitStackOverflowChecks()); 2061 __ Bind(&frame_entry_label_); 2062 2063 if (HasEmptyFrame()) { 2064 return; 2065 } 2066 2067 if (!skip_overflow_check) { 2068 __ AddConstant(IP, SP, -static_cast<int32_t>(GetStackOverflowReservedBytes(kArm))); 2069 __ LoadFromOffset(kLoadWord, IP, IP, 0); 2070 RecordPcInfo(nullptr, 0); 2071 } 2072 2073 __ PushList(core_spill_mask_); 2074 __ cfi().AdjustCFAOffset(kArmWordSize * POPCOUNT(core_spill_mask_)); 2075 __ cfi().RelOffsetForMany(DWARFReg(kMethodRegisterArgument), 0, core_spill_mask_, kArmWordSize); 2076 if (fpu_spill_mask_ != 0) { 2077 SRegister start_register = SRegister(LeastSignificantBit(fpu_spill_mask_)); 2078 __ vpushs(start_register, POPCOUNT(fpu_spill_mask_)); 2079 __ cfi().AdjustCFAOffset(kArmWordSize * POPCOUNT(fpu_spill_mask_)); 2080 __ cfi().RelOffsetForMany(DWARFReg(S0), 0, fpu_spill_mask_, kArmWordSize); 2081 } 2082 2083 if (GetGraph()->HasShouldDeoptimizeFlag()) { 2084 // Initialize should_deoptimize flag to 0. 2085 __ mov(IP, ShifterOperand(0)); 2086 __ StoreToOffset(kStoreWord, IP, SP, -kShouldDeoptimizeFlagSize); 2087 } 2088 2089 int adjust = GetFrameSize() - FrameEntrySpillSize(); 2090 __ AddConstant(SP, -adjust); 2091 __ cfi().AdjustCFAOffset(adjust); 2092 2093 // Save the current method if we need it. Note that we do not 2094 // do this in HCurrentMethod, as the instruction might have been removed 2095 // in the SSA graph. 2096 if (RequiresCurrentMethod()) { 2097 __ StoreToOffset(kStoreWord, kMethodRegisterArgument, SP, 0); 2098 } 2099} 2100 2101void CodeGeneratorARM::GenerateFrameExit() { 2102 if (HasEmptyFrame()) { 2103 __ bx(LR); 2104 return; 2105 } 2106 __ cfi().RememberState(); 2107 int adjust = GetFrameSize() - FrameEntrySpillSize(); 2108 __ AddConstant(SP, adjust); 2109 __ cfi().AdjustCFAOffset(-adjust); 2110 if (fpu_spill_mask_ != 0) { 2111 SRegister start_register = SRegister(LeastSignificantBit(fpu_spill_mask_)); 2112 __ vpops(start_register, POPCOUNT(fpu_spill_mask_)); 2113 __ cfi().AdjustCFAOffset(-static_cast<int>(kArmPointerSize) * POPCOUNT(fpu_spill_mask_)); 2114 __ cfi().RestoreMany(DWARFReg(SRegister(0)), fpu_spill_mask_); 2115 } 2116 // Pop LR into PC to return. 2117 DCHECK_NE(core_spill_mask_ & (1 << LR), 0U); 2118 uint32_t pop_mask = (core_spill_mask_ & (~(1 << LR))) | 1 << PC; 2119 __ PopList(pop_mask); 2120 __ cfi().RestoreState(); 2121 __ cfi().DefCFAOffset(GetFrameSize()); 2122} 2123 2124void CodeGeneratorARM::Bind(HBasicBlock* block) { 2125 Label* label = GetLabelOf(block); 2126 __ BindTrackedLabel(label); 2127} 2128 2129Location InvokeDexCallingConventionVisitorARM::GetNextLocation(Primitive::Type type) { 2130 switch (type) { 2131 case Primitive::kPrimBoolean: 2132 case Primitive::kPrimByte: 2133 case Primitive::kPrimChar: 2134 case Primitive::kPrimShort: 2135 case Primitive::kPrimInt: 2136 case Primitive::kPrimNot: { 2137 uint32_t index = gp_index_++; 2138 uint32_t stack_index = stack_index_++; 2139 if (index < calling_convention.GetNumberOfRegisters()) { 2140 return Location::RegisterLocation(calling_convention.GetRegisterAt(index)); 2141 } else { 2142 return Location::StackSlot(calling_convention.GetStackOffsetOf(stack_index)); 2143 } 2144 } 2145 2146 case Primitive::kPrimLong: { 2147 uint32_t index = gp_index_; 2148 uint32_t stack_index = stack_index_; 2149 gp_index_ += 2; 2150 stack_index_ += 2; 2151 if (index + 1 < calling_convention.GetNumberOfRegisters()) { 2152 if (calling_convention.GetRegisterAt(index) == R1) { 2153 // Skip R1, and use R2_R3 instead. 2154 gp_index_++; 2155 index++; 2156 } 2157 } 2158 if (index + 1 < calling_convention.GetNumberOfRegisters()) { 2159 DCHECK_EQ(calling_convention.GetRegisterAt(index) + 1, 2160 calling_convention.GetRegisterAt(index + 1)); 2161 2162 return Location::RegisterPairLocation(calling_convention.GetRegisterAt(index), 2163 calling_convention.GetRegisterAt(index + 1)); 2164 } else { 2165 return Location::DoubleStackSlot(calling_convention.GetStackOffsetOf(stack_index)); 2166 } 2167 } 2168 2169 case Primitive::kPrimFloat: { 2170 uint32_t stack_index = stack_index_++; 2171 if (float_index_ % 2 == 0) { 2172 float_index_ = std::max(double_index_, float_index_); 2173 } 2174 if (float_index_ < calling_convention.GetNumberOfFpuRegisters()) { 2175 return Location::FpuRegisterLocation(calling_convention.GetFpuRegisterAt(float_index_++)); 2176 } else { 2177 return Location::StackSlot(calling_convention.GetStackOffsetOf(stack_index)); 2178 } 2179 } 2180 2181 case Primitive::kPrimDouble: { 2182 double_index_ = std::max(double_index_, RoundUp(float_index_, 2)); 2183 uint32_t stack_index = stack_index_; 2184 stack_index_ += 2; 2185 if (double_index_ + 1 < calling_convention.GetNumberOfFpuRegisters()) { 2186 uint32_t index = double_index_; 2187 double_index_ += 2; 2188 Location result = Location::FpuRegisterPairLocation( 2189 calling_convention.GetFpuRegisterAt(index), 2190 calling_convention.GetFpuRegisterAt(index + 1)); 2191 DCHECK(ExpectedPairLayout(result)); 2192 return result; 2193 } else { 2194 return Location::DoubleStackSlot(calling_convention.GetStackOffsetOf(stack_index)); 2195 } 2196 } 2197 2198 case Primitive::kPrimVoid: 2199 LOG(FATAL) << "Unexpected parameter type " << type; 2200 break; 2201 } 2202 return Location::NoLocation(); 2203} 2204 2205Location InvokeDexCallingConventionVisitorARM::GetReturnLocation(Primitive::Type type) const { 2206 switch (type) { 2207 case Primitive::kPrimBoolean: 2208 case Primitive::kPrimByte: 2209 case Primitive::kPrimChar: 2210 case Primitive::kPrimShort: 2211 case Primitive::kPrimInt: 2212 case Primitive::kPrimNot: { 2213 return Location::RegisterLocation(R0); 2214 } 2215 2216 case Primitive::kPrimFloat: { 2217 return Location::FpuRegisterLocation(S0); 2218 } 2219 2220 case Primitive::kPrimLong: { 2221 return Location::RegisterPairLocation(R0, R1); 2222 } 2223 2224 case Primitive::kPrimDouble: { 2225 return Location::FpuRegisterPairLocation(S0, S1); 2226 } 2227 2228 case Primitive::kPrimVoid: 2229 return Location::NoLocation(); 2230 } 2231 2232 UNREACHABLE(); 2233} 2234 2235Location InvokeDexCallingConventionVisitorARM::GetMethodLocation() const { 2236 return Location::RegisterLocation(kMethodRegisterArgument); 2237} 2238 2239void CodeGeneratorARM::Move32(Location destination, Location source) { 2240 if (source.Equals(destination)) { 2241 return; 2242 } 2243 if (destination.IsRegister()) { 2244 if (source.IsRegister()) { 2245 __ Mov(destination.AsRegister<Register>(), source.AsRegister<Register>()); 2246 } else if (source.IsFpuRegister()) { 2247 __ vmovrs(destination.AsRegister<Register>(), source.AsFpuRegister<SRegister>()); 2248 } else { 2249 __ LoadFromOffset(kLoadWord, destination.AsRegister<Register>(), SP, source.GetStackIndex()); 2250 } 2251 } else if (destination.IsFpuRegister()) { 2252 if (source.IsRegister()) { 2253 __ vmovsr(destination.AsFpuRegister<SRegister>(), source.AsRegister<Register>()); 2254 } else if (source.IsFpuRegister()) { 2255 __ vmovs(destination.AsFpuRegister<SRegister>(), source.AsFpuRegister<SRegister>()); 2256 } else { 2257 __ LoadSFromOffset(destination.AsFpuRegister<SRegister>(), SP, source.GetStackIndex()); 2258 } 2259 } else { 2260 DCHECK(destination.IsStackSlot()) << destination; 2261 if (source.IsRegister()) { 2262 __ StoreToOffset(kStoreWord, source.AsRegister<Register>(), SP, destination.GetStackIndex()); 2263 } else if (source.IsFpuRegister()) { 2264 __ StoreSToOffset(source.AsFpuRegister<SRegister>(), SP, destination.GetStackIndex()); 2265 } else { 2266 DCHECK(source.IsStackSlot()) << source; 2267 __ LoadFromOffset(kLoadWord, IP, SP, source.GetStackIndex()); 2268 __ StoreToOffset(kStoreWord, IP, SP, destination.GetStackIndex()); 2269 } 2270 } 2271} 2272 2273void CodeGeneratorARM::Move64(Location destination, Location source) { 2274 if (source.Equals(destination)) { 2275 return; 2276 } 2277 if (destination.IsRegisterPair()) { 2278 if (source.IsRegisterPair()) { 2279 EmitParallelMoves( 2280 Location::RegisterLocation(source.AsRegisterPairHigh<Register>()), 2281 Location::RegisterLocation(destination.AsRegisterPairHigh<Register>()), 2282 Primitive::kPrimInt, 2283 Location::RegisterLocation(source.AsRegisterPairLow<Register>()), 2284 Location::RegisterLocation(destination.AsRegisterPairLow<Register>()), 2285 Primitive::kPrimInt); 2286 } else if (source.IsFpuRegister()) { 2287 UNIMPLEMENTED(FATAL); 2288 } else if (source.IsFpuRegisterPair()) { 2289 __ vmovrrd(destination.AsRegisterPairLow<Register>(), 2290 destination.AsRegisterPairHigh<Register>(), 2291 FromLowSToD(source.AsFpuRegisterPairLow<SRegister>())); 2292 } else { 2293 DCHECK(source.IsDoubleStackSlot()); 2294 DCHECK(ExpectedPairLayout(destination)); 2295 __ LoadFromOffset(kLoadWordPair, destination.AsRegisterPairLow<Register>(), 2296 SP, source.GetStackIndex()); 2297 } 2298 } else if (destination.IsFpuRegisterPair()) { 2299 if (source.IsDoubleStackSlot()) { 2300 __ LoadDFromOffset(FromLowSToD(destination.AsFpuRegisterPairLow<SRegister>()), 2301 SP, 2302 source.GetStackIndex()); 2303 } else if (source.IsRegisterPair()) { 2304 __ vmovdrr(FromLowSToD(destination.AsFpuRegisterPairLow<SRegister>()), 2305 source.AsRegisterPairLow<Register>(), 2306 source.AsRegisterPairHigh<Register>()); 2307 } else { 2308 UNIMPLEMENTED(FATAL); 2309 } 2310 } else { 2311 DCHECK(destination.IsDoubleStackSlot()); 2312 if (source.IsRegisterPair()) { 2313 // No conflict possible, so just do the moves. 2314 if (source.AsRegisterPairLow<Register>() == R1) { 2315 DCHECK_EQ(source.AsRegisterPairHigh<Register>(), R2); 2316 __ StoreToOffset(kStoreWord, R1, SP, destination.GetStackIndex()); 2317 __ StoreToOffset(kStoreWord, R2, SP, destination.GetHighStackIndex(kArmWordSize)); 2318 } else { 2319 __ StoreToOffset(kStoreWordPair, source.AsRegisterPairLow<Register>(), 2320 SP, destination.GetStackIndex()); 2321 } 2322 } else if (source.IsFpuRegisterPair()) { 2323 __ StoreDToOffset(FromLowSToD(source.AsFpuRegisterPairLow<SRegister>()), 2324 SP, 2325 destination.GetStackIndex()); 2326 } else { 2327 DCHECK(source.IsDoubleStackSlot()); 2328 EmitParallelMoves( 2329 Location::StackSlot(source.GetStackIndex()), 2330 Location::StackSlot(destination.GetStackIndex()), 2331 Primitive::kPrimInt, 2332 Location::StackSlot(source.GetHighStackIndex(kArmWordSize)), 2333 Location::StackSlot(destination.GetHighStackIndex(kArmWordSize)), 2334 Primitive::kPrimInt); 2335 } 2336 } 2337} 2338 2339void CodeGeneratorARM::MoveConstant(Location location, int32_t value) { 2340 DCHECK(location.IsRegister()); 2341 __ LoadImmediate(location.AsRegister<Register>(), value); 2342} 2343 2344void CodeGeneratorARM::MoveLocation(Location dst, Location src, Primitive::Type dst_type) { 2345 HParallelMove move(GetGraph()->GetArena()); 2346 move.AddMove(src, dst, dst_type, nullptr); 2347 GetMoveResolver()->EmitNativeCode(&move); 2348} 2349 2350void CodeGeneratorARM::AddLocationAsTemp(Location location, LocationSummary* locations) { 2351 if (location.IsRegister()) { 2352 locations->AddTemp(location); 2353 } else if (location.IsRegisterPair()) { 2354 locations->AddTemp(Location::RegisterLocation(location.AsRegisterPairLow<Register>())); 2355 locations->AddTemp(Location::RegisterLocation(location.AsRegisterPairHigh<Register>())); 2356 } else { 2357 UNIMPLEMENTED(FATAL) << "AddLocationAsTemp not implemented for location " << location; 2358 } 2359} 2360 2361void CodeGeneratorARM::InvokeRuntime(QuickEntrypointEnum entrypoint, 2362 HInstruction* instruction, 2363 uint32_t dex_pc, 2364 SlowPathCode* slow_path) { 2365 ValidateInvokeRuntime(entrypoint, instruction, slow_path); 2366 GenerateInvokeRuntime(GetThreadOffset<kArmPointerSize>(entrypoint).Int32Value()); 2367 if (EntrypointRequiresStackMap(entrypoint)) { 2368 RecordPcInfo(instruction, dex_pc, slow_path); 2369 } 2370} 2371 2372void CodeGeneratorARM::InvokeRuntimeWithoutRecordingPcInfo(int32_t entry_point_offset, 2373 HInstruction* instruction, 2374 SlowPathCode* slow_path) { 2375 ValidateInvokeRuntimeWithoutRecordingPcInfo(instruction, slow_path); 2376 GenerateInvokeRuntime(entry_point_offset); 2377} 2378 2379void CodeGeneratorARM::GenerateInvokeRuntime(int32_t entry_point_offset) { 2380 __ LoadFromOffset(kLoadWord, LR, TR, entry_point_offset); 2381 __ blx(LR); 2382} 2383 2384void InstructionCodeGeneratorARM::HandleGoto(HInstruction* got, HBasicBlock* successor) { 2385 DCHECK(!successor->IsExitBlock()); 2386 2387 HBasicBlock* block = got->GetBlock(); 2388 HInstruction* previous = got->GetPrevious(); 2389 2390 HLoopInformation* info = block->GetLoopInformation(); 2391 if (info != nullptr && info->IsBackEdge(*block) && info->HasSuspendCheck()) { 2392 codegen_->ClearSpillSlotsFromLoopPhisInStackMap(info->GetSuspendCheck()); 2393 GenerateSuspendCheck(info->GetSuspendCheck(), successor); 2394 return; 2395 } 2396 2397 if (block->IsEntryBlock() && (previous != nullptr) && previous->IsSuspendCheck()) { 2398 GenerateSuspendCheck(previous->AsSuspendCheck(), nullptr); 2399 } 2400 if (!codegen_->GoesToNextBlock(got->GetBlock(), successor)) { 2401 __ b(codegen_->GetLabelOf(successor)); 2402 } 2403} 2404 2405void LocationsBuilderARM::VisitGoto(HGoto* got) { 2406 got->SetLocations(nullptr); 2407} 2408 2409void InstructionCodeGeneratorARM::VisitGoto(HGoto* got) { 2410 HandleGoto(got, got->GetSuccessor()); 2411} 2412 2413void LocationsBuilderARM::VisitTryBoundary(HTryBoundary* try_boundary) { 2414 try_boundary->SetLocations(nullptr); 2415} 2416 2417void InstructionCodeGeneratorARM::VisitTryBoundary(HTryBoundary* try_boundary) { 2418 HBasicBlock* successor = try_boundary->GetNormalFlowSuccessor(); 2419 if (!successor->IsExitBlock()) { 2420 HandleGoto(try_boundary, successor); 2421 } 2422} 2423 2424void LocationsBuilderARM::VisitExit(HExit* exit) { 2425 exit->SetLocations(nullptr); 2426} 2427 2428void InstructionCodeGeneratorARM::VisitExit(HExit* exit ATTRIBUTE_UNUSED) { 2429} 2430 2431void InstructionCodeGeneratorARM::GenerateLongComparesAndJumps(HCondition* cond, 2432 Label* true_label, 2433 Label* false_label) { 2434 LocationSummary* locations = cond->GetLocations(); 2435 Location left = locations->InAt(0); 2436 Location right = locations->InAt(1); 2437 IfCondition if_cond = cond->GetCondition(); 2438 2439 Register left_high = left.AsRegisterPairHigh<Register>(); 2440 Register left_low = left.AsRegisterPairLow<Register>(); 2441 IfCondition true_high_cond = if_cond; 2442 IfCondition false_high_cond = cond->GetOppositeCondition(); 2443 Condition final_condition = ARMUnsignedCondition(if_cond); // unsigned on lower part 2444 2445 // Set the conditions for the test, remembering that == needs to be 2446 // decided using the low words. 2447 switch (if_cond) { 2448 case kCondEQ: 2449 case kCondNE: 2450 // Nothing to do. 2451 break; 2452 case kCondLT: 2453 false_high_cond = kCondGT; 2454 break; 2455 case kCondLE: 2456 true_high_cond = kCondLT; 2457 break; 2458 case kCondGT: 2459 false_high_cond = kCondLT; 2460 break; 2461 case kCondGE: 2462 true_high_cond = kCondGT; 2463 break; 2464 case kCondB: 2465 false_high_cond = kCondA; 2466 break; 2467 case kCondBE: 2468 true_high_cond = kCondB; 2469 break; 2470 case kCondA: 2471 false_high_cond = kCondB; 2472 break; 2473 case kCondAE: 2474 true_high_cond = kCondA; 2475 break; 2476 } 2477 if (right.IsConstant()) { 2478 int64_t value = right.GetConstant()->AsLongConstant()->GetValue(); 2479 int32_t val_low = Low32Bits(value); 2480 int32_t val_high = High32Bits(value); 2481 2482 __ CmpConstant(left_high, val_high); 2483 if (if_cond == kCondNE) { 2484 __ b(true_label, ARMCondition(true_high_cond)); 2485 } else if (if_cond == kCondEQ) { 2486 __ b(false_label, ARMCondition(false_high_cond)); 2487 } else { 2488 __ b(true_label, ARMCondition(true_high_cond)); 2489 __ b(false_label, ARMCondition(false_high_cond)); 2490 } 2491 // Must be equal high, so compare the lows. 2492 __ CmpConstant(left_low, val_low); 2493 } else { 2494 Register right_high = right.AsRegisterPairHigh<Register>(); 2495 Register right_low = right.AsRegisterPairLow<Register>(); 2496 2497 __ cmp(left_high, ShifterOperand(right_high)); 2498 if (if_cond == kCondNE) { 2499 __ b(true_label, ARMCondition(true_high_cond)); 2500 } else if (if_cond == kCondEQ) { 2501 __ b(false_label, ARMCondition(false_high_cond)); 2502 } else { 2503 __ b(true_label, ARMCondition(true_high_cond)); 2504 __ b(false_label, ARMCondition(false_high_cond)); 2505 } 2506 // Must be equal high, so compare the lows. 2507 __ cmp(left_low, ShifterOperand(right_low)); 2508 } 2509 // The last comparison might be unsigned. 2510 // TODO: optimize cases where this is always true/false 2511 __ b(true_label, final_condition); 2512} 2513 2514void InstructionCodeGeneratorARM::GenerateCompareTestAndBranch(HCondition* condition, 2515 Label* true_target_in, 2516 Label* false_target_in) { 2517 if (CanGenerateTest(condition, codegen_->GetAssembler())) { 2518 Label* non_fallthrough_target; 2519 bool invert; 2520 2521 if (true_target_in == nullptr) { 2522 DCHECK(false_target_in != nullptr); 2523 non_fallthrough_target = false_target_in; 2524 invert = true; 2525 } else { 2526 non_fallthrough_target = true_target_in; 2527 invert = false; 2528 } 2529 2530 const auto cond = GenerateTest(condition, invert, codegen_); 2531 2532 __ b(non_fallthrough_target, cond.first); 2533 2534 if (false_target_in != nullptr && false_target_in != non_fallthrough_target) { 2535 __ b(false_target_in); 2536 } 2537 2538 return; 2539 } 2540 2541 // Generated branching requires both targets to be explicit. If either of the 2542 // targets is nullptr (fallthrough) use and bind `fallthrough_target` instead. 2543 Label fallthrough_target; 2544 Label* true_target = true_target_in == nullptr ? &fallthrough_target : true_target_in; 2545 Label* false_target = false_target_in == nullptr ? &fallthrough_target : false_target_in; 2546 2547 DCHECK_EQ(condition->InputAt(0)->GetType(), Primitive::kPrimLong); 2548 GenerateLongComparesAndJumps(condition, true_target, false_target); 2549 2550 if (false_target != &fallthrough_target) { 2551 __ b(false_target); 2552 } 2553 2554 if (fallthrough_target.IsLinked()) { 2555 __ Bind(&fallthrough_target); 2556 } 2557} 2558 2559void InstructionCodeGeneratorARM::GenerateTestAndBranch(HInstruction* instruction, 2560 size_t condition_input_index, 2561 Label* true_target, 2562 Label* false_target) { 2563 HInstruction* cond = instruction->InputAt(condition_input_index); 2564 2565 if (true_target == nullptr && false_target == nullptr) { 2566 // Nothing to do. The code always falls through. 2567 return; 2568 } else if (cond->IsIntConstant()) { 2569 // Constant condition, statically compared against "true" (integer value 1). 2570 if (cond->AsIntConstant()->IsTrue()) { 2571 if (true_target != nullptr) { 2572 __ b(true_target); 2573 } 2574 } else { 2575 DCHECK(cond->AsIntConstant()->IsFalse()) << cond->AsIntConstant()->GetValue(); 2576 if (false_target != nullptr) { 2577 __ b(false_target); 2578 } 2579 } 2580 return; 2581 } 2582 2583 // The following code generates these patterns: 2584 // (1) true_target == nullptr && false_target != nullptr 2585 // - opposite condition true => branch to false_target 2586 // (2) true_target != nullptr && false_target == nullptr 2587 // - condition true => branch to true_target 2588 // (3) true_target != nullptr && false_target != nullptr 2589 // - condition true => branch to true_target 2590 // - branch to false_target 2591 if (IsBooleanValueOrMaterializedCondition(cond)) { 2592 // Condition has been materialized, compare the output to 0. 2593 Location cond_val = instruction->GetLocations()->InAt(condition_input_index); 2594 DCHECK(cond_val.IsRegister()); 2595 if (true_target == nullptr) { 2596 __ CompareAndBranchIfZero(cond_val.AsRegister<Register>(), false_target); 2597 } else { 2598 __ CompareAndBranchIfNonZero(cond_val.AsRegister<Register>(), true_target); 2599 } 2600 } else { 2601 // Condition has not been materialized. Use its inputs as the comparison and 2602 // its condition as the branch condition. 2603 HCondition* condition = cond->AsCondition(); 2604 2605 // If this is a long or FP comparison that has been folded into 2606 // the HCondition, generate the comparison directly. 2607 Primitive::Type type = condition->InputAt(0)->GetType(); 2608 if (type == Primitive::kPrimLong || Primitive::IsFloatingPointType(type)) { 2609 GenerateCompareTestAndBranch(condition, true_target, false_target); 2610 return; 2611 } 2612 2613 Label* non_fallthrough_target; 2614 Condition arm_cond; 2615 LocationSummary* locations = cond->GetLocations(); 2616 DCHECK(locations->InAt(0).IsRegister()); 2617 Register left = locations->InAt(0).AsRegister<Register>(); 2618 Location right = locations->InAt(1); 2619 2620 if (true_target == nullptr) { 2621 arm_cond = ARMCondition(condition->GetOppositeCondition()); 2622 non_fallthrough_target = false_target; 2623 } else { 2624 arm_cond = ARMCondition(condition->GetCondition()); 2625 non_fallthrough_target = true_target; 2626 } 2627 2628 if (right.IsConstant() && (arm_cond == NE || arm_cond == EQ) && 2629 CodeGenerator::GetInt32ValueOf(right.GetConstant()) == 0) { 2630 if (arm_cond == EQ) { 2631 __ CompareAndBranchIfZero(left, non_fallthrough_target); 2632 } else { 2633 DCHECK_EQ(arm_cond, NE); 2634 __ CompareAndBranchIfNonZero(left, non_fallthrough_target); 2635 } 2636 } else { 2637 if (right.IsRegister()) { 2638 __ cmp(left, ShifterOperand(right.AsRegister<Register>())); 2639 } else { 2640 DCHECK(right.IsConstant()); 2641 __ CmpConstant(left, CodeGenerator::GetInt32ValueOf(right.GetConstant())); 2642 } 2643 2644 __ b(non_fallthrough_target, arm_cond); 2645 } 2646 } 2647 2648 // If neither branch falls through (case 3), the conditional branch to `true_target` 2649 // was already emitted (case 2) and we need to emit a jump to `false_target`. 2650 if (true_target != nullptr && false_target != nullptr) { 2651 __ b(false_target); 2652 } 2653} 2654 2655void LocationsBuilderARM::VisitIf(HIf* if_instr) { 2656 LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(if_instr); 2657 if (IsBooleanValueOrMaterializedCondition(if_instr->InputAt(0))) { 2658 locations->SetInAt(0, Location::RequiresRegister()); 2659 } 2660} 2661 2662void InstructionCodeGeneratorARM::VisitIf(HIf* if_instr) { 2663 HBasicBlock* true_successor = if_instr->IfTrueSuccessor(); 2664 HBasicBlock* false_successor = if_instr->IfFalseSuccessor(); 2665 Label* true_target = codegen_->GoesToNextBlock(if_instr->GetBlock(), true_successor) ? 2666 nullptr : codegen_->GetLabelOf(true_successor); 2667 Label* false_target = codegen_->GoesToNextBlock(if_instr->GetBlock(), false_successor) ? 2668 nullptr : codegen_->GetLabelOf(false_successor); 2669 GenerateTestAndBranch(if_instr, /* condition_input_index */ 0, true_target, false_target); 2670} 2671 2672void LocationsBuilderARM::VisitDeoptimize(HDeoptimize* deoptimize) { 2673 LocationSummary* locations = new (GetGraph()->GetArena()) 2674 LocationSummary(deoptimize, LocationSummary::kCallOnSlowPath); 2675 locations->SetCustomSlowPathCallerSaves(RegisterSet::Empty()); // No caller-save registers. 2676 if (IsBooleanValueOrMaterializedCondition(deoptimize->InputAt(0))) { 2677 locations->SetInAt(0, Location::RequiresRegister()); 2678 } 2679} 2680 2681void InstructionCodeGeneratorARM::VisitDeoptimize(HDeoptimize* deoptimize) { 2682 SlowPathCodeARM* slow_path = deopt_slow_paths_.NewSlowPath<DeoptimizationSlowPathARM>(deoptimize); 2683 GenerateTestAndBranch(deoptimize, 2684 /* condition_input_index */ 0, 2685 slow_path->GetEntryLabel(), 2686 /* false_target */ nullptr); 2687} 2688 2689void LocationsBuilderARM::VisitShouldDeoptimizeFlag(HShouldDeoptimizeFlag* flag) { 2690 LocationSummary* locations = new (GetGraph()->GetArena()) 2691 LocationSummary(flag, LocationSummary::kNoCall); 2692 locations->SetOut(Location::RequiresRegister()); 2693} 2694 2695void InstructionCodeGeneratorARM::VisitShouldDeoptimizeFlag(HShouldDeoptimizeFlag* flag) { 2696 __ LoadFromOffset(kLoadWord, 2697 flag->GetLocations()->Out().AsRegister<Register>(), 2698 SP, 2699 codegen_->GetStackOffsetOfShouldDeoptimizeFlag()); 2700} 2701 2702void LocationsBuilderARM::VisitSelect(HSelect* select) { 2703 LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(select); 2704 const bool is_floating_point = Primitive::IsFloatingPointType(select->GetType()); 2705 2706 if (is_floating_point) { 2707 locations->SetInAt(0, Location::RequiresFpuRegister()); 2708 locations->SetInAt(1, Location::FpuRegisterOrConstant(select->GetTrueValue())); 2709 } else { 2710 locations->SetInAt(0, Location::RequiresRegister()); 2711 locations->SetInAt(1, Arm8BitEncodableConstantOrRegister(select->GetTrueValue())); 2712 } 2713 2714 if (IsBooleanValueOrMaterializedCondition(select->GetCondition())) { 2715 locations->SetInAt(2, Location::RegisterOrConstant(select->GetCondition())); 2716 // The code generator handles overlap with the values, but not with the condition. 2717 locations->SetOut(Location::SameAsFirstInput()); 2718 } else if (is_floating_point) { 2719 locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap); 2720 } else { 2721 if (!locations->InAt(1).IsConstant()) { 2722 locations->SetInAt(0, Arm8BitEncodableConstantOrRegister(select->GetFalseValue())); 2723 } 2724 2725 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 2726 } 2727} 2728 2729void InstructionCodeGeneratorARM::VisitSelect(HSelect* select) { 2730 HInstruction* const condition = select->GetCondition(); 2731 const LocationSummary* const locations = select->GetLocations(); 2732 const Primitive::Type type = select->GetType(); 2733 const Location first = locations->InAt(0); 2734 const Location out = locations->Out(); 2735 const Location second = locations->InAt(1); 2736 Location src; 2737 2738 if (condition->IsIntConstant()) { 2739 if (condition->AsIntConstant()->IsFalse()) { 2740 src = first; 2741 } else { 2742 src = second; 2743 } 2744 2745 codegen_->MoveLocation(out, src, type); 2746 return; 2747 } 2748 2749 if (!Primitive::IsFloatingPointType(type) && 2750 (IsBooleanValueOrMaterializedCondition(condition) || 2751 CanGenerateTest(condition->AsCondition(), codegen_->GetAssembler()))) { 2752 bool invert = false; 2753 2754 if (out.Equals(second)) { 2755 src = first; 2756 invert = true; 2757 } else if (out.Equals(first)) { 2758 src = second; 2759 } else if (second.IsConstant()) { 2760 DCHECK(CanEncodeConstantAs8BitImmediate(second.GetConstant())); 2761 src = second; 2762 } else if (first.IsConstant()) { 2763 DCHECK(CanEncodeConstantAs8BitImmediate(first.GetConstant())); 2764 src = first; 2765 invert = true; 2766 } else { 2767 src = second; 2768 } 2769 2770 if (CanGenerateConditionalMove(out, src)) { 2771 if (!out.Equals(first) && !out.Equals(second)) { 2772 codegen_->MoveLocation(out, src.Equals(first) ? second : first, type); 2773 } 2774 2775 std::pair<Condition, Condition> cond; 2776 2777 if (IsBooleanValueOrMaterializedCondition(condition)) { 2778 __ CmpConstant(locations->InAt(2).AsRegister<Register>(), 0); 2779 cond = invert ? std::make_pair(EQ, NE) : std::make_pair(NE, EQ); 2780 } else { 2781 cond = GenerateTest(condition->AsCondition(), invert, codegen_); 2782 } 2783 2784 if (out.IsRegister()) { 2785 ShifterOperand operand; 2786 2787 if (src.IsConstant()) { 2788 operand = ShifterOperand(CodeGenerator::GetInt32ValueOf(src.GetConstant())); 2789 } else { 2790 DCHECK(src.IsRegister()); 2791 operand = ShifterOperand(src.AsRegister<Register>()); 2792 } 2793 2794 __ it(cond.first); 2795 __ mov(out.AsRegister<Register>(), operand, cond.first); 2796 } else { 2797 DCHECK(out.IsRegisterPair()); 2798 2799 ShifterOperand operand_high; 2800 ShifterOperand operand_low; 2801 2802 if (src.IsConstant()) { 2803 const int64_t value = src.GetConstant()->AsLongConstant()->GetValue(); 2804 2805 operand_high = ShifterOperand(High32Bits(value)); 2806 operand_low = ShifterOperand(Low32Bits(value)); 2807 } else { 2808 DCHECK(src.IsRegisterPair()); 2809 operand_high = ShifterOperand(src.AsRegisterPairHigh<Register>()); 2810 operand_low = ShifterOperand(src.AsRegisterPairLow<Register>()); 2811 } 2812 2813 __ it(cond.first); 2814 __ mov(out.AsRegisterPairLow<Register>(), operand_low, cond.first); 2815 __ it(cond.first); 2816 __ mov(out.AsRegisterPairHigh<Register>(), operand_high, cond.first); 2817 } 2818 2819 return; 2820 } 2821 } 2822 2823 Label* false_target = nullptr; 2824 Label* true_target = nullptr; 2825 Label select_end; 2826 Label* target = codegen_->GetFinalLabel(select, &select_end); 2827 2828 if (out.Equals(second)) { 2829 true_target = target; 2830 src = first; 2831 } else { 2832 false_target = target; 2833 src = second; 2834 2835 if (!out.Equals(first)) { 2836 codegen_->MoveLocation(out, first, type); 2837 } 2838 } 2839 2840 GenerateTestAndBranch(select, 2, true_target, false_target); 2841 codegen_->MoveLocation(out, src, type); 2842 2843 if (select_end.IsLinked()) { 2844 __ Bind(&select_end); 2845 } 2846} 2847 2848void LocationsBuilderARM::VisitNativeDebugInfo(HNativeDebugInfo* info) { 2849 new (GetGraph()->GetArena()) LocationSummary(info); 2850} 2851 2852void InstructionCodeGeneratorARM::VisitNativeDebugInfo(HNativeDebugInfo*) { 2853 // MaybeRecordNativeDebugInfo is already called implicitly in CodeGenerator::Compile. 2854} 2855 2856void CodeGeneratorARM::GenerateNop() { 2857 __ nop(); 2858} 2859 2860void LocationsBuilderARM::HandleCondition(HCondition* cond) { 2861 LocationSummary* locations = 2862 new (GetGraph()->GetArena()) LocationSummary(cond, LocationSummary::kNoCall); 2863 // Handle the long/FP comparisons made in instruction simplification. 2864 switch (cond->InputAt(0)->GetType()) { 2865 case Primitive::kPrimLong: 2866 locations->SetInAt(0, Location::RequiresRegister()); 2867 locations->SetInAt(1, Location::RegisterOrConstant(cond->InputAt(1))); 2868 if (!cond->IsEmittedAtUseSite()) { 2869 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 2870 } 2871 break; 2872 2873 case Primitive::kPrimFloat: 2874 case Primitive::kPrimDouble: 2875 locations->SetInAt(0, Location::RequiresFpuRegister()); 2876 locations->SetInAt(1, ArithmeticZeroOrFpuRegister(cond->InputAt(1))); 2877 if (!cond->IsEmittedAtUseSite()) { 2878 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 2879 } 2880 break; 2881 2882 default: 2883 locations->SetInAt(0, Location::RequiresRegister()); 2884 locations->SetInAt(1, Location::RegisterOrConstant(cond->InputAt(1))); 2885 if (!cond->IsEmittedAtUseSite()) { 2886 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 2887 } 2888 } 2889} 2890 2891void InstructionCodeGeneratorARM::HandleCondition(HCondition* cond) { 2892 if (cond->IsEmittedAtUseSite()) { 2893 return; 2894 } 2895 2896 const Register out = cond->GetLocations()->Out().AsRegister<Register>(); 2897 2898 if (ArmAssembler::IsLowRegister(out) && CanGenerateTest(cond, codegen_->GetAssembler())) { 2899 const auto condition = GenerateTest(cond, false, codegen_); 2900 2901 __ it(condition.first); 2902 __ mov(out, ShifterOperand(1), condition.first); 2903 __ it(condition.second); 2904 __ mov(out, ShifterOperand(0), condition.second); 2905 return; 2906 } 2907 2908 // Convert the jumps into the result. 2909 Label done_label; 2910 Label* const final_label = codegen_->GetFinalLabel(cond, &done_label); 2911 2912 if (cond->InputAt(0)->GetType() == Primitive::kPrimLong) { 2913 Label true_label, false_label; 2914 2915 GenerateLongComparesAndJumps(cond, &true_label, &false_label); 2916 2917 // False case: result = 0. 2918 __ Bind(&false_label); 2919 __ LoadImmediate(out, 0); 2920 __ b(final_label); 2921 2922 // True case: result = 1. 2923 __ Bind(&true_label); 2924 __ LoadImmediate(out, 1); 2925 } else { 2926 DCHECK(CanGenerateTest(cond, codegen_->GetAssembler())); 2927 2928 const auto condition = GenerateTest(cond, false, codegen_); 2929 2930 __ mov(out, ShifterOperand(0), AL, kCcKeep); 2931 __ b(final_label, condition.second); 2932 __ LoadImmediate(out, 1); 2933 } 2934 2935 if (done_label.IsLinked()) { 2936 __ Bind(&done_label); 2937 } 2938} 2939 2940void LocationsBuilderARM::VisitEqual(HEqual* comp) { 2941 HandleCondition(comp); 2942} 2943 2944void InstructionCodeGeneratorARM::VisitEqual(HEqual* comp) { 2945 HandleCondition(comp); 2946} 2947 2948void LocationsBuilderARM::VisitNotEqual(HNotEqual* comp) { 2949 HandleCondition(comp); 2950} 2951 2952void InstructionCodeGeneratorARM::VisitNotEqual(HNotEqual* comp) { 2953 HandleCondition(comp); 2954} 2955 2956void LocationsBuilderARM::VisitLessThan(HLessThan* comp) { 2957 HandleCondition(comp); 2958} 2959 2960void InstructionCodeGeneratorARM::VisitLessThan(HLessThan* comp) { 2961 HandleCondition(comp); 2962} 2963 2964void LocationsBuilderARM::VisitLessThanOrEqual(HLessThanOrEqual* comp) { 2965 HandleCondition(comp); 2966} 2967 2968void InstructionCodeGeneratorARM::VisitLessThanOrEqual(HLessThanOrEqual* comp) { 2969 HandleCondition(comp); 2970} 2971 2972void LocationsBuilderARM::VisitGreaterThan(HGreaterThan* comp) { 2973 HandleCondition(comp); 2974} 2975 2976void InstructionCodeGeneratorARM::VisitGreaterThan(HGreaterThan* comp) { 2977 HandleCondition(comp); 2978} 2979 2980void LocationsBuilderARM::VisitGreaterThanOrEqual(HGreaterThanOrEqual* comp) { 2981 HandleCondition(comp); 2982} 2983 2984void InstructionCodeGeneratorARM::VisitGreaterThanOrEqual(HGreaterThanOrEqual* comp) { 2985 HandleCondition(comp); 2986} 2987 2988void LocationsBuilderARM::VisitBelow(HBelow* comp) { 2989 HandleCondition(comp); 2990} 2991 2992void InstructionCodeGeneratorARM::VisitBelow(HBelow* comp) { 2993 HandleCondition(comp); 2994} 2995 2996void LocationsBuilderARM::VisitBelowOrEqual(HBelowOrEqual* comp) { 2997 HandleCondition(comp); 2998} 2999 3000void InstructionCodeGeneratorARM::VisitBelowOrEqual(HBelowOrEqual* comp) { 3001 HandleCondition(comp); 3002} 3003 3004void LocationsBuilderARM::VisitAbove(HAbove* comp) { 3005 HandleCondition(comp); 3006} 3007 3008void InstructionCodeGeneratorARM::VisitAbove(HAbove* comp) { 3009 HandleCondition(comp); 3010} 3011 3012void LocationsBuilderARM::VisitAboveOrEqual(HAboveOrEqual* comp) { 3013 HandleCondition(comp); 3014} 3015 3016void InstructionCodeGeneratorARM::VisitAboveOrEqual(HAboveOrEqual* comp) { 3017 HandleCondition(comp); 3018} 3019 3020void LocationsBuilderARM::VisitIntConstant(HIntConstant* constant) { 3021 LocationSummary* locations = 3022 new (GetGraph()->GetArena()) LocationSummary(constant, LocationSummary::kNoCall); 3023 locations->SetOut(Location::ConstantLocation(constant)); 3024} 3025 3026void InstructionCodeGeneratorARM::VisitIntConstant(HIntConstant* constant ATTRIBUTE_UNUSED) { 3027 // Will be generated at use site. 3028} 3029 3030void LocationsBuilderARM::VisitNullConstant(HNullConstant* constant) { 3031 LocationSummary* locations = 3032 new (GetGraph()->GetArena()) LocationSummary(constant, LocationSummary::kNoCall); 3033 locations->SetOut(Location::ConstantLocation(constant)); 3034} 3035 3036void InstructionCodeGeneratorARM::VisitNullConstant(HNullConstant* constant ATTRIBUTE_UNUSED) { 3037 // Will be generated at use site. 3038} 3039 3040void LocationsBuilderARM::VisitLongConstant(HLongConstant* constant) { 3041 LocationSummary* locations = 3042 new (GetGraph()->GetArena()) LocationSummary(constant, LocationSummary::kNoCall); 3043 locations->SetOut(Location::ConstantLocation(constant)); 3044} 3045 3046void InstructionCodeGeneratorARM::VisitLongConstant(HLongConstant* constant ATTRIBUTE_UNUSED) { 3047 // Will be generated at use site. 3048} 3049 3050void LocationsBuilderARM::VisitFloatConstant(HFloatConstant* constant) { 3051 LocationSummary* locations = 3052 new (GetGraph()->GetArena()) LocationSummary(constant, LocationSummary::kNoCall); 3053 locations->SetOut(Location::ConstantLocation(constant)); 3054} 3055 3056void InstructionCodeGeneratorARM::VisitFloatConstant(HFloatConstant* constant ATTRIBUTE_UNUSED) { 3057 // Will be generated at use site. 3058} 3059 3060void LocationsBuilderARM::VisitDoubleConstant(HDoubleConstant* constant) { 3061 LocationSummary* locations = 3062 new (GetGraph()->GetArena()) LocationSummary(constant, LocationSummary::kNoCall); 3063 locations->SetOut(Location::ConstantLocation(constant)); 3064} 3065 3066void InstructionCodeGeneratorARM::VisitDoubleConstant(HDoubleConstant* constant ATTRIBUTE_UNUSED) { 3067 // Will be generated at use site. 3068} 3069 3070void LocationsBuilderARM::VisitMemoryBarrier(HMemoryBarrier* memory_barrier) { 3071 memory_barrier->SetLocations(nullptr); 3072} 3073 3074void InstructionCodeGeneratorARM::VisitMemoryBarrier(HMemoryBarrier* memory_barrier) { 3075 codegen_->GenerateMemoryBarrier(memory_barrier->GetBarrierKind()); 3076} 3077 3078void LocationsBuilderARM::VisitReturnVoid(HReturnVoid* ret) { 3079 ret->SetLocations(nullptr); 3080} 3081 3082void InstructionCodeGeneratorARM::VisitReturnVoid(HReturnVoid* ret ATTRIBUTE_UNUSED) { 3083 codegen_->GenerateFrameExit(); 3084} 3085 3086void LocationsBuilderARM::VisitReturn(HReturn* ret) { 3087 LocationSummary* locations = 3088 new (GetGraph()->GetArena()) LocationSummary(ret, LocationSummary::kNoCall); 3089 locations->SetInAt(0, parameter_visitor_.GetReturnLocation(ret->InputAt(0)->GetType())); 3090} 3091 3092void InstructionCodeGeneratorARM::VisitReturn(HReturn* ret ATTRIBUTE_UNUSED) { 3093 codegen_->GenerateFrameExit(); 3094} 3095 3096void LocationsBuilderARM::VisitInvokeUnresolved(HInvokeUnresolved* invoke) { 3097 // The trampoline uses the same calling convention as dex calling conventions, 3098 // except instead of loading arg0/r0 with the target Method*, arg0/r0 will contain 3099 // the method_idx. 3100 HandleInvoke(invoke); 3101} 3102 3103void InstructionCodeGeneratorARM::VisitInvokeUnresolved(HInvokeUnresolved* invoke) { 3104 codegen_->GenerateInvokeUnresolvedRuntimeCall(invoke); 3105} 3106 3107void LocationsBuilderARM::VisitInvokeStaticOrDirect(HInvokeStaticOrDirect* invoke) { 3108 // Explicit clinit checks triggered by static invokes must have been pruned by 3109 // art::PrepareForRegisterAllocation. 3110 DCHECK(!invoke->IsStaticWithExplicitClinitCheck()); 3111 3112 IntrinsicLocationsBuilderARM intrinsic(codegen_); 3113 if (intrinsic.TryDispatch(invoke)) { 3114 if (invoke->GetLocations()->CanCall() && invoke->HasPcRelativeDexCache()) { 3115 invoke->GetLocations()->SetInAt(invoke->GetSpecialInputIndex(), Location::Any()); 3116 } 3117 return; 3118 } 3119 3120 HandleInvoke(invoke); 3121 3122 // For PC-relative dex cache the invoke has an extra input, the PC-relative address base. 3123 if (invoke->HasPcRelativeDexCache()) { 3124 invoke->GetLocations()->SetInAt(invoke->GetSpecialInputIndex(), Location::RequiresRegister()); 3125 } 3126} 3127 3128static bool TryGenerateIntrinsicCode(HInvoke* invoke, CodeGeneratorARM* codegen) { 3129 if (invoke->GetLocations()->Intrinsified()) { 3130 IntrinsicCodeGeneratorARM intrinsic(codegen); 3131 intrinsic.Dispatch(invoke); 3132 return true; 3133 } 3134 return false; 3135} 3136 3137void InstructionCodeGeneratorARM::VisitInvokeStaticOrDirect(HInvokeStaticOrDirect* invoke) { 3138 // Explicit clinit checks triggered by static invokes must have been pruned by 3139 // art::PrepareForRegisterAllocation. 3140 DCHECK(!invoke->IsStaticWithExplicitClinitCheck()); 3141 3142 if (TryGenerateIntrinsicCode(invoke, codegen_)) { 3143 return; 3144 } 3145 3146 LocationSummary* locations = invoke->GetLocations(); 3147 codegen_->GenerateStaticOrDirectCall( 3148 invoke, locations->HasTemps() ? locations->GetTemp(0) : Location::NoLocation()); 3149 codegen_->RecordPcInfo(invoke, invoke->GetDexPc()); 3150} 3151 3152void LocationsBuilderARM::HandleInvoke(HInvoke* invoke) { 3153 InvokeDexCallingConventionVisitorARM calling_convention_visitor; 3154 CodeGenerator::CreateCommonInvokeLocationSummary(invoke, &calling_convention_visitor); 3155} 3156 3157void LocationsBuilderARM::VisitInvokeVirtual(HInvokeVirtual* invoke) { 3158 IntrinsicLocationsBuilderARM intrinsic(codegen_); 3159 if (intrinsic.TryDispatch(invoke)) { 3160 return; 3161 } 3162 3163 HandleInvoke(invoke); 3164} 3165 3166void InstructionCodeGeneratorARM::VisitInvokeVirtual(HInvokeVirtual* invoke) { 3167 if (TryGenerateIntrinsicCode(invoke, codegen_)) { 3168 return; 3169 } 3170 3171 codegen_->GenerateVirtualCall(invoke, invoke->GetLocations()->GetTemp(0)); 3172 DCHECK(!codegen_->IsLeafMethod()); 3173 codegen_->RecordPcInfo(invoke, invoke->GetDexPc()); 3174} 3175 3176void LocationsBuilderARM::VisitInvokeInterface(HInvokeInterface* invoke) { 3177 HandleInvoke(invoke); 3178 // Add the hidden argument. 3179 invoke->GetLocations()->AddTemp(Location::RegisterLocation(R12)); 3180} 3181 3182void InstructionCodeGeneratorARM::VisitInvokeInterface(HInvokeInterface* invoke) { 3183 // TODO: b/18116999, our IMTs can miss an IncompatibleClassChangeError. 3184 LocationSummary* locations = invoke->GetLocations(); 3185 Register temp = locations->GetTemp(0).AsRegister<Register>(); 3186 Register hidden_reg = locations->GetTemp(1).AsRegister<Register>(); 3187 Location receiver = locations->InAt(0); 3188 uint32_t class_offset = mirror::Object::ClassOffset().Int32Value(); 3189 3190 // Set the hidden argument. This is safe to do this here, as R12 3191 // won't be modified thereafter, before the `blx` (call) instruction. 3192 DCHECK_EQ(R12, hidden_reg); 3193 __ LoadImmediate(hidden_reg, invoke->GetDexMethodIndex()); 3194 3195 if (receiver.IsStackSlot()) { 3196 __ LoadFromOffset(kLoadWord, temp, SP, receiver.GetStackIndex()); 3197 // /* HeapReference<Class> */ temp = temp->klass_ 3198 __ LoadFromOffset(kLoadWord, temp, temp, class_offset); 3199 } else { 3200 // /* HeapReference<Class> */ temp = receiver->klass_ 3201 __ LoadFromOffset(kLoadWord, temp, receiver.AsRegister<Register>(), class_offset); 3202 } 3203 codegen_->MaybeRecordImplicitNullCheck(invoke); 3204 // Instead of simply (possibly) unpoisoning `temp` here, we should 3205 // emit a read barrier for the previous class reference load. 3206 // However this is not required in practice, as this is an 3207 // intermediate/temporary reference and because the current 3208 // concurrent copying collector keeps the from-space memory 3209 // intact/accessible until the end of the marking phase (the 3210 // concurrent copying collector may not in the future). 3211 __ MaybeUnpoisonHeapReference(temp); 3212 __ LoadFromOffset(kLoadWord, temp, temp, 3213 mirror::Class::ImtPtrOffset(kArmPointerSize).Uint32Value()); 3214 uint32_t method_offset = static_cast<uint32_t>(ImTable::OffsetOfElement( 3215 invoke->GetImtIndex(), kArmPointerSize)); 3216 // temp = temp->GetImtEntryAt(method_offset); 3217 __ LoadFromOffset(kLoadWord, temp, temp, method_offset); 3218 uint32_t entry_point = 3219 ArtMethod::EntryPointFromQuickCompiledCodeOffset(kArmPointerSize).Int32Value(); 3220 // LR = temp->GetEntryPoint(); 3221 __ LoadFromOffset(kLoadWord, LR, temp, entry_point); 3222 // LR(); 3223 __ blx(LR); 3224 DCHECK(!codegen_->IsLeafMethod()); 3225 codegen_->RecordPcInfo(invoke, invoke->GetDexPc()); 3226} 3227 3228void LocationsBuilderARM::VisitInvokePolymorphic(HInvokePolymorphic* invoke) { 3229 HandleInvoke(invoke); 3230} 3231 3232void InstructionCodeGeneratorARM::VisitInvokePolymorphic(HInvokePolymorphic* invoke) { 3233 codegen_->GenerateInvokePolymorphicCall(invoke); 3234} 3235 3236void LocationsBuilderARM::VisitNeg(HNeg* neg) { 3237 LocationSummary* locations = 3238 new (GetGraph()->GetArena()) LocationSummary(neg, LocationSummary::kNoCall); 3239 switch (neg->GetResultType()) { 3240 case Primitive::kPrimInt: { 3241 locations->SetInAt(0, Location::RequiresRegister()); 3242 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 3243 break; 3244 } 3245 case Primitive::kPrimLong: { 3246 locations->SetInAt(0, Location::RequiresRegister()); 3247 locations->SetOut(Location::RequiresRegister(), Location::kOutputOverlap); 3248 break; 3249 } 3250 3251 case Primitive::kPrimFloat: 3252 case Primitive::kPrimDouble: 3253 locations->SetInAt(0, Location::RequiresFpuRegister()); 3254 locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap); 3255 break; 3256 3257 default: 3258 LOG(FATAL) << "Unexpected neg type " << neg->GetResultType(); 3259 } 3260} 3261 3262void InstructionCodeGeneratorARM::VisitNeg(HNeg* neg) { 3263 LocationSummary* locations = neg->GetLocations(); 3264 Location out = locations->Out(); 3265 Location in = locations->InAt(0); 3266 switch (neg->GetResultType()) { 3267 case Primitive::kPrimInt: 3268 DCHECK(in.IsRegister()); 3269 __ rsb(out.AsRegister<Register>(), in.AsRegister<Register>(), ShifterOperand(0)); 3270 break; 3271 3272 case Primitive::kPrimLong: 3273 DCHECK(in.IsRegisterPair()); 3274 // out.lo = 0 - in.lo (and update the carry/borrow (C) flag) 3275 __ rsbs(out.AsRegisterPairLow<Register>(), 3276 in.AsRegisterPairLow<Register>(), 3277 ShifterOperand(0)); 3278 // We cannot emit an RSC (Reverse Subtract with Carry) 3279 // instruction here, as it does not exist in the Thumb-2 3280 // instruction set. We use the following approach 3281 // using SBC and SUB instead. 3282 // 3283 // out.hi = -C 3284 __ sbc(out.AsRegisterPairHigh<Register>(), 3285 out.AsRegisterPairHigh<Register>(), 3286 ShifterOperand(out.AsRegisterPairHigh<Register>())); 3287 // out.hi = out.hi - in.hi 3288 __ sub(out.AsRegisterPairHigh<Register>(), 3289 out.AsRegisterPairHigh<Register>(), 3290 ShifterOperand(in.AsRegisterPairHigh<Register>())); 3291 break; 3292 3293 case Primitive::kPrimFloat: 3294 DCHECK(in.IsFpuRegister()); 3295 __ vnegs(out.AsFpuRegister<SRegister>(), in.AsFpuRegister<SRegister>()); 3296 break; 3297 3298 case Primitive::kPrimDouble: 3299 DCHECK(in.IsFpuRegisterPair()); 3300 __ vnegd(FromLowSToD(out.AsFpuRegisterPairLow<SRegister>()), 3301 FromLowSToD(in.AsFpuRegisterPairLow<SRegister>())); 3302 break; 3303 3304 default: 3305 LOG(FATAL) << "Unexpected neg type " << neg->GetResultType(); 3306 } 3307} 3308 3309void LocationsBuilderARM::VisitTypeConversion(HTypeConversion* conversion) { 3310 Primitive::Type result_type = conversion->GetResultType(); 3311 Primitive::Type input_type = conversion->GetInputType(); 3312 DCHECK_NE(result_type, input_type); 3313 3314 // The float-to-long, double-to-long and long-to-float type conversions 3315 // rely on a call to the runtime. 3316 LocationSummary::CallKind call_kind = 3317 (((input_type == Primitive::kPrimFloat || input_type == Primitive::kPrimDouble) 3318 && result_type == Primitive::kPrimLong) 3319 || (input_type == Primitive::kPrimLong && result_type == Primitive::kPrimFloat)) 3320 ? LocationSummary::kCallOnMainOnly 3321 : LocationSummary::kNoCall; 3322 LocationSummary* locations = 3323 new (GetGraph()->GetArena()) LocationSummary(conversion, call_kind); 3324 3325 // The Java language does not allow treating boolean as an integral type but 3326 // our bit representation makes it safe. 3327 3328 switch (result_type) { 3329 case Primitive::kPrimByte: 3330 switch (input_type) { 3331 case Primitive::kPrimLong: 3332 // Type conversion from long to byte is a result of code transformations. 3333 case Primitive::kPrimBoolean: 3334 // Boolean input is a result of code transformations. 3335 case Primitive::kPrimShort: 3336 case Primitive::kPrimInt: 3337 case Primitive::kPrimChar: 3338 // Processing a Dex `int-to-byte' instruction. 3339 locations->SetInAt(0, Location::RequiresRegister()); 3340 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 3341 break; 3342 3343 default: 3344 LOG(FATAL) << "Unexpected type conversion from " << input_type 3345 << " to " << result_type; 3346 } 3347 break; 3348 3349 case Primitive::kPrimShort: 3350 switch (input_type) { 3351 case Primitive::kPrimLong: 3352 // Type conversion from long to short is a result of code transformations. 3353 case Primitive::kPrimBoolean: 3354 // Boolean input is a result of code transformations. 3355 case Primitive::kPrimByte: 3356 case Primitive::kPrimInt: 3357 case Primitive::kPrimChar: 3358 // Processing a Dex `int-to-short' instruction. 3359 locations->SetInAt(0, Location::RequiresRegister()); 3360 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 3361 break; 3362 3363 default: 3364 LOG(FATAL) << "Unexpected type conversion from " << input_type 3365 << " to " << result_type; 3366 } 3367 break; 3368 3369 case Primitive::kPrimInt: 3370 switch (input_type) { 3371 case Primitive::kPrimLong: 3372 // Processing a Dex `long-to-int' instruction. 3373 locations->SetInAt(0, Location::Any()); 3374 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 3375 break; 3376 3377 case Primitive::kPrimFloat: 3378 // Processing a Dex `float-to-int' instruction. 3379 locations->SetInAt(0, Location::RequiresFpuRegister()); 3380 locations->SetOut(Location::RequiresRegister()); 3381 locations->AddTemp(Location::RequiresFpuRegister()); 3382 break; 3383 3384 case Primitive::kPrimDouble: 3385 // Processing a Dex `double-to-int' instruction. 3386 locations->SetInAt(0, Location::RequiresFpuRegister()); 3387 locations->SetOut(Location::RequiresRegister()); 3388 locations->AddTemp(Location::RequiresFpuRegister()); 3389 break; 3390 3391 default: 3392 LOG(FATAL) << "Unexpected type conversion from " << input_type 3393 << " to " << result_type; 3394 } 3395 break; 3396 3397 case Primitive::kPrimLong: 3398 switch (input_type) { 3399 case Primitive::kPrimBoolean: 3400 // Boolean input is a result of code transformations. 3401 case Primitive::kPrimByte: 3402 case Primitive::kPrimShort: 3403 case Primitive::kPrimInt: 3404 case Primitive::kPrimChar: 3405 // Processing a Dex `int-to-long' instruction. 3406 locations->SetInAt(0, Location::RequiresRegister()); 3407 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 3408 break; 3409 3410 case Primitive::kPrimFloat: { 3411 // Processing a Dex `float-to-long' instruction. 3412 InvokeRuntimeCallingConvention calling_convention; 3413 locations->SetInAt(0, Location::FpuRegisterLocation( 3414 calling_convention.GetFpuRegisterAt(0))); 3415 locations->SetOut(Location::RegisterPairLocation(R0, R1)); 3416 break; 3417 } 3418 3419 case Primitive::kPrimDouble: { 3420 // Processing a Dex `double-to-long' instruction. 3421 InvokeRuntimeCallingConvention calling_convention; 3422 locations->SetInAt(0, Location::FpuRegisterPairLocation( 3423 calling_convention.GetFpuRegisterAt(0), 3424 calling_convention.GetFpuRegisterAt(1))); 3425 locations->SetOut(Location::RegisterPairLocation(R0, R1)); 3426 break; 3427 } 3428 3429 default: 3430 LOG(FATAL) << "Unexpected type conversion from " << input_type 3431 << " to " << result_type; 3432 } 3433 break; 3434 3435 case Primitive::kPrimChar: 3436 switch (input_type) { 3437 case Primitive::kPrimLong: 3438 // Type conversion from long to char is a result of code transformations. 3439 case Primitive::kPrimBoolean: 3440 // Boolean input is a result of code transformations. 3441 case Primitive::kPrimByte: 3442 case Primitive::kPrimShort: 3443 case Primitive::kPrimInt: 3444 // Processing a Dex `int-to-char' instruction. 3445 locations->SetInAt(0, Location::RequiresRegister()); 3446 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 3447 break; 3448 3449 default: 3450 LOG(FATAL) << "Unexpected type conversion from " << input_type 3451 << " to " << result_type; 3452 } 3453 break; 3454 3455 case Primitive::kPrimFloat: 3456 switch (input_type) { 3457 case Primitive::kPrimBoolean: 3458 // Boolean input is a result of code transformations. 3459 case Primitive::kPrimByte: 3460 case Primitive::kPrimShort: 3461 case Primitive::kPrimInt: 3462 case Primitive::kPrimChar: 3463 // Processing a Dex `int-to-float' instruction. 3464 locations->SetInAt(0, Location::RequiresRegister()); 3465 locations->SetOut(Location::RequiresFpuRegister()); 3466 break; 3467 3468 case Primitive::kPrimLong: { 3469 // Processing a Dex `long-to-float' instruction. 3470 InvokeRuntimeCallingConvention calling_convention; 3471 locations->SetInAt(0, Location::RegisterPairLocation( 3472 calling_convention.GetRegisterAt(0), calling_convention.GetRegisterAt(1))); 3473 locations->SetOut(Location::FpuRegisterLocation(calling_convention.GetFpuRegisterAt(0))); 3474 break; 3475 } 3476 3477 case Primitive::kPrimDouble: 3478 // Processing a Dex `double-to-float' instruction. 3479 locations->SetInAt(0, Location::RequiresFpuRegister()); 3480 locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap); 3481 break; 3482 3483 default: 3484 LOG(FATAL) << "Unexpected type conversion from " << input_type 3485 << " to " << result_type; 3486 }; 3487 break; 3488 3489 case Primitive::kPrimDouble: 3490 switch (input_type) { 3491 case Primitive::kPrimBoolean: 3492 // Boolean input is a result of code transformations. 3493 case Primitive::kPrimByte: 3494 case Primitive::kPrimShort: 3495 case Primitive::kPrimInt: 3496 case Primitive::kPrimChar: 3497 // Processing a Dex `int-to-double' instruction. 3498 locations->SetInAt(0, Location::RequiresRegister()); 3499 locations->SetOut(Location::RequiresFpuRegister()); 3500 break; 3501 3502 case Primitive::kPrimLong: 3503 // Processing a Dex `long-to-double' instruction. 3504 locations->SetInAt(0, Location::RequiresRegister()); 3505 locations->SetOut(Location::RequiresFpuRegister()); 3506 locations->AddTemp(Location::RequiresFpuRegister()); 3507 locations->AddTemp(Location::RequiresFpuRegister()); 3508 break; 3509 3510 case Primitive::kPrimFloat: 3511 // Processing a Dex `float-to-double' instruction. 3512 locations->SetInAt(0, Location::RequiresFpuRegister()); 3513 locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap); 3514 break; 3515 3516 default: 3517 LOG(FATAL) << "Unexpected type conversion from " << input_type 3518 << " to " << result_type; 3519 }; 3520 break; 3521 3522 default: 3523 LOG(FATAL) << "Unexpected type conversion from " << input_type 3524 << " to " << result_type; 3525 } 3526} 3527 3528void InstructionCodeGeneratorARM::VisitTypeConversion(HTypeConversion* conversion) { 3529 LocationSummary* locations = conversion->GetLocations(); 3530 Location out = locations->Out(); 3531 Location in = locations->InAt(0); 3532 Primitive::Type result_type = conversion->GetResultType(); 3533 Primitive::Type input_type = conversion->GetInputType(); 3534 DCHECK_NE(result_type, input_type); 3535 switch (result_type) { 3536 case Primitive::kPrimByte: 3537 switch (input_type) { 3538 case Primitive::kPrimLong: 3539 // Type conversion from long to byte is a result of code transformations. 3540 __ sbfx(out.AsRegister<Register>(), in.AsRegisterPairLow<Register>(), 0, 8); 3541 break; 3542 case Primitive::kPrimBoolean: 3543 // Boolean input is a result of code transformations. 3544 case Primitive::kPrimShort: 3545 case Primitive::kPrimInt: 3546 case Primitive::kPrimChar: 3547 // Processing a Dex `int-to-byte' instruction. 3548 __ sbfx(out.AsRegister<Register>(), in.AsRegister<Register>(), 0, 8); 3549 break; 3550 3551 default: 3552 LOG(FATAL) << "Unexpected type conversion from " << input_type 3553 << " to " << result_type; 3554 } 3555 break; 3556 3557 case Primitive::kPrimShort: 3558 switch (input_type) { 3559 case Primitive::kPrimLong: 3560 // Type conversion from long to short is a result of code transformations. 3561 __ sbfx(out.AsRegister<Register>(), in.AsRegisterPairLow<Register>(), 0, 16); 3562 break; 3563 case Primitive::kPrimBoolean: 3564 // Boolean input is a result of code transformations. 3565 case Primitive::kPrimByte: 3566 case Primitive::kPrimInt: 3567 case Primitive::kPrimChar: 3568 // Processing a Dex `int-to-short' instruction. 3569 __ sbfx(out.AsRegister<Register>(), in.AsRegister<Register>(), 0, 16); 3570 break; 3571 3572 default: 3573 LOG(FATAL) << "Unexpected type conversion from " << input_type 3574 << " to " << result_type; 3575 } 3576 break; 3577 3578 case Primitive::kPrimInt: 3579 switch (input_type) { 3580 case Primitive::kPrimLong: 3581 // Processing a Dex `long-to-int' instruction. 3582 DCHECK(out.IsRegister()); 3583 if (in.IsRegisterPair()) { 3584 __ Mov(out.AsRegister<Register>(), in.AsRegisterPairLow<Register>()); 3585 } else if (in.IsDoubleStackSlot()) { 3586 __ LoadFromOffset(kLoadWord, out.AsRegister<Register>(), SP, in.GetStackIndex()); 3587 } else { 3588 DCHECK(in.IsConstant()); 3589 DCHECK(in.GetConstant()->IsLongConstant()); 3590 int64_t value = in.GetConstant()->AsLongConstant()->GetValue(); 3591 __ LoadImmediate(out.AsRegister<Register>(), static_cast<int32_t>(value)); 3592 } 3593 break; 3594 3595 case Primitive::kPrimFloat: { 3596 // Processing a Dex `float-to-int' instruction. 3597 SRegister temp = locations->GetTemp(0).AsFpuRegisterPairLow<SRegister>(); 3598 __ vcvtis(temp, in.AsFpuRegister<SRegister>()); 3599 __ vmovrs(out.AsRegister<Register>(), temp); 3600 break; 3601 } 3602 3603 case Primitive::kPrimDouble: { 3604 // Processing a Dex `double-to-int' instruction. 3605 SRegister temp_s = locations->GetTemp(0).AsFpuRegisterPairLow<SRegister>(); 3606 __ vcvtid(temp_s, FromLowSToD(in.AsFpuRegisterPairLow<SRegister>())); 3607 __ vmovrs(out.AsRegister<Register>(), temp_s); 3608 break; 3609 } 3610 3611 default: 3612 LOG(FATAL) << "Unexpected type conversion from " << input_type 3613 << " to " << result_type; 3614 } 3615 break; 3616 3617 case Primitive::kPrimLong: 3618 switch (input_type) { 3619 case Primitive::kPrimBoolean: 3620 // Boolean input is a result of code transformations. 3621 case Primitive::kPrimByte: 3622 case Primitive::kPrimShort: 3623 case Primitive::kPrimInt: 3624 case Primitive::kPrimChar: 3625 // Processing a Dex `int-to-long' instruction. 3626 DCHECK(out.IsRegisterPair()); 3627 DCHECK(in.IsRegister()); 3628 __ Mov(out.AsRegisterPairLow<Register>(), in.AsRegister<Register>()); 3629 // Sign extension. 3630 __ Asr(out.AsRegisterPairHigh<Register>(), 3631 out.AsRegisterPairLow<Register>(), 3632 31); 3633 break; 3634 3635 case Primitive::kPrimFloat: 3636 // Processing a Dex `float-to-long' instruction. 3637 codegen_->InvokeRuntime(kQuickF2l, conversion, conversion->GetDexPc()); 3638 CheckEntrypointTypes<kQuickF2l, int64_t, float>(); 3639 break; 3640 3641 case Primitive::kPrimDouble: 3642 // Processing a Dex `double-to-long' instruction. 3643 codegen_->InvokeRuntime(kQuickD2l, conversion, conversion->GetDexPc()); 3644 CheckEntrypointTypes<kQuickD2l, int64_t, double>(); 3645 break; 3646 3647 default: 3648 LOG(FATAL) << "Unexpected type conversion from " << input_type 3649 << " to " << result_type; 3650 } 3651 break; 3652 3653 case Primitive::kPrimChar: 3654 switch (input_type) { 3655 case Primitive::kPrimLong: 3656 // Type conversion from long to char is a result of code transformations. 3657 __ ubfx(out.AsRegister<Register>(), in.AsRegisterPairLow<Register>(), 0, 16); 3658 break; 3659 case Primitive::kPrimBoolean: 3660 // Boolean input is a result of code transformations. 3661 case Primitive::kPrimByte: 3662 case Primitive::kPrimShort: 3663 case Primitive::kPrimInt: 3664 // Processing a Dex `int-to-char' instruction. 3665 __ ubfx(out.AsRegister<Register>(), in.AsRegister<Register>(), 0, 16); 3666 break; 3667 3668 default: 3669 LOG(FATAL) << "Unexpected type conversion from " << input_type 3670 << " to " << result_type; 3671 } 3672 break; 3673 3674 case Primitive::kPrimFloat: 3675 switch (input_type) { 3676 case Primitive::kPrimBoolean: 3677 // Boolean input is a result of code transformations. 3678 case Primitive::kPrimByte: 3679 case Primitive::kPrimShort: 3680 case Primitive::kPrimInt: 3681 case Primitive::kPrimChar: { 3682 // Processing a Dex `int-to-float' instruction. 3683 __ vmovsr(out.AsFpuRegister<SRegister>(), in.AsRegister<Register>()); 3684 __ vcvtsi(out.AsFpuRegister<SRegister>(), out.AsFpuRegister<SRegister>()); 3685 break; 3686 } 3687 3688 case Primitive::kPrimLong: 3689 // Processing a Dex `long-to-float' instruction. 3690 codegen_->InvokeRuntime(kQuickL2f, conversion, conversion->GetDexPc()); 3691 CheckEntrypointTypes<kQuickL2f, float, int64_t>(); 3692 break; 3693 3694 case Primitive::kPrimDouble: 3695 // Processing a Dex `double-to-float' instruction. 3696 __ vcvtsd(out.AsFpuRegister<SRegister>(), 3697 FromLowSToD(in.AsFpuRegisterPairLow<SRegister>())); 3698 break; 3699 3700 default: 3701 LOG(FATAL) << "Unexpected type conversion from " << input_type 3702 << " to " << result_type; 3703 }; 3704 break; 3705 3706 case Primitive::kPrimDouble: 3707 switch (input_type) { 3708 case Primitive::kPrimBoolean: 3709 // Boolean input is a result of code transformations. 3710 case Primitive::kPrimByte: 3711 case Primitive::kPrimShort: 3712 case Primitive::kPrimInt: 3713 case Primitive::kPrimChar: { 3714 // Processing a Dex `int-to-double' instruction. 3715 __ vmovsr(out.AsFpuRegisterPairLow<SRegister>(), in.AsRegister<Register>()); 3716 __ vcvtdi(FromLowSToD(out.AsFpuRegisterPairLow<SRegister>()), 3717 out.AsFpuRegisterPairLow<SRegister>()); 3718 break; 3719 } 3720 3721 case Primitive::kPrimLong: { 3722 // Processing a Dex `long-to-double' instruction. 3723 Register low = in.AsRegisterPairLow<Register>(); 3724 Register high = in.AsRegisterPairHigh<Register>(); 3725 SRegister out_s = out.AsFpuRegisterPairLow<SRegister>(); 3726 DRegister out_d = FromLowSToD(out_s); 3727 SRegister temp_s = locations->GetTemp(0).AsFpuRegisterPairLow<SRegister>(); 3728 DRegister temp_d = FromLowSToD(temp_s); 3729 SRegister constant_s = locations->GetTemp(1).AsFpuRegisterPairLow<SRegister>(); 3730 DRegister constant_d = FromLowSToD(constant_s); 3731 3732 // temp_d = int-to-double(high) 3733 __ vmovsr(temp_s, high); 3734 __ vcvtdi(temp_d, temp_s); 3735 // constant_d = k2Pow32EncodingForDouble 3736 __ LoadDImmediate(constant_d, bit_cast<double, int64_t>(k2Pow32EncodingForDouble)); 3737 // out_d = unsigned-to-double(low) 3738 __ vmovsr(out_s, low); 3739 __ vcvtdu(out_d, out_s); 3740 // out_d += temp_d * constant_d 3741 __ vmlad(out_d, temp_d, constant_d); 3742 break; 3743 } 3744 3745 case Primitive::kPrimFloat: 3746 // Processing a Dex `float-to-double' instruction. 3747 __ vcvtds(FromLowSToD(out.AsFpuRegisterPairLow<SRegister>()), 3748 in.AsFpuRegister<SRegister>()); 3749 break; 3750 3751 default: 3752 LOG(FATAL) << "Unexpected type conversion from " << input_type 3753 << " to " << result_type; 3754 }; 3755 break; 3756 3757 default: 3758 LOG(FATAL) << "Unexpected type conversion from " << input_type 3759 << " to " << result_type; 3760 } 3761} 3762 3763void LocationsBuilderARM::VisitAdd(HAdd* add) { 3764 LocationSummary* locations = 3765 new (GetGraph()->GetArena()) LocationSummary(add, LocationSummary::kNoCall); 3766 switch (add->GetResultType()) { 3767 case Primitive::kPrimInt: { 3768 locations->SetInAt(0, Location::RequiresRegister()); 3769 locations->SetInAt(1, Location::RegisterOrConstant(add->InputAt(1))); 3770 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 3771 break; 3772 } 3773 3774 case Primitive::kPrimLong: { 3775 locations->SetInAt(0, Location::RequiresRegister()); 3776 locations->SetInAt(1, ArmEncodableConstantOrRegister(add->InputAt(1), ADD)); 3777 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 3778 break; 3779 } 3780 3781 case Primitive::kPrimFloat: 3782 case Primitive::kPrimDouble: { 3783 locations->SetInAt(0, Location::RequiresFpuRegister()); 3784 locations->SetInAt(1, Location::RequiresFpuRegister()); 3785 locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap); 3786 break; 3787 } 3788 3789 default: 3790 LOG(FATAL) << "Unexpected add type " << add->GetResultType(); 3791 } 3792} 3793 3794void InstructionCodeGeneratorARM::VisitAdd(HAdd* add) { 3795 LocationSummary* locations = add->GetLocations(); 3796 Location out = locations->Out(); 3797 Location first = locations->InAt(0); 3798 Location second = locations->InAt(1); 3799 switch (add->GetResultType()) { 3800 case Primitive::kPrimInt: 3801 if (second.IsRegister()) { 3802 __ add(out.AsRegister<Register>(), 3803 first.AsRegister<Register>(), 3804 ShifterOperand(second.AsRegister<Register>())); 3805 } else { 3806 __ AddConstant(out.AsRegister<Register>(), 3807 first.AsRegister<Register>(), 3808 second.GetConstant()->AsIntConstant()->GetValue()); 3809 } 3810 break; 3811 3812 case Primitive::kPrimLong: { 3813 if (second.IsConstant()) { 3814 uint64_t value = static_cast<uint64_t>(Int64FromConstant(second.GetConstant())); 3815 GenerateAddLongConst(out, first, value); 3816 } else { 3817 DCHECK(second.IsRegisterPair()); 3818 __ adds(out.AsRegisterPairLow<Register>(), 3819 first.AsRegisterPairLow<Register>(), 3820 ShifterOperand(second.AsRegisterPairLow<Register>())); 3821 __ adc(out.AsRegisterPairHigh<Register>(), 3822 first.AsRegisterPairHigh<Register>(), 3823 ShifterOperand(second.AsRegisterPairHigh<Register>())); 3824 } 3825 break; 3826 } 3827 3828 case Primitive::kPrimFloat: 3829 __ vadds(out.AsFpuRegister<SRegister>(), 3830 first.AsFpuRegister<SRegister>(), 3831 second.AsFpuRegister<SRegister>()); 3832 break; 3833 3834 case Primitive::kPrimDouble: 3835 __ vaddd(FromLowSToD(out.AsFpuRegisterPairLow<SRegister>()), 3836 FromLowSToD(first.AsFpuRegisterPairLow<SRegister>()), 3837 FromLowSToD(second.AsFpuRegisterPairLow<SRegister>())); 3838 break; 3839 3840 default: 3841 LOG(FATAL) << "Unexpected add type " << add->GetResultType(); 3842 } 3843} 3844 3845void LocationsBuilderARM::VisitSub(HSub* sub) { 3846 LocationSummary* locations = 3847 new (GetGraph()->GetArena()) LocationSummary(sub, LocationSummary::kNoCall); 3848 switch (sub->GetResultType()) { 3849 case Primitive::kPrimInt: { 3850 locations->SetInAt(0, Location::RequiresRegister()); 3851 locations->SetInAt(1, Location::RegisterOrConstant(sub->InputAt(1))); 3852 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 3853 break; 3854 } 3855 3856 case Primitive::kPrimLong: { 3857 locations->SetInAt(0, Location::RequiresRegister()); 3858 locations->SetInAt(1, ArmEncodableConstantOrRegister(sub->InputAt(1), SUB)); 3859 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 3860 break; 3861 } 3862 case Primitive::kPrimFloat: 3863 case Primitive::kPrimDouble: { 3864 locations->SetInAt(0, Location::RequiresFpuRegister()); 3865 locations->SetInAt(1, Location::RequiresFpuRegister()); 3866 locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap); 3867 break; 3868 } 3869 default: 3870 LOG(FATAL) << "Unexpected sub type " << sub->GetResultType(); 3871 } 3872} 3873 3874void InstructionCodeGeneratorARM::VisitSub(HSub* sub) { 3875 LocationSummary* locations = sub->GetLocations(); 3876 Location out = locations->Out(); 3877 Location first = locations->InAt(0); 3878 Location second = locations->InAt(1); 3879 switch (sub->GetResultType()) { 3880 case Primitive::kPrimInt: { 3881 if (second.IsRegister()) { 3882 __ sub(out.AsRegister<Register>(), 3883 first.AsRegister<Register>(), 3884 ShifterOperand(second.AsRegister<Register>())); 3885 } else { 3886 __ AddConstant(out.AsRegister<Register>(), 3887 first.AsRegister<Register>(), 3888 -second.GetConstant()->AsIntConstant()->GetValue()); 3889 } 3890 break; 3891 } 3892 3893 case Primitive::kPrimLong: { 3894 if (second.IsConstant()) { 3895 uint64_t value = static_cast<uint64_t>(Int64FromConstant(second.GetConstant())); 3896 GenerateAddLongConst(out, first, -value); 3897 } else { 3898 DCHECK(second.IsRegisterPair()); 3899 __ subs(out.AsRegisterPairLow<Register>(), 3900 first.AsRegisterPairLow<Register>(), 3901 ShifterOperand(second.AsRegisterPairLow<Register>())); 3902 __ sbc(out.AsRegisterPairHigh<Register>(), 3903 first.AsRegisterPairHigh<Register>(), 3904 ShifterOperand(second.AsRegisterPairHigh<Register>())); 3905 } 3906 break; 3907 } 3908 3909 case Primitive::kPrimFloat: { 3910 __ vsubs(out.AsFpuRegister<SRegister>(), 3911 first.AsFpuRegister<SRegister>(), 3912 second.AsFpuRegister<SRegister>()); 3913 break; 3914 } 3915 3916 case Primitive::kPrimDouble: { 3917 __ vsubd(FromLowSToD(out.AsFpuRegisterPairLow<SRegister>()), 3918 FromLowSToD(first.AsFpuRegisterPairLow<SRegister>()), 3919 FromLowSToD(second.AsFpuRegisterPairLow<SRegister>())); 3920 break; 3921 } 3922 3923 3924 default: 3925 LOG(FATAL) << "Unexpected sub type " << sub->GetResultType(); 3926 } 3927} 3928 3929void LocationsBuilderARM::VisitMul(HMul* mul) { 3930 LocationSummary* locations = 3931 new (GetGraph()->GetArena()) LocationSummary(mul, LocationSummary::kNoCall); 3932 switch (mul->GetResultType()) { 3933 case Primitive::kPrimInt: 3934 case Primitive::kPrimLong: { 3935 locations->SetInAt(0, Location::RequiresRegister()); 3936 locations->SetInAt(1, Location::RequiresRegister()); 3937 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 3938 break; 3939 } 3940 3941 case Primitive::kPrimFloat: 3942 case Primitive::kPrimDouble: { 3943 locations->SetInAt(0, Location::RequiresFpuRegister()); 3944 locations->SetInAt(1, Location::RequiresFpuRegister()); 3945 locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap); 3946 break; 3947 } 3948 3949 default: 3950 LOG(FATAL) << "Unexpected mul type " << mul->GetResultType(); 3951 } 3952} 3953 3954void InstructionCodeGeneratorARM::VisitMul(HMul* mul) { 3955 LocationSummary* locations = mul->GetLocations(); 3956 Location out = locations->Out(); 3957 Location first = locations->InAt(0); 3958 Location second = locations->InAt(1); 3959 switch (mul->GetResultType()) { 3960 case Primitive::kPrimInt: { 3961 __ mul(out.AsRegister<Register>(), 3962 first.AsRegister<Register>(), 3963 second.AsRegister<Register>()); 3964 break; 3965 } 3966 case Primitive::kPrimLong: { 3967 Register out_hi = out.AsRegisterPairHigh<Register>(); 3968 Register out_lo = out.AsRegisterPairLow<Register>(); 3969 Register in1_hi = first.AsRegisterPairHigh<Register>(); 3970 Register in1_lo = first.AsRegisterPairLow<Register>(); 3971 Register in2_hi = second.AsRegisterPairHigh<Register>(); 3972 Register in2_lo = second.AsRegisterPairLow<Register>(); 3973 3974 // Extra checks to protect caused by the existence of R1_R2. 3975 // The algorithm is wrong if out.hi is either in1.lo or in2.lo: 3976 // (e.g. in1=r0_r1, in2=r2_r3 and out=r1_r2); 3977 DCHECK_NE(out_hi, in1_lo); 3978 DCHECK_NE(out_hi, in2_lo); 3979 3980 // input: in1 - 64 bits, in2 - 64 bits 3981 // output: out 3982 // formula: out.hi : out.lo = (in1.lo * in2.hi + in1.hi * in2.lo)* 2^32 + in1.lo * in2.lo 3983 // parts: out.hi = in1.lo * in2.hi + in1.hi * in2.lo + (in1.lo * in2.lo)[63:32] 3984 // parts: out.lo = (in1.lo * in2.lo)[31:0] 3985 3986 // IP <- in1.lo * in2.hi 3987 __ mul(IP, in1_lo, in2_hi); 3988 // out.hi <- in1.lo * in2.hi + in1.hi * in2.lo 3989 __ mla(out_hi, in1_hi, in2_lo, IP); 3990 // out.lo <- (in1.lo * in2.lo)[31:0]; 3991 __ umull(out_lo, IP, in1_lo, in2_lo); 3992 // out.hi <- in2.hi * in1.lo + in2.lo * in1.hi + (in1.lo * in2.lo)[63:32] 3993 __ add(out_hi, out_hi, ShifterOperand(IP)); 3994 break; 3995 } 3996 3997 case Primitive::kPrimFloat: { 3998 __ vmuls(out.AsFpuRegister<SRegister>(), 3999 first.AsFpuRegister<SRegister>(), 4000 second.AsFpuRegister<SRegister>()); 4001 break; 4002 } 4003 4004 case Primitive::kPrimDouble: { 4005 __ vmuld(FromLowSToD(out.AsFpuRegisterPairLow<SRegister>()), 4006 FromLowSToD(first.AsFpuRegisterPairLow<SRegister>()), 4007 FromLowSToD(second.AsFpuRegisterPairLow<SRegister>())); 4008 break; 4009 } 4010 4011 default: 4012 LOG(FATAL) << "Unexpected mul type " << mul->GetResultType(); 4013 } 4014} 4015 4016void InstructionCodeGeneratorARM::DivRemOneOrMinusOne(HBinaryOperation* instruction) { 4017 DCHECK(instruction->IsDiv() || instruction->IsRem()); 4018 DCHECK(instruction->GetResultType() == Primitive::kPrimInt); 4019 4020 LocationSummary* locations = instruction->GetLocations(); 4021 Location second = locations->InAt(1); 4022 DCHECK(second.IsConstant()); 4023 4024 Register out = locations->Out().AsRegister<Register>(); 4025 Register dividend = locations->InAt(0).AsRegister<Register>(); 4026 int32_t imm = second.GetConstant()->AsIntConstant()->GetValue(); 4027 DCHECK(imm == 1 || imm == -1); 4028 4029 if (instruction->IsRem()) { 4030 __ LoadImmediate(out, 0); 4031 } else { 4032 if (imm == 1) { 4033 __ Mov(out, dividend); 4034 } else { 4035 __ rsb(out, dividend, ShifterOperand(0)); 4036 } 4037 } 4038} 4039 4040void InstructionCodeGeneratorARM::DivRemByPowerOfTwo(HBinaryOperation* instruction) { 4041 DCHECK(instruction->IsDiv() || instruction->IsRem()); 4042 DCHECK(instruction->GetResultType() == Primitive::kPrimInt); 4043 4044 LocationSummary* locations = instruction->GetLocations(); 4045 Location second = locations->InAt(1); 4046 DCHECK(second.IsConstant()); 4047 4048 Register out = locations->Out().AsRegister<Register>(); 4049 Register dividend = locations->InAt(0).AsRegister<Register>(); 4050 Register temp = locations->GetTemp(0).AsRegister<Register>(); 4051 int32_t imm = second.GetConstant()->AsIntConstant()->GetValue(); 4052 uint32_t abs_imm = static_cast<uint32_t>(AbsOrMin(imm)); 4053 int ctz_imm = CTZ(abs_imm); 4054 4055 if (ctz_imm == 1) { 4056 __ Lsr(temp, dividend, 32 - ctz_imm); 4057 } else { 4058 __ Asr(temp, dividend, 31); 4059 __ Lsr(temp, temp, 32 - ctz_imm); 4060 } 4061 __ add(out, temp, ShifterOperand(dividend)); 4062 4063 if (instruction->IsDiv()) { 4064 __ Asr(out, out, ctz_imm); 4065 if (imm < 0) { 4066 __ rsb(out, out, ShifterOperand(0)); 4067 } 4068 } else { 4069 __ ubfx(out, out, 0, ctz_imm); 4070 __ sub(out, out, ShifterOperand(temp)); 4071 } 4072} 4073 4074void InstructionCodeGeneratorARM::GenerateDivRemWithAnyConstant(HBinaryOperation* instruction) { 4075 DCHECK(instruction->IsDiv() || instruction->IsRem()); 4076 DCHECK(instruction->GetResultType() == Primitive::kPrimInt); 4077 4078 LocationSummary* locations = instruction->GetLocations(); 4079 Location second = locations->InAt(1); 4080 DCHECK(second.IsConstant()); 4081 4082 Register out = locations->Out().AsRegister<Register>(); 4083 Register dividend = locations->InAt(0).AsRegister<Register>(); 4084 Register temp1 = locations->GetTemp(0).AsRegister<Register>(); 4085 Register temp2 = locations->GetTemp(1).AsRegister<Register>(); 4086 int64_t imm = second.GetConstant()->AsIntConstant()->GetValue(); 4087 4088 int64_t magic; 4089 int shift; 4090 CalculateMagicAndShiftForDivRem(imm, false /* is_long */, &magic, &shift); 4091 4092 __ LoadImmediate(temp1, magic); 4093 __ smull(temp2, temp1, dividend, temp1); 4094 4095 if (imm > 0 && magic < 0) { 4096 __ add(temp1, temp1, ShifterOperand(dividend)); 4097 } else if (imm < 0 && magic > 0) { 4098 __ sub(temp1, temp1, ShifterOperand(dividend)); 4099 } 4100 4101 if (shift != 0) { 4102 __ Asr(temp1, temp1, shift); 4103 } 4104 4105 if (instruction->IsDiv()) { 4106 __ sub(out, temp1, ShifterOperand(temp1, ASR, 31)); 4107 } else { 4108 __ sub(temp1, temp1, ShifterOperand(temp1, ASR, 31)); 4109 // TODO: Strength reduction for mls. 4110 __ LoadImmediate(temp2, imm); 4111 __ mls(out, temp1, temp2, dividend); 4112 } 4113} 4114 4115void InstructionCodeGeneratorARM::GenerateDivRemConstantIntegral(HBinaryOperation* instruction) { 4116 DCHECK(instruction->IsDiv() || instruction->IsRem()); 4117 DCHECK(instruction->GetResultType() == Primitive::kPrimInt); 4118 4119 LocationSummary* locations = instruction->GetLocations(); 4120 Location second = locations->InAt(1); 4121 DCHECK(second.IsConstant()); 4122 4123 int32_t imm = second.GetConstant()->AsIntConstant()->GetValue(); 4124 if (imm == 0) { 4125 // Do not generate anything. DivZeroCheck would prevent any code to be executed. 4126 } else if (imm == 1 || imm == -1) { 4127 DivRemOneOrMinusOne(instruction); 4128 } else if (IsPowerOfTwo(AbsOrMin(imm))) { 4129 DivRemByPowerOfTwo(instruction); 4130 } else { 4131 DCHECK(imm <= -2 || imm >= 2); 4132 GenerateDivRemWithAnyConstant(instruction); 4133 } 4134} 4135 4136void LocationsBuilderARM::VisitDiv(HDiv* div) { 4137 LocationSummary::CallKind call_kind = LocationSummary::kNoCall; 4138 if (div->GetResultType() == Primitive::kPrimLong) { 4139 // pLdiv runtime call. 4140 call_kind = LocationSummary::kCallOnMainOnly; 4141 } else if (div->GetResultType() == Primitive::kPrimInt && div->InputAt(1)->IsConstant()) { 4142 // sdiv will be replaced by other instruction sequence. 4143 } else if (div->GetResultType() == Primitive::kPrimInt && 4144 !codegen_->GetInstructionSetFeatures().HasDivideInstruction()) { 4145 // pIdivmod runtime call. 4146 call_kind = LocationSummary::kCallOnMainOnly; 4147 } 4148 4149 LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(div, call_kind); 4150 4151 switch (div->GetResultType()) { 4152 case Primitive::kPrimInt: { 4153 if (div->InputAt(1)->IsConstant()) { 4154 locations->SetInAt(0, Location::RequiresRegister()); 4155 locations->SetInAt(1, Location::ConstantLocation(div->InputAt(1)->AsConstant())); 4156 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 4157 int32_t value = div->InputAt(1)->AsIntConstant()->GetValue(); 4158 if (value == 1 || value == 0 || value == -1) { 4159 // No temp register required. 4160 } else { 4161 locations->AddTemp(Location::RequiresRegister()); 4162 if (!IsPowerOfTwo(AbsOrMin(value))) { 4163 locations->AddTemp(Location::RequiresRegister()); 4164 } 4165 } 4166 } else if (codegen_->GetInstructionSetFeatures().HasDivideInstruction()) { 4167 locations->SetInAt(0, Location::RequiresRegister()); 4168 locations->SetInAt(1, Location::RequiresRegister()); 4169 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 4170 } else { 4171 InvokeRuntimeCallingConvention calling_convention; 4172 locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(0))); 4173 locations->SetInAt(1, Location::RegisterLocation(calling_convention.GetRegisterAt(1))); 4174 // Note: divmod will compute both the quotient and the remainder as the pair R0 and R1, but 4175 // we only need the former. 4176 locations->SetOut(Location::RegisterLocation(R0)); 4177 } 4178 break; 4179 } 4180 case Primitive::kPrimLong: { 4181 InvokeRuntimeCallingConvention calling_convention; 4182 locations->SetInAt(0, Location::RegisterPairLocation( 4183 calling_convention.GetRegisterAt(0), calling_convention.GetRegisterAt(1))); 4184 locations->SetInAt(1, Location::RegisterPairLocation( 4185 calling_convention.GetRegisterAt(2), calling_convention.GetRegisterAt(3))); 4186 locations->SetOut(Location::RegisterPairLocation(R0, R1)); 4187 break; 4188 } 4189 case Primitive::kPrimFloat: 4190 case Primitive::kPrimDouble: { 4191 locations->SetInAt(0, Location::RequiresFpuRegister()); 4192 locations->SetInAt(1, Location::RequiresFpuRegister()); 4193 locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap); 4194 break; 4195 } 4196 4197 default: 4198 LOG(FATAL) << "Unexpected div type " << div->GetResultType(); 4199 } 4200} 4201 4202void InstructionCodeGeneratorARM::VisitDiv(HDiv* div) { 4203 LocationSummary* locations = div->GetLocations(); 4204 Location out = locations->Out(); 4205 Location first = locations->InAt(0); 4206 Location second = locations->InAt(1); 4207 4208 switch (div->GetResultType()) { 4209 case Primitive::kPrimInt: { 4210 if (second.IsConstant()) { 4211 GenerateDivRemConstantIntegral(div); 4212 } else if (codegen_->GetInstructionSetFeatures().HasDivideInstruction()) { 4213 __ sdiv(out.AsRegister<Register>(), 4214 first.AsRegister<Register>(), 4215 second.AsRegister<Register>()); 4216 } else { 4217 InvokeRuntimeCallingConvention calling_convention; 4218 DCHECK_EQ(calling_convention.GetRegisterAt(0), first.AsRegister<Register>()); 4219 DCHECK_EQ(calling_convention.GetRegisterAt(1), second.AsRegister<Register>()); 4220 DCHECK_EQ(R0, out.AsRegister<Register>()); 4221 4222 codegen_->InvokeRuntime(kQuickIdivmod, div, div->GetDexPc()); 4223 CheckEntrypointTypes<kQuickIdivmod, int32_t, int32_t, int32_t>(); 4224 } 4225 break; 4226 } 4227 4228 case Primitive::kPrimLong: { 4229 InvokeRuntimeCallingConvention calling_convention; 4230 DCHECK_EQ(calling_convention.GetRegisterAt(0), first.AsRegisterPairLow<Register>()); 4231 DCHECK_EQ(calling_convention.GetRegisterAt(1), first.AsRegisterPairHigh<Register>()); 4232 DCHECK_EQ(calling_convention.GetRegisterAt(2), second.AsRegisterPairLow<Register>()); 4233 DCHECK_EQ(calling_convention.GetRegisterAt(3), second.AsRegisterPairHigh<Register>()); 4234 DCHECK_EQ(R0, out.AsRegisterPairLow<Register>()); 4235 DCHECK_EQ(R1, out.AsRegisterPairHigh<Register>()); 4236 4237 codegen_->InvokeRuntime(kQuickLdiv, div, div->GetDexPc()); 4238 CheckEntrypointTypes<kQuickLdiv, int64_t, int64_t, int64_t>(); 4239 break; 4240 } 4241 4242 case Primitive::kPrimFloat: { 4243 __ vdivs(out.AsFpuRegister<SRegister>(), 4244 first.AsFpuRegister<SRegister>(), 4245 second.AsFpuRegister<SRegister>()); 4246 break; 4247 } 4248 4249 case Primitive::kPrimDouble: { 4250 __ vdivd(FromLowSToD(out.AsFpuRegisterPairLow<SRegister>()), 4251 FromLowSToD(first.AsFpuRegisterPairLow<SRegister>()), 4252 FromLowSToD(second.AsFpuRegisterPairLow<SRegister>())); 4253 break; 4254 } 4255 4256 default: 4257 LOG(FATAL) << "Unexpected div type " << div->GetResultType(); 4258 } 4259} 4260 4261void LocationsBuilderARM::VisitRem(HRem* rem) { 4262 Primitive::Type type = rem->GetResultType(); 4263 4264 // Most remainders are implemented in the runtime. 4265 LocationSummary::CallKind call_kind = LocationSummary::kCallOnMainOnly; 4266 if (rem->GetResultType() == Primitive::kPrimInt && rem->InputAt(1)->IsConstant()) { 4267 // sdiv will be replaced by other instruction sequence. 4268 call_kind = LocationSummary::kNoCall; 4269 } else if ((rem->GetResultType() == Primitive::kPrimInt) 4270 && codegen_->GetInstructionSetFeatures().HasDivideInstruction()) { 4271 // Have hardware divide instruction for int, do it with three instructions. 4272 call_kind = LocationSummary::kNoCall; 4273 } 4274 4275 LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(rem, call_kind); 4276 4277 switch (type) { 4278 case Primitive::kPrimInt: { 4279 if (rem->InputAt(1)->IsConstant()) { 4280 locations->SetInAt(0, Location::RequiresRegister()); 4281 locations->SetInAt(1, Location::ConstantLocation(rem->InputAt(1)->AsConstant())); 4282 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 4283 int32_t value = rem->InputAt(1)->AsIntConstant()->GetValue(); 4284 if (value == 1 || value == 0 || value == -1) { 4285 // No temp register required. 4286 } else { 4287 locations->AddTemp(Location::RequiresRegister()); 4288 if (!IsPowerOfTwo(AbsOrMin(value))) { 4289 locations->AddTemp(Location::RequiresRegister()); 4290 } 4291 } 4292 } else if (codegen_->GetInstructionSetFeatures().HasDivideInstruction()) { 4293 locations->SetInAt(0, Location::RequiresRegister()); 4294 locations->SetInAt(1, Location::RequiresRegister()); 4295 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 4296 locations->AddTemp(Location::RequiresRegister()); 4297 } else { 4298 InvokeRuntimeCallingConvention calling_convention; 4299 locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(0))); 4300 locations->SetInAt(1, Location::RegisterLocation(calling_convention.GetRegisterAt(1))); 4301 // Note: divmod will compute both the quotient and the remainder as the pair R0 and R1, but 4302 // we only need the latter. 4303 locations->SetOut(Location::RegisterLocation(R1)); 4304 } 4305 break; 4306 } 4307 case Primitive::kPrimLong: { 4308 InvokeRuntimeCallingConvention calling_convention; 4309 locations->SetInAt(0, Location::RegisterPairLocation( 4310 calling_convention.GetRegisterAt(0), calling_convention.GetRegisterAt(1))); 4311 locations->SetInAt(1, Location::RegisterPairLocation( 4312 calling_convention.GetRegisterAt(2), calling_convention.GetRegisterAt(3))); 4313 // The runtime helper puts the output in R2,R3. 4314 locations->SetOut(Location::RegisterPairLocation(R2, R3)); 4315 break; 4316 } 4317 case Primitive::kPrimFloat: { 4318 InvokeRuntimeCallingConvention calling_convention; 4319 locations->SetInAt(0, Location::FpuRegisterLocation(calling_convention.GetFpuRegisterAt(0))); 4320 locations->SetInAt(1, Location::FpuRegisterLocation(calling_convention.GetFpuRegisterAt(1))); 4321 locations->SetOut(Location::FpuRegisterLocation(S0)); 4322 break; 4323 } 4324 4325 case Primitive::kPrimDouble: { 4326 InvokeRuntimeCallingConvention calling_convention; 4327 locations->SetInAt(0, Location::FpuRegisterPairLocation( 4328 calling_convention.GetFpuRegisterAt(0), calling_convention.GetFpuRegisterAt(1))); 4329 locations->SetInAt(1, Location::FpuRegisterPairLocation( 4330 calling_convention.GetFpuRegisterAt(2), calling_convention.GetFpuRegisterAt(3))); 4331 locations->SetOut(Location::Location::FpuRegisterPairLocation(S0, S1)); 4332 break; 4333 } 4334 4335 default: 4336 LOG(FATAL) << "Unexpected rem type " << type; 4337 } 4338} 4339 4340void InstructionCodeGeneratorARM::VisitRem(HRem* rem) { 4341 LocationSummary* locations = rem->GetLocations(); 4342 Location out = locations->Out(); 4343 Location first = locations->InAt(0); 4344 Location second = locations->InAt(1); 4345 4346 Primitive::Type type = rem->GetResultType(); 4347 switch (type) { 4348 case Primitive::kPrimInt: { 4349 if (second.IsConstant()) { 4350 GenerateDivRemConstantIntegral(rem); 4351 } else if (codegen_->GetInstructionSetFeatures().HasDivideInstruction()) { 4352 Register reg1 = first.AsRegister<Register>(); 4353 Register reg2 = second.AsRegister<Register>(); 4354 Register temp = locations->GetTemp(0).AsRegister<Register>(); 4355 4356 // temp = reg1 / reg2 (integer division) 4357 // dest = reg1 - temp * reg2 4358 __ sdiv(temp, reg1, reg2); 4359 __ mls(out.AsRegister<Register>(), temp, reg2, reg1); 4360 } else { 4361 InvokeRuntimeCallingConvention calling_convention; 4362 DCHECK_EQ(calling_convention.GetRegisterAt(0), first.AsRegister<Register>()); 4363 DCHECK_EQ(calling_convention.GetRegisterAt(1), second.AsRegister<Register>()); 4364 DCHECK_EQ(R1, out.AsRegister<Register>()); 4365 4366 codegen_->InvokeRuntime(kQuickIdivmod, rem, rem->GetDexPc()); 4367 CheckEntrypointTypes<kQuickIdivmod, int32_t, int32_t, int32_t>(); 4368 } 4369 break; 4370 } 4371 4372 case Primitive::kPrimLong: { 4373 codegen_->InvokeRuntime(kQuickLmod, rem, rem->GetDexPc()); 4374 CheckEntrypointTypes<kQuickLmod, int64_t, int64_t, int64_t>(); 4375 break; 4376 } 4377 4378 case Primitive::kPrimFloat: { 4379 codegen_->InvokeRuntime(kQuickFmodf, rem, rem->GetDexPc()); 4380 CheckEntrypointTypes<kQuickFmodf, float, float, float>(); 4381 break; 4382 } 4383 4384 case Primitive::kPrimDouble: { 4385 codegen_->InvokeRuntime(kQuickFmod, rem, rem->GetDexPc()); 4386 CheckEntrypointTypes<kQuickFmod, double, double, double>(); 4387 break; 4388 } 4389 4390 default: 4391 LOG(FATAL) << "Unexpected rem type " << type; 4392 } 4393} 4394 4395void LocationsBuilderARM::VisitDivZeroCheck(HDivZeroCheck* instruction) { 4396 LocationSummary* locations = codegen_->CreateThrowingSlowPathLocations(instruction); 4397 locations->SetInAt(0, Location::RegisterOrConstant(instruction->InputAt(0))); 4398} 4399 4400void InstructionCodeGeneratorARM::VisitDivZeroCheck(HDivZeroCheck* instruction) { 4401 SlowPathCodeARM* slow_path = new (GetGraph()->GetArena()) DivZeroCheckSlowPathARM(instruction); 4402 codegen_->AddSlowPath(slow_path); 4403 4404 LocationSummary* locations = instruction->GetLocations(); 4405 Location value = locations->InAt(0); 4406 4407 switch (instruction->GetType()) { 4408 case Primitive::kPrimBoolean: 4409 case Primitive::kPrimByte: 4410 case Primitive::kPrimChar: 4411 case Primitive::kPrimShort: 4412 case Primitive::kPrimInt: { 4413 if (value.IsRegister()) { 4414 __ CompareAndBranchIfZero(value.AsRegister<Register>(), slow_path->GetEntryLabel()); 4415 } else { 4416 DCHECK(value.IsConstant()) << value; 4417 if (value.GetConstant()->AsIntConstant()->GetValue() == 0) { 4418 __ b(slow_path->GetEntryLabel()); 4419 } 4420 } 4421 break; 4422 } 4423 case Primitive::kPrimLong: { 4424 if (value.IsRegisterPair()) { 4425 __ orrs(IP, 4426 value.AsRegisterPairLow<Register>(), 4427 ShifterOperand(value.AsRegisterPairHigh<Register>())); 4428 __ b(slow_path->GetEntryLabel(), EQ); 4429 } else { 4430 DCHECK(value.IsConstant()) << value; 4431 if (value.GetConstant()->AsLongConstant()->GetValue() == 0) { 4432 __ b(slow_path->GetEntryLabel()); 4433 } 4434 } 4435 break; 4436 default: 4437 LOG(FATAL) << "Unexpected type for HDivZeroCheck " << instruction->GetType(); 4438 } 4439 } 4440} 4441 4442void InstructionCodeGeneratorARM::HandleIntegerRotate(LocationSummary* locations) { 4443 Register in = locations->InAt(0).AsRegister<Register>(); 4444 Location rhs = locations->InAt(1); 4445 Register out = locations->Out().AsRegister<Register>(); 4446 4447 if (rhs.IsConstant()) { 4448 // Arm32 and Thumb2 assemblers require a rotation on the interval [1,31], 4449 // so map all rotations to a +ve. equivalent in that range. 4450 // (e.g. left *or* right by -2 bits == 30 bits in the same direction.) 4451 uint32_t rot = CodeGenerator::GetInt32ValueOf(rhs.GetConstant()) & 0x1F; 4452 if (rot) { 4453 // Rotate, mapping left rotations to right equivalents if necessary. 4454 // (e.g. left by 2 bits == right by 30.) 4455 __ Ror(out, in, rot); 4456 } else if (out != in) { 4457 __ Mov(out, in); 4458 } 4459 } else { 4460 __ Ror(out, in, rhs.AsRegister<Register>()); 4461 } 4462} 4463 4464// Gain some speed by mapping all Long rotates onto equivalent pairs of Integer 4465// rotates by swapping input regs (effectively rotating by the first 32-bits of 4466// a larger rotation) or flipping direction (thus treating larger right/left 4467// rotations as sub-word sized rotations in the other direction) as appropriate. 4468void InstructionCodeGeneratorARM::HandleLongRotate(HRor* ror) { 4469 LocationSummary* locations = ror->GetLocations(); 4470 Register in_reg_lo = locations->InAt(0).AsRegisterPairLow<Register>(); 4471 Register in_reg_hi = locations->InAt(0).AsRegisterPairHigh<Register>(); 4472 Location rhs = locations->InAt(1); 4473 Register out_reg_lo = locations->Out().AsRegisterPairLow<Register>(); 4474 Register out_reg_hi = locations->Out().AsRegisterPairHigh<Register>(); 4475 4476 if (rhs.IsConstant()) { 4477 uint64_t rot = CodeGenerator::GetInt64ValueOf(rhs.GetConstant()); 4478 // Map all rotations to +ve. equivalents on the interval [0,63]. 4479 rot &= kMaxLongShiftDistance; 4480 // For rotates over a word in size, 'pre-rotate' by 32-bits to keep rotate 4481 // logic below to a simple pair of binary orr. 4482 // (e.g. 34 bits == in_reg swap + 2 bits right.) 4483 if (rot >= kArmBitsPerWord) { 4484 rot -= kArmBitsPerWord; 4485 std::swap(in_reg_hi, in_reg_lo); 4486 } 4487 // Rotate, or mov to out for zero or word size rotations. 4488 if (rot != 0u) { 4489 __ Lsr(out_reg_hi, in_reg_hi, rot); 4490 __ orr(out_reg_hi, out_reg_hi, ShifterOperand(in_reg_lo, arm::LSL, kArmBitsPerWord - rot)); 4491 __ Lsr(out_reg_lo, in_reg_lo, rot); 4492 __ orr(out_reg_lo, out_reg_lo, ShifterOperand(in_reg_hi, arm::LSL, kArmBitsPerWord - rot)); 4493 } else { 4494 __ Mov(out_reg_lo, in_reg_lo); 4495 __ Mov(out_reg_hi, in_reg_hi); 4496 } 4497 } else { 4498 Register shift_right = locations->GetTemp(0).AsRegister<Register>(); 4499 Register shift_left = locations->GetTemp(1).AsRegister<Register>(); 4500 Label end; 4501 Label shift_by_32_plus_shift_right; 4502 Label* final_label = codegen_->GetFinalLabel(ror, &end); 4503 4504 __ and_(shift_right, rhs.AsRegister<Register>(), ShifterOperand(0x1F)); 4505 __ Lsrs(shift_left, rhs.AsRegister<Register>(), 6); 4506 __ rsb(shift_left, shift_right, ShifterOperand(kArmBitsPerWord), AL, kCcKeep); 4507 __ b(&shift_by_32_plus_shift_right, CC); 4508 4509 // out_reg_hi = (reg_hi << shift_left) | (reg_lo >> shift_right). 4510 // out_reg_lo = (reg_lo << shift_left) | (reg_hi >> shift_right). 4511 __ Lsl(out_reg_hi, in_reg_hi, shift_left); 4512 __ Lsr(out_reg_lo, in_reg_lo, shift_right); 4513 __ add(out_reg_hi, out_reg_hi, ShifterOperand(out_reg_lo)); 4514 __ Lsl(out_reg_lo, in_reg_lo, shift_left); 4515 __ Lsr(shift_left, in_reg_hi, shift_right); 4516 __ add(out_reg_lo, out_reg_lo, ShifterOperand(shift_left)); 4517 __ b(final_label); 4518 4519 __ Bind(&shift_by_32_plus_shift_right); // Shift by 32+shift_right. 4520 // out_reg_hi = (reg_hi >> shift_right) | (reg_lo << shift_left). 4521 // out_reg_lo = (reg_lo >> shift_right) | (reg_hi << shift_left). 4522 __ Lsr(out_reg_hi, in_reg_hi, shift_right); 4523 __ Lsl(out_reg_lo, in_reg_lo, shift_left); 4524 __ add(out_reg_hi, out_reg_hi, ShifterOperand(out_reg_lo)); 4525 __ Lsr(out_reg_lo, in_reg_lo, shift_right); 4526 __ Lsl(shift_right, in_reg_hi, shift_left); 4527 __ add(out_reg_lo, out_reg_lo, ShifterOperand(shift_right)); 4528 4529 if (end.IsLinked()) { 4530 __ Bind(&end); 4531 } 4532 } 4533} 4534 4535void LocationsBuilderARM::VisitRor(HRor* ror) { 4536 LocationSummary* locations = 4537 new (GetGraph()->GetArena()) LocationSummary(ror, LocationSummary::kNoCall); 4538 switch (ror->GetResultType()) { 4539 case Primitive::kPrimInt: { 4540 locations->SetInAt(0, Location::RequiresRegister()); 4541 locations->SetInAt(1, Location::RegisterOrConstant(ror->InputAt(1))); 4542 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 4543 break; 4544 } 4545 case Primitive::kPrimLong: { 4546 locations->SetInAt(0, Location::RequiresRegister()); 4547 if (ror->InputAt(1)->IsConstant()) { 4548 locations->SetInAt(1, Location::ConstantLocation(ror->InputAt(1)->AsConstant())); 4549 } else { 4550 locations->SetInAt(1, Location::RequiresRegister()); 4551 locations->AddTemp(Location::RequiresRegister()); 4552 locations->AddTemp(Location::RequiresRegister()); 4553 } 4554 locations->SetOut(Location::RequiresRegister(), Location::kOutputOverlap); 4555 break; 4556 } 4557 default: 4558 LOG(FATAL) << "Unexpected operation type " << ror->GetResultType(); 4559 } 4560} 4561 4562void InstructionCodeGeneratorARM::VisitRor(HRor* ror) { 4563 LocationSummary* locations = ror->GetLocations(); 4564 Primitive::Type type = ror->GetResultType(); 4565 switch (type) { 4566 case Primitive::kPrimInt: { 4567 HandleIntegerRotate(locations); 4568 break; 4569 } 4570 case Primitive::kPrimLong: { 4571 HandleLongRotate(ror); 4572 break; 4573 } 4574 default: 4575 LOG(FATAL) << "Unexpected operation type " << type; 4576 UNREACHABLE(); 4577 } 4578} 4579 4580void LocationsBuilderARM::HandleShift(HBinaryOperation* op) { 4581 DCHECK(op->IsShl() || op->IsShr() || op->IsUShr()); 4582 4583 LocationSummary* locations = 4584 new (GetGraph()->GetArena()) LocationSummary(op, LocationSummary::kNoCall); 4585 4586 switch (op->GetResultType()) { 4587 case Primitive::kPrimInt: { 4588 locations->SetInAt(0, Location::RequiresRegister()); 4589 if (op->InputAt(1)->IsConstant()) { 4590 locations->SetInAt(1, Location::ConstantLocation(op->InputAt(1)->AsConstant())); 4591 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 4592 } else { 4593 locations->SetInAt(1, Location::RequiresRegister()); 4594 // Make the output overlap, as it will be used to hold the masked 4595 // second input. 4596 locations->SetOut(Location::RequiresRegister(), Location::kOutputOverlap); 4597 } 4598 break; 4599 } 4600 case Primitive::kPrimLong: { 4601 locations->SetInAt(0, Location::RequiresRegister()); 4602 if (op->InputAt(1)->IsConstant()) { 4603 locations->SetInAt(1, Location::ConstantLocation(op->InputAt(1)->AsConstant())); 4604 // For simplicity, use kOutputOverlap even though we only require that low registers 4605 // don't clash with high registers which the register allocator currently guarantees. 4606 locations->SetOut(Location::RequiresRegister(), Location::kOutputOverlap); 4607 } else { 4608 locations->SetInAt(1, Location::RequiresRegister()); 4609 locations->AddTemp(Location::RequiresRegister()); 4610 locations->SetOut(Location::RequiresRegister(), Location::kOutputOverlap); 4611 } 4612 break; 4613 } 4614 default: 4615 LOG(FATAL) << "Unexpected operation type " << op->GetResultType(); 4616 } 4617} 4618 4619void InstructionCodeGeneratorARM::HandleShift(HBinaryOperation* op) { 4620 DCHECK(op->IsShl() || op->IsShr() || op->IsUShr()); 4621 4622 LocationSummary* locations = op->GetLocations(); 4623 Location out = locations->Out(); 4624 Location first = locations->InAt(0); 4625 Location second = locations->InAt(1); 4626 4627 Primitive::Type type = op->GetResultType(); 4628 switch (type) { 4629 case Primitive::kPrimInt: { 4630 Register out_reg = out.AsRegister<Register>(); 4631 Register first_reg = first.AsRegister<Register>(); 4632 if (second.IsRegister()) { 4633 Register second_reg = second.AsRegister<Register>(); 4634 // ARM doesn't mask the shift count so we need to do it ourselves. 4635 __ and_(out_reg, second_reg, ShifterOperand(kMaxIntShiftDistance)); 4636 if (op->IsShl()) { 4637 __ Lsl(out_reg, first_reg, out_reg); 4638 } else if (op->IsShr()) { 4639 __ Asr(out_reg, first_reg, out_reg); 4640 } else { 4641 __ Lsr(out_reg, first_reg, out_reg); 4642 } 4643 } else { 4644 int32_t cst = second.GetConstant()->AsIntConstant()->GetValue(); 4645 uint32_t shift_value = cst & kMaxIntShiftDistance; 4646 if (shift_value == 0) { // ARM does not support shifting with 0 immediate. 4647 __ Mov(out_reg, first_reg); 4648 } else if (op->IsShl()) { 4649 __ Lsl(out_reg, first_reg, shift_value); 4650 } else if (op->IsShr()) { 4651 __ Asr(out_reg, first_reg, shift_value); 4652 } else { 4653 __ Lsr(out_reg, first_reg, shift_value); 4654 } 4655 } 4656 break; 4657 } 4658 case Primitive::kPrimLong: { 4659 Register o_h = out.AsRegisterPairHigh<Register>(); 4660 Register o_l = out.AsRegisterPairLow<Register>(); 4661 4662 Register high = first.AsRegisterPairHigh<Register>(); 4663 Register low = first.AsRegisterPairLow<Register>(); 4664 4665 if (second.IsRegister()) { 4666 Register temp = locations->GetTemp(0).AsRegister<Register>(); 4667 4668 Register second_reg = second.AsRegister<Register>(); 4669 4670 if (op->IsShl()) { 4671 __ and_(o_l, second_reg, ShifterOperand(kMaxLongShiftDistance)); 4672 // Shift the high part 4673 __ Lsl(o_h, high, o_l); 4674 // Shift the low part and `or` what overflew on the high part 4675 __ rsb(temp, o_l, ShifterOperand(kArmBitsPerWord)); 4676 __ Lsr(temp, low, temp); 4677 __ orr(o_h, o_h, ShifterOperand(temp)); 4678 // If the shift is > 32 bits, override the high part 4679 __ subs(temp, o_l, ShifterOperand(kArmBitsPerWord)); 4680 __ it(PL); 4681 __ Lsl(o_h, low, temp, PL); 4682 // Shift the low part 4683 __ Lsl(o_l, low, o_l); 4684 } else if (op->IsShr()) { 4685 __ and_(o_h, second_reg, ShifterOperand(kMaxLongShiftDistance)); 4686 // Shift the low part 4687 __ Lsr(o_l, low, o_h); 4688 // Shift the high part and `or` what underflew on the low part 4689 __ rsb(temp, o_h, ShifterOperand(kArmBitsPerWord)); 4690 __ Lsl(temp, high, temp); 4691 __ orr(o_l, o_l, ShifterOperand(temp)); 4692 // If the shift is > 32 bits, override the low part 4693 __ subs(temp, o_h, ShifterOperand(kArmBitsPerWord)); 4694 __ it(PL); 4695 __ Asr(o_l, high, temp, PL); 4696 // Shift the high part 4697 __ Asr(o_h, high, o_h); 4698 } else { 4699 __ and_(o_h, second_reg, ShifterOperand(kMaxLongShiftDistance)); 4700 // same as Shr except we use `Lsr`s and not `Asr`s 4701 __ Lsr(o_l, low, o_h); 4702 __ rsb(temp, o_h, ShifterOperand(kArmBitsPerWord)); 4703 __ Lsl(temp, high, temp); 4704 __ orr(o_l, o_l, ShifterOperand(temp)); 4705 __ subs(temp, o_h, ShifterOperand(kArmBitsPerWord)); 4706 __ it(PL); 4707 __ Lsr(o_l, high, temp, PL); 4708 __ Lsr(o_h, high, o_h); 4709 } 4710 } else { 4711 // Register allocator doesn't create partial overlap. 4712 DCHECK_NE(o_l, high); 4713 DCHECK_NE(o_h, low); 4714 int32_t cst = second.GetConstant()->AsIntConstant()->GetValue(); 4715 uint32_t shift_value = cst & kMaxLongShiftDistance; 4716 if (shift_value > 32) { 4717 if (op->IsShl()) { 4718 __ Lsl(o_h, low, shift_value - 32); 4719 __ LoadImmediate(o_l, 0); 4720 } else if (op->IsShr()) { 4721 __ Asr(o_l, high, shift_value - 32); 4722 __ Asr(o_h, high, 31); 4723 } else { 4724 __ Lsr(o_l, high, shift_value - 32); 4725 __ LoadImmediate(o_h, 0); 4726 } 4727 } else if (shift_value == 32) { 4728 if (op->IsShl()) { 4729 __ mov(o_h, ShifterOperand(low)); 4730 __ LoadImmediate(o_l, 0); 4731 } else if (op->IsShr()) { 4732 __ mov(o_l, ShifterOperand(high)); 4733 __ Asr(o_h, high, 31); 4734 } else { 4735 __ mov(o_l, ShifterOperand(high)); 4736 __ LoadImmediate(o_h, 0); 4737 } 4738 } else if (shift_value == 1) { 4739 if (op->IsShl()) { 4740 __ Lsls(o_l, low, 1); 4741 __ adc(o_h, high, ShifterOperand(high)); 4742 } else if (op->IsShr()) { 4743 __ Asrs(o_h, high, 1); 4744 __ Rrx(o_l, low); 4745 } else { 4746 __ Lsrs(o_h, high, 1); 4747 __ Rrx(o_l, low); 4748 } 4749 } else { 4750 DCHECK(2 <= shift_value && shift_value < 32) << shift_value; 4751 if (op->IsShl()) { 4752 __ Lsl(o_h, high, shift_value); 4753 __ orr(o_h, o_h, ShifterOperand(low, LSR, 32 - shift_value)); 4754 __ Lsl(o_l, low, shift_value); 4755 } else if (op->IsShr()) { 4756 __ Lsr(o_l, low, shift_value); 4757 __ orr(o_l, o_l, ShifterOperand(high, LSL, 32 - shift_value)); 4758 __ Asr(o_h, high, shift_value); 4759 } else { 4760 __ Lsr(o_l, low, shift_value); 4761 __ orr(o_l, o_l, ShifterOperand(high, LSL, 32 - shift_value)); 4762 __ Lsr(o_h, high, shift_value); 4763 } 4764 } 4765 } 4766 break; 4767 } 4768 default: 4769 LOG(FATAL) << "Unexpected operation type " << type; 4770 UNREACHABLE(); 4771 } 4772} 4773 4774void LocationsBuilderARM::VisitShl(HShl* shl) { 4775 HandleShift(shl); 4776} 4777 4778void InstructionCodeGeneratorARM::VisitShl(HShl* shl) { 4779 HandleShift(shl); 4780} 4781 4782void LocationsBuilderARM::VisitShr(HShr* shr) { 4783 HandleShift(shr); 4784} 4785 4786void InstructionCodeGeneratorARM::VisitShr(HShr* shr) { 4787 HandleShift(shr); 4788} 4789 4790void LocationsBuilderARM::VisitUShr(HUShr* ushr) { 4791 HandleShift(ushr); 4792} 4793 4794void InstructionCodeGeneratorARM::VisitUShr(HUShr* ushr) { 4795 HandleShift(ushr); 4796} 4797 4798void LocationsBuilderARM::VisitNewInstance(HNewInstance* instruction) { 4799 LocationSummary* locations = 4800 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCallOnMainOnly); 4801 if (instruction->IsStringAlloc()) { 4802 locations->AddTemp(Location::RegisterLocation(kMethodRegisterArgument)); 4803 } else { 4804 InvokeRuntimeCallingConvention calling_convention; 4805 locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(0))); 4806 } 4807 locations->SetOut(Location::RegisterLocation(R0)); 4808} 4809 4810void InstructionCodeGeneratorARM::VisitNewInstance(HNewInstance* instruction) { 4811 // Note: if heap poisoning is enabled, the entry point takes cares 4812 // of poisoning the reference. 4813 if (instruction->IsStringAlloc()) { 4814 // String is allocated through StringFactory. Call NewEmptyString entry point. 4815 Register temp = instruction->GetLocations()->GetTemp(0).AsRegister<Register>(); 4816 MemberOffset code_offset = ArtMethod::EntryPointFromQuickCompiledCodeOffset(kArmPointerSize); 4817 __ LoadFromOffset(kLoadWord, temp, TR, QUICK_ENTRY_POINT(pNewEmptyString)); 4818 __ LoadFromOffset(kLoadWord, LR, temp, code_offset.Int32Value()); 4819 __ blx(LR); 4820 codegen_->RecordPcInfo(instruction, instruction->GetDexPc()); 4821 } else { 4822 codegen_->InvokeRuntime(instruction->GetEntrypoint(), instruction, instruction->GetDexPc()); 4823 CheckEntrypointTypes<kQuickAllocObjectWithChecks, void*, mirror::Class*>(); 4824 } 4825} 4826 4827void LocationsBuilderARM::VisitNewArray(HNewArray* instruction) { 4828 LocationSummary* locations = 4829 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCallOnMainOnly); 4830 InvokeRuntimeCallingConvention calling_convention; 4831 locations->SetOut(Location::RegisterLocation(R0)); 4832 locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(0))); 4833 locations->SetInAt(1, Location::RegisterLocation(calling_convention.GetRegisterAt(1))); 4834} 4835 4836void InstructionCodeGeneratorARM::VisitNewArray(HNewArray* instruction) { 4837 // Note: if heap poisoning is enabled, the entry point takes cares 4838 // of poisoning the reference. 4839 QuickEntrypointEnum entrypoint = 4840 CodeGenerator::GetArrayAllocationEntrypoint(instruction->GetLoadClass()->GetClass()); 4841 codegen_->InvokeRuntime(entrypoint, instruction, instruction->GetDexPc()); 4842 CheckEntrypointTypes<kQuickAllocArrayResolved, void*, mirror::Class*, int32_t>(); 4843 DCHECK(!codegen_->IsLeafMethod()); 4844} 4845 4846void LocationsBuilderARM::VisitParameterValue(HParameterValue* instruction) { 4847 LocationSummary* locations = 4848 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall); 4849 Location location = parameter_visitor_.GetNextLocation(instruction->GetType()); 4850 if (location.IsStackSlot()) { 4851 location = Location::StackSlot(location.GetStackIndex() + codegen_->GetFrameSize()); 4852 } else if (location.IsDoubleStackSlot()) { 4853 location = Location::DoubleStackSlot(location.GetStackIndex() + codegen_->GetFrameSize()); 4854 } 4855 locations->SetOut(location); 4856} 4857 4858void InstructionCodeGeneratorARM::VisitParameterValue( 4859 HParameterValue* instruction ATTRIBUTE_UNUSED) { 4860 // Nothing to do, the parameter is already at its location. 4861} 4862 4863void LocationsBuilderARM::VisitCurrentMethod(HCurrentMethod* instruction) { 4864 LocationSummary* locations = 4865 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall); 4866 locations->SetOut(Location::RegisterLocation(kMethodRegisterArgument)); 4867} 4868 4869void InstructionCodeGeneratorARM::VisitCurrentMethod(HCurrentMethod* instruction ATTRIBUTE_UNUSED) { 4870 // Nothing to do, the method is already at its location. 4871} 4872 4873void LocationsBuilderARM::VisitNot(HNot* not_) { 4874 LocationSummary* locations = 4875 new (GetGraph()->GetArena()) LocationSummary(not_, LocationSummary::kNoCall); 4876 locations->SetInAt(0, Location::RequiresRegister()); 4877 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 4878} 4879 4880void InstructionCodeGeneratorARM::VisitNot(HNot* not_) { 4881 LocationSummary* locations = not_->GetLocations(); 4882 Location out = locations->Out(); 4883 Location in = locations->InAt(0); 4884 switch (not_->GetResultType()) { 4885 case Primitive::kPrimInt: 4886 __ mvn(out.AsRegister<Register>(), ShifterOperand(in.AsRegister<Register>())); 4887 break; 4888 4889 case Primitive::kPrimLong: 4890 __ mvn(out.AsRegisterPairLow<Register>(), 4891 ShifterOperand(in.AsRegisterPairLow<Register>())); 4892 __ mvn(out.AsRegisterPairHigh<Register>(), 4893 ShifterOperand(in.AsRegisterPairHigh<Register>())); 4894 break; 4895 4896 default: 4897 LOG(FATAL) << "Unimplemented type for not operation " << not_->GetResultType(); 4898 } 4899} 4900 4901void LocationsBuilderARM::VisitBooleanNot(HBooleanNot* bool_not) { 4902 LocationSummary* locations = 4903 new (GetGraph()->GetArena()) LocationSummary(bool_not, LocationSummary::kNoCall); 4904 locations->SetInAt(0, Location::RequiresRegister()); 4905 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 4906} 4907 4908void InstructionCodeGeneratorARM::VisitBooleanNot(HBooleanNot* bool_not) { 4909 LocationSummary* locations = bool_not->GetLocations(); 4910 Location out = locations->Out(); 4911 Location in = locations->InAt(0); 4912 __ eor(out.AsRegister<Register>(), in.AsRegister<Register>(), ShifterOperand(1)); 4913} 4914 4915void LocationsBuilderARM::VisitCompare(HCompare* compare) { 4916 LocationSummary* locations = 4917 new (GetGraph()->GetArena()) LocationSummary(compare, LocationSummary::kNoCall); 4918 switch (compare->InputAt(0)->GetType()) { 4919 case Primitive::kPrimBoolean: 4920 case Primitive::kPrimByte: 4921 case Primitive::kPrimShort: 4922 case Primitive::kPrimChar: 4923 case Primitive::kPrimInt: 4924 case Primitive::kPrimLong: { 4925 locations->SetInAt(0, Location::RequiresRegister()); 4926 locations->SetInAt(1, Location::RequiresRegister()); 4927 // Output overlaps because it is written before doing the low comparison. 4928 locations->SetOut(Location::RequiresRegister(), Location::kOutputOverlap); 4929 break; 4930 } 4931 case Primitive::kPrimFloat: 4932 case Primitive::kPrimDouble: { 4933 locations->SetInAt(0, Location::RequiresFpuRegister()); 4934 locations->SetInAt(1, ArithmeticZeroOrFpuRegister(compare->InputAt(1))); 4935 locations->SetOut(Location::RequiresRegister()); 4936 break; 4937 } 4938 default: 4939 LOG(FATAL) << "Unexpected type for compare operation " << compare->InputAt(0)->GetType(); 4940 } 4941} 4942 4943void InstructionCodeGeneratorARM::VisitCompare(HCompare* compare) { 4944 LocationSummary* locations = compare->GetLocations(); 4945 Register out = locations->Out().AsRegister<Register>(); 4946 Location left = locations->InAt(0); 4947 Location right = locations->InAt(1); 4948 4949 Label less, greater, done; 4950 Label* final_label = codegen_->GetFinalLabel(compare, &done); 4951 Primitive::Type type = compare->InputAt(0)->GetType(); 4952 Condition less_cond; 4953 switch (type) { 4954 case Primitive::kPrimBoolean: 4955 case Primitive::kPrimByte: 4956 case Primitive::kPrimShort: 4957 case Primitive::kPrimChar: 4958 case Primitive::kPrimInt: { 4959 __ LoadImmediate(out, 0); 4960 __ cmp(left.AsRegister<Register>(), 4961 ShifterOperand(right.AsRegister<Register>())); // Signed compare. 4962 less_cond = LT; 4963 break; 4964 } 4965 case Primitive::kPrimLong: { 4966 __ cmp(left.AsRegisterPairHigh<Register>(), 4967 ShifterOperand(right.AsRegisterPairHigh<Register>())); // Signed compare. 4968 __ b(&less, LT); 4969 __ b(&greater, GT); 4970 // Do LoadImmediate before the last `cmp`, as LoadImmediate might affect the status flags. 4971 __ LoadImmediate(out, 0); 4972 __ cmp(left.AsRegisterPairLow<Register>(), 4973 ShifterOperand(right.AsRegisterPairLow<Register>())); // Unsigned compare. 4974 less_cond = LO; 4975 break; 4976 } 4977 case Primitive::kPrimFloat: 4978 case Primitive::kPrimDouble: { 4979 __ LoadImmediate(out, 0); 4980 GenerateVcmp(compare, codegen_); 4981 __ vmstat(); // transfer FP status register to ARM APSR. 4982 less_cond = ARMFPCondition(kCondLT, compare->IsGtBias()); 4983 break; 4984 } 4985 default: 4986 LOG(FATAL) << "Unexpected compare type " << type; 4987 UNREACHABLE(); 4988 } 4989 4990 __ b(final_label, EQ); 4991 __ b(&less, less_cond); 4992 4993 __ Bind(&greater); 4994 __ LoadImmediate(out, 1); 4995 __ b(final_label); 4996 4997 __ Bind(&less); 4998 __ LoadImmediate(out, -1); 4999 5000 if (done.IsLinked()) { 5001 __ Bind(&done); 5002 } 5003} 5004 5005void LocationsBuilderARM::VisitPhi(HPhi* instruction) { 5006 LocationSummary* locations = 5007 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall); 5008 for (size_t i = 0, e = locations->GetInputCount(); i < e; ++i) { 5009 locations->SetInAt(i, Location::Any()); 5010 } 5011 locations->SetOut(Location::Any()); 5012} 5013 5014void InstructionCodeGeneratorARM::VisitPhi(HPhi* instruction ATTRIBUTE_UNUSED) { 5015 LOG(FATAL) << "Unreachable"; 5016} 5017 5018void CodeGeneratorARM::GenerateMemoryBarrier(MemBarrierKind kind) { 5019 // TODO (ported from quick): revisit ARM barrier kinds. 5020 DmbOptions flavor = DmbOptions::ISH; // Quiet C++ warnings. 5021 switch (kind) { 5022 case MemBarrierKind::kAnyStore: 5023 case MemBarrierKind::kLoadAny: 5024 case MemBarrierKind::kAnyAny: { 5025 flavor = DmbOptions::ISH; 5026 break; 5027 } 5028 case MemBarrierKind::kStoreStore: { 5029 flavor = DmbOptions::ISHST; 5030 break; 5031 } 5032 default: 5033 LOG(FATAL) << "Unexpected memory barrier " << kind; 5034 } 5035 __ dmb(flavor); 5036} 5037 5038void InstructionCodeGeneratorARM::GenerateWideAtomicLoad(Register addr, 5039 uint32_t offset, 5040 Register out_lo, 5041 Register out_hi) { 5042 if (offset != 0) { 5043 // Ensure `out_lo` is different from `addr`, so that loading 5044 // `offset` into `out_lo` does not clutter `addr`. 5045 DCHECK_NE(out_lo, addr); 5046 __ LoadImmediate(out_lo, offset); 5047 __ add(IP, addr, ShifterOperand(out_lo)); 5048 addr = IP; 5049 } 5050 __ ldrexd(out_lo, out_hi, addr); 5051} 5052 5053void InstructionCodeGeneratorARM::GenerateWideAtomicStore(Register addr, 5054 uint32_t offset, 5055 Register value_lo, 5056 Register value_hi, 5057 Register temp1, 5058 Register temp2, 5059 HInstruction* instruction) { 5060 Label fail; 5061 if (offset != 0) { 5062 __ LoadImmediate(temp1, offset); 5063 __ add(IP, addr, ShifterOperand(temp1)); 5064 addr = IP; 5065 } 5066 __ Bind(&fail); 5067 // We need a load followed by store. (The address used in a STREX instruction must 5068 // be the same as the address in the most recently executed LDREX instruction.) 5069 __ ldrexd(temp1, temp2, addr); 5070 codegen_->MaybeRecordImplicitNullCheck(instruction); 5071 __ strexd(temp1, value_lo, value_hi, addr); 5072 __ CompareAndBranchIfNonZero(temp1, &fail); 5073} 5074 5075void LocationsBuilderARM::HandleFieldSet(HInstruction* instruction, const FieldInfo& field_info) { 5076 DCHECK(instruction->IsInstanceFieldSet() || instruction->IsStaticFieldSet()); 5077 5078 LocationSummary* locations = 5079 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall); 5080 locations->SetInAt(0, Location::RequiresRegister()); 5081 5082 Primitive::Type field_type = field_info.GetFieldType(); 5083 if (Primitive::IsFloatingPointType(field_type)) { 5084 locations->SetInAt(1, Location::RequiresFpuRegister()); 5085 } else { 5086 locations->SetInAt(1, Location::RequiresRegister()); 5087 } 5088 5089 bool is_wide = field_type == Primitive::kPrimLong || field_type == Primitive::kPrimDouble; 5090 bool generate_volatile = field_info.IsVolatile() 5091 && is_wide 5092 && !codegen_->GetInstructionSetFeatures().HasAtomicLdrdAndStrd(); 5093 bool needs_write_barrier = 5094 CodeGenerator::StoreNeedsWriteBarrier(field_type, instruction->InputAt(1)); 5095 // Temporary registers for the write barrier. 5096 // TODO: consider renaming StoreNeedsWriteBarrier to StoreNeedsGCMark. 5097 if (needs_write_barrier) { 5098 locations->AddTemp(Location::RequiresRegister()); // Possibly used for reference poisoning too. 5099 locations->AddTemp(Location::RequiresRegister()); 5100 } else if (generate_volatile) { 5101 // ARM encoding have some additional constraints for ldrexd/strexd: 5102 // - registers need to be consecutive 5103 // - the first register should be even but not R14. 5104 // We don't test for ARM yet, and the assertion makes sure that we 5105 // revisit this if we ever enable ARM encoding. 5106 DCHECK_EQ(InstructionSet::kThumb2, codegen_->GetInstructionSet()); 5107 5108 locations->AddTemp(Location::RequiresRegister()); 5109 locations->AddTemp(Location::RequiresRegister()); 5110 if (field_type == Primitive::kPrimDouble) { 5111 // For doubles we need two more registers to copy the value. 5112 locations->AddTemp(Location::RegisterLocation(R2)); 5113 locations->AddTemp(Location::RegisterLocation(R3)); 5114 } 5115 } 5116} 5117 5118void InstructionCodeGeneratorARM::HandleFieldSet(HInstruction* instruction, 5119 const FieldInfo& field_info, 5120 bool value_can_be_null) { 5121 DCHECK(instruction->IsInstanceFieldSet() || instruction->IsStaticFieldSet()); 5122 5123 LocationSummary* locations = instruction->GetLocations(); 5124 Register base = locations->InAt(0).AsRegister<Register>(); 5125 Location value = locations->InAt(1); 5126 5127 bool is_volatile = field_info.IsVolatile(); 5128 bool atomic_ldrd_strd = codegen_->GetInstructionSetFeatures().HasAtomicLdrdAndStrd(); 5129 Primitive::Type field_type = field_info.GetFieldType(); 5130 uint32_t offset = field_info.GetFieldOffset().Uint32Value(); 5131 bool needs_write_barrier = 5132 CodeGenerator::StoreNeedsWriteBarrier(field_type, instruction->InputAt(1)); 5133 5134 if (is_volatile) { 5135 codegen_->GenerateMemoryBarrier(MemBarrierKind::kAnyStore); 5136 } 5137 5138 switch (field_type) { 5139 case Primitive::kPrimBoolean: 5140 case Primitive::kPrimByte: { 5141 __ StoreToOffset(kStoreByte, value.AsRegister<Register>(), base, offset); 5142 break; 5143 } 5144 5145 case Primitive::kPrimShort: 5146 case Primitive::kPrimChar: { 5147 __ StoreToOffset(kStoreHalfword, value.AsRegister<Register>(), base, offset); 5148 break; 5149 } 5150 5151 case Primitive::kPrimInt: 5152 case Primitive::kPrimNot: { 5153 if (kPoisonHeapReferences && needs_write_barrier) { 5154 // Note that in the case where `value` is a null reference, 5155 // we do not enter this block, as a null reference does not 5156 // need poisoning. 5157 DCHECK_EQ(field_type, Primitive::kPrimNot); 5158 Register temp = locations->GetTemp(0).AsRegister<Register>(); 5159 __ Mov(temp, value.AsRegister<Register>()); 5160 __ PoisonHeapReference(temp); 5161 __ StoreToOffset(kStoreWord, temp, base, offset); 5162 } else { 5163 __ StoreToOffset(kStoreWord, value.AsRegister<Register>(), base, offset); 5164 } 5165 break; 5166 } 5167 5168 case Primitive::kPrimLong: { 5169 if (is_volatile && !atomic_ldrd_strd) { 5170 GenerateWideAtomicStore(base, offset, 5171 value.AsRegisterPairLow<Register>(), 5172 value.AsRegisterPairHigh<Register>(), 5173 locations->GetTemp(0).AsRegister<Register>(), 5174 locations->GetTemp(1).AsRegister<Register>(), 5175 instruction); 5176 } else { 5177 __ StoreToOffset(kStoreWordPair, value.AsRegisterPairLow<Register>(), base, offset); 5178 codegen_->MaybeRecordImplicitNullCheck(instruction); 5179 } 5180 break; 5181 } 5182 5183 case Primitive::kPrimFloat: { 5184 __ StoreSToOffset(value.AsFpuRegister<SRegister>(), base, offset); 5185 break; 5186 } 5187 5188 case Primitive::kPrimDouble: { 5189 DRegister value_reg = FromLowSToD(value.AsFpuRegisterPairLow<SRegister>()); 5190 if (is_volatile && !atomic_ldrd_strd) { 5191 Register value_reg_lo = locations->GetTemp(0).AsRegister<Register>(); 5192 Register value_reg_hi = locations->GetTemp(1).AsRegister<Register>(); 5193 5194 __ vmovrrd(value_reg_lo, value_reg_hi, value_reg); 5195 5196 GenerateWideAtomicStore(base, offset, 5197 value_reg_lo, 5198 value_reg_hi, 5199 locations->GetTemp(2).AsRegister<Register>(), 5200 locations->GetTemp(3).AsRegister<Register>(), 5201 instruction); 5202 } else { 5203 __ StoreDToOffset(value_reg, base, offset); 5204 codegen_->MaybeRecordImplicitNullCheck(instruction); 5205 } 5206 break; 5207 } 5208 5209 case Primitive::kPrimVoid: 5210 LOG(FATAL) << "Unreachable type " << field_type; 5211 UNREACHABLE(); 5212 } 5213 5214 // Longs and doubles are handled in the switch. 5215 if (field_type != Primitive::kPrimLong && field_type != Primitive::kPrimDouble) { 5216 codegen_->MaybeRecordImplicitNullCheck(instruction); 5217 } 5218 5219 if (CodeGenerator::StoreNeedsWriteBarrier(field_type, instruction->InputAt(1))) { 5220 Register temp = locations->GetTemp(0).AsRegister<Register>(); 5221 Register card = locations->GetTemp(1).AsRegister<Register>(); 5222 codegen_->MarkGCCard( 5223 temp, card, base, value.AsRegister<Register>(), value_can_be_null); 5224 } 5225 5226 if (is_volatile) { 5227 codegen_->GenerateMemoryBarrier(MemBarrierKind::kAnyAny); 5228 } 5229} 5230 5231void LocationsBuilderARM::HandleFieldGet(HInstruction* instruction, const FieldInfo& field_info) { 5232 DCHECK(instruction->IsInstanceFieldGet() || instruction->IsStaticFieldGet()); 5233 5234 bool object_field_get_with_read_barrier = 5235 kEmitCompilerReadBarrier && (field_info.GetFieldType() == Primitive::kPrimNot); 5236 LocationSummary* locations = 5237 new (GetGraph()->GetArena()) LocationSummary(instruction, 5238 object_field_get_with_read_barrier ? 5239 LocationSummary::kCallOnSlowPath : 5240 LocationSummary::kNoCall); 5241 if (object_field_get_with_read_barrier && kUseBakerReadBarrier) { 5242 locations->SetCustomSlowPathCallerSaves(RegisterSet::Empty()); // No caller-save registers. 5243 } 5244 locations->SetInAt(0, Location::RequiresRegister()); 5245 5246 bool volatile_for_double = field_info.IsVolatile() 5247 && (field_info.GetFieldType() == Primitive::kPrimDouble) 5248 && !codegen_->GetInstructionSetFeatures().HasAtomicLdrdAndStrd(); 5249 // The output overlaps in case of volatile long: we don't want the 5250 // code generated by GenerateWideAtomicLoad to overwrite the 5251 // object's location. Likewise, in the case of an object field get 5252 // with read barriers enabled, we do not want the load to overwrite 5253 // the object's location, as we need it to emit the read barrier. 5254 bool overlap = (field_info.IsVolatile() && (field_info.GetFieldType() == Primitive::kPrimLong)) || 5255 object_field_get_with_read_barrier; 5256 5257 if (Primitive::IsFloatingPointType(instruction->GetType())) { 5258 locations->SetOut(Location::RequiresFpuRegister()); 5259 } else { 5260 locations->SetOut(Location::RequiresRegister(), 5261 (overlap ? Location::kOutputOverlap : Location::kNoOutputOverlap)); 5262 } 5263 if (volatile_for_double) { 5264 // ARM encoding have some additional constraints for ldrexd/strexd: 5265 // - registers need to be consecutive 5266 // - the first register should be even but not R14. 5267 // We don't test for ARM yet, and the assertion makes sure that we 5268 // revisit this if we ever enable ARM encoding. 5269 DCHECK_EQ(InstructionSet::kThumb2, codegen_->GetInstructionSet()); 5270 locations->AddTemp(Location::RequiresRegister()); 5271 locations->AddTemp(Location::RequiresRegister()); 5272 } else if (object_field_get_with_read_barrier && kUseBakerReadBarrier) { 5273 // We need a temporary register for the read barrier marking slow 5274 // path in CodeGeneratorARM::GenerateFieldLoadWithBakerReadBarrier. 5275 locations->AddTemp(Location::RequiresRegister()); 5276 } 5277} 5278 5279Location LocationsBuilderARM::ArithmeticZeroOrFpuRegister(HInstruction* input) { 5280 DCHECK(input->GetType() == Primitive::kPrimDouble || input->GetType() == Primitive::kPrimFloat) 5281 << input->GetType(); 5282 if ((input->IsFloatConstant() && (input->AsFloatConstant()->IsArithmeticZero())) || 5283 (input->IsDoubleConstant() && (input->AsDoubleConstant()->IsArithmeticZero()))) { 5284 return Location::ConstantLocation(input->AsConstant()); 5285 } else { 5286 return Location::RequiresFpuRegister(); 5287 } 5288} 5289 5290Location LocationsBuilderARM::ArmEncodableConstantOrRegister(HInstruction* constant, 5291 Opcode opcode) { 5292 DCHECK(!Primitive::IsFloatingPointType(constant->GetType())); 5293 if (constant->IsConstant() && 5294 CanEncodeConstantAsImmediate(constant->AsConstant(), opcode)) { 5295 return Location::ConstantLocation(constant->AsConstant()); 5296 } 5297 return Location::RequiresRegister(); 5298} 5299 5300bool LocationsBuilderARM::CanEncodeConstantAsImmediate(HConstant* input_cst, 5301 Opcode opcode) { 5302 uint64_t value = static_cast<uint64_t>(Int64FromConstant(input_cst)); 5303 if (Primitive::Is64BitType(input_cst->GetType())) { 5304 Opcode high_opcode = opcode; 5305 SetCc low_set_cc = kCcDontCare; 5306 switch (opcode) { 5307 case SUB: 5308 // Flip the operation to an ADD. 5309 value = -value; 5310 opcode = ADD; 5311 FALLTHROUGH_INTENDED; 5312 case ADD: 5313 if (Low32Bits(value) == 0u) { 5314 return CanEncodeConstantAsImmediate(High32Bits(value), opcode, kCcDontCare); 5315 } 5316 high_opcode = ADC; 5317 low_set_cc = kCcSet; 5318 break; 5319 default: 5320 break; 5321 } 5322 return CanEncodeConstantAsImmediate(Low32Bits(value), opcode, low_set_cc) && 5323 CanEncodeConstantAsImmediate(High32Bits(value), high_opcode, kCcDontCare); 5324 } else { 5325 return CanEncodeConstantAsImmediate(Low32Bits(value), opcode); 5326 } 5327} 5328 5329bool LocationsBuilderARM::CanEncodeConstantAsImmediate(uint32_t value, 5330 Opcode opcode, 5331 SetCc set_cc) { 5332 ShifterOperand so; 5333 ArmAssembler* assembler = codegen_->GetAssembler(); 5334 if (assembler->ShifterOperandCanHold(kNoRegister, kNoRegister, opcode, value, set_cc, &so)) { 5335 return true; 5336 } 5337 Opcode neg_opcode = kNoOperand; 5338 uint32_t neg_value = 0; 5339 switch (opcode) { 5340 case AND: neg_opcode = BIC; neg_value = ~value; break; 5341 case ORR: neg_opcode = ORN; neg_value = ~value; break; 5342 case ADD: neg_opcode = SUB; neg_value = -value; break; 5343 case ADC: neg_opcode = SBC; neg_value = ~value; break; 5344 case SUB: neg_opcode = ADD; neg_value = -value; break; 5345 case SBC: neg_opcode = ADC; neg_value = ~value; break; 5346 case MOV: neg_opcode = MVN; neg_value = ~value; break; 5347 default: 5348 return false; 5349 } 5350 5351 if (assembler->ShifterOperandCanHold(kNoRegister, 5352 kNoRegister, 5353 neg_opcode, 5354 neg_value, 5355 set_cc, 5356 &so)) { 5357 return true; 5358 } 5359 5360 return opcode == AND && IsPowerOfTwo(value + 1); 5361} 5362 5363void InstructionCodeGeneratorARM::HandleFieldGet(HInstruction* instruction, 5364 const FieldInfo& field_info) { 5365 DCHECK(instruction->IsInstanceFieldGet() || instruction->IsStaticFieldGet()); 5366 5367 LocationSummary* locations = instruction->GetLocations(); 5368 Location base_loc = locations->InAt(0); 5369 Register base = base_loc.AsRegister<Register>(); 5370 Location out = locations->Out(); 5371 bool is_volatile = field_info.IsVolatile(); 5372 bool atomic_ldrd_strd = codegen_->GetInstructionSetFeatures().HasAtomicLdrdAndStrd(); 5373 Primitive::Type field_type = field_info.GetFieldType(); 5374 uint32_t offset = field_info.GetFieldOffset().Uint32Value(); 5375 5376 switch (field_type) { 5377 case Primitive::kPrimBoolean: 5378 __ LoadFromOffset(kLoadUnsignedByte, out.AsRegister<Register>(), base, offset); 5379 break; 5380 5381 case Primitive::kPrimByte: 5382 __ LoadFromOffset(kLoadSignedByte, out.AsRegister<Register>(), base, offset); 5383 break; 5384 5385 case Primitive::kPrimShort: 5386 __ LoadFromOffset(kLoadSignedHalfword, out.AsRegister<Register>(), base, offset); 5387 break; 5388 5389 case Primitive::kPrimChar: 5390 __ LoadFromOffset(kLoadUnsignedHalfword, out.AsRegister<Register>(), base, offset); 5391 break; 5392 5393 case Primitive::kPrimInt: 5394 __ LoadFromOffset(kLoadWord, out.AsRegister<Register>(), base, offset); 5395 break; 5396 5397 case Primitive::kPrimNot: { 5398 // /* HeapReference<Object> */ out = *(base + offset) 5399 if (kEmitCompilerReadBarrier && kUseBakerReadBarrier) { 5400 Location temp_loc = locations->GetTemp(0); 5401 // Note that a potential implicit null check is handled in this 5402 // CodeGeneratorARM::GenerateFieldLoadWithBakerReadBarrier call. 5403 codegen_->GenerateFieldLoadWithBakerReadBarrier( 5404 instruction, out, base, offset, temp_loc, /* needs_null_check */ true); 5405 if (is_volatile) { 5406 codegen_->GenerateMemoryBarrier(MemBarrierKind::kLoadAny); 5407 } 5408 } else { 5409 __ LoadFromOffset(kLoadWord, out.AsRegister<Register>(), base, offset); 5410 codegen_->MaybeRecordImplicitNullCheck(instruction); 5411 if (is_volatile) { 5412 codegen_->GenerateMemoryBarrier(MemBarrierKind::kLoadAny); 5413 } 5414 // If read barriers are enabled, emit read barriers other than 5415 // Baker's using a slow path (and also unpoison the loaded 5416 // reference, if heap poisoning is enabled). 5417 codegen_->MaybeGenerateReadBarrierSlow(instruction, out, out, base_loc, offset); 5418 } 5419 break; 5420 } 5421 5422 case Primitive::kPrimLong: 5423 if (is_volatile && !atomic_ldrd_strd) { 5424 GenerateWideAtomicLoad(base, offset, 5425 out.AsRegisterPairLow<Register>(), 5426 out.AsRegisterPairHigh<Register>()); 5427 } else { 5428 __ LoadFromOffset(kLoadWordPair, out.AsRegisterPairLow<Register>(), base, offset); 5429 } 5430 break; 5431 5432 case Primitive::kPrimFloat: 5433 __ LoadSFromOffset(out.AsFpuRegister<SRegister>(), base, offset); 5434 break; 5435 5436 case Primitive::kPrimDouble: { 5437 DRegister out_reg = FromLowSToD(out.AsFpuRegisterPairLow<SRegister>()); 5438 if (is_volatile && !atomic_ldrd_strd) { 5439 Register lo = locations->GetTemp(0).AsRegister<Register>(); 5440 Register hi = locations->GetTemp(1).AsRegister<Register>(); 5441 GenerateWideAtomicLoad(base, offset, lo, hi); 5442 codegen_->MaybeRecordImplicitNullCheck(instruction); 5443 __ vmovdrr(out_reg, lo, hi); 5444 } else { 5445 __ LoadDFromOffset(out_reg, base, offset); 5446 codegen_->MaybeRecordImplicitNullCheck(instruction); 5447 } 5448 break; 5449 } 5450 5451 case Primitive::kPrimVoid: 5452 LOG(FATAL) << "Unreachable type " << field_type; 5453 UNREACHABLE(); 5454 } 5455 5456 if (field_type == Primitive::kPrimNot || field_type == Primitive::kPrimDouble) { 5457 // Potential implicit null checks, in the case of reference or 5458 // double fields, are handled in the previous switch statement. 5459 } else { 5460 codegen_->MaybeRecordImplicitNullCheck(instruction); 5461 } 5462 5463 if (is_volatile) { 5464 if (field_type == Primitive::kPrimNot) { 5465 // Memory barriers, in the case of references, are also handled 5466 // in the previous switch statement. 5467 } else { 5468 codegen_->GenerateMemoryBarrier(MemBarrierKind::kLoadAny); 5469 } 5470 } 5471} 5472 5473void LocationsBuilderARM::VisitInstanceFieldSet(HInstanceFieldSet* instruction) { 5474 HandleFieldSet(instruction, instruction->GetFieldInfo()); 5475} 5476 5477void InstructionCodeGeneratorARM::VisitInstanceFieldSet(HInstanceFieldSet* instruction) { 5478 HandleFieldSet(instruction, instruction->GetFieldInfo(), instruction->GetValueCanBeNull()); 5479} 5480 5481void LocationsBuilderARM::VisitInstanceFieldGet(HInstanceFieldGet* instruction) { 5482 HandleFieldGet(instruction, instruction->GetFieldInfo()); 5483} 5484 5485void InstructionCodeGeneratorARM::VisitInstanceFieldGet(HInstanceFieldGet* instruction) { 5486 HandleFieldGet(instruction, instruction->GetFieldInfo()); 5487} 5488 5489void LocationsBuilderARM::VisitStaticFieldGet(HStaticFieldGet* instruction) { 5490 HandleFieldGet(instruction, instruction->GetFieldInfo()); 5491} 5492 5493void InstructionCodeGeneratorARM::VisitStaticFieldGet(HStaticFieldGet* instruction) { 5494 HandleFieldGet(instruction, instruction->GetFieldInfo()); 5495} 5496 5497void LocationsBuilderARM::VisitStaticFieldSet(HStaticFieldSet* instruction) { 5498 HandleFieldSet(instruction, instruction->GetFieldInfo()); 5499} 5500 5501void InstructionCodeGeneratorARM::VisitStaticFieldSet(HStaticFieldSet* instruction) { 5502 HandleFieldSet(instruction, instruction->GetFieldInfo(), instruction->GetValueCanBeNull()); 5503} 5504 5505void LocationsBuilderARM::VisitUnresolvedInstanceFieldGet( 5506 HUnresolvedInstanceFieldGet* instruction) { 5507 FieldAccessCallingConventionARM calling_convention; 5508 codegen_->CreateUnresolvedFieldLocationSummary( 5509 instruction, instruction->GetFieldType(), calling_convention); 5510} 5511 5512void InstructionCodeGeneratorARM::VisitUnresolvedInstanceFieldGet( 5513 HUnresolvedInstanceFieldGet* instruction) { 5514 FieldAccessCallingConventionARM calling_convention; 5515 codegen_->GenerateUnresolvedFieldAccess(instruction, 5516 instruction->GetFieldType(), 5517 instruction->GetFieldIndex(), 5518 instruction->GetDexPc(), 5519 calling_convention); 5520} 5521 5522void LocationsBuilderARM::VisitUnresolvedInstanceFieldSet( 5523 HUnresolvedInstanceFieldSet* instruction) { 5524 FieldAccessCallingConventionARM calling_convention; 5525 codegen_->CreateUnresolvedFieldLocationSummary( 5526 instruction, instruction->GetFieldType(), calling_convention); 5527} 5528 5529void InstructionCodeGeneratorARM::VisitUnresolvedInstanceFieldSet( 5530 HUnresolvedInstanceFieldSet* instruction) { 5531 FieldAccessCallingConventionARM calling_convention; 5532 codegen_->GenerateUnresolvedFieldAccess(instruction, 5533 instruction->GetFieldType(), 5534 instruction->GetFieldIndex(), 5535 instruction->GetDexPc(), 5536 calling_convention); 5537} 5538 5539void LocationsBuilderARM::VisitUnresolvedStaticFieldGet( 5540 HUnresolvedStaticFieldGet* instruction) { 5541 FieldAccessCallingConventionARM calling_convention; 5542 codegen_->CreateUnresolvedFieldLocationSummary( 5543 instruction, instruction->GetFieldType(), calling_convention); 5544} 5545 5546void InstructionCodeGeneratorARM::VisitUnresolvedStaticFieldGet( 5547 HUnresolvedStaticFieldGet* instruction) { 5548 FieldAccessCallingConventionARM calling_convention; 5549 codegen_->GenerateUnresolvedFieldAccess(instruction, 5550 instruction->GetFieldType(), 5551 instruction->GetFieldIndex(), 5552 instruction->GetDexPc(), 5553 calling_convention); 5554} 5555 5556void LocationsBuilderARM::VisitUnresolvedStaticFieldSet( 5557 HUnresolvedStaticFieldSet* instruction) { 5558 FieldAccessCallingConventionARM calling_convention; 5559 codegen_->CreateUnresolvedFieldLocationSummary( 5560 instruction, instruction->GetFieldType(), calling_convention); 5561} 5562 5563void InstructionCodeGeneratorARM::VisitUnresolvedStaticFieldSet( 5564 HUnresolvedStaticFieldSet* instruction) { 5565 FieldAccessCallingConventionARM calling_convention; 5566 codegen_->GenerateUnresolvedFieldAccess(instruction, 5567 instruction->GetFieldType(), 5568 instruction->GetFieldIndex(), 5569 instruction->GetDexPc(), 5570 calling_convention); 5571} 5572 5573void LocationsBuilderARM::VisitNullCheck(HNullCheck* instruction) { 5574 LocationSummary* locations = codegen_->CreateThrowingSlowPathLocations(instruction); 5575 locations->SetInAt(0, Location::RequiresRegister()); 5576} 5577 5578void CodeGeneratorARM::GenerateImplicitNullCheck(HNullCheck* instruction) { 5579 if (CanMoveNullCheckToUser(instruction)) { 5580 return; 5581 } 5582 Location obj = instruction->GetLocations()->InAt(0); 5583 5584 __ LoadFromOffset(kLoadWord, IP, obj.AsRegister<Register>(), 0); 5585 RecordPcInfo(instruction, instruction->GetDexPc()); 5586} 5587 5588void CodeGeneratorARM::GenerateExplicitNullCheck(HNullCheck* instruction) { 5589 SlowPathCodeARM* slow_path = new (GetGraph()->GetArena()) NullCheckSlowPathARM(instruction); 5590 AddSlowPath(slow_path); 5591 5592 LocationSummary* locations = instruction->GetLocations(); 5593 Location obj = locations->InAt(0); 5594 5595 __ CompareAndBranchIfZero(obj.AsRegister<Register>(), slow_path->GetEntryLabel()); 5596} 5597 5598void InstructionCodeGeneratorARM::VisitNullCheck(HNullCheck* instruction) { 5599 codegen_->GenerateNullCheck(instruction); 5600} 5601 5602static LoadOperandType GetLoadOperandType(Primitive::Type type) { 5603 switch (type) { 5604 case Primitive::kPrimNot: 5605 return kLoadWord; 5606 case Primitive::kPrimBoolean: 5607 return kLoadUnsignedByte; 5608 case Primitive::kPrimByte: 5609 return kLoadSignedByte; 5610 case Primitive::kPrimChar: 5611 return kLoadUnsignedHalfword; 5612 case Primitive::kPrimShort: 5613 return kLoadSignedHalfword; 5614 case Primitive::kPrimInt: 5615 return kLoadWord; 5616 case Primitive::kPrimLong: 5617 return kLoadWordPair; 5618 case Primitive::kPrimFloat: 5619 return kLoadSWord; 5620 case Primitive::kPrimDouble: 5621 return kLoadDWord; 5622 default: 5623 LOG(FATAL) << "Unreachable type " << type; 5624 UNREACHABLE(); 5625 } 5626} 5627 5628static StoreOperandType GetStoreOperandType(Primitive::Type type) { 5629 switch (type) { 5630 case Primitive::kPrimNot: 5631 return kStoreWord; 5632 case Primitive::kPrimBoolean: 5633 case Primitive::kPrimByte: 5634 return kStoreByte; 5635 case Primitive::kPrimChar: 5636 case Primitive::kPrimShort: 5637 return kStoreHalfword; 5638 case Primitive::kPrimInt: 5639 return kStoreWord; 5640 case Primitive::kPrimLong: 5641 return kStoreWordPair; 5642 case Primitive::kPrimFloat: 5643 return kStoreSWord; 5644 case Primitive::kPrimDouble: 5645 return kStoreDWord; 5646 default: 5647 LOG(FATAL) << "Unreachable type " << type; 5648 UNREACHABLE(); 5649 } 5650} 5651 5652void CodeGeneratorARM::LoadFromShiftedRegOffset(Primitive::Type type, 5653 Location out_loc, 5654 Register base, 5655 Register reg_offset, 5656 Condition cond) { 5657 uint32_t shift_count = Primitive::ComponentSizeShift(type); 5658 Address mem_address(base, reg_offset, Shift::LSL, shift_count); 5659 5660 switch (type) { 5661 case Primitive::kPrimByte: 5662 __ ldrsb(out_loc.AsRegister<Register>(), mem_address, cond); 5663 break; 5664 case Primitive::kPrimBoolean: 5665 __ ldrb(out_loc.AsRegister<Register>(), mem_address, cond); 5666 break; 5667 case Primitive::kPrimShort: 5668 __ ldrsh(out_loc.AsRegister<Register>(), mem_address, cond); 5669 break; 5670 case Primitive::kPrimChar: 5671 __ ldrh(out_loc.AsRegister<Register>(), mem_address, cond); 5672 break; 5673 case Primitive::kPrimNot: 5674 case Primitive::kPrimInt: 5675 __ ldr(out_loc.AsRegister<Register>(), mem_address, cond); 5676 break; 5677 // T32 doesn't support LoadFromShiftedRegOffset mem address mode for these types. 5678 case Primitive::kPrimLong: 5679 case Primitive::kPrimFloat: 5680 case Primitive::kPrimDouble: 5681 default: 5682 LOG(FATAL) << "Unreachable type " << type; 5683 UNREACHABLE(); 5684 } 5685} 5686 5687void CodeGeneratorARM::StoreToShiftedRegOffset(Primitive::Type type, 5688 Location loc, 5689 Register base, 5690 Register reg_offset, 5691 Condition cond) { 5692 uint32_t shift_count = Primitive::ComponentSizeShift(type); 5693 Address mem_address(base, reg_offset, Shift::LSL, shift_count); 5694 5695 switch (type) { 5696 case Primitive::kPrimByte: 5697 case Primitive::kPrimBoolean: 5698 __ strb(loc.AsRegister<Register>(), mem_address, cond); 5699 break; 5700 case Primitive::kPrimShort: 5701 case Primitive::kPrimChar: 5702 __ strh(loc.AsRegister<Register>(), mem_address, cond); 5703 break; 5704 case Primitive::kPrimNot: 5705 case Primitive::kPrimInt: 5706 __ str(loc.AsRegister<Register>(), mem_address, cond); 5707 break; 5708 // T32 doesn't support StoreToShiftedRegOffset mem address mode for these types. 5709 case Primitive::kPrimLong: 5710 case Primitive::kPrimFloat: 5711 case Primitive::kPrimDouble: 5712 default: 5713 LOG(FATAL) << "Unreachable type " << type; 5714 UNREACHABLE(); 5715 } 5716} 5717 5718void LocationsBuilderARM::VisitArrayGet(HArrayGet* instruction) { 5719 bool object_array_get_with_read_barrier = 5720 kEmitCompilerReadBarrier && (instruction->GetType() == Primitive::kPrimNot); 5721 LocationSummary* locations = 5722 new (GetGraph()->GetArena()) LocationSummary(instruction, 5723 object_array_get_with_read_barrier ? 5724 LocationSummary::kCallOnSlowPath : 5725 LocationSummary::kNoCall); 5726 if (object_array_get_with_read_barrier && kUseBakerReadBarrier) { 5727 locations->SetCustomSlowPathCallerSaves(RegisterSet::Empty()); // No caller-save registers. 5728 } 5729 locations->SetInAt(0, Location::RequiresRegister()); 5730 locations->SetInAt(1, Location::RegisterOrConstant(instruction->InputAt(1))); 5731 if (Primitive::IsFloatingPointType(instruction->GetType())) { 5732 locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap); 5733 } else { 5734 // The output overlaps in the case of an object array get with 5735 // read barriers enabled: we do not want the move to overwrite the 5736 // array's location, as we need it to emit the read barrier. 5737 locations->SetOut( 5738 Location::RequiresRegister(), 5739 object_array_get_with_read_barrier ? Location::kOutputOverlap : Location::kNoOutputOverlap); 5740 } 5741 // We need a temporary register for the read barrier marking slow 5742 // path in CodeGeneratorARM::GenerateArrayLoadWithBakerReadBarrier. 5743 // Also need for String compression feature. 5744 if ((object_array_get_with_read_barrier && kUseBakerReadBarrier) 5745 || (mirror::kUseStringCompression && instruction->IsStringCharAt())) { 5746 locations->AddTemp(Location::RequiresRegister()); 5747 } 5748} 5749 5750void InstructionCodeGeneratorARM::VisitArrayGet(HArrayGet* instruction) { 5751 LocationSummary* locations = instruction->GetLocations(); 5752 Location obj_loc = locations->InAt(0); 5753 Register obj = obj_loc.AsRegister<Register>(); 5754 Location index = locations->InAt(1); 5755 Location out_loc = locations->Out(); 5756 uint32_t data_offset = CodeGenerator::GetArrayDataOffset(instruction); 5757 Primitive::Type type = instruction->GetType(); 5758 const bool maybe_compressed_char_at = mirror::kUseStringCompression && 5759 instruction->IsStringCharAt(); 5760 HInstruction* array_instr = instruction->GetArray(); 5761 bool has_intermediate_address = array_instr->IsIntermediateAddress(); 5762 5763 switch (type) { 5764 case Primitive::kPrimBoolean: 5765 case Primitive::kPrimByte: 5766 case Primitive::kPrimShort: 5767 case Primitive::kPrimChar: 5768 case Primitive::kPrimInt: { 5769 Register length; 5770 if (maybe_compressed_char_at) { 5771 length = locations->GetTemp(0).AsRegister<Register>(); 5772 uint32_t count_offset = mirror::String::CountOffset().Uint32Value(); 5773 __ LoadFromOffset(kLoadWord, length, obj, count_offset); 5774 codegen_->MaybeRecordImplicitNullCheck(instruction); 5775 } 5776 if (index.IsConstant()) { 5777 int32_t const_index = index.GetConstant()->AsIntConstant()->GetValue(); 5778 if (maybe_compressed_char_at) { 5779 Label uncompressed_load, done; 5780 Label* final_label = codegen_->GetFinalLabel(instruction, &done); 5781 __ Lsrs(length, length, 1u); // LSRS has a 16-bit encoding, TST (immediate) does not. 5782 static_assert(static_cast<uint32_t>(mirror::StringCompressionFlag::kCompressed) == 0u, 5783 "Expecting 0=compressed, 1=uncompressed"); 5784 __ b(&uncompressed_load, CS); 5785 __ LoadFromOffset(kLoadUnsignedByte, 5786 out_loc.AsRegister<Register>(), 5787 obj, 5788 data_offset + const_index); 5789 __ b(final_label); 5790 __ Bind(&uncompressed_load); 5791 __ LoadFromOffset(GetLoadOperandType(Primitive::kPrimChar), 5792 out_loc.AsRegister<Register>(), 5793 obj, 5794 data_offset + (const_index << 1)); 5795 if (done.IsLinked()) { 5796 __ Bind(&done); 5797 } 5798 } else { 5799 uint32_t full_offset = data_offset + (const_index << Primitive::ComponentSizeShift(type)); 5800 5801 LoadOperandType load_type = GetLoadOperandType(type); 5802 __ LoadFromOffset(load_type, out_loc.AsRegister<Register>(), obj, full_offset); 5803 } 5804 } else { 5805 Register temp = IP; 5806 5807 if (has_intermediate_address) { 5808 // We do not need to compute the intermediate address from the array: the 5809 // input instruction has done it already. See the comment in 5810 // `TryExtractArrayAccessAddress()`. 5811 if (kIsDebugBuild) { 5812 HIntermediateAddress* tmp = array_instr->AsIntermediateAddress(); 5813 DCHECK_EQ(tmp->GetOffset()->AsIntConstant()->GetValueAsUint64(), data_offset); 5814 } 5815 temp = obj; 5816 } else { 5817 __ add(temp, obj, ShifterOperand(data_offset)); 5818 } 5819 if (maybe_compressed_char_at) { 5820 Label uncompressed_load, done; 5821 Label* final_label = codegen_->GetFinalLabel(instruction, &done); 5822 __ Lsrs(length, length, 1u); // LSRS has a 16-bit encoding, TST (immediate) does not. 5823 static_assert(static_cast<uint32_t>(mirror::StringCompressionFlag::kCompressed) == 0u, 5824 "Expecting 0=compressed, 1=uncompressed"); 5825 __ b(&uncompressed_load, CS); 5826 __ ldrb(out_loc.AsRegister<Register>(), 5827 Address(temp, index.AsRegister<Register>(), Shift::LSL, 0)); 5828 __ b(final_label); 5829 __ Bind(&uncompressed_load); 5830 __ ldrh(out_loc.AsRegister<Register>(), 5831 Address(temp, index.AsRegister<Register>(), Shift::LSL, 1)); 5832 if (done.IsLinked()) { 5833 __ Bind(&done); 5834 } 5835 } else { 5836 codegen_->LoadFromShiftedRegOffset(type, out_loc, temp, index.AsRegister<Register>()); 5837 } 5838 } 5839 break; 5840 } 5841 5842 case Primitive::kPrimNot: { 5843 // The read barrier instrumentation of object ArrayGet 5844 // instructions does not support the HIntermediateAddress 5845 // instruction. 5846 DCHECK(!(has_intermediate_address && kEmitCompilerReadBarrier)); 5847 5848 static_assert( 5849 sizeof(mirror::HeapReference<mirror::Object>) == sizeof(int32_t), 5850 "art::mirror::HeapReference<art::mirror::Object> and int32_t have different sizes."); 5851 // /* HeapReference<Object> */ out = 5852 // *(obj + data_offset + index * sizeof(HeapReference<Object>)) 5853 if (kEmitCompilerReadBarrier && kUseBakerReadBarrier) { 5854 Location temp = locations->GetTemp(0); 5855 // Note that a potential implicit null check is handled in this 5856 // CodeGeneratorARM::GenerateArrayLoadWithBakerReadBarrier call. 5857 codegen_->GenerateArrayLoadWithBakerReadBarrier( 5858 instruction, out_loc, obj, data_offset, index, temp, /* needs_null_check */ true); 5859 } else { 5860 Register out = out_loc.AsRegister<Register>(); 5861 if (index.IsConstant()) { 5862 size_t offset = 5863 (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4) + data_offset; 5864 __ LoadFromOffset(kLoadWord, out, obj, offset); 5865 codegen_->MaybeRecordImplicitNullCheck(instruction); 5866 // If read barriers are enabled, emit read barriers other than 5867 // Baker's using a slow path (and also unpoison the loaded 5868 // reference, if heap poisoning is enabled). 5869 codegen_->MaybeGenerateReadBarrierSlow(instruction, out_loc, out_loc, obj_loc, offset); 5870 } else { 5871 Register temp = IP; 5872 5873 if (has_intermediate_address) { 5874 // We do not need to compute the intermediate address from the array: the 5875 // input instruction has done it already. See the comment in 5876 // `TryExtractArrayAccessAddress()`. 5877 if (kIsDebugBuild) { 5878 HIntermediateAddress* tmp = array_instr->AsIntermediateAddress(); 5879 DCHECK_EQ(tmp->GetOffset()->AsIntConstant()->GetValueAsUint64(), data_offset); 5880 } 5881 temp = obj; 5882 } else { 5883 __ add(temp, obj, ShifterOperand(data_offset)); 5884 } 5885 codegen_->LoadFromShiftedRegOffset(type, out_loc, temp, index.AsRegister<Register>()); 5886 5887 codegen_->MaybeRecordImplicitNullCheck(instruction); 5888 // If read barriers are enabled, emit read barriers other than 5889 // Baker's using a slow path (and also unpoison the loaded 5890 // reference, if heap poisoning is enabled). 5891 codegen_->MaybeGenerateReadBarrierSlow( 5892 instruction, out_loc, out_loc, obj_loc, data_offset, index); 5893 } 5894 } 5895 break; 5896 } 5897 5898 case Primitive::kPrimLong: { 5899 if (index.IsConstant()) { 5900 size_t offset = 5901 (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_8) + data_offset; 5902 __ LoadFromOffset(kLoadWordPair, out_loc.AsRegisterPairLow<Register>(), obj, offset); 5903 } else { 5904 __ add(IP, obj, ShifterOperand(index.AsRegister<Register>(), LSL, TIMES_8)); 5905 __ LoadFromOffset(kLoadWordPair, out_loc.AsRegisterPairLow<Register>(), IP, data_offset); 5906 } 5907 break; 5908 } 5909 5910 case Primitive::kPrimFloat: { 5911 SRegister out = out_loc.AsFpuRegister<SRegister>(); 5912 if (index.IsConstant()) { 5913 size_t offset = (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4) + data_offset; 5914 __ LoadSFromOffset(out, obj, offset); 5915 } else { 5916 __ add(IP, obj, ShifterOperand(index.AsRegister<Register>(), LSL, TIMES_4)); 5917 __ LoadSFromOffset(out, IP, data_offset); 5918 } 5919 break; 5920 } 5921 5922 case Primitive::kPrimDouble: { 5923 SRegister out = out_loc.AsFpuRegisterPairLow<SRegister>(); 5924 if (index.IsConstant()) { 5925 size_t offset = (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_8) + data_offset; 5926 __ LoadDFromOffset(FromLowSToD(out), obj, offset); 5927 } else { 5928 __ add(IP, obj, ShifterOperand(index.AsRegister<Register>(), LSL, TIMES_8)); 5929 __ LoadDFromOffset(FromLowSToD(out), IP, data_offset); 5930 } 5931 break; 5932 } 5933 5934 case Primitive::kPrimVoid: 5935 LOG(FATAL) << "Unreachable type " << type; 5936 UNREACHABLE(); 5937 } 5938 5939 if (type == Primitive::kPrimNot) { 5940 // Potential implicit null checks, in the case of reference 5941 // arrays, are handled in the previous switch statement. 5942 } else if (!maybe_compressed_char_at) { 5943 codegen_->MaybeRecordImplicitNullCheck(instruction); 5944 } 5945} 5946 5947void LocationsBuilderARM::VisitArraySet(HArraySet* instruction) { 5948 Primitive::Type value_type = instruction->GetComponentType(); 5949 5950 bool needs_write_barrier = 5951 CodeGenerator::StoreNeedsWriteBarrier(value_type, instruction->GetValue()); 5952 bool may_need_runtime_call_for_type_check = instruction->NeedsTypeCheck(); 5953 5954 LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary( 5955 instruction, 5956 may_need_runtime_call_for_type_check ? 5957 LocationSummary::kCallOnSlowPath : 5958 LocationSummary::kNoCall); 5959 5960 locations->SetInAt(0, Location::RequiresRegister()); 5961 locations->SetInAt(1, Location::RegisterOrConstant(instruction->InputAt(1))); 5962 if (Primitive::IsFloatingPointType(value_type)) { 5963 locations->SetInAt(2, Location::RequiresFpuRegister()); 5964 } else { 5965 locations->SetInAt(2, Location::RequiresRegister()); 5966 } 5967 if (needs_write_barrier) { 5968 // Temporary registers for the write barrier. 5969 locations->AddTemp(Location::RequiresRegister()); // Possibly used for ref. poisoning too. 5970 locations->AddTemp(Location::RequiresRegister()); 5971 } 5972} 5973 5974void InstructionCodeGeneratorARM::VisitArraySet(HArraySet* instruction) { 5975 LocationSummary* locations = instruction->GetLocations(); 5976 Location array_loc = locations->InAt(0); 5977 Register array = array_loc.AsRegister<Register>(); 5978 Location index = locations->InAt(1); 5979 Primitive::Type value_type = instruction->GetComponentType(); 5980 bool may_need_runtime_call_for_type_check = instruction->NeedsTypeCheck(); 5981 bool needs_write_barrier = 5982 CodeGenerator::StoreNeedsWriteBarrier(value_type, instruction->GetValue()); 5983 uint32_t data_offset = 5984 mirror::Array::DataOffset(Primitive::ComponentSize(value_type)).Uint32Value(); 5985 Location value_loc = locations->InAt(2); 5986 HInstruction* array_instr = instruction->GetArray(); 5987 bool has_intermediate_address = array_instr->IsIntermediateAddress(); 5988 5989 switch (value_type) { 5990 case Primitive::kPrimBoolean: 5991 case Primitive::kPrimByte: 5992 case Primitive::kPrimShort: 5993 case Primitive::kPrimChar: 5994 case Primitive::kPrimInt: { 5995 if (index.IsConstant()) { 5996 int32_t const_index = index.GetConstant()->AsIntConstant()->GetValue(); 5997 uint32_t full_offset = 5998 data_offset + (const_index << Primitive::ComponentSizeShift(value_type)); 5999 StoreOperandType store_type = GetStoreOperandType(value_type); 6000 __ StoreToOffset(store_type, value_loc.AsRegister<Register>(), array, full_offset); 6001 } else { 6002 Register temp = IP; 6003 6004 if (has_intermediate_address) { 6005 // We do not need to compute the intermediate address from the array: the 6006 // input instruction has done it already. See the comment in 6007 // `TryExtractArrayAccessAddress()`. 6008 if (kIsDebugBuild) { 6009 HIntermediateAddress* tmp = array_instr->AsIntermediateAddress(); 6010 DCHECK(tmp->GetOffset()->AsIntConstant()->GetValueAsUint64() == data_offset); 6011 } 6012 temp = array; 6013 } else { 6014 __ add(temp, array, ShifterOperand(data_offset)); 6015 } 6016 codegen_->StoreToShiftedRegOffset(value_type, 6017 value_loc, 6018 temp, 6019 index.AsRegister<Register>()); 6020 } 6021 break; 6022 } 6023 6024 case Primitive::kPrimNot: { 6025 Register value = value_loc.AsRegister<Register>(); 6026 // TryExtractArrayAccessAddress optimization is never applied for non-primitive ArraySet. 6027 // See the comment in instruction_simplifier_shared.cc. 6028 DCHECK(!has_intermediate_address); 6029 6030 if (instruction->InputAt(2)->IsNullConstant()) { 6031 // Just setting null. 6032 if (index.IsConstant()) { 6033 size_t offset = 6034 (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4) + data_offset; 6035 __ StoreToOffset(kStoreWord, value, array, offset); 6036 } else { 6037 DCHECK(index.IsRegister()) << index; 6038 __ add(IP, array, ShifterOperand(data_offset)); 6039 codegen_->StoreToShiftedRegOffset(value_type, 6040 value_loc, 6041 IP, 6042 index.AsRegister<Register>()); 6043 } 6044 codegen_->MaybeRecordImplicitNullCheck(instruction); 6045 DCHECK(!needs_write_barrier); 6046 DCHECK(!may_need_runtime_call_for_type_check); 6047 break; 6048 } 6049 6050 DCHECK(needs_write_barrier); 6051 Location temp1_loc = locations->GetTemp(0); 6052 Register temp1 = temp1_loc.AsRegister<Register>(); 6053 Location temp2_loc = locations->GetTemp(1); 6054 Register temp2 = temp2_loc.AsRegister<Register>(); 6055 uint32_t class_offset = mirror::Object::ClassOffset().Int32Value(); 6056 uint32_t super_offset = mirror::Class::SuperClassOffset().Int32Value(); 6057 uint32_t component_offset = mirror::Class::ComponentTypeOffset().Int32Value(); 6058 Label done; 6059 Label* final_label = codegen_->GetFinalLabel(instruction, &done); 6060 SlowPathCodeARM* slow_path = nullptr; 6061 6062 if (may_need_runtime_call_for_type_check) { 6063 slow_path = new (GetGraph()->GetArena()) ArraySetSlowPathARM(instruction); 6064 codegen_->AddSlowPath(slow_path); 6065 if (instruction->GetValueCanBeNull()) { 6066 Label non_zero; 6067 __ CompareAndBranchIfNonZero(value, &non_zero); 6068 if (index.IsConstant()) { 6069 size_t offset = 6070 (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4) + data_offset; 6071 __ StoreToOffset(kStoreWord, value, array, offset); 6072 } else { 6073 DCHECK(index.IsRegister()) << index; 6074 __ add(IP, array, ShifterOperand(data_offset)); 6075 codegen_->StoreToShiftedRegOffset(value_type, 6076 value_loc, 6077 IP, 6078 index.AsRegister<Register>()); 6079 } 6080 codegen_->MaybeRecordImplicitNullCheck(instruction); 6081 __ b(final_label); 6082 __ Bind(&non_zero); 6083 } 6084 6085 // Note that when read barriers are enabled, the type checks 6086 // are performed without read barriers. This is fine, even in 6087 // the case where a class object is in the from-space after 6088 // the flip, as a comparison involving such a type would not 6089 // produce a false positive; it may of course produce a false 6090 // negative, in which case we would take the ArraySet slow 6091 // path. 6092 6093 // /* HeapReference<Class> */ temp1 = array->klass_ 6094 __ LoadFromOffset(kLoadWord, temp1, array, class_offset); 6095 codegen_->MaybeRecordImplicitNullCheck(instruction); 6096 __ MaybeUnpoisonHeapReference(temp1); 6097 6098 // /* HeapReference<Class> */ temp1 = temp1->component_type_ 6099 __ LoadFromOffset(kLoadWord, temp1, temp1, component_offset); 6100 // /* HeapReference<Class> */ temp2 = value->klass_ 6101 __ LoadFromOffset(kLoadWord, temp2, value, class_offset); 6102 // If heap poisoning is enabled, no need to unpoison `temp1` 6103 // nor `temp2`, as we are comparing two poisoned references. 6104 __ cmp(temp1, ShifterOperand(temp2)); 6105 6106 if (instruction->StaticTypeOfArrayIsObjectArray()) { 6107 Label do_put; 6108 __ b(&do_put, EQ); 6109 // If heap poisoning is enabled, the `temp1` reference has 6110 // not been unpoisoned yet; unpoison it now. 6111 __ MaybeUnpoisonHeapReference(temp1); 6112 6113 // /* HeapReference<Class> */ temp1 = temp1->super_class_ 6114 __ LoadFromOffset(kLoadWord, temp1, temp1, super_offset); 6115 // If heap poisoning is enabled, no need to unpoison 6116 // `temp1`, as we are comparing against null below. 6117 __ CompareAndBranchIfNonZero(temp1, slow_path->GetEntryLabel()); 6118 __ Bind(&do_put); 6119 } else { 6120 __ b(slow_path->GetEntryLabel(), NE); 6121 } 6122 } 6123 6124 Register source = value; 6125 if (kPoisonHeapReferences) { 6126 // Note that in the case where `value` is a null reference, 6127 // we do not enter this block, as a null reference does not 6128 // need poisoning. 6129 DCHECK_EQ(value_type, Primitive::kPrimNot); 6130 __ Mov(temp1, value); 6131 __ PoisonHeapReference(temp1); 6132 source = temp1; 6133 } 6134 6135 if (index.IsConstant()) { 6136 size_t offset = 6137 (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4) + data_offset; 6138 __ StoreToOffset(kStoreWord, source, array, offset); 6139 } else { 6140 DCHECK(index.IsRegister()) << index; 6141 6142 __ add(IP, array, ShifterOperand(data_offset)); 6143 codegen_->StoreToShiftedRegOffset(value_type, 6144 Location::RegisterLocation(source), 6145 IP, 6146 index.AsRegister<Register>()); 6147 } 6148 6149 if (!may_need_runtime_call_for_type_check) { 6150 codegen_->MaybeRecordImplicitNullCheck(instruction); 6151 } 6152 6153 codegen_->MarkGCCard(temp1, temp2, array, value, instruction->GetValueCanBeNull()); 6154 6155 if (done.IsLinked()) { 6156 __ Bind(&done); 6157 } 6158 6159 if (slow_path != nullptr) { 6160 __ Bind(slow_path->GetExitLabel()); 6161 } 6162 6163 break; 6164 } 6165 6166 case Primitive::kPrimLong: { 6167 Location value = locations->InAt(2); 6168 if (index.IsConstant()) { 6169 size_t offset = 6170 (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_8) + data_offset; 6171 __ StoreToOffset(kStoreWordPair, value.AsRegisterPairLow<Register>(), array, offset); 6172 } else { 6173 __ add(IP, array, ShifterOperand(index.AsRegister<Register>(), LSL, TIMES_8)); 6174 __ StoreToOffset(kStoreWordPair, value.AsRegisterPairLow<Register>(), IP, data_offset); 6175 } 6176 break; 6177 } 6178 6179 case Primitive::kPrimFloat: { 6180 Location value = locations->InAt(2); 6181 DCHECK(value.IsFpuRegister()); 6182 if (index.IsConstant()) { 6183 size_t offset = (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4) + data_offset; 6184 __ StoreSToOffset(value.AsFpuRegister<SRegister>(), array, offset); 6185 } else { 6186 __ add(IP, array, ShifterOperand(index.AsRegister<Register>(), LSL, TIMES_4)); 6187 __ StoreSToOffset(value.AsFpuRegister<SRegister>(), IP, data_offset); 6188 } 6189 break; 6190 } 6191 6192 case Primitive::kPrimDouble: { 6193 Location value = locations->InAt(2); 6194 DCHECK(value.IsFpuRegisterPair()); 6195 if (index.IsConstant()) { 6196 size_t offset = (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_8) + data_offset; 6197 __ StoreDToOffset(FromLowSToD(value.AsFpuRegisterPairLow<SRegister>()), array, offset); 6198 } else { 6199 __ add(IP, array, ShifterOperand(index.AsRegister<Register>(), LSL, TIMES_8)); 6200 __ StoreDToOffset(FromLowSToD(value.AsFpuRegisterPairLow<SRegister>()), IP, data_offset); 6201 } 6202 6203 break; 6204 } 6205 6206 case Primitive::kPrimVoid: 6207 LOG(FATAL) << "Unreachable type " << value_type; 6208 UNREACHABLE(); 6209 } 6210 6211 // Objects are handled in the switch. 6212 if (value_type != Primitive::kPrimNot) { 6213 codegen_->MaybeRecordImplicitNullCheck(instruction); 6214 } 6215} 6216 6217void LocationsBuilderARM::VisitArrayLength(HArrayLength* instruction) { 6218 LocationSummary* locations = 6219 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall); 6220 locations->SetInAt(0, Location::RequiresRegister()); 6221 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 6222} 6223 6224void InstructionCodeGeneratorARM::VisitArrayLength(HArrayLength* instruction) { 6225 LocationSummary* locations = instruction->GetLocations(); 6226 uint32_t offset = CodeGenerator::GetArrayLengthOffset(instruction); 6227 Register obj = locations->InAt(0).AsRegister<Register>(); 6228 Register out = locations->Out().AsRegister<Register>(); 6229 __ LoadFromOffset(kLoadWord, out, obj, offset); 6230 codegen_->MaybeRecordImplicitNullCheck(instruction); 6231 // Mask out compression flag from String's array length. 6232 if (mirror::kUseStringCompression && instruction->IsStringLength()) { 6233 __ Lsr(out, out, 1u); 6234 } 6235} 6236 6237void LocationsBuilderARM::VisitIntermediateAddress(HIntermediateAddress* instruction) { 6238 LocationSummary* locations = 6239 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall); 6240 6241 locations->SetInAt(0, Location::RequiresRegister()); 6242 locations->SetInAt(1, Location::RegisterOrConstant(instruction->GetOffset())); 6243 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 6244} 6245 6246void InstructionCodeGeneratorARM::VisitIntermediateAddress(HIntermediateAddress* instruction) { 6247 LocationSummary* locations = instruction->GetLocations(); 6248 Location out = locations->Out(); 6249 Location first = locations->InAt(0); 6250 Location second = locations->InAt(1); 6251 6252 if (second.IsRegister()) { 6253 __ add(out.AsRegister<Register>(), 6254 first.AsRegister<Register>(), 6255 ShifterOperand(second.AsRegister<Register>())); 6256 } else { 6257 __ AddConstant(out.AsRegister<Register>(), 6258 first.AsRegister<Register>(), 6259 second.GetConstant()->AsIntConstant()->GetValue()); 6260 } 6261} 6262 6263void LocationsBuilderARM::VisitBoundsCheck(HBoundsCheck* instruction) { 6264 RegisterSet caller_saves = RegisterSet::Empty(); 6265 InvokeRuntimeCallingConvention calling_convention; 6266 caller_saves.Add(Location::RegisterLocation(calling_convention.GetRegisterAt(0))); 6267 caller_saves.Add(Location::RegisterLocation(calling_convention.GetRegisterAt(1))); 6268 LocationSummary* locations = codegen_->CreateThrowingSlowPathLocations(instruction, caller_saves); 6269 6270 HInstruction* index = instruction->InputAt(0); 6271 HInstruction* length = instruction->InputAt(1); 6272 // If both index and length are constants we can statically check the bounds. But if at least one 6273 // of them is not encodable ArmEncodableConstantOrRegister will create 6274 // Location::RequiresRegister() which is not desired to happen. Instead we create constant 6275 // locations. 6276 bool both_const = index->IsConstant() && length->IsConstant(); 6277 locations->SetInAt(0, both_const 6278 ? Location::ConstantLocation(index->AsConstant()) 6279 : ArmEncodableConstantOrRegister(index, CMP)); 6280 locations->SetInAt(1, both_const 6281 ? Location::ConstantLocation(length->AsConstant()) 6282 : ArmEncodableConstantOrRegister(length, CMP)); 6283} 6284 6285void InstructionCodeGeneratorARM::VisitBoundsCheck(HBoundsCheck* instruction) { 6286 LocationSummary* locations = instruction->GetLocations(); 6287 Location index_loc = locations->InAt(0); 6288 Location length_loc = locations->InAt(1); 6289 6290 if (length_loc.IsConstant()) { 6291 int32_t length = helpers::Int32ConstantFrom(length_loc); 6292 if (index_loc.IsConstant()) { 6293 // BCE will remove the bounds check if we are guaranteed to pass. 6294 int32_t index = helpers::Int32ConstantFrom(index_loc); 6295 if (index < 0 || index >= length) { 6296 SlowPathCodeARM* slow_path = 6297 new (GetGraph()->GetArena()) BoundsCheckSlowPathARM(instruction); 6298 codegen_->AddSlowPath(slow_path); 6299 __ b(slow_path->GetEntryLabel()); 6300 } else { 6301 // Some optimization after BCE may have generated this, and we should not 6302 // generate a bounds check if it is a valid range. 6303 } 6304 return; 6305 } 6306 6307 SlowPathCodeARM* slow_path = new (GetGraph()->GetArena()) BoundsCheckSlowPathARM(instruction); 6308 __ cmp(index_loc.AsRegister<Register>(), ShifterOperand(length)); 6309 codegen_->AddSlowPath(slow_path); 6310 __ b(slow_path->GetEntryLabel(), HS); 6311 } else { 6312 SlowPathCodeARM* slow_path = new (GetGraph()->GetArena()) BoundsCheckSlowPathARM(instruction); 6313 if (index_loc.IsConstant()) { 6314 int32_t index = helpers::Int32ConstantFrom(index_loc); 6315 __ cmp(length_loc.AsRegister<Register>(), ShifterOperand(index)); 6316 } else { 6317 __ cmp(length_loc.AsRegister<Register>(), ShifterOperand(index_loc.AsRegister<Register>())); 6318 } 6319 codegen_->AddSlowPath(slow_path); 6320 __ b(slow_path->GetEntryLabel(), LS); 6321 } 6322} 6323 6324void CodeGeneratorARM::MarkGCCard(Register temp, 6325 Register card, 6326 Register object, 6327 Register value, 6328 bool can_be_null) { 6329 Label is_null; 6330 if (can_be_null) { 6331 __ CompareAndBranchIfZero(value, &is_null); 6332 } 6333 __ LoadFromOffset(kLoadWord, card, TR, Thread::CardTableOffset<kArmPointerSize>().Int32Value()); 6334 __ Lsr(temp, object, gc::accounting::CardTable::kCardShift); 6335 __ strb(card, Address(card, temp)); 6336 if (can_be_null) { 6337 __ Bind(&is_null); 6338 } 6339} 6340 6341void LocationsBuilderARM::VisitParallelMove(HParallelMove* instruction ATTRIBUTE_UNUSED) { 6342 LOG(FATAL) << "Unreachable"; 6343} 6344 6345void InstructionCodeGeneratorARM::VisitParallelMove(HParallelMove* instruction) { 6346 codegen_->GetMoveResolver()->EmitNativeCode(instruction); 6347} 6348 6349void LocationsBuilderARM::VisitSuspendCheck(HSuspendCheck* instruction) { 6350 LocationSummary* locations = 6351 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCallOnSlowPath); 6352 locations->SetCustomSlowPathCallerSaves(RegisterSet::Empty()); // No caller-save registers. 6353} 6354 6355void InstructionCodeGeneratorARM::VisitSuspendCheck(HSuspendCheck* instruction) { 6356 HBasicBlock* block = instruction->GetBlock(); 6357 if (block->GetLoopInformation() != nullptr) { 6358 DCHECK(block->GetLoopInformation()->GetSuspendCheck() == instruction); 6359 // The back edge will generate the suspend check. 6360 return; 6361 } 6362 if (block->IsEntryBlock() && instruction->GetNext()->IsGoto()) { 6363 // The goto will generate the suspend check. 6364 return; 6365 } 6366 GenerateSuspendCheck(instruction, nullptr); 6367} 6368 6369void InstructionCodeGeneratorARM::GenerateSuspendCheck(HSuspendCheck* instruction, 6370 HBasicBlock* successor) { 6371 SuspendCheckSlowPathARM* slow_path = 6372 down_cast<SuspendCheckSlowPathARM*>(instruction->GetSlowPath()); 6373 if (slow_path == nullptr) { 6374 slow_path = new (GetGraph()->GetArena()) SuspendCheckSlowPathARM(instruction, successor); 6375 instruction->SetSlowPath(slow_path); 6376 codegen_->AddSlowPath(slow_path); 6377 if (successor != nullptr) { 6378 DCHECK(successor->IsLoopHeader()); 6379 codegen_->ClearSpillSlotsFromLoopPhisInStackMap(instruction); 6380 } 6381 } else { 6382 DCHECK_EQ(slow_path->GetSuccessor(), successor); 6383 } 6384 6385 __ LoadFromOffset( 6386 kLoadUnsignedHalfword, IP, TR, Thread::ThreadFlagsOffset<kArmPointerSize>().Int32Value()); 6387 if (successor == nullptr) { 6388 __ CompareAndBranchIfNonZero(IP, slow_path->GetEntryLabel()); 6389 __ Bind(slow_path->GetReturnLabel()); 6390 } else { 6391 __ CompareAndBranchIfZero(IP, codegen_->GetLabelOf(successor)); 6392 __ b(slow_path->GetEntryLabel()); 6393 } 6394} 6395 6396ArmAssembler* ParallelMoveResolverARM::GetAssembler() const { 6397 return codegen_->GetAssembler(); 6398} 6399 6400void ParallelMoveResolverARM::EmitMove(size_t index) { 6401 MoveOperands* move = moves_[index]; 6402 Location source = move->GetSource(); 6403 Location destination = move->GetDestination(); 6404 6405 if (source.IsRegister()) { 6406 if (destination.IsRegister()) { 6407 __ Mov(destination.AsRegister<Register>(), source.AsRegister<Register>()); 6408 } else if (destination.IsFpuRegister()) { 6409 __ vmovsr(destination.AsFpuRegister<SRegister>(), source.AsRegister<Register>()); 6410 } else { 6411 DCHECK(destination.IsStackSlot()); 6412 __ StoreToOffset(kStoreWord, source.AsRegister<Register>(), 6413 SP, destination.GetStackIndex()); 6414 } 6415 } else if (source.IsStackSlot()) { 6416 if (destination.IsRegister()) { 6417 __ LoadFromOffset(kLoadWord, destination.AsRegister<Register>(), 6418 SP, source.GetStackIndex()); 6419 } else if (destination.IsFpuRegister()) { 6420 __ LoadSFromOffset(destination.AsFpuRegister<SRegister>(), SP, source.GetStackIndex()); 6421 } else { 6422 DCHECK(destination.IsStackSlot()); 6423 __ LoadFromOffset(kLoadWord, IP, SP, source.GetStackIndex()); 6424 __ StoreToOffset(kStoreWord, IP, SP, destination.GetStackIndex()); 6425 } 6426 } else if (source.IsFpuRegister()) { 6427 if (destination.IsRegister()) { 6428 __ vmovrs(destination.AsRegister<Register>(), source.AsFpuRegister<SRegister>()); 6429 } else if (destination.IsFpuRegister()) { 6430 __ vmovs(destination.AsFpuRegister<SRegister>(), source.AsFpuRegister<SRegister>()); 6431 } else { 6432 DCHECK(destination.IsStackSlot()); 6433 __ StoreSToOffset(source.AsFpuRegister<SRegister>(), SP, destination.GetStackIndex()); 6434 } 6435 } else if (source.IsDoubleStackSlot()) { 6436 if (destination.IsDoubleStackSlot()) { 6437 __ LoadDFromOffset(DTMP, SP, source.GetStackIndex()); 6438 __ StoreDToOffset(DTMP, SP, destination.GetStackIndex()); 6439 } else if (destination.IsRegisterPair()) { 6440 DCHECK(ExpectedPairLayout(destination)); 6441 __ LoadFromOffset( 6442 kLoadWordPair, destination.AsRegisterPairLow<Register>(), SP, source.GetStackIndex()); 6443 } else { 6444 DCHECK(destination.IsFpuRegisterPair()) << destination; 6445 __ LoadDFromOffset(FromLowSToD(destination.AsFpuRegisterPairLow<SRegister>()), 6446 SP, 6447 source.GetStackIndex()); 6448 } 6449 } else if (source.IsRegisterPair()) { 6450 if (destination.IsRegisterPair()) { 6451 __ Mov(destination.AsRegisterPairLow<Register>(), source.AsRegisterPairLow<Register>()); 6452 __ Mov(destination.AsRegisterPairHigh<Register>(), source.AsRegisterPairHigh<Register>()); 6453 } else if (destination.IsFpuRegisterPair()) { 6454 __ vmovdrr(FromLowSToD(destination.AsFpuRegisterPairLow<SRegister>()), 6455 source.AsRegisterPairLow<Register>(), 6456 source.AsRegisterPairHigh<Register>()); 6457 } else { 6458 DCHECK(destination.IsDoubleStackSlot()) << destination; 6459 DCHECK(ExpectedPairLayout(source)); 6460 __ StoreToOffset( 6461 kStoreWordPair, source.AsRegisterPairLow<Register>(), SP, destination.GetStackIndex()); 6462 } 6463 } else if (source.IsFpuRegisterPair()) { 6464 if (destination.IsRegisterPair()) { 6465 __ vmovrrd(destination.AsRegisterPairLow<Register>(), 6466 destination.AsRegisterPairHigh<Register>(), 6467 FromLowSToD(source.AsFpuRegisterPairLow<SRegister>())); 6468 } else if (destination.IsFpuRegisterPair()) { 6469 __ vmovd(FromLowSToD(destination.AsFpuRegisterPairLow<SRegister>()), 6470 FromLowSToD(source.AsFpuRegisterPairLow<SRegister>())); 6471 } else { 6472 DCHECK(destination.IsDoubleStackSlot()) << destination; 6473 __ StoreDToOffset(FromLowSToD(source.AsFpuRegisterPairLow<SRegister>()), 6474 SP, 6475 destination.GetStackIndex()); 6476 } 6477 } else { 6478 DCHECK(source.IsConstant()) << source; 6479 HConstant* constant = source.GetConstant(); 6480 if (constant->IsIntConstant() || constant->IsNullConstant()) { 6481 int32_t value = CodeGenerator::GetInt32ValueOf(constant); 6482 if (destination.IsRegister()) { 6483 __ LoadImmediate(destination.AsRegister<Register>(), value); 6484 } else { 6485 DCHECK(destination.IsStackSlot()); 6486 __ LoadImmediate(IP, value); 6487 __ StoreToOffset(kStoreWord, IP, SP, destination.GetStackIndex()); 6488 } 6489 } else if (constant->IsLongConstant()) { 6490 int64_t value = constant->AsLongConstant()->GetValue(); 6491 if (destination.IsRegisterPair()) { 6492 __ LoadImmediate(destination.AsRegisterPairLow<Register>(), Low32Bits(value)); 6493 __ LoadImmediate(destination.AsRegisterPairHigh<Register>(), High32Bits(value)); 6494 } else { 6495 DCHECK(destination.IsDoubleStackSlot()) << destination; 6496 __ LoadImmediate(IP, Low32Bits(value)); 6497 __ StoreToOffset(kStoreWord, IP, SP, destination.GetStackIndex()); 6498 __ LoadImmediate(IP, High32Bits(value)); 6499 __ StoreToOffset(kStoreWord, IP, SP, destination.GetHighStackIndex(kArmWordSize)); 6500 } 6501 } else if (constant->IsDoubleConstant()) { 6502 double value = constant->AsDoubleConstant()->GetValue(); 6503 if (destination.IsFpuRegisterPair()) { 6504 __ LoadDImmediate(FromLowSToD(destination.AsFpuRegisterPairLow<SRegister>()), value); 6505 } else { 6506 DCHECK(destination.IsDoubleStackSlot()) << destination; 6507 uint64_t int_value = bit_cast<uint64_t, double>(value); 6508 __ LoadImmediate(IP, Low32Bits(int_value)); 6509 __ StoreToOffset(kStoreWord, IP, SP, destination.GetStackIndex()); 6510 __ LoadImmediate(IP, High32Bits(int_value)); 6511 __ StoreToOffset(kStoreWord, IP, SP, destination.GetHighStackIndex(kArmWordSize)); 6512 } 6513 } else { 6514 DCHECK(constant->IsFloatConstant()) << constant->DebugName(); 6515 float value = constant->AsFloatConstant()->GetValue(); 6516 if (destination.IsFpuRegister()) { 6517 __ LoadSImmediate(destination.AsFpuRegister<SRegister>(), value); 6518 } else { 6519 DCHECK(destination.IsStackSlot()); 6520 __ LoadImmediate(IP, bit_cast<int32_t, float>(value)); 6521 __ StoreToOffset(kStoreWord, IP, SP, destination.GetStackIndex()); 6522 } 6523 } 6524 } 6525} 6526 6527void ParallelMoveResolverARM::Exchange(Register reg, int mem) { 6528 __ Mov(IP, reg); 6529 __ LoadFromOffset(kLoadWord, reg, SP, mem); 6530 __ StoreToOffset(kStoreWord, IP, SP, mem); 6531} 6532 6533void ParallelMoveResolverARM::Exchange(int mem1, int mem2) { 6534 ScratchRegisterScope ensure_scratch(this, IP, R0, codegen_->GetNumberOfCoreRegisters()); 6535 int stack_offset = ensure_scratch.IsSpilled() ? kArmWordSize : 0; 6536 __ LoadFromOffset(kLoadWord, static_cast<Register>(ensure_scratch.GetRegister()), 6537 SP, mem1 + stack_offset); 6538 __ LoadFromOffset(kLoadWord, IP, SP, mem2 + stack_offset); 6539 __ StoreToOffset(kStoreWord, static_cast<Register>(ensure_scratch.GetRegister()), 6540 SP, mem2 + stack_offset); 6541 __ StoreToOffset(kStoreWord, IP, SP, mem1 + stack_offset); 6542} 6543 6544void ParallelMoveResolverARM::EmitSwap(size_t index) { 6545 MoveOperands* move = moves_[index]; 6546 Location source = move->GetSource(); 6547 Location destination = move->GetDestination(); 6548 6549 if (source.IsRegister() && destination.IsRegister()) { 6550 DCHECK_NE(source.AsRegister<Register>(), IP); 6551 DCHECK_NE(destination.AsRegister<Register>(), IP); 6552 __ Mov(IP, source.AsRegister<Register>()); 6553 __ Mov(source.AsRegister<Register>(), destination.AsRegister<Register>()); 6554 __ Mov(destination.AsRegister<Register>(), IP); 6555 } else if (source.IsRegister() && destination.IsStackSlot()) { 6556 Exchange(source.AsRegister<Register>(), destination.GetStackIndex()); 6557 } else if (source.IsStackSlot() && destination.IsRegister()) { 6558 Exchange(destination.AsRegister<Register>(), source.GetStackIndex()); 6559 } else if (source.IsStackSlot() && destination.IsStackSlot()) { 6560 Exchange(source.GetStackIndex(), destination.GetStackIndex()); 6561 } else if (source.IsFpuRegister() && destination.IsFpuRegister()) { 6562 __ vmovrs(IP, source.AsFpuRegister<SRegister>()); 6563 __ vmovs(source.AsFpuRegister<SRegister>(), destination.AsFpuRegister<SRegister>()); 6564 __ vmovsr(destination.AsFpuRegister<SRegister>(), IP); 6565 } else if (source.IsRegisterPair() && destination.IsRegisterPair()) { 6566 __ vmovdrr(DTMP, source.AsRegisterPairLow<Register>(), source.AsRegisterPairHigh<Register>()); 6567 __ Mov(source.AsRegisterPairLow<Register>(), destination.AsRegisterPairLow<Register>()); 6568 __ Mov(source.AsRegisterPairHigh<Register>(), destination.AsRegisterPairHigh<Register>()); 6569 __ vmovrrd(destination.AsRegisterPairLow<Register>(), 6570 destination.AsRegisterPairHigh<Register>(), 6571 DTMP); 6572 } else if (source.IsRegisterPair() || destination.IsRegisterPair()) { 6573 Register low_reg = source.IsRegisterPair() 6574 ? source.AsRegisterPairLow<Register>() 6575 : destination.AsRegisterPairLow<Register>(); 6576 int mem = source.IsRegisterPair() 6577 ? destination.GetStackIndex() 6578 : source.GetStackIndex(); 6579 DCHECK(ExpectedPairLayout(source.IsRegisterPair() ? source : destination)); 6580 __ vmovdrr(DTMP, low_reg, static_cast<Register>(low_reg + 1)); 6581 __ LoadFromOffset(kLoadWordPair, low_reg, SP, mem); 6582 __ StoreDToOffset(DTMP, SP, mem); 6583 } else if (source.IsFpuRegisterPair() && destination.IsFpuRegisterPair()) { 6584 DRegister first = FromLowSToD(source.AsFpuRegisterPairLow<SRegister>()); 6585 DRegister second = FromLowSToD(destination.AsFpuRegisterPairLow<SRegister>()); 6586 __ vmovd(DTMP, first); 6587 __ vmovd(first, second); 6588 __ vmovd(second, DTMP); 6589 } else if (source.IsFpuRegisterPair() || destination.IsFpuRegisterPair()) { 6590 DRegister reg = source.IsFpuRegisterPair() 6591 ? FromLowSToD(source.AsFpuRegisterPairLow<SRegister>()) 6592 : FromLowSToD(destination.AsFpuRegisterPairLow<SRegister>()); 6593 int mem = source.IsFpuRegisterPair() 6594 ? destination.GetStackIndex() 6595 : source.GetStackIndex(); 6596 __ vmovd(DTMP, reg); 6597 __ LoadDFromOffset(reg, SP, mem); 6598 __ StoreDToOffset(DTMP, SP, mem); 6599 } else if (source.IsFpuRegister() || destination.IsFpuRegister()) { 6600 SRegister reg = source.IsFpuRegister() ? source.AsFpuRegister<SRegister>() 6601 : destination.AsFpuRegister<SRegister>(); 6602 int mem = source.IsFpuRegister() 6603 ? destination.GetStackIndex() 6604 : source.GetStackIndex(); 6605 6606 __ vmovrs(IP, reg); 6607 __ LoadSFromOffset(reg, SP, mem); 6608 __ StoreToOffset(kStoreWord, IP, SP, mem); 6609 } else if (source.IsDoubleStackSlot() && destination.IsDoubleStackSlot()) { 6610 Exchange(source.GetStackIndex(), destination.GetStackIndex()); 6611 Exchange(source.GetHighStackIndex(kArmWordSize), destination.GetHighStackIndex(kArmWordSize)); 6612 } else { 6613 LOG(FATAL) << "Unimplemented" << source << " <-> " << destination; 6614 } 6615} 6616 6617void ParallelMoveResolverARM::SpillScratch(int reg) { 6618 __ Push(static_cast<Register>(reg)); 6619} 6620 6621void ParallelMoveResolverARM::RestoreScratch(int reg) { 6622 __ Pop(static_cast<Register>(reg)); 6623} 6624 6625HLoadClass::LoadKind CodeGeneratorARM::GetSupportedLoadClassKind( 6626 HLoadClass::LoadKind desired_class_load_kind) { 6627 switch (desired_class_load_kind) { 6628 case HLoadClass::LoadKind::kInvalid: 6629 LOG(FATAL) << "UNREACHABLE"; 6630 UNREACHABLE(); 6631 case HLoadClass::LoadKind::kReferrersClass: 6632 break; 6633 case HLoadClass::LoadKind::kBootImageLinkTimeAddress: 6634 DCHECK(!GetCompilerOptions().GetCompilePic()); 6635 break; 6636 case HLoadClass::LoadKind::kBootImageLinkTimePcRelative: 6637 DCHECK(GetCompilerOptions().GetCompilePic()); 6638 break; 6639 case HLoadClass::LoadKind::kBootImageAddress: 6640 break; 6641 case HLoadClass::LoadKind::kBssEntry: 6642 DCHECK(!Runtime::Current()->UseJitCompilation()); 6643 break; 6644 case HLoadClass::LoadKind::kJitTableAddress: 6645 DCHECK(Runtime::Current()->UseJitCompilation()); 6646 break; 6647 case HLoadClass::LoadKind::kDexCacheViaMethod: 6648 break; 6649 } 6650 return desired_class_load_kind; 6651} 6652 6653void LocationsBuilderARM::VisitLoadClass(HLoadClass* cls) { 6654 HLoadClass::LoadKind load_kind = cls->GetLoadKind(); 6655 if (load_kind == HLoadClass::LoadKind::kDexCacheViaMethod) { 6656 InvokeRuntimeCallingConvention calling_convention; 6657 CodeGenerator::CreateLoadClassRuntimeCallLocationSummary( 6658 cls, 6659 Location::RegisterLocation(calling_convention.GetRegisterAt(0)), 6660 Location::RegisterLocation(R0)); 6661 DCHECK_EQ(calling_convention.GetRegisterAt(0), R0); 6662 return; 6663 } 6664 DCHECK(!cls->NeedsAccessCheck()); 6665 6666 const bool requires_read_barrier = kEmitCompilerReadBarrier && !cls->IsInBootImage(); 6667 LocationSummary::CallKind call_kind = (cls->NeedsEnvironment() || requires_read_barrier) 6668 ? LocationSummary::kCallOnSlowPath 6669 : LocationSummary::kNoCall; 6670 LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(cls, call_kind); 6671 if (kUseBakerReadBarrier && requires_read_barrier && !cls->NeedsEnvironment()) { 6672 locations->SetCustomSlowPathCallerSaves(RegisterSet::Empty()); // No caller-save registers. 6673 } 6674 6675 if (load_kind == HLoadClass::LoadKind::kReferrersClass) { 6676 locations->SetInAt(0, Location::RequiresRegister()); 6677 } 6678 locations->SetOut(Location::RequiresRegister()); 6679 if (load_kind == HLoadClass::LoadKind::kBssEntry) { 6680 if (!kUseReadBarrier || kUseBakerReadBarrier) { 6681 // Rely on the type resolution or initialization and marking to save everything we need. 6682 // Note that IP may be clobbered by saving/restoring the live register (only one thanks 6683 // to the custom calling convention) or by marking, so we request a different temp. 6684 locations->AddTemp(Location::RequiresRegister()); 6685 RegisterSet caller_saves = RegisterSet::Empty(); 6686 InvokeRuntimeCallingConvention calling_convention; 6687 caller_saves.Add(Location::RegisterLocation(calling_convention.GetRegisterAt(0))); 6688 // TODO: Add GetReturnLocation() to the calling convention so that we can DCHECK() 6689 // that the the kPrimNot result register is the same as the first argument register. 6690 locations->SetCustomSlowPathCallerSaves(caller_saves); 6691 } else { 6692 // For non-Baker read barrier we have a temp-clobbering call. 6693 } 6694 } 6695} 6696 6697// NO_THREAD_SAFETY_ANALYSIS as we manipulate handles whose internal object we know does not 6698// move. 6699void InstructionCodeGeneratorARM::VisitLoadClass(HLoadClass* cls) NO_THREAD_SAFETY_ANALYSIS { 6700 HLoadClass::LoadKind load_kind = cls->GetLoadKind(); 6701 if (load_kind == HLoadClass::LoadKind::kDexCacheViaMethod) { 6702 codegen_->GenerateLoadClassRuntimeCall(cls); 6703 return; 6704 } 6705 DCHECK(!cls->NeedsAccessCheck()); 6706 6707 LocationSummary* locations = cls->GetLocations(); 6708 Location out_loc = locations->Out(); 6709 Register out = out_loc.AsRegister<Register>(); 6710 6711 const ReadBarrierOption read_barrier_option = cls->IsInBootImage() 6712 ? kWithoutReadBarrier 6713 : kCompilerReadBarrierOption; 6714 bool generate_null_check = false; 6715 switch (load_kind) { 6716 case HLoadClass::LoadKind::kReferrersClass: { 6717 DCHECK(!cls->CanCallRuntime()); 6718 DCHECK(!cls->MustGenerateClinitCheck()); 6719 // /* GcRoot<mirror::Class> */ out = current_method->declaring_class_ 6720 Register current_method = locations->InAt(0).AsRegister<Register>(); 6721 GenerateGcRootFieldLoad(cls, 6722 out_loc, 6723 current_method, 6724 ArtMethod::DeclaringClassOffset().Int32Value(), 6725 read_barrier_option); 6726 break; 6727 } 6728 case HLoadClass::LoadKind::kBootImageLinkTimeAddress: { 6729 DCHECK(codegen_->GetCompilerOptions().IsBootImage()); 6730 DCHECK_EQ(read_barrier_option, kWithoutReadBarrier); 6731 __ LoadLiteral(out, codegen_->DeduplicateBootImageTypeLiteral(cls->GetDexFile(), 6732 cls->GetTypeIndex())); 6733 break; 6734 } 6735 case HLoadClass::LoadKind::kBootImageLinkTimePcRelative: { 6736 DCHECK(codegen_->GetCompilerOptions().IsBootImage()); 6737 DCHECK_EQ(read_barrier_option, kWithoutReadBarrier); 6738 CodeGeneratorARM::PcRelativePatchInfo* labels = 6739 codegen_->NewPcRelativeTypePatch(cls->GetDexFile(), cls->GetTypeIndex()); 6740 __ BindTrackedLabel(&labels->movw_label); 6741 __ movw(out, /* placeholder */ 0u); 6742 __ BindTrackedLabel(&labels->movt_label); 6743 __ movt(out, /* placeholder */ 0u); 6744 __ BindTrackedLabel(&labels->add_pc_label); 6745 __ add(out, out, ShifterOperand(PC)); 6746 break; 6747 } 6748 case HLoadClass::LoadKind::kBootImageAddress: { 6749 DCHECK_EQ(read_barrier_option, kWithoutReadBarrier); 6750 uint32_t address = dchecked_integral_cast<uint32_t>( 6751 reinterpret_cast<uintptr_t>(cls->GetClass().Get())); 6752 DCHECK_NE(address, 0u); 6753 __ LoadLiteral(out, codegen_->DeduplicateBootImageAddressLiteral(address)); 6754 break; 6755 } 6756 case HLoadClass::LoadKind::kBssEntry: { 6757 Register temp = (!kUseReadBarrier || kUseBakerReadBarrier) 6758 ? locations->GetTemp(0).AsRegister<Register>() 6759 : out; 6760 CodeGeneratorARM::PcRelativePatchInfo* labels = 6761 codegen_->NewTypeBssEntryPatch(cls->GetDexFile(), cls->GetTypeIndex()); 6762 __ BindTrackedLabel(&labels->movw_label); 6763 __ movw(temp, /* placeholder */ 0u); 6764 __ BindTrackedLabel(&labels->movt_label); 6765 __ movt(temp, /* placeholder */ 0u); 6766 __ BindTrackedLabel(&labels->add_pc_label); 6767 __ add(temp, temp, ShifterOperand(PC)); 6768 GenerateGcRootFieldLoad(cls, out_loc, temp, /* offset */ 0, read_barrier_option); 6769 generate_null_check = true; 6770 break; 6771 } 6772 case HLoadClass::LoadKind::kJitTableAddress: { 6773 __ LoadLiteral(out, codegen_->DeduplicateJitClassLiteral(cls->GetDexFile(), 6774 cls->GetTypeIndex(), 6775 cls->GetClass())); 6776 // /* GcRoot<mirror::Class> */ out = *out 6777 GenerateGcRootFieldLoad(cls, out_loc, out, /* offset */ 0, read_barrier_option); 6778 break; 6779 } 6780 case HLoadClass::LoadKind::kDexCacheViaMethod: 6781 case HLoadClass::LoadKind::kInvalid: 6782 LOG(FATAL) << "UNREACHABLE"; 6783 UNREACHABLE(); 6784 } 6785 6786 if (generate_null_check || cls->MustGenerateClinitCheck()) { 6787 DCHECK(cls->CanCallRuntime()); 6788 SlowPathCodeARM* slow_path = new (GetGraph()->GetArena()) LoadClassSlowPathARM( 6789 cls, cls, cls->GetDexPc(), cls->MustGenerateClinitCheck()); 6790 codegen_->AddSlowPath(slow_path); 6791 if (generate_null_check) { 6792 __ CompareAndBranchIfZero(out, slow_path->GetEntryLabel()); 6793 } 6794 if (cls->MustGenerateClinitCheck()) { 6795 GenerateClassInitializationCheck(slow_path, out); 6796 } else { 6797 __ Bind(slow_path->GetExitLabel()); 6798 } 6799 } 6800} 6801 6802void LocationsBuilderARM::VisitClinitCheck(HClinitCheck* check) { 6803 LocationSummary* locations = 6804 new (GetGraph()->GetArena()) LocationSummary(check, LocationSummary::kCallOnSlowPath); 6805 locations->SetInAt(0, Location::RequiresRegister()); 6806 if (check->HasUses()) { 6807 locations->SetOut(Location::SameAsFirstInput()); 6808 } 6809} 6810 6811void InstructionCodeGeneratorARM::VisitClinitCheck(HClinitCheck* check) { 6812 // We assume the class is not null. 6813 SlowPathCodeARM* slow_path = new (GetGraph()->GetArena()) LoadClassSlowPathARM( 6814 check->GetLoadClass(), check, check->GetDexPc(), true); 6815 codegen_->AddSlowPath(slow_path); 6816 GenerateClassInitializationCheck(slow_path, 6817 check->GetLocations()->InAt(0).AsRegister<Register>()); 6818} 6819 6820void InstructionCodeGeneratorARM::GenerateClassInitializationCheck( 6821 SlowPathCodeARM* slow_path, Register class_reg) { 6822 __ LoadFromOffset(kLoadWord, IP, class_reg, mirror::Class::StatusOffset().Int32Value()); 6823 __ cmp(IP, ShifterOperand(mirror::Class::kStatusInitialized)); 6824 __ b(slow_path->GetEntryLabel(), LT); 6825 // Even if the initialized flag is set, we may be in a situation where caches are not synced 6826 // properly. Therefore, we do a memory fence. 6827 __ dmb(ISH); 6828 __ Bind(slow_path->GetExitLabel()); 6829} 6830 6831HLoadString::LoadKind CodeGeneratorARM::GetSupportedLoadStringKind( 6832 HLoadString::LoadKind desired_string_load_kind) { 6833 switch (desired_string_load_kind) { 6834 case HLoadString::LoadKind::kBootImageLinkTimeAddress: 6835 DCHECK(!GetCompilerOptions().GetCompilePic()); 6836 break; 6837 case HLoadString::LoadKind::kBootImageLinkTimePcRelative: 6838 DCHECK(GetCompilerOptions().GetCompilePic()); 6839 break; 6840 case HLoadString::LoadKind::kBootImageAddress: 6841 break; 6842 case HLoadString::LoadKind::kBssEntry: 6843 DCHECK(!Runtime::Current()->UseJitCompilation()); 6844 break; 6845 case HLoadString::LoadKind::kJitTableAddress: 6846 DCHECK(Runtime::Current()->UseJitCompilation()); 6847 break; 6848 case HLoadString::LoadKind::kDexCacheViaMethod: 6849 break; 6850 } 6851 return desired_string_load_kind; 6852} 6853 6854void LocationsBuilderARM::VisitLoadString(HLoadString* load) { 6855 LocationSummary::CallKind call_kind = CodeGenerator::GetLoadStringCallKind(load); 6856 LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(load, call_kind); 6857 HLoadString::LoadKind load_kind = load->GetLoadKind(); 6858 if (load_kind == HLoadString::LoadKind::kDexCacheViaMethod) { 6859 locations->SetOut(Location::RegisterLocation(R0)); 6860 } else { 6861 locations->SetOut(Location::RequiresRegister()); 6862 if (load_kind == HLoadString::LoadKind::kBssEntry) { 6863 if (!kUseReadBarrier || kUseBakerReadBarrier) { 6864 // Rely on the pResolveString and marking to save everything we need, including temps. 6865 // Note that IP may be clobbered by saving/restoring the live register (only one thanks 6866 // to the custom calling convention) or by marking, so we request a different temp. 6867 locations->AddTemp(Location::RequiresRegister()); 6868 RegisterSet caller_saves = RegisterSet::Empty(); 6869 InvokeRuntimeCallingConvention calling_convention; 6870 caller_saves.Add(Location::RegisterLocation(calling_convention.GetRegisterAt(0))); 6871 // TODO: Add GetReturnLocation() to the calling convention so that we can DCHECK() 6872 // that the the kPrimNot result register is the same as the first argument register. 6873 locations->SetCustomSlowPathCallerSaves(caller_saves); 6874 } else { 6875 // For non-Baker read barrier we have a temp-clobbering call. 6876 } 6877 } 6878 } 6879} 6880 6881// NO_THREAD_SAFETY_ANALYSIS as we manipulate handles whose internal object we know does not 6882// move. 6883void InstructionCodeGeneratorARM::VisitLoadString(HLoadString* load) NO_THREAD_SAFETY_ANALYSIS { 6884 LocationSummary* locations = load->GetLocations(); 6885 Location out_loc = locations->Out(); 6886 Register out = out_loc.AsRegister<Register>(); 6887 HLoadString::LoadKind load_kind = load->GetLoadKind(); 6888 6889 switch (load_kind) { 6890 case HLoadString::LoadKind::kBootImageLinkTimeAddress: { 6891 DCHECK(codegen_->GetCompilerOptions().IsBootImage()); 6892 __ LoadLiteral(out, codegen_->DeduplicateBootImageStringLiteral(load->GetDexFile(), 6893 load->GetStringIndex())); 6894 return; // No dex cache slow path. 6895 } 6896 case HLoadString::LoadKind::kBootImageLinkTimePcRelative: { 6897 DCHECK(codegen_->GetCompilerOptions().IsBootImage()); 6898 CodeGeneratorARM::PcRelativePatchInfo* labels = 6899 codegen_->NewPcRelativeStringPatch(load->GetDexFile(), load->GetStringIndex()); 6900 __ BindTrackedLabel(&labels->movw_label); 6901 __ movw(out, /* placeholder */ 0u); 6902 __ BindTrackedLabel(&labels->movt_label); 6903 __ movt(out, /* placeholder */ 0u); 6904 __ BindTrackedLabel(&labels->add_pc_label); 6905 __ add(out, out, ShifterOperand(PC)); 6906 return; // No dex cache slow path. 6907 } 6908 case HLoadString::LoadKind::kBootImageAddress: { 6909 uint32_t address = dchecked_integral_cast<uint32_t>( 6910 reinterpret_cast<uintptr_t>(load->GetString().Get())); 6911 DCHECK_NE(address, 0u); 6912 __ LoadLiteral(out, codegen_->DeduplicateBootImageAddressLiteral(address)); 6913 return; // No dex cache slow path. 6914 } 6915 case HLoadString::LoadKind::kBssEntry: { 6916 DCHECK(!codegen_->GetCompilerOptions().IsBootImage()); 6917 Register temp = (!kUseReadBarrier || kUseBakerReadBarrier) 6918 ? locations->GetTemp(0).AsRegister<Register>() 6919 : out; 6920 CodeGeneratorARM::PcRelativePatchInfo* labels = 6921 codegen_->NewPcRelativeStringPatch(load->GetDexFile(), load->GetStringIndex()); 6922 __ BindTrackedLabel(&labels->movw_label); 6923 __ movw(temp, /* placeholder */ 0u); 6924 __ BindTrackedLabel(&labels->movt_label); 6925 __ movt(temp, /* placeholder */ 0u); 6926 __ BindTrackedLabel(&labels->add_pc_label); 6927 __ add(temp, temp, ShifterOperand(PC)); 6928 GenerateGcRootFieldLoad(load, out_loc, temp, /* offset */ 0, kCompilerReadBarrierOption); 6929 SlowPathCode* slow_path = new (GetGraph()->GetArena()) LoadStringSlowPathARM(load); 6930 codegen_->AddSlowPath(slow_path); 6931 __ CompareAndBranchIfZero(out, slow_path->GetEntryLabel()); 6932 __ Bind(slow_path->GetExitLabel()); 6933 return; 6934 } 6935 case HLoadString::LoadKind::kJitTableAddress: { 6936 __ LoadLiteral(out, codegen_->DeduplicateJitStringLiteral(load->GetDexFile(), 6937 load->GetStringIndex(), 6938 load->GetString())); 6939 // /* GcRoot<mirror::String> */ out = *out 6940 GenerateGcRootFieldLoad(load, out_loc, out, /* offset */ 0, kCompilerReadBarrierOption); 6941 return; 6942 } 6943 default: 6944 break; 6945 } 6946 6947 // TODO: Consider re-adding the compiler code to do string dex cache lookup again. 6948 DCHECK(load_kind == HLoadString::LoadKind::kDexCacheViaMethod); 6949 InvokeRuntimeCallingConvention calling_convention; 6950 DCHECK_EQ(calling_convention.GetRegisterAt(0), out); 6951 __ LoadImmediate(calling_convention.GetRegisterAt(0), load->GetStringIndex().index_); 6952 codegen_->InvokeRuntime(kQuickResolveString, load, load->GetDexPc()); 6953 CheckEntrypointTypes<kQuickResolveString, void*, uint32_t>(); 6954} 6955 6956static int32_t GetExceptionTlsOffset() { 6957 return Thread::ExceptionOffset<kArmPointerSize>().Int32Value(); 6958} 6959 6960void LocationsBuilderARM::VisitLoadException(HLoadException* load) { 6961 LocationSummary* locations = 6962 new (GetGraph()->GetArena()) LocationSummary(load, LocationSummary::kNoCall); 6963 locations->SetOut(Location::RequiresRegister()); 6964} 6965 6966void InstructionCodeGeneratorARM::VisitLoadException(HLoadException* load) { 6967 Register out = load->GetLocations()->Out().AsRegister<Register>(); 6968 __ LoadFromOffset(kLoadWord, out, TR, GetExceptionTlsOffset()); 6969} 6970 6971void LocationsBuilderARM::VisitClearException(HClearException* clear) { 6972 new (GetGraph()->GetArena()) LocationSummary(clear, LocationSummary::kNoCall); 6973} 6974 6975void InstructionCodeGeneratorARM::VisitClearException(HClearException* clear ATTRIBUTE_UNUSED) { 6976 __ LoadImmediate(IP, 0); 6977 __ StoreToOffset(kStoreWord, IP, TR, GetExceptionTlsOffset()); 6978} 6979 6980void LocationsBuilderARM::VisitThrow(HThrow* instruction) { 6981 LocationSummary* locations = 6982 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCallOnMainOnly); 6983 InvokeRuntimeCallingConvention calling_convention; 6984 locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(0))); 6985} 6986 6987void InstructionCodeGeneratorARM::VisitThrow(HThrow* instruction) { 6988 codegen_->InvokeRuntime(kQuickDeliverException, instruction, instruction->GetDexPc()); 6989 CheckEntrypointTypes<kQuickDeliverException, void, mirror::Object*>(); 6990} 6991 6992// Temp is used for read barrier. 6993static size_t NumberOfInstanceOfTemps(TypeCheckKind type_check_kind) { 6994 if (kEmitCompilerReadBarrier && 6995 (kUseBakerReadBarrier || 6996 type_check_kind == TypeCheckKind::kAbstractClassCheck || 6997 type_check_kind == TypeCheckKind::kClassHierarchyCheck || 6998 type_check_kind == TypeCheckKind::kArrayObjectCheck)) { 6999 return 1; 7000 } 7001 return 0; 7002} 7003 7004// Interface case has 3 temps, one for holding the number of interfaces, one for the current 7005// interface pointer, one for loading the current interface. 7006// The other checks have one temp for loading the object's class. 7007static size_t NumberOfCheckCastTemps(TypeCheckKind type_check_kind) { 7008 if (type_check_kind == TypeCheckKind::kInterfaceCheck) { 7009 return 3; 7010 } 7011 return 1 + NumberOfInstanceOfTemps(type_check_kind); 7012} 7013 7014void LocationsBuilderARM::VisitInstanceOf(HInstanceOf* instruction) { 7015 LocationSummary::CallKind call_kind = LocationSummary::kNoCall; 7016 TypeCheckKind type_check_kind = instruction->GetTypeCheckKind(); 7017 bool baker_read_barrier_slow_path = false; 7018 switch (type_check_kind) { 7019 case TypeCheckKind::kExactCheck: 7020 case TypeCheckKind::kAbstractClassCheck: 7021 case TypeCheckKind::kClassHierarchyCheck: 7022 case TypeCheckKind::kArrayObjectCheck: 7023 call_kind = 7024 kEmitCompilerReadBarrier ? LocationSummary::kCallOnSlowPath : LocationSummary::kNoCall; 7025 baker_read_barrier_slow_path = kUseBakerReadBarrier; 7026 break; 7027 case TypeCheckKind::kArrayCheck: 7028 case TypeCheckKind::kUnresolvedCheck: 7029 case TypeCheckKind::kInterfaceCheck: 7030 call_kind = LocationSummary::kCallOnSlowPath; 7031 break; 7032 } 7033 7034 LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction, call_kind); 7035 if (baker_read_barrier_slow_path) { 7036 locations->SetCustomSlowPathCallerSaves(RegisterSet::Empty()); // No caller-save registers. 7037 } 7038 locations->SetInAt(0, Location::RequiresRegister()); 7039 locations->SetInAt(1, Location::RequiresRegister()); 7040 // The "out" register is used as a temporary, so it overlaps with the inputs. 7041 // Note that TypeCheckSlowPathARM uses this register too. 7042 locations->SetOut(Location::RequiresRegister(), Location::kOutputOverlap); 7043 locations->AddRegisterTemps(NumberOfInstanceOfTemps(type_check_kind)); 7044} 7045 7046void InstructionCodeGeneratorARM::VisitInstanceOf(HInstanceOf* instruction) { 7047 TypeCheckKind type_check_kind = instruction->GetTypeCheckKind(); 7048 LocationSummary* locations = instruction->GetLocations(); 7049 Location obj_loc = locations->InAt(0); 7050 Register obj = obj_loc.AsRegister<Register>(); 7051 Register cls = locations->InAt(1).AsRegister<Register>(); 7052 Location out_loc = locations->Out(); 7053 Register out = out_loc.AsRegister<Register>(); 7054 const size_t num_temps = NumberOfInstanceOfTemps(type_check_kind); 7055 DCHECK_LE(num_temps, 1u); 7056 Location maybe_temp_loc = (num_temps >= 1) ? locations->GetTemp(0) : Location::NoLocation(); 7057 uint32_t class_offset = mirror::Object::ClassOffset().Int32Value(); 7058 uint32_t super_offset = mirror::Class::SuperClassOffset().Int32Value(); 7059 uint32_t component_offset = mirror::Class::ComponentTypeOffset().Int32Value(); 7060 uint32_t primitive_offset = mirror::Class::PrimitiveTypeOffset().Int32Value(); 7061 Label done; 7062 Label* const final_label = codegen_->GetFinalLabel(instruction, &done); 7063 SlowPathCodeARM* slow_path = nullptr; 7064 7065 // Return 0 if `obj` is null. 7066 // avoid null check if we know obj is not null. 7067 if (instruction->MustDoNullCheck()) { 7068 DCHECK_NE(out, obj); 7069 __ LoadImmediate(out, 0); 7070 __ CompareAndBranchIfZero(obj, final_label); 7071 } 7072 7073 switch (type_check_kind) { 7074 case TypeCheckKind::kExactCheck: { 7075 // /* HeapReference<Class> */ out = obj->klass_ 7076 GenerateReferenceLoadTwoRegisters(instruction, 7077 out_loc, 7078 obj_loc, 7079 class_offset, 7080 maybe_temp_loc, 7081 kCompilerReadBarrierOption); 7082 // Classes must be equal for the instanceof to succeed. 7083 __ cmp(out, ShifterOperand(cls)); 7084 // We speculatively set the result to false without changing the condition 7085 // flags, which allows us to avoid some branching later. 7086 __ mov(out, ShifterOperand(0), AL, kCcKeep); 7087 7088 // Since IT blocks longer than a 16-bit instruction are deprecated by ARMv8, 7089 // we check that the output is in a low register, so that a 16-bit MOV 7090 // encoding can be used. 7091 if (ArmAssembler::IsLowRegister(out)) { 7092 __ it(EQ); 7093 __ mov(out, ShifterOperand(1), EQ); 7094 } else { 7095 __ b(final_label, NE); 7096 __ LoadImmediate(out, 1); 7097 } 7098 7099 break; 7100 } 7101 7102 case TypeCheckKind::kAbstractClassCheck: { 7103 // /* HeapReference<Class> */ out = obj->klass_ 7104 GenerateReferenceLoadTwoRegisters(instruction, 7105 out_loc, 7106 obj_loc, 7107 class_offset, 7108 maybe_temp_loc, 7109 kCompilerReadBarrierOption); 7110 // If the class is abstract, we eagerly fetch the super class of the 7111 // object to avoid doing a comparison we know will fail. 7112 Label loop; 7113 __ Bind(&loop); 7114 // /* HeapReference<Class> */ out = out->super_class_ 7115 GenerateReferenceLoadOneRegister(instruction, 7116 out_loc, 7117 super_offset, 7118 maybe_temp_loc, 7119 kCompilerReadBarrierOption); 7120 // If `out` is null, we use it for the result, and jump to the final label. 7121 __ CompareAndBranchIfZero(out, final_label); 7122 __ cmp(out, ShifterOperand(cls)); 7123 __ b(&loop, NE); 7124 __ LoadImmediate(out, 1); 7125 break; 7126 } 7127 7128 case TypeCheckKind::kClassHierarchyCheck: { 7129 // /* HeapReference<Class> */ out = obj->klass_ 7130 GenerateReferenceLoadTwoRegisters(instruction, 7131 out_loc, 7132 obj_loc, 7133 class_offset, 7134 maybe_temp_loc, 7135 kCompilerReadBarrierOption); 7136 // Walk over the class hierarchy to find a match. 7137 Label loop, success; 7138 __ Bind(&loop); 7139 __ cmp(out, ShifterOperand(cls)); 7140 __ b(&success, EQ); 7141 // /* HeapReference<Class> */ out = out->super_class_ 7142 GenerateReferenceLoadOneRegister(instruction, 7143 out_loc, 7144 super_offset, 7145 maybe_temp_loc, 7146 kCompilerReadBarrierOption); 7147 // This is essentially a null check, but it sets the condition flags to the 7148 // proper value for the code that follows the loop, i.e. not `EQ`. 7149 __ cmp(out, ShifterOperand(1)); 7150 __ b(&loop, HS); 7151 7152 // Since IT blocks longer than a 16-bit instruction are deprecated by ARMv8, 7153 // we check that the output is in a low register, so that a 16-bit MOV 7154 // encoding can be used. 7155 if (ArmAssembler::IsLowRegister(out)) { 7156 // If `out` is null, we use it for the result, and the condition flags 7157 // have already been set to `NE`, so the IT block that comes afterwards 7158 // (and which handles the successful case) turns into a NOP (instead of 7159 // overwriting `out`). 7160 __ Bind(&success); 7161 // There is only one branch to the `success` label (which is bound to this 7162 // IT block), and it has the same condition, `EQ`, so in that case the MOV 7163 // is executed. 7164 __ it(EQ); 7165 __ mov(out, ShifterOperand(1), EQ); 7166 } else { 7167 // If `out` is null, we use it for the result, and jump to the final label. 7168 __ b(final_label); 7169 __ Bind(&success); 7170 __ LoadImmediate(out, 1); 7171 } 7172 7173 break; 7174 } 7175 7176 case TypeCheckKind::kArrayObjectCheck: { 7177 // /* HeapReference<Class> */ out = obj->klass_ 7178 GenerateReferenceLoadTwoRegisters(instruction, 7179 out_loc, 7180 obj_loc, 7181 class_offset, 7182 maybe_temp_loc, 7183 kCompilerReadBarrierOption); 7184 // Do an exact check. 7185 Label exact_check; 7186 __ cmp(out, ShifterOperand(cls)); 7187 __ b(&exact_check, EQ); 7188 // Otherwise, we need to check that the object's class is a non-primitive array. 7189 // /* HeapReference<Class> */ out = out->component_type_ 7190 GenerateReferenceLoadOneRegister(instruction, 7191 out_loc, 7192 component_offset, 7193 maybe_temp_loc, 7194 kCompilerReadBarrierOption); 7195 // If `out` is null, we use it for the result, and jump to the final label. 7196 __ CompareAndBranchIfZero(out, final_label); 7197 __ LoadFromOffset(kLoadUnsignedHalfword, out, out, primitive_offset); 7198 static_assert(Primitive::kPrimNot == 0, "Expected 0 for kPrimNot"); 7199 __ cmp(out, ShifterOperand(0)); 7200 // We speculatively set the result to false without changing the condition 7201 // flags, which allows us to avoid some branching later. 7202 __ mov(out, ShifterOperand(0), AL, kCcKeep); 7203 7204 // Since IT blocks longer than a 16-bit instruction are deprecated by ARMv8, 7205 // we check that the output is in a low register, so that a 16-bit MOV 7206 // encoding can be used. 7207 if (ArmAssembler::IsLowRegister(out)) { 7208 __ Bind(&exact_check); 7209 __ it(EQ); 7210 __ mov(out, ShifterOperand(1), EQ); 7211 } else { 7212 __ b(final_label, NE); 7213 __ Bind(&exact_check); 7214 __ LoadImmediate(out, 1); 7215 } 7216 7217 break; 7218 } 7219 7220 case TypeCheckKind::kArrayCheck: { 7221 // No read barrier since the slow path will retry upon failure. 7222 // /* HeapReference<Class> */ out = obj->klass_ 7223 GenerateReferenceLoadTwoRegisters(instruction, 7224 out_loc, 7225 obj_loc, 7226 class_offset, 7227 maybe_temp_loc, 7228 kWithoutReadBarrier); 7229 __ cmp(out, ShifterOperand(cls)); 7230 DCHECK(locations->OnlyCallsOnSlowPath()); 7231 slow_path = new (GetGraph()->GetArena()) TypeCheckSlowPathARM(instruction, 7232 /* is_fatal */ false); 7233 codegen_->AddSlowPath(slow_path); 7234 __ b(slow_path->GetEntryLabel(), NE); 7235 __ LoadImmediate(out, 1); 7236 break; 7237 } 7238 7239 case TypeCheckKind::kUnresolvedCheck: 7240 case TypeCheckKind::kInterfaceCheck: { 7241 // Note that we indeed only call on slow path, but we always go 7242 // into the slow path for the unresolved and interface check 7243 // cases. 7244 // 7245 // We cannot directly call the InstanceofNonTrivial runtime 7246 // entry point without resorting to a type checking slow path 7247 // here (i.e. by calling InvokeRuntime directly), as it would 7248 // require to assign fixed registers for the inputs of this 7249 // HInstanceOf instruction (following the runtime calling 7250 // convention), which might be cluttered by the potential first 7251 // read barrier emission at the beginning of this method. 7252 // 7253 // TODO: Introduce a new runtime entry point taking the object 7254 // to test (instead of its class) as argument, and let it deal 7255 // with the read barrier issues. This will let us refactor this 7256 // case of the `switch` code as it was previously (with a direct 7257 // call to the runtime not using a type checking slow path). 7258 // This should also be beneficial for the other cases above. 7259 DCHECK(locations->OnlyCallsOnSlowPath()); 7260 slow_path = new (GetGraph()->GetArena()) TypeCheckSlowPathARM(instruction, 7261 /* is_fatal */ false); 7262 codegen_->AddSlowPath(slow_path); 7263 __ b(slow_path->GetEntryLabel()); 7264 break; 7265 } 7266 } 7267 7268 if (done.IsLinked()) { 7269 __ Bind(&done); 7270 } 7271 7272 if (slow_path != nullptr) { 7273 __ Bind(slow_path->GetExitLabel()); 7274 } 7275} 7276 7277void LocationsBuilderARM::VisitCheckCast(HCheckCast* instruction) { 7278 LocationSummary::CallKind call_kind = LocationSummary::kNoCall; 7279 bool throws_into_catch = instruction->CanThrowIntoCatchBlock(); 7280 7281 TypeCheckKind type_check_kind = instruction->GetTypeCheckKind(); 7282 switch (type_check_kind) { 7283 case TypeCheckKind::kExactCheck: 7284 case TypeCheckKind::kAbstractClassCheck: 7285 case TypeCheckKind::kClassHierarchyCheck: 7286 case TypeCheckKind::kArrayObjectCheck: 7287 call_kind = (throws_into_catch || kEmitCompilerReadBarrier) ? 7288 LocationSummary::kCallOnSlowPath : 7289 LocationSummary::kNoCall; // In fact, call on a fatal (non-returning) slow path. 7290 break; 7291 case TypeCheckKind::kArrayCheck: 7292 case TypeCheckKind::kUnresolvedCheck: 7293 case TypeCheckKind::kInterfaceCheck: 7294 call_kind = LocationSummary::kCallOnSlowPath; 7295 break; 7296 } 7297 7298 LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction, call_kind); 7299 locations->SetInAt(0, Location::RequiresRegister()); 7300 locations->SetInAt(1, Location::RequiresRegister()); 7301 locations->AddRegisterTemps(NumberOfCheckCastTemps(type_check_kind)); 7302} 7303 7304void InstructionCodeGeneratorARM::VisitCheckCast(HCheckCast* instruction) { 7305 TypeCheckKind type_check_kind = instruction->GetTypeCheckKind(); 7306 LocationSummary* locations = instruction->GetLocations(); 7307 Location obj_loc = locations->InAt(0); 7308 Register obj = obj_loc.AsRegister<Register>(); 7309 Register cls = locations->InAt(1).AsRegister<Register>(); 7310 Location temp_loc = locations->GetTemp(0); 7311 Register temp = temp_loc.AsRegister<Register>(); 7312 const size_t num_temps = NumberOfCheckCastTemps(type_check_kind); 7313 DCHECK_LE(num_temps, 3u); 7314 Location maybe_temp2_loc = (num_temps >= 2) ? locations->GetTemp(1) : Location::NoLocation(); 7315 Location maybe_temp3_loc = (num_temps >= 3) ? locations->GetTemp(2) : Location::NoLocation(); 7316 const uint32_t class_offset = mirror::Object::ClassOffset().Int32Value(); 7317 const uint32_t super_offset = mirror::Class::SuperClassOffset().Int32Value(); 7318 const uint32_t component_offset = mirror::Class::ComponentTypeOffset().Int32Value(); 7319 const uint32_t primitive_offset = mirror::Class::PrimitiveTypeOffset().Int32Value(); 7320 const uint32_t iftable_offset = mirror::Class::IfTableOffset().Uint32Value(); 7321 const uint32_t array_length_offset = mirror::Array::LengthOffset().Uint32Value(); 7322 const uint32_t object_array_data_offset = 7323 mirror::Array::DataOffset(kHeapReferenceSize).Uint32Value(); 7324 7325 // Always false for read barriers since we may need to go to the entrypoint for non-fatal cases 7326 // from false negatives. The false negatives may come from avoiding read barriers below. Avoiding 7327 // read barriers is done for performance and code size reasons. 7328 bool is_type_check_slow_path_fatal = false; 7329 if (!kEmitCompilerReadBarrier) { 7330 is_type_check_slow_path_fatal = 7331 (type_check_kind == TypeCheckKind::kExactCheck || 7332 type_check_kind == TypeCheckKind::kAbstractClassCheck || 7333 type_check_kind == TypeCheckKind::kClassHierarchyCheck || 7334 type_check_kind == TypeCheckKind::kArrayObjectCheck) && 7335 !instruction->CanThrowIntoCatchBlock(); 7336 } 7337 SlowPathCodeARM* type_check_slow_path = 7338 new (GetGraph()->GetArena()) TypeCheckSlowPathARM(instruction, 7339 is_type_check_slow_path_fatal); 7340 codegen_->AddSlowPath(type_check_slow_path); 7341 7342 Label done; 7343 Label* final_label = codegen_->GetFinalLabel(instruction, &done); 7344 // Avoid null check if we know obj is not null. 7345 if (instruction->MustDoNullCheck()) { 7346 __ CompareAndBranchIfZero(obj, final_label); 7347 } 7348 7349 switch (type_check_kind) { 7350 case TypeCheckKind::kExactCheck: 7351 case TypeCheckKind::kArrayCheck: { 7352 // /* HeapReference<Class> */ temp = obj->klass_ 7353 GenerateReferenceLoadTwoRegisters(instruction, 7354 temp_loc, 7355 obj_loc, 7356 class_offset, 7357 maybe_temp2_loc, 7358 kWithoutReadBarrier); 7359 7360 __ cmp(temp, ShifterOperand(cls)); 7361 // Jump to slow path for throwing the exception or doing a 7362 // more involved array check. 7363 __ b(type_check_slow_path->GetEntryLabel(), NE); 7364 break; 7365 } 7366 7367 case TypeCheckKind::kAbstractClassCheck: { 7368 // /* HeapReference<Class> */ temp = obj->klass_ 7369 GenerateReferenceLoadTwoRegisters(instruction, 7370 temp_loc, 7371 obj_loc, 7372 class_offset, 7373 maybe_temp2_loc, 7374 kWithoutReadBarrier); 7375 7376 // If the class is abstract, we eagerly fetch the super class of the 7377 // object to avoid doing a comparison we know will fail. 7378 Label loop; 7379 __ Bind(&loop); 7380 // /* HeapReference<Class> */ temp = temp->super_class_ 7381 GenerateReferenceLoadOneRegister(instruction, 7382 temp_loc, 7383 super_offset, 7384 maybe_temp2_loc, 7385 kWithoutReadBarrier); 7386 7387 // If the class reference currently in `temp` is null, jump to the slow path to throw the 7388 // exception. 7389 __ CompareAndBranchIfZero(temp, type_check_slow_path->GetEntryLabel()); 7390 7391 // Otherwise, compare the classes. 7392 __ cmp(temp, ShifterOperand(cls)); 7393 __ b(&loop, NE); 7394 break; 7395 } 7396 7397 case TypeCheckKind::kClassHierarchyCheck: { 7398 // /* HeapReference<Class> */ temp = obj->klass_ 7399 GenerateReferenceLoadTwoRegisters(instruction, 7400 temp_loc, 7401 obj_loc, 7402 class_offset, 7403 maybe_temp2_loc, 7404 kWithoutReadBarrier); 7405 7406 // Walk over the class hierarchy to find a match. 7407 Label loop; 7408 __ Bind(&loop); 7409 __ cmp(temp, ShifterOperand(cls)); 7410 __ b(final_label, EQ); 7411 7412 // /* HeapReference<Class> */ temp = temp->super_class_ 7413 GenerateReferenceLoadOneRegister(instruction, 7414 temp_loc, 7415 super_offset, 7416 maybe_temp2_loc, 7417 kWithoutReadBarrier); 7418 7419 // If the class reference currently in `temp` is null, jump to the slow path to throw the 7420 // exception. 7421 __ CompareAndBranchIfZero(temp, type_check_slow_path->GetEntryLabel()); 7422 // Otherwise, jump to the beginning of the loop. 7423 __ b(&loop); 7424 break; 7425 } 7426 7427 case TypeCheckKind::kArrayObjectCheck: { 7428 // /* HeapReference<Class> */ temp = obj->klass_ 7429 GenerateReferenceLoadTwoRegisters(instruction, 7430 temp_loc, 7431 obj_loc, 7432 class_offset, 7433 maybe_temp2_loc, 7434 kWithoutReadBarrier); 7435 7436 // Do an exact check. 7437 __ cmp(temp, ShifterOperand(cls)); 7438 __ b(final_label, EQ); 7439 7440 // Otherwise, we need to check that the object's class is a non-primitive array. 7441 // /* HeapReference<Class> */ temp = temp->component_type_ 7442 GenerateReferenceLoadOneRegister(instruction, 7443 temp_loc, 7444 component_offset, 7445 maybe_temp2_loc, 7446 kWithoutReadBarrier); 7447 // If the component type is null, jump to the slow path to throw the exception. 7448 __ CompareAndBranchIfZero(temp, type_check_slow_path->GetEntryLabel()); 7449 // Otherwise,the object is indeed an array, jump to label `check_non_primitive_component_type` 7450 // to further check that this component type is not a primitive type. 7451 __ LoadFromOffset(kLoadUnsignedHalfword, temp, temp, primitive_offset); 7452 static_assert(Primitive::kPrimNot == 0, "Expected 0 for art::Primitive::kPrimNot"); 7453 __ CompareAndBranchIfNonZero(temp, type_check_slow_path->GetEntryLabel()); 7454 break; 7455 } 7456 7457 case TypeCheckKind::kUnresolvedCheck: 7458 // We always go into the type check slow path for the unresolved check case. 7459 // We cannot directly call the CheckCast runtime entry point 7460 // without resorting to a type checking slow path here (i.e. by 7461 // calling InvokeRuntime directly), as it would require to 7462 // assign fixed registers for the inputs of this HInstanceOf 7463 // instruction (following the runtime calling convention), which 7464 // might be cluttered by the potential first read barrier 7465 // emission at the beginning of this method. 7466 7467 __ b(type_check_slow_path->GetEntryLabel()); 7468 break; 7469 7470 case TypeCheckKind::kInterfaceCheck: { 7471 // Avoid read barriers to improve performance of the fast path. We can not get false 7472 // positives by doing this. 7473 // /* HeapReference<Class> */ temp = obj->klass_ 7474 GenerateReferenceLoadTwoRegisters(instruction, 7475 temp_loc, 7476 obj_loc, 7477 class_offset, 7478 maybe_temp2_loc, 7479 kWithoutReadBarrier); 7480 7481 // /* HeapReference<Class> */ temp = temp->iftable_ 7482 GenerateReferenceLoadTwoRegisters(instruction, 7483 temp_loc, 7484 temp_loc, 7485 iftable_offset, 7486 maybe_temp2_loc, 7487 kWithoutReadBarrier); 7488 // Iftable is never null. 7489 __ ldr(maybe_temp2_loc.AsRegister<Register>(), Address(temp, array_length_offset)); 7490 // Loop through the iftable and check if any class matches. 7491 Label start_loop; 7492 __ Bind(&start_loop); 7493 __ CompareAndBranchIfZero(maybe_temp2_loc.AsRegister<Register>(), 7494 type_check_slow_path->GetEntryLabel()); 7495 __ ldr(maybe_temp3_loc.AsRegister<Register>(), Address(temp, object_array_data_offset)); 7496 __ MaybeUnpoisonHeapReference(maybe_temp3_loc.AsRegister<Register>()); 7497 // Go to next interface. 7498 __ add(temp, temp, ShifterOperand(2 * kHeapReferenceSize)); 7499 __ sub(maybe_temp2_loc.AsRegister<Register>(), 7500 maybe_temp2_loc.AsRegister<Register>(), 7501 ShifterOperand(2)); 7502 // Compare the classes and continue the loop if they do not match. 7503 __ cmp(cls, ShifterOperand(maybe_temp3_loc.AsRegister<Register>())); 7504 __ b(&start_loop, NE); 7505 break; 7506 } 7507 } 7508 7509 if (done.IsLinked()) { 7510 __ Bind(&done); 7511 } 7512 7513 __ Bind(type_check_slow_path->GetExitLabel()); 7514} 7515 7516void LocationsBuilderARM::VisitMonitorOperation(HMonitorOperation* instruction) { 7517 LocationSummary* locations = 7518 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCallOnMainOnly); 7519 InvokeRuntimeCallingConvention calling_convention; 7520 locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(0))); 7521} 7522 7523void InstructionCodeGeneratorARM::VisitMonitorOperation(HMonitorOperation* instruction) { 7524 codegen_->InvokeRuntime(instruction->IsEnter() ? kQuickLockObject : kQuickUnlockObject, 7525 instruction, 7526 instruction->GetDexPc()); 7527 if (instruction->IsEnter()) { 7528 CheckEntrypointTypes<kQuickLockObject, void, mirror::Object*>(); 7529 } else { 7530 CheckEntrypointTypes<kQuickUnlockObject, void, mirror::Object*>(); 7531 } 7532} 7533 7534void LocationsBuilderARM::VisitAnd(HAnd* instruction) { HandleBitwiseOperation(instruction, AND); } 7535void LocationsBuilderARM::VisitOr(HOr* instruction) { HandleBitwiseOperation(instruction, ORR); } 7536void LocationsBuilderARM::VisitXor(HXor* instruction) { HandleBitwiseOperation(instruction, EOR); } 7537 7538void LocationsBuilderARM::HandleBitwiseOperation(HBinaryOperation* instruction, Opcode opcode) { 7539 LocationSummary* locations = 7540 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall); 7541 DCHECK(instruction->GetResultType() == Primitive::kPrimInt 7542 || instruction->GetResultType() == Primitive::kPrimLong); 7543 // Note: GVN reorders commutative operations to have the constant on the right hand side. 7544 locations->SetInAt(0, Location::RequiresRegister()); 7545 locations->SetInAt(1, ArmEncodableConstantOrRegister(instruction->InputAt(1), opcode)); 7546 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 7547} 7548 7549void InstructionCodeGeneratorARM::VisitAnd(HAnd* instruction) { 7550 HandleBitwiseOperation(instruction); 7551} 7552 7553void InstructionCodeGeneratorARM::VisitOr(HOr* instruction) { 7554 HandleBitwiseOperation(instruction); 7555} 7556 7557void InstructionCodeGeneratorARM::VisitXor(HXor* instruction) { 7558 HandleBitwiseOperation(instruction); 7559} 7560 7561 7562void LocationsBuilderARM::VisitBitwiseNegatedRight(HBitwiseNegatedRight* instruction) { 7563 LocationSummary* locations = 7564 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall); 7565 DCHECK(instruction->GetResultType() == Primitive::kPrimInt 7566 || instruction->GetResultType() == Primitive::kPrimLong); 7567 7568 locations->SetInAt(0, Location::RequiresRegister()); 7569 locations->SetInAt(1, Location::RequiresRegister()); 7570 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 7571} 7572 7573void InstructionCodeGeneratorARM::VisitBitwiseNegatedRight(HBitwiseNegatedRight* instruction) { 7574 LocationSummary* locations = instruction->GetLocations(); 7575 Location first = locations->InAt(0); 7576 Location second = locations->InAt(1); 7577 Location out = locations->Out(); 7578 7579 if (instruction->GetResultType() == Primitive::kPrimInt) { 7580 Register first_reg = first.AsRegister<Register>(); 7581 ShifterOperand second_reg(second.AsRegister<Register>()); 7582 Register out_reg = out.AsRegister<Register>(); 7583 7584 switch (instruction->GetOpKind()) { 7585 case HInstruction::kAnd: 7586 __ bic(out_reg, first_reg, second_reg); 7587 break; 7588 case HInstruction::kOr: 7589 __ orn(out_reg, first_reg, second_reg); 7590 break; 7591 // There is no EON on arm. 7592 case HInstruction::kXor: 7593 default: 7594 LOG(FATAL) << "Unexpected instruction " << instruction->DebugName(); 7595 UNREACHABLE(); 7596 } 7597 return; 7598 7599 } else { 7600 DCHECK_EQ(instruction->GetResultType(), Primitive::kPrimLong); 7601 Register first_low = first.AsRegisterPairLow<Register>(); 7602 Register first_high = first.AsRegisterPairHigh<Register>(); 7603 ShifterOperand second_low(second.AsRegisterPairLow<Register>()); 7604 ShifterOperand second_high(second.AsRegisterPairHigh<Register>()); 7605 Register out_low = out.AsRegisterPairLow<Register>(); 7606 Register out_high = out.AsRegisterPairHigh<Register>(); 7607 7608 switch (instruction->GetOpKind()) { 7609 case HInstruction::kAnd: 7610 __ bic(out_low, first_low, second_low); 7611 __ bic(out_high, first_high, second_high); 7612 break; 7613 case HInstruction::kOr: 7614 __ orn(out_low, first_low, second_low); 7615 __ orn(out_high, first_high, second_high); 7616 break; 7617 // There is no EON on arm. 7618 case HInstruction::kXor: 7619 default: 7620 LOG(FATAL) << "Unexpected instruction " << instruction->DebugName(); 7621 UNREACHABLE(); 7622 } 7623 } 7624} 7625 7626void LocationsBuilderARM::VisitDataProcWithShifterOp( 7627 HDataProcWithShifterOp* instruction) { 7628 DCHECK(instruction->GetType() == Primitive::kPrimInt || 7629 instruction->GetType() == Primitive::kPrimLong); 7630 LocationSummary* locations = 7631 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall); 7632 const bool overlap = instruction->GetType() == Primitive::kPrimLong && 7633 HDataProcWithShifterOp::IsExtensionOp(instruction->GetOpKind()); 7634 7635 locations->SetInAt(0, Location::RequiresRegister()); 7636 locations->SetInAt(1, Location::RequiresRegister()); 7637 locations->SetOut(Location::RequiresRegister(), 7638 overlap ? Location::kOutputOverlap : Location::kNoOutputOverlap); 7639} 7640 7641void InstructionCodeGeneratorARM::VisitDataProcWithShifterOp( 7642 HDataProcWithShifterOp* instruction) { 7643 const LocationSummary* const locations = instruction->GetLocations(); 7644 const HInstruction::InstructionKind kind = instruction->GetInstrKind(); 7645 const HDataProcWithShifterOp::OpKind op_kind = instruction->GetOpKind(); 7646 const Location left = locations->InAt(0); 7647 const Location right = locations->InAt(1); 7648 const Location out = locations->Out(); 7649 7650 if (instruction->GetType() == Primitive::kPrimInt) { 7651 DCHECK(!HDataProcWithShifterOp::IsExtensionOp(op_kind)); 7652 7653 const Register second = instruction->InputAt(1)->GetType() == Primitive::kPrimLong 7654 ? right.AsRegisterPairLow<Register>() 7655 : right.AsRegister<Register>(); 7656 7657 GenerateDataProcInstruction(kind, 7658 out.AsRegister<Register>(), 7659 left.AsRegister<Register>(), 7660 ShifterOperand(second, 7661 ShiftFromOpKind(op_kind), 7662 instruction->GetShiftAmount()), 7663 codegen_); 7664 } else { 7665 DCHECK_EQ(instruction->GetType(), Primitive::kPrimLong); 7666 7667 if (HDataProcWithShifterOp::IsExtensionOp(op_kind)) { 7668 const Register second = right.AsRegister<Register>(); 7669 7670 DCHECK_NE(out.AsRegisterPairLow<Register>(), second); 7671 GenerateDataProc(kind, 7672 out, 7673 left, 7674 ShifterOperand(second), 7675 ShifterOperand(second, ASR, 31), 7676 codegen_); 7677 } else { 7678 GenerateLongDataProc(instruction, codegen_); 7679 } 7680 } 7681} 7682 7683void InstructionCodeGeneratorARM::GenerateAndConst(Register out, Register first, uint32_t value) { 7684 // Optimize special cases for individual halfs of `and-long` (`and` is simplified earlier). 7685 if (value == 0xffffffffu) { 7686 if (out != first) { 7687 __ mov(out, ShifterOperand(first)); 7688 } 7689 return; 7690 } 7691 if (value == 0u) { 7692 __ mov(out, ShifterOperand(0)); 7693 return; 7694 } 7695 ShifterOperand so; 7696 if (__ ShifterOperandCanHold(kNoRegister, kNoRegister, AND, value, &so)) { 7697 __ and_(out, first, so); 7698 } else if (__ ShifterOperandCanHold(kNoRegister, kNoRegister, BIC, ~value, &so)) { 7699 __ bic(out, first, ShifterOperand(~value)); 7700 } else { 7701 DCHECK(IsPowerOfTwo(value + 1)); 7702 __ ubfx(out, first, 0, WhichPowerOf2(value + 1)); 7703 } 7704} 7705 7706void InstructionCodeGeneratorARM::GenerateOrrConst(Register out, Register first, uint32_t value) { 7707 // Optimize special cases for individual halfs of `or-long` (`or` is simplified earlier). 7708 if (value == 0u) { 7709 if (out != first) { 7710 __ mov(out, ShifterOperand(first)); 7711 } 7712 return; 7713 } 7714 if (value == 0xffffffffu) { 7715 __ mvn(out, ShifterOperand(0)); 7716 return; 7717 } 7718 ShifterOperand so; 7719 if (__ ShifterOperandCanHold(kNoRegister, kNoRegister, ORR, value, &so)) { 7720 __ orr(out, first, so); 7721 } else { 7722 DCHECK(__ ShifterOperandCanHold(kNoRegister, kNoRegister, ORN, ~value, &so)); 7723 __ orn(out, first, ShifterOperand(~value)); 7724 } 7725} 7726 7727void InstructionCodeGeneratorARM::GenerateEorConst(Register out, Register first, uint32_t value) { 7728 // Optimize special case for individual halfs of `xor-long` (`xor` is simplified earlier). 7729 if (value == 0u) { 7730 if (out != first) { 7731 __ mov(out, ShifterOperand(first)); 7732 } 7733 return; 7734 } 7735 __ eor(out, first, ShifterOperand(value)); 7736} 7737 7738void InstructionCodeGeneratorARM::GenerateAddLongConst(Location out, 7739 Location first, 7740 uint64_t value) { 7741 Register out_low = out.AsRegisterPairLow<Register>(); 7742 Register out_high = out.AsRegisterPairHigh<Register>(); 7743 Register first_low = first.AsRegisterPairLow<Register>(); 7744 Register first_high = first.AsRegisterPairHigh<Register>(); 7745 uint32_t value_low = Low32Bits(value); 7746 uint32_t value_high = High32Bits(value); 7747 if (value_low == 0u) { 7748 if (out_low != first_low) { 7749 __ mov(out_low, ShifterOperand(first_low)); 7750 } 7751 __ AddConstant(out_high, first_high, value_high); 7752 return; 7753 } 7754 __ AddConstantSetFlags(out_low, first_low, value_low); 7755 ShifterOperand so; 7756 if (__ ShifterOperandCanHold(out_high, first_high, ADC, value_high, kCcDontCare, &so)) { 7757 __ adc(out_high, first_high, so); 7758 } else if (__ ShifterOperandCanHold(out_low, first_low, SBC, ~value_high, kCcDontCare, &so)) { 7759 __ sbc(out_high, first_high, so); 7760 } else { 7761 LOG(FATAL) << "Unexpected constant " << value_high; 7762 UNREACHABLE(); 7763 } 7764} 7765 7766void InstructionCodeGeneratorARM::HandleBitwiseOperation(HBinaryOperation* instruction) { 7767 LocationSummary* locations = instruction->GetLocations(); 7768 Location first = locations->InAt(0); 7769 Location second = locations->InAt(1); 7770 Location out = locations->Out(); 7771 7772 if (second.IsConstant()) { 7773 uint64_t value = static_cast<uint64_t>(Int64FromConstant(second.GetConstant())); 7774 uint32_t value_low = Low32Bits(value); 7775 if (instruction->GetResultType() == Primitive::kPrimInt) { 7776 Register first_reg = first.AsRegister<Register>(); 7777 Register out_reg = out.AsRegister<Register>(); 7778 if (instruction->IsAnd()) { 7779 GenerateAndConst(out_reg, first_reg, value_low); 7780 } else if (instruction->IsOr()) { 7781 GenerateOrrConst(out_reg, first_reg, value_low); 7782 } else { 7783 DCHECK(instruction->IsXor()); 7784 GenerateEorConst(out_reg, first_reg, value_low); 7785 } 7786 } else { 7787 DCHECK_EQ(instruction->GetResultType(), Primitive::kPrimLong); 7788 uint32_t value_high = High32Bits(value); 7789 Register first_low = first.AsRegisterPairLow<Register>(); 7790 Register first_high = first.AsRegisterPairHigh<Register>(); 7791 Register out_low = out.AsRegisterPairLow<Register>(); 7792 Register out_high = out.AsRegisterPairHigh<Register>(); 7793 if (instruction->IsAnd()) { 7794 GenerateAndConst(out_low, first_low, value_low); 7795 GenerateAndConst(out_high, first_high, value_high); 7796 } else if (instruction->IsOr()) { 7797 GenerateOrrConst(out_low, first_low, value_low); 7798 GenerateOrrConst(out_high, first_high, value_high); 7799 } else { 7800 DCHECK(instruction->IsXor()); 7801 GenerateEorConst(out_low, first_low, value_low); 7802 GenerateEorConst(out_high, first_high, value_high); 7803 } 7804 } 7805 return; 7806 } 7807 7808 if (instruction->GetResultType() == Primitive::kPrimInt) { 7809 Register first_reg = first.AsRegister<Register>(); 7810 ShifterOperand second_reg(second.AsRegister<Register>()); 7811 Register out_reg = out.AsRegister<Register>(); 7812 if (instruction->IsAnd()) { 7813 __ and_(out_reg, first_reg, second_reg); 7814 } else if (instruction->IsOr()) { 7815 __ orr(out_reg, first_reg, second_reg); 7816 } else { 7817 DCHECK(instruction->IsXor()); 7818 __ eor(out_reg, first_reg, second_reg); 7819 } 7820 } else { 7821 DCHECK_EQ(instruction->GetResultType(), Primitive::kPrimLong); 7822 Register first_low = first.AsRegisterPairLow<Register>(); 7823 Register first_high = first.AsRegisterPairHigh<Register>(); 7824 ShifterOperand second_low(second.AsRegisterPairLow<Register>()); 7825 ShifterOperand second_high(second.AsRegisterPairHigh<Register>()); 7826 Register out_low = out.AsRegisterPairLow<Register>(); 7827 Register out_high = out.AsRegisterPairHigh<Register>(); 7828 if (instruction->IsAnd()) { 7829 __ and_(out_low, first_low, second_low); 7830 __ and_(out_high, first_high, second_high); 7831 } else if (instruction->IsOr()) { 7832 __ orr(out_low, first_low, second_low); 7833 __ orr(out_high, first_high, second_high); 7834 } else { 7835 DCHECK(instruction->IsXor()); 7836 __ eor(out_low, first_low, second_low); 7837 __ eor(out_high, first_high, second_high); 7838 } 7839 } 7840} 7841 7842void InstructionCodeGeneratorARM::GenerateReferenceLoadOneRegister( 7843 HInstruction* instruction, 7844 Location out, 7845 uint32_t offset, 7846 Location maybe_temp, 7847 ReadBarrierOption read_barrier_option) { 7848 Register out_reg = out.AsRegister<Register>(); 7849 if (read_barrier_option == kWithReadBarrier) { 7850 CHECK(kEmitCompilerReadBarrier); 7851 DCHECK(maybe_temp.IsRegister()) << maybe_temp; 7852 if (kUseBakerReadBarrier) { 7853 // Load with fast path based Baker's read barrier. 7854 // /* HeapReference<Object> */ out = *(out + offset) 7855 codegen_->GenerateFieldLoadWithBakerReadBarrier( 7856 instruction, out, out_reg, offset, maybe_temp, /* needs_null_check */ false); 7857 } else { 7858 // Load with slow path based read barrier. 7859 // Save the value of `out` into `maybe_temp` before overwriting it 7860 // in the following move operation, as we will need it for the 7861 // read barrier below. 7862 __ Mov(maybe_temp.AsRegister<Register>(), out_reg); 7863 // /* HeapReference<Object> */ out = *(out + offset) 7864 __ LoadFromOffset(kLoadWord, out_reg, out_reg, offset); 7865 codegen_->GenerateReadBarrierSlow(instruction, out, out, maybe_temp, offset); 7866 } 7867 } else { 7868 // Plain load with no read barrier. 7869 // /* HeapReference<Object> */ out = *(out + offset) 7870 __ LoadFromOffset(kLoadWord, out_reg, out_reg, offset); 7871 __ MaybeUnpoisonHeapReference(out_reg); 7872 } 7873} 7874 7875void InstructionCodeGeneratorARM::GenerateReferenceLoadTwoRegisters( 7876 HInstruction* instruction, 7877 Location out, 7878 Location obj, 7879 uint32_t offset, 7880 Location maybe_temp, 7881 ReadBarrierOption read_barrier_option) { 7882 Register out_reg = out.AsRegister<Register>(); 7883 Register obj_reg = obj.AsRegister<Register>(); 7884 if (read_barrier_option == kWithReadBarrier) { 7885 CHECK(kEmitCompilerReadBarrier); 7886 if (kUseBakerReadBarrier) { 7887 DCHECK(maybe_temp.IsRegister()) << maybe_temp; 7888 // Load with fast path based Baker's read barrier. 7889 // /* HeapReference<Object> */ out = *(obj + offset) 7890 codegen_->GenerateFieldLoadWithBakerReadBarrier( 7891 instruction, out, obj_reg, offset, maybe_temp, /* needs_null_check */ false); 7892 } else { 7893 // Load with slow path based read barrier. 7894 // /* HeapReference<Object> */ out = *(obj + offset) 7895 __ LoadFromOffset(kLoadWord, out_reg, obj_reg, offset); 7896 codegen_->GenerateReadBarrierSlow(instruction, out, out, obj, offset); 7897 } 7898 } else { 7899 // Plain load with no read barrier. 7900 // /* HeapReference<Object> */ out = *(obj + offset) 7901 __ LoadFromOffset(kLoadWord, out_reg, obj_reg, offset); 7902 __ MaybeUnpoisonHeapReference(out_reg); 7903 } 7904} 7905 7906void InstructionCodeGeneratorARM::GenerateGcRootFieldLoad(HInstruction* instruction, 7907 Location root, 7908 Register obj, 7909 uint32_t offset, 7910 ReadBarrierOption read_barrier_option) { 7911 Register root_reg = root.AsRegister<Register>(); 7912 if (read_barrier_option == kWithReadBarrier) { 7913 DCHECK(kEmitCompilerReadBarrier); 7914 if (kUseBakerReadBarrier) { 7915 // Fast path implementation of art::ReadBarrier::BarrierForRoot when 7916 // Baker's read barrier are used. 7917 // 7918 // Note that we do not actually check the value of 7919 // `GetIsGcMarking()` to decide whether to mark the loaded GC 7920 // root or not. Instead, we load into `temp` the read barrier 7921 // mark entry point corresponding to register `root`. If `temp` 7922 // is null, it means that `GetIsGcMarking()` is false, and vice 7923 // versa. 7924 // 7925 // temp = Thread::Current()->pReadBarrierMarkReg ## root.reg() 7926 // GcRoot<mirror::Object> root = *(obj+offset); // Original reference load. 7927 // if (temp != nullptr) { // <=> Thread::Current()->GetIsGcMarking() 7928 // // Slow path. 7929 // root = temp(root); // root = ReadBarrier::Mark(root); // Runtime entry point call. 7930 // } 7931 7932 // Slow path marking the GC root `root`. The entrypoint will already be loaded in `temp`. 7933 Location temp = Location::RegisterLocation(LR); 7934 SlowPathCodeARM* slow_path = new (GetGraph()->GetArena()) ReadBarrierMarkSlowPathARM( 7935 instruction, root, /* entrypoint */ temp); 7936 codegen_->AddSlowPath(slow_path); 7937 7938 // temp = Thread::Current()->pReadBarrierMarkReg ## root.reg() 7939 const int32_t entry_point_offset = 7940 CodeGenerator::GetReadBarrierMarkEntryPointsOffset<kArmPointerSize>(root.reg()); 7941 // Loading the entrypoint does not require a load acquire since it is only changed when 7942 // threads are suspended or running a checkpoint. 7943 __ LoadFromOffset(kLoadWord, temp.AsRegister<Register>(), TR, entry_point_offset); 7944 7945 // /* GcRoot<mirror::Object> */ root = *(obj + offset) 7946 __ LoadFromOffset(kLoadWord, root_reg, obj, offset); 7947 static_assert( 7948 sizeof(mirror::CompressedReference<mirror::Object>) == sizeof(GcRoot<mirror::Object>), 7949 "art::mirror::CompressedReference<mirror::Object> and art::GcRoot<mirror::Object> " 7950 "have different sizes."); 7951 static_assert(sizeof(mirror::CompressedReference<mirror::Object>) == sizeof(int32_t), 7952 "art::mirror::CompressedReference<mirror::Object> and int32_t " 7953 "have different sizes."); 7954 7955 // The entrypoint is null when the GC is not marking, this prevents one load compared to 7956 // checking GetIsGcMarking. 7957 __ CompareAndBranchIfNonZero(temp.AsRegister<Register>(), slow_path->GetEntryLabel()); 7958 __ Bind(slow_path->GetExitLabel()); 7959 } else { 7960 // GC root loaded through a slow path for read barriers other 7961 // than Baker's. 7962 // /* GcRoot<mirror::Object>* */ root = obj + offset 7963 __ AddConstant(root_reg, obj, offset); 7964 // /* mirror::Object* */ root = root->Read() 7965 codegen_->GenerateReadBarrierForRootSlow(instruction, root, root); 7966 } 7967 } else { 7968 // Plain GC root load with no read barrier. 7969 // /* GcRoot<mirror::Object> */ root = *(obj + offset) 7970 __ LoadFromOffset(kLoadWord, root_reg, obj, offset); 7971 // Note that GC roots are not affected by heap poisoning, thus we 7972 // do not have to unpoison `root_reg` here. 7973 } 7974} 7975 7976void CodeGeneratorARM::GenerateFieldLoadWithBakerReadBarrier(HInstruction* instruction, 7977 Location ref, 7978 Register obj, 7979 uint32_t offset, 7980 Location temp, 7981 bool needs_null_check) { 7982 DCHECK(kEmitCompilerReadBarrier); 7983 DCHECK(kUseBakerReadBarrier); 7984 7985 // /* HeapReference<Object> */ ref = *(obj + offset) 7986 Location no_index = Location::NoLocation(); 7987 ScaleFactor no_scale_factor = TIMES_1; 7988 GenerateReferenceLoadWithBakerReadBarrier( 7989 instruction, ref, obj, offset, no_index, no_scale_factor, temp, needs_null_check); 7990} 7991 7992void CodeGeneratorARM::GenerateArrayLoadWithBakerReadBarrier(HInstruction* instruction, 7993 Location ref, 7994 Register obj, 7995 uint32_t data_offset, 7996 Location index, 7997 Location temp, 7998 bool needs_null_check) { 7999 DCHECK(kEmitCompilerReadBarrier); 8000 DCHECK(kUseBakerReadBarrier); 8001 8002 static_assert( 8003 sizeof(mirror::HeapReference<mirror::Object>) == sizeof(int32_t), 8004 "art::mirror::HeapReference<art::mirror::Object> and int32_t have different sizes."); 8005 // /* HeapReference<Object> */ ref = 8006 // *(obj + data_offset + index * sizeof(HeapReference<Object>)) 8007 ScaleFactor scale_factor = TIMES_4; 8008 GenerateReferenceLoadWithBakerReadBarrier( 8009 instruction, ref, obj, data_offset, index, scale_factor, temp, needs_null_check); 8010} 8011 8012void CodeGeneratorARM::GenerateReferenceLoadWithBakerReadBarrier(HInstruction* instruction, 8013 Location ref, 8014 Register obj, 8015 uint32_t offset, 8016 Location index, 8017 ScaleFactor scale_factor, 8018 Location temp, 8019 bool needs_null_check, 8020 bool always_update_field, 8021 Register* temp2) { 8022 DCHECK(kEmitCompilerReadBarrier); 8023 DCHECK(kUseBakerReadBarrier); 8024 8025 // Query `art::Thread::Current()->GetIsGcMarking()` to decide 8026 // whether we need to enter the slow path to mark the reference. 8027 // Then, in the slow path, check the gray bit in the lock word of 8028 // the reference's holder (`obj`) to decide whether to mark `ref` or 8029 // not. 8030 // 8031 // Note that we do not actually check the value of `GetIsGcMarking()`; 8032 // instead, we load into `temp3` the read barrier mark entry point 8033 // corresponding to register `ref`. If `temp3` is null, it means 8034 // that `GetIsGcMarking()` is false, and vice versa. 8035 // 8036 // temp3 = Thread::Current()->pReadBarrierMarkReg ## root.reg() 8037 // if (temp3 != nullptr) { // <=> Thread::Current()->GetIsGcMarking() 8038 // // Slow path. 8039 // uint32_t rb_state = Lockword(obj->monitor_).ReadBarrierState(); 8040 // lfence; // Load fence or artificial data dependency to prevent load-load reordering 8041 // HeapReference<mirror::Object> ref = *src; // Original reference load. 8042 // bool is_gray = (rb_state == ReadBarrier::GrayState()); 8043 // if (is_gray) { 8044 // ref = temp3(ref); // ref = ReadBarrier::Mark(ref); // Runtime entry point call. 8045 // } 8046 // } else { 8047 // HeapReference<mirror::Object> ref = *src; // Original reference load. 8048 // } 8049 8050 Register temp_reg = temp.AsRegister<Register>(); 8051 8052 // Slow path marking the object `ref` when the GC is marking. The 8053 // entrypoint will already be loaded in `temp3`. 8054 Location temp3 = Location::RegisterLocation(LR); 8055 SlowPathCodeARM* slow_path; 8056 if (always_update_field) { 8057 DCHECK(temp2 != nullptr); 8058 // LoadReferenceWithBakerReadBarrierAndUpdateFieldSlowPathARM only 8059 // supports address of the form `obj + field_offset`, where `obj` 8060 // is a register and `field_offset` is a register pair (of which 8061 // only the lower half is used). Thus `offset` and `scale_factor` 8062 // above are expected to be null in this code path. 8063 DCHECK_EQ(offset, 0u); 8064 DCHECK_EQ(scale_factor, ScaleFactor::TIMES_1); 8065 Location field_offset = index; 8066 slow_path = 8067 new (GetGraph()->GetArena()) LoadReferenceWithBakerReadBarrierAndUpdateFieldSlowPathARM( 8068 instruction, 8069 ref, 8070 obj, 8071 offset, 8072 /* index */ field_offset, 8073 scale_factor, 8074 needs_null_check, 8075 temp_reg, 8076 *temp2, 8077 /* entrypoint */ temp3); 8078 } else { 8079 slow_path = new (GetGraph()->GetArena()) LoadReferenceWithBakerReadBarrierSlowPathARM( 8080 instruction, 8081 ref, 8082 obj, 8083 offset, 8084 index, 8085 scale_factor, 8086 needs_null_check, 8087 temp_reg, 8088 /* entrypoint */ temp3); 8089 } 8090 AddSlowPath(slow_path); 8091 8092 // temp3 = Thread::Current()->pReadBarrierMarkReg ## ref.reg() 8093 const int32_t entry_point_offset = 8094 CodeGenerator::GetReadBarrierMarkEntryPointsOffset<kArmPointerSize>(ref.reg()); 8095 // Loading the entrypoint does not require a load acquire since it is only changed when 8096 // threads are suspended or running a checkpoint. 8097 __ LoadFromOffset(kLoadWord, temp3.AsRegister<Register>(), TR, entry_point_offset); 8098 // The entrypoint is null when the GC is not marking, this prevents one load compared to 8099 // checking GetIsGcMarking. 8100 __ CompareAndBranchIfNonZero(temp3.AsRegister<Register>(), slow_path->GetEntryLabel()); 8101 // Fast path: just load the reference. 8102 GenerateRawReferenceLoad(instruction, ref, obj, offset, index, scale_factor, needs_null_check); 8103 __ Bind(slow_path->GetExitLabel()); 8104} 8105 8106void CodeGeneratorARM::GenerateRawReferenceLoad(HInstruction* instruction, 8107 Location ref, 8108 Register obj, 8109 uint32_t offset, 8110 Location index, 8111 ScaleFactor scale_factor, 8112 bool needs_null_check) { 8113 Register ref_reg = ref.AsRegister<Register>(); 8114 8115 if (index.IsValid()) { 8116 // Load types involving an "index": ArrayGet, 8117 // UnsafeGetObject/UnsafeGetObjectVolatile and UnsafeCASObject 8118 // intrinsics. 8119 // /* HeapReference<mirror::Object> */ ref = *(obj + offset + (index << scale_factor)) 8120 if (index.IsConstant()) { 8121 size_t computed_offset = 8122 (index.GetConstant()->AsIntConstant()->GetValue() << scale_factor) + offset; 8123 __ LoadFromOffset(kLoadWord, ref_reg, obj, computed_offset); 8124 } else { 8125 // Handle the special case of the 8126 // UnsafeGetObject/UnsafeGetObjectVolatile and UnsafeCASObject 8127 // intrinsics, which use a register pair as index ("long 8128 // offset"), of which only the low part contains data. 8129 Register index_reg = index.IsRegisterPair() 8130 ? index.AsRegisterPairLow<Register>() 8131 : index.AsRegister<Register>(); 8132 __ add(IP, obj, ShifterOperand(index_reg, LSL, scale_factor)); 8133 __ LoadFromOffset(kLoadWord, ref_reg, IP, offset); 8134 } 8135 } else { 8136 // /* HeapReference<mirror::Object> */ ref = *(obj + offset) 8137 __ LoadFromOffset(kLoadWord, ref_reg, obj, offset); 8138 } 8139 8140 if (needs_null_check) { 8141 MaybeRecordImplicitNullCheck(instruction); 8142 } 8143 8144 // Object* ref = ref_addr->AsMirrorPtr() 8145 __ MaybeUnpoisonHeapReference(ref_reg); 8146} 8147 8148void CodeGeneratorARM::GenerateReadBarrierSlow(HInstruction* instruction, 8149 Location out, 8150 Location ref, 8151 Location obj, 8152 uint32_t offset, 8153 Location index) { 8154 DCHECK(kEmitCompilerReadBarrier); 8155 8156 // Insert a slow path based read barrier *after* the reference load. 8157 // 8158 // If heap poisoning is enabled, the unpoisoning of the loaded 8159 // reference will be carried out by the runtime within the slow 8160 // path. 8161 // 8162 // Note that `ref` currently does not get unpoisoned (when heap 8163 // poisoning is enabled), which is alright as the `ref` argument is 8164 // not used by the artReadBarrierSlow entry point. 8165 // 8166 // TODO: Unpoison `ref` when it is used by artReadBarrierSlow. 8167 SlowPathCodeARM* slow_path = new (GetGraph()->GetArena()) 8168 ReadBarrierForHeapReferenceSlowPathARM(instruction, out, ref, obj, offset, index); 8169 AddSlowPath(slow_path); 8170 8171 __ b(slow_path->GetEntryLabel()); 8172 __ Bind(slow_path->GetExitLabel()); 8173} 8174 8175void CodeGeneratorARM::MaybeGenerateReadBarrierSlow(HInstruction* instruction, 8176 Location out, 8177 Location ref, 8178 Location obj, 8179 uint32_t offset, 8180 Location index) { 8181 if (kEmitCompilerReadBarrier) { 8182 // Baker's read barriers shall be handled by the fast path 8183 // (CodeGeneratorARM::GenerateReferenceLoadWithBakerReadBarrier). 8184 DCHECK(!kUseBakerReadBarrier); 8185 // If heap poisoning is enabled, unpoisoning will be taken care of 8186 // by the runtime within the slow path. 8187 GenerateReadBarrierSlow(instruction, out, ref, obj, offset, index); 8188 } else if (kPoisonHeapReferences) { 8189 __ UnpoisonHeapReference(out.AsRegister<Register>()); 8190 } 8191} 8192 8193void CodeGeneratorARM::GenerateReadBarrierForRootSlow(HInstruction* instruction, 8194 Location out, 8195 Location root) { 8196 DCHECK(kEmitCompilerReadBarrier); 8197 8198 // Insert a slow path based read barrier *after* the GC root load. 8199 // 8200 // Note that GC roots are not affected by heap poisoning, so we do 8201 // not need to do anything special for this here. 8202 SlowPathCodeARM* slow_path = 8203 new (GetGraph()->GetArena()) ReadBarrierForRootSlowPathARM(instruction, out, root); 8204 AddSlowPath(slow_path); 8205 8206 __ b(slow_path->GetEntryLabel()); 8207 __ Bind(slow_path->GetExitLabel()); 8208} 8209 8210HInvokeStaticOrDirect::DispatchInfo CodeGeneratorARM::GetSupportedInvokeStaticOrDirectDispatch( 8211 const HInvokeStaticOrDirect::DispatchInfo& desired_dispatch_info, 8212 HInvokeStaticOrDirect* invoke ATTRIBUTE_UNUSED) { 8213 return desired_dispatch_info; 8214} 8215 8216Register CodeGeneratorARM::GetInvokeStaticOrDirectExtraParameter(HInvokeStaticOrDirect* invoke, 8217 Register temp) { 8218 DCHECK_EQ(invoke->InputCount(), invoke->GetNumberOfArguments() + 1u); 8219 Location location = invoke->GetLocations()->InAt(invoke->GetSpecialInputIndex()); 8220 if (!invoke->GetLocations()->Intrinsified()) { 8221 return location.AsRegister<Register>(); 8222 } 8223 // For intrinsics we allow any location, so it may be on the stack. 8224 if (!location.IsRegister()) { 8225 __ LoadFromOffset(kLoadWord, temp, SP, location.GetStackIndex()); 8226 return temp; 8227 } 8228 // For register locations, check if the register was saved. If so, get it from the stack. 8229 // Note: There is a chance that the register was saved but not overwritten, so we could 8230 // save one load. However, since this is just an intrinsic slow path we prefer this 8231 // simple and more robust approach rather that trying to determine if that's the case. 8232 SlowPathCode* slow_path = GetCurrentSlowPath(); 8233 if (slow_path != nullptr && slow_path->IsCoreRegisterSaved(location.AsRegister<Register>())) { 8234 int stack_offset = slow_path->GetStackOffsetOfCoreRegister(location.AsRegister<Register>()); 8235 __ LoadFromOffset(kLoadWord, temp, SP, stack_offset); 8236 return temp; 8237 } 8238 return location.AsRegister<Register>(); 8239} 8240 8241Location CodeGeneratorARM::GenerateCalleeMethodStaticOrDirectCall(HInvokeStaticOrDirect* invoke, 8242 Location temp) { 8243 Location callee_method = temp; // For all kinds except kRecursive, callee will be in temp. 8244 switch (invoke->GetMethodLoadKind()) { 8245 case HInvokeStaticOrDirect::MethodLoadKind::kStringInit: { 8246 uint32_t offset = 8247 GetThreadOffset<kArmPointerSize>(invoke->GetStringInitEntryPoint()).Int32Value(); 8248 // temp = thread->string_init_entrypoint 8249 __ LoadFromOffset(kLoadWord, temp.AsRegister<Register>(), TR, offset); 8250 break; 8251 } 8252 case HInvokeStaticOrDirect::MethodLoadKind::kRecursive: 8253 callee_method = invoke->GetLocations()->InAt(invoke->GetSpecialInputIndex()); 8254 break; 8255 case HInvokeStaticOrDirect::MethodLoadKind::kDirectAddress: 8256 __ LoadImmediate(temp.AsRegister<Register>(), invoke->GetMethodAddress()); 8257 break; 8258 case HInvokeStaticOrDirect::MethodLoadKind::kDexCachePcRelative: { 8259 HArmDexCacheArraysBase* base = 8260 invoke->InputAt(invoke->GetSpecialInputIndex())->AsArmDexCacheArraysBase(); 8261 Register base_reg = GetInvokeStaticOrDirectExtraParameter(invoke, 8262 temp.AsRegister<Register>()); 8263 int32_t offset = invoke->GetDexCacheArrayOffset() - base->GetElementOffset(); 8264 __ LoadFromOffset(kLoadWord, temp.AsRegister<Register>(), base_reg, offset); 8265 break; 8266 } 8267 case HInvokeStaticOrDirect::MethodLoadKind::kDexCacheViaMethod: { 8268 Location current_method = invoke->GetLocations()->InAt(invoke->GetSpecialInputIndex()); 8269 Register method_reg; 8270 Register reg = temp.AsRegister<Register>(); 8271 if (current_method.IsRegister()) { 8272 method_reg = current_method.AsRegister<Register>(); 8273 } else { 8274 DCHECK(invoke->GetLocations()->Intrinsified()); 8275 DCHECK(!current_method.IsValid()); 8276 method_reg = reg; 8277 __ LoadFromOffset(kLoadWord, reg, SP, kCurrentMethodStackOffset); 8278 } 8279 // /* ArtMethod*[] */ temp = temp.ptr_sized_fields_->dex_cache_resolved_methods_; 8280 __ LoadFromOffset(kLoadWord, 8281 reg, 8282 method_reg, 8283 ArtMethod::DexCacheResolvedMethodsOffset(kArmPointerSize).Int32Value()); 8284 // temp = temp[index_in_cache]; 8285 // Note: Don't use invoke->GetTargetMethod() as it may point to a different dex file. 8286 uint32_t index_in_cache = invoke->GetDexMethodIndex(); 8287 __ LoadFromOffset(kLoadWord, reg, reg, CodeGenerator::GetCachePointerOffset(index_in_cache)); 8288 break; 8289 } 8290 } 8291 return callee_method; 8292} 8293 8294void CodeGeneratorARM::GenerateStaticOrDirectCall(HInvokeStaticOrDirect* invoke, Location temp) { 8295 Location callee_method = GenerateCalleeMethodStaticOrDirectCall(invoke, temp); 8296 8297 switch (invoke->GetCodePtrLocation()) { 8298 case HInvokeStaticOrDirect::CodePtrLocation::kCallSelf: 8299 __ bl(GetFrameEntryLabel()); 8300 break; 8301 case HInvokeStaticOrDirect::CodePtrLocation::kCallArtMethod: 8302 // LR = callee_method->entry_point_from_quick_compiled_code_ 8303 __ LoadFromOffset( 8304 kLoadWord, LR, callee_method.AsRegister<Register>(), 8305 ArtMethod::EntryPointFromQuickCompiledCodeOffset(kArmPointerSize).Int32Value()); 8306 // LR() 8307 __ blx(LR); 8308 break; 8309 } 8310 8311 DCHECK(!IsLeafMethod()); 8312} 8313 8314void CodeGeneratorARM::GenerateVirtualCall(HInvokeVirtual* invoke, Location temp_location) { 8315 Register temp = temp_location.AsRegister<Register>(); 8316 uint32_t method_offset = mirror::Class::EmbeddedVTableEntryOffset( 8317 invoke->GetVTableIndex(), kArmPointerSize).Uint32Value(); 8318 8319 // Use the calling convention instead of the location of the receiver, as 8320 // intrinsics may have put the receiver in a different register. In the intrinsics 8321 // slow path, the arguments have been moved to the right place, so here we are 8322 // guaranteed that the receiver is the first register of the calling convention. 8323 InvokeDexCallingConvention calling_convention; 8324 Register receiver = calling_convention.GetRegisterAt(0); 8325 uint32_t class_offset = mirror::Object::ClassOffset().Int32Value(); 8326 // /* HeapReference<Class> */ temp = receiver->klass_ 8327 __ LoadFromOffset(kLoadWord, temp, receiver, class_offset); 8328 MaybeRecordImplicitNullCheck(invoke); 8329 // Instead of simply (possibly) unpoisoning `temp` here, we should 8330 // emit a read barrier for the previous class reference load. 8331 // However this is not required in practice, as this is an 8332 // intermediate/temporary reference and because the current 8333 // concurrent copying collector keeps the from-space memory 8334 // intact/accessible until the end of the marking phase (the 8335 // concurrent copying collector may not in the future). 8336 __ MaybeUnpoisonHeapReference(temp); 8337 // temp = temp->GetMethodAt(method_offset); 8338 uint32_t entry_point = ArtMethod::EntryPointFromQuickCompiledCodeOffset( 8339 kArmPointerSize).Int32Value(); 8340 __ LoadFromOffset(kLoadWord, temp, temp, method_offset); 8341 // LR = temp->GetEntryPoint(); 8342 __ LoadFromOffset(kLoadWord, LR, temp, entry_point); 8343 // LR(); 8344 __ blx(LR); 8345} 8346 8347CodeGeneratorARM::PcRelativePatchInfo* CodeGeneratorARM::NewPcRelativeStringPatch( 8348 const DexFile& dex_file, dex::StringIndex string_index) { 8349 return NewPcRelativePatch(dex_file, string_index.index_, &pc_relative_string_patches_); 8350} 8351 8352CodeGeneratorARM::PcRelativePatchInfo* CodeGeneratorARM::NewPcRelativeTypePatch( 8353 const DexFile& dex_file, dex::TypeIndex type_index) { 8354 return NewPcRelativePatch(dex_file, type_index.index_, &pc_relative_type_patches_); 8355} 8356 8357CodeGeneratorARM::PcRelativePatchInfo* CodeGeneratorARM::NewTypeBssEntryPatch( 8358 const DexFile& dex_file, dex::TypeIndex type_index) { 8359 return NewPcRelativePatch(dex_file, type_index.index_, &type_bss_entry_patches_); 8360} 8361 8362CodeGeneratorARM::PcRelativePatchInfo* CodeGeneratorARM::NewPcRelativeDexCacheArrayPatch( 8363 const DexFile& dex_file, uint32_t element_offset) { 8364 return NewPcRelativePatch(dex_file, element_offset, &pc_relative_dex_cache_patches_); 8365} 8366 8367CodeGeneratorARM::PcRelativePatchInfo* CodeGeneratorARM::NewPcRelativePatch( 8368 const DexFile& dex_file, uint32_t offset_or_index, ArenaDeque<PcRelativePatchInfo>* patches) { 8369 patches->emplace_back(dex_file, offset_or_index); 8370 return &patches->back(); 8371} 8372 8373Literal* CodeGeneratorARM::DeduplicateBootImageStringLiteral(const DexFile& dex_file, 8374 dex::StringIndex string_index) { 8375 return boot_image_string_patches_.GetOrCreate( 8376 StringReference(&dex_file, string_index), 8377 [this]() { return __ NewLiteral<uint32_t>(/* placeholder */ 0u); }); 8378} 8379 8380Literal* CodeGeneratorARM::DeduplicateBootImageTypeLiteral(const DexFile& dex_file, 8381 dex::TypeIndex type_index) { 8382 return boot_image_type_patches_.GetOrCreate( 8383 TypeReference(&dex_file, type_index), 8384 [this]() { return __ NewLiteral<uint32_t>(/* placeholder */ 0u); }); 8385} 8386 8387Literal* CodeGeneratorARM::DeduplicateBootImageAddressLiteral(uint32_t address) { 8388 return DeduplicateUint32Literal(dchecked_integral_cast<uint32_t>(address), &uint32_literals_); 8389} 8390 8391Literal* CodeGeneratorARM::DeduplicateJitStringLiteral(const DexFile& dex_file, 8392 dex::StringIndex string_index, 8393 Handle<mirror::String> handle) { 8394 jit_string_roots_.Overwrite(StringReference(&dex_file, string_index), 8395 reinterpret_cast64<uint64_t>(handle.GetReference())); 8396 return jit_string_patches_.GetOrCreate( 8397 StringReference(&dex_file, string_index), 8398 [this]() { return __ NewLiteral<uint32_t>(/* placeholder */ 0u); }); 8399} 8400 8401Literal* CodeGeneratorARM::DeduplicateJitClassLiteral(const DexFile& dex_file, 8402 dex::TypeIndex type_index, 8403 Handle<mirror::Class> handle) { 8404 jit_class_roots_.Overwrite(TypeReference(&dex_file, type_index), 8405 reinterpret_cast64<uint64_t>(handle.GetReference())); 8406 return jit_class_patches_.GetOrCreate( 8407 TypeReference(&dex_file, type_index), 8408 [this]() { return __ NewLiteral<uint32_t>(/* placeholder */ 0u); }); 8409} 8410 8411template <LinkerPatch (*Factory)(size_t, const DexFile*, uint32_t, uint32_t)> 8412inline void CodeGeneratorARM::EmitPcRelativeLinkerPatches( 8413 const ArenaDeque<PcRelativePatchInfo>& infos, 8414 ArenaVector<LinkerPatch>* linker_patches) { 8415 for (const PcRelativePatchInfo& info : infos) { 8416 const DexFile& dex_file = info.target_dex_file; 8417 size_t offset_or_index = info.offset_or_index; 8418 DCHECK(info.add_pc_label.IsBound()); 8419 uint32_t add_pc_offset = dchecked_integral_cast<uint32_t>(info.add_pc_label.Position()); 8420 // Add MOVW patch. 8421 DCHECK(info.movw_label.IsBound()); 8422 uint32_t movw_offset = dchecked_integral_cast<uint32_t>(info.movw_label.Position()); 8423 linker_patches->push_back(Factory(movw_offset, &dex_file, add_pc_offset, offset_or_index)); 8424 // Add MOVT patch. 8425 DCHECK(info.movt_label.IsBound()); 8426 uint32_t movt_offset = dchecked_integral_cast<uint32_t>(info.movt_label.Position()); 8427 linker_patches->push_back(Factory(movt_offset, &dex_file, add_pc_offset, offset_or_index)); 8428 } 8429} 8430 8431void CodeGeneratorARM::EmitLinkerPatches(ArenaVector<LinkerPatch>* linker_patches) { 8432 DCHECK(linker_patches->empty()); 8433 size_t size = 8434 /* MOVW+MOVT for each entry */ 2u * pc_relative_dex_cache_patches_.size() + 8435 boot_image_string_patches_.size() + 8436 /* MOVW+MOVT for each entry */ 2u * pc_relative_string_patches_.size() + 8437 boot_image_type_patches_.size() + 8438 /* MOVW+MOVT for each entry */ 2u * pc_relative_type_patches_.size() + 8439 /* MOVW+MOVT for each entry */ 2u * type_bss_entry_patches_.size(); 8440 linker_patches->reserve(size); 8441 EmitPcRelativeLinkerPatches<LinkerPatch::DexCacheArrayPatch>(pc_relative_dex_cache_patches_, 8442 linker_patches); 8443 for (const auto& entry : boot_image_string_patches_) { 8444 const StringReference& target_string = entry.first; 8445 Literal* literal = entry.second; 8446 DCHECK(literal->GetLabel()->IsBound()); 8447 uint32_t literal_offset = literal->GetLabel()->Position(); 8448 linker_patches->push_back(LinkerPatch::StringPatch(literal_offset, 8449 target_string.dex_file, 8450 target_string.string_index.index_)); 8451 } 8452 if (!GetCompilerOptions().IsBootImage()) { 8453 DCHECK(pc_relative_type_patches_.empty()); 8454 EmitPcRelativeLinkerPatches<LinkerPatch::StringBssEntryPatch>(pc_relative_string_patches_, 8455 linker_patches); 8456 } else { 8457 EmitPcRelativeLinkerPatches<LinkerPatch::RelativeTypePatch>(pc_relative_type_patches_, 8458 linker_patches); 8459 EmitPcRelativeLinkerPatches<LinkerPatch::RelativeStringPatch>(pc_relative_string_patches_, 8460 linker_patches); 8461 } 8462 EmitPcRelativeLinkerPatches<LinkerPatch::TypeBssEntryPatch>(type_bss_entry_patches_, 8463 linker_patches); 8464 for (const auto& entry : boot_image_type_patches_) { 8465 const TypeReference& target_type = entry.first; 8466 Literal* literal = entry.second; 8467 DCHECK(literal->GetLabel()->IsBound()); 8468 uint32_t literal_offset = literal->GetLabel()->Position(); 8469 linker_patches->push_back(LinkerPatch::TypePatch(literal_offset, 8470 target_type.dex_file, 8471 target_type.type_index.index_)); 8472 } 8473 DCHECK_EQ(size, linker_patches->size()); 8474} 8475 8476Literal* CodeGeneratorARM::DeduplicateUint32Literal(uint32_t value, Uint32ToLiteralMap* map) { 8477 return map->GetOrCreate( 8478 value, 8479 [this, value]() { return __ NewLiteral<uint32_t>(value); }); 8480} 8481 8482Literal* CodeGeneratorARM::DeduplicateMethodLiteral(MethodReference target_method, 8483 MethodToLiteralMap* map) { 8484 return map->GetOrCreate( 8485 target_method, 8486 [this]() { return __ NewLiteral<uint32_t>(/* placeholder */ 0u); }); 8487} 8488 8489void LocationsBuilderARM::VisitMultiplyAccumulate(HMultiplyAccumulate* instr) { 8490 LocationSummary* locations = 8491 new (GetGraph()->GetArena()) LocationSummary(instr, LocationSummary::kNoCall); 8492 locations->SetInAt(HMultiplyAccumulate::kInputAccumulatorIndex, 8493 Location::RequiresRegister()); 8494 locations->SetInAt(HMultiplyAccumulate::kInputMulLeftIndex, Location::RequiresRegister()); 8495 locations->SetInAt(HMultiplyAccumulate::kInputMulRightIndex, Location::RequiresRegister()); 8496 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 8497} 8498 8499void InstructionCodeGeneratorARM::VisitMultiplyAccumulate(HMultiplyAccumulate* instr) { 8500 LocationSummary* locations = instr->GetLocations(); 8501 Register res = locations->Out().AsRegister<Register>(); 8502 Register accumulator = 8503 locations->InAt(HMultiplyAccumulate::kInputAccumulatorIndex).AsRegister<Register>(); 8504 Register mul_left = 8505 locations->InAt(HMultiplyAccumulate::kInputMulLeftIndex).AsRegister<Register>(); 8506 Register mul_right = 8507 locations->InAt(HMultiplyAccumulate::kInputMulRightIndex).AsRegister<Register>(); 8508 8509 if (instr->GetOpKind() == HInstruction::kAdd) { 8510 __ mla(res, mul_left, mul_right, accumulator); 8511 } else { 8512 __ mls(res, mul_left, mul_right, accumulator); 8513 } 8514} 8515 8516void LocationsBuilderARM::VisitBoundType(HBoundType* instruction ATTRIBUTE_UNUSED) { 8517 // Nothing to do, this should be removed during prepare for register allocator. 8518 LOG(FATAL) << "Unreachable"; 8519} 8520 8521void InstructionCodeGeneratorARM::VisitBoundType(HBoundType* instruction ATTRIBUTE_UNUSED) { 8522 // Nothing to do, this should be removed during prepare for register allocator. 8523 LOG(FATAL) << "Unreachable"; 8524} 8525 8526// Simple implementation of packed switch - generate cascaded compare/jumps. 8527void LocationsBuilderARM::VisitPackedSwitch(HPackedSwitch* switch_instr) { 8528 LocationSummary* locations = 8529 new (GetGraph()->GetArena()) LocationSummary(switch_instr, LocationSummary::kNoCall); 8530 locations->SetInAt(0, Location::RequiresRegister()); 8531 if (switch_instr->GetNumEntries() > kPackedSwitchCompareJumpThreshold && 8532 codegen_->GetAssembler()->IsThumb()) { 8533 locations->AddTemp(Location::RequiresRegister()); // We need a temp for the table base. 8534 if (switch_instr->GetStartValue() != 0) { 8535 locations->AddTemp(Location::RequiresRegister()); // We need a temp for the bias. 8536 } 8537 } 8538} 8539 8540void InstructionCodeGeneratorARM::VisitPackedSwitch(HPackedSwitch* switch_instr) { 8541 int32_t lower_bound = switch_instr->GetStartValue(); 8542 uint32_t num_entries = switch_instr->GetNumEntries(); 8543 LocationSummary* locations = switch_instr->GetLocations(); 8544 Register value_reg = locations->InAt(0).AsRegister<Register>(); 8545 HBasicBlock* default_block = switch_instr->GetDefaultBlock(); 8546 8547 if (num_entries <= kPackedSwitchCompareJumpThreshold || !codegen_->GetAssembler()->IsThumb()) { 8548 // Create a series of compare/jumps. 8549 Register temp_reg = IP; 8550 // Note: It is fine for the below AddConstantSetFlags() using IP register to temporarily store 8551 // the immediate, because IP is used as the destination register. For the other 8552 // AddConstantSetFlags() and GenerateCompareWithImmediate(), the immediate values are constant, 8553 // and they can be encoded in the instruction without making use of IP register. 8554 __ AddConstantSetFlags(temp_reg, value_reg, -lower_bound); 8555 8556 const ArenaVector<HBasicBlock*>& successors = switch_instr->GetBlock()->GetSuccessors(); 8557 // Jump to successors[0] if value == lower_bound. 8558 __ b(codegen_->GetLabelOf(successors[0]), EQ); 8559 int32_t last_index = 0; 8560 for (; num_entries - last_index > 2; last_index += 2) { 8561 __ AddConstantSetFlags(temp_reg, temp_reg, -2); 8562 // Jump to successors[last_index + 1] if value < case_value[last_index + 2]. 8563 __ b(codegen_->GetLabelOf(successors[last_index + 1]), LO); 8564 // Jump to successors[last_index + 2] if value == case_value[last_index + 2]. 8565 __ b(codegen_->GetLabelOf(successors[last_index + 2]), EQ); 8566 } 8567 if (num_entries - last_index == 2) { 8568 // The last missing case_value. 8569 __ CmpConstant(temp_reg, 1); 8570 __ b(codegen_->GetLabelOf(successors[last_index + 1]), EQ); 8571 } 8572 8573 // And the default for any other value. 8574 if (!codegen_->GoesToNextBlock(switch_instr->GetBlock(), default_block)) { 8575 __ b(codegen_->GetLabelOf(default_block)); 8576 } 8577 } else { 8578 // Create a table lookup. 8579 Register temp_reg = locations->GetTemp(0).AsRegister<Register>(); 8580 8581 // Materialize a pointer to the switch table 8582 std::vector<Label*> labels(num_entries); 8583 const ArenaVector<HBasicBlock*>& successors = switch_instr->GetBlock()->GetSuccessors(); 8584 for (uint32_t i = 0; i < num_entries; i++) { 8585 labels[i] = codegen_->GetLabelOf(successors[i]); 8586 } 8587 JumpTable* table = __ CreateJumpTable(std::move(labels), temp_reg); 8588 8589 // Remove the bias. 8590 Register key_reg; 8591 if (lower_bound != 0) { 8592 key_reg = locations->GetTemp(1).AsRegister<Register>(); 8593 __ AddConstant(key_reg, value_reg, -lower_bound); 8594 } else { 8595 key_reg = value_reg; 8596 } 8597 8598 // Check whether the value is in the table, jump to default block if not. 8599 __ CmpConstant(key_reg, num_entries - 1); 8600 __ b(codegen_->GetLabelOf(default_block), Condition::HI); 8601 8602 // Load the displacement from the table. 8603 __ ldr(temp_reg, Address(temp_reg, key_reg, Shift::LSL, 2)); 8604 8605 // Dispatch is a direct add to the PC (for Thumb2). 8606 __ EmitJumpTableDispatch(table, temp_reg); 8607 } 8608} 8609 8610void LocationsBuilderARM::VisitArmDexCacheArraysBase(HArmDexCacheArraysBase* base) { 8611 LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(base); 8612 locations->SetOut(Location::RequiresRegister()); 8613} 8614 8615void InstructionCodeGeneratorARM::VisitArmDexCacheArraysBase(HArmDexCacheArraysBase* base) { 8616 Register base_reg = base->GetLocations()->Out().AsRegister<Register>(); 8617 CodeGeneratorARM::PcRelativePatchInfo* labels = 8618 codegen_->NewPcRelativeDexCacheArrayPatch(base->GetDexFile(), base->GetElementOffset()); 8619 __ BindTrackedLabel(&labels->movw_label); 8620 __ movw(base_reg, /* placeholder */ 0u); 8621 __ BindTrackedLabel(&labels->movt_label); 8622 __ movt(base_reg, /* placeholder */ 0u); 8623 __ BindTrackedLabel(&labels->add_pc_label); 8624 __ add(base_reg, base_reg, ShifterOperand(PC)); 8625} 8626 8627void CodeGeneratorARM::MoveFromReturnRegister(Location trg, Primitive::Type type) { 8628 if (!trg.IsValid()) { 8629 DCHECK_EQ(type, Primitive::kPrimVoid); 8630 return; 8631 } 8632 8633 DCHECK_NE(type, Primitive::kPrimVoid); 8634 8635 Location return_loc = InvokeDexCallingConventionVisitorARM().GetReturnLocation(type); 8636 if (return_loc.Equals(trg)) { 8637 return; 8638 } 8639 8640 // TODO: Consider pairs in the parallel move resolver, then this could be nicely merged 8641 // with the last branch. 8642 if (type == Primitive::kPrimLong) { 8643 HParallelMove parallel_move(GetGraph()->GetArena()); 8644 parallel_move.AddMove(return_loc.ToLow(), trg.ToLow(), Primitive::kPrimInt, nullptr); 8645 parallel_move.AddMove(return_loc.ToHigh(), trg.ToHigh(), Primitive::kPrimInt, nullptr); 8646 GetMoveResolver()->EmitNativeCode(¶llel_move); 8647 } else if (type == Primitive::kPrimDouble) { 8648 HParallelMove parallel_move(GetGraph()->GetArena()); 8649 parallel_move.AddMove(return_loc.ToLow(), trg.ToLow(), Primitive::kPrimFloat, nullptr); 8650 parallel_move.AddMove(return_loc.ToHigh(), trg.ToHigh(), Primitive::kPrimFloat, nullptr); 8651 GetMoveResolver()->EmitNativeCode(¶llel_move); 8652 } else { 8653 // Let the parallel move resolver take care of all of this. 8654 HParallelMove parallel_move(GetGraph()->GetArena()); 8655 parallel_move.AddMove(return_loc, trg, type, nullptr); 8656 GetMoveResolver()->EmitNativeCode(¶llel_move); 8657 } 8658} 8659 8660void LocationsBuilderARM::VisitClassTableGet(HClassTableGet* instruction) { 8661 LocationSummary* locations = 8662 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall); 8663 locations->SetInAt(0, Location::RequiresRegister()); 8664 locations->SetOut(Location::RequiresRegister()); 8665} 8666 8667void InstructionCodeGeneratorARM::VisitClassTableGet(HClassTableGet* instruction) { 8668 LocationSummary* locations = instruction->GetLocations(); 8669 if (instruction->GetTableKind() == HClassTableGet::TableKind::kVTable) { 8670 uint32_t method_offset = mirror::Class::EmbeddedVTableEntryOffset( 8671 instruction->GetIndex(), kArmPointerSize).SizeValue(); 8672 __ LoadFromOffset(kLoadWord, 8673 locations->Out().AsRegister<Register>(), 8674 locations->InAt(0).AsRegister<Register>(), 8675 method_offset); 8676 } else { 8677 uint32_t method_offset = static_cast<uint32_t>(ImTable::OffsetOfElement( 8678 instruction->GetIndex(), kArmPointerSize)); 8679 __ LoadFromOffset(kLoadWord, 8680 locations->Out().AsRegister<Register>(), 8681 locations->InAt(0).AsRegister<Register>(), 8682 mirror::Class::ImtPtrOffset(kArmPointerSize).Uint32Value()); 8683 __ LoadFromOffset(kLoadWord, 8684 locations->Out().AsRegister<Register>(), 8685 locations->Out().AsRegister<Register>(), 8686 method_offset); 8687 } 8688} 8689 8690static void PatchJitRootUse(uint8_t* code, 8691 const uint8_t* roots_data, 8692 Literal* literal, 8693 uint64_t index_in_table) { 8694 DCHECK(literal->GetLabel()->IsBound()); 8695 uint32_t literal_offset = literal->GetLabel()->Position(); 8696 uintptr_t address = 8697 reinterpret_cast<uintptr_t>(roots_data) + index_in_table * sizeof(GcRoot<mirror::Object>); 8698 uint8_t* data = code + literal_offset; 8699 reinterpret_cast<uint32_t*>(data)[0] = dchecked_integral_cast<uint32_t>(address); 8700} 8701 8702void CodeGeneratorARM::EmitJitRootPatches(uint8_t* code, const uint8_t* roots_data) { 8703 for (const auto& entry : jit_string_patches_) { 8704 const auto& it = jit_string_roots_.find(entry.first); 8705 DCHECK(it != jit_string_roots_.end()); 8706 PatchJitRootUse(code, roots_data, entry.second, it->second); 8707 } 8708 for (const auto& entry : jit_class_patches_) { 8709 const auto& it = jit_class_roots_.find(entry.first); 8710 DCHECK(it != jit_class_roots_.end()); 8711 PatchJitRootUse(code, roots_data, entry.second, it->second); 8712 } 8713} 8714 8715#undef __ 8716#undef QUICK_ENTRY_POINT 8717 8718} // namespace arm 8719} // namespace art 8720