code_generator_arm.cc revision 2dd053d61c3971fa5b5e179e0a2b5368409c9ba3
1/* 2 * Copyright (C) 2014 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17#include "code_generator_arm.h" 18 19#include "arch/arm/instruction_set_features_arm.h" 20#include "art_method.h" 21#include "code_generator_utils.h" 22#include "common_arm.h" 23#include "compiled_method.h" 24#include "entrypoints/quick/quick_entrypoints.h" 25#include "gc/accounting/card_table.h" 26#include "intrinsics.h" 27#include "intrinsics_arm.h" 28#include "mirror/array-inl.h" 29#include "mirror/class-inl.h" 30#include "thread.h" 31#include "utils/arm/assembler_arm.h" 32#include "utils/arm/managed_register_arm.h" 33#include "utils/assembler.h" 34#include "utils/stack_checks.h" 35 36namespace art { 37 38template<class MirrorType> 39class GcRoot; 40 41namespace arm { 42 43static bool ExpectedPairLayout(Location location) { 44 // We expected this for both core and fpu register pairs. 45 return ((location.low() & 1) == 0) && (location.low() + 1 == location.high()); 46} 47 48static constexpr int kCurrentMethodStackOffset = 0; 49static constexpr Register kMethodRegisterArgument = R0; 50 51static constexpr Register kCoreAlwaysSpillRegister = R5; 52static constexpr Register kCoreCalleeSaves[] = 53 { R5, R6, R7, R8, R10, R11, LR }; 54static constexpr SRegister kFpuCalleeSaves[] = 55 { S16, S17, S18, S19, S20, S21, S22, S23, S24, S25, S26, S27, S28, S29, S30, S31 }; 56 57// D31 cannot be split into two S registers, and the register allocator only works on 58// S registers. Therefore there is no need to block it. 59static constexpr DRegister DTMP = D31; 60 61static constexpr uint32_t kPackedSwitchCompareJumpThreshold = 7; 62 63// NOLINT on __ macro to suppress wrong warning/fix (misc-macro-parentheses) from clang-tidy. 64#define __ down_cast<ArmAssembler*>(codegen->GetAssembler())-> // NOLINT 65#define QUICK_ENTRY_POINT(x) QUICK_ENTRYPOINT_OFFSET(kArmPointerSize, x).Int32Value() 66 67static constexpr int kRegListThreshold = 4; 68 69// SaveLiveRegisters and RestoreLiveRegisters from SlowPathCodeARM operate on sets of S registers, 70// for each live D registers they treat two corresponding S registers as live ones. 71// 72// Two following functions (SaveContiguousSRegisterList, RestoreContiguousSRegisterList) build 73// from a list of contiguous S registers a list of contiguous D registers (processing first/last 74// S registers corner cases) and save/restore this new list treating them as D registers. 75// - decreasing code size 76// - avoiding hazards on Cortex-A57, when a pair of S registers for an actual live D register is 77// restored and then used in regular non SlowPath code as D register. 78// 79// For the following example (v means the S register is live): 80// D names: | D0 | D1 | D2 | D4 | ... 81// S names: | S0 | S1 | S2 | S3 | S4 | S5 | S6 | S7 | ... 82// Live? | | v | v | v | v | v | v | | ... 83// 84// S1 and S6 will be saved/restored independently; D registers list (D1, D2) will be processed 85// as D registers. 86static size_t SaveContiguousSRegisterList(size_t first, 87 size_t last, 88 CodeGenerator* codegen, 89 size_t stack_offset) { 90 DCHECK_LE(first, last); 91 if ((first == last) && (first == 0)) { 92 stack_offset += codegen->SaveFloatingPointRegister(stack_offset, first); 93 return stack_offset; 94 } 95 if (first % 2 == 1) { 96 stack_offset += codegen->SaveFloatingPointRegister(stack_offset, first++); 97 } 98 99 bool save_last = false; 100 if (last % 2 == 0) { 101 save_last = true; 102 --last; 103 } 104 105 if (first < last) { 106 DRegister d_reg = static_cast<DRegister>(first / 2); 107 DCHECK_EQ((last - first + 1) % 2, 0u); 108 size_t number_of_d_regs = (last - first + 1) / 2; 109 110 if (number_of_d_regs == 1) { 111 __ StoreDToOffset(d_reg, SP, stack_offset); 112 } else if (number_of_d_regs > 1) { 113 __ add(IP, SP, ShifterOperand(stack_offset)); 114 __ vstmiad(IP, d_reg, number_of_d_regs); 115 } 116 stack_offset += number_of_d_regs * kArmWordSize * 2; 117 } 118 119 if (save_last) { 120 stack_offset += codegen->SaveFloatingPointRegister(stack_offset, last + 1); 121 } 122 123 return stack_offset; 124} 125 126static size_t RestoreContiguousSRegisterList(size_t first, 127 size_t last, 128 CodeGenerator* codegen, 129 size_t stack_offset) { 130 DCHECK_LE(first, last); 131 if ((first == last) && (first == 0)) { 132 stack_offset += codegen->RestoreFloatingPointRegister(stack_offset, first); 133 return stack_offset; 134 } 135 if (first % 2 == 1) { 136 stack_offset += codegen->RestoreFloatingPointRegister(stack_offset, first++); 137 } 138 139 bool restore_last = false; 140 if (last % 2 == 0) { 141 restore_last = true; 142 --last; 143 } 144 145 if (first < last) { 146 DRegister d_reg = static_cast<DRegister>(first / 2); 147 DCHECK_EQ((last - first + 1) % 2, 0u); 148 size_t number_of_d_regs = (last - first + 1) / 2; 149 if (number_of_d_regs == 1) { 150 __ LoadDFromOffset(d_reg, SP, stack_offset); 151 } else if (number_of_d_regs > 1) { 152 __ add(IP, SP, ShifterOperand(stack_offset)); 153 __ vldmiad(IP, d_reg, number_of_d_regs); 154 } 155 stack_offset += number_of_d_regs * kArmWordSize * 2; 156 } 157 158 if (restore_last) { 159 stack_offset += codegen->RestoreFloatingPointRegister(stack_offset, last + 1); 160 } 161 162 return stack_offset; 163} 164 165void SlowPathCodeARM::SaveLiveRegisters(CodeGenerator* codegen, LocationSummary* locations) { 166 size_t stack_offset = codegen->GetFirstRegisterSlotInSlowPath(); 167 size_t orig_offset = stack_offset; 168 169 const uint32_t core_spills = codegen->GetSlowPathSpills(locations, /* core_registers */ true); 170 for (uint32_t i : LowToHighBits(core_spills)) { 171 // If the register holds an object, update the stack mask. 172 if (locations->RegisterContainsObject(i)) { 173 locations->SetStackBit(stack_offset / kVRegSize); 174 } 175 DCHECK_LT(stack_offset, codegen->GetFrameSize() - codegen->FrameEntrySpillSize()); 176 DCHECK_LT(i, kMaximumNumberOfExpectedRegisters); 177 saved_core_stack_offsets_[i] = stack_offset; 178 stack_offset += kArmWordSize; 179 } 180 181 int reg_num = POPCOUNT(core_spills); 182 if (reg_num != 0) { 183 if (reg_num > kRegListThreshold) { 184 __ StoreList(RegList(core_spills), orig_offset); 185 } else { 186 stack_offset = orig_offset; 187 for (uint32_t i : LowToHighBits(core_spills)) { 188 stack_offset += codegen->SaveCoreRegister(stack_offset, i); 189 } 190 } 191 } 192 193 uint32_t fp_spills = codegen->GetSlowPathSpills(locations, /* core_registers */ false); 194 orig_offset = stack_offset; 195 for (uint32_t i : LowToHighBits(fp_spills)) { 196 DCHECK_LT(i, kMaximumNumberOfExpectedRegisters); 197 saved_fpu_stack_offsets_[i] = stack_offset; 198 stack_offset += kArmWordSize; 199 } 200 201 stack_offset = orig_offset; 202 while (fp_spills != 0u) { 203 uint32_t begin = CTZ(fp_spills); 204 uint32_t tmp = fp_spills + (1u << begin); 205 fp_spills &= tmp; // Clear the contiguous range of 1s. 206 uint32_t end = (tmp == 0u) ? 32u : CTZ(tmp); // CTZ(0) is undefined. 207 stack_offset = SaveContiguousSRegisterList(begin, end - 1, codegen, stack_offset); 208 } 209 DCHECK_LE(stack_offset, codegen->GetFrameSize() - codegen->FrameEntrySpillSize()); 210} 211 212void SlowPathCodeARM::RestoreLiveRegisters(CodeGenerator* codegen, LocationSummary* locations) { 213 size_t stack_offset = codegen->GetFirstRegisterSlotInSlowPath(); 214 size_t orig_offset = stack_offset; 215 216 const uint32_t core_spills = codegen->GetSlowPathSpills(locations, /* core_registers */ true); 217 for (uint32_t i : LowToHighBits(core_spills)) { 218 DCHECK_LT(stack_offset, codegen->GetFrameSize() - codegen->FrameEntrySpillSize()); 219 DCHECK_LT(i, kMaximumNumberOfExpectedRegisters); 220 stack_offset += kArmWordSize; 221 } 222 223 int reg_num = POPCOUNT(core_spills); 224 if (reg_num != 0) { 225 if (reg_num > kRegListThreshold) { 226 __ LoadList(RegList(core_spills), orig_offset); 227 } else { 228 stack_offset = orig_offset; 229 for (uint32_t i : LowToHighBits(core_spills)) { 230 stack_offset += codegen->RestoreCoreRegister(stack_offset, i); 231 } 232 } 233 } 234 235 uint32_t fp_spills = codegen->GetSlowPathSpills(locations, /* core_registers */ false); 236 while (fp_spills != 0u) { 237 uint32_t begin = CTZ(fp_spills); 238 uint32_t tmp = fp_spills + (1u << begin); 239 fp_spills &= tmp; // Clear the contiguous range of 1s. 240 uint32_t end = (tmp == 0u) ? 32u : CTZ(tmp); // CTZ(0) is undefined. 241 stack_offset = RestoreContiguousSRegisterList(begin, end - 1, codegen, stack_offset); 242 } 243 DCHECK_LE(stack_offset, codegen->GetFrameSize() - codegen->FrameEntrySpillSize()); 244} 245 246class NullCheckSlowPathARM : public SlowPathCodeARM { 247 public: 248 explicit NullCheckSlowPathARM(HNullCheck* instruction) : SlowPathCodeARM(instruction) {} 249 250 void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { 251 CodeGeneratorARM* arm_codegen = down_cast<CodeGeneratorARM*>(codegen); 252 __ Bind(GetEntryLabel()); 253 if (instruction_->CanThrowIntoCatchBlock()) { 254 // Live registers will be restored in the catch block if caught. 255 SaveLiveRegisters(codegen, instruction_->GetLocations()); 256 } 257 arm_codegen->InvokeRuntime(kQuickThrowNullPointer, 258 instruction_, 259 instruction_->GetDexPc(), 260 this); 261 CheckEntrypointTypes<kQuickThrowNullPointer, void, void>(); 262 } 263 264 bool IsFatal() const OVERRIDE { return true; } 265 266 const char* GetDescription() const OVERRIDE { return "NullCheckSlowPathARM"; } 267 268 private: 269 DISALLOW_COPY_AND_ASSIGN(NullCheckSlowPathARM); 270}; 271 272class DivZeroCheckSlowPathARM : public SlowPathCodeARM { 273 public: 274 explicit DivZeroCheckSlowPathARM(HDivZeroCheck* instruction) : SlowPathCodeARM(instruction) {} 275 276 void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { 277 CodeGeneratorARM* arm_codegen = down_cast<CodeGeneratorARM*>(codegen); 278 __ Bind(GetEntryLabel()); 279 arm_codegen->InvokeRuntime(kQuickThrowDivZero, instruction_, instruction_->GetDexPc(), this); 280 CheckEntrypointTypes<kQuickThrowDivZero, void, void>(); 281 } 282 283 bool IsFatal() const OVERRIDE { return true; } 284 285 const char* GetDescription() const OVERRIDE { return "DivZeroCheckSlowPathARM"; } 286 287 private: 288 DISALLOW_COPY_AND_ASSIGN(DivZeroCheckSlowPathARM); 289}; 290 291class SuspendCheckSlowPathARM : public SlowPathCodeARM { 292 public: 293 SuspendCheckSlowPathARM(HSuspendCheck* instruction, HBasicBlock* successor) 294 : SlowPathCodeARM(instruction), successor_(successor) {} 295 296 void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { 297 CodeGeneratorARM* arm_codegen = down_cast<CodeGeneratorARM*>(codegen); 298 __ Bind(GetEntryLabel()); 299 arm_codegen->InvokeRuntime(kQuickTestSuspend, instruction_, instruction_->GetDexPc(), this); 300 CheckEntrypointTypes<kQuickTestSuspend, void, void>(); 301 if (successor_ == nullptr) { 302 __ b(GetReturnLabel()); 303 } else { 304 __ b(arm_codegen->GetLabelOf(successor_)); 305 } 306 } 307 308 Label* GetReturnLabel() { 309 DCHECK(successor_ == nullptr); 310 return &return_label_; 311 } 312 313 HBasicBlock* GetSuccessor() const { 314 return successor_; 315 } 316 317 const char* GetDescription() const OVERRIDE { return "SuspendCheckSlowPathARM"; } 318 319 private: 320 // If not null, the block to branch to after the suspend check. 321 HBasicBlock* const successor_; 322 323 // If `successor_` is null, the label to branch to after the suspend check. 324 Label return_label_; 325 326 DISALLOW_COPY_AND_ASSIGN(SuspendCheckSlowPathARM); 327}; 328 329class BoundsCheckSlowPathARM : public SlowPathCodeARM { 330 public: 331 explicit BoundsCheckSlowPathARM(HBoundsCheck* instruction) 332 : SlowPathCodeARM(instruction) {} 333 334 void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { 335 CodeGeneratorARM* arm_codegen = down_cast<CodeGeneratorARM*>(codegen); 336 LocationSummary* locations = instruction_->GetLocations(); 337 338 __ Bind(GetEntryLabel()); 339 if (instruction_->CanThrowIntoCatchBlock()) { 340 // Live registers will be restored in the catch block if caught. 341 SaveLiveRegisters(codegen, instruction_->GetLocations()); 342 } 343 // We're moving two locations to locations that could overlap, so we need a parallel 344 // move resolver. 345 InvokeRuntimeCallingConvention calling_convention; 346 codegen->EmitParallelMoves( 347 locations->InAt(0), 348 Location::RegisterLocation(calling_convention.GetRegisterAt(0)), 349 Primitive::kPrimInt, 350 locations->InAt(1), 351 Location::RegisterLocation(calling_convention.GetRegisterAt(1)), 352 Primitive::kPrimInt); 353 QuickEntrypointEnum entrypoint = instruction_->AsBoundsCheck()->IsStringCharAt() 354 ? kQuickThrowStringBounds 355 : kQuickThrowArrayBounds; 356 arm_codegen->InvokeRuntime(entrypoint, instruction_, instruction_->GetDexPc(), this); 357 CheckEntrypointTypes<kQuickThrowStringBounds, void, int32_t, int32_t>(); 358 CheckEntrypointTypes<kQuickThrowArrayBounds, void, int32_t, int32_t>(); 359 } 360 361 bool IsFatal() const OVERRIDE { return true; } 362 363 const char* GetDescription() const OVERRIDE { return "BoundsCheckSlowPathARM"; } 364 365 private: 366 DISALLOW_COPY_AND_ASSIGN(BoundsCheckSlowPathARM); 367}; 368 369class LoadClassSlowPathARM : public SlowPathCodeARM { 370 public: 371 LoadClassSlowPathARM(HLoadClass* cls, HInstruction* at, uint32_t dex_pc, bool do_clinit) 372 : SlowPathCodeARM(at), cls_(cls), dex_pc_(dex_pc), do_clinit_(do_clinit) { 373 DCHECK(at->IsLoadClass() || at->IsClinitCheck()); 374 } 375 376 void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { 377 LocationSummary* locations = instruction_->GetLocations(); 378 Location out = locations->Out(); 379 constexpr bool call_saves_everything_except_r0 = (!kUseReadBarrier || kUseBakerReadBarrier); 380 381 CodeGeneratorARM* arm_codegen = down_cast<CodeGeneratorARM*>(codegen); 382 __ Bind(GetEntryLabel()); 383 SaveLiveRegisters(codegen, locations); 384 385 InvokeRuntimeCallingConvention calling_convention; 386 // For HLoadClass/kBssEntry/kSaveEverything, make sure we preserve the address of the entry. 387 DCHECK_EQ(instruction_->IsLoadClass(), cls_ == instruction_); 388 bool is_load_class_bss_entry = 389 (cls_ == instruction_) && (cls_->GetLoadKind() == HLoadClass::LoadKind::kBssEntry); 390 Register entry_address = kNoRegister; 391 if (is_load_class_bss_entry && call_saves_everything_except_r0) { 392 Register temp = locations->GetTemp(0).AsRegister<Register>(); 393 // In the unlucky case that the `temp` is R0, we preserve the address in `out` across 394 // the kSaveEverything call. 395 bool temp_is_r0 = (temp == calling_convention.GetRegisterAt(0)); 396 entry_address = temp_is_r0 ? out.AsRegister<Register>() : temp; 397 DCHECK_NE(entry_address, calling_convention.GetRegisterAt(0)); 398 if (temp_is_r0) { 399 __ mov(entry_address, ShifterOperand(temp)); 400 } 401 } 402 dex::TypeIndex type_index = cls_->GetTypeIndex(); 403 __ LoadImmediate(calling_convention.GetRegisterAt(0), type_index.index_); 404 QuickEntrypointEnum entrypoint = do_clinit_ ? kQuickInitializeStaticStorage 405 : kQuickInitializeType; 406 arm_codegen->InvokeRuntime(entrypoint, instruction_, dex_pc_, this); 407 if (do_clinit_) { 408 CheckEntrypointTypes<kQuickInitializeStaticStorage, void*, uint32_t>(); 409 } else { 410 CheckEntrypointTypes<kQuickInitializeType, void*, uint32_t>(); 411 } 412 413 // For HLoadClass/kBssEntry, store the resolved Class to the BSS entry. 414 if (is_load_class_bss_entry) { 415 if (call_saves_everything_except_r0) { 416 // The class entry address was preserved in `entry_address` thanks to kSaveEverything. 417 __ str(R0, Address(entry_address)); 418 } else { 419 // For non-Baker read barrier, we need to re-calculate the address of the string entry. 420 Register temp = IP; 421 CodeGeneratorARM::PcRelativePatchInfo* labels = 422 arm_codegen->NewTypeBssEntryPatch(cls_->GetDexFile(), type_index); 423 __ BindTrackedLabel(&labels->movw_label); 424 __ movw(temp, /* placeholder */ 0u); 425 __ BindTrackedLabel(&labels->movt_label); 426 __ movt(temp, /* placeholder */ 0u); 427 __ BindTrackedLabel(&labels->add_pc_label); 428 __ add(temp, temp, ShifterOperand(PC)); 429 __ str(R0, Address(temp)); 430 } 431 } 432 // Move the class to the desired location. 433 if (out.IsValid()) { 434 DCHECK(out.IsRegister() && !locations->GetLiveRegisters()->ContainsCoreRegister(out.reg())); 435 arm_codegen->Move32(locations->Out(), Location::RegisterLocation(R0)); 436 } 437 RestoreLiveRegisters(codegen, locations); 438 __ b(GetExitLabel()); 439 } 440 441 const char* GetDescription() const OVERRIDE { return "LoadClassSlowPathARM"; } 442 443 private: 444 // The class this slow path will load. 445 HLoadClass* const cls_; 446 447 // The dex PC of `at_`. 448 const uint32_t dex_pc_; 449 450 // Whether to initialize the class. 451 const bool do_clinit_; 452 453 DISALLOW_COPY_AND_ASSIGN(LoadClassSlowPathARM); 454}; 455 456class LoadStringSlowPathARM : public SlowPathCodeARM { 457 public: 458 explicit LoadStringSlowPathARM(HLoadString* instruction) : SlowPathCodeARM(instruction) {} 459 460 void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { 461 DCHECK(instruction_->IsLoadString()); 462 DCHECK_EQ(instruction_->AsLoadString()->GetLoadKind(), HLoadString::LoadKind::kBssEntry); 463 LocationSummary* locations = instruction_->GetLocations(); 464 DCHECK(!locations->GetLiveRegisters()->ContainsCoreRegister(locations->Out().reg())); 465 HLoadString* load = instruction_->AsLoadString(); 466 const dex::StringIndex string_index = load->GetStringIndex(); 467 Register out = locations->Out().AsRegister<Register>(); 468 constexpr bool call_saves_everything_except_r0 = (!kUseReadBarrier || kUseBakerReadBarrier); 469 470 CodeGeneratorARM* arm_codegen = down_cast<CodeGeneratorARM*>(codegen); 471 __ Bind(GetEntryLabel()); 472 SaveLiveRegisters(codegen, locations); 473 474 InvokeRuntimeCallingConvention calling_convention; 475 // In the unlucky case that the `temp` is R0, we preserve the address in `out` across 476 // the kSaveEverything call. 477 Register entry_address = kNoRegister; 478 if (call_saves_everything_except_r0) { 479 Register temp = locations->GetTemp(0).AsRegister<Register>(); 480 bool temp_is_r0 = (temp == calling_convention.GetRegisterAt(0)); 481 entry_address = temp_is_r0 ? out : temp; 482 DCHECK_NE(entry_address, calling_convention.GetRegisterAt(0)); 483 if (temp_is_r0) { 484 __ mov(entry_address, ShifterOperand(temp)); 485 } 486 } 487 488 __ LoadImmediate(calling_convention.GetRegisterAt(0), string_index.index_); 489 arm_codegen->InvokeRuntime(kQuickResolveString, instruction_, instruction_->GetDexPc(), this); 490 CheckEntrypointTypes<kQuickResolveString, void*, uint32_t>(); 491 492 // Store the resolved String to the .bss entry. 493 if (call_saves_everything_except_r0) { 494 // The string entry address was preserved in `entry_address` thanks to kSaveEverything. 495 __ str(R0, Address(entry_address)); 496 } else { 497 // For non-Baker read barrier, we need to re-calculate the address of the string entry. 498 Register temp = IP; 499 CodeGeneratorARM::PcRelativePatchInfo* labels = 500 arm_codegen->NewPcRelativeStringPatch(load->GetDexFile(), string_index); 501 __ BindTrackedLabel(&labels->movw_label); 502 __ movw(temp, /* placeholder */ 0u); 503 __ BindTrackedLabel(&labels->movt_label); 504 __ movt(temp, /* placeholder */ 0u); 505 __ BindTrackedLabel(&labels->add_pc_label); 506 __ add(temp, temp, ShifterOperand(PC)); 507 __ str(R0, Address(temp)); 508 } 509 510 arm_codegen->Move32(locations->Out(), Location::RegisterLocation(R0)); 511 RestoreLiveRegisters(codegen, locations); 512 513 __ b(GetExitLabel()); 514 } 515 516 const char* GetDescription() const OVERRIDE { return "LoadStringSlowPathARM"; } 517 518 private: 519 DISALLOW_COPY_AND_ASSIGN(LoadStringSlowPathARM); 520}; 521 522class TypeCheckSlowPathARM : public SlowPathCodeARM { 523 public: 524 TypeCheckSlowPathARM(HInstruction* instruction, bool is_fatal) 525 : SlowPathCodeARM(instruction), is_fatal_(is_fatal) {} 526 527 void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { 528 LocationSummary* locations = instruction_->GetLocations(); 529 DCHECK(instruction_->IsCheckCast() 530 || !locations->GetLiveRegisters()->ContainsCoreRegister(locations->Out().reg())); 531 532 CodeGeneratorARM* arm_codegen = down_cast<CodeGeneratorARM*>(codegen); 533 __ Bind(GetEntryLabel()); 534 535 if (!is_fatal_) { 536 SaveLiveRegisters(codegen, locations); 537 } 538 539 // We're moving two locations to locations that could overlap, so we need a parallel 540 // move resolver. 541 InvokeRuntimeCallingConvention calling_convention; 542 codegen->EmitParallelMoves(locations->InAt(0), 543 Location::RegisterLocation(calling_convention.GetRegisterAt(0)), 544 Primitive::kPrimNot, 545 locations->InAt(1), 546 Location::RegisterLocation(calling_convention.GetRegisterAt(1)), 547 Primitive::kPrimNot); 548 if (instruction_->IsInstanceOf()) { 549 arm_codegen->InvokeRuntime(kQuickInstanceofNonTrivial, 550 instruction_, 551 instruction_->GetDexPc(), 552 this); 553 CheckEntrypointTypes<kQuickInstanceofNonTrivial, size_t, mirror::Object*, mirror::Class*>(); 554 arm_codegen->Move32(locations->Out(), Location::RegisterLocation(R0)); 555 } else { 556 DCHECK(instruction_->IsCheckCast()); 557 arm_codegen->InvokeRuntime(kQuickCheckInstanceOf, 558 instruction_, 559 instruction_->GetDexPc(), 560 this); 561 CheckEntrypointTypes<kQuickCheckInstanceOf, void, mirror::Object*, mirror::Class*>(); 562 } 563 564 if (!is_fatal_) { 565 RestoreLiveRegisters(codegen, locations); 566 __ b(GetExitLabel()); 567 } 568 } 569 570 const char* GetDescription() const OVERRIDE { return "TypeCheckSlowPathARM"; } 571 572 bool IsFatal() const OVERRIDE { return is_fatal_; } 573 574 private: 575 const bool is_fatal_; 576 577 DISALLOW_COPY_AND_ASSIGN(TypeCheckSlowPathARM); 578}; 579 580class DeoptimizationSlowPathARM : public SlowPathCodeARM { 581 public: 582 explicit DeoptimizationSlowPathARM(HDeoptimize* instruction) 583 : SlowPathCodeARM(instruction) {} 584 585 void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { 586 CodeGeneratorARM* arm_codegen = down_cast<CodeGeneratorARM*>(codegen); 587 __ Bind(GetEntryLabel()); 588 arm_codegen->InvokeRuntime(kQuickDeoptimize, instruction_, instruction_->GetDexPc(), this); 589 CheckEntrypointTypes<kQuickDeoptimize, void, void>(); 590 } 591 592 const char* GetDescription() const OVERRIDE { return "DeoptimizationSlowPathARM"; } 593 594 private: 595 DISALLOW_COPY_AND_ASSIGN(DeoptimizationSlowPathARM); 596}; 597 598class ArraySetSlowPathARM : public SlowPathCodeARM { 599 public: 600 explicit ArraySetSlowPathARM(HInstruction* instruction) : SlowPathCodeARM(instruction) {} 601 602 void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { 603 LocationSummary* locations = instruction_->GetLocations(); 604 __ Bind(GetEntryLabel()); 605 SaveLiveRegisters(codegen, locations); 606 607 InvokeRuntimeCallingConvention calling_convention; 608 HParallelMove parallel_move(codegen->GetGraph()->GetArena()); 609 parallel_move.AddMove( 610 locations->InAt(0), 611 Location::RegisterLocation(calling_convention.GetRegisterAt(0)), 612 Primitive::kPrimNot, 613 nullptr); 614 parallel_move.AddMove( 615 locations->InAt(1), 616 Location::RegisterLocation(calling_convention.GetRegisterAt(1)), 617 Primitive::kPrimInt, 618 nullptr); 619 parallel_move.AddMove( 620 locations->InAt(2), 621 Location::RegisterLocation(calling_convention.GetRegisterAt(2)), 622 Primitive::kPrimNot, 623 nullptr); 624 codegen->GetMoveResolver()->EmitNativeCode(¶llel_move); 625 626 CodeGeneratorARM* arm_codegen = down_cast<CodeGeneratorARM*>(codegen); 627 arm_codegen->InvokeRuntime(kQuickAputObject, instruction_, instruction_->GetDexPc(), this); 628 CheckEntrypointTypes<kQuickAputObject, void, mirror::Array*, int32_t, mirror::Object*>(); 629 RestoreLiveRegisters(codegen, locations); 630 __ b(GetExitLabel()); 631 } 632 633 const char* GetDescription() const OVERRIDE { return "ArraySetSlowPathARM"; } 634 635 private: 636 DISALLOW_COPY_AND_ASSIGN(ArraySetSlowPathARM); 637}; 638 639// Abstract base class for read barrier slow paths marking a reference 640// `ref`. 641// 642// Argument `entrypoint` must be a register location holding the read 643// barrier marking runtime entry point to be invoked. 644class ReadBarrierMarkSlowPathBaseARM : public SlowPathCodeARM { 645 protected: 646 ReadBarrierMarkSlowPathBaseARM(HInstruction* instruction, Location ref, Location entrypoint) 647 : SlowPathCodeARM(instruction), ref_(ref), entrypoint_(entrypoint) { 648 DCHECK(kEmitCompilerReadBarrier); 649 } 650 651 const char* GetDescription() const OVERRIDE { return "ReadBarrierMarkSlowPathBaseARM"; } 652 653 // Generate assembly code calling the read barrier marking runtime 654 // entry point (ReadBarrierMarkRegX). 655 void GenerateReadBarrierMarkRuntimeCall(CodeGenerator* codegen) { 656 Register ref_reg = ref_.AsRegister<Register>(); 657 658 // No need to save live registers; it's taken care of by the 659 // entrypoint. Also, there is no need to update the stack mask, 660 // as this runtime call will not trigger a garbage collection. 661 CodeGeneratorARM* arm_codegen = down_cast<CodeGeneratorARM*>(codegen); 662 DCHECK_NE(ref_reg, SP); 663 DCHECK_NE(ref_reg, LR); 664 DCHECK_NE(ref_reg, PC); 665 // IP is used internally by the ReadBarrierMarkRegX entry point 666 // as a temporary, it cannot be the entry point's input/output. 667 DCHECK_NE(ref_reg, IP); 668 DCHECK(0 <= ref_reg && ref_reg < kNumberOfCoreRegisters) << ref_reg; 669 // "Compact" slow path, saving two moves. 670 // 671 // Instead of using the standard runtime calling convention (input 672 // and output in R0): 673 // 674 // R0 <- ref 675 // R0 <- ReadBarrierMark(R0) 676 // ref <- R0 677 // 678 // we just use rX (the register containing `ref`) as input and output 679 // of a dedicated entrypoint: 680 // 681 // rX <- ReadBarrierMarkRegX(rX) 682 // 683 if (entrypoint_.IsValid()) { 684 arm_codegen->ValidateInvokeRuntimeWithoutRecordingPcInfo(instruction_, this); 685 __ blx(entrypoint_.AsRegister<Register>()); 686 } else { 687 // Entrypoint is not already loaded, load from the thread. 688 int32_t entry_point_offset = 689 CodeGenerator::GetReadBarrierMarkEntryPointsOffset<kArmPointerSize>(ref_reg); 690 // This runtime call does not require a stack map. 691 arm_codegen->InvokeRuntimeWithoutRecordingPcInfo(entry_point_offset, instruction_, this); 692 } 693 } 694 695 // The location (register) of the marked object reference. 696 const Location ref_; 697 698 // The location of the entrypoint if it is already loaded. 699 const Location entrypoint_; 700 701 private: 702 DISALLOW_COPY_AND_ASSIGN(ReadBarrierMarkSlowPathBaseARM); 703}; 704 705// Slow path marking an object reference `ref` during a read 706// barrier. The field `obj.field` in the object `obj` holding this 707// reference does not get updated by this slow path after marking. 708// 709// This means that after the execution of this slow path, `ref` will 710// always be up-to-date, but `obj.field` may not; i.e., after the 711// flip, `ref` will be a to-space reference, but `obj.field` will 712// probably still be a from-space reference (unless it gets updated by 713// another thread, or if another thread installed another object 714// reference (different from `ref`) in `obj.field`). 715// 716// If `entrypoint` is a valid location it is assumed to already be 717// holding the entrypoint. The case where the entrypoint is passed in 718// is when the decision to mark is based on whether the GC is marking. 719class ReadBarrierMarkSlowPathARM : public ReadBarrierMarkSlowPathBaseARM { 720 public: 721 ReadBarrierMarkSlowPathARM(HInstruction* instruction, 722 Location ref, 723 Location entrypoint = Location::NoLocation()) 724 : ReadBarrierMarkSlowPathBaseARM(instruction, ref, entrypoint) { 725 DCHECK(kEmitCompilerReadBarrier); 726 } 727 728 const char* GetDescription() const OVERRIDE { return "ReadBarrierMarkSlowPathARM"; } 729 730 void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { 731 LocationSummary* locations = instruction_->GetLocations(); 732 DCHECK(locations->CanCall()); 733 if (kIsDebugBuild) { 734 Register ref_reg = ref_.AsRegister<Register>(); 735 DCHECK(!locations->GetLiveRegisters()->ContainsCoreRegister(ref_reg)) << ref_reg; 736 } 737 DCHECK(instruction_->IsLoadClass() || instruction_->IsLoadString()) 738 << "Unexpected instruction in read barrier marking slow path: " 739 << instruction_->DebugName(); 740 741 __ Bind(GetEntryLabel()); 742 GenerateReadBarrierMarkRuntimeCall(codegen); 743 __ b(GetExitLabel()); 744 } 745 746 private: 747 DISALLOW_COPY_AND_ASSIGN(ReadBarrierMarkSlowPathARM); 748}; 749 750// Slow path loading `obj`'s lock word, loading a reference from 751// object `*(obj + offset + (index << scale_factor))` into `ref`, and 752// marking `ref` if `obj` is gray according to the lock word (Baker 753// read barrier). The field `obj.field` in the object `obj` holding 754// this reference does not get updated by this slow path after marking 755// (see LoadReferenceWithBakerReadBarrierAndUpdateFieldSlowPathARM 756// below for that). 757// 758// This means that after the execution of this slow path, `ref` will 759// always be up-to-date, but `obj.field` may not; i.e., after the 760// flip, `ref` will be a to-space reference, but `obj.field` will 761// probably still be a from-space reference (unless it gets updated by 762// another thread, or if another thread installed another object 763// reference (different from `ref`) in `obj.field`). 764// 765// Argument `entrypoint` must be a register location holding the read 766// barrier marking runtime entry point to be invoked. 767class LoadReferenceWithBakerReadBarrierSlowPathARM : public ReadBarrierMarkSlowPathBaseARM { 768 public: 769 LoadReferenceWithBakerReadBarrierSlowPathARM(HInstruction* instruction, 770 Location ref, 771 Register obj, 772 uint32_t offset, 773 Location index, 774 ScaleFactor scale_factor, 775 bool needs_null_check, 776 Register temp, 777 Location entrypoint) 778 : ReadBarrierMarkSlowPathBaseARM(instruction, ref, entrypoint), 779 obj_(obj), 780 offset_(offset), 781 index_(index), 782 scale_factor_(scale_factor), 783 needs_null_check_(needs_null_check), 784 temp_(temp) { 785 DCHECK(kEmitCompilerReadBarrier); 786 DCHECK(kUseBakerReadBarrier); 787 } 788 789 const char* GetDescription() const OVERRIDE { 790 return "LoadReferenceWithBakerReadBarrierSlowPathARM"; 791 } 792 793 void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { 794 LocationSummary* locations = instruction_->GetLocations(); 795 Register ref_reg = ref_.AsRegister<Register>(); 796 DCHECK(locations->CanCall()); 797 DCHECK(!locations->GetLiveRegisters()->ContainsCoreRegister(ref_reg)) << ref_reg; 798 DCHECK_NE(ref_reg, temp_); 799 DCHECK(instruction_->IsInstanceFieldGet() || 800 instruction_->IsStaticFieldGet() || 801 instruction_->IsArrayGet() || 802 instruction_->IsArraySet() || 803 instruction_->IsInstanceOf() || 804 instruction_->IsCheckCast() || 805 (instruction_->IsInvokeVirtual() && instruction_->GetLocations()->Intrinsified()) || 806 (instruction_->IsInvokeStaticOrDirect() && instruction_->GetLocations()->Intrinsified())) 807 << "Unexpected instruction in read barrier marking slow path: " 808 << instruction_->DebugName(); 809 // The read barrier instrumentation of object ArrayGet 810 // instructions does not support the HIntermediateAddress 811 // instruction. 812 DCHECK(!(instruction_->IsArrayGet() && 813 instruction_->AsArrayGet()->GetArray()->IsIntermediateAddress())); 814 815 __ Bind(GetEntryLabel()); 816 817 // When using MaybeGenerateReadBarrierSlow, the read barrier call is 818 // inserted after the original load. However, in fast path based 819 // Baker's read barriers, we need to perform the load of 820 // mirror::Object::monitor_ *before* the original reference load. 821 // This load-load ordering is required by the read barrier. 822 // The fast path/slow path (for Baker's algorithm) should look like: 823 // 824 // uint32_t rb_state = Lockword(obj->monitor_).ReadBarrierState(); 825 // lfence; // Load fence or artificial data dependency to prevent load-load reordering 826 // HeapReference<mirror::Object> ref = *src; // Original reference load. 827 // bool is_gray = (rb_state == ReadBarrier::GrayState()); 828 // if (is_gray) { 829 // ref = entrypoint(ref); // ref = ReadBarrier::Mark(ref); // Runtime entry point call. 830 // } 831 // 832 // Note: the original implementation in ReadBarrier::Barrier is 833 // slightly more complex as it performs additional checks that we do 834 // not do here for performance reasons. 835 836 // /* int32_t */ monitor = obj->monitor_ 837 uint32_t monitor_offset = mirror::Object::MonitorOffset().Int32Value(); 838 __ LoadFromOffset(kLoadWord, temp_, obj_, monitor_offset); 839 if (needs_null_check_) { 840 codegen->MaybeRecordImplicitNullCheck(instruction_); 841 } 842 // /* LockWord */ lock_word = LockWord(monitor) 843 static_assert(sizeof(LockWord) == sizeof(int32_t), 844 "art::LockWord and int32_t have different sizes."); 845 846 // Introduce a dependency on the lock_word including the rb_state, 847 // which shall prevent load-load reordering without using 848 // a memory barrier (which would be more expensive). 849 // `obj` is unchanged by this operation, but its value now depends 850 // on `temp`. 851 __ add(obj_, obj_, ShifterOperand(temp_, LSR, 32)); 852 853 // The actual reference load. 854 // A possible implicit null check has already been handled above. 855 CodeGeneratorARM* arm_codegen = down_cast<CodeGeneratorARM*>(codegen); 856 arm_codegen->GenerateRawReferenceLoad( 857 instruction_, ref_, obj_, offset_, index_, scale_factor_, /* needs_null_check */ false); 858 859 // Mark the object `ref` when `obj` is gray. 860 // 861 // if (rb_state == ReadBarrier::GrayState()) 862 // ref = ReadBarrier::Mark(ref); 863 // 864 // Given the numeric representation, it's enough to check the low bit of the 865 // rb_state. We do that by shifting the bit out of the lock word with LSRS 866 // which can be a 16-bit instruction unlike the TST immediate. 867 static_assert(ReadBarrier::WhiteState() == 0, "Expecting white to have value 0"); 868 static_assert(ReadBarrier::GrayState() == 1, "Expecting gray to have value 1"); 869 __ Lsrs(temp_, temp_, LockWord::kReadBarrierStateShift + 1); 870 __ b(GetExitLabel(), CC); // Carry flag is the last bit shifted out by LSRS. 871 GenerateReadBarrierMarkRuntimeCall(codegen); 872 873 __ b(GetExitLabel()); 874 } 875 876 private: 877 // The register containing the object holding the marked object reference field. 878 Register obj_; 879 // The offset, index and scale factor to access the reference in `obj_`. 880 uint32_t offset_; 881 Location index_; 882 ScaleFactor scale_factor_; 883 // Is a null check required? 884 bool needs_null_check_; 885 // A temporary register used to hold the lock word of `obj_`. 886 Register temp_; 887 888 DISALLOW_COPY_AND_ASSIGN(LoadReferenceWithBakerReadBarrierSlowPathARM); 889}; 890 891// Slow path loading `obj`'s lock word, loading a reference from 892// object `*(obj + offset + (index << scale_factor))` into `ref`, and 893// marking `ref` if `obj` is gray according to the lock word (Baker 894// read barrier). If needed, this slow path also atomically updates 895// the field `obj.field` in the object `obj` holding this reference 896// after marking (contrary to 897// LoadReferenceWithBakerReadBarrierSlowPathARM above, which never 898// tries to update `obj.field`). 899// 900// This means that after the execution of this slow path, both `ref` 901// and `obj.field` will be up-to-date; i.e., after the flip, both will 902// hold the same to-space reference (unless another thread installed 903// another object reference (different from `ref`) in `obj.field`). 904// 905// Argument `entrypoint` must be a register location holding the read 906// barrier marking runtime entry point to be invoked. 907class LoadReferenceWithBakerReadBarrierAndUpdateFieldSlowPathARM 908 : public ReadBarrierMarkSlowPathBaseARM { 909 public: 910 LoadReferenceWithBakerReadBarrierAndUpdateFieldSlowPathARM(HInstruction* instruction, 911 Location ref, 912 Register obj, 913 uint32_t offset, 914 Location index, 915 ScaleFactor scale_factor, 916 bool needs_null_check, 917 Register temp1, 918 Register temp2, 919 Location entrypoint) 920 : ReadBarrierMarkSlowPathBaseARM(instruction, ref, entrypoint), 921 obj_(obj), 922 offset_(offset), 923 index_(index), 924 scale_factor_(scale_factor), 925 needs_null_check_(needs_null_check), 926 temp1_(temp1), 927 temp2_(temp2) { 928 DCHECK(kEmitCompilerReadBarrier); 929 DCHECK(kUseBakerReadBarrier); 930 } 931 932 const char* GetDescription() const OVERRIDE { 933 return "LoadReferenceWithBakerReadBarrierAndUpdateFieldSlowPathARM"; 934 } 935 936 void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { 937 LocationSummary* locations = instruction_->GetLocations(); 938 Register ref_reg = ref_.AsRegister<Register>(); 939 DCHECK(locations->CanCall()); 940 DCHECK(!locations->GetLiveRegisters()->ContainsCoreRegister(ref_reg)) << ref_reg; 941 DCHECK_NE(ref_reg, temp1_); 942 943 // This slow path is only used by the UnsafeCASObject intrinsic at the moment. 944 DCHECK((instruction_->IsInvokeVirtual() && instruction_->GetLocations()->Intrinsified())) 945 << "Unexpected instruction in read barrier marking and field updating slow path: " 946 << instruction_->DebugName(); 947 DCHECK(instruction_->GetLocations()->Intrinsified()); 948 DCHECK_EQ(instruction_->AsInvoke()->GetIntrinsic(), Intrinsics::kUnsafeCASObject); 949 DCHECK_EQ(offset_, 0u); 950 DCHECK_EQ(scale_factor_, ScaleFactor::TIMES_1); 951 // The location of the offset of the marked reference field within `obj_`. 952 Location field_offset = index_; 953 DCHECK(field_offset.IsRegisterPair()) << field_offset; 954 955 __ Bind(GetEntryLabel()); 956 957 // /* int32_t */ monitor = obj->monitor_ 958 uint32_t monitor_offset = mirror::Object::MonitorOffset().Int32Value(); 959 __ LoadFromOffset(kLoadWord, temp1_, obj_, monitor_offset); 960 if (needs_null_check_) { 961 codegen->MaybeRecordImplicitNullCheck(instruction_); 962 } 963 // /* LockWord */ lock_word = LockWord(monitor) 964 static_assert(sizeof(LockWord) == sizeof(int32_t), 965 "art::LockWord and int32_t have different sizes."); 966 967 // Introduce a dependency on the lock_word including the rb_state, 968 // which shall prevent load-load reordering without using 969 // a memory barrier (which would be more expensive). 970 // `obj` is unchanged by this operation, but its value now depends 971 // on `temp1`. 972 __ add(obj_, obj_, ShifterOperand(temp1_, LSR, 32)); 973 974 // The actual reference load. 975 // A possible implicit null check has already been handled above. 976 CodeGeneratorARM* arm_codegen = down_cast<CodeGeneratorARM*>(codegen); 977 arm_codegen->GenerateRawReferenceLoad( 978 instruction_, ref_, obj_, offset_, index_, scale_factor_, /* needs_null_check */ false); 979 980 // Mark the object `ref` when `obj` is gray. 981 // 982 // if (rb_state == ReadBarrier::GrayState()) 983 // ref = ReadBarrier::Mark(ref); 984 // 985 // Given the numeric representation, it's enough to check the low bit of the 986 // rb_state. We do that by shifting the bit out of the lock word with LSRS 987 // which can be a 16-bit instruction unlike the TST immediate. 988 static_assert(ReadBarrier::WhiteState() == 0, "Expecting white to have value 0"); 989 static_assert(ReadBarrier::GrayState() == 1, "Expecting gray to have value 1"); 990 __ Lsrs(temp1_, temp1_, LockWord::kReadBarrierStateShift + 1); 991 __ b(GetExitLabel(), CC); // Carry flag is the last bit shifted out by LSRS. 992 993 // Save the old value of the reference before marking it. 994 // Note that we cannot use IP to save the old reference, as IP is 995 // used internally by the ReadBarrierMarkRegX entry point, and we 996 // need the old reference after the call to that entry point. 997 DCHECK_NE(temp1_, IP); 998 __ Mov(temp1_, ref_reg); 999 1000 GenerateReadBarrierMarkRuntimeCall(codegen); 1001 1002 // If the new reference is different from the old reference, 1003 // update the field in the holder (`*(obj_ + field_offset)`). 1004 // 1005 // Note that this field could also hold a different object, if 1006 // another thread had concurrently changed it. In that case, the 1007 // LDREX/SUBS/ITNE sequence of instructions in the compare-and-set 1008 // (CAS) operation below would abort the CAS, leaving the field 1009 // as-is. 1010 __ cmp(temp1_, ShifterOperand(ref_reg)); 1011 __ b(GetExitLabel(), EQ); 1012 1013 // Update the the holder's field atomically. This may fail if 1014 // mutator updates before us, but it's OK. This is achieved 1015 // using a strong compare-and-set (CAS) operation with relaxed 1016 // memory synchronization ordering, where the expected value is 1017 // the old reference and the desired value is the new reference. 1018 1019 // Convenience aliases. 1020 Register base = obj_; 1021 // The UnsafeCASObject intrinsic uses a register pair as field 1022 // offset ("long offset"), of which only the low part contains 1023 // data. 1024 Register offset = field_offset.AsRegisterPairLow<Register>(); 1025 Register expected = temp1_; 1026 Register value = ref_reg; 1027 Register tmp_ptr = IP; // Pointer to actual memory. 1028 Register tmp = temp2_; // Value in memory. 1029 1030 __ add(tmp_ptr, base, ShifterOperand(offset)); 1031 1032 if (kPoisonHeapReferences) { 1033 __ PoisonHeapReference(expected); 1034 if (value == expected) { 1035 // Do not poison `value`, as it is the same register as 1036 // `expected`, which has just been poisoned. 1037 } else { 1038 __ PoisonHeapReference(value); 1039 } 1040 } 1041 1042 // do { 1043 // tmp = [r_ptr] - expected; 1044 // } while (tmp == 0 && failure([r_ptr] <- r_new_value)); 1045 1046 Label loop_head, exit_loop; 1047 __ Bind(&loop_head); 1048 1049 __ ldrex(tmp, tmp_ptr); 1050 1051 __ subs(tmp, tmp, ShifterOperand(expected)); 1052 1053 __ it(NE); 1054 __ clrex(NE); 1055 1056 __ b(&exit_loop, NE); 1057 1058 __ strex(tmp, value, tmp_ptr); 1059 __ cmp(tmp, ShifterOperand(1)); 1060 __ b(&loop_head, EQ); 1061 1062 __ Bind(&exit_loop); 1063 1064 if (kPoisonHeapReferences) { 1065 __ UnpoisonHeapReference(expected); 1066 if (value == expected) { 1067 // Do not unpoison `value`, as it is the same register as 1068 // `expected`, which has just been unpoisoned. 1069 } else { 1070 __ UnpoisonHeapReference(value); 1071 } 1072 } 1073 1074 __ b(GetExitLabel()); 1075 } 1076 1077 private: 1078 // The register containing the object holding the marked object reference field. 1079 const Register obj_; 1080 // The offset, index and scale factor to access the reference in `obj_`. 1081 uint32_t offset_; 1082 Location index_; 1083 ScaleFactor scale_factor_; 1084 // Is a null check required? 1085 bool needs_null_check_; 1086 // A temporary register used to hold the lock word of `obj_`; and 1087 // also to hold the original reference value, when the reference is 1088 // marked. 1089 const Register temp1_; 1090 // A temporary register used in the implementation of the CAS, to 1091 // update the object's reference field. 1092 const Register temp2_; 1093 1094 DISALLOW_COPY_AND_ASSIGN(LoadReferenceWithBakerReadBarrierAndUpdateFieldSlowPathARM); 1095}; 1096 1097// Slow path generating a read barrier for a heap reference. 1098class ReadBarrierForHeapReferenceSlowPathARM : public SlowPathCodeARM { 1099 public: 1100 ReadBarrierForHeapReferenceSlowPathARM(HInstruction* instruction, 1101 Location out, 1102 Location ref, 1103 Location obj, 1104 uint32_t offset, 1105 Location index) 1106 : SlowPathCodeARM(instruction), 1107 out_(out), 1108 ref_(ref), 1109 obj_(obj), 1110 offset_(offset), 1111 index_(index) { 1112 DCHECK(kEmitCompilerReadBarrier); 1113 // If `obj` is equal to `out` or `ref`, it means the initial object 1114 // has been overwritten by (or after) the heap object reference load 1115 // to be instrumented, e.g.: 1116 // 1117 // __ LoadFromOffset(kLoadWord, out, out, offset); 1118 // codegen_->GenerateReadBarrierSlow(instruction, out_loc, out_loc, out_loc, offset); 1119 // 1120 // In that case, we have lost the information about the original 1121 // object, and the emitted read barrier cannot work properly. 1122 DCHECK(!obj.Equals(out)) << "obj=" << obj << " out=" << out; 1123 DCHECK(!obj.Equals(ref)) << "obj=" << obj << " ref=" << ref; 1124 } 1125 1126 void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { 1127 CodeGeneratorARM* arm_codegen = down_cast<CodeGeneratorARM*>(codegen); 1128 LocationSummary* locations = instruction_->GetLocations(); 1129 Register reg_out = out_.AsRegister<Register>(); 1130 DCHECK(locations->CanCall()); 1131 DCHECK(!locations->GetLiveRegisters()->ContainsCoreRegister(reg_out)); 1132 DCHECK(instruction_->IsInstanceFieldGet() || 1133 instruction_->IsStaticFieldGet() || 1134 instruction_->IsArrayGet() || 1135 instruction_->IsInstanceOf() || 1136 instruction_->IsCheckCast() || 1137 (instruction_->IsInvokeVirtual()) && instruction_->GetLocations()->Intrinsified()) 1138 << "Unexpected instruction in read barrier for heap reference slow path: " 1139 << instruction_->DebugName(); 1140 // The read barrier instrumentation of object ArrayGet 1141 // instructions does not support the HIntermediateAddress 1142 // instruction. 1143 DCHECK(!(instruction_->IsArrayGet() && 1144 instruction_->AsArrayGet()->GetArray()->IsIntermediateAddress())); 1145 1146 __ Bind(GetEntryLabel()); 1147 SaveLiveRegisters(codegen, locations); 1148 1149 // We may have to change the index's value, but as `index_` is a 1150 // constant member (like other "inputs" of this slow path), 1151 // introduce a copy of it, `index`. 1152 Location index = index_; 1153 if (index_.IsValid()) { 1154 // Handle `index_` for HArrayGet and UnsafeGetObject/UnsafeGetObjectVolatile intrinsics. 1155 if (instruction_->IsArrayGet()) { 1156 // Compute the actual memory offset and store it in `index`. 1157 Register index_reg = index_.AsRegister<Register>(); 1158 DCHECK(locations->GetLiveRegisters()->ContainsCoreRegister(index_reg)); 1159 if (codegen->IsCoreCalleeSaveRegister(index_reg)) { 1160 // We are about to change the value of `index_reg` (see the 1161 // calls to art::arm::Thumb2Assembler::Lsl and 1162 // art::arm::Thumb2Assembler::AddConstant below), but it has 1163 // not been saved by the previous call to 1164 // art::SlowPathCode::SaveLiveRegisters, as it is a 1165 // callee-save register -- 1166 // art::SlowPathCode::SaveLiveRegisters does not consider 1167 // callee-save registers, as it has been designed with the 1168 // assumption that callee-save registers are supposed to be 1169 // handled by the called function. So, as a callee-save 1170 // register, `index_reg` _would_ eventually be saved onto 1171 // the stack, but it would be too late: we would have 1172 // changed its value earlier. Therefore, we manually save 1173 // it here into another freely available register, 1174 // `free_reg`, chosen of course among the caller-save 1175 // registers (as a callee-save `free_reg` register would 1176 // exhibit the same problem). 1177 // 1178 // Note we could have requested a temporary register from 1179 // the register allocator instead; but we prefer not to, as 1180 // this is a slow path, and we know we can find a 1181 // caller-save register that is available. 1182 Register free_reg = FindAvailableCallerSaveRegister(codegen); 1183 __ Mov(free_reg, index_reg); 1184 index_reg = free_reg; 1185 index = Location::RegisterLocation(index_reg); 1186 } else { 1187 // The initial register stored in `index_` has already been 1188 // saved in the call to art::SlowPathCode::SaveLiveRegisters 1189 // (as it is not a callee-save register), so we can freely 1190 // use it. 1191 } 1192 // Shifting the index value contained in `index_reg` by the scale 1193 // factor (2) cannot overflow in practice, as the runtime is 1194 // unable to allocate object arrays with a size larger than 1195 // 2^26 - 1 (that is, 2^28 - 4 bytes). 1196 __ Lsl(index_reg, index_reg, TIMES_4); 1197 static_assert( 1198 sizeof(mirror::HeapReference<mirror::Object>) == sizeof(int32_t), 1199 "art::mirror::HeapReference<art::mirror::Object> and int32_t have different sizes."); 1200 __ AddConstant(index_reg, index_reg, offset_); 1201 } else { 1202 // In the case of the UnsafeGetObject/UnsafeGetObjectVolatile 1203 // intrinsics, `index_` is not shifted by a scale factor of 2 1204 // (as in the case of ArrayGet), as it is actually an offset 1205 // to an object field within an object. 1206 DCHECK(instruction_->IsInvoke()) << instruction_->DebugName(); 1207 DCHECK(instruction_->GetLocations()->Intrinsified()); 1208 DCHECK((instruction_->AsInvoke()->GetIntrinsic() == Intrinsics::kUnsafeGetObject) || 1209 (instruction_->AsInvoke()->GetIntrinsic() == Intrinsics::kUnsafeGetObjectVolatile)) 1210 << instruction_->AsInvoke()->GetIntrinsic(); 1211 DCHECK_EQ(offset_, 0U); 1212 DCHECK(index_.IsRegisterPair()); 1213 // UnsafeGet's offset location is a register pair, the low 1214 // part contains the correct offset. 1215 index = index_.ToLow(); 1216 } 1217 } 1218 1219 // We're moving two or three locations to locations that could 1220 // overlap, so we need a parallel move resolver. 1221 InvokeRuntimeCallingConvention calling_convention; 1222 HParallelMove parallel_move(codegen->GetGraph()->GetArena()); 1223 parallel_move.AddMove(ref_, 1224 Location::RegisterLocation(calling_convention.GetRegisterAt(0)), 1225 Primitive::kPrimNot, 1226 nullptr); 1227 parallel_move.AddMove(obj_, 1228 Location::RegisterLocation(calling_convention.GetRegisterAt(1)), 1229 Primitive::kPrimNot, 1230 nullptr); 1231 if (index.IsValid()) { 1232 parallel_move.AddMove(index, 1233 Location::RegisterLocation(calling_convention.GetRegisterAt(2)), 1234 Primitive::kPrimInt, 1235 nullptr); 1236 codegen->GetMoveResolver()->EmitNativeCode(¶llel_move); 1237 } else { 1238 codegen->GetMoveResolver()->EmitNativeCode(¶llel_move); 1239 __ LoadImmediate(calling_convention.GetRegisterAt(2), offset_); 1240 } 1241 arm_codegen->InvokeRuntime(kQuickReadBarrierSlow, instruction_, instruction_->GetDexPc(), this); 1242 CheckEntrypointTypes< 1243 kQuickReadBarrierSlow, mirror::Object*, mirror::Object*, mirror::Object*, uint32_t>(); 1244 arm_codegen->Move32(out_, Location::RegisterLocation(R0)); 1245 1246 RestoreLiveRegisters(codegen, locations); 1247 __ b(GetExitLabel()); 1248 } 1249 1250 const char* GetDescription() const OVERRIDE { return "ReadBarrierForHeapReferenceSlowPathARM"; } 1251 1252 private: 1253 Register FindAvailableCallerSaveRegister(CodeGenerator* codegen) { 1254 size_t ref = static_cast<int>(ref_.AsRegister<Register>()); 1255 size_t obj = static_cast<int>(obj_.AsRegister<Register>()); 1256 for (size_t i = 0, e = codegen->GetNumberOfCoreRegisters(); i < e; ++i) { 1257 if (i != ref && i != obj && !codegen->IsCoreCalleeSaveRegister(i)) { 1258 return static_cast<Register>(i); 1259 } 1260 } 1261 // We shall never fail to find a free caller-save register, as 1262 // there are more than two core caller-save registers on ARM 1263 // (meaning it is possible to find one which is different from 1264 // `ref` and `obj`). 1265 DCHECK_GT(codegen->GetNumberOfCoreCallerSaveRegisters(), 2u); 1266 LOG(FATAL) << "Could not find a free caller-save register"; 1267 UNREACHABLE(); 1268 } 1269 1270 const Location out_; 1271 const Location ref_; 1272 const Location obj_; 1273 const uint32_t offset_; 1274 // An additional location containing an index to an array. 1275 // Only used for HArrayGet and the UnsafeGetObject & 1276 // UnsafeGetObjectVolatile intrinsics. 1277 const Location index_; 1278 1279 DISALLOW_COPY_AND_ASSIGN(ReadBarrierForHeapReferenceSlowPathARM); 1280}; 1281 1282// Slow path generating a read barrier for a GC root. 1283class ReadBarrierForRootSlowPathARM : public SlowPathCodeARM { 1284 public: 1285 ReadBarrierForRootSlowPathARM(HInstruction* instruction, Location out, Location root) 1286 : SlowPathCodeARM(instruction), out_(out), root_(root) { 1287 DCHECK(kEmitCompilerReadBarrier); 1288 } 1289 1290 void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { 1291 LocationSummary* locations = instruction_->GetLocations(); 1292 Register reg_out = out_.AsRegister<Register>(); 1293 DCHECK(locations->CanCall()); 1294 DCHECK(!locations->GetLiveRegisters()->ContainsCoreRegister(reg_out)); 1295 DCHECK(instruction_->IsLoadClass() || instruction_->IsLoadString()) 1296 << "Unexpected instruction in read barrier for GC root slow path: " 1297 << instruction_->DebugName(); 1298 1299 __ Bind(GetEntryLabel()); 1300 SaveLiveRegisters(codegen, locations); 1301 1302 InvokeRuntimeCallingConvention calling_convention; 1303 CodeGeneratorARM* arm_codegen = down_cast<CodeGeneratorARM*>(codegen); 1304 arm_codegen->Move32(Location::RegisterLocation(calling_convention.GetRegisterAt(0)), root_); 1305 arm_codegen->InvokeRuntime(kQuickReadBarrierForRootSlow, 1306 instruction_, 1307 instruction_->GetDexPc(), 1308 this); 1309 CheckEntrypointTypes<kQuickReadBarrierForRootSlow, mirror::Object*, GcRoot<mirror::Object>*>(); 1310 arm_codegen->Move32(out_, Location::RegisterLocation(R0)); 1311 1312 RestoreLiveRegisters(codegen, locations); 1313 __ b(GetExitLabel()); 1314 } 1315 1316 const char* GetDescription() const OVERRIDE { return "ReadBarrierForRootSlowPathARM"; } 1317 1318 private: 1319 const Location out_; 1320 const Location root_; 1321 1322 DISALLOW_COPY_AND_ASSIGN(ReadBarrierForRootSlowPathARM); 1323}; 1324 1325inline Condition ARMCondition(IfCondition cond) { 1326 switch (cond) { 1327 case kCondEQ: return EQ; 1328 case kCondNE: return NE; 1329 case kCondLT: return LT; 1330 case kCondLE: return LE; 1331 case kCondGT: return GT; 1332 case kCondGE: return GE; 1333 case kCondB: return LO; 1334 case kCondBE: return LS; 1335 case kCondA: return HI; 1336 case kCondAE: return HS; 1337 } 1338 LOG(FATAL) << "Unreachable"; 1339 UNREACHABLE(); 1340} 1341 1342// Maps signed condition to unsigned condition. 1343inline Condition ARMUnsignedCondition(IfCondition cond) { 1344 switch (cond) { 1345 case kCondEQ: return EQ; 1346 case kCondNE: return NE; 1347 // Signed to unsigned. 1348 case kCondLT: return LO; 1349 case kCondLE: return LS; 1350 case kCondGT: return HI; 1351 case kCondGE: return HS; 1352 // Unsigned remain unchanged. 1353 case kCondB: return LO; 1354 case kCondBE: return LS; 1355 case kCondA: return HI; 1356 case kCondAE: return HS; 1357 } 1358 LOG(FATAL) << "Unreachable"; 1359 UNREACHABLE(); 1360} 1361 1362inline Condition ARMFPCondition(IfCondition cond, bool gt_bias) { 1363 // The ARM condition codes can express all the necessary branches, see the 1364 // "Meaning (floating-point)" column in the table A8-1 of the ARMv7 reference manual. 1365 // There is no dex instruction or HIR that would need the missing conditions 1366 // "equal or unordered" or "not equal". 1367 switch (cond) { 1368 case kCondEQ: return EQ; 1369 case kCondNE: return NE /* unordered */; 1370 case kCondLT: return gt_bias ? CC : LT /* unordered */; 1371 case kCondLE: return gt_bias ? LS : LE /* unordered */; 1372 case kCondGT: return gt_bias ? HI /* unordered */ : GT; 1373 case kCondGE: return gt_bias ? CS /* unordered */ : GE; 1374 default: 1375 LOG(FATAL) << "UNREACHABLE"; 1376 UNREACHABLE(); 1377 } 1378} 1379 1380inline Shift ShiftFromOpKind(HDataProcWithShifterOp::OpKind op_kind) { 1381 switch (op_kind) { 1382 case HDataProcWithShifterOp::kASR: return ASR; 1383 case HDataProcWithShifterOp::kLSL: return LSL; 1384 case HDataProcWithShifterOp::kLSR: return LSR; 1385 default: 1386 LOG(FATAL) << "Unexpected op kind " << op_kind; 1387 UNREACHABLE(); 1388 } 1389} 1390 1391static void GenerateDataProcInstruction(HInstruction::InstructionKind kind, 1392 Register out, 1393 Register first, 1394 const ShifterOperand& second, 1395 CodeGeneratorARM* codegen) { 1396 if (second.IsImmediate() && second.GetImmediate() == 0) { 1397 const ShifterOperand in = kind == HInstruction::kAnd 1398 ? ShifterOperand(0) 1399 : ShifterOperand(first); 1400 1401 __ mov(out, in); 1402 } else { 1403 switch (kind) { 1404 case HInstruction::kAdd: 1405 __ add(out, first, second); 1406 break; 1407 case HInstruction::kAnd: 1408 __ and_(out, first, second); 1409 break; 1410 case HInstruction::kOr: 1411 __ orr(out, first, second); 1412 break; 1413 case HInstruction::kSub: 1414 __ sub(out, first, second); 1415 break; 1416 case HInstruction::kXor: 1417 __ eor(out, first, second); 1418 break; 1419 default: 1420 LOG(FATAL) << "Unexpected instruction kind: " << kind; 1421 UNREACHABLE(); 1422 } 1423 } 1424} 1425 1426static void GenerateDataProc(HInstruction::InstructionKind kind, 1427 const Location& out, 1428 const Location& first, 1429 const ShifterOperand& second_lo, 1430 const ShifterOperand& second_hi, 1431 CodeGeneratorARM* codegen) { 1432 const Register first_hi = first.AsRegisterPairHigh<Register>(); 1433 const Register first_lo = first.AsRegisterPairLow<Register>(); 1434 const Register out_hi = out.AsRegisterPairHigh<Register>(); 1435 const Register out_lo = out.AsRegisterPairLow<Register>(); 1436 1437 if (kind == HInstruction::kAdd) { 1438 __ adds(out_lo, first_lo, second_lo); 1439 __ adc(out_hi, first_hi, second_hi); 1440 } else if (kind == HInstruction::kSub) { 1441 __ subs(out_lo, first_lo, second_lo); 1442 __ sbc(out_hi, first_hi, second_hi); 1443 } else { 1444 GenerateDataProcInstruction(kind, out_lo, first_lo, second_lo, codegen); 1445 GenerateDataProcInstruction(kind, out_hi, first_hi, second_hi, codegen); 1446 } 1447} 1448 1449static ShifterOperand GetShifterOperand(Register rm, Shift shift, uint32_t shift_imm) { 1450 return shift_imm == 0 ? ShifterOperand(rm) : ShifterOperand(rm, shift, shift_imm); 1451} 1452 1453static void GenerateLongDataProc(HDataProcWithShifterOp* instruction, CodeGeneratorARM* codegen) { 1454 DCHECK_EQ(instruction->GetType(), Primitive::kPrimLong); 1455 DCHECK(HDataProcWithShifterOp::IsShiftOp(instruction->GetOpKind())); 1456 1457 const LocationSummary* const locations = instruction->GetLocations(); 1458 const uint32_t shift_value = instruction->GetShiftAmount(); 1459 const HInstruction::InstructionKind kind = instruction->GetInstrKind(); 1460 const Location first = locations->InAt(0); 1461 const Location second = locations->InAt(1); 1462 const Location out = locations->Out(); 1463 const Register first_hi = first.AsRegisterPairHigh<Register>(); 1464 const Register first_lo = first.AsRegisterPairLow<Register>(); 1465 const Register out_hi = out.AsRegisterPairHigh<Register>(); 1466 const Register out_lo = out.AsRegisterPairLow<Register>(); 1467 const Register second_hi = second.AsRegisterPairHigh<Register>(); 1468 const Register second_lo = second.AsRegisterPairLow<Register>(); 1469 const Shift shift = ShiftFromOpKind(instruction->GetOpKind()); 1470 1471 if (shift_value >= 32) { 1472 if (shift == LSL) { 1473 GenerateDataProcInstruction(kind, 1474 out_hi, 1475 first_hi, 1476 ShifterOperand(second_lo, LSL, shift_value - 32), 1477 codegen); 1478 GenerateDataProcInstruction(kind, 1479 out_lo, 1480 first_lo, 1481 ShifterOperand(0), 1482 codegen); 1483 } else if (shift == ASR) { 1484 GenerateDataProc(kind, 1485 out, 1486 first, 1487 GetShifterOperand(second_hi, ASR, shift_value - 32), 1488 ShifterOperand(second_hi, ASR, 31), 1489 codegen); 1490 } else { 1491 DCHECK_EQ(shift, LSR); 1492 GenerateDataProc(kind, 1493 out, 1494 first, 1495 GetShifterOperand(second_hi, LSR, shift_value - 32), 1496 ShifterOperand(0), 1497 codegen); 1498 } 1499 } else { 1500 DCHECK_GT(shift_value, 1U); 1501 DCHECK_LT(shift_value, 32U); 1502 1503 if (shift == LSL) { 1504 // We are not doing this for HInstruction::kAdd because the output will require 1505 // Location::kOutputOverlap; not applicable to other cases. 1506 if (kind == HInstruction::kOr || kind == HInstruction::kXor) { 1507 GenerateDataProcInstruction(kind, 1508 out_hi, 1509 first_hi, 1510 ShifterOperand(second_hi, LSL, shift_value), 1511 codegen); 1512 GenerateDataProcInstruction(kind, 1513 out_hi, 1514 out_hi, 1515 ShifterOperand(second_lo, LSR, 32 - shift_value), 1516 codegen); 1517 GenerateDataProcInstruction(kind, 1518 out_lo, 1519 first_lo, 1520 ShifterOperand(second_lo, LSL, shift_value), 1521 codegen); 1522 } else { 1523 __ Lsl(IP, second_hi, shift_value); 1524 __ orr(IP, IP, ShifterOperand(second_lo, LSR, 32 - shift_value)); 1525 GenerateDataProc(kind, 1526 out, 1527 first, 1528 ShifterOperand(second_lo, LSL, shift_value), 1529 ShifterOperand(IP), 1530 codegen); 1531 } 1532 } else { 1533 DCHECK(shift == ASR || shift == LSR); 1534 1535 // We are not doing this for HInstruction::kAdd because the output will require 1536 // Location::kOutputOverlap; not applicable to other cases. 1537 if (kind == HInstruction::kOr || kind == HInstruction::kXor) { 1538 GenerateDataProcInstruction(kind, 1539 out_lo, 1540 first_lo, 1541 ShifterOperand(second_lo, LSR, shift_value), 1542 codegen); 1543 GenerateDataProcInstruction(kind, 1544 out_lo, 1545 out_lo, 1546 ShifterOperand(second_hi, LSL, 32 - shift_value), 1547 codegen); 1548 GenerateDataProcInstruction(kind, 1549 out_hi, 1550 first_hi, 1551 ShifterOperand(second_hi, shift, shift_value), 1552 codegen); 1553 } else { 1554 __ Lsr(IP, second_lo, shift_value); 1555 __ orr(IP, IP, ShifterOperand(second_hi, LSL, 32 - shift_value)); 1556 GenerateDataProc(kind, 1557 out, 1558 first, 1559 ShifterOperand(IP), 1560 ShifterOperand(second_hi, shift, shift_value), 1561 codegen); 1562 } 1563 } 1564 } 1565} 1566 1567static void GenerateVcmp(HInstruction* instruction, CodeGeneratorARM* codegen) { 1568 Primitive::Type type = instruction->InputAt(0)->GetType(); 1569 Location lhs_loc = instruction->GetLocations()->InAt(0); 1570 Location rhs_loc = instruction->GetLocations()->InAt(1); 1571 if (rhs_loc.IsConstant()) { 1572 // 0.0 is the only immediate that can be encoded directly in 1573 // a VCMP instruction. 1574 // 1575 // Both the JLS (section 15.20.1) and the JVMS (section 6.5) 1576 // specify that in a floating-point comparison, positive zero 1577 // and negative zero are considered equal, so we can use the 1578 // literal 0.0 for both cases here. 1579 // 1580 // Note however that some methods (Float.equal, Float.compare, 1581 // Float.compareTo, Double.equal, Double.compare, 1582 // Double.compareTo, Math.max, Math.min, StrictMath.max, 1583 // StrictMath.min) consider 0.0 to be (strictly) greater than 1584 // -0.0. So if we ever translate calls to these methods into a 1585 // HCompare instruction, we must handle the -0.0 case with 1586 // care here. 1587 DCHECK(rhs_loc.GetConstant()->IsArithmeticZero()); 1588 if (type == Primitive::kPrimFloat) { 1589 __ vcmpsz(lhs_loc.AsFpuRegister<SRegister>()); 1590 } else { 1591 DCHECK_EQ(type, Primitive::kPrimDouble); 1592 __ vcmpdz(FromLowSToD(lhs_loc.AsFpuRegisterPairLow<SRegister>())); 1593 } 1594 } else { 1595 if (type == Primitive::kPrimFloat) { 1596 __ vcmps(lhs_loc.AsFpuRegister<SRegister>(), rhs_loc.AsFpuRegister<SRegister>()); 1597 } else { 1598 DCHECK_EQ(type, Primitive::kPrimDouble); 1599 __ vcmpd(FromLowSToD(lhs_loc.AsFpuRegisterPairLow<SRegister>()), 1600 FromLowSToD(rhs_loc.AsFpuRegisterPairLow<SRegister>())); 1601 } 1602 } 1603} 1604 1605static Condition GenerateLongTestConstant(HCondition* condition, 1606 bool invert, 1607 CodeGeneratorARM* codegen) { 1608 DCHECK_EQ(condition->GetLeft()->GetType(), Primitive::kPrimLong); 1609 1610 const LocationSummary* const locations = condition->GetLocations(); 1611 IfCondition cond = invert ? condition->GetOppositeCondition() : condition->GetCondition(); 1612 Condition ret = EQ; 1613 const Location left = locations->InAt(0); 1614 const Location right = locations->InAt(1); 1615 1616 DCHECK(right.IsConstant()); 1617 1618 const Register left_high = left.AsRegisterPairHigh<Register>(); 1619 const Register left_low = left.AsRegisterPairLow<Register>(); 1620 int64_t value = right.GetConstant()->AsLongConstant()->GetValue(); 1621 1622 switch (cond) { 1623 case kCondEQ: 1624 case kCondNE: 1625 case kCondB: 1626 case kCondBE: 1627 case kCondA: 1628 case kCondAE: 1629 __ CmpConstant(left_high, High32Bits(value)); 1630 __ it(EQ); 1631 __ cmp(left_low, ShifterOperand(Low32Bits(value)), EQ); 1632 ret = ARMUnsignedCondition(cond); 1633 break; 1634 case kCondLE: 1635 case kCondGT: 1636 // Trivially true or false. 1637 if (value == std::numeric_limits<int64_t>::max()) { 1638 __ cmp(left_low, ShifterOperand(left_low)); 1639 ret = cond == kCondLE ? EQ : NE; 1640 break; 1641 } 1642 1643 if (cond == kCondLE) { 1644 cond = kCondLT; 1645 } else { 1646 DCHECK_EQ(cond, kCondGT); 1647 cond = kCondGE; 1648 } 1649 1650 value++; 1651 FALLTHROUGH_INTENDED; 1652 case kCondGE: 1653 case kCondLT: 1654 __ CmpConstant(left_low, Low32Bits(value)); 1655 __ sbcs(IP, left_high, ShifterOperand(High32Bits(value))); 1656 ret = ARMCondition(cond); 1657 break; 1658 default: 1659 LOG(FATAL) << "Unreachable"; 1660 UNREACHABLE(); 1661 } 1662 1663 return ret; 1664} 1665 1666static Condition GenerateLongTest(HCondition* condition, 1667 bool invert, 1668 CodeGeneratorARM* codegen) { 1669 DCHECK_EQ(condition->GetLeft()->GetType(), Primitive::kPrimLong); 1670 1671 const LocationSummary* const locations = condition->GetLocations(); 1672 IfCondition cond = invert ? condition->GetOppositeCondition() : condition->GetCondition(); 1673 Condition ret = EQ; 1674 Location left = locations->InAt(0); 1675 Location right = locations->InAt(1); 1676 1677 DCHECK(right.IsRegisterPair()); 1678 1679 switch (cond) { 1680 case kCondEQ: 1681 case kCondNE: 1682 case kCondB: 1683 case kCondBE: 1684 case kCondA: 1685 case kCondAE: 1686 __ cmp(left.AsRegisterPairHigh<Register>(), 1687 ShifterOperand(right.AsRegisterPairHigh<Register>())); 1688 __ it(EQ); 1689 __ cmp(left.AsRegisterPairLow<Register>(), 1690 ShifterOperand(right.AsRegisterPairLow<Register>()), 1691 EQ); 1692 ret = ARMUnsignedCondition(cond); 1693 break; 1694 case kCondLE: 1695 case kCondGT: 1696 if (cond == kCondLE) { 1697 cond = kCondGE; 1698 } else { 1699 DCHECK_EQ(cond, kCondGT); 1700 cond = kCondLT; 1701 } 1702 1703 std::swap(left, right); 1704 FALLTHROUGH_INTENDED; 1705 case kCondGE: 1706 case kCondLT: 1707 __ cmp(left.AsRegisterPairLow<Register>(), 1708 ShifterOperand(right.AsRegisterPairLow<Register>())); 1709 __ sbcs(IP, 1710 left.AsRegisterPairHigh<Register>(), 1711 ShifterOperand(right.AsRegisterPairHigh<Register>())); 1712 ret = ARMCondition(cond); 1713 break; 1714 default: 1715 LOG(FATAL) << "Unreachable"; 1716 UNREACHABLE(); 1717 } 1718 1719 return ret; 1720} 1721 1722static Condition GenerateTest(HInstruction* instruction, 1723 Location loc, 1724 bool invert, 1725 CodeGeneratorARM* codegen) { 1726 DCHECK(!instruction->IsConstant()); 1727 1728 Condition ret = invert ? EQ : NE; 1729 1730 if (IsBooleanValueOrMaterializedCondition(instruction)) { 1731 __ CmpConstant(loc.AsRegister<Register>(), 0); 1732 } else { 1733 HCondition* const condition = instruction->AsCondition(); 1734 const LocationSummary* const locations = condition->GetLocations(); 1735 const Primitive::Type type = condition->GetLeft()->GetType(); 1736 const IfCondition cond = invert ? condition->GetOppositeCondition() : condition->GetCondition(); 1737 const Location right = locations->InAt(1); 1738 1739 if (type == Primitive::kPrimLong) { 1740 ret = condition->GetLocations()->InAt(1).IsConstant() 1741 ? GenerateLongTestConstant(condition, invert, codegen) 1742 : GenerateLongTest(condition, invert, codegen); 1743 } else if (Primitive::IsFloatingPointType(type)) { 1744 GenerateVcmp(condition, codegen); 1745 __ vmstat(); 1746 ret = ARMFPCondition(cond, condition->IsGtBias()); 1747 } else { 1748 DCHECK(Primitive::IsIntegralType(type) || type == Primitive::kPrimNot) << type; 1749 1750 const Register left = locations->InAt(0).AsRegister<Register>(); 1751 1752 if (right.IsRegister()) { 1753 __ cmp(left, ShifterOperand(right.AsRegister<Register>())); 1754 } else { 1755 DCHECK(right.IsConstant()); 1756 __ CmpConstant(left, CodeGenerator::GetInt32ValueOf(right.GetConstant())); 1757 } 1758 1759 ret = ARMCondition(cond); 1760 } 1761 } 1762 1763 return ret; 1764} 1765 1766static bool CanGenerateTest(HInstruction* condition, ArmAssembler* assembler) { 1767 if (!IsBooleanValueOrMaterializedCondition(condition)) { 1768 const HCondition* const cond = condition->AsCondition(); 1769 1770 if (cond->GetLeft()->GetType() == Primitive::kPrimLong) { 1771 const LocationSummary* const locations = cond->GetLocations(); 1772 const IfCondition c = cond->GetCondition(); 1773 1774 if (locations->InAt(1).IsConstant()) { 1775 const int64_t value = locations->InAt(1).GetConstant()->AsLongConstant()->GetValue(); 1776 ShifterOperand so; 1777 1778 if (c < kCondLT || c > kCondGE) { 1779 // Since IT blocks longer than a 16-bit instruction are deprecated by ARMv8, 1780 // we check that the least significant half of the first input to be compared 1781 // is in a low register (the other half is read outside an IT block), and 1782 // the constant fits in an 8-bit unsigned integer, so that a 16-bit CMP 1783 // encoding can be used. 1784 if (!ArmAssembler::IsLowRegister(locations->InAt(0).AsRegisterPairLow<Register>()) || 1785 !IsUint<8>(Low32Bits(value))) { 1786 return false; 1787 } 1788 } else if (c == kCondLE || c == kCondGT) { 1789 if (value < std::numeric_limits<int64_t>::max() && 1790 !assembler->ShifterOperandCanHold(kNoRegister, 1791 kNoRegister, 1792 SBC, 1793 High32Bits(value + 1), 1794 kCcSet, 1795 &so)) { 1796 return false; 1797 } 1798 } else if (!assembler->ShifterOperandCanHold(kNoRegister, 1799 kNoRegister, 1800 SBC, 1801 High32Bits(value), 1802 kCcSet, 1803 &so)) { 1804 return false; 1805 } 1806 } 1807 } 1808 } 1809 1810 return true; 1811} 1812 1813static bool CanEncodeConstantAs8BitImmediate(HConstant* constant) { 1814 const Primitive::Type type = constant->GetType(); 1815 bool ret = false; 1816 1817 DCHECK(Primitive::IsIntegralType(type) || type == Primitive::kPrimNot) << type; 1818 1819 if (type == Primitive::kPrimLong) { 1820 const uint64_t value = constant->AsLongConstant()->GetValueAsUint64(); 1821 1822 ret = IsUint<8>(Low32Bits(value)) && IsUint<8>(High32Bits(value)); 1823 } else { 1824 ret = IsUint<8>(CodeGenerator::GetInt32ValueOf(constant)); 1825 } 1826 1827 return ret; 1828} 1829 1830static Location Arm8BitEncodableConstantOrRegister(HInstruction* constant) { 1831 DCHECK(!Primitive::IsFloatingPointType(constant->GetType())); 1832 1833 if (constant->IsConstant() && CanEncodeConstantAs8BitImmediate(constant->AsConstant())) { 1834 return Location::ConstantLocation(constant->AsConstant()); 1835 } 1836 1837 return Location::RequiresRegister(); 1838} 1839 1840static bool CanGenerateConditionalMove(const Location& out, const Location& src) { 1841 // Since IT blocks longer than a 16-bit instruction are deprecated by ARMv8, 1842 // we check that we are not dealing with floating-point output (there is no 1843 // 16-bit VMOV encoding). 1844 if (!out.IsRegister() && !out.IsRegisterPair()) { 1845 return false; 1846 } 1847 1848 // For constants, we also check that the output is in one or two low registers, 1849 // and that the constants fit in an 8-bit unsigned integer, so that a 16-bit 1850 // MOV encoding can be used. 1851 if (src.IsConstant()) { 1852 if (!CanEncodeConstantAs8BitImmediate(src.GetConstant())) { 1853 return false; 1854 } 1855 1856 if (out.IsRegister()) { 1857 if (!ArmAssembler::IsLowRegister(out.AsRegister<Register>())) { 1858 return false; 1859 } 1860 } else { 1861 DCHECK(out.IsRegisterPair()); 1862 1863 if (!ArmAssembler::IsLowRegister(out.AsRegisterPairHigh<Register>())) { 1864 return false; 1865 } 1866 } 1867 } 1868 1869 return true; 1870} 1871 1872#undef __ 1873// NOLINT on __ macro to suppress wrong warning/fix (misc-macro-parentheses) from clang-tidy. 1874#define __ down_cast<ArmAssembler*>(GetAssembler())-> // NOLINT 1875 1876Label* CodeGeneratorARM::GetFinalLabel(HInstruction* instruction, Label* final_label) { 1877 DCHECK(!instruction->IsControlFlow() && !instruction->IsSuspendCheck()); 1878 1879 const HBasicBlock* const block = instruction->GetBlock(); 1880 const HLoopInformation* const info = block->GetLoopInformation(); 1881 HInstruction* const next = instruction->GetNext(); 1882 1883 // Avoid a branch to a branch. 1884 if (next->IsGoto() && (info == nullptr || 1885 !info->IsBackEdge(*block) || 1886 !info->HasSuspendCheck())) { 1887 final_label = GetLabelOf(next->AsGoto()->GetSuccessor()); 1888 } 1889 1890 return final_label; 1891} 1892 1893void CodeGeneratorARM::DumpCoreRegister(std::ostream& stream, int reg) const { 1894 stream << Register(reg); 1895} 1896 1897void CodeGeneratorARM::DumpFloatingPointRegister(std::ostream& stream, int reg) const { 1898 stream << SRegister(reg); 1899} 1900 1901size_t CodeGeneratorARM::SaveCoreRegister(size_t stack_index, uint32_t reg_id) { 1902 __ StoreToOffset(kStoreWord, static_cast<Register>(reg_id), SP, stack_index); 1903 return kArmWordSize; 1904} 1905 1906size_t CodeGeneratorARM::RestoreCoreRegister(size_t stack_index, uint32_t reg_id) { 1907 __ LoadFromOffset(kLoadWord, static_cast<Register>(reg_id), SP, stack_index); 1908 return kArmWordSize; 1909} 1910 1911size_t CodeGeneratorARM::SaveFloatingPointRegister(size_t stack_index, uint32_t reg_id) { 1912 __ StoreSToOffset(static_cast<SRegister>(reg_id), SP, stack_index); 1913 return kArmWordSize; 1914} 1915 1916size_t CodeGeneratorARM::RestoreFloatingPointRegister(size_t stack_index, uint32_t reg_id) { 1917 __ LoadSFromOffset(static_cast<SRegister>(reg_id), SP, stack_index); 1918 return kArmWordSize; 1919} 1920 1921CodeGeneratorARM::CodeGeneratorARM(HGraph* graph, 1922 const ArmInstructionSetFeatures& isa_features, 1923 const CompilerOptions& compiler_options, 1924 OptimizingCompilerStats* stats) 1925 : CodeGenerator(graph, 1926 kNumberOfCoreRegisters, 1927 kNumberOfSRegisters, 1928 kNumberOfRegisterPairs, 1929 ComputeRegisterMask(reinterpret_cast<const int*>(kCoreCalleeSaves), 1930 arraysize(kCoreCalleeSaves)), 1931 ComputeRegisterMask(reinterpret_cast<const int*>(kFpuCalleeSaves), 1932 arraysize(kFpuCalleeSaves)), 1933 compiler_options, 1934 stats), 1935 block_labels_(nullptr), 1936 location_builder_(graph, this), 1937 instruction_visitor_(graph, this), 1938 move_resolver_(graph->GetArena(), this), 1939 assembler_(graph->GetArena()), 1940 isa_features_(isa_features), 1941 uint32_literals_(std::less<uint32_t>(), 1942 graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), 1943 pc_relative_dex_cache_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), 1944 boot_image_string_patches_(StringReferenceValueComparator(), 1945 graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), 1946 pc_relative_string_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), 1947 boot_image_type_patches_(TypeReferenceValueComparator(), 1948 graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), 1949 pc_relative_type_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), 1950 type_bss_entry_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), 1951 jit_string_patches_(StringReferenceValueComparator(), 1952 graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), 1953 jit_class_patches_(TypeReferenceValueComparator(), 1954 graph->GetArena()->Adapter(kArenaAllocCodeGenerator)) { 1955 // Always save the LR register to mimic Quick. 1956 AddAllocatedRegister(Location::RegisterLocation(LR)); 1957} 1958 1959void CodeGeneratorARM::Finalize(CodeAllocator* allocator) { 1960 // Ensure that we fix up branches and literal loads and emit the literal pool. 1961 __ FinalizeCode(); 1962 1963 // Adjust native pc offsets in stack maps. 1964 for (size_t i = 0, num = stack_map_stream_.GetNumberOfStackMaps(); i != num; ++i) { 1965 uint32_t old_position = 1966 stack_map_stream_.GetStackMap(i).native_pc_code_offset.Uint32Value(kThumb2); 1967 uint32_t new_position = __ GetAdjustedPosition(old_position); 1968 stack_map_stream_.SetStackMapNativePcOffset(i, new_position); 1969 } 1970 // Adjust pc offsets for the disassembly information. 1971 if (disasm_info_ != nullptr) { 1972 GeneratedCodeInterval* frame_entry_interval = disasm_info_->GetFrameEntryInterval(); 1973 frame_entry_interval->start = __ GetAdjustedPosition(frame_entry_interval->start); 1974 frame_entry_interval->end = __ GetAdjustedPosition(frame_entry_interval->end); 1975 for (auto& it : *disasm_info_->GetInstructionIntervals()) { 1976 it.second.start = __ GetAdjustedPosition(it.second.start); 1977 it.second.end = __ GetAdjustedPosition(it.second.end); 1978 } 1979 for (auto& it : *disasm_info_->GetSlowPathIntervals()) { 1980 it.code_interval.start = __ GetAdjustedPosition(it.code_interval.start); 1981 it.code_interval.end = __ GetAdjustedPosition(it.code_interval.end); 1982 } 1983 } 1984 1985 CodeGenerator::Finalize(allocator); 1986} 1987 1988void CodeGeneratorARM::SetupBlockedRegisters() const { 1989 // Stack register, LR and PC are always reserved. 1990 blocked_core_registers_[SP] = true; 1991 blocked_core_registers_[LR] = true; 1992 blocked_core_registers_[PC] = true; 1993 1994 // Reserve thread register. 1995 blocked_core_registers_[TR] = true; 1996 1997 // Reserve temp register. 1998 blocked_core_registers_[IP] = true; 1999 2000 if (GetGraph()->IsDebuggable()) { 2001 // Stubs do not save callee-save floating point registers. If the graph 2002 // is debuggable, we need to deal with these registers differently. For 2003 // now, just block them. 2004 for (size_t i = 0; i < arraysize(kFpuCalleeSaves); ++i) { 2005 blocked_fpu_registers_[kFpuCalleeSaves[i]] = true; 2006 } 2007 } 2008} 2009 2010InstructionCodeGeneratorARM::InstructionCodeGeneratorARM(HGraph* graph, CodeGeneratorARM* codegen) 2011 : InstructionCodeGenerator(graph, codegen), 2012 assembler_(codegen->GetAssembler()), 2013 codegen_(codegen) {} 2014 2015void CodeGeneratorARM::ComputeSpillMask() { 2016 core_spill_mask_ = allocated_registers_.GetCoreRegisters() & core_callee_save_mask_; 2017 DCHECK_NE(core_spill_mask_, 0u) << "At least the return address register must be saved"; 2018 // There is no easy instruction to restore just the PC on thumb2. We spill and 2019 // restore another arbitrary register. 2020 core_spill_mask_ |= (1 << kCoreAlwaysSpillRegister); 2021 fpu_spill_mask_ = allocated_registers_.GetFloatingPointRegisters() & fpu_callee_save_mask_; 2022 // We use vpush and vpop for saving and restoring floating point registers, which take 2023 // a SRegister and the number of registers to save/restore after that SRegister. We 2024 // therefore update the `fpu_spill_mask_` to also contain those registers not allocated, 2025 // but in the range. 2026 if (fpu_spill_mask_ != 0) { 2027 uint32_t least_significant_bit = LeastSignificantBit(fpu_spill_mask_); 2028 uint32_t most_significant_bit = MostSignificantBit(fpu_spill_mask_); 2029 for (uint32_t i = least_significant_bit + 1 ; i < most_significant_bit; ++i) { 2030 fpu_spill_mask_ |= (1 << i); 2031 } 2032 } 2033} 2034 2035static dwarf::Reg DWARFReg(Register reg) { 2036 return dwarf::Reg::ArmCore(static_cast<int>(reg)); 2037} 2038 2039static dwarf::Reg DWARFReg(SRegister reg) { 2040 return dwarf::Reg::ArmFp(static_cast<int>(reg)); 2041} 2042 2043void CodeGeneratorARM::GenerateFrameEntry() { 2044 bool skip_overflow_check = 2045 IsLeafMethod() && !FrameNeedsStackCheck(GetFrameSize(), InstructionSet::kArm); 2046 DCHECK(GetCompilerOptions().GetImplicitStackOverflowChecks()); 2047 __ Bind(&frame_entry_label_); 2048 2049 if (HasEmptyFrame()) { 2050 return; 2051 } 2052 2053 if (!skip_overflow_check) { 2054 __ AddConstant(IP, SP, -static_cast<int32_t>(GetStackOverflowReservedBytes(kArm))); 2055 __ LoadFromOffset(kLoadWord, IP, IP, 0); 2056 RecordPcInfo(nullptr, 0); 2057 } 2058 2059 __ PushList(core_spill_mask_); 2060 __ cfi().AdjustCFAOffset(kArmWordSize * POPCOUNT(core_spill_mask_)); 2061 __ cfi().RelOffsetForMany(DWARFReg(kMethodRegisterArgument), 0, core_spill_mask_, kArmWordSize); 2062 if (fpu_spill_mask_ != 0) { 2063 SRegister start_register = SRegister(LeastSignificantBit(fpu_spill_mask_)); 2064 __ vpushs(start_register, POPCOUNT(fpu_spill_mask_)); 2065 __ cfi().AdjustCFAOffset(kArmWordSize * POPCOUNT(fpu_spill_mask_)); 2066 __ cfi().RelOffsetForMany(DWARFReg(S0), 0, fpu_spill_mask_, kArmWordSize); 2067 } 2068 2069 if (GetGraph()->HasShouldDeoptimizeFlag()) { 2070 // Initialize should_deoptimize flag to 0. 2071 __ mov(IP, ShifterOperand(0)); 2072 __ StoreToOffset(kStoreWord, IP, SP, -kShouldDeoptimizeFlagSize); 2073 } 2074 2075 int adjust = GetFrameSize() - FrameEntrySpillSize(); 2076 __ AddConstant(SP, -adjust); 2077 __ cfi().AdjustCFAOffset(adjust); 2078 2079 // Save the current method if we need it. Note that we do not 2080 // do this in HCurrentMethod, as the instruction might have been removed 2081 // in the SSA graph. 2082 if (RequiresCurrentMethod()) { 2083 __ StoreToOffset(kStoreWord, kMethodRegisterArgument, SP, 0); 2084 } 2085} 2086 2087void CodeGeneratorARM::GenerateFrameExit() { 2088 if (HasEmptyFrame()) { 2089 __ bx(LR); 2090 return; 2091 } 2092 __ cfi().RememberState(); 2093 int adjust = GetFrameSize() - FrameEntrySpillSize(); 2094 __ AddConstant(SP, adjust); 2095 __ cfi().AdjustCFAOffset(-adjust); 2096 if (fpu_spill_mask_ != 0) { 2097 SRegister start_register = SRegister(LeastSignificantBit(fpu_spill_mask_)); 2098 __ vpops(start_register, POPCOUNT(fpu_spill_mask_)); 2099 __ cfi().AdjustCFAOffset(-static_cast<int>(kArmPointerSize) * POPCOUNT(fpu_spill_mask_)); 2100 __ cfi().RestoreMany(DWARFReg(SRegister(0)), fpu_spill_mask_); 2101 } 2102 // Pop LR into PC to return. 2103 DCHECK_NE(core_spill_mask_ & (1 << LR), 0U); 2104 uint32_t pop_mask = (core_spill_mask_ & (~(1 << LR))) | 1 << PC; 2105 __ PopList(pop_mask); 2106 __ cfi().RestoreState(); 2107 __ cfi().DefCFAOffset(GetFrameSize()); 2108} 2109 2110void CodeGeneratorARM::Bind(HBasicBlock* block) { 2111 Label* label = GetLabelOf(block); 2112 __ BindTrackedLabel(label); 2113} 2114 2115Location InvokeDexCallingConventionVisitorARM::GetNextLocation(Primitive::Type type) { 2116 switch (type) { 2117 case Primitive::kPrimBoolean: 2118 case Primitive::kPrimByte: 2119 case Primitive::kPrimChar: 2120 case Primitive::kPrimShort: 2121 case Primitive::kPrimInt: 2122 case Primitive::kPrimNot: { 2123 uint32_t index = gp_index_++; 2124 uint32_t stack_index = stack_index_++; 2125 if (index < calling_convention.GetNumberOfRegisters()) { 2126 return Location::RegisterLocation(calling_convention.GetRegisterAt(index)); 2127 } else { 2128 return Location::StackSlot(calling_convention.GetStackOffsetOf(stack_index)); 2129 } 2130 } 2131 2132 case Primitive::kPrimLong: { 2133 uint32_t index = gp_index_; 2134 uint32_t stack_index = stack_index_; 2135 gp_index_ += 2; 2136 stack_index_ += 2; 2137 if (index + 1 < calling_convention.GetNumberOfRegisters()) { 2138 if (calling_convention.GetRegisterAt(index) == R1) { 2139 // Skip R1, and use R2_R3 instead. 2140 gp_index_++; 2141 index++; 2142 } 2143 } 2144 if (index + 1 < calling_convention.GetNumberOfRegisters()) { 2145 DCHECK_EQ(calling_convention.GetRegisterAt(index) + 1, 2146 calling_convention.GetRegisterAt(index + 1)); 2147 2148 return Location::RegisterPairLocation(calling_convention.GetRegisterAt(index), 2149 calling_convention.GetRegisterAt(index + 1)); 2150 } else { 2151 return Location::DoubleStackSlot(calling_convention.GetStackOffsetOf(stack_index)); 2152 } 2153 } 2154 2155 case Primitive::kPrimFloat: { 2156 uint32_t stack_index = stack_index_++; 2157 if (float_index_ % 2 == 0) { 2158 float_index_ = std::max(double_index_, float_index_); 2159 } 2160 if (float_index_ < calling_convention.GetNumberOfFpuRegisters()) { 2161 return Location::FpuRegisterLocation(calling_convention.GetFpuRegisterAt(float_index_++)); 2162 } else { 2163 return Location::StackSlot(calling_convention.GetStackOffsetOf(stack_index)); 2164 } 2165 } 2166 2167 case Primitive::kPrimDouble: { 2168 double_index_ = std::max(double_index_, RoundUp(float_index_, 2)); 2169 uint32_t stack_index = stack_index_; 2170 stack_index_ += 2; 2171 if (double_index_ + 1 < calling_convention.GetNumberOfFpuRegisters()) { 2172 uint32_t index = double_index_; 2173 double_index_ += 2; 2174 Location result = Location::FpuRegisterPairLocation( 2175 calling_convention.GetFpuRegisterAt(index), 2176 calling_convention.GetFpuRegisterAt(index + 1)); 2177 DCHECK(ExpectedPairLayout(result)); 2178 return result; 2179 } else { 2180 return Location::DoubleStackSlot(calling_convention.GetStackOffsetOf(stack_index)); 2181 } 2182 } 2183 2184 case Primitive::kPrimVoid: 2185 LOG(FATAL) << "Unexpected parameter type " << type; 2186 break; 2187 } 2188 return Location::NoLocation(); 2189} 2190 2191Location InvokeDexCallingConventionVisitorARM::GetReturnLocation(Primitive::Type type) const { 2192 switch (type) { 2193 case Primitive::kPrimBoolean: 2194 case Primitive::kPrimByte: 2195 case Primitive::kPrimChar: 2196 case Primitive::kPrimShort: 2197 case Primitive::kPrimInt: 2198 case Primitive::kPrimNot: { 2199 return Location::RegisterLocation(R0); 2200 } 2201 2202 case Primitive::kPrimFloat: { 2203 return Location::FpuRegisterLocation(S0); 2204 } 2205 2206 case Primitive::kPrimLong: { 2207 return Location::RegisterPairLocation(R0, R1); 2208 } 2209 2210 case Primitive::kPrimDouble: { 2211 return Location::FpuRegisterPairLocation(S0, S1); 2212 } 2213 2214 case Primitive::kPrimVoid: 2215 return Location::NoLocation(); 2216 } 2217 2218 UNREACHABLE(); 2219} 2220 2221Location InvokeDexCallingConventionVisitorARM::GetMethodLocation() const { 2222 return Location::RegisterLocation(kMethodRegisterArgument); 2223} 2224 2225void CodeGeneratorARM::Move32(Location destination, Location source) { 2226 if (source.Equals(destination)) { 2227 return; 2228 } 2229 if (destination.IsRegister()) { 2230 if (source.IsRegister()) { 2231 __ Mov(destination.AsRegister<Register>(), source.AsRegister<Register>()); 2232 } else if (source.IsFpuRegister()) { 2233 __ vmovrs(destination.AsRegister<Register>(), source.AsFpuRegister<SRegister>()); 2234 } else { 2235 __ LoadFromOffset(kLoadWord, destination.AsRegister<Register>(), SP, source.GetStackIndex()); 2236 } 2237 } else if (destination.IsFpuRegister()) { 2238 if (source.IsRegister()) { 2239 __ vmovsr(destination.AsFpuRegister<SRegister>(), source.AsRegister<Register>()); 2240 } else if (source.IsFpuRegister()) { 2241 __ vmovs(destination.AsFpuRegister<SRegister>(), source.AsFpuRegister<SRegister>()); 2242 } else { 2243 __ LoadSFromOffset(destination.AsFpuRegister<SRegister>(), SP, source.GetStackIndex()); 2244 } 2245 } else { 2246 DCHECK(destination.IsStackSlot()) << destination; 2247 if (source.IsRegister()) { 2248 __ StoreToOffset(kStoreWord, source.AsRegister<Register>(), SP, destination.GetStackIndex()); 2249 } else if (source.IsFpuRegister()) { 2250 __ StoreSToOffset(source.AsFpuRegister<SRegister>(), SP, destination.GetStackIndex()); 2251 } else { 2252 DCHECK(source.IsStackSlot()) << source; 2253 __ LoadFromOffset(kLoadWord, IP, SP, source.GetStackIndex()); 2254 __ StoreToOffset(kStoreWord, IP, SP, destination.GetStackIndex()); 2255 } 2256 } 2257} 2258 2259void CodeGeneratorARM::Move64(Location destination, Location source) { 2260 if (source.Equals(destination)) { 2261 return; 2262 } 2263 if (destination.IsRegisterPair()) { 2264 if (source.IsRegisterPair()) { 2265 EmitParallelMoves( 2266 Location::RegisterLocation(source.AsRegisterPairHigh<Register>()), 2267 Location::RegisterLocation(destination.AsRegisterPairHigh<Register>()), 2268 Primitive::kPrimInt, 2269 Location::RegisterLocation(source.AsRegisterPairLow<Register>()), 2270 Location::RegisterLocation(destination.AsRegisterPairLow<Register>()), 2271 Primitive::kPrimInt); 2272 } else if (source.IsFpuRegister()) { 2273 UNIMPLEMENTED(FATAL); 2274 } else if (source.IsFpuRegisterPair()) { 2275 __ vmovrrd(destination.AsRegisterPairLow<Register>(), 2276 destination.AsRegisterPairHigh<Register>(), 2277 FromLowSToD(source.AsFpuRegisterPairLow<SRegister>())); 2278 } else { 2279 DCHECK(source.IsDoubleStackSlot()); 2280 DCHECK(ExpectedPairLayout(destination)); 2281 __ LoadFromOffset(kLoadWordPair, destination.AsRegisterPairLow<Register>(), 2282 SP, source.GetStackIndex()); 2283 } 2284 } else if (destination.IsFpuRegisterPair()) { 2285 if (source.IsDoubleStackSlot()) { 2286 __ LoadDFromOffset(FromLowSToD(destination.AsFpuRegisterPairLow<SRegister>()), 2287 SP, 2288 source.GetStackIndex()); 2289 } else if (source.IsRegisterPair()) { 2290 __ vmovdrr(FromLowSToD(destination.AsFpuRegisterPairLow<SRegister>()), 2291 source.AsRegisterPairLow<Register>(), 2292 source.AsRegisterPairHigh<Register>()); 2293 } else { 2294 UNIMPLEMENTED(FATAL); 2295 } 2296 } else { 2297 DCHECK(destination.IsDoubleStackSlot()); 2298 if (source.IsRegisterPair()) { 2299 // No conflict possible, so just do the moves. 2300 if (source.AsRegisterPairLow<Register>() == R1) { 2301 DCHECK_EQ(source.AsRegisterPairHigh<Register>(), R2); 2302 __ StoreToOffset(kStoreWord, R1, SP, destination.GetStackIndex()); 2303 __ StoreToOffset(kStoreWord, R2, SP, destination.GetHighStackIndex(kArmWordSize)); 2304 } else { 2305 __ StoreToOffset(kStoreWordPair, source.AsRegisterPairLow<Register>(), 2306 SP, destination.GetStackIndex()); 2307 } 2308 } else if (source.IsFpuRegisterPair()) { 2309 __ StoreDToOffset(FromLowSToD(source.AsFpuRegisterPairLow<SRegister>()), 2310 SP, 2311 destination.GetStackIndex()); 2312 } else { 2313 DCHECK(source.IsDoubleStackSlot()); 2314 EmitParallelMoves( 2315 Location::StackSlot(source.GetStackIndex()), 2316 Location::StackSlot(destination.GetStackIndex()), 2317 Primitive::kPrimInt, 2318 Location::StackSlot(source.GetHighStackIndex(kArmWordSize)), 2319 Location::StackSlot(destination.GetHighStackIndex(kArmWordSize)), 2320 Primitive::kPrimInt); 2321 } 2322 } 2323} 2324 2325void CodeGeneratorARM::MoveConstant(Location location, int32_t value) { 2326 DCHECK(location.IsRegister()); 2327 __ LoadImmediate(location.AsRegister<Register>(), value); 2328} 2329 2330void CodeGeneratorARM::MoveLocation(Location dst, Location src, Primitive::Type dst_type) { 2331 HParallelMove move(GetGraph()->GetArena()); 2332 move.AddMove(src, dst, dst_type, nullptr); 2333 GetMoveResolver()->EmitNativeCode(&move); 2334} 2335 2336void CodeGeneratorARM::AddLocationAsTemp(Location location, LocationSummary* locations) { 2337 if (location.IsRegister()) { 2338 locations->AddTemp(location); 2339 } else if (location.IsRegisterPair()) { 2340 locations->AddTemp(Location::RegisterLocation(location.AsRegisterPairLow<Register>())); 2341 locations->AddTemp(Location::RegisterLocation(location.AsRegisterPairHigh<Register>())); 2342 } else { 2343 UNIMPLEMENTED(FATAL) << "AddLocationAsTemp not implemented for location " << location; 2344 } 2345} 2346 2347void CodeGeneratorARM::InvokeRuntime(QuickEntrypointEnum entrypoint, 2348 HInstruction* instruction, 2349 uint32_t dex_pc, 2350 SlowPathCode* slow_path) { 2351 ValidateInvokeRuntime(entrypoint, instruction, slow_path); 2352 GenerateInvokeRuntime(GetThreadOffset<kArmPointerSize>(entrypoint).Int32Value()); 2353 if (EntrypointRequiresStackMap(entrypoint)) { 2354 RecordPcInfo(instruction, dex_pc, slow_path); 2355 } 2356} 2357 2358void CodeGeneratorARM::InvokeRuntimeWithoutRecordingPcInfo(int32_t entry_point_offset, 2359 HInstruction* instruction, 2360 SlowPathCode* slow_path) { 2361 ValidateInvokeRuntimeWithoutRecordingPcInfo(instruction, slow_path); 2362 GenerateInvokeRuntime(entry_point_offset); 2363} 2364 2365void CodeGeneratorARM::GenerateInvokeRuntime(int32_t entry_point_offset) { 2366 __ LoadFromOffset(kLoadWord, LR, TR, entry_point_offset); 2367 __ blx(LR); 2368} 2369 2370void InstructionCodeGeneratorARM::HandleGoto(HInstruction* got, HBasicBlock* successor) { 2371 DCHECK(!successor->IsExitBlock()); 2372 2373 HBasicBlock* block = got->GetBlock(); 2374 HInstruction* previous = got->GetPrevious(); 2375 2376 HLoopInformation* info = block->GetLoopInformation(); 2377 if (info != nullptr && info->IsBackEdge(*block) && info->HasSuspendCheck()) { 2378 codegen_->ClearSpillSlotsFromLoopPhisInStackMap(info->GetSuspendCheck()); 2379 GenerateSuspendCheck(info->GetSuspendCheck(), successor); 2380 return; 2381 } 2382 2383 if (block->IsEntryBlock() && (previous != nullptr) && previous->IsSuspendCheck()) { 2384 GenerateSuspendCheck(previous->AsSuspendCheck(), nullptr); 2385 } 2386 if (!codegen_->GoesToNextBlock(got->GetBlock(), successor)) { 2387 __ b(codegen_->GetLabelOf(successor)); 2388 } 2389} 2390 2391void LocationsBuilderARM::VisitGoto(HGoto* got) { 2392 got->SetLocations(nullptr); 2393} 2394 2395void InstructionCodeGeneratorARM::VisitGoto(HGoto* got) { 2396 HandleGoto(got, got->GetSuccessor()); 2397} 2398 2399void LocationsBuilderARM::VisitTryBoundary(HTryBoundary* try_boundary) { 2400 try_boundary->SetLocations(nullptr); 2401} 2402 2403void InstructionCodeGeneratorARM::VisitTryBoundary(HTryBoundary* try_boundary) { 2404 HBasicBlock* successor = try_boundary->GetNormalFlowSuccessor(); 2405 if (!successor->IsExitBlock()) { 2406 HandleGoto(try_boundary, successor); 2407 } 2408} 2409 2410void LocationsBuilderARM::VisitExit(HExit* exit) { 2411 exit->SetLocations(nullptr); 2412} 2413 2414void InstructionCodeGeneratorARM::VisitExit(HExit* exit ATTRIBUTE_UNUSED) { 2415} 2416 2417void InstructionCodeGeneratorARM::GenerateFPJumps(HCondition* cond, 2418 Label* true_label, 2419 Label* false_label ATTRIBUTE_UNUSED) { 2420 __ vmstat(); // transfer FP status register to ARM APSR. 2421 __ b(true_label, ARMFPCondition(cond->GetCondition(), cond->IsGtBias())); 2422} 2423 2424void InstructionCodeGeneratorARM::GenerateLongComparesAndJumps(HCondition* cond, 2425 Label* true_label, 2426 Label* false_label) { 2427 LocationSummary* locations = cond->GetLocations(); 2428 Location left = locations->InAt(0); 2429 Location right = locations->InAt(1); 2430 IfCondition if_cond = cond->GetCondition(); 2431 2432 Register left_high = left.AsRegisterPairHigh<Register>(); 2433 Register left_low = left.AsRegisterPairLow<Register>(); 2434 IfCondition true_high_cond = if_cond; 2435 IfCondition false_high_cond = cond->GetOppositeCondition(); 2436 Condition final_condition = ARMUnsignedCondition(if_cond); // unsigned on lower part 2437 2438 // Set the conditions for the test, remembering that == needs to be 2439 // decided using the low words. 2440 // TODO: consider avoiding jumps with temporary and CMP low+SBC high 2441 switch (if_cond) { 2442 case kCondEQ: 2443 case kCondNE: 2444 // Nothing to do. 2445 break; 2446 case kCondLT: 2447 false_high_cond = kCondGT; 2448 break; 2449 case kCondLE: 2450 true_high_cond = kCondLT; 2451 break; 2452 case kCondGT: 2453 false_high_cond = kCondLT; 2454 break; 2455 case kCondGE: 2456 true_high_cond = kCondGT; 2457 break; 2458 case kCondB: 2459 false_high_cond = kCondA; 2460 break; 2461 case kCondBE: 2462 true_high_cond = kCondB; 2463 break; 2464 case kCondA: 2465 false_high_cond = kCondB; 2466 break; 2467 case kCondAE: 2468 true_high_cond = kCondA; 2469 break; 2470 } 2471 if (right.IsConstant()) { 2472 int64_t value = right.GetConstant()->AsLongConstant()->GetValue(); 2473 int32_t val_low = Low32Bits(value); 2474 int32_t val_high = High32Bits(value); 2475 2476 __ CmpConstant(left_high, val_high); 2477 if (if_cond == kCondNE) { 2478 __ b(true_label, ARMCondition(true_high_cond)); 2479 } else if (if_cond == kCondEQ) { 2480 __ b(false_label, ARMCondition(false_high_cond)); 2481 } else { 2482 __ b(true_label, ARMCondition(true_high_cond)); 2483 __ b(false_label, ARMCondition(false_high_cond)); 2484 } 2485 // Must be equal high, so compare the lows. 2486 __ CmpConstant(left_low, val_low); 2487 } else { 2488 Register right_high = right.AsRegisterPairHigh<Register>(); 2489 Register right_low = right.AsRegisterPairLow<Register>(); 2490 2491 __ cmp(left_high, ShifterOperand(right_high)); 2492 if (if_cond == kCondNE) { 2493 __ b(true_label, ARMCondition(true_high_cond)); 2494 } else if (if_cond == kCondEQ) { 2495 __ b(false_label, ARMCondition(false_high_cond)); 2496 } else { 2497 __ b(true_label, ARMCondition(true_high_cond)); 2498 __ b(false_label, ARMCondition(false_high_cond)); 2499 } 2500 // Must be equal high, so compare the lows. 2501 __ cmp(left_low, ShifterOperand(right_low)); 2502 } 2503 // The last comparison might be unsigned. 2504 // TODO: optimize cases where this is always true/false 2505 __ b(true_label, final_condition); 2506} 2507 2508void InstructionCodeGeneratorARM::GenerateCompareTestAndBranch(HCondition* condition, 2509 Label* true_target_in, 2510 Label* false_target_in) { 2511 // Generated branching requires both targets to be explicit. If either of the 2512 // targets is nullptr (fallthrough) use and bind `fallthrough_target` instead. 2513 Label fallthrough_target; 2514 Label* true_target = true_target_in == nullptr ? &fallthrough_target : true_target_in; 2515 Label* false_target = false_target_in == nullptr ? &fallthrough_target : false_target_in; 2516 2517 Primitive::Type type = condition->InputAt(0)->GetType(); 2518 switch (type) { 2519 case Primitive::kPrimLong: 2520 GenerateLongComparesAndJumps(condition, true_target, false_target); 2521 break; 2522 case Primitive::kPrimFloat: 2523 case Primitive::kPrimDouble: 2524 GenerateVcmp(condition, codegen_); 2525 GenerateFPJumps(condition, true_target, false_target); 2526 break; 2527 default: 2528 LOG(FATAL) << "Unexpected compare type " << type; 2529 } 2530 2531 if (false_target != &fallthrough_target) { 2532 __ b(false_target); 2533 } 2534 2535 if (fallthrough_target.IsLinked()) { 2536 __ Bind(&fallthrough_target); 2537 } 2538} 2539 2540void InstructionCodeGeneratorARM::GenerateTestAndBranch(HInstruction* instruction, 2541 size_t condition_input_index, 2542 Label* true_target, 2543 Label* false_target) { 2544 HInstruction* cond = instruction->InputAt(condition_input_index); 2545 2546 if (true_target == nullptr && false_target == nullptr) { 2547 // Nothing to do. The code always falls through. 2548 return; 2549 } else if (cond->IsIntConstant()) { 2550 // Constant condition, statically compared against "true" (integer value 1). 2551 if (cond->AsIntConstant()->IsTrue()) { 2552 if (true_target != nullptr) { 2553 __ b(true_target); 2554 } 2555 } else { 2556 DCHECK(cond->AsIntConstant()->IsFalse()) << cond->AsIntConstant()->GetValue(); 2557 if (false_target != nullptr) { 2558 __ b(false_target); 2559 } 2560 } 2561 return; 2562 } 2563 2564 // The following code generates these patterns: 2565 // (1) true_target == nullptr && false_target != nullptr 2566 // - opposite condition true => branch to false_target 2567 // (2) true_target != nullptr && false_target == nullptr 2568 // - condition true => branch to true_target 2569 // (3) true_target != nullptr && false_target != nullptr 2570 // - condition true => branch to true_target 2571 // - branch to false_target 2572 if (IsBooleanValueOrMaterializedCondition(cond)) { 2573 // Condition has been materialized, compare the output to 0. 2574 Location cond_val = instruction->GetLocations()->InAt(condition_input_index); 2575 DCHECK(cond_val.IsRegister()); 2576 if (true_target == nullptr) { 2577 __ CompareAndBranchIfZero(cond_val.AsRegister<Register>(), false_target); 2578 } else { 2579 __ CompareAndBranchIfNonZero(cond_val.AsRegister<Register>(), true_target); 2580 } 2581 } else { 2582 // Condition has not been materialized. Use its inputs as the comparison and 2583 // its condition as the branch condition. 2584 HCondition* condition = cond->AsCondition(); 2585 2586 // If this is a long or FP comparison that has been folded into 2587 // the HCondition, generate the comparison directly. 2588 Primitive::Type type = condition->InputAt(0)->GetType(); 2589 if (type == Primitive::kPrimLong || Primitive::IsFloatingPointType(type)) { 2590 GenerateCompareTestAndBranch(condition, true_target, false_target); 2591 return; 2592 } 2593 2594 Label* non_fallthrough_target; 2595 Condition arm_cond; 2596 LocationSummary* locations = cond->GetLocations(); 2597 DCHECK(locations->InAt(0).IsRegister()); 2598 Register left = locations->InAt(0).AsRegister<Register>(); 2599 Location right = locations->InAt(1); 2600 2601 if (true_target == nullptr) { 2602 arm_cond = ARMCondition(condition->GetOppositeCondition()); 2603 non_fallthrough_target = false_target; 2604 } else { 2605 arm_cond = ARMCondition(condition->GetCondition()); 2606 non_fallthrough_target = true_target; 2607 } 2608 2609 if (right.IsConstant() && (arm_cond == NE || arm_cond == EQ) && 2610 CodeGenerator::GetInt32ValueOf(right.GetConstant()) == 0) { 2611 if (arm_cond == EQ) { 2612 __ CompareAndBranchIfZero(left, non_fallthrough_target); 2613 } else { 2614 DCHECK_EQ(arm_cond, NE); 2615 __ CompareAndBranchIfNonZero(left, non_fallthrough_target); 2616 } 2617 } else { 2618 if (right.IsRegister()) { 2619 __ cmp(left, ShifterOperand(right.AsRegister<Register>())); 2620 } else { 2621 DCHECK(right.IsConstant()); 2622 __ CmpConstant(left, CodeGenerator::GetInt32ValueOf(right.GetConstant())); 2623 } 2624 2625 __ b(non_fallthrough_target, arm_cond); 2626 } 2627 } 2628 2629 // If neither branch falls through (case 3), the conditional branch to `true_target` 2630 // was already emitted (case 2) and we need to emit a jump to `false_target`. 2631 if (true_target != nullptr && false_target != nullptr) { 2632 __ b(false_target); 2633 } 2634} 2635 2636void LocationsBuilderARM::VisitIf(HIf* if_instr) { 2637 LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(if_instr); 2638 if (IsBooleanValueOrMaterializedCondition(if_instr->InputAt(0))) { 2639 locations->SetInAt(0, Location::RequiresRegister()); 2640 } 2641} 2642 2643void InstructionCodeGeneratorARM::VisitIf(HIf* if_instr) { 2644 HBasicBlock* true_successor = if_instr->IfTrueSuccessor(); 2645 HBasicBlock* false_successor = if_instr->IfFalseSuccessor(); 2646 Label* true_target = codegen_->GoesToNextBlock(if_instr->GetBlock(), true_successor) ? 2647 nullptr : codegen_->GetLabelOf(true_successor); 2648 Label* false_target = codegen_->GoesToNextBlock(if_instr->GetBlock(), false_successor) ? 2649 nullptr : codegen_->GetLabelOf(false_successor); 2650 GenerateTestAndBranch(if_instr, /* condition_input_index */ 0, true_target, false_target); 2651} 2652 2653void LocationsBuilderARM::VisitDeoptimize(HDeoptimize* deoptimize) { 2654 LocationSummary* locations = new (GetGraph()->GetArena()) 2655 LocationSummary(deoptimize, LocationSummary::kCallOnSlowPath); 2656 locations->SetCustomSlowPathCallerSaves(RegisterSet::Empty()); // No caller-save registers. 2657 if (IsBooleanValueOrMaterializedCondition(deoptimize->InputAt(0))) { 2658 locations->SetInAt(0, Location::RequiresRegister()); 2659 } 2660} 2661 2662void InstructionCodeGeneratorARM::VisitDeoptimize(HDeoptimize* deoptimize) { 2663 SlowPathCodeARM* slow_path = deopt_slow_paths_.NewSlowPath<DeoptimizationSlowPathARM>(deoptimize); 2664 GenerateTestAndBranch(deoptimize, 2665 /* condition_input_index */ 0, 2666 slow_path->GetEntryLabel(), 2667 /* false_target */ nullptr); 2668} 2669 2670void LocationsBuilderARM::VisitShouldDeoptimizeFlag(HShouldDeoptimizeFlag* flag) { 2671 LocationSummary* locations = new (GetGraph()->GetArena()) 2672 LocationSummary(flag, LocationSummary::kNoCall); 2673 locations->SetOut(Location::RequiresRegister()); 2674} 2675 2676void InstructionCodeGeneratorARM::VisitShouldDeoptimizeFlag(HShouldDeoptimizeFlag* flag) { 2677 __ LoadFromOffset(kLoadWord, 2678 flag->GetLocations()->Out().AsRegister<Register>(), 2679 SP, 2680 codegen_->GetStackOffsetOfShouldDeoptimizeFlag()); 2681} 2682 2683void LocationsBuilderARM::VisitSelect(HSelect* select) { 2684 LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(select); 2685 const bool is_floating_point = Primitive::IsFloatingPointType(select->GetType()); 2686 2687 if (is_floating_point) { 2688 locations->SetInAt(0, Location::RequiresFpuRegister()); 2689 locations->SetInAt(1, Location::FpuRegisterOrConstant(select->GetTrueValue())); 2690 } else { 2691 locations->SetInAt(0, Location::RequiresRegister()); 2692 locations->SetInAt(1, Arm8BitEncodableConstantOrRegister(select->GetTrueValue())); 2693 } 2694 2695 if (IsBooleanValueOrMaterializedCondition(select->GetCondition())) { 2696 locations->SetInAt(2, Location::RegisterOrConstant(select->GetCondition())); 2697 // The code generator handles overlap with the values, but not with the condition. 2698 locations->SetOut(Location::SameAsFirstInput()); 2699 } else if (is_floating_point) { 2700 locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap); 2701 } else { 2702 if (!locations->InAt(1).IsConstant()) { 2703 locations->SetInAt(0, Arm8BitEncodableConstantOrRegister(select->GetFalseValue())); 2704 } 2705 2706 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 2707 } 2708} 2709 2710void InstructionCodeGeneratorARM::VisitSelect(HSelect* select) { 2711 HInstruction* const condition = select->GetCondition(); 2712 const LocationSummary* const locations = select->GetLocations(); 2713 const Primitive::Type type = select->GetType(); 2714 const Location first = locations->InAt(0); 2715 const Location out = locations->Out(); 2716 const Location second = locations->InAt(1); 2717 Location src; 2718 2719 if (condition->IsIntConstant()) { 2720 if (condition->AsIntConstant()->IsFalse()) { 2721 src = first; 2722 } else { 2723 src = second; 2724 } 2725 2726 codegen_->MoveLocation(out, src, type); 2727 return; 2728 } 2729 2730 if (!Primitive::IsFloatingPointType(type) && 2731 CanGenerateTest(condition, codegen_->GetAssembler())) { 2732 bool invert = false; 2733 2734 if (out.Equals(second)) { 2735 src = first; 2736 invert = true; 2737 } else if (out.Equals(first)) { 2738 src = second; 2739 } else if (second.IsConstant()) { 2740 DCHECK(CanEncodeConstantAs8BitImmediate(second.GetConstant())); 2741 src = second; 2742 } else if (first.IsConstant()) { 2743 DCHECK(CanEncodeConstantAs8BitImmediate(first.GetConstant())); 2744 src = first; 2745 invert = true; 2746 } else { 2747 src = second; 2748 } 2749 2750 if (CanGenerateConditionalMove(out, src)) { 2751 if (!out.Equals(first) && !out.Equals(second)) { 2752 codegen_->MoveLocation(out, src.Equals(first) ? second : first, type); 2753 } 2754 2755 const Condition cond = GenerateTest(condition, locations->InAt(2), invert, codegen_); 2756 2757 if (out.IsRegister()) { 2758 ShifterOperand operand; 2759 2760 if (src.IsConstant()) { 2761 operand = ShifterOperand(CodeGenerator::GetInt32ValueOf(src.GetConstant())); 2762 } else { 2763 DCHECK(src.IsRegister()); 2764 operand = ShifterOperand(src.AsRegister<Register>()); 2765 } 2766 2767 __ it(cond); 2768 __ mov(out.AsRegister<Register>(), operand, cond); 2769 } else { 2770 DCHECK(out.IsRegisterPair()); 2771 2772 ShifterOperand operand_high; 2773 ShifterOperand operand_low; 2774 2775 if (src.IsConstant()) { 2776 const int64_t value = src.GetConstant()->AsLongConstant()->GetValue(); 2777 2778 operand_high = ShifterOperand(High32Bits(value)); 2779 operand_low = ShifterOperand(Low32Bits(value)); 2780 } else { 2781 DCHECK(src.IsRegisterPair()); 2782 operand_high = ShifterOperand(src.AsRegisterPairHigh<Register>()); 2783 operand_low = ShifterOperand(src.AsRegisterPairLow<Register>()); 2784 } 2785 2786 __ it(cond); 2787 __ mov(out.AsRegisterPairLow<Register>(), operand_low, cond); 2788 __ it(cond); 2789 __ mov(out.AsRegisterPairHigh<Register>(), operand_high, cond); 2790 } 2791 2792 return; 2793 } 2794 } 2795 2796 Label* false_target = nullptr; 2797 Label* true_target = nullptr; 2798 Label select_end; 2799 Label* target = codegen_->GetFinalLabel(select, &select_end); 2800 2801 if (out.Equals(second)) { 2802 true_target = target; 2803 src = first; 2804 } else { 2805 false_target = target; 2806 src = second; 2807 2808 if (!out.Equals(first)) { 2809 codegen_->MoveLocation(out, first, type); 2810 } 2811 } 2812 2813 GenerateTestAndBranch(select, 2, true_target, false_target); 2814 codegen_->MoveLocation(out, src, type); 2815 2816 if (select_end.IsLinked()) { 2817 __ Bind(&select_end); 2818 } 2819} 2820 2821void LocationsBuilderARM::VisitNativeDebugInfo(HNativeDebugInfo* info) { 2822 new (GetGraph()->GetArena()) LocationSummary(info); 2823} 2824 2825void InstructionCodeGeneratorARM::VisitNativeDebugInfo(HNativeDebugInfo*) { 2826 // MaybeRecordNativeDebugInfo is already called implicitly in CodeGenerator::Compile. 2827} 2828 2829void CodeGeneratorARM::GenerateNop() { 2830 __ nop(); 2831} 2832 2833void LocationsBuilderARM::HandleCondition(HCondition* cond) { 2834 LocationSummary* locations = 2835 new (GetGraph()->GetArena()) LocationSummary(cond, LocationSummary::kNoCall); 2836 // Handle the long/FP comparisons made in instruction simplification. 2837 switch (cond->InputAt(0)->GetType()) { 2838 case Primitive::kPrimLong: 2839 locations->SetInAt(0, Location::RequiresRegister()); 2840 locations->SetInAt(1, Location::RegisterOrConstant(cond->InputAt(1))); 2841 if (!cond->IsEmittedAtUseSite()) { 2842 locations->SetOut(Location::RequiresRegister(), Location::kOutputOverlap); 2843 } 2844 break; 2845 2846 case Primitive::kPrimFloat: 2847 case Primitive::kPrimDouble: 2848 locations->SetInAt(0, Location::RequiresFpuRegister()); 2849 locations->SetInAt(1, ArithmeticZeroOrFpuRegister(cond->InputAt(1))); 2850 if (!cond->IsEmittedAtUseSite()) { 2851 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 2852 } 2853 break; 2854 2855 default: 2856 locations->SetInAt(0, Location::RequiresRegister()); 2857 locations->SetInAt(1, Location::RegisterOrConstant(cond->InputAt(1))); 2858 if (!cond->IsEmittedAtUseSite()) { 2859 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 2860 } 2861 } 2862} 2863 2864void InstructionCodeGeneratorARM::HandleCondition(HCondition* cond) { 2865 if (cond->IsEmittedAtUseSite()) { 2866 return; 2867 } 2868 2869 LocationSummary* locations = cond->GetLocations(); 2870 Location left = locations->InAt(0); 2871 Location right = locations->InAt(1); 2872 Register out = locations->Out().AsRegister<Register>(); 2873 Label true_label, false_label; 2874 2875 switch (cond->InputAt(0)->GetType()) { 2876 default: { 2877 // Integer case. 2878 if (right.IsRegister()) { 2879 __ cmp(left.AsRegister<Register>(), ShifterOperand(right.AsRegister<Register>())); 2880 } else { 2881 DCHECK(right.IsConstant()); 2882 __ CmpConstant(left.AsRegister<Register>(), 2883 CodeGenerator::GetInt32ValueOf(right.GetConstant())); 2884 } 2885 __ it(ARMCondition(cond->GetCondition()), kItElse); 2886 __ mov(locations->Out().AsRegister<Register>(), ShifterOperand(1), 2887 ARMCondition(cond->GetCondition())); 2888 __ mov(locations->Out().AsRegister<Register>(), ShifterOperand(0), 2889 ARMCondition(cond->GetOppositeCondition())); 2890 return; 2891 } 2892 case Primitive::kPrimLong: 2893 GenerateLongComparesAndJumps(cond, &true_label, &false_label); 2894 break; 2895 case Primitive::kPrimFloat: 2896 case Primitive::kPrimDouble: 2897 GenerateVcmp(cond, codegen_); 2898 GenerateFPJumps(cond, &true_label, &false_label); 2899 break; 2900 } 2901 2902 // Convert the jumps into the result. 2903 Label done_label; 2904 2905 // False case: result = 0. 2906 __ Bind(&false_label); 2907 __ LoadImmediate(out, 0); 2908 __ b(&done_label); 2909 2910 // True case: result = 1. 2911 __ Bind(&true_label); 2912 __ LoadImmediate(out, 1); 2913 __ Bind(&done_label); 2914} 2915 2916void LocationsBuilderARM::VisitEqual(HEqual* comp) { 2917 HandleCondition(comp); 2918} 2919 2920void InstructionCodeGeneratorARM::VisitEqual(HEqual* comp) { 2921 HandleCondition(comp); 2922} 2923 2924void LocationsBuilderARM::VisitNotEqual(HNotEqual* comp) { 2925 HandleCondition(comp); 2926} 2927 2928void InstructionCodeGeneratorARM::VisitNotEqual(HNotEqual* comp) { 2929 HandleCondition(comp); 2930} 2931 2932void LocationsBuilderARM::VisitLessThan(HLessThan* comp) { 2933 HandleCondition(comp); 2934} 2935 2936void InstructionCodeGeneratorARM::VisitLessThan(HLessThan* comp) { 2937 HandleCondition(comp); 2938} 2939 2940void LocationsBuilderARM::VisitLessThanOrEqual(HLessThanOrEqual* comp) { 2941 HandleCondition(comp); 2942} 2943 2944void InstructionCodeGeneratorARM::VisitLessThanOrEqual(HLessThanOrEqual* comp) { 2945 HandleCondition(comp); 2946} 2947 2948void LocationsBuilderARM::VisitGreaterThan(HGreaterThan* comp) { 2949 HandleCondition(comp); 2950} 2951 2952void InstructionCodeGeneratorARM::VisitGreaterThan(HGreaterThan* comp) { 2953 HandleCondition(comp); 2954} 2955 2956void LocationsBuilderARM::VisitGreaterThanOrEqual(HGreaterThanOrEqual* comp) { 2957 HandleCondition(comp); 2958} 2959 2960void InstructionCodeGeneratorARM::VisitGreaterThanOrEqual(HGreaterThanOrEqual* comp) { 2961 HandleCondition(comp); 2962} 2963 2964void LocationsBuilderARM::VisitBelow(HBelow* comp) { 2965 HandleCondition(comp); 2966} 2967 2968void InstructionCodeGeneratorARM::VisitBelow(HBelow* comp) { 2969 HandleCondition(comp); 2970} 2971 2972void LocationsBuilderARM::VisitBelowOrEqual(HBelowOrEqual* comp) { 2973 HandleCondition(comp); 2974} 2975 2976void InstructionCodeGeneratorARM::VisitBelowOrEqual(HBelowOrEqual* comp) { 2977 HandleCondition(comp); 2978} 2979 2980void LocationsBuilderARM::VisitAbove(HAbove* comp) { 2981 HandleCondition(comp); 2982} 2983 2984void InstructionCodeGeneratorARM::VisitAbove(HAbove* comp) { 2985 HandleCondition(comp); 2986} 2987 2988void LocationsBuilderARM::VisitAboveOrEqual(HAboveOrEqual* comp) { 2989 HandleCondition(comp); 2990} 2991 2992void InstructionCodeGeneratorARM::VisitAboveOrEqual(HAboveOrEqual* comp) { 2993 HandleCondition(comp); 2994} 2995 2996void LocationsBuilderARM::VisitIntConstant(HIntConstant* constant) { 2997 LocationSummary* locations = 2998 new (GetGraph()->GetArena()) LocationSummary(constant, LocationSummary::kNoCall); 2999 locations->SetOut(Location::ConstantLocation(constant)); 3000} 3001 3002void InstructionCodeGeneratorARM::VisitIntConstant(HIntConstant* constant ATTRIBUTE_UNUSED) { 3003 // Will be generated at use site. 3004} 3005 3006void LocationsBuilderARM::VisitNullConstant(HNullConstant* constant) { 3007 LocationSummary* locations = 3008 new (GetGraph()->GetArena()) LocationSummary(constant, LocationSummary::kNoCall); 3009 locations->SetOut(Location::ConstantLocation(constant)); 3010} 3011 3012void InstructionCodeGeneratorARM::VisitNullConstant(HNullConstant* constant ATTRIBUTE_UNUSED) { 3013 // Will be generated at use site. 3014} 3015 3016void LocationsBuilderARM::VisitLongConstant(HLongConstant* constant) { 3017 LocationSummary* locations = 3018 new (GetGraph()->GetArena()) LocationSummary(constant, LocationSummary::kNoCall); 3019 locations->SetOut(Location::ConstantLocation(constant)); 3020} 3021 3022void InstructionCodeGeneratorARM::VisitLongConstant(HLongConstant* constant ATTRIBUTE_UNUSED) { 3023 // Will be generated at use site. 3024} 3025 3026void LocationsBuilderARM::VisitFloatConstant(HFloatConstant* constant) { 3027 LocationSummary* locations = 3028 new (GetGraph()->GetArena()) LocationSummary(constant, LocationSummary::kNoCall); 3029 locations->SetOut(Location::ConstantLocation(constant)); 3030} 3031 3032void InstructionCodeGeneratorARM::VisitFloatConstant(HFloatConstant* constant ATTRIBUTE_UNUSED) { 3033 // Will be generated at use site. 3034} 3035 3036void LocationsBuilderARM::VisitDoubleConstant(HDoubleConstant* constant) { 3037 LocationSummary* locations = 3038 new (GetGraph()->GetArena()) LocationSummary(constant, LocationSummary::kNoCall); 3039 locations->SetOut(Location::ConstantLocation(constant)); 3040} 3041 3042void InstructionCodeGeneratorARM::VisitDoubleConstant(HDoubleConstant* constant ATTRIBUTE_UNUSED) { 3043 // Will be generated at use site. 3044} 3045 3046void LocationsBuilderARM::VisitMemoryBarrier(HMemoryBarrier* memory_barrier) { 3047 memory_barrier->SetLocations(nullptr); 3048} 3049 3050void InstructionCodeGeneratorARM::VisitMemoryBarrier(HMemoryBarrier* memory_barrier) { 3051 codegen_->GenerateMemoryBarrier(memory_barrier->GetBarrierKind()); 3052} 3053 3054void LocationsBuilderARM::VisitReturnVoid(HReturnVoid* ret) { 3055 ret->SetLocations(nullptr); 3056} 3057 3058void InstructionCodeGeneratorARM::VisitReturnVoid(HReturnVoid* ret ATTRIBUTE_UNUSED) { 3059 codegen_->GenerateFrameExit(); 3060} 3061 3062void LocationsBuilderARM::VisitReturn(HReturn* ret) { 3063 LocationSummary* locations = 3064 new (GetGraph()->GetArena()) LocationSummary(ret, LocationSummary::kNoCall); 3065 locations->SetInAt(0, parameter_visitor_.GetReturnLocation(ret->InputAt(0)->GetType())); 3066} 3067 3068void InstructionCodeGeneratorARM::VisitReturn(HReturn* ret ATTRIBUTE_UNUSED) { 3069 codegen_->GenerateFrameExit(); 3070} 3071 3072void LocationsBuilderARM::VisitInvokeUnresolved(HInvokeUnresolved* invoke) { 3073 // The trampoline uses the same calling convention as dex calling conventions, 3074 // except instead of loading arg0/r0 with the target Method*, arg0/r0 will contain 3075 // the method_idx. 3076 HandleInvoke(invoke); 3077} 3078 3079void InstructionCodeGeneratorARM::VisitInvokeUnresolved(HInvokeUnresolved* invoke) { 3080 codegen_->GenerateInvokeUnresolvedRuntimeCall(invoke); 3081} 3082 3083void LocationsBuilderARM::VisitInvokeStaticOrDirect(HInvokeStaticOrDirect* invoke) { 3084 // Explicit clinit checks triggered by static invokes must have been pruned by 3085 // art::PrepareForRegisterAllocation. 3086 DCHECK(!invoke->IsStaticWithExplicitClinitCheck()); 3087 3088 IntrinsicLocationsBuilderARM intrinsic(codegen_); 3089 if (intrinsic.TryDispatch(invoke)) { 3090 if (invoke->GetLocations()->CanCall() && invoke->HasPcRelativeDexCache()) { 3091 invoke->GetLocations()->SetInAt(invoke->GetSpecialInputIndex(), Location::Any()); 3092 } 3093 return; 3094 } 3095 3096 HandleInvoke(invoke); 3097 3098 // For PC-relative dex cache the invoke has an extra input, the PC-relative address base. 3099 if (invoke->HasPcRelativeDexCache()) { 3100 invoke->GetLocations()->SetInAt(invoke->GetSpecialInputIndex(), Location::RequiresRegister()); 3101 } 3102} 3103 3104static bool TryGenerateIntrinsicCode(HInvoke* invoke, CodeGeneratorARM* codegen) { 3105 if (invoke->GetLocations()->Intrinsified()) { 3106 IntrinsicCodeGeneratorARM intrinsic(codegen); 3107 intrinsic.Dispatch(invoke); 3108 return true; 3109 } 3110 return false; 3111} 3112 3113void InstructionCodeGeneratorARM::VisitInvokeStaticOrDirect(HInvokeStaticOrDirect* invoke) { 3114 // Explicit clinit checks triggered by static invokes must have been pruned by 3115 // art::PrepareForRegisterAllocation. 3116 DCHECK(!invoke->IsStaticWithExplicitClinitCheck()); 3117 3118 if (TryGenerateIntrinsicCode(invoke, codegen_)) { 3119 return; 3120 } 3121 3122 LocationSummary* locations = invoke->GetLocations(); 3123 codegen_->GenerateStaticOrDirectCall( 3124 invoke, locations->HasTemps() ? locations->GetTemp(0) : Location::NoLocation()); 3125 codegen_->RecordPcInfo(invoke, invoke->GetDexPc()); 3126} 3127 3128void LocationsBuilderARM::HandleInvoke(HInvoke* invoke) { 3129 InvokeDexCallingConventionVisitorARM calling_convention_visitor; 3130 CodeGenerator::CreateCommonInvokeLocationSummary(invoke, &calling_convention_visitor); 3131} 3132 3133void LocationsBuilderARM::VisitInvokeVirtual(HInvokeVirtual* invoke) { 3134 IntrinsicLocationsBuilderARM intrinsic(codegen_); 3135 if (intrinsic.TryDispatch(invoke)) { 3136 return; 3137 } 3138 3139 HandleInvoke(invoke); 3140} 3141 3142void InstructionCodeGeneratorARM::VisitInvokeVirtual(HInvokeVirtual* invoke) { 3143 if (TryGenerateIntrinsicCode(invoke, codegen_)) { 3144 return; 3145 } 3146 3147 codegen_->GenerateVirtualCall(invoke, invoke->GetLocations()->GetTemp(0)); 3148 DCHECK(!codegen_->IsLeafMethod()); 3149 codegen_->RecordPcInfo(invoke, invoke->GetDexPc()); 3150} 3151 3152void LocationsBuilderARM::VisitInvokeInterface(HInvokeInterface* invoke) { 3153 HandleInvoke(invoke); 3154 // Add the hidden argument. 3155 invoke->GetLocations()->AddTemp(Location::RegisterLocation(R12)); 3156} 3157 3158void InstructionCodeGeneratorARM::VisitInvokeInterface(HInvokeInterface* invoke) { 3159 // TODO: b/18116999, our IMTs can miss an IncompatibleClassChangeError. 3160 LocationSummary* locations = invoke->GetLocations(); 3161 Register temp = locations->GetTemp(0).AsRegister<Register>(); 3162 Register hidden_reg = locations->GetTemp(1).AsRegister<Register>(); 3163 Location receiver = locations->InAt(0); 3164 uint32_t class_offset = mirror::Object::ClassOffset().Int32Value(); 3165 3166 // Set the hidden argument. This is safe to do this here, as R12 3167 // won't be modified thereafter, before the `blx` (call) instruction. 3168 DCHECK_EQ(R12, hidden_reg); 3169 __ LoadImmediate(hidden_reg, invoke->GetDexMethodIndex()); 3170 3171 if (receiver.IsStackSlot()) { 3172 __ LoadFromOffset(kLoadWord, temp, SP, receiver.GetStackIndex()); 3173 // /* HeapReference<Class> */ temp = temp->klass_ 3174 __ LoadFromOffset(kLoadWord, temp, temp, class_offset); 3175 } else { 3176 // /* HeapReference<Class> */ temp = receiver->klass_ 3177 __ LoadFromOffset(kLoadWord, temp, receiver.AsRegister<Register>(), class_offset); 3178 } 3179 codegen_->MaybeRecordImplicitNullCheck(invoke); 3180 // Instead of simply (possibly) unpoisoning `temp` here, we should 3181 // emit a read barrier for the previous class reference load. 3182 // However this is not required in practice, as this is an 3183 // intermediate/temporary reference and because the current 3184 // concurrent copying collector keeps the from-space memory 3185 // intact/accessible until the end of the marking phase (the 3186 // concurrent copying collector may not in the future). 3187 __ MaybeUnpoisonHeapReference(temp); 3188 __ LoadFromOffset(kLoadWord, temp, temp, 3189 mirror::Class::ImtPtrOffset(kArmPointerSize).Uint32Value()); 3190 uint32_t method_offset = static_cast<uint32_t>(ImTable::OffsetOfElement( 3191 invoke->GetImtIndex(), kArmPointerSize)); 3192 // temp = temp->GetImtEntryAt(method_offset); 3193 __ LoadFromOffset(kLoadWord, temp, temp, method_offset); 3194 uint32_t entry_point = 3195 ArtMethod::EntryPointFromQuickCompiledCodeOffset(kArmPointerSize).Int32Value(); 3196 // LR = temp->GetEntryPoint(); 3197 __ LoadFromOffset(kLoadWord, LR, temp, entry_point); 3198 // LR(); 3199 __ blx(LR); 3200 DCHECK(!codegen_->IsLeafMethod()); 3201 codegen_->RecordPcInfo(invoke, invoke->GetDexPc()); 3202} 3203 3204void LocationsBuilderARM::VisitInvokePolymorphic(HInvokePolymorphic* invoke) { 3205 HandleInvoke(invoke); 3206} 3207 3208void InstructionCodeGeneratorARM::VisitInvokePolymorphic(HInvokePolymorphic* invoke) { 3209 codegen_->GenerateInvokePolymorphicCall(invoke); 3210} 3211 3212void LocationsBuilderARM::VisitNeg(HNeg* neg) { 3213 LocationSummary* locations = 3214 new (GetGraph()->GetArena()) LocationSummary(neg, LocationSummary::kNoCall); 3215 switch (neg->GetResultType()) { 3216 case Primitive::kPrimInt: { 3217 locations->SetInAt(0, Location::RequiresRegister()); 3218 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 3219 break; 3220 } 3221 case Primitive::kPrimLong: { 3222 locations->SetInAt(0, Location::RequiresRegister()); 3223 locations->SetOut(Location::RequiresRegister(), Location::kOutputOverlap); 3224 break; 3225 } 3226 3227 case Primitive::kPrimFloat: 3228 case Primitive::kPrimDouble: 3229 locations->SetInAt(0, Location::RequiresFpuRegister()); 3230 locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap); 3231 break; 3232 3233 default: 3234 LOG(FATAL) << "Unexpected neg type " << neg->GetResultType(); 3235 } 3236} 3237 3238void InstructionCodeGeneratorARM::VisitNeg(HNeg* neg) { 3239 LocationSummary* locations = neg->GetLocations(); 3240 Location out = locations->Out(); 3241 Location in = locations->InAt(0); 3242 switch (neg->GetResultType()) { 3243 case Primitive::kPrimInt: 3244 DCHECK(in.IsRegister()); 3245 __ rsb(out.AsRegister<Register>(), in.AsRegister<Register>(), ShifterOperand(0)); 3246 break; 3247 3248 case Primitive::kPrimLong: 3249 DCHECK(in.IsRegisterPair()); 3250 // out.lo = 0 - in.lo (and update the carry/borrow (C) flag) 3251 __ rsbs(out.AsRegisterPairLow<Register>(), 3252 in.AsRegisterPairLow<Register>(), 3253 ShifterOperand(0)); 3254 // We cannot emit an RSC (Reverse Subtract with Carry) 3255 // instruction here, as it does not exist in the Thumb-2 3256 // instruction set. We use the following approach 3257 // using SBC and SUB instead. 3258 // 3259 // out.hi = -C 3260 __ sbc(out.AsRegisterPairHigh<Register>(), 3261 out.AsRegisterPairHigh<Register>(), 3262 ShifterOperand(out.AsRegisterPairHigh<Register>())); 3263 // out.hi = out.hi - in.hi 3264 __ sub(out.AsRegisterPairHigh<Register>(), 3265 out.AsRegisterPairHigh<Register>(), 3266 ShifterOperand(in.AsRegisterPairHigh<Register>())); 3267 break; 3268 3269 case Primitive::kPrimFloat: 3270 DCHECK(in.IsFpuRegister()); 3271 __ vnegs(out.AsFpuRegister<SRegister>(), in.AsFpuRegister<SRegister>()); 3272 break; 3273 3274 case Primitive::kPrimDouble: 3275 DCHECK(in.IsFpuRegisterPair()); 3276 __ vnegd(FromLowSToD(out.AsFpuRegisterPairLow<SRegister>()), 3277 FromLowSToD(in.AsFpuRegisterPairLow<SRegister>())); 3278 break; 3279 3280 default: 3281 LOG(FATAL) << "Unexpected neg type " << neg->GetResultType(); 3282 } 3283} 3284 3285void LocationsBuilderARM::VisitTypeConversion(HTypeConversion* conversion) { 3286 Primitive::Type result_type = conversion->GetResultType(); 3287 Primitive::Type input_type = conversion->GetInputType(); 3288 DCHECK_NE(result_type, input_type); 3289 3290 // The float-to-long, double-to-long and long-to-float type conversions 3291 // rely on a call to the runtime. 3292 LocationSummary::CallKind call_kind = 3293 (((input_type == Primitive::kPrimFloat || input_type == Primitive::kPrimDouble) 3294 && result_type == Primitive::kPrimLong) 3295 || (input_type == Primitive::kPrimLong && result_type == Primitive::kPrimFloat)) 3296 ? LocationSummary::kCallOnMainOnly 3297 : LocationSummary::kNoCall; 3298 LocationSummary* locations = 3299 new (GetGraph()->GetArena()) LocationSummary(conversion, call_kind); 3300 3301 // The Java language does not allow treating boolean as an integral type but 3302 // our bit representation makes it safe. 3303 3304 switch (result_type) { 3305 case Primitive::kPrimByte: 3306 switch (input_type) { 3307 case Primitive::kPrimLong: 3308 // Type conversion from long to byte is a result of code transformations. 3309 case Primitive::kPrimBoolean: 3310 // Boolean input is a result of code transformations. 3311 case Primitive::kPrimShort: 3312 case Primitive::kPrimInt: 3313 case Primitive::kPrimChar: 3314 // Processing a Dex `int-to-byte' instruction. 3315 locations->SetInAt(0, Location::RequiresRegister()); 3316 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 3317 break; 3318 3319 default: 3320 LOG(FATAL) << "Unexpected type conversion from " << input_type 3321 << " to " << result_type; 3322 } 3323 break; 3324 3325 case Primitive::kPrimShort: 3326 switch (input_type) { 3327 case Primitive::kPrimLong: 3328 // Type conversion from long to short is a result of code transformations. 3329 case Primitive::kPrimBoolean: 3330 // Boolean input is a result of code transformations. 3331 case Primitive::kPrimByte: 3332 case Primitive::kPrimInt: 3333 case Primitive::kPrimChar: 3334 // Processing a Dex `int-to-short' instruction. 3335 locations->SetInAt(0, Location::RequiresRegister()); 3336 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 3337 break; 3338 3339 default: 3340 LOG(FATAL) << "Unexpected type conversion from " << input_type 3341 << " to " << result_type; 3342 } 3343 break; 3344 3345 case Primitive::kPrimInt: 3346 switch (input_type) { 3347 case Primitive::kPrimLong: 3348 // Processing a Dex `long-to-int' instruction. 3349 locations->SetInAt(0, Location::Any()); 3350 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 3351 break; 3352 3353 case Primitive::kPrimFloat: 3354 // Processing a Dex `float-to-int' instruction. 3355 locations->SetInAt(0, Location::RequiresFpuRegister()); 3356 locations->SetOut(Location::RequiresRegister()); 3357 locations->AddTemp(Location::RequiresFpuRegister()); 3358 break; 3359 3360 case Primitive::kPrimDouble: 3361 // Processing a Dex `double-to-int' instruction. 3362 locations->SetInAt(0, Location::RequiresFpuRegister()); 3363 locations->SetOut(Location::RequiresRegister()); 3364 locations->AddTemp(Location::RequiresFpuRegister()); 3365 break; 3366 3367 default: 3368 LOG(FATAL) << "Unexpected type conversion from " << input_type 3369 << " to " << result_type; 3370 } 3371 break; 3372 3373 case Primitive::kPrimLong: 3374 switch (input_type) { 3375 case Primitive::kPrimBoolean: 3376 // Boolean input is a result of code transformations. 3377 case Primitive::kPrimByte: 3378 case Primitive::kPrimShort: 3379 case Primitive::kPrimInt: 3380 case Primitive::kPrimChar: 3381 // Processing a Dex `int-to-long' instruction. 3382 locations->SetInAt(0, Location::RequiresRegister()); 3383 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 3384 break; 3385 3386 case Primitive::kPrimFloat: { 3387 // Processing a Dex `float-to-long' instruction. 3388 InvokeRuntimeCallingConvention calling_convention; 3389 locations->SetInAt(0, Location::FpuRegisterLocation( 3390 calling_convention.GetFpuRegisterAt(0))); 3391 locations->SetOut(Location::RegisterPairLocation(R0, R1)); 3392 break; 3393 } 3394 3395 case Primitive::kPrimDouble: { 3396 // Processing a Dex `double-to-long' instruction. 3397 InvokeRuntimeCallingConvention calling_convention; 3398 locations->SetInAt(0, Location::FpuRegisterPairLocation( 3399 calling_convention.GetFpuRegisterAt(0), 3400 calling_convention.GetFpuRegisterAt(1))); 3401 locations->SetOut(Location::RegisterPairLocation(R0, R1)); 3402 break; 3403 } 3404 3405 default: 3406 LOG(FATAL) << "Unexpected type conversion from " << input_type 3407 << " to " << result_type; 3408 } 3409 break; 3410 3411 case Primitive::kPrimChar: 3412 switch (input_type) { 3413 case Primitive::kPrimLong: 3414 // Type conversion from long to char is a result of code transformations. 3415 case Primitive::kPrimBoolean: 3416 // Boolean input is a result of code transformations. 3417 case Primitive::kPrimByte: 3418 case Primitive::kPrimShort: 3419 case Primitive::kPrimInt: 3420 // Processing a Dex `int-to-char' instruction. 3421 locations->SetInAt(0, Location::RequiresRegister()); 3422 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 3423 break; 3424 3425 default: 3426 LOG(FATAL) << "Unexpected type conversion from " << input_type 3427 << " to " << result_type; 3428 } 3429 break; 3430 3431 case Primitive::kPrimFloat: 3432 switch (input_type) { 3433 case Primitive::kPrimBoolean: 3434 // Boolean input is a result of code transformations. 3435 case Primitive::kPrimByte: 3436 case Primitive::kPrimShort: 3437 case Primitive::kPrimInt: 3438 case Primitive::kPrimChar: 3439 // Processing a Dex `int-to-float' instruction. 3440 locations->SetInAt(0, Location::RequiresRegister()); 3441 locations->SetOut(Location::RequiresFpuRegister()); 3442 break; 3443 3444 case Primitive::kPrimLong: { 3445 // Processing a Dex `long-to-float' instruction. 3446 InvokeRuntimeCallingConvention calling_convention; 3447 locations->SetInAt(0, Location::RegisterPairLocation( 3448 calling_convention.GetRegisterAt(0), calling_convention.GetRegisterAt(1))); 3449 locations->SetOut(Location::FpuRegisterLocation(calling_convention.GetFpuRegisterAt(0))); 3450 break; 3451 } 3452 3453 case Primitive::kPrimDouble: 3454 // Processing a Dex `double-to-float' instruction. 3455 locations->SetInAt(0, Location::RequiresFpuRegister()); 3456 locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap); 3457 break; 3458 3459 default: 3460 LOG(FATAL) << "Unexpected type conversion from " << input_type 3461 << " to " << result_type; 3462 }; 3463 break; 3464 3465 case Primitive::kPrimDouble: 3466 switch (input_type) { 3467 case Primitive::kPrimBoolean: 3468 // Boolean input is a result of code transformations. 3469 case Primitive::kPrimByte: 3470 case Primitive::kPrimShort: 3471 case Primitive::kPrimInt: 3472 case Primitive::kPrimChar: 3473 // Processing a Dex `int-to-double' instruction. 3474 locations->SetInAt(0, Location::RequiresRegister()); 3475 locations->SetOut(Location::RequiresFpuRegister()); 3476 break; 3477 3478 case Primitive::kPrimLong: 3479 // Processing a Dex `long-to-double' instruction. 3480 locations->SetInAt(0, Location::RequiresRegister()); 3481 locations->SetOut(Location::RequiresFpuRegister()); 3482 locations->AddTemp(Location::RequiresFpuRegister()); 3483 locations->AddTemp(Location::RequiresFpuRegister()); 3484 break; 3485 3486 case Primitive::kPrimFloat: 3487 // Processing a Dex `float-to-double' instruction. 3488 locations->SetInAt(0, Location::RequiresFpuRegister()); 3489 locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap); 3490 break; 3491 3492 default: 3493 LOG(FATAL) << "Unexpected type conversion from " << input_type 3494 << " to " << result_type; 3495 }; 3496 break; 3497 3498 default: 3499 LOG(FATAL) << "Unexpected type conversion from " << input_type 3500 << " to " << result_type; 3501 } 3502} 3503 3504void InstructionCodeGeneratorARM::VisitTypeConversion(HTypeConversion* conversion) { 3505 LocationSummary* locations = conversion->GetLocations(); 3506 Location out = locations->Out(); 3507 Location in = locations->InAt(0); 3508 Primitive::Type result_type = conversion->GetResultType(); 3509 Primitive::Type input_type = conversion->GetInputType(); 3510 DCHECK_NE(result_type, input_type); 3511 switch (result_type) { 3512 case Primitive::kPrimByte: 3513 switch (input_type) { 3514 case Primitive::kPrimLong: 3515 // Type conversion from long to byte is a result of code transformations. 3516 __ sbfx(out.AsRegister<Register>(), in.AsRegisterPairLow<Register>(), 0, 8); 3517 break; 3518 case Primitive::kPrimBoolean: 3519 // Boolean input is a result of code transformations. 3520 case Primitive::kPrimShort: 3521 case Primitive::kPrimInt: 3522 case Primitive::kPrimChar: 3523 // Processing a Dex `int-to-byte' instruction. 3524 __ sbfx(out.AsRegister<Register>(), in.AsRegister<Register>(), 0, 8); 3525 break; 3526 3527 default: 3528 LOG(FATAL) << "Unexpected type conversion from " << input_type 3529 << " to " << result_type; 3530 } 3531 break; 3532 3533 case Primitive::kPrimShort: 3534 switch (input_type) { 3535 case Primitive::kPrimLong: 3536 // Type conversion from long to short is a result of code transformations. 3537 __ sbfx(out.AsRegister<Register>(), in.AsRegisterPairLow<Register>(), 0, 16); 3538 break; 3539 case Primitive::kPrimBoolean: 3540 // Boolean input is a result of code transformations. 3541 case Primitive::kPrimByte: 3542 case Primitive::kPrimInt: 3543 case Primitive::kPrimChar: 3544 // Processing a Dex `int-to-short' instruction. 3545 __ sbfx(out.AsRegister<Register>(), in.AsRegister<Register>(), 0, 16); 3546 break; 3547 3548 default: 3549 LOG(FATAL) << "Unexpected type conversion from " << input_type 3550 << " to " << result_type; 3551 } 3552 break; 3553 3554 case Primitive::kPrimInt: 3555 switch (input_type) { 3556 case Primitive::kPrimLong: 3557 // Processing a Dex `long-to-int' instruction. 3558 DCHECK(out.IsRegister()); 3559 if (in.IsRegisterPair()) { 3560 __ Mov(out.AsRegister<Register>(), in.AsRegisterPairLow<Register>()); 3561 } else if (in.IsDoubleStackSlot()) { 3562 __ LoadFromOffset(kLoadWord, out.AsRegister<Register>(), SP, in.GetStackIndex()); 3563 } else { 3564 DCHECK(in.IsConstant()); 3565 DCHECK(in.GetConstant()->IsLongConstant()); 3566 int64_t value = in.GetConstant()->AsLongConstant()->GetValue(); 3567 __ LoadImmediate(out.AsRegister<Register>(), static_cast<int32_t>(value)); 3568 } 3569 break; 3570 3571 case Primitive::kPrimFloat: { 3572 // Processing a Dex `float-to-int' instruction. 3573 SRegister temp = locations->GetTemp(0).AsFpuRegisterPairLow<SRegister>(); 3574 __ vcvtis(temp, in.AsFpuRegister<SRegister>()); 3575 __ vmovrs(out.AsRegister<Register>(), temp); 3576 break; 3577 } 3578 3579 case Primitive::kPrimDouble: { 3580 // Processing a Dex `double-to-int' instruction. 3581 SRegister temp_s = locations->GetTemp(0).AsFpuRegisterPairLow<SRegister>(); 3582 __ vcvtid(temp_s, FromLowSToD(in.AsFpuRegisterPairLow<SRegister>())); 3583 __ vmovrs(out.AsRegister<Register>(), temp_s); 3584 break; 3585 } 3586 3587 default: 3588 LOG(FATAL) << "Unexpected type conversion from " << input_type 3589 << " to " << result_type; 3590 } 3591 break; 3592 3593 case Primitive::kPrimLong: 3594 switch (input_type) { 3595 case Primitive::kPrimBoolean: 3596 // Boolean input is a result of code transformations. 3597 case Primitive::kPrimByte: 3598 case Primitive::kPrimShort: 3599 case Primitive::kPrimInt: 3600 case Primitive::kPrimChar: 3601 // Processing a Dex `int-to-long' instruction. 3602 DCHECK(out.IsRegisterPair()); 3603 DCHECK(in.IsRegister()); 3604 __ Mov(out.AsRegisterPairLow<Register>(), in.AsRegister<Register>()); 3605 // Sign extension. 3606 __ Asr(out.AsRegisterPairHigh<Register>(), 3607 out.AsRegisterPairLow<Register>(), 3608 31); 3609 break; 3610 3611 case Primitive::kPrimFloat: 3612 // Processing a Dex `float-to-long' instruction. 3613 codegen_->InvokeRuntime(kQuickF2l, conversion, conversion->GetDexPc()); 3614 CheckEntrypointTypes<kQuickF2l, int64_t, float>(); 3615 break; 3616 3617 case Primitive::kPrimDouble: 3618 // Processing a Dex `double-to-long' instruction. 3619 codegen_->InvokeRuntime(kQuickD2l, conversion, conversion->GetDexPc()); 3620 CheckEntrypointTypes<kQuickD2l, int64_t, double>(); 3621 break; 3622 3623 default: 3624 LOG(FATAL) << "Unexpected type conversion from " << input_type 3625 << " to " << result_type; 3626 } 3627 break; 3628 3629 case Primitive::kPrimChar: 3630 switch (input_type) { 3631 case Primitive::kPrimLong: 3632 // Type conversion from long to char is a result of code transformations. 3633 __ ubfx(out.AsRegister<Register>(), in.AsRegisterPairLow<Register>(), 0, 16); 3634 break; 3635 case Primitive::kPrimBoolean: 3636 // Boolean input is a result of code transformations. 3637 case Primitive::kPrimByte: 3638 case Primitive::kPrimShort: 3639 case Primitive::kPrimInt: 3640 // Processing a Dex `int-to-char' instruction. 3641 __ ubfx(out.AsRegister<Register>(), in.AsRegister<Register>(), 0, 16); 3642 break; 3643 3644 default: 3645 LOG(FATAL) << "Unexpected type conversion from " << input_type 3646 << " to " << result_type; 3647 } 3648 break; 3649 3650 case Primitive::kPrimFloat: 3651 switch (input_type) { 3652 case Primitive::kPrimBoolean: 3653 // Boolean input is a result of code transformations. 3654 case Primitive::kPrimByte: 3655 case Primitive::kPrimShort: 3656 case Primitive::kPrimInt: 3657 case Primitive::kPrimChar: { 3658 // Processing a Dex `int-to-float' instruction. 3659 __ vmovsr(out.AsFpuRegister<SRegister>(), in.AsRegister<Register>()); 3660 __ vcvtsi(out.AsFpuRegister<SRegister>(), out.AsFpuRegister<SRegister>()); 3661 break; 3662 } 3663 3664 case Primitive::kPrimLong: 3665 // Processing a Dex `long-to-float' instruction. 3666 codegen_->InvokeRuntime(kQuickL2f, conversion, conversion->GetDexPc()); 3667 CheckEntrypointTypes<kQuickL2f, float, int64_t>(); 3668 break; 3669 3670 case Primitive::kPrimDouble: 3671 // Processing a Dex `double-to-float' instruction. 3672 __ vcvtsd(out.AsFpuRegister<SRegister>(), 3673 FromLowSToD(in.AsFpuRegisterPairLow<SRegister>())); 3674 break; 3675 3676 default: 3677 LOG(FATAL) << "Unexpected type conversion from " << input_type 3678 << " to " << result_type; 3679 }; 3680 break; 3681 3682 case Primitive::kPrimDouble: 3683 switch (input_type) { 3684 case Primitive::kPrimBoolean: 3685 // Boolean input is a result of code transformations. 3686 case Primitive::kPrimByte: 3687 case Primitive::kPrimShort: 3688 case Primitive::kPrimInt: 3689 case Primitive::kPrimChar: { 3690 // Processing a Dex `int-to-double' instruction. 3691 __ vmovsr(out.AsFpuRegisterPairLow<SRegister>(), in.AsRegister<Register>()); 3692 __ vcvtdi(FromLowSToD(out.AsFpuRegisterPairLow<SRegister>()), 3693 out.AsFpuRegisterPairLow<SRegister>()); 3694 break; 3695 } 3696 3697 case Primitive::kPrimLong: { 3698 // Processing a Dex `long-to-double' instruction. 3699 Register low = in.AsRegisterPairLow<Register>(); 3700 Register high = in.AsRegisterPairHigh<Register>(); 3701 SRegister out_s = out.AsFpuRegisterPairLow<SRegister>(); 3702 DRegister out_d = FromLowSToD(out_s); 3703 SRegister temp_s = locations->GetTemp(0).AsFpuRegisterPairLow<SRegister>(); 3704 DRegister temp_d = FromLowSToD(temp_s); 3705 SRegister constant_s = locations->GetTemp(1).AsFpuRegisterPairLow<SRegister>(); 3706 DRegister constant_d = FromLowSToD(constant_s); 3707 3708 // temp_d = int-to-double(high) 3709 __ vmovsr(temp_s, high); 3710 __ vcvtdi(temp_d, temp_s); 3711 // constant_d = k2Pow32EncodingForDouble 3712 __ LoadDImmediate(constant_d, bit_cast<double, int64_t>(k2Pow32EncodingForDouble)); 3713 // out_d = unsigned-to-double(low) 3714 __ vmovsr(out_s, low); 3715 __ vcvtdu(out_d, out_s); 3716 // out_d += temp_d * constant_d 3717 __ vmlad(out_d, temp_d, constant_d); 3718 break; 3719 } 3720 3721 case Primitive::kPrimFloat: 3722 // Processing a Dex `float-to-double' instruction. 3723 __ vcvtds(FromLowSToD(out.AsFpuRegisterPairLow<SRegister>()), 3724 in.AsFpuRegister<SRegister>()); 3725 break; 3726 3727 default: 3728 LOG(FATAL) << "Unexpected type conversion from " << input_type 3729 << " to " << result_type; 3730 }; 3731 break; 3732 3733 default: 3734 LOG(FATAL) << "Unexpected type conversion from " << input_type 3735 << " to " << result_type; 3736 } 3737} 3738 3739void LocationsBuilderARM::VisitAdd(HAdd* add) { 3740 LocationSummary* locations = 3741 new (GetGraph()->GetArena()) LocationSummary(add, LocationSummary::kNoCall); 3742 switch (add->GetResultType()) { 3743 case Primitive::kPrimInt: { 3744 locations->SetInAt(0, Location::RequiresRegister()); 3745 locations->SetInAt(1, Location::RegisterOrConstant(add->InputAt(1))); 3746 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 3747 break; 3748 } 3749 3750 case Primitive::kPrimLong: { 3751 locations->SetInAt(0, Location::RequiresRegister()); 3752 locations->SetInAt(1, ArmEncodableConstantOrRegister(add->InputAt(1), ADD)); 3753 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 3754 break; 3755 } 3756 3757 case Primitive::kPrimFloat: 3758 case Primitive::kPrimDouble: { 3759 locations->SetInAt(0, Location::RequiresFpuRegister()); 3760 locations->SetInAt(1, Location::RequiresFpuRegister()); 3761 locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap); 3762 break; 3763 } 3764 3765 default: 3766 LOG(FATAL) << "Unexpected add type " << add->GetResultType(); 3767 } 3768} 3769 3770void InstructionCodeGeneratorARM::VisitAdd(HAdd* add) { 3771 LocationSummary* locations = add->GetLocations(); 3772 Location out = locations->Out(); 3773 Location first = locations->InAt(0); 3774 Location second = locations->InAt(1); 3775 switch (add->GetResultType()) { 3776 case Primitive::kPrimInt: 3777 if (second.IsRegister()) { 3778 __ add(out.AsRegister<Register>(), 3779 first.AsRegister<Register>(), 3780 ShifterOperand(second.AsRegister<Register>())); 3781 } else { 3782 __ AddConstant(out.AsRegister<Register>(), 3783 first.AsRegister<Register>(), 3784 second.GetConstant()->AsIntConstant()->GetValue()); 3785 } 3786 break; 3787 3788 case Primitive::kPrimLong: { 3789 if (second.IsConstant()) { 3790 uint64_t value = static_cast<uint64_t>(Int64FromConstant(second.GetConstant())); 3791 GenerateAddLongConst(out, first, value); 3792 } else { 3793 DCHECK(second.IsRegisterPair()); 3794 __ adds(out.AsRegisterPairLow<Register>(), 3795 first.AsRegisterPairLow<Register>(), 3796 ShifterOperand(second.AsRegisterPairLow<Register>())); 3797 __ adc(out.AsRegisterPairHigh<Register>(), 3798 first.AsRegisterPairHigh<Register>(), 3799 ShifterOperand(second.AsRegisterPairHigh<Register>())); 3800 } 3801 break; 3802 } 3803 3804 case Primitive::kPrimFloat: 3805 __ vadds(out.AsFpuRegister<SRegister>(), 3806 first.AsFpuRegister<SRegister>(), 3807 second.AsFpuRegister<SRegister>()); 3808 break; 3809 3810 case Primitive::kPrimDouble: 3811 __ vaddd(FromLowSToD(out.AsFpuRegisterPairLow<SRegister>()), 3812 FromLowSToD(first.AsFpuRegisterPairLow<SRegister>()), 3813 FromLowSToD(second.AsFpuRegisterPairLow<SRegister>())); 3814 break; 3815 3816 default: 3817 LOG(FATAL) << "Unexpected add type " << add->GetResultType(); 3818 } 3819} 3820 3821void LocationsBuilderARM::VisitSub(HSub* sub) { 3822 LocationSummary* locations = 3823 new (GetGraph()->GetArena()) LocationSummary(sub, LocationSummary::kNoCall); 3824 switch (sub->GetResultType()) { 3825 case Primitive::kPrimInt: { 3826 locations->SetInAt(0, Location::RequiresRegister()); 3827 locations->SetInAt(1, Location::RegisterOrConstant(sub->InputAt(1))); 3828 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 3829 break; 3830 } 3831 3832 case Primitive::kPrimLong: { 3833 locations->SetInAt(0, Location::RequiresRegister()); 3834 locations->SetInAt(1, ArmEncodableConstantOrRegister(sub->InputAt(1), SUB)); 3835 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 3836 break; 3837 } 3838 case Primitive::kPrimFloat: 3839 case Primitive::kPrimDouble: { 3840 locations->SetInAt(0, Location::RequiresFpuRegister()); 3841 locations->SetInAt(1, Location::RequiresFpuRegister()); 3842 locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap); 3843 break; 3844 } 3845 default: 3846 LOG(FATAL) << "Unexpected sub type " << sub->GetResultType(); 3847 } 3848} 3849 3850void InstructionCodeGeneratorARM::VisitSub(HSub* sub) { 3851 LocationSummary* locations = sub->GetLocations(); 3852 Location out = locations->Out(); 3853 Location first = locations->InAt(0); 3854 Location second = locations->InAt(1); 3855 switch (sub->GetResultType()) { 3856 case Primitive::kPrimInt: { 3857 if (second.IsRegister()) { 3858 __ sub(out.AsRegister<Register>(), 3859 first.AsRegister<Register>(), 3860 ShifterOperand(second.AsRegister<Register>())); 3861 } else { 3862 __ AddConstant(out.AsRegister<Register>(), 3863 first.AsRegister<Register>(), 3864 -second.GetConstant()->AsIntConstant()->GetValue()); 3865 } 3866 break; 3867 } 3868 3869 case Primitive::kPrimLong: { 3870 if (second.IsConstant()) { 3871 uint64_t value = static_cast<uint64_t>(Int64FromConstant(second.GetConstant())); 3872 GenerateAddLongConst(out, first, -value); 3873 } else { 3874 DCHECK(second.IsRegisterPair()); 3875 __ subs(out.AsRegisterPairLow<Register>(), 3876 first.AsRegisterPairLow<Register>(), 3877 ShifterOperand(second.AsRegisterPairLow<Register>())); 3878 __ sbc(out.AsRegisterPairHigh<Register>(), 3879 first.AsRegisterPairHigh<Register>(), 3880 ShifterOperand(second.AsRegisterPairHigh<Register>())); 3881 } 3882 break; 3883 } 3884 3885 case Primitive::kPrimFloat: { 3886 __ vsubs(out.AsFpuRegister<SRegister>(), 3887 first.AsFpuRegister<SRegister>(), 3888 second.AsFpuRegister<SRegister>()); 3889 break; 3890 } 3891 3892 case Primitive::kPrimDouble: { 3893 __ vsubd(FromLowSToD(out.AsFpuRegisterPairLow<SRegister>()), 3894 FromLowSToD(first.AsFpuRegisterPairLow<SRegister>()), 3895 FromLowSToD(second.AsFpuRegisterPairLow<SRegister>())); 3896 break; 3897 } 3898 3899 3900 default: 3901 LOG(FATAL) << "Unexpected sub type " << sub->GetResultType(); 3902 } 3903} 3904 3905void LocationsBuilderARM::VisitMul(HMul* mul) { 3906 LocationSummary* locations = 3907 new (GetGraph()->GetArena()) LocationSummary(mul, LocationSummary::kNoCall); 3908 switch (mul->GetResultType()) { 3909 case Primitive::kPrimInt: 3910 case Primitive::kPrimLong: { 3911 locations->SetInAt(0, Location::RequiresRegister()); 3912 locations->SetInAt(1, Location::RequiresRegister()); 3913 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 3914 break; 3915 } 3916 3917 case Primitive::kPrimFloat: 3918 case Primitive::kPrimDouble: { 3919 locations->SetInAt(0, Location::RequiresFpuRegister()); 3920 locations->SetInAt(1, Location::RequiresFpuRegister()); 3921 locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap); 3922 break; 3923 } 3924 3925 default: 3926 LOG(FATAL) << "Unexpected mul type " << mul->GetResultType(); 3927 } 3928} 3929 3930void InstructionCodeGeneratorARM::VisitMul(HMul* mul) { 3931 LocationSummary* locations = mul->GetLocations(); 3932 Location out = locations->Out(); 3933 Location first = locations->InAt(0); 3934 Location second = locations->InAt(1); 3935 switch (mul->GetResultType()) { 3936 case Primitive::kPrimInt: { 3937 __ mul(out.AsRegister<Register>(), 3938 first.AsRegister<Register>(), 3939 second.AsRegister<Register>()); 3940 break; 3941 } 3942 case Primitive::kPrimLong: { 3943 Register out_hi = out.AsRegisterPairHigh<Register>(); 3944 Register out_lo = out.AsRegisterPairLow<Register>(); 3945 Register in1_hi = first.AsRegisterPairHigh<Register>(); 3946 Register in1_lo = first.AsRegisterPairLow<Register>(); 3947 Register in2_hi = second.AsRegisterPairHigh<Register>(); 3948 Register in2_lo = second.AsRegisterPairLow<Register>(); 3949 3950 // Extra checks to protect caused by the existence of R1_R2. 3951 // The algorithm is wrong if out.hi is either in1.lo or in2.lo: 3952 // (e.g. in1=r0_r1, in2=r2_r3 and out=r1_r2); 3953 DCHECK_NE(out_hi, in1_lo); 3954 DCHECK_NE(out_hi, in2_lo); 3955 3956 // input: in1 - 64 bits, in2 - 64 bits 3957 // output: out 3958 // formula: out.hi : out.lo = (in1.lo * in2.hi + in1.hi * in2.lo)* 2^32 + in1.lo * in2.lo 3959 // parts: out.hi = in1.lo * in2.hi + in1.hi * in2.lo + (in1.lo * in2.lo)[63:32] 3960 // parts: out.lo = (in1.lo * in2.lo)[31:0] 3961 3962 // IP <- in1.lo * in2.hi 3963 __ mul(IP, in1_lo, in2_hi); 3964 // out.hi <- in1.lo * in2.hi + in1.hi * in2.lo 3965 __ mla(out_hi, in1_hi, in2_lo, IP); 3966 // out.lo <- (in1.lo * in2.lo)[31:0]; 3967 __ umull(out_lo, IP, in1_lo, in2_lo); 3968 // out.hi <- in2.hi * in1.lo + in2.lo * in1.hi + (in1.lo * in2.lo)[63:32] 3969 __ add(out_hi, out_hi, ShifterOperand(IP)); 3970 break; 3971 } 3972 3973 case Primitive::kPrimFloat: { 3974 __ vmuls(out.AsFpuRegister<SRegister>(), 3975 first.AsFpuRegister<SRegister>(), 3976 second.AsFpuRegister<SRegister>()); 3977 break; 3978 } 3979 3980 case Primitive::kPrimDouble: { 3981 __ vmuld(FromLowSToD(out.AsFpuRegisterPairLow<SRegister>()), 3982 FromLowSToD(first.AsFpuRegisterPairLow<SRegister>()), 3983 FromLowSToD(second.AsFpuRegisterPairLow<SRegister>())); 3984 break; 3985 } 3986 3987 default: 3988 LOG(FATAL) << "Unexpected mul type " << mul->GetResultType(); 3989 } 3990} 3991 3992void InstructionCodeGeneratorARM::DivRemOneOrMinusOne(HBinaryOperation* instruction) { 3993 DCHECK(instruction->IsDiv() || instruction->IsRem()); 3994 DCHECK(instruction->GetResultType() == Primitive::kPrimInt); 3995 3996 LocationSummary* locations = instruction->GetLocations(); 3997 Location second = locations->InAt(1); 3998 DCHECK(second.IsConstant()); 3999 4000 Register out = locations->Out().AsRegister<Register>(); 4001 Register dividend = locations->InAt(0).AsRegister<Register>(); 4002 int32_t imm = second.GetConstant()->AsIntConstant()->GetValue(); 4003 DCHECK(imm == 1 || imm == -1); 4004 4005 if (instruction->IsRem()) { 4006 __ LoadImmediate(out, 0); 4007 } else { 4008 if (imm == 1) { 4009 __ Mov(out, dividend); 4010 } else { 4011 __ rsb(out, dividend, ShifterOperand(0)); 4012 } 4013 } 4014} 4015 4016void InstructionCodeGeneratorARM::DivRemByPowerOfTwo(HBinaryOperation* instruction) { 4017 DCHECK(instruction->IsDiv() || instruction->IsRem()); 4018 DCHECK(instruction->GetResultType() == Primitive::kPrimInt); 4019 4020 LocationSummary* locations = instruction->GetLocations(); 4021 Location second = locations->InAt(1); 4022 DCHECK(second.IsConstant()); 4023 4024 Register out = locations->Out().AsRegister<Register>(); 4025 Register dividend = locations->InAt(0).AsRegister<Register>(); 4026 Register temp = locations->GetTemp(0).AsRegister<Register>(); 4027 int32_t imm = second.GetConstant()->AsIntConstant()->GetValue(); 4028 uint32_t abs_imm = static_cast<uint32_t>(AbsOrMin(imm)); 4029 int ctz_imm = CTZ(abs_imm); 4030 4031 if (ctz_imm == 1) { 4032 __ Lsr(temp, dividend, 32 - ctz_imm); 4033 } else { 4034 __ Asr(temp, dividend, 31); 4035 __ Lsr(temp, temp, 32 - ctz_imm); 4036 } 4037 __ add(out, temp, ShifterOperand(dividend)); 4038 4039 if (instruction->IsDiv()) { 4040 __ Asr(out, out, ctz_imm); 4041 if (imm < 0) { 4042 __ rsb(out, out, ShifterOperand(0)); 4043 } 4044 } else { 4045 __ ubfx(out, out, 0, ctz_imm); 4046 __ sub(out, out, ShifterOperand(temp)); 4047 } 4048} 4049 4050void InstructionCodeGeneratorARM::GenerateDivRemWithAnyConstant(HBinaryOperation* instruction) { 4051 DCHECK(instruction->IsDiv() || instruction->IsRem()); 4052 DCHECK(instruction->GetResultType() == Primitive::kPrimInt); 4053 4054 LocationSummary* locations = instruction->GetLocations(); 4055 Location second = locations->InAt(1); 4056 DCHECK(second.IsConstant()); 4057 4058 Register out = locations->Out().AsRegister<Register>(); 4059 Register dividend = locations->InAt(0).AsRegister<Register>(); 4060 Register temp1 = locations->GetTemp(0).AsRegister<Register>(); 4061 Register temp2 = locations->GetTemp(1).AsRegister<Register>(); 4062 int64_t imm = second.GetConstant()->AsIntConstant()->GetValue(); 4063 4064 int64_t magic; 4065 int shift; 4066 CalculateMagicAndShiftForDivRem(imm, false /* is_long */, &magic, &shift); 4067 4068 __ LoadImmediate(temp1, magic); 4069 __ smull(temp2, temp1, dividend, temp1); 4070 4071 if (imm > 0 && magic < 0) { 4072 __ add(temp1, temp1, ShifterOperand(dividend)); 4073 } else if (imm < 0 && magic > 0) { 4074 __ sub(temp1, temp1, ShifterOperand(dividend)); 4075 } 4076 4077 if (shift != 0) { 4078 __ Asr(temp1, temp1, shift); 4079 } 4080 4081 if (instruction->IsDiv()) { 4082 __ sub(out, temp1, ShifterOperand(temp1, ASR, 31)); 4083 } else { 4084 __ sub(temp1, temp1, ShifterOperand(temp1, ASR, 31)); 4085 // TODO: Strength reduction for mls. 4086 __ LoadImmediate(temp2, imm); 4087 __ mls(out, temp1, temp2, dividend); 4088 } 4089} 4090 4091void InstructionCodeGeneratorARM::GenerateDivRemConstantIntegral(HBinaryOperation* instruction) { 4092 DCHECK(instruction->IsDiv() || instruction->IsRem()); 4093 DCHECK(instruction->GetResultType() == Primitive::kPrimInt); 4094 4095 LocationSummary* locations = instruction->GetLocations(); 4096 Location second = locations->InAt(1); 4097 DCHECK(second.IsConstant()); 4098 4099 int32_t imm = second.GetConstant()->AsIntConstant()->GetValue(); 4100 if (imm == 0) { 4101 // Do not generate anything. DivZeroCheck would prevent any code to be executed. 4102 } else if (imm == 1 || imm == -1) { 4103 DivRemOneOrMinusOne(instruction); 4104 } else if (IsPowerOfTwo(AbsOrMin(imm))) { 4105 DivRemByPowerOfTwo(instruction); 4106 } else { 4107 DCHECK(imm <= -2 || imm >= 2); 4108 GenerateDivRemWithAnyConstant(instruction); 4109 } 4110} 4111 4112void LocationsBuilderARM::VisitDiv(HDiv* div) { 4113 LocationSummary::CallKind call_kind = LocationSummary::kNoCall; 4114 if (div->GetResultType() == Primitive::kPrimLong) { 4115 // pLdiv runtime call. 4116 call_kind = LocationSummary::kCallOnMainOnly; 4117 } else if (div->GetResultType() == Primitive::kPrimInt && div->InputAt(1)->IsConstant()) { 4118 // sdiv will be replaced by other instruction sequence. 4119 } else if (div->GetResultType() == Primitive::kPrimInt && 4120 !codegen_->GetInstructionSetFeatures().HasDivideInstruction()) { 4121 // pIdivmod runtime call. 4122 call_kind = LocationSummary::kCallOnMainOnly; 4123 } 4124 4125 LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(div, call_kind); 4126 4127 switch (div->GetResultType()) { 4128 case Primitive::kPrimInt: { 4129 if (div->InputAt(1)->IsConstant()) { 4130 locations->SetInAt(0, Location::RequiresRegister()); 4131 locations->SetInAt(1, Location::ConstantLocation(div->InputAt(1)->AsConstant())); 4132 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 4133 int32_t value = div->InputAt(1)->AsIntConstant()->GetValue(); 4134 if (value == 1 || value == 0 || value == -1) { 4135 // No temp register required. 4136 } else { 4137 locations->AddTemp(Location::RequiresRegister()); 4138 if (!IsPowerOfTwo(AbsOrMin(value))) { 4139 locations->AddTemp(Location::RequiresRegister()); 4140 } 4141 } 4142 } else if (codegen_->GetInstructionSetFeatures().HasDivideInstruction()) { 4143 locations->SetInAt(0, Location::RequiresRegister()); 4144 locations->SetInAt(1, Location::RequiresRegister()); 4145 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 4146 } else { 4147 InvokeRuntimeCallingConvention calling_convention; 4148 locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(0))); 4149 locations->SetInAt(1, Location::RegisterLocation(calling_convention.GetRegisterAt(1))); 4150 // Note: divmod will compute both the quotient and the remainder as the pair R0 and R1, but 4151 // we only need the former. 4152 locations->SetOut(Location::RegisterLocation(R0)); 4153 } 4154 break; 4155 } 4156 case Primitive::kPrimLong: { 4157 InvokeRuntimeCallingConvention calling_convention; 4158 locations->SetInAt(0, Location::RegisterPairLocation( 4159 calling_convention.GetRegisterAt(0), calling_convention.GetRegisterAt(1))); 4160 locations->SetInAt(1, Location::RegisterPairLocation( 4161 calling_convention.GetRegisterAt(2), calling_convention.GetRegisterAt(3))); 4162 locations->SetOut(Location::RegisterPairLocation(R0, R1)); 4163 break; 4164 } 4165 case Primitive::kPrimFloat: 4166 case Primitive::kPrimDouble: { 4167 locations->SetInAt(0, Location::RequiresFpuRegister()); 4168 locations->SetInAt(1, Location::RequiresFpuRegister()); 4169 locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap); 4170 break; 4171 } 4172 4173 default: 4174 LOG(FATAL) << "Unexpected div type " << div->GetResultType(); 4175 } 4176} 4177 4178void InstructionCodeGeneratorARM::VisitDiv(HDiv* div) { 4179 LocationSummary* locations = div->GetLocations(); 4180 Location out = locations->Out(); 4181 Location first = locations->InAt(0); 4182 Location second = locations->InAt(1); 4183 4184 switch (div->GetResultType()) { 4185 case Primitive::kPrimInt: { 4186 if (second.IsConstant()) { 4187 GenerateDivRemConstantIntegral(div); 4188 } else if (codegen_->GetInstructionSetFeatures().HasDivideInstruction()) { 4189 __ sdiv(out.AsRegister<Register>(), 4190 first.AsRegister<Register>(), 4191 second.AsRegister<Register>()); 4192 } else { 4193 InvokeRuntimeCallingConvention calling_convention; 4194 DCHECK_EQ(calling_convention.GetRegisterAt(0), first.AsRegister<Register>()); 4195 DCHECK_EQ(calling_convention.GetRegisterAt(1), second.AsRegister<Register>()); 4196 DCHECK_EQ(R0, out.AsRegister<Register>()); 4197 4198 codegen_->InvokeRuntime(kQuickIdivmod, div, div->GetDexPc()); 4199 CheckEntrypointTypes<kQuickIdivmod, int32_t, int32_t, int32_t>(); 4200 } 4201 break; 4202 } 4203 4204 case Primitive::kPrimLong: { 4205 InvokeRuntimeCallingConvention calling_convention; 4206 DCHECK_EQ(calling_convention.GetRegisterAt(0), first.AsRegisterPairLow<Register>()); 4207 DCHECK_EQ(calling_convention.GetRegisterAt(1), first.AsRegisterPairHigh<Register>()); 4208 DCHECK_EQ(calling_convention.GetRegisterAt(2), second.AsRegisterPairLow<Register>()); 4209 DCHECK_EQ(calling_convention.GetRegisterAt(3), second.AsRegisterPairHigh<Register>()); 4210 DCHECK_EQ(R0, out.AsRegisterPairLow<Register>()); 4211 DCHECK_EQ(R1, out.AsRegisterPairHigh<Register>()); 4212 4213 codegen_->InvokeRuntime(kQuickLdiv, div, div->GetDexPc()); 4214 CheckEntrypointTypes<kQuickLdiv, int64_t, int64_t, int64_t>(); 4215 break; 4216 } 4217 4218 case Primitive::kPrimFloat: { 4219 __ vdivs(out.AsFpuRegister<SRegister>(), 4220 first.AsFpuRegister<SRegister>(), 4221 second.AsFpuRegister<SRegister>()); 4222 break; 4223 } 4224 4225 case Primitive::kPrimDouble: { 4226 __ vdivd(FromLowSToD(out.AsFpuRegisterPairLow<SRegister>()), 4227 FromLowSToD(first.AsFpuRegisterPairLow<SRegister>()), 4228 FromLowSToD(second.AsFpuRegisterPairLow<SRegister>())); 4229 break; 4230 } 4231 4232 default: 4233 LOG(FATAL) << "Unexpected div type " << div->GetResultType(); 4234 } 4235} 4236 4237void LocationsBuilderARM::VisitRem(HRem* rem) { 4238 Primitive::Type type = rem->GetResultType(); 4239 4240 // Most remainders are implemented in the runtime. 4241 LocationSummary::CallKind call_kind = LocationSummary::kCallOnMainOnly; 4242 if (rem->GetResultType() == Primitive::kPrimInt && rem->InputAt(1)->IsConstant()) { 4243 // sdiv will be replaced by other instruction sequence. 4244 call_kind = LocationSummary::kNoCall; 4245 } else if ((rem->GetResultType() == Primitive::kPrimInt) 4246 && codegen_->GetInstructionSetFeatures().HasDivideInstruction()) { 4247 // Have hardware divide instruction for int, do it with three instructions. 4248 call_kind = LocationSummary::kNoCall; 4249 } 4250 4251 LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(rem, call_kind); 4252 4253 switch (type) { 4254 case Primitive::kPrimInt: { 4255 if (rem->InputAt(1)->IsConstant()) { 4256 locations->SetInAt(0, Location::RequiresRegister()); 4257 locations->SetInAt(1, Location::ConstantLocation(rem->InputAt(1)->AsConstant())); 4258 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 4259 int32_t value = rem->InputAt(1)->AsIntConstant()->GetValue(); 4260 if (value == 1 || value == 0 || value == -1) { 4261 // No temp register required. 4262 } else { 4263 locations->AddTemp(Location::RequiresRegister()); 4264 if (!IsPowerOfTwo(AbsOrMin(value))) { 4265 locations->AddTemp(Location::RequiresRegister()); 4266 } 4267 } 4268 } else if (codegen_->GetInstructionSetFeatures().HasDivideInstruction()) { 4269 locations->SetInAt(0, Location::RequiresRegister()); 4270 locations->SetInAt(1, Location::RequiresRegister()); 4271 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 4272 locations->AddTemp(Location::RequiresRegister()); 4273 } else { 4274 InvokeRuntimeCallingConvention calling_convention; 4275 locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(0))); 4276 locations->SetInAt(1, Location::RegisterLocation(calling_convention.GetRegisterAt(1))); 4277 // Note: divmod will compute both the quotient and the remainder as the pair R0 and R1, but 4278 // we only need the latter. 4279 locations->SetOut(Location::RegisterLocation(R1)); 4280 } 4281 break; 4282 } 4283 case Primitive::kPrimLong: { 4284 InvokeRuntimeCallingConvention calling_convention; 4285 locations->SetInAt(0, Location::RegisterPairLocation( 4286 calling_convention.GetRegisterAt(0), calling_convention.GetRegisterAt(1))); 4287 locations->SetInAt(1, Location::RegisterPairLocation( 4288 calling_convention.GetRegisterAt(2), calling_convention.GetRegisterAt(3))); 4289 // The runtime helper puts the output in R2,R3. 4290 locations->SetOut(Location::RegisterPairLocation(R2, R3)); 4291 break; 4292 } 4293 case Primitive::kPrimFloat: { 4294 InvokeRuntimeCallingConvention calling_convention; 4295 locations->SetInAt(0, Location::FpuRegisterLocation(calling_convention.GetFpuRegisterAt(0))); 4296 locations->SetInAt(1, Location::FpuRegisterLocation(calling_convention.GetFpuRegisterAt(1))); 4297 locations->SetOut(Location::FpuRegisterLocation(S0)); 4298 break; 4299 } 4300 4301 case Primitive::kPrimDouble: { 4302 InvokeRuntimeCallingConvention calling_convention; 4303 locations->SetInAt(0, Location::FpuRegisterPairLocation( 4304 calling_convention.GetFpuRegisterAt(0), calling_convention.GetFpuRegisterAt(1))); 4305 locations->SetInAt(1, Location::FpuRegisterPairLocation( 4306 calling_convention.GetFpuRegisterAt(2), calling_convention.GetFpuRegisterAt(3))); 4307 locations->SetOut(Location::Location::FpuRegisterPairLocation(S0, S1)); 4308 break; 4309 } 4310 4311 default: 4312 LOG(FATAL) << "Unexpected rem type " << type; 4313 } 4314} 4315 4316void InstructionCodeGeneratorARM::VisitRem(HRem* rem) { 4317 LocationSummary* locations = rem->GetLocations(); 4318 Location out = locations->Out(); 4319 Location first = locations->InAt(0); 4320 Location second = locations->InAt(1); 4321 4322 Primitive::Type type = rem->GetResultType(); 4323 switch (type) { 4324 case Primitive::kPrimInt: { 4325 if (second.IsConstant()) { 4326 GenerateDivRemConstantIntegral(rem); 4327 } else if (codegen_->GetInstructionSetFeatures().HasDivideInstruction()) { 4328 Register reg1 = first.AsRegister<Register>(); 4329 Register reg2 = second.AsRegister<Register>(); 4330 Register temp = locations->GetTemp(0).AsRegister<Register>(); 4331 4332 // temp = reg1 / reg2 (integer division) 4333 // dest = reg1 - temp * reg2 4334 __ sdiv(temp, reg1, reg2); 4335 __ mls(out.AsRegister<Register>(), temp, reg2, reg1); 4336 } else { 4337 InvokeRuntimeCallingConvention calling_convention; 4338 DCHECK_EQ(calling_convention.GetRegisterAt(0), first.AsRegister<Register>()); 4339 DCHECK_EQ(calling_convention.GetRegisterAt(1), second.AsRegister<Register>()); 4340 DCHECK_EQ(R1, out.AsRegister<Register>()); 4341 4342 codegen_->InvokeRuntime(kQuickIdivmod, rem, rem->GetDexPc()); 4343 CheckEntrypointTypes<kQuickIdivmod, int32_t, int32_t, int32_t>(); 4344 } 4345 break; 4346 } 4347 4348 case Primitive::kPrimLong: { 4349 codegen_->InvokeRuntime(kQuickLmod, rem, rem->GetDexPc()); 4350 CheckEntrypointTypes<kQuickLmod, int64_t, int64_t, int64_t>(); 4351 break; 4352 } 4353 4354 case Primitive::kPrimFloat: { 4355 codegen_->InvokeRuntime(kQuickFmodf, rem, rem->GetDexPc()); 4356 CheckEntrypointTypes<kQuickFmodf, float, float, float>(); 4357 break; 4358 } 4359 4360 case Primitive::kPrimDouble: { 4361 codegen_->InvokeRuntime(kQuickFmod, rem, rem->GetDexPc()); 4362 CheckEntrypointTypes<kQuickFmod, double, double, double>(); 4363 break; 4364 } 4365 4366 default: 4367 LOG(FATAL) << "Unexpected rem type " << type; 4368 } 4369} 4370 4371void LocationsBuilderARM::VisitDivZeroCheck(HDivZeroCheck* instruction) { 4372 LocationSummary* locations = codegen_->CreateThrowingSlowPathLocations(instruction); 4373 locations->SetInAt(0, Location::RegisterOrConstant(instruction->InputAt(0))); 4374} 4375 4376void InstructionCodeGeneratorARM::VisitDivZeroCheck(HDivZeroCheck* instruction) { 4377 SlowPathCodeARM* slow_path = new (GetGraph()->GetArena()) DivZeroCheckSlowPathARM(instruction); 4378 codegen_->AddSlowPath(slow_path); 4379 4380 LocationSummary* locations = instruction->GetLocations(); 4381 Location value = locations->InAt(0); 4382 4383 switch (instruction->GetType()) { 4384 case Primitive::kPrimBoolean: 4385 case Primitive::kPrimByte: 4386 case Primitive::kPrimChar: 4387 case Primitive::kPrimShort: 4388 case Primitive::kPrimInt: { 4389 if (value.IsRegister()) { 4390 __ CompareAndBranchIfZero(value.AsRegister<Register>(), slow_path->GetEntryLabel()); 4391 } else { 4392 DCHECK(value.IsConstant()) << value; 4393 if (value.GetConstant()->AsIntConstant()->GetValue() == 0) { 4394 __ b(slow_path->GetEntryLabel()); 4395 } 4396 } 4397 break; 4398 } 4399 case Primitive::kPrimLong: { 4400 if (value.IsRegisterPair()) { 4401 __ orrs(IP, 4402 value.AsRegisterPairLow<Register>(), 4403 ShifterOperand(value.AsRegisterPairHigh<Register>())); 4404 __ b(slow_path->GetEntryLabel(), EQ); 4405 } else { 4406 DCHECK(value.IsConstant()) << value; 4407 if (value.GetConstant()->AsLongConstant()->GetValue() == 0) { 4408 __ b(slow_path->GetEntryLabel()); 4409 } 4410 } 4411 break; 4412 default: 4413 LOG(FATAL) << "Unexpected type for HDivZeroCheck " << instruction->GetType(); 4414 } 4415 } 4416} 4417 4418void InstructionCodeGeneratorARM::HandleIntegerRotate(LocationSummary* locations) { 4419 Register in = locations->InAt(0).AsRegister<Register>(); 4420 Location rhs = locations->InAt(1); 4421 Register out = locations->Out().AsRegister<Register>(); 4422 4423 if (rhs.IsConstant()) { 4424 // Arm32 and Thumb2 assemblers require a rotation on the interval [1,31], 4425 // so map all rotations to a +ve. equivalent in that range. 4426 // (e.g. left *or* right by -2 bits == 30 bits in the same direction.) 4427 uint32_t rot = CodeGenerator::GetInt32ValueOf(rhs.GetConstant()) & 0x1F; 4428 if (rot) { 4429 // Rotate, mapping left rotations to right equivalents if necessary. 4430 // (e.g. left by 2 bits == right by 30.) 4431 __ Ror(out, in, rot); 4432 } else if (out != in) { 4433 __ Mov(out, in); 4434 } 4435 } else { 4436 __ Ror(out, in, rhs.AsRegister<Register>()); 4437 } 4438} 4439 4440// Gain some speed by mapping all Long rotates onto equivalent pairs of Integer 4441// rotates by swapping input regs (effectively rotating by the first 32-bits of 4442// a larger rotation) or flipping direction (thus treating larger right/left 4443// rotations as sub-word sized rotations in the other direction) as appropriate. 4444void InstructionCodeGeneratorARM::HandleLongRotate(LocationSummary* locations) { 4445 Register in_reg_lo = locations->InAt(0).AsRegisterPairLow<Register>(); 4446 Register in_reg_hi = locations->InAt(0).AsRegisterPairHigh<Register>(); 4447 Location rhs = locations->InAt(1); 4448 Register out_reg_lo = locations->Out().AsRegisterPairLow<Register>(); 4449 Register out_reg_hi = locations->Out().AsRegisterPairHigh<Register>(); 4450 4451 if (rhs.IsConstant()) { 4452 uint64_t rot = CodeGenerator::GetInt64ValueOf(rhs.GetConstant()); 4453 // Map all rotations to +ve. equivalents on the interval [0,63]. 4454 rot &= kMaxLongShiftDistance; 4455 // For rotates over a word in size, 'pre-rotate' by 32-bits to keep rotate 4456 // logic below to a simple pair of binary orr. 4457 // (e.g. 34 bits == in_reg swap + 2 bits right.) 4458 if (rot >= kArmBitsPerWord) { 4459 rot -= kArmBitsPerWord; 4460 std::swap(in_reg_hi, in_reg_lo); 4461 } 4462 // Rotate, or mov to out for zero or word size rotations. 4463 if (rot != 0u) { 4464 __ Lsr(out_reg_hi, in_reg_hi, rot); 4465 __ orr(out_reg_hi, out_reg_hi, ShifterOperand(in_reg_lo, arm::LSL, kArmBitsPerWord - rot)); 4466 __ Lsr(out_reg_lo, in_reg_lo, rot); 4467 __ orr(out_reg_lo, out_reg_lo, ShifterOperand(in_reg_hi, arm::LSL, kArmBitsPerWord - rot)); 4468 } else { 4469 __ Mov(out_reg_lo, in_reg_lo); 4470 __ Mov(out_reg_hi, in_reg_hi); 4471 } 4472 } else { 4473 Register shift_right = locations->GetTemp(0).AsRegister<Register>(); 4474 Register shift_left = locations->GetTemp(1).AsRegister<Register>(); 4475 Label end; 4476 Label shift_by_32_plus_shift_right; 4477 4478 __ and_(shift_right, rhs.AsRegister<Register>(), ShifterOperand(0x1F)); 4479 __ Lsrs(shift_left, rhs.AsRegister<Register>(), 6); 4480 __ rsb(shift_left, shift_right, ShifterOperand(kArmBitsPerWord), AL, kCcKeep); 4481 __ b(&shift_by_32_plus_shift_right, CC); 4482 4483 // out_reg_hi = (reg_hi << shift_left) | (reg_lo >> shift_right). 4484 // out_reg_lo = (reg_lo << shift_left) | (reg_hi >> shift_right). 4485 __ Lsl(out_reg_hi, in_reg_hi, shift_left); 4486 __ Lsr(out_reg_lo, in_reg_lo, shift_right); 4487 __ add(out_reg_hi, out_reg_hi, ShifterOperand(out_reg_lo)); 4488 __ Lsl(out_reg_lo, in_reg_lo, shift_left); 4489 __ Lsr(shift_left, in_reg_hi, shift_right); 4490 __ add(out_reg_lo, out_reg_lo, ShifterOperand(shift_left)); 4491 __ b(&end); 4492 4493 __ Bind(&shift_by_32_plus_shift_right); // Shift by 32+shift_right. 4494 // out_reg_hi = (reg_hi >> shift_right) | (reg_lo << shift_left). 4495 // out_reg_lo = (reg_lo >> shift_right) | (reg_hi << shift_left). 4496 __ Lsr(out_reg_hi, in_reg_hi, shift_right); 4497 __ Lsl(out_reg_lo, in_reg_lo, shift_left); 4498 __ add(out_reg_hi, out_reg_hi, ShifterOperand(out_reg_lo)); 4499 __ Lsr(out_reg_lo, in_reg_lo, shift_right); 4500 __ Lsl(shift_right, in_reg_hi, shift_left); 4501 __ add(out_reg_lo, out_reg_lo, ShifterOperand(shift_right)); 4502 4503 __ Bind(&end); 4504 } 4505} 4506 4507void LocationsBuilderARM::VisitRor(HRor* ror) { 4508 LocationSummary* locations = 4509 new (GetGraph()->GetArena()) LocationSummary(ror, LocationSummary::kNoCall); 4510 switch (ror->GetResultType()) { 4511 case Primitive::kPrimInt: { 4512 locations->SetInAt(0, Location::RequiresRegister()); 4513 locations->SetInAt(1, Location::RegisterOrConstant(ror->InputAt(1))); 4514 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 4515 break; 4516 } 4517 case Primitive::kPrimLong: { 4518 locations->SetInAt(0, Location::RequiresRegister()); 4519 if (ror->InputAt(1)->IsConstant()) { 4520 locations->SetInAt(1, Location::ConstantLocation(ror->InputAt(1)->AsConstant())); 4521 } else { 4522 locations->SetInAt(1, Location::RequiresRegister()); 4523 locations->AddTemp(Location::RequiresRegister()); 4524 locations->AddTemp(Location::RequiresRegister()); 4525 } 4526 locations->SetOut(Location::RequiresRegister(), Location::kOutputOverlap); 4527 break; 4528 } 4529 default: 4530 LOG(FATAL) << "Unexpected operation type " << ror->GetResultType(); 4531 } 4532} 4533 4534void InstructionCodeGeneratorARM::VisitRor(HRor* ror) { 4535 LocationSummary* locations = ror->GetLocations(); 4536 Primitive::Type type = ror->GetResultType(); 4537 switch (type) { 4538 case Primitive::kPrimInt: { 4539 HandleIntegerRotate(locations); 4540 break; 4541 } 4542 case Primitive::kPrimLong: { 4543 HandleLongRotate(locations); 4544 break; 4545 } 4546 default: 4547 LOG(FATAL) << "Unexpected operation type " << type; 4548 UNREACHABLE(); 4549 } 4550} 4551 4552void LocationsBuilderARM::HandleShift(HBinaryOperation* op) { 4553 DCHECK(op->IsShl() || op->IsShr() || op->IsUShr()); 4554 4555 LocationSummary* locations = 4556 new (GetGraph()->GetArena()) LocationSummary(op, LocationSummary::kNoCall); 4557 4558 switch (op->GetResultType()) { 4559 case Primitive::kPrimInt: { 4560 locations->SetInAt(0, Location::RequiresRegister()); 4561 if (op->InputAt(1)->IsConstant()) { 4562 locations->SetInAt(1, Location::ConstantLocation(op->InputAt(1)->AsConstant())); 4563 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 4564 } else { 4565 locations->SetInAt(1, Location::RequiresRegister()); 4566 // Make the output overlap, as it will be used to hold the masked 4567 // second input. 4568 locations->SetOut(Location::RequiresRegister(), Location::kOutputOverlap); 4569 } 4570 break; 4571 } 4572 case Primitive::kPrimLong: { 4573 locations->SetInAt(0, Location::RequiresRegister()); 4574 if (op->InputAt(1)->IsConstant()) { 4575 locations->SetInAt(1, Location::ConstantLocation(op->InputAt(1)->AsConstant())); 4576 // For simplicity, use kOutputOverlap even though we only require that low registers 4577 // don't clash with high registers which the register allocator currently guarantees. 4578 locations->SetOut(Location::RequiresRegister(), Location::kOutputOverlap); 4579 } else { 4580 locations->SetInAt(1, Location::RequiresRegister()); 4581 locations->AddTemp(Location::RequiresRegister()); 4582 locations->SetOut(Location::RequiresRegister(), Location::kOutputOverlap); 4583 } 4584 break; 4585 } 4586 default: 4587 LOG(FATAL) << "Unexpected operation type " << op->GetResultType(); 4588 } 4589} 4590 4591void InstructionCodeGeneratorARM::HandleShift(HBinaryOperation* op) { 4592 DCHECK(op->IsShl() || op->IsShr() || op->IsUShr()); 4593 4594 LocationSummary* locations = op->GetLocations(); 4595 Location out = locations->Out(); 4596 Location first = locations->InAt(0); 4597 Location second = locations->InAt(1); 4598 4599 Primitive::Type type = op->GetResultType(); 4600 switch (type) { 4601 case Primitive::kPrimInt: { 4602 Register out_reg = out.AsRegister<Register>(); 4603 Register first_reg = first.AsRegister<Register>(); 4604 if (second.IsRegister()) { 4605 Register second_reg = second.AsRegister<Register>(); 4606 // ARM doesn't mask the shift count so we need to do it ourselves. 4607 __ and_(out_reg, second_reg, ShifterOperand(kMaxIntShiftDistance)); 4608 if (op->IsShl()) { 4609 __ Lsl(out_reg, first_reg, out_reg); 4610 } else if (op->IsShr()) { 4611 __ Asr(out_reg, first_reg, out_reg); 4612 } else { 4613 __ Lsr(out_reg, first_reg, out_reg); 4614 } 4615 } else { 4616 int32_t cst = second.GetConstant()->AsIntConstant()->GetValue(); 4617 uint32_t shift_value = cst & kMaxIntShiftDistance; 4618 if (shift_value == 0) { // ARM does not support shifting with 0 immediate. 4619 __ Mov(out_reg, first_reg); 4620 } else if (op->IsShl()) { 4621 __ Lsl(out_reg, first_reg, shift_value); 4622 } else if (op->IsShr()) { 4623 __ Asr(out_reg, first_reg, shift_value); 4624 } else { 4625 __ Lsr(out_reg, first_reg, shift_value); 4626 } 4627 } 4628 break; 4629 } 4630 case Primitive::kPrimLong: { 4631 Register o_h = out.AsRegisterPairHigh<Register>(); 4632 Register o_l = out.AsRegisterPairLow<Register>(); 4633 4634 Register high = first.AsRegisterPairHigh<Register>(); 4635 Register low = first.AsRegisterPairLow<Register>(); 4636 4637 if (second.IsRegister()) { 4638 Register temp = locations->GetTemp(0).AsRegister<Register>(); 4639 4640 Register second_reg = second.AsRegister<Register>(); 4641 4642 if (op->IsShl()) { 4643 __ and_(o_l, second_reg, ShifterOperand(kMaxLongShiftDistance)); 4644 // Shift the high part 4645 __ Lsl(o_h, high, o_l); 4646 // Shift the low part and `or` what overflew on the high part 4647 __ rsb(temp, o_l, ShifterOperand(kArmBitsPerWord)); 4648 __ Lsr(temp, low, temp); 4649 __ orr(o_h, o_h, ShifterOperand(temp)); 4650 // If the shift is > 32 bits, override the high part 4651 __ subs(temp, o_l, ShifterOperand(kArmBitsPerWord)); 4652 __ it(PL); 4653 __ Lsl(o_h, low, temp, PL); 4654 // Shift the low part 4655 __ Lsl(o_l, low, o_l); 4656 } else if (op->IsShr()) { 4657 __ and_(o_h, second_reg, ShifterOperand(kMaxLongShiftDistance)); 4658 // Shift the low part 4659 __ Lsr(o_l, low, o_h); 4660 // Shift the high part and `or` what underflew on the low part 4661 __ rsb(temp, o_h, ShifterOperand(kArmBitsPerWord)); 4662 __ Lsl(temp, high, temp); 4663 __ orr(o_l, o_l, ShifterOperand(temp)); 4664 // If the shift is > 32 bits, override the low part 4665 __ subs(temp, o_h, ShifterOperand(kArmBitsPerWord)); 4666 __ it(PL); 4667 __ Asr(o_l, high, temp, PL); 4668 // Shift the high part 4669 __ Asr(o_h, high, o_h); 4670 } else { 4671 __ and_(o_h, second_reg, ShifterOperand(kMaxLongShiftDistance)); 4672 // same as Shr except we use `Lsr`s and not `Asr`s 4673 __ Lsr(o_l, low, o_h); 4674 __ rsb(temp, o_h, ShifterOperand(kArmBitsPerWord)); 4675 __ Lsl(temp, high, temp); 4676 __ orr(o_l, o_l, ShifterOperand(temp)); 4677 __ subs(temp, o_h, ShifterOperand(kArmBitsPerWord)); 4678 __ it(PL); 4679 __ Lsr(o_l, high, temp, PL); 4680 __ Lsr(o_h, high, o_h); 4681 } 4682 } else { 4683 // Register allocator doesn't create partial overlap. 4684 DCHECK_NE(o_l, high); 4685 DCHECK_NE(o_h, low); 4686 int32_t cst = second.GetConstant()->AsIntConstant()->GetValue(); 4687 uint32_t shift_value = cst & kMaxLongShiftDistance; 4688 if (shift_value > 32) { 4689 if (op->IsShl()) { 4690 __ Lsl(o_h, low, shift_value - 32); 4691 __ LoadImmediate(o_l, 0); 4692 } else if (op->IsShr()) { 4693 __ Asr(o_l, high, shift_value - 32); 4694 __ Asr(o_h, high, 31); 4695 } else { 4696 __ Lsr(o_l, high, shift_value - 32); 4697 __ LoadImmediate(o_h, 0); 4698 } 4699 } else if (shift_value == 32) { 4700 if (op->IsShl()) { 4701 __ mov(o_h, ShifterOperand(low)); 4702 __ LoadImmediate(o_l, 0); 4703 } else if (op->IsShr()) { 4704 __ mov(o_l, ShifterOperand(high)); 4705 __ Asr(o_h, high, 31); 4706 } else { 4707 __ mov(o_l, ShifterOperand(high)); 4708 __ LoadImmediate(o_h, 0); 4709 } 4710 } else if (shift_value == 1) { 4711 if (op->IsShl()) { 4712 __ Lsls(o_l, low, 1); 4713 __ adc(o_h, high, ShifterOperand(high)); 4714 } else if (op->IsShr()) { 4715 __ Asrs(o_h, high, 1); 4716 __ Rrx(o_l, low); 4717 } else { 4718 __ Lsrs(o_h, high, 1); 4719 __ Rrx(o_l, low); 4720 } 4721 } else { 4722 DCHECK(2 <= shift_value && shift_value < 32) << shift_value; 4723 if (op->IsShl()) { 4724 __ Lsl(o_h, high, shift_value); 4725 __ orr(o_h, o_h, ShifterOperand(low, LSR, 32 - shift_value)); 4726 __ Lsl(o_l, low, shift_value); 4727 } else if (op->IsShr()) { 4728 __ Lsr(o_l, low, shift_value); 4729 __ orr(o_l, o_l, ShifterOperand(high, LSL, 32 - shift_value)); 4730 __ Asr(o_h, high, shift_value); 4731 } else { 4732 __ Lsr(o_l, low, shift_value); 4733 __ orr(o_l, o_l, ShifterOperand(high, LSL, 32 - shift_value)); 4734 __ Lsr(o_h, high, shift_value); 4735 } 4736 } 4737 } 4738 break; 4739 } 4740 default: 4741 LOG(FATAL) << "Unexpected operation type " << type; 4742 UNREACHABLE(); 4743 } 4744} 4745 4746void LocationsBuilderARM::VisitShl(HShl* shl) { 4747 HandleShift(shl); 4748} 4749 4750void InstructionCodeGeneratorARM::VisitShl(HShl* shl) { 4751 HandleShift(shl); 4752} 4753 4754void LocationsBuilderARM::VisitShr(HShr* shr) { 4755 HandleShift(shr); 4756} 4757 4758void InstructionCodeGeneratorARM::VisitShr(HShr* shr) { 4759 HandleShift(shr); 4760} 4761 4762void LocationsBuilderARM::VisitUShr(HUShr* ushr) { 4763 HandleShift(ushr); 4764} 4765 4766void InstructionCodeGeneratorARM::VisitUShr(HUShr* ushr) { 4767 HandleShift(ushr); 4768} 4769 4770void LocationsBuilderARM::VisitNewInstance(HNewInstance* instruction) { 4771 LocationSummary* locations = 4772 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCallOnMainOnly); 4773 if (instruction->IsStringAlloc()) { 4774 locations->AddTemp(Location::RegisterLocation(kMethodRegisterArgument)); 4775 } else { 4776 InvokeRuntimeCallingConvention calling_convention; 4777 locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(0))); 4778 } 4779 locations->SetOut(Location::RegisterLocation(R0)); 4780} 4781 4782void InstructionCodeGeneratorARM::VisitNewInstance(HNewInstance* instruction) { 4783 // Note: if heap poisoning is enabled, the entry point takes cares 4784 // of poisoning the reference. 4785 if (instruction->IsStringAlloc()) { 4786 // String is allocated through StringFactory. Call NewEmptyString entry point. 4787 Register temp = instruction->GetLocations()->GetTemp(0).AsRegister<Register>(); 4788 MemberOffset code_offset = ArtMethod::EntryPointFromQuickCompiledCodeOffset(kArmPointerSize); 4789 __ LoadFromOffset(kLoadWord, temp, TR, QUICK_ENTRY_POINT(pNewEmptyString)); 4790 __ LoadFromOffset(kLoadWord, LR, temp, code_offset.Int32Value()); 4791 __ blx(LR); 4792 codegen_->RecordPcInfo(instruction, instruction->GetDexPc()); 4793 } else { 4794 codegen_->InvokeRuntime(instruction->GetEntrypoint(), instruction, instruction->GetDexPc()); 4795 CheckEntrypointTypes<kQuickAllocObjectWithChecks, void*, mirror::Class*>(); 4796 } 4797} 4798 4799void LocationsBuilderARM::VisitNewArray(HNewArray* instruction) { 4800 LocationSummary* locations = 4801 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCallOnMainOnly); 4802 InvokeRuntimeCallingConvention calling_convention; 4803 locations->SetOut(Location::RegisterLocation(R0)); 4804 locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(0))); 4805 locations->SetInAt(1, Location::RegisterLocation(calling_convention.GetRegisterAt(1))); 4806} 4807 4808void InstructionCodeGeneratorARM::VisitNewArray(HNewArray* instruction) { 4809 // Note: if heap poisoning is enabled, the entry point takes cares 4810 // of poisoning the reference. 4811 QuickEntrypointEnum entrypoint = 4812 CodeGenerator::GetArrayAllocationEntrypoint(instruction->GetLoadClass()->GetClass()); 4813 codegen_->InvokeRuntime(entrypoint, instruction, instruction->GetDexPc()); 4814 CheckEntrypointTypes<kQuickAllocArrayResolved, void*, mirror::Class*, int32_t>(); 4815 DCHECK(!codegen_->IsLeafMethod()); 4816} 4817 4818void LocationsBuilderARM::VisitParameterValue(HParameterValue* instruction) { 4819 LocationSummary* locations = 4820 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall); 4821 Location location = parameter_visitor_.GetNextLocation(instruction->GetType()); 4822 if (location.IsStackSlot()) { 4823 location = Location::StackSlot(location.GetStackIndex() + codegen_->GetFrameSize()); 4824 } else if (location.IsDoubleStackSlot()) { 4825 location = Location::DoubleStackSlot(location.GetStackIndex() + codegen_->GetFrameSize()); 4826 } 4827 locations->SetOut(location); 4828} 4829 4830void InstructionCodeGeneratorARM::VisitParameterValue( 4831 HParameterValue* instruction ATTRIBUTE_UNUSED) { 4832 // Nothing to do, the parameter is already at its location. 4833} 4834 4835void LocationsBuilderARM::VisitCurrentMethod(HCurrentMethod* instruction) { 4836 LocationSummary* locations = 4837 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall); 4838 locations->SetOut(Location::RegisterLocation(kMethodRegisterArgument)); 4839} 4840 4841void InstructionCodeGeneratorARM::VisitCurrentMethod(HCurrentMethod* instruction ATTRIBUTE_UNUSED) { 4842 // Nothing to do, the method is already at its location. 4843} 4844 4845void LocationsBuilderARM::VisitNot(HNot* not_) { 4846 LocationSummary* locations = 4847 new (GetGraph()->GetArena()) LocationSummary(not_, LocationSummary::kNoCall); 4848 locations->SetInAt(0, Location::RequiresRegister()); 4849 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 4850} 4851 4852void InstructionCodeGeneratorARM::VisitNot(HNot* not_) { 4853 LocationSummary* locations = not_->GetLocations(); 4854 Location out = locations->Out(); 4855 Location in = locations->InAt(0); 4856 switch (not_->GetResultType()) { 4857 case Primitive::kPrimInt: 4858 __ mvn(out.AsRegister<Register>(), ShifterOperand(in.AsRegister<Register>())); 4859 break; 4860 4861 case Primitive::kPrimLong: 4862 __ mvn(out.AsRegisterPairLow<Register>(), 4863 ShifterOperand(in.AsRegisterPairLow<Register>())); 4864 __ mvn(out.AsRegisterPairHigh<Register>(), 4865 ShifterOperand(in.AsRegisterPairHigh<Register>())); 4866 break; 4867 4868 default: 4869 LOG(FATAL) << "Unimplemented type for not operation " << not_->GetResultType(); 4870 } 4871} 4872 4873void LocationsBuilderARM::VisitBooleanNot(HBooleanNot* bool_not) { 4874 LocationSummary* locations = 4875 new (GetGraph()->GetArena()) LocationSummary(bool_not, LocationSummary::kNoCall); 4876 locations->SetInAt(0, Location::RequiresRegister()); 4877 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 4878} 4879 4880void InstructionCodeGeneratorARM::VisitBooleanNot(HBooleanNot* bool_not) { 4881 LocationSummary* locations = bool_not->GetLocations(); 4882 Location out = locations->Out(); 4883 Location in = locations->InAt(0); 4884 __ eor(out.AsRegister<Register>(), in.AsRegister<Register>(), ShifterOperand(1)); 4885} 4886 4887void LocationsBuilderARM::VisitCompare(HCompare* compare) { 4888 LocationSummary* locations = 4889 new (GetGraph()->GetArena()) LocationSummary(compare, LocationSummary::kNoCall); 4890 switch (compare->InputAt(0)->GetType()) { 4891 case Primitive::kPrimBoolean: 4892 case Primitive::kPrimByte: 4893 case Primitive::kPrimShort: 4894 case Primitive::kPrimChar: 4895 case Primitive::kPrimInt: 4896 case Primitive::kPrimLong: { 4897 locations->SetInAt(0, Location::RequiresRegister()); 4898 locations->SetInAt(1, Location::RequiresRegister()); 4899 // Output overlaps because it is written before doing the low comparison. 4900 locations->SetOut(Location::RequiresRegister(), Location::kOutputOverlap); 4901 break; 4902 } 4903 case Primitive::kPrimFloat: 4904 case Primitive::kPrimDouble: { 4905 locations->SetInAt(0, Location::RequiresFpuRegister()); 4906 locations->SetInAt(1, ArithmeticZeroOrFpuRegister(compare->InputAt(1))); 4907 locations->SetOut(Location::RequiresRegister()); 4908 break; 4909 } 4910 default: 4911 LOG(FATAL) << "Unexpected type for compare operation " << compare->InputAt(0)->GetType(); 4912 } 4913} 4914 4915void InstructionCodeGeneratorARM::VisitCompare(HCompare* compare) { 4916 LocationSummary* locations = compare->GetLocations(); 4917 Register out = locations->Out().AsRegister<Register>(); 4918 Location left = locations->InAt(0); 4919 Location right = locations->InAt(1); 4920 4921 Label less, greater, done; 4922 Primitive::Type type = compare->InputAt(0)->GetType(); 4923 Condition less_cond; 4924 switch (type) { 4925 case Primitive::kPrimBoolean: 4926 case Primitive::kPrimByte: 4927 case Primitive::kPrimShort: 4928 case Primitive::kPrimChar: 4929 case Primitive::kPrimInt: { 4930 __ LoadImmediate(out, 0); 4931 __ cmp(left.AsRegister<Register>(), 4932 ShifterOperand(right.AsRegister<Register>())); // Signed compare. 4933 less_cond = LT; 4934 break; 4935 } 4936 case Primitive::kPrimLong: { 4937 __ cmp(left.AsRegisterPairHigh<Register>(), 4938 ShifterOperand(right.AsRegisterPairHigh<Register>())); // Signed compare. 4939 __ b(&less, LT); 4940 __ b(&greater, GT); 4941 // Do LoadImmediate before the last `cmp`, as LoadImmediate might affect the status flags. 4942 __ LoadImmediate(out, 0); 4943 __ cmp(left.AsRegisterPairLow<Register>(), 4944 ShifterOperand(right.AsRegisterPairLow<Register>())); // Unsigned compare. 4945 less_cond = LO; 4946 break; 4947 } 4948 case Primitive::kPrimFloat: 4949 case Primitive::kPrimDouble: { 4950 __ LoadImmediate(out, 0); 4951 GenerateVcmp(compare, codegen_); 4952 __ vmstat(); // transfer FP status register to ARM APSR. 4953 less_cond = ARMFPCondition(kCondLT, compare->IsGtBias()); 4954 break; 4955 } 4956 default: 4957 LOG(FATAL) << "Unexpected compare type " << type; 4958 UNREACHABLE(); 4959 } 4960 4961 __ b(&done, EQ); 4962 __ b(&less, less_cond); 4963 4964 __ Bind(&greater); 4965 __ LoadImmediate(out, 1); 4966 __ b(&done); 4967 4968 __ Bind(&less); 4969 __ LoadImmediate(out, -1); 4970 4971 __ Bind(&done); 4972} 4973 4974void LocationsBuilderARM::VisitPhi(HPhi* instruction) { 4975 LocationSummary* locations = 4976 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall); 4977 for (size_t i = 0, e = locations->GetInputCount(); i < e; ++i) { 4978 locations->SetInAt(i, Location::Any()); 4979 } 4980 locations->SetOut(Location::Any()); 4981} 4982 4983void InstructionCodeGeneratorARM::VisitPhi(HPhi* instruction ATTRIBUTE_UNUSED) { 4984 LOG(FATAL) << "Unreachable"; 4985} 4986 4987void CodeGeneratorARM::GenerateMemoryBarrier(MemBarrierKind kind) { 4988 // TODO (ported from quick): revisit ARM barrier kinds. 4989 DmbOptions flavor = DmbOptions::ISH; // Quiet C++ warnings. 4990 switch (kind) { 4991 case MemBarrierKind::kAnyStore: 4992 case MemBarrierKind::kLoadAny: 4993 case MemBarrierKind::kAnyAny: { 4994 flavor = DmbOptions::ISH; 4995 break; 4996 } 4997 case MemBarrierKind::kStoreStore: { 4998 flavor = DmbOptions::ISHST; 4999 break; 5000 } 5001 default: 5002 LOG(FATAL) << "Unexpected memory barrier " << kind; 5003 } 5004 __ dmb(flavor); 5005} 5006 5007void InstructionCodeGeneratorARM::GenerateWideAtomicLoad(Register addr, 5008 uint32_t offset, 5009 Register out_lo, 5010 Register out_hi) { 5011 if (offset != 0) { 5012 // Ensure `out_lo` is different from `addr`, so that loading 5013 // `offset` into `out_lo` does not clutter `addr`. 5014 DCHECK_NE(out_lo, addr); 5015 __ LoadImmediate(out_lo, offset); 5016 __ add(IP, addr, ShifterOperand(out_lo)); 5017 addr = IP; 5018 } 5019 __ ldrexd(out_lo, out_hi, addr); 5020} 5021 5022void InstructionCodeGeneratorARM::GenerateWideAtomicStore(Register addr, 5023 uint32_t offset, 5024 Register value_lo, 5025 Register value_hi, 5026 Register temp1, 5027 Register temp2, 5028 HInstruction* instruction) { 5029 Label fail; 5030 if (offset != 0) { 5031 __ LoadImmediate(temp1, offset); 5032 __ add(IP, addr, ShifterOperand(temp1)); 5033 addr = IP; 5034 } 5035 __ Bind(&fail); 5036 // We need a load followed by store. (The address used in a STREX instruction must 5037 // be the same as the address in the most recently executed LDREX instruction.) 5038 __ ldrexd(temp1, temp2, addr); 5039 codegen_->MaybeRecordImplicitNullCheck(instruction); 5040 __ strexd(temp1, value_lo, value_hi, addr); 5041 __ CompareAndBranchIfNonZero(temp1, &fail); 5042} 5043 5044void LocationsBuilderARM::HandleFieldSet(HInstruction* instruction, const FieldInfo& field_info) { 5045 DCHECK(instruction->IsInstanceFieldSet() || instruction->IsStaticFieldSet()); 5046 5047 LocationSummary* locations = 5048 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall); 5049 locations->SetInAt(0, Location::RequiresRegister()); 5050 5051 Primitive::Type field_type = field_info.GetFieldType(); 5052 if (Primitive::IsFloatingPointType(field_type)) { 5053 locations->SetInAt(1, Location::RequiresFpuRegister()); 5054 } else { 5055 locations->SetInAt(1, Location::RequiresRegister()); 5056 } 5057 5058 bool is_wide = field_type == Primitive::kPrimLong || field_type == Primitive::kPrimDouble; 5059 bool generate_volatile = field_info.IsVolatile() 5060 && is_wide 5061 && !codegen_->GetInstructionSetFeatures().HasAtomicLdrdAndStrd(); 5062 bool needs_write_barrier = 5063 CodeGenerator::StoreNeedsWriteBarrier(field_type, instruction->InputAt(1)); 5064 // Temporary registers for the write barrier. 5065 // TODO: consider renaming StoreNeedsWriteBarrier to StoreNeedsGCMark. 5066 if (needs_write_barrier) { 5067 locations->AddTemp(Location::RequiresRegister()); // Possibly used for reference poisoning too. 5068 locations->AddTemp(Location::RequiresRegister()); 5069 } else if (generate_volatile) { 5070 // ARM encoding have some additional constraints for ldrexd/strexd: 5071 // - registers need to be consecutive 5072 // - the first register should be even but not R14. 5073 // We don't test for ARM yet, and the assertion makes sure that we 5074 // revisit this if we ever enable ARM encoding. 5075 DCHECK_EQ(InstructionSet::kThumb2, codegen_->GetInstructionSet()); 5076 5077 locations->AddTemp(Location::RequiresRegister()); 5078 locations->AddTemp(Location::RequiresRegister()); 5079 if (field_type == Primitive::kPrimDouble) { 5080 // For doubles we need two more registers to copy the value. 5081 locations->AddTemp(Location::RegisterLocation(R2)); 5082 locations->AddTemp(Location::RegisterLocation(R3)); 5083 } 5084 } 5085} 5086 5087void InstructionCodeGeneratorARM::HandleFieldSet(HInstruction* instruction, 5088 const FieldInfo& field_info, 5089 bool value_can_be_null) { 5090 DCHECK(instruction->IsInstanceFieldSet() || instruction->IsStaticFieldSet()); 5091 5092 LocationSummary* locations = instruction->GetLocations(); 5093 Register base = locations->InAt(0).AsRegister<Register>(); 5094 Location value = locations->InAt(1); 5095 5096 bool is_volatile = field_info.IsVolatile(); 5097 bool atomic_ldrd_strd = codegen_->GetInstructionSetFeatures().HasAtomicLdrdAndStrd(); 5098 Primitive::Type field_type = field_info.GetFieldType(); 5099 uint32_t offset = field_info.GetFieldOffset().Uint32Value(); 5100 bool needs_write_barrier = 5101 CodeGenerator::StoreNeedsWriteBarrier(field_type, instruction->InputAt(1)); 5102 5103 if (is_volatile) { 5104 codegen_->GenerateMemoryBarrier(MemBarrierKind::kAnyStore); 5105 } 5106 5107 switch (field_type) { 5108 case Primitive::kPrimBoolean: 5109 case Primitive::kPrimByte: { 5110 __ StoreToOffset(kStoreByte, value.AsRegister<Register>(), base, offset); 5111 break; 5112 } 5113 5114 case Primitive::kPrimShort: 5115 case Primitive::kPrimChar: { 5116 __ StoreToOffset(kStoreHalfword, value.AsRegister<Register>(), base, offset); 5117 break; 5118 } 5119 5120 case Primitive::kPrimInt: 5121 case Primitive::kPrimNot: { 5122 if (kPoisonHeapReferences && needs_write_barrier) { 5123 // Note that in the case where `value` is a null reference, 5124 // we do not enter this block, as a null reference does not 5125 // need poisoning. 5126 DCHECK_EQ(field_type, Primitive::kPrimNot); 5127 Register temp = locations->GetTemp(0).AsRegister<Register>(); 5128 __ Mov(temp, value.AsRegister<Register>()); 5129 __ PoisonHeapReference(temp); 5130 __ StoreToOffset(kStoreWord, temp, base, offset); 5131 } else { 5132 __ StoreToOffset(kStoreWord, value.AsRegister<Register>(), base, offset); 5133 } 5134 break; 5135 } 5136 5137 case Primitive::kPrimLong: { 5138 if (is_volatile && !atomic_ldrd_strd) { 5139 GenerateWideAtomicStore(base, offset, 5140 value.AsRegisterPairLow<Register>(), 5141 value.AsRegisterPairHigh<Register>(), 5142 locations->GetTemp(0).AsRegister<Register>(), 5143 locations->GetTemp(1).AsRegister<Register>(), 5144 instruction); 5145 } else { 5146 __ StoreToOffset(kStoreWordPair, value.AsRegisterPairLow<Register>(), base, offset); 5147 codegen_->MaybeRecordImplicitNullCheck(instruction); 5148 } 5149 break; 5150 } 5151 5152 case Primitive::kPrimFloat: { 5153 __ StoreSToOffset(value.AsFpuRegister<SRegister>(), base, offset); 5154 break; 5155 } 5156 5157 case Primitive::kPrimDouble: { 5158 DRegister value_reg = FromLowSToD(value.AsFpuRegisterPairLow<SRegister>()); 5159 if (is_volatile && !atomic_ldrd_strd) { 5160 Register value_reg_lo = locations->GetTemp(0).AsRegister<Register>(); 5161 Register value_reg_hi = locations->GetTemp(1).AsRegister<Register>(); 5162 5163 __ vmovrrd(value_reg_lo, value_reg_hi, value_reg); 5164 5165 GenerateWideAtomicStore(base, offset, 5166 value_reg_lo, 5167 value_reg_hi, 5168 locations->GetTemp(2).AsRegister<Register>(), 5169 locations->GetTemp(3).AsRegister<Register>(), 5170 instruction); 5171 } else { 5172 __ StoreDToOffset(value_reg, base, offset); 5173 codegen_->MaybeRecordImplicitNullCheck(instruction); 5174 } 5175 break; 5176 } 5177 5178 case Primitive::kPrimVoid: 5179 LOG(FATAL) << "Unreachable type " << field_type; 5180 UNREACHABLE(); 5181 } 5182 5183 // Longs and doubles are handled in the switch. 5184 if (field_type != Primitive::kPrimLong && field_type != Primitive::kPrimDouble) { 5185 codegen_->MaybeRecordImplicitNullCheck(instruction); 5186 } 5187 5188 if (CodeGenerator::StoreNeedsWriteBarrier(field_type, instruction->InputAt(1))) { 5189 Register temp = locations->GetTemp(0).AsRegister<Register>(); 5190 Register card = locations->GetTemp(1).AsRegister<Register>(); 5191 codegen_->MarkGCCard( 5192 temp, card, base, value.AsRegister<Register>(), value_can_be_null); 5193 } 5194 5195 if (is_volatile) { 5196 codegen_->GenerateMemoryBarrier(MemBarrierKind::kAnyAny); 5197 } 5198} 5199 5200void LocationsBuilderARM::HandleFieldGet(HInstruction* instruction, const FieldInfo& field_info) { 5201 DCHECK(instruction->IsInstanceFieldGet() || instruction->IsStaticFieldGet()); 5202 5203 bool object_field_get_with_read_barrier = 5204 kEmitCompilerReadBarrier && (field_info.GetFieldType() == Primitive::kPrimNot); 5205 LocationSummary* locations = 5206 new (GetGraph()->GetArena()) LocationSummary(instruction, 5207 object_field_get_with_read_barrier ? 5208 LocationSummary::kCallOnSlowPath : 5209 LocationSummary::kNoCall); 5210 if (object_field_get_with_read_barrier && kUseBakerReadBarrier) { 5211 locations->SetCustomSlowPathCallerSaves(RegisterSet::Empty()); // No caller-save registers. 5212 } 5213 locations->SetInAt(0, Location::RequiresRegister()); 5214 5215 bool volatile_for_double = field_info.IsVolatile() 5216 && (field_info.GetFieldType() == Primitive::kPrimDouble) 5217 && !codegen_->GetInstructionSetFeatures().HasAtomicLdrdAndStrd(); 5218 // The output overlaps in case of volatile long: we don't want the 5219 // code generated by GenerateWideAtomicLoad to overwrite the 5220 // object's location. Likewise, in the case of an object field get 5221 // with read barriers enabled, we do not want the load to overwrite 5222 // the object's location, as we need it to emit the read barrier. 5223 bool overlap = (field_info.IsVolatile() && (field_info.GetFieldType() == Primitive::kPrimLong)) || 5224 object_field_get_with_read_barrier; 5225 5226 if (Primitive::IsFloatingPointType(instruction->GetType())) { 5227 locations->SetOut(Location::RequiresFpuRegister()); 5228 } else { 5229 locations->SetOut(Location::RequiresRegister(), 5230 (overlap ? Location::kOutputOverlap : Location::kNoOutputOverlap)); 5231 } 5232 if (volatile_for_double) { 5233 // ARM encoding have some additional constraints for ldrexd/strexd: 5234 // - registers need to be consecutive 5235 // - the first register should be even but not R14. 5236 // We don't test for ARM yet, and the assertion makes sure that we 5237 // revisit this if we ever enable ARM encoding. 5238 DCHECK_EQ(InstructionSet::kThumb2, codegen_->GetInstructionSet()); 5239 locations->AddTemp(Location::RequiresRegister()); 5240 locations->AddTemp(Location::RequiresRegister()); 5241 } else if (object_field_get_with_read_barrier && kUseBakerReadBarrier) { 5242 // We need a temporary register for the read barrier marking slow 5243 // path in CodeGeneratorARM::GenerateFieldLoadWithBakerReadBarrier. 5244 locations->AddTemp(Location::RequiresRegister()); 5245 } 5246} 5247 5248Location LocationsBuilderARM::ArithmeticZeroOrFpuRegister(HInstruction* input) { 5249 DCHECK(input->GetType() == Primitive::kPrimDouble || input->GetType() == Primitive::kPrimFloat) 5250 << input->GetType(); 5251 if ((input->IsFloatConstant() && (input->AsFloatConstant()->IsArithmeticZero())) || 5252 (input->IsDoubleConstant() && (input->AsDoubleConstant()->IsArithmeticZero()))) { 5253 return Location::ConstantLocation(input->AsConstant()); 5254 } else { 5255 return Location::RequiresFpuRegister(); 5256 } 5257} 5258 5259Location LocationsBuilderARM::ArmEncodableConstantOrRegister(HInstruction* constant, 5260 Opcode opcode) { 5261 DCHECK(!Primitive::IsFloatingPointType(constant->GetType())); 5262 if (constant->IsConstant() && 5263 CanEncodeConstantAsImmediate(constant->AsConstant(), opcode)) { 5264 return Location::ConstantLocation(constant->AsConstant()); 5265 } 5266 return Location::RequiresRegister(); 5267} 5268 5269bool LocationsBuilderARM::CanEncodeConstantAsImmediate(HConstant* input_cst, 5270 Opcode opcode) { 5271 uint64_t value = static_cast<uint64_t>(Int64FromConstant(input_cst)); 5272 if (Primitive::Is64BitType(input_cst->GetType())) { 5273 Opcode high_opcode = opcode; 5274 SetCc low_set_cc = kCcDontCare; 5275 switch (opcode) { 5276 case SUB: 5277 // Flip the operation to an ADD. 5278 value = -value; 5279 opcode = ADD; 5280 FALLTHROUGH_INTENDED; 5281 case ADD: 5282 if (Low32Bits(value) == 0u) { 5283 return CanEncodeConstantAsImmediate(High32Bits(value), opcode, kCcDontCare); 5284 } 5285 high_opcode = ADC; 5286 low_set_cc = kCcSet; 5287 break; 5288 default: 5289 break; 5290 } 5291 return CanEncodeConstantAsImmediate(Low32Bits(value), opcode, low_set_cc) && 5292 CanEncodeConstantAsImmediate(High32Bits(value), high_opcode, kCcDontCare); 5293 } else { 5294 return CanEncodeConstantAsImmediate(Low32Bits(value), opcode); 5295 } 5296} 5297 5298bool LocationsBuilderARM::CanEncodeConstantAsImmediate(uint32_t value, 5299 Opcode opcode, 5300 SetCc set_cc) { 5301 ShifterOperand so; 5302 ArmAssembler* assembler = codegen_->GetAssembler(); 5303 if (assembler->ShifterOperandCanHold(kNoRegister, kNoRegister, opcode, value, set_cc, &so)) { 5304 return true; 5305 } 5306 Opcode neg_opcode = kNoOperand; 5307 switch (opcode) { 5308 case AND: neg_opcode = BIC; value = ~value; break; 5309 case ORR: neg_opcode = ORN; value = ~value; break; 5310 case ADD: neg_opcode = SUB; value = -value; break; 5311 case ADC: neg_opcode = SBC; value = ~value; break; 5312 case SUB: neg_opcode = ADD; value = -value; break; 5313 case SBC: neg_opcode = ADC; value = ~value; break; 5314 case MOV: neg_opcode = MVN; value = ~value; break; 5315 default: 5316 return false; 5317 } 5318 return assembler->ShifterOperandCanHold(kNoRegister, kNoRegister, neg_opcode, value, set_cc, &so); 5319} 5320 5321void InstructionCodeGeneratorARM::HandleFieldGet(HInstruction* instruction, 5322 const FieldInfo& field_info) { 5323 DCHECK(instruction->IsInstanceFieldGet() || instruction->IsStaticFieldGet()); 5324 5325 LocationSummary* locations = instruction->GetLocations(); 5326 Location base_loc = locations->InAt(0); 5327 Register base = base_loc.AsRegister<Register>(); 5328 Location out = locations->Out(); 5329 bool is_volatile = field_info.IsVolatile(); 5330 bool atomic_ldrd_strd = codegen_->GetInstructionSetFeatures().HasAtomicLdrdAndStrd(); 5331 Primitive::Type field_type = field_info.GetFieldType(); 5332 uint32_t offset = field_info.GetFieldOffset().Uint32Value(); 5333 5334 switch (field_type) { 5335 case Primitive::kPrimBoolean: 5336 __ LoadFromOffset(kLoadUnsignedByte, out.AsRegister<Register>(), base, offset); 5337 break; 5338 5339 case Primitive::kPrimByte: 5340 __ LoadFromOffset(kLoadSignedByte, out.AsRegister<Register>(), base, offset); 5341 break; 5342 5343 case Primitive::kPrimShort: 5344 __ LoadFromOffset(kLoadSignedHalfword, out.AsRegister<Register>(), base, offset); 5345 break; 5346 5347 case Primitive::kPrimChar: 5348 __ LoadFromOffset(kLoadUnsignedHalfword, out.AsRegister<Register>(), base, offset); 5349 break; 5350 5351 case Primitive::kPrimInt: 5352 __ LoadFromOffset(kLoadWord, out.AsRegister<Register>(), base, offset); 5353 break; 5354 5355 case Primitive::kPrimNot: { 5356 // /* HeapReference<Object> */ out = *(base + offset) 5357 if (kEmitCompilerReadBarrier && kUseBakerReadBarrier) { 5358 Location temp_loc = locations->GetTemp(0); 5359 // Note that a potential implicit null check is handled in this 5360 // CodeGeneratorARM::GenerateFieldLoadWithBakerReadBarrier call. 5361 codegen_->GenerateFieldLoadWithBakerReadBarrier( 5362 instruction, out, base, offset, temp_loc, /* needs_null_check */ true); 5363 if (is_volatile) { 5364 codegen_->GenerateMemoryBarrier(MemBarrierKind::kLoadAny); 5365 } 5366 } else { 5367 __ LoadFromOffset(kLoadWord, out.AsRegister<Register>(), base, offset); 5368 codegen_->MaybeRecordImplicitNullCheck(instruction); 5369 if (is_volatile) { 5370 codegen_->GenerateMemoryBarrier(MemBarrierKind::kLoadAny); 5371 } 5372 // If read barriers are enabled, emit read barriers other than 5373 // Baker's using a slow path (and also unpoison the loaded 5374 // reference, if heap poisoning is enabled). 5375 codegen_->MaybeGenerateReadBarrierSlow(instruction, out, out, base_loc, offset); 5376 } 5377 break; 5378 } 5379 5380 case Primitive::kPrimLong: 5381 if (is_volatile && !atomic_ldrd_strd) { 5382 GenerateWideAtomicLoad(base, offset, 5383 out.AsRegisterPairLow<Register>(), 5384 out.AsRegisterPairHigh<Register>()); 5385 } else { 5386 __ LoadFromOffset(kLoadWordPair, out.AsRegisterPairLow<Register>(), base, offset); 5387 } 5388 break; 5389 5390 case Primitive::kPrimFloat: 5391 __ LoadSFromOffset(out.AsFpuRegister<SRegister>(), base, offset); 5392 break; 5393 5394 case Primitive::kPrimDouble: { 5395 DRegister out_reg = FromLowSToD(out.AsFpuRegisterPairLow<SRegister>()); 5396 if (is_volatile && !atomic_ldrd_strd) { 5397 Register lo = locations->GetTemp(0).AsRegister<Register>(); 5398 Register hi = locations->GetTemp(1).AsRegister<Register>(); 5399 GenerateWideAtomicLoad(base, offset, lo, hi); 5400 codegen_->MaybeRecordImplicitNullCheck(instruction); 5401 __ vmovdrr(out_reg, lo, hi); 5402 } else { 5403 __ LoadDFromOffset(out_reg, base, offset); 5404 codegen_->MaybeRecordImplicitNullCheck(instruction); 5405 } 5406 break; 5407 } 5408 5409 case Primitive::kPrimVoid: 5410 LOG(FATAL) << "Unreachable type " << field_type; 5411 UNREACHABLE(); 5412 } 5413 5414 if (field_type == Primitive::kPrimNot || field_type == Primitive::kPrimDouble) { 5415 // Potential implicit null checks, in the case of reference or 5416 // double fields, are handled in the previous switch statement. 5417 } else { 5418 codegen_->MaybeRecordImplicitNullCheck(instruction); 5419 } 5420 5421 if (is_volatile) { 5422 if (field_type == Primitive::kPrimNot) { 5423 // Memory barriers, in the case of references, are also handled 5424 // in the previous switch statement. 5425 } else { 5426 codegen_->GenerateMemoryBarrier(MemBarrierKind::kLoadAny); 5427 } 5428 } 5429} 5430 5431void LocationsBuilderARM::VisitInstanceFieldSet(HInstanceFieldSet* instruction) { 5432 HandleFieldSet(instruction, instruction->GetFieldInfo()); 5433} 5434 5435void InstructionCodeGeneratorARM::VisitInstanceFieldSet(HInstanceFieldSet* instruction) { 5436 HandleFieldSet(instruction, instruction->GetFieldInfo(), instruction->GetValueCanBeNull()); 5437} 5438 5439void LocationsBuilderARM::VisitInstanceFieldGet(HInstanceFieldGet* instruction) { 5440 HandleFieldGet(instruction, instruction->GetFieldInfo()); 5441} 5442 5443void InstructionCodeGeneratorARM::VisitInstanceFieldGet(HInstanceFieldGet* instruction) { 5444 HandleFieldGet(instruction, instruction->GetFieldInfo()); 5445} 5446 5447void LocationsBuilderARM::VisitStaticFieldGet(HStaticFieldGet* instruction) { 5448 HandleFieldGet(instruction, instruction->GetFieldInfo()); 5449} 5450 5451void InstructionCodeGeneratorARM::VisitStaticFieldGet(HStaticFieldGet* instruction) { 5452 HandleFieldGet(instruction, instruction->GetFieldInfo()); 5453} 5454 5455void LocationsBuilderARM::VisitStaticFieldSet(HStaticFieldSet* instruction) { 5456 HandleFieldSet(instruction, instruction->GetFieldInfo()); 5457} 5458 5459void InstructionCodeGeneratorARM::VisitStaticFieldSet(HStaticFieldSet* instruction) { 5460 HandleFieldSet(instruction, instruction->GetFieldInfo(), instruction->GetValueCanBeNull()); 5461} 5462 5463void LocationsBuilderARM::VisitUnresolvedInstanceFieldGet( 5464 HUnresolvedInstanceFieldGet* instruction) { 5465 FieldAccessCallingConventionARM calling_convention; 5466 codegen_->CreateUnresolvedFieldLocationSummary( 5467 instruction, instruction->GetFieldType(), calling_convention); 5468} 5469 5470void InstructionCodeGeneratorARM::VisitUnresolvedInstanceFieldGet( 5471 HUnresolvedInstanceFieldGet* instruction) { 5472 FieldAccessCallingConventionARM calling_convention; 5473 codegen_->GenerateUnresolvedFieldAccess(instruction, 5474 instruction->GetFieldType(), 5475 instruction->GetFieldIndex(), 5476 instruction->GetDexPc(), 5477 calling_convention); 5478} 5479 5480void LocationsBuilderARM::VisitUnresolvedInstanceFieldSet( 5481 HUnresolvedInstanceFieldSet* instruction) { 5482 FieldAccessCallingConventionARM calling_convention; 5483 codegen_->CreateUnresolvedFieldLocationSummary( 5484 instruction, instruction->GetFieldType(), calling_convention); 5485} 5486 5487void InstructionCodeGeneratorARM::VisitUnresolvedInstanceFieldSet( 5488 HUnresolvedInstanceFieldSet* instruction) { 5489 FieldAccessCallingConventionARM calling_convention; 5490 codegen_->GenerateUnresolvedFieldAccess(instruction, 5491 instruction->GetFieldType(), 5492 instruction->GetFieldIndex(), 5493 instruction->GetDexPc(), 5494 calling_convention); 5495} 5496 5497void LocationsBuilderARM::VisitUnresolvedStaticFieldGet( 5498 HUnresolvedStaticFieldGet* instruction) { 5499 FieldAccessCallingConventionARM calling_convention; 5500 codegen_->CreateUnresolvedFieldLocationSummary( 5501 instruction, instruction->GetFieldType(), calling_convention); 5502} 5503 5504void InstructionCodeGeneratorARM::VisitUnresolvedStaticFieldGet( 5505 HUnresolvedStaticFieldGet* instruction) { 5506 FieldAccessCallingConventionARM calling_convention; 5507 codegen_->GenerateUnresolvedFieldAccess(instruction, 5508 instruction->GetFieldType(), 5509 instruction->GetFieldIndex(), 5510 instruction->GetDexPc(), 5511 calling_convention); 5512} 5513 5514void LocationsBuilderARM::VisitUnresolvedStaticFieldSet( 5515 HUnresolvedStaticFieldSet* instruction) { 5516 FieldAccessCallingConventionARM calling_convention; 5517 codegen_->CreateUnresolvedFieldLocationSummary( 5518 instruction, instruction->GetFieldType(), calling_convention); 5519} 5520 5521void InstructionCodeGeneratorARM::VisitUnresolvedStaticFieldSet( 5522 HUnresolvedStaticFieldSet* instruction) { 5523 FieldAccessCallingConventionARM calling_convention; 5524 codegen_->GenerateUnresolvedFieldAccess(instruction, 5525 instruction->GetFieldType(), 5526 instruction->GetFieldIndex(), 5527 instruction->GetDexPc(), 5528 calling_convention); 5529} 5530 5531void LocationsBuilderARM::VisitNullCheck(HNullCheck* instruction) { 5532 LocationSummary* locations = codegen_->CreateThrowingSlowPathLocations(instruction); 5533 locations->SetInAt(0, Location::RequiresRegister()); 5534} 5535 5536void CodeGeneratorARM::GenerateImplicitNullCheck(HNullCheck* instruction) { 5537 if (CanMoveNullCheckToUser(instruction)) { 5538 return; 5539 } 5540 Location obj = instruction->GetLocations()->InAt(0); 5541 5542 __ LoadFromOffset(kLoadWord, IP, obj.AsRegister<Register>(), 0); 5543 RecordPcInfo(instruction, instruction->GetDexPc()); 5544} 5545 5546void CodeGeneratorARM::GenerateExplicitNullCheck(HNullCheck* instruction) { 5547 SlowPathCodeARM* slow_path = new (GetGraph()->GetArena()) NullCheckSlowPathARM(instruction); 5548 AddSlowPath(slow_path); 5549 5550 LocationSummary* locations = instruction->GetLocations(); 5551 Location obj = locations->InAt(0); 5552 5553 __ CompareAndBranchIfZero(obj.AsRegister<Register>(), slow_path->GetEntryLabel()); 5554} 5555 5556void InstructionCodeGeneratorARM::VisitNullCheck(HNullCheck* instruction) { 5557 codegen_->GenerateNullCheck(instruction); 5558} 5559 5560static LoadOperandType GetLoadOperandType(Primitive::Type type) { 5561 switch (type) { 5562 case Primitive::kPrimNot: 5563 return kLoadWord; 5564 case Primitive::kPrimBoolean: 5565 return kLoadUnsignedByte; 5566 case Primitive::kPrimByte: 5567 return kLoadSignedByte; 5568 case Primitive::kPrimChar: 5569 return kLoadUnsignedHalfword; 5570 case Primitive::kPrimShort: 5571 return kLoadSignedHalfword; 5572 case Primitive::kPrimInt: 5573 return kLoadWord; 5574 case Primitive::kPrimLong: 5575 return kLoadWordPair; 5576 case Primitive::kPrimFloat: 5577 return kLoadSWord; 5578 case Primitive::kPrimDouble: 5579 return kLoadDWord; 5580 default: 5581 LOG(FATAL) << "Unreachable type " << type; 5582 UNREACHABLE(); 5583 } 5584} 5585 5586static StoreOperandType GetStoreOperandType(Primitive::Type type) { 5587 switch (type) { 5588 case Primitive::kPrimNot: 5589 return kStoreWord; 5590 case Primitive::kPrimBoolean: 5591 case Primitive::kPrimByte: 5592 return kStoreByte; 5593 case Primitive::kPrimChar: 5594 case Primitive::kPrimShort: 5595 return kStoreHalfword; 5596 case Primitive::kPrimInt: 5597 return kStoreWord; 5598 case Primitive::kPrimLong: 5599 return kStoreWordPair; 5600 case Primitive::kPrimFloat: 5601 return kStoreSWord; 5602 case Primitive::kPrimDouble: 5603 return kStoreDWord; 5604 default: 5605 LOG(FATAL) << "Unreachable type " << type; 5606 UNREACHABLE(); 5607 } 5608} 5609 5610void CodeGeneratorARM::LoadFromShiftedRegOffset(Primitive::Type type, 5611 Location out_loc, 5612 Register base, 5613 Register reg_offset, 5614 Condition cond) { 5615 uint32_t shift_count = Primitive::ComponentSizeShift(type); 5616 Address mem_address(base, reg_offset, Shift::LSL, shift_count); 5617 5618 switch (type) { 5619 case Primitive::kPrimByte: 5620 __ ldrsb(out_loc.AsRegister<Register>(), mem_address, cond); 5621 break; 5622 case Primitive::kPrimBoolean: 5623 __ ldrb(out_loc.AsRegister<Register>(), mem_address, cond); 5624 break; 5625 case Primitive::kPrimShort: 5626 __ ldrsh(out_loc.AsRegister<Register>(), mem_address, cond); 5627 break; 5628 case Primitive::kPrimChar: 5629 __ ldrh(out_loc.AsRegister<Register>(), mem_address, cond); 5630 break; 5631 case Primitive::kPrimNot: 5632 case Primitive::kPrimInt: 5633 __ ldr(out_loc.AsRegister<Register>(), mem_address, cond); 5634 break; 5635 // T32 doesn't support LoadFromShiftedRegOffset mem address mode for these types. 5636 case Primitive::kPrimLong: 5637 case Primitive::kPrimFloat: 5638 case Primitive::kPrimDouble: 5639 default: 5640 LOG(FATAL) << "Unreachable type " << type; 5641 UNREACHABLE(); 5642 } 5643} 5644 5645void CodeGeneratorARM::StoreToShiftedRegOffset(Primitive::Type type, 5646 Location loc, 5647 Register base, 5648 Register reg_offset, 5649 Condition cond) { 5650 uint32_t shift_count = Primitive::ComponentSizeShift(type); 5651 Address mem_address(base, reg_offset, Shift::LSL, shift_count); 5652 5653 switch (type) { 5654 case Primitive::kPrimByte: 5655 case Primitive::kPrimBoolean: 5656 __ strb(loc.AsRegister<Register>(), mem_address, cond); 5657 break; 5658 case Primitive::kPrimShort: 5659 case Primitive::kPrimChar: 5660 __ strh(loc.AsRegister<Register>(), mem_address, cond); 5661 break; 5662 case Primitive::kPrimNot: 5663 case Primitive::kPrimInt: 5664 __ str(loc.AsRegister<Register>(), mem_address, cond); 5665 break; 5666 // T32 doesn't support StoreToShiftedRegOffset mem address mode for these types. 5667 case Primitive::kPrimLong: 5668 case Primitive::kPrimFloat: 5669 case Primitive::kPrimDouble: 5670 default: 5671 LOG(FATAL) << "Unreachable type " << type; 5672 UNREACHABLE(); 5673 } 5674} 5675 5676void LocationsBuilderARM::VisitArrayGet(HArrayGet* instruction) { 5677 bool object_array_get_with_read_barrier = 5678 kEmitCompilerReadBarrier && (instruction->GetType() == Primitive::kPrimNot); 5679 LocationSummary* locations = 5680 new (GetGraph()->GetArena()) LocationSummary(instruction, 5681 object_array_get_with_read_barrier ? 5682 LocationSummary::kCallOnSlowPath : 5683 LocationSummary::kNoCall); 5684 if (object_array_get_with_read_barrier && kUseBakerReadBarrier) { 5685 locations->SetCustomSlowPathCallerSaves(RegisterSet::Empty()); // No caller-save registers. 5686 } 5687 locations->SetInAt(0, Location::RequiresRegister()); 5688 locations->SetInAt(1, Location::RegisterOrConstant(instruction->InputAt(1))); 5689 if (Primitive::IsFloatingPointType(instruction->GetType())) { 5690 locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap); 5691 } else { 5692 // The output overlaps in the case of an object array get with 5693 // read barriers enabled: we do not want the move to overwrite the 5694 // array's location, as we need it to emit the read barrier. 5695 locations->SetOut( 5696 Location::RequiresRegister(), 5697 object_array_get_with_read_barrier ? Location::kOutputOverlap : Location::kNoOutputOverlap); 5698 } 5699 // We need a temporary register for the read barrier marking slow 5700 // path in CodeGeneratorARM::GenerateArrayLoadWithBakerReadBarrier. 5701 // Also need for String compression feature. 5702 if ((object_array_get_with_read_barrier && kUseBakerReadBarrier) 5703 || (mirror::kUseStringCompression && instruction->IsStringCharAt())) { 5704 locations->AddTemp(Location::RequiresRegister()); 5705 } 5706} 5707 5708void InstructionCodeGeneratorARM::VisitArrayGet(HArrayGet* instruction) { 5709 LocationSummary* locations = instruction->GetLocations(); 5710 Location obj_loc = locations->InAt(0); 5711 Register obj = obj_loc.AsRegister<Register>(); 5712 Location index = locations->InAt(1); 5713 Location out_loc = locations->Out(); 5714 uint32_t data_offset = CodeGenerator::GetArrayDataOffset(instruction); 5715 Primitive::Type type = instruction->GetType(); 5716 const bool maybe_compressed_char_at = mirror::kUseStringCompression && 5717 instruction->IsStringCharAt(); 5718 HInstruction* array_instr = instruction->GetArray(); 5719 bool has_intermediate_address = array_instr->IsIntermediateAddress(); 5720 5721 switch (type) { 5722 case Primitive::kPrimBoolean: 5723 case Primitive::kPrimByte: 5724 case Primitive::kPrimShort: 5725 case Primitive::kPrimChar: 5726 case Primitive::kPrimInt: { 5727 Register length; 5728 if (maybe_compressed_char_at) { 5729 length = locations->GetTemp(0).AsRegister<Register>(); 5730 uint32_t count_offset = mirror::String::CountOffset().Uint32Value(); 5731 __ LoadFromOffset(kLoadWord, length, obj, count_offset); 5732 codegen_->MaybeRecordImplicitNullCheck(instruction); 5733 } 5734 if (index.IsConstant()) { 5735 int32_t const_index = index.GetConstant()->AsIntConstant()->GetValue(); 5736 if (maybe_compressed_char_at) { 5737 Label uncompressed_load, done; 5738 __ Lsrs(length, length, 1u); // LSRS has a 16-bit encoding, TST (immediate) does not. 5739 static_assert(static_cast<uint32_t>(mirror::StringCompressionFlag::kCompressed) == 0u, 5740 "Expecting 0=compressed, 1=uncompressed"); 5741 __ b(&uncompressed_load, CS); 5742 __ LoadFromOffset(kLoadUnsignedByte, 5743 out_loc.AsRegister<Register>(), 5744 obj, 5745 data_offset + const_index); 5746 __ b(&done); 5747 __ Bind(&uncompressed_load); 5748 __ LoadFromOffset(GetLoadOperandType(Primitive::kPrimChar), 5749 out_loc.AsRegister<Register>(), 5750 obj, 5751 data_offset + (const_index << 1)); 5752 __ Bind(&done); 5753 } else { 5754 uint32_t full_offset = data_offset + (const_index << Primitive::ComponentSizeShift(type)); 5755 5756 LoadOperandType load_type = GetLoadOperandType(type); 5757 __ LoadFromOffset(load_type, out_loc.AsRegister<Register>(), obj, full_offset); 5758 } 5759 } else { 5760 Register temp = IP; 5761 5762 if (has_intermediate_address) { 5763 // We do not need to compute the intermediate address from the array: the 5764 // input instruction has done it already. See the comment in 5765 // `TryExtractArrayAccessAddress()`. 5766 if (kIsDebugBuild) { 5767 HIntermediateAddress* tmp = array_instr->AsIntermediateAddress(); 5768 DCHECK_EQ(tmp->GetOffset()->AsIntConstant()->GetValueAsUint64(), data_offset); 5769 } 5770 temp = obj; 5771 } else { 5772 __ add(temp, obj, ShifterOperand(data_offset)); 5773 } 5774 if (maybe_compressed_char_at) { 5775 Label uncompressed_load, done; 5776 __ Lsrs(length, length, 1u); // LSRS has a 16-bit encoding, TST (immediate) does not. 5777 static_assert(static_cast<uint32_t>(mirror::StringCompressionFlag::kCompressed) == 0u, 5778 "Expecting 0=compressed, 1=uncompressed"); 5779 __ b(&uncompressed_load, CS); 5780 __ ldrb(out_loc.AsRegister<Register>(), 5781 Address(temp, index.AsRegister<Register>(), Shift::LSL, 0)); 5782 __ b(&done); 5783 __ Bind(&uncompressed_load); 5784 __ ldrh(out_loc.AsRegister<Register>(), 5785 Address(temp, index.AsRegister<Register>(), Shift::LSL, 1)); 5786 __ Bind(&done); 5787 } else { 5788 codegen_->LoadFromShiftedRegOffset(type, out_loc, temp, index.AsRegister<Register>()); 5789 } 5790 } 5791 break; 5792 } 5793 5794 case Primitive::kPrimNot: { 5795 // The read barrier instrumentation of object ArrayGet 5796 // instructions does not support the HIntermediateAddress 5797 // instruction. 5798 DCHECK(!(has_intermediate_address && kEmitCompilerReadBarrier)); 5799 5800 static_assert( 5801 sizeof(mirror::HeapReference<mirror::Object>) == sizeof(int32_t), 5802 "art::mirror::HeapReference<art::mirror::Object> and int32_t have different sizes."); 5803 // /* HeapReference<Object> */ out = 5804 // *(obj + data_offset + index * sizeof(HeapReference<Object>)) 5805 if (kEmitCompilerReadBarrier && kUseBakerReadBarrier) { 5806 Location temp = locations->GetTemp(0); 5807 // Note that a potential implicit null check is handled in this 5808 // CodeGeneratorARM::GenerateArrayLoadWithBakerReadBarrier call. 5809 codegen_->GenerateArrayLoadWithBakerReadBarrier( 5810 instruction, out_loc, obj, data_offset, index, temp, /* needs_null_check */ true); 5811 } else { 5812 Register out = out_loc.AsRegister<Register>(); 5813 if (index.IsConstant()) { 5814 size_t offset = 5815 (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4) + data_offset; 5816 __ LoadFromOffset(kLoadWord, out, obj, offset); 5817 codegen_->MaybeRecordImplicitNullCheck(instruction); 5818 // If read barriers are enabled, emit read barriers other than 5819 // Baker's using a slow path (and also unpoison the loaded 5820 // reference, if heap poisoning is enabled). 5821 codegen_->MaybeGenerateReadBarrierSlow(instruction, out_loc, out_loc, obj_loc, offset); 5822 } else { 5823 Register temp = IP; 5824 5825 if (has_intermediate_address) { 5826 // We do not need to compute the intermediate address from the array: the 5827 // input instruction has done it already. See the comment in 5828 // `TryExtractArrayAccessAddress()`. 5829 if (kIsDebugBuild) { 5830 HIntermediateAddress* tmp = array_instr->AsIntermediateAddress(); 5831 DCHECK_EQ(tmp->GetOffset()->AsIntConstant()->GetValueAsUint64(), data_offset); 5832 } 5833 temp = obj; 5834 } else { 5835 __ add(temp, obj, ShifterOperand(data_offset)); 5836 } 5837 codegen_->LoadFromShiftedRegOffset(type, out_loc, temp, index.AsRegister<Register>()); 5838 5839 codegen_->MaybeRecordImplicitNullCheck(instruction); 5840 // If read barriers are enabled, emit read barriers other than 5841 // Baker's using a slow path (and also unpoison the loaded 5842 // reference, if heap poisoning is enabled). 5843 codegen_->MaybeGenerateReadBarrierSlow( 5844 instruction, out_loc, out_loc, obj_loc, data_offset, index); 5845 } 5846 } 5847 break; 5848 } 5849 5850 case Primitive::kPrimLong: { 5851 if (index.IsConstant()) { 5852 size_t offset = 5853 (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_8) + data_offset; 5854 __ LoadFromOffset(kLoadWordPair, out_loc.AsRegisterPairLow<Register>(), obj, offset); 5855 } else { 5856 __ add(IP, obj, ShifterOperand(index.AsRegister<Register>(), LSL, TIMES_8)); 5857 __ LoadFromOffset(kLoadWordPair, out_loc.AsRegisterPairLow<Register>(), IP, data_offset); 5858 } 5859 break; 5860 } 5861 5862 case Primitive::kPrimFloat: { 5863 SRegister out = out_loc.AsFpuRegister<SRegister>(); 5864 if (index.IsConstant()) { 5865 size_t offset = (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4) + data_offset; 5866 __ LoadSFromOffset(out, obj, offset); 5867 } else { 5868 __ add(IP, obj, ShifterOperand(index.AsRegister<Register>(), LSL, TIMES_4)); 5869 __ LoadSFromOffset(out, IP, data_offset); 5870 } 5871 break; 5872 } 5873 5874 case Primitive::kPrimDouble: { 5875 SRegister out = out_loc.AsFpuRegisterPairLow<SRegister>(); 5876 if (index.IsConstant()) { 5877 size_t offset = (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_8) + data_offset; 5878 __ LoadDFromOffset(FromLowSToD(out), obj, offset); 5879 } else { 5880 __ add(IP, obj, ShifterOperand(index.AsRegister<Register>(), LSL, TIMES_8)); 5881 __ LoadDFromOffset(FromLowSToD(out), IP, data_offset); 5882 } 5883 break; 5884 } 5885 5886 case Primitive::kPrimVoid: 5887 LOG(FATAL) << "Unreachable type " << type; 5888 UNREACHABLE(); 5889 } 5890 5891 if (type == Primitive::kPrimNot) { 5892 // Potential implicit null checks, in the case of reference 5893 // arrays, are handled in the previous switch statement. 5894 } else if (!maybe_compressed_char_at) { 5895 codegen_->MaybeRecordImplicitNullCheck(instruction); 5896 } 5897} 5898 5899void LocationsBuilderARM::VisitArraySet(HArraySet* instruction) { 5900 Primitive::Type value_type = instruction->GetComponentType(); 5901 5902 bool needs_write_barrier = 5903 CodeGenerator::StoreNeedsWriteBarrier(value_type, instruction->GetValue()); 5904 bool may_need_runtime_call_for_type_check = instruction->NeedsTypeCheck(); 5905 5906 LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary( 5907 instruction, 5908 may_need_runtime_call_for_type_check ? 5909 LocationSummary::kCallOnSlowPath : 5910 LocationSummary::kNoCall); 5911 5912 locations->SetInAt(0, Location::RequiresRegister()); 5913 locations->SetInAt(1, Location::RegisterOrConstant(instruction->InputAt(1))); 5914 if (Primitive::IsFloatingPointType(value_type)) { 5915 locations->SetInAt(2, Location::RequiresFpuRegister()); 5916 } else { 5917 locations->SetInAt(2, Location::RequiresRegister()); 5918 } 5919 if (needs_write_barrier) { 5920 // Temporary registers for the write barrier. 5921 locations->AddTemp(Location::RequiresRegister()); // Possibly used for ref. poisoning too. 5922 locations->AddTemp(Location::RequiresRegister()); 5923 } 5924} 5925 5926void InstructionCodeGeneratorARM::VisitArraySet(HArraySet* instruction) { 5927 LocationSummary* locations = instruction->GetLocations(); 5928 Location array_loc = locations->InAt(0); 5929 Register array = array_loc.AsRegister<Register>(); 5930 Location index = locations->InAt(1); 5931 Primitive::Type value_type = instruction->GetComponentType(); 5932 bool may_need_runtime_call_for_type_check = instruction->NeedsTypeCheck(); 5933 bool needs_write_barrier = 5934 CodeGenerator::StoreNeedsWriteBarrier(value_type, instruction->GetValue()); 5935 uint32_t data_offset = 5936 mirror::Array::DataOffset(Primitive::ComponentSize(value_type)).Uint32Value(); 5937 Location value_loc = locations->InAt(2); 5938 HInstruction* array_instr = instruction->GetArray(); 5939 bool has_intermediate_address = array_instr->IsIntermediateAddress(); 5940 5941 switch (value_type) { 5942 case Primitive::kPrimBoolean: 5943 case Primitive::kPrimByte: 5944 case Primitive::kPrimShort: 5945 case Primitive::kPrimChar: 5946 case Primitive::kPrimInt: { 5947 if (index.IsConstant()) { 5948 int32_t const_index = index.GetConstant()->AsIntConstant()->GetValue(); 5949 uint32_t full_offset = 5950 data_offset + (const_index << Primitive::ComponentSizeShift(value_type)); 5951 StoreOperandType store_type = GetStoreOperandType(value_type); 5952 __ StoreToOffset(store_type, value_loc.AsRegister<Register>(), array, full_offset); 5953 } else { 5954 Register temp = IP; 5955 5956 if (has_intermediate_address) { 5957 // We do not need to compute the intermediate address from the array: the 5958 // input instruction has done it already. See the comment in 5959 // `TryExtractArrayAccessAddress()`. 5960 if (kIsDebugBuild) { 5961 HIntermediateAddress* tmp = array_instr->AsIntermediateAddress(); 5962 DCHECK(tmp->GetOffset()->AsIntConstant()->GetValueAsUint64() == data_offset); 5963 } 5964 temp = array; 5965 } else { 5966 __ add(temp, array, ShifterOperand(data_offset)); 5967 } 5968 codegen_->StoreToShiftedRegOffset(value_type, 5969 value_loc, 5970 temp, 5971 index.AsRegister<Register>()); 5972 } 5973 break; 5974 } 5975 5976 case Primitive::kPrimNot: { 5977 Register value = value_loc.AsRegister<Register>(); 5978 // TryExtractArrayAccessAddress optimization is never applied for non-primitive ArraySet. 5979 // See the comment in instruction_simplifier_shared.cc. 5980 DCHECK(!has_intermediate_address); 5981 5982 if (instruction->InputAt(2)->IsNullConstant()) { 5983 // Just setting null. 5984 if (index.IsConstant()) { 5985 size_t offset = 5986 (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4) + data_offset; 5987 __ StoreToOffset(kStoreWord, value, array, offset); 5988 } else { 5989 DCHECK(index.IsRegister()) << index; 5990 __ add(IP, array, ShifterOperand(data_offset)); 5991 codegen_->StoreToShiftedRegOffset(value_type, 5992 value_loc, 5993 IP, 5994 index.AsRegister<Register>()); 5995 } 5996 codegen_->MaybeRecordImplicitNullCheck(instruction); 5997 DCHECK(!needs_write_barrier); 5998 DCHECK(!may_need_runtime_call_for_type_check); 5999 break; 6000 } 6001 6002 DCHECK(needs_write_barrier); 6003 Location temp1_loc = locations->GetTemp(0); 6004 Register temp1 = temp1_loc.AsRegister<Register>(); 6005 Location temp2_loc = locations->GetTemp(1); 6006 Register temp2 = temp2_loc.AsRegister<Register>(); 6007 uint32_t class_offset = mirror::Object::ClassOffset().Int32Value(); 6008 uint32_t super_offset = mirror::Class::SuperClassOffset().Int32Value(); 6009 uint32_t component_offset = mirror::Class::ComponentTypeOffset().Int32Value(); 6010 Label done; 6011 SlowPathCodeARM* slow_path = nullptr; 6012 6013 if (may_need_runtime_call_for_type_check) { 6014 slow_path = new (GetGraph()->GetArena()) ArraySetSlowPathARM(instruction); 6015 codegen_->AddSlowPath(slow_path); 6016 if (instruction->GetValueCanBeNull()) { 6017 Label non_zero; 6018 __ CompareAndBranchIfNonZero(value, &non_zero); 6019 if (index.IsConstant()) { 6020 size_t offset = 6021 (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4) + data_offset; 6022 __ StoreToOffset(kStoreWord, value, array, offset); 6023 } else { 6024 DCHECK(index.IsRegister()) << index; 6025 __ add(IP, array, ShifterOperand(data_offset)); 6026 codegen_->StoreToShiftedRegOffset(value_type, 6027 value_loc, 6028 IP, 6029 index.AsRegister<Register>()); 6030 } 6031 codegen_->MaybeRecordImplicitNullCheck(instruction); 6032 __ b(&done); 6033 __ Bind(&non_zero); 6034 } 6035 6036 // Note that when read barriers are enabled, the type checks 6037 // are performed without read barriers. This is fine, even in 6038 // the case where a class object is in the from-space after 6039 // the flip, as a comparison involving such a type would not 6040 // produce a false positive; it may of course produce a false 6041 // negative, in which case we would take the ArraySet slow 6042 // path. 6043 6044 // /* HeapReference<Class> */ temp1 = array->klass_ 6045 __ LoadFromOffset(kLoadWord, temp1, array, class_offset); 6046 codegen_->MaybeRecordImplicitNullCheck(instruction); 6047 __ MaybeUnpoisonHeapReference(temp1); 6048 6049 // /* HeapReference<Class> */ temp1 = temp1->component_type_ 6050 __ LoadFromOffset(kLoadWord, temp1, temp1, component_offset); 6051 // /* HeapReference<Class> */ temp2 = value->klass_ 6052 __ LoadFromOffset(kLoadWord, temp2, value, class_offset); 6053 // If heap poisoning is enabled, no need to unpoison `temp1` 6054 // nor `temp2`, as we are comparing two poisoned references. 6055 __ cmp(temp1, ShifterOperand(temp2)); 6056 6057 if (instruction->StaticTypeOfArrayIsObjectArray()) { 6058 Label do_put; 6059 __ b(&do_put, EQ); 6060 // If heap poisoning is enabled, the `temp1` reference has 6061 // not been unpoisoned yet; unpoison it now. 6062 __ MaybeUnpoisonHeapReference(temp1); 6063 6064 // /* HeapReference<Class> */ temp1 = temp1->super_class_ 6065 __ LoadFromOffset(kLoadWord, temp1, temp1, super_offset); 6066 // If heap poisoning is enabled, no need to unpoison 6067 // `temp1`, as we are comparing against null below. 6068 __ CompareAndBranchIfNonZero(temp1, slow_path->GetEntryLabel()); 6069 __ Bind(&do_put); 6070 } else { 6071 __ b(slow_path->GetEntryLabel(), NE); 6072 } 6073 } 6074 6075 Register source = value; 6076 if (kPoisonHeapReferences) { 6077 // Note that in the case where `value` is a null reference, 6078 // we do not enter this block, as a null reference does not 6079 // need poisoning. 6080 DCHECK_EQ(value_type, Primitive::kPrimNot); 6081 __ Mov(temp1, value); 6082 __ PoisonHeapReference(temp1); 6083 source = temp1; 6084 } 6085 6086 if (index.IsConstant()) { 6087 size_t offset = 6088 (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4) + data_offset; 6089 __ StoreToOffset(kStoreWord, source, array, offset); 6090 } else { 6091 DCHECK(index.IsRegister()) << index; 6092 6093 __ add(IP, array, ShifterOperand(data_offset)); 6094 codegen_->StoreToShiftedRegOffset(value_type, 6095 Location::RegisterLocation(source), 6096 IP, 6097 index.AsRegister<Register>()); 6098 } 6099 6100 if (!may_need_runtime_call_for_type_check) { 6101 codegen_->MaybeRecordImplicitNullCheck(instruction); 6102 } 6103 6104 codegen_->MarkGCCard(temp1, temp2, array, value, instruction->GetValueCanBeNull()); 6105 6106 if (done.IsLinked()) { 6107 __ Bind(&done); 6108 } 6109 6110 if (slow_path != nullptr) { 6111 __ Bind(slow_path->GetExitLabel()); 6112 } 6113 6114 break; 6115 } 6116 6117 case Primitive::kPrimLong: { 6118 Location value = locations->InAt(2); 6119 if (index.IsConstant()) { 6120 size_t offset = 6121 (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_8) + data_offset; 6122 __ StoreToOffset(kStoreWordPair, value.AsRegisterPairLow<Register>(), array, offset); 6123 } else { 6124 __ add(IP, array, ShifterOperand(index.AsRegister<Register>(), LSL, TIMES_8)); 6125 __ StoreToOffset(kStoreWordPair, value.AsRegisterPairLow<Register>(), IP, data_offset); 6126 } 6127 break; 6128 } 6129 6130 case Primitive::kPrimFloat: { 6131 Location value = locations->InAt(2); 6132 DCHECK(value.IsFpuRegister()); 6133 if (index.IsConstant()) { 6134 size_t offset = (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4) + data_offset; 6135 __ StoreSToOffset(value.AsFpuRegister<SRegister>(), array, offset); 6136 } else { 6137 __ add(IP, array, ShifterOperand(index.AsRegister<Register>(), LSL, TIMES_4)); 6138 __ StoreSToOffset(value.AsFpuRegister<SRegister>(), IP, data_offset); 6139 } 6140 break; 6141 } 6142 6143 case Primitive::kPrimDouble: { 6144 Location value = locations->InAt(2); 6145 DCHECK(value.IsFpuRegisterPair()); 6146 if (index.IsConstant()) { 6147 size_t offset = (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_8) + data_offset; 6148 __ StoreDToOffset(FromLowSToD(value.AsFpuRegisterPairLow<SRegister>()), array, offset); 6149 } else { 6150 __ add(IP, array, ShifterOperand(index.AsRegister<Register>(), LSL, TIMES_8)); 6151 __ StoreDToOffset(FromLowSToD(value.AsFpuRegisterPairLow<SRegister>()), IP, data_offset); 6152 } 6153 6154 break; 6155 } 6156 6157 case Primitive::kPrimVoid: 6158 LOG(FATAL) << "Unreachable type " << value_type; 6159 UNREACHABLE(); 6160 } 6161 6162 // Objects are handled in the switch. 6163 if (value_type != Primitive::kPrimNot) { 6164 codegen_->MaybeRecordImplicitNullCheck(instruction); 6165 } 6166} 6167 6168void LocationsBuilderARM::VisitArrayLength(HArrayLength* instruction) { 6169 LocationSummary* locations = 6170 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall); 6171 locations->SetInAt(0, Location::RequiresRegister()); 6172 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 6173} 6174 6175void InstructionCodeGeneratorARM::VisitArrayLength(HArrayLength* instruction) { 6176 LocationSummary* locations = instruction->GetLocations(); 6177 uint32_t offset = CodeGenerator::GetArrayLengthOffset(instruction); 6178 Register obj = locations->InAt(0).AsRegister<Register>(); 6179 Register out = locations->Out().AsRegister<Register>(); 6180 __ LoadFromOffset(kLoadWord, out, obj, offset); 6181 codegen_->MaybeRecordImplicitNullCheck(instruction); 6182 // Mask out compression flag from String's array length. 6183 if (mirror::kUseStringCompression && instruction->IsStringLength()) { 6184 __ Lsr(out, out, 1u); 6185 } 6186} 6187 6188void LocationsBuilderARM::VisitIntermediateAddress(HIntermediateAddress* instruction) { 6189 LocationSummary* locations = 6190 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall); 6191 6192 locations->SetInAt(0, Location::RequiresRegister()); 6193 locations->SetInAt(1, Location::RegisterOrConstant(instruction->GetOffset())); 6194 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 6195} 6196 6197void InstructionCodeGeneratorARM::VisitIntermediateAddress(HIntermediateAddress* instruction) { 6198 LocationSummary* locations = instruction->GetLocations(); 6199 Location out = locations->Out(); 6200 Location first = locations->InAt(0); 6201 Location second = locations->InAt(1); 6202 6203 if (second.IsRegister()) { 6204 __ add(out.AsRegister<Register>(), 6205 first.AsRegister<Register>(), 6206 ShifterOperand(second.AsRegister<Register>())); 6207 } else { 6208 __ AddConstant(out.AsRegister<Register>(), 6209 first.AsRegister<Register>(), 6210 second.GetConstant()->AsIntConstant()->GetValue()); 6211 } 6212} 6213 6214void LocationsBuilderARM::VisitBoundsCheck(HBoundsCheck* instruction) { 6215 RegisterSet caller_saves = RegisterSet::Empty(); 6216 InvokeRuntimeCallingConvention calling_convention; 6217 caller_saves.Add(Location::RegisterLocation(calling_convention.GetRegisterAt(0))); 6218 caller_saves.Add(Location::RegisterLocation(calling_convention.GetRegisterAt(1))); 6219 LocationSummary* locations = codegen_->CreateThrowingSlowPathLocations(instruction, caller_saves); 6220 6221 HInstruction* index = instruction->InputAt(0); 6222 HInstruction* length = instruction->InputAt(1); 6223 // If both index and length are constants we can statically check the bounds. But if at least one 6224 // of them is not encodable ArmEncodableConstantOrRegister will create 6225 // Location::RequiresRegister() which is not desired to happen. Instead we create constant 6226 // locations. 6227 bool both_const = index->IsConstant() && length->IsConstant(); 6228 locations->SetInAt(0, both_const 6229 ? Location::ConstantLocation(index->AsConstant()) 6230 : ArmEncodableConstantOrRegister(index, CMP)); 6231 locations->SetInAt(1, both_const 6232 ? Location::ConstantLocation(length->AsConstant()) 6233 : ArmEncodableConstantOrRegister(length, CMP)); 6234} 6235 6236void InstructionCodeGeneratorARM::VisitBoundsCheck(HBoundsCheck* instruction) { 6237 LocationSummary* locations = instruction->GetLocations(); 6238 Location index_loc = locations->InAt(0); 6239 Location length_loc = locations->InAt(1); 6240 6241 if (length_loc.IsConstant()) { 6242 int32_t length = helpers::Int32ConstantFrom(length_loc); 6243 if (index_loc.IsConstant()) { 6244 // BCE will remove the bounds check if we are guaranteed to pass. 6245 int32_t index = helpers::Int32ConstantFrom(index_loc); 6246 if (index < 0 || index >= length) { 6247 SlowPathCodeARM* slow_path = 6248 new (GetGraph()->GetArena()) BoundsCheckSlowPathARM(instruction); 6249 codegen_->AddSlowPath(slow_path); 6250 __ b(slow_path->GetEntryLabel()); 6251 } else { 6252 // Some optimization after BCE may have generated this, and we should not 6253 // generate a bounds check if it is a valid range. 6254 } 6255 return; 6256 } 6257 6258 SlowPathCodeARM* slow_path = new (GetGraph()->GetArena()) BoundsCheckSlowPathARM(instruction); 6259 __ cmp(index_loc.AsRegister<Register>(), ShifterOperand(length)); 6260 codegen_->AddSlowPath(slow_path); 6261 __ b(slow_path->GetEntryLabel(), HS); 6262 } else { 6263 SlowPathCodeARM* slow_path = new (GetGraph()->GetArena()) BoundsCheckSlowPathARM(instruction); 6264 if (index_loc.IsConstant()) { 6265 int32_t index = helpers::Int32ConstantFrom(index_loc); 6266 __ cmp(length_loc.AsRegister<Register>(), ShifterOperand(index)); 6267 } else { 6268 __ cmp(length_loc.AsRegister<Register>(), ShifterOperand(index_loc.AsRegister<Register>())); 6269 } 6270 codegen_->AddSlowPath(slow_path); 6271 __ b(slow_path->GetEntryLabel(), LS); 6272 } 6273} 6274 6275void CodeGeneratorARM::MarkGCCard(Register temp, 6276 Register card, 6277 Register object, 6278 Register value, 6279 bool can_be_null) { 6280 Label is_null; 6281 if (can_be_null) { 6282 __ CompareAndBranchIfZero(value, &is_null); 6283 } 6284 __ LoadFromOffset(kLoadWord, card, TR, Thread::CardTableOffset<kArmPointerSize>().Int32Value()); 6285 __ Lsr(temp, object, gc::accounting::CardTable::kCardShift); 6286 __ strb(card, Address(card, temp)); 6287 if (can_be_null) { 6288 __ Bind(&is_null); 6289 } 6290} 6291 6292void LocationsBuilderARM::VisitParallelMove(HParallelMove* instruction ATTRIBUTE_UNUSED) { 6293 LOG(FATAL) << "Unreachable"; 6294} 6295 6296void InstructionCodeGeneratorARM::VisitParallelMove(HParallelMove* instruction) { 6297 codegen_->GetMoveResolver()->EmitNativeCode(instruction); 6298} 6299 6300void LocationsBuilderARM::VisitSuspendCheck(HSuspendCheck* instruction) { 6301 LocationSummary* locations = 6302 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCallOnSlowPath); 6303 locations->SetCustomSlowPathCallerSaves(RegisterSet::Empty()); // No caller-save registers. 6304} 6305 6306void InstructionCodeGeneratorARM::VisitSuspendCheck(HSuspendCheck* instruction) { 6307 HBasicBlock* block = instruction->GetBlock(); 6308 if (block->GetLoopInformation() != nullptr) { 6309 DCHECK(block->GetLoopInformation()->GetSuspendCheck() == instruction); 6310 // The back edge will generate the suspend check. 6311 return; 6312 } 6313 if (block->IsEntryBlock() && instruction->GetNext()->IsGoto()) { 6314 // The goto will generate the suspend check. 6315 return; 6316 } 6317 GenerateSuspendCheck(instruction, nullptr); 6318} 6319 6320void InstructionCodeGeneratorARM::GenerateSuspendCheck(HSuspendCheck* instruction, 6321 HBasicBlock* successor) { 6322 SuspendCheckSlowPathARM* slow_path = 6323 down_cast<SuspendCheckSlowPathARM*>(instruction->GetSlowPath()); 6324 if (slow_path == nullptr) { 6325 slow_path = new (GetGraph()->GetArena()) SuspendCheckSlowPathARM(instruction, successor); 6326 instruction->SetSlowPath(slow_path); 6327 codegen_->AddSlowPath(slow_path); 6328 if (successor != nullptr) { 6329 DCHECK(successor->IsLoopHeader()); 6330 codegen_->ClearSpillSlotsFromLoopPhisInStackMap(instruction); 6331 } 6332 } else { 6333 DCHECK_EQ(slow_path->GetSuccessor(), successor); 6334 } 6335 6336 __ LoadFromOffset( 6337 kLoadUnsignedHalfword, IP, TR, Thread::ThreadFlagsOffset<kArmPointerSize>().Int32Value()); 6338 if (successor == nullptr) { 6339 __ CompareAndBranchIfNonZero(IP, slow_path->GetEntryLabel()); 6340 __ Bind(slow_path->GetReturnLabel()); 6341 } else { 6342 __ CompareAndBranchIfZero(IP, codegen_->GetLabelOf(successor)); 6343 __ b(slow_path->GetEntryLabel()); 6344 } 6345} 6346 6347ArmAssembler* ParallelMoveResolverARM::GetAssembler() const { 6348 return codegen_->GetAssembler(); 6349} 6350 6351void ParallelMoveResolverARM::EmitMove(size_t index) { 6352 MoveOperands* move = moves_[index]; 6353 Location source = move->GetSource(); 6354 Location destination = move->GetDestination(); 6355 6356 if (source.IsRegister()) { 6357 if (destination.IsRegister()) { 6358 __ Mov(destination.AsRegister<Register>(), source.AsRegister<Register>()); 6359 } else if (destination.IsFpuRegister()) { 6360 __ vmovsr(destination.AsFpuRegister<SRegister>(), source.AsRegister<Register>()); 6361 } else { 6362 DCHECK(destination.IsStackSlot()); 6363 __ StoreToOffset(kStoreWord, source.AsRegister<Register>(), 6364 SP, destination.GetStackIndex()); 6365 } 6366 } else if (source.IsStackSlot()) { 6367 if (destination.IsRegister()) { 6368 __ LoadFromOffset(kLoadWord, destination.AsRegister<Register>(), 6369 SP, source.GetStackIndex()); 6370 } else if (destination.IsFpuRegister()) { 6371 __ LoadSFromOffset(destination.AsFpuRegister<SRegister>(), SP, source.GetStackIndex()); 6372 } else { 6373 DCHECK(destination.IsStackSlot()); 6374 __ LoadFromOffset(kLoadWord, IP, SP, source.GetStackIndex()); 6375 __ StoreToOffset(kStoreWord, IP, SP, destination.GetStackIndex()); 6376 } 6377 } else if (source.IsFpuRegister()) { 6378 if (destination.IsRegister()) { 6379 __ vmovrs(destination.AsRegister<Register>(), source.AsFpuRegister<SRegister>()); 6380 } else if (destination.IsFpuRegister()) { 6381 __ vmovs(destination.AsFpuRegister<SRegister>(), source.AsFpuRegister<SRegister>()); 6382 } else { 6383 DCHECK(destination.IsStackSlot()); 6384 __ StoreSToOffset(source.AsFpuRegister<SRegister>(), SP, destination.GetStackIndex()); 6385 } 6386 } else if (source.IsDoubleStackSlot()) { 6387 if (destination.IsDoubleStackSlot()) { 6388 __ LoadDFromOffset(DTMP, SP, source.GetStackIndex()); 6389 __ StoreDToOffset(DTMP, SP, destination.GetStackIndex()); 6390 } else if (destination.IsRegisterPair()) { 6391 DCHECK(ExpectedPairLayout(destination)); 6392 __ LoadFromOffset( 6393 kLoadWordPair, destination.AsRegisterPairLow<Register>(), SP, source.GetStackIndex()); 6394 } else { 6395 DCHECK(destination.IsFpuRegisterPair()) << destination; 6396 __ LoadDFromOffset(FromLowSToD(destination.AsFpuRegisterPairLow<SRegister>()), 6397 SP, 6398 source.GetStackIndex()); 6399 } 6400 } else if (source.IsRegisterPair()) { 6401 if (destination.IsRegisterPair()) { 6402 __ Mov(destination.AsRegisterPairLow<Register>(), source.AsRegisterPairLow<Register>()); 6403 __ Mov(destination.AsRegisterPairHigh<Register>(), source.AsRegisterPairHigh<Register>()); 6404 } else if (destination.IsFpuRegisterPair()) { 6405 __ vmovdrr(FromLowSToD(destination.AsFpuRegisterPairLow<SRegister>()), 6406 source.AsRegisterPairLow<Register>(), 6407 source.AsRegisterPairHigh<Register>()); 6408 } else { 6409 DCHECK(destination.IsDoubleStackSlot()) << destination; 6410 DCHECK(ExpectedPairLayout(source)); 6411 __ StoreToOffset( 6412 kStoreWordPair, source.AsRegisterPairLow<Register>(), SP, destination.GetStackIndex()); 6413 } 6414 } else if (source.IsFpuRegisterPair()) { 6415 if (destination.IsRegisterPair()) { 6416 __ vmovrrd(destination.AsRegisterPairLow<Register>(), 6417 destination.AsRegisterPairHigh<Register>(), 6418 FromLowSToD(source.AsFpuRegisterPairLow<SRegister>())); 6419 } else if (destination.IsFpuRegisterPair()) { 6420 __ vmovd(FromLowSToD(destination.AsFpuRegisterPairLow<SRegister>()), 6421 FromLowSToD(source.AsFpuRegisterPairLow<SRegister>())); 6422 } else { 6423 DCHECK(destination.IsDoubleStackSlot()) << destination; 6424 __ StoreDToOffset(FromLowSToD(source.AsFpuRegisterPairLow<SRegister>()), 6425 SP, 6426 destination.GetStackIndex()); 6427 } 6428 } else { 6429 DCHECK(source.IsConstant()) << source; 6430 HConstant* constant = source.GetConstant(); 6431 if (constant->IsIntConstant() || constant->IsNullConstant()) { 6432 int32_t value = CodeGenerator::GetInt32ValueOf(constant); 6433 if (destination.IsRegister()) { 6434 __ LoadImmediate(destination.AsRegister<Register>(), value); 6435 } else { 6436 DCHECK(destination.IsStackSlot()); 6437 __ LoadImmediate(IP, value); 6438 __ StoreToOffset(kStoreWord, IP, SP, destination.GetStackIndex()); 6439 } 6440 } else if (constant->IsLongConstant()) { 6441 int64_t value = constant->AsLongConstant()->GetValue(); 6442 if (destination.IsRegisterPair()) { 6443 __ LoadImmediate(destination.AsRegisterPairLow<Register>(), Low32Bits(value)); 6444 __ LoadImmediate(destination.AsRegisterPairHigh<Register>(), High32Bits(value)); 6445 } else { 6446 DCHECK(destination.IsDoubleStackSlot()) << destination; 6447 __ LoadImmediate(IP, Low32Bits(value)); 6448 __ StoreToOffset(kStoreWord, IP, SP, destination.GetStackIndex()); 6449 __ LoadImmediate(IP, High32Bits(value)); 6450 __ StoreToOffset(kStoreWord, IP, SP, destination.GetHighStackIndex(kArmWordSize)); 6451 } 6452 } else if (constant->IsDoubleConstant()) { 6453 double value = constant->AsDoubleConstant()->GetValue(); 6454 if (destination.IsFpuRegisterPair()) { 6455 __ LoadDImmediate(FromLowSToD(destination.AsFpuRegisterPairLow<SRegister>()), value); 6456 } else { 6457 DCHECK(destination.IsDoubleStackSlot()) << destination; 6458 uint64_t int_value = bit_cast<uint64_t, double>(value); 6459 __ LoadImmediate(IP, Low32Bits(int_value)); 6460 __ StoreToOffset(kStoreWord, IP, SP, destination.GetStackIndex()); 6461 __ LoadImmediate(IP, High32Bits(int_value)); 6462 __ StoreToOffset(kStoreWord, IP, SP, destination.GetHighStackIndex(kArmWordSize)); 6463 } 6464 } else { 6465 DCHECK(constant->IsFloatConstant()) << constant->DebugName(); 6466 float value = constant->AsFloatConstant()->GetValue(); 6467 if (destination.IsFpuRegister()) { 6468 __ LoadSImmediate(destination.AsFpuRegister<SRegister>(), value); 6469 } else { 6470 DCHECK(destination.IsStackSlot()); 6471 __ LoadImmediate(IP, bit_cast<int32_t, float>(value)); 6472 __ StoreToOffset(kStoreWord, IP, SP, destination.GetStackIndex()); 6473 } 6474 } 6475 } 6476} 6477 6478void ParallelMoveResolverARM::Exchange(Register reg, int mem) { 6479 __ Mov(IP, reg); 6480 __ LoadFromOffset(kLoadWord, reg, SP, mem); 6481 __ StoreToOffset(kStoreWord, IP, SP, mem); 6482} 6483 6484void ParallelMoveResolverARM::Exchange(int mem1, int mem2) { 6485 ScratchRegisterScope ensure_scratch(this, IP, R0, codegen_->GetNumberOfCoreRegisters()); 6486 int stack_offset = ensure_scratch.IsSpilled() ? kArmWordSize : 0; 6487 __ LoadFromOffset(kLoadWord, static_cast<Register>(ensure_scratch.GetRegister()), 6488 SP, mem1 + stack_offset); 6489 __ LoadFromOffset(kLoadWord, IP, SP, mem2 + stack_offset); 6490 __ StoreToOffset(kStoreWord, static_cast<Register>(ensure_scratch.GetRegister()), 6491 SP, mem2 + stack_offset); 6492 __ StoreToOffset(kStoreWord, IP, SP, mem1 + stack_offset); 6493} 6494 6495void ParallelMoveResolverARM::EmitSwap(size_t index) { 6496 MoveOperands* move = moves_[index]; 6497 Location source = move->GetSource(); 6498 Location destination = move->GetDestination(); 6499 6500 if (source.IsRegister() && destination.IsRegister()) { 6501 DCHECK_NE(source.AsRegister<Register>(), IP); 6502 DCHECK_NE(destination.AsRegister<Register>(), IP); 6503 __ Mov(IP, source.AsRegister<Register>()); 6504 __ Mov(source.AsRegister<Register>(), destination.AsRegister<Register>()); 6505 __ Mov(destination.AsRegister<Register>(), IP); 6506 } else if (source.IsRegister() && destination.IsStackSlot()) { 6507 Exchange(source.AsRegister<Register>(), destination.GetStackIndex()); 6508 } else if (source.IsStackSlot() && destination.IsRegister()) { 6509 Exchange(destination.AsRegister<Register>(), source.GetStackIndex()); 6510 } else if (source.IsStackSlot() && destination.IsStackSlot()) { 6511 Exchange(source.GetStackIndex(), destination.GetStackIndex()); 6512 } else if (source.IsFpuRegister() && destination.IsFpuRegister()) { 6513 __ vmovrs(IP, source.AsFpuRegister<SRegister>()); 6514 __ vmovs(source.AsFpuRegister<SRegister>(), destination.AsFpuRegister<SRegister>()); 6515 __ vmovsr(destination.AsFpuRegister<SRegister>(), IP); 6516 } else if (source.IsRegisterPair() && destination.IsRegisterPair()) { 6517 __ vmovdrr(DTMP, source.AsRegisterPairLow<Register>(), source.AsRegisterPairHigh<Register>()); 6518 __ Mov(source.AsRegisterPairLow<Register>(), destination.AsRegisterPairLow<Register>()); 6519 __ Mov(source.AsRegisterPairHigh<Register>(), destination.AsRegisterPairHigh<Register>()); 6520 __ vmovrrd(destination.AsRegisterPairLow<Register>(), 6521 destination.AsRegisterPairHigh<Register>(), 6522 DTMP); 6523 } else if (source.IsRegisterPair() || destination.IsRegisterPair()) { 6524 Register low_reg = source.IsRegisterPair() 6525 ? source.AsRegisterPairLow<Register>() 6526 : destination.AsRegisterPairLow<Register>(); 6527 int mem = source.IsRegisterPair() 6528 ? destination.GetStackIndex() 6529 : source.GetStackIndex(); 6530 DCHECK(ExpectedPairLayout(source.IsRegisterPair() ? source : destination)); 6531 __ vmovdrr(DTMP, low_reg, static_cast<Register>(low_reg + 1)); 6532 __ LoadFromOffset(kLoadWordPair, low_reg, SP, mem); 6533 __ StoreDToOffset(DTMP, SP, mem); 6534 } else if (source.IsFpuRegisterPair() && destination.IsFpuRegisterPair()) { 6535 DRegister first = FromLowSToD(source.AsFpuRegisterPairLow<SRegister>()); 6536 DRegister second = FromLowSToD(destination.AsFpuRegisterPairLow<SRegister>()); 6537 __ vmovd(DTMP, first); 6538 __ vmovd(first, second); 6539 __ vmovd(second, DTMP); 6540 } else if (source.IsFpuRegisterPair() || destination.IsFpuRegisterPair()) { 6541 DRegister reg = source.IsFpuRegisterPair() 6542 ? FromLowSToD(source.AsFpuRegisterPairLow<SRegister>()) 6543 : FromLowSToD(destination.AsFpuRegisterPairLow<SRegister>()); 6544 int mem = source.IsFpuRegisterPair() 6545 ? destination.GetStackIndex() 6546 : source.GetStackIndex(); 6547 __ vmovd(DTMP, reg); 6548 __ LoadDFromOffset(reg, SP, mem); 6549 __ StoreDToOffset(DTMP, SP, mem); 6550 } else if (source.IsFpuRegister() || destination.IsFpuRegister()) { 6551 SRegister reg = source.IsFpuRegister() ? source.AsFpuRegister<SRegister>() 6552 : destination.AsFpuRegister<SRegister>(); 6553 int mem = source.IsFpuRegister() 6554 ? destination.GetStackIndex() 6555 : source.GetStackIndex(); 6556 6557 __ vmovrs(IP, reg); 6558 __ LoadSFromOffset(reg, SP, mem); 6559 __ StoreToOffset(kStoreWord, IP, SP, mem); 6560 } else if (source.IsDoubleStackSlot() && destination.IsDoubleStackSlot()) { 6561 Exchange(source.GetStackIndex(), destination.GetStackIndex()); 6562 Exchange(source.GetHighStackIndex(kArmWordSize), destination.GetHighStackIndex(kArmWordSize)); 6563 } else { 6564 LOG(FATAL) << "Unimplemented" << source << " <-> " << destination; 6565 } 6566} 6567 6568void ParallelMoveResolverARM::SpillScratch(int reg) { 6569 __ Push(static_cast<Register>(reg)); 6570} 6571 6572void ParallelMoveResolverARM::RestoreScratch(int reg) { 6573 __ Pop(static_cast<Register>(reg)); 6574} 6575 6576HLoadClass::LoadKind CodeGeneratorARM::GetSupportedLoadClassKind( 6577 HLoadClass::LoadKind desired_class_load_kind) { 6578 switch (desired_class_load_kind) { 6579 case HLoadClass::LoadKind::kInvalid: 6580 LOG(FATAL) << "UNREACHABLE"; 6581 UNREACHABLE(); 6582 case HLoadClass::LoadKind::kReferrersClass: 6583 break; 6584 case HLoadClass::LoadKind::kBootImageLinkTimeAddress: 6585 DCHECK(!GetCompilerOptions().GetCompilePic()); 6586 break; 6587 case HLoadClass::LoadKind::kBootImageLinkTimePcRelative: 6588 DCHECK(GetCompilerOptions().GetCompilePic()); 6589 break; 6590 case HLoadClass::LoadKind::kBootImageAddress: 6591 break; 6592 case HLoadClass::LoadKind::kBssEntry: 6593 DCHECK(!Runtime::Current()->UseJitCompilation()); 6594 break; 6595 case HLoadClass::LoadKind::kJitTableAddress: 6596 DCHECK(Runtime::Current()->UseJitCompilation()); 6597 break; 6598 case HLoadClass::LoadKind::kDexCacheViaMethod: 6599 break; 6600 } 6601 return desired_class_load_kind; 6602} 6603 6604void LocationsBuilderARM::VisitLoadClass(HLoadClass* cls) { 6605 HLoadClass::LoadKind load_kind = cls->GetLoadKind(); 6606 if (load_kind == HLoadClass::LoadKind::kDexCacheViaMethod) { 6607 InvokeRuntimeCallingConvention calling_convention; 6608 CodeGenerator::CreateLoadClassRuntimeCallLocationSummary( 6609 cls, 6610 Location::RegisterLocation(calling_convention.GetRegisterAt(0)), 6611 Location::RegisterLocation(R0)); 6612 DCHECK_EQ(calling_convention.GetRegisterAt(0), R0); 6613 return; 6614 } 6615 DCHECK(!cls->NeedsAccessCheck()); 6616 6617 const bool requires_read_barrier = kEmitCompilerReadBarrier && !cls->IsInBootImage(); 6618 LocationSummary::CallKind call_kind = (cls->NeedsEnvironment() || requires_read_barrier) 6619 ? LocationSummary::kCallOnSlowPath 6620 : LocationSummary::kNoCall; 6621 LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(cls, call_kind); 6622 if (kUseBakerReadBarrier && requires_read_barrier && !cls->NeedsEnvironment()) { 6623 locations->SetCustomSlowPathCallerSaves(RegisterSet::Empty()); // No caller-save registers. 6624 } 6625 6626 if (load_kind == HLoadClass::LoadKind::kReferrersClass) { 6627 locations->SetInAt(0, Location::RequiresRegister()); 6628 } 6629 locations->SetOut(Location::RequiresRegister()); 6630 if (load_kind == HLoadClass::LoadKind::kBssEntry) { 6631 if (!kUseReadBarrier || kUseBakerReadBarrier) { 6632 // Rely on the type resolution or initialization and marking to save everything we need. 6633 // Note that IP may be clobbered by saving/restoring the live register (only one thanks 6634 // to the custom calling convention) or by marking, so we request a different temp. 6635 locations->AddTemp(Location::RequiresRegister()); 6636 RegisterSet caller_saves = RegisterSet::Empty(); 6637 InvokeRuntimeCallingConvention calling_convention; 6638 caller_saves.Add(Location::RegisterLocation(calling_convention.GetRegisterAt(0))); 6639 // TODO: Add GetReturnLocation() to the calling convention so that we can DCHECK() 6640 // that the the kPrimNot result register is the same as the first argument register. 6641 locations->SetCustomSlowPathCallerSaves(caller_saves); 6642 } else { 6643 // For non-Baker read barrier we have a temp-clobbering call. 6644 } 6645 } 6646} 6647 6648// NO_THREAD_SAFETY_ANALYSIS as we manipulate handles whose internal object we know does not 6649// move. 6650void InstructionCodeGeneratorARM::VisitLoadClass(HLoadClass* cls) NO_THREAD_SAFETY_ANALYSIS { 6651 HLoadClass::LoadKind load_kind = cls->GetLoadKind(); 6652 if (load_kind == HLoadClass::LoadKind::kDexCacheViaMethod) { 6653 codegen_->GenerateLoadClassRuntimeCall(cls); 6654 return; 6655 } 6656 DCHECK(!cls->NeedsAccessCheck()); 6657 6658 LocationSummary* locations = cls->GetLocations(); 6659 Location out_loc = locations->Out(); 6660 Register out = out_loc.AsRegister<Register>(); 6661 6662 const ReadBarrierOption read_barrier_option = cls->IsInBootImage() 6663 ? kWithoutReadBarrier 6664 : kCompilerReadBarrierOption; 6665 bool generate_null_check = false; 6666 switch (load_kind) { 6667 case HLoadClass::LoadKind::kReferrersClass: { 6668 DCHECK(!cls->CanCallRuntime()); 6669 DCHECK(!cls->MustGenerateClinitCheck()); 6670 // /* GcRoot<mirror::Class> */ out = current_method->declaring_class_ 6671 Register current_method = locations->InAt(0).AsRegister<Register>(); 6672 GenerateGcRootFieldLoad(cls, 6673 out_loc, 6674 current_method, 6675 ArtMethod::DeclaringClassOffset().Int32Value(), 6676 read_barrier_option); 6677 break; 6678 } 6679 case HLoadClass::LoadKind::kBootImageLinkTimeAddress: { 6680 DCHECK(codegen_->GetCompilerOptions().IsBootImage()); 6681 DCHECK_EQ(read_barrier_option, kWithoutReadBarrier); 6682 __ LoadLiteral(out, codegen_->DeduplicateBootImageTypeLiteral(cls->GetDexFile(), 6683 cls->GetTypeIndex())); 6684 break; 6685 } 6686 case HLoadClass::LoadKind::kBootImageLinkTimePcRelative: { 6687 DCHECK(codegen_->GetCompilerOptions().IsBootImage()); 6688 DCHECK_EQ(read_barrier_option, kWithoutReadBarrier); 6689 CodeGeneratorARM::PcRelativePatchInfo* labels = 6690 codegen_->NewPcRelativeTypePatch(cls->GetDexFile(), cls->GetTypeIndex()); 6691 __ BindTrackedLabel(&labels->movw_label); 6692 __ movw(out, /* placeholder */ 0u); 6693 __ BindTrackedLabel(&labels->movt_label); 6694 __ movt(out, /* placeholder */ 0u); 6695 __ BindTrackedLabel(&labels->add_pc_label); 6696 __ add(out, out, ShifterOperand(PC)); 6697 break; 6698 } 6699 case HLoadClass::LoadKind::kBootImageAddress: { 6700 DCHECK_EQ(read_barrier_option, kWithoutReadBarrier); 6701 uint32_t address = dchecked_integral_cast<uint32_t>( 6702 reinterpret_cast<uintptr_t>(cls->GetClass().Get())); 6703 DCHECK_NE(address, 0u); 6704 __ LoadLiteral(out, codegen_->DeduplicateBootImageAddressLiteral(address)); 6705 break; 6706 } 6707 case HLoadClass::LoadKind::kBssEntry: { 6708 Register temp = (!kUseReadBarrier || kUseBakerReadBarrier) 6709 ? locations->GetTemp(0).AsRegister<Register>() 6710 : out; 6711 CodeGeneratorARM::PcRelativePatchInfo* labels = 6712 codegen_->NewTypeBssEntryPatch(cls->GetDexFile(), cls->GetTypeIndex()); 6713 __ BindTrackedLabel(&labels->movw_label); 6714 __ movw(temp, /* placeholder */ 0u); 6715 __ BindTrackedLabel(&labels->movt_label); 6716 __ movt(temp, /* placeholder */ 0u); 6717 __ BindTrackedLabel(&labels->add_pc_label); 6718 __ add(temp, temp, ShifterOperand(PC)); 6719 GenerateGcRootFieldLoad(cls, out_loc, temp, /* offset */ 0, read_barrier_option); 6720 generate_null_check = true; 6721 break; 6722 } 6723 case HLoadClass::LoadKind::kJitTableAddress: { 6724 __ LoadLiteral(out, codegen_->DeduplicateJitClassLiteral(cls->GetDexFile(), 6725 cls->GetTypeIndex(), 6726 cls->GetClass())); 6727 // /* GcRoot<mirror::Class> */ out = *out 6728 GenerateGcRootFieldLoad(cls, out_loc, out, /* offset */ 0, read_barrier_option); 6729 break; 6730 } 6731 case HLoadClass::LoadKind::kDexCacheViaMethod: 6732 case HLoadClass::LoadKind::kInvalid: 6733 LOG(FATAL) << "UNREACHABLE"; 6734 UNREACHABLE(); 6735 } 6736 6737 if (generate_null_check || cls->MustGenerateClinitCheck()) { 6738 DCHECK(cls->CanCallRuntime()); 6739 SlowPathCodeARM* slow_path = new (GetGraph()->GetArena()) LoadClassSlowPathARM( 6740 cls, cls, cls->GetDexPc(), cls->MustGenerateClinitCheck()); 6741 codegen_->AddSlowPath(slow_path); 6742 if (generate_null_check) { 6743 __ CompareAndBranchIfZero(out, slow_path->GetEntryLabel()); 6744 } 6745 if (cls->MustGenerateClinitCheck()) { 6746 GenerateClassInitializationCheck(slow_path, out); 6747 } else { 6748 __ Bind(slow_path->GetExitLabel()); 6749 } 6750 } 6751} 6752 6753void LocationsBuilderARM::VisitClinitCheck(HClinitCheck* check) { 6754 LocationSummary* locations = 6755 new (GetGraph()->GetArena()) LocationSummary(check, LocationSummary::kCallOnSlowPath); 6756 locations->SetInAt(0, Location::RequiresRegister()); 6757 if (check->HasUses()) { 6758 locations->SetOut(Location::SameAsFirstInput()); 6759 } 6760} 6761 6762void InstructionCodeGeneratorARM::VisitClinitCheck(HClinitCheck* check) { 6763 // We assume the class is not null. 6764 SlowPathCodeARM* slow_path = new (GetGraph()->GetArena()) LoadClassSlowPathARM( 6765 check->GetLoadClass(), check, check->GetDexPc(), true); 6766 codegen_->AddSlowPath(slow_path); 6767 GenerateClassInitializationCheck(slow_path, 6768 check->GetLocations()->InAt(0).AsRegister<Register>()); 6769} 6770 6771void InstructionCodeGeneratorARM::GenerateClassInitializationCheck( 6772 SlowPathCodeARM* slow_path, Register class_reg) { 6773 __ LoadFromOffset(kLoadWord, IP, class_reg, mirror::Class::StatusOffset().Int32Value()); 6774 __ cmp(IP, ShifterOperand(mirror::Class::kStatusInitialized)); 6775 __ b(slow_path->GetEntryLabel(), LT); 6776 // Even if the initialized flag is set, we may be in a situation where caches are not synced 6777 // properly. Therefore, we do a memory fence. 6778 __ dmb(ISH); 6779 __ Bind(slow_path->GetExitLabel()); 6780} 6781 6782HLoadString::LoadKind CodeGeneratorARM::GetSupportedLoadStringKind( 6783 HLoadString::LoadKind desired_string_load_kind) { 6784 switch (desired_string_load_kind) { 6785 case HLoadString::LoadKind::kBootImageLinkTimeAddress: 6786 DCHECK(!GetCompilerOptions().GetCompilePic()); 6787 break; 6788 case HLoadString::LoadKind::kBootImageLinkTimePcRelative: 6789 DCHECK(GetCompilerOptions().GetCompilePic()); 6790 break; 6791 case HLoadString::LoadKind::kBootImageAddress: 6792 break; 6793 case HLoadString::LoadKind::kBssEntry: 6794 DCHECK(!Runtime::Current()->UseJitCompilation()); 6795 break; 6796 case HLoadString::LoadKind::kJitTableAddress: 6797 DCHECK(Runtime::Current()->UseJitCompilation()); 6798 break; 6799 case HLoadString::LoadKind::kDexCacheViaMethod: 6800 break; 6801 } 6802 return desired_string_load_kind; 6803} 6804 6805void LocationsBuilderARM::VisitLoadString(HLoadString* load) { 6806 LocationSummary::CallKind call_kind = CodeGenerator::GetLoadStringCallKind(load); 6807 LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(load, call_kind); 6808 HLoadString::LoadKind load_kind = load->GetLoadKind(); 6809 if (load_kind == HLoadString::LoadKind::kDexCacheViaMethod) { 6810 locations->SetOut(Location::RegisterLocation(R0)); 6811 } else { 6812 locations->SetOut(Location::RequiresRegister()); 6813 if (load_kind == HLoadString::LoadKind::kBssEntry) { 6814 if (!kUseReadBarrier || kUseBakerReadBarrier) { 6815 // Rely on the pResolveString and marking to save everything we need, including temps. 6816 // Note that IP may be clobbered by saving/restoring the live register (only one thanks 6817 // to the custom calling convention) or by marking, so we request a different temp. 6818 locations->AddTemp(Location::RequiresRegister()); 6819 RegisterSet caller_saves = RegisterSet::Empty(); 6820 InvokeRuntimeCallingConvention calling_convention; 6821 caller_saves.Add(Location::RegisterLocation(calling_convention.GetRegisterAt(0))); 6822 // TODO: Add GetReturnLocation() to the calling convention so that we can DCHECK() 6823 // that the the kPrimNot result register is the same as the first argument register. 6824 locations->SetCustomSlowPathCallerSaves(caller_saves); 6825 } else { 6826 // For non-Baker read barrier we have a temp-clobbering call. 6827 } 6828 } 6829 } 6830} 6831 6832// NO_THREAD_SAFETY_ANALYSIS as we manipulate handles whose internal object we know does not 6833// move. 6834void InstructionCodeGeneratorARM::VisitLoadString(HLoadString* load) NO_THREAD_SAFETY_ANALYSIS { 6835 LocationSummary* locations = load->GetLocations(); 6836 Location out_loc = locations->Out(); 6837 Register out = out_loc.AsRegister<Register>(); 6838 HLoadString::LoadKind load_kind = load->GetLoadKind(); 6839 6840 switch (load_kind) { 6841 case HLoadString::LoadKind::kBootImageLinkTimeAddress: { 6842 DCHECK(codegen_->GetCompilerOptions().IsBootImage()); 6843 __ LoadLiteral(out, codegen_->DeduplicateBootImageStringLiteral(load->GetDexFile(), 6844 load->GetStringIndex())); 6845 return; // No dex cache slow path. 6846 } 6847 case HLoadString::LoadKind::kBootImageLinkTimePcRelative: { 6848 DCHECK(codegen_->GetCompilerOptions().IsBootImage()); 6849 CodeGeneratorARM::PcRelativePatchInfo* labels = 6850 codegen_->NewPcRelativeStringPatch(load->GetDexFile(), load->GetStringIndex()); 6851 __ BindTrackedLabel(&labels->movw_label); 6852 __ movw(out, /* placeholder */ 0u); 6853 __ BindTrackedLabel(&labels->movt_label); 6854 __ movt(out, /* placeholder */ 0u); 6855 __ BindTrackedLabel(&labels->add_pc_label); 6856 __ add(out, out, ShifterOperand(PC)); 6857 return; // No dex cache slow path. 6858 } 6859 case HLoadString::LoadKind::kBootImageAddress: { 6860 uint32_t address = dchecked_integral_cast<uint32_t>( 6861 reinterpret_cast<uintptr_t>(load->GetString().Get())); 6862 DCHECK_NE(address, 0u); 6863 __ LoadLiteral(out, codegen_->DeduplicateBootImageAddressLiteral(address)); 6864 return; // No dex cache slow path. 6865 } 6866 case HLoadString::LoadKind::kBssEntry: { 6867 DCHECK(!codegen_->GetCompilerOptions().IsBootImage()); 6868 Register temp = (!kUseReadBarrier || kUseBakerReadBarrier) 6869 ? locations->GetTemp(0).AsRegister<Register>() 6870 : out; 6871 CodeGeneratorARM::PcRelativePatchInfo* labels = 6872 codegen_->NewPcRelativeStringPatch(load->GetDexFile(), load->GetStringIndex()); 6873 __ BindTrackedLabel(&labels->movw_label); 6874 __ movw(temp, /* placeholder */ 0u); 6875 __ BindTrackedLabel(&labels->movt_label); 6876 __ movt(temp, /* placeholder */ 0u); 6877 __ BindTrackedLabel(&labels->add_pc_label); 6878 __ add(temp, temp, ShifterOperand(PC)); 6879 GenerateGcRootFieldLoad(load, out_loc, temp, /* offset */ 0, kCompilerReadBarrierOption); 6880 SlowPathCode* slow_path = new (GetGraph()->GetArena()) LoadStringSlowPathARM(load); 6881 codegen_->AddSlowPath(slow_path); 6882 __ CompareAndBranchIfZero(out, slow_path->GetEntryLabel()); 6883 __ Bind(slow_path->GetExitLabel()); 6884 return; 6885 } 6886 case HLoadString::LoadKind::kJitTableAddress: { 6887 __ LoadLiteral(out, codegen_->DeduplicateJitStringLiteral(load->GetDexFile(), 6888 load->GetStringIndex(), 6889 load->GetString())); 6890 // /* GcRoot<mirror::String> */ out = *out 6891 GenerateGcRootFieldLoad(load, out_loc, out, /* offset */ 0, kCompilerReadBarrierOption); 6892 return; 6893 } 6894 default: 6895 break; 6896 } 6897 6898 // TODO: Consider re-adding the compiler code to do string dex cache lookup again. 6899 DCHECK(load_kind == HLoadString::LoadKind::kDexCacheViaMethod); 6900 InvokeRuntimeCallingConvention calling_convention; 6901 DCHECK_EQ(calling_convention.GetRegisterAt(0), out); 6902 __ LoadImmediate(calling_convention.GetRegisterAt(0), load->GetStringIndex().index_); 6903 codegen_->InvokeRuntime(kQuickResolveString, load, load->GetDexPc()); 6904 CheckEntrypointTypes<kQuickResolveString, void*, uint32_t>(); 6905} 6906 6907static int32_t GetExceptionTlsOffset() { 6908 return Thread::ExceptionOffset<kArmPointerSize>().Int32Value(); 6909} 6910 6911void LocationsBuilderARM::VisitLoadException(HLoadException* load) { 6912 LocationSummary* locations = 6913 new (GetGraph()->GetArena()) LocationSummary(load, LocationSummary::kNoCall); 6914 locations->SetOut(Location::RequiresRegister()); 6915} 6916 6917void InstructionCodeGeneratorARM::VisitLoadException(HLoadException* load) { 6918 Register out = load->GetLocations()->Out().AsRegister<Register>(); 6919 __ LoadFromOffset(kLoadWord, out, TR, GetExceptionTlsOffset()); 6920} 6921 6922void LocationsBuilderARM::VisitClearException(HClearException* clear) { 6923 new (GetGraph()->GetArena()) LocationSummary(clear, LocationSummary::kNoCall); 6924} 6925 6926void InstructionCodeGeneratorARM::VisitClearException(HClearException* clear ATTRIBUTE_UNUSED) { 6927 __ LoadImmediate(IP, 0); 6928 __ StoreToOffset(kStoreWord, IP, TR, GetExceptionTlsOffset()); 6929} 6930 6931void LocationsBuilderARM::VisitThrow(HThrow* instruction) { 6932 LocationSummary* locations = 6933 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCallOnMainOnly); 6934 InvokeRuntimeCallingConvention calling_convention; 6935 locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(0))); 6936} 6937 6938void InstructionCodeGeneratorARM::VisitThrow(HThrow* instruction) { 6939 codegen_->InvokeRuntime(kQuickDeliverException, instruction, instruction->GetDexPc()); 6940 CheckEntrypointTypes<kQuickDeliverException, void, mirror::Object*>(); 6941} 6942 6943// Temp is used for read barrier. 6944static size_t NumberOfInstanceOfTemps(TypeCheckKind type_check_kind) { 6945 if (kEmitCompilerReadBarrier && 6946 (kUseBakerReadBarrier || 6947 type_check_kind == TypeCheckKind::kAbstractClassCheck || 6948 type_check_kind == TypeCheckKind::kClassHierarchyCheck || 6949 type_check_kind == TypeCheckKind::kArrayObjectCheck)) { 6950 return 1; 6951 } 6952 return 0; 6953} 6954 6955// Interface case has 3 temps, one for holding the number of interfaces, one for the current 6956// interface pointer, one for loading the current interface. 6957// The other checks have one temp for loading the object's class. 6958static size_t NumberOfCheckCastTemps(TypeCheckKind type_check_kind) { 6959 if (type_check_kind == TypeCheckKind::kInterfaceCheck) { 6960 return 3; 6961 } 6962 return 1 + NumberOfInstanceOfTemps(type_check_kind); 6963} 6964 6965void LocationsBuilderARM::VisitInstanceOf(HInstanceOf* instruction) { 6966 LocationSummary::CallKind call_kind = LocationSummary::kNoCall; 6967 TypeCheckKind type_check_kind = instruction->GetTypeCheckKind(); 6968 bool baker_read_barrier_slow_path = false; 6969 switch (type_check_kind) { 6970 case TypeCheckKind::kExactCheck: 6971 case TypeCheckKind::kAbstractClassCheck: 6972 case TypeCheckKind::kClassHierarchyCheck: 6973 case TypeCheckKind::kArrayObjectCheck: 6974 call_kind = 6975 kEmitCompilerReadBarrier ? LocationSummary::kCallOnSlowPath : LocationSummary::kNoCall; 6976 baker_read_barrier_slow_path = kUseBakerReadBarrier; 6977 break; 6978 case TypeCheckKind::kArrayCheck: 6979 case TypeCheckKind::kUnresolvedCheck: 6980 case TypeCheckKind::kInterfaceCheck: 6981 call_kind = LocationSummary::kCallOnSlowPath; 6982 break; 6983 } 6984 6985 LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction, call_kind); 6986 if (baker_read_barrier_slow_path) { 6987 locations->SetCustomSlowPathCallerSaves(RegisterSet::Empty()); // No caller-save registers. 6988 } 6989 locations->SetInAt(0, Location::RequiresRegister()); 6990 locations->SetInAt(1, Location::RequiresRegister()); 6991 // The "out" register is used as a temporary, so it overlaps with the inputs. 6992 // Note that TypeCheckSlowPathARM uses this register too. 6993 locations->SetOut(Location::RequiresRegister(), Location::kOutputOverlap); 6994 locations->AddRegisterTemps(NumberOfInstanceOfTemps(type_check_kind)); 6995} 6996 6997void InstructionCodeGeneratorARM::VisitInstanceOf(HInstanceOf* instruction) { 6998 TypeCheckKind type_check_kind = instruction->GetTypeCheckKind(); 6999 LocationSummary* locations = instruction->GetLocations(); 7000 Location obj_loc = locations->InAt(0); 7001 Register obj = obj_loc.AsRegister<Register>(); 7002 Register cls = locations->InAt(1).AsRegister<Register>(); 7003 Location out_loc = locations->Out(); 7004 Register out = out_loc.AsRegister<Register>(); 7005 const size_t num_temps = NumberOfInstanceOfTemps(type_check_kind); 7006 DCHECK_LE(num_temps, 1u); 7007 Location maybe_temp_loc = (num_temps >= 1) ? locations->GetTemp(0) : Location::NoLocation(); 7008 uint32_t class_offset = mirror::Object::ClassOffset().Int32Value(); 7009 uint32_t super_offset = mirror::Class::SuperClassOffset().Int32Value(); 7010 uint32_t component_offset = mirror::Class::ComponentTypeOffset().Int32Value(); 7011 uint32_t primitive_offset = mirror::Class::PrimitiveTypeOffset().Int32Value(); 7012 Label done, zero; 7013 SlowPathCodeARM* slow_path = nullptr; 7014 7015 // Return 0 if `obj` is null. 7016 // avoid null check if we know obj is not null. 7017 if (instruction->MustDoNullCheck()) { 7018 __ CompareAndBranchIfZero(obj, &zero); 7019 } 7020 7021 switch (type_check_kind) { 7022 case TypeCheckKind::kExactCheck: { 7023 // /* HeapReference<Class> */ out = obj->klass_ 7024 GenerateReferenceLoadTwoRegisters(instruction, 7025 out_loc, 7026 obj_loc, 7027 class_offset, 7028 maybe_temp_loc, 7029 kCompilerReadBarrierOption); 7030 __ cmp(out, ShifterOperand(cls)); 7031 // Classes must be equal for the instanceof to succeed. 7032 __ b(&zero, NE); 7033 __ LoadImmediate(out, 1); 7034 __ b(&done); 7035 break; 7036 } 7037 7038 case TypeCheckKind::kAbstractClassCheck: { 7039 // /* HeapReference<Class> */ out = obj->klass_ 7040 GenerateReferenceLoadTwoRegisters(instruction, 7041 out_loc, 7042 obj_loc, 7043 class_offset, 7044 maybe_temp_loc, 7045 kCompilerReadBarrierOption); 7046 // If the class is abstract, we eagerly fetch the super class of the 7047 // object to avoid doing a comparison we know will fail. 7048 Label loop; 7049 __ Bind(&loop); 7050 // /* HeapReference<Class> */ out = out->super_class_ 7051 GenerateReferenceLoadOneRegister(instruction, 7052 out_loc, 7053 super_offset, 7054 maybe_temp_loc, 7055 kCompilerReadBarrierOption); 7056 // If `out` is null, we use it for the result, and jump to `done`. 7057 __ CompareAndBranchIfZero(out, &done); 7058 __ cmp(out, ShifterOperand(cls)); 7059 __ b(&loop, NE); 7060 __ LoadImmediate(out, 1); 7061 if (zero.IsLinked()) { 7062 __ b(&done); 7063 } 7064 break; 7065 } 7066 7067 case TypeCheckKind::kClassHierarchyCheck: { 7068 // /* HeapReference<Class> */ out = obj->klass_ 7069 GenerateReferenceLoadTwoRegisters(instruction, 7070 out_loc, 7071 obj_loc, 7072 class_offset, 7073 maybe_temp_loc, 7074 kCompilerReadBarrierOption); 7075 // Walk over the class hierarchy to find a match. 7076 Label loop, success; 7077 __ Bind(&loop); 7078 __ cmp(out, ShifterOperand(cls)); 7079 __ b(&success, EQ); 7080 // /* HeapReference<Class> */ out = out->super_class_ 7081 GenerateReferenceLoadOneRegister(instruction, 7082 out_loc, 7083 super_offset, 7084 maybe_temp_loc, 7085 kCompilerReadBarrierOption); 7086 __ CompareAndBranchIfNonZero(out, &loop); 7087 // If `out` is null, we use it for the result, and jump to `done`. 7088 __ b(&done); 7089 __ Bind(&success); 7090 __ LoadImmediate(out, 1); 7091 if (zero.IsLinked()) { 7092 __ b(&done); 7093 } 7094 break; 7095 } 7096 7097 case TypeCheckKind::kArrayObjectCheck: { 7098 // /* HeapReference<Class> */ out = obj->klass_ 7099 GenerateReferenceLoadTwoRegisters(instruction, 7100 out_loc, 7101 obj_loc, 7102 class_offset, 7103 maybe_temp_loc, 7104 kCompilerReadBarrierOption); 7105 // Do an exact check. 7106 Label exact_check; 7107 __ cmp(out, ShifterOperand(cls)); 7108 __ b(&exact_check, EQ); 7109 // Otherwise, we need to check that the object's class is a non-primitive array. 7110 // /* HeapReference<Class> */ out = out->component_type_ 7111 GenerateReferenceLoadOneRegister(instruction, 7112 out_loc, 7113 component_offset, 7114 maybe_temp_loc, 7115 kCompilerReadBarrierOption); 7116 // If `out` is null, we use it for the result, and jump to `done`. 7117 __ CompareAndBranchIfZero(out, &done); 7118 __ LoadFromOffset(kLoadUnsignedHalfword, out, out, primitive_offset); 7119 static_assert(Primitive::kPrimNot == 0, "Expected 0 for kPrimNot"); 7120 __ CompareAndBranchIfNonZero(out, &zero); 7121 __ Bind(&exact_check); 7122 __ LoadImmediate(out, 1); 7123 __ b(&done); 7124 break; 7125 } 7126 7127 case TypeCheckKind::kArrayCheck: { 7128 // No read barrier since the slow path will retry upon failure. 7129 // /* HeapReference<Class> */ out = obj->klass_ 7130 GenerateReferenceLoadTwoRegisters(instruction, 7131 out_loc, 7132 obj_loc, 7133 class_offset, 7134 maybe_temp_loc, 7135 kWithoutReadBarrier); 7136 __ cmp(out, ShifterOperand(cls)); 7137 DCHECK(locations->OnlyCallsOnSlowPath()); 7138 slow_path = new (GetGraph()->GetArena()) TypeCheckSlowPathARM(instruction, 7139 /* is_fatal */ false); 7140 codegen_->AddSlowPath(slow_path); 7141 __ b(slow_path->GetEntryLabel(), NE); 7142 __ LoadImmediate(out, 1); 7143 if (zero.IsLinked()) { 7144 __ b(&done); 7145 } 7146 break; 7147 } 7148 7149 case TypeCheckKind::kUnresolvedCheck: 7150 case TypeCheckKind::kInterfaceCheck: { 7151 // Note that we indeed only call on slow path, but we always go 7152 // into the slow path for the unresolved and interface check 7153 // cases. 7154 // 7155 // We cannot directly call the InstanceofNonTrivial runtime 7156 // entry point without resorting to a type checking slow path 7157 // here (i.e. by calling InvokeRuntime directly), as it would 7158 // require to assign fixed registers for the inputs of this 7159 // HInstanceOf instruction (following the runtime calling 7160 // convention), which might be cluttered by the potential first 7161 // read barrier emission at the beginning of this method. 7162 // 7163 // TODO: Introduce a new runtime entry point taking the object 7164 // to test (instead of its class) as argument, and let it deal 7165 // with the read barrier issues. This will let us refactor this 7166 // case of the `switch` code as it was previously (with a direct 7167 // call to the runtime not using a type checking slow path). 7168 // This should also be beneficial for the other cases above. 7169 DCHECK(locations->OnlyCallsOnSlowPath()); 7170 slow_path = new (GetGraph()->GetArena()) TypeCheckSlowPathARM(instruction, 7171 /* is_fatal */ false); 7172 codegen_->AddSlowPath(slow_path); 7173 __ b(slow_path->GetEntryLabel()); 7174 if (zero.IsLinked()) { 7175 __ b(&done); 7176 } 7177 break; 7178 } 7179 } 7180 7181 if (zero.IsLinked()) { 7182 __ Bind(&zero); 7183 __ LoadImmediate(out, 0); 7184 } 7185 7186 if (done.IsLinked()) { 7187 __ Bind(&done); 7188 } 7189 7190 if (slow_path != nullptr) { 7191 __ Bind(slow_path->GetExitLabel()); 7192 } 7193} 7194 7195void LocationsBuilderARM::VisitCheckCast(HCheckCast* instruction) { 7196 LocationSummary::CallKind call_kind = LocationSummary::kNoCall; 7197 bool throws_into_catch = instruction->CanThrowIntoCatchBlock(); 7198 7199 TypeCheckKind type_check_kind = instruction->GetTypeCheckKind(); 7200 switch (type_check_kind) { 7201 case TypeCheckKind::kExactCheck: 7202 case TypeCheckKind::kAbstractClassCheck: 7203 case TypeCheckKind::kClassHierarchyCheck: 7204 case TypeCheckKind::kArrayObjectCheck: 7205 call_kind = (throws_into_catch || kEmitCompilerReadBarrier) ? 7206 LocationSummary::kCallOnSlowPath : 7207 LocationSummary::kNoCall; // In fact, call on a fatal (non-returning) slow path. 7208 break; 7209 case TypeCheckKind::kArrayCheck: 7210 case TypeCheckKind::kUnresolvedCheck: 7211 case TypeCheckKind::kInterfaceCheck: 7212 call_kind = LocationSummary::kCallOnSlowPath; 7213 break; 7214 } 7215 7216 LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction, call_kind); 7217 locations->SetInAt(0, Location::RequiresRegister()); 7218 locations->SetInAt(1, Location::RequiresRegister()); 7219 locations->AddRegisterTemps(NumberOfCheckCastTemps(type_check_kind)); 7220} 7221 7222void InstructionCodeGeneratorARM::VisitCheckCast(HCheckCast* instruction) { 7223 TypeCheckKind type_check_kind = instruction->GetTypeCheckKind(); 7224 LocationSummary* locations = instruction->GetLocations(); 7225 Location obj_loc = locations->InAt(0); 7226 Register obj = obj_loc.AsRegister<Register>(); 7227 Register cls = locations->InAt(1).AsRegister<Register>(); 7228 Location temp_loc = locations->GetTemp(0); 7229 Register temp = temp_loc.AsRegister<Register>(); 7230 const size_t num_temps = NumberOfCheckCastTemps(type_check_kind); 7231 DCHECK_LE(num_temps, 3u); 7232 Location maybe_temp2_loc = (num_temps >= 2) ? locations->GetTemp(1) : Location::NoLocation(); 7233 Location maybe_temp3_loc = (num_temps >= 3) ? locations->GetTemp(2) : Location::NoLocation(); 7234 const uint32_t class_offset = mirror::Object::ClassOffset().Int32Value(); 7235 const uint32_t super_offset = mirror::Class::SuperClassOffset().Int32Value(); 7236 const uint32_t component_offset = mirror::Class::ComponentTypeOffset().Int32Value(); 7237 const uint32_t primitive_offset = mirror::Class::PrimitiveTypeOffset().Int32Value(); 7238 const uint32_t iftable_offset = mirror::Class::IfTableOffset().Uint32Value(); 7239 const uint32_t array_length_offset = mirror::Array::LengthOffset().Uint32Value(); 7240 const uint32_t object_array_data_offset = 7241 mirror::Array::DataOffset(kHeapReferenceSize).Uint32Value(); 7242 7243 // Always false for read barriers since we may need to go to the entrypoint for non-fatal cases 7244 // from false negatives. The false negatives may come from avoiding read barriers below. Avoiding 7245 // read barriers is done for performance and code size reasons. 7246 bool is_type_check_slow_path_fatal = false; 7247 if (!kEmitCompilerReadBarrier) { 7248 is_type_check_slow_path_fatal = 7249 (type_check_kind == TypeCheckKind::kExactCheck || 7250 type_check_kind == TypeCheckKind::kAbstractClassCheck || 7251 type_check_kind == TypeCheckKind::kClassHierarchyCheck || 7252 type_check_kind == TypeCheckKind::kArrayObjectCheck) && 7253 !instruction->CanThrowIntoCatchBlock(); 7254 } 7255 SlowPathCodeARM* type_check_slow_path = 7256 new (GetGraph()->GetArena()) TypeCheckSlowPathARM(instruction, 7257 is_type_check_slow_path_fatal); 7258 codegen_->AddSlowPath(type_check_slow_path); 7259 7260 Label done; 7261 // Avoid null check if we know obj is not null. 7262 if (instruction->MustDoNullCheck()) { 7263 __ CompareAndBranchIfZero(obj, &done); 7264 } 7265 7266 switch (type_check_kind) { 7267 case TypeCheckKind::kExactCheck: 7268 case TypeCheckKind::kArrayCheck: { 7269 // /* HeapReference<Class> */ temp = obj->klass_ 7270 GenerateReferenceLoadTwoRegisters(instruction, 7271 temp_loc, 7272 obj_loc, 7273 class_offset, 7274 maybe_temp2_loc, 7275 kWithoutReadBarrier); 7276 7277 __ cmp(temp, ShifterOperand(cls)); 7278 // Jump to slow path for throwing the exception or doing a 7279 // more involved array check. 7280 __ b(type_check_slow_path->GetEntryLabel(), NE); 7281 break; 7282 } 7283 7284 case TypeCheckKind::kAbstractClassCheck: { 7285 // /* HeapReference<Class> */ temp = obj->klass_ 7286 GenerateReferenceLoadTwoRegisters(instruction, 7287 temp_loc, 7288 obj_loc, 7289 class_offset, 7290 maybe_temp2_loc, 7291 kWithoutReadBarrier); 7292 7293 // If the class is abstract, we eagerly fetch the super class of the 7294 // object to avoid doing a comparison we know will fail. 7295 Label loop; 7296 __ Bind(&loop); 7297 // /* HeapReference<Class> */ temp = temp->super_class_ 7298 GenerateReferenceLoadOneRegister(instruction, 7299 temp_loc, 7300 super_offset, 7301 maybe_temp2_loc, 7302 kWithoutReadBarrier); 7303 7304 // If the class reference currently in `temp` is null, jump to the slow path to throw the 7305 // exception. 7306 __ CompareAndBranchIfZero(temp, type_check_slow_path->GetEntryLabel()); 7307 7308 // Otherwise, compare the classes. 7309 __ cmp(temp, ShifterOperand(cls)); 7310 __ b(&loop, NE); 7311 break; 7312 } 7313 7314 case TypeCheckKind::kClassHierarchyCheck: { 7315 // /* HeapReference<Class> */ temp = obj->klass_ 7316 GenerateReferenceLoadTwoRegisters(instruction, 7317 temp_loc, 7318 obj_loc, 7319 class_offset, 7320 maybe_temp2_loc, 7321 kWithoutReadBarrier); 7322 7323 // Walk over the class hierarchy to find a match. 7324 Label loop; 7325 __ Bind(&loop); 7326 __ cmp(temp, ShifterOperand(cls)); 7327 __ b(&done, EQ); 7328 7329 // /* HeapReference<Class> */ temp = temp->super_class_ 7330 GenerateReferenceLoadOneRegister(instruction, 7331 temp_loc, 7332 super_offset, 7333 maybe_temp2_loc, 7334 kWithoutReadBarrier); 7335 7336 // If the class reference currently in `temp` is null, jump to the slow path to throw the 7337 // exception. 7338 __ CompareAndBranchIfZero(temp, type_check_slow_path->GetEntryLabel()); 7339 // Otherwise, jump to the beginning of the loop. 7340 __ b(&loop); 7341 break; 7342 } 7343 7344 case TypeCheckKind::kArrayObjectCheck: { 7345 // /* HeapReference<Class> */ temp = obj->klass_ 7346 GenerateReferenceLoadTwoRegisters(instruction, 7347 temp_loc, 7348 obj_loc, 7349 class_offset, 7350 maybe_temp2_loc, 7351 kWithoutReadBarrier); 7352 7353 // Do an exact check. 7354 __ cmp(temp, ShifterOperand(cls)); 7355 __ b(&done, EQ); 7356 7357 // Otherwise, we need to check that the object's class is a non-primitive array. 7358 // /* HeapReference<Class> */ temp = temp->component_type_ 7359 GenerateReferenceLoadOneRegister(instruction, 7360 temp_loc, 7361 component_offset, 7362 maybe_temp2_loc, 7363 kWithoutReadBarrier); 7364 // If the component type is null, jump to the slow path to throw the exception. 7365 __ CompareAndBranchIfZero(temp, type_check_slow_path->GetEntryLabel()); 7366 // Otherwise,the object is indeed an array, jump to label `check_non_primitive_component_type` 7367 // to further check that this component type is not a primitive type. 7368 __ LoadFromOffset(kLoadUnsignedHalfword, temp, temp, primitive_offset); 7369 static_assert(Primitive::kPrimNot == 0, "Expected 0 for art::Primitive::kPrimNot"); 7370 __ CompareAndBranchIfNonZero(temp, type_check_slow_path->GetEntryLabel()); 7371 break; 7372 } 7373 7374 case TypeCheckKind::kUnresolvedCheck: 7375 // We always go into the type check slow path for the unresolved check case. 7376 // We cannot directly call the CheckCast runtime entry point 7377 // without resorting to a type checking slow path here (i.e. by 7378 // calling InvokeRuntime directly), as it would require to 7379 // assign fixed registers for the inputs of this HInstanceOf 7380 // instruction (following the runtime calling convention), which 7381 // might be cluttered by the potential first read barrier 7382 // emission at the beginning of this method. 7383 7384 __ b(type_check_slow_path->GetEntryLabel()); 7385 break; 7386 7387 case TypeCheckKind::kInterfaceCheck: { 7388 // Avoid read barriers to improve performance of the fast path. We can not get false 7389 // positives by doing this. 7390 // /* HeapReference<Class> */ temp = obj->klass_ 7391 GenerateReferenceLoadTwoRegisters(instruction, 7392 temp_loc, 7393 obj_loc, 7394 class_offset, 7395 maybe_temp2_loc, 7396 kWithoutReadBarrier); 7397 7398 // /* HeapReference<Class> */ temp = temp->iftable_ 7399 GenerateReferenceLoadTwoRegisters(instruction, 7400 temp_loc, 7401 temp_loc, 7402 iftable_offset, 7403 maybe_temp2_loc, 7404 kWithoutReadBarrier); 7405 // Iftable is never null. 7406 __ ldr(maybe_temp2_loc.AsRegister<Register>(), Address(temp, array_length_offset)); 7407 // Loop through the iftable and check if any class matches. 7408 Label start_loop; 7409 __ Bind(&start_loop); 7410 __ CompareAndBranchIfZero(maybe_temp2_loc.AsRegister<Register>(), 7411 type_check_slow_path->GetEntryLabel()); 7412 __ ldr(maybe_temp3_loc.AsRegister<Register>(), Address(temp, object_array_data_offset)); 7413 __ MaybeUnpoisonHeapReference(maybe_temp3_loc.AsRegister<Register>()); 7414 // Go to next interface. 7415 __ add(temp, temp, ShifterOperand(2 * kHeapReferenceSize)); 7416 __ sub(maybe_temp2_loc.AsRegister<Register>(), 7417 maybe_temp2_loc.AsRegister<Register>(), 7418 ShifterOperand(2)); 7419 // Compare the classes and continue the loop if they do not match. 7420 __ cmp(cls, ShifterOperand(maybe_temp3_loc.AsRegister<Register>())); 7421 __ b(&start_loop, NE); 7422 break; 7423 } 7424 } 7425 __ Bind(&done); 7426 7427 __ Bind(type_check_slow_path->GetExitLabel()); 7428} 7429 7430void LocationsBuilderARM::VisitMonitorOperation(HMonitorOperation* instruction) { 7431 LocationSummary* locations = 7432 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCallOnMainOnly); 7433 InvokeRuntimeCallingConvention calling_convention; 7434 locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(0))); 7435} 7436 7437void InstructionCodeGeneratorARM::VisitMonitorOperation(HMonitorOperation* instruction) { 7438 codegen_->InvokeRuntime(instruction->IsEnter() ? kQuickLockObject : kQuickUnlockObject, 7439 instruction, 7440 instruction->GetDexPc()); 7441 if (instruction->IsEnter()) { 7442 CheckEntrypointTypes<kQuickLockObject, void, mirror::Object*>(); 7443 } else { 7444 CheckEntrypointTypes<kQuickUnlockObject, void, mirror::Object*>(); 7445 } 7446} 7447 7448void LocationsBuilderARM::VisitAnd(HAnd* instruction) { HandleBitwiseOperation(instruction, AND); } 7449void LocationsBuilderARM::VisitOr(HOr* instruction) { HandleBitwiseOperation(instruction, ORR); } 7450void LocationsBuilderARM::VisitXor(HXor* instruction) { HandleBitwiseOperation(instruction, EOR); } 7451 7452void LocationsBuilderARM::HandleBitwiseOperation(HBinaryOperation* instruction, Opcode opcode) { 7453 LocationSummary* locations = 7454 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall); 7455 DCHECK(instruction->GetResultType() == Primitive::kPrimInt 7456 || instruction->GetResultType() == Primitive::kPrimLong); 7457 // Note: GVN reorders commutative operations to have the constant on the right hand side. 7458 locations->SetInAt(0, Location::RequiresRegister()); 7459 locations->SetInAt(1, ArmEncodableConstantOrRegister(instruction->InputAt(1), opcode)); 7460 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 7461} 7462 7463void InstructionCodeGeneratorARM::VisitAnd(HAnd* instruction) { 7464 HandleBitwiseOperation(instruction); 7465} 7466 7467void InstructionCodeGeneratorARM::VisitOr(HOr* instruction) { 7468 HandleBitwiseOperation(instruction); 7469} 7470 7471void InstructionCodeGeneratorARM::VisitXor(HXor* instruction) { 7472 HandleBitwiseOperation(instruction); 7473} 7474 7475 7476void LocationsBuilderARM::VisitBitwiseNegatedRight(HBitwiseNegatedRight* instruction) { 7477 LocationSummary* locations = 7478 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall); 7479 DCHECK(instruction->GetResultType() == Primitive::kPrimInt 7480 || instruction->GetResultType() == Primitive::kPrimLong); 7481 7482 locations->SetInAt(0, Location::RequiresRegister()); 7483 locations->SetInAt(1, Location::RequiresRegister()); 7484 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 7485} 7486 7487void InstructionCodeGeneratorARM::VisitBitwiseNegatedRight(HBitwiseNegatedRight* instruction) { 7488 LocationSummary* locations = instruction->GetLocations(); 7489 Location first = locations->InAt(0); 7490 Location second = locations->InAt(1); 7491 Location out = locations->Out(); 7492 7493 if (instruction->GetResultType() == Primitive::kPrimInt) { 7494 Register first_reg = first.AsRegister<Register>(); 7495 ShifterOperand second_reg(second.AsRegister<Register>()); 7496 Register out_reg = out.AsRegister<Register>(); 7497 7498 switch (instruction->GetOpKind()) { 7499 case HInstruction::kAnd: 7500 __ bic(out_reg, first_reg, second_reg); 7501 break; 7502 case HInstruction::kOr: 7503 __ orn(out_reg, first_reg, second_reg); 7504 break; 7505 // There is no EON on arm. 7506 case HInstruction::kXor: 7507 default: 7508 LOG(FATAL) << "Unexpected instruction " << instruction->DebugName(); 7509 UNREACHABLE(); 7510 } 7511 return; 7512 7513 } else { 7514 DCHECK_EQ(instruction->GetResultType(), Primitive::kPrimLong); 7515 Register first_low = first.AsRegisterPairLow<Register>(); 7516 Register first_high = first.AsRegisterPairHigh<Register>(); 7517 ShifterOperand second_low(second.AsRegisterPairLow<Register>()); 7518 ShifterOperand second_high(second.AsRegisterPairHigh<Register>()); 7519 Register out_low = out.AsRegisterPairLow<Register>(); 7520 Register out_high = out.AsRegisterPairHigh<Register>(); 7521 7522 switch (instruction->GetOpKind()) { 7523 case HInstruction::kAnd: 7524 __ bic(out_low, first_low, second_low); 7525 __ bic(out_high, first_high, second_high); 7526 break; 7527 case HInstruction::kOr: 7528 __ orn(out_low, first_low, second_low); 7529 __ orn(out_high, first_high, second_high); 7530 break; 7531 // There is no EON on arm. 7532 case HInstruction::kXor: 7533 default: 7534 LOG(FATAL) << "Unexpected instruction " << instruction->DebugName(); 7535 UNREACHABLE(); 7536 } 7537 } 7538} 7539 7540void LocationsBuilderARM::VisitDataProcWithShifterOp( 7541 HDataProcWithShifterOp* instruction) { 7542 DCHECK(instruction->GetType() == Primitive::kPrimInt || 7543 instruction->GetType() == Primitive::kPrimLong); 7544 LocationSummary* locations = 7545 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall); 7546 const bool overlap = instruction->GetType() == Primitive::kPrimLong && 7547 HDataProcWithShifterOp::IsExtensionOp(instruction->GetOpKind()); 7548 7549 locations->SetInAt(0, Location::RequiresRegister()); 7550 locations->SetInAt(1, Location::RequiresRegister()); 7551 locations->SetOut(Location::RequiresRegister(), 7552 overlap ? Location::kOutputOverlap : Location::kNoOutputOverlap); 7553} 7554 7555void InstructionCodeGeneratorARM::VisitDataProcWithShifterOp( 7556 HDataProcWithShifterOp* instruction) { 7557 const LocationSummary* const locations = instruction->GetLocations(); 7558 const HInstruction::InstructionKind kind = instruction->GetInstrKind(); 7559 const HDataProcWithShifterOp::OpKind op_kind = instruction->GetOpKind(); 7560 const Location left = locations->InAt(0); 7561 const Location right = locations->InAt(1); 7562 const Location out = locations->Out(); 7563 7564 if (instruction->GetType() == Primitive::kPrimInt) { 7565 DCHECK(!HDataProcWithShifterOp::IsExtensionOp(op_kind)); 7566 7567 const Register second = instruction->InputAt(1)->GetType() == Primitive::kPrimLong 7568 ? right.AsRegisterPairLow<Register>() 7569 : right.AsRegister<Register>(); 7570 7571 GenerateDataProcInstruction(kind, 7572 out.AsRegister<Register>(), 7573 left.AsRegister<Register>(), 7574 ShifterOperand(second, 7575 ShiftFromOpKind(op_kind), 7576 instruction->GetShiftAmount()), 7577 codegen_); 7578 } else { 7579 DCHECK_EQ(instruction->GetType(), Primitive::kPrimLong); 7580 7581 if (HDataProcWithShifterOp::IsExtensionOp(op_kind)) { 7582 const Register second = right.AsRegister<Register>(); 7583 7584 DCHECK_NE(out.AsRegisterPairLow<Register>(), second); 7585 GenerateDataProc(kind, 7586 out, 7587 left, 7588 ShifterOperand(second), 7589 ShifterOperand(second, ASR, 31), 7590 codegen_); 7591 } else { 7592 GenerateLongDataProc(instruction, codegen_); 7593 } 7594 } 7595} 7596 7597void InstructionCodeGeneratorARM::GenerateAndConst(Register out, Register first, uint32_t value) { 7598 // Optimize special cases for individual halfs of `and-long` (`and` is simplified earlier). 7599 if (value == 0xffffffffu) { 7600 if (out != first) { 7601 __ mov(out, ShifterOperand(first)); 7602 } 7603 return; 7604 } 7605 if (value == 0u) { 7606 __ mov(out, ShifterOperand(0)); 7607 return; 7608 } 7609 ShifterOperand so; 7610 if (__ ShifterOperandCanHold(kNoRegister, kNoRegister, AND, value, &so)) { 7611 __ and_(out, first, so); 7612 } else { 7613 DCHECK(__ ShifterOperandCanHold(kNoRegister, kNoRegister, BIC, ~value, &so)); 7614 __ bic(out, first, ShifterOperand(~value)); 7615 } 7616} 7617 7618void InstructionCodeGeneratorARM::GenerateOrrConst(Register out, Register first, uint32_t value) { 7619 // Optimize special cases for individual halfs of `or-long` (`or` is simplified earlier). 7620 if (value == 0u) { 7621 if (out != first) { 7622 __ mov(out, ShifterOperand(first)); 7623 } 7624 return; 7625 } 7626 if (value == 0xffffffffu) { 7627 __ mvn(out, ShifterOperand(0)); 7628 return; 7629 } 7630 ShifterOperand so; 7631 if (__ ShifterOperandCanHold(kNoRegister, kNoRegister, ORR, value, &so)) { 7632 __ orr(out, first, so); 7633 } else { 7634 DCHECK(__ ShifterOperandCanHold(kNoRegister, kNoRegister, ORN, ~value, &so)); 7635 __ orn(out, first, ShifterOperand(~value)); 7636 } 7637} 7638 7639void InstructionCodeGeneratorARM::GenerateEorConst(Register out, Register first, uint32_t value) { 7640 // Optimize special case for individual halfs of `xor-long` (`xor` is simplified earlier). 7641 if (value == 0u) { 7642 if (out != first) { 7643 __ mov(out, ShifterOperand(first)); 7644 } 7645 return; 7646 } 7647 __ eor(out, first, ShifterOperand(value)); 7648} 7649 7650void InstructionCodeGeneratorARM::GenerateAddLongConst(Location out, 7651 Location first, 7652 uint64_t value) { 7653 Register out_low = out.AsRegisterPairLow<Register>(); 7654 Register out_high = out.AsRegisterPairHigh<Register>(); 7655 Register first_low = first.AsRegisterPairLow<Register>(); 7656 Register first_high = first.AsRegisterPairHigh<Register>(); 7657 uint32_t value_low = Low32Bits(value); 7658 uint32_t value_high = High32Bits(value); 7659 if (value_low == 0u) { 7660 if (out_low != first_low) { 7661 __ mov(out_low, ShifterOperand(first_low)); 7662 } 7663 __ AddConstant(out_high, first_high, value_high); 7664 return; 7665 } 7666 __ AddConstantSetFlags(out_low, first_low, value_low); 7667 ShifterOperand so; 7668 if (__ ShifterOperandCanHold(out_high, first_high, ADC, value_high, kCcDontCare, &so)) { 7669 __ adc(out_high, first_high, so); 7670 } else if (__ ShifterOperandCanHold(out_low, first_low, SBC, ~value_high, kCcDontCare, &so)) { 7671 __ sbc(out_high, first_high, so); 7672 } else { 7673 LOG(FATAL) << "Unexpected constant " << value_high; 7674 UNREACHABLE(); 7675 } 7676} 7677 7678void InstructionCodeGeneratorARM::HandleBitwiseOperation(HBinaryOperation* instruction) { 7679 LocationSummary* locations = instruction->GetLocations(); 7680 Location first = locations->InAt(0); 7681 Location second = locations->InAt(1); 7682 Location out = locations->Out(); 7683 7684 if (second.IsConstant()) { 7685 uint64_t value = static_cast<uint64_t>(Int64FromConstant(second.GetConstant())); 7686 uint32_t value_low = Low32Bits(value); 7687 if (instruction->GetResultType() == Primitive::kPrimInt) { 7688 Register first_reg = first.AsRegister<Register>(); 7689 Register out_reg = out.AsRegister<Register>(); 7690 if (instruction->IsAnd()) { 7691 GenerateAndConst(out_reg, first_reg, value_low); 7692 } else if (instruction->IsOr()) { 7693 GenerateOrrConst(out_reg, first_reg, value_low); 7694 } else { 7695 DCHECK(instruction->IsXor()); 7696 GenerateEorConst(out_reg, first_reg, value_low); 7697 } 7698 } else { 7699 DCHECK_EQ(instruction->GetResultType(), Primitive::kPrimLong); 7700 uint32_t value_high = High32Bits(value); 7701 Register first_low = first.AsRegisterPairLow<Register>(); 7702 Register first_high = first.AsRegisterPairHigh<Register>(); 7703 Register out_low = out.AsRegisterPairLow<Register>(); 7704 Register out_high = out.AsRegisterPairHigh<Register>(); 7705 if (instruction->IsAnd()) { 7706 GenerateAndConst(out_low, first_low, value_low); 7707 GenerateAndConst(out_high, first_high, value_high); 7708 } else if (instruction->IsOr()) { 7709 GenerateOrrConst(out_low, first_low, value_low); 7710 GenerateOrrConst(out_high, first_high, value_high); 7711 } else { 7712 DCHECK(instruction->IsXor()); 7713 GenerateEorConst(out_low, first_low, value_low); 7714 GenerateEorConst(out_high, first_high, value_high); 7715 } 7716 } 7717 return; 7718 } 7719 7720 if (instruction->GetResultType() == Primitive::kPrimInt) { 7721 Register first_reg = first.AsRegister<Register>(); 7722 ShifterOperand second_reg(second.AsRegister<Register>()); 7723 Register out_reg = out.AsRegister<Register>(); 7724 if (instruction->IsAnd()) { 7725 __ and_(out_reg, first_reg, second_reg); 7726 } else if (instruction->IsOr()) { 7727 __ orr(out_reg, first_reg, second_reg); 7728 } else { 7729 DCHECK(instruction->IsXor()); 7730 __ eor(out_reg, first_reg, second_reg); 7731 } 7732 } else { 7733 DCHECK_EQ(instruction->GetResultType(), Primitive::kPrimLong); 7734 Register first_low = first.AsRegisterPairLow<Register>(); 7735 Register first_high = first.AsRegisterPairHigh<Register>(); 7736 ShifterOperand second_low(second.AsRegisterPairLow<Register>()); 7737 ShifterOperand second_high(second.AsRegisterPairHigh<Register>()); 7738 Register out_low = out.AsRegisterPairLow<Register>(); 7739 Register out_high = out.AsRegisterPairHigh<Register>(); 7740 if (instruction->IsAnd()) { 7741 __ and_(out_low, first_low, second_low); 7742 __ and_(out_high, first_high, second_high); 7743 } else if (instruction->IsOr()) { 7744 __ orr(out_low, first_low, second_low); 7745 __ orr(out_high, first_high, second_high); 7746 } else { 7747 DCHECK(instruction->IsXor()); 7748 __ eor(out_low, first_low, second_low); 7749 __ eor(out_high, first_high, second_high); 7750 } 7751 } 7752} 7753 7754void InstructionCodeGeneratorARM::GenerateReferenceLoadOneRegister( 7755 HInstruction* instruction, 7756 Location out, 7757 uint32_t offset, 7758 Location maybe_temp, 7759 ReadBarrierOption read_barrier_option) { 7760 Register out_reg = out.AsRegister<Register>(); 7761 if (read_barrier_option == kWithReadBarrier) { 7762 CHECK(kEmitCompilerReadBarrier); 7763 DCHECK(maybe_temp.IsRegister()) << maybe_temp; 7764 if (kUseBakerReadBarrier) { 7765 // Load with fast path based Baker's read barrier. 7766 // /* HeapReference<Object> */ out = *(out + offset) 7767 codegen_->GenerateFieldLoadWithBakerReadBarrier( 7768 instruction, out, out_reg, offset, maybe_temp, /* needs_null_check */ false); 7769 } else { 7770 // Load with slow path based read barrier. 7771 // Save the value of `out` into `maybe_temp` before overwriting it 7772 // in the following move operation, as we will need it for the 7773 // read barrier below. 7774 __ Mov(maybe_temp.AsRegister<Register>(), out_reg); 7775 // /* HeapReference<Object> */ out = *(out + offset) 7776 __ LoadFromOffset(kLoadWord, out_reg, out_reg, offset); 7777 codegen_->GenerateReadBarrierSlow(instruction, out, out, maybe_temp, offset); 7778 } 7779 } else { 7780 // Plain load with no read barrier. 7781 // /* HeapReference<Object> */ out = *(out + offset) 7782 __ LoadFromOffset(kLoadWord, out_reg, out_reg, offset); 7783 __ MaybeUnpoisonHeapReference(out_reg); 7784 } 7785} 7786 7787void InstructionCodeGeneratorARM::GenerateReferenceLoadTwoRegisters( 7788 HInstruction* instruction, 7789 Location out, 7790 Location obj, 7791 uint32_t offset, 7792 Location maybe_temp, 7793 ReadBarrierOption read_barrier_option) { 7794 Register out_reg = out.AsRegister<Register>(); 7795 Register obj_reg = obj.AsRegister<Register>(); 7796 if (read_barrier_option == kWithReadBarrier) { 7797 CHECK(kEmitCompilerReadBarrier); 7798 if (kUseBakerReadBarrier) { 7799 DCHECK(maybe_temp.IsRegister()) << maybe_temp; 7800 // Load with fast path based Baker's read barrier. 7801 // /* HeapReference<Object> */ out = *(obj + offset) 7802 codegen_->GenerateFieldLoadWithBakerReadBarrier( 7803 instruction, out, obj_reg, offset, maybe_temp, /* needs_null_check */ false); 7804 } else { 7805 // Load with slow path based read barrier. 7806 // /* HeapReference<Object> */ out = *(obj + offset) 7807 __ LoadFromOffset(kLoadWord, out_reg, obj_reg, offset); 7808 codegen_->GenerateReadBarrierSlow(instruction, out, out, obj, offset); 7809 } 7810 } else { 7811 // Plain load with no read barrier. 7812 // /* HeapReference<Object> */ out = *(obj + offset) 7813 __ LoadFromOffset(kLoadWord, out_reg, obj_reg, offset); 7814 __ MaybeUnpoisonHeapReference(out_reg); 7815 } 7816} 7817 7818void InstructionCodeGeneratorARM::GenerateGcRootFieldLoad(HInstruction* instruction, 7819 Location root, 7820 Register obj, 7821 uint32_t offset, 7822 ReadBarrierOption read_barrier_option) { 7823 Register root_reg = root.AsRegister<Register>(); 7824 if (read_barrier_option == kWithReadBarrier) { 7825 DCHECK(kEmitCompilerReadBarrier); 7826 if (kUseBakerReadBarrier) { 7827 // Fast path implementation of art::ReadBarrier::BarrierForRoot when 7828 // Baker's read barrier are used. 7829 // 7830 // Note that we do not actually check the value of 7831 // `GetIsGcMarking()` to decide whether to mark the loaded GC 7832 // root or not. Instead, we load into `temp` the read barrier 7833 // mark entry point corresponding to register `root`. If `temp` 7834 // is null, it means that `GetIsGcMarking()` is false, and vice 7835 // versa. 7836 // 7837 // temp = Thread::Current()->pReadBarrierMarkReg ## root.reg() 7838 // GcRoot<mirror::Object> root = *(obj+offset); // Original reference load. 7839 // if (temp != nullptr) { // <=> Thread::Current()->GetIsGcMarking() 7840 // // Slow path. 7841 // root = temp(root); // root = ReadBarrier::Mark(root); // Runtime entry point call. 7842 // } 7843 7844 // Slow path marking the GC root `root`. The entrypoint will already be loaded in `temp`. 7845 Location temp = Location::RegisterLocation(LR); 7846 SlowPathCodeARM* slow_path = new (GetGraph()->GetArena()) ReadBarrierMarkSlowPathARM( 7847 instruction, root, /* entrypoint */ temp); 7848 codegen_->AddSlowPath(slow_path); 7849 7850 // temp = Thread::Current()->pReadBarrierMarkReg ## root.reg() 7851 const int32_t entry_point_offset = 7852 CodeGenerator::GetReadBarrierMarkEntryPointsOffset<kArmPointerSize>(root.reg()); 7853 // Loading the entrypoint does not require a load acquire since it is only changed when 7854 // threads are suspended or running a checkpoint. 7855 __ LoadFromOffset(kLoadWord, temp.AsRegister<Register>(), TR, entry_point_offset); 7856 7857 // /* GcRoot<mirror::Object> */ root = *(obj + offset) 7858 __ LoadFromOffset(kLoadWord, root_reg, obj, offset); 7859 static_assert( 7860 sizeof(mirror::CompressedReference<mirror::Object>) == sizeof(GcRoot<mirror::Object>), 7861 "art::mirror::CompressedReference<mirror::Object> and art::GcRoot<mirror::Object> " 7862 "have different sizes."); 7863 static_assert(sizeof(mirror::CompressedReference<mirror::Object>) == sizeof(int32_t), 7864 "art::mirror::CompressedReference<mirror::Object> and int32_t " 7865 "have different sizes."); 7866 7867 // The entrypoint is null when the GC is not marking, this prevents one load compared to 7868 // checking GetIsGcMarking. 7869 __ CompareAndBranchIfNonZero(temp.AsRegister<Register>(), slow_path->GetEntryLabel()); 7870 __ Bind(slow_path->GetExitLabel()); 7871 } else { 7872 // GC root loaded through a slow path for read barriers other 7873 // than Baker's. 7874 // /* GcRoot<mirror::Object>* */ root = obj + offset 7875 __ AddConstant(root_reg, obj, offset); 7876 // /* mirror::Object* */ root = root->Read() 7877 codegen_->GenerateReadBarrierForRootSlow(instruction, root, root); 7878 } 7879 } else { 7880 // Plain GC root load with no read barrier. 7881 // /* GcRoot<mirror::Object> */ root = *(obj + offset) 7882 __ LoadFromOffset(kLoadWord, root_reg, obj, offset); 7883 // Note that GC roots are not affected by heap poisoning, thus we 7884 // do not have to unpoison `root_reg` here. 7885 } 7886} 7887 7888void CodeGeneratorARM::GenerateFieldLoadWithBakerReadBarrier(HInstruction* instruction, 7889 Location ref, 7890 Register obj, 7891 uint32_t offset, 7892 Location temp, 7893 bool needs_null_check) { 7894 DCHECK(kEmitCompilerReadBarrier); 7895 DCHECK(kUseBakerReadBarrier); 7896 7897 // /* HeapReference<Object> */ ref = *(obj + offset) 7898 Location no_index = Location::NoLocation(); 7899 ScaleFactor no_scale_factor = TIMES_1; 7900 GenerateReferenceLoadWithBakerReadBarrier( 7901 instruction, ref, obj, offset, no_index, no_scale_factor, temp, needs_null_check); 7902} 7903 7904void CodeGeneratorARM::GenerateArrayLoadWithBakerReadBarrier(HInstruction* instruction, 7905 Location ref, 7906 Register obj, 7907 uint32_t data_offset, 7908 Location index, 7909 Location temp, 7910 bool needs_null_check) { 7911 DCHECK(kEmitCompilerReadBarrier); 7912 DCHECK(kUseBakerReadBarrier); 7913 7914 static_assert( 7915 sizeof(mirror::HeapReference<mirror::Object>) == sizeof(int32_t), 7916 "art::mirror::HeapReference<art::mirror::Object> and int32_t have different sizes."); 7917 // /* HeapReference<Object> */ ref = 7918 // *(obj + data_offset + index * sizeof(HeapReference<Object>)) 7919 ScaleFactor scale_factor = TIMES_4; 7920 GenerateReferenceLoadWithBakerReadBarrier( 7921 instruction, ref, obj, data_offset, index, scale_factor, temp, needs_null_check); 7922} 7923 7924void CodeGeneratorARM::GenerateReferenceLoadWithBakerReadBarrier(HInstruction* instruction, 7925 Location ref, 7926 Register obj, 7927 uint32_t offset, 7928 Location index, 7929 ScaleFactor scale_factor, 7930 Location temp, 7931 bool needs_null_check, 7932 bool always_update_field, 7933 Register* temp2) { 7934 DCHECK(kEmitCompilerReadBarrier); 7935 DCHECK(kUseBakerReadBarrier); 7936 7937 // Query `art::Thread::Current()->GetIsGcMarking()` to decide 7938 // whether we need to enter the slow path to mark the reference. 7939 // Then, in the slow path, check the gray bit in the lock word of 7940 // the reference's holder (`obj`) to decide whether to mark `ref` or 7941 // not. 7942 // 7943 // Note that we do not actually check the value of `GetIsGcMarking()`; 7944 // instead, we load into `temp3` the read barrier mark entry point 7945 // corresponding to register `ref`. If `temp3` is null, it means 7946 // that `GetIsGcMarking()` is false, and vice versa. 7947 // 7948 // temp3 = Thread::Current()->pReadBarrierMarkReg ## root.reg() 7949 // if (temp3 != nullptr) { // <=> Thread::Current()->GetIsGcMarking() 7950 // // Slow path. 7951 // uint32_t rb_state = Lockword(obj->monitor_).ReadBarrierState(); 7952 // lfence; // Load fence or artificial data dependency to prevent load-load reordering 7953 // HeapReference<mirror::Object> ref = *src; // Original reference load. 7954 // bool is_gray = (rb_state == ReadBarrier::GrayState()); 7955 // if (is_gray) { 7956 // ref = temp3(ref); // ref = ReadBarrier::Mark(ref); // Runtime entry point call. 7957 // } 7958 // } else { 7959 // HeapReference<mirror::Object> ref = *src; // Original reference load. 7960 // } 7961 7962 Register temp_reg = temp.AsRegister<Register>(); 7963 7964 // Slow path marking the object `ref` when the GC is marking. The 7965 // entrypoint will already be loaded in `temp3`. 7966 Location temp3 = Location::RegisterLocation(LR); 7967 SlowPathCodeARM* slow_path; 7968 if (always_update_field) { 7969 DCHECK(temp2 != nullptr); 7970 // LoadReferenceWithBakerReadBarrierAndUpdateFieldSlowPathARM only 7971 // supports address of the form `obj + field_offset`, where `obj` 7972 // is a register and `field_offset` is a register pair (of which 7973 // only the lower half is used). Thus `offset` and `scale_factor` 7974 // above are expected to be null in this code path. 7975 DCHECK_EQ(offset, 0u); 7976 DCHECK_EQ(scale_factor, ScaleFactor::TIMES_1); 7977 Location field_offset = index; 7978 slow_path = 7979 new (GetGraph()->GetArena()) LoadReferenceWithBakerReadBarrierAndUpdateFieldSlowPathARM( 7980 instruction, 7981 ref, 7982 obj, 7983 offset, 7984 /* index */ field_offset, 7985 scale_factor, 7986 needs_null_check, 7987 temp_reg, 7988 *temp2, 7989 /* entrypoint */ temp3); 7990 } else { 7991 slow_path = new (GetGraph()->GetArena()) LoadReferenceWithBakerReadBarrierSlowPathARM( 7992 instruction, 7993 ref, 7994 obj, 7995 offset, 7996 index, 7997 scale_factor, 7998 needs_null_check, 7999 temp_reg, 8000 /* entrypoint */ temp3); 8001 } 8002 AddSlowPath(slow_path); 8003 8004 // temp3 = Thread::Current()->pReadBarrierMarkReg ## ref.reg() 8005 const int32_t entry_point_offset = 8006 CodeGenerator::GetReadBarrierMarkEntryPointsOffset<kArmPointerSize>(ref.reg()); 8007 // Loading the entrypoint does not require a load acquire since it is only changed when 8008 // threads are suspended or running a checkpoint. 8009 __ LoadFromOffset(kLoadWord, temp3.AsRegister<Register>(), TR, entry_point_offset); 8010 // The entrypoint is null when the GC is not marking, this prevents one load compared to 8011 // checking GetIsGcMarking. 8012 __ CompareAndBranchIfNonZero(temp3.AsRegister<Register>(), slow_path->GetEntryLabel()); 8013 // Fast path: just load the reference. 8014 GenerateRawReferenceLoad(instruction, ref, obj, offset, index, scale_factor, needs_null_check); 8015 __ Bind(slow_path->GetExitLabel()); 8016} 8017 8018void CodeGeneratorARM::GenerateRawReferenceLoad(HInstruction* instruction, 8019 Location ref, 8020 Register obj, 8021 uint32_t offset, 8022 Location index, 8023 ScaleFactor scale_factor, 8024 bool needs_null_check) { 8025 Register ref_reg = ref.AsRegister<Register>(); 8026 8027 if (index.IsValid()) { 8028 // Load types involving an "index": ArrayGet, 8029 // UnsafeGetObject/UnsafeGetObjectVolatile and UnsafeCASObject 8030 // intrinsics. 8031 // /* HeapReference<mirror::Object> */ ref = *(obj + offset + (index << scale_factor)) 8032 if (index.IsConstant()) { 8033 size_t computed_offset = 8034 (index.GetConstant()->AsIntConstant()->GetValue() << scale_factor) + offset; 8035 __ LoadFromOffset(kLoadWord, ref_reg, obj, computed_offset); 8036 } else { 8037 // Handle the special case of the 8038 // UnsafeGetObject/UnsafeGetObjectVolatile and UnsafeCASObject 8039 // intrinsics, which use a register pair as index ("long 8040 // offset"), of which only the low part contains data. 8041 Register index_reg = index.IsRegisterPair() 8042 ? index.AsRegisterPairLow<Register>() 8043 : index.AsRegister<Register>(); 8044 __ add(IP, obj, ShifterOperand(index_reg, LSL, scale_factor)); 8045 __ LoadFromOffset(kLoadWord, ref_reg, IP, offset); 8046 } 8047 } else { 8048 // /* HeapReference<mirror::Object> */ ref = *(obj + offset) 8049 __ LoadFromOffset(kLoadWord, ref_reg, obj, offset); 8050 } 8051 8052 if (needs_null_check) { 8053 MaybeRecordImplicitNullCheck(instruction); 8054 } 8055 8056 // Object* ref = ref_addr->AsMirrorPtr() 8057 __ MaybeUnpoisonHeapReference(ref_reg); 8058} 8059 8060void CodeGeneratorARM::GenerateReadBarrierSlow(HInstruction* instruction, 8061 Location out, 8062 Location ref, 8063 Location obj, 8064 uint32_t offset, 8065 Location index) { 8066 DCHECK(kEmitCompilerReadBarrier); 8067 8068 // Insert a slow path based read barrier *after* the reference load. 8069 // 8070 // If heap poisoning is enabled, the unpoisoning of the loaded 8071 // reference will be carried out by the runtime within the slow 8072 // path. 8073 // 8074 // Note that `ref` currently does not get unpoisoned (when heap 8075 // poisoning is enabled), which is alright as the `ref` argument is 8076 // not used by the artReadBarrierSlow entry point. 8077 // 8078 // TODO: Unpoison `ref` when it is used by artReadBarrierSlow. 8079 SlowPathCodeARM* slow_path = new (GetGraph()->GetArena()) 8080 ReadBarrierForHeapReferenceSlowPathARM(instruction, out, ref, obj, offset, index); 8081 AddSlowPath(slow_path); 8082 8083 __ b(slow_path->GetEntryLabel()); 8084 __ Bind(slow_path->GetExitLabel()); 8085} 8086 8087void CodeGeneratorARM::MaybeGenerateReadBarrierSlow(HInstruction* instruction, 8088 Location out, 8089 Location ref, 8090 Location obj, 8091 uint32_t offset, 8092 Location index) { 8093 if (kEmitCompilerReadBarrier) { 8094 // Baker's read barriers shall be handled by the fast path 8095 // (CodeGeneratorARM::GenerateReferenceLoadWithBakerReadBarrier). 8096 DCHECK(!kUseBakerReadBarrier); 8097 // If heap poisoning is enabled, unpoisoning will be taken care of 8098 // by the runtime within the slow path. 8099 GenerateReadBarrierSlow(instruction, out, ref, obj, offset, index); 8100 } else if (kPoisonHeapReferences) { 8101 __ UnpoisonHeapReference(out.AsRegister<Register>()); 8102 } 8103} 8104 8105void CodeGeneratorARM::GenerateReadBarrierForRootSlow(HInstruction* instruction, 8106 Location out, 8107 Location root) { 8108 DCHECK(kEmitCompilerReadBarrier); 8109 8110 // Insert a slow path based read barrier *after* the GC root load. 8111 // 8112 // Note that GC roots are not affected by heap poisoning, so we do 8113 // not need to do anything special for this here. 8114 SlowPathCodeARM* slow_path = 8115 new (GetGraph()->GetArena()) ReadBarrierForRootSlowPathARM(instruction, out, root); 8116 AddSlowPath(slow_path); 8117 8118 __ b(slow_path->GetEntryLabel()); 8119 __ Bind(slow_path->GetExitLabel()); 8120} 8121 8122HInvokeStaticOrDirect::DispatchInfo CodeGeneratorARM::GetSupportedInvokeStaticOrDirectDispatch( 8123 const HInvokeStaticOrDirect::DispatchInfo& desired_dispatch_info, 8124 HInvokeStaticOrDirect* invoke ATTRIBUTE_UNUSED) { 8125 return desired_dispatch_info; 8126} 8127 8128Register CodeGeneratorARM::GetInvokeStaticOrDirectExtraParameter(HInvokeStaticOrDirect* invoke, 8129 Register temp) { 8130 DCHECK_EQ(invoke->InputCount(), invoke->GetNumberOfArguments() + 1u); 8131 Location location = invoke->GetLocations()->InAt(invoke->GetSpecialInputIndex()); 8132 if (!invoke->GetLocations()->Intrinsified()) { 8133 return location.AsRegister<Register>(); 8134 } 8135 // For intrinsics we allow any location, so it may be on the stack. 8136 if (!location.IsRegister()) { 8137 __ LoadFromOffset(kLoadWord, temp, SP, location.GetStackIndex()); 8138 return temp; 8139 } 8140 // For register locations, check if the register was saved. If so, get it from the stack. 8141 // Note: There is a chance that the register was saved but not overwritten, so we could 8142 // save one load. However, since this is just an intrinsic slow path we prefer this 8143 // simple and more robust approach rather that trying to determine if that's the case. 8144 SlowPathCode* slow_path = GetCurrentSlowPath(); 8145 if (slow_path != nullptr && slow_path->IsCoreRegisterSaved(location.AsRegister<Register>())) { 8146 int stack_offset = slow_path->GetStackOffsetOfCoreRegister(location.AsRegister<Register>()); 8147 __ LoadFromOffset(kLoadWord, temp, SP, stack_offset); 8148 return temp; 8149 } 8150 return location.AsRegister<Register>(); 8151} 8152 8153Location CodeGeneratorARM::GenerateCalleeMethodStaticOrDirectCall(HInvokeStaticOrDirect* invoke, 8154 Location temp) { 8155 Location callee_method = temp; // For all kinds except kRecursive, callee will be in temp. 8156 switch (invoke->GetMethodLoadKind()) { 8157 case HInvokeStaticOrDirect::MethodLoadKind::kStringInit: { 8158 uint32_t offset = 8159 GetThreadOffset<kArmPointerSize>(invoke->GetStringInitEntryPoint()).Int32Value(); 8160 // temp = thread->string_init_entrypoint 8161 __ LoadFromOffset(kLoadWord, temp.AsRegister<Register>(), TR, offset); 8162 break; 8163 } 8164 case HInvokeStaticOrDirect::MethodLoadKind::kRecursive: 8165 callee_method = invoke->GetLocations()->InAt(invoke->GetSpecialInputIndex()); 8166 break; 8167 case HInvokeStaticOrDirect::MethodLoadKind::kDirectAddress: 8168 __ LoadImmediate(temp.AsRegister<Register>(), invoke->GetMethodAddress()); 8169 break; 8170 case HInvokeStaticOrDirect::MethodLoadKind::kDexCachePcRelative: { 8171 HArmDexCacheArraysBase* base = 8172 invoke->InputAt(invoke->GetSpecialInputIndex())->AsArmDexCacheArraysBase(); 8173 Register base_reg = GetInvokeStaticOrDirectExtraParameter(invoke, 8174 temp.AsRegister<Register>()); 8175 int32_t offset = invoke->GetDexCacheArrayOffset() - base->GetElementOffset(); 8176 __ LoadFromOffset(kLoadWord, temp.AsRegister<Register>(), base_reg, offset); 8177 break; 8178 } 8179 case HInvokeStaticOrDirect::MethodLoadKind::kDexCacheViaMethod: { 8180 Location current_method = invoke->GetLocations()->InAt(invoke->GetSpecialInputIndex()); 8181 Register method_reg; 8182 Register reg = temp.AsRegister<Register>(); 8183 if (current_method.IsRegister()) { 8184 method_reg = current_method.AsRegister<Register>(); 8185 } else { 8186 DCHECK(invoke->GetLocations()->Intrinsified()); 8187 DCHECK(!current_method.IsValid()); 8188 method_reg = reg; 8189 __ LoadFromOffset(kLoadWord, reg, SP, kCurrentMethodStackOffset); 8190 } 8191 // /* ArtMethod*[] */ temp = temp.ptr_sized_fields_->dex_cache_resolved_methods_; 8192 __ LoadFromOffset(kLoadWord, 8193 reg, 8194 method_reg, 8195 ArtMethod::DexCacheResolvedMethodsOffset(kArmPointerSize).Int32Value()); 8196 // temp = temp[index_in_cache]; 8197 // Note: Don't use invoke->GetTargetMethod() as it may point to a different dex file. 8198 uint32_t index_in_cache = invoke->GetDexMethodIndex(); 8199 __ LoadFromOffset(kLoadWord, reg, reg, CodeGenerator::GetCachePointerOffset(index_in_cache)); 8200 break; 8201 } 8202 } 8203 return callee_method; 8204} 8205 8206void CodeGeneratorARM::GenerateStaticOrDirectCall(HInvokeStaticOrDirect* invoke, Location temp) { 8207 Location callee_method = GenerateCalleeMethodStaticOrDirectCall(invoke, temp); 8208 8209 switch (invoke->GetCodePtrLocation()) { 8210 case HInvokeStaticOrDirect::CodePtrLocation::kCallSelf: 8211 __ bl(GetFrameEntryLabel()); 8212 break; 8213 case HInvokeStaticOrDirect::CodePtrLocation::kCallArtMethod: 8214 // LR = callee_method->entry_point_from_quick_compiled_code_ 8215 __ LoadFromOffset( 8216 kLoadWord, LR, callee_method.AsRegister<Register>(), 8217 ArtMethod::EntryPointFromQuickCompiledCodeOffset(kArmPointerSize).Int32Value()); 8218 // LR() 8219 __ blx(LR); 8220 break; 8221 } 8222 8223 DCHECK(!IsLeafMethod()); 8224} 8225 8226void CodeGeneratorARM::GenerateVirtualCall(HInvokeVirtual* invoke, Location temp_location) { 8227 Register temp = temp_location.AsRegister<Register>(); 8228 uint32_t method_offset = mirror::Class::EmbeddedVTableEntryOffset( 8229 invoke->GetVTableIndex(), kArmPointerSize).Uint32Value(); 8230 8231 // Use the calling convention instead of the location of the receiver, as 8232 // intrinsics may have put the receiver in a different register. In the intrinsics 8233 // slow path, the arguments have been moved to the right place, so here we are 8234 // guaranteed that the receiver is the first register of the calling convention. 8235 InvokeDexCallingConvention calling_convention; 8236 Register receiver = calling_convention.GetRegisterAt(0); 8237 uint32_t class_offset = mirror::Object::ClassOffset().Int32Value(); 8238 // /* HeapReference<Class> */ temp = receiver->klass_ 8239 __ LoadFromOffset(kLoadWord, temp, receiver, class_offset); 8240 MaybeRecordImplicitNullCheck(invoke); 8241 // Instead of simply (possibly) unpoisoning `temp` here, we should 8242 // emit a read barrier for the previous class reference load. 8243 // However this is not required in practice, as this is an 8244 // intermediate/temporary reference and because the current 8245 // concurrent copying collector keeps the from-space memory 8246 // intact/accessible until the end of the marking phase (the 8247 // concurrent copying collector may not in the future). 8248 __ MaybeUnpoisonHeapReference(temp); 8249 // temp = temp->GetMethodAt(method_offset); 8250 uint32_t entry_point = ArtMethod::EntryPointFromQuickCompiledCodeOffset( 8251 kArmPointerSize).Int32Value(); 8252 __ LoadFromOffset(kLoadWord, temp, temp, method_offset); 8253 // LR = temp->GetEntryPoint(); 8254 __ LoadFromOffset(kLoadWord, LR, temp, entry_point); 8255 // LR(); 8256 __ blx(LR); 8257} 8258 8259CodeGeneratorARM::PcRelativePatchInfo* CodeGeneratorARM::NewPcRelativeStringPatch( 8260 const DexFile& dex_file, dex::StringIndex string_index) { 8261 return NewPcRelativePatch(dex_file, string_index.index_, &pc_relative_string_patches_); 8262} 8263 8264CodeGeneratorARM::PcRelativePatchInfo* CodeGeneratorARM::NewPcRelativeTypePatch( 8265 const DexFile& dex_file, dex::TypeIndex type_index) { 8266 return NewPcRelativePatch(dex_file, type_index.index_, &pc_relative_type_patches_); 8267} 8268 8269CodeGeneratorARM::PcRelativePatchInfo* CodeGeneratorARM::NewTypeBssEntryPatch( 8270 const DexFile& dex_file, dex::TypeIndex type_index) { 8271 return NewPcRelativePatch(dex_file, type_index.index_, &type_bss_entry_patches_); 8272} 8273 8274CodeGeneratorARM::PcRelativePatchInfo* CodeGeneratorARM::NewPcRelativeDexCacheArrayPatch( 8275 const DexFile& dex_file, uint32_t element_offset) { 8276 return NewPcRelativePatch(dex_file, element_offset, &pc_relative_dex_cache_patches_); 8277} 8278 8279CodeGeneratorARM::PcRelativePatchInfo* CodeGeneratorARM::NewPcRelativePatch( 8280 const DexFile& dex_file, uint32_t offset_or_index, ArenaDeque<PcRelativePatchInfo>* patches) { 8281 patches->emplace_back(dex_file, offset_or_index); 8282 return &patches->back(); 8283} 8284 8285Literal* CodeGeneratorARM::DeduplicateBootImageStringLiteral(const DexFile& dex_file, 8286 dex::StringIndex string_index) { 8287 return boot_image_string_patches_.GetOrCreate( 8288 StringReference(&dex_file, string_index), 8289 [this]() { return __ NewLiteral<uint32_t>(/* placeholder */ 0u); }); 8290} 8291 8292Literal* CodeGeneratorARM::DeduplicateBootImageTypeLiteral(const DexFile& dex_file, 8293 dex::TypeIndex type_index) { 8294 return boot_image_type_patches_.GetOrCreate( 8295 TypeReference(&dex_file, type_index), 8296 [this]() { return __ NewLiteral<uint32_t>(/* placeholder */ 0u); }); 8297} 8298 8299Literal* CodeGeneratorARM::DeduplicateBootImageAddressLiteral(uint32_t address) { 8300 return DeduplicateUint32Literal(dchecked_integral_cast<uint32_t>(address), &uint32_literals_); 8301} 8302 8303Literal* CodeGeneratorARM::DeduplicateJitStringLiteral(const DexFile& dex_file, 8304 dex::StringIndex string_index, 8305 Handle<mirror::String> handle) { 8306 jit_string_roots_.Overwrite(StringReference(&dex_file, string_index), 8307 reinterpret_cast64<uint64_t>(handle.GetReference())); 8308 return jit_string_patches_.GetOrCreate( 8309 StringReference(&dex_file, string_index), 8310 [this]() { return __ NewLiteral<uint32_t>(/* placeholder */ 0u); }); 8311} 8312 8313Literal* CodeGeneratorARM::DeduplicateJitClassLiteral(const DexFile& dex_file, 8314 dex::TypeIndex type_index, 8315 Handle<mirror::Class> handle) { 8316 jit_class_roots_.Overwrite(TypeReference(&dex_file, type_index), 8317 reinterpret_cast64<uint64_t>(handle.GetReference())); 8318 return jit_class_patches_.GetOrCreate( 8319 TypeReference(&dex_file, type_index), 8320 [this]() { return __ NewLiteral<uint32_t>(/* placeholder */ 0u); }); 8321} 8322 8323template <LinkerPatch (*Factory)(size_t, const DexFile*, uint32_t, uint32_t)> 8324inline void CodeGeneratorARM::EmitPcRelativeLinkerPatches( 8325 const ArenaDeque<PcRelativePatchInfo>& infos, 8326 ArenaVector<LinkerPatch>* linker_patches) { 8327 for (const PcRelativePatchInfo& info : infos) { 8328 const DexFile& dex_file = info.target_dex_file; 8329 size_t offset_or_index = info.offset_or_index; 8330 DCHECK(info.add_pc_label.IsBound()); 8331 uint32_t add_pc_offset = dchecked_integral_cast<uint32_t>(info.add_pc_label.Position()); 8332 // Add MOVW patch. 8333 DCHECK(info.movw_label.IsBound()); 8334 uint32_t movw_offset = dchecked_integral_cast<uint32_t>(info.movw_label.Position()); 8335 linker_patches->push_back(Factory(movw_offset, &dex_file, add_pc_offset, offset_or_index)); 8336 // Add MOVT patch. 8337 DCHECK(info.movt_label.IsBound()); 8338 uint32_t movt_offset = dchecked_integral_cast<uint32_t>(info.movt_label.Position()); 8339 linker_patches->push_back(Factory(movt_offset, &dex_file, add_pc_offset, offset_or_index)); 8340 } 8341} 8342 8343void CodeGeneratorARM::EmitLinkerPatches(ArenaVector<LinkerPatch>* linker_patches) { 8344 DCHECK(linker_patches->empty()); 8345 size_t size = 8346 /* MOVW+MOVT for each entry */ 2u * pc_relative_dex_cache_patches_.size() + 8347 boot_image_string_patches_.size() + 8348 /* MOVW+MOVT for each entry */ 2u * pc_relative_string_patches_.size() + 8349 boot_image_type_patches_.size() + 8350 /* MOVW+MOVT for each entry */ 2u * pc_relative_type_patches_.size() + 8351 /* MOVW+MOVT for each entry */ 2u * type_bss_entry_patches_.size(); 8352 linker_patches->reserve(size); 8353 EmitPcRelativeLinkerPatches<LinkerPatch::DexCacheArrayPatch>(pc_relative_dex_cache_patches_, 8354 linker_patches); 8355 for (const auto& entry : boot_image_string_patches_) { 8356 const StringReference& target_string = entry.first; 8357 Literal* literal = entry.second; 8358 DCHECK(literal->GetLabel()->IsBound()); 8359 uint32_t literal_offset = literal->GetLabel()->Position(); 8360 linker_patches->push_back(LinkerPatch::StringPatch(literal_offset, 8361 target_string.dex_file, 8362 target_string.string_index.index_)); 8363 } 8364 if (!GetCompilerOptions().IsBootImage()) { 8365 DCHECK(pc_relative_type_patches_.empty()); 8366 EmitPcRelativeLinkerPatches<LinkerPatch::StringBssEntryPatch>(pc_relative_string_patches_, 8367 linker_patches); 8368 } else { 8369 EmitPcRelativeLinkerPatches<LinkerPatch::RelativeTypePatch>(pc_relative_type_patches_, 8370 linker_patches); 8371 EmitPcRelativeLinkerPatches<LinkerPatch::RelativeStringPatch>(pc_relative_string_patches_, 8372 linker_patches); 8373 } 8374 EmitPcRelativeLinkerPatches<LinkerPatch::TypeBssEntryPatch>(type_bss_entry_patches_, 8375 linker_patches); 8376 for (const auto& entry : boot_image_type_patches_) { 8377 const TypeReference& target_type = entry.first; 8378 Literal* literal = entry.second; 8379 DCHECK(literal->GetLabel()->IsBound()); 8380 uint32_t literal_offset = literal->GetLabel()->Position(); 8381 linker_patches->push_back(LinkerPatch::TypePatch(literal_offset, 8382 target_type.dex_file, 8383 target_type.type_index.index_)); 8384 } 8385 DCHECK_EQ(size, linker_patches->size()); 8386} 8387 8388Literal* CodeGeneratorARM::DeduplicateUint32Literal(uint32_t value, Uint32ToLiteralMap* map) { 8389 return map->GetOrCreate( 8390 value, 8391 [this, value]() { return __ NewLiteral<uint32_t>(value); }); 8392} 8393 8394Literal* CodeGeneratorARM::DeduplicateMethodLiteral(MethodReference target_method, 8395 MethodToLiteralMap* map) { 8396 return map->GetOrCreate( 8397 target_method, 8398 [this]() { return __ NewLiteral<uint32_t>(/* placeholder */ 0u); }); 8399} 8400 8401void LocationsBuilderARM::VisitMultiplyAccumulate(HMultiplyAccumulate* instr) { 8402 LocationSummary* locations = 8403 new (GetGraph()->GetArena()) LocationSummary(instr, LocationSummary::kNoCall); 8404 locations->SetInAt(HMultiplyAccumulate::kInputAccumulatorIndex, 8405 Location::RequiresRegister()); 8406 locations->SetInAt(HMultiplyAccumulate::kInputMulLeftIndex, Location::RequiresRegister()); 8407 locations->SetInAt(HMultiplyAccumulate::kInputMulRightIndex, Location::RequiresRegister()); 8408 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 8409} 8410 8411void InstructionCodeGeneratorARM::VisitMultiplyAccumulate(HMultiplyAccumulate* instr) { 8412 LocationSummary* locations = instr->GetLocations(); 8413 Register res = locations->Out().AsRegister<Register>(); 8414 Register accumulator = 8415 locations->InAt(HMultiplyAccumulate::kInputAccumulatorIndex).AsRegister<Register>(); 8416 Register mul_left = 8417 locations->InAt(HMultiplyAccumulate::kInputMulLeftIndex).AsRegister<Register>(); 8418 Register mul_right = 8419 locations->InAt(HMultiplyAccumulate::kInputMulRightIndex).AsRegister<Register>(); 8420 8421 if (instr->GetOpKind() == HInstruction::kAdd) { 8422 __ mla(res, mul_left, mul_right, accumulator); 8423 } else { 8424 __ mls(res, mul_left, mul_right, accumulator); 8425 } 8426} 8427 8428void LocationsBuilderARM::VisitBoundType(HBoundType* instruction ATTRIBUTE_UNUSED) { 8429 // Nothing to do, this should be removed during prepare for register allocator. 8430 LOG(FATAL) << "Unreachable"; 8431} 8432 8433void InstructionCodeGeneratorARM::VisitBoundType(HBoundType* instruction ATTRIBUTE_UNUSED) { 8434 // Nothing to do, this should be removed during prepare for register allocator. 8435 LOG(FATAL) << "Unreachable"; 8436} 8437 8438// Simple implementation of packed switch - generate cascaded compare/jumps. 8439void LocationsBuilderARM::VisitPackedSwitch(HPackedSwitch* switch_instr) { 8440 LocationSummary* locations = 8441 new (GetGraph()->GetArena()) LocationSummary(switch_instr, LocationSummary::kNoCall); 8442 locations->SetInAt(0, Location::RequiresRegister()); 8443 if (switch_instr->GetNumEntries() > kPackedSwitchCompareJumpThreshold && 8444 codegen_->GetAssembler()->IsThumb()) { 8445 locations->AddTemp(Location::RequiresRegister()); // We need a temp for the table base. 8446 if (switch_instr->GetStartValue() != 0) { 8447 locations->AddTemp(Location::RequiresRegister()); // We need a temp for the bias. 8448 } 8449 } 8450} 8451 8452void InstructionCodeGeneratorARM::VisitPackedSwitch(HPackedSwitch* switch_instr) { 8453 int32_t lower_bound = switch_instr->GetStartValue(); 8454 uint32_t num_entries = switch_instr->GetNumEntries(); 8455 LocationSummary* locations = switch_instr->GetLocations(); 8456 Register value_reg = locations->InAt(0).AsRegister<Register>(); 8457 HBasicBlock* default_block = switch_instr->GetDefaultBlock(); 8458 8459 if (num_entries <= kPackedSwitchCompareJumpThreshold || !codegen_->GetAssembler()->IsThumb()) { 8460 // Create a series of compare/jumps. 8461 Register temp_reg = IP; 8462 // Note: It is fine for the below AddConstantSetFlags() using IP register to temporarily store 8463 // the immediate, because IP is used as the destination register. For the other 8464 // AddConstantSetFlags() and GenerateCompareWithImmediate(), the immediate values are constant, 8465 // and they can be encoded in the instruction without making use of IP register. 8466 __ AddConstantSetFlags(temp_reg, value_reg, -lower_bound); 8467 8468 const ArenaVector<HBasicBlock*>& successors = switch_instr->GetBlock()->GetSuccessors(); 8469 // Jump to successors[0] if value == lower_bound. 8470 __ b(codegen_->GetLabelOf(successors[0]), EQ); 8471 int32_t last_index = 0; 8472 for (; num_entries - last_index > 2; last_index += 2) { 8473 __ AddConstantSetFlags(temp_reg, temp_reg, -2); 8474 // Jump to successors[last_index + 1] if value < case_value[last_index + 2]. 8475 __ b(codegen_->GetLabelOf(successors[last_index + 1]), LO); 8476 // Jump to successors[last_index + 2] if value == case_value[last_index + 2]. 8477 __ b(codegen_->GetLabelOf(successors[last_index + 2]), EQ); 8478 } 8479 if (num_entries - last_index == 2) { 8480 // The last missing case_value. 8481 __ CmpConstant(temp_reg, 1); 8482 __ b(codegen_->GetLabelOf(successors[last_index + 1]), EQ); 8483 } 8484 8485 // And the default for any other value. 8486 if (!codegen_->GoesToNextBlock(switch_instr->GetBlock(), default_block)) { 8487 __ b(codegen_->GetLabelOf(default_block)); 8488 } 8489 } else { 8490 // Create a table lookup. 8491 Register temp_reg = locations->GetTemp(0).AsRegister<Register>(); 8492 8493 // Materialize a pointer to the switch table 8494 std::vector<Label*> labels(num_entries); 8495 const ArenaVector<HBasicBlock*>& successors = switch_instr->GetBlock()->GetSuccessors(); 8496 for (uint32_t i = 0; i < num_entries; i++) { 8497 labels[i] = codegen_->GetLabelOf(successors[i]); 8498 } 8499 JumpTable* table = __ CreateJumpTable(std::move(labels), temp_reg); 8500 8501 // Remove the bias. 8502 Register key_reg; 8503 if (lower_bound != 0) { 8504 key_reg = locations->GetTemp(1).AsRegister<Register>(); 8505 __ AddConstant(key_reg, value_reg, -lower_bound); 8506 } else { 8507 key_reg = value_reg; 8508 } 8509 8510 // Check whether the value is in the table, jump to default block if not. 8511 __ CmpConstant(key_reg, num_entries - 1); 8512 __ b(codegen_->GetLabelOf(default_block), Condition::HI); 8513 8514 // Load the displacement from the table. 8515 __ ldr(temp_reg, Address(temp_reg, key_reg, Shift::LSL, 2)); 8516 8517 // Dispatch is a direct add to the PC (for Thumb2). 8518 __ EmitJumpTableDispatch(table, temp_reg); 8519 } 8520} 8521 8522void LocationsBuilderARM::VisitArmDexCacheArraysBase(HArmDexCacheArraysBase* base) { 8523 LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(base); 8524 locations->SetOut(Location::RequiresRegister()); 8525} 8526 8527void InstructionCodeGeneratorARM::VisitArmDexCacheArraysBase(HArmDexCacheArraysBase* base) { 8528 Register base_reg = base->GetLocations()->Out().AsRegister<Register>(); 8529 CodeGeneratorARM::PcRelativePatchInfo* labels = 8530 codegen_->NewPcRelativeDexCacheArrayPatch(base->GetDexFile(), base->GetElementOffset()); 8531 __ BindTrackedLabel(&labels->movw_label); 8532 __ movw(base_reg, /* placeholder */ 0u); 8533 __ BindTrackedLabel(&labels->movt_label); 8534 __ movt(base_reg, /* placeholder */ 0u); 8535 __ BindTrackedLabel(&labels->add_pc_label); 8536 __ add(base_reg, base_reg, ShifterOperand(PC)); 8537} 8538 8539void CodeGeneratorARM::MoveFromReturnRegister(Location trg, Primitive::Type type) { 8540 if (!trg.IsValid()) { 8541 DCHECK_EQ(type, Primitive::kPrimVoid); 8542 return; 8543 } 8544 8545 DCHECK_NE(type, Primitive::kPrimVoid); 8546 8547 Location return_loc = InvokeDexCallingConventionVisitorARM().GetReturnLocation(type); 8548 if (return_loc.Equals(trg)) { 8549 return; 8550 } 8551 8552 // TODO: Consider pairs in the parallel move resolver, then this could be nicely merged 8553 // with the last branch. 8554 if (type == Primitive::kPrimLong) { 8555 HParallelMove parallel_move(GetGraph()->GetArena()); 8556 parallel_move.AddMove(return_loc.ToLow(), trg.ToLow(), Primitive::kPrimInt, nullptr); 8557 parallel_move.AddMove(return_loc.ToHigh(), trg.ToHigh(), Primitive::kPrimInt, nullptr); 8558 GetMoveResolver()->EmitNativeCode(¶llel_move); 8559 } else if (type == Primitive::kPrimDouble) { 8560 HParallelMove parallel_move(GetGraph()->GetArena()); 8561 parallel_move.AddMove(return_loc.ToLow(), trg.ToLow(), Primitive::kPrimFloat, nullptr); 8562 parallel_move.AddMove(return_loc.ToHigh(), trg.ToHigh(), Primitive::kPrimFloat, nullptr); 8563 GetMoveResolver()->EmitNativeCode(¶llel_move); 8564 } else { 8565 // Let the parallel move resolver take care of all of this. 8566 HParallelMove parallel_move(GetGraph()->GetArena()); 8567 parallel_move.AddMove(return_loc, trg, type, nullptr); 8568 GetMoveResolver()->EmitNativeCode(¶llel_move); 8569 } 8570} 8571 8572void LocationsBuilderARM::VisitClassTableGet(HClassTableGet* instruction) { 8573 LocationSummary* locations = 8574 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall); 8575 locations->SetInAt(0, Location::RequiresRegister()); 8576 locations->SetOut(Location::RequiresRegister()); 8577} 8578 8579void InstructionCodeGeneratorARM::VisitClassTableGet(HClassTableGet* instruction) { 8580 LocationSummary* locations = instruction->GetLocations(); 8581 if (instruction->GetTableKind() == HClassTableGet::TableKind::kVTable) { 8582 uint32_t method_offset = mirror::Class::EmbeddedVTableEntryOffset( 8583 instruction->GetIndex(), kArmPointerSize).SizeValue(); 8584 __ LoadFromOffset(kLoadWord, 8585 locations->Out().AsRegister<Register>(), 8586 locations->InAt(0).AsRegister<Register>(), 8587 method_offset); 8588 } else { 8589 uint32_t method_offset = static_cast<uint32_t>(ImTable::OffsetOfElement( 8590 instruction->GetIndex(), kArmPointerSize)); 8591 __ LoadFromOffset(kLoadWord, 8592 locations->Out().AsRegister<Register>(), 8593 locations->InAt(0).AsRegister<Register>(), 8594 mirror::Class::ImtPtrOffset(kArmPointerSize).Uint32Value()); 8595 __ LoadFromOffset(kLoadWord, 8596 locations->Out().AsRegister<Register>(), 8597 locations->Out().AsRegister<Register>(), 8598 method_offset); 8599 } 8600} 8601 8602static void PatchJitRootUse(uint8_t* code, 8603 const uint8_t* roots_data, 8604 Literal* literal, 8605 uint64_t index_in_table) { 8606 DCHECK(literal->GetLabel()->IsBound()); 8607 uint32_t literal_offset = literal->GetLabel()->Position(); 8608 uintptr_t address = 8609 reinterpret_cast<uintptr_t>(roots_data) + index_in_table * sizeof(GcRoot<mirror::Object>); 8610 uint8_t* data = code + literal_offset; 8611 reinterpret_cast<uint32_t*>(data)[0] = dchecked_integral_cast<uint32_t>(address); 8612} 8613 8614void CodeGeneratorARM::EmitJitRootPatches(uint8_t* code, const uint8_t* roots_data) { 8615 for (const auto& entry : jit_string_patches_) { 8616 const auto& it = jit_string_roots_.find(entry.first); 8617 DCHECK(it != jit_string_roots_.end()); 8618 PatchJitRootUse(code, roots_data, entry.second, it->second); 8619 } 8620 for (const auto& entry : jit_class_patches_) { 8621 const auto& it = jit_class_roots_.find(entry.first); 8622 DCHECK(it != jit_class_roots_.end()); 8623 PatchJitRootUse(code, roots_data, entry.second, it->second); 8624 } 8625} 8626 8627#undef __ 8628#undef QUICK_ENTRY_POINT 8629 8630} // namespace arm 8631} // namespace art 8632