code_generator_arm.cc revision 6f64420701c1e1af3bb51bce6a9b608b5bb79fb5
1/* 2 * Copyright (C) 2014 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17#include "code_generator_arm.h" 18 19#include "arch/arm/instruction_set_features_arm.h" 20#include "art_method.h" 21#include "code_generator_utils.h" 22#include "common_arm.h" 23#include "compiled_method.h" 24#include "entrypoints/quick/quick_entrypoints.h" 25#include "gc/accounting/card_table.h" 26#include "intrinsics.h" 27#include "intrinsics_arm.h" 28#include "mirror/array-inl.h" 29#include "mirror/class-inl.h" 30#include "thread.h" 31#include "utils/arm/assembler_arm.h" 32#include "utils/arm/managed_register_arm.h" 33#include "utils/assembler.h" 34#include "utils/stack_checks.h" 35 36namespace art { 37 38template<class MirrorType> 39class GcRoot; 40 41namespace arm { 42 43static bool ExpectedPairLayout(Location location) { 44 // We expected this for both core and fpu register pairs. 45 return ((location.low() & 1) == 0) && (location.low() + 1 == location.high()); 46} 47 48static constexpr int kCurrentMethodStackOffset = 0; 49static constexpr Register kMethodRegisterArgument = R0; 50 51static constexpr Register kCoreAlwaysSpillRegister = R5; 52static constexpr Register kCoreCalleeSaves[] = 53 { R5, R6, R7, R8, R10, R11, LR }; 54static constexpr SRegister kFpuCalleeSaves[] = 55 { S16, S17, S18, S19, S20, S21, S22, S23, S24, S25, S26, S27, S28, S29, S30, S31 }; 56 57// D31 cannot be split into two S registers, and the register allocator only works on 58// S registers. Therefore there is no need to block it. 59static constexpr DRegister DTMP = D31; 60 61static constexpr uint32_t kPackedSwitchCompareJumpThreshold = 7; 62 63// NOLINT on __ macro to suppress wrong warning/fix (misc-macro-parentheses) from clang-tidy. 64#define __ down_cast<ArmAssembler*>(codegen->GetAssembler())-> // NOLINT 65#define QUICK_ENTRY_POINT(x) QUICK_ENTRYPOINT_OFFSET(kArmPointerSize, x).Int32Value() 66 67static constexpr int kRegListThreshold = 4; 68 69// SaveLiveRegisters and RestoreLiveRegisters from SlowPathCodeARM operate on sets of S registers, 70// for each live D registers they treat two corresponding S registers as live ones. 71// 72// Two following functions (SaveContiguousSRegisterList, RestoreContiguousSRegisterList) build 73// from a list of contiguous S registers a list of contiguous D registers (processing first/last 74// S registers corner cases) and save/restore this new list treating them as D registers. 75// - decreasing code size 76// - avoiding hazards on Cortex-A57, when a pair of S registers for an actual live D register is 77// restored and then used in regular non SlowPath code as D register. 78// 79// For the following example (v means the S register is live): 80// D names: | D0 | D1 | D2 | D4 | ... 81// S names: | S0 | S1 | S2 | S3 | S4 | S5 | S6 | S7 | ... 82// Live? | | v | v | v | v | v | v | | ... 83// 84// S1 and S6 will be saved/restored independently; D registers list (D1, D2) will be processed 85// as D registers. 86static size_t SaveContiguousSRegisterList(size_t first, 87 size_t last, 88 CodeGenerator* codegen, 89 size_t stack_offset) { 90 DCHECK_LE(first, last); 91 if ((first == last) && (first == 0)) { 92 stack_offset += codegen->SaveFloatingPointRegister(stack_offset, first); 93 return stack_offset; 94 } 95 if (first % 2 == 1) { 96 stack_offset += codegen->SaveFloatingPointRegister(stack_offset, first++); 97 } 98 99 bool save_last = false; 100 if (last % 2 == 0) { 101 save_last = true; 102 --last; 103 } 104 105 if (first < last) { 106 DRegister d_reg = static_cast<DRegister>(first / 2); 107 DCHECK_EQ((last - first + 1) % 2, 0u); 108 size_t number_of_d_regs = (last - first + 1) / 2; 109 110 if (number_of_d_regs == 1) { 111 __ StoreDToOffset(d_reg, SP, stack_offset); 112 } else if (number_of_d_regs > 1) { 113 __ add(IP, SP, ShifterOperand(stack_offset)); 114 __ vstmiad(IP, d_reg, number_of_d_regs); 115 } 116 stack_offset += number_of_d_regs * kArmWordSize * 2; 117 } 118 119 if (save_last) { 120 stack_offset += codegen->SaveFloatingPointRegister(stack_offset, last + 1); 121 } 122 123 return stack_offset; 124} 125 126static size_t RestoreContiguousSRegisterList(size_t first, 127 size_t last, 128 CodeGenerator* codegen, 129 size_t stack_offset) { 130 DCHECK_LE(first, last); 131 if ((first == last) && (first == 0)) { 132 stack_offset += codegen->RestoreFloatingPointRegister(stack_offset, first); 133 return stack_offset; 134 } 135 if (first % 2 == 1) { 136 stack_offset += codegen->RestoreFloatingPointRegister(stack_offset, first++); 137 } 138 139 bool restore_last = false; 140 if (last % 2 == 0) { 141 restore_last = true; 142 --last; 143 } 144 145 if (first < last) { 146 DRegister d_reg = static_cast<DRegister>(first / 2); 147 DCHECK_EQ((last - first + 1) % 2, 0u); 148 size_t number_of_d_regs = (last - first + 1) / 2; 149 if (number_of_d_regs == 1) { 150 __ LoadDFromOffset(d_reg, SP, stack_offset); 151 } else if (number_of_d_regs > 1) { 152 __ add(IP, SP, ShifterOperand(stack_offset)); 153 __ vldmiad(IP, d_reg, number_of_d_regs); 154 } 155 stack_offset += number_of_d_regs * kArmWordSize * 2; 156 } 157 158 if (restore_last) { 159 stack_offset += codegen->RestoreFloatingPointRegister(stack_offset, last + 1); 160 } 161 162 return stack_offset; 163} 164 165void SlowPathCodeARM::SaveLiveRegisters(CodeGenerator* codegen, LocationSummary* locations) { 166 size_t stack_offset = codegen->GetFirstRegisterSlotInSlowPath(); 167 size_t orig_offset = stack_offset; 168 169 const uint32_t core_spills = codegen->GetSlowPathSpills(locations, /* core_registers */ true); 170 for (uint32_t i : LowToHighBits(core_spills)) { 171 // If the register holds an object, update the stack mask. 172 if (locations->RegisterContainsObject(i)) { 173 locations->SetStackBit(stack_offset / kVRegSize); 174 } 175 DCHECK_LT(stack_offset, codegen->GetFrameSize() - codegen->FrameEntrySpillSize()); 176 DCHECK_LT(i, kMaximumNumberOfExpectedRegisters); 177 saved_core_stack_offsets_[i] = stack_offset; 178 stack_offset += kArmWordSize; 179 } 180 181 int reg_num = POPCOUNT(core_spills); 182 if (reg_num != 0) { 183 if (reg_num > kRegListThreshold) { 184 __ StoreList(RegList(core_spills), orig_offset); 185 } else { 186 stack_offset = orig_offset; 187 for (uint32_t i : LowToHighBits(core_spills)) { 188 stack_offset += codegen->SaveCoreRegister(stack_offset, i); 189 } 190 } 191 } 192 193 uint32_t fp_spills = codegen->GetSlowPathSpills(locations, /* core_registers */ false); 194 orig_offset = stack_offset; 195 for (uint32_t i : LowToHighBits(fp_spills)) { 196 DCHECK_LT(i, kMaximumNumberOfExpectedRegisters); 197 saved_fpu_stack_offsets_[i] = stack_offset; 198 stack_offset += kArmWordSize; 199 } 200 201 stack_offset = orig_offset; 202 while (fp_spills != 0u) { 203 uint32_t begin = CTZ(fp_spills); 204 uint32_t tmp = fp_spills + (1u << begin); 205 fp_spills &= tmp; // Clear the contiguous range of 1s. 206 uint32_t end = (tmp == 0u) ? 32u : CTZ(tmp); // CTZ(0) is undefined. 207 stack_offset = SaveContiguousSRegisterList(begin, end - 1, codegen, stack_offset); 208 } 209 DCHECK_LE(stack_offset, codegen->GetFrameSize() - codegen->FrameEntrySpillSize()); 210} 211 212void SlowPathCodeARM::RestoreLiveRegisters(CodeGenerator* codegen, LocationSummary* locations) { 213 size_t stack_offset = codegen->GetFirstRegisterSlotInSlowPath(); 214 size_t orig_offset = stack_offset; 215 216 const uint32_t core_spills = codegen->GetSlowPathSpills(locations, /* core_registers */ true); 217 for (uint32_t i : LowToHighBits(core_spills)) { 218 DCHECK_LT(stack_offset, codegen->GetFrameSize() - codegen->FrameEntrySpillSize()); 219 DCHECK_LT(i, kMaximumNumberOfExpectedRegisters); 220 stack_offset += kArmWordSize; 221 } 222 223 int reg_num = POPCOUNT(core_spills); 224 if (reg_num != 0) { 225 if (reg_num > kRegListThreshold) { 226 __ LoadList(RegList(core_spills), orig_offset); 227 } else { 228 stack_offset = orig_offset; 229 for (uint32_t i : LowToHighBits(core_spills)) { 230 stack_offset += codegen->RestoreCoreRegister(stack_offset, i); 231 } 232 } 233 } 234 235 uint32_t fp_spills = codegen->GetSlowPathSpills(locations, /* core_registers */ false); 236 while (fp_spills != 0u) { 237 uint32_t begin = CTZ(fp_spills); 238 uint32_t tmp = fp_spills + (1u << begin); 239 fp_spills &= tmp; // Clear the contiguous range of 1s. 240 uint32_t end = (tmp == 0u) ? 32u : CTZ(tmp); // CTZ(0) is undefined. 241 stack_offset = RestoreContiguousSRegisterList(begin, end - 1, codegen, stack_offset); 242 } 243 DCHECK_LE(stack_offset, codegen->GetFrameSize() - codegen->FrameEntrySpillSize()); 244} 245 246class NullCheckSlowPathARM : public SlowPathCodeARM { 247 public: 248 explicit NullCheckSlowPathARM(HNullCheck* instruction) : SlowPathCodeARM(instruction) {} 249 250 void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { 251 CodeGeneratorARM* arm_codegen = down_cast<CodeGeneratorARM*>(codegen); 252 __ Bind(GetEntryLabel()); 253 if (instruction_->CanThrowIntoCatchBlock()) { 254 // Live registers will be restored in the catch block if caught. 255 SaveLiveRegisters(codegen, instruction_->GetLocations()); 256 } 257 arm_codegen->InvokeRuntime(kQuickThrowNullPointer, 258 instruction_, 259 instruction_->GetDexPc(), 260 this); 261 CheckEntrypointTypes<kQuickThrowNullPointer, void, void>(); 262 } 263 264 bool IsFatal() const OVERRIDE { return true; } 265 266 const char* GetDescription() const OVERRIDE { return "NullCheckSlowPathARM"; } 267 268 private: 269 DISALLOW_COPY_AND_ASSIGN(NullCheckSlowPathARM); 270}; 271 272class DivZeroCheckSlowPathARM : public SlowPathCodeARM { 273 public: 274 explicit DivZeroCheckSlowPathARM(HDivZeroCheck* instruction) : SlowPathCodeARM(instruction) {} 275 276 void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { 277 CodeGeneratorARM* arm_codegen = down_cast<CodeGeneratorARM*>(codegen); 278 __ Bind(GetEntryLabel()); 279 arm_codegen->InvokeRuntime(kQuickThrowDivZero, instruction_, instruction_->GetDexPc(), this); 280 CheckEntrypointTypes<kQuickThrowDivZero, void, void>(); 281 } 282 283 bool IsFatal() const OVERRIDE { return true; } 284 285 const char* GetDescription() const OVERRIDE { return "DivZeroCheckSlowPathARM"; } 286 287 private: 288 DISALLOW_COPY_AND_ASSIGN(DivZeroCheckSlowPathARM); 289}; 290 291class SuspendCheckSlowPathARM : public SlowPathCodeARM { 292 public: 293 SuspendCheckSlowPathARM(HSuspendCheck* instruction, HBasicBlock* successor) 294 : SlowPathCodeARM(instruction), successor_(successor) {} 295 296 void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { 297 CodeGeneratorARM* arm_codegen = down_cast<CodeGeneratorARM*>(codegen); 298 __ Bind(GetEntryLabel()); 299 arm_codegen->InvokeRuntime(kQuickTestSuspend, instruction_, instruction_->GetDexPc(), this); 300 CheckEntrypointTypes<kQuickTestSuspend, void, void>(); 301 if (successor_ == nullptr) { 302 __ b(GetReturnLabel()); 303 } else { 304 __ b(arm_codegen->GetLabelOf(successor_)); 305 } 306 } 307 308 Label* GetReturnLabel() { 309 DCHECK(successor_ == nullptr); 310 return &return_label_; 311 } 312 313 HBasicBlock* GetSuccessor() const { 314 return successor_; 315 } 316 317 const char* GetDescription() const OVERRIDE { return "SuspendCheckSlowPathARM"; } 318 319 private: 320 // If not null, the block to branch to after the suspend check. 321 HBasicBlock* const successor_; 322 323 // If `successor_` is null, the label to branch to after the suspend check. 324 Label return_label_; 325 326 DISALLOW_COPY_AND_ASSIGN(SuspendCheckSlowPathARM); 327}; 328 329class BoundsCheckSlowPathARM : public SlowPathCodeARM { 330 public: 331 explicit BoundsCheckSlowPathARM(HBoundsCheck* instruction) 332 : SlowPathCodeARM(instruction) {} 333 334 void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { 335 CodeGeneratorARM* arm_codegen = down_cast<CodeGeneratorARM*>(codegen); 336 LocationSummary* locations = instruction_->GetLocations(); 337 338 __ Bind(GetEntryLabel()); 339 if (instruction_->CanThrowIntoCatchBlock()) { 340 // Live registers will be restored in the catch block if caught. 341 SaveLiveRegisters(codegen, instruction_->GetLocations()); 342 } 343 // We're moving two locations to locations that could overlap, so we need a parallel 344 // move resolver. 345 InvokeRuntimeCallingConvention calling_convention; 346 codegen->EmitParallelMoves( 347 locations->InAt(0), 348 Location::RegisterLocation(calling_convention.GetRegisterAt(0)), 349 Primitive::kPrimInt, 350 locations->InAt(1), 351 Location::RegisterLocation(calling_convention.GetRegisterAt(1)), 352 Primitive::kPrimInt); 353 QuickEntrypointEnum entrypoint = instruction_->AsBoundsCheck()->IsStringCharAt() 354 ? kQuickThrowStringBounds 355 : kQuickThrowArrayBounds; 356 arm_codegen->InvokeRuntime(entrypoint, instruction_, instruction_->GetDexPc(), this); 357 CheckEntrypointTypes<kQuickThrowStringBounds, void, int32_t, int32_t>(); 358 CheckEntrypointTypes<kQuickThrowArrayBounds, void, int32_t, int32_t>(); 359 } 360 361 bool IsFatal() const OVERRIDE { return true; } 362 363 const char* GetDescription() const OVERRIDE { return "BoundsCheckSlowPathARM"; } 364 365 private: 366 DISALLOW_COPY_AND_ASSIGN(BoundsCheckSlowPathARM); 367}; 368 369class LoadClassSlowPathARM : public SlowPathCodeARM { 370 public: 371 LoadClassSlowPathARM(HLoadClass* cls, HInstruction* at, uint32_t dex_pc, bool do_clinit) 372 : SlowPathCodeARM(at), cls_(cls), dex_pc_(dex_pc), do_clinit_(do_clinit) { 373 DCHECK(at->IsLoadClass() || at->IsClinitCheck()); 374 } 375 376 void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { 377 LocationSummary* locations = instruction_->GetLocations(); 378 Location out = locations->Out(); 379 constexpr bool call_saves_everything_except_r0 = (!kUseReadBarrier || kUseBakerReadBarrier); 380 381 CodeGeneratorARM* arm_codegen = down_cast<CodeGeneratorARM*>(codegen); 382 __ Bind(GetEntryLabel()); 383 SaveLiveRegisters(codegen, locations); 384 385 InvokeRuntimeCallingConvention calling_convention; 386 // For HLoadClass/kBssEntry/kSaveEverything, make sure we preserve the address of the entry. 387 DCHECK_EQ(instruction_->IsLoadClass(), cls_ == instruction_); 388 bool is_load_class_bss_entry = 389 (cls_ == instruction_) && (cls_->GetLoadKind() == HLoadClass::LoadKind::kBssEntry); 390 Register entry_address = kNoRegister; 391 if (is_load_class_bss_entry && call_saves_everything_except_r0) { 392 Register temp = locations->GetTemp(0).AsRegister<Register>(); 393 // In the unlucky case that the `temp` is R0, we preserve the address in `out` across 394 // the kSaveEverything call. 395 bool temp_is_r0 = (temp == calling_convention.GetRegisterAt(0)); 396 entry_address = temp_is_r0 ? out.AsRegister<Register>() : temp; 397 DCHECK_NE(entry_address, calling_convention.GetRegisterAt(0)); 398 if (temp_is_r0) { 399 __ mov(entry_address, ShifterOperand(temp)); 400 } 401 } 402 dex::TypeIndex type_index = cls_->GetTypeIndex(); 403 __ LoadImmediate(calling_convention.GetRegisterAt(0), type_index.index_); 404 QuickEntrypointEnum entrypoint = do_clinit_ ? kQuickInitializeStaticStorage 405 : kQuickInitializeType; 406 arm_codegen->InvokeRuntime(entrypoint, instruction_, dex_pc_, this); 407 if (do_clinit_) { 408 CheckEntrypointTypes<kQuickInitializeStaticStorage, void*, uint32_t>(); 409 } else { 410 CheckEntrypointTypes<kQuickInitializeType, void*, uint32_t>(); 411 } 412 413 // For HLoadClass/kBssEntry, store the resolved Class to the BSS entry. 414 if (is_load_class_bss_entry) { 415 if (call_saves_everything_except_r0) { 416 // The class entry address was preserved in `entry_address` thanks to kSaveEverything. 417 __ str(R0, Address(entry_address)); 418 } else { 419 // For non-Baker read barrier, we need to re-calculate the address of the string entry. 420 Register temp = IP; 421 CodeGeneratorARM::PcRelativePatchInfo* labels = 422 arm_codegen->NewTypeBssEntryPatch(cls_->GetDexFile(), type_index); 423 __ BindTrackedLabel(&labels->movw_label); 424 __ movw(temp, /* placeholder */ 0u); 425 __ BindTrackedLabel(&labels->movt_label); 426 __ movt(temp, /* placeholder */ 0u); 427 __ BindTrackedLabel(&labels->add_pc_label); 428 __ add(temp, temp, ShifterOperand(PC)); 429 __ str(R0, Address(temp)); 430 } 431 } 432 // Move the class to the desired location. 433 if (out.IsValid()) { 434 DCHECK(out.IsRegister() && !locations->GetLiveRegisters()->ContainsCoreRegister(out.reg())); 435 arm_codegen->Move32(locations->Out(), Location::RegisterLocation(R0)); 436 } 437 RestoreLiveRegisters(codegen, locations); 438 __ b(GetExitLabel()); 439 } 440 441 const char* GetDescription() const OVERRIDE { return "LoadClassSlowPathARM"; } 442 443 private: 444 // The class this slow path will load. 445 HLoadClass* const cls_; 446 447 // The dex PC of `at_`. 448 const uint32_t dex_pc_; 449 450 // Whether to initialize the class. 451 const bool do_clinit_; 452 453 DISALLOW_COPY_AND_ASSIGN(LoadClassSlowPathARM); 454}; 455 456class LoadStringSlowPathARM : public SlowPathCodeARM { 457 public: 458 explicit LoadStringSlowPathARM(HLoadString* instruction) : SlowPathCodeARM(instruction) {} 459 460 void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { 461 DCHECK(instruction_->IsLoadString()); 462 DCHECK_EQ(instruction_->AsLoadString()->GetLoadKind(), HLoadString::LoadKind::kBssEntry); 463 LocationSummary* locations = instruction_->GetLocations(); 464 DCHECK(!locations->GetLiveRegisters()->ContainsCoreRegister(locations->Out().reg())); 465 HLoadString* load = instruction_->AsLoadString(); 466 const dex::StringIndex string_index = load->GetStringIndex(); 467 Register out = locations->Out().AsRegister<Register>(); 468 constexpr bool call_saves_everything_except_r0 = (!kUseReadBarrier || kUseBakerReadBarrier); 469 470 CodeGeneratorARM* arm_codegen = down_cast<CodeGeneratorARM*>(codegen); 471 __ Bind(GetEntryLabel()); 472 SaveLiveRegisters(codegen, locations); 473 474 InvokeRuntimeCallingConvention calling_convention; 475 // In the unlucky case that the `temp` is R0, we preserve the address in `out` across 476 // the kSaveEverything call. 477 Register entry_address = kNoRegister; 478 if (call_saves_everything_except_r0) { 479 Register temp = locations->GetTemp(0).AsRegister<Register>(); 480 bool temp_is_r0 = (temp == calling_convention.GetRegisterAt(0)); 481 entry_address = temp_is_r0 ? out : temp; 482 DCHECK_NE(entry_address, calling_convention.GetRegisterAt(0)); 483 if (temp_is_r0) { 484 __ mov(entry_address, ShifterOperand(temp)); 485 } 486 } 487 488 __ LoadImmediate(calling_convention.GetRegisterAt(0), string_index.index_); 489 arm_codegen->InvokeRuntime(kQuickResolveString, instruction_, instruction_->GetDexPc(), this); 490 CheckEntrypointTypes<kQuickResolveString, void*, uint32_t>(); 491 492 // Store the resolved String to the .bss entry. 493 if (call_saves_everything_except_r0) { 494 // The string entry address was preserved in `entry_address` thanks to kSaveEverything. 495 __ str(R0, Address(entry_address)); 496 } else { 497 // For non-Baker read barrier, we need to re-calculate the address of the string entry. 498 Register temp = IP; 499 CodeGeneratorARM::PcRelativePatchInfo* labels = 500 arm_codegen->NewPcRelativeStringPatch(load->GetDexFile(), string_index); 501 __ BindTrackedLabel(&labels->movw_label); 502 __ movw(temp, /* placeholder */ 0u); 503 __ BindTrackedLabel(&labels->movt_label); 504 __ movt(temp, /* placeholder */ 0u); 505 __ BindTrackedLabel(&labels->add_pc_label); 506 __ add(temp, temp, ShifterOperand(PC)); 507 __ str(R0, Address(temp)); 508 } 509 510 arm_codegen->Move32(locations->Out(), Location::RegisterLocation(R0)); 511 RestoreLiveRegisters(codegen, locations); 512 513 __ b(GetExitLabel()); 514 } 515 516 const char* GetDescription() const OVERRIDE { return "LoadStringSlowPathARM"; } 517 518 private: 519 DISALLOW_COPY_AND_ASSIGN(LoadStringSlowPathARM); 520}; 521 522class TypeCheckSlowPathARM : public SlowPathCodeARM { 523 public: 524 TypeCheckSlowPathARM(HInstruction* instruction, bool is_fatal) 525 : SlowPathCodeARM(instruction), is_fatal_(is_fatal) {} 526 527 void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { 528 LocationSummary* locations = instruction_->GetLocations(); 529 DCHECK(instruction_->IsCheckCast() 530 || !locations->GetLiveRegisters()->ContainsCoreRegister(locations->Out().reg())); 531 532 CodeGeneratorARM* arm_codegen = down_cast<CodeGeneratorARM*>(codegen); 533 __ Bind(GetEntryLabel()); 534 535 if (!is_fatal_) { 536 SaveLiveRegisters(codegen, locations); 537 } 538 539 // We're moving two locations to locations that could overlap, so we need a parallel 540 // move resolver. 541 InvokeRuntimeCallingConvention calling_convention; 542 codegen->EmitParallelMoves(locations->InAt(0), 543 Location::RegisterLocation(calling_convention.GetRegisterAt(0)), 544 Primitive::kPrimNot, 545 locations->InAt(1), 546 Location::RegisterLocation(calling_convention.GetRegisterAt(1)), 547 Primitive::kPrimNot); 548 if (instruction_->IsInstanceOf()) { 549 arm_codegen->InvokeRuntime(kQuickInstanceofNonTrivial, 550 instruction_, 551 instruction_->GetDexPc(), 552 this); 553 CheckEntrypointTypes<kQuickInstanceofNonTrivial, size_t, mirror::Object*, mirror::Class*>(); 554 arm_codegen->Move32(locations->Out(), Location::RegisterLocation(R0)); 555 } else { 556 DCHECK(instruction_->IsCheckCast()); 557 arm_codegen->InvokeRuntime(kQuickCheckInstanceOf, 558 instruction_, 559 instruction_->GetDexPc(), 560 this); 561 CheckEntrypointTypes<kQuickCheckInstanceOf, void, mirror::Object*, mirror::Class*>(); 562 } 563 564 if (!is_fatal_) { 565 RestoreLiveRegisters(codegen, locations); 566 __ b(GetExitLabel()); 567 } 568 } 569 570 const char* GetDescription() const OVERRIDE { return "TypeCheckSlowPathARM"; } 571 572 bool IsFatal() const OVERRIDE { return is_fatal_; } 573 574 private: 575 const bool is_fatal_; 576 577 DISALLOW_COPY_AND_ASSIGN(TypeCheckSlowPathARM); 578}; 579 580class DeoptimizationSlowPathARM : public SlowPathCodeARM { 581 public: 582 explicit DeoptimizationSlowPathARM(HDeoptimize* instruction) 583 : SlowPathCodeARM(instruction) {} 584 585 void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { 586 CodeGeneratorARM* arm_codegen = down_cast<CodeGeneratorARM*>(codegen); 587 __ Bind(GetEntryLabel()); 588 arm_codegen->InvokeRuntime(kQuickDeoptimize, instruction_, instruction_->GetDexPc(), this); 589 CheckEntrypointTypes<kQuickDeoptimize, void, void>(); 590 } 591 592 const char* GetDescription() const OVERRIDE { return "DeoptimizationSlowPathARM"; } 593 594 private: 595 DISALLOW_COPY_AND_ASSIGN(DeoptimizationSlowPathARM); 596}; 597 598class ArraySetSlowPathARM : public SlowPathCodeARM { 599 public: 600 explicit ArraySetSlowPathARM(HInstruction* instruction) : SlowPathCodeARM(instruction) {} 601 602 void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { 603 LocationSummary* locations = instruction_->GetLocations(); 604 __ Bind(GetEntryLabel()); 605 SaveLiveRegisters(codegen, locations); 606 607 InvokeRuntimeCallingConvention calling_convention; 608 HParallelMove parallel_move(codegen->GetGraph()->GetArena()); 609 parallel_move.AddMove( 610 locations->InAt(0), 611 Location::RegisterLocation(calling_convention.GetRegisterAt(0)), 612 Primitive::kPrimNot, 613 nullptr); 614 parallel_move.AddMove( 615 locations->InAt(1), 616 Location::RegisterLocation(calling_convention.GetRegisterAt(1)), 617 Primitive::kPrimInt, 618 nullptr); 619 parallel_move.AddMove( 620 locations->InAt(2), 621 Location::RegisterLocation(calling_convention.GetRegisterAt(2)), 622 Primitive::kPrimNot, 623 nullptr); 624 codegen->GetMoveResolver()->EmitNativeCode(¶llel_move); 625 626 CodeGeneratorARM* arm_codegen = down_cast<CodeGeneratorARM*>(codegen); 627 arm_codegen->InvokeRuntime(kQuickAputObject, instruction_, instruction_->GetDexPc(), this); 628 CheckEntrypointTypes<kQuickAputObject, void, mirror::Array*, int32_t, mirror::Object*>(); 629 RestoreLiveRegisters(codegen, locations); 630 __ b(GetExitLabel()); 631 } 632 633 const char* GetDescription() const OVERRIDE { return "ArraySetSlowPathARM"; } 634 635 private: 636 DISALLOW_COPY_AND_ASSIGN(ArraySetSlowPathARM); 637}; 638 639// Abstract base class for read barrier slow paths marking a reference 640// `ref`. 641// 642// Argument `entrypoint` must be a register location holding the read 643// barrier marking runtime entry point to be invoked. 644class ReadBarrierMarkSlowPathBaseARM : public SlowPathCodeARM { 645 protected: 646 ReadBarrierMarkSlowPathBaseARM(HInstruction* instruction, Location ref, Location entrypoint) 647 : SlowPathCodeARM(instruction), ref_(ref), entrypoint_(entrypoint) { 648 DCHECK(kEmitCompilerReadBarrier); 649 } 650 651 const char* GetDescription() const OVERRIDE { return "ReadBarrierMarkSlowPathBaseARM"; } 652 653 // Generate assembly code calling the read barrier marking runtime 654 // entry point (ReadBarrierMarkRegX). 655 void GenerateReadBarrierMarkRuntimeCall(CodeGenerator* codegen) { 656 Register ref_reg = ref_.AsRegister<Register>(); 657 658 // No need to save live registers; it's taken care of by the 659 // entrypoint. Also, there is no need to update the stack mask, 660 // as this runtime call will not trigger a garbage collection. 661 CodeGeneratorARM* arm_codegen = down_cast<CodeGeneratorARM*>(codegen); 662 DCHECK_NE(ref_reg, SP); 663 DCHECK_NE(ref_reg, LR); 664 DCHECK_NE(ref_reg, PC); 665 // IP is used internally by the ReadBarrierMarkRegX entry point 666 // as a temporary, it cannot be the entry point's input/output. 667 DCHECK_NE(ref_reg, IP); 668 DCHECK(0 <= ref_reg && ref_reg < kNumberOfCoreRegisters) << ref_reg; 669 // "Compact" slow path, saving two moves. 670 // 671 // Instead of using the standard runtime calling convention (input 672 // and output in R0): 673 // 674 // R0 <- ref 675 // R0 <- ReadBarrierMark(R0) 676 // ref <- R0 677 // 678 // we just use rX (the register containing `ref`) as input and output 679 // of a dedicated entrypoint: 680 // 681 // rX <- ReadBarrierMarkRegX(rX) 682 // 683 if (entrypoint_.IsValid()) { 684 arm_codegen->ValidateInvokeRuntimeWithoutRecordingPcInfo(instruction_, this); 685 __ blx(entrypoint_.AsRegister<Register>()); 686 } else { 687 // Entrypoint is not already loaded, load from the thread. 688 int32_t entry_point_offset = 689 CodeGenerator::GetReadBarrierMarkEntryPointsOffset<kArmPointerSize>(ref_reg); 690 // This runtime call does not require a stack map. 691 arm_codegen->InvokeRuntimeWithoutRecordingPcInfo(entry_point_offset, instruction_, this); 692 } 693 } 694 695 // The location (register) of the marked object reference. 696 const Location ref_; 697 698 // The location of the entrypoint if it is already loaded. 699 const Location entrypoint_; 700 701 private: 702 DISALLOW_COPY_AND_ASSIGN(ReadBarrierMarkSlowPathBaseARM); 703}; 704 705// Slow path marking an object reference `ref` during a read 706// barrier. The field `obj.field` in the object `obj` holding this 707// reference does not get updated by this slow path after marking. 708// 709// This means that after the execution of this slow path, `ref` will 710// always be up-to-date, but `obj.field` may not; i.e., after the 711// flip, `ref` will be a to-space reference, but `obj.field` will 712// probably still be a from-space reference (unless it gets updated by 713// another thread, or if another thread installed another object 714// reference (different from `ref`) in `obj.field`). 715// 716// If `entrypoint` is a valid location it is assumed to already be 717// holding the entrypoint. The case where the entrypoint is passed in 718// is when the decision to mark is based on whether the GC is marking. 719class ReadBarrierMarkSlowPathARM : public ReadBarrierMarkSlowPathBaseARM { 720 public: 721 ReadBarrierMarkSlowPathARM(HInstruction* instruction, 722 Location ref, 723 Location entrypoint = Location::NoLocation()) 724 : ReadBarrierMarkSlowPathBaseARM(instruction, ref, entrypoint) { 725 DCHECK(kEmitCompilerReadBarrier); 726 } 727 728 const char* GetDescription() const OVERRIDE { return "ReadBarrierMarkSlowPathARM"; } 729 730 void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { 731 LocationSummary* locations = instruction_->GetLocations(); 732 DCHECK(locations->CanCall()); 733 if (kIsDebugBuild) { 734 Register ref_reg = ref_.AsRegister<Register>(); 735 DCHECK(!locations->GetLiveRegisters()->ContainsCoreRegister(ref_reg)) << ref_reg; 736 } 737 DCHECK(instruction_->IsLoadClass() || instruction_->IsLoadString()) 738 << "Unexpected instruction in read barrier marking slow path: " 739 << instruction_->DebugName(); 740 741 __ Bind(GetEntryLabel()); 742 GenerateReadBarrierMarkRuntimeCall(codegen); 743 __ b(GetExitLabel()); 744 } 745 746 private: 747 DISALLOW_COPY_AND_ASSIGN(ReadBarrierMarkSlowPathARM); 748}; 749 750// Slow path loading `obj`'s lock word, loading a reference from 751// object `*(obj + offset + (index << scale_factor))` into `ref`, and 752// marking `ref` if `obj` is gray according to the lock word (Baker 753// read barrier). The field `obj.field` in the object `obj` holding 754// this reference does not get updated by this slow path after marking 755// (see LoadReferenceWithBakerReadBarrierAndUpdateFieldSlowPathARM 756// below for that). 757// 758// This means that after the execution of this slow path, `ref` will 759// always be up-to-date, but `obj.field` may not; i.e., after the 760// flip, `ref` will be a to-space reference, but `obj.field` will 761// probably still be a from-space reference (unless it gets updated by 762// another thread, or if another thread installed another object 763// reference (different from `ref`) in `obj.field`). 764// 765// Argument `entrypoint` must be a register location holding the read 766// barrier marking runtime entry point to be invoked. 767class LoadReferenceWithBakerReadBarrierSlowPathARM : public ReadBarrierMarkSlowPathBaseARM { 768 public: 769 LoadReferenceWithBakerReadBarrierSlowPathARM(HInstruction* instruction, 770 Location ref, 771 Register obj, 772 uint32_t offset, 773 Location index, 774 ScaleFactor scale_factor, 775 bool needs_null_check, 776 Register temp, 777 Location entrypoint) 778 : ReadBarrierMarkSlowPathBaseARM(instruction, ref, entrypoint), 779 obj_(obj), 780 offset_(offset), 781 index_(index), 782 scale_factor_(scale_factor), 783 needs_null_check_(needs_null_check), 784 temp_(temp) { 785 DCHECK(kEmitCompilerReadBarrier); 786 DCHECK(kUseBakerReadBarrier); 787 } 788 789 const char* GetDescription() const OVERRIDE { 790 return "LoadReferenceWithBakerReadBarrierSlowPathARM"; 791 } 792 793 void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { 794 LocationSummary* locations = instruction_->GetLocations(); 795 Register ref_reg = ref_.AsRegister<Register>(); 796 DCHECK(locations->CanCall()); 797 DCHECK(!locations->GetLiveRegisters()->ContainsCoreRegister(ref_reg)) << ref_reg; 798 DCHECK_NE(ref_reg, temp_); 799 DCHECK(instruction_->IsInstanceFieldGet() || 800 instruction_->IsStaticFieldGet() || 801 instruction_->IsArrayGet() || 802 instruction_->IsArraySet() || 803 instruction_->IsInstanceOf() || 804 instruction_->IsCheckCast() || 805 (instruction_->IsInvokeVirtual() && instruction_->GetLocations()->Intrinsified()) || 806 (instruction_->IsInvokeStaticOrDirect() && instruction_->GetLocations()->Intrinsified())) 807 << "Unexpected instruction in read barrier marking slow path: " 808 << instruction_->DebugName(); 809 // The read barrier instrumentation of object ArrayGet 810 // instructions does not support the HIntermediateAddress 811 // instruction. 812 DCHECK(!(instruction_->IsArrayGet() && 813 instruction_->AsArrayGet()->GetArray()->IsIntermediateAddress())); 814 815 __ Bind(GetEntryLabel()); 816 817 // When using MaybeGenerateReadBarrierSlow, the read barrier call is 818 // inserted after the original load. However, in fast path based 819 // Baker's read barriers, we need to perform the load of 820 // mirror::Object::monitor_ *before* the original reference load. 821 // This load-load ordering is required by the read barrier. 822 // The fast path/slow path (for Baker's algorithm) should look like: 823 // 824 // uint32_t rb_state = Lockword(obj->monitor_).ReadBarrierState(); 825 // lfence; // Load fence or artificial data dependency to prevent load-load reordering 826 // HeapReference<mirror::Object> ref = *src; // Original reference load. 827 // bool is_gray = (rb_state == ReadBarrier::GrayState()); 828 // if (is_gray) { 829 // ref = entrypoint(ref); // ref = ReadBarrier::Mark(ref); // Runtime entry point call. 830 // } 831 // 832 // Note: the original implementation in ReadBarrier::Barrier is 833 // slightly more complex as it performs additional checks that we do 834 // not do here for performance reasons. 835 836 // /* int32_t */ monitor = obj->monitor_ 837 uint32_t monitor_offset = mirror::Object::MonitorOffset().Int32Value(); 838 __ LoadFromOffset(kLoadWord, temp_, obj_, monitor_offset); 839 if (needs_null_check_) { 840 codegen->MaybeRecordImplicitNullCheck(instruction_); 841 } 842 // /* LockWord */ lock_word = LockWord(monitor) 843 static_assert(sizeof(LockWord) == sizeof(int32_t), 844 "art::LockWord and int32_t have different sizes."); 845 846 // Introduce a dependency on the lock_word including the rb_state, 847 // which shall prevent load-load reordering without using 848 // a memory barrier (which would be more expensive). 849 // `obj` is unchanged by this operation, but its value now depends 850 // on `temp`. 851 __ add(obj_, obj_, ShifterOperand(temp_, LSR, 32)); 852 853 // The actual reference load. 854 // A possible implicit null check has already been handled above. 855 CodeGeneratorARM* arm_codegen = down_cast<CodeGeneratorARM*>(codegen); 856 arm_codegen->GenerateRawReferenceLoad( 857 instruction_, ref_, obj_, offset_, index_, scale_factor_, /* needs_null_check */ false); 858 859 // Mark the object `ref` when `obj` is gray. 860 // 861 // if (rb_state == ReadBarrier::GrayState()) 862 // ref = ReadBarrier::Mark(ref); 863 // 864 // Given the numeric representation, it's enough to check the low bit of the 865 // rb_state. We do that by shifting the bit out of the lock word with LSRS 866 // which can be a 16-bit instruction unlike the TST immediate. 867 static_assert(ReadBarrier::WhiteState() == 0, "Expecting white to have value 0"); 868 static_assert(ReadBarrier::GrayState() == 1, "Expecting gray to have value 1"); 869 __ Lsrs(temp_, temp_, LockWord::kReadBarrierStateShift + 1); 870 __ b(GetExitLabel(), CC); // Carry flag is the last bit shifted out by LSRS. 871 GenerateReadBarrierMarkRuntimeCall(codegen); 872 873 __ b(GetExitLabel()); 874 } 875 876 private: 877 // The register containing the object holding the marked object reference field. 878 Register obj_; 879 // The offset, index and scale factor to access the reference in `obj_`. 880 uint32_t offset_; 881 Location index_; 882 ScaleFactor scale_factor_; 883 // Is a null check required? 884 bool needs_null_check_; 885 // A temporary register used to hold the lock word of `obj_`. 886 Register temp_; 887 888 DISALLOW_COPY_AND_ASSIGN(LoadReferenceWithBakerReadBarrierSlowPathARM); 889}; 890 891// Slow path loading `obj`'s lock word, loading a reference from 892// object `*(obj + offset + (index << scale_factor))` into `ref`, and 893// marking `ref` if `obj` is gray according to the lock word (Baker 894// read barrier). If needed, this slow path also atomically updates 895// the field `obj.field` in the object `obj` holding this reference 896// after marking (contrary to 897// LoadReferenceWithBakerReadBarrierSlowPathARM above, which never 898// tries to update `obj.field`). 899// 900// This means that after the execution of this slow path, both `ref` 901// and `obj.field` will be up-to-date; i.e., after the flip, both will 902// hold the same to-space reference (unless another thread installed 903// another object reference (different from `ref`) in `obj.field`). 904// 905// Argument `entrypoint` must be a register location holding the read 906// barrier marking runtime entry point to be invoked. 907class LoadReferenceWithBakerReadBarrierAndUpdateFieldSlowPathARM 908 : public ReadBarrierMarkSlowPathBaseARM { 909 public: 910 LoadReferenceWithBakerReadBarrierAndUpdateFieldSlowPathARM(HInstruction* instruction, 911 Location ref, 912 Register obj, 913 uint32_t offset, 914 Location index, 915 ScaleFactor scale_factor, 916 bool needs_null_check, 917 Register temp1, 918 Register temp2, 919 Location entrypoint) 920 : ReadBarrierMarkSlowPathBaseARM(instruction, ref, entrypoint), 921 obj_(obj), 922 offset_(offset), 923 index_(index), 924 scale_factor_(scale_factor), 925 needs_null_check_(needs_null_check), 926 temp1_(temp1), 927 temp2_(temp2) { 928 DCHECK(kEmitCompilerReadBarrier); 929 DCHECK(kUseBakerReadBarrier); 930 } 931 932 const char* GetDescription() const OVERRIDE { 933 return "LoadReferenceWithBakerReadBarrierAndUpdateFieldSlowPathARM"; 934 } 935 936 void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { 937 LocationSummary* locations = instruction_->GetLocations(); 938 Register ref_reg = ref_.AsRegister<Register>(); 939 DCHECK(locations->CanCall()); 940 DCHECK(!locations->GetLiveRegisters()->ContainsCoreRegister(ref_reg)) << ref_reg; 941 DCHECK_NE(ref_reg, temp1_); 942 943 // This slow path is only used by the UnsafeCASObject intrinsic at the moment. 944 DCHECK((instruction_->IsInvokeVirtual() && instruction_->GetLocations()->Intrinsified())) 945 << "Unexpected instruction in read barrier marking and field updating slow path: " 946 << instruction_->DebugName(); 947 DCHECK(instruction_->GetLocations()->Intrinsified()); 948 DCHECK_EQ(instruction_->AsInvoke()->GetIntrinsic(), Intrinsics::kUnsafeCASObject); 949 DCHECK_EQ(offset_, 0u); 950 DCHECK_EQ(scale_factor_, ScaleFactor::TIMES_1); 951 // The location of the offset of the marked reference field within `obj_`. 952 Location field_offset = index_; 953 DCHECK(field_offset.IsRegisterPair()) << field_offset; 954 955 __ Bind(GetEntryLabel()); 956 957 // /* int32_t */ monitor = obj->monitor_ 958 uint32_t monitor_offset = mirror::Object::MonitorOffset().Int32Value(); 959 __ LoadFromOffset(kLoadWord, temp1_, obj_, monitor_offset); 960 if (needs_null_check_) { 961 codegen->MaybeRecordImplicitNullCheck(instruction_); 962 } 963 // /* LockWord */ lock_word = LockWord(monitor) 964 static_assert(sizeof(LockWord) == sizeof(int32_t), 965 "art::LockWord and int32_t have different sizes."); 966 967 // Introduce a dependency on the lock_word including the rb_state, 968 // which shall prevent load-load reordering without using 969 // a memory barrier (which would be more expensive). 970 // `obj` is unchanged by this operation, but its value now depends 971 // on `temp1`. 972 __ add(obj_, obj_, ShifterOperand(temp1_, LSR, 32)); 973 974 // The actual reference load. 975 // A possible implicit null check has already been handled above. 976 CodeGeneratorARM* arm_codegen = down_cast<CodeGeneratorARM*>(codegen); 977 arm_codegen->GenerateRawReferenceLoad( 978 instruction_, ref_, obj_, offset_, index_, scale_factor_, /* needs_null_check */ false); 979 980 // Mark the object `ref` when `obj` is gray. 981 // 982 // if (rb_state == ReadBarrier::GrayState()) 983 // ref = ReadBarrier::Mark(ref); 984 // 985 // Given the numeric representation, it's enough to check the low bit of the 986 // rb_state. We do that by shifting the bit out of the lock word with LSRS 987 // which can be a 16-bit instruction unlike the TST immediate. 988 static_assert(ReadBarrier::WhiteState() == 0, "Expecting white to have value 0"); 989 static_assert(ReadBarrier::GrayState() == 1, "Expecting gray to have value 1"); 990 __ Lsrs(temp1_, temp1_, LockWord::kReadBarrierStateShift + 1); 991 __ b(GetExitLabel(), CC); // Carry flag is the last bit shifted out by LSRS. 992 993 // Save the old value of the reference before marking it. 994 // Note that we cannot use IP to save the old reference, as IP is 995 // used internally by the ReadBarrierMarkRegX entry point, and we 996 // need the old reference after the call to that entry point. 997 DCHECK_NE(temp1_, IP); 998 __ Mov(temp1_, ref_reg); 999 1000 GenerateReadBarrierMarkRuntimeCall(codegen); 1001 1002 // If the new reference is different from the old reference, 1003 // update the field in the holder (`*(obj_ + field_offset)`). 1004 // 1005 // Note that this field could also hold a different object, if 1006 // another thread had concurrently changed it. In that case, the 1007 // LDREX/SUBS/ITNE sequence of instructions in the compare-and-set 1008 // (CAS) operation below would abort the CAS, leaving the field 1009 // as-is. 1010 __ cmp(temp1_, ShifterOperand(ref_reg)); 1011 __ b(GetExitLabel(), EQ); 1012 1013 // Update the the holder's field atomically. This may fail if 1014 // mutator updates before us, but it's OK. This is achieved 1015 // using a strong compare-and-set (CAS) operation with relaxed 1016 // memory synchronization ordering, where the expected value is 1017 // the old reference and the desired value is the new reference. 1018 1019 // Convenience aliases. 1020 Register base = obj_; 1021 // The UnsafeCASObject intrinsic uses a register pair as field 1022 // offset ("long offset"), of which only the low part contains 1023 // data. 1024 Register offset = field_offset.AsRegisterPairLow<Register>(); 1025 Register expected = temp1_; 1026 Register value = ref_reg; 1027 Register tmp_ptr = IP; // Pointer to actual memory. 1028 Register tmp = temp2_; // Value in memory. 1029 1030 __ add(tmp_ptr, base, ShifterOperand(offset)); 1031 1032 if (kPoisonHeapReferences) { 1033 __ PoisonHeapReference(expected); 1034 if (value == expected) { 1035 // Do not poison `value`, as it is the same register as 1036 // `expected`, which has just been poisoned. 1037 } else { 1038 __ PoisonHeapReference(value); 1039 } 1040 } 1041 1042 // do { 1043 // tmp = [r_ptr] - expected; 1044 // } while (tmp == 0 && failure([r_ptr] <- r_new_value)); 1045 1046 Label loop_head, exit_loop; 1047 __ Bind(&loop_head); 1048 1049 __ ldrex(tmp, tmp_ptr); 1050 1051 __ subs(tmp, tmp, ShifterOperand(expected)); 1052 1053 __ it(NE); 1054 __ clrex(NE); 1055 1056 __ b(&exit_loop, NE); 1057 1058 __ strex(tmp, value, tmp_ptr); 1059 __ cmp(tmp, ShifterOperand(1)); 1060 __ b(&loop_head, EQ); 1061 1062 __ Bind(&exit_loop); 1063 1064 if (kPoisonHeapReferences) { 1065 __ UnpoisonHeapReference(expected); 1066 if (value == expected) { 1067 // Do not unpoison `value`, as it is the same register as 1068 // `expected`, which has just been unpoisoned. 1069 } else { 1070 __ UnpoisonHeapReference(value); 1071 } 1072 } 1073 1074 __ b(GetExitLabel()); 1075 } 1076 1077 private: 1078 // The register containing the object holding the marked object reference field. 1079 const Register obj_; 1080 // The offset, index and scale factor to access the reference in `obj_`. 1081 uint32_t offset_; 1082 Location index_; 1083 ScaleFactor scale_factor_; 1084 // Is a null check required? 1085 bool needs_null_check_; 1086 // A temporary register used to hold the lock word of `obj_`; and 1087 // also to hold the original reference value, when the reference is 1088 // marked. 1089 const Register temp1_; 1090 // A temporary register used in the implementation of the CAS, to 1091 // update the object's reference field. 1092 const Register temp2_; 1093 1094 DISALLOW_COPY_AND_ASSIGN(LoadReferenceWithBakerReadBarrierAndUpdateFieldSlowPathARM); 1095}; 1096 1097// Slow path generating a read barrier for a heap reference. 1098class ReadBarrierForHeapReferenceSlowPathARM : public SlowPathCodeARM { 1099 public: 1100 ReadBarrierForHeapReferenceSlowPathARM(HInstruction* instruction, 1101 Location out, 1102 Location ref, 1103 Location obj, 1104 uint32_t offset, 1105 Location index) 1106 : SlowPathCodeARM(instruction), 1107 out_(out), 1108 ref_(ref), 1109 obj_(obj), 1110 offset_(offset), 1111 index_(index) { 1112 DCHECK(kEmitCompilerReadBarrier); 1113 // If `obj` is equal to `out` or `ref`, it means the initial object 1114 // has been overwritten by (or after) the heap object reference load 1115 // to be instrumented, e.g.: 1116 // 1117 // __ LoadFromOffset(kLoadWord, out, out, offset); 1118 // codegen_->GenerateReadBarrierSlow(instruction, out_loc, out_loc, out_loc, offset); 1119 // 1120 // In that case, we have lost the information about the original 1121 // object, and the emitted read barrier cannot work properly. 1122 DCHECK(!obj.Equals(out)) << "obj=" << obj << " out=" << out; 1123 DCHECK(!obj.Equals(ref)) << "obj=" << obj << " ref=" << ref; 1124 } 1125 1126 void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { 1127 CodeGeneratorARM* arm_codegen = down_cast<CodeGeneratorARM*>(codegen); 1128 LocationSummary* locations = instruction_->GetLocations(); 1129 Register reg_out = out_.AsRegister<Register>(); 1130 DCHECK(locations->CanCall()); 1131 DCHECK(!locations->GetLiveRegisters()->ContainsCoreRegister(reg_out)); 1132 DCHECK(instruction_->IsInstanceFieldGet() || 1133 instruction_->IsStaticFieldGet() || 1134 instruction_->IsArrayGet() || 1135 instruction_->IsInstanceOf() || 1136 instruction_->IsCheckCast() || 1137 (instruction_->IsInvokeVirtual()) && instruction_->GetLocations()->Intrinsified()) 1138 << "Unexpected instruction in read barrier for heap reference slow path: " 1139 << instruction_->DebugName(); 1140 // The read barrier instrumentation of object ArrayGet 1141 // instructions does not support the HIntermediateAddress 1142 // instruction. 1143 DCHECK(!(instruction_->IsArrayGet() && 1144 instruction_->AsArrayGet()->GetArray()->IsIntermediateAddress())); 1145 1146 __ Bind(GetEntryLabel()); 1147 SaveLiveRegisters(codegen, locations); 1148 1149 // We may have to change the index's value, but as `index_` is a 1150 // constant member (like other "inputs" of this slow path), 1151 // introduce a copy of it, `index`. 1152 Location index = index_; 1153 if (index_.IsValid()) { 1154 // Handle `index_` for HArrayGet and UnsafeGetObject/UnsafeGetObjectVolatile intrinsics. 1155 if (instruction_->IsArrayGet()) { 1156 // Compute the actual memory offset and store it in `index`. 1157 Register index_reg = index_.AsRegister<Register>(); 1158 DCHECK(locations->GetLiveRegisters()->ContainsCoreRegister(index_reg)); 1159 if (codegen->IsCoreCalleeSaveRegister(index_reg)) { 1160 // We are about to change the value of `index_reg` (see the 1161 // calls to art::arm::Thumb2Assembler::Lsl and 1162 // art::arm::Thumb2Assembler::AddConstant below), but it has 1163 // not been saved by the previous call to 1164 // art::SlowPathCode::SaveLiveRegisters, as it is a 1165 // callee-save register -- 1166 // art::SlowPathCode::SaveLiveRegisters does not consider 1167 // callee-save registers, as it has been designed with the 1168 // assumption that callee-save registers are supposed to be 1169 // handled by the called function. So, as a callee-save 1170 // register, `index_reg` _would_ eventually be saved onto 1171 // the stack, but it would be too late: we would have 1172 // changed its value earlier. Therefore, we manually save 1173 // it here into another freely available register, 1174 // `free_reg`, chosen of course among the caller-save 1175 // registers (as a callee-save `free_reg` register would 1176 // exhibit the same problem). 1177 // 1178 // Note we could have requested a temporary register from 1179 // the register allocator instead; but we prefer not to, as 1180 // this is a slow path, and we know we can find a 1181 // caller-save register that is available. 1182 Register free_reg = FindAvailableCallerSaveRegister(codegen); 1183 __ Mov(free_reg, index_reg); 1184 index_reg = free_reg; 1185 index = Location::RegisterLocation(index_reg); 1186 } else { 1187 // The initial register stored in `index_` has already been 1188 // saved in the call to art::SlowPathCode::SaveLiveRegisters 1189 // (as it is not a callee-save register), so we can freely 1190 // use it. 1191 } 1192 // Shifting the index value contained in `index_reg` by the scale 1193 // factor (2) cannot overflow in practice, as the runtime is 1194 // unable to allocate object arrays with a size larger than 1195 // 2^26 - 1 (that is, 2^28 - 4 bytes). 1196 __ Lsl(index_reg, index_reg, TIMES_4); 1197 static_assert( 1198 sizeof(mirror::HeapReference<mirror::Object>) == sizeof(int32_t), 1199 "art::mirror::HeapReference<art::mirror::Object> and int32_t have different sizes."); 1200 __ AddConstant(index_reg, index_reg, offset_); 1201 } else { 1202 // In the case of the UnsafeGetObject/UnsafeGetObjectVolatile 1203 // intrinsics, `index_` is not shifted by a scale factor of 2 1204 // (as in the case of ArrayGet), as it is actually an offset 1205 // to an object field within an object. 1206 DCHECK(instruction_->IsInvoke()) << instruction_->DebugName(); 1207 DCHECK(instruction_->GetLocations()->Intrinsified()); 1208 DCHECK((instruction_->AsInvoke()->GetIntrinsic() == Intrinsics::kUnsafeGetObject) || 1209 (instruction_->AsInvoke()->GetIntrinsic() == Intrinsics::kUnsafeGetObjectVolatile)) 1210 << instruction_->AsInvoke()->GetIntrinsic(); 1211 DCHECK_EQ(offset_, 0U); 1212 DCHECK(index_.IsRegisterPair()); 1213 // UnsafeGet's offset location is a register pair, the low 1214 // part contains the correct offset. 1215 index = index_.ToLow(); 1216 } 1217 } 1218 1219 // We're moving two or three locations to locations that could 1220 // overlap, so we need a parallel move resolver. 1221 InvokeRuntimeCallingConvention calling_convention; 1222 HParallelMove parallel_move(codegen->GetGraph()->GetArena()); 1223 parallel_move.AddMove(ref_, 1224 Location::RegisterLocation(calling_convention.GetRegisterAt(0)), 1225 Primitive::kPrimNot, 1226 nullptr); 1227 parallel_move.AddMove(obj_, 1228 Location::RegisterLocation(calling_convention.GetRegisterAt(1)), 1229 Primitive::kPrimNot, 1230 nullptr); 1231 if (index.IsValid()) { 1232 parallel_move.AddMove(index, 1233 Location::RegisterLocation(calling_convention.GetRegisterAt(2)), 1234 Primitive::kPrimInt, 1235 nullptr); 1236 codegen->GetMoveResolver()->EmitNativeCode(¶llel_move); 1237 } else { 1238 codegen->GetMoveResolver()->EmitNativeCode(¶llel_move); 1239 __ LoadImmediate(calling_convention.GetRegisterAt(2), offset_); 1240 } 1241 arm_codegen->InvokeRuntime(kQuickReadBarrierSlow, instruction_, instruction_->GetDexPc(), this); 1242 CheckEntrypointTypes< 1243 kQuickReadBarrierSlow, mirror::Object*, mirror::Object*, mirror::Object*, uint32_t>(); 1244 arm_codegen->Move32(out_, Location::RegisterLocation(R0)); 1245 1246 RestoreLiveRegisters(codegen, locations); 1247 __ b(GetExitLabel()); 1248 } 1249 1250 const char* GetDescription() const OVERRIDE { return "ReadBarrierForHeapReferenceSlowPathARM"; } 1251 1252 private: 1253 Register FindAvailableCallerSaveRegister(CodeGenerator* codegen) { 1254 size_t ref = static_cast<int>(ref_.AsRegister<Register>()); 1255 size_t obj = static_cast<int>(obj_.AsRegister<Register>()); 1256 for (size_t i = 0, e = codegen->GetNumberOfCoreRegisters(); i < e; ++i) { 1257 if (i != ref && i != obj && !codegen->IsCoreCalleeSaveRegister(i)) { 1258 return static_cast<Register>(i); 1259 } 1260 } 1261 // We shall never fail to find a free caller-save register, as 1262 // there are more than two core caller-save registers on ARM 1263 // (meaning it is possible to find one which is different from 1264 // `ref` and `obj`). 1265 DCHECK_GT(codegen->GetNumberOfCoreCallerSaveRegisters(), 2u); 1266 LOG(FATAL) << "Could not find a free caller-save register"; 1267 UNREACHABLE(); 1268 } 1269 1270 const Location out_; 1271 const Location ref_; 1272 const Location obj_; 1273 const uint32_t offset_; 1274 // An additional location containing an index to an array. 1275 // Only used for HArrayGet and the UnsafeGetObject & 1276 // UnsafeGetObjectVolatile intrinsics. 1277 const Location index_; 1278 1279 DISALLOW_COPY_AND_ASSIGN(ReadBarrierForHeapReferenceSlowPathARM); 1280}; 1281 1282// Slow path generating a read barrier for a GC root. 1283class ReadBarrierForRootSlowPathARM : public SlowPathCodeARM { 1284 public: 1285 ReadBarrierForRootSlowPathARM(HInstruction* instruction, Location out, Location root) 1286 : SlowPathCodeARM(instruction), out_(out), root_(root) { 1287 DCHECK(kEmitCompilerReadBarrier); 1288 } 1289 1290 void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { 1291 LocationSummary* locations = instruction_->GetLocations(); 1292 Register reg_out = out_.AsRegister<Register>(); 1293 DCHECK(locations->CanCall()); 1294 DCHECK(!locations->GetLiveRegisters()->ContainsCoreRegister(reg_out)); 1295 DCHECK(instruction_->IsLoadClass() || instruction_->IsLoadString()) 1296 << "Unexpected instruction in read barrier for GC root slow path: " 1297 << instruction_->DebugName(); 1298 1299 __ Bind(GetEntryLabel()); 1300 SaveLiveRegisters(codegen, locations); 1301 1302 InvokeRuntimeCallingConvention calling_convention; 1303 CodeGeneratorARM* arm_codegen = down_cast<CodeGeneratorARM*>(codegen); 1304 arm_codegen->Move32(Location::RegisterLocation(calling_convention.GetRegisterAt(0)), root_); 1305 arm_codegen->InvokeRuntime(kQuickReadBarrierForRootSlow, 1306 instruction_, 1307 instruction_->GetDexPc(), 1308 this); 1309 CheckEntrypointTypes<kQuickReadBarrierForRootSlow, mirror::Object*, GcRoot<mirror::Object>*>(); 1310 arm_codegen->Move32(out_, Location::RegisterLocation(R0)); 1311 1312 RestoreLiveRegisters(codegen, locations); 1313 __ b(GetExitLabel()); 1314 } 1315 1316 const char* GetDescription() const OVERRIDE { return "ReadBarrierForRootSlowPathARM"; } 1317 1318 private: 1319 const Location out_; 1320 const Location root_; 1321 1322 DISALLOW_COPY_AND_ASSIGN(ReadBarrierForRootSlowPathARM); 1323}; 1324 1325inline Condition ARMCondition(IfCondition cond) { 1326 switch (cond) { 1327 case kCondEQ: return EQ; 1328 case kCondNE: return NE; 1329 case kCondLT: return LT; 1330 case kCondLE: return LE; 1331 case kCondGT: return GT; 1332 case kCondGE: return GE; 1333 case kCondB: return LO; 1334 case kCondBE: return LS; 1335 case kCondA: return HI; 1336 case kCondAE: return HS; 1337 } 1338 LOG(FATAL) << "Unreachable"; 1339 UNREACHABLE(); 1340} 1341 1342// Maps signed condition to unsigned condition. 1343inline Condition ARMUnsignedCondition(IfCondition cond) { 1344 switch (cond) { 1345 case kCondEQ: return EQ; 1346 case kCondNE: return NE; 1347 // Signed to unsigned. 1348 case kCondLT: return LO; 1349 case kCondLE: return LS; 1350 case kCondGT: return HI; 1351 case kCondGE: return HS; 1352 // Unsigned remain unchanged. 1353 case kCondB: return LO; 1354 case kCondBE: return LS; 1355 case kCondA: return HI; 1356 case kCondAE: return HS; 1357 } 1358 LOG(FATAL) << "Unreachable"; 1359 UNREACHABLE(); 1360} 1361 1362inline Condition ARMFPCondition(IfCondition cond, bool gt_bias) { 1363 // The ARM condition codes can express all the necessary branches, see the 1364 // "Meaning (floating-point)" column in the table A8-1 of the ARMv7 reference manual. 1365 // There is no dex instruction or HIR that would need the missing conditions 1366 // "equal or unordered" or "not equal". 1367 switch (cond) { 1368 case kCondEQ: return EQ; 1369 case kCondNE: return NE /* unordered */; 1370 case kCondLT: return gt_bias ? CC : LT /* unordered */; 1371 case kCondLE: return gt_bias ? LS : LE /* unordered */; 1372 case kCondGT: return gt_bias ? HI /* unordered */ : GT; 1373 case kCondGE: return gt_bias ? CS /* unordered */ : GE; 1374 default: 1375 LOG(FATAL) << "UNREACHABLE"; 1376 UNREACHABLE(); 1377 } 1378} 1379 1380inline Shift ShiftFromOpKind(HDataProcWithShifterOp::OpKind op_kind) { 1381 switch (op_kind) { 1382 case HDataProcWithShifterOp::kASR: return ASR; 1383 case HDataProcWithShifterOp::kLSL: return LSL; 1384 case HDataProcWithShifterOp::kLSR: return LSR; 1385 default: 1386 LOG(FATAL) << "Unexpected op kind " << op_kind; 1387 UNREACHABLE(); 1388 } 1389} 1390 1391static void GenerateDataProcInstruction(HInstruction::InstructionKind kind, 1392 Register out, 1393 Register first, 1394 const ShifterOperand& second, 1395 CodeGeneratorARM* codegen) { 1396 if (second.IsImmediate() && second.GetImmediate() == 0) { 1397 const ShifterOperand in = kind == HInstruction::kAnd 1398 ? ShifterOperand(0) 1399 : ShifterOperand(first); 1400 1401 __ mov(out, in); 1402 } else { 1403 switch (kind) { 1404 case HInstruction::kAdd: 1405 __ add(out, first, second); 1406 break; 1407 case HInstruction::kAnd: 1408 __ and_(out, first, second); 1409 break; 1410 case HInstruction::kOr: 1411 __ orr(out, first, second); 1412 break; 1413 case HInstruction::kSub: 1414 __ sub(out, first, second); 1415 break; 1416 case HInstruction::kXor: 1417 __ eor(out, first, second); 1418 break; 1419 default: 1420 LOG(FATAL) << "Unexpected instruction kind: " << kind; 1421 UNREACHABLE(); 1422 } 1423 } 1424} 1425 1426static void GenerateDataProc(HInstruction::InstructionKind kind, 1427 const Location& out, 1428 const Location& first, 1429 const ShifterOperand& second_lo, 1430 const ShifterOperand& second_hi, 1431 CodeGeneratorARM* codegen) { 1432 const Register first_hi = first.AsRegisterPairHigh<Register>(); 1433 const Register first_lo = first.AsRegisterPairLow<Register>(); 1434 const Register out_hi = out.AsRegisterPairHigh<Register>(); 1435 const Register out_lo = out.AsRegisterPairLow<Register>(); 1436 1437 if (kind == HInstruction::kAdd) { 1438 __ adds(out_lo, first_lo, second_lo); 1439 __ adc(out_hi, first_hi, second_hi); 1440 } else if (kind == HInstruction::kSub) { 1441 __ subs(out_lo, first_lo, second_lo); 1442 __ sbc(out_hi, first_hi, second_hi); 1443 } else { 1444 GenerateDataProcInstruction(kind, out_lo, first_lo, second_lo, codegen); 1445 GenerateDataProcInstruction(kind, out_hi, first_hi, second_hi, codegen); 1446 } 1447} 1448 1449static ShifterOperand GetShifterOperand(Register rm, Shift shift, uint32_t shift_imm) { 1450 return shift_imm == 0 ? ShifterOperand(rm) : ShifterOperand(rm, shift, shift_imm); 1451} 1452 1453static void GenerateLongDataProc(HDataProcWithShifterOp* instruction, CodeGeneratorARM* codegen) { 1454 DCHECK_EQ(instruction->GetType(), Primitive::kPrimLong); 1455 DCHECK(HDataProcWithShifterOp::IsShiftOp(instruction->GetOpKind())); 1456 1457 const LocationSummary* const locations = instruction->GetLocations(); 1458 const uint32_t shift_value = instruction->GetShiftAmount(); 1459 const HInstruction::InstructionKind kind = instruction->GetInstrKind(); 1460 const Location first = locations->InAt(0); 1461 const Location second = locations->InAt(1); 1462 const Location out = locations->Out(); 1463 const Register first_hi = first.AsRegisterPairHigh<Register>(); 1464 const Register first_lo = first.AsRegisterPairLow<Register>(); 1465 const Register out_hi = out.AsRegisterPairHigh<Register>(); 1466 const Register out_lo = out.AsRegisterPairLow<Register>(); 1467 const Register second_hi = second.AsRegisterPairHigh<Register>(); 1468 const Register second_lo = second.AsRegisterPairLow<Register>(); 1469 const Shift shift = ShiftFromOpKind(instruction->GetOpKind()); 1470 1471 if (shift_value >= 32) { 1472 if (shift == LSL) { 1473 GenerateDataProcInstruction(kind, 1474 out_hi, 1475 first_hi, 1476 ShifterOperand(second_lo, LSL, shift_value - 32), 1477 codegen); 1478 GenerateDataProcInstruction(kind, 1479 out_lo, 1480 first_lo, 1481 ShifterOperand(0), 1482 codegen); 1483 } else if (shift == ASR) { 1484 GenerateDataProc(kind, 1485 out, 1486 first, 1487 GetShifterOperand(second_hi, ASR, shift_value - 32), 1488 ShifterOperand(second_hi, ASR, 31), 1489 codegen); 1490 } else { 1491 DCHECK_EQ(shift, LSR); 1492 GenerateDataProc(kind, 1493 out, 1494 first, 1495 GetShifterOperand(second_hi, LSR, shift_value - 32), 1496 ShifterOperand(0), 1497 codegen); 1498 } 1499 } else { 1500 DCHECK_GT(shift_value, 1U); 1501 DCHECK_LT(shift_value, 32U); 1502 1503 if (shift == LSL) { 1504 // We are not doing this for HInstruction::kAdd because the output will require 1505 // Location::kOutputOverlap; not applicable to other cases. 1506 if (kind == HInstruction::kOr || kind == HInstruction::kXor) { 1507 GenerateDataProcInstruction(kind, 1508 out_hi, 1509 first_hi, 1510 ShifterOperand(second_hi, LSL, shift_value), 1511 codegen); 1512 GenerateDataProcInstruction(kind, 1513 out_hi, 1514 out_hi, 1515 ShifterOperand(second_lo, LSR, 32 - shift_value), 1516 codegen); 1517 GenerateDataProcInstruction(kind, 1518 out_lo, 1519 first_lo, 1520 ShifterOperand(second_lo, LSL, shift_value), 1521 codegen); 1522 } else { 1523 __ Lsl(IP, second_hi, shift_value); 1524 __ orr(IP, IP, ShifterOperand(second_lo, LSR, 32 - shift_value)); 1525 GenerateDataProc(kind, 1526 out, 1527 first, 1528 ShifterOperand(second_lo, LSL, shift_value), 1529 ShifterOperand(IP), 1530 codegen); 1531 } 1532 } else { 1533 DCHECK(shift == ASR || shift == LSR); 1534 1535 // We are not doing this for HInstruction::kAdd because the output will require 1536 // Location::kOutputOverlap; not applicable to other cases. 1537 if (kind == HInstruction::kOr || kind == HInstruction::kXor) { 1538 GenerateDataProcInstruction(kind, 1539 out_lo, 1540 first_lo, 1541 ShifterOperand(second_lo, LSR, shift_value), 1542 codegen); 1543 GenerateDataProcInstruction(kind, 1544 out_lo, 1545 out_lo, 1546 ShifterOperand(second_hi, LSL, 32 - shift_value), 1547 codegen); 1548 GenerateDataProcInstruction(kind, 1549 out_hi, 1550 first_hi, 1551 ShifterOperand(second_hi, shift, shift_value), 1552 codegen); 1553 } else { 1554 __ Lsr(IP, second_lo, shift_value); 1555 __ orr(IP, IP, ShifterOperand(second_hi, LSL, 32 - shift_value)); 1556 GenerateDataProc(kind, 1557 out, 1558 first, 1559 ShifterOperand(IP), 1560 ShifterOperand(second_hi, shift, shift_value), 1561 codegen); 1562 } 1563 } 1564 } 1565} 1566 1567static void GenerateVcmp(HInstruction* instruction, CodeGeneratorARM* codegen) { 1568 Primitive::Type type = instruction->InputAt(0)->GetType(); 1569 Location lhs_loc = instruction->GetLocations()->InAt(0); 1570 Location rhs_loc = instruction->GetLocations()->InAt(1); 1571 if (rhs_loc.IsConstant()) { 1572 // 0.0 is the only immediate that can be encoded directly in 1573 // a VCMP instruction. 1574 // 1575 // Both the JLS (section 15.20.1) and the JVMS (section 6.5) 1576 // specify that in a floating-point comparison, positive zero 1577 // and negative zero are considered equal, so we can use the 1578 // literal 0.0 for both cases here. 1579 // 1580 // Note however that some methods (Float.equal, Float.compare, 1581 // Float.compareTo, Double.equal, Double.compare, 1582 // Double.compareTo, Math.max, Math.min, StrictMath.max, 1583 // StrictMath.min) consider 0.0 to be (strictly) greater than 1584 // -0.0. So if we ever translate calls to these methods into a 1585 // HCompare instruction, we must handle the -0.0 case with 1586 // care here. 1587 DCHECK(rhs_loc.GetConstant()->IsArithmeticZero()); 1588 if (type == Primitive::kPrimFloat) { 1589 __ vcmpsz(lhs_loc.AsFpuRegister<SRegister>()); 1590 } else { 1591 DCHECK_EQ(type, Primitive::kPrimDouble); 1592 __ vcmpdz(FromLowSToD(lhs_loc.AsFpuRegisterPairLow<SRegister>())); 1593 } 1594 } else { 1595 if (type == Primitive::kPrimFloat) { 1596 __ vcmps(lhs_loc.AsFpuRegister<SRegister>(), rhs_loc.AsFpuRegister<SRegister>()); 1597 } else { 1598 DCHECK_EQ(type, Primitive::kPrimDouble); 1599 __ vcmpd(FromLowSToD(lhs_loc.AsFpuRegisterPairLow<SRegister>()), 1600 FromLowSToD(rhs_loc.AsFpuRegisterPairLow<SRegister>())); 1601 } 1602 } 1603} 1604 1605static Condition GenerateLongTestConstant(HCondition* condition, 1606 bool invert, 1607 CodeGeneratorARM* codegen) { 1608 DCHECK_EQ(condition->GetLeft()->GetType(), Primitive::kPrimLong); 1609 1610 const LocationSummary* const locations = condition->GetLocations(); 1611 IfCondition cond = invert ? condition->GetOppositeCondition() : condition->GetCondition(); 1612 Condition ret = EQ; 1613 const Location left = locations->InAt(0); 1614 const Location right = locations->InAt(1); 1615 1616 DCHECK(right.IsConstant()); 1617 1618 const Register left_high = left.AsRegisterPairHigh<Register>(); 1619 const Register left_low = left.AsRegisterPairLow<Register>(); 1620 int64_t value = right.GetConstant()->AsLongConstant()->GetValue(); 1621 1622 switch (cond) { 1623 case kCondEQ: 1624 case kCondNE: 1625 case kCondB: 1626 case kCondBE: 1627 case kCondA: 1628 case kCondAE: 1629 __ CmpConstant(left_high, High32Bits(value)); 1630 __ it(EQ); 1631 __ cmp(left_low, ShifterOperand(Low32Bits(value)), EQ); 1632 ret = ARMUnsignedCondition(cond); 1633 break; 1634 case kCondLE: 1635 case kCondGT: 1636 // Trivially true or false. 1637 if (value == std::numeric_limits<int64_t>::max()) { 1638 __ cmp(left_low, ShifterOperand(left_low)); 1639 ret = cond == kCondLE ? EQ : NE; 1640 break; 1641 } 1642 1643 if (cond == kCondLE) { 1644 cond = kCondLT; 1645 } else { 1646 DCHECK_EQ(cond, kCondGT); 1647 cond = kCondGE; 1648 } 1649 1650 value++; 1651 FALLTHROUGH_INTENDED; 1652 case kCondGE: 1653 case kCondLT: 1654 __ CmpConstant(left_low, Low32Bits(value)); 1655 __ sbcs(IP, left_high, ShifterOperand(High32Bits(value))); 1656 ret = ARMCondition(cond); 1657 break; 1658 default: 1659 LOG(FATAL) << "Unreachable"; 1660 UNREACHABLE(); 1661 } 1662 1663 return ret; 1664} 1665 1666static Condition GenerateLongTest(HCondition* condition, 1667 bool invert, 1668 CodeGeneratorARM* codegen) { 1669 DCHECK_EQ(condition->GetLeft()->GetType(), Primitive::kPrimLong); 1670 1671 const LocationSummary* const locations = condition->GetLocations(); 1672 IfCondition cond = invert ? condition->GetOppositeCondition() : condition->GetCondition(); 1673 Condition ret = EQ; 1674 Location left = locations->InAt(0); 1675 Location right = locations->InAt(1); 1676 1677 DCHECK(right.IsRegisterPair()); 1678 1679 switch (cond) { 1680 case kCondEQ: 1681 case kCondNE: 1682 case kCondB: 1683 case kCondBE: 1684 case kCondA: 1685 case kCondAE: 1686 __ cmp(left.AsRegisterPairHigh<Register>(), 1687 ShifterOperand(right.AsRegisterPairHigh<Register>())); 1688 __ it(EQ); 1689 __ cmp(left.AsRegisterPairLow<Register>(), 1690 ShifterOperand(right.AsRegisterPairLow<Register>()), 1691 EQ); 1692 ret = ARMUnsignedCondition(cond); 1693 break; 1694 case kCondLE: 1695 case kCondGT: 1696 if (cond == kCondLE) { 1697 cond = kCondGE; 1698 } else { 1699 DCHECK_EQ(cond, kCondGT); 1700 cond = kCondLT; 1701 } 1702 1703 std::swap(left, right); 1704 FALLTHROUGH_INTENDED; 1705 case kCondGE: 1706 case kCondLT: 1707 __ cmp(left.AsRegisterPairLow<Register>(), 1708 ShifterOperand(right.AsRegisterPairLow<Register>())); 1709 __ sbcs(IP, 1710 left.AsRegisterPairHigh<Register>(), 1711 ShifterOperand(right.AsRegisterPairHigh<Register>())); 1712 ret = ARMCondition(cond); 1713 break; 1714 default: 1715 LOG(FATAL) << "Unreachable"; 1716 UNREACHABLE(); 1717 } 1718 1719 return ret; 1720} 1721 1722static Condition GenerateTest(HInstruction* instruction, 1723 Location loc, 1724 bool invert, 1725 CodeGeneratorARM* codegen) { 1726 DCHECK(!instruction->IsConstant()); 1727 1728 Condition ret = invert ? EQ : NE; 1729 1730 if (IsBooleanValueOrMaterializedCondition(instruction)) { 1731 __ CmpConstant(loc.AsRegister<Register>(), 0); 1732 } else { 1733 HCondition* const condition = instruction->AsCondition(); 1734 const LocationSummary* const locations = condition->GetLocations(); 1735 const Primitive::Type type = condition->GetLeft()->GetType(); 1736 const IfCondition cond = invert ? condition->GetOppositeCondition() : condition->GetCondition(); 1737 const Location right = locations->InAt(1); 1738 1739 if (type == Primitive::kPrimLong) { 1740 ret = condition->GetLocations()->InAt(1).IsConstant() 1741 ? GenerateLongTestConstant(condition, invert, codegen) 1742 : GenerateLongTest(condition, invert, codegen); 1743 } else if (Primitive::IsFloatingPointType(type)) { 1744 GenerateVcmp(condition, codegen); 1745 __ vmstat(); 1746 ret = ARMFPCondition(cond, condition->IsGtBias()); 1747 } else { 1748 DCHECK(Primitive::IsIntegralType(type) || type == Primitive::kPrimNot) << type; 1749 1750 const Register left = locations->InAt(0).AsRegister<Register>(); 1751 1752 if (right.IsRegister()) { 1753 __ cmp(left, ShifterOperand(right.AsRegister<Register>())); 1754 } else { 1755 DCHECK(right.IsConstant()); 1756 __ CmpConstant(left, CodeGenerator::GetInt32ValueOf(right.GetConstant())); 1757 } 1758 1759 ret = ARMCondition(cond); 1760 } 1761 } 1762 1763 return ret; 1764} 1765 1766static bool CanGenerateTest(HInstruction* condition, ArmAssembler* assembler) { 1767 if (!IsBooleanValueOrMaterializedCondition(condition)) { 1768 const HCondition* const cond = condition->AsCondition(); 1769 1770 if (cond->GetLeft()->GetType() == Primitive::kPrimLong) { 1771 const LocationSummary* const locations = cond->GetLocations(); 1772 const IfCondition c = cond->GetCondition(); 1773 1774 if (locations->InAt(1).IsConstant()) { 1775 const int64_t value = locations->InAt(1).GetConstant()->AsLongConstant()->GetValue(); 1776 ShifterOperand so; 1777 1778 if (c < kCondLT || c > kCondGE) { 1779 // Since IT blocks longer than a 16-bit instruction are deprecated by ARMv8, 1780 // we check that the least significant half of the first input to be compared 1781 // is in a low register (the other half is read outside an IT block), and 1782 // the constant fits in an 8-bit unsigned integer, so that a 16-bit CMP 1783 // encoding can be used. 1784 if (!ArmAssembler::IsLowRegister(locations->InAt(0).AsRegisterPairLow<Register>()) || 1785 !IsUint<8>(Low32Bits(value))) { 1786 return false; 1787 } 1788 } else if (c == kCondLE || c == kCondGT) { 1789 if (value < std::numeric_limits<int64_t>::max() && 1790 !assembler->ShifterOperandCanHold(kNoRegister, 1791 kNoRegister, 1792 SBC, 1793 High32Bits(value + 1), 1794 kCcSet, 1795 &so)) { 1796 return false; 1797 } 1798 } else if (!assembler->ShifterOperandCanHold(kNoRegister, 1799 kNoRegister, 1800 SBC, 1801 High32Bits(value), 1802 kCcSet, 1803 &so)) { 1804 return false; 1805 } 1806 } 1807 } 1808 } 1809 1810 return true; 1811} 1812 1813static bool CanEncodeConstantAs8BitImmediate(HConstant* constant) { 1814 const Primitive::Type type = constant->GetType(); 1815 bool ret = false; 1816 1817 DCHECK(Primitive::IsIntegralType(type) || type == Primitive::kPrimNot) << type; 1818 1819 if (type == Primitive::kPrimLong) { 1820 const uint64_t value = constant->AsLongConstant()->GetValueAsUint64(); 1821 1822 ret = IsUint<8>(Low32Bits(value)) && IsUint<8>(High32Bits(value)); 1823 } else { 1824 ret = IsUint<8>(CodeGenerator::GetInt32ValueOf(constant)); 1825 } 1826 1827 return ret; 1828} 1829 1830static Location Arm8BitEncodableConstantOrRegister(HInstruction* constant) { 1831 DCHECK(!Primitive::IsFloatingPointType(constant->GetType())); 1832 1833 if (constant->IsConstant() && CanEncodeConstantAs8BitImmediate(constant->AsConstant())) { 1834 return Location::ConstantLocation(constant->AsConstant()); 1835 } 1836 1837 return Location::RequiresRegister(); 1838} 1839 1840static bool CanGenerateConditionalMove(const Location& out, const Location& src) { 1841 // Since IT blocks longer than a 16-bit instruction are deprecated by ARMv8, 1842 // we check that we are not dealing with floating-point output (there is no 1843 // 16-bit VMOV encoding). 1844 if (!out.IsRegister() && !out.IsRegisterPair()) { 1845 return false; 1846 } 1847 1848 // For constants, we also check that the output is in one or two low registers, 1849 // and that the constants fit in an 8-bit unsigned integer, so that a 16-bit 1850 // MOV encoding can be used. 1851 if (src.IsConstant()) { 1852 if (!CanEncodeConstantAs8BitImmediate(src.GetConstant())) { 1853 return false; 1854 } 1855 1856 if (out.IsRegister()) { 1857 if (!ArmAssembler::IsLowRegister(out.AsRegister<Register>())) { 1858 return false; 1859 } 1860 } else { 1861 DCHECK(out.IsRegisterPair()); 1862 1863 if (!ArmAssembler::IsLowRegister(out.AsRegisterPairHigh<Register>())) { 1864 return false; 1865 } 1866 } 1867 } 1868 1869 return true; 1870} 1871 1872#undef __ 1873// NOLINT on __ macro to suppress wrong warning/fix (misc-macro-parentheses) from clang-tidy. 1874#define __ down_cast<ArmAssembler*>(GetAssembler())-> // NOLINT 1875 1876Label* CodeGeneratorARM::GetFinalLabel(HInstruction* instruction, Label* final_label) { 1877 DCHECK(!instruction->IsControlFlow() && !instruction->IsSuspendCheck()); 1878 DCHECK(!instruction->IsInvoke() || !instruction->GetLocations()->CanCall()); 1879 1880 const HBasicBlock* const block = instruction->GetBlock(); 1881 const HLoopInformation* const info = block->GetLoopInformation(); 1882 HInstruction* const next = instruction->GetNext(); 1883 1884 // Avoid a branch to a branch. 1885 if (next->IsGoto() && (info == nullptr || 1886 !info->IsBackEdge(*block) || 1887 !info->HasSuspendCheck())) { 1888 final_label = GetLabelOf(next->AsGoto()->GetSuccessor()); 1889 } 1890 1891 return final_label; 1892} 1893 1894void CodeGeneratorARM::DumpCoreRegister(std::ostream& stream, int reg) const { 1895 stream << Register(reg); 1896} 1897 1898void CodeGeneratorARM::DumpFloatingPointRegister(std::ostream& stream, int reg) const { 1899 stream << SRegister(reg); 1900} 1901 1902size_t CodeGeneratorARM::SaveCoreRegister(size_t stack_index, uint32_t reg_id) { 1903 __ StoreToOffset(kStoreWord, static_cast<Register>(reg_id), SP, stack_index); 1904 return kArmWordSize; 1905} 1906 1907size_t CodeGeneratorARM::RestoreCoreRegister(size_t stack_index, uint32_t reg_id) { 1908 __ LoadFromOffset(kLoadWord, static_cast<Register>(reg_id), SP, stack_index); 1909 return kArmWordSize; 1910} 1911 1912size_t CodeGeneratorARM::SaveFloatingPointRegister(size_t stack_index, uint32_t reg_id) { 1913 __ StoreSToOffset(static_cast<SRegister>(reg_id), SP, stack_index); 1914 return kArmWordSize; 1915} 1916 1917size_t CodeGeneratorARM::RestoreFloatingPointRegister(size_t stack_index, uint32_t reg_id) { 1918 __ LoadSFromOffset(static_cast<SRegister>(reg_id), SP, stack_index); 1919 return kArmWordSize; 1920} 1921 1922CodeGeneratorARM::CodeGeneratorARM(HGraph* graph, 1923 const ArmInstructionSetFeatures& isa_features, 1924 const CompilerOptions& compiler_options, 1925 OptimizingCompilerStats* stats) 1926 : CodeGenerator(graph, 1927 kNumberOfCoreRegisters, 1928 kNumberOfSRegisters, 1929 kNumberOfRegisterPairs, 1930 ComputeRegisterMask(reinterpret_cast<const int*>(kCoreCalleeSaves), 1931 arraysize(kCoreCalleeSaves)), 1932 ComputeRegisterMask(reinterpret_cast<const int*>(kFpuCalleeSaves), 1933 arraysize(kFpuCalleeSaves)), 1934 compiler_options, 1935 stats), 1936 block_labels_(nullptr), 1937 location_builder_(graph, this), 1938 instruction_visitor_(graph, this), 1939 move_resolver_(graph->GetArena(), this), 1940 assembler_(graph->GetArena()), 1941 isa_features_(isa_features), 1942 uint32_literals_(std::less<uint32_t>(), 1943 graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), 1944 pc_relative_dex_cache_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), 1945 boot_image_string_patches_(StringReferenceValueComparator(), 1946 graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), 1947 pc_relative_string_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), 1948 boot_image_type_patches_(TypeReferenceValueComparator(), 1949 graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), 1950 pc_relative_type_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), 1951 type_bss_entry_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), 1952 jit_string_patches_(StringReferenceValueComparator(), 1953 graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), 1954 jit_class_patches_(TypeReferenceValueComparator(), 1955 graph->GetArena()->Adapter(kArenaAllocCodeGenerator)) { 1956 // Always save the LR register to mimic Quick. 1957 AddAllocatedRegister(Location::RegisterLocation(LR)); 1958} 1959 1960void CodeGeneratorARM::Finalize(CodeAllocator* allocator) { 1961 // Ensure that we fix up branches and literal loads and emit the literal pool. 1962 __ FinalizeCode(); 1963 1964 // Adjust native pc offsets in stack maps. 1965 for (size_t i = 0, num = stack_map_stream_.GetNumberOfStackMaps(); i != num; ++i) { 1966 uint32_t old_position = 1967 stack_map_stream_.GetStackMap(i).native_pc_code_offset.Uint32Value(kThumb2); 1968 uint32_t new_position = __ GetAdjustedPosition(old_position); 1969 stack_map_stream_.SetStackMapNativePcOffset(i, new_position); 1970 } 1971 // Adjust pc offsets for the disassembly information. 1972 if (disasm_info_ != nullptr) { 1973 GeneratedCodeInterval* frame_entry_interval = disasm_info_->GetFrameEntryInterval(); 1974 frame_entry_interval->start = __ GetAdjustedPosition(frame_entry_interval->start); 1975 frame_entry_interval->end = __ GetAdjustedPosition(frame_entry_interval->end); 1976 for (auto& it : *disasm_info_->GetInstructionIntervals()) { 1977 it.second.start = __ GetAdjustedPosition(it.second.start); 1978 it.second.end = __ GetAdjustedPosition(it.second.end); 1979 } 1980 for (auto& it : *disasm_info_->GetSlowPathIntervals()) { 1981 it.code_interval.start = __ GetAdjustedPosition(it.code_interval.start); 1982 it.code_interval.end = __ GetAdjustedPosition(it.code_interval.end); 1983 } 1984 } 1985 1986 CodeGenerator::Finalize(allocator); 1987} 1988 1989void CodeGeneratorARM::SetupBlockedRegisters() const { 1990 // Stack register, LR and PC are always reserved. 1991 blocked_core_registers_[SP] = true; 1992 blocked_core_registers_[LR] = true; 1993 blocked_core_registers_[PC] = true; 1994 1995 // Reserve thread register. 1996 blocked_core_registers_[TR] = true; 1997 1998 // Reserve temp register. 1999 blocked_core_registers_[IP] = true; 2000 2001 if (GetGraph()->IsDebuggable()) { 2002 // Stubs do not save callee-save floating point registers. If the graph 2003 // is debuggable, we need to deal with these registers differently. For 2004 // now, just block them. 2005 for (size_t i = 0; i < arraysize(kFpuCalleeSaves); ++i) { 2006 blocked_fpu_registers_[kFpuCalleeSaves[i]] = true; 2007 } 2008 } 2009} 2010 2011InstructionCodeGeneratorARM::InstructionCodeGeneratorARM(HGraph* graph, CodeGeneratorARM* codegen) 2012 : InstructionCodeGenerator(graph, codegen), 2013 assembler_(codegen->GetAssembler()), 2014 codegen_(codegen) {} 2015 2016void CodeGeneratorARM::ComputeSpillMask() { 2017 core_spill_mask_ = allocated_registers_.GetCoreRegisters() & core_callee_save_mask_; 2018 DCHECK_NE(core_spill_mask_, 0u) << "At least the return address register must be saved"; 2019 // There is no easy instruction to restore just the PC on thumb2. We spill and 2020 // restore another arbitrary register. 2021 core_spill_mask_ |= (1 << kCoreAlwaysSpillRegister); 2022 fpu_spill_mask_ = allocated_registers_.GetFloatingPointRegisters() & fpu_callee_save_mask_; 2023 // We use vpush and vpop for saving and restoring floating point registers, which take 2024 // a SRegister and the number of registers to save/restore after that SRegister. We 2025 // therefore update the `fpu_spill_mask_` to also contain those registers not allocated, 2026 // but in the range. 2027 if (fpu_spill_mask_ != 0) { 2028 uint32_t least_significant_bit = LeastSignificantBit(fpu_spill_mask_); 2029 uint32_t most_significant_bit = MostSignificantBit(fpu_spill_mask_); 2030 for (uint32_t i = least_significant_bit + 1 ; i < most_significant_bit; ++i) { 2031 fpu_spill_mask_ |= (1 << i); 2032 } 2033 } 2034} 2035 2036static dwarf::Reg DWARFReg(Register reg) { 2037 return dwarf::Reg::ArmCore(static_cast<int>(reg)); 2038} 2039 2040static dwarf::Reg DWARFReg(SRegister reg) { 2041 return dwarf::Reg::ArmFp(static_cast<int>(reg)); 2042} 2043 2044void CodeGeneratorARM::GenerateFrameEntry() { 2045 bool skip_overflow_check = 2046 IsLeafMethod() && !FrameNeedsStackCheck(GetFrameSize(), InstructionSet::kArm); 2047 DCHECK(GetCompilerOptions().GetImplicitStackOverflowChecks()); 2048 __ Bind(&frame_entry_label_); 2049 2050 if (HasEmptyFrame()) { 2051 return; 2052 } 2053 2054 if (!skip_overflow_check) { 2055 __ AddConstant(IP, SP, -static_cast<int32_t>(GetStackOverflowReservedBytes(kArm))); 2056 __ LoadFromOffset(kLoadWord, IP, IP, 0); 2057 RecordPcInfo(nullptr, 0); 2058 } 2059 2060 __ PushList(core_spill_mask_); 2061 __ cfi().AdjustCFAOffset(kArmWordSize * POPCOUNT(core_spill_mask_)); 2062 __ cfi().RelOffsetForMany(DWARFReg(kMethodRegisterArgument), 0, core_spill_mask_, kArmWordSize); 2063 if (fpu_spill_mask_ != 0) { 2064 SRegister start_register = SRegister(LeastSignificantBit(fpu_spill_mask_)); 2065 __ vpushs(start_register, POPCOUNT(fpu_spill_mask_)); 2066 __ cfi().AdjustCFAOffset(kArmWordSize * POPCOUNT(fpu_spill_mask_)); 2067 __ cfi().RelOffsetForMany(DWARFReg(S0), 0, fpu_spill_mask_, kArmWordSize); 2068 } 2069 2070 if (GetGraph()->HasShouldDeoptimizeFlag()) { 2071 // Initialize should_deoptimize flag to 0. 2072 __ mov(IP, ShifterOperand(0)); 2073 __ StoreToOffset(kStoreWord, IP, SP, -kShouldDeoptimizeFlagSize); 2074 } 2075 2076 int adjust = GetFrameSize() - FrameEntrySpillSize(); 2077 __ AddConstant(SP, -adjust); 2078 __ cfi().AdjustCFAOffset(adjust); 2079 2080 // Save the current method if we need it. Note that we do not 2081 // do this in HCurrentMethod, as the instruction might have been removed 2082 // in the SSA graph. 2083 if (RequiresCurrentMethod()) { 2084 __ StoreToOffset(kStoreWord, kMethodRegisterArgument, SP, 0); 2085 } 2086} 2087 2088void CodeGeneratorARM::GenerateFrameExit() { 2089 if (HasEmptyFrame()) { 2090 __ bx(LR); 2091 return; 2092 } 2093 __ cfi().RememberState(); 2094 int adjust = GetFrameSize() - FrameEntrySpillSize(); 2095 __ AddConstant(SP, adjust); 2096 __ cfi().AdjustCFAOffset(-adjust); 2097 if (fpu_spill_mask_ != 0) { 2098 SRegister start_register = SRegister(LeastSignificantBit(fpu_spill_mask_)); 2099 __ vpops(start_register, POPCOUNT(fpu_spill_mask_)); 2100 __ cfi().AdjustCFAOffset(-static_cast<int>(kArmPointerSize) * POPCOUNT(fpu_spill_mask_)); 2101 __ cfi().RestoreMany(DWARFReg(SRegister(0)), fpu_spill_mask_); 2102 } 2103 // Pop LR into PC to return. 2104 DCHECK_NE(core_spill_mask_ & (1 << LR), 0U); 2105 uint32_t pop_mask = (core_spill_mask_ & (~(1 << LR))) | 1 << PC; 2106 __ PopList(pop_mask); 2107 __ cfi().RestoreState(); 2108 __ cfi().DefCFAOffset(GetFrameSize()); 2109} 2110 2111void CodeGeneratorARM::Bind(HBasicBlock* block) { 2112 Label* label = GetLabelOf(block); 2113 __ BindTrackedLabel(label); 2114} 2115 2116Location InvokeDexCallingConventionVisitorARM::GetNextLocation(Primitive::Type type) { 2117 switch (type) { 2118 case Primitive::kPrimBoolean: 2119 case Primitive::kPrimByte: 2120 case Primitive::kPrimChar: 2121 case Primitive::kPrimShort: 2122 case Primitive::kPrimInt: 2123 case Primitive::kPrimNot: { 2124 uint32_t index = gp_index_++; 2125 uint32_t stack_index = stack_index_++; 2126 if (index < calling_convention.GetNumberOfRegisters()) { 2127 return Location::RegisterLocation(calling_convention.GetRegisterAt(index)); 2128 } else { 2129 return Location::StackSlot(calling_convention.GetStackOffsetOf(stack_index)); 2130 } 2131 } 2132 2133 case Primitive::kPrimLong: { 2134 uint32_t index = gp_index_; 2135 uint32_t stack_index = stack_index_; 2136 gp_index_ += 2; 2137 stack_index_ += 2; 2138 if (index + 1 < calling_convention.GetNumberOfRegisters()) { 2139 if (calling_convention.GetRegisterAt(index) == R1) { 2140 // Skip R1, and use R2_R3 instead. 2141 gp_index_++; 2142 index++; 2143 } 2144 } 2145 if (index + 1 < calling_convention.GetNumberOfRegisters()) { 2146 DCHECK_EQ(calling_convention.GetRegisterAt(index) + 1, 2147 calling_convention.GetRegisterAt(index + 1)); 2148 2149 return Location::RegisterPairLocation(calling_convention.GetRegisterAt(index), 2150 calling_convention.GetRegisterAt(index + 1)); 2151 } else { 2152 return Location::DoubleStackSlot(calling_convention.GetStackOffsetOf(stack_index)); 2153 } 2154 } 2155 2156 case Primitive::kPrimFloat: { 2157 uint32_t stack_index = stack_index_++; 2158 if (float_index_ % 2 == 0) { 2159 float_index_ = std::max(double_index_, float_index_); 2160 } 2161 if (float_index_ < calling_convention.GetNumberOfFpuRegisters()) { 2162 return Location::FpuRegisterLocation(calling_convention.GetFpuRegisterAt(float_index_++)); 2163 } else { 2164 return Location::StackSlot(calling_convention.GetStackOffsetOf(stack_index)); 2165 } 2166 } 2167 2168 case Primitive::kPrimDouble: { 2169 double_index_ = std::max(double_index_, RoundUp(float_index_, 2)); 2170 uint32_t stack_index = stack_index_; 2171 stack_index_ += 2; 2172 if (double_index_ + 1 < calling_convention.GetNumberOfFpuRegisters()) { 2173 uint32_t index = double_index_; 2174 double_index_ += 2; 2175 Location result = Location::FpuRegisterPairLocation( 2176 calling_convention.GetFpuRegisterAt(index), 2177 calling_convention.GetFpuRegisterAt(index + 1)); 2178 DCHECK(ExpectedPairLayout(result)); 2179 return result; 2180 } else { 2181 return Location::DoubleStackSlot(calling_convention.GetStackOffsetOf(stack_index)); 2182 } 2183 } 2184 2185 case Primitive::kPrimVoid: 2186 LOG(FATAL) << "Unexpected parameter type " << type; 2187 break; 2188 } 2189 return Location::NoLocation(); 2190} 2191 2192Location InvokeDexCallingConventionVisitorARM::GetReturnLocation(Primitive::Type type) const { 2193 switch (type) { 2194 case Primitive::kPrimBoolean: 2195 case Primitive::kPrimByte: 2196 case Primitive::kPrimChar: 2197 case Primitive::kPrimShort: 2198 case Primitive::kPrimInt: 2199 case Primitive::kPrimNot: { 2200 return Location::RegisterLocation(R0); 2201 } 2202 2203 case Primitive::kPrimFloat: { 2204 return Location::FpuRegisterLocation(S0); 2205 } 2206 2207 case Primitive::kPrimLong: { 2208 return Location::RegisterPairLocation(R0, R1); 2209 } 2210 2211 case Primitive::kPrimDouble: { 2212 return Location::FpuRegisterPairLocation(S0, S1); 2213 } 2214 2215 case Primitive::kPrimVoid: 2216 return Location::NoLocation(); 2217 } 2218 2219 UNREACHABLE(); 2220} 2221 2222Location InvokeDexCallingConventionVisitorARM::GetMethodLocation() const { 2223 return Location::RegisterLocation(kMethodRegisterArgument); 2224} 2225 2226void CodeGeneratorARM::Move32(Location destination, Location source) { 2227 if (source.Equals(destination)) { 2228 return; 2229 } 2230 if (destination.IsRegister()) { 2231 if (source.IsRegister()) { 2232 __ Mov(destination.AsRegister<Register>(), source.AsRegister<Register>()); 2233 } else if (source.IsFpuRegister()) { 2234 __ vmovrs(destination.AsRegister<Register>(), source.AsFpuRegister<SRegister>()); 2235 } else { 2236 __ LoadFromOffset(kLoadWord, destination.AsRegister<Register>(), SP, source.GetStackIndex()); 2237 } 2238 } else if (destination.IsFpuRegister()) { 2239 if (source.IsRegister()) { 2240 __ vmovsr(destination.AsFpuRegister<SRegister>(), source.AsRegister<Register>()); 2241 } else if (source.IsFpuRegister()) { 2242 __ vmovs(destination.AsFpuRegister<SRegister>(), source.AsFpuRegister<SRegister>()); 2243 } else { 2244 __ LoadSFromOffset(destination.AsFpuRegister<SRegister>(), SP, source.GetStackIndex()); 2245 } 2246 } else { 2247 DCHECK(destination.IsStackSlot()) << destination; 2248 if (source.IsRegister()) { 2249 __ StoreToOffset(kStoreWord, source.AsRegister<Register>(), SP, destination.GetStackIndex()); 2250 } else if (source.IsFpuRegister()) { 2251 __ StoreSToOffset(source.AsFpuRegister<SRegister>(), SP, destination.GetStackIndex()); 2252 } else { 2253 DCHECK(source.IsStackSlot()) << source; 2254 __ LoadFromOffset(kLoadWord, IP, SP, source.GetStackIndex()); 2255 __ StoreToOffset(kStoreWord, IP, SP, destination.GetStackIndex()); 2256 } 2257 } 2258} 2259 2260void CodeGeneratorARM::Move64(Location destination, Location source) { 2261 if (source.Equals(destination)) { 2262 return; 2263 } 2264 if (destination.IsRegisterPair()) { 2265 if (source.IsRegisterPair()) { 2266 EmitParallelMoves( 2267 Location::RegisterLocation(source.AsRegisterPairHigh<Register>()), 2268 Location::RegisterLocation(destination.AsRegisterPairHigh<Register>()), 2269 Primitive::kPrimInt, 2270 Location::RegisterLocation(source.AsRegisterPairLow<Register>()), 2271 Location::RegisterLocation(destination.AsRegisterPairLow<Register>()), 2272 Primitive::kPrimInt); 2273 } else if (source.IsFpuRegister()) { 2274 UNIMPLEMENTED(FATAL); 2275 } else if (source.IsFpuRegisterPair()) { 2276 __ vmovrrd(destination.AsRegisterPairLow<Register>(), 2277 destination.AsRegisterPairHigh<Register>(), 2278 FromLowSToD(source.AsFpuRegisterPairLow<SRegister>())); 2279 } else { 2280 DCHECK(source.IsDoubleStackSlot()); 2281 DCHECK(ExpectedPairLayout(destination)); 2282 __ LoadFromOffset(kLoadWordPair, destination.AsRegisterPairLow<Register>(), 2283 SP, source.GetStackIndex()); 2284 } 2285 } else if (destination.IsFpuRegisterPair()) { 2286 if (source.IsDoubleStackSlot()) { 2287 __ LoadDFromOffset(FromLowSToD(destination.AsFpuRegisterPairLow<SRegister>()), 2288 SP, 2289 source.GetStackIndex()); 2290 } else if (source.IsRegisterPair()) { 2291 __ vmovdrr(FromLowSToD(destination.AsFpuRegisterPairLow<SRegister>()), 2292 source.AsRegisterPairLow<Register>(), 2293 source.AsRegisterPairHigh<Register>()); 2294 } else { 2295 UNIMPLEMENTED(FATAL); 2296 } 2297 } else { 2298 DCHECK(destination.IsDoubleStackSlot()); 2299 if (source.IsRegisterPair()) { 2300 // No conflict possible, so just do the moves. 2301 if (source.AsRegisterPairLow<Register>() == R1) { 2302 DCHECK_EQ(source.AsRegisterPairHigh<Register>(), R2); 2303 __ StoreToOffset(kStoreWord, R1, SP, destination.GetStackIndex()); 2304 __ StoreToOffset(kStoreWord, R2, SP, destination.GetHighStackIndex(kArmWordSize)); 2305 } else { 2306 __ StoreToOffset(kStoreWordPair, source.AsRegisterPairLow<Register>(), 2307 SP, destination.GetStackIndex()); 2308 } 2309 } else if (source.IsFpuRegisterPair()) { 2310 __ StoreDToOffset(FromLowSToD(source.AsFpuRegisterPairLow<SRegister>()), 2311 SP, 2312 destination.GetStackIndex()); 2313 } else { 2314 DCHECK(source.IsDoubleStackSlot()); 2315 EmitParallelMoves( 2316 Location::StackSlot(source.GetStackIndex()), 2317 Location::StackSlot(destination.GetStackIndex()), 2318 Primitive::kPrimInt, 2319 Location::StackSlot(source.GetHighStackIndex(kArmWordSize)), 2320 Location::StackSlot(destination.GetHighStackIndex(kArmWordSize)), 2321 Primitive::kPrimInt); 2322 } 2323 } 2324} 2325 2326void CodeGeneratorARM::MoveConstant(Location location, int32_t value) { 2327 DCHECK(location.IsRegister()); 2328 __ LoadImmediate(location.AsRegister<Register>(), value); 2329} 2330 2331void CodeGeneratorARM::MoveLocation(Location dst, Location src, Primitive::Type dst_type) { 2332 HParallelMove move(GetGraph()->GetArena()); 2333 move.AddMove(src, dst, dst_type, nullptr); 2334 GetMoveResolver()->EmitNativeCode(&move); 2335} 2336 2337void CodeGeneratorARM::AddLocationAsTemp(Location location, LocationSummary* locations) { 2338 if (location.IsRegister()) { 2339 locations->AddTemp(location); 2340 } else if (location.IsRegisterPair()) { 2341 locations->AddTemp(Location::RegisterLocation(location.AsRegisterPairLow<Register>())); 2342 locations->AddTemp(Location::RegisterLocation(location.AsRegisterPairHigh<Register>())); 2343 } else { 2344 UNIMPLEMENTED(FATAL) << "AddLocationAsTemp not implemented for location " << location; 2345 } 2346} 2347 2348void CodeGeneratorARM::InvokeRuntime(QuickEntrypointEnum entrypoint, 2349 HInstruction* instruction, 2350 uint32_t dex_pc, 2351 SlowPathCode* slow_path) { 2352 ValidateInvokeRuntime(entrypoint, instruction, slow_path); 2353 GenerateInvokeRuntime(GetThreadOffset<kArmPointerSize>(entrypoint).Int32Value()); 2354 if (EntrypointRequiresStackMap(entrypoint)) { 2355 RecordPcInfo(instruction, dex_pc, slow_path); 2356 } 2357} 2358 2359void CodeGeneratorARM::InvokeRuntimeWithoutRecordingPcInfo(int32_t entry_point_offset, 2360 HInstruction* instruction, 2361 SlowPathCode* slow_path) { 2362 ValidateInvokeRuntimeWithoutRecordingPcInfo(instruction, slow_path); 2363 GenerateInvokeRuntime(entry_point_offset); 2364} 2365 2366void CodeGeneratorARM::GenerateInvokeRuntime(int32_t entry_point_offset) { 2367 __ LoadFromOffset(kLoadWord, LR, TR, entry_point_offset); 2368 __ blx(LR); 2369} 2370 2371void InstructionCodeGeneratorARM::HandleGoto(HInstruction* got, HBasicBlock* successor) { 2372 DCHECK(!successor->IsExitBlock()); 2373 2374 HBasicBlock* block = got->GetBlock(); 2375 HInstruction* previous = got->GetPrevious(); 2376 2377 HLoopInformation* info = block->GetLoopInformation(); 2378 if (info != nullptr && info->IsBackEdge(*block) && info->HasSuspendCheck()) { 2379 codegen_->ClearSpillSlotsFromLoopPhisInStackMap(info->GetSuspendCheck()); 2380 GenerateSuspendCheck(info->GetSuspendCheck(), successor); 2381 return; 2382 } 2383 2384 if (block->IsEntryBlock() && (previous != nullptr) && previous->IsSuspendCheck()) { 2385 GenerateSuspendCheck(previous->AsSuspendCheck(), nullptr); 2386 } 2387 if (!codegen_->GoesToNextBlock(got->GetBlock(), successor)) { 2388 __ b(codegen_->GetLabelOf(successor)); 2389 } 2390} 2391 2392void LocationsBuilderARM::VisitGoto(HGoto* got) { 2393 got->SetLocations(nullptr); 2394} 2395 2396void InstructionCodeGeneratorARM::VisitGoto(HGoto* got) { 2397 HandleGoto(got, got->GetSuccessor()); 2398} 2399 2400void LocationsBuilderARM::VisitTryBoundary(HTryBoundary* try_boundary) { 2401 try_boundary->SetLocations(nullptr); 2402} 2403 2404void InstructionCodeGeneratorARM::VisitTryBoundary(HTryBoundary* try_boundary) { 2405 HBasicBlock* successor = try_boundary->GetNormalFlowSuccessor(); 2406 if (!successor->IsExitBlock()) { 2407 HandleGoto(try_boundary, successor); 2408 } 2409} 2410 2411void LocationsBuilderARM::VisitExit(HExit* exit) { 2412 exit->SetLocations(nullptr); 2413} 2414 2415void InstructionCodeGeneratorARM::VisitExit(HExit* exit ATTRIBUTE_UNUSED) { 2416} 2417 2418void InstructionCodeGeneratorARM::GenerateFPJumps(HCondition* cond, 2419 Label* true_label, 2420 Label* false_label ATTRIBUTE_UNUSED) { 2421 __ vmstat(); // transfer FP status register to ARM APSR. 2422 __ b(true_label, ARMFPCondition(cond->GetCondition(), cond->IsGtBias())); 2423} 2424 2425void InstructionCodeGeneratorARM::GenerateLongComparesAndJumps(HCondition* cond, 2426 Label* true_label, 2427 Label* false_label) { 2428 LocationSummary* locations = cond->GetLocations(); 2429 Location left = locations->InAt(0); 2430 Location right = locations->InAt(1); 2431 IfCondition if_cond = cond->GetCondition(); 2432 2433 Register left_high = left.AsRegisterPairHigh<Register>(); 2434 Register left_low = left.AsRegisterPairLow<Register>(); 2435 IfCondition true_high_cond = if_cond; 2436 IfCondition false_high_cond = cond->GetOppositeCondition(); 2437 Condition final_condition = ARMUnsignedCondition(if_cond); // unsigned on lower part 2438 2439 // Set the conditions for the test, remembering that == needs to be 2440 // decided using the low words. 2441 // TODO: consider avoiding jumps with temporary and CMP low+SBC high 2442 switch (if_cond) { 2443 case kCondEQ: 2444 case kCondNE: 2445 // Nothing to do. 2446 break; 2447 case kCondLT: 2448 false_high_cond = kCondGT; 2449 break; 2450 case kCondLE: 2451 true_high_cond = kCondLT; 2452 break; 2453 case kCondGT: 2454 false_high_cond = kCondLT; 2455 break; 2456 case kCondGE: 2457 true_high_cond = kCondGT; 2458 break; 2459 case kCondB: 2460 false_high_cond = kCondA; 2461 break; 2462 case kCondBE: 2463 true_high_cond = kCondB; 2464 break; 2465 case kCondA: 2466 false_high_cond = kCondB; 2467 break; 2468 case kCondAE: 2469 true_high_cond = kCondA; 2470 break; 2471 } 2472 if (right.IsConstant()) { 2473 int64_t value = right.GetConstant()->AsLongConstant()->GetValue(); 2474 int32_t val_low = Low32Bits(value); 2475 int32_t val_high = High32Bits(value); 2476 2477 __ CmpConstant(left_high, val_high); 2478 if (if_cond == kCondNE) { 2479 __ b(true_label, ARMCondition(true_high_cond)); 2480 } else if (if_cond == kCondEQ) { 2481 __ b(false_label, ARMCondition(false_high_cond)); 2482 } else { 2483 __ b(true_label, ARMCondition(true_high_cond)); 2484 __ b(false_label, ARMCondition(false_high_cond)); 2485 } 2486 // Must be equal high, so compare the lows. 2487 __ CmpConstant(left_low, val_low); 2488 } else { 2489 Register right_high = right.AsRegisterPairHigh<Register>(); 2490 Register right_low = right.AsRegisterPairLow<Register>(); 2491 2492 __ cmp(left_high, ShifterOperand(right_high)); 2493 if (if_cond == kCondNE) { 2494 __ b(true_label, ARMCondition(true_high_cond)); 2495 } else if (if_cond == kCondEQ) { 2496 __ b(false_label, ARMCondition(false_high_cond)); 2497 } else { 2498 __ b(true_label, ARMCondition(true_high_cond)); 2499 __ b(false_label, ARMCondition(false_high_cond)); 2500 } 2501 // Must be equal high, so compare the lows. 2502 __ cmp(left_low, ShifterOperand(right_low)); 2503 } 2504 // The last comparison might be unsigned. 2505 // TODO: optimize cases where this is always true/false 2506 __ b(true_label, final_condition); 2507} 2508 2509void InstructionCodeGeneratorARM::GenerateCompareTestAndBranch(HCondition* condition, 2510 Label* true_target_in, 2511 Label* false_target_in) { 2512 // Generated branching requires both targets to be explicit. If either of the 2513 // targets is nullptr (fallthrough) use and bind `fallthrough_target` instead. 2514 Label fallthrough_target; 2515 Label* true_target = true_target_in == nullptr ? &fallthrough_target : true_target_in; 2516 Label* false_target = false_target_in == nullptr ? &fallthrough_target : false_target_in; 2517 2518 Primitive::Type type = condition->InputAt(0)->GetType(); 2519 switch (type) { 2520 case Primitive::kPrimLong: 2521 GenerateLongComparesAndJumps(condition, true_target, false_target); 2522 break; 2523 case Primitive::kPrimFloat: 2524 case Primitive::kPrimDouble: 2525 GenerateVcmp(condition, codegen_); 2526 GenerateFPJumps(condition, true_target, false_target); 2527 break; 2528 default: 2529 LOG(FATAL) << "Unexpected compare type " << type; 2530 } 2531 2532 if (false_target != &fallthrough_target) { 2533 __ b(false_target); 2534 } 2535 2536 if (fallthrough_target.IsLinked()) { 2537 __ Bind(&fallthrough_target); 2538 } 2539} 2540 2541void InstructionCodeGeneratorARM::GenerateTestAndBranch(HInstruction* instruction, 2542 size_t condition_input_index, 2543 Label* true_target, 2544 Label* false_target) { 2545 HInstruction* cond = instruction->InputAt(condition_input_index); 2546 2547 if (true_target == nullptr && false_target == nullptr) { 2548 // Nothing to do. The code always falls through. 2549 return; 2550 } else if (cond->IsIntConstant()) { 2551 // Constant condition, statically compared against "true" (integer value 1). 2552 if (cond->AsIntConstant()->IsTrue()) { 2553 if (true_target != nullptr) { 2554 __ b(true_target); 2555 } 2556 } else { 2557 DCHECK(cond->AsIntConstant()->IsFalse()) << cond->AsIntConstant()->GetValue(); 2558 if (false_target != nullptr) { 2559 __ b(false_target); 2560 } 2561 } 2562 return; 2563 } 2564 2565 // The following code generates these patterns: 2566 // (1) true_target == nullptr && false_target != nullptr 2567 // - opposite condition true => branch to false_target 2568 // (2) true_target != nullptr && false_target == nullptr 2569 // - condition true => branch to true_target 2570 // (3) true_target != nullptr && false_target != nullptr 2571 // - condition true => branch to true_target 2572 // - branch to false_target 2573 if (IsBooleanValueOrMaterializedCondition(cond)) { 2574 // Condition has been materialized, compare the output to 0. 2575 Location cond_val = instruction->GetLocations()->InAt(condition_input_index); 2576 DCHECK(cond_val.IsRegister()); 2577 if (true_target == nullptr) { 2578 __ CompareAndBranchIfZero(cond_val.AsRegister<Register>(), false_target); 2579 } else { 2580 __ CompareAndBranchIfNonZero(cond_val.AsRegister<Register>(), true_target); 2581 } 2582 } else { 2583 // Condition has not been materialized. Use its inputs as the comparison and 2584 // its condition as the branch condition. 2585 HCondition* condition = cond->AsCondition(); 2586 2587 // If this is a long or FP comparison that has been folded into 2588 // the HCondition, generate the comparison directly. 2589 Primitive::Type type = condition->InputAt(0)->GetType(); 2590 if (type == Primitive::kPrimLong || Primitive::IsFloatingPointType(type)) { 2591 GenerateCompareTestAndBranch(condition, true_target, false_target); 2592 return; 2593 } 2594 2595 Label* non_fallthrough_target; 2596 Condition arm_cond; 2597 LocationSummary* locations = cond->GetLocations(); 2598 DCHECK(locations->InAt(0).IsRegister()); 2599 Register left = locations->InAt(0).AsRegister<Register>(); 2600 Location right = locations->InAt(1); 2601 2602 if (true_target == nullptr) { 2603 arm_cond = ARMCondition(condition->GetOppositeCondition()); 2604 non_fallthrough_target = false_target; 2605 } else { 2606 arm_cond = ARMCondition(condition->GetCondition()); 2607 non_fallthrough_target = true_target; 2608 } 2609 2610 if (right.IsConstant() && (arm_cond == NE || arm_cond == EQ) && 2611 CodeGenerator::GetInt32ValueOf(right.GetConstant()) == 0) { 2612 if (arm_cond == EQ) { 2613 __ CompareAndBranchIfZero(left, non_fallthrough_target); 2614 } else { 2615 DCHECK_EQ(arm_cond, NE); 2616 __ CompareAndBranchIfNonZero(left, non_fallthrough_target); 2617 } 2618 } else { 2619 if (right.IsRegister()) { 2620 __ cmp(left, ShifterOperand(right.AsRegister<Register>())); 2621 } else { 2622 DCHECK(right.IsConstant()); 2623 __ CmpConstant(left, CodeGenerator::GetInt32ValueOf(right.GetConstant())); 2624 } 2625 2626 __ b(non_fallthrough_target, arm_cond); 2627 } 2628 } 2629 2630 // If neither branch falls through (case 3), the conditional branch to `true_target` 2631 // was already emitted (case 2) and we need to emit a jump to `false_target`. 2632 if (true_target != nullptr && false_target != nullptr) { 2633 __ b(false_target); 2634 } 2635} 2636 2637void LocationsBuilderARM::VisitIf(HIf* if_instr) { 2638 LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(if_instr); 2639 if (IsBooleanValueOrMaterializedCondition(if_instr->InputAt(0))) { 2640 locations->SetInAt(0, Location::RequiresRegister()); 2641 } 2642} 2643 2644void InstructionCodeGeneratorARM::VisitIf(HIf* if_instr) { 2645 HBasicBlock* true_successor = if_instr->IfTrueSuccessor(); 2646 HBasicBlock* false_successor = if_instr->IfFalseSuccessor(); 2647 Label* true_target = codegen_->GoesToNextBlock(if_instr->GetBlock(), true_successor) ? 2648 nullptr : codegen_->GetLabelOf(true_successor); 2649 Label* false_target = codegen_->GoesToNextBlock(if_instr->GetBlock(), false_successor) ? 2650 nullptr : codegen_->GetLabelOf(false_successor); 2651 GenerateTestAndBranch(if_instr, /* condition_input_index */ 0, true_target, false_target); 2652} 2653 2654void LocationsBuilderARM::VisitDeoptimize(HDeoptimize* deoptimize) { 2655 LocationSummary* locations = new (GetGraph()->GetArena()) 2656 LocationSummary(deoptimize, LocationSummary::kCallOnSlowPath); 2657 locations->SetCustomSlowPathCallerSaves(RegisterSet::Empty()); // No caller-save registers. 2658 if (IsBooleanValueOrMaterializedCondition(deoptimize->InputAt(0))) { 2659 locations->SetInAt(0, Location::RequiresRegister()); 2660 } 2661} 2662 2663void InstructionCodeGeneratorARM::VisitDeoptimize(HDeoptimize* deoptimize) { 2664 SlowPathCodeARM* slow_path = deopt_slow_paths_.NewSlowPath<DeoptimizationSlowPathARM>(deoptimize); 2665 GenerateTestAndBranch(deoptimize, 2666 /* condition_input_index */ 0, 2667 slow_path->GetEntryLabel(), 2668 /* false_target */ nullptr); 2669} 2670 2671void LocationsBuilderARM::VisitShouldDeoptimizeFlag(HShouldDeoptimizeFlag* flag) { 2672 LocationSummary* locations = new (GetGraph()->GetArena()) 2673 LocationSummary(flag, LocationSummary::kNoCall); 2674 locations->SetOut(Location::RequiresRegister()); 2675} 2676 2677void InstructionCodeGeneratorARM::VisitShouldDeoptimizeFlag(HShouldDeoptimizeFlag* flag) { 2678 __ LoadFromOffset(kLoadWord, 2679 flag->GetLocations()->Out().AsRegister<Register>(), 2680 SP, 2681 codegen_->GetStackOffsetOfShouldDeoptimizeFlag()); 2682} 2683 2684void LocationsBuilderARM::VisitSelect(HSelect* select) { 2685 LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(select); 2686 const bool is_floating_point = Primitive::IsFloatingPointType(select->GetType()); 2687 2688 if (is_floating_point) { 2689 locations->SetInAt(0, Location::RequiresFpuRegister()); 2690 locations->SetInAt(1, Location::FpuRegisterOrConstant(select->GetTrueValue())); 2691 } else { 2692 locations->SetInAt(0, Location::RequiresRegister()); 2693 locations->SetInAt(1, Arm8BitEncodableConstantOrRegister(select->GetTrueValue())); 2694 } 2695 2696 if (IsBooleanValueOrMaterializedCondition(select->GetCondition())) { 2697 locations->SetInAt(2, Location::RegisterOrConstant(select->GetCondition())); 2698 // The code generator handles overlap with the values, but not with the condition. 2699 locations->SetOut(Location::SameAsFirstInput()); 2700 } else if (is_floating_point) { 2701 locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap); 2702 } else { 2703 if (!locations->InAt(1).IsConstant()) { 2704 locations->SetInAt(0, Arm8BitEncodableConstantOrRegister(select->GetFalseValue())); 2705 } 2706 2707 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 2708 } 2709} 2710 2711void InstructionCodeGeneratorARM::VisitSelect(HSelect* select) { 2712 HInstruction* const condition = select->GetCondition(); 2713 const LocationSummary* const locations = select->GetLocations(); 2714 const Primitive::Type type = select->GetType(); 2715 const Location first = locations->InAt(0); 2716 const Location out = locations->Out(); 2717 const Location second = locations->InAt(1); 2718 Location src; 2719 2720 if (condition->IsIntConstant()) { 2721 if (condition->AsIntConstant()->IsFalse()) { 2722 src = first; 2723 } else { 2724 src = second; 2725 } 2726 2727 codegen_->MoveLocation(out, src, type); 2728 return; 2729 } 2730 2731 if (!Primitive::IsFloatingPointType(type) && 2732 CanGenerateTest(condition, codegen_->GetAssembler())) { 2733 bool invert = false; 2734 2735 if (out.Equals(second)) { 2736 src = first; 2737 invert = true; 2738 } else if (out.Equals(first)) { 2739 src = second; 2740 } else if (second.IsConstant()) { 2741 DCHECK(CanEncodeConstantAs8BitImmediate(second.GetConstant())); 2742 src = second; 2743 } else if (first.IsConstant()) { 2744 DCHECK(CanEncodeConstantAs8BitImmediate(first.GetConstant())); 2745 src = first; 2746 invert = true; 2747 } else { 2748 src = second; 2749 } 2750 2751 if (CanGenerateConditionalMove(out, src)) { 2752 if (!out.Equals(first) && !out.Equals(second)) { 2753 codegen_->MoveLocation(out, src.Equals(first) ? second : first, type); 2754 } 2755 2756 const Condition cond = GenerateTest(condition, locations->InAt(2), invert, codegen_); 2757 2758 if (out.IsRegister()) { 2759 ShifterOperand operand; 2760 2761 if (src.IsConstant()) { 2762 operand = ShifterOperand(CodeGenerator::GetInt32ValueOf(src.GetConstant())); 2763 } else { 2764 DCHECK(src.IsRegister()); 2765 operand = ShifterOperand(src.AsRegister<Register>()); 2766 } 2767 2768 __ it(cond); 2769 __ mov(out.AsRegister<Register>(), operand, cond); 2770 } else { 2771 DCHECK(out.IsRegisterPair()); 2772 2773 ShifterOperand operand_high; 2774 ShifterOperand operand_low; 2775 2776 if (src.IsConstant()) { 2777 const int64_t value = src.GetConstant()->AsLongConstant()->GetValue(); 2778 2779 operand_high = ShifterOperand(High32Bits(value)); 2780 operand_low = ShifterOperand(Low32Bits(value)); 2781 } else { 2782 DCHECK(src.IsRegisterPair()); 2783 operand_high = ShifterOperand(src.AsRegisterPairHigh<Register>()); 2784 operand_low = ShifterOperand(src.AsRegisterPairLow<Register>()); 2785 } 2786 2787 __ it(cond); 2788 __ mov(out.AsRegisterPairLow<Register>(), operand_low, cond); 2789 __ it(cond); 2790 __ mov(out.AsRegisterPairHigh<Register>(), operand_high, cond); 2791 } 2792 2793 return; 2794 } 2795 } 2796 2797 Label* false_target = nullptr; 2798 Label* true_target = nullptr; 2799 Label select_end; 2800 Label* target = codegen_->GetFinalLabel(select, &select_end); 2801 2802 if (out.Equals(second)) { 2803 true_target = target; 2804 src = first; 2805 } else { 2806 false_target = target; 2807 src = second; 2808 2809 if (!out.Equals(first)) { 2810 codegen_->MoveLocation(out, first, type); 2811 } 2812 } 2813 2814 GenerateTestAndBranch(select, 2, true_target, false_target); 2815 codegen_->MoveLocation(out, src, type); 2816 2817 if (select_end.IsLinked()) { 2818 __ Bind(&select_end); 2819 } 2820} 2821 2822void LocationsBuilderARM::VisitNativeDebugInfo(HNativeDebugInfo* info) { 2823 new (GetGraph()->GetArena()) LocationSummary(info); 2824} 2825 2826void InstructionCodeGeneratorARM::VisitNativeDebugInfo(HNativeDebugInfo*) { 2827 // MaybeRecordNativeDebugInfo is already called implicitly in CodeGenerator::Compile. 2828} 2829 2830void CodeGeneratorARM::GenerateNop() { 2831 __ nop(); 2832} 2833 2834void LocationsBuilderARM::HandleCondition(HCondition* cond) { 2835 LocationSummary* locations = 2836 new (GetGraph()->GetArena()) LocationSummary(cond, LocationSummary::kNoCall); 2837 // Handle the long/FP comparisons made in instruction simplification. 2838 switch (cond->InputAt(0)->GetType()) { 2839 case Primitive::kPrimLong: 2840 locations->SetInAt(0, Location::RequiresRegister()); 2841 locations->SetInAt(1, Location::RegisterOrConstant(cond->InputAt(1))); 2842 if (!cond->IsEmittedAtUseSite()) { 2843 locations->SetOut(Location::RequiresRegister(), Location::kOutputOverlap); 2844 } 2845 break; 2846 2847 case Primitive::kPrimFloat: 2848 case Primitive::kPrimDouble: 2849 locations->SetInAt(0, Location::RequiresFpuRegister()); 2850 locations->SetInAt(1, ArithmeticZeroOrFpuRegister(cond->InputAt(1))); 2851 if (!cond->IsEmittedAtUseSite()) { 2852 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 2853 } 2854 break; 2855 2856 default: 2857 locations->SetInAt(0, Location::RequiresRegister()); 2858 locations->SetInAt(1, Location::RegisterOrConstant(cond->InputAt(1))); 2859 if (!cond->IsEmittedAtUseSite()) { 2860 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 2861 } 2862 } 2863} 2864 2865void InstructionCodeGeneratorARM::HandleCondition(HCondition* cond) { 2866 if (cond->IsEmittedAtUseSite()) { 2867 return; 2868 } 2869 2870 LocationSummary* locations = cond->GetLocations(); 2871 Location left = locations->InAt(0); 2872 Location right = locations->InAt(1); 2873 Register out = locations->Out().AsRegister<Register>(); 2874 Label true_label, false_label; 2875 2876 switch (cond->InputAt(0)->GetType()) { 2877 default: { 2878 // Integer case. 2879 if (right.IsRegister()) { 2880 __ cmp(left.AsRegister<Register>(), ShifterOperand(right.AsRegister<Register>())); 2881 } else { 2882 DCHECK(right.IsConstant()); 2883 __ CmpConstant(left.AsRegister<Register>(), 2884 CodeGenerator::GetInt32ValueOf(right.GetConstant())); 2885 } 2886 __ it(ARMCondition(cond->GetCondition()), kItElse); 2887 __ mov(locations->Out().AsRegister<Register>(), ShifterOperand(1), 2888 ARMCondition(cond->GetCondition())); 2889 __ mov(locations->Out().AsRegister<Register>(), ShifterOperand(0), 2890 ARMCondition(cond->GetOppositeCondition())); 2891 return; 2892 } 2893 case Primitive::kPrimLong: 2894 GenerateLongComparesAndJumps(cond, &true_label, &false_label); 2895 break; 2896 case Primitive::kPrimFloat: 2897 case Primitive::kPrimDouble: 2898 GenerateVcmp(cond, codegen_); 2899 GenerateFPJumps(cond, &true_label, &false_label); 2900 break; 2901 } 2902 2903 // Convert the jumps into the result. 2904 Label done_label; 2905 Label* final_label = codegen_->GetFinalLabel(cond, &done_label); 2906 2907 // False case: result = 0. 2908 __ Bind(&false_label); 2909 __ LoadImmediate(out, 0); 2910 __ b(final_label); 2911 2912 // True case: result = 1. 2913 __ Bind(&true_label); 2914 __ LoadImmediate(out, 1); 2915 2916 if (done_label.IsLinked()) { 2917 __ Bind(&done_label); 2918 } 2919} 2920 2921void LocationsBuilderARM::VisitEqual(HEqual* comp) { 2922 HandleCondition(comp); 2923} 2924 2925void InstructionCodeGeneratorARM::VisitEqual(HEqual* comp) { 2926 HandleCondition(comp); 2927} 2928 2929void LocationsBuilderARM::VisitNotEqual(HNotEqual* comp) { 2930 HandleCondition(comp); 2931} 2932 2933void InstructionCodeGeneratorARM::VisitNotEqual(HNotEqual* comp) { 2934 HandleCondition(comp); 2935} 2936 2937void LocationsBuilderARM::VisitLessThan(HLessThan* comp) { 2938 HandleCondition(comp); 2939} 2940 2941void InstructionCodeGeneratorARM::VisitLessThan(HLessThan* comp) { 2942 HandleCondition(comp); 2943} 2944 2945void LocationsBuilderARM::VisitLessThanOrEqual(HLessThanOrEqual* comp) { 2946 HandleCondition(comp); 2947} 2948 2949void InstructionCodeGeneratorARM::VisitLessThanOrEqual(HLessThanOrEqual* comp) { 2950 HandleCondition(comp); 2951} 2952 2953void LocationsBuilderARM::VisitGreaterThan(HGreaterThan* comp) { 2954 HandleCondition(comp); 2955} 2956 2957void InstructionCodeGeneratorARM::VisitGreaterThan(HGreaterThan* comp) { 2958 HandleCondition(comp); 2959} 2960 2961void LocationsBuilderARM::VisitGreaterThanOrEqual(HGreaterThanOrEqual* comp) { 2962 HandleCondition(comp); 2963} 2964 2965void InstructionCodeGeneratorARM::VisitGreaterThanOrEqual(HGreaterThanOrEqual* comp) { 2966 HandleCondition(comp); 2967} 2968 2969void LocationsBuilderARM::VisitBelow(HBelow* comp) { 2970 HandleCondition(comp); 2971} 2972 2973void InstructionCodeGeneratorARM::VisitBelow(HBelow* comp) { 2974 HandleCondition(comp); 2975} 2976 2977void LocationsBuilderARM::VisitBelowOrEqual(HBelowOrEqual* comp) { 2978 HandleCondition(comp); 2979} 2980 2981void InstructionCodeGeneratorARM::VisitBelowOrEqual(HBelowOrEqual* comp) { 2982 HandleCondition(comp); 2983} 2984 2985void LocationsBuilderARM::VisitAbove(HAbove* comp) { 2986 HandleCondition(comp); 2987} 2988 2989void InstructionCodeGeneratorARM::VisitAbove(HAbove* comp) { 2990 HandleCondition(comp); 2991} 2992 2993void LocationsBuilderARM::VisitAboveOrEqual(HAboveOrEqual* comp) { 2994 HandleCondition(comp); 2995} 2996 2997void InstructionCodeGeneratorARM::VisitAboveOrEqual(HAboveOrEqual* comp) { 2998 HandleCondition(comp); 2999} 3000 3001void LocationsBuilderARM::VisitIntConstant(HIntConstant* constant) { 3002 LocationSummary* locations = 3003 new (GetGraph()->GetArena()) LocationSummary(constant, LocationSummary::kNoCall); 3004 locations->SetOut(Location::ConstantLocation(constant)); 3005} 3006 3007void InstructionCodeGeneratorARM::VisitIntConstant(HIntConstant* constant ATTRIBUTE_UNUSED) { 3008 // Will be generated at use site. 3009} 3010 3011void LocationsBuilderARM::VisitNullConstant(HNullConstant* constant) { 3012 LocationSummary* locations = 3013 new (GetGraph()->GetArena()) LocationSummary(constant, LocationSummary::kNoCall); 3014 locations->SetOut(Location::ConstantLocation(constant)); 3015} 3016 3017void InstructionCodeGeneratorARM::VisitNullConstant(HNullConstant* constant ATTRIBUTE_UNUSED) { 3018 // Will be generated at use site. 3019} 3020 3021void LocationsBuilderARM::VisitLongConstant(HLongConstant* constant) { 3022 LocationSummary* locations = 3023 new (GetGraph()->GetArena()) LocationSummary(constant, LocationSummary::kNoCall); 3024 locations->SetOut(Location::ConstantLocation(constant)); 3025} 3026 3027void InstructionCodeGeneratorARM::VisitLongConstant(HLongConstant* constant ATTRIBUTE_UNUSED) { 3028 // Will be generated at use site. 3029} 3030 3031void LocationsBuilderARM::VisitFloatConstant(HFloatConstant* constant) { 3032 LocationSummary* locations = 3033 new (GetGraph()->GetArena()) LocationSummary(constant, LocationSummary::kNoCall); 3034 locations->SetOut(Location::ConstantLocation(constant)); 3035} 3036 3037void InstructionCodeGeneratorARM::VisitFloatConstant(HFloatConstant* constant ATTRIBUTE_UNUSED) { 3038 // Will be generated at use site. 3039} 3040 3041void LocationsBuilderARM::VisitDoubleConstant(HDoubleConstant* constant) { 3042 LocationSummary* locations = 3043 new (GetGraph()->GetArena()) LocationSummary(constant, LocationSummary::kNoCall); 3044 locations->SetOut(Location::ConstantLocation(constant)); 3045} 3046 3047void InstructionCodeGeneratorARM::VisitDoubleConstant(HDoubleConstant* constant ATTRIBUTE_UNUSED) { 3048 // Will be generated at use site. 3049} 3050 3051void LocationsBuilderARM::VisitMemoryBarrier(HMemoryBarrier* memory_barrier) { 3052 memory_barrier->SetLocations(nullptr); 3053} 3054 3055void InstructionCodeGeneratorARM::VisitMemoryBarrier(HMemoryBarrier* memory_barrier) { 3056 codegen_->GenerateMemoryBarrier(memory_barrier->GetBarrierKind()); 3057} 3058 3059void LocationsBuilderARM::VisitReturnVoid(HReturnVoid* ret) { 3060 ret->SetLocations(nullptr); 3061} 3062 3063void InstructionCodeGeneratorARM::VisitReturnVoid(HReturnVoid* ret ATTRIBUTE_UNUSED) { 3064 codegen_->GenerateFrameExit(); 3065} 3066 3067void LocationsBuilderARM::VisitReturn(HReturn* ret) { 3068 LocationSummary* locations = 3069 new (GetGraph()->GetArena()) LocationSummary(ret, LocationSummary::kNoCall); 3070 locations->SetInAt(0, parameter_visitor_.GetReturnLocation(ret->InputAt(0)->GetType())); 3071} 3072 3073void InstructionCodeGeneratorARM::VisitReturn(HReturn* ret ATTRIBUTE_UNUSED) { 3074 codegen_->GenerateFrameExit(); 3075} 3076 3077void LocationsBuilderARM::VisitInvokeUnresolved(HInvokeUnresolved* invoke) { 3078 // The trampoline uses the same calling convention as dex calling conventions, 3079 // except instead of loading arg0/r0 with the target Method*, arg0/r0 will contain 3080 // the method_idx. 3081 HandleInvoke(invoke); 3082} 3083 3084void InstructionCodeGeneratorARM::VisitInvokeUnresolved(HInvokeUnresolved* invoke) { 3085 codegen_->GenerateInvokeUnresolvedRuntimeCall(invoke); 3086} 3087 3088void LocationsBuilderARM::VisitInvokeStaticOrDirect(HInvokeStaticOrDirect* invoke) { 3089 // Explicit clinit checks triggered by static invokes must have been pruned by 3090 // art::PrepareForRegisterAllocation. 3091 DCHECK(!invoke->IsStaticWithExplicitClinitCheck()); 3092 3093 IntrinsicLocationsBuilderARM intrinsic(codegen_); 3094 if (intrinsic.TryDispatch(invoke)) { 3095 if (invoke->GetLocations()->CanCall() && invoke->HasPcRelativeDexCache()) { 3096 invoke->GetLocations()->SetInAt(invoke->GetSpecialInputIndex(), Location::Any()); 3097 } 3098 return; 3099 } 3100 3101 HandleInvoke(invoke); 3102 3103 // For PC-relative dex cache the invoke has an extra input, the PC-relative address base. 3104 if (invoke->HasPcRelativeDexCache()) { 3105 invoke->GetLocations()->SetInAt(invoke->GetSpecialInputIndex(), Location::RequiresRegister()); 3106 } 3107} 3108 3109static bool TryGenerateIntrinsicCode(HInvoke* invoke, CodeGeneratorARM* codegen) { 3110 if (invoke->GetLocations()->Intrinsified()) { 3111 IntrinsicCodeGeneratorARM intrinsic(codegen); 3112 intrinsic.Dispatch(invoke); 3113 return true; 3114 } 3115 return false; 3116} 3117 3118void InstructionCodeGeneratorARM::VisitInvokeStaticOrDirect(HInvokeStaticOrDirect* invoke) { 3119 // Explicit clinit checks triggered by static invokes must have been pruned by 3120 // art::PrepareForRegisterAllocation. 3121 DCHECK(!invoke->IsStaticWithExplicitClinitCheck()); 3122 3123 if (TryGenerateIntrinsicCode(invoke, codegen_)) { 3124 return; 3125 } 3126 3127 LocationSummary* locations = invoke->GetLocations(); 3128 codegen_->GenerateStaticOrDirectCall( 3129 invoke, locations->HasTemps() ? locations->GetTemp(0) : Location::NoLocation()); 3130 codegen_->RecordPcInfo(invoke, invoke->GetDexPc()); 3131} 3132 3133void LocationsBuilderARM::HandleInvoke(HInvoke* invoke) { 3134 InvokeDexCallingConventionVisitorARM calling_convention_visitor; 3135 CodeGenerator::CreateCommonInvokeLocationSummary(invoke, &calling_convention_visitor); 3136} 3137 3138void LocationsBuilderARM::VisitInvokeVirtual(HInvokeVirtual* invoke) { 3139 IntrinsicLocationsBuilderARM intrinsic(codegen_); 3140 if (intrinsic.TryDispatch(invoke)) { 3141 return; 3142 } 3143 3144 HandleInvoke(invoke); 3145} 3146 3147void InstructionCodeGeneratorARM::VisitInvokeVirtual(HInvokeVirtual* invoke) { 3148 if (TryGenerateIntrinsicCode(invoke, codegen_)) { 3149 return; 3150 } 3151 3152 codegen_->GenerateVirtualCall(invoke, invoke->GetLocations()->GetTemp(0)); 3153 DCHECK(!codegen_->IsLeafMethod()); 3154 codegen_->RecordPcInfo(invoke, invoke->GetDexPc()); 3155} 3156 3157void LocationsBuilderARM::VisitInvokeInterface(HInvokeInterface* invoke) { 3158 HandleInvoke(invoke); 3159 // Add the hidden argument. 3160 invoke->GetLocations()->AddTemp(Location::RegisterLocation(R12)); 3161} 3162 3163void InstructionCodeGeneratorARM::VisitInvokeInterface(HInvokeInterface* invoke) { 3164 // TODO: b/18116999, our IMTs can miss an IncompatibleClassChangeError. 3165 LocationSummary* locations = invoke->GetLocations(); 3166 Register temp = locations->GetTemp(0).AsRegister<Register>(); 3167 Register hidden_reg = locations->GetTemp(1).AsRegister<Register>(); 3168 Location receiver = locations->InAt(0); 3169 uint32_t class_offset = mirror::Object::ClassOffset().Int32Value(); 3170 3171 // Set the hidden argument. This is safe to do this here, as R12 3172 // won't be modified thereafter, before the `blx` (call) instruction. 3173 DCHECK_EQ(R12, hidden_reg); 3174 __ LoadImmediate(hidden_reg, invoke->GetDexMethodIndex()); 3175 3176 if (receiver.IsStackSlot()) { 3177 __ LoadFromOffset(kLoadWord, temp, SP, receiver.GetStackIndex()); 3178 // /* HeapReference<Class> */ temp = temp->klass_ 3179 __ LoadFromOffset(kLoadWord, temp, temp, class_offset); 3180 } else { 3181 // /* HeapReference<Class> */ temp = receiver->klass_ 3182 __ LoadFromOffset(kLoadWord, temp, receiver.AsRegister<Register>(), class_offset); 3183 } 3184 codegen_->MaybeRecordImplicitNullCheck(invoke); 3185 // Instead of simply (possibly) unpoisoning `temp` here, we should 3186 // emit a read barrier for the previous class reference load. 3187 // However this is not required in practice, as this is an 3188 // intermediate/temporary reference and because the current 3189 // concurrent copying collector keeps the from-space memory 3190 // intact/accessible until the end of the marking phase (the 3191 // concurrent copying collector may not in the future). 3192 __ MaybeUnpoisonHeapReference(temp); 3193 __ LoadFromOffset(kLoadWord, temp, temp, 3194 mirror::Class::ImtPtrOffset(kArmPointerSize).Uint32Value()); 3195 uint32_t method_offset = static_cast<uint32_t>(ImTable::OffsetOfElement( 3196 invoke->GetImtIndex(), kArmPointerSize)); 3197 // temp = temp->GetImtEntryAt(method_offset); 3198 __ LoadFromOffset(kLoadWord, temp, temp, method_offset); 3199 uint32_t entry_point = 3200 ArtMethod::EntryPointFromQuickCompiledCodeOffset(kArmPointerSize).Int32Value(); 3201 // LR = temp->GetEntryPoint(); 3202 __ LoadFromOffset(kLoadWord, LR, temp, entry_point); 3203 // LR(); 3204 __ blx(LR); 3205 DCHECK(!codegen_->IsLeafMethod()); 3206 codegen_->RecordPcInfo(invoke, invoke->GetDexPc()); 3207} 3208 3209void LocationsBuilderARM::VisitInvokePolymorphic(HInvokePolymorphic* invoke) { 3210 HandleInvoke(invoke); 3211} 3212 3213void InstructionCodeGeneratorARM::VisitInvokePolymorphic(HInvokePolymorphic* invoke) { 3214 codegen_->GenerateInvokePolymorphicCall(invoke); 3215} 3216 3217void LocationsBuilderARM::VisitNeg(HNeg* neg) { 3218 LocationSummary* locations = 3219 new (GetGraph()->GetArena()) LocationSummary(neg, LocationSummary::kNoCall); 3220 switch (neg->GetResultType()) { 3221 case Primitive::kPrimInt: { 3222 locations->SetInAt(0, Location::RequiresRegister()); 3223 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 3224 break; 3225 } 3226 case Primitive::kPrimLong: { 3227 locations->SetInAt(0, Location::RequiresRegister()); 3228 locations->SetOut(Location::RequiresRegister(), Location::kOutputOverlap); 3229 break; 3230 } 3231 3232 case Primitive::kPrimFloat: 3233 case Primitive::kPrimDouble: 3234 locations->SetInAt(0, Location::RequiresFpuRegister()); 3235 locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap); 3236 break; 3237 3238 default: 3239 LOG(FATAL) << "Unexpected neg type " << neg->GetResultType(); 3240 } 3241} 3242 3243void InstructionCodeGeneratorARM::VisitNeg(HNeg* neg) { 3244 LocationSummary* locations = neg->GetLocations(); 3245 Location out = locations->Out(); 3246 Location in = locations->InAt(0); 3247 switch (neg->GetResultType()) { 3248 case Primitive::kPrimInt: 3249 DCHECK(in.IsRegister()); 3250 __ rsb(out.AsRegister<Register>(), in.AsRegister<Register>(), ShifterOperand(0)); 3251 break; 3252 3253 case Primitive::kPrimLong: 3254 DCHECK(in.IsRegisterPair()); 3255 // out.lo = 0 - in.lo (and update the carry/borrow (C) flag) 3256 __ rsbs(out.AsRegisterPairLow<Register>(), 3257 in.AsRegisterPairLow<Register>(), 3258 ShifterOperand(0)); 3259 // We cannot emit an RSC (Reverse Subtract with Carry) 3260 // instruction here, as it does not exist in the Thumb-2 3261 // instruction set. We use the following approach 3262 // using SBC and SUB instead. 3263 // 3264 // out.hi = -C 3265 __ sbc(out.AsRegisterPairHigh<Register>(), 3266 out.AsRegisterPairHigh<Register>(), 3267 ShifterOperand(out.AsRegisterPairHigh<Register>())); 3268 // out.hi = out.hi - in.hi 3269 __ sub(out.AsRegisterPairHigh<Register>(), 3270 out.AsRegisterPairHigh<Register>(), 3271 ShifterOperand(in.AsRegisterPairHigh<Register>())); 3272 break; 3273 3274 case Primitive::kPrimFloat: 3275 DCHECK(in.IsFpuRegister()); 3276 __ vnegs(out.AsFpuRegister<SRegister>(), in.AsFpuRegister<SRegister>()); 3277 break; 3278 3279 case Primitive::kPrimDouble: 3280 DCHECK(in.IsFpuRegisterPair()); 3281 __ vnegd(FromLowSToD(out.AsFpuRegisterPairLow<SRegister>()), 3282 FromLowSToD(in.AsFpuRegisterPairLow<SRegister>())); 3283 break; 3284 3285 default: 3286 LOG(FATAL) << "Unexpected neg type " << neg->GetResultType(); 3287 } 3288} 3289 3290void LocationsBuilderARM::VisitTypeConversion(HTypeConversion* conversion) { 3291 Primitive::Type result_type = conversion->GetResultType(); 3292 Primitive::Type input_type = conversion->GetInputType(); 3293 DCHECK_NE(result_type, input_type); 3294 3295 // The float-to-long, double-to-long and long-to-float type conversions 3296 // rely on a call to the runtime. 3297 LocationSummary::CallKind call_kind = 3298 (((input_type == Primitive::kPrimFloat || input_type == Primitive::kPrimDouble) 3299 && result_type == Primitive::kPrimLong) 3300 || (input_type == Primitive::kPrimLong && result_type == Primitive::kPrimFloat)) 3301 ? LocationSummary::kCallOnMainOnly 3302 : LocationSummary::kNoCall; 3303 LocationSummary* locations = 3304 new (GetGraph()->GetArena()) LocationSummary(conversion, call_kind); 3305 3306 // The Java language does not allow treating boolean as an integral type but 3307 // our bit representation makes it safe. 3308 3309 switch (result_type) { 3310 case Primitive::kPrimByte: 3311 switch (input_type) { 3312 case Primitive::kPrimLong: 3313 // Type conversion from long to byte is a result of code transformations. 3314 case Primitive::kPrimBoolean: 3315 // Boolean input is a result of code transformations. 3316 case Primitive::kPrimShort: 3317 case Primitive::kPrimInt: 3318 case Primitive::kPrimChar: 3319 // Processing a Dex `int-to-byte' instruction. 3320 locations->SetInAt(0, Location::RequiresRegister()); 3321 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 3322 break; 3323 3324 default: 3325 LOG(FATAL) << "Unexpected type conversion from " << input_type 3326 << " to " << result_type; 3327 } 3328 break; 3329 3330 case Primitive::kPrimShort: 3331 switch (input_type) { 3332 case Primitive::kPrimLong: 3333 // Type conversion from long to short is a result of code transformations. 3334 case Primitive::kPrimBoolean: 3335 // Boolean input is a result of code transformations. 3336 case Primitive::kPrimByte: 3337 case Primitive::kPrimInt: 3338 case Primitive::kPrimChar: 3339 // Processing a Dex `int-to-short' instruction. 3340 locations->SetInAt(0, Location::RequiresRegister()); 3341 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 3342 break; 3343 3344 default: 3345 LOG(FATAL) << "Unexpected type conversion from " << input_type 3346 << " to " << result_type; 3347 } 3348 break; 3349 3350 case Primitive::kPrimInt: 3351 switch (input_type) { 3352 case Primitive::kPrimLong: 3353 // Processing a Dex `long-to-int' instruction. 3354 locations->SetInAt(0, Location::Any()); 3355 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 3356 break; 3357 3358 case Primitive::kPrimFloat: 3359 // Processing a Dex `float-to-int' instruction. 3360 locations->SetInAt(0, Location::RequiresFpuRegister()); 3361 locations->SetOut(Location::RequiresRegister()); 3362 locations->AddTemp(Location::RequiresFpuRegister()); 3363 break; 3364 3365 case Primitive::kPrimDouble: 3366 // Processing a Dex `double-to-int' instruction. 3367 locations->SetInAt(0, Location::RequiresFpuRegister()); 3368 locations->SetOut(Location::RequiresRegister()); 3369 locations->AddTemp(Location::RequiresFpuRegister()); 3370 break; 3371 3372 default: 3373 LOG(FATAL) << "Unexpected type conversion from " << input_type 3374 << " to " << result_type; 3375 } 3376 break; 3377 3378 case Primitive::kPrimLong: 3379 switch (input_type) { 3380 case Primitive::kPrimBoolean: 3381 // Boolean input is a result of code transformations. 3382 case Primitive::kPrimByte: 3383 case Primitive::kPrimShort: 3384 case Primitive::kPrimInt: 3385 case Primitive::kPrimChar: 3386 // Processing a Dex `int-to-long' instruction. 3387 locations->SetInAt(0, Location::RequiresRegister()); 3388 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 3389 break; 3390 3391 case Primitive::kPrimFloat: { 3392 // Processing a Dex `float-to-long' instruction. 3393 InvokeRuntimeCallingConvention calling_convention; 3394 locations->SetInAt(0, Location::FpuRegisterLocation( 3395 calling_convention.GetFpuRegisterAt(0))); 3396 locations->SetOut(Location::RegisterPairLocation(R0, R1)); 3397 break; 3398 } 3399 3400 case Primitive::kPrimDouble: { 3401 // Processing a Dex `double-to-long' instruction. 3402 InvokeRuntimeCallingConvention calling_convention; 3403 locations->SetInAt(0, Location::FpuRegisterPairLocation( 3404 calling_convention.GetFpuRegisterAt(0), 3405 calling_convention.GetFpuRegisterAt(1))); 3406 locations->SetOut(Location::RegisterPairLocation(R0, R1)); 3407 break; 3408 } 3409 3410 default: 3411 LOG(FATAL) << "Unexpected type conversion from " << input_type 3412 << " to " << result_type; 3413 } 3414 break; 3415 3416 case Primitive::kPrimChar: 3417 switch (input_type) { 3418 case Primitive::kPrimLong: 3419 // Type conversion from long to char is a result of code transformations. 3420 case Primitive::kPrimBoolean: 3421 // Boolean input is a result of code transformations. 3422 case Primitive::kPrimByte: 3423 case Primitive::kPrimShort: 3424 case Primitive::kPrimInt: 3425 // Processing a Dex `int-to-char' instruction. 3426 locations->SetInAt(0, Location::RequiresRegister()); 3427 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 3428 break; 3429 3430 default: 3431 LOG(FATAL) << "Unexpected type conversion from " << input_type 3432 << " to " << result_type; 3433 } 3434 break; 3435 3436 case Primitive::kPrimFloat: 3437 switch (input_type) { 3438 case Primitive::kPrimBoolean: 3439 // Boolean input is a result of code transformations. 3440 case Primitive::kPrimByte: 3441 case Primitive::kPrimShort: 3442 case Primitive::kPrimInt: 3443 case Primitive::kPrimChar: 3444 // Processing a Dex `int-to-float' instruction. 3445 locations->SetInAt(0, Location::RequiresRegister()); 3446 locations->SetOut(Location::RequiresFpuRegister()); 3447 break; 3448 3449 case Primitive::kPrimLong: { 3450 // Processing a Dex `long-to-float' instruction. 3451 InvokeRuntimeCallingConvention calling_convention; 3452 locations->SetInAt(0, Location::RegisterPairLocation( 3453 calling_convention.GetRegisterAt(0), calling_convention.GetRegisterAt(1))); 3454 locations->SetOut(Location::FpuRegisterLocation(calling_convention.GetFpuRegisterAt(0))); 3455 break; 3456 } 3457 3458 case Primitive::kPrimDouble: 3459 // Processing a Dex `double-to-float' instruction. 3460 locations->SetInAt(0, Location::RequiresFpuRegister()); 3461 locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap); 3462 break; 3463 3464 default: 3465 LOG(FATAL) << "Unexpected type conversion from " << input_type 3466 << " to " << result_type; 3467 }; 3468 break; 3469 3470 case Primitive::kPrimDouble: 3471 switch (input_type) { 3472 case Primitive::kPrimBoolean: 3473 // Boolean input is a result of code transformations. 3474 case Primitive::kPrimByte: 3475 case Primitive::kPrimShort: 3476 case Primitive::kPrimInt: 3477 case Primitive::kPrimChar: 3478 // Processing a Dex `int-to-double' instruction. 3479 locations->SetInAt(0, Location::RequiresRegister()); 3480 locations->SetOut(Location::RequiresFpuRegister()); 3481 break; 3482 3483 case Primitive::kPrimLong: 3484 // Processing a Dex `long-to-double' instruction. 3485 locations->SetInAt(0, Location::RequiresRegister()); 3486 locations->SetOut(Location::RequiresFpuRegister()); 3487 locations->AddTemp(Location::RequiresFpuRegister()); 3488 locations->AddTemp(Location::RequiresFpuRegister()); 3489 break; 3490 3491 case Primitive::kPrimFloat: 3492 // Processing a Dex `float-to-double' instruction. 3493 locations->SetInAt(0, Location::RequiresFpuRegister()); 3494 locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap); 3495 break; 3496 3497 default: 3498 LOG(FATAL) << "Unexpected type conversion from " << input_type 3499 << " to " << result_type; 3500 }; 3501 break; 3502 3503 default: 3504 LOG(FATAL) << "Unexpected type conversion from " << input_type 3505 << " to " << result_type; 3506 } 3507} 3508 3509void InstructionCodeGeneratorARM::VisitTypeConversion(HTypeConversion* conversion) { 3510 LocationSummary* locations = conversion->GetLocations(); 3511 Location out = locations->Out(); 3512 Location in = locations->InAt(0); 3513 Primitive::Type result_type = conversion->GetResultType(); 3514 Primitive::Type input_type = conversion->GetInputType(); 3515 DCHECK_NE(result_type, input_type); 3516 switch (result_type) { 3517 case Primitive::kPrimByte: 3518 switch (input_type) { 3519 case Primitive::kPrimLong: 3520 // Type conversion from long to byte is a result of code transformations. 3521 __ sbfx(out.AsRegister<Register>(), in.AsRegisterPairLow<Register>(), 0, 8); 3522 break; 3523 case Primitive::kPrimBoolean: 3524 // Boolean input is a result of code transformations. 3525 case Primitive::kPrimShort: 3526 case Primitive::kPrimInt: 3527 case Primitive::kPrimChar: 3528 // Processing a Dex `int-to-byte' instruction. 3529 __ sbfx(out.AsRegister<Register>(), in.AsRegister<Register>(), 0, 8); 3530 break; 3531 3532 default: 3533 LOG(FATAL) << "Unexpected type conversion from " << input_type 3534 << " to " << result_type; 3535 } 3536 break; 3537 3538 case Primitive::kPrimShort: 3539 switch (input_type) { 3540 case Primitive::kPrimLong: 3541 // Type conversion from long to short is a result of code transformations. 3542 __ sbfx(out.AsRegister<Register>(), in.AsRegisterPairLow<Register>(), 0, 16); 3543 break; 3544 case Primitive::kPrimBoolean: 3545 // Boolean input is a result of code transformations. 3546 case Primitive::kPrimByte: 3547 case Primitive::kPrimInt: 3548 case Primitive::kPrimChar: 3549 // Processing a Dex `int-to-short' instruction. 3550 __ sbfx(out.AsRegister<Register>(), in.AsRegister<Register>(), 0, 16); 3551 break; 3552 3553 default: 3554 LOG(FATAL) << "Unexpected type conversion from " << input_type 3555 << " to " << result_type; 3556 } 3557 break; 3558 3559 case Primitive::kPrimInt: 3560 switch (input_type) { 3561 case Primitive::kPrimLong: 3562 // Processing a Dex `long-to-int' instruction. 3563 DCHECK(out.IsRegister()); 3564 if (in.IsRegisterPair()) { 3565 __ Mov(out.AsRegister<Register>(), in.AsRegisterPairLow<Register>()); 3566 } else if (in.IsDoubleStackSlot()) { 3567 __ LoadFromOffset(kLoadWord, out.AsRegister<Register>(), SP, in.GetStackIndex()); 3568 } else { 3569 DCHECK(in.IsConstant()); 3570 DCHECK(in.GetConstant()->IsLongConstant()); 3571 int64_t value = in.GetConstant()->AsLongConstant()->GetValue(); 3572 __ LoadImmediate(out.AsRegister<Register>(), static_cast<int32_t>(value)); 3573 } 3574 break; 3575 3576 case Primitive::kPrimFloat: { 3577 // Processing a Dex `float-to-int' instruction. 3578 SRegister temp = locations->GetTemp(0).AsFpuRegisterPairLow<SRegister>(); 3579 __ vcvtis(temp, in.AsFpuRegister<SRegister>()); 3580 __ vmovrs(out.AsRegister<Register>(), temp); 3581 break; 3582 } 3583 3584 case Primitive::kPrimDouble: { 3585 // Processing a Dex `double-to-int' instruction. 3586 SRegister temp_s = locations->GetTemp(0).AsFpuRegisterPairLow<SRegister>(); 3587 __ vcvtid(temp_s, FromLowSToD(in.AsFpuRegisterPairLow<SRegister>())); 3588 __ vmovrs(out.AsRegister<Register>(), temp_s); 3589 break; 3590 } 3591 3592 default: 3593 LOG(FATAL) << "Unexpected type conversion from " << input_type 3594 << " to " << result_type; 3595 } 3596 break; 3597 3598 case Primitive::kPrimLong: 3599 switch (input_type) { 3600 case Primitive::kPrimBoolean: 3601 // Boolean input is a result of code transformations. 3602 case Primitive::kPrimByte: 3603 case Primitive::kPrimShort: 3604 case Primitive::kPrimInt: 3605 case Primitive::kPrimChar: 3606 // Processing a Dex `int-to-long' instruction. 3607 DCHECK(out.IsRegisterPair()); 3608 DCHECK(in.IsRegister()); 3609 __ Mov(out.AsRegisterPairLow<Register>(), in.AsRegister<Register>()); 3610 // Sign extension. 3611 __ Asr(out.AsRegisterPairHigh<Register>(), 3612 out.AsRegisterPairLow<Register>(), 3613 31); 3614 break; 3615 3616 case Primitive::kPrimFloat: 3617 // Processing a Dex `float-to-long' instruction. 3618 codegen_->InvokeRuntime(kQuickF2l, conversion, conversion->GetDexPc()); 3619 CheckEntrypointTypes<kQuickF2l, int64_t, float>(); 3620 break; 3621 3622 case Primitive::kPrimDouble: 3623 // Processing a Dex `double-to-long' instruction. 3624 codegen_->InvokeRuntime(kQuickD2l, conversion, conversion->GetDexPc()); 3625 CheckEntrypointTypes<kQuickD2l, int64_t, double>(); 3626 break; 3627 3628 default: 3629 LOG(FATAL) << "Unexpected type conversion from " << input_type 3630 << " to " << result_type; 3631 } 3632 break; 3633 3634 case Primitive::kPrimChar: 3635 switch (input_type) { 3636 case Primitive::kPrimLong: 3637 // Type conversion from long to char is a result of code transformations. 3638 __ ubfx(out.AsRegister<Register>(), in.AsRegisterPairLow<Register>(), 0, 16); 3639 break; 3640 case Primitive::kPrimBoolean: 3641 // Boolean input is a result of code transformations. 3642 case Primitive::kPrimByte: 3643 case Primitive::kPrimShort: 3644 case Primitive::kPrimInt: 3645 // Processing a Dex `int-to-char' instruction. 3646 __ ubfx(out.AsRegister<Register>(), in.AsRegister<Register>(), 0, 16); 3647 break; 3648 3649 default: 3650 LOG(FATAL) << "Unexpected type conversion from " << input_type 3651 << " to " << result_type; 3652 } 3653 break; 3654 3655 case Primitive::kPrimFloat: 3656 switch (input_type) { 3657 case Primitive::kPrimBoolean: 3658 // Boolean input is a result of code transformations. 3659 case Primitive::kPrimByte: 3660 case Primitive::kPrimShort: 3661 case Primitive::kPrimInt: 3662 case Primitive::kPrimChar: { 3663 // Processing a Dex `int-to-float' instruction. 3664 __ vmovsr(out.AsFpuRegister<SRegister>(), in.AsRegister<Register>()); 3665 __ vcvtsi(out.AsFpuRegister<SRegister>(), out.AsFpuRegister<SRegister>()); 3666 break; 3667 } 3668 3669 case Primitive::kPrimLong: 3670 // Processing a Dex `long-to-float' instruction. 3671 codegen_->InvokeRuntime(kQuickL2f, conversion, conversion->GetDexPc()); 3672 CheckEntrypointTypes<kQuickL2f, float, int64_t>(); 3673 break; 3674 3675 case Primitive::kPrimDouble: 3676 // Processing a Dex `double-to-float' instruction. 3677 __ vcvtsd(out.AsFpuRegister<SRegister>(), 3678 FromLowSToD(in.AsFpuRegisterPairLow<SRegister>())); 3679 break; 3680 3681 default: 3682 LOG(FATAL) << "Unexpected type conversion from " << input_type 3683 << " to " << result_type; 3684 }; 3685 break; 3686 3687 case Primitive::kPrimDouble: 3688 switch (input_type) { 3689 case Primitive::kPrimBoolean: 3690 // Boolean input is a result of code transformations. 3691 case Primitive::kPrimByte: 3692 case Primitive::kPrimShort: 3693 case Primitive::kPrimInt: 3694 case Primitive::kPrimChar: { 3695 // Processing a Dex `int-to-double' instruction. 3696 __ vmovsr(out.AsFpuRegisterPairLow<SRegister>(), in.AsRegister<Register>()); 3697 __ vcvtdi(FromLowSToD(out.AsFpuRegisterPairLow<SRegister>()), 3698 out.AsFpuRegisterPairLow<SRegister>()); 3699 break; 3700 } 3701 3702 case Primitive::kPrimLong: { 3703 // Processing a Dex `long-to-double' instruction. 3704 Register low = in.AsRegisterPairLow<Register>(); 3705 Register high = in.AsRegisterPairHigh<Register>(); 3706 SRegister out_s = out.AsFpuRegisterPairLow<SRegister>(); 3707 DRegister out_d = FromLowSToD(out_s); 3708 SRegister temp_s = locations->GetTemp(0).AsFpuRegisterPairLow<SRegister>(); 3709 DRegister temp_d = FromLowSToD(temp_s); 3710 SRegister constant_s = locations->GetTemp(1).AsFpuRegisterPairLow<SRegister>(); 3711 DRegister constant_d = FromLowSToD(constant_s); 3712 3713 // temp_d = int-to-double(high) 3714 __ vmovsr(temp_s, high); 3715 __ vcvtdi(temp_d, temp_s); 3716 // constant_d = k2Pow32EncodingForDouble 3717 __ LoadDImmediate(constant_d, bit_cast<double, int64_t>(k2Pow32EncodingForDouble)); 3718 // out_d = unsigned-to-double(low) 3719 __ vmovsr(out_s, low); 3720 __ vcvtdu(out_d, out_s); 3721 // out_d += temp_d * constant_d 3722 __ vmlad(out_d, temp_d, constant_d); 3723 break; 3724 } 3725 3726 case Primitive::kPrimFloat: 3727 // Processing a Dex `float-to-double' instruction. 3728 __ vcvtds(FromLowSToD(out.AsFpuRegisterPairLow<SRegister>()), 3729 in.AsFpuRegister<SRegister>()); 3730 break; 3731 3732 default: 3733 LOG(FATAL) << "Unexpected type conversion from " << input_type 3734 << " to " << result_type; 3735 }; 3736 break; 3737 3738 default: 3739 LOG(FATAL) << "Unexpected type conversion from " << input_type 3740 << " to " << result_type; 3741 } 3742} 3743 3744void LocationsBuilderARM::VisitAdd(HAdd* add) { 3745 LocationSummary* locations = 3746 new (GetGraph()->GetArena()) LocationSummary(add, LocationSummary::kNoCall); 3747 switch (add->GetResultType()) { 3748 case Primitive::kPrimInt: { 3749 locations->SetInAt(0, Location::RequiresRegister()); 3750 locations->SetInAt(1, Location::RegisterOrConstant(add->InputAt(1))); 3751 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 3752 break; 3753 } 3754 3755 case Primitive::kPrimLong: { 3756 locations->SetInAt(0, Location::RequiresRegister()); 3757 locations->SetInAt(1, ArmEncodableConstantOrRegister(add->InputAt(1), ADD)); 3758 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 3759 break; 3760 } 3761 3762 case Primitive::kPrimFloat: 3763 case Primitive::kPrimDouble: { 3764 locations->SetInAt(0, Location::RequiresFpuRegister()); 3765 locations->SetInAt(1, Location::RequiresFpuRegister()); 3766 locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap); 3767 break; 3768 } 3769 3770 default: 3771 LOG(FATAL) << "Unexpected add type " << add->GetResultType(); 3772 } 3773} 3774 3775void InstructionCodeGeneratorARM::VisitAdd(HAdd* add) { 3776 LocationSummary* locations = add->GetLocations(); 3777 Location out = locations->Out(); 3778 Location first = locations->InAt(0); 3779 Location second = locations->InAt(1); 3780 switch (add->GetResultType()) { 3781 case Primitive::kPrimInt: 3782 if (second.IsRegister()) { 3783 __ add(out.AsRegister<Register>(), 3784 first.AsRegister<Register>(), 3785 ShifterOperand(second.AsRegister<Register>())); 3786 } else { 3787 __ AddConstant(out.AsRegister<Register>(), 3788 first.AsRegister<Register>(), 3789 second.GetConstant()->AsIntConstant()->GetValue()); 3790 } 3791 break; 3792 3793 case Primitive::kPrimLong: { 3794 if (second.IsConstant()) { 3795 uint64_t value = static_cast<uint64_t>(Int64FromConstant(second.GetConstant())); 3796 GenerateAddLongConst(out, first, value); 3797 } else { 3798 DCHECK(second.IsRegisterPair()); 3799 __ adds(out.AsRegisterPairLow<Register>(), 3800 first.AsRegisterPairLow<Register>(), 3801 ShifterOperand(second.AsRegisterPairLow<Register>())); 3802 __ adc(out.AsRegisterPairHigh<Register>(), 3803 first.AsRegisterPairHigh<Register>(), 3804 ShifterOperand(second.AsRegisterPairHigh<Register>())); 3805 } 3806 break; 3807 } 3808 3809 case Primitive::kPrimFloat: 3810 __ vadds(out.AsFpuRegister<SRegister>(), 3811 first.AsFpuRegister<SRegister>(), 3812 second.AsFpuRegister<SRegister>()); 3813 break; 3814 3815 case Primitive::kPrimDouble: 3816 __ vaddd(FromLowSToD(out.AsFpuRegisterPairLow<SRegister>()), 3817 FromLowSToD(first.AsFpuRegisterPairLow<SRegister>()), 3818 FromLowSToD(second.AsFpuRegisterPairLow<SRegister>())); 3819 break; 3820 3821 default: 3822 LOG(FATAL) << "Unexpected add type " << add->GetResultType(); 3823 } 3824} 3825 3826void LocationsBuilderARM::VisitSub(HSub* sub) { 3827 LocationSummary* locations = 3828 new (GetGraph()->GetArena()) LocationSummary(sub, LocationSummary::kNoCall); 3829 switch (sub->GetResultType()) { 3830 case Primitive::kPrimInt: { 3831 locations->SetInAt(0, Location::RequiresRegister()); 3832 locations->SetInAt(1, Location::RegisterOrConstant(sub->InputAt(1))); 3833 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 3834 break; 3835 } 3836 3837 case Primitive::kPrimLong: { 3838 locations->SetInAt(0, Location::RequiresRegister()); 3839 locations->SetInAt(1, ArmEncodableConstantOrRegister(sub->InputAt(1), SUB)); 3840 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 3841 break; 3842 } 3843 case Primitive::kPrimFloat: 3844 case Primitive::kPrimDouble: { 3845 locations->SetInAt(0, Location::RequiresFpuRegister()); 3846 locations->SetInAt(1, Location::RequiresFpuRegister()); 3847 locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap); 3848 break; 3849 } 3850 default: 3851 LOG(FATAL) << "Unexpected sub type " << sub->GetResultType(); 3852 } 3853} 3854 3855void InstructionCodeGeneratorARM::VisitSub(HSub* sub) { 3856 LocationSummary* locations = sub->GetLocations(); 3857 Location out = locations->Out(); 3858 Location first = locations->InAt(0); 3859 Location second = locations->InAt(1); 3860 switch (sub->GetResultType()) { 3861 case Primitive::kPrimInt: { 3862 if (second.IsRegister()) { 3863 __ sub(out.AsRegister<Register>(), 3864 first.AsRegister<Register>(), 3865 ShifterOperand(second.AsRegister<Register>())); 3866 } else { 3867 __ AddConstant(out.AsRegister<Register>(), 3868 first.AsRegister<Register>(), 3869 -second.GetConstant()->AsIntConstant()->GetValue()); 3870 } 3871 break; 3872 } 3873 3874 case Primitive::kPrimLong: { 3875 if (second.IsConstant()) { 3876 uint64_t value = static_cast<uint64_t>(Int64FromConstant(second.GetConstant())); 3877 GenerateAddLongConst(out, first, -value); 3878 } else { 3879 DCHECK(second.IsRegisterPair()); 3880 __ subs(out.AsRegisterPairLow<Register>(), 3881 first.AsRegisterPairLow<Register>(), 3882 ShifterOperand(second.AsRegisterPairLow<Register>())); 3883 __ sbc(out.AsRegisterPairHigh<Register>(), 3884 first.AsRegisterPairHigh<Register>(), 3885 ShifterOperand(second.AsRegisterPairHigh<Register>())); 3886 } 3887 break; 3888 } 3889 3890 case Primitive::kPrimFloat: { 3891 __ vsubs(out.AsFpuRegister<SRegister>(), 3892 first.AsFpuRegister<SRegister>(), 3893 second.AsFpuRegister<SRegister>()); 3894 break; 3895 } 3896 3897 case Primitive::kPrimDouble: { 3898 __ vsubd(FromLowSToD(out.AsFpuRegisterPairLow<SRegister>()), 3899 FromLowSToD(first.AsFpuRegisterPairLow<SRegister>()), 3900 FromLowSToD(second.AsFpuRegisterPairLow<SRegister>())); 3901 break; 3902 } 3903 3904 3905 default: 3906 LOG(FATAL) << "Unexpected sub type " << sub->GetResultType(); 3907 } 3908} 3909 3910void LocationsBuilderARM::VisitMul(HMul* mul) { 3911 LocationSummary* locations = 3912 new (GetGraph()->GetArena()) LocationSummary(mul, LocationSummary::kNoCall); 3913 switch (mul->GetResultType()) { 3914 case Primitive::kPrimInt: 3915 case Primitive::kPrimLong: { 3916 locations->SetInAt(0, Location::RequiresRegister()); 3917 locations->SetInAt(1, Location::RequiresRegister()); 3918 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 3919 break; 3920 } 3921 3922 case Primitive::kPrimFloat: 3923 case Primitive::kPrimDouble: { 3924 locations->SetInAt(0, Location::RequiresFpuRegister()); 3925 locations->SetInAt(1, Location::RequiresFpuRegister()); 3926 locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap); 3927 break; 3928 } 3929 3930 default: 3931 LOG(FATAL) << "Unexpected mul type " << mul->GetResultType(); 3932 } 3933} 3934 3935void InstructionCodeGeneratorARM::VisitMul(HMul* mul) { 3936 LocationSummary* locations = mul->GetLocations(); 3937 Location out = locations->Out(); 3938 Location first = locations->InAt(0); 3939 Location second = locations->InAt(1); 3940 switch (mul->GetResultType()) { 3941 case Primitive::kPrimInt: { 3942 __ mul(out.AsRegister<Register>(), 3943 first.AsRegister<Register>(), 3944 second.AsRegister<Register>()); 3945 break; 3946 } 3947 case Primitive::kPrimLong: { 3948 Register out_hi = out.AsRegisterPairHigh<Register>(); 3949 Register out_lo = out.AsRegisterPairLow<Register>(); 3950 Register in1_hi = first.AsRegisterPairHigh<Register>(); 3951 Register in1_lo = first.AsRegisterPairLow<Register>(); 3952 Register in2_hi = second.AsRegisterPairHigh<Register>(); 3953 Register in2_lo = second.AsRegisterPairLow<Register>(); 3954 3955 // Extra checks to protect caused by the existence of R1_R2. 3956 // The algorithm is wrong if out.hi is either in1.lo or in2.lo: 3957 // (e.g. in1=r0_r1, in2=r2_r3 and out=r1_r2); 3958 DCHECK_NE(out_hi, in1_lo); 3959 DCHECK_NE(out_hi, in2_lo); 3960 3961 // input: in1 - 64 bits, in2 - 64 bits 3962 // output: out 3963 // formula: out.hi : out.lo = (in1.lo * in2.hi + in1.hi * in2.lo)* 2^32 + in1.lo * in2.lo 3964 // parts: out.hi = in1.lo * in2.hi + in1.hi * in2.lo + (in1.lo * in2.lo)[63:32] 3965 // parts: out.lo = (in1.lo * in2.lo)[31:0] 3966 3967 // IP <- in1.lo * in2.hi 3968 __ mul(IP, in1_lo, in2_hi); 3969 // out.hi <- in1.lo * in2.hi + in1.hi * in2.lo 3970 __ mla(out_hi, in1_hi, in2_lo, IP); 3971 // out.lo <- (in1.lo * in2.lo)[31:0]; 3972 __ umull(out_lo, IP, in1_lo, in2_lo); 3973 // out.hi <- in2.hi * in1.lo + in2.lo * in1.hi + (in1.lo * in2.lo)[63:32] 3974 __ add(out_hi, out_hi, ShifterOperand(IP)); 3975 break; 3976 } 3977 3978 case Primitive::kPrimFloat: { 3979 __ vmuls(out.AsFpuRegister<SRegister>(), 3980 first.AsFpuRegister<SRegister>(), 3981 second.AsFpuRegister<SRegister>()); 3982 break; 3983 } 3984 3985 case Primitive::kPrimDouble: { 3986 __ vmuld(FromLowSToD(out.AsFpuRegisterPairLow<SRegister>()), 3987 FromLowSToD(first.AsFpuRegisterPairLow<SRegister>()), 3988 FromLowSToD(second.AsFpuRegisterPairLow<SRegister>())); 3989 break; 3990 } 3991 3992 default: 3993 LOG(FATAL) << "Unexpected mul type " << mul->GetResultType(); 3994 } 3995} 3996 3997void InstructionCodeGeneratorARM::DivRemOneOrMinusOne(HBinaryOperation* instruction) { 3998 DCHECK(instruction->IsDiv() || instruction->IsRem()); 3999 DCHECK(instruction->GetResultType() == Primitive::kPrimInt); 4000 4001 LocationSummary* locations = instruction->GetLocations(); 4002 Location second = locations->InAt(1); 4003 DCHECK(second.IsConstant()); 4004 4005 Register out = locations->Out().AsRegister<Register>(); 4006 Register dividend = locations->InAt(0).AsRegister<Register>(); 4007 int32_t imm = second.GetConstant()->AsIntConstant()->GetValue(); 4008 DCHECK(imm == 1 || imm == -1); 4009 4010 if (instruction->IsRem()) { 4011 __ LoadImmediate(out, 0); 4012 } else { 4013 if (imm == 1) { 4014 __ Mov(out, dividend); 4015 } else { 4016 __ rsb(out, dividend, ShifterOperand(0)); 4017 } 4018 } 4019} 4020 4021void InstructionCodeGeneratorARM::DivRemByPowerOfTwo(HBinaryOperation* instruction) { 4022 DCHECK(instruction->IsDiv() || instruction->IsRem()); 4023 DCHECK(instruction->GetResultType() == Primitive::kPrimInt); 4024 4025 LocationSummary* locations = instruction->GetLocations(); 4026 Location second = locations->InAt(1); 4027 DCHECK(second.IsConstant()); 4028 4029 Register out = locations->Out().AsRegister<Register>(); 4030 Register dividend = locations->InAt(0).AsRegister<Register>(); 4031 Register temp = locations->GetTemp(0).AsRegister<Register>(); 4032 int32_t imm = second.GetConstant()->AsIntConstant()->GetValue(); 4033 uint32_t abs_imm = static_cast<uint32_t>(AbsOrMin(imm)); 4034 int ctz_imm = CTZ(abs_imm); 4035 4036 if (ctz_imm == 1) { 4037 __ Lsr(temp, dividend, 32 - ctz_imm); 4038 } else { 4039 __ Asr(temp, dividend, 31); 4040 __ Lsr(temp, temp, 32 - ctz_imm); 4041 } 4042 __ add(out, temp, ShifterOperand(dividend)); 4043 4044 if (instruction->IsDiv()) { 4045 __ Asr(out, out, ctz_imm); 4046 if (imm < 0) { 4047 __ rsb(out, out, ShifterOperand(0)); 4048 } 4049 } else { 4050 __ ubfx(out, out, 0, ctz_imm); 4051 __ sub(out, out, ShifterOperand(temp)); 4052 } 4053} 4054 4055void InstructionCodeGeneratorARM::GenerateDivRemWithAnyConstant(HBinaryOperation* instruction) { 4056 DCHECK(instruction->IsDiv() || instruction->IsRem()); 4057 DCHECK(instruction->GetResultType() == Primitive::kPrimInt); 4058 4059 LocationSummary* locations = instruction->GetLocations(); 4060 Location second = locations->InAt(1); 4061 DCHECK(second.IsConstant()); 4062 4063 Register out = locations->Out().AsRegister<Register>(); 4064 Register dividend = locations->InAt(0).AsRegister<Register>(); 4065 Register temp1 = locations->GetTemp(0).AsRegister<Register>(); 4066 Register temp2 = locations->GetTemp(1).AsRegister<Register>(); 4067 int64_t imm = second.GetConstant()->AsIntConstant()->GetValue(); 4068 4069 int64_t magic; 4070 int shift; 4071 CalculateMagicAndShiftForDivRem(imm, false /* is_long */, &magic, &shift); 4072 4073 __ LoadImmediate(temp1, magic); 4074 __ smull(temp2, temp1, dividend, temp1); 4075 4076 if (imm > 0 && magic < 0) { 4077 __ add(temp1, temp1, ShifterOperand(dividend)); 4078 } else if (imm < 0 && magic > 0) { 4079 __ sub(temp1, temp1, ShifterOperand(dividend)); 4080 } 4081 4082 if (shift != 0) { 4083 __ Asr(temp1, temp1, shift); 4084 } 4085 4086 if (instruction->IsDiv()) { 4087 __ sub(out, temp1, ShifterOperand(temp1, ASR, 31)); 4088 } else { 4089 __ sub(temp1, temp1, ShifterOperand(temp1, ASR, 31)); 4090 // TODO: Strength reduction for mls. 4091 __ LoadImmediate(temp2, imm); 4092 __ mls(out, temp1, temp2, dividend); 4093 } 4094} 4095 4096void InstructionCodeGeneratorARM::GenerateDivRemConstantIntegral(HBinaryOperation* instruction) { 4097 DCHECK(instruction->IsDiv() || instruction->IsRem()); 4098 DCHECK(instruction->GetResultType() == Primitive::kPrimInt); 4099 4100 LocationSummary* locations = instruction->GetLocations(); 4101 Location second = locations->InAt(1); 4102 DCHECK(second.IsConstant()); 4103 4104 int32_t imm = second.GetConstant()->AsIntConstant()->GetValue(); 4105 if (imm == 0) { 4106 // Do not generate anything. DivZeroCheck would prevent any code to be executed. 4107 } else if (imm == 1 || imm == -1) { 4108 DivRemOneOrMinusOne(instruction); 4109 } else if (IsPowerOfTwo(AbsOrMin(imm))) { 4110 DivRemByPowerOfTwo(instruction); 4111 } else { 4112 DCHECK(imm <= -2 || imm >= 2); 4113 GenerateDivRemWithAnyConstant(instruction); 4114 } 4115} 4116 4117void LocationsBuilderARM::VisitDiv(HDiv* div) { 4118 LocationSummary::CallKind call_kind = LocationSummary::kNoCall; 4119 if (div->GetResultType() == Primitive::kPrimLong) { 4120 // pLdiv runtime call. 4121 call_kind = LocationSummary::kCallOnMainOnly; 4122 } else if (div->GetResultType() == Primitive::kPrimInt && div->InputAt(1)->IsConstant()) { 4123 // sdiv will be replaced by other instruction sequence. 4124 } else if (div->GetResultType() == Primitive::kPrimInt && 4125 !codegen_->GetInstructionSetFeatures().HasDivideInstruction()) { 4126 // pIdivmod runtime call. 4127 call_kind = LocationSummary::kCallOnMainOnly; 4128 } 4129 4130 LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(div, call_kind); 4131 4132 switch (div->GetResultType()) { 4133 case Primitive::kPrimInt: { 4134 if (div->InputAt(1)->IsConstant()) { 4135 locations->SetInAt(0, Location::RequiresRegister()); 4136 locations->SetInAt(1, Location::ConstantLocation(div->InputAt(1)->AsConstant())); 4137 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 4138 int32_t value = div->InputAt(1)->AsIntConstant()->GetValue(); 4139 if (value == 1 || value == 0 || value == -1) { 4140 // No temp register required. 4141 } else { 4142 locations->AddTemp(Location::RequiresRegister()); 4143 if (!IsPowerOfTwo(AbsOrMin(value))) { 4144 locations->AddTemp(Location::RequiresRegister()); 4145 } 4146 } 4147 } else if (codegen_->GetInstructionSetFeatures().HasDivideInstruction()) { 4148 locations->SetInAt(0, Location::RequiresRegister()); 4149 locations->SetInAt(1, Location::RequiresRegister()); 4150 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 4151 } else { 4152 InvokeRuntimeCallingConvention calling_convention; 4153 locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(0))); 4154 locations->SetInAt(1, Location::RegisterLocation(calling_convention.GetRegisterAt(1))); 4155 // Note: divmod will compute both the quotient and the remainder as the pair R0 and R1, but 4156 // we only need the former. 4157 locations->SetOut(Location::RegisterLocation(R0)); 4158 } 4159 break; 4160 } 4161 case Primitive::kPrimLong: { 4162 InvokeRuntimeCallingConvention calling_convention; 4163 locations->SetInAt(0, Location::RegisterPairLocation( 4164 calling_convention.GetRegisterAt(0), calling_convention.GetRegisterAt(1))); 4165 locations->SetInAt(1, Location::RegisterPairLocation( 4166 calling_convention.GetRegisterAt(2), calling_convention.GetRegisterAt(3))); 4167 locations->SetOut(Location::RegisterPairLocation(R0, R1)); 4168 break; 4169 } 4170 case Primitive::kPrimFloat: 4171 case Primitive::kPrimDouble: { 4172 locations->SetInAt(0, Location::RequiresFpuRegister()); 4173 locations->SetInAt(1, Location::RequiresFpuRegister()); 4174 locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap); 4175 break; 4176 } 4177 4178 default: 4179 LOG(FATAL) << "Unexpected div type " << div->GetResultType(); 4180 } 4181} 4182 4183void InstructionCodeGeneratorARM::VisitDiv(HDiv* div) { 4184 LocationSummary* locations = div->GetLocations(); 4185 Location out = locations->Out(); 4186 Location first = locations->InAt(0); 4187 Location second = locations->InAt(1); 4188 4189 switch (div->GetResultType()) { 4190 case Primitive::kPrimInt: { 4191 if (second.IsConstant()) { 4192 GenerateDivRemConstantIntegral(div); 4193 } else if (codegen_->GetInstructionSetFeatures().HasDivideInstruction()) { 4194 __ sdiv(out.AsRegister<Register>(), 4195 first.AsRegister<Register>(), 4196 second.AsRegister<Register>()); 4197 } else { 4198 InvokeRuntimeCallingConvention calling_convention; 4199 DCHECK_EQ(calling_convention.GetRegisterAt(0), first.AsRegister<Register>()); 4200 DCHECK_EQ(calling_convention.GetRegisterAt(1), second.AsRegister<Register>()); 4201 DCHECK_EQ(R0, out.AsRegister<Register>()); 4202 4203 codegen_->InvokeRuntime(kQuickIdivmod, div, div->GetDexPc()); 4204 CheckEntrypointTypes<kQuickIdivmod, int32_t, int32_t, int32_t>(); 4205 } 4206 break; 4207 } 4208 4209 case Primitive::kPrimLong: { 4210 InvokeRuntimeCallingConvention calling_convention; 4211 DCHECK_EQ(calling_convention.GetRegisterAt(0), first.AsRegisterPairLow<Register>()); 4212 DCHECK_EQ(calling_convention.GetRegisterAt(1), first.AsRegisterPairHigh<Register>()); 4213 DCHECK_EQ(calling_convention.GetRegisterAt(2), second.AsRegisterPairLow<Register>()); 4214 DCHECK_EQ(calling_convention.GetRegisterAt(3), second.AsRegisterPairHigh<Register>()); 4215 DCHECK_EQ(R0, out.AsRegisterPairLow<Register>()); 4216 DCHECK_EQ(R1, out.AsRegisterPairHigh<Register>()); 4217 4218 codegen_->InvokeRuntime(kQuickLdiv, div, div->GetDexPc()); 4219 CheckEntrypointTypes<kQuickLdiv, int64_t, int64_t, int64_t>(); 4220 break; 4221 } 4222 4223 case Primitive::kPrimFloat: { 4224 __ vdivs(out.AsFpuRegister<SRegister>(), 4225 first.AsFpuRegister<SRegister>(), 4226 second.AsFpuRegister<SRegister>()); 4227 break; 4228 } 4229 4230 case Primitive::kPrimDouble: { 4231 __ vdivd(FromLowSToD(out.AsFpuRegisterPairLow<SRegister>()), 4232 FromLowSToD(first.AsFpuRegisterPairLow<SRegister>()), 4233 FromLowSToD(second.AsFpuRegisterPairLow<SRegister>())); 4234 break; 4235 } 4236 4237 default: 4238 LOG(FATAL) << "Unexpected div type " << div->GetResultType(); 4239 } 4240} 4241 4242void LocationsBuilderARM::VisitRem(HRem* rem) { 4243 Primitive::Type type = rem->GetResultType(); 4244 4245 // Most remainders are implemented in the runtime. 4246 LocationSummary::CallKind call_kind = LocationSummary::kCallOnMainOnly; 4247 if (rem->GetResultType() == Primitive::kPrimInt && rem->InputAt(1)->IsConstant()) { 4248 // sdiv will be replaced by other instruction sequence. 4249 call_kind = LocationSummary::kNoCall; 4250 } else if ((rem->GetResultType() == Primitive::kPrimInt) 4251 && codegen_->GetInstructionSetFeatures().HasDivideInstruction()) { 4252 // Have hardware divide instruction for int, do it with three instructions. 4253 call_kind = LocationSummary::kNoCall; 4254 } 4255 4256 LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(rem, call_kind); 4257 4258 switch (type) { 4259 case Primitive::kPrimInt: { 4260 if (rem->InputAt(1)->IsConstant()) { 4261 locations->SetInAt(0, Location::RequiresRegister()); 4262 locations->SetInAt(1, Location::ConstantLocation(rem->InputAt(1)->AsConstant())); 4263 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 4264 int32_t value = rem->InputAt(1)->AsIntConstant()->GetValue(); 4265 if (value == 1 || value == 0 || value == -1) { 4266 // No temp register required. 4267 } else { 4268 locations->AddTemp(Location::RequiresRegister()); 4269 if (!IsPowerOfTwo(AbsOrMin(value))) { 4270 locations->AddTemp(Location::RequiresRegister()); 4271 } 4272 } 4273 } else if (codegen_->GetInstructionSetFeatures().HasDivideInstruction()) { 4274 locations->SetInAt(0, Location::RequiresRegister()); 4275 locations->SetInAt(1, Location::RequiresRegister()); 4276 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 4277 locations->AddTemp(Location::RequiresRegister()); 4278 } else { 4279 InvokeRuntimeCallingConvention calling_convention; 4280 locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(0))); 4281 locations->SetInAt(1, Location::RegisterLocation(calling_convention.GetRegisterAt(1))); 4282 // Note: divmod will compute both the quotient and the remainder as the pair R0 and R1, but 4283 // we only need the latter. 4284 locations->SetOut(Location::RegisterLocation(R1)); 4285 } 4286 break; 4287 } 4288 case Primitive::kPrimLong: { 4289 InvokeRuntimeCallingConvention calling_convention; 4290 locations->SetInAt(0, Location::RegisterPairLocation( 4291 calling_convention.GetRegisterAt(0), calling_convention.GetRegisterAt(1))); 4292 locations->SetInAt(1, Location::RegisterPairLocation( 4293 calling_convention.GetRegisterAt(2), calling_convention.GetRegisterAt(3))); 4294 // The runtime helper puts the output in R2,R3. 4295 locations->SetOut(Location::RegisterPairLocation(R2, R3)); 4296 break; 4297 } 4298 case Primitive::kPrimFloat: { 4299 InvokeRuntimeCallingConvention calling_convention; 4300 locations->SetInAt(0, Location::FpuRegisterLocation(calling_convention.GetFpuRegisterAt(0))); 4301 locations->SetInAt(1, Location::FpuRegisterLocation(calling_convention.GetFpuRegisterAt(1))); 4302 locations->SetOut(Location::FpuRegisterLocation(S0)); 4303 break; 4304 } 4305 4306 case Primitive::kPrimDouble: { 4307 InvokeRuntimeCallingConvention calling_convention; 4308 locations->SetInAt(0, Location::FpuRegisterPairLocation( 4309 calling_convention.GetFpuRegisterAt(0), calling_convention.GetFpuRegisterAt(1))); 4310 locations->SetInAt(1, Location::FpuRegisterPairLocation( 4311 calling_convention.GetFpuRegisterAt(2), calling_convention.GetFpuRegisterAt(3))); 4312 locations->SetOut(Location::Location::FpuRegisterPairLocation(S0, S1)); 4313 break; 4314 } 4315 4316 default: 4317 LOG(FATAL) << "Unexpected rem type " << type; 4318 } 4319} 4320 4321void InstructionCodeGeneratorARM::VisitRem(HRem* rem) { 4322 LocationSummary* locations = rem->GetLocations(); 4323 Location out = locations->Out(); 4324 Location first = locations->InAt(0); 4325 Location second = locations->InAt(1); 4326 4327 Primitive::Type type = rem->GetResultType(); 4328 switch (type) { 4329 case Primitive::kPrimInt: { 4330 if (second.IsConstant()) { 4331 GenerateDivRemConstantIntegral(rem); 4332 } else if (codegen_->GetInstructionSetFeatures().HasDivideInstruction()) { 4333 Register reg1 = first.AsRegister<Register>(); 4334 Register reg2 = second.AsRegister<Register>(); 4335 Register temp = locations->GetTemp(0).AsRegister<Register>(); 4336 4337 // temp = reg1 / reg2 (integer division) 4338 // dest = reg1 - temp * reg2 4339 __ sdiv(temp, reg1, reg2); 4340 __ mls(out.AsRegister<Register>(), temp, reg2, reg1); 4341 } else { 4342 InvokeRuntimeCallingConvention calling_convention; 4343 DCHECK_EQ(calling_convention.GetRegisterAt(0), first.AsRegister<Register>()); 4344 DCHECK_EQ(calling_convention.GetRegisterAt(1), second.AsRegister<Register>()); 4345 DCHECK_EQ(R1, out.AsRegister<Register>()); 4346 4347 codegen_->InvokeRuntime(kQuickIdivmod, rem, rem->GetDexPc()); 4348 CheckEntrypointTypes<kQuickIdivmod, int32_t, int32_t, int32_t>(); 4349 } 4350 break; 4351 } 4352 4353 case Primitive::kPrimLong: { 4354 codegen_->InvokeRuntime(kQuickLmod, rem, rem->GetDexPc()); 4355 CheckEntrypointTypes<kQuickLmod, int64_t, int64_t, int64_t>(); 4356 break; 4357 } 4358 4359 case Primitive::kPrimFloat: { 4360 codegen_->InvokeRuntime(kQuickFmodf, rem, rem->GetDexPc()); 4361 CheckEntrypointTypes<kQuickFmodf, float, float, float>(); 4362 break; 4363 } 4364 4365 case Primitive::kPrimDouble: { 4366 codegen_->InvokeRuntime(kQuickFmod, rem, rem->GetDexPc()); 4367 CheckEntrypointTypes<kQuickFmod, double, double, double>(); 4368 break; 4369 } 4370 4371 default: 4372 LOG(FATAL) << "Unexpected rem type " << type; 4373 } 4374} 4375 4376void LocationsBuilderARM::VisitDivZeroCheck(HDivZeroCheck* instruction) { 4377 LocationSummary* locations = codegen_->CreateThrowingSlowPathLocations(instruction); 4378 locations->SetInAt(0, Location::RegisterOrConstant(instruction->InputAt(0))); 4379} 4380 4381void InstructionCodeGeneratorARM::VisitDivZeroCheck(HDivZeroCheck* instruction) { 4382 SlowPathCodeARM* slow_path = new (GetGraph()->GetArena()) DivZeroCheckSlowPathARM(instruction); 4383 codegen_->AddSlowPath(slow_path); 4384 4385 LocationSummary* locations = instruction->GetLocations(); 4386 Location value = locations->InAt(0); 4387 4388 switch (instruction->GetType()) { 4389 case Primitive::kPrimBoolean: 4390 case Primitive::kPrimByte: 4391 case Primitive::kPrimChar: 4392 case Primitive::kPrimShort: 4393 case Primitive::kPrimInt: { 4394 if (value.IsRegister()) { 4395 __ CompareAndBranchIfZero(value.AsRegister<Register>(), slow_path->GetEntryLabel()); 4396 } else { 4397 DCHECK(value.IsConstant()) << value; 4398 if (value.GetConstant()->AsIntConstant()->GetValue() == 0) { 4399 __ b(slow_path->GetEntryLabel()); 4400 } 4401 } 4402 break; 4403 } 4404 case Primitive::kPrimLong: { 4405 if (value.IsRegisterPair()) { 4406 __ orrs(IP, 4407 value.AsRegisterPairLow<Register>(), 4408 ShifterOperand(value.AsRegisterPairHigh<Register>())); 4409 __ b(slow_path->GetEntryLabel(), EQ); 4410 } else { 4411 DCHECK(value.IsConstant()) << value; 4412 if (value.GetConstant()->AsLongConstant()->GetValue() == 0) { 4413 __ b(slow_path->GetEntryLabel()); 4414 } 4415 } 4416 break; 4417 default: 4418 LOG(FATAL) << "Unexpected type for HDivZeroCheck " << instruction->GetType(); 4419 } 4420 } 4421} 4422 4423void InstructionCodeGeneratorARM::HandleIntegerRotate(LocationSummary* locations) { 4424 Register in = locations->InAt(0).AsRegister<Register>(); 4425 Location rhs = locations->InAt(1); 4426 Register out = locations->Out().AsRegister<Register>(); 4427 4428 if (rhs.IsConstant()) { 4429 // Arm32 and Thumb2 assemblers require a rotation on the interval [1,31], 4430 // so map all rotations to a +ve. equivalent in that range. 4431 // (e.g. left *or* right by -2 bits == 30 bits in the same direction.) 4432 uint32_t rot = CodeGenerator::GetInt32ValueOf(rhs.GetConstant()) & 0x1F; 4433 if (rot) { 4434 // Rotate, mapping left rotations to right equivalents if necessary. 4435 // (e.g. left by 2 bits == right by 30.) 4436 __ Ror(out, in, rot); 4437 } else if (out != in) { 4438 __ Mov(out, in); 4439 } 4440 } else { 4441 __ Ror(out, in, rhs.AsRegister<Register>()); 4442 } 4443} 4444 4445// Gain some speed by mapping all Long rotates onto equivalent pairs of Integer 4446// rotates by swapping input regs (effectively rotating by the first 32-bits of 4447// a larger rotation) or flipping direction (thus treating larger right/left 4448// rotations as sub-word sized rotations in the other direction) as appropriate. 4449void InstructionCodeGeneratorARM::HandleLongRotate(HRor* ror) { 4450 LocationSummary* locations = ror->GetLocations(); 4451 Register in_reg_lo = locations->InAt(0).AsRegisterPairLow<Register>(); 4452 Register in_reg_hi = locations->InAt(0).AsRegisterPairHigh<Register>(); 4453 Location rhs = locations->InAt(1); 4454 Register out_reg_lo = locations->Out().AsRegisterPairLow<Register>(); 4455 Register out_reg_hi = locations->Out().AsRegisterPairHigh<Register>(); 4456 4457 if (rhs.IsConstant()) { 4458 uint64_t rot = CodeGenerator::GetInt64ValueOf(rhs.GetConstant()); 4459 // Map all rotations to +ve. equivalents on the interval [0,63]. 4460 rot &= kMaxLongShiftDistance; 4461 // For rotates over a word in size, 'pre-rotate' by 32-bits to keep rotate 4462 // logic below to a simple pair of binary orr. 4463 // (e.g. 34 bits == in_reg swap + 2 bits right.) 4464 if (rot >= kArmBitsPerWord) { 4465 rot -= kArmBitsPerWord; 4466 std::swap(in_reg_hi, in_reg_lo); 4467 } 4468 // Rotate, or mov to out for zero or word size rotations. 4469 if (rot != 0u) { 4470 __ Lsr(out_reg_hi, in_reg_hi, rot); 4471 __ orr(out_reg_hi, out_reg_hi, ShifterOperand(in_reg_lo, arm::LSL, kArmBitsPerWord - rot)); 4472 __ Lsr(out_reg_lo, in_reg_lo, rot); 4473 __ orr(out_reg_lo, out_reg_lo, ShifterOperand(in_reg_hi, arm::LSL, kArmBitsPerWord - rot)); 4474 } else { 4475 __ Mov(out_reg_lo, in_reg_lo); 4476 __ Mov(out_reg_hi, in_reg_hi); 4477 } 4478 } else { 4479 Register shift_right = locations->GetTemp(0).AsRegister<Register>(); 4480 Register shift_left = locations->GetTemp(1).AsRegister<Register>(); 4481 Label end; 4482 Label shift_by_32_plus_shift_right; 4483 Label* final_label = codegen_->GetFinalLabel(ror, &end); 4484 4485 __ and_(shift_right, rhs.AsRegister<Register>(), ShifterOperand(0x1F)); 4486 __ Lsrs(shift_left, rhs.AsRegister<Register>(), 6); 4487 __ rsb(shift_left, shift_right, ShifterOperand(kArmBitsPerWord), AL, kCcKeep); 4488 __ b(&shift_by_32_plus_shift_right, CC); 4489 4490 // out_reg_hi = (reg_hi << shift_left) | (reg_lo >> shift_right). 4491 // out_reg_lo = (reg_lo << shift_left) | (reg_hi >> shift_right). 4492 __ Lsl(out_reg_hi, in_reg_hi, shift_left); 4493 __ Lsr(out_reg_lo, in_reg_lo, shift_right); 4494 __ add(out_reg_hi, out_reg_hi, ShifterOperand(out_reg_lo)); 4495 __ Lsl(out_reg_lo, in_reg_lo, shift_left); 4496 __ Lsr(shift_left, in_reg_hi, shift_right); 4497 __ add(out_reg_lo, out_reg_lo, ShifterOperand(shift_left)); 4498 __ b(final_label); 4499 4500 __ Bind(&shift_by_32_plus_shift_right); // Shift by 32+shift_right. 4501 // out_reg_hi = (reg_hi >> shift_right) | (reg_lo << shift_left). 4502 // out_reg_lo = (reg_lo >> shift_right) | (reg_hi << shift_left). 4503 __ Lsr(out_reg_hi, in_reg_hi, shift_right); 4504 __ Lsl(out_reg_lo, in_reg_lo, shift_left); 4505 __ add(out_reg_hi, out_reg_hi, ShifterOperand(out_reg_lo)); 4506 __ Lsr(out_reg_lo, in_reg_lo, shift_right); 4507 __ Lsl(shift_right, in_reg_hi, shift_left); 4508 __ add(out_reg_lo, out_reg_lo, ShifterOperand(shift_right)); 4509 4510 if (end.IsLinked()) { 4511 __ Bind(&end); 4512 } 4513 } 4514} 4515 4516void LocationsBuilderARM::VisitRor(HRor* ror) { 4517 LocationSummary* locations = 4518 new (GetGraph()->GetArena()) LocationSummary(ror, LocationSummary::kNoCall); 4519 switch (ror->GetResultType()) { 4520 case Primitive::kPrimInt: { 4521 locations->SetInAt(0, Location::RequiresRegister()); 4522 locations->SetInAt(1, Location::RegisterOrConstant(ror->InputAt(1))); 4523 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 4524 break; 4525 } 4526 case Primitive::kPrimLong: { 4527 locations->SetInAt(0, Location::RequiresRegister()); 4528 if (ror->InputAt(1)->IsConstant()) { 4529 locations->SetInAt(1, Location::ConstantLocation(ror->InputAt(1)->AsConstant())); 4530 } else { 4531 locations->SetInAt(1, Location::RequiresRegister()); 4532 locations->AddTemp(Location::RequiresRegister()); 4533 locations->AddTemp(Location::RequiresRegister()); 4534 } 4535 locations->SetOut(Location::RequiresRegister(), Location::kOutputOverlap); 4536 break; 4537 } 4538 default: 4539 LOG(FATAL) << "Unexpected operation type " << ror->GetResultType(); 4540 } 4541} 4542 4543void InstructionCodeGeneratorARM::VisitRor(HRor* ror) { 4544 LocationSummary* locations = ror->GetLocations(); 4545 Primitive::Type type = ror->GetResultType(); 4546 switch (type) { 4547 case Primitive::kPrimInt: { 4548 HandleIntegerRotate(locations); 4549 break; 4550 } 4551 case Primitive::kPrimLong: { 4552 HandleLongRotate(ror); 4553 break; 4554 } 4555 default: 4556 LOG(FATAL) << "Unexpected operation type " << type; 4557 UNREACHABLE(); 4558 } 4559} 4560 4561void LocationsBuilderARM::HandleShift(HBinaryOperation* op) { 4562 DCHECK(op->IsShl() || op->IsShr() || op->IsUShr()); 4563 4564 LocationSummary* locations = 4565 new (GetGraph()->GetArena()) LocationSummary(op, LocationSummary::kNoCall); 4566 4567 switch (op->GetResultType()) { 4568 case Primitive::kPrimInt: { 4569 locations->SetInAt(0, Location::RequiresRegister()); 4570 if (op->InputAt(1)->IsConstant()) { 4571 locations->SetInAt(1, Location::ConstantLocation(op->InputAt(1)->AsConstant())); 4572 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 4573 } else { 4574 locations->SetInAt(1, Location::RequiresRegister()); 4575 // Make the output overlap, as it will be used to hold the masked 4576 // second input. 4577 locations->SetOut(Location::RequiresRegister(), Location::kOutputOverlap); 4578 } 4579 break; 4580 } 4581 case Primitive::kPrimLong: { 4582 locations->SetInAt(0, Location::RequiresRegister()); 4583 if (op->InputAt(1)->IsConstant()) { 4584 locations->SetInAt(1, Location::ConstantLocation(op->InputAt(1)->AsConstant())); 4585 // For simplicity, use kOutputOverlap even though we only require that low registers 4586 // don't clash with high registers which the register allocator currently guarantees. 4587 locations->SetOut(Location::RequiresRegister(), Location::kOutputOverlap); 4588 } else { 4589 locations->SetInAt(1, Location::RequiresRegister()); 4590 locations->AddTemp(Location::RequiresRegister()); 4591 locations->SetOut(Location::RequiresRegister(), Location::kOutputOverlap); 4592 } 4593 break; 4594 } 4595 default: 4596 LOG(FATAL) << "Unexpected operation type " << op->GetResultType(); 4597 } 4598} 4599 4600void InstructionCodeGeneratorARM::HandleShift(HBinaryOperation* op) { 4601 DCHECK(op->IsShl() || op->IsShr() || op->IsUShr()); 4602 4603 LocationSummary* locations = op->GetLocations(); 4604 Location out = locations->Out(); 4605 Location first = locations->InAt(0); 4606 Location second = locations->InAt(1); 4607 4608 Primitive::Type type = op->GetResultType(); 4609 switch (type) { 4610 case Primitive::kPrimInt: { 4611 Register out_reg = out.AsRegister<Register>(); 4612 Register first_reg = first.AsRegister<Register>(); 4613 if (second.IsRegister()) { 4614 Register second_reg = second.AsRegister<Register>(); 4615 // ARM doesn't mask the shift count so we need to do it ourselves. 4616 __ and_(out_reg, second_reg, ShifterOperand(kMaxIntShiftDistance)); 4617 if (op->IsShl()) { 4618 __ Lsl(out_reg, first_reg, out_reg); 4619 } else if (op->IsShr()) { 4620 __ Asr(out_reg, first_reg, out_reg); 4621 } else { 4622 __ Lsr(out_reg, first_reg, out_reg); 4623 } 4624 } else { 4625 int32_t cst = second.GetConstant()->AsIntConstant()->GetValue(); 4626 uint32_t shift_value = cst & kMaxIntShiftDistance; 4627 if (shift_value == 0) { // ARM does not support shifting with 0 immediate. 4628 __ Mov(out_reg, first_reg); 4629 } else if (op->IsShl()) { 4630 __ Lsl(out_reg, first_reg, shift_value); 4631 } else if (op->IsShr()) { 4632 __ Asr(out_reg, first_reg, shift_value); 4633 } else { 4634 __ Lsr(out_reg, first_reg, shift_value); 4635 } 4636 } 4637 break; 4638 } 4639 case Primitive::kPrimLong: { 4640 Register o_h = out.AsRegisterPairHigh<Register>(); 4641 Register o_l = out.AsRegisterPairLow<Register>(); 4642 4643 Register high = first.AsRegisterPairHigh<Register>(); 4644 Register low = first.AsRegisterPairLow<Register>(); 4645 4646 if (second.IsRegister()) { 4647 Register temp = locations->GetTemp(0).AsRegister<Register>(); 4648 4649 Register second_reg = second.AsRegister<Register>(); 4650 4651 if (op->IsShl()) { 4652 __ and_(o_l, second_reg, ShifterOperand(kMaxLongShiftDistance)); 4653 // Shift the high part 4654 __ Lsl(o_h, high, o_l); 4655 // Shift the low part and `or` what overflew on the high part 4656 __ rsb(temp, o_l, ShifterOperand(kArmBitsPerWord)); 4657 __ Lsr(temp, low, temp); 4658 __ orr(o_h, o_h, ShifterOperand(temp)); 4659 // If the shift is > 32 bits, override the high part 4660 __ subs(temp, o_l, ShifterOperand(kArmBitsPerWord)); 4661 __ it(PL); 4662 __ Lsl(o_h, low, temp, PL); 4663 // Shift the low part 4664 __ Lsl(o_l, low, o_l); 4665 } else if (op->IsShr()) { 4666 __ and_(o_h, second_reg, ShifterOperand(kMaxLongShiftDistance)); 4667 // Shift the low part 4668 __ Lsr(o_l, low, o_h); 4669 // Shift the high part and `or` what underflew on the low part 4670 __ rsb(temp, o_h, ShifterOperand(kArmBitsPerWord)); 4671 __ Lsl(temp, high, temp); 4672 __ orr(o_l, o_l, ShifterOperand(temp)); 4673 // If the shift is > 32 bits, override the low part 4674 __ subs(temp, o_h, ShifterOperand(kArmBitsPerWord)); 4675 __ it(PL); 4676 __ Asr(o_l, high, temp, PL); 4677 // Shift the high part 4678 __ Asr(o_h, high, o_h); 4679 } else { 4680 __ and_(o_h, second_reg, ShifterOperand(kMaxLongShiftDistance)); 4681 // same as Shr except we use `Lsr`s and not `Asr`s 4682 __ Lsr(o_l, low, o_h); 4683 __ rsb(temp, o_h, ShifterOperand(kArmBitsPerWord)); 4684 __ Lsl(temp, high, temp); 4685 __ orr(o_l, o_l, ShifterOperand(temp)); 4686 __ subs(temp, o_h, ShifterOperand(kArmBitsPerWord)); 4687 __ it(PL); 4688 __ Lsr(o_l, high, temp, PL); 4689 __ Lsr(o_h, high, o_h); 4690 } 4691 } else { 4692 // Register allocator doesn't create partial overlap. 4693 DCHECK_NE(o_l, high); 4694 DCHECK_NE(o_h, low); 4695 int32_t cst = second.GetConstant()->AsIntConstant()->GetValue(); 4696 uint32_t shift_value = cst & kMaxLongShiftDistance; 4697 if (shift_value > 32) { 4698 if (op->IsShl()) { 4699 __ Lsl(o_h, low, shift_value - 32); 4700 __ LoadImmediate(o_l, 0); 4701 } else if (op->IsShr()) { 4702 __ Asr(o_l, high, shift_value - 32); 4703 __ Asr(o_h, high, 31); 4704 } else { 4705 __ Lsr(o_l, high, shift_value - 32); 4706 __ LoadImmediate(o_h, 0); 4707 } 4708 } else if (shift_value == 32) { 4709 if (op->IsShl()) { 4710 __ mov(o_h, ShifterOperand(low)); 4711 __ LoadImmediate(o_l, 0); 4712 } else if (op->IsShr()) { 4713 __ mov(o_l, ShifterOperand(high)); 4714 __ Asr(o_h, high, 31); 4715 } else { 4716 __ mov(o_l, ShifterOperand(high)); 4717 __ LoadImmediate(o_h, 0); 4718 } 4719 } else if (shift_value == 1) { 4720 if (op->IsShl()) { 4721 __ Lsls(o_l, low, 1); 4722 __ adc(o_h, high, ShifterOperand(high)); 4723 } else if (op->IsShr()) { 4724 __ Asrs(o_h, high, 1); 4725 __ Rrx(o_l, low); 4726 } else { 4727 __ Lsrs(o_h, high, 1); 4728 __ Rrx(o_l, low); 4729 } 4730 } else { 4731 DCHECK(2 <= shift_value && shift_value < 32) << shift_value; 4732 if (op->IsShl()) { 4733 __ Lsl(o_h, high, shift_value); 4734 __ orr(o_h, o_h, ShifterOperand(low, LSR, 32 - shift_value)); 4735 __ Lsl(o_l, low, shift_value); 4736 } else if (op->IsShr()) { 4737 __ Lsr(o_l, low, shift_value); 4738 __ orr(o_l, o_l, ShifterOperand(high, LSL, 32 - shift_value)); 4739 __ Asr(o_h, high, shift_value); 4740 } else { 4741 __ Lsr(o_l, low, shift_value); 4742 __ orr(o_l, o_l, ShifterOperand(high, LSL, 32 - shift_value)); 4743 __ Lsr(o_h, high, shift_value); 4744 } 4745 } 4746 } 4747 break; 4748 } 4749 default: 4750 LOG(FATAL) << "Unexpected operation type " << type; 4751 UNREACHABLE(); 4752 } 4753} 4754 4755void LocationsBuilderARM::VisitShl(HShl* shl) { 4756 HandleShift(shl); 4757} 4758 4759void InstructionCodeGeneratorARM::VisitShl(HShl* shl) { 4760 HandleShift(shl); 4761} 4762 4763void LocationsBuilderARM::VisitShr(HShr* shr) { 4764 HandleShift(shr); 4765} 4766 4767void InstructionCodeGeneratorARM::VisitShr(HShr* shr) { 4768 HandleShift(shr); 4769} 4770 4771void LocationsBuilderARM::VisitUShr(HUShr* ushr) { 4772 HandleShift(ushr); 4773} 4774 4775void InstructionCodeGeneratorARM::VisitUShr(HUShr* ushr) { 4776 HandleShift(ushr); 4777} 4778 4779void LocationsBuilderARM::VisitNewInstance(HNewInstance* instruction) { 4780 LocationSummary* locations = 4781 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCallOnMainOnly); 4782 if (instruction->IsStringAlloc()) { 4783 locations->AddTemp(Location::RegisterLocation(kMethodRegisterArgument)); 4784 } else { 4785 InvokeRuntimeCallingConvention calling_convention; 4786 locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(0))); 4787 } 4788 locations->SetOut(Location::RegisterLocation(R0)); 4789} 4790 4791void InstructionCodeGeneratorARM::VisitNewInstance(HNewInstance* instruction) { 4792 // Note: if heap poisoning is enabled, the entry point takes cares 4793 // of poisoning the reference. 4794 if (instruction->IsStringAlloc()) { 4795 // String is allocated through StringFactory. Call NewEmptyString entry point. 4796 Register temp = instruction->GetLocations()->GetTemp(0).AsRegister<Register>(); 4797 MemberOffset code_offset = ArtMethod::EntryPointFromQuickCompiledCodeOffset(kArmPointerSize); 4798 __ LoadFromOffset(kLoadWord, temp, TR, QUICK_ENTRY_POINT(pNewEmptyString)); 4799 __ LoadFromOffset(kLoadWord, LR, temp, code_offset.Int32Value()); 4800 __ blx(LR); 4801 codegen_->RecordPcInfo(instruction, instruction->GetDexPc()); 4802 } else { 4803 codegen_->InvokeRuntime(instruction->GetEntrypoint(), instruction, instruction->GetDexPc()); 4804 CheckEntrypointTypes<kQuickAllocObjectWithChecks, void*, mirror::Class*>(); 4805 } 4806} 4807 4808void LocationsBuilderARM::VisitNewArray(HNewArray* instruction) { 4809 LocationSummary* locations = 4810 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCallOnMainOnly); 4811 InvokeRuntimeCallingConvention calling_convention; 4812 locations->SetOut(Location::RegisterLocation(R0)); 4813 locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(0))); 4814 locations->SetInAt(1, Location::RegisterLocation(calling_convention.GetRegisterAt(1))); 4815} 4816 4817void InstructionCodeGeneratorARM::VisitNewArray(HNewArray* instruction) { 4818 // Note: if heap poisoning is enabled, the entry point takes cares 4819 // of poisoning the reference. 4820 QuickEntrypointEnum entrypoint = 4821 CodeGenerator::GetArrayAllocationEntrypoint(instruction->GetLoadClass()->GetClass()); 4822 codegen_->InvokeRuntime(entrypoint, instruction, instruction->GetDexPc()); 4823 CheckEntrypointTypes<kQuickAllocArrayResolved, void*, mirror::Class*, int32_t>(); 4824 DCHECK(!codegen_->IsLeafMethod()); 4825} 4826 4827void LocationsBuilderARM::VisitParameterValue(HParameterValue* instruction) { 4828 LocationSummary* locations = 4829 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall); 4830 Location location = parameter_visitor_.GetNextLocation(instruction->GetType()); 4831 if (location.IsStackSlot()) { 4832 location = Location::StackSlot(location.GetStackIndex() + codegen_->GetFrameSize()); 4833 } else if (location.IsDoubleStackSlot()) { 4834 location = Location::DoubleStackSlot(location.GetStackIndex() + codegen_->GetFrameSize()); 4835 } 4836 locations->SetOut(location); 4837} 4838 4839void InstructionCodeGeneratorARM::VisitParameterValue( 4840 HParameterValue* instruction ATTRIBUTE_UNUSED) { 4841 // Nothing to do, the parameter is already at its location. 4842} 4843 4844void LocationsBuilderARM::VisitCurrentMethod(HCurrentMethod* instruction) { 4845 LocationSummary* locations = 4846 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall); 4847 locations->SetOut(Location::RegisterLocation(kMethodRegisterArgument)); 4848} 4849 4850void InstructionCodeGeneratorARM::VisitCurrentMethod(HCurrentMethod* instruction ATTRIBUTE_UNUSED) { 4851 // Nothing to do, the method is already at its location. 4852} 4853 4854void LocationsBuilderARM::VisitNot(HNot* not_) { 4855 LocationSummary* locations = 4856 new (GetGraph()->GetArena()) LocationSummary(not_, LocationSummary::kNoCall); 4857 locations->SetInAt(0, Location::RequiresRegister()); 4858 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 4859} 4860 4861void InstructionCodeGeneratorARM::VisitNot(HNot* not_) { 4862 LocationSummary* locations = not_->GetLocations(); 4863 Location out = locations->Out(); 4864 Location in = locations->InAt(0); 4865 switch (not_->GetResultType()) { 4866 case Primitive::kPrimInt: 4867 __ mvn(out.AsRegister<Register>(), ShifterOperand(in.AsRegister<Register>())); 4868 break; 4869 4870 case Primitive::kPrimLong: 4871 __ mvn(out.AsRegisterPairLow<Register>(), 4872 ShifterOperand(in.AsRegisterPairLow<Register>())); 4873 __ mvn(out.AsRegisterPairHigh<Register>(), 4874 ShifterOperand(in.AsRegisterPairHigh<Register>())); 4875 break; 4876 4877 default: 4878 LOG(FATAL) << "Unimplemented type for not operation " << not_->GetResultType(); 4879 } 4880} 4881 4882void LocationsBuilderARM::VisitBooleanNot(HBooleanNot* bool_not) { 4883 LocationSummary* locations = 4884 new (GetGraph()->GetArena()) LocationSummary(bool_not, LocationSummary::kNoCall); 4885 locations->SetInAt(0, Location::RequiresRegister()); 4886 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 4887} 4888 4889void InstructionCodeGeneratorARM::VisitBooleanNot(HBooleanNot* bool_not) { 4890 LocationSummary* locations = bool_not->GetLocations(); 4891 Location out = locations->Out(); 4892 Location in = locations->InAt(0); 4893 __ eor(out.AsRegister<Register>(), in.AsRegister<Register>(), ShifterOperand(1)); 4894} 4895 4896void LocationsBuilderARM::VisitCompare(HCompare* compare) { 4897 LocationSummary* locations = 4898 new (GetGraph()->GetArena()) LocationSummary(compare, LocationSummary::kNoCall); 4899 switch (compare->InputAt(0)->GetType()) { 4900 case Primitive::kPrimBoolean: 4901 case Primitive::kPrimByte: 4902 case Primitive::kPrimShort: 4903 case Primitive::kPrimChar: 4904 case Primitive::kPrimInt: 4905 case Primitive::kPrimLong: { 4906 locations->SetInAt(0, Location::RequiresRegister()); 4907 locations->SetInAt(1, Location::RequiresRegister()); 4908 // Output overlaps because it is written before doing the low comparison. 4909 locations->SetOut(Location::RequiresRegister(), Location::kOutputOverlap); 4910 break; 4911 } 4912 case Primitive::kPrimFloat: 4913 case Primitive::kPrimDouble: { 4914 locations->SetInAt(0, Location::RequiresFpuRegister()); 4915 locations->SetInAt(1, ArithmeticZeroOrFpuRegister(compare->InputAt(1))); 4916 locations->SetOut(Location::RequiresRegister()); 4917 break; 4918 } 4919 default: 4920 LOG(FATAL) << "Unexpected type for compare operation " << compare->InputAt(0)->GetType(); 4921 } 4922} 4923 4924void InstructionCodeGeneratorARM::VisitCompare(HCompare* compare) { 4925 LocationSummary* locations = compare->GetLocations(); 4926 Register out = locations->Out().AsRegister<Register>(); 4927 Location left = locations->InAt(0); 4928 Location right = locations->InAt(1); 4929 4930 Label less, greater, done; 4931 Label* final_label = codegen_->GetFinalLabel(compare, &done); 4932 Primitive::Type type = compare->InputAt(0)->GetType(); 4933 Condition less_cond; 4934 switch (type) { 4935 case Primitive::kPrimBoolean: 4936 case Primitive::kPrimByte: 4937 case Primitive::kPrimShort: 4938 case Primitive::kPrimChar: 4939 case Primitive::kPrimInt: { 4940 __ LoadImmediate(out, 0); 4941 __ cmp(left.AsRegister<Register>(), 4942 ShifterOperand(right.AsRegister<Register>())); // Signed compare. 4943 less_cond = LT; 4944 break; 4945 } 4946 case Primitive::kPrimLong: { 4947 __ cmp(left.AsRegisterPairHigh<Register>(), 4948 ShifterOperand(right.AsRegisterPairHigh<Register>())); // Signed compare. 4949 __ b(&less, LT); 4950 __ b(&greater, GT); 4951 // Do LoadImmediate before the last `cmp`, as LoadImmediate might affect the status flags. 4952 __ LoadImmediate(out, 0); 4953 __ cmp(left.AsRegisterPairLow<Register>(), 4954 ShifterOperand(right.AsRegisterPairLow<Register>())); // Unsigned compare. 4955 less_cond = LO; 4956 break; 4957 } 4958 case Primitive::kPrimFloat: 4959 case Primitive::kPrimDouble: { 4960 __ LoadImmediate(out, 0); 4961 GenerateVcmp(compare, codegen_); 4962 __ vmstat(); // transfer FP status register to ARM APSR. 4963 less_cond = ARMFPCondition(kCondLT, compare->IsGtBias()); 4964 break; 4965 } 4966 default: 4967 LOG(FATAL) << "Unexpected compare type " << type; 4968 UNREACHABLE(); 4969 } 4970 4971 __ b(final_label, EQ); 4972 __ b(&less, less_cond); 4973 4974 __ Bind(&greater); 4975 __ LoadImmediate(out, 1); 4976 __ b(final_label); 4977 4978 __ Bind(&less); 4979 __ LoadImmediate(out, -1); 4980 4981 if (done.IsLinked()) { 4982 __ Bind(&done); 4983 } 4984} 4985 4986void LocationsBuilderARM::VisitPhi(HPhi* instruction) { 4987 LocationSummary* locations = 4988 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall); 4989 for (size_t i = 0, e = locations->GetInputCount(); i < e; ++i) { 4990 locations->SetInAt(i, Location::Any()); 4991 } 4992 locations->SetOut(Location::Any()); 4993} 4994 4995void InstructionCodeGeneratorARM::VisitPhi(HPhi* instruction ATTRIBUTE_UNUSED) { 4996 LOG(FATAL) << "Unreachable"; 4997} 4998 4999void CodeGeneratorARM::GenerateMemoryBarrier(MemBarrierKind kind) { 5000 // TODO (ported from quick): revisit ARM barrier kinds. 5001 DmbOptions flavor = DmbOptions::ISH; // Quiet C++ warnings. 5002 switch (kind) { 5003 case MemBarrierKind::kAnyStore: 5004 case MemBarrierKind::kLoadAny: 5005 case MemBarrierKind::kAnyAny: { 5006 flavor = DmbOptions::ISH; 5007 break; 5008 } 5009 case MemBarrierKind::kStoreStore: { 5010 flavor = DmbOptions::ISHST; 5011 break; 5012 } 5013 default: 5014 LOG(FATAL) << "Unexpected memory barrier " << kind; 5015 } 5016 __ dmb(flavor); 5017} 5018 5019void InstructionCodeGeneratorARM::GenerateWideAtomicLoad(Register addr, 5020 uint32_t offset, 5021 Register out_lo, 5022 Register out_hi) { 5023 if (offset != 0) { 5024 // Ensure `out_lo` is different from `addr`, so that loading 5025 // `offset` into `out_lo` does not clutter `addr`. 5026 DCHECK_NE(out_lo, addr); 5027 __ LoadImmediate(out_lo, offset); 5028 __ add(IP, addr, ShifterOperand(out_lo)); 5029 addr = IP; 5030 } 5031 __ ldrexd(out_lo, out_hi, addr); 5032} 5033 5034void InstructionCodeGeneratorARM::GenerateWideAtomicStore(Register addr, 5035 uint32_t offset, 5036 Register value_lo, 5037 Register value_hi, 5038 Register temp1, 5039 Register temp2, 5040 HInstruction* instruction) { 5041 Label fail; 5042 if (offset != 0) { 5043 __ LoadImmediate(temp1, offset); 5044 __ add(IP, addr, ShifterOperand(temp1)); 5045 addr = IP; 5046 } 5047 __ Bind(&fail); 5048 // We need a load followed by store. (The address used in a STREX instruction must 5049 // be the same as the address in the most recently executed LDREX instruction.) 5050 __ ldrexd(temp1, temp2, addr); 5051 codegen_->MaybeRecordImplicitNullCheck(instruction); 5052 __ strexd(temp1, value_lo, value_hi, addr); 5053 __ CompareAndBranchIfNonZero(temp1, &fail); 5054} 5055 5056void LocationsBuilderARM::HandleFieldSet(HInstruction* instruction, const FieldInfo& field_info) { 5057 DCHECK(instruction->IsInstanceFieldSet() || instruction->IsStaticFieldSet()); 5058 5059 LocationSummary* locations = 5060 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall); 5061 locations->SetInAt(0, Location::RequiresRegister()); 5062 5063 Primitive::Type field_type = field_info.GetFieldType(); 5064 if (Primitive::IsFloatingPointType(field_type)) { 5065 locations->SetInAt(1, Location::RequiresFpuRegister()); 5066 } else { 5067 locations->SetInAt(1, Location::RequiresRegister()); 5068 } 5069 5070 bool is_wide = field_type == Primitive::kPrimLong || field_type == Primitive::kPrimDouble; 5071 bool generate_volatile = field_info.IsVolatile() 5072 && is_wide 5073 && !codegen_->GetInstructionSetFeatures().HasAtomicLdrdAndStrd(); 5074 bool needs_write_barrier = 5075 CodeGenerator::StoreNeedsWriteBarrier(field_type, instruction->InputAt(1)); 5076 // Temporary registers for the write barrier. 5077 // TODO: consider renaming StoreNeedsWriteBarrier to StoreNeedsGCMark. 5078 if (needs_write_barrier) { 5079 locations->AddTemp(Location::RequiresRegister()); // Possibly used for reference poisoning too. 5080 locations->AddTemp(Location::RequiresRegister()); 5081 } else if (generate_volatile) { 5082 // ARM encoding have some additional constraints for ldrexd/strexd: 5083 // - registers need to be consecutive 5084 // - the first register should be even but not R14. 5085 // We don't test for ARM yet, and the assertion makes sure that we 5086 // revisit this if we ever enable ARM encoding. 5087 DCHECK_EQ(InstructionSet::kThumb2, codegen_->GetInstructionSet()); 5088 5089 locations->AddTemp(Location::RequiresRegister()); 5090 locations->AddTemp(Location::RequiresRegister()); 5091 if (field_type == Primitive::kPrimDouble) { 5092 // For doubles we need two more registers to copy the value. 5093 locations->AddTemp(Location::RegisterLocation(R2)); 5094 locations->AddTemp(Location::RegisterLocation(R3)); 5095 } 5096 } 5097} 5098 5099void InstructionCodeGeneratorARM::HandleFieldSet(HInstruction* instruction, 5100 const FieldInfo& field_info, 5101 bool value_can_be_null) { 5102 DCHECK(instruction->IsInstanceFieldSet() || instruction->IsStaticFieldSet()); 5103 5104 LocationSummary* locations = instruction->GetLocations(); 5105 Register base = locations->InAt(0).AsRegister<Register>(); 5106 Location value = locations->InAt(1); 5107 5108 bool is_volatile = field_info.IsVolatile(); 5109 bool atomic_ldrd_strd = codegen_->GetInstructionSetFeatures().HasAtomicLdrdAndStrd(); 5110 Primitive::Type field_type = field_info.GetFieldType(); 5111 uint32_t offset = field_info.GetFieldOffset().Uint32Value(); 5112 bool needs_write_barrier = 5113 CodeGenerator::StoreNeedsWriteBarrier(field_type, instruction->InputAt(1)); 5114 5115 if (is_volatile) { 5116 codegen_->GenerateMemoryBarrier(MemBarrierKind::kAnyStore); 5117 } 5118 5119 switch (field_type) { 5120 case Primitive::kPrimBoolean: 5121 case Primitive::kPrimByte: { 5122 __ StoreToOffset(kStoreByte, value.AsRegister<Register>(), base, offset); 5123 break; 5124 } 5125 5126 case Primitive::kPrimShort: 5127 case Primitive::kPrimChar: { 5128 __ StoreToOffset(kStoreHalfword, value.AsRegister<Register>(), base, offset); 5129 break; 5130 } 5131 5132 case Primitive::kPrimInt: 5133 case Primitive::kPrimNot: { 5134 if (kPoisonHeapReferences && needs_write_barrier) { 5135 // Note that in the case where `value` is a null reference, 5136 // we do not enter this block, as a null reference does not 5137 // need poisoning. 5138 DCHECK_EQ(field_type, Primitive::kPrimNot); 5139 Register temp = locations->GetTemp(0).AsRegister<Register>(); 5140 __ Mov(temp, value.AsRegister<Register>()); 5141 __ PoisonHeapReference(temp); 5142 __ StoreToOffset(kStoreWord, temp, base, offset); 5143 } else { 5144 __ StoreToOffset(kStoreWord, value.AsRegister<Register>(), base, offset); 5145 } 5146 break; 5147 } 5148 5149 case Primitive::kPrimLong: { 5150 if (is_volatile && !atomic_ldrd_strd) { 5151 GenerateWideAtomicStore(base, offset, 5152 value.AsRegisterPairLow<Register>(), 5153 value.AsRegisterPairHigh<Register>(), 5154 locations->GetTemp(0).AsRegister<Register>(), 5155 locations->GetTemp(1).AsRegister<Register>(), 5156 instruction); 5157 } else { 5158 __ StoreToOffset(kStoreWordPair, value.AsRegisterPairLow<Register>(), base, offset); 5159 codegen_->MaybeRecordImplicitNullCheck(instruction); 5160 } 5161 break; 5162 } 5163 5164 case Primitive::kPrimFloat: { 5165 __ StoreSToOffset(value.AsFpuRegister<SRegister>(), base, offset); 5166 break; 5167 } 5168 5169 case Primitive::kPrimDouble: { 5170 DRegister value_reg = FromLowSToD(value.AsFpuRegisterPairLow<SRegister>()); 5171 if (is_volatile && !atomic_ldrd_strd) { 5172 Register value_reg_lo = locations->GetTemp(0).AsRegister<Register>(); 5173 Register value_reg_hi = locations->GetTemp(1).AsRegister<Register>(); 5174 5175 __ vmovrrd(value_reg_lo, value_reg_hi, value_reg); 5176 5177 GenerateWideAtomicStore(base, offset, 5178 value_reg_lo, 5179 value_reg_hi, 5180 locations->GetTemp(2).AsRegister<Register>(), 5181 locations->GetTemp(3).AsRegister<Register>(), 5182 instruction); 5183 } else { 5184 __ StoreDToOffset(value_reg, base, offset); 5185 codegen_->MaybeRecordImplicitNullCheck(instruction); 5186 } 5187 break; 5188 } 5189 5190 case Primitive::kPrimVoid: 5191 LOG(FATAL) << "Unreachable type " << field_type; 5192 UNREACHABLE(); 5193 } 5194 5195 // Longs and doubles are handled in the switch. 5196 if (field_type != Primitive::kPrimLong && field_type != Primitive::kPrimDouble) { 5197 codegen_->MaybeRecordImplicitNullCheck(instruction); 5198 } 5199 5200 if (CodeGenerator::StoreNeedsWriteBarrier(field_type, instruction->InputAt(1))) { 5201 Register temp = locations->GetTemp(0).AsRegister<Register>(); 5202 Register card = locations->GetTemp(1).AsRegister<Register>(); 5203 codegen_->MarkGCCard( 5204 temp, card, base, value.AsRegister<Register>(), value_can_be_null); 5205 } 5206 5207 if (is_volatile) { 5208 codegen_->GenerateMemoryBarrier(MemBarrierKind::kAnyAny); 5209 } 5210} 5211 5212void LocationsBuilderARM::HandleFieldGet(HInstruction* instruction, const FieldInfo& field_info) { 5213 DCHECK(instruction->IsInstanceFieldGet() || instruction->IsStaticFieldGet()); 5214 5215 bool object_field_get_with_read_barrier = 5216 kEmitCompilerReadBarrier && (field_info.GetFieldType() == Primitive::kPrimNot); 5217 LocationSummary* locations = 5218 new (GetGraph()->GetArena()) LocationSummary(instruction, 5219 object_field_get_with_read_barrier ? 5220 LocationSummary::kCallOnSlowPath : 5221 LocationSummary::kNoCall); 5222 if (object_field_get_with_read_barrier && kUseBakerReadBarrier) { 5223 locations->SetCustomSlowPathCallerSaves(RegisterSet::Empty()); // No caller-save registers. 5224 } 5225 locations->SetInAt(0, Location::RequiresRegister()); 5226 5227 bool volatile_for_double = field_info.IsVolatile() 5228 && (field_info.GetFieldType() == Primitive::kPrimDouble) 5229 && !codegen_->GetInstructionSetFeatures().HasAtomicLdrdAndStrd(); 5230 // The output overlaps in case of volatile long: we don't want the 5231 // code generated by GenerateWideAtomicLoad to overwrite the 5232 // object's location. Likewise, in the case of an object field get 5233 // with read barriers enabled, we do not want the load to overwrite 5234 // the object's location, as we need it to emit the read barrier. 5235 bool overlap = (field_info.IsVolatile() && (field_info.GetFieldType() == Primitive::kPrimLong)) || 5236 object_field_get_with_read_barrier; 5237 5238 if (Primitive::IsFloatingPointType(instruction->GetType())) { 5239 locations->SetOut(Location::RequiresFpuRegister()); 5240 } else { 5241 locations->SetOut(Location::RequiresRegister(), 5242 (overlap ? Location::kOutputOverlap : Location::kNoOutputOverlap)); 5243 } 5244 if (volatile_for_double) { 5245 // ARM encoding have some additional constraints for ldrexd/strexd: 5246 // - registers need to be consecutive 5247 // - the first register should be even but not R14. 5248 // We don't test for ARM yet, and the assertion makes sure that we 5249 // revisit this if we ever enable ARM encoding. 5250 DCHECK_EQ(InstructionSet::kThumb2, codegen_->GetInstructionSet()); 5251 locations->AddTemp(Location::RequiresRegister()); 5252 locations->AddTemp(Location::RequiresRegister()); 5253 } else if (object_field_get_with_read_barrier && kUseBakerReadBarrier) { 5254 // We need a temporary register for the read barrier marking slow 5255 // path in CodeGeneratorARM::GenerateFieldLoadWithBakerReadBarrier. 5256 locations->AddTemp(Location::RequiresRegister()); 5257 } 5258} 5259 5260Location LocationsBuilderARM::ArithmeticZeroOrFpuRegister(HInstruction* input) { 5261 DCHECK(input->GetType() == Primitive::kPrimDouble || input->GetType() == Primitive::kPrimFloat) 5262 << input->GetType(); 5263 if ((input->IsFloatConstant() && (input->AsFloatConstant()->IsArithmeticZero())) || 5264 (input->IsDoubleConstant() && (input->AsDoubleConstant()->IsArithmeticZero()))) { 5265 return Location::ConstantLocation(input->AsConstant()); 5266 } else { 5267 return Location::RequiresFpuRegister(); 5268 } 5269} 5270 5271Location LocationsBuilderARM::ArmEncodableConstantOrRegister(HInstruction* constant, 5272 Opcode opcode) { 5273 DCHECK(!Primitive::IsFloatingPointType(constant->GetType())); 5274 if (constant->IsConstant() && 5275 CanEncodeConstantAsImmediate(constant->AsConstant(), opcode)) { 5276 return Location::ConstantLocation(constant->AsConstant()); 5277 } 5278 return Location::RequiresRegister(); 5279} 5280 5281bool LocationsBuilderARM::CanEncodeConstantAsImmediate(HConstant* input_cst, 5282 Opcode opcode) { 5283 uint64_t value = static_cast<uint64_t>(Int64FromConstant(input_cst)); 5284 if (Primitive::Is64BitType(input_cst->GetType())) { 5285 Opcode high_opcode = opcode; 5286 SetCc low_set_cc = kCcDontCare; 5287 switch (opcode) { 5288 case SUB: 5289 // Flip the operation to an ADD. 5290 value = -value; 5291 opcode = ADD; 5292 FALLTHROUGH_INTENDED; 5293 case ADD: 5294 if (Low32Bits(value) == 0u) { 5295 return CanEncodeConstantAsImmediate(High32Bits(value), opcode, kCcDontCare); 5296 } 5297 high_opcode = ADC; 5298 low_set_cc = kCcSet; 5299 break; 5300 default: 5301 break; 5302 } 5303 return CanEncodeConstantAsImmediate(Low32Bits(value), opcode, low_set_cc) && 5304 CanEncodeConstantAsImmediate(High32Bits(value), high_opcode, kCcDontCare); 5305 } else { 5306 return CanEncodeConstantAsImmediate(Low32Bits(value), opcode); 5307 } 5308} 5309 5310bool LocationsBuilderARM::CanEncodeConstantAsImmediate(uint32_t value, 5311 Opcode opcode, 5312 SetCc set_cc) { 5313 ShifterOperand so; 5314 ArmAssembler* assembler = codegen_->GetAssembler(); 5315 if (assembler->ShifterOperandCanHold(kNoRegister, kNoRegister, opcode, value, set_cc, &so)) { 5316 return true; 5317 } 5318 Opcode neg_opcode = kNoOperand; 5319 uint32_t neg_value = 0; 5320 switch (opcode) { 5321 case AND: neg_opcode = BIC; neg_value = ~value; break; 5322 case ORR: neg_opcode = ORN; neg_value = ~value; break; 5323 case ADD: neg_opcode = SUB; neg_value = -value; break; 5324 case ADC: neg_opcode = SBC; neg_value = ~value; break; 5325 case SUB: neg_opcode = ADD; neg_value = -value; break; 5326 case SBC: neg_opcode = ADC; neg_value = ~value; break; 5327 case MOV: neg_opcode = MVN; neg_value = ~value; break; 5328 default: 5329 return false; 5330 } 5331 5332 if (assembler->ShifterOperandCanHold(kNoRegister, 5333 kNoRegister, 5334 neg_opcode, 5335 neg_value, 5336 set_cc, 5337 &so)) { 5338 return true; 5339 } 5340 5341 return opcode == AND && IsPowerOfTwo(value + 1); 5342} 5343 5344void InstructionCodeGeneratorARM::HandleFieldGet(HInstruction* instruction, 5345 const FieldInfo& field_info) { 5346 DCHECK(instruction->IsInstanceFieldGet() || instruction->IsStaticFieldGet()); 5347 5348 LocationSummary* locations = instruction->GetLocations(); 5349 Location base_loc = locations->InAt(0); 5350 Register base = base_loc.AsRegister<Register>(); 5351 Location out = locations->Out(); 5352 bool is_volatile = field_info.IsVolatile(); 5353 bool atomic_ldrd_strd = codegen_->GetInstructionSetFeatures().HasAtomicLdrdAndStrd(); 5354 Primitive::Type field_type = field_info.GetFieldType(); 5355 uint32_t offset = field_info.GetFieldOffset().Uint32Value(); 5356 5357 switch (field_type) { 5358 case Primitive::kPrimBoolean: 5359 __ LoadFromOffset(kLoadUnsignedByte, out.AsRegister<Register>(), base, offset); 5360 break; 5361 5362 case Primitive::kPrimByte: 5363 __ LoadFromOffset(kLoadSignedByte, out.AsRegister<Register>(), base, offset); 5364 break; 5365 5366 case Primitive::kPrimShort: 5367 __ LoadFromOffset(kLoadSignedHalfword, out.AsRegister<Register>(), base, offset); 5368 break; 5369 5370 case Primitive::kPrimChar: 5371 __ LoadFromOffset(kLoadUnsignedHalfword, out.AsRegister<Register>(), base, offset); 5372 break; 5373 5374 case Primitive::kPrimInt: 5375 __ LoadFromOffset(kLoadWord, out.AsRegister<Register>(), base, offset); 5376 break; 5377 5378 case Primitive::kPrimNot: { 5379 // /* HeapReference<Object> */ out = *(base + offset) 5380 if (kEmitCompilerReadBarrier && kUseBakerReadBarrier) { 5381 Location temp_loc = locations->GetTemp(0); 5382 // Note that a potential implicit null check is handled in this 5383 // CodeGeneratorARM::GenerateFieldLoadWithBakerReadBarrier call. 5384 codegen_->GenerateFieldLoadWithBakerReadBarrier( 5385 instruction, out, base, offset, temp_loc, /* needs_null_check */ true); 5386 if (is_volatile) { 5387 codegen_->GenerateMemoryBarrier(MemBarrierKind::kLoadAny); 5388 } 5389 } else { 5390 __ LoadFromOffset(kLoadWord, out.AsRegister<Register>(), base, offset); 5391 codegen_->MaybeRecordImplicitNullCheck(instruction); 5392 if (is_volatile) { 5393 codegen_->GenerateMemoryBarrier(MemBarrierKind::kLoadAny); 5394 } 5395 // If read barriers are enabled, emit read barriers other than 5396 // Baker's using a slow path (and also unpoison the loaded 5397 // reference, if heap poisoning is enabled). 5398 codegen_->MaybeGenerateReadBarrierSlow(instruction, out, out, base_loc, offset); 5399 } 5400 break; 5401 } 5402 5403 case Primitive::kPrimLong: 5404 if (is_volatile && !atomic_ldrd_strd) { 5405 GenerateWideAtomicLoad(base, offset, 5406 out.AsRegisterPairLow<Register>(), 5407 out.AsRegisterPairHigh<Register>()); 5408 } else { 5409 __ LoadFromOffset(kLoadWordPair, out.AsRegisterPairLow<Register>(), base, offset); 5410 } 5411 break; 5412 5413 case Primitive::kPrimFloat: 5414 __ LoadSFromOffset(out.AsFpuRegister<SRegister>(), base, offset); 5415 break; 5416 5417 case Primitive::kPrimDouble: { 5418 DRegister out_reg = FromLowSToD(out.AsFpuRegisterPairLow<SRegister>()); 5419 if (is_volatile && !atomic_ldrd_strd) { 5420 Register lo = locations->GetTemp(0).AsRegister<Register>(); 5421 Register hi = locations->GetTemp(1).AsRegister<Register>(); 5422 GenerateWideAtomicLoad(base, offset, lo, hi); 5423 codegen_->MaybeRecordImplicitNullCheck(instruction); 5424 __ vmovdrr(out_reg, lo, hi); 5425 } else { 5426 __ LoadDFromOffset(out_reg, base, offset); 5427 codegen_->MaybeRecordImplicitNullCheck(instruction); 5428 } 5429 break; 5430 } 5431 5432 case Primitive::kPrimVoid: 5433 LOG(FATAL) << "Unreachable type " << field_type; 5434 UNREACHABLE(); 5435 } 5436 5437 if (field_type == Primitive::kPrimNot || field_type == Primitive::kPrimDouble) { 5438 // Potential implicit null checks, in the case of reference or 5439 // double fields, are handled in the previous switch statement. 5440 } else { 5441 codegen_->MaybeRecordImplicitNullCheck(instruction); 5442 } 5443 5444 if (is_volatile) { 5445 if (field_type == Primitive::kPrimNot) { 5446 // Memory barriers, in the case of references, are also handled 5447 // in the previous switch statement. 5448 } else { 5449 codegen_->GenerateMemoryBarrier(MemBarrierKind::kLoadAny); 5450 } 5451 } 5452} 5453 5454void LocationsBuilderARM::VisitInstanceFieldSet(HInstanceFieldSet* instruction) { 5455 HandleFieldSet(instruction, instruction->GetFieldInfo()); 5456} 5457 5458void InstructionCodeGeneratorARM::VisitInstanceFieldSet(HInstanceFieldSet* instruction) { 5459 HandleFieldSet(instruction, instruction->GetFieldInfo(), instruction->GetValueCanBeNull()); 5460} 5461 5462void LocationsBuilderARM::VisitInstanceFieldGet(HInstanceFieldGet* instruction) { 5463 HandleFieldGet(instruction, instruction->GetFieldInfo()); 5464} 5465 5466void InstructionCodeGeneratorARM::VisitInstanceFieldGet(HInstanceFieldGet* instruction) { 5467 HandleFieldGet(instruction, instruction->GetFieldInfo()); 5468} 5469 5470void LocationsBuilderARM::VisitStaticFieldGet(HStaticFieldGet* instruction) { 5471 HandleFieldGet(instruction, instruction->GetFieldInfo()); 5472} 5473 5474void InstructionCodeGeneratorARM::VisitStaticFieldGet(HStaticFieldGet* instruction) { 5475 HandleFieldGet(instruction, instruction->GetFieldInfo()); 5476} 5477 5478void LocationsBuilderARM::VisitStaticFieldSet(HStaticFieldSet* instruction) { 5479 HandleFieldSet(instruction, instruction->GetFieldInfo()); 5480} 5481 5482void InstructionCodeGeneratorARM::VisitStaticFieldSet(HStaticFieldSet* instruction) { 5483 HandleFieldSet(instruction, instruction->GetFieldInfo(), instruction->GetValueCanBeNull()); 5484} 5485 5486void LocationsBuilderARM::VisitUnresolvedInstanceFieldGet( 5487 HUnresolvedInstanceFieldGet* instruction) { 5488 FieldAccessCallingConventionARM calling_convention; 5489 codegen_->CreateUnresolvedFieldLocationSummary( 5490 instruction, instruction->GetFieldType(), calling_convention); 5491} 5492 5493void InstructionCodeGeneratorARM::VisitUnresolvedInstanceFieldGet( 5494 HUnresolvedInstanceFieldGet* instruction) { 5495 FieldAccessCallingConventionARM calling_convention; 5496 codegen_->GenerateUnresolvedFieldAccess(instruction, 5497 instruction->GetFieldType(), 5498 instruction->GetFieldIndex(), 5499 instruction->GetDexPc(), 5500 calling_convention); 5501} 5502 5503void LocationsBuilderARM::VisitUnresolvedInstanceFieldSet( 5504 HUnresolvedInstanceFieldSet* instruction) { 5505 FieldAccessCallingConventionARM calling_convention; 5506 codegen_->CreateUnresolvedFieldLocationSummary( 5507 instruction, instruction->GetFieldType(), calling_convention); 5508} 5509 5510void InstructionCodeGeneratorARM::VisitUnresolvedInstanceFieldSet( 5511 HUnresolvedInstanceFieldSet* instruction) { 5512 FieldAccessCallingConventionARM calling_convention; 5513 codegen_->GenerateUnresolvedFieldAccess(instruction, 5514 instruction->GetFieldType(), 5515 instruction->GetFieldIndex(), 5516 instruction->GetDexPc(), 5517 calling_convention); 5518} 5519 5520void LocationsBuilderARM::VisitUnresolvedStaticFieldGet( 5521 HUnresolvedStaticFieldGet* instruction) { 5522 FieldAccessCallingConventionARM calling_convention; 5523 codegen_->CreateUnresolvedFieldLocationSummary( 5524 instruction, instruction->GetFieldType(), calling_convention); 5525} 5526 5527void InstructionCodeGeneratorARM::VisitUnresolvedStaticFieldGet( 5528 HUnresolvedStaticFieldGet* instruction) { 5529 FieldAccessCallingConventionARM calling_convention; 5530 codegen_->GenerateUnresolvedFieldAccess(instruction, 5531 instruction->GetFieldType(), 5532 instruction->GetFieldIndex(), 5533 instruction->GetDexPc(), 5534 calling_convention); 5535} 5536 5537void LocationsBuilderARM::VisitUnresolvedStaticFieldSet( 5538 HUnresolvedStaticFieldSet* instruction) { 5539 FieldAccessCallingConventionARM calling_convention; 5540 codegen_->CreateUnresolvedFieldLocationSummary( 5541 instruction, instruction->GetFieldType(), calling_convention); 5542} 5543 5544void InstructionCodeGeneratorARM::VisitUnresolvedStaticFieldSet( 5545 HUnresolvedStaticFieldSet* instruction) { 5546 FieldAccessCallingConventionARM calling_convention; 5547 codegen_->GenerateUnresolvedFieldAccess(instruction, 5548 instruction->GetFieldType(), 5549 instruction->GetFieldIndex(), 5550 instruction->GetDexPc(), 5551 calling_convention); 5552} 5553 5554void LocationsBuilderARM::VisitNullCheck(HNullCheck* instruction) { 5555 LocationSummary* locations = codegen_->CreateThrowingSlowPathLocations(instruction); 5556 locations->SetInAt(0, Location::RequiresRegister()); 5557} 5558 5559void CodeGeneratorARM::GenerateImplicitNullCheck(HNullCheck* instruction) { 5560 if (CanMoveNullCheckToUser(instruction)) { 5561 return; 5562 } 5563 Location obj = instruction->GetLocations()->InAt(0); 5564 5565 __ LoadFromOffset(kLoadWord, IP, obj.AsRegister<Register>(), 0); 5566 RecordPcInfo(instruction, instruction->GetDexPc()); 5567} 5568 5569void CodeGeneratorARM::GenerateExplicitNullCheck(HNullCheck* instruction) { 5570 SlowPathCodeARM* slow_path = new (GetGraph()->GetArena()) NullCheckSlowPathARM(instruction); 5571 AddSlowPath(slow_path); 5572 5573 LocationSummary* locations = instruction->GetLocations(); 5574 Location obj = locations->InAt(0); 5575 5576 __ CompareAndBranchIfZero(obj.AsRegister<Register>(), slow_path->GetEntryLabel()); 5577} 5578 5579void InstructionCodeGeneratorARM::VisitNullCheck(HNullCheck* instruction) { 5580 codegen_->GenerateNullCheck(instruction); 5581} 5582 5583static LoadOperandType GetLoadOperandType(Primitive::Type type) { 5584 switch (type) { 5585 case Primitive::kPrimNot: 5586 return kLoadWord; 5587 case Primitive::kPrimBoolean: 5588 return kLoadUnsignedByte; 5589 case Primitive::kPrimByte: 5590 return kLoadSignedByte; 5591 case Primitive::kPrimChar: 5592 return kLoadUnsignedHalfword; 5593 case Primitive::kPrimShort: 5594 return kLoadSignedHalfword; 5595 case Primitive::kPrimInt: 5596 return kLoadWord; 5597 case Primitive::kPrimLong: 5598 return kLoadWordPair; 5599 case Primitive::kPrimFloat: 5600 return kLoadSWord; 5601 case Primitive::kPrimDouble: 5602 return kLoadDWord; 5603 default: 5604 LOG(FATAL) << "Unreachable type " << type; 5605 UNREACHABLE(); 5606 } 5607} 5608 5609static StoreOperandType GetStoreOperandType(Primitive::Type type) { 5610 switch (type) { 5611 case Primitive::kPrimNot: 5612 return kStoreWord; 5613 case Primitive::kPrimBoolean: 5614 case Primitive::kPrimByte: 5615 return kStoreByte; 5616 case Primitive::kPrimChar: 5617 case Primitive::kPrimShort: 5618 return kStoreHalfword; 5619 case Primitive::kPrimInt: 5620 return kStoreWord; 5621 case Primitive::kPrimLong: 5622 return kStoreWordPair; 5623 case Primitive::kPrimFloat: 5624 return kStoreSWord; 5625 case Primitive::kPrimDouble: 5626 return kStoreDWord; 5627 default: 5628 LOG(FATAL) << "Unreachable type " << type; 5629 UNREACHABLE(); 5630 } 5631} 5632 5633void CodeGeneratorARM::LoadFromShiftedRegOffset(Primitive::Type type, 5634 Location out_loc, 5635 Register base, 5636 Register reg_offset, 5637 Condition cond) { 5638 uint32_t shift_count = Primitive::ComponentSizeShift(type); 5639 Address mem_address(base, reg_offset, Shift::LSL, shift_count); 5640 5641 switch (type) { 5642 case Primitive::kPrimByte: 5643 __ ldrsb(out_loc.AsRegister<Register>(), mem_address, cond); 5644 break; 5645 case Primitive::kPrimBoolean: 5646 __ ldrb(out_loc.AsRegister<Register>(), mem_address, cond); 5647 break; 5648 case Primitive::kPrimShort: 5649 __ ldrsh(out_loc.AsRegister<Register>(), mem_address, cond); 5650 break; 5651 case Primitive::kPrimChar: 5652 __ ldrh(out_loc.AsRegister<Register>(), mem_address, cond); 5653 break; 5654 case Primitive::kPrimNot: 5655 case Primitive::kPrimInt: 5656 __ ldr(out_loc.AsRegister<Register>(), mem_address, cond); 5657 break; 5658 // T32 doesn't support LoadFromShiftedRegOffset mem address mode for these types. 5659 case Primitive::kPrimLong: 5660 case Primitive::kPrimFloat: 5661 case Primitive::kPrimDouble: 5662 default: 5663 LOG(FATAL) << "Unreachable type " << type; 5664 UNREACHABLE(); 5665 } 5666} 5667 5668void CodeGeneratorARM::StoreToShiftedRegOffset(Primitive::Type type, 5669 Location loc, 5670 Register base, 5671 Register reg_offset, 5672 Condition cond) { 5673 uint32_t shift_count = Primitive::ComponentSizeShift(type); 5674 Address mem_address(base, reg_offset, Shift::LSL, shift_count); 5675 5676 switch (type) { 5677 case Primitive::kPrimByte: 5678 case Primitive::kPrimBoolean: 5679 __ strb(loc.AsRegister<Register>(), mem_address, cond); 5680 break; 5681 case Primitive::kPrimShort: 5682 case Primitive::kPrimChar: 5683 __ strh(loc.AsRegister<Register>(), mem_address, cond); 5684 break; 5685 case Primitive::kPrimNot: 5686 case Primitive::kPrimInt: 5687 __ str(loc.AsRegister<Register>(), mem_address, cond); 5688 break; 5689 // T32 doesn't support StoreToShiftedRegOffset mem address mode for these types. 5690 case Primitive::kPrimLong: 5691 case Primitive::kPrimFloat: 5692 case Primitive::kPrimDouble: 5693 default: 5694 LOG(FATAL) << "Unreachable type " << type; 5695 UNREACHABLE(); 5696 } 5697} 5698 5699void LocationsBuilderARM::VisitArrayGet(HArrayGet* instruction) { 5700 bool object_array_get_with_read_barrier = 5701 kEmitCompilerReadBarrier && (instruction->GetType() == Primitive::kPrimNot); 5702 LocationSummary* locations = 5703 new (GetGraph()->GetArena()) LocationSummary(instruction, 5704 object_array_get_with_read_barrier ? 5705 LocationSummary::kCallOnSlowPath : 5706 LocationSummary::kNoCall); 5707 if (object_array_get_with_read_barrier && kUseBakerReadBarrier) { 5708 locations->SetCustomSlowPathCallerSaves(RegisterSet::Empty()); // No caller-save registers. 5709 } 5710 locations->SetInAt(0, Location::RequiresRegister()); 5711 locations->SetInAt(1, Location::RegisterOrConstant(instruction->InputAt(1))); 5712 if (Primitive::IsFloatingPointType(instruction->GetType())) { 5713 locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap); 5714 } else { 5715 // The output overlaps in the case of an object array get with 5716 // read barriers enabled: we do not want the move to overwrite the 5717 // array's location, as we need it to emit the read barrier. 5718 locations->SetOut( 5719 Location::RequiresRegister(), 5720 object_array_get_with_read_barrier ? Location::kOutputOverlap : Location::kNoOutputOverlap); 5721 } 5722 // We need a temporary register for the read barrier marking slow 5723 // path in CodeGeneratorARM::GenerateArrayLoadWithBakerReadBarrier. 5724 // Also need for String compression feature. 5725 if ((object_array_get_with_read_barrier && kUseBakerReadBarrier) 5726 || (mirror::kUseStringCompression && instruction->IsStringCharAt())) { 5727 locations->AddTemp(Location::RequiresRegister()); 5728 } 5729} 5730 5731void InstructionCodeGeneratorARM::VisitArrayGet(HArrayGet* instruction) { 5732 LocationSummary* locations = instruction->GetLocations(); 5733 Location obj_loc = locations->InAt(0); 5734 Register obj = obj_loc.AsRegister<Register>(); 5735 Location index = locations->InAt(1); 5736 Location out_loc = locations->Out(); 5737 uint32_t data_offset = CodeGenerator::GetArrayDataOffset(instruction); 5738 Primitive::Type type = instruction->GetType(); 5739 const bool maybe_compressed_char_at = mirror::kUseStringCompression && 5740 instruction->IsStringCharAt(); 5741 HInstruction* array_instr = instruction->GetArray(); 5742 bool has_intermediate_address = array_instr->IsIntermediateAddress(); 5743 5744 switch (type) { 5745 case Primitive::kPrimBoolean: 5746 case Primitive::kPrimByte: 5747 case Primitive::kPrimShort: 5748 case Primitive::kPrimChar: 5749 case Primitive::kPrimInt: { 5750 Register length; 5751 if (maybe_compressed_char_at) { 5752 length = locations->GetTemp(0).AsRegister<Register>(); 5753 uint32_t count_offset = mirror::String::CountOffset().Uint32Value(); 5754 __ LoadFromOffset(kLoadWord, length, obj, count_offset); 5755 codegen_->MaybeRecordImplicitNullCheck(instruction); 5756 } 5757 if (index.IsConstant()) { 5758 int32_t const_index = index.GetConstant()->AsIntConstant()->GetValue(); 5759 if (maybe_compressed_char_at) { 5760 Label uncompressed_load, done; 5761 Label* final_label = codegen_->GetFinalLabel(instruction, &done); 5762 __ Lsrs(length, length, 1u); // LSRS has a 16-bit encoding, TST (immediate) does not. 5763 static_assert(static_cast<uint32_t>(mirror::StringCompressionFlag::kCompressed) == 0u, 5764 "Expecting 0=compressed, 1=uncompressed"); 5765 __ b(&uncompressed_load, CS); 5766 __ LoadFromOffset(kLoadUnsignedByte, 5767 out_loc.AsRegister<Register>(), 5768 obj, 5769 data_offset + const_index); 5770 __ b(final_label); 5771 __ Bind(&uncompressed_load); 5772 __ LoadFromOffset(GetLoadOperandType(Primitive::kPrimChar), 5773 out_loc.AsRegister<Register>(), 5774 obj, 5775 data_offset + (const_index << 1)); 5776 if (done.IsLinked()) { 5777 __ Bind(&done); 5778 } 5779 } else { 5780 uint32_t full_offset = data_offset + (const_index << Primitive::ComponentSizeShift(type)); 5781 5782 LoadOperandType load_type = GetLoadOperandType(type); 5783 __ LoadFromOffset(load_type, out_loc.AsRegister<Register>(), obj, full_offset); 5784 } 5785 } else { 5786 Register temp = IP; 5787 5788 if (has_intermediate_address) { 5789 // We do not need to compute the intermediate address from the array: the 5790 // input instruction has done it already. See the comment in 5791 // `TryExtractArrayAccessAddress()`. 5792 if (kIsDebugBuild) { 5793 HIntermediateAddress* tmp = array_instr->AsIntermediateAddress(); 5794 DCHECK_EQ(tmp->GetOffset()->AsIntConstant()->GetValueAsUint64(), data_offset); 5795 } 5796 temp = obj; 5797 } else { 5798 __ add(temp, obj, ShifterOperand(data_offset)); 5799 } 5800 if (maybe_compressed_char_at) { 5801 Label uncompressed_load, done; 5802 Label* final_label = codegen_->GetFinalLabel(instruction, &done); 5803 __ Lsrs(length, length, 1u); // LSRS has a 16-bit encoding, TST (immediate) does not. 5804 static_assert(static_cast<uint32_t>(mirror::StringCompressionFlag::kCompressed) == 0u, 5805 "Expecting 0=compressed, 1=uncompressed"); 5806 __ b(&uncompressed_load, CS); 5807 __ ldrb(out_loc.AsRegister<Register>(), 5808 Address(temp, index.AsRegister<Register>(), Shift::LSL, 0)); 5809 __ b(final_label); 5810 __ Bind(&uncompressed_load); 5811 __ ldrh(out_loc.AsRegister<Register>(), 5812 Address(temp, index.AsRegister<Register>(), Shift::LSL, 1)); 5813 if (done.IsLinked()) { 5814 __ Bind(&done); 5815 } 5816 } else { 5817 codegen_->LoadFromShiftedRegOffset(type, out_loc, temp, index.AsRegister<Register>()); 5818 } 5819 } 5820 break; 5821 } 5822 5823 case Primitive::kPrimNot: { 5824 // The read barrier instrumentation of object ArrayGet 5825 // instructions does not support the HIntermediateAddress 5826 // instruction. 5827 DCHECK(!(has_intermediate_address && kEmitCompilerReadBarrier)); 5828 5829 static_assert( 5830 sizeof(mirror::HeapReference<mirror::Object>) == sizeof(int32_t), 5831 "art::mirror::HeapReference<art::mirror::Object> and int32_t have different sizes."); 5832 // /* HeapReference<Object> */ out = 5833 // *(obj + data_offset + index * sizeof(HeapReference<Object>)) 5834 if (kEmitCompilerReadBarrier && kUseBakerReadBarrier) { 5835 Location temp = locations->GetTemp(0); 5836 // Note that a potential implicit null check is handled in this 5837 // CodeGeneratorARM::GenerateArrayLoadWithBakerReadBarrier call. 5838 codegen_->GenerateArrayLoadWithBakerReadBarrier( 5839 instruction, out_loc, obj, data_offset, index, temp, /* needs_null_check */ true); 5840 } else { 5841 Register out = out_loc.AsRegister<Register>(); 5842 if (index.IsConstant()) { 5843 size_t offset = 5844 (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4) + data_offset; 5845 __ LoadFromOffset(kLoadWord, out, obj, offset); 5846 codegen_->MaybeRecordImplicitNullCheck(instruction); 5847 // If read barriers are enabled, emit read barriers other than 5848 // Baker's using a slow path (and also unpoison the loaded 5849 // reference, if heap poisoning is enabled). 5850 codegen_->MaybeGenerateReadBarrierSlow(instruction, out_loc, out_loc, obj_loc, offset); 5851 } else { 5852 Register temp = IP; 5853 5854 if (has_intermediate_address) { 5855 // We do not need to compute the intermediate address from the array: the 5856 // input instruction has done it already. See the comment in 5857 // `TryExtractArrayAccessAddress()`. 5858 if (kIsDebugBuild) { 5859 HIntermediateAddress* tmp = array_instr->AsIntermediateAddress(); 5860 DCHECK_EQ(tmp->GetOffset()->AsIntConstant()->GetValueAsUint64(), data_offset); 5861 } 5862 temp = obj; 5863 } else { 5864 __ add(temp, obj, ShifterOperand(data_offset)); 5865 } 5866 codegen_->LoadFromShiftedRegOffset(type, out_loc, temp, index.AsRegister<Register>()); 5867 5868 codegen_->MaybeRecordImplicitNullCheck(instruction); 5869 // If read barriers are enabled, emit read barriers other than 5870 // Baker's using a slow path (and also unpoison the loaded 5871 // reference, if heap poisoning is enabled). 5872 codegen_->MaybeGenerateReadBarrierSlow( 5873 instruction, out_loc, out_loc, obj_loc, data_offset, index); 5874 } 5875 } 5876 break; 5877 } 5878 5879 case Primitive::kPrimLong: { 5880 if (index.IsConstant()) { 5881 size_t offset = 5882 (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_8) + data_offset; 5883 __ LoadFromOffset(kLoadWordPair, out_loc.AsRegisterPairLow<Register>(), obj, offset); 5884 } else { 5885 __ add(IP, obj, ShifterOperand(index.AsRegister<Register>(), LSL, TIMES_8)); 5886 __ LoadFromOffset(kLoadWordPair, out_loc.AsRegisterPairLow<Register>(), IP, data_offset); 5887 } 5888 break; 5889 } 5890 5891 case Primitive::kPrimFloat: { 5892 SRegister out = out_loc.AsFpuRegister<SRegister>(); 5893 if (index.IsConstant()) { 5894 size_t offset = (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4) + data_offset; 5895 __ LoadSFromOffset(out, obj, offset); 5896 } else { 5897 __ add(IP, obj, ShifterOperand(index.AsRegister<Register>(), LSL, TIMES_4)); 5898 __ LoadSFromOffset(out, IP, data_offset); 5899 } 5900 break; 5901 } 5902 5903 case Primitive::kPrimDouble: { 5904 SRegister out = out_loc.AsFpuRegisterPairLow<SRegister>(); 5905 if (index.IsConstant()) { 5906 size_t offset = (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_8) + data_offset; 5907 __ LoadDFromOffset(FromLowSToD(out), obj, offset); 5908 } else { 5909 __ add(IP, obj, ShifterOperand(index.AsRegister<Register>(), LSL, TIMES_8)); 5910 __ LoadDFromOffset(FromLowSToD(out), IP, data_offset); 5911 } 5912 break; 5913 } 5914 5915 case Primitive::kPrimVoid: 5916 LOG(FATAL) << "Unreachable type " << type; 5917 UNREACHABLE(); 5918 } 5919 5920 if (type == Primitive::kPrimNot) { 5921 // Potential implicit null checks, in the case of reference 5922 // arrays, are handled in the previous switch statement. 5923 } else if (!maybe_compressed_char_at) { 5924 codegen_->MaybeRecordImplicitNullCheck(instruction); 5925 } 5926} 5927 5928void LocationsBuilderARM::VisitArraySet(HArraySet* instruction) { 5929 Primitive::Type value_type = instruction->GetComponentType(); 5930 5931 bool needs_write_barrier = 5932 CodeGenerator::StoreNeedsWriteBarrier(value_type, instruction->GetValue()); 5933 bool may_need_runtime_call_for_type_check = instruction->NeedsTypeCheck(); 5934 5935 LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary( 5936 instruction, 5937 may_need_runtime_call_for_type_check ? 5938 LocationSummary::kCallOnSlowPath : 5939 LocationSummary::kNoCall); 5940 5941 locations->SetInAt(0, Location::RequiresRegister()); 5942 locations->SetInAt(1, Location::RegisterOrConstant(instruction->InputAt(1))); 5943 if (Primitive::IsFloatingPointType(value_type)) { 5944 locations->SetInAt(2, Location::RequiresFpuRegister()); 5945 } else { 5946 locations->SetInAt(2, Location::RequiresRegister()); 5947 } 5948 if (needs_write_barrier) { 5949 // Temporary registers for the write barrier. 5950 locations->AddTemp(Location::RequiresRegister()); // Possibly used for ref. poisoning too. 5951 locations->AddTemp(Location::RequiresRegister()); 5952 } 5953} 5954 5955void InstructionCodeGeneratorARM::VisitArraySet(HArraySet* instruction) { 5956 LocationSummary* locations = instruction->GetLocations(); 5957 Location array_loc = locations->InAt(0); 5958 Register array = array_loc.AsRegister<Register>(); 5959 Location index = locations->InAt(1); 5960 Primitive::Type value_type = instruction->GetComponentType(); 5961 bool may_need_runtime_call_for_type_check = instruction->NeedsTypeCheck(); 5962 bool needs_write_barrier = 5963 CodeGenerator::StoreNeedsWriteBarrier(value_type, instruction->GetValue()); 5964 uint32_t data_offset = 5965 mirror::Array::DataOffset(Primitive::ComponentSize(value_type)).Uint32Value(); 5966 Location value_loc = locations->InAt(2); 5967 HInstruction* array_instr = instruction->GetArray(); 5968 bool has_intermediate_address = array_instr->IsIntermediateAddress(); 5969 5970 switch (value_type) { 5971 case Primitive::kPrimBoolean: 5972 case Primitive::kPrimByte: 5973 case Primitive::kPrimShort: 5974 case Primitive::kPrimChar: 5975 case Primitive::kPrimInt: { 5976 if (index.IsConstant()) { 5977 int32_t const_index = index.GetConstant()->AsIntConstant()->GetValue(); 5978 uint32_t full_offset = 5979 data_offset + (const_index << Primitive::ComponentSizeShift(value_type)); 5980 StoreOperandType store_type = GetStoreOperandType(value_type); 5981 __ StoreToOffset(store_type, value_loc.AsRegister<Register>(), array, full_offset); 5982 } else { 5983 Register temp = IP; 5984 5985 if (has_intermediate_address) { 5986 // We do not need to compute the intermediate address from the array: the 5987 // input instruction has done it already. See the comment in 5988 // `TryExtractArrayAccessAddress()`. 5989 if (kIsDebugBuild) { 5990 HIntermediateAddress* tmp = array_instr->AsIntermediateAddress(); 5991 DCHECK(tmp->GetOffset()->AsIntConstant()->GetValueAsUint64() == data_offset); 5992 } 5993 temp = array; 5994 } else { 5995 __ add(temp, array, ShifterOperand(data_offset)); 5996 } 5997 codegen_->StoreToShiftedRegOffset(value_type, 5998 value_loc, 5999 temp, 6000 index.AsRegister<Register>()); 6001 } 6002 break; 6003 } 6004 6005 case Primitive::kPrimNot: { 6006 Register value = value_loc.AsRegister<Register>(); 6007 // TryExtractArrayAccessAddress optimization is never applied for non-primitive ArraySet. 6008 // See the comment in instruction_simplifier_shared.cc. 6009 DCHECK(!has_intermediate_address); 6010 6011 if (instruction->InputAt(2)->IsNullConstant()) { 6012 // Just setting null. 6013 if (index.IsConstant()) { 6014 size_t offset = 6015 (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4) + data_offset; 6016 __ StoreToOffset(kStoreWord, value, array, offset); 6017 } else { 6018 DCHECK(index.IsRegister()) << index; 6019 __ add(IP, array, ShifterOperand(data_offset)); 6020 codegen_->StoreToShiftedRegOffset(value_type, 6021 value_loc, 6022 IP, 6023 index.AsRegister<Register>()); 6024 } 6025 codegen_->MaybeRecordImplicitNullCheck(instruction); 6026 DCHECK(!needs_write_barrier); 6027 DCHECK(!may_need_runtime_call_for_type_check); 6028 break; 6029 } 6030 6031 DCHECK(needs_write_barrier); 6032 Location temp1_loc = locations->GetTemp(0); 6033 Register temp1 = temp1_loc.AsRegister<Register>(); 6034 Location temp2_loc = locations->GetTemp(1); 6035 Register temp2 = temp2_loc.AsRegister<Register>(); 6036 uint32_t class_offset = mirror::Object::ClassOffset().Int32Value(); 6037 uint32_t super_offset = mirror::Class::SuperClassOffset().Int32Value(); 6038 uint32_t component_offset = mirror::Class::ComponentTypeOffset().Int32Value(); 6039 Label done; 6040 Label* final_label = codegen_->GetFinalLabel(instruction, &done); 6041 SlowPathCodeARM* slow_path = nullptr; 6042 6043 if (may_need_runtime_call_for_type_check) { 6044 slow_path = new (GetGraph()->GetArena()) ArraySetSlowPathARM(instruction); 6045 codegen_->AddSlowPath(slow_path); 6046 if (instruction->GetValueCanBeNull()) { 6047 Label non_zero; 6048 __ CompareAndBranchIfNonZero(value, &non_zero); 6049 if (index.IsConstant()) { 6050 size_t offset = 6051 (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4) + data_offset; 6052 __ StoreToOffset(kStoreWord, value, array, offset); 6053 } else { 6054 DCHECK(index.IsRegister()) << index; 6055 __ add(IP, array, ShifterOperand(data_offset)); 6056 codegen_->StoreToShiftedRegOffset(value_type, 6057 value_loc, 6058 IP, 6059 index.AsRegister<Register>()); 6060 } 6061 codegen_->MaybeRecordImplicitNullCheck(instruction); 6062 __ b(final_label); 6063 __ Bind(&non_zero); 6064 } 6065 6066 // Note that when read barriers are enabled, the type checks 6067 // are performed without read barriers. This is fine, even in 6068 // the case where a class object is in the from-space after 6069 // the flip, as a comparison involving such a type would not 6070 // produce a false positive; it may of course produce a false 6071 // negative, in which case we would take the ArraySet slow 6072 // path. 6073 6074 // /* HeapReference<Class> */ temp1 = array->klass_ 6075 __ LoadFromOffset(kLoadWord, temp1, array, class_offset); 6076 codegen_->MaybeRecordImplicitNullCheck(instruction); 6077 __ MaybeUnpoisonHeapReference(temp1); 6078 6079 // /* HeapReference<Class> */ temp1 = temp1->component_type_ 6080 __ LoadFromOffset(kLoadWord, temp1, temp1, component_offset); 6081 // /* HeapReference<Class> */ temp2 = value->klass_ 6082 __ LoadFromOffset(kLoadWord, temp2, value, class_offset); 6083 // If heap poisoning is enabled, no need to unpoison `temp1` 6084 // nor `temp2`, as we are comparing two poisoned references. 6085 __ cmp(temp1, ShifterOperand(temp2)); 6086 6087 if (instruction->StaticTypeOfArrayIsObjectArray()) { 6088 Label do_put; 6089 __ b(&do_put, EQ); 6090 // If heap poisoning is enabled, the `temp1` reference has 6091 // not been unpoisoned yet; unpoison it now. 6092 __ MaybeUnpoisonHeapReference(temp1); 6093 6094 // /* HeapReference<Class> */ temp1 = temp1->super_class_ 6095 __ LoadFromOffset(kLoadWord, temp1, temp1, super_offset); 6096 // If heap poisoning is enabled, no need to unpoison 6097 // `temp1`, as we are comparing against null below. 6098 __ CompareAndBranchIfNonZero(temp1, slow_path->GetEntryLabel()); 6099 __ Bind(&do_put); 6100 } else { 6101 __ b(slow_path->GetEntryLabel(), NE); 6102 } 6103 } 6104 6105 Register source = value; 6106 if (kPoisonHeapReferences) { 6107 // Note that in the case where `value` is a null reference, 6108 // we do not enter this block, as a null reference does not 6109 // need poisoning. 6110 DCHECK_EQ(value_type, Primitive::kPrimNot); 6111 __ Mov(temp1, value); 6112 __ PoisonHeapReference(temp1); 6113 source = temp1; 6114 } 6115 6116 if (index.IsConstant()) { 6117 size_t offset = 6118 (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4) + data_offset; 6119 __ StoreToOffset(kStoreWord, source, array, offset); 6120 } else { 6121 DCHECK(index.IsRegister()) << index; 6122 6123 __ add(IP, array, ShifterOperand(data_offset)); 6124 codegen_->StoreToShiftedRegOffset(value_type, 6125 Location::RegisterLocation(source), 6126 IP, 6127 index.AsRegister<Register>()); 6128 } 6129 6130 if (!may_need_runtime_call_for_type_check) { 6131 codegen_->MaybeRecordImplicitNullCheck(instruction); 6132 } 6133 6134 codegen_->MarkGCCard(temp1, temp2, array, value, instruction->GetValueCanBeNull()); 6135 6136 if (done.IsLinked()) { 6137 __ Bind(&done); 6138 } 6139 6140 if (slow_path != nullptr) { 6141 __ Bind(slow_path->GetExitLabel()); 6142 } 6143 6144 break; 6145 } 6146 6147 case Primitive::kPrimLong: { 6148 Location value = locations->InAt(2); 6149 if (index.IsConstant()) { 6150 size_t offset = 6151 (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_8) + data_offset; 6152 __ StoreToOffset(kStoreWordPair, value.AsRegisterPairLow<Register>(), array, offset); 6153 } else { 6154 __ add(IP, array, ShifterOperand(index.AsRegister<Register>(), LSL, TIMES_8)); 6155 __ StoreToOffset(kStoreWordPair, value.AsRegisterPairLow<Register>(), IP, data_offset); 6156 } 6157 break; 6158 } 6159 6160 case Primitive::kPrimFloat: { 6161 Location value = locations->InAt(2); 6162 DCHECK(value.IsFpuRegister()); 6163 if (index.IsConstant()) { 6164 size_t offset = (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4) + data_offset; 6165 __ StoreSToOffset(value.AsFpuRegister<SRegister>(), array, offset); 6166 } else { 6167 __ add(IP, array, ShifterOperand(index.AsRegister<Register>(), LSL, TIMES_4)); 6168 __ StoreSToOffset(value.AsFpuRegister<SRegister>(), IP, data_offset); 6169 } 6170 break; 6171 } 6172 6173 case Primitive::kPrimDouble: { 6174 Location value = locations->InAt(2); 6175 DCHECK(value.IsFpuRegisterPair()); 6176 if (index.IsConstant()) { 6177 size_t offset = (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_8) + data_offset; 6178 __ StoreDToOffset(FromLowSToD(value.AsFpuRegisterPairLow<SRegister>()), array, offset); 6179 } else { 6180 __ add(IP, array, ShifterOperand(index.AsRegister<Register>(), LSL, TIMES_8)); 6181 __ StoreDToOffset(FromLowSToD(value.AsFpuRegisterPairLow<SRegister>()), IP, data_offset); 6182 } 6183 6184 break; 6185 } 6186 6187 case Primitive::kPrimVoid: 6188 LOG(FATAL) << "Unreachable type " << value_type; 6189 UNREACHABLE(); 6190 } 6191 6192 // Objects are handled in the switch. 6193 if (value_type != Primitive::kPrimNot) { 6194 codegen_->MaybeRecordImplicitNullCheck(instruction); 6195 } 6196} 6197 6198void LocationsBuilderARM::VisitArrayLength(HArrayLength* instruction) { 6199 LocationSummary* locations = 6200 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall); 6201 locations->SetInAt(0, Location::RequiresRegister()); 6202 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 6203} 6204 6205void InstructionCodeGeneratorARM::VisitArrayLength(HArrayLength* instruction) { 6206 LocationSummary* locations = instruction->GetLocations(); 6207 uint32_t offset = CodeGenerator::GetArrayLengthOffset(instruction); 6208 Register obj = locations->InAt(0).AsRegister<Register>(); 6209 Register out = locations->Out().AsRegister<Register>(); 6210 __ LoadFromOffset(kLoadWord, out, obj, offset); 6211 codegen_->MaybeRecordImplicitNullCheck(instruction); 6212 // Mask out compression flag from String's array length. 6213 if (mirror::kUseStringCompression && instruction->IsStringLength()) { 6214 __ Lsr(out, out, 1u); 6215 } 6216} 6217 6218void LocationsBuilderARM::VisitIntermediateAddress(HIntermediateAddress* instruction) { 6219 LocationSummary* locations = 6220 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall); 6221 6222 locations->SetInAt(0, Location::RequiresRegister()); 6223 locations->SetInAt(1, Location::RegisterOrConstant(instruction->GetOffset())); 6224 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 6225} 6226 6227void InstructionCodeGeneratorARM::VisitIntermediateAddress(HIntermediateAddress* instruction) { 6228 LocationSummary* locations = instruction->GetLocations(); 6229 Location out = locations->Out(); 6230 Location first = locations->InAt(0); 6231 Location second = locations->InAt(1); 6232 6233 if (second.IsRegister()) { 6234 __ add(out.AsRegister<Register>(), 6235 first.AsRegister<Register>(), 6236 ShifterOperand(second.AsRegister<Register>())); 6237 } else { 6238 __ AddConstant(out.AsRegister<Register>(), 6239 first.AsRegister<Register>(), 6240 second.GetConstant()->AsIntConstant()->GetValue()); 6241 } 6242} 6243 6244void LocationsBuilderARM::VisitBoundsCheck(HBoundsCheck* instruction) { 6245 RegisterSet caller_saves = RegisterSet::Empty(); 6246 InvokeRuntimeCallingConvention calling_convention; 6247 caller_saves.Add(Location::RegisterLocation(calling_convention.GetRegisterAt(0))); 6248 caller_saves.Add(Location::RegisterLocation(calling_convention.GetRegisterAt(1))); 6249 LocationSummary* locations = codegen_->CreateThrowingSlowPathLocations(instruction, caller_saves); 6250 6251 HInstruction* index = instruction->InputAt(0); 6252 HInstruction* length = instruction->InputAt(1); 6253 // If both index and length are constants we can statically check the bounds. But if at least one 6254 // of them is not encodable ArmEncodableConstantOrRegister will create 6255 // Location::RequiresRegister() which is not desired to happen. Instead we create constant 6256 // locations. 6257 bool both_const = index->IsConstant() && length->IsConstant(); 6258 locations->SetInAt(0, both_const 6259 ? Location::ConstantLocation(index->AsConstant()) 6260 : ArmEncodableConstantOrRegister(index, CMP)); 6261 locations->SetInAt(1, both_const 6262 ? Location::ConstantLocation(length->AsConstant()) 6263 : ArmEncodableConstantOrRegister(length, CMP)); 6264} 6265 6266void InstructionCodeGeneratorARM::VisitBoundsCheck(HBoundsCheck* instruction) { 6267 LocationSummary* locations = instruction->GetLocations(); 6268 Location index_loc = locations->InAt(0); 6269 Location length_loc = locations->InAt(1); 6270 6271 if (length_loc.IsConstant()) { 6272 int32_t length = helpers::Int32ConstantFrom(length_loc); 6273 if (index_loc.IsConstant()) { 6274 // BCE will remove the bounds check if we are guaranteed to pass. 6275 int32_t index = helpers::Int32ConstantFrom(index_loc); 6276 if (index < 0 || index >= length) { 6277 SlowPathCodeARM* slow_path = 6278 new (GetGraph()->GetArena()) BoundsCheckSlowPathARM(instruction); 6279 codegen_->AddSlowPath(slow_path); 6280 __ b(slow_path->GetEntryLabel()); 6281 } else { 6282 // Some optimization after BCE may have generated this, and we should not 6283 // generate a bounds check if it is a valid range. 6284 } 6285 return; 6286 } 6287 6288 SlowPathCodeARM* slow_path = new (GetGraph()->GetArena()) BoundsCheckSlowPathARM(instruction); 6289 __ cmp(index_loc.AsRegister<Register>(), ShifterOperand(length)); 6290 codegen_->AddSlowPath(slow_path); 6291 __ b(slow_path->GetEntryLabel(), HS); 6292 } else { 6293 SlowPathCodeARM* slow_path = new (GetGraph()->GetArena()) BoundsCheckSlowPathARM(instruction); 6294 if (index_loc.IsConstant()) { 6295 int32_t index = helpers::Int32ConstantFrom(index_loc); 6296 __ cmp(length_loc.AsRegister<Register>(), ShifterOperand(index)); 6297 } else { 6298 __ cmp(length_loc.AsRegister<Register>(), ShifterOperand(index_loc.AsRegister<Register>())); 6299 } 6300 codegen_->AddSlowPath(slow_path); 6301 __ b(slow_path->GetEntryLabel(), LS); 6302 } 6303} 6304 6305void CodeGeneratorARM::MarkGCCard(Register temp, 6306 Register card, 6307 Register object, 6308 Register value, 6309 bool can_be_null) { 6310 Label is_null; 6311 if (can_be_null) { 6312 __ CompareAndBranchIfZero(value, &is_null); 6313 } 6314 __ LoadFromOffset(kLoadWord, card, TR, Thread::CardTableOffset<kArmPointerSize>().Int32Value()); 6315 __ Lsr(temp, object, gc::accounting::CardTable::kCardShift); 6316 __ strb(card, Address(card, temp)); 6317 if (can_be_null) { 6318 __ Bind(&is_null); 6319 } 6320} 6321 6322void LocationsBuilderARM::VisitParallelMove(HParallelMove* instruction ATTRIBUTE_UNUSED) { 6323 LOG(FATAL) << "Unreachable"; 6324} 6325 6326void InstructionCodeGeneratorARM::VisitParallelMove(HParallelMove* instruction) { 6327 codegen_->GetMoveResolver()->EmitNativeCode(instruction); 6328} 6329 6330void LocationsBuilderARM::VisitSuspendCheck(HSuspendCheck* instruction) { 6331 LocationSummary* locations = 6332 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCallOnSlowPath); 6333 locations->SetCustomSlowPathCallerSaves(RegisterSet::Empty()); // No caller-save registers. 6334} 6335 6336void InstructionCodeGeneratorARM::VisitSuspendCheck(HSuspendCheck* instruction) { 6337 HBasicBlock* block = instruction->GetBlock(); 6338 if (block->GetLoopInformation() != nullptr) { 6339 DCHECK(block->GetLoopInformation()->GetSuspendCheck() == instruction); 6340 // The back edge will generate the suspend check. 6341 return; 6342 } 6343 if (block->IsEntryBlock() && instruction->GetNext()->IsGoto()) { 6344 // The goto will generate the suspend check. 6345 return; 6346 } 6347 GenerateSuspendCheck(instruction, nullptr); 6348} 6349 6350void InstructionCodeGeneratorARM::GenerateSuspendCheck(HSuspendCheck* instruction, 6351 HBasicBlock* successor) { 6352 SuspendCheckSlowPathARM* slow_path = 6353 down_cast<SuspendCheckSlowPathARM*>(instruction->GetSlowPath()); 6354 if (slow_path == nullptr) { 6355 slow_path = new (GetGraph()->GetArena()) SuspendCheckSlowPathARM(instruction, successor); 6356 instruction->SetSlowPath(slow_path); 6357 codegen_->AddSlowPath(slow_path); 6358 if (successor != nullptr) { 6359 DCHECK(successor->IsLoopHeader()); 6360 codegen_->ClearSpillSlotsFromLoopPhisInStackMap(instruction); 6361 } 6362 } else { 6363 DCHECK_EQ(slow_path->GetSuccessor(), successor); 6364 } 6365 6366 __ LoadFromOffset( 6367 kLoadUnsignedHalfword, IP, TR, Thread::ThreadFlagsOffset<kArmPointerSize>().Int32Value()); 6368 if (successor == nullptr) { 6369 __ CompareAndBranchIfNonZero(IP, slow_path->GetEntryLabel()); 6370 __ Bind(slow_path->GetReturnLabel()); 6371 } else { 6372 __ CompareAndBranchIfZero(IP, codegen_->GetLabelOf(successor)); 6373 __ b(slow_path->GetEntryLabel()); 6374 } 6375} 6376 6377ArmAssembler* ParallelMoveResolverARM::GetAssembler() const { 6378 return codegen_->GetAssembler(); 6379} 6380 6381void ParallelMoveResolverARM::EmitMove(size_t index) { 6382 MoveOperands* move = moves_[index]; 6383 Location source = move->GetSource(); 6384 Location destination = move->GetDestination(); 6385 6386 if (source.IsRegister()) { 6387 if (destination.IsRegister()) { 6388 __ Mov(destination.AsRegister<Register>(), source.AsRegister<Register>()); 6389 } else if (destination.IsFpuRegister()) { 6390 __ vmovsr(destination.AsFpuRegister<SRegister>(), source.AsRegister<Register>()); 6391 } else { 6392 DCHECK(destination.IsStackSlot()); 6393 __ StoreToOffset(kStoreWord, source.AsRegister<Register>(), 6394 SP, destination.GetStackIndex()); 6395 } 6396 } else if (source.IsStackSlot()) { 6397 if (destination.IsRegister()) { 6398 __ LoadFromOffset(kLoadWord, destination.AsRegister<Register>(), 6399 SP, source.GetStackIndex()); 6400 } else if (destination.IsFpuRegister()) { 6401 __ LoadSFromOffset(destination.AsFpuRegister<SRegister>(), SP, source.GetStackIndex()); 6402 } else { 6403 DCHECK(destination.IsStackSlot()); 6404 __ LoadFromOffset(kLoadWord, IP, SP, source.GetStackIndex()); 6405 __ StoreToOffset(kStoreWord, IP, SP, destination.GetStackIndex()); 6406 } 6407 } else if (source.IsFpuRegister()) { 6408 if (destination.IsRegister()) { 6409 __ vmovrs(destination.AsRegister<Register>(), source.AsFpuRegister<SRegister>()); 6410 } else if (destination.IsFpuRegister()) { 6411 __ vmovs(destination.AsFpuRegister<SRegister>(), source.AsFpuRegister<SRegister>()); 6412 } else { 6413 DCHECK(destination.IsStackSlot()); 6414 __ StoreSToOffset(source.AsFpuRegister<SRegister>(), SP, destination.GetStackIndex()); 6415 } 6416 } else if (source.IsDoubleStackSlot()) { 6417 if (destination.IsDoubleStackSlot()) { 6418 __ LoadDFromOffset(DTMP, SP, source.GetStackIndex()); 6419 __ StoreDToOffset(DTMP, SP, destination.GetStackIndex()); 6420 } else if (destination.IsRegisterPair()) { 6421 DCHECK(ExpectedPairLayout(destination)); 6422 __ LoadFromOffset( 6423 kLoadWordPair, destination.AsRegisterPairLow<Register>(), SP, source.GetStackIndex()); 6424 } else { 6425 DCHECK(destination.IsFpuRegisterPair()) << destination; 6426 __ LoadDFromOffset(FromLowSToD(destination.AsFpuRegisterPairLow<SRegister>()), 6427 SP, 6428 source.GetStackIndex()); 6429 } 6430 } else if (source.IsRegisterPair()) { 6431 if (destination.IsRegisterPair()) { 6432 __ Mov(destination.AsRegisterPairLow<Register>(), source.AsRegisterPairLow<Register>()); 6433 __ Mov(destination.AsRegisterPairHigh<Register>(), source.AsRegisterPairHigh<Register>()); 6434 } else if (destination.IsFpuRegisterPair()) { 6435 __ vmovdrr(FromLowSToD(destination.AsFpuRegisterPairLow<SRegister>()), 6436 source.AsRegisterPairLow<Register>(), 6437 source.AsRegisterPairHigh<Register>()); 6438 } else { 6439 DCHECK(destination.IsDoubleStackSlot()) << destination; 6440 DCHECK(ExpectedPairLayout(source)); 6441 __ StoreToOffset( 6442 kStoreWordPair, source.AsRegisterPairLow<Register>(), SP, destination.GetStackIndex()); 6443 } 6444 } else if (source.IsFpuRegisterPair()) { 6445 if (destination.IsRegisterPair()) { 6446 __ vmovrrd(destination.AsRegisterPairLow<Register>(), 6447 destination.AsRegisterPairHigh<Register>(), 6448 FromLowSToD(source.AsFpuRegisterPairLow<SRegister>())); 6449 } else if (destination.IsFpuRegisterPair()) { 6450 __ vmovd(FromLowSToD(destination.AsFpuRegisterPairLow<SRegister>()), 6451 FromLowSToD(source.AsFpuRegisterPairLow<SRegister>())); 6452 } else { 6453 DCHECK(destination.IsDoubleStackSlot()) << destination; 6454 __ StoreDToOffset(FromLowSToD(source.AsFpuRegisterPairLow<SRegister>()), 6455 SP, 6456 destination.GetStackIndex()); 6457 } 6458 } else { 6459 DCHECK(source.IsConstant()) << source; 6460 HConstant* constant = source.GetConstant(); 6461 if (constant->IsIntConstant() || constant->IsNullConstant()) { 6462 int32_t value = CodeGenerator::GetInt32ValueOf(constant); 6463 if (destination.IsRegister()) { 6464 __ LoadImmediate(destination.AsRegister<Register>(), value); 6465 } else { 6466 DCHECK(destination.IsStackSlot()); 6467 __ LoadImmediate(IP, value); 6468 __ StoreToOffset(kStoreWord, IP, SP, destination.GetStackIndex()); 6469 } 6470 } else if (constant->IsLongConstant()) { 6471 int64_t value = constant->AsLongConstant()->GetValue(); 6472 if (destination.IsRegisterPair()) { 6473 __ LoadImmediate(destination.AsRegisterPairLow<Register>(), Low32Bits(value)); 6474 __ LoadImmediate(destination.AsRegisterPairHigh<Register>(), High32Bits(value)); 6475 } else { 6476 DCHECK(destination.IsDoubleStackSlot()) << destination; 6477 __ LoadImmediate(IP, Low32Bits(value)); 6478 __ StoreToOffset(kStoreWord, IP, SP, destination.GetStackIndex()); 6479 __ LoadImmediate(IP, High32Bits(value)); 6480 __ StoreToOffset(kStoreWord, IP, SP, destination.GetHighStackIndex(kArmWordSize)); 6481 } 6482 } else if (constant->IsDoubleConstant()) { 6483 double value = constant->AsDoubleConstant()->GetValue(); 6484 if (destination.IsFpuRegisterPair()) { 6485 __ LoadDImmediate(FromLowSToD(destination.AsFpuRegisterPairLow<SRegister>()), value); 6486 } else { 6487 DCHECK(destination.IsDoubleStackSlot()) << destination; 6488 uint64_t int_value = bit_cast<uint64_t, double>(value); 6489 __ LoadImmediate(IP, Low32Bits(int_value)); 6490 __ StoreToOffset(kStoreWord, IP, SP, destination.GetStackIndex()); 6491 __ LoadImmediate(IP, High32Bits(int_value)); 6492 __ StoreToOffset(kStoreWord, IP, SP, destination.GetHighStackIndex(kArmWordSize)); 6493 } 6494 } else { 6495 DCHECK(constant->IsFloatConstant()) << constant->DebugName(); 6496 float value = constant->AsFloatConstant()->GetValue(); 6497 if (destination.IsFpuRegister()) { 6498 __ LoadSImmediate(destination.AsFpuRegister<SRegister>(), value); 6499 } else { 6500 DCHECK(destination.IsStackSlot()); 6501 __ LoadImmediate(IP, bit_cast<int32_t, float>(value)); 6502 __ StoreToOffset(kStoreWord, IP, SP, destination.GetStackIndex()); 6503 } 6504 } 6505 } 6506} 6507 6508void ParallelMoveResolverARM::Exchange(Register reg, int mem) { 6509 __ Mov(IP, reg); 6510 __ LoadFromOffset(kLoadWord, reg, SP, mem); 6511 __ StoreToOffset(kStoreWord, IP, SP, mem); 6512} 6513 6514void ParallelMoveResolverARM::Exchange(int mem1, int mem2) { 6515 ScratchRegisterScope ensure_scratch(this, IP, R0, codegen_->GetNumberOfCoreRegisters()); 6516 int stack_offset = ensure_scratch.IsSpilled() ? kArmWordSize : 0; 6517 __ LoadFromOffset(kLoadWord, static_cast<Register>(ensure_scratch.GetRegister()), 6518 SP, mem1 + stack_offset); 6519 __ LoadFromOffset(kLoadWord, IP, SP, mem2 + stack_offset); 6520 __ StoreToOffset(kStoreWord, static_cast<Register>(ensure_scratch.GetRegister()), 6521 SP, mem2 + stack_offset); 6522 __ StoreToOffset(kStoreWord, IP, SP, mem1 + stack_offset); 6523} 6524 6525void ParallelMoveResolverARM::EmitSwap(size_t index) { 6526 MoveOperands* move = moves_[index]; 6527 Location source = move->GetSource(); 6528 Location destination = move->GetDestination(); 6529 6530 if (source.IsRegister() && destination.IsRegister()) { 6531 DCHECK_NE(source.AsRegister<Register>(), IP); 6532 DCHECK_NE(destination.AsRegister<Register>(), IP); 6533 __ Mov(IP, source.AsRegister<Register>()); 6534 __ Mov(source.AsRegister<Register>(), destination.AsRegister<Register>()); 6535 __ Mov(destination.AsRegister<Register>(), IP); 6536 } else if (source.IsRegister() && destination.IsStackSlot()) { 6537 Exchange(source.AsRegister<Register>(), destination.GetStackIndex()); 6538 } else if (source.IsStackSlot() && destination.IsRegister()) { 6539 Exchange(destination.AsRegister<Register>(), source.GetStackIndex()); 6540 } else if (source.IsStackSlot() && destination.IsStackSlot()) { 6541 Exchange(source.GetStackIndex(), destination.GetStackIndex()); 6542 } else if (source.IsFpuRegister() && destination.IsFpuRegister()) { 6543 __ vmovrs(IP, source.AsFpuRegister<SRegister>()); 6544 __ vmovs(source.AsFpuRegister<SRegister>(), destination.AsFpuRegister<SRegister>()); 6545 __ vmovsr(destination.AsFpuRegister<SRegister>(), IP); 6546 } else if (source.IsRegisterPair() && destination.IsRegisterPair()) { 6547 __ vmovdrr(DTMP, source.AsRegisterPairLow<Register>(), source.AsRegisterPairHigh<Register>()); 6548 __ Mov(source.AsRegisterPairLow<Register>(), destination.AsRegisterPairLow<Register>()); 6549 __ Mov(source.AsRegisterPairHigh<Register>(), destination.AsRegisterPairHigh<Register>()); 6550 __ vmovrrd(destination.AsRegisterPairLow<Register>(), 6551 destination.AsRegisterPairHigh<Register>(), 6552 DTMP); 6553 } else if (source.IsRegisterPair() || destination.IsRegisterPair()) { 6554 Register low_reg = source.IsRegisterPair() 6555 ? source.AsRegisterPairLow<Register>() 6556 : destination.AsRegisterPairLow<Register>(); 6557 int mem = source.IsRegisterPair() 6558 ? destination.GetStackIndex() 6559 : source.GetStackIndex(); 6560 DCHECK(ExpectedPairLayout(source.IsRegisterPair() ? source : destination)); 6561 __ vmovdrr(DTMP, low_reg, static_cast<Register>(low_reg + 1)); 6562 __ LoadFromOffset(kLoadWordPair, low_reg, SP, mem); 6563 __ StoreDToOffset(DTMP, SP, mem); 6564 } else if (source.IsFpuRegisterPair() && destination.IsFpuRegisterPair()) { 6565 DRegister first = FromLowSToD(source.AsFpuRegisterPairLow<SRegister>()); 6566 DRegister second = FromLowSToD(destination.AsFpuRegisterPairLow<SRegister>()); 6567 __ vmovd(DTMP, first); 6568 __ vmovd(first, second); 6569 __ vmovd(second, DTMP); 6570 } else if (source.IsFpuRegisterPair() || destination.IsFpuRegisterPair()) { 6571 DRegister reg = source.IsFpuRegisterPair() 6572 ? FromLowSToD(source.AsFpuRegisterPairLow<SRegister>()) 6573 : FromLowSToD(destination.AsFpuRegisterPairLow<SRegister>()); 6574 int mem = source.IsFpuRegisterPair() 6575 ? destination.GetStackIndex() 6576 : source.GetStackIndex(); 6577 __ vmovd(DTMP, reg); 6578 __ LoadDFromOffset(reg, SP, mem); 6579 __ StoreDToOffset(DTMP, SP, mem); 6580 } else if (source.IsFpuRegister() || destination.IsFpuRegister()) { 6581 SRegister reg = source.IsFpuRegister() ? source.AsFpuRegister<SRegister>() 6582 : destination.AsFpuRegister<SRegister>(); 6583 int mem = source.IsFpuRegister() 6584 ? destination.GetStackIndex() 6585 : source.GetStackIndex(); 6586 6587 __ vmovrs(IP, reg); 6588 __ LoadSFromOffset(reg, SP, mem); 6589 __ StoreToOffset(kStoreWord, IP, SP, mem); 6590 } else if (source.IsDoubleStackSlot() && destination.IsDoubleStackSlot()) { 6591 Exchange(source.GetStackIndex(), destination.GetStackIndex()); 6592 Exchange(source.GetHighStackIndex(kArmWordSize), destination.GetHighStackIndex(kArmWordSize)); 6593 } else { 6594 LOG(FATAL) << "Unimplemented" << source << " <-> " << destination; 6595 } 6596} 6597 6598void ParallelMoveResolverARM::SpillScratch(int reg) { 6599 __ Push(static_cast<Register>(reg)); 6600} 6601 6602void ParallelMoveResolverARM::RestoreScratch(int reg) { 6603 __ Pop(static_cast<Register>(reg)); 6604} 6605 6606HLoadClass::LoadKind CodeGeneratorARM::GetSupportedLoadClassKind( 6607 HLoadClass::LoadKind desired_class_load_kind) { 6608 switch (desired_class_load_kind) { 6609 case HLoadClass::LoadKind::kInvalid: 6610 LOG(FATAL) << "UNREACHABLE"; 6611 UNREACHABLE(); 6612 case HLoadClass::LoadKind::kReferrersClass: 6613 break; 6614 case HLoadClass::LoadKind::kBootImageLinkTimeAddress: 6615 DCHECK(!GetCompilerOptions().GetCompilePic()); 6616 break; 6617 case HLoadClass::LoadKind::kBootImageLinkTimePcRelative: 6618 DCHECK(GetCompilerOptions().GetCompilePic()); 6619 break; 6620 case HLoadClass::LoadKind::kBootImageAddress: 6621 break; 6622 case HLoadClass::LoadKind::kBssEntry: 6623 DCHECK(!Runtime::Current()->UseJitCompilation()); 6624 break; 6625 case HLoadClass::LoadKind::kJitTableAddress: 6626 DCHECK(Runtime::Current()->UseJitCompilation()); 6627 break; 6628 case HLoadClass::LoadKind::kDexCacheViaMethod: 6629 break; 6630 } 6631 return desired_class_load_kind; 6632} 6633 6634void LocationsBuilderARM::VisitLoadClass(HLoadClass* cls) { 6635 HLoadClass::LoadKind load_kind = cls->GetLoadKind(); 6636 if (load_kind == HLoadClass::LoadKind::kDexCacheViaMethod) { 6637 InvokeRuntimeCallingConvention calling_convention; 6638 CodeGenerator::CreateLoadClassRuntimeCallLocationSummary( 6639 cls, 6640 Location::RegisterLocation(calling_convention.GetRegisterAt(0)), 6641 Location::RegisterLocation(R0)); 6642 DCHECK_EQ(calling_convention.GetRegisterAt(0), R0); 6643 return; 6644 } 6645 DCHECK(!cls->NeedsAccessCheck()); 6646 6647 const bool requires_read_barrier = kEmitCompilerReadBarrier && !cls->IsInBootImage(); 6648 LocationSummary::CallKind call_kind = (cls->NeedsEnvironment() || requires_read_barrier) 6649 ? LocationSummary::kCallOnSlowPath 6650 : LocationSummary::kNoCall; 6651 LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(cls, call_kind); 6652 if (kUseBakerReadBarrier && requires_read_barrier && !cls->NeedsEnvironment()) { 6653 locations->SetCustomSlowPathCallerSaves(RegisterSet::Empty()); // No caller-save registers. 6654 } 6655 6656 if (load_kind == HLoadClass::LoadKind::kReferrersClass) { 6657 locations->SetInAt(0, Location::RequiresRegister()); 6658 } 6659 locations->SetOut(Location::RequiresRegister()); 6660 if (load_kind == HLoadClass::LoadKind::kBssEntry) { 6661 if (!kUseReadBarrier || kUseBakerReadBarrier) { 6662 // Rely on the type resolution or initialization and marking to save everything we need. 6663 // Note that IP may be clobbered by saving/restoring the live register (only one thanks 6664 // to the custom calling convention) or by marking, so we request a different temp. 6665 locations->AddTemp(Location::RequiresRegister()); 6666 RegisterSet caller_saves = RegisterSet::Empty(); 6667 InvokeRuntimeCallingConvention calling_convention; 6668 caller_saves.Add(Location::RegisterLocation(calling_convention.GetRegisterAt(0))); 6669 // TODO: Add GetReturnLocation() to the calling convention so that we can DCHECK() 6670 // that the the kPrimNot result register is the same as the first argument register. 6671 locations->SetCustomSlowPathCallerSaves(caller_saves); 6672 } else { 6673 // For non-Baker read barrier we have a temp-clobbering call. 6674 } 6675 } 6676} 6677 6678// NO_THREAD_SAFETY_ANALYSIS as we manipulate handles whose internal object we know does not 6679// move. 6680void InstructionCodeGeneratorARM::VisitLoadClass(HLoadClass* cls) NO_THREAD_SAFETY_ANALYSIS { 6681 HLoadClass::LoadKind load_kind = cls->GetLoadKind(); 6682 if (load_kind == HLoadClass::LoadKind::kDexCacheViaMethod) { 6683 codegen_->GenerateLoadClassRuntimeCall(cls); 6684 return; 6685 } 6686 DCHECK(!cls->NeedsAccessCheck()); 6687 6688 LocationSummary* locations = cls->GetLocations(); 6689 Location out_loc = locations->Out(); 6690 Register out = out_loc.AsRegister<Register>(); 6691 6692 const ReadBarrierOption read_barrier_option = cls->IsInBootImage() 6693 ? kWithoutReadBarrier 6694 : kCompilerReadBarrierOption; 6695 bool generate_null_check = false; 6696 switch (load_kind) { 6697 case HLoadClass::LoadKind::kReferrersClass: { 6698 DCHECK(!cls->CanCallRuntime()); 6699 DCHECK(!cls->MustGenerateClinitCheck()); 6700 // /* GcRoot<mirror::Class> */ out = current_method->declaring_class_ 6701 Register current_method = locations->InAt(0).AsRegister<Register>(); 6702 GenerateGcRootFieldLoad(cls, 6703 out_loc, 6704 current_method, 6705 ArtMethod::DeclaringClassOffset().Int32Value(), 6706 read_barrier_option); 6707 break; 6708 } 6709 case HLoadClass::LoadKind::kBootImageLinkTimeAddress: { 6710 DCHECK(codegen_->GetCompilerOptions().IsBootImage()); 6711 DCHECK_EQ(read_barrier_option, kWithoutReadBarrier); 6712 __ LoadLiteral(out, codegen_->DeduplicateBootImageTypeLiteral(cls->GetDexFile(), 6713 cls->GetTypeIndex())); 6714 break; 6715 } 6716 case HLoadClass::LoadKind::kBootImageLinkTimePcRelative: { 6717 DCHECK(codegen_->GetCompilerOptions().IsBootImage()); 6718 DCHECK_EQ(read_barrier_option, kWithoutReadBarrier); 6719 CodeGeneratorARM::PcRelativePatchInfo* labels = 6720 codegen_->NewPcRelativeTypePatch(cls->GetDexFile(), cls->GetTypeIndex()); 6721 __ BindTrackedLabel(&labels->movw_label); 6722 __ movw(out, /* placeholder */ 0u); 6723 __ BindTrackedLabel(&labels->movt_label); 6724 __ movt(out, /* placeholder */ 0u); 6725 __ BindTrackedLabel(&labels->add_pc_label); 6726 __ add(out, out, ShifterOperand(PC)); 6727 break; 6728 } 6729 case HLoadClass::LoadKind::kBootImageAddress: { 6730 DCHECK_EQ(read_barrier_option, kWithoutReadBarrier); 6731 uint32_t address = dchecked_integral_cast<uint32_t>( 6732 reinterpret_cast<uintptr_t>(cls->GetClass().Get())); 6733 DCHECK_NE(address, 0u); 6734 __ LoadLiteral(out, codegen_->DeduplicateBootImageAddressLiteral(address)); 6735 break; 6736 } 6737 case HLoadClass::LoadKind::kBssEntry: { 6738 Register temp = (!kUseReadBarrier || kUseBakerReadBarrier) 6739 ? locations->GetTemp(0).AsRegister<Register>() 6740 : out; 6741 CodeGeneratorARM::PcRelativePatchInfo* labels = 6742 codegen_->NewTypeBssEntryPatch(cls->GetDexFile(), cls->GetTypeIndex()); 6743 __ BindTrackedLabel(&labels->movw_label); 6744 __ movw(temp, /* placeholder */ 0u); 6745 __ BindTrackedLabel(&labels->movt_label); 6746 __ movt(temp, /* placeholder */ 0u); 6747 __ BindTrackedLabel(&labels->add_pc_label); 6748 __ add(temp, temp, ShifterOperand(PC)); 6749 GenerateGcRootFieldLoad(cls, out_loc, temp, /* offset */ 0, read_barrier_option); 6750 generate_null_check = true; 6751 break; 6752 } 6753 case HLoadClass::LoadKind::kJitTableAddress: { 6754 __ LoadLiteral(out, codegen_->DeduplicateJitClassLiteral(cls->GetDexFile(), 6755 cls->GetTypeIndex(), 6756 cls->GetClass())); 6757 // /* GcRoot<mirror::Class> */ out = *out 6758 GenerateGcRootFieldLoad(cls, out_loc, out, /* offset */ 0, read_barrier_option); 6759 break; 6760 } 6761 case HLoadClass::LoadKind::kDexCacheViaMethod: 6762 case HLoadClass::LoadKind::kInvalid: 6763 LOG(FATAL) << "UNREACHABLE"; 6764 UNREACHABLE(); 6765 } 6766 6767 if (generate_null_check || cls->MustGenerateClinitCheck()) { 6768 DCHECK(cls->CanCallRuntime()); 6769 SlowPathCodeARM* slow_path = new (GetGraph()->GetArena()) LoadClassSlowPathARM( 6770 cls, cls, cls->GetDexPc(), cls->MustGenerateClinitCheck()); 6771 codegen_->AddSlowPath(slow_path); 6772 if (generate_null_check) { 6773 __ CompareAndBranchIfZero(out, slow_path->GetEntryLabel()); 6774 } 6775 if (cls->MustGenerateClinitCheck()) { 6776 GenerateClassInitializationCheck(slow_path, out); 6777 } else { 6778 __ Bind(slow_path->GetExitLabel()); 6779 } 6780 } 6781} 6782 6783void LocationsBuilderARM::VisitClinitCheck(HClinitCheck* check) { 6784 LocationSummary* locations = 6785 new (GetGraph()->GetArena()) LocationSummary(check, LocationSummary::kCallOnSlowPath); 6786 locations->SetInAt(0, Location::RequiresRegister()); 6787 if (check->HasUses()) { 6788 locations->SetOut(Location::SameAsFirstInput()); 6789 } 6790} 6791 6792void InstructionCodeGeneratorARM::VisitClinitCheck(HClinitCheck* check) { 6793 // We assume the class is not null. 6794 SlowPathCodeARM* slow_path = new (GetGraph()->GetArena()) LoadClassSlowPathARM( 6795 check->GetLoadClass(), check, check->GetDexPc(), true); 6796 codegen_->AddSlowPath(slow_path); 6797 GenerateClassInitializationCheck(slow_path, 6798 check->GetLocations()->InAt(0).AsRegister<Register>()); 6799} 6800 6801void InstructionCodeGeneratorARM::GenerateClassInitializationCheck( 6802 SlowPathCodeARM* slow_path, Register class_reg) { 6803 __ LoadFromOffset(kLoadWord, IP, class_reg, mirror::Class::StatusOffset().Int32Value()); 6804 __ cmp(IP, ShifterOperand(mirror::Class::kStatusInitialized)); 6805 __ b(slow_path->GetEntryLabel(), LT); 6806 // Even if the initialized flag is set, we may be in a situation where caches are not synced 6807 // properly. Therefore, we do a memory fence. 6808 __ dmb(ISH); 6809 __ Bind(slow_path->GetExitLabel()); 6810} 6811 6812HLoadString::LoadKind CodeGeneratorARM::GetSupportedLoadStringKind( 6813 HLoadString::LoadKind desired_string_load_kind) { 6814 switch (desired_string_load_kind) { 6815 case HLoadString::LoadKind::kBootImageLinkTimeAddress: 6816 DCHECK(!GetCompilerOptions().GetCompilePic()); 6817 break; 6818 case HLoadString::LoadKind::kBootImageLinkTimePcRelative: 6819 DCHECK(GetCompilerOptions().GetCompilePic()); 6820 break; 6821 case HLoadString::LoadKind::kBootImageAddress: 6822 break; 6823 case HLoadString::LoadKind::kBssEntry: 6824 DCHECK(!Runtime::Current()->UseJitCompilation()); 6825 break; 6826 case HLoadString::LoadKind::kJitTableAddress: 6827 DCHECK(Runtime::Current()->UseJitCompilation()); 6828 break; 6829 case HLoadString::LoadKind::kDexCacheViaMethod: 6830 break; 6831 } 6832 return desired_string_load_kind; 6833} 6834 6835void LocationsBuilderARM::VisitLoadString(HLoadString* load) { 6836 LocationSummary::CallKind call_kind = CodeGenerator::GetLoadStringCallKind(load); 6837 LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(load, call_kind); 6838 HLoadString::LoadKind load_kind = load->GetLoadKind(); 6839 if (load_kind == HLoadString::LoadKind::kDexCacheViaMethod) { 6840 locations->SetOut(Location::RegisterLocation(R0)); 6841 } else { 6842 locations->SetOut(Location::RequiresRegister()); 6843 if (load_kind == HLoadString::LoadKind::kBssEntry) { 6844 if (!kUseReadBarrier || kUseBakerReadBarrier) { 6845 // Rely on the pResolveString and marking to save everything we need, including temps. 6846 // Note that IP may be clobbered by saving/restoring the live register (only one thanks 6847 // to the custom calling convention) or by marking, so we request a different temp. 6848 locations->AddTemp(Location::RequiresRegister()); 6849 RegisterSet caller_saves = RegisterSet::Empty(); 6850 InvokeRuntimeCallingConvention calling_convention; 6851 caller_saves.Add(Location::RegisterLocation(calling_convention.GetRegisterAt(0))); 6852 // TODO: Add GetReturnLocation() to the calling convention so that we can DCHECK() 6853 // that the the kPrimNot result register is the same as the first argument register. 6854 locations->SetCustomSlowPathCallerSaves(caller_saves); 6855 } else { 6856 // For non-Baker read barrier we have a temp-clobbering call. 6857 } 6858 } 6859 } 6860} 6861 6862// NO_THREAD_SAFETY_ANALYSIS as we manipulate handles whose internal object we know does not 6863// move. 6864void InstructionCodeGeneratorARM::VisitLoadString(HLoadString* load) NO_THREAD_SAFETY_ANALYSIS { 6865 LocationSummary* locations = load->GetLocations(); 6866 Location out_loc = locations->Out(); 6867 Register out = out_loc.AsRegister<Register>(); 6868 HLoadString::LoadKind load_kind = load->GetLoadKind(); 6869 6870 switch (load_kind) { 6871 case HLoadString::LoadKind::kBootImageLinkTimeAddress: { 6872 DCHECK(codegen_->GetCompilerOptions().IsBootImage()); 6873 __ LoadLiteral(out, codegen_->DeduplicateBootImageStringLiteral(load->GetDexFile(), 6874 load->GetStringIndex())); 6875 return; // No dex cache slow path. 6876 } 6877 case HLoadString::LoadKind::kBootImageLinkTimePcRelative: { 6878 DCHECK(codegen_->GetCompilerOptions().IsBootImage()); 6879 CodeGeneratorARM::PcRelativePatchInfo* labels = 6880 codegen_->NewPcRelativeStringPatch(load->GetDexFile(), load->GetStringIndex()); 6881 __ BindTrackedLabel(&labels->movw_label); 6882 __ movw(out, /* placeholder */ 0u); 6883 __ BindTrackedLabel(&labels->movt_label); 6884 __ movt(out, /* placeholder */ 0u); 6885 __ BindTrackedLabel(&labels->add_pc_label); 6886 __ add(out, out, ShifterOperand(PC)); 6887 return; // No dex cache slow path. 6888 } 6889 case HLoadString::LoadKind::kBootImageAddress: { 6890 uint32_t address = dchecked_integral_cast<uint32_t>( 6891 reinterpret_cast<uintptr_t>(load->GetString().Get())); 6892 DCHECK_NE(address, 0u); 6893 __ LoadLiteral(out, codegen_->DeduplicateBootImageAddressLiteral(address)); 6894 return; // No dex cache slow path. 6895 } 6896 case HLoadString::LoadKind::kBssEntry: { 6897 DCHECK(!codegen_->GetCompilerOptions().IsBootImage()); 6898 Register temp = (!kUseReadBarrier || kUseBakerReadBarrier) 6899 ? locations->GetTemp(0).AsRegister<Register>() 6900 : out; 6901 CodeGeneratorARM::PcRelativePatchInfo* labels = 6902 codegen_->NewPcRelativeStringPatch(load->GetDexFile(), load->GetStringIndex()); 6903 __ BindTrackedLabel(&labels->movw_label); 6904 __ movw(temp, /* placeholder */ 0u); 6905 __ BindTrackedLabel(&labels->movt_label); 6906 __ movt(temp, /* placeholder */ 0u); 6907 __ BindTrackedLabel(&labels->add_pc_label); 6908 __ add(temp, temp, ShifterOperand(PC)); 6909 GenerateGcRootFieldLoad(load, out_loc, temp, /* offset */ 0, kCompilerReadBarrierOption); 6910 SlowPathCode* slow_path = new (GetGraph()->GetArena()) LoadStringSlowPathARM(load); 6911 codegen_->AddSlowPath(slow_path); 6912 __ CompareAndBranchIfZero(out, slow_path->GetEntryLabel()); 6913 __ Bind(slow_path->GetExitLabel()); 6914 return; 6915 } 6916 case HLoadString::LoadKind::kJitTableAddress: { 6917 __ LoadLiteral(out, codegen_->DeduplicateJitStringLiteral(load->GetDexFile(), 6918 load->GetStringIndex(), 6919 load->GetString())); 6920 // /* GcRoot<mirror::String> */ out = *out 6921 GenerateGcRootFieldLoad(load, out_loc, out, /* offset */ 0, kCompilerReadBarrierOption); 6922 return; 6923 } 6924 default: 6925 break; 6926 } 6927 6928 // TODO: Consider re-adding the compiler code to do string dex cache lookup again. 6929 DCHECK(load_kind == HLoadString::LoadKind::kDexCacheViaMethod); 6930 InvokeRuntimeCallingConvention calling_convention; 6931 DCHECK_EQ(calling_convention.GetRegisterAt(0), out); 6932 __ LoadImmediate(calling_convention.GetRegisterAt(0), load->GetStringIndex().index_); 6933 codegen_->InvokeRuntime(kQuickResolveString, load, load->GetDexPc()); 6934 CheckEntrypointTypes<kQuickResolveString, void*, uint32_t>(); 6935} 6936 6937static int32_t GetExceptionTlsOffset() { 6938 return Thread::ExceptionOffset<kArmPointerSize>().Int32Value(); 6939} 6940 6941void LocationsBuilderARM::VisitLoadException(HLoadException* load) { 6942 LocationSummary* locations = 6943 new (GetGraph()->GetArena()) LocationSummary(load, LocationSummary::kNoCall); 6944 locations->SetOut(Location::RequiresRegister()); 6945} 6946 6947void InstructionCodeGeneratorARM::VisitLoadException(HLoadException* load) { 6948 Register out = load->GetLocations()->Out().AsRegister<Register>(); 6949 __ LoadFromOffset(kLoadWord, out, TR, GetExceptionTlsOffset()); 6950} 6951 6952void LocationsBuilderARM::VisitClearException(HClearException* clear) { 6953 new (GetGraph()->GetArena()) LocationSummary(clear, LocationSummary::kNoCall); 6954} 6955 6956void InstructionCodeGeneratorARM::VisitClearException(HClearException* clear ATTRIBUTE_UNUSED) { 6957 __ LoadImmediate(IP, 0); 6958 __ StoreToOffset(kStoreWord, IP, TR, GetExceptionTlsOffset()); 6959} 6960 6961void LocationsBuilderARM::VisitThrow(HThrow* instruction) { 6962 LocationSummary* locations = 6963 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCallOnMainOnly); 6964 InvokeRuntimeCallingConvention calling_convention; 6965 locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(0))); 6966} 6967 6968void InstructionCodeGeneratorARM::VisitThrow(HThrow* instruction) { 6969 codegen_->InvokeRuntime(kQuickDeliverException, instruction, instruction->GetDexPc()); 6970 CheckEntrypointTypes<kQuickDeliverException, void, mirror::Object*>(); 6971} 6972 6973// Temp is used for read barrier. 6974static size_t NumberOfInstanceOfTemps(TypeCheckKind type_check_kind) { 6975 if (kEmitCompilerReadBarrier && 6976 (kUseBakerReadBarrier || 6977 type_check_kind == TypeCheckKind::kAbstractClassCheck || 6978 type_check_kind == TypeCheckKind::kClassHierarchyCheck || 6979 type_check_kind == TypeCheckKind::kArrayObjectCheck)) { 6980 return 1; 6981 } 6982 return 0; 6983} 6984 6985// Interface case has 3 temps, one for holding the number of interfaces, one for the current 6986// interface pointer, one for loading the current interface. 6987// The other checks have one temp for loading the object's class. 6988static size_t NumberOfCheckCastTemps(TypeCheckKind type_check_kind) { 6989 if (type_check_kind == TypeCheckKind::kInterfaceCheck) { 6990 return 3; 6991 } 6992 return 1 + NumberOfInstanceOfTemps(type_check_kind); 6993} 6994 6995void LocationsBuilderARM::VisitInstanceOf(HInstanceOf* instruction) { 6996 LocationSummary::CallKind call_kind = LocationSummary::kNoCall; 6997 TypeCheckKind type_check_kind = instruction->GetTypeCheckKind(); 6998 bool baker_read_barrier_slow_path = false; 6999 switch (type_check_kind) { 7000 case TypeCheckKind::kExactCheck: 7001 case TypeCheckKind::kAbstractClassCheck: 7002 case TypeCheckKind::kClassHierarchyCheck: 7003 case TypeCheckKind::kArrayObjectCheck: 7004 call_kind = 7005 kEmitCompilerReadBarrier ? LocationSummary::kCallOnSlowPath : LocationSummary::kNoCall; 7006 baker_read_barrier_slow_path = kUseBakerReadBarrier; 7007 break; 7008 case TypeCheckKind::kArrayCheck: 7009 case TypeCheckKind::kUnresolvedCheck: 7010 case TypeCheckKind::kInterfaceCheck: 7011 call_kind = LocationSummary::kCallOnSlowPath; 7012 break; 7013 } 7014 7015 LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction, call_kind); 7016 if (baker_read_barrier_slow_path) { 7017 locations->SetCustomSlowPathCallerSaves(RegisterSet::Empty()); // No caller-save registers. 7018 } 7019 locations->SetInAt(0, Location::RequiresRegister()); 7020 locations->SetInAt(1, Location::RequiresRegister()); 7021 // The "out" register is used as a temporary, so it overlaps with the inputs. 7022 // Note that TypeCheckSlowPathARM uses this register too. 7023 locations->SetOut(Location::RequiresRegister(), Location::kOutputOverlap); 7024 locations->AddRegisterTemps(NumberOfInstanceOfTemps(type_check_kind)); 7025} 7026 7027void InstructionCodeGeneratorARM::VisitInstanceOf(HInstanceOf* instruction) { 7028 TypeCheckKind type_check_kind = instruction->GetTypeCheckKind(); 7029 LocationSummary* locations = instruction->GetLocations(); 7030 Location obj_loc = locations->InAt(0); 7031 Register obj = obj_loc.AsRegister<Register>(); 7032 Register cls = locations->InAt(1).AsRegister<Register>(); 7033 Location out_loc = locations->Out(); 7034 Register out = out_loc.AsRegister<Register>(); 7035 const size_t num_temps = NumberOfInstanceOfTemps(type_check_kind); 7036 DCHECK_LE(num_temps, 1u); 7037 Location maybe_temp_loc = (num_temps >= 1) ? locations->GetTemp(0) : Location::NoLocation(); 7038 uint32_t class_offset = mirror::Object::ClassOffset().Int32Value(); 7039 uint32_t super_offset = mirror::Class::SuperClassOffset().Int32Value(); 7040 uint32_t component_offset = mirror::Class::ComponentTypeOffset().Int32Value(); 7041 uint32_t primitive_offset = mirror::Class::PrimitiveTypeOffset().Int32Value(); 7042 Label done, zero; 7043 Label* final_label = codegen_->GetFinalLabel(instruction, &done); 7044 SlowPathCodeARM* slow_path = nullptr; 7045 7046 // Return 0 if `obj` is null. 7047 // avoid null check if we know obj is not null. 7048 if (instruction->MustDoNullCheck()) { 7049 __ CompareAndBranchIfZero(obj, &zero); 7050 } 7051 7052 switch (type_check_kind) { 7053 case TypeCheckKind::kExactCheck: { 7054 // /* HeapReference<Class> */ out = obj->klass_ 7055 GenerateReferenceLoadTwoRegisters(instruction, 7056 out_loc, 7057 obj_loc, 7058 class_offset, 7059 maybe_temp_loc, 7060 kCompilerReadBarrierOption); 7061 __ cmp(out, ShifterOperand(cls)); 7062 // Classes must be equal for the instanceof to succeed. 7063 __ b(&zero, NE); 7064 __ LoadImmediate(out, 1); 7065 __ b(final_label); 7066 break; 7067 } 7068 7069 case TypeCheckKind::kAbstractClassCheck: { 7070 // /* HeapReference<Class> */ out = obj->klass_ 7071 GenerateReferenceLoadTwoRegisters(instruction, 7072 out_loc, 7073 obj_loc, 7074 class_offset, 7075 maybe_temp_loc, 7076 kCompilerReadBarrierOption); 7077 // If the class is abstract, we eagerly fetch the super class of the 7078 // object to avoid doing a comparison we know will fail. 7079 Label loop; 7080 __ Bind(&loop); 7081 // /* HeapReference<Class> */ out = out->super_class_ 7082 GenerateReferenceLoadOneRegister(instruction, 7083 out_loc, 7084 super_offset, 7085 maybe_temp_loc, 7086 kCompilerReadBarrierOption); 7087 // If `out` is null, we use it for the result, and jump to `done`. 7088 __ CompareAndBranchIfZero(out, final_label); 7089 __ cmp(out, ShifterOperand(cls)); 7090 __ b(&loop, NE); 7091 __ LoadImmediate(out, 1); 7092 if (zero.IsLinked()) { 7093 __ b(final_label); 7094 } 7095 break; 7096 } 7097 7098 case TypeCheckKind::kClassHierarchyCheck: { 7099 // /* HeapReference<Class> */ out = obj->klass_ 7100 GenerateReferenceLoadTwoRegisters(instruction, 7101 out_loc, 7102 obj_loc, 7103 class_offset, 7104 maybe_temp_loc, 7105 kCompilerReadBarrierOption); 7106 // Walk over the class hierarchy to find a match. 7107 Label loop, success; 7108 __ Bind(&loop); 7109 __ cmp(out, ShifterOperand(cls)); 7110 __ b(&success, EQ); 7111 // /* HeapReference<Class> */ out = out->super_class_ 7112 GenerateReferenceLoadOneRegister(instruction, 7113 out_loc, 7114 super_offset, 7115 maybe_temp_loc, 7116 kCompilerReadBarrierOption); 7117 __ CompareAndBranchIfNonZero(out, &loop); 7118 // If `out` is null, we use it for the result, and jump to `done`. 7119 __ b(final_label); 7120 __ Bind(&success); 7121 __ LoadImmediate(out, 1); 7122 if (zero.IsLinked()) { 7123 __ b(final_label); 7124 } 7125 break; 7126 } 7127 7128 case TypeCheckKind::kArrayObjectCheck: { 7129 // /* HeapReference<Class> */ out = obj->klass_ 7130 GenerateReferenceLoadTwoRegisters(instruction, 7131 out_loc, 7132 obj_loc, 7133 class_offset, 7134 maybe_temp_loc, 7135 kCompilerReadBarrierOption); 7136 // Do an exact check. 7137 Label exact_check; 7138 __ cmp(out, ShifterOperand(cls)); 7139 __ b(&exact_check, EQ); 7140 // Otherwise, we need to check that the object's class is a non-primitive array. 7141 // /* HeapReference<Class> */ out = out->component_type_ 7142 GenerateReferenceLoadOneRegister(instruction, 7143 out_loc, 7144 component_offset, 7145 maybe_temp_loc, 7146 kCompilerReadBarrierOption); 7147 // If `out` is null, we use it for the result, and jump to `done`. 7148 __ CompareAndBranchIfZero(out, final_label); 7149 __ LoadFromOffset(kLoadUnsignedHalfword, out, out, primitive_offset); 7150 static_assert(Primitive::kPrimNot == 0, "Expected 0 for kPrimNot"); 7151 __ CompareAndBranchIfNonZero(out, &zero); 7152 __ Bind(&exact_check); 7153 __ LoadImmediate(out, 1); 7154 __ b(final_label); 7155 break; 7156 } 7157 7158 case TypeCheckKind::kArrayCheck: { 7159 // No read barrier since the slow path will retry upon failure. 7160 // /* HeapReference<Class> */ out = obj->klass_ 7161 GenerateReferenceLoadTwoRegisters(instruction, 7162 out_loc, 7163 obj_loc, 7164 class_offset, 7165 maybe_temp_loc, 7166 kWithoutReadBarrier); 7167 __ cmp(out, ShifterOperand(cls)); 7168 DCHECK(locations->OnlyCallsOnSlowPath()); 7169 slow_path = new (GetGraph()->GetArena()) TypeCheckSlowPathARM(instruction, 7170 /* is_fatal */ false); 7171 codegen_->AddSlowPath(slow_path); 7172 __ b(slow_path->GetEntryLabel(), NE); 7173 __ LoadImmediate(out, 1); 7174 if (zero.IsLinked()) { 7175 __ b(final_label); 7176 } 7177 break; 7178 } 7179 7180 case TypeCheckKind::kUnresolvedCheck: 7181 case TypeCheckKind::kInterfaceCheck: { 7182 // Note that we indeed only call on slow path, but we always go 7183 // into the slow path for the unresolved and interface check 7184 // cases. 7185 // 7186 // We cannot directly call the InstanceofNonTrivial runtime 7187 // entry point without resorting to a type checking slow path 7188 // here (i.e. by calling InvokeRuntime directly), as it would 7189 // require to assign fixed registers for the inputs of this 7190 // HInstanceOf instruction (following the runtime calling 7191 // convention), which might be cluttered by the potential first 7192 // read barrier emission at the beginning of this method. 7193 // 7194 // TODO: Introduce a new runtime entry point taking the object 7195 // to test (instead of its class) as argument, and let it deal 7196 // with the read barrier issues. This will let us refactor this 7197 // case of the `switch` code as it was previously (with a direct 7198 // call to the runtime not using a type checking slow path). 7199 // This should also be beneficial for the other cases above. 7200 DCHECK(locations->OnlyCallsOnSlowPath()); 7201 slow_path = new (GetGraph()->GetArena()) TypeCheckSlowPathARM(instruction, 7202 /* is_fatal */ false); 7203 codegen_->AddSlowPath(slow_path); 7204 __ b(slow_path->GetEntryLabel()); 7205 if (zero.IsLinked()) { 7206 __ b(final_label); 7207 } 7208 break; 7209 } 7210 } 7211 7212 if (zero.IsLinked()) { 7213 __ Bind(&zero); 7214 __ LoadImmediate(out, 0); 7215 } 7216 7217 if (done.IsLinked()) { 7218 __ Bind(&done); 7219 } 7220 7221 if (slow_path != nullptr) { 7222 __ Bind(slow_path->GetExitLabel()); 7223 } 7224} 7225 7226void LocationsBuilderARM::VisitCheckCast(HCheckCast* instruction) { 7227 LocationSummary::CallKind call_kind = LocationSummary::kNoCall; 7228 bool throws_into_catch = instruction->CanThrowIntoCatchBlock(); 7229 7230 TypeCheckKind type_check_kind = instruction->GetTypeCheckKind(); 7231 switch (type_check_kind) { 7232 case TypeCheckKind::kExactCheck: 7233 case TypeCheckKind::kAbstractClassCheck: 7234 case TypeCheckKind::kClassHierarchyCheck: 7235 case TypeCheckKind::kArrayObjectCheck: 7236 call_kind = (throws_into_catch || kEmitCompilerReadBarrier) ? 7237 LocationSummary::kCallOnSlowPath : 7238 LocationSummary::kNoCall; // In fact, call on a fatal (non-returning) slow path. 7239 break; 7240 case TypeCheckKind::kArrayCheck: 7241 case TypeCheckKind::kUnresolvedCheck: 7242 case TypeCheckKind::kInterfaceCheck: 7243 call_kind = LocationSummary::kCallOnSlowPath; 7244 break; 7245 } 7246 7247 LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction, call_kind); 7248 locations->SetInAt(0, Location::RequiresRegister()); 7249 locations->SetInAt(1, Location::RequiresRegister()); 7250 locations->AddRegisterTemps(NumberOfCheckCastTemps(type_check_kind)); 7251} 7252 7253void InstructionCodeGeneratorARM::VisitCheckCast(HCheckCast* instruction) { 7254 TypeCheckKind type_check_kind = instruction->GetTypeCheckKind(); 7255 LocationSummary* locations = instruction->GetLocations(); 7256 Location obj_loc = locations->InAt(0); 7257 Register obj = obj_loc.AsRegister<Register>(); 7258 Register cls = locations->InAt(1).AsRegister<Register>(); 7259 Location temp_loc = locations->GetTemp(0); 7260 Register temp = temp_loc.AsRegister<Register>(); 7261 const size_t num_temps = NumberOfCheckCastTemps(type_check_kind); 7262 DCHECK_LE(num_temps, 3u); 7263 Location maybe_temp2_loc = (num_temps >= 2) ? locations->GetTemp(1) : Location::NoLocation(); 7264 Location maybe_temp3_loc = (num_temps >= 3) ? locations->GetTemp(2) : Location::NoLocation(); 7265 const uint32_t class_offset = mirror::Object::ClassOffset().Int32Value(); 7266 const uint32_t super_offset = mirror::Class::SuperClassOffset().Int32Value(); 7267 const uint32_t component_offset = mirror::Class::ComponentTypeOffset().Int32Value(); 7268 const uint32_t primitive_offset = mirror::Class::PrimitiveTypeOffset().Int32Value(); 7269 const uint32_t iftable_offset = mirror::Class::IfTableOffset().Uint32Value(); 7270 const uint32_t array_length_offset = mirror::Array::LengthOffset().Uint32Value(); 7271 const uint32_t object_array_data_offset = 7272 mirror::Array::DataOffset(kHeapReferenceSize).Uint32Value(); 7273 7274 // Always false for read barriers since we may need to go to the entrypoint for non-fatal cases 7275 // from false negatives. The false negatives may come from avoiding read barriers below. Avoiding 7276 // read barriers is done for performance and code size reasons. 7277 bool is_type_check_slow_path_fatal = false; 7278 if (!kEmitCompilerReadBarrier) { 7279 is_type_check_slow_path_fatal = 7280 (type_check_kind == TypeCheckKind::kExactCheck || 7281 type_check_kind == TypeCheckKind::kAbstractClassCheck || 7282 type_check_kind == TypeCheckKind::kClassHierarchyCheck || 7283 type_check_kind == TypeCheckKind::kArrayObjectCheck) && 7284 !instruction->CanThrowIntoCatchBlock(); 7285 } 7286 SlowPathCodeARM* type_check_slow_path = 7287 new (GetGraph()->GetArena()) TypeCheckSlowPathARM(instruction, 7288 is_type_check_slow_path_fatal); 7289 codegen_->AddSlowPath(type_check_slow_path); 7290 7291 Label done; 7292 Label* final_label = codegen_->GetFinalLabel(instruction, &done); 7293 // Avoid null check if we know obj is not null. 7294 if (instruction->MustDoNullCheck()) { 7295 __ CompareAndBranchIfZero(obj, final_label); 7296 } 7297 7298 switch (type_check_kind) { 7299 case TypeCheckKind::kExactCheck: 7300 case TypeCheckKind::kArrayCheck: { 7301 // /* HeapReference<Class> */ temp = obj->klass_ 7302 GenerateReferenceLoadTwoRegisters(instruction, 7303 temp_loc, 7304 obj_loc, 7305 class_offset, 7306 maybe_temp2_loc, 7307 kWithoutReadBarrier); 7308 7309 __ cmp(temp, ShifterOperand(cls)); 7310 // Jump to slow path for throwing the exception or doing a 7311 // more involved array check. 7312 __ b(type_check_slow_path->GetEntryLabel(), NE); 7313 break; 7314 } 7315 7316 case TypeCheckKind::kAbstractClassCheck: { 7317 // /* HeapReference<Class> */ temp = obj->klass_ 7318 GenerateReferenceLoadTwoRegisters(instruction, 7319 temp_loc, 7320 obj_loc, 7321 class_offset, 7322 maybe_temp2_loc, 7323 kWithoutReadBarrier); 7324 7325 // If the class is abstract, we eagerly fetch the super class of the 7326 // object to avoid doing a comparison we know will fail. 7327 Label loop; 7328 __ Bind(&loop); 7329 // /* HeapReference<Class> */ temp = temp->super_class_ 7330 GenerateReferenceLoadOneRegister(instruction, 7331 temp_loc, 7332 super_offset, 7333 maybe_temp2_loc, 7334 kWithoutReadBarrier); 7335 7336 // If the class reference currently in `temp` is null, jump to the slow path to throw the 7337 // exception. 7338 __ CompareAndBranchIfZero(temp, type_check_slow_path->GetEntryLabel()); 7339 7340 // Otherwise, compare the classes. 7341 __ cmp(temp, ShifterOperand(cls)); 7342 __ b(&loop, NE); 7343 break; 7344 } 7345 7346 case TypeCheckKind::kClassHierarchyCheck: { 7347 // /* HeapReference<Class> */ temp = obj->klass_ 7348 GenerateReferenceLoadTwoRegisters(instruction, 7349 temp_loc, 7350 obj_loc, 7351 class_offset, 7352 maybe_temp2_loc, 7353 kWithoutReadBarrier); 7354 7355 // Walk over the class hierarchy to find a match. 7356 Label loop; 7357 __ Bind(&loop); 7358 __ cmp(temp, ShifterOperand(cls)); 7359 __ b(final_label, EQ); 7360 7361 // /* HeapReference<Class> */ temp = temp->super_class_ 7362 GenerateReferenceLoadOneRegister(instruction, 7363 temp_loc, 7364 super_offset, 7365 maybe_temp2_loc, 7366 kWithoutReadBarrier); 7367 7368 // If the class reference currently in `temp` is null, jump to the slow path to throw the 7369 // exception. 7370 __ CompareAndBranchIfZero(temp, type_check_slow_path->GetEntryLabel()); 7371 // Otherwise, jump to the beginning of the loop. 7372 __ b(&loop); 7373 break; 7374 } 7375 7376 case TypeCheckKind::kArrayObjectCheck: { 7377 // /* HeapReference<Class> */ temp = obj->klass_ 7378 GenerateReferenceLoadTwoRegisters(instruction, 7379 temp_loc, 7380 obj_loc, 7381 class_offset, 7382 maybe_temp2_loc, 7383 kWithoutReadBarrier); 7384 7385 // Do an exact check. 7386 __ cmp(temp, ShifterOperand(cls)); 7387 __ b(final_label, EQ); 7388 7389 // Otherwise, we need to check that the object's class is a non-primitive array. 7390 // /* HeapReference<Class> */ temp = temp->component_type_ 7391 GenerateReferenceLoadOneRegister(instruction, 7392 temp_loc, 7393 component_offset, 7394 maybe_temp2_loc, 7395 kWithoutReadBarrier); 7396 // If the component type is null, jump to the slow path to throw the exception. 7397 __ CompareAndBranchIfZero(temp, type_check_slow_path->GetEntryLabel()); 7398 // Otherwise,the object is indeed an array, jump to label `check_non_primitive_component_type` 7399 // to further check that this component type is not a primitive type. 7400 __ LoadFromOffset(kLoadUnsignedHalfword, temp, temp, primitive_offset); 7401 static_assert(Primitive::kPrimNot == 0, "Expected 0 for art::Primitive::kPrimNot"); 7402 __ CompareAndBranchIfNonZero(temp, type_check_slow_path->GetEntryLabel()); 7403 break; 7404 } 7405 7406 case TypeCheckKind::kUnresolvedCheck: 7407 // We always go into the type check slow path for the unresolved check case. 7408 // We cannot directly call the CheckCast runtime entry point 7409 // without resorting to a type checking slow path here (i.e. by 7410 // calling InvokeRuntime directly), as it would require to 7411 // assign fixed registers for the inputs of this HInstanceOf 7412 // instruction (following the runtime calling convention), which 7413 // might be cluttered by the potential first read barrier 7414 // emission at the beginning of this method. 7415 7416 __ b(type_check_slow_path->GetEntryLabel()); 7417 break; 7418 7419 case TypeCheckKind::kInterfaceCheck: { 7420 // Avoid read barriers to improve performance of the fast path. We can not get false 7421 // positives by doing this. 7422 // /* HeapReference<Class> */ temp = obj->klass_ 7423 GenerateReferenceLoadTwoRegisters(instruction, 7424 temp_loc, 7425 obj_loc, 7426 class_offset, 7427 maybe_temp2_loc, 7428 kWithoutReadBarrier); 7429 7430 // /* HeapReference<Class> */ temp = temp->iftable_ 7431 GenerateReferenceLoadTwoRegisters(instruction, 7432 temp_loc, 7433 temp_loc, 7434 iftable_offset, 7435 maybe_temp2_loc, 7436 kWithoutReadBarrier); 7437 // Iftable is never null. 7438 __ ldr(maybe_temp2_loc.AsRegister<Register>(), Address(temp, array_length_offset)); 7439 // Loop through the iftable and check if any class matches. 7440 Label start_loop; 7441 __ Bind(&start_loop); 7442 __ CompareAndBranchIfZero(maybe_temp2_loc.AsRegister<Register>(), 7443 type_check_slow_path->GetEntryLabel()); 7444 __ ldr(maybe_temp3_loc.AsRegister<Register>(), Address(temp, object_array_data_offset)); 7445 __ MaybeUnpoisonHeapReference(maybe_temp3_loc.AsRegister<Register>()); 7446 // Go to next interface. 7447 __ add(temp, temp, ShifterOperand(2 * kHeapReferenceSize)); 7448 __ sub(maybe_temp2_loc.AsRegister<Register>(), 7449 maybe_temp2_loc.AsRegister<Register>(), 7450 ShifterOperand(2)); 7451 // Compare the classes and continue the loop if they do not match. 7452 __ cmp(cls, ShifterOperand(maybe_temp3_loc.AsRegister<Register>())); 7453 __ b(&start_loop, NE); 7454 break; 7455 } 7456 } 7457 7458 if (done.IsLinked()) { 7459 __ Bind(&done); 7460 } 7461 7462 __ Bind(type_check_slow_path->GetExitLabel()); 7463} 7464 7465void LocationsBuilderARM::VisitMonitorOperation(HMonitorOperation* instruction) { 7466 LocationSummary* locations = 7467 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCallOnMainOnly); 7468 InvokeRuntimeCallingConvention calling_convention; 7469 locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(0))); 7470} 7471 7472void InstructionCodeGeneratorARM::VisitMonitorOperation(HMonitorOperation* instruction) { 7473 codegen_->InvokeRuntime(instruction->IsEnter() ? kQuickLockObject : kQuickUnlockObject, 7474 instruction, 7475 instruction->GetDexPc()); 7476 if (instruction->IsEnter()) { 7477 CheckEntrypointTypes<kQuickLockObject, void, mirror::Object*>(); 7478 } else { 7479 CheckEntrypointTypes<kQuickUnlockObject, void, mirror::Object*>(); 7480 } 7481} 7482 7483void LocationsBuilderARM::VisitAnd(HAnd* instruction) { HandleBitwiseOperation(instruction, AND); } 7484void LocationsBuilderARM::VisitOr(HOr* instruction) { HandleBitwiseOperation(instruction, ORR); } 7485void LocationsBuilderARM::VisitXor(HXor* instruction) { HandleBitwiseOperation(instruction, EOR); } 7486 7487void LocationsBuilderARM::HandleBitwiseOperation(HBinaryOperation* instruction, Opcode opcode) { 7488 LocationSummary* locations = 7489 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall); 7490 DCHECK(instruction->GetResultType() == Primitive::kPrimInt 7491 || instruction->GetResultType() == Primitive::kPrimLong); 7492 // Note: GVN reorders commutative operations to have the constant on the right hand side. 7493 locations->SetInAt(0, Location::RequiresRegister()); 7494 locations->SetInAt(1, ArmEncodableConstantOrRegister(instruction->InputAt(1), opcode)); 7495 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 7496} 7497 7498void InstructionCodeGeneratorARM::VisitAnd(HAnd* instruction) { 7499 HandleBitwiseOperation(instruction); 7500} 7501 7502void InstructionCodeGeneratorARM::VisitOr(HOr* instruction) { 7503 HandleBitwiseOperation(instruction); 7504} 7505 7506void InstructionCodeGeneratorARM::VisitXor(HXor* instruction) { 7507 HandleBitwiseOperation(instruction); 7508} 7509 7510 7511void LocationsBuilderARM::VisitBitwiseNegatedRight(HBitwiseNegatedRight* instruction) { 7512 LocationSummary* locations = 7513 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall); 7514 DCHECK(instruction->GetResultType() == Primitive::kPrimInt 7515 || instruction->GetResultType() == Primitive::kPrimLong); 7516 7517 locations->SetInAt(0, Location::RequiresRegister()); 7518 locations->SetInAt(1, Location::RequiresRegister()); 7519 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 7520} 7521 7522void InstructionCodeGeneratorARM::VisitBitwiseNegatedRight(HBitwiseNegatedRight* instruction) { 7523 LocationSummary* locations = instruction->GetLocations(); 7524 Location first = locations->InAt(0); 7525 Location second = locations->InAt(1); 7526 Location out = locations->Out(); 7527 7528 if (instruction->GetResultType() == Primitive::kPrimInt) { 7529 Register first_reg = first.AsRegister<Register>(); 7530 ShifterOperand second_reg(second.AsRegister<Register>()); 7531 Register out_reg = out.AsRegister<Register>(); 7532 7533 switch (instruction->GetOpKind()) { 7534 case HInstruction::kAnd: 7535 __ bic(out_reg, first_reg, second_reg); 7536 break; 7537 case HInstruction::kOr: 7538 __ orn(out_reg, first_reg, second_reg); 7539 break; 7540 // There is no EON on arm. 7541 case HInstruction::kXor: 7542 default: 7543 LOG(FATAL) << "Unexpected instruction " << instruction->DebugName(); 7544 UNREACHABLE(); 7545 } 7546 return; 7547 7548 } else { 7549 DCHECK_EQ(instruction->GetResultType(), Primitive::kPrimLong); 7550 Register first_low = first.AsRegisterPairLow<Register>(); 7551 Register first_high = first.AsRegisterPairHigh<Register>(); 7552 ShifterOperand second_low(second.AsRegisterPairLow<Register>()); 7553 ShifterOperand second_high(second.AsRegisterPairHigh<Register>()); 7554 Register out_low = out.AsRegisterPairLow<Register>(); 7555 Register out_high = out.AsRegisterPairHigh<Register>(); 7556 7557 switch (instruction->GetOpKind()) { 7558 case HInstruction::kAnd: 7559 __ bic(out_low, first_low, second_low); 7560 __ bic(out_high, first_high, second_high); 7561 break; 7562 case HInstruction::kOr: 7563 __ orn(out_low, first_low, second_low); 7564 __ orn(out_high, first_high, second_high); 7565 break; 7566 // There is no EON on arm. 7567 case HInstruction::kXor: 7568 default: 7569 LOG(FATAL) << "Unexpected instruction " << instruction->DebugName(); 7570 UNREACHABLE(); 7571 } 7572 } 7573} 7574 7575void LocationsBuilderARM::VisitDataProcWithShifterOp( 7576 HDataProcWithShifterOp* instruction) { 7577 DCHECK(instruction->GetType() == Primitive::kPrimInt || 7578 instruction->GetType() == Primitive::kPrimLong); 7579 LocationSummary* locations = 7580 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall); 7581 const bool overlap = instruction->GetType() == Primitive::kPrimLong && 7582 HDataProcWithShifterOp::IsExtensionOp(instruction->GetOpKind()); 7583 7584 locations->SetInAt(0, Location::RequiresRegister()); 7585 locations->SetInAt(1, Location::RequiresRegister()); 7586 locations->SetOut(Location::RequiresRegister(), 7587 overlap ? Location::kOutputOverlap : Location::kNoOutputOverlap); 7588} 7589 7590void InstructionCodeGeneratorARM::VisitDataProcWithShifterOp( 7591 HDataProcWithShifterOp* instruction) { 7592 const LocationSummary* const locations = instruction->GetLocations(); 7593 const HInstruction::InstructionKind kind = instruction->GetInstrKind(); 7594 const HDataProcWithShifterOp::OpKind op_kind = instruction->GetOpKind(); 7595 const Location left = locations->InAt(0); 7596 const Location right = locations->InAt(1); 7597 const Location out = locations->Out(); 7598 7599 if (instruction->GetType() == Primitive::kPrimInt) { 7600 DCHECK(!HDataProcWithShifterOp::IsExtensionOp(op_kind)); 7601 7602 const Register second = instruction->InputAt(1)->GetType() == Primitive::kPrimLong 7603 ? right.AsRegisterPairLow<Register>() 7604 : right.AsRegister<Register>(); 7605 7606 GenerateDataProcInstruction(kind, 7607 out.AsRegister<Register>(), 7608 left.AsRegister<Register>(), 7609 ShifterOperand(second, 7610 ShiftFromOpKind(op_kind), 7611 instruction->GetShiftAmount()), 7612 codegen_); 7613 } else { 7614 DCHECK_EQ(instruction->GetType(), Primitive::kPrimLong); 7615 7616 if (HDataProcWithShifterOp::IsExtensionOp(op_kind)) { 7617 const Register second = right.AsRegister<Register>(); 7618 7619 DCHECK_NE(out.AsRegisterPairLow<Register>(), second); 7620 GenerateDataProc(kind, 7621 out, 7622 left, 7623 ShifterOperand(second), 7624 ShifterOperand(second, ASR, 31), 7625 codegen_); 7626 } else { 7627 GenerateLongDataProc(instruction, codegen_); 7628 } 7629 } 7630} 7631 7632void InstructionCodeGeneratorARM::GenerateAndConst(Register out, Register first, uint32_t value) { 7633 // Optimize special cases for individual halfs of `and-long` (`and` is simplified earlier). 7634 if (value == 0xffffffffu) { 7635 if (out != first) { 7636 __ mov(out, ShifterOperand(first)); 7637 } 7638 return; 7639 } 7640 if (value == 0u) { 7641 __ mov(out, ShifterOperand(0)); 7642 return; 7643 } 7644 ShifterOperand so; 7645 if (__ ShifterOperandCanHold(kNoRegister, kNoRegister, AND, value, &so)) { 7646 __ and_(out, first, so); 7647 } else if (__ ShifterOperandCanHold(kNoRegister, kNoRegister, BIC, ~value, &so)) { 7648 __ bic(out, first, ShifterOperand(~value)); 7649 } else { 7650 DCHECK(IsPowerOfTwo(value + 1)); 7651 __ ubfx(out, first, 0, WhichPowerOf2(value + 1)); 7652 } 7653} 7654 7655void InstructionCodeGeneratorARM::GenerateOrrConst(Register out, Register first, uint32_t value) { 7656 // Optimize special cases for individual halfs of `or-long` (`or` is simplified earlier). 7657 if (value == 0u) { 7658 if (out != first) { 7659 __ mov(out, ShifterOperand(first)); 7660 } 7661 return; 7662 } 7663 if (value == 0xffffffffu) { 7664 __ mvn(out, ShifterOperand(0)); 7665 return; 7666 } 7667 ShifterOperand so; 7668 if (__ ShifterOperandCanHold(kNoRegister, kNoRegister, ORR, value, &so)) { 7669 __ orr(out, first, so); 7670 } else { 7671 DCHECK(__ ShifterOperandCanHold(kNoRegister, kNoRegister, ORN, ~value, &so)); 7672 __ orn(out, first, ShifterOperand(~value)); 7673 } 7674} 7675 7676void InstructionCodeGeneratorARM::GenerateEorConst(Register out, Register first, uint32_t value) { 7677 // Optimize special case for individual halfs of `xor-long` (`xor` is simplified earlier). 7678 if (value == 0u) { 7679 if (out != first) { 7680 __ mov(out, ShifterOperand(first)); 7681 } 7682 return; 7683 } 7684 __ eor(out, first, ShifterOperand(value)); 7685} 7686 7687void InstructionCodeGeneratorARM::GenerateAddLongConst(Location out, 7688 Location first, 7689 uint64_t value) { 7690 Register out_low = out.AsRegisterPairLow<Register>(); 7691 Register out_high = out.AsRegisterPairHigh<Register>(); 7692 Register first_low = first.AsRegisterPairLow<Register>(); 7693 Register first_high = first.AsRegisterPairHigh<Register>(); 7694 uint32_t value_low = Low32Bits(value); 7695 uint32_t value_high = High32Bits(value); 7696 if (value_low == 0u) { 7697 if (out_low != first_low) { 7698 __ mov(out_low, ShifterOperand(first_low)); 7699 } 7700 __ AddConstant(out_high, first_high, value_high); 7701 return; 7702 } 7703 __ AddConstantSetFlags(out_low, first_low, value_low); 7704 ShifterOperand so; 7705 if (__ ShifterOperandCanHold(out_high, first_high, ADC, value_high, kCcDontCare, &so)) { 7706 __ adc(out_high, first_high, so); 7707 } else if (__ ShifterOperandCanHold(out_low, first_low, SBC, ~value_high, kCcDontCare, &so)) { 7708 __ sbc(out_high, first_high, so); 7709 } else { 7710 LOG(FATAL) << "Unexpected constant " << value_high; 7711 UNREACHABLE(); 7712 } 7713} 7714 7715void InstructionCodeGeneratorARM::HandleBitwiseOperation(HBinaryOperation* instruction) { 7716 LocationSummary* locations = instruction->GetLocations(); 7717 Location first = locations->InAt(0); 7718 Location second = locations->InAt(1); 7719 Location out = locations->Out(); 7720 7721 if (second.IsConstant()) { 7722 uint64_t value = static_cast<uint64_t>(Int64FromConstant(second.GetConstant())); 7723 uint32_t value_low = Low32Bits(value); 7724 if (instruction->GetResultType() == Primitive::kPrimInt) { 7725 Register first_reg = first.AsRegister<Register>(); 7726 Register out_reg = out.AsRegister<Register>(); 7727 if (instruction->IsAnd()) { 7728 GenerateAndConst(out_reg, first_reg, value_low); 7729 } else if (instruction->IsOr()) { 7730 GenerateOrrConst(out_reg, first_reg, value_low); 7731 } else { 7732 DCHECK(instruction->IsXor()); 7733 GenerateEorConst(out_reg, first_reg, value_low); 7734 } 7735 } else { 7736 DCHECK_EQ(instruction->GetResultType(), Primitive::kPrimLong); 7737 uint32_t value_high = High32Bits(value); 7738 Register first_low = first.AsRegisterPairLow<Register>(); 7739 Register first_high = first.AsRegisterPairHigh<Register>(); 7740 Register out_low = out.AsRegisterPairLow<Register>(); 7741 Register out_high = out.AsRegisterPairHigh<Register>(); 7742 if (instruction->IsAnd()) { 7743 GenerateAndConst(out_low, first_low, value_low); 7744 GenerateAndConst(out_high, first_high, value_high); 7745 } else if (instruction->IsOr()) { 7746 GenerateOrrConst(out_low, first_low, value_low); 7747 GenerateOrrConst(out_high, first_high, value_high); 7748 } else { 7749 DCHECK(instruction->IsXor()); 7750 GenerateEorConst(out_low, first_low, value_low); 7751 GenerateEorConst(out_high, first_high, value_high); 7752 } 7753 } 7754 return; 7755 } 7756 7757 if (instruction->GetResultType() == Primitive::kPrimInt) { 7758 Register first_reg = first.AsRegister<Register>(); 7759 ShifterOperand second_reg(second.AsRegister<Register>()); 7760 Register out_reg = out.AsRegister<Register>(); 7761 if (instruction->IsAnd()) { 7762 __ and_(out_reg, first_reg, second_reg); 7763 } else if (instruction->IsOr()) { 7764 __ orr(out_reg, first_reg, second_reg); 7765 } else { 7766 DCHECK(instruction->IsXor()); 7767 __ eor(out_reg, first_reg, second_reg); 7768 } 7769 } else { 7770 DCHECK_EQ(instruction->GetResultType(), Primitive::kPrimLong); 7771 Register first_low = first.AsRegisterPairLow<Register>(); 7772 Register first_high = first.AsRegisterPairHigh<Register>(); 7773 ShifterOperand second_low(second.AsRegisterPairLow<Register>()); 7774 ShifterOperand second_high(second.AsRegisterPairHigh<Register>()); 7775 Register out_low = out.AsRegisterPairLow<Register>(); 7776 Register out_high = out.AsRegisterPairHigh<Register>(); 7777 if (instruction->IsAnd()) { 7778 __ and_(out_low, first_low, second_low); 7779 __ and_(out_high, first_high, second_high); 7780 } else if (instruction->IsOr()) { 7781 __ orr(out_low, first_low, second_low); 7782 __ orr(out_high, first_high, second_high); 7783 } else { 7784 DCHECK(instruction->IsXor()); 7785 __ eor(out_low, first_low, second_low); 7786 __ eor(out_high, first_high, second_high); 7787 } 7788 } 7789} 7790 7791void InstructionCodeGeneratorARM::GenerateReferenceLoadOneRegister( 7792 HInstruction* instruction, 7793 Location out, 7794 uint32_t offset, 7795 Location maybe_temp, 7796 ReadBarrierOption read_barrier_option) { 7797 Register out_reg = out.AsRegister<Register>(); 7798 if (read_barrier_option == kWithReadBarrier) { 7799 CHECK(kEmitCompilerReadBarrier); 7800 DCHECK(maybe_temp.IsRegister()) << maybe_temp; 7801 if (kUseBakerReadBarrier) { 7802 // Load with fast path based Baker's read barrier. 7803 // /* HeapReference<Object> */ out = *(out + offset) 7804 codegen_->GenerateFieldLoadWithBakerReadBarrier( 7805 instruction, out, out_reg, offset, maybe_temp, /* needs_null_check */ false); 7806 } else { 7807 // Load with slow path based read barrier. 7808 // Save the value of `out` into `maybe_temp` before overwriting it 7809 // in the following move operation, as we will need it for the 7810 // read barrier below. 7811 __ Mov(maybe_temp.AsRegister<Register>(), out_reg); 7812 // /* HeapReference<Object> */ out = *(out + offset) 7813 __ LoadFromOffset(kLoadWord, out_reg, out_reg, offset); 7814 codegen_->GenerateReadBarrierSlow(instruction, out, out, maybe_temp, offset); 7815 } 7816 } else { 7817 // Plain load with no read barrier. 7818 // /* HeapReference<Object> */ out = *(out + offset) 7819 __ LoadFromOffset(kLoadWord, out_reg, out_reg, offset); 7820 __ MaybeUnpoisonHeapReference(out_reg); 7821 } 7822} 7823 7824void InstructionCodeGeneratorARM::GenerateReferenceLoadTwoRegisters( 7825 HInstruction* instruction, 7826 Location out, 7827 Location obj, 7828 uint32_t offset, 7829 Location maybe_temp, 7830 ReadBarrierOption read_barrier_option) { 7831 Register out_reg = out.AsRegister<Register>(); 7832 Register obj_reg = obj.AsRegister<Register>(); 7833 if (read_barrier_option == kWithReadBarrier) { 7834 CHECK(kEmitCompilerReadBarrier); 7835 if (kUseBakerReadBarrier) { 7836 DCHECK(maybe_temp.IsRegister()) << maybe_temp; 7837 // Load with fast path based Baker's read barrier. 7838 // /* HeapReference<Object> */ out = *(obj + offset) 7839 codegen_->GenerateFieldLoadWithBakerReadBarrier( 7840 instruction, out, obj_reg, offset, maybe_temp, /* needs_null_check */ false); 7841 } else { 7842 // Load with slow path based read barrier. 7843 // /* HeapReference<Object> */ out = *(obj + offset) 7844 __ LoadFromOffset(kLoadWord, out_reg, obj_reg, offset); 7845 codegen_->GenerateReadBarrierSlow(instruction, out, out, obj, offset); 7846 } 7847 } else { 7848 // Plain load with no read barrier. 7849 // /* HeapReference<Object> */ out = *(obj + offset) 7850 __ LoadFromOffset(kLoadWord, out_reg, obj_reg, offset); 7851 __ MaybeUnpoisonHeapReference(out_reg); 7852 } 7853} 7854 7855void InstructionCodeGeneratorARM::GenerateGcRootFieldLoad(HInstruction* instruction, 7856 Location root, 7857 Register obj, 7858 uint32_t offset, 7859 ReadBarrierOption read_barrier_option) { 7860 Register root_reg = root.AsRegister<Register>(); 7861 if (read_barrier_option == kWithReadBarrier) { 7862 DCHECK(kEmitCompilerReadBarrier); 7863 if (kUseBakerReadBarrier) { 7864 // Fast path implementation of art::ReadBarrier::BarrierForRoot when 7865 // Baker's read barrier are used. 7866 // 7867 // Note that we do not actually check the value of 7868 // `GetIsGcMarking()` to decide whether to mark the loaded GC 7869 // root or not. Instead, we load into `temp` the read barrier 7870 // mark entry point corresponding to register `root`. If `temp` 7871 // is null, it means that `GetIsGcMarking()` is false, and vice 7872 // versa. 7873 // 7874 // temp = Thread::Current()->pReadBarrierMarkReg ## root.reg() 7875 // GcRoot<mirror::Object> root = *(obj+offset); // Original reference load. 7876 // if (temp != nullptr) { // <=> Thread::Current()->GetIsGcMarking() 7877 // // Slow path. 7878 // root = temp(root); // root = ReadBarrier::Mark(root); // Runtime entry point call. 7879 // } 7880 7881 // Slow path marking the GC root `root`. The entrypoint will already be loaded in `temp`. 7882 Location temp = Location::RegisterLocation(LR); 7883 SlowPathCodeARM* slow_path = new (GetGraph()->GetArena()) ReadBarrierMarkSlowPathARM( 7884 instruction, root, /* entrypoint */ temp); 7885 codegen_->AddSlowPath(slow_path); 7886 7887 // temp = Thread::Current()->pReadBarrierMarkReg ## root.reg() 7888 const int32_t entry_point_offset = 7889 CodeGenerator::GetReadBarrierMarkEntryPointsOffset<kArmPointerSize>(root.reg()); 7890 // Loading the entrypoint does not require a load acquire since it is only changed when 7891 // threads are suspended or running a checkpoint. 7892 __ LoadFromOffset(kLoadWord, temp.AsRegister<Register>(), TR, entry_point_offset); 7893 7894 // /* GcRoot<mirror::Object> */ root = *(obj + offset) 7895 __ LoadFromOffset(kLoadWord, root_reg, obj, offset); 7896 static_assert( 7897 sizeof(mirror::CompressedReference<mirror::Object>) == sizeof(GcRoot<mirror::Object>), 7898 "art::mirror::CompressedReference<mirror::Object> and art::GcRoot<mirror::Object> " 7899 "have different sizes."); 7900 static_assert(sizeof(mirror::CompressedReference<mirror::Object>) == sizeof(int32_t), 7901 "art::mirror::CompressedReference<mirror::Object> and int32_t " 7902 "have different sizes."); 7903 7904 // The entrypoint is null when the GC is not marking, this prevents one load compared to 7905 // checking GetIsGcMarking. 7906 __ CompareAndBranchIfNonZero(temp.AsRegister<Register>(), slow_path->GetEntryLabel()); 7907 __ Bind(slow_path->GetExitLabel()); 7908 } else { 7909 // GC root loaded through a slow path for read barriers other 7910 // than Baker's. 7911 // /* GcRoot<mirror::Object>* */ root = obj + offset 7912 __ AddConstant(root_reg, obj, offset); 7913 // /* mirror::Object* */ root = root->Read() 7914 codegen_->GenerateReadBarrierForRootSlow(instruction, root, root); 7915 } 7916 } else { 7917 // Plain GC root load with no read barrier. 7918 // /* GcRoot<mirror::Object> */ root = *(obj + offset) 7919 __ LoadFromOffset(kLoadWord, root_reg, obj, offset); 7920 // Note that GC roots are not affected by heap poisoning, thus we 7921 // do not have to unpoison `root_reg` here. 7922 } 7923} 7924 7925void CodeGeneratorARM::GenerateFieldLoadWithBakerReadBarrier(HInstruction* instruction, 7926 Location ref, 7927 Register obj, 7928 uint32_t offset, 7929 Location temp, 7930 bool needs_null_check) { 7931 DCHECK(kEmitCompilerReadBarrier); 7932 DCHECK(kUseBakerReadBarrier); 7933 7934 // /* HeapReference<Object> */ ref = *(obj + offset) 7935 Location no_index = Location::NoLocation(); 7936 ScaleFactor no_scale_factor = TIMES_1; 7937 GenerateReferenceLoadWithBakerReadBarrier( 7938 instruction, ref, obj, offset, no_index, no_scale_factor, temp, needs_null_check); 7939} 7940 7941void CodeGeneratorARM::GenerateArrayLoadWithBakerReadBarrier(HInstruction* instruction, 7942 Location ref, 7943 Register obj, 7944 uint32_t data_offset, 7945 Location index, 7946 Location temp, 7947 bool needs_null_check) { 7948 DCHECK(kEmitCompilerReadBarrier); 7949 DCHECK(kUseBakerReadBarrier); 7950 7951 static_assert( 7952 sizeof(mirror::HeapReference<mirror::Object>) == sizeof(int32_t), 7953 "art::mirror::HeapReference<art::mirror::Object> and int32_t have different sizes."); 7954 // /* HeapReference<Object> */ ref = 7955 // *(obj + data_offset + index * sizeof(HeapReference<Object>)) 7956 ScaleFactor scale_factor = TIMES_4; 7957 GenerateReferenceLoadWithBakerReadBarrier( 7958 instruction, ref, obj, data_offset, index, scale_factor, temp, needs_null_check); 7959} 7960 7961void CodeGeneratorARM::GenerateReferenceLoadWithBakerReadBarrier(HInstruction* instruction, 7962 Location ref, 7963 Register obj, 7964 uint32_t offset, 7965 Location index, 7966 ScaleFactor scale_factor, 7967 Location temp, 7968 bool needs_null_check, 7969 bool always_update_field, 7970 Register* temp2) { 7971 DCHECK(kEmitCompilerReadBarrier); 7972 DCHECK(kUseBakerReadBarrier); 7973 7974 // Query `art::Thread::Current()->GetIsGcMarking()` to decide 7975 // whether we need to enter the slow path to mark the reference. 7976 // Then, in the slow path, check the gray bit in the lock word of 7977 // the reference's holder (`obj`) to decide whether to mark `ref` or 7978 // not. 7979 // 7980 // Note that we do not actually check the value of `GetIsGcMarking()`; 7981 // instead, we load into `temp3` the read barrier mark entry point 7982 // corresponding to register `ref`. If `temp3` is null, it means 7983 // that `GetIsGcMarking()` is false, and vice versa. 7984 // 7985 // temp3 = Thread::Current()->pReadBarrierMarkReg ## root.reg() 7986 // if (temp3 != nullptr) { // <=> Thread::Current()->GetIsGcMarking() 7987 // // Slow path. 7988 // uint32_t rb_state = Lockword(obj->monitor_).ReadBarrierState(); 7989 // lfence; // Load fence or artificial data dependency to prevent load-load reordering 7990 // HeapReference<mirror::Object> ref = *src; // Original reference load. 7991 // bool is_gray = (rb_state == ReadBarrier::GrayState()); 7992 // if (is_gray) { 7993 // ref = temp3(ref); // ref = ReadBarrier::Mark(ref); // Runtime entry point call. 7994 // } 7995 // } else { 7996 // HeapReference<mirror::Object> ref = *src; // Original reference load. 7997 // } 7998 7999 Register temp_reg = temp.AsRegister<Register>(); 8000 8001 // Slow path marking the object `ref` when the GC is marking. The 8002 // entrypoint will already be loaded in `temp3`. 8003 Location temp3 = Location::RegisterLocation(LR); 8004 SlowPathCodeARM* slow_path; 8005 if (always_update_field) { 8006 DCHECK(temp2 != nullptr); 8007 // LoadReferenceWithBakerReadBarrierAndUpdateFieldSlowPathARM only 8008 // supports address of the form `obj + field_offset`, where `obj` 8009 // is a register and `field_offset` is a register pair (of which 8010 // only the lower half is used). Thus `offset` and `scale_factor` 8011 // above are expected to be null in this code path. 8012 DCHECK_EQ(offset, 0u); 8013 DCHECK_EQ(scale_factor, ScaleFactor::TIMES_1); 8014 Location field_offset = index; 8015 slow_path = 8016 new (GetGraph()->GetArena()) LoadReferenceWithBakerReadBarrierAndUpdateFieldSlowPathARM( 8017 instruction, 8018 ref, 8019 obj, 8020 offset, 8021 /* index */ field_offset, 8022 scale_factor, 8023 needs_null_check, 8024 temp_reg, 8025 *temp2, 8026 /* entrypoint */ temp3); 8027 } else { 8028 slow_path = new (GetGraph()->GetArena()) LoadReferenceWithBakerReadBarrierSlowPathARM( 8029 instruction, 8030 ref, 8031 obj, 8032 offset, 8033 index, 8034 scale_factor, 8035 needs_null_check, 8036 temp_reg, 8037 /* entrypoint */ temp3); 8038 } 8039 AddSlowPath(slow_path); 8040 8041 // temp3 = Thread::Current()->pReadBarrierMarkReg ## ref.reg() 8042 const int32_t entry_point_offset = 8043 CodeGenerator::GetReadBarrierMarkEntryPointsOffset<kArmPointerSize>(ref.reg()); 8044 // Loading the entrypoint does not require a load acquire since it is only changed when 8045 // threads are suspended or running a checkpoint. 8046 __ LoadFromOffset(kLoadWord, temp3.AsRegister<Register>(), TR, entry_point_offset); 8047 // The entrypoint is null when the GC is not marking, this prevents one load compared to 8048 // checking GetIsGcMarking. 8049 __ CompareAndBranchIfNonZero(temp3.AsRegister<Register>(), slow_path->GetEntryLabel()); 8050 // Fast path: just load the reference. 8051 GenerateRawReferenceLoad(instruction, ref, obj, offset, index, scale_factor, needs_null_check); 8052 __ Bind(slow_path->GetExitLabel()); 8053} 8054 8055void CodeGeneratorARM::GenerateRawReferenceLoad(HInstruction* instruction, 8056 Location ref, 8057 Register obj, 8058 uint32_t offset, 8059 Location index, 8060 ScaleFactor scale_factor, 8061 bool needs_null_check) { 8062 Register ref_reg = ref.AsRegister<Register>(); 8063 8064 if (index.IsValid()) { 8065 // Load types involving an "index": ArrayGet, 8066 // UnsafeGetObject/UnsafeGetObjectVolatile and UnsafeCASObject 8067 // intrinsics. 8068 // /* HeapReference<mirror::Object> */ ref = *(obj + offset + (index << scale_factor)) 8069 if (index.IsConstant()) { 8070 size_t computed_offset = 8071 (index.GetConstant()->AsIntConstant()->GetValue() << scale_factor) + offset; 8072 __ LoadFromOffset(kLoadWord, ref_reg, obj, computed_offset); 8073 } else { 8074 // Handle the special case of the 8075 // UnsafeGetObject/UnsafeGetObjectVolatile and UnsafeCASObject 8076 // intrinsics, which use a register pair as index ("long 8077 // offset"), of which only the low part contains data. 8078 Register index_reg = index.IsRegisterPair() 8079 ? index.AsRegisterPairLow<Register>() 8080 : index.AsRegister<Register>(); 8081 __ add(IP, obj, ShifterOperand(index_reg, LSL, scale_factor)); 8082 __ LoadFromOffset(kLoadWord, ref_reg, IP, offset); 8083 } 8084 } else { 8085 // /* HeapReference<mirror::Object> */ ref = *(obj + offset) 8086 __ LoadFromOffset(kLoadWord, ref_reg, obj, offset); 8087 } 8088 8089 if (needs_null_check) { 8090 MaybeRecordImplicitNullCheck(instruction); 8091 } 8092 8093 // Object* ref = ref_addr->AsMirrorPtr() 8094 __ MaybeUnpoisonHeapReference(ref_reg); 8095} 8096 8097void CodeGeneratorARM::GenerateReadBarrierSlow(HInstruction* instruction, 8098 Location out, 8099 Location ref, 8100 Location obj, 8101 uint32_t offset, 8102 Location index) { 8103 DCHECK(kEmitCompilerReadBarrier); 8104 8105 // Insert a slow path based read barrier *after* the reference load. 8106 // 8107 // If heap poisoning is enabled, the unpoisoning of the loaded 8108 // reference will be carried out by the runtime within the slow 8109 // path. 8110 // 8111 // Note that `ref` currently does not get unpoisoned (when heap 8112 // poisoning is enabled), which is alright as the `ref` argument is 8113 // not used by the artReadBarrierSlow entry point. 8114 // 8115 // TODO: Unpoison `ref` when it is used by artReadBarrierSlow. 8116 SlowPathCodeARM* slow_path = new (GetGraph()->GetArena()) 8117 ReadBarrierForHeapReferenceSlowPathARM(instruction, out, ref, obj, offset, index); 8118 AddSlowPath(slow_path); 8119 8120 __ b(slow_path->GetEntryLabel()); 8121 __ Bind(slow_path->GetExitLabel()); 8122} 8123 8124void CodeGeneratorARM::MaybeGenerateReadBarrierSlow(HInstruction* instruction, 8125 Location out, 8126 Location ref, 8127 Location obj, 8128 uint32_t offset, 8129 Location index) { 8130 if (kEmitCompilerReadBarrier) { 8131 // Baker's read barriers shall be handled by the fast path 8132 // (CodeGeneratorARM::GenerateReferenceLoadWithBakerReadBarrier). 8133 DCHECK(!kUseBakerReadBarrier); 8134 // If heap poisoning is enabled, unpoisoning will be taken care of 8135 // by the runtime within the slow path. 8136 GenerateReadBarrierSlow(instruction, out, ref, obj, offset, index); 8137 } else if (kPoisonHeapReferences) { 8138 __ UnpoisonHeapReference(out.AsRegister<Register>()); 8139 } 8140} 8141 8142void CodeGeneratorARM::GenerateReadBarrierForRootSlow(HInstruction* instruction, 8143 Location out, 8144 Location root) { 8145 DCHECK(kEmitCompilerReadBarrier); 8146 8147 // Insert a slow path based read barrier *after* the GC root load. 8148 // 8149 // Note that GC roots are not affected by heap poisoning, so we do 8150 // not need to do anything special for this here. 8151 SlowPathCodeARM* slow_path = 8152 new (GetGraph()->GetArena()) ReadBarrierForRootSlowPathARM(instruction, out, root); 8153 AddSlowPath(slow_path); 8154 8155 __ b(slow_path->GetEntryLabel()); 8156 __ Bind(slow_path->GetExitLabel()); 8157} 8158 8159HInvokeStaticOrDirect::DispatchInfo CodeGeneratorARM::GetSupportedInvokeStaticOrDirectDispatch( 8160 const HInvokeStaticOrDirect::DispatchInfo& desired_dispatch_info, 8161 HInvokeStaticOrDirect* invoke ATTRIBUTE_UNUSED) { 8162 return desired_dispatch_info; 8163} 8164 8165Register CodeGeneratorARM::GetInvokeStaticOrDirectExtraParameter(HInvokeStaticOrDirect* invoke, 8166 Register temp) { 8167 DCHECK_EQ(invoke->InputCount(), invoke->GetNumberOfArguments() + 1u); 8168 Location location = invoke->GetLocations()->InAt(invoke->GetSpecialInputIndex()); 8169 if (!invoke->GetLocations()->Intrinsified()) { 8170 return location.AsRegister<Register>(); 8171 } 8172 // For intrinsics we allow any location, so it may be on the stack. 8173 if (!location.IsRegister()) { 8174 __ LoadFromOffset(kLoadWord, temp, SP, location.GetStackIndex()); 8175 return temp; 8176 } 8177 // For register locations, check if the register was saved. If so, get it from the stack. 8178 // Note: There is a chance that the register was saved but not overwritten, so we could 8179 // save one load. However, since this is just an intrinsic slow path we prefer this 8180 // simple and more robust approach rather that trying to determine if that's the case. 8181 SlowPathCode* slow_path = GetCurrentSlowPath(); 8182 if (slow_path != nullptr && slow_path->IsCoreRegisterSaved(location.AsRegister<Register>())) { 8183 int stack_offset = slow_path->GetStackOffsetOfCoreRegister(location.AsRegister<Register>()); 8184 __ LoadFromOffset(kLoadWord, temp, SP, stack_offset); 8185 return temp; 8186 } 8187 return location.AsRegister<Register>(); 8188} 8189 8190Location CodeGeneratorARM::GenerateCalleeMethodStaticOrDirectCall(HInvokeStaticOrDirect* invoke, 8191 Location temp) { 8192 Location callee_method = temp; // For all kinds except kRecursive, callee will be in temp. 8193 switch (invoke->GetMethodLoadKind()) { 8194 case HInvokeStaticOrDirect::MethodLoadKind::kStringInit: { 8195 uint32_t offset = 8196 GetThreadOffset<kArmPointerSize>(invoke->GetStringInitEntryPoint()).Int32Value(); 8197 // temp = thread->string_init_entrypoint 8198 __ LoadFromOffset(kLoadWord, temp.AsRegister<Register>(), TR, offset); 8199 break; 8200 } 8201 case HInvokeStaticOrDirect::MethodLoadKind::kRecursive: 8202 callee_method = invoke->GetLocations()->InAt(invoke->GetSpecialInputIndex()); 8203 break; 8204 case HInvokeStaticOrDirect::MethodLoadKind::kDirectAddress: 8205 __ LoadImmediate(temp.AsRegister<Register>(), invoke->GetMethodAddress()); 8206 break; 8207 case HInvokeStaticOrDirect::MethodLoadKind::kDexCachePcRelative: { 8208 HArmDexCacheArraysBase* base = 8209 invoke->InputAt(invoke->GetSpecialInputIndex())->AsArmDexCacheArraysBase(); 8210 Register base_reg = GetInvokeStaticOrDirectExtraParameter(invoke, 8211 temp.AsRegister<Register>()); 8212 int32_t offset = invoke->GetDexCacheArrayOffset() - base->GetElementOffset(); 8213 __ LoadFromOffset(kLoadWord, temp.AsRegister<Register>(), base_reg, offset); 8214 break; 8215 } 8216 case HInvokeStaticOrDirect::MethodLoadKind::kDexCacheViaMethod: { 8217 Location current_method = invoke->GetLocations()->InAt(invoke->GetSpecialInputIndex()); 8218 Register method_reg; 8219 Register reg = temp.AsRegister<Register>(); 8220 if (current_method.IsRegister()) { 8221 method_reg = current_method.AsRegister<Register>(); 8222 } else { 8223 DCHECK(invoke->GetLocations()->Intrinsified()); 8224 DCHECK(!current_method.IsValid()); 8225 method_reg = reg; 8226 __ LoadFromOffset(kLoadWord, reg, SP, kCurrentMethodStackOffset); 8227 } 8228 // /* ArtMethod*[] */ temp = temp.ptr_sized_fields_->dex_cache_resolved_methods_; 8229 __ LoadFromOffset(kLoadWord, 8230 reg, 8231 method_reg, 8232 ArtMethod::DexCacheResolvedMethodsOffset(kArmPointerSize).Int32Value()); 8233 // temp = temp[index_in_cache]; 8234 // Note: Don't use invoke->GetTargetMethod() as it may point to a different dex file. 8235 uint32_t index_in_cache = invoke->GetDexMethodIndex(); 8236 __ LoadFromOffset(kLoadWord, reg, reg, CodeGenerator::GetCachePointerOffset(index_in_cache)); 8237 break; 8238 } 8239 } 8240 return callee_method; 8241} 8242 8243void CodeGeneratorARM::GenerateStaticOrDirectCall(HInvokeStaticOrDirect* invoke, Location temp) { 8244 Location callee_method = GenerateCalleeMethodStaticOrDirectCall(invoke, temp); 8245 8246 switch (invoke->GetCodePtrLocation()) { 8247 case HInvokeStaticOrDirect::CodePtrLocation::kCallSelf: 8248 __ bl(GetFrameEntryLabel()); 8249 break; 8250 case HInvokeStaticOrDirect::CodePtrLocation::kCallArtMethod: 8251 // LR = callee_method->entry_point_from_quick_compiled_code_ 8252 __ LoadFromOffset( 8253 kLoadWord, LR, callee_method.AsRegister<Register>(), 8254 ArtMethod::EntryPointFromQuickCompiledCodeOffset(kArmPointerSize).Int32Value()); 8255 // LR() 8256 __ blx(LR); 8257 break; 8258 } 8259 8260 DCHECK(!IsLeafMethod()); 8261} 8262 8263void CodeGeneratorARM::GenerateVirtualCall(HInvokeVirtual* invoke, Location temp_location) { 8264 Register temp = temp_location.AsRegister<Register>(); 8265 uint32_t method_offset = mirror::Class::EmbeddedVTableEntryOffset( 8266 invoke->GetVTableIndex(), kArmPointerSize).Uint32Value(); 8267 8268 // Use the calling convention instead of the location of the receiver, as 8269 // intrinsics may have put the receiver in a different register. In the intrinsics 8270 // slow path, the arguments have been moved to the right place, so here we are 8271 // guaranteed that the receiver is the first register of the calling convention. 8272 InvokeDexCallingConvention calling_convention; 8273 Register receiver = calling_convention.GetRegisterAt(0); 8274 uint32_t class_offset = mirror::Object::ClassOffset().Int32Value(); 8275 // /* HeapReference<Class> */ temp = receiver->klass_ 8276 __ LoadFromOffset(kLoadWord, temp, receiver, class_offset); 8277 MaybeRecordImplicitNullCheck(invoke); 8278 // Instead of simply (possibly) unpoisoning `temp` here, we should 8279 // emit a read barrier for the previous class reference load. 8280 // However this is not required in practice, as this is an 8281 // intermediate/temporary reference and because the current 8282 // concurrent copying collector keeps the from-space memory 8283 // intact/accessible until the end of the marking phase (the 8284 // concurrent copying collector may not in the future). 8285 __ MaybeUnpoisonHeapReference(temp); 8286 // temp = temp->GetMethodAt(method_offset); 8287 uint32_t entry_point = ArtMethod::EntryPointFromQuickCompiledCodeOffset( 8288 kArmPointerSize).Int32Value(); 8289 __ LoadFromOffset(kLoadWord, temp, temp, method_offset); 8290 // LR = temp->GetEntryPoint(); 8291 __ LoadFromOffset(kLoadWord, LR, temp, entry_point); 8292 // LR(); 8293 __ blx(LR); 8294} 8295 8296CodeGeneratorARM::PcRelativePatchInfo* CodeGeneratorARM::NewPcRelativeStringPatch( 8297 const DexFile& dex_file, dex::StringIndex string_index) { 8298 return NewPcRelativePatch(dex_file, string_index.index_, &pc_relative_string_patches_); 8299} 8300 8301CodeGeneratorARM::PcRelativePatchInfo* CodeGeneratorARM::NewPcRelativeTypePatch( 8302 const DexFile& dex_file, dex::TypeIndex type_index) { 8303 return NewPcRelativePatch(dex_file, type_index.index_, &pc_relative_type_patches_); 8304} 8305 8306CodeGeneratorARM::PcRelativePatchInfo* CodeGeneratorARM::NewTypeBssEntryPatch( 8307 const DexFile& dex_file, dex::TypeIndex type_index) { 8308 return NewPcRelativePatch(dex_file, type_index.index_, &type_bss_entry_patches_); 8309} 8310 8311CodeGeneratorARM::PcRelativePatchInfo* CodeGeneratorARM::NewPcRelativeDexCacheArrayPatch( 8312 const DexFile& dex_file, uint32_t element_offset) { 8313 return NewPcRelativePatch(dex_file, element_offset, &pc_relative_dex_cache_patches_); 8314} 8315 8316CodeGeneratorARM::PcRelativePatchInfo* CodeGeneratorARM::NewPcRelativePatch( 8317 const DexFile& dex_file, uint32_t offset_or_index, ArenaDeque<PcRelativePatchInfo>* patches) { 8318 patches->emplace_back(dex_file, offset_or_index); 8319 return &patches->back(); 8320} 8321 8322Literal* CodeGeneratorARM::DeduplicateBootImageStringLiteral(const DexFile& dex_file, 8323 dex::StringIndex string_index) { 8324 return boot_image_string_patches_.GetOrCreate( 8325 StringReference(&dex_file, string_index), 8326 [this]() { return __ NewLiteral<uint32_t>(/* placeholder */ 0u); }); 8327} 8328 8329Literal* CodeGeneratorARM::DeduplicateBootImageTypeLiteral(const DexFile& dex_file, 8330 dex::TypeIndex type_index) { 8331 return boot_image_type_patches_.GetOrCreate( 8332 TypeReference(&dex_file, type_index), 8333 [this]() { return __ NewLiteral<uint32_t>(/* placeholder */ 0u); }); 8334} 8335 8336Literal* CodeGeneratorARM::DeduplicateBootImageAddressLiteral(uint32_t address) { 8337 return DeduplicateUint32Literal(dchecked_integral_cast<uint32_t>(address), &uint32_literals_); 8338} 8339 8340Literal* CodeGeneratorARM::DeduplicateJitStringLiteral(const DexFile& dex_file, 8341 dex::StringIndex string_index, 8342 Handle<mirror::String> handle) { 8343 jit_string_roots_.Overwrite(StringReference(&dex_file, string_index), 8344 reinterpret_cast64<uint64_t>(handle.GetReference())); 8345 return jit_string_patches_.GetOrCreate( 8346 StringReference(&dex_file, string_index), 8347 [this]() { return __ NewLiteral<uint32_t>(/* placeholder */ 0u); }); 8348} 8349 8350Literal* CodeGeneratorARM::DeduplicateJitClassLiteral(const DexFile& dex_file, 8351 dex::TypeIndex type_index, 8352 Handle<mirror::Class> handle) { 8353 jit_class_roots_.Overwrite(TypeReference(&dex_file, type_index), 8354 reinterpret_cast64<uint64_t>(handle.GetReference())); 8355 return jit_class_patches_.GetOrCreate( 8356 TypeReference(&dex_file, type_index), 8357 [this]() { return __ NewLiteral<uint32_t>(/* placeholder */ 0u); }); 8358} 8359 8360template <LinkerPatch (*Factory)(size_t, const DexFile*, uint32_t, uint32_t)> 8361inline void CodeGeneratorARM::EmitPcRelativeLinkerPatches( 8362 const ArenaDeque<PcRelativePatchInfo>& infos, 8363 ArenaVector<LinkerPatch>* linker_patches) { 8364 for (const PcRelativePatchInfo& info : infos) { 8365 const DexFile& dex_file = info.target_dex_file; 8366 size_t offset_or_index = info.offset_or_index; 8367 DCHECK(info.add_pc_label.IsBound()); 8368 uint32_t add_pc_offset = dchecked_integral_cast<uint32_t>(info.add_pc_label.Position()); 8369 // Add MOVW patch. 8370 DCHECK(info.movw_label.IsBound()); 8371 uint32_t movw_offset = dchecked_integral_cast<uint32_t>(info.movw_label.Position()); 8372 linker_patches->push_back(Factory(movw_offset, &dex_file, add_pc_offset, offset_or_index)); 8373 // Add MOVT patch. 8374 DCHECK(info.movt_label.IsBound()); 8375 uint32_t movt_offset = dchecked_integral_cast<uint32_t>(info.movt_label.Position()); 8376 linker_patches->push_back(Factory(movt_offset, &dex_file, add_pc_offset, offset_or_index)); 8377 } 8378} 8379 8380void CodeGeneratorARM::EmitLinkerPatches(ArenaVector<LinkerPatch>* linker_patches) { 8381 DCHECK(linker_patches->empty()); 8382 size_t size = 8383 /* MOVW+MOVT for each entry */ 2u * pc_relative_dex_cache_patches_.size() + 8384 boot_image_string_patches_.size() + 8385 /* MOVW+MOVT for each entry */ 2u * pc_relative_string_patches_.size() + 8386 boot_image_type_patches_.size() + 8387 /* MOVW+MOVT for each entry */ 2u * pc_relative_type_patches_.size() + 8388 /* MOVW+MOVT for each entry */ 2u * type_bss_entry_patches_.size(); 8389 linker_patches->reserve(size); 8390 EmitPcRelativeLinkerPatches<LinkerPatch::DexCacheArrayPatch>(pc_relative_dex_cache_patches_, 8391 linker_patches); 8392 for (const auto& entry : boot_image_string_patches_) { 8393 const StringReference& target_string = entry.first; 8394 Literal* literal = entry.second; 8395 DCHECK(literal->GetLabel()->IsBound()); 8396 uint32_t literal_offset = literal->GetLabel()->Position(); 8397 linker_patches->push_back(LinkerPatch::StringPatch(literal_offset, 8398 target_string.dex_file, 8399 target_string.string_index.index_)); 8400 } 8401 if (!GetCompilerOptions().IsBootImage()) { 8402 DCHECK(pc_relative_type_patches_.empty()); 8403 EmitPcRelativeLinkerPatches<LinkerPatch::StringBssEntryPatch>(pc_relative_string_patches_, 8404 linker_patches); 8405 } else { 8406 EmitPcRelativeLinkerPatches<LinkerPatch::RelativeTypePatch>(pc_relative_type_patches_, 8407 linker_patches); 8408 EmitPcRelativeLinkerPatches<LinkerPatch::RelativeStringPatch>(pc_relative_string_patches_, 8409 linker_patches); 8410 } 8411 EmitPcRelativeLinkerPatches<LinkerPatch::TypeBssEntryPatch>(type_bss_entry_patches_, 8412 linker_patches); 8413 for (const auto& entry : boot_image_type_patches_) { 8414 const TypeReference& target_type = entry.first; 8415 Literal* literal = entry.second; 8416 DCHECK(literal->GetLabel()->IsBound()); 8417 uint32_t literal_offset = literal->GetLabel()->Position(); 8418 linker_patches->push_back(LinkerPatch::TypePatch(literal_offset, 8419 target_type.dex_file, 8420 target_type.type_index.index_)); 8421 } 8422 DCHECK_EQ(size, linker_patches->size()); 8423} 8424 8425Literal* CodeGeneratorARM::DeduplicateUint32Literal(uint32_t value, Uint32ToLiteralMap* map) { 8426 return map->GetOrCreate( 8427 value, 8428 [this, value]() { return __ NewLiteral<uint32_t>(value); }); 8429} 8430 8431Literal* CodeGeneratorARM::DeduplicateMethodLiteral(MethodReference target_method, 8432 MethodToLiteralMap* map) { 8433 return map->GetOrCreate( 8434 target_method, 8435 [this]() { return __ NewLiteral<uint32_t>(/* placeholder */ 0u); }); 8436} 8437 8438void LocationsBuilderARM::VisitMultiplyAccumulate(HMultiplyAccumulate* instr) { 8439 LocationSummary* locations = 8440 new (GetGraph()->GetArena()) LocationSummary(instr, LocationSummary::kNoCall); 8441 locations->SetInAt(HMultiplyAccumulate::kInputAccumulatorIndex, 8442 Location::RequiresRegister()); 8443 locations->SetInAt(HMultiplyAccumulate::kInputMulLeftIndex, Location::RequiresRegister()); 8444 locations->SetInAt(HMultiplyAccumulate::kInputMulRightIndex, Location::RequiresRegister()); 8445 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 8446} 8447 8448void InstructionCodeGeneratorARM::VisitMultiplyAccumulate(HMultiplyAccumulate* instr) { 8449 LocationSummary* locations = instr->GetLocations(); 8450 Register res = locations->Out().AsRegister<Register>(); 8451 Register accumulator = 8452 locations->InAt(HMultiplyAccumulate::kInputAccumulatorIndex).AsRegister<Register>(); 8453 Register mul_left = 8454 locations->InAt(HMultiplyAccumulate::kInputMulLeftIndex).AsRegister<Register>(); 8455 Register mul_right = 8456 locations->InAt(HMultiplyAccumulate::kInputMulRightIndex).AsRegister<Register>(); 8457 8458 if (instr->GetOpKind() == HInstruction::kAdd) { 8459 __ mla(res, mul_left, mul_right, accumulator); 8460 } else { 8461 __ mls(res, mul_left, mul_right, accumulator); 8462 } 8463} 8464 8465void LocationsBuilderARM::VisitBoundType(HBoundType* instruction ATTRIBUTE_UNUSED) { 8466 // Nothing to do, this should be removed during prepare for register allocator. 8467 LOG(FATAL) << "Unreachable"; 8468} 8469 8470void InstructionCodeGeneratorARM::VisitBoundType(HBoundType* instruction ATTRIBUTE_UNUSED) { 8471 // Nothing to do, this should be removed during prepare for register allocator. 8472 LOG(FATAL) << "Unreachable"; 8473} 8474 8475// Simple implementation of packed switch - generate cascaded compare/jumps. 8476void LocationsBuilderARM::VisitPackedSwitch(HPackedSwitch* switch_instr) { 8477 LocationSummary* locations = 8478 new (GetGraph()->GetArena()) LocationSummary(switch_instr, LocationSummary::kNoCall); 8479 locations->SetInAt(0, Location::RequiresRegister()); 8480 if (switch_instr->GetNumEntries() > kPackedSwitchCompareJumpThreshold && 8481 codegen_->GetAssembler()->IsThumb()) { 8482 locations->AddTemp(Location::RequiresRegister()); // We need a temp for the table base. 8483 if (switch_instr->GetStartValue() != 0) { 8484 locations->AddTemp(Location::RequiresRegister()); // We need a temp for the bias. 8485 } 8486 } 8487} 8488 8489void InstructionCodeGeneratorARM::VisitPackedSwitch(HPackedSwitch* switch_instr) { 8490 int32_t lower_bound = switch_instr->GetStartValue(); 8491 uint32_t num_entries = switch_instr->GetNumEntries(); 8492 LocationSummary* locations = switch_instr->GetLocations(); 8493 Register value_reg = locations->InAt(0).AsRegister<Register>(); 8494 HBasicBlock* default_block = switch_instr->GetDefaultBlock(); 8495 8496 if (num_entries <= kPackedSwitchCompareJumpThreshold || !codegen_->GetAssembler()->IsThumb()) { 8497 // Create a series of compare/jumps. 8498 Register temp_reg = IP; 8499 // Note: It is fine for the below AddConstantSetFlags() using IP register to temporarily store 8500 // the immediate, because IP is used as the destination register. For the other 8501 // AddConstantSetFlags() and GenerateCompareWithImmediate(), the immediate values are constant, 8502 // and they can be encoded in the instruction without making use of IP register. 8503 __ AddConstantSetFlags(temp_reg, value_reg, -lower_bound); 8504 8505 const ArenaVector<HBasicBlock*>& successors = switch_instr->GetBlock()->GetSuccessors(); 8506 // Jump to successors[0] if value == lower_bound. 8507 __ b(codegen_->GetLabelOf(successors[0]), EQ); 8508 int32_t last_index = 0; 8509 for (; num_entries - last_index > 2; last_index += 2) { 8510 __ AddConstantSetFlags(temp_reg, temp_reg, -2); 8511 // Jump to successors[last_index + 1] if value < case_value[last_index + 2]. 8512 __ b(codegen_->GetLabelOf(successors[last_index + 1]), LO); 8513 // Jump to successors[last_index + 2] if value == case_value[last_index + 2]. 8514 __ b(codegen_->GetLabelOf(successors[last_index + 2]), EQ); 8515 } 8516 if (num_entries - last_index == 2) { 8517 // The last missing case_value. 8518 __ CmpConstant(temp_reg, 1); 8519 __ b(codegen_->GetLabelOf(successors[last_index + 1]), EQ); 8520 } 8521 8522 // And the default for any other value. 8523 if (!codegen_->GoesToNextBlock(switch_instr->GetBlock(), default_block)) { 8524 __ b(codegen_->GetLabelOf(default_block)); 8525 } 8526 } else { 8527 // Create a table lookup. 8528 Register temp_reg = locations->GetTemp(0).AsRegister<Register>(); 8529 8530 // Materialize a pointer to the switch table 8531 std::vector<Label*> labels(num_entries); 8532 const ArenaVector<HBasicBlock*>& successors = switch_instr->GetBlock()->GetSuccessors(); 8533 for (uint32_t i = 0; i < num_entries; i++) { 8534 labels[i] = codegen_->GetLabelOf(successors[i]); 8535 } 8536 JumpTable* table = __ CreateJumpTable(std::move(labels), temp_reg); 8537 8538 // Remove the bias. 8539 Register key_reg; 8540 if (lower_bound != 0) { 8541 key_reg = locations->GetTemp(1).AsRegister<Register>(); 8542 __ AddConstant(key_reg, value_reg, -lower_bound); 8543 } else { 8544 key_reg = value_reg; 8545 } 8546 8547 // Check whether the value is in the table, jump to default block if not. 8548 __ CmpConstant(key_reg, num_entries - 1); 8549 __ b(codegen_->GetLabelOf(default_block), Condition::HI); 8550 8551 // Load the displacement from the table. 8552 __ ldr(temp_reg, Address(temp_reg, key_reg, Shift::LSL, 2)); 8553 8554 // Dispatch is a direct add to the PC (for Thumb2). 8555 __ EmitJumpTableDispatch(table, temp_reg); 8556 } 8557} 8558 8559void LocationsBuilderARM::VisitArmDexCacheArraysBase(HArmDexCacheArraysBase* base) { 8560 LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(base); 8561 locations->SetOut(Location::RequiresRegister()); 8562} 8563 8564void InstructionCodeGeneratorARM::VisitArmDexCacheArraysBase(HArmDexCacheArraysBase* base) { 8565 Register base_reg = base->GetLocations()->Out().AsRegister<Register>(); 8566 CodeGeneratorARM::PcRelativePatchInfo* labels = 8567 codegen_->NewPcRelativeDexCacheArrayPatch(base->GetDexFile(), base->GetElementOffset()); 8568 __ BindTrackedLabel(&labels->movw_label); 8569 __ movw(base_reg, /* placeholder */ 0u); 8570 __ BindTrackedLabel(&labels->movt_label); 8571 __ movt(base_reg, /* placeholder */ 0u); 8572 __ BindTrackedLabel(&labels->add_pc_label); 8573 __ add(base_reg, base_reg, ShifterOperand(PC)); 8574} 8575 8576void CodeGeneratorARM::MoveFromReturnRegister(Location trg, Primitive::Type type) { 8577 if (!trg.IsValid()) { 8578 DCHECK_EQ(type, Primitive::kPrimVoid); 8579 return; 8580 } 8581 8582 DCHECK_NE(type, Primitive::kPrimVoid); 8583 8584 Location return_loc = InvokeDexCallingConventionVisitorARM().GetReturnLocation(type); 8585 if (return_loc.Equals(trg)) { 8586 return; 8587 } 8588 8589 // TODO: Consider pairs in the parallel move resolver, then this could be nicely merged 8590 // with the last branch. 8591 if (type == Primitive::kPrimLong) { 8592 HParallelMove parallel_move(GetGraph()->GetArena()); 8593 parallel_move.AddMove(return_loc.ToLow(), trg.ToLow(), Primitive::kPrimInt, nullptr); 8594 parallel_move.AddMove(return_loc.ToHigh(), trg.ToHigh(), Primitive::kPrimInt, nullptr); 8595 GetMoveResolver()->EmitNativeCode(¶llel_move); 8596 } else if (type == Primitive::kPrimDouble) { 8597 HParallelMove parallel_move(GetGraph()->GetArena()); 8598 parallel_move.AddMove(return_loc.ToLow(), trg.ToLow(), Primitive::kPrimFloat, nullptr); 8599 parallel_move.AddMove(return_loc.ToHigh(), trg.ToHigh(), Primitive::kPrimFloat, nullptr); 8600 GetMoveResolver()->EmitNativeCode(¶llel_move); 8601 } else { 8602 // Let the parallel move resolver take care of all of this. 8603 HParallelMove parallel_move(GetGraph()->GetArena()); 8604 parallel_move.AddMove(return_loc, trg, type, nullptr); 8605 GetMoveResolver()->EmitNativeCode(¶llel_move); 8606 } 8607} 8608 8609void LocationsBuilderARM::VisitClassTableGet(HClassTableGet* instruction) { 8610 LocationSummary* locations = 8611 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall); 8612 locations->SetInAt(0, Location::RequiresRegister()); 8613 locations->SetOut(Location::RequiresRegister()); 8614} 8615 8616void InstructionCodeGeneratorARM::VisitClassTableGet(HClassTableGet* instruction) { 8617 LocationSummary* locations = instruction->GetLocations(); 8618 if (instruction->GetTableKind() == HClassTableGet::TableKind::kVTable) { 8619 uint32_t method_offset = mirror::Class::EmbeddedVTableEntryOffset( 8620 instruction->GetIndex(), kArmPointerSize).SizeValue(); 8621 __ LoadFromOffset(kLoadWord, 8622 locations->Out().AsRegister<Register>(), 8623 locations->InAt(0).AsRegister<Register>(), 8624 method_offset); 8625 } else { 8626 uint32_t method_offset = static_cast<uint32_t>(ImTable::OffsetOfElement( 8627 instruction->GetIndex(), kArmPointerSize)); 8628 __ LoadFromOffset(kLoadWord, 8629 locations->Out().AsRegister<Register>(), 8630 locations->InAt(0).AsRegister<Register>(), 8631 mirror::Class::ImtPtrOffset(kArmPointerSize).Uint32Value()); 8632 __ LoadFromOffset(kLoadWord, 8633 locations->Out().AsRegister<Register>(), 8634 locations->Out().AsRegister<Register>(), 8635 method_offset); 8636 } 8637} 8638 8639static void PatchJitRootUse(uint8_t* code, 8640 const uint8_t* roots_data, 8641 Literal* literal, 8642 uint64_t index_in_table) { 8643 DCHECK(literal->GetLabel()->IsBound()); 8644 uint32_t literal_offset = literal->GetLabel()->Position(); 8645 uintptr_t address = 8646 reinterpret_cast<uintptr_t>(roots_data) + index_in_table * sizeof(GcRoot<mirror::Object>); 8647 uint8_t* data = code + literal_offset; 8648 reinterpret_cast<uint32_t*>(data)[0] = dchecked_integral_cast<uint32_t>(address); 8649} 8650 8651void CodeGeneratorARM::EmitJitRootPatches(uint8_t* code, const uint8_t* roots_data) { 8652 for (const auto& entry : jit_string_patches_) { 8653 const auto& it = jit_string_roots_.find(entry.first); 8654 DCHECK(it != jit_string_roots_.end()); 8655 PatchJitRootUse(code, roots_data, entry.second, it->second); 8656 } 8657 for (const auto& entry : jit_class_patches_) { 8658 const auto& it = jit_class_roots_.find(entry.first); 8659 DCHECK(it != jit_class_roots_.end()); 8660 PatchJitRootUse(code, roots_data, entry.second, it->second); 8661 } 8662} 8663 8664#undef __ 8665#undef QUICK_ENTRY_POINT 8666 8667} // namespace arm 8668} // namespace art 8669