code_generator_arm.cc revision dbdba3776f2a5ba6b0400883d0c6fac474ffe2a3
1/* 2 * Copyright (C) 2014 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17#include "code_generator_arm.h" 18 19#include "arch/arm/instruction_set_features_arm.h" 20#include "art_method.h" 21#include "code_generator_utils.h" 22#include "common_arm.h" 23#include "compiled_method.h" 24#include "entrypoints/quick/quick_entrypoints.h" 25#include "gc/accounting/card_table.h" 26#include "intrinsics.h" 27#include "intrinsics_arm.h" 28#include "mirror/array-inl.h" 29#include "mirror/class-inl.h" 30#include "thread.h" 31#include "utils/arm/assembler_arm.h" 32#include "utils/arm/managed_register_arm.h" 33#include "utils/assembler.h" 34#include "utils/stack_checks.h" 35 36namespace art { 37 38template<class MirrorType> 39class GcRoot; 40 41namespace arm { 42 43static bool ExpectedPairLayout(Location location) { 44 // We expected this for both core and fpu register pairs. 45 return ((location.low() & 1) == 0) && (location.low() + 1 == location.high()); 46} 47 48static constexpr int kCurrentMethodStackOffset = 0; 49static constexpr Register kMethodRegisterArgument = R0; 50 51static constexpr Register kCoreAlwaysSpillRegister = R5; 52static constexpr Register kCoreCalleeSaves[] = 53 { R5, R6, R7, R8, R10, R11, LR }; 54static constexpr SRegister kFpuCalleeSaves[] = 55 { S16, S17, S18, S19, S20, S21, S22, S23, S24, S25, S26, S27, S28, S29, S30, S31 }; 56 57// D31 cannot be split into two S registers, and the register allocator only works on 58// S registers. Therefore there is no need to block it. 59static constexpr DRegister DTMP = D31; 60 61static constexpr uint32_t kPackedSwitchCompareJumpThreshold = 7; 62 63// NOLINT on __ macro to suppress wrong warning/fix (misc-macro-parentheses) from clang-tidy. 64#define __ down_cast<ArmAssembler*>(codegen->GetAssembler())-> // NOLINT 65#define QUICK_ENTRY_POINT(x) QUICK_ENTRYPOINT_OFFSET(kArmPointerSize, x).Int32Value() 66 67static constexpr int kRegListThreshold = 4; 68 69// SaveLiveRegisters and RestoreLiveRegisters from SlowPathCodeARM operate on sets of S registers, 70// for each live D registers they treat two corresponding S registers as live ones. 71// 72// Two following functions (SaveContiguousSRegisterList, RestoreContiguousSRegisterList) build 73// from a list of contiguous S registers a list of contiguous D registers (processing first/last 74// S registers corner cases) and save/restore this new list treating them as D registers. 75// - decreasing code size 76// - avoiding hazards on Cortex-A57, when a pair of S registers for an actual live D register is 77// restored and then used in regular non SlowPath code as D register. 78// 79// For the following example (v means the S register is live): 80// D names: | D0 | D1 | D2 | D4 | ... 81// S names: | S0 | S1 | S2 | S3 | S4 | S5 | S6 | S7 | ... 82// Live? | | v | v | v | v | v | v | | ... 83// 84// S1 and S6 will be saved/restored independently; D registers list (D1, D2) will be processed 85// as D registers. 86static size_t SaveContiguousSRegisterList(size_t first, 87 size_t last, 88 CodeGenerator* codegen, 89 size_t stack_offset) { 90 DCHECK_LE(first, last); 91 if ((first == last) && (first == 0)) { 92 stack_offset += codegen->SaveFloatingPointRegister(stack_offset, first); 93 return stack_offset; 94 } 95 if (first % 2 == 1) { 96 stack_offset += codegen->SaveFloatingPointRegister(stack_offset, first++); 97 } 98 99 bool save_last = false; 100 if (last % 2 == 0) { 101 save_last = true; 102 --last; 103 } 104 105 if (first < last) { 106 DRegister d_reg = static_cast<DRegister>(first / 2); 107 DCHECK_EQ((last - first + 1) % 2, 0u); 108 size_t number_of_d_regs = (last - first + 1) / 2; 109 110 if (number_of_d_regs == 1) { 111 __ StoreDToOffset(d_reg, SP, stack_offset); 112 } else if (number_of_d_regs > 1) { 113 __ add(IP, SP, ShifterOperand(stack_offset)); 114 __ vstmiad(IP, d_reg, number_of_d_regs); 115 } 116 stack_offset += number_of_d_regs * kArmWordSize * 2; 117 } 118 119 if (save_last) { 120 stack_offset += codegen->SaveFloatingPointRegister(stack_offset, last + 1); 121 } 122 123 return stack_offset; 124} 125 126static size_t RestoreContiguousSRegisterList(size_t first, 127 size_t last, 128 CodeGenerator* codegen, 129 size_t stack_offset) { 130 DCHECK_LE(first, last); 131 if ((first == last) && (first == 0)) { 132 stack_offset += codegen->RestoreFloatingPointRegister(stack_offset, first); 133 return stack_offset; 134 } 135 if (first % 2 == 1) { 136 stack_offset += codegen->RestoreFloatingPointRegister(stack_offset, first++); 137 } 138 139 bool restore_last = false; 140 if (last % 2 == 0) { 141 restore_last = true; 142 --last; 143 } 144 145 if (first < last) { 146 DRegister d_reg = static_cast<DRegister>(first / 2); 147 DCHECK_EQ((last - first + 1) % 2, 0u); 148 size_t number_of_d_regs = (last - first + 1) / 2; 149 if (number_of_d_regs == 1) { 150 __ LoadDFromOffset(d_reg, SP, stack_offset); 151 } else if (number_of_d_regs > 1) { 152 __ add(IP, SP, ShifterOperand(stack_offset)); 153 __ vldmiad(IP, d_reg, number_of_d_regs); 154 } 155 stack_offset += number_of_d_regs * kArmWordSize * 2; 156 } 157 158 if (restore_last) { 159 stack_offset += codegen->RestoreFloatingPointRegister(stack_offset, last + 1); 160 } 161 162 return stack_offset; 163} 164 165void SlowPathCodeARM::SaveLiveRegisters(CodeGenerator* codegen, LocationSummary* locations) { 166 size_t stack_offset = codegen->GetFirstRegisterSlotInSlowPath(); 167 size_t orig_offset = stack_offset; 168 169 const uint32_t core_spills = codegen->GetSlowPathSpills(locations, /* core_registers */ true); 170 for (uint32_t i : LowToHighBits(core_spills)) { 171 // If the register holds an object, update the stack mask. 172 if (locations->RegisterContainsObject(i)) { 173 locations->SetStackBit(stack_offset / kVRegSize); 174 } 175 DCHECK_LT(stack_offset, codegen->GetFrameSize() - codegen->FrameEntrySpillSize()); 176 DCHECK_LT(i, kMaximumNumberOfExpectedRegisters); 177 saved_core_stack_offsets_[i] = stack_offset; 178 stack_offset += kArmWordSize; 179 } 180 181 int reg_num = POPCOUNT(core_spills); 182 if (reg_num != 0) { 183 if (reg_num > kRegListThreshold) { 184 __ StoreList(RegList(core_spills), orig_offset); 185 } else { 186 stack_offset = orig_offset; 187 for (uint32_t i : LowToHighBits(core_spills)) { 188 stack_offset += codegen->SaveCoreRegister(stack_offset, i); 189 } 190 } 191 } 192 193 uint32_t fp_spills = codegen->GetSlowPathSpills(locations, /* core_registers */ false); 194 orig_offset = stack_offset; 195 for (uint32_t i : LowToHighBits(fp_spills)) { 196 DCHECK_LT(i, kMaximumNumberOfExpectedRegisters); 197 saved_fpu_stack_offsets_[i] = stack_offset; 198 stack_offset += kArmWordSize; 199 } 200 201 stack_offset = orig_offset; 202 while (fp_spills != 0u) { 203 uint32_t begin = CTZ(fp_spills); 204 uint32_t tmp = fp_spills + (1u << begin); 205 fp_spills &= tmp; // Clear the contiguous range of 1s. 206 uint32_t end = (tmp == 0u) ? 32u : CTZ(tmp); // CTZ(0) is undefined. 207 stack_offset = SaveContiguousSRegisterList(begin, end - 1, codegen, stack_offset); 208 } 209 DCHECK_LE(stack_offset, codegen->GetFrameSize() - codegen->FrameEntrySpillSize()); 210} 211 212void SlowPathCodeARM::RestoreLiveRegisters(CodeGenerator* codegen, LocationSummary* locations) { 213 size_t stack_offset = codegen->GetFirstRegisterSlotInSlowPath(); 214 size_t orig_offset = stack_offset; 215 216 const uint32_t core_spills = codegen->GetSlowPathSpills(locations, /* core_registers */ true); 217 for (uint32_t i : LowToHighBits(core_spills)) { 218 DCHECK_LT(stack_offset, codegen->GetFrameSize() - codegen->FrameEntrySpillSize()); 219 DCHECK_LT(i, kMaximumNumberOfExpectedRegisters); 220 stack_offset += kArmWordSize; 221 } 222 223 int reg_num = POPCOUNT(core_spills); 224 if (reg_num != 0) { 225 if (reg_num > kRegListThreshold) { 226 __ LoadList(RegList(core_spills), orig_offset); 227 } else { 228 stack_offset = orig_offset; 229 for (uint32_t i : LowToHighBits(core_spills)) { 230 stack_offset += codegen->RestoreCoreRegister(stack_offset, i); 231 } 232 } 233 } 234 235 uint32_t fp_spills = codegen->GetSlowPathSpills(locations, /* core_registers */ false); 236 while (fp_spills != 0u) { 237 uint32_t begin = CTZ(fp_spills); 238 uint32_t tmp = fp_spills + (1u << begin); 239 fp_spills &= tmp; // Clear the contiguous range of 1s. 240 uint32_t end = (tmp == 0u) ? 32u : CTZ(tmp); // CTZ(0) is undefined. 241 stack_offset = RestoreContiguousSRegisterList(begin, end - 1, codegen, stack_offset); 242 } 243 DCHECK_LE(stack_offset, codegen->GetFrameSize() - codegen->FrameEntrySpillSize()); 244} 245 246class NullCheckSlowPathARM : public SlowPathCodeARM { 247 public: 248 explicit NullCheckSlowPathARM(HNullCheck* instruction) : SlowPathCodeARM(instruction) {} 249 250 void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { 251 CodeGeneratorARM* arm_codegen = down_cast<CodeGeneratorARM*>(codegen); 252 __ Bind(GetEntryLabel()); 253 if (instruction_->CanThrowIntoCatchBlock()) { 254 // Live registers will be restored in the catch block if caught. 255 SaveLiveRegisters(codegen, instruction_->GetLocations()); 256 } 257 arm_codegen->InvokeRuntime(kQuickThrowNullPointer, 258 instruction_, 259 instruction_->GetDexPc(), 260 this); 261 CheckEntrypointTypes<kQuickThrowNullPointer, void, void>(); 262 } 263 264 bool IsFatal() const OVERRIDE { return true; } 265 266 const char* GetDescription() const OVERRIDE { return "NullCheckSlowPathARM"; } 267 268 private: 269 DISALLOW_COPY_AND_ASSIGN(NullCheckSlowPathARM); 270}; 271 272class DivZeroCheckSlowPathARM : public SlowPathCodeARM { 273 public: 274 explicit DivZeroCheckSlowPathARM(HDivZeroCheck* instruction) : SlowPathCodeARM(instruction) {} 275 276 void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { 277 CodeGeneratorARM* arm_codegen = down_cast<CodeGeneratorARM*>(codegen); 278 __ Bind(GetEntryLabel()); 279 arm_codegen->InvokeRuntime(kQuickThrowDivZero, instruction_, instruction_->GetDexPc(), this); 280 CheckEntrypointTypes<kQuickThrowDivZero, void, void>(); 281 } 282 283 bool IsFatal() const OVERRIDE { return true; } 284 285 const char* GetDescription() const OVERRIDE { return "DivZeroCheckSlowPathARM"; } 286 287 private: 288 DISALLOW_COPY_AND_ASSIGN(DivZeroCheckSlowPathARM); 289}; 290 291class SuspendCheckSlowPathARM : public SlowPathCodeARM { 292 public: 293 SuspendCheckSlowPathARM(HSuspendCheck* instruction, HBasicBlock* successor) 294 : SlowPathCodeARM(instruction), successor_(successor) {} 295 296 void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { 297 CodeGeneratorARM* arm_codegen = down_cast<CodeGeneratorARM*>(codegen); 298 __ Bind(GetEntryLabel()); 299 arm_codegen->InvokeRuntime(kQuickTestSuspend, instruction_, instruction_->GetDexPc(), this); 300 CheckEntrypointTypes<kQuickTestSuspend, void, void>(); 301 if (successor_ == nullptr) { 302 __ b(GetReturnLabel()); 303 } else { 304 __ b(arm_codegen->GetLabelOf(successor_)); 305 } 306 } 307 308 Label* GetReturnLabel() { 309 DCHECK(successor_ == nullptr); 310 return &return_label_; 311 } 312 313 HBasicBlock* GetSuccessor() const { 314 return successor_; 315 } 316 317 const char* GetDescription() const OVERRIDE { return "SuspendCheckSlowPathARM"; } 318 319 private: 320 // If not null, the block to branch to after the suspend check. 321 HBasicBlock* const successor_; 322 323 // If `successor_` is null, the label to branch to after the suspend check. 324 Label return_label_; 325 326 DISALLOW_COPY_AND_ASSIGN(SuspendCheckSlowPathARM); 327}; 328 329class BoundsCheckSlowPathARM : public SlowPathCodeARM { 330 public: 331 explicit BoundsCheckSlowPathARM(HBoundsCheck* instruction) 332 : SlowPathCodeARM(instruction) {} 333 334 void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { 335 CodeGeneratorARM* arm_codegen = down_cast<CodeGeneratorARM*>(codegen); 336 LocationSummary* locations = instruction_->GetLocations(); 337 338 __ Bind(GetEntryLabel()); 339 if (instruction_->CanThrowIntoCatchBlock()) { 340 // Live registers will be restored in the catch block if caught. 341 SaveLiveRegisters(codegen, instruction_->GetLocations()); 342 } 343 // We're moving two locations to locations that could overlap, so we need a parallel 344 // move resolver. 345 InvokeRuntimeCallingConvention calling_convention; 346 codegen->EmitParallelMoves( 347 locations->InAt(0), 348 Location::RegisterLocation(calling_convention.GetRegisterAt(0)), 349 Primitive::kPrimInt, 350 locations->InAt(1), 351 Location::RegisterLocation(calling_convention.GetRegisterAt(1)), 352 Primitive::kPrimInt); 353 QuickEntrypointEnum entrypoint = instruction_->AsBoundsCheck()->IsStringCharAt() 354 ? kQuickThrowStringBounds 355 : kQuickThrowArrayBounds; 356 arm_codegen->InvokeRuntime(entrypoint, instruction_, instruction_->GetDexPc(), this); 357 CheckEntrypointTypes<kQuickThrowStringBounds, void, int32_t, int32_t>(); 358 CheckEntrypointTypes<kQuickThrowArrayBounds, void, int32_t, int32_t>(); 359 } 360 361 bool IsFatal() const OVERRIDE { return true; } 362 363 const char* GetDescription() const OVERRIDE { return "BoundsCheckSlowPathARM"; } 364 365 private: 366 DISALLOW_COPY_AND_ASSIGN(BoundsCheckSlowPathARM); 367}; 368 369class LoadClassSlowPathARM : public SlowPathCodeARM { 370 public: 371 LoadClassSlowPathARM(HLoadClass* cls, HInstruction* at, uint32_t dex_pc, bool do_clinit) 372 : SlowPathCodeARM(at), cls_(cls), dex_pc_(dex_pc), do_clinit_(do_clinit) { 373 DCHECK(at->IsLoadClass() || at->IsClinitCheck()); 374 } 375 376 void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { 377 LocationSummary* locations = instruction_->GetLocations(); 378 Location out = locations->Out(); 379 constexpr bool call_saves_everything_except_r0 = (!kUseReadBarrier || kUseBakerReadBarrier); 380 381 CodeGeneratorARM* arm_codegen = down_cast<CodeGeneratorARM*>(codegen); 382 __ Bind(GetEntryLabel()); 383 SaveLiveRegisters(codegen, locations); 384 385 InvokeRuntimeCallingConvention calling_convention; 386 // For HLoadClass/kBssEntry/kSaveEverything, make sure we preserve the address of the entry. 387 DCHECK_EQ(instruction_->IsLoadClass(), cls_ == instruction_); 388 bool is_load_class_bss_entry = 389 (cls_ == instruction_) && (cls_->GetLoadKind() == HLoadClass::LoadKind::kBssEntry); 390 Register entry_address = kNoRegister; 391 if (is_load_class_bss_entry && call_saves_everything_except_r0) { 392 Register temp = locations->GetTemp(0).AsRegister<Register>(); 393 // In the unlucky case that the `temp` is R0, we preserve the address in `out` across 394 // the kSaveEverything call. 395 bool temp_is_r0 = (temp == calling_convention.GetRegisterAt(0)); 396 entry_address = temp_is_r0 ? out.AsRegister<Register>() : temp; 397 DCHECK_NE(entry_address, calling_convention.GetRegisterAt(0)); 398 if (temp_is_r0) { 399 __ mov(entry_address, ShifterOperand(temp)); 400 } 401 } 402 dex::TypeIndex type_index = cls_->GetTypeIndex(); 403 __ LoadImmediate(calling_convention.GetRegisterAt(0), type_index.index_); 404 QuickEntrypointEnum entrypoint = do_clinit_ ? kQuickInitializeStaticStorage 405 : kQuickInitializeType; 406 arm_codegen->InvokeRuntime(entrypoint, instruction_, dex_pc_, this); 407 if (do_clinit_) { 408 CheckEntrypointTypes<kQuickInitializeStaticStorage, void*, uint32_t>(); 409 } else { 410 CheckEntrypointTypes<kQuickInitializeType, void*, uint32_t>(); 411 } 412 413 // For HLoadClass/kBssEntry, store the resolved Class to the BSS entry. 414 if (is_load_class_bss_entry) { 415 if (call_saves_everything_except_r0) { 416 // The class entry address was preserved in `entry_address` thanks to kSaveEverything. 417 __ str(R0, Address(entry_address)); 418 } else { 419 // For non-Baker read barrier, we need to re-calculate the address of the string entry. 420 Register temp = IP; 421 CodeGeneratorARM::PcRelativePatchInfo* labels = 422 arm_codegen->NewTypeBssEntryPatch(cls_->GetDexFile(), type_index); 423 __ BindTrackedLabel(&labels->movw_label); 424 __ movw(temp, /* placeholder */ 0u); 425 __ BindTrackedLabel(&labels->movt_label); 426 __ movt(temp, /* placeholder */ 0u); 427 __ BindTrackedLabel(&labels->add_pc_label); 428 __ add(temp, temp, ShifterOperand(PC)); 429 __ str(R0, Address(temp)); 430 } 431 } 432 // Move the class to the desired location. 433 if (out.IsValid()) { 434 DCHECK(out.IsRegister() && !locations->GetLiveRegisters()->ContainsCoreRegister(out.reg())); 435 arm_codegen->Move32(locations->Out(), Location::RegisterLocation(R0)); 436 } 437 RestoreLiveRegisters(codegen, locations); 438 __ b(GetExitLabel()); 439 } 440 441 const char* GetDescription() const OVERRIDE { return "LoadClassSlowPathARM"; } 442 443 private: 444 // The class this slow path will load. 445 HLoadClass* const cls_; 446 447 // The dex PC of `at_`. 448 const uint32_t dex_pc_; 449 450 // Whether to initialize the class. 451 const bool do_clinit_; 452 453 DISALLOW_COPY_AND_ASSIGN(LoadClassSlowPathARM); 454}; 455 456class LoadStringSlowPathARM : public SlowPathCodeARM { 457 public: 458 explicit LoadStringSlowPathARM(HLoadString* instruction) : SlowPathCodeARM(instruction) {} 459 460 void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { 461 DCHECK(instruction_->IsLoadString()); 462 DCHECK_EQ(instruction_->AsLoadString()->GetLoadKind(), HLoadString::LoadKind::kBssEntry); 463 LocationSummary* locations = instruction_->GetLocations(); 464 DCHECK(!locations->GetLiveRegisters()->ContainsCoreRegister(locations->Out().reg())); 465 HLoadString* load = instruction_->AsLoadString(); 466 const dex::StringIndex string_index = load->GetStringIndex(); 467 Register out = locations->Out().AsRegister<Register>(); 468 constexpr bool call_saves_everything_except_r0 = (!kUseReadBarrier || kUseBakerReadBarrier); 469 470 CodeGeneratorARM* arm_codegen = down_cast<CodeGeneratorARM*>(codegen); 471 __ Bind(GetEntryLabel()); 472 SaveLiveRegisters(codegen, locations); 473 474 InvokeRuntimeCallingConvention calling_convention; 475 // In the unlucky case that the `temp` is R0, we preserve the address in `out` across 476 // the kSaveEverything call. 477 Register entry_address = kNoRegister; 478 if (call_saves_everything_except_r0) { 479 Register temp = locations->GetTemp(0).AsRegister<Register>(); 480 bool temp_is_r0 = (temp == calling_convention.GetRegisterAt(0)); 481 entry_address = temp_is_r0 ? out : temp; 482 DCHECK_NE(entry_address, calling_convention.GetRegisterAt(0)); 483 if (temp_is_r0) { 484 __ mov(entry_address, ShifterOperand(temp)); 485 } 486 } 487 488 __ LoadImmediate(calling_convention.GetRegisterAt(0), string_index.index_); 489 arm_codegen->InvokeRuntime(kQuickResolveString, instruction_, instruction_->GetDexPc(), this); 490 CheckEntrypointTypes<kQuickResolveString, void*, uint32_t>(); 491 492 // Store the resolved String to the .bss entry. 493 if (call_saves_everything_except_r0) { 494 // The string entry address was preserved in `entry_address` thanks to kSaveEverything. 495 __ str(R0, Address(entry_address)); 496 } else { 497 // For non-Baker read barrier, we need to re-calculate the address of the string entry. 498 Register temp = IP; 499 CodeGeneratorARM::PcRelativePatchInfo* labels = 500 arm_codegen->NewPcRelativeStringPatch(load->GetDexFile(), string_index); 501 __ BindTrackedLabel(&labels->movw_label); 502 __ movw(temp, /* placeholder */ 0u); 503 __ BindTrackedLabel(&labels->movt_label); 504 __ movt(temp, /* placeholder */ 0u); 505 __ BindTrackedLabel(&labels->add_pc_label); 506 __ add(temp, temp, ShifterOperand(PC)); 507 __ str(R0, Address(temp)); 508 } 509 510 arm_codegen->Move32(locations->Out(), Location::RegisterLocation(R0)); 511 RestoreLiveRegisters(codegen, locations); 512 513 __ b(GetExitLabel()); 514 } 515 516 const char* GetDescription() const OVERRIDE { return "LoadStringSlowPathARM"; } 517 518 private: 519 DISALLOW_COPY_AND_ASSIGN(LoadStringSlowPathARM); 520}; 521 522class TypeCheckSlowPathARM : public SlowPathCodeARM { 523 public: 524 TypeCheckSlowPathARM(HInstruction* instruction, bool is_fatal) 525 : SlowPathCodeARM(instruction), is_fatal_(is_fatal) {} 526 527 void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { 528 LocationSummary* locations = instruction_->GetLocations(); 529 DCHECK(instruction_->IsCheckCast() 530 || !locations->GetLiveRegisters()->ContainsCoreRegister(locations->Out().reg())); 531 532 CodeGeneratorARM* arm_codegen = down_cast<CodeGeneratorARM*>(codegen); 533 __ Bind(GetEntryLabel()); 534 535 if (!is_fatal_) { 536 SaveLiveRegisters(codegen, locations); 537 } 538 539 // We're moving two locations to locations that could overlap, so we need a parallel 540 // move resolver. 541 InvokeRuntimeCallingConvention calling_convention; 542 codegen->EmitParallelMoves(locations->InAt(0), 543 Location::RegisterLocation(calling_convention.GetRegisterAt(0)), 544 Primitive::kPrimNot, 545 locations->InAt(1), 546 Location::RegisterLocation(calling_convention.GetRegisterAt(1)), 547 Primitive::kPrimNot); 548 if (instruction_->IsInstanceOf()) { 549 arm_codegen->InvokeRuntime(kQuickInstanceofNonTrivial, 550 instruction_, 551 instruction_->GetDexPc(), 552 this); 553 CheckEntrypointTypes<kQuickInstanceofNonTrivial, size_t, mirror::Object*, mirror::Class*>(); 554 arm_codegen->Move32(locations->Out(), Location::RegisterLocation(R0)); 555 } else { 556 DCHECK(instruction_->IsCheckCast()); 557 arm_codegen->InvokeRuntime(kQuickCheckInstanceOf, 558 instruction_, 559 instruction_->GetDexPc(), 560 this); 561 CheckEntrypointTypes<kQuickCheckInstanceOf, void, mirror::Object*, mirror::Class*>(); 562 } 563 564 if (!is_fatal_) { 565 RestoreLiveRegisters(codegen, locations); 566 __ b(GetExitLabel()); 567 } 568 } 569 570 const char* GetDescription() const OVERRIDE { return "TypeCheckSlowPathARM"; } 571 572 bool IsFatal() const OVERRIDE { return is_fatal_; } 573 574 private: 575 const bool is_fatal_; 576 577 DISALLOW_COPY_AND_ASSIGN(TypeCheckSlowPathARM); 578}; 579 580class DeoptimizationSlowPathARM : public SlowPathCodeARM { 581 public: 582 explicit DeoptimizationSlowPathARM(HDeoptimize* instruction) 583 : SlowPathCodeARM(instruction) {} 584 585 void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { 586 CodeGeneratorARM* arm_codegen = down_cast<CodeGeneratorARM*>(codegen); 587 __ Bind(GetEntryLabel()); 588 arm_codegen->InvokeRuntime(kQuickDeoptimize, instruction_, instruction_->GetDexPc(), this); 589 CheckEntrypointTypes<kQuickDeoptimize, void, void>(); 590 } 591 592 const char* GetDescription() const OVERRIDE { return "DeoptimizationSlowPathARM"; } 593 594 private: 595 DISALLOW_COPY_AND_ASSIGN(DeoptimizationSlowPathARM); 596}; 597 598class ArraySetSlowPathARM : public SlowPathCodeARM { 599 public: 600 explicit ArraySetSlowPathARM(HInstruction* instruction) : SlowPathCodeARM(instruction) {} 601 602 void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { 603 LocationSummary* locations = instruction_->GetLocations(); 604 __ Bind(GetEntryLabel()); 605 SaveLiveRegisters(codegen, locations); 606 607 InvokeRuntimeCallingConvention calling_convention; 608 HParallelMove parallel_move(codegen->GetGraph()->GetArena()); 609 parallel_move.AddMove( 610 locations->InAt(0), 611 Location::RegisterLocation(calling_convention.GetRegisterAt(0)), 612 Primitive::kPrimNot, 613 nullptr); 614 parallel_move.AddMove( 615 locations->InAt(1), 616 Location::RegisterLocation(calling_convention.GetRegisterAt(1)), 617 Primitive::kPrimInt, 618 nullptr); 619 parallel_move.AddMove( 620 locations->InAt(2), 621 Location::RegisterLocation(calling_convention.GetRegisterAt(2)), 622 Primitive::kPrimNot, 623 nullptr); 624 codegen->GetMoveResolver()->EmitNativeCode(¶llel_move); 625 626 CodeGeneratorARM* arm_codegen = down_cast<CodeGeneratorARM*>(codegen); 627 arm_codegen->InvokeRuntime(kQuickAputObject, instruction_, instruction_->GetDexPc(), this); 628 CheckEntrypointTypes<kQuickAputObject, void, mirror::Array*, int32_t, mirror::Object*>(); 629 RestoreLiveRegisters(codegen, locations); 630 __ b(GetExitLabel()); 631 } 632 633 const char* GetDescription() const OVERRIDE { return "ArraySetSlowPathARM"; } 634 635 private: 636 DISALLOW_COPY_AND_ASSIGN(ArraySetSlowPathARM); 637}; 638 639// Slow path marking an object reference `ref` during a read 640// barrier. The field `obj.field` in the object `obj` holding this 641// reference does not get updated by this slow path after marking (see 642// ReadBarrierMarkAndUpdateFieldSlowPathARM below for that). 643// 644// This means that after the execution of this slow path, `ref` will 645// always be up-to-date, but `obj.field` may not; i.e., after the 646// flip, `ref` will be a to-space reference, but `obj.field` will 647// probably still be a from-space reference (unless it gets updated by 648// another thread, or if another thread installed another object 649// reference (different from `ref`) in `obj.field`). 650// 651// If `entrypoint` is a valid location it is assumed to already be 652// holding the entrypoint. The case where the entrypoint is passed in 653// is for the GcRoot read barrier. 654class ReadBarrierMarkSlowPathARM : public SlowPathCodeARM { 655 public: 656 ReadBarrierMarkSlowPathARM(HInstruction* instruction, 657 Location ref, 658 Location entrypoint = Location::NoLocation()) 659 : SlowPathCodeARM(instruction), ref_(ref), entrypoint_(entrypoint) { 660 DCHECK(kEmitCompilerReadBarrier); 661 } 662 663 const char* GetDescription() const OVERRIDE { return "ReadBarrierMarkSlowPathARM"; } 664 665 void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { 666 LocationSummary* locations = instruction_->GetLocations(); 667 Register ref_reg = ref_.AsRegister<Register>(); 668 DCHECK(locations->CanCall()); 669 DCHECK(!locations->GetLiveRegisters()->ContainsCoreRegister(ref_reg)) << ref_reg; 670 DCHECK(instruction_->IsInstanceFieldGet() || 671 instruction_->IsStaticFieldGet() || 672 instruction_->IsArrayGet() || 673 instruction_->IsArraySet() || 674 instruction_->IsLoadClass() || 675 instruction_->IsLoadString() || 676 instruction_->IsInstanceOf() || 677 instruction_->IsCheckCast() || 678 (instruction_->IsInvokeVirtual() && instruction_->GetLocations()->Intrinsified()) || 679 (instruction_->IsInvokeStaticOrDirect() && instruction_->GetLocations()->Intrinsified())) 680 << "Unexpected instruction in read barrier marking slow path: " 681 << instruction_->DebugName(); 682 // The read barrier instrumentation of object ArrayGet 683 // instructions does not support the HIntermediateAddress 684 // instruction. 685 DCHECK(!(instruction_->IsArrayGet() && 686 instruction_->AsArrayGet()->GetArray()->IsIntermediateAddress())); 687 688 __ Bind(GetEntryLabel()); 689 // No need to save live registers; it's taken care of by the 690 // entrypoint. Also, there is no need to update the stack mask, 691 // as this runtime call will not trigger a garbage collection. 692 CodeGeneratorARM* arm_codegen = down_cast<CodeGeneratorARM*>(codegen); 693 DCHECK_NE(ref_reg, SP); 694 DCHECK_NE(ref_reg, LR); 695 DCHECK_NE(ref_reg, PC); 696 // IP is used internally by the ReadBarrierMarkRegX entry point 697 // as a temporary, it cannot be the entry point's input/output. 698 DCHECK_NE(ref_reg, IP); 699 DCHECK(0 <= ref_reg && ref_reg < kNumberOfCoreRegisters) << ref_reg; 700 // "Compact" slow path, saving two moves. 701 // 702 // Instead of using the standard runtime calling convention (input 703 // and output in R0): 704 // 705 // R0 <- ref 706 // R0 <- ReadBarrierMark(R0) 707 // ref <- R0 708 // 709 // we just use rX (the register containing `ref`) as input and output 710 // of a dedicated entrypoint: 711 // 712 // rX <- ReadBarrierMarkRegX(rX) 713 // 714 if (entrypoint_.IsValid()) { 715 arm_codegen->ValidateInvokeRuntimeWithoutRecordingPcInfo(instruction_, this); 716 __ blx(entrypoint_.AsRegister<Register>()); 717 } else { 718 int32_t entry_point_offset = 719 CodeGenerator::GetReadBarrierMarkEntryPointsOffset<kArmPointerSize>(ref_reg); 720 // This runtime call does not require a stack map. 721 arm_codegen->InvokeRuntimeWithoutRecordingPcInfo(entry_point_offset, instruction_, this); 722 } 723 __ b(GetExitLabel()); 724 } 725 726 private: 727 // The location (register) of the marked object reference. 728 const Location ref_; 729 730 // The location of the entrypoint if already loaded. 731 const Location entrypoint_; 732 733 DISALLOW_COPY_AND_ASSIGN(ReadBarrierMarkSlowPathARM); 734}; 735 736// Slow path marking an object reference `ref` during a read barrier, 737// and if needed, atomically updating the field `obj.field` in the 738// object `obj` holding this reference after marking (contrary to 739// ReadBarrierMarkSlowPathARM above, which never tries to update 740// `obj.field`). 741// 742// This means that after the execution of this slow path, both `ref` 743// and `obj.field` will be up-to-date; i.e., after the flip, both will 744// hold the same to-space reference (unless another thread installed 745// another object reference (different from `ref`) in `obj.field`). 746class ReadBarrierMarkAndUpdateFieldSlowPathARM : public SlowPathCodeARM { 747 public: 748 ReadBarrierMarkAndUpdateFieldSlowPathARM(HInstruction* instruction, 749 Location ref, 750 Register obj, 751 Location field_offset, 752 Register temp1, 753 Register temp2) 754 : SlowPathCodeARM(instruction), 755 ref_(ref), 756 obj_(obj), 757 field_offset_(field_offset), 758 temp1_(temp1), 759 temp2_(temp2) { 760 DCHECK(kEmitCompilerReadBarrier); 761 } 762 763 const char* GetDescription() const OVERRIDE { return "ReadBarrierMarkAndUpdateFieldSlowPathARM"; } 764 765 void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { 766 LocationSummary* locations = instruction_->GetLocations(); 767 Register ref_reg = ref_.AsRegister<Register>(); 768 DCHECK(locations->CanCall()); 769 DCHECK(!locations->GetLiveRegisters()->ContainsCoreRegister(ref_reg)) << ref_reg; 770 // This slow path is only used by the UnsafeCASObject intrinsic. 771 DCHECK((instruction_->IsInvokeVirtual() && instruction_->GetLocations()->Intrinsified())) 772 << "Unexpected instruction in read barrier marking and field updating slow path: " 773 << instruction_->DebugName(); 774 DCHECK(instruction_->GetLocations()->Intrinsified()); 775 DCHECK_EQ(instruction_->AsInvoke()->GetIntrinsic(), Intrinsics::kUnsafeCASObject); 776 DCHECK(field_offset_.IsRegisterPair()) << field_offset_; 777 778 __ Bind(GetEntryLabel()); 779 780 // Save the old reference. 781 // Note that we cannot use IP to save the old reference, as IP is 782 // used internally by the ReadBarrierMarkRegX entry point, and we 783 // need the old reference after the call to that entry point. 784 DCHECK_NE(temp1_, IP); 785 __ Mov(temp1_, ref_reg); 786 787 // No need to save live registers; it's taken care of by the 788 // entrypoint. Also, there is no need to update the stack mask, 789 // as this runtime call will not trigger a garbage collection. 790 CodeGeneratorARM* arm_codegen = down_cast<CodeGeneratorARM*>(codegen); 791 DCHECK_NE(ref_reg, SP); 792 DCHECK_NE(ref_reg, LR); 793 DCHECK_NE(ref_reg, PC); 794 // IP is used internally by the ReadBarrierMarkRegX entry point 795 // as a temporary, it cannot be the entry point's input/output. 796 DCHECK_NE(ref_reg, IP); 797 DCHECK(0 <= ref_reg && ref_reg < kNumberOfCoreRegisters) << ref_reg; 798 // "Compact" slow path, saving two moves. 799 // 800 // Instead of using the standard runtime calling convention (input 801 // and output in R0): 802 // 803 // R0 <- ref 804 // R0 <- ReadBarrierMark(R0) 805 // ref <- R0 806 // 807 // we just use rX (the register containing `ref`) as input and output 808 // of a dedicated entrypoint: 809 // 810 // rX <- ReadBarrierMarkRegX(rX) 811 // 812 int32_t entry_point_offset = 813 CodeGenerator::GetReadBarrierMarkEntryPointsOffset<kArmPointerSize>(ref_reg); 814 // This runtime call does not require a stack map. 815 arm_codegen->InvokeRuntimeWithoutRecordingPcInfo(entry_point_offset, instruction_, this); 816 817 // If the new reference is different from the old reference, 818 // update the field in the holder (`*(obj_ + field_offset_)`). 819 // 820 // Note that this field could also hold a different object, if 821 // another thread had concurrently changed it. In that case, the 822 // LDREX/SUBS/ITNE sequence of instructions in the compare-and-set 823 // (CAS) operation below would abort the CAS, leaving the field 824 // as-is. 825 Label done; 826 __ cmp(temp1_, ShifterOperand(ref_reg)); 827 __ b(&done, EQ); 828 829 // Update the the holder's field atomically. This may fail if 830 // mutator updates before us, but it's OK. This is achieved 831 // using a strong compare-and-set (CAS) operation with relaxed 832 // memory synchronization ordering, where the expected value is 833 // the old reference and the desired value is the new reference. 834 835 // Convenience aliases. 836 Register base = obj_; 837 // The UnsafeCASObject intrinsic uses a register pair as field 838 // offset ("long offset"), of which only the low part contains 839 // data. 840 Register offset = field_offset_.AsRegisterPairLow<Register>(); 841 Register expected = temp1_; 842 Register value = ref_reg; 843 Register tmp_ptr = IP; // Pointer to actual memory. 844 Register tmp = temp2_; // Value in memory. 845 846 __ add(tmp_ptr, base, ShifterOperand(offset)); 847 848 if (kPoisonHeapReferences) { 849 __ PoisonHeapReference(expected); 850 if (value == expected) { 851 // Do not poison `value`, as it is the same register as 852 // `expected`, which has just been poisoned. 853 } else { 854 __ PoisonHeapReference(value); 855 } 856 } 857 858 // do { 859 // tmp = [r_ptr] - expected; 860 // } while (tmp == 0 && failure([r_ptr] <- r_new_value)); 861 862 Label loop_head, exit_loop; 863 __ Bind(&loop_head); 864 865 __ ldrex(tmp, tmp_ptr); 866 867 __ subs(tmp, tmp, ShifterOperand(expected)); 868 869 __ it(NE); 870 __ clrex(NE); 871 872 __ b(&exit_loop, NE); 873 874 __ strex(tmp, value, tmp_ptr); 875 __ cmp(tmp, ShifterOperand(1)); 876 __ b(&loop_head, EQ); 877 878 __ Bind(&exit_loop); 879 880 if (kPoisonHeapReferences) { 881 __ UnpoisonHeapReference(expected); 882 if (value == expected) { 883 // Do not unpoison `value`, as it is the same register as 884 // `expected`, which has just been unpoisoned. 885 } else { 886 __ UnpoisonHeapReference(value); 887 } 888 } 889 890 __ Bind(&done); 891 __ b(GetExitLabel()); 892 } 893 894 private: 895 // The location (register) of the marked object reference. 896 const Location ref_; 897 // The register containing the object holding the marked object reference field. 898 const Register obj_; 899 // The location of the offset of the marked reference field within `obj_`. 900 Location field_offset_; 901 902 const Register temp1_; 903 const Register temp2_; 904 905 DISALLOW_COPY_AND_ASSIGN(ReadBarrierMarkAndUpdateFieldSlowPathARM); 906}; 907 908// Slow path generating a read barrier for a heap reference. 909class ReadBarrierForHeapReferenceSlowPathARM : public SlowPathCodeARM { 910 public: 911 ReadBarrierForHeapReferenceSlowPathARM(HInstruction* instruction, 912 Location out, 913 Location ref, 914 Location obj, 915 uint32_t offset, 916 Location index) 917 : SlowPathCodeARM(instruction), 918 out_(out), 919 ref_(ref), 920 obj_(obj), 921 offset_(offset), 922 index_(index) { 923 DCHECK(kEmitCompilerReadBarrier); 924 // If `obj` is equal to `out` or `ref`, it means the initial object 925 // has been overwritten by (or after) the heap object reference load 926 // to be instrumented, e.g.: 927 // 928 // __ LoadFromOffset(kLoadWord, out, out, offset); 929 // codegen_->GenerateReadBarrierSlow(instruction, out_loc, out_loc, out_loc, offset); 930 // 931 // In that case, we have lost the information about the original 932 // object, and the emitted read barrier cannot work properly. 933 DCHECK(!obj.Equals(out)) << "obj=" << obj << " out=" << out; 934 DCHECK(!obj.Equals(ref)) << "obj=" << obj << " ref=" << ref; 935 } 936 937 void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { 938 CodeGeneratorARM* arm_codegen = down_cast<CodeGeneratorARM*>(codegen); 939 LocationSummary* locations = instruction_->GetLocations(); 940 Register reg_out = out_.AsRegister<Register>(); 941 DCHECK(locations->CanCall()); 942 DCHECK(!locations->GetLiveRegisters()->ContainsCoreRegister(reg_out)); 943 DCHECK(instruction_->IsInstanceFieldGet() || 944 instruction_->IsStaticFieldGet() || 945 instruction_->IsArrayGet() || 946 instruction_->IsInstanceOf() || 947 instruction_->IsCheckCast() || 948 (instruction_->IsInvokeVirtual()) && instruction_->GetLocations()->Intrinsified()) 949 << "Unexpected instruction in read barrier for heap reference slow path: " 950 << instruction_->DebugName(); 951 // The read barrier instrumentation of object ArrayGet 952 // instructions does not support the HIntermediateAddress 953 // instruction. 954 DCHECK(!(instruction_->IsArrayGet() && 955 instruction_->AsArrayGet()->GetArray()->IsIntermediateAddress())); 956 957 __ Bind(GetEntryLabel()); 958 SaveLiveRegisters(codegen, locations); 959 960 // We may have to change the index's value, but as `index_` is a 961 // constant member (like other "inputs" of this slow path), 962 // introduce a copy of it, `index`. 963 Location index = index_; 964 if (index_.IsValid()) { 965 // Handle `index_` for HArrayGet and UnsafeGetObject/UnsafeGetObjectVolatile intrinsics. 966 if (instruction_->IsArrayGet()) { 967 // Compute the actual memory offset and store it in `index`. 968 Register index_reg = index_.AsRegister<Register>(); 969 DCHECK(locations->GetLiveRegisters()->ContainsCoreRegister(index_reg)); 970 if (codegen->IsCoreCalleeSaveRegister(index_reg)) { 971 // We are about to change the value of `index_reg` (see the 972 // calls to art::arm::Thumb2Assembler::Lsl and 973 // art::arm::Thumb2Assembler::AddConstant below), but it has 974 // not been saved by the previous call to 975 // art::SlowPathCode::SaveLiveRegisters, as it is a 976 // callee-save register -- 977 // art::SlowPathCode::SaveLiveRegisters does not consider 978 // callee-save registers, as it has been designed with the 979 // assumption that callee-save registers are supposed to be 980 // handled by the called function. So, as a callee-save 981 // register, `index_reg` _would_ eventually be saved onto 982 // the stack, but it would be too late: we would have 983 // changed its value earlier. Therefore, we manually save 984 // it here into another freely available register, 985 // `free_reg`, chosen of course among the caller-save 986 // registers (as a callee-save `free_reg` register would 987 // exhibit the same problem). 988 // 989 // Note we could have requested a temporary register from 990 // the register allocator instead; but we prefer not to, as 991 // this is a slow path, and we know we can find a 992 // caller-save register that is available. 993 Register free_reg = FindAvailableCallerSaveRegister(codegen); 994 __ Mov(free_reg, index_reg); 995 index_reg = free_reg; 996 index = Location::RegisterLocation(index_reg); 997 } else { 998 // The initial register stored in `index_` has already been 999 // saved in the call to art::SlowPathCode::SaveLiveRegisters 1000 // (as it is not a callee-save register), so we can freely 1001 // use it. 1002 } 1003 // Shifting the index value contained in `index_reg` by the scale 1004 // factor (2) cannot overflow in practice, as the runtime is 1005 // unable to allocate object arrays with a size larger than 1006 // 2^26 - 1 (that is, 2^28 - 4 bytes). 1007 __ Lsl(index_reg, index_reg, TIMES_4); 1008 static_assert( 1009 sizeof(mirror::HeapReference<mirror::Object>) == sizeof(int32_t), 1010 "art::mirror::HeapReference<art::mirror::Object> and int32_t have different sizes."); 1011 __ AddConstant(index_reg, index_reg, offset_); 1012 } else { 1013 // In the case of the UnsafeGetObject/UnsafeGetObjectVolatile 1014 // intrinsics, `index_` is not shifted by a scale factor of 2 1015 // (as in the case of ArrayGet), as it is actually an offset 1016 // to an object field within an object. 1017 DCHECK(instruction_->IsInvoke()) << instruction_->DebugName(); 1018 DCHECK(instruction_->GetLocations()->Intrinsified()); 1019 DCHECK((instruction_->AsInvoke()->GetIntrinsic() == Intrinsics::kUnsafeGetObject) || 1020 (instruction_->AsInvoke()->GetIntrinsic() == Intrinsics::kUnsafeGetObjectVolatile)) 1021 << instruction_->AsInvoke()->GetIntrinsic(); 1022 DCHECK_EQ(offset_, 0U); 1023 DCHECK(index_.IsRegisterPair()); 1024 // UnsafeGet's offset location is a register pair, the low 1025 // part contains the correct offset. 1026 index = index_.ToLow(); 1027 } 1028 } 1029 1030 // We're moving two or three locations to locations that could 1031 // overlap, so we need a parallel move resolver. 1032 InvokeRuntimeCallingConvention calling_convention; 1033 HParallelMove parallel_move(codegen->GetGraph()->GetArena()); 1034 parallel_move.AddMove(ref_, 1035 Location::RegisterLocation(calling_convention.GetRegisterAt(0)), 1036 Primitive::kPrimNot, 1037 nullptr); 1038 parallel_move.AddMove(obj_, 1039 Location::RegisterLocation(calling_convention.GetRegisterAt(1)), 1040 Primitive::kPrimNot, 1041 nullptr); 1042 if (index.IsValid()) { 1043 parallel_move.AddMove(index, 1044 Location::RegisterLocation(calling_convention.GetRegisterAt(2)), 1045 Primitive::kPrimInt, 1046 nullptr); 1047 codegen->GetMoveResolver()->EmitNativeCode(¶llel_move); 1048 } else { 1049 codegen->GetMoveResolver()->EmitNativeCode(¶llel_move); 1050 __ LoadImmediate(calling_convention.GetRegisterAt(2), offset_); 1051 } 1052 arm_codegen->InvokeRuntime(kQuickReadBarrierSlow, instruction_, instruction_->GetDexPc(), this); 1053 CheckEntrypointTypes< 1054 kQuickReadBarrierSlow, mirror::Object*, mirror::Object*, mirror::Object*, uint32_t>(); 1055 arm_codegen->Move32(out_, Location::RegisterLocation(R0)); 1056 1057 RestoreLiveRegisters(codegen, locations); 1058 __ b(GetExitLabel()); 1059 } 1060 1061 const char* GetDescription() const OVERRIDE { return "ReadBarrierForHeapReferenceSlowPathARM"; } 1062 1063 private: 1064 Register FindAvailableCallerSaveRegister(CodeGenerator* codegen) { 1065 size_t ref = static_cast<int>(ref_.AsRegister<Register>()); 1066 size_t obj = static_cast<int>(obj_.AsRegister<Register>()); 1067 for (size_t i = 0, e = codegen->GetNumberOfCoreRegisters(); i < e; ++i) { 1068 if (i != ref && i != obj && !codegen->IsCoreCalleeSaveRegister(i)) { 1069 return static_cast<Register>(i); 1070 } 1071 } 1072 // We shall never fail to find a free caller-save register, as 1073 // there are more than two core caller-save registers on ARM 1074 // (meaning it is possible to find one which is different from 1075 // `ref` and `obj`). 1076 DCHECK_GT(codegen->GetNumberOfCoreCallerSaveRegisters(), 2u); 1077 LOG(FATAL) << "Could not find a free caller-save register"; 1078 UNREACHABLE(); 1079 } 1080 1081 const Location out_; 1082 const Location ref_; 1083 const Location obj_; 1084 const uint32_t offset_; 1085 // An additional location containing an index to an array. 1086 // Only used for HArrayGet and the UnsafeGetObject & 1087 // UnsafeGetObjectVolatile intrinsics. 1088 const Location index_; 1089 1090 DISALLOW_COPY_AND_ASSIGN(ReadBarrierForHeapReferenceSlowPathARM); 1091}; 1092 1093// Slow path generating a read barrier for a GC root. 1094class ReadBarrierForRootSlowPathARM : public SlowPathCodeARM { 1095 public: 1096 ReadBarrierForRootSlowPathARM(HInstruction* instruction, Location out, Location root) 1097 : SlowPathCodeARM(instruction), out_(out), root_(root) { 1098 DCHECK(kEmitCompilerReadBarrier); 1099 } 1100 1101 void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { 1102 LocationSummary* locations = instruction_->GetLocations(); 1103 Register reg_out = out_.AsRegister<Register>(); 1104 DCHECK(locations->CanCall()); 1105 DCHECK(!locations->GetLiveRegisters()->ContainsCoreRegister(reg_out)); 1106 DCHECK(instruction_->IsLoadClass() || instruction_->IsLoadString()) 1107 << "Unexpected instruction in read barrier for GC root slow path: " 1108 << instruction_->DebugName(); 1109 1110 __ Bind(GetEntryLabel()); 1111 SaveLiveRegisters(codegen, locations); 1112 1113 InvokeRuntimeCallingConvention calling_convention; 1114 CodeGeneratorARM* arm_codegen = down_cast<CodeGeneratorARM*>(codegen); 1115 arm_codegen->Move32(Location::RegisterLocation(calling_convention.GetRegisterAt(0)), root_); 1116 arm_codegen->InvokeRuntime(kQuickReadBarrierForRootSlow, 1117 instruction_, 1118 instruction_->GetDexPc(), 1119 this); 1120 CheckEntrypointTypes<kQuickReadBarrierForRootSlow, mirror::Object*, GcRoot<mirror::Object>*>(); 1121 arm_codegen->Move32(out_, Location::RegisterLocation(R0)); 1122 1123 RestoreLiveRegisters(codegen, locations); 1124 __ b(GetExitLabel()); 1125 } 1126 1127 const char* GetDescription() const OVERRIDE { return "ReadBarrierForRootSlowPathARM"; } 1128 1129 private: 1130 const Location out_; 1131 const Location root_; 1132 1133 DISALLOW_COPY_AND_ASSIGN(ReadBarrierForRootSlowPathARM); 1134}; 1135 1136inline Condition ARMCondition(IfCondition cond) { 1137 switch (cond) { 1138 case kCondEQ: return EQ; 1139 case kCondNE: return NE; 1140 case kCondLT: return LT; 1141 case kCondLE: return LE; 1142 case kCondGT: return GT; 1143 case kCondGE: return GE; 1144 case kCondB: return LO; 1145 case kCondBE: return LS; 1146 case kCondA: return HI; 1147 case kCondAE: return HS; 1148 } 1149 LOG(FATAL) << "Unreachable"; 1150 UNREACHABLE(); 1151} 1152 1153// Maps signed condition to unsigned condition. 1154inline Condition ARMUnsignedCondition(IfCondition cond) { 1155 switch (cond) { 1156 case kCondEQ: return EQ; 1157 case kCondNE: return NE; 1158 // Signed to unsigned. 1159 case kCondLT: return LO; 1160 case kCondLE: return LS; 1161 case kCondGT: return HI; 1162 case kCondGE: return HS; 1163 // Unsigned remain unchanged. 1164 case kCondB: return LO; 1165 case kCondBE: return LS; 1166 case kCondA: return HI; 1167 case kCondAE: return HS; 1168 } 1169 LOG(FATAL) << "Unreachable"; 1170 UNREACHABLE(); 1171} 1172 1173inline Condition ARMFPCondition(IfCondition cond, bool gt_bias) { 1174 // The ARM condition codes can express all the necessary branches, see the 1175 // "Meaning (floating-point)" column in the table A8-1 of the ARMv7 reference manual. 1176 // There is no dex instruction or HIR that would need the missing conditions 1177 // "equal or unordered" or "not equal". 1178 switch (cond) { 1179 case kCondEQ: return EQ; 1180 case kCondNE: return NE /* unordered */; 1181 case kCondLT: return gt_bias ? CC : LT /* unordered */; 1182 case kCondLE: return gt_bias ? LS : LE /* unordered */; 1183 case kCondGT: return gt_bias ? HI /* unordered */ : GT; 1184 case kCondGE: return gt_bias ? CS /* unordered */ : GE; 1185 default: 1186 LOG(FATAL) << "UNREACHABLE"; 1187 UNREACHABLE(); 1188 } 1189} 1190 1191inline Shift ShiftFromOpKind(HDataProcWithShifterOp::OpKind op_kind) { 1192 switch (op_kind) { 1193 case HDataProcWithShifterOp::kASR: return ASR; 1194 case HDataProcWithShifterOp::kLSL: return LSL; 1195 case HDataProcWithShifterOp::kLSR: return LSR; 1196 default: 1197 LOG(FATAL) << "Unexpected op kind " << op_kind; 1198 UNREACHABLE(); 1199 } 1200} 1201 1202static void GenerateDataProcInstruction(HInstruction::InstructionKind kind, 1203 Register out, 1204 Register first, 1205 const ShifterOperand& second, 1206 CodeGeneratorARM* codegen) { 1207 if (second.IsImmediate() && second.GetImmediate() == 0) { 1208 const ShifterOperand in = kind == HInstruction::kAnd 1209 ? ShifterOperand(0) 1210 : ShifterOperand(first); 1211 1212 __ mov(out, in); 1213 } else { 1214 switch (kind) { 1215 case HInstruction::kAdd: 1216 __ add(out, first, second); 1217 break; 1218 case HInstruction::kAnd: 1219 __ and_(out, first, second); 1220 break; 1221 case HInstruction::kOr: 1222 __ orr(out, first, second); 1223 break; 1224 case HInstruction::kSub: 1225 __ sub(out, first, second); 1226 break; 1227 case HInstruction::kXor: 1228 __ eor(out, first, second); 1229 break; 1230 default: 1231 LOG(FATAL) << "Unexpected instruction kind: " << kind; 1232 UNREACHABLE(); 1233 } 1234 } 1235} 1236 1237static void GenerateDataProc(HInstruction::InstructionKind kind, 1238 const Location& out, 1239 const Location& first, 1240 const ShifterOperand& second_lo, 1241 const ShifterOperand& second_hi, 1242 CodeGeneratorARM* codegen) { 1243 const Register first_hi = first.AsRegisterPairHigh<Register>(); 1244 const Register first_lo = first.AsRegisterPairLow<Register>(); 1245 const Register out_hi = out.AsRegisterPairHigh<Register>(); 1246 const Register out_lo = out.AsRegisterPairLow<Register>(); 1247 1248 if (kind == HInstruction::kAdd) { 1249 __ adds(out_lo, first_lo, second_lo); 1250 __ adc(out_hi, first_hi, second_hi); 1251 } else if (kind == HInstruction::kSub) { 1252 __ subs(out_lo, first_lo, second_lo); 1253 __ sbc(out_hi, first_hi, second_hi); 1254 } else { 1255 GenerateDataProcInstruction(kind, out_lo, first_lo, second_lo, codegen); 1256 GenerateDataProcInstruction(kind, out_hi, first_hi, second_hi, codegen); 1257 } 1258} 1259 1260static ShifterOperand GetShifterOperand(Register rm, Shift shift, uint32_t shift_imm) { 1261 return shift_imm == 0 ? ShifterOperand(rm) : ShifterOperand(rm, shift, shift_imm); 1262} 1263 1264static void GenerateLongDataProc(HDataProcWithShifterOp* instruction, CodeGeneratorARM* codegen) { 1265 DCHECK_EQ(instruction->GetType(), Primitive::kPrimLong); 1266 DCHECK(HDataProcWithShifterOp::IsShiftOp(instruction->GetOpKind())); 1267 1268 const LocationSummary* const locations = instruction->GetLocations(); 1269 const uint32_t shift_value = instruction->GetShiftAmount(); 1270 const HInstruction::InstructionKind kind = instruction->GetInstrKind(); 1271 const Location first = locations->InAt(0); 1272 const Location second = locations->InAt(1); 1273 const Location out = locations->Out(); 1274 const Register first_hi = first.AsRegisterPairHigh<Register>(); 1275 const Register first_lo = first.AsRegisterPairLow<Register>(); 1276 const Register out_hi = out.AsRegisterPairHigh<Register>(); 1277 const Register out_lo = out.AsRegisterPairLow<Register>(); 1278 const Register second_hi = second.AsRegisterPairHigh<Register>(); 1279 const Register second_lo = second.AsRegisterPairLow<Register>(); 1280 const Shift shift = ShiftFromOpKind(instruction->GetOpKind()); 1281 1282 if (shift_value >= 32) { 1283 if (shift == LSL) { 1284 GenerateDataProcInstruction(kind, 1285 out_hi, 1286 first_hi, 1287 ShifterOperand(second_lo, LSL, shift_value - 32), 1288 codegen); 1289 GenerateDataProcInstruction(kind, 1290 out_lo, 1291 first_lo, 1292 ShifterOperand(0), 1293 codegen); 1294 } else if (shift == ASR) { 1295 GenerateDataProc(kind, 1296 out, 1297 first, 1298 GetShifterOperand(second_hi, ASR, shift_value - 32), 1299 ShifterOperand(second_hi, ASR, 31), 1300 codegen); 1301 } else { 1302 DCHECK_EQ(shift, LSR); 1303 GenerateDataProc(kind, 1304 out, 1305 first, 1306 GetShifterOperand(second_hi, LSR, shift_value - 32), 1307 ShifterOperand(0), 1308 codegen); 1309 } 1310 } else { 1311 DCHECK_GT(shift_value, 1U); 1312 DCHECK_LT(shift_value, 32U); 1313 1314 if (shift == LSL) { 1315 // We are not doing this for HInstruction::kAdd because the output will require 1316 // Location::kOutputOverlap; not applicable to other cases. 1317 if (kind == HInstruction::kOr || kind == HInstruction::kXor) { 1318 GenerateDataProcInstruction(kind, 1319 out_hi, 1320 first_hi, 1321 ShifterOperand(second_hi, LSL, shift_value), 1322 codegen); 1323 GenerateDataProcInstruction(kind, 1324 out_hi, 1325 out_hi, 1326 ShifterOperand(second_lo, LSR, 32 - shift_value), 1327 codegen); 1328 GenerateDataProcInstruction(kind, 1329 out_lo, 1330 first_lo, 1331 ShifterOperand(second_lo, LSL, shift_value), 1332 codegen); 1333 } else { 1334 __ Lsl(IP, second_hi, shift_value); 1335 __ orr(IP, IP, ShifterOperand(second_lo, LSR, 32 - shift_value)); 1336 GenerateDataProc(kind, 1337 out, 1338 first, 1339 ShifterOperand(second_lo, LSL, shift_value), 1340 ShifterOperand(IP), 1341 codegen); 1342 } 1343 } else { 1344 DCHECK(shift == ASR || shift == LSR); 1345 1346 // We are not doing this for HInstruction::kAdd because the output will require 1347 // Location::kOutputOverlap; not applicable to other cases. 1348 if (kind == HInstruction::kOr || kind == HInstruction::kXor) { 1349 GenerateDataProcInstruction(kind, 1350 out_lo, 1351 first_lo, 1352 ShifterOperand(second_lo, LSR, shift_value), 1353 codegen); 1354 GenerateDataProcInstruction(kind, 1355 out_lo, 1356 out_lo, 1357 ShifterOperand(second_hi, LSL, 32 - shift_value), 1358 codegen); 1359 GenerateDataProcInstruction(kind, 1360 out_hi, 1361 first_hi, 1362 ShifterOperand(second_hi, shift, shift_value), 1363 codegen); 1364 } else { 1365 __ Lsr(IP, second_lo, shift_value); 1366 __ orr(IP, IP, ShifterOperand(second_hi, LSL, 32 - shift_value)); 1367 GenerateDataProc(kind, 1368 out, 1369 first, 1370 ShifterOperand(IP), 1371 ShifterOperand(second_hi, shift, shift_value), 1372 codegen); 1373 } 1374 } 1375 } 1376} 1377 1378#undef __ 1379// NOLINT on __ macro to suppress wrong warning/fix (misc-macro-parentheses) from clang-tidy. 1380#define __ down_cast<ArmAssembler*>(GetAssembler())-> // NOLINT 1381 1382void CodeGeneratorARM::DumpCoreRegister(std::ostream& stream, int reg) const { 1383 stream << Register(reg); 1384} 1385 1386void CodeGeneratorARM::DumpFloatingPointRegister(std::ostream& stream, int reg) const { 1387 stream << SRegister(reg); 1388} 1389 1390size_t CodeGeneratorARM::SaveCoreRegister(size_t stack_index, uint32_t reg_id) { 1391 __ StoreToOffset(kStoreWord, static_cast<Register>(reg_id), SP, stack_index); 1392 return kArmWordSize; 1393} 1394 1395size_t CodeGeneratorARM::RestoreCoreRegister(size_t stack_index, uint32_t reg_id) { 1396 __ LoadFromOffset(kLoadWord, static_cast<Register>(reg_id), SP, stack_index); 1397 return kArmWordSize; 1398} 1399 1400size_t CodeGeneratorARM::SaveFloatingPointRegister(size_t stack_index, uint32_t reg_id) { 1401 __ StoreSToOffset(static_cast<SRegister>(reg_id), SP, stack_index); 1402 return kArmWordSize; 1403} 1404 1405size_t CodeGeneratorARM::RestoreFloatingPointRegister(size_t stack_index, uint32_t reg_id) { 1406 __ LoadSFromOffset(static_cast<SRegister>(reg_id), SP, stack_index); 1407 return kArmWordSize; 1408} 1409 1410CodeGeneratorARM::CodeGeneratorARM(HGraph* graph, 1411 const ArmInstructionSetFeatures& isa_features, 1412 const CompilerOptions& compiler_options, 1413 OptimizingCompilerStats* stats) 1414 : CodeGenerator(graph, 1415 kNumberOfCoreRegisters, 1416 kNumberOfSRegisters, 1417 kNumberOfRegisterPairs, 1418 ComputeRegisterMask(reinterpret_cast<const int*>(kCoreCalleeSaves), 1419 arraysize(kCoreCalleeSaves)), 1420 ComputeRegisterMask(reinterpret_cast<const int*>(kFpuCalleeSaves), 1421 arraysize(kFpuCalleeSaves)), 1422 compiler_options, 1423 stats), 1424 block_labels_(nullptr), 1425 location_builder_(graph, this), 1426 instruction_visitor_(graph, this), 1427 move_resolver_(graph->GetArena(), this), 1428 assembler_(graph->GetArena()), 1429 isa_features_(isa_features), 1430 uint32_literals_(std::less<uint32_t>(), 1431 graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), 1432 pc_relative_dex_cache_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), 1433 boot_image_string_patches_(StringReferenceValueComparator(), 1434 graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), 1435 pc_relative_string_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), 1436 boot_image_type_patches_(TypeReferenceValueComparator(), 1437 graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), 1438 pc_relative_type_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), 1439 type_bss_entry_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), 1440 boot_image_address_patches_(std::less<uint32_t>(), 1441 graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), 1442 jit_string_patches_(StringReferenceValueComparator(), 1443 graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), 1444 jit_class_patches_(TypeReferenceValueComparator(), 1445 graph->GetArena()->Adapter(kArenaAllocCodeGenerator)) { 1446 // Always save the LR register to mimic Quick. 1447 AddAllocatedRegister(Location::RegisterLocation(LR)); 1448} 1449 1450void CodeGeneratorARM::Finalize(CodeAllocator* allocator) { 1451 // Ensure that we fix up branches and literal loads and emit the literal pool. 1452 __ FinalizeCode(); 1453 1454 // Adjust native pc offsets in stack maps. 1455 for (size_t i = 0, num = stack_map_stream_.GetNumberOfStackMaps(); i != num; ++i) { 1456 uint32_t old_position = 1457 stack_map_stream_.GetStackMap(i).native_pc_code_offset.Uint32Value(kThumb2); 1458 uint32_t new_position = __ GetAdjustedPosition(old_position); 1459 stack_map_stream_.SetStackMapNativePcOffset(i, new_position); 1460 } 1461 // Adjust pc offsets for the disassembly information. 1462 if (disasm_info_ != nullptr) { 1463 GeneratedCodeInterval* frame_entry_interval = disasm_info_->GetFrameEntryInterval(); 1464 frame_entry_interval->start = __ GetAdjustedPosition(frame_entry_interval->start); 1465 frame_entry_interval->end = __ GetAdjustedPosition(frame_entry_interval->end); 1466 for (auto& it : *disasm_info_->GetInstructionIntervals()) { 1467 it.second.start = __ GetAdjustedPosition(it.second.start); 1468 it.second.end = __ GetAdjustedPosition(it.second.end); 1469 } 1470 for (auto& it : *disasm_info_->GetSlowPathIntervals()) { 1471 it.code_interval.start = __ GetAdjustedPosition(it.code_interval.start); 1472 it.code_interval.end = __ GetAdjustedPosition(it.code_interval.end); 1473 } 1474 } 1475 1476 CodeGenerator::Finalize(allocator); 1477} 1478 1479void CodeGeneratorARM::SetupBlockedRegisters() const { 1480 // Stack register, LR and PC are always reserved. 1481 blocked_core_registers_[SP] = true; 1482 blocked_core_registers_[LR] = true; 1483 blocked_core_registers_[PC] = true; 1484 1485 // Reserve thread register. 1486 blocked_core_registers_[TR] = true; 1487 1488 // Reserve temp register. 1489 blocked_core_registers_[IP] = true; 1490 1491 if (GetGraph()->IsDebuggable()) { 1492 // Stubs do not save callee-save floating point registers. If the graph 1493 // is debuggable, we need to deal with these registers differently. For 1494 // now, just block them. 1495 for (size_t i = 0; i < arraysize(kFpuCalleeSaves); ++i) { 1496 blocked_fpu_registers_[kFpuCalleeSaves[i]] = true; 1497 } 1498 } 1499} 1500 1501InstructionCodeGeneratorARM::InstructionCodeGeneratorARM(HGraph* graph, CodeGeneratorARM* codegen) 1502 : InstructionCodeGenerator(graph, codegen), 1503 assembler_(codegen->GetAssembler()), 1504 codegen_(codegen) {} 1505 1506void CodeGeneratorARM::ComputeSpillMask() { 1507 core_spill_mask_ = allocated_registers_.GetCoreRegisters() & core_callee_save_mask_; 1508 DCHECK_NE(core_spill_mask_, 0u) << "At least the return address register must be saved"; 1509 // There is no easy instruction to restore just the PC on thumb2. We spill and 1510 // restore another arbitrary register. 1511 core_spill_mask_ |= (1 << kCoreAlwaysSpillRegister); 1512 fpu_spill_mask_ = allocated_registers_.GetFloatingPointRegisters() & fpu_callee_save_mask_; 1513 // We use vpush and vpop for saving and restoring floating point registers, which take 1514 // a SRegister and the number of registers to save/restore after that SRegister. We 1515 // therefore update the `fpu_spill_mask_` to also contain those registers not allocated, 1516 // but in the range. 1517 if (fpu_spill_mask_ != 0) { 1518 uint32_t least_significant_bit = LeastSignificantBit(fpu_spill_mask_); 1519 uint32_t most_significant_bit = MostSignificantBit(fpu_spill_mask_); 1520 for (uint32_t i = least_significant_bit + 1 ; i < most_significant_bit; ++i) { 1521 fpu_spill_mask_ |= (1 << i); 1522 } 1523 } 1524} 1525 1526static dwarf::Reg DWARFReg(Register reg) { 1527 return dwarf::Reg::ArmCore(static_cast<int>(reg)); 1528} 1529 1530static dwarf::Reg DWARFReg(SRegister reg) { 1531 return dwarf::Reg::ArmFp(static_cast<int>(reg)); 1532} 1533 1534void CodeGeneratorARM::GenerateFrameEntry() { 1535 bool skip_overflow_check = 1536 IsLeafMethod() && !FrameNeedsStackCheck(GetFrameSize(), InstructionSet::kArm); 1537 DCHECK(GetCompilerOptions().GetImplicitStackOverflowChecks()); 1538 __ Bind(&frame_entry_label_); 1539 1540 if (HasEmptyFrame()) { 1541 return; 1542 } 1543 1544 if (!skip_overflow_check) { 1545 __ AddConstant(IP, SP, -static_cast<int32_t>(GetStackOverflowReservedBytes(kArm))); 1546 __ LoadFromOffset(kLoadWord, IP, IP, 0); 1547 RecordPcInfo(nullptr, 0); 1548 } 1549 1550 __ PushList(core_spill_mask_); 1551 __ cfi().AdjustCFAOffset(kArmWordSize * POPCOUNT(core_spill_mask_)); 1552 __ cfi().RelOffsetForMany(DWARFReg(kMethodRegisterArgument), 0, core_spill_mask_, kArmWordSize); 1553 if (fpu_spill_mask_ != 0) { 1554 SRegister start_register = SRegister(LeastSignificantBit(fpu_spill_mask_)); 1555 __ vpushs(start_register, POPCOUNT(fpu_spill_mask_)); 1556 __ cfi().AdjustCFAOffset(kArmWordSize * POPCOUNT(fpu_spill_mask_)); 1557 __ cfi().RelOffsetForMany(DWARFReg(S0), 0, fpu_spill_mask_, kArmWordSize); 1558 } 1559 1560 if (GetGraph()->HasShouldDeoptimizeFlag()) { 1561 // Initialize should_deoptimize flag to 0. 1562 __ mov(IP, ShifterOperand(0)); 1563 __ StoreToOffset(kStoreWord, IP, SP, -kShouldDeoptimizeFlagSize); 1564 } 1565 1566 int adjust = GetFrameSize() - FrameEntrySpillSize(); 1567 __ AddConstant(SP, -adjust); 1568 __ cfi().AdjustCFAOffset(adjust); 1569 1570 // Save the current method if we need it. Note that we do not 1571 // do this in HCurrentMethod, as the instruction might have been removed 1572 // in the SSA graph. 1573 if (RequiresCurrentMethod()) { 1574 __ StoreToOffset(kStoreWord, kMethodRegisterArgument, SP, 0); 1575 } 1576} 1577 1578void CodeGeneratorARM::GenerateFrameExit() { 1579 if (HasEmptyFrame()) { 1580 __ bx(LR); 1581 return; 1582 } 1583 __ cfi().RememberState(); 1584 int adjust = GetFrameSize() - FrameEntrySpillSize(); 1585 __ AddConstant(SP, adjust); 1586 __ cfi().AdjustCFAOffset(-adjust); 1587 if (fpu_spill_mask_ != 0) { 1588 SRegister start_register = SRegister(LeastSignificantBit(fpu_spill_mask_)); 1589 __ vpops(start_register, POPCOUNT(fpu_spill_mask_)); 1590 __ cfi().AdjustCFAOffset(-static_cast<int>(kArmPointerSize) * POPCOUNT(fpu_spill_mask_)); 1591 __ cfi().RestoreMany(DWARFReg(SRegister(0)), fpu_spill_mask_); 1592 } 1593 // Pop LR into PC to return. 1594 DCHECK_NE(core_spill_mask_ & (1 << LR), 0U); 1595 uint32_t pop_mask = (core_spill_mask_ & (~(1 << LR))) | 1 << PC; 1596 __ PopList(pop_mask); 1597 __ cfi().RestoreState(); 1598 __ cfi().DefCFAOffset(GetFrameSize()); 1599} 1600 1601void CodeGeneratorARM::Bind(HBasicBlock* block) { 1602 Label* label = GetLabelOf(block); 1603 __ BindTrackedLabel(label); 1604} 1605 1606Location InvokeDexCallingConventionVisitorARM::GetNextLocation(Primitive::Type type) { 1607 switch (type) { 1608 case Primitive::kPrimBoolean: 1609 case Primitive::kPrimByte: 1610 case Primitive::kPrimChar: 1611 case Primitive::kPrimShort: 1612 case Primitive::kPrimInt: 1613 case Primitive::kPrimNot: { 1614 uint32_t index = gp_index_++; 1615 uint32_t stack_index = stack_index_++; 1616 if (index < calling_convention.GetNumberOfRegisters()) { 1617 return Location::RegisterLocation(calling_convention.GetRegisterAt(index)); 1618 } else { 1619 return Location::StackSlot(calling_convention.GetStackOffsetOf(stack_index)); 1620 } 1621 } 1622 1623 case Primitive::kPrimLong: { 1624 uint32_t index = gp_index_; 1625 uint32_t stack_index = stack_index_; 1626 gp_index_ += 2; 1627 stack_index_ += 2; 1628 if (index + 1 < calling_convention.GetNumberOfRegisters()) { 1629 if (calling_convention.GetRegisterAt(index) == R1) { 1630 // Skip R1, and use R2_R3 instead. 1631 gp_index_++; 1632 index++; 1633 } 1634 } 1635 if (index + 1 < calling_convention.GetNumberOfRegisters()) { 1636 DCHECK_EQ(calling_convention.GetRegisterAt(index) + 1, 1637 calling_convention.GetRegisterAt(index + 1)); 1638 1639 return Location::RegisterPairLocation(calling_convention.GetRegisterAt(index), 1640 calling_convention.GetRegisterAt(index + 1)); 1641 } else { 1642 return Location::DoubleStackSlot(calling_convention.GetStackOffsetOf(stack_index)); 1643 } 1644 } 1645 1646 case Primitive::kPrimFloat: { 1647 uint32_t stack_index = stack_index_++; 1648 if (float_index_ % 2 == 0) { 1649 float_index_ = std::max(double_index_, float_index_); 1650 } 1651 if (float_index_ < calling_convention.GetNumberOfFpuRegisters()) { 1652 return Location::FpuRegisterLocation(calling_convention.GetFpuRegisterAt(float_index_++)); 1653 } else { 1654 return Location::StackSlot(calling_convention.GetStackOffsetOf(stack_index)); 1655 } 1656 } 1657 1658 case Primitive::kPrimDouble: { 1659 double_index_ = std::max(double_index_, RoundUp(float_index_, 2)); 1660 uint32_t stack_index = stack_index_; 1661 stack_index_ += 2; 1662 if (double_index_ + 1 < calling_convention.GetNumberOfFpuRegisters()) { 1663 uint32_t index = double_index_; 1664 double_index_ += 2; 1665 Location result = Location::FpuRegisterPairLocation( 1666 calling_convention.GetFpuRegisterAt(index), 1667 calling_convention.GetFpuRegisterAt(index + 1)); 1668 DCHECK(ExpectedPairLayout(result)); 1669 return result; 1670 } else { 1671 return Location::DoubleStackSlot(calling_convention.GetStackOffsetOf(stack_index)); 1672 } 1673 } 1674 1675 case Primitive::kPrimVoid: 1676 LOG(FATAL) << "Unexpected parameter type " << type; 1677 break; 1678 } 1679 return Location::NoLocation(); 1680} 1681 1682Location InvokeDexCallingConventionVisitorARM::GetReturnLocation(Primitive::Type type) const { 1683 switch (type) { 1684 case Primitive::kPrimBoolean: 1685 case Primitive::kPrimByte: 1686 case Primitive::kPrimChar: 1687 case Primitive::kPrimShort: 1688 case Primitive::kPrimInt: 1689 case Primitive::kPrimNot: { 1690 return Location::RegisterLocation(R0); 1691 } 1692 1693 case Primitive::kPrimFloat: { 1694 return Location::FpuRegisterLocation(S0); 1695 } 1696 1697 case Primitive::kPrimLong: { 1698 return Location::RegisterPairLocation(R0, R1); 1699 } 1700 1701 case Primitive::kPrimDouble: { 1702 return Location::FpuRegisterPairLocation(S0, S1); 1703 } 1704 1705 case Primitive::kPrimVoid: 1706 return Location::NoLocation(); 1707 } 1708 1709 UNREACHABLE(); 1710} 1711 1712Location InvokeDexCallingConventionVisitorARM::GetMethodLocation() const { 1713 return Location::RegisterLocation(kMethodRegisterArgument); 1714} 1715 1716void CodeGeneratorARM::Move32(Location destination, Location source) { 1717 if (source.Equals(destination)) { 1718 return; 1719 } 1720 if (destination.IsRegister()) { 1721 if (source.IsRegister()) { 1722 __ Mov(destination.AsRegister<Register>(), source.AsRegister<Register>()); 1723 } else if (source.IsFpuRegister()) { 1724 __ vmovrs(destination.AsRegister<Register>(), source.AsFpuRegister<SRegister>()); 1725 } else { 1726 __ LoadFromOffset(kLoadWord, destination.AsRegister<Register>(), SP, source.GetStackIndex()); 1727 } 1728 } else if (destination.IsFpuRegister()) { 1729 if (source.IsRegister()) { 1730 __ vmovsr(destination.AsFpuRegister<SRegister>(), source.AsRegister<Register>()); 1731 } else if (source.IsFpuRegister()) { 1732 __ vmovs(destination.AsFpuRegister<SRegister>(), source.AsFpuRegister<SRegister>()); 1733 } else { 1734 __ LoadSFromOffset(destination.AsFpuRegister<SRegister>(), SP, source.GetStackIndex()); 1735 } 1736 } else { 1737 DCHECK(destination.IsStackSlot()) << destination; 1738 if (source.IsRegister()) { 1739 __ StoreToOffset(kStoreWord, source.AsRegister<Register>(), SP, destination.GetStackIndex()); 1740 } else if (source.IsFpuRegister()) { 1741 __ StoreSToOffset(source.AsFpuRegister<SRegister>(), SP, destination.GetStackIndex()); 1742 } else { 1743 DCHECK(source.IsStackSlot()) << source; 1744 __ LoadFromOffset(kLoadWord, IP, SP, source.GetStackIndex()); 1745 __ StoreToOffset(kStoreWord, IP, SP, destination.GetStackIndex()); 1746 } 1747 } 1748} 1749 1750void CodeGeneratorARM::Move64(Location destination, Location source) { 1751 if (source.Equals(destination)) { 1752 return; 1753 } 1754 if (destination.IsRegisterPair()) { 1755 if (source.IsRegisterPair()) { 1756 EmitParallelMoves( 1757 Location::RegisterLocation(source.AsRegisterPairHigh<Register>()), 1758 Location::RegisterLocation(destination.AsRegisterPairHigh<Register>()), 1759 Primitive::kPrimInt, 1760 Location::RegisterLocation(source.AsRegisterPairLow<Register>()), 1761 Location::RegisterLocation(destination.AsRegisterPairLow<Register>()), 1762 Primitive::kPrimInt); 1763 } else if (source.IsFpuRegister()) { 1764 UNIMPLEMENTED(FATAL); 1765 } else if (source.IsFpuRegisterPair()) { 1766 __ vmovrrd(destination.AsRegisterPairLow<Register>(), 1767 destination.AsRegisterPairHigh<Register>(), 1768 FromLowSToD(source.AsFpuRegisterPairLow<SRegister>())); 1769 } else { 1770 DCHECK(source.IsDoubleStackSlot()); 1771 DCHECK(ExpectedPairLayout(destination)); 1772 __ LoadFromOffset(kLoadWordPair, destination.AsRegisterPairLow<Register>(), 1773 SP, source.GetStackIndex()); 1774 } 1775 } else if (destination.IsFpuRegisterPair()) { 1776 if (source.IsDoubleStackSlot()) { 1777 __ LoadDFromOffset(FromLowSToD(destination.AsFpuRegisterPairLow<SRegister>()), 1778 SP, 1779 source.GetStackIndex()); 1780 } else if (source.IsRegisterPair()) { 1781 __ vmovdrr(FromLowSToD(destination.AsFpuRegisterPairLow<SRegister>()), 1782 source.AsRegisterPairLow<Register>(), 1783 source.AsRegisterPairHigh<Register>()); 1784 } else { 1785 UNIMPLEMENTED(FATAL); 1786 } 1787 } else { 1788 DCHECK(destination.IsDoubleStackSlot()); 1789 if (source.IsRegisterPair()) { 1790 // No conflict possible, so just do the moves. 1791 if (source.AsRegisterPairLow<Register>() == R1) { 1792 DCHECK_EQ(source.AsRegisterPairHigh<Register>(), R2); 1793 __ StoreToOffset(kStoreWord, R1, SP, destination.GetStackIndex()); 1794 __ StoreToOffset(kStoreWord, R2, SP, destination.GetHighStackIndex(kArmWordSize)); 1795 } else { 1796 __ StoreToOffset(kStoreWordPair, source.AsRegisterPairLow<Register>(), 1797 SP, destination.GetStackIndex()); 1798 } 1799 } else if (source.IsFpuRegisterPair()) { 1800 __ StoreDToOffset(FromLowSToD(source.AsFpuRegisterPairLow<SRegister>()), 1801 SP, 1802 destination.GetStackIndex()); 1803 } else { 1804 DCHECK(source.IsDoubleStackSlot()); 1805 EmitParallelMoves( 1806 Location::StackSlot(source.GetStackIndex()), 1807 Location::StackSlot(destination.GetStackIndex()), 1808 Primitive::kPrimInt, 1809 Location::StackSlot(source.GetHighStackIndex(kArmWordSize)), 1810 Location::StackSlot(destination.GetHighStackIndex(kArmWordSize)), 1811 Primitive::kPrimInt); 1812 } 1813 } 1814} 1815 1816void CodeGeneratorARM::MoveConstant(Location location, int32_t value) { 1817 DCHECK(location.IsRegister()); 1818 __ LoadImmediate(location.AsRegister<Register>(), value); 1819} 1820 1821void CodeGeneratorARM::MoveLocation(Location dst, Location src, Primitive::Type dst_type) { 1822 HParallelMove move(GetGraph()->GetArena()); 1823 move.AddMove(src, dst, dst_type, nullptr); 1824 GetMoveResolver()->EmitNativeCode(&move); 1825} 1826 1827void CodeGeneratorARM::AddLocationAsTemp(Location location, LocationSummary* locations) { 1828 if (location.IsRegister()) { 1829 locations->AddTemp(location); 1830 } else if (location.IsRegisterPair()) { 1831 locations->AddTemp(Location::RegisterLocation(location.AsRegisterPairLow<Register>())); 1832 locations->AddTemp(Location::RegisterLocation(location.AsRegisterPairHigh<Register>())); 1833 } else { 1834 UNIMPLEMENTED(FATAL) << "AddLocationAsTemp not implemented for location " << location; 1835 } 1836} 1837 1838void CodeGeneratorARM::InvokeRuntime(QuickEntrypointEnum entrypoint, 1839 HInstruction* instruction, 1840 uint32_t dex_pc, 1841 SlowPathCode* slow_path) { 1842 ValidateInvokeRuntime(entrypoint, instruction, slow_path); 1843 GenerateInvokeRuntime(GetThreadOffset<kArmPointerSize>(entrypoint).Int32Value()); 1844 if (EntrypointRequiresStackMap(entrypoint)) { 1845 RecordPcInfo(instruction, dex_pc, slow_path); 1846 } 1847} 1848 1849void CodeGeneratorARM::InvokeRuntimeWithoutRecordingPcInfo(int32_t entry_point_offset, 1850 HInstruction* instruction, 1851 SlowPathCode* slow_path) { 1852 ValidateInvokeRuntimeWithoutRecordingPcInfo(instruction, slow_path); 1853 GenerateInvokeRuntime(entry_point_offset); 1854} 1855 1856void CodeGeneratorARM::GenerateInvokeRuntime(int32_t entry_point_offset) { 1857 __ LoadFromOffset(kLoadWord, LR, TR, entry_point_offset); 1858 __ blx(LR); 1859} 1860 1861void InstructionCodeGeneratorARM::HandleGoto(HInstruction* got, HBasicBlock* successor) { 1862 DCHECK(!successor->IsExitBlock()); 1863 1864 HBasicBlock* block = got->GetBlock(); 1865 HInstruction* previous = got->GetPrevious(); 1866 1867 HLoopInformation* info = block->GetLoopInformation(); 1868 if (info != nullptr && info->IsBackEdge(*block) && info->HasSuspendCheck()) { 1869 codegen_->ClearSpillSlotsFromLoopPhisInStackMap(info->GetSuspendCheck()); 1870 GenerateSuspendCheck(info->GetSuspendCheck(), successor); 1871 return; 1872 } 1873 1874 if (block->IsEntryBlock() && (previous != nullptr) && previous->IsSuspendCheck()) { 1875 GenerateSuspendCheck(previous->AsSuspendCheck(), nullptr); 1876 } 1877 if (!codegen_->GoesToNextBlock(got->GetBlock(), successor)) { 1878 __ b(codegen_->GetLabelOf(successor)); 1879 } 1880} 1881 1882void LocationsBuilderARM::VisitGoto(HGoto* got) { 1883 got->SetLocations(nullptr); 1884} 1885 1886void InstructionCodeGeneratorARM::VisitGoto(HGoto* got) { 1887 HandleGoto(got, got->GetSuccessor()); 1888} 1889 1890void LocationsBuilderARM::VisitTryBoundary(HTryBoundary* try_boundary) { 1891 try_boundary->SetLocations(nullptr); 1892} 1893 1894void InstructionCodeGeneratorARM::VisitTryBoundary(HTryBoundary* try_boundary) { 1895 HBasicBlock* successor = try_boundary->GetNormalFlowSuccessor(); 1896 if (!successor->IsExitBlock()) { 1897 HandleGoto(try_boundary, successor); 1898 } 1899} 1900 1901void LocationsBuilderARM::VisitExit(HExit* exit) { 1902 exit->SetLocations(nullptr); 1903} 1904 1905void InstructionCodeGeneratorARM::VisitExit(HExit* exit ATTRIBUTE_UNUSED) { 1906} 1907 1908void InstructionCodeGeneratorARM::GenerateVcmp(HInstruction* instruction) { 1909 Primitive::Type type = instruction->InputAt(0)->GetType(); 1910 Location lhs_loc = instruction->GetLocations()->InAt(0); 1911 Location rhs_loc = instruction->GetLocations()->InAt(1); 1912 if (rhs_loc.IsConstant()) { 1913 // 0.0 is the only immediate that can be encoded directly in 1914 // a VCMP instruction. 1915 // 1916 // Both the JLS (section 15.20.1) and the JVMS (section 6.5) 1917 // specify that in a floating-point comparison, positive zero 1918 // and negative zero are considered equal, so we can use the 1919 // literal 0.0 for both cases here. 1920 // 1921 // Note however that some methods (Float.equal, Float.compare, 1922 // Float.compareTo, Double.equal, Double.compare, 1923 // Double.compareTo, Math.max, Math.min, StrictMath.max, 1924 // StrictMath.min) consider 0.0 to be (strictly) greater than 1925 // -0.0. So if we ever translate calls to these methods into a 1926 // HCompare instruction, we must handle the -0.0 case with 1927 // care here. 1928 DCHECK(rhs_loc.GetConstant()->IsArithmeticZero()); 1929 if (type == Primitive::kPrimFloat) { 1930 __ vcmpsz(lhs_loc.AsFpuRegister<SRegister>()); 1931 } else { 1932 DCHECK_EQ(type, Primitive::kPrimDouble); 1933 __ vcmpdz(FromLowSToD(lhs_loc.AsFpuRegisterPairLow<SRegister>())); 1934 } 1935 } else { 1936 if (type == Primitive::kPrimFloat) { 1937 __ vcmps(lhs_loc.AsFpuRegister<SRegister>(), rhs_loc.AsFpuRegister<SRegister>()); 1938 } else { 1939 DCHECK_EQ(type, Primitive::kPrimDouble); 1940 __ vcmpd(FromLowSToD(lhs_loc.AsFpuRegisterPairLow<SRegister>()), 1941 FromLowSToD(rhs_loc.AsFpuRegisterPairLow<SRegister>())); 1942 } 1943 } 1944} 1945 1946void InstructionCodeGeneratorARM::GenerateFPJumps(HCondition* cond, 1947 Label* true_label, 1948 Label* false_label ATTRIBUTE_UNUSED) { 1949 __ vmstat(); // transfer FP status register to ARM APSR. 1950 __ b(true_label, ARMFPCondition(cond->GetCondition(), cond->IsGtBias())); 1951} 1952 1953void InstructionCodeGeneratorARM::GenerateLongComparesAndJumps(HCondition* cond, 1954 Label* true_label, 1955 Label* false_label) { 1956 LocationSummary* locations = cond->GetLocations(); 1957 Location left = locations->InAt(0); 1958 Location right = locations->InAt(1); 1959 IfCondition if_cond = cond->GetCondition(); 1960 1961 Register left_high = left.AsRegisterPairHigh<Register>(); 1962 Register left_low = left.AsRegisterPairLow<Register>(); 1963 IfCondition true_high_cond = if_cond; 1964 IfCondition false_high_cond = cond->GetOppositeCondition(); 1965 Condition final_condition = ARMUnsignedCondition(if_cond); // unsigned on lower part 1966 1967 // Set the conditions for the test, remembering that == needs to be 1968 // decided using the low words. 1969 // TODO: consider avoiding jumps with temporary and CMP low+SBC high 1970 switch (if_cond) { 1971 case kCondEQ: 1972 case kCondNE: 1973 // Nothing to do. 1974 break; 1975 case kCondLT: 1976 false_high_cond = kCondGT; 1977 break; 1978 case kCondLE: 1979 true_high_cond = kCondLT; 1980 break; 1981 case kCondGT: 1982 false_high_cond = kCondLT; 1983 break; 1984 case kCondGE: 1985 true_high_cond = kCondGT; 1986 break; 1987 case kCondB: 1988 false_high_cond = kCondA; 1989 break; 1990 case kCondBE: 1991 true_high_cond = kCondB; 1992 break; 1993 case kCondA: 1994 false_high_cond = kCondB; 1995 break; 1996 case kCondAE: 1997 true_high_cond = kCondA; 1998 break; 1999 } 2000 if (right.IsConstant()) { 2001 int64_t value = right.GetConstant()->AsLongConstant()->GetValue(); 2002 int32_t val_low = Low32Bits(value); 2003 int32_t val_high = High32Bits(value); 2004 2005 __ CmpConstant(left_high, val_high); 2006 if (if_cond == kCondNE) { 2007 __ b(true_label, ARMCondition(true_high_cond)); 2008 } else if (if_cond == kCondEQ) { 2009 __ b(false_label, ARMCondition(false_high_cond)); 2010 } else { 2011 __ b(true_label, ARMCondition(true_high_cond)); 2012 __ b(false_label, ARMCondition(false_high_cond)); 2013 } 2014 // Must be equal high, so compare the lows. 2015 __ CmpConstant(left_low, val_low); 2016 } else { 2017 Register right_high = right.AsRegisterPairHigh<Register>(); 2018 Register right_low = right.AsRegisterPairLow<Register>(); 2019 2020 __ cmp(left_high, ShifterOperand(right_high)); 2021 if (if_cond == kCondNE) { 2022 __ b(true_label, ARMCondition(true_high_cond)); 2023 } else if (if_cond == kCondEQ) { 2024 __ b(false_label, ARMCondition(false_high_cond)); 2025 } else { 2026 __ b(true_label, ARMCondition(true_high_cond)); 2027 __ b(false_label, ARMCondition(false_high_cond)); 2028 } 2029 // Must be equal high, so compare the lows. 2030 __ cmp(left_low, ShifterOperand(right_low)); 2031 } 2032 // The last comparison might be unsigned. 2033 // TODO: optimize cases where this is always true/false 2034 __ b(true_label, final_condition); 2035} 2036 2037void InstructionCodeGeneratorARM::GenerateCompareTestAndBranch(HCondition* condition, 2038 Label* true_target_in, 2039 Label* false_target_in) { 2040 // Generated branching requires both targets to be explicit. If either of the 2041 // targets is nullptr (fallthrough) use and bind `fallthrough_target` instead. 2042 Label fallthrough_target; 2043 Label* true_target = true_target_in == nullptr ? &fallthrough_target : true_target_in; 2044 Label* false_target = false_target_in == nullptr ? &fallthrough_target : false_target_in; 2045 2046 Primitive::Type type = condition->InputAt(0)->GetType(); 2047 switch (type) { 2048 case Primitive::kPrimLong: 2049 GenerateLongComparesAndJumps(condition, true_target, false_target); 2050 break; 2051 case Primitive::kPrimFloat: 2052 case Primitive::kPrimDouble: 2053 GenerateVcmp(condition); 2054 GenerateFPJumps(condition, true_target, false_target); 2055 break; 2056 default: 2057 LOG(FATAL) << "Unexpected compare type " << type; 2058 } 2059 2060 if (false_target != &fallthrough_target) { 2061 __ b(false_target); 2062 } 2063 2064 if (fallthrough_target.IsLinked()) { 2065 __ Bind(&fallthrough_target); 2066 } 2067} 2068 2069void InstructionCodeGeneratorARM::GenerateTestAndBranch(HInstruction* instruction, 2070 size_t condition_input_index, 2071 Label* true_target, 2072 Label* false_target) { 2073 HInstruction* cond = instruction->InputAt(condition_input_index); 2074 2075 if (true_target == nullptr && false_target == nullptr) { 2076 // Nothing to do. The code always falls through. 2077 return; 2078 } else if (cond->IsIntConstant()) { 2079 // Constant condition, statically compared against "true" (integer value 1). 2080 if (cond->AsIntConstant()->IsTrue()) { 2081 if (true_target != nullptr) { 2082 __ b(true_target); 2083 } 2084 } else { 2085 DCHECK(cond->AsIntConstant()->IsFalse()) << cond->AsIntConstant()->GetValue(); 2086 if (false_target != nullptr) { 2087 __ b(false_target); 2088 } 2089 } 2090 return; 2091 } 2092 2093 // The following code generates these patterns: 2094 // (1) true_target == nullptr && false_target != nullptr 2095 // - opposite condition true => branch to false_target 2096 // (2) true_target != nullptr && false_target == nullptr 2097 // - condition true => branch to true_target 2098 // (3) true_target != nullptr && false_target != nullptr 2099 // - condition true => branch to true_target 2100 // - branch to false_target 2101 if (IsBooleanValueOrMaterializedCondition(cond)) { 2102 // Condition has been materialized, compare the output to 0. 2103 Location cond_val = instruction->GetLocations()->InAt(condition_input_index); 2104 DCHECK(cond_val.IsRegister()); 2105 if (true_target == nullptr) { 2106 __ CompareAndBranchIfZero(cond_val.AsRegister<Register>(), false_target); 2107 } else { 2108 __ CompareAndBranchIfNonZero(cond_val.AsRegister<Register>(), true_target); 2109 } 2110 } else { 2111 // Condition has not been materialized. Use its inputs as the comparison and 2112 // its condition as the branch condition. 2113 HCondition* condition = cond->AsCondition(); 2114 2115 // If this is a long or FP comparison that has been folded into 2116 // the HCondition, generate the comparison directly. 2117 Primitive::Type type = condition->InputAt(0)->GetType(); 2118 if (type == Primitive::kPrimLong || Primitive::IsFloatingPointType(type)) { 2119 GenerateCompareTestAndBranch(condition, true_target, false_target); 2120 return; 2121 } 2122 2123 LocationSummary* locations = cond->GetLocations(); 2124 DCHECK(locations->InAt(0).IsRegister()); 2125 Register left = locations->InAt(0).AsRegister<Register>(); 2126 Location right = locations->InAt(1); 2127 if (right.IsRegister()) { 2128 __ cmp(left, ShifterOperand(right.AsRegister<Register>())); 2129 } else { 2130 DCHECK(right.IsConstant()); 2131 __ CmpConstant(left, CodeGenerator::GetInt32ValueOf(right.GetConstant())); 2132 } 2133 if (true_target == nullptr) { 2134 __ b(false_target, ARMCondition(condition->GetOppositeCondition())); 2135 } else { 2136 __ b(true_target, ARMCondition(condition->GetCondition())); 2137 } 2138 } 2139 2140 // If neither branch falls through (case 3), the conditional branch to `true_target` 2141 // was already emitted (case 2) and we need to emit a jump to `false_target`. 2142 if (true_target != nullptr && false_target != nullptr) { 2143 __ b(false_target); 2144 } 2145} 2146 2147void LocationsBuilderARM::VisitIf(HIf* if_instr) { 2148 LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(if_instr); 2149 if (IsBooleanValueOrMaterializedCondition(if_instr->InputAt(0))) { 2150 locations->SetInAt(0, Location::RequiresRegister()); 2151 } 2152} 2153 2154void InstructionCodeGeneratorARM::VisitIf(HIf* if_instr) { 2155 HBasicBlock* true_successor = if_instr->IfTrueSuccessor(); 2156 HBasicBlock* false_successor = if_instr->IfFalseSuccessor(); 2157 Label* true_target = codegen_->GoesToNextBlock(if_instr->GetBlock(), true_successor) ? 2158 nullptr : codegen_->GetLabelOf(true_successor); 2159 Label* false_target = codegen_->GoesToNextBlock(if_instr->GetBlock(), false_successor) ? 2160 nullptr : codegen_->GetLabelOf(false_successor); 2161 GenerateTestAndBranch(if_instr, /* condition_input_index */ 0, true_target, false_target); 2162} 2163 2164void LocationsBuilderARM::VisitDeoptimize(HDeoptimize* deoptimize) { 2165 LocationSummary* locations = new (GetGraph()->GetArena()) 2166 LocationSummary(deoptimize, LocationSummary::kCallOnSlowPath); 2167 locations->SetCustomSlowPathCallerSaves(RegisterSet::Empty()); // No caller-save registers. 2168 if (IsBooleanValueOrMaterializedCondition(deoptimize->InputAt(0))) { 2169 locations->SetInAt(0, Location::RequiresRegister()); 2170 } 2171} 2172 2173void InstructionCodeGeneratorARM::VisitDeoptimize(HDeoptimize* deoptimize) { 2174 SlowPathCodeARM* slow_path = deopt_slow_paths_.NewSlowPath<DeoptimizationSlowPathARM>(deoptimize); 2175 GenerateTestAndBranch(deoptimize, 2176 /* condition_input_index */ 0, 2177 slow_path->GetEntryLabel(), 2178 /* false_target */ nullptr); 2179} 2180 2181void LocationsBuilderARM::VisitShouldDeoptimizeFlag(HShouldDeoptimizeFlag* flag) { 2182 LocationSummary* locations = new (GetGraph()->GetArena()) 2183 LocationSummary(flag, LocationSummary::kNoCall); 2184 locations->SetOut(Location::RequiresRegister()); 2185} 2186 2187void InstructionCodeGeneratorARM::VisitShouldDeoptimizeFlag(HShouldDeoptimizeFlag* flag) { 2188 __ LoadFromOffset(kLoadWord, 2189 flag->GetLocations()->Out().AsRegister<Register>(), 2190 SP, 2191 codegen_->GetStackOffsetOfShouldDeoptimizeFlag()); 2192} 2193 2194void LocationsBuilderARM::VisitSelect(HSelect* select) { 2195 LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(select); 2196 if (Primitive::IsFloatingPointType(select->GetType())) { 2197 locations->SetInAt(0, Location::RequiresFpuRegister()); 2198 locations->SetInAt(1, Location::RequiresFpuRegister()); 2199 } else { 2200 locations->SetInAt(0, Location::RequiresRegister()); 2201 locations->SetInAt(1, Location::RequiresRegister()); 2202 } 2203 if (IsBooleanValueOrMaterializedCondition(select->GetCondition())) { 2204 locations->SetInAt(2, Location::RequiresRegister()); 2205 } 2206 locations->SetOut(Location::SameAsFirstInput()); 2207} 2208 2209void InstructionCodeGeneratorARM::VisitSelect(HSelect* select) { 2210 LocationSummary* locations = select->GetLocations(); 2211 Label false_target; 2212 GenerateTestAndBranch(select, 2213 /* condition_input_index */ 2, 2214 /* true_target */ nullptr, 2215 &false_target); 2216 codegen_->MoveLocation(locations->Out(), locations->InAt(1), select->GetType()); 2217 __ Bind(&false_target); 2218} 2219 2220void LocationsBuilderARM::VisitNativeDebugInfo(HNativeDebugInfo* info) { 2221 new (GetGraph()->GetArena()) LocationSummary(info); 2222} 2223 2224void InstructionCodeGeneratorARM::VisitNativeDebugInfo(HNativeDebugInfo*) { 2225 // MaybeRecordNativeDebugInfo is already called implicitly in CodeGenerator::Compile. 2226} 2227 2228void CodeGeneratorARM::GenerateNop() { 2229 __ nop(); 2230} 2231 2232void LocationsBuilderARM::HandleCondition(HCondition* cond) { 2233 LocationSummary* locations = 2234 new (GetGraph()->GetArena()) LocationSummary(cond, LocationSummary::kNoCall); 2235 // Handle the long/FP comparisons made in instruction simplification. 2236 switch (cond->InputAt(0)->GetType()) { 2237 case Primitive::kPrimLong: 2238 locations->SetInAt(0, Location::RequiresRegister()); 2239 locations->SetInAt(1, Location::RegisterOrConstant(cond->InputAt(1))); 2240 if (!cond->IsEmittedAtUseSite()) { 2241 locations->SetOut(Location::RequiresRegister(), Location::kOutputOverlap); 2242 } 2243 break; 2244 2245 case Primitive::kPrimFloat: 2246 case Primitive::kPrimDouble: 2247 locations->SetInAt(0, Location::RequiresFpuRegister()); 2248 locations->SetInAt(1, ArithmeticZeroOrFpuRegister(cond->InputAt(1))); 2249 if (!cond->IsEmittedAtUseSite()) { 2250 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 2251 } 2252 break; 2253 2254 default: 2255 locations->SetInAt(0, Location::RequiresRegister()); 2256 locations->SetInAt(1, Location::RegisterOrConstant(cond->InputAt(1))); 2257 if (!cond->IsEmittedAtUseSite()) { 2258 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 2259 } 2260 } 2261} 2262 2263void InstructionCodeGeneratorARM::HandleCondition(HCondition* cond) { 2264 if (cond->IsEmittedAtUseSite()) { 2265 return; 2266 } 2267 2268 LocationSummary* locations = cond->GetLocations(); 2269 Location left = locations->InAt(0); 2270 Location right = locations->InAt(1); 2271 Register out = locations->Out().AsRegister<Register>(); 2272 Label true_label, false_label; 2273 2274 switch (cond->InputAt(0)->GetType()) { 2275 default: { 2276 // Integer case. 2277 if (right.IsRegister()) { 2278 __ cmp(left.AsRegister<Register>(), ShifterOperand(right.AsRegister<Register>())); 2279 } else { 2280 DCHECK(right.IsConstant()); 2281 __ CmpConstant(left.AsRegister<Register>(), 2282 CodeGenerator::GetInt32ValueOf(right.GetConstant())); 2283 } 2284 __ it(ARMCondition(cond->GetCondition()), kItElse); 2285 __ mov(locations->Out().AsRegister<Register>(), ShifterOperand(1), 2286 ARMCondition(cond->GetCondition())); 2287 __ mov(locations->Out().AsRegister<Register>(), ShifterOperand(0), 2288 ARMCondition(cond->GetOppositeCondition())); 2289 return; 2290 } 2291 case Primitive::kPrimLong: 2292 GenerateLongComparesAndJumps(cond, &true_label, &false_label); 2293 break; 2294 case Primitive::kPrimFloat: 2295 case Primitive::kPrimDouble: 2296 GenerateVcmp(cond); 2297 GenerateFPJumps(cond, &true_label, &false_label); 2298 break; 2299 } 2300 2301 // Convert the jumps into the result. 2302 Label done_label; 2303 2304 // False case: result = 0. 2305 __ Bind(&false_label); 2306 __ LoadImmediate(out, 0); 2307 __ b(&done_label); 2308 2309 // True case: result = 1. 2310 __ Bind(&true_label); 2311 __ LoadImmediate(out, 1); 2312 __ Bind(&done_label); 2313} 2314 2315void LocationsBuilderARM::VisitEqual(HEqual* comp) { 2316 HandleCondition(comp); 2317} 2318 2319void InstructionCodeGeneratorARM::VisitEqual(HEqual* comp) { 2320 HandleCondition(comp); 2321} 2322 2323void LocationsBuilderARM::VisitNotEqual(HNotEqual* comp) { 2324 HandleCondition(comp); 2325} 2326 2327void InstructionCodeGeneratorARM::VisitNotEqual(HNotEqual* comp) { 2328 HandleCondition(comp); 2329} 2330 2331void LocationsBuilderARM::VisitLessThan(HLessThan* comp) { 2332 HandleCondition(comp); 2333} 2334 2335void InstructionCodeGeneratorARM::VisitLessThan(HLessThan* comp) { 2336 HandleCondition(comp); 2337} 2338 2339void LocationsBuilderARM::VisitLessThanOrEqual(HLessThanOrEqual* comp) { 2340 HandleCondition(comp); 2341} 2342 2343void InstructionCodeGeneratorARM::VisitLessThanOrEqual(HLessThanOrEqual* comp) { 2344 HandleCondition(comp); 2345} 2346 2347void LocationsBuilderARM::VisitGreaterThan(HGreaterThan* comp) { 2348 HandleCondition(comp); 2349} 2350 2351void InstructionCodeGeneratorARM::VisitGreaterThan(HGreaterThan* comp) { 2352 HandleCondition(comp); 2353} 2354 2355void LocationsBuilderARM::VisitGreaterThanOrEqual(HGreaterThanOrEqual* comp) { 2356 HandleCondition(comp); 2357} 2358 2359void InstructionCodeGeneratorARM::VisitGreaterThanOrEqual(HGreaterThanOrEqual* comp) { 2360 HandleCondition(comp); 2361} 2362 2363void LocationsBuilderARM::VisitBelow(HBelow* comp) { 2364 HandleCondition(comp); 2365} 2366 2367void InstructionCodeGeneratorARM::VisitBelow(HBelow* comp) { 2368 HandleCondition(comp); 2369} 2370 2371void LocationsBuilderARM::VisitBelowOrEqual(HBelowOrEqual* comp) { 2372 HandleCondition(comp); 2373} 2374 2375void InstructionCodeGeneratorARM::VisitBelowOrEqual(HBelowOrEqual* comp) { 2376 HandleCondition(comp); 2377} 2378 2379void LocationsBuilderARM::VisitAbove(HAbove* comp) { 2380 HandleCondition(comp); 2381} 2382 2383void InstructionCodeGeneratorARM::VisitAbove(HAbove* comp) { 2384 HandleCondition(comp); 2385} 2386 2387void LocationsBuilderARM::VisitAboveOrEqual(HAboveOrEqual* comp) { 2388 HandleCondition(comp); 2389} 2390 2391void InstructionCodeGeneratorARM::VisitAboveOrEqual(HAboveOrEqual* comp) { 2392 HandleCondition(comp); 2393} 2394 2395void LocationsBuilderARM::VisitIntConstant(HIntConstant* constant) { 2396 LocationSummary* locations = 2397 new (GetGraph()->GetArena()) LocationSummary(constant, LocationSummary::kNoCall); 2398 locations->SetOut(Location::ConstantLocation(constant)); 2399} 2400 2401void InstructionCodeGeneratorARM::VisitIntConstant(HIntConstant* constant ATTRIBUTE_UNUSED) { 2402 // Will be generated at use site. 2403} 2404 2405void LocationsBuilderARM::VisitNullConstant(HNullConstant* constant) { 2406 LocationSummary* locations = 2407 new (GetGraph()->GetArena()) LocationSummary(constant, LocationSummary::kNoCall); 2408 locations->SetOut(Location::ConstantLocation(constant)); 2409} 2410 2411void InstructionCodeGeneratorARM::VisitNullConstant(HNullConstant* constant ATTRIBUTE_UNUSED) { 2412 // Will be generated at use site. 2413} 2414 2415void LocationsBuilderARM::VisitLongConstant(HLongConstant* constant) { 2416 LocationSummary* locations = 2417 new (GetGraph()->GetArena()) LocationSummary(constant, LocationSummary::kNoCall); 2418 locations->SetOut(Location::ConstantLocation(constant)); 2419} 2420 2421void InstructionCodeGeneratorARM::VisitLongConstant(HLongConstant* constant ATTRIBUTE_UNUSED) { 2422 // Will be generated at use site. 2423} 2424 2425void LocationsBuilderARM::VisitFloatConstant(HFloatConstant* constant) { 2426 LocationSummary* locations = 2427 new (GetGraph()->GetArena()) LocationSummary(constant, LocationSummary::kNoCall); 2428 locations->SetOut(Location::ConstantLocation(constant)); 2429} 2430 2431void InstructionCodeGeneratorARM::VisitFloatConstant(HFloatConstant* constant ATTRIBUTE_UNUSED) { 2432 // Will be generated at use site. 2433} 2434 2435void LocationsBuilderARM::VisitDoubleConstant(HDoubleConstant* constant) { 2436 LocationSummary* locations = 2437 new (GetGraph()->GetArena()) LocationSummary(constant, LocationSummary::kNoCall); 2438 locations->SetOut(Location::ConstantLocation(constant)); 2439} 2440 2441void InstructionCodeGeneratorARM::VisitDoubleConstant(HDoubleConstant* constant ATTRIBUTE_UNUSED) { 2442 // Will be generated at use site. 2443} 2444 2445void LocationsBuilderARM::VisitMemoryBarrier(HMemoryBarrier* memory_barrier) { 2446 memory_barrier->SetLocations(nullptr); 2447} 2448 2449void InstructionCodeGeneratorARM::VisitMemoryBarrier(HMemoryBarrier* memory_barrier) { 2450 codegen_->GenerateMemoryBarrier(memory_barrier->GetBarrierKind()); 2451} 2452 2453void LocationsBuilderARM::VisitReturnVoid(HReturnVoid* ret) { 2454 ret->SetLocations(nullptr); 2455} 2456 2457void InstructionCodeGeneratorARM::VisitReturnVoid(HReturnVoid* ret ATTRIBUTE_UNUSED) { 2458 codegen_->GenerateFrameExit(); 2459} 2460 2461void LocationsBuilderARM::VisitReturn(HReturn* ret) { 2462 LocationSummary* locations = 2463 new (GetGraph()->GetArena()) LocationSummary(ret, LocationSummary::kNoCall); 2464 locations->SetInAt(0, parameter_visitor_.GetReturnLocation(ret->InputAt(0)->GetType())); 2465} 2466 2467void InstructionCodeGeneratorARM::VisitReturn(HReturn* ret ATTRIBUTE_UNUSED) { 2468 codegen_->GenerateFrameExit(); 2469} 2470 2471void LocationsBuilderARM::VisitInvokeUnresolved(HInvokeUnresolved* invoke) { 2472 // The trampoline uses the same calling convention as dex calling conventions, 2473 // except instead of loading arg0/r0 with the target Method*, arg0/r0 will contain 2474 // the method_idx. 2475 HandleInvoke(invoke); 2476} 2477 2478void InstructionCodeGeneratorARM::VisitInvokeUnresolved(HInvokeUnresolved* invoke) { 2479 codegen_->GenerateInvokeUnresolvedRuntimeCall(invoke); 2480} 2481 2482void LocationsBuilderARM::VisitInvokeStaticOrDirect(HInvokeStaticOrDirect* invoke) { 2483 // Explicit clinit checks triggered by static invokes must have been pruned by 2484 // art::PrepareForRegisterAllocation. 2485 DCHECK(!invoke->IsStaticWithExplicitClinitCheck()); 2486 2487 IntrinsicLocationsBuilderARM intrinsic(codegen_); 2488 if (intrinsic.TryDispatch(invoke)) { 2489 if (invoke->GetLocations()->CanCall() && invoke->HasPcRelativeDexCache()) { 2490 invoke->GetLocations()->SetInAt(invoke->GetSpecialInputIndex(), Location::Any()); 2491 } 2492 return; 2493 } 2494 2495 HandleInvoke(invoke); 2496 2497 // For PC-relative dex cache the invoke has an extra input, the PC-relative address base. 2498 if (invoke->HasPcRelativeDexCache()) { 2499 invoke->GetLocations()->SetInAt(invoke->GetSpecialInputIndex(), Location::RequiresRegister()); 2500 } 2501} 2502 2503static bool TryGenerateIntrinsicCode(HInvoke* invoke, CodeGeneratorARM* codegen) { 2504 if (invoke->GetLocations()->Intrinsified()) { 2505 IntrinsicCodeGeneratorARM intrinsic(codegen); 2506 intrinsic.Dispatch(invoke); 2507 return true; 2508 } 2509 return false; 2510} 2511 2512void InstructionCodeGeneratorARM::VisitInvokeStaticOrDirect(HInvokeStaticOrDirect* invoke) { 2513 // Explicit clinit checks triggered by static invokes must have been pruned by 2514 // art::PrepareForRegisterAllocation. 2515 DCHECK(!invoke->IsStaticWithExplicitClinitCheck()); 2516 2517 if (TryGenerateIntrinsicCode(invoke, codegen_)) { 2518 return; 2519 } 2520 2521 LocationSummary* locations = invoke->GetLocations(); 2522 codegen_->GenerateStaticOrDirectCall( 2523 invoke, locations->HasTemps() ? locations->GetTemp(0) : Location::NoLocation()); 2524 codegen_->RecordPcInfo(invoke, invoke->GetDexPc()); 2525} 2526 2527void LocationsBuilderARM::HandleInvoke(HInvoke* invoke) { 2528 InvokeDexCallingConventionVisitorARM calling_convention_visitor; 2529 CodeGenerator::CreateCommonInvokeLocationSummary(invoke, &calling_convention_visitor); 2530} 2531 2532void LocationsBuilderARM::VisitInvokeVirtual(HInvokeVirtual* invoke) { 2533 IntrinsicLocationsBuilderARM intrinsic(codegen_); 2534 if (intrinsic.TryDispatch(invoke)) { 2535 return; 2536 } 2537 2538 HandleInvoke(invoke); 2539} 2540 2541void InstructionCodeGeneratorARM::VisitInvokeVirtual(HInvokeVirtual* invoke) { 2542 if (TryGenerateIntrinsicCode(invoke, codegen_)) { 2543 return; 2544 } 2545 2546 codegen_->GenerateVirtualCall(invoke, invoke->GetLocations()->GetTemp(0)); 2547 DCHECK(!codegen_->IsLeafMethod()); 2548 codegen_->RecordPcInfo(invoke, invoke->GetDexPc()); 2549} 2550 2551void LocationsBuilderARM::VisitInvokeInterface(HInvokeInterface* invoke) { 2552 HandleInvoke(invoke); 2553 // Add the hidden argument. 2554 invoke->GetLocations()->AddTemp(Location::RegisterLocation(R12)); 2555} 2556 2557void InstructionCodeGeneratorARM::VisitInvokeInterface(HInvokeInterface* invoke) { 2558 // TODO: b/18116999, our IMTs can miss an IncompatibleClassChangeError. 2559 LocationSummary* locations = invoke->GetLocations(); 2560 Register temp = locations->GetTemp(0).AsRegister<Register>(); 2561 Register hidden_reg = locations->GetTemp(1).AsRegister<Register>(); 2562 Location receiver = locations->InAt(0); 2563 uint32_t class_offset = mirror::Object::ClassOffset().Int32Value(); 2564 2565 // Set the hidden argument. This is safe to do this here, as R12 2566 // won't be modified thereafter, before the `blx` (call) instruction. 2567 DCHECK_EQ(R12, hidden_reg); 2568 __ LoadImmediate(hidden_reg, invoke->GetDexMethodIndex()); 2569 2570 if (receiver.IsStackSlot()) { 2571 __ LoadFromOffset(kLoadWord, temp, SP, receiver.GetStackIndex()); 2572 // /* HeapReference<Class> */ temp = temp->klass_ 2573 __ LoadFromOffset(kLoadWord, temp, temp, class_offset); 2574 } else { 2575 // /* HeapReference<Class> */ temp = receiver->klass_ 2576 __ LoadFromOffset(kLoadWord, temp, receiver.AsRegister<Register>(), class_offset); 2577 } 2578 codegen_->MaybeRecordImplicitNullCheck(invoke); 2579 // Instead of simply (possibly) unpoisoning `temp` here, we should 2580 // emit a read barrier for the previous class reference load. 2581 // However this is not required in practice, as this is an 2582 // intermediate/temporary reference and because the current 2583 // concurrent copying collector keeps the from-space memory 2584 // intact/accessible until the end of the marking phase (the 2585 // concurrent copying collector may not in the future). 2586 __ MaybeUnpoisonHeapReference(temp); 2587 __ LoadFromOffset(kLoadWord, temp, temp, 2588 mirror::Class::ImtPtrOffset(kArmPointerSize).Uint32Value()); 2589 uint32_t method_offset = static_cast<uint32_t>(ImTable::OffsetOfElement( 2590 invoke->GetImtIndex(), kArmPointerSize)); 2591 // temp = temp->GetImtEntryAt(method_offset); 2592 __ LoadFromOffset(kLoadWord, temp, temp, method_offset); 2593 uint32_t entry_point = 2594 ArtMethod::EntryPointFromQuickCompiledCodeOffset(kArmPointerSize).Int32Value(); 2595 // LR = temp->GetEntryPoint(); 2596 __ LoadFromOffset(kLoadWord, LR, temp, entry_point); 2597 // LR(); 2598 __ blx(LR); 2599 DCHECK(!codegen_->IsLeafMethod()); 2600 codegen_->RecordPcInfo(invoke, invoke->GetDexPc()); 2601} 2602 2603void LocationsBuilderARM::VisitInvokePolymorphic(HInvokePolymorphic* invoke) { 2604 HandleInvoke(invoke); 2605} 2606 2607void InstructionCodeGeneratorARM::VisitInvokePolymorphic(HInvokePolymorphic* invoke) { 2608 codegen_->GenerateInvokePolymorphicCall(invoke); 2609} 2610 2611void LocationsBuilderARM::VisitNeg(HNeg* neg) { 2612 LocationSummary* locations = 2613 new (GetGraph()->GetArena()) LocationSummary(neg, LocationSummary::kNoCall); 2614 switch (neg->GetResultType()) { 2615 case Primitive::kPrimInt: { 2616 locations->SetInAt(0, Location::RequiresRegister()); 2617 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 2618 break; 2619 } 2620 case Primitive::kPrimLong: { 2621 locations->SetInAt(0, Location::RequiresRegister()); 2622 locations->SetOut(Location::RequiresRegister(), Location::kOutputOverlap); 2623 break; 2624 } 2625 2626 case Primitive::kPrimFloat: 2627 case Primitive::kPrimDouble: 2628 locations->SetInAt(0, Location::RequiresFpuRegister()); 2629 locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap); 2630 break; 2631 2632 default: 2633 LOG(FATAL) << "Unexpected neg type " << neg->GetResultType(); 2634 } 2635} 2636 2637void InstructionCodeGeneratorARM::VisitNeg(HNeg* neg) { 2638 LocationSummary* locations = neg->GetLocations(); 2639 Location out = locations->Out(); 2640 Location in = locations->InAt(0); 2641 switch (neg->GetResultType()) { 2642 case Primitive::kPrimInt: 2643 DCHECK(in.IsRegister()); 2644 __ rsb(out.AsRegister<Register>(), in.AsRegister<Register>(), ShifterOperand(0)); 2645 break; 2646 2647 case Primitive::kPrimLong: 2648 DCHECK(in.IsRegisterPair()); 2649 // out.lo = 0 - in.lo (and update the carry/borrow (C) flag) 2650 __ rsbs(out.AsRegisterPairLow<Register>(), 2651 in.AsRegisterPairLow<Register>(), 2652 ShifterOperand(0)); 2653 // We cannot emit an RSC (Reverse Subtract with Carry) 2654 // instruction here, as it does not exist in the Thumb-2 2655 // instruction set. We use the following approach 2656 // using SBC and SUB instead. 2657 // 2658 // out.hi = -C 2659 __ sbc(out.AsRegisterPairHigh<Register>(), 2660 out.AsRegisterPairHigh<Register>(), 2661 ShifterOperand(out.AsRegisterPairHigh<Register>())); 2662 // out.hi = out.hi - in.hi 2663 __ sub(out.AsRegisterPairHigh<Register>(), 2664 out.AsRegisterPairHigh<Register>(), 2665 ShifterOperand(in.AsRegisterPairHigh<Register>())); 2666 break; 2667 2668 case Primitive::kPrimFloat: 2669 DCHECK(in.IsFpuRegister()); 2670 __ vnegs(out.AsFpuRegister<SRegister>(), in.AsFpuRegister<SRegister>()); 2671 break; 2672 2673 case Primitive::kPrimDouble: 2674 DCHECK(in.IsFpuRegisterPair()); 2675 __ vnegd(FromLowSToD(out.AsFpuRegisterPairLow<SRegister>()), 2676 FromLowSToD(in.AsFpuRegisterPairLow<SRegister>())); 2677 break; 2678 2679 default: 2680 LOG(FATAL) << "Unexpected neg type " << neg->GetResultType(); 2681 } 2682} 2683 2684void LocationsBuilderARM::VisitTypeConversion(HTypeConversion* conversion) { 2685 Primitive::Type result_type = conversion->GetResultType(); 2686 Primitive::Type input_type = conversion->GetInputType(); 2687 DCHECK_NE(result_type, input_type); 2688 2689 // The float-to-long, double-to-long and long-to-float type conversions 2690 // rely on a call to the runtime. 2691 LocationSummary::CallKind call_kind = 2692 (((input_type == Primitive::kPrimFloat || input_type == Primitive::kPrimDouble) 2693 && result_type == Primitive::kPrimLong) 2694 || (input_type == Primitive::kPrimLong && result_type == Primitive::kPrimFloat)) 2695 ? LocationSummary::kCallOnMainOnly 2696 : LocationSummary::kNoCall; 2697 LocationSummary* locations = 2698 new (GetGraph()->GetArena()) LocationSummary(conversion, call_kind); 2699 2700 // The Java language does not allow treating boolean as an integral type but 2701 // our bit representation makes it safe. 2702 2703 switch (result_type) { 2704 case Primitive::kPrimByte: 2705 switch (input_type) { 2706 case Primitive::kPrimLong: 2707 // Type conversion from long to byte is a result of code transformations. 2708 case Primitive::kPrimBoolean: 2709 // Boolean input is a result of code transformations. 2710 case Primitive::kPrimShort: 2711 case Primitive::kPrimInt: 2712 case Primitive::kPrimChar: 2713 // Processing a Dex `int-to-byte' instruction. 2714 locations->SetInAt(0, Location::RequiresRegister()); 2715 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 2716 break; 2717 2718 default: 2719 LOG(FATAL) << "Unexpected type conversion from " << input_type 2720 << " to " << result_type; 2721 } 2722 break; 2723 2724 case Primitive::kPrimShort: 2725 switch (input_type) { 2726 case Primitive::kPrimLong: 2727 // Type conversion from long to short is a result of code transformations. 2728 case Primitive::kPrimBoolean: 2729 // Boolean input is a result of code transformations. 2730 case Primitive::kPrimByte: 2731 case Primitive::kPrimInt: 2732 case Primitive::kPrimChar: 2733 // Processing a Dex `int-to-short' instruction. 2734 locations->SetInAt(0, Location::RequiresRegister()); 2735 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 2736 break; 2737 2738 default: 2739 LOG(FATAL) << "Unexpected type conversion from " << input_type 2740 << " to " << result_type; 2741 } 2742 break; 2743 2744 case Primitive::kPrimInt: 2745 switch (input_type) { 2746 case Primitive::kPrimLong: 2747 // Processing a Dex `long-to-int' instruction. 2748 locations->SetInAt(0, Location::Any()); 2749 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 2750 break; 2751 2752 case Primitive::kPrimFloat: 2753 // Processing a Dex `float-to-int' instruction. 2754 locations->SetInAt(0, Location::RequiresFpuRegister()); 2755 locations->SetOut(Location::RequiresRegister()); 2756 locations->AddTemp(Location::RequiresFpuRegister()); 2757 break; 2758 2759 case Primitive::kPrimDouble: 2760 // Processing a Dex `double-to-int' instruction. 2761 locations->SetInAt(0, Location::RequiresFpuRegister()); 2762 locations->SetOut(Location::RequiresRegister()); 2763 locations->AddTemp(Location::RequiresFpuRegister()); 2764 break; 2765 2766 default: 2767 LOG(FATAL) << "Unexpected type conversion from " << input_type 2768 << " to " << result_type; 2769 } 2770 break; 2771 2772 case Primitive::kPrimLong: 2773 switch (input_type) { 2774 case Primitive::kPrimBoolean: 2775 // Boolean input is a result of code transformations. 2776 case Primitive::kPrimByte: 2777 case Primitive::kPrimShort: 2778 case Primitive::kPrimInt: 2779 case Primitive::kPrimChar: 2780 // Processing a Dex `int-to-long' instruction. 2781 locations->SetInAt(0, Location::RequiresRegister()); 2782 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 2783 break; 2784 2785 case Primitive::kPrimFloat: { 2786 // Processing a Dex `float-to-long' instruction. 2787 InvokeRuntimeCallingConvention calling_convention; 2788 locations->SetInAt(0, Location::FpuRegisterLocation( 2789 calling_convention.GetFpuRegisterAt(0))); 2790 locations->SetOut(Location::RegisterPairLocation(R0, R1)); 2791 break; 2792 } 2793 2794 case Primitive::kPrimDouble: { 2795 // Processing a Dex `double-to-long' instruction. 2796 InvokeRuntimeCallingConvention calling_convention; 2797 locations->SetInAt(0, Location::FpuRegisterPairLocation( 2798 calling_convention.GetFpuRegisterAt(0), 2799 calling_convention.GetFpuRegisterAt(1))); 2800 locations->SetOut(Location::RegisterPairLocation(R0, R1)); 2801 break; 2802 } 2803 2804 default: 2805 LOG(FATAL) << "Unexpected type conversion from " << input_type 2806 << " to " << result_type; 2807 } 2808 break; 2809 2810 case Primitive::kPrimChar: 2811 switch (input_type) { 2812 case Primitive::kPrimLong: 2813 // Type conversion from long to char is a result of code transformations. 2814 case Primitive::kPrimBoolean: 2815 // Boolean input is a result of code transformations. 2816 case Primitive::kPrimByte: 2817 case Primitive::kPrimShort: 2818 case Primitive::kPrimInt: 2819 // Processing a Dex `int-to-char' instruction. 2820 locations->SetInAt(0, Location::RequiresRegister()); 2821 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 2822 break; 2823 2824 default: 2825 LOG(FATAL) << "Unexpected type conversion from " << input_type 2826 << " to " << result_type; 2827 } 2828 break; 2829 2830 case Primitive::kPrimFloat: 2831 switch (input_type) { 2832 case Primitive::kPrimBoolean: 2833 // Boolean input is a result of code transformations. 2834 case Primitive::kPrimByte: 2835 case Primitive::kPrimShort: 2836 case Primitive::kPrimInt: 2837 case Primitive::kPrimChar: 2838 // Processing a Dex `int-to-float' instruction. 2839 locations->SetInAt(0, Location::RequiresRegister()); 2840 locations->SetOut(Location::RequiresFpuRegister()); 2841 break; 2842 2843 case Primitive::kPrimLong: { 2844 // Processing a Dex `long-to-float' instruction. 2845 InvokeRuntimeCallingConvention calling_convention; 2846 locations->SetInAt(0, Location::RegisterPairLocation( 2847 calling_convention.GetRegisterAt(0), calling_convention.GetRegisterAt(1))); 2848 locations->SetOut(Location::FpuRegisterLocation(calling_convention.GetFpuRegisterAt(0))); 2849 break; 2850 } 2851 2852 case Primitive::kPrimDouble: 2853 // Processing a Dex `double-to-float' instruction. 2854 locations->SetInAt(0, Location::RequiresFpuRegister()); 2855 locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap); 2856 break; 2857 2858 default: 2859 LOG(FATAL) << "Unexpected type conversion from " << input_type 2860 << " to " << result_type; 2861 }; 2862 break; 2863 2864 case Primitive::kPrimDouble: 2865 switch (input_type) { 2866 case Primitive::kPrimBoolean: 2867 // Boolean input is a result of code transformations. 2868 case Primitive::kPrimByte: 2869 case Primitive::kPrimShort: 2870 case Primitive::kPrimInt: 2871 case Primitive::kPrimChar: 2872 // Processing a Dex `int-to-double' instruction. 2873 locations->SetInAt(0, Location::RequiresRegister()); 2874 locations->SetOut(Location::RequiresFpuRegister()); 2875 break; 2876 2877 case Primitive::kPrimLong: 2878 // Processing a Dex `long-to-double' instruction. 2879 locations->SetInAt(0, Location::RequiresRegister()); 2880 locations->SetOut(Location::RequiresFpuRegister()); 2881 locations->AddTemp(Location::RequiresFpuRegister()); 2882 locations->AddTemp(Location::RequiresFpuRegister()); 2883 break; 2884 2885 case Primitive::kPrimFloat: 2886 // Processing a Dex `float-to-double' instruction. 2887 locations->SetInAt(0, Location::RequiresFpuRegister()); 2888 locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap); 2889 break; 2890 2891 default: 2892 LOG(FATAL) << "Unexpected type conversion from " << input_type 2893 << " to " << result_type; 2894 }; 2895 break; 2896 2897 default: 2898 LOG(FATAL) << "Unexpected type conversion from " << input_type 2899 << " to " << result_type; 2900 } 2901} 2902 2903void InstructionCodeGeneratorARM::VisitTypeConversion(HTypeConversion* conversion) { 2904 LocationSummary* locations = conversion->GetLocations(); 2905 Location out = locations->Out(); 2906 Location in = locations->InAt(0); 2907 Primitive::Type result_type = conversion->GetResultType(); 2908 Primitive::Type input_type = conversion->GetInputType(); 2909 DCHECK_NE(result_type, input_type); 2910 switch (result_type) { 2911 case Primitive::kPrimByte: 2912 switch (input_type) { 2913 case Primitive::kPrimLong: 2914 // Type conversion from long to byte is a result of code transformations. 2915 __ sbfx(out.AsRegister<Register>(), in.AsRegisterPairLow<Register>(), 0, 8); 2916 break; 2917 case Primitive::kPrimBoolean: 2918 // Boolean input is a result of code transformations. 2919 case Primitive::kPrimShort: 2920 case Primitive::kPrimInt: 2921 case Primitive::kPrimChar: 2922 // Processing a Dex `int-to-byte' instruction. 2923 __ sbfx(out.AsRegister<Register>(), in.AsRegister<Register>(), 0, 8); 2924 break; 2925 2926 default: 2927 LOG(FATAL) << "Unexpected type conversion from " << input_type 2928 << " to " << result_type; 2929 } 2930 break; 2931 2932 case Primitive::kPrimShort: 2933 switch (input_type) { 2934 case Primitive::kPrimLong: 2935 // Type conversion from long to short is a result of code transformations. 2936 __ sbfx(out.AsRegister<Register>(), in.AsRegisterPairLow<Register>(), 0, 16); 2937 break; 2938 case Primitive::kPrimBoolean: 2939 // Boolean input is a result of code transformations. 2940 case Primitive::kPrimByte: 2941 case Primitive::kPrimInt: 2942 case Primitive::kPrimChar: 2943 // Processing a Dex `int-to-short' instruction. 2944 __ sbfx(out.AsRegister<Register>(), in.AsRegister<Register>(), 0, 16); 2945 break; 2946 2947 default: 2948 LOG(FATAL) << "Unexpected type conversion from " << input_type 2949 << " to " << result_type; 2950 } 2951 break; 2952 2953 case Primitive::kPrimInt: 2954 switch (input_type) { 2955 case Primitive::kPrimLong: 2956 // Processing a Dex `long-to-int' instruction. 2957 DCHECK(out.IsRegister()); 2958 if (in.IsRegisterPair()) { 2959 __ Mov(out.AsRegister<Register>(), in.AsRegisterPairLow<Register>()); 2960 } else if (in.IsDoubleStackSlot()) { 2961 __ LoadFromOffset(kLoadWord, out.AsRegister<Register>(), SP, in.GetStackIndex()); 2962 } else { 2963 DCHECK(in.IsConstant()); 2964 DCHECK(in.GetConstant()->IsLongConstant()); 2965 int64_t value = in.GetConstant()->AsLongConstant()->GetValue(); 2966 __ LoadImmediate(out.AsRegister<Register>(), static_cast<int32_t>(value)); 2967 } 2968 break; 2969 2970 case Primitive::kPrimFloat: { 2971 // Processing a Dex `float-to-int' instruction. 2972 SRegister temp = locations->GetTemp(0).AsFpuRegisterPairLow<SRegister>(); 2973 __ vcvtis(temp, in.AsFpuRegister<SRegister>()); 2974 __ vmovrs(out.AsRegister<Register>(), temp); 2975 break; 2976 } 2977 2978 case Primitive::kPrimDouble: { 2979 // Processing a Dex `double-to-int' instruction. 2980 SRegister temp_s = locations->GetTemp(0).AsFpuRegisterPairLow<SRegister>(); 2981 __ vcvtid(temp_s, FromLowSToD(in.AsFpuRegisterPairLow<SRegister>())); 2982 __ vmovrs(out.AsRegister<Register>(), temp_s); 2983 break; 2984 } 2985 2986 default: 2987 LOG(FATAL) << "Unexpected type conversion from " << input_type 2988 << " to " << result_type; 2989 } 2990 break; 2991 2992 case Primitive::kPrimLong: 2993 switch (input_type) { 2994 case Primitive::kPrimBoolean: 2995 // Boolean input is a result of code transformations. 2996 case Primitive::kPrimByte: 2997 case Primitive::kPrimShort: 2998 case Primitive::kPrimInt: 2999 case Primitive::kPrimChar: 3000 // Processing a Dex `int-to-long' instruction. 3001 DCHECK(out.IsRegisterPair()); 3002 DCHECK(in.IsRegister()); 3003 __ Mov(out.AsRegisterPairLow<Register>(), in.AsRegister<Register>()); 3004 // Sign extension. 3005 __ Asr(out.AsRegisterPairHigh<Register>(), 3006 out.AsRegisterPairLow<Register>(), 3007 31); 3008 break; 3009 3010 case Primitive::kPrimFloat: 3011 // Processing a Dex `float-to-long' instruction. 3012 codegen_->InvokeRuntime(kQuickF2l, conversion, conversion->GetDexPc()); 3013 CheckEntrypointTypes<kQuickF2l, int64_t, float>(); 3014 break; 3015 3016 case Primitive::kPrimDouble: 3017 // Processing a Dex `double-to-long' instruction. 3018 codegen_->InvokeRuntime(kQuickD2l, conversion, conversion->GetDexPc()); 3019 CheckEntrypointTypes<kQuickD2l, int64_t, double>(); 3020 break; 3021 3022 default: 3023 LOG(FATAL) << "Unexpected type conversion from " << input_type 3024 << " to " << result_type; 3025 } 3026 break; 3027 3028 case Primitive::kPrimChar: 3029 switch (input_type) { 3030 case Primitive::kPrimLong: 3031 // Type conversion from long to char is a result of code transformations. 3032 __ ubfx(out.AsRegister<Register>(), in.AsRegisterPairLow<Register>(), 0, 16); 3033 break; 3034 case Primitive::kPrimBoolean: 3035 // Boolean input is a result of code transformations. 3036 case Primitive::kPrimByte: 3037 case Primitive::kPrimShort: 3038 case Primitive::kPrimInt: 3039 // Processing a Dex `int-to-char' instruction. 3040 __ ubfx(out.AsRegister<Register>(), in.AsRegister<Register>(), 0, 16); 3041 break; 3042 3043 default: 3044 LOG(FATAL) << "Unexpected type conversion from " << input_type 3045 << " to " << result_type; 3046 } 3047 break; 3048 3049 case Primitive::kPrimFloat: 3050 switch (input_type) { 3051 case Primitive::kPrimBoolean: 3052 // Boolean input is a result of code transformations. 3053 case Primitive::kPrimByte: 3054 case Primitive::kPrimShort: 3055 case Primitive::kPrimInt: 3056 case Primitive::kPrimChar: { 3057 // Processing a Dex `int-to-float' instruction. 3058 __ vmovsr(out.AsFpuRegister<SRegister>(), in.AsRegister<Register>()); 3059 __ vcvtsi(out.AsFpuRegister<SRegister>(), out.AsFpuRegister<SRegister>()); 3060 break; 3061 } 3062 3063 case Primitive::kPrimLong: 3064 // Processing a Dex `long-to-float' instruction. 3065 codegen_->InvokeRuntime(kQuickL2f, conversion, conversion->GetDexPc()); 3066 CheckEntrypointTypes<kQuickL2f, float, int64_t>(); 3067 break; 3068 3069 case Primitive::kPrimDouble: 3070 // Processing a Dex `double-to-float' instruction. 3071 __ vcvtsd(out.AsFpuRegister<SRegister>(), 3072 FromLowSToD(in.AsFpuRegisterPairLow<SRegister>())); 3073 break; 3074 3075 default: 3076 LOG(FATAL) << "Unexpected type conversion from " << input_type 3077 << " to " << result_type; 3078 }; 3079 break; 3080 3081 case Primitive::kPrimDouble: 3082 switch (input_type) { 3083 case Primitive::kPrimBoolean: 3084 // Boolean input is a result of code transformations. 3085 case Primitive::kPrimByte: 3086 case Primitive::kPrimShort: 3087 case Primitive::kPrimInt: 3088 case Primitive::kPrimChar: { 3089 // Processing a Dex `int-to-double' instruction. 3090 __ vmovsr(out.AsFpuRegisterPairLow<SRegister>(), in.AsRegister<Register>()); 3091 __ vcvtdi(FromLowSToD(out.AsFpuRegisterPairLow<SRegister>()), 3092 out.AsFpuRegisterPairLow<SRegister>()); 3093 break; 3094 } 3095 3096 case Primitive::kPrimLong: { 3097 // Processing a Dex `long-to-double' instruction. 3098 Register low = in.AsRegisterPairLow<Register>(); 3099 Register high = in.AsRegisterPairHigh<Register>(); 3100 SRegister out_s = out.AsFpuRegisterPairLow<SRegister>(); 3101 DRegister out_d = FromLowSToD(out_s); 3102 SRegister temp_s = locations->GetTemp(0).AsFpuRegisterPairLow<SRegister>(); 3103 DRegister temp_d = FromLowSToD(temp_s); 3104 SRegister constant_s = locations->GetTemp(1).AsFpuRegisterPairLow<SRegister>(); 3105 DRegister constant_d = FromLowSToD(constant_s); 3106 3107 // temp_d = int-to-double(high) 3108 __ vmovsr(temp_s, high); 3109 __ vcvtdi(temp_d, temp_s); 3110 // constant_d = k2Pow32EncodingForDouble 3111 __ LoadDImmediate(constant_d, bit_cast<double, int64_t>(k2Pow32EncodingForDouble)); 3112 // out_d = unsigned-to-double(low) 3113 __ vmovsr(out_s, low); 3114 __ vcvtdu(out_d, out_s); 3115 // out_d += temp_d * constant_d 3116 __ vmlad(out_d, temp_d, constant_d); 3117 break; 3118 } 3119 3120 case Primitive::kPrimFloat: 3121 // Processing a Dex `float-to-double' instruction. 3122 __ vcvtds(FromLowSToD(out.AsFpuRegisterPairLow<SRegister>()), 3123 in.AsFpuRegister<SRegister>()); 3124 break; 3125 3126 default: 3127 LOG(FATAL) << "Unexpected type conversion from " << input_type 3128 << " to " << result_type; 3129 }; 3130 break; 3131 3132 default: 3133 LOG(FATAL) << "Unexpected type conversion from " << input_type 3134 << " to " << result_type; 3135 } 3136} 3137 3138void LocationsBuilderARM::VisitAdd(HAdd* add) { 3139 LocationSummary* locations = 3140 new (GetGraph()->GetArena()) LocationSummary(add, LocationSummary::kNoCall); 3141 switch (add->GetResultType()) { 3142 case Primitive::kPrimInt: { 3143 locations->SetInAt(0, Location::RequiresRegister()); 3144 locations->SetInAt(1, Location::RegisterOrConstant(add->InputAt(1))); 3145 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 3146 break; 3147 } 3148 3149 case Primitive::kPrimLong: { 3150 locations->SetInAt(0, Location::RequiresRegister()); 3151 locations->SetInAt(1, ArmEncodableConstantOrRegister(add->InputAt(1), ADD)); 3152 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 3153 break; 3154 } 3155 3156 case Primitive::kPrimFloat: 3157 case Primitive::kPrimDouble: { 3158 locations->SetInAt(0, Location::RequiresFpuRegister()); 3159 locations->SetInAt(1, Location::RequiresFpuRegister()); 3160 locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap); 3161 break; 3162 } 3163 3164 default: 3165 LOG(FATAL) << "Unexpected add type " << add->GetResultType(); 3166 } 3167} 3168 3169void InstructionCodeGeneratorARM::VisitAdd(HAdd* add) { 3170 LocationSummary* locations = add->GetLocations(); 3171 Location out = locations->Out(); 3172 Location first = locations->InAt(0); 3173 Location second = locations->InAt(1); 3174 switch (add->GetResultType()) { 3175 case Primitive::kPrimInt: 3176 if (second.IsRegister()) { 3177 __ add(out.AsRegister<Register>(), 3178 first.AsRegister<Register>(), 3179 ShifterOperand(second.AsRegister<Register>())); 3180 } else { 3181 __ AddConstant(out.AsRegister<Register>(), 3182 first.AsRegister<Register>(), 3183 second.GetConstant()->AsIntConstant()->GetValue()); 3184 } 3185 break; 3186 3187 case Primitive::kPrimLong: { 3188 if (second.IsConstant()) { 3189 uint64_t value = static_cast<uint64_t>(Int64FromConstant(second.GetConstant())); 3190 GenerateAddLongConst(out, first, value); 3191 } else { 3192 DCHECK(second.IsRegisterPair()); 3193 __ adds(out.AsRegisterPairLow<Register>(), 3194 first.AsRegisterPairLow<Register>(), 3195 ShifterOperand(second.AsRegisterPairLow<Register>())); 3196 __ adc(out.AsRegisterPairHigh<Register>(), 3197 first.AsRegisterPairHigh<Register>(), 3198 ShifterOperand(second.AsRegisterPairHigh<Register>())); 3199 } 3200 break; 3201 } 3202 3203 case Primitive::kPrimFloat: 3204 __ vadds(out.AsFpuRegister<SRegister>(), 3205 first.AsFpuRegister<SRegister>(), 3206 second.AsFpuRegister<SRegister>()); 3207 break; 3208 3209 case Primitive::kPrimDouble: 3210 __ vaddd(FromLowSToD(out.AsFpuRegisterPairLow<SRegister>()), 3211 FromLowSToD(first.AsFpuRegisterPairLow<SRegister>()), 3212 FromLowSToD(second.AsFpuRegisterPairLow<SRegister>())); 3213 break; 3214 3215 default: 3216 LOG(FATAL) << "Unexpected add type " << add->GetResultType(); 3217 } 3218} 3219 3220void LocationsBuilderARM::VisitSub(HSub* sub) { 3221 LocationSummary* locations = 3222 new (GetGraph()->GetArena()) LocationSummary(sub, LocationSummary::kNoCall); 3223 switch (sub->GetResultType()) { 3224 case Primitive::kPrimInt: { 3225 locations->SetInAt(0, Location::RequiresRegister()); 3226 locations->SetInAt(1, Location::RegisterOrConstant(sub->InputAt(1))); 3227 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 3228 break; 3229 } 3230 3231 case Primitive::kPrimLong: { 3232 locations->SetInAt(0, Location::RequiresRegister()); 3233 locations->SetInAt(1, ArmEncodableConstantOrRegister(sub->InputAt(1), SUB)); 3234 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 3235 break; 3236 } 3237 case Primitive::kPrimFloat: 3238 case Primitive::kPrimDouble: { 3239 locations->SetInAt(0, Location::RequiresFpuRegister()); 3240 locations->SetInAt(1, Location::RequiresFpuRegister()); 3241 locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap); 3242 break; 3243 } 3244 default: 3245 LOG(FATAL) << "Unexpected sub type " << sub->GetResultType(); 3246 } 3247} 3248 3249void InstructionCodeGeneratorARM::VisitSub(HSub* sub) { 3250 LocationSummary* locations = sub->GetLocations(); 3251 Location out = locations->Out(); 3252 Location first = locations->InAt(0); 3253 Location second = locations->InAt(1); 3254 switch (sub->GetResultType()) { 3255 case Primitive::kPrimInt: { 3256 if (second.IsRegister()) { 3257 __ sub(out.AsRegister<Register>(), 3258 first.AsRegister<Register>(), 3259 ShifterOperand(second.AsRegister<Register>())); 3260 } else { 3261 __ AddConstant(out.AsRegister<Register>(), 3262 first.AsRegister<Register>(), 3263 -second.GetConstant()->AsIntConstant()->GetValue()); 3264 } 3265 break; 3266 } 3267 3268 case Primitive::kPrimLong: { 3269 if (second.IsConstant()) { 3270 uint64_t value = static_cast<uint64_t>(Int64FromConstant(second.GetConstant())); 3271 GenerateAddLongConst(out, first, -value); 3272 } else { 3273 DCHECK(second.IsRegisterPair()); 3274 __ subs(out.AsRegisterPairLow<Register>(), 3275 first.AsRegisterPairLow<Register>(), 3276 ShifterOperand(second.AsRegisterPairLow<Register>())); 3277 __ sbc(out.AsRegisterPairHigh<Register>(), 3278 first.AsRegisterPairHigh<Register>(), 3279 ShifterOperand(second.AsRegisterPairHigh<Register>())); 3280 } 3281 break; 3282 } 3283 3284 case Primitive::kPrimFloat: { 3285 __ vsubs(out.AsFpuRegister<SRegister>(), 3286 first.AsFpuRegister<SRegister>(), 3287 second.AsFpuRegister<SRegister>()); 3288 break; 3289 } 3290 3291 case Primitive::kPrimDouble: { 3292 __ vsubd(FromLowSToD(out.AsFpuRegisterPairLow<SRegister>()), 3293 FromLowSToD(first.AsFpuRegisterPairLow<SRegister>()), 3294 FromLowSToD(second.AsFpuRegisterPairLow<SRegister>())); 3295 break; 3296 } 3297 3298 3299 default: 3300 LOG(FATAL) << "Unexpected sub type " << sub->GetResultType(); 3301 } 3302} 3303 3304void LocationsBuilderARM::VisitMul(HMul* mul) { 3305 LocationSummary* locations = 3306 new (GetGraph()->GetArena()) LocationSummary(mul, LocationSummary::kNoCall); 3307 switch (mul->GetResultType()) { 3308 case Primitive::kPrimInt: 3309 case Primitive::kPrimLong: { 3310 locations->SetInAt(0, Location::RequiresRegister()); 3311 locations->SetInAt(1, Location::RequiresRegister()); 3312 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 3313 break; 3314 } 3315 3316 case Primitive::kPrimFloat: 3317 case Primitive::kPrimDouble: { 3318 locations->SetInAt(0, Location::RequiresFpuRegister()); 3319 locations->SetInAt(1, Location::RequiresFpuRegister()); 3320 locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap); 3321 break; 3322 } 3323 3324 default: 3325 LOG(FATAL) << "Unexpected mul type " << mul->GetResultType(); 3326 } 3327} 3328 3329void InstructionCodeGeneratorARM::VisitMul(HMul* mul) { 3330 LocationSummary* locations = mul->GetLocations(); 3331 Location out = locations->Out(); 3332 Location first = locations->InAt(0); 3333 Location second = locations->InAt(1); 3334 switch (mul->GetResultType()) { 3335 case Primitive::kPrimInt: { 3336 __ mul(out.AsRegister<Register>(), 3337 first.AsRegister<Register>(), 3338 second.AsRegister<Register>()); 3339 break; 3340 } 3341 case Primitive::kPrimLong: { 3342 Register out_hi = out.AsRegisterPairHigh<Register>(); 3343 Register out_lo = out.AsRegisterPairLow<Register>(); 3344 Register in1_hi = first.AsRegisterPairHigh<Register>(); 3345 Register in1_lo = first.AsRegisterPairLow<Register>(); 3346 Register in2_hi = second.AsRegisterPairHigh<Register>(); 3347 Register in2_lo = second.AsRegisterPairLow<Register>(); 3348 3349 // Extra checks to protect caused by the existence of R1_R2. 3350 // The algorithm is wrong if out.hi is either in1.lo or in2.lo: 3351 // (e.g. in1=r0_r1, in2=r2_r3 and out=r1_r2); 3352 DCHECK_NE(out_hi, in1_lo); 3353 DCHECK_NE(out_hi, in2_lo); 3354 3355 // input: in1 - 64 bits, in2 - 64 bits 3356 // output: out 3357 // formula: out.hi : out.lo = (in1.lo * in2.hi + in1.hi * in2.lo)* 2^32 + in1.lo * in2.lo 3358 // parts: out.hi = in1.lo * in2.hi + in1.hi * in2.lo + (in1.lo * in2.lo)[63:32] 3359 // parts: out.lo = (in1.lo * in2.lo)[31:0] 3360 3361 // IP <- in1.lo * in2.hi 3362 __ mul(IP, in1_lo, in2_hi); 3363 // out.hi <- in1.lo * in2.hi + in1.hi * in2.lo 3364 __ mla(out_hi, in1_hi, in2_lo, IP); 3365 // out.lo <- (in1.lo * in2.lo)[31:0]; 3366 __ umull(out_lo, IP, in1_lo, in2_lo); 3367 // out.hi <- in2.hi * in1.lo + in2.lo * in1.hi + (in1.lo * in2.lo)[63:32] 3368 __ add(out_hi, out_hi, ShifterOperand(IP)); 3369 break; 3370 } 3371 3372 case Primitive::kPrimFloat: { 3373 __ vmuls(out.AsFpuRegister<SRegister>(), 3374 first.AsFpuRegister<SRegister>(), 3375 second.AsFpuRegister<SRegister>()); 3376 break; 3377 } 3378 3379 case Primitive::kPrimDouble: { 3380 __ vmuld(FromLowSToD(out.AsFpuRegisterPairLow<SRegister>()), 3381 FromLowSToD(first.AsFpuRegisterPairLow<SRegister>()), 3382 FromLowSToD(second.AsFpuRegisterPairLow<SRegister>())); 3383 break; 3384 } 3385 3386 default: 3387 LOG(FATAL) << "Unexpected mul type " << mul->GetResultType(); 3388 } 3389} 3390 3391void InstructionCodeGeneratorARM::DivRemOneOrMinusOne(HBinaryOperation* instruction) { 3392 DCHECK(instruction->IsDiv() || instruction->IsRem()); 3393 DCHECK(instruction->GetResultType() == Primitive::kPrimInt); 3394 3395 LocationSummary* locations = instruction->GetLocations(); 3396 Location second = locations->InAt(1); 3397 DCHECK(second.IsConstant()); 3398 3399 Register out = locations->Out().AsRegister<Register>(); 3400 Register dividend = locations->InAt(0).AsRegister<Register>(); 3401 int32_t imm = second.GetConstant()->AsIntConstant()->GetValue(); 3402 DCHECK(imm == 1 || imm == -1); 3403 3404 if (instruction->IsRem()) { 3405 __ LoadImmediate(out, 0); 3406 } else { 3407 if (imm == 1) { 3408 __ Mov(out, dividend); 3409 } else { 3410 __ rsb(out, dividend, ShifterOperand(0)); 3411 } 3412 } 3413} 3414 3415void InstructionCodeGeneratorARM::DivRemByPowerOfTwo(HBinaryOperation* instruction) { 3416 DCHECK(instruction->IsDiv() || instruction->IsRem()); 3417 DCHECK(instruction->GetResultType() == Primitive::kPrimInt); 3418 3419 LocationSummary* locations = instruction->GetLocations(); 3420 Location second = locations->InAt(1); 3421 DCHECK(second.IsConstant()); 3422 3423 Register out = locations->Out().AsRegister<Register>(); 3424 Register dividend = locations->InAt(0).AsRegister<Register>(); 3425 Register temp = locations->GetTemp(0).AsRegister<Register>(); 3426 int32_t imm = second.GetConstant()->AsIntConstant()->GetValue(); 3427 uint32_t abs_imm = static_cast<uint32_t>(AbsOrMin(imm)); 3428 int ctz_imm = CTZ(abs_imm); 3429 3430 if (ctz_imm == 1) { 3431 __ Lsr(temp, dividend, 32 - ctz_imm); 3432 } else { 3433 __ Asr(temp, dividend, 31); 3434 __ Lsr(temp, temp, 32 - ctz_imm); 3435 } 3436 __ add(out, temp, ShifterOperand(dividend)); 3437 3438 if (instruction->IsDiv()) { 3439 __ Asr(out, out, ctz_imm); 3440 if (imm < 0) { 3441 __ rsb(out, out, ShifterOperand(0)); 3442 } 3443 } else { 3444 __ ubfx(out, out, 0, ctz_imm); 3445 __ sub(out, out, ShifterOperand(temp)); 3446 } 3447} 3448 3449void InstructionCodeGeneratorARM::GenerateDivRemWithAnyConstant(HBinaryOperation* instruction) { 3450 DCHECK(instruction->IsDiv() || instruction->IsRem()); 3451 DCHECK(instruction->GetResultType() == Primitive::kPrimInt); 3452 3453 LocationSummary* locations = instruction->GetLocations(); 3454 Location second = locations->InAt(1); 3455 DCHECK(second.IsConstant()); 3456 3457 Register out = locations->Out().AsRegister<Register>(); 3458 Register dividend = locations->InAt(0).AsRegister<Register>(); 3459 Register temp1 = locations->GetTemp(0).AsRegister<Register>(); 3460 Register temp2 = locations->GetTemp(1).AsRegister<Register>(); 3461 int64_t imm = second.GetConstant()->AsIntConstant()->GetValue(); 3462 3463 int64_t magic; 3464 int shift; 3465 CalculateMagicAndShiftForDivRem(imm, false /* is_long */, &magic, &shift); 3466 3467 __ LoadImmediate(temp1, magic); 3468 __ smull(temp2, temp1, dividend, temp1); 3469 3470 if (imm > 0 && magic < 0) { 3471 __ add(temp1, temp1, ShifterOperand(dividend)); 3472 } else if (imm < 0 && magic > 0) { 3473 __ sub(temp1, temp1, ShifterOperand(dividend)); 3474 } 3475 3476 if (shift != 0) { 3477 __ Asr(temp1, temp1, shift); 3478 } 3479 3480 if (instruction->IsDiv()) { 3481 __ sub(out, temp1, ShifterOperand(temp1, ASR, 31)); 3482 } else { 3483 __ sub(temp1, temp1, ShifterOperand(temp1, ASR, 31)); 3484 // TODO: Strength reduction for mls. 3485 __ LoadImmediate(temp2, imm); 3486 __ mls(out, temp1, temp2, dividend); 3487 } 3488} 3489 3490void InstructionCodeGeneratorARM::GenerateDivRemConstantIntegral(HBinaryOperation* instruction) { 3491 DCHECK(instruction->IsDiv() || instruction->IsRem()); 3492 DCHECK(instruction->GetResultType() == Primitive::kPrimInt); 3493 3494 LocationSummary* locations = instruction->GetLocations(); 3495 Location second = locations->InAt(1); 3496 DCHECK(second.IsConstant()); 3497 3498 int32_t imm = second.GetConstant()->AsIntConstant()->GetValue(); 3499 if (imm == 0) { 3500 // Do not generate anything. DivZeroCheck would prevent any code to be executed. 3501 } else if (imm == 1 || imm == -1) { 3502 DivRemOneOrMinusOne(instruction); 3503 } else if (IsPowerOfTwo(AbsOrMin(imm))) { 3504 DivRemByPowerOfTwo(instruction); 3505 } else { 3506 DCHECK(imm <= -2 || imm >= 2); 3507 GenerateDivRemWithAnyConstant(instruction); 3508 } 3509} 3510 3511void LocationsBuilderARM::VisitDiv(HDiv* div) { 3512 LocationSummary::CallKind call_kind = LocationSummary::kNoCall; 3513 if (div->GetResultType() == Primitive::kPrimLong) { 3514 // pLdiv runtime call. 3515 call_kind = LocationSummary::kCallOnMainOnly; 3516 } else if (div->GetResultType() == Primitive::kPrimInt && div->InputAt(1)->IsConstant()) { 3517 // sdiv will be replaced by other instruction sequence. 3518 } else if (div->GetResultType() == Primitive::kPrimInt && 3519 !codegen_->GetInstructionSetFeatures().HasDivideInstruction()) { 3520 // pIdivmod runtime call. 3521 call_kind = LocationSummary::kCallOnMainOnly; 3522 } 3523 3524 LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(div, call_kind); 3525 3526 switch (div->GetResultType()) { 3527 case Primitive::kPrimInt: { 3528 if (div->InputAt(1)->IsConstant()) { 3529 locations->SetInAt(0, Location::RequiresRegister()); 3530 locations->SetInAt(1, Location::ConstantLocation(div->InputAt(1)->AsConstant())); 3531 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 3532 int32_t value = div->InputAt(1)->AsIntConstant()->GetValue(); 3533 if (value == 1 || value == 0 || value == -1) { 3534 // No temp register required. 3535 } else { 3536 locations->AddTemp(Location::RequiresRegister()); 3537 if (!IsPowerOfTwo(AbsOrMin(value))) { 3538 locations->AddTemp(Location::RequiresRegister()); 3539 } 3540 } 3541 } else if (codegen_->GetInstructionSetFeatures().HasDivideInstruction()) { 3542 locations->SetInAt(0, Location::RequiresRegister()); 3543 locations->SetInAt(1, Location::RequiresRegister()); 3544 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 3545 } else { 3546 InvokeRuntimeCallingConvention calling_convention; 3547 locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(0))); 3548 locations->SetInAt(1, Location::RegisterLocation(calling_convention.GetRegisterAt(1))); 3549 // Note: divmod will compute both the quotient and the remainder as the pair R0 and R1, but 3550 // we only need the former. 3551 locations->SetOut(Location::RegisterLocation(R0)); 3552 } 3553 break; 3554 } 3555 case Primitive::kPrimLong: { 3556 InvokeRuntimeCallingConvention calling_convention; 3557 locations->SetInAt(0, Location::RegisterPairLocation( 3558 calling_convention.GetRegisterAt(0), calling_convention.GetRegisterAt(1))); 3559 locations->SetInAt(1, Location::RegisterPairLocation( 3560 calling_convention.GetRegisterAt(2), calling_convention.GetRegisterAt(3))); 3561 locations->SetOut(Location::RegisterPairLocation(R0, R1)); 3562 break; 3563 } 3564 case Primitive::kPrimFloat: 3565 case Primitive::kPrimDouble: { 3566 locations->SetInAt(0, Location::RequiresFpuRegister()); 3567 locations->SetInAt(1, Location::RequiresFpuRegister()); 3568 locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap); 3569 break; 3570 } 3571 3572 default: 3573 LOG(FATAL) << "Unexpected div type " << div->GetResultType(); 3574 } 3575} 3576 3577void InstructionCodeGeneratorARM::VisitDiv(HDiv* div) { 3578 LocationSummary* locations = div->GetLocations(); 3579 Location out = locations->Out(); 3580 Location first = locations->InAt(0); 3581 Location second = locations->InAt(1); 3582 3583 switch (div->GetResultType()) { 3584 case Primitive::kPrimInt: { 3585 if (second.IsConstant()) { 3586 GenerateDivRemConstantIntegral(div); 3587 } else if (codegen_->GetInstructionSetFeatures().HasDivideInstruction()) { 3588 __ sdiv(out.AsRegister<Register>(), 3589 first.AsRegister<Register>(), 3590 second.AsRegister<Register>()); 3591 } else { 3592 InvokeRuntimeCallingConvention calling_convention; 3593 DCHECK_EQ(calling_convention.GetRegisterAt(0), first.AsRegister<Register>()); 3594 DCHECK_EQ(calling_convention.GetRegisterAt(1), second.AsRegister<Register>()); 3595 DCHECK_EQ(R0, out.AsRegister<Register>()); 3596 3597 codegen_->InvokeRuntime(kQuickIdivmod, div, div->GetDexPc()); 3598 CheckEntrypointTypes<kQuickIdivmod, int32_t, int32_t, int32_t>(); 3599 } 3600 break; 3601 } 3602 3603 case Primitive::kPrimLong: { 3604 InvokeRuntimeCallingConvention calling_convention; 3605 DCHECK_EQ(calling_convention.GetRegisterAt(0), first.AsRegisterPairLow<Register>()); 3606 DCHECK_EQ(calling_convention.GetRegisterAt(1), first.AsRegisterPairHigh<Register>()); 3607 DCHECK_EQ(calling_convention.GetRegisterAt(2), second.AsRegisterPairLow<Register>()); 3608 DCHECK_EQ(calling_convention.GetRegisterAt(3), second.AsRegisterPairHigh<Register>()); 3609 DCHECK_EQ(R0, out.AsRegisterPairLow<Register>()); 3610 DCHECK_EQ(R1, out.AsRegisterPairHigh<Register>()); 3611 3612 codegen_->InvokeRuntime(kQuickLdiv, div, div->GetDexPc()); 3613 CheckEntrypointTypes<kQuickLdiv, int64_t, int64_t, int64_t>(); 3614 break; 3615 } 3616 3617 case Primitive::kPrimFloat: { 3618 __ vdivs(out.AsFpuRegister<SRegister>(), 3619 first.AsFpuRegister<SRegister>(), 3620 second.AsFpuRegister<SRegister>()); 3621 break; 3622 } 3623 3624 case Primitive::kPrimDouble: { 3625 __ vdivd(FromLowSToD(out.AsFpuRegisterPairLow<SRegister>()), 3626 FromLowSToD(first.AsFpuRegisterPairLow<SRegister>()), 3627 FromLowSToD(second.AsFpuRegisterPairLow<SRegister>())); 3628 break; 3629 } 3630 3631 default: 3632 LOG(FATAL) << "Unexpected div type " << div->GetResultType(); 3633 } 3634} 3635 3636void LocationsBuilderARM::VisitRem(HRem* rem) { 3637 Primitive::Type type = rem->GetResultType(); 3638 3639 // Most remainders are implemented in the runtime. 3640 LocationSummary::CallKind call_kind = LocationSummary::kCallOnMainOnly; 3641 if (rem->GetResultType() == Primitive::kPrimInt && rem->InputAt(1)->IsConstant()) { 3642 // sdiv will be replaced by other instruction sequence. 3643 call_kind = LocationSummary::kNoCall; 3644 } else if ((rem->GetResultType() == Primitive::kPrimInt) 3645 && codegen_->GetInstructionSetFeatures().HasDivideInstruction()) { 3646 // Have hardware divide instruction for int, do it with three instructions. 3647 call_kind = LocationSummary::kNoCall; 3648 } 3649 3650 LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(rem, call_kind); 3651 3652 switch (type) { 3653 case Primitive::kPrimInt: { 3654 if (rem->InputAt(1)->IsConstant()) { 3655 locations->SetInAt(0, Location::RequiresRegister()); 3656 locations->SetInAt(1, Location::ConstantLocation(rem->InputAt(1)->AsConstant())); 3657 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 3658 int32_t value = rem->InputAt(1)->AsIntConstant()->GetValue(); 3659 if (value == 1 || value == 0 || value == -1) { 3660 // No temp register required. 3661 } else { 3662 locations->AddTemp(Location::RequiresRegister()); 3663 if (!IsPowerOfTwo(AbsOrMin(value))) { 3664 locations->AddTemp(Location::RequiresRegister()); 3665 } 3666 } 3667 } else if (codegen_->GetInstructionSetFeatures().HasDivideInstruction()) { 3668 locations->SetInAt(0, Location::RequiresRegister()); 3669 locations->SetInAt(1, Location::RequiresRegister()); 3670 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 3671 locations->AddTemp(Location::RequiresRegister()); 3672 } else { 3673 InvokeRuntimeCallingConvention calling_convention; 3674 locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(0))); 3675 locations->SetInAt(1, Location::RegisterLocation(calling_convention.GetRegisterAt(1))); 3676 // Note: divmod will compute both the quotient and the remainder as the pair R0 and R1, but 3677 // we only need the latter. 3678 locations->SetOut(Location::RegisterLocation(R1)); 3679 } 3680 break; 3681 } 3682 case Primitive::kPrimLong: { 3683 InvokeRuntimeCallingConvention calling_convention; 3684 locations->SetInAt(0, Location::RegisterPairLocation( 3685 calling_convention.GetRegisterAt(0), calling_convention.GetRegisterAt(1))); 3686 locations->SetInAt(1, Location::RegisterPairLocation( 3687 calling_convention.GetRegisterAt(2), calling_convention.GetRegisterAt(3))); 3688 // The runtime helper puts the output in R2,R3. 3689 locations->SetOut(Location::RegisterPairLocation(R2, R3)); 3690 break; 3691 } 3692 case Primitive::kPrimFloat: { 3693 InvokeRuntimeCallingConvention calling_convention; 3694 locations->SetInAt(0, Location::FpuRegisterLocation(calling_convention.GetFpuRegisterAt(0))); 3695 locations->SetInAt(1, Location::FpuRegisterLocation(calling_convention.GetFpuRegisterAt(1))); 3696 locations->SetOut(Location::FpuRegisterLocation(S0)); 3697 break; 3698 } 3699 3700 case Primitive::kPrimDouble: { 3701 InvokeRuntimeCallingConvention calling_convention; 3702 locations->SetInAt(0, Location::FpuRegisterPairLocation( 3703 calling_convention.GetFpuRegisterAt(0), calling_convention.GetFpuRegisterAt(1))); 3704 locations->SetInAt(1, Location::FpuRegisterPairLocation( 3705 calling_convention.GetFpuRegisterAt(2), calling_convention.GetFpuRegisterAt(3))); 3706 locations->SetOut(Location::Location::FpuRegisterPairLocation(S0, S1)); 3707 break; 3708 } 3709 3710 default: 3711 LOG(FATAL) << "Unexpected rem type " << type; 3712 } 3713} 3714 3715void InstructionCodeGeneratorARM::VisitRem(HRem* rem) { 3716 LocationSummary* locations = rem->GetLocations(); 3717 Location out = locations->Out(); 3718 Location first = locations->InAt(0); 3719 Location second = locations->InAt(1); 3720 3721 Primitive::Type type = rem->GetResultType(); 3722 switch (type) { 3723 case Primitive::kPrimInt: { 3724 if (second.IsConstant()) { 3725 GenerateDivRemConstantIntegral(rem); 3726 } else if (codegen_->GetInstructionSetFeatures().HasDivideInstruction()) { 3727 Register reg1 = first.AsRegister<Register>(); 3728 Register reg2 = second.AsRegister<Register>(); 3729 Register temp = locations->GetTemp(0).AsRegister<Register>(); 3730 3731 // temp = reg1 / reg2 (integer division) 3732 // dest = reg1 - temp * reg2 3733 __ sdiv(temp, reg1, reg2); 3734 __ mls(out.AsRegister<Register>(), temp, reg2, reg1); 3735 } else { 3736 InvokeRuntimeCallingConvention calling_convention; 3737 DCHECK_EQ(calling_convention.GetRegisterAt(0), first.AsRegister<Register>()); 3738 DCHECK_EQ(calling_convention.GetRegisterAt(1), second.AsRegister<Register>()); 3739 DCHECK_EQ(R1, out.AsRegister<Register>()); 3740 3741 codegen_->InvokeRuntime(kQuickIdivmod, rem, rem->GetDexPc()); 3742 CheckEntrypointTypes<kQuickIdivmod, int32_t, int32_t, int32_t>(); 3743 } 3744 break; 3745 } 3746 3747 case Primitive::kPrimLong: { 3748 codegen_->InvokeRuntime(kQuickLmod, rem, rem->GetDexPc()); 3749 CheckEntrypointTypes<kQuickLmod, int64_t, int64_t, int64_t>(); 3750 break; 3751 } 3752 3753 case Primitive::kPrimFloat: { 3754 codegen_->InvokeRuntime(kQuickFmodf, rem, rem->GetDexPc()); 3755 CheckEntrypointTypes<kQuickFmodf, float, float, float>(); 3756 break; 3757 } 3758 3759 case Primitive::kPrimDouble: { 3760 codegen_->InvokeRuntime(kQuickFmod, rem, rem->GetDexPc()); 3761 CheckEntrypointTypes<kQuickFmod, double, double, double>(); 3762 break; 3763 } 3764 3765 default: 3766 LOG(FATAL) << "Unexpected rem type " << type; 3767 } 3768} 3769 3770void LocationsBuilderARM::VisitDivZeroCheck(HDivZeroCheck* instruction) { 3771 LocationSummary* locations = codegen_->CreateThrowingSlowPathLocations(instruction); 3772 locations->SetInAt(0, Location::RegisterOrConstant(instruction->InputAt(0))); 3773} 3774 3775void InstructionCodeGeneratorARM::VisitDivZeroCheck(HDivZeroCheck* instruction) { 3776 SlowPathCodeARM* slow_path = new (GetGraph()->GetArena()) DivZeroCheckSlowPathARM(instruction); 3777 codegen_->AddSlowPath(slow_path); 3778 3779 LocationSummary* locations = instruction->GetLocations(); 3780 Location value = locations->InAt(0); 3781 3782 switch (instruction->GetType()) { 3783 case Primitive::kPrimBoolean: 3784 case Primitive::kPrimByte: 3785 case Primitive::kPrimChar: 3786 case Primitive::kPrimShort: 3787 case Primitive::kPrimInt: { 3788 if (value.IsRegister()) { 3789 __ CompareAndBranchIfZero(value.AsRegister<Register>(), slow_path->GetEntryLabel()); 3790 } else { 3791 DCHECK(value.IsConstant()) << value; 3792 if (value.GetConstant()->AsIntConstant()->GetValue() == 0) { 3793 __ b(slow_path->GetEntryLabel()); 3794 } 3795 } 3796 break; 3797 } 3798 case Primitive::kPrimLong: { 3799 if (value.IsRegisterPair()) { 3800 __ orrs(IP, 3801 value.AsRegisterPairLow<Register>(), 3802 ShifterOperand(value.AsRegisterPairHigh<Register>())); 3803 __ b(slow_path->GetEntryLabel(), EQ); 3804 } else { 3805 DCHECK(value.IsConstant()) << value; 3806 if (value.GetConstant()->AsLongConstant()->GetValue() == 0) { 3807 __ b(slow_path->GetEntryLabel()); 3808 } 3809 } 3810 break; 3811 default: 3812 LOG(FATAL) << "Unexpected type for HDivZeroCheck " << instruction->GetType(); 3813 } 3814 } 3815} 3816 3817void InstructionCodeGeneratorARM::HandleIntegerRotate(LocationSummary* locations) { 3818 Register in = locations->InAt(0).AsRegister<Register>(); 3819 Location rhs = locations->InAt(1); 3820 Register out = locations->Out().AsRegister<Register>(); 3821 3822 if (rhs.IsConstant()) { 3823 // Arm32 and Thumb2 assemblers require a rotation on the interval [1,31], 3824 // so map all rotations to a +ve. equivalent in that range. 3825 // (e.g. left *or* right by -2 bits == 30 bits in the same direction.) 3826 uint32_t rot = CodeGenerator::GetInt32ValueOf(rhs.GetConstant()) & 0x1F; 3827 if (rot) { 3828 // Rotate, mapping left rotations to right equivalents if necessary. 3829 // (e.g. left by 2 bits == right by 30.) 3830 __ Ror(out, in, rot); 3831 } else if (out != in) { 3832 __ Mov(out, in); 3833 } 3834 } else { 3835 __ Ror(out, in, rhs.AsRegister<Register>()); 3836 } 3837} 3838 3839// Gain some speed by mapping all Long rotates onto equivalent pairs of Integer 3840// rotates by swapping input regs (effectively rotating by the first 32-bits of 3841// a larger rotation) or flipping direction (thus treating larger right/left 3842// rotations as sub-word sized rotations in the other direction) as appropriate. 3843void InstructionCodeGeneratorARM::HandleLongRotate(LocationSummary* locations) { 3844 Register in_reg_lo = locations->InAt(0).AsRegisterPairLow<Register>(); 3845 Register in_reg_hi = locations->InAt(0).AsRegisterPairHigh<Register>(); 3846 Location rhs = locations->InAt(1); 3847 Register out_reg_lo = locations->Out().AsRegisterPairLow<Register>(); 3848 Register out_reg_hi = locations->Out().AsRegisterPairHigh<Register>(); 3849 3850 if (rhs.IsConstant()) { 3851 uint64_t rot = CodeGenerator::GetInt64ValueOf(rhs.GetConstant()); 3852 // Map all rotations to +ve. equivalents on the interval [0,63]. 3853 rot &= kMaxLongShiftDistance; 3854 // For rotates over a word in size, 'pre-rotate' by 32-bits to keep rotate 3855 // logic below to a simple pair of binary orr. 3856 // (e.g. 34 bits == in_reg swap + 2 bits right.) 3857 if (rot >= kArmBitsPerWord) { 3858 rot -= kArmBitsPerWord; 3859 std::swap(in_reg_hi, in_reg_lo); 3860 } 3861 // Rotate, or mov to out for zero or word size rotations. 3862 if (rot != 0u) { 3863 __ Lsr(out_reg_hi, in_reg_hi, rot); 3864 __ orr(out_reg_hi, out_reg_hi, ShifterOperand(in_reg_lo, arm::LSL, kArmBitsPerWord - rot)); 3865 __ Lsr(out_reg_lo, in_reg_lo, rot); 3866 __ orr(out_reg_lo, out_reg_lo, ShifterOperand(in_reg_hi, arm::LSL, kArmBitsPerWord - rot)); 3867 } else { 3868 __ Mov(out_reg_lo, in_reg_lo); 3869 __ Mov(out_reg_hi, in_reg_hi); 3870 } 3871 } else { 3872 Register shift_right = locations->GetTemp(0).AsRegister<Register>(); 3873 Register shift_left = locations->GetTemp(1).AsRegister<Register>(); 3874 Label end; 3875 Label shift_by_32_plus_shift_right; 3876 3877 __ and_(shift_right, rhs.AsRegister<Register>(), ShifterOperand(0x1F)); 3878 __ Lsrs(shift_left, rhs.AsRegister<Register>(), 6); 3879 __ rsb(shift_left, shift_right, ShifterOperand(kArmBitsPerWord), AL, kCcKeep); 3880 __ b(&shift_by_32_plus_shift_right, CC); 3881 3882 // out_reg_hi = (reg_hi << shift_left) | (reg_lo >> shift_right). 3883 // out_reg_lo = (reg_lo << shift_left) | (reg_hi >> shift_right). 3884 __ Lsl(out_reg_hi, in_reg_hi, shift_left); 3885 __ Lsr(out_reg_lo, in_reg_lo, shift_right); 3886 __ add(out_reg_hi, out_reg_hi, ShifterOperand(out_reg_lo)); 3887 __ Lsl(out_reg_lo, in_reg_lo, shift_left); 3888 __ Lsr(shift_left, in_reg_hi, shift_right); 3889 __ add(out_reg_lo, out_reg_lo, ShifterOperand(shift_left)); 3890 __ b(&end); 3891 3892 __ Bind(&shift_by_32_plus_shift_right); // Shift by 32+shift_right. 3893 // out_reg_hi = (reg_hi >> shift_right) | (reg_lo << shift_left). 3894 // out_reg_lo = (reg_lo >> shift_right) | (reg_hi << shift_left). 3895 __ Lsr(out_reg_hi, in_reg_hi, shift_right); 3896 __ Lsl(out_reg_lo, in_reg_lo, shift_left); 3897 __ add(out_reg_hi, out_reg_hi, ShifterOperand(out_reg_lo)); 3898 __ Lsr(out_reg_lo, in_reg_lo, shift_right); 3899 __ Lsl(shift_right, in_reg_hi, shift_left); 3900 __ add(out_reg_lo, out_reg_lo, ShifterOperand(shift_right)); 3901 3902 __ Bind(&end); 3903 } 3904} 3905 3906void LocationsBuilderARM::VisitRor(HRor* ror) { 3907 LocationSummary* locations = 3908 new (GetGraph()->GetArena()) LocationSummary(ror, LocationSummary::kNoCall); 3909 switch (ror->GetResultType()) { 3910 case Primitive::kPrimInt: { 3911 locations->SetInAt(0, Location::RequiresRegister()); 3912 locations->SetInAt(1, Location::RegisterOrConstant(ror->InputAt(1))); 3913 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 3914 break; 3915 } 3916 case Primitive::kPrimLong: { 3917 locations->SetInAt(0, Location::RequiresRegister()); 3918 if (ror->InputAt(1)->IsConstant()) { 3919 locations->SetInAt(1, Location::ConstantLocation(ror->InputAt(1)->AsConstant())); 3920 } else { 3921 locations->SetInAt(1, Location::RequiresRegister()); 3922 locations->AddTemp(Location::RequiresRegister()); 3923 locations->AddTemp(Location::RequiresRegister()); 3924 } 3925 locations->SetOut(Location::RequiresRegister(), Location::kOutputOverlap); 3926 break; 3927 } 3928 default: 3929 LOG(FATAL) << "Unexpected operation type " << ror->GetResultType(); 3930 } 3931} 3932 3933void InstructionCodeGeneratorARM::VisitRor(HRor* ror) { 3934 LocationSummary* locations = ror->GetLocations(); 3935 Primitive::Type type = ror->GetResultType(); 3936 switch (type) { 3937 case Primitive::kPrimInt: { 3938 HandleIntegerRotate(locations); 3939 break; 3940 } 3941 case Primitive::kPrimLong: { 3942 HandleLongRotate(locations); 3943 break; 3944 } 3945 default: 3946 LOG(FATAL) << "Unexpected operation type " << type; 3947 UNREACHABLE(); 3948 } 3949} 3950 3951void LocationsBuilderARM::HandleShift(HBinaryOperation* op) { 3952 DCHECK(op->IsShl() || op->IsShr() || op->IsUShr()); 3953 3954 LocationSummary* locations = 3955 new (GetGraph()->GetArena()) LocationSummary(op, LocationSummary::kNoCall); 3956 3957 switch (op->GetResultType()) { 3958 case Primitive::kPrimInt: { 3959 locations->SetInAt(0, Location::RequiresRegister()); 3960 if (op->InputAt(1)->IsConstant()) { 3961 locations->SetInAt(1, Location::ConstantLocation(op->InputAt(1)->AsConstant())); 3962 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 3963 } else { 3964 locations->SetInAt(1, Location::RequiresRegister()); 3965 // Make the output overlap, as it will be used to hold the masked 3966 // second input. 3967 locations->SetOut(Location::RequiresRegister(), Location::kOutputOverlap); 3968 } 3969 break; 3970 } 3971 case Primitive::kPrimLong: { 3972 locations->SetInAt(0, Location::RequiresRegister()); 3973 if (op->InputAt(1)->IsConstant()) { 3974 locations->SetInAt(1, Location::ConstantLocation(op->InputAt(1)->AsConstant())); 3975 // For simplicity, use kOutputOverlap even though we only require that low registers 3976 // don't clash with high registers which the register allocator currently guarantees. 3977 locations->SetOut(Location::RequiresRegister(), Location::kOutputOverlap); 3978 } else { 3979 locations->SetInAt(1, Location::RequiresRegister()); 3980 locations->AddTemp(Location::RequiresRegister()); 3981 locations->SetOut(Location::RequiresRegister(), Location::kOutputOverlap); 3982 } 3983 break; 3984 } 3985 default: 3986 LOG(FATAL) << "Unexpected operation type " << op->GetResultType(); 3987 } 3988} 3989 3990void InstructionCodeGeneratorARM::HandleShift(HBinaryOperation* op) { 3991 DCHECK(op->IsShl() || op->IsShr() || op->IsUShr()); 3992 3993 LocationSummary* locations = op->GetLocations(); 3994 Location out = locations->Out(); 3995 Location first = locations->InAt(0); 3996 Location second = locations->InAt(1); 3997 3998 Primitive::Type type = op->GetResultType(); 3999 switch (type) { 4000 case Primitive::kPrimInt: { 4001 Register out_reg = out.AsRegister<Register>(); 4002 Register first_reg = first.AsRegister<Register>(); 4003 if (second.IsRegister()) { 4004 Register second_reg = second.AsRegister<Register>(); 4005 // ARM doesn't mask the shift count so we need to do it ourselves. 4006 __ and_(out_reg, second_reg, ShifterOperand(kMaxIntShiftDistance)); 4007 if (op->IsShl()) { 4008 __ Lsl(out_reg, first_reg, out_reg); 4009 } else if (op->IsShr()) { 4010 __ Asr(out_reg, first_reg, out_reg); 4011 } else { 4012 __ Lsr(out_reg, first_reg, out_reg); 4013 } 4014 } else { 4015 int32_t cst = second.GetConstant()->AsIntConstant()->GetValue(); 4016 uint32_t shift_value = cst & kMaxIntShiftDistance; 4017 if (shift_value == 0) { // ARM does not support shifting with 0 immediate. 4018 __ Mov(out_reg, first_reg); 4019 } else if (op->IsShl()) { 4020 __ Lsl(out_reg, first_reg, shift_value); 4021 } else if (op->IsShr()) { 4022 __ Asr(out_reg, first_reg, shift_value); 4023 } else { 4024 __ Lsr(out_reg, first_reg, shift_value); 4025 } 4026 } 4027 break; 4028 } 4029 case Primitive::kPrimLong: { 4030 Register o_h = out.AsRegisterPairHigh<Register>(); 4031 Register o_l = out.AsRegisterPairLow<Register>(); 4032 4033 Register high = first.AsRegisterPairHigh<Register>(); 4034 Register low = first.AsRegisterPairLow<Register>(); 4035 4036 if (second.IsRegister()) { 4037 Register temp = locations->GetTemp(0).AsRegister<Register>(); 4038 4039 Register second_reg = second.AsRegister<Register>(); 4040 4041 if (op->IsShl()) { 4042 __ and_(o_l, second_reg, ShifterOperand(kMaxLongShiftDistance)); 4043 // Shift the high part 4044 __ Lsl(o_h, high, o_l); 4045 // Shift the low part and `or` what overflew on the high part 4046 __ rsb(temp, o_l, ShifterOperand(kArmBitsPerWord)); 4047 __ Lsr(temp, low, temp); 4048 __ orr(o_h, o_h, ShifterOperand(temp)); 4049 // If the shift is > 32 bits, override the high part 4050 __ subs(temp, o_l, ShifterOperand(kArmBitsPerWord)); 4051 __ it(PL); 4052 __ Lsl(o_h, low, temp, PL); 4053 // Shift the low part 4054 __ Lsl(o_l, low, o_l); 4055 } else if (op->IsShr()) { 4056 __ and_(o_h, second_reg, ShifterOperand(kMaxLongShiftDistance)); 4057 // Shift the low part 4058 __ Lsr(o_l, low, o_h); 4059 // Shift the high part and `or` what underflew on the low part 4060 __ rsb(temp, o_h, ShifterOperand(kArmBitsPerWord)); 4061 __ Lsl(temp, high, temp); 4062 __ orr(o_l, o_l, ShifterOperand(temp)); 4063 // If the shift is > 32 bits, override the low part 4064 __ subs(temp, o_h, ShifterOperand(kArmBitsPerWord)); 4065 __ it(PL); 4066 __ Asr(o_l, high, temp, PL); 4067 // Shift the high part 4068 __ Asr(o_h, high, o_h); 4069 } else { 4070 __ and_(o_h, second_reg, ShifterOperand(kMaxLongShiftDistance)); 4071 // same as Shr except we use `Lsr`s and not `Asr`s 4072 __ Lsr(o_l, low, o_h); 4073 __ rsb(temp, o_h, ShifterOperand(kArmBitsPerWord)); 4074 __ Lsl(temp, high, temp); 4075 __ orr(o_l, o_l, ShifterOperand(temp)); 4076 __ subs(temp, o_h, ShifterOperand(kArmBitsPerWord)); 4077 __ it(PL); 4078 __ Lsr(o_l, high, temp, PL); 4079 __ Lsr(o_h, high, o_h); 4080 } 4081 } else { 4082 // Register allocator doesn't create partial overlap. 4083 DCHECK_NE(o_l, high); 4084 DCHECK_NE(o_h, low); 4085 int32_t cst = second.GetConstant()->AsIntConstant()->GetValue(); 4086 uint32_t shift_value = cst & kMaxLongShiftDistance; 4087 if (shift_value > 32) { 4088 if (op->IsShl()) { 4089 __ Lsl(o_h, low, shift_value - 32); 4090 __ LoadImmediate(o_l, 0); 4091 } else if (op->IsShr()) { 4092 __ Asr(o_l, high, shift_value - 32); 4093 __ Asr(o_h, high, 31); 4094 } else { 4095 __ Lsr(o_l, high, shift_value - 32); 4096 __ LoadImmediate(o_h, 0); 4097 } 4098 } else if (shift_value == 32) { 4099 if (op->IsShl()) { 4100 __ mov(o_h, ShifterOperand(low)); 4101 __ LoadImmediate(o_l, 0); 4102 } else if (op->IsShr()) { 4103 __ mov(o_l, ShifterOperand(high)); 4104 __ Asr(o_h, high, 31); 4105 } else { 4106 __ mov(o_l, ShifterOperand(high)); 4107 __ LoadImmediate(o_h, 0); 4108 } 4109 } else if (shift_value == 1) { 4110 if (op->IsShl()) { 4111 __ Lsls(o_l, low, 1); 4112 __ adc(o_h, high, ShifterOperand(high)); 4113 } else if (op->IsShr()) { 4114 __ Asrs(o_h, high, 1); 4115 __ Rrx(o_l, low); 4116 } else { 4117 __ Lsrs(o_h, high, 1); 4118 __ Rrx(o_l, low); 4119 } 4120 } else { 4121 DCHECK(2 <= shift_value && shift_value < 32) << shift_value; 4122 if (op->IsShl()) { 4123 __ Lsl(o_h, high, shift_value); 4124 __ orr(o_h, o_h, ShifterOperand(low, LSR, 32 - shift_value)); 4125 __ Lsl(o_l, low, shift_value); 4126 } else if (op->IsShr()) { 4127 __ Lsr(o_l, low, shift_value); 4128 __ orr(o_l, o_l, ShifterOperand(high, LSL, 32 - shift_value)); 4129 __ Asr(o_h, high, shift_value); 4130 } else { 4131 __ Lsr(o_l, low, shift_value); 4132 __ orr(o_l, o_l, ShifterOperand(high, LSL, 32 - shift_value)); 4133 __ Lsr(o_h, high, shift_value); 4134 } 4135 } 4136 } 4137 break; 4138 } 4139 default: 4140 LOG(FATAL) << "Unexpected operation type " << type; 4141 UNREACHABLE(); 4142 } 4143} 4144 4145void LocationsBuilderARM::VisitShl(HShl* shl) { 4146 HandleShift(shl); 4147} 4148 4149void InstructionCodeGeneratorARM::VisitShl(HShl* shl) { 4150 HandleShift(shl); 4151} 4152 4153void LocationsBuilderARM::VisitShr(HShr* shr) { 4154 HandleShift(shr); 4155} 4156 4157void InstructionCodeGeneratorARM::VisitShr(HShr* shr) { 4158 HandleShift(shr); 4159} 4160 4161void LocationsBuilderARM::VisitUShr(HUShr* ushr) { 4162 HandleShift(ushr); 4163} 4164 4165void InstructionCodeGeneratorARM::VisitUShr(HUShr* ushr) { 4166 HandleShift(ushr); 4167} 4168 4169void LocationsBuilderARM::VisitNewInstance(HNewInstance* instruction) { 4170 LocationSummary* locations = 4171 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCallOnMainOnly); 4172 if (instruction->IsStringAlloc()) { 4173 locations->AddTemp(Location::RegisterLocation(kMethodRegisterArgument)); 4174 } else { 4175 InvokeRuntimeCallingConvention calling_convention; 4176 locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(0))); 4177 } 4178 locations->SetOut(Location::RegisterLocation(R0)); 4179} 4180 4181void InstructionCodeGeneratorARM::VisitNewInstance(HNewInstance* instruction) { 4182 // Note: if heap poisoning is enabled, the entry point takes cares 4183 // of poisoning the reference. 4184 if (instruction->IsStringAlloc()) { 4185 // String is allocated through StringFactory. Call NewEmptyString entry point. 4186 Register temp = instruction->GetLocations()->GetTemp(0).AsRegister<Register>(); 4187 MemberOffset code_offset = ArtMethod::EntryPointFromQuickCompiledCodeOffset(kArmPointerSize); 4188 __ LoadFromOffset(kLoadWord, temp, TR, QUICK_ENTRY_POINT(pNewEmptyString)); 4189 __ LoadFromOffset(kLoadWord, LR, temp, code_offset.Int32Value()); 4190 __ blx(LR); 4191 codegen_->RecordPcInfo(instruction, instruction->GetDexPc()); 4192 } else { 4193 codegen_->InvokeRuntime(instruction->GetEntrypoint(), instruction, instruction->GetDexPc()); 4194 CheckEntrypointTypes<kQuickAllocObjectWithChecks, void*, mirror::Class*>(); 4195 } 4196} 4197 4198void LocationsBuilderARM::VisitNewArray(HNewArray* instruction) { 4199 LocationSummary* locations = 4200 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCallOnMainOnly); 4201 InvokeRuntimeCallingConvention calling_convention; 4202 locations->SetOut(Location::RegisterLocation(R0)); 4203 locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(0))); 4204 locations->SetInAt(1, Location::RegisterLocation(calling_convention.GetRegisterAt(1))); 4205} 4206 4207void InstructionCodeGeneratorARM::VisitNewArray(HNewArray* instruction) { 4208 // Note: if heap poisoning is enabled, the entry point takes cares 4209 // of poisoning the reference. 4210 QuickEntrypointEnum entrypoint = 4211 CodeGenerator::GetArrayAllocationEntrypoint(instruction->GetLoadClass()->GetClass()); 4212 codegen_->InvokeRuntime(entrypoint, instruction, instruction->GetDexPc()); 4213 CheckEntrypointTypes<kQuickAllocArrayResolved, void*, mirror::Class*, int32_t>(); 4214 DCHECK(!codegen_->IsLeafMethod()); 4215} 4216 4217void LocationsBuilderARM::VisitParameterValue(HParameterValue* instruction) { 4218 LocationSummary* locations = 4219 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall); 4220 Location location = parameter_visitor_.GetNextLocation(instruction->GetType()); 4221 if (location.IsStackSlot()) { 4222 location = Location::StackSlot(location.GetStackIndex() + codegen_->GetFrameSize()); 4223 } else if (location.IsDoubleStackSlot()) { 4224 location = Location::DoubleStackSlot(location.GetStackIndex() + codegen_->GetFrameSize()); 4225 } 4226 locations->SetOut(location); 4227} 4228 4229void InstructionCodeGeneratorARM::VisitParameterValue( 4230 HParameterValue* instruction ATTRIBUTE_UNUSED) { 4231 // Nothing to do, the parameter is already at its location. 4232} 4233 4234void LocationsBuilderARM::VisitCurrentMethod(HCurrentMethod* instruction) { 4235 LocationSummary* locations = 4236 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall); 4237 locations->SetOut(Location::RegisterLocation(kMethodRegisterArgument)); 4238} 4239 4240void InstructionCodeGeneratorARM::VisitCurrentMethod(HCurrentMethod* instruction ATTRIBUTE_UNUSED) { 4241 // Nothing to do, the method is already at its location. 4242} 4243 4244void LocationsBuilderARM::VisitNot(HNot* not_) { 4245 LocationSummary* locations = 4246 new (GetGraph()->GetArena()) LocationSummary(not_, LocationSummary::kNoCall); 4247 locations->SetInAt(0, Location::RequiresRegister()); 4248 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 4249} 4250 4251void InstructionCodeGeneratorARM::VisitNot(HNot* not_) { 4252 LocationSummary* locations = not_->GetLocations(); 4253 Location out = locations->Out(); 4254 Location in = locations->InAt(0); 4255 switch (not_->GetResultType()) { 4256 case Primitive::kPrimInt: 4257 __ mvn(out.AsRegister<Register>(), ShifterOperand(in.AsRegister<Register>())); 4258 break; 4259 4260 case Primitive::kPrimLong: 4261 __ mvn(out.AsRegisterPairLow<Register>(), 4262 ShifterOperand(in.AsRegisterPairLow<Register>())); 4263 __ mvn(out.AsRegisterPairHigh<Register>(), 4264 ShifterOperand(in.AsRegisterPairHigh<Register>())); 4265 break; 4266 4267 default: 4268 LOG(FATAL) << "Unimplemented type for not operation " << not_->GetResultType(); 4269 } 4270} 4271 4272void LocationsBuilderARM::VisitBooleanNot(HBooleanNot* bool_not) { 4273 LocationSummary* locations = 4274 new (GetGraph()->GetArena()) LocationSummary(bool_not, LocationSummary::kNoCall); 4275 locations->SetInAt(0, Location::RequiresRegister()); 4276 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 4277} 4278 4279void InstructionCodeGeneratorARM::VisitBooleanNot(HBooleanNot* bool_not) { 4280 LocationSummary* locations = bool_not->GetLocations(); 4281 Location out = locations->Out(); 4282 Location in = locations->InAt(0); 4283 __ eor(out.AsRegister<Register>(), in.AsRegister<Register>(), ShifterOperand(1)); 4284} 4285 4286void LocationsBuilderARM::VisitCompare(HCompare* compare) { 4287 LocationSummary* locations = 4288 new (GetGraph()->GetArena()) LocationSummary(compare, LocationSummary::kNoCall); 4289 switch (compare->InputAt(0)->GetType()) { 4290 case Primitive::kPrimBoolean: 4291 case Primitive::kPrimByte: 4292 case Primitive::kPrimShort: 4293 case Primitive::kPrimChar: 4294 case Primitive::kPrimInt: 4295 case Primitive::kPrimLong: { 4296 locations->SetInAt(0, Location::RequiresRegister()); 4297 locations->SetInAt(1, Location::RequiresRegister()); 4298 // Output overlaps because it is written before doing the low comparison. 4299 locations->SetOut(Location::RequiresRegister(), Location::kOutputOverlap); 4300 break; 4301 } 4302 case Primitive::kPrimFloat: 4303 case Primitive::kPrimDouble: { 4304 locations->SetInAt(0, Location::RequiresFpuRegister()); 4305 locations->SetInAt(1, ArithmeticZeroOrFpuRegister(compare->InputAt(1))); 4306 locations->SetOut(Location::RequiresRegister()); 4307 break; 4308 } 4309 default: 4310 LOG(FATAL) << "Unexpected type for compare operation " << compare->InputAt(0)->GetType(); 4311 } 4312} 4313 4314void InstructionCodeGeneratorARM::VisitCompare(HCompare* compare) { 4315 LocationSummary* locations = compare->GetLocations(); 4316 Register out = locations->Out().AsRegister<Register>(); 4317 Location left = locations->InAt(0); 4318 Location right = locations->InAt(1); 4319 4320 Label less, greater, done; 4321 Primitive::Type type = compare->InputAt(0)->GetType(); 4322 Condition less_cond; 4323 switch (type) { 4324 case Primitive::kPrimBoolean: 4325 case Primitive::kPrimByte: 4326 case Primitive::kPrimShort: 4327 case Primitive::kPrimChar: 4328 case Primitive::kPrimInt: { 4329 __ LoadImmediate(out, 0); 4330 __ cmp(left.AsRegister<Register>(), 4331 ShifterOperand(right.AsRegister<Register>())); // Signed compare. 4332 less_cond = LT; 4333 break; 4334 } 4335 case Primitive::kPrimLong: { 4336 __ cmp(left.AsRegisterPairHigh<Register>(), 4337 ShifterOperand(right.AsRegisterPairHigh<Register>())); // Signed compare. 4338 __ b(&less, LT); 4339 __ b(&greater, GT); 4340 // Do LoadImmediate before the last `cmp`, as LoadImmediate might affect the status flags. 4341 __ LoadImmediate(out, 0); 4342 __ cmp(left.AsRegisterPairLow<Register>(), 4343 ShifterOperand(right.AsRegisterPairLow<Register>())); // Unsigned compare. 4344 less_cond = LO; 4345 break; 4346 } 4347 case Primitive::kPrimFloat: 4348 case Primitive::kPrimDouble: { 4349 __ LoadImmediate(out, 0); 4350 GenerateVcmp(compare); 4351 __ vmstat(); // transfer FP status register to ARM APSR. 4352 less_cond = ARMFPCondition(kCondLT, compare->IsGtBias()); 4353 break; 4354 } 4355 default: 4356 LOG(FATAL) << "Unexpected compare type " << type; 4357 UNREACHABLE(); 4358 } 4359 4360 __ b(&done, EQ); 4361 __ b(&less, less_cond); 4362 4363 __ Bind(&greater); 4364 __ LoadImmediate(out, 1); 4365 __ b(&done); 4366 4367 __ Bind(&less); 4368 __ LoadImmediate(out, -1); 4369 4370 __ Bind(&done); 4371} 4372 4373void LocationsBuilderARM::VisitPhi(HPhi* instruction) { 4374 LocationSummary* locations = 4375 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall); 4376 for (size_t i = 0, e = locations->GetInputCount(); i < e; ++i) { 4377 locations->SetInAt(i, Location::Any()); 4378 } 4379 locations->SetOut(Location::Any()); 4380} 4381 4382void InstructionCodeGeneratorARM::VisitPhi(HPhi* instruction ATTRIBUTE_UNUSED) { 4383 LOG(FATAL) << "Unreachable"; 4384} 4385 4386void CodeGeneratorARM::GenerateMemoryBarrier(MemBarrierKind kind) { 4387 // TODO (ported from quick): revisit ARM barrier kinds. 4388 DmbOptions flavor = DmbOptions::ISH; // Quiet C++ warnings. 4389 switch (kind) { 4390 case MemBarrierKind::kAnyStore: 4391 case MemBarrierKind::kLoadAny: 4392 case MemBarrierKind::kAnyAny: { 4393 flavor = DmbOptions::ISH; 4394 break; 4395 } 4396 case MemBarrierKind::kStoreStore: { 4397 flavor = DmbOptions::ISHST; 4398 break; 4399 } 4400 default: 4401 LOG(FATAL) << "Unexpected memory barrier " << kind; 4402 } 4403 __ dmb(flavor); 4404} 4405 4406void InstructionCodeGeneratorARM::GenerateWideAtomicLoad(Register addr, 4407 uint32_t offset, 4408 Register out_lo, 4409 Register out_hi) { 4410 if (offset != 0) { 4411 // Ensure `out_lo` is different from `addr`, so that loading 4412 // `offset` into `out_lo` does not clutter `addr`. 4413 DCHECK_NE(out_lo, addr); 4414 __ LoadImmediate(out_lo, offset); 4415 __ add(IP, addr, ShifterOperand(out_lo)); 4416 addr = IP; 4417 } 4418 __ ldrexd(out_lo, out_hi, addr); 4419} 4420 4421void InstructionCodeGeneratorARM::GenerateWideAtomicStore(Register addr, 4422 uint32_t offset, 4423 Register value_lo, 4424 Register value_hi, 4425 Register temp1, 4426 Register temp2, 4427 HInstruction* instruction) { 4428 Label fail; 4429 if (offset != 0) { 4430 __ LoadImmediate(temp1, offset); 4431 __ add(IP, addr, ShifterOperand(temp1)); 4432 addr = IP; 4433 } 4434 __ Bind(&fail); 4435 // We need a load followed by store. (The address used in a STREX instruction must 4436 // be the same as the address in the most recently executed LDREX instruction.) 4437 __ ldrexd(temp1, temp2, addr); 4438 codegen_->MaybeRecordImplicitNullCheck(instruction); 4439 __ strexd(temp1, value_lo, value_hi, addr); 4440 __ CompareAndBranchIfNonZero(temp1, &fail); 4441} 4442 4443void LocationsBuilderARM::HandleFieldSet(HInstruction* instruction, const FieldInfo& field_info) { 4444 DCHECK(instruction->IsInstanceFieldSet() || instruction->IsStaticFieldSet()); 4445 4446 LocationSummary* locations = 4447 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall); 4448 locations->SetInAt(0, Location::RequiresRegister()); 4449 4450 Primitive::Type field_type = field_info.GetFieldType(); 4451 if (Primitive::IsFloatingPointType(field_type)) { 4452 locations->SetInAt(1, Location::RequiresFpuRegister()); 4453 } else { 4454 locations->SetInAt(1, Location::RequiresRegister()); 4455 } 4456 4457 bool is_wide = field_type == Primitive::kPrimLong || field_type == Primitive::kPrimDouble; 4458 bool generate_volatile = field_info.IsVolatile() 4459 && is_wide 4460 && !codegen_->GetInstructionSetFeatures().HasAtomicLdrdAndStrd(); 4461 bool needs_write_barrier = 4462 CodeGenerator::StoreNeedsWriteBarrier(field_type, instruction->InputAt(1)); 4463 // Temporary registers for the write barrier. 4464 // TODO: consider renaming StoreNeedsWriteBarrier to StoreNeedsGCMark. 4465 if (needs_write_barrier) { 4466 locations->AddTemp(Location::RequiresRegister()); // Possibly used for reference poisoning too. 4467 locations->AddTemp(Location::RequiresRegister()); 4468 } else if (generate_volatile) { 4469 // ARM encoding have some additional constraints for ldrexd/strexd: 4470 // - registers need to be consecutive 4471 // - the first register should be even but not R14. 4472 // We don't test for ARM yet, and the assertion makes sure that we 4473 // revisit this if we ever enable ARM encoding. 4474 DCHECK_EQ(InstructionSet::kThumb2, codegen_->GetInstructionSet()); 4475 4476 locations->AddTemp(Location::RequiresRegister()); 4477 locations->AddTemp(Location::RequiresRegister()); 4478 if (field_type == Primitive::kPrimDouble) { 4479 // For doubles we need two more registers to copy the value. 4480 locations->AddTemp(Location::RegisterLocation(R2)); 4481 locations->AddTemp(Location::RegisterLocation(R3)); 4482 } 4483 } 4484} 4485 4486void InstructionCodeGeneratorARM::HandleFieldSet(HInstruction* instruction, 4487 const FieldInfo& field_info, 4488 bool value_can_be_null) { 4489 DCHECK(instruction->IsInstanceFieldSet() || instruction->IsStaticFieldSet()); 4490 4491 LocationSummary* locations = instruction->GetLocations(); 4492 Register base = locations->InAt(0).AsRegister<Register>(); 4493 Location value = locations->InAt(1); 4494 4495 bool is_volatile = field_info.IsVolatile(); 4496 bool atomic_ldrd_strd = codegen_->GetInstructionSetFeatures().HasAtomicLdrdAndStrd(); 4497 Primitive::Type field_type = field_info.GetFieldType(); 4498 uint32_t offset = field_info.GetFieldOffset().Uint32Value(); 4499 bool needs_write_barrier = 4500 CodeGenerator::StoreNeedsWriteBarrier(field_type, instruction->InputAt(1)); 4501 4502 if (is_volatile) { 4503 codegen_->GenerateMemoryBarrier(MemBarrierKind::kAnyStore); 4504 } 4505 4506 switch (field_type) { 4507 case Primitive::kPrimBoolean: 4508 case Primitive::kPrimByte: { 4509 __ StoreToOffset(kStoreByte, value.AsRegister<Register>(), base, offset); 4510 break; 4511 } 4512 4513 case Primitive::kPrimShort: 4514 case Primitive::kPrimChar: { 4515 __ StoreToOffset(kStoreHalfword, value.AsRegister<Register>(), base, offset); 4516 break; 4517 } 4518 4519 case Primitive::kPrimInt: 4520 case Primitive::kPrimNot: { 4521 if (kPoisonHeapReferences && needs_write_barrier) { 4522 // Note that in the case where `value` is a null reference, 4523 // we do not enter this block, as a null reference does not 4524 // need poisoning. 4525 DCHECK_EQ(field_type, Primitive::kPrimNot); 4526 Register temp = locations->GetTemp(0).AsRegister<Register>(); 4527 __ Mov(temp, value.AsRegister<Register>()); 4528 __ PoisonHeapReference(temp); 4529 __ StoreToOffset(kStoreWord, temp, base, offset); 4530 } else { 4531 __ StoreToOffset(kStoreWord, value.AsRegister<Register>(), base, offset); 4532 } 4533 break; 4534 } 4535 4536 case Primitive::kPrimLong: { 4537 if (is_volatile && !atomic_ldrd_strd) { 4538 GenerateWideAtomicStore(base, offset, 4539 value.AsRegisterPairLow<Register>(), 4540 value.AsRegisterPairHigh<Register>(), 4541 locations->GetTemp(0).AsRegister<Register>(), 4542 locations->GetTemp(1).AsRegister<Register>(), 4543 instruction); 4544 } else { 4545 __ StoreToOffset(kStoreWordPair, value.AsRegisterPairLow<Register>(), base, offset); 4546 codegen_->MaybeRecordImplicitNullCheck(instruction); 4547 } 4548 break; 4549 } 4550 4551 case Primitive::kPrimFloat: { 4552 __ StoreSToOffset(value.AsFpuRegister<SRegister>(), base, offset); 4553 break; 4554 } 4555 4556 case Primitive::kPrimDouble: { 4557 DRegister value_reg = FromLowSToD(value.AsFpuRegisterPairLow<SRegister>()); 4558 if (is_volatile && !atomic_ldrd_strd) { 4559 Register value_reg_lo = locations->GetTemp(0).AsRegister<Register>(); 4560 Register value_reg_hi = locations->GetTemp(1).AsRegister<Register>(); 4561 4562 __ vmovrrd(value_reg_lo, value_reg_hi, value_reg); 4563 4564 GenerateWideAtomicStore(base, offset, 4565 value_reg_lo, 4566 value_reg_hi, 4567 locations->GetTemp(2).AsRegister<Register>(), 4568 locations->GetTemp(3).AsRegister<Register>(), 4569 instruction); 4570 } else { 4571 __ StoreDToOffset(value_reg, base, offset); 4572 codegen_->MaybeRecordImplicitNullCheck(instruction); 4573 } 4574 break; 4575 } 4576 4577 case Primitive::kPrimVoid: 4578 LOG(FATAL) << "Unreachable type " << field_type; 4579 UNREACHABLE(); 4580 } 4581 4582 // Longs and doubles are handled in the switch. 4583 if (field_type != Primitive::kPrimLong && field_type != Primitive::kPrimDouble) { 4584 codegen_->MaybeRecordImplicitNullCheck(instruction); 4585 } 4586 4587 if (CodeGenerator::StoreNeedsWriteBarrier(field_type, instruction->InputAt(1))) { 4588 Register temp = locations->GetTemp(0).AsRegister<Register>(); 4589 Register card = locations->GetTemp(1).AsRegister<Register>(); 4590 codegen_->MarkGCCard( 4591 temp, card, base, value.AsRegister<Register>(), value_can_be_null); 4592 } 4593 4594 if (is_volatile) { 4595 codegen_->GenerateMemoryBarrier(MemBarrierKind::kAnyAny); 4596 } 4597} 4598 4599void LocationsBuilderARM::HandleFieldGet(HInstruction* instruction, const FieldInfo& field_info) { 4600 DCHECK(instruction->IsInstanceFieldGet() || instruction->IsStaticFieldGet()); 4601 4602 bool object_field_get_with_read_barrier = 4603 kEmitCompilerReadBarrier && (field_info.GetFieldType() == Primitive::kPrimNot); 4604 LocationSummary* locations = 4605 new (GetGraph()->GetArena()) LocationSummary(instruction, 4606 object_field_get_with_read_barrier ? 4607 LocationSummary::kCallOnSlowPath : 4608 LocationSummary::kNoCall); 4609 if (object_field_get_with_read_barrier && kUseBakerReadBarrier) { 4610 locations->SetCustomSlowPathCallerSaves(RegisterSet::Empty()); // No caller-save registers. 4611 } 4612 locations->SetInAt(0, Location::RequiresRegister()); 4613 4614 bool volatile_for_double = field_info.IsVolatile() 4615 && (field_info.GetFieldType() == Primitive::kPrimDouble) 4616 && !codegen_->GetInstructionSetFeatures().HasAtomicLdrdAndStrd(); 4617 // The output overlaps in case of volatile long: we don't want the 4618 // code generated by GenerateWideAtomicLoad to overwrite the 4619 // object's location. Likewise, in the case of an object field get 4620 // with read barriers enabled, we do not want the load to overwrite 4621 // the object's location, as we need it to emit the read barrier. 4622 bool overlap = (field_info.IsVolatile() && (field_info.GetFieldType() == Primitive::kPrimLong)) || 4623 object_field_get_with_read_barrier; 4624 4625 if (Primitive::IsFloatingPointType(instruction->GetType())) { 4626 locations->SetOut(Location::RequiresFpuRegister()); 4627 } else { 4628 locations->SetOut(Location::RequiresRegister(), 4629 (overlap ? Location::kOutputOverlap : Location::kNoOutputOverlap)); 4630 } 4631 if (volatile_for_double) { 4632 // ARM encoding have some additional constraints for ldrexd/strexd: 4633 // - registers need to be consecutive 4634 // - the first register should be even but not R14. 4635 // We don't test for ARM yet, and the assertion makes sure that we 4636 // revisit this if we ever enable ARM encoding. 4637 DCHECK_EQ(InstructionSet::kThumb2, codegen_->GetInstructionSet()); 4638 locations->AddTemp(Location::RequiresRegister()); 4639 locations->AddTemp(Location::RequiresRegister()); 4640 } else if (object_field_get_with_read_barrier && kUseBakerReadBarrier) { 4641 // We need a temporary register for the read barrier marking slow 4642 // path in CodeGeneratorARM::GenerateFieldLoadWithBakerReadBarrier. 4643 locations->AddTemp(Location::RequiresRegister()); 4644 } 4645} 4646 4647Location LocationsBuilderARM::ArithmeticZeroOrFpuRegister(HInstruction* input) { 4648 DCHECK(input->GetType() == Primitive::kPrimDouble || input->GetType() == Primitive::kPrimFloat) 4649 << input->GetType(); 4650 if ((input->IsFloatConstant() && (input->AsFloatConstant()->IsArithmeticZero())) || 4651 (input->IsDoubleConstant() && (input->AsDoubleConstant()->IsArithmeticZero()))) { 4652 return Location::ConstantLocation(input->AsConstant()); 4653 } else { 4654 return Location::RequiresFpuRegister(); 4655 } 4656} 4657 4658Location LocationsBuilderARM::ArmEncodableConstantOrRegister(HInstruction* constant, 4659 Opcode opcode) { 4660 DCHECK(!Primitive::IsFloatingPointType(constant->GetType())); 4661 if (constant->IsConstant() && 4662 CanEncodeConstantAsImmediate(constant->AsConstant(), opcode)) { 4663 return Location::ConstantLocation(constant->AsConstant()); 4664 } 4665 return Location::RequiresRegister(); 4666} 4667 4668bool LocationsBuilderARM::CanEncodeConstantAsImmediate(HConstant* input_cst, 4669 Opcode opcode) { 4670 uint64_t value = static_cast<uint64_t>(Int64FromConstant(input_cst)); 4671 if (Primitive::Is64BitType(input_cst->GetType())) { 4672 Opcode high_opcode = opcode; 4673 SetCc low_set_cc = kCcDontCare; 4674 switch (opcode) { 4675 case SUB: 4676 // Flip the operation to an ADD. 4677 value = -value; 4678 opcode = ADD; 4679 FALLTHROUGH_INTENDED; 4680 case ADD: 4681 if (Low32Bits(value) == 0u) { 4682 return CanEncodeConstantAsImmediate(High32Bits(value), opcode, kCcDontCare); 4683 } 4684 high_opcode = ADC; 4685 low_set_cc = kCcSet; 4686 break; 4687 default: 4688 break; 4689 } 4690 return CanEncodeConstantAsImmediate(Low32Bits(value), opcode, low_set_cc) && 4691 CanEncodeConstantAsImmediate(High32Bits(value), high_opcode, kCcDontCare); 4692 } else { 4693 return CanEncodeConstantAsImmediate(Low32Bits(value), opcode); 4694 } 4695} 4696 4697bool LocationsBuilderARM::CanEncodeConstantAsImmediate(uint32_t value, 4698 Opcode opcode, 4699 SetCc set_cc) { 4700 ShifterOperand so; 4701 ArmAssembler* assembler = codegen_->GetAssembler(); 4702 if (assembler->ShifterOperandCanHold(kNoRegister, kNoRegister, opcode, value, set_cc, &so)) { 4703 return true; 4704 } 4705 Opcode neg_opcode = kNoOperand; 4706 switch (opcode) { 4707 case AND: neg_opcode = BIC; value = ~value; break; 4708 case ORR: neg_opcode = ORN; value = ~value; break; 4709 case ADD: neg_opcode = SUB; value = -value; break; 4710 case ADC: neg_opcode = SBC; value = ~value; break; 4711 case SUB: neg_opcode = ADD; value = -value; break; 4712 case SBC: neg_opcode = ADC; value = ~value; break; 4713 default: 4714 return false; 4715 } 4716 return assembler->ShifterOperandCanHold(kNoRegister, kNoRegister, neg_opcode, value, set_cc, &so); 4717} 4718 4719void InstructionCodeGeneratorARM::HandleFieldGet(HInstruction* instruction, 4720 const FieldInfo& field_info) { 4721 DCHECK(instruction->IsInstanceFieldGet() || instruction->IsStaticFieldGet()); 4722 4723 LocationSummary* locations = instruction->GetLocations(); 4724 Location base_loc = locations->InAt(0); 4725 Register base = base_loc.AsRegister<Register>(); 4726 Location out = locations->Out(); 4727 bool is_volatile = field_info.IsVolatile(); 4728 bool atomic_ldrd_strd = codegen_->GetInstructionSetFeatures().HasAtomicLdrdAndStrd(); 4729 Primitive::Type field_type = field_info.GetFieldType(); 4730 uint32_t offset = field_info.GetFieldOffset().Uint32Value(); 4731 4732 switch (field_type) { 4733 case Primitive::kPrimBoolean: 4734 __ LoadFromOffset(kLoadUnsignedByte, out.AsRegister<Register>(), base, offset); 4735 break; 4736 4737 case Primitive::kPrimByte: 4738 __ LoadFromOffset(kLoadSignedByte, out.AsRegister<Register>(), base, offset); 4739 break; 4740 4741 case Primitive::kPrimShort: 4742 __ LoadFromOffset(kLoadSignedHalfword, out.AsRegister<Register>(), base, offset); 4743 break; 4744 4745 case Primitive::kPrimChar: 4746 __ LoadFromOffset(kLoadUnsignedHalfword, out.AsRegister<Register>(), base, offset); 4747 break; 4748 4749 case Primitive::kPrimInt: 4750 __ LoadFromOffset(kLoadWord, out.AsRegister<Register>(), base, offset); 4751 break; 4752 4753 case Primitive::kPrimNot: { 4754 // /* HeapReference<Object> */ out = *(base + offset) 4755 if (kEmitCompilerReadBarrier && kUseBakerReadBarrier) { 4756 Location temp_loc = locations->GetTemp(0); 4757 // Note that a potential implicit null check is handled in this 4758 // CodeGeneratorARM::GenerateFieldLoadWithBakerReadBarrier call. 4759 codegen_->GenerateFieldLoadWithBakerReadBarrier( 4760 instruction, out, base, offset, temp_loc, /* needs_null_check */ true); 4761 if (is_volatile) { 4762 codegen_->GenerateMemoryBarrier(MemBarrierKind::kLoadAny); 4763 } 4764 } else { 4765 __ LoadFromOffset(kLoadWord, out.AsRegister<Register>(), base, offset); 4766 codegen_->MaybeRecordImplicitNullCheck(instruction); 4767 if (is_volatile) { 4768 codegen_->GenerateMemoryBarrier(MemBarrierKind::kLoadAny); 4769 } 4770 // If read barriers are enabled, emit read barriers other than 4771 // Baker's using a slow path (and also unpoison the loaded 4772 // reference, if heap poisoning is enabled). 4773 codegen_->MaybeGenerateReadBarrierSlow(instruction, out, out, base_loc, offset); 4774 } 4775 break; 4776 } 4777 4778 case Primitive::kPrimLong: 4779 if (is_volatile && !atomic_ldrd_strd) { 4780 GenerateWideAtomicLoad(base, offset, 4781 out.AsRegisterPairLow<Register>(), 4782 out.AsRegisterPairHigh<Register>()); 4783 } else { 4784 __ LoadFromOffset(kLoadWordPair, out.AsRegisterPairLow<Register>(), base, offset); 4785 } 4786 break; 4787 4788 case Primitive::kPrimFloat: 4789 __ LoadSFromOffset(out.AsFpuRegister<SRegister>(), base, offset); 4790 break; 4791 4792 case Primitive::kPrimDouble: { 4793 DRegister out_reg = FromLowSToD(out.AsFpuRegisterPairLow<SRegister>()); 4794 if (is_volatile && !atomic_ldrd_strd) { 4795 Register lo = locations->GetTemp(0).AsRegister<Register>(); 4796 Register hi = locations->GetTemp(1).AsRegister<Register>(); 4797 GenerateWideAtomicLoad(base, offset, lo, hi); 4798 codegen_->MaybeRecordImplicitNullCheck(instruction); 4799 __ vmovdrr(out_reg, lo, hi); 4800 } else { 4801 __ LoadDFromOffset(out_reg, base, offset); 4802 codegen_->MaybeRecordImplicitNullCheck(instruction); 4803 } 4804 break; 4805 } 4806 4807 case Primitive::kPrimVoid: 4808 LOG(FATAL) << "Unreachable type " << field_type; 4809 UNREACHABLE(); 4810 } 4811 4812 if (field_type == Primitive::kPrimNot || field_type == Primitive::kPrimDouble) { 4813 // Potential implicit null checks, in the case of reference or 4814 // double fields, are handled in the previous switch statement. 4815 } else { 4816 codegen_->MaybeRecordImplicitNullCheck(instruction); 4817 } 4818 4819 if (is_volatile) { 4820 if (field_type == Primitive::kPrimNot) { 4821 // Memory barriers, in the case of references, are also handled 4822 // in the previous switch statement. 4823 } else { 4824 codegen_->GenerateMemoryBarrier(MemBarrierKind::kLoadAny); 4825 } 4826 } 4827} 4828 4829void LocationsBuilderARM::VisitInstanceFieldSet(HInstanceFieldSet* instruction) { 4830 HandleFieldSet(instruction, instruction->GetFieldInfo()); 4831} 4832 4833void InstructionCodeGeneratorARM::VisitInstanceFieldSet(HInstanceFieldSet* instruction) { 4834 HandleFieldSet(instruction, instruction->GetFieldInfo(), instruction->GetValueCanBeNull()); 4835} 4836 4837void LocationsBuilderARM::VisitInstanceFieldGet(HInstanceFieldGet* instruction) { 4838 HandleFieldGet(instruction, instruction->GetFieldInfo()); 4839} 4840 4841void InstructionCodeGeneratorARM::VisitInstanceFieldGet(HInstanceFieldGet* instruction) { 4842 HandleFieldGet(instruction, instruction->GetFieldInfo()); 4843} 4844 4845void LocationsBuilderARM::VisitStaticFieldGet(HStaticFieldGet* instruction) { 4846 HandleFieldGet(instruction, instruction->GetFieldInfo()); 4847} 4848 4849void InstructionCodeGeneratorARM::VisitStaticFieldGet(HStaticFieldGet* instruction) { 4850 HandleFieldGet(instruction, instruction->GetFieldInfo()); 4851} 4852 4853void LocationsBuilderARM::VisitStaticFieldSet(HStaticFieldSet* instruction) { 4854 HandleFieldSet(instruction, instruction->GetFieldInfo()); 4855} 4856 4857void InstructionCodeGeneratorARM::VisitStaticFieldSet(HStaticFieldSet* instruction) { 4858 HandleFieldSet(instruction, instruction->GetFieldInfo(), instruction->GetValueCanBeNull()); 4859} 4860 4861void LocationsBuilderARM::VisitUnresolvedInstanceFieldGet( 4862 HUnresolvedInstanceFieldGet* instruction) { 4863 FieldAccessCallingConventionARM calling_convention; 4864 codegen_->CreateUnresolvedFieldLocationSummary( 4865 instruction, instruction->GetFieldType(), calling_convention); 4866} 4867 4868void InstructionCodeGeneratorARM::VisitUnresolvedInstanceFieldGet( 4869 HUnresolvedInstanceFieldGet* instruction) { 4870 FieldAccessCallingConventionARM calling_convention; 4871 codegen_->GenerateUnresolvedFieldAccess(instruction, 4872 instruction->GetFieldType(), 4873 instruction->GetFieldIndex(), 4874 instruction->GetDexPc(), 4875 calling_convention); 4876} 4877 4878void LocationsBuilderARM::VisitUnresolvedInstanceFieldSet( 4879 HUnresolvedInstanceFieldSet* instruction) { 4880 FieldAccessCallingConventionARM calling_convention; 4881 codegen_->CreateUnresolvedFieldLocationSummary( 4882 instruction, instruction->GetFieldType(), calling_convention); 4883} 4884 4885void InstructionCodeGeneratorARM::VisitUnresolvedInstanceFieldSet( 4886 HUnresolvedInstanceFieldSet* instruction) { 4887 FieldAccessCallingConventionARM calling_convention; 4888 codegen_->GenerateUnresolvedFieldAccess(instruction, 4889 instruction->GetFieldType(), 4890 instruction->GetFieldIndex(), 4891 instruction->GetDexPc(), 4892 calling_convention); 4893} 4894 4895void LocationsBuilderARM::VisitUnresolvedStaticFieldGet( 4896 HUnresolvedStaticFieldGet* instruction) { 4897 FieldAccessCallingConventionARM calling_convention; 4898 codegen_->CreateUnresolvedFieldLocationSummary( 4899 instruction, instruction->GetFieldType(), calling_convention); 4900} 4901 4902void InstructionCodeGeneratorARM::VisitUnresolvedStaticFieldGet( 4903 HUnresolvedStaticFieldGet* instruction) { 4904 FieldAccessCallingConventionARM calling_convention; 4905 codegen_->GenerateUnresolvedFieldAccess(instruction, 4906 instruction->GetFieldType(), 4907 instruction->GetFieldIndex(), 4908 instruction->GetDexPc(), 4909 calling_convention); 4910} 4911 4912void LocationsBuilderARM::VisitUnresolvedStaticFieldSet( 4913 HUnresolvedStaticFieldSet* instruction) { 4914 FieldAccessCallingConventionARM calling_convention; 4915 codegen_->CreateUnresolvedFieldLocationSummary( 4916 instruction, instruction->GetFieldType(), calling_convention); 4917} 4918 4919void InstructionCodeGeneratorARM::VisitUnresolvedStaticFieldSet( 4920 HUnresolvedStaticFieldSet* instruction) { 4921 FieldAccessCallingConventionARM calling_convention; 4922 codegen_->GenerateUnresolvedFieldAccess(instruction, 4923 instruction->GetFieldType(), 4924 instruction->GetFieldIndex(), 4925 instruction->GetDexPc(), 4926 calling_convention); 4927} 4928 4929void LocationsBuilderARM::VisitNullCheck(HNullCheck* instruction) { 4930 LocationSummary* locations = codegen_->CreateThrowingSlowPathLocations(instruction); 4931 locations->SetInAt(0, Location::RequiresRegister()); 4932} 4933 4934void CodeGeneratorARM::GenerateImplicitNullCheck(HNullCheck* instruction) { 4935 if (CanMoveNullCheckToUser(instruction)) { 4936 return; 4937 } 4938 Location obj = instruction->GetLocations()->InAt(0); 4939 4940 __ LoadFromOffset(kLoadWord, IP, obj.AsRegister<Register>(), 0); 4941 RecordPcInfo(instruction, instruction->GetDexPc()); 4942} 4943 4944void CodeGeneratorARM::GenerateExplicitNullCheck(HNullCheck* instruction) { 4945 SlowPathCodeARM* slow_path = new (GetGraph()->GetArena()) NullCheckSlowPathARM(instruction); 4946 AddSlowPath(slow_path); 4947 4948 LocationSummary* locations = instruction->GetLocations(); 4949 Location obj = locations->InAt(0); 4950 4951 __ CompareAndBranchIfZero(obj.AsRegister<Register>(), slow_path->GetEntryLabel()); 4952} 4953 4954void InstructionCodeGeneratorARM::VisitNullCheck(HNullCheck* instruction) { 4955 codegen_->GenerateNullCheck(instruction); 4956} 4957 4958static LoadOperandType GetLoadOperandType(Primitive::Type type) { 4959 switch (type) { 4960 case Primitive::kPrimNot: 4961 return kLoadWord; 4962 case Primitive::kPrimBoolean: 4963 return kLoadUnsignedByte; 4964 case Primitive::kPrimByte: 4965 return kLoadSignedByte; 4966 case Primitive::kPrimChar: 4967 return kLoadUnsignedHalfword; 4968 case Primitive::kPrimShort: 4969 return kLoadSignedHalfword; 4970 case Primitive::kPrimInt: 4971 return kLoadWord; 4972 case Primitive::kPrimLong: 4973 return kLoadWordPair; 4974 case Primitive::kPrimFloat: 4975 return kLoadSWord; 4976 case Primitive::kPrimDouble: 4977 return kLoadDWord; 4978 default: 4979 LOG(FATAL) << "Unreachable type " << type; 4980 UNREACHABLE(); 4981 } 4982} 4983 4984static StoreOperandType GetStoreOperandType(Primitive::Type type) { 4985 switch (type) { 4986 case Primitive::kPrimNot: 4987 return kStoreWord; 4988 case Primitive::kPrimBoolean: 4989 case Primitive::kPrimByte: 4990 return kStoreByte; 4991 case Primitive::kPrimChar: 4992 case Primitive::kPrimShort: 4993 return kStoreHalfword; 4994 case Primitive::kPrimInt: 4995 return kStoreWord; 4996 case Primitive::kPrimLong: 4997 return kStoreWordPair; 4998 case Primitive::kPrimFloat: 4999 return kStoreSWord; 5000 case Primitive::kPrimDouble: 5001 return kStoreDWord; 5002 default: 5003 LOG(FATAL) << "Unreachable type " << type; 5004 UNREACHABLE(); 5005 } 5006} 5007 5008void CodeGeneratorARM::LoadFromShiftedRegOffset(Primitive::Type type, 5009 Location out_loc, 5010 Register base, 5011 Register reg_offset, 5012 Condition cond) { 5013 uint32_t shift_count = Primitive::ComponentSizeShift(type); 5014 Address mem_address(base, reg_offset, Shift::LSL, shift_count); 5015 5016 switch (type) { 5017 case Primitive::kPrimByte: 5018 __ ldrsb(out_loc.AsRegister<Register>(), mem_address, cond); 5019 break; 5020 case Primitive::kPrimBoolean: 5021 __ ldrb(out_loc.AsRegister<Register>(), mem_address, cond); 5022 break; 5023 case Primitive::kPrimShort: 5024 __ ldrsh(out_loc.AsRegister<Register>(), mem_address, cond); 5025 break; 5026 case Primitive::kPrimChar: 5027 __ ldrh(out_loc.AsRegister<Register>(), mem_address, cond); 5028 break; 5029 case Primitive::kPrimNot: 5030 case Primitive::kPrimInt: 5031 __ ldr(out_loc.AsRegister<Register>(), mem_address, cond); 5032 break; 5033 // T32 doesn't support LoadFromShiftedRegOffset mem address mode for these types. 5034 case Primitive::kPrimLong: 5035 case Primitive::kPrimFloat: 5036 case Primitive::kPrimDouble: 5037 default: 5038 LOG(FATAL) << "Unreachable type " << type; 5039 UNREACHABLE(); 5040 } 5041} 5042 5043void CodeGeneratorARM::StoreToShiftedRegOffset(Primitive::Type type, 5044 Location loc, 5045 Register base, 5046 Register reg_offset, 5047 Condition cond) { 5048 uint32_t shift_count = Primitive::ComponentSizeShift(type); 5049 Address mem_address(base, reg_offset, Shift::LSL, shift_count); 5050 5051 switch (type) { 5052 case Primitive::kPrimByte: 5053 case Primitive::kPrimBoolean: 5054 __ strb(loc.AsRegister<Register>(), mem_address, cond); 5055 break; 5056 case Primitive::kPrimShort: 5057 case Primitive::kPrimChar: 5058 __ strh(loc.AsRegister<Register>(), mem_address, cond); 5059 break; 5060 case Primitive::kPrimNot: 5061 case Primitive::kPrimInt: 5062 __ str(loc.AsRegister<Register>(), mem_address, cond); 5063 break; 5064 // T32 doesn't support StoreToShiftedRegOffset mem address mode for these types. 5065 case Primitive::kPrimLong: 5066 case Primitive::kPrimFloat: 5067 case Primitive::kPrimDouble: 5068 default: 5069 LOG(FATAL) << "Unreachable type " << type; 5070 UNREACHABLE(); 5071 } 5072} 5073 5074void LocationsBuilderARM::VisitArrayGet(HArrayGet* instruction) { 5075 bool object_array_get_with_read_barrier = 5076 kEmitCompilerReadBarrier && (instruction->GetType() == Primitive::kPrimNot); 5077 LocationSummary* locations = 5078 new (GetGraph()->GetArena()) LocationSummary(instruction, 5079 object_array_get_with_read_barrier ? 5080 LocationSummary::kCallOnSlowPath : 5081 LocationSummary::kNoCall); 5082 if (object_array_get_with_read_barrier && kUseBakerReadBarrier) { 5083 locations->SetCustomSlowPathCallerSaves(RegisterSet::Empty()); // No caller-save registers. 5084 } 5085 locations->SetInAt(0, Location::RequiresRegister()); 5086 locations->SetInAt(1, Location::RegisterOrConstant(instruction->InputAt(1))); 5087 if (Primitive::IsFloatingPointType(instruction->GetType())) { 5088 locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap); 5089 } else { 5090 // The output overlaps in the case of an object array get with 5091 // read barriers enabled: we do not want the move to overwrite the 5092 // array's location, as we need it to emit the read barrier. 5093 locations->SetOut( 5094 Location::RequiresRegister(), 5095 object_array_get_with_read_barrier ? Location::kOutputOverlap : Location::kNoOutputOverlap); 5096 } 5097 // We need a temporary register for the read barrier marking slow 5098 // path in CodeGeneratorARM::GenerateArrayLoadWithBakerReadBarrier. 5099 // Also need for String compression feature. 5100 if ((object_array_get_with_read_barrier && kUseBakerReadBarrier) 5101 || (mirror::kUseStringCompression && instruction->IsStringCharAt())) { 5102 locations->AddTemp(Location::RequiresRegister()); 5103 } 5104} 5105 5106void InstructionCodeGeneratorARM::VisitArrayGet(HArrayGet* instruction) { 5107 LocationSummary* locations = instruction->GetLocations(); 5108 Location obj_loc = locations->InAt(0); 5109 Register obj = obj_loc.AsRegister<Register>(); 5110 Location index = locations->InAt(1); 5111 Location out_loc = locations->Out(); 5112 uint32_t data_offset = CodeGenerator::GetArrayDataOffset(instruction); 5113 Primitive::Type type = instruction->GetType(); 5114 const bool maybe_compressed_char_at = mirror::kUseStringCompression && 5115 instruction->IsStringCharAt(); 5116 HInstruction* array_instr = instruction->GetArray(); 5117 bool has_intermediate_address = array_instr->IsIntermediateAddress(); 5118 5119 switch (type) { 5120 case Primitive::kPrimBoolean: 5121 case Primitive::kPrimByte: 5122 case Primitive::kPrimShort: 5123 case Primitive::kPrimChar: 5124 case Primitive::kPrimInt: { 5125 Register length; 5126 if (maybe_compressed_char_at) { 5127 length = locations->GetTemp(0).AsRegister<Register>(); 5128 uint32_t count_offset = mirror::String::CountOffset().Uint32Value(); 5129 __ LoadFromOffset(kLoadWord, length, obj, count_offset); 5130 codegen_->MaybeRecordImplicitNullCheck(instruction); 5131 } 5132 if (index.IsConstant()) { 5133 int32_t const_index = index.GetConstant()->AsIntConstant()->GetValue(); 5134 if (maybe_compressed_char_at) { 5135 Label uncompressed_load, done; 5136 __ Lsrs(length, length, 1u); // LSRS has a 16-bit encoding, TST (immediate) does not. 5137 static_assert(static_cast<uint32_t>(mirror::StringCompressionFlag::kCompressed) == 0u, 5138 "Expecting 0=compressed, 1=uncompressed"); 5139 __ b(&uncompressed_load, CS); 5140 __ LoadFromOffset(kLoadUnsignedByte, 5141 out_loc.AsRegister<Register>(), 5142 obj, 5143 data_offset + const_index); 5144 __ b(&done); 5145 __ Bind(&uncompressed_load); 5146 __ LoadFromOffset(GetLoadOperandType(Primitive::kPrimChar), 5147 out_loc.AsRegister<Register>(), 5148 obj, 5149 data_offset + (const_index << 1)); 5150 __ Bind(&done); 5151 } else { 5152 uint32_t full_offset = data_offset + (const_index << Primitive::ComponentSizeShift(type)); 5153 5154 LoadOperandType load_type = GetLoadOperandType(type); 5155 __ LoadFromOffset(load_type, out_loc.AsRegister<Register>(), obj, full_offset); 5156 } 5157 } else { 5158 Register temp = IP; 5159 5160 if (has_intermediate_address) { 5161 // We do not need to compute the intermediate address from the array: the 5162 // input instruction has done it already. See the comment in 5163 // `TryExtractArrayAccessAddress()`. 5164 if (kIsDebugBuild) { 5165 HIntermediateAddress* tmp = array_instr->AsIntermediateAddress(); 5166 DCHECK_EQ(tmp->GetOffset()->AsIntConstant()->GetValueAsUint64(), data_offset); 5167 } 5168 temp = obj; 5169 } else { 5170 __ add(temp, obj, ShifterOperand(data_offset)); 5171 } 5172 if (maybe_compressed_char_at) { 5173 Label uncompressed_load, done; 5174 __ Lsrs(length, length, 1u); // LSRS has a 16-bit encoding, TST (immediate) does not. 5175 static_assert(static_cast<uint32_t>(mirror::StringCompressionFlag::kCompressed) == 0u, 5176 "Expecting 0=compressed, 1=uncompressed"); 5177 __ b(&uncompressed_load, CS); 5178 __ ldrb(out_loc.AsRegister<Register>(), 5179 Address(temp, index.AsRegister<Register>(), Shift::LSL, 0)); 5180 __ b(&done); 5181 __ Bind(&uncompressed_load); 5182 __ ldrh(out_loc.AsRegister<Register>(), 5183 Address(temp, index.AsRegister<Register>(), Shift::LSL, 1)); 5184 __ Bind(&done); 5185 } else { 5186 codegen_->LoadFromShiftedRegOffset(type, out_loc, temp, index.AsRegister<Register>()); 5187 } 5188 } 5189 break; 5190 } 5191 5192 case Primitive::kPrimNot: { 5193 // The read barrier instrumentation of object ArrayGet 5194 // instructions does not support the HIntermediateAddress 5195 // instruction. 5196 DCHECK(!(has_intermediate_address && kEmitCompilerReadBarrier)); 5197 5198 static_assert( 5199 sizeof(mirror::HeapReference<mirror::Object>) == sizeof(int32_t), 5200 "art::mirror::HeapReference<art::mirror::Object> and int32_t have different sizes."); 5201 // /* HeapReference<Object> */ out = 5202 // *(obj + data_offset + index * sizeof(HeapReference<Object>)) 5203 if (kEmitCompilerReadBarrier && kUseBakerReadBarrier) { 5204 Location temp = locations->GetTemp(0); 5205 // Note that a potential implicit null check is handled in this 5206 // CodeGeneratorARM::GenerateArrayLoadWithBakerReadBarrier call. 5207 codegen_->GenerateArrayLoadWithBakerReadBarrier( 5208 instruction, out_loc, obj, data_offset, index, temp, /* needs_null_check */ true); 5209 } else { 5210 Register out = out_loc.AsRegister<Register>(); 5211 if (index.IsConstant()) { 5212 size_t offset = 5213 (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4) + data_offset; 5214 __ LoadFromOffset(kLoadWord, out, obj, offset); 5215 codegen_->MaybeRecordImplicitNullCheck(instruction); 5216 // If read barriers are enabled, emit read barriers other than 5217 // Baker's using a slow path (and also unpoison the loaded 5218 // reference, if heap poisoning is enabled). 5219 codegen_->MaybeGenerateReadBarrierSlow(instruction, out_loc, out_loc, obj_loc, offset); 5220 } else { 5221 Register temp = IP; 5222 5223 if (has_intermediate_address) { 5224 // We do not need to compute the intermediate address from the array: the 5225 // input instruction has done it already. See the comment in 5226 // `TryExtractArrayAccessAddress()`. 5227 if (kIsDebugBuild) { 5228 HIntermediateAddress* tmp = array_instr->AsIntermediateAddress(); 5229 DCHECK_EQ(tmp->GetOffset()->AsIntConstant()->GetValueAsUint64(), data_offset); 5230 } 5231 temp = obj; 5232 } else { 5233 __ add(temp, obj, ShifterOperand(data_offset)); 5234 } 5235 codegen_->LoadFromShiftedRegOffset(type, out_loc, temp, index.AsRegister<Register>()); 5236 5237 codegen_->MaybeRecordImplicitNullCheck(instruction); 5238 // If read barriers are enabled, emit read barriers other than 5239 // Baker's using a slow path (and also unpoison the loaded 5240 // reference, if heap poisoning is enabled). 5241 codegen_->MaybeGenerateReadBarrierSlow( 5242 instruction, out_loc, out_loc, obj_loc, data_offset, index); 5243 } 5244 } 5245 break; 5246 } 5247 5248 case Primitive::kPrimLong: { 5249 if (index.IsConstant()) { 5250 size_t offset = 5251 (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_8) + data_offset; 5252 __ LoadFromOffset(kLoadWordPair, out_loc.AsRegisterPairLow<Register>(), obj, offset); 5253 } else { 5254 __ add(IP, obj, ShifterOperand(index.AsRegister<Register>(), LSL, TIMES_8)); 5255 __ LoadFromOffset(kLoadWordPair, out_loc.AsRegisterPairLow<Register>(), IP, data_offset); 5256 } 5257 break; 5258 } 5259 5260 case Primitive::kPrimFloat: { 5261 SRegister out = out_loc.AsFpuRegister<SRegister>(); 5262 if (index.IsConstant()) { 5263 size_t offset = (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4) + data_offset; 5264 __ LoadSFromOffset(out, obj, offset); 5265 } else { 5266 __ add(IP, obj, ShifterOperand(index.AsRegister<Register>(), LSL, TIMES_4)); 5267 __ LoadSFromOffset(out, IP, data_offset); 5268 } 5269 break; 5270 } 5271 5272 case Primitive::kPrimDouble: { 5273 SRegister out = out_loc.AsFpuRegisterPairLow<SRegister>(); 5274 if (index.IsConstant()) { 5275 size_t offset = (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_8) + data_offset; 5276 __ LoadDFromOffset(FromLowSToD(out), obj, offset); 5277 } else { 5278 __ add(IP, obj, ShifterOperand(index.AsRegister<Register>(), LSL, TIMES_8)); 5279 __ LoadDFromOffset(FromLowSToD(out), IP, data_offset); 5280 } 5281 break; 5282 } 5283 5284 case Primitive::kPrimVoid: 5285 LOG(FATAL) << "Unreachable type " << type; 5286 UNREACHABLE(); 5287 } 5288 5289 if (type == Primitive::kPrimNot) { 5290 // Potential implicit null checks, in the case of reference 5291 // arrays, are handled in the previous switch statement. 5292 } else if (!maybe_compressed_char_at) { 5293 codegen_->MaybeRecordImplicitNullCheck(instruction); 5294 } 5295} 5296 5297void LocationsBuilderARM::VisitArraySet(HArraySet* instruction) { 5298 Primitive::Type value_type = instruction->GetComponentType(); 5299 5300 bool needs_write_barrier = 5301 CodeGenerator::StoreNeedsWriteBarrier(value_type, instruction->GetValue()); 5302 bool may_need_runtime_call_for_type_check = instruction->NeedsTypeCheck(); 5303 5304 LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary( 5305 instruction, 5306 may_need_runtime_call_for_type_check ? 5307 LocationSummary::kCallOnSlowPath : 5308 LocationSummary::kNoCall); 5309 5310 locations->SetInAt(0, Location::RequiresRegister()); 5311 locations->SetInAt(1, Location::RegisterOrConstant(instruction->InputAt(1))); 5312 if (Primitive::IsFloatingPointType(value_type)) { 5313 locations->SetInAt(2, Location::RequiresFpuRegister()); 5314 } else { 5315 locations->SetInAt(2, Location::RequiresRegister()); 5316 } 5317 if (needs_write_barrier) { 5318 // Temporary registers for the write barrier. 5319 locations->AddTemp(Location::RequiresRegister()); // Possibly used for ref. poisoning too. 5320 locations->AddTemp(Location::RequiresRegister()); 5321 } 5322} 5323 5324void InstructionCodeGeneratorARM::VisitArraySet(HArraySet* instruction) { 5325 LocationSummary* locations = instruction->GetLocations(); 5326 Location array_loc = locations->InAt(0); 5327 Register array = array_loc.AsRegister<Register>(); 5328 Location index = locations->InAt(1); 5329 Primitive::Type value_type = instruction->GetComponentType(); 5330 bool may_need_runtime_call_for_type_check = instruction->NeedsTypeCheck(); 5331 bool needs_write_barrier = 5332 CodeGenerator::StoreNeedsWriteBarrier(value_type, instruction->GetValue()); 5333 uint32_t data_offset = 5334 mirror::Array::DataOffset(Primitive::ComponentSize(value_type)).Uint32Value(); 5335 Location value_loc = locations->InAt(2); 5336 HInstruction* array_instr = instruction->GetArray(); 5337 bool has_intermediate_address = array_instr->IsIntermediateAddress(); 5338 5339 switch (value_type) { 5340 case Primitive::kPrimBoolean: 5341 case Primitive::kPrimByte: 5342 case Primitive::kPrimShort: 5343 case Primitive::kPrimChar: 5344 case Primitive::kPrimInt: { 5345 if (index.IsConstant()) { 5346 int32_t const_index = index.GetConstant()->AsIntConstant()->GetValue(); 5347 uint32_t full_offset = 5348 data_offset + (const_index << Primitive::ComponentSizeShift(value_type)); 5349 StoreOperandType store_type = GetStoreOperandType(value_type); 5350 __ StoreToOffset(store_type, value_loc.AsRegister<Register>(), array, full_offset); 5351 } else { 5352 Register temp = IP; 5353 5354 if (has_intermediate_address) { 5355 // We do not need to compute the intermediate address from the array: the 5356 // input instruction has done it already. See the comment in 5357 // `TryExtractArrayAccessAddress()`. 5358 if (kIsDebugBuild) { 5359 HIntermediateAddress* tmp = array_instr->AsIntermediateAddress(); 5360 DCHECK(tmp->GetOffset()->AsIntConstant()->GetValueAsUint64() == data_offset); 5361 } 5362 temp = array; 5363 } else { 5364 __ add(temp, array, ShifterOperand(data_offset)); 5365 } 5366 codegen_->StoreToShiftedRegOffset(value_type, 5367 value_loc, 5368 temp, 5369 index.AsRegister<Register>()); 5370 } 5371 break; 5372 } 5373 5374 case Primitive::kPrimNot: { 5375 Register value = value_loc.AsRegister<Register>(); 5376 // TryExtractArrayAccessAddress optimization is never applied for non-primitive ArraySet. 5377 // See the comment in instruction_simplifier_shared.cc. 5378 DCHECK(!has_intermediate_address); 5379 5380 if (instruction->InputAt(2)->IsNullConstant()) { 5381 // Just setting null. 5382 if (index.IsConstant()) { 5383 size_t offset = 5384 (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4) + data_offset; 5385 __ StoreToOffset(kStoreWord, value, array, offset); 5386 } else { 5387 DCHECK(index.IsRegister()) << index; 5388 __ add(IP, array, ShifterOperand(data_offset)); 5389 codegen_->StoreToShiftedRegOffset(value_type, 5390 value_loc, 5391 IP, 5392 index.AsRegister<Register>()); 5393 } 5394 codegen_->MaybeRecordImplicitNullCheck(instruction); 5395 DCHECK(!needs_write_barrier); 5396 DCHECK(!may_need_runtime_call_for_type_check); 5397 break; 5398 } 5399 5400 DCHECK(needs_write_barrier); 5401 Location temp1_loc = locations->GetTemp(0); 5402 Register temp1 = temp1_loc.AsRegister<Register>(); 5403 Location temp2_loc = locations->GetTemp(1); 5404 Register temp2 = temp2_loc.AsRegister<Register>(); 5405 uint32_t class_offset = mirror::Object::ClassOffset().Int32Value(); 5406 uint32_t super_offset = mirror::Class::SuperClassOffset().Int32Value(); 5407 uint32_t component_offset = mirror::Class::ComponentTypeOffset().Int32Value(); 5408 Label done; 5409 SlowPathCodeARM* slow_path = nullptr; 5410 5411 if (may_need_runtime_call_for_type_check) { 5412 slow_path = new (GetGraph()->GetArena()) ArraySetSlowPathARM(instruction); 5413 codegen_->AddSlowPath(slow_path); 5414 if (instruction->GetValueCanBeNull()) { 5415 Label non_zero; 5416 __ CompareAndBranchIfNonZero(value, &non_zero); 5417 if (index.IsConstant()) { 5418 size_t offset = 5419 (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4) + data_offset; 5420 __ StoreToOffset(kStoreWord, value, array, offset); 5421 } else { 5422 DCHECK(index.IsRegister()) << index; 5423 __ add(IP, array, ShifterOperand(data_offset)); 5424 codegen_->StoreToShiftedRegOffset(value_type, 5425 value_loc, 5426 IP, 5427 index.AsRegister<Register>()); 5428 } 5429 codegen_->MaybeRecordImplicitNullCheck(instruction); 5430 __ b(&done); 5431 __ Bind(&non_zero); 5432 } 5433 5434 // Note that when read barriers are enabled, the type checks 5435 // are performed without read barriers. This is fine, even in 5436 // the case where a class object is in the from-space after 5437 // the flip, as a comparison involving such a type would not 5438 // produce a false positive; it may of course produce a false 5439 // negative, in which case we would take the ArraySet slow 5440 // path. 5441 5442 // /* HeapReference<Class> */ temp1 = array->klass_ 5443 __ LoadFromOffset(kLoadWord, temp1, array, class_offset); 5444 codegen_->MaybeRecordImplicitNullCheck(instruction); 5445 __ MaybeUnpoisonHeapReference(temp1); 5446 5447 // /* HeapReference<Class> */ temp1 = temp1->component_type_ 5448 __ LoadFromOffset(kLoadWord, temp1, temp1, component_offset); 5449 // /* HeapReference<Class> */ temp2 = value->klass_ 5450 __ LoadFromOffset(kLoadWord, temp2, value, class_offset); 5451 // If heap poisoning is enabled, no need to unpoison `temp1` 5452 // nor `temp2`, as we are comparing two poisoned references. 5453 __ cmp(temp1, ShifterOperand(temp2)); 5454 5455 if (instruction->StaticTypeOfArrayIsObjectArray()) { 5456 Label do_put; 5457 __ b(&do_put, EQ); 5458 // If heap poisoning is enabled, the `temp1` reference has 5459 // not been unpoisoned yet; unpoison it now. 5460 __ MaybeUnpoisonHeapReference(temp1); 5461 5462 // /* HeapReference<Class> */ temp1 = temp1->super_class_ 5463 __ LoadFromOffset(kLoadWord, temp1, temp1, super_offset); 5464 // If heap poisoning is enabled, no need to unpoison 5465 // `temp1`, as we are comparing against null below. 5466 __ CompareAndBranchIfNonZero(temp1, slow_path->GetEntryLabel()); 5467 __ Bind(&do_put); 5468 } else { 5469 __ b(slow_path->GetEntryLabel(), NE); 5470 } 5471 } 5472 5473 Register source = value; 5474 if (kPoisonHeapReferences) { 5475 // Note that in the case where `value` is a null reference, 5476 // we do not enter this block, as a null reference does not 5477 // need poisoning. 5478 DCHECK_EQ(value_type, Primitive::kPrimNot); 5479 __ Mov(temp1, value); 5480 __ PoisonHeapReference(temp1); 5481 source = temp1; 5482 } 5483 5484 if (index.IsConstant()) { 5485 size_t offset = 5486 (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4) + data_offset; 5487 __ StoreToOffset(kStoreWord, source, array, offset); 5488 } else { 5489 DCHECK(index.IsRegister()) << index; 5490 5491 __ add(IP, array, ShifterOperand(data_offset)); 5492 codegen_->StoreToShiftedRegOffset(value_type, 5493 Location::RegisterLocation(source), 5494 IP, 5495 index.AsRegister<Register>()); 5496 } 5497 5498 if (!may_need_runtime_call_for_type_check) { 5499 codegen_->MaybeRecordImplicitNullCheck(instruction); 5500 } 5501 5502 codegen_->MarkGCCard(temp1, temp2, array, value, instruction->GetValueCanBeNull()); 5503 5504 if (done.IsLinked()) { 5505 __ Bind(&done); 5506 } 5507 5508 if (slow_path != nullptr) { 5509 __ Bind(slow_path->GetExitLabel()); 5510 } 5511 5512 break; 5513 } 5514 5515 case Primitive::kPrimLong: { 5516 Location value = locations->InAt(2); 5517 if (index.IsConstant()) { 5518 size_t offset = 5519 (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_8) + data_offset; 5520 __ StoreToOffset(kStoreWordPair, value.AsRegisterPairLow<Register>(), array, offset); 5521 } else { 5522 __ add(IP, array, ShifterOperand(index.AsRegister<Register>(), LSL, TIMES_8)); 5523 __ StoreToOffset(kStoreWordPair, value.AsRegisterPairLow<Register>(), IP, data_offset); 5524 } 5525 break; 5526 } 5527 5528 case Primitive::kPrimFloat: { 5529 Location value = locations->InAt(2); 5530 DCHECK(value.IsFpuRegister()); 5531 if (index.IsConstant()) { 5532 size_t offset = (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4) + data_offset; 5533 __ StoreSToOffset(value.AsFpuRegister<SRegister>(), array, offset); 5534 } else { 5535 __ add(IP, array, ShifterOperand(index.AsRegister<Register>(), LSL, TIMES_4)); 5536 __ StoreSToOffset(value.AsFpuRegister<SRegister>(), IP, data_offset); 5537 } 5538 break; 5539 } 5540 5541 case Primitive::kPrimDouble: { 5542 Location value = locations->InAt(2); 5543 DCHECK(value.IsFpuRegisterPair()); 5544 if (index.IsConstant()) { 5545 size_t offset = (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_8) + data_offset; 5546 __ StoreDToOffset(FromLowSToD(value.AsFpuRegisterPairLow<SRegister>()), array, offset); 5547 } else { 5548 __ add(IP, array, ShifterOperand(index.AsRegister<Register>(), LSL, TIMES_8)); 5549 __ StoreDToOffset(FromLowSToD(value.AsFpuRegisterPairLow<SRegister>()), IP, data_offset); 5550 } 5551 5552 break; 5553 } 5554 5555 case Primitive::kPrimVoid: 5556 LOG(FATAL) << "Unreachable type " << value_type; 5557 UNREACHABLE(); 5558 } 5559 5560 // Objects are handled in the switch. 5561 if (value_type != Primitive::kPrimNot) { 5562 codegen_->MaybeRecordImplicitNullCheck(instruction); 5563 } 5564} 5565 5566void LocationsBuilderARM::VisitArrayLength(HArrayLength* instruction) { 5567 LocationSummary* locations = 5568 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall); 5569 locations->SetInAt(0, Location::RequiresRegister()); 5570 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 5571} 5572 5573void InstructionCodeGeneratorARM::VisitArrayLength(HArrayLength* instruction) { 5574 LocationSummary* locations = instruction->GetLocations(); 5575 uint32_t offset = CodeGenerator::GetArrayLengthOffset(instruction); 5576 Register obj = locations->InAt(0).AsRegister<Register>(); 5577 Register out = locations->Out().AsRegister<Register>(); 5578 __ LoadFromOffset(kLoadWord, out, obj, offset); 5579 codegen_->MaybeRecordImplicitNullCheck(instruction); 5580 // Mask out compression flag from String's array length. 5581 if (mirror::kUseStringCompression && instruction->IsStringLength()) { 5582 __ Lsr(out, out, 1u); 5583 } 5584} 5585 5586void LocationsBuilderARM::VisitIntermediateAddress(HIntermediateAddress* instruction) { 5587 LocationSummary* locations = 5588 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall); 5589 5590 locations->SetInAt(0, Location::RequiresRegister()); 5591 locations->SetInAt(1, Location::RegisterOrConstant(instruction->GetOffset())); 5592 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 5593} 5594 5595void InstructionCodeGeneratorARM::VisitIntermediateAddress(HIntermediateAddress* instruction) { 5596 LocationSummary* locations = instruction->GetLocations(); 5597 Location out = locations->Out(); 5598 Location first = locations->InAt(0); 5599 Location second = locations->InAt(1); 5600 5601 if (second.IsRegister()) { 5602 __ add(out.AsRegister<Register>(), 5603 first.AsRegister<Register>(), 5604 ShifterOperand(second.AsRegister<Register>())); 5605 } else { 5606 __ AddConstant(out.AsRegister<Register>(), 5607 first.AsRegister<Register>(), 5608 second.GetConstant()->AsIntConstant()->GetValue()); 5609 } 5610} 5611 5612void LocationsBuilderARM::VisitBoundsCheck(HBoundsCheck* instruction) { 5613 RegisterSet caller_saves = RegisterSet::Empty(); 5614 InvokeRuntimeCallingConvention calling_convention; 5615 caller_saves.Add(Location::RegisterLocation(calling_convention.GetRegisterAt(0))); 5616 caller_saves.Add(Location::RegisterLocation(calling_convention.GetRegisterAt(1))); 5617 LocationSummary* locations = codegen_->CreateThrowingSlowPathLocations(instruction, caller_saves); 5618 locations->SetInAt(0, Location::RequiresRegister()); 5619 locations->SetInAt(1, Location::RequiresRegister()); 5620} 5621 5622void InstructionCodeGeneratorARM::VisitBoundsCheck(HBoundsCheck* instruction) { 5623 LocationSummary* locations = instruction->GetLocations(); 5624 SlowPathCodeARM* slow_path = 5625 new (GetGraph()->GetArena()) BoundsCheckSlowPathARM(instruction); 5626 codegen_->AddSlowPath(slow_path); 5627 5628 Register index = locations->InAt(0).AsRegister<Register>(); 5629 Register length = locations->InAt(1).AsRegister<Register>(); 5630 5631 __ cmp(index, ShifterOperand(length)); 5632 __ b(slow_path->GetEntryLabel(), HS); 5633} 5634 5635void CodeGeneratorARM::MarkGCCard(Register temp, 5636 Register card, 5637 Register object, 5638 Register value, 5639 bool can_be_null) { 5640 Label is_null; 5641 if (can_be_null) { 5642 __ CompareAndBranchIfZero(value, &is_null); 5643 } 5644 __ LoadFromOffset(kLoadWord, card, TR, Thread::CardTableOffset<kArmPointerSize>().Int32Value()); 5645 __ Lsr(temp, object, gc::accounting::CardTable::kCardShift); 5646 __ strb(card, Address(card, temp)); 5647 if (can_be_null) { 5648 __ Bind(&is_null); 5649 } 5650} 5651 5652void LocationsBuilderARM::VisitParallelMove(HParallelMove* instruction ATTRIBUTE_UNUSED) { 5653 LOG(FATAL) << "Unreachable"; 5654} 5655 5656void InstructionCodeGeneratorARM::VisitParallelMove(HParallelMove* instruction) { 5657 codegen_->GetMoveResolver()->EmitNativeCode(instruction); 5658} 5659 5660void LocationsBuilderARM::VisitSuspendCheck(HSuspendCheck* instruction) { 5661 LocationSummary* locations = 5662 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCallOnSlowPath); 5663 locations->SetCustomSlowPathCallerSaves(RegisterSet::Empty()); // No caller-save registers. 5664} 5665 5666void InstructionCodeGeneratorARM::VisitSuspendCheck(HSuspendCheck* instruction) { 5667 HBasicBlock* block = instruction->GetBlock(); 5668 if (block->GetLoopInformation() != nullptr) { 5669 DCHECK(block->GetLoopInformation()->GetSuspendCheck() == instruction); 5670 // The back edge will generate the suspend check. 5671 return; 5672 } 5673 if (block->IsEntryBlock() && instruction->GetNext()->IsGoto()) { 5674 // The goto will generate the suspend check. 5675 return; 5676 } 5677 GenerateSuspendCheck(instruction, nullptr); 5678} 5679 5680void InstructionCodeGeneratorARM::GenerateSuspendCheck(HSuspendCheck* instruction, 5681 HBasicBlock* successor) { 5682 SuspendCheckSlowPathARM* slow_path = 5683 down_cast<SuspendCheckSlowPathARM*>(instruction->GetSlowPath()); 5684 if (slow_path == nullptr) { 5685 slow_path = new (GetGraph()->GetArena()) SuspendCheckSlowPathARM(instruction, successor); 5686 instruction->SetSlowPath(slow_path); 5687 codegen_->AddSlowPath(slow_path); 5688 if (successor != nullptr) { 5689 DCHECK(successor->IsLoopHeader()); 5690 codegen_->ClearSpillSlotsFromLoopPhisInStackMap(instruction); 5691 } 5692 } else { 5693 DCHECK_EQ(slow_path->GetSuccessor(), successor); 5694 } 5695 5696 __ LoadFromOffset( 5697 kLoadUnsignedHalfword, IP, TR, Thread::ThreadFlagsOffset<kArmPointerSize>().Int32Value()); 5698 if (successor == nullptr) { 5699 __ CompareAndBranchIfNonZero(IP, slow_path->GetEntryLabel()); 5700 __ Bind(slow_path->GetReturnLabel()); 5701 } else { 5702 __ CompareAndBranchIfZero(IP, codegen_->GetLabelOf(successor)); 5703 __ b(slow_path->GetEntryLabel()); 5704 } 5705} 5706 5707ArmAssembler* ParallelMoveResolverARM::GetAssembler() const { 5708 return codegen_->GetAssembler(); 5709} 5710 5711void ParallelMoveResolverARM::EmitMove(size_t index) { 5712 MoveOperands* move = moves_[index]; 5713 Location source = move->GetSource(); 5714 Location destination = move->GetDestination(); 5715 5716 if (source.IsRegister()) { 5717 if (destination.IsRegister()) { 5718 __ Mov(destination.AsRegister<Register>(), source.AsRegister<Register>()); 5719 } else if (destination.IsFpuRegister()) { 5720 __ vmovsr(destination.AsFpuRegister<SRegister>(), source.AsRegister<Register>()); 5721 } else { 5722 DCHECK(destination.IsStackSlot()); 5723 __ StoreToOffset(kStoreWord, source.AsRegister<Register>(), 5724 SP, destination.GetStackIndex()); 5725 } 5726 } else if (source.IsStackSlot()) { 5727 if (destination.IsRegister()) { 5728 __ LoadFromOffset(kLoadWord, destination.AsRegister<Register>(), 5729 SP, source.GetStackIndex()); 5730 } else if (destination.IsFpuRegister()) { 5731 __ LoadSFromOffset(destination.AsFpuRegister<SRegister>(), SP, source.GetStackIndex()); 5732 } else { 5733 DCHECK(destination.IsStackSlot()); 5734 __ LoadFromOffset(kLoadWord, IP, SP, source.GetStackIndex()); 5735 __ StoreToOffset(kStoreWord, IP, SP, destination.GetStackIndex()); 5736 } 5737 } else if (source.IsFpuRegister()) { 5738 if (destination.IsRegister()) { 5739 __ vmovrs(destination.AsRegister<Register>(), source.AsFpuRegister<SRegister>()); 5740 } else if (destination.IsFpuRegister()) { 5741 __ vmovs(destination.AsFpuRegister<SRegister>(), source.AsFpuRegister<SRegister>()); 5742 } else { 5743 DCHECK(destination.IsStackSlot()); 5744 __ StoreSToOffset(source.AsFpuRegister<SRegister>(), SP, destination.GetStackIndex()); 5745 } 5746 } else if (source.IsDoubleStackSlot()) { 5747 if (destination.IsDoubleStackSlot()) { 5748 __ LoadDFromOffset(DTMP, SP, source.GetStackIndex()); 5749 __ StoreDToOffset(DTMP, SP, destination.GetStackIndex()); 5750 } else if (destination.IsRegisterPair()) { 5751 DCHECK(ExpectedPairLayout(destination)); 5752 __ LoadFromOffset( 5753 kLoadWordPair, destination.AsRegisterPairLow<Register>(), SP, source.GetStackIndex()); 5754 } else { 5755 DCHECK(destination.IsFpuRegisterPair()) << destination; 5756 __ LoadDFromOffset(FromLowSToD(destination.AsFpuRegisterPairLow<SRegister>()), 5757 SP, 5758 source.GetStackIndex()); 5759 } 5760 } else if (source.IsRegisterPair()) { 5761 if (destination.IsRegisterPair()) { 5762 __ Mov(destination.AsRegisterPairLow<Register>(), source.AsRegisterPairLow<Register>()); 5763 __ Mov(destination.AsRegisterPairHigh<Register>(), source.AsRegisterPairHigh<Register>()); 5764 } else if (destination.IsFpuRegisterPair()) { 5765 __ vmovdrr(FromLowSToD(destination.AsFpuRegisterPairLow<SRegister>()), 5766 source.AsRegisterPairLow<Register>(), 5767 source.AsRegisterPairHigh<Register>()); 5768 } else { 5769 DCHECK(destination.IsDoubleStackSlot()) << destination; 5770 DCHECK(ExpectedPairLayout(source)); 5771 __ StoreToOffset( 5772 kStoreWordPair, source.AsRegisterPairLow<Register>(), SP, destination.GetStackIndex()); 5773 } 5774 } else if (source.IsFpuRegisterPair()) { 5775 if (destination.IsRegisterPair()) { 5776 __ vmovrrd(destination.AsRegisterPairLow<Register>(), 5777 destination.AsRegisterPairHigh<Register>(), 5778 FromLowSToD(source.AsFpuRegisterPairLow<SRegister>())); 5779 } else if (destination.IsFpuRegisterPair()) { 5780 __ vmovd(FromLowSToD(destination.AsFpuRegisterPairLow<SRegister>()), 5781 FromLowSToD(source.AsFpuRegisterPairLow<SRegister>())); 5782 } else { 5783 DCHECK(destination.IsDoubleStackSlot()) << destination; 5784 __ StoreDToOffset(FromLowSToD(source.AsFpuRegisterPairLow<SRegister>()), 5785 SP, 5786 destination.GetStackIndex()); 5787 } 5788 } else { 5789 DCHECK(source.IsConstant()) << source; 5790 HConstant* constant = source.GetConstant(); 5791 if (constant->IsIntConstant() || constant->IsNullConstant()) { 5792 int32_t value = CodeGenerator::GetInt32ValueOf(constant); 5793 if (destination.IsRegister()) { 5794 __ LoadImmediate(destination.AsRegister<Register>(), value); 5795 } else { 5796 DCHECK(destination.IsStackSlot()); 5797 __ LoadImmediate(IP, value); 5798 __ StoreToOffset(kStoreWord, IP, SP, destination.GetStackIndex()); 5799 } 5800 } else if (constant->IsLongConstant()) { 5801 int64_t value = constant->AsLongConstant()->GetValue(); 5802 if (destination.IsRegisterPair()) { 5803 __ LoadImmediate(destination.AsRegisterPairLow<Register>(), Low32Bits(value)); 5804 __ LoadImmediate(destination.AsRegisterPairHigh<Register>(), High32Bits(value)); 5805 } else { 5806 DCHECK(destination.IsDoubleStackSlot()) << destination; 5807 __ LoadImmediate(IP, Low32Bits(value)); 5808 __ StoreToOffset(kStoreWord, IP, SP, destination.GetStackIndex()); 5809 __ LoadImmediate(IP, High32Bits(value)); 5810 __ StoreToOffset(kStoreWord, IP, SP, destination.GetHighStackIndex(kArmWordSize)); 5811 } 5812 } else if (constant->IsDoubleConstant()) { 5813 double value = constant->AsDoubleConstant()->GetValue(); 5814 if (destination.IsFpuRegisterPair()) { 5815 __ LoadDImmediate(FromLowSToD(destination.AsFpuRegisterPairLow<SRegister>()), value); 5816 } else { 5817 DCHECK(destination.IsDoubleStackSlot()) << destination; 5818 uint64_t int_value = bit_cast<uint64_t, double>(value); 5819 __ LoadImmediate(IP, Low32Bits(int_value)); 5820 __ StoreToOffset(kStoreWord, IP, SP, destination.GetStackIndex()); 5821 __ LoadImmediate(IP, High32Bits(int_value)); 5822 __ StoreToOffset(kStoreWord, IP, SP, destination.GetHighStackIndex(kArmWordSize)); 5823 } 5824 } else { 5825 DCHECK(constant->IsFloatConstant()) << constant->DebugName(); 5826 float value = constant->AsFloatConstant()->GetValue(); 5827 if (destination.IsFpuRegister()) { 5828 __ LoadSImmediate(destination.AsFpuRegister<SRegister>(), value); 5829 } else { 5830 DCHECK(destination.IsStackSlot()); 5831 __ LoadImmediate(IP, bit_cast<int32_t, float>(value)); 5832 __ StoreToOffset(kStoreWord, IP, SP, destination.GetStackIndex()); 5833 } 5834 } 5835 } 5836} 5837 5838void ParallelMoveResolverARM::Exchange(Register reg, int mem) { 5839 __ Mov(IP, reg); 5840 __ LoadFromOffset(kLoadWord, reg, SP, mem); 5841 __ StoreToOffset(kStoreWord, IP, SP, mem); 5842} 5843 5844void ParallelMoveResolverARM::Exchange(int mem1, int mem2) { 5845 ScratchRegisterScope ensure_scratch(this, IP, R0, codegen_->GetNumberOfCoreRegisters()); 5846 int stack_offset = ensure_scratch.IsSpilled() ? kArmWordSize : 0; 5847 __ LoadFromOffset(kLoadWord, static_cast<Register>(ensure_scratch.GetRegister()), 5848 SP, mem1 + stack_offset); 5849 __ LoadFromOffset(kLoadWord, IP, SP, mem2 + stack_offset); 5850 __ StoreToOffset(kStoreWord, static_cast<Register>(ensure_scratch.GetRegister()), 5851 SP, mem2 + stack_offset); 5852 __ StoreToOffset(kStoreWord, IP, SP, mem1 + stack_offset); 5853} 5854 5855void ParallelMoveResolverARM::EmitSwap(size_t index) { 5856 MoveOperands* move = moves_[index]; 5857 Location source = move->GetSource(); 5858 Location destination = move->GetDestination(); 5859 5860 if (source.IsRegister() && destination.IsRegister()) { 5861 DCHECK_NE(source.AsRegister<Register>(), IP); 5862 DCHECK_NE(destination.AsRegister<Register>(), IP); 5863 __ Mov(IP, source.AsRegister<Register>()); 5864 __ Mov(source.AsRegister<Register>(), destination.AsRegister<Register>()); 5865 __ Mov(destination.AsRegister<Register>(), IP); 5866 } else if (source.IsRegister() && destination.IsStackSlot()) { 5867 Exchange(source.AsRegister<Register>(), destination.GetStackIndex()); 5868 } else if (source.IsStackSlot() && destination.IsRegister()) { 5869 Exchange(destination.AsRegister<Register>(), source.GetStackIndex()); 5870 } else if (source.IsStackSlot() && destination.IsStackSlot()) { 5871 Exchange(source.GetStackIndex(), destination.GetStackIndex()); 5872 } else if (source.IsFpuRegister() && destination.IsFpuRegister()) { 5873 __ vmovrs(IP, source.AsFpuRegister<SRegister>()); 5874 __ vmovs(source.AsFpuRegister<SRegister>(), destination.AsFpuRegister<SRegister>()); 5875 __ vmovsr(destination.AsFpuRegister<SRegister>(), IP); 5876 } else if (source.IsRegisterPair() && destination.IsRegisterPair()) { 5877 __ vmovdrr(DTMP, source.AsRegisterPairLow<Register>(), source.AsRegisterPairHigh<Register>()); 5878 __ Mov(source.AsRegisterPairLow<Register>(), destination.AsRegisterPairLow<Register>()); 5879 __ Mov(source.AsRegisterPairHigh<Register>(), destination.AsRegisterPairHigh<Register>()); 5880 __ vmovrrd(destination.AsRegisterPairLow<Register>(), 5881 destination.AsRegisterPairHigh<Register>(), 5882 DTMP); 5883 } else if (source.IsRegisterPair() || destination.IsRegisterPair()) { 5884 Register low_reg = source.IsRegisterPair() 5885 ? source.AsRegisterPairLow<Register>() 5886 : destination.AsRegisterPairLow<Register>(); 5887 int mem = source.IsRegisterPair() 5888 ? destination.GetStackIndex() 5889 : source.GetStackIndex(); 5890 DCHECK(ExpectedPairLayout(source.IsRegisterPair() ? source : destination)); 5891 __ vmovdrr(DTMP, low_reg, static_cast<Register>(low_reg + 1)); 5892 __ LoadFromOffset(kLoadWordPair, low_reg, SP, mem); 5893 __ StoreDToOffset(DTMP, SP, mem); 5894 } else if (source.IsFpuRegisterPair() && destination.IsFpuRegisterPair()) { 5895 DRegister first = FromLowSToD(source.AsFpuRegisterPairLow<SRegister>()); 5896 DRegister second = FromLowSToD(destination.AsFpuRegisterPairLow<SRegister>()); 5897 __ vmovd(DTMP, first); 5898 __ vmovd(first, second); 5899 __ vmovd(second, DTMP); 5900 } else if (source.IsFpuRegisterPair() || destination.IsFpuRegisterPair()) { 5901 DRegister reg = source.IsFpuRegisterPair() 5902 ? FromLowSToD(source.AsFpuRegisterPairLow<SRegister>()) 5903 : FromLowSToD(destination.AsFpuRegisterPairLow<SRegister>()); 5904 int mem = source.IsFpuRegisterPair() 5905 ? destination.GetStackIndex() 5906 : source.GetStackIndex(); 5907 __ vmovd(DTMP, reg); 5908 __ LoadDFromOffset(reg, SP, mem); 5909 __ StoreDToOffset(DTMP, SP, mem); 5910 } else if (source.IsFpuRegister() || destination.IsFpuRegister()) { 5911 SRegister reg = source.IsFpuRegister() ? source.AsFpuRegister<SRegister>() 5912 : destination.AsFpuRegister<SRegister>(); 5913 int mem = source.IsFpuRegister() 5914 ? destination.GetStackIndex() 5915 : source.GetStackIndex(); 5916 5917 __ vmovrs(IP, reg); 5918 __ LoadSFromOffset(reg, SP, mem); 5919 __ StoreToOffset(kStoreWord, IP, SP, mem); 5920 } else if (source.IsDoubleStackSlot() && destination.IsDoubleStackSlot()) { 5921 Exchange(source.GetStackIndex(), destination.GetStackIndex()); 5922 Exchange(source.GetHighStackIndex(kArmWordSize), destination.GetHighStackIndex(kArmWordSize)); 5923 } else { 5924 LOG(FATAL) << "Unimplemented" << source << " <-> " << destination; 5925 } 5926} 5927 5928void ParallelMoveResolverARM::SpillScratch(int reg) { 5929 __ Push(static_cast<Register>(reg)); 5930} 5931 5932void ParallelMoveResolverARM::RestoreScratch(int reg) { 5933 __ Pop(static_cast<Register>(reg)); 5934} 5935 5936HLoadClass::LoadKind CodeGeneratorARM::GetSupportedLoadClassKind( 5937 HLoadClass::LoadKind desired_class_load_kind) { 5938 switch (desired_class_load_kind) { 5939 case HLoadClass::LoadKind::kInvalid: 5940 LOG(FATAL) << "UNREACHABLE"; 5941 UNREACHABLE(); 5942 case HLoadClass::LoadKind::kReferrersClass: 5943 break; 5944 case HLoadClass::LoadKind::kBootImageLinkTimeAddress: 5945 DCHECK(!GetCompilerOptions().GetCompilePic()); 5946 break; 5947 case HLoadClass::LoadKind::kBootImageLinkTimePcRelative: 5948 DCHECK(GetCompilerOptions().GetCompilePic()); 5949 break; 5950 case HLoadClass::LoadKind::kBootImageAddress: 5951 break; 5952 case HLoadClass::LoadKind::kBssEntry: 5953 DCHECK(!Runtime::Current()->UseJitCompilation()); 5954 break; 5955 case HLoadClass::LoadKind::kJitTableAddress: 5956 DCHECK(Runtime::Current()->UseJitCompilation()); 5957 break; 5958 case HLoadClass::LoadKind::kDexCacheViaMethod: 5959 break; 5960 } 5961 return desired_class_load_kind; 5962} 5963 5964void LocationsBuilderARM::VisitLoadClass(HLoadClass* cls) { 5965 HLoadClass::LoadKind load_kind = cls->GetLoadKind(); 5966 if (load_kind == HLoadClass::LoadKind::kDexCacheViaMethod) { 5967 InvokeRuntimeCallingConvention calling_convention; 5968 CodeGenerator::CreateLoadClassRuntimeCallLocationSummary( 5969 cls, 5970 Location::RegisterLocation(calling_convention.GetRegisterAt(0)), 5971 Location::RegisterLocation(R0)); 5972 DCHECK_EQ(calling_convention.GetRegisterAt(0), R0); 5973 return; 5974 } 5975 DCHECK(!cls->NeedsAccessCheck()); 5976 5977 const bool requires_read_barrier = kEmitCompilerReadBarrier && !cls->IsInBootImage(); 5978 LocationSummary::CallKind call_kind = (cls->NeedsEnvironment() || requires_read_barrier) 5979 ? LocationSummary::kCallOnSlowPath 5980 : LocationSummary::kNoCall; 5981 LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(cls, call_kind); 5982 if (kUseBakerReadBarrier && requires_read_barrier && !cls->NeedsEnvironment()) { 5983 locations->SetCustomSlowPathCallerSaves(RegisterSet::Empty()); // No caller-save registers. 5984 } 5985 5986 if (load_kind == HLoadClass::LoadKind::kReferrersClass) { 5987 locations->SetInAt(0, Location::RequiresRegister()); 5988 } 5989 locations->SetOut(Location::RequiresRegister()); 5990 if (load_kind == HLoadClass::LoadKind::kBssEntry) { 5991 if (!kUseReadBarrier || kUseBakerReadBarrier) { 5992 // Rely on the type resolution or initialization and marking to save everything we need. 5993 // Note that IP may be clobbered by saving/restoring the live register (only one thanks 5994 // to the custom calling convention) or by marking, so we request a different temp. 5995 locations->AddTemp(Location::RequiresRegister()); 5996 RegisterSet caller_saves = RegisterSet::Empty(); 5997 InvokeRuntimeCallingConvention calling_convention; 5998 caller_saves.Add(Location::RegisterLocation(calling_convention.GetRegisterAt(0))); 5999 // TODO: Add GetReturnLocation() to the calling convention so that we can DCHECK() 6000 // that the the kPrimNot result register is the same as the first argument register. 6001 locations->SetCustomSlowPathCallerSaves(caller_saves); 6002 } else { 6003 // For non-Baker read barrier we have a temp-clobbering call. 6004 } 6005 } 6006} 6007 6008// NO_THREAD_SAFETY_ANALYSIS as we manipulate handles whose internal object we know does not 6009// move. 6010void InstructionCodeGeneratorARM::VisitLoadClass(HLoadClass* cls) NO_THREAD_SAFETY_ANALYSIS { 6011 HLoadClass::LoadKind load_kind = cls->GetLoadKind(); 6012 if (load_kind == HLoadClass::LoadKind::kDexCacheViaMethod) { 6013 codegen_->GenerateLoadClassRuntimeCall(cls); 6014 return; 6015 } 6016 DCHECK(!cls->NeedsAccessCheck()); 6017 6018 LocationSummary* locations = cls->GetLocations(); 6019 Location out_loc = locations->Out(); 6020 Register out = out_loc.AsRegister<Register>(); 6021 6022 const ReadBarrierOption read_barrier_option = cls->IsInBootImage() 6023 ? kWithoutReadBarrier 6024 : kCompilerReadBarrierOption; 6025 bool generate_null_check = false; 6026 switch (load_kind) { 6027 case HLoadClass::LoadKind::kReferrersClass: { 6028 DCHECK(!cls->CanCallRuntime()); 6029 DCHECK(!cls->MustGenerateClinitCheck()); 6030 // /* GcRoot<mirror::Class> */ out = current_method->declaring_class_ 6031 Register current_method = locations->InAt(0).AsRegister<Register>(); 6032 GenerateGcRootFieldLoad(cls, 6033 out_loc, 6034 current_method, 6035 ArtMethod::DeclaringClassOffset().Int32Value(), 6036 read_barrier_option); 6037 break; 6038 } 6039 case HLoadClass::LoadKind::kBootImageLinkTimeAddress: { 6040 DCHECK(codegen_->GetCompilerOptions().IsBootImage()); 6041 DCHECK_EQ(read_barrier_option, kWithoutReadBarrier); 6042 __ LoadLiteral(out, codegen_->DeduplicateBootImageTypeLiteral(cls->GetDexFile(), 6043 cls->GetTypeIndex())); 6044 break; 6045 } 6046 case HLoadClass::LoadKind::kBootImageLinkTimePcRelative: { 6047 DCHECK(codegen_->GetCompilerOptions().IsBootImage()); 6048 DCHECK_EQ(read_barrier_option, kWithoutReadBarrier); 6049 CodeGeneratorARM::PcRelativePatchInfo* labels = 6050 codegen_->NewPcRelativeTypePatch(cls->GetDexFile(), cls->GetTypeIndex()); 6051 __ BindTrackedLabel(&labels->movw_label); 6052 __ movw(out, /* placeholder */ 0u); 6053 __ BindTrackedLabel(&labels->movt_label); 6054 __ movt(out, /* placeholder */ 0u); 6055 __ BindTrackedLabel(&labels->add_pc_label); 6056 __ add(out, out, ShifterOperand(PC)); 6057 break; 6058 } 6059 case HLoadClass::LoadKind::kBootImageAddress: { 6060 DCHECK_EQ(read_barrier_option, kWithoutReadBarrier); 6061 uint32_t address = dchecked_integral_cast<uint32_t>( 6062 reinterpret_cast<uintptr_t>(cls->GetClass().Get())); 6063 DCHECK_NE(address, 0u); 6064 __ LoadLiteral(out, codegen_->DeduplicateBootImageAddressLiteral(address)); 6065 break; 6066 } 6067 case HLoadClass::LoadKind::kBssEntry: { 6068 Register temp = (!kUseReadBarrier || kUseBakerReadBarrier) 6069 ? locations->GetTemp(0).AsRegister<Register>() 6070 : out; 6071 CodeGeneratorARM::PcRelativePatchInfo* labels = 6072 codegen_->NewTypeBssEntryPatch(cls->GetDexFile(), cls->GetTypeIndex()); 6073 __ BindTrackedLabel(&labels->movw_label); 6074 __ movw(temp, /* placeholder */ 0u); 6075 __ BindTrackedLabel(&labels->movt_label); 6076 __ movt(temp, /* placeholder */ 0u); 6077 __ BindTrackedLabel(&labels->add_pc_label); 6078 __ add(temp, temp, ShifterOperand(PC)); 6079 GenerateGcRootFieldLoad(cls, out_loc, temp, /* offset */ 0, read_barrier_option); 6080 generate_null_check = true; 6081 break; 6082 } 6083 case HLoadClass::LoadKind::kJitTableAddress: { 6084 __ LoadLiteral(out, codegen_->DeduplicateJitClassLiteral(cls->GetDexFile(), 6085 cls->GetTypeIndex(), 6086 cls->GetClass())); 6087 // /* GcRoot<mirror::Class> */ out = *out 6088 GenerateGcRootFieldLoad(cls, out_loc, out, /* offset */ 0, read_barrier_option); 6089 break; 6090 } 6091 case HLoadClass::LoadKind::kDexCacheViaMethod: 6092 case HLoadClass::LoadKind::kInvalid: 6093 LOG(FATAL) << "UNREACHABLE"; 6094 UNREACHABLE(); 6095 } 6096 6097 if (generate_null_check || cls->MustGenerateClinitCheck()) { 6098 DCHECK(cls->CanCallRuntime()); 6099 SlowPathCodeARM* slow_path = new (GetGraph()->GetArena()) LoadClassSlowPathARM( 6100 cls, cls, cls->GetDexPc(), cls->MustGenerateClinitCheck()); 6101 codegen_->AddSlowPath(slow_path); 6102 if (generate_null_check) { 6103 __ CompareAndBranchIfZero(out, slow_path->GetEntryLabel()); 6104 } 6105 if (cls->MustGenerateClinitCheck()) { 6106 GenerateClassInitializationCheck(slow_path, out); 6107 } else { 6108 __ Bind(slow_path->GetExitLabel()); 6109 } 6110 } 6111} 6112 6113void LocationsBuilderARM::VisitClinitCheck(HClinitCheck* check) { 6114 LocationSummary* locations = 6115 new (GetGraph()->GetArena()) LocationSummary(check, LocationSummary::kCallOnSlowPath); 6116 locations->SetInAt(0, Location::RequiresRegister()); 6117 if (check->HasUses()) { 6118 locations->SetOut(Location::SameAsFirstInput()); 6119 } 6120} 6121 6122void InstructionCodeGeneratorARM::VisitClinitCheck(HClinitCheck* check) { 6123 // We assume the class is not null. 6124 SlowPathCodeARM* slow_path = new (GetGraph()->GetArena()) LoadClassSlowPathARM( 6125 check->GetLoadClass(), check, check->GetDexPc(), true); 6126 codegen_->AddSlowPath(slow_path); 6127 GenerateClassInitializationCheck(slow_path, 6128 check->GetLocations()->InAt(0).AsRegister<Register>()); 6129} 6130 6131void InstructionCodeGeneratorARM::GenerateClassInitializationCheck( 6132 SlowPathCodeARM* slow_path, Register class_reg) { 6133 __ LoadFromOffset(kLoadWord, IP, class_reg, mirror::Class::StatusOffset().Int32Value()); 6134 __ cmp(IP, ShifterOperand(mirror::Class::kStatusInitialized)); 6135 __ b(slow_path->GetEntryLabel(), LT); 6136 // Even if the initialized flag is set, we may be in a situation where caches are not synced 6137 // properly. Therefore, we do a memory fence. 6138 __ dmb(ISH); 6139 __ Bind(slow_path->GetExitLabel()); 6140} 6141 6142HLoadString::LoadKind CodeGeneratorARM::GetSupportedLoadStringKind( 6143 HLoadString::LoadKind desired_string_load_kind) { 6144 switch (desired_string_load_kind) { 6145 case HLoadString::LoadKind::kBootImageLinkTimeAddress: 6146 DCHECK(!GetCompilerOptions().GetCompilePic()); 6147 break; 6148 case HLoadString::LoadKind::kBootImageLinkTimePcRelative: 6149 DCHECK(GetCompilerOptions().GetCompilePic()); 6150 break; 6151 case HLoadString::LoadKind::kBootImageAddress: 6152 break; 6153 case HLoadString::LoadKind::kBssEntry: 6154 DCHECK(!Runtime::Current()->UseJitCompilation()); 6155 break; 6156 case HLoadString::LoadKind::kJitTableAddress: 6157 DCHECK(Runtime::Current()->UseJitCompilation()); 6158 break; 6159 case HLoadString::LoadKind::kDexCacheViaMethod: 6160 break; 6161 } 6162 return desired_string_load_kind; 6163} 6164 6165void LocationsBuilderARM::VisitLoadString(HLoadString* load) { 6166 LocationSummary::CallKind call_kind = CodeGenerator::GetLoadStringCallKind(load); 6167 LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(load, call_kind); 6168 HLoadString::LoadKind load_kind = load->GetLoadKind(); 6169 if (load_kind == HLoadString::LoadKind::kDexCacheViaMethod) { 6170 locations->SetOut(Location::RegisterLocation(R0)); 6171 } else { 6172 locations->SetOut(Location::RequiresRegister()); 6173 if (load_kind == HLoadString::LoadKind::kBssEntry) { 6174 if (!kUseReadBarrier || kUseBakerReadBarrier) { 6175 // Rely on the pResolveString and marking to save everything we need, including temps. 6176 // Note that IP may be clobbered by saving/restoring the live register (only one thanks 6177 // to the custom calling convention) or by marking, so we request a different temp. 6178 locations->AddTemp(Location::RequiresRegister()); 6179 RegisterSet caller_saves = RegisterSet::Empty(); 6180 InvokeRuntimeCallingConvention calling_convention; 6181 caller_saves.Add(Location::RegisterLocation(calling_convention.GetRegisterAt(0))); 6182 // TODO: Add GetReturnLocation() to the calling convention so that we can DCHECK() 6183 // that the the kPrimNot result register is the same as the first argument register. 6184 locations->SetCustomSlowPathCallerSaves(caller_saves); 6185 } else { 6186 // For non-Baker read barrier we have a temp-clobbering call. 6187 } 6188 } 6189 } 6190} 6191 6192// NO_THREAD_SAFETY_ANALYSIS as we manipulate handles whose internal object we know does not 6193// move. 6194void InstructionCodeGeneratorARM::VisitLoadString(HLoadString* load) NO_THREAD_SAFETY_ANALYSIS { 6195 LocationSummary* locations = load->GetLocations(); 6196 Location out_loc = locations->Out(); 6197 Register out = out_loc.AsRegister<Register>(); 6198 HLoadString::LoadKind load_kind = load->GetLoadKind(); 6199 6200 switch (load_kind) { 6201 case HLoadString::LoadKind::kBootImageLinkTimeAddress: { 6202 DCHECK(codegen_->GetCompilerOptions().IsBootImage()); 6203 __ LoadLiteral(out, codegen_->DeduplicateBootImageStringLiteral(load->GetDexFile(), 6204 load->GetStringIndex())); 6205 return; // No dex cache slow path. 6206 } 6207 case HLoadString::LoadKind::kBootImageLinkTimePcRelative: { 6208 DCHECK(codegen_->GetCompilerOptions().IsBootImage()); 6209 CodeGeneratorARM::PcRelativePatchInfo* labels = 6210 codegen_->NewPcRelativeStringPatch(load->GetDexFile(), load->GetStringIndex()); 6211 __ BindTrackedLabel(&labels->movw_label); 6212 __ movw(out, /* placeholder */ 0u); 6213 __ BindTrackedLabel(&labels->movt_label); 6214 __ movt(out, /* placeholder */ 0u); 6215 __ BindTrackedLabel(&labels->add_pc_label); 6216 __ add(out, out, ShifterOperand(PC)); 6217 return; // No dex cache slow path. 6218 } 6219 case HLoadString::LoadKind::kBootImageAddress: { 6220 uint32_t address = dchecked_integral_cast<uint32_t>( 6221 reinterpret_cast<uintptr_t>(load->GetString().Get())); 6222 DCHECK_NE(address, 0u); 6223 __ LoadLiteral(out, codegen_->DeduplicateBootImageAddressLiteral(address)); 6224 return; // No dex cache slow path. 6225 } 6226 case HLoadString::LoadKind::kBssEntry: { 6227 DCHECK(!codegen_->GetCompilerOptions().IsBootImage()); 6228 Register temp = (!kUseReadBarrier || kUseBakerReadBarrier) 6229 ? locations->GetTemp(0).AsRegister<Register>() 6230 : out; 6231 CodeGeneratorARM::PcRelativePatchInfo* labels = 6232 codegen_->NewPcRelativeStringPatch(load->GetDexFile(), load->GetStringIndex()); 6233 __ BindTrackedLabel(&labels->movw_label); 6234 __ movw(temp, /* placeholder */ 0u); 6235 __ BindTrackedLabel(&labels->movt_label); 6236 __ movt(temp, /* placeholder */ 0u); 6237 __ BindTrackedLabel(&labels->add_pc_label); 6238 __ add(temp, temp, ShifterOperand(PC)); 6239 GenerateGcRootFieldLoad(load, out_loc, temp, /* offset */ 0, kCompilerReadBarrierOption); 6240 SlowPathCode* slow_path = new (GetGraph()->GetArena()) LoadStringSlowPathARM(load); 6241 codegen_->AddSlowPath(slow_path); 6242 __ CompareAndBranchIfZero(out, slow_path->GetEntryLabel()); 6243 __ Bind(slow_path->GetExitLabel()); 6244 return; 6245 } 6246 case HLoadString::LoadKind::kJitTableAddress: { 6247 __ LoadLiteral(out, codegen_->DeduplicateJitStringLiteral(load->GetDexFile(), 6248 load->GetStringIndex(), 6249 load->GetString())); 6250 // /* GcRoot<mirror::String> */ out = *out 6251 GenerateGcRootFieldLoad(load, out_loc, out, /* offset */ 0, kCompilerReadBarrierOption); 6252 return; 6253 } 6254 default: 6255 break; 6256 } 6257 6258 // TODO: Consider re-adding the compiler code to do string dex cache lookup again. 6259 DCHECK(load_kind == HLoadString::LoadKind::kDexCacheViaMethod); 6260 InvokeRuntimeCallingConvention calling_convention; 6261 DCHECK_EQ(calling_convention.GetRegisterAt(0), out); 6262 __ LoadImmediate(calling_convention.GetRegisterAt(0), load->GetStringIndex().index_); 6263 codegen_->InvokeRuntime(kQuickResolveString, load, load->GetDexPc()); 6264 CheckEntrypointTypes<kQuickResolveString, void*, uint32_t>(); 6265} 6266 6267static int32_t GetExceptionTlsOffset() { 6268 return Thread::ExceptionOffset<kArmPointerSize>().Int32Value(); 6269} 6270 6271void LocationsBuilderARM::VisitLoadException(HLoadException* load) { 6272 LocationSummary* locations = 6273 new (GetGraph()->GetArena()) LocationSummary(load, LocationSummary::kNoCall); 6274 locations->SetOut(Location::RequiresRegister()); 6275} 6276 6277void InstructionCodeGeneratorARM::VisitLoadException(HLoadException* load) { 6278 Register out = load->GetLocations()->Out().AsRegister<Register>(); 6279 __ LoadFromOffset(kLoadWord, out, TR, GetExceptionTlsOffset()); 6280} 6281 6282void LocationsBuilderARM::VisitClearException(HClearException* clear) { 6283 new (GetGraph()->GetArena()) LocationSummary(clear, LocationSummary::kNoCall); 6284} 6285 6286void InstructionCodeGeneratorARM::VisitClearException(HClearException* clear ATTRIBUTE_UNUSED) { 6287 __ LoadImmediate(IP, 0); 6288 __ StoreToOffset(kStoreWord, IP, TR, GetExceptionTlsOffset()); 6289} 6290 6291void LocationsBuilderARM::VisitThrow(HThrow* instruction) { 6292 LocationSummary* locations = 6293 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCallOnMainOnly); 6294 InvokeRuntimeCallingConvention calling_convention; 6295 locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(0))); 6296} 6297 6298void InstructionCodeGeneratorARM::VisitThrow(HThrow* instruction) { 6299 codegen_->InvokeRuntime(kQuickDeliverException, instruction, instruction->GetDexPc()); 6300 CheckEntrypointTypes<kQuickDeliverException, void, mirror::Object*>(); 6301} 6302 6303// Temp is used for read barrier. 6304static size_t NumberOfInstanceOfTemps(TypeCheckKind type_check_kind) { 6305 if (kEmitCompilerReadBarrier && 6306 (kUseBakerReadBarrier || 6307 type_check_kind == TypeCheckKind::kAbstractClassCheck || 6308 type_check_kind == TypeCheckKind::kClassHierarchyCheck || 6309 type_check_kind == TypeCheckKind::kArrayObjectCheck)) { 6310 return 1; 6311 } 6312 return 0; 6313} 6314 6315// Interface case has 3 temps, one for holding the number of interfaces, one for the current 6316// interface pointer, one for loading the current interface. 6317// The other checks have one temp for loading the object's class. 6318static size_t NumberOfCheckCastTemps(TypeCheckKind type_check_kind) { 6319 if (type_check_kind == TypeCheckKind::kInterfaceCheck) { 6320 return 3; 6321 } 6322 return 1 + NumberOfInstanceOfTemps(type_check_kind); 6323} 6324 6325void LocationsBuilderARM::VisitInstanceOf(HInstanceOf* instruction) { 6326 LocationSummary::CallKind call_kind = LocationSummary::kNoCall; 6327 TypeCheckKind type_check_kind = instruction->GetTypeCheckKind(); 6328 bool baker_read_barrier_slow_path = false; 6329 switch (type_check_kind) { 6330 case TypeCheckKind::kExactCheck: 6331 case TypeCheckKind::kAbstractClassCheck: 6332 case TypeCheckKind::kClassHierarchyCheck: 6333 case TypeCheckKind::kArrayObjectCheck: 6334 call_kind = 6335 kEmitCompilerReadBarrier ? LocationSummary::kCallOnSlowPath : LocationSummary::kNoCall; 6336 baker_read_barrier_slow_path = kUseBakerReadBarrier; 6337 break; 6338 case TypeCheckKind::kArrayCheck: 6339 case TypeCheckKind::kUnresolvedCheck: 6340 case TypeCheckKind::kInterfaceCheck: 6341 call_kind = LocationSummary::kCallOnSlowPath; 6342 break; 6343 } 6344 6345 LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction, call_kind); 6346 if (baker_read_barrier_slow_path) { 6347 locations->SetCustomSlowPathCallerSaves(RegisterSet::Empty()); // No caller-save registers. 6348 } 6349 locations->SetInAt(0, Location::RequiresRegister()); 6350 locations->SetInAt(1, Location::RequiresRegister()); 6351 // The "out" register is used as a temporary, so it overlaps with the inputs. 6352 // Note that TypeCheckSlowPathARM uses this register too. 6353 locations->SetOut(Location::RequiresRegister(), Location::kOutputOverlap); 6354 locations->AddRegisterTemps(NumberOfInstanceOfTemps(type_check_kind)); 6355} 6356 6357void InstructionCodeGeneratorARM::VisitInstanceOf(HInstanceOf* instruction) { 6358 TypeCheckKind type_check_kind = instruction->GetTypeCheckKind(); 6359 LocationSummary* locations = instruction->GetLocations(); 6360 Location obj_loc = locations->InAt(0); 6361 Register obj = obj_loc.AsRegister<Register>(); 6362 Register cls = locations->InAt(1).AsRegister<Register>(); 6363 Location out_loc = locations->Out(); 6364 Register out = out_loc.AsRegister<Register>(); 6365 const size_t num_temps = NumberOfInstanceOfTemps(type_check_kind); 6366 DCHECK_LE(num_temps, 1u); 6367 Location maybe_temp_loc = (num_temps >= 1) ? locations->GetTemp(0) : Location::NoLocation(); 6368 uint32_t class_offset = mirror::Object::ClassOffset().Int32Value(); 6369 uint32_t super_offset = mirror::Class::SuperClassOffset().Int32Value(); 6370 uint32_t component_offset = mirror::Class::ComponentTypeOffset().Int32Value(); 6371 uint32_t primitive_offset = mirror::Class::PrimitiveTypeOffset().Int32Value(); 6372 Label done, zero; 6373 SlowPathCodeARM* slow_path = nullptr; 6374 6375 // Return 0 if `obj` is null. 6376 // avoid null check if we know obj is not null. 6377 if (instruction->MustDoNullCheck()) { 6378 __ CompareAndBranchIfZero(obj, &zero); 6379 } 6380 6381 switch (type_check_kind) { 6382 case TypeCheckKind::kExactCheck: { 6383 // /* HeapReference<Class> */ out = obj->klass_ 6384 GenerateReferenceLoadTwoRegisters(instruction, 6385 out_loc, 6386 obj_loc, 6387 class_offset, 6388 maybe_temp_loc, 6389 kCompilerReadBarrierOption); 6390 __ cmp(out, ShifterOperand(cls)); 6391 // Classes must be equal for the instanceof to succeed. 6392 __ b(&zero, NE); 6393 __ LoadImmediate(out, 1); 6394 __ b(&done); 6395 break; 6396 } 6397 6398 case TypeCheckKind::kAbstractClassCheck: { 6399 // /* HeapReference<Class> */ out = obj->klass_ 6400 GenerateReferenceLoadTwoRegisters(instruction, 6401 out_loc, 6402 obj_loc, 6403 class_offset, 6404 maybe_temp_loc, 6405 kCompilerReadBarrierOption); 6406 // If the class is abstract, we eagerly fetch the super class of the 6407 // object to avoid doing a comparison we know will fail. 6408 Label loop; 6409 __ Bind(&loop); 6410 // /* HeapReference<Class> */ out = out->super_class_ 6411 GenerateReferenceLoadOneRegister(instruction, 6412 out_loc, 6413 super_offset, 6414 maybe_temp_loc, 6415 kCompilerReadBarrierOption); 6416 // If `out` is null, we use it for the result, and jump to `done`. 6417 __ CompareAndBranchIfZero(out, &done); 6418 __ cmp(out, ShifterOperand(cls)); 6419 __ b(&loop, NE); 6420 __ LoadImmediate(out, 1); 6421 if (zero.IsLinked()) { 6422 __ b(&done); 6423 } 6424 break; 6425 } 6426 6427 case TypeCheckKind::kClassHierarchyCheck: { 6428 // /* HeapReference<Class> */ out = obj->klass_ 6429 GenerateReferenceLoadTwoRegisters(instruction, 6430 out_loc, 6431 obj_loc, 6432 class_offset, 6433 maybe_temp_loc, 6434 kCompilerReadBarrierOption); 6435 // Walk over the class hierarchy to find a match. 6436 Label loop, success; 6437 __ Bind(&loop); 6438 __ cmp(out, ShifterOperand(cls)); 6439 __ b(&success, EQ); 6440 // /* HeapReference<Class> */ out = out->super_class_ 6441 GenerateReferenceLoadOneRegister(instruction, 6442 out_loc, 6443 super_offset, 6444 maybe_temp_loc, 6445 kCompilerReadBarrierOption); 6446 __ CompareAndBranchIfNonZero(out, &loop); 6447 // If `out` is null, we use it for the result, and jump to `done`. 6448 __ b(&done); 6449 __ Bind(&success); 6450 __ LoadImmediate(out, 1); 6451 if (zero.IsLinked()) { 6452 __ b(&done); 6453 } 6454 break; 6455 } 6456 6457 case TypeCheckKind::kArrayObjectCheck: { 6458 // /* HeapReference<Class> */ out = obj->klass_ 6459 GenerateReferenceLoadTwoRegisters(instruction, 6460 out_loc, 6461 obj_loc, 6462 class_offset, 6463 maybe_temp_loc, 6464 kCompilerReadBarrierOption); 6465 // Do an exact check. 6466 Label exact_check; 6467 __ cmp(out, ShifterOperand(cls)); 6468 __ b(&exact_check, EQ); 6469 // Otherwise, we need to check that the object's class is a non-primitive array. 6470 // /* HeapReference<Class> */ out = out->component_type_ 6471 GenerateReferenceLoadOneRegister(instruction, 6472 out_loc, 6473 component_offset, 6474 maybe_temp_loc, 6475 kCompilerReadBarrierOption); 6476 // If `out` is null, we use it for the result, and jump to `done`. 6477 __ CompareAndBranchIfZero(out, &done); 6478 __ LoadFromOffset(kLoadUnsignedHalfword, out, out, primitive_offset); 6479 static_assert(Primitive::kPrimNot == 0, "Expected 0 for kPrimNot"); 6480 __ CompareAndBranchIfNonZero(out, &zero); 6481 __ Bind(&exact_check); 6482 __ LoadImmediate(out, 1); 6483 __ b(&done); 6484 break; 6485 } 6486 6487 case TypeCheckKind::kArrayCheck: { 6488 // No read barrier since the slow path will retry upon failure. 6489 // /* HeapReference<Class> */ out = obj->klass_ 6490 GenerateReferenceLoadTwoRegisters(instruction, 6491 out_loc, 6492 obj_loc, 6493 class_offset, 6494 maybe_temp_loc, 6495 kWithoutReadBarrier); 6496 __ cmp(out, ShifterOperand(cls)); 6497 DCHECK(locations->OnlyCallsOnSlowPath()); 6498 slow_path = new (GetGraph()->GetArena()) TypeCheckSlowPathARM(instruction, 6499 /* is_fatal */ false); 6500 codegen_->AddSlowPath(slow_path); 6501 __ b(slow_path->GetEntryLabel(), NE); 6502 __ LoadImmediate(out, 1); 6503 if (zero.IsLinked()) { 6504 __ b(&done); 6505 } 6506 break; 6507 } 6508 6509 case TypeCheckKind::kUnresolvedCheck: 6510 case TypeCheckKind::kInterfaceCheck: { 6511 // Note that we indeed only call on slow path, but we always go 6512 // into the slow path for the unresolved and interface check 6513 // cases. 6514 // 6515 // We cannot directly call the InstanceofNonTrivial runtime 6516 // entry point without resorting to a type checking slow path 6517 // here (i.e. by calling InvokeRuntime directly), as it would 6518 // require to assign fixed registers for the inputs of this 6519 // HInstanceOf instruction (following the runtime calling 6520 // convention), which might be cluttered by the potential first 6521 // read barrier emission at the beginning of this method. 6522 // 6523 // TODO: Introduce a new runtime entry point taking the object 6524 // to test (instead of its class) as argument, and let it deal 6525 // with the read barrier issues. This will let us refactor this 6526 // case of the `switch` code as it was previously (with a direct 6527 // call to the runtime not using a type checking slow path). 6528 // This should also be beneficial for the other cases above. 6529 DCHECK(locations->OnlyCallsOnSlowPath()); 6530 slow_path = new (GetGraph()->GetArena()) TypeCheckSlowPathARM(instruction, 6531 /* is_fatal */ false); 6532 codegen_->AddSlowPath(slow_path); 6533 __ b(slow_path->GetEntryLabel()); 6534 if (zero.IsLinked()) { 6535 __ b(&done); 6536 } 6537 break; 6538 } 6539 } 6540 6541 if (zero.IsLinked()) { 6542 __ Bind(&zero); 6543 __ LoadImmediate(out, 0); 6544 } 6545 6546 if (done.IsLinked()) { 6547 __ Bind(&done); 6548 } 6549 6550 if (slow_path != nullptr) { 6551 __ Bind(slow_path->GetExitLabel()); 6552 } 6553} 6554 6555void LocationsBuilderARM::VisitCheckCast(HCheckCast* instruction) { 6556 LocationSummary::CallKind call_kind = LocationSummary::kNoCall; 6557 bool throws_into_catch = instruction->CanThrowIntoCatchBlock(); 6558 6559 TypeCheckKind type_check_kind = instruction->GetTypeCheckKind(); 6560 switch (type_check_kind) { 6561 case TypeCheckKind::kExactCheck: 6562 case TypeCheckKind::kAbstractClassCheck: 6563 case TypeCheckKind::kClassHierarchyCheck: 6564 case TypeCheckKind::kArrayObjectCheck: 6565 call_kind = (throws_into_catch || kEmitCompilerReadBarrier) ? 6566 LocationSummary::kCallOnSlowPath : 6567 LocationSummary::kNoCall; // In fact, call on a fatal (non-returning) slow path. 6568 break; 6569 case TypeCheckKind::kArrayCheck: 6570 case TypeCheckKind::kUnresolvedCheck: 6571 case TypeCheckKind::kInterfaceCheck: 6572 call_kind = LocationSummary::kCallOnSlowPath; 6573 break; 6574 } 6575 6576 LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction, call_kind); 6577 locations->SetInAt(0, Location::RequiresRegister()); 6578 locations->SetInAt(1, Location::RequiresRegister()); 6579 locations->AddRegisterTemps(NumberOfCheckCastTemps(type_check_kind)); 6580} 6581 6582void InstructionCodeGeneratorARM::VisitCheckCast(HCheckCast* instruction) { 6583 TypeCheckKind type_check_kind = instruction->GetTypeCheckKind(); 6584 LocationSummary* locations = instruction->GetLocations(); 6585 Location obj_loc = locations->InAt(0); 6586 Register obj = obj_loc.AsRegister<Register>(); 6587 Register cls = locations->InAt(1).AsRegister<Register>(); 6588 Location temp_loc = locations->GetTemp(0); 6589 Register temp = temp_loc.AsRegister<Register>(); 6590 const size_t num_temps = NumberOfCheckCastTemps(type_check_kind); 6591 DCHECK_LE(num_temps, 3u); 6592 Location maybe_temp2_loc = (num_temps >= 2) ? locations->GetTemp(1) : Location::NoLocation(); 6593 Location maybe_temp3_loc = (num_temps >= 3) ? locations->GetTemp(2) : Location::NoLocation(); 6594 const uint32_t class_offset = mirror::Object::ClassOffset().Int32Value(); 6595 const uint32_t super_offset = mirror::Class::SuperClassOffset().Int32Value(); 6596 const uint32_t component_offset = mirror::Class::ComponentTypeOffset().Int32Value(); 6597 const uint32_t primitive_offset = mirror::Class::PrimitiveTypeOffset().Int32Value(); 6598 const uint32_t iftable_offset = mirror::Class::IfTableOffset().Uint32Value(); 6599 const uint32_t array_length_offset = mirror::Array::LengthOffset().Uint32Value(); 6600 const uint32_t object_array_data_offset = 6601 mirror::Array::DataOffset(kHeapReferenceSize).Uint32Value(); 6602 6603 // Always false for read barriers since we may need to go to the entrypoint for non-fatal cases 6604 // from false negatives. The false negatives may come from avoiding read barriers below. Avoiding 6605 // read barriers is done for performance and code size reasons. 6606 bool is_type_check_slow_path_fatal = false; 6607 if (!kEmitCompilerReadBarrier) { 6608 is_type_check_slow_path_fatal = 6609 (type_check_kind == TypeCheckKind::kExactCheck || 6610 type_check_kind == TypeCheckKind::kAbstractClassCheck || 6611 type_check_kind == TypeCheckKind::kClassHierarchyCheck || 6612 type_check_kind == TypeCheckKind::kArrayObjectCheck) && 6613 !instruction->CanThrowIntoCatchBlock(); 6614 } 6615 SlowPathCodeARM* type_check_slow_path = 6616 new (GetGraph()->GetArena()) TypeCheckSlowPathARM(instruction, 6617 is_type_check_slow_path_fatal); 6618 codegen_->AddSlowPath(type_check_slow_path); 6619 6620 Label done; 6621 // Avoid null check if we know obj is not null. 6622 if (instruction->MustDoNullCheck()) { 6623 __ CompareAndBranchIfZero(obj, &done); 6624 } 6625 6626 switch (type_check_kind) { 6627 case TypeCheckKind::kExactCheck: 6628 case TypeCheckKind::kArrayCheck: { 6629 // /* HeapReference<Class> */ temp = obj->klass_ 6630 GenerateReferenceLoadTwoRegisters(instruction, 6631 temp_loc, 6632 obj_loc, 6633 class_offset, 6634 maybe_temp2_loc, 6635 kWithoutReadBarrier); 6636 6637 __ cmp(temp, ShifterOperand(cls)); 6638 // Jump to slow path for throwing the exception or doing a 6639 // more involved array check. 6640 __ b(type_check_slow_path->GetEntryLabel(), NE); 6641 break; 6642 } 6643 6644 case TypeCheckKind::kAbstractClassCheck: { 6645 // /* HeapReference<Class> */ temp = obj->klass_ 6646 GenerateReferenceLoadTwoRegisters(instruction, 6647 temp_loc, 6648 obj_loc, 6649 class_offset, 6650 maybe_temp2_loc, 6651 kWithoutReadBarrier); 6652 6653 // If the class is abstract, we eagerly fetch the super class of the 6654 // object to avoid doing a comparison we know will fail. 6655 Label loop; 6656 __ Bind(&loop); 6657 // /* HeapReference<Class> */ temp = temp->super_class_ 6658 GenerateReferenceLoadOneRegister(instruction, 6659 temp_loc, 6660 super_offset, 6661 maybe_temp2_loc, 6662 kWithoutReadBarrier); 6663 6664 // If the class reference currently in `temp` is null, jump to the slow path to throw the 6665 // exception. 6666 __ CompareAndBranchIfZero(temp, type_check_slow_path->GetEntryLabel()); 6667 6668 // Otherwise, compare the classes. 6669 __ cmp(temp, ShifterOperand(cls)); 6670 __ b(&loop, NE); 6671 break; 6672 } 6673 6674 case TypeCheckKind::kClassHierarchyCheck: { 6675 // /* HeapReference<Class> */ temp = obj->klass_ 6676 GenerateReferenceLoadTwoRegisters(instruction, 6677 temp_loc, 6678 obj_loc, 6679 class_offset, 6680 maybe_temp2_loc, 6681 kWithoutReadBarrier); 6682 6683 // Walk over the class hierarchy to find a match. 6684 Label loop; 6685 __ Bind(&loop); 6686 __ cmp(temp, ShifterOperand(cls)); 6687 __ b(&done, EQ); 6688 6689 // /* HeapReference<Class> */ temp = temp->super_class_ 6690 GenerateReferenceLoadOneRegister(instruction, 6691 temp_loc, 6692 super_offset, 6693 maybe_temp2_loc, 6694 kWithoutReadBarrier); 6695 6696 // If the class reference currently in `temp` is null, jump to the slow path to throw the 6697 // exception. 6698 __ CompareAndBranchIfZero(temp, type_check_slow_path->GetEntryLabel()); 6699 // Otherwise, jump to the beginning of the loop. 6700 __ b(&loop); 6701 break; 6702 } 6703 6704 case TypeCheckKind::kArrayObjectCheck: { 6705 // /* HeapReference<Class> */ temp = obj->klass_ 6706 GenerateReferenceLoadTwoRegisters(instruction, 6707 temp_loc, 6708 obj_loc, 6709 class_offset, 6710 maybe_temp2_loc, 6711 kWithoutReadBarrier); 6712 6713 // Do an exact check. 6714 __ cmp(temp, ShifterOperand(cls)); 6715 __ b(&done, EQ); 6716 6717 // Otherwise, we need to check that the object's class is a non-primitive array. 6718 // /* HeapReference<Class> */ temp = temp->component_type_ 6719 GenerateReferenceLoadOneRegister(instruction, 6720 temp_loc, 6721 component_offset, 6722 maybe_temp2_loc, 6723 kWithoutReadBarrier); 6724 // If the component type is null, jump to the slow path to throw the exception. 6725 __ CompareAndBranchIfZero(temp, type_check_slow_path->GetEntryLabel()); 6726 // Otherwise,the object is indeed an array, jump to label `check_non_primitive_component_type` 6727 // to further check that this component type is not a primitive type. 6728 __ LoadFromOffset(kLoadUnsignedHalfword, temp, temp, primitive_offset); 6729 static_assert(Primitive::kPrimNot == 0, "Expected 0 for art::Primitive::kPrimNot"); 6730 __ CompareAndBranchIfNonZero(temp, type_check_slow_path->GetEntryLabel()); 6731 break; 6732 } 6733 6734 case TypeCheckKind::kUnresolvedCheck: 6735 // We always go into the type check slow path for the unresolved check case. 6736 // We cannot directly call the CheckCast runtime entry point 6737 // without resorting to a type checking slow path here (i.e. by 6738 // calling InvokeRuntime directly), as it would require to 6739 // assign fixed registers for the inputs of this HInstanceOf 6740 // instruction (following the runtime calling convention), which 6741 // might be cluttered by the potential first read barrier 6742 // emission at the beginning of this method. 6743 6744 __ b(type_check_slow_path->GetEntryLabel()); 6745 break; 6746 6747 case TypeCheckKind::kInterfaceCheck: { 6748 // Avoid read barriers to improve performance of the fast path. We can not get false 6749 // positives by doing this. 6750 // /* HeapReference<Class> */ temp = obj->klass_ 6751 GenerateReferenceLoadTwoRegisters(instruction, 6752 temp_loc, 6753 obj_loc, 6754 class_offset, 6755 maybe_temp2_loc, 6756 kWithoutReadBarrier); 6757 6758 // /* HeapReference<Class> */ temp = temp->iftable_ 6759 GenerateReferenceLoadTwoRegisters(instruction, 6760 temp_loc, 6761 temp_loc, 6762 iftable_offset, 6763 maybe_temp2_loc, 6764 kWithoutReadBarrier); 6765 // Iftable is never null. 6766 __ ldr(maybe_temp2_loc.AsRegister<Register>(), Address(temp, array_length_offset)); 6767 // Loop through the iftable and check if any class matches. 6768 Label start_loop; 6769 __ Bind(&start_loop); 6770 __ CompareAndBranchIfZero(maybe_temp2_loc.AsRegister<Register>(), 6771 type_check_slow_path->GetEntryLabel()); 6772 __ ldr(maybe_temp3_loc.AsRegister<Register>(), Address(temp, object_array_data_offset)); 6773 __ MaybeUnpoisonHeapReference(maybe_temp3_loc.AsRegister<Register>()); 6774 // Go to next interface. 6775 __ add(temp, temp, ShifterOperand(2 * kHeapReferenceSize)); 6776 __ sub(maybe_temp2_loc.AsRegister<Register>(), 6777 maybe_temp2_loc.AsRegister<Register>(), 6778 ShifterOperand(2)); 6779 // Compare the classes and continue the loop if they do not match. 6780 __ cmp(cls, ShifterOperand(maybe_temp3_loc.AsRegister<Register>())); 6781 __ b(&start_loop, NE); 6782 break; 6783 } 6784 } 6785 __ Bind(&done); 6786 6787 __ Bind(type_check_slow_path->GetExitLabel()); 6788} 6789 6790void LocationsBuilderARM::VisitMonitorOperation(HMonitorOperation* instruction) { 6791 LocationSummary* locations = 6792 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCallOnMainOnly); 6793 InvokeRuntimeCallingConvention calling_convention; 6794 locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(0))); 6795} 6796 6797void InstructionCodeGeneratorARM::VisitMonitorOperation(HMonitorOperation* instruction) { 6798 codegen_->InvokeRuntime(instruction->IsEnter() ? kQuickLockObject : kQuickUnlockObject, 6799 instruction, 6800 instruction->GetDexPc()); 6801 if (instruction->IsEnter()) { 6802 CheckEntrypointTypes<kQuickLockObject, void, mirror::Object*>(); 6803 } else { 6804 CheckEntrypointTypes<kQuickUnlockObject, void, mirror::Object*>(); 6805 } 6806} 6807 6808void LocationsBuilderARM::VisitAnd(HAnd* instruction) { HandleBitwiseOperation(instruction, AND); } 6809void LocationsBuilderARM::VisitOr(HOr* instruction) { HandleBitwiseOperation(instruction, ORR); } 6810void LocationsBuilderARM::VisitXor(HXor* instruction) { HandleBitwiseOperation(instruction, EOR); } 6811 6812void LocationsBuilderARM::HandleBitwiseOperation(HBinaryOperation* instruction, Opcode opcode) { 6813 LocationSummary* locations = 6814 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall); 6815 DCHECK(instruction->GetResultType() == Primitive::kPrimInt 6816 || instruction->GetResultType() == Primitive::kPrimLong); 6817 // Note: GVN reorders commutative operations to have the constant on the right hand side. 6818 locations->SetInAt(0, Location::RequiresRegister()); 6819 locations->SetInAt(1, ArmEncodableConstantOrRegister(instruction->InputAt(1), opcode)); 6820 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 6821} 6822 6823void InstructionCodeGeneratorARM::VisitAnd(HAnd* instruction) { 6824 HandleBitwiseOperation(instruction); 6825} 6826 6827void InstructionCodeGeneratorARM::VisitOr(HOr* instruction) { 6828 HandleBitwiseOperation(instruction); 6829} 6830 6831void InstructionCodeGeneratorARM::VisitXor(HXor* instruction) { 6832 HandleBitwiseOperation(instruction); 6833} 6834 6835 6836void LocationsBuilderARM::VisitBitwiseNegatedRight(HBitwiseNegatedRight* instruction) { 6837 LocationSummary* locations = 6838 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall); 6839 DCHECK(instruction->GetResultType() == Primitive::kPrimInt 6840 || instruction->GetResultType() == Primitive::kPrimLong); 6841 6842 locations->SetInAt(0, Location::RequiresRegister()); 6843 locations->SetInAt(1, Location::RequiresRegister()); 6844 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 6845} 6846 6847void InstructionCodeGeneratorARM::VisitBitwiseNegatedRight(HBitwiseNegatedRight* instruction) { 6848 LocationSummary* locations = instruction->GetLocations(); 6849 Location first = locations->InAt(0); 6850 Location second = locations->InAt(1); 6851 Location out = locations->Out(); 6852 6853 if (instruction->GetResultType() == Primitive::kPrimInt) { 6854 Register first_reg = first.AsRegister<Register>(); 6855 ShifterOperand second_reg(second.AsRegister<Register>()); 6856 Register out_reg = out.AsRegister<Register>(); 6857 6858 switch (instruction->GetOpKind()) { 6859 case HInstruction::kAnd: 6860 __ bic(out_reg, first_reg, second_reg); 6861 break; 6862 case HInstruction::kOr: 6863 __ orn(out_reg, first_reg, second_reg); 6864 break; 6865 // There is no EON on arm. 6866 case HInstruction::kXor: 6867 default: 6868 LOG(FATAL) << "Unexpected instruction " << instruction->DebugName(); 6869 UNREACHABLE(); 6870 } 6871 return; 6872 6873 } else { 6874 DCHECK_EQ(instruction->GetResultType(), Primitive::kPrimLong); 6875 Register first_low = first.AsRegisterPairLow<Register>(); 6876 Register first_high = first.AsRegisterPairHigh<Register>(); 6877 ShifterOperand second_low(second.AsRegisterPairLow<Register>()); 6878 ShifterOperand second_high(second.AsRegisterPairHigh<Register>()); 6879 Register out_low = out.AsRegisterPairLow<Register>(); 6880 Register out_high = out.AsRegisterPairHigh<Register>(); 6881 6882 switch (instruction->GetOpKind()) { 6883 case HInstruction::kAnd: 6884 __ bic(out_low, first_low, second_low); 6885 __ bic(out_high, first_high, second_high); 6886 break; 6887 case HInstruction::kOr: 6888 __ orn(out_low, first_low, second_low); 6889 __ orn(out_high, first_high, second_high); 6890 break; 6891 // There is no EON on arm. 6892 case HInstruction::kXor: 6893 default: 6894 LOG(FATAL) << "Unexpected instruction " << instruction->DebugName(); 6895 UNREACHABLE(); 6896 } 6897 } 6898} 6899 6900void LocationsBuilderARM::VisitDataProcWithShifterOp( 6901 HDataProcWithShifterOp* instruction) { 6902 DCHECK(instruction->GetType() == Primitive::kPrimInt || 6903 instruction->GetType() == Primitive::kPrimLong); 6904 LocationSummary* locations = 6905 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall); 6906 const bool overlap = instruction->GetType() == Primitive::kPrimLong && 6907 HDataProcWithShifterOp::IsExtensionOp(instruction->GetOpKind()); 6908 6909 locations->SetInAt(0, Location::RequiresRegister()); 6910 locations->SetInAt(1, Location::RequiresRegister()); 6911 locations->SetOut(Location::RequiresRegister(), 6912 overlap ? Location::kOutputOverlap : Location::kNoOutputOverlap); 6913} 6914 6915void InstructionCodeGeneratorARM::VisitDataProcWithShifterOp( 6916 HDataProcWithShifterOp* instruction) { 6917 const LocationSummary* const locations = instruction->GetLocations(); 6918 const HInstruction::InstructionKind kind = instruction->GetInstrKind(); 6919 const HDataProcWithShifterOp::OpKind op_kind = instruction->GetOpKind(); 6920 const Location left = locations->InAt(0); 6921 const Location right = locations->InAt(1); 6922 const Location out = locations->Out(); 6923 6924 if (instruction->GetType() == Primitive::kPrimInt) { 6925 DCHECK(!HDataProcWithShifterOp::IsExtensionOp(op_kind)); 6926 6927 const Register second = instruction->InputAt(1)->GetType() == Primitive::kPrimLong 6928 ? right.AsRegisterPairLow<Register>() 6929 : right.AsRegister<Register>(); 6930 6931 GenerateDataProcInstruction(kind, 6932 out.AsRegister<Register>(), 6933 left.AsRegister<Register>(), 6934 ShifterOperand(second, 6935 ShiftFromOpKind(op_kind), 6936 instruction->GetShiftAmount()), 6937 codegen_); 6938 } else { 6939 DCHECK_EQ(instruction->GetType(), Primitive::kPrimLong); 6940 6941 if (HDataProcWithShifterOp::IsExtensionOp(op_kind)) { 6942 const Register second = right.AsRegister<Register>(); 6943 6944 DCHECK_NE(out.AsRegisterPairLow<Register>(), second); 6945 GenerateDataProc(kind, 6946 out, 6947 left, 6948 ShifterOperand(second), 6949 ShifterOperand(second, ASR, 31), 6950 codegen_); 6951 } else { 6952 GenerateLongDataProc(instruction, codegen_); 6953 } 6954 } 6955} 6956 6957void InstructionCodeGeneratorARM::GenerateAndConst(Register out, Register first, uint32_t value) { 6958 // Optimize special cases for individual halfs of `and-long` (`and` is simplified earlier). 6959 if (value == 0xffffffffu) { 6960 if (out != first) { 6961 __ mov(out, ShifterOperand(first)); 6962 } 6963 return; 6964 } 6965 if (value == 0u) { 6966 __ mov(out, ShifterOperand(0)); 6967 return; 6968 } 6969 ShifterOperand so; 6970 if (__ ShifterOperandCanHold(kNoRegister, kNoRegister, AND, value, &so)) { 6971 __ and_(out, first, so); 6972 } else { 6973 DCHECK(__ ShifterOperandCanHold(kNoRegister, kNoRegister, BIC, ~value, &so)); 6974 __ bic(out, first, ShifterOperand(~value)); 6975 } 6976} 6977 6978void InstructionCodeGeneratorARM::GenerateOrrConst(Register out, Register first, uint32_t value) { 6979 // Optimize special cases for individual halfs of `or-long` (`or` is simplified earlier). 6980 if (value == 0u) { 6981 if (out != first) { 6982 __ mov(out, ShifterOperand(first)); 6983 } 6984 return; 6985 } 6986 if (value == 0xffffffffu) { 6987 __ mvn(out, ShifterOperand(0)); 6988 return; 6989 } 6990 ShifterOperand so; 6991 if (__ ShifterOperandCanHold(kNoRegister, kNoRegister, ORR, value, &so)) { 6992 __ orr(out, first, so); 6993 } else { 6994 DCHECK(__ ShifterOperandCanHold(kNoRegister, kNoRegister, ORN, ~value, &so)); 6995 __ orn(out, first, ShifterOperand(~value)); 6996 } 6997} 6998 6999void InstructionCodeGeneratorARM::GenerateEorConst(Register out, Register first, uint32_t value) { 7000 // Optimize special case for individual halfs of `xor-long` (`xor` is simplified earlier). 7001 if (value == 0u) { 7002 if (out != first) { 7003 __ mov(out, ShifterOperand(first)); 7004 } 7005 return; 7006 } 7007 __ eor(out, first, ShifterOperand(value)); 7008} 7009 7010void InstructionCodeGeneratorARM::GenerateAddLongConst(Location out, 7011 Location first, 7012 uint64_t value) { 7013 Register out_low = out.AsRegisterPairLow<Register>(); 7014 Register out_high = out.AsRegisterPairHigh<Register>(); 7015 Register first_low = first.AsRegisterPairLow<Register>(); 7016 Register first_high = first.AsRegisterPairHigh<Register>(); 7017 uint32_t value_low = Low32Bits(value); 7018 uint32_t value_high = High32Bits(value); 7019 if (value_low == 0u) { 7020 if (out_low != first_low) { 7021 __ mov(out_low, ShifterOperand(first_low)); 7022 } 7023 __ AddConstant(out_high, first_high, value_high); 7024 return; 7025 } 7026 __ AddConstantSetFlags(out_low, first_low, value_low); 7027 ShifterOperand so; 7028 if (__ ShifterOperandCanHold(out_high, first_high, ADC, value_high, kCcDontCare, &so)) { 7029 __ adc(out_high, first_high, so); 7030 } else if (__ ShifterOperandCanHold(out_low, first_low, SBC, ~value_high, kCcDontCare, &so)) { 7031 __ sbc(out_high, first_high, so); 7032 } else { 7033 LOG(FATAL) << "Unexpected constant " << value_high; 7034 UNREACHABLE(); 7035 } 7036} 7037 7038void InstructionCodeGeneratorARM::HandleBitwiseOperation(HBinaryOperation* instruction) { 7039 LocationSummary* locations = instruction->GetLocations(); 7040 Location first = locations->InAt(0); 7041 Location second = locations->InAt(1); 7042 Location out = locations->Out(); 7043 7044 if (second.IsConstant()) { 7045 uint64_t value = static_cast<uint64_t>(Int64FromConstant(second.GetConstant())); 7046 uint32_t value_low = Low32Bits(value); 7047 if (instruction->GetResultType() == Primitive::kPrimInt) { 7048 Register first_reg = first.AsRegister<Register>(); 7049 Register out_reg = out.AsRegister<Register>(); 7050 if (instruction->IsAnd()) { 7051 GenerateAndConst(out_reg, first_reg, value_low); 7052 } else if (instruction->IsOr()) { 7053 GenerateOrrConst(out_reg, first_reg, value_low); 7054 } else { 7055 DCHECK(instruction->IsXor()); 7056 GenerateEorConst(out_reg, first_reg, value_low); 7057 } 7058 } else { 7059 DCHECK_EQ(instruction->GetResultType(), Primitive::kPrimLong); 7060 uint32_t value_high = High32Bits(value); 7061 Register first_low = first.AsRegisterPairLow<Register>(); 7062 Register first_high = first.AsRegisterPairHigh<Register>(); 7063 Register out_low = out.AsRegisterPairLow<Register>(); 7064 Register out_high = out.AsRegisterPairHigh<Register>(); 7065 if (instruction->IsAnd()) { 7066 GenerateAndConst(out_low, first_low, value_low); 7067 GenerateAndConst(out_high, first_high, value_high); 7068 } else if (instruction->IsOr()) { 7069 GenerateOrrConst(out_low, first_low, value_low); 7070 GenerateOrrConst(out_high, first_high, value_high); 7071 } else { 7072 DCHECK(instruction->IsXor()); 7073 GenerateEorConst(out_low, first_low, value_low); 7074 GenerateEorConst(out_high, first_high, value_high); 7075 } 7076 } 7077 return; 7078 } 7079 7080 if (instruction->GetResultType() == Primitive::kPrimInt) { 7081 Register first_reg = first.AsRegister<Register>(); 7082 ShifterOperand second_reg(second.AsRegister<Register>()); 7083 Register out_reg = out.AsRegister<Register>(); 7084 if (instruction->IsAnd()) { 7085 __ and_(out_reg, first_reg, second_reg); 7086 } else if (instruction->IsOr()) { 7087 __ orr(out_reg, first_reg, second_reg); 7088 } else { 7089 DCHECK(instruction->IsXor()); 7090 __ eor(out_reg, first_reg, second_reg); 7091 } 7092 } else { 7093 DCHECK_EQ(instruction->GetResultType(), Primitive::kPrimLong); 7094 Register first_low = first.AsRegisterPairLow<Register>(); 7095 Register first_high = first.AsRegisterPairHigh<Register>(); 7096 ShifterOperand second_low(second.AsRegisterPairLow<Register>()); 7097 ShifterOperand second_high(second.AsRegisterPairHigh<Register>()); 7098 Register out_low = out.AsRegisterPairLow<Register>(); 7099 Register out_high = out.AsRegisterPairHigh<Register>(); 7100 if (instruction->IsAnd()) { 7101 __ and_(out_low, first_low, second_low); 7102 __ and_(out_high, first_high, second_high); 7103 } else if (instruction->IsOr()) { 7104 __ orr(out_low, first_low, second_low); 7105 __ orr(out_high, first_high, second_high); 7106 } else { 7107 DCHECK(instruction->IsXor()); 7108 __ eor(out_low, first_low, second_low); 7109 __ eor(out_high, first_high, second_high); 7110 } 7111 } 7112} 7113 7114void InstructionCodeGeneratorARM::GenerateReferenceLoadOneRegister( 7115 HInstruction* instruction, 7116 Location out, 7117 uint32_t offset, 7118 Location maybe_temp, 7119 ReadBarrierOption read_barrier_option) { 7120 Register out_reg = out.AsRegister<Register>(); 7121 if (read_barrier_option == kWithReadBarrier) { 7122 CHECK(kEmitCompilerReadBarrier); 7123 DCHECK(maybe_temp.IsRegister()) << maybe_temp; 7124 if (kUseBakerReadBarrier) { 7125 // Load with fast path based Baker's read barrier. 7126 // /* HeapReference<Object> */ out = *(out + offset) 7127 codegen_->GenerateFieldLoadWithBakerReadBarrier( 7128 instruction, out, out_reg, offset, maybe_temp, /* needs_null_check */ false); 7129 } else { 7130 // Load with slow path based read barrier. 7131 // Save the value of `out` into `maybe_temp` before overwriting it 7132 // in the following move operation, as we will need it for the 7133 // read barrier below. 7134 __ Mov(maybe_temp.AsRegister<Register>(), out_reg); 7135 // /* HeapReference<Object> */ out = *(out + offset) 7136 __ LoadFromOffset(kLoadWord, out_reg, out_reg, offset); 7137 codegen_->GenerateReadBarrierSlow(instruction, out, out, maybe_temp, offset); 7138 } 7139 } else { 7140 // Plain load with no read barrier. 7141 // /* HeapReference<Object> */ out = *(out + offset) 7142 __ LoadFromOffset(kLoadWord, out_reg, out_reg, offset); 7143 __ MaybeUnpoisonHeapReference(out_reg); 7144 } 7145} 7146 7147void InstructionCodeGeneratorARM::GenerateReferenceLoadTwoRegisters( 7148 HInstruction* instruction, 7149 Location out, 7150 Location obj, 7151 uint32_t offset, 7152 Location maybe_temp, 7153 ReadBarrierOption read_barrier_option) { 7154 Register out_reg = out.AsRegister<Register>(); 7155 Register obj_reg = obj.AsRegister<Register>(); 7156 if (read_barrier_option == kWithReadBarrier) { 7157 CHECK(kEmitCompilerReadBarrier); 7158 if (kUseBakerReadBarrier) { 7159 DCHECK(maybe_temp.IsRegister()) << maybe_temp; 7160 // Load with fast path based Baker's read barrier. 7161 // /* HeapReference<Object> */ out = *(obj + offset) 7162 codegen_->GenerateFieldLoadWithBakerReadBarrier( 7163 instruction, out, obj_reg, offset, maybe_temp, /* needs_null_check */ false); 7164 } else { 7165 // Load with slow path based read barrier. 7166 // /* HeapReference<Object> */ out = *(obj + offset) 7167 __ LoadFromOffset(kLoadWord, out_reg, obj_reg, offset); 7168 codegen_->GenerateReadBarrierSlow(instruction, out, out, obj, offset); 7169 } 7170 } else { 7171 // Plain load with no read barrier. 7172 // /* HeapReference<Object> */ out = *(obj + offset) 7173 __ LoadFromOffset(kLoadWord, out_reg, obj_reg, offset); 7174 __ MaybeUnpoisonHeapReference(out_reg); 7175 } 7176} 7177 7178void InstructionCodeGeneratorARM::GenerateGcRootFieldLoad(HInstruction* instruction, 7179 Location root, 7180 Register obj, 7181 uint32_t offset, 7182 ReadBarrierOption read_barrier_option) { 7183 Register root_reg = root.AsRegister<Register>(); 7184 if (read_barrier_option == kWithReadBarrier) { 7185 DCHECK(kEmitCompilerReadBarrier); 7186 if (kUseBakerReadBarrier) { 7187 // Fast path implementation of art::ReadBarrier::BarrierForRoot when 7188 // Baker's read barrier are used: 7189 // 7190 // root = obj.field; 7191 // temp = Thread::Current()->pReadBarrierMarkReg ## root.reg() 7192 // if (temp != null) { 7193 // root = temp(root) 7194 // } 7195 7196 // /* GcRoot<mirror::Object> */ root = *(obj + offset) 7197 __ LoadFromOffset(kLoadWord, root_reg, obj, offset); 7198 static_assert( 7199 sizeof(mirror::CompressedReference<mirror::Object>) == sizeof(GcRoot<mirror::Object>), 7200 "art::mirror::CompressedReference<mirror::Object> and art::GcRoot<mirror::Object> " 7201 "have different sizes."); 7202 static_assert(sizeof(mirror::CompressedReference<mirror::Object>) == sizeof(int32_t), 7203 "art::mirror::CompressedReference<mirror::Object> and int32_t " 7204 "have different sizes."); 7205 7206 // Slow path marking the GC root `root`. 7207 Location temp = Location::RegisterLocation(LR); 7208 SlowPathCodeARM* slow_path = 7209 new (GetGraph()->GetArena()) ReadBarrierMarkSlowPathARM( 7210 instruction, 7211 root, 7212 /*entrypoint*/ temp); 7213 codegen_->AddSlowPath(slow_path); 7214 7215 // temp = Thread::Current()->pReadBarrierMarkReg ## root.reg() 7216 const int32_t entry_point_offset = 7217 CodeGenerator::GetReadBarrierMarkEntryPointsOffset<kArmPointerSize>(root.reg()); 7218 // Loading the entrypoint does not require a load acquire since it is only changed when 7219 // threads are suspended or running a checkpoint. 7220 __ LoadFromOffset(kLoadWord, temp.AsRegister<Register>(), TR, entry_point_offset); 7221 // The entrypoint is null when the GC is not marking, this prevents one load compared to 7222 // checking GetIsGcMarking. 7223 __ CompareAndBranchIfNonZero(temp.AsRegister<Register>(), slow_path->GetEntryLabel()); 7224 __ Bind(slow_path->GetExitLabel()); 7225 } else { 7226 // GC root loaded through a slow path for read barriers other 7227 // than Baker's. 7228 // /* GcRoot<mirror::Object>* */ root = obj + offset 7229 __ AddConstant(root_reg, obj, offset); 7230 // /* mirror::Object* */ root = root->Read() 7231 codegen_->GenerateReadBarrierForRootSlow(instruction, root, root); 7232 } 7233 } else { 7234 // Plain GC root load with no read barrier. 7235 // /* GcRoot<mirror::Object> */ root = *(obj + offset) 7236 __ LoadFromOffset(kLoadWord, root_reg, obj, offset); 7237 // Note that GC roots are not affected by heap poisoning, thus we 7238 // do not have to unpoison `root_reg` here. 7239 } 7240} 7241 7242void CodeGeneratorARM::GenerateFieldLoadWithBakerReadBarrier(HInstruction* instruction, 7243 Location ref, 7244 Register obj, 7245 uint32_t offset, 7246 Location temp, 7247 bool needs_null_check) { 7248 DCHECK(kEmitCompilerReadBarrier); 7249 DCHECK(kUseBakerReadBarrier); 7250 7251 // /* HeapReference<Object> */ ref = *(obj + offset) 7252 Location no_index = Location::NoLocation(); 7253 ScaleFactor no_scale_factor = TIMES_1; 7254 GenerateReferenceLoadWithBakerReadBarrier( 7255 instruction, ref, obj, offset, no_index, no_scale_factor, temp, needs_null_check); 7256} 7257 7258void CodeGeneratorARM::GenerateArrayLoadWithBakerReadBarrier(HInstruction* instruction, 7259 Location ref, 7260 Register obj, 7261 uint32_t data_offset, 7262 Location index, 7263 Location temp, 7264 bool needs_null_check) { 7265 DCHECK(kEmitCompilerReadBarrier); 7266 DCHECK(kUseBakerReadBarrier); 7267 7268 static_assert( 7269 sizeof(mirror::HeapReference<mirror::Object>) == sizeof(int32_t), 7270 "art::mirror::HeapReference<art::mirror::Object> and int32_t have different sizes."); 7271 // /* HeapReference<Object> */ ref = 7272 // *(obj + data_offset + index * sizeof(HeapReference<Object>)) 7273 ScaleFactor scale_factor = TIMES_4; 7274 GenerateReferenceLoadWithBakerReadBarrier( 7275 instruction, ref, obj, data_offset, index, scale_factor, temp, needs_null_check); 7276} 7277 7278void CodeGeneratorARM::GenerateReferenceLoadWithBakerReadBarrier(HInstruction* instruction, 7279 Location ref, 7280 Register obj, 7281 uint32_t offset, 7282 Location index, 7283 ScaleFactor scale_factor, 7284 Location temp, 7285 bool needs_null_check, 7286 bool always_update_field, 7287 Register* temp2) { 7288 DCHECK(kEmitCompilerReadBarrier); 7289 DCHECK(kUseBakerReadBarrier); 7290 7291 // In slow path based read barriers, the read barrier call is 7292 // inserted after the original load. However, in fast path based 7293 // Baker's read barriers, we need to perform the load of 7294 // mirror::Object::monitor_ *before* the original reference load. 7295 // This load-load ordering is required by the read barrier. 7296 // The fast path/slow path (for Baker's algorithm) should look like: 7297 // 7298 // uint32_t rb_state = Lockword(obj->monitor_).ReadBarrierState(); 7299 // lfence; // Load fence or artificial data dependency to prevent load-load reordering 7300 // HeapReference<Object> ref = *src; // Original reference load. 7301 // bool is_gray = (rb_state == ReadBarrier::GrayState()); 7302 // if (is_gray) { 7303 // ref = ReadBarrier::Mark(ref); // Performed by runtime entrypoint slow path. 7304 // } 7305 // 7306 // Note: the original implementation in ReadBarrier::Barrier is 7307 // slightly more complex as it performs additional checks that we do 7308 // not do here for performance reasons. 7309 7310 Register ref_reg = ref.AsRegister<Register>(); 7311 Register temp_reg = temp.AsRegister<Register>(); 7312 uint32_t monitor_offset = mirror::Object::MonitorOffset().Int32Value(); 7313 7314 // /* int32_t */ monitor = obj->monitor_ 7315 __ LoadFromOffset(kLoadWord, temp_reg, obj, monitor_offset); 7316 if (needs_null_check) { 7317 MaybeRecordImplicitNullCheck(instruction); 7318 } 7319 // /* LockWord */ lock_word = LockWord(monitor) 7320 static_assert(sizeof(LockWord) == sizeof(int32_t), 7321 "art::LockWord and int32_t have different sizes."); 7322 7323 // Introduce a dependency on the lock_word including the rb_state, 7324 // which shall prevent load-load reordering without using 7325 // a memory barrier (which would be more expensive). 7326 // `obj` is unchanged by this operation, but its value now depends 7327 // on `temp_reg`. 7328 __ add(obj, obj, ShifterOperand(temp_reg, LSR, 32)); 7329 7330 // The actual reference load. 7331 if (index.IsValid()) { 7332 // Load types involving an "index": ArrayGet, 7333 // UnsafeGetObject/UnsafeGetObjectVolatile and UnsafeCASObject 7334 // intrinsics. 7335 // /* HeapReference<Object> */ ref = *(obj + offset + (index << scale_factor)) 7336 if (index.IsConstant()) { 7337 size_t computed_offset = 7338 (index.GetConstant()->AsIntConstant()->GetValue() << scale_factor) + offset; 7339 __ LoadFromOffset(kLoadWord, ref_reg, obj, computed_offset); 7340 } else { 7341 // Handle the special case of the 7342 // UnsafeGetObject/UnsafeGetObjectVolatile and UnsafeCASObject 7343 // intrinsics, which use a register pair as index ("long 7344 // offset"), of which only the low part contains data. 7345 Register index_reg = index.IsRegisterPair() 7346 ? index.AsRegisterPairLow<Register>() 7347 : index.AsRegister<Register>(); 7348 __ add(IP, obj, ShifterOperand(index_reg, LSL, scale_factor)); 7349 __ LoadFromOffset(kLoadWord, ref_reg, IP, offset); 7350 } 7351 } else { 7352 // /* HeapReference<Object> */ ref = *(obj + offset) 7353 __ LoadFromOffset(kLoadWord, ref_reg, obj, offset); 7354 } 7355 7356 // Object* ref = ref_addr->AsMirrorPtr() 7357 __ MaybeUnpoisonHeapReference(ref_reg); 7358 7359 // Slow path marking the object `ref` when it is gray. 7360 SlowPathCodeARM* slow_path; 7361 if (always_update_field) { 7362 DCHECK(temp2 != nullptr); 7363 // ReadBarrierMarkAndUpdateFieldSlowPathARM only supports address 7364 // of the form `obj + field_offset`, where `obj` is a register and 7365 // `field_offset` is a register pair (of which only the lower half 7366 // is used). Thus `offset` and `scale_factor` above are expected 7367 // to be null in this code path. 7368 DCHECK_EQ(offset, 0u); 7369 DCHECK_EQ(scale_factor, ScaleFactor::TIMES_1); 7370 slow_path = new (GetGraph()->GetArena()) ReadBarrierMarkAndUpdateFieldSlowPathARM( 7371 instruction, ref, obj, /* field_offset */ index, temp_reg, *temp2); 7372 } else { 7373 slow_path = new (GetGraph()->GetArena()) ReadBarrierMarkSlowPathARM(instruction, ref); 7374 } 7375 AddSlowPath(slow_path); 7376 7377 // if (rb_state == ReadBarrier::GrayState()) 7378 // ref = ReadBarrier::Mark(ref); 7379 // Given the numeric representation, it's enough to check the low bit of the 7380 // rb_state. We do that by shifting the bit out of the lock word with LSRS 7381 // which can be a 16-bit instruction unlike the TST immediate. 7382 static_assert(ReadBarrier::WhiteState() == 0, "Expecting white to have value 0"); 7383 static_assert(ReadBarrier::GrayState() == 1, "Expecting gray to have value 1"); 7384 __ Lsrs(temp_reg, temp_reg, LockWord::kReadBarrierStateShift + 1); 7385 __ b(slow_path->GetEntryLabel(), CS); // Carry flag is the last bit shifted out by LSRS. 7386 __ Bind(slow_path->GetExitLabel()); 7387} 7388 7389void CodeGeneratorARM::GenerateReadBarrierSlow(HInstruction* instruction, 7390 Location out, 7391 Location ref, 7392 Location obj, 7393 uint32_t offset, 7394 Location index) { 7395 DCHECK(kEmitCompilerReadBarrier); 7396 7397 // Insert a slow path based read barrier *after* the reference load. 7398 // 7399 // If heap poisoning is enabled, the unpoisoning of the loaded 7400 // reference will be carried out by the runtime within the slow 7401 // path. 7402 // 7403 // Note that `ref` currently does not get unpoisoned (when heap 7404 // poisoning is enabled), which is alright as the `ref` argument is 7405 // not used by the artReadBarrierSlow entry point. 7406 // 7407 // TODO: Unpoison `ref` when it is used by artReadBarrierSlow. 7408 SlowPathCodeARM* slow_path = new (GetGraph()->GetArena()) 7409 ReadBarrierForHeapReferenceSlowPathARM(instruction, out, ref, obj, offset, index); 7410 AddSlowPath(slow_path); 7411 7412 __ b(slow_path->GetEntryLabel()); 7413 __ Bind(slow_path->GetExitLabel()); 7414} 7415 7416void CodeGeneratorARM::MaybeGenerateReadBarrierSlow(HInstruction* instruction, 7417 Location out, 7418 Location ref, 7419 Location obj, 7420 uint32_t offset, 7421 Location index) { 7422 if (kEmitCompilerReadBarrier) { 7423 // Baker's read barriers shall be handled by the fast path 7424 // (CodeGeneratorARM::GenerateReferenceLoadWithBakerReadBarrier). 7425 DCHECK(!kUseBakerReadBarrier); 7426 // If heap poisoning is enabled, unpoisoning will be taken care of 7427 // by the runtime within the slow path. 7428 GenerateReadBarrierSlow(instruction, out, ref, obj, offset, index); 7429 } else if (kPoisonHeapReferences) { 7430 __ UnpoisonHeapReference(out.AsRegister<Register>()); 7431 } 7432} 7433 7434void CodeGeneratorARM::GenerateReadBarrierForRootSlow(HInstruction* instruction, 7435 Location out, 7436 Location root) { 7437 DCHECK(kEmitCompilerReadBarrier); 7438 7439 // Insert a slow path based read barrier *after* the GC root load. 7440 // 7441 // Note that GC roots are not affected by heap poisoning, so we do 7442 // not need to do anything special for this here. 7443 SlowPathCodeARM* slow_path = 7444 new (GetGraph()->GetArena()) ReadBarrierForRootSlowPathARM(instruction, out, root); 7445 AddSlowPath(slow_path); 7446 7447 __ b(slow_path->GetEntryLabel()); 7448 __ Bind(slow_path->GetExitLabel()); 7449} 7450 7451HInvokeStaticOrDirect::DispatchInfo CodeGeneratorARM::GetSupportedInvokeStaticOrDirectDispatch( 7452 const HInvokeStaticOrDirect::DispatchInfo& desired_dispatch_info, 7453 HInvokeStaticOrDirect* invoke ATTRIBUTE_UNUSED) { 7454 return desired_dispatch_info; 7455} 7456 7457Register CodeGeneratorARM::GetInvokeStaticOrDirectExtraParameter(HInvokeStaticOrDirect* invoke, 7458 Register temp) { 7459 DCHECK_EQ(invoke->InputCount(), invoke->GetNumberOfArguments() + 1u); 7460 Location location = invoke->GetLocations()->InAt(invoke->GetSpecialInputIndex()); 7461 if (!invoke->GetLocations()->Intrinsified()) { 7462 return location.AsRegister<Register>(); 7463 } 7464 // For intrinsics we allow any location, so it may be on the stack. 7465 if (!location.IsRegister()) { 7466 __ LoadFromOffset(kLoadWord, temp, SP, location.GetStackIndex()); 7467 return temp; 7468 } 7469 // For register locations, check if the register was saved. If so, get it from the stack. 7470 // Note: There is a chance that the register was saved but not overwritten, so we could 7471 // save one load. However, since this is just an intrinsic slow path we prefer this 7472 // simple and more robust approach rather that trying to determine if that's the case. 7473 SlowPathCode* slow_path = GetCurrentSlowPath(); 7474 if (slow_path != nullptr && slow_path->IsCoreRegisterSaved(location.AsRegister<Register>())) { 7475 int stack_offset = slow_path->GetStackOffsetOfCoreRegister(location.AsRegister<Register>()); 7476 __ LoadFromOffset(kLoadWord, temp, SP, stack_offset); 7477 return temp; 7478 } 7479 return location.AsRegister<Register>(); 7480} 7481 7482Location CodeGeneratorARM::GenerateCalleeMethodStaticOrDirectCall(HInvokeStaticOrDirect* invoke, 7483 Location temp) { 7484 Location callee_method = temp; // For all kinds except kRecursive, callee will be in temp. 7485 switch (invoke->GetMethodLoadKind()) { 7486 case HInvokeStaticOrDirect::MethodLoadKind::kStringInit: { 7487 uint32_t offset = 7488 GetThreadOffset<kArmPointerSize>(invoke->GetStringInitEntryPoint()).Int32Value(); 7489 // temp = thread->string_init_entrypoint 7490 __ LoadFromOffset(kLoadWord, temp.AsRegister<Register>(), TR, offset); 7491 break; 7492 } 7493 case HInvokeStaticOrDirect::MethodLoadKind::kRecursive: 7494 callee_method = invoke->GetLocations()->InAt(invoke->GetSpecialInputIndex()); 7495 break; 7496 case HInvokeStaticOrDirect::MethodLoadKind::kDirectAddress: 7497 __ LoadImmediate(temp.AsRegister<Register>(), invoke->GetMethodAddress()); 7498 break; 7499 case HInvokeStaticOrDirect::MethodLoadKind::kDexCachePcRelative: { 7500 HArmDexCacheArraysBase* base = 7501 invoke->InputAt(invoke->GetSpecialInputIndex())->AsArmDexCacheArraysBase(); 7502 Register base_reg = GetInvokeStaticOrDirectExtraParameter(invoke, 7503 temp.AsRegister<Register>()); 7504 int32_t offset = invoke->GetDexCacheArrayOffset() - base->GetElementOffset(); 7505 __ LoadFromOffset(kLoadWord, temp.AsRegister<Register>(), base_reg, offset); 7506 break; 7507 } 7508 case HInvokeStaticOrDirect::MethodLoadKind::kDexCacheViaMethod: { 7509 Location current_method = invoke->GetLocations()->InAt(invoke->GetSpecialInputIndex()); 7510 Register method_reg; 7511 Register reg = temp.AsRegister<Register>(); 7512 if (current_method.IsRegister()) { 7513 method_reg = current_method.AsRegister<Register>(); 7514 } else { 7515 DCHECK(invoke->GetLocations()->Intrinsified()); 7516 DCHECK(!current_method.IsValid()); 7517 method_reg = reg; 7518 __ LoadFromOffset(kLoadWord, reg, SP, kCurrentMethodStackOffset); 7519 } 7520 // /* ArtMethod*[] */ temp = temp.ptr_sized_fields_->dex_cache_resolved_methods_; 7521 __ LoadFromOffset(kLoadWord, 7522 reg, 7523 method_reg, 7524 ArtMethod::DexCacheResolvedMethodsOffset(kArmPointerSize).Int32Value()); 7525 // temp = temp[index_in_cache]; 7526 // Note: Don't use invoke->GetTargetMethod() as it may point to a different dex file. 7527 uint32_t index_in_cache = invoke->GetDexMethodIndex(); 7528 __ LoadFromOffset(kLoadWord, reg, reg, CodeGenerator::GetCachePointerOffset(index_in_cache)); 7529 break; 7530 } 7531 } 7532 return callee_method; 7533} 7534 7535void CodeGeneratorARM::GenerateStaticOrDirectCall(HInvokeStaticOrDirect* invoke, Location temp) { 7536 Location callee_method = GenerateCalleeMethodStaticOrDirectCall(invoke, temp); 7537 7538 switch (invoke->GetCodePtrLocation()) { 7539 case HInvokeStaticOrDirect::CodePtrLocation::kCallSelf: 7540 __ bl(GetFrameEntryLabel()); 7541 break; 7542 case HInvokeStaticOrDirect::CodePtrLocation::kCallArtMethod: 7543 // LR = callee_method->entry_point_from_quick_compiled_code_ 7544 __ LoadFromOffset( 7545 kLoadWord, LR, callee_method.AsRegister<Register>(), 7546 ArtMethod::EntryPointFromQuickCompiledCodeOffset(kArmPointerSize).Int32Value()); 7547 // LR() 7548 __ blx(LR); 7549 break; 7550 } 7551 7552 DCHECK(!IsLeafMethod()); 7553} 7554 7555void CodeGeneratorARM::GenerateVirtualCall(HInvokeVirtual* invoke, Location temp_location) { 7556 Register temp = temp_location.AsRegister<Register>(); 7557 uint32_t method_offset = mirror::Class::EmbeddedVTableEntryOffset( 7558 invoke->GetVTableIndex(), kArmPointerSize).Uint32Value(); 7559 7560 // Use the calling convention instead of the location of the receiver, as 7561 // intrinsics may have put the receiver in a different register. In the intrinsics 7562 // slow path, the arguments have been moved to the right place, so here we are 7563 // guaranteed that the receiver is the first register of the calling convention. 7564 InvokeDexCallingConvention calling_convention; 7565 Register receiver = calling_convention.GetRegisterAt(0); 7566 uint32_t class_offset = mirror::Object::ClassOffset().Int32Value(); 7567 // /* HeapReference<Class> */ temp = receiver->klass_ 7568 __ LoadFromOffset(kLoadWord, temp, receiver, class_offset); 7569 MaybeRecordImplicitNullCheck(invoke); 7570 // Instead of simply (possibly) unpoisoning `temp` here, we should 7571 // emit a read barrier for the previous class reference load. 7572 // However this is not required in practice, as this is an 7573 // intermediate/temporary reference and because the current 7574 // concurrent copying collector keeps the from-space memory 7575 // intact/accessible until the end of the marking phase (the 7576 // concurrent copying collector may not in the future). 7577 __ MaybeUnpoisonHeapReference(temp); 7578 // temp = temp->GetMethodAt(method_offset); 7579 uint32_t entry_point = ArtMethod::EntryPointFromQuickCompiledCodeOffset( 7580 kArmPointerSize).Int32Value(); 7581 __ LoadFromOffset(kLoadWord, temp, temp, method_offset); 7582 // LR = temp->GetEntryPoint(); 7583 __ LoadFromOffset(kLoadWord, LR, temp, entry_point); 7584 // LR(); 7585 __ blx(LR); 7586} 7587 7588CodeGeneratorARM::PcRelativePatchInfo* CodeGeneratorARM::NewPcRelativeStringPatch( 7589 const DexFile& dex_file, dex::StringIndex string_index) { 7590 return NewPcRelativePatch(dex_file, string_index.index_, &pc_relative_string_patches_); 7591} 7592 7593CodeGeneratorARM::PcRelativePatchInfo* CodeGeneratorARM::NewPcRelativeTypePatch( 7594 const DexFile& dex_file, dex::TypeIndex type_index) { 7595 return NewPcRelativePatch(dex_file, type_index.index_, &pc_relative_type_patches_); 7596} 7597 7598CodeGeneratorARM::PcRelativePatchInfo* CodeGeneratorARM::NewTypeBssEntryPatch( 7599 const DexFile& dex_file, dex::TypeIndex type_index) { 7600 return NewPcRelativePatch(dex_file, type_index.index_, &type_bss_entry_patches_); 7601} 7602 7603CodeGeneratorARM::PcRelativePatchInfo* CodeGeneratorARM::NewPcRelativeDexCacheArrayPatch( 7604 const DexFile& dex_file, uint32_t element_offset) { 7605 return NewPcRelativePatch(dex_file, element_offset, &pc_relative_dex_cache_patches_); 7606} 7607 7608CodeGeneratorARM::PcRelativePatchInfo* CodeGeneratorARM::NewPcRelativePatch( 7609 const DexFile& dex_file, uint32_t offset_or_index, ArenaDeque<PcRelativePatchInfo>* patches) { 7610 patches->emplace_back(dex_file, offset_or_index); 7611 return &patches->back(); 7612} 7613 7614Literal* CodeGeneratorARM::DeduplicateBootImageStringLiteral(const DexFile& dex_file, 7615 dex::StringIndex string_index) { 7616 return boot_image_string_patches_.GetOrCreate( 7617 StringReference(&dex_file, string_index), 7618 [this]() { return __ NewLiteral<uint32_t>(/* placeholder */ 0u); }); 7619} 7620 7621Literal* CodeGeneratorARM::DeduplicateBootImageTypeLiteral(const DexFile& dex_file, 7622 dex::TypeIndex type_index) { 7623 return boot_image_type_patches_.GetOrCreate( 7624 TypeReference(&dex_file, type_index), 7625 [this]() { return __ NewLiteral<uint32_t>(/* placeholder */ 0u); }); 7626} 7627 7628Literal* CodeGeneratorARM::DeduplicateBootImageAddressLiteral(uint32_t address) { 7629 bool needs_patch = GetCompilerOptions().GetIncludePatchInformation(); 7630 Uint32ToLiteralMap* map = needs_patch ? &boot_image_address_patches_ : &uint32_literals_; 7631 return DeduplicateUint32Literal(dchecked_integral_cast<uint32_t>(address), map); 7632} 7633 7634Literal* CodeGeneratorARM::DeduplicateJitStringLiteral(const DexFile& dex_file, 7635 dex::StringIndex string_index, 7636 Handle<mirror::String> handle) { 7637 jit_string_roots_.Overwrite(StringReference(&dex_file, string_index), 7638 reinterpret_cast64<uint64_t>(handle.GetReference())); 7639 return jit_string_patches_.GetOrCreate( 7640 StringReference(&dex_file, string_index), 7641 [this]() { return __ NewLiteral<uint32_t>(/* placeholder */ 0u); }); 7642} 7643 7644Literal* CodeGeneratorARM::DeduplicateJitClassLiteral(const DexFile& dex_file, 7645 dex::TypeIndex type_index, 7646 Handle<mirror::Class> handle) { 7647 jit_class_roots_.Overwrite(TypeReference(&dex_file, type_index), 7648 reinterpret_cast64<uint64_t>(handle.GetReference())); 7649 return jit_class_patches_.GetOrCreate( 7650 TypeReference(&dex_file, type_index), 7651 [this]() { return __ NewLiteral<uint32_t>(/* placeholder */ 0u); }); 7652} 7653 7654template <LinkerPatch (*Factory)(size_t, const DexFile*, uint32_t, uint32_t)> 7655inline void CodeGeneratorARM::EmitPcRelativeLinkerPatches( 7656 const ArenaDeque<PcRelativePatchInfo>& infos, 7657 ArenaVector<LinkerPatch>* linker_patches) { 7658 for (const PcRelativePatchInfo& info : infos) { 7659 const DexFile& dex_file = info.target_dex_file; 7660 size_t offset_or_index = info.offset_or_index; 7661 DCHECK(info.add_pc_label.IsBound()); 7662 uint32_t add_pc_offset = dchecked_integral_cast<uint32_t>(info.add_pc_label.Position()); 7663 // Add MOVW patch. 7664 DCHECK(info.movw_label.IsBound()); 7665 uint32_t movw_offset = dchecked_integral_cast<uint32_t>(info.movw_label.Position()); 7666 linker_patches->push_back(Factory(movw_offset, &dex_file, add_pc_offset, offset_or_index)); 7667 // Add MOVT patch. 7668 DCHECK(info.movt_label.IsBound()); 7669 uint32_t movt_offset = dchecked_integral_cast<uint32_t>(info.movt_label.Position()); 7670 linker_patches->push_back(Factory(movt_offset, &dex_file, add_pc_offset, offset_or_index)); 7671 } 7672} 7673 7674void CodeGeneratorARM::EmitLinkerPatches(ArenaVector<LinkerPatch>* linker_patches) { 7675 DCHECK(linker_patches->empty()); 7676 size_t size = 7677 /* MOVW+MOVT for each entry */ 2u * pc_relative_dex_cache_patches_.size() + 7678 boot_image_string_patches_.size() + 7679 /* MOVW+MOVT for each entry */ 2u * pc_relative_string_patches_.size() + 7680 boot_image_type_patches_.size() + 7681 /* MOVW+MOVT for each entry */ 2u * pc_relative_type_patches_.size() + 7682 /* MOVW+MOVT for each entry */ 2u * type_bss_entry_patches_.size() + 7683 boot_image_address_patches_.size(); 7684 linker_patches->reserve(size); 7685 EmitPcRelativeLinkerPatches<LinkerPatch::DexCacheArrayPatch>(pc_relative_dex_cache_patches_, 7686 linker_patches); 7687 for (const auto& entry : boot_image_string_patches_) { 7688 const StringReference& target_string = entry.first; 7689 Literal* literal = entry.second; 7690 DCHECK(literal->GetLabel()->IsBound()); 7691 uint32_t literal_offset = literal->GetLabel()->Position(); 7692 linker_patches->push_back(LinkerPatch::StringPatch(literal_offset, 7693 target_string.dex_file, 7694 target_string.string_index.index_)); 7695 } 7696 if (!GetCompilerOptions().IsBootImage()) { 7697 DCHECK(pc_relative_type_patches_.empty()); 7698 EmitPcRelativeLinkerPatches<LinkerPatch::StringBssEntryPatch>(pc_relative_string_patches_, 7699 linker_patches); 7700 } else { 7701 EmitPcRelativeLinkerPatches<LinkerPatch::RelativeTypePatch>(pc_relative_type_patches_, 7702 linker_patches); 7703 EmitPcRelativeLinkerPatches<LinkerPatch::RelativeStringPatch>(pc_relative_string_patches_, 7704 linker_patches); 7705 } 7706 EmitPcRelativeLinkerPatches<LinkerPatch::TypeBssEntryPatch>(type_bss_entry_patches_, 7707 linker_patches); 7708 for (const auto& entry : boot_image_type_patches_) { 7709 const TypeReference& target_type = entry.first; 7710 Literal* literal = entry.second; 7711 DCHECK(literal->GetLabel()->IsBound()); 7712 uint32_t literal_offset = literal->GetLabel()->Position(); 7713 linker_patches->push_back(LinkerPatch::TypePatch(literal_offset, 7714 target_type.dex_file, 7715 target_type.type_index.index_)); 7716 } 7717 for (const auto& entry : boot_image_address_patches_) { 7718 DCHECK(GetCompilerOptions().GetIncludePatchInformation()); 7719 Literal* literal = entry.second; 7720 DCHECK(literal->GetLabel()->IsBound()); 7721 uint32_t literal_offset = literal->GetLabel()->Position(); 7722 linker_patches->push_back(LinkerPatch::RecordPosition(literal_offset)); 7723 } 7724 DCHECK_EQ(size, linker_patches->size()); 7725} 7726 7727Literal* CodeGeneratorARM::DeduplicateUint32Literal(uint32_t value, Uint32ToLiteralMap* map) { 7728 return map->GetOrCreate( 7729 value, 7730 [this, value]() { return __ NewLiteral<uint32_t>(value); }); 7731} 7732 7733Literal* CodeGeneratorARM::DeduplicateMethodLiteral(MethodReference target_method, 7734 MethodToLiteralMap* map) { 7735 return map->GetOrCreate( 7736 target_method, 7737 [this]() { return __ NewLiteral<uint32_t>(/* placeholder */ 0u); }); 7738} 7739 7740void LocationsBuilderARM::VisitMultiplyAccumulate(HMultiplyAccumulate* instr) { 7741 LocationSummary* locations = 7742 new (GetGraph()->GetArena()) LocationSummary(instr, LocationSummary::kNoCall); 7743 locations->SetInAt(HMultiplyAccumulate::kInputAccumulatorIndex, 7744 Location::RequiresRegister()); 7745 locations->SetInAt(HMultiplyAccumulate::kInputMulLeftIndex, Location::RequiresRegister()); 7746 locations->SetInAt(HMultiplyAccumulate::kInputMulRightIndex, Location::RequiresRegister()); 7747 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 7748} 7749 7750void InstructionCodeGeneratorARM::VisitMultiplyAccumulate(HMultiplyAccumulate* instr) { 7751 LocationSummary* locations = instr->GetLocations(); 7752 Register res = locations->Out().AsRegister<Register>(); 7753 Register accumulator = 7754 locations->InAt(HMultiplyAccumulate::kInputAccumulatorIndex).AsRegister<Register>(); 7755 Register mul_left = 7756 locations->InAt(HMultiplyAccumulate::kInputMulLeftIndex).AsRegister<Register>(); 7757 Register mul_right = 7758 locations->InAt(HMultiplyAccumulate::kInputMulRightIndex).AsRegister<Register>(); 7759 7760 if (instr->GetOpKind() == HInstruction::kAdd) { 7761 __ mla(res, mul_left, mul_right, accumulator); 7762 } else { 7763 __ mls(res, mul_left, mul_right, accumulator); 7764 } 7765} 7766 7767void LocationsBuilderARM::VisitBoundType(HBoundType* instruction ATTRIBUTE_UNUSED) { 7768 // Nothing to do, this should be removed during prepare for register allocator. 7769 LOG(FATAL) << "Unreachable"; 7770} 7771 7772void InstructionCodeGeneratorARM::VisitBoundType(HBoundType* instruction ATTRIBUTE_UNUSED) { 7773 // Nothing to do, this should be removed during prepare for register allocator. 7774 LOG(FATAL) << "Unreachable"; 7775} 7776 7777// Simple implementation of packed switch - generate cascaded compare/jumps. 7778void LocationsBuilderARM::VisitPackedSwitch(HPackedSwitch* switch_instr) { 7779 LocationSummary* locations = 7780 new (GetGraph()->GetArena()) LocationSummary(switch_instr, LocationSummary::kNoCall); 7781 locations->SetInAt(0, Location::RequiresRegister()); 7782 if (switch_instr->GetNumEntries() > kPackedSwitchCompareJumpThreshold && 7783 codegen_->GetAssembler()->IsThumb()) { 7784 locations->AddTemp(Location::RequiresRegister()); // We need a temp for the table base. 7785 if (switch_instr->GetStartValue() != 0) { 7786 locations->AddTemp(Location::RequiresRegister()); // We need a temp for the bias. 7787 } 7788 } 7789} 7790 7791void InstructionCodeGeneratorARM::VisitPackedSwitch(HPackedSwitch* switch_instr) { 7792 int32_t lower_bound = switch_instr->GetStartValue(); 7793 uint32_t num_entries = switch_instr->GetNumEntries(); 7794 LocationSummary* locations = switch_instr->GetLocations(); 7795 Register value_reg = locations->InAt(0).AsRegister<Register>(); 7796 HBasicBlock* default_block = switch_instr->GetDefaultBlock(); 7797 7798 if (num_entries <= kPackedSwitchCompareJumpThreshold || !codegen_->GetAssembler()->IsThumb()) { 7799 // Create a series of compare/jumps. 7800 Register temp_reg = IP; 7801 // Note: It is fine for the below AddConstantSetFlags() using IP register to temporarily store 7802 // the immediate, because IP is used as the destination register. For the other 7803 // AddConstantSetFlags() and GenerateCompareWithImmediate(), the immediate values are constant, 7804 // and they can be encoded in the instruction without making use of IP register. 7805 __ AddConstantSetFlags(temp_reg, value_reg, -lower_bound); 7806 7807 const ArenaVector<HBasicBlock*>& successors = switch_instr->GetBlock()->GetSuccessors(); 7808 // Jump to successors[0] if value == lower_bound. 7809 __ b(codegen_->GetLabelOf(successors[0]), EQ); 7810 int32_t last_index = 0; 7811 for (; num_entries - last_index > 2; last_index += 2) { 7812 __ AddConstantSetFlags(temp_reg, temp_reg, -2); 7813 // Jump to successors[last_index + 1] if value < case_value[last_index + 2]. 7814 __ b(codegen_->GetLabelOf(successors[last_index + 1]), LO); 7815 // Jump to successors[last_index + 2] if value == case_value[last_index + 2]. 7816 __ b(codegen_->GetLabelOf(successors[last_index + 2]), EQ); 7817 } 7818 if (num_entries - last_index == 2) { 7819 // The last missing case_value. 7820 __ CmpConstant(temp_reg, 1); 7821 __ b(codegen_->GetLabelOf(successors[last_index + 1]), EQ); 7822 } 7823 7824 // And the default for any other value. 7825 if (!codegen_->GoesToNextBlock(switch_instr->GetBlock(), default_block)) { 7826 __ b(codegen_->GetLabelOf(default_block)); 7827 } 7828 } else { 7829 // Create a table lookup. 7830 Register temp_reg = locations->GetTemp(0).AsRegister<Register>(); 7831 7832 // Materialize a pointer to the switch table 7833 std::vector<Label*> labels(num_entries); 7834 const ArenaVector<HBasicBlock*>& successors = switch_instr->GetBlock()->GetSuccessors(); 7835 for (uint32_t i = 0; i < num_entries; i++) { 7836 labels[i] = codegen_->GetLabelOf(successors[i]); 7837 } 7838 JumpTable* table = __ CreateJumpTable(std::move(labels), temp_reg); 7839 7840 // Remove the bias. 7841 Register key_reg; 7842 if (lower_bound != 0) { 7843 key_reg = locations->GetTemp(1).AsRegister<Register>(); 7844 __ AddConstant(key_reg, value_reg, -lower_bound); 7845 } else { 7846 key_reg = value_reg; 7847 } 7848 7849 // Check whether the value is in the table, jump to default block if not. 7850 __ CmpConstant(key_reg, num_entries - 1); 7851 __ b(codegen_->GetLabelOf(default_block), Condition::HI); 7852 7853 // Load the displacement from the table. 7854 __ ldr(temp_reg, Address(temp_reg, key_reg, Shift::LSL, 2)); 7855 7856 // Dispatch is a direct add to the PC (for Thumb2). 7857 __ EmitJumpTableDispatch(table, temp_reg); 7858 } 7859} 7860 7861void LocationsBuilderARM::VisitArmDexCacheArraysBase(HArmDexCacheArraysBase* base) { 7862 LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(base); 7863 locations->SetOut(Location::RequiresRegister()); 7864} 7865 7866void InstructionCodeGeneratorARM::VisitArmDexCacheArraysBase(HArmDexCacheArraysBase* base) { 7867 Register base_reg = base->GetLocations()->Out().AsRegister<Register>(); 7868 CodeGeneratorARM::PcRelativePatchInfo* labels = 7869 codegen_->NewPcRelativeDexCacheArrayPatch(base->GetDexFile(), base->GetElementOffset()); 7870 __ BindTrackedLabel(&labels->movw_label); 7871 __ movw(base_reg, /* placeholder */ 0u); 7872 __ BindTrackedLabel(&labels->movt_label); 7873 __ movt(base_reg, /* placeholder */ 0u); 7874 __ BindTrackedLabel(&labels->add_pc_label); 7875 __ add(base_reg, base_reg, ShifterOperand(PC)); 7876} 7877 7878void CodeGeneratorARM::MoveFromReturnRegister(Location trg, Primitive::Type type) { 7879 if (!trg.IsValid()) { 7880 DCHECK_EQ(type, Primitive::kPrimVoid); 7881 return; 7882 } 7883 7884 DCHECK_NE(type, Primitive::kPrimVoid); 7885 7886 Location return_loc = InvokeDexCallingConventionVisitorARM().GetReturnLocation(type); 7887 if (return_loc.Equals(trg)) { 7888 return; 7889 } 7890 7891 // TODO: Consider pairs in the parallel move resolver, then this could be nicely merged 7892 // with the last branch. 7893 if (type == Primitive::kPrimLong) { 7894 HParallelMove parallel_move(GetGraph()->GetArena()); 7895 parallel_move.AddMove(return_loc.ToLow(), trg.ToLow(), Primitive::kPrimInt, nullptr); 7896 parallel_move.AddMove(return_loc.ToHigh(), trg.ToHigh(), Primitive::kPrimInt, nullptr); 7897 GetMoveResolver()->EmitNativeCode(¶llel_move); 7898 } else if (type == Primitive::kPrimDouble) { 7899 HParallelMove parallel_move(GetGraph()->GetArena()); 7900 parallel_move.AddMove(return_loc.ToLow(), trg.ToLow(), Primitive::kPrimFloat, nullptr); 7901 parallel_move.AddMove(return_loc.ToHigh(), trg.ToHigh(), Primitive::kPrimFloat, nullptr); 7902 GetMoveResolver()->EmitNativeCode(¶llel_move); 7903 } else { 7904 // Let the parallel move resolver take care of all of this. 7905 HParallelMove parallel_move(GetGraph()->GetArena()); 7906 parallel_move.AddMove(return_loc, trg, type, nullptr); 7907 GetMoveResolver()->EmitNativeCode(¶llel_move); 7908 } 7909} 7910 7911void LocationsBuilderARM::VisitClassTableGet(HClassTableGet* instruction) { 7912 LocationSummary* locations = 7913 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall); 7914 locations->SetInAt(0, Location::RequiresRegister()); 7915 locations->SetOut(Location::RequiresRegister()); 7916} 7917 7918void InstructionCodeGeneratorARM::VisitClassTableGet(HClassTableGet* instruction) { 7919 LocationSummary* locations = instruction->GetLocations(); 7920 if (instruction->GetTableKind() == HClassTableGet::TableKind::kVTable) { 7921 uint32_t method_offset = mirror::Class::EmbeddedVTableEntryOffset( 7922 instruction->GetIndex(), kArmPointerSize).SizeValue(); 7923 __ LoadFromOffset(kLoadWord, 7924 locations->Out().AsRegister<Register>(), 7925 locations->InAt(0).AsRegister<Register>(), 7926 method_offset); 7927 } else { 7928 uint32_t method_offset = static_cast<uint32_t>(ImTable::OffsetOfElement( 7929 instruction->GetIndex(), kArmPointerSize)); 7930 __ LoadFromOffset(kLoadWord, 7931 locations->Out().AsRegister<Register>(), 7932 locations->InAt(0).AsRegister<Register>(), 7933 mirror::Class::ImtPtrOffset(kArmPointerSize).Uint32Value()); 7934 __ LoadFromOffset(kLoadWord, 7935 locations->Out().AsRegister<Register>(), 7936 locations->Out().AsRegister<Register>(), 7937 method_offset); 7938 } 7939} 7940 7941static void PatchJitRootUse(uint8_t* code, 7942 const uint8_t* roots_data, 7943 Literal* literal, 7944 uint64_t index_in_table) { 7945 DCHECK(literal->GetLabel()->IsBound()); 7946 uint32_t literal_offset = literal->GetLabel()->Position(); 7947 uintptr_t address = 7948 reinterpret_cast<uintptr_t>(roots_data) + index_in_table * sizeof(GcRoot<mirror::Object>); 7949 uint8_t* data = code + literal_offset; 7950 reinterpret_cast<uint32_t*>(data)[0] = dchecked_integral_cast<uint32_t>(address); 7951} 7952 7953void CodeGeneratorARM::EmitJitRootPatches(uint8_t* code, const uint8_t* roots_data) { 7954 for (const auto& entry : jit_string_patches_) { 7955 const auto& it = jit_string_roots_.find(entry.first); 7956 DCHECK(it != jit_string_roots_.end()); 7957 PatchJitRootUse(code, roots_data, entry.second, it->second); 7958 } 7959 for (const auto& entry : jit_class_patches_) { 7960 const auto& it = jit_class_roots_.find(entry.first); 7961 DCHECK(it != jit_class_roots_.end()); 7962 PatchJitRootUse(code, roots_data, entry.second, it->second); 7963 } 7964} 7965 7966#undef __ 7967#undef QUICK_ENTRY_POINT 7968 7969} // namespace arm 7970} // namespace art 7971