code_generator_arm.cc revision 1372c9f40df1e47bf775f1466bbb96f472b6b9ed
1/* 2 * Copyright (C) 2014 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17#include "code_generator_arm.h" 18 19#include "arch/arm/instruction_set_features_arm.h" 20#include "art_method.h" 21#include "code_generator_utils.h" 22#include "common_arm.h" 23#include "compiled_method.h" 24#include "entrypoints/quick/quick_entrypoints.h" 25#include "gc/accounting/card_table.h" 26#include "intrinsics.h" 27#include "intrinsics_arm.h" 28#include "mirror/array-inl.h" 29#include "mirror/class-inl.h" 30#include "thread.h" 31#include "utils/arm/assembler_arm.h" 32#include "utils/arm/managed_register_arm.h" 33#include "utils/assembler.h" 34#include "utils/stack_checks.h" 35 36namespace art { 37 38template<class MirrorType> 39class GcRoot; 40 41namespace arm { 42 43static bool ExpectedPairLayout(Location location) { 44 // We expected this for both core and fpu register pairs. 45 return ((location.low() & 1) == 0) && (location.low() + 1 == location.high()); 46} 47 48static constexpr int kCurrentMethodStackOffset = 0; 49static constexpr Register kMethodRegisterArgument = R0; 50 51static constexpr Register kCoreAlwaysSpillRegister = R5; 52static constexpr Register kCoreCalleeSaves[] = 53 { R5, R6, R7, R8, R10, R11, LR }; 54static constexpr SRegister kFpuCalleeSaves[] = 55 { S16, S17, S18, S19, S20, S21, S22, S23, S24, S25, S26, S27, S28, S29, S30, S31 }; 56 57// D31 cannot be split into two S registers, and the register allocator only works on 58// S registers. Therefore there is no need to block it. 59static constexpr DRegister DTMP = D31; 60 61static constexpr uint32_t kPackedSwitchCompareJumpThreshold = 7; 62 63// NOLINT on __ macro to suppress wrong warning/fix (misc-macro-parentheses) from clang-tidy. 64#define __ down_cast<ArmAssembler*>(codegen->GetAssembler())-> // NOLINT 65#define QUICK_ENTRY_POINT(x) QUICK_ENTRYPOINT_OFFSET(kArmPointerSize, x).Int32Value() 66 67static constexpr int kRegListThreshold = 4; 68 69// SaveLiveRegisters and RestoreLiveRegisters from SlowPathCodeARM operate on sets of S registers, 70// for each live D registers they treat two corresponding S registers as live ones. 71// 72// Two following functions (SaveContiguousSRegisterList, RestoreContiguousSRegisterList) build 73// from a list of contiguous S registers a list of contiguous D registers (processing first/last 74// S registers corner cases) and save/restore this new list treating them as D registers. 75// - decreasing code size 76// - avoiding hazards on Cortex-A57, when a pair of S registers for an actual live D register is 77// restored and then used in regular non SlowPath code as D register. 78// 79// For the following example (v means the S register is live): 80// D names: | D0 | D1 | D2 | D4 | ... 81// S names: | S0 | S1 | S2 | S3 | S4 | S5 | S6 | S7 | ... 82// Live? | | v | v | v | v | v | v | | ... 83// 84// S1 and S6 will be saved/restored independently; D registers list (D1, D2) will be processed 85// as D registers. 86static size_t SaveContiguousSRegisterList(size_t first, 87 size_t last, 88 CodeGenerator* codegen, 89 size_t stack_offset) { 90 DCHECK_LE(first, last); 91 if ((first == last) && (first == 0)) { 92 stack_offset += codegen->SaveFloatingPointRegister(stack_offset, first); 93 return stack_offset; 94 } 95 if (first % 2 == 1) { 96 stack_offset += codegen->SaveFloatingPointRegister(stack_offset, first++); 97 } 98 99 bool save_last = false; 100 if (last % 2 == 0) { 101 save_last = true; 102 --last; 103 } 104 105 if (first < last) { 106 DRegister d_reg = static_cast<DRegister>(first / 2); 107 DCHECK_EQ((last - first + 1) % 2, 0u); 108 size_t number_of_d_regs = (last - first + 1) / 2; 109 110 if (number_of_d_regs == 1) { 111 __ StoreDToOffset(d_reg, SP, stack_offset); 112 } else if (number_of_d_regs > 1) { 113 __ add(IP, SP, ShifterOperand(stack_offset)); 114 __ vstmiad(IP, d_reg, number_of_d_regs); 115 } 116 stack_offset += number_of_d_regs * kArmWordSize * 2; 117 } 118 119 if (save_last) { 120 stack_offset += codegen->SaveFloatingPointRegister(stack_offset, last + 1); 121 } 122 123 return stack_offset; 124} 125 126static size_t RestoreContiguousSRegisterList(size_t first, 127 size_t last, 128 CodeGenerator* codegen, 129 size_t stack_offset) { 130 DCHECK_LE(first, last); 131 if ((first == last) && (first == 0)) { 132 stack_offset += codegen->RestoreFloatingPointRegister(stack_offset, first); 133 return stack_offset; 134 } 135 if (first % 2 == 1) { 136 stack_offset += codegen->RestoreFloatingPointRegister(stack_offset, first++); 137 } 138 139 bool restore_last = false; 140 if (last % 2 == 0) { 141 restore_last = true; 142 --last; 143 } 144 145 if (first < last) { 146 DRegister d_reg = static_cast<DRegister>(first / 2); 147 DCHECK_EQ((last - first + 1) % 2, 0u); 148 size_t number_of_d_regs = (last - first + 1) / 2; 149 if (number_of_d_regs == 1) { 150 __ LoadDFromOffset(d_reg, SP, stack_offset); 151 } else if (number_of_d_regs > 1) { 152 __ add(IP, SP, ShifterOperand(stack_offset)); 153 __ vldmiad(IP, d_reg, number_of_d_regs); 154 } 155 stack_offset += number_of_d_regs * kArmWordSize * 2; 156 } 157 158 if (restore_last) { 159 stack_offset += codegen->RestoreFloatingPointRegister(stack_offset, last + 1); 160 } 161 162 return stack_offset; 163} 164 165void SlowPathCodeARM::SaveLiveRegisters(CodeGenerator* codegen, LocationSummary* locations) { 166 size_t stack_offset = codegen->GetFirstRegisterSlotInSlowPath(); 167 size_t orig_offset = stack_offset; 168 169 const uint32_t core_spills = codegen->GetSlowPathSpills(locations, /* core_registers */ true); 170 for (uint32_t i : LowToHighBits(core_spills)) { 171 // If the register holds an object, update the stack mask. 172 if (locations->RegisterContainsObject(i)) { 173 locations->SetStackBit(stack_offset / kVRegSize); 174 } 175 DCHECK_LT(stack_offset, codegen->GetFrameSize() - codegen->FrameEntrySpillSize()); 176 DCHECK_LT(i, kMaximumNumberOfExpectedRegisters); 177 saved_core_stack_offsets_[i] = stack_offset; 178 stack_offset += kArmWordSize; 179 } 180 181 int reg_num = POPCOUNT(core_spills); 182 if (reg_num != 0) { 183 if (reg_num > kRegListThreshold) { 184 __ StoreList(RegList(core_spills), orig_offset); 185 } else { 186 stack_offset = orig_offset; 187 for (uint32_t i : LowToHighBits(core_spills)) { 188 stack_offset += codegen->SaveCoreRegister(stack_offset, i); 189 } 190 } 191 } 192 193 uint32_t fp_spills = codegen->GetSlowPathSpills(locations, /* core_registers */ false); 194 orig_offset = stack_offset; 195 for (uint32_t i : LowToHighBits(fp_spills)) { 196 DCHECK_LT(i, kMaximumNumberOfExpectedRegisters); 197 saved_fpu_stack_offsets_[i] = stack_offset; 198 stack_offset += kArmWordSize; 199 } 200 201 stack_offset = orig_offset; 202 while (fp_spills != 0u) { 203 uint32_t begin = CTZ(fp_spills); 204 uint32_t tmp = fp_spills + (1u << begin); 205 fp_spills &= tmp; // Clear the contiguous range of 1s. 206 uint32_t end = (tmp == 0u) ? 32u : CTZ(tmp); // CTZ(0) is undefined. 207 stack_offset = SaveContiguousSRegisterList(begin, end - 1, codegen, stack_offset); 208 } 209 DCHECK_LE(stack_offset, codegen->GetFrameSize() - codegen->FrameEntrySpillSize()); 210} 211 212void SlowPathCodeARM::RestoreLiveRegisters(CodeGenerator* codegen, LocationSummary* locations) { 213 size_t stack_offset = codegen->GetFirstRegisterSlotInSlowPath(); 214 size_t orig_offset = stack_offset; 215 216 const uint32_t core_spills = codegen->GetSlowPathSpills(locations, /* core_registers */ true); 217 for (uint32_t i : LowToHighBits(core_spills)) { 218 DCHECK_LT(stack_offset, codegen->GetFrameSize() - codegen->FrameEntrySpillSize()); 219 DCHECK_LT(i, kMaximumNumberOfExpectedRegisters); 220 stack_offset += kArmWordSize; 221 } 222 223 int reg_num = POPCOUNT(core_spills); 224 if (reg_num != 0) { 225 if (reg_num > kRegListThreshold) { 226 __ LoadList(RegList(core_spills), orig_offset); 227 } else { 228 stack_offset = orig_offset; 229 for (uint32_t i : LowToHighBits(core_spills)) { 230 stack_offset += codegen->RestoreCoreRegister(stack_offset, i); 231 } 232 } 233 } 234 235 uint32_t fp_spills = codegen->GetSlowPathSpills(locations, /* core_registers */ false); 236 while (fp_spills != 0u) { 237 uint32_t begin = CTZ(fp_spills); 238 uint32_t tmp = fp_spills + (1u << begin); 239 fp_spills &= tmp; // Clear the contiguous range of 1s. 240 uint32_t end = (tmp == 0u) ? 32u : CTZ(tmp); // CTZ(0) is undefined. 241 stack_offset = RestoreContiguousSRegisterList(begin, end - 1, codegen, stack_offset); 242 } 243 DCHECK_LE(stack_offset, codegen->GetFrameSize() - codegen->FrameEntrySpillSize()); 244} 245 246class NullCheckSlowPathARM : public SlowPathCodeARM { 247 public: 248 explicit NullCheckSlowPathARM(HNullCheck* instruction) : SlowPathCodeARM(instruction) {} 249 250 void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { 251 CodeGeneratorARM* arm_codegen = down_cast<CodeGeneratorARM*>(codegen); 252 __ Bind(GetEntryLabel()); 253 if (instruction_->CanThrowIntoCatchBlock()) { 254 // Live registers will be restored in the catch block if caught. 255 SaveLiveRegisters(codegen, instruction_->GetLocations()); 256 } 257 arm_codegen->InvokeRuntime(kQuickThrowNullPointer, 258 instruction_, 259 instruction_->GetDexPc(), 260 this); 261 CheckEntrypointTypes<kQuickThrowNullPointer, void, void>(); 262 } 263 264 bool IsFatal() const OVERRIDE { return true; } 265 266 const char* GetDescription() const OVERRIDE { return "NullCheckSlowPathARM"; } 267 268 private: 269 DISALLOW_COPY_AND_ASSIGN(NullCheckSlowPathARM); 270}; 271 272class DivZeroCheckSlowPathARM : public SlowPathCodeARM { 273 public: 274 explicit DivZeroCheckSlowPathARM(HDivZeroCheck* instruction) : SlowPathCodeARM(instruction) {} 275 276 void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { 277 CodeGeneratorARM* arm_codegen = down_cast<CodeGeneratorARM*>(codegen); 278 __ Bind(GetEntryLabel()); 279 arm_codegen->InvokeRuntime(kQuickThrowDivZero, instruction_, instruction_->GetDexPc(), this); 280 CheckEntrypointTypes<kQuickThrowDivZero, void, void>(); 281 } 282 283 bool IsFatal() const OVERRIDE { return true; } 284 285 const char* GetDescription() const OVERRIDE { return "DivZeroCheckSlowPathARM"; } 286 287 private: 288 DISALLOW_COPY_AND_ASSIGN(DivZeroCheckSlowPathARM); 289}; 290 291class SuspendCheckSlowPathARM : public SlowPathCodeARM { 292 public: 293 SuspendCheckSlowPathARM(HSuspendCheck* instruction, HBasicBlock* successor) 294 : SlowPathCodeARM(instruction), successor_(successor) {} 295 296 void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { 297 CodeGeneratorARM* arm_codegen = down_cast<CodeGeneratorARM*>(codegen); 298 __ Bind(GetEntryLabel()); 299 arm_codegen->InvokeRuntime(kQuickTestSuspend, instruction_, instruction_->GetDexPc(), this); 300 CheckEntrypointTypes<kQuickTestSuspend, void, void>(); 301 if (successor_ == nullptr) { 302 __ b(GetReturnLabel()); 303 } else { 304 __ b(arm_codegen->GetLabelOf(successor_)); 305 } 306 } 307 308 Label* GetReturnLabel() { 309 DCHECK(successor_ == nullptr); 310 return &return_label_; 311 } 312 313 HBasicBlock* GetSuccessor() const { 314 return successor_; 315 } 316 317 const char* GetDescription() const OVERRIDE { return "SuspendCheckSlowPathARM"; } 318 319 private: 320 // If not null, the block to branch to after the suspend check. 321 HBasicBlock* const successor_; 322 323 // If `successor_` is null, the label to branch to after the suspend check. 324 Label return_label_; 325 326 DISALLOW_COPY_AND_ASSIGN(SuspendCheckSlowPathARM); 327}; 328 329class BoundsCheckSlowPathARM : public SlowPathCodeARM { 330 public: 331 explicit BoundsCheckSlowPathARM(HBoundsCheck* instruction) 332 : SlowPathCodeARM(instruction) {} 333 334 void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { 335 CodeGeneratorARM* arm_codegen = down_cast<CodeGeneratorARM*>(codegen); 336 LocationSummary* locations = instruction_->GetLocations(); 337 338 __ Bind(GetEntryLabel()); 339 if (instruction_->CanThrowIntoCatchBlock()) { 340 // Live registers will be restored in the catch block if caught. 341 SaveLiveRegisters(codegen, instruction_->GetLocations()); 342 } 343 // We're moving two locations to locations that could overlap, so we need a parallel 344 // move resolver. 345 InvokeRuntimeCallingConvention calling_convention; 346 codegen->EmitParallelMoves( 347 locations->InAt(0), 348 Location::RegisterLocation(calling_convention.GetRegisterAt(0)), 349 Primitive::kPrimInt, 350 locations->InAt(1), 351 Location::RegisterLocation(calling_convention.GetRegisterAt(1)), 352 Primitive::kPrimInt); 353 QuickEntrypointEnum entrypoint = instruction_->AsBoundsCheck()->IsStringCharAt() 354 ? kQuickThrowStringBounds 355 : kQuickThrowArrayBounds; 356 arm_codegen->InvokeRuntime(entrypoint, instruction_, instruction_->GetDexPc(), this); 357 CheckEntrypointTypes<kQuickThrowStringBounds, void, int32_t, int32_t>(); 358 CheckEntrypointTypes<kQuickThrowArrayBounds, void, int32_t, int32_t>(); 359 } 360 361 bool IsFatal() const OVERRIDE { return true; } 362 363 const char* GetDescription() const OVERRIDE { return "BoundsCheckSlowPathARM"; } 364 365 private: 366 DISALLOW_COPY_AND_ASSIGN(BoundsCheckSlowPathARM); 367}; 368 369class LoadClassSlowPathARM : public SlowPathCodeARM { 370 public: 371 LoadClassSlowPathARM(HLoadClass* cls, HInstruction* at, uint32_t dex_pc, bool do_clinit) 372 : SlowPathCodeARM(at), cls_(cls), dex_pc_(dex_pc), do_clinit_(do_clinit) { 373 DCHECK(at->IsLoadClass() || at->IsClinitCheck()); 374 } 375 376 void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { 377 LocationSummary* locations = instruction_->GetLocations(); 378 Location out = locations->Out(); 379 constexpr bool call_saves_everything_except_r0 = (!kUseReadBarrier || kUseBakerReadBarrier); 380 381 CodeGeneratorARM* arm_codegen = down_cast<CodeGeneratorARM*>(codegen); 382 __ Bind(GetEntryLabel()); 383 SaveLiveRegisters(codegen, locations); 384 385 InvokeRuntimeCallingConvention calling_convention; 386 // For HLoadClass/kBssEntry/kSaveEverything, make sure we preserve the address of the entry. 387 DCHECK_EQ(instruction_->IsLoadClass(), cls_ == instruction_); 388 bool is_load_class_bss_entry = 389 (cls_ == instruction_) && (cls_->GetLoadKind() == HLoadClass::LoadKind::kBssEntry); 390 Register entry_address = kNoRegister; 391 if (is_load_class_bss_entry && call_saves_everything_except_r0) { 392 Register temp = locations->GetTemp(0).AsRegister<Register>(); 393 // In the unlucky case that the `temp` is R0, we preserve the address in `out` across 394 // the kSaveEverything call. 395 bool temp_is_r0 = (temp == calling_convention.GetRegisterAt(0)); 396 entry_address = temp_is_r0 ? out.AsRegister<Register>() : temp; 397 DCHECK_NE(entry_address, calling_convention.GetRegisterAt(0)); 398 if (temp_is_r0) { 399 __ mov(entry_address, ShifterOperand(temp)); 400 } 401 } 402 dex::TypeIndex type_index = cls_->GetTypeIndex(); 403 __ LoadImmediate(calling_convention.GetRegisterAt(0), type_index.index_); 404 QuickEntrypointEnum entrypoint = do_clinit_ ? kQuickInitializeStaticStorage 405 : kQuickInitializeType; 406 arm_codegen->InvokeRuntime(entrypoint, instruction_, dex_pc_, this); 407 if (do_clinit_) { 408 CheckEntrypointTypes<kQuickInitializeStaticStorage, void*, uint32_t>(); 409 } else { 410 CheckEntrypointTypes<kQuickInitializeType, void*, uint32_t>(); 411 } 412 413 // For HLoadClass/kBssEntry, store the resolved Class to the BSS entry. 414 if (is_load_class_bss_entry) { 415 if (call_saves_everything_except_r0) { 416 // The class entry address was preserved in `entry_address` thanks to kSaveEverything. 417 __ str(R0, Address(entry_address)); 418 } else { 419 // For non-Baker read barrier, we need to re-calculate the address of the string entry. 420 Register temp = IP; 421 CodeGeneratorARM::PcRelativePatchInfo* labels = 422 arm_codegen->NewTypeBssEntryPatch(cls_->GetDexFile(), type_index); 423 __ BindTrackedLabel(&labels->movw_label); 424 __ movw(temp, /* placeholder */ 0u); 425 __ BindTrackedLabel(&labels->movt_label); 426 __ movt(temp, /* placeholder */ 0u); 427 __ BindTrackedLabel(&labels->add_pc_label); 428 __ add(temp, temp, ShifterOperand(PC)); 429 __ str(R0, Address(temp)); 430 } 431 } 432 // Move the class to the desired location. 433 if (out.IsValid()) { 434 DCHECK(out.IsRegister() && !locations->GetLiveRegisters()->ContainsCoreRegister(out.reg())); 435 arm_codegen->Move32(locations->Out(), Location::RegisterLocation(R0)); 436 } 437 RestoreLiveRegisters(codegen, locations); 438 __ b(GetExitLabel()); 439 } 440 441 const char* GetDescription() const OVERRIDE { return "LoadClassSlowPathARM"; } 442 443 private: 444 // The class this slow path will load. 445 HLoadClass* const cls_; 446 447 // The dex PC of `at_`. 448 const uint32_t dex_pc_; 449 450 // Whether to initialize the class. 451 const bool do_clinit_; 452 453 DISALLOW_COPY_AND_ASSIGN(LoadClassSlowPathARM); 454}; 455 456class LoadStringSlowPathARM : public SlowPathCodeARM { 457 public: 458 explicit LoadStringSlowPathARM(HLoadString* instruction) : SlowPathCodeARM(instruction) {} 459 460 void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { 461 DCHECK(instruction_->IsLoadString()); 462 DCHECK_EQ(instruction_->AsLoadString()->GetLoadKind(), HLoadString::LoadKind::kBssEntry); 463 LocationSummary* locations = instruction_->GetLocations(); 464 DCHECK(!locations->GetLiveRegisters()->ContainsCoreRegister(locations->Out().reg())); 465 HLoadString* load = instruction_->AsLoadString(); 466 const dex::StringIndex string_index = load->GetStringIndex(); 467 Register out = locations->Out().AsRegister<Register>(); 468 constexpr bool call_saves_everything_except_r0 = (!kUseReadBarrier || kUseBakerReadBarrier); 469 470 CodeGeneratorARM* arm_codegen = down_cast<CodeGeneratorARM*>(codegen); 471 __ Bind(GetEntryLabel()); 472 SaveLiveRegisters(codegen, locations); 473 474 InvokeRuntimeCallingConvention calling_convention; 475 // In the unlucky case that the `temp` is R0, we preserve the address in `out` across 476 // the kSaveEverything call. 477 Register entry_address = kNoRegister; 478 if (call_saves_everything_except_r0) { 479 Register temp = locations->GetTemp(0).AsRegister<Register>(); 480 bool temp_is_r0 = (temp == calling_convention.GetRegisterAt(0)); 481 entry_address = temp_is_r0 ? out : temp; 482 DCHECK_NE(entry_address, calling_convention.GetRegisterAt(0)); 483 if (temp_is_r0) { 484 __ mov(entry_address, ShifterOperand(temp)); 485 } 486 } 487 488 __ LoadImmediate(calling_convention.GetRegisterAt(0), string_index.index_); 489 arm_codegen->InvokeRuntime(kQuickResolveString, instruction_, instruction_->GetDexPc(), this); 490 CheckEntrypointTypes<kQuickResolveString, void*, uint32_t>(); 491 492 // Store the resolved String to the .bss entry. 493 if (call_saves_everything_except_r0) { 494 // The string entry address was preserved in `entry_address` thanks to kSaveEverything. 495 __ str(R0, Address(entry_address)); 496 } else { 497 // For non-Baker read barrier, we need to re-calculate the address of the string entry. 498 Register temp = IP; 499 CodeGeneratorARM::PcRelativePatchInfo* labels = 500 arm_codegen->NewPcRelativeStringPatch(load->GetDexFile(), string_index); 501 __ BindTrackedLabel(&labels->movw_label); 502 __ movw(temp, /* placeholder */ 0u); 503 __ BindTrackedLabel(&labels->movt_label); 504 __ movt(temp, /* placeholder */ 0u); 505 __ BindTrackedLabel(&labels->add_pc_label); 506 __ add(temp, temp, ShifterOperand(PC)); 507 __ str(R0, Address(temp)); 508 } 509 510 arm_codegen->Move32(locations->Out(), Location::RegisterLocation(R0)); 511 RestoreLiveRegisters(codegen, locations); 512 513 __ b(GetExitLabel()); 514 } 515 516 const char* GetDescription() const OVERRIDE { return "LoadStringSlowPathARM"; } 517 518 private: 519 DISALLOW_COPY_AND_ASSIGN(LoadStringSlowPathARM); 520}; 521 522class TypeCheckSlowPathARM : public SlowPathCodeARM { 523 public: 524 TypeCheckSlowPathARM(HInstruction* instruction, bool is_fatal) 525 : SlowPathCodeARM(instruction), is_fatal_(is_fatal) {} 526 527 void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { 528 LocationSummary* locations = instruction_->GetLocations(); 529 DCHECK(instruction_->IsCheckCast() 530 || !locations->GetLiveRegisters()->ContainsCoreRegister(locations->Out().reg())); 531 532 CodeGeneratorARM* arm_codegen = down_cast<CodeGeneratorARM*>(codegen); 533 __ Bind(GetEntryLabel()); 534 535 if (!is_fatal_) { 536 SaveLiveRegisters(codegen, locations); 537 } 538 539 // We're moving two locations to locations that could overlap, so we need a parallel 540 // move resolver. 541 InvokeRuntimeCallingConvention calling_convention; 542 codegen->EmitParallelMoves(locations->InAt(0), 543 Location::RegisterLocation(calling_convention.GetRegisterAt(0)), 544 Primitive::kPrimNot, 545 locations->InAt(1), 546 Location::RegisterLocation(calling_convention.GetRegisterAt(1)), 547 Primitive::kPrimNot); 548 if (instruction_->IsInstanceOf()) { 549 arm_codegen->InvokeRuntime(kQuickInstanceofNonTrivial, 550 instruction_, 551 instruction_->GetDexPc(), 552 this); 553 CheckEntrypointTypes<kQuickInstanceofNonTrivial, size_t, mirror::Object*, mirror::Class*>(); 554 arm_codegen->Move32(locations->Out(), Location::RegisterLocation(R0)); 555 } else { 556 DCHECK(instruction_->IsCheckCast()); 557 arm_codegen->InvokeRuntime(kQuickCheckInstanceOf, 558 instruction_, 559 instruction_->GetDexPc(), 560 this); 561 CheckEntrypointTypes<kQuickCheckInstanceOf, void, mirror::Object*, mirror::Class*>(); 562 } 563 564 if (!is_fatal_) { 565 RestoreLiveRegisters(codegen, locations); 566 __ b(GetExitLabel()); 567 } 568 } 569 570 const char* GetDescription() const OVERRIDE { return "TypeCheckSlowPathARM"; } 571 572 bool IsFatal() const OVERRIDE { return is_fatal_; } 573 574 private: 575 const bool is_fatal_; 576 577 DISALLOW_COPY_AND_ASSIGN(TypeCheckSlowPathARM); 578}; 579 580class DeoptimizationSlowPathARM : public SlowPathCodeARM { 581 public: 582 explicit DeoptimizationSlowPathARM(HDeoptimize* instruction) 583 : SlowPathCodeARM(instruction) {} 584 585 void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { 586 CodeGeneratorARM* arm_codegen = down_cast<CodeGeneratorARM*>(codegen); 587 __ Bind(GetEntryLabel()); 588 arm_codegen->InvokeRuntime(kQuickDeoptimize, instruction_, instruction_->GetDexPc(), this); 589 CheckEntrypointTypes<kQuickDeoptimize, void, void>(); 590 } 591 592 const char* GetDescription() const OVERRIDE { return "DeoptimizationSlowPathARM"; } 593 594 private: 595 DISALLOW_COPY_AND_ASSIGN(DeoptimizationSlowPathARM); 596}; 597 598class ArraySetSlowPathARM : public SlowPathCodeARM { 599 public: 600 explicit ArraySetSlowPathARM(HInstruction* instruction) : SlowPathCodeARM(instruction) {} 601 602 void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { 603 LocationSummary* locations = instruction_->GetLocations(); 604 __ Bind(GetEntryLabel()); 605 SaveLiveRegisters(codegen, locations); 606 607 InvokeRuntimeCallingConvention calling_convention; 608 HParallelMove parallel_move(codegen->GetGraph()->GetArena()); 609 parallel_move.AddMove( 610 locations->InAt(0), 611 Location::RegisterLocation(calling_convention.GetRegisterAt(0)), 612 Primitive::kPrimNot, 613 nullptr); 614 parallel_move.AddMove( 615 locations->InAt(1), 616 Location::RegisterLocation(calling_convention.GetRegisterAt(1)), 617 Primitive::kPrimInt, 618 nullptr); 619 parallel_move.AddMove( 620 locations->InAt(2), 621 Location::RegisterLocation(calling_convention.GetRegisterAt(2)), 622 Primitive::kPrimNot, 623 nullptr); 624 codegen->GetMoveResolver()->EmitNativeCode(¶llel_move); 625 626 CodeGeneratorARM* arm_codegen = down_cast<CodeGeneratorARM*>(codegen); 627 arm_codegen->InvokeRuntime(kQuickAputObject, instruction_, instruction_->GetDexPc(), this); 628 CheckEntrypointTypes<kQuickAputObject, void, mirror::Array*, int32_t, mirror::Object*>(); 629 RestoreLiveRegisters(codegen, locations); 630 __ b(GetExitLabel()); 631 } 632 633 const char* GetDescription() const OVERRIDE { return "ArraySetSlowPathARM"; } 634 635 private: 636 DISALLOW_COPY_AND_ASSIGN(ArraySetSlowPathARM); 637}; 638 639// Slow path marking an object reference `ref` during a read 640// barrier. The field `obj.field` in the object `obj` holding this 641// reference does not get updated by this slow path after marking (see 642// ReadBarrierMarkAndUpdateFieldSlowPathARM below for that). 643// 644// This means that after the execution of this slow path, `ref` will 645// always be up-to-date, but `obj.field` may not; i.e., after the 646// flip, `ref` will be a to-space reference, but `obj.field` will 647// probably still be a from-space reference (unless it gets updated by 648// another thread, or if another thread installed another object 649// reference (different from `ref`) in `obj.field`). 650// 651// If `entrypoint` is a valid location it is assumed to already be 652// holding the entrypoint. The case where the entrypoint is passed in 653// is when the decision to mark is based on whether the GC is marking. 654class ReadBarrierMarkSlowPathARM : public SlowPathCodeARM { 655 public: 656 ReadBarrierMarkSlowPathARM(HInstruction* instruction, 657 Location ref, 658 Location entrypoint = Location::NoLocation()) 659 : SlowPathCodeARM(instruction), ref_(ref), entrypoint_(entrypoint) { 660 DCHECK(kEmitCompilerReadBarrier); 661 } 662 663 const char* GetDescription() const OVERRIDE { return "ReadBarrierMarkSlowPathARM"; } 664 665 void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { 666 LocationSummary* locations = instruction_->GetLocations(); 667 Register ref_reg = ref_.AsRegister<Register>(); 668 DCHECK(locations->CanCall()); 669 DCHECK(!locations->GetLiveRegisters()->ContainsCoreRegister(ref_reg)) << ref_reg; 670 DCHECK(instruction_->IsInstanceFieldGet() || 671 instruction_->IsStaticFieldGet() || 672 instruction_->IsArrayGet() || 673 instruction_->IsArraySet() || 674 instruction_->IsLoadClass() || 675 instruction_->IsLoadString() || 676 instruction_->IsInstanceOf() || 677 instruction_->IsCheckCast() || 678 (instruction_->IsInvokeVirtual() && instruction_->GetLocations()->Intrinsified()) || 679 (instruction_->IsInvokeStaticOrDirect() && instruction_->GetLocations()->Intrinsified())) 680 << "Unexpected instruction in read barrier marking slow path: " 681 << instruction_->DebugName(); 682 // The read barrier instrumentation of object ArrayGet 683 // instructions does not support the HIntermediateAddress 684 // instruction. 685 DCHECK(!(instruction_->IsArrayGet() && 686 instruction_->AsArrayGet()->GetArray()->IsIntermediateAddress())); 687 688 __ Bind(GetEntryLabel()); 689 // No need to save live registers; it's taken care of by the 690 // entrypoint. Also, there is no need to update the stack mask, 691 // as this runtime call will not trigger a garbage collection. 692 CodeGeneratorARM* arm_codegen = down_cast<CodeGeneratorARM*>(codegen); 693 DCHECK_NE(ref_reg, SP); 694 DCHECK_NE(ref_reg, LR); 695 DCHECK_NE(ref_reg, PC); 696 // IP is used internally by the ReadBarrierMarkRegX entry point 697 // as a temporary, it cannot be the entry point's input/output. 698 DCHECK_NE(ref_reg, IP); 699 DCHECK(0 <= ref_reg && ref_reg < kNumberOfCoreRegisters) << ref_reg; 700 // "Compact" slow path, saving two moves. 701 // 702 // Instead of using the standard runtime calling convention (input 703 // and output in R0): 704 // 705 // R0 <- ref 706 // R0 <- ReadBarrierMark(R0) 707 // ref <- R0 708 // 709 // we just use rX (the register containing `ref`) as input and output 710 // of a dedicated entrypoint: 711 // 712 // rX <- ReadBarrierMarkRegX(rX) 713 // 714 if (entrypoint_.IsValid()) { 715 arm_codegen->ValidateInvokeRuntimeWithoutRecordingPcInfo(instruction_, this); 716 __ blx(entrypoint_.AsRegister<Register>()); 717 } else { 718 // Entrypoint is not already loaded, load from the thread. 719 int32_t entry_point_offset = 720 CodeGenerator::GetReadBarrierMarkEntryPointsOffset<kArmPointerSize>(ref_reg); 721 // This runtime call does not require a stack map. 722 arm_codegen->InvokeRuntimeWithoutRecordingPcInfo(entry_point_offset, instruction_, this); 723 } 724 __ b(GetExitLabel()); 725 } 726 727 private: 728 // The location (register) of the marked object reference. 729 const Location ref_; 730 731 // The location of the entrypoint if already loaded. 732 const Location entrypoint_; 733 734 DISALLOW_COPY_AND_ASSIGN(ReadBarrierMarkSlowPathARM); 735}; 736 737// Slow path marking an object reference `ref` during a read barrier, 738// and if needed, atomically updating the field `obj.field` in the 739// object `obj` holding this reference after marking (contrary to 740// ReadBarrierMarkSlowPathARM above, which never tries to update 741// `obj.field`). 742// 743// This means that after the execution of this slow path, both `ref` 744// and `obj.field` will be up-to-date; i.e., after the flip, both will 745// hold the same to-space reference (unless another thread installed 746// another object reference (different from `ref`) in `obj.field`). 747// 748// If `entrypoint` is a valid location it is assumed to already be 749// holding the entrypoint. The case where the entrypoint is passed in 750// is when the decision to mark is based on whether the GC is marking. 751class ReadBarrierMarkAndUpdateFieldSlowPathARM : public SlowPathCodeARM { 752 public: 753 ReadBarrierMarkAndUpdateFieldSlowPathARM(HInstruction* instruction, 754 Location ref, 755 Register obj, 756 Location field_offset, 757 Register temp1, 758 Register temp2, 759 Location entrypoint = Location::NoLocation()) 760 : SlowPathCodeARM(instruction), 761 ref_(ref), 762 obj_(obj), 763 field_offset_(field_offset), 764 temp1_(temp1), 765 temp2_(temp2), 766 entrypoint_(entrypoint) { 767 DCHECK(kEmitCompilerReadBarrier); 768 } 769 770 const char* GetDescription() const OVERRIDE { return "ReadBarrierMarkAndUpdateFieldSlowPathARM"; } 771 772 void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { 773 LocationSummary* locations = instruction_->GetLocations(); 774 Register ref_reg = ref_.AsRegister<Register>(); 775 DCHECK(locations->CanCall()); 776 DCHECK(!locations->GetLiveRegisters()->ContainsCoreRegister(ref_reg)) << ref_reg; 777 // This slow path is only used by the UnsafeCASObject intrinsic. 778 DCHECK((instruction_->IsInvokeVirtual() && instruction_->GetLocations()->Intrinsified())) 779 << "Unexpected instruction in read barrier marking and field updating slow path: " 780 << instruction_->DebugName(); 781 DCHECK(instruction_->GetLocations()->Intrinsified()); 782 DCHECK_EQ(instruction_->AsInvoke()->GetIntrinsic(), Intrinsics::kUnsafeCASObject); 783 DCHECK(field_offset_.IsRegisterPair()) << field_offset_; 784 785 __ Bind(GetEntryLabel()); 786 787 // Save the old reference. 788 // Note that we cannot use IP to save the old reference, as IP is 789 // used internally by the ReadBarrierMarkRegX entry point, and we 790 // need the old reference after the call to that entry point. 791 DCHECK_NE(temp1_, IP); 792 __ Mov(temp1_, ref_reg); 793 794 // No need to save live registers; it's taken care of by the 795 // entrypoint. Also, there is no need to update the stack mask, 796 // as this runtime call will not trigger a garbage collection. 797 CodeGeneratorARM* arm_codegen = down_cast<CodeGeneratorARM*>(codegen); 798 DCHECK_NE(ref_reg, SP); 799 DCHECK_NE(ref_reg, LR); 800 DCHECK_NE(ref_reg, PC); 801 // IP is used internally by the ReadBarrierMarkRegX entry point 802 // as a temporary, it cannot be the entry point's input/output. 803 DCHECK_NE(ref_reg, IP); 804 DCHECK(0 <= ref_reg && ref_reg < kNumberOfCoreRegisters) << ref_reg; 805 // "Compact" slow path, saving two moves. 806 // 807 // Instead of using the standard runtime calling convention (input 808 // and output in R0): 809 // 810 // R0 <- ref 811 // R0 <- ReadBarrierMark(R0) 812 // ref <- R0 813 // 814 // we just use rX (the register containing `ref`) as input and output 815 // of a dedicated entrypoint: 816 // 817 // rX <- ReadBarrierMarkRegX(rX) 818 // 819 if (entrypoint_.IsValid()) { 820 arm_codegen->ValidateInvokeRuntimeWithoutRecordingPcInfo(instruction_, this); 821 __ blx(entrypoint_.AsRegister<Register>()); 822 } else { 823 // Entrypoint is not already loaded, load from the thread. 824 int32_t entry_point_offset = 825 CodeGenerator::GetReadBarrierMarkEntryPointsOffset<kArmPointerSize>(ref_reg); 826 // This runtime call does not require a stack map. 827 arm_codegen->InvokeRuntimeWithoutRecordingPcInfo(entry_point_offset, instruction_, this); 828 } 829 830 // If the new reference is different from the old reference, 831 // update the field in the holder (`*(obj_ + field_offset_)`). 832 // 833 // Note that this field could also hold a different object, if 834 // another thread had concurrently changed it. In that case, the 835 // LDREX/SUBS/ITNE sequence of instructions in the compare-and-set 836 // (CAS) operation below would abort the CAS, leaving the field 837 // as-is. 838 Label done; 839 __ cmp(temp1_, ShifterOperand(ref_reg)); 840 __ b(&done, EQ); 841 842 // Update the the holder's field atomically. This may fail if 843 // mutator updates before us, but it's OK. This is achieved 844 // using a strong compare-and-set (CAS) operation with relaxed 845 // memory synchronization ordering, where the expected value is 846 // the old reference and the desired value is the new reference. 847 848 // Convenience aliases. 849 Register base = obj_; 850 // The UnsafeCASObject intrinsic uses a register pair as field 851 // offset ("long offset"), of which only the low part contains 852 // data. 853 Register offset = field_offset_.AsRegisterPairLow<Register>(); 854 Register expected = temp1_; 855 Register value = ref_reg; 856 Register tmp_ptr = IP; // Pointer to actual memory. 857 Register tmp = temp2_; // Value in memory. 858 859 __ add(tmp_ptr, base, ShifterOperand(offset)); 860 861 if (kPoisonHeapReferences) { 862 __ PoisonHeapReference(expected); 863 if (value == expected) { 864 // Do not poison `value`, as it is the same register as 865 // `expected`, which has just been poisoned. 866 } else { 867 __ PoisonHeapReference(value); 868 } 869 } 870 871 // do { 872 // tmp = [r_ptr] - expected; 873 // } while (tmp == 0 && failure([r_ptr] <- r_new_value)); 874 875 Label loop_head, exit_loop; 876 __ Bind(&loop_head); 877 878 __ ldrex(tmp, tmp_ptr); 879 880 __ subs(tmp, tmp, ShifterOperand(expected)); 881 882 __ it(NE); 883 __ clrex(NE); 884 885 __ b(&exit_loop, NE); 886 887 __ strex(tmp, value, tmp_ptr); 888 __ cmp(tmp, ShifterOperand(1)); 889 __ b(&loop_head, EQ); 890 891 __ Bind(&exit_loop); 892 893 if (kPoisonHeapReferences) { 894 __ UnpoisonHeapReference(expected); 895 if (value == expected) { 896 // Do not unpoison `value`, as it is the same register as 897 // `expected`, which has just been unpoisoned. 898 } else { 899 __ UnpoisonHeapReference(value); 900 } 901 } 902 903 __ Bind(&done); 904 __ b(GetExitLabel()); 905 } 906 907 private: 908 // The location (register) of the marked object reference. 909 const Location ref_; 910 // The register containing the object holding the marked object reference field. 911 const Register obj_; 912 // The location of the offset of the marked reference field within `obj_`. 913 Location field_offset_; 914 915 const Register temp1_; 916 const Register temp2_; 917 918 // The location of the entrypoint if already loaded. 919 const Location entrypoint_; 920 921 DISALLOW_COPY_AND_ASSIGN(ReadBarrierMarkAndUpdateFieldSlowPathARM); 922}; 923 924// Slow path generating a read barrier for a heap reference. 925class ReadBarrierForHeapReferenceSlowPathARM : public SlowPathCodeARM { 926 public: 927 ReadBarrierForHeapReferenceSlowPathARM(HInstruction* instruction, 928 Location out, 929 Location ref, 930 Location obj, 931 uint32_t offset, 932 Location index) 933 : SlowPathCodeARM(instruction), 934 out_(out), 935 ref_(ref), 936 obj_(obj), 937 offset_(offset), 938 index_(index) { 939 DCHECK(kEmitCompilerReadBarrier); 940 // If `obj` is equal to `out` or `ref`, it means the initial object 941 // has been overwritten by (or after) the heap object reference load 942 // to be instrumented, e.g.: 943 // 944 // __ LoadFromOffset(kLoadWord, out, out, offset); 945 // codegen_->GenerateReadBarrierSlow(instruction, out_loc, out_loc, out_loc, offset); 946 // 947 // In that case, we have lost the information about the original 948 // object, and the emitted read barrier cannot work properly. 949 DCHECK(!obj.Equals(out)) << "obj=" << obj << " out=" << out; 950 DCHECK(!obj.Equals(ref)) << "obj=" << obj << " ref=" << ref; 951 } 952 953 void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { 954 CodeGeneratorARM* arm_codegen = down_cast<CodeGeneratorARM*>(codegen); 955 LocationSummary* locations = instruction_->GetLocations(); 956 Register reg_out = out_.AsRegister<Register>(); 957 DCHECK(locations->CanCall()); 958 DCHECK(!locations->GetLiveRegisters()->ContainsCoreRegister(reg_out)); 959 DCHECK(instruction_->IsInstanceFieldGet() || 960 instruction_->IsStaticFieldGet() || 961 instruction_->IsArrayGet() || 962 instruction_->IsInstanceOf() || 963 instruction_->IsCheckCast() || 964 (instruction_->IsInvokeVirtual()) && instruction_->GetLocations()->Intrinsified()) 965 << "Unexpected instruction in read barrier for heap reference slow path: " 966 << instruction_->DebugName(); 967 // The read barrier instrumentation of object ArrayGet 968 // instructions does not support the HIntermediateAddress 969 // instruction. 970 DCHECK(!(instruction_->IsArrayGet() && 971 instruction_->AsArrayGet()->GetArray()->IsIntermediateAddress())); 972 973 __ Bind(GetEntryLabel()); 974 SaveLiveRegisters(codegen, locations); 975 976 // We may have to change the index's value, but as `index_` is a 977 // constant member (like other "inputs" of this slow path), 978 // introduce a copy of it, `index`. 979 Location index = index_; 980 if (index_.IsValid()) { 981 // Handle `index_` for HArrayGet and UnsafeGetObject/UnsafeGetObjectVolatile intrinsics. 982 if (instruction_->IsArrayGet()) { 983 // Compute the actual memory offset and store it in `index`. 984 Register index_reg = index_.AsRegister<Register>(); 985 DCHECK(locations->GetLiveRegisters()->ContainsCoreRegister(index_reg)); 986 if (codegen->IsCoreCalleeSaveRegister(index_reg)) { 987 // We are about to change the value of `index_reg` (see the 988 // calls to art::arm::Thumb2Assembler::Lsl and 989 // art::arm::Thumb2Assembler::AddConstant below), but it has 990 // not been saved by the previous call to 991 // art::SlowPathCode::SaveLiveRegisters, as it is a 992 // callee-save register -- 993 // art::SlowPathCode::SaveLiveRegisters does not consider 994 // callee-save registers, as it has been designed with the 995 // assumption that callee-save registers are supposed to be 996 // handled by the called function. So, as a callee-save 997 // register, `index_reg` _would_ eventually be saved onto 998 // the stack, but it would be too late: we would have 999 // changed its value earlier. Therefore, we manually save 1000 // it here into another freely available register, 1001 // `free_reg`, chosen of course among the caller-save 1002 // registers (as a callee-save `free_reg` register would 1003 // exhibit the same problem). 1004 // 1005 // Note we could have requested a temporary register from 1006 // the register allocator instead; but we prefer not to, as 1007 // this is a slow path, and we know we can find a 1008 // caller-save register that is available. 1009 Register free_reg = FindAvailableCallerSaveRegister(codegen); 1010 __ Mov(free_reg, index_reg); 1011 index_reg = free_reg; 1012 index = Location::RegisterLocation(index_reg); 1013 } else { 1014 // The initial register stored in `index_` has already been 1015 // saved in the call to art::SlowPathCode::SaveLiveRegisters 1016 // (as it is not a callee-save register), so we can freely 1017 // use it. 1018 } 1019 // Shifting the index value contained in `index_reg` by the scale 1020 // factor (2) cannot overflow in practice, as the runtime is 1021 // unable to allocate object arrays with a size larger than 1022 // 2^26 - 1 (that is, 2^28 - 4 bytes). 1023 __ Lsl(index_reg, index_reg, TIMES_4); 1024 static_assert( 1025 sizeof(mirror::HeapReference<mirror::Object>) == sizeof(int32_t), 1026 "art::mirror::HeapReference<art::mirror::Object> and int32_t have different sizes."); 1027 __ AddConstant(index_reg, index_reg, offset_); 1028 } else { 1029 // In the case of the UnsafeGetObject/UnsafeGetObjectVolatile 1030 // intrinsics, `index_` is not shifted by a scale factor of 2 1031 // (as in the case of ArrayGet), as it is actually an offset 1032 // to an object field within an object. 1033 DCHECK(instruction_->IsInvoke()) << instruction_->DebugName(); 1034 DCHECK(instruction_->GetLocations()->Intrinsified()); 1035 DCHECK((instruction_->AsInvoke()->GetIntrinsic() == Intrinsics::kUnsafeGetObject) || 1036 (instruction_->AsInvoke()->GetIntrinsic() == Intrinsics::kUnsafeGetObjectVolatile)) 1037 << instruction_->AsInvoke()->GetIntrinsic(); 1038 DCHECK_EQ(offset_, 0U); 1039 DCHECK(index_.IsRegisterPair()); 1040 // UnsafeGet's offset location is a register pair, the low 1041 // part contains the correct offset. 1042 index = index_.ToLow(); 1043 } 1044 } 1045 1046 // We're moving two or three locations to locations that could 1047 // overlap, so we need a parallel move resolver. 1048 InvokeRuntimeCallingConvention calling_convention; 1049 HParallelMove parallel_move(codegen->GetGraph()->GetArena()); 1050 parallel_move.AddMove(ref_, 1051 Location::RegisterLocation(calling_convention.GetRegisterAt(0)), 1052 Primitive::kPrimNot, 1053 nullptr); 1054 parallel_move.AddMove(obj_, 1055 Location::RegisterLocation(calling_convention.GetRegisterAt(1)), 1056 Primitive::kPrimNot, 1057 nullptr); 1058 if (index.IsValid()) { 1059 parallel_move.AddMove(index, 1060 Location::RegisterLocation(calling_convention.GetRegisterAt(2)), 1061 Primitive::kPrimInt, 1062 nullptr); 1063 codegen->GetMoveResolver()->EmitNativeCode(¶llel_move); 1064 } else { 1065 codegen->GetMoveResolver()->EmitNativeCode(¶llel_move); 1066 __ LoadImmediate(calling_convention.GetRegisterAt(2), offset_); 1067 } 1068 arm_codegen->InvokeRuntime(kQuickReadBarrierSlow, instruction_, instruction_->GetDexPc(), this); 1069 CheckEntrypointTypes< 1070 kQuickReadBarrierSlow, mirror::Object*, mirror::Object*, mirror::Object*, uint32_t>(); 1071 arm_codegen->Move32(out_, Location::RegisterLocation(R0)); 1072 1073 RestoreLiveRegisters(codegen, locations); 1074 __ b(GetExitLabel()); 1075 } 1076 1077 const char* GetDescription() const OVERRIDE { return "ReadBarrierForHeapReferenceSlowPathARM"; } 1078 1079 private: 1080 Register FindAvailableCallerSaveRegister(CodeGenerator* codegen) { 1081 size_t ref = static_cast<int>(ref_.AsRegister<Register>()); 1082 size_t obj = static_cast<int>(obj_.AsRegister<Register>()); 1083 for (size_t i = 0, e = codegen->GetNumberOfCoreRegisters(); i < e; ++i) { 1084 if (i != ref && i != obj && !codegen->IsCoreCalleeSaveRegister(i)) { 1085 return static_cast<Register>(i); 1086 } 1087 } 1088 // We shall never fail to find a free caller-save register, as 1089 // there are more than two core caller-save registers on ARM 1090 // (meaning it is possible to find one which is different from 1091 // `ref` and `obj`). 1092 DCHECK_GT(codegen->GetNumberOfCoreCallerSaveRegisters(), 2u); 1093 LOG(FATAL) << "Could not find a free caller-save register"; 1094 UNREACHABLE(); 1095 } 1096 1097 const Location out_; 1098 const Location ref_; 1099 const Location obj_; 1100 const uint32_t offset_; 1101 // An additional location containing an index to an array. 1102 // Only used for HArrayGet and the UnsafeGetObject & 1103 // UnsafeGetObjectVolatile intrinsics. 1104 const Location index_; 1105 1106 DISALLOW_COPY_AND_ASSIGN(ReadBarrierForHeapReferenceSlowPathARM); 1107}; 1108 1109// Slow path generating a read barrier for a GC root. 1110class ReadBarrierForRootSlowPathARM : public SlowPathCodeARM { 1111 public: 1112 ReadBarrierForRootSlowPathARM(HInstruction* instruction, Location out, Location root) 1113 : SlowPathCodeARM(instruction), out_(out), root_(root) { 1114 DCHECK(kEmitCompilerReadBarrier); 1115 } 1116 1117 void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { 1118 LocationSummary* locations = instruction_->GetLocations(); 1119 Register reg_out = out_.AsRegister<Register>(); 1120 DCHECK(locations->CanCall()); 1121 DCHECK(!locations->GetLiveRegisters()->ContainsCoreRegister(reg_out)); 1122 DCHECK(instruction_->IsLoadClass() || instruction_->IsLoadString()) 1123 << "Unexpected instruction in read barrier for GC root slow path: " 1124 << instruction_->DebugName(); 1125 1126 __ Bind(GetEntryLabel()); 1127 SaveLiveRegisters(codegen, locations); 1128 1129 InvokeRuntimeCallingConvention calling_convention; 1130 CodeGeneratorARM* arm_codegen = down_cast<CodeGeneratorARM*>(codegen); 1131 arm_codegen->Move32(Location::RegisterLocation(calling_convention.GetRegisterAt(0)), root_); 1132 arm_codegen->InvokeRuntime(kQuickReadBarrierForRootSlow, 1133 instruction_, 1134 instruction_->GetDexPc(), 1135 this); 1136 CheckEntrypointTypes<kQuickReadBarrierForRootSlow, mirror::Object*, GcRoot<mirror::Object>*>(); 1137 arm_codegen->Move32(out_, Location::RegisterLocation(R0)); 1138 1139 RestoreLiveRegisters(codegen, locations); 1140 __ b(GetExitLabel()); 1141 } 1142 1143 const char* GetDescription() const OVERRIDE { return "ReadBarrierForRootSlowPathARM"; } 1144 1145 private: 1146 const Location out_; 1147 const Location root_; 1148 1149 DISALLOW_COPY_AND_ASSIGN(ReadBarrierForRootSlowPathARM); 1150}; 1151 1152inline Condition ARMCondition(IfCondition cond) { 1153 switch (cond) { 1154 case kCondEQ: return EQ; 1155 case kCondNE: return NE; 1156 case kCondLT: return LT; 1157 case kCondLE: return LE; 1158 case kCondGT: return GT; 1159 case kCondGE: return GE; 1160 case kCondB: return LO; 1161 case kCondBE: return LS; 1162 case kCondA: return HI; 1163 case kCondAE: return HS; 1164 } 1165 LOG(FATAL) << "Unreachable"; 1166 UNREACHABLE(); 1167} 1168 1169// Maps signed condition to unsigned condition. 1170inline Condition ARMUnsignedCondition(IfCondition cond) { 1171 switch (cond) { 1172 case kCondEQ: return EQ; 1173 case kCondNE: return NE; 1174 // Signed to unsigned. 1175 case kCondLT: return LO; 1176 case kCondLE: return LS; 1177 case kCondGT: return HI; 1178 case kCondGE: return HS; 1179 // Unsigned remain unchanged. 1180 case kCondB: return LO; 1181 case kCondBE: return LS; 1182 case kCondA: return HI; 1183 case kCondAE: return HS; 1184 } 1185 LOG(FATAL) << "Unreachable"; 1186 UNREACHABLE(); 1187} 1188 1189inline Condition ARMFPCondition(IfCondition cond, bool gt_bias) { 1190 // The ARM condition codes can express all the necessary branches, see the 1191 // "Meaning (floating-point)" column in the table A8-1 of the ARMv7 reference manual. 1192 // There is no dex instruction or HIR that would need the missing conditions 1193 // "equal or unordered" or "not equal". 1194 switch (cond) { 1195 case kCondEQ: return EQ; 1196 case kCondNE: return NE /* unordered */; 1197 case kCondLT: return gt_bias ? CC : LT /* unordered */; 1198 case kCondLE: return gt_bias ? LS : LE /* unordered */; 1199 case kCondGT: return gt_bias ? HI /* unordered */ : GT; 1200 case kCondGE: return gt_bias ? CS /* unordered */ : GE; 1201 default: 1202 LOG(FATAL) << "UNREACHABLE"; 1203 UNREACHABLE(); 1204 } 1205} 1206 1207inline Shift ShiftFromOpKind(HDataProcWithShifterOp::OpKind op_kind) { 1208 switch (op_kind) { 1209 case HDataProcWithShifterOp::kASR: return ASR; 1210 case HDataProcWithShifterOp::kLSL: return LSL; 1211 case HDataProcWithShifterOp::kLSR: return LSR; 1212 default: 1213 LOG(FATAL) << "Unexpected op kind " << op_kind; 1214 UNREACHABLE(); 1215 } 1216} 1217 1218static void GenerateDataProcInstruction(HInstruction::InstructionKind kind, 1219 Register out, 1220 Register first, 1221 const ShifterOperand& second, 1222 CodeGeneratorARM* codegen) { 1223 if (second.IsImmediate() && second.GetImmediate() == 0) { 1224 const ShifterOperand in = kind == HInstruction::kAnd 1225 ? ShifterOperand(0) 1226 : ShifterOperand(first); 1227 1228 __ mov(out, in); 1229 } else { 1230 switch (kind) { 1231 case HInstruction::kAdd: 1232 __ add(out, first, second); 1233 break; 1234 case HInstruction::kAnd: 1235 __ and_(out, first, second); 1236 break; 1237 case HInstruction::kOr: 1238 __ orr(out, first, second); 1239 break; 1240 case HInstruction::kSub: 1241 __ sub(out, first, second); 1242 break; 1243 case HInstruction::kXor: 1244 __ eor(out, first, second); 1245 break; 1246 default: 1247 LOG(FATAL) << "Unexpected instruction kind: " << kind; 1248 UNREACHABLE(); 1249 } 1250 } 1251} 1252 1253static void GenerateDataProc(HInstruction::InstructionKind kind, 1254 const Location& out, 1255 const Location& first, 1256 const ShifterOperand& second_lo, 1257 const ShifterOperand& second_hi, 1258 CodeGeneratorARM* codegen) { 1259 const Register first_hi = first.AsRegisterPairHigh<Register>(); 1260 const Register first_lo = first.AsRegisterPairLow<Register>(); 1261 const Register out_hi = out.AsRegisterPairHigh<Register>(); 1262 const Register out_lo = out.AsRegisterPairLow<Register>(); 1263 1264 if (kind == HInstruction::kAdd) { 1265 __ adds(out_lo, first_lo, second_lo); 1266 __ adc(out_hi, first_hi, second_hi); 1267 } else if (kind == HInstruction::kSub) { 1268 __ subs(out_lo, first_lo, second_lo); 1269 __ sbc(out_hi, first_hi, second_hi); 1270 } else { 1271 GenerateDataProcInstruction(kind, out_lo, first_lo, second_lo, codegen); 1272 GenerateDataProcInstruction(kind, out_hi, first_hi, second_hi, codegen); 1273 } 1274} 1275 1276static ShifterOperand GetShifterOperand(Register rm, Shift shift, uint32_t shift_imm) { 1277 return shift_imm == 0 ? ShifterOperand(rm) : ShifterOperand(rm, shift, shift_imm); 1278} 1279 1280static void GenerateLongDataProc(HDataProcWithShifterOp* instruction, CodeGeneratorARM* codegen) { 1281 DCHECK_EQ(instruction->GetType(), Primitive::kPrimLong); 1282 DCHECK(HDataProcWithShifterOp::IsShiftOp(instruction->GetOpKind())); 1283 1284 const LocationSummary* const locations = instruction->GetLocations(); 1285 const uint32_t shift_value = instruction->GetShiftAmount(); 1286 const HInstruction::InstructionKind kind = instruction->GetInstrKind(); 1287 const Location first = locations->InAt(0); 1288 const Location second = locations->InAt(1); 1289 const Location out = locations->Out(); 1290 const Register first_hi = first.AsRegisterPairHigh<Register>(); 1291 const Register first_lo = first.AsRegisterPairLow<Register>(); 1292 const Register out_hi = out.AsRegisterPairHigh<Register>(); 1293 const Register out_lo = out.AsRegisterPairLow<Register>(); 1294 const Register second_hi = second.AsRegisterPairHigh<Register>(); 1295 const Register second_lo = second.AsRegisterPairLow<Register>(); 1296 const Shift shift = ShiftFromOpKind(instruction->GetOpKind()); 1297 1298 if (shift_value >= 32) { 1299 if (shift == LSL) { 1300 GenerateDataProcInstruction(kind, 1301 out_hi, 1302 first_hi, 1303 ShifterOperand(second_lo, LSL, shift_value - 32), 1304 codegen); 1305 GenerateDataProcInstruction(kind, 1306 out_lo, 1307 first_lo, 1308 ShifterOperand(0), 1309 codegen); 1310 } else if (shift == ASR) { 1311 GenerateDataProc(kind, 1312 out, 1313 first, 1314 GetShifterOperand(second_hi, ASR, shift_value - 32), 1315 ShifterOperand(second_hi, ASR, 31), 1316 codegen); 1317 } else { 1318 DCHECK_EQ(shift, LSR); 1319 GenerateDataProc(kind, 1320 out, 1321 first, 1322 GetShifterOperand(second_hi, LSR, shift_value - 32), 1323 ShifterOperand(0), 1324 codegen); 1325 } 1326 } else { 1327 DCHECK_GT(shift_value, 1U); 1328 DCHECK_LT(shift_value, 32U); 1329 1330 if (shift == LSL) { 1331 // We are not doing this for HInstruction::kAdd because the output will require 1332 // Location::kOutputOverlap; not applicable to other cases. 1333 if (kind == HInstruction::kOr || kind == HInstruction::kXor) { 1334 GenerateDataProcInstruction(kind, 1335 out_hi, 1336 first_hi, 1337 ShifterOperand(second_hi, LSL, shift_value), 1338 codegen); 1339 GenerateDataProcInstruction(kind, 1340 out_hi, 1341 out_hi, 1342 ShifterOperand(second_lo, LSR, 32 - shift_value), 1343 codegen); 1344 GenerateDataProcInstruction(kind, 1345 out_lo, 1346 first_lo, 1347 ShifterOperand(second_lo, LSL, shift_value), 1348 codegen); 1349 } else { 1350 __ Lsl(IP, second_hi, shift_value); 1351 __ orr(IP, IP, ShifterOperand(second_lo, LSR, 32 - shift_value)); 1352 GenerateDataProc(kind, 1353 out, 1354 first, 1355 ShifterOperand(second_lo, LSL, shift_value), 1356 ShifterOperand(IP), 1357 codegen); 1358 } 1359 } else { 1360 DCHECK(shift == ASR || shift == LSR); 1361 1362 // We are not doing this for HInstruction::kAdd because the output will require 1363 // Location::kOutputOverlap; not applicable to other cases. 1364 if (kind == HInstruction::kOr || kind == HInstruction::kXor) { 1365 GenerateDataProcInstruction(kind, 1366 out_lo, 1367 first_lo, 1368 ShifterOperand(second_lo, LSR, shift_value), 1369 codegen); 1370 GenerateDataProcInstruction(kind, 1371 out_lo, 1372 out_lo, 1373 ShifterOperand(second_hi, LSL, 32 - shift_value), 1374 codegen); 1375 GenerateDataProcInstruction(kind, 1376 out_hi, 1377 first_hi, 1378 ShifterOperand(second_hi, shift, shift_value), 1379 codegen); 1380 } else { 1381 __ Lsr(IP, second_lo, shift_value); 1382 __ orr(IP, IP, ShifterOperand(second_hi, LSL, 32 - shift_value)); 1383 GenerateDataProc(kind, 1384 out, 1385 first, 1386 ShifterOperand(IP), 1387 ShifterOperand(second_hi, shift, shift_value), 1388 codegen); 1389 } 1390 } 1391 } 1392} 1393 1394#undef __ 1395// NOLINT on __ macro to suppress wrong warning/fix (misc-macro-parentheses) from clang-tidy. 1396#define __ down_cast<ArmAssembler*>(GetAssembler())-> // NOLINT 1397 1398void CodeGeneratorARM::DumpCoreRegister(std::ostream& stream, int reg) const { 1399 stream << Register(reg); 1400} 1401 1402void CodeGeneratorARM::DumpFloatingPointRegister(std::ostream& stream, int reg) const { 1403 stream << SRegister(reg); 1404} 1405 1406size_t CodeGeneratorARM::SaveCoreRegister(size_t stack_index, uint32_t reg_id) { 1407 __ StoreToOffset(kStoreWord, static_cast<Register>(reg_id), SP, stack_index); 1408 return kArmWordSize; 1409} 1410 1411size_t CodeGeneratorARM::RestoreCoreRegister(size_t stack_index, uint32_t reg_id) { 1412 __ LoadFromOffset(kLoadWord, static_cast<Register>(reg_id), SP, stack_index); 1413 return kArmWordSize; 1414} 1415 1416size_t CodeGeneratorARM::SaveFloatingPointRegister(size_t stack_index, uint32_t reg_id) { 1417 __ StoreSToOffset(static_cast<SRegister>(reg_id), SP, stack_index); 1418 return kArmWordSize; 1419} 1420 1421size_t CodeGeneratorARM::RestoreFloatingPointRegister(size_t stack_index, uint32_t reg_id) { 1422 __ LoadSFromOffset(static_cast<SRegister>(reg_id), SP, stack_index); 1423 return kArmWordSize; 1424} 1425 1426CodeGeneratorARM::CodeGeneratorARM(HGraph* graph, 1427 const ArmInstructionSetFeatures& isa_features, 1428 const CompilerOptions& compiler_options, 1429 OptimizingCompilerStats* stats) 1430 : CodeGenerator(graph, 1431 kNumberOfCoreRegisters, 1432 kNumberOfSRegisters, 1433 kNumberOfRegisterPairs, 1434 ComputeRegisterMask(reinterpret_cast<const int*>(kCoreCalleeSaves), 1435 arraysize(kCoreCalleeSaves)), 1436 ComputeRegisterMask(reinterpret_cast<const int*>(kFpuCalleeSaves), 1437 arraysize(kFpuCalleeSaves)), 1438 compiler_options, 1439 stats), 1440 block_labels_(nullptr), 1441 location_builder_(graph, this), 1442 instruction_visitor_(graph, this), 1443 move_resolver_(graph->GetArena(), this), 1444 assembler_(graph->GetArena()), 1445 isa_features_(isa_features), 1446 uint32_literals_(std::less<uint32_t>(), 1447 graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), 1448 pc_relative_dex_cache_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), 1449 boot_image_string_patches_(StringReferenceValueComparator(), 1450 graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), 1451 pc_relative_string_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), 1452 boot_image_type_patches_(TypeReferenceValueComparator(), 1453 graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), 1454 pc_relative_type_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), 1455 type_bss_entry_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), 1456 boot_image_address_patches_(std::less<uint32_t>(), 1457 graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), 1458 jit_string_patches_(StringReferenceValueComparator(), 1459 graph->GetArena()->Adapter(kArenaAllocCodeGenerator)), 1460 jit_class_patches_(TypeReferenceValueComparator(), 1461 graph->GetArena()->Adapter(kArenaAllocCodeGenerator)) { 1462 // Always save the LR register to mimic Quick. 1463 AddAllocatedRegister(Location::RegisterLocation(LR)); 1464} 1465 1466void CodeGeneratorARM::Finalize(CodeAllocator* allocator) { 1467 // Ensure that we fix up branches and literal loads and emit the literal pool. 1468 __ FinalizeCode(); 1469 1470 // Adjust native pc offsets in stack maps. 1471 for (size_t i = 0, num = stack_map_stream_.GetNumberOfStackMaps(); i != num; ++i) { 1472 uint32_t old_position = 1473 stack_map_stream_.GetStackMap(i).native_pc_code_offset.Uint32Value(kThumb2); 1474 uint32_t new_position = __ GetAdjustedPosition(old_position); 1475 stack_map_stream_.SetStackMapNativePcOffset(i, new_position); 1476 } 1477 // Adjust pc offsets for the disassembly information. 1478 if (disasm_info_ != nullptr) { 1479 GeneratedCodeInterval* frame_entry_interval = disasm_info_->GetFrameEntryInterval(); 1480 frame_entry_interval->start = __ GetAdjustedPosition(frame_entry_interval->start); 1481 frame_entry_interval->end = __ GetAdjustedPosition(frame_entry_interval->end); 1482 for (auto& it : *disasm_info_->GetInstructionIntervals()) { 1483 it.second.start = __ GetAdjustedPosition(it.second.start); 1484 it.second.end = __ GetAdjustedPosition(it.second.end); 1485 } 1486 for (auto& it : *disasm_info_->GetSlowPathIntervals()) { 1487 it.code_interval.start = __ GetAdjustedPosition(it.code_interval.start); 1488 it.code_interval.end = __ GetAdjustedPosition(it.code_interval.end); 1489 } 1490 } 1491 1492 CodeGenerator::Finalize(allocator); 1493} 1494 1495void CodeGeneratorARM::SetupBlockedRegisters() const { 1496 // Stack register, LR and PC are always reserved. 1497 blocked_core_registers_[SP] = true; 1498 blocked_core_registers_[LR] = true; 1499 blocked_core_registers_[PC] = true; 1500 1501 // Reserve thread register. 1502 blocked_core_registers_[TR] = true; 1503 1504 // Reserve temp register. 1505 blocked_core_registers_[IP] = true; 1506 1507 if (GetGraph()->IsDebuggable()) { 1508 // Stubs do not save callee-save floating point registers. If the graph 1509 // is debuggable, we need to deal with these registers differently. For 1510 // now, just block them. 1511 for (size_t i = 0; i < arraysize(kFpuCalleeSaves); ++i) { 1512 blocked_fpu_registers_[kFpuCalleeSaves[i]] = true; 1513 } 1514 } 1515} 1516 1517InstructionCodeGeneratorARM::InstructionCodeGeneratorARM(HGraph* graph, CodeGeneratorARM* codegen) 1518 : InstructionCodeGenerator(graph, codegen), 1519 assembler_(codegen->GetAssembler()), 1520 codegen_(codegen) {} 1521 1522void CodeGeneratorARM::ComputeSpillMask() { 1523 core_spill_mask_ = allocated_registers_.GetCoreRegisters() & core_callee_save_mask_; 1524 DCHECK_NE(core_spill_mask_, 0u) << "At least the return address register must be saved"; 1525 // There is no easy instruction to restore just the PC on thumb2. We spill and 1526 // restore another arbitrary register. 1527 core_spill_mask_ |= (1 << kCoreAlwaysSpillRegister); 1528 fpu_spill_mask_ = allocated_registers_.GetFloatingPointRegisters() & fpu_callee_save_mask_; 1529 // We use vpush and vpop for saving and restoring floating point registers, which take 1530 // a SRegister and the number of registers to save/restore after that SRegister. We 1531 // therefore update the `fpu_spill_mask_` to also contain those registers not allocated, 1532 // but in the range. 1533 if (fpu_spill_mask_ != 0) { 1534 uint32_t least_significant_bit = LeastSignificantBit(fpu_spill_mask_); 1535 uint32_t most_significant_bit = MostSignificantBit(fpu_spill_mask_); 1536 for (uint32_t i = least_significant_bit + 1 ; i < most_significant_bit; ++i) { 1537 fpu_spill_mask_ |= (1 << i); 1538 } 1539 } 1540} 1541 1542static dwarf::Reg DWARFReg(Register reg) { 1543 return dwarf::Reg::ArmCore(static_cast<int>(reg)); 1544} 1545 1546static dwarf::Reg DWARFReg(SRegister reg) { 1547 return dwarf::Reg::ArmFp(static_cast<int>(reg)); 1548} 1549 1550void CodeGeneratorARM::GenerateFrameEntry() { 1551 bool skip_overflow_check = 1552 IsLeafMethod() && !FrameNeedsStackCheck(GetFrameSize(), InstructionSet::kArm); 1553 DCHECK(GetCompilerOptions().GetImplicitStackOverflowChecks()); 1554 __ Bind(&frame_entry_label_); 1555 1556 if (HasEmptyFrame()) { 1557 return; 1558 } 1559 1560 if (!skip_overflow_check) { 1561 __ AddConstant(IP, SP, -static_cast<int32_t>(GetStackOverflowReservedBytes(kArm))); 1562 __ LoadFromOffset(kLoadWord, IP, IP, 0); 1563 RecordPcInfo(nullptr, 0); 1564 } 1565 1566 __ PushList(core_spill_mask_); 1567 __ cfi().AdjustCFAOffset(kArmWordSize * POPCOUNT(core_spill_mask_)); 1568 __ cfi().RelOffsetForMany(DWARFReg(kMethodRegisterArgument), 0, core_spill_mask_, kArmWordSize); 1569 if (fpu_spill_mask_ != 0) { 1570 SRegister start_register = SRegister(LeastSignificantBit(fpu_spill_mask_)); 1571 __ vpushs(start_register, POPCOUNT(fpu_spill_mask_)); 1572 __ cfi().AdjustCFAOffset(kArmWordSize * POPCOUNT(fpu_spill_mask_)); 1573 __ cfi().RelOffsetForMany(DWARFReg(S0), 0, fpu_spill_mask_, kArmWordSize); 1574 } 1575 1576 if (GetGraph()->HasShouldDeoptimizeFlag()) { 1577 // Initialize should_deoptimize flag to 0. 1578 __ mov(IP, ShifterOperand(0)); 1579 __ StoreToOffset(kStoreWord, IP, SP, -kShouldDeoptimizeFlagSize); 1580 } 1581 1582 int adjust = GetFrameSize() - FrameEntrySpillSize(); 1583 __ AddConstant(SP, -adjust); 1584 __ cfi().AdjustCFAOffset(adjust); 1585 1586 // Save the current method if we need it. Note that we do not 1587 // do this in HCurrentMethod, as the instruction might have been removed 1588 // in the SSA graph. 1589 if (RequiresCurrentMethod()) { 1590 __ StoreToOffset(kStoreWord, kMethodRegisterArgument, SP, 0); 1591 } 1592} 1593 1594void CodeGeneratorARM::GenerateFrameExit() { 1595 if (HasEmptyFrame()) { 1596 __ bx(LR); 1597 return; 1598 } 1599 __ cfi().RememberState(); 1600 int adjust = GetFrameSize() - FrameEntrySpillSize(); 1601 __ AddConstant(SP, adjust); 1602 __ cfi().AdjustCFAOffset(-adjust); 1603 if (fpu_spill_mask_ != 0) { 1604 SRegister start_register = SRegister(LeastSignificantBit(fpu_spill_mask_)); 1605 __ vpops(start_register, POPCOUNT(fpu_spill_mask_)); 1606 __ cfi().AdjustCFAOffset(-static_cast<int>(kArmPointerSize) * POPCOUNT(fpu_spill_mask_)); 1607 __ cfi().RestoreMany(DWARFReg(SRegister(0)), fpu_spill_mask_); 1608 } 1609 // Pop LR into PC to return. 1610 DCHECK_NE(core_spill_mask_ & (1 << LR), 0U); 1611 uint32_t pop_mask = (core_spill_mask_ & (~(1 << LR))) | 1 << PC; 1612 __ PopList(pop_mask); 1613 __ cfi().RestoreState(); 1614 __ cfi().DefCFAOffset(GetFrameSize()); 1615} 1616 1617void CodeGeneratorARM::Bind(HBasicBlock* block) { 1618 Label* label = GetLabelOf(block); 1619 __ BindTrackedLabel(label); 1620} 1621 1622Location InvokeDexCallingConventionVisitorARM::GetNextLocation(Primitive::Type type) { 1623 switch (type) { 1624 case Primitive::kPrimBoolean: 1625 case Primitive::kPrimByte: 1626 case Primitive::kPrimChar: 1627 case Primitive::kPrimShort: 1628 case Primitive::kPrimInt: 1629 case Primitive::kPrimNot: { 1630 uint32_t index = gp_index_++; 1631 uint32_t stack_index = stack_index_++; 1632 if (index < calling_convention.GetNumberOfRegisters()) { 1633 return Location::RegisterLocation(calling_convention.GetRegisterAt(index)); 1634 } else { 1635 return Location::StackSlot(calling_convention.GetStackOffsetOf(stack_index)); 1636 } 1637 } 1638 1639 case Primitive::kPrimLong: { 1640 uint32_t index = gp_index_; 1641 uint32_t stack_index = stack_index_; 1642 gp_index_ += 2; 1643 stack_index_ += 2; 1644 if (index + 1 < calling_convention.GetNumberOfRegisters()) { 1645 if (calling_convention.GetRegisterAt(index) == R1) { 1646 // Skip R1, and use R2_R3 instead. 1647 gp_index_++; 1648 index++; 1649 } 1650 } 1651 if (index + 1 < calling_convention.GetNumberOfRegisters()) { 1652 DCHECK_EQ(calling_convention.GetRegisterAt(index) + 1, 1653 calling_convention.GetRegisterAt(index + 1)); 1654 1655 return Location::RegisterPairLocation(calling_convention.GetRegisterAt(index), 1656 calling_convention.GetRegisterAt(index + 1)); 1657 } else { 1658 return Location::DoubleStackSlot(calling_convention.GetStackOffsetOf(stack_index)); 1659 } 1660 } 1661 1662 case Primitive::kPrimFloat: { 1663 uint32_t stack_index = stack_index_++; 1664 if (float_index_ % 2 == 0) { 1665 float_index_ = std::max(double_index_, float_index_); 1666 } 1667 if (float_index_ < calling_convention.GetNumberOfFpuRegisters()) { 1668 return Location::FpuRegisterLocation(calling_convention.GetFpuRegisterAt(float_index_++)); 1669 } else { 1670 return Location::StackSlot(calling_convention.GetStackOffsetOf(stack_index)); 1671 } 1672 } 1673 1674 case Primitive::kPrimDouble: { 1675 double_index_ = std::max(double_index_, RoundUp(float_index_, 2)); 1676 uint32_t stack_index = stack_index_; 1677 stack_index_ += 2; 1678 if (double_index_ + 1 < calling_convention.GetNumberOfFpuRegisters()) { 1679 uint32_t index = double_index_; 1680 double_index_ += 2; 1681 Location result = Location::FpuRegisterPairLocation( 1682 calling_convention.GetFpuRegisterAt(index), 1683 calling_convention.GetFpuRegisterAt(index + 1)); 1684 DCHECK(ExpectedPairLayout(result)); 1685 return result; 1686 } else { 1687 return Location::DoubleStackSlot(calling_convention.GetStackOffsetOf(stack_index)); 1688 } 1689 } 1690 1691 case Primitive::kPrimVoid: 1692 LOG(FATAL) << "Unexpected parameter type " << type; 1693 break; 1694 } 1695 return Location::NoLocation(); 1696} 1697 1698Location InvokeDexCallingConventionVisitorARM::GetReturnLocation(Primitive::Type type) const { 1699 switch (type) { 1700 case Primitive::kPrimBoolean: 1701 case Primitive::kPrimByte: 1702 case Primitive::kPrimChar: 1703 case Primitive::kPrimShort: 1704 case Primitive::kPrimInt: 1705 case Primitive::kPrimNot: { 1706 return Location::RegisterLocation(R0); 1707 } 1708 1709 case Primitive::kPrimFloat: { 1710 return Location::FpuRegisterLocation(S0); 1711 } 1712 1713 case Primitive::kPrimLong: { 1714 return Location::RegisterPairLocation(R0, R1); 1715 } 1716 1717 case Primitive::kPrimDouble: { 1718 return Location::FpuRegisterPairLocation(S0, S1); 1719 } 1720 1721 case Primitive::kPrimVoid: 1722 return Location::NoLocation(); 1723 } 1724 1725 UNREACHABLE(); 1726} 1727 1728Location InvokeDexCallingConventionVisitorARM::GetMethodLocation() const { 1729 return Location::RegisterLocation(kMethodRegisterArgument); 1730} 1731 1732void CodeGeneratorARM::Move32(Location destination, Location source) { 1733 if (source.Equals(destination)) { 1734 return; 1735 } 1736 if (destination.IsRegister()) { 1737 if (source.IsRegister()) { 1738 __ Mov(destination.AsRegister<Register>(), source.AsRegister<Register>()); 1739 } else if (source.IsFpuRegister()) { 1740 __ vmovrs(destination.AsRegister<Register>(), source.AsFpuRegister<SRegister>()); 1741 } else { 1742 __ LoadFromOffset(kLoadWord, destination.AsRegister<Register>(), SP, source.GetStackIndex()); 1743 } 1744 } else if (destination.IsFpuRegister()) { 1745 if (source.IsRegister()) { 1746 __ vmovsr(destination.AsFpuRegister<SRegister>(), source.AsRegister<Register>()); 1747 } else if (source.IsFpuRegister()) { 1748 __ vmovs(destination.AsFpuRegister<SRegister>(), source.AsFpuRegister<SRegister>()); 1749 } else { 1750 __ LoadSFromOffset(destination.AsFpuRegister<SRegister>(), SP, source.GetStackIndex()); 1751 } 1752 } else { 1753 DCHECK(destination.IsStackSlot()) << destination; 1754 if (source.IsRegister()) { 1755 __ StoreToOffset(kStoreWord, source.AsRegister<Register>(), SP, destination.GetStackIndex()); 1756 } else if (source.IsFpuRegister()) { 1757 __ StoreSToOffset(source.AsFpuRegister<SRegister>(), SP, destination.GetStackIndex()); 1758 } else { 1759 DCHECK(source.IsStackSlot()) << source; 1760 __ LoadFromOffset(kLoadWord, IP, SP, source.GetStackIndex()); 1761 __ StoreToOffset(kStoreWord, IP, SP, destination.GetStackIndex()); 1762 } 1763 } 1764} 1765 1766void CodeGeneratorARM::Move64(Location destination, Location source) { 1767 if (source.Equals(destination)) { 1768 return; 1769 } 1770 if (destination.IsRegisterPair()) { 1771 if (source.IsRegisterPair()) { 1772 EmitParallelMoves( 1773 Location::RegisterLocation(source.AsRegisterPairHigh<Register>()), 1774 Location::RegisterLocation(destination.AsRegisterPairHigh<Register>()), 1775 Primitive::kPrimInt, 1776 Location::RegisterLocation(source.AsRegisterPairLow<Register>()), 1777 Location::RegisterLocation(destination.AsRegisterPairLow<Register>()), 1778 Primitive::kPrimInt); 1779 } else if (source.IsFpuRegister()) { 1780 UNIMPLEMENTED(FATAL); 1781 } else if (source.IsFpuRegisterPair()) { 1782 __ vmovrrd(destination.AsRegisterPairLow<Register>(), 1783 destination.AsRegisterPairHigh<Register>(), 1784 FromLowSToD(source.AsFpuRegisterPairLow<SRegister>())); 1785 } else { 1786 DCHECK(source.IsDoubleStackSlot()); 1787 DCHECK(ExpectedPairLayout(destination)); 1788 __ LoadFromOffset(kLoadWordPair, destination.AsRegisterPairLow<Register>(), 1789 SP, source.GetStackIndex()); 1790 } 1791 } else if (destination.IsFpuRegisterPair()) { 1792 if (source.IsDoubleStackSlot()) { 1793 __ LoadDFromOffset(FromLowSToD(destination.AsFpuRegisterPairLow<SRegister>()), 1794 SP, 1795 source.GetStackIndex()); 1796 } else if (source.IsRegisterPair()) { 1797 __ vmovdrr(FromLowSToD(destination.AsFpuRegisterPairLow<SRegister>()), 1798 source.AsRegisterPairLow<Register>(), 1799 source.AsRegisterPairHigh<Register>()); 1800 } else { 1801 UNIMPLEMENTED(FATAL); 1802 } 1803 } else { 1804 DCHECK(destination.IsDoubleStackSlot()); 1805 if (source.IsRegisterPair()) { 1806 // No conflict possible, so just do the moves. 1807 if (source.AsRegisterPairLow<Register>() == R1) { 1808 DCHECK_EQ(source.AsRegisterPairHigh<Register>(), R2); 1809 __ StoreToOffset(kStoreWord, R1, SP, destination.GetStackIndex()); 1810 __ StoreToOffset(kStoreWord, R2, SP, destination.GetHighStackIndex(kArmWordSize)); 1811 } else { 1812 __ StoreToOffset(kStoreWordPair, source.AsRegisterPairLow<Register>(), 1813 SP, destination.GetStackIndex()); 1814 } 1815 } else if (source.IsFpuRegisterPair()) { 1816 __ StoreDToOffset(FromLowSToD(source.AsFpuRegisterPairLow<SRegister>()), 1817 SP, 1818 destination.GetStackIndex()); 1819 } else { 1820 DCHECK(source.IsDoubleStackSlot()); 1821 EmitParallelMoves( 1822 Location::StackSlot(source.GetStackIndex()), 1823 Location::StackSlot(destination.GetStackIndex()), 1824 Primitive::kPrimInt, 1825 Location::StackSlot(source.GetHighStackIndex(kArmWordSize)), 1826 Location::StackSlot(destination.GetHighStackIndex(kArmWordSize)), 1827 Primitive::kPrimInt); 1828 } 1829 } 1830} 1831 1832void CodeGeneratorARM::MoveConstant(Location location, int32_t value) { 1833 DCHECK(location.IsRegister()); 1834 __ LoadImmediate(location.AsRegister<Register>(), value); 1835} 1836 1837void CodeGeneratorARM::MoveLocation(Location dst, Location src, Primitive::Type dst_type) { 1838 HParallelMove move(GetGraph()->GetArena()); 1839 move.AddMove(src, dst, dst_type, nullptr); 1840 GetMoveResolver()->EmitNativeCode(&move); 1841} 1842 1843void CodeGeneratorARM::AddLocationAsTemp(Location location, LocationSummary* locations) { 1844 if (location.IsRegister()) { 1845 locations->AddTemp(location); 1846 } else if (location.IsRegisterPair()) { 1847 locations->AddTemp(Location::RegisterLocation(location.AsRegisterPairLow<Register>())); 1848 locations->AddTemp(Location::RegisterLocation(location.AsRegisterPairHigh<Register>())); 1849 } else { 1850 UNIMPLEMENTED(FATAL) << "AddLocationAsTemp not implemented for location " << location; 1851 } 1852} 1853 1854void CodeGeneratorARM::InvokeRuntime(QuickEntrypointEnum entrypoint, 1855 HInstruction* instruction, 1856 uint32_t dex_pc, 1857 SlowPathCode* slow_path) { 1858 ValidateInvokeRuntime(entrypoint, instruction, slow_path); 1859 GenerateInvokeRuntime(GetThreadOffset<kArmPointerSize>(entrypoint).Int32Value()); 1860 if (EntrypointRequiresStackMap(entrypoint)) { 1861 RecordPcInfo(instruction, dex_pc, slow_path); 1862 } 1863} 1864 1865void CodeGeneratorARM::InvokeRuntimeWithoutRecordingPcInfo(int32_t entry_point_offset, 1866 HInstruction* instruction, 1867 SlowPathCode* slow_path) { 1868 ValidateInvokeRuntimeWithoutRecordingPcInfo(instruction, slow_path); 1869 GenerateInvokeRuntime(entry_point_offset); 1870} 1871 1872void CodeGeneratorARM::GenerateInvokeRuntime(int32_t entry_point_offset) { 1873 __ LoadFromOffset(kLoadWord, LR, TR, entry_point_offset); 1874 __ blx(LR); 1875} 1876 1877void InstructionCodeGeneratorARM::HandleGoto(HInstruction* got, HBasicBlock* successor) { 1878 DCHECK(!successor->IsExitBlock()); 1879 1880 HBasicBlock* block = got->GetBlock(); 1881 HInstruction* previous = got->GetPrevious(); 1882 1883 HLoopInformation* info = block->GetLoopInformation(); 1884 if (info != nullptr && info->IsBackEdge(*block) && info->HasSuspendCheck()) { 1885 codegen_->ClearSpillSlotsFromLoopPhisInStackMap(info->GetSuspendCheck()); 1886 GenerateSuspendCheck(info->GetSuspendCheck(), successor); 1887 return; 1888 } 1889 1890 if (block->IsEntryBlock() && (previous != nullptr) && previous->IsSuspendCheck()) { 1891 GenerateSuspendCheck(previous->AsSuspendCheck(), nullptr); 1892 } 1893 if (!codegen_->GoesToNextBlock(got->GetBlock(), successor)) { 1894 __ b(codegen_->GetLabelOf(successor)); 1895 } 1896} 1897 1898void LocationsBuilderARM::VisitGoto(HGoto* got) { 1899 got->SetLocations(nullptr); 1900} 1901 1902void InstructionCodeGeneratorARM::VisitGoto(HGoto* got) { 1903 HandleGoto(got, got->GetSuccessor()); 1904} 1905 1906void LocationsBuilderARM::VisitTryBoundary(HTryBoundary* try_boundary) { 1907 try_boundary->SetLocations(nullptr); 1908} 1909 1910void InstructionCodeGeneratorARM::VisitTryBoundary(HTryBoundary* try_boundary) { 1911 HBasicBlock* successor = try_boundary->GetNormalFlowSuccessor(); 1912 if (!successor->IsExitBlock()) { 1913 HandleGoto(try_boundary, successor); 1914 } 1915} 1916 1917void LocationsBuilderARM::VisitExit(HExit* exit) { 1918 exit->SetLocations(nullptr); 1919} 1920 1921void InstructionCodeGeneratorARM::VisitExit(HExit* exit ATTRIBUTE_UNUSED) { 1922} 1923 1924void InstructionCodeGeneratorARM::GenerateVcmp(HInstruction* instruction) { 1925 Primitive::Type type = instruction->InputAt(0)->GetType(); 1926 Location lhs_loc = instruction->GetLocations()->InAt(0); 1927 Location rhs_loc = instruction->GetLocations()->InAt(1); 1928 if (rhs_loc.IsConstant()) { 1929 // 0.0 is the only immediate that can be encoded directly in 1930 // a VCMP instruction. 1931 // 1932 // Both the JLS (section 15.20.1) and the JVMS (section 6.5) 1933 // specify that in a floating-point comparison, positive zero 1934 // and negative zero are considered equal, so we can use the 1935 // literal 0.0 for both cases here. 1936 // 1937 // Note however that some methods (Float.equal, Float.compare, 1938 // Float.compareTo, Double.equal, Double.compare, 1939 // Double.compareTo, Math.max, Math.min, StrictMath.max, 1940 // StrictMath.min) consider 0.0 to be (strictly) greater than 1941 // -0.0. So if we ever translate calls to these methods into a 1942 // HCompare instruction, we must handle the -0.0 case with 1943 // care here. 1944 DCHECK(rhs_loc.GetConstant()->IsArithmeticZero()); 1945 if (type == Primitive::kPrimFloat) { 1946 __ vcmpsz(lhs_loc.AsFpuRegister<SRegister>()); 1947 } else { 1948 DCHECK_EQ(type, Primitive::kPrimDouble); 1949 __ vcmpdz(FromLowSToD(lhs_loc.AsFpuRegisterPairLow<SRegister>())); 1950 } 1951 } else { 1952 if (type == Primitive::kPrimFloat) { 1953 __ vcmps(lhs_loc.AsFpuRegister<SRegister>(), rhs_loc.AsFpuRegister<SRegister>()); 1954 } else { 1955 DCHECK_EQ(type, Primitive::kPrimDouble); 1956 __ vcmpd(FromLowSToD(lhs_loc.AsFpuRegisterPairLow<SRegister>()), 1957 FromLowSToD(rhs_loc.AsFpuRegisterPairLow<SRegister>())); 1958 } 1959 } 1960} 1961 1962void InstructionCodeGeneratorARM::GenerateFPJumps(HCondition* cond, 1963 Label* true_label, 1964 Label* false_label ATTRIBUTE_UNUSED) { 1965 __ vmstat(); // transfer FP status register to ARM APSR. 1966 __ b(true_label, ARMFPCondition(cond->GetCondition(), cond->IsGtBias())); 1967} 1968 1969void InstructionCodeGeneratorARM::GenerateLongComparesAndJumps(HCondition* cond, 1970 Label* true_label, 1971 Label* false_label) { 1972 LocationSummary* locations = cond->GetLocations(); 1973 Location left = locations->InAt(0); 1974 Location right = locations->InAt(1); 1975 IfCondition if_cond = cond->GetCondition(); 1976 1977 Register left_high = left.AsRegisterPairHigh<Register>(); 1978 Register left_low = left.AsRegisterPairLow<Register>(); 1979 IfCondition true_high_cond = if_cond; 1980 IfCondition false_high_cond = cond->GetOppositeCondition(); 1981 Condition final_condition = ARMUnsignedCondition(if_cond); // unsigned on lower part 1982 1983 // Set the conditions for the test, remembering that == needs to be 1984 // decided using the low words. 1985 // TODO: consider avoiding jumps with temporary and CMP low+SBC high 1986 switch (if_cond) { 1987 case kCondEQ: 1988 case kCondNE: 1989 // Nothing to do. 1990 break; 1991 case kCondLT: 1992 false_high_cond = kCondGT; 1993 break; 1994 case kCondLE: 1995 true_high_cond = kCondLT; 1996 break; 1997 case kCondGT: 1998 false_high_cond = kCondLT; 1999 break; 2000 case kCondGE: 2001 true_high_cond = kCondGT; 2002 break; 2003 case kCondB: 2004 false_high_cond = kCondA; 2005 break; 2006 case kCondBE: 2007 true_high_cond = kCondB; 2008 break; 2009 case kCondA: 2010 false_high_cond = kCondB; 2011 break; 2012 case kCondAE: 2013 true_high_cond = kCondA; 2014 break; 2015 } 2016 if (right.IsConstant()) { 2017 int64_t value = right.GetConstant()->AsLongConstant()->GetValue(); 2018 int32_t val_low = Low32Bits(value); 2019 int32_t val_high = High32Bits(value); 2020 2021 __ CmpConstant(left_high, val_high); 2022 if (if_cond == kCondNE) { 2023 __ b(true_label, ARMCondition(true_high_cond)); 2024 } else if (if_cond == kCondEQ) { 2025 __ b(false_label, ARMCondition(false_high_cond)); 2026 } else { 2027 __ b(true_label, ARMCondition(true_high_cond)); 2028 __ b(false_label, ARMCondition(false_high_cond)); 2029 } 2030 // Must be equal high, so compare the lows. 2031 __ CmpConstant(left_low, val_low); 2032 } else { 2033 Register right_high = right.AsRegisterPairHigh<Register>(); 2034 Register right_low = right.AsRegisterPairLow<Register>(); 2035 2036 __ cmp(left_high, ShifterOperand(right_high)); 2037 if (if_cond == kCondNE) { 2038 __ b(true_label, ARMCondition(true_high_cond)); 2039 } else if (if_cond == kCondEQ) { 2040 __ b(false_label, ARMCondition(false_high_cond)); 2041 } else { 2042 __ b(true_label, ARMCondition(true_high_cond)); 2043 __ b(false_label, ARMCondition(false_high_cond)); 2044 } 2045 // Must be equal high, so compare the lows. 2046 __ cmp(left_low, ShifterOperand(right_low)); 2047 } 2048 // The last comparison might be unsigned. 2049 // TODO: optimize cases where this is always true/false 2050 __ b(true_label, final_condition); 2051} 2052 2053void InstructionCodeGeneratorARM::GenerateCompareTestAndBranch(HCondition* condition, 2054 Label* true_target_in, 2055 Label* false_target_in) { 2056 // Generated branching requires both targets to be explicit. If either of the 2057 // targets is nullptr (fallthrough) use and bind `fallthrough_target` instead. 2058 Label fallthrough_target; 2059 Label* true_target = true_target_in == nullptr ? &fallthrough_target : true_target_in; 2060 Label* false_target = false_target_in == nullptr ? &fallthrough_target : false_target_in; 2061 2062 Primitive::Type type = condition->InputAt(0)->GetType(); 2063 switch (type) { 2064 case Primitive::kPrimLong: 2065 GenerateLongComparesAndJumps(condition, true_target, false_target); 2066 break; 2067 case Primitive::kPrimFloat: 2068 case Primitive::kPrimDouble: 2069 GenerateVcmp(condition); 2070 GenerateFPJumps(condition, true_target, false_target); 2071 break; 2072 default: 2073 LOG(FATAL) << "Unexpected compare type " << type; 2074 } 2075 2076 if (false_target != &fallthrough_target) { 2077 __ b(false_target); 2078 } 2079 2080 if (fallthrough_target.IsLinked()) { 2081 __ Bind(&fallthrough_target); 2082 } 2083} 2084 2085void InstructionCodeGeneratorARM::GenerateTestAndBranch(HInstruction* instruction, 2086 size_t condition_input_index, 2087 Label* true_target, 2088 Label* false_target) { 2089 HInstruction* cond = instruction->InputAt(condition_input_index); 2090 2091 if (true_target == nullptr && false_target == nullptr) { 2092 // Nothing to do. The code always falls through. 2093 return; 2094 } else if (cond->IsIntConstant()) { 2095 // Constant condition, statically compared against "true" (integer value 1). 2096 if (cond->AsIntConstant()->IsTrue()) { 2097 if (true_target != nullptr) { 2098 __ b(true_target); 2099 } 2100 } else { 2101 DCHECK(cond->AsIntConstant()->IsFalse()) << cond->AsIntConstant()->GetValue(); 2102 if (false_target != nullptr) { 2103 __ b(false_target); 2104 } 2105 } 2106 return; 2107 } 2108 2109 // The following code generates these patterns: 2110 // (1) true_target == nullptr && false_target != nullptr 2111 // - opposite condition true => branch to false_target 2112 // (2) true_target != nullptr && false_target == nullptr 2113 // - condition true => branch to true_target 2114 // (3) true_target != nullptr && false_target != nullptr 2115 // - condition true => branch to true_target 2116 // - branch to false_target 2117 if (IsBooleanValueOrMaterializedCondition(cond)) { 2118 // Condition has been materialized, compare the output to 0. 2119 Location cond_val = instruction->GetLocations()->InAt(condition_input_index); 2120 DCHECK(cond_val.IsRegister()); 2121 if (true_target == nullptr) { 2122 __ CompareAndBranchIfZero(cond_val.AsRegister<Register>(), false_target); 2123 } else { 2124 __ CompareAndBranchIfNonZero(cond_val.AsRegister<Register>(), true_target); 2125 } 2126 } else { 2127 // Condition has not been materialized. Use its inputs as the comparison and 2128 // its condition as the branch condition. 2129 HCondition* condition = cond->AsCondition(); 2130 2131 // If this is a long or FP comparison that has been folded into 2132 // the HCondition, generate the comparison directly. 2133 Primitive::Type type = condition->InputAt(0)->GetType(); 2134 if (type == Primitive::kPrimLong || Primitive::IsFloatingPointType(type)) { 2135 GenerateCompareTestAndBranch(condition, true_target, false_target); 2136 return; 2137 } 2138 2139 LocationSummary* locations = cond->GetLocations(); 2140 DCHECK(locations->InAt(0).IsRegister()); 2141 Register left = locations->InAt(0).AsRegister<Register>(); 2142 Location right = locations->InAt(1); 2143 if (right.IsRegister()) { 2144 __ cmp(left, ShifterOperand(right.AsRegister<Register>())); 2145 } else { 2146 DCHECK(right.IsConstant()); 2147 __ CmpConstant(left, CodeGenerator::GetInt32ValueOf(right.GetConstant())); 2148 } 2149 if (true_target == nullptr) { 2150 __ b(false_target, ARMCondition(condition->GetOppositeCondition())); 2151 } else { 2152 __ b(true_target, ARMCondition(condition->GetCondition())); 2153 } 2154 } 2155 2156 // If neither branch falls through (case 3), the conditional branch to `true_target` 2157 // was already emitted (case 2) and we need to emit a jump to `false_target`. 2158 if (true_target != nullptr && false_target != nullptr) { 2159 __ b(false_target); 2160 } 2161} 2162 2163void LocationsBuilderARM::VisitIf(HIf* if_instr) { 2164 LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(if_instr); 2165 if (IsBooleanValueOrMaterializedCondition(if_instr->InputAt(0))) { 2166 locations->SetInAt(0, Location::RequiresRegister()); 2167 } 2168} 2169 2170void InstructionCodeGeneratorARM::VisitIf(HIf* if_instr) { 2171 HBasicBlock* true_successor = if_instr->IfTrueSuccessor(); 2172 HBasicBlock* false_successor = if_instr->IfFalseSuccessor(); 2173 Label* true_target = codegen_->GoesToNextBlock(if_instr->GetBlock(), true_successor) ? 2174 nullptr : codegen_->GetLabelOf(true_successor); 2175 Label* false_target = codegen_->GoesToNextBlock(if_instr->GetBlock(), false_successor) ? 2176 nullptr : codegen_->GetLabelOf(false_successor); 2177 GenerateTestAndBranch(if_instr, /* condition_input_index */ 0, true_target, false_target); 2178} 2179 2180void LocationsBuilderARM::VisitDeoptimize(HDeoptimize* deoptimize) { 2181 LocationSummary* locations = new (GetGraph()->GetArena()) 2182 LocationSummary(deoptimize, LocationSummary::kCallOnSlowPath); 2183 locations->SetCustomSlowPathCallerSaves(RegisterSet::Empty()); // No caller-save registers. 2184 if (IsBooleanValueOrMaterializedCondition(deoptimize->InputAt(0))) { 2185 locations->SetInAt(0, Location::RequiresRegister()); 2186 } 2187} 2188 2189void InstructionCodeGeneratorARM::VisitDeoptimize(HDeoptimize* deoptimize) { 2190 SlowPathCodeARM* slow_path = deopt_slow_paths_.NewSlowPath<DeoptimizationSlowPathARM>(deoptimize); 2191 GenerateTestAndBranch(deoptimize, 2192 /* condition_input_index */ 0, 2193 slow_path->GetEntryLabel(), 2194 /* false_target */ nullptr); 2195} 2196 2197void LocationsBuilderARM::VisitShouldDeoptimizeFlag(HShouldDeoptimizeFlag* flag) { 2198 LocationSummary* locations = new (GetGraph()->GetArena()) 2199 LocationSummary(flag, LocationSummary::kNoCall); 2200 locations->SetOut(Location::RequiresRegister()); 2201} 2202 2203void InstructionCodeGeneratorARM::VisitShouldDeoptimizeFlag(HShouldDeoptimizeFlag* flag) { 2204 __ LoadFromOffset(kLoadWord, 2205 flag->GetLocations()->Out().AsRegister<Register>(), 2206 SP, 2207 codegen_->GetStackOffsetOfShouldDeoptimizeFlag()); 2208} 2209 2210void LocationsBuilderARM::VisitSelect(HSelect* select) { 2211 LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(select); 2212 if (Primitive::IsFloatingPointType(select->GetType())) { 2213 locations->SetInAt(0, Location::RequiresFpuRegister()); 2214 locations->SetInAt(1, Location::RequiresFpuRegister()); 2215 } else { 2216 locations->SetInAt(0, Location::RequiresRegister()); 2217 locations->SetInAt(1, Location::RequiresRegister()); 2218 } 2219 if (IsBooleanValueOrMaterializedCondition(select->GetCondition())) { 2220 locations->SetInAt(2, Location::RequiresRegister()); 2221 } 2222 locations->SetOut(Location::SameAsFirstInput()); 2223} 2224 2225void InstructionCodeGeneratorARM::VisitSelect(HSelect* select) { 2226 LocationSummary* locations = select->GetLocations(); 2227 Label false_target; 2228 GenerateTestAndBranch(select, 2229 /* condition_input_index */ 2, 2230 /* true_target */ nullptr, 2231 &false_target); 2232 codegen_->MoveLocation(locations->Out(), locations->InAt(1), select->GetType()); 2233 __ Bind(&false_target); 2234} 2235 2236void LocationsBuilderARM::VisitNativeDebugInfo(HNativeDebugInfo* info) { 2237 new (GetGraph()->GetArena()) LocationSummary(info); 2238} 2239 2240void InstructionCodeGeneratorARM::VisitNativeDebugInfo(HNativeDebugInfo*) { 2241 // MaybeRecordNativeDebugInfo is already called implicitly in CodeGenerator::Compile. 2242} 2243 2244void CodeGeneratorARM::GenerateNop() { 2245 __ nop(); 2246} 2247 2248void LocationsBuilderARM::HandleCondition(HCondition* cond) { 2249 LocationSummary* locations = 2250 new (GetGraph()->GetArena()) LocationSummary(cond, LocationSummary::kNoCall); 2251 // Handle the long/FP comparisons made in instruction simplification. 2252 switch (cond->InputAt(0)->GetType()) { 2253 case Primitive::kPrimLong: 2254 locations->SetInAt(0, Location::RequiresRegister()); 2255 locations->SetInAt(1, Location::RegisterOrConstant(cond->InputAt(1))); 2256 if (!cond->IsEmittedAtUseSite()) { 2257 locations->SetOut(Location::RequiresRegister(), Location::kOutputOverlap); 2258 } 2259 break; 2260 2261 case Primitive::kPrimFloat: 2262 case Primitive::kPrimDouble: 2263 locations->SetInAt(0, Location::RequiresFpuRegister()); 2264 locations->SetInAt(1, ArithmeticZeroOrFpuRegister(cond->InputAt(1))); 2265 if (!cond->IsEmittedAtUseSite()) { 2266 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 2267 } 2268 break; 2269 2270 default: 2271 locations->SetInAt(0, Location::RequiresRegister()); 2272 locations->SetInAt(1, Location::RegisterOrConstant(cond->InputAt(1))); 2273 if (!cond->IsEmittedAtUseSite()) { 2274 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 2275 } 2276 } 2277} 2278 2279void InstructionCodeGeneratorARM::HandleCondition(HCondition* cond) { 2280 if (cond->IsEmittedAtUseSite()) { 2281 return; 2282 } 2283 2284 LocationSummary* locations = cond->GetLocations(); 2285 Location left = locations->InAt(0); 2286 Location right = locations->InAt(1); 2287 Register out = locations->Out().AsRegister<Register>(); 2288 Label true_label, false_label; 2289 2290 switch (cond->InputAt(0)->GetType()) { 2291 default: { 2292 // Integer case. 2293 if (right.IsRegister()) { 2294 __ cmp(left.AsRegister<Register>(), ShifterOperand(right.AsRegister<Register>())); 2295 } else { 2296 DCHECK(right.IsConstant()); 2297 __ CmpConstant(left.AsRegister<Register>(), 2298 CodeGenerator::GetInt32ValueOf(right.GetConstant())); 2299 } 2300 __ it(ARMCondition(cond->GetCondition()), kItElse); 2301 __ mov(locations->Out().AsRegister<Register>(), ShifterOperand(1), 2302 ARMCondition(cond->GetCondition())); 2303 __ mov(locations->Out().AsRegister<Register>(), ShifterOperand(0), 2304 ARMCondition(cond->GetOppositeCondition())); 2305 return; 2306 } 2307 case Primitive::kPrimLong: 2308 GenerateLongComparesAndJumps(cond, &true_label, &false_label); 2309 break; 2310 case Primitive::kPrimFloat: 2311 case Primitive::kPrimDouble: 2312 GenerateVcmp(cond); 2313 GenerateFPJumps(cond, &true_label, &false_label); 2314 break; 2315 } 2316 2317 // Convert the jumps into the result. 2318 Label done_label; 2319 2320 // False case: result = 0. 2321 __ Bind(&false_label); 2322 __ LoadImmediate(out, 0); 2323 __ b(&done_label); 2324 2325 // True case: result = 1. 2326 __ Bind(&true_label); 2327 __ LoadImmediate(out, 1); 2328 __ Bind(&done_label); 2329} 2330 2331void LocationsBuilderARM::VisitEqual(HEqual* comp) { 2332 HandleCondition(comp); 2333} 2334 2335void InstructionCodeGeneratorARM::VisitEqual(HEqual* comp) { 2336 HandleCondition(comp); 2337} 2338 2339void LocationsBuilderARM::VisitNotEqual(HNotEqual* comp) { 2340 HandleCondition(comp); 2341} 2342 2343void InstructionCodeGeneratorARM::VisitNotEqual(HNotEqual* comp) { 2344 HandleCondition(comp); 2345} 2346 2347void LocationsBuilderARM::VisitLessThan(HLessThan* comp) { 2348 HandleCondition(comp); 2349} 2350 2351void InstructionCodeGeneratorARM::VisitLessThan(HLessThan* comp) { 2352 HandleCondition(comp); 2353} 2354 2355void LocationsBuilderARM::VisitLessThanOrEqual(HLessThanOrEqual* comp) { 2356 HandleCondition(comp); 2357} 2358 2359void InstructionCodeGeneratorARM::VisitLessThanOrEqual(HLessThanOrEqual* comp) { 2360 HandleCondition(comp); 2361} 2362 2363void LocationsBuilderARM::VisitGreaterThan(HGreaterThan* comp) { 2364 HandleCondition(comp); 2365} 2366 2367void InstructionCodeGeneratorARM::VisitGreaterThan(HGreaterThan* comp) { 2368 HandleCondition(comp); 2369} 2370 2371void LocationsBuilderARM::VisitGreaterThanOrEqual(HGreaterThanOrEqual* comp) { 2372 HandleCondition(comp); 2373} 2374 2375void InstructionCodeGeneratorARM::VisitGreaterThanOrEqual(HGreaterThanOrEqual* comp) { 2376 HandleCondition(comp); 2377} 2378 2379void LocationsBuilderARM::VisitBelow(HBelow* comp) { 2380 HandleCondition(comp); 2381} 2382 2383void InstructionCodeGeneratorARM::VisitBelow(HBelow* comp) { 2384 HandleCondition(comp); 2385} 2386 2387void LocationsBuilderARM::VisitBelowOrEqual(HBelowOrEqual* comp) { 2388 HandleCondition(comp); 2389} 2390 2391void InstructionCodeGeneratorARM::VisitBelowOrEqual(HBelowOrEqual* comp) { 2392 HandleCondition(comp); 2393} 2394 2395void LocationsBuilderARM::VisitAbove(HAbove* comp) { 2396 HandleCondition(comp); 2397} 2398 2399void InstructionCodeGeneratorARM::VisitAbove(HAbove* comp) { 2400 HandleCondition(comp); 2401} 2402 2403void LocationsBuilderARM::VisitAboveOrEqual(HAboveOrEqual* comp) { 2404 HandleCondition(comp); 2405} 2406 2407void InstructionCodeGeneratorARM::VisitAboveOrEqual(HAboveOrEqual* comp) { 2408 HandleCondition(comp); 2409} 2410 2411void LocationsBuilderARM::VisitIntConstant(HIntConstant* constant) { 2412 LocationSummary* locations = 2413 new (GetGraph()->GetArena()) LocationSummary(constant, LocationSummary::kNoCall); 2414 locations->SetOut(Location::ConstantLocation(constant)); 2415} 2416 2417void InstructionCodeGeneratorARM::VisitIntConstant(HIntConstant* constant ATTRIBUTE_UNUSED) { 2418 // Will be generated at use site. 2419} 2420 2421void LocationsBuilderARM::VisitNullConstant(HNullConstant* constant) { 2422 LocationSummary* locations = 2423 new (GetGraph()->GetArena()) LocationSummary(constant, LocationSummary::kNoCall); 2424 locations->SetOut(Location::ConstantLocation(constant)); 2425} 2426 2427void InstructionCodeGeneratorARM::VisitNullConstant(HNullConstant* constant ATTRIBUTE_UNUSED) { 2428 // Will be generated at use site. 2429} 2430 2431void LocationsBuilderARM::VisitLongConstant(HLongConstant* constant) { 2432 LocationSummary* locations = 2433 new (GetGraph()->GetArena()) LocationSummary(constant, LocationSummary::kNoCall); 2434 locations->SetOut(Location::ConstantLocation(constant)); 2435} 2436 2437void InstructionCodeGeneratorARM::VisitLongConstant(HLongConstant* constant ATTRIBUTE_UNUSED) { 2438 // Will be generated at use site. 2439} 2440 2441void LocationsBuilderARM::VisitFloatConstant(HFloatConstant* constant) { 2442 LocationSummary* locations = 2443 new (GetGraph()->GetArena()) LocationSummary(constant, LocationSummary::kNoCall); 2444 locations->SetOut(Location::ConstantLocation(constant)); 2445} 2446 2447void InstructionCodeGeneratorARM::VisitFloatConstant(HFloatConstant* constant ATTRIBUTE_UNUSED) { 2448 // Will be generated at use site. 2449} 2450 2451void LocationsBuilderARM::VisitDoubleConstant(HDoubleConstant* constant) { 2452 LocationSummary* locations = 2453 new (GetGraph()->GetArena()) LocationSummary(constant, LocationSummary::kNoCall); 2454 locations->SetOut(Location::ConstantLocation(constant)); 2455} 2456 2457void InstructionCodeGeneratorARM::VisitDoubleConstant(HDoubleConstant* constant ATTRIBUTE_UNUSED) { 2458 // Will be generated at use site. 2459} 2460 2461void LocationsBuilderARM::VisitMemoryBarrier(HMemoryBarrier* memory_barrier) { 2462 memory_barrier->SetLocations(nullptr); 2463} 2464 2465void InstructionCodeGeneratorARM::VisitMemoryBarrier(HMemoryBarrier* memory_barrier) { 2466 codegen_->GenerateMemoryBarrier(memory_barrier->GetBarrierKind()); 2467} 2468 2469void LocationsBuilderARM::VisitReturnVoid(HReturnVoid* ret) { 2470 ret->SetLocations(nullptr); 2471} 2472 2473void InstructionCodeGeneratorARM::VisitReturnVoid(HReturnVoid* ret ATTRIBUTE_UNUSED) { 2474 codegen_->GenerateFrameExit(); 2475} 2476 2477void LocationsBuilderARM::VisitReturn(HReturn* ret) { 2478 LocationSummary* locations = 2479 new (GetGraph()->GetArena()) LocationSummary(ret, LocationSummary::kNoCall); 2480 locations->SetInAt(0, parameter_visitor_.GetReturnLocation(ret->InputAt(0)->GetType())); 2481} 2482 2483void InstructionCodeGeneratorARM::VisitReturn(HReturn* ret ATTRIBUTE_UNUSED) { 2484 codegen_->GenerateFrameExit(); 2485} 2486 2487void LocationsBuilderARM::VisitInvokeUnresolved(HInvokeUnresolved* invoke) { 2488 // The trampoline uses the same calling convention as dex calling conventions, 2489 // except instead of loading arg0/r0 with the target Method*, arg0/r0 will contain 2490 // the method_idx. 2491 HandleInvoke(invoke); 2492} 2493 2494void InstructionCodeGeneratorARM::VisitInvokeUnresolved(HInvokeUnresolved* invoke) { 2495 codegen_->GenerateInvokeUnresolvedRuntimeCall(invoke); 2496} 2497 2498void LocationsBuilderARM::VisitInvokeStaticOrDirect(HInvokeStaticOrDirect* invoke) { 2499 // Explicit clinit checks triggered by static invokes must have been pruned by 2500 // art::PrepareForRegisterAllocation. 2501 DCHECK(!invoke->IsStaticWithExplicitClinitCheck()); 2502 2503 IntrinsicLocationsBuilderARM intrinsic(codegen_); 2504 if (intrinsic.TryDispatch(invoke)) { 2505 if (invoke->GetLocations()->CanCall() && invoke->HasPcRelativeDexCache()) { 2506 invoke->GetLocations()->SetInAt(invoke->GetSpecialInputIndex(), Location::Any()); 2507 } 2508 return; 2509 } 2510 2511 HandleInvoke(invoke); 2512 2513 // For PC-relative dex cache the invoke has an extra input, the PC-relative address base. 2514 if (invoke->HasPcRelativeDexCache()) { 2515 invoke->GetLocations()->SetInAt(invoke->GetSpecialInputIndex(), Location::RequiresRegister()); 2516 } 2517} 2518 2519static bool TryGenerateIntrinsicCode(HInvoke* invoke, CodeGeneratorARM* codegen) { 2520 if (invoke->GetLocations()->Intrinsified()) { 2521 IntrinsicCodeGeneratorARM intrinsic(codegen); 2522 intrinsic.Dispatch(invoke); 2523 return true; 2524 } 2525 return false; 2526} 2527 2528void InstructionCodeGeneratorARM::VisitInvokeStaticOrDirect(HInvokeStaticOrDirect* invoke) { 2529 // Explicit clinit checks triggered by static invokes must have been pruned by 2530 // art::PrepareForRegisterAllocation. 2531 DCHECK(!invoke->IsStaticWithExplicitClinitCheck()); 2532 2533 if (TryGenerateIntrinsicCode(invoke, codegen_)) { 2534 return; 2535 } 2536 2537 LocationSummary* locations = invoke->GetLocations(); 2538 codegen_->GenerateStaticOrDirectCall( 2539 invoke, locations->HasTemps() ? locations->GetTemp(0) : Location::NoLocation()); 2540 codegen_->RecordPcInfo(invoke, invoke->GetDexPc()); 2541} 2542 2543void LocationsBuilderARM::HandleInvoke(HInvoke* invoke) { 2544 InvokeDexCallingConventionVisitorARM calling_convention_visitor; 2545 CodeGenerator::CreateCommonInvokeLocationSummary(invoke, &calling_convention_visitor); 2546} 2547 2548void LocationsBuilderARM::VisitInvokeVirtual(HInvokeVirtual* invoke) { 2549 IntrinsicLocationsBuilderARM intrinsic(codegen_); 2550 if (intrinsic.TryDispatch(invoke)) { 2551 return; 2552 } 2553 2554 HandleInvoke(invoke); 2555} 2556 2557void InstructionCodeGeneratorARM::VisitInvokeVirtual(HInvokeVirtual* invoke) { 2558 if (TryGenerateIntrinsicCode(invoke, codegen_)) { 2559 return; 2560 } 2561 2562 codegen_->GenerateVirtualCall(invoke, invoke->GetLocations()->GetTemp(0)); 2563 DCHECK(!codegen_->IsLeafMethod()); 2564 codegen_->RecordPcInfo(invoke, invoke->GetDexPc()); 2565} 2566 2567void LocationsBuilderARM::VisitInvokeInterface(HInvokeInterface* invoke) { 2568 HandleInvoke(invoke); 2569 // Add the hidden argument. 2570 invoke->GetLocations()->AddTemp(Location::RegisterLocation(R12)); 2571} 2572 2573void InstructionCodeGeneratorARM::VisitInvokeInterface(HInvokeInterface* invoke) { 2574 // TODO: b/18116999, our IMTs can miss an IncompatibleClassChangeError. 2575 LocationSummary* locations = invoke->GetLocations(); 2576 Register temp = locations->GetTemp(0).AsRegister<Register>(); 2577 Register hidden_reg = locations->GetTemp(1).AsRegister<Register>(); 2578 Location receiver = locations->InAt(0); 2579 uint32_t class_offset = mirror::Object::ClassOffset().Int32Value(); 2580 2581 // Set the hidden argument. This is safe to do this here, as R12 2582 // won't be modified thereafter, before the `blx` (call) instruction. 2583 DCHECK_EQ(R12, hidden_reg); 2584 __ LoadImmediate(hidden_reg, invoke->GetDexMethodIndex()); 2585 2586 if (receiver.IsStackSlot()) { 2587 __ LoadFromOffset(kLoadWord, temp, SP, receiver.GetStackIndex()); 2588 // /* HeapReference<Class> */ temp = temp->klass_ 2589 __ LoadFromOffset(kLoadWord, temp, temp, class_offset); 2590 } else { 2591 // /* HeapReference<Class> */ temp = receiver->klass_ 2592 __ LoadFromOffset(kLoadWord, temp, receiver.AsRegister<Register>(), class_offset); 2593 } 2594 codegen_->MaybeRecordImplicitNullCheck(invoke); 2595 // Instead of simply (possibly) unpoisoning `temp` here, we should 2596 // emit a read barrier for the previous class reference load. 2597 // However this is not required in practice, as this is an 2598 // intermediate/temporary reference and because the current 2599 // concurrent copying collector keeps the from-space memory 2600 // intact/accessible until the end of the marking phase (the 2601 // concurrent copying collector may not in the future). 2602 __ MaybeUnpoisonHeapReference(temp); 2603 __ LoadFromOffset(kLoadWord, temp, temp, 2604 mirror::Class::ImtPtrOffset(kArmPointerSize).Uint32Value()); 2605 uint32_t method_offset = static_cast<uint32_t>(ImTable::OffsetOfElement( 2606 invoke->GetImtIndex(), kArmPointerSize)); 2607 // temp = temp->GetImtEntryAt(method_offset); 2608 __ LoadFromOffset(kLoadWord, temp, temp, method_offset); 2609 uint32_t entry_point = 2610 ArtMethod::EntryPointFromQuickCompiledCodeOffset(kArmPointerSize).Int32Value(); 2611 // LR = temp->GetEntryPoint(); 2612 __ LoadFromOffset(kLoadWord, LR, temp, entry_point); 2613 // LR(); 2614 __ blx(LR); 2615 DCHECK(!codegen_->IsLeafMethod()); 2616 codegen_->RecordPcInfo(invoke, invoke->GetDexPc()); 2617} 2618 2619void LocationsBuilderARM::VisitInvokePolymorphic(HInvokePolymorphic* invoke) { 2620 HandleInvoke(invoke); 2621} 2622 2623void InstructionCodeGeneratorARM::VisitInvokePolymorphic(HInvokePolymorphic* invoke) { 2624 codegen_->GenerateInvokePolymorphicCall(invoke); 2625} 2626 2627void LocationsBuilderARM::VisitNeg(HNeg* neg) { 2628 LocationSummary* locations = 2629 new (GetGraph()->GetArena()) LocationSummary(neg, LocationSummary::kNoCall); 2630 switch (neg->GetResultType()) { 2631 case Primitive::kPrimInt: { 2632 locations->SetInAt(0, Location::RequiresRegister()); 2633 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 2634 break; 2635 } 2636 case Primitive::kPrimLong: { 2637 locations->SetInAt(0, Location::RequiresRegister()); 2638 locations->SetOut(Location::RequiresRegister(), Location::kOutputOverlap); 2639 break; 2640 } 2641 2642 case Primitive::kPrimFloat: 2643 case Primitive::kPrimDouble: 2644 locations->SetInAt(0, Location::RequiresFpuRegister()); 2645 locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap); 2646 break; 2647 2648 default: 2649 LOG(FATAL) << "Unexpected neg type " << neg->GetResultType(); 2650 } 2651} 2652 2653void InstructionCodeGeneratorARM::VisitNeg(HNeg* neg) { 2654 LocationSummary* locations = neg->GetLocations(); 2655 Location out = locations->Out(); 2656 Location in = locations->InAt(0); 2657 switch (neg->GetResultType()) { 2658 case Primitive::kPrimInt: 2659 DCHECK(in.IsRegister()); 2660 __ rsb(out.AsRegister<Register>(), in.AsRegister<Register>(), ShifterOperand(0)); 2661 break; 2662 2663 case Primitive::kPrimLong: 2664 DCHECK(in.IsRegisterPair()); 2665 // out.lo = 0 - in.lo (and update the carry/borrow (C) flag) 2666 __ rsbs(out.AsRegisterPairLow<Register>(), 2667 in.AsRegisterPairLow<Register>(), 2668 ShifterOperand(0)); 2669 // We cannot emit an RSC (Reverse Subtract with Carry) 2670 // instruction here, as it does not exist in the Thumb-2 2671 // instruction set. We use the following approach 2672 // using SBC and SUB instead. 2673 // 2674 // out.hi = -C 2675 __ sbc(out.AsRegisterPairHigh<Register>(), 2676 out.AsRegisterPairHigh<Register>(), 2677 ShifterOperand(out.AsRegisterPairHigh<Register>())); 2678 // out.hi = out.hi - in.hi 2679 __ sub(out.AsRegisterPairHigh<Register>(), 2680 out.AsRegisterPairHigh<Register>(), 2681 ShifterOperand(in.AsRegisterPairHigh<Register>())); 2682 break; 2683 2684 case Primitive::kPrimFloat: 2685 DCHECK(in.IsFpuRegister()); 2686 __ vnegs(out.AsFpuRegister<SRegister>(), in.AsFpuRegister<SRegister>()); 2687 break; 2688 2689 case Primitive::kPrimDouble: 2690 DCHECK(in.IsFpuRegisterPair()); 2691 __ vnegd(FromLowSToD(out.AsFpuRegisterPairLow<SRegister>()), 2692 FromLowSToD(in.AsFpuRegisterPairLow<SRegister>())); 2693 break; 2694 2695 default: 2696 LOG(FATAL) << "Unexpected neg type " << neg->GetResultType(); 2697 } 2698} 2699 2700void LocationsBuilderARM::VisitTypeConversion(HTypeConversion* conversion) { 2701 Primitive::Type result_type = conversion->GetResultType(); 2702 Primitive::Type input_type = conversion->GetInputType(); 2703 DCHECK_NE(result_type, input_type); 2704 2705 // The float-to-long, double-to-long and long-to-float type conversions 2706 // rely on a call to the runtime. 2707 LocationSummary::CallKind call_kind = 2708 (((input_type == Primitive::kPrimFloat || input_type == Primitive::kPrimDouble) 2709 && result_type == Primitive::kPrimLong) 2710 || (input_type == Primitive::kPrimLong && result_type == Primitive::kPrimFloat)) 2711 ? LocationSummary::kCallOnMainOnly 2712 : LocationSummary::kNoCall; 2713 LocationSummary* locations = 2714 new (GetGraph()->GetArena()) LocationSummary(conversion, call_kind); 2715 2716 // The Java language does not allow treating boolean as an integral type but 2717 // our bit representation makes it safe. 2718 2719 switch (result_type) { 2720 case Primitive::kPrimByte: 2721 switch (input_type) { 2722 case Primitive::kPrimLong: 2723 // Type conversion from long to byte is a result of code transformations. 2724 case Primitive::kPrimBoolean: 2725 // Boolean input is a result of code transformations. 2726 case Primitive::kPrimShort: 2727 case Primitive::kPrimInt: 2728 case Primitive::kPrimChar: 2729 // Processing a Dex `int-to-byte' instruction. 2730 locations->SetInAt(0, Location::RequiresRegister()); 2731 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 2732 break; 2733 2734 default: 2735 LOG(FATAL) << "Unexpected type conversion from " << input_type 2736 << " to " << result_type; 2737 } 2738 break; 2739 2740 case Primitive::kPrimShort: 2741 switch (input_type) { 2742 case Primitive::kPrimLong: 2743 // Type conversion from long to short is a result of code transformations. 2744 case Primitive::kPrimBoolean: 2745 // Boolean input is a result of code transformations. 2746 case Primitive::kPrimByte: 2747 case Primitive::kPrimInt: 2748 case Primitive::kPrimChar: 2749 // Processing a Dex `int-to-short' instruction. 2750 locations->SetInAt(0, Location::RequiresRegister()); 2751 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 2752 break; 2753 2754 default: 2755 LOG(FATAL) << "Unexpected type conversion from " << input_type 2756 << " to " << result_type; 2757 } 2758 break; 2759 2760 case Primitive::kPrimInt: 2761 switch (input_type) { 2762 case Primitive::kPrimLong: 2763 // Processing a Dex `long-to-int' instruction. 2764 locations->SetInAt(0, Location::Any()); 2765 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 2766 break; 2767 2768 case Primitive::kPrimFloat: 2769 // Processing a Dex `float-to-int' instruction. 2770 locations->SetInAt(0, Location::RequiresFpuRegister()); 2771 locations->SetOut(Location::RequiresRegister()); 2772 locations->AddTemp(Location::RequiresFpuRegister()); 2773 break; 2774 2775 case Primitive::kPrimDouble: 2776 // Processing a Dex `double-to-int' instruction. 2777 locations->SetInAt(0, Location::RequiresFpuRegister()); 2778 locations->SetOut(Location::RequiresRegister()); 2779 locations->AddTemp(Location::RequiresFpuRegister()); 2780 break; 2781 2782 default: 2783 LOG(FATAL) << "Unexpected type conversion from " << input_type 2784 << " to " << result_type; 2785 } 2786 break; 2787 2788 case Primitive::kPrimLong: 2789 switch (input_type) { 2790 case Primitive::kPrimBoolean: 2791 // Boolean input is a result of code transformations. 2792 case Primitive::kPrimByte: 2793 case Primitive::kPrimShort: 2794 case Primitive::kPrimInt: 2795 case Primitive::kPrimChar: 2796 // Processing a Dex `int-to-long' instruction. 2797 locations->SetInAt(0, Location::RequiresRegister()); 2798 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 2799 break; 2800 2801 case Primitive::kPrimFloat: { 2802 // Processing a Dex `float-to-long' instruction. 2803 InvokeRuntimeCallingConvention calling_convention; 2804 locations->SetInAt(0, Location::FpuRegisterLocation( 2805 calling_convention.GetFpuRegisterAt(0))); 2806 locations->SetOut(Location::RegisterPairLocation(R0, R1)); 2807 break; 2808 } 2809 2810 case Primitive::kPrimDouble: { 2811 // Processing a Dex `double-to-long' instruction. 2812 InvokeRuntimeCallingConvention calling_convention; 2813 locations->SetInAt(0, Location::FpuRegisterPairLocation( 2814 calling_convention.GetFpuRegisterAt(0), 2815 calling_convention.GetFpuRegisterAt(1))); 2816 locations->SetOut(Location::RegisterPairLocation(R0, R1)); 2817 break; 2818 } 2819 2820 default: 2821 LOG(FATAL) << "Unexpected type conversion from " << input_type 2822 << " to " << result_type; 2823 } 2824 break; 2825 2826 case Primitive::kPrimChar: 2827 switch (input_type) { 2828 case Primitive::kPrimLong: 2829 // Type conversion from long to char is a result of code transformations. 2830 case Primitive::kPrimBoolean: 2831 // Boolean input is a result of code transformations. 2832 case Primitive::kPrimByte: 2833 case Primitive::kPrimShort: 2834 case Primitive::kPrimInt: 2835 // Processing a Dex `int-to-char' instruction. 2836 locations->SetInAt(0, Location::RequiresRegister()); 2837 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 2838 break; 2839 2840 default: 2841 LOG(FATAL) << "Unexpected type conversion from " << input_type 2842 << " to " << result_type; 2843 } 2844 break; 2845 2846 case Primitive::kPrimFloat: 2847 switch (input_type) { 2848 case Primitive::kPrimBoolean: 2849 // Boolean input is a result of code transformations. 2850 case Primitive::kPrimByte: 2851 case Primitive::kPrimShort: 2852 case Primitive::kPrimInt: 2853 case Primitive::kPrimChar: 2854 // Processing a Dex `int-to-float' instruction. 2855 locations->SetInAt(0, Location::RequiresRegister()); 2856 locations->SetOut(Location::RequiresFpuRegister()); 2857 break; 2858 2859 case Primitive::kPrimLong: { 2860 // Processing a Dex `long-to-float' instruction. 2861 InvokeRuntimeCallingConvention calling_convention; 2862 locations->SetInAt(0, Location::RegisterPairLocation( 2863 calling_convention.GetRegisterAt(0), calling_convention.GetRegisterAt(1))); 2864 locations->SetOut(Location::FpuRegisterLocation(calling_convention.GetFpuRegisterAt(0))); 2865 break; 2866 } 2867 2868 case Primitive::kPrimDouble: 2869 // Processing a Dex `double-to-float' instruction. 2870 locations->SetInAt(0, Location::RequiresFpuRegister()); 2871 locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap); 2872 break; 2873 2874 default: 2875 LOG(FATAL) << "Unexpected type conversion from " << input_type 2876 << " to " << result_type; 2877 }; 2878 break; 2879 2880 case Primitive::kPrimDouble: 2881 switch (input_type) { 2882 case Primitive::kPrimBoolean: 2883 // Boolean input is a result of code transformations. 2884 case Primitive::kPrimByte: 2885 case Primitive::kPrimShort: 2886 case Primitive::kPrimInt: 2887 case Primitive::kPrimChar: 2888 // Processing a Dex `int-to-double' instruction. 2889 locations->SetInAt(0, Location::RequiresRegister()); 2890 locations->SetOut(Location::RequiresFpuRegister()); 2891 break; 2892 2893 case Primitive::kPrimLong: 2894 // Processing a Dex `long-to-double' instruction. 2895 locations->SetInAt(0, Location::RequiresRegister()); 2896 locations->SetOut(Location::RequiresFpuRegister()); 2897 locations->AddTemp(Location::RequiresFpuRegister()); 2898 locations->AddTemp(Location::RequiresFpuRegister()); 2899 break; 2900 2901 case Primitive::kPrimFloat: 2902 // Processing a Dex `float-to-double' instruction. 2903 locations->SetInAt(0, Location::RequiresFpuRegister()); 2904 locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap); 2905 break; 2906 2907 default: 2908 LOG(FATAL) << "Unexpected type conversion from " << input_type 2909 << " to " << result_type; 2910 }; 2911 break; 2912 2913 default: 2914 LOG(FATAL) << "Unexpected type conversion from " << input_type 2915 << " to " << result_type; 2916 } 2917} 2918 2919void InstructionCodeGeneratorARM::VisitTypeConversion(HTypeConversion* conversion) { 2920 LocationSummary* locations = conversion->GetLocations(); 2921 Location out = locations->Out(); 2922 Location in = locations->InAt(0); 2923 Primitive::Type result_type = conversion->GetResultType(); 2924 Primitive::Type input_type = conversion->GetInputType(); 2925 DCHECK_NE(result_type, input_type); 2926 switch (result_type) { 2927 case Primitive::kPrimByte: 2928 switch (input_type) { 2929 case Primitive::kPrimLong: 2930 // Type conversion from long to byte is a result of code transformations. 2931 __ sbfx(out.AsRegister<Register>(), in.AsRegisterPairLow<Register>(), 0, 8); 2932 break; 2933 case Primitive::kPrimBoolean: 2934 // Boolean input is a result of code transformations. 2935 case Primitive::kPrimShort: 2936 case Primitive::kPrimInt: 2937 case Primitive::kPrimChar: 2938 // Processing a Dex `int-to-byte' instruction. 2939 __ sbfx(out.AsRegister<Register>(), in.AsRegister<Register>(), 0, 8); 2940 break; 2941 2942 default: 2943 LOG(FATAL) << "Unexpected type conversion from " << input_type 2944 << " to " << result_type; 2945 } 2946 break; 2947 2948 case Primitive::kPrimShort: 2949 switch (input_type) { 2950 case Primitive::kPrimLong: 2951 // Type conversion from long to short is a result of code transformations. 2952 __ sbfx(out.AsRegister<Register>(), in.AsRegisterPairLow<Register>(), 0, 16); 2953 break; 2954 case Primitive::kPrimBoolean: 2955 // Boolean input is a result of code transformations. 2956 case Primitive::kPrimByte: 2957 case Primitive::kPrimInt: 2958 case Primitive::kPrimChar: 2959 // Processing a Dex `int-to-short' instruction. 2960 __ sbfx(out.AsRegister<Register>(), in.AsRegister<Register>(), 0, 16); 2961 break; 2962 2963 default: 2964 LOG(FATAL) << "Unexpected type conversion from " << input_type 2965 << " to " << result_type; 2966 } 2967 break; 2968 2969 case Primitive::kPrimInt: 2970 switch (input_type) { 2971 case Primitive::kPrimLong: 2972 // Processing a Dex `long-to-int' instruction. 2973 DCHECK(out.IsRegister()); 2974 if (in.IsRegisterPair()) { 2975 __ Mov(out.AsRegister<Register>(), in.AsRegisterPairLow<Register>()); 2976 } else if (in.IsDoubleStackSlot()) { 2977 __ LoadFromOffset(kLoadWord, out.AsRegister<Register>(), SP, in.GetStackIndex()); 2978 } else { 2979 DCHECK(in.IsConstant()); 2980 DCHECK(in.GetConstant()->IsLongConstant()); 2981 int64_t value = in.GetConstant()->AsLongConstant()->GetValue(); 2982 __ LoadImmediate(out.AsRegister<Register>(), static_cast<int32_t>(value)); 2983 } 2984 break; 2985 2986 case Primitive::kPrimFloat: { 2987 // Processing a Dex `float-to-int' instruction. 2988 SRegister temp = locations->GetTemp(0).AsFpuRegisterPairLow<SRegister>(); 2989 __ vcvtis(temp, in.AsFpuRegister<SRegister>()); 2990 __ vmovrs(out.AsRegister<Register>(), temp); 2991 break; 2992 } 2993 2994 case Primitive::kPrimDouble: { 2995 // Processing a Dex `double-to-int' instruction. 2996 SRegister temp_s = locations->GetTemp(0).AsFpuRegisterPairLow<SRegister>(); 2997 __ vcvtid(temp_s, FromLowSToD(in.AsFpuRegisterPairLow<SRegister>())); 2998 __ vmovrs(out.AsRegister<Register>(), temp_s); 2999 break; 3000 } 3001 3002 default: 3003 LOG(FATAL) << "Unexpected type conversion from " << input_type 3004 << " to " << result_type; 3005 } 3006 break; 3007 3008 case Primitive::kPrimLong: 3009 switch (input_type) { 3010 case Primitive::kPrimBoolean: 3011 // Boolean input is a result of code transformations. 3012 case Primitive::kPrimByte: 3013 case Primitive::kPrimShort: 3014 case Primitive::kPrimInt: 3015 case Primitive::kPrimChar: 3016 // Processing a Dex `int-to-long' instruction. 3017 DCHECK(out.IsRegisterPair()); 3018 DCHECK(in.IsRegister()); 3019 __ Mov(out.AsRegisterPairLow<Register>(), in.AsRegister<Register>()); 3020 // Sign extension. 3021 __ Asr(out.AsRegisterPairHigh<Register>(), 3022 out.AsRegisterPairLow<Register>(), 3023 31); 3024 break; 3025 3026 case Primitive::kPrimFloat: 3027 // Processing a Dex `float-to-long' instruction. 3028 codegen_->InvokeRuntime(kQuickF2l, conversion, conversion->GetDexPc()); 3029 CheckEntrypointTypes<kQuickF2l, int64_t, float>(); 3030 break; 3031 3032 case Primitive::kPrimDouble: 3033 // Processing a Dex `double-to-long' instruction. 3034 codegen_->InvokeRuntime(kQuickD2l, conversion, conversion->GetDexPc()); 3035 CheckEntrypointTypes<kQuickD2l, int64_t, double>(); 3036 break; 3037 3038 default: 3039 LOG(FATAL) << "Unexpected type conversion from " << input_type 3040 << " to " << result_type; 3041 } 3042 break; 3043 3044 case Primitive::kPrimChar: 3045 switch (input_type) { 3046 case Primitive::kPrimLong: 3047 // Type conversion from long to char is a result of code transformations. 3048 __ ubfx(out.AsRegister<Register>(), in.AsRegisterPairLow<Register>(), 0, 16); 3049 break; 3050 case Primitive::kPrimBoolean: 3051 // Boolean input is a result of code transformations. 3052 case Primitive::kPrimByte: 3053 case Primitive::kPrimShort: 3054 case Primitive::kPrimInt: 3055 // Processing a Dex `int-to-char' instruction. 3056 __ ubfx(out.AsRegister<Register>(), in.AsRegister<Register>(), 0, 16); 3057 break; 3058 3059 default: 3060 LOG(FATAL) << "Unexpected type conversion from " << input_type 3061 << " to " << result_type; 3062 } 3063 break; 3064 3065 case Primitive::kPrimFloat: 3066 switch (input_type) { 3067 case Primitive::kPrimBoolean: 3068 // Boolean input is a result of code transformations. 3069 case Primitive::kPrimByte: 3070 case Primitive::kPrimShort: 3071 case Primitive::kPrimInt: 3072 case Primitive::kPrimChar: { 3073 // Processing a Dex `int-to-float' instruction. 3074 __ vmovsr(out.AsFpuRegister<SRegister>(), in.AsRegister<Register>()); 3075 __ vcvtsi(out.AsFpuRegister<SRegister>(), out.AsFpuRegister<SRegister>()); 3076 break; 3077 } 3078 3079 case Primitive::kPrimLong: 3080 // Processing a Dex `long-to-float' instruction. 3081 codegen_->InvokeRuntime(kQuickL2f, conversion, conversion->GetDexPc()); 3082 CheckEntrypointTypes<kQuickL2f, float, int64_t>(); 3083 break; 3084 3085 case Primitive::kPrimDouble: 3086 // Processing a Dex `double-to-float' instruction. 3087 __ vcvtsd(out.AsFpuRegister<SRegister>(), 3088 FromLowSToD(in.AsFpuRegisterPairLow<SRegister>())); 3089 break; 3090 3091 default: 3092 LOG(FATAL) << "Unexpected type conversion from " << input_type 3093 << " to " << result_type; 3094 }; 3095 break; 3096 3097 case Primitive::kPrimDouble: 3098 switch (input_type) { 3099 case Primitive::kPrimBoolean: 3100 // Boolean input is a result of code transformations. 3101 case Primitive::kPrimByte: 3102 case Primitive::kPrimShort: 3103 case Primitive::kPrimInt: 3104 case Primitive::kPrimChar: { 3105 // Processing a Dex `int-to-double' instruction. 3106 __ vmovsr(out.AsFpuRegisterPairLow<SRegister>(), in.AsRegister<Register>()); 3107 __ vcvtdi(FromLowSToD(out.AsFpuRegisterPairLow<SRegister>()), 3108 out.AsFpuRegisterPairLow<SRegister>()); 3109 break; 3110 } 3111 3112 case Primitive::kPrimLong: { 3113 // Processing a Dex `long-to-double' instruction. 3114 Register low = in.AsRegisterPairLow<Register>(); 3115 Register high = in.AsRegisterPairHigh<Register>(); 3116 SRegister out_s = out.AsFpuRegisterPairLow<SRegister>(); 3117 DRegister out_d = FromLowSToD(out_s); 3118 SRegister temp_s = locations->GetTemp(0).AsFpuRegisterPairLow<SRegister>(); 3119 DRegister temp_d = FromLowSToD(temp_s); 3120 SRegister constant_s = locations->GetTemp(1).AsFpuRegisterPairLow<SRegister>(); 3121 DRegister constant_d = FromLowSToD(constant_s); 3122 3123 // temp_d = int-to-double(high) 3124 __ vmovsr(temp_s, high); 3125 __ vcvtdi(temp_d, temp_s); 3126 // constant_d = k2Pow32EncodingForDouble 3127 __ LoadDImmediate(constant_d, bit_cast<double, int64_t>(k2Pow32EncodingForDouble)); 3128 // out_d = unsigned-to-double(low) 3129 __ vmovsr(out_s, low); 3130 __ vcvtdu(out_d, out_s); 3131 // out_d += temp_d * constant_d 3132 __ vmlad(out_d, temp_d, constant_d); 3133 break; 3134 } 3135 3136 case Primitive::kPrimFloat: 3137 // Processing a Dex `float-to-double' instruction. 3138 __ vcvtds(FromLowSToD(out.AsFpuRegisterPairLow<SRegister>()), 3139 in.AsFpuRegister<SRegister>()); 3140 break; 3141 3142 default: 3143 LOG(FATAL) << "Unexpected type conversion from " << input_type 3144 << " to " << result_type; 3145 }; 3146 break; 3147 3148 default: 3149 LOG(FATAL) << "Unexpected type conversion from " << input_type 3150 << " to " << result_type; 3151 } 3152} 3153 3154void LocationsBuilderARM::VisitAdd(HAdd* add) { 3155 LocationSummary* locations = 3156 new (GetGraph()->GetArena()) LocationSummary(add, LocationSummary::kNoCall); 3157 switch (add->GetResultType()) { 3158 case Primitive::kPrimInt: { 3159 locations->SetInAt(0, Location::RequiresRegister()); 3160 locations->SetInAt(1, Location::RegisterOrConstant(add->InputAt(1))); 3161 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 3162 break; 3163 } 3164 3165 case Primitive::kPrimLong: { 3166 locations->SetInAt(0, Location::RequiresRegister()); 3167 locations->SetInAt(1, ArmEncodableConstantOrRegister(add->InputAt(1), ADD)); 3168 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 3169 break; 3170 } 3171 3172 case Primitive::kPrimFloat: 3173 case Primitive::kPrimDouble: { 3174 locations->SetInAt(0, Location::RequiresFpuRegister()); 3175 locations->SetInAt(1, Location::RequiresFpuRegister()); 3176 locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap); 3177 break; 3178 } 3179 3180 default: 3181 LOG(FATAL) << "Unexpected add type " << add->GetResultType(); 3182 } 3183} 3184 3185void InstructionCodeGeneratorARM::VisitAdd(HAdd* add) { 3186 LocationSummary* locations = add->GetLocations(); 3187 Location out = locations->Out(); 3188 Location first = locations->InAt(0); 3189 Location second = locations->InAt(1); 3190 switch (add->GetResultType()) { 3191 case Primitive::kPrimInt: 3192 if (second.IsRegister()) { 3193 __ add(out.AsRegister<Register>(), 3194 first.AsRegister<Register>(), 3195 ShifterOperand(second.AsRegister<Register>())); 3196 } else { 3197 __ AddConstant(out.AsRegister<Register>(), 3198 first.AsRegister<Register>(), 3199 second.GetConstant()->AsIntConstant()->GetValue()); 3200 } 3201 break; 3202 3203 case Primitive::kPrimLong: { 3204 if (second.IsConstant()) { 3205 uint64_t value = static_cast<uint64_t>(Int64FromConstant(second.GetConstant())); 3206 GenerateAddLongConst(out, first, value); 3207 } else { 3208 DCHECK(second.IsRegisterPair()); 3209 __ adds(out.AsRegisterPairLow<Register>(), 3210 first.AsRegisterPairLow<Register>(), 3211 ShifterOperand(second.AsRegisterPairLow<Register>())); 3212 __ adc(out.AsRegisterPairHigh<Register>(), 3213 first.AsRegisterPairHigh<Register>(), 3214 ShifterOperand(second.AsRegisterPairHigh<Register>())); 3215 } 3216 break; 3217 } 3218 3219 case Primitive::kPrimFloat: 3220 __ vadds(out.AsFpuRegister<SRegister>(), 3221 first.AsFpuRegister<SRegister>(), 3222 second.AsFpuRegister<SRegister>()); 3223 break; 3224 3225 case Primitive::kPrimDouble: 3226 __ vaddd(FromLowSToD(out.AsFpuRegisterPairLow<SRegister>()), 3227 FromLowSToD(first.AsFpuRegisterPairLow<SRegister>()), 3228 FromLowSToD(second.AsFpuRegisterPairLow<SRegister>())); 3229 break; 3230 3231 default: 3232 LOG(FATAL) << "Unexpected add type " << add->GetResultType(); 3233 } 3234} 3235 3236void LocationsBuilderARM::VisitSub(HSub* sub) { 3237 LocationSummary* locations = 3238 new (GetGraph()->GetArena()) LocationSummary(sub, LocationSummary::kNoCall); 3239 switch (sub->GetResultType()) { 3240 case Primitive::kPrimInt: { 3241 locations->SetInAt(0, Location::RequiresRegister()); 3242 locations->SetInAt(1, Location::RegisterOrConstant(sub->InputAt(1))); 3243 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 3244 break; 3245 } 3246 3247 case Primitive::kPrimLong: { 3248 locations->SetInAt(0, Location::RequiresRegister()); 3249 locations->SetInAt(1, ArmEncodableConstantOrRegister(sub->InputAt(1), SUB)); 3250 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 3251 break; 3252 } 3253 case Primitive::kPrimFloat: 3254 case Primitive::kPrimDouble: { 3255 locations->SetInAt(0, Location::RequiresFpuRegister()); 3256 locations->SetInAt(1, Location::RequiresFpuRegister()); 3257 locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap); 3258 break; 3259 } 3260 default: 3261 LOG(FATAL) << "Unexpected sub type " << sub->GetResultType(); 3262 } 3263} 3264 3265void InstructionCodeGeneratorARM::VisitSub(HSub* sub) { 3266 LocationSummary* locations = sub->GetLocations(); 3267 Location out = locations->Out(); 3268 Location first = locations->InAt(0); 3269 Location second = locations->InAt(1); 3270 switch (sub->GetResultType()) { 3271 case Primitive::kPrimInt: { 3272 if (second.IsRegister()) { 3273 __ sub(out.AsRegister<Register>(), 3274 first.AsRegister<Register>(), 3275 ShifterOperand(second.AsRegister<Register>())); 3276 } else { 3277 __ AddConstant(out.AsRegister<Register>(), 3278 first.AsRegister<Register>(), 3279 -second.GetConstant()->AsIntConstant()->GetValue()); 3280 } 3281 break; 3282 } 3283 3284 case Primitive::kPrimLong: { 3285 if (second.IsConstant()) { 3286 uint64_t value = static_cast<uint64_t>(Int64FromConstant(second.GetConstant())); 3287 GenerateAddLongConst(out, first, -value); 3288 } else { 3289 DCHECK(second.IsRegisterPair()); 3290 __ subs(out.AsRegisterPairLow<Register>(), 3291 first.AsRegisterPairLow<Register>(), 3292 ShifterOperand(second.AsRegisterPairLow<Register>())); 3293 __ sbc(out.AsRegisterPairHigh<Register>(), 3294 first.AsRegisterPairHigh<Register>(), 3295 ShifterOperand(second.AsRegisterPairHigh<Register>())); 3296 } 3297 break; 3298 } 3299 3300 case Primitive::kPrimFloat: { 3301 __ vsubs(out.AsFpuRegister<SRegister>(), 3302 first.AsFpuRegister<SRegister>(), 3303 second.AsFpuRegister<SRegister>()); 3304 break; 3305 } 3306 3307 case Primitive::kPrimDouble: { 3308 __ vsubd(FromLowSToD(out.AsFpuRegisterPairLow<SRegister>()), 3309 FromLowSToD(first.AsFpuRegisterPairLow<SRegister>()), 3310 FromLowSToD(second.AsFpuRegisterPairLow<SRegister>())); 3311 break; 3312 } 3313 3314 3315 default: 3316 LOG(FATAL) << "Unexpected sub type " << sub->GetResultType(); 3317 } 3318} 3319 3320void LocationsBuilderARM::VisitMul(HMul* mul) { 3321 LocationSummary* locations = 3322 new (GetGraph()->GetArena()) LocationSummary(mul, LocationSummary::kNoCall); 3323 switch (mul->GetResultType()) { 3324 case Primitive::kPrimInt: 3325 case Primitive::kPrimLong: { 3326 locations->SetInAt(0, Location::RequiresRegister()); 3327 locations->SetInAt(1, Location::RequiresRegister()); 3328 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 3329 break; 3330 } 3331 3332 case Primitive::kPrimFloat: 3333 case Primitive::kPrimDouble: { 3334 locations->SetInAt(0, Location::RequiresFpuRegister()); 3335 locations->SetInAt(1, Location::RequiresFpuRegister()); 3336 locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap); 3337 break; 3338 } 3339 3340 default: 3341 LOG(FATAL) << "Unexpected mul type " << mul->GetResultType(); 3342 } 3343} 3344 3345void InstructionCodeGeneratorARM::VisitMul(HMul* mul) { 3346 LocationSummary* locations = mul->GetLocations(); 3347 Location out = locations->Out(); 3348 Location first = locations->InAt(0); 3349 Location second = locations->InAt(1); 3350 switch (mul->GetResultType()) { 3351 case Primitive::kPrimInt: { 3352 __ mul(out.AsRegister<Register>(), 3353 first.AsRegister<Register>(), 3354 second.AsRegister<Register>()); 3355 break; 3356 } 3357 case Primitive::kPrimLong: { 3358 Register out_hi = out.AsRegisterPairHigh<Register>(); 3359 Register out_lo = out.AsRegisterPairLow<Register>(); 3360 Register in1_hi = first.AsRegisterPairHigh<Register>(); 3361 Register in1_lo = first.AsRegisterPairLow<Register>(); 3362 Register in2_hi = second.AsRegisterPairHigh<Register>(); 3363 Register in2_lo = second.AsRegisterPairLow<Register>(); 3364 3365 // Extra checks to protect caused by the existence of R1_R2. 3366 // The algorithm is wrong if out.hi is either in1.lo or in2.lo: 3367 // (e.g. in1=r0_r1, in2=r2_r3 and out=r1_r2); 3368 DCHECK_NE(out_hi, in1_lo); 3369 DCHECK_NE(out_hi, in2_lo); 3370 3371 // input: in1 - 64 bits, in2 - 64 bits 3372 // output: out 3373 // formula: out.hi : out.lo = (in1.lo * in2.hi + in1.hi * in2.lo)* 2^32 + in1.lo * in2.lo 3374 // parts: out.hi = in1.lo * in2.hi + in1.hi * in2.lo + (in1.lo * in2.lo)[63:32] 3375 // parts: out.lo = (in1.lo * in2.lo)[31:0] 3376 3377 // IP <- in1.lo * in2.hi 3378 __ mul(IP, in1_lo, in2_hi); 3379 // out.hi <- in1.lo * in2.hi + in1.hi * in2.lo 3380 __ mla(out_hi, in1_hi, in2_lo, IP); 3381 // out.lo <- (in1.lo * in2.lo)[31:0]; 3382 __ umull(out_lo, IP, in1_lo, in2_lo); 3383 // out.hi <- in2.hi * in1.lo + in2.lo * in1.hi + (in1.lo * in2.lo)[63:32] 3384 __ add(out_hi, out_hi, ShifterOperand(IP)); 3385 break; 3386 } 3387 3388 case Primitive::kPrimFloat: { 3389 __ vmuls(out.AsFpuRegister<SRegister>(), 3390 first.AsFpuRegister<SRegister>(), 3391 second.AsFpuRegister<SRegister>()); 3392 break; 3393 } 3394 3395 case Primitive::kPrimDouble: { 3396 __ vmuld(FromLowSToD(out.AsFpuRegisterPairLow<SRegister>()), 3397 FromLowSToD(first.AsFpuRegisterPairLow<SRegister>()), 3398 FromLowSToD(second.AsFpuRegisterPairLow<SRegister>())); 3399 break; 3400 } 3401 3402 default: 3403 LOG(FATAL) << "Unexpected mul type " << mul->GetResultType(); 3404 } 3405} 3406 3407void InstructionCodeGeneratorARM::DivRemOneOrMinusOne(HBinaryOperation* instruction) { 3408 DCHECK(instruction->IsDiv() || instruction->IsRem()); 3409 DCHECK(instruction->GetResultType() == Primitive::kPrimInt); 3410 3411 LocationSummary* locations = instruction->GetLocations(); 3412 Location second = locations->InAt(1); 3413 DCHECK(second.IsConstant()); 3414 3415 Register out = locations->Out().AsRegister<Register>(); 3416 Register dividend = locations->InAt(0).AsRegister<Register>(); 3417 int32_t imm = second.GetConstant()->AsIntConstant()->GetValue(); 3418 DCHECK(imm == 1 || imm == -1); 3419 3420 if (instruction->IsRem()) { 3421 __ LoadImmediate(out, 0); 3422 } else { 3423 if (imm == 1) { 3424 __ Mov(out, dividend); 3425 } else { 3426 __ rsb(out, dividend, ShifterOperand(0)); 3427 } 3428 } 3429} 3430 3431void InstructionCodeGeneratorARM::DivRemByPowerOfTwo(HBinaryOperation* instruction) { 3432 DCHECK(instruction->IsDiv() || instruction->IsRem()); 3433 DCHECK(instruction->GetResultType() == Primitive::kPrimInt); 3434 3435 LocationSummary* locations = instruction->GetLocations(); 3436 Location second = locations->InAt(1); 3437 DCHECK(second.IsConstant()); 3438 3439 Register out = locations->Out().AsRegister<Register>(); 3440 Register dividend = locations->InAt(0).AsRegister<Register>(); 3441 Register temp = locations->GetTemp(0).AsRegister<Register>(); 3442 int32_t imm = second.GetConstant()->AsIntConstant()->GetValue(); 3443 uint32_t abs_imm = static_cast<uint32_t>(AbsOrMin(imm)); 3444 int ctz_imm = CTZ(abs_imm); 3445 3446 if (ctz_imm == 1) { 3447 __ Lsr(temp, dividend, 32 - ctz_imm); 3448 } else { 3449 __ Asr(temp, dividend, 31); 3450 __ Lsr(temp, temp, 32 - ctz_imm); 3451 } 3452 __ add(out, temp, ShifterOperand(dividend)); 3453 3454 if (instruction->IsDiv()) { 3455 __ Asr(out, out, ctz_imm); 3456 if (imm < 0) { 3457 __ rsb(out, out, ShifterOperand(0)); 3458 } 3459 } else { 3460 __ ubfx(out, out, 0, ctz_imm); 3461 __ sub(out, out, ShifterOperand(temp)); 3462 } 3463} 3464 3465void InstructionCodeGeneratorARM::GenerateDivRemWithAnyConstant(HBinaryOperation* instruction) { 3466 DCHECK(instruction->IsDiv() || instruction->IsRem()); 3467 DCHECK(instruction->GetResultType() == Primitive::kPrimInt); 3468 3469 LocationSummary* locations = instruction->GetLocations(); 3470 Location second = locations->InAt(1); 3471 DCHECK(second.IsConstant()); 3472 3473 Register out = locations->Out().AsRegister<Register>(); 3474 Register dividend = locations->InAt(0).AsRegister<Register>(); 3475 Register temp1 = locations->GetTemp(0).AsRegister<Register>(); 3476 Register temp2 = locations->GetTemp(1).AsRegister<Register>(); 3477 int64_t imm = second.GetConstant()->AsIntConstant()->GetValue(); 3478 3479 int64_t magic; 3480 int shift; 3481 CalculateMagicAndShiftForDivRem(imm, false /* is_long */, &magic, &shift); 3482 3483 __ LoadImmediate(temp1, magic); 3484 __ smull(temp2, temp1, dividend, temp1); 3485 3486 if (imm > 0 && magic < 0) { 3487 __ add(temp1, temp1, ShifterOperand(dividend)); 3488 } else if (imm < 0 && magic > 0) { 3489 __ sub(temp1, temp1, ShifterOperand(dividend)); 3490 } 3491 3492 if (shift != 0) { 3493 __ Asr(temp1, temp1, shift); 3494 } 3495 3496 if (instruction->IsDiv()) { 3497 __ sub(out, temp1, ShifterOperand(temp1, ASR, 31)); 3498 } else { 3499 __ sub(temp1, temp1, ShifterOperand(temp1, ASR, 31)); 3500 // TODO: Strength reduction for mls. 3501 __ LoadImmediate(temp2, imm); 3502 __ mls(out, temp1, temp2, dividend); 3503 } 3504} 3505 3506void InstructionCodeGeneratorARM::GenerateDivRemConstantIntegral(HBinaryOperation* instruction) { 3507 DCHECK(instruction->IsDiv() || instruction->IsRem()); 3508 DCHECK(instruction->GetResultType() == Primitive::kPrimInt); 3509 3510 LocationSummary* locations = instruction->GetLocations(); 3511 Location second = locations->InAt(1); 3512 DCHECK(second.IsConstant()); 3513 3514 int32_t imm = second.GetConstant()->AsIntConstant()->GetValue(); 3515 if (imm == 0) { 3516 // Do not generate anything. DivZeroCheck would prevent any code to be executed. 3517 } else if (imm == 1 || imm == -1) { 3518 DivRemOneOrMinusOne(instruction); 3519 } else if (IsPowerOfTwo(AbsOrMin(imm))) { 3520 DivRemByPowerOfTwo(instruction); 3521 } else { 3522 DCHECK(imm <= -2 || imm >= 2); 3523 GenerateDivRemWithAnyConstant(instruction); 3524 } 3525} 3526 3527void LocationsBuilderARM::VisitDiv(HDiv* div) { 3528 LocationSummary::CallKind call_kind = LocationSummary::kNoCall; 3529 if (div->GetResultType() == Primitive::kPrimLong) { 3530 // pLdiv runtime call. 3531 call_kind = LocationSummary::kCallOnMainOnly; 3532 } else if (div->GetResultType() == Primitive::kPrimInt && div->InputAt(1)->IsConstant()) { 3533 // sdiv will be replaced by other instruction sequence. 3534 } else if (div->GetResultType() == Primitive::kPrimInt && 3535 !codegen_->GetInstructionSetFeatures().HasDivideInstruction()) { 3536 // pIdivmod runtime call. 3537 call_kind = LocationSummary::kCallOnMainOnly; 3538 } 3539 3540 LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(div, call_kind); 3541 3542 switch (div->GetResultType()) { 3543 case Primitive::kPrimInt: { 3544 if (div->InputAt(1)->IsConstant()) { 3545 locations->SetInAt(0, Location::RequiresRegister()); 3546 locations->SetInAt(1, Location::ConstantLocation(div->InputAt(1)->AsConstant())); 3547 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 3548 int32_t value = div->InputAt(1)->AsIntConstant()->GetValue(); 3549 if (value == 1 || value == 0 || value == -1) { 3550 // No temp register required. 3551 } else { 3552 locations->AddTemp(Location::RequiresRegister()); 3553 if (!IsPowerOfTwo(AbsOrMin(value))) { 3554 locations->AddTemp(Location::RequiresRegister()); 3555 } 3556 } 3557 } else if (codegen_->GetInstructionSetFeatures().HasDivideInstruction()) { 3558 locations->SetInAt(0, Location::RequiresRegister()); 3559 locations->SetInAt(1, Location::RequiresRegister()); 3560 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 3561 } else { 3562 InvokeRuntimeCallingConvention calling_convention; 3563 locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(0))); 3564 locations->SetInAt(1, Location::RegisterLocation(calling_convention.GetRegisterAt(1))); 3565 // Note: divmod will compute both the quotient and the remainder as the pair R0 and R1, but 3566 // we only need the former. 3567 locations->SetOut(Location::RegisterLocation(R0)); 3568 } 3569 break; 3570 } 3571 case Primitive::kPrimLong: { 3572 InvokeRuntimeCallingConvention calling_convention; 3573 locations->SetInAt(0, Location::RegisterPairLocation( 3574 calling_convention.GetRegisterAt(0), calling_convention.GetRegisterAt(1))); 3575 locations->SetInAt(1, Location::RegisterPairLocation( 3576 calling_convention.GetRegisterAt(2), calling_convention.GetRegisterAt(3))); 3577 locations->SetOut(Location::RegisterPairLocation(R0, R1)); 3578 break; 3579 } 3580 case Primitive::kPrimFloat: 3581 case Primitive::kPrimDouble: { 3582 locations->SetInAt(0, Location::RequiresFpuRegister()); 3583 locations->SetInAt(1, Location::RequiresFpuRegister()); 3584 locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap); 3585 break; 3586 } 3587 3588 default: 3589 LOG(FATAL) << "Unexpected div type " << div->GetResultType(); 3590 } 3591} 3592 3593void InstructionCodeGeneratorARM::VisitDiv(HDiv* div) { 3594 LocationSummary* locations = div->GetLocations(); 3595 Location out = locations->Out(); 3596 Location first = locations->InAt(0); 3597 Location second = locations->InAt(1); 3598 3599 switch (div->GetResultType()) { 3600 case Primitive::kPrimInt: { 3601 if (second.IsConstant()) { 3602 GenerateDivRemConstantIntegral(div); 3603 } else if (codegen_->GetInstructionSetFeatures().HasDivideInstruction()) { 3604 __ sdiv(out.AsRegister<Register>(), 3605 first.AsRegister<Register>(), 3606 second.AsRegister<Register>()); 3607 } else { 3608 InvokeRuntimeCallingConvention calling_convention; 3609 DCHECK_EQ(calling_convention.GetRegisterAt(0), first.AsRegister<Register>()); 3610 DCHECK_EQ(calling_convention.GetRegisterAt(1), second.AsRegister<Register>()); 3611 DCHECK_EQ(R0, out.AsRegister<Register>()); 3612 3613 codegen_->InvokeRuntime(kQuickIdivmod, div, div->GetDexPc()); 3614 CheckEntrypointTypes<kQuickIdivmod, int32_t, int32_t, int32_t>(); 3615 } 3616 break; 3617 } 3618 3619 case Primitive::kPrimLong: { 3620 InvokeRuntimeCallingConvention calling_convention; 3621 DCHECK_EQ(calling_convention.GetRegisterAt(0), first.AsRegisterPairLow<Register>()); 3622 DCHECK_EQ(calling_convention.GetRegisterAt(1), first.AsRegisterPairHigh<Register>()); 3623 DCHECK_EQ(calling_convention.GetRegisterAt(2), second.AsRegisterPairLow<Register>()); 3624 DCHECK_EQ(calling_convention.GetRegisterAt(3), second.AsRegisterPairHigh<Register>()); 3625 DCHECK_EQ(R0, out.AsRegisterPairLow<Register>()); 3626 DCHECK_EQ(R1, out.AsRegisterPairHigh<Register>()); 3627 3628 codegen_->InvokeRuntime(kQuickLdiv, div, div->GetDexPc()); 3629 CheckEntrypointTypes<kQuickLdiv, int64_t, int64_t, int64_t>(); 3630 break; 3631 } 3632 3633 case Primitive::kPrimFloat: { 3634 __ vdivs(out.AsFpuRegister<SRegister>(), 3635 first.AsFpuRegister<SRegister>(), 3636 second.AsFpuRegister<SRegister>()); 3637 break; 3638 } 3639 3640 case Primitive::kPrimDouble: { 3641 __ vdivd(FromLowSToD(out.AsFpuRegisterPairLow<SRegister>()), 3642 FromLowSToD(first.AsFpuRegisterPairLow<SRegister>()), 3643 FromLowSToD(second.AsFpuRegisterPairLow<SRegister>())); 3644 break; 3645 } 3646 3647 default: 3648 LOG(FATAL) << "Unexpected div type " << div->GetResultType(); 3649 } 3650} 3651 3652void LocationsBuilderARM::VisitRem(HRem* rem) { 3653 Primitive::Type type = rem->GetResultType(); 3654 3655 // Most remainders are implemented in the runtime. 3656 LocationSummary::CallKind call_kind = LocationSummary::kCallOnMainOnly; 3657 if (rem->GetResultType() == Primitive::kPrimInt && rem->InputAt(1)->IsConstant()) { 3658 // sdiv will be replaced by other instruction sequence. 3659 call_kind = LocationSummary::kNoCall; 3660 } else if ((rem->GetResultType() == Primitive::kPrimInt) 3661 && codegen_->GetInstructionSetFeatures().HasDivideInstruction()) { 3662 // Have hardware divide instruction for int, do it with three instructions. 3663 call_kind = LocationSummary::kNoCall; 3664 } 3665 3666 LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(rem, call_kind); 3667 3668 switch (type) { 3669 case Primitive::kPrimInt: { 3670 if (rem->InputAt(1)->IsConstant()) { 3671 locations->SetInAt(0, Location::RequiresRegister()); 3672 locations->SetInAt(1, Location::ConstantLocation(rem->InputAt(1)->AsConstant())); 3673 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 3674 int32_t value = rem->InputAt(1)->AsIntConstant()->GetValue(); 3675 if (value == 1 || value == 0 || value == -1) { 3676 // No temp register required. 3677 } else { 3678 locations->AddTemp(Location::RequiresRegister()); 3679 if (!IsPowerOfTwo(AbsOrMin(value))) { 3680 locations->AddTemp(Location::RequiresRegister()); 3681 } 3682 } 3683 } else if (codegen_->GetInstructionSetFeatures().HasDivideInstruction()) { 3684 locations->SetInAt(0, Location::RequiresRegister()); 3685 locations->SetInAt(1, Location::RequiresRegister()); 3686 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 3687 locations->AddTemp(Location::RequiresRegister()); 3688 } else { 3689 InvokeRuntimeCallingConvention calling_convention; 3690 locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(0))); 3691 locations->SetInAt(1, Location::RegisterLocation(calling_convention.GetRegisterAt(1))); 3692 // Note: divmod will compute both the quotient and the remainder as the pair R0 and R1, but 3693 // we only need the latter. 3694 locations->SetOut(Location::RegisterLocation(R1)); 3695 } 3696 break; 3697 } 3698 case Primitive::kPrimLong: { 3699 InvokeRuntimeCallingConvention calling_convention; 3700 locations->SetInAt(0, Location::RegisterPairLocation( 3701 calling_convention.GetRegisterAt(0), calling_convention.GetRegisterAt(1))); 3702 locations->SetInAt(1, Location::RegisterPairLocation( 3703 calling_convention.GetRegisterAt(2), calling_convention.GetRegisterAt(3))); 3704 // The runtime helper puts the output in R2,R3. 3705 locations->SetOut(Location::RegisterPairLocation(R2, R3)); 3706 break; 3707 } 3708 case Primitive::kPrimFloat: { 3709 InvokeRuntimeCallingConvention calling_convention; 3710 locations->SetInAt(0, Location::FpuRegisterLocation(calling_convention.GetFpuRegisterAt(0))); 3711 locations->SetInAt(1, Location::FpuRegisterLocation(calling_convention.GetFpuRegisterAt(1))); 3712 locations->SetOut(Location::FpuRegisterLocation(S0)); 3713 break; 3714 } 3715 3716 case Primitive::kPrimDouble: { 3717 InvokeRuntimeCallingConvention calling_convention; 3718 locations->SetInAt(0, Location::FpuRegisterPairLocation( 3719 calling_convention.GetFpuRegisterAt(0), calling_convention.GetFpuRegisterAt(1))); 3720 locations->SetInAt(1, Location::FpuRegisterPairLocation( 3721 calling_convention.GetFpuRegisterAt(2), calling_convention.GetFpuRegisterAt(3))); 3722 locations->SetOut(Location::Location::FpuRegisterPairLocation(S0, S1)); 3723 break; 3724 } 3725 3726 default: 3727 LOG(FATAL) << "Unexpected rem type " << type; 3728 } 3729} 3730 3731void InstructionCodeGeneratorARM::VisitRem(HRem* rem) { 3732 LocationSummary* locations = rem->GetLocations(); 3733 Location out = locations->Out(); 3734 Location first = locations->InAt(0); 3735 Location second = locations->InAt(1); 3736 3737 Primitive::Type type = rem->GetResultType(); 3738 switch (type) { 3739 case Primitive::kPrimInt: { 3740 if (second.IsConstant()) { 3741 GenerateDivRemConstantIntegral(rem); 3742 } else if (codegen_->GetInstructionSetFeatures().HasDivideInstruction()) { 3743 Register reg1 = first.AsRegister<Register>(); 3744 Register reg2 = second.AsRegister<Register>(); 3745 Register temp = locations->GetTemp(0).AsRegister<Register>(); 3746 3747 // temp = reg1 / reg2 (integer division) 3748 // dest = reg1 - temp * reg2 3749 __ sdiv(temp, reg1, reg2); 3750 __ mls(out.AsRegister<Register>(), temp, reg2, reg1); 3751 } else { 3752 InvokeRuntimeCallingConvention calling_convention; 3753 DCHECK_EQ(calling_convention.GetRegisterAt(0), first.AsRegister<Register>()); 3754 DCHECK_EQ(calling_convention.GetRegisterAt(1), second.AsRegister<Register>()); 3755 DCHECK_EQ(R1, out.AsRegister<Register>()); 3756 3757 codegen_->InvokeRuntime(kQuickIdivmod, rem, rem->GetDexPc()); 3758 CheckEntrypointTypes<kQuickIdivmod, int32_t, int32_t, int32_t>(); 3759 } 3760 break; 3761 } 3762 3763 case Primitive::kPrimLong: { 3764 codegen_->InvokeRuntime(kQuickLmod, rem, rem->GetDexPc()); 3765 CheckEntrypointTypes<kQuickLmod, int64_t, int64_t, int64_t>(); 3766 break; 3767 } 3768 3769 case Primitive::kPrimFloat: { 3770 codegen_->InvokeRuntime(kQuickFmodf, rem, rem->GetDexPc()); 3771 CheckEntrypointTypes<kQuickFmodf, float, float, float>(); 3772 break; 3773 } 3774 3775 case Primitive::kPrimDouble: { 3776 codegen_->InvokeRuntime(kQuickFmod, rem, rem->GetDexPc()); 3777 CheckEntrypointTypes<kQuickFmod, double, double, double>(); 3778 break; 3779 } 3780 3781 default: 3782 LOG(FATAL) << "Unexpected rem type " << type; 3783 } 3784} 3785 3786void LocationsBuilderARM::VisitDivZeroCheck(HDivZeroCheck* instruction) { 3787 LocationSummary* locations = codegen_->CreateThrowingSlowPathLocations(instruction); 3788 locations->SetInAt(0, Location::RegisterOrConstant(instruction->InputAt(0))); 3789} 3790 3791void InstructionCodeGeneratorARM::VisitDivZeroCheck(HDivZeroCheck* instruction) { 3792 SlowPathCodeARM* slow_path = new (GetGraph()->GetArena()) DivZeroCheckSlowPathARM(instruction); 3793 codegen_->AddSlowPath(slow_path); 3794 3795 LocationSummary* locations = instruction->GetLocations(); 3796 Location value = locations->InAt(0); 3797 3798 switch (instruction->GetType()) { 3799 case Primitive::kPrimBoolean: 3800 case Primitive::kPrimByte: 3801 case Primitive::kPrimChar: 3802 case Primitive::kPrimShort: 3803 case Primitive::kPrimInt: { 3804 if (value.IsRegister()) { 3805 __ CompareAndBranchIfZero(value.AsRegister<Register>(), slow_path->GetEntryLabel()); 3806 } else { 3807 DCHECK(value.IsConstant()) << value; 3808 if (value.GetConstant()->AsIntConstant()->GetValue() == 0) { 3809 __ b(slow_path->GetEntryLabel()); 3810 } 3811 } 3812 break; 3813 } 3814 case Primitive::kPrimLong: { 3815 if (value.IsRegisterPair()) { 3816 __ orrs(IP, 3817 value.AsRegisterPairLow<Register>(), 3818 ShifterOperand(value.AsRegisterPairHigh<Register>())); 3819 __ b(slow_path->GetEntryLabel(), EQ); 3820 } else { 3821 DCHECK(value.IsConstant()) << value; 3822 if (value.GetConstant()->AsLongConstant()->GetValue() == 0) { 3823 __ b(slow_path->GetEntryLabel()); 3824 } 3825 } 3826 break; 3827 default: 3828 LOG(FATAL) << "Unexpected type for HDivZeroCheck " << instruction->GetType(); 3829 } 3830 } 3831} 3832 3833void InstructionCodeGeneratorARM::HandleIntegerRotate(LocationSummary* locations) { 3834 Register in = locations->InAt(0).AsRegister<Register>(); 3835 Location rhs = locations->InAt(1); 3836 Register out = locations->Out().AsRegister<Register>(); 3837 3838 if (rhs.IsConstant()) { 3839 // Arm32 and Thumb2 assemblers require a rotation on the interval [1,31], 3840 // so map all rotations to a +ve. equivalent in that range. 3841 // (e.g. left *or* right by -2 bits == 30 bits in the same direction.) 3842 uint32_t rot = CodeGenerator::GetInt32ValueOf(rhs.GetConstant()) & 0x1F; 3843 if (rot) { 3844 // Rotate, mapping left rotations to right equivalents if necessary. 3845 // (e.g. left by 2 bits == right by 30.) 3846 __ Ror(out, in, rot); 3847 } else if (out != in) { 3848 __ Mov(out, in); 3849 } 3850 } else { 3851 __ Ror(out, in, rhs.AsRegister<Register>()); 3852 } 3853} 3854 3855// Gain some speed by mapping all Long rotates onto equivalent pairs of Integer 3856// rotates by swapping input regs (effectively rotating by the first 32-bits of 3857// a larger rotation) or flipping direction (thus treating larger right/left 3858// rotations as sub-word sized rotations in the other direction) as appropriate. 3859void InstructionCodeGeneratorARM::HandleLongRotate(LocationSummary* locations) { 3860 Register in_reg_lo = locations->InAt(0).AsRegisterPairLow<Register>(); 3861 Register in_reg_hi = locations->InAt(0).AsRegisterPairHigh<Register>(); 3862 Location rhs = locations->InAt(1); 3863 Register out_reg_lo = locations->Out().AsRegisterPairLow<Register>(); 3864 Register out_reg_hi = locations->Out().AsRegisterPairHigh<Register>(); 3865 3866 if (rhs.IsConstant()) { 3867 uint64_t rot = CodeGenerator::GetInt64ValueOf(rhs.GetConstant()); 3868 // Map all rotations to +ve. equivalents on the interval [0,63]. 3869 rot &= kMaxLongShiftDistance; 3870 // For rotates over a word in size, 'pre-rotate' by 32-bits to keep rotate 3871 // logic below to a simple pair of binary orr. 3872 // (e.g. 34 bits == in_reg swap + 2 bits right.) 3873 if (rot >= kArmBitsPerWord) { 3874 rot -= kArmBitsPerWord; 3875 std::swap(in_reg_hi, in_reg_lo); 3876 } 3877 // Rotate, or mov to out for zero or word size rotations. 3878 if (rot != 0u) { 3879 __ Lsr(out_reg_hi, in_reg_hi, rot); 3880 __ orr(out_reg_hi, out_reg_hi, ShifterOperand(in_reg_lo, arm::LSL, kArmBitsPerWord - rot)); 3881 __ Lsr(out_reg_lo, in_reg_lo, rot); 3882 __ orr(out_reg_lo, out_reg_lo, ShifterOperand(in_reg_hi, arm::LSL, kArmBitsPerWord - rot)); 3883 } else { 3884 __ Mov(out_reg_lo, in_reg_lo); 3885 __ Mov(out_reg_hi, in_reg_hi); 3886 } 3887 } else { 3888 Register shift_right = locations->GetTemp(0).AsRegister<Register>(); 3889 Register shift_left = locations->GetTemp(1).AsRegister<Register>(); 3890 Label end; 3891 Label shift_by_32_plus_shift_right; 3892 3893 __ and_(shift_right, rhs.AsRegister<Register>(), ShifterOperand(0x1F)); 3894 __ Lsrs(shift_left, rhs.AsRegister<Register>(), 6); 3895 __ rsb(shift_left, shift_right, ShifterOperand(kArmBitsPerWord), AL, kCcKeep); 3896 __ b(&shift_by_32_plus_shift_right, CC); 3897 3898 // out_reg_hi = (reg_hi << shift_left) | (reg_lo >> shift_right). 3899 // out_reg_lo = (reg_lo << shift_left) | (reg_hi >> shift_right). 3900 __ Lsl(out_reg_hi, in_reg_hi, shift_left); 3901 __ Lsr(out_reg_lo, in_reg_lo, shift_right); 3902 __ add(out_reg_hi, out_reg_hi, ShifterOperand(out_reg_lo)); 3903 __ Lsl(out_reg_lo, in_reg_lo, shift_left); 3904 __ Lsr(shift_left, in_reg_hi, shift_right); 3905 __ add(out_reg_lo, out_reg_lo, ShifterOperand(shift_left)); 3906 __ b(&end); 3907 3908 __ Bind(&shift_by_32_plus_shift_right); // Shift by 32+shift_right. 3909 // out_reg_hi = (reg_hi >> shift_right) | (reg_lo << shift_left). 3910 // out_reg_lo = (reg_lo >> shift_right) | (reg_hi << shift_left). 3911 __ Lsr(out_reg_hi, in_reg_hi, shift_right); 3912 __ Lsl(out_reg_lo, in_reg_lo, shift_left); 3913 __ add(out_reg_hi, out_reg_hi, ShifterOperand(out_reg_lo)); 3914 __ Lsr(out_reg_lo, in_reg_lo, shift_right); 3915 __ Lsl(shift_right, in_reg_hi, shift_left); 3916 __ add(out_reg_lo, out_reg_lo, ShifterOperand(shift_right)); 3917 3918 __ Bind(&end); 3919 } 3920} 3921 3922void LocationsBuilderARM::VisitRor(HRor* ror) { 3923 LocationSummary* locations = 3924 new (GetGraph()->GetArena()) LocationSummary(ror, LocationSummary::kNoCall); 3925 switch (ror->GetResultType()) { 3926 case Primitive::kPrimInt: { 3927 locations->SetInAt(0, Location::RequiresRegister()); 3928 locations->SetInAt(1, Location::RegisterOrConstant(ror->InputAt(1))); 3929 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 3930 break; 3931 } 3932 case Primitive::kPrimLong: { 3933 locations->SetInAt(0, Location::RequiresRegister()); 3934 if (ror->InputAt(1)->IsConstant()) { 3935 locations->SetInAt(1, Location::ConstantLocation(ror->InputAt(1)->AsConstant())); 3936 } else { 3937 locations->SetInAt(1, Location::RequiresRegister()); 3938 locations->AddTemp(Location::RequiresRegister()); 3939 locations->AddTemp(Location::RequiresRegister()); 3940 } 3941 locations->SetOut(Location::RequiresRegister(), Location::kOutputOverlap); 3942 break; 3943 } 3944 default: 3945 LOG(FATAL) << "Unexpected operation type " << ror->GetResultType(); 3946 } 3947} 3948 3949void InstructionCodeGeneratorARM::VisitRor(HRor* ror) { 3950 LocationSummary* locations = ror->GetLocations(); 3951 Primitive::Type type = ror->GetResultType(); 3952 switch (type) { 3953 case Primitive::kPrimInt: { 3954 HandleIntegerRotate(locations); 3955 break; 3956 } 3957 case Primitive::kPrimLong: { 3958 HandleLongRotate(locations); 3959 break; 3960 } 3961 default: 3962 LOG(FATAL) << "Unexpected operation type " << type; 3963 UNREACHABLE(); 3964 } 3965} 3966 3967void LocationsBuilderARM::HandleShift(HBinaryOperation* op) { 3968 DCHECK(op->IsShl() || op->IsShr() || op->IsUShr()); 3969 3970 LocationSummary* locations = 3971 new (GetGraph()->GetArena()) LocationSummary(op, LocationSummary::kNoCall); 3972 3973 switch (op->GetResultType()) { 3974 case Primitive::kPrimInt: { 3975 locations->SetInAt(0, Location::RequiresRegister()); 3976 if (op->InputAt(1)->IsConstant()) { 3977 locations->SetInAt(1, Location::ConstantLocation(op->InputAt(1)->AsConstant())); 3978 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 3979 } else { 3980 locations->SetInAt(1, Location::RequiresRegister()); 3981 // Make the output overlap, as it will be used to hold the masked 3982 // second input. 3983 locations->SetOut(Location::RequiresRegister(), Location::kOutputOverlap); 3984 } 3985 break; 3986 } 3987 case Primitive::kPrimLong: { 3988 locations->SetInAt(0, Location::RequiresRegister()); 3989 if (op->InputAt(1)->IsConstant()) { 3990 locations->SetInAt(1, Location::ConstantLocation(op->InputAt(1)->AsConstant())); 3991 // For simplicity, use kOutputOverlap even though we only require that low registers 3992 // don't clash with high registers which the register allocator currently guarantees. 3993 locations->SetOut(Location::RequiresRegister(), Location::kOutputOverlap); 3994 } else { 3995 locations->SetInAt(1, Location::RequiresRegister()); 3996 locations->AddTemp(Location::RequiresRegister()); 3997 locations->SetOut(Location::RequiresRegister(), Location::kOutputOverlap); 3998 } 3999 break; 4000 } 4001 default: 4002 LOG(FATAL) << "Unexpected operation type " << op->GetResultType(); 4003 } 4004} 4005 4006void InstructionCodeGeneratorARM::HandleShift(HBinaryOperation* op) { 4007 DCHECK(op->IsShl() || op->IsShr() || op->IsUShr()); 4008 4009 LocationSummary* locations = op->GetLocations(); 4010 Location out = locations->Out(); 4011 Location first = locations->InAt(0); 4012 Location second = locations->InAt(1); 4013 4014 Primitive::Type type = op->GetResultType(); 4015 switch (type) { 4016 case Primitive::kPrimInt: { 4017 Register out_reg = out.AsRegister<Register>(); 4018 Register first_reg = first.AsRegister<Register>(); 4019 if (second.IsRegister()) { 4020 Register second_reg = second.AsRegister<Register>(); 4021 // ARM doesn't mask the shift count so we need to do it ourselves. 4022 __ and_(out_reg, second_reg, ShifterOperand(kMaxIntShiftDistance)); 4023 if (op->IsShl()) { 4024 __ Lsl(out_reg, first_reg, out_reg); 4025 } else if (op->IsShr()) { 4026 __ Asr(out_reg, first_reg, out_reg); 4027 } else { 4028 __ Lsr(out_reg, first_reg, out_reg); 4029 } 4030 } else { 4031 int32_t cst = second.GetConstant()->AsIntConstant()->GetValue(); 4032 uint32_t shift_value = cst & kMaxIntShiftDistance; 4033 if (shift_value == 0) { // ARM does not support shifting with 0 immediate. 4034 __ Mov(out_reg, first_reg); 4035 } else if (op->IsShl()) { 4036 __ Lsl(out_reg, first_reg, shift_value); 4037 } else if (op->IsShr()) { 4038 __ Asr(out_reg, first_reg, shift_value); 4039 } else { 4040 __ Lsr(out_reg, first_reg, shift_value); 4041 } 4042 } 4043 break; 4044 } 4045 case Primitive::kPrimLong: { 4046 Register o_h = out.AsRegisterPairHigh<Register>(); 4047 Register o_l = out.AsRegisterPairLow<Register>(); 4048 4049 Register high = first.AsRegisterPairHigh<Register>(); 4050 Register low = first.AsRegisterPairLow<Register>(); 4051 4052 if (second.IsRegister()) { 4053 Register temp = locations->GetTemp(0).AsRegister<Register>(); 4054 4055 Register second_reg = second.AsRegister<Register>(); 4056 4057 if (op->IsShl()) { 4058 __ and_(o_l, second_reg, ShifterOperand(kMaxLongShiftDistance)); 4059 // Shift the high part 4060 __ Lsl(o_h, high, o_l); 4061 // Shift the low part and `or` what overflew on the high part 4062 __ rsb(temp, o_l, ShifterOperand(kArmBitsPerWord)); 4063 __ Lsr(temp, low, temp); 4064 __ orr(o_h, o_h, ShifterOperand(temp)); 4065 // If the shift is > 32 bits, override the high part 4066 __ subs(temp, o_l, ShifterOperand(kArmBitsPerWord)); 4067 __ it(PL); 4068 __ Lsl(o_h, low, temp, PL); 4069 // Shift the low part 4070 __ Lsl(o_l, low, o_l); 4071 } else if (op->IsShr()) { 4072 __ and_(o_h, second_reg, ShifterOperand(kMaxLongShiftDistance)); 4073 // Shift the low part 4074 __ Lsr(o_l, low, o_h); 4075 // Shift the high part and `or` what underflew on the low part 4076 __ rsb(temp, o_h, ShifterOperand(kArmBitsPerWord)); 4077 __ Lsl(temp, high, temp); 4078 __ orr(o_l, o_l, ShifterOperand(temp)); 4079 // If the shift is > 32 bits, override the low part 4080 __ subs(temp, o_h, ShifterOperand(kArmBitsPerWord)); 4081 __ it(PL); 4082 __ Asr(o_l, high, temp, PL); 4083 // Shift the high part 4084 __ Asr(o_h, high, o_h); 4085 } else { 4086 __ and_(o_h, second_reg, ShifterOperand(kMaxLongShiftDistance)); 4087 // same as Shr except we use `Lsr`s and not `Asr`s 4088 __ Lsr(o_l, low, o_h); 4089 __ rsb(temp, o_h, ShifterOperand(kArmBitsPerWord)); 4090 __ Lsl(temp, high, temp); 4091 __ orr(o_l, o_l, ShifterOperand(temp)); 4092 __ subs(temp, o_h, ShifterOperand(kArmBitsPerWord)); 4093 __ it(PL); 4094 __ Lsr(o_l, high, temp, PL); 4095 __ Lsr(o_h, high, o_h); 4096 } 4097 } else { 4098 // Register allocator doesn't create partial overlap. 4099 DCHECK_NE(o_l, high); 4100 DCHECK_NE(o_h, low); 4101 int32_t cst = second.GetConstant()->AsIntConstant()->GetValue(); 4102 uint32_t shift_value = cst & kMaxLongShiftDistance; 4103 if (shift_value > 32) { 4104 if (op->IsShl()) { 4105 __ Lsl(o_h, low, shift_value - 32); 4106 __ LoadImmediate(o_l, 0); 4107 } else if (op->IsShr()) { 4108 __ Asr(o_l, high, shift_value - 32); 4109 __ Asr(o_h, high, 31); 4110 } else { 4111 __ Lsr(o_l, high, shift_value - 32); 4112 __ LoadImmediate(o_h, 0); 4113 } 4114 } else if (shift_value == 32) { 4115 if (op->IsShl()) { 4116 __ mov(o_h, ShifterOperand(low)); 4117 __ LoadImmediate(o_l, 0); 4118 } else if (op->IsShr()) { 4119 __ mov(o_l, ShifterOperand(high)); 4120 __ Asr(o_h, high, 31); 4121 } else { 4122 __ mov(o_l, ShifterOperand(high)); 4123 __ LoadImmediate(o_h, 0); 4124 } 4125 } else if (shift_value == 1) { 4126 if (op->IsShl()) { 4127 __ Lsls(o_l, low, 1); 4128 __ adc(o_h, high, ShifterOperand(high)); 4129 } else if (op->IsShr()) { 4130 __ Asrs(o_h, high, 1); 4131 __ Rrx(o_l, low); 4132 } else { 4133 __ Lsrs(o_h, high, 1); 4134 __ Rrx(o_l, low); 4135 } 4136 } else { 4137 DCHECK(2 <= shift_value && shift_value < 32) << shift_value; 4138 if (op->IsShl()) { 4139 __ Lsl(o_h, high, shift_value); 4140 __ orr(o_h, o_h, ShifterOperand(low, LSR, 32 - shift_value)); 4141 __ Lsl(o_l, low, shift_value); 4142 } else if (op->IsShr()) { 4143 __ Lsr(o_l, low, shift_value); 4144 __ orr(o_l, o_l, ShifterOperand(high, LSL, 32 - shift_value)); 4145 __ Asr(o_h, high, shift_value); 4146 } else { 4147 __ Lsr(o_l, low, shift_value); 4148 __ orr(o_l, o_l, ShifterOperand(high, LSL, 32 - shift_value)); 4149 __ Lsr(o_h, high, shift_value); 4150 } 4151 } 4152 } 4153 break; 4154 } 4155 default: 4156 LOG(FATAL) << "Unexpected operation type " << type; 4157 UNREACHABLE(); 4158 } 4159} 4160 4161void LocationsBuilderARM::VisitShl(HShl* shl) { 4162 HandleShift(shl); 4163} 4164 4165void InstructionCodeGeneratorARM::VisitShl(HShl* shl) { 4166 HandleShift(shl); 4167} 4168 4169void LocationsBuilderARM::VisitShr(HShr* shr) { 4170 HandleShift(shr); 4171} 4172 4173void InstructionCodeGeneratorARM::VisitShr(HShr* shr) { 4174 HandleShift(shr); 4175} 4176 4177void LocationsBuilderARM::VisitUShr(HUShr* ushr) { 4178 HandleShift(ushr); 4179} 4180 4181void InstructionCodeGeneratorARM::VisitUShr(HUShr* ushr) { 4182 HandleShift(ushr); 4183} 4184 4185void LocationsBuilderARM::VisitNewInstance(HNewInstance* instruction) { 4186 LocationSummary* locations = 4187 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCallOnMainOnly); 4188 if (instruction->IsStringAlloc()) { 4189 locations->AddTemp(Location::RegisterLocation(kMethodRegisterArgument)); 4190 } else { 4191 InvokeRuntimeCallingConvention calling_convention; 4192 locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(0))); 4193 } 4194 locations->SetOut(Location::RegisterLocation(R0)); 4195} 4196 4197void InstructionCodeGeneratorARM::VisitNewInstance(HNewInstance* instruction) { 4198 // Note: if heap poisoning is enabled, the entry point takes cares 4199 // of poisoning the reference. 4200 if (instruction->IsStringAlloc()) { 4201 // String is allocated through StringFactory. Call NewEmptyString entry point. 4202 Register temp = instruction->GetLocations()->GetTemp(0).AsRegister<Register>(); 4203 MemberOffset code_offset = ArtMethod::EntryPointFromQuickCompiledCodeOffset(kArmPointerSize); 4204 __ LoadFromOffset(kLoadWord, temp, TR, QUICK_ENTRY_POINT(pNewEmptyString)); 4205 __ LoadFromOffset(kLoadWord, LR, temp, code_offset.Int32Value()); 4206 __ blx(LR); 4207 codegen_->RecordPcInfo(instruction, instruction->GetDexPc()); 4208 } else { 4209 codegen_->InvokeRuntime(instruction->GetEntrypoint(), instruction, instruction->GetDexPc()); 4210 CheckEntrypointTypes<kQuickAllocObjectWithChecks, void*, mirror::Class*>(); 4211 } 4212} 4213 4214void LocationsBuilderARM::VisitNewArray(HNewArray* instruction) { 4215 LocationSummary* locations = 4216 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCallOnMainOnly); 4217 InvokeRuntimeCallingConvention calling_convention; 4218 locations->SetOut(Location::RegisterLocation(R0)); 4219 locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(0))); 4220 locations->SetInAt(1, Location::RegisterLocation(calling_convention.GetRegisterAt(1))); 4221} 4222 4223void InstructionCodeGeneratorARM::VisitNewArray(HNewArray* instruction) { 4224 // Note: if heap poisoning is enabled, the entry point takes cares 4225 // of poisoning the reference. 4226 QuickEntrypointEnum entrypoint = 4227 CodeGenerator::GetArrayAllocationEntrypoint(instruction->GetLoadClass()->GetClass()); 4228 codegen_->InvokeRuntime(entrypoint, instruction, instruction->GetDexPc()); 4229 CheckEntrypointTypes<kQuickAllocArrayResolved, void*, mirror::Class*, int32_t>(); 4230 DCHECK(!codegen_->IsLeafMethod()); 4231} 4232 4233void LocationsBuilderARM::VisitParameterValue(HParameterValue* instruction) { 4234 LocationSummary* locations = 4235 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall); 4236 Location location = parameter_visitor_.GetNextLocation(instruction->GetType()); 4237 if (location.IsStackSlot()) { 4238 location = Location::StackSlot(location.GetStackIndex() + codegen_->GetFrameSize()); 4239 } else if (location.IsDoubleStackSlot()) { 4240 location = Location::DoubleStackSlot(location.GetStackIndex() + codegen_->GetFrameSize()); 4241 } 4242 locations->SetOut(location); 4243} 4244 4245void InstructionCodeGeneratorARM::VisitParameterValue( 4246 HParameterValue* instruction ATTRIBUTE_UNUSED) { 4247 // Nothing to do, the parameter is already at its location. 4248} 4249 4250void LocationsBuilderARM::VisitCurrentMethod(HCurrentMethod* instruction) { 4251 LocationSummary* locations = 4252 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall); 4253 locations->SetOut(Location::RegisterLocation(kMethodRegisterArgument)); 4254} 4255 4256void InstructionCodeGeneratorARM::VisitCurrentMethod(HCurrentMethod* instruction ATTRIBUTE_UNUSED) { 4257 // Nothing to do, the method is already at its location. 4258} 4259 4260void LocationsBuilderARM::VisitNot(HNot* not_) { 4261 LocationSummary* locations = 4262 new (GetGraph()->GetArena()) LocationSummary(not_, LocationSummary::kNoCall); 4263 locations->SetInAt(0, Location::RequiresRegister()); 4264 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 4265} 4266 4267void InstructionCodeGeneratorARM::VisitNot(HNot* not_) { 4268 LocationSummary* locations = not_->GetLocations(); 4269 Location out = locations->Out(); 4270 Location in = locations->InAt(0); 4271 switch (not_->GetResultType()) { 4272 case Primitive::kPrimInt: 4273 __ mvn(out.AsRegister<Register>(), ShifterOperand(in.AsRegister<Register>())); 4274 break; 4275 4276 case Primitive::kPrimLong: 4277 __ mvn(out.AsRegisterPairLow<Register>(), 4278 ShifterOperand(in.AsRegisterPairLow<Register>())); 4279 __ mvn(out.AsRegisterPairHigh<Register>(), 4280 ShifterOperand(in.AsRegisterPairHigh<Register>())); 4281 break; 4282 4283 default: 4284 LOG(FATAL) << "Unimplemented type for not operation " << not_->GetResultType(); 4285 } 4286} 4287 4288void LocationsBuilderARM::VisitBooleanNot(HBooleanNot* bool_not) { 4289 LocationSummary* locations = 4290 new (GetGraph()->GetArena()) LocationSummary(bool_not, LocationSummary::kNoCall); 4291 locations->SetInAt(0, Location::RequiresRegister()); 4292 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 4293} 4294 4295void InstructionCodeGeneratorARM::VisitBooleanNot(HBooleanNot* bool_not) { 4296 LocationSummary* locations = bool_not->GetLocations(); 4297 Location out = locations->Out(); 4298 Location in = locations->InAt(0); 4299 __ eor(out.AsRegister<Register>(), in.AsRegister<Register>(), ShifterOperand(1)); 4300} 4301 4302void LocationsBuilderARM::VisitCompare(HCompare* compare) { 4303 LocationSummary* locations = 4304 new (GetGraph()->GetArena()) LocationSummary(compare, LocationSummary::kNoCall); 4305 switch (compare->InputAt(0)->GetType()) { 4306 case Primitive::kPrimBoolean: 4307 case Primitive::kPrimByte: 4308 case Primitive::kPrimShort: 4309 case Primitive::kPrimChar: 4310 case Primitive::kPrimInt: 4311 case Primitive::kPrimLong: { 4312 locations->SetInAt(0, Location::RequiresRegister()); 4313 locations->SetInAt(1, Location::RequiresRegister()); 4314 // Output overlaps because it is written before doing the low comparison. 4315 locations->SetOut(Location::RequiresRegister(), Location::kOutputOverlap); 4316 break; 4317 } 4318 case Primitive::kPrimFloat: 4319 case Primitive::kPrimDouble: { 4320 locations->SetInAt(0, Location::RequiresFpuRegister()); 4321 locations->SetInAt(1, ArithmeticZeroOrFpuRegister(compare->InputAt(1))); 4322 locations->SetOut(Location::RequiresRegister()); 4323 break; 4324 } 4325 default: 4326 LOG(FATAL) << "Unexpected type for compare operation " << compare->InputAt(0)->GetType(); 4327 } 4328} 4329 4330void InstructionCodeGeneratorARM::VisitCompare(HCompare* compare) { 4331 LocationSummary* locations = compare->GetLocations(); 4332 Register out = locations->Out().AsRegister<Register>(); 4333 Location left = locations->InAt(0); 4334 Location right = locations->InAt(1); 4335 4336 Label less, greater, done; 4337 Primitive::Type type = compare->InputAt(0)->GetType(); 4338 Condition less_cond; 4339 switch (type) { 4340 case Primitive::kPrimBoolean: 4341 case Primitive::kPrimByte: 4342 case Primitive::kPrimShort: 4343 case Primitive::kPrimChar: 4344 case Primitive::kPrimInt: { 4345 __ LoadImmediate(out, 0); 4346 __ cmp(left.AsRegister<Register>(), 4347 ShifterOperand(right.AsRegister<Register>())); // Signed compare. 4348 less_cond = LT; 4349 break; 4350 } 4351 case Primitive::kPrimLong: { 4352 __ cmp(left.AsRegisterPairHigh<Register>(), 4353 ShifterOperand(right.AsRegisterPairHigh<Register>())); // Signed compare. 4354 __ b(&less, LT); 4355 __ b(&greater, GT); 4356 // Do LoadImmediate before the last `cmp`, as LoadImmediate might affect the status flags. 4357 __ LoadImmediate(out, 0); 4358 __ cmp(left.AsRegisterPairLow<Register>(), 4359 ShifterOperand(right.AsRegisterPairLow<Register>())); // Unsigned compare. 4360 less_cond = LO; 4361 break; 4362 } 4363 case Primitive::kPrimFloat: 4364 case Primitive::kPrimDouble: { 4365 __ LoadImmediate(out, 0); 4366 GenerateVcmp(compare); 4367 __ vmstat(); // transfer FP status register to ARM APSR. 4368 less_cond = ARMFPCondition(kCondLT, compare->IsGtBias()); 4369 break; 4370 } 4371 default: 4372 LOG(FATAL) << "Unexpected compare type " << type; 4373 UNREACHABLE(); 4374 } 4375 4376 __ b(&done, EQ); 4377 __ b(&less, less_cond); 4378 4379 __ Bind(&greater); 4380 __ LoadImmediate(out, 1); 4381 __ b(&done); 4382 4383 __ Bind(&less); 4384 __ LoadImmediate(out, -1); 4385 4386 __ Bind(&done); 4387} 4388 4389void LocationsBuilderARM::VisitPhi(HPhi* instruction) { 4390 LocationSummary* locations = 4391 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall); 4392 for (size_t i = 0, e = locations->GetInputCount(); i < e; ++i) { 4393 locations->SetInAt(i, Location::Any()); 4394 } 4395 locations->SetOut(Location::Any()); 4396} 4397 4398void InstructionCodeGeneratorARM::VisitPhi(HPhi* instruction ATTRIBUTE_UNUSED) { 4399 LOG(FATAL) << "Unreachable"; 4400} 4401 4402void CodeGeneratorARM::GenerateMemoryBarrier(MemBarrierKind kind) { 4403 // TODO (ported from quick): revisit ARM barrier kinds. 4404 DmbOptions flavor = DmbOptions::ISH; // Quiet C++ warnings. 4405 switch (kind) { 4406 case MemBarrierKind::kAnyStore: 4407 case MemBarrierKind::kLoadAny: 4408 case MemBarrierKind::kAnyAny: { 4409 flavor = DmbOptions::ISH; 4410 break; 4411 } 4412 case MemBarrierKind::kStoreStore: { 4413 flavor = DmbOptions::ISHST; 4414 break; 4415 } 4416 default: 4417 LOG(FATAL) << "Unexpected memory barrier " << kind; 4418 } 4419 __ dmb(flavor); 4420} 4421 4422void InstructionCodeGeneratorARM::GenerateWideAtomicLoad(Register addr, 4423 uint32_t offset, 4424 Register out_lo, 4425 Register out_hi) { 4426 if (offset != 0) { 4427 // Ensure `out_lo` is different from `addr`, so that loading 4428 // `offset` into `out_lo` does not clutter `addr`. 4429 DCHECK_NE(out_lo, addr); 4430 __ LoadImmediate(out_lo, offset); 4431 __ add(IP, addr, ShifterOperand(out_lo)); 4432 addr = IP; 4433 } 4434 __ ldrexd(out_lo, out_hi, addr); 4435} 4436 4437void InstructionCodeGeneratorARM::GenerateWideAtomicStore(Register addr, 4438 uint32_t offset, 4439 Register value_lo, 4440 Register value_hi, 4441 Register temp1, 4442 Register temp2, 4443 HInstruction* instruction) { 4444 Label fail; 4445 if (offset != 0) { 4446 __ LoadImmediate(temp1, offset); 4447 __ add(IP, addr, ShifterOperand(temp1)); 4448 addr = IP; 4449 } 4450 __ Bind(&fail); 4451 // We need a load followed by store. (The address used in a STREX instruction must 4452 // be the same as the address in the most recently executed LDREX instruction.) 4453 __ ldrexd(temp1, temp2, addr); 4454 codegen_->MaybeRecordImplicitNullCheck(instruction); 4455 __ strexd(temp1, value_lo, value_hi, addr); 4456 __ CompareAndBranchIfNonZero(temp1, &fail); 4457} 4458 4459void LocationsBuilderARM::HandleFieldSet(HInstruction* instruction, const FieldInfo& field_info) { 4460 DCHECK(instruction->IsInstanceFieldSet() || instruction->IsStaticFieldSet()); 4461 4462 LocationSummary* locations = 4463 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall); 4464 locations->SetInAt(0, Location::RequiresRegister()); 4465 4466 Primitive::Type field_type = field_info.GetFieldType(); 4467 if (Primitive::IsFloatingPointType(field_type)) { 4468 locations->SetInAt(1, Location::RequiresFpuRegister()); 4469 } else { 4470 locations->SetInAt(1, Location::RequiresRegister()); 4471 } 4472 4473 bool is_wide = field_type == Primitive::kPrimLong || field_type == Primitive::kPrimDouble; 4474 bool generate_volatile = field_info.IsVolatile() 4475 && is_wide 4476 && !codegen_->GetInstructionSetFeatures().HasAtomicLdrdAndStrd(); 4477 bool needs_write_barrier = 4478 CodeGenerator::StoreNeedsWriteBarrier(field_type, instruction->InputAt(1)); 4479 // Temporary registers for the write barrier. 4480 // TODO: consider renaming StoreNeedsWriteBarrier to StoreNeedsGCMark. 4481 if (needs_write_barrier) { 4482 locations->AddTemp(Location::RequiresRegister()); // Possibly used for reference poisoning too. 4483 locations->AddTemp(Location::RequiresRegister()); 4484 } else if (generate_volatile) { 4485 // ARM encoding have some additional constraints for ldrexd/strexd: 4486 // - registers need to be consecutive 4487 // - the first register should be even but not R14. 4488 // We don't test for ARM yet, and the assertion makes sure that we 4489 // revisit this if we ever enable ARM encoding. 4490 DCHECK_EQ(InstructionSet::kThumb2, codegen_->GetInstructionSet()); 4491 4492 locations->AddTemp(Location::RequiresRegister()); 4493 locations->AddTemp(Location::RequiresRegister()); 4494 if (field_type == Primitive::kPrimDouble) { 4495 // For doubles we need two more registers to copy the value. 4496 locations->AddTemp(Location::RegisterLocation(R2)); 4497 locations->AddTemp(Location::RegisterLocation(R3)); 4498 } 4499 } 4500} 4501 4502void InstructionCodeGeneratorARM::HandleFieldSet(HInstruction* instruction, 4503 const FieldInfo& field_info, 4504 bool value_can_be_null) { 4505 DCHECK(instruction->IsInstanceFieldSet() || instruction->IsStaticFieldSet()); 4506 4507 LocationSummary* locations = instruction->GetLocations(); 4508 Register base = locations->InAt(0).AsRegister<Register>(); 4509 Location value = locations->InAt(1); 4510 4511 bool is_volatile = field_info.IsVolatile(); 4512 bool atomic_ldrd_strd = codegen_->GetInstructionSetFeatures().HasAtomicLdrdAndStrd(); 4513 Primitive::Type field_type = field_info.GetFieldType(); 4514 uint32_t offset = field_info.GetFieldOffset().Uint32Value(); 4515 bool needs_write_barrier = 4516 CodeGenerator::StoreNeedsWriteBarrier(field_type, instruction->InputAt(1)); 4517 4518 if (is_volatile) { 4519 codegen_->GenerateMemoryBarrier(MemBarrierKind::kAnyStore); 4520 } 4521 4522 switch (field_type) { 4523 case Primitive::kPrimBoolean: 4524 case Primitive::kPrimByte: { 4525 __ StoreToOffset(kStoreByte, value.AsRegister<Register>(), base, offset); 4526 break; 4527 } 4528 4529 case Primitive::kPrimShort: 4530 case Primitive::kPrimChar: { 4531 __ StoreToOffset(kStoreHalfword, value.AsRegister<Register>(), base, offset); 4532 break; 4533 } 4534 4535 case Primitive::kPrimInt: 4536 case Primitive::kPrimNot: { 4537 if (kPoisonHeapReferences && needs_write_barrier) { 4538 // Note that in the case where `value` is a null reference, 4539 // we do not enter this block, as a null reference does not 4540 // need poisoning. 4541 DCHECK_EQ(field_type, Primitive::kPrimNot); 4542 Register temp = locations->GetTemp(0).AsRegister<Register>(); 4543 __ Mov(temp, value.AsRegister<Register>()); 4544 __ PoisonHeapReference(temp); 4545 __ StoreToOffset(kStoreWord, temp, base, offset); 4546 } else { 4547 __ StoreToOffset(kStoreWord, value.AsRegister<Register>(), base, offset); 4548 } 4549 break; 4550 } 4551 4552 case Primitive::kPrimLong: { 4553 if (is_volatile && !atomic_ldrd_strd) { 4554 GenerateWideAtomicStore(base, offset, 4555 value.AsRegisterPairLow<Register>(), 4556 value.AsRegisterPairHigh<Register>(), 4557 locations->GetTemp(0).AsRegister<Register>(), 4558 locations->GetTemp(1).AsRegister<Register>(), 4559 instruction); 4560 } else { 4561 __ StoreToOffset(kStoreWordPair, value.AsRegisterPairLow<Register>(), base, offset); 4562 codegen_->MaybeRecordImplicitNullCheck(instruction); 4563 } 4564 break; 4565 } 4566 4567 case Primitive::kPrimFloat: { 4568 __ StoreSToOffset(value.AsFpuRegister<SRegister>(), base, offset); 4569 break; 4570 } 4571 4572 case Primitive::kPrimDouble: { 4573 DRegister value_reg = FromLowSToD(value.AsFpuRegisterPairLow<SRegister>()); 4574 if (is_volatile && !atomic_ldrd_strd) { 4575 Register value_reg_lo = locations->GetTemp(0).AsRegister<Register>(); 4576 Register value_reg_hi = locations->GetTemp(1).AsRegister<Register>(); 4577 4578 __ vmovrrd(value_reg_lo, value_reg_hi, value_reg); 4579 4580 GenerateWideAtomicStore(base, offset, 4581 value_reg_lo, 4582 value_reg_hi, 4583 locations->GetTemp(2).AsRegister<Register>(), 4584 locations->GetTemp(3).AsRegister<Register>(), 4585 instruction); 4586 } else { 4587 __ StoreDToOffset(value_reg, base, offset); 4588 codegen_->MaybeRecordImplicitNullCheck(instruction); 4589 } 4590 break; 4591 } 4592 4593 case Primitive::kPrimVoid: 4594 LOG(FATAL) << "Unreachable type " << field_type; 4595 UNREACHABLE(); 4596 } 4597 4598 // Longs and doubles are handled in the switch. 4599 if (field_type != Primitive::kPrimLong && field_type != Primitive::kPrimDouble) { 4600 codegen_->MaybeRecordImplicitNullCheck(instruction); 4601 } 4602 4603 if (CodeGenerator::StoreNeedsWriteBarrier(field_type, instruction->InputAt(1))) { 4604 Register temp = locations->GetTemp(0).AsRegister<Register>(); 4605 Register card = locations->GetTemp(1).AsRegister<Register>(); 4606 codegen_->MarkGCCard( 4607 temp, card, base, value.AsRegister<Register>(), value_can_be_null); 4608 } 4609 4610 if (is_volatile) { 4611 codegen_->GenerateMemoryBarrier(MemBarrierKind::kAnyAny); 4612 } 4613} 4614 4615void LocationsBuilderARM::HandleFieldGet(HInstruction* instruction, const FieldInfo& field_info) { 4616 DCHECK(instruction->IsInstanceFieldGet() || instruction->IsStaticFieldGet()); 4617 4618 bool object_field_get_with_read_barrier = 4619 kEmitCompilerReadBarrier && (field_info.GetFieldType() == Primitive::kPrimNot); 4620 LocationSummary* locations = 4621 new (GetGraph()->GetArena()) LocationSummary(instruction, 4622 object_field_get_with_read_barrier ? 4623 LocationSummary::kCallOnSlowPath : 4624 LocationSummary::kNoCall); 4625 if (object_field_get_with_read_barrier && kUseBakerReadBarrier) { 4626 locations->SetCustomSlowPathCallerSaves(RegisterSet::Empty()); // No caller-save registers. 4627 } 4628 locations->SetInAt(0, Location::RequiresRegister()); 4629 4630 bool volatile_for_double = field_info.IsVolatile() 4631 && (field_info.GetFieldType() == Primitive::kPrimDouble) 4632 && !codegen_->GetInstructionSetFeatures().HasAtomicLdrdAndStrd(); 4633 // The output overlaps in case of volatile long: we don't want the 4634 // code generated by GenerateWideAtomicLoad to overwrite the 4635 // object's location. Likewise, in the case of an object field get 4636 // with read barriers enabled, we do not want the load to overwrite 4637 // the object's location, as we need it to emit the read barrier. 4638 bool overlap = (field_info.IsVolatile() && (field_info.GetFieldType() == Primitive::kPrimLong)) || 4639 object_field_get_with_read_barrier; 4640 4641 if (Primitive::IsFloatingPointType(instruction->GetType())) { 4642 locations->SetOut(Location::RequiresFpuRegister()); 4643 } else { 4644 locations->SetOut(Location::RequiresRegister(), 4645 (overlap ? Location::kOutputOverlap : Location::kNoOutputOverlap)); 4646 } 4647 if (volatile_for_double) { 4648 // ARM encoding have some additional constraints for ldrexd/strexd: 4649 // - registers need to be consecutive 4650 // - the first register should be even but not R14. 4651 // We don't test for ARM yet, and the assertion makes sure that we 4652 // revisit this if we ever enable ARM encoding. 4653 DCHECK_EQ(InstructionSet::kThumb2, codegen_->GetInstructionSet()); 4654 locations->AddTemp(Location::RequiresRegister()); 4655 locations->AddTemp(Location::RequiresRegister()); 4656 } else if (object_field_get_with_read_barrier && kUseBakerReadBarrier) { 4657 // We need a temporary register for the read barrier marking slow 4658 // path in CodeGeneratorARM::GenerateFieldLoadWithBakerReadBarrier. 4659 locations->AddTemp(Location::RequiresRegister()); 4660 } 4661} 4662 4663Location LocationsBuilderARM::ArithmeticZeroOrFpuRegister(HInstruction* input) { 4664 DCHECK(input->GetType() == Primitive::kPrimDouble || input->GetType() == Primitive::kPrimFloat) 4665 << input->GetType(); 4666 if ((input->IsFloatConstant() && (input->AsFloatConstant()->IsArithmeticZero())) || 4667 (input->IsDoubleConstant() && (input->AsDoubleConstant()->IsArithmeticZero()))) { 4668 return Location::ConstantLocation(input->AsConstant()); 4669 } else { 4670 return Location::RequiresFpuRegister(); 4671 } 4672} 4673 4674Location LocationsBuilderARM::ArmEncodableConstantOrRegister(HInstruction* constant, 4675 Opcode opcode) { 4676 DCHECK(!Primitive::IsFloatingPointType(constant->GetType())); 4677 if (constant->IsConstant() && 4678 CanEncodeConstantAsImmediate(constant->AsConstant(), opcode)) { 4679 return Location::ConstantLocation(constant->AsConstant()); 4680 } 4681 return Location::RequiresRegister(); 4682} 4683 4684bool LocationsBuilderARM::CanEncodeConstantAsImmediate(HConstant* input_cst, 4685 Opcode opcode) { 4686 uint64_t value = static_cast<uint64_t>(Int64FromConstant(input_cst)); 4687 if (Primitive::Is64BitType(input_cst->GetType())) { 4688 Opcode high_opcode = opcode; 4689 SetCc low_set_cc = kCcDontCare; 4690 switch (opcode) { 4691 case SUB: 4692 // Flip the operation to an ADD. 4693 value = -value; 4694 opcode = ADD; 4695 FALLTHROUGH_INTENDED; 4696 case ADD: 4697 if (Low32Bits(value) == 0u) { 4698 return CanEncodeConstantAsImmediate(High32Bits(value), opcode, kCcDontCare); 4699 } 4700 high_opcode = ADC; 4701 low_set_cc = kCcSet; 4702 break; 4703 default: 4704 break; 4705 } 4706 return CanEncodeConstantAsImmediate(Low32Bits(value), opcode, low_set_cc) && 4707 CanEncodeConstantAsImmediate(High32Bits(value), high_opcode, kCcDontCare); 4708 } else { 4709 return CanEncodeConstantAsImmediate(Low32Bits(value), opcode); 4710 } 4711} 4712 4713bool LocationsBuilderARM::CanEncodeConstantAsImmediate(uint32_t value, 4714 Opcode opcode, 4715 SetCc set_cc) { 4716 ShifterOperand so; 4717 ArmAssembler* assembler = codegen_->GetAssembler(); 4718 if (assembler->ShifterOperandCanHold(kNoRegister, kNoRegister, opcode, value, set_cc, &so)) { 4719 return true; 4720 } 4721 Opcode neg_opcode = kNoOperand; 4722 switch (opcode) { 4723 case AND: neg_opcode = BIC; value = ~value; break; 4724 case ORR: neg_opcode = ORN; value = ~value; break; 4725 case ADD: neg_opcode = SUB; value = -value; break; 4726 case ADC: neg_opcode = SBC; value = ~value; break; 4727 case SUB: neg_opcode = ADD; value = -value; break; 4728 case SBC: neg_opcode = ADC; value = ~value; break; 4729 default: 4730 return false; 4731 } 4732 return assembler->ShifterOperandCanHold(kNoRegister, kNoRegister, neg_opcode, value, set_cc, &so); 4733} 4734 4735void InstructionCodeGeneratorARM::HandleFieldGet(HInstruction* instruction, 4736 const FieldInfo& field_info) { 4737 DCHECK(instruction->IsInstanceFieldGet() || instruction->IsStaticFieldGet()); 4738 4739 LocationSummary* locations = instruction->GetLocations(); 4740 Location base_loc = locations->InAt(0); 4741 Register base = base_loc.AsRegister<Register>(); 4742 Location out = locations->Out(); 4743 bool is_volatile = field_info.IsVolatile(); 4744 bool atomic_ldrd_strd = codegen_->GetInstructionSetFeatures().HasAtomicLdrdAndStrd(); 4745 Primitive::Type field_type = field_info.GetFieldType(); 4746 uint32_t offset = field_info.GetFieldOffset().Uint32Value(); 4747 4748 switch (field_type) { 4749 case Primitive::kPrimBoolean: 4750 __ LoadFromOffset(kLoadUnsignedByte, out.AsRegister<Register>(), base, offset); 4751 break; 4752 4753 case Primitive::kPrimByte: 4754 __ LoadFromOffset(kLoadSignedByte, out.AsRegister<Register>(), base, offset); 4755 break; 4756 4757 case Primitive::kPrimShort: 4758 __ LoadFromOffset(kLoadSignedHalfword, out.AsRegister<Register>(), base, offset); 4759 break; 4760 4761 case Primitive::kPrimChar: 4762 __ LoadFromOffset(kLoadUnsignedHalfword, out.AsRegister<Register>(), base, offset); 4763 break; 4764 4765 case Primitive::kPrimInt: 4766 __ LoadFromOffset(kLoadWord, out.AsRegister<Register>(), base, offset); 4767 break; 4768 4769 case Primitive::kPrimNot: { 4770 // /* HeapReference<Object> */ out = *(base + offset) 4771 if (kEmitCompilerReadBarrier && kUseBakerReadBarrier) { 4772 Location temp_loc = locations->GetTemp(0); 4773 // Note that a potential implicit null check is handled in this 4774 // CodeGeneratorARM::GenerateFieldLoadWithBakerReadBarrier call. 4775 codegen_->GenerateFieldLoadWithBakerReadBarrier( 4776 instruction, out, base, offset, temp_loc, /* needs_null_check */ true); 4777 if (is_volatile) { 4778 codegen_->GenerateMemoryBarrier(MemBarrierKind::kLoadAny); 4779 } 4780 } else { 4781 __ LoadFromOffset(kLoadWord, out.AsRegister<Register>(), base, offset); 4782 codegen_->MaybeRecordImplicitNullCheck(instruction); 4783 if (is_volatile) { 4784 codegen_->GenerateMemoryBarrier(MemBarrierKind::kLoadAny); 4785 } 4786 // If read barriers are enabled, emit read barriers other than 4787 // Baker's using a slow path (and also unpoison the loaded 4788 // reference, if heap poisoning is enabled). 4789 codegen_->MaybeGenerateReadBarrierSlow(instruction, out, out, base_loc, offset); 4790 } 4791 break; 4792 } 4793 4794 case Primitive::kPrimLong: 4795 if (is_volatile && !atomic_ldrd_strd) { 4796 GenerateWideAtomicLoad(base, offset, 4797 out.AsRegisterPairLow<Register>(), 4798 out.AsRegisterPairHigh<Register>()); 4799 } else { 4800 __ LoadFromOffset(kLoadWordPair, out.AsRegisterPairLow<Register>(), base, offset); 4801 } 4802 break; 4803 4804 case Primitive::kPrimFloat: 4805 __ LoadSFromOffset(out.AsFpuRegister<SRegister>(), base, offset); 4806 break; 4807 4808 case Primitive::kPrimDouble: { 4809 DRegister out_reg = FromLowSToD(out.AsFpuRegisterPairLow<SRegister>()); 4810 if (is_volatile && !atomic_ldrd_strd) { 4811 Register lo = locations->GetTemp(0).AsRegister<Register>(); 4812 Register hi = locations->GetTemp(1).AsRegister<Register>(); 4813 GenerateWideAtomicLoad(base, offset, lo, hi); 4814 codegen_->MaybeRecordImplicitNullCheck(instruction); 4815 __ vmovdrr(out_reg, lo, hi); 4816 } else { 4817 __ LoadDFromOffset(out_reg, base, offset); 4818 codegen_->MaybeRecordImplicitNullCheck(instruction); 4819 } 4820 break; 4821 } 4822 4823 case Primitive::kPrimVoid: 4824 LOG(FATAL) << "Unreachable type " << field_type; 4825 UNREACHABLE(); 4826 } 4827 4828 if (field_type == Primitive::kPrimNot || field_type == Primitive::kPrimDouble) { 4829 // Potential implicit null checks, in the case of reference or 4830 // double fields, are handled in the previous switch statement. 4831 } else { 4832 codegen_->MaybeRecordImplicitNullCheck(instruction); 4833 } 4834 4835 if (is_volatile) { 4836 if (field_type == Primitive::kPrimNot) { 4837 // Memory barriers, in the case of references, are also handled 4838 // in the previous switch statement. 4839 } else { 4840 codegen_->GenerateMemoryBarrier(MemBarrierKind::kLoadAny); 4841 } 4842 } 4843} 4844 4845void LocationsBuilderARM::VisitInstanceFieldSet(HInstanceFieldSet* instruction) { 4846 HandleFieldSet(instruction, instruction->GetFieldInfo()); 4847} 4848 4849void InstructionCodeGeneratorARM::VisitInstanceFieldSet(HInstanceFieldSet* instruction) { 4850 HandleFieldSet(instruction, instruction->GetFieldInfo(), instruction->GetValueCanBeNull()); 4851} 4852 4853void LocationsBuilderARM::VisitInstanceFieldGet(HInstanceFieldGet* instruction) { 4854 HandleFieldGet(instruction, instruction->GetFieldInfo()); 4855} 4856 4857void InstructionCodeGeneratorARM::VisitInstanceFieldGet(HInstanceFieldGet* instruction) { 4858 HandleFieldGet(instruction, instruction->GetFieldInfo()); 4859} 4860 4861void LocationsBuilderARM::VisitStaticFieldGet(HStaticFieldGet* instruction) { 4862 HandleFieldGet(instruction, instruction->GetFieldInfo()); 4863} 4864 4865void InstructionCodeGeneratorARM::VisitStaticFieldGet(HStaticFieldGet* instruction) { 4866 HandleFieldGet(instruction, instruction->GetFieldInfo()); 4867} 4868 4869void LocationsBuilderARM::VisitStaticFieldSet(HStaticFieldSet* instruction) { 4870 HandleFieldSet(instruction, instruction->GetFieldInfo()); 4871} 4872 4873void InstructionCodeGeneratorARM::VisitStaticFieldSet(HStaticFieldSet* instruction) { 4874 HandleFieldSet(instruction, instruction->GetFieldInfo(), instruction->GetValueCanBeNull()); 4875} 4876 4877void LocationsBuilderARM::VisitUnresolvedInstanceFieldGet( 4878 HUnresolvedInstanceFieldGet* instruction) { 4879 FieldAccessCallingConventionARM calling_convention; 4880 codegen_->CreateUnresolvedFieldLocationSummary( 4881 instruction, instruction->GetFieldType(), calling_convention); 4882} 4883 4884void InstructionCodeGeneratorARM::VisitUnresolvedInstanceFieldGet( 4885 HUnresolvedInstanceFieldGet* instruction) { 4886 FieldAccessCallingConventionARM calling_convention; 4887 codegen_->GenerateUnresolvedFieldAccess(instruction, 4888 instruction->GetFieldType(), 4889 instruction->GetFieldIndex(), 4890 instruction->GetDexPc(), 4891 calling_convention); 4892} 4893 4894void LocationsBuilderARM::VisitUnresolvedInstanceFieldSet( 4895 HUnresolvedInstanceFieldSet* instruction) { 4896 FieldAccessCallingConventionARM calling_convention; 4897 codegen_->CreateUnresolvedFieldLocationSummary( 4898 instruction, instruction->GetFieldType(), calling_convention); 4899} 4900 4901void InstructionCodeGeneratorARM::VisitUnresolvedInstanceFieldSet( 4902 HUnresolvedInstanceFieldSet* instruction) { 4903 FieldAccessCallingConventionARM calling_convention; 4904 codegen_->GenerateUnresolvedFieldAccess(instruction, 4905 instruction->GetFieldType(), 4906 instruction->GetFieldIndex(), 4907 instruction->GetDexPc(), 4908 calling_convention); 4909} 4910 4911void LocationsBuilderARM::VisitUnresolvedStaticFieldGet( 4912 HUnresolvedStaticFieldGet* instruction) { 4913 FieldAccessCallingConventionARM calling_convention; 4914 codegen_->CreateUnresolvedFieldLocationSummary( 4915 instruction, instruction->GetFieldType(), calling_convention); 4916} 4917 4918void InstructionCodeGeneratorARM::VisitUnresolvedStaticFieldGet( 4919 HUnresolvedStaticFieldGet* instruction) { 4920 FieldAccessCallingConventionARM calling_convention; 4921 codegen_->GenerateUnresolvedFieldAccess(instruction, 4922 instruction->GetFieldType(), 4923 instruction->GetFieldIndex(), 4924 instruction->GetDexPc(), 4925 calling_convention); 4926} 4927 4928void LocationsBuilderARM::VisitUnresolvedStaticFieldSet( 4929 HUnresolvedStaticFieldSet* instruction) { 4930 FieldAccessCallingConventionARM calling_convention; 4931 codegen_->CreateUnresolvedFieldLocationSummary( 4932 instruction, instruction->GetFieldType(), calling_convention); 4933} 4934 4935void InstructionCodeGeneratorARM::VisitUnresolvedStaticFieldSet( 4936 HUnresolvedStaticFieldSet* instruction) { 4937 FieldAccessCallingConventionARM calling_convention; 4938 codegen_->GenerateUnresolvedFieldAccess(instruction, 4939 instruction->GetFieldType(), 4940 instruction->GetFieldIndex(), 4941 instruction->GetDexPc(), 4942 calling_convention); 4943} 4944 4945void LocationsBuilderARM::VisitNullCheck(HNullCheck* instruction) { 4946 LocationSummary* locations = codegen_->CreateThrowingSlowPathLocations(instruction); 4947 locations->SetInAt(0, Location::RequiresRegister()); 4948} 4949 4950void CodeGeneratorARM::GenerateImplicitNullCheck(HNullCheck* instruction) { 4951 if (CanMoveNullCheckToUser(instruction)) { 4952 return; 4953 } 4954 Location obj = instruction->GetLocations()->InAt(0); 4955 4956 __ LoadFromOffset(kLoadWord, IP, obj.AsRegister<Register>(), 0); 4957 RecordPcInfo(instruction, instruction->GetDexPc()); 4958} 4959 4960void CodeGeneratorARM::GenerateExplicitNullCheck(HNullCheck* instruction) { 4961 SlowPathCodeARM* slow_path = new (GetGraph()->GetArena()) NullCheckSlowPathARM(instruction); 4962 AddSlowPath(slow_path); 4963 4964 LocationSummary* locations = instruction->GetLocations(); 4965 Location obj = locations->InAt(0); 4966 4967 __ CompareAndBranchIfZero(obj.AsRegister<Register>(), slow_path->GetEntryLabel()); 4968} 4969 4970void InstructionCodeGeneratorARM::VisitNullCheck(HNullCheck* instruction) { 4971 codegen_->GenerateNullCheck(instruction); 4972} 4973 4974static LoadOperandType GetLoadOperandType(Primitive::Type type) { 4975 switch (type) { 4976 case Primitive::kPrimNot: 4977 return kLoadWord; 4978 case Primitive::kPrimBoolean: 4979 return kLoadUnsignedByte; 4980 case Primitive::kPrimByte: 4981 return kLoadSignedByte; 4982 case Primitive::kPrimChar: 4983 return kLoadUnsignedHalfword; 4984 case Primitive::kPrimShort: 4985 return kLoadSignedHalfword; 4986 case Primitive::kPrimInt: 4987 return kLoadWord; 4988 case Primitive::kPrimLong: 4989 return kLoadWordPair; 4990 case Primitive::kPrimFloat: 4991 return kLoadSWord; 4992 case Primitive::kPrimDouble: 4993 return kLoadDWord; 4994 default: 4995 LOG(FATAL) << "Unreachable type " << type; 4996 UNREACHABLE(); 4997 } 4998} 4999 5000static StoreOperandType GetStoreOperandType(Primitive::Type type) { 5001 switch (type) { 5002 case Primitive::kPrimNot: 5003 return kStoreWord; 5004 case Primitive::kPrimBoolean: 5005 case Primitive::kPrimByte: 5006 return kStoreByte; 5007 case Primitive::kPrimChar: 5008 case Primitive::kPrimShort: 5009 return kStoreHalfword; 5010 case Primitive::kPrimInt: 5011 return kStoreWord; 5012 case Primitive::kPrimLong: 5013 return kStoreWordPair; 5014 case Primitive::kPrimFloat: 5015 return kStoreSWord; 5016 case Primitive::kPrimDouble: 5017 return kStoreDWord; 5018 default: 5019 LOG(FATAL) << "Unreachable type " << type; 5020 UNREACHABLE(); 5021 } 5022} 5023 5024void CodeGeneratorARM::LoadFromShiftedRegOffset(Primitive::Type type, 5025 Location out_loc, 5026 Register base, 5027 Register reg_offset, 5028 Condition cond) { 5029 uint32_t shift_count = Primitive::ComponentSizeShift(type); 5030 Address mem_address(base, reg_offset, Shift::LSL, shift_count); 5031 5032 switch (type) { 5033 case Primitive::kPrimByte: 5034 __ ldrsb(out_loc.AsRegister<Register>(), mem_address, cond); 5035 break; 5036 case Primitive::kPrimBoolean: 5037 __ ldrb(out_loc.AsRegister<Register>(), mem_address, cond); 5038 break; 5039 case Primitive::kPrimShort: 5040 __ ldrsh(out_loc.AsRegister<Register>(), mem_address, cond); 5041 break; 5042 case Primitive::kPrimChar: 5043 __ ldrh(out_loc.AsRegister<Register>(), mem_address, cond); 5044 break; 5045 case Primitive::kPrimNot: 5046 case Primitive::kPrimInt: 5047 __ ldr(out_loc.AsRegister<Register>(), mem_address, cond); 5048 break; 5049 // T32 doesn't support LoadFromShiftedRegOffset mem address mode for these types. 5050 case Primitive::kPrimLong: 5051 case Primitive::kPrimFloat: 5052 case Primitive::kPrimDouble: 5053 default: 5054 LOG(FATAL) << "Unreachable type " << type; 5055 UNREACHABLE(); 5056 } 5057} 5058 5059void CodeGeneratorARM::StoreToShiftedRegOffset(Primitive::Type type, 5060 Location loc, 5061 Register base, 5062 Register reg_offset, 5063 Condition cond) { 5064 uint32_t shift_count = Primitive::ComponentSizeShift(type); 5065 Address mem_address(base, reg_offset, Shift::LSL, shift_count); 5066 5067 switch (type) { 5068 case Primitive::kPrimByte: 5069 case Primitive::kPrimBoolean: 5070 __ strb(loc.AsRegister<Register>(), mem_address, cond); 5071 break; 5072 case Primitive::kPrimShort: 5073 case Primitive::kPrimChar: 5074 __ strh(loc.AsRegister<Register>(), mem_address, cond); 5075 break; 5076 case Primitive::kPrimNot: 5077 case Primitive::kPrimInt: 5078 __ str(loc.AsRegister<Register>(), mem_address, cond); 5079 break; 5080 // T32 doesn't support StoreToShiftedRegOffset mem address mode for these types. 5081 case Primitive::kPrimLong: 5082 case Primitive::kPrimFloat: 5083 case Primitive::kPrimDouble: 5084 default: 5085 LOG(FATAL) << "Unreachable type " << type; 5086 UNREACHABLE(); 5087 } 5088} 5089 5090void LocationsBuilderARM::VisitArrayGet(HArrayGet* instruction) { 5091 bool object_array_get_with_read_barrier = 5092 kEmitCompilerReadBarrier && (instruction->GetType() == Primitive::kPrimNot); 5093 LocationSummary* locations = 5094 new (GetGraph()->GetArena()) LocationSummary(instruction, 5095 object_array_get_with_read_barrier ? 5096 LocationSummary::kCallOnSlowPath : 5097 LocationSummary::kNoCall); 5098 if (object_array_get_with_read_barrier && kUseBakerReadBarrier) { 5099 locations->SetCustomSlowPathCallerSaves(RegisterSet::Empty()); // No caller-save registers. 5100 } 5101 locations->SetInAt(0, Location::RequiresRegister()); 5102 locations->SetInAt(1, Location::RegisterOrConstant(instruction->InputAt(1))); 5103 if (Primitive::IsFloatingPointType(instruction->GetType())) { 5104 locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap); 5105 } else { 5106 // The output overlaps in the case of an object array get with 5107 // read barriers enabled: we do not want the move to overwrite the 5108 // array's location, as we need it to emit the read barrier. 5109 locations->SetOut( 5110 Location::RequiresRegister(), 5111 object_array_get_with_read_barrier ? Location::kOutputOverlap : Location::kNoOutputOverlap); 5112 } 5113 // We need a temporary register for the read barrier marking slow 5114 // path in CodeGeneratorARM::GenerateArrayLoadWithBakerReadBarrier. 5115 // Also need for String compression feature. 5116 if ((object_array_get_with_read_barrier && kUseBakerReadBarrier) 5117 || (mirror::kUseStringCompression && instruction->IsStringCharAt())) { 5118 locations->AddTemp(Location::RequiresRegister()); 5119 } 5120} 5121 5122void InstructionCodeGeneratorARM::VisitArrayGet(HArrayGet* instruction) { 5123 LocationSummary* locations = instruction->GetLocations(); 5124 Location obj_loc = locations->InAt(0); 5125 Register obj = obj_loc.AsRegister<Register>(); 5126 Location index = locations->InAt(1); 5127 Location out_loc = locations->Out(); 5128 uint32_t data_offset = CodeGenerator::GetArrayDataOffset(instruction); 5129 Primitive::Type type = instruction->GetType(); 5130 const bool maybe_compressed_char_at = mirror::kUseStringCompression && 5131 instruction->IsStringCharAt(); 5132 HInstruction* array_instr = instruction->GetArray(); 5133 bool has_intermediate_address = array_instr->IsIntermediateAddress(); 5134 5135 switch (type) { 5136 case Primitive::kPrimBoolean: 5137 case Primitive::kPrimByte: 5138 case Primitive::kPrimShort: 5139 case Primitive::kPrimChar: 5140 case Primitive::kPrimInt: { 5141 Register length; 5142 if (maybe_compressed_char_at) { 5143 length = locations->GetTemp(0).AsRegister<Register>(); 5144 uint32_t count_offset = mirror::String::CountOffset().Uint32Value(); 5145 __ LoadFromOffset(kLoadWord, length, obj, count_offset); 5146 codegen_->MaybeRecordImplicitNullCheck(instruction); 5147 } 5148 if (index.IsConstant()) { 5149 int32_t const_index = index.GetConstant()->AsIntConstant()->GetValue(); 5150 if (maybe_compressed_char_at) { 5151 Label uncompressed_load, done; 5152 __ Lsrs(length, length, 1u); // LSRS has a 16-bit encoding, TST (immediate) does not. 5153 static_assert(static_cast<uint32_t>(mirror::StringCompressionFlag::kCompressed) == 0u, 5154 "Expecting 0=compressed, 1=uncompressed"); 5155 __ b(&uncompressed_load, CS); 5156 __ LoadFromOffset(kLoadUnsignedByte, 5157 out_loc.AsRegister<Register>(), 5158 obj, 5159 data_offset + const_index); 5160 __ b(&done); 5161 __ Bind(&uncompressed_load); 5162 __ LoadFromOffset(GetLoadOperandType(Primitive::kPrimChar), 5163 out_loc.AsRegister<Register>(), 5164 obj, 5165 data_offset + (const_index << 1)); 5166 __ Bind(&done); 5167 } else { 5168 uint32_t full_offset = data_offset + (const_index << Primitive::ComponentSizeShift(type)); 5169 5170 LoadOperandType load_type = GetLoadOperandType(type); 5171 __ LoadFromOffset(load_type, out_loc.AsRegister<Register>(), obj, full_offset); 5172 } 5173 } else { 5174 Register temp = IP; 5175 5176 if (has_intermediate_address) { 5177 // We do not need to compute the intermediate address from the array: the 5178 // input instruction has done it already. See the comment in 5179 // `TryExtractArrayAccessAddress()`. 5180 if (kIsDebugBuild) { 5181 HIntermediateAddress* tmp = array_instr->AsIntermediateAddress(); 5182 DCHECK_EQ(tmp->GetOffset()->AsIntConstant()->GetValueAsUint64(), data_offset); 5183 } 5184 temp = obj; 5185 } else { 5186 __ add(temp, obj, ShifterOperand(data_offset)); 5187 } 5188 if (maybe_compressed_char_at) { 5189 Label uncompressed_load, done; 5190 __ Lsrs(length, length, 1u); // LSRS has a 16-bit encoding, TST (immediate) does not. 5191 static_assert(static_cast<uint32_t>(mirror::StringCompressionFlag::kCompressed) == 0u, 5192 "Expecting 0=compressed, 1=uncompressed"); 5193 __ b(&uncompressed_load, CS); 5194 __ ldrb(out_loc.AsRegister<Register>(), 5195 Address(temp, index.AsRegister<Register>(), Shift::LSL, 0)); 5196 __ b(&done); 5197 __ Bind(&uncompressed_load); 5198 __ ldrh(out_loc.AsRegister<Register>(), 5199 Address(temp, index.AsRegister<Register>(), Shift::LSL, 1)); 5200 __ Bind(&done); 5201 } else { 5202 codegen_->LoadFromShiftedRegOffset(type, out_loc, temp, index.AsRegister<Register>()); 5203 } 5204 } 5205 break; 5206 } 5207 5208 case Primitive::kPrimNot: { 5209 // The read barrier instrumentation of object ArrayGet 5210 // instructions does not support the HIntermediateAddress 5211 // instruction. 5212 DCHECK(!(has_intermediate_address && kEmitCompilerReadBarrier)); 5213 5214 static_assert( 5215 sizeof(mirror::HeapReference<mirror::Object>) == sizeof(int32_t), 5216 "art::mirror::HeapReference<art::mirror::Object> and int32_t have different sizes."); 5217 // /* HeapReference<Object> */ out = 5218 // *(obj + data_offset + index * sizeof(HeapReference<Object>)) 5219 if (kEmitCompilerReadBarrier && kUseBakerReadBarrier) { 5220 Location temp = locations->GetTemp(0); 5221 // Note that a potential implicit null check is handled in this 5222 // CodeGeneratorARM::GenerateArrayLoadWithBakerReadBarrier call. 5223 codegen_->GenerateArrayLoadWithBakerReadBarrier( 5224 instruction, out_loc, obj, data_offset, index, temp, /* needs_null_check */ true); 5225 } else { 5226 Register out = out_loc.AsRegister<Register>(); 5227 if (index.IsConstant()) { 5228 size_t offset = 5229 (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4) + data_offset; 5230 __ LoadFromOffset(kLoadWord, out, obj, offset); 5231 codegen_->MaybeRecordImplicitNullCheck(instruction); 5232 // If read barriers are enabled, emit read barriers other than 5233 // Baker's using a slow path (and also unpoison the loaded 5234 // reference, if heap poisoning is enabled). 5235 codegen_->MaybeGenerateReadBarrierSlow(instruction, out_loc, out_loc, obj_loc, offset); 5236 } else { 5237 Register temp = IP; 5238 5239 if (has_intermediate_address) { 5240 // We do not need to compute the intermediate address from the array: the 5241 // input instruction has done it already. See the comment in 5242 // `TryExtractArrayAccessAddress()`. 5243 if (kIsDebugBuild) { 5244 HIntermediateAddress* tmp = array_instr->AsIntermediateAddress(); 5245 DCHECK_EQ(tmp->GetOffset()->AsIntConstant()->GetValueAsUint64(), data_offset); 5246 } 5247 temp = obj; 5248 } else { 5249 __ add(temp, obj, ShifterOperand(data_offset)); 5250 } 5251 codegen_->LoadFromShiftedRegOffset(type, out_loc, temp, index.AsRegister<Register>()); 5252 5253 codegen_->MaybeRecordImplicitNullCheck(instruction); 5254 // If read barriers are enabled, emit read barriers other than 5255 // Baker's using a slow path (and also unpoison the loaded 5256 // reference, if heap poisoning is enabled). 5257 codegen_->MaybeGenerateReadBarrierSlow( 5258 instruction, out_loc, out_loc, obj_loc, data_offset, index); 5259 } 5260 } 5261 break; 5262 } 5263 5264 case Primitive::kPrimLong: { 5265 if (index.IsConstant()) { 5266 size_t offset = 5267 (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_8) + data_offset; 5268 __ LoadFromOffset(kLoadWordPair, out_loc.AsRegisterPairLow<Register>(), obj, offset); 5269 } else { 5270 __ add(IP, obj, ShifterOperand(index.AsRegister<Register>(), LSL, TIMES_8)); 5271 __ LoadFromOffset(kLoadWordPair, out_loc.AsRegisterPairLow<Register>(), IP, data_offset); 5272 } 5273 break; 5274 } 5275 5276 case Primitive::kPrimFloat: { 5277 SRegister out = out_loc.AsFpuRegister<SRegister>(); 5278 if (index.IsConstant()) { 5279 size_t offset = (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4) + data_offset; 5280 __ LoadSFromOffset(out, obj, offset); 5281 } else { 5282 __ add(IP, obj, ShifterOperand(index.AsRegister<Register>(), LSL, TIMES_4)); 5283 __ LoadSFromOffset(out, IP, data_offset); 5284 } 5285 break; 5286 } 5287 5288 case Primitive::kPrimDouble: { 5289 SRegister out = out_loc.AsFpuRegisterPairLow<SRegister>(); 5290 if (index.IsConstant()) { 5291 size_t offset = (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_8) + data_offset; 5292 __ LoadDFromOffset(FromLowSToD(out), obj, offset); 5293 } else { 5294 __ add(IP, obj, ShifterOperand(index.AsRegister<Register>(), LSL, TIMES_8)); 5295 __ LoadDFromOffset(FromLowSToD(out), IP, data_offset); 5296 } 5297 break; 5298 } 5299 5300 case Primitive::kPrimVoid: 5301 LOG(FATAL) << "Unreachable type " << type; 5302 UNREACHABLE(); 5303 } 5304 5305 if (type == Primitive::kPrimNot) { 5306 // Potential implicit null checks, in the case of reference 5307 // arrays, are handled in the previous switch statement. 5308 } else if (!maybe_compressed_char_at) { 5309 codegen_->MaybeRecordImplicitNullCheck(instruction); 5310 } 5311} 5312 5313void LocationsBuilderARM::VisitArraySet(HArraySet* instruction) { 5314 Primitive::Type value_type = instruction->GetComponentType(); 5315 5316 bool needs_write_barrier = 5317 CodeGenerator::StoreNeedsWriteBarrier(value_type, instruction->GetValue()); 5318 bool may_need_runtime_call_for_type_check = instruction->NeedsTypeCheck(); 5319 5320 LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary( 5321 instruction, 5322 may_need_runtime_call_for_type_check ? 5323 LocationSummary::kCallOnSlowPath : 5324 LocationSummary::kNoCall); 5325 5326 locations->SetInAt(0, Location::RequiresRegister()); 5327 locations->SetInAt(1, Location::RegisterOrConstant(instruction->InputAt(1))); 5328 if (Primitive::IsFloatingPointType(value_type)) { 5329 locations->SetInAt(2, Location::RequiresFpuRegister()); 5330 } else { 5331 locations->SetInAt(2, Location::RequiresRegister()); 5332 } 5333 if (needs_write_barrier) { 5334 // Temporary registers for the write barrier. 5335 locations->AddTemp(Location::RequiresRegister()); // Possibly used for ref. poisoning too. 5336 locations->AddTemp(Location::RequiresRegister()); 5337 } 5338} 5339 5340void InstructionCodeGeneratorARM::VisitArraySet(HArraySet* instruction) { 5341 LocationSummary* locations = instruction->GetLocations(); 5342 Location array_loc = locations->InAt(0); 5343 Register array = array_loc.AsRegister<Register>(); 5344 Location index = locations->InAt(1); 5345 Primitive::Type value_type = instruction->GetComponentType(); 5346 bool may_need_runtime_call_for_type_check = instruction->NeedsTypeCheck(); 5347 bool needs_write_barrier = 5348 CodeGenerator::StoreNeedsWriteBarrier(value_type, instruction->GetValue()); 5349 uint32_t data_offset = 5350 mirror::Array::DataOffset(Primitive::ComponentSize(value_type)).Uint32Value(); 5351 Location value_loc = locations->InAt(2); 5352 HInstruction* array_instr = instruction->GetArray(); 5353 bool has_intermediate_address = array_instr->IsIntermediateAddress(); 5354 5355 switch (value_type) { 5356 case Primitive::kPrimBoolean: 5357 case Primitive::kPrimByte: 5358 case Primitive::kPrimShort: 5359 case Primitive::kPrimChar: 5360 case Primitive::kPrimInt: { 5361 if (index.IsConstant()) { 5362 int32_t const_index = index.GetConstant()->AsIntConstant()->GetValue(); 5363 uint32_t full_offset = 5364 data_offset + (const_index << Primitive::ComponentSizeShift(value_type)); 5365 StoreOperandType store_type = GetStoreOperandType(value_type); 5366 __ StoreToOffset(store_type, value_loc.AsRegister<Register>(), array, full_offset); 5367 } else { 5368 Register temp = IP; 5369 5370 if (has_intermediate_address) { 5371 // We do not need to compute the intermediate address from the array: the 5372 // input instruction has done it already. See the comment in 5373 // `TryExtractArrayAccessAddress()`. 5374 if (kIsDebugBuild) { 5375 HIntermediateAddress* tmp = array_instr->AsIntermediateAddress(); 5376 DCHECK(tmp->GetOffset()->AsIntConstant()->GetValueAsUint64() == data_offset); 5377 } 5378 temp = array; 5379 } else { 5380 __ add(temp, array, ShifterOperand(data_offset)); 5381 } 5382 codegen_->StoreToShiftedRegOffset(value_type, 5383 value_loc, 5384 temp, 5385 index.AsRegister<Register>()); 5386 } 5387 break; 5388 } 5389 5390 case Primitive::kPrimNot: { 5391 Register value = value_loc.AsRegister<Register>(); 5392 // TryExtractArrayAccessAddress optimization is never applied for non-primitive ArraySet. 5393 // See the comment in instruction_simplifier_shared.cc. 5394 DCHECK(!has_intermediate_address); 5395 5396 if (instruction->InputAt(2)->IsNullConstant()) { 5397 // Just setting null. 5398 if (index.IsConstant()) { 5399 size_t offset = 5400 (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4) + data_offset; 5401 __ StoreToOffset(kStoreWord, value, array, offset); 5402 } else { 5403 DCHECK(index.IsRegister()) << index; 5404 __ add(IP, array, ShifterOperand(data_offset)); 5405 codegen_->StoreToShiftedRegOffset(value_type, 5406 value_loc, 5407 IP, 5408 index.AsRegister<Register>()); 5409 } 5410 codegen_->MaybeRecordImplicitNullCheck(instruction); 5411 DCHECK(!needs_write_barrier); 5412 DCHECK(!may_need_runtime_call_for_type_check); 5413 break; 5414 } 5415 5416 DCHECK(needs_write_barrier); 5417 Location temp1_loc = locations->GetTemp(0); 5418 Register temp1 = temp1_loc.AsRegister<Register>(); 5419 Location temp2_loc = locations->GetTemp(1); 5420 Register temp2 = temp2_loc.AsRegister<Register>(); 5421 uint32_t class_offset = mirror::Object::ClassOffset().Int32Value(); 5422 uint32_t super_offset = mirror::Class::SuperClassOffset().Int32Value(); 5423 uint32_t component_offset = mirror::Class::ComponentTypeOffset().Int32Value(); 5424 Label done; 5425 SlowPathCodeARM* slow_path = nullptr; 5426 5427 if (may_need_runtime_call_for_type_check) { 5428 slow_path = new (GetGraph()->GetArena()) ArraySetSlowPathARM(instruction); 5429 codegen_->AddSlowPath(slow_path); 5430 if (instruction->GetValueCanBeNull()) { 5431 Label non_zero; 5432 __ CompareAndBranchIfNonZero(value, &non_zero); 5433 if (index.IsConstant()) { 5434 size_t offset = 5435 (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4) + data_offset; 5436 __ StoreToOffset(kStoreWord, value, array, offset); 5437 } else { 5438 DCHECK(index.IsRegister()) << index; 5439 __ add(IP, array, ShifterOperand(data_offset)); 5440 codegen_->StoreToShiftedRegOffset(value_type, 5441 value_loc, 5442 IP, 5443 index.AsRegister<Register>()); 5444 } 5445 codegen_->MaybeRecordImplicitNullCheck(instruction); 5446 __ b(&done); 5447 __ Bind(&non_zero); 5448 } 5449 5450 // Note that when read barriers are enabled, the type checks 5451 // are performed without read barriers. This is fine, even in 5452 // the case where a class object is in the from-space after 5453 // the flip, as a comparison involving such a type would not 5454 // produce a false positive; it may of course produce a false 5455 // negative, in which case we would take the ArraySet slow 5456 // path. 5457 5458 // /* HeapReference<Class> */ temp1 = array->klass_ 5459 __ LoadFromOffset(kLoadWord, temp1, array, class_offset); 5460 codegen_->MaybeRecordImplicitNullCheck(instruction); 5461 __ MaybeUnpoisonHeapReference(temp1); 5462 5463 // /* HeapReference<Class> */ temp1 = temp1->component_type_ 5464 __ LoadFromOffset(kLoadWord, temp1, temp1, component_offset); 5465 // /* HeapReference<Class> */ temp2 = value->klass_ 5466 __ LoadFromOffset(kLoadWord, temp2, value, class_offset); 5467 // If heap poisoning is enabled, no need to unpoison `temp1` 5468 // nor `temp2`, as we are comparing two poisoned references. 5469 __ cmp(temp1, ShifterOperand(temp2)); 5470 5471 if (instruction->StaticTypeOfArrayIsObjectArray()) { 5472 Label do_put; 5473 __ b(&do_put, EQ); 5474 // If heap poisoning is enabled, the `temp1` reference has 5475 // not been unpoisoned yet; unpoison it now. 5476 __ MaybeUnpoisonHeapReference(temp1); 5477 5478 // /* HeapReference<Class> */ temp1 = temp1->super_class_ 5479 __ LoadFromOffset(kLoadWord, temp1, temp1, super_offset); 5480 // If heap poisoning is enabled, no need to unpoison 5481 // `temp1`, as we are comparing against null below. 5482 __ CompareAndBranchIfNonZero(temp1, slow_path->GetEntryLabel()); 5483 __ Bind(&do_put); 5484 } else { 5485 __ b(slow_path->GetEntryLabel(), NE); 5486 } 5487 } 5488 5489 Register source = value; 5490 if (kPoisonHeapReferences) { 5491 // Note that in the case where `value` is a null reference, 5492 // we do not enter this block, as a null reference does not 5493 // need poisoning. 5494 DCHECK_EQ(value_type, Primitive::kPrimNot); 5495 __ Mov(temp1, value); 5496 __ PoisonHeapReference(temp1); 5497 source = temp1; 5498 } 5499 5500 if (index.IsConstant()) { 5501 size_t offset = 5502 (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4) + data_offset; 5503 __ StoreToOffset(kStoreWord, source, array, offset); 5504 } else { 5505 DCHECK(index.IsRegister()) << index; 5506 5507 __ add(IP, array, ShifterOperand(data_offset)); 5508 codegen_->StoreToShiftedRegOffset(value_type, 5509 Location::RegisterLocation(source), 5510 IP, 5511 index.AsRegister<Register>()); 5512 } 5513 5514 if (!may_need_runtime_call_for_type_check) { 5515 codegen_->MaybeRecordImplicitNullCheck(instruction); 5516 } 5517 5518 codegen_->MarkGCCard(temp1, temp2, array, value, instruction->GetValueCanBeNull()); 5519 5520 if (done.IsLinked()) { 5521 __ Bind(&done); 5522 } 5523 5524 if (slow_path != nullptr) { 5525 __ Bind(slow_path->GetExitLabel()); 5526 } 5527 5528 break; 5529 } 5530 5531 case Primitive::kPrimLong: { 5532 Location value = locations->InAt(2); 5533 if (index.IsConstant()) { 5534 size_t offset = 5535 (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_8) + data_offset; 5536 __ StoreToOffset(kStoreWordPair, value.AsRegisterPairLow<Register>(), array, offset); 5537 } else { 5538 __ add(IP, array, ShifterOperand(index.AsRegister<Register>(), LSL, TIMES_8)); 5539 __ StoreToOffset(kStoreWordPair, value.AsRegisterPairLow<Register>(), IP, data_offset); 5540 } 5541 break; 5542 } 5543 5544 case Primitive::kPrimFloat: { 5545 Location value = locations->InAt(2); 5546 DCHECK(value.IsFpuRegister()); 5547 if (index.IsConstant()) { 5548 size_t offset = (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_4) + data_offset; 5549 __ StoreSToOffset(value.AsFpuRegister<SRegister>(), array, offset); 5550 } else { 5551 __ add(IP, array, ShifterOperand(index.AsRegister<Register>(), LSL, TIMES_4)); 5552 __ StoreSToOffset(value.AsFpuRegister<SRegister>(), IP, data_offset); 5553 } 5554 break; 5555 } 5556 5557 case Primitive::kPrimDouble: { 5558 Location value = locations->InAt(2); 5559 DCHECK(value.IsFpuRegisterPair()); 5560 if (index.IsConstant()) { 5561 size_t offset = (index.GetConstant()->AsIntConstant()->GetValue() << TIMES_8) + data_offset; 5562 __ StoreDToOffset(FromLowSToD(value.AsFpuRegisterPairLow<SRegister>()), array, offset); 5563 } else { 5564 __ add(IP, array, ShifterOperand(index.AsRegister<Register>(), LSL, TIMES_8)); 5565 __ StoreDToOffset(FromLowSToD(value.AsFpuRegisterPairLow<SRegister>()), IP, data_offset); 5566 } 5567 5568 break; 5569 } 5570 5571 case Primitive::kPrimVoid: 5572 LOG(FATAL) << "Unreachable type " << value_type; 5573 UNREACHABLE(); 5574 } 5575 5576 // Objects are handled in the switch. 5577 if (value_type != Primitive::kPrimNot) { 5578 codegen_->MaybeRecordImplicitNullCheck(instruction); 5579 } 5580} 5581 5582void LocationsBuilderARM::VisitArrayLength(HArrayLength* instruction) { 5583 LocationSummary* locations = 5584 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall); 5585 locations->SetInAt(0, Location::RequiresRegister()); 5586 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 5587} 5588 5589void InstructionCodeGeneratorARM::VisitArrayLength(HArrayLength* instruction) { 5590 LocationSummary* locations = instruction->GetLocations(); 5591 uint32_t offset = CodeGenerator::GetArrayLengthOffset(instruction); 5592 Register obj = locations->InAt(0).AsRegister<Register>(); 5593 Register out = locations->Out().AsRegister<Register>(); 5594 __ LoadFromOffset(kLoadWord, out, obj, offset); 5595 codegen_->MaybeRecordImplicitNullCheck(instruction); 5596 // Mask out compression flag from String's array length. 5597 if (mirror::kUseStringCompression && instruction->IsStringLength()) { 5598 __ Lsr(out, out, 1u); 5599 } 5600} 5601 5602void LocationsBuilderARM::VisitIntermediateAddress(HIntermediateAddress* instruction) { 5603 LocationSummary* locations = 5604 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall); 5605 5606 locations->SetInAt(0, Location::RequiresRegister()); 5607 locations->SetInAt(1, Location::RegisterOrConstant(instruction->GetOffset())); 5608 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 5609} 5610 5611void InstructionCodeGeneratorARM::VisitIntermediateAddress(HIntermediateAddress* instruction) { 5612 LocationSummary* locations = instruction->GetLocations(); 5613 Location out = locations->Out(); 5614 Location first = locations->InAt(0); 5615 Location second = locations->InAt(1); 5616 5617 if (second.IsRegister()) { 5618 __ add(out.AsRegister<Register>(), 5619 first.AsRegister<Register>(), 5620 ShifterOperand(second.AsRegister<Register>())); 5621 } else { 5622 __ AddConstant(out.AsRegister<Register>(), 5623 first.AsRegister<Register>(), 5624 second.GetConstant()->AsIntConstant()->GetValue()); 5625 } 5626} 5627 5628void LocationsBuilderARM::VisitBoundsCheck(HBoundsCheck* instruction) { 5629 RegisterSet caller_saves = RegisterSet::Empty(); 5630 InvokeRuntimeCallingConvention calling_convention; 5631 caller_saves.Add(Location::RegisterLocation(calling_convention.GetRegisterAt(0))); 5632 caller_saves.Add(Location::RegisterLocation(calling_convention.GetRegisterAt(1))); 5633 LocationSummary* locations = codegen_->CreateThrowingSlowPathLocations(instruction, caller_saves); 5634 locations->SetInAt(0, Location::RequiresRegister()); 5635 locations->SetInAt(1, Location::RequiresRegister()); 5636} 5637 5638void InstructionCodeGeneratorARM::VisitBoundsCheck(HBoundsCheck* instruction) { 5639 LocationSummary* locations = instruction->GetLocations(); 5640 SlowPathCodeARM* slow_path = 5641 new (GetGraph()->GetArena()) BoundsCheckSlowPathARM(instruction); 5642 codegen_->AddSlowPath(slow_path); 5643 5644 Register index = locations->InAt(0).AsRegister<Register>(); 5645 Register length = locations->InAt(1).AsRegister<Register>(); 5646 5647 __ cmp(index, ShifterOperand(length)); 5648 __ b(slow_path->GetEntryLabel(), HS); 5649} 5650 5651void CodeGeneratorARM::MarkGCCard(Register temp, 5652 Register card, 5653 Register object, 5654 Register value, 5655 bool can_be_null) { 5656 Label is_null; 5657 if (can_be_null) { 5658 __ CompareAndBranchIfZero(value, &is_null); 5659 } 5660 __ LoadFromOffset(kLoadWord, card, TR, Thread::CardTableOffset<kArmPointerSize>().Int32Value()); 5661 __ Lsr(temp, object, gc::accounting::CardTable::kCardShift); 5662 __ strb(card, Address(card, temp)); 5663 if (can_be_null) { 5664 __ Bind(&is_null); 5665 } 5666} 5667 5668void LocationsBuilderARM::VisitParallelMove(HParallelMove* instruction ATTRIBUTE_UNUSED) { 5669 LOG(FATAL) << "Unreachable"; 5670} 5671 5672void InstructionCodeGeneratorARM::VisitParallelMove(HParallelMove* instruction) { 5673 codegen_->GetMoveResolver()->EmitNativeCode(instruction); 5674} 5675 5676void LocationsBuilderARM::VisitSuspendCheck(HSuspendCheck* instruction) { 5677 LocationSummary* locations = 5678 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCallOnSlowPath); 5679 locations->SetCustomSlowPathCallerSaves(RegisterSet::Empty()); // No caller-save registers. 5680} 5681 5682void InstructionCodeGeneratorARM::VisitSuspendCheck(HSuspendCheck* instruction) { 5683 HBasicBlock* block = instruction->GetBlock(); 5684 if (block->GetLoopInformation() != nullptr) { 5685 DCHECK(block->GetLoopInformation()->GetSuspendCheck() == instruction); 5686 // The back edge will generate the suspend check. 5687 return; 5688 } 5689 if (block->IsEntryBlock() && instruction->GetNext()->IsGoto()) { 5690 // The goto will generate the suspend check. 5691 return; 5692 } 5693 GenerateSuspendCheck(instruction, nullptr); 5694} 5695 5696void InstructionCodeGeneratorARM::GenerateSuspendCheck(HSuspendCheck* instruction, 5697 HBasicBlock* successor) { 5698 SuspendCheckSlowPathARM* slow_path = 5699 down_cast<SuspendCheckSlowPathARM*>(instruction->GetSlowPath()); 5700 if (slow_path == nullptr) { 5701 slow_path = new (GetGraph()->GetArena()) SuspendCheckSlowPathARM(instruction, successor); 5702 instruction->SetSlowPath(slow_path); 5703 codegen_->AddSlowPath(slow_path); 5704 if (successor != nullptr) { 5705 DCHECK(successor->IsLoopHeader()); 5706 codegen_->ClearSpillSlotsFromLoopPhisInStackMap(instruction); 5707 } 5708 } else { 5709 DCHECK_EQ(slow_path->GetSuccessor(), successor); 5710 } 5711 5712 __ LoadFromOffset( 5713 kLoadUnsignedHalfword, IP, TR, Thread::ThreadFlagsOffset<kArmPointerSize>().Int32Value()); 5714 if (successor == nullptr) { 5715 __ CompareAndBranchIfNonZero(IP, slow_path->GetEntryLabel()); 5716 __ Bind(slow_path->GetReturnLabel()); 5717 } else { 5718 __ CompareAndBranchIfZero(IP, codegen_->GetLabelOf(successor)); 5719 __ b(slow_path->GetEntryLabel()); 5720 } 5721} 5722 5723ArmAssembler* ParallelMoveResolverARM::GetAssembler() const { 5724 return codegen_->GetAssembler(); 5725} 5726 5727void ParallelMoveResolverARM::EmitMove(size_t index) { 5728 MoveOperands* move = moves_[index]; 5729 Location source = move->GetSource(); 5730 Location destination = move->GetDestination(); 5731 5732 if (source.IsRegister()) { 5733 if (destination.IsRegister()) { 5734 __ Mov(destination.AsRegister<Register>(), source.AsRegister<Register>()); 5735 } else if (destination.IsFpuRegister()) { 5736 __ vmovsr(destination.AsFpuRegister<SRegister>(), source.AsRegister<Register>()); 5737 } else { 5738 DCHECK(destination.IsStackSlot()); 5739 __ StoreToOffset(kStoreWord, source.AsRegister<Register>(), 5740 SP, destination.GetStackIndex()); 5741 } 5742 } else if (source.IsStackSlot()) { 5743 if (destination.IsRegister()) { 5744 __ LoadFromOffset(kLoadWord, destination.AsRegister<Register>(), 5745 SP, source.GetStackIndex()); 5746 } else if (destination.IsFpuRegister()) { 5747 __ LoadSFromOffset(destination.AsFpuRegister<SRegister>(), SP, source.GetStackIndex()); 5748 } else { 5749 DCHECK(destination.IsStackSlot()); 5750 __ LoadFromOffset(kLoadWord, IP, SP, source.GetStackIndex()); 5751 __ StoreToOffset(kStoreWord, IP, SP, destination.GetStackIndex()); 5752 } 5753 } else if (source.IsFpuRegister()) { 5754 if (destination.IsRegister()) { 5755 __ vmovrs(destination.AsRegister<Register>(), source.AsFpuRegister<SRegister>()); 5756 } else if (destination.IsFpuRegister()) { 5757 __ vmovs(destination.AsFpuRegister<SRegister>(), source.AsFpuRegister<SRegister>()); 5758 } else { 5759 DCHECK(destination.IsStackSlot()); 5760 __ StoreSToOffset(source.AsFpuRegister<SRegister>(), SP, destination.GetStackIndex()); 5761 } 5762 } else if (source.IsDoubleStackSlot()) { 5763 if (destination.IsDoubleStackSlot()) { 5764 __ LoadDFromOffset(DTMP, SP, source.GetStackIndex()); 5765 __ StoreDToOffset(DTMP, SP, destination.GetStackIndex()); 5766 } else if (destination.IsRegisterPair()) { 5767 DCHECK(ExpectedPairLayout(destination)); 5768 __ LoadFromOffset( 5769 kLoadWordPair, destination.AsRegisterPairLow<Register>(), SP, source.GetStackIndex()); 5770 } else { 5771 DCHECK(destination.IsFpuRegisterPair()) << destination; 5772 __ LoadDFromOffset(FromLowSToD(destination.AsFpuRegisterPairLow<SRegister>()), 5773 SP, 5774 source.GetStackIndex()); 5775 } 5776 } else if (source.IsRegisterPair()) { 5777 if (destination.IsRegisterPair()) { 5778 __ Mov(destination.AsRegisterPairLow<Register>(), source.AsRegisterPairLow<Register>()); 5779 __ Mov(destination.AsRegisterPairHigh<Register>(), source.AsRegisterPairHigh<Register>()); 5780 } else if (destination.IsFpuRegisterPair()) { 5781 __ vmovdrr(FromLowSToD(destination.AsFpuRegisterPairLow<SRegister>()), 5782 source.AsRegisterPairLow<Register>(), 5783 source.AsRegisterPairHigh<Register>()); 5784 } else { 5785 DCHECK(destination.IsDoubleStackSlot()) << destination; 5786 DCHECK(ExpectedPairLayout(source)); 5787 __ StoreToOffset( 5788 kStoreWordPair, source.AsRegisterPairLow<Register>(), SP, destination.GetStackIndex()); 5789 } 5790 } else if (source.IsFpuRegisterPair()) { 5791 if (destination.IsRegisterPair()) { 5792 __ vmovrrd(destination.AsRegisterPairLow<Register>(), 5793 destination.AsRegisterPairHigh<Register>(), 5794 FromLowSToD(source.AsFpuRegisterPairLow<SRegister>())); 5795 } else if (destination.IsFpuRegisterPair()) { 5796 __ vmovd(FromLowSToD(destination.AsFpuRegisterPairLow<SRegister>()), 5797 FromLowSToD(source.AsFpuRegisterPairLow<SRegister>())); 5798 } else { 5799 DCHECK(destination.IsDoubleStackSlot()) << destination; 5800 __ StoreDToOffset(FromLowSToD(source.AsFpuRegisterPairLow<SRegister>()), 5801 SP, 5802 destination.GetStackIndex()); 5803 } 5804 } else { 5805 DCHECK(source.IsConstant()) << source; 5806 HConstant* constant = source.GetConstant(); 5807 if (constant->IsIntConstant() || constant->IsNullConstant()) { 5808 int32_t value = CodeGenerator::GetInt32ValueOf(constant); 5809 if (destination.IsRegister()) { 5810 __ LoadImmediate(destination.AsRegister<Register>(), value); 5811 } else { 5812 DCHECK(destination.IsStackSlot()); 5813 __ LoadImmediate(IP, value); 5814 __ StoreToOffset(kStoreWord, IP, SP, destination.GetStackIndex()); 5815 } 5816 } else if (constant->IsLongConstant()) { 5817 int64_t value = constant->AsLongConstant()->GetValue(); 5818 if (destination.IsRegisterPair()) { 5819 __ LoadImmediate(destination.AsRegisterPairLow<Register>(), Low32Bits(value)); 5820 __ LoadImmediate(destination.AsRegisterPairHigh<Register>(), High32Bits(value)); 5821 } else { 5822 DCHECK(destination.IsDoubleStackSlot()) << destination; 5823 __ LoadImmediate(IP, Low32Bits(value)); 5824 __ StoreToOffset(kStoreWord, IP, SP, destination.GetStackIndex()); 5825 __ LoadImmediate(IP, High32Bits(value)); 5826 __ StoreToOffset(kStoreWord, IP, SP, destination.GetHighStackIndex(kArmWordSize)); 5827 } 5828 } else if (constant->IsDoubleConstant()) { 5829 double value = constant->AsDoubleConstant()->GetValue(); 5830 if (destination.IsFpuRegisterPair()) { 5831 __ LoadDImmediate(FromLowSToD(destination.AsFpuRegisterPairLow<SRegister>()), value); 5832 } else { 5833 DCHECK(destination.IsDoubleStackSlot()) << destination; 5834 uint64_t int_value = bit_cast<uint64_t, double>(value); 5835 __ LoadImmediate(IP, Low32Bits(int_value)); 5836 __ StoreToOffset(kStoreWord, IP, SP, destination.GetStackIndex()); 5837 __ LoadImmediate(IP, High32Bits(int_value)); 5838 __ StoreToOffset(kStoreWord, IP, SP, destination.GetHighStackIndex(kArmWordSize)); 5839 } 5840 } else { 5841 DCHECK(constant->IsFloatConstant()) << constant->DebugName(); 5842 float value = constant->AsFloatConstant()->GetValue(); 5843 if (destination.IsFpuRegister()) { 5844 __ LoadSImmediate(destination.AsFpuRegister<SRegister>(), value); 5845 } else { 5846 DCHECK(destination.IsStackSlot()); 5847 __ LoadImmediate(IP, bit_cast<int32_t, float>(value)); 5848 __ StoreToOffset(kStoreWord, IP, SP, destination.GetStackIndex()); 5849 } 5850 } 5851 } 5852} 5853 5854void ParallelMoveResolverARM::Exchange(Register reg, int mem) { 5855 __ Mov(IP, reg); 5856 __ LoadFromOffset(kLoadWord, reg, SP, mem); 5857 __ StoreToOffset(kStoreWord, IP, SP, mem); 5858} 5859 5860void ParallelMoveResolverARM::Exchange(int mem1, int mem2) { 5861 ScratchRegisterScope ensure_scratch(this, IP, R0, codegen_->GetNumberOfCoreRegisters()); 5862 int stack_offset = ensure_scratch.IsSpilled() ? kArmWordSize : 0; 5863 __ LoadFromOffset(kLoadWord, static_cast<Register>(ensure_scratch.GetRegister()), 5864 SP, mem1 + stack_offset); 5865 __ LoadFromOffset(kLoadWord, IP, SP, mem2 + stack_offset); 5866 __ StoreToOffset(kStoreWord, static_cast<Register>(ensure_scratch.GetRegister()), 5867 SP, mem2 + stack_offset); 5868 __ StoreToOffset(kStoreWord, IP, SP, mem1 + stack_offset); 5869} 5870 5871void ParallelMoveResolverARM::EmitSwap(size_t index) { 5872 MoveOperands* move = moves_[index]; 5873 Location source = move->GetSource(); 5874 Location destination = move->GetDestination(); 5875 5876 if (source.IsRegister() && destination.IsRegister()) { 5877 DCHECK_NE(source.AsRegister<Register>(), IP); 5878 DCHECK_NE(destination.AsRegister<Register>(), IP); 5879 __ Mov(IP, source.AsRegister<Register>()); 5880 __ Mov(source.AsRegister<Register>(), destination.AsRegister<Register>()); 5881 __ Mov(destination.AsRegister<Register>(), IP); 5882 } else if (source.IsRegister() && destination.IsStackSlot()) { 5883 Exchange(source.AsRegister<Register>(), destination.GetStackIndex()); 5884 } else if (source.IsStackSlot() && destination.IsRegister()) { 5885 Exchange(destination.AsRegister<Register>(), source.GetStackIndex()); 5886 } else if (source.IsStackSlot() && destination.IsStackSlot()) { 5887 Exchange(source.GetStackIndex(), destination.GetStackIndex()); 5888 } else if (source.IsFpuRegister() && destination.IsFpuRegister()) { 5889 __ vmovrs(IP, source.AsFpuRegister<SRegister>()); 5890 __ vmovs(source.AsFpuRegister<SRegister>(), destination.AsFpuRegister<SRegister>()); 5891 __ vmovsr(destination.AsFpuRegister<SRegister>(), IP); 5892 } else if (source.IsRegisterPair() && destination.IsRegisterPair()) { 5893 __ vmovdrr(DTMP, source.AsRegisterPairLow<Register>(), source.AsRegisterPairHigh<Register>()); 5894 __ Mov(source.AsRegisterPairLow<Register>(), destination.AsRegisterPairLow<Register>()); 5895 __ Mov(source.AsRegisterPairHigh<Register>(), destination.AsRegisterPairHigh<Register>()); 5896 __ vmovrrd(destination.AsRegisterPairLow<Register>(), 5897 destination.AsRegisterPairHigh<Register>(), 5898 DTMP); 5899 } else if (source.IsRegisterPair() || destination.IsRegisterPair()) { 5900 Register low_reg = source.IsRegisterPair() 5901 ? source.AsRegisterPairLow<Register>() 5902 : destination.AsRegisterPairLow<Register>(); 5903 int mem = source.IsRegisterPair() 5904 ? destination.GetStackIndex() 5905 : source.GetStackIndex(); 5906 DCHECK(ExpectedPairLayout(source.IsRegisterPair() ? source : destination)); 5907 __ vmovdrr(DTMP, low_reg, static_cast<Register>(low_reg + 1)); 5908 __ LoadFromOffset(kLoadWordPair, low_reg, SP, mem); 5909 __ StoreDToOffset(DTMP, SP, mem); 5910 } else if (source.IsFpuRegisterPair() && destination.IsFpuRegisterPair()) { 5911 DRegister first = FromLowSToD(source.AsFpuRegisterPairLow<SRegister>()); 5912 DRegister second = FromLowSToD(destination.AsFpuRegisterPairLow<SRegister>()); 5913 __ vmovd(DTMP, first); 5914 __ vmovd(first, second); 5915 __ vmovd(second, DTMP); 5916 } else if (source.IsFpuRegisterPair() || destination.IsFpuRegisterPair()) { 5917 DRegister reg = source.IsFpuRegisterPair() 5918 ? FromLowSToD(source.AsFpuRegisterPairLow<SRegister>()) 5919 : FromLowSToD(destination.AsFpuRegisterPairLow<SRegister>()); 5920 int mem = source.IsFpuRegisterPair() 5921 ? destination.GetStackIndex() 5922 : source.GetStackIndex(); 5923 __ vmovd(DTMP, reg); 5924 __ LoadDFromOffset(reg, SP, mem); 5925 __ StoreDToOffset(DTMP, SP, mem); 5926 } else if (source.IsFpuRegister() || destination.IsFpuRegister()) { 5927 SRegister reg = source.IsFpuRegister() ? source.AsFpuRegister<SRegister>() 5928 : destination.AsFpuRegister<SRegister>(); 5929 int mem = source.IsFpuRegister() 5930 ? destination.GetStackIndex() 5931 : source.GetStackIndex(); 5932 5933 __ vmovrs(IP, reg); 5934 __ LoadSFromOffset(reg, SP, mem); 5935 __ StoreToOffset(kStoreWord, IP, SP, mem); 5936 } else if (source.IsDoubleStackSlot() && destination.IsDoubleStackSlot()) { 5937 Exchange(source.GetStackIndex(), destination.GetStackIndex()); 5938 Exchange(source.GetHighStackIndex(kArmWordSize), destination.GetHighStackIndex(kArmWordSize)); 5939 } else { 5940 LOG(FATAL) << "Unimplemented" << source << " <-> " << destination; 5941 } 5942} 5943 5944void ParallelMoveResolverARM::SpillScratch(int reg) { 5945 __ Push(static_cast<Register>(reg)); 5946} 5947 5948void ParallelMoveResolverARM::RestoreScratch(int reg) { 5949 __ Pop(static_cast<Register>(reg)); 5950} 5951 5952HLoadClass::LoadKind CodeGeneratorARM::GetSupportedLoadClassKind( 5953 HLoadClass::LoadKind desired_class_load_kind) { 5954 switch (desired_class_load_kind) { 5955 case HLoadClass::LoadKind::kInvalid: 5956 LOG(FATAL) << "UNREACHABLE"; 5957 UNREACHABLE(); 5958 case HLoadClass::LoadKind::kReferrersClass: 5959 break; 5960 case HLoadClass::LoadKind::kBootImageLinkTimeAddress: 5961 DCHECK(!GetCompilerOptions().GetCompilePic()); 5962 break; 5963 case HLoadClass::LoadKind::kBootImageLinkTimePcRelative: 5964 DCHECK(GetCompilerOptions().GetCompilePic()); 5965 break; 5966 case HLoadClass::LoadKind::kBootImageAddress: 5967 break; 5968 case HLoadClass::LoadKind::kBssEntry: 5969 DCHECK(!Runtime::Current()->UseJitCompilation()); 5970 break; 5971 case HLoadClass::LoadKind::kJitTableAddress: 5972 DCHECK(Runtime::Current()->UseJitCompilation()); 5973 break; 5974 case HLoadClass::LoadKind::kDexCacheViaMethod: 5975 break; 5976 } 5977 return desired_class_load_kind; 5978} 5979 5980void LocationsBuilderARM::VisitLoadClass(HLoadClass* cls) { 5981 HLoadClass::LoadKind load_kind = cls->GetLoadKind(); 5982 if (load_kind == HLoadClass::LoadKind::kDexCacheViaMethod) { 5983 InvokeRuntimeCallingConvention calling_convention; 5984 CodeGenerator::CreateLoadClassRuntimeCallLocationSummary( 5985 cls, 5986 Location::RegisterLocation(calling_convention.GetRegisterAt(0)), 5987 Location::RegisterLocation(R0)); 5988 DCHECK_EQ(calling_convention.GetRegisterAt(0), R0); 5989 return; 5990 } 5991 DCHECK(!cls->NeedsAccessCheck()); 5992 5993 const bool requires_read_barrier = kEmitCompilerReadBarrier && !cls->IsInBootImage(); 5994 LocationSummary::CallKind call_kind = (cls->NeedsEnvironment() || requires_read_barrier) 5995 ? LocationSummary::kCallOnSlowPath 5996 : LocationSummary::kNoCall; 5997 LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(cls, call_kind); 5998 if (kUseBakerReadBarrier && requires_read_barrier && !cls->NeedsEnvironment()) { 5999 locations->SetCustomSlowPathCallerSaves(RegisterSet::Empty()); // No caller-save registers. 6000 } 6001 6002 if (load_kind == HLoadClass::LoadKind::kReferrersClass) { 6003 locations->SetInAt(0, Location::RequiresRegister()); 6004 } 6005 locations->SetOut(Location::RequiresRegister()); 6006 if (load_kind == HLoadClass::LoadKind::kBssEntry) { 6007 if (!kUseReadBarrier || kUseBakerReadBarrier) { 6008 // Rely on the type resolution or initialization and marking to save everything we need. 6009 // Note that IP may be clobbered by saving/restoring the live register (only one thanks 6010 // to the custom calling convention) or by marking, so we request a different temp. 6011 locations->AddTemp(Location::RequiresRegister()); 6012 RegisterSet caller_saves = RegisterSet::Empty(); 6013 InvokeRuntimeCallingConvention calling_convention; 6014 caller_saves.Add(Location::RegisterLocation(calling_convention.GetRegisterAt(0))); 6015 // TODO: Add GetReturnLocation() to the calling convention so that we can DCHECK() 6016 // that the the kPrimNot result register is the same as the first argument register. 6017 locations->SetCustomSlowPathCallerSaves(caller_saves); 6018 } else { 6019 // For non-Baker read barrier we have a temp-clobbering call. 6020 } 6021 } 6022} 6023 6024// NO_THREAD_SAFETY_ANALYSIS as we manipulate handles whose internal object we know does not 6025// move. 6026void InstructionCodeGeneratorARM::VisitLoadClass(HLoadClass* cls) NO_THREAD_SAFETY_ANALYSIS { 6027 HLoadClass::LoadKind load_kind = cls->GetLoadKind(); 6028 if (load_kind == HLoadClass::LoadKind::kDexCacheViaMethod) { 6029 codegen_->GenerateLoadClassRuntimeCall(cls); 6030 return; 6031 } 6032 DCHECK(!cls->NeedsAccessCheck()); 6033 6034 LocationSummary* locations = cls->GetLocations(); 6035 Location out_loc = locations->Out(); 6036 Register out = out_loc.AsRegister<Register>(); 6037 6038 const ReadBarrierOption read_barrier_option = cls->IsInBootImage() 6039 ? kWithoutReadBarrier 6040 : kCompilerReadBarrierOption; 6041 bool generate_null_check = false; 6042 switch (load_kind) { 6043 case HLoadClass::LoadKind::kReferrersClass: { 6044 DCHECK(!cls->CanCallRuntime()); 6045 DCHECK(!cls->MustGenerateClinitCheck()); 6046 // /* GcRoot<mirror::Class> */ out = current_method->declaring_class_ 6047 Register current_method = locations->InAt(0).AsRegister<Register>(); 6048 GenerateGcRootFieldLoad(cls, 6049 out_loc, 6050 current_method, 6051 ArtMethod::DeclaringClassOffset().Int32Value(), 6052 read_barrier_option); 6053 break; 6054 } 6055 case HLoadClass::LoadKind::kBootImageLinkTimeAddress: { 6056 DCHECK(codegen_->GetCompilerOptions().IsBootImage()); 6057 DCHECK_EQ(read_barrier_option, kWithoutReadBarrier); 6058 __ LoadLiteral(out, codegen_->DeduplicateBootImageTypeLiteral(cls->GetDexFile(), 6059 cls->GetTypeIndex())); 6060 break; 6061 } 6062 case HLoadClass::LoadKind::kBootImageLinkTimePcRelative: { 6063 DCHECK(codegen_->GetCompilerOptions().IsBootImage()); 6064 DCHECK_EQ(read_barrier_option, kWithoutReadBarrier); 6065 CodeGeneratorARM::PcRelativePatchInfo* labels = 6066 codegen_->NewPcRelativeTypePatch(cls->GetDexFile(), cls->GetTypeIndex()); 6067 __ BindTrackedLabel(&labels->movw_label); 6068 __ movw(out, /* placeholder */ 0u); 6069 __ BindTrackedLabel(&labels->movt_label); 6070 __ movt(out, /* placeholder */ 0u); 6071 __ BindTrackedLabel(&labels->add_pc_label); 6072 __ add(out, out, ShifterOperand(PC)); 6073 break; 6074 } 6075 case HLoadClass::LoadKind::kBootImageAddress: { 6076 DCHECK_EQ(read_barrier_option, kWithoutReadBarrier); 6077 uint32_t address = dchecked_integral_cast<uint32_t>( 6078 reinterpret_cast<uintptr_t>(cls->GetClass().Get())); 6079 DCHECK_NE(address, 0u); 6080 __ LoadLiteral(out, codegen_->DeduplicateBootImageAddressLiteral(address)); 6081 break; 6082 } 6083 case HLoadClass::LoadKind::kBssEntry: { 6084 Register temp = (!kUseReadBarrier || kUseBakerReadBarrier) 6085 ? locations->GetTemp(0).AsRegister<Register>() 6086 : out; 6087 CodeGeneratorARM::PcRelativePatchInfo* labels = 6088 codegen_->NewTypeBssEntryPatch(cls->GetDexFile(), cls->GetTypeIndex()); 6089 __ BindTrackedLabel(&labels->movw_label); 6090 __ movw(temp, /* placeholder */ 0u); 6091 __ BindTrackedLabel(&labels->movt_label); 6092 __ movt(temp, /* placeholder */ 0u); 6093 __ BindTrackedLabel(&labels->add_pc_label); 6094 __ add(temp, temp, ShifterOperand(PC)); 6095 GenerateGcRootFieldLoad(cls, out_loc, temp, /* offset */ 0, read_barrier_option); 6096 generate_null_check = true; 6097 break; 6098 } 6099 case HLoadClass::LoadKind::kJitTableAddress: { 6100 __ LoadLiteral(out, codegen_->DeduplicateJitClassLiteral(cls->GetDexFile(), 6101 cls->GetTypeIndex(), 6102 cls->GetClass())); 6103 // /* GcRoot<mirror::Class> */ out = *out 6104 GenerateGcRootFieldLoad(cls, out_loc, out, /* offset */ 0, read_barrier_option); 6105 break; 6106 } 6107 case HLoadClass::LoadKind::kDexCacheViaMethod: 6108 case HLoadClass::LoadKind::kInvalid: 6109 LOG(FATAL) << "UNREACHABLE"; 6110 UNREACHABLE(); 6111 } 6112 6113 if (generate_null_check || cls->MustGenerateClinitCheck()) { 6114 DCHECK(cls->CanCallRuntime()); 6115 SlowPathCodeARM* slow_path = new (GetGraph()->GetArena()) LoadClassSlowPathARM( 6116 cls, cls, cls->GetDexPc(), cls->MustGenerateClinitCheck()); 6117 codegen_->AddSlowPath(slow_path); 6118 if (generate_null_check) { 6119 __ CompareAndBranchIfZero(out, slow_path->GetEntryLabel()); 6120 } 6121 if (cls->MustGenerateClinitCheck()) { 6122 GenerateClassInitializationCheck(slow_path, out); 6123 } else { 6124 __ Bind(slow_path->GetExitLabel()); 6125 } 6126 } 6127} 6128 6129void LocationsBuilderARM::VisitClinitCheck(HClinitCheck* check) { 6130 LocationSummary* locations = 6131 new (GetGraph()->GetArena()) LocationSummary(check, LocationSummary::kCallOnSlowPath); 6132 locations->SetInAt(0, Location::RequiresRegister()); 6133 if (check->HasUses()) { 6134 locations->SetOut(Location::SameAsFirstInput()); 6135 } 6136} 6137 6138void InstructionCodeGeneratorARM::VisitClinitCheck(HClinitCheck* check) { 6139 // We assume the class is not null. 6140 SlowPathCodeARM* slow_path = new (GetGraph()->GetArena()) LoadClassSlowPathARM( 6141 check->GetLoadClass(), check, check->GetDexPc(), true); 6142 codegen_->AddSlowPath(slow_path); 6143 GenerateClassInitializationCheck(slow_path, 6144 check->GetLocations()->InAt(0).AsRegister<Register>()); 6145} 6146 6147void InstructionCodeGeneratorARM::GenerateClassInitializationCheck( 6148 SlowPathCodeARM* slow_path, Register class_reg) { 6149 __ LoadFromOffset(kLoadWord, IP, class_reg, mirror::Class::StatusOffset().Int32Value()); 6150 __ cmp(IP, ShifterOperand(mirror::Class::kStatusInitialized)); 6151 __ b(slow_path->GetEntryLabel(), LT); 6152 // Even if the initialized flag is set, we may be in a situation where caches are not synced 6153 // properly. Therefore, we do a memory fence. 6154 __ dmb(ISH); 6155 __ Bind(slow_path->GetExitLabel()); 6156} 6157 6158HLoadString::LoadKind CodeGeneratorARM::GetSupportedLoadStringKind( 6159 HLoadString::LoadKind desired_string_load_kind) { 6160 switch (desired_string_load_kind) { 6161 case HLoadString::LoadKind::kBootImageLinkTimeAddress: 6162 DCHECK(!GetCompilerOptions().GetCompilePic()); 6163 break; 6164 case HLoadString::LoadKind::kBootImageLinkTimePcRelative: 6165 DCHECK(GetCompilerOptions().GetCompilePic()); 6166 break; 6167 case HLoadString::LoadKind::kBootImageAddress: 6168 break; 6169 case HLoadString::LoadKind::kBssEntry: 6170 DCHECK(!Runtime::Current()->UseJitCompilation()); 6171 break; 6172 case HLoadString::LoadKind::kJitTableAddress: 6173 DCHECK(Runtime::Current()->UseJitCompilation()); 6174 break; 6175 case HLoadString::LoadKind::kDexCacheViaMethod: 6176 break; 6177 } 6178 return desired_string_load_kind; 6179} 6180 6181void LocationsBuilderARM::VisitLoadString(HLoadString* load) { 6182 LocationSummary::CallKind call_kind = CodeGenerator::GetLoadStringCallKind(load); 6183 LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(load, call_kind); 6184 HLoadString::LoadKind load_kind = load->GetLoadKind(); 6185 if (load_kind == HLoadString::LoadKind::kDexCacheViaMethod) { 6186 locations->SetOut(Location::RegisterLocation(R0)); 6187 } else { 6188 locations->SetOut(Location::RequiresRegister()); 6189 if (load_kind == HLoadString::LoadKind::kBssEntry) { 6190 if (!kUseReadBarrier || kUseBakerReadBarrier) { 6191 // Rely on the pResolveString and marking to save everything we need, including temps. 6192 // Note that IP may be clobbered by saving/restoring the live register (only one thanks 6193 // to the custom calling convention) or by marking, so we request a different temp. 6194 locations->AddTemp(Location::RequiresRegister()); 6195 RegisterSet caller_saves = RegisterSet::Empty(); 6196 InvokeRuntimeCallingConvention calling_convention; 6197 caller_saves.Add(Location::RegisterLocation(calling_convention.GetRegisterAt(0))); 6198 // TODO: Add GetReturnLocation() to the calling convention so that we can DCHECK() 6199 // that the the kPrimNot result register is the same as the first argument register. 6200 locations->SetCustomSlowPathCallerSaves(caller_saves); 6201 } else { 6202 // For non-Baker read barrier we have a temp-clobbering call. 6203 } 6204 } 6205 } 6206} 6207 6208// NO_THREAD_SAFETY_ANALYSIS as we manipulate handles whose internal object we know does not 6209// move. 6210void InstructionCodeGeneratorARM::VisitLoadString(HLoadString* load) NO_THREAD_SAFETY_ANALYSIS { 6211 LocationSummary* locations = load->GetLocations(); 6212 Location out_loc = locations->Out(); 6213 Register out = out_loc.AsRegister<Register>(); 6214 HLoadString::LoadKind load_kind = load->GetLoadKind(); 6215 6216 switch (load_kind) { 6217 case HLoadString::LoadKind::kBootImageLinkTimeAddress: { 6218 DCHECK(codegen_->GetCompilerOptions().IsBootImage()); 6219 __ LoadLiteral(out, codegen_->DeduplicateBootImageStringLiteral(load->GetDexFile(), 6220 load->GetStringIndex())); 6221 return; // No dex cache slow path. 6222 } 6223 case HLoadString::LoadKind::kBootImageLinkTimePcRelative: { 6224 DCHECK(codegen_->GetCompilerOptions().IsBootImage()); 6225 CodeGeneratorARM::PcRelativePatchInfo* labels = 6226 codegen_->NewPcRelativeStringPatch(load->GetDexFile(), load->GetStringIndex()); 6227 __ BindTrackedLabel(&labels->movw_label); 6228 __ movw(out, /* placeholder */ 0u); 6229 __ BindTrackedLabel(&labels->movt_label); 6230 __ movt(out, /* placeholder */ 0u); 6231 __ BindTrackedLabel(&labels->add_pc_label); 6232 __ add(out, out, ShifterOperand(PC)); 6233 return; // No dex cache slow path. 6234 } 6235 case HLoadString::LoadKind::kBootImageAddress: { 6236 uint32_t address = dchecked_integral_cast<uint32_t>( 6237 reinterpret_cast<uintptr_t>(load->GetString().Get())); 6238 DCHECK_NE(address, 0u); 6239 __ LoadLiteral(out, codegen_->DeduplicateBootImageAddressLiteral(address)); 6240 return; // No dex cache slow path. 6241 } 6242 case HLoadString::LoadKind::kBssEntry: { 6243 DCHECK(!codegen_->GetCompilerOptions().IsBootImage()); 6244 Register temp = (!kUseReadBarrier || kUseBakerReadBarrier) 6245 ? locations->GetTemp(0).AsRegister<Register>() 6246 : out; 6247 CodeGeneratorARM::PcRelativePatchInfo* labels = 6248 codegen_->NewPcRelativeStringPatch(load->GetDexFile(), load->GetStringIndex()); 6249 __ BindTrackedLabel(&labels->movw_label); 6250 __ movw(temp, /* placeholder */ 0u); 6251 __ BindTrackedLabel(&labels->movt_label); 6252 __ movt(temp, /* placeholder */ 0u); 6253 __ BindTrackedLabel(&labels->add_pc_label); 6254 __ add(temp, temp, ShifterOperand(PC)); 6255 GenerateGcRootFieldLoad(load, out_loc, temp, /* offset */ 0, kCompilerReadBarrierOption); 6256 SlowPathCode* slow_path = new (GetGraph()->GetArena()) LoadStringSlowPathARM(load); 6257 codegen_->AddSlowPath(slow_path); 6258 __ CompareAndBranchIfZero(out, slow_path->GetEntryLabel()); 6259 __ Bind(slow_path->GetExitLabel()); 6260 return; 6261 } 6262 case HLoadString::LoadKind::kJitTableAddress: { 6263 __ LoadLiteral(out, codegen_->DeduplicateJitStringLiteral(load->GetDexFile(), 6264 load->GetStringIndex(), 6265 load->GetString())); 6266 // /* GcRoot<mirror::String> */ out = *out 6267 GenerateGcRootFieldLoad(load, out_loc, out, /* offset */ 0, kCompilerReadBarrierOption); 6268 return; 6269 } 6270 default: 6271 break; 6272 } 6273 6274 // TODO: Consider re-adding the compiler code to do string dex cache lookup again. 6275 DCHECK(load_kind == HLoadString::LoadKind::kDexCacheViaMethod); 6276 InvokeRuntimeCallingConvention calling_convention; 6277 DCHECK_EQ(calling_convention.GetRegisterAt(0), out); 6278 __ LoadImmediate(calling_convention.GetRegisterAt(0), load->GetStringIndex().index_); 6279 codegen_->InvokeRuntime(kQuickResolveString, load, load->GetDexPc()); 6280 CheckEntrypointTypes<kQuickResolveString, void*, uint32_t>(); 6281} 6282 6283static int32_t GetExceptionTlsOffset() { 6284 return Thread::ExceptionOffset<kArmPointerSize>().Int32Value(); 6285} 6286 6287void LocationsBuilderARM::VisitLoadException(HLoadException* load) { 6288 LocationSummary* locations = 6289 new (GetGraph()->GetArena()) LocationSummary(load, LocationSummary::kNoCall); 6290 locations->SetOut(Location::RequiresRegister()); 6291} 6292 6293void InstructionCodeGeneratorARM::VisitLoadException(HLoadException* load) { 6294 Register out = load->GetLocations()->Out().AsRegister<Register>(); 6295 __ LoadFromOffset(kLoadWord, out, TR, GetExceptionTlsOffset()); 6296} 6297 6298void LocationsBuilderARM::VisitClearException(HClearException* clear) { 6299 new (GetGraph()->GetArena()) LocationSummary(clear, LocationSummary::kNoCall); 6300} 6301 6302void InstructionCodeGeneratorARM::VisitClearException(HClearException* clear ATTRIBUTE_UNUSED) { 6303 __ LoadImmediate(IP, 0); 6304 __ StoreToOffset(kStoreWord, IP, TR, GetExceptionTlsOffset()); 6305} 6306 6307void LocationsBuilderARM::VisitThrow(HThrow* instruction) { 6308 LocationSummary* locations = 6309 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCallOnMainOnly); 6310 InvokeRuntimeCallingConvention calling_convention; 6311 locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(0))); 6312} 6313 6314void InstructionCodeGeneratorARM::VisitThrow(HThrow* instruction) { 6315 codegen_->InvokeRuntime(kQuickDeliverException, instruction, instruction->GetDexPc()); 6316 CheckEntrypointTypes<kQuickDeliverException, void, mirror::Object*>(); 6317} 6318 6319// Temp is used for read barrier. 6320static size_t NumberOfInstanceOfTemps(TypeCheckKind type_check_kind) { 6321 if (kEmitCompilerReadBarrier && 6322 (kUseBakerReadBarrier || 6323 type_check_kind == TypeCheckKind::kAbstractClassCheck || 6324 type_check_kind == TypeCheckKind::kClassHierarchyCheck || 6325 type_check_kind == TypeCheckKind::kArrayObjectCheck)) { 6326 return 1; 6327 } 6328 return 0; 6329} 6330 6331// Interface case has 3 temps, one for holding the number of interfaces, one for the current 6332// interface pointer, one for loading the current interface. 6333// The other checks have one temp for loading the object's class. 6334static size_t NumberOfCheckCastTemps(TypeCheckKind type_check_kind) { 6335 if (type_check_kind == TypeCheckKind::kInterfaceCheck) { 6336 return 3; 6337 } 6338 return 1 + NumberOfInstanceOfTemps(type_check_kind); 6339} 6340 6341void LocationsBuilderARM::VisitInstanceOf(HInstanceOf* instruction) { 6342 LocationSummary::CallKind call_kind = LocationSummary::kNoCall; 6343 TypeCheckKind type_check_kind = instruction->GetTypeCheckKind(); 6344 bool baker_read_barrier_slow_path = false; 6345 switch (type_check_kind) { 6346 case TypeCheckKind::kExactCheck: 6347 case TypeCheckKind::kAbstractClassCheck: 6348 case TypeCheckKind::kClassHierarchyCheck: 6349 case TypeCheckKind::kArrayObjectCheck: 6350 call_kind = 6351 kEmitCompilerReadBarrier ? LocationSummary::kCallOnSlowPath : LocationSummary::kNoCall; 6352 baker_read_barrier_slow_path = kUseBakerReadBarrier; 6353 break; 6354 case TypeCheckKind::kArrayCheck: 6355 case TypeCheckKind::kUnresolvedCheck: 6356 case TypeCheckKind::kInterfaceCheck: 6357 call_kind = LocationSummary::kCallOnSlowPath; 6358 break; 6359 } 6360 6361 LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction, call_kind); 6362 if (baker_read_barrier_slow_path) { 6363 locations->SetCustomSlowPathCallerSaves(RegisterSet::Empty()); // No caller-save registers. 6364 } 6365 locations->SetInAt(0, Location::RequiresRegister()); 6366 locations->SetInAt(1, Location::RequiresRegister()); 6367 // The "out" register is used as a temporary, so it overlaps with the inputs. 6368 // Note that TypeCheckSlowPathARM uses this register too. 6369 locations->SetOut(Location::RequiresRegister(), Location::kOutputOverlap); 6370 locations->AddRegisterTemps(NumberOfInstanceOfTemps(type_check_kind)); 6371} 6372 6373void InstructionCodeGeneratorARM::VisitInstanceOf(HInstanceOf* instruction) { 6374 TypeCheckKind type_check_kind = instruction->GetTypeCheckKind(); 6375 LocationSummary* locations = instruction->GetLocations(); 6376 Location obj_loc = locations->InAt(0); 6377 Register obj = obj_loc.AsRegister<Register>(); 6378 Register cls = locations->InAt(1).AsRegister<Register>(); 6379 Location out_loc = locations->Out(); 6380 Register out = out_loc.AsRegister<Register>(); 6381 const size_t num_temps = NumberOfInstanceOfTemps(type_check_kind); 6382 DCHECK_LE(num_temps, 1u); 6383 Location maybe_temp_loc = (num_temps >= 1) ? locations->GetTemp(0) : Location::NoLocation(); 6384 uint32_t class_offset = mirror::Object::ClassOffset().Int32Value(); 6385 uint32_t super_offset = mirror::Class::SuperClassOffset().Int32Value(); 6386 uint32_t component_offset = mirror::Class::ComponentTypeOffset().Int32Value(); 6387 uint32_t primitive_offset = mirror::Class::PrimitiveTypeOffset().Int32Value(); 6388 Label done, zero; 6389 SlowPathCodeARM* slow_path = nullptr; 6390 6391 // Return 0 if `obj` is null. 6392 // avoid null check if we know obj is not null. 6393 if (instruction->MustDoNullCheck()) { 6394 __ CompareAndBranchIfZero(obj, &zero); 6395 } 6396 6397 switch (type_check_kind) { 6398 case TypeCheckKind::kExactCheck: { 6399 // /* HeapReference<Class> */ out = obj->klass_ 6400 GenerateReferenceLoadTwoRegisters(instruction, 6401 out_loc, 6402 obj_loc, 6403 class_offset, 6404 maybe_temp_loc, 6405 kCompilerReadBarrierOption); 6406 __ cmp(out, ShifterOperand(cls)); 6407 // Classes must be equal for the instanceof to succeed. 6408 __ b(&zero, NE); 6409 __ LoadImmediate(out, 1); 6410 __ b(&done); 6411 break; 6412 } 6413 6414 case TypeCheckKind::kAbstractClassCheck: { 6415 // /* HeapReference<Class> */ out = obj->klass_ 6416 GenerateReferenceLoadTwoRegisters(instruction, 6417 out_loc, 6418 obj_loc, 6419 class_offset, 6420 maybe_temp_loc, 6421 kCompilerReadBarrierOption); 6422 // If the class is abstract, we eagerly fetch the super class of the 6423 // object to avoid doing a comparison we know will fail. 6424 Label loop; 6425 __ Bind(&loop); 6426 // /* HeapReference<Class> */ out = out->super_class_ 6427 GenerateReferenceLoadOneRegister(instruction, 6428 out_loc, 6429 super_offset, 6430 maybe_temp_loc, 6431 kCompilerReadBarrierOption); 6432 // If `out` is null, we use it for the result, and jump to `done`. 6433 __ CompareAndBranchIfZero(out, &done); 6434 __ cmp(out, ShifterOperand(cls)); 6435 __ b(&loop, NE); 6436 __ LoadImmediate(out, 1); 6437 if (zero.IsLinked()) { 6438 __ b(&done); 6439 } 6440 break; 6441 } 6442 6443 case TypeCheckKind::kClassHierarchyCheck: { 6444 // /* HeapReference<Class> */ out = obj->klass_ 6445 GenerateReferenceLoadTwoRegisters(instruction, 6446 out_loc, 6447 obj_loc, 6448 class_offset, 6449 maybe_temp_loc, 6450 kCompilerReadBarrierOption); 6451 // Walk over the class hierarchy to find a match. 6452 Label loop, success; 6453 __ Bind(&loop); 6454 __ cmp(out, ShifterOperand(cls)); 6455 __ b(&success, EQ); 6456 // /* HeapReference<Class> */ out = out->super_class_ 6457 GenerateReferenceLoadOneRegister(instruction, 6458 out_loc, 6459 super_offset, 6460 maybe_temp_loc, 6461 kCompilerReadBarrierOption); 6462 __ CompareAndBranchIfNonZero(out, &loop); 6463 // If `out` is null, we use it for the result, and jump to `done`. 6464 __ b(&done); 6465 __ Bind(&success); 6466 __ LoadImmediate(out, 1); 6467 if (zero.IsLinked()) { 6468 __ b(&done); 6469 } 6470 break; 6471 } 6472 6473 case TypeCheckKind::kArrayObjectCheck: { 6474 // /* HeapReference<Class> */ out = obj->klass_ 6475 GenerateReferenceLoadTwoRegisters(instruction, 6476 out_loc, 6477 obj_loc, 6478 class_offset, 6479 maybe_temp_loc, 6480 kCompilerReadBarrierOption); 6481 // Do an exact check. 6482 Label exact_check; 6483 __ cmp(out, ShifterOperand(cls)); 6484 __ b(&exact_check, EQ); 6485 // Otherwise, we need to check that the object's class is a non-primitive array. 6486 // /* HeapReference<Class> */ out = out->component_type_ 6487 GenerateReferenceLoadOneRegister(instruction, 6488 out_loc, 6489 component_offset, 6490 maybe_temp_loc, 6491 kCompilerReadBarrierOption); 6492 // If `out` is null, we use it for the result, and jump to `done`. 6493 __ CompareAndBranchIfZero(out, &done); 6494 __ LoadFromOffset(kLoadUnsignedHalfword, out, out, primitive_offset); 6495 static_assert(Primitive::kPrimNot == 0, "Expected 0 for kPrimNot"); 6496 __ CompareAndBranchIfNonZero(out, &zero); 6497 __ Bind(&exact_check); 6498 __ LoadImmediate(out, 1); 6499 __ b(&done); 6500 break; 6501 } 6502 6503 case TypeCheckKind::kArrayCheck: { 6504 // No read barrier since the slow path will retry upon failure. 6505 // /* HeapReference<Class> */ out = obj->klass_ 6506 GenerateReferenceLoadTwoRegisters(instruction, 6507 out_loc, 6508 obj_loc, 6509 class_offset, 6510 maybe_temp_loc, 6511 kWithoutReadBarrier); 6512 __ cmp(out, ShifterOperand(cls)); 6513 DCHECK(locations->OnlyCallsOnSlowPath()); 6514 slow_path = new (GetGraph()->GetArena()) TypeCheckSlowPathARM(instruction, 6515 /* is_fatal */ false); 6516 codegen_->AddSlowPath(slow_path); 6517 __ b(slow_path->GetEntryLabel(), NE); 6518 __ LoadImmediate(out, 1); 6519 if (zero.IsLinked()) { 6520 __ b(&done); 6521 } 6522 break; 6523 } 6524 6525 case TypeCheckKind::kUnresolvedCheck: 6526 case TypeCheckKind::kInterfaceCheck: { 6527 // Note that we indeed only call on slow path, but we always go 6528 // into the slow path for the unresolved and interface check 6529 // cases. 6530 // 6531 // We cannot directly call the InstanceofNonTrivial runtime 6532 // entry point without resorting to a type checking slow path 6533 // here (i.e. by calling InvokeRuntime directly), as it would 6534 // require to assign fixed registers for the inputs of this 6535 // HInstanceOf instruction (following the runtime calling 6536 // convention), which might be cluttered by the potential first 6537 // read barrier emission at the beginning of this method. 6538 // 6539 // TODO: Introduce a new runtime entry point taking the object 6540 // to test (instead of its class) as argument, and let it deal 6541 // with the read barrier issues. This will let us refactor this 6542 // case of the `switch` code as it was previously (with a direct 6543 // call to the runtime not using a type checking slow path). 6544 // This should also be beneficial for the other cases above. 6545 DCHECK(locations->OnlyCallsOnSlowPath()); 6546 slow_path = new (GetGraph()->GetArena()) TypeCheckSlowPathARM(instruction, 6547 /* is_fatal */ false); 6548 codegen_->AddSlowPath(slow_path); 6549 __ b(slow_path->GetEntryLabel()); 6550 if (zero.IsLinked()) { 6551 __ b(&done); 6552 } 6553 break; 6554 } 6555 } 6556 6557 if (zero.IsLinked()) { 6558 __ Bind(&zero); 6559 __ LoadImmediate(out, 0); 6560 } 6561 6562 if (done.IsLinked()) { 6563 __ Bind(&done); 6564 } 6565 6566 if (slow_path != nullptr) { 6567 __ Bind(slow_path->GetExitLabel()); 6568 } 6569} 6570 6571void LocationsBuilderARM::VisitCheckCast(HCheckCast* instruction) { 6572 LocationSummary::CallKind call_kind = LocationSummary::kNoCall; 6573 bool throws_into_catch = instruction->CanThrowIntoCatchBlock(); 6574 6575 TypeCheckKind type_check_kind = instruction->GetTypeCheckKind(); 6576 switch (type_check_kind) { 6577 case TypeCheckKind::kExactCheck: 6578 case TypeCheckKind::kAbstractClassCheck: 6579 case TypeCheckKind::kClassHierarchyCheck: 6580 case TypeCheckKind::kArrayObjectCheck: 6581 call_kind = (throws_into_catch || kEmitCompilerReadBarrier) ? 6582 LocationSummary::kCallOnSlowPath : 6583 LocationSummary::kNoCall; // In fact, call on a fatal (non-returning) slow path. 6584 break; 6585 case TypeCheckKind::kArrayCheck: 6586 case TypeCheckKind::kUnresolvedCheck: 6587 case TypeCheckKind::kInterfaceCheck: 6588 call_kind = LocationSummary::kCallOnSlowPath; 6589 break; 6590 } 6591 6592 LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction, call_kind); 6593 locations->SetInAt(0, Location::RequiresRegister()); 6594 locations->SetInAt(1, Location::RequiresRegister()); 6595 locations->AddRegisterTemps(NumberOfCheckCastTemps(type_check_kind)); 6596} 6597 6598void InstructionCodeGeneratorARM::VisitCheckCast(HCheckCast* instruction) { 6599 TypeCheckKind type_check_kind = instruction->GetTypeCheckKind(); 6600 LocationSummary* locations = instruction->GetLocations(); 6601 Location obj_loc = locations->InAt(0); 6602 Register obj = obj_loc.AsRegister<Register>(); 6603 Register cls = locations->InAt(1).AsRegister<Register>(); 6604 Location temp_loc = locations->GetTemp(0); 6605 Register temp = temp_loc.AsRegister<Register>(); 6606 const size_t num_temps = NumberOfCheckCastTemps(type_check_kind); 6607 DCHECK_LE(num_temps, 3u); 6608 Location maybe_temp2_loc = (num_temps >= 2) ? locations->GetTemp(1) : Location::NoLocation(); 6609 Location maybe_temp3_loc = (num_temps >= 3) ? locations->GetTemp(2) : Location::NoLocation(); 6610 const uint32_t class_offset = mirror::Object::ClassOffset().Int32Value(); 6611 const uint32_t super_offset = mirror::Class::SuperClassOffset().Int32Value(); 6612 const uint32_t component_offset = mirror::Class::ComponentTypeOffset().Int32Value(); 6613 const uint32_t primitive_offset = mirror::Class::PrimitiveTypeOffset().Int32Value(); 6614 const uint32_t iftable_offset = mirror::Class::IfTableOffset().Uint32Value(); 6615 const uint32_t array_length_offset = mirror::Array::LengthOffset().Uint32Value(); 6616 const uint32_t object_array_data_offset = 6617 mirror::Array::DataOffset(kHeapReferenceSize).Uint32Value(); 6618 6619 // Always false for read barriers since we may need to go to the entrypoint for non-fatal cases 6620 // from false negatives. The false negatives may come from avoiding read barriers below. Avoiding 6621 // read barriers is done for performance and code size reasons. 6622 bool is_type_check_slow_path_fatal = false; 6623 if (!kEmitCompilerReadBarrier) { 6624 is_type_check_slow_path_fatal = 6625 (type_check_kind == TypeCheckKind::kExactCheck || 6626 type_check_kind == TypeCheckKind::kAbstractClassCheck || 6627 type_check_kind == TypeCheckKind::kClassHierarchyCheck || 6628 type_check_kind == TypeCheckKind::kArrayObjectCheck) && 6629 !instruction->CanThrowIntoCatchBlock(); 6630 } 6631 SlowPathCodeARM* type_check_slow_path = 6632 new (GetGraph()->GetArena()) TypeCheckSlowPathARM(instruction, 6633 is_type_check_slow_path_fatal); 6634 codegen_->AddSlowPath(type_check_slow_path); 6635 6636 Label done; 6637 // Avoid null check if we know obj is not null. 6638 if (instruction->MustDoNullCheck()) { 6639 __ CompareAndBranchIfZero(obj, &done); 6640 } 6641 6642 switch (type_check_kind) { 6643 case TypeCheckKind::kExactCheck: 6644 case TypeCheckKind::kArrayCheck: { 6645 // /* HeapReference<Class> */ temp = obj->klass_ 6646 GenerateReferenceLoadTwoRegisters(instruction, 6647 temp_loc, 6648 obj_loc, 6649 class_offset, 6650 maybe_temp2_loc, 6651 kWithoutReadBarrier); 6652 6653 __ cmp(temp, ShifterOperand(cls)); 6654 // Jump to slow path for throwing the exception or doing a 6655 // more involved array check. 6656 __ b(type_check_slow_path->GetEntryLabel(), NE); 6657 break; 6658 } 6659 6660 case TypeCheckKind::kAbstractClassCheck: { 6661 // /* HeapReference<Class> */ temp = obj->klass_ 6662 GenerateReferenceLoadTwoRegisters(instruction, 6663 temp_loc, 6664 obj_loc, 6665 class_offset, 6666 maybe_temp2_loc, 6667 kWithoutReadBarrier); 6668 6669 // If the class is abstract, we eagerly fetch the super class of the 6670 // object to avoid doing a comparison we know will fail. 6671 Label loop; 6672 __ Bind(&loop); 6673 // /* HeapReference<Class> */ temp = temp->super_class_ 6674 GenerateReferenceLoadOneRegister(instruction, 6675 temp_loc, 6676 super_offset, 6677 maybe_temp2_loc, 6678 kWithoutReadBarrier); 6679 6680 // If the class reference currently in `temp` is null, jump to the slow path to throw the 6681 // exception. 6682 __ CompareAndBranchIfZero(temp, type_check_slow_path->GetEntryLabel()); 6683 6684 // Otherwise, compare the classes. 6685 __ cmp(temp, ShifterOperand(cls)); 6686 __ b(&loop, NE); 6687 break; 6688 } 6689 6690 case TypeCheckKind::kClassHierarchyCheck: { 6691 // /* HeapReference<Class> */ temp = obj->klass_ 6692 GenerateReferenceLoadTwoRegisters(instruction, 6693 temp_loc, 6694 obj_loc, 6695 class_offset, 6696 maybe_temp2_loc, 6697 kWithoutReadBarrier); 6698 6699 // Walk over the class hierarchy to find a match. 6700 Label loop; 6701 __ Bind(&loop); 6702 __ cmp(temp, ShifterOperand(cls)); 6703 __ b(&done, EQ); 6704 6705 // /* HeapReference<Class> */ temp = temp->super_class_ 6706 GenerateReferenceLoadOneRegister(instruction, 6707 temp_loc, 6708 super_offset, 6709 maybe_temp2_loc, 6710 kWithoutReadBarrier); 6711 6712 // If the class reference currently in `temp` is null, jump to the slow path to throw the 6713 // exception. 6714 __ CompareAndBranchIfZero(temp, type_check_slow_path->GetEntryLabel()); 6715 // Otherwise, jump to the beginning of the loop. 6716 __ b(&loop); 6717 break; 6718 } 6719 6720 case TypeCheckKind::kArrayObjectCheck: { 6721 // /* HeapReference<Class> */ temp = obj->klass_ 6722 GenerateReferenceLoadTwoRegisters(instruction, 6723 temp_loc, 6724 obj_loc, 6725 class_offset, 6726 maybe_temp2_loc, 6727 kWithoutReadBarrier); 6728 6729 // Do an exact check. 6730 __ cmp(temp, ShifterOperand(cls)); 6731 __ b(&done, EQ); 6732 6733 // Otherwise, we need to check that the object's class is a non-primitive array. 6734 // /* HeapReference<Class> */ temp = temp->component_type_ 6735 GenerateReferenceLoadOneRegister(instruction, 6736 temp_loc, 6737 component_offset, 6738 maybe_temp2_loc, 6739 kWithoutReadBarrier); 6740 // If the component type is null, jump to the slow path to throw the exception. 6741 __ CompareAndBranchIfZero(temp, type_check_slow_path->GetEntryLabel()); 6742 // Otherwise,the object is indeed an array, jump to label `check_non_primitive_component_type` 6743 // to further check that this component type is not a primitive type. 6744 __ LoadFromOffset(kLoadUnsignedHalfword, temp, temp, primitive_offset); 6745 static_assert(Primitive::kPrimNot == 0, "Expected 0 for art::Primitive::kPrimNot"); 6746 __ CompareAndBranchIfNonZero(temp, type_check_slow_path->GetEntryLabel()); 6747 break; 6748 } 6749 6750 case TypeCheckKind::kUnresolvedCheck: 6751 // We always go into the type check slow path for the unresolved check case. 6752 // We cannot directly call the CheckCast runtime entry point 6753 // without resorting to a type checking slow path here (i.e. by 6754 // calling InvokeRuntime directly), as it would require to 6755 // assign fixed registers for the inputs of this HInstanceOf 6756 // instruction (following the runtime calling convention), which 6757 // might be cluttered by the potential first read barrier 6758 // emission at the beginning of this method. 6759 6760 __ b(type_check_slow_path->GetEntryLabel()); 6761 break; 6762 6763 case TypeCheckKind::kInterfaceCheck: { 6764 // Avoid read barriers to improve performance of the fast path. We can not get false 6765 // positives by doing this. 6766 // /* HeapReference<Class> */ temp = obj->klass_ 6767 GenerateReferenceLoadTwoRegisters(instruction, 6768 temp_loc, 6769 obj_loc, 6770 class_offset, 6771 maybe_temp2_loc, 6772 kWithoutReadBarrier); 6773 6774 // /* HeapReference<Class> */ temp = temp->iftable_ 6775 GenerateReferenceLoadTwoRegisters(instruction, 6776 temp_loc, 6777 temp_loc, 6778 iftable_offset, 6779 maybe_temp2_loc, 6780 kWithoutReadBarrier); 6781 // Iftable is never null. 6782 __ ldr(maybe_temp2_loc.AsRegister<Register>(), Address(temp, array_length_offset)); 6783 // Loop through the iftable and check if any class matches. 6784 Label start_loop; 6785 __ Bind(&start_loop); 6786 __ CompareAndBranchIfZero(maybe_temp2_loc.AsRegister<Register>(), 6787 type_check_slow_path->GetEntryLabel()); 6788 __ ldr(maybe_temp3_loc.AsRegister<Register>(), Address(temp, object_array_data_offset)); 6789 __ MaybeUnpoisonHeapReference(maybe_temp3_loc.AsRegister<Register>()); 6790 // Go to next interface. 6791 __ add(temp, temp, ShifterOperand(2 * kHeapReferenceSize)); 6792 __ sub(maybe_temp2_loc.AsRegister<Register>(), 6793 maybe_temp2_loc.AsRegister<Register>(), 6794 ShifterOperand(2)); 6795 // Compare the classes and continue the loop if they do not match. 6796 __ cmp(cls, ShifterOperand(maybe_temp3_loc.AsRegister<Register>())); 6797 __ b(&start_loop, NE); 6798 break; 6799 } 6800 } 6801 __ Bind(&done); 6802 6803 __ Bind(type_check_slow_path->GetExitLabel()); 6804} 6805 6806void LocationsBuilderARM::VisitMonitorOperation(HMonitorOperation* instruction) { 6807 LocationSummary* locations = 6808 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCallOnMainOnly); 6809 InvokeRuntimeCallingConvention calling_convention; 6810 locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(0))); 6811} 6812 6813void InstructionCodeGeneratorARM::VisitMonitorOperation(HMonitorOperation* instruction) { 6814 codegen_->InvokeRuntime(instruction->IsEnter() ? kQuickLockObject : kQuickUnlockObject, 6815 instruction, 6816 instruction->GetDexPc()); 6817 if (instruction->IsEnter()) { 6818 CheckEntrypointTypes<kQuickLockObject, void, mirror::Object*>(); 6819 } else { 6820 CheckEntrypointTypes<kQuickUnlockObject, void, mirror::Object*>(); 6821 } 6822} 6823 6824void LocationsBuilderARM::VisitAnd(HAnd* instruction) { HandleBitwiseOperation(instruction, AND); } 6825void LocationsBuilderARM::VisitOr(HOr* instruction) { HandleBitwiseOperation(instruction, ORR); } 6826void LocationsBuilderARM::VisitXor(HXor* instruction) { HandleBitwiseOperation(instruction, EOR); } 6827 6828void LocationsBuilderARM::HandleBitwiseOperation(HBinaryOperation* instruction, Opcode opcode) { 6829 LocationSummary* locations = 6830 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall); 6831 DCHECK(instruction->GetResultType() == Primitive::kPrimInt 6832 || instruction->GetResultType() == Primitive::kPrimLong); 6833 // Note: GVN reorders commutative operations to have the constant on the right hand side. 6834 locations->SetInAt(0, Location::RequiresRegister()); 6835 locations->SetInAt(1, ArmEncodableConstantOrRegister(instruction->InputAt(1), opcode)); 6836 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 6837} 6838 6839void InstructionCodeGeneratorARM::VisitAnd(HAnd* instruction) { 6840 HandleBitwiseOperation(instruction); 6841} 6842 6843void InstructionCodeGeneratorARM::VisitOr(HOr* instruction) { 6844 HandleBitwiseOperation(instruction); 6845} 6846 6847void InstructionCodeGeneratorARM::VisitXor(HXor* instruction) { 6848 HandleBitwiseOperation(instruction); 6849} 6850 6851 6852void LocationsBuilderARM::VisitBitwiseNegatedRight(HBitwiseNegatedRight* instruction) { 6853 LocationSummary* locations = 6854 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall); 6855 DCHECK(instruction->GetResultType() == Primitive::kPrimInt 6856 || instruction->GetResultType() == Primitive::kPrimLong); 6857 6858 locations->SetInAt(0, Location::RequiresRegister()); 6859 locations->SetInAt(1, Location::RequiresRegister()); 6860 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 6861} 6862 6863void InstructionCodeGeneratorARM::VisitBitwiseNegatedRight(HBitwiseNegatedRight* instruction) { 6864 LocationSummary* locations = instruction->GetLocations(); 6865 Location first = locations->InAt(0); 6866 Location second = locations->InAt(1); 6867 Location out = locations->Out(); 6868 6869 if (instruction->GetResultType() == Primitive::kPrimInt) { 6870 Register first_reg = first.AsRegister<Register>(); 6871 ShifterOperand second_reg(second.AsRegister<Register>()); 6872 Register out_reg = out.AsRegister<Register>(); 6873 6874 switch (instruction->GetOpKind()) { 6875 case HInstruction::kAnd: 6876 __ bic(out_reg, first_reg, second_reg); 6877 break; 6878 case HInstruction::kOr: 6879 __ orn(out_reg, first_reg, second_reg); 6880 break; 6881 // There is no EON on arm. 6882 case HInstruction::kXor: 6883 default: 6884 LOG(FATAL) << "Unexpected instruction " << instruction->DebugName(); 6885 UNREACHABLE(); 6886 } 6887 return; 6888 6889 } else { 6890 DCHECK_EQ(instruction->GetResultType(), Primitive::kPrimLong); 6891 Register first_low = first.AsRegisterPairLow<Register>(); 6892 Register first_high = first.AsRegisterPairHigh<Register>(); 6893 ShifterOperand second_low(second.AsRegisterPairLow<Register>()); 6894 ShifterOperand second_high(second.AsRegisterPairHigh<Register>()); 6895 Register out_low = out.AsRegisterPairLow<Register>(); 6896 Register out_high = out.AsRegisterPairHigh<Register>(); 6897 6898 switch (instruction->GetOpKind()) { 6899 case HInstruction::kAnd: 6900 __ bic(out_low, first_low, second_low); 6901 __ bic(out_high, first_high, second_high); 6902 break; 6903 case HInstruction::kOr: 6904 __ orn(out_low, first_low, second_low); 6905 __ orn(out_high, first_high, second_high); 6906 break; 6907 // There is no EON on arm. 6908 case HInstruction::kXor: 6909 default: 6910 LOG(FATAL) << "Unexpected instruction " << instruction->DebugName(); 6911 UNREACHABLE(); 6912 } 6913 } 6914} 6915 6916void LocationsBuilderARM::VisitDataProcWithShifterOp( 6917 HDataProcWithShifterOp* instruction) { 6918 DCHECK(instruction->GetType() == Primitive::kPrimInt || 6919 instruction->GetType() == Primitive::kPrimLong); 6920 LocationSummary* locations = 6921 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall); 6922 const bool overlap = instruction->GetType() == Primitive::kPrimLong && 6923 HDataProcWithShifterOp::IsExtensionOp(instruction->GetOpKind()); 6924 6925 locations->SetInAt(0, Location::RequiresRegister()); 6926 locations->SetInAt(1, Location::RequiresRegister()); 6927 locations->SetOut(Location::RequiresRegister(), 6928 overlap ? Location::kOutputOverlap : Location::kNoOutputOverlap); 6929} 6930 6931void InstructionCodeGeneratorARM::VisitDataProcWithShifterOp( 6932 HDataProcWithShifterOp* instruction) { 6933 const LocationSummary* const locations = instruction->GetLocations(); 6934 const HInstruction::InstructionKind kind = instruction->GetInstrKind(); 6935 const HDataProcWithShifterOp::OpKind op_kind = instruction->GetOpKind(); 6936 const Location left = locations->InAt(0); 6937 const Location right = locations->InAt(1); 6938 const Location out = locations->Out(); 6939 6940 if (instruction->GetType() == Primitive::kPrimInt) { 6941 DCHECK(!HDataProcWithShifterOp::IsExtensionOp(op_kind)); 6942 6943 const Register second = instruction->InputAt(1)->GetType() == Primitive::kPrimLong 6944 ? right.AsRegisterPairLow<Register>() 6945 : right.AsRegister<Register>(); 6946 6947 GenerateDataProcInstruction(kind, 6948 out.AsRegister<Register>(), 6949 left.AsRegister<Register>(), 6950 ShifterOperand(second, 6951 ShiftFromOpKind(op_kind), 6952 instruction->GetShiftAmount()), 6953 codegen_); 6954 } else { 6955 DCHECK_EQ(instruction->GetType(), Primitive::kPrimLong); 6956 6957 if (HDataProcWithShifterOp::IsExtensionOp(op_kind)) { 6958 const Register second = right.AsRegister<Register>(); 6959 6960 DCHECK_NE(out.AsRegisterPairLow<Register>(), second); 6961 GenerateDataProc(kind, 6962 out, 6963 left, 6964 ShifterOperand(second), 6965 ShifterOperand(second, ASR, 31), 6966 codegen_); 6967 } else { 6968 GenerateLongDataProc(instruction, codegen_); 6969 } 6970 } 6971} 6972 6973void InstructionCodeGeneratorARM::GenerateAndConst(Register out, Register first, uint32_t value) { 6974 // Optimize special cases for individual halfs of `and-long` (`and` is simplified earlier). 6975 if (value == 0xffffffffu) { 6976 if (out != first) { 6977 __ mov(out, ShifterOperand(first)); 6978 } 6979 return; 6980 } 6981 if (value == 0u) { 6982 __ mov(out, ShifterOperand(0)); 6983 return; 6984 } 6985 ShifterOperand so; 6986 if (__ ShifterOperandCanHold(kNoRegister, kNoRegister, AND, value, &so)) { 6987 __ and_(out, first, so); 6988 } else { 6989 DCHECK(__ ShifterOperandCanHold(kNoRegister, kNoRegister, BIC, ~value, &so)); 6990 __ bic(out, first, ShifterOperand(~value)); 6991 } 6992} 6993 6994void InstructionCodeGeneratorARM::GenerateOrrConst(Register out, Register first, uint32_t value) { 6995 // Optimize special cases for individual halfs of `or-long` (`or` is simplified earlier). 6996 if (value == 0u) { 6997 if (out != first) { 6998 __ mov(out, ShifterOperand(first)); 6999 } 7000 return; 7001 } 7002 if (value == 0xffffffffu) { 7003 __ mvn(out, ShifterOperand(0)); 7004 return; 7005 } 7006 ShifterOperand so; 7007 if (__ ShifterOperandCanHold(kNoRegister, kNoRegister, ORR, value, &so)) { 7008 __ orr(out, first, so); 7009 } else { 7010 DCHECK(__ ShifterOperandCanHold(kNoRegister, kNoRegister, ORN, ~value, &so)); 7011 __ orn(out, first, ShifterOperand(~value)); 7012 } 7013} 7014 7015void InstructionCodeGeneratorARM::GenerateEorConst(Register out, Register first, uint32_t value) { 7016 // Optimize special case for individual halfs of `xor-long` (`xor` is simplified earlier). 7017 if (value == 0u) { 7018 if (out != first) { 7019 __ mov(out, ShifterOperand(first)); 7020 } 7021 return; 7022 } 7023 __ eor(out, first, ShifterOperand(value)); 7024} 7025 7026void InstructionCodeGeneratorARM::GenerateAddLongConst(Location out, 7027 Location first, 7028 uint64_t value) { 7029 Register out_low = out.AsRegisterPairLow<Register>(); 7030 Register out_high = out.AsRegisterPairHigh<Register>(); 7031 Register first_low = first.AsRegisterPairLow<Register>(); 7032 Register first_high = first.AsRegisterPairHigh<Register>(); 7033 uint32_t value_low = Low32Bits(value); 7034 uint32_t value_high = High32Bits(value); 7035 if (value_low == 0u) { 7036 if (out_low != first_low) { 7037 __ mov(out_low, ShifterOperand(first_low)); 7038 } 7039 __ AddConstant(out_high, first_high, value_high); 7040 return; 7041 } 7042 __ AddConstantSetFlags(out_low, first_low, value_low); 7043 ShifterOperand so; 7044 if (__ ShifterOperandCanHold(out_high, first_high, ADC, value_high, kCcDontCare, &so)) { 7045 __ adc(out_high, first_high, so); 7046 } else if (__ ShifterOperandCanHold(out_low, first_low, SBC, ~value_high, kCcDontCare, &so)) { 7047 __ sbc(out_high, first_high, so); 7048 } else { 7049 LOG(FATAL) << "Unexpected constant " << value_high; 7050 UNREACHABLE(); 7051 } 7052} 7053 7054void InstructionCodeGeneratorARM::HandleBitwiseOperation(HBinaryOperation* instruction) { 7055 LocationSummary* locations = instruction->GetLocations(); 7056 Location first = locations->InAt(0); 7057 Location second = locations->InAt(1); 7058 Location out = locations->Out(); 7059 7060 if (second.IsConstant()) { 7061 uint64_t value = static_cast<uint64_t>(Int64FromConstant(second.GetConstant())); 7062 uint32_t value_low = Low32Bits(value); 7063 if (instruction->GetResultType() == Primitive::kPrimInt) { 7064 Register first_reg = first.AsRegister<Register>(); 7065 Register out_reg = out.AsRegister<Register>(); 7066 if (instruction->IsAnd()) { 7067 GenerateAndConst(out_reg, first_reg, value_low); 7068 } else if (instruction->IsOr()) { 7069 GenerateOrrConst(out_reg, first_reg, value_low); 7070 } else { 7071 DCHECK(instruction->IsXor()); 7072 GenerateEorConst(out_reg, first_reg, value_low); 7073 } 7074 } else { 7075 DCHECK_EQ(instruction->GetResultType(), Primitive::kPrimLong); 7076 uint32_t value_high = High32Bits(value); 7077 Register first_low = first.AsRegisterPairLow<Register>(); 7078 Register first_high = first.AsRegisterPairHigh<Register>(); 7079 Register out_low = out.AsRegisterPairLow<Register>(); 7080 Register out_high = out.AsRegisterPairHigh<Register>(); 7081 if (instruction->IsAnd()) { 7082 GenerateAndConst(out_low, first_low, value_low); 7083 GenerateAndConst(out_high, first_high, value_high); 7084 } else if (instruction->IsOr()) { 7085 GenerateOrrConst(out_low, first_low, value_low); 7086 GenerateOrrConst(out_high, first_high, value_high); 7087 } else { 7088 DCHECK(instruction->IsXor()); 7089 GenerateEorConst(out_low, first_low, value_low); 7090 GenerateEorConst(out_high, first_high, value_high); 7091 } 7092 } 7093 return; 7094 } 7095 7096 if (instruction->GetResultType() == Primitive::kPrimInt) { 7097 Register first_reg = first.AsRegister<Register>(); 7098 ShifterOperand second_reg(second.AsRegister<Register>()); 7099 Register out_reg = out.AsRegister<Register>(); 7100 if (instruction->IsAnd()) { 7101 __ and_(out_reg, first_reg, second_reg); 7102 } else if (instruction->IsOr()) { 7103 __ orr(out_reg, first_reg, second_reg); 7104 } else { 7105 DCHECK(instruction->IsXor()); 7106 __ eor(out_reg, first_reg, second_reg); 7107 } 7108 } else { 7109 DCHECK_EQ(instruction->GetResultType(), Primitive::kPrimLong); 7110 Register first_low = first.AsRegisterPairLow<Register>(); 7111 Register first_high = first.AsRegisterPairHigh<Register>(); 7112 ShifterOperand second_low(second.AsRegisterPairLow<Register>()); 7113 ShifterOperand second_high(second.AsRegisterPairHigh<Register>()); 7114 Register out_low = out.AsRegisterPairLow<Register>(); 7115 Register out_high = out.AsRegisterPairHigh<Register>(); 7116 if (instruction->IsAnd()) { 7117 __ and_(out_low, first_low, second_low); 7118 __ and_(out_high, first_high, second_high); 7119 } else if (instruction->IsOr()) { 7120 __ orr(out_low, first_low, second_low); 7121 __ orr(out_high, first_high, second_high); 7122 } else { 7123 DCHECK(instruction->IsXor()); 7124 __ eor(out_low, first_low, second_low); 7125 __ eor(out_high, first_high, second_high); 7126 } 7127 } 7128} 7129 7130void InstructionCodeGeneratorARM::GenerateReferenceLoadOneRegister( 7131 HInstruction* instruction, 7132 Location out, 7133 uint32_t offset, 7134 Location maybe_temp, 7135 ReadBarrierOption read_barrier_option) { 7136 Register out_reg = out.AsRegister<Register>(); 7137 if (read_barrier_option == kWithReadBarrier) { 7138 CHECK(kEmitCompilerReadBarrier); 7139 DCHECK(maybe_temp.IsRegister()) << maybe_temp; 7140 if (kUseBakerReadBarrier) { 7141 // Load with fast path based Baker's read barrier. 7142 // /* HeapReference<Object> */ out = *(out + offset) 7143 codegen_->GenerateFieldLoadWithBakerReadBarrier( 7144 instruction, out, out_reg, offset, maybe_temp, /* needs_null_check */ false); 7145 } else { 7146 // Load with slow path based read barrier. 7147 // Save the value of `out` into `maybe_temp` before overwriting it 7148 // in the following move operation, as we will need it for the 7149 // read barrier below. 7150 __ Mov(maybe_temp.AsRegister<Register>(), out_reg); 7151 // /* HeapReference<Object> */ out = *(out + offset) 7152 __ LoadFromOffset(kLoadWord, out_reg, out_reg, offset); 7153 codegen_->GenerateReadBarrierSlow(instruction, out, out, maybe_temp, offset); 7154 } 7155 } else { 7156 // Plain load with no read barrier. 7157 // /* HeapReference<Object> */ out = *(out + offset) 7158 __ LoadFromOffset(kLoadWord, out_reg, out_reg, offset); 7159 __ MaybeUnpoisonHeapReference(out_reg); 7160 } 7161} 7162 7163void InstructionCodeGeneratorARM::GenerateReferenceLoadTwoRegisters( 7164 HInstruction* instruction, 7165 Location out, 7166 Location obj, 7167 uint32_t offset, 7168 Location maybe_temp, 7169 ReadBarrierOption read_barrier_option) { 7170 Register out_reg = out.AsRegister<Register>(); 7171 Register obj_reg = obj.AsRegister<Register>(); 7172 if (read_barrier_option == kWithReadBarrier) { 7173 CHECK(kEmitCompilerReadBarrier); 7174 if (kUseBakerReadBarrier) { 7175 DCHECK(maybe_temp.IsRegister()) << maybe_temp; 7176 // Load with fast path based Baker's read barrier. 7177 // /* HeapReference<Object> */ out = *(obj + offset) 7178 codegen_->GenerateFieldLoadWithBakerReadBarrier( 7179 instruction, out, obj_reg, offset, maybe_temp, /* needs_null_check */ false); 7180 } else { 7181 // Load with slow path based read barrier. 7182 // /* HeapReference<Object> */ out = *(obj + offset) 7183 __ LoadFromOffset(kLoadWord, out_reg, obj_reg, offset); 7184 codegen_->GenerateReadBarrierSlow(instruction, out, out, obj, offset); 7185 } 7186 } else { 7187 // Plain load with no read barrier. 7188 // /* HeapReference<Object> */ out = *(obj + offset) 7189 __ LoadFromOffset(kLoadWord, out_reg, obj_reg, offset); 7190 __ MaybeUnpoisonHeapReference(out_reg); 7191 } 7192} 7193 7194void InstructionCodeGeneratorARM::GenerateGcRootFieldLoad(HInstruction* instruction, 7195 Location root, 7196 Register obj, 7197 uint32_t offset, 7198 ReadBarrierOption read_barrier_option) { 7199 Register root_reg = root.AsRegister<Register>(); 7200 if (read_barrier_option == kWithReadBarrier) { 7201 DCHECK(kEmitCompilerReadBarrier); 7202 if (kUseBakerReadBarrier) { 7203 // Fast path implementation of art::ReadBarrier::BarrierForRoot when 7204 // Baker's read barrier are used. 7205 // 7206 // Note that we do not actually check the value of 7207 // `GetIsGcMarking()` to decide whether to mark the loaded GC 7208 // root or not. Instead, we load into `temp` the read barrier 7209 // mark entry point corresponding to register `root`. If `temp` 7210 // is null, it means that `GetIsGcMarking()` is false, and vice 7211 // versa. 7212 // 7213 // temp = Thread::Current()->pReadBarrierMarkReg ## root.reg() 7214 // GcRoot<mirror::Object> root = *(obj+offset); // Original reference load. 7215 // if (temp != nullptr) { // <=> Thread::Current()->GetIsGcMarking() 7216 // // Slow path. 7217 // root = temp(root); // root = ReadBarrier::Mark(root); // Runtime entry point call. 7218 // } 7219 7220 // Slow path marking the GC root `root`. The entrypoint will already be loaded in `temp`. 7221 Location temp = Location::RegisterLocation(LR); 7222 SlowPathCodeARM* slow_path = new (GetGraph()->GetArena()) ReadBarrierMarkSlowPathARM( 7223 instruction, root, /* entrypoint */ temp); 7224 codegen_->AddSlowPath(slow_path); 7225 7226 // temp = Thread::Current()->pReadBarrierMarkReg ## root.reg() 7227 const int32_t entry_point_offset = 7228 CodeGenerator::GetReadBarrierMarkEntryPointsOffset<kArmPointerSize>(root.reg()); 7229 // Loading the entrypoint does not require a load acquire since it is only changed when 7230 // threads are suspended or running a checkpoint. 7231 __ LoadFromOffset(kLoadWord, temp.AsRegister<Register>(), TR, entry_point_offset); 7232 7233 // /* GcRoot<mirror::Object> */ root = *(obj + offset) 7234 __ LoadFromOffset(kLoadWord, root_reg, obj, offset); 7235 static_assert( 7236 sizeof(mirror::CompressedReference<mirror::Object>) == sizeof(GcRoot<mirror::Object>), 7237 "art::mirror::CompressedReference<mirror::Object> and art::GcRoot<mirror::Object> " 7238 "have different sizes."); 7239 static_assert(sizeof(mirror::CompressedReference<mirror::Object>) == sizeof(int32_t), 7240 "art::mirror::CompressedReference<mirror::Object> and int32_t " 7241 "have different sizes."); 7242 7243 // The entrypoint is null when the GC is not marking, this prevents one load compared to 7244 // checking GetIsGcMarking. 7245 __ CompareAndBranchIfNonZero(temp.AsRegister<Register>(), slow_path->GetEntryLabel()); 7246 __ Bind(slow_path->GetExitLabel()); 7247 } else { 7248 // GC root loaded through a slow path for read barriers other 7249 // than Baker's. 7250 // /* GcRoot<mirror::Object>* */ root = obj + offset 7251 __ AddConstant(root_reg, obj, offset); 7252 // /* mirror::Object* */ root = root->Read() 7253 codegen_->GenerateReadBarrierForRootSlow(instruction, root, root); 7254 } 7255 } else { 7256 // Plain GC root load with no read barrier. 7257 // /* GcRoot<mirror::Object> */ root = *(obj + offset) 7258 __ LoadFromOffset(kLoadWord, root_reg, obj, offset); 7259 // Note that GC roots are not affected by heap poisoning, thus we 7260 // do not have to unpoison `root_reg` here. 7261 } 7262} 7263 7264void CodeGeneratorARM::GenerateFieldLoadWithBakerReadBarrier(HInstruction* instruction, 7265 Location ref, 7266 Register obj, 7267 uint32_t offset, 7268 Location temp, 7269 bool needs_null_check) { 7270 DCHECK(kEmitCompilerReadBarrier); 7271 DCHECK(kUseBakerReadBarrier); 7272 7273 // /* HeapReference<Object> */ ref = *(obj + offset) 7274 Location no_index = Location::NoLocation(); 7275 ScaleFactor no_scale_factor = TIMES_1; 7276 GenerateReferenceLoadWithBakerReadBarrier( 7277 instruction, ref, obj, offset, no_index, no_scale_factor, temp, needs_null_check); 7278} 7279 7280void CodeGeneratorARM::GenerateArrayLoadWithBakerReadBarrier(HInstruction* instruction, 7281 Location ref, 7282 Register obj, 7283 uint32_t data_offset, 7284 Location index, 7285 Location temp, 7286 bool needs_null_check) { 7287 DCHECK(kEmitCompilerReadBarrier); 7288 DCHECK(kUseBakerReadBarrier); 7289 7290 static_assert( 7291 sizeof(mirror::HeapReference<mirror::Object>) == sizeof(int32_t), 7292 "art::mirror::HeapReference<art::mirror::Object> and int32_t have different sizes."); 7293 // /* HeapReference<Object> */ ref = 7294 // *(obj + data_offset + index * sizeof(HeapReference<Object>)) 7295 ScaleFactor scale_factor = TIMES_4; 7296 GenerateReferenceLoadWithBakerReadBarrier( 7297 instruction, ref, obj, data_offset, index, scale_factor, temp, needs_null_check); 7298} 7299 7300void CodeGeneratorARM::GenerateReferenceLoadWithBakerReadBarrier(HInstruction* instruction, 7301 Location ref, 7302 Register obj, 7303 uint32_t offset, 7304 Location index, 7305 ScaleFactor scale_factor, 7306 Location temp, 7307 bool needs_null_check, 7308 bool always_update_field, 7309 Register* temp2) { 7310 DCHECK(kEmitCompilerReadBarrier); 7311 DCHECK(kUseBakerReadBarrier); 7312 7313 // After loading the reference from `obj.field` into `ref`, query 7314 // `art::Thread::Current()->GetIsGcMarking()` to decide whether we 7315 // need to enter the slow path to mark the reference. This 7316 // optimistic strategy (we expect the GC to not be marking most of 7317 // the time) does not check `obj`'s lock word (to see if it is a 7318 // gray object or not), so may sometimes mark an already marked 7319 // object. 7320 // 7321 // Note that we do not actually check the value of `GetIsGcMarking()`; 7322 // instead, we load into `temp3` the read barrier mark entry point 7323 // corresponding to register `ref`. If `temp3` is null, it means 7324 // that `GetIsGcMarking()` is false, and vice versa. 7325 // 7326 // temp3 = Thread::Current()->pReadBarrierMarkReg ## root.reg() 7327 // HeapReference<mirror::Object> ref = *src; // Original reference load. 7328 // if (temp3 != nullptr) { // <=> Thread::Current()->GetIsGcMarking() 7329 // // Slow path. 7330 // ref = temp3(ref); // ref = ReadBarrier::Mark(ref); // Runtime entry point call. 7331 // } 7332 7333 // TODO: This temp register is only necessary when 7334 // `always_update_field` is true; make it optional (like `temp2`). 7335 Register temp_reg = temp.AsRegister<Register>(); 7336 7337 // Slow path marking the object `ref` when the GC is marking. The 7338 // entrypoint will already be loaded in `temp3`. 7339 Location temp3 = Location::RegisterLocation(LR); 7340 SlowPathCodeARM* slow_path; 7341 if (always_update_field) { 7342 DCHECK(temp2 != nullptr); 7343 // ReadBarrierMarkAndUpdateFieldSlowPathARM only supports address 7344 // of the form `obj + field_offset`, where `obj` is a register and 7345 // `field_offset` is a register pair (of which only the lower half 7346 // is used). Thus `offset` and `scale_factor` above are expected 7347 // to be null in this code path. 7348 DCHECK_EQ(offset, 0u); 7349 DCHECK_EQ(scale_factor, ScaleFactor::TIMES_1); 7350 slow_path = new (GetGraph()->GetArena()) ReadBarrierMarkAndUpdateFieldSlowPathARM( 7351 instruction, ref, obj, /* field_offset */ index, temp_reg, *temp2, /* entrypoint */ temp3); 7352 } else { 7353 slow_path = new (GetGraph()->GetArena()) ReadBarrierMarkSlowPathARM( 7354 instruction, ref, /* entrypoint */ temp3); 7355 } 7356 AddSlowPath(slow_path); 7357 7358 // temp3 = Thread::Current()->pReadBarrierMarkReg ## ref.reg() 7359 const int32_t entry_point_offset = 7360 CodeGenerator::GetReadBarrierMarkEntryPointsOffset<kArmPointerSize>(ref.reg()); 7361 // Loading the entrypoint does not require a load acquire since it is only changed when 7362 // threads are suspended or running a checkpoint. 7363 __ LoadFromOffset(kLoadWord, temp3.AsRegister<Register>(), TR, entry_point_offset); 7364 // The reference load. 7365 GenerateRawReferenceLoad(instruction, ref, obj, offset, index, scale_factor, needs_null_check); 7366 // The entrypoint is null when the GC is not marking, this prevents one load compared to 7367 // checking GetIsGcMarking. 7368 __ CompareAndBranchIfNonZero(temp3.AsRegister<Register>(), slow_path->GetEntryLabel()); 7369 __ Bind(slow_path->GetExitLabel()); 7370} 7371 7372void CodeGeneratorARM::GenerateRawReferenceLoad(HInstruction* instruction, 7373 Location ref, 7374 Register obj, 7375 uint32_t offset, 7376 Location index, 7377 ScaleFactor scale_factor, 7378 bool needs_null_check) { 7379 Register ref_reg = ref.AsRegister<Register>(); 7380 7381 if (index.IsValid()) { 7382 // Load types involving an "index": ArrayGet, 7383 // UnsafeGetObject/UnsafeGetObjectVolatile and UnsafeCASObject 7384 // intrinsics. 7385 // /* HeapReference<mirror::Object> */ ref = *(obj + offset + (index << scale_factor)) 7386 if (index.IsConstant()) { 7387 size_t computed_offset = 7388 (index.GetConstant()->AsIntConstant()->GetValue() << scale_factor) + offset; 7389 __ LoadFromOffset(kLoadWord, ref_reg, obj, computed_offset); 7390 } else { 7391 // Handle the special case of the 7392 // UnsafeGetObject/UnsafeGetObjectVolatile and UnsafeCASObject 7393 // intrinsics, which use a register pair as index ("long 7394 // offset"), of which only the low part contains data. 7395 Register index_reg = index.IsRegisterPair() 7396 ? index.AsRegisterPairLow<Register>() 7397 : index.AsRegister<Register>(); 7398 __ add(IP, obj, ShifterOperand(index_reg, LSL, scale_factor)); 7399 __ LoadFromOffset(kLoadWord, ref_reg, IP, offset); 7400 } 7401 } else { 7402 // /* HeapReference<mirror::Object> */ ref = *(obj + offset) 7403 __ LoadFromOffset(kLoadWord, ref_reg, obj, offset); 7404 } 7405 7406 if (needs_null_check) { 7407 MaybeRecordImplicitNullCheck(instruction); 7408 } 7409 7410 // Object* ref = ref_addr->AsMirrorPtr() 7411 __ MaybeUnpoisonHeapReference(ref_reg); 7412} 7413 7414void CodeGeneratorARM::GenerateReadBarrierSlow(HInstruction* instruction, 7415 Location out, 7416 Location ref, 7417 Location obj, 7418 uint32_t offset, 7419 Location index) { 7420 DCHECK(kEmitCompilerReadBarrier); 7421 7422 // Insert a slow path based read barrier *after* the reference load. 7423 // 7424 // If heap poisoning is enabled, the unpoisoning of the loaded 7425 // reference will be carried out by the runtime within the slow 7426 // path. 7427 // 7428 // Note that `ref` currently does not get unpoisoned (when heap 7429 // poisoning is enabled), which is alright as the `ref` argument is 7430 // not used by the artReadBarrierSlow entry point. 7431 // 7432 // TODO: Unpoison `ref` when it is used by artReadBarrierSlow. 7433 SlowPathCodeARM* slow_path = new (GetGraph()->GetArena()) 7434 ReadBarrierForHeapReferenceSlowPathARM(instruction, out, ref, obj, offset, index); 7435 AddSlowPath(slow_path); 7436 7437 __ b(slow_path->GetEntryLabel()); 7438 __ Bind(slow_path->GetExitLabel()); 7439} 7440 7441void CodeGeneratorARM::MaybeGenerateReadBarrierSlow(HInstruction* instruction, 7442 Location out, 7443 Location ref, 7444 Location obj, 7445 uint32_t offset, 7446 Location index) { 7447 if (kEmitCompilerReadBarrier) { 7448 // Baker's read barriers shall be handled by the fast path 7449 // (CodeGeneratorARM::GenerateReferenceLoadWithBakerReadBarrier). 7450 DCHECK(!kUseBakerReadBarrier); 7451 // If heap poisoning is enabled, unpoisoning will be taken care of 7452 // by the runtime within the slow path. 7453 GenerateReadBarrierSlow(instruction, out, ref, obj, offset, index); 7454 } else if (kPoisonHeapReferences) { 7455 __ UnpoisonHeapReference(out.AsRegister<Register>()); 7456 } 7457} 7458 7459void CodeGeneratorARM::GenerateReadBarrierForRootSlow(HInstruction* instruction, 7460 Location out, 7461 Location root) { 7462 DCHECK(kEmitCompilerReadBarrier); 7463 7464 // Insert a slow path based read barrier *after* the GC root load. 7465 // 7466 // Note that GC roots are not affected by heap poisoning, so we do 7467 // not need to do anything special for this here. 7468 SlowPathCodeARM* slow_path = 7469 new (GetGraph()->GetArena()) ReadBarrierForRootSlowPathARM(instruction, out, root); 7470 AddSlowPath(slow_path); 7471 7472 __ b(slow_path->GetEntryLabel()); 7473 __ Bind(slow_path->GetExitLabel()); 7474} 7475 7476HInvokeStaticOrDirect::DispatchInfo CodeGeneratorARM::GetSupportedInvokeStaticOrDirectDispatch( 7477 const HInvokeStaticOrDirect::DispatchInfo& desired_dispatch_info, 7478 HInvokeStaticOrDirect* invoke ATTRIBUTE_UNUSED) { 7479 return desired_dispatch_info; 7480} 7481 7482Register CodeGeneratorARM::GetInvokeStaticOrDirectExtraParameter(HInvokeStaticOrDirect* invoke, 7483 Register temp) { 7484 DCHECK_EQ(invoke->InputCount(), invoke->GetNumberOfArguments() + 1u); 7485 Location location = invoke->GetLocations()->InAt(invoke->GetSpecialInputIndex()); 7486 if (!invoke->GetLocations()->Intrinsified()) { 7487 return location.AsRegister<Register>(); 7488 } 7489 // For intrinsics we allow any location, so it may be on the stack. 7490 if (!location.IsRegister()) { 7491 __ LoadFromOffset(kLoadWord, temp, SP, location.GetStackIndex()); 7492 return temp; 7493 } 7494 // For register locations, check if the register was saved. If so, get it from the stack. 7495 // Note: There is a chance that the register was saved but not overwritten, so we could 7496 // save one load. However, since this is just an intrinsic slow path we prefer this 7497 // simple and more robust approach rather that trying to determine if that's the case. 7498 SlowPathCode* slow_path = GetCurrentSlowPath(); 7499 if (slow_path != nullptr && slow_path->IsCoreRegisterSaved(location.AsRegister<Register>())) { 7500 int stack_offset = slow_path->GetStackOffsetOfCoreRegister(location.AsRegister<Register>()); 7501 __ LoadFromOffset(kLoadWord, temp, SP, stack_offset); 7502 return temp; 7503 } 7504 return location.AsRegister<Register>(); 7505} 7506 7507Location CodeGeneratorARM::GenerateCalleeMethodStaticOrDirectCall(HInvokeStaticOrDirect* invoke, 7508 Location temp) { 7509 Location callee_method = temp; // For all kinds except kRecursive, callee will be in temp. 7510 switch (invoke->GetMethodLoadKind()) { 7511 case HInvokeStaticOrDirect::MethodLoadKind::kStringInit: { 7512 uint32_t offset = 7513 GetThreadOffset<kArmPointerSize>(invoke->GetStringInitEntryPoint()).Int32Value(); 7514 // temp = thread->string_init_entrypoint 7515 __ LoadFromOffset(kLoadWord, temp.AsRegister<Register>(), TR, offset); 7516 break; 7517 } 7518 case HInvokeStaticOrDirect::MethodLoadKind::kRecursive: 7519 callee_method = invoke->GetLocations()->InAt(invoke->GetSpecialInputIndex()); 7520 break; 7521 case HInvokeStaticOrDirect::MethodLoadKind::kDirectAddress: 7522 __ LoadImmediate(temp.AsRegister<Register>(), invoke->GetMethodAddress()); 7523 break; 7524 case HInvokeStaticOrDirect::MethodLoadKind::kDexCachePcRelative: { 7525 HArmDexCacheArraysBase* base = 7526 invoke->InputAt(invoke->GetSpecialInputIndex())->AsArmDexCacheArraysBase(); 7527 Register base_reg = GetInvokeStaticOrDirectExtraParameter(invoke, 7528 temp.AsRegister<Register>()); 7529 int32_t offset = invoke->GetDexCacheArrayOffset() - base->GetElementOffset(); 7530 __ LoadFromOffset(kLoadWord, temp.AsRegister<Register>(), base_reg, offset); 7531 break; 7532 } 7533 case HInvokeStaticOrDirect::MethodLoadKind::kDexCacheViaMethod: { 7534 Location current_method = invoke->GetLocations()->InAt(invoke->GetSpecialInputIndex()); 7535 Register method_reg; 7536 Register reg = temp.AsRegister<Register>(); 7537 if (current_method.IsRegister()) { 7538 method_reg = current_method.AsRegister<Register>(); 7539 } else { 7540 DCHECK(invoke->GetLocations()->Intrinsified()); 7541 DCHECK(!current_method.IsValid()); 7542 method_reg = reg; 7543 __ LoadFromOffset(kLoadWord, reg, SP, kCurrentMethodStackOffset); 7544 } 7545 // /* ArtMethod*[] */ temp = temp.ptr_sized_fields_->dex_cache_resolved_methods_; 7546 __ LoadFromOffset(kLoadWord, 7547 reg, 7548 method_reg, 7549 ArtMethod::DexCacheResolvedMethodsOffset(kArmPointerSize).Int32Value()); 7550 // temp = temp[index_in_cache]; 7551 // Note: Don't use invoke->GetTargetMethod() as it may point to a different dex file. 7552 uint32_t index_in_cache = invoke->GetDexMethodIndex(); 7553 __ LoadFromOffset(kLoadWord, reg, reg, CodeGenerator::GetCachePointerOffset(index_in_cache)); 7554 break; 7555 } 7556 } 7557 return callee_method; 7558} 7559 7560void CodeGeneratorARM::GenerateStaticOrDirectCall(HInvokeStaticOrDirect* invoke, Location temp) { 7561 Location callee_method = GenerateCalleeMethodStaticOrDirectCall(invoke, temp); 7562 7563 switch (invoke->GetCodePtrLocation()) { 7564 case HInvokeStaticOrDirect::CodePtrLocation::kCallSelf: 7565 __ bl(GetFrameEntryLabel()); 7566 break; 7567 case HInvokeStaticOrDirect::CodePtrLocation::kCallArtMethod: 7568 // LR = callee_method->entry_point_from_quick_compiled_code_ 7569 __ LoadFromOffset( 7570 kLoadWord, LR, callee_method.AsRegister<Register>(), 7571 ArtMethod::EntryPointFromQuickCompiledCodeOffset(kArmPointerSize).Int32Value()); 7572 // LR() 7573 __ blx(LR); 7574 break; 7575 } 7576 7577 DCHECK(!IsLeafMethod()); 7578} 7579 7580void CodeGeneratorARM::GenerateVirtualCall(HInvokeVirtual* invoke, Location temp_location) { 7581 Register temp = temp_location.AsRegister<Register>(); 7582 uint32_t method_offset = mirror::Class::EmbeddedVTableEntryOffset( 7583 invoke->GetVTableIndex(), kArmPointerSize).Uint32Value(); 7584 7585 // Use the calling convention instead of the location of the receiver, as 7586 // intrinsics may have put the receiver in a different register. In the intrinsics 7587 // slow path, the arguments have been moved to the right place, so here we are 7588 // guaranteed that the receiver is the first register of the calling convention. 7589 InvokeDexCallingConvention calling_convention; 7590 Register receiver = calling_convention.GetRegisterAt(0); 7591 uint32_t class_offset = mirror::Object::ClassOffset().Int32Value(); 7592 // /* HeapReference<Class> */ temp = receiver->klass_ 7593 __ LoadFromOffset(kLoadWord, temp, receiver, class_offset); 7594 MaybeRecordImplicitNullCheck(invoke); 7595 // Instead of simply (possibly) unpoisoning `temp` here, we should 7596 // emit a read barrier for the previous class reference load. 7597 // However this is not required in practice, as this is an 7598 // intermediate/temporary reference and because the current 7599 // concurrent copying collector keeps the from-space memory 7600 // intact/accessible until the end of the marking phase (the 7601 // concurrent copying collector may not in the future). 7602 __ MaybeUnpoisonHeapReference(temp); 7603 // temp = temp->GetMethodAt(method_offset); 7604 uint32_t entry_point = ArtMethod::EntryPointFromQuickCompiledCodeOffset( 7605 kArmPointerSize).Int32Value(); 7606 __ LoadFromOffset(kLoadWord, temp, temp, method_offset); 7607 // LR = temp->GetEntryPoint(); 7608 __ LoadFromOffset(kLoadWord, LR, temp, entry_point); 7609 // LR(); 7610 __ blx(LR); 7611} 7612 7613CodeGeneratorARM::PcRelativePatchInfo* CodeGeneratorARM::NewPcRelativeStringPatch( 7614 const DexFile& dex_file, dex::StringIndex string_index) { 7615 return NewPcRelativePatch(dex_file, string_index.index_, &pc_relative_string_patches_); 7616} 7617 7618CodeGeneratorARM::PcRelativePatchInfo* CodeGeneratorARM::NewPcRelativeTypePatch( 7619 const DexFile& dex_file, dex::TypeIndex type_index) { 7620 return NewPcRelativePatch(dex_file, type_index.index_, &pc_relative_type_patches_); 7621} 7622 7623CodeGeneratorARM::PcRelativePatchInfo* CodeGeneratorARM::NewTypeBssEntryPatch( 7624 const DexFile& dex_file, dex::TypeIndex type_index) { 7625 return NewPcRelativePatch(dex_file, type_index.index_, &type_bss_entry_patches_); 7626} 7627 7628CodeGeneratorARM::PcRelativePatchInfo* CodeGeneratorARM::NewPcRelativeDexCacheArrayPatch( 7629 const DexFile& dex_file, uint32_t element_offset) { 7630 return NewPcRelativePatch(dex_file, element_offset, &pc_relative_dex_cache_patches_); 7631} 7632 7633CodeGeneratorARM::PcRelativePatchInfo* CodeGeneratorARM::NewPcRelativePatch( 7634 const DexFile& dex_file, uint32_t offset_or_index, ArenaDeque<PcRelativePatchInfo>* patches) { 7635 patches->emplace_back(dex_file, offset_or_index); 7636 return &patches->back(); 7637} 7638 7639Literal* CodeGeneratorARM::DeduplicateBootImageStringLiteral(const DexFile& dex_file, 7640 dex::StringIndex string_index) { 7641 return boot_image_string_patches_.GetOrCreate( 7642 StringReference(&dex_file, string_index), 7643 [this]() { return __ NewLiteral<uint32_t>(/* placeholder */ 0u); }); 7644} 7645 7646Literal* CodeGeneratorARM::DeduplicateBootImageTypeLiteral(const DexFile& dex_file, 7647 dex::TypeIndex type_index) { 7648 return boot_image_type_patches_.GetOrCreate( 7649 TypeReference(&dex_file, type_index), 7650 [this]() { return __ NewLiteral<uint32_t>(/* placeholder */ 0u); }); 7651} 7652 7653Literal* CodeGeneratorARM::DeduplicateBootImageAddressLiteral(uint32_t address) { 7654 bool needs_patch = GetCompilerOptions().GetIncludePatchInformation(); 7655 Uint32ToLiteralMap* map = needs_patch ? &boot_image_address_patches_ : &uint32_literals_; 7656 return DeduplicateUint32Literal(dchecked_integral_cast<uint32_t>(address), map); 7657} 7658 7659Literal* CodeGeneratorARM::DeduplicateJitStringLiteral(const DexFile& dex_file, 7660 dex::StringIndex string_index, 7661 Handle<mirror::String> handle) { 7662 jit_string_roots_.Overwrite(StringReference(&dex_file, string_index), 7663 reinterpret_cast64<uint64_t>(handle.GetReference())); 7664 return jit_string_patches_.GetOrCreate( 7665 StringReference(&dex_file, string_index), 7666 [this]() { return __ NewLiteral<uint32_t>(/* placeholder */ 0u); }); 7667} 7668 7669Literal* CodeGeneratorARM::DeduplicateJitClassLiteral(const DexFile& dex_file, 7670 dex::TypeIndex type_index, 7671 Handle<mirror::Class> handle) { 7672 jit_class_roots_.Overwrite(TypeReference(&dex_file, type_index), 7673 reinterpret_cast64<uint64_t>(handle.GetReference())); 7674 return jit_class_patches_.GetOrCreate( 7675 TypeReference(&dex_file, type_index), 7676 [this]() { return __ NewLiteral<uint32_t>(/* placeholder */ 0u); }); 7677} 7678 7679template <LinkerPatch (*Factory)(size_t, const DexFile*, uint32_t, uint32_t)> 7680inline void CodeGeneratorARM::EmitPcRelativeLinkerPatches( 7681 const ArenaDeque<PcRelativePatchInfo>& infos, 7682 ArenaVector<LinkerPatch>* linker_patches) { 7683 for (const PcRelativePatchInfo& info : infos) { 7684 const DexFile& dex_file = info.target_dex_file; 7685 size_t offset_or_index = info.offset_or_index; 7686 DCHECK(info.add_pc_label.IsBound()); 7687 uint32_t add_pc_offset = dchecked_integral_cast<uint32_t>(info.add_pc_label.Position()); 7688 // Add MOVW patch. 7689 DCHECK(info.movw_label.IsBound()); 7690 uint32_t movw_offset = dchecked_integral_cast<uint32_t>(info.movw_label.Position()); 7691 linker_patches->push_back(Factory(movw_offset, &dex_file, add_pc_offset, offset_or_index)); 7692 // Add MOVT patch. 7693 DCHECK(info.movt_label.IsBound()); 7694 uint32_t movt_offset = dchecked_integral_cast<uint32_t>(info.movt_label.Position()); 7695 linker_patches->push_back(Factory(movt_offset, &dex_file, add_pc_offset, offset_or_index)); 7696 } 7697} 7698 7699void CodeGeneratorARM::EmitLinkerPatches(ArenaVector<LinkerPatch>* linker_patches) { 7700 DCHECK(linker_patches->empty()); 7701 size_t size = 7702 /* MOVW+MOVT for each entry */ 2u * pc_relative_dex_cache_patches_.size() + 7703 boot_image_string_patches_.size() + 7704 /* MOVW+MOVT for each entry */ 2u * pc_relative_string_patches_.size() + 7705 boot_image_type_patches_.size() + 7706 /* MOVW+MOVT for each entry */ 2u * pc_relative_type_patches_.size() + 7707 /* MOVW+MOVT for each entry */ 2u * type_bss_entry_patches_.size() + 7708 boot_image_address_patches_.size(); 7709 linker_patches->reserve(size); 7710 EmitPcRelativeLinkerPatches<LinkerPatch::DexCacheArrayPatch>(pc_relative_dex_cache_patches_, 7711 linker_patches); 7712 for (const auto& entry : boot_image_string_patches_) { 7713 const StringReference& target_string = entry.first; 7714 Literal* literal = entry.second; 7715 DCHECK(literal->GetLabel()->IsBound()); 7716 uint32_t literal_offset = literal->GetLabel()->Position(); 7717 linker_patches->push_back(LinkerPatch::StringPatch(literal_offset, 7718 target_string.dex_file, 7719 target_string.string_index.index_)); 7720 } 7721 if (!GetCompilerOptions().IsBootImage()) { 7722 DCHECK(pc_relative_type_patches_.empty()); 7723 EmitPcRelativeLinkerPatches<LinkerPatch::StringBssEntryPatch>(pc_relative_string_patches_, 7724 linker_patches); 7725 } else { 7726 EmitPcRelativeLinkerPatches<LinkerPatch::RelativeTypePatch>(pc_relative_type_patches_, 7727 linker_patches); 7728 EmitPcRelativeLinkerPatches<LinkerPatch::RelativeStringPatch>(pc_relative_string_patches_, 7729 linker_patches); 7730 } 7731 EmitPcRelativeLinkerPatches<LinkerPatch::TypeBssEntryPatch>(type_bss_entry_patches_, 7732 linker_patches); 7733 for (const auto& entry : boot_image_type_patches_) { 7734 const TypeReference& target_type = entry.first; 7735 Literal* literal = entry.second; 7736 DCHECK(literal->GetLabel()->IsBound()); 7737 uint32_t literal_offset = literal->GetLabel()->Position(); 7738 linker_patches->push_back(LinkerPatch::TypePatch(literal_offset, 7739 target_type.dex_file, 7740 target_type.type_index.index_)); 7741 } 7742 for (const auto& entry : boot_image_address_patches_) { 7743 DCHECK(GetCompilerOptions().GetIncludePatchInformation()); 7744 Literal* literal = entry.second; 7745 DCHECK(literal->GetLabel()->IsBound()); 7746 uint32_t literal_offset = literal->GetLabel()->Position(); 7747 linker_patches->push_back(LinkerPatch::RecordPosition(literal_offset)); 7748 } 7749 DCHECK_EQ(size, linker_patches->size()); 7750} 7751 7752Literal* CodeGeneratorARM::DeduplicateUint32Literal(uint32_t value, Uint32ToLiteralMap* map) { 7753 return map->GetOrCreate( 7754 value, 7755 [this, value]() { return __ NewLiteral<uint32_t>(value); }); 7756} 7757 7758Literal* CodeGeneratorARM::DeduplicateMethodLiteral(MethodReference target_method, 7759 MethodToLiteralMap* map) { 7760 return map->GetOrCreate( 7761 target_method, 7762 [this]() { return __ NewLiteral<uint32_t>(/* placeholder */ 0u); }); 7763} 7764 7765void LocationsBuilderARM::VisitMultiplyAccumulate(HMultiplyAccumulate* instr) { 7766 LocationSummary* locations = 7767 new (GetGraph()->GetArena()) LocationSummary(instr, LocationSummary::kNoCall); 7768 locations->SetInAt(HMultiplyAccumulate::kInputAccumulatorIndex, 7769 Location::RequiresRegister()); 7770 locations->SetInAt(HMultiplyAccumulate::kInputMulLeftIndex, Location::RequiresRegister()); 7771 locations->SetInAt(HMultiplyAccumulate::kInputMulRightIndex, Location::RequiresRegister()); 7772 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 7773} 7774 7775void InstructionCodeGeneratorARM::VisitMultiplyAccumulate(HMultiplyAccumulate* instr) { 7776 LocationSummary* locations = instr->GetLocations(); 7777 Register res = locations->Out().AsRegister<Register>(); 7778 Register accumulator = 7779 locations->InAt(HMultiplyAccumulate::kInputAccumulatorIndex).AsRegister<Register>(); 7780 Register mul_left = 7781 locations->InAt(HMultiplyAccumulate::kInputMulLeftIndex).AsRegister<Register>(); 7782 Register mul_right = 7783 locations->InAt(HMultiplyAccumulate::kInputMulRightIndex).AsRegister<Register>(); 7784 7785 if (instr->GetOpKind() == HInstruction::kAdd) { 7786 __ mla(res, mul_left, mul_right, accumulator); 7787 } else { 7788 __ mls(res, mul_left, mul_right, accumulator); 7789 } 7790} 7791 7792void LocationsBuilderARM::VisitBoundType(HBoundType* instruction ATTRIBUTE_UNUSED) { 7793 // Nothing to do, this should be removed during prepare for register allocator. 7794 LOG(FATAL) << "Unreachable"; 7795} 7796 7797void InstructionCodeGeneratorARM::VisitBoundType(HBoundType* instruction ATTRIBUTE_UNUSED) { 7798 // Nothing to do, this should be removed during prepare for register allocator. 7799 LOG(FATAL) << "Unreachable"; 7800} 7801 7802// Simple implementation of packed switch - generate cascaded compare/jumps. 7803void LocationsBuilderARM::VisitPackedSwitch(HPackedSwitch* switch_instr) { 7804 LocationSummary* locations = 7805 new (GetGraph()->GetArena()) LocationSummary(switch_instr, LocationSummary::kNoCall); 7806 locations->SetInAt(0, Location::RequiresRegister()); 7807 if (switch_instr->GetNumEntries() > kPackedSwitchCompareJumpThreshold && 7808 codegen_->GetAssembler()->IsThumb()) { 7809 locations->AddTemp(Location::RequiresRegister()); // We need a temp for the table base. 7810 if (switch_instr->GetStartValue() != 0) { 7811 locations->AddTemp(Location::RequiresRegister()); // We need a temp for the bias. 7812 } 7813 } 7814} 7815 7816void InstructionCodeGeneratorARM::VisitPackedSwitch(HPackedSwitch* switch_instr) { 7817 int32_t lower_bound = switch_instr->GetStartValue(); 7818 uint32_t num_entries = switch_instr->GetNumEntries(); 7819 LocationSummary* locations = switch_instr->GetLocations(); 7820 Register value_reg = locations->InAt(0).AsRegister<Register>(); 7821 HBasicBlock* default_block = switch_instr->GetDefaultBlock(); 7822 7823 if (num_entries <= kPackedSwitchCompareJumpThreshold || !codegen_->GetAssembler()->IsThumb()) { 7824 // Create a series of compare/jumps. 7825 Register temp_reg = IP; 7826 // Note: It is fine for the below AddConstantSetFlags() using IP register to temporarily store 7827 // the immediate, because IP is used as the destination register. For the other 7828 // AddConstantSetFlags() and GenerateCompareWithImmediate(), the immediate values are constant, 7829 // and they can be encoded in the instruction without making use of IP register. 7830 __ AddConstantSetFlags(temp_reg, value_reg, -lower_bound); 7831 7832 const ArenaVector<HBasicBlock*>& successors = switch_instr->GetBlock()->GetSuccessors(); 7833 // Jump to successors[0] if value == lower_bound. 7834 __ b(codegen_->GetLabelOf(successors[0]), EQ); 7835 int32_t last_index = 0; 7836 for (; num_entries - last_index > 2; last_index += 2) { 7837 __ AddConstantSetFlags(temp_reg, temp_reg, -2); 7838 // Jump to successors[last_index + 1] if value < case_value[last_index + 2]. 7839 __ b(codegen_->GetLabelOf(successors[last_index + 1]), LO); 7840 // Jump to successors[last_index + 2] if value == case_value[last_index + 2]. 7841 __ b(codegen_->GetLabelOf(successors[last_index + 2]), EQ); 7842 } 7843 if (num_entries - last_index == 2) { 7844 // The last missing case_value. 7845 __ CmpConstant(temp_reg, 1); 7846 __ b(codegen_->GetLabelOf(successors[last_index + 1]), EQ); 7847 } 7848 7849 // And the default for any other value. 7850 if (!codegen_->GoesToNextBlock(switch_instr->GetBlock(), default_block)) { 7851 __ b(codegen_->GetLabelOf(default_block)); 7852 } 7853 } else { 7854 // Create a table lookup. 7855 Register temp_reg = locations->GetTemp(0).AsRegister<Register>(); 7856 7857 // Materialize a pointer to the switch table 7858 std::vector<Label*> labels(num_entries); 7859 const ArenaVector<HBasicBlock*>& successors = switch_instr->GetBlock()->GetSuccessors(); 7860 for (uint32_t i = 0; i < num_entries; i++) { 7861 labels[i] = codegen_->GetLabelOf(successors[i]); 7862 } 7863 JumpTable* table = __ CreateJumpTable(std::move(labels), temp_reg); 7864 7865 // Remove the bias. 7866 Register key_reg; 7867 if (lower_bound != 0) { 7868 key_reg = locations->GetTemp(1).AsRegister<Register>(); 7869 __ AddConstant(key_reg, value_reg, -lower_bound); 7870 } else { 7871 key_reg = value_reg; 7872 } 7873 7874 // Check whether the value is in the table, jump to default block if not. 7875 __ CmpConstant(key_reg, num_entries - 1); 7876 __ b(codegen_->GetLabelOf(default_block), Condition::HI); 7877 7878 // Load the displacement from the table. 7879 __ ldr(temp_reg, Address(temp_reg, key_reg, Shift::LSL, 2)); 7880 7881 // Dispatch is a direct add to the PC (for Thumb2). 7882 __ EmitJumpTableDispatch(table, temp_reg); 7883 } 7884} 7885 7886void LocationsBuilderARM::VisitArmDexCacheArraysBase(HArmDexCacheArraysBase* base) { 7887 LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(base); 7888 locations->SetOut(Location::RequiresRegister()); 7889} 7890 7891void InstructionCodeGeneratorARM::VisitArmDexCacheArraysBase(HArmDexCacheArraysBase* base) { 7892 Register base_reg = base->GetLocations()->Out().AsRegister<Register>(); 7893 CodeGeneratorARM::PcRelativePatchInfo* labels = 7894 codegen_->NewPcRelativeDexCacheArrayPatch(base->GetDexFile(), base->GetElementOffset()); 7895 __ BindTrackedLabel(&labels->movw_label); 7896 __ movw(base_reg, /* placeholder */ 0u); 7897 __ BindTrackedLabel(&labels->movt_label); 7898 __ movt(base_reg, /* placeholder */ 0u); 7899 __ BindTrackedLabel(&labels->add_pc_label); 7900 __ add(base_reg, base_reg, ShifterOperand(PC)); 7901} 7902 7903void CodeGeneratorARM::MoveFromReturnRegister(Location trg, Primitive::Type type) { 7904 if (!trg.IsValid()) { 7905 DCHECK_EQ(type, Primitive::kPrimVoid); 7906 return; 7907 } 7908 7909 DCHECK_NE(type, Primitive::kPrimVoid); 7910 7911 Location return_loc = InvokeDexCallingConventionVisitorARM().GetReturnLocation(type); 7912 if (return_loc.Equals(trg)) { 7913 return; 7914 } 7915 7916 // TODO: Consider pairs in the parallel move resolver, then this could be nicely merged 7917 // with the last branch. 7918 if (type == Primitive::kPrimLong) { 7919 HParallelMove parallel_move(GetGraph()->GetArena()); 7920 parallel_move.AddMove(return_loc.ToLow(), trg.ToLow(), Primitive::kPrimInt, nullptr); 7921 parallel_move.AddMove(return_loc.ToHigh(), trg.ToHigh(), Primitive::kPrimInt, nullptr); 7922 GetMoveResolver()->EmitNativeCode(¶llel_move); 7923 } else if (type == Primitive::kPrimDouble) { 7924 HParallelMove parallel_move(GetGraph()->GetArena()); 7925 parallel_move.AddMove(return_loc.ToLow(), trg.ToLow(), Primitive::kPrimFloat, nullptr); 7926 parallel_move.AddMove(return_loc.ToHigh(), trg.ToHigh(), Primitive::kPrimFloat, nullptr); 7927 GetMoveResolver()->EmitNativeCode(¶llel_move); 7928 } else { 7929 // Let the parallel move resolver take care of all of this. 7930 HParallelMove parallel_move(GetGraph()->GetArena()); 7931 parallel_move.AddMove(return_loc, trg, type, nullptr); 7932 GetMoveResolver()->EmitNativeCode(¶llel_move); 7933 } 7934} 7935 7936void LocationsBuilderARM::VisitClassTableGet(HClassTableGet* instruction) { 7937 LocationSummary* locations = 7938 new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall); 7939 locations->SetInAt(0, Location::RequiresRegister()); 7940 locations->SetOut(Location::RequiresRegister()); 7941} 7942 7943void InstructionCodeGeneratorARM::VisitClassTableGet(HClassTableGet* instruction) { 7944 LocationSummary* locations = instruction->GetLocations(); 7945 if (instruction->GetTableKind() == HClassTableGet::TableKind::kVTable) { 7946 uint32_t method_offset = mirror::Class::EmbeddedVTableEntryOffset( 7947 instruction->GetIndex(), kArmPointerSize).SizeValue(); 7948 __ LoadFromOffset(kLoadWord, 7949 locations->Out().AsRegister<Register>(), 7950 locations->InAt(0).AsRegister<Register>(), 7951 method_offset); 7952 } else { 7953 uint32_t method_offset = static_cast<uint32_t>(ImTable::OffsetOfElement( 7954 instruction->GetIndex(), kArmPointerSize)); 7955 __ LoadFromOffset(kLoadWord, 7956 locations->Out().AsRegister<Register>(), 7957 locations->InAt(0).AsRegister<Register>(), 7958 mirror::Class::ImtPtrOffset(kArmPointerSize).Uint32Value()); 7959 __ LoadFromOffset(kLoadWord, 7960 locations->Out().AsRegister<Register>(), 7961 locations->Out().AsRegister<Register>(), 7962 method_offset); 7963 } 7964} 7965 7966static void PatchJitRootUse(uint8_t* code, 7967 const uint8_t* roots_data, 7968 Literal* literal, 7969 uint64_t index_in_table) { 7970 DCHECK(literal->GetLabel()->IsBound()); 7971 uint32_t literal_offset = literal->GetLabel()->Position(); 7972 uintptr_t address = 7973 reinterpret_cast<uintptr_t>(roots_data) + index_in_table * sizeof(GcRoot<mirror::Object>); 7974 uint8_t* data = code + literal_offset; 7975 reinterpret_cast<uint32_t*>(data)[0] = dchecked_integral_cast<uint32_t>(address); 7976} 7977 7978void CodeGeneratorARM::EmitJitRootPatches(uint8_t* code, const uint8_t* roots_data) { 7979 for (const auto& entry : jit_string_patches_) { 7980 const auto& it = jit_string_roots_.find(entry.first); 7981 DCHECK(it != jit_string_roots_.end()); 7982 PatchJitRootUse(code, roots_data, entry.second, it->second); 7983 } 7984 for (const auto& entry : jit_class_patches_) { 7985 const auto& it = jit_class_roots_.find(entry.first); 7986 DCHECK(it != jit_class_roots_.end()); 7987 PatchJitRootUse(code, roots_data, entry.second, it->second); 7988 } 7989} 7990 7991#undef __ 7992#undef QUICK_ENTRY_POINT 7993 7994} // namespace arm 7995} // namespace art 7996