code_generator_arm64.cc revision 6dc01748c61a7ad41d4ab701d3e27897bd39a899 
1/*
2 * Copyright (C) 2014 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *      http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17#include "code_generator_arm64.h"
18
19#include "arch/arm64/instruction_set_features_arm64.h"
20#include "art_method.h"
21#include "code_generator_utils.h"
22#include "compiled_method.h"
23#include "entrypoints/quick/quick_entrypoints.h"
24#include "entrypoints/quick/quick_entrypoints_enum.h"
25#include "gc/accounting/card_table.h"
26#include "intrinsics.h"
27#include "intrinsics_arm64.h"
28#include "mirror/array-inl.h"
29#include "mirror/class-inl.h"
30#include "offsets.h"
31#include "thread.h"
32#include "utils/arm64/assembler_arm64.h"
33#include "utils/assembler.h"
34#include "utils/stack_checks.h"
35
36
37using namespace vixl;   // NOLINT(build/namespaces)
38
39#ifdef __
40#error "ARM64 Codegen VIXL macro-assembler macro already defined."
41#endif
42
43namespace art {
44
45namespace arm64 {
46
47using helpers::CPURegisterFrom;
48using helpers::DRegisterFrom;
49using helpers::FPRegisterFrom;
50using helpers::HeapOperand;
51using helpers::HeapOperandFrom;
52using helpers::InputCPURegisterAt;
53using helpers::InputFPRegisterAt;
54using helpers::InputRegisterAt;
55using helpers::InputOperandAt;
56using helpers::Int64ConstantFrom;
57using helpers::LocationFrom;
58using helpers::OperandFromMemOperand;
59using helpers::OutputCPURegister;
60using helpers::OutputFPRegister;
61using helpers::OutputRegister;
62using helpers::RegisterFrom;
63using helpers::StackOperandFrom;
64using helpers::VIXLRegCodeFromART;
65using helpers::WRegisterFrom;
66using helpers::XRegisterFrom;
67using helpers::ARM64EncodableConstantOrRegister;
68using helpers::ArtVixlRegCodeCoherentForRegSet;
69
70static constexpr int kCurrentMethodStackOffset = 0;
71
72inline Condition ARM64Condition(IfCondition cond) {
73  switch (cond) {
74    case kCondEQ: return eq;
75    case kCondNE: return ne;
76    case kCondLT: return lt;
77    case kCondLE: return le;
78    case kCondGT: return gt;
79    case kCondGE: return ge;
80    case kCondB:  return lo;
81    case kCondBE: return ls;
82    case kCondA:  return hi;
83    case kCondAE: return hs;
84  }
85  LOG(FATAL) << "Unreachable";
86  UNREACHABLE();
87}
88
89Location ARM64ReturnLocation(Primitive::Type return_type) {
90  // Note that in practice, `LocationFrom(x0)` and `LocationFrom(w0)` create the
91  // same Location object, and so do `LocationFrom(d0)` and `LocationFrom(s0)`,
92  // but we use the exact registers for clarity.
93  if (return_type == Primitive::kPrimFloat) {
94    return LocationFrom(s0);
95  } else if (return_type == Primitive::kPrimDouble) {
96    return LocationFrom(d0);
97  } else if (return_type == Primitive::kPrimLong) {
98    return LocationFrom(x0);
99  } else if (return_type == Primitive::kPrimVoid) {
100    return Location::NoLocation();
101  } else {
102    return LocationFrom(w0);
103  }
104}
105
106Location InvokeRuntimeCallingConvention::GetReturnLocation(Primitive::Type return_type) {
107  return ARM64ReturnLocation(return_type);
108}
109
110#define __ down_cast<CodeGeneratorARM64*>(codegen)->GetVIXLAssembler()->
111#define QUICK_ENTRY_POINT(x) QUICK_ENTRYPOINT_OFFSET(kArm64WordSize, x).Int32Value()
112
113// Calculate memory accessing operand for save/restore live registers.
114static void SaveRestoreLiveRegistersHelper(CodeGenerator* codegen,
115                                           RegisterSet* register_set,
116                                           int64_t spill_offset,
117                                           bool is_save) {
118  DCHECK(ArtVixlRegCodeCoherentForRegSet(register_set->GetCoreRegisters(),
119                                         codegen->GetNumberOfCoreRegisters(),
120                                         register_set->GetFloatingPointRegisters(),
121                                         codegen->GetNumberOfFloatingPointRegisters()));
122
123  CPURegList core_list = CPURegList(CPURegister::kRegister, kXRegSize,
124      register_set->GetCoreRegisters() & (~callee_saved_core_registers.list()));
125  CPURegList fp_list = CPURegList(CPURegister::kFPRegister, kDRegSize,
126      register_set->GetFloatingPointRegisters() & (~callee_saved_fp_registers.list()));
127
128  MacroAssembler* masm = down_cast<CodeGeneratorARM64*>(codegen)->GetVIXLAssembler();
129  UseScratchRegisterScope temps(masm);
130
131  Register base = masm->StackPointer();
132  int64_t core_spill_size = core_list.TotalSizeInBytes();
133  int64_t fp_spill_size = fp_list.TotalSizeInBytes();
134  int64_t reg_size = kXRegSizeInBytes;
135  int64_t max_ls_pair_offset = spill_offset + core_spill_size + fp_spill_size - 2 * reg_size;
136  uint32_t ls_access_size = WhichPowerOf2(reg_size);
137  if (((core_list.Count() > 1) || (fp_list.Count() > 1)) &&
138      !masm->IsImmLSPair(max_ls_pair_offset, ls_access_size)) {
139    // If the offset does not fit in the instruction's immediate field, use an alternate register
140    // to compute the base address(float point registers spill base address).
141    Register new_base = temps.AcquireSameSizeAs(base);
142    __ Add(new_base, base, Operand(spill_offset + core_spill_size));
143    base = new_base;
144    spill_offset = -core_spill_size;
145    int64_t new_max_ls_pair_offset = fp_spill_size - 2 * reg_size;
146    DCHECK(masm->IsImmLSPair(spill_offset, ls_access_size));
147    DCHECK(masm->IsImmLSPair(new_max_ls_pair_offset, ls_access_size));
148  }
149
150  if (is_save) {
151    __ StoreCPURegList(core_list, MemOperand(base, spill_offset));
152    __ StoreCPURegList(fp_list, MemOperand(base, spill_offset + core_spill_size));
153  } else {
154    __ LoadCPURegList(core_list, MemOperand(base, spill_offset));
155    __ LoadCPURegList(fp_list, MemOperand(base, spill_offset + core_spill_size));
156  }
157}
158
159void SlowPathCodeARM64::SaveLiveRegisters(CodeGenerator* codegen, LocationSummary* locations) {
160  RegisterSet* register_set = locations->GetLiveRegisters();
161  size_t stack_offset = codegen->GetFirstRegisterSlotInSlowPath();
162  for (size_t i = 0, e = codegen->GetNumberOfCoreRegisters(); i < e; ++i) {
163    if (!codegen->IsCoreCalleeSaveRegister(i) && register_set->ContainsCoreRegister(i)) {
164      // If the register holds an object, update the stack mask.
165      if (locations->RegisterContainsObject(i)) {
166        locations->SetStackBit(stack_offset / kVRegSize);
167      }
168      DCHECK_LT(stack_offset, codegen->GetFrameSize() - codegen->FrameEntrySpillSize());
169      DCHECK_LT(i, kMaximumNumberOfExpectedRegisters);
170      saved_core_stack_offsets_[i] = stack_offset;
171      stack_offset += kXRegSizeInBytes;
172    }
173  }
174
175  for (size_t i = 0, e = codegen->GetNumberOfFloatingPointRegisters(); i < e; ++i) {
176    if (!codegen->IsFloatingPointCalleeSaveRegister(i) &&
177        register_set->ContainsFloatingPointRegister(i)) {
178      DCHECK_LT(stack_offset, codegen->GetFrameSize() - codegen->FrameEntrySpillSize());
179      DCHECK_LT(i, kMaximumNumberOfExpectedRegisters);
180      saved_fpu_stack_offsets_[i] = stack_offset;
181      stack_offset += kDRegSizeInBytes;
182    }
183  }
184
185  SaveRestoreLiveRegistersHelper(codegen, register_set,
186                                 codegen->GetFirstRegisterSlotInSlowPath(), true /* is_save */);
187}
188
189void SlowPathCodeARM64::RestoreLiveRegisters(CodeGenerator* codegen, LocationSummary* locations) {
190  RegisterSet* register_set = locations->GetLiveRegisters();
191  SaveRestoreLiveRegistersHelper(codegen, register_set,
192                                 codegen->GetFirstRegisterSlotInSlowPath(), false /* is_save */);
193}
194
195class BoundsCheckSlowPathARM64 : public SlowPathCodeARM64 {
196 public:
197  explicit BoundsCheckSlowPathARM64(HBoundsCheck* instruction) : instruction_(instruction) {}
198
199  void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
200    LocationSummary* locations = instruction_->GetLocations();
201    CodeGeneratorARM64* arm64_codegen = down_cast<CodeGeneratorARM64*>(codegen);
202
203    __ Bind(GetEntryLabel());
204    if (instruction_->CanThrowIntoCatchBlock()) {
205      // Live registers will be restored in the catch block if caught.
206      SaveLiveRegisters(codegen, instruction_->GetLocations());
207    }
208    // We're moving two locations to locations that could overlap, so we need a parallel
209    // move resolver.
210    InvokeRuntimeCallingConvention calling_convention;
211    codegen->EmitParallelMoves(
212        locations->InAt(0), LocationFrom(calling_convention.GetRegisterAt(0)), Primitive::kPrimInt,
213        locations->InAt(1), LocationFrom(calling_convention.GetRegisterAt(1)), Primitive::kPrimInt);
214    arm64_codegen->InvokeRuntime(
215        QUICK_ENTRY_POINT(pThrowArrayBounds), instruction_, instruction_->GetDexPc(), this);
216    CheckEntrypointTypes<kQuickThrowArrayBounds, void, int32_t, int32_t>();
217  }
218
219  bool IsFatal() const OVERRIDE { return true; }
220
221  const char* GetDescription() const OVERRIDE { return "BoundsCheckSlowPathARM64"; }
222
223 private:
224  HBoundsCheck* const instruction_;
225
226  DISALLOW_COPY_AND_ASSIGN(BoundsCheckSlowPathARM64);
227};
228
229class DivZeroCheckSlowPathARM64 : public SlowPathCodeARM64 {
230 public:
231  explicit DivZeroCheckSlowPathARM64(HDivZeroCheck* instruction) : instruction_(instruction) {}
232
233  void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
234    CodeGeneratorARM64* arm64_codegen = down_cast<CodeGeneratorARM64*>(codegen);
235    __ Bind(GetEntryLabel());
236    if (instruction_->CanThrowIntoCatchBlock()) {
237      // Live registers will be restored in the catch block if caught.
238      SaveLiveRegisters(codegen, instruction_->GetLocations());
239    }
240    arm64_codegen->InvokeRuntime(
241        QUICK_ENTRY_POINT(pThrowDivZero), instruction_, instruction_->GetDexPc(), this);
242    CheckEntrypointTypes<kQuickThrowDivZero, void, void>();
243  }
244
245  bool IsFatal() const OVERRIDE { return true; }
246
247  const char* GetDescription() const OVERRIDE { return "DivZeroCheckSlowPathARM64"; }
248
249 private:
250  HDivZeroCheck* const instruction_;
251  DISALLOW_COPY_AND_ASSIGN(DivZeroCheckSlowPathARM64);
252};
253
254class LoadClassSlowPathARM64 : public SlowPathCodeARM64 {
255 public:
256  LoadClassSlowPathARM64(HLoadClass* cls,
257                         HInstruction* at,
258                         uint32_t dex_pc,
259                         bool do_clinit)
260      : cls_(cls), at_(at), dex_pc_(dex_pc), do_clinit_(do_clinit) {
261    DCHECK(at->IsLoadClass() || at->IsClinitCheck());
262  }
263
264  void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
265    LocationSummary* locations = at_->GetLocations();
266    CodeGeneratorARM64* arm64_codegen = down_cast<CodeGeneratorARM64*>(codegen);
267
268    __ Bind(GetEntryLabel());
269    SaveLiveRegisters(codegen, locations);
270
271    InvokeRuntimeCallingConvention calling_convention;
272    __ Mov(calling_convention.GetRegisterAt(0).W(), cls_->GetTypeIndex());
273    int32_t entry_point_offset = do_clinit_ ? QUICK_ENTRY_POINT(pInitializeStaticStorage)
274                                            : QUICK_ENTRY_POINT(pInitializeType);
275    arm64_codegen->InvokeRuntime(entry_point_offset, at_, dex_pc_, this);
276    if (do_clinit_) {
277      CheckEntrypointTypes<kQuickInitializeStaticStorage, void*, uint32_t>();
278    } else {
279      CheckEntrypointTypes<kQuickInitializeType, void*, uint32_t>();
280    }
281
282    // Move the class to the desired location.
283    Location out = locations->Out();
284    if (out.IsValid()) {
285      DCHECK(out.IsRegister() && !locations->GetLiveRegisters()->ContainsCoreRegister(out.reg()));
286      Primitive::Type type = at_->GetType();
287      arm64_codegen->MoveLocation(out, calling_convention.GetReturnLocation(type), type);
288    }
289
290    RestoreLiveRegisters(codegen, locations);
291    __ B(GetExitLabel());
292  }
293
294  const char* GetDescription() const OVERRIDE { return "LoadClassSlowPathARM64"; }
295
296 private:
297  // The class this slow path will load.
298  HLoadClass* const cls_;
299
300  // The instruction where this slow path is happening.
301  // (Might be the load class or an initialization check).
302  HInstruction* const at_;
303
304  // The dex PC of `at_`.
305  const uint32_t dex_pc_;
306
307  // Whether to initialize the class.
308  const bool do_clinit_;
309
310  DISALLOW_COPY_AND_ASSIGN(LoadClassSlowPathARM64);
311};
312
313class LoadStringSlowPathARM64 : public SlowPathCodeARM64 {
314 public:
315  explicit LoadStringSlowPathARM64(HLoadString* instruction) : instruction_(instruction) {}
316
317  void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
318    LocationSummary* locations = instruction_->GetLocations();
319    DCHECK(!locations->GetLiveRegisters()->ContainsCoreRegister(locations->Out().reg()));
320    CodeGeneratorARM64* arm64_codegen = down_cast<CodeGeneratorARM64*>(codegen);
321
322    __ Bind(GetEntryLabel());
323    SaveLiveRegisters(codegen, locations);
324
325    InvokeRuntimeCallingConvention calling_convention;
326    __ Mov(calling_convention.GetRegisterAt(0).W(), instruction_->GetStringIndex());
327    arm64_codegen->InvokeRuntime(
328        QUICK_ENTRY_POINT(pResolveString), instruction_, instruction_->GetDexPc(), this);
329    CheckEntrypointTypes<kQuickResolveString, void*, uint32_t>();
330    Primitive::Type type = instruction_->GetType();
331    arm64_codegen->MoveLocation(locations->Out(), calling_convention.GetReturnLocation(type), type);
332
333    RestoreLiveRegisters(codegen, locations);
334    __ B(GetExitLabel());
335  }
336
337  const char* GetDescription() const OVERRIDE { return "LoadStringSlowPathARM64"; }
338
339 private:
340  HLoadString* const instruction_;
341
342  DISALLOW_COPY_AND_ASSIGN(LoadStringSlowPathARM64);
343};
344
345class NullCheckSlowPathARM64 : public SlowPathCodeARM64 {
346 public:
347  explicit NullCheckSlowPathARM64(HNullCheck* instr) : instruction_(instr) {}
348
349  void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
350    CodeGeneratorARM64* arm64_codegen = down_cast<CodeGeneratorARM64*>(codegen);
351    __ Bind(GetEntryLabel());
352    if (instruction_->CanThrowIntoCatchBlock()) {
353      // Live registers will be restored in the catch block if caught.
354      SaveLiveRegisters(codegen, instruction_->GetLocations());
355    }
356    arm64_codegen->InvokeRuntime(
357        QUICK_ENTRY_POINT(pThrowNullPointer), instruction_, instruction_->GetDexPc(), this);
358    CheckEntrypointTypes<kQuickThrowNullPointer, void, void>();
359  }
360
361  bool IsFatal() const OVERRIDE { return true; }
362
363  const char* GetDescription() const OVERRIDE { return "NullCheckSlowPathARM64"; }
364
365 private:
366  HNullCheck* const instruction_;
367
368  DISALLOW_COPY_AND_ASSIGN(NullCheckSlowPathARM64);
369};
370
371class SuspendCheckSlowPathARM64 : public SlowPathCodeARM64 {
372 public:
373  SuspendCheckSlowPathARM64(HSuspendCheck* instruction, HBasicBlock* successor)
374      : instruction_(instruction), successor_(successor) {}
375
376  void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
377    CodeGeneratorARM64* arm64_codegen = down_cast<CodeGeneratorARM64*>(codegen);
378    __ Bind(GetEntryLabel());
379    SaveLiveRegisters(codegen, instruction_->GetLocations());
380    arm64_codegen->InvokeRuntime(
381        QUICK_ENTRY_POINT(pTestSuspend), instruction_, instruction_->GetDexPc(), this);
382    CheckEntrypointTypes<kQuickTestSuspend, void, void>();
383    RestoreLiveRegisters(codegen, instruction_->GetLocations());
384    if (successor_ == nullptr) {
385      __ B(GetReturnLabel());
386    } else {
387      __ B(arm64_codegen->GetLabelOf(successor_));
388    }
389  }
390
391  vixl::Label* GetReturnLabel() {
392    DCHECK(successor_ == nullptr);
393    return &return_label_;
394  }
395
396  HBasicBlock* GetSuccessor() const {
397    return successor_;
398  }
399
400  const char* GetDescription() const OVERRIDE { return "SuspendCheckSlowPathARM64"; }
401
402 private:
403  HSuspendCheck* const instruction_;
404  // If not null, the block to branch to after the suspend check.
405  HBasicBlock* const successor_;
406
407  // If `successor_` is null, the label to branch to after the suspend check.
408  vixl::Label return_label_;
409
410  DISALLOW_COPY_AND_ASSIGN(SuspendCheckSlowPathARM64);
411};
412
413class TypeCheckSlowPathARM64 : public SlowPathCodeARM64 {
414 public:
415  TypeCheckSlowPathARM64(HInstruction* instruction, bool is_fatal)
416      : instruction_(instruction), is_fatal_(is_fatal) {}
417
418  void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
419    LocationSummary* locations = instruction_->GetLocations();
420    Location class_to_check = locations->InAt(1);
421    Location object_class = instruction_->IsCheckCast() ? locations->GetTemp(0)
422                                                        : locations->Out();
423    DCHECK(instruction_->IsCheckCast()
424           || !locations->GetLiveRegisters()->ContainsCoreRegister(locations->Out().reg()));
425    CodeGeneratorARM64* arm64_codegen = down_cast<CodeGeneratorARM64*>(codegen);
426    uint32_t dex_pc = instruction_->GetDexPc();
427
428    __ Bind(GetEntryLabel());
429
430    if (instruction_->IsCheckCast()) {
431      // The codegen for the instruction overwrites `temp`, so put it back in place.
432      Register obj = InputRegisterAt(instruction_, 0);
433      Register temp = WRegisterFrom(locations->GetTemp(0));
434      uint32_t class_offset = mirror::Object::ClassOffset().Int32Value();
435      __ Ldr(temp, HeapOperand(obj, class_offset));
436      arm64_codegen->GetAssembler()->MaybeUnpoisonHeapReference(temp);
437    }
438
439    if (!is_fatal_) {
440      SaveLiveRegisters(codegen, locations);
441    }
442
443    // We're moving two locations to locations that could overlap, so we need a parallel
444    // move resolver.
445    InvokeRuntimeCallingConvention calling_convention;
446    codegen->EmitParallelMoves(
447        class_to_check, LocationFrom(calling_convention.GetRegisterAt(0)), Primitive::kPrimNot,
448        object_class, LocationFrom(calling_convention.GetRegisterAt(1)), Primitive::kPrimNot);
449
450    if (instruction_->IsInstanceOf()) {
451      arm64_codegen->InvokeRuntime(
452          QUICK_ENTRY_POINT(pInstanceofNonTrivial), instruction_, dex_pc, this);
453      Primitive::Type ret_type = instruction_->GetType();
454      Location ret_loc = calling_convention.GetReturnLocation(ret_type);
455      arm64_codegen->MoveLocation(locations->Out(), ret_loc, ret_type);
456      CheckEntrypointTypes<kQuickInstanceofNonTrivial, uint32_t,
457                           const mirror::Class*, const mirror::Class*>();
458    } else {
459      DCHECK(instruction_->IsCheckCast());
460      arm64_codegen->InvokeRuntime(QUICK_ENTRY_POINT(pCheckCast), instruction_, dex_pc, this);
461      CheckEntrypointTypes<kQuickCheckCast, void, const mirror::Class*, const mirror::Class*>();
462    }
463
464    if (!is_fatal_) {
465      RestoreLiveRegisters(codegen, locations);
466      __ B(GetExitLabel());
467    }
468  }
469
470  const char* GetDescription() const OVERRIDE { return "TypeCheckSlowPathARM64"; }
471  bool IsFatal() const { return is_fatal_; }
472
473 private:
474  HInstruction* const instruction_;
475  const bool is_fatal_;
476
477  DISALLOW_COPY_AND_ASSIGN(TypeCheckSlowPathARM64);
478};
479
480class DeoptimizationSlowPathARM64 : public SlowPathCodeARM64 {
481 public:
482  explicit DeoptimizationSlowPathARM64(HInstruction* instruction)
483      : instruction_(instruction) {}
484
485  void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
486    __ Bind(GetEntryLabel());
487    SaveLiveRegisters(codegen, instruction_->GetLocations());
488    DCHECK(instruction_->IsDeoptimize());
489    HDeoptimize* deoptimize = instruction_->AsDeoptimize();
490    uint32_t dex_pc = deoptimize->GetDexPc();
491    CodeGeneratorARM64* arm64_codegen = down_cast<CodeGeneratorARM64*>(codegen);
492    arm64_codegen->InvokeRuntime(QUICK_ENTRY_POINT(pDeoptimize), instruction_, dex_pc, this);
493  }
494
495  const char* GetDescription() const OVERRIDE { return "DeoptimizationSlowPathARM64"; }
496
497 private:
498  HInstruction* const instruction_;
499  DISALLOW_COPY_AND_ASSIGN(DeoptimizationSlowPathARM64);
500};
501
502class ArraySetSlowPathARM64 : public SlowPathCodeARM64 {
503 public:
504  explicit ArraySetSlowPathARM64(HInstruction* instruction) : instruction_(instruction) {}
505
506  void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
507    LocationSummary* locations = instruction_->GetLocations();
508    __ Bind(GetEntryLabel());
509    SaveLiveRegisters(codegen, locations);
510
511    InvokeRuntimeCallingConvention calling_convention;
512    HParallelMove parallel_move(codegen->GetGraph()->GetArena());
513    parallel_move.AddMove(
514        locations->InAt(0),
515        LocationFrom(calling_convention.GetRegisterAt(0)),
516        Primitive::kPrimNot,
517        nullptr);
518    parallel_move.AddMove(
519        locations->InAt(1),
520        LocationFrom(calling_convention.GetRegisterAt(1)),
521        Primitive::kPrimInt,
522        nullptr);
523    parallel_move.AddMove(
524        locations->InAt(2),
525        LocationFrom(calling_convention.GetRegisterAt(2)),
526        Primitive::kPrimNot,
527        nullptr);
528    codegen->GetMoveResolver()->EmitNativeCode(&parallel_move);
529
530    CodeGeneratorARM64* arm64_codegen = down_cast<CodeGeneratorARM64*>(codegen);
531    arm64_codegen->InvokeRuntime(QUICK_ENTRY_POINT(pAputObject),
532                                 instruction_,
533                                 instruction_->GetDexPc(),
534                                 this);
535    CheckEntrypointTypes<kQuickAputObject, void, mirror::Array*, int32_t, mirror::Object*>();
536    RestoreLiveRegisters(codegen, locations);
537    __ B(GetExitLabel());
538  }
539
540  const char* GetDescription() const OVERRIDE { return "ArraySetSlowPathARM64"; }
541
542 private:
543  HInstruction* const instruction_;
544
545  DISALLOW_COPY_AND_ASSIGN(ArraySetSlowPathARM64);
546};
547
548#undef __
549
550Location InvokeDexCallingConventionVisitorARM64::GetNextLocation(Primitive::Type type) {
551  Location next_location;
552  if (type == Primitive::kPrimVoid) {
553    LOG(FATAL) << "Unreachable type " << type;
554  }
555
556  if (Primitive::IsFloatingPointType(type) &&
557      (float_index_ < calling_convention.GetNumberOfFpuRegisters())) {
558    next_location = LocationFrom(calling_convention.GetFpuRegisterAt(float_index_++));
559  } else if (!Primitive::IsFloatingPointType(type) &&
560             (gp_index_ < calling_convention.GetNumberOfRegisters())) {
561    next_location = LocationFrom(calling_convention.GetRegisterAt(gp_index_++));
562  } else {
563    size_t stack_offset = calling_convention.GetStackOffsetOf(stack_index_);
564    next_location = Primitive::Is64BitType(type) ? Location::DoubleStackSlot(stack_offset)
565                                                 : Location::StackSlot(stack_offset);
566  }
567
568  // Space on the stack is reserved for all arguments.
569  stack_index_ += Primitive::Is64BitType(type) ? 2 : 1;
570  return next_location;
571}
572
573Location InvokeDexCallingConventionVisitorARM64::GetMethodLocation() const {
574  return LocationFrom(kArtMethodRegister);
575}
576
577CodeGeneratorARM64::CodeGeneratorARM64(HGraph* graph,
578                                       const Arm64InstructionSetFeatures& isa_features,
579                                       const CompilerOptions& compiler_options,
580                                       OptimizingCompilerStats* stats)
581    : CodeGenerator(graph,
582                    kNumberOfAllocatableRegisters,
583                    kNumberOfAllocatableFPRegisters,
584                    kNumberOfAllocatableRegisterPairs,
585                    callee_saved_core_registers.list(),
586                    callee_saved_fp_registers.list(),
587                    compiler_options,
588                    stats),
589      block_labels_(nullptr),
590      location_builder_(graph, this),
591      instruction_visitor_(graph, this),
592      move_resolver_(graph->GetArena(), this),
593      isa_features_(isa_features),
594      uint64_literals_(std::less<uint64_t>(),
595                       graph->GetArena()->Adapter(kArenaAllocCodeGenerator)),
596      method_patches_(MethodReferenceComparator(),
597                      graph->GetArena()->Adapter(kArenaAllocCodeGenerator)),
598      call_patches_(MethodReferenceComparator(),
599                    graph->GetArena()->Adapter(kArenaAllocCodeGenerator)),
600      relative_call_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)),
601      pc_relative_dex_cache_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)) {
602  // Save the link register (containing the return address) to mimic Quick.
603  AddAllocatedRegister(LocationFrom(lr));
604}
605
606#undef __
607#define __ GetVIXLAssembler()->
608
609void CodeGeneratorARM64::Finalize(CodeAllocator* allocator) {
610  // Ensure we emit the literal pool.
611  __ FinalizeCode();
612
613  CodeGenerator::Finalize(allocator);
614}
615
616void ParallelMoveResolverARM64::PrepareForEmitNativeCode() {
617  // Note: There are 6 kinds of moves:
618  // 1. constant -> GPR/FPR (non-cycle)
619  // 2. constant -> stack (non-cycle)
620  // 3. GPR/FPR -> GPR/FPR
621  // 4. GPR/FPR -> stack
622  // 5. stack -> GPR/FPR
623  // 6. stack -> stack (non-cycle)
624  // Case 1, 2 and 6 should never be included in a dependency cycle on ARM64. For case 3, 4, and 5
625  // VIXL uses at most 1 GPR. VIXL has 2 GPR and 1 FPR temps, and there should be no intersecting
626  // cycles on ARM64, so we always have 1 GPR and 1 FPR available VIXL temps to resolve the
627  // dependency.
628  vixl_temps_.Open(GetVIXLAssembler());
629}
630
631void ParallelMoveResolverARM64::FinishEmitNativeCode() {
632  vixl_temps_.Close();
633}
634
635Location ParallelMoveResolverARM64::AllocateScratchLocationFor(Location::Kind kind) {
636  DCHECK(kind == Location::kRegister || kind == Location::kFpuRegister ||
637         kind == Location::kStackSlot || kind == Location::kDoubleStackSlot);
638  kind = (kind == Location::kFpuRegister) ? Location::kFpuRegister : Location::kRegister;
639  Location scratch = GetScratchLocation(kind);
640  if (!scratch.Equals(Location::NoLocation())) {
641    return scratch;
642  }
643  // Allocate from VIXL temp registers.
644  if (kind == Location::kRegister) {
645    scratch = LocationFrom(vixl_temps_.AcquireX());
646  } else {
647    DCHECK(kind == Location::kFpuRegister);
648    scratch = LocationFrom(vixl_temps_.AcquireD());
649  }
650  AddScratchLocation(scratch);
651  return scratch;
652}
653
654void ParallelMoveResolverARM64::FreeScratchLocation(Location loc) {
655  if (loc.IsRegister()) {
656    vixl_temps_.Release(XRegisterFrom(loc));
657  } else {
658    DCHECK(loc.IsFpuRegister());
659    vixl_temps_.Release(DRegisterFrom(loc));
660  }
661  RemoveScratchLocation(loc);
662}
663
664void ParallelMoveResolverARM64::EmitMove(size_t index) {
665  MoveOperands* move = moves_[index];
666  codegen_->MoveLocation(move->GetDestination(), move->GetSource(), Primitive::kPrimVoid);
667}
668
669void CodeGeneratorARM64::GenerateFrameEntry() {
670  MacroAssembler* masm = GetVIXLAssembler();
671  BlockPoolsScope block_pools(masm);
672  __ Bind(&frame_entry_label_);
673
674  bool do_overflow_check = FrameNeedsStackCheck(GetFrameSize(), kArm64) || !IsLeafMethod();
675  if (do_overflow_check) {
676    UseScratchRegisterScope temps(masm);
677    Register temp = temps.AcquireX();
678    DCHECK(GetCompilerOptions().GetImplicitStackOverflowChecks());
679    __ Sub(temp, sp, static_cast<int32_t>(GetStackOverflowReservedBytes(kArm64)));
680    __ Ldr(wzr, MemOperand(temp, 0));
681    RecordPcInfo(nullptr, 0);
682  }
683
684  if (!HasEmptyFrame()) {
685    int frame_size = GetFrameSize();
686    // Stack layout:
687    //      sp[frame_size - 8]        : lr.
688    //      ...                       : other preserved core registers.
689    //      ...                       : other preserved fp registers.
690    //      ...                       : reserved frame space.
691    //      sp[0]                     : current method.
692    __ Str(kArtMethodRegister, MemOperand(sp, -frame_size, PreIndex));
693    GetAssembler()->cfi().AdjustCFAOffset(frame_size);
694    GetAssembler()->SpillRegisters(GetFramePreservedCoreRegisters(),
695        frame_size - GetCoreSpillSize());
696    GetAssembler()->SpillRegisters(GetFramePreservedFPRegisters(),
697        frame_size - FrameEntrySpillSize());
698  }
699}
700
701void CodeGeneratorARM64::GenerateFrameExit() {
702  BlockPoolsScope block_pools(GetVIXLAssembler());
703  GetAssembler()->cfi().RememberState();
704  if (!HasEmptyFrame()) {
705    int frame_size = GetFrameSize();
706    GetAssembler()->UnspillRegisters(GetFramePreservedFPRegisters(),
707        frame_size - FrameEntrySpillSize());
708    GetAssembler()->UnspillRegisters(GetFramePreservedCoreRegisters(),
709        frame_size - GetCoreSpillSize());
710    __ Drop(frame_size);
711    GetAssembler()->cfi().AdjustCFAOffset(-frame_size);
712  }
713  __ Ret();
714  GetAssembler()->cfi().RestoreState();
715  GetAssembler()->cfi().DefCFAOffset(GetFrameSize());
716}
717
718vixl::CPURegList CodeGeneratorARM64::GetFramePreservedCoreRegisters() const {
719  DCHECK(ArtVixlRegCodeCoherentForRegSet(core_spill_mask_, GetNumberOfCoreRegisters(), 0, 0));
720  return vixl::CPURegList(vixl::CPURegister::kRegister, vixl::kXRegSize,
721                          core_spill_mask_);
722}
723
724vixl::CPURegList CodeGeneratorARM64::GetFramePreservedFPRegisters() const {
725  DCHECK(ArtVixlRegCodeCoherentForRegSet(0, 0, fpu_spill_mask_,
726                                         GetNumberOfFloatingPointRegisters()));
727  return vixl::CPURegList(vixl::CPURegister::kFPRegister, vixl::kDRegSize,
728                          fpu_spill_mask_);
729}
730
731void CodeGeneratorARM64::Bind(HBasicBlock* block) {
732  __ Bind(GetLabelOf(block));
733}
734
735void CodeGeneratorARM64::Move(HInstruction* instruction,
736                              Location location,
737                              HInstruction* move_for) {
738  LocationSummary* locations = instruction->GetLocations();
739  Primitive::Type type = instruction->GetType();
740  DCHECK_NE(type, Primitive::kPrimVoid);
741
742  if (instruction->IsFakeString()) {
743    // The fake string is an alias for null.
744    DCHECK(IsBaseline());
745    instruction = locations->Out().GetConstant();
746    DCHECK(instruction->IsNullConstant()) << instruction->DebugName();
747  }
748
749  if (instruction->IsCurrentMethod()) {
750    MoveLocation(location,
751                 Location::DoubleStackSlot(kCurrentMethodStackOffset),
752                 Primitive::kPrimVoid);
753  } else if (locations != nullptr && locations->Out().Equals(location)) {
754    return;
755  } else if (instruction->IsIntConstant()
756             || instruction->IsLongConstant()
757             || instruction->IsNullConstant()) {
758    int64_t value = GetInt64ValueOf(instruction->AsConstant());
759    if (location.IsRegister()) {
760      Register dst = RegisterFrom(location, type);
761      DCHECK(((instruction->IsIntConstant() || instruction->IsNullConstant()) && dst.Is32Bits()) ||
762             (instruction->IsLongConstant() && dst.Is64Bits()));
763      __ Mov(dst, value);
764    } else {
765      DCHECK(location.IsStackSlot() || location.IsDoubleStackSlot());
766      UseScratchRegisterScope temps(GetVIXLAssembler());
767      Register temp = (instruction->IsIntConstant() || instruction->IsNullConstant())
768          ? temps.AcquireW()
769          : temps.AcquireX();
770      __ Mov(temp, value);
771      __ Str(temp, StackOperandFrom(location));
772    }
773  } else if (instruction->IsTemporary()) {
774    Location temp_location = GetTemporaryLocation(instruction->AsTemporary());
775    MoveLocation(location, temp_location, type);
776  } else if (instruction->IsLoadLocal()) {
777    uint32_t stack_slot = GetStackSlot(instruction->AsLoadLocal()->GetLocal());
778    if (Primitive::Is64BitType(type)) {
779      MoveLocation(location, Location::DoubleStackSlot(stack_slot), type);
780    } else {
781      MoveLocation(location, Location::StackSlot(stack_slot), type);
782    }
783
784  } else {
785    DCHECK((instruction->GetNext() == move_for) || instruction->GetNext()->IsTemporary());
786    MoveLocation(location, locations->Out(), type);
787  }
788}
789
790void CodeGeneratorARM64::MoveConstant(Location location, int32_t value) {
791  DCHECK(location.IsRegister());
792  __ Mov(RegisterFrom(location, Primitive::kPrimInt), value);
793}
794
795void CodeGeneratorARM64::AddLocationAsTemp(Location location, LocationSummary* locations) {
796  if (location.IsRegister()) {
797    locations->AddTemp(location);
798  } else {
799    UNIMPLEMENTED(FATAL) << "AddLocationAsTemp not implemented for location " << location;
800  }
801}
802
803Location CodeGeneratorARM64::GetStackLocation(HLoadLocal* load) const {
804  Primitive::Type type = load->GetType();
805
806  switch (type) {
807    case Primitive::kPrimNot:
808    case Primitive::kPrimInt:
809    case Primitive::kPrimFloat:
810      return Location::StackSlot(GetStackSlot(load->GetLocal()));
811
812    case Primitive::kPrimLong:
813    case Primitive::kPrimDouble:
814      return Location::DoubleStackSlot(GetStackSlot(load->GetLocal()));
815
816    case Primitive::kPrimBoolean:
817    case Primitive::kPrimByte:
818    case Primitive::kPrimChar:
819    case Primitive::kPrimShort:
820    case Primitive::kPrimVoid:
821      LOG(FATAL) << "Unexpected type " << type;
822  }
823
824  LOG(FATAL) << "Unreachable";
825  return Location::NoLocation();
826}
827
828void CodeGeneratorARM64::MarkGCCard(Register object, Register value, bool value_can_be_null) {
829  UseScratchRegisterScope temps(GetVIXLAssembler());
830  Register card = temps.AcquireX();
831  Register temp = temps.AcquireW();   // Index within the CardTable - 32bit.
832  vixl::Label done;
833  if (value_can_be_null) {
834    __ Cbz(value, &done);
835  }
836  __ Ldr(card, MemOperand(tr, Thread::CardTableOffset<kArm64WordSize>().Int32Value()));
837  __ Lsr(temp, object, gc::accounting::CardTable::kCardShift);
838  __ Strb(card, MemOperand(card, temp.X()));
839  if (value_can_be_null) {
840    __ Bind(&done);
841  }
842}
843
844void CodeGeneratorARM64::SetupBlockedRegisters(bool is_baseline) const {
845  // Blocked core registers:
846  //      lr        : Runtime reserved.
847  //      tr        : Runtime reserved.
848  //      xSuspend  : Runtime reserved. TODO: Unblock this when the runtime stops using it.
849  //      ip1       : VIXL core temp.
850  //      ip0       : VIXL core temp.
851  //
852  // Blocked fp registers:
853  //      d31       : VIXL fp temp.
854  CPURegList reserved_core_registers = vixl_reserved_core_registers;
855  reserved_core_registers.Combine(runtime_reserved_core_registers);
856  while (!reserved_core_registers.IsEmpty()) {
857    blocked_core_registers_[reserved_core_registers.PopLowestIndex().code()] = true;
858  }
859
860  CPURegList reserved_fp_registers = vixl_reserved_fp_registers;
861  while (!reserved_fp_registers.IsEmpty()) {
862    blocked_fpu_registers_[reserved_fp_registers.PopLowestIndex().code()] = true;
863  }
864
865  if (is_baseline) {
866    CPURegList reserved_core_baseline_registers = callee_saved_core_registers;
867    while (!reserved_core_baseline_registers.IsEmpty()) {
868      blocked_core_registers_[reserved_core_baseline_registers.PopLowestIndex().code()] = true;
869    }
870  }
871
872  if (is_baseline || GetGraph()->IsDebuggable()) {
873    // Stubs do not save callee-save floating point registers. If the graph
874    // is debuggable, we need to deal with these registers differently. For
875    // now, just block them.
876    CPURegList reserved_fp_baseline_registers = callee_saved_fp_registers;
877    while (!reserved_fp_baseline_registers.IsEmpty()) {
878      blocked_fpu_registers_[reserved_fp_baseline_registers.PopLowestIndex().code()] = true;
879    }
880  }
881}
882
883Location CodeGeneratorARM64::AllocateFreeRegister(Primitive::Type type) const {
884  if (type == Primitive::kPrimVoid) {
885    LOG(FATAL) << "Unreachable type " << type;
886  }
887
888  if (Primitive::IsFloatingPointType(type)) {
889    ssize_t reg = FindFreeEntry(blocked_fpu_registers_, kNumberOfAllocatableFPRegisters);
890    DCHECK_NE(reg, -1);
891    return Location::FpuRegisterLocation(reg);
892  } else {
893    ssize_t reg = FindFreeEntry(blocked_core_registers_, kNumberOfAllocatableRegisters);
894    DCHECK_NE(reg, -1);
895    return Location::RegisterLocation(reg);
896  }
897}
898
899size_t CodeGeneratorARM64::SaveCoreRegister(size_t stack_index, uint32_t reg_id) {
900  Register reg = Register(VIXLRegCodeFromART(reg_id), kXRegSize);
901  __ Str(reg, MemOperand(sp, stack_index));
902  return kArm64WordSize;
903}
904
905size_t CodeGeneratorARM64::RestoreCoreRegister(size_t stack_index, uint32_t reg_id) {
906  Register reg = Register(VIXLRegCodeFromART(reg_id), kXRegSize);
907  __ Ldr(reg, MemOperand(sp, stack_index));
908  return kArm64WordSize;
909}
910
911size_t CodeGeneratorARM64::SaveFloatingPointRegister(size_t stack_index, uint32_t reg_id) {
912  FPRegister reg = FPRegister(reg_id, kDRegSize);
913  __ Str(reg, MemOperand(sp, stack_index));
914  return kArm64WordSize;
915}
916
917size_t CodeGeneratorARM64::RestoreFloatingPointRegister(size_t stack_index, uint32_t reg_id) {
918  FPRegister reg = FPRegister(reg_id, kDRegSize);
919  __ Ldr(reg, MemOperand(sp, stack_index));
920  return kArm64WordSize;
921}
922
923void CodeGeneratorARM64::DumpCoreRegister(std::ostream& stream, int reg) const {
924  stream << XRegister(reg);
925}
926
927void CodeGeneratorARM64::DumpFloatingPointRegister(std::ostream& stream, int reg) const {
928  stream << DRegister(reg);
929}
930
931void CodeGeneratorARM64::MoveConstant(CPURegister destination, HConstant* constant) {
932  if (constant->IsIntConstant()) {
933    __ Mov(Register(destination), constant->AsIntConstant()->GetValue());
934  } else if (constant->IsLongConstant()) {
935    __ Mov(Register(destination), constant->AsLongConstant()->GetValue());
936  } else if (constant->IsNullConstant()) {
937    __ Mov(Register(destination), 0);
938  } else if (constant->IsFloatConstant()) {
939    __ Fmov(FPRegister(destination), constant->AsFloatConstant()->GetValue());
940  } else {
941    DCHECK(constant->IsDoubleConstant());
942    __ Fmov(FPRegister(destination), constant->AsDoubleConstant()->GetValue());
943  }
944}
945
946
947static bool CoherentConstantAndType(Location constant, Primitive::Type type) {
948  DCHECK(constant.IsConstant());
949  HConstant* cst = constant.GetConstant();
950  return (cst->IsIntConstant() && type == Primitive::kPrimInt) ||
951         // Null is mapped to a core W register, which we associate with kPrimInt.
952         (cst->IsNullConstant() && type == Primitive::kPrimInt) ||
953         (cst->IsLongConstant() && type == Primitive::kPrimLong) ||
954         (cst->IsFloatConstant() && type == Primitive::kPrimFloat) ||
955         (cst->IsDoubleConstant() && type == Primitive::kPrimDouble);
956}
957
958void CodeGeneratorARM64::MoveLocation(Location destination,
959                                      Location source,
960                                      Primitive::Type dst_type) {
961  if (source.Equals(destination)) {
962    return;
963  }
964
965  // A valid move can always be inferred from the destination and source
966  // locations. When moving from and to a register, the argument type can be
967  // used to generate 32bit instead of 64bit moves. In debug mode we also
968  // checks the coherency of the locations and the type.
969  bool unspecified_type = (dst_type == Primitive::kPrimVoid);
970
971  if (destination.IsRegister() || destination.IsFpuRegister()) {
972    if (unspecified_type) {
973      HConstant* src_cst = source.IsConstant() ? source.GetConstant() : nullptr;
974      if (source.IsStackSlot() ||
975          (src_cst != nullptr && (src_cst->IsIntConstant()
976                                  || src_cst->IsFloatConstant()
977                                  || src_cst->IsNullConstant()))) {
978        // For stack slots and 32bit constants, a 64bit type is appropriate.
979        dst_type = destination.IsRegister() ? Primitive::kPrimInt : Primitive::kPrimFloat;
980      } else {
981        // If the source is a double stack slot or a 64bit constant, a 64bit
982        // type is appropriate. Else the source is a register, and since the
983        // type has not been specified, we chose a 64bit type to force a 64bit
984        // move.
985        dst_type = destination.IsRegister() ? Primitive::kPrimLong : Primitive::kPrimDouble;
986      }
987    }
988    DCHECK((destination.IsFpuRegister() && Primitive::IsFloatingPointType(dst_type)) ||
989           (destination.IsRegister() && !Primitive::IsFloatingPointType(dst_type)));
990    CPURegister dst = CPURegisterFrom(destination, dst_type);
991    if (source.IsStackSlot() || source.IsDoubleStackSlot()) {
992      DCHECK(dst.Is64Bits() == source.IsDoubleStackSlot());
993      __ Ldr(dst, StackOperandFrom(source));
994    } else if (source.IsConstant()) {
995      DCHECK(CoherentConstantAndType(source, dst_type));
996      MoveConstant(dst, source.GetConstant());
997    } else if (source.IsRegister()) {
998      if (destination.IsRegister()) {
999        __ Mov(Register(dst), RegisterFrom(source, dst_type));
1000      } else {
1001        DCHECK(destination.IsFpuRegister());
1002        Primitive::Type source_type = Primitive::Is64BitType(dst_type)
1003            ? Primitive::kPrimLong
1004            : Primitive::kPrimInt;
1005        __ Fmov(FPRegisterFrom(destination, dst_type), RegisterFrom(source, source_type));
1006      }
1007    } else {
1008      DCHECK(source.IsFpuRegister());
1009      if (destination.IsRegister()) {
1010        Primitive::Type source_type = Primitive::Is64BitType(dst_type)
1011            ? Primitive::kPrimDouble
1012            : Primitive::kPrimFloat;
1013        __ Fmov(RegisterFrom(destination, dst_type), FPRegisterFrom(source, source_type));
1014      } else {
1015        DCHECK(destination.IsFpuRegister());
1016        __ Fmov(FPRegister(dst), FPRegisterFrom(source, dst_type));
1017      }
1018    }
1019  } else {  // The destination is not a register. It must be a stack slot.
1020    DCHECK(destination.IsStackSlot() || destination.IsDoubleStackSlot());
1021    if (source.IsRegister() || source.IsFpuRegister()) {
1022      if (unspecified_type) {
1023        if (source.IsRegister()) {
1024          dst_type = destination.IsStackSlot() ? Primitive::kPrimInt : Primitive::kPrimLong;
1025        } else {
1026          dst_type = destination.IsStackSlot() ? Primitive::kPrimFloat : Primitive::kPrimDouble;
1027        }
1028      }
1029      DCHECK((destination.IsDoubleStackSlot() == Primitive::Is64BitType(dst_type)) &&
1030             (source.IsFpuRegister() == Primitive::IsFloatingPointType(dst_type)));
1031      __ Str(CPURegisterFrom(source, dst_type), StackOperandFrom(destination));
1032    } else if (source.IsConstant()) {
1033      DCHECK(unspecified_type || CoherentConstantAndType(source, dst_type))
1034          << source << " " << dst_type;
1035      UseScratchRegisterScope temps(GetVIXLAssembler());
1036      HConstant* src_cst = source.GetConstant();
1037      CPURegister temp;
1038      if (src_cst->IsIntConstant() || src_cst->IsNullConstant()) {
1039        temp = temps.AcquireW();
1040      } else if (src_cst->IsLongConstant()) {
1041        temp = temps.AcquireX();
1042      } else if (src_cst->IsFloatConstant()) {
1043        temp = temps.AcquireS();
1044      } else {
1045        DCHECK(src_cst->IsDoubleConstant());
1046        temp = temps.AcquireD();
1047      }
1048      MoveConstant(temp, src_cst);
1049      __ Str(temp, StackOperandFrom(destination));
1050    } else {
1051      DCHECK(source.IsStackSlot() || source.IsDoubleStackSlot());
1052      DCHECK(source.IsDoubleStackSlot() == destination.IsDoubleStackSlot());
1053      UseScratchRegisterScope temps(GetVIXLAssembler());
1054      // There is generally less pressure on FP registers.
1055      FPRegister temp = destination.IsDoubleStackSlot() ? temps.AcquireD() : temps.AcquireS();
1056      __ Ldr(temp, StackOperandFrom(source));
1057      __ Str(temp, StackOperandFrom(destination));
1058    }
1059  }
1060}
1061
1062void CodeGeneratorARM64::Load(Primitive::Type type,
1063                              CPURegister dst,
1064                              const MemOperand& src) {
1065  switch (type) {
1066    case Primitive::kPrimBoolean:
1067      __ Ldrb(Register(dst), src);
1068      break;
1069    case Primitive::kPrimByte:
1070      __ Ldrsb(Register(dst), src);
1071      break;
1072    case Primitive::kPrimShort:
1073      __ Ldrsh(Register(dst), src);
1074      break;
1075    case Primitive::kPrimChar:
1076      __ Ldrh(Register(dst), src);
1077      break;
1078    case Primitive::kPrimInt:
1079    case Primitive::kPrimNot:
1080    case Primitive::kPrimLong:
1081    case Primitive::kPrimFloat:
1082    case Primitive::kPrimDouble:
1083      DCHECK_EQ(dst.Is64Bits(), Primitive::Is64BitType(type));
1084      __ Ldr(dst, src);
1085      break;
1086    case Primitive::kPrimVoid:
1087      LOG(FATAL) << "Unreachable type " << type;
1088  }
1089}
1090
1091void CodeGeneratorARM64::LoadAcquire(HInstruction* instruction,
1092                                     CPURegister dst,
1093                                     const MemOperand& src) {
1094  MacroAssembler* masm = GetVIXLAssembler();
1095  BlockPoolsScope block_pools(masm);
1096  UseScratchRegisterScope temps(masm);
1097  Register temp_base = temps.AcquireX();
1098  Primitive::Type type = instruction->GetType();
1099
1100  DCHECK(!src.IsPreIndex());
1101  DCHECK(!src.IsPostIndex());
1102
1103  // TODO(vixl): Let the MacroAssembler handle MemOperand.
1104  __ Add(temp_base, src.base(), OperandFromMemOperand(src));
1105  MemOperand base = MemOperand(temp_base);
1106  switch (type) {
1107    case Primitive::kPrimBoolean:
1108      __ Ldarb(Register(dst), base);
1109      MaybeRecordImplicitNullCheck(instruction);
1110      break;
1111    case Primitive::kPrimByte:
1112      __ Ldarb(Register(dst), base);
1113      MaybeRecordImplicitNullCheck(instruction);
1114      __ Sbfx(Register(dst), Register(dst), 0, Primitive::ComponentSize(type) * kBitsPerByte);
1115      break;
1116    case Primitive::kPrimChar:
1117      __ Ldarh(Register(dst), base);
1118      MaybeRecordImplicitNullCheck(instruction);
1119      break;
1120    case Primitive::kPrimShort:
1121      __ Ldarh(Register(dst), base);
1122      MaybeRecordImplicitNullCheck(instruction);
1123      __ Sbfx(Register(dst), Register(dst), 0, Primitive::ComponentSize(type) * kBitsPerByte);
1124      break;
1125    case Primitive::kPrimInt:
1126    case Primitive::kPrimNot:
1127    case Primitive::kPrimLong:
1128      DCHECK_EQ(dst.Is64Bits(), Primitive::Is64BitType(type));
1129      __ Ldar(Register(dst), base);
1130      MaybeRecordImplicitNullCheck(instruction);
1131      break;
1132    case Primitive::kPrimFloat:
1133    case Primitive::kPrimDouble: {
1134      DCHECK(dst.IsFPRegister());
1135      DCHECK_EQ(dst.Is64Bits(), Primitive::Is64BitType(type));
1136
1137      Register temp = dst.Is64Bits() ? temps.AcquireX() : temps.AcquireW();
1138      __ Ldar(temp, base);
1139      MaybeRecordImplicitNullCheck(instruction);
1140      __ Fmov(FPRegister(dst), temp);
1141      break;
1142    }
1143    case Primitive::kPrimVoid:
1144      LOG(FATAL) << "Unreachable type " << type;
1145  }
1146}
1147
1148void CodeGeneratorARM64::Store(Primitive::Type type,
1149                               CPURegister src,
1150                               const MemOperand& dst) {
1151  switch (type) {
1152    case Primitive::kPrimBoolean:
1153    case Primitive::kPrimByte:
1154      __ Strb(Register(src), dst);
1155      break;
1156    case Primitive::kPrimChar:
1157    case Primitive::kPrimShort:
1158      __ Strh(Register(src), dst);
1159      break;
1160    case Primitive::kPrimInt:
1161    case Primitive::kPrimNot:
1162    case Primitive::kPrimLong:
1163    case Primitive::kPrimFloat:
1164    case Primitive::kPrimDouble:
1165      DCHECK_EQ(src.Is64Bits(), Primitive::Is64BitType(type));
1166      __ Str(src, dst);
1167      break;
1168    case Primitive::kPrimVoid:
1169      LOG(FATAL) << "Unreachable type " << type;
1170  }
1171}
1172
1173void CodeGeneratorARM64::StoreRelease(Primitive::Type type,
1174                                      CPURegister src,
1175                                      const MemOperand& dst) {
1176  UseScratchRegisterScope temps(GetVIXLAssembler());
1177  Register temp_base = temps.AcquireX();
1178
1179  DCHECK(!dst.IsPreIndex());
1180  DCHECK(!dst.IsPostIndex());
1181
1182  // TODO(vixl): Let the MacroAssembler handle this.
1183  Operand op = OperandFromMemOperand(dst);
1184  __ Add(temp_base, dst.base(), op);
1185  MemOperand base = MemOperand(temp_base);
1186  switch (type) {
1187    case Primitive::kPrimBoolean:
1188    case Primitive::kPrimByte:
1189      __ Stlrb(Register(src), base);
1190      break;
1191    case Primitive::kPrimChar:
1192    case Primitive::kPrimShort:
1193      __ Stlrh(Register(src), base);
1194      break;
1195    case Primitive::kPrimInt:
1196    case Primitive::kPrimNot:
1197    case Primitive::kPrimLong:
1198      DCHECK_EQ(src.Is64Bits(), Primitive::Is64BitType(type));
1199      __ Stlr(Register(src), base);
1200      break;
1201    case Primitive::kPrimFloat:
1202    case Primitive::kPrimDouble: {
1203      DCHECK(src.IsFPRegister());
1204      DCHECK_EQ(src.Is64Bits(), Primitive::Is64BitType(type));
1205
1206      Register temp = src.Is64Bits() ? temps.AcquireX() : temps.AcquireW();
1207      __ Fmov(temp, FPRegister(src));
1208      __ Stlr(temp, base);
1209      break;
1210    }
1211    case Primitive::kPrimVoid:
1212      LOG(FATAL) << "Unreachable type " << type;
1213  }
1214}
1215
1216void CodeGeneratorARM64::InvokeRuntime(QuickEntrypointEnum entrypoint,
1217                                       HInstruction* instruction,
1218                                       uint32_t dex_pc,
1219                                       SlowPathCode* slow_path) {
1220  InvokeRuntime(GetThreadOffset<kArm64WordSize>(entrypoint).Int32Value(),
1221                instruction,
1222                dex_pc,
1223                slow_path);
1224}
1225
1226void CodeGeneratorARM64::InvokeRuntime(int32_t entry_point_offset,
1227                                       HInstruction* instruction,
1228                                       uint32_t dex_pc,
1229                                       SlowPathCode* slow_path) {
1230  ValidateInvokeRuntime(instruction, slow_path);
1231  BlockPoolsScope block_pools(GetVIXLAssembler());
1232  __ Ldr(lr, MemOperand(tr, entry_point_offset));
1233  __ Blr(lr);
1234  RecordPcInfo(instruction, dex_pc, slow_path);
1235}
1236
1237void InstructionCodeGeneratorARM64::GenerateClassInitializationCheck(SlowPathCodeARM64* slow_path,
1238                                                                     vixl::Register class_reg) {
1239  UseScratchRegisterScope temps(GetVIXLAssembler());
1240  Register temp = temps.AcquireW();
1241  size_t status_offset = mirror::Class::StatusOffset().SizeValue();
1242  bool use_acquire_release = codegen_->GetInstructionSetFeatures().PreferAcquireRelease();
1243
1244  // Even if the initialized flag is set, we need to ensure consistent memory ordering.
1245  if (use_acquire_release) {
1246    // TODO(vixl): Let the MacroAssembler handle MemOperand.
1247    __ Add(temp, class_reg, status_offset);
1248    __ Ldar(temp, HeapOperand(temp));
1249    __ Cmp(temp, mirror::Class::kStatusInitialized);
1250    __ B(lt, slow_path->GetEntryLabel());
1251  } else {
1252    __ Ldr(temp, HeapOperand(class_reg, status_offset));
1253    __ Cmp(temp, mirror::Class::kStatusInitialized);
1254    __ B(lt, slow_path->GetEntryLabel());
1255    __ Dmb(InnerShareable, BarrierReads);
1256  }
1257  __ Bind(slow_path->GetExitLabel());
1258}
1259
1260void InstructionCodeGeneratorARM64::GenerateMemoryBarrier(MemBarrierKind kind) {
1261  BarrierType type = BarrierAll;
1262
1263  switch (kind) {
1264    case MemBarrierKind::kAnyAny:
1265    case MemBarrierKind::kAnyStore: {
1266      type = BarrierAll;
1267      break;
1268    }
1269    case MemBarrierKind::kLoadAny: {
1270      type = BarrierReads;
1271      break;
1272    }
1273    case MemBarrierKind::kStoreStore: {
1274      type = BarrierWrites;
1275      break;
1276    }
1277    default:
1278      LOG(FATAL) << "Unexpected memory barrier " << kind;
1279  }
1280  __ Dmb(InnerShareable, type);
1281}
1282
1283void InstructionCodeGeneratorARM64::GenerateSuspendCheck(HSuspendCheck* instruction,
1284                                                         HBasicBlock* successor) {
1285  SuspendCheckSlowPathARM64* slow_path =
1286      down_cast<SuspendCheckSlowPathARM64*>(instruction->GetSlowPath());
1287  if (slow_path == nullptr) {
1288    slow_path = new (GetGraph()->GetArena()) SuspendCheckSlowPathARM64(instruction, successor);
1289    instruction->SetSlowPath(slow_path);
1290    codegen_->AddSlowPath(slow_path);
1291    if (successor != nullptr) {
1292      DCHECK(successor->IsLoopHeader());
1293      codegen_->ClearSpillSlotsFromLoopPhisInStackMap(instruction);
1294    }
1295  } else {
1296    DCHECK_EQ(slow_path->GetSuccessor(), successor);
1297  }
1298
1299  UseScratchRegisterScope temps(codegen_->GetVIXLAssembler());
1300  Register temp = temps.AcquireW();
1301
1302  __ Ldrh(temp, MemOperand(tr, Thread::ThreadFlagsOffset<kArm64WordSize>().SizeValue()));
1303  if (successor == nullptr) {
1304    __ Cbnz(temp, slow_path->GetEntryLabel());
1305    __ Bind(slow_path->GetReturnLabel());
1306  } else {
1307    __ Cbz(temp, codegen_->GetLabelOf(successor));
1308    __ B(slow_path->GetEntryLabel());
1309    // slow_path will return to GetLabelOf(successor).
1310  }
1311}
1312
1313InstructionCodeGeneratorARM64::InstructionCodeGeneratorARM64(HGraph* graph,
1314                                                             CodeGeneratorARM64* codegen)
1315      : HGraphVisitor(graph),
1316        assembler_(codegen->GetAssembler()),
1317        codegen_(codegen) {}
1318
1319#define FOR_EACH_UNIMPLEMENTED_INSTRUCTION(M)              \
1320  /* No unimplemented IR. */
1321
1322#define UNIMPLEMENTED_INSTRUCTION_BREAK_CODE(name) name##UnimplementedInstructionBreakCode
1323
1324enum UnimplementedInstructionBreakCode {
1325  // Using a base helps identify when we hit such breakpoints.
1326  UnimplementedInstructionBreakCodeBaseCode = 0x900,
1327#define ENUM_UNIMPLEMENTED_INSTRUCTION(name) UNIMPLEMENTED_INSTRUCTION_BREAK_CODE(name),
1328  FOR_EACH_UNIMPLEMENTED_INSTRUCTION(ENUM_UNIMPLEMENTED_INSTRUCTION)
1329#undef ENUM_UNIMPLEMENTED_INSTRUCTION
1330};
1331
1332#define DEFINE_UNIMPLEMENTED_INSTRUCTION_VISITORS(name)                               \
1333  void InstructionCodeGeneratorARM64::Visit##name(H##name* instr ATTRIBUTE_UNUSED) {  \
1334    __ Brk(UNIMPLEMENTED_INSTRUCTION_BREAK_CODE(name));                               \
1335  }                                                                                   \
1336  void LocationsBuilderARM64::Visit##name(H##name* instr) {                           \
1337    LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instr); \
1338    locations->SetOut(Location::Any());                                               \
1339  }
1340  FOR_EACH_UNIMPLEMENTED_INSTRUCTION(DEFINE_UNIMPLEMENTED_INSTRUCTION_VISITORS)
1341#undef DEFINE_UNIMPLEMENTED_INSTRUCTION_VISITORS
1342
1343#undef UNIMPLEMENTED_INSTRUCTION_BREAK_CODE
1344#undef FOR_EACH_UNIMPLEMENTED_INSTRUCTION
1345
1346void LocationsBuilderARM64::HandleBinaryOp(HBinaryOperation* instr) {
1347  DCHECK_EQ(instr->InputCount(), 2U);
1348  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instr);
1349  Primitive::Type type = instr->GetResultType();
1350  switch (type) {
1351    case Primitive::kPrimInt:
1352    case Primitive::kPrimLong:
1353      locations->SetInAt(0, Location::RequiresRegister());
1354      locations->SetInAt(1, ARM64EncodableConstantOrRegister(instr->InputAt(1), instr));
1355      locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
1356      break;
1357
1358    case Primitive::kPrimFloat:
1359    case Primitive::kPrimDouble:
1360      locations->SetInAt(0, Location::RequiresFpuRegister());
1361      locations->SetInAt(1, Location::RequiresFpuRegister());
1362      locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap);
1363      break;
1364
1365    default:
1366      LOG(FATAL) << "Unexpected " << instr->DebugName() << " type " << type;
1367  }
1368}
1369
1370void LocationsBuilderARM64::HandleFieldGet(HInstruction* instruction) {
1371  LocationSummary* locations =
1372      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
1373  locations->SetInAt(0, Location::RequiresRegister());
1374  if (Primitive::IsFloatingPointType(instruction->GetType())) {
1375    locations->SetOut(Location::RequiresFpuRegister());
1376  } else {
1377    locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
1378  }
1379}
1380
1381void InstructionCodeGeneratorARM64::HandleFieldGet(HInstruction* instruction,
1382                                                   const FieldInfo& field_info) {
1383  DCHECK(instruction->IsInstanceFieldGet() || instruction->IsStaticFieldGet());
1384  Primitive::Type field_type = field_info.GetFieldType();
1385  BlockPoolsScope block_pools(GetVIXLAssembler());
1386
1387  MemOperand field = HeapOperand(InputRegisterAt(instruction, 0), field_info.GetFieldOffset());
1388  bool use_acquire_release = codegen_->GetInstructionSetFeatures().PreferAcquireRelease();
1389
1390  if (field_info.IsVolatile()) {
1391    if (use_acquire_release) {
1392      // NB: LoadAcquire will record the pc info if needed.
1393      codegen_->LoadAcquire(instruction, OutputCPURegister(instruction), field);
1394    } else {
1395      codegen_->Load(field_type, OutputCPURegister(instruction), field);
1396      codegen_->MaybeRecordImplicitNullCheck(instruction);
1397      // For IRIW sequential consistency kLoadAny is not sufficient.
1398      GenerateMemoryBarrier(MemBarrierKind::kAnyAny);
1399    }
1400  } else {
1401    codegen_->Load(field_type, OutputCPURegister(instruction), field);
1402    codegen_->MaybeRecordImplicitNullCheck(instruction);
1403  }
1404
1405  if (field_type == Primitive::kPrimNot) {
1406    GetAssembler()->MaybeUnpoisonHeapReference(OutputCPURegister(instruction).W());
1407  }
1408}
1409
1410void LocationsBuilderARM64::HandleFieldSet(HInstruction* instruction) {
1411  LocationSummary* locations =
1412      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
1413  locations->SetInAt(0, Location::RequiresRegister());
1414  if (Primitive::IsFloatingPointType(instruction->InputAt(1)->GetType())) {
1415    locations->SetInAt(1, Location::RequiresFpuRegister());
1416  } else {
1417    locations->SetInAt(1, Location::RequiresRegister());
1418  }
1419}
1420
1421void InstructionCodeGeneratorARM64::HandleFieldSet(HInstruction* instruction,
1422                                                   const FieldInfo& field_info,
1423                                                   bool value_can_be_null) {
1424  DCHECK(instruction->IsInstanceFieldSet() || instruction->IsStaticFieldSet());
1425  BlockPoolsScope block_pools(GetVIXLAssembler());
1426
1427  Register obj = InputRegisterAt(instruction, 0);
1428  CPURegister value = InputCPURegisterAt(instruction, 1);
1429  CPURegister source = value;
1430  Offset offset = field_info.GetFieldOffset();
1431  Primitive::Type field_type = field_info.GetFieldType();
1432  bool use_acquire_release = codegen_->GetInstructionSetFeatures().PreferAcquireRelease();
1433
1434  {
1435    // We use a block to end the scratch scope before the write barrier, thus
1436    // freeing the temporary registers so they can be used in `MarkGCCard`.
1437    UseScratchRegisterScope temps(GetVIXLAssembler());
1438
1439    if (kPoisonHeapReferences && field_type == Primitive::kPrimNot) {
1440      DCHECK(value.IsW());
1441      Register temp = temps.AcquireW();
1442      __ Mov(temp, value.W());
1443      GetAssembler()->PoisonHeapReference(temp.W());
1444      source = temp;
1445    }
1446
1447    if (field_info.IsVolatile()) {
1448      if (use_acquire_release) {
1449        codegen_->StoreRelease(field_type, source, HeapOperand(obj, offset));
1450        codegen_->MaybeRecordImplicitNullCheck(instruction);
1451      } else {
1452        GenerateMemoryBarrier(MemBarrierKind::kAnyStore);
1453        codegen_->Store(field_type, source, HeapOperand(obj, offset));
1454        codegen_->MaybeRecordImplicitNullCheck(instruction);
1455        GenerateMemoryBarrier(MemBarrierKind::kAnyAny);
1456      }
1457    } else {
1458      codegen_->Store(field_type, source, HeapOperand(obj, offset));
1459      codegen_->MaybeRecordImplicitNullCheck(instruction);
1460    }
1461  }
1462
1463  if (CodeGenerator::StoreNeedsWriteBarrier(field_type, instruction->InputAt(1))) {
1464    codegen_->MarkGCCard(obj, Register(value), value_can_be_null);
1465  }
1466}
1467
1468void InstructionCodeGeneratorARM64::HandleBinaryOp(HBinaryOperation* instr) {
1469  Primitive::Type type = instr->GetType();
1470
1471  switch (type) {
1472    case Primitive::kPrimInt:
1473    case Primitive::kPrimLong: {
1474      Register dst = OutputRegister(instr);
1475      Register lhs = InputRegisterAt(instr, 0);
1476      Operand rhs = InputOperandAt(instr, 1);
1477      if (instr->IsAdd()) {
1478        __ Add(dst, lhs, rhs);
1479      } else if (instr->IsAnd()) {
1480        __ And(dst, lhs, rhs);
1481      } else if (instr->IsOr()) {
1482        __ Orr(dst, lhs, rhs);
1483      } else if (instr->IsSub()) {
1484        __ Sub(dst, lhs, rhs);
1485      } else {
1486        DCHECK(instr->IsXor());
1487        __ Eor(dst, lhs, rhs);
1488      }
1489      break;
1490    }
1491    case Primitive::kPrimFloat:
1492    case Primitive::kPrimDouble: {
1493      FPRegister dst = OutputFPRegister(instr);
1494      FPRegister lhs = InputFPRegisterAt(instr, 0);
1495      FPRegister rhs = InputFPRegisterAt(instr, 1);
1496      if (instr->IsAdd()) {
1497        __ Fadd(dst, lhs, rhs);
1498      } else if (instr->IsSub()) {
1499        __ Fsub(dst, lhs, rhs);
1500      } else {
1501        LOG(FATAL) << "Unexpected floating-point binary operation";
1502      }
1503      break;
1504    }
1505    default:
1506      LOG(FATAL) << "Unexpected binary operation type " << type;
1507  }
1508}
1509
1510void LocationsBuilderARM64::HandleShift(HBinaryOperation* instr) {
1511  DCHECK(instr->IsShl() || instr->IsShr() || instr->IsUShr());
1512
1513  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instr);
1514  Primitive::Type type = instr->GetResultType();
1515  switch (type) {
1516    case Primitive::kPrimInt:
1517    case Primitive::kPrimLong: {
1518      locations->SetInAt(0, Location::RequiresRegister());
1519      locations->SetInAt(1, Location::RegisterOrConstant(instr->InputAt(1)));
1520      locations->SetOut(Location::RequiresRegister());
1521      break;
1522    }
1523    default:
1524      LOG(FATAL) << "Unexpected shift type " << type;
1525  }
1526}
1527
1528void InstructionCodeGeneratorARM64::HandleShift(HBinaryOperation* instr) {
1529  DCHECK(instr->IsShl() || instr->IsShr() || instr->IsUShr());
1530
1531  Primitive::Type type = instr->GetType();
1532  switch (type) {
1533    case Primitive::kPrimInt:
1534    case Primitive::kPrimLong: {
1535      Register dst = OutputRegister(instr);
1536      Register lhs = InputRegisterAt(instr, 0);
1537      Operand rhs = InputOperandAt(instr, 1);
1538      if (rhs.IsImmediate()) {
1539        uint32_t shift_value = (type == Primitive::kPrimInt)
1540          ? static_cast<uint32_t>(rhs.immediate() & kMaxIntShiftValue)
1541          : static_cast<uint32_t>(rhs.immediate() & kMaxLongShiftValue);
1542        if (instr->IsShl()) {
1543          __ Lsl(dst, lhs, shift_value);
1544        } else if (instr->IsShr()) {
1545          __ Asr(dst, lhs, shift_value);
1546        } else {
1547          __ Lsr(dst, lhs, shift_value);
1548        }
1549      } else {
1550        Register rhs_reg = dst.IsX() ? rhs.reg().X() : rhs.reg().W();
1551
1552        if (instr->IsShl()) {
1553          __ Lsl(dst, lhs, rhs_reg);
1554        } else if (instr->IsShr()) {
1555          __ Asr(dst, lhs, rhs_reg);
1556        } else {
1557          __ Lsr(dst, lhs, rhs_reg);
1558        }
1559      }
1560      break;
1561    }
1562    default:
1563      LOG(FATAL) << "Unexpected shift operation type " << type;
1564  }
1565}
1566
1567void LocationsBuilderARM64::VisitAdd(HAdd* instruction) {
1568  HandleBinaryOp(instruction);
1569}
1570
1571void InstructionCodeGeneratorARM64::VisitAdd(HAdd* instruction) {
1572  HandleBinaryOp(instruction);
1573}
1574
1575void LocationsBuilderARM64::VisitAnd(HAnd* instruction) {
1576  HandleBinaryOp(instruction);
1577}
1578
1579void InstructionCodeGeneratorARM64::VisitAnd(HAnd* instruction) {
1580  HandleBinaryOp(instruction);
1581}
1582
1583void LocationsBuilderARM64::VisitArm64IntermediateAddress(HArm64IntermediateAddress* instruction) {
1584  LocationSummary* locations =
1585      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
1586  locations->SetInAt(0, Location::RequiresRegister());
1587  locations->SetInAt(1, ARM64EncodableConstantOrRegister(instruction->GetOffset(), instruction));
1588  locations->SetOut(Location::RequiresRegister());
1589}
1590
1591void InstructionCodeGeneratorARM64::VisitArm64IntermediateAddress(
1592    HArm64IntermediateAddress* instruction) {
1593  __ Add(OutputRegister(instruction),
1594         InputRegisterAt(instruction, 0),
1595         Operand(InputOperandAt(instruction, 1)));
1596}
1597
1598void LocationsBuilderARM64::VisitArrayGet(HArrayGet* instruction) {
1599  LocationSummary* locations =
1600      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
1601  locations->SetInAt(0, Location::RequiresRegister());
1602  locations->SetInAt(1, Location::RegisterOrConstant(instruction->InputAt(1)));
1603  if (Primitive::IsFloatingPointType(instruction->GetType())) {
1604    locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap);
1605  } else {
1606    locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
1607  }
1608}
1609
1610void InstructionCodeGeneratorARM64::VisitArrayGet(HArrayGet* instruction) {
1611  Primitive::Type type = instruction->GetType();
1612  Register obj = InputRegisterAt(instruction, 0);
1613  Location index = instruction->GetLocations()->InAt(1);
1614  size_t offset = mirror::Array::DataOffset(Primitive::ComponentSize(type)).Uint32Value();
1615  MemOperand source = HeapOperand(obj);
1616  CPURegister dest = OutputCPURegister(instruction);
1617
1618  MacroAssembler* masm = GetVIXLAssembler();
1619  UseScratchRegisterScope temps(masm);
1620  // Block pools between `Load` and `MaybeRecordImplicitNullCheck`.
1621  BlockPoolsScope block_pools(masm);
1622
1623  if (index.IsConstant()) {
1624    offset += Int64ConstantFrom(index) << Primitive::ComponentSizeShift(type);
1625    source = HeapOperand(obj, offset);
1626  } else {
1627    Register temp = temps.AcquireSameSizeAs(obj);
1628    if (instruction->GetArray()->IsArm64IntermediateAddress()) {
1629      // We do not need to compute the intermediate address from the array: the
1630      // input instruction has done it already. See the comment in
1631      // `InstructionSimplifierArm64::TryExtractArrayAccessAddress()`.
1632      if (kIsDebugBuild) {
1633        HArm64IntermediateAddress* tmp = instruction->GetArray()->AsArm64IntermediateAddress();
1634        DCHECK(tmp->GetOffset()->AsIntConstant()->GetValueAsUint64() == offset);
1635      }
1636      temp = obj;
1637    } else {
1638      __ Add(temp, obj, offset);
1639    }
1640    source = HeapOperand(temp, XRegisterFrom(index), LSL, Primitive::ComponentSizeShift(type));
1641  }
1642
1643  codegen_->Load(type, dest, source);
1644  codegen_->MaybeRecordImplicitNullCheck(instruction);
1645
1646  if (instruction->GetType() == Primitive::kPrimNot) {
1647    GetAssembler()->MaybeUnpoisonHeapReference(dest.W());
1648  }
1649}
1650
1651void LocationsBuilderARM64::VisitArrayLength(HArrayLength* instruction) {
1652  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction);
1653  locations->SetInAt(0, Location::RequiresRegister());
1654  locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
1655}
1656
1657void InstructionCodeGeneratorARM64::VisitArrayLength(HArrayLength* instruction) {
1658  BlockPoolsScope block_pools(GetVIXLAssembler());
1659  __ Ldr(OutputRegister(instruction),
1660         HeapOperand(InputRegisterAt(instruction, 0), mirror::Array::LengthOffset()));
1661  codegen_->MaybeRecordImplicitNullCheck(instruction);
1662}
1663
1664void LocationsBuilderARM64::VisitArraySet(HArraySet* instruction) {
1665  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(
1666      instruction,
1667      instruction->NeedsTypeCheck() ? LocationSummary::kCallOnSlowPath : LocationSummary::kNoCall);
1668  locations->SetInAt(0, Location::RequiresRegister());
1669  locations->SetInAt(1, Location::RegisterOrConstant(instruction->InputAt(1)));
1670  if (Primitive::IsFloatingPointType(instruction->InputAt(2)->GetType())) {
1671    locations->SetInAt(2, Location::RequiresFpuRegister());
1672  } else {
1673    locations->SetInAt(2, Location::RequiresRegister());
1674  }
1675}
1676
1677void InstructionCodeGeneratorARM64::VisitArraySet(HArraySet* instruction) {
1678  Primitive::Type value_type = instruction->GetComponentType();
1679  LocationSummary* locations = instruction->GetLocations();
1680  bool may_need_runtime_call = locations->CanCall();
1681  bool needs_write_barrier =
1682      CodeGenerator::StoreNeedsWriteBarrier(value_type, instruction->GetValue());
1683
1684  Register array = InputRegisterAt(instruction, 0);
1685  CPURegister value = InputCPURegisterAt(instruction, 2);
1686  CPURegister source = value;
1687  Location index = locations->InAt(1);
1688  size_t offset = mirror::Array::DataOffset(Primitive::ComponentSize(value_type)).Uint32Value();
1689  MemOperand destination = HeapOperand(array);
1690  MacroAssembler* masm = GetVIXLAssembler();
1691  BlockPoolsScope block_pools(masm);
1692
1693  if (!needs_write_barrier) {
1694    DCHECK(!may_need_runtime_call);
1695    if (index.IsConstant()) {
1696      offset += Int64ConstantFrom(index) << Primitive::ComponentSizeShift(value_type);
1697      destination = HeapOperand(array, offset);
1698    } else {
1699      UseScratchRegisterScope temps(masm);
1700      Register temp = temps.AcquireSameSizeAs(array);
1701      if (instruction->GetArray()->IsArm64IntermediateAddress()) {
1702        // We do not need to compute the intermediate address from the array: the
1703        // input instruction has done it already. See the comment in
1704        // `InstructionSimplifierArm64::TryExtractArrayAccessAddress()`.
1705        if (kIsDebugBuild) {
1706          HArm64IntermediateAddress* tmp = instruction->GetArray()->AsArm64IntermediateAddress();
1707          DCHECK(tmp->GetOffset()->AsIntConstant()->GetValueAsUint64() == offset);
1708        }
1709        temp = array;
1710      } else {
1711        __ Add(temp, array, offset);
1712      }
1713      destination = HeapOperand(temp,
1714                                XRegisterFrom(index),
1715                                LSL,
1716                                Primitive::ComponentSizeShift(value_type));
1717    }
1718    codegen_->Store(value_type, value, destination);
1719    codegen_->MaybeRecordImplicitNullCheck(instruction);
1720  } else {
1721    DCHECK(needs_write_barrier);
1722    DCHECK(!instruction->GetArray()->IsArm64IntermediateAddress());
1723    vixl::Label done;
1724    SlowPathCodeARM64* slow_path = nullptr;
1725    {
1726      // We use a block to end the scratch scope before the write barrier, thus
1727      // freeing the temporary registers so they can be used in `MarkGCCard`.
1728      UseScratchRegisterScope temps(masm);
1729      Register temp = temps.AcquireSameSizeAs(array);
1730      if (index.IsConstant()) {
1731        offset += Int64ConstantFrom(index) << Primitive::ComponentSizeShift(value_type);
1732        destination = HeapOperand(array, offset);
1733      } else {
1734        destination = HeapOperand(temp,
1735                                  XRegisterFrom(index),
1736                                  LSL,
1737                                  Primitive::ComponentSizeShift(value_type));
1738      }
1739
1740      uint32_t class_offset = mirror::Object::ClassOffset().Int32Value();
1741      uint32_t super_offset = mirror::Class::SuperClassOffset().Int32Value();
1742      uint32_t component_offset = mirror::Class::ComponentTypeOffset().Int32Value();
1743
1744      if (may_need_runtime_call) {
1745        slow_path = new (GetGraph()->GetArena()) ArraySetSlowPathARM64(instruction);
1746        codegen_->AddSlowPath(slow_path);
1747        if (instruction->GetValueCanBeNull()) {
1748          vixl::Label non_zero;
1749          __ Cbnz(Register(value), &non_zero);
1750          if (!index.IsConstant()) {
1751            __ Add(temp, array, offset);
1752          }
1753          __ Str(wzr, destination);
1754          codegen_->MaybeRecordImplicitNullCheck(instruction);
1755          __ B(&done);
1756          __ Bind(&non_zero);
1757        }
1758
1759        Register temp2 = temps.AcquireSameSizeAs(array);
1760        __ Ldr(temp, HeapOperand(array, class_offset));
1761        codegen_->MaybeRecordImplicitNullCheck(instruction);
1762        GetAssembler()->MaybeUnpoisonHeapReference(temp);
1763        __ Ldr(temp, HeapOperand(temp, component_offset));
1764        __ Ldr(temp2, HeapOperand(Register(value), class_offset));
1765        // No need to poison/unpoison, we're comparing two poisoned references.
1766        __ Cmp(temp, temp2);
1767        if (instruction->StaticTypeOfArrayIsObjectArray()) {
1768          vixl::Label do_put;
1769          __ B(eq, &do_put);
1770          GetAssembler()->MaybeUnpoisonHeapReference(temp);
1771          __ Ldr(temp, HeapOperand(temp, super_offset));
1772          // No need to unpoison, we're comparing against null.
1773          __ Cbnz(temp, slow_path->GetEntryLabel());
1774          __ Bind(&do_put);
1775        } else {
1776          __ B(ne, slow_path->GetEntryLabel());
1777        }
1778        temps.Release(temp2);
1779      }
1780
1781      if (kPoisonHeapReferences) {
1782        Register temp2 = temps.AcquireSameSizeAs(array);
1783          DCHECK(value.IsW());
1784        __ Mov(temp2, value.W());
1785        GetAssembler()->PoisonHeapReference(temp2);
1786        source = temp2;
1787      }
1788
1789      if (!index.IsConstant()) {
1790        __ Add(temp, array, offset);
1791      }
1792      __ Str(source, destination);
1793
1794      if (!may_need_runtime_call) {
1795        codegen_->MaybeRecordImplicitNullCheck(instruction);
1796      }
1797    }
1798
1799    codegen_->MarkGCCard(array, value.W(), instruction->GetValueCanBeNull());
1800
1801    if (done.IsLinked()) {
1802      __ Bind(&done);
1803    }
1804
1805    if (slow_path != nullptr) {
1806      __ Bind(slow_path->GetExitLabel());
1807    }
1808  }
1809}
1810
1811void LocationsBuilderARM64::VisitBoundsCheck(HBoundsCheck* instruction) {
1812  LocationSummary::CallKind call_kind = instruction->CanThrowIntoCatchBlock()
1813      ? LocationSummary::kCallOnSlowPath
1814      : LocationSummary::kNoCall;
1815  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction, call_kind);
1816  locations->SetInAt(0, Location::RequiresRegister());
1817  locations->SetInAt(1, ARM64EncodableConstantOrRegister(instruction->InputAt(1), instruction));
1818  if (instruction->HasUses()) {
1819    locations->SetOut(Location::SameAsFirstInput());
1820  }
1821}
1822
1823void InstructionCodeGeneratorARM64::VisitBoundsCheck(HBoundsCheck* instruction) {
1824  BoundsCheckSlowPathARM64* slow_path =
1825      new (GetGraph()->GetArena()) BoundsCheckSlowPathARM64(instruction);
1826  codegen_->AddSlowPath(slow_path);
1827
1828  __ Cmp(InputRegisterAt(instruction, 0), InputOperandAt(instruction, 1));
1829  __ B(slow_path->GetEntryLabel(), hs);
1830}
1831
1832void LocationsBuilderARM64::VisitClinitCheck(HClinitCheck* check) {
1833  LocationSummary* locations =
1834      new (GetGraph()->GetArena()) LocationSummary(check, LocationSummary::kCallOnSlowPath);
1835  locations->SetInAt(0, Location::RequiresRegister());
1836  if (check->HasUses()) {
1837    locations->SetOut(Location::SameAsFirstInput());
1838  }
1839}
1840
1841void InstructionCodeGeneratorARM64::VisitClinitCheck(HClinitCheck* check) {
1842  // We assume the class is not null.
1843  SlowPathCodeARM64* slow_path = new (GetGraph()->GetArena()) LoadClassSlowPathARM64(
1844      check->GetLoadClass(), check, check->GetDexPc(), true);
1845  codegen_->AddSlowPath(slow_path);
1846  GenerateClassInitializationCheck(slow_path, InputRegisterAt(check, 0));
1847}
1848
1849static bool IsFloatingPointZeroConstant(HInstruction* instruction) {
1850  return (instruction->IsFloatConstant() && (instruction->AsFloatConstant()->GetValue() == 0.0f))
1851      || (instruction->IsDoubleConstant() && (instruction->AsDoubleConstant()->GetValue() == 0.0));
1852}
1853
1854void LocationsBuilderARM64::VisitCompare(HCompare* compare) {
1855  LocationSummary* locations =
1856      new (GetGraph()->GetArena()) LocationSummary(compare, LocationSummary::kNoCall);
1857  Primitive::Type in_type = compare->InputAt(0)->GetType();
1858  switch (in_type) {
1859    case Primitive::kPrimLong: {
1860      locations->SetInAt(0, Location::RequiresRegister());
1861      locations->SetInAt(1, ARM64EncodableConstantOrRegister(compare->InputAt(1), compare));
1862      locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
1863      break;
1864    }
1865    case Primitive::kPrimFloat:
1866    case Primitive::kPrimDouble: {
1867      locations->SetInAt(0, Location::RequiresFpuRegister());
1868      locations->SetInAt(1,
1869                         IsFloatingPointZeroConstant(compare->InputAt(1))
1870                             ? Location::ConstantLocation(compare->InputAt(1)->AsConstant())
1871                             : Location::RequiresFpuRegister());
1872      locations->SetOut(Location::RequiresRegister());
1873      break;
1874    }
1875    default:
1876      LOG(FATAL) << "Unexpected type for compare operation " << in_type;
1877  }
1878}
1879
1880void InstructionCodeGeneratorARM64::VisitCompare(HCompare* compare) {
1881  Primitive::Type in_type = compare->InputAt(0)->GetType();
1882
1883  //  0 if: left == right
1884  //  1 if: left  > right
1885  // -1 if: left  < right
1886  switch (in_type) {
1887    case Primitive::kPrimLong: {
1888      Register result = OutputRegister(compare);
1889      Register left = InputRegisterAt(compare, 0);
1890      Operand right = InputOperandAt(compare, 1);
1891
1892      __ Cmp(left, right);
1893      __ Cset(result, ne);
1894      __ Cneg(result, result, lt);
1895      break;
1896    }
1897    case Primitive::kPrimFloat:
1898    case Primitive::kPrimDouble: {
1899      Register result = OutputRegister(compare);
1900      FPRegister left = InputFPRegisterAt(compare, 0);
1901      if (compare->GetLocations()->InAt(1).IsConstant()) {
1902        DCHECK(IsFloatingPointZeroConstant(compare->GetLocations()->InAt(1).GetConstant()));
1903        // 0.0 is the only immediate that can be encoded directly in an FCMP instruction.
1904        __ Fcmp(left, 0.0);
1905      } else {
1906        __ Fcmp(left, InputFPRegisterAt(compare, 1));
1907      }
1908      if (compare->IsGtBias()) {
1909        __ Cset(result, ne);
1910      } else {
1911        __ Csetm(result, ne);
1912      }
1913      __ Cneg(result, result, compare->IsGtBias() ? mi : gt);
1914      break;
1915    }
1916    default:
1917      LOG(FATAL) << "Unimplemented compare type " << in_type;
1918  }
1919}
1920
1921void LocationsBuilderARM64::VisitCondition(HCondition* instruction) {
1922  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction);
1923
1924  if (Primitive::IsFloatingPointType(instruction->InputAt(0)->GetType())) {
1925    locations->SetInAt(0, Location::RequiresFpuRegister());
1926    locations->SetInAt(1,
1927                       IsFloatingPointZeroConstant(instruction->InputAt(1))
1928                           ? Location::ConstantLocation(instruction->InputAt(1)->AsConstant())
1929                           : Location::RequiresFpuRegister());
1930  } else {
1931    // Integer cases.
1932    locations->SetInAt(0, Location::RequiresRegister());
1933    locations->SetInAt(1, ARM64EncodableConstantOrRegister(instruction->InputAt(1), instruction));
1934  }
1935
1936  if (instruction->NeedsMaterialization()) {
1937    locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
1938  }
1939}
1940
1941void InstructionCodeGeneratorARM64::VisitCondition(HCondition* instruction) {
1942  if (!instruction->NeedsMaterialization()) {
1943    return;
1944  }
1945
1946  LocationSummary* locations = instruction->GetLocations();
1947  Register res = RegisterFrom(locations->Out(), instruction->GetType());
1948  IfCondition if_cond = instruction->GetCondition();
1949  Condition arm64_cond = ARM64Condition(if_cond);
1950
1951  if (Primitive::IsFloatingPointType(instruction->InputAt(0)->GetType())) {
1952    FPRegister lhs = InputFPRegisterAt(instruction, 0);
1953    if (locations->InAt(1).IsConstant()) {
1954      DCHECK(IsFloatingPointZeroConstant(locations->InAt(1).GetConstant()));
1955      // 0.0 is the only immediate that can be encoded directly in an FCMP instruction.
1956      __ Fcmp(lhs, 0.0);
1957    } else {
1958      __ Fcmp(lhs, InputFPRegisterAt(instruction, 1));
1959    }
1960    __ Cset(res, arm64_cond);
1961    if (instruction->IsFPConditionTrueIfNaN()) {
1962      // res = IsUnordered(arm64_cond) ? 1 : res  <=>  res = IsNotUnordered(arm64_cond) ? res : 1
1963      __ Csel(res, res, Operand(1), vc);  // VC for "not unordered".
1964    } else if (instruction->IsFPConditionFalseIfNaN()) {
1965      // res = IsUnordered(arm64_cond) ? 0 : res  <=>  res = IsNotUnordered(arm64_cond) ? res : 0
1966      __ Csel(res, res, Operand(0), vc);  // VC for "not unordered".
1967    }
1968  } else {
1969    // Integer cases.
1970    Register lhs = InputRegisterAt(instruction, 0);
1971    Operand rhs = InputOperandAt(instruction, 1);
1972    __ Cmp(lhs, rhs);
1973    __ Cset(res, arm64_cond);
1974  }
1975}
1976
1977#define FOR_EACH_CONDITION_INSTRUCTION(M)                                                \
1978  M(Equal)                                                                               \
1979  M(NotEqual)                                                                            \
1980  M(LessThan)                                                                            \
1981  M(LessThanOrEqual)                                                                     \
1982  M(GreaterThan)                                                                         \
1983  M(GreaterThanOrEqual)                                                                  \
1984  M(Below)                                                                               \
1985  M(BelowOrEqual)                                                                        \
1986  M(Above)                                                                               \
1987  M(AboveOrEqual)
1988#define DEFINE_CONDITION_VISITORS(Name)                                                  \
1989void LocationsBuilderARM64::Visit##Name(H##Name* comp) { VisitCondition(comp); }         \
1990void InstructionCodeGeneratorARM64::Visit##Name(H##Name* comp) { VisitCondition(comp); }
1991FOR_EACH_CONDITION_INSTRUCTION(DEFINE_CONDITION_VISITORS)
1992#undef DEFINE_CONDITION_VISITORS
1993#undef FOR_EACH_CONDITION_INSTRUCTION
1994
1995void InstructionCodeGeneratorARM64::DivRemOneOrMinusOne(HBinaryOperation* instruction) {
1996  DCHECK(instruction->IsDiv() || instruction->IsRem());
1997
1998  LocationSummary* locations = instruction->GetLocations();
1999  Location second = locations->InAt(1);
2000  DCHECK(second.IsConstant());
2001
2002  Register out = OutputRegister(instruction);
2003  Register dividend = InputRegisterAt(instruction, 0);
2004  int64_t imm = Int64FromConstant(second.GetConstant());
2005  DCHECK(imm == 1 || imm == -1);
2006
2007  if (instruction->IsRem()) {
2008    __ Mov(out, 0);
2009  } else {
2010    if (imm == 1) {
2011      __ Mov(out, dividend);
2012    } else {
2013      __ Neg(out, dividend);
2014    }
2015  }
2016}
2017
2018void InstructionCodeGeneratorARM64::DivRemByPowerOfTwo(HBinaryOperation* instruction) {
2019  DCHECK(instruction->IsDiv() || instruction->IsRem());
2020
2021  LocationSummary* locations = instruction->GetLocations();
2022  Location second = locations->InAt(1);
2023  DCHECK(second.IsConstant());
2024
2025  Register out = OutputRegister(instruction);
2026  Register dividend = InputRegisterAt(instruction, 0);
2027  int64_t imm = Int64FromConstant(second.GetConstant());
2028  uint64_t abs_imm = static_cast<uint64_t>(std::abs(imm));
2029  DCHECK(IsPowerOfTwo(abs_imm));
2030  int ctz_imm = CTZ(abs_imm);
2031
2032  UseScratchRegisterScope temps(GetVIXLAssembler());
2033  Register temp = temps.AcquireSameSizeAs(out);
2034
2035  if (instruction->IsDiv()) {
2036    __ Add(temp, dividend, abs_imm - 1);
2037    __ Cmp(dividend, 0);
2038    __ Csel(out, temp, dividend, lt);
2039    if (imm > 0) {
2040      __ Asr(out, out, ctz_imm);
2041    } else {
2042      __ Neg(out, Operand(out, ASR, ctz_imm));
2043    }
2044  } else {
2045    int bits = instruction->GetResultType() == Primitive::kPrimInt ? 32 : 64;
2046    __ Asr(temp, dividend, bits - 1);
2047    __ Lsr(temp, temp, bits - ctz_imm);
2048    __ Add(out, dividend, temp);
2049    __ And(out, out, abs_imm - 1);
2050    __ Sub(out, out, temp);
2051  }
2052}
2053
2054void InstructionCodeGeneratorARM64::GenerateDivRemWithAnyConstant(HBinaryOperation* instruction) {
2055  DCHECK(instruction->IsDiv() || instruction->IsRem());
2056
2057  LocationSummary* locations = instruction->GetLocations();
2058  Location second = locations->InAt(1);
2059  DCHECK(second.IsConstant());
2060
2061  Register out = OutputRegister(instruction);
2062  Register dividend = InputRegisterAt(instruction, 0);
2063  int64_t imm = Int64FromConstant(second.GetConstant());
2064
2065  Primitive::Type type = instruction->GetResultType();
2066  DCHECK(type == Primitive::kPrimInt || type == Primitive::kPrimLong);
2067
2068  int64_t magic;
2069  int shift;
2070  CalculateMagicAndShiftForDivRem(imm, type == Primitive::kPrimLong /* is_long */, &magic, &shift);
2071
2072  UseScratchRegisterScope temps(GetVIXLAssembler());
2073  Register temp = temps.AcquireSameSizeAs(out);
2074
2075  // temp = get_high(dividend * magic)
2076  __ Mov(temp, magic);
2077  if (type == Primitive::kPrimLong) {
2078    __ Smulh(temp, dividend, temp);
2079  } else {
2080    __ Smull(temp.X(), dividend, temp);
2081    __ Lsr(temp.X(), temp.X(), 32);
2082  }
2083
2084  if (imm > 0 && magic < 0) {
2085    __ Add(temp, temp, dividend);
2086  } else if (imm < 0 && magic > 0) {
2087    __ Sub(temp, temp, dividend);
2088  }
2089
2090  if (shift != 0) {
2091    __ Asr(temp, temp, shift);
2092  }
2093
2094  if (instruction->IsDiv()) {
2095    __ Sub(out, temp, Operand(temp, ASR, type == Primitive::kPrimLong ? 63 : 31));
2096  } else {
2097    __ Sub(temp, temp, Operand(temp, ASR, type == Primitive::kPrimLong ? 63 : 31));
2098    // TODO: Strength reduction for msub.
2099    Register temp_imm = temps.AcquireSameSizeAs(out);
2100    __ Mov(temp_imm, imm);
2101    __ Msub(out, temp, temp_imm, dividend);
2102  }
2103}
2104
2105void InstructionCodeGeneratorARM64::GenerateDivRemIntegral(HBinaryOperation* instruction) {
2106  DCHECK(instruction->IsDiv() || instruction->IsRem());
2107  Primitive::Type type = instruction->GetResultType();
2108  DCHECK(type == Primitive::kPrimInt || Primitive::kPrimLong);
2109
2110  LocationSummary* locations = instruction->GetLocations();
2111  Register out = OutputRegister(instruction);
2112  Location second = locations->InAt(1);
2113
2114  if (second.IsConstant()) {
2115    int64_t imm = Int64FromConstant(second.GetConstant());
2116
2117    if (imm == 0) {
2118      // Do not generate anything. DivZeroCheck would prevent any code to be executed.
2119    } else if (imm == 1 || imm == -1) {
2120      DivRemOneOrMinusOne(instruction);
2121    } else if (IsPowerOfTwo(std::abs(imm))) {
2122      DivRemByPowerOfTwo(instruction);
2123    } else {
2124      DCHECK(imm <= -2 || imm >= 2);
2125      GenerateDivRemWithAnyConstant(instruction);
2126    }
2127  } else {
2128    Register dividend = InputRegisterAt(instruction, 0);
2129    Register divisor = InputRegisterAt(instruction, 1);
2130    if (instruction->IsDiv()) {
2131      __ Sdiv(out, dividend, divisor);
2132    } else {
2133      UseScratchRegisterScope temps(GetVIXLAssembler());
2134      Register temp = temps.AcquireSameSizeAs(out);
2135      __ Sdiv(temp, dividend, divisor);
2136      __ Msub(out, temp, divisor, dividend);
2137    }
2138  }
2139}
2140
2141void LocationsBuilderARM64::VisitDiv(HDiv* div) {
2142  LocationSummary* locations =
2143      new (GetGraph()->GetArena()) LocationSummary(div, LocationSummary::kNoCall);
2144  switch (div->GetResultType()) {
2145    case Primitive::kPrimInt:
2146    case Primitive::kPrimLong:
2147      locations->SetInAt(0, Location::RequiresRegister());
2148      locations->SetInAt(1, Location::RegisterOrConstant(div->InputAt(1)));
2149      locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
2150      break;
2151
2152    case Primitive::kPrimFloat:
2153    case Primitive::kPrimDouble:
2154      locations->SetInAt(0, Location::RequiresFpuRegister());
2155      locations->SetInAt(1, Location::RequiresFpuRegister());
2156      locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap);
2157      break;
2158
2159    default:
2160      LOG(FATAL) << "Unexpected div type " << div->GetResultType();
2161  }
2162}
2163
2164void InstructionCodeGeneratorARM64::VisitDiv(HDiv* div) {
2165  Primitive::Type type = div->GetResultType();
2166  switch (type) {
2167    case Primitive::kPrimInt:
2168    case Primitive::kPrimLong:
2169      GenerateDivRemIntegral(div);
2170      break;
2171
2172    case Primitive::kPrimFloat:
2173    case Primitive::kPrimDouble:
2174      __ Fdiv(OutputFPRegister(div), InputFPRegisterAt(div, 0), InputFPRegisterAt(div, 1));
2175      break;
2176
2177    default:
2178      LOG(FATAL) << "Unexpected div type " << type;
2179  }
2180}
2181
2182void LocationsBuilderARM64::VisitDivZeroCheck(HDivZeroCheck* instruction) {
2183  LocationSummary::CallKind call_kind = instruction->CanThrowIntoCatchBlock()
2184      ? LocationSummary::kCallOnSlowPath
2185      : LocationSummary::kNoCall;
2186  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction, call_kind);
2187  locations->SetInAt(0, Location::RegisterOrConstant(instruction->InputAt(0)));
2188  if (instruction->HasUses()) {
2189    locations->SetOut(Location::SameAsFirstInput());
2190  }
2191}
2192
2193void InstructionCodeGeneratorARM64::VisitDivZeroCheck(HDivZeroCheck* instruction) {
2194  SlowPathCodeARM64* slow_path =
2195      new (GetGraph()->GetArena()) DivZeroCheckSlowPathARM64(instruction);
2196  codegen_->AddSlowPath(slow_path);
2197  Location value = instruction->GetLocations()->InAt(0);
2198
2199  Primitive::Type type = instruction->GetType();
2200
2201  if ((type == Primitive::kPrimBoolean) || !Primitive::IsIntegralType(type)) {
2202      LOG(FATAL) << "Unexpected type " << type << " for DivZeroCheck.";
2203    return;
2204  }
2205
2206  if (value.IsConstant()) {
2207    int64_t divisor = Int64ConstantFrom(value);
2208    if (divisor == 0) {
2209      __ B(slow_path->GetEntryLabel());
2210    } else {
2211      // A division by a non-null constant is valid. We don't need to perform
2212      // any check, so simply fall through.
2213    }
2214  } else {
2215    __ Cbz(InputRegisterAt(instruction, 0), slow_path->GetEntryLabel());
2216  }
2217}
2218
2219void LocationsBuilderARM64::VisitDoubleConstant(HDoubleConstant* constant) {
2220  LocationSummary* locations =
2221      new (GetGraph()->GetArena()) LocationSummary(constant, LocationSummary::kNoCall);
2222  locations->SetOut(Location::ConstantLocation(constant));
2223}
2224
2225void InstructionCodeGeneratorARM64::VisitDoubleConstant(
2226    HDoubleConstant* constant ATTRIBUTE_UNUSED) {
2227  // Will be generated at use site.
2228}
2229
2230void LocationsBuilderARM64::VisitExit(HExit* exit) {
2231  exit->SetLocations(nullptr);
2232}
2233
2234void InstructionCodeGeneratorARM64::VisitExit(HExit* exit ATTRIBUTE_UNUSED) {
2235}
2236
2237void LocationsBuilderARM64::VisitFloatConstant(HFloatConstant* constant) {
2238  LocationSummary* locations =
2239      new (GetGraph()->GetArena()) LocationSummary(constant, LocationSummary::kNoCall);
2240  locations->SetOut(Location::ConstantLocation(constant));
2241}
2242
2243void InstructionCodeGeneratorARM64::VisitFloatConstant(HFloatConstant* constant ATTRIBUTE_UNUSED) {
2244  // Will be generated at use site.
2245}
2246
2247void InstructionCodeGeneratorARM64::HandleGoto(HInstruction* got, HBasicBlock* successor) {
2248  DCHECK(!successor->IsExitBlock());
2249  HBasicBlock* block = got->GetBlock();
2250  HInstruction* previous = got->GetPrevious();
2251  HLoopInformation* info = block->GetLoopInformation();
2252
2253  if (info != nullptr && info->IsBackEdge(*block) && info->HasSuspendCheck()) {
2254    codegen_->ClearSpillSlotsFromLoopPhisInStackMap(info->GetSuspendCheck());
2255    GenerateSuspendCheck(info->GetSuspendCheck(), successor);
2256    return;
2257  }
2258  if (block->IsEntryBlock() && (previous != nullptr) && previous->IsSuspendCheck()) {
2259    GenerateSuspendCheck(previous->AsSuspendCheck(), nullptr);
2260  }
2261  if (!codegen_->GoesToNextBlock(block, successor)) {
2262    __ B(codegen_->GetLabelOf(successor));
2263  }
2264}
2265
2266void LocationsBuilderARM64::VisitGoto(HGoto* got) {
2267  got->SetLocations(nullptr);
2268}
2269
2270void InstructionCodeGeneratorARM64::VisitGoto(HGoto* got) {
2271  HandleGoto(got, got->GetSuccessor());
2272}
2273
2274void LocationsBuilderARM64::VisitTryBoundary(HTryBoundary* try_boundary) {
2275  try_boundary->SetLocations(nullptr);
2276}
2277
2278void InstructionCodeGeneratorARM64::VisitTryBoundary(HTryBoundary* try_boundary) {
2279  HBasicBlock* successor = try_boundary->GetNormalFlowSuccessor();
2280  if (!successor->IsExitBlock()) {
2281    HandleGoto(try_boundary, successor);
2282  }
2283}
2284
2285void InstructionCodeGeneratorARM64::GenerateTestAndBranch(HInstruction* instruction,
2286                                                          vixl::Label* true_target,
2287                                                          vixl::Label* false_target,
2288                                                          vixl::Label* always_true_target) {
2289  HInstruction* cond = instruction->InputAt(0);
2290  HCondition* condition = cond->AsCondition();
2291
2292  if (cond->IsIntConstant()) {
2293    int32_t cond_value = cond->AsIntConstant()->GetValue();
2294    if (cond_value == 1) {
2295      if (always_true_target != nullptr) {
2296        __ B(always_true_target);
2297      }
2298      return;
2299    } else {
2300      DCHECK_EQ(cond_value, 0);
2301    }
2302  } else if (!cond->IsCondition() || condition->NeedsMaterialization()) {
2303    // The condition instruction has been materialized, compare the output to 0.
2304    Location cond_val = instruction->GetLocations()->InAt(0);
2305    DCHECK(cond_val.IsRegister());
2306    __ Cbnz(InputRegisterAt(instruction, 0), true_target);
2307  } else {
2308    // The condition instruction has not been materialized, use its inputs as
2309    // the comparison and its condition as the branch condition.
2310    Primitive::Type type =
2311        cond->IsCondition() ? cond->InputAt(0)->GetType() : Primitive::kPrimInt;
2312
2313    if (Primitive::IsFloatingPointType(type)) {
2314      // FP compares don't like null false_targets.
2315      if (false_target == nullptr) {
2316        false_target = codegen_->GetLabelOf(instruction->AsIf()->IfFalseSuccessor());
2317      }
2318      FPRegister lhs = InputFPRegisterAt(condition, 0);
2319      if (condition->GetLocations()->InAt(1).IsConstant()) {
2320        DCHECK(IsFloatingPointZeroConstant(condition->GetLocations()->InAt(1).GetConstant()));
2321        // 0.0 is the only immediate that can be encoded directly in an FCMP instruction.
2322        __ Fcmp(lhs, 0.0);
2323      } else {
2324        __ Fcmp(lhs, InputFPRegisterAt(condition, 1));
2325      }
2326      if (condition->IsFPConditionTrueIfNaN()) {
2327        __ B(vs, true_target);  // VS for unordered.
2328      } else if (condition->IsFPConditionFalseIfNaN()) {
2329        __ B(vs, false_target);  // VS for unordered.
2330      }
2331      __ B(ARM64Condition(condition->GetCondition()), true_target);
2332    } else {
2333      // Integer cases.
2334      Register lhs = InputRegisterAt(condition, 0);
2335      Operand rhs = InputOperandAt(condition, 1);
2336      Condition arm64_cond = ARM64Condition(condition->GetCondition());
2337      if ((arm64_cond != gt && arm64_cond != le) && rhs.IsImmediate() && (rhs.immediate() == 0)) {
2338        switch (arm64_cond) {
2339          case eq:
2340            __ Cbz(lhs, true_target);
2341            break;
2342          case ne:
2343            __ Cbnz(lhs, true_target);
2344            break;
2345          case lt:
2346            // Test the sign bit and branch accordingly.
2347            __ Tbnz(lhs, (lhs.IsX() ? kXRegSize : kWRegSize) - 1, true_target);
2348            break;
2349          case ge:
2350            // Test the sign bit and branch accordingly.
2351            __ Tbz(lhs, (lhs.IsX() ? kXRegSize : kWRegSize) - 1, true_target);
2352            break;
2353          default:
2354            // Without the `static_cast` the compiler throws an error for
2355            // `-Werror=sign-promo`.
2356            LOG(FATAL) << "Unexpected condition: " << static_cast<int>(arm64_cond);
2357        }
2358      } else {
2359        __ Cmp(lhs, rhs);
2360        __ B(arm64_cond, true_target);
2361      }
2362    }
2363  }
2364  if (false_target != nullptr) {
2365    __ B(false_target);
2366  }
2367}
2368
2369void LocationsBuilderARM64::VisitIf(HIf* if_instr) {
2370  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(if_instr);
2371  HInstruction* cond = if_instr->InputAt(0);
2372  if (!cond->IsCondition() || cond->AsCondition()->NeedsMaterialization()) {
2373    locations->SetInAt(0, Location::RequiresRegister());
2374  }
2375}
2376
2377void InstructionCodeGeneratorARM64::VisitIf(HIf* if_instr) {
2378  vixl::Label* true_target = codegen_->GetLabelOf(if_instr->IfTrueSuccessor());
2379  vixl::Label* false_target = codegen_->GetLabelOf(if_instr->IfFalseSuccessor());
2380  vixl::Label* always_true_target = true_target;
2381  if (codegen_->GoesToNextBlock(if_instr->GetBlock(),
2382                                if_instr->IfTrueSuccessor())) {
2383    always_true_target = nullptr;
2384  }
2385  if (codegen_->GoesToNextBlock(if_instr->GetBlock(),
2386                                if_instr->IfFalseSuccessor())) {
2387    false_target = nullptr;
2388  }
2389  GenerateTestAndBranch(if_instr, true_target, false_target, always_true_target);
2390}
2391
2392void LocationsBuilderARM64::VisitDeoptimize(HDeoptimize* deoptimize) {
2393  LocationSummary* locations = new (GetGraph()->GetArena())
2394      LocationSummary(deoptimize, LocationSummary::kCallOnSlowPath);
2395  HInstruction* cond = deoptimize->InputAt(0);
2396  if (!cond->IsCondition() || cond->AsCondition()->NeedsMaterialization()) {
2397    locations->SetInAt(0, Location::RequiresRegister());
2398  }
2399}
2400
2401void InstructionCodeGeneratorARM64::VisitDeoptimize(HDeoptimize* deoptimize) {
2402  SlowPathCodeARM64* slow_path = new (GetGraph()->GetArena())
2403      DeoptimizationSlowPathARM64(deoptimize);
2404  codegen_->AddSlowPath(slow_path);
2405  vixl::Label* slow_path_entry = slow_path->GetEntryLabel();
2406  GenerateTestAndBranch(deoptimize, slow_path_entry, nullptr, slow_path_entry);
2407}
2408
2409void LocationsBuilderARM64::VisitInstanceFieldGet(HInstanceFieldGet* instruction) {
2410  HandleFieldGet(instruction);
2411}
2412
2413void InstructionCodeGeneratorARM64::VisitInstanceFieldGet(HInstanceFieldGet* instruction) {
2414  HandleFieldGet(instruction, instruction->GetFieldInfo());
2415}
2416
2417void LocationsBuilderARM64::VisitInstanceFieldSet(HInstanceFieldSet* instruction) {
2418  HandleFieldSet(instruction);
2419}
2420
2421void InstructionCodeGeneratorARM64::VisitInstanceFieldSet(HInstanceFieldSet* instruction) {
2422  HandleFieldSet(instruction, instruction->GetFieldInfo(), instruction->GetValueCanBeNull());
2423}
2424
2425void LocationsBuilderARM64::VisitInstanceOf(HInstanceOf* instruction) {
2426  LocationSummary::CallKind call_kind = LocationSummary::kNoCall;
2427  switch (instruction->GetTypeCheckKind()) {
2428    case TypeCheckKind::kExactCheck:
2429    case TypeCheckKind::kAbstractClassCheck:
2430    case TypeCheckKind::kClassHierarchyCheck:
2431    case TypeCheckKind::kArrayObjectCheck:
2432      call_kind = LocationSummary::kNoCall;
2433      break;
2434    case TypeCheckKind::kUnresolvedCheck:
2435    case TypeCheckKind::kInterfaceCheck:
2436      call_kind = LocationSummary::kCall;
2437      break;
2438    case TypeCheckKind::kArrayCheck:
2439      call_kind = LocationSummary::kCallOnSlowPath;
2440      break;
2441  }
2442  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction, call_kind);
2443  if (call_kind != LocationSummary::kCall) {
2444    locations->SetInAt(0, Location::RequiresRegister());
2445    locations->SetInAt(1, Location::RequiresRegister());
2446    // The out register is used as a temporary, so it overlaps with the inputs.
2447    // Note that TypeCheckSlowPathARM64 uses this register too.
2448    locations->SetOut(Location::RequiresRegister(), Location::kOutputOverlap);
2449  } else {
2450    InvokeRuntimeCallingConvention calling_convention;
2451    locations->SetInAt(1, LocationFrom(calling_convention.GetRegisterAt(0)));
2452    locations->SetInAt(0, LocationFrom(calling_convention.GetRegisterAt(1)));
2453    locations->SetOut(calling_convention.GetReturnLocation(Primitive::kPrimInt));
2454  }
2455}
2456
2457void InstructionCodeGeneratorARM64::VisitInstanceOf(HInstanceOf* instruction) {
2458  LocationSummary* locations = instruction->GetLocations();
2459  Register obj = InputRegisterAt(instruction, 0);
2460  Register cls = InputRegisterAt(instruction, 1);
2461  Register out = OutputRegister(instruction);
2462  uint32_t class_offset = mirror::Object::ClassOffset().Int32Value();
2463  uint32_t super_offset = mirror::Class::SuperClassOffset().Int32Value();
2464  uint32_t component_offset = mirror::Class::ComponentTypeOffset().Int32Value();
2465  uint32_t primitive_offset = mirror::Class::PrimitiveTypeOffset().Int32Value();
2466
2467  vixl::Label done, zero;
2468  SlowPathCodeARM64* slow_path = nullptr;
2469
2470  // Return 0 if `obj` is null.
2471  // Avoid null check if we know `obj` is not null.
2472  if (instruction->MustDoNullCheck()) {
2473    __ Cbz(obj, &zero);
2474  }
2475
2476  // In case of an interface/unresolved check, we put the object class into the object register.
2477  // This is safe, as the register is caller-save, and the object must be in another
2478  // register if it survives the runtime call.
2479  Register target = (instruction->GetTypeCheckKind() == TypeCheckKind::kInterfaceCheck) ||
2480      (instruction->GetTypeCheckKind() == TypeCheckKind::kUnresolvedCheck)
2481      ? obj
2482      : out;
2483  __ Ldr(target, HeapOperand(obj.W(), class_offset));
2484  GetAssembler()->MaybeUnpoisonHeapReference(target);
2485
2486  switch (instruction->GetTypeCheckKind()) {
2487    case TypeCheckKind::kExactCheck: {
2488      __ Cmp(out, cls);
2489      __ Cset(out, eq);
2490      if (zero.IsLinked()) {
2491        __ B(&done);
2492      }
2493      break;
2494    }
2495    case TypeCheckKind::kAbstractClassCheck: {
2496      // If the class is abstract, we eagerly fetch the super class of the
2497      // object to avoid doing a comparison we know will fail.
2498      vixl::Label loop, success;
2499      __ Bind(&loop);
2500      __ Ldr(out, HeapOperand(out, super_offset));
2501      GetAssembler()->MaybeUnpoisonHeapReference(out);
2502      // If `out` is null, we use it for the result, and jump to `done`.
2503      __ Cbz(out, &done);
2504      __ Cmp(out, cls);
2505      __ B(ne, &loop);
2506      __ Mov(out, 1);
2507      if (zero.IsLinked()) {
2508        __ B(&done);
2509      }
2510      break;
2511    }
2512    case TypeCheckKind::kClassHierarchyCheck: {
2513      // Walk over the class hierarchy to find a match.
2514      vixl::Label loop, success;
2515      __ Bind(&loop);
2516      __ Cmp(out, cls);
2517      __ B(eq, &success);
2518      __ Ldr(out, HeapOperand(out, super_offset));
2519      GetAssembler()->MaybeUnpoisonHeapReference(out);
2520      __ Cbnz(out, &loop);
2521      // If `out` is null, we use it for the result, and jump to `done`.
2522      __ B(&done);
2523      __ Bind(&success);
2524      __ Mov(out, 1);
2525      if (zero.IsLinked()) {
2526        __ B(&done);
2527      }
2528      break;
2529    }
2530    case TypeCheckKind::kArrayObjectCheck: {
2531      // Do an exact check.
2532      vixl::Label exact_check;
2533      __ Cmp(out, cls);
2534      __ B(eq, &exact_check);
2535      // Otherwise, we need to check that the object's class is a non primitive array.
2536      __ Ldr(out, HeapOperand(out, component_offset));
2537      GetAssembler()->MaybeUnpoisonHeapReference(out);
2538      // If `out` is null, we use it for the result, and jump to `done`.
2539      __ Cbz(out, &done);
2540      __ Ldrh(out, HeapOperand(out, primitive_offset));
2541      static_assert(Primitive::kPrimNot == 0, "Expected 0 for kPrimNot");
2542      __ Cbnz(out, &zero);
2543      __ Bind(&exact_check);
2544      __ Mov(out, 1);
2545      __ B(&done);
2546      break;
2547    }
2548    case TypeCheckKind::kArrayCheck: {
2549      __ Cmp(out, cls);
2550      DCHECK(locations->OnlyCallsOnSlowPath());
2551      slow_path = new (GetGraph()->GetArena()) TypeCheckSlowPathARM64(
2552          instruction, /* is_fatal */ false);
2553      codegen_->AddSlowPath(slow_path);
2554      __ B(ne, slow_path->GetEntryLabel());
2555      __ Mov(out, 1);
2556      if (zero.IsLinked()) {
2557        __ B(&done);
2558      }
2559      break;
2560    }
2561    case TypeCheckKind::kUnresolvedCheck:
2562    case TypeCheckKind::kInterfaceCheck:
2563    default: {
2564      codegen_->InvokeRuntime(QUICK_ENTRY_POINT(pInstanceofNonTrivial),
2565                              instruction,
2566                              instruction->GetDexPc(),
2567                              nullptr);
2568      if (zero.IsLinked()) {
2569        __ B(&done);
2570      }
2571      break;
2572    }
2573  }
2574
2575  if (zero.IsLinked()) {
2576    __ Bind(&zero);
2577    __ Mov(out, 0);
2578  }
2579
2580  if (done.IsLinked()) {
2581    __ Bind(&done);
2582  }
2583
2584  if (slow_path != nullptr) {
2585    __ Bind(slow_path->GetExitLabel());
2586  }
2587}
2588
2589void LocationsBuilderARM64::VisitCheckCast(HCheckCast* instruction) {
2590  LocationSummary::CallKind call_kind = LocationSummary::kNoCall;
2591  bool throws_into_catch = instruction->CanThrowIntoCatchBlock();
2592
2593  switch (instruction->GetTypeCheckKind()) {
2594    case TypeCheckKind::kExactCheck:
2595    case TypeCheckKind::kAbstractClassCheck:
2596    case TypeCheckKind::kClassHierarchyCheck:
2597    case TypeCheckKind::kArrayObjectCheck:
2598      call_kind = throws_into_catch
2599          ? LocationSummary::kCallOnSlowPath
2600          : LocationSummary::kNoCall;
2601      break;
2602    case TypeCheckKind::kUnresolvedCheck:
2603    case TypeCheckKind::kInterfaceCheck:
2604      call_kind = LocationSummary::kCall;
2605      break;
2606    case TypeCheckKind::kArrayCheck:
2607      call_kind = LocationSummary::kCallOnSlowPath;
2608      break;
2609  }
2610
2611  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(
2612      instruction, call_kind);
2613  if (call_kind != LocationSummary::kCall) {
2614    locations->SetInAt(0, Location::RequiresRegister());
2615    locations->SetInAt(1, Location::RequiresRegister());
2616    // Note that TypeCheckSlowPathARM64 uses this register too.
2617    locations->AddTemp(Location::RequiresRegister());
2618  } else {
2619    InvokeRuntimeCallingConvention calling_convention;
2620    locations->SetInAt(1, LocationFrom(calling_convention.GetRegisterAt(0)));
2621    locations->SetInAt(0, LocationFrom(calling_convention.GetRegisterAt(1)));
2622  }
2623}
2624
2625void InstructionCodeGeneratorARM64::VisitCheckCast(HCheckCast* instruction) {
2626  LocationSummary* locations = instruction->GetLocations();
2627  Register obj = InputRegisterAt(instruction, 0);
2628  Register cls = InputRegisterAt(instruction, 1);
2629  Register temp;
2630  if (!locations->WillCall()) {
2631    temp = WRegisterFrom(instruction->GetLocations()->GetTemp(0));
2632  }
2633
2634  uint32_t class_offset = mirror::Object::ClassOffset().Int32Value();
2635  uint32_t super_offset = mirror::Class::SuperClassOffset().Int32Value();
2636  uint32_t component_offset = mirror::Class::ComponentTypeOffset().Int32Value();
2637  uint32_t primitive_offset = mirror::Class::PrimitiveTypeOffset().Int32Value();
2638  SlowPathCodeARM64* slow_path = nullptr;
2639
2640  if (!locations->WillCall()) {
2641    slow_path = new (GetGraph()->GetArena()) TypeCheckSlowPathARM64(
2642        instruction, !locations->CanCall());
2643    codegen_->AddSlowPath(slow_path);
2644  }
2645
2646  vixl::Label done;
2647  // Avoid null check if we know obj is not null.
2648  if (instruction->MustDoNullCheck()) {
2649    __ Cbz(obj, &done);
2650  }
2651
2652  if (locations->WillCall()) {
2653    __ Ldr(obj, HeapOperand(obj, class_offset));
2654    GetAssembler()->MaybeUnpoisonHeapReference(obj);
2655  } else {
2656    __ Ldr(temp, HeapOperand(obj, class_offset));
2657    GetAssembler()->MaybeUnpoisonHeapReference(temp);
2658  }
2659
2660  switch (instruction->GetTypeCheckKind()) {
2661    case TypeCheckKind::kExactCheck:
2662    case TypeCheckKind::kArrayCheck: {
2663      __ Cmp(temp, cls);
2664      // Jump to slow path for throwing the exception or doing a
2665      // more involved array check.
2666      __ B(ne, slow_path->GetEntryLabel());
2667      break;
2668    }
2669    case TypeCheckKind::kAbstractClassCheck: {
2670      // If the class is abstract, we eagerly fetch the super class of the
2671      // object to avoid doing a comparison we know will fail.
2672      vixl::Label loop;
2673      __ Bind(&loop);
2674      __ Ldr(temp, HeapOperand(temp, super_offset));
2675      GetAssembler()->MaybeUnpoisonHeapReference(temp);
2676      // Jump to the slow path to throw the exception.
2677      __ Cbz(temp, slow_path->GetEntryLabel());
2678      __ Cmp(temp, cls);
2679      __ B(ne, &loop);
2680      break;
2681    }
2682    case TypeCheckKind::kClassHierarchyCheck: {
2683      // Walk over the class hierarchy to find a match.
2684      vixl::Label loop;
2685      __ Bind(&loop);
2686      __ Cmp(temp, cls);
2687      __ B(eq, &done);
2688      __ Ldr(temp, HeapOperand(temp, super_offset));
2689      GetAssembler()->MaybeUnpoisonHeapReference(temp);
2690      __ Cbnz(temp, &loop);
2691      // Jump to the slow path to throw the exception.
2692      __ B(slow_path->GetEntryLabel());
2693      break;
2694    }
2695    case TypeCheckKind::kArrayObjectCheck: {
2696      // Do an exact check.
2697      __ Cmp(temp, cls);
2698      __ B(eq, &done);
2699      // Otherwise, we need to check that the object's class is a non primitive array.
2700      __ Ldr(temp, HeapOperand(temp, component_offset));
2701      GetAssembler()->MaybeUnpoisonHeapReference(temp);
2702      __ Cbz(temp, slow_path->GetEntryLabel());
2703      __ Ldrh(temp, HeapOperand(temp, primitive_offset));
2704      static_assert(Primitive::kPrimNot == 0, "Expected 0 for kPrimNot");
2705      __ Cbnz(temp, slow_path->GetEntryLabel());
2706      break;
2707    }
2708    case TypeCheckKind::kUnresolvedCheck:
2709    case TypeCheckKind::kInterfaceCheck:
2710    default:
2711      codegen_->InvokeRuntime(QUICK_ENTRY_POINT(pCheckCast),
2712                              instruction,
2713                              instruction->GetDexPc(),
2714                              nullptr);
2715      break;
2716  }
2717  __ Bind(&done);
2718
2719  if (slow_path != nullptr) {
2720    __ Bind(slow_path->GetExitLabel());
2721  }
2722}
2723
2724void LocationsBuilderARM64::VisitIntConstant(HIntConstant* constant) {
2725  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(constant);
2726  locations->SetOut(Location::ConstantLocation(constant));
2727}
2728
2729void InstructionCodeGeneratorARM64::VisitIntConstant(HIntConstant* constant ATTRIBUTE_UNUSED) {
2730  // Will be generated at use site.
2731}
2732
2733void LocationsBuilderARM64::VisitNullConstant(HNullConstant* constant) {
2734  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(constant);
2735  locations->SetOut(Location::ConstantLocation(constant));
2736}
2737
2738void InstructionCodeGeneratorARM64::VisitNullConstant(HNullConstant* constant ATTRIBUTE_UNUSED) {
2739  // Will be generated at use site.
2740}
2741
2742void LocationsBuilderARM64::VisitInvokeUnresolved(HInvokeUnresolved* invoke) {
2743  // The trampoline uses the same calling convention as dex calling conventions,
2744  // except instead of loading arg0/r0 with the target Method*, arg0/r0 will contain
2745  // the method_idx.
2746  HandleInvoke(invoke);
2747}
2748
2749void InstructionCodeGeneratorARM64::VisitInvokeUnresolved(HInvokeUnresolved* invoke) {
2750  codegen_->GenerateInvokeUnresolvedRuntimeCall(invoke);
2751}
2752
2753void LocationsBuilderARM64::HandleInvoke(HInvoke* invoke) {
2754  InvokeDexCallingConventionVisitorARM64 calling_convention_visitor;
2755  CodeGenerator::CreateCommonInvokeLocationSummary(invoke, &calling_convention_visitor);
2756}
2757
2758void LocationsBuilderARM64::VisitInvokeInterface(HInvokeInterface* invoke) {
2759  HandleInvoke(invoke);
2760}
2761
2762void InstructionCodeGeneratorARM64::VisitInvokeInterface(HInvokeInterface* invoke) {
2763  // TODO: b/18116999, our IMTs can miss an IncompatibleClassChangeError.
2764  Register temp = XRegisterFrom(invoke->GetLocations()->GetTemp(0));
2765  uint32_t method_offset = mirror::Class::EmbeddedImTableEntryOffset(
2766      invoke->GetImtIndex() % mirror::Class::kImtSize, kArm64PointerSize).Uint32Value();
2767  Location receiver = invoke->GetLocations()->InAt(0);
2768  Offset class_offset = mirror::Object::ClassOffset();
2769  Offset entry_point = ArtMethod::EntryPointFromQuickCompiledCodeOffset(kArm64WordSize);
2770
2771  // The register ip1 is required to be used for the hidden argument in
2772  // art_quick_imt_conflict_trampoline, so prevent VIXL from using it.
2773  MacroAssembler* masm = GetVIXLAssembler();
2774  UseScratchRegisterScope scratch_scope(masm);
2775  BlockPoolsScope block_pools(masm);
2776  scratch_scope.Exclude(ip1);
2777  __ Mov(ip1, invoke->GetDexMethodIndex());
2778
2779  // temp = object->GetClass();
2780  if (receiver.IsStackSlot()) {
2781    __ Ldr(temp.W(), StackOperandFrom(receiver));
2782    __ Ldr(temp.W(), HeapOperand(temp.W(), class_offset));
2783  } else {
2784    __ Ldr(temp.W(), HeapOperandFrom(receiver, class_offset));
2785  }
2786  codegen_->MaybeRecordImplicitNullCheck(invoke);
2787  GetAssembler()->MaybeUnpoisonHeapReference(temp.W());
2788  // temp = temp->GetImtEntryAt(method_offset);
2789  __ Ldr(temp, MemOperand(temp, method_offset));
2790  // lr = temp->GetEntryPoint();
2791  __ Ldr(lr, MemOperand(temp, entry_point.Int32Value()));
2792  // lr();
2793  __ Blr(lr);
2794  DCHECK(!codegen_->IsLeafMethod());
2795  codegen_->RecordPcInfo(invoke, invoke->GetDexPc());
2796}
2797
2798void LocationsBuilderARM64::VisitInvokeVirtual(HInvokeVirtual* invoke) {
2799  IntrinsicLocationsBuilderARM64 intrinsic(GetGraph()->GetArena());
2800  if (intrinsic.TryDispatch(invoke)) {
2801    return;
2802  }
2803
2804  HandleInvoke(invoke);
2805}
2806
2807void LocationsBuilderARM64::VisitInvokeStaticOrDirect(HInvokeStaticOrDirect* invoke) {
2808  // When we do not run baseline, explicit clinit checks triggered by static
2809  // invokes must have been pruned by art::PrepareForRegisterAllocation.
2810  DCHECK(codegen_->IsBaseline() || !invoke->IsStaticWithExplicitClinitCheck());
2811
2812  IntrinsicLocationsBuilderARM64 intrinsic(GetGraph()->GetArena());
2813  if (intrinsic.TryDispatch(invoke)) {
2814    return;
2815  }
2816
2817  HandleInvoke(invoke);
2818}
2819
2820static bool TryGenerateIntrinsicCode(HInvoke* invoke, CodeGeneratorARM64* codegen) {
2821  if (invoke->GetLocations()->Intrinsified()) {
2822    IntrinsicCodeGeneratorARM64 intrinsic(codegen);
2823    intrinsic.Dispatch(invoke);
2824    return true;
2825  }
2826  return false;
2827}
2828
2829HInvokeStaticOrDirect::DispatchInfo CodeGeneratorARM64::GetSupportedInvokeStaticOrDirectDispatch(
2830      const HInvokeStaticOrDirect::DispatchInfo& desired_dispatch_info,
2831      MethodReference target_method ATTRIBUTE_UNUSED) {
2832  // On arm64 we support all dispatch types.
2833  return desired_dispatch_info;
2834}
2835
2836void CodeGeneratorARM64::GenerateStaticOrDirectCall(HInvokeStaticOrDirect* invoke, Location temp) {
2837  // For better instruction scheduling we load the direct code pointer before the method pointer.
2838  bool direct_code_loaded = false;
2839  switch (invoke->GetCodePtrLocation()) {
2840    case HInvokeStaticOrDirect::CodePtrLocation::kCallDirectWithFixup:
2841      // LR = code address from literal pool with link-time patch.
2842      __ Ldr(lr, DeduplicateMethodCodeLiteral(invoke->GetTargetMethod()));
2843      direct_code_loaded = true;
2844      break;
2845    case HInvokeStaticOrDirect::CodePtrLocation::kCallDirect:
2846      // LR = invoke->GetDirectCodePtr();
2847      __ Ldr(lr, DeduplicateUint64Literal(invoke->GetDirectCodePtr()));
2848      direct_code_loaded = true;
2849      break;
2850    default:
2851      break;
2852  }
2853
2854  // Make sure that ArtMethod* is passed in kArtMethodRegister as per the calling convention.
2855  Location callee_method = temp;  // For all kinds except kRecursive, callee will be in temp.
2856  switch (invoke->GetMethodLoadKind()) {
2857    case HInvokeStaticOrDirect::MethodLoadKind::kStringInit:
2858      // temp = thread->string_init_entrypoint
2859      __ Ldr(XRegisterFrom(temp), MemOperand(tr, invoke->GetStringInitOffset()));
2860      break;
2861    case HInvokeStaticOrDirect::MethodLoadKind::kRecursive:
2862      callee_method = invoke->GetLocations()->InAt(invoke->GetCurrentMethodInputIndex());
2863      break;
2864    case HInvokeStaticOrDirect::MethodLoadKind::kDirectAddress:
2865      // Load method address from literal pool.
2866      __ Ldr(XRegisterFrom(temp), DeduplicateUint64Literal(invoke->GetMethodAddress()));
2867      break;
2868    case HInvokeStaticOrDirect::MethodLoadKind::kDirectAddressWithFixup:
2869      // Load method address from literal pool with a link-time patch.
2870      __ Ldr(XRegisterFrom(temp),
2871             DeduplicateMethodAddressLiteral(invoke->GetTargetMethod()));
2872      break;
2873    case HInvokeStaticOrDirect::MethodLoadKind::kDexCachePcRelative: {
2874      // Add ADRP with its PC-relative DexCache access patch.
2875      pc_relative_dex_cache_patches_.emplace_back(*invoke->GetTargetMethod().dex_file,
2876                                                  invoke->GetDexCacheArrayOffset());
2877      vixl::Label* pc_insn_label = &pc_relative_dex_cache_patches_.back().label;
2878      {
2879        vixl::SingleEmissionCheckScope guard(GetVIXLAssembler());
2880        __ Bind(pc_insn_label);
2881        __ adrp(XRegisterFrom(temp), 0);
2882      }
2883      pc_relative_dex_cache_patches_.back().pc_insn_label = pc_insn_label;
2884      // Add LDR with its PC-relative DexCache access patch.
2885      pc_relative_dex_cache_patches_.emplace_back(*invoke->GetTargetMethod().dex_file,
2886                                                  invoke->GetDexCacheArrayOffset());
2887      {
2888        vixl::SingleEmissionCheckScope guard(GetVIXLAssembler());
2889        __ Bind(&pc_relative_dex_cache_patches_.back().label);
2890        __ ldr(XRegisterFrom(temp), MemOperand(XRegisterFrom(temp), 0));
2891        pc_relative_dex_cache_patches_.back().pc_insn_label = pc_insn_label;
2892      }
2893      break;
2894    }
2895    case HInvokeStaticOrDirect::MethodLoadKind::kDexCacheViaMethod: {
2896      Location current_method = invoke->GetLocations()->InAt(invoke->GetCurrentMethodInputIndex());
2897      Register reg = XRegisterFrom(temp);
2898      Register method_reg;
2899      if (current_method.IsRegister()) {
2900        method_reg = XRegisterFrom(current_method);
2901      } else {
2902        DCHECK(invoke->GetLocations()->Intrinsified());
2903        DCHECK(!current_method.IsValid());
2904        method_reg = reg;
2905        __ Ldr(reg.X(), MemOperand(sp, kCurrentMethodStackOffset));
2906      }
2907
2908      // temp = current_method->dex_cache_resolved_methods_;
2909      __ Ldr(reg.X(),
2910             MemOperand(method_reg.X(),
2911                        ArtMethod::DexCacheResolvedMethodsOffset(kArm64WordSize).Int32Value()));
2912      // temp = temp[index_in_cache];
2913      uint32_t index_in_cache = invoke->GetTargetMethod().dex_method_index;
2914    __ Ldr(reg.X(), MemOperand(reg.X(), GetCachePointerOffset(index_in_cache)));
2915      break;
2916    }
2917  }
2918
2919  switch (invoke->GetCodePtrLocation()) {
2920    case HInvokeStaticOrDirect::CodePtrLocation::kCallSelf:
2921      __ Bl(&frame_entry_label_);
2922      break;
2923    case HInvokeStaticOrDirect::CodePtrLocation::kCallPCRelative: {
2924      relative_call_patches_.emplace_back(invoke->GetTargetMethod());
2925      vixl::Label* label = &relative_call_patches_.back().label;
2926      vixl::SingleEmissionCheckScope guard(GetVIXLAssembler());
2927      __ Bind(label);
2928      __ bl(0);  // Branch and link to itself. This will be overriden at link time.
2929      break;
2930    }
2931    case HInvokeStaticOrDirect::CodePtrLocation::kCallDirectWithFixup:
2932    case HInvokeStaticOrDirect::CodePtrLocation::kCallDirect:
2933      // LR prepared above for better instruction scheduling.
2934      DCHECK(direct_code_loaded);
2935      // lr()
2936      __ Blr(lr);
2937      break;
2938    case HInvokeStaticOrDirect::CodePtrLocation::kCallArtMethod:
2939      // LR = callee_method->entry_point_from_quick_compiled_code_;
2940      __ Ldr(lr, MemOperand(
2941          XRegisterFrom(callee_method),
2942          ArtMethod::EntryPointFromQuickCompiledCodeOffset(kArm64WordSize).Int32Value()));
2943      // lr()
2944      __ Blr(lr);
2945      break;
2946  }
2947
2948  DCHECK(!IsLeafMethod());
2949}
2950
2951void CodeGeneratorARM64::GenerateVirtualCall(HInvokeVirtual* invoke, Location temp_in) {
2952  LocationSummary* locations = invoke->GetLocations();
2953  Location receiver = locations->InAt(0);
2954  Register temp = XRegisterFrom(temp_in);
2955  size_t method_offset = mirror::Class::EmbeddedVTableEntryOffset(
2956      invoke->GetVTableIndex(), kArm64PointerSize).SizeValue();
2957  Offset class_offset = mirror::Object::ClassOffset();
2958  Offset entry_point = ArtMethod::EntryPointFromQuickCompiledCodeOffset(kArm64WordSize);
2959
2960  BlockPoolsScope block_pools(GetVIXLAssembler());
2961
2962  DCHECK(receiver.IsRegister());
2963  __ Ldr(temp.W(), HeapOperandFrom(receiver, class_offset));
2964  MaybeRecordImplicitNullCheck(invoke);
2965  GetAssembler()->MaybeUnpoisonHeapReference(temp.W());
2966  // temp = temp->GetMethodAt(method_offset);
2967  __ Ldr(temp, MemOperand(temp, method_offset));
2968  // lr = temp->GetEntryPoint();
2969  __ Ldr(lr, MemOperand(temp, entry_point.SizeValue()));
2970  // lr();
2971  __ Blr(lr);
2972}
2973
2974void CodeGeneratorARM64::EmitLinkerPatches(ArenaVector<LinkerPatch>* linker_patches) {
2975  DCHECK(linker_patches->empty());
2976  size_t size =
2977      method_patches_.size() +
2978      call_patches_.size() +
2979      relative_call_patches_.size() +
2980      pc_relative_dex_cache_patches_.size();
2981  linker_patches->reserve(size);
2982  for (const auto& entry : method_patches_) {
2983    const MethodReference& target_method = entry.first;
2984    vixl::Literal<uint64_t>* literal = entry.second;
2985    linker_patches->push_back(LinkerPatch::MethodPatch(literal->offset(),
2986                                                       target_method.dex_file,
2987                                                       target_method.dex_method_index));
2988  }
2989  for (const auto& entry : call_patches_) {
2990    const MethodReference& target_method = entry.first;
2991    vixl::Literal<uint64_t>* literal = entry.second;
2992    linker_patches->push_back(LinkerPatch::CodePatch(literal->offset(),
2993                                                     target_method.dex_file,
2994                                                     target_method.dex_method_index));
2995  }
2996  for (const MethodPatchInfo<vixl::Label>& info : relative_call_patches_) {
2997    linker_patches->push_back(LinkerPatch::RelativeCodePatch(info.label.location(),
2998                                                             info.target_method.dex_file,
2999                                                             info.target_method.dex_method_index));
3000  }
3001  for (const PcRelativeDexCacheAccessInfo& info : pc_relative_dex_cache_patches_) {
3002    linker_patches->push_back(LinkerPatch::DexCacheArrayPatch(info.label.location(),
3003                                                              &info.target_dex_file,
3004                                                              info.pc_insn_label->location(),
3005                                                              info.element_offset));
3006  }
3007}
3008
3009vixl::Literal<uint64_t>* CodeGeneratorARM64::DeduplicateUint64Literal(uint64_t value) {
3010  // Look up the literal for value.
3011  auto lb = uint64_literals_.lower_bound(value);
3012  if (lb != uint64_literals_.end() && !uint64_literals_.key_comp()(value, lb->first)) {
3013    return lb->second;
3014  }
3015  // We don't have a literal for this value, insert a new one.
3016  vixl::Literal<uint64_t>* literal = __ CreateLiteralDestroyedWithPool<uint64_t>(value);
3017  uint64_literals_.PutBefore(lb, value, literal);
3018  return literal;
3019}
3020
3021vixl::Literal<uint64_t>* CodeGeneratorARM64::DeduplicateMethodLiteral(
3022    MethodReference target_method,
3023    MethodToLiteralMap* map) {
3024  // Look up the literal for target_method.
3025  auto lb = map->lower_bound(target_method);
3026  if (lb != map->end() && !map->key_comp()(target_method, lb->first)) {
3027    return lb->second;
3028  }
3029  // We don't have a literal for this method yet, insert a new one.
3030  vixl::Literal<uint64_t>* literal = __ CreateLiteralDestroyedWithPool<uint64_t>(0u);
3031  map->PutBefore(lb, target_method, literal);
3032  return literal;
3033}
3034
3035vixl::Literal<uint64_t>* CodeGeneratorARM64::DeduplicateMethodAddressLiteral(
3036    MethodReference target_method) {
3037  return DeduplicateMethodLiteral(target_method, &method_patches_);
3038}
3039
3040vixl::Literal<uint64_t>* CodeGeneratorARM64::DeduplicateMethodCodeLiteral(
3041    MethodReference target_method) {
3042  return DeduplicateMethodLiteral(target_method, &call_patches_);
3043}
3044
3045
3046void InstructionCodeGeneratorARM64::VisitInvokeStaticOrDirect(HInvokeStaticOrDirect* invoke) {
3047  // When we do not run baseline, explicit clinit checks triggered by static
3048  // invokes must have been pruned by art::PrepareForRegisterAllocation.
3049  DCHECK(codegen_->IsBaseline() || !invoke->IsStaticWithExplicitClinitCheck());
3050
3051  if (TryGenerateIntrinsicCode(invoke, codegen_)) {
3052    return;
3053  }
3054
3055  BlockPoolsScope block_pools(GetVIXLAssembler());
3056  LocationSummary* locations = invoke->GetLocations();
3057  codegen_->GenerateStaticOrDirectCall(
3058      invoke, locations->HasTemps() ? locations->GetTemp(0) : Location::NoLocation());
3059  codegen_->RecordPcInfo(invoke, invoke->GetDexPc());
3060}
3061
3062void InstructionCodeGeneratorARM64::VisitInvokeVirtual(HInvokeVirtual* invoke) {
3063  if (TryGenerateIntrinsicCode(invoke, codegen_)) {
3064    return;
3065  }
3066
3067  codegen_->GenerateVirtualCall(invoke, invoke->GetLocations()->GetTemp(0));
3068  DCHECK(!codegen_->IsLeafMethod());
3069  codegen_->RecordPcInfo(invoke, invoke->GetDexPc());
3070}
3071
3072void LocationsBuilderARM64::VisitLoadClass(HLoadClass* cls) {
3073  InvokeRuntimeCallingConvention calling_convention;
3074  CodeGenerator::CreateLoadClassLocationSummary(
3075      cls,
3076      LocationFrom(calling_convention.GetRegisterAt(0)),
3077      LocationFrom(vixl::x0));
3078}
3079
3080void InstructionCodeGeneratorARM64::VisitLoadClass(HLoadClass* cls) {
3081  if (cls->NeedsAccessCheck()) {
3082    codegen_->MoveConstant(cls->GetLocations()->GetTemp(0), cls->GetTypeIndex());
3083    codegen_->InvokeRuntime(QUICK_ENTRY_POINT(pInitializeTypeAndVerifyAccess),
3084                            cls,
3085                            cls->GetDexPc(),
3086                            nullptr);
3087    return;
3088  }
3089
3090  Register out = OutputRegister(cls);
3091  Register current_method = InputRegisterAt(cls, 0);
3092  if (cls->IsReferrersClass()) {
3093    DCHECK(!cls->CanCallRuntime());
3094    DCHECK(!cls->MustGenerateClinitCheck());
3095    __ Ldr(out, MemOperand(current_method, ArtMethod::DeclaringClassOffset().Int32Value()));
3096  } else {
3097    DCHECK(cls->CanCallRuntime());
3098    MemberOffset resolved_types_offset = ArtMethod::DexCacheResolvedTypesOffset(kArm64PointerSize);
3099    __ Ldr(out.X(), MemOperand(current_method, resolved_types_offset.Int32Value()));
3100    __ Ldr(out, MemOperand(out.X(), CodeGenerator::GetCacheOffset(cls->GetTypeIndex())));
3101    // TODO: We will need a read barrier here.
3102
3103    SlowPathCodeARM64* slow_path = new (GetGraph()->GetArena()) LoadClassSlowPathARM64(
3104        cls, cls, cls->GetDexPc(), cls->MustGenerateClinitCheck());
3105    codegen_->AddSlowPath(slow_path);
3106    __ Cbz(out, slow_path->GetEntryLabel());
3107    if (cls->MustGenerateClinitCheck()) {
3108      GenerateClassInitializationCheck(slow_path, out);
3109    } else {
3110      __ Bind(slow_path->GetExitLabel());
3111    }
3112  }
3113}
3114
3115static MemOperand GetExceptionTlsAddress() {
3116  return MemOperand(tr, Thread::ExceptionOffset<kArm64WordSize>().Int32Value());
3117}
3118
3119void LocationsBuilderARM64::VisitLoadException(HLoadException* load) {
3120  LocationSummary* locations =
3121      new (GetGraph()->GetArena()) LocationSummary(load, LocationSummary::kNoCall);
3122  locations->SetOut(Location::RequiresRegister());
3123}
3124
3125void InstructionCodeGeneratorARM64::VisitLoadException(HLoadException* instruction) {
3126  __ Ldr(OutputRegister(instruction), GetExceptionTlsAddress());
3127}
3128
3129void LocationsBuilderARM64::VisitClearException(HClearException* clear) {
3130  new (GetGraph()->GetArena()) LocationSummary(clear, LocationSummary::kNoCall);
3131}
3132
3133void InstructionCodeGeneratorARM64::VisitClearException(HClearException* clear ATTRIBUTE_UNUSED) {
3134  __ Str(wzr, GetExceptionTlsAddress());
3135}
3136
3137void LocationsBuilderARM64::VisitLoadLocal(HLoadLocal* load) {
3138  load->SetLocations(nullptr);
3139}
3140
3141void InstructionCodeGeneratorARM64::VisitLoadLocal(HLoadLocal* load ATTRIBUTE_UNUSED) {
3142  // Nothing to do, this is driven by the code generator.
3143}
3144
3145void LocationsBuilderARM64::VisitLoadString(HLoadString* load) {
3146  LocationSummary* locations =
3147      new (GetGraph()->GetArena()) LocationSummary(load, LocationSummary::kCallOnSlowPath);
3148  locations->SetInAt(0, Location::RequiresRegister());
3149  locations->SetOut(Location::RequiresRegister());
3150}
3151
3152void InstructionCodeGeneratorARM64::VisitLoadString(HLoadString* load) {
3153  SlowPathCodeARM64* slow_path = new (GetGraph()->GetArena()) LoadStringSlowPathARM64(load);
3154  codegen_->AddSlowPath(slow_path);
3155
3156  Register out = OutputRegister(load);
3157  Register current_method = InputRegisterAt(load, 0);
3158  __ Ldr(out, MemOperand(current_method, ArtMethod::DeclaringClassOffset().Int32Value()));
3159  __ Ldr(out.X(), HeapOperand(out, mirror::Class::DexCacheStringsOffset()));
3160  __ Ldr(out, MemOperand(out.X(), CodeGenerator::GetCacheOffset(load->GetStringIndex())));
3161  // TODO: We will need a read barrier here.
3162  __ Cbz(out, slow_path->GetEntryLabel());
3163  __ Bind(slow_path->GetExitLabel());
3164}
3165
3166void LocationsBuilderARM64::VisitLocal(HLocal* local) {
3167  local->SetLocations(nullptr);
3168}
3169
3170void InstructionCodeGeneratorARM64::VisitLocal(HLocal* local) {
3171  DCHECK_EQ(local->GetBlock(), GetGraph()->GetEntryBlock());
3172}
3173
3174void LocationsBuilderARM64::VisitLongConstant(HLongConstant* constant) {
3175  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(constant);
3176  locations->SetOut(Location::ConstantLocation(constant));
3177}
3178
3179void InstructionCodeGeneratorARM64::VisitLongConstant(HLongConstant* constant ATTRIBUTE_UNUSED) {
3180  // Will be generated at use site.
3181}
3182
3183void LocationsBuilderARM64::VisitMonitorOperation(HMonitorOperation* instruction) {
3184  LocationSummary* locations =
3185      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCall);
3186  InvokeRuntimeCallingConvention calling_convention;
3187  locations->SetInAt(0, LocationFrom(calling_convention.GetRegisterAt(0)));
3188}
3189
3190void InstructionCodeGeneratorARM64::VisitMonitorOperation(HMonitorOperation* instruction) {
3191  codegen_->InvokeRuntime(instruction->IsEnter()
3192        ? QUICK_ENTRY_POINT(pLockObject) : QUICK_ENTRY_POINT(pUnlockObject),
3193      instruction,
3194      instruction->GetDexPc(),
3195      nullptr);
3196  CheckEntrypointTypes<kQuickLockObject, void, mirror::Object*>();
3197}
3198
3199void LocationsBuilderARM64::VisitMul(HMul* mul) {
3200  LocationSummary* locations =
3201      new (GetGraph()->GetArena()) LocationSummary(mul, LocationSummary::kNoCall);
3202  switch (mul->GetResultType()) {
3203    case Primitive::kPrimInt:
3204    case Primitive::kPrimLong:
3205      locations->SetInAt(0, Location::RequiresRegister());
3206      locations->SetInAt(1, Location::RequiresRegister());
3207      locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
3208      break;
3209
3210    case Primitive::kPrimFloat:
3211    case Primitive::kPrimDouble:
3212      locations->SetInAt(0, Location::RequiresFpuRegister());
3213      locations->SetInAt(1, Location::RequiresFpuRegister());
3214      locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap);
3215      break;
3216
3217    default:
3218      LOG(FATAL) << "Unexpected mul type " << mul->GetResultType();
3219  }
3220}
3221
3222void InstructionCodeGeneratorARM64::VisitMul(HMul* mul) {
3223  switch (mul->GetResultType()) {
3224    case Primitive::kPrimInt:
3225    case Primitive::kPrimLong:
3226      __ Mul(OutputRegister(mul), InputRegisterAt(mul, 0), InputRegisterAt(mul, 1));
3227      break;
3228
3229    case Primitive::kPrimFloat:
3230    case Primitive::kPrimDouble:
3231      __ Fmul(OutputFPRegister(mul), InputFPRegisterAt(mul, 0), InputFPRegisterAt(mul, 1));
3232      break;
3233
3234    default:
3235      LOG(FATAL) << "Unexpected mul type " << mul->GetResultType();
3236  }
3237}
3238
3239void LocationsBuilderARM64::VisitNeg(HNeg* neg) {
3240  LocationSummary* locations =
3241      new (GetGraph()->GetArena()) LocationSummary(neg, LocationSummary::kNoCall);
3242  switch (neg->GetResultType()) {
3243    case Primitive::kPrimInt:
3244    case Primitive::kPrimLong:
3245      locations->SetInAt(0, ARM64EncodableConstantOrRegister(neg->InputAt(0), neg));
3246      locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
3247      break;
3248
3249    case Primitive::kPrimFloat:
3250    case Primitive::kPrimDouble:
3251      locations->SetInAt(0, Location::RequiresFpuRegister());
3252      locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap);
3253      break;
3254
3255    default:
3256      LOG(FATAL) << "Unexpected neg type " << neg->GetResultType();
3257  }
3258}
3259
3260void InstructionCodeGeneratorARM64::VisitNeg(HNeg* neg) {
3261  switch (neg->GetResultType()) {
3262    case Primitive::kPrimInt:
3263    case Primitive::kPrimLong:
3264      __ Neg(OutputRegister(neg), InputOperandAt(neg, 0));
3265      break;
3266
3267    case Primitive::kPrimFloat:
3268    case Primitive::kPrimDouble:
3269      __ Fneg(OutputFPRegister(neg), InputFPRegisterAt(neg, 0));
3270      break;
3271
3272    default:
3273      LOG(FATAL) << "Unexpected neg type " << neg->GetResultType();
3274  }
3275}
3276
3277void LocationsBuilderARM64::VisitNewArray(HNewArray* instruction) {
3278  LocationSummary* locations =
3279      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCall);
3280  InvokeRuntimeCallingConvention calling_convention;
3281  locations->AddTemp(LocationFrom(calling_convention.GetRegisterAt(0)));
3282  locations->SetOut(LocationFrom(x0));
3283  locations->SetInAt(0, LocationFrom(calling_convention.GetRegisterAt(1)));
3284  locations->SetInAt(1, LocationFrom(calling_convention.GetRegisterAt(2)));
3285  CheckEntrypointTypes<kQuickAllocArrayWithAccessCheck,
3286                       void*, uint32_t, int32_t, ArtMethod*>();
3287}
3288
3289void InstructionCodeGeneratorARM64::VisitNewArray(HNewArray* instruction) {
3290  LocationSummary* locations = instruction->GetLocations();
3291  InvokeRuntimeCallingConvention calling_convention;
3292  Register type_index = RegisterFrom(locations->GetTemp(0), Primitive::kPrimInt);
3293  DCHECK(type_index.Is(w0));
3294  __ Mov(type_index, instruction->GetTypeIndex());
3295  // Note: if heap poisoning is enabled, the entry point takes cares
3296  // of poisoning the reference.
3297  codegen_->InvokeRuntime(instruction->GetEntrypoint(),
3298                          instruction,
3299                          instruction->GetDexPc(),
3300                          nullptr);
3301  CheckEntrypointTypes<kQuickAllocArrayWithAccessCheck, void*, uint32_t, int32_t, ArtMethod*>();
3302}
3303
3304void LocationsBuilderARM64::VisitNewInstance(HNewInstance* instruction) {
3305  LocationSummary* locations =
3306      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCall);
3307  InvokeRuntimeCallingConvention calling_convention;
3308  locations->AddTemp(LocationFrom(calling_convention.GetRegisterAt(0)));
3309  locations->SetInAt(0, LocationFrom(calling_convention.GetRegisterAt(1)));
3310  locations->SetOut(calling_convention.GetReturnLocation(Primitive::kPrimNot));
3311  CheckEntrypointTypes<kQuickAllocObjectWithAccessCheck, void*, uint32_t, ArtMethod*>();
3312}
3313
3314void InstructionCodeGeneratorARM64::VisitNewInstance(HNewInstance* instruction) {
3315  LocationSummary* locations = instruction->GetLocations();
3316  Register type_index = RegisterFrom(locations->GetTemp(0), Primitive::kPrimInt);
3317  DCHECK(type_index.Is(w0));
3318  __ Mov(type_index, instruction->GetTypeIndex());
3319  // Note: if heap poisoning is enabled, the entry point takes cares
3320  // of poisoning the reference.
3321  codegen_->InvokeRuntime(instruction->GetEntrypoint(),
3322                          instruction,
3323                          instruction->GetDexPc(),
3324                          nullptr);
3325  CheckEntrypointTypes<kQuickAllocObjectWithAccessCheck, void*, uint32_t, ArtMethod*>();
3326}
3327
3328void LocationsBuilderARM64::VisitNot(HNot* instruction) {
3329  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction);
3330  locations->SetInAt(0, Location::RequiresRegister());
3331  locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
3332}
3333
3334void InstructionCodeGeneratorARM64::VisitNot(HNot* instruction) {
3335  switch (instruction->GetResultType()) {
3336    case Primitive::kPrimInt:
3337    case Primitive::kPrimLong:
3338      __ Mvn(OutputRegister(instruction), InputOperandAt(instruction, 0));
3339      break;
3340
3341    default:
3342      LOG(FATAL) << "Unexpected type for not operation " << instruction->GetResultType();
3343  }
3344}
3345
3346void LocationsBuilderARM64::VisitBooleanNot(HBooleanNot* instruction) {
3347  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction);
3348  locations->SetInAt(0, Location::RequiresRegister());
3349  locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
3350}
3351
3352void InstructionCodeGeneratorARM64::VisitBooleanNot(HBooleanNot* instruction) {
3353  __ Eor(OutputRegister(instruction), InputRegisterAt(instruction, 0), vixl::Operand(1));
3354}
3355
3356void LocationsBuilderARM64::VisitNullCheck(HNullCheck* instruction) {
3357  LocationSummary::CallKind call_kind = instruction->CanThrowIntoCatchBlock()
3358      ? LocationSummary::kCallOnSlowPath
3359      : LocationSummary::kNoCall;
3360  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction, call_kind);
3361  locations->SetInAt(0, Location::RequiresRegister());
3362  if (instruction->HasUses()) {
3363    locations->SetOut(Location::SameAsFirstInput());
3364  }
3365}
3366
3367void InstructionCodeGeneratorARM64::GenerateImplicitNullCheck(HNullCheck* instruction) {
3368  if (codegen_->CanMoveNullCheckToUser(instruction)) {
3369    return;
3370  }
3371
3372  BlockPoolsScope block_pools(GetVIXLAssembler());
3373  Location obj = instruction->GetLocations()->InAt(0);
3374  __ Ldr(wzr, HeapOperandFrom(obj, Offset(0)));
3375  codegen_->RecordPcInfo(instruction, instruction->GetDexPc());
3376}
3377
3378void InstructionCodeGeneratorARM64::GenerateExplicitNullCheck(HNullCheck* instruction) {
3379  SlowPathCodeARM64* slow_path = new (GetGraph()->GetArena()) NullCheckSlowPathARM64(instruction);
3380  codegen_->AddSlowPath(slow_path);
3381
3382  LocationSummary* locations = instruction->GetLocations();
3383  Location obj = locations->InAt(0);
3384
3385  __ Cbz(RegisterFrom(obj, instruction->InputAt(0)->GetType()), slow_path->GetEntryLabel());
3386}
3387
3388void InstructionCodeGeneratorARM64::VisitNullCheck(HNullCheck* instruction) {
3389  if (codegen_->IsImplicitNullCheckAllowed(instruction)) {
3390    GenerateImplicitNullCheck(instruction);
3391  } else {
3392    GenerateExplicitNullCheck(instruction);
3393  }
3394}
3395
3396void LocationsBuilderARM64::VisitOr(HOr* instruction) {
3397  HandleBinaryOp(instruction);
3398}
3399
3400void InstructionCodeGeneratorARM64::VisitOr(HOr* instruction) {
3401  HandleBinaryOp(instruction);
3402}
3403
3404void LocationsBuilderARM64::VisitParallelMove(HParallelMove* instruction ATTRIBUTE_UNUSED) {
3405  LOG(FATAL) << "Unreachable";
3406}
3407
3408void InstructionCodeGeneratorARM64::VisitParallelMove(HParallelMove* instruction) {
3409  codegen_->GetMoveResolver()->EmitNativeCode(instruction);
3410}
3411
3412void LocationsBuilderARM64::VisitParameterValue(HParameterValue* instruction) {
3413  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction);
3414  Location location = parameter_visitor_.GetNextLocation(instruction->GetType());
3415  if (location.IsStackSlot()) {
3416    location = Location::StackSlot(location.GetStackIndex() + codegen_->GetFrameSize());
3417  } else if (location.IsDoubleStackSlot()) {
3418    location = Location::DoubleStackSlot(location.GetStackIndex() + codegen_->GetFrameSize());
3419  }
3420  locations->SetOut(location);
3421}
3422
3423void InstructionCodeGeneratorARM64::VisitParameterValue(
3424    HParameterValue* instruction ATTRIBUTE_UNUSED) {
3425  // Nothing to do, the parameter is already at its location.
3426}
3427
3428void LocationsBuilderARM64::VisitCurrentMethod(HCurrentMethod* instruction) {
3429  LocationSummary* locations =
3430      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
3431  locations->SetOut(LocationFrom(kArtMethodRegister));
3432}
3433
3434void InstructionCodeGeneratorARM64::VisitCurrentMethod(
3435    HCurrentMethod* instruction ATTRIBUTE_UNUSED) {
3436  // Nothing to do, the method is already at its location.
3437}
3438
3439void LocationsBuilderARM64::VisitPhi(HPhi* instruction) {
3440  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction);
3441  for (size_t i = 0, e = instruction->InputCount(); i < e; ++i) {
3442    locations->SetInAt(i, Location::Any());
3443  }
3444  locations->SetOut(Location::Any());
3445}
3446
3447void InstructionCodeGeneratorARM64::VisitPhi(HPhi* instruction ATTRIBUTE_UNUSED) {
3448  LOG(FATAL) << "Unreachable";
3449}
3450
3451void LocationsBuilderARM64::VisitRem(HRem* rem) {
3452  Primitive::Type type = rem->GetResultType();
3453  LocationSummary::CallKind call_kind =
3454      Primitive::IsFloatingPointType(type) ? LocationSummary::kCall : LocationSummary::kNoCall;
3455  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(rem, call_kind);
3456
3457  switch (type) {
3458    case Primitive::kPrimInt:
3459    case Primitive::kPrimLong:
3460      locations->SetInAt(0, Location::RequiresRegister());
3461      locations->SetInAt(1, Location::RegisterOrConstant(rem->InputAt(1)));
3462      locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
3463      break;
3464
3465    case Primitive::kPrimFloat:
3466    case Primitive::kPrimDouble: {
3467      InvokeRuntimeCallingConvention calling_convention;
3468      locations->SetInAt(0, LocationFrom(calling_convention.GetFpuRegisterAt(0)));
3469      locations->SetInAt(1, LocationFrom(calling_convention.GetFpuRegisterAt(1)));
3470      locations->SetOut(calling_convention.GetReturnLocation(type));
3471
3472      break;
3473    }
3474
3475    default:
3476      LOG(FATAL) << "Unexpected rem type " << type;
3477  }
3478}
3479
3480void InstructionCodeGeneratorARM64::VisitRem(HRem* rem) {
3481  Primitive::Type type = rem->GetResultType();
3482
3483  switch (type) {
3484    case Primitive::kPrimInt:
3485    case Primitive::kPrimLong: {
3486      GenerateDivRemIntegral(rem);
3487      break;
3488    }
3489
3490    case Primitive::kPrimFloat:
3491    case Primitive::kPrimDouble: {
3492      int32_t entry_offset = (type == Primitive::kPrimFloat) ? QUICK_ENTRY_POINT(pFmodf)
3493                                                             : QUICK_ENTRY_POINT(pFmod);
3494      codegen_->InvokeRuntime(entry_offset, rem, rem->GetDexPc(), nullptr);
3495      break;
3496    }
3497
3498    default:
3499      LOG(FATAL) << "Unexpected rem type " << type;
3500  }
3501}
3502
3503void LocationsBuilderARM64::VisitMemoryBarrier(HMemoryBarrier* memory_barrier) {
3504  memory_barrier->SetLocations(nullptr);
3505}
3506
3507void InstructionCodeGeneratorARM64::VisitMemoryBarrier(HMemoryBarrier* memory_barrier) {
3508  GenerateMemoryBarrier(memory_barrier->GetBarrierKind());
3509}
3510
3511void LocationsBuilderARM64::VisitReturn(HReturn* instruction) {
3512  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction);
3513  Primitive::Type return_type = instruction->InputAt(0)->GetType();
3514  locations->SetInAt(0, ARM64ReturnLocation(return_type));
3515}
3516
3517void InstructionCodeGeneratorARM64::VisitReturn(HReturn* instruction ATTRIBUTE_UNUSED) {
3518  codegen_->GenerateFrameExit();
3519}
3520
3521void LocationsBuilderARM64::VisitReturnVoid(HReturnVoid* instruction) {
3522  instruction->SetLocations(nullptr);
3523}
3524
3525void InstructionCodeGeneratorARM64::VisitReturnVoid(HReturnVoid* instruction ATTRIBUTE_UNUSED) {
3526  codegen_->GenerateFrameExit();
3527}
3528
3529void LocationsBuilderARM64::VisitShl(HShl* shl) {
3530  HandleShift(shl);
3531}
3532
3533void InstructionCodeGeneratorARM64::VisitShl(HShl* shl) {
3534  HandleShift(shl);
3535}
3536
3537void LocationsBuilderARM64::VisitShr(HShr* shr) {
3538  HandleShift(shr);
3539}
3540
3541void InstructionCodeGeneratorARM64::VisitShr(HShr* shr) {
3542  HandleShift(shr);
3543}
3544
3545void LocationsBuilderARM64::VisitStoreLocal(HStoreLocal* store) {
3546  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(store);
3547  Primitive::Type field_type = store->InputAt(1)->GetType();
3548  switch (field_type) {
3549    case Primitive::kPrimNot:
3550    case Primitive::kPrimBoolean:
3551    case Primitive::kPrimByte:
3552    case Primitive::kPrimChar:
3553    case Primitive::kPrimShort:
3554    case Primitive::kPrimInt:
3555    case Primitive::kPrimFloat:
3556      locations->SetInAt(1, Location::StackSlot(codegen_->GetStackSlot(store->GetLocal())));
3557      break;
3558
3559    case Primitive::kPrimLong:
3560    case Primitive::kPrimDouble:
3561      locations->SetInAt(1, Location::DoubleStackSlot(codegen_->GetStackSlot(store->GetLocal())));
3562      break;
3563
3564    default:
3565      LOG(FATAL) << "Unimplemented local type " << field_type;
3566  }
3567}
3568
3569void InstructionCodeGeneratorARM64::VisitStoreLocal(HStoreLocal* store ATTRIBUTE_UNUSED) {
3570}
3571
3572void LocationsBuilderARM64::VisitSub(HSub* instruction) {
3573  HandleBinaryOp(instruction);
3574}
3575
3576void InstructionCodeGeneratorARM64::VisitSub(HSub* instruction) {
3577  HandleBinaryOp(instruction);
3578}
3579
3580void LocationsBuilderARM64::VisitStaticFieldGet(HStaticFieldGet* instruction) {
3581  HandleFieldGet(instruction);
3582}
3583
3584void InstructionCodeGeneratorARM64::VisitStaticFieldGet(HStaticFieldGet* instruction) {
3585  HandleFieldGet(instruction, instruction->GetFieldInfo());
3586}
3587
3588void LocationsBuilderARM64::VisitStaticFieldSet(HStaticFieldSet* instruction) {
3589  HandleFieldSet(instruction);
3590}
3591
3592void InstructionCodeGeneratorARM64::VisitStaticFieldSet(HStaticFieldSet* instruction) {
3593  HandleFieldSet(instruction, instruction->GetFieldInfo(), instruction->GetValueCanBeNull());
3594}
3595
3596void LocationsBuilderARM64::VisitUnresolvedInstanceFieldGet(
3597    HUnresolvedInstanceFieldGet* instruction) {
3598  FieldAccessCallingConventionARM64 calling_convention;
3599  codegen_->CreateUnresolvedFieldLocationSummary(
3600      instruction, instruction->GetFieldType(), calling_convention);
3601}
3602
3603void InstructionCodeGeneratorARM64::VisitUnresolvedInstanceFieldGet(
3604    HUnresolvedInstanceFieldGet* instruction) {
3605  FieldAccessCallingConventionARM64 calling_convention;
3606  codegen_->GenerateUnresolvedFieldAccess(instruction,
3607                                          instruction->GetFieldType(),
3608                                          instruction->GetFieldIndex(),
3609                                          instruction->GetDexPc(),
3610                                          calling_convention);
3611}
3612
3613void LocationsBuilderARM64::VisitUnresolvedInstanceFieldSet(
3614    HUnresolvedInstanceFieldSet* instruction) {
3615  FieldAccessCallingConventionARM64 calling_convention;
3616  codegen_->CreateUnresolvedFieldLocationSummary(
3617      instruction, instruction->GetFieldType(), calling_convention);
3618}
3619
3620void InstructionCodeGeneratorARM64::VisitUnresolvedInstanceFieldSet(
3621    HUnresolvedInstanceFieldSet* instruction) {
3622  FieldAccessCallingConventionARM64 calling_convention;
3623  codegen_->GenerateUnresolvedFieldAccess(instruction,
3624                                          instruction->GetFieldType(),
3625                                          instruction->GetFieldIndex(),
3626                                          instruction->GetDexPc(),
3627                                          calling_convention);
3628}
3629
3630void LocationsBuilderARM64::VisitUnresolvedStaticFieldGet(
3631    HUnresolvedStaticFieldGet* instruction) {
3632  FieldAccessCallingConventionARM64 calling_convention;
3633  codegen_->CreateUnresolvedFieldLocationSummary(
3634      instruction, instruction->GetFieldType(), calling_convention);
3635}
3636
3637void InstructionCodeGeneratorARM64::VisitUnresolvedStaticFieldGet(
3638    HUnresolvedStaticFieldGet* instruction) {
3639  FieldAccessCallingConventionARM64 calling_convention;
3640  codegen_->GenerateUnresolvedFieldAccess(instruction,
3641                                          instruction->GetFieldType(),
3642                                          instruction->GetFieldIndex(),
3643                                          instruction->GetDexPc(),
3644                                          calling_convention);
3645}
3646
3647void LocationsBuilderARM64::VisitUnresolvedStaticFieldSet(
3648    HUnresolvedStaticFieldSet* instruction) {
3649  FieldAccessCallingConventionARM64 calling_convention;
3650  codegen_->CreateUnresolvedFieldLocationSummary(
3651      instruction, instruction->GetFieldType(), calling_convention);
3652}
3653
3654void InstructionCodeGeneratorARM64::VisitUnresolvedStaticFieldSet(
3655    HUnresolvedStaticFieldSet* instruction) {
3656  FieldAccessCallingConventionARM64 calling_convention;
3657  codegen_->GenerateUnresolvedFieldAccess(instruction,
3658                                          instruction->GetFieldType(),
3659                                          instruction->GetFieldIndex(),
3660                                          instruction->GetDexPc(),
3661                                          calling_convention);
3662}
3663
3664void LocationsBuilderARM64::VisitSuspendCheck(HSuspendCheck* instruction) {
3665  new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCallOnSlowPath);
3666}
3667
3668void InstructionCodeGeneratorARM64::VisitSuspendCheck(HSuspendCheck* instruction) {
3669  HBasicBlock* block = instruction->GetBlock();
3670  if (block->GetLoopInformation() != nullptr) {
3671    DCHECK(block->GetLoopInformation()->GetSuspendCheck() == instruction);
3672    // The back edge will generate the suspend check.
3673    return;
3674  }
3675  if (block->IsEntryBlock() && instruction->GetNext()->IsGoto()) {
3676    // The goto will generate the suspend check.
3677    return;
3678  }
3679  GenerateSuspendCheck(instruction, nullptr);
3680}
3681
3682void LocationsBuilderARM64::VisitTemporary(HTemporary* temp) {
3683  temp->SetLocations(nullptr);
3684}
3685
3686void InstructionCodeGeneratorARM64::VisitTemporary(HTemporary* temp ATTRIBUTE_UNUSED) {
3687  // Nothing to do, this is driven by the code generator.
3688}
3689
3690void LocationsBuilderARM64::VisitThrow(HThrow* instruction) {
3691  LocationSummary* locations =
3692      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCall);
3693  InvokeRuntimeCallingConvention calling_convention;
3694  locations->SetInAt(0, LocationFrom(calling_convention.GetRegisterAt(0)));
3695}
3696
3697void InstructionCodeGeneratorARM64::VisitThrow(HThrow* instruction) {
3698  codegen_->InvokeRuntime(
3699      QUICK_ENTRY_POINT(pDeliverException), instruction, instruction->GetDexPc(), nullptr);
3700  CheckEntrypointTypes<kQuickDeliverException, void, mirror::Object*>();
3701}
3702
3703void LocationsBuilderARM64::VisitTypeConversion(HTypeConversion* conversion) {
3704  LocationSummary* locations =
3705      new (GetGraph()->GetArena()) LocationSummary(conversion, LocationSummary::kNoCall);
3706  Primitive::Type input_type = conversion->GetInputType();
3707  Primitive::Type result_type = conversion->GetResultType();
3708  DCHECK_NE(input_type, result_type);
3709  if ((input_type == Primitive::kPrimNot) || (input_type == Primitive::kPrimVoid) ||
3710      (result_type == Primitive::kPrimNot) || (result_type == Primitive::kPrimVoid)) {
3711    LOG(FATAL) << "Unexpected type conversion from " << input_type << " to " << result_type;
3712  }
3713
3714  if (Primitive::IsFloatingPointType(input_type)) {
3715    locations->SetInAt(0, Location::RequiresFpuRegister());
3716  } else {
3717    locations->SetInAt(0, Location::RequiresRegister());
3718  }
3719
3720  if (Primitive::IsFloatingPointType(result_type)) {
3721    locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap);
3722  } else {
3723    locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
3724  }
3725}
3726
3727void InstructionCodeGeneratorARM64::VisitTypeConversion(HTypeConversion* conversion) {
3728  Primitive::Type result_type = conversion->GetResultType();
3729  Primitive::Type input_type = conversion->GetInputType();
3730
3731  DCHECK_NE(input_type, result_type);
3732
3733  if (Primitive::IsIntegralType(result_type) && Primitive::IsIntegralType(input_type)) {
3734    int result_size = Primitive::ComponentSize(result_type);
3735    int input_size = Primitive::ComponentSize(input_type);
3736    int min_size = std::min(result_size, input_size);
3737    Register output = OutputRegister(conversion);
3738    Register source = InputRegisterAt(conversion, 0);
3739    if ((result_type == Primitive::kPrimChar) && (input_size < result_size)) {
3740      __ Ubfx(output, source, 0, result_size * kBitsPerByte);
3741    } else if (result_type == Primitive::kPrimInt && input_type == Primitive::kPrimLong) {
3742      // 'int' values are used directly as W registers, discarding the top
3743      // bits, so we don't need to sign-extend and can just perform a move.
3744      // We do not pass the `kDiscardForSameWReg` argument to force clearing the
3745      // top 32 bits of the target register. We theoretically could leave those
3746      // bits unchanged, but we would have to make sure that no code uses a
3747      // 32bit input value as a 64bit value assuming that the top 32 bits are
3748      // zero.
3749      __ Mov(output.W(), source.W());
3750    } else if ((result_type == Primitive::kPrimChar) ||
3751               ((input_type == Primitive::kPrimChar) && (result_size > input_size))) {
3752      __ Ubfx(output, output.IsX() ? source.X() : source.W(), 0, min_size * kBitsPerByte);
3753    } else {
3754      __ Sbfx(output, output.IsX() ? source.X() : source.W(), 0, min_size * kBitsPerByte);
3755    }
3756  } else if (Primitive::IsFloatingPointType(result_type) && Primitive::IsIntegralType(input_type)) {
3757    __ Scvtf(OutputFPRegister(conversion), InputRegisterAt(conversion, 0));
3758  } else if (Primitive::IsIntegralType(result_type) && Primitive::IsFloatingPointType(input_type)) {
3759    CHECK(result_type == Primitive::kPrimInt || result_type == Primitive::kPrimLong);
3760    __ Fcvtzs(OutputRegister(conversion), InputFPRegisterAt(conversion, 0));
3761  } else if (Primitive::IsFloatingPointType(result_type) &&
3762             Primitive::IsFloatingPointType(input_type)) {
3763    __ Fcvt(OutputFPRegister(conversion), InputFPRegisterAt(conversion, 0));
3764  } else {
3765    LOG(FATAL) << "Unexpected or unimplemented type conversion from " << input_type
3766                << " to " << result_type;
3767  }
3768}
3769
3770void LocationsBuilderARM64::VisitUShr(HUShr* ushr) {
3771  HandleShift(ushr);
3772}
3773
3774void InstructionCodeGeneratorARM64::VisitUShr(HUShr* ushr) {
3775  HandleShift(ushr);
3776}
3777
3778void LocationsBuilderARM64::VisitXor(HXor* instruction) {
3779  HandleBinaryOp(instruction);
3780}
3781
3782void InstructionCodeGeneratorARM64::VisitXor(HXor* instruction) {
3783  HandleBinaryOp(instruction);
3784}
3785
3786void LocationsBuilderARM64::VisitBoundType(HBoundType* instruction ATTRIBUTE_UNUSED) {
3787  // Nothing to do, this should be removed during prepare for register allocator.
3788  LOG(FATAL) << "Unreachable";
3789}
3790
3791void InstructionCodeGeneratorARM64::VisitBoundType(HBoundType* instruction ATTRIBUTE_UNUSED) {
3792  // Nothing to do, this should be removed during prepare for register allocator.
3793  LOG(FATAL) << "Unreachable";
3794}
3795
3796void LocationsBuilderARM64::VisitFakeString(HFakeString* instruction) {
3797  DCHECK(codegen_->IsBaseline());
3798  LocationSummary* locations =
3799      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
3800  locations->SetOut(Location::ConstantLocation(GetGraph()->GetNullConstant()));
3801}
3802
3803void InstructionCodeGeneratorARM64::VisitFakeString(HFakeString* instruction ATTRIBUTE_UNUSED) {
3804  DCHECK(codegen_->IsBaseline());
3805  // Will be generated at use site.
3806}
3807
3808// Simple implementation of packed switch - generate cascaded compare/jumps.
3809void LocationsBuilderARM64::VisitPackedSwitch(HPackedSwitch* switch_instr) {
3810  LocationSummary* locations =
3811      new (GetGraph()->GetArena()) LocationSummary(switch_instr, LocationSummary::kNoCall);
3812  locations->SetInAt(0, Location::RequiresRegister());
3813}
3814
3815void InstructionCodeGeneratorARM64::VisitPackedSwitch(HPackedSwitch* switch_instr) {
3816  int32_t lower_bound = switch_instr->GetStartValue();
3817  int32_t num_entries = switch_instr->GetNumEntries();
3818  Register value_reg = InputRegisterAt(switch_instr, 0);
3819  HBasicBlock* default_block = switch_instr->GetDefaultBlock();
3820
3821  // Create a series of compare/jumps.
3822  const ArenaVector<HBasicBlock*>& successors = switch_instr->GetBlock()->GetSuccessors();
3823  for (int32_t i = 0; i < num_entries; i++) {
3824    int32_t case_value = lower_bound + i;
3825    vixl::Label* succ = codegen_->GetLabelOf(successors[i]);
3826    if (case_value == 0) {
3827      __ Cbz(value_reg, succ);
3828    } else {
3829      __ Cmp(value_reg, vixl::Operand(case_value));
3830      __ B(eq, succ);
3831    }
3832  }
3833
3834  // And the default for any other value.
3835  if (!codegen_->GoesToNextBlock(switch_instr->GetBlock(), default_block)) {
3836    __ B(codegen_->GetLabelOf(default_block));
3837  }
3838}
3839
3840#undef __
3841#undef QUICK_ENTRY_POINT
3842
3843}  // namespace arm64
3844}  // namespace art
3845