code_generator_arm64.cc revision 729645a937eb9f04a311b3c22471dcf3ebe9bcec 
1/*
2 * Copyright (C) 2014 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *      http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17#include "code_generator_arm64.h"
18
19#include "arch/arm64/instruction_set_features_arm64.h"
20#include "art_method.h"
21#include "code_generator_utils.h"
22#include "compiled_method.h"
23#include "entrypoints/quick/quick_entrypoints.h"
24#include "entrypoints/quick/quick_entrypoints_enum.h"
25#include "gc/accounting/card_table.h"
26#include "intrinsics.h"
27#include "intrinsics_arm64.h"
28#include "mirror/array-inl.h"
29#include "mirror/class-inl.h"
30#include "offsets.h"
31#include "thread.h"
32#include "utils/arm64/assembler_arm64.h"
33#include "utils/assembler.h"
34#include "utils/stack_checks.h"
35
36
37using namespace vixl;   // NOLINT(build/namespaces)
38
39#ifdef __
40#error "ARM64 Codegen VIXL macro-assembler macro already defined."
41#endif
42
43namespace art {
44
45namespace arm64 {
46
47using helpers::CPURegisterFrom;
48using helpers::DRegisterFrom;
49using helpers::FPRegisterFrom;
50using helpers::HeapOperand;
51using helpers::HeapOperandFrom;
52using helpers::InputCPURegisterAt;
53using helpers::InputFPRegisterAt;
54using helpers::InputRegisterAt;
55using helpers::InputOperandAt;
56using helpers::Int64ConstantFrom;
57using helpers::LocationFrom;
58using helpers::OperandFromMemOperand;
59using helpers::OutputCPURegister;
60using helpers::OutputFPRegister;
61using helpers::OutputRegister;
62using helpers::RegisterFrom;
63using helpers::StackOperandFrom;
64using helpers::VIXLRegCodeFromART;
65using helpers::WRegisterFrom;
66using helpers::XRegisterFrom;
67using helpers::ARM64EncodableConstantOrRegister;
68using helpers::ArtVixlRegCodeCoherentForRegSet;
69
70static constexpr int kCurrentMethodStackOffset = 0;
71// The compare/jump sequence will generate about (2 * num_entries + 1) instructions. While jump
72// table version generates 7 instructions and num_entries literals. Compare/jump sequence will
73// generates less code/data with a small num_entries.
74static constexpr uint32_t kPackedSwitchJumpTableThreshold = 6;
75
76inline Condition ARM64Condition(IfCondition cond) {
77  switch (cond) {
78    case kCondEQ: return eq;
79    case kCondNE: return ne;
80    case kCondLT: return lt;
81    case kCondLE: return le;
82    case kCondGT: return gt;
83    case kCondGE: return ge;
84    case kCondB:  return lo;
85    case kCondBE: return ls;
86    case kCondA:  return hi;
87    case kCondAE: return hs;
88  }
89  LOG(FATAL) << "Unreachable";
90  UNREACHABLE();
91}
92
93Location ARM64ReturnLocation(Primitive::Type return_type) {
94  // Note that in practice, `LocationFrom(x0)` and `LocationFrom(w0)` create the
95  // same Location object, and so do `LocationFrom(d0)` and `LocationFrom(s0)`,
96  // but we use the exact registers for clarity.
97  if (return_type == Primitive::kPrimFloat) {
98    return LocationFrom(s0);
99  } else if (return_type == Primitive::kPrimDouble) {
100    return LocationFrom(d0);
101  } else if (return_type == Primitive::kPrimLong) {
102    return LocationFrom(x0);
103  } else if (return_type == Primitive::kPrimVoid) {
104    return Location::NoLocation();
105  } else {
106    return LocationFrom(w0);
107  }
108}
109
110Location InvokeRuntimeCallingConvention::GetReturnLocation(Primitive::Type return_type) {
111  return ARM64ReturnLocation(return_type);
112}
113
114#define __ down_cast<CodeGeneratorARM64*>(codegen)->GetVIXLAssembler()->
115#define QUICK_ENTRY_POINT(x) QUICK_ENTRYPOINT_OFFSET(kArm64WordSize, x).Int32Value()
116
117// Calculate memory accessing operand for save/restore live registers.
118static void SaveRestoreLiveRegistersHelper(CodeGenerator* codegen,
119                                           RegisterSet* register_set,
120                                           int64_t spill_offset,
121                                           bool is_save) {
122  DCHECK(ArtVixlRegCodeCoherentForRegSet(register_set->GetCoreRegisters(),
123                                         codegen->GetNumberOfCoreRegisters(),
124                                         register_set->GetFloatingPointRegisters(),
125                                         codegen->GetNumberOfFloatingPointRegisters()));
126
127  CPURegList core_list = CPURegList(CPURegister::kRegister, kXRegSize,
128      register_set->GetCoreRegisters() & (~callee_saved_core_registers.list()));
129  CPURegList fp_list = CPURegList(CPURegister::kFPRegister, kDRegSize,
130      register_set->GetFloatingPointRegisters() & (~callee_saved_fp_registers.list()));
131
132  MacroAssembler* masm = down_cast<CodeGeneratorARM64*>(codegen)->GetVIXLAssembler();
133  UseScratchRegisterScope temps(masm);
134
135  Register base = masm->StackPointer();
136  int64_t core_spill_size = core_list.TotalSizeInBytes();
137  int64_t fp_spill_size = fp_list.TotalSizeInBytes();
138  int64_t reg_size = kXRegSizeInBytes;
139  int64_t max_ls_pair_offset = spill_offset + core_spill_size + fp_spill_size - 2 * reg_size;
140  uint32_t ls_access_size = WhichPowerOf2(reg_size);
141  if (((core_list.Count() > 1) || (fp_list.Count() > 1)) &&
142      !masm->IsImmLSPair(max_ls_pair_offset, ls_access_size)) {
143    // If the offset does not fit in the instruction's immediate field, use an alternate register
144    // to compute the base address(float point registers spill base address).
145    Register new_base = temps.AcquireSameSizeAs(base);
146    __ Add(new_base, base, Operand(spill_offset + core_spill_size));
147    base = new_base;
148    spill_offset = -core_spill_size;
149    int64_t new_max_ls_pair_offset = fp_spill_size - 2 * reg_size;
150    DCHECK(masm->IsImmLSPair(spill_offset, ls_access_size));
151    DCHECK(masm->IsImmLSPair(new_max_ls_pair_offset, ls_access_size));
152  }
153
154  if (is_save) {
155    __ StoreCPURegList(core_list, MemOperand(base, spill_offset));
156    __ StoreCPURegList(fp_list, MemOperand(base, spill_offset + core_spill_size));
157  } else {
158    __ LoadCPURegList(core_list, MemOperand(base, spill_offset));
159    __ LoadCPURegList(fp_list, MemOperand(base, spill_offset + core_spill_size));
160  }
161}
162
163void SlowPathCodeARM64::SaveLiveRegisters(CodeGenerator* codegen, LocationSummary* locations) {
164  RegisterSet* register_set = locations->GetLiveRegisters();
165  size_t stack_offset = codegen->GetFirstRegisterSlotInSlowPath();
166  for (size_t i = 0, e = codegen->GetNumberOfCoreRegisters(); i < e; ++i) {
167    if (!codegen->IsCoreCalleeSaveRegister(i) && register_set->ContainsCoreRegister(i)) {
168      // If the register holds an object, update the stack mask.
169      if (locations->RegisterContainsObject(i)) {
170        locations->SetStackBit(stack_offset / kVRegSize);
171      }
172      DCHECK_LT(stack_offset, codegen->GetFrameSize() - codegen->FrameEntrySpillSize());
173      DCHECK_LT(i, kMaximumNumberOfExpectedRegisters);
174      saved_core_stack_offsets_[i] = stack_offset;
175      stack_offset += kXRegSizeInBytes;
176    }
177  }
178
179  for (size_t i = 0, e = codegen->GetNumberOfFloatingPointRegisters(); i < e; ++i) {
180    if (!codegen->IsFloatingPointCalleeSaveRegister(i) &&
181        register_set->ContainsFloatingPointRegister(i)) {
182      DCHECK_LT(stack_offset, codegen->GetFrameSize() - codegen->FrameEntrySpillSize());
183      DCHECK_LT(i, kMaximumNumberOfExpectedRegisters);
184      saved_fpu_stack_offsets_[i] = stack_offset;
185      stack_offset += kDRegSizeInBytes;
186    }
187  }
188
189  SaveRestoreLiveRegistersHelper(codegen, register_set,
190                                 codegen->GetFirstRegisterSlotInSlowPath(), true /* is_save */);
191}
192
193void SlowPathCodeARM64::RestoreLiveRegisters(CodeGenerator* codegen, LocationSummary* locations) {
194  RegisterSet* register_set = locations->GetLiveRegisters();
195  SaveRestoreLiveRegistersHelper(codegen, register_set,
196                                 codegen->GetFirstRegisterSlotInSlowPath(), false /* is_save */);
197}
198
199class BoundsCheckSlowPathARM64 : public SlowPathCodeARM64 {
200 public:
201  explicit BoundsCheckSlowPathARM64(HBoundsCheck* instruction) : instruction_(instruction) {}
202
203  void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
204    LocationSummary* locations = instruction_->GetLocations();
205    CodeGeneratorARM64* arm64_codegen = down_cast<CodeGeneratorARM64*>(codegen);
206
207    __ Bind(GetEntryLabel());
208    if (instruction_->CanThrowIntoCatchBlock()) {
209      // Live registers will be restored in the catch block if caught.
210      SaveLiveRegisters(codegen, instruction_->GetLocations());
211    }
212    // We're moving two locations to locations that could overlap, so we need a parallel
213    // move resolver.
214    InvokeRuntimeCallingConvention calling_convention;
215    codegen->EmitParallelMoves(
216        locations->InAt(0), LocationFrom(calling_convention.GetRegisterAt(0)), Primitive::kPrimInt,
217        locations->InAt(1), LocationFrom(calling_convention.GetRegisterAt(1)), Primitive::kPrimInt);
218    arm64_codegen->InvokeRuntime(
219        QUICK_ENTRY_POINT(pThrowArrayBounds), instruction_, instruction_->GetDexPc(), this);
220    CheckEntrypointTypes<kQuickThrowArrayBounds, void, int32_t, int32_t>();
221  }
222
223  bool IsFatal() const OVERRIDE { return true; }
224
225  const char* GetDescription() const OVERRIDE { return "BoundsCheckSlowPathARM64"; }
226
227 private:
228  HBoundsCheck* const instruction_;
229
230  DISALLOW_COPY_AND_ASSIGN(BoundsCheckSlowPathARM64);
231};
232
233class DivZeroCheckSlowPathARM64 : public SlowPathCodeARM64 {
234 public:
235  explicit DivZeroCheckSlowPathARM64(HDivZeroCheck* instruction) : instruction_(instruction) {}
236
237  void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
238    CodeGeneratorARM64* arm64_codegen = down_cast<CodeGeneratorARM64*>(codegen);
239    __ Bind(GetEntryLabel());
240    if (instruction_->CanThrowIntoCatchBlock()) {
241      // Live registers will be restored in the catch block if caught.
242      SaveLiveRegisters(codegen, instruction_->GetLocations());
243    }
244    arm64_codegen->InvokeRuntime(
245        QUICK_ENTRY_POINT(pThrowDivZero), instruction_, instruction_->GetDexPc(), this);
246    CheckEntrypointTypes<kQuickThrowDivZero, void, void>();
247  }
248
249  bool IsFatal() const OVERRIDE { return true; }
250
251  const char* GetDescription() const OVERRIDE { return "DivZeroCheckSlowPathARM64"; }
252
253 private:
254  HDivZeroCheck* const instruction_;
255  DISALLOW_COPY_AND_ASSIGN(DivZeroCheckSlowPathARM64);
256};
257
258class LoadClassSlowPathARM64 : public SlowPathCodeARM64 {
259 public:
260  LoadClassSlowPathARM64(HLoadClass* cls,
261                         HInstruction* at,
262                         uint32_t dex_pc,
263                         bool do_clinit)
264      : cls_(cls), at_(at), dex_pc_(dex_pc), do_clinit_(do_clinit) {
265    DCHECK(at->IsLoadClass() || at->IsClinitCheck());
266  }
267
268  void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
269    LocationSummary* locations = at_->GetLocations();
270    CodeGeneratorARM64* arm64_codegen = down_cast<CodeGeneratorARM64*>(codegen);
271
272    __ Bind(GetEntryLabel());
273    SaveLiveRegisters(codegen, locations);
274
275    InvokeRuntimeCallingConvention calling_convention;
276    __ Mov(calling_convention.GetRegisterAt(0).W(), cls_->GetTypeIndex());
277    int32_t entry_point_offset = do_clinit_ ? QUICK_ENTRY_POINT(pInitializeStaticStorage)
278                                            : QUICK_ENTRY_POINT(pInitializeType);
279    arm64_codegen->InvokeRuntime(entry_point_offset, at_, dex_pc_, this);
280    if (do_clinit_) {
281      CheckEntrypointTypes<kQuickInitializeStaticStorage, void*, uint32_t>();
282    } else {
283      CheckEntrypointTypes<kQuickInitializeType, void*, uint32_t>();
284    }
285
286    // Move the class to the desired location.
287    Location out = locations->Out();
288    if (out.IsValid()) {
289      DCHECK(out.IsRegister() && !locations->GetLiveRegisters()->ContainsCoreRegister(out.reg()));
290      Primitive::Type type = at_->GetType();
291      arm64_codegen->MoveLocation(out, calling_convention.GetReturnLocation(type), type);
292    }
293
294    RestoreLiveRegisters(codegen, locations);
295    __ B(GetExitLabel());
296  }
297
298  const char* GetDescription() const OVERRIDE { return "LoadClassSlowPathARM64"; }
299
300 private:
301  // The class this slow path will load.
302  HLoadClass* const cls_;
303
304  // The instruction where this slow path is happening.
305  // (Might be the load class or an initialization check).
306  HInstruction* const at_;
307
308  // The dex PC of `at_`.
309  const uint32_t dex_pc_;
310
311  // Whether to initialize the class.
312  const bool do_clinit_;
313
314  DISALLOW_COPY_AND_ASSIGN(LoadClassSlowPathARM64);
315};
316
317class LoadStringSlowPathARM64 : public SlowPathCodeARM64 {
318 public:
319  explicit LoadStringSlowPathARM64(HLoadString* instruction) : instruction_(instruction) {}
320
321  void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
322    LocationSummary* locations = instruction_->GetLocations();
323    DCHECK(!locations->GetLiveRegisters()->ContainsCoreRegister(locations->Out().reg()));
324    CodeGeneratorARM64* arm64_codegen = down_cast<CodeGeneratorARM64*>(codegen);
325
326    __ Bind(GetEntryLabel());
327    SaveLiveRegisters(codegen, locations);
328
329    InvokeRuntimeCallingConvention calling_convention;
330    __ Mov(calling_convention.GetRegisterAt(0).W(), instruction_->GetStringIndex());
331    arm64_codegen->InvokeRuntime(
332        QUICK_ENTRY_POINT(pResolveString), instruction_, instruction_->GetDexPc(), this);
333    CheckEntrypointTypes<kQuickResolveString, void*, uint32_t>();
334    Primitive::Type type = instruction_->GetType();
335    arm64_codegen->MoveLocation(locations->Out(), calling_convention.GetReturnLocation(type), type);
336
337    RestoreLiveRegisters(codegen, locations);
338    __ B(GetExitLabel());
339  }
340
341  const char* GetDescription() const OVERRIDE { return "LoadStringSlowPathARM64"; }
342
343 private:
344  HLoadString* const instruction_;
345
346  DISALLOW_COPY_AND_ASSIGN(LoadStringSlowPathARM64);
347};
348
349class NullCheckSlowPathARM64 : public SlowPathCodeARM64 {
350 public:
351  explicit NullCheckSlowPathARM64(HNullCheck* instr) : instruction_(instr) {}
352
353  void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
354    CodeGeneratorARM64* arm64_codegen = down_cast<CodeGeneratorARM64*>(codegen);
355    __ Bind(GetEntryLabel());
356    if (instruction_->CanThrowIntoCatchBlock()) {
357      // Live registers will be restored in the catch block if caught.
358      SaveLiveRegisters(codegen, instruction_->GetLocations());
359    }
360    arm64_codegen->InvokeRuntime(
361        QUICK_ENTRY_POINT(pThrowNullPointer), instruction_, instruction_->GetDexPc(), this);
362    CheckEntrypointTypes<kQuickThrowNullPointer, void, void>();
363  }
364
365  bool IsFatal() const OVERRIDE { return true; }
366
367  const char* GetDescription() const OVERRIDE { return "NullCheckSlowPathARM64"; }
368
369 private:
370  HNullCheck* const instruction_;
371
372  DISALLOW_COPY_AND_ASSIGN(NullCheckSlowPathARM64);
373};
374
375class SuspendCheckSlowPathARM64 : public SlowPathCodeARM64 {
376 public:
377  SuspendCheckSlowPathARM64(HSuspendCheck* instruction, HBasicBlock* successor)
378      : instruction_(instruction), successor_(successor) {}
379
380  void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
381    CodeGeneratorARM64* arm64_codegen = down_cast<CodeGeneratorARM64*>(codegen);
382    __ Bind(GetEntryLabel());
383    SaveLiveRegisters(codegen, instruction_->GetLocations());
384    arm64_codegen->InvokeRuntime(
385        QUICK_ENTRY_POINT(pTestSuspend), instruction_, instruction_->GetDexPc(), this);
386    CheckEntrypointTypes<kQuickTestSuspend, void, void>();
387    RestoreLiveRegisters(codegen, instruction_->GetLocations());
388    if (successor_ == nullptr) {
389      __ B(GetReturnLabel());
390    } else {
391      __ B(arm64_codegen->GetLabelOf(successor_));
392    }
393  }
394
395  vixl::Label* GetReturnLabel() {
396    DCHECK(successor_ == nullptr);
397    return &return_label_;
398  }
399
400  HBasicBlock* GetSuccessor() const {
401    return successor_;
402  }
403
404  const char* GetDescription() const OVERRIDE { return "SuspendCheckSlowPathARM64"; }
405
406 private:
407  HSuspendCheck* const instruction_;
408  // If not null, the block to branch to after the suspend check.
409  HBasicBlock* const successor_;
410
411  // If `successor_` is null, the label to branch to after the suspend check.
412  vixl::Label return_label_;
413
414  DISALLOW_COPY_AND_ASSIGN(SuspendCheckSlowPathARM64);
415};
416
417class TypeCheckSlowPathARM64 : public SlowPathCodeARM64 {
418 public:
419  TypeCheckSlowPathARM64(HInstruction* instruction, bool is_fatal)
420      : instruction_(instruction), is_fatal_(is_fatal) {}
421
422  void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
423    LocationSummary* locations = instruction_->GetLocations();
424    Location class_to_check = locations->InAt(1);
425    Location object_class = instruction_->IsCheckCast() ? locations->GetTemp(0)
426                                                        : locations->Out();
427    DCHECK(instruction_->IsCheckCast()
428           || !locations->GetLiveRegisters()->ContainsCoreRegister(locations->Out().reg()));
429    CodeGeneratorARM64* arm64_codegen = down_cast<CodeGeneratorARM64*>(codegen);
430    uint32_t dex_pc = instruction_->GetDexPc();
431
432    __ Bind(GetEntryLabel());
433
434    if (instruction_->IsCheckCast()) {
435      // The codegen for the instruction overwrites `temp`, so put it back in place.
436      Register obj = InputRegisterAt(instruction_, 0);
437      Register temp = WRegisterFrom(locations->GetTemp(0));
438      uint32_t class_offset = mirror::Object::ClassOffset().Int32Value();
439      __ Ldr(temp, HeapOperand(obj, class_offset));
440      arm64_codegen->GetAssembler()->MaybeUnpoisonHeapReference(temp);
441    }
442
443    if (!is_fatal_) {
444      SaveLiveRegisters(codegen, locations);
445    }
446
447    // We're moving two locations to locations that could overlap, so we need a parallel
448    // move resolver.
449    InvokeRuntimeCallingConvention calling_convention;
450    codegen->EmitParallelMoves(
451        class_to_check, LocationFrom(calling_convention.GetRegisterAt(0)), Primitive::kPrimNot,
452        object_class, LocationFrom(calling_convention.GetRegisterAt(1)), Primitive::kPrimNot);
453
454    if (instruction_->IsInstanceOf()) {
455      arm64_codegen->InvokeRuntime(
456          QUICK_ENTRY_POINT(pInstanceofNonTrivial), instruction_, dex_pc, this);
457      Primitive::Type ret_type = instruction_->GetType();
458      Location ret_loc = calling_convention.GetReturnLocation(ret_type);
459      arm64_codegen->MoveLocation(locations->Out(), ret_loc, ret_type);
460      CheckEntrypointTypes<kQuickInstanceofNonTrivial, uint32_t,
461                           const mirror::Class*, const mirror::Class*>();
462    } else {
463      DCHECK(instruction_->IsCheckCast());
464      arm64_codegen->InvokeRuntime(QUICK_ENTRY_POINT(pCheckCast), instruction_, dex_pc, this);
465      CheckEntrypointTypes<kQuickCheckCast, void, const mirror::Class*, const mirror::Class*>();
466    }
467
468    if (!is_fatal_) {
469      RestoreLiveRegisters(codegen, locations);
470      __ B(GetExitLabel());
471    }
472  }
473
474  const char* GetDescription() const OVERRIDE { return "TypeCheckSlowPathARM64"; }
475  bool IsFatal() const { return is_fatal_; }
476
477 private:
478  HInstruction* const instruction_;
479  const bool is_fatal_;
480
481  DISALLOW_COPY_AND_ASSIGN(TypeCheckSlowPathARM64);
482};
483
484class DeoptimizationSlowPathARM64 : public SlowPathCodeARM64 {
485 public:
486  explicit DeoptimizationSlowPathARM64(HInstruction* instruction)
487      : instruction_(instruction) {}
488
489  void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
490    __ Bind(GetEntryLabel());
491    SaveLiveRegisters(codegen, instruction_->GetLocations());
492    DCHECK(instruction_->IsDeoptimize());
493    HDeoptimize* deoptimize = instruction_->AsDeoptimize();
494    uint32_t dex_pc = deoptimize->GetDexPc();
495    CodeGeneratorARM64* arm64_codegen = down_cast<CodeGeneratorARM64*>(codegen);
496    arm64_codegen->InvokeRuntime(QUICK_ENTRY_POINT(pDeoptimize), instruction_, dex_pc, this);
497  }
498
499  const char* GetDescription() const OVERRIDE { return "DeoptimizationSlowPathARM64"; }
500
501 private:
502  HInstruction* const instruction_;
503  DISALLOW_COPY_AND_ASSIGN(DeoptimizationSlowPathARM64);
504};
505
506class ArraySetSlowPathARM64 : public SlowPathCodeARM64 {
507 public:
508  explicit ArraySetSlowPathARM64(HInstruction* instruction) : instruction_(instruction) {}
509
510  void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
511    LocationSummary* locations = instruction_->GetLocations();
512    __ Bind(GetEntryLabel());
513    SaveLiveRegisters(codegen, locations);
514
515    InvokeRuntimeCallingConvention calling_convention;
516    HParallelMove parallel_move(codegen->GetGraph()->GetArena());
517    parallel_move.AddMove(
518        locations->InAt(0),
519        LocationFrom(calling_convention.GetRegisterAt(0)),
520        Primitive::kPrimNot,
521        nullptr);
522    parallel_move.AddMove(
523        locations->InAt(1),
524        LocationFrom(calling_convention.GetRegisterAt(1)),
525        Primitive::kPrimInt,
526        nullptr);
527    parallel_move.AddMove(
528        locations->InAt(2),
529        LocationFrom(calling_convention.GetRegisterAt(2)),
530        Primitive::kPrimNot,
531        nullptr);
532    codegen->GetMoveResolver()->EmitNativeCode(&parallel_move);
533
534    CodeGeneratorARM64* arm64_codegen = down_cast<CodeGeneratorARM64*>(codegen);
535    arm64_codegen->InvokeRuntime(QUICK_ENTRY_POINT(pAputObject),
536                                 instruction_,
537                                 instruction_->GetDexPc(),
538                                 this);
539    CheckEntrypointTypes<kQuickAputObject, void, mirror::Array*, int32_t, mirror::Object*>();
540    RestoreLiveRegisters(codegen, locations);
541    __ B(GetExitLabel());
542  }
543
544  const char* GetDescription() const OVERRIDE { return "ArraySetSlowPathARM64"; }
545
546 private:
547  HInstruction* const instruction_;
548
549  DISALLOW_COPY_AND_ASSIGN(ArraySetSlowPathARM64);
550};
551
552void JumpTableARM64::EmitTable(CodeGeneratorARM64* codegen) {
553  uint32_t num_entries = switch_instr_->GetNumEntries();
554  DCHECK_GE(num_entries, kPackedSwitchJumpTableThreshold);
555
556  // We are about to use the assembler to place literals directly. Make sure we have enough
557  // underlying code buffer and we have generated the jump table with right size.
558  CodeBufferCheckScope scope(codegen->GetVIXLAssembler(), num_entries * sizeof(int32_t),
559                             CodeBufferCheckScope::kCheck, CodeBufferCheckScope::kExactSize);
560
561  __ Bind(&table_start_);
562  const ArenaVector<HBasicBlock*>& successors = switch_instr_->GetBlock()->GetSuccessors();
563  for (uint32_t i = 0; i < num_entries; i++) {
564    vixl::Label* target_label = codegen->GetLabelOf(successors[i]);
565    DCHECK(target_label->IsBound());
566    ptrdiff_t jump_offset = target_label->location() - table_start_.location();
567    DCHECK_GT(jump_offset, std::numeric_limits<int32_t>::min());
568    DCHECK_LE(jump_offset, std::numeric_limits<int32_t>::max());
569    Literal<int32_t> literal(jump_offset);
570    __ place(&literal);
571  }
572}
573
574#undef __
575
576Location InvokeDexCallingConventionVisitorARM64::GetNextLocation(Primitive::Type type) {
577  Location next_location;
578  if (type == Primitive::kPrimVoid) {
579    LOG(FATAL) << "Unreachable type " << type;
580  }
581
582  if (Primitive::IsFloatingPointType(type) &&
583      (float_index_ < calling_convention.GetNumberOfFpuRegisters())) {
584    next_location = LocationFrom(calling_convention.GetFpuRegisterAt(float_index_++));
585  } else if (!Primitive::IsFloatingPointType(type) &&
586             (gp_index_ < calling_convention.GetNumberOfRegisters())) {
587    next_location = LocationFrom(calling_convention.GetRegisterAt(gp_index_++));
588  } else {
589    size_t stack_offset = calling_convention.GetStackOffsetOf(stack_index_);
590    next_location = Primitive::Is64BitType(type) ? Location::DoubleStackSlot(stack_offset)
591                                                 : Location::StackSlot(stack_offset);
592  }
593
594  // Space on the stack is reserved for all arguments.
595  stack_index_ += Primitive::Is64BitType(type) ? 2 : 1;
596  return next_location;
597}
598
599Location InvokeDexCallingConventionVisitorARM64::GetMethodLocation() const {
600  return LocationFrom(kArtMethodRegister);
601}
602
603CodeGeneratorARM64::CodeGeneratorARM64(HGraph* graph,
604                                       const Arm64InstructionSetFeatures& isa_features,
605                                       const CompilerOptions& compiler_options,
606                                       OptimizingCompilerStats* stats)
607    : CodeGenerator(graph,
608                    kNumberOfAllocatableRegisters,
609                    kNumberOfAllocatableFPRegisters,
610                    kNumberOfAllocatableRegisterPairs,
611                    callee_saved_core_registers.list(),
612                    callee_saved_fp_registers.list(),
613                    compiler_options,
614                    stats),
615      block_labels_(nullptr),
616      jump_tables_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)),
617      location_builder_(graph, this),
618      instruction_visitor_(graph, this),
619      move_resolver_(graph->GetArena(), this),
620      isa_features_(isa_features),
621      uint64_literals_(std::less<uint64_t>(),
622                       graph->GetArena()->Adapter(kArenaAllocCodeGenerator)),
623      method_patches_(MethodReferenceComparator(),
624                      graph->GetArena()->Adapter(kArenaAllocCodeGenerator)),
625      call_patches_(MethodReferenceComparator(),
626                    graph->GetArena()->Adapter(kArenaAllocCodeGenerator)),
627      relative_call_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)),
628      pc_relative_dex_cache_patches_(graph->GetArena()->Adapter(kArenaAllocCodeGenerator)) {
629  // Save the link register (containing the return address) to mimic Quick.
630  AddAllocatedRegister(LocationFrom(lr));
631}
632
633#define __ GetVIXLAssembler()->
634
635void CodeGeneratorARM64::EmitJumpTables() {
636  for (auto jump_table : jump_tables_) {
637    jump_table->EmitTable(this);
638  }
639}
640
641void CodeGeneratorARM64::Finalize(CodeAllocator* allocator) {
642  EmitJumpTables();
643  // Ensure we emit the literal pool.
644  __ FinalizeCode();
645
646  CodeGenerator::Finalize(allocator);
647}
648
649void ParallelMoveResolverARM64::PrepareForEmitNativeCode() {
650  // Note: There are 6 kinds of moves:
651  // 1. constant -> GPR/FPR (non-cycle)
652  // 2. constant -> stack (non-cycle)
653  // 3. GPR/FPR -> GPR/FPR
654  // 4. GPR/FPR -> stack
655  // 5. stack -> GPR/FPR
656  // 6. stack -> stack (non-cycle)
657  // Case 1, 2 and 6 should never be included in a dependency cycle on ARM64. For case 3, 4, and 5
658  // VIXL uses at most 1 GPR. VIXL has 2 GPR and 1 FPR temps, and there should be no intersecting
659  // cycles on ARM64, so we always have 1 GPR and 1 FPR available VIXL temps to resolve the
660  // dependency.
661  vixl_temps_.Open(GetVIXLAssembler());
662}
663
664void ParallelMoveResolverARM64::FinishEmitNativeCode() {
665  vixl_temps_.Close();
666}
667
668Location ParallelMoveResolverARM64::AllocateScratchLocationFor(Location::Kind kind) {
669  DCHECK(kind == Location::kRegister || kind == Location::kFpuRegister ||
670         kind == Location::kStackSlot || kind == Location::kDoubleStackSlot);
671  kind = (kind == Location::kFpuRegister) ? Location::kFpuRegister : Location::kRegister;
672  Location scratch = GetScratchLocation(kind);
673  if (!scratch.Equals(Location::NoLocation())) {
674    return scratch;
675  }
676  // Allocate from VIXL temp registers.
677  if (kind == Location::kRegister) {
678    scratch = LocationFrom(vixl_temps_.AcquireX());
679  } else {
680    DCHECK(kind == Location::kFpuRegister);
681    scratch = LocationFrom(vixl_temps_.AcquireD());
682  }
683  AddScratchLocation(scratch);
684  return scratch;
685}
686
687void ParallelMoveResolverARM64::FreeScratchLocation(Location loc) {
688  if (loc.IsRegister()) {
689    vixl_temps_.Release(XRegisterFrom(loc));
690  } else {
691    DCHECK(loc.IsFpuRegister());
692    vixl_temps_.Release(DRegisterFrom(loc));
693  }
694  RemoveScratchLocation(loc);
695}
696
697void ParallelMoveResolverARM64::EmitMove(size_t index) {
698  MoveOperands* move = moves_[index];
699  codegen_->MoveLocation(move->GetDestination(), move->GetSource(), Primitive::kPrimVoid);
700}
701
702void CodeGeneratorARM64::GenerateFrameEntry() {
703  MacroAssembler* masm = GetVIXLAssembler();
704  BlockPoolsScope block_pools(masm);
705  __ Bind(&frame_entry_label_);
706
707  bool do_overflow_check = FrameNeedsStackCheck(GetFrameSize(), kArm64) || !IsLeafMethod();
708  if (do_overflow_check) {
709    UseScratchRegisterScope temps(masm);
710    Register temp = temps.AcquireX();
711    DCHECK(GetCompilerOptions().GetImplicitStackOverflowChecks());
712    __ Sub(temp, sp, static_cast<int32_t>(GetStackOverflowReservedBytes(kArm64)));
713    __ Ldr(wzr, MemOperand(temp, 0));
714    RecordPcInfo(nullptr, 0);
715  }
716
717  if (!HasEmptyFrame()) {
718    int frame_size = GetFrameSize();
719    // Stack layout:
720    //      sp[frame_size - 8]        : lr.
721    //      ...                       : other preserved core registers.
722    //      ...                       : other preserved fp registers.
723    //      ...                       : reserved frame space.
724    //      sp[0]                     : current method.
725    __ Str(kArtMethodRegister, MemOperand(sp, -frame_size, PreIndex));
726    GetAssembler()->cfi().AdjustCFAOffset(frame_size);
727    GetAssembler()->SpillRegisters(GetFramePreservedCoreRegisters(),
728        frame_size - GetCoreSpillSize());
729    GetAssembler()->SpillRegisters(GetFramePreservedFPRegisters(),
730        frame_size - FrameEntrySpillSize());
731  }
732}
733
734void CodeGeneratorARM64::GenerateFrameExit() {
735  BlockPoolsScope block_pools(GetVIXLAssembler());
736  GetAssembler()->cfi().RememberState();
737  if (!HasEmptyFrame()) {
738    int frame_size = GetFrameSize();
739    GetAssembler()->UnspillRegisters(GetFramePreservedFPRegisters(),
740        frame_size - FrameEntrySpillSize());
741    GetAssembler()->UnspillRegisters(GetFramePreservedCoreRegisters(),
742        frame_size - GetCoreSpillSize());
743    __ Drop(frame_size);
744    GetAssembler()->cfi().AdjustCFAOffset(-frame_size);
745  }
746  __ Ret();
747  GetAssembler()->cfi().RestoreState();
748  GetAssembler()->cfi().DefCFAOffset(GetFrameSize());
749}
750
751vixl::CPURegList CodeGeneratorARM64::GetFramePreservedCoreRegisters() const {
752  DCHECK(ArtVixlRegCodeCoherentForRegSet(core_spill_mask_, GetNumberOfCoreRegisters(), 0, 0));
753  return vixl::CPURegList(vixl::CPURegister::kRegister, vixl::kXRegSize,
754                          core_spill_mask_);
755}
756
757vixl::CPURegList CodeGeneratorARM64::GetFramePreservedFPRegisters() const {
758  DCHECK(ArtVixlRegCodeCoherentForRegSet(0, 0, fpu_spill_mask_,
759                                         GetNumberOfFloatingPointRegisters()));
760  return vixl::CPURegList(vixl::CPURegister::kFPRegister, vixl::kDRegSize,
761                          fpu_spill_mask_);
762}
763
764void CodeGeneratorARM64::Bind(HBasicBlock* block) {
765  __ Bind(GetLabelOf(block));
766}
767
768void CodeGeneratorARM64::Move(HInstruction* instruction,
769                              Location location,
770                              HInstruction* move_for) {
771  LocationSummary* locations = instruction->GetLocations();
772  Primitive::Type type = instruction->GetType();
773  DCHECK_NE(type, Primitive::kPrimVoid);
774
775  if (instruction->IsFakeString()) {
776    // The fake string is an alias for null.
777    DCHECK(IsBaseline());
778    instruction = locations->Out().GetConstant();
779    DCHECK(instruction->IsNullConstant()) << instruction->DebugName();
780  }
781
782  if (instruction->IsCurrentMethod()) {
783    MoveLocation(location,
784                 Location::DoubleStackSlot(kCurrentMethodStackOffset),
785                 Primitive::kPrimVoid);
786  } else if (locations != nullptr && locations->Out().Equals(location)) {
787    return;
788  } else if (instruction->IsIntConstant()
789             || instruction->IsLongConstant()
790             || instruction->IsNullConstant()) {
791    int64_t value = GetInt64ValueOf(instruction->AsConstant());
792    if (location.IsRegister()) {
793      Register dst = RegisterFrom(location, type);
794      DCHECK(((instruction->IsIntConstant() || instruction->IsNullConstant()) && dst.Is32Bits()) ||
795             (instruction->IsLongConstant() && dst.Is64Bits()));
796      __ Mov(dst, value);
797    } else {
798      DCHECK(location.IsStackSlot() || location.IsDoubleStackSlot());
799      UseScratchRegisterScope temps(GetVIXLAssembler());
800      Register temp = (instruction->IsIntConstant() || instruction->IsNullConstant())
801          ? temps.AcquireW()
802          : temps.AcquireX();
803      __ Mov(temp, value);
804      __ Str(temp, StackOperandFrom(location));
805    }
806  } else if (instruction->IsTemporary()) {
807    Location temp_location = GetTemporaryLocation(instruction->AsTemporary());
808    MoveLocation(location, temp_location, type);
809  } else if (instruction->IsLoadLocal()) {
810    uint32_t stack_slot = GetStackSlot(instruction->AsLoadLocal()->GetLocal());
811    if (Primitive::Is64BitType(type)) {
812      MoveLocation(location, Location::DoubleStackSlot(stack_slot), type);
813    } else {
814      MoveLocation(location, Location::StackSlot(stack_slot), type);
815    }
816
817  } else {
818    DCHECK((instruction->GetNext() == move_for) || instruction->GetNext()->IsTemporary());
819    MoveLocation(location, locations->Out(), type);
820  }
821}
822
823void CodeGeneratorARM64::MoveConstant(Location location, int32_t value) {
824  DCHECK(location.IsRegister());
825  __ Mov(RegisterFrom(location, Primitive::kPrimInt), value);
826}
827
828void CodeGeneratorARM64::AddLocationAsTemp(Location location, LocationSummary* locations) {
829  if (location.IsRegister()) {
830    locations->AddTemp(location);
831  } else {
832    UNIMPLEMENTED(FATAL) << "AddLocationAsTemp not implemented for location " << location;
833  }
834}
835
836Location CodeGeneratorARM64::GetStackLocation(HLoadLocal* load) const {
837  Primitive::Type type = load->GetType();
838
839  switch (type) {
840    case Primitive::kPrimNot:
841    case Primitive::kPrimInt:
842    case Primitive::kPrimFloat:
843      return Location::StackSlot(GetStackSlot(load->GetLocal()));
844
845    case Primitive::kPrimLong:
846    case Primitive::kPrimDouble:
847      return Location::DoubleStackSlot(GetStackSlot(load->GetLocal()));
848
849    case Primitive::kPrimBoolean:
850    case Primitive::kPrimByte:
851    case Primitive::kPrimChar:
852    case Primitive::kPrimShort:
853    case Primitive::kPrimVoid:
854      LOG(FATAL) << "Unexpected type " << type;
855  }
856
857  LOG(FATAL) << "Unreachable";
858  return Location::NoLocation();
859}
860
861void CodeGeneratorARM64::MarkGCCard(Register object, Register value, bool value_can_be_null) {
862  UseScratchRegisterScope temps(GetVIXLAssembler());
863  Register card = temps.AcquireX();
864  Register temp = temps.AcquireW();   // Index within the CardTable - 32bit.
865  vixl::Label done;
866  if (value_can_be_null) {
867    __ Cbz(value, &done);
868  }
869  __ Ldr(card, MemOperand(tr, Thread::CardTableOffset<kArm64WordSize>().Int32Value()));
870  __ Lsr(temp, object, gc::accounting::CardTable::kCardShift);
871  __ Strb(card, MemOperand(card, temp.X()));
872  if (value_can_be_null) {
873    __ Bind(&done);
874  }
875}
876
877void CodeGeneratorARM64::SetupBlockedRegisters(bool is_baseline) const {
878  // Blocked core registers:
879  //      lr        : Runtime reserved.
880  //      tr        : Runtime reserved.
881  //      xSuspend  : Runtime reserved. TODO: Unblock this when the runtime stops using it.
882  //      ip1       : VIXL core temp.
883  //      ip0       : VIXL core temp.
884  //
885  // Blocked fp registers:
886  //      d31       : VIXL fp temp.
887  CPURegList reserved_core_registers = vixl_reserved_core_registers;
888  reserved_core_registers.Combine(runtime_reserved_core_registers);
889  while (!reserved_core_registers.IsEmpty()) {
890    blocked_core_registers_[reserved_core_registers.PopLowestIndex().code()] = true;
891  }
892
893  CPURegList reserved_fp_registers = vixl_reserved_fp_registers;
894  while (!reserved_fp_registers.IsEmpty()) {
895    blocked_fpu_registers_[reserved_fp_registers.PopLowestIndex().code()] = true;
896  }
897
898  if (is_baseline) {
899    CPURegList reserved_core_baseline_registers = callee_saved_core_registers;
900    while (!reserved_core_baseline_registers.IsEmpty()) {
901      blocked_core_registers_[reserved_core_baseline_registers.PopLowestIndex().code()] = true;
902    }
903  }
904
905  if (is_baseline || GetGraph()->IsDebuggable()) {
906    // Stubs do not save callee-save floating point registers. If the graph
907    // is debuggable, we need to deal with these registers differently. For
908    // now, just block them.
909    CPURegList reserved_fp_baseline_registers = callee_saved_fp_registers;
910    while (!reserved_fp_baseline_registers.IsEmpty()) {
911      blocked_fpu_registers_[reserved_fp_baseline_registers.PopLowestIndex().code()] = true;
912    }
913  }
914}
915
916Location CodeGeneratorARM64::AllocateFreeRegister(Primitive::Type type) const {
917  if (type == Primitive::kPrimVoid) {
918    LOG(FATAL) << "Unreachable type " << type;
919  }
920
921  if (Primitive::IsFloatingPointType(type)) {
922    ssize_t reg = FindFreeEntry(blocked_fpu_registers_, kNumberOfAllocatableFPRegisters);
923    DCHECK_NE(reg, -1);
924    return Location::FpuRegisterLocation(reg);
925  } else {
926    ssize_t reg = FindFreeEntry(blocked_core_registers_, kNumberOfAllocatableRegisters);
927    DCHECK_NE(reg, -1);
928    return Location::RegisterLocation(reg);
929  }
930}
931
932size_t CodeGeneratorARM64::SaveCoreRegister(size_t stack_index, uint32_t reg_id) {
933  Register reg = Register(VIXLRegCodeFromART(reg_id), kXRegSize);
934  __ Str(reg, MemOperand(sp, stack_index));
935  return kArm64WordSize;
936}
937
938size_t CodeGeneratorARM64::RestoreCoreRegister(size_t stack_index, uint32_t reg_id) {
939  Register reg = Register(VIXLRegCodeFromART(reg_id), kXRegSize);
940  __ Ldr(reg, MemOperand(sp, stack_index));
941  return kArm64WordSize;
942}
943
944size_t CodeGeneratorARM64::SaveFloatingPointRegister(size_t stack_index, uint32_t reg_id) {
945  FPRegister reg = FPRegister(reg_id, kDRegSize);
946  __ Str(reg, MemOperand(sp, stack_index));
947  return kArm64WordSize;
948}
949
950size_t CodeGeneratorARM64::RestoreFloatingPointRegister(size_t stack_index, uint32_t reg_id) {
951  FPRegister reg = FPRegister(reg_id, kDRegSize);
952  __ Ldr(reg, MemOperand(sp, stack_index));
953  return kArm64WordSize;
954}
955
956void CodeGeneratorARM64::DumpCoreRegister(std::ostream& stream, int reg) const {
957  stream << XRegister(reg);
958}
959
960void CodeGeneratorARM64::DumpFloatingPointRegister(std::ostream& stream, int reg) const {
961  stream << DRegister(reg);
962}
963
964void CodeGeneratorARM64::MoveConstant(CPURegister destination, HConstant* constant) {
965  if (constant->IsIntConstant()) {
966    __ Mov(Register(destination), constant->AsIntConstant()->GetValue());
967  } else if (constant->IsLongConstant()) {
968    __ Mov(Register(destination), constant->AsLongConstant()->GetValue());
969  } else if (constant->IsNullConstant()) {
970    __ Mov(Register(destination), 0);
971  } else if (constant->IsFloatConstant()) {
972    __ Fmov(FPRegister(destination), constant->AsFloatConstant()->GetValue());
973  } else {
974    DCHECK(constant->IsDoubleConstant());
975    __ Fmov(FPRegister(destination), constant->AsDoubleConstant()->GetValue());
976  }
977}
978
979
980static bool CoherentConstantAndType(Location constant, Primitive::Type type) {
981  DCHECK(constant.IsConstant());
982  HConstant* cst = constant.GetConstant();
983  return (cst->IsIntConstant() && type == Primitive::kPrimInt) ||
984         // Null is mapped to a core W register, which we associate with kPrimInt.
985         (cst->IsNullConstant() && type == Primitive::kPrimInt) ||
986         (cst->IsLongConstant() && type == Primitive::kPrimLong) ||
987         (cst->IsFloatConstant() && type == Primitive::kPrimFloat) ||
988         (cst->IsDoubleConstant() && type == Primitive::kPrimDouble);
989}
990
991void CodeGeneratorARM64::MoveLocation(Location destination,
992                                      Location source,
993                                      Primitive::Type dst_type) {
994  if (source.Equals(destination)) {
995    return;
996  }
997
998  // A valid move can always be inferred from the destination and source
999  // locations. When moving from and to a register, the argument type can be
1000  // used to generate 32bit instead of 64bit moves. In debug mode we also
1001  // checks the coherency of the locations and the type.
1002  bool unspecified_type = (dst_type == Primitive::kPrimVoid);
1003
1004  if (destination.IsRegister() || destination.IsFpuRegister()) {
1005    if (unspecified_type) {
1006      HConstant* src_cst = source.IsConstant() ? source.GetConstant() : nullptr;
1007      if (source.IsStackSlot() ||
1008          (src_cst != nullptr && (src_cst->IsIntConstant()
1009                                  || src_cst->IsFloatConstant()
1010                                  || src_cst->IsNullConstant()))) {
1011        // For stack slots and 32bit constants, a 64bit type is appropriate.
1012        dst_type = destination.IsRegister() ? Primitive::kPrimInt : Primitive::kPrimFloat;
1013      } else {
1014        // If the source is a double stack slot or a 64bit constant, a 64bit
1015        // type is appropriate. Else the source is a register, and since the
1016        // type has not been specified, we chose a 64bit type to force a 64bit
1017        // move.
1018        dst_type = destination.IsRegister() ? Primitive::kPrimLong : Primitive::kPrimDouble;
1019      }
1020    }
1021    DCHECK((destination.IsFpuRegister() && Primitive::IsFloatingPointType(dst_type)) ||
1022           (destination.IsRegister() && !Primitive::IsFloatingPointType(dst_type)));
1023    CPURegister dst = CPURegisterFrom(destination, dst_type);
1024    if (source.IsStackSlot() || source.IsDoubleStackSlot()) {
1025      DCHECK(dst.Is64Bits() == source.IsDoubleStackSlot());
1026      __ Ldr(dst, StackOperandFrom(source));
1027    } else if (source.IsConstant()) {
1028      DCHECK(CoherentConstantAndType(source, dst_type));
1029      MoveConstant(dst, source.GetConstant());
1030    } else if (source.IsRegister()) {
1031      if (destination.IsRegister()) {
1032        __ Mov(Register(dst), RegisterFrom(source, dst_type));
1033      } else {
1034        DCHECK(destination.IsFpuRegister());
1035        Primitive::Type source_type = Primitive::Is64BitType(dst_type)
1036            ? Primitive::kPrimLong
1037            : Primitive::kPrimInt;
1038        __ Fmov(FPRegisterFrom(destination, dst_type), RegisterFrom(source, source_type));
1039      }
1040    } else {
1041      DCHECK(source.IsFpuRegister());
1042      if (destination.IsRegister()) {
1043        Primitive::Type source_type = Primitive::Is64BitType(dst_type)
1044            ? Primitive::kPrimDouble
1045            : Primitive::kPrimFloat;
1046        __ Fmov(RegisterFrom(destination, dst_type), FPRegisterFrom(source, source_type));
1047      } else {
1048        DCHECK(destination.IsFpuRegister());
1049        __ Fmov(FPRegister(dst), FPRegisterFrom(source, dst_type));
1050      }
1051    }
1052  } else {  // The destination is not a register. It must be a stack slot.
1053    DCHECK(destination.IsStackSlot() || destination.IsDoubleStackSlot());
1054    if (source.IsRegister() || source.IsFpuRegister()) {
1055      if (unspecified_type) {
1056        if (source.IsRegister()) {
1057          dst_type = destination.IsStackSlot() ? Primitive::kPrimInt : Primitive::kPrimLong;
1058        } else {
1059          dst_type = destination.IsStackSlot() ? Primitive::kPrimFloat : Primitive::kPrimDouble;
1060        }
1061      }
1062      DCHECK((destination.IsDoubleStackSlot() == Primitive::Is64BitType(dst_type)) &&
1063             (source.IsFpuRegister() == Primitive::IsFloatingPointType(dst_type)));
1064      __ Str(CPURegisterFrom(source, dst_type), StackOperandFrom(destination));
1065    } else if (source.IsConstant()) {
1066      DCHECK(unspecified_type || CoherentConstantAndType(source, dst_type))
1067          << source << " " << dst_type;
1068      UseScratchRegisterScope temps(GetVIXLAssembler());
1069      HConstant* src_cst = source.GetConstant();
1070      CPURegister temp;
1071      if (src_cst->IsIntConstant() || src_cst->IsNullConstant()) {
1072        temp = temps.AcquireW();
1073      } else if (src_cst->IsLongConstant()) {
1074        temp = temps.AcquireX();
1075      } else if (src_cst->IsFloatConstant()) {
1076        temp = temps.AcquireS();
1077      } else {
1078        DCHECK(src_cst->IsDoubleConstant());
1079        temp = temps.AcquireD();
1080      }
1081      MoveConstant(temp, src_cst);
1082      __ Str(temp, StackOperandFrom(destination));
1083    } else {
1084      DCHECK(source.IsStackSlot() || source.IsDoubleStackSlot());
1085      DCHECK(source.IsDoubleStackSlot() == destination.IsDoubleStackSlot());
1086      UseScratchRegisterScope temps(GetVIXLAssembler());
1087      // There is generally less pressure on FP registers.
1088      FPRegister temp = destination.IsDoubleStackSlot() ? temps.AcquireD() : temps.AcquireS();
1089      __ Ldr(temp, StackOperandFrom(source));
1090      __ Str(temp, StackOperandFrom(destination));
1091    }
1092  }
1093}
1094
1095void CodeGeneratorARM64::Load(Primitive::Type type,
1096                              CPURegister dst,
1097                              const MemOperand& src) {
1098  switch (type) {
1099    case Primitive::kPrimBoolean:
1100      __ Ldrb(Register(dst), src);
1101      break;
1102    case Primitive::kPrimByte:
1103      __ Ldrsb(Register(dst), src);
1104      break;
1105    case Primitive::kPrimShort:
1106      __ Ldrsh(Register(dst), src);
1107      break;
1108    case Primitive::kPrimChar:
1109      __ Ldrh(Register(dst), src);
1110      break;
1111    case Primitive::kPrimInt:
1112    case Primitive::kPrimNot:
1113    case Primitive::kPrimLong:
1114    case Primitive::kPrimFloat:
1115    case Primitive::kPrimDouble:
1116      DCHECK_EQ(dst.Is64Bits(), Primitive::Is64BitType(type));
1117      __ Ldr(dst, src);
1118      break;
1119    case Primitive::kPrimVoid:
1120      LOG(FATAL) << "Unreachable type " << type;
1121  }
1122}
1123
1124void CodeGeneratorARM64::LoadAcquire(HInstruction* instruction,
1125                                     CPURegister dst,
1126                                     const MemOperand& src) {
1127  MacroAssembler* masm = GetVIXLAssembler();
1128  BlockPoolsScope block_pools(masm);
1129  UseScratchRegisterScope temps(masm);
1130  Register temp_base = temps.AcquireX();
1131  Primitive::Type type = instruction->GetType();
1132
1133  DCHECK(!src.IsPreIndex());
1134  DCHECK(!src.IsPostIndex());
1135
1136  // TODO(vixl): Let the MacroAssembler handle MemOperand.
1137  __ Add(temp_base, src.base(), OperandFromMemOperand(src));
1138  MemOperand base = MemOperand(temp_base);
1139  switch (type) {
1140    case Primitive::kPrimBoolean:
1141      __ Ldarb(Register(dst), base);
1142      MaybeRecordImplicitNullCheck(instruction);
1143      break;
1144    case Primitive::kPrimByte:
1145      __ Ldarb(Register(dst), base);
1146      MaybeRecordImplicitNullCheck(instruction);
1147      __ Sbfx(Register(dst), Register(dst), 0, Primitive::ComponentSize(type) * kBitsPerByte);
1148      break;
1149    case Primitive::kPrimChar:
1150      __ Ldarh(Register(dst), base);
1151      MaybeRecordImplicitNullCheck(instruction);
1152      break;
1153    case Primitive::kPrimShort:
1154      __ Ldarh(Register(dst), base);
1155      MaybeRecordImplicitNullCheck(instruction);
1156      __ Sbfx(Register(dst), Register(dst), 0, Primitive::ComponentSize(type) * kBitsPerByte);
1157      break;
1158    case Primitive::kPrimInt:
1159    case Primitive::kPrimNot:
1160    case Primitive::kPrimLong:
1161      DCHECK_EQ(dst.Is64Bits(), Primitive::Is64BitType(type));
1162      __ Ldar(Register(dst), base);
1163      MaybeRecordImplicitNullCheck(instruction);
1164      break;
1165    case Primitive::kPrimFloat:
1166    case Primitive::kPrimDouble: {
1167      DCHECK(dst.IsFPRegister());
1168      DCHECK_EQ(dst.Is64Bits(), Primitive::Is64BitType(type));
1169
1170      Register temp = dst.Is64Bits() ? temps.AcquireX() : temps.AcquireW();
1171      __ Ldar(temp, base);
1172      MaybeRecordImplicitNullCheck(instruction);
1173      __ Fmov(FPRegister(dst), temp);
1174      break;
1175    }
1176    case Primitive::kPrimVoid:
1177      LOG(FATAL) << "Unreachable type " << type;
1178  }
1179}
1180
1181void CodeGeneratorARM64::Store(Primitive::Type type,
1182                               CPURegister src,
1183                               const MemOperand& dst) {
1184  switch (type) {
1185    case Primitive::kPrimBoolean:
1186    case Primitive::kPrimByte:
1187      __ Strb(Register(src), dst);
1188      break;
1189    case Primitive::kPrimChar:
1190    case Primitive::kPrimShort:
1191      __ Strh(Register(src), dst);
1192      break;
1193    case Primitive::kPrimInt:
1194    case Primitive::kPrimNot:
1195    case Primitive::kPrimLong:
1196    case Primitive::kPrimFloat:
1197    case Primitive::kPrimDouble:
1198      DCHECK_EQ(src.Is64Bits(), Primitive::Is64BitType(type));
1199      __ Str(src, dst);
1200      break;
1201    case Primitive::kPrimVoid:
1202      LOG(FATAL) << "Unreachable type " << type;
1203  }
1204}
1205
1206void CodeGeneratorARM64::StoreRelease(Primitive::Type type,
1207                                      CPURegister src,
1208                                      const MemOperand& dst) {
1209  UseScratchRegisterScope temps(GetVIXLAssembler());
1210  Register temp_base = temps.AcquireX();
1211
1212  DCHECK(!dst.IsPreIndex());
1213  DCHECK(!dst.IsPostIndex());
1214
1215  // TODO(vixl): Let the MacroAssembler handle this.
1216  Operand op = OperandFromMemOperand(dst);
1217  __ Add(temp_base, dst.base(), op);
1218  MemOperand base = MemOperand(temp_base);
1219  switch (type) {
1220    case Primitive::kPrimBoolean:
1221    case Primitive::kPrimByte:
1222      __ Stlrb(Register(src), base);
1223      break;
1224    case Primitive::kPrimChar:
1225    case Primitive::kPrimShort:
1226      __ Stlrh(Register(src), base);
1227      break;
1228    case Primitive::kPrimInt:
1229    case Primitive::kPrimNot:
1230    case Primitive::kPrimLong:
1231      DCHECK_EQ(src.Is64Bits(), Primitive::Is64BitType(type));
1232      __ Stlr(Register(src), base);
1233      break;
1234    case Primitive::kPrimFloat:
1235    case Primitive::kPrimDouble: {
1236      DCHECK(src.IsFPRegister());
1237      DCHECK_EQ(src.Is64Bits(), Primitive::Is64BitType(type));
1238
1239      Register temp = src.Is64Bits() ? temps.AcquireX() : temps.AcquireW();
1240      __ Fmov(temp, FPRegister(src));
1241      __ Stlr(temp, base);
1242      break;
1243    }
1244    case Primitive::kPrimVoid:
1245      LOG(FATAL) << "Unreachable type " << type;
1246  }
1247}
1248
1249void CodeGeneratorARM64::InvokeRuntime(QuickEntrypointEnum entrypoint,
1250                                       HInstruction* instruction,
1251                                       uint32_t dex_pc,
1252                                       SlowPathCode* slow_path) {
1253  InvokeRuntime(GetThreadOffset<kArm64WordSize>(entrypoint).Int32Value(),
1254                instruction,
1255                dex_pc,
1256                slow_path);
1257}
1258
1259void CodeGeneratorARM64::InvokeRuntime(int32_t entry_point_offset,
1260                                       HInstruction* instruction,
1261                                       uint32_t dex_pc,
1262                                       SlowPathCode* slow_path) {
1263  ValidateInvokeRuntime(instruction, slow_path);
1264  BlockPoolsScope block_pools(GetVIXLAssembler());
1265  __ Ldr(lr, MemOperand(tr, entry_point_offset));
1266  __ Blr(lr);
1267  RecordPcInfo(instruction, dex_pc, slow_path);
1268}
1269
1270void InstructionCodeGeneratorARM64::GenerateClassInitializationCheck(SlowPathCodeARM64* slow_path,
1271                                                                     vixl::Register class_reg) {
1272  UseScratchRegisterScope temps(GetVIXLAssembler());
1273  Register temp = temps.AcquireW();
1274  size_t status_offset = mirror::Class::StatusOffset().SizeValue();
1275  bool use_acquire_release = codegen_->GetInstructionSetFeatures().PreferAcquireRelease();
1276
1277  // Even if the initialized flag is set, we need to ensure consistent memory ordering.
1278  if (use_acquire_release) {
1279    // TODO(vixl): Let the MacroAssembler handle MemOperand.
1280    __ Add(temp, class_reg, status_offset);
1281    __ Ldar(temp, HeapOperand(temp));
1282    __ Cmp(temp, mirror::Class::kStatusInitialized);
1283    __ B(lt, slow_path->GetEntryLabel());
1284  } else {
1285    __ Ldr(temp, HeapOperand(class_reg, status_offset));
1286    __ Cmp(temp, mirror::Class::kStatusInitialized);
1287    __ B(lt, slow_path->GetEntryLabel());
1288    __ Dmb(InnerShareable, BarrierReads);
1289  }
1290  __ Bind(slow_path->GetExitLabel());
1291}
1292
1293void InstructionCodeGeneratorARM64::GenerateMemoryBarrier(MemBarrierKind kind) {
1294  BarrierType type = BarrierAll;
1295
1296  switch (kind) {
1297    case MemBarrierKind::kAnyAny:
1298    case MemBarrierKind::kAnyStore: {
1299      type = BarrierAll;
1300      break;
1301    }
1302    case MemBarrierKind::kLoadAny: {
1303      type = BarrierReads;
1304      break;
1305    }
1306    case MemBarrierKind::kStoreStore: {
1307      type = BarrierWrites;
1308      break;
1309    }
1310    default:
1311      LOG(FATAL) << "Unexpected memory barrier " << kind;
1312  }
1313  __ Dmb(InnerShareable, type);
1314}
1315
1316void InstructionCodeGeneratorARM64::GenerateSuspendCheck(HSuspendCheck* instruction,
1317                                                         HBasicBlock* successor) {
1318  SuspendCheckSlowPathARM64* slow_path =
1319      down_cast<SuspendCheckSlowPathARM64*>(instruction->GetSlowPath());
1320  if (slow_path == nullptr) {
1321    slow_path = new (GetGraph()->GetArena()) SuspendCheckSlowPathARM64(instruction, successor);
1322    instruction->SetSlowPath(slow_path);
1323    codegen_->AddSlowPath(slow_path);
1324    if (successor != nullptr) {
1325      DCHECK(successor->IsLoopHeader());
1326      codegen_->ClearSpillSlotsFromLoopPhisInStackMap(instruction);
1327    }
1328  } else {
1329    DCHECK_EQ(slow_path->GetSuccessor(), successor);
1330  }
1331
1332  UseScratchRegisterScope temps(codegen_->GetVIXLAssembler());
1333  Register temp = temps.AcquireW();
1334
1335  __ Ldrh(temp, MemOperand(tr, Thread::ThreadFlagsOffset<kArm64WordSize>().SizeValue()));
1336  if (successor == nullptr) {
1337    __ Cbnz(temp, slow_path->GetEntryLabel());
1338    __ Bind(slow_path->GetReturnLabel());
1339  } else {
1340    __ Cbz(temp, codegen_->GetLabelOf(successor));
1341    __ B(slow_path->GetEntryLabel());
1342    // slow_path will return to GetLabelOf(successor).
1343  }
1344}
1345
1346InstructionCodeGeneratorARM64::InstructionCodeGeneratorARM64(HGraph* graph,
1347                                                             CodeGeneratorARM64* codegen)
1348      : HGraphVisitor(graph),
1349        assembler_(codegen->GetAssembler()),
1350        codegen_(codegen) {}
1351
1352#define FOR_EACH_UNIMPLEMENTED_INSTRUCTION(M)              \
1353  /* No unimplemented IR. */
1354
1355#define UNIMPLEMENTED_INSTRUCTION_BREAK_CODE(name) name##UnimplementedInstructionBreakCode
1356
1357enum UnimplementedInstructionBreakCode {
1358  // Using a base helps identify when we hit such breakpoints.
1359  UnimplementedInstructionBreakCodeBaseCode = 0x900,
1360#define ENUM_UNIMPLEMENTED_INSTRUCTION(name) UNIMPLEMENTED_INSTRUCTION_BREAK_CODE(name),
1361  FOR_EACH_UNIMPLEMENTED_INSTRUCTION(ENUM_UNIMPLEMENTED_INSTRUCTION)
1362#undef ENUM_UNIMPLEMENTED_INSTRUCTION
1363};
1364
1365#define DEFINE_UNIMPLEMENTED_INSTRUCTION_VISITORS(name)                               \
1366  void InstructionCodeGeneratorARM64::Visit##name(H##name* instr ATTRIBUTE_UNUSED) {  \
1367    __ Brk(UNIMPLEMENTED_INSTRUCTION_BREAK_CODE(name));                               \
1368  }                                                                                   \
1369  void LocationsBuilderARM64::Visit##name(H##name* instr) {                           \
1370    LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instr); \
1371    locations->SetOut(Location::Any());                                               \
1372  }
1373  FOR_EACH_UNIMPLEMENTED_INSTRUCTION(DEFINE_UNIMPLEMENTED_INSTRUCTION_VISITORS)
1374#undef DEFINE_UNIMPLEMENTED_INSTRUCTION_VISITORS
1375
1376#undef UNIMPLEMENTED_INSTRUCTION_BREAK_CODE
1377#undef FOR_EACH_UNIMPLEMENTED_INSTRUCTION
1378
1379void LocationsBuilderARM64::HandleBinaryOp(HBinaryOperation* instr) {
1380  DCHECK_EQ(instr->InputCount(), 2U);
1381  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instr);
1382  Primitive::Type type = instr->GetResultType();
1383  switch (type) {
1384    case Primitive::kPrimInt:
1385    case Primitive::kPrimLong:
1386      locations->SetInAt(0, Location::RequiresRegister());
1387      locations->SetInAt(1, ARM64EncodableConstantOrRegister(instr->InputAt(1), instr));
1388      locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
1389      break;
1390
1391    case Primitive::kPrimFloat:
1392    case Primitive::kPrimDouble:
1393      locations->SetInAt(0, Location::RequiresFpuRegister());
1394      locations->SetInAt(1, Location::RequiresFpuRegister());
1395      locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap);
1396      break;
1397
1398    default:
1399      LOG(FATAL) << "Unexpected " << instr->DebugName() << " type " << type;
1400  }
1401}
1402
1403void LocationsBuilderARM64::HandleFieldGet(HInstruction* instruction) {
1404  LocationSummary* locations =
1405      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
1406  locations->SetInAt(0, Location::RequiresRegister());
1407  if (Primitive::IsFloatingPointType(instruction->GetType())) {
1408    locations->SetOut(Location::RequiresFpuRegister());
1409  } else {
1410    locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
1411  }
1412}
1413
1414void InstructionCodeGeneratorARM64::HandleFieldGet(HInstruction* instruction,
1415                                                   const FieldInfo& field_info) {
1416  DCHECK(instruction->IsInstanceFieldGet() || instruction->IsStaticFieldGet());
1417  Primitive::Type field_type = field_info.GetFieldType();
1418  BlockPoolsScope block_pools(GetVIXLAssembler());
1419
1420  MemOperand field = HeapOperand(InputRegisterAt(instruction, 0), field_info.GetFieldOffset());
1421  bool use_acquire_release = codegen_->GetInstructionSetFeatures().PreferAcquireRelease();
1422
1423  if (field_info.IsVolatile()) {
1424    if (use_acquire_release) {
1425      // NB: LoadAcquire will record the pc info if needed.
1426      codegen_->LoadAcquire(instruction, OutputCPURegister(instruction), field);
1427    } else {
1428      codegen_->Load(field_type, OutputCPURegister(instruction), field);
1429      codegen_->MaybeRecordImplicitNullCheck(instruction);
1430      // For IRIW sequential consistency kLoadAny is not sufficient.
1431      GenerateMemoryBarrier(MemBarrierKind::kAnyAny);
1432    }
1433  } else {
1434    codegen_->Load(field_type, OutputCPURegister(instruction), field);
1435    codegen_->MaybeRecordImplicitNullCheck(instruction);
1436  }
1437
1438  if (field_type == Primitive::kPrimNot) {
1439    GetAssembler()->MaybeUnpoisonHeapReference(OutputCPURegister(instruction).W());
1440  }
1441}
1442
1443void LocationsBuilderARM64::HandleFieldSet(HInstruction* instruction) {
1444  LocationSummary* locations =
1445      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
1446  locations->SetInAt(0, Location::RequiresRegister());
1447  if (Primitive::IsFloatingPointType(instruction->InputAt(1)->GetType())) {
1448    locations->SetInAt(1, Location::RequiresFpuRegister());
1449  } else {
1450    locations->SetInAt(1, Location::RequiresRegister());
1451  }
1452}
1453
1454void InstructionCodeGeneratorARM64::HandleFieldSet(HInstruction* instruction,
1455                                                   const FieldInfo& field_info,
1456                                                   bool value_can_be_null) {
1457  DCHECK(instruction->IsInstanceFieldSet() || instruction->IsStaticFieldSet());
1458  BlockPoolsScope block_pools(GetVIXLAssembler());
1459
1460  Register obj = InputRegisterAt(instruction, 0);
1461  CPURegister value = InputCPURegisterAt(instruction, 1);
1462  CPURegister source = value;
1463  Offset offset = field_info.GetFieldOffset();
1464  Primitive::Type field_type = field_info.GetFieldType();
1465  bool use_acquire_release = codegen_->GetInstructionSetFeatures().PreferAcquireRelease();
1466
1467  {
1468    // We use a block to end the scratch scope before the write barrier, thus
1469    // freeing the temporary registers so they can be used in `MarkGCCard`.
1470    UseScratchRegisterScope temps(GetVIXLAssembler());
1471
1472    if (kPoisonHeapReferences && field_type == Primitive::kPrimNot) {
1473      DCHECK(value.IsW());
1474      Register temp = temps.AcquireW();
1475      __ Mov(temp, value.W());
1476      GetAssembler()->PoisonHeapReference(temp.W());
1477      source = temp;
1478    }
1479
1480    if (field_info.IsVolatile()) {
1481      if (use_acquire_release) {
1482        codegen_->StoreRelease(field_type, source, HeapOperand(obj, offset));
1483        codegen_->MaybeRecordImplicitNullCheck(instruction);
1484      } else {
1485        GenerateMemoryBarrier(MemBarrierKind::kAnyStore);
1486        codegen_->Store(field_type, source, HeapOperand(obj, offset));
1487        codegen_->MaybeRecordImplicitNullCheck(instruction);
1488        GenerateMemoryBarrier(MemBarrierKind::kAnyAny);
1489      }
1490    } else {
1491      codegen_->Store(field_type, source, HeapOperand(obj, offset));
1492      codegen_->MaybeRecordImplicitNullCheck(instruction);
1493    }
1494  }
1495
1496  if (CodeGenerator::StoreNeedsWriteBarrier(field_type, instruction->InputAt(1))) {
1497    codegen_->MarkGCCard(obj, Register(value), value_can_be_null);
1498  }
1499}
1500
1501void InstructionCodeGeneratorARM64::HandleBinaryOp(HBinaryOperation* instr) {
1502  Primitive::Type type = instr->GetType();
1503
1504  switch (type) {
1505    case Primitive::kPrimInt:
1506    case Primitive::kPrimLong: {
1507      Register dst = OutputRegister(instr);
1508      Register lhs = InputRegisterAt(instr, 0);
1509      Operand rhs = InputOperandAt(instr, 1);
1510      if (instr->IsAdd()) {
1511        __ Add(dst, lhs, rhs);
1512      } else if (instr->IsAnd()) {
1513        __ And(dst, lhs, rhs);
1514      } else if (instr->IsOr()) {
1515        __ Orr(dst, lhs, rhs);
1516      } else if (instr->IsSub()) {
1517        __ Sub(dst, lhs, rhs);
1518      } else {
1519        DCHECK(instr->IsXor());
1520        __ Eor(dst, lhs, rhs);
1521      }
1522      break;
1523    }
1524    case Primitive::kPrimFloat:
1525    case Primitive::kPrimDouble: {
1526      FPRegister dst = OutputFPRegister(instr);
1527      FPRegister lhs = InputFPRegisterAt(instr, 0);
1528      FPRegister rhs = InputFPRegisterAt(instr, 1);
1529      if (instr->IsAdd()) {
1530        __ Fadd(dst, lhs, rhs);
1531      } else if (instr->IsSub()) {
1532        __ Fsub(dst, lhs, rhs);
1533      } else {
1534        LOG(FATAL) << "Unexpected floating-point binary operation";
1535      }
1536      break;
1537    }
1538    default:
1539      LOG(FATAL) << "Unexpected binary operation type " << type;
1540  }
1541}
1542
1543void LocationsBuilderARM64::HandleShift(HBinaryOperation* instr) {
1544  DCHECK(instr->IsShl() || instr->IsShr() || instr->IsUShr());
1545
1546  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instr);
1547  Primitive::Type type = instr->GetResultType();
1548  switch (type) {
1549    case Primitive::kPrimInt:
1550    case Primitive::kPrimLong: {
1551      locations->SetInAt(0, Location::RequiresRegister());
1552      locations->SetInAt(1, Location::RegisterOrConstant(instr->InputAt(1)));
1553      locations->SetOut(Location::RequiresRegister());
1554      break;
1555    }
1556    default:
1557      LOG(FATAL) << "Unexpected shift type " << type;
1558  }
1559}
1560
1561void InstructionCodeGeneratorARM64::HandleShift(HBinaryOperation* instr) {
1562  DCHECK(instr->IsShl() || instr->IsShr() || instr->IsUShr());
1563
1564  Primitive::Type type = instr->GetType();
1565  switch (type) {
1566    case Primitive::kPrimInt:
1567    case Primitive::kPrimLong: {
1568      Register dst = OutputRegister(instr);
1569      Register lhs = InputRegisterAt(instr, 0);
1570      Operand rhs = InputOperandAt(instr, 1);
1571      if (rhs.IsImmediate()) {
1572        uint32_t shift_value = (type == Primitive::kPrimInt)
1573          ? static_cast<uint32_t>(rhs.immediate() & kMaxIntShiftValue)
1574          : static_cast<uint32_t>(rhs.immediate() & kMaxLongShiftValue);
1575        if (instr->IsShl()) {
1576          __ Lsl(dst, lhs, shift_value);
1577        } else if (instr->IsShr()) {
1578          __ Asr(dst, lhs, shift_value);
1579        } else {
1580          __ Lsr(dst, lhs, shift_value);
1581        }
1582      } else {
1583        Register rhs_reg = dst.IsX() ? rhs.reg().X() : rhs.reg().W();
1584
1585        if (instr->IsShl()) {
1586          __ Lsl(dst, lhs, rhs_reg);
1587        } else if (instr->IsShr()) {
1588          __ Asr(dst, lhs, rhs_reg);
1589        } else {
1590          __ Lsr(dst, lhs, rhs_reg);
1591        }
1592      }
1593      break;
1594    }
1595    default:
1596      LOG(FATAL) << "Unexpected shift operation type " << type;
1597  }
1598}
1599
1600void LocationsBuilderARM64::VisitAdd(HAdd* instruction) {
1601  HandleBinaryOp(instruction);
1602}
1603
1604void InstructionCodeGeneratorARM64::VisitAdd(HAdd* instruction) {
1605  HandleBinaryOp(instruction);
1606}
1607
1608void LocationsBuilderARM64::VisitAnd(HAnd* instruction) {
1609  HandleBinaryOp(instruction);
1610}
1611
1612void InstructionCodeGeneratorARM64::VisitAnd(HAnd* instruction) {
1613  HandleBinaryOp(instruction);
1614}
1615
1616void LocationsBuilderARM64::VisitArm64IntermediateAddress(HArm64IntermediateAddress* instruction) {
1617  LocationSummary* locations =
1618      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
1619  locations->SetInAt(0, Location::RequiresRegister());
1620  locations->SetInAt(1, ARM64EncodableConstantOrRegister(instruction->GetOffset(), instruction));
1621  locations->SetOut(Location::RequiresRegister());
1622}
1623
1624void InstructionCodeGeneratorARM64::VisitArm64IntermediateAddress(
1625    HArm64IntermediateAddress* instruction) {
1626  __ Add(OutputRegister(instruction),
1627         InputRegisterAt(instruction, 0),
1628         Operand(InputOperandAt(instruction, 1)));
1629}
1630
1631void LocationsBuilderARM64::VisitArrayGet(HArrayGet* instruction) {
1632  LocationSummary* locations =
1633      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
1634  locations->SetInAt(0, Location::RequiresRegister());
1635  locations->SetInAt(1, Location::RegisterOrConstant(instruction->InputAt(1)));
1636  if (Primitive::IsFloatingPointType(instruction->GetType())) {
1637    locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap);
1638  } else {
1639    locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
1640  }
1641}
1642
1643void InstructionCodeGeneratorARM64::VisitArrayGet(HArrayGet* instruction) {
1644  Primitive::Type type = instruction->GetType();
1645  Register obj = InputRegisterAt(instruction, 0);
1646  Location index = instruction->GetLocations()->InAt(1);
1647  size_t offset = mirror::Array::DataOffset(Primitive::ComponentSize(type)).Uint32Value();
1648  MemOperand source = HeapOperand(obj);
1649  CPURegister dest = OutputCPURegister(instruction);
1650
1651  MacroAssembler* masm = GetVIXLAssembler();
1652  UseScratchRegisterScope temps(masm);
1653  // Block pools between `Load` and `MaybeRecordImplicitNullCheck`.
1654  BlockPoolsScope block_pools(masm);
1655
1656  if (index.IsConstant()) {
1657    offset += Int64ConstantFrom(index) << Primitive::ComponentSizeShift(type);
1658    source = HeapOperand(obj, offset);
1659  } else {
1660    Register temp = temps.AcquireSameSizeAs(obj);
1661    if (instruction->GetArray()->IsArm64IntermediateAddress()) {
1662      // We do not need to compute the intermediate address from the array: the
1663      // input instruction has done it already. See the comment in
1664      // `InstructionSimplifierArm64::TryExtractArrayAccessAddress()`.
1665      if (kIsDebugBuild) {
1666        HArm64IntermediateAddress* tmp = instruction->GetArray()->AsArm64IntermediateAddress();
1667        DCHECK(tmp->GetOffset()->AsIntConstant()->GetValueAsUint64() == offset);
1668      }
1669      temp = obj;
1670    } else {
1671      __ Add(temp, obj, offset);
1672    }
1673    source = HeapOperand(temp, XRegisterFrom(index), LSL, Primitive::ComponentSizeShift(type));
1674  }
1675
1676  codegen_->Load(type, dest, source);
1677  codegen_->MaybeRecordImplicitNullCheck(instruction);
1678
1679  if (instruction->GetType() == Primitive::kPrimNot) {
1680    GetAssembler()->MaybeUnpoisonHeapReference(dest.W());
1681  }
1682}
1683
1684void LocationsBuilderARM64::VisitArrayLength(HArrayLength* instruction) {
1685  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction);
1686  locations->SetInAt(0, Location::RequiresRegister());
1687  locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
1688}
1689
1690void InstructionCodeGeneratorARM64::VisitArrayLength(HArrayLength* instruction) {
1691  BlockPoolsScope block_pools(GetVIXLAssembler());
1692  __ Ldr(OutputRegister(instruction),
1693         HeapOperand(InputRegisterAt(instruction, 0), mirror::Array::LengthOffset()));
1694  codegen_->MaybeRecordImplicitNullCheck(instruction);
1695}
1696
1697void LocationsBuilderARM64::VisitArraySet(HArraySet* instruction) {
1698  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(
1699      instruction,
1700      instruction->NeedsTypeCheck() ? LocationSummary::kCallOnSlowPath : LocationSummary::kNoCall);
1701  locations->SetInAt(0, Location::RequiresRegister());
1702  locations->SetInAt(1, Location::RegisterOrConstant(instruction->InputAt(1)));
1703  if (Primitive::IsFloatingPointType(instruction->InputAt(2)->GetType())) {
1704    locations->SetInAt(2, Location::RequiresFpuRegister());
1705  } else {
1706    locations->SetInAt(2, Location::RequiresRegister());
1707  }
1708}
1709
1710void InstructionCodeGeneratorARM64::VisitArraySet(HArraySet* instruction) {
1711  Primitive::Type value_type = instruction->GetComponentType();
1712  LocationSummary* locations = instruction->GetLocations();
1713  bool may_need_runtime_call = locations->CanCall();
1714  bool needs_write_barrier =
1715      CodeGenerator::StoreNeedsWriteBarrier(value_type, instruction->GetValue());
1716
1717  Register array = InputRegisterAt(instruction, 0);
1718  CPURegister value = InputCPURegisterAt(instruction, 2);
1719  CPURegister source = value;
1720  Location index = locations->InAt(1);
1721  size_t offset = mirror::Array::DataOffset(Primitive::ComponentSize(value_type)).Uint32Value();
1722  MemOperand destination = HeapOperand(array);
1723  MacroAssembler* masm = GetVIXLAssembler();
1724  BlockPoolsScope block_pools(masm);
1725
1726  if (!needs_write_barrier) {
1727    DCHECK(!may_need_runtime_call);
1728    if (index.IsConstant()) {
1729      offset += Int64ConstantFrom(index) << Primitive::ComponentSizeShift(value_type);
1730      destination = HeapOperand(array, offset);
1731    } else {
1732      UseScratchRegisterScope temps(masm);
1733      Register temp = temps.AcquireSameSizeAs(array);
1734      if (instruction->GetArray()->IsArm64IntermediateAddress()) {
1735        // We do not need to compute the intermediate address from the array: the
1736        // input instruction has done it already. See the comment in
1737        // `InstructionSimplifierArm64::TryExtractArrayAccessAddress()`.
1738        if (kIsDebugBuild) {
1739          HArm64IntermediateAddress* tmp = instruction->GetArray()->AsArm64IntermediateAddress();
1740          DCHECK(tmp->GetOffset()->AsIntConstant()->GetValueAsUint64() == offset);
1741        }
1742        temp = array;
1743      } else {
1744        __ Add(temp, array, offset);
1745      }
1746      destination = HeapOperand(temp,
1747                                XRegisterFrom(index),
1748                                LSL,
1749                                Primitive::ComponentSizeShift(value_type));
1750    }
1751    codegen_->Store(value_type, value, destination);
1752    codegen_->MaybeRecordImplicitNullCheck(instruction);
1753  } else {
1754    DCHECK(needs_write_barrier);
1755    DCHECK(!instruction->GetArray()->IsArm64IntermediateAddress());
1756    vixl::Label done;
1757    SlowPathCodeARM64* slow_path = nullptr;
1758    {
1759      // We use a block to end the scratch scope before the write barrier, thus
1760      // freeing the temporary registers so they can be used in `MarkGCCard`.
1761      UseScratchRegisterScope temps(masm);
1762      Register temp = temps.AcquireSameSizeAs(array);
1763      if (index.IsConstant()) {
1764        offset += Int64ConstantFrom(index) << Primitive::ComponentSizeShift(value_type);
1765        destination = HeapOperand(array, offset);
1766      } else {
1767        destination = HeapOperand(temp,
1768                                  XRegisterFrom(index),
1769                                  LSL,
1770                                  Primitive::ComponentSizeShift(value_type));
1771      }
1772
1773      uint32_t class_offset = mirror::Object::ClassOffset().Int32Value();
1774      uint32_t super_offset = mirror::Class::SuperClassOffset().Int32Value();
1775      uint32_t component_offset = mirror::Class::ComponentTypeOffset().Int32Value();
1776
1777      if (may_need_runtime_call) {
1778        slow_path = new (GetGraph()->GetArena()) ArraySetSlowPathARM64(instruction);
1779        codegen_->AddSlowPath(slow_path);
1780        if (instruction->GetValueCanBeNull()) {
1781          vixl::Label non_zero;
1782          __ Cbnz(Register(value), &non_zero);
1783          if (!index.IsConstant()) {
1784            __ Add(temp, array, offset);
1785          }
1786          __ Str(wzr, destination);
1787          codegen_->MaybeRecordImplicitNullCheck(instruction);
1788          __ B(&done);
1789          __ Bind(&non_zero);
1790        }
1791
1792        Register temp2 = temps.AcquireSameSizeAs(array);
1793        __ Ldr(temp, HeapOperand(array, class_offset));
1794        codegen_->MaybeRecordImplicitNullCheck(instruction);
1795        GetAssembler()->MaybeUnpoisonHeapReference(temp);
1796        __ Ldr(temp, HeapOperand(temp, component_offset));
1797        __ Ldr(temp2, HeapOperand(Register(value), class_offset));
1798        // No need to poison/unpoison, we're comparing two poisoned references.
1799        __ Cmp(temp, temp2);
1800        if (instruction->StaticTypeOfArrayIsObjectArray()) {
1801          vixl::Label do_put;
1802          __ B(eq, &do_put);
1803          GetAssembler()->MaybeUnpoisonHeapReference(temp);
1804          __ Ldr(temp, HeapOperand(temp, super_offset));
1805          // No need to unpoison, we're comparing against null.
1806          __ Cbnz(temp, slow_path->GetEntryLabel());
1807          __ Bind(&do_put);
1808        } else {
1809          __ B(ne, slow_path->GetEntryLabel());
1810        }
1811        temps.Release(temp2);
1812      }
1813
1814      if (kPoisonHeapReferences) {
1815        Register temp2 = temps.AcquireSameSizeAs(array);
1816          DCHECK(value.IsW());
1817        __ Mov(temp2, value.W());
1818        GetAssembler()->PoisonHeapReference(temp2);
1819        source = temp2;
1820      }
1821
1822      if (!index.IsConstant()) {
1823        __ Add(temp, array, offset);
1824      }
1825      __ Str(source, destination);
1826
1827      if (!may_need_runtime_call) {
1828        codegen_->MaybeRecordImplicitNullCheck(instruction);
1829      }
1830    }
1831
1832    codegen_->MarkGCCard(array, value.W(), instruction->GetValueCanBeNull());
1833
1834    if (done.IsLinked()) {
1835      __ Bind(&done);
1836    }
1837
1838    if (slow_path != nullptr) {
1839      __ Bind(slow_path->GetExitLabel());
1840    }
1841  }
1842}
1843
1844void LocationsBuilderARM64::VisitBoundsCheck(HBoundsCheck* instruction) {
1845  LocationSummary::CallKind call_kind = instruction->CanThrowIntoCatchBlock()
1846      ? LocationSummary::kCallOnSlowPath
1847      : LocationSummary::kNoCall;
1848  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction, call_kind);
1849  locations->SetInAt(0, Location::RequiresRegister());
1850  locations->SetInAt(1, ARM64EncodableConstantOrRegister(instruction->InputAt(1), instruction));
1851  if (instruction->HasUses()) {
1852    locations->SetOut(Location::SameAsFirstInput());
1853  }
1854}
1855
1856void InstructionCodeGeneratorARM64::VisitBoundsCheck(HBoundsCheck* instruction) {
1857  BoundsCheckSlowPathARM64* slow_path =
1858      new (GetGraph()->GetArena()) BoundsCheckSlowPathARM64(instruction);
1859  codegen_->AddSlowPath(slow_path);
1860
1861  __ Cmp(InputRegisterAt(instruction, 0), InputOperandAt(instruction, 1));
1862  __ B(slow_path->GetEntryLabel(), hs);
1863}
1864
1865void LocationsBuilderARM64::VisitClinitCheck(HClinitCheck* check) {
1866  LocationSummary* locations =
1867      new (GetGraph()->GetArena()) LocationSummary(check, LocationSummary::kCallOnSlowPath);
1868  locations->SetInAt(0, Location::RequiresRegister());
1869  if (check->HasUses()) {
1870    locations->SetOut(Location::SameAsFirstInput());
1871  }
1872}
1873
1874void InstructionCodeGeneratorARM64::VisitClinitCheck(HClinitCheck* check) {
1875  // We assume the class is not null.
1876  SlowPathCodeARM64* slow_path = new (GetGraph()->GetArena()) LoadClassSlowPathARM64(
1877      check->GetLoadClass(), check, check->GetDexPc(), true);
1878  codegen_->AddSlowPath(slow_path);
1879  GenerateClassInitializationCheck(slow_path, InputRegisterAt(check, 0));
1880}
1881
1882static bool IsFloatingPointZeroConstant(HInstruction* instruction) {
1883  return (instruction->IsFloatConstant() && (instruction->AsFloatConstant()->GetValue() == 0.0f))
1884      || (instruction->IsDoubleConstant() && (instruction->AsDoubleConstant()->GetValue() == 0.0));
1885}
1886
1887void LocationsBuilderARM64::VisitCompare(HCompare* compare) {
1888  LocationSummary* locations =
1889      new (GetGraph()->GetArena()) LocationSummary(compare, LocationSummary::kNoCall);
1890  Primitive::Type in_type = compare->InputAt(0)->GetType();
1891  switch (in_type) {
1892    case Primitive::kPrimLong: {
1893      locations->SetInAt(0, Location::RequiresRegister());
1894      locations->SetInAt(1, ARM64EncodableConstantOrRegister(compare->InputAt(1), compare));
1895      locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
1896      break;
1897    }
1898    case Primitive::kPrimFloat:
1899    case Primitive::kPrimDouble: {
1900      locations->SetInAt(0, Location::RequiresFpuRegister());
1901      locations->SetInAt(1,
1902                         IsFloatingPointZeroConstant(compare->InputAt(1))
1903                             ? Location::ConstantLocation(compare->InputAt(1)->AsConstant())
1904                             : Location::RequiresFpuRegister());
1905      locations->SetOut(Location::RequiresRegister());
1906      break;
1907    }
1908    default:
1909      LOG(FATAL) << "Unexpected type for compare operation " << in_type;
1910  }
1911}
1912
1913void InstructionCodeGeneratorARM64::VisitCompare(HCompare* compare) {
1914  Primitive::Type in_type = compare->InputAt(0)->GetType();
1915
1916  //  0 if: left == right
1917  //  1 if: left  > right
1918  // -1 if: left  < right
1919  switch (in_type) {
1920    case Primitive::kPrimLong: {
1921      Register result = OutputRegister(compare);
1922      Register left = InputRegisterAt(compare, 0);
1923      Operand right = InputOperandAt(compare, 1);
1924
1925      __ Cmp(left, right);
1926      __ Cset(result, ne);
1927      __ Cneg(result, result, lt);
1928      break;
1929    }
1930    case Primitive::kPrimFloat:
1931    case Primitive::kPrimDouble: {
1932      Register result = OutputRegister(compare);
1933      FPRegister left = InputFPRegisterAt(compare, 0);
1934      if (compare->GetLocations()->InAt(1).IsConstant()) {
1935        DCHECK(IsFloatingPointZeroConstant(compare->GetLocations()->InAt(1).GetConstant()));
1936        // 0.0 is the only immediate that can be encoded directly in an FCMP instruction.
1937        __ Fcmp(left, 0.0);
1938      } else {
1939        __ Fcmp(left, InputFPRegisterAt(compare, 1));
1940      }
1941      if (compare->IsGtBias()) {
1942        __ Cset(result, ne);
1943      } else {
1944        __ Csetm(result, ne);
1945      }
1946      __ Cneg(result, result, compare->IsGtBias() ? mi : gt);
1947      break;
1948    }
1949    default:
1950      LOG(FATAL) << "Unimplemented compare type " << in_type;
1951  }
1952}
1953
1954void LocationsBuilderARM64::VisitCondition(HCondition* instruction) {
1955  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction);
1956
1957  if (Primitive::IsFloatingPointType(instruction->InputAt(0)->GetType())) {
1958    locations->SetInAt(0, Location::RequiresFpuRegister());
1959    locations->SetInAt(1,
1960                       IsFloatingPointZeroConstant(instruction->InputAt(1))
1961                           ? Location::ConstantLocation(instruction->InputAt(1)->AsConstant())
1962                           : Location::RequiresFpuRegister());
1963  } else {
1964    // Integer cases.
1965    locations->SetInAt(0, Location::RequiresRegister());
1966    locations->SetInAt(1, ARM64EncodableConstantOrRegister(instruction->InputAt(1), instruction));
1967  }
1968
1969  if (instruction->NeedsMaterialization()) {
1970    locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
1971  }
1972}
1973
1974void InstructionCodeGeneratorARM64::VisitCondition(HCondition* instruction) {
1975  if (!instruction->NeedsMaterialization()) {
1976    return;
1977  }
1978
1979  LocationSummary* locations = instruction->GetLocations();
1980  Register res = RegisterFrom(locations->Out(), instruction->GetType());
1981  IfCondition if_cond = instruction->GetCondition();
1982  Condition arm64_cond = ARM64Condition(if_cond);
1983
1984  if (Primitive::IsFloatingPointType(instruction->InputAt(0)->GetType())) {
1985    FPRegister lhs = InputFPRegisterAt(instruction, 0);
1986    if (locations->InAt(1).IsConstant()) {
1987      DCHECK(IsFloatingPointZeroConstant(locations->InAt(1).GetConstant()));
1988      // 0.0 is the only immediate that can be encoded directly in an FCMP instruction.
1989      __ Fcmp(lhs, 0.0);
1990    } else {
1991      __ Fcmp(lhs, InputFPRegisterAt(instruction, 1));
1992    }
1993    __ Cset(res, arm64_cond);
1994    if (instruction->IsFPConditionTrueIfNaN()) {
1995      // res = IsUnordered(arm64_cond) ? 1 : res  <=>  res = IsNotUnordered(arm64_cond) ? res : 1
1996      __ Csel(res, res, Operand(1), vc);  // VC for "not unordered".
1997    } else if (instruction->IsFPConditionFalseIfNaN()) {
1998      // res = IsUnordered(arm64_cond) ? 0 : res  <=>  res = IsNotUnordered(arm64_cond) ? res : 0
1999      __ Csel(res, res, Operand(0), vc);  // VC for "not unordered".
2000    }
2001  } else {
2002    // Integer cases.
2003    Register lhs = InputRegisterAt(instruction, 0);
2004    Operand rhs = InputOperandAt(instruction, 1);
2005    __ Cmp(lhs, rhs);
2006    __ Cset(res, arm64_cond);
2007  }
2008}
2009
2010#define FOR_EACH_CONDITION_INSTRUCTION(M)                                                \
2011  M(Equal)                                                                               \
2012  M(NotEqual)                                                                            \
2013  M(LessThan)                                                                            \
2014  M(LessThanOrEqual)                                                                     \
2015  M(GreaterThan)                                                                         \
2016  M(GreaterThanOrEqual)                                                                  \
2017  M(Below)                                                                               \
2018  M(BelowOrEqual)                                                                        \
2019  M(Above)                                                                               \
2020  M(AboveOrEqual)
2021#define DEFINE_CONDITION_VISITORS(Name)                                                  \
2022void LocationsBuilderARM64::Visit##Name(H##Name* comp) { VisitCondition(comp); }         \
2023void InstructionCodeGeneratorARM64::Visit##Name(H##Name* comp) { VisitCondition(comp); }
2024FOR_EACH_CONDITION_INSTRUCTION(DEFINE_CONDITION_VISITORS)
2025#undef DEFINE_CONDITION_VISITORS
2026#undef FOR_EACH_CONDITION_INSTRUCTION
2027
2028void InstructionCodeGeneratorARM64::DivRemOneOrMinusOne(HBinaryOperation* instruction) {
2029  DCHECK(instruction->IsDiv() || instruction->IsRem());
2030
2031  LocationSummary* locations = instruction->GetLocations();
2032  Location second = locations->InAt(1);
2033  DCHECK(second.IsConstant());
2034
2035  Register out = OutputRegister(instruction);
2036  Register dividend = InputRegisterAt(instruction, 0);
2037  int64_t imm = Int64FromConstant(second.GetConstant());
2038  DCHECK(imm == 1 || imm == -1);
2039
2040  if (instruction->IsRem()) {
2041    __ Mov(out, 0);
2042  } else {
2043    if (imm == 1) {
2044      __ Mov(out, dividend);
2045    } else {
2046      __ Neg(out, dividend);
2047    }
2048  }
2049}
2050
2051void InstructionCodeGeneratorARM64::DivRemByPowerOfTwo(HBinaryOperation* instruction) {
2052  DCHECK(instruction->IsDiv() || instruction->IsRem());
2053
2054  LocationSummary* locations = instruction->GetLocations();
2055  Location second = locations->InAt(1);
2056  DCHECK(second.IsConstant());
2057
2058  Register out = OutputRegister(instruction);
2059  Register dividend = InputRegisterAt(instruction, 0);
2060  int64_t imm = Int64FromConstant(second.GetConstant());
2061  uint64_t abs_imm = static_cast<uint64_t>(std::abs(imm));
2062  DCHECK(IsPowerOfTwo(abs_imm));
2063  int ctz_imm = CTZ(abs_imm);
2064
2065  UseScratchRegisterScope temps(GetVIXLAssembler());
2066  Register temp = temps.AcquireSameSizeAs(out);
2067
2068  if (instruction->IsDiv()) {
2069    __ Add(temp, dividend, abs_imm - 1);
2070    __ Cmp(dividend, 0);
2071    __ Csel(out, temp, dividend, lt);
2072    if (imm > 0) {
2073      __ Asr(out, out, ctz_imm);
2074    } else {
2075      __ Neg(out, Operand(out, ASR, ctz_imm));
2076    }
2077  } else {
2078    int bits = instruction->GetResultType() == Primitive::kPrimInt ? 32 : 64;
2079    __ Asr(temp, dividend, bits - 1);
2080    __ Lsr(temp, temp, bits - ctz_imm);
2081    __ Add(out, dividend, temp);
2082    __ And(out, out, abs_imm - 1);
2083    __ Sub(out, out, temp);
2084  }
2085}
2086
2087void InstructionCodeGeneratorARM64::GenerateDivRemWithAnyConstant(HBinaryOperation* instruction) {
2088  DCHECK(instruction->IsDiv() || instruction->IsRem());
2089
2090  LocationSummary* locations = instruction->GetLocations();
2091  Location second = locations->InAt(1);
2092  DCHECK(second.IsConstant());
2093
2094  Register out = OutputRegister(instruction);
2095  Register dividend = InputRegisterAt(instruction, 0);
2096  int64_t imm = Int64FromConstant(second.GetConstant());
2097
2098  Primitive::Type type = instruction->GetResultType();
2099  DCHECK(type == Primitive::kPrimInt || type == Primitive::kPrimLong);
2100
2101  int64_t magic;
2102  int shift;
2103  CalculateMagicAndShiftForDivRem(imm, type == Primitive::kPrimLong /* is_long */, &magic, &shift);
2104
2105  UseScratchRegisterScope temps(GetVIXLAssembler());
2106  Register temp = temps.AcquireSameSizeAs(out);
2107
2108  // temp = get_high(dividend * magic)
2109  __ Mov(temp, magic);
2110  if (type == Primitive::kPrimLong) {
2111    __ Smulh(temp, dividend, temp);
2112  } else {
2113    __ Smull(temp.X(), dividend, temp);
2114    __ Lsr(temp.X(), temp.X(), 32);
2115  }
2116
2117  if (imm > 0 && magic < 0) {
2118    __ Add(temp, temp, dividend);
2119  } else if (imm < 0 && magic > 0) {
2120    __ Sub(temp, temp, dividend);
2121  }
2122
2123  if (shift != 0) {
2124    __ Asr(temp, temp, shift);
2125  }
2126
2127  if (instruction->IsDiv()) {
2128    __ Sub(out, temp, Operand(temp, ASR, type == Primitive::kPrimLong ? 63 : 31));
2129  } else {
2130    __ Sub(temp, temp, Operand(temp, ASR, type == Primitive::kPrimLong ? 63 : 31));
2131    // TODO: Strength reduction for msub.
2132    Register temp_imm = temps.AcquireSameSizeAs(out);
2133    __ Mov(temp_imm, imm);
2134    __ Msub(out, temp, temp_imm, dividend);
2135  }
2136}
2137
2138void InstructionCodeGeneratorARM64::GenerateDivRemIntegral(HBinaryOperation* instruction) {
2139  DCHECK(instruction->IsDiv() || instruction->IsRem());
2140  Primitive::Type type = instruction->GetResultType();
2141  DCHECK(type == Primitive::kPrimInt || Primitive::kPrimLong);
2142
2143  LocationSummary* locations = instruction->GetLocations();
2144  Register out = OutputRegister(instruction);
2145  Location second = locations->InAt(1);
2146
2147  if (second.IsConstant()) {
2148    int64_t imm = Int64FromConstant(second.GetConstant());
2149
2150    if (imm == 0) {
2151      // Do not generate anything. DivZeroCheck would prevent any code to be executed.
2152    } else if (imm == 1 || imm == -1) {
2153      DivRemOneOrMinusOne(instruction);
2154    } else if (IsPowerOfTwo(std::abs(imm))) {
2155      DivRemByPowerOfTwo(instruction);
2156    } else {
2157      DCHECK(imm <= -2 || imm >= 2);
2158      GenerateDivRemWithAnyConstant(instruction);
2159    }
2160  } else {
2161    Register dividend = InputRegisterAt(instruction, 0);
2162    Register divisor = InputRegisterAt(instruction, 1);
2163    if (instruction->IsDiv()) {
2164      __ Sdiv(out, dividend, divisor);
2165    } else {
2166      UseScratchRegisterScope temps(GetVIXLAssembler());
2167      Register temp = temps.AcquireSameSizeAs(out);
2168      __ Sdiv(temp, dividend, divisor);
2169      __ Msub(out, temp, divisor, dividend);
2170    }
2171  }
2172}
2173
2174void LocationsBuilderARM64::VisitDiv(HDiv* div) {
2175  LocationSummary* locations =
2176      new (GetGraph()->GetArena()) LocationSummary(div, LocationSummary::kNoCall);
2177  switch (div->GetResultType()) {
2178    case Primitive::kPrimInt:
2179    case Primitive::kPrimLong:
2180      locations->SetInAt(0, Location::RequiresRegister());
2181      locations->SetInAt(1, Location::RegisterOrConstant(div->InputAt(1)));
2182      locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
2183      break;
2184
2185    case Primitive::kPrimFloat:
2186    case Primitive::kPrimDouble:
2187      locations->SetInAt(0, Location::RequiresFpuRegister());
2188      locations->SetInAt(1, Location::RequiresFpuRegister());
2189      locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap);
2190      break;
2191
2192    default:
2193      LOG(FATAL) << "Unexpected div type " << div->GetResultType();
2194  }
2195}
2196
2197void InstructionCodeGeneratorARM64::VisitDiv(HDiv* div) {
2198  Primitive::Type type = div->GetResultType();
2199  switch (type) {
2200    case Primitive::kPrimInt:
2201    case Primitive::kPrimLong:
2202      GenerateDivRemIntegral(div);
2203      break;
2204
2205    case Primitive::kPrimFloat:
2206    case Primitive::kPrimDouble:
2207      __ Fdiv(OutputFPRegister(div), InputFPRegisterAt(div, 0), InputFPRegisterAt(div, 1));
2208      break;
2209
2210    default:
2211      LOG(FATAL) << "Unexpected div type " << type;
2212  }
2213}
2214
2215void LocationsBuilderARM64::VisitDivZeroCheck(HDivZeroCheck* instruction) {
2216  LocationSummary::CallKind call_kind = instruction->CanThrowIntoCatchBlock()
2217      ? LocationSummary::kCallOnSlowPath
2218      : LocationSummary::kNoCall;
2219  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction, call_kind);
2220  locations->SetInAt(0, Location::RegisterOrConstant(instruction->InputAt(0)));
2221  if (instruction->HasUses()) {
2222    locations->SetOut(Location::SameAsFirstInput());
2223  }
2224}
2225
2226void InstructionCodeGeneratorARM64::VisitDivZeroCheck(HDivZeroCheck* instruction) {
2227  SlowPathCodeARM64* slow_path =
2228      new (GetGraph()->GetArena()) DivZeroCheckSlowPathARM64(instruction);
2229  codegen_->AddSlowPath(slow_path);
2230  Location value = instruction->GetLocations()->InAt(0);
2231
2232  Primitive::Type type = instruction->GetType();
2233
2234  if ((type == Primitive::kPrimBoolean) || !Primitive::IsIntegralType(type)) {
2235      LOG(FATAL) << "Unexpected type " << type << " for DivZeroCheck.";
2236    return;
2237  }
2238
2239  if (value.IsConstant()) {
2240    int64_t divisor = Int64ConstantFrom(value);
2241    if (divisor == 0) {
2242      __ B(slow_path->GetEntryLabel());
2243    } else {
2244      // A division by a non-null constant is valid. We don't need to perform
2245      // any check, so simply fall through.
2246    }
2247  } else {
2248    __ Cbz(InputRegisterAt(instruction, 0), slow_path->GetEntryLabel());
2249  }
2250}
2251
2252void LocationsBuilderARM64::VisitDoubleConstant(HDoubleConstant* constant) {
2253  LocationSummary* locations =
2254      new (GetGraph()->GetArena()) LocationSummary(constant, LocationSummary::kNoCall);
2255  locations->SetOut(Location::ConstantLocation(constant));
2256}
2257
2258void InstructionCodeGeneratorARM64::VisitDoubleConstant(
2259    HDoubleConstant* constant ATTRIBUTE_UNUSED) {
2260  // Will be generated at use site.
2261}
2262
2263void LocationsBuilderARM64::VisitExit(HExit* exit) {
2264  exit->SetLocations(nullptr);
2265}
2266
2267void InstructionCodeGeneratorARM64::VisitExit(HExit* exit ATTRIBUTE_UNUSED) {
2268}
2269
2270void LocationsBuilderARM64::VisitFloatConstant(HFloatConstant* constant) {
2271  LocationSummary* locations =
2272      new (GetGraph()->GetArena()) LocationSummary(constant, LocationSummary::kNoCall);
2273  locations->SetOut(Location::ConstantLocation(constant));
2274}
2275
2276void InstructionCodeGeneratorARM64::VisitFloatConstant(HFloatConstant* constant ATTRIBUTE_UNUSED) {
2277  // Will be generated at use site.
2278}
2279
2280void InstructionCodeGeneratorARM64::HandleGoto(HInstruction* got, HBasicBlock* successor) {
2281  DCHECK(!successor->IsExitBlock());
2282  HBasicBlock* block = got->GetBlock();
2283  HInstruction* previous = got->GetPrevious();
2284  HLoopInformation* info = block->GetLoopInformation();
2285
2286  if (info != nullptr && info->IsBackEdge(*block) && info->HasSuspendCheck()) {
2287    codegen_->ClearSpillSlotsFromLoopPhisInStackMap(info->GetSuspendCheck());
2288    GenerateSuspendCheck(info->GetSuspendCheck(), successor);
2289    return;
2290  }
2291  if (block->IsEntryBlock() && (previous != nullptr) && previous->IsSuspendCheck()) {
2292    GenerateSuspendCheck(previous->AsSuspendCheck(), nullptr);
2293  }
2294  if (!codegen_->GoesToNextBlock(block, successor)) {
2295    __ B(codegen_->GetLabelOf(successor));
2296  }
2297}
2298
2299void LocationsBuilderARM64::VisitGoto(HGoto* got) {
2300  got->SetLocations(nullptr);
2301}
2302
2303void InstructionCodeGeneratorARM64::VisitGoto(HGoto* got) {
2304  HandleGoto(got, got->GetSuccessor());
2305}
2306
2307void LocationsBuilderARM64::VisitTryBoundary(HTryBoundary* try_boundary) {
2308  try_boundary->SetLocations(nullptr);
2309}
2310
2311void InstructionCodeGeneratorARM64::VisitTryBoundary(HTryBoundary* try_boundary) {
2312  HBasicBlock* successor = try_boundary->GetNormalFlowSuccessor();
2313  if (!successor->IsExitBlock()) {
2314    HandleGoto(try_boundary, successor);
2315  }
2316}
2317
2318void InstructionCodeGeneratorARM64::GenerateTestAndBranch(HInstruction* instruction,
2319                                                          size_t condition_input_index,
2320                                                          vixl::Label* true_target,
2321                                                          vixl::Label* false_target) {
2322  // FP branching requires both targets to be explicit. If either of the targets
2323  // is nullptr (fallthrough) use and bind `fallthrough_target` instead.
2324  vixl::Label fallthrough_target;
2325  HInstruction* cond = instruction->InputAt(condition_input_index);
2326
2327  if (true_target == nullptr && false_target == nullptr) {
2328    // Nothing to do. The code always falls through.
2329    return;
2330  } else if (cond->IsIntConstant()) {
2331    // Constant condition, statically compared against 1.
2332    if (cond->AsIntConstant()->IsOne()) {
2333      if (true_target != nullptr) {
2334        __ B(true_target);
2335      }
2336    } else {
2337      DCHECK(cond->AsIntConstant()->IsZero());
2338      if (false_target != nullptr) {
2339        __ B(false_target);
2340      }
2341    }
2342    return;
2343  }
2344
2345  // The following code generates these patterns:
2346  //  (1) true_target == nullptr && false_target != nullptr
2347  //        - opposite condition true => branch to false_target
2348  //  (2) true_target != nullptr && false_target == nullptr
2349  //        - condition true => branch to true_target
2350  //  (3) true_target != nullptr && false_target != nullptr
2351  //        - condition true => branch to true_target
2352  //        - branch to false_target
2353  if (IsBooleanValueOrMaterializedCondition(cond)) {
2354    // The condition instruction has been materialized, compare the output to 0.
2355    Location cond_val = instruction->GetLocations()->InAt(condition_input_index);
2356    DCHECK(cond_val.IsRegister());
2357      if (true_target == nullptr) {
2358      __ Cbz(InputRegisterAt(instruction, condition_input_index), false_target);
2359    } else {
2360      __ Cbnz(InputRegisterAt(instruction, condition_input_index), true_target);
2361    }
2362  } else {
2363    // The condition instruction has not been materialized, use its inputs as
2364    // the comparison and its condition as the branch condition.
2365    HCondition* condition = cond->AsCondition();
2366
2367    Primitive::Type type = condition->InputAt(0)->GetType();
2368    if (Primitive::IsFloatingPointType(type)) {
2369      FPRegister lhs = InputFPRegisterAt(condition, 0);
2370      if (condition->GetLocations()->InAt(1).IsConstant()) {
2371        DCHECK(IsFloatingPointZeroConstant(condition->GetLocations()->InAt(1).GetConstant()));
2372        // 0.0 is the only immediate that can be encoded directly in an FCMP instruction.
2373        __ Fcmp(lhs, 0.0);
2374      } else {
2375        __ Fcmp(lhs, InputFPRegisterAt(condition, 1));
2376      }
2377      if (condition->IsFPConditionTrueIfNaN()) {
2378        __ B(vs, true_target == nullptr ? &fallthrough_target : true_target);
2379      } else if (condition->IsFPConditionFalseIfNaN()) {
2380        __ B(vs, false_target == nullptr ? &fallthrough_target : false_target);
2381      }
2382      if (true_target == nullptr) {
2383        __ B(ARM64Condition(condition->GetOppositeCondition()), false_target);
2384      } else {
2385        __ B(ARM64Condition(condition->GetCondition()), true_target);
2386      }
2387    } else {
2388      // Integer cases.
2389      Register lhs = InputRegisterAt(condition, 0);
2390      Operand rhs = InputOperandAt(condition, 1);
2391
2392      Condition arm64_cond;
2393      vixl::Label* non_fallthrough_target;
2394      if (true_target == nullptr) {
2395        arm64_cond = ARM64Condition(condition->GetOppositeCondition());
2396        non_fallthrough_target = false_target;
2397      } else {
2398        arm64_cond = ARM64Condition(condition->GetCondition());
2399        non_fallthrough_target = true_target;
2400      }
2401
2402      if ((arm64_cond != gt && arm64_cond != le) && rhs.IsImmediate() && (rhs.immediate() == 0)) {
2403        switch (arm64_cond) {
2404          case eq:
2405            __ Cbz(lhs, non_fallthrough_target);
2406            break;
2407          case ne:
2408            __ Cbnz(lhs, non_fallthrough_target);
2409            break;
2410          case lt:
2411            // Test the sign bit and branch accordingly.
2412            __ Tbnz(lhs, (lhs.IsX() ? kXRegSize : kWRegSize) - 1, non_fallthrough_target);
2413            break;
2414          case ge:
2415            // Test the sign bit and branch accordingly.
2416            __ Tbz(lhs, (lhs.IsX() ? kXRegSize : kWRegSize) - 1, non_fallthrough_target);
2417            break;
2418          default:
2419            // Without the `static_cast` the compiler throws an error for
2420            // `-Werror=sign-promo`.
2421            LOG(FATAL) << "Unexpected condition: " << static_cast<int>(arm64_cond);
2422        }
2423      } else {
2424        __ Cmp(lhs, rhs);
2425        __ B(arm64_cond, non_fallthrough_target);
2426      }
2427    }
2428  }
2429
2430  // If neither branch falls through (case 3), the conditional branch to `true_target`
2431  // was already emitted (case 2) and we need to emit a jump to `false_target`.
2432  if (true_target != nullptr && false_target != nullptr) {
2433    __ B(false_target);
2434  }
2435
2436  if (fallthrough_target.IsLinked()) {
2437    __ Bind(&fallthrough_target);
2438  }
2439}
2440
2441void LocationsBuilderARM64::VisitIf(HIf* if_instr) {
2442  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(if_instr);
2443  if (IsBooleanValueOrMaterializedCondition(if_instr->InputAt(0))) {
2444    locations->SetInAt(0, Location::RequiresRegister());
2445  }
2446}
2447
2448void InstructionCodeGeneratorARM64::VisitIf(HIf* if_instr) {
2449  HBasicBlock* true_successor = if_instr->IfTrueSuccessor();
2450  HBasicBlock* false_successor = if_instr->IfFalseSuccessor();
2451  vixl::Label* true_target = codegen_->GoesToNextBlock(if_instr->GetBlock(), true_successor) ?
2452      nullptr : codegen_->GetLabelOf(true_successor);
2453  vixl::Label* false_target = codegen_->GoesToNextBlock(if_instr->GetBlock(), false_successor) ?
2454      nullptr : codegen_->GetLabelOf(false_successor);
2455  GenerateTestAndBranch(if_instr, /* condition_input_index */ 0, true_target, false_target);
2456}
2457
2458void LocationsBuilderARM64::VisitDeoptimize(HDeoptimize* deoptimize) {
2459  LocationSummary* locations = new (GetGraph()->GetArena())
2460      LocationSummary(deoptimize, LocationSummary::kCallOnSlowPath);
2461  if (IsBooleanValueOrMaterializedCondition(deoptimize->InputAt(0))) {
2462    locations->SetInAt(0, Location::RequiresRegister());
2463  }
2464}
2465
2466void InstructionCodeGeneratorARM64::VisitDeoptimize(HDeoptimize* deoptimize) {
2467  SlowPathCodeARM64* slow_path = new (GetGraph()->GetArena())
2468      DeoptimizationSlowPathARM64(deoptimize);
2469  codegen_->AddSlowPath(slow_path);
2470  GenerateTestAndBranch(deoptimize,
2471                        /* condition_input_index */ 0,
2472                        slow_path->GetEntryLabel(),
2473                        /* false_target */ nullptr);
2474}
2475
2476void LocationsBuilderARM64::VisitInstanceFieldGet(HInstanceFieldGet* instruction) {
2477  HandleFieldGet(instruction);
2478}
2479
2480void InstructionCodeGeneratorARM64::VisitInstanceFieldGet(HInstanceFieldGet* instruction) {
2481  HandleFieldGet(instruction, instruction->GetFieldInfo());
2482}
2483
2484void LocationsBuilderARM64::VisitInstanceFieldSet(HInstanceFieldSet* instruction) {
2485  HandleFieldSet(instruction);
2486}
2487
2488void InstructionCodeGeneratorARM64::VisitInstanceFieldSet(HInstanceFieldSet* instruction) {
2489  HandleFieldSet(instruction, instruction->GetFieldInfo(), instruction->GetValueCanBeNull());
2490}
2491
2492void LocationsBuilderARM64::VisitInstanceOf(HInstanceOf* instruction) {
2493  LocationSummary::CallKind call_kind = LocationSummary::kNoCall;
2494  switch (instruction->GetTypeCheckKind()) {
2495    case TypeCheckKind::kExactCheck:
2496    case TypeCheckKind::kAbstractClassCheck:
2497    case TypeCheckKind::kClassHierarchyCheck:
2498    case TypeCheckKind::kArrayObjectCheck:
2499      call_kind = LocationSummary::kNoCall;
2500      break;
2501    case TypeCheckKind::kUnresolvedCheck:
2502    case TypeCheckKind::kInterfaceCheck:
2503      call_kind = LocationSummary::kCall;
2504      break;
2505    case TypeCheckKind::kArrayCheck:
2506      call_kind = LocationSummary::kCallOnSlowPath;
2507      break;
2508  }
2509  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction, call_kind);
2510  if (call_kind != LocationSummary::kCall) {
2511    locations->SetInAt(0, Location::RequiresRegister());
2512    locations->SetInAt(1, Location::RequiresRegister());
2513    // The out register is used as a temporary, so it overlaps with the inputs.
2514    // Note that TypeCheckSlowPathARM64 uses this register too.
2515    locations->SetOut(Location::RequiresRegister(), Location::kOutputOverlap);
2516  } else {
2517    InvokeRuntimeCallingConvention calling_convention;
2518    locations->SetInAt(1, LocationFrom(calling_convention.GetRegisterAt(0)));
2519    locations->SetInAt(0, LocationFrom(calling_convention.GetRegisterAt(1)));
2520    locations->SetOut(calling_convention.GetReturnLocation(Primitive::kPrimInt));
2521  }
2522}
2523
2524void InstructionCodeGeneratorARM64::VisitInstanceOf(HInstanceOf* instruction) {
2525  LocationSummary* locations = instruction->GetLocations();
2526  Register obj = InputRegisterAt(instruction, 0);
2527  Register cls = InputRegisterAt(instruction, 1);
2528  Register out = OutputRegister(instruction);
2529  uint32_t class_offset = mirror::Object::ClassOffset().Int32Value();
2530  uint32_t super_offset = mirror::Class::SuperClassOffset().Int32Value();
2531  uint32_t component_offset = mirror::Class::ComponentTypeOffset().Int32Value();
2532  uint32_t primitive_offset = mirror::Class::PrimitiveTypeOffset().Int32Value();
2533
2534  vixl::Label done, zero;
2535  SlowPathCodeARM64* slow_path = nullptr;
2536
2537  // Return 0 if `obj` is null.
2538  // Avoid null check if we know `obj` is not null.
2539  if (instruction->MustDoNullCheck()) {
2540    __ Cbz(obj, &zero);
2541  }
2542
2543  // In case of an interface/unresolved check, we put the object class into the object register.
2544  // This is safe, as the register is caller-save, and the object must be in another
2545  // register if it survives the runtime call.
2546  Register target = (instruction->GetTypeCheckKind() == TypeCheckKind::kInterfaceCheck) ||
2547      (instruction->GetTypeCheckKind() == TypeCheckKind::kUnresolvedCheck)
2548      ? obj
2549      : out;
2550  __ Ldr(target, HeapOperand(obj.W(), class_offset));
2551  GetAssembler()->MaybeUnpoisonHeapReference(target);
2552
2553  switch (instruction->GetTypeCheckKind()) {
2554    case TypeCheckKind::kExactCheck: {
2555      __ Cmp(out, cls);
2556      __ Cset(out, eq);
2557      if (zero.IsLinked()) {
2558        __ B(&done);
2559      }
2560      break;
2561    }
2562    case TypeCheckKind::kAbstractClassCheck: {
2563      // If the class is abstract, we eagerly fetch the super class of the
2564      // object to avoid doing a comparison we know will fail.
2565      vixl::Label loop, success;
2566      __ Bind(&loop);
2567      __ Ldr(out, HeapOperand(out, super_offset));
2568      GetAssembler()->MaybeUnpoisonHeapReference(out);
2569      // If `out` is null, we use it for the result, and jump to `done`.
2570      __ Cbz(out, &done);
2571      __ Cmp(out, cls);
2572      __ B(ne, &loop);
2573      __ Mov(out, 1);
2574      if (zero.IsLinked()) {
2575        __ B(&done);
2576      }
2577      break;
2578    }
2579    case TypeCheckKind::kClassHierarchyCheck: {
2580      // Walk over the class hierarchy to find a match.
2581      vixl::Label loop, success;
2582      __ Bind(&loop);
2583      __ Cmp(out, cls);
2584      __ B(eq, &success);
2585      __ Ldr(out, HeapOperand(out, super_offset));
2586      GetAssembler()->MaybeUnpoisonHeapReference(out);
2587      __ Cbnz(out, &loop);
2588      // If `out` is null, we use it for the result, and jump to `done`.
2589      __ B(&done);
2590      __ Bind(&success);
2591      __ Mov(out, 1);
2592      if (zero.IsLinked()) {
2593        __ B(&done);
2594      }
2595      break;
2596    }
2597    case TypeCheckKind::kArrayObjectCheck: {
2598      // Do an exact check.
2599      vixl::Label exact_check;
2600      __ Cmp(out, cls);
2601      __ B(eq, &exact_check);
2602      // Otherwise, we need to check that the object's class is a non primitive array.
2603      __ Ldr(out, HeapOperand(out, component_offset));
2604      GetAssembler()->MaybeUnpoisonHeapReference(out);
2605      // If `out` is null, we use it for the result, and jump to `done`.
2606      __ Cbz(out, &done);
2607      __ Ldrh(out, HeapOperand(out, primitive_offset));
2608      static_assert(Primitive::kPrimNot == 0, "Expected 0 for kPrimNot");
2609      __ Cbnz(out, &zero);
2610      __ Bind(&exact_check);
2611      __ Mov(out, 1);
2612      __ B(&done);
2613      break;
2614    }
2615    case TypeCheckKind::kArrayCheck: {
2616      __ Cmp(out, cls);
2617      DCHECK(locations->OnlyCallsOnSlowPath());
2618      slow_path = new (GetGraph()->GetArena()) TypeCheckSlowPathARM64(
2619          instruction, /* is_fatal */ false);
2620      codegen_->AddSlowPath(slow_path);
2621      __ B(ne, slow_path->GetEntryLabel());
2622      __ Mov(out, 1);
2623      if (zero.IsLinked()) {
2624        __ B(&done);
2625      }
2626      break;
2627    }
2628    case TypeCheckKind::kUnresolvedCheck:
2629    case TypeCheckKind::kInterfaceCheck:
2630    default: {
2631      codegen_->InvokeRuntime(QUICK_ENTRY_POINT(pInstanceofNonTrivial),
2632                              instruction,
2633                              instruction->GetDexPc(),
2634                              nullptr);
2635      if (zero.IsLinked()) {
2636        __ B(&done);
2637      }
2638      break;
2639    }
2640  }
2641
2642  if (zero.IsLinked()) {
2643    __ Bind(&zero);
2644    __ Mov(out, 0);
2645  }
2646
2647  if (done.IsLinked()) {
2648    __ Bind(&done);
2649  }
2650
2651  if (slow_path != nullptr) {
2652    __ Bind(slow_path->GetExitLabel());
2653  }
2654}
2655
2656void LocationsBuilderARM64::VisitCheckCast(HCheckCast* instruction) {
2657  LocationSummary::CallKind call_kind = LocationSummary::kNoCall;
2658  bool throws_into_catch = instruction->CanThrowIntoCatchBlock();
2659
2660  switch (instruction->GetTypeCheckKind()) {
2661    case TypeCheckKind::kExactCheck:
2662    case TypeCheckKind::kAbstractClassCheck:
2663    case TypeCheckKind::kClassHierarchyCheck:
2664    case TypeCheckKind::kArrayObjectCheck:
2665      call_kind = throws_into_catch
2666          ? LocationSummary::kCallOnSlowPath
2667          : LocationSummary::kNoCall;
2668      break;
2669    case TypeCheckKind::kUnresolvedCheck:
2670    case TypeCheckKind::kInterfaceCheck:
2671      call_kind = LocationSummary::kCall;
2672      break;
2673    case TypeCheckKind::kArrayCheck:
2674      call_kind = LocationSummary::kCallOnSlowPath;
2675      break;
2676  }
2677
2678  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(
2679      instruction, call_kind);
2680  if (call_kind != LocationSummary::kCall) {
2681    locations->SetInAt(0, Location::RequiresRegister());
2682    locations->SetInAt(1, Location::RequiresRegister());
2683    // Note that TypeCheckSlowPathARM64 uses this register too.
2684    locations->AddTemp(Location::RequiresRegister());
2685  } else {
2686    InvokeRuntimeCallingConvention calling_convention;
2687    locations->SetInAt(1, LocationFrom(calling_convention.GetRegisterAt(0)));
2688    locations->SetInAt(0, LocationFrom(calling_convention.GetRegisterAt(1)));
2689  }
2690}
2691
2692void InstructionCodeGeneratorARM64::VisitCheckCast(HCheckCast* instruction) {
2693  LocationSummary* locations = instruction->GetLocations();
2694  Register obj = InputRegisterAt(instruction, 0);
2695  Register cls = InputRegisterAt(instruction, 1);
2696  Register temp;
2697  if (!locations->WillCall()) {
2698    temp = WRegisterFrom(instruction->GetLocations()->GetTemp(0));
2699  }
2700
2701  uint32_t class_offset = mirror::Object::ClassOffset().Int32Value();
2702  uint32_t super_offset = mirror::Class::SuperClassOffset().Int32Value();
2703  uint32_t component_offset = mirror::Class::ComponentTypeOffset().Int32Value();
2704  uint32_t primitive_offset = mirror::Class::PrimitiveTypeOffset().Int32Value();
2705  SlowPathCodeARM64* slow_path = nullptr;
2706
2707  if (!locations->WillCall()) {
2708    slow_path = new (GetGraph()->GetArena()) TypeCheckSlowPathARM64(
2709        instruction, !locations->CanCall());
2710    codegen_->AddSlowPath(slow_path);
2711  }
2712
2713  vixl::Label done;
2714  // Avoid null check if we know obj is not null.
2715  if (instruction->MustDoNullCheck()) {
2716    __ Cbz(obj, &done);
2717  }
2718
2719  if (locations->WillCall()) {
2720    __ Ldr(obj, HeapOperand(obj, class_offset));
2721    GetAssembler()->MaybeUnpoisonHeapReference(obj);
2722  } else {
2723    __ Ldr(temp, HeapOperand(obj, class_offset));
2724    GetAssembler()->MaybeUnpoisonHeapReference(temp);
2725  }
2726
2727  switch (instruction->GetTypeCheckKind()) {
2728    case TypeCheckKind::kExactCheck:
2729    case TypeCheckKind::kArrayCheck: {
2730      __ Cmp(temp, cls);
2731      // Jump to slow path for throwing the exception or doing a
2732      // more involved array check.
2733      __ B(ne, slow_path->GetEntryLabel());
2734      break;
2735    }
2736    case TypeCheckKind::kAbstractClassCheck: {
2737      // If the class is abstract, we eagerly fetch the super class of the
2738      // object to avoid doing a comparison we know will fail.
2739      vixl::Label loop;
2740      __ Bind(&loop);
2741      __ Ldr(temp, HeapOperand(temp, super_offset));
2742      GetAssembler()->MaybeUnpoisonHeapReference(temp);
2743      // Jump to the slow path to throw the exception.
2744      __ Cbz(temp, slow_path->GetEntryLabel());
2745      __ Cmp(temp, cls);
2746      __ B(ne, &loop);
2747      break;
2748    }
2749    case TypeCheckKind::kClassHierarchyCheck: {
2750      // Walk over the class hierarchy to find a match.
2751      vixl::Label loop;
2752      __ Bind(&loop);
2753      __ Cmp(temp, cls);
2754      __ B(eq, &done);
2755      __ Ldr(temp, HeapOperand(temp, super_offset));
2756      GetAssembler()->MaybeUnpoisonHeapReference(temp);
2757      __ Cbnz(temp, &loop);
2758      // Jump to the slow path to throw the exception.
2759      __ B(slow_path->GetEntryLabel());
2760      break;
2761    }
2762    case TypeCheckKind::kArrayObjectCheck: {
2763      // Do an exact check.
2764      __ Cmp(temp, cls);
2765      __ B(eq, &done);
2766      // Otherwise, we need to check that the object's class is a non primitive array.
2767      __ Ldr(temp, HeapOperand(temp, component_offset));
2768      GetAssembler()->MaybeUnpoisonHeapReference(temp);
2769      __ Cbz(temp, slow_path->GetEntryLabel());
2770      __ Ldrh(temp, HeapOperand(temp, primitive_offset));
2771      static_assert(Primitive::kPrimNot == 0, "Expected 0 for kPrimNot");
2772      __ Cbnz(temp, slow_path->GetEntryLabel());
2773      break;
2774    }
2775    case TypeCheckKind::kUnresolvedCheck:
2776    case TypeCheckKind::kInterfaceCheck:
2777    default:
2778      codegen_->InvokeRuntime(QUICK_ENTRY_POINT(pCheckCast),
2779                              instruction,
2780                              instruction->GetDexPc(),
2781                              nullptr);
2782      break;
2783  }
2784  __ Bind(&done);
2785
2786  if (slow_path != nullptr) {
2787    __ Bind(slow_path->GetExitLabel());
2788  }
2789}
2790
2791void LocationsBuilderARM64::VisitIntConstant(HIntConstant* constant) {
2792  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(constant);
2793  locations->SetOut(Location::ConstantLocation(constant));
2794}
2795
2796void InstructionCodeGeneratorARM64::VisitIntConstant(HIntConstant* constant ATTRIBUTE_UNUSED) {
2797  // Will be generated at use site.
2798}
2799
2800void LocationsBuilderARM64::VisitNullConstant(HNullConstant* constant) {
2801  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(constant);
2802  locations->SetOut(Location::ConstantLocation(constant));
2803}
2804
2805void InstructionCodeGeneratorARM64::VisitNullConstant(HNullConstant* constant ATTRIBUTE_UNUSED) {
2806  // Will be generated at use site.
2807}
2808
2809void LocationsBuilderARM64::VisitInvokeUnresolved(HInvokeUnresolved* invoke) {
2810  // The trampoline uses the same calling convention as dex calling conventions,
2811  // except instead of loading arg0/r0 with the target Method*, arg0/r0 will contain
2812  // the method_idx.
2813  HandleInvoke(invoke);
2814}
2815
2816void InstructionCodeGeneratorARM64::VisitInvokeUnresolved(HInvokeUnresolved* invoke) {
2817  codegen_->GenerateInvokeUnresolvedRuntimeCall(invoke);
2818}
2819
2820void LocationsBuilderARM64::HandleInvoke(HInvoke* invoke) {
2821  InvokeDexCallingConventionVisitorARM64 calling_convention_visitor;
2822  CodeGenerator::CreateCommonInvokeLocationSummary(invoke, &calling_convention_visitor);
2823}
2824
2825void LocationsBuilderARM64::VisitInvokeInterface(HInvokeInterface* invoke) {
2826  HandleInvoke(invoke);
2827}
2828
2829void InstructionCodeGeneratorARM64::VisitInvokeInterface(HInvokeInterface* invoke) {
2830  // TODO: b/18116999, our IMTs can miss an IncompatibleClassChangeError.
2831  Register temp = XRegisterFrom(invoke->GetLocations()->GetTemp(0));
2832  uint32_t method_offset = mirror::Class::EmbeddedImTableEntryOffset(
2833      invoke->GetImtIndex() % mirror::Class::kImtSize, kArm64PointerSize).Uint32Value();
2834  Location receiver = invoke->GetLocations()->InAt(0);
2835  Offset class_offset = mirror::Object::ClassOffset();
2836  Offset entry_point = ArtMethod::EntryPointFromQuickCompiledCodeOffset(kArm64WordSize);
2837
2838  // The register ip1 is required to be used for the hidden argument in
2839  // art_quick_imt_conflict_trampoline, so prevent VIXL from using it.
2840  MacroAssembler* masm = GetVIXLAssembler();
2841  UseScratchRegisterScope scratch_scope(masm);
2842  BlockPoolsScope block_pools(masm);
2843  scratch_scope.Exclude(ip1);
2844  __ Mov(ip1, invoke->GetDexMethodIndex());
2845
2846  // temp = object->GetClass();
2847  if (receiver.IsStackSlot()) {
2848    __ Ldr(temp.W(), StackOperandFrom(receiver));
2849    __ Ldr(temp.W(), HeapOperand(temp.W(), class_offset));
2850  } else {
2851    __ Ldr(temp.W(), HeapOperandFrom(receiver, class_offset));
2852  }
2853  codegen_->MaybeRecordImplicitNullCheck(invoke);
2854  GetAssembler()->MaybeUnpoisonHeapReference(temp.W());
2855  // temp = temp->GetImtEntryAt(method_offset);
2856  __ Ldr(temp, MemOperand(temp, method_offset));
2857  // lr = temp->GetEntryPoint();
2858  __ Ldr(lr, MemOperand(temp, entry_point.Int32Value()));
2859  // lr();
2860  __ Blr(lr);
2861  DCHECK(!codegen_->IsLeafMethod());
2862  codegen_->RecordPcInfo(invoke, invoke->GetDexPc());
2863}
2864
2865void LocationsBuilderARM64::VisitInvokeVirtual(HInvokeVirtual* invoke) {
2866  IntrinsicLocationsBuilderARM64 intrinsic(GetGraph()->GetArena());
2867  if (intrinsic.TryDispatch(invoke)) {
2868    return;
2869  }
2870
2871  HandleInvoke(invoke);
2872}
2873
2874void LocationsBuilderARM64::VisitInvokeStaticOrDirect(HInvokeStaticOrDirect* invoke) {
2875  // When we do not run baseline, explicit clinit checks triggered by static
2876  // invokes must have been pruned by art::PrepareForRegisterAllocation.
2877  DCHECK(codegen_->IsBaseline() || !invoke->IsStaticWithExplicitClinitCheck());
2878
2879  IntrinsicLocationsBuilderARM64 intrinsic(GetGraph()->GetArena());
2880  if (intrinsic.TryDispatch(invoke)) {
2881    return;
2882  }
2883
2884  HandleInvoke(invoke);
2885}
2886
2887static bool TryGenerateIntrinsicCode(HInvoke* invoke, CodeGeneratorARM64* codegen) {
2888  if (invoke->GetLocations()->Intrinsified()) {
2889    IntrinsicCodeGeneratorARM64 intrinsic(codegen);
2890    intrinsic.Dispatch(invoke);
2891    return true;
2892  }
2893  return false;
2894}
2895
2896HInvokeStaticOrDirect::DispatchInfo CodeGeneratorARM64::GetSupportedInvokeStaticOrDirectDispatch(
2897      const HInvokeStaticOrDirect::DispatchInfo& desired_dispatch_info,
2898      MethodReference target_method ATTRIBUTE_UNUSED) {
2899  // On arm64 we support all dispatch types.
2900  return desired_dispatch_info;
2901}
2902
2903void CodeGeneratorARM64::GenerateStaticOrDirectCall(HInvokeStaticOrDirect* invoke, Location temp) {
2904  // For better instruction scheduling we load the direct code pointer before the method pointer.
2905  bool direct_code_loaded = false;
2906  switch (invoke->GetCodePtrLocation()) {
2907    case HInvokeStaticOrDirect::CodePtrLocation::kCallDirectWithFixup:
2908      // LR = code address from literal pool with link-time patch.
2909      __ Ldr(lr, DeduplicateMethodCodeLiteral(invoke->GetTargetMethod()));
2910      direct_code_loaded = true;
2911      break;
2912    case HInvokeStaticOrDirect::CodePtrLocation::kCallDirect:
2913      // LR = invoke->GetDirectCodePtr();
2914      __ Ldr(lr, DeduplicateUint64Literal(invoke->GetDirectCodePtr()));
2915      direct_code_loaded = true;
2916      break;
2917    default:
2918      break;
2919  }
2920
2921  // Make sure that ArtMethod* is passed in kArtMethodRegister as per the calling convention.
2922  Location callee_method = temp;  // For all kinds except kRecursive, callee will be in temp.
2923  switch (invoke->GetMethodLoadKind()) {
2924    case HInvokeStaticOrDirect::MethodLoadKind::kStringInit:
2925      // temp = thread->string_init_entrypoint
2926      __ Ldr(XRegisterFrom(temp), MemOperand(tr, invoke->GetStringInitOffset()));
2927      break;
2928    case HInvokeStaticOrDirect::MethodLoadKind::kRecursive:
2929      callee_method = invoke->GetLocations()->InAt(invoke->GetCurrentMethodInputIndex());
2930      break;
2931    case HInvokeStaticOrDirect::MethodLoadKind::kDirectAddress:
2932      // Load method address from literal pool.
2933      __ Ldr(XRegisterFrom(temp), DeduplicateUint64Literal(invoke->GetMethodAddress()));
2934      break;
2935    case HInvokeStaticOrDirect::MethodLoadKind::kDirectAddressWithFixup:
2936      // Load method address from literal pool with a link-time patch.
2937      __ Ldr(XRegisterFrom(temp),
2938             DeduplicateMethodAddressLiteral(invoke->GetTargetMethod()));
2939      break;
2940    case HInvokeStaticOrDirect::MethodLoadKind::kDexCachePcRelative: {
2941      // Add ADRP with its PC-relative DexCache access patch.
2942      pc_relative_dex_cache_patches_.emplace_back(*invoke->GetTargetMethod().dex_file,
2943                                                  invoke->GetDexCacheArrayOffset());
2944      vixl::Label* pc_insn_label = &pc_relative_dex_cache_patches_.back().label;
2945      {
2946        vixl::SingleEmissionCheckScope guard(GetVIXLAssembler());
2947        __ Bind(pc_insn_label);
2948        __ adrp(XRegisterFrom(temp), 0);
2949      }
2950      pc_relative_dex_cache_patches_.back().pc_insn_label = pc_insn_label;
2951      // Add LDR with its PC-relative DexCache access patch.
2952      pc_relative_dex_cache_patches_.emplace_back(*invoke->GetTargetMethod().dex_file,
2953                                                  invoke->GetDexCacheArrayOffset());
2954      {
2955        vixl::SingleEmissionCheckScope guard(GetVIXLAssembler());
2956        __ Bind(&pc_relative_dex_cache_patches_.back().label);
2957        __ ldr(XRegisterFrom(temp), MemOperand(XRegisterFrom(temp), 0));
2958        pc_relative_dex_cache_patches_.back().pc_insn_label = pc_insn_label;
2959      }
2960      break;
2961    }
2962    case HInvokeStaticOrDirect::MethodLoadKind::kDexCacheViaMethod: {
2963      Location current_method = invoke->GetLocations()->InAt(invoke->GetCurrentMethodInputIndex());
2964      Register reg = XRegisterFrom(temp);
2965      Register method_reg;
2966      if (current_method.IsRegister()) {
2967        method_reg = XRegisterFrom(current_method);
2968      } else {
2969        DCHECK(invoke->GetLocations()->Intrinsified());
2970        DCHECK(!current_method.IsValid());
2971        method_reg = reg;
2972        __ Ldr(reg.X(), MemOperand(sp, kCurrentMethodStackOffset));
2973      }
2974
2975      // temp = current_method->dex_cache_resolved_methods_;
2976      __ Ldr(reg.X(),
2977             MemOperand(method_reg.X(),
2978                        ArtMethod::DexCacheResolvedMethodsOffset(kArm64WordSize).Int32Value()));
2979      // temp = temp[index_in_cache];
2980      uint32_t index_in_cache = invoke->GetTargetMethod().dex_method_index;
2981    __ Ldr(reg.X(), MemOperand(reg.X(), GetCachePointerOffset(index_in_cache)));
2982      break;
2983    }
2984  }
2985
2986  switch (invoke->GetCodePtrLocation()) {
2987    case HInvokeStaticOrDirect::CodePtrLocation::kCallSelf:
2988      __ Bl(&frame_entry_label_);
2989      break;
2990    case HInvokeStaticOrDirect::CodePtrLocation::kCallPCRelative: {
2991      relative_call_patches_.emplace_back(invoke->GetTargetMethod());
2992      vixl::Label* label = &relative_call_patches_.back().label;
2993      vixl::SingleEmissionCheckScope guard(GetVIXLAssembler());
2994      __ Bind(label);
2995      __ bl(0);  // Branch and link to itself. This will be overriden at link time.
2996      break;
2997    }
2998    case HInvokeStaticOrDirect::CodePtrLocation::kCallDirectWithFixup:
2999    case HInvokeStaticOrDirect::CodePtrLocation::kCallDirect:
3000      // LR prepared above for better instruction scheduling.
3001      DCHECK(direct_code_loaded);
3002      // lr()
3003      __ Blr(lr);
3004      break;
3005    case HInvokeStaticOrDirect::CodePtrLocation::kCallArtMethod:
3006      // LR = callee_method->entry_point_from_quick_compiled_code_;
3007      __ Ldr(lr, MemOperand(
3008          XRegisterFrom(callee_method),
3009          ArtMethod::EntryPointFromQuickCompiledCodeOffset(kArm64WordSize).Int32Value()));
3010      // lr()
3011      __ Blr(lr);
3012      break;
3013  }
3014
3015  DCHECK(!IsLeafMethod());
3016}
3017
3018void CodeGeneratorARM64::GenerateVirtualCall(HInvokeVirtual* invoke, Location temp_in) {
3019  LocationSummary* locations = invoke->GetLocations();
3020  Location receiver = locations->InAt(0);
3021  Register temp = XRegisterFrom(temp_in);
3022  size_t method_offset = mirror::Class::EmbeddedVTableEntryOffset(
3023      invoke->GetVTableIndex(), kArm64PointerSize).SizeValue();
3024  Offset class_offset = mirror::Object::ClassOffset();
3025  Offset entry_point = ArtMethod::EntryPointFromQuickCompiledCodeOffset(kArm64WordSize);
3026
3027  BlockPoolsScope block_pools(GetVIXLAssembler());
3028
3029  DCHECK(receiver.IsRegister());
3030  __ Ldr(temp.W(), HeapOperandFrom(receiver, class_offset));
3031  MaybeRecordImplicitNullCheck(invoke);
3032  GetAssembler()->MaybeUnpoisonHeapReference(temp.W());
3033  // temp = temp->GetMethodAt(method_offset);
3034  __ Ldr(temp, MemOperand(temp, method_offset));
3035  // lr = temp->GetEntryPoint();
3036  __ Ldr(lr, MemOperand(temp, entry_point.SizeValue()));
3037  // lr();
3038  __ Blr(lr);
3039}
3040
3041void CodeGeneratorARM64::EmitLinkerPatches(ArenaVector<LinkerPatch>* linker_patches) {
3042  DCHECK(linker_patches->empty());
3043  size_t size =
3044      method_patches_.size() +
3045      call_patches_.size() +
3046      relative_call_patches_.size() +
3047      pc_relative_dex_cache_patches_.size();
3048  linker_patches->reserve(size);
3049  for (const auto& entry : method_patches_) {
3050    const MethodReference& target_method = entry.first;
3051    vixl::Literal<uint64_t>* literal = entry.second;
3052    linker_patches->push_back(LinkerPatch::MethodPatch(literal->offset(),
3053                                                       target_method.dex_file,
3054                                                       target_method.dex_method_index));
3055  }
3056  for (const auto& entry : call_patches_) {
3057    const MethodReference& target_method = entry.first;
3058    vixl::Literal<uint64_t>* literal = entry.second;
3059    linker_patches->push_back(LinkerPatch::CodePatch(literal->offset(),
3060                                                     target_method.dex_file,
3061                                                     target_method.dex_method_index));
3062  }
3063  for (const MethodPatchInfo<vixl::Label>& info : relative_call_patches_) {
3064    linker_patches->push_back(LinkerPatch::RelativeCodePatch(info.label.location(),
3065                                                             info.target_method.dex_file,
3066                                                             info.target_method.dex_method_index));
3067  }
3068  for (const PcRelativeDexCacheAccessInfo& info : pc_relative_dex_cache_patches_) {
3069    linker_patches->push_back(LinkerPatch::DexCacheArrayPatch(info.label.location(),
3070                                                              &info.target_dex_file,
3071                                                              info.pc_insn_label->location(),
3072                                                              info.element_offset));
3073  }
3074}
3075
3076vixl::Literal<uint64_t>* CodeGeneratorARM64::DeduplicateUint64Literal(uint64_t value) {
3077  // Look up the literal for value.
3078  auto lb = uint64_literals_.lower_bound(value);
3079  if (lb != uint64_literals_.end() && !uint64_literals_.key_comp()(value, lb->first)) {
3080    return lb->second;
3081  }
3082  // We don't have a literal for this value, insert a new one.
3083  vixl::Literal<uint64_t>* literal = __ CreateLiteralDestroyedWithPool<uint64_t>(value);
3084  uint64_literals_.PutBefore(lb, value, literal);
3085  return literal;
3086}
3087
3088vixl::Literal<uint64_t>* CodeGeneratorARM64::DeduplicateMethodLiteral(
3089    MethodReference target_method,
3090    MethodToLiteralMap* map) {
3091  // Look up the literal for target_method.
3092  auto lb = map->lower_bound(target_method);
3093  if (lb != map->end() && !map->key_comp()(target_method, lb->first)) {
3094    return lb->second;
3095  }
3096  // We don't have a literal for this method yet, insert a new one.
3097  vixl::Literal<uint64_t>* literal = __ CreateLiteralDestroyedWithPool<uint64_t>(0u);
3098  map->PutBefore(lb, target_method, literal);
3099  return literal;
3100}
3101
3102vixl::Literal<uint64_t>* CodeGeneratorARM64::DeduplicateMethodAddressLiteral(
3103    MethodReference target_method) {
3104  return DeduplicateMethodLiteral(target_method, &method_patches_);
3105}
3106
3107vixl::Literal<uint64_t>* CodeGeneratorARM64::DeduplicateMethodCodeLiteral(
3108    MethodReference target_method) {
3109  return DeduplicateMethodLiteral(target_method, &call_patches_);
3110}
3111
3112
3113void InstructionCodeGeneratorARM64::VisitInvokeStaticOrDirect(HInvokeStaticOrDirect* invoke) {
3114  // When we do not run baseline, explicit clinit checks triggered by static
3115  // invokes must have been pruned by art::PrepareForRegisterAllocation.
3116  DCHECK(codegen_->IsBaseline() || !invoke->IsStaticWithExplicitClinitCheck());
3117
3118  if (TryGenerateIntrinsicCode(invoke, codegen_)) {
3119    return;
3120  }
3121
3122  BlockPoolsScope block_pools(GetVIXLAssembler());
3123  LocationSummary* locations = invoke->GetLocations();
3124  codegen_->GenerateStaticOrDirectCall(
3125      invoke, locations->HasTemps() ? locations->GetTemp(0) : Location::NoLocation());
3126  codegen_->RecordPcInfo(invoke, invoke->GetDexPc());
3127}
3128
3129void InstructionCodeGeneratorARM64::VisitInvokeVirtual(HInvokeVirtual* invoke) {
3130  if (TryGenerateIntrinsicCode(invoke, codegen_)) {
3131    return;
3132  }
3133
3134  codegen_->GenerateVirtualCall(invoke, invoke->GetLocations()->GetTemp(0));
3135  DCHECK(!codegen_->IsLeafMethod());
3136  codegen_->RecordPcInfo(invoke, invoke->GetDexPc());
3137}
3138
3139void LocationsBuilderARM64::VisitLoadClass(HLoadClass* cls) {
3140  InvokeRuntimeCallingConvention calling_convention;
3141  CodeGenerator::CreateLoadClassLocationSummary(
3142      cls,
3143      LocationFrom(calling_convention.GetRegisterAt(0)),
3144      LocationFrom(vixl::x0));
3145}
3146
3147void InstructionCodeGeneratorARM64::VisitLoadClass(HLoadClass* cls) {
3148  if (cls->NeedsAccessCheck()) {
3149    codegen_->MoveConstant(cls->GetLocations()->GetTemp(0), cls->GetTypeIndex());
3150    codegen_->InvokeRuntime(QUICK_ENTRY_POINT(pInitializeTypeAndVerifyAccess),
3151                            cls,
3152                            cls->GetDexPc(),
3153                            nullptr);
3154    return;
3155  }
3156
3157  Register out = OutputRegister(cls);
3158  Register current_method = InputRegisterAt(cls, 0);
3159  if (cls->IsReferrersClass()) {
3160    DCHECK(!cls->CanCallRuntime());
3161    DCHECK(!cls->MustGenerateClinitCheck());
3162    __ Ldr(out, MemOperand(current_method, ArtMethod::DeclaringClassOffset().Int32Value()));
3163  } else {
3164    DCHECK(cls->CanCallRuntime());
3165    MemberOffset resolved_types_offset = ArtMethod::DexCacheResolvedTypesOffset(kArm64PointerSize);
3166    __ Ldr(out.X(), MemOperand(current_method, resolved_types_offset.Int32Value()));
3167    __ Ldr(out, MemOperand(out.X(), CodeGenerator::GetCacheOffset(cls->GetTypeIndex())));
3168    // TODO: We will need a read barrier here.
3169
3170    SlowPathCodeARM64* slow_path = new (GetGraph()->GetArena()) LoadClassSlowPathARM64(
3171        cls, cls, cls->GetDexPc(), cls->MustGenerateClinitCheck());
3172    codegen_->AddSlowPath(slow_path);
3173    __ Cbz(out, slow_path->GetEntryLabel());
3174    if (cls->MustGenerateClinitCheck()) {
3175      GenerateClassInitializationCheck(slow_path, out);
3176    } else {
3177      __ Bind(slow_path->GetExitLabel());
3178    }
3179  }
3180}
3181
3182static MemOperand GetExceptionTlsAddress() {
3183  return MemOperand(tr, Thread::ExceptionOffset<kArm64WordSize>().Int32Value());
3184}
3185
3186void LocationsBuilderARM64::VisitLoadException(HLoadException* load) {
3187  LocationSummary* locations =
3188      new (GetGraph()->GetArena()) LocationSummary(load, LocationSummary::kNoCall);
3189  locations->SetOut(Location::RequiresRegister());
3190}
3191
3192void InstructionCodeGeneratorARM64::VisitLoadException(HLoadException* instruction) {
3193  __ Ldr(OutputRegister(instruction), GetExceptionTlsAddress());
3194}
3195
3196void LocationsBuilderARM64::VisitClearException(HClearException* clear) {
3197  new (GetGraph()->GetArena()) LocationSummary(clear, LocationSummary::kNoCall);
3198}
3199
3200void InstructionCodeGeneratorARM64::VisitClearException(HClearException* clear ATTRIBUTE_UNUSED) {
3201  __ Str(wzr, GetExceptionTlsAddress());
3202}
3203
3204void LocationsBuilderARM64::VisitLoadLocal(HLoadLocal* load) {
3205  load->SetLocations(nullptr);
3206}
3207
3208void InstructionCodeGeneratorARM64::VisitLoadLocal(HLoadLocal* load ATTRIBUTE_UNUSED) {
3209  // Nothing to do, this is driven by the code generator.
3210}
3211
3212void LocationsBuilderARM64::VisitLoadString(HLoadString* load) {
3213  LocationSummary* locations =
3214      new (GetGraph()->GetArena()) LocationSummary(load, LocationSummary::kCallOnSlowPath);
3215  locations->SetInAt(0, Location::RequiresRegister());
3216  locations->SetOut(Location::RequiresRegister());
3217}
3218
3219void InstructionCodeGeneratorARM64::VisitLoadString(HLoadString* load) {
3220  SlowPathCodeARM64* slow_path = new (GetGraph()->GetArena()) LoadStringSlowPathARM64(load);
3221  codegen_->AddSlowPath(slow_path);
3222
3223  Register out = OutputRegister(load);
3224  Register current_method = InputRegisterAt(load, 0);
3225  __ Ldr(out, MemOperand(current_method, ArtMethod::DeclaringClassOffset().Int32Value()));
3226  __ Ldr(out.X(), HeapOperand(out, mirror::Class::DexCacheStringsOffset()));
3227  __ Ldr(out, MemOperand(out.X(), CodeGenerator::GetCacheOffset(load->GetStringIndex())));
3228  // TODO: We will need a read barrier here.
3229  __ Cbz(out, slow_path->GetEntryLabel());
3230  __ Bind(slow_path->GetExitLabel());
3231}
3232
3233void LocationsBuilderARM64::VisitLocal(HLocal* local) {
3234  local->SetLocations(nullptr);
3235}
3236
3237void InstructionCodeGeneratorARM64::VisitLocal(HLocal* local) {
3238  DCHECK_EQ(local->GetBlock(), GetGraph()->GetEntryBlock());
3239}
3240
3241void LocationsBuilderARM64::VisitLongConstant(HLongConstant* constant) {
3242  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(constant);
3243  locations->SetOut(Location::ConstantLocation(constant));
3244}
3245
3246void InstructionCodeGeneratorARM64::VisitLongConstant(HLongConstant* constant ATTRIBUTE_UNUSED) {
3247  // Will be generated at use site.
3248}
3249
3250void LocationsBuilderARM64::VisitMonitorOperation(HMonitorOperation* instruction) {
3251  LocationSummary* locations =
3252      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCall);
3253  InvokeRuntimeCallingConvention calling_convention;
3254  locations->SetInAt(0, LocationFrom(calling_convention.GetRegisterAt(0)));
3255}
3256
3257void InstructionCodeGeneratorARM64::VisitMonitorOperation(HMonitorOperation* instruction) {
3258  codegen_->InvokeRuntime(instruction->IsEnter()
3259        ? QUICK_ENTRY_POINT(pLockObject) : QUICK_ENTRY_POINT(pUnlockObject),
3260      instruction,
3261      instruction->GetDexPc(),
3262      nullptr);
3263  CheckEntrypointTypes<kQuickLockObject, void, mirror::Object*>();
3264}
3265
3266void LocationsBuilderARM64::VisitMul(HMul* mul) {
3267  LocationSummary* locations =
3268      new (GetGraph()->GetArena()) LocationSummary(mul, LocationSummary::kNoCall);
3269  switch (mul->GetResultType()) {
3270    case Primitive::kPrimInt:
3271    case Primitive::kPrimLong:
3272      locations->SetInAt(0, Location::RequiresRegister());
3273      locations->SetInAt(1, Location::RequiresRegister());
3274      locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
3275      break;
3276
3277    case Primitive::kPrimFloat:
3278    case Primitive::kPrimDouble:
3279      locations->SetInAt(0, Location::RequiresFpuRegister());
3280      locations->SetInAt(1, Location::RequiresFpuRegister());
3281      locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap);
3282      break;
3283
3284    default:
3285      LOG(FATAL) << "Unexpected mul type " << mul->GetResultType();
3286  }
3287}
3288
3289void InstructionCodeGeneratorARM64::VisitMul(HMul* mul) {
3290  switch (mul->GetResultType()) {
3291    case Primitive::kPrimInt:
3292    case Primitive::kPrimLong:
3293      __ Mul(OutputRegister(mul), InputRegisterAt(mul, 0), InputRegisterAt(mul, 1));
3294      break;
3295
3296    case Primitive::kPrimFloat:
3297    case Primitive::kPrimDouble:
3298      __ Fmul(OutputFPRegister(mul), InputFPRegisterAt(mul, 0), InputFPRegisterAt(mul, 1));
3299      break;
3300
3301    default:
3302      LOG(FATAL) << "Unexpected mul type " << mul->GetResultType();
3303  }
3304}
3305
3306void LocationsBuilderARM64::VisitNeg(HNeg* neg) {
3307  LocationSummary* locations =
3308      new (GetGraph()->GetArena()) LocationSummary(neg, LocationSummary::kNoCall);
3309  switch (neg->GetResultType()) {
3310    case Primitive::kPrimInt:
3311    case Primitive::kPrimLong:
3312      locations->SetInAt(0, ARM64EncodableConstantOrRegister(neg->InputAt(0), neg));
3313      locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
3314      break;
3315
3316    case Primitive::kPrimFloat:
3317    case Primitive::kPrimDouble:
3318      locations->SetInAt(0, Location::RequiresFpuRegister());
3319      locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap);
3320      break;
3321
3322    default:
3323      LOG(FATAL) << "Unexpected neg type " << neg->GetResultType();
3324  }
3325}
3326
3327void InstructionCodeGeneratorARM64::VisitNeg(HNeg* neg) {
3328  switch (neg->GetResultType()) {
3329    case Primitive::kPrimInt:
3330    case Primitive::kPrimLong:
3331      __ Neg(OutputRegister(neg), InputOperandAt(neg, 0));
3332      break;
3333
3334    case Primitive::kPrimFloat:
3335    case Primitive::kPrimDouble:
3336      __ Fneg(OutputFPRegister(neg), InputFPRegisterAt(neg, 0));
3337      break;
3338
3339    default:
3340      LOG(FATAL) << "Unexpected neg type " << neg->GetResultType();
3341  }
3342}
3343
3344void LocationsBuilderARM64::VisitNewArray(HNewArray* instruction) {
3345  LocationSummary* locations =
3346      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCall);
3347  InvokeRuntimeCallingConvention calling_convention;
3348  locations->AddTemp(LocationFrom(calling_convention.GetRegisterAt(0)));
3349  locations->SetOut(LocationFrom(x0));
3350  locations->SetInAt(0, LocationFrom(calling_convention.GetRegisterAt(1)));
3351  locations->SetInAt(1, LocationFrom(calling_convention.GetRegisterAt(2)));
3352  CheckEntrypointTypes<kQuickAllocArrayWithAccessCheck,
3353                       void*, uint32_t, int32_t, ArtMethod*>();
3354}
3355
3356void InstructionCodeGeneratorARM64::VisitNewArray(HNewArray* instruction) {
3357  LocationSummary* locations = instruction->GetLocations();
3358  InvokeRuntimeCallingConvention calling_convention;
3359  Register type_index = RegisterFrom(locations->GetTemp(0), Primitive::kPrimInt);
3360  DCHECK(type_index.Is(w0));
3361  __ Mov(type_index, instruction->GetTypeIndex());
3362  // Note: if heap poisoning is enabled, the entry point takes cares
3363  // of poisoning the reference.
3364  codegen_->InvokeRuntime(instruction->GetEntrypoint(),
3365                          instruction,
3366                          instruction->GetDexPc(),
3367                          nullptr);
3368  CheckEntrypointTypes<kQuickAllocArrayWithAccessCheck, void*, uint32_t, int32_t, ArtMethod*>();
3369}
3370
3371void LocationsBuilderARM64::VisitNewInstance(HNewInstance* instruction) {
3372  LocationSummary* locations =
3373      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCall);
3374  InvokeRuntimeCallingConvention calling_convention;
3375  locations->SetInAt(0, LocationFrom(calling_convention.GetRegisterAt(0)));
3376  locations->SetInAt(1, LocationFrom(calling_convention.GetRegisterAt(1)));
3377  locations->SetOut(calling_convention.GetReturnLocation(Primitive::kPrimNot));
3378  CheckEntrypointTypes<kQuickAllocObjectWithAccessCheck, void*, uint32_t, ArtMethod*>();
3379}
3380
3381void InstructionCodeGeneratorARM64::VisitNewInstance(HNewInstance* instruction) {
3382  // Note: if heap poisoning is enabled, the entry point takes cares
3383  // of poisoning the reference.
3384  codegen_->InvokeRuntime(instruction->GetEntrypoint(),
3385                          instruction,
3386                          instruction->GetDexPc(),
3387                          nullptr);
3388  CheckEntrypointTypes<kQuickAllocObjectWithAccessCheck, void*, uint32_t, ArtMethod*>();
3389}
3390
3391void LocationsBuilderARM64::VisitNot(HNot* instruction) {
3392  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction);
3393  locations->SetInAt(0, Location::RequiresRegister());
3394  locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
3395}
3396
3397void InstructionCodeGeneratorARM64::VisitNot(HNot* instruction) {
3398  switch (instruction->GetResultType()) {
3399    case Primitive::kPrimInt:
3400    case Primitive::kPrimLong:
3401      __ Mvn(OutputRegister(instruction), InputOperandAt(instruction, 0));
3402      break;
3403
3404    default:
3405      LOG(FATAL) << "Unexpected type for not operation " << instruction->GetResultType();
3406  }
3407}
3408
3409void LocationsBuilderARM64::VisitBooleanNot(HBooleanNot* instruction) {
3410  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction);
3411  locations->SetInAt(0, Location::RequiresRegister());
3412  locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
3413}
3414
3415void InstructionCodeGeneratorARM64::VisitBooleanNot(HBooleanNot* instruction) {
3416  __ Eor(OutputRegister(instruction), InputRegisterAt(instruction, 0), vixl::Operand(1));
3417}
3418
3419void LocationsBuilderARM64::VisitNullCheck(HNullCheck* instruction) {
3420  LocationSummary::CallKind call_kind = instruction->CanThrowIntoCatchBlock()
3421      ? LocationSummary::kCallOnSlowPath
3422      : LocationSummary::kNoCall;
3423  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction, call_kind);
3424  locations->SetInAt(0, Location::RequiresRegister());
3425  if (instruction->HasUses()) {
3426    locations->SetOut(Location::SameAsFirstInput());
3427  }
3428}
3429
3430void InstructionCodeGeneratorARM64::GenerateImplicitNullCheck(HNullCheck* instruction) {
3431  if (codegen_->CanMoveNullCheckToUser(instruction)) {
3432    return;
3433  }
3434
3435  BlockPoolsScope block_pools(GetVIXLAssembler());
3436  Location obj = instruction->GetLocations()->InAt(0);
3437  __ Ldr(wzr, HeapOperandFrom(obj, Offset(0)));
3438  codegen_->RecordPcInfo(instruction, instruction->GetDexPc());
3439}
3440
3441void InstructionCodeGeneratorARM64::GenerateExplicitNullCheck(HNullCheck* instruction) {
3442  SlowPathCodeARM64* slow_path = new (GetGraph()->GetArena()) NullCheckSlowPathARM64(instruction);
3443  codegen_->AddSlowPath(slow_path);
3444
3445  LocationSummary* locations = instruction->GetLocations();
3446  Location obj = locations->InAt(0);
3447
3448  __ Cbz(RegisterFrom(obj, instruction->InputAt(0)->GetType()), slow_path->GetEntryLabel());
3449}
3450
3451void InstructionCodeGeneratorARM64::VisitNullCheck(HNullCheck* instruction) {
3452  if (codegen_->IsImplicitNullCheckAllowed(instruction)) {
3453    GenerateImplicitNullCheck(instruction);
3454  } else {
3455    GenerateExplicitNullCheck(instruction);
3456  }
3457}
3458
3459void LocationsBuilderARM64::VisitOr(HOr* instruction) {
3460  HandleBinaryOp(instruction);
3461}
3462
3463void InstructionCodeGeneratorARM64::VisitOr(HOr* instruction) {
3464  HandleBinaryOp(instruction);
3465}
3466
3467void LocationsBuilderARM64::VisitParallelMove(HParallelMove* instruction ATTRIBUTE_UNUSED) {
3468  LOG(FATAL) << "Unreachable";
3469}
3470
3471void InstructionCodeGeneratorARM64::VisitParallelMove(HParallelMove* instruction) {
3472  codegen_->GetMoveResolver()->EmitNativeCode(instruction);
3473}
3474
3475void LocationsBuilderARM64::VisitParameterValue(HParameterValue* instruction) {
3476  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction);
3477  Location location = parameter_visitor_.GetNextLocation(instruction->GetType());
3478  if (location.IsStackSlot()) {
3479    location = Location::StackSlot(location.GetStackIndex() + codegen_->GetFrameSize());
3480  } else if (location.IsDoubleStackSlot()) {
3481    location = Location::DoubleStackSlot(location.GetStackIndex() + codegen_->GetFrameSize());
3482  }
3483  locations->SetOut(location);
3484}
3485
3486void InstructionCodeGeneratorARM64::VisitParameterValue(
3487    HParameterValue* instruction ATTRIBUTE_UNUSED) {
3488  // Nothing to do, the parameter is already at its location.
3489}
3490
3491void LocationsBuilderARM64::VisitCurrentMethod(HCurrentMethod* instruction) {
3492  LocationSummary* locations =
3493      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
3494  locations->SetOut(LocationFrom(kArtMethodRegister));
3495}
3496
3497void InstructionCodeGeneratorARM64::VisitCurrentMethod(
3498    HCurrentMethod* instruction ATTRIBUTE_UNUSED) {
3499  // Nothing to do, the method is already at its location.
3500}
3501
3502void LocationsBuilderARM64::VisitPhi(HPhi* instruction) {
3503  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction);
3504  for (size_t i = 0, e = instruction->InputCount(); i < e; ++i) {
3505    locations->SetInAt(i, Location::Any());
3506  }
3507  locations->SetOut(Location::Any());
3508}
3509
3510void InstructionCodeGeneratorARM64::VisitPhi(HPhi* instruction ATTRIBUTE_UNUSED) {
3511  LOG(FATAL) << "Unreachable";
3512}
3513
3514void LocationsBuilderARM64::VisitRem(HRem* rem) {
3515  Primitive::Type type = rem->GetResultType();
3516  LocationSummary::CallKind call_kind =
3517      Primitive::IsFloatingPointType(type) ? LocationSummary::kCall : LocationSummary::kNoCall;
3518  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(rem, call_kind);
3519
3520  switch (type) {
3521    case Primitive::kPrimInt:
3522    case Primitive::kPrimLong:
3523      locations->SetInAt(0, Location::RequiresRegister());
3524      locations->SetInAt(1, Location::RegisterOrConstant(rem->InputAt(1)));
3525      locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
3526      break;
3527
3528    case Primitive::kPrimFloat:
3529    case Primitive::kPrimDouble: {
3530      InvokeRuntimeCallingConvention calling_convention;
3531      locations->SetInAt(0, LocationFrom(calling_convention.GetFpuRegisterAt(0)));
3532      locations->SetInAt(1, LocationFrom(calling_convention.GetFpuRegisterAt(1)));
3533      locations->SetOut(calling_convention.GetReturnLocation(type));
3534
3535      break;
3536    }
3537
3538    default:
3539      LOG(FATAL) << "Unexpected rem type " << type;
3540  }
3541}
3542
3543void InstructionCodeGeneratorARM64::VisitRem(HRem* rem) {
3544  Primitive::Type type = rem->GetResultType();
3545
3546  switch (type) {
3547    case Primitive::kPrimInt:
3548    case Primitive::kPrimLong: {
3549      GenerateDivRemIntegral(rem);
3550      break;
3551    }
3552
3553    case Primitive::kPrimFloat:
3554    case Primitive::kPrimDouble: {
3555      int32_t entry_offset = (type == Primitive::kPrimFloat) ? QUICK_ENTRY_POINT(pFmodf)
3556                                                             : QUICK_ENTRY_POINT(pFmod);
3557      codegen_->InvokeRuntime(entry_offset, rem, rem->GetDexPc(), nullptr);
3558      break;
3559    }
3560
3561    default:
3562      LOG(FATAL) << "Unexpected rem type " << type;
3563  }
3564}
3565
3566void LocationsBuilderARM64::VisitMemoryBarrier(HMemoryBarrier* memory_barrier) {
3567  memory_barrier->SetLocations(nullptr);
3568}
3569
3570void InstructionCodeGeneratorARM64::VisitMemoryBarrier(HMemoryBarrier* memory_barrier) {
3571  GenerateMemoryBarrier(memory_barrier->GetBarrierKind());
3572}
3573
3574void LocationsBuilderARM64::VisitReturn(HReturn* instruction) {
3575  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction);
3576  Primitive::Type return_type = instruction->InputAt(0)->GetType();
3577  locations->SetInAt(0, ARM64ReturnLocation(return_type));
3578}
3579
3580void InstructionCodeGeneratorARM64::VisitReturn(HReturn* instruction ATTRIBUTE_UNUSED) {
3581  codegen_->GenerateFrameExit();
3582}
3583
3584void LocationsBuilderARM64::VisitReturnVoid(HReturnVoid* instruction) {
3585  instruction->SetLocations(nullptr);
3586}
3587
3588void InstructionCodeGeneratorARM64::VisitReturnVoid(HReturnVoid* instruction ATTRIBUTE_UNUSED) {
3589  codegen_->GenerateFrameExit();
3590}
3591
3592void LocationsBuilderARM64::VisitShl(HShl* shl) {
3593  HandleShift(shl);
3594}
3595
3596void InstructionCodeGeneratorARM64::VisitShl(HShl* shl) {
3597  HandleShift(shl);
3598}
3599
3600void LocationsBuilderARM64::VisitShr(HShr* shr) {
3601  HandleShift(shr);
3602}
3603
3604void InstructionCodeGeneratorARM64::VisitShr(HShr* shr) {
3605  HandleShift(shr);
3606}
3607
3608void LocationsBuilderARM64::VisitStoreLocal(HStoreLocal* store) {
3609  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(store);
3610  Primitive::Type field_type = store->InputAt(1)->GetType();
3611  switch (field_type) {
3612    case Primitive::kPrimNot:
3613    case Primitive::kPrimBoolean:
3614    case Primitive::kPrimByte:
3615    case Primitive::kPrimChar:
3616    case Primitive::kPrimShort:
3617    case Primitive::kPrimInt:
3618    case Primitive::kPrimFloat:
3619      locations->SetInAt(1, Location::StackSlot(codegen_->GetStackSlot(store->GetLocal())));
3620      break;
3621
3622    case Primitive::kPrimLong:
3623    case Primitive::kPrimDouble:
3624      locations->SetInAt(1, Location::DoubleStackSlot(codegen_->GetStackSlot(store->GetLocal())));
3625      break;
3626
3627    default:
3628      LOG(FATAL) << "Unimplemented local type " << field_type;
3629  }
3630}
3631
3632void InstructionCodeGeneratorARM64::VisitStoreLocal(HStoreLocal* store ATTRIBUTE_UNUSED) {
3633}
3634
3635void LocationsBuilderARM64::VisitSub(HSub* instruction) {
3636  HandleBinaryOp(instruction);
3637}
3638
3639void InstructionCodeGeneratorARM64::VisitSub(HSub* instruction) {
3640  HandleBinaryOp(instruction);
3641}
3642
3643void LocationsBuilderARM64::VisitStaticFieldGet(HStaticFieldGet* instruction) {
3644  HandleFieldGet(instruction);
3645}
3646
3647void InstructionCodeGeneratorARM64::VisitStaticFieldGet(HStaticFieldGet* instruction) {
3648  HandleFieldGet(instruction, instruction->GetFieldInfo());
3649}
3650
3651void LocationsBuilderARM64::VisitStaticFieldSet(HStaticFieldSet* instruction) {
3652  HandleFieldSet(instruction);
3653}
3654
3655void InstructionCodeGeneratorARM64::VisitStaticFieldSet(HStaticFieldSet* instruction) {
3656  HandleFieldSet(instruction, instruction->GetFieldInfo(), instruction->GetValueCanBeNull());
3657}
3658
3659void LocationsBuilderARM64::VisitUnresolvedInstanceFieldGet(
3660    HUnresolvedInstanceFieldGet* instruction) {
3661  FieldAccessCallingConventionARM64 calling_convention;
3662  codegen_->CreateUnresolvedFieldLocationSummary(
3663      instruction, instruction->GetFieldType(), calling_convention);
3664}
3665
3666void InstructionCodeGeneratorARM64::VisitUnresolvedInstanceFieldGet(
3667    HUnresolvedInstanceFieldGet* instruction) {
3668  FieldAccessCallingConventionARM64 calling_convention;
3669  codegen_->GenerateUnresolvedFieldAccess(instruction,
3670                                          instruction->GetFieldType(),
3671                                          instruction->GetFieldIndex(),
3672                                          instruction->GetDexPc(),
3673                                          calling_convention);
3674}
3675
3676void LocationsBuilderARM64::VisitUnresolvedInstanceFieldSet(
3677    HUnresolvedInstanceFieldSet* instruction) {
3678  FieldAccessCallingConventionARM64 calling_convention;
3679  codegen_->CreateUnresolvedFieldLocationSummary(
3680      instruction, instruction->GetFieldType(), calling_convention);
3681}
3682
3683void InstructionCodeGeneratorARM64::VisitUnresolvedInstanceFieldSet(
3684    HUnresolvedInstanceFieldSet* instruction) {
3685  FieldAccessCallingConventionARM64 calling_convention;
3686  codegen_->GenerateUnresolvedFieldAccess(instruction,
3687                                          instruction->GetFieldType(),
3688                                          instruction->GetFieldIndex(),
3689                                          instruction->GetDexPc(),
3690                                          calling_convention);
3691}
3692
3693void LocationsBuilderARM64::VisitUnresolvedStaticFieldGet(
3694    HUnresolvedStaticFieldGet* instruction) {
3695  FieldAccessCallingConventionARM64 calling_convention;
3696  codegen_->CreateUnresolvedFieldLocationSummary(
3697      instruction, instruction->GetFieldType(), calling_convention);
3698}
3699
3700void InstructionCodeGeneratorARM64::VisitUnresolvedStaticFieldGet(
3701    HUnresolvedStaticFieldGet* instruction) {
3702  FieldAccessCallingConventionARM64 calling_convention;
3703  codegen_->GenerateUnresolvedFieldAccess(instruction,
3704                                          instruction->GetFieldType(),
3705                                          instruction->GetFieldIndex(),
3706                                          instruction->GetDexPc(),
3707                                          calling_convention);
3708}
3709
3710void LocationsBuilderARM64::VisitUnresolvedStaticFieldSet(
3711    HUnresolvedStaticFieldSet* instruction) {
3712  FieldAccessCallingConventionARM64 calling_convention;
3713  codegen_->CreateUnresolvedFieldLocationSummary(
3714      instruction, instruction->GetFieldType(), calling_convention);
3715}
3716
3717void InstructionCodeGeneratorARM64::VisitUnresolvedStaticFieldSet(
3718    HUnresolvedStaticFieldSet* instruction) {
3719  FieldAccessCallingConventionARM64 calling_convention;
3720  codegen_->GenerateUnresolvedFieldAccess(instruction,
3721                                          instruction->GetFieldType(),
3722                                          instruction->GetFieldIndex(),
3723                                          instruction->GetDexPc(),
3724                                          calling_convention);
3725}
3726
3727void LocationsBuilderARM64::VisitSuspendCheck(HSuspendCheck* instruction) {
3728  new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCallOnSlowPath);
3729}
3730
3731void InstructionCodeGeneratorARM64::VisitSuspendCheck(HSuspendCheck* instruction) {
3732  HBasicBlock* block = instruction->GetBlock();
3733  if (block->GetLoopInformation() != nullptr) {
3734    DCHECK(block->GetLoopInformation()->GetSuspendCheck() == instruction);
3735    // The back edge will generate the suspend check.
3736    return;
3737  }
3738  if (block->IsEntryBlock() && instruction->GetNext()->IsGoto()) {
3739    // The goto will generate the suspend check.
3740    return;
3741  }
3742  GenerateSuspendCheck(instruction, nullptr);
3743}
3744
3745void LocationsBuilderARM64::VisitTemporary(HTemporary* temp) {
3746  temp->SetLocations(nullptr);
3747}
3748
3749void InstructionCodeGeneratorARM64::VisitTemporary(HTemporary* temp ATTRIBUTE_UNUSED) {
3750  // Nothing to do, this is driven by the code generator.
3751}
3752
3753void LocationsBuilderARM64::VisitThrow(HThrow* instruction) {
3754  LocationSummary* locations =
3755      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCall);
3756  InvokeRuntimeCallingConvention calling_convention;
3757  locations->SetInAt(0, LocationFrom(calling_convention.GetRegisterAt(0)));
3758}
3759
3760void InstructionCodeGeneratorARM64::VisitThrow(HThrow* instruction) {
3761  codegen_->InvokeRuntime(
3762      QUICK_ENTRY_POINT(pDeliverException), instruction, instruction->GetDexPc(), nullptr);
3763  CheckEntrypointTypes<kQuickDeliverException, void, mirror::Object*>();
3764}
3765
3766void LocationsBuilderARM64::VisitTypeConversion(HTypeConversion* conversion) {
3767  LocationSummary* locations =
3768      new (GetGraph()->GetArena()) LocationSummary(conversion, LocationSummary::kNoCall);
3769  Primitive::Type input_type = conversion->GetInputType();
3770  Primitive::Type result_type = conversion->GetResultType();
3771  DCHECK_NE(input_type, result_type);
3772  if ((input_type == Primitive::kPrimNot) || (input_type == Primitive::kPrimVoid) ||
3773      (result_type == Primitive::kPrimNot) || (result_type == Primitive::kPrimVoid)) {
3774    LOG(FATAL) << "Unexpected type conversion from " << input_type << " to " << result_type;
3775  }
3776
3777  if (Primitive::IsFloatingPointType(input_type)) {
3778    locations->SetInAt(0, Location::RequiresFpuRegister());
3779  } else {
3780    locations->SetInAt(0, Location::RequiresRegister());
3781  }
3782
3783  if (Primitive::IsFloatingPointType(result_type)) {
3784    locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap);
3785  } else {
3786    locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
3787  }
3788}
3789
3790void InstructionCodeGeneratorARM64::VisitTypeConversion(HTypeConversion* conversion) {
3791  Primitive::Type result_type = conversion->GetResultType();
3792  Primitive::Type input_type = conversion->GetInputType();
3793
3794  DCHECK_NE(input_type, result_type);
3795
3796  if (Primitive::IsIntegralType(result_type) && Primitive::IsIntegralType(input_type)) {
3797    int result_size = Primitive::ComponentSize(result_type);
3798    int input_size = Primitive::ComponentSize(input_type);
3799    int min_size = std::min(result_size, input_size);
3800    Register output = OutputRegister(conversion);
3801    Register source = InputRegisterAt(conversion, 0);
3802    if ((result_type == Primitive::kPrimChar) && (input_size < result_size)) {
3803      __ Ubfx(output, source, 0, result_size * kBitsPerByte);
3804    } else if (result_type == Primitive::kPrimInt && input_type == Primitive::kPrimLong) {
3805      // 'int' values are used directly as W registers, discarding the top
3806      // bits, so we don't need to sign-extend and can just perform a move.
3807      // We do not pass the `kDiscardForSameWReg` argument to force clearing the
3808      // top 32 bits of the target register. We theoretically could leave those
3809      // bits unchanged, but we would have to make sure that no code uses a
3810      // 32bit input value as a 64bit value assuming that the top 32 bits are
3811      // zero.
3812      __ Mov(output.W(), source.W());
3813    } else if ((result_type == Primitive::kPrimChar) ||
3814               ((input_type == Primitive::kPrimChar) && (result_size > input_size))) {
3815      __ Ubfx(output, output.IsX() ? source.X() : source.W(), 0, min_size * kBitsPerByte);
3816    } else {
3817      __ Sbfx(output, output.IsX() ? source.X() : source.W(), 0, min_size * kBitsPerByte);
3818    }
3819  } else if (Primitive::IsFloatingPointType(result_type) && Primitive::IsIntegralType(input_type)) {
3820    __ Scvtf(OutputFPRegister(conversion), InputRegisterAt(conversion, 0));
3821  } else if (Primitive::IsIntegralType(result_type) && Primitive::IsFloatingPointType(input_type)) {
3822    CHECK(result_type == Primitive::kPrimInt || result_type == Primitive::kPrimLong);
3823    __ Fcvtzs(OutputRegister(conversion), InputFPRegisterAt(conversion, 0));
3824  } else if (Primitive::IsFloatingPointType(result_type) &&
3825             Primitive::IsFloatingPointType(input_type)) {
3826    __ Fcvt(OutputFPRegister(conversion), InputFPRegisterAt(conversion, 0));
3827  } else {
3828    LOG(FATAL) << "Unexpected or unimplemented type conversion from " << input_type
3829                << " to " << result_type;
3830  }
3831}
3832
3833void LocationsBuilderARM64::VisitUShr(HUShr* ushr) {
3834  HandleShift(ushr);
3835}
3836
3837void InstructionCodeGeneratorARM64::VisitUShr(HUShr* ushr) {
3838  HandleShift(ushr);
3839}
3840
3841void LocationsBuilderARM64::VisitXor(HXor* instruction) {
3842  HandleBinaryOp(instruction);
3843}
3844
3845void InstructionCodeGeneratorARM64::VisitXor(HXor* instruction) {
3846  HandleBinaryOp(instruction);
3847}
3848
3849void LocationsBuilderARM64::VisitBoundType(HBoundType* instruction ATTRIBUTE_UNUSED) {
3850  // Nothing to do, this should be removed during prepare for register allocator.
3851  LOG(FATAL) << "Unreachable";
3852}
3853
3854void InstructionCodeGeneratorARM64::VisitBoundType(HBoundType* instruction ATTRIBUTE_UNUSED) {
3855  // Nothing to do, this should be removed during prepare for register allocator.
3856  LOG(FATAL) << "Unreachable";
3857}
3858
3859void LocationsBuilderARM64::VisitFakeString(HFakeString* instruction) {
3860  DCHECK(codegen_->IsBaseline());
3861  LocationSummary* locations =
3862      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
3863  locations->SetOut(Location::ConstantLocation(GetGraph()->GetNullConstant()));
3864}
3865
3866void InstructionCodeGeneratorARM64::VisitFakeString(HFakeString* instruction ATTRIBUTE_UNUSED) {
3867  DCHECK(codegen_->IsBaseline());
3868  // Will be generated at use site.
3869}
3870
3871// Simple implementation of packed switch - generate cascaded compare/jumps.
3872void LocationsBuilderARM64::VisitPackedSwitch(HPackedSwitch* switch_instr) {
3873  LocationSummary* locations =
3874      new (GetGraph()->GetArena()) LocationSummary(switch_instr, LocationSummary::kNoCall);
3875  locations->SetInAt(0, Location::RequiresRegister());
3876}
3877
3878void InstructionCodeGeneratorARM64::VisitPackedSwitch(HPackedSwitch* switch_instr) {
3879  int32_t lower_bound = switch_instr->GetStartValue();
3880  uint32_t num_entries = switch_instr->GetNumEntries();
3881  Register value_reg = InputRegisterAt(switch_instr, 0);
3882  HBasicBlock* default_block = switch_instr->GetDefaultBlock();
3883
3884  // Roughly set 16 as max average assemblies generated per HIR in a graph.
3885  static constexpr int32_t kMaxExpectedSizePerHInstruction = 16 * vixl::kInstructionSize;
3886  // ADR has a limited range(+/-1MB), so we set a threshold for the number of HIRs in the graph to
3887  // make sure we don't emit it if the target may run out of range.
3888  // TODO: Instead of emitting all jump tables at the end of the code, we could keep track of ADR
3889  // ranges and emit the tables only as required.
3890  static constexpr int32_t kJumpTableInstructionThreshold = 1* MB / kMaxExpectedSizePerHInstruction;
3891
3892  if (num_entries < kPackedSwitchJumpTableThreshold ||
3893      // Current instruction id is an upper bound of the number of HIRs in the graph.
3894      GetGraph()->GetCurrentInstructionId() > kJumpTableInstructionThreshold) {
3895    // Create a series of compare/jumps.
3896    const ArenaVector<HBasicBlock*>& successors = switch_instr->GetBlock()->GetSuccessors();
3897    for (uint32_t i = 0; i < num_entries; i++) {
3898      int32_t case_value = lower_bound + i;
3899      vixl::Label* succ = codegen_->GetLabelOf(successors[i]);
3900      if (case_value == 0) {
3901        __ Cbz(value_reg, succ);
3902      } else {
3903        __ Cmp(value_reg, Operand(case_value));
3904        __ B(eq, succ);
3905      }
3906    }
3907
3908    // And the default for any other value.
3909    if (!codegen_->GoesToNextBlock(switch_instr->GetBlock(), default_block)) {
3910      __ B(codegen_->GetLabelOf(default_block));
3911    }
3912  } else {
3913    JumpTableARM64* jump_table = new (GetGraph()->GetArena()) JumpTableARM64(switch_instr);
3914    codegen_->AddJumpTable(jump_table);
3915
3916    UseScratchRegisterScope temps(codegen_->GetVIXLAssembler());
3917
3918    // Below instructions should use at most one blocked register. Since there are two blocked
3919    // registers, we are free to block one.
3920    Register temp_w = temps.AcquireW();
3921    Register index;
3922    // Remove the bias.
3923    if (lower_bound != 0) {
3924      index = temp_w;
3925      __ Sub(index, value_reg, Operand(lower_bound));
3926    } else {
3927      index = value_reg;
3928    }
3929
3930    // Jump to default block if index is out of the range.
3931    __ Cmp(index, Operand(num_entries));
3932    __ B(hs, codegen_->GetLabelOf(default_block));
3933
3934    // In current VIXL implementation, it won't require any blocked registers to encode the
3935    // immediate value for Adr. So we are free to use both VIXL blocked registers to reduce the
3936    // register pressure.
3937    Register table_base = temps.AcquireX();
3938    // Load jump offset from the table.
3939    __ Adr(table_base, jump_table->GetTableStartLabel());
3940    Register jump_offset = temp_w;
3941    __ Ldr(jump_offset, MemOperand(table_base, index, UXTW, 2));
3942
3943    // Jump to target block by branching to table_base(pc related) + offset.
3944    Register target_address = table_base;
3945    __ Add(target_address, table_base, Operand(jump_offset, SXTW));
3946    __ Br(target_address);
3947  }
3948}
3949
3950#undef __
3951#undef QUICK_ENTRY_POINT
3952
3953}  // namespace arm64
3954}  // namespace art
3955