code_generator_arm64.cc revision b022fa1300e6d78639b3b910af0cf85c43df44bb 
1/*
2 * Copyright (C) 2014 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *      http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17#include "code_generator_arm64.h"
18
19#include "arch/arm64/instruction_set_features_arm64.h"
20#include "art_method.h"
21#include "code_generator_utils.h"
22#include "common_arm64.h"
23#include "compiled_method.h"
24#include "entrypoints/quick/quick_entrypoints.h"
25#include "entrypoints/quick/quick_entrypoints_enum.h"
26#include "gc/accounting/card_table.h"
27#include "intrinsics.h"
28#include "intrinsics_arm64.h"
29#include "mirror/array-inl.h"
30#include "mirror/class-inl.h"
31#include "offsets.h"
32#include "thread.h"
33#include "utils/arm64/assembler_arm64.h"
34#include "utils/assembler.h"
35#include "utils/stack_checks.h"
36
37
38using namespace vixl;   // NOLINT(build/namespaces)
39
40#ifdef __
41#error "ARM64 Codegen VIXL macro-assembler macro already defined."
42#endif
43
44namespace art {
45
46namespace arm64 {
47
48using helpers::CPURegisterFrom;
49using helpers::DRegisterFrom;
50using helpers::FPRegisterFrom;
51using helpers::HeapOperand;
52using helpers::HeapOperandFrom;
53using helpers::InputCPURegisterAt;
54using helpers::InputFPRegisterAt;
55using helpers::InputRegisterAt;
56using helpers::InputOperandAt;
57using helpers::Int64ConstantFrom;
58using helpers::LocationFrom;
59using helpers::OperandFromMemOperand;
60using helpers::OutputCPURegister;
61using helpers::OutputFPRegister;
62using helpers::OutputRegister;
63using helpers::RegisterFrom;
64using helpers::StackOperandFrom;
65using helpers::VIXLRegCodeFromART;
66using helpers::WRegisterFrom;
67using helpers::XRegisterFrom;
68using helpers::ARM64EncodableConstantOrRegister;
69using helpers::ArtVixlRegCodeCoherentForRegSet;
70
71static constexpr int kCurrentMethodStackOffset = 0;
72
73inline Condition ARM64Condition(IfCondition cond) {
74  switch (cond) {
75    case kCondEQ: return eq;
76    case kCondNE: return ne;
77    case kCondLT: return lt;
78    case kCondLE: return le;
79    case kCondGT: return gt;
80    case kCondGE: return ge;
81  }
82  LOG(FATAL) << "Unreachable";
83  UNREACHABLE();
84}
85
86Location ARM64ReturnLocation(Primitive::Type return_type) {
87  // Note that in practice, `LocationFrom(x0)` and `LocationFrom(w0)` create the
88  // same Location object, and so do `LocationFrom(d0)` and `LocationFrom(s0)`,
89  // but we use the exact registers for clarity.
90  if (return_type == Primitive::kPrimFloat) {
91    return LocationFrom(s0);
92  } else if (return_type == Primitive::kPrimDouble) {
93    return LocationFrom(d0);
94  } else if (return_type == Primitive::kPrimLong) {
95    return LocationFrom(x0);
96  } else if (return_type == Primitive::kPrimVoid) {
97    return Location::NoLocation();
98  } else {
99    return LocationFrom(w0);
100  }
101}
102
103Location InvokeRuntimeCallingConvention::GetReturnLocation(Primitive::Type return_type) {
104  return ARM64ReturnLocation(return_type);
105}
106
107#define __ down_cast<CodeGeneratorARM64*>(codegen)->GetVIXLAssembler()->
108#define QUICK_ENTRY_POINT(x) QUICK_ENTRYPOINT_OFFSET(kArm64WordSize, x).Int32Value()
109
110// Calculate memory accessing operand for save/restore live registers.
111static void SaveRestoreLiveRegistersHelper(CodeGenerator* codegen,
112                                           RegisterSet* register_set,
113                                           int64_t spill_offset,
114                                           bool is_save) {
115  DCHECK(ArtVixlRegCodeCoherentForRegSet(register_set->GetCoreRegisters(),
116                                         codegen->GetNumberOfCoreRegisters(),
117                                         register_set->GetFloatingPointRegisters(),
118                                         codegen->GetNumberOfFloatingPointRegisters()));
119
120  CPURegList core_list = CPURegList(CPURegister::kRegister, kXRegSize,
121      register_set->GetCoreRegisters() & (~callee_saved_core_registers.list()));
122  CPURegList fp_list = CPURegList(CPURegister::kFPRegister, kDRegSize,
123      register_set->GetFloatingPointRegisters() & (~callee_saved_fp_registers.list()));
124
125  MacroAssembler* masm = down_cast<CodeGeneratorARM64*>(codegen)->GetVIXLAssembler();
126  UseScratchRegisterScope temps(masm);
127
128  Register base = masm->StackPointer();
129  int64_t core_spill_size = core_list.TotalSizeInBytes();
130  int64_t fp_spill_size = fp_list.TotalSizeInBytes();
131  int64_t reg_size = kXRegSizeInBytes;
132  int64_t max_ls_pair_offset = spill_offset + core_spill_size + fp_spill_size - 2 * reg_size;
133  uint32_t ls_access_size = WhichPowerOf2(reg_size);
134  if (((core_list.Count() > 1) || (fp_list.Count() > 1)) &&
135      !masm->IsImmLSPair(max_ls_pair_offset, ls_access_size)) {
136    // If the offset does not fit in the instruction's immediate field, use an alternate register
137    // to compute the base address(float point registers spill base address).
138    Register new_base = temps.AcquireSameSizeAs(base);
139    __ Add(new_base, base, Operand(spill_offset + core_spill_size));
140    base = new_base;
141    spill_offset = -core_spill_size;
142    int64_t new_max_ls_pair_offset = fp_spill_size - 2 * reg_size;
143    DCHECK(masm->IsImmLSPair(spill_offset, ls_access_size));
144    DCHECK(masm->IsImmLSPair(new_max_ls_pair_offset, ls_access_size));
145  }
146
147  if (is_save) {
148    __ StoreCPURegList(core_list, MemOperand(base, spill_offset));
149    __ StoreCPURegList(fp_list, MemOperand(base, spill_offset + core_spill_size));
150  } else {
151    __ LoadCPURegList(core_list, MemOperand(base, spill_offset));
152    __ LoadCPURegList(fp_list, MemOperand(base, spill_offset + core_spill_size));
153  }
154}
155
156void SlowPathCodeARM64::SaveLiveRegisters(CodeGenerator* codegen, LocationSummary* locations) {
157  RegisterSet* register_set = locations->GetLiveRegisters();
158  size_t stack_offset = codegen->GetFirstRegisterSlotInSlowPath();
159  for (size_t i = 0, e = codegen->GetNumberOfCoreRegisters(); i < e; ++i) {
160    if (!codegen->IsCoreCalleeSaveRegister(i) && register_set->ContainsCoreRegister(i)) {
161      // If the register holds an object, update the stack mask.
162      if (locations->RegisterContainsObject(i)) {
163        locations->SetStackBit(stack_offset / kVRegSize);
164      }
165      DCHECK_LT(stack_offset, codegen->GetFrameSize() - codegen->FrameEntrySpillSize());
166      DCHECK_LT(i, kMaximumNumberOfExpectedRegisters);
167      saved_core_stack_offsets_[i] = stack_offset;
168      stack_offset += kXRegSizeInBytes;
169    }
170  }
171
172  for (size_t i = 0, e = codegen->GetNumberOfFloatingPointRegisters(); i < e; ++i) {
173    if (!codegen->IsFloatingPointCalleeSaveRegister(i) &&
174        register_set->ContainsFloatingPointRegister(i)) {
175      DCHECK_LT(stack_offset, codegen->GetFrameSize() - codegen->FrameEntrySpillSize());
176      DCHECK_LT(i, kMaximumNumberOfExpectedRegisters);
177      saved_fpu_stack_offsets_[i] = stack_offset;
178      stack_offset += kDRegSizeInBytes;
179    }
180  }
181
182  SaveRestoreLiveRegistersHelper(codegen, register_set,
183                                 codegen->GetFirstRegisterSlotInSlowPath(), true /* is_save */);
184}
185
186void SlowPathCodeARM64::RestoreLiveRegisters(CodeGenerator* codegen, LocationSummary* locations) {
187  RegisterSet* register_set = locations->GetLiveRegisters();
188  SaveRestoreLiveRegistersHelper(codegen, register_set,
189                                 codegen->GetFirstRegisterSlotInSlowPath(), false /* is_save */);
190}
191
192class BoundsCheckSlowPathARM64 : public SlowPathCodeARM64 {
193 public:
194  explicit BoundsCheckSlowPathARM64(HBoundsCheck* instruction) : instruction_(instruction) {}
195
196  void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
197    LocationSummary* locations = instruction_->GetLocations();
198    CodeGeneratorARM64* arm64_codegen = down_cast<CodeGeneratorARM64*>(codegen);
199
200    __ Bind(GetEntryLabel());
201    if (instruction_->CanThrowIntoCatchBlock()) {
202      // Live registers will be restored in the catch block if caught.
203      SaveLiveRegisters(codegen, instruction_->GetLocations());
204    }
205    // We're moving two locations to locations that could overlap, so we need a parallel
206    // move resolver.
207    InvokeRuntimeCallingConvention calling_convention;
208    codegen->EmitParallelMoves(
209        locations->InAt(0), LocationFrom(calling_convention.GetRegisterAt(0)), Primitive::kPrimInt,
210        locations->InAt(1), LocationFrom(calling_convention.GetRegisterAt(1)), Primitive::kPrimInt);
211    arm64_codegen->InvokeRuntime(
212        QUICK_ENTRY_POINT(pThrowArrayBounds), instruction_, instruction_->GetDexPc(), this);
213    CheckEntrypointTypes<kQuickThrowArrayBounds, void, int32_t, int32_t>();
214  }
215
216  bool IsFatal() const OVERRIDE { return true; }
217
218  const char* GetDescription() const OVERRIDE { return "BoundsCheckSlowPathARM64"; }
219
220 private:
221  HBoundsCheck* const instruction_;
222
223  DISALLOW_COPY_AND_ASSIGN(BoundsCheckSlowPathARM64);
224};
225
226class DivZeroCheckSlowPathARM64 : public SlowPathCodeARM64 {
227 public:
228  explicit DivZeroCheckSlowPathARM64(HDivZeroCheck* instruction) : instruction_(instruction) {}
229
230  void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
231    CodeGeneratorARM64* arm64_codegen = down_cast<CodeGeneratorARM64*>(codegen);
232    __ Bind(GetEntryLabel());
233    if (instruction_->CanThrowIntoCatchBlock()) {
234      // Live registers will be restored in the catch block if caught.
235      SaveLiveRegisters(codegen, instruction_->GetLocations());
236    }
237    arm64_codegen->InvokeRuntime(
238        QUICK_ENTRY_POINT(pThrowDivZero), instruction_, instruction_->GetDexPc(), this);
239    CheckEntrypointTypes<kQuickThrowDivZero, void, void>();
240  }
241
242  bool IsFatal() const OVERRIDE { return true; }
243
244  const char* GetDescription() const OVERRIDE { return "DivZeroCheckSlowPathARM64"; }
245
246 private:
247  HDivZeroCheck* const instruction_;
248  DISALLOW_COPY_AND_ASSIGN(DivZeroCheckSlowPathARM64);
249};
250
251class LoadClassSlowPathARM64 : public SlowPathCodeARM64 {
252 public:
253  LoadClassSlowPathARM64(HLoadClass* cls,
254                         HInstruction* at,
255                         uint32_t dex_pc,
256                         bool do_clinit)
257      : cls_(cls), at_(at), dex_pc_(dex_pc), do_clinit_(do_clinit) {
258    DCHECK(at->IsLoadClass() || at->IsClinitCheck());
259  }
260
261  void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
262    LocationSummary* locations = at_->GetLocations();
263    CodeGeneratorARM64* arm64_codegen = down_cast<CodeGeneratorARM64*>(codegen);
264
265    __ Bind(GetEntryLabel());
266    SaveLiveRegisters(codegen, locations);
267
268    InvokeRuntimeCallingConvention calling_convention;
269    __ Mov(calling_convention.GetRegisterAt(0).W(), cls_->GetTypeIndex());
270    int32_t entry_point_offset = do_clinit_ ? QUICK_ENTRY_POINT(pInitializeStaticStorage)
271                                            : QUICK_ENTRY_POINT(pInitializeType);
272    arm64_codegen->InvokeRuntime(entry_point_offset, at_, dex_pc_, this);
273    if (do_clinit_) {
274      CheckEntrypointTypes<kQuickInitializeStaticStorage, void*, uint32_t>();
275    } else {
276      CheckEntrypointTypes<kQuickInitializeType, void*, uint32_t>();
277    }
278
279    // Move the class to the desired location.
280    Location out = locations->Out();
281    if (out.IsValid()) {
282      DCHECK(out.IsRegister() && !locations->GetLiveRegisters()->ContainsCoreRegister(out.reg()));
283      Primitive::Type type = at_->GetType();
284      arm64_codegen->MoveLocation(out, calling_convention.GetReturnLocation(type), type);
285    }
286
287    RestoreLiveRegisters(codegen, locations);
288    __ B(GetExitLabel());
289  }
290
291  const char* GetDescription() const OVERRIDE { return "LoadClassSlowPathARM64"; }
292
293 private:
294  // The class this slow path will load.
295  HLoadClass* const cls_;
296
297  // The instruction where this slow path is happening.
298  // (Might be the load class or an initialization check).
299  HInstruction* const at_;
300
301  // The dex PC of `at_`.
302  const uint32_t dex_pc_;
303
304  // Whether to initialize the class.
305  const bool do_clinit_;
306
307  DISALLOW_COPY_AND_ASSIGN(LoadClassSlowPathARM64);
308};
309
310class LoadStringSlowPathARM64 : public SlowPathCodeARM64 {
311 public:
312  explicit LoadStringSlowPathARM64(HLoadString* instruction) : instruction_(instruction) {}
313
314  void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
315    LocationSummary* locations = instruction_->GetLocations();
316    DCHECK(!locations->GetLiveRegisters()->ContainsCoreRegister(locations->Out().reg()));
317    CodeGeneratorARM64* arm64_codegen = down_cast<CodeGeneratorARM64*>(codegen);
318
319    __ Bind(GetEntryLabel());
320    SaveLiveRegisters(codegen, locations);
321
322    InvokeRuntimeCallingConvention calling_convention;
323    __ Mov(calling_convention.GetRegisterAt(0).W(), instruction_->GetStringIndex());
324    arm64_codegen->InvokeRuntime(
325        QUICK_ENTRY_POINT(pResolveString), instruction_, instruction_->GetDexPc(), this);
326    CheckEntrypointTypes<kQuickResolveString, void*, uint32_t>();
327    Primitive::Type type = instruction_->GetType();
328    arm64_codegen->MoveLocation(locations->Out(), calling_convention.GetReturnLocation(type), type);
329
330    RestoreLiveRegisters(codegen, locations);
331    __ B(GetExitLabel());
332  }
333
334  const char* GetDescription() const OVERRIDE { return "LoadStringSlowPathARM64"; }
335
336 private:
337  HLoadString* const instruction_;
338
339  DISALLOW_COPY_AND_ASSIGN(LoadStringSlowPathARM64);
340};
341
342class NullCheckSlowPathARM64 : public SlowPathCodeARM64 {
343 public:
344  explicit NullCheckSlowPathARM64(HNullCheck* instr) : instruction_(instr) {}
345
346  void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
347    CodeGeneratorARM64* arm64_codegen = down_cast<CodeGeneratorARM64*>(codegen);
348    __ Bind(GetEntryLabel());
349    if (instruction_->CanThrowIntoCatchBlock()) {
350      // Live registers will be restored in the catch block if caught.
351      SaveLiveRegisters(codegen, instruction_->GetLocations());
352    }
353    arm64_codegen->InvokeRuntime(
354        QUICK_ENTRY_POINT(pThrowNullPointer), instruction_, instruction_->GetDexPc(), this);
355    CheckEntrypointTypes<kQuickThrowNullPointer, void, void>();
356  }
357
358  bool IsFatal() const OVERRIDE { return true; }
359
360  const char* GetDescription() const OVERRIDE { return "NullCheckSlowPathARM64"; }
361
362 private:
363  HNullCheck* const instruction_;
364
365  DISALLOW_COPY_AND_ASSIGN(NullCheckSlowPathARM64);
366};
367
368class SuspendCheckSlowPathARM64 : public SlowPathCodeARM64 {
369 public:
370  SuspendCheckSlowPathARM64(HSuspendCheck* instruction, HBasicBlock* successor)
371      : instruction_(instruction), successor_(successor) {}
372
373  void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
374    CodeGeneratorARM64* arm64_codegen = down_cast<CodeGeneratorARM64*>(codegen);
375    __ Bind(GetEntryLabel());
376    SaveLiveRegisters(codegen, instruction_->GetLocations());
377    arm64_codegen->InvokeRuntime(
378        QUICK_ENTRY_POINT(pTestSuspend), instruction_, instruction_->GetDexPc(), this);
379    CheckEntrypointTypes<kQuickTestSuspend, void, void>();
380    RestoreLiveRegisters(codegen, instruction_->GetLocations());
381    if (successor_ == nullptr) {
382      __ B(GetReturnLabel());
383    } else {
384      __ B(arm64_codegen->GetLabelOf(successor_));
385    }
386  }
387
388  vixl::Label* GetReturnLabel() {
389    DCHECK(successor_ == nullptr);
390    return &return_label_;
391  }
392
393  HBasicBlock* GetSuccessor() const {
394    return successor_;
395  }
396
397  const char* GetDescription() const OVERRIDE { return "SuspendCheckSlowPathARM64"; }
398
399 private:
400  HSuspendCheck* const instruction_;
401  // If not null, the block to branch to after the suspend check.
402  HBasicBlock* const successor_;
403
404  // If `successor_` is null, the label to branch to after the suspend check.
405  vixl::Label return_label_;
406
407  DISALLOW_COPY_AND_ASSIGN(SuspendCheckSlowPathARM64);
408};
409
410class TypeCheckSlowPathARM64 : public SlowPathCodeARM64 {
411 public:
412  explicit TypeCheckSlowPathARM64(HInstruction* instruction) : instruction_(instruction) {}
413
414  void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
415    LocationSummary* locations = instruction_->GetLocations();
416    Location class_to_check = locations->InAt(1);
417    Location object_class = instruction_->IsCheckCast() ? locations->GetTemp(0)
418                                                        : locations->Out();
419    DCHECK(instruction_->IsCheckCast()
420           || !locations->GetLiveRegisters()->ContainsCoreRegister(locations->Out().reg()));
421    CodeGeneratorARM64* arm64_codegen = down_cast<CodeGeneratorARM64*>(codegen);
422    uint32_t dex_pc = instruction_->GetDexPc();
423
424    __ Bind(GetEntryLabel());
425    SaveLiveRegisters(codegen, locations);
426
427    // We're moving two locations to locations that could overlap, so we need a parallel
428    // move resolver.
429    InvokeRuntimeCallingConvention calling_convention;
430    codegen->EmitParallelMoves(
431        class_to_check, LocationFrom(calling_convention.GetRegisterAt(0)), Primitive::kPrimNot,
432        object_class, LocationFrom(calling_convention.GetRegisterAt(1)), Primitive::kPrimNot);
433
434    if (instruction_->IsInstanceOf()) {
435      arm64_codegen->InvokeRuntime(
436          QUICK_ENTRY_POINT(pInstanceofNonTrivial), instruction_, dex_pc, this);
437      Primitive::Type ret_type = instruction_->GetType();
438      Location ret_loc = calling_convention.GetReturnLocation(ret_type);
439      arm64_codegen->MoveLocation(locations->Out(), ret_loc, ret_type);
440      CheckEntrypointTypes<kQuickInstanceofNonTrivial, uint32_t,
441                           const mirror::Class*, const mirror::Class*>();
442    } else {
443      DCHECK(instruction_->IsCheckCast());
444      arm64_codegen->InvokeRuntime(QUICK_ENTRY_POINT(pCheckCast), instruction_, dex_pc, this);
445      CheckEntrypointTypes<kQuickCheckCast, void, const mirror::Class*, const mirror::Class*>();
446    }
447
448    RestoreLiveRegisters(codegen, locations);
449    __ B(GetExitLabel());
450  }
451
452  const char* GetDescription() const OVERRIDE { return "TypeCheckSlowPathARM64"; }
453
454 private:
455  HInstruction* const instruction_;
456
457  DISALLOW_COPY_AND_ASSIGN(TypeCheckSlowPathARM64);
458};
459
460class DeoptimizationSlowPathARM64 : public SlowPathCodeARM64 {
461 public:
462  explicit DeoptimizationSlowPathARM64(HInstruction* instruction)
463    : instruction_(instruction) {}
464
465  void EmitNativeCode(CodeGenerator* codegen) OVERRIDE {
466    __ Bind(GetEntryLabel());
467    SaveLiveRegisters(codegen, instruction_->GetLocations());
468    DCHECK(instruction_->IsDeoptimize());
469    HDeoptimize* deoptimize = instruction_->AsDeoptimize();
470    uint32_t dex_pc = deoptimize->GetDexPc();
471    CodeGeneratorARM64* arm64_codegen = down_cast<CodeGeneratorARM64*>(codegen);
472    arm64_codegen->InvokeRuntime(QUICK_ENTRY_POINT(pDeoptimize), instruction_, dex_pc, this);
473  }
474
475  const char* GetDescription() const OVERRIDE { return "DeoptimizationSlowPathARM64"; }
476
477 private:
478  HInstruction* const instruction_;
479  DISALLOW_COPY_AND_ASSIGN(DeoptimizationSlowPathARM64);
480};
481
482#undef __
483
484Location InvokeDexCallingConventionVisitorARM64::GetNextLocation(Primitive::Type type) {
485  Location next_location;
486  if (type == Primitive::kPrimVoid) {
487    LOG(FATAL) << "Unreachable type " << type;
488  }
489
490  if (Primitive::IsFloatingPointType(type) &&
491      (float_index_ < calling_convention.GetNumberOfFpuRegisters())) {
492    next_location = LocationFrom(calling_convention.GetFpuRegisterAt(float_index_++));
493  } else if (!Primitive::IsFloatingPointType(type) &&
494             (gp_index_ < calling_convention.GetNumberOfRegisters())) {
495    next_location = LocationFrom(calling_convention.GetRegisterAt(gp_index_++));
496  } else {
497    size_t stack_offset = calling_convention.GetStackOffsetOf(stack_index_);
498    next_location = Primitive::Is64BitType(type) ? Location::DoubleStackSlot(stack_offset)
499                                                 : Location::StackSlot(stack_offset);
500  }
501
502  // Space on the stack is reserved for all arguments.
503  stack_index_ += Primitive::Is64BitType(type) ? 2 : 1;
504  return next_location;
505}
506
507Location InvokeDexCallingConventionVisitorARM64::GetMethodLocation() const {
508  return LocationFrom(kArtMethodRegister);
509}
510
511CodeGeneratorARM64::CodeGeneratorARM64(HGraph* graph,
512                                       const Arm64InstructionSetFeatures& isa_features,
513                                       const CompilerOptions& compiler_options)
514    : CodeGenerator(graph,
515                    kNumberOfAllocatableRegisters,
516                    kNumberOfAllocatableFPRegisters,
517                    kNumberOfAllocatableRegisterPairs,
518                    callee_saved_core_registers.list(),
519                    callee_saved_fp_registers.list(),
520                    compiler_options),
521      block_labels_(nullptr),
522      location_builder_(graph, this),
523      instruction_visitor_(graph, this),
524      move_resolver_(graph->GetArena(), this),
525      isa_features_(isa_features),
526      uint64_literals_(std::less<uint64_t>(), graph->GetArena()->Adapter()),
527      method_patches_(MethodReferenceComparator(), graph->GetArena()->Adapter()),
528      call_patches_(MethodReferenceComparator(), graph->GetArena()->Adapter()),
529      relative_call_patches_(graph->GetArena()->Adapter()),
530      pc_rel_dex_cache_patches_(graph->GetArena()->Adapter()) {
531  // Save the link register (containing the return address) to mimic Quick.
532  AddAllocatedRegister(LocationFrom(lr));
533}
534
535#undef __
536#define __ GetVIXLAssembler()->
537
538void CodeGeneratorARM64::Finalize(CodeAllocator* allocator) {
539  // Ensure we emit the literal pool.
540  __ FinalizeCode();
541
542  CodeGenerator::Finalize(allocator);
543}
544
545void ParallelMoveResolverARM64::PrepareForEmitNativeCode() {
546  // Note: There are 6 kinds of moves:
547  // 1. constant -> GPR/FPR (non-cycle)
548  // 2. constant -> stack (non-cycle)
549  // 3. GPR/FPR -> GPR/FPR
550  // 4. GPR/FPR -> stack
551  // 5. stack -> GPR/FPR
552  // 6. stack -> stack (non-cycle)
553  // Case 1, 2 and 6 should never be included in a dependency cycle on ARM64. For case 3, 4, and 5
554  // VIXL uses at most 1 GPR. VIXL has 2 GPR and 1 FPR temps, and there should be no intersecting
555  // cycles on ARM64, so we always have 1 GPR and 1 FPR available VIXL temps to resolve the
556  // dependency.
557  vixl_temps_.Open(GetVIXLAssembler());
558}
559
560void ParallelMoveResolverARM64::FinishEmitNativeCode() {
561  vixl_temps_.Close();
562}
563
564Location ParallelMoveResolverARM64::AllocateScratchLocationFor(Location::Kind kind) {
565  DCHECK(kind == Location::kRegister || kind == Location::kFpuRegister ||
566         kind == Location::kStackSlot || kind == Location::kDoubleStackSlot);
567  kind = (kind == Location::kFpuRegister) ? Location::kFpuRegister : Location::kRegister;
568  Location scratch = GetScratchLocation(kind);
569  if (!scratch.Equals(Location::NoLocation())) {
570    return scratch;
571  }
572  // Allocate from VIXL temp registers.
573  if (kind == Location::kRegister) {
574    scratch = LocationFrom(vixl_temps_.AcquireX());
575  } else {
576    DCHECK(kind == Location::kFpuRegister);
577    scratch = LocationFrom(vixl_temps_.AcquireD());
578  }
579  AddScratchLocation(scratch);
580  return scratch;
581}
582
583void ParallelMoveResolverARM64::FreeScratchLocation(Location loc) {
584  if (loc.IsRegister()) {
585    vixl_temps_.Release(XRegisterFrom(loc));
586  } else {
587    DCHECK(loc.IsFpuRegister());
588    vixl_temps_.Release(DRegisterFrom(loc));
589  }
590  RemoveScratchLocation(loc);
591}
592
593void ParallelMoveResolverARM64::EmitMove(size_t index) {
594  MoveOperands* move = moves_.Get(index);
595  codegen_->MoveLocation(move->GetDestination(), move->GetSource());
596}
597
598void CodeGeneratorARM64::GenerateFrameEntry() {
599  MacroAssembler* masm = GetVIXLAssembler();
600  BlockPoolsScope block_pools(masm);
601  __ Bind(&frame_entry_label_);
602
603  bool do_overflow_check = FrameNeedsStackCheck(GetFrameSize(), kArm64) || !IsLeafMethod();
604  if (do_overflow_check) {
605    UseScratchRegisterScope temps(masm);
606    Register temp = temps.AcquireX();
607    DCHECK(GetCompilerOptions().GetImplicitStackOverflowChecks());
608    __ Sub(temp, sp, static_cast<int32_t>(GetStackOverflowReservedBytes(kArm64)));
609    __ Ldr(wzr, MemOperand(temp, 0));
610    RecordPcInfo(nullptr, 0);
611  }
612
613  if (!HasEmptyFrame()) {
614    int frame_size = GetFrameSize();
615    // Stack layout:
616    //      sp[frame_size - 8]        : lr.
617    //      ...                       : other preserved core registers.
618    //      ...                       : other preserved fp registers.
619    //      ...                       : reserved frame space.
620    //      sp[0]                     : current method.
621    __ Str(kArtMethodRegister, MemOperand(sp, -frame_size, PreIndex));
622    GetAssembler()->cfi().AdjustCFAOffset(frame_size);
623    GetAssembler()->SpillRegisters(GetFramePreservedCoreRegisters(),
624        frame_size - GetCoreSpillSize());
625    GetAssembler()->SpillRegisters(GetFramePreservedFPRegisters(),
626        frame_size - FrameEntrySpillSize());
627  }
628}
629
630void CodeGeneratorARM64::GenerateFrameExit() {
631  BlockPoolsScope block_pools(GetVIXLAssembler());
632  GetAssembler()->cfi().RememberState();
633  if (!HasEmptyFrame()) {
634    int frame_size = GetFrameSize();
635    GetAssembler()->UnspillRegisters(GetFramePreservedFPRegisters(),
636        frame_size - FrameEntrySpillSize());
637    GetAssembler()->UnspillRegisters(GetFramePreservedCoreRegisters(),
638        frame_size - GetCoreSpillSize());
639    __ Drop(frame_size);
640    GetAssembler()->cfi().AdjustCFAOffset(-frame_size);
641  }
642  __ Ret();
643  GetAssembler()->cfi().RestoreState();
644  GetAssembler()->cfi().DefCFAOffset(GetFrameSize());
645}
646
647vixl::CPURegList CodeGeneratorARM64::GetFramePreservedCoreRegisters() const {
648  DCHECK(ArtVixlRegCodeCoherentForRegSet(core_spill_mask_, GetNumberOfCoreRegisters(), 0, 0));
649  return vixl::CPURegList(vixl::CPURegister::kRegister, vixl::kXRegSize,
650                          core_spill_mask_);
651}
652
653vixl::CPURegList CodeGeneratorARM64::GetFramePreservedFPRegisters() const {
654  DCHECK(ArtVixlRegCodeCoherentForRegSet(0, 0, fpu_spill_mask_,
655                                         GetNumberOfFloatingPointRegisters()));
656  return vixl::CPURegList(vixl::CPURegister::kFPRegister, vixl::kDRegSize,
657                          fpu_spill_mask_);
658}
659
660void CodeGeneratorARM64::Bind(HBasicBlock* block) {
661  __ Bind(GetLabelOf(block));
662}
663
664void CodeGeneratorARM64::Move(HInstruction* instruction,
665                              Location location,
666                              HInstruction* move_for) {
667  LocationSummary* locations = instruction->GetLocations();
668  Primitive::Type type = instruction->GetType();
669  DCHECK_NE(type, Primitive::kPrimVoid);
670
671  if (instruction->IsFakeString()) {
672    // The fake string is an alias for null.
673    DCHECK(IsBaseline());
674    instruction = locations->Out().GetConstant();
675    DCHECK(instruction->IsNullConstant()) << instruction->DebugName();
676  }
677
678  if (instruction->IsCurrentMethod()) {
679    MoveLocation(location, Location::DoubleStackSlot(kCurrentMethodStackOffset));
680  } else if (locations != nullptr && locations->Out().Equals(location)) {
681    return;
682  } else if (instruction->IsIntConstant()
683             || instruction->IsLongConstant()
684             || instruction->IsNullConstant()) {
685    int64_t value = GetInt64ValueOf(instruction->AsConstant());
686    if (location.IsRegister()) {
687      Register dst = RegisterFrom(location, type);
688      DCHECK(((instruction->IsIntConstant() || instruction->IsNullConstant()) && dst.Is32Bits()) ||
689             (instruction->IsLongConstant() && dst.Is64Bits()));
690      __ Mov(dst, value);
691    } else {
692      DCHECK(location.IsStackSlot() || location.IsDoubleStackSlot());
693      UseScratchRegisterScope temps(GetVIXLAssembler());
694      Register temp = (instruction->IsIntConstant() || instruction->IsNullConstant())
695          ? temps.AcquireW()
696          : temps.AcquireX();
697      __ Mov(temp, value);
698      __ Str(temp, StackOperandFrom(location));
699    }
700  } else if (instruction->IsTemporary()) {
701    Location temp_location = GetTemporaryLocation(instruction->AsTemporary());
702    MoveLocation(location, temp_location, type);
703  } else if (instruction->IsLoadLocal()) {
704    uint32_t stack_slot = GetStackSlot(instruction->AsLoadLocal()->GetLocal());
705    if (Primitive::Is64BitType(type)) {
706      MoveLocation(location, Location::DoubleStackSlot(stack_slot), type);
707    } else {
708      MoveLocation(location, Location::StackSlot(stack_slot), type);
709    }
710
711  } else {
712    DCHECK((instruction->GetNext() == move_for) || instruction->GetNext()->IsTemporary());
713    MoveLocation(location, locations->Out(), type);
714  }
715}
716
717Location CodeGeneratorARM64::GetStackLocation(HLoadLocal* load) const {
718  Primitive::Type type = load->GetType();
719
720  switch (type) {
721    case Primitive::kPrimNot:
722    case Primitive::kPrimInt:
723    case Primitive::kPrimFloat:
724      return Location::StackSlot(GetStackSlot(load->GetLocal()));
725
726    case Primitive::kPrimLong:
727    case Primitive::kPrimDouble:
728      return Location::DoubleStackSlot(GetStackSlot(load->GetLocal()));
729
730    case Primitive::kPrimBoolean:
731    case Primitive::kPrimByte:
732    case Primitive::kPrimChar:
733    case Primitive::kPrimShort:
734    case Primitive::kPrimVoid:
735      LOG(FATAL) << "Unexpected type " << type;
736  }
737
738  LOG(FATAL) << "Unreachable";
739  return Location::NoLocation();
740}
741
742void CodeGeneratorARM64::MarkGCCard(Register object, Register value, bool value_can_be_null) {
743  UseScratchRegisterScope temps(GetVIXLAssembler());
744  Register card = temps.AcquireX();
745  Register temp = temps.AcquireW();   // Index within the CardTable - 32bit.
746  vixl::Label done;
747  if (value_can_be_null) {
748    __ Cbz(value, &done);
749  }
750  __ Ldr(card, MemOperand(tr, Thread::CardTableOffset<kArm64WordSize>().Int32Value()));
751  __ Lsr(temp, object, gc::accounting::CardTable::kCardShift);
752  __ Strb(card, MemOperand(card, temp.X()));
753  if (value_can_be_null) {
754    __ Bind(&done);
755  }
756}
757
758void CodeGeneratorARM64::SetupBlockedRegisters(bool is_baseline) const {
759  // Blocked core registers:
760  //      lr        : Runtime reserved.
761  //      tr        : Runtime reserved.
762  //      xSuspend  : Runtime reserved. TODO: Unblock this when the runtime stops using it.
763  //      ip1       : VIXL core temp.
764  //      ip0       : VIXL core temp.
765  //
766  // Blocked fp registers:
767  //      d31       : VIXL fp temp.
768  CPURegList reserved_core_registers = vixl_reserved_core_registers;
769  reserved_core_registers.Combine(runtime_reserved_core_registers);
770  while (!reserved_core_registers.IsEmpty()) {
771    blocked_core_registers_[reserved_core_registers.PopLowestIndex().code()] = true;
772  }
773
774  CPURegList reserved_fp_registers = vixl_reserved_fp_registers;
775  while (!reserved_fp_registers.IsEmpty()) {
776    blocked_fpu_registers_[reserved_fp_registers.PopLowestIndex().code()] = true;
777  }
778
779  if (is_baseline) {
780    CPURegList reserved_core_baseline_registers = callee_saved_core_registers;
781    while (!reserved_core_baseline_registers.IsEmpty()) {
782      blocked_core_registers_[reserved_core_baseline_registers.PopLowestIndex().code()] = true;
783    }
784
785    CPURegList reserved_fp_baseline_registers = callee_saved_fp_registers;
786    while (!reserved_fp_baseline_registers.IsEmpty()) {
787      blocked_fpu_registers_[reserved_fp_baseline_registers.PopLowestIndex().code()] = true;
788    }
789  }
790}
791
792Location CodeGeneratorARM64::AllocateFreeRegister(Primitive::Type type) const {
793  if (type == Primitive::kPrimVoid) {
794    LOG(FATAL) << "Unreachable type " << type;
795  }
796
797  if (Primitive::IsFloatingPointType(type)) {
798    ssize_t reg = FindFreeEntry(blocked_fpu_registers_, kNumberOfAllocatableFPRegisters);
799    DCHECK_NE(reg, -1);
800    return Location::FpuRegisterLocation(reg);
801  } else {
802    ssize_t reg = FindFreeEntry(blocked_core_registers_, kNumberOfAllocatableRegisters);
803    DCHECK_NE(reg, -1);
804    return Location::RegisterLocation(reg);
805  }
806}
807
808size_t CodeGeneratorARM64::SaveCoreRegister(size_t stack_index, uint32_t reg_id) {
809  Register reg = Register(VIXLRegCodeFromART(reg_id), kXRegSize);
810  __ Str(reg, MemOperand(sp, stack_index));
811  return kArm64WordSize;
812}
813
814size_t CodeGeneratorARM64::RestoreCoreRegister(size_t stack_index, uint32_t reg_id) {
815  Register reg = Register(VIXLRegCodeFromART(reg_id), kXRegSize);
816  __ Ldr(reg, MemOperand(sp, stack_index));
817  return kArm64WordSize;
818}
819
820size_t CodeGeneratorARM64::SaveFloatingPointRegister(size_t stack_index, uint32_t reg_id) {
821  FPRegister reg = FPRegister(reg_id, kDRegSize);
822  __ Str(reg, MemOperand(sp, stack_index));
823  return kArm64WordSize;
824}
825
826size_t CodeGeneratorARM64::RestoreFloatingPointRegister(size_t stack_index, uint32_t reg_id) {
827  FPRegister reg = FPRegister(reg_id, kDRegSize);
828  __ Ldr(reg, MemOperand(sp, stack_index));
829  return kArm64WordSize;
830}
831
832void CodeGeneratorARM64::DumpCoreRegister(std::ostream& stream, int reg) const {
833  stream << XRegister(reg);
834}
835
836void CodeGeneratorARM64::DumpFloatingPointRegister(std::ostream& stream, int reg) const {
837  stream << DRegister(reg);
838}
839
840void CodeGeneratorARM64::MoveConstant(CPURegister destination, HConstant* constant) {
841  if (constant->IsIntConstant()) {
842    __ Mov(Register(destination), constant->AsIntConstant()->GetValue());
843  } else if (constant->IsLongConstant()) {
844    __ Mov(Register(destination), constant->AsLongConstant()->GetValue());
845  } else if (constant->IsNullConstant()) {
846    __ Mov(Register(destination), 0);
847  } else if (constant->IsFloatConstant()) {
848    __ Fmov(FPRegister(destination), constant->AsFloatConstant()->GetValue());
849  } else {
850    DCHECK(constant->IsDoubleConstant());
851    __ Fmov(FPRegister(destination), constant->AsDoubleConstant()->GetValue());
852  }
853}
854
855
856static bool CoherentConstantAndType(Location constant, Primitive::Type type) {
857  DCHECK(constant.IsConstant());
858  HConstant* cst = constant.GetConstant();
859  return (cst->IsIntConstant() && type == Primitive::kPrimInt) ||
860         // Null is mapped to a core W register, which we associate with kPrimInt.
861         (cst->IsNullConstant() && type == Primitive::kPrimInt) ||
862         (cst->IsLongConstant() && type == Primitive::kPrimLong) ||
863         (cst->IsFloatConstant() && type == Primitive::kPrimFloat) ||
864         (cst->IsDoubleConstant() && type == Primitive::kPrimDouble);
865}
866
867void CodeGeneratorARM64::MoveLocation(Location destination, Location source, Primitive::Type type) {
868  if (source.Equals(destination)) {
869    return;
870  }
871
872  // A valid move can always be inferred from the destination and source
873  // locations. When moving from and to a register, the argument type can be
874  // used to generate 32bit instead of 64bit moves. In debug mode we also
875  // checks the coherency of the locations and the type.
876  bool unspecified_type = (type == Primitive::kPrimVoid);
877
878  if (destination.IsRegister() || destination.IsFpuRegister()) {
879    if (unspecified_type) {
880      HConstant* src_cst = source.IsConstant() ? source.GetConstant() : nullptr;
881      if (source.IsStackSlot() ||
882          (src_cst != nullptr && (src_cst->IsIntConstant()
883                                  || src_cst->IsFloatConstant()
884                                  || src_cst->IsNullConstant()))) {
885        // For stack slots and 32bit constants, a 64bit type is appropriate.
886        type = destination.IsRegister() ? Primitive::kPrimInt : Primitive::kPrimFloat;
887      } else {
888        // If the source is a double stack slot or a 64bit constant, a 64bit
889        // type is appropriate. Else the source is a register, and since the
890        // type has not been specified, we chose a 64bit type to force a 64bit
891        // move.
892        type = destination.IsRegister() ? Primitive::kPrimLong : Primitive::kPrimDouble;
893      }
894    }
895    DCHECK((destination.IsFpuRegister() && Primitive::IsFloatingPointType(type)) ||
896           (destination.IsRegister() && !Primitive::IsFloatingPointType(type)));
897    CPURegister dst = CPURegisterFrom(destination, type);
898    if (source.IsStackSlot() || source.IsDoubleStackSlot()) {
899      DCHECK(dst.Is64Bits() == source.IsDoubleStackSlot());
900      __ Ldr(dst, StackOperandFrom(source));
901    } else if (source.IsConstant()) {
902      DCHECK(CoherentConstantAndType(source, type));
903      MoveConstant(dst, source.GetConstant());
904    } else {
905      if (destination.IsRegister()) {
906        __ Mov(Register(dst), RegisterFrom(source, type));
907      } else {
908        DCHECK(destination.IsFpuRegister());
909        __ Fmov(FPRegister(dst), FPRegisterFrom(source, type));
910      }
911    }
912  } else {  // The destination is not a register. It must be a stack slot.
913    DCHECK(destination.IsStackSlot() || destination.IsDoubleStackSlot());
914    if (source.IsRegister() || source.IsFpuRegister()) {
915      if (unspecified_type) {
916        if (source.IsRegister()) {
917          type = destination.IsStackSlot() ? Primitive::kPrimInt : Primitive::kPrimLong;
918        } else {
919          type = destination.IsStackSlot() ? Primitive::kPrimFloat : Primitive::kPrimDouble;
920        }
921      }
922      DCHECK((destination.IsDoubleStackSlot() == Primitive::Is64BitType(type)) &&
923             (source.IsFpuRegister() == Primitive::IsFloatingPointType(type)));
924      __ Str(CPURegisterFrom(source, type), StackOperandFrom(destination));
925    } else if (source.IsConstant()) {
926      DCHECK(unspecified_type || CoherentConstantAndType(source, type)) << source << " " << type;
927      UseScratchRegisterScope temps(GetVIXLAssembler());
928      HConstant* src_cst = source.GetConstant();
929      CPURegister temp;
930      if (src_cst->IsIntConstant() || src_cst->IsNullConstant()) {
931        temp = temps.AcquireW();
932      } else if (src_cst->IsLongConstant()) {
933        temp = temps.AcquireX();
934      } else if (src_cst->IsFloatConstant()) {
935        temp = temps.AcquireS();
936      } else {
937        DCHECK(src_cst->IsDoubleConstant());
938        temp = temps.AcquireD();
939      }
940      MoveConstant(temp, src_cst);
941      __ Str(temp, StackOperandFrom(destination));
942    } else {
943      DCHECK(source.IsStackSlot() || source.IsDoubleStackSlot());
944      DCHECK(source.IsDoubleStackSlot() == destination.IsDoubleStackSlot());
945      UseScratchRegisterScope temps(GetVIXLAssembler());
946      // There is generally less pressure on FP registers.
947      FPRegister temp = destination.IsDoubleStackSlot() ? temps.AcquireD() : temps.AcquireS();
948      __ Ldr(temp, StackOperandFrom(source));
949      __ Str(temp, StackOperandFrom(destination));
950    }
951  }
952}
953
954void CodeGeneratorARM64::Load(Primitive::Type type,
955                              CPURegister dst,
956                              const MemOperand& src) {
957  switch (type) {
958    case Primitive::kPrimBoolean:
959      __ Ldrb(Register(dst), src);
960      break;
961    case Primitive::kPrimByte:
962      __ Ldrsb(Register(dst), src);
963      break;
964    case Primitive::kPrimShort:
965      __ Ldrsh(Register(dst), src);
966      break;
967    case Primitive::kPrimChar:
968      __ Ldrh(Register(dst), src);
969      break;
970    case Primitive::kPrimInt:
971    case Primitive::kPrimNot:
972    case Primitive::kPrimLong:
973    case Primitive::kPrimFloat:
974    case Primitive::kPrimDouble:
975      DCHECK_EQ(dst.Is64Bits(), Primitive::Is64BitType(type));
976      __ Ldr(dst, src);
977      break;
978    case Primitive::kPrimVoid:
979      LOG(FATAL) << "Unreachable type " << type;
980  }
981}
982
983void CodeGeneratorARM64::LoadAcquire(HInstruction* instruction,
984                                     CPURegister dst,
985                                     const MemOperand& src) {
986  MacroAssembler* masm = GetVIXLAssembler();
987  BlockPoolsScope block_pools(masm);
988  UseScratchRegisterScope temps(masm);
989  Register temp_base = temps.AcquireX();
990  Primitive::Type type = instruction->GetType();
991
992  DCHECK(!src.IsPreIndex());
993  DCHECK(!src.IsPostIndex());
994
995  // TODO(vixl): Let the MacroAssembler handle MemOperand.
996  __ Add(temp_base, src.base(), OperandFromMemOperand(src));
997  MemOperand base = MemOperand(temp_base);
998  switch (type) {
999    case Primitive::kPrimBoolean:
1000      __ Ldarb(Register(dst), base);
1001      MaybeRecordImplicitNullCheck(instruction);
1002      break;
1003    case Primitive::kPrimByte:
1004      __ Ldarb(Register(dst), base);
1005      MaybeRecordImplicitNullCheck(instruction);
1006      __ Sbfx(Register(dst), Register(dst), 0, Primitive::ComponentSize(type) * kBitsPerByte);
1007      break;
1008    case Primitive::kPrimChar:
1009      __ Ldarh(Register(dst), base);
1010      MaybeRecordImplicitNullCheck(instruction);
1011      break;
1012    case Primitive::kPrimShort:
1013      __ Ldarh(Register(dst), base);
1014      MaybeRecordImplicitNullCheck(instruction);
1015      __ Sbfx(Register(dst), Register(dst), 0, Primitive::ComponentSize(type) * kBitsPerByte);
1016      break;
1017    case Primitive::kPrimInt:
1018    case Primitive::kPrimNot:
1019    case Primitive::kPrimLong:
1020      DCHECK_EQ(dst.Is64Bits(), Primitive::Is64BitType(type));
1021      __ Ldar(Register(dst), base);
1022      MaybeRecordImplicitNullCheck(instruction);
1023      break;
1024    case Primitive::kPrimFloat:
1025    case Primitive::kPrimDouble: {
1026      DCHECK(dst.IsFPRegister());
1027      DCHECK_EQ(dst.Is64Bits(), Primitive::Is64BitType(type));
1028
1029      Register temp = dst.Is64Bits() ? temps.AcquireX() : temps.AcquireW();
1030      __ Ldar(temp, base);
1031      MaybeRecordImplicitNullCheck(instruction);
1032      __ Fmov(FPRegister(dst), temp);
1033      break;
1034    }
1035    case Primitive::kPrimVoid:
1036      LOG(FATAL) << "Unreachable type " << type;
1037  }
1038}
1039
1040void CodeGeneratorARM64::Store(Primitive::Type type,
1041                               CPURegister src,
1042                               const MemOperand& dst) {
1043  switch (type) {
1044    case Primitive::kPrimBoolean:
1045    case Primitive::kPrimByte:
1046      __ Strb(Register(src), dst);
1047      break;
1048    case Primitive::kPrimChar:
1049    case Primitive::kPrimShort:
1050      __ Strh(Register(src), dst);
1051      break;
1052    case Primitive::kPrimInt:
1053    case Primitive::kPrimNot:
1054    case Primitive::kPrimLong:
1055    case Primitive::kPrimFloat:
1056    case Primitive::kPrimDouble:
1057      DCHECK_EQ(src.Is64Bits(), Primitive::Is64BitType(type));
1058      __ Str(src, dst);
1059      break;
1060    case Primitive::kPrimVoid:
1061      LOG(FATAL) << "Unreachable type " << type;
1062  }
1063}
1064
1065void CodeGeneratorARM64::StoreRelease(Primitive::Type type,
1066                                      CPURegister src,
1067                                      const MemOperand& dst) {
1068  UseScratchRegisterScope temps(GetVIXLAssembler());
1069  Register temp_base = temps.AcquireX();
1070
1071  DCHECK(!dst.IsPreIndex());
1072  DCHECK(!dst.IsPostIndex());
1073
1074  // TODO(vixl): Let the MacroAssembler handle this.
1075  Operand op = OperandFromMemOperand(dst);
1076  __ Add(temp_base, dst.base(), op);
1077  MemOperand base = MemOperand(temp_base);
1078  switch (type) {
1079    case Primitive::kPrimBoolean:
1080    case Primitive::kPrimByte:
1081      __ Stlrb(Register(src), base);
1082      break;
1083    case Primitive::kPrimChar:
1084    case Primitive::kPrimShort:
1085      __ Stlrh(Register(src), base);
1086      break;
1087    case Primitive::kPrimInt:
1088    case Primitive::kPrimNot:
1089    case Primitive::kPrimLong:
1090      DCHECK_EQ(src.Is64Bits(), Primitive::Is64BitType(type));
1091      __ Stlr(Register(src), base);
1092      break;
1093    case Primitive::kPrimFloat:
1094    case Primitive::kPrimDouble: {
1095      DCHECK(src.IsFPRegister());
1096      DCHECK_EQ(src.Is64Bits(), Primitive::Is64BitType(type));
1097
1098      Register temp = src.Is64Bits() ? temps.AcquireX() : temps.AcquireW();
1099      __ Fmov(temp, FPRegister(src));
1100      __ Stlr(temp, base);
1101      break;
1102    }
1103    case Primitive::kPrimVoid:
1104      LOG(FATAL) << "Unreachable type " << type;
1105  }
1106}
1107
1108void CodeGeneratorARM64::InvokeRuntime(int32_t entry_point_offset,
1109                                       HInstruction* instruction,
1110                                       uint32_t dex_pc,
1111                                       SlowPathCode* slow_path) {
1112  ValidateInvokeRuntime(instruction, slow_path);
1113  BlockPoolsScope block_pools(GetVIXLAssembler());
1114  __ Ldr(lr, MemOperand(tr, entry_point_offset));
1115  __ Blr(lr);
1116  RecordPcInfo(instruction, dex_pc, slow_path);
1117}
1118
1119void InstructionCodeGeneratorARM64::GenerateClassInitializationCheck(SlowPathCodeARM64* slow_path,
1120                                                                     vixl::Register class_reg) {
1121  UseScratchRegisterScope temps(GetVIXLAssembler());
1122  Register temp = temps.AcquireW();
1123  size_t status_offset = mirror::Class::StatusOffset().SizeValue();
1124  bool use_acquire_release = codegen_->GetInstructionSetFeatures().PreferAcquireRelease();
1125
1126  // Even if the initialized flag is set, we need to ensure consistent memory ordering.
1127  if (use_acquire_release) {
1128    // TODO(vixl): Let the MacroAssembler handle MemOperand.
1129    __ Add(temp, class_reg, status_offset);
1130    __ Ldar(temp, HeapOperand(temp));
1131    __ Cmp(temp, mirror::Class::kStatusInitialized);
1132    __ B(lt, slow_path->GetEntryLabel());
1133  } else {
1134    __ Ldr(temp, HeapOperand(class_reg, status_offset));
1135    __ Cmp(temp, mirror::Class::kStatusInitialized);
1136    __ B(lt, slow_path->GetEntryLabel());
1137    __ Dmb(InnerShareable, BarrierReads);
1138  }
1139  __ Bind(slow_path->GetExitLabel());
1140}
1141
1142void InstructionCodeGeneratorARM64::GenerateMemoryBarrier(MemBarrierKind kind) {
1143  BarrierType type = BarrierAll;
1144
1145  switch (kind) {
1146    case MemBarrierKind::kAnyAny:
1147    case MemBarrierKind::kAnyStore: {
1148      type = BarrierAll;
1149      break;
1150    }
1151    case MemBarrierKind::kLoadAny: {
1152      type = BarrierReads;
1153      break;
1154    }
1155    case MemBarrierKind::kStoreStore: {
1156      type = BarrierWrites;
1157      break;
1158    }
1159    default:
1160      LOG(FATAL) << "Unexpected memory barrier " << kind;
1161  }
1162  __ Dmb(InnerShareable, type);
1163}
1164
1165void InstructionCodeGeneratorARM64::GenerateSuspendCheck(HSuspendCheck* instruction,
1166                                                         HBasicBlock* successor) {
1167  SuspendCheckSlowPathARM64* slow_path =
1168      down_cast<SuspendCheckSlowPathARM64*>(instruction->GetSlowPath());
1169  if (slow_path == nullptr) {
1170    slow_path = new (GetGraph()->GetArena()) SuspendCheckSlowPathARM64(instruction, successor);
1171    instruction->SetSlowPath(slow_path);
1172    codegen_->AddSlowPath(slow_path);
1173    if (successor != nullptr) {
1174      DCHECK(successor->IsLoopHeader());
1175      codegen_->ClearSpillSlotsFromLoopPhisInStackMap(instruction);
1176    }
1177  } else {
1178    DCHECK_EQ(slow_path->GetSuccessor(), successor);
1179  }
1180
1181  UseScratchRegisterScope temps(codegen_->GetVIXLAssembler());
1182  Register temp = temps.AcquireW();
1183
1184  __ Ldrh(temp, MemOperand(tr, Thread::ThreadFlagsOffset<kArm64WordSize>().SizeValue()));
1185  if (successor == nullptr) {
1186    __ Cbnz(temp, slow_path->GetEntryLabel());
1187    __ Bind(slow_path->GetReturnLabel());
1188  } else {
1189    __ Cbz(temp, codegen_->GetLabelOf(successor));
1190    __ B(slow_path->GetEntryLabel());
1191    // slow_path will return to GetLabelOf(successor).
1192  }
1193}
1194
1195InstructionCodeGeneratorARM64::InstructionCodeGeneratorARM64(HGraph* graph,
1196                                                             CodeGeneratorARM64* codegen)
1197      : HGraphVisitor(graph),
1198        assembler_(codegen->GetAssembler()),
1199        codegen_(codegen) {}
1200
1201#define FOR_EACH_UNIMPLEMENTED_INSTRUCTION(M)              \
1202  /* No unimplemented IR. */
1203
1204#define UNIMPLEMENTED_INSTRUCTION_BREAK_CODE(name) name##UnimplementedInstructionBreakCode
1205
1206enum UnimplementedInstructionBreakCode {
1207  // Using a base helps identify when we hit such breakpoints.
1208  UnimplementedInstructionBreakCodeBaseCode = 0x900,
1209#define ENUM_UNIMPLEMENTED_INSTRUCTION(name) UNIMPLEMENTED_INSTRUCTION_BREAK_CODE(name),
1210  FOR_EACH_UNIMPLEMENTED_INSTRUCTION(ENUM_UNIMPLEMENTED_INSTRUCTION)
1211#undef ENUM_UNIMPLEMENTED_INSTRUCTION
1212};
1213
1214#define DEFINE_UNIMPLEMENTED_INSTRUCTION_VISITORS(name)                               \
1215  void InstructionCodeGeneratorARM64::Visit##name(H##name* instr) {                   \
1216    UNUSED(instr);                                                                    \
1217    __ Brk(UNIMPLEMENTED_INSTRUCTION_BREAK_CODE(name));                               \
1218  }                                                                                   \
1219  void LocationsBuilderARM64::Visit##name(H##name* instr) {                           \
1220    LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instr); \
1221    locations->SetOut(Location::Any());                                               \
1222  }
1223  FOR_EACH_UNIMPLEMENTED_INSTRUCTION(DEFINE_UNIMPLEMENTED_INSTRUCTION_VISITORS)
1224#undef DEFINE_UNIMPLEMENTED_INSTRUCTION_VISITORS
1225
1226#undef UNIMPLEMENTED_INSTRUCTION_BREAK_CODE
1227#undef FOR_EACH_UNIMPLEMENTED_INSTRUCTION
1228
1229void LocationsBuilderARM64::HandleBinaryOp(HBinaryOperation* instr) {
1230  DCHECK_EQ(instr->InputCount(), 2U);
1231  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instr);
1232  Primitive::Type type = instr->GetResultType();
1233  switch (type) {
1234    case Primitive::kPrimInt:
1235    case Primitive::kPrimLong:
1236      locations->SetInAt(0, Location::RequiresRegister());
1237      locations->SetInAt(1, ARM64EncodableConstantOrRegister(instr->InputAt(1), instr));
1238      locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
1239      break;
1240
1241    case Primitive::kPrimFloat:
1242    case Primitive::kPrimDouble:
1243      locations->SetInAt(0, Location::RequiresFpuRegister());
1244      locations->SetInAt(1, Location::RequiresFpuRegister());
1245      locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap);
1246      break;
1247
1248    default:
1249      LOG(FATAL) << "Unexpected " << instr->DebugName() << " type " << type;
1250  }
1251}
1252
1253void LocationsBuilderARM64::HandleFieldGet(HInstruction* instruction) {
1254  LocationSummary* locations =
1255      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
1256  locations->SetInAt(0, Location::RequiresRegister());
1257  if (Primitive::IsFloatingPointType(instruction->GetType())) {
1258    locations->SetOut(Location::RequiresFpuRegister());
1259  } else {
1260    locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
1261  }
1262}
1263
1264void InstructionCodeGeneratorARM64::HandleFieldGet(HInstruction* instruction,
1265                                                   const FieldInfo& field_info) {
1266  DCHECK(instruction->IsInstanceFieldGet() || instruction->IsStaticFieldGet());
1267  Primitive::Type field_type = field_info.GetFieldType();
1268  BlockPoolsScope block_pools(GetVIXLAssembler());
1269
1270  MemOperand field = HeapOperand(InputRegisterAt(instruction, 0), field_info.GetFieldOffset());
1271  bool use_acquire_release = codegen_->GetInstructionSetFeatures().PreferAcquireRelease();
1272
1273  if (field_info.IsVolatile()) {
1274    if (use_acquire_release) {
1275      // NB: LoadAcquire will record the pc info if needed.
1276      codegen_->LoadAcquire(instruction, OutputCPURegister(instruction), field);
1277    } else {
1278      codegen_->Load(field_type, OutputCPURegister(instruction), field);
1279      codegen_->MaybeRecordImplicitNullCheck(instruction);
1280      // For IRIW sequential consistency kLoadAny is not sufficient.
1281      GenerateMemoryBarrier(MemBarrierKind::kAnyAny);
1282    }
1283  } else {
1284    codegen_->Load(field_type, OutputCPURegister(instruction), field);
1285    codegen_->MaybeRecordImplicitNullCheck(instruction);
1286  }
1287
1288  if (field_type == Primitive::kPrimNot) {
1289    GetAssembler()->MaybeUnpoisonHeapReference(OutputCPURegister(instruction).W());
1290  }
1291}
1292
1293void LocationsBuilderARM64::HandleFieldSet(HInstruction* instruction) {
1294  LocationSummary* locations =
1295      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
1296  locations->SetInAt(0, Location::RequiresRegister());
1297  if (Primitive::IsFloatingPointType(instruction->InputAt(1)->GetType())) {
1298    locations->SetInAt(1, Location::RequiresFpuRegister());
1299  } else {
1300    locations->SetInAt(1, Location::RequiresRegister());
1301  }
1302}
1303
1304void InstructionCodeGeneratorARM64::HandleFieldSet(HInstruction* instruction,
1305                                                   const FieldInfo& field_info,
1306                                                   bool value_can_be_null) {
1307  DCHECK(instruction->IsInstanceFieldSet() || instruction->IsStaticFieldSet());
1308  BlockPoolsScope block_pools(GetVIXLAssembler());
1309
1310  Register obj = InputRegisterAt(instruction, 0);
1311  CPURegister value = InputCPURegisterAt(instruction, 1);
1312  CPURegister source = value;
1313  Offset offset = field_info.GetFieldOffset();
1314  Primitive::Type field_type = field_info.GetFieldType();
1315  bool use_acquire_release = codegen_->GetInstructionSetFeatures().PreferAcquireRelease();
1316
1317  {
1318    // We use a block to end the scratch scope before the write barrier, thus
1319    // freeing the temporary registers so they can be used in `MarkGCCard`.
1320    UseScratchRegisterScope temps(GetVIXLAssembler());
1321
1322    if (kPoisonHeapReferences && field_type == Primitive::kPrimNot) {
1323      DCHECK(value.IsW());
1324      Register temp = temps.AcquireW();
1325      __ Mov(temp, value.W());
1326      GetAssembler()->PoisonHeapReference(temp.W());
1327      source = temp;
1328    }
1329
1330    if (field_info.IsVolatile()) {
1331      if (use_acquire_release) {
1332        codegen_->StoreRelease(field_type, source, HeapOperand(obj, offset));
1333        codegen_->MaybeRecordImplicitNullCheck(instruction);
1334      } else {
1335        GenerateMemoryBarrier(MemBarrierKind::kAnyStore);
1336        codegen_->Store(field_type, source, HeapOperand(obj, offset));
1337        codegen_->MaybeRecordImplicitNullCheck(instruction);
1338        GenerateMemoryBarrier(MemBarrierKind::kAnyAny);
1339      }
1340    } else {
1341      codegen_->Store(field_type, source, HeapOperand(obj, offset));
1342      codegen_->MaybeRecordImplicitNullCheck(instruction);
1343    }
1344  }
1345
1346  if (CodeGenerator::StoreNeedsWriteBarrier(field_type, instruction->InputAt(1))) {
1347    codegen_->MarkGCCard(obj, Register(value), value_can_be_null);
1348  }
1349}
1350
1351void InstructionCodeGeneratorARM64::HandleBinaryOp(HBinaryOperation* instr) {
1352  Primitive::Type type = instr->GetType();
1353
1354  switch (type) {
1355    case Primitive::kPrimInt:
1356    case Primitive::kPrimLong: {
1357      Register dst = OutputRegister(instr);
1358      Register lhs = InputRegisterAt(instr, 0);
1359      Operand rhs = InputOperandAt(instr, 1);
1360      if (instr->IsAdd()) {
1361        __ Add(dst, lhs, rhs);
1362      } else if (instr->IsAnd()) {
1363        __ And(dst, lhs, rhs);
1364      } else if (instr->IsOr()) {
1365        __ Orr(dst, lhs, rhs);
1366      } else if (instr->IsSub()) {
1367        __ Sub(dst, lhs, rhs);
1368      } else {
1369        DCHECK(instr->IsXor());
1370        __ Eor(dst, lhs, rhs);
1371      }
1372      break;
1373    }
1374    case Primitive::kPrimFloat:
1375    case Primitive::kPrimDouble: {
1376      FPRegister dst = OutputFPRegister(instr);
1377      FPRegister lhs = InputFPRegisterAt(instr, 0);
1378      FPRegister rhs = InputFPRegisterAt(instr, 1);
1379      if (instr->IsAdd()) {
1380        __ Fadd(dst, lhs, rhs);
1381      } else if (instr->IsSub()) {
1382        __ Fsub(dst, lhs, rhs);
1383      } else {
1384        LOG(FATAL) << "Unexpected floating-point binary operation";
1385      }
1386      break;
1387    }
1388    default:
1389      LOG(FATAL) << "Unexpected binary operation type " << type;
1390  }
1391}
1392
1393void LocationsBuilderARM64::HandleShift(HBinaryOperation* instr) {
1394  DCHECK(instr->IsShl() || instr->IsShr() || instr->IsUShr());
1395
1396  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instr);
1397  Primitive::Type type = instr->GetResultType();
1398  switch (type) {
1399    case Primitive::kPrimInt:
1400    case Primitive::kPrimLong: {
1401      locations->SetInAt(0, Location::RequiresRegister());
1402      locations->SetInAt(1, Location::RegisterOrConstant(instr->InputAt(1)));
1403      locations->SetOut(Location::RequiresRegister());
1404      break;
1405    }
1406    default:
1407      LOG(FATAL) << "Unexpected shift type " << type;
1408  }
1409}
1410
1411void InstructionCodeGeneratorARM64::HandleShift(HBinaryOperation* instr) {
1412  DCHECK(instr->IsShl() || instr->IsShr() || instr->IsUShr());
1413
1414  Primitive::Type type = instr->GetType();
1415  switch (type) {
1416    case Primitive::kPrimInt:
1417    case Primitive::kPrimLong: {
1418      Register dst = OutputRegister(instr);
1419      Register lhs = InputRegisterAt(instr, 0);
1420      Operand rhs = InputOperandAt(instr, 1);
1421      if (rhs.IsImmediate()) {
1422        uint32_t shift_value = (type == Primitive::kPrimInt)
1423          ? static_cast<uint32_t>(rhs.immediate() & kMaxIntShiftValue)
1424          : static_cast<uint32_t>(rhs.immediate() & kMaxLongShiftValue);
1425        if (instr->IsShl()) {
1426          __ Lsl(dst, lhs, shift_value);
1427        } else if (instr->IsShr()) {
1428          __ Asr(dst, lhs, shift_value);
1429        } else {
1430          __ Lsr(dst, lhs, shift_value);
1431        }
1432      } else {
1433        Register rhs_reg = dst.IsX() ? rhs.reg().X() : rhs.reg().W();
1434
1435        if (instr->IsShl()) {
1436          __ Lsl(dst, lhs, rhs_reg);
1437        } else if (instr->IsShr()) {
1438          __ Asr(dst, lhs, rhs_reg);
1439        } else {
1440          __ Lsr(dst, lhs, rhs_reg);
1441        }
1442      }
1443      break;
1444    }
1445    default:
1446      LOG(FATAL) << "Unexpected shift operation type " << type;
1447  }
1448}
1449
1450void LocationsBuilderARM64::VisitAdd(HAdd* instruction) {
1451  HandleBinaryOp(instruction);
1452}
1453
1454void InstructionCodeGeneratorARM64::VisitAdd(HAdd* instruction) {
1455  HandleBinaryOp(instruction);
1456}
1457
1458void LocationsBuilderARM64::VisitAnd(HAnd* instruction) {
1459  HandleBinaryOp(instruction);
1460}
1461
1462void InstructionCodeGeneratorARM64::VisitAnd(HAnd* instruction) {
1463  HandleBinaryOp(instruction);
1464}
1465
1466void LocationsBuilderARM64::VisitArrayGet(HArrayGet* instruction) {
1467  LocationSummary* locations =
1468      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
1469  locations->SetInAt(0, Location::RequiresRegister());
1470  locations->SetInAt(1, Location::RegisterOrConstant(instruction->InputAt(1)));
1471  if (Primitive::IsFloatingPointType(instruction->GetType())) {
1472    locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap);
1473  } else {
1474    locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
1475  }
1476}
1477
1478void InstructionCodeGeneratorARM64::VisitArrayGet(HArrayGet* instruction) {
1479  LocationSummary* locations = instruction->GetLocations();
1480  Primitive::Type type = instruction->GetType();
1481  Register obj = InputRegisterAt(instruction, 0);
1482  Location index = locations->InAt(1);
1483  size_t offset = mirror::Array::DataOffset(Primitive::ComponentSize(type)).Uint32Value();
1484  MemOperand source = HeapOperand(obj);
1485  MacroAssembler* masm = GetVIXLAssembler();
1486  UseScratchRegisterScope temps(masm);
1487  BlockPoolsScope block_pools(masm);
1488
1489  if (index.IsConstant()) {
1490    offset += Int64ConstantFrom(index) << Primitive::ComponentSizeShift(type);
1491    source = HeapOperand(obj, offset);
1492  } else {
1493    Register temp = temps.AcquireSameSizeAs(obj);
1494    __ Add(temp, obj, offset);
1495    source = HeapOperand(temp, XRegisterFrom(index), LSL, Primitive::ComponentSizeShift(type));
1496  }
1497
1498  codegen_->Load(type, OutputCPURegister(instruction), source);
1499  codegen_->MaybeRecordImplicitNullCheck(instruction);
1500
1501  if (type == Primitive::kPrimNot) {
1502    GetAssembler()->MaybeUnpoisonHeapReference(OutputCPURegister(instruction).W());
1503  }
1504}
1505
1506void LocationsBuilderARM64::VisitArrayLength(HArrayLength* instruction) {
1507  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction);
1508  locations->SetInAt(0, Location::RequiresRegister());
1509  locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
1510}
1511
1512void InstructionCodeGeneratorARM64::VisitArrayLength(HArrayLength* instruction) {
1513  BlockPoolsScope block_pools(GetVIXLAssembler());
1514  __ Ldr(OutputRegister(instruction),
1515         HeapOperand(InputRegisterAt(instruction, 0), mirror::Array::LengthOffset()));
1516  codegen_->MaybeRecordImplicitNullCheck(instruction);
1517}
1518
1519void LocationsBuilderARM64::VisitArraySet(HArraySet* instruction) {
1520  if (instruction->NeedsTypeCheck()) {
1521    LocationSummary* locations =
1522        new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCall);
1523    InvokeRuntimeCallingConvention calling_convention;
1524    locations->SetInAt(0, LocationFrom(calling_convention.GetRegisterAt(0)));
1525    locations->SetInAt(1, LocationFrom(calling_convention.GetRegisterAt(1)));
1526    locations->SetInAt(2, LocationFrom(calling_convention.GetRegisterAt(2)));
1527  } else {
1528    LocationSummary* locations =
1529        new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
1530    locations->SetInAt(0, Location::RequiresRegister());
1531    locations->SetInAt(1, Location::RegisterOrConstant(instruction->InputAt(1)));
1532    if (Primitive::IsFloatingPointType(instruction->InputAt(2)->GetType())) {
1533      locations->SetInAt(2, Location::RequiresFpuRegister());
1534    } else {
1535      locations->SetInAt(2, Location::RequiresRegister());
1536    }
1537  }
1538}
1539
1540void InstructionCodeGeneratorARM64::VisitArraySet(HArraySet* instruction) {
1541  Primitive::Type value_type = instruction->GetComponentType();
1542  LocationSummary* locations = instruction->GetLocations();
1543  bool needs_runtime_call = locations->WillCall();
1544
1545  if (needs_runtime_call) {
1546    // Note: if heap poisoning is enabled, pAputObject takes cares
1547    // of poisoning the reference.
1548    codegen_->InvokeRuntime(
1549        QUICK_ENTRY_POINT(pAputObject), instruction, instruction->GetDexPc(), nullptr);
1550    CheckEntrypointTypes<kQuickAputObject, void, mirror::Array*, int32_t, mirror::Object*>();
1551  } else {
1552    Register obj = InputRegisterAt(instruction, 0);
1553    CPURegister value = InputCPURegisterAt(instruction, 2);
1554    CPURegister source = value;
1555    Location index = locations->InAt(1);
1556    size_t offset = mirror::Array::DataOffset(Primitive::ComponentSize(value_type)).Uint32Value();
1557    MemOperand destination = HeapOperand(obj);
1558    MacroAssembler* masm = GetVIXLAssembler();
1559    BlockPoolsScope block_pools(masm);
1560    {
1561      // We use a block to end the scratch scope before the write barrier, thus
1562      // freeing the temporary registers so they can be used in `MarkGCCard`.
1563      UseScratchRegisterScope temps(masm);
1564
1565      if (kPoisonHeapReferences && value_type == Primitive::kPrimNot) {
1566        DCHECK(value.IsW());
1567        Register temp = temps.AcquireW();
1568        __ Mov(temp, value.W());
1569        GetAssembler()->PoisonHeapReference(temp.W());
1570        source = temp;
1571      }
1572
1573      if (index.IsConstant()) {
1574        offset += Int64ConstantFrom(index) << Primitive::ComponentSizeShift(value_type);
1575        destination = HeapOperand(obj, offset);
1576      } else {
1577        Register temp = temps.AcquireSameSizeAs(obj);
1578        __ Add(temp, obj, offset);
1579        destination = HeapOperand(temp,
1580                                  XRegisterFrom(index),
1581                                  LSL,
1582                                  Primitive::ComponentSizeShift(value_type));
1583      }
1584
1585      codegen_->Store(value_type, source, destination);
1586      codegen_->MaybeRecordImplicitNullCheck(instruction);
1587    }
1588    if (CodeGenerator::StoreNeedsWriteBarrier(value_type, instruction->GetValue())) {
1589      codegen_->MarkGCCard(obj, value.W(), instruction->GetValueCanBeNull());
1590    }
1591  }
1592}
1593
1594void LocationsBuilderARM64::VisitBoundsCheck(HBoundsCheck* instruction) {
1595  LocationSummary::CallKind call_kind = instruction->CanThrowIntoCatchBlock()
1596      ? LocationSummary::kCallOnSlowPath
1597      : LocationSummary::kNoCall;
1598  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction, call_kind);
1599  locations->SetInAt(0, Location::RequiresRegister());
1600  locations->SetInAt(1, ARM64EncodableConstantOrRegister(instruction->InputAt(1), instruction));
1601  if (instruction->HasUses()) {
1602    locations->SetOut(Location::SameAsFirstInput());
1603  }
1604}
1605
1606void InstructionCodeGeneratorARM64::VisitBoundsCheck(HBoundsCheck* instruction) {
1607  BoundsCheckSlowPathARM64* slow_path =
1608      new (GetGraph()->GetArena()) BoundsCheckSlowPathARM64(instruction);
1609  codegen_->AddSlowPath(slow_path);
1610
1611  __ Cmp(InputRegisterAt(instruction, 0), InputOperandAt(instruction, 1));
1612  __ B(slow_path->GetEntryLabel(), hs);
1613}
1614
1615void LocationsBuilderARM64::VisitCheckCast(HCheckCast* instruction) {
1616  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(
1617      instruction, LocationSummary::kCallOnSlowPath);
1618  locations->SetInAt(0, Location::RequiresRegister());
1619  locations->SetInAt(1, Location::RequiresRegister());
1620  // Note that TypeCheckSlowPathARM64 uses this register too.
1621  locations->AddTemp(Location::RequiresRegister());
1622}
1623
1624void InstructionCodeGeneratorARM64::VisitCheckCast(HCheckCast* instruction) {
1625  Register obj = InputRegisterAt(instruction, 0);;
1626  Register cls = InputRegisterAt(instruction, 1);;
1627  Register obj_cls = WRegisterFrom(instruction->GetLocations()->GetTemp(0));
1628
1629  SlowPathCodeARM64* slow_path =
1630      new (GetGraph()->GetArena()) TypeCheckSlowPathARM64(instruction);
1631  codegen_->AddSlowPath(slow_path);
1632
1633  // Avoid null check if we know obj is not null.
1634  if (instruction->MustDoNullCheck()) {
1635    __ Cbz(obj, slow_path->GetExitLabel());
1636  }
1637  // Compare the class of `obj` with `cls`.
1638  __ Ldr(obj_cls, HeapOperand(obj, mirror::Object::ClassOffset()));
1639  GetAssembler()->MaybeUnpoisonHeapReference(obj_cls.W());
1640  __ Cmp(obj_cls, cls);
1641  // The checkcast succeeds if the classes are equal (fast path).
1642  // Otherwise, we need to go into the slow path to check the types.
1643  __ B(ne, slow_path->GetEntryLabel());
1644  __ Bind(slow_path->GetExitLabel());
1645}
1646
1647void LocationsBuilderARM64::VisitClinitCheck(HClinitCheck* check) {
1648  LocationSummary* locations =
1649      new (GetGraph()->GetArena()) LocationSummary(check, LocationSummary::kCallOnSlowPath);
1650  locations->SetInAt(0, Location::RequiresRegister());
1651  if (check->HasUses()) {
1652    locations->SetOut(Location::SameAsFirstInput());
1653  }
1654}
1655
1656void InstructionCodeGeneratorARM64::VisitClinitCheck(HClinitCheck* check) {
1657  // We assume the class is not null.
1658  SlowPathCodeARM64* slow_path = new (GetGraph()->GetArena()) LoadClassSlowPathARM64(
1659      check->GetLoadClass(), check, check->GetDexPc(), true);
1660  codegen_->AddSlowPath(slow_path);
1661  GenerateClassInitializationCheck(slow_path, InputRegisterAt(check, 0));
1662}
1663
1664static bool IsFloatingPointZeroConstant(HInstruction* instruction) {
1665  return (instruction->IsFloatConstant() && (instruction->AsFloatConstant()->GetValue() == 0.0f))
1666      || (instruction->IsDoubleConstant() && (instruction->AsDoubleConstant()->GetValue() == 0.0));
1667}
1668
1669void LocationsBuilderARM64::VisitCompare(HCompare* compare) {
1670  LocationSummary* locations =
1671      new (GetGraph()->GetArena()) LocationSummary(compare, LocationSummary::kNoCall);
1672  Primitive::Type in_type = compare->InputAt(0)->GetType();
1673  switch (in_type) {
1674    case Primitive::kPrimLong: {
1675      locations->SetInAt(0, Location::RequiresRegister());
1676      locations->SetInAt(1, ARM64EncodableConstantOrRegister(compare->InputAt(1), compare));
1677      locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
1678      break;
1679    }
1680    case Primitive::kPrimFloat:
1681    case Primitive::kPrimDouble: {
1682      locations->SetInAt(0, Location::RequiresFpuRegister());
1683      locations->SetInAt(1,
1684                         IsFloatingPointZeroConstant(compare->InputAt(1))
1685                             ? Location::ConstantLocation(compare->InputAt(1)->AsConstant())
1686                             : Location::RequiresFpuRegister());
1687      locations->SetOut(Location::RequiresRegister());
1688      break;
1689    }
1690    default:
1691      LOG(FATAL) << "Unexpected type for compare operation " << in_type;
1692  }
1693}
1694
1695void InstructionCodeGeneratorARM64::VisitCompare(HCompare* compare) {
1696  Primitive::Type in_type = compare->InputAt(0)->GetType();
1697
1698  //  0 if: left == right
1699  //  1 if: left  > right
1700  // -1 if: left  < right
1701  switch (in_type) {
1702    case Primitive::kPrimLong: {
1703      Register result = OutputRegister(compare);
1704      Register left = InputRegisterAt(compare, 0);
1705      Operand right = InputOperandAt(compare, 1);
1706
1707      __ Cmp(left, right);
1708      __ Cset(result, ne);
1709      __ Cneg(result, result, lt);
1710      break;
1711    }
1712    case Primitive::kPrimFloat:
1713    case Primitive::kPrimDouble: {
1714      Register result = OutputRegister(compare);
1715      FPRegister left = InputFPRegisterAt(compare, 0);
1716      if (compare->GetLocations()->InAt(1).IsConstant()) {
1717        DCHECK(IsFloatingPointZeroConstant(compare->GetLocations()->InAt(1).GetConstant()));
1718        // 0.0 is the only immediate that can be encoded directly in an FCMP instruction.
1719        __ Fcmp(left, 0.0);
1720      } else {
1721        __ Fcmp(left, InputFPRegisterAt(compare, 1));
1722      }
1723      if (compare->IsGtBias()) {
1724        __ Cset(result, ne);
1725      } else {
1726        __ Csetm(result, ne);
1727      }
1728      __ Cneg(result, result, compare->IsGtBias() ? mi : gt);
1729      break;
1730    }
1731    default:
1732      LOG(FATAL) << "Unimplemented compare type " << in_type;
1733  }
1734}
1735
1736void LocationsBuilderARM64::VisitCondition(HCondition* instruction) {
1737  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction);
1738
1739  if (Primitive::IsFloatingPointType(instruction->InputAt(0)->GetType())) {
1740    locations->SetInAt(0, Location::RequiresFpuRegister());
1741    locations->SetInAt(1,
1742                       IsFloatingPointZeroConstant(instruction->InputAt(1))
1743                           ? Location::ConstantLocation(instruction->InputAt(1)->AsConstant())
1744                           : Location::RequiresFpuRegister());
1745  } else {
1746    // Integer cases.
1747    locations->SetInAt(0, Location::RequiresRegister());
1748    locations->SetInAt(1, ARM64EncodableConstantOrRegister(instruction->InputAt(1), instruction));
1749  }
1750
1751  if (instruction->NeedsMaterialization()) {
1752    locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
1753  }
1754}
1755
1756void InstructionCodeGeneratorARM64::VisitCondition(HCondition* instruction) {
1757  if (!instruction->NeedsMaterialization()) {
1758    return;
1759  }
1760
1761  LocationSummary* locations = instruction->GetLocations();
1762  Register res = RegisterFrom(locations->Out(), instruction->GetType());
1763  IfCondition if_cond = instruction->GetCondition();
1764  Condition arm64_cond = ARM64Condition(if_cond);
1765
1766  if (Primitive::IsFloatingPointType(instruction->InputAt(0)->GetType())) {
1767    FPRegister lhs = InputFPRegisterAt(instruction, 0);
1768    if (locations->InAt(1).IsConstant()) {
1769      DCHECK(IsFloatingPointZeroConstant(locations->InAt(1).GetConstant()));
1770      // 0.0 is the only immediate that can be encoded directly in an FCMP instruction.
1771      __ Fcmp(lhs, 0.0);
1772    } else {
1773      __ Fcmp(lhs, InputFPRegisterAt(instruction, 1));
1774    }
1775    __ Cset(res, arm64_cond);
1776    if (instruction->IsFPConditionTrueIfNaN()) {
1777      // res = IsUnordered(arm64_cond) ? 1 : res  <=>  res = IsNotUnordered(arm64_cond) ? res : 1
1778      __ Csel(res, res, Operand(1), vc);  // VC for "not unordered".
1779    } else if (instruction->IsFPConditionFalseIfNaN()) {
1780      // res = IsUnordered(arm64_cond) ? 0 : res  <=>  res = IsNotUnordered(arm64_cond) ? res : 0
1781      __ Csel(res, res, Operand(0), vc);  // VC for "not unordered".
1782    }
1783  } else {
1784    // Integer cases.
1785    Register lhs = InputRegisterAt(instruction, 0);
1786    Operand rhs = InputOperandAt(instruction, 1);
1787    __ Cmp(lhs, rhs);
1788    __ Cset(res, arm64_cond);
1789  }
1790}
1791
1792#define FOR_EACH_CONDITION_INSTRUCTION(M)                                                \
1793  M(Equal)                                                                               \
1794  M(NotEqual)                                                                            \
1795  M(LessThan)                                                                            \
1796  M(LessThanOrEqual)                                                                     \
1797  M(GreaterThan)                                                                         \
1798  M(GreaterThanOrEqual)
1799#define DEFINE_CONDITION_VISITORS(Name)                                                  \
1800void LocationsBuilderARM64::Visit##Name(H##Name* comp) { VisitCondition(comp); }         \
1801void InstructionCodeGeneratorARM64::Visit##Name(H##Name* comp) { VisitCondition(comp); }
1802FOR_EACH_CONDITION_INSTRUCTION(DEFINE_CONDITION_VISITORS)
1803#undef DEFINE_CONDITION_VISITORS
1804#undef FOR_EACH_CONDITION_INSTRUCTION
1805
1806void InstructionCodeGeneratorARM64::DivRemOneOrMinusOne(HBinaryOperation* instruction) {
1807  DCHECK(instruction->IsDiv() || instruction->IsRem());
1808
1809  LocationSummary* locations = instruction->GetLocations();
1810  Location second = locations->InAt(1);
1811  DCHECK(second.IsConstant());
1812
1813  Register out = OutputRegister(instruction);
1814  Register dividend = InputRegisterAt(instruction, 0);
1815  int64_t imm = Int64FromConstant(second.GetConstant());
1816  DCHECK(imm == 1 || imm == -1);
1817
1818  if (instruction->IsRem()) {
1819    __ Mov(out, 0);
1820  } else {
1821    if (imm == 1) {
1822      __ Mov(out, dividend);
1823    } else {
1824      __ Neg(out, dividend);
1825    }
1826  }
1827}
1828
1829void InstructionCodeGeneratorARM64::DivRemByPowerOfTwo(HBinaryOperation* instruction) {
1830  DCHECK(instruction->IsDiv() || instruction->IsRem());
1831
1832  LocationSummary* locations = instruction->GetLocations();
1833  Location second = locations->InAt(1);
1834  DCHECK(second.IsConstant());
1835
1836  Register out = OutputRegister(instruction);
1837  Register dividend = InputRegisterAt(instruction, 0);
1838  int64_t imm = Int64FromConstant(second.GetConstant());
1839  uint64_t abs_imm = static_cast<uint64_t>(std::abs(imm));
1840  DCHECK(IsPowerOfTwo(abs_imm));
1841  int ctz_imm = CTZ(abs_imm);
1842
1843  UseScratchRegisterScope temps(GetVIXLAssembler());
1844  Register temp = temps.AcquireSameSizeAs(out);
1845
1846  if (instruction->IsDiv()) {
1847    __ Add(temp, dividend, abs_imm - 1);
1848    __ Cmp(dividend, 0);
1849    __ Csel(out, temp, dividend, lt);
1850    if (imm > 0) {
1851      __ Asr(out, out, ctz_imm);
1852    } else {
1853      __ Neg(out, Operand(out, ASR, ctz_imm));
1854    }
1855  } else {
1856    int bits = instruction->GetResultType() == Primitive::kPrimInt ? 32 : 64;
1857    __ Asr(temp, dividend, bits - 1);
1858    __ Lsr(temp, temp, bits - ctz_imm);
1859    __ Add(out, dividend, temp);
1860    __ And(out, out, abs_imm - 1);
1861    __ Sub(out, out, temp);
1862  }
1863}
1864
1865void InstructionCodeGeneratorARM64::GenerateDivRemWithAnyConstant(HBinaryOperation* instruction) {
1866  DCHECK(instruction->IsDiv() || instruction->IsRem());
1867
1868  LocationSummary* locations = instruction->GetLocations();
1869  Location second = locations->InAt(1);
1870  DCHECK(second.IsConstant());
1871
1872  Register out = OutputRegister(instruction);
1873  Register dividend = InputRegisterAt(instruction, 0);
1874  int64_t imm = Int64FromConstant(second.GetConstant());
1875
1876  Primitive::Type type = instruction->GetResultType();
1877  DCHECK(type == Primitive::kPrimInt || type == Primitive::kPrimLong);
1878
1879  int64_t magic;
1880  int shift;
1881  CalculateMagicAndShiftForDivRem(imm, type == Primitive::kPrimLong /* is_long */, &magic, &shift);
1882
1883  UseScratchRegisterScope temps(GetVIXLAssembler());
1884  Register temp = temps.AcquireSameSizeAs(out);
1885
1886  // temp = get_high(dividend * magic)
1887  __ Mov(temp, magic);
1888  if (type == Primitive::kPrimLong) {
1889    __ Smulh(temp, dividend, temp);
1890  } else {
1891    __ Smull(temp.X(), dividend, temp);
1892    __ Lsr(temp.X(), temp.X(), 32);
1893  }
1894
1895  if (imm > 0 && magic < 0) {
1896    __ Add(temp, temp, dividend);
1897  } else if (imm < 0 && magic > 0) {
1898    __ Sub(temp, temp, dividend);
1899  }
1900
1901  if (shift != 0) {
1902    __ Asr(temp, temp, shift);
1903  }
1904
1905  if (instruction->IsDiv()) {
1906    __ Sub(out, temp, Operand(temp, ASR, type == Primitive::kPrimLong ? 63 : 31));
1907  } else {
1908    __ Sub(temp, temp, Operand(temp, ASR, type == Primitive::kPrimLong ? 63 : 31));
1909    // TODO: Strength reduction for msub.
1910    Register temp_imm = temps.AcquireSameSizeAs(out);
1911    __ Mov(temp_imm, imm);
1912    __ Msub(out, temp, temp_imm, dividend);
1913  }
1914}
1915
1916void InstructionCodeGeneratorARM64::GenerateDivRemIntegral(HBinaryOperation* instruction) {
1917  DCHECK(instruction->IsDiv() || instruction->IsRem());
1918  Primitive::Type type = instruction->GetResultType();
1919  DCHECK(type == Primitive::kPrimInt || Primitive::kPrimLong);
1920
1921  LocationSummary* locations = instruction->GetLocations();
1922  Register out = OutputRegister(instruction);
1923  Location second = locations->InAt(1);
1924
1925  if (second.IsConstant()) {
1926    int64_t imm = Int64FromConstant(second.GetConstant());
1927
1928    if (imm == 0) {
1929      // Do not generate anything. DivZeroCheck would prevent any code to be executed.
1930    } else if (imm == 1 || imm == -1) {
1931      DivRemOneOrMinusOne(instruction);
1932    } else if (IsPowerOfTwo(std::abs(imm))) {
1933      DivRemByPowerOfTwo(instruction);
1934    } else {
1935      DCHECK(imm <= -2 || imm >= 2);
1936      GenerateDivRemWithAnyConstant(instruction);
1937    }
1938  } else {
1939    Register dividend = InputRegisterAt(instruction, 0);
1940    Register divisor = InputRegisterAt(instruction, 1);
1941    if (instruction->IsDiv()) {
1942      __ Sdiv(out, dividend, divisor);
1943    } else {
1944      UseScratchRegisterScope temps(GetVIXLAssembler());
1945      Register temp = temps.AcquireSameSizeAs(out);
1946      __ Sdiv(temp, dividend, divisor);
1947      __ Msub(out, temp, divisor, dividend);
1948    }
1949  }
1950}
1951
1952void LocationsBuilderARM64::VisitDiv(HDiv* div) {
1953  LocationSummary* locations =
1954      new (GetGraph()->GetArena()) LocationSummary(div, LocationSummary::kNoCall);
1955  switch (div->GetResultType()) {
1956    case Primitive::kPrimInt:
1957    case Primitive::kPrimLong:
1958      locations->SetInAt(0, Location::RequiresRegister());
1959      locations->SetInAt(1, Location::RegisterOrConstant(div->InputAt(1)));
1960      locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
1961      break;
1962
1963    case Primitive::kPrimFloat:
1964    case Primitive::kPrimDouble:
1965      locations->SetInAt(0, Location::RequiresFpuRegister());
1966      locations->SetInAt(1, Location::RequiresFpuRegister());
1967      locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap);
1968      break;
1969
1970    default:
1971      LOG(FATAL) << "Unexpected div type " << div->GetResultType();
1972  }
1973}
1974
1975void InstructionCodeGeneratorARM64::VisitDiv(HDiv* div) {
1976  Primitive::Type type = div->GetResultType();
1977  switch (type) {
1978    case Primitive::kPrimInt:
1979    case Primitive::kPrimLong:
1980      GenerateDivRemIntegral(div);
1981      break;
1982
1983    case Primitive::kPrimFloat:
1984    case Primitive::kPrimDouble:
1985      __ Fdiv(OutputFPRegister(div), InputFPRegisterAt(div, 0), InputFPRegisterAt(div, 1));
1986      break;
1987
1988    default:
1989      LOG(FATAL) << "Unexpected div type " << type;
1990  }
1991}
1992
1993void LocationsBuilderARM64::VisitDivZeroCheck(HDivZeroCheck* instruction) {
1994  LocationSummary::CallKind call_kind = instruction->CanThrowIntoCatchBlock()
1995      ? LocationSummary::kCallOnSlowPath
1996      : LocationSummary::kNoCall;
1997  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction, call_kind);
1998  locations->SetInAt(0, Location::RegisterOrConstant(instruction->InputAt(0)));
1999  if (instruction->HasUses()) {
2000    locations->SetOut(Location::SameAsFirstInput());
2001  }
2002}
2003
2004void InstructionCodeGeneratorARM64::VisitDivZeroCheck(HDivZeroCheck* instruction) {
2005  SlowPathCodeARM64* slow_path =
2006      new (GetGraph()->GetArena()) DivZeroCheckSlowPathARM64(instruction);
2007  codegen_->AddSlowPath(slow_path);
2008  Location value = instruction->GetLocations()->InAt(0);
2009
2010  Primitive::Type type = instruction->GetType();
2011
2012  if ((type == Primitive::kPrimBoolean) || !Primitive::IsIntegralType(type)) {
2013      LOG(FATAL) << "Unexpected type " << type << " for DivZeroCheck.";
2014    return;
2015  }
2016
2017  if (value.IsConstant()) {
2018    int64_t divisor = Int64ConstantFrom(value);
2019    if (divisor == 0) {
2020      __ B(slow_path->GetEntryLabel());
2021    } else {
2022      // A division by a non-null constant is valid. We don't need to perform
2023      // any check, so simply fall through.
2024    }
2025  } else {
2026    __ Cbz(InputRegisterAt(instruction, 0), slow_path->GetEntryLabel());
2027  }
2028}
2029
2030void LocationsBuilderARM64::VisitDoubleConstant(HDoubleConstant* constant) {
2031  LocationSummary* locations =
2032      new (GetGraph()->GetArena()) LocationSummary(constant, LocationSummary::kNoCall);
2033  locations->SetOut(Location::ConstantLocation(constant));
2034}
2035
2036void InstructionCodeGeneratorARM64::VisitDoubleConstant(HDoubleConstant* constant) {
2037  UNUSED(constant);
2038  // Will be generated at use site.
2039}
2040
2041void LocationsBuilderARM64::VisitExit(HExit* exit) {
2042  exit->SetLocations(nullptr);
2043}
2044
2045void InstructionCodeGeneratorARM64::VisitExit(HExit* exit) {
2046  UNUSED(exit);
2047}
2048
2049void LocationsBuilderARM64::VisitFloatConstant(HFloatConstant* constant) {
2050  LocationSummary* locations =
2051      new (GetGraph()->GetArena()) LocationSummary(constant, LocationSummary::kNoCall);
2052  locations->SetOut(Location::ConstantLocation(constant));
2053}
2054
2055void InstructionCodeGeneratorARM64::VisitFloatConstant(HFloatConstant* constant) {
2056  UNUSED(constant);
2057  // Will be generated at use site.
2058}
2059
2060void InstructionCodeGeneratorARM64::HandleGoto(HInstruction* got, HBasicBlock* successor) {
2061  DCHECK(!successor->IsExitBlock());
2062  HBasicBlock* block = got->GetBlock();
2063  HInstruction* previous = got->GetPrevious();
2064  HLoopInformation* info = block->GetLoopInformation();
2065
2066  if (info != nullptr && info->IsBackEdge(*block) && info->HasSuspendCheck()) {
2067    codegen_->ClearSpillSlotsFromLoopPhisInStackMap(info->GetSuspendCheck());
2068    GenerateSuspendCheck(info->GetSuspendCheck(), successor);
2069    return;
2070  }
2071  if (block->IsEntryBlock() && (previous != nullptr) && previous->IsSuspendCheck()) {
2072    GenerateSuspendCheck(previous->AsSuspendCheck(), nullptr);
2073  }
2074  if (!codegen_->GoesToNextBlock(block, successor)) {
2075    __ B(codegen_->GetLabelOf(successor));
2076  }
2077}
2078
2079void LocationsBuilderARM64::VisitGoto(HGoto* got) {
2080  got->SetLocations(nullptr);
2081}
2082
2083void InstructionCodeGeneratorARM64::VisitGoto(HGoto* got) {
2084  HandleGoto(got, got->GetSuccessor());
2085}
2086
2087void LocationsBuilderARM64::VisitTryBoundary(HTryBoundary* try_boundary) {
2088  try_boundary->SetLocations(nullptr);
2089}
2090
2091void InstructionCodeGeneratorARM64::VisitTryBoundary(HTryBoundary* try_boundary) {
2092  HBasicBlock* successor = try_boundary->GetNormalFlowSuccessor();
2093  if (!successor->IsExitBlock()) {
2094    HandleGoto(try_boundary, successor);
2095  }
2096}
2097
2098void InstructionCodeGeneratorARM64::GenerateTestAndBranch(HInstruction* instruction,
2099                                                          vixl::Label* true_target,
2100                                                          vixl::Label* false_target,
2101                                                          vixl::Label* always_true_target) {
2102  HInstruction* cond = instruction->InputAt(0);
2103  HCondition* condition = cond->AsCondition();
2104
2105  if (cond->IsIntConstant()) {
2106    int32_t cond_value = cond->AsIntConstant()->GetValue();
2107    if (cond_value == 1) {
2108      if (always_true_target != nullptr) {
2109        __ B(always_true_target);
2110      }
2111      return;
2112    } else {
2113      DCHECK_EQ(cond_value, 0);
2114    }
2115  } else if (!cond->IsCondition() || condition->NeedsMaterialization()) {
2116    // The condition instruction has been materialized, compare the output to 0.
2117    Location cond_val = instruction->GetLocations()->InAt(0);
2118    DCHECK(cond_val.IsRegister());
2119    __ Cbnz(InputRegisterAt(instruction, 0), true_target);
2120  } else {
2121    // The condition instruction has not been materialized, use its inputs as
2122    // the comparison and its condition as the branch condition.
2123    Primitive::Type type =
2124        cond->IsCondition() ? cond->InputAt(0)->GetType() : Primitive::kPrimInt;
2125
2126    if (Primitive::IsFloatingPointType(type)) {
2127      // FP compares don't like null false_targets.
2128      if (false_target == nullptr) {
2129        false_target = codegen_->GetLabelOf(instruction->AsIf()->IfFalseSuccessor());
2130      }
2131      FPRegister lhs = InputFPRegisterAt(condition, 0);
2132      if (condition->GetLocations()->InAt(1).IsConstant()) {
2133        DCHECK(IsFloatingPointZeroConstant(condition->GetLocations()->InAt(1).GetConstant()));
2134        // 0.0 is the only immediate that can be encoded directly in an FCMP instruction.
2135        __ Fcmp(lhs, 0.0);
2136      } else {
2137        __ Fcmp(lhs, InputFPRegisterAt(condition, 1));
2138      }
2139      if (condition->IsFPConditionTrueIfNaN()) {
2140        __ B(vs, true_target);  // VS for unordered.
2141      } else if (condition->IsFPConditionFalseIfNaN()) {
2142        __ B(vs, false_target);  // VS for unordered.
2143      }
2144      __ B(ARM64Condition(condition->GetCondition()), true_target);
2145    } else {
2146      // Integer cases.
2147      Register lhs = InputRegisterAt(condition, 0);
2148      Operand rhs = InputOperandAt(condition, 1);
2149      Condition arm64_cond = ARM64Condition(condition->GetCondition());
2150      if ((arm64_cond != gt && arm64_cond != le) && rhs.IsImmediate() && (rhs.immediate() == 0)) {
2151        switch (arm64_cond) {
2152          case eq:
2153            __ Cbz(lhs, true_target);
2154            break;
2155          case ne:
2156            __ Cbnz(lhs, true_target);
2157            break;
2158          case lt:
2159            // Test the sign bit and branch accordingly.
2160            __ Tbnz(lhs, (lhs.IsX() ? kXRegSize : kWRegSize) - 1, true_target);
2161            break;
2162          case ge:
2163            // Test the sign bit and branch accordingly.
2164            __ Tbz(lhs, (lhs.IsX() ? kXRegSize : kWRegSize) - 1, true_target);
2165            break;
2166          default:
2167            // Without the `static_cast` the compiler throws an error for
2168            // `-Werror=sign-promo`.
2169            LOG(FATAL) << "Unexpected condition: " << static_cast<int>(arm64_cond);
2170        }
2171      } else {
2172        __ Cmp(lhs, rhs);
2173        __ B(arm64_cond, true_target);
2174      }
2175    }
2176  }
2177  if (false_target != nullptr) {
2178    __ B(false_target);
2179  }
2180}
2181
2182void LocationsBuilderARM64::VisitIf(HIf* if_instr) {
2183  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(if_instr);
2184  HInstruction* cond = if_instr->InputAt(0);
2185  if (!cond->IsCondition() || cond->AsCondition()->NeedsMaterialization()) {
2186    locations->SetInAt(0, Location::RequiresRegister());
2187  }
2188}
2189
2190void InstructionCodeGeneratorARM64::VisitIf(HIf* if_instr) {
2191  vixl::Label* true_target = codegen_->GetLabelOf(if_instr->IfTrueSuccessor());
2192  vixl::Label* false_target = codegen_->GetLabelOf(if_instr->IfFalseSuccessor());
2193  vixl::Label* always_true_target = true_target;
2194  if (codegen_->GoesToNextBlock(if_instr->GetBlock(),
2195                                if_instr->IfTrueSuccessor())) {
2196    always_true_target = nullptr;
2197  }
2198  if (codegen_->GoesToNextBlock(if_instr->GetBlock(),
2199                                if_instr->IfFalseSuccessor())) {
2200    false_target = nullptr;
2201  }
2202  GenerateTestAndBranch(if_instr, true_target, false_target, always_true_target);
2203}
2204
2205void LocationsBuilderARM64::VisitDeoptimize(HDeoptimize* deoptimize) {
2206  LocationSummary* locations = new (GetGraph()->GetArena())
2207      LocationSummary(deoptimize, LocationSummary::kCallOnSlowPath);
2208  HInstruction* cond = deoptimize->InputAt(0);
2209  DCHECK(cond->IsCondition());
2210  if (cond->AsCondition()->NeedsMaterialization()) {
2211    locations->SetInAt(0, Location::RequiresRegister());
2212  }
2213}
2214
2215void InstructionCodeGeneratorARM64::VisitDeoptimize(HDeoptimize* deoptimize) {
2216  SlowPathCodeARM64* slow_path = new (GetGraph()->GetArena())
2217      DeoptimizationSlowPathARM64(deoptimize);
2218  codegen_->AddSlowPath(slow_path);
2219  vixl::Label* slow_path_entry = slow_path->GetEntryLabel();
2220  GenerateTestAndBranch(deoptimize, slow_path_entry, nullptr, slow_path_entry);
2221}
2222
2223void LocationsBuilderARM64::VisitInstanceFieldGet(HInstanceFieldGet* instruction) {
2224  HandleFieldGet(instruction);
2225}
2226
2227void InstructionCodeGeneratorARM64::VisitInstanceFieldGet(HInstanceFieldGet* instruction) {
2228  HandleFieldGet(instruction, instruction->GetFieldInfo());
2229}
2230
2231void LocationsBuilderARM64::VisitInstanceFieldSet(HInstanceFieldSet* instruction) {
2232  HandleFieldSet(instruction);
2233}
2234
2235void InstructionCodeGeneratorARM64::VisitInstanceFieldSet(HInstanceFieldSet* instruction) {
2236  HandleFieldSet(instruction, instruction->GetFieldInfo(), instruction->GetValueCanBeNull());
2237}
2238
2239void LocationsBuilderARM64::VisitInstanceOf(HInstanceOf* instruction) {
2240  LocationSummary::CallKind call_kind =
2241      instruction->IsClassFinal() ? LocationSummary::kNoCall : LocationSummary::kCallOnSlowPath;
2242  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction, call_kind);
2243  locations->SetInAt(0, Location::RequiresRegister());
2244  locations->SetInAt(1, Location::RequiresRegister());
2245  // The output does overlap inputs.
2246  // Note that TypeCheckSlowPathARM64 uses this register too.
2247  locations->SetOut(Location::RequiresRegister(), Location::kOutputOverlap);
2248}
2249
2250void InstructionCodeGeneratorARM64::VisitInstanceOf(HInstanceOf* instruction) {
2251  LocationSummary* locations = instruction->GetLocations();
2252  Register obj = InputRegisterAt(instruction, 0);;
2253  Register cls = InputRegisterAt(instruction, 1);;
2254  Register out = OutputRegister(instruction);
2255
2256  vixl::Label done;
2257
2258  // Return 0 if `obj` is null.
2259  // Avoid null check if we know `obj` is not null.
2260  if (instruction->MustDoNullCheck()) {
2261    __ Mov(out, 0);
2262    __ Cbz(obj, &done);
2263  }
2264
2265  // Compare the class of `obj` with `cls`.
2266  __ Ldr(out, HeapOperand(obj, mirror::Object::ClassOffset()));
2267  GetAssembler()->MaybeUnpoisonHeapReference(out.W());
2268  __ Cmp(out, cls);
2269  if (instruction->IsClassFinal()) {
2270    // Classes must be equal for the instanceof to succeed.
2271    __ Cset(out, eq);
2272  } else {
2273    // If the classes are not equal, we go into a slow path.
2274    DCHECK(locations->OnlyCallsOnSlowPath());
2275    SlowPathCodeARM64* slow_path =
2276        new (GetGraph()->GetArena()) TypeCheckSlowPathARM64(instruction);
2277    codegen_->AddSlowPath(slow_path);
2278    __ B(ne, slow_path->GetEntryLabel());
2279    __ Mov(out, 1);
2280    __ Bind(slow_path->GetExitLabel());
2281  }
2282
2283  __ Bind(&done);
2284}
2285
2286void LocationsBuilderARM64::VisitIntConstant(HIntConstant* constant) {
2287  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(constant);
2288  locations->SetOut(Location::ConstantLocation(constant));
2289}
2290
2291void InstructionCodeGeneratorARM64::VisitIntConstant(HIntConstant* constant) {
2292  // Will be generated at use site.
2293  UNUSED(constant);
2294}
2295
2296void LocationsBuilderARM64::VisitNullConstant(HNullConstant* constant) {
2297  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(constant);
2298  locations->SetOut(Location::ConstantLocation(constant));
2299}
2300
2301void InstructionCodeGeneratorARM64::VisitNullConstant(HNullConstant* constant) {
2302  // Will be generated at use site.
2303  UNUSED(constant);
2304}
2305
2306void LocationsBuilderARM64::HandleInvoke(HInvoke* invoke) {
2307  InvokeDexCallingConventionVisitorARM64 calling_convention_visitor;
2308  CodeGenerator::CreateCommonInvokeLocationSummary(invoke, &calling_convention_visitor);
2309}
2310
2311void LocationsBuilderARM64::VisitInvokeInterface(HInvokeInterface* invoke) {
2312  HandleInvoke(invoke);
2313}
2314
2315void InstructionCodeGeneratorARM64::VisitInvokeInterface(HInvokeInterface* invoke) {
2316  // TODO: b/18116999, our IMTs can miss an IncompatibleClassChangeError.
2317  Register temp = XRegisterFrom(invoke->GetLocations()->GetTemp(0));
2318  uint32_t method_offset = mirror::Class::EmbeddedImTableEntryOffset(
2319      invoke->GetImtIndex() % mirror::Class::kImtSize, kArm64PointerSize).Uint32Value();
2320  Location receiver = invoke->GetLocations()->InAt(0);
2321  Offset class_offset = mirror::Object::ClassOffset();
2322  Offset entry_point = ArtMethod::EntryPointFromQuickCompiledCodeOffset(kArm64WordSize);
2323
2324  // The register ip1 is required to be used for the hidden argument in
2325  // art_quick_imt_conflict_trampoline, so prevent VIXL from using it.
2326  MacroAssembler* masm = GetVIXLAssembler();
2327  UseScratchRegisterScope scratch_scope(masm);
2328  BlockPoolsScope block_pools(masm);
2329  scratch_scope.Exclude(ip1);
2330  __ Mov(ip1, invoke->GetDexMethodIndex());
2331
2332  // temp = object->GetClass();
2333  if (receiver.IsStackSlot()) {
2334    __ Ldr(temp.W(), StackOperandFrom(receiver));
2335    __ Ldr(temp.W(), HeapOperand(temp.W(), class_offset));
2336  } else {
2337    __ Ldr(temp.W(), HeapOperandFrom(receiver, class_offset));
2338  }
2339  codegen_->MaybeRecordImplicitNullCheck(invoke);
2340  GetAssembler()->MaybeUnpoisonHeapReference(temp.W());
2341  // temp = temp->GetImtEntryAt(method_offset);
2342  __ Ldr(temp, MemOperand(temp, method_offset));
2343  // lr = temp->GetEntryPoint();
2344  __ Ldr(lr, MemOperand(temp, entry_point.Int32Value()));
2345  // lr();
2346  __ Blr(lr);
2347  DCHECK(!codegen_->IsLeafMethod());
2348  codegen_->RecordPcInfo(invoke, invoke->GetDexPc());
2349}
2350
2351void LocationsBuilderARM64::VisitInvokeVirtual(HInvokeVirtual* invoke) {
2352  IntrinsicLocationsBuilderARM64 intrinsic(GetGraph()->GetArena());
2353  if (intrinsic.TryDispatch(invoke)) {
2354    return;
2355  }
2356
2357  HandleInvoke(invoke);
2358}
2359
2360void LocationsBuilderARM64::VisitInvokeStaticOrDirect(HInvokeStaticOrDirect* invoke) {
2361  // When we do not run baseline, explicit clinit checks triggered by static
2362  // invokes must have been pruned by art::PrepareForRegisterAllocation.
2363  DCHECK(codegen_->IsBaseline() || !invoke->IsStaticWithExplicitClinitCheck());
2364
2365  IntrinsicLocationsBuilderARM64 intrinsic(GetGraph()->GetArena());
2366  if (intrinsic.TryDispatch(invoke)) {
2367    return;
2368  }
2369
2370  HandleInvoke(invoke);
2371}
2372
2373static bool TryGenerateIntrinsicCode(HInvoke* invoke, CodeGeneratorARM64* codegen) {
2374  if (invoke->GetLocations()->Intrinsified()) {
2375    IntrinsicCodeGeneratorARM64 intrinsic(codegen);
2376    intrinsic.Dispatch(invoke);
2377    return true;
2378  }
2379  return false;
2380}
2381
2382void CodeGeneratorARM64::GenerateStaticOrDirectCall(HInvokeStaticOrDirect* invoke, Location temp) {
2383  // For better instruction scheduling we load the direct code pointer before the method pointer.
2384  bool direct_code_loaded = false;
2385  switch (invoke->GetCodePtrLocation()) {
2386    case HInvokeStaticOrDirect::CodePtrLocation::kCallDirectWithFixup:
2387      // LR = code address from literal pool with link-time patch.
2388      __ Ldr(lr, DeduplicateMethodCodeLiteral(invoke->GetTargetMethod()));
2389      direct_code_loaded = true;
2390      break;
2391    case HInvokeStaticOrDirect::CodePtrLocation::kCallDirect:
2392      // LR = invoke->GetDirectCodePtr();
2393      __ Ldr(lr, DeduplicateUint64Literal(invoke->GetDirectCodePtr()));
2394      direct_code_loaded = true;
2395      break;
2396    default:
2397      break;
2398  }
2399
2400  // Make sure that ArtMethod* is passed in kArtMethodRegister as per the calling convention.
2401  Location callee_method = temp;  // For all kinds except kRecursive, callee will be in temp.
2402  switch (invoke->GetMethodLoadKind()) {
2403    case HInvokeStaticOrDirect::MethodLoadKind::kStringInit:
2404      // temp = thread->string_init_entrypoint
2405      __ Ldr(XRegisterFrom(temp).X(), MemOperand(tr, invoke->GetStringInitOffset()));
2406      break;
2407    case HInvokeStaticOrDirect::MethodLoadKind::kRecursive:
2408      callee_method = invoke->GetLocations()->InAt(invoke->GetCurrentMethodInputIndex());
2409      break;
2410    case HInvokeStaticOrDirect::MethodLoadKind::kDirectAddress:
2411      // Load method address from literal pool.
2412      __ Ldr(XRegisterFrom(temp).X(), DeduplicateUint64Literal(invoke->GetMethodAddress()));
2413      break;
2414    case HInvokeStaticOrDirect::MethodLoadKind::kDirectAddressWithFixup:
2415      // Load method address from literal pool with a link-time patch.
2416      __ Ldr(XRegisterFrom(temp).X(),
2417             DeduplicateMethodAddressLiteral(invoke->GetTargetMethod()));
2418      break;
2419    case HInvokeStaticOrDirect::MethodLoadKind::kDexCachePcRelative: {
2420      // Add ADRP with its PC-relative DexCache access patch.
2421      pc_rel_dex_cache_patches_.emplace_back(*invoke->GetTargetMethod().dex_file,
2422                                             invoke->GetDexCacheArrayOffset());
2423      vixl::Label* pc_insn_label = &pc_rel_dex_cache_patches_.back().label;
2424      {
2425        vixl::SingleEmissionCheckScope guard(GetVIXLAssembler());
2426        __ adrp(XRegisterFrom(temp).X(), 0);
2427      }
2428      __ Bind(pc_insn_label);  // Bind after ADRP.
2429      pc_rel_dex_cache_patches_.back().pc_insn_label = pc_insn_label;
2430      // Add LDR with its PC-relative DexCache access patch.
2431      pc_rel_dex_cache_patches_.emplace_back(*invoke->GetTargetMethod().dex_file,
2432                                             invoke->GetDexCacheArrayOffset());
2433      __ Ldr(XRegisterFrom(temp).X(), MemOperand(XRegisterFrom(temp).X(), 0));
2434      __ Bind(&pc_rel_dex_cache_patches_.back().label);  // Bind after LDR.
2435      pc_rel_dex_cache_patches_.back().pc_insn_label = pc_insn_label;
2436      break;
2437    }
2438    case HInvokeStaticOrDirect::MethodLoadKind::kDexCacheViaMethod: {
2439      Location current_method = invoke->GetLocations()->InAt(invoke->GetCurrentMethodInputIndex());
2440      Register reg = XRegisterFrom(temp);
2441      Register method_reg;
2442      if (current_method.IsRegister()) {
2443        method_reg = XRegisterFrom(current_method);
2444      } else {
2445        DCHECK(invoke->GetLocations()->Intrinsified());
2446        DCHECK(!current_method.IsValid());
2447        method_reg = reg;
2448        __ Ldr(reg.X(), MemOperand(sp, kCurrentMethodStackOffset));
2449      }
2450
2451      // temp = current_method->dex_cache_resolved_methods_;
2452      __ Ldr(reg.X(),
2453             MemOperand(method_reg.X(),
2454                        ArtMethod::DexCacheResolvedMethodsOffset(kArm64WordSize).Int32Value()));
2455      // temp = temp[index_in_cache];
2456      uint32_t index_in_cache = invoke->GetTargetMethod().dex_method_index;
2457    __ Ldr(reg.X(), MemOperand(reg.X(), GetCachePointerOffset(index_in_cache)));
2458      break;
2459    }
2460  }
2461
2462  switch (invoke->GetCodePtrLocation()) {
2463    case HInvokeStaticOrDirect::CodePtrLocation::kCallSelf:
2464      __ Bl(&frame_entry_label_);
2465      break;
2466    case HInvokeStaticOrDirect::CodePtrLocation::kCallPCRelative: {
2467      relative_call_patches_.emplace_back(invoke->GetTargetMethod());
2468      vixl::Label* label = &relative_call_patches_.back().label;
2469      __ Bl(label);  // Arbitrarily branch to the instruction after BL, override at link time.
2470      __ Bind(label);  // Bind after BL.
2471      break;
2472    }
2473    case HInvokeStaticOrDirect::CodePtrLocation::kCallDirectWithFixup:
2474    case HInvokeStaticOrDirect::CodePtrLocation::kCallDirect:
2475      // LR prepared above for better instruction scheduling.
2476      DCHECK(direct_code_loaded);
2477      // lr()
2478      __ Blr(lr);
2479      break;
2480    case HInvokeStaticOrDirect::CodePtrLocation::kCallArtMethod:
2481      // LR = callee_method->entry_point_from_quick_compiled_code_;
2482      __ Ldr(lr, MemOperand(
2483          XRegisterFrom(callee_method).X(),
2484          ArtMethod::EntryPointFromQuickCompiledCodeOffset(kArm64WordSize).Int32Value()));
2485      // lr()
2486      __ Blr(lr);
2487      break;
2488  }
2489
2490  DCHECK(!IsLeafMethod());
2491}
2492
2493void CodeGeneratorARM64::GenerateVirtualCall(HInvokeVirtual* invoke, Location temp_in) {
2494  LocationSummary* locations = invoke->GetLocations();
2495  Location receiver = locations->InAt(0);
2496  Register temp = XRegisterFrom(temp_in);
2497  size_t method_offset = mirror::Class::EmbeddedVTableEntryOffset(
2498      invoke->GetVTableIndex(), kArm64PointerSize).SizeValue();
2499  Offset class_offset = mirror::Object::ClassOffset();
2500  Offset entry_point = ArtMethod::EntryPointFromQuickCompiledCodeOffset(kArm64WordSize);
2501
2502  BlockPoolsScope block_pools(GetVIXLAssembler());
2503
2504  DCHECK(receiver.IsRegister());
2505  __ Ldr(temp.W(), HeapOperandFrom(receiver, class_offset));
2506  MaybeRecordImplicitNullCheck(invoke);
2507  GetAssembler()->MaybeUnpoisonHeapReference(temp.W());
2508  // temp = temp->GetMethodAt(method_offset);
2509  __ Ldr(temp, MemOperand(temp, method_offset));
2510  // lr = temp->GetEntryPoint();
2511  __ Ldr(lr, MemOperand(temp, entry_point.SizeValue()));
2512  // lr();
2513  __ Blr(lr);
2514}
2515
2516void CodeGeneratorARM64::EmitLinkerPatches(ArenaVector<LinkerPatch>* linker_patches) {
2517  DCHECK(linker_patches->empty());
2518  size_t size =
2519      method_patches_.size() +
2520      call_patches_.size() +
2521      relative_call_patches_.size() +
2522      pc_rel_dex_cache_patches_.size();
2523  linker_patches->reserve(size);
2524  for (const auto& entry : method_patches_) {
2525    const MethodReference& target_method = entry.first;
2526    vixl::Literal<uint64_t>* literal = entry.second;
2527    linker_patches->push_back(LinkerPatch::MethodPatch(literal->offset(),
2528                                                       target_method.dex_file,
2529                                                       target_method.dex_method_index));
2530  }
2531  for (const auto& entry : call_patches_) {
2532    const MethodReference& target_method = entry.first;
2533    vixl::Literal<uint64_t>* literal = entry.second;
2534    linker_patches->push_back(LinkerPatch::CodePatch(literal->offset(),
2535                                                     target_method.dex_file,
2536                                                     target_method.dex_method_index));
2537  }
2538  for (const MethodPatchInfo<vixl::Label>& info : relative_call_patches_) {
2539    linker_patches->push_back(LinkerPatch::RelativeCodePatch(info.label.location() - 4u,
2540                                                             info.target_method.dex_file,
2541                                                             info.target_method.dex_method_index));
2542  }
2543  for (const PcRelativeDexCacheAccessInfo& info : pc_rel_dex_cache_patches_) {
2544    linker_patches->push_back(LinkerPatch::DexCacheArrayPatch(info.label.location() - 4u,
2545                                                              &info.target_dex_file,
2546                                                              info.pc_insn_label->location() - 4u,
2547                                                              info.element_offset));
2548  }
2549}
2550
2551vixl::Literal<uint64_t>* CodeGeneratorARM64::DeduplicateUint64Literal(uint64_t value) {
2552  // Look up the literal for value.
2553  auto lb = uint64_literals_.lower_bound(value);
2554  if (lb != uint64_literals_.end() && !uint64_literals_.key_comp()(value, lb->first)) {
2555    return lb->second;
2556  }
2557  // We don't have a literal for this value, insert a new one.
2558  vixl::Literal<uint64_t>* literal = __ CreateLiteralDestroyedWithPool<uint64_t>(value);
2559  uint64_literals_.PutBefore(lb, value, literal);
2560  return literal;
2561}
2562
2563vixl::Literal<uint64_t>* CodeGeneratorARM64::DeduplicateMethodLiteral(
2564    MethodReference target_method,
2565    MethodToLiteralMap* map) {
2566  // Look up the literal for target_method.
2567  auto lb = map->lower_bound(target_method);
2568  if (lb != map->end() && !map->key_comp()(target_method, lb->first)) {
2569    return lb->second;
2570  }
2571  // We don't have a literal for this method yet, insert a new one.
2572  vixl::Literal<uint64_t>* literal = __ CreateLiteralDestroyedWithPool<uint64_t>(0u);
2573  map->PutBefore(lb, target_method, literal);
2574  return literal;
2575}
2576
2577vixl::Literal<uint64_t>* CodeGeneratorARM64::DeduplicateMethodAddressLiteral(
2578    MethodReference target_method) {
2579  return DeduplicateMethodLiteral(target_method, &method_patches_);
2580}
2581
2582vixl::Literal<uint64_t>* CodeGeneratorARM64::DeduplicateMethodCodeLiteral(
2583    MethodReference target_method) {
2584  return DeduplicateMethodLiteral(target_method, &call_patches_);
2585}
2586
2587
2588void InstructionCodeGeneratorARM64::VisitInvokeStaticOrDirect(HInvokeStaticOrDirect* invoke) {
2589  // When we do not run baseline, explicit clinit checks triggered by static
2590  // invokes must have been pruned by art::PrepareForRegisterAllocation.
2591  DCHECK(codegen_->IsBaseline() || !invoke->IsStaticWithExplicitClinitCheck());
2592
2593  if (TryGenerateIntrinsicCode(invoke, codegen_)) {
2594    return;
2595  }
2596
2597  BlockPoolsScope block_pools(GetVIXLAssembler());
2598  LocationSummary* locations = invoke->GetLocations();
2599  codegen_->GenerateStaticOrDirectCall(
2600      invoke, locations->HasTemps() ? locations->GetTemp(0) : Location::NoLocation());
2601  codegen_->RecordPcInfo(invoke, invoke->GetDexPc());
2602}
2603
2604void InstructionCodeGeneratorARM64::VisitInvokeVirtual(HInvokeVirtual* invoke) {
2605  if (TryGenerateIntrinsicCode(invoke, codegen_)) {
2606    return;
2607  }
2608
2609  codegen_->GenerateVirtualCall(invoke, invoke->GetLocations()->GetTemp(0));
2610  DCHECK(!codegen_->IsLeafMethod());
2611  codegen_->RecordPcInfo(invoke, invoke->GetDexPc());
2612}
2613
2614void LocationsBuilderARM64::VisitLoadClass(HLoadClass* cls) {
2615  LocationSummary::CallKind call_kind = cls->CanCallRuntime() ? LocationSummary::kCallOnSlowPath
2616                                                              : LocationSummary::kNoCall;
2617  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(cls, call_kind);
2618  locations->SetInAt(0, Location::RequiresRegister());
2619  locations->SetOut(Location::RequiresRegister());
2620}
2621
2622void InstructionCodeGeneratorARM64::VisitLoadClass(HLoadClass* cls) {
2623  Register out = OutputRegister(cls);
2624  Register current_method = InputRegisterAt(cls, 0);
2625  if (cls->IsReferrersClass()) {
2626    DCHECK(!cls->CanCallRuntime());
2627    DCHECK(!cls->MustGenerateClinitCheck());
2628    __ Ldr(out, MemOperand(current_method, ArtMethod::DeclaringClassOffset().Int32Value()));
2629  } else {
2630    DCHECK(cls->CanCallRuntime());
2631    MemberOffset resolved_types_offset = ArtMethod::DexCacheResolvedTypesOffset(kArm64PointerSize);
2632    __ Ldr(out.X(), MemOperand(current_method, resolved_types_offset.Int32Value()));
2633    __ Ldr(out, MemOperand(out.X(), CodeGenerator::GetCacheOffset(cls->GetTypeIndex())));
2634    // TODO: We will need a read barrier here.
2635
2636    SlowPathCodeARM64* slow_path = new (GetGraph()->GetArena()) LoadClassSlowPathARM64(
2637        cls, cls, cls->GetDexPc(), cls->MustGenerateClinitCheck());
2638    codegen_->AddSlowPath(slow_path);
2639    __ Cbz(out, slow_path->GetEntryLabel());
2640    if (cls->MustGenerateClinitCheck()) {
2641      GenerateClassInitializationCheck(slow_path, out);
2642    } else {
2643      __ Bind(slow_path->GetExitLabel());
2644    }
2645  }
2646}
2647
2648static MemOperand GetExceptionTlsAddress() {
2649  return MemOperand(tr, Thread::ExceptionOffset<kArm64WordSize>().Int32Value());
2650}
2651
2652void LocationsBuilderARM64::VisitLoadException(HLoadException* load) {
2653  LocationSummary* locations =
2654      new (GetGraph()->GetArena()) LocationSummary(load, LocationSummary::kNoCall);
2655  locations->SetOut(Location::RequiresRegister());
2656}
2657
2658void InstructionCodeGeneratorARM64::VisitLoadException(HLoadException* instruction) {
2659  __ Ldr(OutputRegister(instruction), GetExceptionTlsAddress());
2660}
2661
2662void LocationsBuilderARM64::VisitClearException(HClearException* clear) {
2663  new (GetGraph()->GetArena()) LocationSummary(clear, LocationSummary::kNoCall);
2664}
2665
2666void InstructionCodeGeneratorARM64::VisitClearException(HClearException* clear ATTRIBUTE_UNUSED) {
2667  __ Str(wzr, GetExceptionTlsAddress());
2668}
2669
2670void LocationsBuilderARM64::VisitLoadLocal(HLoadLocal* load) {
2671  load->SetLocations(nullptr);
2672}
2673
2674void InstructionCodeGeneratorARM64::VisitLoadLocal(HLoadLocal* load) {
2675  // Nothing to do, this is driven by the code generator.
2676  UNUSED(load);
2677}
2678
2679void LocationsBuilderARM64::VisitLoadString(HLoadString* load) {
2680  LocationSummary* locations =
2681      new (GetGraph()->GetArena()) LocationSummary(load, LocationSummary::kCallOnSlowPath);
2682  locations->SetInAt(0, Location::RequiresRegister());
2683  locations->SetOut(Location::RequiresRegister());
2684}
2685
2686void InstructionCodeGeneratorARM64::VisitLoadString(HLoadString* load) {
2687  SlowPathCodeARM64* slow_path = new (GetGraph()->GetArena()) LoadStringSlowPathARM64(load);
2688  codegen_->AddSlowPath(slow_path);
2689
2690  Register out = OutputRegister(load);
2691  Register current_method = InputRegisterAt(load, 0);
2692  __ Ldr(out, MemOperand(current_method, ArtMethod::DeclaringClassOffset().Int32Value()));
2693  __ Ldr(out.X(), HeapOperand(out, mirror::Class::DexCacheStringsOffset()));
2694  __ Ldr(out, MemOperand(out.X(), CodeGenerator::GetCacheOffset(load->GetStringIndex())));
2695  // TODO: We will need a read barrier here.
2696  __ Cbz(out, slow_path->GetEntryLabel());
2697  __ Bind(slow_path->GetExitLabel());
2698}
2699
2700void LocationsBuilderARM64::VisitLocal(HLocal* local) {
2701  local->SetLocations(nullptr);
2702}
2703
2704void InstructionCodeGeneratorARM64::VisitLocal(HLocal* local) {
2705  DCHECK_EQ(local->GetBlock(), GetGraph()->GetEntryBlock());
2706}
2707
2708void LocationsBuilderARM64::VisitLongConstant(HLongConstant* constant) {
2709  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(constant);
2710  locations->SetOut(Location::ConstantLocation(constant));
2711}
2712
2713void InstructionCodeGeneratorARM64::VisitLongConstant(HLongConstant* constant) {
2714  // Will be generated at use site.
2715  UNUSED(constant);
2716}
2717
2718void LocationsBuilderARM64::VisitMonitorOperation(HMonitorOperation* instruction) {
2719  LocationSummary* locations =
2720      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCall);
2721  InvokeRuntimeCallingConvention calling_convention;
2722  locations->SetInAt(0, LocationFrom(calling_convention.GetRegisterAt(0)));
2723}
2724
2725void InstructionCodeGeneratorARM64::VisitMonitorOperation(HMonitorOperation* instruction) {
2726  codegen_->InvokeRuntime(instruction->IsEnter()
2727        ? QUICK_ENTRY_POINT(pLockObject) : QUICK_ENTRY_POINT(pUnlockObject),
2728      instruction,
2729      instruction->GetDexPc(),
2730      nullptr);
2731  CheckEntrypointTypes<kQuickLockObject, void, mirror::Object*>();
2732}
2733
2734void LocationsBuilderARM64::VisitMul(HMul* mul) {
2735  LocationSummary* locations =
2736      new (GetGraph()->GetArena()) LocationSummary(mul, LocationSummary::kNoCall);
2737  switch (mul->GetResultType()) {
2738    case Primitive::kPrimInt:
2739    case Primitive::kPrimLong:
2740      locations->SetInAt(0, Location::RequiresRegister());
2741      locations->SetInAt(1, Location::RequiresRegister());
2742      locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
2743      break;
2744
2745    case Primitive::kPrimFloat:
2746    case Primitive::kPrimDouble:
2747      locations->SetInAt(0, Location::RequiresFpuRegister());
2748      locations->SetInAt(1, Location::RequiresFpuRegister());
2749      locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap);
2750      break;
2751
2752    default:
2753      LOG(FATAL) << "Unexpected mul type " << mul->GetResultType();
2754  }
2755}
2756
2757void InstructionCodeGeneratorARM64::VisitMul(HMul* mul) {
2758  switch (mul->GetResultType()) {
2759    case Primitive::kPrimInt:
2760    case Primitive::kPrimLong:
2761      __ Mul(OutputRegister(mul), InputRegisterAt(mul, 0), InputRegisterAt(mul, 1));
2762      break;
2763
2764    case Primitive::kPrimFloat:
2765    case Primitive::kPrimDouble:
2766      __ Fmul(OutputFPRegister(mul), InputFPRegisterAt(mul, 0), InputFPRegisterAt(mul, 1));
2767      break;
2768
2769    default:
2770      LOG(FATAL) << "Unexpected mul type " << mul->GetResultType();
2771  }
2772}
2773
2774void LocationsBuilderARM64::VisitNeg(HNeg* neg) {
2775  LocationSummary* locations =
2776      new (GetGraph()->GetArena()) LocationSummary(neg, LocationSummary::kNoCall);
2777  switch (neg->GetResultType()) {
2778    case Primitive::kPrimInt:
2779    case Primitive::kPrimLong:
2780      locations->SetInAt(0, ARM64EncodableConstantOrRegister(neg->InputAt(0), neg));
2781      locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
2782      break;
2783
2784    case Primitive::kPrimFloat:
2785    case Primitive::kPrimDouble:
2786      locations->SetInAt(0, Location::RequiresFpuRegister());
2787      locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap);
2788      break;
2789
2790    default:
2791      LOG(FATAL) << "Unexpected neg type " << neg->GetResultType();
2792  }
2793}
2794
2795void InstructionCodeGeneratorARM64::VisitNeg(HNeg* neg) {
2796  switch (neg->GetResultType()) {
2797    case Primitive::kPrimInt:
2798    case Primitive::kPrimLong:
2799      __ Neg(OutputRegister(neg), InputOperandAt(neg, 0));
2800      break;
2801
2802    case Primitive::kPrimFloat:
2803    case Primitive::kPrimDouble:
2804      __ Fneg(OutputFPRegister(neg), InputFPRegisterAt(neg, 0));
2805      break;
2806
2807    default:
2808      LOG(FATAL) << "Unexpected neg type " << neg->GetResultType();
2809  }
2810}
2811
2812void LocationsBuilderARM64::VisitNewArray(HNewArray* instruction) {
2813  LocationSummary* locations =
2814      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCall);
2815  InvokeRuntimeCallingConvention calling_convention;
2816  locations->AddTemp(LocationFrom(calling_convention.GetRegisterAt(0)));
2817  locations->SetOut(LocationFrom(x0));
2818  locations->SetInAt(0, LocationFrom(calling_convention.GetRegisterAt(1)));
2819  locations->SetInAt(1, LocationFrom(calling_convention.GetRegisterAt(2)));
2820  CheckEntrypointTypes<kQuickAllocArrayWithAccessCheck,
2821                       void*, uint32_t, int32_t, ArtMethod*>();
2822}
2823
2824void InstructionCodeGeneratorARM64::VisitNewArray(HNewArray* instruction) {
2825  LocationSummary* locations = instruction->GetLocations();
2826  InvokeRuntimeCallingConvention calling_convention;
2827  Register type_index = RegisterFrom(locations->GetTemp(0), Primitive::kPrimInt);
2828  DCHECK(type_index.Is(w0));
2829  __ Mov(type_index, instruction->GetTypeIndex());
2830  // Note: if heap poisoning is enabled, the entry point takes cares
2831  // of poisoning the reference.
2832  codegen_->InvokeRuntime(
2833      GetThreadOffset<kArm64WordSize>(instruction->GetEntrypoint()).Int32Value(),
2834      instruction,
2835      instruction->GetDexPc(),
2836      nullptr);
2837  CheckEntrypointTypes<kQuickAllocArrayWithAccessCheck, void*, uint32_t, int32_t, ArtMethod*>();
2838}
2839
2840void LocationsBuilderARM64::VisitNewInstance(HNewInstance* instruction) {
2841  LocationSummary* locations =
2842      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCall);
2843  InvokeRuntimeCallingConvention calling_convention;
2844  locations->AddTemp(LocationFrom(calling_convention.GetRegisterAt(0)));
2845  locations->SetInAt(0, LocationFrom(calling_convention.GetRegisterAt(1)));
2846  locations->SetOut(calling_convention.GetReturnLocation(Primitive::kPrimNot));
2847  CheckEntrypointTypes<kQuickAllocObjectWithAccessCheck, void*, uint32_t, ArtMethod*>();
2848}
2849
2850void InstructionCodeGeneratorARM64::VisitNewInstance(HNewInstance* instruction) {
2851  LocationSummary* locations = instruction->GetLocations();
2852  Register type_index = RegisterFrom(locations->GetTemp(0), Primitive::kPrimInt);
2853  DCHECK(type_index.Is(w0));
2854  __ Mov(type_index, instruction->GetTypeIndex());
2855  // Note: if heap poisoning is enabled, the entry point takes cares
2856  // of poisoning the reference.
2857  codegen_->InvokeRuntime(
2858      GetThreadOffset<kArm64WordSize>(instruction->GetEntrypoint()).Int32Value(),
2859      instruction,
2860      instruction->GetDexPc(),
2861      nullptr);
2862  CheckEntrypointTypes<kQuickAllocObjectWithAccessCheck, void*, uint32_t, ArtMethod*>();
2863}
2864
2865void LocationsBuilderARM64::VisitNot(HNot* instruction) {
2866  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction);
2867  locations->SetInAt(0, Location::RequiresRegister());
2868  locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
2869}
2870
2871void InstructionCodeGeneratorARM64::VisitNot(HNot* instruction) {
2872  switch (instruction->GetResultType()) {
2873    case Primitive::kPrimInt:
2874    case Primitive::kPrimLong:
2875      __ Mvn(OutputRegister(instruction), InputOperandAt(instruction, 0));
2876      break;
2877
2878    default:
2879      LOG(FATAL) << "Unexpected type for not operation " << instruction->GetResultType();
2880  }
2881}
2882
2883void LocationsBuilderARM64::VisitBooleanNot(HBooleanNot* instruction) {
2884  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction);
2885  locations->SetInAt(0, Location::RequiresRegister());
2886  locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
2887}
2888
2889void InstructionCodeGeneratorARM64::VisitBooleanNot(HBooleanNot* instruction) {
2890  __ Eor(OutputRegister(instruction), InputRegisterAt(instruction, 0), vixl::Operand(1));
2891}
2892
2893void LocationsBuilderARM64::VisitNullCheck(HNullCheck* instruction) {
2894  LocationSummary::CallKind call_kind = instruction->CanThrowIntoCatchBlock()
2895      ? LocationSummary::kCallOnSlowPath
2896      : LocationSummary::kNoCall;
2897  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction, call_kind);
2898  locations->SetInAt(0, Location::RequiresRegister());
2899  if (instruction->HasUses()) {
2900    locations->SetOut(Location::SameAsFirstInput());
2901  }
2902}
2903
2904void InstructionCodeGeneratorARM64::GenerateImplicitNullCheck(HNullCheck* instruction) {
2905  if (codegen_->CanMoveNullCheckToUser(instruction)) {
2906    return;
2907  }
2908
2909  BlockPoolsScope block_pools(GetVIXLAssembler());
2910  Location obj = instruction->GetLocations()->InAt(0);
2911  __ Ldr(wzr, HeapOperandFrom(obj, Offset(0)));
2912  codegen_->RecordPcInfo(instruction, instruction->GetDexPc());
2913}
2914
2915void InstructionCodeGeneratorARM64::GenerateExplicitNullCheck(HNullCheck* instruction) {
2916  SlowPathCodeARM64* slow_path = new (GetGraph()->GetArena()) NullCheckSlowPathARM64(instruction);
2917  codegen_->AddSlowPath(slow_path);
2918
2919  LocationSummary* locations = instruction->GetLocations();
2920  Location obj = locations->InAt(0);
2921
2922  __ Cbz(RegisterFrom(obj, instruction->InputAt(0)->GetType()), slow_path->GetEntryLabel());
2923}
2924
2925void InstructionCodeGeneratorARM64::VisitNullCheck(HNullCheck* instruction) {
2926  if (codegen_->IsImplicitNullCheckAllowed(instruction)) {
2927    GenerateImplicitNullCheck(instruction);
2928  } else {
2929    GenerateExplicitNullCheck(instruction);
2930  }
2931}
2932
2933void LocationsBuilderARM64::VisitOr(HOr* instruction) {
2934  HandleBinaryOp(instruction);
2935}
2936
2937void InstructionCodeGeneratorARM64::VisitOr(HOr* instruction) {
2938  HandleBinaryOp(instruction);
2939}
2940
2941void LocationsBuilderARM64::VisitParallelMove(HParallelMove* instruction ATTRIBUTE_UNUSED) {
2942  LOG(FATAL) << "Unreachable";
2943}
2944
2945void InstructionCodeGeneratorARM64::VisitParallelMove(HParallelMove* instruction) {
2946  codegen_->GetMoveResolver()->EmitNativeCode(instruction);
2947}
2948
2949void LocationsBuilderARM64::VisitParameterValue(HParameterValue* instruction) {
2950  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction);
2951  Location location = parameter_visitor_.GetNextLocation(instruction->GetType());
2952  if (location.IsStackSlot()) {
2953    location = Location::StackSlot(location.GetStackIndex() + codegen_->GetFrameSize());
2954  } else if (location.IsDoubleStackSlot()) {
2955    location = Location::DoubleStackSlot(location.GetStackIndex() + codegen_->GetFrameSize());
2956  }
2957  locations->SetOut(location);
2958}
2959
2960void InstructionCodeGeneratorARM64::VisitParameterValue(
2961    HParameterValue* instruction ATTRIBUTE_UNUSED) {
2962  // Nothing to do, the parameter is already at its location.
2963}
2964
2965void LocationsBuilderARM64::VisitCurrentMethod(HCurrentMethod* instruction) {
2966  LocationSummary* locations =
2967      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
2968  locations->SetOut(LocationFrom(kArtMethodRegister));
2969}
2970
2971void InstructionCodeGeneratorARM64::VisitCurrentMethod(
2972    HCurrentMethod* instruction ATTRIBUTE_UNUSED) {
2973  // Nothing to do, the method is already at its location.
2974}
2975
2976void LocationsBuilderARM64::VisitPhi(HPhi* instruction) {
2977  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction);
2978  for (size_t i = 0, e = instruction->InputCount(); i < e; ++i) {
2979    locations->SetInAt(i, Location::Any());
2980  }
2981  locations->SetOut(Location::Any());
2982}
2983
2984void InstructionCodeGeneratorARM64::VisitPhi(HPhi* instruction) {
2985  UNUSED(instruction);
2986  LOG(FATAL) << "Unreachable";
2987}
2988
2989void LocationsBuilderARM64::VisitRem(HRem* rem) {
2990  Primitive::Type type = rem->GetResultType();
2991  LocationSummary::CallKind call_kind =
2992      Primitive::IsFloatingPointType(type) ? LocationSummary::kCall : LocationSummary::kNoCall;
2993  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(rem, call_kind);
2994
2995  switch (type) {
2996    case Primitive::kPrimInt:
2997    case Primitive::kPrimLong:
2998      locations->SetInAt(0, Location::RequiresRegister());
2999      locations->SetInAt(1, Location::RegisterOrConstant(rem->InputAt(1)));
3000      locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
3001      break;
3002
3003    case Primitive::kPrimFloat:
3004    case Primitive::kPrimDouble: {
3005      InvokeRuntimeCallingConvention calling_convention;
3006      locations->SetInAt(0, LocationFrom(calling_convention.GetFpuRegisterAt(0)));
3007      locations->SetInAt(1, LocationFrom(calling_convention.GetFpuRegisterAt(1)));
3008      locations->SetOut(calling_convention.GetReturnLocation(type));
3009
3010      break;
3011    }
3012
3013    default:
3014      LOG(FATAL) << "Unexpected rem type " << type;
3015  }
3016}
3017
3018void InstructionCodeGeneratorARM64::VisitRem(HRem* rem) {
3019  Primitive::Type type = rem->GetResultType();
3020
3021  switch (type) {
3022    case Primitive::kPrimInt:
3023    case Primitive::kPrimLong: {
3024      GenerateDivRemIntegral(rem);
3025      break;
3026    }
3027
3028    case Primitive::kPrimFloat:
3029    case Primitive::kPrimDouble: {
3030      int32_t entry_offset = (type == Primitive::kPrimFloat) ? QUICK_ENTRY_POINT(pFmodf)
3031                                                             : QUICK_ENTRY_POINT(pFmod);
3032      codegen_->InvokeRuntime(entry_offset, rem, rem->GetDexPc(), nullptr);
3033      break;
3034    }
3035
3036    default:
3037      LOG(FATAL) << "Unexpected rem type " << type;
3038  }
3039}
3040
3041void LocationsBuilderARM64::VisitMemoryBarrier(HMemoryBarrier* memory_barrier) {
3042  memory_barrier->SetLocations(nullptr);
3043}
3044
3045void InstructionCodeGeneratorARM64::VisitMemoryBarrier(HMemoryBarrier* memory_barrier) {
3046  GenerateMemoryBarrier(memory_barrier->GetBarrierKind());
3047}
3048
3049void LocationsBuilderARM64::VisitReturn(HReturn* instruction) {
3050  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(instruction);
3051  Primitive::Type return_type = instruction->InputAt(0)->GetType();
3052  locations->SetInAt(0, ARM64ReturnLocation(return_type));
3053}
3054
3055void InstructionCodeGeneratorARM64::VisitReturn(HReturn* instruction) {
3056  UNUSED(instruction);
3057  codegen_->GenerateFrameExit();
3058}
3059
3060void LocationsBuilderARM64::VisitReturnVoid(HReturnVoid* instruction) {
3061  instruction->SetLocations(nullptr);
3062}
3063
3064void InstructionCodeGeneratorARM64::VisitReturnVoid(HReturnVoid* instruction) {
3065  UNUSED(instruction);
3066  codegen_->GenerateFrameExit();
3067}
3068
3069void LocationsBuilderARM64::VisitShl(HShl* shl) {
3070  HandleShift(shl);
3071}
3072
3073void InstructionCodeGeneratorARM64::VisitShl(HShl* shl) {
3074  HandleShift(shl);
3075}
3076
3077void LocationsBuilderARM64::VisitShr(HShr* shr) {
3078  HandleShift(shr);
3079}
3080
3081void InstructionCodeGeneratorARM64::VisitShr(HShr* shr) {
3082  HandleShift(shr);
3083}
3084
3085void LocationsBuilderARM64::VisitStoreLocal(HStoreLocal* store) {
3086  LocationSummary* locations = new (GetGraph()->GetArena()) LocationSummary(store);
3087  Primitive::Type field_type = store->InputAt(1)->GetType();
3088  switch (field_type) {
3089    case Primitive::kPrimNot:
3090    case Primitive::kPrimBoolean:
3091    case Primitive::kPrimByte:
3092    case Primitive::kPrimChar:
3093    case Primitive::kPrimShort:
3094    case Primitive::kPrimInt:
3095    case Primitive::kPrimFloat:
3096      locations->SetInAt(1, Location::StackSlot(codegen_->GetStackSlot(store->GetLocal())));
3097      break;
3098
3099    case Primitive::kPrimLong:
3100    case Primitive::kPrimDouble:
3101      locations->SetInAt(1, Location::DoubleStackSlot(codegen_->GetStackSlot(store->GetLocal())));
3102      break;
3103
3104    default:
3105      LOG(FATAL) << "Unimplemented local type " << field_type;
3106  }
3107}
3108
3109void InstructionCodeGeneratorARM64::VisitStoreLocal(HStoreLocal* store) {
3110  UNUSED(store);
3111}
3112
3113void LocationsBuilderARM64::VisitSub(HSub* instruction) {
3114  HandleBinaryOp(instruction);
3115}
3116
3117void InstructionCodeGeneratorARM64::VisitSub(HSub* instruction) {
3118  HandleBinaryOp(instruction);
3119}
3120
3121void LocationsBuilderARM64::VisitStaticFieldGet(HStaticFieldGet* instruction) {
3122  HandleFieldGet(instruction);
3123}
3124
3125void InstructionCodeGeneratorARM64::VisitStaticFieldGet(HStaticFieldGet* instruction) {
3126  HandleFieldGet(instruction, instruction->GetFieldInfo());
3127}
3128
3129void LocationsBuilderARM64::VisitStaticFieldSet(HStaticFieldSet* instruction) {
3130  HandleFieldSet(instruction);
3131}
3132
3133void InstructionCodeGeneratorARM64::VisitStaticFieldSet(HStaticFieldSet* instruction) {
3134  HandleFieldSet(instruction, instruction->GetFieldInfo(), instruction->GetValueCanBeNull());
3135}
3136
3137void LocationsBuilderARM64::VisitSuspendCheck(HSuspendCheck* instruction) {
3138  new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCallOnSlowPath);
3139}
3140
3141void InstructionCodeGeneratorARM64::VisitSuspendCheck(HSuspendCheck* instruction) {
3142  HBasicBlock* block = instruction->GetBlock();
3143  if (block->GetLoopInformation() != nullptr) {
3144    DCHECK(block->GetLoopInformation()->GetSuspendCheck() == instruction);
3145    // The back edge will generate the suspend check.
3146    return;
3147  }
3148  if (block->IsEntryBlock() && instruction->GetNext()->IsGoto()) {
3149    // The goto will generate the suspend check.
3150    return;
3151  }
3152  GenerateSuspendCheck(instruction, nullptr);
3153}
3154
3155void LocationsBuilderARM64::VisitTemporary(HTemporary* temp) {
3156  temp->SetLocations(nullptr);
3157}
3158
3159void InstructionCodeGeneratorARM64::VisitTemporary(HTemporary* temp) {
3160  // Nothing to do, this is driven by the code generator.
3161  UNUSED(temp);
3162}
3163
3164void LocationsBuilderARM64::VisitThrow(HThrow* instruction) {
3165  LocationSummary* locations =
3166      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kCall);
3167  InvokeRuntimeCallingConvention calling_convention;
3168  locations->SetInAt(0, LocationFrom(calling_convention.GetRegisterAt(0)));
3169}
3170
3171void InstructionCodeGeneratorARM64::VisitThrow(HThrow* instruction) {
3172  codegen_->InvokeRuntime(
3173      QUICK_ENTRY_POINT(pDeliverException), instruction, instruction->GetDexPc(), nullptr);
3174  CheckEntrypointTypes<kQuickDeliverException, void, mirror::Object*>();
3175}
3176
3177void LocationsBuilderARM64::VisitTypeConversion(HTypeConversion* conversion) {
3178  LocationSummary* locations =
3179      new (GetGraph()->GetArena()) LocationSummary(conversion, LocationSummary::kNoCall);
3180  Primitive::Type input_type = conversion->GetInputType();
3181  Primitive::Type result_type = conversion->GetResultType();
3182  DCHECK_NE(input_type, result_type);
3183  if ((input_type == Primitive::kPrimNot) || (input_type == Primitive::kPrimVoid) ||
3184      (result_type == Primitive::kPrimNot) || (result_type == Primitive::kPrimVoid)) {
3185    LOG(FATAL) << "Unexpected type conversion from " << input_type << " to " << result_type;
3186  }
3187
3188  if (Primitive::IsFloatingPointType(input_type)) {
3189    locations->SetInAt(0, Location::RequiresFpuRegister());
3190  } else {
3191    locations->SetInAt(0, Location::RequiresRegister());
3192  }
3193
3194  if (Primitive::IsFloatingPointType(result_type)) {
3195    locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap);
3196  } else {
3197    locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
3198  }
3199}
3200
3201void InstructionCodeGeneratorARM64::VisitTypeConversion(HTypeConversion* conversion) {
3202  Primitive::Type result_type = conversion->GetResultType();
3203  Primitive::Type input_type = conversion->GetInputType();
3204
3205  DCHECK_NE(input_type, result_type);
3206
3207  if (Primitive::IsIntegralType(result_type) && Primitive::IsIntegralType(input_type)) {
3208    int result_size = Primitive::ComponentSize(result_type);
3209    int input_size = Primitive::ComponentSize(input_type);
3210    int min_size = std::min(result_size, input_size);
3211    Register output = OutputRegister(conversion);
3212    Register source = InputRegisterAt(conversion, 0);
3213    if ((result_type == Primitive::kPrimChar) && (input_size < result_size)) {
3214      __ Ubfx(output, source, 0, result_size * kBitsPerByte);
3215    } else if (result_type == Primitive::kPrimInt && input_type == Primitive::kPrimLong) {
3216      // 'int' values are used directly as W registers, discarding the top
3217      // bits, so we don't need to sign-extend and can just perform a move.
3218      // We do not pass the `kDiscardForSameWReg` argument to force clearing the
3219      // top 32 bits of the target register. We theoretically could leave those
3220      // bits unchanged, but we would have to make sure that no code uses a
3221      // 32bit input value as a 64bit value assuming that the top 32 bits are
3222      // zero.
3223      __ Mov(output.W(), source.W());
3224    } else if ((result_type == Primitive::kPrimChar) ||
3225               ((input_type == Primitive::kPrimChar) && (result_size > input_size))) {
3226      __ Ubfx(output, output.IsX() ? source.X() : source.W(), 0, min_size * kBitsPerByte);
3227    } else {
3228      __ Sbfx(output, output.IsX() ? source.X() : source.W(), 0, min_size * kBitsPerByte);
3229    }
3230  } else if (Primitive::IsFloatingPointType(result_type) && Primitive::IsIntegralType(input_type)) {
3231    __ Scvtf(OutputFPRegister(conversion), InputRegisterAt(conversion, 0));
3232  } else if (Primitive::IsIntegralType(result_type) && Primitive::IsFloatingPointType(input_type)) {
3233    CHECK(result_type == Primitive::kPrimInt || result_type == Primitive::kPrimLong);
3234    __ Fcvtzs(OutputRegister(conversion), InputFPRegisterAt(conversion, 0));
3235  } else if (Primitive::IsFloatingPointType(result_type) &&
3236             Primitive::IsFloatingPointType(input_type)) {
3237    __ Fcvt(OutputFPRegister(conversion), InputFPRegisterAt(conversion, 0));
3238  } else {
3239    LOG(FATAL) << "Unexpected or unimplemented type conversion from " << input_type
3240                << " to " << result_type;
3241  }
3242}
3243
3244void LocationsBuilderARM64::VisitUShr(HUShr* ushr) {
3245  HandleShift(ushr);
3246}
3247
3248void InstructionCodeGeneratorARM64::VisitUShr(HUShr* ushr) {
3249  HandleShift(ushr);
3250}
3251
3252void LocationsBuilderARM64::VisitXor(HXor* instruction) {
3253  HandleBinaryOp(instruction);
3254}
3255
3256void InstructionCodeGeneratorARM64::VisitXor(HXor* instruction) {
3257  HandleBinaryOp(instruction);
3258}
3259
3260void LocationsBuilderARM64::VisitBoundType(HBoundType* instruction) {
3261  // Nothing to do, this should be removed during prepare for register allocator.
3262  UNUSED(instruction);
3263  LOG(FATAL) << "Unreachable";
3264}
3265
3266void InstructionCodeGeneratorARM64::VisitBoundType(HBoundType* instruction) {
3267  // Nothing to do, this should be removed during prepare for register allocator.
3268  UNUSED(instruction);
3269  LOG(FATAL) << "Unreachable";
3270}
3271
3272void LocationsBuilderARM64::VisitFakeString(HFakeString* instruction) {
3273  DCHECK(codegen_->IsBaseline());
3274  LocationSummary* locations =
3275      new (GetGraph()->GetArena()) LocationSummary(instruction, LocationSummary::kNoCall);
3276  locations->SetOut(Location::ConstantLocation(GetGraph()->GetNullConstant()));
3277}
3278
3279void InstructionCodeGeneratorARM64::VisitFakeString(HFakeString* instruction ATTRIBUTE_UNUSED) {
3280  DCHECK(codegen_->IsBaseline());
3281  // Will be generated at use site.
3282}
3283
3284#undef __
3285#undef QUICK_ENTRY_POINT
3286
3287}  // namespace arm64
3288}  // namespace art
3289