intrinsics_arm64.cc revision 4bedb3845ac33c95cb779987abd4e76a88b19989 
1/*
2 * Copyright (C) 2015 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *      http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17#include "intrinsics_arm64.h"
18
19#include "arch/arm64/instruction_set_features_arm64.h"
20#include "art_method.h"
21#include "code_generator_arm64.h"
22#include "common_arm64.h"
23#include "entrypoints/quick/quick_entrypoints.h"
24#include "intrinsics.h"
25#include "mirror/array-inl.h"
26#include "mirror/string.h"
27#include "thread.h"
28#include "utils/arm64/assembler_arm64.h"
29#include "utils/arm64/constants_arm64.h"
30
31#include "vixl/a64/disasm-a64.h"
32#include "vixl/a64/macro-assembler-a64.h"
33
34using namespace vixl;   // NOLINT(build/namespaces)
35
36namespace art {
37
38namespace arm64 {
39
40using helpers::DRegisterFrom;
41using helpers::FPRegisterFrom;
42using helpers::HeapOperand;
43using helpers::LocationFrom;
44using helpers::OperandFrom;
45using helpers::RegisterFrom;
46using helpers::SRegisterFrom;
47using helpers::WRegisterFrom;
48using helpers::XRegisterFrom;
49
50namespace {
51
52ALWAYS_INLINE inline MemOperand AbsoluteHeapOperandFrom(Location location, size_t offset = 0) {
53  return MemOperand(XRegisterFrom(location), offset);
54}
55
56}  // namespace
57
58vixl::MacroAssembler* IntrinsicCodeGeneratorARM64::GetVIXLAssembler() {
59  return codegen_->GetAssembler()->vixl_masm_;
60}
61
62ArenaAllocator* IntrinsicCodeGeneratorARM64::GetAllocator() {
63  return codegen_->GetGraph()->GetArena();
64}
65
66#define __ codegen->GetAssembler()->vixl_masm_->
67
68static void MoveFromReturnRegister(Location trg,
69                                   Primitive::Type type,
70                                   CodeGeneratorARM64* codegen) {
71  if (!trg.IsValid()) {
72    DCHECK(type == Primitive::kPrimVoid);
73    return;
74  }
75
76  DCHECK_NE(type, Primitive::kPrimVoid);
77
78  if (Primitive::IsIntegralType(type) || type == Primitive::kPrimNot) {
79    Register trg_reg = RegisterFrom(trg, type);
80    Register res_reg = RegisterFrom(ARM64ReturnLocation(type), type);
81    __ Mov(trg_reg, res_reg, kDiscardForSameWReg);
82  } else {
83    FPRegister trg_reg = FPRegisterFrom(trg, type);
84    FPRegister res_reg = FPRegisterFrom(ARM64ReturnLocation(type), type);
85    __ Fmov(trg_reg, res_reg);
86  }
87}
88
89static void MoveArguments(HInvoke* invoke, CodeGeneratorARM64* codegen) {
90  InvokeDexCallingConventionVisitorARM64 calling_convention_visitor;
91  IntrinsicVisitor::MoveArguments(invoke, codegen, &calling_convention_visitor);
92}
93
94// Slow-path for fallback (calling the managed code to handle the intrinsic) in an intrinsified
95// call. This will copy the arguments into the positions for a regular call.
96//
97// Note: The actual parameters are required to be in the locations given by the invoke's location
98//       summary. If an intrinsic modifies those locations before a slowpath call, they must be
99//       restored!
100class IntrinsicSlowPathARM64 : public SlowPathCodeARM64 {
101 public:
102  explicit IntrinsicSlowPathARM64(HInvoke* invoke) : invoke_(invoke) { }
103
104  void EmitNativeCode(CodeGenerator* codegen_in) OVERRIDE {
105    CodeGeneratorARM64* codegen = down_cast<CodeGeneratorARM64*>(codegen_in);
106    __ Bind(GetEntryLabel());
107
108    SaveLiveRegisters(codegen, invoke_->GetLocations());
109
110    MoveArguments(invoke_, codegen);
111
112    if (invoke_->IsInvokeStaticOrDirect()) {
113      codegen->GenerateStaticOrDirectCall(invoke_->AsInvokeStaticOrDirect(),
114                                          LocationFrom(kArtMethodRegister));
115    } else {
116      codegen->GenerateVirtualCall(invoke_->AsInvokeVirtual(), LocationFrom(kArtMethodRegister));
117    }
118    codegen->RecordPcInfo(invoke_, invoke_->GetDexPc(), this);
119
120    // Copy the result back to the expected output.
121    Location out = invoke_->GetLocations()->Out();
122    if (out.IsValid()) {
123      DCHECK(out.IsRegister());  // TODO: Replace this when we support output in memory.
124      DCHECK(!invoke_->GetLocations()->GetLiveRegisters()->ContainsCoreRegister(out.reg()));
125      MoveFromReturnRegister(out, invoke_->GetType(), codegen);
126    }
127
128    RestoreLiveRegisters(codegen, invoke_->GetLocations());
129    __ B(GetExitLabel());
130  }
131
132  const char* GetDescription() const OVERRIDE { return "IntrinsicSlowPathARM64"; }
133
134 private:
135  // The instruction where this slow path is happening.
136  HInvoke* const invoke_;
137
138  DISALLOW_COPY_AND_ASSIGN(IntrinsicSlowPathARM64);
139};
140
141#undef __
142
143bool IntrinsicLocationsBuilderARM64::TryDispatch(HInvoke* invoke) {
144  Dispatch(invoke);
145  LocationSummary* res = invoke->GetLocations();
146  if (res == nullptr) {
147    return false;
148  }
149  if (kEmitCompilerReadBarrier && res->CanCall()) {
150    // Generating an intrinsic for this HInvoke may produce an
151    // IntrinsicSlowPathARM64 slow path.  Currently this approach
152    // does not work when using read barriers, as the emitted
153    // calling sequence will make use of another slow path
154    // (ReadBarrierForRootSlowPathARM64 for HInvokeStaticOrDirect,
155    // ReadBarrierSlowPathARM64 for HInvokeVirtual).  So we bail
156    // out in this case.
157    //
158    // TODO: Find a way to have intrinsics work with read barriers.
159    invoke->SetLocations(nullptr);
160    return false;
161  }
162  return res->Intrinsified();
163}
164
165#define __ masm->
166
167static void CreateFPToIntLocations(ArenaAllocator* arena, HInvoke* invoke) {
168  LocationSummary* locations = new (arena) LocationSummary(invoke,
169                                                           LocationSummary::kNoCall,
170                                                           kIntrinsified);
171  locations->SetInAt(0, Location::RequiresFpuRegister());
172  locations->SetOut(Location::RequiresRegister());
173}
174
175static void CreateIntToFPLocations(ArenaAllocator* arena, HInvoke* invoke) {
176  LocationSummary* locations = new (arena) LocationSummary(invoke,
177                                                           LocationSummary::kNoCall,
178                                                           kIntrinsified);
179  locations->SetInAt(0, Location::RequiresRegister());
180  locations->SetOut(Location::RequiresFpuRegister());
181}
182
183static void MoveFPToInt(LocationSummary* locations, bool is64bit, vixl::MacroAssembler* masm) {
184  Location input = locations->InAt(0);
185  Location output = locations->Out();
186  __ Fmov(is64bit ? XRegisterFrom(output) : WRegisterFrom(output),
187          is64bit ? DRegisterFrom(input) : SRegisterFrom(input));
188}
189
190static void MoveIntToFP(LocationSummary* locations, bool is64bit, vixl::MacroAssembler* masm) {
191  Location input = locations->InAt(0);
192  Location output = locations->Out();
193  __ Fmov(is64bit ? DRegisterFrom(output) : SRegisterFrom(output),
194          is64bit ? XRegisterFrom(input) : WRegisterFrom(input));
195}
196
197void IntrinsicLocationsBuilderARM64::VisitDoubleDoubleToRawLongBits(HInvoke* invoke) {
198  CreateFPToIntLocations(arena_, invoke);
199}
200void IntrinsicLocationsBuilderARM64::VisitDoubleLongBitsToDouble(HInvoke* invoke) {
201  CreateIntToFPLocations(arena_, invoke);
202}
203
204void IntrinsicCodeGeneratorARM64::VisitDoubleDoubleToRawLongBits(HInvoke* invoke) {
205  MoveFPToInt(invoke->GetLocations(), /* is64bit */ true, GetVIXLAssembler());
206}
207void IntrinsicCodeGeneratorARM64::VisitDoubleLongBitsToDouble(HInvoke* invoke) {
208  MoveIntToFP(invoke->GetLocations(), /* is64bit */ true, GetVIXLAssembler());
209}
210
211void IntrinsicLocationsBuilderARM64::VisitFloatFloatToRawIntBits(HInvoke* invoke) {
212  CreateFPToIntLocations(arena_, invoke);
213}
214void IntrinsicLocationsBuilderARM64::VisitFloatIntBitsToFloat(HInvoke* invoke) {
215  CreateIntToFPLocations(arena_, invoke);
216}
217
218void IntrinsicCodeGeneratorARM64::VisitFloatFloatToRawIntBits(HInvoke* invoke) {
219  MoveFPToInt(invoke->GetLocations(), /* is64bit */ false, GetVIXLAssembler());
220}
221void IntrinsicCodeGeneratorARM64::VisitFloatIntBitsToFloat(HInvoke* invoke) {
222  MoveIntToFP(invoke->GetLocations(), /* is64bit */ false, GetVIXLAssembler());
223}
224
225static void CreateIntToIntLocations(ArenaAllocator* arena, HInvoke* invoke) {
226  LocationSummary* locations = new (arena) LocationSummary(invoke,
227                                                           LocationSummary::kNoCall,
228                                                           kIntrinsified);
229  locations->SetInAt(0, Location::RequiresRegister());
230  locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
231}
232
233static void GenReverseBytes(LocationSummary* locations,
234                            Primitive::Type type,
235                            vixl::MacroAssembler* masm) {
236  Location in = locations->InAt(0);
237  Location out = locations->Out();
238
239  switch (type) {
240    case Primitive::kPrimShort:
241      __ Rev16(WRegisterFrom(out), WRegisterFrom(in));
242      __ Sxth(WRegisterFrom(out), WRegisterFrom(out));
243      break;
244    case Primitive::kPrimInt:
245    case Primitive::kPrimLong:
246      __ Rev(RegisterFrom(out, type), RegisterFrom(in, type));
247      break;
248    default:
249      LOG(FATAL) << "Unexpected size for reverse-bytes: " << type;
250      UNREACHABLE();
251  }
252}
253
254void IntrinsicLocationsBuilderARM64::VisitIntegerReverseBytes(HInvoke* invoke) {
255  CreateIntToIntLocations(arena_, invoke);
256}
257
258void IntrinsicCodeGeneratorARM64::VisitIntegerReverseBytes(HInvoke* invoke) {
259  GenReverseBytes(invoke->GetLocations(), Primitive::kPrimInt, GetVIXLAssembler());
260}
261
262void IntrinsicLocationsBuilderARM64::VisitLongReverseBytes(HInvoke* invoke) {
263  CreateIntToIntLocations(arena_, invoke);
264}
265
266void IntrinsicCodeGeneratorARM64::VisitLongReverseBytes(HInvoke* invoke) {
267  GenReverseBytes(invoke->GetLocations(), Primitive::kPrimLong, GetVIXLAssembler());
268}
269
270void IntrinsicLocationsBuilderARM64::VisitShortReverseBytes(HInvoke* invoke) {
271  CreateIntToIntLocations(arena_, invoke);
272}
273
274void IntrinsicCodeGeneratorARM64::VisitShortReverseBytes(HInvoke* invoke) {
275  GenReverseBytes(invoke->GetLocations(), Primitive::kPrimShort, GetVIXLAssembler());
276}
277
278static void GenNumberOfLeadingZeros(LocationSummary* locations,
279                                    Primitive::Type type,
280                                    vixl::MacroAssembler* masm) {
281  DCHECK(type == Primitive::kPrimInt || type == Primitive::kPrimLong);
282
283  Location in = locations->InAt(0);
284  Location out = locations->Out();
285
286  __ Clz(RegisterFrom(out, type), RegisterFrom(in, type));
287}
288
289void IntrinsicLocationsBuilderARM64::VisitIntegerNumberOfLeadingZeros(HInvoke* invoke) {
290  CreateIntToIntLocations(arena_, invoke);
291}
292
293void IntrinsicCodeGeneratorARM64::VisitIntegerNumberOfLeadingZeros(HInvoke* invoke) {
294  GenNumberOfLeadingZeros(invoke->GetLocations(), Primitive::kPrimInt, GetVIXLAssembler());
295}
296
297void IntrinsicLocationsBuilderARM64::VisitLongNumberOfLeadingZeros(HInvoke* invoke) {
298  CreateIntToIntLocations(arena_, invoke);
299}
300
301void IntrinsicCodeGeneratorARM64::VisitLongNumberOfLeadingZeros(HInvoke* invoke) {
302  GenNumberOfLeadingZeros(invoke->GetLocations(), Primitive::kPrimLong, GetVIXLAssembler());
303}
304
305static void GenNumberOfTrailingZeros(LocationSummary* locations,
306                                     Primitive::Type type,
307                                     vixl::MacroAssembler* masm) {
308  DCHECK(type == Primitive::kPrimInt || type == Primitive::kPrimLong);
309
310  Location in = locations->InAt(0);
311  Location out = locations->Out();
312
313  __ Rbit(RegisterFrom(out, type), RegisterFrom(in, type));
314  __ Clz(RegisterFrom(out, type), RegisterFrom(out, type));
315}
316
317void IntrinsicLocationsBuilderARM64::VisitIntegerNumberOfTrailingZeros(HInvoke* invoke) {
318  CreateIntToIntLocations(arena_, invoke);
319}
320
321void IntrinsicCodeGeneratorARM64::VisitIntegerNumberOfTrailingZeros(HInvoke* invoke) {
322  GenNumberOfTrailingZeros(invoke->GetLocations(), Primitive::kPrimInt, GetVIXLAssembler());
323}
324
325void IntrinsicLocationsBuilderARM64::VisitLongNumberOfTrailingZeros(HInvoke* invoke) {
326  CreateIntToIntLocations(arena_, invoke);
327}
328
329void IntrinsicCodeGeneratorARM64::VisitLongNumberOfTrailingZeros(HInvoke* invoke) {
330  GenNumberOfTrailingZeros(invoke->GetLocations(), Primitive::kPrimLong, GetVIXLAssembler());
331}
332
333static void GenReverse(LocationSummary* locations,
334                       Primitive::Type type,
335                       vixl::MacroAssembler* masm) {
336  DCHECK(type == Primitive::kPrimInt || type == Primitive::kPrimLong);
337
338  Location in = locations->InAt(0);
339  Location out = locations->Out();
340
341  __ Rbit(RegisterFrom(out, type), RegisterFrom(in, type));
342}
343
344void IntrinsicLocationsBuilderARM64::VisitIntegerReverse(HInvoke* invoke) {
345  CreateIntToIntLocations(arena_, invoke);
346}
347
348void IntrinsicCodeGeneratorARM64::VisitIntegerReverse(HInvoke* invoke) {
349  GenReverse(invoke->GetLocations(), Primitive::kPrimInt, GetVIXLAssembler());
350}
351
352void IntrinsicLocationsBuilderARM64::VisitLongReverse(HInvoke* invoke) {
353  CreateIntToIntLocations(arena_, invoke);
354}
355
356void IntrinsicCodeGeneratorARM64::VisitLongReverse(HInvoke* invoke) {
357  GenReverse(invoke->GetLocations(), Primitive::kPrimLong, GetVIXLAssembler());
358}
359
360static void CreateFPToFPLocations(ArenaAllocator* arena, HInvoke* invoke) {
361  LocationSummary* locations = new (arena) LocationSummary(invoke,
362                                                           LocationSummary::kNoCall,
363                                                           kIntrinsified);
364  locations->SetInAt(0, Location::RequiresFpuRegister());
365  locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap);
366}
367
368static void MathAbsFP(LocationSummary* locations, bool is64bit, vixl::MacroAssembler* masm) {
369  Location in = locations->InAt(0);
370  Location out = locations->Out();
371
372  FPRegister in_reg = is64bit ? DRegisterFrom(in) : SRegisterFrom(in);
373  FPRegister out_reg = is64bit ? DRegisterFrom(out) : SRegisterFrom(out);
374
375  __ Fabs(out_reg, in_reg);
376}
377
378void IntrinsicLocationsBuilderARM64::VisitMathAbsDouble(HInvoke* invoke) {
379  CreateFPToFPLocations(arena_, invoke);
380}
381
382void IntrinsicCodeGeneratorARM64::VisitMathAbsDouble(HInvoke* invoke) {
383  MathAbsFP(invoke->GetLocations(), /* is64bit */ true, GetVIXLAssembler());
384}
385
386void IntrinsicLocationsBuilderARM64::VisitMathAbsFloat(HInvoke* invoke) {
387  CreateFPToFPLocations(arena_, invoke);
388}
389
390void IntrinsicCodeGeneratorARM64::VisitMathAbsFloat(HInvoke* invoke) {
391  MathAbsFP(invoke->GetLocations(), /* is64bit */ false, GetVIXLAssembler());
392}
393
394static void CreateIntToInt(ArenaAllocator* arena, HInvoke* invoke) {
395  LocationSummary* locations = new (arena) LocationSummary(invoke,
396                                                           LocationSummary::kNoCall,
397                                                           kIntrinsified);
398  locations->SetInAt(0, Location::RequiresRegister());
399  locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
400}
401
402static void GenAbsInteger(LocationSummary* locations,
403                          bool is64bit,
404                          vixl::MacroAssembler* masm) {
405  Location in = locations->InAt(0);
406  Location output = locations->Out();
407
408  Register in_reg = is64bit ? XRegisterFrom(in) : WRegisterFrom(in);
409  Register out_reg = is64bit ? XRegisterFrom(output) : WRegisterFrom(output);
410
411  __ Cmp(in_reg, Operand(0));
412  __ Cneg(out_reg, in_reg, lt);
413}
414
415void IntrinsicLocationsBuilderARM64::VisitMathAbsInt(HInvoke* invoke) {
416  CreateIntToInt(arena_, invoke);
417}
418
419void IntrinsicCodeGeneratorARM64::VisitMathAbsInt(HInvoke* invoke) {
420  GenAbsInteger(invoke->GetLocations(), /* is64bit */ false, GetVIXLAssembler());
421}
422
423void IntrinsicLocationsBuilderARM64::VisitMathAbsLong(HInvoke* invoke) {
424  CreateIntToInt(arena_, invoke);
425}
426
427void IntrinsicCodeGeneratorARM64::VisitMathAbsLong(HInvoke* invoke) {
428  GenAbsInteger(invoke->GetLocations(), /* is64bit */ true, GetVIXLAssembler());
429}
430
431static void GenMinMaxFP(LocationSummary* locations,
432                        bool is_min,
433                        bool is_double,
434                        vixl::MacroAssembler* masm) {
435  Location op1 = locations->InAt(0);
436  Location op2 = locations->InAt(1);
437  Location out = locations->Out();
438
439  FPRegister op1_reg = is_double ? DRegisterFrom(op1) : SRegisterFrom(op1);
440  FPRegister op2_reg = is_double ? DRegisterFrom(op2) : SRegisterFrom(op2);
441  FPRegister out_reg = is_double ? DRegisterFrom(out) : SRegisterFrom(out);
442  if (is_min) {
443    __ Fmin(out_reg, op1_reg, op2_reg);
444  } else {
445    __ Fmax(out_reg, op1_reg, op2_reg);
446  }
447}
448
449static void CreateFPFPToFPLocations(ArenaAllocator* arena, HInvoke* invoke) {
450  LocationSummary* locations = new (arena) LocationSummary(invoke,
451                                                           LocationSummary::kNoCall,
452                                                           kIntrinsified);
453  locations->SetInAt(0, Location::RequiresFpuRegister());
454  locations->SetInAt(1, Location::RequiresFpuRegister());
455  locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap);
456}
457
458void IntrinsicLocationsBuilderARM64::VisitMathMinDoubleDouble(HInvoke* invoke) {
459  CreateFPFPToFPLocations(arena_, invoke);
460}
461
462void IntrinsicCodeGeneratorARM64::VisitMathMinDoubleDouble(HInvoke* invoke) {
463  GenMinMaxFP(invoke->GetLocations(), /* is_min */ true, /* is_double */ true, GetVIXLAssembler());
464}
465
466void IntrinsicLocationsBuilderARM64::VisitMathMinFloatFloat(HInvoke* invoke) {
467  CreateFPFPToFPLocations(arena_, invoke);
468}
469
470void IntrinsicCodeGeneratorARM64::VisitMathMinFloatFloat(HInvoke* invoke) {
471  GenMinMaxFP(invoke->GetLocations(), /* is_min */ true, /* is_double */ false, GetVIXLAssembler());
472}
473
474void IntrinsicLocationsBuilderARM64::VisitMathMaxDoubleDouble(HInvoke* invoke) {
475  CreateFPFPToFPLocations(arena_, invoke);
476}
477
478void IntrinsicCodeGeneratorARM64::VisitMathMaxDoubleDouble(HInvoke* invoke) {
479  GenMinMaxFP(invoke->GetLocations(), /* is_min */ false, /* is_double */ true, GetVIXLAssembler());
480}
481
482void IntrinsicLocationsBuilderARM64::VisitMathMaxFloatFloat(HInvoke* invoke) {
483  CreateFPFPToFPLocations(arena_, invoke);
484}
485
486void IntrinsicCodeGeneratorARM64::VisitMathMaxFloatFloat(HInvoke* invoke) {
487  GenMinMaxFP(
488      invoke->GetLocations(), /* is_min */ false, /* is_double */ false, GetVIXLAssembler());
489}
490
491static void GenMinMax(LocationSummary* locations,
492                      bool is_min,
493                      bool is_long,
494                      vixl::MacroAssembler* masm) {
495  Location op1 = locations->InAt(0);
496  Location op2 = locations->InAt(1);
497  Location out = locations->Out();
498
499  Register op1_reg = is_long ? XRegisterFrom(op1) : WRegisterFrom(op1);
500  Register op2_reg = is_long ? XRegisterFrom(op2) : WRegisterFrom(op2);
501  Register out_reg = is_long ? XRegisterFrom(out) : WRegisterFrom(out);
502
503  __ Cmp(op1_reg, op2_reg);
504  __ Csel(out_reg, op1_reg, op2_reg, is_min ? lt : gt);
505}
506
507static void CreateIntIntToIntLocations(ArenaAllocator* arena, HInvoke* invoke) {
508  LocationSummary* locations = new (arena) LocationSummary(invoke,
509                                                           LocationSummary::kNoCall,
510                                                           kIntrinsified);
511  locations->SetInAt(0, Location::RequiresRegister());
512  locations->SetInAt(1, Location::RequiresRegister());
513  locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
514}
515
516void IntrinsicLocationsBuilderARM64::VisitMathMinIntInt(HInvoke* invoke) {
517  CreateIntIntToIntLocations(arena_, invoke);
518}
519
520void IntrinsicCodeGeneratorARM64::VisitMathMinIntInt(HInvoke* invoke) {
521  GenMinMax(invoke->GetLocations(), /* is_min */ true, /* is_long */ false, GetVIXLAssembler());
522}
523
524void IntrinsicLocationsBuilderARM64::VisitMathMinLongLong(HInvoke* invoke) {
525  CreateIntIntToIntLocations(arena_, invoke);
526}
527
528void IntrinsicCodeGeneratorARM64::VisitMathMinLongLong(HInvoke* invoke) {
529  GenMinMax(invoke->GetLocations(), /* is_min */ true, /* is_long */ true, GetVIXLAssembler());
530}
531
532void IntrinsicLocationsBuilderARM64::VisitMathMaxIntInt(HInvoke* invoke) {
533  CreateIntIntToIntLocations(arena_, invoke);
534}
535
536void IntrinsicCodeGeneratorARM64::VisitMathMaxIntInt(HInvoke* invoke) {
537  GenMinMax(invoke->GetLocations(), /* is_min */ false, /* is_long */ false, GetVIXLAssembler());
538}
539
540void IntrinsicLocationsBuilderARM64::VisitMathMaxLongLong(HInvoke* invoke) {
541  CreateIntIntToIntLocations(arena_, invoke);
542}
543
544void IntrinsicCodeGeneratorARM64::VisitMathMaxLongLong(HInvoke* invoke) {
545  GenMinMax(invoke->GetLocations(), /* is_min */ false, /* is_long */ true, GetVIXLAssembler());
546}
547
548void IntrinsicLocationsBuilderARM64::VisitMathSqrt(HInvoke* invoke) {
549  CreateFPToFPLocations(arena_, invoke);
550}
551
552void IntrinsicCodeGeneratorARM64::VisitMathSqrt(HInvoke* invoke) {
553  LocationSummary* locations = invoke->GetLocations();
554  vixl::MacroAssembler* masm = GetVIXLAssembler();
555  __ Fsqrt(DRegisterFrom(locations->Out()), DRegisterFrom(locations->InAt(0)));
556}
557
558void IntrinsicLocationsBuilderARM64::VisitMathCeil(HInvoke* invoke) {
559  CreateFPToFPLocations(arena_, invoke);
560}
561
562void IntrinsicCodeGeneratorARM64::VisitMathCeil(HInvoke* invoke) {
563  LocationSummary* locations = invoke->GetLocations();
564  vixl::MacroAssembler* masm = GetVIXLAssembler();
565  __ Frintp(DRegisterFrom(locations->Out()), DRegisterFrom(locations->InAt(0)));
566}
567
568void IntrinsicLocationsBuilderARM64::VisitMathFloor(HInvoke* invoke) {
569  CreateFPToFPLocations(arena_, invoke);
570}
571
572void IntrinsicCodeGeneratorARM64::VisitMathFloor(HInvoke* invoke) {
573  LocationSummary* locations = invoke->GetLocations();
574  vixl::MacroAssembler* masm = GetVIXLAssembler();
575  __ Frintm(DRegisterFrom(locations->Out()), DRegisterFrom(locations->InAt(0)));
576}
577
578void IntrinsicLocationsBuilderARM64::VisitMathRint(HInvoke* invoke) {
579  CreateFPToFPLocations(arena_, invoke);
580}
581
582void IntrinsicCodeGeneratorARM64::VisitMathRint(HInvoke* invoke) {
583  LocationSummary* locations = invoke->GetLocations();
584  vixl::MacroAssembler* masm = GetVIXLAssembler();
585  __ Frintn(DRegisterFrom(locations->Out()), DRegisterFrom(locations->InAt(0)));
586}
587
588static void CreateFPToIntPlusTempLocations(ArenaAllocator* arena, HInvoke* invoke) {
589  LocationSummary* locations = new (arena) LocationSummary(invoke,
590                                                           LocationSummary::kNoCall,
591                                                           kIntrinsified);
592  locations->SetInAt(0, Location::RequiresFpuRegister());
593  locations->SetOut(Location::RequiresRegister());
594}
595
596static void GenMathRound(LocationSummary* locations,
597                         bool is_double,
598                         vixl::MacroAssembler* masm) {
599  FPRegister in_reg = is_double ?
600      DRegisterFrom(locations->InAt(0)) : SRegisterFrom(locations->InAt(0));
601  Register out_reg = is_double ?
602      XRegisterFrom(locations->Out()) : WRegisterFrom(locations->Out());
603  UseScratchRegisterScope temps(masm);
604  FPRegister temp1_reg = temps.AcquireSameSizeAs(in_reg);
605
606  // 0.5 can be encoded as an immediate, so use fmov.
607  if (is_double) {
608    __ Fmov(temp1_reg, static_cast<double>(0.5));
609  } else {
610    __ Fmov(temp1_reg, static_cast<float>(0.5));
611  }
612  __ Fadd(temp1_reg, in_reg, temp1_reg);
613  __ Fcvtms(out_reg, temp1_reg);
614}
615
616void IntrinsicLocationsBuilderARM64::VisitMathRoundDouble(HInvoke* invoke) {
617  // See intrinsics.h.
618  if (kRoundIsPlusPointFive) {
619    CreateFPToIntPlusTempLocations(arena_, invoke);
620  }
621}
622
623void IntrinsicCodeGeneratorARM64::VisitMathRoundDouble(HInvoke* invoke) {
624  GenMathRound(invoke->GetLocations(), /* is_double */ true, GetVIXLAssembler());
625}
626
627void IntrinsicLocationsBuilderARM64::VisitMathRoundFloat(HInvoke* invoke) {
628  // See intrinsics.h.
629  if (kRoundIsPlusPointFive) {
630    CreateFPToIntPlusTempLocations(arena_, invoke);
631  }
632}
633
634void IntrinsicCodeGeneratorARM64::VisitMathRoundFloat(HInvoke* invoke) {
635  GenMathRound(invoke->GetLocations(), /* is_double */ false, GetVIXLAssembler());
636}
637
638void IntrinsicLocationsBuilderARM64::VisitMemoryPeekByte(HInvoke* invoke) {
639  CreateIntToIntLocations(arena_, invoke);
640}
641
642void IntrinsicCodeGeneratorARM64::VisitMemoryPeekByte(HInvoke* invoke) {
643  vixl::MacroAssembler* masm = GetVIXLAssembler();
644  __ Ldrsb(WRegisterFrom(invoke->GetLocations()->Out()),
645          AbsoluteHeapOperandFrom(invoke->GetLocations()->InAt(0), 0));
646}
647
648void IntrinsicLocationsBuilderARM64::VisitMemoryPeekIntNative(HInvoke* invoke) {
649  CreateIntToIntLocations(arena_, invoke);
650}
651
652void IntrinsicCodeGeneratorARM64::VisitMemoryPeekIntNative(HInvoke* invoke) {
653  vixl::MacroAssembler* masm = GetVIXLAssembler();
654  __ Ldr(WRegisterFrom(invoke->GetLocations()->Out()),
655         AbsoluteHeapOperandFrom(invoke->GetLocations()->InAt(0), 0));
656}
657
658void IntrinsicLocationsBuilderARM64::VisitMemoryPeekLongNative(HInvoke* invoke) {
659  CreateIntToIntLocations(arena_, invoke);
660}
661
662void IntrinsicCodeGeneratorARM64::VisitMemoryPeekLongNative(HInvoke* invoke) {
663  vixl::MacroAssembler* masm = GetVIXLAssembler();
664  __ Ldr(XRegisterFrom(invoke->GetLocations()->Out()),
665         AbsoluteHeapOperandFrom(invoke->GetLocations()->InAt(0), 0));
666}
667
668void IntrinsicLocationsBuilderARM64::VisitMemoryPeekShortNative(HInvoke* invoke) {
669  CreateIntToIntLocations(arena_, invoke);
670}
671
672void IntrinsicCodeGeneratorARM64::VisitMemoryPeekShortNative(HInvoke* invoke) {
673  vixl::MacroAssembler* masm = GetVIXLAssembler();
674  __ Ldrsh(WRegisterFrom(invoke->GetLocations()->Out()),
675           AbsoluteHeapOperandFrom(invoke->GetLocations()->InAt(0), 0));
676}
677
678static void CreateIntIntToVoidLocations(ArenaAllocator* arena, HInvoke* invoke) {
679  LocationSummary* locations = new (arena) LocationSummary(invoke,
680                                                           LocationSummary::kNoCall,
681                                                           kIntrinsified);
682  locations->SetInAt(0, Location::RequiresRegister());
683  locations->SetInAt(1, Location::RequiresRegister());
684}
685
686void IntrinsicLocationsBuilderARM64::VisitMemoryPokeByte(HInvoke* invoke) {
687  CreateIntIntToVoidLocations(arena_, invoke);
688}
689
690void IntrinsicCodeGeneratorARM64::VisitMemoryPokeByte(HInvoke* invoke) {
691  vixl::MacroAssembler* masm = GetVIXLAssembler();
692  __ Strb(WRegisterFrom(invoke->GetLocations()->InAt(1)),
693          AbsoluteHeapOperandFrom(invoke->GetLocations()->InAt(0), 0));
694}
695
696void IntrinsicLocationsBuilderARM64::VisitMemoryPokeIntNative(HInvoke* invoke) {
697  CreateIntIntToVoidLocations(arena_, invoke);
698}
699
700void IntrinsicCodeGeneratorARM64::VisitMemoryPokeIntNative(HInvoke* invoke) {
701  vixl::MacroAssembler* masm = GetVIXLAssembler();
702  __ Str(WRegisterFrom(invoke->GetLocations()->InAt(1)),
703         AbsoluteHeapOperandFrom(invoke->GetLocations()->InAt(0), 0));
704}
705
706void IntrinsicLocationsBuilderARM64::VisitMemoryPokeLongNative(HInvoke* invoke) {
707  CreateIntIntToVoidLocations(arena_, invoke);
708}
709
710void IntrinsicCodeGeneratorARM64::VisitMemoryPokeLongNative(HInvoke* invoke) {
711  vixl::MacroAssembler* masm = GetVIXLAssembler();
712  __ Str(XRegisterFrom(invoke->GetLocations()->InAt(1)),
713         AbsoluteHeapOperandFrom(invoke->GetLocations()->InAt(0), 0));
714}
715
716void IntrinsicLocationsBuilderARM64::VisitMemoryPokeShortNative(HInvoke* invoke) {
717  CreateIntIntToVoidLocations(arena_, invoke);
718}
719
720void IntrinsicCodeGeneratorARM64::VisitMemoryPokeShortNative(HInvoke* invoke) {
721  vixl::MacroAssembler* masm = GetVIXLAssembler();
722  __ Strh(WRegisterFrom(invoke->GetLocations()->InAt(1)),
723          AbsoluteHeapOperandFrom(invoke->GetLocations()->InAt(0), 0));
724}
725
726void IntrinsicLocationsBuilderARM64::VisitThreadCurrentThread(HInvoke* invoke) {
727  LocationSummary* locations = new (arena_) LocationSummary(invoke,
728                                                            LocationSummary::kNoCall,
729                                                            kIntrinsified);
730  locations->SetOut(Location::RequiresRegister());
731}
732
733void IntrinsicCodeGeneratorARM64::VisitThreadCurrentThread(HInvoke* invoke) {
734  codegen_->Load(Primitive::kPrimNot, WRegisterFrom(invoke->GetLocations()->Out()),
735                 MemOperand(tr, Thread::PeerOffset<8>().Int32Value()));
736}
737
738static void GenUnsafeGet(HInvoke* invoke,
739                         Primitive::Type type,
740                         bool is_volatile,
741                         CodeGeneratorARM64* codegen) {
742  LocationSummary* locations = invoke->GetLocations();
743  DCHECK((type == Primitive::kPrimInt) ||
744         (type == Primitive::kPrimLong) ||
745         (type == Primitive::kPrimNot));
746  vixl::MacroAssembler* masm = codegen->GetAssembler()->vixl_masm_;
747  Location base_loc = locations->InAt(1);
748  Register base = WRegisterFrom(base_loc);      // Object pointer.
749  Location offset_loc = locations->InAt(2);
750  Register offset = XRegisterFrom(offset_loc);  // Long offset.
751  Location trg_loc = locations->Out();
752  Register trg = RegisterFrom(trg_loc, type);
753  bool use_acquire_release = codegen->GetInstructionSetFeatures().PreferAcquireRelease();
754
755  MemOperand mem_op(base.X(), offset);
756  if (is_volatile) {
757    if (use_acquire_release) {
758      codegen->LoadAcquire(invoke, trg, mem_op);
759    } else {
760      codegen->Load(type, trg, mem_op);
761      __ Dmb(InnerShareable, BarrierReads);
762    }
763  } else {
764    codegen->Load(type, trg, mem_op);
765  }
766
767  if (type == Primitive::kPrimNot) {
768    DCHECK(trg.IsW());
769    codegen->MaybeGenerateReadBarrier(invoke, trg_loc, trg_loc, base_loc, 0U, offset_loc);
770  }
771}
772
773static void CreateIntIntIntToIntLocations(ArenaAllocator* arena, HInvoke* invoke) {
774  bool can_call = kEmitCompilerReadBarrier &&
775      (invoke->GetIntrinsic() == Intrinsics::kUnsafeGetObject ||
776       invoke->GetIntrinsic() == Intrinsics::kUnsafeGetObjectVolatile);
777  LocationSummary* locations = new (arena) LocationSummary(invoke,
778                                                           can_call ?
779                                                               LocationSummary::kCallOnSlowPath :
780                                                               LocationSummary::kNoCall,
781                                                           kIntrinsified);
782  locations->SetInAt(0, Location::NoLocation());        // Unused receiver.
783  locations->SetInAt(1, Location::RequiresRegister());
784  locations->SetInAt(2, Location::RequiresRegister());
785  locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
786}
787
788void IntrinsicLocationsBuilderARM64::VisitUnsafeGet(HInvoke* invoke) {
789  CreateIntIntIntToIntLocations(arena_, invoke);
790}
791void IntrinsicLocationsBuilderARM64::VisitUnsafeGetVolatile(HInvoke* invoke) {
792  CreateIntIntIntToIntLocations(arena_, invoke);
793}
794void IntrinsicLocationsBuilderARM64::VisitUnsafeGetLong(HInvoke* invoke) {
795  CreateIntIntIntToIntLocations(arena_, invoke);
796}
797void IntrinsicLocationsBuilderARM64::VisitUnsafeGetLongVolatile(HInvoke* invoke) {
798  CreateIntIntIntToIntLocations(arena_, invoke);
799}
800void IntrinsicLocationsBuilderARM64::VisitUnsafeGetObject(HInvoke* invoke) {
801  CreateIntIntIntToIntLocations(arena_, invoke);
802}
803void IntrinsicLocationsBuilderARM64::VisitUnsafeGetObjectVolatile(HInvoke* invoke) {
804  CreateIntIntIntToIntLocations(arena_, invoke);
805}
806
807void IntrinsicCodeGeneratorARM64::VisitUnsafeGet(HInvoke* invoke) {
808  GenUnsafeGet(invoke, Primitive::kPrimInt, /* is_volatile */ false, codegen_);
809}
810void IntrinsicCodeGeneratorARM64::VisitUnsafeGetVolatile(HInvoke* invoke) {
811  GenUnsafeGet(invoke, Primitive::kPrimInt, /* is_volatile */ true, codegen_);
812}
813void IntrinsicCodeGeneratorARM64::VisitUnsafeGetLong(HInvoke* invoke) {
814  GenUnsafeGet(invoke, Primitive::kPrimLong, /* is_volatile */ false, codegen_);
815}
816void IntrinsicCodeGeneratorARM64::VisitUnsafeGetLongVolatile(HInvoke* invoke) {
817  GenUnsafeGet(invoke, Primitive::kPrimLong, /* is_volatile */ true, codegen_);
818}
819void IntrinsicCodeGeneratorARM64::VisitUnsafeGetObject(HInvoke* invoke) {
820  GenUnsafeGet(invoke, Primitive::kPrimNot, /* is_volatile */ false, codegen_);
821}
822void IntrinsicCodeGeneratorARM64::VisitUnsafeGetObjectVolatile(HInvoke* invoke) {
823  GenUnsafeGet(invoke, Primitive::kPrimNot, /* is_volatile */ true, codegen_);
824}
825
826static void CreateIntIntIntIntToVoid(ArenaAllocator* arena, HInvoke* invoke) {
827  LocationSummary* locations = new (arena) LocationSummary(invoke,
828                                                           LocationSummary::kNoCall,
829                                                           kIntrinsified);
830  locations->SetInAt(0, Location::NoLocation());        // Unused receiver.
831  locations->SetInAt(1, Location::RequiresRegister());
832  locations->SetInAt(2, Location::RequiresRegister());
833  locations->SetInAt(3, Location::RequiresRegister());
834}
835
836void IntrinsicLocationsBuilderARM64::VisitUnsafePut(HInvoke* invoke) {
837  CreateIntIntIntIntToVoid(arena_, invoke);
838}
839void IntrinsicLocationsBuilderARM64::VisitUnsafePutOrdered(HInvoke* invoke) {
840  CreateIntIntIntIntToVoid(arena_, invoke);
841}
842void IntrinsicLocationsBuilderARM64::VisitUnsafePutVolatile(HInvoke* invoke) {
843  CreateIntIntIntIntToVoid(arena_, invoke);
844}
845void IntrinsicLocationsBuilderARM64::VisitUnsafePutObject(HInvoke* invoke) {
846  CreateIntIntIntIntToVoid(arena_, invoke);
847}
848void IntrinsicLocationsBuilderARM64::VisitUnsafePutObjectOrdered(HInvoke* invoke) {
849  CreateIntIntIntIntToVoid(arena_, invoke);
850}
851void IntrinsicLocationsBuilderARM64::VisitUnsafePutObjectVolatile(HInvoke* invoke) {
852  CreateIntIntIntIntToVoid(arena_, invoke);
853}
854void IntrinsicLocationsBuilderARM64::VisitUnsafePutLong(HInvoke* invoke) {
855  CreateIntIntIntIntToVoid(arena_, invoke);
856}
857void IntrinsicLocationsBuilderARM64::VisitUnsafePutLongOrdered(HInvoke* invoke) {
858  CreateIntIntIntIntToVoid(arena_, invoke);
859}
860void IntrinsicLocationsBuilderARM64::VisitUnsafePutLongVolatile(HInvoke* invoke) {
861  CreateIntIntIntIntToVoid(arena_, invoke);
862}
863
864static void GenUnsafePut(LocationSummary* locations,
865                         Primitive::Type type,
866                         bool is_volatile,
867                         bool is_ordered,
868                         CodeGeneratorARM64* codegen) {
869  vixl::MacroAssembler* masm = codegen->GetAssembler()->vixl_masm_;
870
871  Register base = WRegisterFrom(locations->InAt(1));    // Object pointer.
872  Register offset = XRegisterFrom(locations->InAt(2));  // Long offset.
873  Register value = RegisterFrom(locations->InAt(3), type);
874  Register source = value;
875  bool use_acquire_release = codegen->GetInstructionSetFeatures().PreferAcquireRelease();
876
877  MemOperand mem_op(base.X(), offset);
878
879  {
880    // We use a block to end the scratch scope before the write barrier, thus
881    // freeing the temporary registers so they can be used in `MarkGCCard`.
882    UseScratchRegisterScope temps(masm);
883
884    if (kPoisonHeapReferences && type == Primitive::kPrimNot) {
885      DCHECK(value.IsW());
886      Register temp = temps.AcquireW();
887      __ Mov(temp.W(), value.W());
888      codegen->GetAssembler()->PoisonHeapReference(temp.W());
889      source = temp;
890    }
891
892    if (is_volatile || is_ordered) {
893      if (use_acquire_release) {
894        codegen->StoreRelease(type, source, mem_op);
895      } else {
896        __ Dmb(InnerShareable, BarrierAll);
897        codegen->Store(type, source, mem_op);
898        if (is_volatile) {
899          __ Dmb(InnerShareable, BarrierReads);
900        }
901      }
902    } else {
903      codegen->Store(type, source, mem_op);
904    }
905  }
906
907  if (type == Primitive::kPrimNot) {
908    bool value_can_be_null = true;  // TODO: Worth finding out this information?
909    codegen->MarkGCCard(base, value, value_can_be_null);
910  }
911}
912
913void IntrinsicCodeGeneratorARM64::VisitUnsafePut(HInvoke* invoke) {
914  GenUnsafePut(invoke->GetLocations(),
915               Primitive::kPrimInt,
916               /* is_volatile */ false,
917               /* is_ordered */ false,
918               codegen_);
919}
920void IntrinsicCodeGeneratorARM64::VisitUnsafePutOrdered(HInvoke* invoke) {
921  GenUnsafePut(invoke->GetLocations(),
922               Primitive::kPrimInt,
923               /* is_volatile */ false,
924               /* is_ordered */ true,
925               codegen_);
926}
927void IntrinsicCodeGeneratorARM64::VisitUnsafePutVolatile(HInvoke* invoke) {
928  GenUnsafePut(invoke->GetLocations(),
929               Primitive::kPrimInt,
930               /* is_volatile */ true,
931               /* is_ordered */ false,
932               codegen_);
933}
934void IntrinsicCodeGeneratorARM64::VisitUnsafePutObject(HInvoke* invoke) {
935  GenUnsafePut(invoke->GetLocations(),
936               Primitive::kPrimNot,
937               /* is_volatile */ false,
938               /* is_ordered */ false,
939               codegen_);
940}
941void IntrinsicCodeGeneratorARM64::VisitUnsafePutObjectOrdered(HInvoke* invoke) {
942  GenUnsafePut(invoke->GetLocations(),
943               Primitive::kPrimNot,
944               /* is_volatile */ false,
945               /* is_ordered */ true,
946               codegen_);
947}
948void IntrinsicCodeGeneratorARM64::VisitUnsafePutObjectVolatile(HInvoke* invoke) {
949  GenUnsafePut(invoke->GetLocations(),
950               Primitive::kPrimNot,
951               /* is_volatile */ true,
952               /* is_ordered */ false,
953               codegen_);
954}
955void IntrinsicCodeGeneratorARM64::VisitUnsafePutLong(HInvoke* invoke) {
956  GenUnsafePut(invoke->GetLocations(),
957               Primitive::kPrimLong,
958               /* is_volatile */ false,
959               /* is_ordered */ false,
960               codegen_);
961}
962void IntrinsicCodeGeneratorARM64::VisitUnsafePutLongOrdered(HInvoke* invoke) {
963  GenUnsafePut(invoke->GetLocations(),
964               Primitive::kPrimLong,
965               /* is_volatile */ false,
966               /* is_ordered */ true,
967               codegen_);
968}
969void IntrinsicCodeGeneratorARM64::VisitUnsafePutLongVolatile(HInvoke* invoke) {
970  GenUnsafePut(invoke->GetLocations(),
971               Primitive::kPrimLong,
972               /* is_volatile */ true,
973               /* is_ordered */ false,
974               codegen_);
975}
976
977static void CreateIntIntIntIntIntToInt(ArenaAllocator* arena, HInvoke* invoke) {
978  LocationSummary* locations = new (arena) LocationSummary(invoke,
979                                                           LocationSummary::kNoCall,
980                                                           kIntrinsified);
981  locations->SetInAt(0, Location::NoLocation());        // Unused receiver.
982  locations->SetInAt(1, Location::RequiresRegister());
983  locations->SetInAt(2, Location::RequiresRegister());
984  locations->SetInAt(3, Location::RequiresRegister());
985  locations->SetInAt(4, Location::RequiresRegister());
986
987  locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
988}
989
990static void GenCas(LocationSummary* locations, Primitive::Type type, CodeGeneratorARM64* codegen) {
991  bool use_acquire_release = codegen->GetInstructionSetFeatures().PreferAcquireRelease();
992  vixl::MacroAssembler* masm = codegen->GetAssembler()->vixl_masm_;
993
994  Register out = WRegisterFrom(locations->Out());                  // Boolean result.
995
996  Register base = WRegisterFrom(locations->InAt(1));               // Object pointer.
997  Register offset = XRegisterFrom(locations->InAt(2));             // Long offset.
998  Register expected = RegisterFrom(locations->InAt(3), type);      // Expected.
999  Register value = RegisterFrom(locations->InAt(4), type);         // Value.
1000
1001  // This needs to be before the temp registers, as MarkGCCard also uses VIXL temps.
1002  if (type == Primitive::kPrimNot) {
1003    // Mark card for object assuming new value is stored.
1004    bool value_can_be_null = true;  // TODO: Worth finding out this information?
1005    codegen->MarkGCCard(base, value, value_can_be_null);
1006  }
1007
1008  UseScratchRegisterScope temps(masm);
1009  Register tmp_ptr = temps.AcquireX();                             // Pointer to actual memory.
1010  Register tmp_value = temps.AcquireSameSizeAs(value);             // Value in memory.
1011
1012  Register tmp_32 = tmp_value.W();
1013
1014  __ Add(tmp_ptr, base.X(), Operand(offset));
1015
1016  if (kPoisonHeapReferences && type == Primitive::kPrimNot) {
1017    codegen->GetAssembler()->PoisonHeapReference(expected);
1018    codegen->GetAssembler()->PoisonHeapReference(value);
1019  }
1020
1021  // do {
1022  //   tmp_value = [tmp_ptr] - expected;
1023  // } while (tmp_value == 0 && failure([tmp_ptr] <- r_new_value));
1024  // result = tmp_value != 0;
1025
1026  vixl::Label loop_head, exit_loop;
1027  if (use_acquire_release) {
1028    __ Bind(&loop_head);
1029    __ Ldaxr(tmp_value, MemOperand(tmp_ptr));
1030    // TODO: Do we need a read barrier here when `type == Primitive::kPrimNot`?
1031    // Note that this code is not (yet) used when read barriers are
1032    // enabled (see IntrinsicLocationsBuilderARM64::VisitUnsafeCASObject).
1033    __ Cmp(tmp_value, expected);
1034    __ B(&exit_loop, ne);
1035    __ Stlxr(tmp_32, value, MemOperand(tmp_ptr));
1036    __ Cbnz(tmp_32, &loop_head);
1037  } else {
1038    // Emit a `Dmb(InnerShareable, BarrierAll)` (DMB ISH) instruction
1039    // instead of a `Dmb(InnerShareable, BarrierWrites)` (DMB ISHST)
1040    // one, as the latter allows a preceding load to be delayed past
1041    // the STXR instruction below.
1042    __ Dmb(InnerShareable, BarrierAll);
1043    __ Bind(&loop_head);
1044    // TODO: When `type == Primitive::kPrimNot`, add a read barrier for
1045    // the reference stored in the object before attempting the CAS,
1046    // similar to the one in the art::Unsafe_compareAndSwapObject JNI
1047    // implementation.
1048    //
1049    // Note that this code is not (yet) used when read barriers are
1050    // enabled (see IntrinsicLocationsBuilderARM64::VisitUnsafeCASObject).
1051    DCHECK(!(type == Primitive::kPrimNot && kEmitCompilerReadBarrier));
1052    __ Ldxr(tmp_value, MemOperand(tmp_ptr));
1053    __ Cmp(tmp_value, expected);
1054    __ B(&exit_loop, ne);
1055    __ Stxr(tmp_32, value, MemOperand(tmp_ptr));
1056    __ Cbnz(tmp_32, &loop_head);
1057    __ Dmb(InnerShareable, BarrierAll);
1058  }
1059  __ Bind(&exit_loop);
1060  __ Cset(out, eq);
1061
1062  if (kPoisonHeapReferences && type == Primitive::kPrimNot) {
1063    codegen->GetAssembler()->UnpoisonHeapReference(value);
1064    codegen->GetAssembler()->UnpoisonHeapReference(expected);
1065  }
1066}
1067
1068void IntrinsicLocationsBuilderARM64::VisitUnsafeCASInt(HInvoke* invoke) {
1069  CreateIntIntIntIntIntToInt(arena_, invoke);
1070}
1071void IntrinsicLocationsBuilderARM64::VisitUnsafeCASLong(HInvoke* invoke) {
1072  CreateIntIntIntIntIntToInt(arena_, invoke);
1073}
1074void IntrinsicLocationsBuilderARM64::VisitUnsafeCASObject(HInvoke* invoke) {
1075  // The UnsafeCASObject intrinsic is missing a read barrier, and
1076  // therefore sometimes does not work as expected (b/25883050).
1077  // Turn it off temporarily as a quick fix, until the read barrier is
1078  // implemented (see TODO in GenCAS below).
1079  //
1080  // Also, the UnsafeCASObject intrinsic does not always work when heap
1081  // poisoning is enabled (it breaks run-test 004-UnsafeTest); turn it
1082  // off temporarily as a quick fix (b/26204023).
1083  //
1084  // TODO(rpl): Fix these two issues and re-enable this intrinsic.
1085  if (kEmitCompilerReadBarrier || kPoisonHeapReferences) {
1086    return;
1087  }
1088
1089  CreateIntIntIntIntIntToInt(arena_, invoke);
1090}
1091
1092void IntrinsicCodeGeneratorARM64::VisitUnsafeCASInt(HInvoke* invoke) {
1093  GenCas(invoke->GetLocations(), Primitive::kPrimInt, codegen_);
1094}
1095void IntrinsicCodeGeneratorARM64::VisitUnsafeCASLong(HInvoke* invoke) {
1096  GenCas(invoke->GetLocations(), Primitive::kPrimLong, codegen_);
1097}
1098void IntrinsicCodeGeneratorARM64::VisitUnsafeCASObject(HInvoke* invoke) {
1099  GenCas(invoke->GetLocations(), Primitive::kPrimNot, codegen_);
1100}
1101
1102void IntrinsicLocationsBuilderARM64::VisitStringCharAt(HInvoke* invoke) {
1103  LocationSummary* locations = new (arena_) LocationSummary(invoke,
1104                                                            LocationSummary::kCallOnSlowPath,
1105                                                            kIntrinsified);
1106  locations->SetInAt(0, Location::RequiresRegister());
1107  locations->SetInAt(1, Location::RequiresRegister());
1108  // In case we need to go in the slow path, we can't have the output be the same
1109  // as the input: the current liveness analysis considers the input to be live
1110  // at the point of the call.
1111  locations->SetOut(Location::RequiresRegister(), Location::kOutputOverlap);
1112}
1113
1114void IntrinsicCodeGeneratorARM64::VisitStringCharAt(HInvoke* invoke) {
1115  vixl::MacroAssembler* masm = GetVIXLAssembler();
1116  LocationSummary* locations = invoke->GetLocations();
1117
1118  // Location of reference to data array
1119  const MemberOffset value_offset = mirror::String::ValueOffset();
1120  // Location of count
1121  const MemberOffset count_offset = mirror::String::CountOffset();
1122
1123  Register obj = WRegisterFrom(locations->InAt(0));  // String object pointer.
1124  Register idx = WRegisterFrom(locations->InAt(1));  // Index of character.
1125  Register out = WRegisterFrom(locations->Out());    // Result character.
1126
1127  UseScratchRegisterScope temps(masm);
1128  Register temp = temps.AcquireW();
1129  Register array_temp = temps.AcquireW();            // We can trade this for worse scheduling.
1130
1131  // TODO: Maybe we can support range check elimination. Overall, though, I think it's not worth
1132  //       the cost.
1133  // TODO: For simplicity, the index parameter is requested in a register, so different from Quick
1134  //       we will not optimize the code for constants (which would save a register).
1135
1136  SlowPathCodeARM64* slow_path = new (GetAllocator()) IntrinsicSlowPathARM64(invoke);
1137  codegen_->AddSlowPath(slow_path);
1138
1139  __ Ldr(temp, HeapOperand(obj, count_offset));          // temp = str.length.
1140  codegen_->MaybeRecordImplicitNullCheck(invoke);
1141  __ Cmp(idx, temp);
1142  __ B(hs, slow_path->GetEntryLabel());
1143
1144  __ Add(array_temp, obj, Operand(value_offset.Int32Value()));  // array_temp := str.value.
1145
1146  // Load the value.
1147  __ Ldrh(out, MemOperand(array_temp.X(), idx, UXTW, 1));  // out := array_temp[idx].
1148
1149  __ Bind(slow_path->GetExitLabel());
1150}
1151
1152void IntrinsicLocationsBuilderARM64::VisitStringCompareTo(HInvoke* invoke) {
1153  LocationSummary* locations = new (arena_) LocationSummary(invoke,
1154                                                            LocationSummary::kCall,
1155                                                            kIntrinsified);
1156  InvokeRuntimeCallingConvention calling_convention;
1157  locations->SetInAt(0, LocationFrom(calling_convention.GetRegisterAt(0)));
1158  locations->SetInAt(1, LocationFrom(calling_convention.GetRegisterAt(1)));
1159  locations->SetOut(calling_convention.GetReturnLocation(Primitive::kPrimInt));
1160}
1161
1162void IntrinsicCodeGeneratorARM64::VisitStringCompareTo(HInvoke* invoke) {
1163  vixl::MacroAssembler* masm = GetVIXLAssembler();
1164  LocationSummary* locations = invoke->GetLocations();
1165
1166  // Note that the null check must have been done earlier.
1167  DCHECK(!invoke->CanDoImplicitNullCheckOn(invoke->InputAt(0)));
1168
1169  Register argument = WRegisterFrom(locations->InAt(1));
1170  __ Cmp(argument, 0);
1171  SlowPathCodeARM64* slow_path = new (GetAllocator()) IntrinsicSlowPathARM64(invoke);
1172  codegen_->AddSlowPath(slow_path);
1173  __ B(eq, slow_path->GetEntryLabel());
1174
1175  __ Ldr(
1176      lr, MemOperand(tr, QUICK_ENTRYPOINT_OFFSET(kArm64WordSize, pStringCompareTo).Int32Value()));
1177  __ Blr(lr);
1178  __ Bind(slow_path->GetExitLabel());
1179}
1180
1181void IntrinsicLocationsBuilderARM64::VisitStringEquals(HInvoke* invoke) {
1182  LocationSummary* locations = new (arena_) LocationSummary(invoke,
1183                                                            LocationSummary::kNoCall,
1184                                                            kIntrinsified);
1185  locations->SetInAt(0, Location::RequiresRegister());
1186  locations->SetInAt(1, Location::RequiresRegister());
1187  // Temporary registers to store lengths of strings and for calculations.
1188  locations->AddTemp(Location::RequiresRegister());
1189  locations->AddTemp(Location::RequiresRegister());
1190
1191  locations->SetOut(Location::RequiresRegister(), Location::kOutputOverlap);
1192}
1193
1194void IntrinsicCodeGeneratorARM64::VisitStringEquals(HInvoke* invoke) {
1195  vixl::MacroAssembler* masm = GetVIXLAssembler();
1196  LocationSummary* locations = invoke->GetLocations();
1197
1198  Register str = WRegisterFrom(locations->InAt(0));
1199  Register arg = WRegisterFrom(locations->InAt(1));
1200  Register out = XRegisterFrom(locations->Out());
1201
1202  UseScratchRegisterScope scratch_scope(masm);
1203  Register temp = scratch_scope.AcquireW();
1204  Register temp1 = WRegisterFrom(locations->GetTemp(0));
1205  Register temp2 = WRegisterFrom(locations->GetTemp(1));
1206
1207  vixl::Label loop;
1208  vixl::Label end;
1209  vixl::Label return_true;
1210  vixl::Label return_false;
1211
1212  // Get offsets of count, value, and class fields within a string object.
1213  const int32_t count_offset = mirror::String::CountOffset().Int32Value();
1214  const int32_t value_offset = mirror::String::ValueOffset().Int32Value();
1215  const int32_t class_offset = mirror::Object::ClassOffset().Int32Value();
1216
1217  // Note that the null check must have been done earlier.
1218  DCHECK(!invoke->CanDoImplicitNullCheckOn(invoke->InputAt(0)));
1219
1220  // Check if input is null, return false if it is.
1221  __ Cbz(arg, &return_false);
1222
1223  // Reference equality check, return true if same reference.
1224  __ Cmp(str, arg);
1225  __ B(&return_true, eq);
1226
1227  // Instanceof check for the argument by comparing class fields.
1228  // All string objects must have the same type since String cannot be subclassed.
1229  // Receiver must be a string object, so its class field is equal to all strings' class fields.
1230  // If the argument is a string object, its class field must be equal to receiver's class field.
1231  __ Ldr(temp, MemOperand(str.X(), class_offset));
1232  __ Ldr(temp1, MemOperand(arg.X(), class_offset));
1233  __ Cmp(temp, temp1);
1234  __ B(&return_false, ne);
1235
1236  // Load lengths of this and argument strings.
1237  __ Ldr(temp, MemOperand(str.X(), count_offset));
1238  __ Ldr(temp1, MemOperand(arg.X(), count_offset));
1239  // Check if lengths are equal, return false if they're not.
1240  __ Cmp(temp, temp1);
1241  __ B(&return_false, ne);
1242  // Store offset of string value in preparation for comparison loop
1243  __ Mov(temp1, value_offset);
1244  // Return true if both strings are empty.
1245  __ Cbz(temp, &return_true);
1246
1247  // Assertions that must hold in order to compare strings 4 characters at a time.
1248  DCHECK_ALIGNED(value_offset, 8);
1249  static_assert(IsAligned<8>(kObjectAlignment), "String of odd length is not zero padded");
1250
1251  temp1 = temp1.X();
1252  temp2 = temp2.X();
1253
1254  // Loop to compare strings 4 characters at a time starting at the beginning of the string.
1255  // Ok to do this because strings are zero-padded to be 8-byte aligned.
1256  __ Bind(&loop);
1257  __ Ldr(out, MemOperand(str.X(), temp1));
1258  __ Ldr(temp2, MemOperand(arg.X(), temp1));
1259  __ Add(temp1, temp1, Operand(sizeof(uint64_t)));
1260  __ Cmp(out, temp2);
1261  __ B(&return_false, ne);
1262  __ Sub(temp, temp, Operand(4), SetFlags);
1263  __ B(&loop, gt);
1264
1265  // Return true and exit the function.
1266  // If loop does not result in returning false, we return true.
1267  __ Bind(&return_true);
1268  __ Mov(out, 1);
1269  __ B(&end);
1270
1271  // Return false and exit the function.
1272  __ Bind(&return_false);
1273  __ Mov(out, 0);
1274  __ Bind(&end);
1275}
1276
1277static void GenerateVisitStringIndexOf(HInvoke* invoke,
1278                                       vixl::MacroAssembler* masm,
1279                                       CodeGeneratorARM64* codegen,
1280                                       ArenaAllocator* allocator,
1281                                       bool start_at_zero) {
1282  LocationSummary* locations = invoke->GetLocations();
1283  Register tmp_reg = WRegisterFrom(locations->GetTemp(0));
1284
1285  // Note that the null check must have been done earlier.
1286  DCHECK(!invoke->CanDoImplicitNullCheckOn(invoke->InputAt(0)));
1287
1288  // Check for code points > 0xFFFF. Either a slow-path check when we don't know statically,
1289  // or directly dispatch if we have a constant.
1290  SlowPathCodeARM64* slow_path = nullptr;
1291  if (invoke->InputAt(1)->IsIntConstant()) {
1292    if (static_cast<uint32_t>(invoke->InputAt(1)->AsIntConstant()->GetValue()) > 0xFFFFU) {
1293      // Always needs the slow-path. We could directly dispatch to it, but this case should be
1294      // rare, so for simplicity just put the full slow-path down and branch unconditionally.
1295      slow_path = new (allocator) IntrinsicSlowPathARM64(invoke);
1296      codegen->AddSlowPath(slow_path);
1297      __ B(slow_path->GetEntryLabel());
1298      __ Bind(slow_path->GetExitLabel());
1299      return;
1300    }
1301  } else {
1302    Register char_reg = WRegisterFrom(locations->InAt(1));
1303    __ Mov(tmp_reg, 0xFFFF);
1304    __ Cmp(char_reg, Operand(tmp_reg));
1305    slow_path = new (allocator) IntrinsicSlowPathARM64(invoke);
1306    codegen->AddSlowPath(slow_path);
1307    __ B(hi, slow_path->GetEntryLabel());
1308  }
1309
1310  if (start_at_zero) {
1311    // Start-index = 0.
1312    __ Mov(tmp_reg, 0);
1313  }
1314
1315  __ Ldr(lr, MemOperand(tr, QUICK_ENTRYPOINT_OFFSET(kArm64WordSize, pIndexOf).Int32Value()));
1316  __ Blr(lr);
1317
1318  if (slow_path != nullptr) {
1319    __ Bind(slow_path->GetExitLabel());
1320  }
1321}
1322
1323void IntrinsicLocationsBuilderARM64::VisitStringIndexOf(HInvoke* invoke) {
1324  LocationSummary* locations = new (arena_) LocationSummary(invoke,
1325                                                            LocationSummary::kCall,
1326                                                            kIntrinsified);
1327  // We have a hand-crafted assembly stub that follows the runtime calling convention. So it's
1328  // best to align the inputs accordingly.
1329  InvokeRuntimeCallingConvention calling_convention;
1330  locations->SetInAt(0, LocationFrom(calling_convention.GetRegisterAt(0)));
1331  locations->SetInAt(1, LocationFrom(calling_convention.GetRegisterAt(1)));
1332  locations->SetOut(calling_convention.GetReturnLocation(Primitive::kPrimInt));
1333
1334  // Need a temp for slow-path codepoint compare, and need to send start_index=0.
1335  locations->AddTemp(LocationFrom(calling_convention.GetRegisterAt(2)));
1336}
1337
1338void IntrinsicCodeGeneratorARM64::VisitStringIndexOf(HInvoke* invoke) {
1339  GenerateVisitStringIndexOf(
1340      invoke, GetVIXLAssembler(), codegen_, GetAllocator(), /* start_at_zero */ true);
1341}
1342
1343void IntrinsicLocationsBuilderARM64::VisitStringIndexOfAfter(HInvoke* invoke) {
1344  LocationSummary* locations = new (arena_) LocationSummary(invoke,
1345                                                            LocationSummary::kCall,
1346                                                            kIntrinsified);
1347  // We have a hand-crafted assembly stub that follows the runtime calling convention. So it's
1348  // best to align the inputs accordingly.
1349  InvokeRuntimeCallingConvention calling_convention;
1350  locations->SetInAt(0, LocationFrom(calling_convention.GetRegisterAt(0)));
1351  locations->SetInAt(1, LocationFrom(calling_convention.GetRegisterAt(1)));
1352  locations->SetInAt(2, LocationFrom(calling_convention.GetRegisterAt(2)));
1353  locations->SetOut(calling_convention.GetReturnLocation(Primitive::kPrimInt));
1354
1355  // Need a temp for slow-path codepoint compare.
1356  locations->AddTemp(Location::RequiresRegister());
1357}
1358
1359void IntrinsicCodeGeneratorARM64::VisitStringIndexOfAfter(HInvoke* invoke) {
1360  GenerateVisitStringIndexOf(
1361      invoke, GetVIXLAssembler(), codegen_, GetAllocator(), /* start_at_zero */ false);
1362}
1363
1364void IntrinsicLocationsBuilderARM64::VisitStringNewStringFromBytes(HInvoke* invoke) {
1365  LocationSummary* locations = new (arena_) LocationSummary(invoke,
1366                                                            LocationSummary::kCall,
1367                                                            kIntrinsified);
1368  InvokeRuntimeCallingConvention calling_convention;
1369  locations->SetInAt(0, LocationFrom(calling_convention.GetRegisterAt(0)));
1370  locations->SetInAt(1, LocationFrom(calling_convention.GetRegisterAt(1)));
1371  locations->SetInAt(2, LocationFrom(calling_convention.GetRegisterAt(2)));
1372  locations->SetInAt(3, LocationFrom(calling_convention.GetRegisterAt(3)));
1373  locations->SetOut(calling_convention.GetReturnLocation(Primitive::kPrimNot));
1374}
1375
1376void IntrinsicCodeGeneratorARM64::VisitStringNewStringFromBytes(HInvoke* invoke) {
1377  vixl::MacroAssembler* masm = GetVIXLAssembler();
1378  LocationSummary* locations = invoke->GetLocations();
1379
1380  Register byte_array = WRegisterFrom(locations->InAt(0));
1381  __ Cmp(byte_array, 0);
1382  SlowPathCodeARM64* slow_path = new (GetAllocator()) IntrinsicSlowPathARM64(invoke);
1383  codegen_->AddSlowPath(slow_path);
1384  __ B(eq, slow_path->GetEntryLabel());
1385
1386  __ Ldr(lr,
1387      MemOperand(tr, QUICK_ENTRYPOINT_OFFSET(kArm64WordSize, pAllocStringFromBytes).Int32Value()));
1388  codegen_->RecordPcInfo(invoke, invoke->GetDexPc());
1389  __ Blr(lr);
1390  __ Bind(slow_path->GetExitLabel());
1391}
1392
1393void IntrinsicLocationsBuilderARM64::VisitStringNewStringFromChars(HInvoke* invoke) {
1394  LocationSummary* locations = new (arena_) LocationSummary(invoke,
1395                                                            LocationSummary::kCall,
1396                                                            kIntrinsified);
1397  InvokeRuntimeCallingConvention calling_convention;
1398  locations->SetInAt(0, LocationFrom(calling_convention.GetRegisterAt(0)));
1399  locations->SetInAt(1, LocationFrom(calling_convention.GetRegisterAt(1)));
1400  locations->SetInAt(2, LocationFrom(calling_convention.GetRegisterAt(2)));
1401  locations->SetOut(calling_convention.GetReturnLocation(Primitive::kPrimNot));
1402}
1403
1404void IntrinsicCodeGeneratorARM64::VisitStringNewStringFromChars(HInvoke* invoke) {
1405  vixl::MacroAssembler* masm = GetVIXLAssembler();
1406
1407  __ Ldr(lr,
1408      MemOperand(tr, QUICK_ENTRYPOINT_OFFSET(kArm64WordSize, pAllocStringFromChars).Int32Value()));
1409  codegen_->RecordPcInfo(invoke, invoke->GetDexPc());
1410  __ Blr(lr);
1411}
1412
1413void IntrinsicLocationsBuilderARM64::VisitStringNewStringFromString(HInvoke* invoke) {
1414  // The inputs plus one temp.
1415  LocationSummary* locations = new (arena_) LocationSummary(invoke,
1416                                                            LocationSummary::kCall,
1417                                                            kIntrinsified);
1418  InvokeRuntimeCallingConvention calling_convention;
1419  locations->SetInAt(0, LocationFrom(calling_convention.GetRegisterAt(0)));
1420  locations->SetInAt(1, LocationFrom(calling_convention.GetRegisterAt(1)));
1421  locations->SetInAt(2, LocationFrom(calling_convention.GetRegisterAt(2)));
1422  locations->SetOut(calling_convention.GetReturnLocation(Primitive::kPrimNot));
1423}
1424
1425void IntrinsicCodeGeneratorARM64::VisitStringNewStringFromString(HInvoke* invoke) {
1426  vixl::MacroAssembler* masm = GetVIXLAssembler();
1427  LocationSummary* locations = invoke->GetLocations();
1428
1429  Register string_to_copy = WRegisterFrom(locations->InAt(0));
1430  __ Cmp(string_to_copy, 0);
1431  SlowPathCodeARM64* slow_path = new (GetAllocator()) IntrinsicSlowPathARM64(invoke);
1432  codegen_->AddSlowPath(slow_path);
1433  __ B(eq, slow_path->GetEntryLabel());
1434
1435  __ Ldr(lr,
1436      MemOperand(tr, QUICK_ENTRYPOINT_OFFSET(kArm64WordSize, pAllocStringFromString).Int32Value()));
1437  codegen_->RecordPcInfo(invoke, invoke->GetDexPc());
1438  __ Blr(lr);
1439  __ Bind(slow_path->GetExitLabel());
1440}
1441
1442// Unimplemented intrinsics.
1443
1444#define UNIMPLEMENTED_INTRINSIC(Name)                                                  \
1445void IntrinsicLocationsBuilderARM64::Visit ## Name(HInvoke* invoke ATTRIBUTE_UNUSED) { \
1446}                                                                                      \
1447void IntrinsicCodeGeneratorARM64::Visit ## Name(HInvoke* invoke ATTRIBUTE_UNUSED) {    \
1448}
1449
1450UNIMPLEMENTED_INTRINSIC(IntegerRotateLeft)
1451UNIMPLEMENTED_INTRINSIC(IntegerRotateRight)
1452UNIMPLEMENTED_INTRINSIC(LongRotateLeft)
1453UNIMPLEMENTED_INTRINSIC(LongRotateRight)
1454UNIMPLEMENTED_INTRINSIC(SystemArrayCopyChar)
1455UNIMPLEMENTED_INTRINSIC(SystemArrayCopy)
1456UNIMPLEMENTED_INTRINSIC(ReferenceGetReferent)
1457UNIMPLEMENTED_INTRINSIC(StringGetCharsNoCheck)
1458
1459UNIMPLEMENTED_INTRINSIC(MathCos)
1460UNIMPLEMENTED_INTRINSIC(MathSin)
1461UNIMPLEMENTED_INTRINSIC(MathAcos)
1462UNIMPLEMENTED_INTRINSIC(MathAsin)
1463UNIMPLEMENTED_INTRINSIC(MathAtan)
1464UNIMPLEMENTED_INTRINSIC(MathAtan2)
1465UNIMPLEMENTED_INTRINSIC(MathCbrt)
1466UNIMPLEMENTED_INTRINSIC(MathCosh)
1467UNIMPLEMENTED_INTRINSIC(MathExp)
1468UNIMPLEMENTED_INTRINSIC(MathExpm1)
1469UNIMPLEMENTED_INTRINSIC(MathHypot)
1470UNIMPLEMENTED_INTRINSIC(MathLog)
1471UNIMPLEMENTED_INTRINSIC(MathLog10)
1472UNIMPLEMENTED_INTRINSIC(MathNextAfter)
1473UNIMPLEMENTED_INTRINSIC(MathSinh)
1474UNIMPLEMENTED_INTRINSIC(MathTan)
1475UNIMPLEMENTED_INTRINSIC(MathTanh)
1476
1477#undef UNIMPLEMENTED_INTRINSIC
1478
1479#undef __
1480
1481}  // namespace arm64
1482}  // namespace art
1483