intrinsics_arm64.cc revision 2f9fcc999fab4ba6cd86c30e664325b47b9618e5 
1/*
2 * Copyright (C) 2015 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *      http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17#include "intrinsics_arm64.h"
18
19#include "arch/arm64/instruction_set_features_arm64.h"
20#include "art_method.h"
21#include "code_generator_arm64.h"
22#include "common_arm64.h"
23#include "entrypoints/quick/quick_entrypoints.h"
24#include "intrinsics.h"
25#include "mirror/array-inl.h"
26#include "mirror/string.h"
27#include "thread.h"
28#include "utils/arm64/assembler_arm64.h"
29#include "utils/arm64/constants_arm64.h"
30
31#include "vixl/a64/disasm-a64.h"
32#include "vixl/a64/macro-assembler-a64.h"
33
34using namespace vixl;   // NOLINT(build/namespaces)
35
36namespace art {
37
38namespace arm64 {
39
40using helpers::DRegisterFrom;
41using helpers::FPRegisterFrom;
42using helpers::HeapOperand;
43using helpers::LocationFrom;
44using helpers::OperandFrom;
45using helpers::RegisterFrom;
46using helpers::SRegisterFrom;
47using helpers::WRegisterFrom;
48using helpers::XRegisterFrom;
49
50namespace {
51
52ALWAYS_INLINE inline MemOperand AbsoluteHeapOperandFrom(Location location, size_t offset = 0) {
53  return MemOperand(XRegisterFrom(location), offset);
54}
55
56}  // namespace
57
58vixl::MacroAssembler* IntrinsicCodeGeneratorARM64::GetVIXLAssembler() {
59  return codegen_->GetAssembler()->vixl_masm_;
60}
61
62ArenaAllocator* IntrinsicCodeGeneratorARM64::GetAllocator() {
63  return codegen_->GetGraph()->GetArena();
64}
65
66#define __ codegen->GetAssembler()->vixl_masm_->
67
68static void MoveFromReturnRegister(Location trg,
69                                   Primitive::Type type,
70                                   CodeGeneratorARM64* codegen) {
71  if (!trg.IsValid()) {
72    DCHECK(type == Primitive::kPrimVoid);
73    return;
74  }
75
76  DCHECK_NE(type, Primitive::kPrimVoid);
77
78  if (Primitive::IsIntegralType(type) || type == Primitive::kPrimNot) {
79    Register trg_reg = RegisterFrom(trg, type);
80    Register res_reg = RegisterFrom(ARM64ReturnLocation(type), type);
81    __ Mov(trg_reg, res_reg, kDiscardForSameWReg);
82  } else {
83    FPRegister trg_reg = FPRegisterFrom(trg, type);
84    FPRegister res_reg = FPRegisterFrom(ARM64ReturnLocation(type), type);
85    __ Fmov(trg_reg, res_reg);
86  }
87}
88
89static void MoveArguments(HInvoke* invoke, CodeGeneratorARM64* codegen) {
90  InvokeDexCallingConventionVisitorARM64 calling_convention_visitor;
91  IntrinsicVisitor::MoveArguments(invoke, codegen, &calling_convention_visitor);
92}
93
94// Slow-path for fallback (calling the managed code to handle the intrinsic) in an intrinsified
95// call. This will copy the arguments into the positions for a regular call.
96//
97// Note: The actual parameters are required to be in the locations given by the invoke's location
98//       summary. If an intrinsic modifies those locations before a slowpath call, they must be
99//       restored!
100class IntrinsicSlowPathARM64 : public SlowPathCodeARM64 {
101 public:
102  explicit IntrinsicSlowPathARM64(HInvoke* invoke)
103      : SlowPathCodeARM64(invoke), invoke_(invoke) { }
104
105  void EmitNativeCode(CodeGenerator* codegen_in) OVERRIDE {
106    CodeGeneratorARM64* codegen = down_cast<CodeGeneratorARM64*>(codegen_in);
107    __ Bind(GetEntryLabel());
108
109    SaveLiveRegisters(codegen, invoke_->GetLocations());
110
111    MoveArguments(invoke_, codegen);
112
113    if (invoke_->IsInvokeStaticOrDirect()) {
114      codegen->GenerateStaticOrDirectCall(invoke_->AsInvokeStaticOrDirect(),
115                                          LocationFrom(kArtMethodRegister));
116    } else {
117      codegen->GenerateVirtualCall(invoke_->AsInvokeVirtual(), LocationFrom(kArtMethodRegister));
118    }
119    codegen->RecordPcInfo(invoke_, invoke_->GetDexPc(), this);
120
121    // Copy the result back to the expected output.
122    Location out = invoke_->GetLocations()->Out();
123    if (out.IsValid()) {
124      DCHECK(out.IsRegister());  // TODO: Replace this when we support output in memory.
125      DCHECK(!invoke_->GetLocations()->GetLiveRegisters()->ContainsCoreRegister(out.reg()));
126      MoveFromReturnRegister(out, invoke_->GetType(), codegen);
127    }
128
129    RestoreLiveRegisters(codegen, invoke_->GetLocations());
130    __ B(GetExitLabel());
131  }
132
133  const char* GetDescription() const OVERRIDE { return "IntrinsicSlowPathARM64"; }
134
135 private:
136  // The instruction where this slow path is happening.
137  HInvoke* const invoke_;
138
139  DISALLOW_COPY_AND_ASSIGN(IntrinsicSlowPathARM64);
140};
141
142#undef __
143
144bool IntrinsicLocationsBuilderARM64::TryDispatch(HInvoke* invoke) {
145  Dispatch(invoke);
146  LocationSummary* res = invoke->GetLocations();
147  if (res == nullptr) {
148    return false;
149  }
150  if (kEmitCompilerReadBarrier && res->CanCall()) {
151    // Generating an intrinsic for this HInvoke may produce an
152    // IntrinsicSlowPathARM64 slow path.  Currently this approach
153    // does not work when using read barriers, as the emitted
154    // calling sequence will make use of another slow path
155    // (ReadBarrierForRootSlowPathARM64 for HInvokeStaticOrDirect,
156    // ReadBarrierSlowPathARM64 for HInvokeVirtual).  So we bail
157    // out in this case.
158    //
159    // TODO: Find a way to have intrinsics work with read barriers.
160    invoke->SetLocations(nullptr);
161    return false;
162  }
163  return res->Intrinsified();
164}
165
166#define __ masm->
167
168static void CreateFPToIntLocations(ArenaAllocator* arena, HInvoke* invoke) {
169  LocationSummary* locations = new (arena) LocationSummary(invoke,
170                                                           LocationSummary::kNoCall,
171                                                           kIntrinsified);
172  locations->SetInAt(0, Location::RequiresFpuRegister());
173  locations->SetOut(Location::RequiresRegister());
174}
175
176static void CreateIntToFPLocations(ArenaAllocator* arena, HInvoke* invoke) {
177  LocationSummary* locations = new (arena) LocationSummary(invoke,
178                                                           LocationSummary::kNoCall,
179                                                           kIntrinsified);
180  locations->SetInAt(0, Location::RequiresRegister());
181  locations->SetOut(Location::RequiresFpuRegister());
182}
183
184static void MoveFPToInt(LocationSummary* locations, bool is64bit, vixl::MacroAssembler* masm) {
185  Location input = locations->InAt(0);
186  Location output = locations->Out();
187  __ Fmov(is64bit ? XRegisterFrom(output) : WRegisterFrom(output),
188          is64bit ? DRegisterFrom(input) : SRegisterFrom(input));
189}
190
191static void MoveIntToFP(LocationSummary* locations, bool is64bit, vixl::MacroAssembler* masm) {
192  Location input = locations->InAt(0);
193  Location output = locations->Out();
194  __ Fmov(is64bit ? DRegisterFrom(output) : SRegisterFrom(output),
195          is64bit ? XRegisterFrom(input) : WRegisterFrom(input));
196}
197
198void IntrinsicLocationsBuilderARM64::VisitDoubleDoubleToRawLongBits(HInvoke* invoke) {
199  CreateFPToIntLocations(arena_, invoke);
200}
201void IntrinsicLocationsBuilderARM64::VisitDoubleLongBitsToDouble(HInvoke* invoke) {
202  CreateIntToFPLocations(arena_, invoke);
203}
204
205void IntrinsicCodeGeneratorARM64::VisitDoubleDoubleToRawLongBits(HInvoke* invoke) {
206  MoveFPToInt(invoke->GetLocations(), /* is64bit */ true, GetVIXLAssembler());
207}
208void IntrinsicCodeGeneratorARM64::VisitDoubleLongBitsToDouble(HInvoke* invoke) {
209  MoveIntToFP(invoke->GetLocations(), /* is64bit */ true, GetVIXLAssembler());
210}
211
212void IntrinsicLocationsBuilderARM64::VisitFloatFloatToRawIntBits(HInvoke* invoke) {
213  CreateFPToIntLocations(arena_, invoke);
214}
215void IntrinsicLocationsBuilderARM64::VisitFloatIntBitsToFloat(HInvoke* invoke) {
216  CreateIntToFPLocations(arena_, invoke);
217}
218
219void IntrinsicCodeGeneratorARM64::VisitFloatFloatToRawIntBits(HInvoke* invoke) {
220  MoveFPToInt(invoke->GetLocations(), /* is64bit */ false, GetVIXLAssembler());
221}
222void IntrinsicCodeGeneratorARM64::VisitFloatIntBitsToFloat(HInvoke* invoke) {
223  MoveIntToFP(invoke->GetLocations(), /* is64bit */ false, GetVIXLAssembler());
224}
225
226static void CreateIntToIntLocations(ArenaAllocator* arena, HInvoke* invoke) {
227  LocationSummary* locations = new (arena) LocationSummary(invoke,
228                                                           LocationSummary::kNoCall,
229                                                           kIntrinsified);
230  locations->SetInAt(0, Location::RequiresRegister());
231  locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
232}
233
234static void GenReverseBytes(LocationSummary* locations,
235                            Primitive::Type type,
236                            vixl::MacroAssembler* masm) {
237  Location in = locations->InAt(0);
238  Location out = locations->Out();
239
240  switch (type) {
241    case Primitive::kPrimShort:
242      __ Rev16(WRegisterFrom(out), WRegisterFrom(in));
243      __ Sxth(WRegisterFrom(out), WRegisterFrom(out));
244      break;
245    case Primitive::kPrimInt:
246    case Primitive::kPrimLong:
247      __ Rev(RegisterFrom(out, type), RegisterFrom(in, type));
248      break;
249    default:
250      LOG(FATAL) << "Unexpected size for reverse-bytes: " << type;
251      UNREACHABLE();
252  }
253}
254
255void IntrinsicLocationsBuilderARM64::VisitIntegerReverseBytes(HInvoke* invoke) {
256  CreateIntToIntLocations(arena_, invoke);
257}
258
259void IntrinsicCodeGeneratorARM64::VisitIntegerReverseBytes(HInvoke* invoke) {
260  GenReverseBytes(invoke->GetLocations(), Primitive::kPrimInt, GetVIXLAssembler());
261}
262
263void IntrinsicLocationsBuilderARM64::VisitLongReverseBytes(HInvoke* invoke) {
264  CreateIntToIntLocations(arena_, invoke);
265}
266
267void IntrinsicCodeGeneratorARM64::VisitLongReverseBytes(HInvoke* invoke) {
268  GenReverseBytes(invoke->GetLocations(), Primitive::kPrimLong, GetVIXLAssembler());
269}
270
271void IntrinsicLocationsBuilderARM64::VisitShortReverseBytes(HInvoke* invoke) {
272  CreateIntToIntLocations(arena_, invoke);
273}
274
275void IntrinsicCodeGeneratorARM64::VisitShortReverseBytes(HInvoke* invoke) {
276  GenReverseBytes(invoke->GetLocations(), Primitive::kPrimShort, GetVIXLAssembler());
277}
278
279static void CreateIntIntToIntLocations(ArenaAllocator* arena, HInvoke* invoke) {
280  LocationSummary* locations = new (arena) LocationSummary(invoke,
281                                                           LocationSummary::kNoCall,
282                                                           kIntrinsified);
283  locations->SetInAt(0, Location::RequiresRegister());
284  locations->SetInAt(1, Location::RequiresRegister());
285  locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
286}
287
288static void GenNumberOfLeadingZeros(LocationSummary* locations,
289                                    Primitive::Type type,
290                                    vixl::MacroAssembler* masm) {
291  DCHECK(type == Primitive::kPrimInt || type == Primitive::kPrimLong);
292
293  Location in = locations->InAt(0);
294  Location out = locations->Out();
295
296  __ Clz(RegisterFrom(out, type), RegisterFrom(in, type));
297}
298
299void IntrinsicLocationsBuilderARM64::VisitIntegerNumberOfLeadingZeros(HInvoke* invoke) {
300  CreateIntToIntLocations(arena_, invoke);
301}
302
303void IntrinsicCodeGeneratorARM64::VisitIntegerNumberOfLeadingZeros(HInvoke* invoke) {
304  GenNumberOfLeadingZeros(invoke->GetLocations(), Primitive::kPrimInt, GetVIXLAssembler());
305}
306
307void IntrinsicLocationsBuilderARM64::VisitLongNumberOfLeadingZeros(HInvoke* invoke) {
308  CreateIntToIntLocations(arena_, invoke);
309}
310
311void IntrinsicCodeGeneratorARM64::VisitLongNumberOfLeadingZeros(HInvoke* invoke) {
312  GenNumberOfLeadingZeros(invoke->GetLocations(), Primitive::kPrimLong, GetVIXLAssembler());
313}
314
315static void GenNumberOfTrailingZeros(LocationSummary* locations,
316                                     Primitive::Type type,
317                                     vixl::MacroAssembler* masm) {
318  DCHECK(type == Primitive::kPrimInt || type == Primitive::kPrimLong);
319
320  Location in = locations->InAt(0);
321  Location out = locations->Out();
322
323  __ Rbit(RegisterFrom(out, type), RegisterFrom(in, type));
324  __ Clz(RegisterFrom(out, type), RegisterFrom(out, type));
325}
326
327void IntrinsicLocationsBuilderARM64::VisitIntegerNumberOfTrailingZeros(HInvoke* invoke) {
328  CreateIntToIntLocations(arena_, invoke);
329}
330
331void IntrinsicCodeGeneratorARM64::VisitIntegerNumberOfTrailingZeros(HInvoke* invoke) {
332  GenNumberOfTrailingZeros(invoke->GetLocations(), Primitive::kPrimInt, GetVIXLAssembler());
333}
334
335void IntrinsicLocationsBuilderARM64::VisitLongNumberOfTrailingZeros(HInvoke* invoke) {
336  CreateIntToIntLocations(arena_, invoke);
337}
338
339void IntrinsicCodeGeneratorARM64::VisitLongNumberOfTrailingZeros(HInvoke* invoke) {
340  GenNumberOfTrailingZeros(invoke->GetLocations(), Primitive::kPrimLong, GetVIXLAssembler());
341}
342
343static void GenReverse(LocationSummary* locations,
344                       Primitive::Type type,
345                       vixl::MacroAssembler* masm) {
346  DCHECK(type == Primitive::kPrimInt || type == Primitive::kPrimLong);
347
348  Location in = locations->InAt(0);
349  Location out = locations->Out();
350
351  __ Rbit(RegisterFrom(out, type), RegisterFrom(in, type));
352}
353
354void IntrinsicLocationsBuilderARM64::VisitIntegerReverse(HInvoke* invoke) {
355  CreateIntToIntLocations(arena_, invoke);
356}
357
358void IntrinsicCodeGeneratorARM64::VisitIntegerReverse(HInvoke* invoke) {
359  GenReverse(invoke->GetLocations(), Primitive::kPrimInt, GetVIXLAssembler());
360}
361
362void IntrinsicLocationsBuilderARM64::VisitLongReverse(HInvoke* invoke) {
363  CreateIntToIntLocations(arena_, invoke);
364}
365
366void IntrinsicCodeGeneratorARM64::VisitLongReverse(HInvoke* invoke) {
367  GenReverse(invoke->GetLocations(), Primitive::kPrimLong, GetVIXLAssembler());
368}
369
370static void CreateFPToFPLocations(ArenaAllocator* arena, HInvoke* invoke) {
371  LocationSummary* locations = new (arena) LocationSummary(invoke,
372                                                           LocationSummary::kNoCall,
373                                                           kIntrinsified);
374  locations->SetInAt(0, Location::RequiresFpuRegister());
375  locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap);
376}
377
378static void MathAbsFP(LocationSummary* locations, bool is64bit, vixl::MacroAssembler* masm) {
379  Location in = locations->InAt(0);
380  Location out = locations->Out();
381
382  FPRegister in_reg = is64bit ? DRegisterFrom(in) : SRegisterFrom(in);
383  FPRegister out_reg = is64bit ? DRegisterFrom(out) : SRegisterFrom(out);
384
385  __ Fabs(out_reg, in_reg);
386}
387
388void IntrinsicLocationsBuilderARM64::VisitMathAbsDouble(HInvoke* invoke) {
389  CreateFPToFPLocations(arena_, invoke);
390}
391
392void IntrinsicCodeGeneratorARM64::VisitMathAbsDouble(HInvoke* invoke) {
393  MathAbsFP(invoke->GetLocations(), /* is64bit */ true, GetVIXLAssembler());
394}
395
396void IntrinsicLocationsBuilderARM64::VisitMathAbsFloat(HInvoke* invoke) {
397  CreateFPToFPLocations(arena_, invoke);
398}
399
400void IntrinsicCodeGeneratorARM64::VisitMathAbsFloat(HInvoke* invoke) {
401  MathAbsFP(invoke->GetLocations(), /* is64bit */ false, GetVIXLAssembler());
402}
403
404static void CreateIntToInt(ArenaAllocator* arena, HInvoke* invoke) {
405  LocationSummary* locations = new (arena) LocationSummary(invoke,
406                                                           LocationSummary::kNoCall,
407                                                           kIntrinsified);
408  locations->SetInAt(0, Location::RequiresRegister());
409  locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
410}
411
412static void GenAbsInteger(LocationSummary* locations,
413                          bool is64bit,
414                          vixl::MacroAssembler* masm) {
415  Location in = locations->InAt(0);
416  Location output = locations->Out();
417
418  Register in_reg = is64bit ? XRegisterFrom(in) : WRegisterFrom(in);
419  Register out_reg = is64bit ? XRegisterFrom(output) : WRegisterFrom(output);
420
421  __ Cmp(in_reg, Operand(0));
422  __ Cneg(out_reg, in_reg, lt);
423}
424
425void IntrinsicLocationsBuilderARM64::VisitMathAbsInt(HInvoke* invoke) {
426  CreateIntToInt(arena_, invoke);
427}
428
429void IntrinsicCodeGeneratorARM64::VisitMathAbsInt(HInvoke* invoke) {
430  GenAbsInteger(invoke->GetLocations(), /* is64bit */ false, GetVIXLAssembler());
431}
432
433void IntrinsicLocationsBuilderARM64::VisitMathAbsLong(HInvoke* invoke) {
434  CreateIntToInt(arena_, invoke);
435}
436
437void IntrinsicCodeGeneratorARM64::VisitMathAbsLong(HInvoke* invoke) {
438  GenAbsInteger(invoke->GetLocations(), /* is64bit */ true, GetVIXLAssembler());
439}
440
441static void GenMinMaxFP(LocationSummary* locations,
442                        bool is_min,
443                        bool is_double,
444                        vixl::MacroAssembler* masm) {
445  Location op1 = locations->InAt(0);
446  Location op2 = locations->InAt(1);
447  Location out = locations->Out();
448
449  FPRegister op1_reg = is_double ? DRegisterFrom(op1) : SRegisterFrom(op1);
450  FPRegister op2_reg = is_double ? DRegisterFrom(op2) : SRegisterFrom(op2);
451  FPRegister out_reg = is_double ? DRegisterFrom(out) : SRegisterFrom(out);
452  if (is_min) {
453    __ Fmin(out_reg, op1_reg, op2_reg);
454  } else {
455    __ Fmax(out_reg, op1_reg, op2_reg);
456  }
457}
458
459static void CreateFPFPToFPLocations(ArenaAllocator* arena, HInvoke* invoke) {
460  LocationSummary* locations = new (arena) LocationSummary(invoke,
461                                                           LocationSummary::kNoCall,
462                                                           kIntrinsified);
463  locations->SetInAt(0, Location::RequiresFpuRegister());
464  locations->SetInAt(1, Location::RequiresFpuRegister());
465  locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap);
466}
467
468void IntrinsicLocationsBuilderARM64::VisitMathMinDoubleDouble(HInvoke* invoke) {
469  CreateFPFPToFPLocations(arena_, invoke);
470}
471
472void IntrinsicCodeGeneratorARM64::VisitMathMinDoubleDouble(HInvoke* invoke) {
473  GenMinMaxFP(invoke->GetLocations(), /* is_min */ true, /* is_double */ true, GetVIXLAssembler());
474}
475
476void IntrinsicLocationsBuilderARM64::VisitMathMinFloatFloat(HInvoke* invoke) {
477  CreateFPFPToFPLocations(arena_, invoke);
478}
479
480void IntrinsicCodeGeneratorARM64::VisitMathMinFloatFloat(HInvoke* invoke) {
481  GenMinMaxFP(invoke->GetLocations(), /* is_min */ true, /* is_double */ false, GetVIXLAssembler());
482}
483
484void IntrinsicLocationsBuilderARM64::VisitMathMaxDoubleDouble(HInvoke* invoke) {
485  CreateFPFPToFPLocations(arena_, invoke);
486}
487
488void IntrinsicCodeGeneratorARM64::VisitMathMaxDoubleDouble(HInvoke* invoke) {
489  GenMinMaxFP(invoke->GetLocations(), /* is_min */ false, /* is_double */ true, GetVIXLAssembler());
490}
491
492void IntrinsicLocationsBuilderARM64::VisitMathMaxFloatFloat(HInvoke* invoke) {
493  CreateFPFPToFPLocations(arena_, invoke);
494}
495
496void IntrinsicCodeGeneratorARM64::VisitMathMaxFloatFloat(HInvoke* invoke) {
497  GenMinMaxFP(
498      invoke->GetLocations(), /* is_min */ false, /* is_double */ false, GetVIXLAssembler());
499}
500
501static void GenMinMax(LocationSummary* locations,
502                      bool is_min,
503                      bool is_long,
504                      vixl::MacroAssembler* masm) {
505  Location op1 = locations->InAt(0);
506  Location op2 = locations->InAt(1);
507  Location out = locations->Out();
508
509  Register op1_reg = is_long ? XRegisterFrom(op1) : WRegisterFrom(op1);
510  Register op2_reg = is_long ? XRegisterFrom(op2) : WRegisterFrom(op2);
511  Register out_reg = is_long ? XRegisterFrom(out) : WRegisterFrom(out);
512
513  __ Cmp(op1_reg, op2_reg);
514  __ Csel(out_reg, op1_reg, op2_reg, is_min ? lt : gt);
515}
516
517void IntrinsicLocationsBuilderARM64::VisitMathMinIntInt(HInvoke* invoke) {
518  CreateIntIntToIntLocations(arena_, invoke);
519}
520
521void IntrinsicCodeGeneratorARM64::VisitMathMinIntInt(HInvoke* invoke) {
522  GenMinMax(invoke->GetLocations(), /* is_min */ true, /* is_long */ false, GetVIXLAssembler());
523}
524
525void IntrinsicLocationsBuilderARM64::VisitMathMinLongLong(HInvoke* invoke) {
526  CreateIntIntToIntLocations(arena_, invoke);
527}
528
529void IntrinsicCodeGeneratorARM64::VisitMathMinLongLong(HInvoke* invoke) {
530  GenMinMax(invoke->GetLocations(), /* is_min */ true, /* is_long */ true, GetVIXLAssembler());
531}
532
533void IntrinsicLocationsBuilderARM64::VisitMathMaxIntInt(HInvoke* invoke) {
534  CreateIntIntToIntLocations(arena_, invoke);
535}
536
537void IntrinsicCodeGeneratorARM64::VisitMathMaxIntInt(HInvoke* invoke) {
538  GenMinMax(invoke->GetLocations(), /* is_min */ false, /* is_long */ false, GetVIXLAssembler());
539}
540
541void IntrinsicLocationsBuilderARM64::VisitMathMaxLongLong(HInvoke* invoke) {
542  CreateIntIntToIntLocations(arena_, invoke);
543}
544
545void IntrinsicCodeGeneratorARM64::VisitMathMaxLongLong(HInvoke* invoke) {
546  GenMinMax(invoke->GetLocations(), /* is_min */ false, /* is_long */ true, GetVIXLAssembler());
547}
548
549void IntrinsicLocationsBuilderARM64::VisitMathSqrt(HInvoke* invoke) {
550  CreateFPToFPLocations(arena_, invoke);
551}
552
553void IntrinsicCodeGeneratorARM64::VisitMathSqrt(HInvoke* invoke) {
554  LocationSummary* locations = invoke->GetLocations();
555  vixl::MacroAssembler* masm = GetVIXLAssembler();
556  __ Fsqrt(DRegisterFrom(locations->Out()), DRegisterFrom(locations->InAt(0)));
557}
558
559void IntrinsicLocationsBuilderARM64::VisitMathCeil(HInvoke* invoke) {
560  CreateFPToFPLocations(arena_, invoke);
561}
562
563void IntrinsicCodeGeneratorARM64::VisitMathCeil(HInvoke* invoke) {
564  LocationSummary* locations = invoke->GetLocations();
565  vixl::MacroAssembler* masm = GetVIXLAssembler();
566  __ Frintp(DRegisterFrom(locations->Out()), DRegisterFrom(locations->InAt(0)));
567}
568
569void IntrinsicLocationsBuilderARM64::VisitMathFloor(HInvoke* invoke) {
570  CreateFPToFPLocations(arena_, invoke);
571}
572
573void IntrinsicCodeGeneratorARM64::VisitMathFloor(HInvoke* invoke) {
574  LocationSummary* locations = invoke->GetLocations();
575  vixl::MacroAssembler* masm = GetVIXLAssembler();
576  __ Frintm(DRegisterFrom(locations->Out()), DRegisterFrom(locations->InAt(0)));
577}
578
579void IntrinsicLocationsBuilderARM64::VisitMathRint(HInvoke* invoke) {
580  CreateFPToFPLocations(arena_, invoke);
581}
582
583void IntrinsicCodeGeneratorARM64::VisitMathRint(HInvoke* invoke) {
584  LocationSummary* locations = invoke->GetLocations();
585  vixl::MacroAssembler* masm = GetVIXLAssembler();
586  __ Frintn(DRegisterFrom(locations->Out()), DRegisterFrom(locations->InAt(0)));
587}
588
589static void CreateFPToIntPlusTempLocations(ArenaAllocator* arena, HInvoke* invoke) {
590  LocationSummary* locations = new (arena) LocationSummary(invoke,
591                                                           LocationSummary::kNoCall,
592                                                           kIntrinsified);
593  locations->SetInAt(0, Location::RequiresFpuRegister());
594  locations->SetOut(Location::RequiresRegister());
595}
596
597static void GenMathRound(LocationSummary* locations,
598                         bool is_double,
599                         vixl::MacroAssembler* masm) {
600  FPRegister in_reg = is_double ?
601      DRegisterFrom(locations->InAt(0)) : SRegisterFrom(locations->InAt(0));
602  Register out_reg = is_double ?
603      XRegisterFrom(locations->Out()) : WRegisterFrom(locations->Out());
604  UseScratchRegisterScope temps(masm);
605  FPRegister temp1_reg = temps.AcquireSameSizeAs(in_reg);
606
607  // 0.5 can be encoded as an immediate, so use fmov.
608  if (is_double) {
609    __ Fmov(temp1_reg, static_cast<double>(0.5));
610  } else {
611    __ Fmov(temp1_reg, static_cast<float>(0.5));
612  }
613  __ Fadd(temp1_reg, in_reg, temp1_reg);
614  __ Fcvtms(out_reg, temp1_reg);
615}
616
617void IntrinsicLocationsBuilderARM64::VisitMathRoundDouble(HInvoke* invoke) {
618  // See intrinsics.h.
619  if (kRoundIsPlusPointFive) {
620    CreateFPToIntPlusTempLocations(arena_, invoke);
621  }
622}
623
624void IntrinsicCodeGeneratorARM64::VisitMathRoundDouble(HInvoke* invoke) {
625  GenMathRound(invoke->GetLocations(), /* is_double */ true, GetVIXLAssembler());
626}
627
628void IntrinsicLocationsBuilderARM64::VisitMathRoundFloat(HInvoke* invoke) {
629  // See intrinsics.h.
630  if (kRoundIsPlusPointFive) {
631    CreateFPToIntPlusTempLocations(arena_, invoke);
632  }
633}
634
635void IntrinsicCodeGeneratorARM64::VisitMathRoundFloat(HInvoke* invoke) {
636  GenMathRound(invoke->GetLocations(), /* is_double */ false, GetVIXLAssembler());
637}
638
639void IntrinsicLocationsBuilderARM64::VisitMemoryPeekByte(HInvoke* invoke) {
640  CreateIntToIntLocations(arena_, invoke);
641}
642
643void IntrinsicCodeGeneratorARM64::VisitMemoryPeekByte(HInvoke* invoke) {
644  vixl::MacroAssembler* masm = GetVIXLAssembler();
645  __ Ldrsb(WRegisterFrom(invoke->GetLocations()->Out()),
646          AbsoluteHeapOperandFrom(invoke->GetLocations()->InAt(0), 0));
647}
648
649void IntrinsicLocationsBuilderARM64::VisitMemoryPeekIntNative(HInvoke* invoke) {
650  CreateIntToIntLocations(arena_, invoke);
651}
652
653void IntrinsicCodeGeneratorARM64::VisitMemoryPeekIntNative(HInvoke* invoke) {
654  vixl::MacroAssembler* masm = GetVIXLAssembler();
655  __ Ldr(WRegisterFrom(invoke->GetLocations()->Out()),
656         AbsoluteHeapOperandFrom(invoke->GetLocations()->InAt(0), 0));
657}
658
659void IntrinsicLocationsBuilderARM64::VisitMemoryPeekLongNative(HInvoke* invoke) {
660  CreateIntToIntLocations(arena_, invoke);
661}
662
663void IntrinsicCodeGeneratorARM64::VisitMemoryPeekLongNative(HInvoke* invoke) {
664  vixl::MacroAssembler* masm = GetVIXLAssembler();
665  __ Ldr(XRegisterFrom(invoke->GetLocations()->Out()),
666         AbsoluteHeapOperandFrom(invoke->GetLocations()->InAt(0), 0));
667}
668
669void IntrinsicLocationsBuilderARM64::VisitMemoryPeekShortNative(HInvoke* invoke) {
670  CreateIntToIntLocations(arena_, invoke);
671}
672
673void IntrinsicCodeGeneratorARM64::VisitMemoryPeekShortNative(HInvoke* invoke) {
674  vixl::MacroAssembler* masm = GetVIXLAssembler();
675  __ Ldrsh(WRegisterFrom(invoke->GetLocations()->Out()),
676           AbsoluteHeapOperandFrom(invoke->GetLocations()->InAt(0), 0));
677}
678
679static void CreateIntIntToVoidLocations(ArenaAllocator* arena, HInvoke* invoke) {
680  LocationSummary* locations = new (arena) LocationSummary(invoke,
681                                                           LocationSummary::kNoCall,
682                                                           kIntrinsified);
683  locations->SetInAt(0, Location::RequiresRegister());
684  locations->SetInAt(1, Location::RequiresRegister());
685}
686
687void IntrinsicLocationsBuilderARM64::VisitMemoryPokeByte(HInvoke* invoke) {
688  CreateIntIntToVoidLocations(arena_, invoke);
689}
690
691void IntrinsicCodeGeneratorARM64::VisitMemoryPokeByte(HInvoke* invoke) {
692  vixl::MacroAssembler* masm = GetVIXLAssembler();
693  __ Strb(WRegisterFrom(invoke->GetLocations()->InAt(1)),
694          AbsoluteHeapOperandFrom(invoke->GetLocations()->InAt(0), 0));
695}
696
697void IntrinsicLocationsBuilderARM64::VisitMemoryPokeIntNative(HInvoke* invoke) {
698  CreateIntIntToVoidLocations(arena_, invoke);
699}
700
701void IntrinsicCodeGeneratorARM64::VisitMemoryPokeIntNative(HInvoke* invoke) {
702  vixl::MacroAssembler* masm = GetVIXLAssembler();
703  __ Str(WRegisterFrom(invoke->GetLocations()->InAt(1)),
704         AbsoluteHeapOperandFrom(invoke->GetLocations()->InAt(0), 0));
705}
706
707void IntrinsicLocationsBuilderARM64::VisitMemoryPokeLongNative(HInvoke* invoke) {
708  CreateIntIntToVoidLocations(arena_, invoke);
709}
710
711void IntrinsicCodeGeneratorARM64::VisitMemoryPokeLongNative(HInvoke* invoke) {
712  vixl::MacroAssembler* masm = GetVIXLAssembler();
713  __ Str(XRegisterFrom(invoke->GetLocations()->InAt(1)),
714         AbsoluteHeapOperandFrom(invoke->GetLocations()->InAt(0), 0));
715}
716
717void IntrinsicLocationsBuilderARM64::VisitMemoryPokeShortNative(HInvoke* invoke) {
718  CreateIntIntToVoidLocations(arena_, invoke);
719}
720
721void IntrinsicCodeGeneratorARM64::VisitMemoryPokeShortNative(HInvoke* invoke) {
722  vixl::MacroAssembler* masm = GetVIXLAssembler();
723  __ Strh(WRegisterFrom(invoke->GetLocations()->InAt(1)),
724          AbsoluteHeapOperandFrom(invoke->GetLocations()->InAt(0), 0));
725}
726
727void IntrinsicLocationsBuilderARM64::VisitThreadCurrentThread(HInvoke* invoke) {
728  LocationSummary* locations = new (arena_) LocationSummary(invoke,
729                                                            LocationSummary::kNoCall,
730                                                            kIntrinsified);
731  locations->SetOut(Location::RequiresRegister());
732}
733
734void IntrinsicCodeGeneratorARM64::VisitThreadCurrentThread(HInvoke* invoke) {
735  codegen_->Load(Primitive::kPrimNot, WRegisterFrom(invoke->GetLocations()->Out()),
736                 MemOperand(tr, Thread::PeerOffset<8>().Int32Value()));
737}
738
739static void GenUnsafeGet(HInvoke* invoke,
740                         Primitive::Type type,
741                         bool is_volatile,
742                         CodeGeneratorARM64* codegen) {
743  LocationSummary* locations = invoke->GetLocations();
744  DCHECK((type == Primitive::kPrimInt) ||
745         (type == Primitive::kPrimLong) ||
746         (type == Primitive::kPrimNot));
747  vixl::MacroAssembler* masm = codegen->GetAssembler()->vixl_masm_;
748  Location base_loc = locations->InAt(1);
749  Register base = WRegisterFrom(base_loc);      // Object pointer.
750  Location offset_loc = locations->InAt(2);
751  Register offset = XRegisterFrom(offset_loc);  // Long offset.
752  Location trg_loc = locations->Out();
753  Register trg = RegisterFrom(trg_loc, type);
754
755  if (type == Primitive::kPrimNot && kEmitCompilerReadBarrier && kUseBakerReadBarrier) {
756    // UnsafeGetObject/UnsafeGetObjectVolatile with Baker's read barrier case.
757    UseScratchRegisterScope temps(masm);
758    Register temp = temps.AcquireW();
759    codegen->GenerateArrayLoadWithBakerReadBarrier(
760        invoke, trg_loc, base, 0U, offset_loc, temp, /* needs_null_check */ false);
761  } else {
762    // Other cases.
763    MemOperand mem_op(base.X(), offset);
764    if (is_volatile) {
765      codegen->LoadAcquire(invoke, trg, mem_op, /* needs_null_check */ true);
766    } else {
767      codegen->Load(type, trg, mem_op);
768    }
769
770    if (type == Primitive::kPrimNot) {
771      DCHECK(trg.IsW());
772      codegen->MaybeGenerateReadBarrierSlow(invoke, trg_loc, trg_loc, base_loc, 0U, offset_loc);
773    }
774  }
775}
776
777static void CreateIntIntIntToIntLocations(ArenaAllocator* arena, HInvoke* invoke) {
778  bool can_call = kEmitCompilerReadBarrier &&
779      (invoke->GetIntrinsic() == Intrinsics::kUnsafeGetObject ||
780       invoke->GetIntrinsic() == Intrinsics::kUnsafeGetObjectVolatile);
781  LocationSummary* locations = new (arena) LocationSummary(invoke,
782                                                           can_call ?
783                                                               LocationSummary::kCallOnSlowPath :
784                                                               LocationSummary::kNoCall,
785                                                           kIntrinsified);
786  locations->SetInAt(0, Location::NoLocation());        // Unused receiver.
787  locations->SetInAt(1, Location::RequiresRegister());
788  locations->SetInAt(2, Location::RequiresRegister());
789  locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
790}
791
792void IntrinsicLocationsBuilderARM64::VisitUnsafeGet(HInvoke* invoke) {
793  CreateIntIntIntToIntLocations(arena_, invoke);
794}
795void IntrinsicLocationsBuilderARM64::VisitUnsafeGetVolatile(HInvoke* invoke) {
796  CreateIntIntIntToIntLocations(arena_, invoke);
797}
798void IntrinsicLocationsBuilderARM64::VisitUnsafeGetLong(HInvoke* invoke) {
799  CreateIntIntIntToIntLocations(arena_, invoke);
800}
801void IntrinsicLocationsBuilderARM64::VisitUnsafeGetLongVolatile(HInvoke* invoke) {
802  CreateIntIntIntToIntLocations(arena_, invoke);
803}
804void IntrinsicLocationsBuilderARM64::VisitUnsafeGetObject(HInvoke* invoke) {
805  CreateIntIntIntToIntLocations(arena_, invoke);
806}
807void IntrinsicLocationsBuilderARM64::VisitUnsafeGetObjectVolatile(HInvoke* invoke) {
808  CreateIntIntIntToIntLocations(arena_, invoke);
809}
810
811void IntrinsicCodeGeneratorARM64::VisitUnsafeGet(HInvoke* invoke) {
812  GenUnsafeGet(invoke, Primitive::kPrimInt, /* is_volatile */ false, codegen_);
813}
814void IntrinsicCodeGeneratorARM64::VisitUnsafeGetVolatile(HInvoke* invoke) {
815  GenUnsafeGet(invoke, Primitive::kPrimInt, /* is_volatile */ true, codegen_);
816}
817void IntrinsicCodeGeneratorARM64::VisitUnsafeGetLong(HInvoke* invoke) {
818  GenUnsafeGet(invoke, Primitive::kPrimLong, /* is_volatile */ false, codegen_);
819}
820void IntrinsicCodeGeneratorARM64::VisitUnsafeGetLongVolatile(HInvoke* invoke) {
821  GenUnsafeGet(invoke, Primitive::kPrimLong, /* is_volatile */ true, codegen_);
822}
823void IntrinsicCodeGeneratorARM64::VisitUnsafeGetObject(HInvoke* invoke) {
824  GenUnsafeGet(invoke, Primitive::kPrimNot, /* is_volatile */ false, codegen_);
825}
826void IntrinsicCodeGeneratorARM64::VisitUnsafeGetObjectVolatile(HInvoke* invoke) {
827  GenUnsafeGet(invoke, Primitive::kPrimNot, /* is_volatile */ true, codegen_);
828}
829
830static void CreateIntIntIntIntToVoid(ArenaAllocator* arena, HInvoke* invoke) {
831  LocationSummary* locations = new (arena) LocationSummary(invoke,
832                                                           LocationSummary::kNoCall,
833                                                           kIntrinsified);
834  locations->SetInAt(0, Location::NoLocation());        // Unused receiver.
835  locations->SetInAt(1, Location::RequiresRegister());
836  locations->SetInAt(2, Location::RequiresRegister());
837  locations->SetInAt(3, Location::RequiresRegister());
838}
839
840void IntrinsicLocationsBuilderARM64::VisitUnsafePut(HInvoke* invoke) {
841  CreateIntIntIntIntToVoid(arena_, invoke);
842}
843void IntrinsicLocationsBuilderARM64::VisitUnsafePutOrdered(HInvoke* invoke) {
844  CreateIntIntIntIntToVoid(arena_, invoke);
845}
846void IntrinsicLocationsBuilderARM64::VisitUnsafePutVolatile(HInvoke* invoke) {
847  CreateIntIntIntIntToVoid(arena_, invoke);
848}
849void IntrinsicLocationsBuilderARM64::VisitUnsafePutObject(HInvoke* invoke) {
850  CreateIntIntIntIntToVoid(arena_, invoke);
851}
852void IntrinsicLocationsBuilderARM64::VisitUnsafePutObjectOrdered(HInvoke* invoke) {
853  CreateIntIntIntIntToVoid(arena_, invoke);
854}
855void IntrinsicLocationsBuilderARM64::VisitUnsafePutObjectVolatile(HInvoke* invoke) {
856  CreateIntIntIntIntToVoid(arena_, invoke);
857}
858void IntrinsicLocationsBuilderARM64::VisitUnsafePutLong(HInvoke* invoke) {
859  CreateIntIntIntIntToVoid(arena_, invoke);
860}
861void IntrinsicLocationsBuilderARM64::VisitUnsafePutLongOrdered(HInvoke* invoke) {
862  CreateIntIntIntIntToVoid(arena_, invoke);
863}
864void IntrinsicLocationsBuilderARM64::VisitUnsafePutLongVolatile(HInvoke* invoke) {
865  CreateIntIntIntIntToVoid(arena_, invoke);
866}
867
868static void GenUnsafePut(LocationSummary* locations,
869                         Primitive::Type type,
870                         bool is_volatile,
871                         bool is_ordered,
872                         CodeGeneratorARM64* codegen) {
873  vixl::MacroAssembler* masm = codegen->GetAssembler()->vixl_masm_;
874
875  Register base = WRegisterFrom(locations->InAt(1));    // Object pointer.
876  Register offset = XRegisterFrom(locations->InAt(2));  // Long offset.
877  Register value = RegisterFrom(locations->InAt(3), type);
878  Register source = value;
879  MemOperand mem_op(base.X(), offset);
880
881  {
882    // We use a block to end the scratch scope before the write barrier, thus
883    // freeing the temporary registers so they can be used in `MarkGCCard`.
884    UseScratchRegisterScope temps(masm);
885
886    if (kPoisonHeapReferences && type == Primitive::kPrimNot) {
887      DCHECK(value.IsW());
888      Register temp = temps.AcquireW();
889      __ Mov(temp.W(), value.W());
890      codegen->GetAssembler()->PoisonHeapReference(temp.W());
891      source = temp;
892    }
893
894    if (is_volatile || is_ordered) {
895      codegen->StoreRelease(type, source, mem_op);
896    } else {
897      codegen->Store(type, source, mem_op);
898    }
899  }
900
901  if (type == Primitive::kPrimNot) {
902    bool value_can_be_null = true;  // TODO: Worth finding out this information?
903    codegen->MarkGCCard(base, value, value_can_be_null);
904  }
905}
906
907void IntrinsicCodeGeneratorARM64::VisitUnsafePut(HInvoke* invoke) {
908  GenUnsafePut(invoke->GetLocations(),
909               Primitive::kPrimInt,
910               /* is_volatile */ false,
911               /* is_ordered */ false,
912               codegen_);
913}
914void IntrinsicCodeGeneratorARM64::VisitUnsafePutOrdered(HInvoke* invoke) {
915  GenUnsafePut(invoke->GetLocations(),
916               Primitive::kPrimInt,
917               /* is_volatile */ false,
918               /* is_ordered */ true,
919               codegen_);
920}
921void IntrinsicCodeGeneratorARM64::VisitUnsafePutVolatile(HInvoke* invoke) {
922  GenUnsafePut(invoke->GetLocations(),
923               Primitive::kPrimInt,
924               /* is_volatile */ true,
925               /* is_ordered */ false,
926               codegen_);
927}
928void IntrinsicCodeGeneratorARM64::VisitUnsafePutObject(HInvoke* invoke) {
929  GenUnsafePut(invoke->GetLocations(),
930               Primitive::kPrimNot,
931               /* is_volatile */ false,
932               /* is_ordered */ false,
933               codegen_);
934}
935void IntrinsicCodeGeneratorARM64::VisitUnsafePutObjectOrdered(HInvoke* invoke) {
936  GenUnsafePut(invoke->GetLocations(),
937               Primitive::kPrimNot,
938               /* is_volatile */ false,
939               /* is_ordered */ true,
940               codegen_);
941}
942void IntrinsicCodeGeneratorARM64::VisitUnsafePutObjectVolatile(HInvoke* invoke) {
943  GenUnsafePut(invoke->GetLocations(),
944               Primitive::kPrimNot,
945               /* is_volatile */ true,
946               /* is_ordered */ false,
947               codegen_);
948}
949void IntrinsicCodeGeneratorARM64::VisitUnsafePutLong(HInvoke* invoke) {
950  GenUnsafePut(invoke->GetLocations(),
951               Primitive::kPrimLong,
952               /* is_volatile */ false,
953               /* is_ordered */ false,
954               codegen_);
955}
956void IntrinsicCodeGeneratorARM64::VisitUnsafePutLongOrdered(HInvoke* invoke) {
957  GenUnsafePut(invoke->GetLocations(),
958               Primitive::kPrimLong,
959               /* is_volatile */ false,
960               /* is_ordered */ true,
961               codegen_);
962}
963void IntrinsicCodeGeneratorARM64::VisitUnsafePutLongVolatile(HInvoke* invoke) {
964  GenUnsafePut(invoke->GetLocations(),
965               Primitive::kPrimLong,
966               /* is_volatile */ true,
967               /* is_ordered */ false,
968               codegen_);
969}
970
971static void CreateIntIntIntIntIntToInt(ArenaAllocator* arena,
972                                       HInvoke* invoke,
973                                       Primitive::Type type) {
974  LocationSummary* locations = new (arena) LocationSummary(invoke,
975                                                           LocationSummary::kNoCall,
976                                                           kIntrinsified);
977  locations->SetInAt(0, Location::NoLocation());        // Unused receiver.
978  locations->SetInAt(1, Location::RequiresRegister());
979  locations->SetInAt(2, Location::RequiresRegister());
980  locations->SetInAt(3, Location::RequiresRegister());
981  locations->SetInAt(4, Location::RequiresRegister());
982
983  // If heap poisoning is enabled, we don't want the unpoisoning
984  // operations to potentially clobber the output.
985  Location::OutputOverlap overlaps = (kPoisonHeapReferences && type == Primitive::kPrimNot)
986      ? Location::kOutputOverlap
987      : Location::kNoOutputOverlap;
988  locations->SetOut(Location::RequiresRegister(), overlaps);
989}
990
991static void GenCas(LocationSummary* locations, Primitive::Type type, CodeGeneratorARM64* codegen) {
992  vixl::MacroAssembler* masm = codegen->GetAssembler()->vixl_masm_;
993
994  Register out = WRegisterFrom(locations->Out());                  // Boolean result.
995
996  Register base = WRegisterFrom(locations->InAt(1));               // Object pointer.
997  Register offset = XRegisterFrom(locations->InAt(2));             // Long offset.
998  Register expected = RegisterFrom(locations->InAt(3), type);      // Expected.
999  Register value = RegisterFrom(locations->InAt(4), type);         // Value.
1000
1001  // This needs to be before the temp registers, as MarkGCCard also uses VIXL temps.
1002  if (type == Primitive::kPrimNot) {
1003    // Mark card for object assuming new value is stored.
1004    bool value_can_be_null = true;  // TODO: Worth finding out this information?
1005    codegen->MarkGCCard(base, value, value_can_be_null);
1006  }
1007
1008  UseScratchRegisterScope temps(masm);
1009  Register tmp_ptr = temps.AcquireX();                             // Pointer to actual memory.
1010  Register tmp_value = temps.AcquireSameSizeAs(value);             // Value in memory.
1011
1012  Register tmp_32 = tmp_value.W();
1013
1014  __ Add(tmp_ptr, base.X(), Operand(offset));
1015
1016  if (kPoisonHeapReferences && type == Primitive::kPrimNot) {
1017    codegen->GetAssembler()->PoisonHeapReference(expected);
1018    if (value.Is(expected)) {
1019      // Do not poison `value`, as it is the same register as
1020      // `expected`, which has just been poisoned.
1021    } else {
1022      codegen->GetAssembler()->PoisonHeapReference(value);
1023    }
1024  }
1025
1026  // do {
1027  //   tmp_value = [tmp_ptr] - expected;
1028  // } while (tmp_value == 0 && failure([tmp_ptr] <- r_new_value));
1029  // result = tmp_value != 0;
1030
1031  vixl::Label loop_head, exit_loop;
1032  __ Bind(&loop_head);
1033  // TODO: When `type == Primitive::kPrimNot`, add a read barrier for
1034  // the reference stored in the object before attempting the CAS,
1035  // similar to the one in the art::Unsafe_compareAndSwapObject JNI
1036  // implementation.
1037  //
1038  // Note that this code is not (yet) used when read barriers are
1039  // enabled (see IntrinsicLocationsBuilderARM64::VisitUnsafeCASObject).
1040  DCHECK(!(type == Primitive::kPrimNot && kEmitCompilerReadBarrier));
1041  __ Ldaxr(tmp_value, MemOperand(tmp_ptr));
1042  __ Cmp(tmp_value, expected);
1043  __ B(&exit_loop, ne);
1044  __ Stlxr(tmp_32, value, MemOperand(tmp_ptr));
1045  __ Cbnz(tmp_32, &loop_head);
1046  __ Bind(&exit_loop);
1047  __ Cset(out, eq);
1048
1049  if (kPoisonHeapReferences && type == Primitive::kPrimNot) {
1050    codegen->GetAssembler()->UnpoisonHeapReference(expected);
1051    if (value.Is(expected)) {
1052      // Do not unpoison `value`, as it is the same register as
1053      // `expected`, which has just been unpoisoned.
1054    } else {
1055      codegen->GetAssembler()->UnpoisonHeapReference(value);
1056    }
1057  }
1058}
1059
1060void IntrinsicLocationsBuilderARM64::VisitUnsafeCASInt(HInvoke* invoke) {
1061  CreateIntIntIntIntIntToInt(arena_, invoke, Primitive::kPrimInt);
1062}
1063void IntrinsicLocationsBuilderARM64::VisitUnsafeCASLong(HInvoke* invoke) {
1064  CreateIntIntIntIntIntToInt(arena_, invoke, Primitive::kPrimLong);
1065}
1066void IntrinsicLocationsBuilderARM64::VisitUnsafeCASObject(HInvoke* invoke) {
1067  // The UnsafeCASObject intrinsic is missing a read barrier, and
1068  // therefore sometimes does not work as expected (b/25883050).
1069  // Turn it off temporarily as a quick fix, until the read barrier is
1070  // implemented (see TODO in GenCAS below).
1071  //
1072  // TODO(rpl): Fix this issue and re-enable this intrinsic with read barriers.
1073  if (kEmitCompilerReadBarrier) {
1074    return;
1075  }
1076
1077  CreateIntIntIntIntIntToInt(arena_, invoke, Primitive::kPrimNot);
1078}
1079
1080void IntrinsicCodeGeneratorARM64::VisitUnsafeCASInt(HInvoke* invoke) {
1081  GenCas(invoke->GetLocations(), Primitive::kPrimInt, codegen_);
1082}
1083void IntrinsicCodeGeneratorARM64::VisitUnsafeCASLong(HInvoke* invoke) {
1084  GenCas(invoke->GetLocations(), Primitive::kPrimLong, codegen_);
1085}
1086void IntrinsicCodeGeneratorARM64::VisitUnsafeCASObject(HInvoke* invoke) {
1087  GenCas(invoke->GetLocations(), Primitive::kPrimNot, codegen_);
1088}
1089
1090void IntrinsicLocationsBuilderARM64::VisitStringCharAt(HInvoke* invoke) {
1091  LocationSummary* locations = new (arena_) LocationSummary(invoke,
1092                                                            LocationSummary::kCallOnSlowPath,
1093                                                            kIntrinsified);
1094  locations->SetInAt(0, Location::RequiresRegister());
1095  locations->SetInAt(1, Location::RequiresRegister());
1096  // In case we need to go in the slow path, we can't have the output be the same
1097  // as the input: the current liveness analysis considers the input to be live
1098  // at the point of the call.
1099  locations->SetOut(Location::RequiresRegister(), Location::kOutputOverlap);
1100}
1101
1102void IntrinsicCodeGeneratorARM64::VisitStringCharAt(HInvoke* invoke) {
1103  vixl::MacroAssembler* masm = GetVIXLAssembler();
1104  LocationSummary* locations = invoke->GetLocations();
1105
1106  // Location of reference to data array
1107  const MemberOffset value_offset = mirror::String::ValueOffset();
1108  // Location of count
1109  const MemberOffset count_offset = mirror::String::CountOffset();
1110
1111  Register obj = WRegisterFrom(locations->InAt(0));  // String object pointer.
1112  Register idx = WRegisterFrom(locations->InAt(1));  // Index of character.
1113  Register out = WRegisterFrom(locations->Out());    // Result character.
1114
1115  UseScratchRegisterScope temps(masm);
1116  Register temp = temps.AcquireW();
1117  Register array_temp = temps.AcquireW();            // We can trade this for worse scheduling.
1118
1119  // TODO: Maybe we can support range check elimination. Overall, though, I think it's not worth
1120  //       the cost.
1121  // TODO: For simplicity, the index parameter is requested in a register, so different from Quick
1122  //       we will not optimize the code for constants (which would save a register).
1123
1124  SlowPathCodeARM64* slow_path = new (GetAllocator()) IntrinsicSlowPathARM64(invoke);
1125  codegen_->AddSlowPath(slow_path);
1126
1127  __ Ldr(temp, HeapOperand(obj, count_offset));          // temp = str.length.
1128  codegen_->MaybeRecordImplicitNullCheck(invoke);
1129  __ Cmp(idx, temp);
1130  __ B(hs, slow_path->GetEntryLabel());
1131
1132  __ Add(array_temp, obj, Operand(value_offset.Int32Value()));  // array_temp := str.value.
1133
1134  // Load the value.
1135  __ Ldrh(out, MemOperand(array_temp.X(), idx, UXTW, 1));  // out := array_temp[idx].
1136
1137  __ Bind(slow_path->GetExitLabel());
1138}
1139
1140void IntrinsicLocationsBuilderARM64::VisitStringCompareTo(HInvoke* invoke) {
1141  LocationSummary* locations = new (arena_) LocationSummary(invoke,
1142                                                            LocationSummary::kCall,
1143                                                            kIntrinsified);
1144  InvokeRuntimeCallingConvention calling_convention;
1145  locations->SetInAt(0, LocationFrom(calling_convention.GetRegisterAt(0)));
1146  locations->SetInAt(1, LocationFrom(calling_convention.GetRegisterAt(1)));
1147  locations->SetOut(calling_convention.GetReturnLocation(Primitive::kPrimInt));
1148}
1149
1150void IntrinsicCodeGeneratorARM64::VisitStringCompareTo(HInvoke* invoke) {
1151  vixl::MacroAssembler* masm = GetVIXLAssembler();
1152  LocationSummary* locations = invoke->GetLocations();
1153
1154  // Note that the null check must have been done earlier.
1155  DCHECK(!invoke->CanDoImplicitNullCheckOn(invoke->InputAt(0)));
1156
1157  Register argument = WRegisterFrom(locations->InAt(1));
1158  __ Cmp(argument, 0);
1159  SlowPathCodeARM64* slow_path = new (GetAllocator()) IntrinsicSlowPathARM64(invoke);
1160  codegen_->AddSlowPath(slow_path);
1161  __ B(eq, slow_path->GetEntryLabel());
1162
1163  __ Ldr(
1164      lr, MemOperand(tr, QUICK_ENTRYPOINT_OFFSET(kArm64WordSize, pStringCompareTo).Int32Value()));
1165  __ Blr(lr);
1166  __ Bind(slow_path->GetExitLabel());
1167}
1168
1169void IntrinsicLocationsBuilderARM64::VisitStringEquals(HInvoke* invoke) {
1170  LocationSummary* locations = new (arena_) LocationSummary(invoke,
1171                                                            LocationSummary::kNoCall,
1172                                                            kIntrinsified);
1173  locations->SetInAt(0, Location::RequiresRegister());
1174  locations->SetInAt(1, Location::RequiresRegister());
1175  // Temporary registers to store lengths of strings and for calculations.
1176  locations->AddTemp(Location::RequiresRegister());
1177  locations->AddTemp(Location::RequiresRegister());
1178
1179  locations->SetOut(Location::RequiresRegister(), Location::kOutputOverlap);
1180}
1181
1182void IntrinsicCodeGeneratorARM64::VisitStringEquals(HInvoke* invoke) {
1183  vixl::MacroAssembler* masm = GetVIXLAssembler();
1184  LocationSummary* locations = invoke->GetLocations();
1185
1186  Register str = WRegisterFrom(locations->InAt(0));
1187  Register arg = WRegisterFrom(locations->InAt(1));
1188  Register out = XRegisterFrom(locations->Out());
1189
1190  UseScratchRegisterScope scratch_scope(masm);
1191  Register temp = scratch_scope.AcquireW();
1192  Register temp1 = WRegisterFrom(locations->GetTemp(0));
1193  Register temp2 = WRegisterFrom(locations->GetTemp(1));
1194
1195  vixl::Label loop;
1196  vixl::Label end;
1197  vixl::Label return_true;
1198  vixl::Label return_false;
1199
1200  // Get offsets of count, value, and class fields within a string object.
1201  const int32_t count_offset = mirror::String::CountOffset().Int32Value();
1202  const int32_t value_offset = mirror::String::ValueOffset().Int32Value();
1203  const int32_t class_offset = mirror::Object::ClassOffset().Int32Value();
1204
1205  // Note that the null check must have been done earlier.
1206  DCHECK(!invoke->CanDoImplicitNullCheckOn(invoke->InputAt(0)));
1207
1208  // Check if input is null, return false if it is.
1209  __ Cbz(arg, &return_false);
1210
1211  // Reference equality check, return true if same reference.
1212  __ Cmp(str, arg);
1213  __ B(&return_true, eq);
1214
1215  // Instanceof check for the argument by comparing class fields.
1216  // All string objects must have the same type since String cannot be subclassed.
1217  // Receiver must be a string object, so its class field is equal to all strings' class fields.
1218  // If the argument is a string object, its class field must be equal to receiver's class field.
1219  __ Ldr(temp, MemOperand(str.X(), class_offset));
1220  __ Ldr(temp1, MemOperand(arg.X(), class_offset));
1221  __ Cmp(temp, temp1);
1222  __ B(&return_false, ne);
1223
1224  // Load lengths of this and argument strings.
1225  __ Ldr(temp, MemOperand(str.X(), count_offset));
1226  __ Ldr(temp1, MemOperand(arg.X(), count_offset));
1227  // Check if lengths are equal, return false if they're not.
1228  __ Cmp(temp, temp1);
1229  __ B(&return_false, ne);
1230  // Store offset of string value in preparation for comparison loop
1231  __ Mov(temp1, value_offset);
1232  // Return true if both strings are empty.
1233  __ Cbz(temp, &return_true);
1234
1235  // Assertions that must hold in order to compare strings 4 characters at a time.
1236  DCHECK_ALIGNED(value_offset, 8);
1237  static_assert(IsAligned<8>(kObjectAlignment), "String of odd length is not zero padded");
1238
1239  temp1 = temp1.X();
1240  temp2 = temp2.X();
1241
1242  // Loop to compare strings 4 characters at a time starting at the beginning of the string.
1243  // Ok to do this because strings are zero-padded to be 8-byte aligned.
1244  __ Bind(&loop);
1245  __ Ldr(out, MemOperand(str.X(), temp1));
1246  __ Ldr(temp2, MemOperand(arg.X(), temp1));
1247  __ Add(temp1, temp1, Operand(sizeof(uint64_t)));
1248  __ Cmp(out, temp2);
1249  __ B(&return_false, ne);
1250  __ Sub(temp, temp, Operand(4), SetFlags);
1251  __ B(&loop, gt);
1252
1253  // Return true and exit the function.
1254  // If loop does not result in returning false, we return true.
1255  __ Bind(&return_true);
1256  __ Mov(out, 1);
1257  __ B(&end);
1258
1259  // Return false and exit the function.
1260  __ Bind(&return_false);
1261  __ Mov(out, 0);
1262  __ Bind(&end);
1263}
1264
1265static void GenerateVisitStringIndexOf(HInvoke* invoke,
1266                                       vixl::MacroAssembler* masm,
1267                                       CodeGeneratorARM64* codegen,
1268                                       ArenaAllocator* allocator,
1269                                       bool start_at_zero) {
1270  LocationSummary* locations = invoke->GetLocations();
1271  Register tmp_reg = WRegisterFrom(locations->GetTemp(0));
1272
1273  // Note that the null check must have been done earlier.
1274  DCHECK(!invoke->CanDoImplicitNullCheckOn(invoke->InputAt(0)));
1275
1276  // Check for code points > 0xFFFF. Either a slow-path check when we don't know statically,
1277  // or directly dispatch if we have a constant.
1278  SlowPathCodeARM64* slow_path = nullptr;
1279  if (invoke->InputAt(1)->IsIntConstant()) {
1280    if (static_cast<uint32_t>(invoke->InputAt(1)->AsIntConstant()->GetValue()) > 0xFFFFU) {
1281      // Always needs the slow-path. We could directly dispatch to it, but this case should be
1282      // rare, so for simplicity just put the full slow-path down and branch unconditionally.
1283      slow_path = new (allocator) IntrinsicSlowPathARM64(invoke);
1284      codegen->AddSlowPath(slow_path);
1285      __ B(slow_path->GetEntryLabel());
1286      __ Bind(slow_path->GetExitLabel());
1287      return;
1288    }
1289  } else {
1290    Register char_reg = WRegisterFrom(locations->InAt(1));
1291    __ Mov(tmp_reg, 0xFFFF);
1292    __ Cmp(char_reg, Operand(tmp_reg));
1293    slow_path = new (allocator) IntrinsicSlowPathARM64(invoke);
1294    codegen->AddSlowPath(slow_path);
1295    __ B(hi, slow_path->GetEntryLabel());
1296  }
1297
1298  if (start_at_zero) {
1299    // Start-index = 0.
1300    __ Mov(tmp_reg, 0);
1301  }
1302
1303  __ Ldr(lr, MemOperand(tr, QUICK_ENTRYPOINT_OFFSET(kArm64WordSize, pIndexOf).Int32Value()));
1304  CheckEntrypointTypes<kQuickIndexOf, int32_t, void*, uint32_t, uint32_t>();
1305  __ Blr(lr);
1306
1307  if (slow_path != nullptr) {
1308    __ Bind(slow_path->GetExitLabel());
1309  }
1310}
1311
1312void IntrinsicLocationsBuilderARM64::VisitStringIndexOf(HInvoke* invoke) {
1313  LocationSummary* locations = new (arena_) LocationSummary(invoke,
1314                                                            LocationSummary::kCall,
1315                                                            kIntrinsified);
1316  // We have a hand-crafted assembly stub that follows the runtime calling convention. So it's
1317  // best to align the inputs accordingly.
1318  InvokeRuntimeCallingConvention calling_convention;
1319  locations->SetInAt(0, LocationFrom(calling_convention.GetRegisterAt(0)));
1320  locations->SetInAt(1, LocationFrom(calling_convention.GetRegisterAt(1)));
1321  locations->SetOut(calling_convention.GetReturnLocation(Primitive::kPrimInt));
1322
1323  // Need a temp for slow-path codepoint compare, and need to send start_index=0.
1324  locations->AddTemp(LocationFrom(calling_convention.GetRegisterAt(2)));
1325}
1326
1327void IntrinsicCodeGeneratorARM64::VisitStringIndexOf(HInvoke* invoke) {
1328  GenerateVisitStringIndexOf(
1329      invoke, GetVIXLAssembler(), codegen_, GetAllocator(), /* start_at_zero */ true);
1330}
1331
1332void IntrinsicLocationsBuilderARM64::VisitStringIndexOfAfter(HInvoke* invoke) {
1333  LocationSummary* locations = new (arena_) LocationSummary(invoke,
1334                                                            LocationSummary::kCall,
1335                                                            kIntrinsified);
1336  // We have a hand-crafted assembly stub that follows the runtime calling convention. So it's
1337  // best to align the inputs accordingly.
1338  InvokeRuntimeCallingConvention calling_convention;
1339  locations->SetInAt(0, LocationFrom(calling_convention.GetRegisterAt(0)));
1340  locations->SetInAt(1, LocationFrom(calling_convention.GetRegisterAt(1)));
1341  locations->SetInAt(2, LocationFrom(calling_convention.GetRegisterAt(2)));
1342  locations->SetOut(calling_convention.GetReturnLocation(Primitive::kPrimInt));
1343
1344  // Need a temp for slow-path codepoint compare.
1345  locations->AddTemp(Location::RequiresRegister());
1346}
1347
1348void IntrinsicCodeGeneratorARM64::VisitStringIndexOfAfter(HInvoke* invoke) {
1349  GenerateVisitStringIndexOf(
1350      invoke, GetVIXLAssembler(), codegen_, GetAllocator(), /* start_at_zero */ false);
1351}
1352
1353void IntrinsicLocationsBuilderARM64::VisitStringNewStringFromBytes(HInvoke* invoke) {
1354  LocationSummary* locations = new (arena_) LocationSummary(invoke,
1355                                                            LocationSummary::kCall,
1356                                                            kIntrinsified);
1357  InvokeRuntimeCallingConvention calling_convention;
1358  locations->SetInAt(0, LocationFrom(calling_convention.GetRegisterAt(0)));
1359  locations->SetInAt(1, LocationFrom(calling_convention.GetRegisterAt(1)));
1360  locations->SetInAt(2, LocationFrom(calling_convention.GetRegisterAt(2)));
1361  locations->SetInAt(3, LocationFrom(calling_convention.GetRegisterAt(3)));
1362  locations->SetOut(calling_convention.GetReturnLocation(Primitive::kPrimNot));
1363}
1364
1365void IntrinsicCodeGeneratorARM64::VisitStringNewStringFromBytes(HInvoke* invoke) {
1366  vixl::MacroAssembler* masm = GetVIXLAssembler();
1367  LocationSummary* locations = invoke->GetLocations();
1368
1369  Register byte_array = WRegisterFrom(locations->InAt(0));
1370  __ Cmp(byte_array, 0);
1371  SlowPathCodeARM64* slow_path = new (GetAllocator()) IntrinsicSlowPathARM64(invoke);
1372  codegen_->AddSlowPath(slow_path);
1373  __ B(eq, slow_path->GetEntryLabel());
1374
1375  __ Ldr(lr,
1376      MemOperand(tr, QUICK_ENTRYPOINT_OFFSET(kArm64WordSize, pAllocStringFromBytes).Int32Value()));
1377  codegen_->RecordPcInfo(invoke, invoke->GetDexPc());
1378  __ Blr(lr);
1379  __ Bind(slow_path->GetExitLabel());
1380}
1381
1382void IntrinsicLocationsBuilderARM64::VisitStringNewStringFromChars(HInvoke* invoke) {
1383  LocationSummary* locations = new (arena_) LocationSummary(invoke,
1384                                                            LocationSummary::kCall,
1385                                                            kIntrinsified);
1386  InvokeRuntimeCallingConvention calling_convention;
1387  locations->SetInAt(0, LocationFrom(calling_convention.GetRegisterAt(0)));
1388  locations->SetInAt(1, LocationFrom(calling_convention.GetRegisterAt(1)));
1389  locations->SetInAt(2, LocationFrom(calling_convention.GetRegisterAt(2)));
1390  locations->SetOut(calling_convention.GetReturnLocation(Primitive::kPrimNot));
1391}
1392
1393void IntrinsicCodeGeneratorARM64::VisitStringNewStringFromChars(HInvoke* invoke) {
1394  vixl::MacroAssembler* masm = GetVIXLAssembler();
1395
1396  // No need to emit code checking whether `locations->InAt(2)` is a null
1397  // pointer, as callers of the native method
1398  //
1399  //   java.lang.StringFactory.newStringFromChars(int offset, int charCount, char[] data)
1400  //
1401  // all include a null check on `data` before calling that method.
1402  __ Ldr(lr,
1403      MemOperand(tr, QUICK_ENTRYPOINT_OFFSET(kArm64WordSize, pAllocStringFromChars).Int32Value()));
1404  codegen_->RecordPcInfo(invoke, invoke->GetDexPc());
1405  __ Blr(lr);
1406}
1407
1408void IntrinsicLocationsBuilderARM64::VisitStringNewStringFromString(HInvoke* invoke) {
1409  // The inputs plus one temp.
1410  LocationSummary* locations = new (arena_) LocationSummary(invoke,
1411                                                            LocationSummary::kCall,
1412                                                            kIntrinsified);
1413  InvokeRuntimeCallingConvention calling_convention;
1414  locations->SetInAt(0, LocationFrom(calling_convention.GetRegisterAt(0)));
1415  locations->SetInAt(1, LocationFrom(calling_convention.GetRegisterAt(1)));
1416  locations->SetInAt(2, LocationFrom(calling_convention.GetRegisterAt(2)));
1417  locations->SetOut(calling_convention.GetReturnLocation(Primitive::kPrimNot));
1418}
1419
1420void IntrinsicCodeGeneratorARM64::VisitStringNewStringFromString(HInvoke* invoke) {
1421  vixl::MacroAssembler* masm = GetVIXLAssembler();
1422  LocationSummary* locations = invoke->GetLocations();
1423
1424  Register string_to_copy = WRegisterFrom(locations->InAt(0));
1425  __ Cmp(string_to_copy, 0);
1426  SlowPathCodeARM64* slow_path = new (GetAllocator()) IntrinsicSlowPathARM64(invoke);
1427  codegen_->AddSlowPath(slow_path);
1428  __ B(eq, slow_path->GetEntryLabel());
1429
1430  __ Ldr(lr,
1431      MemOperand(tr, QUICK_ENTRYPOINT_OFFSET(kArm64WordSize, pAllocStringFromString).Int32Value()));
1432  codegen_->RecordPcInfo(invoke, invoke->GetDexPc());
1433  __ Blr(lr);
1434  __ Bind(slow_path->GetExitLabel());
1435}
1436
1437static void CreateFPToFPCallLocations(ArenaAllocator* arena, HInvoke* invoke) {
1438  DCHECK_EQ(invoke->GetNumberOfArguments(), 1U);
1439  DCHECK(Primitive::IsFloatingPointType(invoke->InputAt(0)->GetType()));
1440  DCHECK(Primitive::IsFloatingPointType(invoke->GetType()));
1441
1442  LocationSummary* const locations = new (arena) LocationSummary(invoke,
1443                                                                 LocationSummary::kCall,
1444                                                                 kIntrinsified);
1445  InvokeRuntimeCallingConvention calling_convention;
1446
1447  locations->SetInAt(0, LocationFrom(calling_convention.GetFpuRegisterAt(0)));
1448  locations->SetOut(calling_convention.GetReturnLocation(invoke->GetType()));
1449}
1450
1451static void CreateFPFPToFPCallLocations(ArenaAllocator* arena, HInvoke* invoke) {
1452  DCHECK_EQ(invoke->GetNumberOfArguments(), 2U);
1453  DCHECK(Primitive::IsFloatingPointType(invoke->InputAt(0)->GetType()));
1454  DCHECK(Primitive::IsFloatingPointType(invoke->InputAt(1)->GetType()));
1455  DCHECK(Primitive::IsFloatingPointType(invoke->GetType()));
1456
1457  LocationSummary* const locations = new (arena) LocationSummary(invoke,
1458                                                                 LocationSummary::kCall,
1459                                                                 kIntrinsified);
1460  InvokeRuntimeCallingConvention calling_convention;
1461
1462  locations->SetInAt(0, LocationFrom(calling_convention.GetFpuRegisterAt(0)));
1463  locations->SetInAt(1, LocationFrom(calling_convention.GetFpuRegisterAt(1)));
1464  locations->SetOut(calling_convention.GetReturnLocation(invoke->GetType()));
1465}
1466
1467static void GenFPToFPCall(HInvoke* invoke,
1468                          vixl::MacroAssembler* masm,
1469                          CodeGeneratorARM64* codegen,
1470                          QuickEntrypointEnum entry) {
1471  __ Ldr(lr, MemOperand(tr, GetThreadOffset<kArm64WordSize>(entry).Int32Value()));
1472  __ Blr(lr);
1473  codegen->RecordPcInfo(invoke, invoke->GetDexPc());
1474}
1475
1476void IntrinsicLocationsBuilderARM64::VisitMathCos(HInvoke* invoke) {
1477  CreateFPToFPCallLocations(arena_, invoke);
1478}
1479
1480void IntrinsicCodeGeneratorARM64::VisitMathCos(HInvoke* invoke) {
1481  GenFPToFPCall(invoke, GetVIXLAssembler(), codegen_, kQuickCos);
1482}
1483
1484void IntrinsicLocationsBuilderARM64::VisitMathSin(HInvoke* invoke) {
1485  CreateFPToFPCallLocations(arena_, invoke);
1486}
1487
1488void IntrinsicCodeGeneratorARM64::VisitMathSin(HInvoke* invoke) {
1489  GenFPToFPCall(invoke, GetVIXLAssembler(), codegen_, kQuickSin);
1490}
1491
1492void IntrinsicLocationsBuilderARM64::VisitMathAcos(HInvoke* invoke) {
1493  CreateFPToFPCallLocations(arena_, invoke);
1494}
1495
1496void IntrinsicCodeGeneratorARM64::VisitMathAcos(HInvoke* invoke) {
1497  GenFPToFPCall(invoke, GetVIXLAssembler(), codegen_, kQuickAcos);
1498}
1499
1500void IntrinsicLocationsBuilderARM64::VisitMathAsin(HInvoke* invoke) {
1501  CreateFPToFPCallLocations(arena_, invoke);
1502}
1503
1504void IntrinsicCodeGeneratorARM64::VisitMathAsin(HInvoke* invoke) {
1505  GenFPToFPCall(invoke, GetVIXLAssembler(), codegen_, kQuickAsin);
1506}
1507
1508void IntrinsicLocationsBuilderARM64::VisitMathAtan(HInvoke* invoke) {
1509  CreateFPToFPCallLocations(arena_, invoke);
1510}
1511
1512void IntrinsicCodeGeneratorARM64::VisitMathAtan(HInvoke* invoke) {
1513  GenFPToFPCall(invoke, GetVIXLAssembler(), codegen_, kQuickAtan);
1514}
1515
1516void IntrinsicLocationsBuilderARM64::VisitMathCbrt(HInvoke* invoke) {
1517  CreateFPToFPCallLocations(arena_, invoke);
1518}
1519
1520void IntrinsicCodeGeneratorARM64::VisitMathCbrt(HInvoke* invoke) {
1521  GenFPToFPCall(invoke, GetVIXLAssembler(), codegen_, kQuickCbrt);
1522}
1523
1524void IntrinsicLocationsBuilderARM64::VisitMathCosh(HInvoke* invoke) {
1525  CreateFPToFPCallLocations(arena_, invoke);
1526}
1527
1528void IntrinsicCodeGeneratorARM64::VisitMathCosh(HInvoke* invoke) {
1529  GenFPToFPCall(invoke, GetVIXLAssembler(), codegen_, kQuickCosh);
1530}
1531
1532void IntrinsicLocationsBuilderARM64::VisitMathExp(HInvoke* invoke) {
1533  CreateFPToFPCallLocations(arena_, invoke);
1534}
1535
1536void IntrinsicCodeGeneratorARM64::VisitMathExp(HInvoke* invoke) {
1537  GenFPToFPCall(invoke, GetVIXLAssembler(), codegen_, kQuickExp);
1538}
1539
1540void IntrinsicLocationsBuilderARM64::VisitMathExpm1(HInvoke* invoke) {
1541  CreateFPToFPCallLocations(arena_, invoke);
1542}
1543
1544void IntrinsicCodeGeneratorARM64::VisitMathExpm1(HInvoke* invoke) {
1545  GenFPToFPCall(invoke, GetVIXLAssembler(), codegen_, kQuickExpm1);
1546}
1547
1548void IntrinsicLocationsBuilderARM64::VisitMathLog(HInvoke* invoke) {
1549  CreateFPToFPCallLocations(arena_, invoke);
1550}
1551
1552void IntrinsicCodeGeneratorARM64::VisitMathLog(HInvoke* invoke) {
1553  GenFPToFPCall(invoke, GetVIXLAssembler(), codegen_, kQuickLog);
1554}
1555
1556void IntrinsicLocationsBuilderARM64::VisitMathLog10(HInvoke* invoke) {
1557  CreateFPToFPCallLocations(arena_, invoke);
1558}
1559
1560void IntrinsicCodeGeneratorARM64::VisitMathLog10(HInvoke* invoke) {
1561  GenFPToFPCall(invoke, GetVIXLAssembler(), codegen_, kQuickLog10);
1562}
1563
1564void IntrinsicLocationsBuilderARM64::VisitMathSinh(HInvoke* invoke) {
1565  CreateFPToFPCallLocations(arena_, invoke);
1566}
1567
1568void IntrinsicCodeGeneratorARM64::VisitMathSinh(HInvoke* invoke) {
1569  GenFPToFPCall(invoke, GetVIXLAssembler(), codegen_, kQuickSinh);
1570}
1571
1572void IntrinsicLocationsBuilderARM64::VisitMathTan(HInvoke* invoke) {
1573  CreateFPToFPCallLocations(arena_, invoke);
1574}
1575
1576void IntrinsicCodeGeneratorARM64::VisitMathTan(HInvoke* invoke) {
1577  GenFPToFPCall(invoke, GetVIXLAssembler(), codegen_, kQuickTan);
1578}
1579
1580void IntrinsicLocationsBuilderARM64::VisitMathTanh(HInvoke* invoke) {
1581  CreateFPToFPCallLocations(arena_, invoke);
1582}
1583
1584void IntrinsicCodeGeneratorARM64::VisitMathTanh(HInvoke* invoke) {
1585  GenFPToFPCall(invoke, GetVIXLAssembler(), codegen_, kQuickTanh);
1586}
1587
1588void IntrinsicLocationsBuilderARM64::VisitMathAtan2(HInvoke* invoke) {
1589  CreateFPFPToFPCallLocations(arena_, invoke);
1590}
1591
1592void IntrinsicCodeGeneratorARM64::VisitMathAtan2(HInvoke* invoke) {
1593  GenFPToFPCall(invoke, GetVIXLAssembler(), codegen_, kQuickAtan2);
1594}
1595
1596void IntrinsicLocationsBuilderARM64::VisitMathHypot(HInvoke* invoke) {
1597  CreateFPFPToFPCallLocations(arena_, invoke);
1598}
1599
1600void IntrinsicCodeGeneratorARM64::VisitMathHypot(HInvoke* invoke) {
1601  GenFPToFPCall(invoke, GetVIXLAssembler(), codegen_, kQuickHypot);
1602}
1603
1604void IntrinsicLocationsBuilderARM64::VisitMathNextAfter(HInvoke* invoke) {
1605  CreateFPFPToFPCallLocations(arena_, invoke);
1606}
1607
1608void IntrinsicCodeGeneratorARM64::VisitMathNextAfter(HInvoke* invoke) {
1609  GenFPToFPCall(invoke, GetVIXLAssembler(), codegen_, kQuickNextAfter);
1610}
1611
1612void IntrinsicLocationsBuilderARM64::VisitStringGetCharsNoCheck(HInvoke* invoke) {
1613  LocationSummary* locations = new (arena_) LocationSummary(invoke,
1614                                                            LocationSummary::kNoCall,
1615                                                            kIntrinsified);
1616  locations->SetInAt(0, Location::RequiresRegister());
1617  locations->SetInAt(1, Location::RequiresRegister());
1618  locations->SetInAt(2, Location::RequiresRegister());
1619  locations->SetInAt(3, Location::RequiresRegister());
1620  locations->SetInAt(4, Location::RequiresRegister());
1621
1622  locations->AddTemp(Location::RequiresRegister());
1623  locations->AddTemp(Location::RequiresRegister());
1624}
1625
1626void IntrinsicCodeGeneratorARM64::VisitStringGetCharsNoCheck(HInvoke* invoke) {
1627  vixl::MacroAssembler* masm = GetVIXLAssembler();
1628  LocationSummary* locations = invoke->GetLocations();
1629
1630  // Check assumption that sizeof(Char) is 2 (used in scaling below).
1631  const size_t char_size = Primitive::ComponentSize(Primitive::kPrimChar);
1632  DCHECK_EQ(char_size, 2u);
1633
1634  // Location of data in char array buffer.
1635  const uint32_t data_offset = mirror::Array::DataOffset(char_size).Uint32Value();
1636
1637  // Location of char array data in string.
1638  const uint32_t value_offset = mirror::String::ValueOffset().Uint32Value();
1639
1640  // void getCharsNoCheck(int srcBegin, int srcEnd, char[] dst, int dstBegin);
1641  // Since getChars() calls getCharsNoCheck() - we use registers rather than constants.
1642  Register srcObj = XRegisterFrom(locations->InAt(0));
1643  Register srcBegin = XRegisterFrom(locations->InAt(1));
1644  Register srcEnd = XRegisterFrom(locations->InAt(2));
1645  Register dstObj = XRegisterFrom(locations->InAt(3));
1646  Register dstBegin = XRegisterFrom(locations->InAt(4));
1647
1648  Register src_ptr = XRegisterFrom(locations->GetTemp(0));
1649  Register src_ptr_end = XRegisterFrom(locations->GetTemp(1));
1650
1651  UseScratchRegisterScope temps(masm);
1652  Register dst_ptr = temps.AcquireX();
1653  Register tmp = temps.AcquireW();
1654
1655  // src range to copy.
1656  __ Add(src_ptr, srcObj, Operand(value_offset));
1657  __ Add(src_ptr_end, src_ptr, Operand(srcEnd, LSL, 1));
1658  __ Add(src_ptr, src_ptr, Operand(srcBegin, LSL, 1));
1659
1660  // dst to be copied.
1661  __ Add(dst_ptr, dstObj, Operand(data_offset));
1662  __ Add(dst_ptr, dst_ptr, Operand(dstBegin, LSL, 1));
1663
1664  // Do the copy.
1665  vixl::Label loop, done;
1666  __ Bind(&loop);
1667  __ Cmp(src_ptr, src_ptr_end);
1668  __ B(&done, eq);
1669  __ Ldrh(tmp, MemOperand(src_ptr, char_size, vixl::PostIndex));
1670  __ Strh(tmp, MemOperand(dst_ptr, char_size, vixl::PostIndex));
1671  __ B(&loop);
1672  __ Bind(&done);
1673}
1674
1675UNIMPLEMENTED_INTRINSIC(ARM64, IntegerBitCount)
1676UNIMPLEMENTED_INTRINSIC(ARM64, LongBitCount)
1677UNIMPLEMENTED_INTRINSIC(ARM64, SystemArrayCopyChar)
1678UNIMPLEMENTED_INTRINSIC(ARM64, SystemArrayCopy)
1679UNIMPLEMENTED_INTRINSIC(ARM64, ReferenceGetReferent)
1680UNIMPLEMENTED_INTRINSIC(ARM64, FloatIsInfinite)
1681UNIMPLEMENTED_INTRINSIC(ARM64, DoubleIsInfinite)
1682UNIMPLEMENTED_INTRINSIC(ARM64, IntegerHighestOneBit)
1683UNIMPLEMENTED_INTRINSIC(ARM64, LongHighestOneBit)
1684UNIMPLEMENTED_INTRINSIC(ARM64, IntegerLowestOneBit)
1685UNIMPLEMENTED_INTRINSIC(ARM64, LongLowestOneBit)
1686
1687UNREACHABLE_INTRINSICS(ARM64)
1688
1689#undef __
1690
1691}  // namespace arm64
1692}  // namespace art
1693