intrinsics_arm64.cc revision 40041c9a38e3961d8675d117517719458a115520 
1/*
2 * Copyright (C) 2015 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *      http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17#include "intrinsics_arm64.h"
18
19#include "arch/arm64/instruction_set_features_arm64.h"
20#include "art_method.h"
21#include "code_generator_arm64.h"
22#include "common_arm64.h"
23#include "entrypoints/quick/quick_entrypoints.h"
24#include "intrinsics.h"
25#include "mirror/array-inl.h"
26#include "mirror/string.h"
27#include "thread.h"
28#include "utils/arm64/assembler_arm64.h"
29#include "utils/arm64/constants_arm64.h"
30
31#include "vixl/a64/disasm-a64.h"
32#include "vixl/a64/macro-assembler-a64.h"
33
34using namespace vixl;   // NOLINT(build/namespaces)
35
36namespace art {
37
38namespace arm64 {
39
40using helpers::DRegisterFrom;
41using helpers::FPRegisterFrom;
42using helpers::HeapOperand;
43using helpers::LocationFrom;
44using helpers::OperandFrom;
45using helpers::RegisterFrom;
46using helpers::SRegisterFrom;
47using helpers::WRegisterFrom;
48using helpers::XRegisterFrom;
49
50namespace {
51
52ALWAYS_INLINE inline MemOperand AbsoluteHeapOperandFrom(Location location, size_t offset = 0) {
53  return MemOperand(XRegisterFrom(location), offset);
54}
55
56}  // namespace
57
58vixl::MacroAssembler* IntrinsicCodeGeneratorARM64::GetVIXLAssembler() {
59  return codegen_->GetAssembler()->vixl_masm_;
60}
61
62ArenaAllocator* IntrinsicCodeGeneratorARM64::GetAllocator() {
63  return codegen_->GetGraph()->GetArena();
64}
65
66#define __ codegen->GetAssembler()->vixl_masm_->
67
68static void MoveFromReturnRegister(Location trg,
69                                   Primitive::Type type,
70                                   CodeGeneratorARM64* codegen) {
71  if (!trg.IsValid()) {
72    DCHECK(type == Primitive::kPrimVoid);
73    return;
74  }
75
76  DCHECK_NE(type, Primitive::kPrimVoid);
77
78  if (Primitive::IsIntegralType(type) || type == Primitive::kPrimNot) {
79    Register trg_reg = RegisterFrom(trg, type);
80    Register res_reg = RegisterFrom(ARM64ReturnLocation(type), type);
81    __ Mov(trg_reg, res_reg, kDiscardForSameWReg);
82  } else {
83    FPRegister trg_reg = FPRegisterFrom(trg, type);
84    FPRegister res_reg = FPRegisterFrom(ARM64ReturnLocation(type), type);
85    __ Fmov(trg_reg, res_reg);
86  }
87}
88
89static void MoveArguments(HInvoke* invoke, CodeGeneratorARM64* codegen) {
90  InvokeDexCallingConventionVisitorARM64 calling_convention_visitor;
91  IntrinsicVisitor::MoveArguments(invoke, codegen, &calling_convention_visitor);
92}
93
94// Slow-path for fallback (calling the managed code to handle the intrinsic) in an intrinsified
95// call. This will copy the arguments into the positions for a regular call.
96//
97// Note: The actual parameters are required to be in the locations given by the invoke's location
98//       summary. If an intrinsic modifies those locations before a slowpath call, they must be
99//       restored!
100class IntrinsicSlowPathARM64 : public SlowPathCodeARM64 {
101 public:
102  explicit IntrinsicSlowPathARM64(HInvoke* invoke) : invoke_(invoke) { }
103
104  void EmitNativeCode(CodeGenerator* codegen_in) OVERRIDE {
105    CodeGeneratorARM64* codegen = down_cast<CodeGeneratorARM64*>(codegen_in);
106    __ Bind(GetEntryLabel());
107
108    SaveLiveRegisters(codegen, invoke_->GetLocations());
109
110    MoveArguments(invoke_, codegen);
111
112    if (invoke_->IsInvokeStaticOrDirect()) {
113      codegen->GenerateStaticOrDirectCall(invoke_->AsInvokeStaticOrDirect(),
114                                          LocationFrom(kArtMethodRegister));
115    } else {
116      codegen->GenerateVirtualCall(invoke_->AsInvokeVirtual(), LocationFrom(kArtMethodRegister));
117    }
118    codegen->RecordPcInfo(invoke_, invoke_->GetDexPc(), this);
119
120    // Copy the result back to the expected output.
121    Location out = invoke_->GetLocations()->Out();
122    if (out.IsValid()) {
123      DCHECK(out.IsRegister());  // TODO: Replace this when we support output in memory.
124      DCHECK(!invoke_->GetLocations()->GetLiveRegisters()->ContainsCoreRegister(out.reg()));
125      MoveFromReturnRegister(out, invoke_->GetType(), codegen);
126    }
127
128    RestoreLiveRegisters(codegen, invoke_->GetLocations());
129    __ B(GetExitLabel());
130  }
131
132  const char* GetDescription() const OVERRIDE { return "IntrinsicSlowPathARM64"; }
133
134 private:
135  // The instruction where this slow path is happening.
136  HInvoke* const invoke_;
137
138  DISALLOW_COPY_AND_ASSIGN(IntrinsicSlowPathARM64);
139};
140
141#undef __
142
143bool IntrinsicLocationsBuilderARM64::TryDispatch(HInvoke* invoke) {
144  Dispatch(invoke);
145  LocationSummary* res = invoke->GetLocations();
146  if (res == nullptr) {
147    return false;
148  }
149  if (kEmitCompilerReadBarrier && res->CanCall()) {
150    // Generating an intrinsic for this HInvoke may produce an
151    // IntrinsicSlowPathARM64 slow path.  Currently this approach
152    // does not work when using read barriers, as the emitted
153    // calling sequence will make use of another slow path
154    // (ReadBarrierForRootSlowPathARM64 for HInvokeStaticOrDirect,
155    // ReadBarrierSlowPathARM64 for HInvokeVirtual).  So we bail
156    // out in this case.
157    //
158    // TODO: Find a way to have intrinsics work with read barriers.
159    invoke->SetLocations(nullptr);
160    return false;
161  }
162  return res->Intrinsified();
163}
164
165#define __ masm->
166
167static void CreateFPToIntLocations(ArenaAllocator* arena, HInvoke* invoke) {
168  LocationSummary* locations = new (arena) LocationSummary(invoke,
169                                                           LocationSummary::kNoCall,
170                                                           kIntrinsified);
171  locations->SetInAt(0, Location::RequiresFpuRegister());
172  locations->SetOut(Location::RequiresRegister());
173}
174
175static void CreateIntToFPLocations(ArenaAllocator* arena, HInvoke* invoke) {
176  LocationSummary* locations = new (arena) LocationSummary(invoke,
177                                                           LocationSummary::kNoCall,
178                                                           kIntrinsified);
179  locations->SetInAt(0, Location::RequiresRegister());
180  locations->SetOut(Location::RequiresFpuRegister());
181}
182
183static void MoveFPToInt(LocationSummary* locations, bool is64bit, vixl::MacroAssembler* masm) {
184  Location input = locations->InAt(0);
185  Location output = locations->Out();
186  __ Fmov(is64bit ? XRegisterFrom(output) : WRegisterFrom(output),
187          is64bit ? DRegisterFrom(input) : SRegisterFrom(input));
188}
189
190static void MoveIntToFP(LocationSummary* locations, bool is64bit, vixl::MacroAssembler* masm) {
191  Location input = locations->InAt(0);
192  Location output = locations->Out();
193  __ Fmov(is64bit ? DRegisterFrom(output) : SRegisterFrom(output),
194          is64bit ? XRegisterFrom(input) : WRegisterFrom(input));
195}
196
197void IntrinsicLocationsBuilderARM64::VisitDoubleDoubleToRawLongBits(HInvoke* invoke) {
198  CreateFPToIntLocations(arena_, invoke);
199}
200void IntrinsicLocationsBuilderARM64::VisitDoubleLongBitsToDouble(HInvoke* invoke) {
201  CreateIntToFPLocations(arena_, invoke);
202}
203
204void IntrinsicCodeGeneratorARM64::VisitDoubleDoubleToRawLongBits(HInvoke* invoke) {
205  MoveFPToInt(invoke->GetLocations(), /* is64bit */ true, GetVIXLAssembler());
206}
207void IntrinsicCodeGeneratorARM64::VisitDoubleLongBitsToDouble(HInvoke* invoke) {
208  MoveIntToFP(invoke->GetLocations(), /* is64bit */ true, GetVIXLAssembler());
209}
210
211void IntrinsicLocationsBuilderARM64::VisitFloatFloatToRawIntBits(HInvoke* invoke) {
212  CreateFPToIntLocations(arena_, invoke);
213}
214void IntrinsicLocationsBuilderARM64::VisitFloatIntBitsToFloat(HInvoke* invoke) {
215  CreateIntToFPLocations(arena_, invoke);
216}
217
218void IntrinsicCodeGeneratorARM64::VisitFloatFloatToRawIntBits(HInvoke* invoke) {
219  MoveFPToInt(invoke->GetLocations(), /* is64bit */ false, GetVIXLAssembler());
220}
221void IntrinsicCodeGeneratorARM64::VisitFloatIntBitsToFloat(HInvoke* invoke) {
222  MoveIntToFP(invoke->GetLocations(), /* is64bit */ false, GetVIXLAssembler());
223}
224
225static void CreateIntToIntLocations(ArenaAllocator* arena, HInvoke* invoke) {
226  LocationSummary* locations = new (arena) LocationSummary(invoke,
227                                                           LocationSummary::kNoCall,
228                                                           kIntrinsified);
229  locations->SetInAt(0, Location::RequiresRegister());
230  locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
231}
232
233static void GenReverseBytes(LocationSummary* locations,
234                            Primitive::Type type,
235                            vixl::MacroAssembler* masm) {
236  Location in = locations->InAt(0);
237  Location out = locations->Out();
238
239  switch (type) {
240    case Primitive::kPrimShort:
241      __ Rev16(WRegisterFrom(out), WRegisterFrom(in));
242      __ Sxth(WRegisterFrom(out), WRegisterFrom(out));
243      break;
244    case Primitive::kPrimInt:
245    case Primitive::kPrimLong:
246      __ Rev(RegisterFrom(out, type), RegisterFrom(in, type));
247      break;
248    default:
249      LOG(FATAL) << "Unexpected size for reverse-bytes: " << type;
250      UNREACHABLE();
251  }
252}
253
254void IntrinsicLocationsBuilderARM64::VisitIntegerReverseBytes(HInvoke* invoke) {
255  CreateIntToIntLocations(arena_, invoke);
256}
257
258void IntrinsicCodeGeneratorARM64::VisitIntegerReverseBytes(HInvoke* invoke) {
259  GenReverseBytes(invoke->GetLocations(), Primitive::kPrimInt, GetVIXLAssembler());
260}
261
262void IntrinsicLocationsBuilderARM64::VisitLongReverseBytes(HInvoke* invoke) {
263  CreateIntToIntLocations(arena_, invoke);
264}
265
266void IntrinsicCodeGeneratorARM64::VisitLongReverseBytes(HInvoke* invoke) {
267  GenReverseBytes(invoke->GetLocations(), Primitive::kPrimLong, GetVIXLAssembler());
268}
269
270void IntrinsicLocationsBuilderARM64::VisitShortReverseBytes(HInvoke* invoke) {
271  CreateIntToIntLocations(arena_, invoke);
272}
273
274void IntrinsicCodeGeneratorARM64::VisitShortReverseBytes(HInvoke* invoke) {
275  GenReverseBytes(invoke->GetLocations(), Primitive::kPrimShort, GetVIXLAssembler());
276}
277
278static void GenNumberOfLeadingZeros(LocationSummary* locations,
279                                    Primitive::Type type,
280                                    vixl::MacroAssembler* masm) {
281  DCHECK(type == Primitive::kPrimInt || type == Primitive::kPrimLong);
282
283  Location in = locations->InAt(0);
284  Location out = locations->Out();
285
286  __ Clz(RegisterFrom(out, type), RegisterFrom(in, type));
287}
288
289void IntrinsicLocationsBuilderARM64::VisitIntegerNumberOfLeadingZeros(HInvoke* invoke) {
290  CreateIntToIntLocations(arena_, invoke);
291}
292
293void IntrinsicCodeGeneratorARM64::VisitIntegerNumberOfLeadingZeros(HInvoke* invoke) {
294  GenNumberOfLeadingZeros(invoke->GetLocations(), Primitive::kPrimInt, GetVIXLAssembler());
295}
296
297void IntrinsicLocationsBuilderARM64::VisitLongNumberOfLeadingZeros(HInvoke* invoke) {
298  CreateIntToIntLocations(arena_, invoke);
299}
300
301void IntrinsicCodeGeneratorARM64::VisitLongNumberOfLeadingZeros(HInvoke* invoke) {
302  GenNumberOfLeadingZeros(invoke->GetLocations(), Primitive::kPrimLong, GetVIXLAssembler());
303}
304
305static void GenNumberOfTrailingZeros(LocationSummary* locations,
306                                     Primitive::Type type,
307                                     vixl::MacroAssembler* masm) {
308  DCHECK(type == Primitive::kPrimInt || type == Primitive::kPrimLong);
309
310  Location in = locations->InAt(0);
311  Location out = locations->Out();
312
313  __ Rbit(RegisterFrom(out, type), RegisterFrom(in, type));
314  __ Clz(RegisterFrom(out, type), RegisterFrom(out, type));
315}
316
317void IntrinsicLocationsBuilderARM64::VisitIntegerNumberOfTrailingZeros(HInvoke* invoke) {
318  CreateIntToIntLocations(arena_, invoke);
319}
320
321void IntrinsicCodeGeneratorARM64::VisitIntegerNumberOfTrailingZeros(HInvoke* invoke) {
322  GenNumberOfTrailingZeros(invoke->GetLocations(), Primitive::kPrimInt, GetVIXLAssembler());
323}
324
325void IntrinsicLocationsBuilderARM64::VisitLongNumberOfTrailingZeros(HInvoke* invoke) {
326  CreateIntToIntLocations(arena_, invoke);
327}
328
329void IntrinsicCodeGeneratorARM64::VisitLongNumberOfTrailingZeros(HInvoke* invoke) {
330  GenNumberOfTrailingZeros(invoke->GetLocations(), Primitive::kPrimLong, GetVIXLAssembler());
331}
332
333static void GenReverse(LocationSummary* locations,
334                       Primitive::Type type,
335                       vixl::MacroAssembler* masm) {
336  DCHECK(type == Primitive::kPrimInt || type == Primitive::kPrimLong);
337
338  Location in = locations->InAt(0);
339  Location out = locations->Out();
340
341  __ Rbit(RegisterFrom(out, type), RegisterFrom(in, type));
342}
343
344void IntrinsicLocationsBuilderARM64::VisitIntegerReverse(HInvoke* invoke) {
345  CreateIntToIntLocations(arena_, invoke);
346}
347
348void IntrinsicCodeGeneratorARM64::VisitIntegerReverse(HInvoke* invoke) {
349  GenReverse(invoke->GetLocations(), Primitive::kPrimInt, GetVIXLAssembler());
350}
351
352void IntrinsicLocationsBuilderARM64::VisitLongReverse(HInvoke* invoke) {
353  CreateIntToIntLocations(arena_, invoke);
354}
355
356void IntrinsicCodeGeneratorARM64::VisitLongReverse(HInvoke* invoke) {
357  GenReverse(invoke->GetLocations(), Primitive::kPrimLong, GetVIXLAssembler());
358}
359
360static void CreateFPToFPLocations(ArenaAllocator* arena, HInvoke* invoke) {
361  LocationSummary* locations = new (arena) LocationSummary(invoke,
362                                                           LocationSummary::kNoCall,
363                                                           kIntrinsified);
364  locations->SetInAt(0, Location::RequiresFpuRegister());
365  locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap);
366}
367
368static void MathAbsFP(LocationSummary* locations, bool is64bit, vixl::MacroAssembler* masm) {
369  Location in = locations->InAt(0);
370  Location out = locations->Out();
371
372  FPRegister in_reg = is64bit ? DRegisterFrom(in) : SRegisterFrom(in);
373  FPRegister out_reg = is64bit ? DRegisterFrom(out) : SRegisterFrom(out);
374
375  __ Fabs(out_reg, in_reg);
376}
377
378void IntrinsicLocationsBuilderARM64::VisitMathAbsDouble(HInvoke* invoke) {
379  CreateFPToFPLocations(arena_, invoke);
380}
381
382void IntrinsicCodeGeneratorARM64::VisitMathAbsDouble(HInvoke* invoke) {
383  MathAbsFP(invoke->GetLocations(), /* is64bit */ true, GetVIXLAssembler());
384}
385
386void IntrinsicLocationsBuilderARM64::VisitMathAbsFloat(HInvoke* invoke) {
387  CreateFPToFPLocations(arena_, invoke);
388}
389
390void IntrinsicCodeGeneratorARM64::VisitMathAbsFloat(HInvoke* invoke) {
391  MathAbsFP(invoke->GetLocations(), /* is64bit */ false, GetVIXLAssembler());
392}
393
394static void CreateIntToInt(ArenaAllocator* arena, HInvoke* invoke) {
395  LocationSummary* locations = new (arena) LocationSummary(invoke,
396                                                           LocationSummary::kNoCall,
397                                                           kIntrinsified);
398  locations->SetInAt(0, Location::RequiresRegister());
399  locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
400}
401
402static void GenAbsInteger(LocationSummary* locations,
403                          bool is64bit,
404                          vixl::MacroAssembler* masm) {
405  Location in = locations->InAt(0);
406  Location output = locations->Out();
407
408  Register in_reg = is64bit ? XRegisterFrom(in) : WRegisterFrom(in);
409  Register out_reg = is64bit ? XRegisterFrom(output) : WRegisterFrom(output);
410
411  __ Cmp(in_reg, Operand(0));
412  __ Cneg(out_reg, in_reg, lt);
413}
414
415void IntrinsicLocationsBuilderARM64::VisitMathAbsInt(HInvoke* invoke) {
416  CreateIntToInt(arena_, invoke);
417}
418
419void IntrinsicCodeGeneratorARM64::VisitMathAbsInt(HInvoke* invoke) {
420  GenAbsInteger(invoke->GetLocations(), /* is64bit */ false, GetVIXLAssembler());
421}
422
423void IntrinsicLocationsBuilderARM64::VisitMathAbsLong(HInvoke* invoke) {
424  CreateIntToInt(arena_, invoke);
425}
426
427void IntrinsicCodeGeneratorARM64::VisitMathAbsLong(HInvoke* invoke) {
428  GenAbsInteger(invoke->GetLocations(), /* is64bit */ true, GetVIXLAssembler());
429}
430
431static void GenMinMaxFP(LocationSummary* locations,
432                        bool is_min,
433                        bool is_double,
434                        vixl::MacroAssembler* masm) {
435  Location op1 = locations->InAt(0);
436  Location op2 = locations->InAt(1);
437  Location out = locations->Out();
438
439  FPRegister op1_reg = is_double ? DRegisterFrom(op1) : SRegisterFrom(op1);
440  FPRegister op2_reg = is_double ? DRegisterFrom(op2) : SRegisterFrom(op2);
441  FPRegister out_reg = is_double ? DRegisterFrom(out) : SRegisterFrom(out);
442  if (is_min) {
443    __ Fmin(out_reg, op1_reg, op2_reg);
444  } else {
445    __ Fmax(out_reg, op1_reg, op2_reg);
446  }
447}
448
449static void CreateFPFPToFPLocations(ArenaAllocator* arena, HInvoke* invoke) {
450  LocationSummary* locations = new (arena) LocationSummary(invoke,
451                                                           LocationSummary::kNoCall,
452                                                           kIntrinsified);
453  locations->SetInAt(0, Location::RequiresFpuRegister());
454  locations->SetInAt(1, Location::RequiresFpuRegister());
455  locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap);
456}
457
458void IntrinsicLocationsBuilderARM64::VisitMathMinDoubleDouble(HInvoke* invoke) {
459  CreateFPFPToFPLocations(arena_, invoke);
460}
461
462void IntrinsicCodeGeneratorARM64::VisitMathMinDoubleDouble(HInvoke* invoke) {
463  GenMinMaxFP(invoke->GetLocations(), /* is_min */ true, /* is_double */ true, GetVIXLAssembler());
464}
465
466void IntrinsicLocationsBuilderARM64::VisitMathMinFloatFloat(HInvoke* invoke) {
467  CreateFPFPToFPLocations(arena_, invoke);
468}
469
470void IntrinsicCodeGeneratorARM64::VisitMathMinFloatFloat(HInvoke* invoke) {
471  GenMinMaxFP(invoke->GetLocations(), /* is_min */ true, /* is_double */ false, GetVIXLAssembler());
472}
473
474void IntrinsicLocationsBuilderARM64::VisitMathMaxDoubleDouble(HInvoke* invoke) {
475  CreateFPFPToFPLocations(arena_, invoke);
476}
477
478void IntrinsicCodeGeneratorARM64::VisitMathMaxDoubleDouble(HInvoke* invoke) {
479  GenMinMaxFP(invoke->GetLocations(), /* is_min */ false, /* is_double */ true, GetVIXLAssembler());
480}
481
482void IntrinsicLocationsBuilderARM64::VisitMathMaxFloatFloat(HInvoke* invoke) {
483  CreateFPFPToFPLocations(arena_, invoke);
484}
485
486void IntrinsicCodeGeneratorARM64::VisitMathMaxFloatFloat(HInvoke* invoke) {
487  GenMinMaxFP(
488      invoke->GetLocations(), /* is_min */ false, /* is_double */ false, GetVIXLAssembler());
489}
490
491static void GenMinMax(LocationSummary* locations,
492                      bool is_min,
493                      bool is_long,
494                      vixl::MacroAssembler* masm) {
495  Location op1 = locations->InAt(0);
496  Location op2 = locations->InAt(1);
497  Location out = locations->Out();
498
499  Register op1_reg = is_long ? XRegisterFrom(op1) : WRegisterFrom(op1);
500  Register op2_reg = is_long ? XRegisterFrom(op2) : WRegisterFrom(op2);
501  Register out_reg = is_long ? XRegisterFrom(out) : WRegisterFrom(out);
502
503  __ Cmp(op1_reg, op2_reg);
504  __ Csel(out_reg, op1_reg, op2_reg, is_min ? lt : gt);
505}
506
507static void CreateIntIntToIntLocations(ArenaAllocator* arena, HInvoke* invoke) {
508  LocationSummary* locations = new (arena) LocationSummary(invoke,
509                                                           LocationSummary::kNoCall,
510                                                           kIntrinsified);
511  locations->SetInAt(0, Location::RequiresRegister());
512  locations->SetInAt(1, Location::RequiresRegister());
513  locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
514}
515
516void IntrinsicLocationsBuilderARM64::VisitMathMinIntInt(HInvoke* invoke) {
517  CreateIntIntToIntLocations(arena_, invoke);
518}
519
520void IntrinsicCodeGeneratorARM64::VisitMathMinIntInt(HInvoke* invoke) {
521  GenMinMax(invoke->GetLocations(), /* is_min */ true, /* is_long */ false, GetVIXLAssembler());
522}
523
524void IntrinsicLocationsBuilderARM64::VisitMathMinLongLong(HInvoke* invoke) {
525  CreateIntIntToIntLocations(arena_, invoke);
526}
527
528void IntrinsicCodeGeneratorARM64::VisitMathMinLongLong(HInvoke* invoke) {
529  GenMinMax(invoke->GetLocations(), /* is_min */ true, /* is_long */ true, GetVIXLAssembler());
530}
531
532void IntrinsicLocationsBuilderARM64::VisitMathMaxIntInt(HInvoke* invoke) {
533  CreateIntIntToIntLocations(arena_, invoke);
534}
535
536void IntrinsicCodeGeneratorARM64::VisitMathMaxIntInt(HInvoke* invoke) {
537  GenMinMax(invoke->GetLocations(), /* is_min */ false, /* is_long */ false, GetVIXLAssembler());
538}
539
540void IntrinsicLocationsBuilderARM64::VisitMathMaxLongLong(HInvoke* invoke) {
541  CreateIntIntToIntLocations(arena_, invoke);
542}
543
544void IntrinsicCodeGeneratorARM64::VisitMathMaxLongLong(HInvoke* invoke) {
545  GenMinMax(invoke->GetLocations(), /* is_min */ false, /* is_long */ true, GetVIXLAssembler());
546}
547
548void IntrinsicLocationsBuilderARM64::VisitMathSqrt(HInvoke* invoke) {
549  CreateFPToFPLocations(arena_, invoke);
550}
551
552void IntrinsicCodeGeneratorARM64::VisitMathSqrt(HInvoke* invoke) {
553  LocationSummary* locations = invoke->GetLocations();
554  vixl::MacroAssembler* masm = GetVIXLAssembler();
555  __ Fsqrt(DRegisterFrom(locations->Out()), DRegisterFrom(locations->InAt(0)));
556}
557
558void IntrinsicLocationsBuilderARM64::VisitMathCeil(HInvoke* invoke) {
559  CreateFPToFPLocations(arena_, invoke);
560}
561
562void IntrinsicCodeGeneratorARM64::VisitMathCeil(HInvoke* invoke) {
563  LocationSummary* locations = invoke->GetLocations();
564  vixl::MacroAssembler* masm = GetVIXLAssembler();
565  __ Frintp(DRegisterFrom(locations->Out()), DRegisterFrom(locations->InAt(0)));
566}
567
568void IntrinsicLocationsBuilderARM64::VisitMathFloor(HInvoke* invoke) {
569  CreateFPToFPLocations(arena_, invoke);
570}
571
572void IntrinsicCodeGeneratorARM64::VisitMathFloor(HInvoke* invoke) {
573  LocationSummary* locations = invoke->GetLocations();
574  vixl::MacroAssembler* masm = GetVIXLAssembler();
575  __ Frintm(DRegisterFrom(locations->Out()), DRegisterFrom(locations->InAt(0)));
576}
577
578void IntrinsicLocationsBuilderARM64::VisitMathRint(HInvoke* invoke) {
579  CreateFPToFPLocations(arena_, invoke);
580}
581
582void IntrinsicCodeGeneratorARM64::VisitMathRint(HInvoke* invoke) {
583  LocationSummary* locations = invoke->GetLocations();
584  vixl::MacroAssembler* masm = GetVIXLAssembler();
585  __ Frintn(DRegisterFrom(locations->Out()), DRegisterFrom(locations->InAt(0)));
586}
587
588static void CreateFPToIntPlusTempLocations(ArenaAllocator* arena, HInvoke* invoke) {
589  LocationSummary* locations = new (arena) LocationSummary(invoke,
590                                                           LocationSummary::kNoCall,
591                                                           kIntrinsified);
592  locations->SetInAt(0, Location::RequiresFpuRegister());
593  locations->SetOut(Location::RequiresRegister());
594}
595
596static void GenMathRound(LocationSummary* locations,
597                         bool is_double,
598                         vixl::MacroAssembler* masm) {
599  FPRegister in_reg = is_double ?
600      DRegisterFrom(locations->InAt(0)) : SRegisterFrom(locations->InAt(0));
601  Register out_reg = is_double ?
602      XRegisterFrom(locations->Out()) : WRegisterFrom(locations->Out());
603  UseScratchRegisterScope temps(masm);
604  FPRegister temp1_reg = temps.AcquireSameSizeAs(in_reg);
605
606  // 0.5 can be encoded as an immediate, so use fmov.
607  if (is_double) {
608    __ Fmov(temp1_reg, static_cast<double>(0.5));
609  } else {
610    __ Fmov(temp1_reg, static_cast<float>(0.5));
611  }
612  __ Fadd(temp1_reg, in_reg, temp1_reg);
613  __ Fcvtms(out_reg, temp1_reg);
614}
615
616void IntrinsicLocationsBuilderARM64::VisitMathRoundDouble(HInvoke* invoke) {
617  if ((true)) {
618    // TODO(26327751): Re-enable?
619    return;
620  }
621  CreateFPToIntPlusTempLocations(arena_, invoke);
622}
623
624void IntrinsicCodeGeneratorARM64::VisitMathRoundDouble(HInvoke* invoke) {
625  GenMathRound(invoke->GetLocations(), /* is_double */ true, GetVIXLAssembler());
626}
627
628void IntrinsicLocationsBuilderARM64::VisitMathRoundFloat(HInvoke* invoke) {
629  if ((true)) {
630    // TODO(26327751): Re-enable?
631    return;
632  }
633  CreateFPToIntPlusTempLocations(arena_, invoke);
634}
635
636void IntrinsicCodeGeneratorARM64::VisitMathRoundFloat(HInvoke* invoke) {
637  GenMathRound(invoke->GetLocations(), /* is_double */ false, GetVIXLAssembler());
638}
639
640void IntrinsicLocationsBuilderARM64::VisitMemoryPeekByte(HInvoke* invoke) {
641  CreateIntToIntLocations(arena_, invoke);
642}
643
644void IntrinsicCodeGeneratorARM64::VisitMemoryPeekByte(HInvoke* invoke) {
645  vixl::MacroAssembler* masm = GetVIXLAssembler();
646  __ Ldrsb(WRegisterFrom(invoke->GetLocations()->Out()),
647          AbsoluteHeapOperandFrom(invoke->GetLocations()->InAt(0), 0));
648}
649
650void IntrinsicLocationsBuilderARM64::VisitMemoryPeekIntNative(HInvoke* invoke) {
651  CreateIntToIntLocations(arena_, invoke);
652}
653
654void IntrinsicCodeGeneratorARM64::VisitMemoryPeekIntNative(HInvoke* invoke) {
655  vixl::MacroAssembler* masm = GetVIXLAssembler();
656  __ Ldr(WRegisterFrom(invoke->GetLocations()->Out()),
657         AbsoluteHeapOperandFrom(invoke->GetLocations()->InAt(0), 0));
658}
659
660void IntrinsicLocationsBuilderARM64::VisitMemoryPeekLongNative(HInvoke* invoke) {
661  CreateIntToIntLocations(arena_, invoke);
662}
663
664void IntrinsicCodeGeneratorARM64::VisitMemoryPeekLongNative(HInvoke* invoke) {
665  vixl::MacroAssembler* masm = GetVIXLAssembler();
666  __ Ldr(XRegisterFrom(invoke->GetLocations()->Out()),
667         AbsoluteHeapOperandFrom(invoke->GetLocations()->InAt(0), 0));
668}
669
670void IntrinsicLocationsBuilderARM64::VisitMemoryPeekShortNative(HInvoke* invoke) {
671  CreateIntToIntLocations(arena_, invoke);
672}
673
674void IntrinsicCodeGeneratorARM64::VisitMemoryPeekShortNative(HInvoke* invoke) {
675  vixl::MacroAssembler* masm = GetVIXLAssembler();
676  __ Ldrsh(WRegisterFrom(invoke->GetLocations()->Out()),
677           AbsoluteHeapOperandFrom(invoke->GetLocations()->InAt(0), 0));
678}
679
680static void CreateIntIntToVoidLocations(ArenaAllocator* arena, HInvoke* invoke) {
681  LocationSummary* locations = new (arena) LocationSummary(invoke,
682                                                           LocationSummary::kNoCall,
683                                                           kIntrinsified);
684  locations->SetInAt(0, Location::RequiresRegister());
685  locations->SetInAt(1, Location::RequiresRegister());
686}
687
688void IntrinsicLocationsBuilderARM64::VisitMemoryPokeByte(HInvoke* invoke) {
689  CreateIntIntToVoidLocations(arena_, invoke);
690}
691
692void IntrinsicCodeGeneratorARM64::VisitMemoryPokeByte(HInvoke* invoke) {
693  vixl::MacroAssembler* masm = GetVIXLAssembler();
694  __ Strb(WRegisterFrom(invoke->GetLocations()->InAt(1)),
695          AbsoluteHeapOperandFrom(invoke->GetLocations()->InAt(0), 0));
696}
697
698void IntrinsicLocationsBuilderARM64::VisitMemoryPokeIntNative(HInvoke* invoke) {
699  CreateIntIntToVoidLocations(arena_, invoke);
700}
701
702void IntrinsicCodeGeneratorARM64::VisitMemoryPokeIntNative(HInvoke* invoke) {
703  vixl::MacroAssembler* masm = GetVIXLAssembler();
704  __ Str(WRegisterFrom(invoke->GetLocations()->InAt(1)),
705         AbsoluteHeapOperandFrom(invoke->GetLocations()->InAt(0), 0));
706}
707
708void IntrinsicLocationsBuilderARM64::VisitMemoryPokeLongNative(HInvoke* invoke) {
709  CreateIntIntToVoidLocations(arena_, invoke);
710}
711
712void IntrinsicCodeGeneratorARM64::VisitMemoryPokeLongNative(HInvoke* invoke) {
713  vixl::MacroAssembler* masm = GetVIXLAssembler();
714  __ Str(XRegisterFrom(invoke->GetLocations()->InAt(1)),
715         AbsoluteHeapOperandFrom(invoke->GetLocations()->InAt(0), 0));
716}
717
718void IntrinsicLocationsBuilderARM64::VisitMemoryPokeShortNative(HInvoke* invoke) {
719  CreateIntIntToVoidLocations(arena_, invoke);
720}
721
722void IntrinsicCodeGeneratorARM64::VisitMemoryPokeShortNative(HInvoke* invoke) {
723  vixl::MacroAssembler* masm = GetVIXLAssembler();
724  __ Strh(WRegisterFrom(invoke->GetLocations()->InAt(1)),
725          AbsoluteHeapOperandFrom(invoke->GetLocations()->InAt(0), 0));
726}
727
728void IntrinsicLocationsBuilderARM64::VisitThreadCurrentThread(HInvoke* invoke) {
729  LocationSummary* locations = new (arena_) LocationSummary(invoke,
730                                                            LocationSummary::kNoCall,
731                                                            kIntrinsified);
732  locations->SetOut(Location::RequiresRegister());
733}
734
735void IntrinsicCodeGeneratorARM64::VisitThreadCurrentThread(HInvoke* invoke) {
736  codegen_->Load(Primitive::kPrimNot, WRegisterFrom(invoke->GetLocations()->Out()),
737                 MemOperand(tr, Thread::PeerOffset<8>().Int32Value()));
738}
739
740static void GenUnsafeGet(HInvoke* invoke,
741                         Primitive::Type type,
742                         bool is_volatile,
743                         CodeGeneratorARM64* codegen) {
744  LocationSummary* locations = invoke->GetLocations();
745  DCHECK((type == Primitive::kPrimInt) ||
746         (type == Primitive::kPrimLong) ||
747         (type == Primitive::kPrimNot));
748  vixl::MacroAssembler* masm = codegen->GetAssembler()->vixl_masm_;
749  Location base_loc = locations->InAt(1);
750  Register base = WRegisterFrom(base_loc);      // Object pointer.
751  Location offset_loc = locations->InAt(2);
752  Register offset = XRegisterFrom(offset_loc);  // Long offset.
753  Location trg_loc = locations->Out();
754  Register trg = RegisterFrom(trg_loc, type);
755  bool use_acquire_release = codegen->GetInstructionSetFeatures().PreferAcquireRelease();
756
757  MemOperand mem_op(base.X(), offset);
758  if (is_volatile) {
759    if (use_acquire_release) {
760      codegen->LoadAcquire(invoke, trg, mem_op);
761    } else {
762      codegen->Load(type, trg, mem_op);
763      __ Dmb(InnerShareable, BarrierReads);
764    }
765  } else {
766    codegen->Load(type, trg, mem_op);
767  }
768
769  if (type == Primitive::kPrimNot) {
770    DCHECK(trg.IsW());
771    codegen->MaybeGenerateReadBarrier(invoke, trg_loc, trg_loc, base_loc, 0U, offset_loc);
772  }
773}
774
775static void CreateIntIntIntToIntLocations(ArenaAllocator* arena, HInvoke* invoke) {
776  bool can_call = kEmitCompilerReadBarrier &&
777      (invoke->GetIntrinsic() == Intrinsics::kUnsafeGetObject ||
778       invoke->GetIntrinsic() == Intrinsics::kUnsafeGetObjectVolatile);
779  LocationSummary* locations = new (arena) LocationSummary(invoke,
780                                                           can_call ?
781                                                               LocationSummary::kCallOnSlowPath :
782                                                               LocationSummary::kNoCall,
783                                                           kIntrinsified);
784  locations->SetInAt(0, Location::NoLocation());        // Unused receiver.
785  locations->SetInAt(1, Location::RequiresRegister());
786  locations->SetInAt(2, Location::RequiresRegister());
787  locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
788}
789
790void IntrinsicLocationsBuilderARM64::VisitUnsafeGet(HInvoke* invoke) {
791  CreateIntIntIntToIntLocations(arena_, invoke);
792}
793void IntrinsicLocationsBuilderARM64::VisitUnsafeGetVolatile(HInvoke* invoke) {
794  CreateIntIntIntToIntLocations(arena_, invoke);
795}
796void IntrinsicLocationsBuilderARM64::VisitUnsafeGetLong(HInvoke* invoke) {
797  CreateIntIntIntToIntLocations(arena_, invoke);
798}
799void IntrinsicLocationsBuilderARM64::VisitUnsafeGetLongVolatile(HInvoke* invoke) {
800  CreateIntIntIntToIntLocations(arena_, invoke);
801}
802void IntrinsicLocationsBuilderARM64::VisitUnsafeGetObject(HInvoke* invoke) {
803  CreateIntIntIntToIntLocations(arena_, invoke);
804}
805void IntrinsicLocationsBuilderARM64::VisitUnsafeGetObjectVolatile(HInvoke* invoke) {
806  CreateIntIntIntToIntLocations(arena_, invoke);
807}
808
809void IntrinsicCodeGeneratorARM64::VisitUnsafeGet(HInvoke* invoke) {
810  GenUnsafeGet(invoke, Primitive::kPrimInt, /* is_volatile */ false, codegen_);
811}
812void IntrinsicCodeGeneratorARM64::VisitUnsafeGetVolatile(HInvoke* invoke) {
813  GenUnsafeGet(invoke, Primitive::kPrimInt, /* is_volatile */ true, codegen_);
814}
815void IntrinsicCodeGeneratorARM64::VisitUnsafeGetLong(HInvoke* invoke) {
816  GenUnsafeGet(invoke, Primitive::kPrimLong, /* is_volatile */ false, codegen_);
817}
818void IntrinsicCodeGeneratorARM64::VisitUnsafeGetLongVolatile(HInvoke* invoke) {
819  GenUnsafeGet(invoke, Primitive::kPrimLong, /* is_volatile */ true, codegen_);
820}
821void IntrinsicCodeGeneratorARM64::VisitUnsafeGetObject(HInvoke* invoke) {
822  GenUnsafeGet(invoke, Primitive::kPrimNot, /* is_volatile */ false, codegen_);
823}
824void IntrinsicCodeGeneratorARM64::VisitUnsafeGetObjectVolatile(HInvoke* invoke) {
825  GenUnsafeGet(invoke, Primitive::kPrimNot, /* is_volatile */ true, codegen_);
826}
827
828static void CreateIntIntIntIntToVoid(ArenaAllocator* arena, HInvoke* invoke) {
829  LocationSummary* locations = new (arena) LocationSummary(invoke,
830                                                           LocationSummary::kNoCall,
831                                                           kIntrinsified);
832  locations->SetInAt(0, Location::NoLocation());        // Unused receiver.
833  locations->SetInAt(1, Location::RequiresRegister());
834  locations->SetInAt(2, Location::RequiresRegister());
835  locations->SetInAt(3, Location::RequiresRegister());
836}
837
838void IntrinsicLocationsBuilderARM64::VisitUnsafePut(HInvoke* invoke) {
839  CreateIntIntIntIntToVoid(arena_, invoke);
840}
841void IntrinsicLocationsBuilderARM64::VisitUnsafePutOrdered(HInvoke* invoke) {
842  CreateIntIntIntIntToVoid(arena_, invoke);
843}
844void IntrinsicLocationsBuilderARM64::VisitUnsafePutVolatile(HInvoke* invoke) {
845  CreateIntIntIntIntToVoid(arena_, invoke);
846}
847void IntrinsicLocationsBuilderARM64::VisitUnsafePutObject(HInvoke* invoke) {
848  CreateIntIntIntIntToVoid(arena_, invoke);
849}
850void IntrinsicLocationsBuilderARM64::VisitUnsafePutObjectOrdered(HInvoke* invoke) {
851  CreateIntIntIntIntToVoid(arena_, invoke);
852}
853void IntrinsicLocationsBuilderARM64::VisitUnsafePutObjectVolatile(HInvoke* invoke) {
854  CreateIntIntIntIntToVoid(arena_, invoke);
855}
856void IntrinsicLocationsBuilderARM64::VisitUnsafePutLong(HInvoke* invoke) {
857  CreateIntIntIntIntToVoid(arena_, invoke);
858}
859void IntrinsicLocationsBuilderARM64::VisitUnsafePutLongOrdered(HInvoke* invoke) {
860  CreateIntIntIntIntToVoid(arena_, invoke);
861}
862void IntrinsicLocationsBuilderARM64::VisitUnsafePutLongVolatile(HInvoke* invoke) {
863  CreateIntIntIntIntToVoid(arena_, invoke);
864}
865
866static void GenUnsafePut(LocationSummary* locations,
867                         Primitive::Type type,
868                         bool is_volatile,
869                         bool is_ordered,
870                         CodeGeneratorARM64* codegen) {
871  vixl::MacroAssembler* masm = codegen->GetAssembler()->vixl_masm_;
872
873  Register base = WRegisterFrom(locations->InAt(1));    // Object pointer.
874  Register offset = XRegisterFrom(locations->InAt(2));  // Long offset.
875  Register value = RegisterFrom(locations->InAt(3), type);
876  Register source = value;
877  bool use_acquire_release = codegen->GetInstructionSetFeatures().PreferAcquireRelease();
878
879  MemOperand mem_op(base.X(), offset);
880
881  {
882    // We use a block to end the scratch scope before the write barrier, thus
883    // freeing the temporary registers so they can be used in `MarkGCCard`.
884    UseScratchRegisterScope temps(masm);
885
886    if (kPoisonHeapReferences && type == Primitive::kPrimNot) {
887      DCHECK(value.IsW());
888      Register temp = temps.AcquireW();
889      __ Mov(temp.W(), value.W());
890      codegen->GetAssembler()->PoisonHeapReference(temp.W());
891      source = temp;
892    }
893
894    if (is_volatile || is_ordered) {
895      if (use_acquire_release) {
896        codegen->StoreRelease(type, source, mem_op);
897      } else {
898        __ Dmb(InnerShareable, BarrierAll);
899        codegen->Store(type, source, mem_op);
900        if (is_volatile) {
901          __ Dmb(InnerShareable, BarrierReads);
902        }
903      }
904    } else {
905      codegen->Store(type, source, mem_op);
906    }
907  }
908
909  if (type == Primitive::kPrimNot) {
910    bool value_can_be_null = true;  // TODO: Worth finding out this information?
911    codegen->MarkGCCard(base, value, value_can_be_null);
912  }
913}
914
915void IntrinsicCodeGeneratorARM64::VisitUnsafePut(HInvoke* invoke) {
916  GenUnsafePut(invoke->GetLocations(),
917               Primitive::kPrimInt,
918               /* is_volatile */ false,
919               /* is_ordered */ false,
920               codegen_);
921}
922void IntrinsicCodeGeneratorARM64::VisitUnsafePutOrdered(HInvoke* invoke) {
923  GenUnsafePut(invoke->GetLocations(),
924               Primitive::kPrimInt,
925               /* is_volatile */ false,
926               /* is_ordered */ true,
927               codegen_);
928}
929void IntrinsicCodeGeneratorARM64::VisitUnsafePutVolatile(HInvoke* invoke) {
930  GenUnsafePut(invoke->GetLocations(),
931               Primitive::kPrimInt,
932               /* is_volatile */ true,
933               /* is_ordered */ false,
934               codegen_);
935}
936void IntrinsicCodeGeneratorARM64::VisitUnsafePutObject(HInvoke* invoke) {
937  GenUnsafePut(invoke->GetLocations(),
938               Primitive::kPrimNot,
939               /* is_volatile */ false,
940               /* is_ordered */ false,
941               codegen_);
942}
943void IntrinsicCodeGeneratorARM64::VisitUnsafePutObjectOrdered(HInvoke* invoke) {
944  GenUnsafePut(invoke->GetLocations(),
945               Primitive::kPrimNot,
946               /* is_volatile */ false,
947               /* is_ordered */ true,
948               codegen_);
949}
950void IntrinsicCodeGeneratorARM64::VisitUnsafePutObjectVolatile(HInvoke* invoke) {
951  GenUnsafePut(invoke->GetLocations(),
952               Primitive::kPrimNot,
953               /* is_volatile */ true,
954               /* is_ordered */ false,
955               codegen_);
956}
957void IntrinsicCodeGeneratorARM64::VisitUnsafePutLong(HInvoke* invoke) {
958  GenUnsafePut(invoke->GetLocations(),
959               Primitive::kPrimLong,
960               /* is_volatile */ false,
961               /* is_ordered */ false,
962               codegen_);
963}
964void IntrinsicCodeGeneratorARM64::VisitUnsafePutLongOrdered(HInvoke* invoke) {
965  GenUnsafePut(invoke->GetLocations(),
966               Primitive::kPrimLong,
967               /* is_volatile */ false,
968               /* is_ordered */ true,
969               codegen_);
970}
971void IntrinsicCodeGeneratorARM64::VisitUnsafePutLongVolatile(HInvoke* invoke) {
972  GenUnsafePut(invoke->GetLocations(),
973               Primitive::kPrimLong,
974               /* is_volatile */ true,
975               /* is_ordered */ false,
976               codegen_);
977}
978
979static void CreateIntIntIntIntIntToInt(ArenaAllocator* arena, HInvoke* invoke) {
980  LocationSummary* locations = new (arena) LocationSummary(invoke,
981                                                           LocationSummary::kNoCall,
982                                                           kIntrinsified);
983  locations->SetInAt(0, Location::NoLocation());        // Unused receiver.
984  locations->SetInAt(1, Location::RequiresRegister());
985  locations->SetInAt(2, Location::RequiresRegister());
986  locations->SetInAt(3, Location::RequiresRegister());
987  locations->SetInAt(4, Location::RequiresRegister());
988
989  locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
990}
991
992static void GenCas(LocationSummary* locations, Primitive::Type type, CodeGeneratorARM64* codegen) {
993  bool use_acquire_release = codegen->GetInstructionSetFeatures().PreferAcquireRelease();
994  vixl::MacroAssembler* masm = codegen->GetAssembler()->vixl_masm_;
995
996  Register out = WRegisterFrom(locations->Out());                  // Boolean result.
997
998  Register base = WRegisterFrom(locations->InAt(1));               // Object pointer.
999  Register offset = XRegisterFrom(locations->InAt(2));             // Long offset.
1000  Register expected = RegisterFrom(locations->InAt(3), type);      // Expected.
1001  Register value = RegisterFrom(locations->InAt(4), type);         // Value.
1002
1003  // This needs to be before the temp registers, as MarkGCCard also uses VIXL temps.
1004  if (type == Primitive::kPrimNot) {
1005    // Mark card for object assuming new value is stored.
1006    bool value_can_be_null = true;  // TODO: Worth finding out this information?
1007    codegen->MarkGCCard(base, value, value_can_be_null);
1008  }
1009
1010  UseScratchRegisterScope temps(masm);
1011  Register tmp_ptr = temps.AcquireX();                             // Pointer to actual memory.
1012  Register tmp_value = temps.AcquireSameSizeAs(value);             // Value in memory.
1013
1014  Register tmp_32 = tmp_value.W();
1015
1016  __ Add(tmp_ptr, base.X(), Operand(offset));
1017
1018  if (kPoisonHeapReferences && type == Primitive::kPrimNot) {
1019    codegen->GetAssembler()->PoisonHeapReference(expected);
1020    codegen->GetAssembler()->PoisonHeapReference(value);
1021  }
1022
1023  // do {
1024  //   tmp_value = [tmp_ptr] - expected;
1025  // } while (tmp_value == 0 && failure([tmp_ptr] <- r_new_value));
1026  // result = tmp_value != 0;
1027
1028  vixl::Label loop_head, exit_loop;
1029  if (use_acquire_release) {
1030    __ Bind(&loop_head);
1031    __ Ldaxr(tmp_value, MemOperand(tmp_ptr));
1032    // TODO: Do we need a read barrier here when `type == Primitive::kPrimNot`?
1033    // Note that this code is not (yet) used when read barriers are
1034    // enabled (see IntrinsicLocationsBuilderARM64::VisitUnsafeCASObject).
1035    __ Cmp(tmp_value, expected);
1036    __ B(&exit_loop, ne);
1037    __ Stlxr(tmp_32, value, MemOperand(tmp_ptr));
1038    __ Cbnz(tmp_32, &loop_head);
1039  } else {
1040    __ Dmb(InnerShareable, BarrierWrites);
1041    __ Bind(&loop_head);
1042    // TODO: When `type == Primitive::kPrimNot`, add a read barrier for
1043    // the reference stored in the object before attempting the CAS,
1044    // similar to the one in the art::Unsafe_compareAndSwapObject JNI
1045    // implementation.
1046    //
1047    // Note that this code is not (yet) used when read barriers are
1048    // enabled (see IntrinsicLocationsBuilderARM64::VisitUnsafeCASObject).
1049    DCHECK(!(type == Primitive::kPrimNot && kEmitCompilerReadBarrier));
1050    __ Ldxr(tmp_value, MemOperand(tmp_ptr));
1051    __ Cmp(tmp_value, expected);
1052    __ B(&exit_loop, ne);
1053    __ Stxr(tmp_32, value, MemOperand(tmp_ptr));
1054    __ Cbnz(tmp_32, &loop_head);
1055    __ Dmb(InnerShareable, BarrierAll);
1056  }
1057  __ Bind(&exit_loop);
1058  __ Cset(out, eq);
1059
1060  if (kPoisonHeapReferences && type == Primitive::kPrimNot) {
1061    codegen->GetAssembler()->UnpoisonHeapReference(value);
1062    codegen->GetAssembler()->UnpoisonHeapReference(expected);
1063  }
1064}
1065
1066void IntrinsicLocationsBuilderARM64::VisitUnsafeCASInt(HInvoke* invoke) {
1067  CreateIntIntIntIntIntToInt(arena_, invoke);
1068}
1069void IntrinsicLocationsBuilderARM64::VisitUnsafeCASLong(HInvoke* invoke) {
1070  CreateIntIntIntIntIntToInt(arena_, invoke);
1071}
1072void IntrinsicLocationsBuilderARM64::VisitUnsafeCASObject(HInvoke* invoke) {
1073  // The UnsafeCASObject intrinsic is missing a read barrier, and
1074  // therefore sometimes does not work as expected (b/25883050).
1075  // Turn it off temporarily as a quick fix, until the read barrier is
1076  // implemented (see TODO in GenCAS below).
1077  //
1078  // Also, the UnsafeCASObject intrinsic does not always work when heap
1079  // poisoning is enabled (it breaks run-test 004-UnsafeTest); turn it
1080  // off temporarily as a quick fix (b/26204023).
1081  //
1082  // TODO(rpl): Fix these two issues and re-enable this intrinsic.
1083  if (kEmitCompilerReadBarrier || kPoisonHeapReferences) {
1084    return;
1085  }
1086
1087  CreateIntIntIntIntIntToInt(arena_, invoke);
1088}
1089
1090void IntrinsicCodeGeneratorARM64::VisitUnsafeCASInt(HInvoke* invoke) {
1091  GenCas(invoke->GetLocations(), Primitive::kPrimInt, codegen_);
1092}
1093void IntrinsicCodeGeneratorARM64::VisitUnsafeCASLong(HInvoke* invoke) {
1094  GenCas(invoke->GetLocations(), Primitive::kPrimLong, codegen_);
1095}
1096void IntrinsicCodeGeneratorARM64::VisitUnsafeCASObject(HInvoke* invoke) {
1097  GenCas(invoke->GetLocations(), Primitive::kPrimNot, codegen_);
1098}
1099
1100void IntrinsicLocationsBuilderARM64::VisitStringCharAt(HInvoke* invoke) {
1101  LocationSummary* locations = new (arena_) LocationSummary(invoke,
1102                                                            LocationSummary::kCallOnSlowPath,
1103                                                            kIntrinsified);
1104  locations->SetInAt(0, Location::RequiresRegister());
1105  locations->SetInAt(1, Location::RequiresRegister());
1106  // In case we need to go in the slow path, we can't have the output be the same
1107  // as the input: the current liveness analysis considers the input to be live
1108  // at the point of the call.
1109  locations->SetOut(Location::RequiresRegister(), Location::kOutputOverlap);
1110}
1111
1112void IntrinsicCodeGeneratorARM64::VisitStringCharAt(HInvoke* invoke) {
1113  vixl::MacroAssembler* masm = GetVIXLAssembler();
1114  LocationSummary* locations = invoke->GetLocations();
1115
1116  // Location of reference to data array
1117  const MemberOffset value_offset = mirror::String::ValueOffset();
1118  // Location of count
1119  const MemberOffset count_offset = mirror::String::CountOffset();
1120
1121  Register obj = WRegisterFrom(locations->InAt(0));  // String object pointer.
1122  Register idx = WRegisterFrom(locations->InAt(1));  // Index of character.
1123  Register out = WRegisterFrom(locations->Out());    // Result character.
1124
1125  UseScratchRegisterScope temps(masm);
1126  Register temp = temps.AcquireW();
1127  Register array_temp = temps.AcquireW();            // We can trade this for worse scheduling.
1128
1129  // TODO: Maybe we can support range check elimination. Overall, though, I think it's not worth
1130  //       the cost.
1131  // TODO: For simplicity, the index parameter is requested in a register, so different from Quick
1132  //       we will not optimize the code for constants (which would save a register).
1133
1134  SlowPathCodeARM64* slow_path = new (GetAllocator()) IntrinsicSlowPathARM64(invoke);
1135  codegen_->AddSlowPath(slow_path);
1136
1137  __ Ldr(temp, HeapOperand(obj, count_offset));          // temp = str.length.
1138  codegen_->MaybeRecordImplicitNullCheck(invoke);
1139  __ Cmp(idx, temp);
1140  __ B(hs, slow_path->GetEntryLabel());
1141
1142  __ Add(array_temp, obj, Operand(value_offset.Int32Value()));  // array_temp := str.value.
1143
1144  // Load the value.
1145  __ Ldrh(out, MemOperand(array_temp.X(), idx, UXTW, 1));  // out := array_temp[idx].
1146
1147  __ Bind(slow_path->GetExitLabel());
1148}
1149
1150void IntrinsicLocationsBuilderARM64::VisitStringCompareTo(HInvoke* invoke) {
1151  LocationSummary* locations = new (arena_) LocationSummary(invoke,
1152                                                            LocationSummary::kCall,
1153                                                            kIntrinsified);
1154  InvokeRuntimeCallingConvention calling_convention;
1155  locations->SetInAt(0, LocationFrom(calling_convention.GetRegisterAt(0)));
1156  locations->SetInAt(1, LocationFrom(calling_convention.GetRegisterAt(1)));
1157  locations->SetOut(calling_convention.GetReturnLocation(Primitive::kPrimInt));
1158}
1159
1160void IntrinsicCodeGeneratorARM64::VisitStringCompareTo(HInvoke* invoke) {
1161  vixl::MacroAssembler* masm = GetVIXLAssembler();
1162  LocationSummary* locations = invoke->GetLocations();
1163
1164  // Note that the null check must have been done earlier.
1165  DCHECK(!invoke->CanDoImplicitNullCheckOn(invoke->InputAt(0)));
1166
1167  Register argument = WRegisterFrom(locations->InAt(1));
1168  __ Cmp(argument, 0);
1169  SlowPathCodeARM64* slow_path = new (GetAllocator()) IntrinsicSlowPathARM64(invoke);
1170  codegen_->AddSlowPath(slow_path);
1171  __ B(eq, slow_path->GetEntryLabel());
1172
1173  __ Ldr(
1174      lr, MemOperand(tr, QUICK_ENTRYPOINT_OFFSET(kArm64WordSize, pStringCompareTo).Int32Value()));
1175  __ Blr(lr);
1176  __ Bind(slow_path->GetExitLabel());
1177}
1178
1179void IntrinsicLocationsBuilderARM64::VisitStringEquals(HInvoke* invoke) {
1180  LocationSummary* locations = new (arena_) LocationSummary(invoke,
1181                                                            LocationSummary::kNoCall,
1182                                                            kIntrinsified);
1183  locations->SetInAt(0, Location::RequiresRegister());
1184  locations->SetInAt(1, Location::RequiresRegister());
1185  // Temporary registers to store lengths of strings and for calculations.
1186  locations->AddTemp(Location::RequiresRegister());
1187  locations->AddTemp(Location::RequiresRegister());
1188
1189  locations->SetOut(Location::RequiresRegister(), Location::kOutputOverlap);
1190}
1191
1192void IntrinsicCodeGeneratorARM64::VisitStringEquals(HInvoke* invoke) {
1193  vixl::MacroAssembler* masm = GetVIXLAssembler();
1194  LocationSummary* locations = invoke->GetLocations();
1195
1196  Register str = WRegisterFrom(locations->InAt(0));
1197  Register arg = WRegisterFrom(locations->InAt(1));
1198  Register out = XRegisterFrom(locations->Out());
1199
1200  UseScratchRegisterScope scratch_scope(masm);
1201  Register temp = scratch_scope.AcquireW();
1202  Register temp1 = WRegisterFrom(locations->GetTemp(0));
1203  Register temp2 = WRegisterFrom(locations->GetTemp(1));
1204
1205  vixl::Label loop;
1206  vixl::Label end;
1207  vixl::Label return_true;
1208  vixl::Label return_false;
1209
1210  // Get offsets of count, value, and class fields within a string object.
1211  const int32_t count_offset = mirror::String::CountOffset().Int32Value();
1212  const int32_t value_offset = mirror::String::ValueOffset().Int32Value();
1213  const int32_t class_offset = mirror::Object::ClassOffset().Int32Value();
1214
1215  // Note that the null check must have been done earlier.
1216  DCHECK(!invoke->CanDoImplicitNullCheckOn(invoke->InputAt(0)));
1217
1218  // Check if input is null, return false if it is.
1219  __ Cbz(arg, &return_false);
1220
1221  // Reference equality check, return true if same reference.
1222  __ Cmp(str, arg);
1223  __ B(&return_true, eq);
1224
1225  // Instanceof check for the argument by comparing class fields.
1226  // All string objects must have the same type since String cannot be subclassed.
1227  // Receiver must be a string object, so its class field is equal to all strings' class fields.
1228  // If the argument is a string object, its class field must be equal to receiver's class field.
1229  __ Ldr(temp, MemOperand(str.X(), class_offset));
1230  __ Ldr(temp1, MemOperand(arg.X(), class_offset));
1231  __ Cmp(temp, temp1);
1232  __ B(&return_false, ne);
1233
1234  // Load lengths of this and argument strings.
1235  __ Ldr(temp, MemOperand(str.X(), count_offset));
1236  __ Ldr(temp1, MemOperand(arg.X(), count_offset));
1237  // Check if lengths are equal, return false if they're not.
1238  __ Cmp(temp, temp1);
1239  __ B(&return_false, ne);
1240  // Store offset of string value in preparation for comparison loop
1241  __ Mov(temp1, value_offset);
1242  // Return true if both strings are empty.
1243  __ Cbz(temp, &return_true);
1244
1245  // Assertions that must hold in order to compare strings 4 characters at a time.
1246  DCHECK_ALIGNED(value_offset, 8);
1247  static_assert(IsAligned<8>(kObjectAlignment), "String of odd length is not zero padded");
1248
1249  temp1 = temp1.X();
1250  temp2 = temp2.X();
1251
1252  // Loop to compare strings 4 characters at a time starting at the beginning of the string.
1253  // Ok to do this because strings are zero-padded to be 8-byte aligned.
1254  __ Bind(&loop);
1255  __ Ldr(out, MemOperand(str.X(), temp1));
1256  __ Ldr(temp2, MemOperand(arg.X(), temp1));
1257  __ Add(temp1, temp1, Operand(sizeof(uint64_t)));
1258  __ Cmp(out, temp2);
1259  __ B(&return_false, ne);
1260  __ Sub(temp, temp, Operand(4), SetFlags);
1261  __ B(&loop, gt);
1262
1263  // Return true and exit the function.
1264  // If loop does not result in returning false, we return true.
1265  __ Bind(&return_true);
1266  __ Mov(out, 1);
1267  __ B(&end);
1268
1269  // Return false and exit the function.
1270  __ Bind(&return_false);
1271  __ Mov(out, 0);
1272  __ Bind(&end);
1273}
1274
1275static void GenerateVisitStringIndexOf(HInvoke* invoke,
1276                                       vixl::MacroAssembler* masm,
1277                                       CodeGeneratorARM64* codegen,
1278                                       ArenaAllocator* allocator,
1279                                       bool start_at_zero) {
1280  LocationSummary* locations = invoke->GetLocations();
1281  Register tmp_reg = WRegisterFrom(locations->GetTemp(0));
1282
1283  // Note that the null check must have been done earlier.
1284  DCHECK(!invoke->CanDoImplicitNullCheckOn(invoke->InputAt(0)));
1285
1286  // Check for code points > 0xFFFF. Either a slow-path check when we don't know statically,
1287  // or directly dispatch if we have a constant.
1288  SlowPathCodeARM64* slow_path = nullptr;
1289  if (invoke->InputAt(1)->IsIntConstant()) {
1290    if (static_cast<uint32_t>(invoke->InputAt(1)->AsIntConstant()->GetValue()) > 0xFFFFU) {
1291      // Always needs the slow-path. We could directly dispatch to it, but this case should be
1292      // rare, so for simplicity just put the full slow-path down and branch unconditionally.
1293      slow_path = new (allocator) IntrinsicSlowPathARM64(invoke);
1294      codegen->AddSlowPath(slow_path);
1295      __ B(slow_path->GetEntryLabel());
1296      __ Bind(slow_path->GetExitLabel());
1297      return;
1298    }
1299  } else {
1300    Register char_reg = WRegisterFrom(locations->InAt(1));
1301    __ Mov(tmp_reg, 0xFFFF);
1302    __ Cmp(char_reg, Operand(tmp_reg));
1303    slow_path = new (allocator) IntrinsicSlowPathARM64(invoke);
1304    codegen->AddSlowPath(slow_path);
1305    __ B(hi, slow_path->GetEntryLabel());
1306  }
1307
1308  if (start_at_zero) {
1309    // Start-index = 0.
1310    __ Mov(tmp_reg, 0);
1311  }
1312
1313  __ Ldr(lr, MemOperand(tr, QUICK_ENTRYPOINT_OFFSET(kArm64WordSize, pIndexOf).Int32Value()));
1314  __ Blr(lr);
1315
1316  if (slow_path != nullptr) {
1317    __ Bind(slow_path->GetExitLabel());
1318  }
1319}
1320
1321void IntrinsicLocationsBuilderARM64::VisitStringIndexOf(HInvoke* invoke) {
1322  LocationSummary* locations = new (arena_) LocationSummary(invoke,
1323                                                            LocationSummary::kCall,
1324                                                            kIntrinsified);
1325  // We have a hand-crafted assembly stub that follows the runtime calling convention. So it's
1326  // best to align the inputs accordingly.
1327  InvokeRuntimeCallingConvention calling_convention;
1328  locations->SetInAt(0, LocationFrom(calling_convention.GetRegisterAt(0)));
1329  locations->SetInAt(1, LocationFrom(calling_convention.GetRegisterAt(1)));
1330  locations->SetOut(calling_convention.GetReturnLocation(Primitive::kPrimInt));
1331
1332  // Need a temp for slow-path codepoint compare, and need to send start_index=0.
1333  locations->AddTemp(LocationFrom(calling_convention.GetRegisterAt(2)));
1334}
1335
1336void IntrinsicCodeGeneratorARM64::VisitStringIndexOf(HInvoke* invoke) {
1337  GenerateVisitStringIndexOf(
1338      invoke, GetVIXLAssembler(), codegen_, GetAllocator(), /* start_at_zero */ true);
1339}
1340
1341void IntrinsicLocationsBuilderARM64::VisitStringIndexOfAfter(HInvoke* invoke) {
1342  LocationSummary* locations = new (arena_) LocationSummary(invoke,
1343                                                            LocationSummary::kCall,
1344                                                            kIntrinsified);
1345  // We have a hand-crafted assembly stub that follows the runtime calling convention. So it's
1346  // best to align the inputs accordingly.
1347  InvokeRuntimeCallingConvention calling_convention;
1348  locations->SetInAt(0, LocationFrom(calling_convention.GetRegisterAt(0)));
1349  locations->SetInAt(1, LocationFrom(calling_convention.GetRegisterAt(1)));
1350  locations->SetInAt(2, LocationFrom(calling_convention.GetRegisterAt(2)));
1351  locations->SetOut(calling_convention.GetReturnLocation(Primitive::kPrimInt));
1352
1353  // Need a temp for slow-path codepoint compare.
1354  locations->AddTemp(Location::RequiresRegister());
1355}
1356
1357void IntrinsicCodeGeneratorARM64::VisitStringIndexOfAfter(HInvoke* invoke) {
1358  GenerateVisitStringIndexOf(
1359      invoke, GetVIXLAssembler(), codegen_, GetAllocator(), /* start_at_zero */ false);
1360}
1361
1362void IntrinsicLocationsBuilderARM64::VisitStringNewStringFromBytes(HInvoke* invoke) {
1363  LocationSummary* locations = new (arena_) LocationSummary(invoke,
1364                                                            LocationSummary::kCall,
1365                                                            kIntrinsified);
1366  InvokeRuntimeCallingConvention calling_convention;
1367  locations->SetInAt(0, LocationFrom(calling_convention.GetRegisterAt(0)));
1368  locations->SetInAt(1, LocationFrom(calling_convention.GetRegisterAt(1)));
1369  locations->SetInAt(2, LocationFrom(calling_convention.GetRegisterAt(2)));
1370  locations->SetInAt(3, LocationFrom(calling_convention.GetRegisterAt(3)));
1371  locations->SetOut(calling_convention.GetReturnLocation(Primitive::kPrimNot));
1372}
1373
1374void IntrinsicCodeGeneratorARM64::VisitStringNewStringFromBytes(HInvoke* invoke) {
1375  vixl::MacroAssembler* masm = GetVIXLAssembler();
1376  LocationSummary* locations = invoke->GetLocations();
1377
1378  Register byte_array = WRegisterFrom(locations->InAt(0));
1379  __ Cmp(byte_array, 0);
1380  SlowPathCodeARM64* slow_path = new (GetAllocator()) IntrinsicSlowPathARM64(invoke);
1381  codegen_->AddSlowPath(slow_path);
1382  __ B(eq, slow_path->GetEntryLabel());
1383
1384  __ Ldr(lr,
1385      MemOperand(tr, QUICK_ENTRYPOINT_OFFSET(kArm64WordSize, pAllocStringFromBytes).Int32Value()));
1386  codegen_->RecordPcInfo(invoke, invoke->GetDexPc());
1387  __ Blr(lr);
1388  __ Bind(slow_path->GetExitLabel());
1389}
1390
1391void IntrinsicLocationsBuilderARM64::VisitStringNewStringFromChars(HInvoke* invoke) {
1392  LocationSummary* locations = new (arena_) LocationSummary(invoke,
1393                                                            LocationSummary::kCall,
1394                                                            kIntrinsified);
1395  InvokeRuntimeCallingConvention calling_convention;
1396  locations->SetInAt(0, LocationFrom(calling_convention.GetRegisterAt(0)));
1397  locations->SetInAt(1, LocationFrom(calling_convention.GetRegisterAt(1)));
1398  locations->SetInAt(2, LocationFrom(calling_convention.GetRegisterAt(2)));
1399  locations->SetOut(calling_convention.GetReturnLocation(Primitive::kPrimNot));
1400}
1401
1402void IntrinsicCodeGeneratorARM64::VisitStringNewStringFromChars(HInvoke* invoke) {
1403  vixl::MacroAssembler* masm = GetVIXLAssembler();
1404
1405  __ Ldr(lr,
1406      MemOperand(tr, QUICK_ENTRYPOINT_OFFSET(kArm64WordSize, pAllocStringFromChars).Int32Value()));
1407  codegen_->RecordPcInfo(invoke, invoke->GetDexPc());
1408  __ Blr(lr);
1409}
1410
1411void IntrinsicLocationsBuilderARM64::VisitStringNewStringFromString(HInvoke* invoke) {
1412  // The inputs plus one temp.
1413  LocationSummary* locations = new (arena_) LocationSummary(invoke,
1414                                                            LocationSummary::kCall,
1415                                                            kIntrinsified);
1416  InvokeRuntimeCallingConvention calling_convention;
1417  locations->SetInAt(0, LocationFrom(calling_convention.GetRegisterAt(0)));
1418  locations->SetInAt(1, LocationFrom(calling_convention.GetRegisterAt(1)));
1419  locations->SetInAt(2, LocationFrom(calling_convention.GetRegisterAt(2)));
1420  locations->SetOut(calling_convention.GetReturnLocation(Primitive::kPrimNot));
1421}
1422
1423void IntrinsicCodeGeneratorARM64::VisitStringNewStringFromString(HInvoke* invoke) {
1424  vixl::MacroAssembler* masm = GetVIXLAssembler();
1425  LocationSummary* locations = invoke->GetLocations();
1426
1427  Register string_to_copy = WRegisterFrom(locations->InAt(0));
1428  __ Cmp(string_to_copy, 0);
1429  SlowPathCodeARM64* slow_path = new (GetAllocator()) IntrinsicSlowPathARM64(invoke);
1430  codegen_->AddSlowPath(slow_path);
1431  __ B(eq, slow_path->GetEntryLabel());
1432
1433  __ Ldr(lr,
1434      MemOperand(tr, QUICK_ENTRYPOINT_OFFSET(kArm64WordSize, pAllocStringFromString).Int32Value()));
1435  codegen_->RecordPcInfo(invoke, invoke->GetDexPc());
1436  __ Blr(lr);
1437  __ Bind(slow_path->GetExitLabel());
1438}
1439
1440// Unimplemented intrinsics.
1441
1442#define UNIMPLEMENTED_INTRINSIC(Name)                                                  \
1443void IntrinsicLocationsBuilderARM64::Visit ## Name(HInvoke* invoke ATTRIBUTE_UNUSED) { \
1444}                                                                                      \
1445void IntrinsicCodeGeneratorARM64::Visit ## Name(HInvoke* invoke ATTRIBUTE_UNUSED) {    \
1446}
1447
1448UNIMPLEMENTED_INTRINSIC(IntegerRotateLeft)
1449UNIMPLEMENTED_INTRINSIC(IntegerRotateRight)
1450UNIMPLEMENTED_INTRINSIC(LongRotateLeft)
1451UNIMPLEMENTED_INTRINSIC(LongRotateRight)
1452UNIMPLEMENTED_INTRINSIC(SystemArrayCopyChar)
1453UNIMPLEMENTED_INTRINSIC(SystemArrayCopy)
1454UNIMPLEMENTED_INTRINSIC(ReferenceGetReferent)
1455UNIMPLEMENTED_INTRINSIC(StringGetCharsNoCheck)
1456
1457UNIMPLEMENTED_INTRINSIC(MathCos)
1458UNIMPLEMENTED_INTRINSIC(MathSin)
1459UNIMPLEMENTED_INTRINSIC(MathAcos)
1460UNIMPLEMENTED_INTRINSIC(MathAsin)
1461UNIMPLEMENTED_INTRINSIC(MathAtan)
1462UNIMPLEMENTED_INTRINSIC(MathAtan2)
1463UNIMPLEMENTED_INTRINSIC(MathCbrt)
1464UNIMPLEMENTED_INTRINSIC(MathCosh)
1465UNIMPLEMENTED_INTRINSIC(MathExp)
1466UNIMPLEMENTED_INTRINSIC(MathExpm1)
1467UNIMPLEMENTED_INTRINSIC(MathHypot)
1468UNIMPLEMENTED_INTRINSIC(MathLog)
1469UNIMPLEMENTED_INTRINSIC(MathLog10)
1470UNIMPLEMENTED_INTRINSIC(MathNextAfter)
1471UNIMPLEMENTED_INTRINSIC(MathSinh)
1472UNIMPLEMENTED_INTRINSIC(MathTan)
1473UNIMPLEMENTED_INTRINSIC(MathTanh)
1474
1475#undef UNIMPLEMENTED_INTRINSIC
1476
1477#undef __
1478
1479}  // namespace arm64
1480}  // namespace art
1481