intrinsics_arm64.cc revision 391b866ce55b8e78b1f9a6b98321d837256e8d66 
1/*
2 * Copyright (C) 2015 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *      http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17#include "intrinsics_arm64.h"
18
19#include "arch/arm64/instruction_set_features_arm64.h"
20#include "art_method.h"
21#include "code_generator_arm64.h"
22#include "common_arm64.h"
23#include "entrypoints/quick/quick_entrypoints.h"
24#include "intrinsics.h"
25#include "mirror/array-inl.h"
26#include "mirror/string.h"
27#include "thread.h"
28#include "utils/arm64/assembler_arm64.h"
29#include "utils/arm64/constants_arm64.h"
30
31#include "vixl/a64/disasm-a64.h"
32#include "vixl/a64/macro-assembler-a64.h"
33
34using namespace vixl;   // NOLINT(build/namespaces)
35
36namespace art {
37
38namespace arm64 {
39
40using helpers::DRegisterFrom;
41using helpers::FPRegisterFrom;
42using helpers::HeapOperand;
43using helpers::LocationFrom;
44using helpers::OperandFrom;
45using helpers::RegisterFrom;
46using helpers::SRegisterFrom;
47using helpers::WRegisterFrom;
48using helpers::XRegisterFrom;
49
50namespace {
51
52ALWAYS_INLINE inline MemOperand AbsoluteHeapOperandFrom(Location location, size_t offset = 0) {
53  return MemOperand(XRegisterFrom(location), offset);
54}
55
56}  // namespace
57
58vixl::MacroAssembler* IntrinsicCodeGeneratorARM64::GetVIXLAssembler() {
59  return codegen_->GetAssembler()->vixl_masm_;
60}
61
62ArenaAllocator* IntrinsicCodeGeneratorARM64::GetAllocator() {
63  return codegen_->GetGraph()->GetArena();
64}
65
66#define __ codegen->GetAssembler()->vixl_masm_->
67
68static void MoveFromReturnRegister(Location trg,
69                                   Primitive::Type type,
70                                   CodeGeneratorARM64* codegen) {
71  if (!trg.IsValid()) {
72    DCHECK(type == Primitive::kPrimVoid);
73    return;
74  }
75
76  DCHECK_NE(type, Primitive::kPrimVoid);
77
78  if (Primitive::IsIntegralType(type) || type == Primitive::kPrimNot) {
79    Register trg_reg = RegisterFrom(trg, type);
80    Register res_reg = RegisterFrom(ARM64ReturnLocation(type), type);
81    __ Mov(trg_reg, res_reg, kDiscardForSameWReg);
82  } else {
83    FPRegister trg_reg = FPRegisterFrom(trg, type);
84    FPRegister res_reg = FPRegisterFrom(ARM64ReturnLocation(type), type);
85    __ Fmov(trg_reg, res_reg);
86  }
87}
88
89static void MoveArguments(HInvoke* invoke, CodeGeneratorARM64* codegen) {
90  InvokeDexCallingConventionVisitorARM64 calling_convention_visitor;
91  IntrinsicVisitor::MoveArguments(invoke, codegen, &calling_convention_visitor);
92}
93
94// Slow-path for fallback (calling the managed code to handle the intrinsic) in an intrinsified
95// call. This will copy the arguments into the positions for a regular call.
96//
97// Note: The actual parameters are required to be in the locations given by the invoke's location
98//       summary. If an intrinsic modifies those locations before a slowpath call, they must be
99//       restored!
100class IntrinsicSlowPathARM64 : public SlowPathCodeARM64 {
101 public:
102  explicit IntrinsicSlowPathARM64(HInvoke* invoke) : invoke_(invoke) { }
103
104  void EmitNativeCode(CodeGenerator* codegen_in) OVERRIDE {
105    CodeGeneratorARM64* codegen = down_cast<CodeGeneratorARM64*>(codegen_in);
106    __ Bind(GetEntryLabel());
107
108    SaveLiveRegisters(codegen, invoke_->GetLocations());
109
110    MoveArguments(invoke_, codegen);
111
112    if (invoke_->IsInvokeStaticOrDirect()) {
113      codegen->GenerateStaticOrDirectCall(invoke_->AsInvokeStaticOrDirect(),
114                                          LocationFrom(kArtMethodRegister));
115    } else {
116      codegen->GenerateVirtualCall(invoke_->AsInvokeVirtual(), LocationFrom(kArtMethodRegister));
117    }
118    codegen->RecordPcInfo(invoke_, invoke_->GetDexPc(), this);
119
120    // Copy the result back to the expected output.
121    Location out = invoke_->GetLocations()->Out();
122    if (out.IsValid()) {
123      DCHECK(out.IsRegister());  // TODO: Replace this when we support output in memory.
124      DCHECK(!invoke_->GetLocations()->GetLiveRegisters()->ContainsCoreRegister(out.reg()));
125      MoveFromReturnRegister(out, invoke_->GetType(), codegen);
126    }
127
128    RestoreLiveRegisters(codegen, invoke_->GetLocations());
129    __ B(GetExitLabel());
130  }
131
132  const char* GetDescription() const OVERRIDE { return "IntrinsicSlowPathARM64"; }
133
134 private:
135  // The instruction where this slow path is happening.
136  HInvoke* const invoke_;
137
138  DISALLOW_COPY_AND_ASSIGN(IntrinsicSlowPathARM64);
139};
140
141#undef __
142
143bool IntrinsicLocationsBuilderARM64::TryDispatch(HInvoke* invoke) {
144  Dispatch(invoke);
145  LocationSummary* res = invoke->GetLocations();
146  if (res == nullptr) {
147    return false;
148  }
149  if (kEmitCompilerReadBarrier && res->CanCall()) {
150    // Generating an intrinsic for this HInvoke may produce an
151    // IntrinsicSlowPathARM64 slow path.  Currently this approach
152    // does not work when using read barriers, as the emitted
153    // calling sequence will make use of another slow path
154    // (ReadBarrierForRootSlowPathARM64 for HInvokeStaticOrDirect,
155    // ReadBarrierSlowPathARM64 for HInvokeVirtual).  So we bail
156    // out in this case.
157    //
158    // TODO: Find a way to have intrinsics work with read barriers.
159    invoke->SetLocations(nullptr);
160    return false;
161  }
162  return res->Intrinsified();
163}
164
165#define __ masm->
166
167static void CreateFPToIntLocations(ArenaAllocator* arena, HInvoke* invoke) {
168  LocationSummary* locations = new (arena) LocationSummary(invoke,
169                                                           LocationSummary::kNoCall,
170                                                           kIntrinsified);
171  locations->SetInAt(0, Location::RequiresFpuRegister());
172  locations->SetOut(Location::RequiresRegister());
173}
174
175static void CreateIntToFPLocations(ArenaAllocator* arena, HInvoke* invoke) {
176  LocationSummary* locations = new (arena) LocationSummary(invoke,
177                                                           LocationSummary::kNoCall,
178                                                           kIntrinsified);
179  locations->SetInAt(0, Location::RequiresRegister());
180  locations->SetOut(Location::RequiresFpuRegister());
181}
182
183static void MoveFPToInt(LocationSummary* locations, bool is64bit, vixl::MacroAssembler* masm) {
184  Location input = locations->InAt(0);
185  Location output = locations->Out();
186  __ Fmov(is64bit ? XRegisterFrom(output) : WRegisterFrom(output),
187          is64bit ? DRegisterFrom(input) : SRegisterFrom(input));
188}
189
190static void MoveIntToFP(LocationSummary* locations, bool is64bit, vixl::MacroAssembler* masm) {
191  Location input = locations->InAt(0);
192  Location output = locations->Out();
193  __ Fmov(is64bit ? DRegisterFrom(output) : SRegisterFrom(output),
194          is64bit ? XRegisterFrom(input) : WRegisterFrom(input));
195}
196
197void IntrinsicLocationsBuilderARM64::VisitDoubleDoubleToRawLongBits(HInvoke* invoke) {
198  CreateFPToIntLocations(arena_, invoke);
199}
200void IntrinsicLocationsBuilderARM64::VisitDoubleLongBitsToDouble(HInvoke* invoke) {
201  CreateIntToFPLocations(arena_, invoke);
202}
203
204void IntrinsicCodeGeneratorARM64::VisitDoubleDoubleToRawLongBits(HInvoke* invoke) {
205  MoveFPToInt(invoke->GetLocations(), /* is64bit */ true, GetVIXLAssembler());
206}
207void IntrinsicCodeGeneratorARM64::VisitDoubleLongBitsToDouble(HInvoke* invoke) {
208  MoveIntToFP(invoke->GetLocations(), /* is64bit */ true, GetVIXLAssembler());
209}
210
211void IntrinsicLocationsBuilderARM64::VisitFloatFloatToRawIntBits(HInvoke* invoke) {
212  CreateFPToIntLocations(arena_, invoke);
213}
214void IntrinsicLocationsBuilderARM64::VisitFloatIntBitsToFloat(HInvoke* invoke) {
215  CreateIntToFPLocations(arena_, invoke);
216}
217
218void IntrinsicCodeGeneratorARM64::VisitFloatFloatToRawIntBits(HInvoke* invoke) {
219  MoveFPToInt(invoke->GetLocations(), /* is64bit */ false, GetVIXLAssembler());
220}
221void IntrinsicCodeGeneratorARM64::VisitFloatIntBitsToFloat(HInvoke* invoke) {
222  MoveIntToFP(invoke->GetLocations(), /* is64bit */ false, GetVIXLAssembler());
223}
224
225static void CreateIntToIntLocations(ArenaAllocator* arena, HInvoke* invoke) {
226  LocationSummary* locations = new (arena) LocationSummary(invoke,
227                                                           LocationSummary::kNoCall,
228                                                           kIntrinsified);
229  locations->SetInAt(0, Location::RequiresRegister());
230  locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
231}
232
233static void GenReverseBytes(LocationSummary* locations,
234                            Primitive::Type type,
235                            vixl::MacroAssembler* masm) {
236  Location in = locations->InAt(0);
237  Location out = locations->Out();
238
239  switch (type) {
240    case Primitive::kPrimShort:
241      __ Rev16(WRegisterFrom(out), WRegisterFrom(in));
242      __ Sxth(WRegisterFrom(out), WRegisterFrom(out));
243      break;
244    case Primitive::kPrimInt:
245    case Primitive::kPrimLong:
246      __ Rev(RegisterFrom(out, type), RegisterFrom(in, type));
247      break;
248    default:
249      LOG(FATAL) << "Unexpected size for reverse-bytes: " << type;
250      UNREACHABLE();
251  }
252}
253
254void IntrinsicLocationsBuilderARM64::VisitIntegerReverseBytes(HInvoke* invoke) {
255  CreateIntToIntLocations(arena_, invoke);
256}
257
258void IntrinsicCodeGeneratorARM64::VisitIntegerReverseBytes(HInvoke* invoke) {
259  GenReverseBytes(invoke->GetLocations(), Primitive::kPrimInt, GetVIXLAssembler());
260}
261
262void IntrinsicLocationsBuilderARM64::VisitLongReverseBytes(HInvoke* invoke) {
263  CreateIntToIntLocations(arena_, invoke);
264}
265
266void IntrinsicCodeGeneratorARM64::VisitLongReverseBytes(HInvoke* invoke) {
267  GenReverseBytes(invoke->GetLocations(), Primitive::kPrimLong, GetVIXLAssembler());
268}
269
270void IntrinsicLocationsBuilderARM64::VisitShortReverseBytes(HInvoke* invoke) {
271  CreateIntToIntLocations(arena_, invoke);
272}
273
274void IntrinsicCodeGeneratorARM64::VisitShortReverseBytes(HInvoke* invoke) {
275  GenReverseBytes(invoke->GetLocations(), Primitive::kPrimShort, GetVIXLAssembler());
276}
277
278static void GenNumberOfLeadingZeros(LocationSummary* locations,
279                                    Primitive::Type type,
280                                    vixl::MacroAssembler* masm) {
281  DCHECK(type == Primitive::kPrimInt || type == Primitive::kPrimLong);
282
283  Location in = locations->InAt(0);
284  Location out = locations->Out();
285
286  __ Clz(RegisterFrom(out, type), RegisterFrom(in, type));
287}
288
289void IntrinsicLocationsBuilderARM64::VisitIntegerNumberOfLeadingZeros(HInvoke* invoke) {
290  CreateIntToIntLocations(arena_, invoke);
291}
292
293void IntrinsicCodeGeneratorARM64::VisitIntegerNumberOfLeadingZeros(HInvoke* invoke) {
294  GenNumberOfLeadingZeros(invoke->GetLocations(), Primitive::kPrimInt, GetVIXLAssembler());
295}
296
297void IntrinsicLocationsBuilderARM64::VisitLongNumberOfLeadingZeros(HInvoke* invoke) {
298  CreateIntToIntLocations(arena_, invoke);
299}
300
301void IntrinsicCodeGeneratorARM64::VisitLongNumberOfLeadingZeros(HInvoke* invoke) {
302  GenNumberOfLeadingZeros(invoke->GetLocations(), Primitive::kPrimLong, GetVIXLAssembler());
303}
304
305static void GenNumberOfTrailingZeros(LocationSummary* locations,
306                                     Primitive::Type type,
307                                     vixl::MacroAssembler* masm) {
308  DCHECK(type == Primitive::kPrimInt || type == Primitive::kPrimLong);
309
310  Location in = locations->InAt(0);
311  Location out = locations->Out();
312
313  __ Rbit(RegisterFrom(out, type), RegisterFrom(in, type));
314  __ Clz(RegisterFrom(out, type), RegisterFrom(out, type));
315}
316
317void IntrinsicLocationsBuilderARM64::VisitIntegerNumberOfTrailingZeros(HInvoke* invoke) {
318  CreateIntToIntLocations(arena_, invoke);
319}
320
321void IntrinsicCodeGeneratorARM64::VisitIntegerNumberOfTrailingZeros(HInvoke* invoke) {
322  GenNumberOfTrailingZeros(invoke->GetLocations(), Primitive::kPrimInt, GetVIXLAssembler());
323}
324
325void IntrinsicLocationsBuilderARM64::VisitLongNumberOfTrailingZeros(HInvoke* invoke) {
326  CreateIntToIntLocations(arena_, invoke);
327}
328
329void IntrinsicCodeGeneratorARM64::VisitLongNumberOfTrailingZeros(HInvoke* invoke) {
330  GenNumberOfTrailingZeros(invoke->GetLocations(), Primitive::kPrimLong, GetVIXLAssembler());
331}
332
333static void GenReverse(LocationSummary* locations,
334                       Primitive::Type type,
335                       vixl::MacroAssembler* masm) {
336  DCHECK(type == Primitive::kPrimInt || type == Primitive::kPrimLong);
337
338  Location in = locations->InAt(0);
339  Location out = locations->Out();
340
341  __ Rbit(RegisterFrom(out, type), RegisterFrom(in, type));
342}
343
344void IntrinsicLocationsBuilderARM64::VisitIntegerReverse(HInvoke* invoke) {
345  CreateIntToIntLocations(arena_, invoke);
346}
347
348void IntrinsicCodeGeneratorARM64::VisitIntegerReverse(HInvoke* invoke) {
349  GenReverse(invoke->GetLocations(), Primitive::kPrimInt, GetVIXLAssembler());
350}
351
352void IntrinsicLocationsBuilderARM64::VisitLongReverse(HInvoke* invoke) {
353  CreateIntToIntLocations(arena_, invoke);
354}
355
356void IntrinsicCodeGeneratorARM64::VisitLongReverse(HInvoke* invoke) {
357  GenReverse(invoke->GetLocations(), Primitive::kPrimLong, GetVIXLAssembler());
358}
359
360static void CreateFPToFPLocations(ArenaAllocator* arena, HInvoke* invoke) {
361  LocationSummary* locations = new (arena) LocationSummary(invoke,
362                                                           LocationSummary::kNoCall,
363                                                           kIntrinsified);
364  locations->SetInAt(0, Location::RequiresFpuRegister());
365  locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap);
366}
367
368static void MathAbsFP(LocationSummary* locations, bool is64bit, vixl::MacroAssembler* masm) {
369  Location in = locations->InAt(0);
370  Location out = locations->Out();
371
372  FPRegister in_reg = is64bit ? DRegisterFrom(in) : SRegisterFrom(in);
373  FPRegister out_reg = is64bit ? DRegisterFrom(out) : SRegisterFrom(out);
374
375  __ Fabs(out_reg, in_reg);
376}
377
378void IntrinsicLocationsBuilderARM64::VisitMathAbsDouble(HInvoke* invoke) {
379  CreateFPToFPLocations(arena_, invoke);
380}
381
382void IntrinsicCodeGeneratorARM64::VisitMathAbsDouble(HInvoke* invoke) {
383  MathAbsFP(invoke->GetLocations(), /* is64bit */ true, GetVIXLAssembler());
384}
385
386void IntrinsicLocationsBuilderARM64::VisitMathAbsFloat(HInvoke* invoke) {
387  CreateFPToFPLocations(arena_, invoke);
388}
389
390void IntrinsicCodeGeneratorARM64::VisitMathAbsFloat(HInvoke* invoke) {
391  MathAbsFP(invoke->GetLocations(), /* is64bit */ false, GetVIXLAssembler());
392}
393
394static void CreateIntToInt(ArenaAllocator* arena, HInvoke* invoke) {
395  LocationSummary* locations = new (arena) LocationSummary(invoke,
396                                                           LocationSummary::kNoCall,
397                                                           kIntrinsified);
398  locations->SetInAt(0, Location::RequiresRegister());
399  locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
400}
401
402static void GenAbsInteger(LocationSummary* locations,
403                          bool is64bit,
404                          vixl::MacroAssembler* masm) {
405  Location in = locations->InAt(0);
406  Location output = locations->Out();
407
408  Register in_reg = is64bit ? XRegisterFrom(in) : WRegisterFrom(in);
409  Register out_reg = is64bit ? XRegisterFrom(output) : WRegisterFrom(output);
410
411  __ Cmp(in_reg, Operand(0));
412  __ Cneg(out_reg, in_reg, lt);
413}
414
415void IntrinsicLocationsBuilderARM64::VisitMathAbsInt(HInvoke* invoke) {
416  CreateIntToInt(arena_, invoke);
417}
418
419void IntrinsicCodeGeneratorARM64::VisitMathAbsInt(HInvoke* invoke) {
420  GenAbsInteger(invoke->GetLocations(), /* is64bit */ false, GetVIXLAssembler());
421}
422
423void IntrinsicLocationsBuilderARM64::VisitMathAbsLong(HInvoke* invoke) {
424  CreateIntToInt(arena_, invoke);
425}
426
427void IntrinsicCodeGeneratorARM64::VisitMathAbsLong(HInvoke* invoke) {
428  GenAbsInteger(invoke->GetLocations(), /* is64bit */ true, GetVIXLAssembler());
429}
430
431static void GenMinMaxFP(LocationSummary* locations,
432                        bool is_min,
433                        bool is_double,
434                        vixl::MacroAssembler* masm) {
435  Location op1 = locations->InAt(0);
436  Location op2 = locations->InAt(1);
437  Location out = locations->Out();
438
439  FPRegister op1_reg = is_double ? DRegisterFrom(op1) : SRegisterFrom(op1);
440  FPRegister op2_reg = is_double ? DRegisterFrom(op2) : SRegisterFrom(op2);
441  FPRegister out_reg = is_double ? DRegisterFrom(out) : SRegisterFrom(out);
442  if (is_min) {
443    __ Fmin(out_reg, op1_reg, op2_reg);
444  } else {
445    __ Fmax(out_reg, op1_reg, op2_reg);
446  }
447}
448
449static void CreateFPFPToFPLocations(ArenaAllocator* arena, HInvoke* invoke) {
450  LocationSummary* locations = new (arena) LocationSummary(invoke,
451                                                           LocationSummary::kNoCall,
452                                                           kIntrinsified);
453  locations->SetInAt(0, Location::RequiresFpuRegister());
454  locations->SetInAt(1, Location::RequiresFpuRegister());
455  locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap);
456}
457
458void IntrinsicLocationsBuilderARM64::VisitMathMinDoubleDouble(HInvoke* invoke) {
459  CreateFPFPToFPLocations(arena_, invoke);
460}
461
462void IntrinsicCodeGeneratorARM64::VisitMathMinDoubleDouble(HInvoke* invoke) {
463  GenMinMaxFP(invoke->GetLocations(), /* is_min */ true, /* is_double */ true, GetVIXLAssembler());
464}
465
466void IntrinsicLocationsBuilderARM64::VisitMathMinFloatFloat(HInvoke* invoke) {
467  CreateFPFPToFPLocations(arena_, invoke);
468}
469
470void IntrinsicCodeGeneratorARM64::VisitMathMinFloatFloat(HInvoke* invoke) {
471  GenMinMaxFP(invoke->GetLocations(), /* is_min */ true, /* is_double */ false, GetVIXLAssembler());
472}
473
474void IntrinsicLocationsBuilderARM64::VisitMathMaxDoubleDouble(HInvoke* invoke) {
475  CreateFPFPToFPLocations(arena_, invoke);
476}
477
478void IntrinsicCodeGeneratorARM64::VisitMathMaxDoubleDouble(HInvoke* invoke) {
479  GenMinMaxFP(invoke->GetLocations(), /* is_min */ false, /* is_double */ true, GetVIXLAssembler());
480}
481
482void IntrinsicLocationsBuilderARM64::VisitMathMaxFloatFloat(HInvoke* invoke) {
483  CreateFPFPToFPLocations(arena_, invoke);
484}
485
486void IntrinsicCodeGeneratorARM64::VisitMathMaxFloatFloat(HInvoke* invoke) {
487  GenMinMaxFP(
488      invoke->GetLocations(), /* is_min */ false, /* is_double */ false, GetVIXLAssembler());
489}
490
491static void GenMinMax(LocationSummary* locations,
492                      bool is_min,
493                      bool is_long,
494                      vixl::MacroAssembler* masm) {
495  Location op1 = locations->InAt(0);
496  Location op2 = locations->InAt(1);
497  Location out = locations->Out();
498
499  Register op1_reg = is_long ? XRegisterFrom(op1) : WRegisterFrom(op1);
500  Register op2_reg = is_long ? XRegisterFrom(op2) : WRegisterFrom(op2);
501  Register out_reg = is_long ? XRegisterFrom(out) : WRegisterFrom(out);
502
503  __ Cmp(op1_reg, op2_reg);
504  __ Csel(out_reg, op1_reg, op2_reg, is_min ? lt : gt);
505}
506
507static void CreateIntIntToIntLocations(ArenaAllocator* arena, HInvoke* invoke) {
508  LocationSummary* locations = new (arena) LocationSummary(invoke,
509                                                           LocationSummary::kNoCall,
510                                                           kIntrinsified);
511  locations->SetInAt(0, Location::RequiresRegister());
512  locations->SetInAt(1, Location::RequiresRegister());
513  locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
514}
515
516void IntrinsicLocationsBuilderARM64::VisitMathMinIntInt(HInvoke* invoke) {
517  CreateIntIntToIntLocations(arena_, invoke);
518}
519
520void IntrinsicCodeGeneratorARM64::VisitMathMinIntInt(HInvoke* invoke) {
521  GenMinMax(invoke->GetLocations(), /* is_min */ true, /* is_long */ false, GetVIXLAssembler());
522}
523
524void IntrinsicLocationsBuilderARM64::VisitMathMinLongLong(HInvoke* invoke) {
525  CreateIntIntToIntLocations(arena_, invoke);
526}
527
528void IntrinsicCodeGeneratorARM64::VisitMathMinLongLong(HInvoke* invoke) {
529  GenMinMax(invoke->GetLocations(), /* is_min */ true, /* is_long */ true, GetVIXLAssembler());
530}
531
532void IntrinsicLocationsBuilderARM64::VisitMathMaxIntInt(HInvoke* invoke) {
533  CreateIntIntToIntLocations(arena_, invoke);
534}
535
536void IntrinsicCodeGeneratorARM64::VisitMathMaxIntInt(HInvoke* invoke) {
537  GenMinMax(invoke->GetLocations(), /* is_min */ false, /* is_long */ false, GetVIXLAssembler());
538}
539
540void IntrinsicLocationsBuilderARM64::VisitMathMaxLongLong(HInvoke* invoke) {
541  CreateIntIntToIntLocations(arena_, invoke);
542}
543
544void IntrinsicCodeGeneratorARM64::VisitMathMaxLongLong(HInvoke* invoke) {
545  GenMinMax(invoke->GetLocations(), /* is_min */ false, /* is_long */ true, GetVIXLAssembler());
546}
547
548void IntrinsicLocationsBuilderARM64::VisitMathSqrt(HInvoke* invoke) {
549  CreateFPToFPLocations(arena_, invoke);
550}
551
552void IntrinsicCodeGeneratorARM64::VisitMathSqrt(HInvoke* invoke) {
553  LocationSummary* locations = invoke->GetLocations();
554  vixl::MacroAssembler* masm = GetVIXLAssembler();
555  __ Fsqrt(DRegisterFrom(locations->Out()), DRegisterFrom(locations->InAt(0)));
556}
557
558void IntrinsicLocationsBuilderARM64::VisitMathCeil(HInvoke* invoke) {
559  CreateFPToFPLocations(arena_, invoke);
560}
561
562void IntrinsicCodeGeneratorARM64::VisitMathCeil(HInvoke* invoke) {
563  LocationSummary* locations = invoke->GetLocations();
564  vixl::MacroAssembler* masm = GetVIXLAssembler();
565  __ Frintp(DRegisterFrom(locations->Out()), DRegisterFrom(locations->InAt(0)));
566}
567
568void IntrinsicLocationsBuilderARM64::VisitMathFloor(HInvoke* invoke) {
569  CreateFPToFPLocations(arena_, invoke);
570}
571
572void IntrinsicCodeGeneratorARM64::VisitMathFloor(HInvoke* invoke) {
573  LocationSummary* locations = invoke->GetLocations();
574  vixl::MacroAssembler* masm = GetVIXLAssembler();
575  __ Frintm(DRegisterFrom(locations->Out()), DRegisterFrom(locations->InAt(0)));
576}
577
578void IntrinsicLocationsBuilderARM64::VisitMathRint(HInvoke* invoke) {
579  CreateFPToFPLocations(arena_, invoke);
580}
581
582void IntrinsicCodeGeneratorARM64::VisitMathRint(HInvoke* invoke) {
583  LocationSummary* locations = invoke->GetLocations();
584  vixl::MacroAssembler* masm = GetVIXLAssembler();
585  __ Frintn(DRegisterFrom(locations->Out()), DRegisterFrom(locations->InAt(0)));
586}
587
588static void CreateFPToIntPlusTempLocations(ArenaAllocator* arena, HInvoke* invoke) {
589  LocationSummary* locations = new (arena) LocationSummary(invoke,
590                                                           LocationSummary::kNoCall,
591                                                           kIntrinsified);
592  locations->SetInAt(0, Location::RequiresFpuRegister());
593  locations->SetOut(Location::RequiresRegister());
594}
595
596static void GenMathRound(LocationSummary* locations,
597                         bool is_double,
598                         vixl::MacroAssembler* masm) {
599  FPRegister in_reg = is_double ?
600      DRegisterFrom(locations->InAt(0)) : SRegisterFrom(locations->InAt(0));
601  Register out_reg = is_double ?
602      XRegisterFrom(locations->Out()) : WRegisterFrom(locations->Out());
603  UseScratchRegisterScope temps(masm);
604  FPRegister temp1_reg = temps.AcquireSameSizeAs(in_reg);
605
606  // 0.5 can be encoded as an immediate, so use fmov.
607  if (is_double) {
608    __ Fmov(temp1_reg, static_cast<double>(0.5));
609  } else {
610    __ Fmov(temp1_reg, static_cast<float>(0.5));
611  }
612  __ Fadd(temp1_reg, in_reg, temp1_reg);
613  __ Fcvtms(out_reg, temp1_reg);
614}
615
616void IntrinsicLocationsBuilderARM64::VisitMathRoundDouble(HInvoke* invoke) {
617  CreateFPToIntPlusTempLocations(arena_, invoke);
618}
619
620void IntrinsicCodeGeneratorARM64::VisitMathRoundDouble(HInvoke* invoke) {
621  GenMathRound(invoke->GetLocations(), /* is_double */ true, GetVIXLAssembler());
622}
623
624void IntrinsicLocationsBuilderARM64::VisitMathRoundFloat(HInvoke* invoke) {
625  CreateFPToIntPlusTempLocations(arena_, invoke);
626}
627
628void IntrinsicCodeGeneratorARM64::VisitMathRoundFloat(HInvoke* invoke) {
629  GenMathRound(invoke->GetLocations(), /* is_double */ false, GetVIXLAssembler());
630}
631
632void IntrinsicLocationsBuilderARM64::VisitMemoryPeekByte(HInvoke* invoke) {
633  CreateIntToIntLocations(arena_, invoke);
634}
635
636void IntrinsicCodeGeneratorARM64::VisitMemoryPeekByte(HInvoke* invoke) {
637  vixl::MacroAssembler* masm = GetVIXLAssembler();
638  __ Ldrsb(WRegisterFrom(invoke->GetLocations()->Out()),
639          AbsoluteHeapOperandFrom(invoke->GetLocations()->InAt(0), 0));
640}
641
642void IntrinsicLocationsBuilderARM64::VisitMemoryPeekIntNative(HInvoke* invoke) {
643  CreateIntToIntLocations(arena_, invoke);
644}
645
646void IntrinsicCodeGeneratorARM64::VisitMemoryPeekIntNative(HInvoke* invoke) {
647  vixl::MacroAssembler* masm = GetVIXLAssembler();
648  __ Ldr(WRegisterFrom(invoke->GetLocations()->Out()),
649         AbsoluteHeapOperandFrom(invoke->GetLocations()->InAt(0), 0));
650}
651
652void IntrinsicLocationsBuilderARM64::VisitMemoryPeekLongNative(HInvoke* invoke) {
653  CreateIntToIntLocations(arena_, invoke);
654}
655
656void IntrinsicCodeGeneratorARM64::VisitMemoryPeekLongNative(HInvoke* invoke) {
657  vixl::MacroAssembler* masm = GetVIXLAssembler();
658  __ Ldr(XRegisterFrom(invoke->GetLocations()->Out()),
659         AbsoluteHeapOperandFrom(invoke->GetLocations()->InAt(0), 0));
660}
661
662void IntrinsicLocationsBuilderARM64::VisitMemoryPeekShortNative(HInvoke* invoke) {
663  CreateIntToIntLocations(arena_, invoke);
664}
665
666void IntrinsicCodeGeneratorARM64::VisitMemoryPeekShortNative(HInvoke* invoke) {
667  vixl::MacroAssembler* masm = GetVIXLAssembler();
668  __ Ldrsh(WRegisterFrom(invoke->GetLocations()->Out()),
669           AbsoluteHeapOperandFrom(invoke->GetLocations()->InAt(0), 0));
670}
671
672static void CreateIntIntToVoidLocations(ArenaAllocator* arena, HInvoke* invoke) {
673  LocationSummary* locations = new (arena) LocationSummary(invoke,
674                                                           LocationSummary::kNoCall,
675                                                           kIntrinsified);
676  locations->SetInAt(0, Location::RequiresRegister());
677  locations->SetInAt(1, Location::RequiresRegister());
678}
679
680void IntrinsicLocationsBuilderARM64::VisitMemoryPokeByte(HInvoke* invoke) {
681  CreateIntIntToVoidLocations(arena_, invoke);
682}
683
684void IntrinsicCodeGeneratorARM64::VisitMemoryPokeByte(HInvoke* invoke) {
685  vixl::MacroAssembler* masm = GetVIXLAssembler();
686  __ Strb(WRegisterFrom(invoke->GetLocations()->InAt(1)),
687          AbsoluteHeapOperandFrom(invoke->GetLocations()->InAt(0), 0));
688}
689
690void IntrinsicLocationsBuilderARM64::VisitMemoryPokeIntNative(HInvoke* invoke) {
691  CreateIntIntToVoidLocations(arena_, invoke);
692}
693
694void IntrinsicCodeGeneratorARM64::VisitMemoryPokeIntNative(HInvoke* invoke) {
695  vixl::MacroAssembler* masm = GetVIXLAssembler();
696  __ Str(WRegisterFrom(invoke->GetLocations()->InAt(1)),
697         AbsoluteHeapOperandFrom(invoke->GetLocations()->InAt(0), 0));
698}
699
700void IntrinsicLocationsBuilderARM64::VisitMemoryPokeLongNative(HInvoke* invoke) {
701  CreateIntIntToVoidLocations(arena_, invoke);
702}
703
704void IntrinsicCodeGeneratorARM64::VisitMemoryPokeLongNative(HInvoke* invoke) {
705  vixl::MacroAssembler* masm = GetVIXLAssembler();
706  __ Str(XRegisterFrom(invoke->GetLocations()->InAt(1)),
707         AbsoluteHeapOperandFrom(invoke->GetLocations()->InAt(0), 0));
708}
709
710void IntrinsicLocationsBuilderARM64::VisitMemoryPokeShortNative(HInvoke* invoke) {
711  CreateIntIntToVoidLocations(arena_, invoke);
712}
713
714void IntrinsicCodeGeneratorARM64::VisitMemoryPokeShortNative(HInvoke* invoke) {
715  vixl::MacroAssembler* masm = GetVIXLAssembler();
716  __ Strh(WRegisterFrom(invoke->GetLocations()->InAt(1)),
717          AbsoluteHeapOperandFrom(invoke->GetLocations()->InAt(0), 0));
718}
719
720void IntrinsicLocationsBuilderARM64::VisitThreadCurrentThread(HInvoke* invoke) {
721  LocationSummary* locations = new (arena_) LocationSummary(invoke,
722                                                            LocationSummary::kNoCall,
723                                                            kIntrinsified);
724  locations->SetOut(Location::RequiresRegister());
725}
726
727void IntrinsicCodeGeneratorARM64::VisitThreadCurrentThread(HInvoke* invoke) {
728  codegen_->Load(Primitive::kPrimNot, WRegisterFrom(invoke->GetLocations()->Out()),
729                 MemOperand(tr, Thread::PeerOffset<8>().Int32Value()));
730}
731
732static void GenUnsafeGet(HInvoke* invoke,
733                         Primitive::Type type,
734                         bool is_volatile,
735                         CodeGeneratorARM64* codegen) {
736  LocationSummary* locations = invoke->GetLocations();
737  DCHECK((type == Primitive::kPrimInt) ||
738         (type == Primitive::kPrimLong) ||
739         (type == Primitive::kPrimNot));
740  vixl::MacroAssembler* masm = codegen->GetAssembler()->vixl_masm_;
741  Location base_loc = locations->InAt(1);
742  Register base = WRegisterFrom(base_loc);      // Object pointer.
743  Location offset_loc = locations->InAt(2);
744  Register offset = XRegisterFrom(offset_loc);  // Long offset.
745  Location trg_loc = locations->Out();
746  Register trg = RegisterFrom(trg_loc, type);
747  bool use_acquire_release = codegen->GetInstructionSetFeatures().PreferAcquireRelease();
748
749  MemOperand mem_op(base.X(), offset);
750  if (is_volatile) {
751    if (use_acquire_release) {
752      codegen->LoadAcquire(invoke, trg, mem_op);
753    } else {
754      codegen->Load(type, trg, mem_op);
755      __ Dmb(InnerShareable, BarrierReads);
756    }
757  } else {
758    codegen->Load(type, trg, mem_op);
759  }
760
761  if (type == Primitive::kPrimNot) {
762    DCHECK(trg.IsW());
763    codegen->MaybeGenerateReadBarrier(invoke, trg_loc, trg_loc, base_loc, 0U, offset_loc);
764  }
765}
766
767static void CreateIntIntIntToIntLocations(ArenaAllocator* arena, HInvoke* invoke) {
768  bool can_call = kEmitCompilerReadBarrier &&
769      (invoke->GetIntrinsic() == Intrinsics::kUnsafeGetObject ||
770       invoke->GetIntrinsic() == Intrinsics::kUnsafeGetObjectVolatile);
771  LocationSummary* locations = new (arena) LocationSummary(invoke,
772                                                           can_call ?
773                                                               LocationSummary::kCallOnSlowPath :
774                                                               LocationSummary::kNoCall,
775                                                           kIntrinsified);
776  locations->SetInAt(0, Location::NoLocation());        // Unused receiver.
777  locations->SetInAt(1, Location::RequiresRegister());
778  locations->SetInAt(2, Location::RequiresRegister());
779  locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
780}
781
782void IntrinsicLocationsBuilderARM64::VisitUnsafeGet(HInvoke* invoke) {
783  CreateIntIntIntToIntLocations(arena_, invoke);
784}
785void IntrinsicLocationsBuilderARM64::VisitUnsafeGetVolatile(HInvoke* invoke) {
786  CreateIntIntIntToIntLocations(arena_, invoke);
787}
788void IntrinsicLocationsBuilderARM64::VisitUnsafeGetLong(HInvoke* invoke) {
789  CreateIntIntIntToIntLocations(arena_, invoke);
790}
791void IntrinsicLocationsBuilderARM64::VisitUnsafeGetLongVolatile(HInvoke* invoke) {
792  CreateIntIntIntToIntLocations(arena_, invoke);
793}
794void IntrinsicLocationsBuilderARM64::VisitUnsafeGetObject(HInvoke* invoke) {
795  CreateIntIntIntToIntLocations(arena_, invoke);
796}
797void IntrinsicLocationsBuilderARM64::VisitUnsafeGetObjectVolatile(HInvoke* invoke) {
798  CreateIntIntIntToIntLocations(arena_, invoke);
799}
800
801void IntrinsicCodeGeneratorARM64::VisitUnsafeGet(HInvoke* invoke) {
802  GenUnsafeGet(invoke, Primitive::kPrimInt, /* is_volatile */ false, codegen_);
803}
804void IntrinsicCodeGeneratorARM64::VisitUnsafeGetVolatile(HInvoke* invoke) {
805  GenUnsafeGet(invoke, Primitive::kPrimInt, /* is_volatile */ true, codegen_);
806}
807void IntrinsicCodeGeneratorARM64::VisitUnsafeGetLong(HInvoke* invoke) {
808  GenUnsafeGet(invoke, Primitive::kPrimLong, /* is_volatile */ false, codegen_);
809}
810void IntrinsicCodeGeneratorARM64::VisitUnsafeGetLongVolatile(HInvoke* invoke) {
811  GenUnsafeGet(invoke, Primitive::kPrimLong, /* is_volatile */ true, codegen_);
812}
813void IntrinsicCodeGeneratorARM64::VisitUnsafeGetObject(HInvoke* invoke) {
814  GenUnsafeGet(invoke, Primitive::kPrimNot, /* is_volatile */ false, codegen_);
815}
816void IntrinsicCodeGeneratorARM64::VisitUnsafeGetObjectVolatile(HInvoke* invoke) {
817  GenUnsafeGet(invoke, Primitive::kPrimNot, /* is_volatile */ true, codegen_);
818}
819
820static void CreateIntIntIntIntToVoid(ArenaAllocator* arena, HInvoke* invoke) {
821  LocationSummary* locations = new (arena) LocationSummary(invoke,
822                                                           LocationSummary::kNoCall,
823                                                           kIntrinsified);
824  locations->SetInAt(0, Location::NoLocation());        // Unused receiver.
825  locations->SetInAt(1, Location::RequiresRegister());
826  locations->SetInAt(2, Location::RequiresRegister());
827  locations->SetInAt(3, Location::RequiresRegister());
828}
829
830void IntrinsicLocationsBuilderARM64::VisitUnsafePut(HInvoke* invoke) {
831  CreateIntIntIntIntToVoid(arena_, invoke);
832}
833void IntrinsicLocationsBuilderARM64::VisitUnsafePutOrdered(HInvoke* invoke) {
834  CreateIntIntIntIntToVoid(arena_, invoke);
835}
836void IntrinsicLocationsBuilderARM64::VisitUnsafePutVolatile(HInvoke* invoke) {
837  CreateIntIntIntIntToVoid(arena_, invoke);
838}
839void IntrinsicLocationsBuilderARM64::VisitUnsafePutObject(HInvoke* invoke) {
840  CreateIntIntIntIntToVoid(arena_, invoke);
841}
842void IntrinsicLocationsBuilderARM64::VisitUnsafePutObjectOrdered(HInvoke* invoke) {
843  CreateIntIntIntIntToVoid(arena_, invoke);
844}
845void IntrinsicLocationsBuilderARM64::VisitUnsafePutObjectVolatile(HInvoke* invoke) {
846  CreateIntIntIntIntToVoid(arena_, invoke);
847}
848void IntrinsicLocationsBuilderARM64::VisitUnsafePutLong(HInvoke* invoke) {
849  CreateIntIntIntIntToVoid(arena_, invoke);
850}
851void IntrinsicLocationsBuilderARM64::VisitUnsafePutLongOrdered(HInvoke* invoke) {
852  CreateIntIntIntIntToVoid(arena_, invoke);
853}
854void IntrinsicLocationsBuilderARM64::VisitUnsafePutLongVolatile(HInvoke* invoke) {
855  CreateIntIntIntIntToVoid(arena_, invoke);
856}
857
858static void GenUnsafePut(LocationSummary* locations,
859                         Primitive::Type type,
860                         bool is_volatile,
861                         bool is_ordered,
862                         CodeGeneratorARM64* codegen) {
863  vixl::MacroAssembler* masm = codegen->GetAssembler()->vixl_masm_;
864
865  Register base = WRegisterFrom(locations->InAt(1));    // Object pointer.
866  Register offset = XRegisterFrom(locations->InAt(2));  // Long offset.
867  Register value = RegisterFrom(locations->InAt(3), type);
868  Register source = value;
869  bool use_acquire_release = codegen->GetInstructionSetFeatures().PreferAcquireRelease();
870
871  MemOperand mem_op(base.X(), offset);
872
873  {
874    // We use a block to end the scratch scope before the write barrier, thus
875    // freeing the temporary registers so they can be used in `MarkGCCard`.
876    UseScratchRegisterScope temps(masm);
877
878    if (kPoisonHeapReferences && type == Primitive::kPrimNot) {
879      DCHECK(value.IsW());
880      Register temp = temps.AcquireW();
881      __ Mov(temp.W(), value.W());
882      codegen->GetAssembler()->PoisonHeapReference(temp.W());
883      source = temp;
884    }
885
886    if (is_volatile || is_ordered) {
887      if (use_acquire_release) {
888        codegen->StoreRelease(type, source, mem_op);
889      } else {
890        __ Dmb(InnerShareable, BarrierAll);
891        codegen->Store(type, source, mem_op);
892        if (is_volatile) {
893          __ Dmb(InnerShareable, BarrierReads);
894        }
895      }
896    } else {
897      codegen->Store(type, source, mem_op);
898    }
899  }
900
901  if (type == Primitive::kPrimNot) {
902    bool value_can_be_null = true;  // TODO: Worth finding out this information?
903    codegen->MarkGCCard(base, value, value_can_be_null);
904  }
905}
906
907void IntrinsicCodeGeneratorARM64::VisitUnsafePut(HInvoke* invoke) {
908  GenUnsafePut(invoke->GetLocations(),
909               Primitive::kPrimInt,
910               /* is_volatile */ false,
911               /* is_ordered */ false,
912               codegen_);
913}
914void IntrinsicCodeGeneratorARM64::VisitUnsafePutOrdered(HInvoke* invoke) {
915  GenUnsafePut(invoke->GetLocations(),
916               Primitive::kPrimInt,
917               /* is_volatile */ false,
918               /* is_ordered */ true,
919               codegen_);
920}
921void IntrinsicCodeGeneratorARM64::VisitUnsafePutVolatile(HInvoke* invoke) {
922  GenUnsafePut(invoke->GetLocations(),
923               Primitive::kPrimInt,
924               /* is_volatile */ true,
925               /* is_ordered */ false,
926               codegen_);
927}
928void IntrinsicCodeGeneratorARM64::VisitUnsafePutObject(HInvoke* invoke) {
929  GenUnsafePut(invoke->GetLocations(),
930               Primitive::kPrimNot,
931               /* is_volatile */ false,
932               /* is_ordered */ false,
933               codegen_);
934}
935void IntrinsicCodeGeneratorARM64::VisitUnsafePutObjectOrdered(HInvoke* invoke) {
936  GenUnsafePut(invoke->GetLocations(),
937               Primitive::kPrimNot,
938               /* is_volatile */ false,
939               /* is_ordered */ true,
940               codegen_);
941}
942void IntrinsicCodeGeneratorARM64::VisitUnsafePutObjectVolatile(HInvoke* invoke) {
943  GenUnsafePut(invoke->GetLocations(),
944               Primitive::kPrimNot,
945               /* is_volatile */ true,
946               /* is_ordered */ false,
947               codegen_);
948}
949void IntrinsicCodeGeneratorARM64::VisitUnsafePutLong(HInvoke* invoke) {
950  GenUnsafePut(invoke->GetLocations(),
951               Primitive::kPrimLong,
952               /* is_volatile */ false,
953               /* is_ordered */ false,
954               codegen_);
955}
956void IntrinsicCodeGeneratorARM64::VisitUnsafePutLongOrdered(HInvoke* invoke) {
957  GenUnsafePut(invoke->GetLocations(),
958               Primitive::kPrimLong,
959               /* is_volatile */ false,
960               /* is_ordered */ true,
961               codegen_);
962}
963void IntrinsicCodeGeneratorARM64::VisitUnsafePutLongVolatile(HInvoke* invoke) {
964  GenUnsafePut(invoke->GetLocations(),
965               Primitive::kPrimLong,
966               /* is_volatile */ true,
967               /* is_ordered */ false,
968               codegen_);
969}
970
971static void CreateIntIntIntIntIntToInt(ArenaAllocator* arena, HInvoke* invoke) {
972  LocationSummary* locations = new (arena) LocationSummary(invoke,
973                                                           LocationSummary::kNoCall,
974                                                           kIntrinsified);
975  locations->SetInAt(0, Location::NoLocation());        // Unused receiver.
976  locations->SetInAt(1, Location::RequiresRegister());
977  locations->SetInAt(2, Location::RequiresRegister());
978  locations->SetInAt(3, Location::RequiresRegister());
979  locations->SetInAt(4, Location::RequiresRegister());
980
981  locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
982}
983
984static void GenCas(LocationSummary* locations, Primitive::Type type, CodeGeneratorARM64* codegen) {
985  bool use_acquire_release = codegen->GetInstructionSetFeatures().PreferAcquireRelease();
986  vixl::MacroAssembler* masm = codegen->GetAssembler()->vixl_masm_;
987
988  Register out = WRegisterFrom(locations->Out());                  // Boolean result.
989
990  Register base = WRegisterFrom(locations->InAt(1));               // Object pointer.
991  Register offset = XRegisterFrom(locations->InAt(2));             // Long offset.
992  Register expected = RegisterFrom(locations->InAt(3), type);      // Expected.
993  Register value = RegisterFrom(locations->InAt(4), type);         // Value.
994
995  // This needs to be before the temp registers, as MarkGCCard also uses VIXL temps.
996  if (type == Primitive::kPrimNot) {
997    // Mark card for object assuming new value is stored.
998    bool value_can_be_null = true;  // TODO: Worth finding out this information?
999    codegen->MarkGCCard(base, value, value_can_be_null);
1000  }
1001
1002  UseScratchRegisterScope temps(masm);
1003  Register tmp_ptr = temps.AcquireX();                             // Pointer to actual memory.
1004  Register tmp_value = temps.AcquireSameSizeAs(value);             // Value in memory.
1005
1006  Register tmp_32 = tmp_value.W();
1007
1008  __ Add(tmp_ptr, base.X(), Operand(offset));
1009
1010  if (kPoisonHeapReferences && type == Primitive::kPrimNot) {
1011    codegen->GetAssembler()->PoisonHeapReference(expected);
1012    codegen->GetAssembler()->PoisonHeapReference(value);
1013  }
1014
1015  // do {
1016  //   tmp_value = [tmp_ptr] - expected;
1017  // } while (tmp_value == 0 && failure([tmp_ptr] <- r_new_value));
1018  // result = tmp_value != 0;
1019
1020  vixl::Label loop_head, exit_loop;
1021  if (use_acquire_release) {
1022    __ Bind(&loop_head);
1023    __ Ldaxr(tmp_value, MemOperand(tmp_ptr));
1024    // TODO: Do we need a read barrier here when `type == Primitive::kPrimNot`?
1025    // Note that this code is not (yet) used when read barriers are
1026    // enabled (see IntrinsicLocationsBuilderARM64::VisitUnsafeCASObject).
1027    __ Cmp(tmp_value, expected);
1028    __ B(&exit_loop, ne);
1029    __ Stlxr(tmp_32, value, MemOperand(tmp_ptr));
1030    __ Cbnz(tmp_32, &loop_head);
1031  } else {
1032    __ Dmb(InnerShareable, BarrierWrites);
1033    __ Bind(&loop_head);
1034    // TODO: When `type == Primitive::kPrimNot`, add a read barrier for
1035    // the reference stored in the object before attempting the CAS,
1036    // similar to the one in the art::Unsafe_compareAndSwapObject JNI
1037    // implementation.
1038    //
1039    // Note that this code is not (yet) used when read barriers are
1040    // enabled (see IntrinsicLocationsBuilderARM64::VisitUnsafeCASObject).
1041    DCHECK(!(type == Primitive::kPrimNot && kEmitCompilerReadBarrier));
1042    __ Ldxr(tmp_value, MemOperand(tmp_ptr));
1043    __ Cmp(tmp_value, expected);
1044    __ B(&exit_loop, ne);
1045    __ Stxr(tmp_32, value, MemOperand(tmp_ptr));
1046    __ Cbnz(tmp_32, &loop_head);
1047    __ Dmb(InnerShareable, BarrierAll);
1048  }
1049  __ Bind(&exit_loop);
1050  __ Cset(out, eq);
1051
1052  if (kPoisonHeapReferences && type == Primitive::kPrimNot) {
1053    codegen->GetAssembler()->UnpoisonHeapReference(value);
1054    codegen->GetAssembler()->UnpoisonHeapReference(expected);
1055  }
1056}
1057
1058void IntrinsicLocationsBuilderARM64::VisitUnsafeCASInt(HInvoke* invoke) {
1059  CreateIntIntIntIntIntToInt(arena_, invoke);
1060}
1061void IntrinsicLocationsBuilderARM64::VisitUnsafeCASLong(HInvoke* invoke) {
1062  CreateIntIntIntIntIntToInt(arena_, invoke);
1063}
1064void IntrinsicLocationsBuilderARM64::VisitUnsafeCASObject(HInvoke* invoke) {
1065  // The UnsafeCASObject intrinsic is missing a read barrier, and
1066  // therefore sometimes does not work as expected (b/25883050).
1067  // Turn it off temporarily as a quick fix, until the read barrier is
1068  // implemented (see TODO in GenCAS below).
1069  //
1070  // Also, the UnsafeCASObject intrinsic does not always work when heap
1071  // poisoning is enabled (it breaks run-test 004-UnsafeTest); turn it
1072  // off temporarily as a quick fix (b/26204023).
1073  //
1074  // TODO(rpl): Fix these two issues and re-enable this intrinsic.
1075  if (kEmitCompilerReadBarrier || kPoisonHeapReferences) {
1076    return;
1077  }
1078
1079  CreateIntIntIntIntIntToInt(arena_, invoke);
1080}
1081
1082void IntrinsicCodeGeneratorARM64::VisitUnsafeCASInt(HInvoke* invoke) {
1083  GenCas(invoke->GetLocations(), Primitive::kPrimInt, codegen_);
1084}
1085void IntrinsicCodeGeneratorARM64::VisitUnsafeCASLong(HInvoke* invoke) {
1086  GenCas(invoke->GetLocations(), Primitive::kPrimLong, codegen_);
1087}
1088void IntrinsicCodeGeneratorARM64::VisitUnsafeCASObject(HInvoke* invoke) {
1089  GenCas(invoke->GetLocations(), Primitive::kPrimNot, codegen_);
1090}
1091
1092void IntrinsicLocationsBuilderARM64::VisitStringCharAt(HInvoke* invoke) {
1093  LocationSummary* locations = new (arena_) LocationSummary(invoke,
1094                                                            LocationSummary::kCallOnSlowPath,
1095                                                            kIntrinsified);
1096  locations->SetInAt(0, Location::RequiresRegister());
1097  locations->SetInAt(1, Location::RequiresRegister());
1098  // In case we need to go in the slow path, we can't have the output be the same
1099  // as the input: the current liveness analysis considers the input to be live
1100  // at the point of the call.
1101  locations->SetOut(Location::RequiresRegister(), Location::kOutputOverlap);
1102}
1103
1104void IntrinsicCodeGeneratorARM64::VisitStringCharAt(HInvoke* invoke) {
1105  vixl::MacroAssembler* masm = GetVIXLAssembler();
1106  LocationSummary* locations = invoke->GetLocations();
1107
1108  // Location of reference to data array
1109  const MemberOffset value_offset = mirror::String::ValueOffset();
1110  // Location of count
1111  const MemberOffset count_offset = mirror::String::CountOffset();
1112
1113  Register obj = WRegisterFrom(locations->InAt(0));  // String object pointer.
1114  Register idx = WRegisterFrom(locations->InAt(1));  // Index of character.
1115  Register out = WRegisterFrom(locations->Out());    // Result character.
1116
1117  UseScratchRegisterScope temps(masm);
1118  Register temp = temps.AcquireW();
1119  Register array_temp = temps.AcquireW();            // We can trade this for worse scheduling.
1120
1121  // TODO: Maybe we can support range check elimination. Overall, though, I think it's not worth
1122  //       the cost.
1123  // TODO: For simplicity, the index parameter is requested in a register, so different from Quick
1124  //       we will not optimize the code for constants (which would save a register).
1125
1126  SlowPathCodeARM64* slow_path = new (GetAllocator()) IntrinsicSlowPathARM64(invoke);
1127  codegen_->AddSlowPath(slow_path);
1128
1129  __ Ldr(temp, HeapOperand(obj, count_offset));          // temp = str.length.
1130  codegen_->MaybeRecordImplicitNullCheck(invoke);
1131  __ Cmp(idx, temp);
1132  __ B(hs, slow_path->GetEntryLabel());
1133
1134  __ Add(array_temp, obj, Operand(value_offset.Int32Value()));  // array_temp := str.value.
1135
1136  // Load the value.
1137  __ Ldrh(out, MemOperand(array_temp.X(), idx, UXTW, 1));  // out := array_temp[idx].
1138
1139  __ Bind(slow_path->GetExitLabel());
1140}
1141
1142void IntrinsicLocationsBuilderARM64::VisitStringCompareTo(HInvoke* invoke) {
1143  LocationSummary* locations = new (arena_) LocationSummary(invoke,
1144                                                            LocationSummary::kCall,
1145                                                            kIntrinsified);
1146  InvokeRuntimeCallingConvention calling_convention;
1147  locations->SetInAt(0, LocationFrom(calling_convention.GetRegisterAt(0)));
1148  locations->SetInAt(1, LocationFrom(calling_convention.GetRegisterAt(1)));
1149  locations->SetOut(calling_convention.GetReturnLocation(Primitive::kPrimInt));
1150}
1151
1152void IntrinsicCodeGeneratorARM64::VisitStringCompareTo(HInvoke* invoke) {
1153  vixl::MacroAssembler* masm = GetVIXLAssembler();
1154  LocationSummary* locations = invoke->GetLocations();
1155
1156  // Note that the null check must have been done earlier.
1157  DCHECK(!invoke->CanDoImplicitNullCheckOn(invoke->InputAt(0)));
1158
1159  Register argument = WRegisterFrom(locations->InAt(1));
1160  __ Cmp(argument, 0);
1161  SlowPathCodeARM64* slow_path = new (GetAllocator()) IntrinsicSlowPathARM64(invoke);
1162  codegen_->AddSlowPath(slow_path);
1163  __ B(eq, slow_path->GetEntryLabel());
1164
1165  __ Ldr(
1166      lr, MemOperand(tr, QUICK_ENTRYPOINT_OFFSET(kArm64WordSize, pStringCompareTo).Int32Value()));
1167  __ Blr(lr);
1168  __ Bind(slow_path->GetExitLabel());
1169}
1170
1171void IntrinsicLocationsBuilderARM64::VisitStringEquals(HInvoke* invoke) {
1172  LocationSummary* locations = new (arena_) LocationSummary(invoke,
1173                                                            LocationSummary::kNoCall,
1174                                                            kIntrinsified);
1175  locations->SetInAt(0, Location::RequiresRegister());
1176  locations->SetInAt(1, Location::RequiresRegister());
1177  // Temporary registers to store lengths of strings and for calculations.
1178  locations->AddTemp(Location::RequiresRegister());
1179  locations->AddTemp(Location::RequiresRegister());
1180
1181  locations->SetOut(Location::RequiresRegister(), Location::kOutputOverlap);
1182}
1183
1184void IntrinsicCodeGeneratorARM64::VisitStringEquals(HInvoke* invoke) {
1185  vixl::MacroAssembler* masm = GetVIXLAssembler();
1186  LocationSummary* locations = invoke->GetLocations();
1187
1188  Register str = WRegisterFrom(locations->InAt(0));
1189  Register arg = WRegisterFrom(locations->InAt(1));
1190  Register out = XRegisterFrom(locations->Out());
1191
1192  UseScratchRegisterScope scratch_scope(masm);
1193  Register temp = scratch_scope.AcquireW();
1194  Register temp1 = WRegisterFrom(locations->GetTemp(0));
1195  Register temp2 = WRegisterFrom(locations->GetTemp(1));
1196
1197  vixl::Label loop;
1198  vixl::Label end;
1199  vixl::Label return_true;
1200  vixl::Label return_false;
1201
1202  // Get offsets of count, value, and class fields within a string object.
1203  const int32_t count_offset = mirror::String::CountOffset().Int32Value();
1204  const int32_t value_offset = mirror::String::ValueOffset().Int32Value();
1205  const int32_t class_offset = mirror::Object::ClassOffset().Int32Value();
1206
1207  // Note that the null check must have been done earlier.
1208  DCHECK(!invoke->CanDoImplicitNullCheckOn(invoke->InputAt(0)));
1209
1210  // Check if input is null, return false if it is.
1211  __ Cbz(arg, &return_false);
1212
1213  // Reference equality check, return true if same reference.
1214  __ Cmp(str, arg);
1215  __ B(&return_true, eq);
1216
1217  // Instanceof check for the argument by comparing class fields.
1218  // All string objects must have the same type since String cannot be subclassed.
1219  // Receiver must be a string object, so its class field is equal to all strings' class fields.
1220  // If the argument is a string object, its class field must be equal to receiver's class field.
1221  __ Ldr(temp, MemOperand(str.X(), class_offset));
1222  __ Ldr(temp1, MemOperand(arg.X(), class_offset));
1223  __ Cmp(temp, temp1);
1224  __ B(&return_false, ne);
1225
1226  // Load lengths of this and argument strings.
1227  __ Ldr(temp, MemOperand(str.X(), count_offset));
1228  __ Ldr(temp1, MemOperand(arg.X(), count_offset));
1229  // Check if lengths are equal, return false if they're not.
1230  __ Cmp(temp, temp1);
1231  __ B(&return_false, ne);
1232  // Store offset of string value in preparation for comparison loop
1233  __ Mov(temp1, value_offset);
1234  // Return true if both strings are empty.
1235  __ Cbz(temp, &return_true);
1236
1237  // Assertions that must hold in order to compare strings 4 characters at a time.
1238  DCHECK_ALIGNED(value_offset, 8);
1239  static_assert(IsAligned<8>(kObjectAlignment), "String of odd length is not zero padded");
1240
1241  temp1 = temp1.X();
1242  temp2 = temp2.X();
1243
1244  // Loop to compare strings 4 characters at a time starting at the beginning of the string.
1245  // Ok to do this because strings are zero-padded to be 8-byte aligned.
1246  __ Bind(&loop);
1247  __ Ldr(out, MemOperand(str.X(), temp1));
1248  __ Ldr(temp2, MemOperand(arg.X(), temp1));
1249  __ Add(temp1, temp1, Operand(sizeof(uint64_t)));
1250  __ Cmp(out, temp2);
1251  __ B(&return_false, ne);
1252  __ Sub(temp, temp, Operand(4), SetFlags);
1253  __ B(&loop, gt);
1254
1255  // Return true and exit the function.
1256  // If loop does not result in returning false, we return true.
1257  __ Bind(&return_true);
1258  __ Mov(out, 1);
1259  __ B(&end);
1260
1261  // Return false and exit the function.
1262  __ Bind(&return_false);
1263  __ Mov(out, 0);
1264  __ Bind(&end);
1265}
1266
1267static void GenerateVisitStringIndexOf(HInvoke* invoke,
1268                                       vixl::MacroAssembler* masm,
1269                                       CodeGeneratorARM64* codegen,
1270                                       ArenaAllocator* allocator,
1271                                       bool start_at_zero) {
1272  LocationSummary* locations = invoke->GetLocations();
1273  Register tmp_reg = WRegisterFrom(locations->GetTemp(0));
1274
1275  // Note that the null check must have been done earlier.
1276  DCHECK(!invoke->CanDoImplicitNullCheckOn(invoke->InputAt(0)));
1277
1278  // Check for code points > 0xFFFF. Either a slow-path check when we don't know statically,
1279  // or directly dispatch if we have a constant.
1280  SlowPathCodeARM64* slow_path = nullptr;
1281  if (invoke->InputAt(1)->IsIntConstant()) {
1282    if (static_cast<uint32_t>(invoke->InputAt(1)->AsIntConstant()->GetValue()) > 0xFFFFU) {
1283      // Always needs the slow-path. We could directly dispatch to it, but this case should be
1284      // rare, so for simplicity just put the full slow-path down and branch unconditionally.
1285      slow_path = new (allocator) IntrinsicSlowPathARM64(invoke);
1286      codegen->AddSlowPath(slow_path);
1287      __ B(slow_path->GetEntryLabel());
1288      __ Bind(slow_path->GetExitLabel());
1289      return;
1290    }
1291  } else {
1292    Register char_reg = WRegisterFrom(locations->InAt(1));
1293    __ Mov(tmp_reg, 0xFFFF);
1294    __ Cmp(char_reg, Operand(tmp_reg));
1295    slow_path = new (allocator) IntrinsicSlowPathARM64(invoke);
1296    codegen->AddSlowPath(slow_path);
1297    __ B(hi, slow_path->GetEntryLabel());
1298  }
1299
1300  if (start_at_zero) {
1301    // Start-index = 0.
1302    __ Mov(tmp_reg, 0);
1303  }
1304
1305  __ Ldr(lr, MemOperand(tr, QUICK_ENTRYPOINT_OFFSET(kArm64WordSize, pIndexOf).Int32Value()));
1306  __ Blr(lr);
1307
1308  if (slow_path != nullptr) {
1309    __ Bind(slow_path->GetExitLabel());
1310  }
1311}
1312
1313void IntrinsicLocationsBuilderARM64::VisitStringIndexOf(HInvoke* invoke) {
1314  LocationSummary* locations = new (arena_) LocationSummary(invoke,
1315                                                            LocationSummary::kCall,
1316                                                            kIntrinsified);
1317  // We have a hand-crafted assembly stub that follows the runtime calling convention. So it's
1318  // best to align the inputs accordingly.
1319  InvokeRuntimeCallingConvention calling_convention;
1320  locations->SetInAt(0, LocationFrom(calling_convention.GetRegisterAt(0)));
1321  locations->SetInAt(1, LocationFrom(calling_convention.GetRegisterAt(1)));
1322  locations->SetOut(calling_convention.GetReturnLocation(Primitive::kPrimInt));
1323
1324  // Need a temp for slow-path codepoint compare, and need to send start_index=0.
1325  locations->AddTemp(LocationFrom(calling_convention.GetRegisterAt(2)));
1326}
1327
1328void IntrinsicCodeGeneratorARM64::VisitStringIndexOf(HInvoke* invoke) {
1329  GenerateVisitStringIndexOf(
1330      invoke, GetVIXLAssembler(), codegen_, GetAllocator(), /* start_at_zero */ true);
1331}
1332
1333void IntrinsicLocationsBuilderARM64::VisitStringIndexOfAfter(HInvoke* invoke) {
1334  LocationSummary* locations = new (arena_) LocationSummary(invoke,
1335                                                            LocationSummary::kCall,
1336                                                            kIntrinsified);
1337  // We have a hand-crafted assembly stub that follows the runtime calling convention. So it's
1338  // best to align the inputs accordingly.
1339  InvokeRuntimeCallingConvention calling_convention;
1340  locations->SetInAt(0, LocationFrom(calling_convention.GetRegisterAt(0)));
1341  locations->SetInAt(1, LocationFrom(calling_convention.GetRegisterAt(1)));
1342  locations->SetInAt(2, LocationFrom(calling_convention.GetRegisterAt(2)));
1343  locations->SetOut(calling_convention.GetReturnLocation(Primitive::kPrimInt));
1344
1345  // Need a temp for slow-path codepoint compare.
1346  locations->AddTemp(Location::RequiresRegister());
1347}
1348
1349void IntrinsicCodeGeneratorARM64::VisitStringIndexOfAfter(HInvoke* invoke) {
1350  GenerateVisitStringIndexOf(
1351      invoke, GetVIXLAssembler(), codegen_, GetAllocator(), /* start_at_zero */ false);
1352}
1353
1354void IntrinsicLocationsBuilderARM64::VisitStringNewStringFromBytes(HInvoke* invoke) {
1355  LocationSummary* locations = new (arena_) LocationSummary(invoke,
1356                                                            LocationSummary::kCall,
1357                                                            kIntrinsified);
1358  InvokeRuntimeCallingConvention calling_convention;
1359  locations->SetInAt(0, LocationFrom(calling_convention.GetRegisterAt(0)));
1360  locations->SetInAt(1, LocationFrom(calling_convention.GetRegisterAt(1)));
1361  locations->SetInAt(2, LocationFrom(calling_convention.GetRegisterAt(2)));
1362  locations->SetInAt(3, LocationFrom(calling_convention.GetRegisterAt(3)));
1363  locations->SetOut(calling_convention.GetReturnLocation(Primitive::kPrimNot));
1364}
1365
1366void IntrinsicCodeGeneratorARM64::VisitStringNewStringFromBytes(HInvoke* invoke) {
1367  vixl::MacroAssembler* masm = GetVIXLAssembler();
1368  LocationSummary* locations = invoke->GetLocations();
1369
1370  Register byte_array = WRegisterFrom(locations->InAt(0));
1371  __ Cmp(byte_array, 0);
1372  SlowPathCodeARM64* slow_path = new (GetAllocator()) IntrinsicSlowPathARM64(invoke);
1373  codegen_->AddSlowPath(slow_path);
1374  __ B(eq, slow_path->GetEntryLabel());
1375
1376  __ Ldr(lr,
1377      MemOperand(tr, QUICK_ENTRYPOINT_OFFSET(kArm64WordSize, pAllocStringFromBytes).Int32Value()));
1378  codegen_->RecordPcInfo(invoke, invoke->GetDexPc());
1379  __ Blr(lr);
1380  __ Bind(slow_path->GetExitLabel());
1381}
1382
1383void IntrinsicLocationsBuilderARM64::VisitStringNewStringFromChars(HInvoke* invoke) {
1384  LocationSummary* locations = new (arena_) LocationSummary(invoke,
1385                                                            LocationSummary::kCall,
1386                                                            kIntrinsified);
1387  InvokeRuntimeCallingConvention calling_convention;
1388  locations->SetInAt(0, LocationFrom(calling_convention.GetRegisterAt(0)));
1389  locations->SetInAt(1, LocationFrom(calling_convention.GetRegisterAt(1)));
1390  locations->SetInAt(2, LocationFrom(calling_convention.GetRegisterAt(2)));
1391  locations->SetOut(calling_convention.GetReturnLocation(Primitive::kPrimNot));
1392}
1393
1394void IntrinsicCodeGeneratorARM64::VisitStringNewStringFromChars(HInvoke* invoke) {
1395  vixl::MacroAssembler* masm = GetVIXLAssembler();
1396
1397  __ Ldr(lr,
1398      MemOperand(tr, QUICK_ENTRYPOINT_OFFSET(kArm64WordSize, pAllocStringFromChars).Int32Value()));
1399  codegen_->RecordPcInfo(invoke, invoke->GetDexPc());
1400  __ Blr(lr);
1401}
1402
1403void IntrinsicLocationsBuilderARM64::VisitStringNewStringFromString(HInvoke* invoke) {
1404  // The inputs plus one temp.
1405  LocationSummary* locations = new (arena_) LocationSummary(invoke,
1406                                                            LocationSummary::kCall,
1407                                                            kIntrinsified);
1408  InvokeRuntimeCallingConvention calling_convention;
1409  locations->SetInAt(0, LocationFrom(calling_convention.GetRegisterAt(0)));
1410  locations->SetInAt(1, LocationFrom(calling_convention.GetRegisterAt(1)));
1411  locations->SetInAt(2, LocationFrom(calling_convention.GetRegisterAt(2)));
1412  locations->SetOut(calling_convention.GetReturnLocation(Primitive::kPrimNot));
1413}
1414
1415void IntrinsicCodeGeneratorARM64::VisitStringNewStringFromString(HInvoke* invoke) {
1416  vixl::MacroAssembler* masm = GetVIXLAssembler();
1417  LocationSummary* locations = invoke->GetLocations();
1418
1419  Register string_to_copy = WRegisterFrom(locations->InAt(0));
1420  __ Cmp(string_to_copy, 0);
1421  SlowPathCodeARM64* slow_path = new (GetAllocator()) IntrinsicSlowPathARM64(invoke);
1422  codegen_->AddSlowPath(slow_path);
1423  __ B(eq, slow_path->GetEntryLabel());
1424
1425  __ Ldr(lr,
1426      MemOperand(tr, QUICK_ENTRYPOINT_OFFSET(kArm64WordSize, pAllocStringFromString).Int32Value()));
1427  codegen_->RecordPcInfo(invoke, invoke->GetDexPc());
1428  __ Blr(lr);
1429  __ Bind(slow_path->GetExitLabel());
1430}
1431
1432// Unimplemented intrinsics.
1433
1434#define UNIMPLEMENTED_INTRINSIC(Name)                                                  \
1435void IntrinsicLocationsBuilderARM64::Visit ## Name(HInvoke* invoke ATTRIBUTE_UNUSED) { \
1436}                                                                                      \
1437void IntrinsicCodeGeneratorARM64::Visit ## Name(HInvoke* invoke ATTRIBUTE_UNUSED) {    \
1438}
1439
1440UNIMPLEMENTED_INTRINSIC(IntegerRotateLeft)
1441UNIMPLEMENTED_INTRINSIC(IntegerRotateRight)
1442UNIMPLEMENTED_INTRINSIC(LongRotateLeft)
1443UNIMPLEMENTED_INTRINSIC(LongRotateRight)
1444UNIMPLEMENTED_INTRINSIC(SystemArrayCopyChar)
1445UNIMPLEMENTED_INTRINSIC(SystemArrayCopy)
1446UNIMPLEMENTED_INTRINSIC(ReferenceGetReferent)
1447UNIMPLEMENTED_INTRINSIC(StringGetCharsNoCheck)
1448
1449UNIMPLEMENTED_INTRINSIC(MathCos)
1450UNIMPLEMENTED_INTRINSIC(MathSin)
1451UNIMPLEMENTED_INTRINSIC(MathAcos)
1452UNIMPLEMENTED_INTRINSIC(MathAsin)
1453UNIMPLEMENTED_INTRINSIC(MathAtan)
1454UNIMPLEMENTED_INTRINSIC(MathAtan2)
1455UNIMPLEMENTED_INTRINSIC(MathCbrt)
1456UNIMPLEMENTED_INTRINSIC(MathCosh)
1457UNIMPLEMENTED_INTRINSIC(MathExp)
1458UNIMPLEMENTED_INTRINSIC(MathExpm1)
1459UNIMPLEMENTED_INTRINSIC(MathHypot)
1460UNIMPLEMENTED_INTRINSIC(MathLog)
1461UNIMPLEMENTED_INTRINSIC(MathLog10)
1462UNIMPLEMENTED_INTRINSIC(MathNextAfter)
1463UNIMPLEMENTED_INTRINSIC(MathSinh)
1464UNIMPLEMENTED_INTRINSIC(MathTan)
1465UNIMPLEMENTED_INTRINSIC(MathTanh)
1466
1467#undef UNIMPLEMENTED_INTRINSIC
1468
1469#undef __
1470
1471}  // namespace arm64
1472}  // namespace art
1473