intrinsics_arm64.cc revision 611d3395e9efc0ab8dbfa4a197fa022fbd8c7204 
1/*
2 * Copyright (C) 2015 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *      http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17#include "intrinsics_arm64.h"
18
19#include "arch/arm64/instruction_set_features_arm64.h"
20#include "art_method.h"
21#include "code_generator_arm64.h"
22#include "common_arm64.h"
23#include "entrypoints/quick/quick_entrypoints.h"
24#include "intrinsics.h"
25#include "mirror/array-inl.h"
26#include "mirror/string.h"
27#include "thread.h"
28#include "utils/arm64/assembler_arm64.h"
29#include "utils/arm64/constants_arm64.h"
30
31#include "vixl/a64/disasm-a64.h"
32#include "vixl/a64/macro-assembler-a64.h"
33
34using namespace vixl;   // NOLINT(build/namespaces)
35
36namespace art {
37
38namespace arm64 {
39
40using helpers::DRegisterFrom;
41using helpers::FPRegisterFrom;
42using helpers::HeapOperand;
43using helpers::LocationFrom;
44using helpers::RegisterFrom;
45using helpers::SRegisterFrom;
46using helpers::WRegisterFrom;
47using helpers::XRegisterFrom;
48
49
50namespace {
51
52ALWAYS_INLINE inline MemOperand AbsoluteHeapOperandFrom(Location location, size_t offset = 0) {
53  return MemOperand(XRegisterFrom(location), offset);
54}
55
56}  // namespace
57
58vixl::MacroAssembler* IntrinsicCodeGeneratorARM64::GetVIXLAssembler() {
59  return codegen_->GetAssembler()->vixl_masm_;
60}
61
62ArenaAllocator* IntrinsicCodeGeneratorARM64::GetAllocator() {
63  return codegen_->GetGraph()->GetArena();
64}
65
66#define __ codegen->GetAssembler()->vixl_masm_->
67
68static void MoveFromReturnRegister(Location trg,
69                                   Primitive::Type type,
70                                   CodeGeneratorARM64* codegen) {
71  if (!trg.IsValid()) {
72    DCHECK(type == Primitive::kPrimVoid);
73    return;
74  }
75
76  DCHECK_NE(type, Primitive::kPrimVoid);
77
78  if (Primitive::IsIntegralType(type) || type == Primitive::kPrimNot) {
79    Register trg_reg = RegisterFrom(trg, type);
80    Register res_reg = RegisterFrom(ARM64ReturnLocation(type), type);
81    __ Mov(trg_reg, res_reg, kDiscardForSameWReg);
82  } else {
83    FPRegister trg_reg = FPRegisterFrom(trg, type);
84    FPRegister res_reg = FPRegisterFrom(ARM64ReturnLocation(type), type);
85    __ Fmov(trg_reg, res_reg);
86  }
87}
88
89static void MoveArguments(HInvoke* invoke, CodeGeneratorARM64* codegen) {
90  InvokeDexCallingConventionVisitorARM64 calling_convention_visitor;
91  IntrinsicVisitor::MoveArguments(invoke, codegen, &calling_convention_visitor);
92}
93
94// Slow-path for fallback (calling the managed code to handle the intrinsic) in an intrinsified
95// call. This will copy the arguments into the positions for a regular call.
96//
97// Note: The actual parameters are required to be in the locations given by the invoke's location
98//       summary. If an intrinsic modifies those locations before a slowpath call, they must be
99//       restored!
100class IntrinsicSlowPathARM64 : public SlowPathCodeARM64 {
101 public:
102  explicit IntrinsicSlowPathARM64(HInvoke* invoke) : invoke_(invoke) { }
103
104  void EmitNativeCode(CodeGenerator* codegen_in) OVERRIDE {
105    CodeGeneratorARM64* codegen = down_cast<CodeGeneratorARM64*>(codegen_in);
106    __ Bind(GetEntryLabel());
107
108    SaveLiveRegisters(codegen, invoke_->GetLocations());
109
110    MoveArguments(invoke_, codegen);
111
112    if (invoke_->IsInvokeStaticOrDirect()) {
113      codegen->GenerateStaticOrDirectCall(invoke_->AsInvokeStaticOrDirect(),
114                                          LocationFrom(kArtMethodRegister));
115      RecordPcInfo(codegen, invoke_, invoke_->GetDexPc());
116    } else {
117      UNIMPLEMENTED(FATAL) << "Non-direct intrinsic slow-path not yet implemented";
118      UNREACHABLE();
119    }
120
121    // Copy the result back to the expected output.
122    Location out = invoke_->GetLocations()->Out();
123    if (out.IsValid()) {
124      DCHECK(out.IsRegister());  // TODO: Replace this when we support output in memory.
125      DCHECK(!invoke_->GetLocations()->GetLiveRegisters()->ContainsCoreRegister(out.reg()));
126      MoveFromReturnRegister(out, invoke_->GetType(), codegen);
127    }
128
129    RestoreLiveRegisters(codegen, invoke_->GetLocations());
130    __ B(GetExitLabel());
131  }
132
133  const char* GetDescription() const OVERRIDE { return "IntrinsicSlowPathARM64"; }
134
135 private:
136  // The instruction where this slow path is happening.
137  HInvoke* const invoke_;
138
139  DISALLOW_COPY_AND_ASSIGN(IntrinsicSlowPathARM64);
140};
141
142#undef __
143
144bool IntrinsicLocationsBuilderARM64::TryDispatch(HInvoke* invoke) {
145  Dispatch(invoke);
146  LocationSummary* res = invoke->GetLocations();
147  return res != nullptr && res->Intrinsified();
148}
149
150#define __ masm->
151
152static void CreateFPToIntLocations(ArenaAllocator* arena, HInvoke* invoke) {
153  LocationSummary* locations = new (arena) LocationSummary(invoke,
154                                                           LocationSummary::kNoCall,
155                                                           kIntrinsified);
156  locations->SetInAt(0, Location::RequiresFpuRegister());
157  locations->SetOut(Location::RequiresRegister());
158}
159
160static void CreateIntToFPLocations(ArenaAllocator* arena, HInvoke* invoke) {
161  LocationSummary* locations = new (arena) LocationSummary(invoke,
162                                                           LocationSummary::kNoCall,
163                                                           kIntrinsified);
164  locations->SetInAt(0, Location::RequiresRegister());
165  locations->SetOut(Location::RequiresFpuRegister());
166}
167
168static void MoveFPToInt(LocationSummary* locations, bool is64bit, vixl::MacroAssembler* masm) {
169  Location input = locations->InAt(0);
170  Location output = locations->Out();
171  __ Fmov(is64bit ? XRegisterFrom(output) : WRegisterFrom(output),
172          is64bit ? DRegisterFrom(input) : SRegisterFrom(input));
173}
174
175static void MoveIntToFP(LocationSummary* locations, bool is64bit, vixl::MacroAssembler* masm) {
176  Location input = locations->InAt(0);
177  Location output = locations->Out();
178  __ Fmov(is64bit ? DRegisterFrom(output) : SRegisterFrom(output),
179          is64bit ? XRegisterFrom(input) : WRegisterFrom(input));
180}
181
182void IntrinsicLocationsBuilderARM64::VisitDoubleDoubleToRawLongBits(HInvoke* invoke) {
183  CreateFPToIntLocations(arena_, invoke);
184}
185void IntrinsicLocationsBuilderARM64::VisitDoubleLongBitsToDouble(HInvoke* invoke) {
186  CreateIntToFPLocations(arena_, invoke);
187}
188
189void IntrinsicCodeGeneratorARM64::VisitDoubleDoubleToRawLongBits(HInvoke* invoke) {
190  MoveFPToInt(invoke->GetLocations(), true, GetVIXLAssembler());
191}
192void IntrinsicCodeGeneratorARM64::VisitDoubleLongBitsToDouble(HInvoke* invoke) {
193  MoveIntToFP(invoke->GetLocations(), true, GetVIXLAssembler());
194}
195
196void IntrinsicLocationsBuilderARM64::VisitFloatFloatToRawIntBits(HInvoke* invoke) {
197  CreateFPToIntLocations(arena_, invoke);
198}
199void IntrinsicLocationsBuilderARM64::VisitFloatIntBitsToFloat(HInvoke* invoke) {
200  CreateIntToFPLocations(arena_, invoke);
201}
202
203void IntrinsicCodeGeneratorARM64::VisitFloatFloatToRawIntBits(HInvoke* invoke) {
204  MoveFPToInt(invoke->GetLocations(), false, GetVIXLAssembler());
205}
206void IntrinsicCodeGeneratorARM64::VisitFloatIntBitsToFloat(HInvoke* invoke) {
207  MoveIntToFP(invoke->GetLocations(), false, GetVIXLAssembler());
208}
209
210static void CreateIntToIntLocations(ArenaAllocator* arena, HInvoke* invoke) {
211  LocationSummary* locations = new (arena) LocationSummary(invoke,
212                                                           LocationSummary::kNoCall,
213                                                           kIntrinsified);
214  locations->SetInAt(0, Location::RequiresRegister());
215  locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
216}
217
218static void GenReverseBytes(LocationSummary* locations,
219                            Primitive::Type type,
220                            vixl::MacroAssembler* masm) {
221  Location in = locations->InAt(0);
222  Location out = locations->Out();
223
224  switch (type) {
225    case Primitive::kPrimShort:
226      __ Rev16(WRegisterFrom(out), WRegisterFrom(in));
227      __ Sxth(WRegisterFrom(out), WRegisterFrom(out));
228      break;
229    case Primitive::kPrimInt:
230    case Primitive::kPrimLong:
231      __ Rev(RegisterFrom(out, type), RegisterFrom(in, type));
232      break;
233    default:
234      LOG(FATAL) << "Unexpected size for reverse-bytes: " << type;
235      UNREACHABLE();
236  }
237}
238
239void IntrinsicLocationsBuilderARM64::VisitIntegerReverseBytes(HInvoke* invoke) {
240  CreateIntToIntLocations(arena_, invoke);
241}
242
243void IntrinsicCodeGeneratorARM64::VisitIntegerReverseBytes(HInvoke* invoke) {
244  GenReverseBytes(invoke->GetLocations(), Primitive::kPrimInt, GetVIXLAssembler());
245}
246
247void IntrinsicLocationsBuilderARM64::VisitLongReverseBytes(HInvoke* invoke) {
248  CreateIntToIntLocations(arena_, invoke);
249}
250
251void IntrinsicCodeGeneratorARM64::VisitLongReverseBytes(HInvoke* invoke) {
252  GenReverseBytes(invoke->GetLocations(), Primitive::kPrimLong, GetVIXLAssembler());
253}
254
255void IntrinsicLocationsBuilderARM64::VisitShortReverseBytes(HInvoke* invoke) {
256  CreateIntToIntLocations(arena_, invoke);
257}
258
259void IntrinsicCodeGeneratorARM64::VisitShortReverseBytes(HInvoke* invoke) {
260  GenReverseBytes(invoke->GetLocations(), Primitive::kPrimShort, GetVIXLAssembler());
261}
262
263static void GenNumberOfLeadingZeros(LocationSummary* locations,
264                                    Primitive::Type type,
265                                    vixl::MacroAssembler* masm) {
266  DCHECK(type == Primitive::kPrimInt || type == Primitive::kPrimLong);
267
268  Location in = locations->InAt(0);
269  Location out = locations->Out();
270
271  __ Clz(RegisterFrom(out, type), RegisterFrom(in, type));
272}
273
274void IntrinsicLocationsBuilderARM64::VisitIntegerNumberOfLeadingZeros(HInvoke* invoke) {
275  CreateIntToIntLocations(arena_, invoke);
276}
277
278void IntrinsicCodeGeneratorARM64::VisitIntegerNumberOfLeadingZeros(HInvoke* invoke) {
279  GenNumberOfLeadingZeros(invoke->GetLocations(), Primitive::kPrimInt, GetVIXLAssembler());
280}
281
282void IntrinsicLocationsBuilderARM64::VisitLongNumberOfLeadingZeros(HInvoke* invoke) {
283  CreateIntToIntLocations(arena_, invoke);
284}
285
286void IntrinsicCodeGeneratorARM64::VisitLongNumberOfLeadingZeros(HInvoke* invoke) {
287  GenNumberOfLeadingZeros(invoke->GetLocations(), Primitive::kPrimLong, GetVIXLAssembler());
288}
289
290static void GenReverse(LocationSummary* locations,
291                       Primitive::Type type,
292                       vixl::MacroAssembler* masm) {
293  DCHECK(type == Primitive::kPrimInt || type == Primitive::kPrimLong);
294
295  Location in = locations->InAt(0);
296  Location out = locations->Out();
297
298  __ Rbit(RegisterFrom(out, type), RegisterFrom(in, type));
299}
300
301void IntrinsicLocationsBuilderARM64::VisitIntegerReverse(HInvoke* invoke) {
302  CreateIntToIntLocations(arena_, invoke);
303}
304
305void IntrinsicCodeGeneratorARM64::VisitIntegerReverse(HInvoke* invoke) {
306  GenReverse(invoke->GetLocations(), Primitive::kPrimInt, GetVIXLAssembler());
307}
308
309void IntrinsicLocationsBuilderARM64::VisitLongReverse(HInvoke* invoke) {
310  CreateIntToIntLocations(arena_, invoke);
311}
312
313void IntrinsicCodeGeneratorARM64::VisitLongReverse(HInvoke* invoke) {
314  GenReverse(invoke->GetLocations(), Primitive::kPrimLong, GetVIXLAssembler());
315}
316
317static void CreateFPToFPLocations(ArenaAllocator* arena, HInvoke* invoke) {
318  LocationSummary* locations = new (arena) LocationSummary(invoke,
319                                                           LocationSummary::kNoCall,
320                                                           kIntrinsified);
321  locations->SetInAt(0, Location::RequiresFpuRegister());
322  locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap);
323}
324
325static void MathAbsFP(LocationSummary* locations, bool is64bit, vixl::MacroAssembler* masm) {
326  Location in = locations->InAt(0);
327  Location out = locations->Out();
328
329  FPRegister in_reg = is64bit ? DRegisterFrom(in) : SRegisterFrom(in);
330  FPRegister out_reg = is64bit ? DRegisterFrom(out) : SRegisterFrom(out);
331
332  __ Fabs(out_reg, in_reg);
333}
334
335void IntrinsicLocationsBuilderARM64::VisitMathAbsDouble(HInvoke* invoke) {
336  CreateFPToFPLocations(arena_, invoke);
337}
338
339void IntrinsicCodeGeneratorARM64::VisitMathAbsDouble(HInvoke* invoke) {
340  MathAbsFP(invoke->GetLocations(), true, GetVIXLAssembler());
341}
342
343void IntrinsicLocationsBuilderARM64::VisitMathAbsFloat(HInvoke* invoke) {
344  CreateFPToFPLocations(arena_, invoke);
345}
346
347void IntrinsicCodeGeneratorARM64::VisitMathAbsFloat(HInvoke* invoke) {
348  MathAbsFP(invoke->GetLocations(), false, GetVIXLAssembler());
349}
350
351static void CreateIntToInt(ArenaAllocator* arena, HInvoke* invoke) {
352  LocationSummary* locations = new (arena) LocationSummary(invoke,
353                                                           LocationSummary::kNoCall,
354                                                           kIntrinsified);
355  locations->SetInAt(0, Location::RequiresRegister());
356  locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
357}
358
359static void GenAbsInteger(LocationSummary* locations,
360                          bool is64bit,
361                          vixl::MacroAssembler* masm) {
362  Location in = locations->InAt(0);
363  Location output = locations->Out();
364
365  Register in_reg = is64bit ? XRegisterFrom(in) : WRegisterFrom(in);
366  Register out_reg = is64bit ? XRegisterFrom(output) : WRegisterFrom(output);
367
368  __ Cmp(in_reg, Operand(0));
369  __ Cneg(out_reg, in_reg, lt);
370}
371
372void IntrinsicLocationsBuilderARM64::VisitMathAbsInt(HInvoke* invoke) {
373  CreateIntToInt(arena_, invoke);
374}
375
376void IntrinsicCodeGeneratorARM64::VisitMathAbsInt(HInvoke* invoke) {
377  GenAbsInteger(invoke->GetLocations(), false, GetVIXLAssembler());
378}
379
380void IntrinsicLocationsBuilderARM64::VisitMathAbsLong(HInvoke* invoke) {
381  CreateIntToInt(arena_, invoke);
382}
383
384void IntrinsicCodeGeneratorARM64::VisitMathAbsLong(HInvoke* invoke) {
385  GenAbsInteger(invoke->GetLocations(), true, GetVIXLAssembler());
386}
387
388static void GenMinMaxFP(LocationSummary* locations,
389                        bool is_min,
390                        bool is_double,
391                        vixl::MacroAssembler* masm) {
392  Location op1 = locations->InAt(0);
393  Location op2 = locations->InAt(1);
394  Location out = locations->Out();
395
396  FPRegister op1_reg = is_double ? DRegisterFrom(op1) : SRegisterFrom(op1);
397  FPRegister op2_reg = is_double ? DRegisterFrom(op2) : SRegisterFrom(op2);
398  FPRegister out_reg = is_double ? DRegisterFrom(out) : SRegisterFrom(out);
399  if (is_min) {
400    __ Fmin(out_reg, op1_reg, op2_reg);
401  } else {
402    __ Fmax(out_reg, op1_reg, op2_reg);
403  }
404}
405
406static void CreateFPFPToFPLocations(ArenaAllocator* arena, HInvoke* invoke) {
407  LocationSummary* locations = new (arena) LocationSummary(invoke,
408                                                           LocationSummary::kNoCall,
409                                                           kIntrinsified);
410  locations->SetInAt(0, Location::RequiresFpuRegister());
411  locations->SetInAt(1, Location::RequiresFpuRegister());
412  locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap);
413}
414
415void IntrinsicLocationsBuilderARM64::VisitMathMinDoubleDouble(HInvoke* invoke) {
416  CreateFPFPToFPLocations(arena_, invoke);
417}
418
419void IntrinsicCodeGeneratorARM64::VisitMathMinDoubleDouble(HInvoke* invoke) {
420  GenMinMaxFP(invoke->GetLocations(), true, true, GetVIXLAssembler());
421}
422
423void IntrinsicLocationsBuilderARM64::VisitMathMinFloatFloat(HInvoke* invoke) {
424  CreateFPFPToFPLocations(arena_, invoke);
425}
426
427void IntrinsicCodeGeneratorARM64::VisitMathMinFloatFloat(HInvoke* invoke) {
428  GenMinMaxFP(invoke->GetLocations(), true, false, GetVIXLAssembler());
429}
430
431void IntrinsicLocationsBuilderARM64::VisitMathMaxDoubleDouble(HInvoke* invoke) {
432  CreateFPFPToFPLocations(arena_, invoke);
433}
434
435void IntrinsicCodeGeneratorARM64::VisitMathMaxDoubleDouble(HInvoke* invoke) {
436  GenMinMaxFP(invoke->GetLocations(), false, true, GetVIXLAssembler());
437}
438
439void IntrinsicLocationsBuilderARM64::VisitMathMaxFloatFloat(HInvoke* invoke) {
440  CreateFPFPToFPLocations(arena_, invoke);
441}
442
443void IntrinsicCodeGeneratorARM64::VisitMathMaxFloatFloat(HInvoke* invoke) {
444  GenMinMaxFP(invoke->GetLocations(), false, false, GetVIXLAssembler());
445}
446
447static void GenMinMax(LocationSummary* locations,
448                      bool is_min,
449                      bool is_long,
450                      vixl::MacroAssembler* masm) {
451  Location op1 = locations->InAt(0);
452  Location op2 = locations->InAt(1);
453  Location out = locations->Out();
454
455  Register op1_reg = is_long ? XRegisterFrom(op1) : WRegisterFrom(op1);
456  Register op2_reg = is_long ? XRegisterFrom(op2) : WRegisterFrom(op2);
457  Register out_reg = is_long ? XRegisterFrom(out) : WRegisterFrom(out);
458
459  __ Cmp(op1_reg, op2_reg);
460  __ Csel(out_reg, op1_reg, op2_reg, is_min ? lt : gt);
461}
462
463static void CreateIntIntToIntLocations(ArenaAllocator* arena, HInvoke* invoke) {
464  LocationSummary* locations = new (arena) LocationSummary(invoke,
465                                                           LocationSummary::kNoCall,
466                                                           kIntrinsified);
467  locations->SetInAt(0, Location::RequiresRegister());
468  locations->SetInAt(1, Location::RequiresRegister());
469  locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
470}
471
472void IntrinsicLocationsBuilderARM64::VisitMathMinIntInt(HInvoke* invoke) {
473  CreateIntIntToIntLocations(arena_, invoke);
474}
475
476void IntrinsicCodeGeneratorARM64::VisitMathMinIntInt(HInvoke* invoke) {
477  GenMinMax(invoke->GetLocations(), true, false, GetVIXLAssembler());
478}
479
480void IntrinsicLocationsBuilderARM64::VisitMathMinLongLong(HInvoke* invoke) {
481  CreateIntIntToIntLocations(arena_, invoke);
482}
483
484void IntrinsicCodeGeneratorARM64::VisitMathMinLongLong(HInvoke* invoke) {
485  GenMinMax(invoke->GetLocations(), true, true, GetVIXLAssembler());
486}
487
488void IntrinsicLocationsBuilderARM64::VisitMathMaxIntInt(HInvoke* invoke) {
489  CreateIntIntToIntLocations(arena_, invoke);
490}
491
492void IntrinsicCodeGeneratorARM64::VisitMathMaxIntInt(HInvoke* invoke) {
493  GenMinMax(invoke->GetLocations(), false, false, GetVIXLAssembler());
494}
495
496void IntrinsicLocationsBuilderARM64::VisitMathMaxLongLong(HInvoke* invoke) {
497  CreateIntIntToIntLocations(arena_, invoke);
498}
499
500void IntrinsicCodeGeneratorARM64::VisitMathMaxLongLong(HInvoke* invoke) {
501  GenMinMax(invoke->GetLocations(), false, true, GetVIXLAssembler());
502}
503
504void IntrinsicLocationsBuilderARM64::VisitMathSqrt(HInvoke* invoke) {
505  CreateFPToFPLocations(arena_, invoke);
506}
507
508void IntrinsicCodeGeneratorARM64::VisitMathSqrt(HInvoke* invoke) {
509  LocationSummary* locations = invoke->GetLocations();
510  vixl::MacroAssembler* masm = GetVIXLAssembler();
511  __ Fsqrt(DRegisterFrom(locations->Out()), DRegisterFrom(locations->InAt(0)));
512}
513
514void IntrinsicLocationsBuilderARM64::VisitMathCeil(HInvoke* invoke) {
515  CreateFPToFPLocations(arena_, invoke);
516}
517
518void IntrinsicCodeGeneratorARM64::VisitMathCeil(HInvoke* invoke) {
519  LocationSummary* locations = invoke->GetLocations();
520  vixl::MacroAssembler* masm = GetVIXLAssembler();
521  __ Frintp(DRegisterFrom(locations->Out()), DRegisterFrom(locations->InAt(0)));
522}
523
524void IntrinsicLocationsBuilderARM64::VisitMathFloor(HInvoke* invoke) {
525  CreateFPToFPLocations(arena_, invoke);
526}
527
528void IntrinsicCodeGeneratorARM64::VisitMathFloor(HInvoke* invoke) {
529  LocationSummary* locations = invoke->GetLocations();
530  vixl::MacroAssembler* masm = GetVIXLAssembler();
531  __ Frintm(DRegisterFrom(locations->Out()), DRegisterFrom(locations->InAt(0)));
532}
533
534void IntrinsicLocationsBuilderARM64::VisitMathRint(HInvoke* invoke) {
535  CreateFPToFPLocations(arena_, invoke);
536}
537
538void IntrinsicCodeGeneratorARM64::VisitMathRint(HInvoke* invoke) {
539  LocationSummary* locations = invoke->GetLocations();
540  vixl::MacroAssembler* masm = GetVIXLAssembler();
541  __ Frintn(DRegisterFrom(locations->Out()), DRegisterFrom(locations->InAt(0)));
542}
543
544static void CreateFPToIntPlusTempLocations(ArenaAllocator* arena, HInvoke* invoke) {
545  LocationSummary* locations = new (arena) LocationSummary(invoke,
546                                                           LocationSummary::kNoCall,
547                                                           kIntrinsified);
548  locations->SetInAt(0, Location::RequiresFpuRegister());
549  locations->SetOut(Location::RequiresRegister());
550}
551
552static void GenMathRound(LocationSummary* locations,
553                         bool is_double,
554                         vixl::MacroAssembler* masm) {
555  FPRegister in_reg = is_double ?
556      DRegisterFrom(locations->InAt(0)) : SRegisterFrom(locations->InAt(0));
557  Register out_reg = is_double ?
558      XRegisterFrom(locations->Out()) : WRegisterFrom(locations->Out());
559  UseScratchRegisterScope temps(masm);
560  FPRegister temp1_reg = temps.AcquireSameSizeAs(in_reg);
561
562  // 0.5 can be encoded as an immediate, so use fmov.
563  if (is_double) {
564    __ Fmov(temp1_reg, static_cast<double>(0.5));
565  } else {
566    __ Fmov(temp1_reg, static_cast<float>(0.5));
567  }
568  __ Fadd(temp1_reg, in_reg, temp1_reg);
569  __ Fcvtms(out_reg, temp1_reg);
570}
571
572void IntrinsicLocationsBuilderARM64::VisitMathRoundDouble(HInvoke* invoke) {
573  CreateFPToIntPlusTempLocations(arena_, invoke);
574}
575
576void IntrinsicCodeGeneratorARM64::VisitMathRoundDouble(HInvoke* invoke) {
577  GenMathRound(invoke->GetLocations(), true, GetVIXLAssembler());
578}
579
580void IntrinsicLocationsBuilderARM64::VisitMathRoundFloat(HInvoke* invoke) {
581  CreateFPToIntPlusTempLocations(arena_, invoke);
582}
583
584void IntrinsicCodeGeneratorARM64::VisitMathRoundFloat(HInvoke* invoke) {
585  GenMathRound(invoke->GetLocations(), false, GetVIXLAssembler());
586}
587
588void IntrinsicLocationsBuilderARM64::VisitMemoryPeekByte(HInvoke* invoke) {
589  CreateIntToIntLocations(arena_, invoke);
590}
591
592void IntrinsicCodeGeneratorARM64::VisitMemoryPeekByte(HInvoke* invoke) {
593  vixl::MacroAssembler* masm = GetVIXLAssembler();
594  __ Ldrsb(WRegisterFrom(invoke->GetLocations()->Out()),
595          AbsoluteHeapOperandFrom(invoke->GetLocations()->InAt(0), 0));
596}
597
598void IntrinsicLocationsBuilderARM64::VisitMemoryPeekIntNative(HInvoke* invoke) {
599  CreateIntToIntLocations(arena_, invoke);
600}
601
602void IntrinsicCodeGeneratorARM64::VisitMemoryPeekIntNative(HInvoke* invoke) {
603  vixl::MacroAssembler* masm = GetVIXLAssembler();
604  __ Ldr(WRegisterFrom(invoke->GetLocations()->Out()),
605         AbsoluteHeapOperandFrom(invoke->GetLocations()->InAt(0), 0));
606}
607
608void IntrinsicLocationsBuilderARM64::VisitMemoryPeekLongNative(HInvoke* invoke) {
609  CreateIntToIntLocations(arena_, invoke);
610}
611
612void IntrinsicCodeGeneratorARM64::VisitMemoryPeekLongNative(HInvoke* invoke) {
613  vixl::MacroAssembler* masm = GetVIXLAssembler();
614  __ Ldr(XRegisterFrom(invoke->GetLocations()->Out()),
615         AbsoluteHeapOperandFrom(invoke->GetLocations()->InAt(0), 0));
616}
617
618void IntrinsicLocationsBuilderARM64::VisitMemoryPeekShortNative(HInvoke* invoke) {
619  CreateIntToIntLocations(arena_, invoke);
620}
621
622void IntrinsicCodeGeneratorARM64::VisitMemoryPeekShortNative(HInvoke* invoke) {
623  vixl::MacroAssembler* masm = GetVIXLAssembler();
624  __ Ldrsh(WRegisterFrom(invoke->GetLocations()->Out()),
625           AbsoluteHeapOperandFrom(invoke->GetLocations()->InAt(0), 0));
626}
627
628static void CreateIntIntToVoidLocations(ArenaAllocator* arena, HInvoke* invoke) {
629  LocationSummary* locations = new (arena) LocationSummary(invoke,
630                                                           LocationSummary::kNoCall,
631                                                           kIntrinsified);
632  locations->SetInAt(0, Location::RequiresRegister());
633  locations->SetInAt(1, Location::RequiresRegister());
634}
635
636void IntrinsicLocationsBuilderARM64::VisitMemoryPokeByte(HInvoke* invoke) {
637  CreateIntIntToVoidLocations(arena_, invoke);
638}
639
640void IntrinsicCodeGeneratorARM64::VisitMemoryPokeByte(HInvoke* invoke) {
641  vixl::MacroAssembler* masm = GetVIXLAssembler();
642  __ Strb(WRegisterFrom(invoke->GetLocations()->InAt(1)),
643          AbsoluteHeapOperandFrom(invoke->GetLocations()->InAt(0), 0));
644}
645
646void IntrinsicLocationsBuilderARM64::VisitMemoryPokeIntNative(HInvoke* invoke) {
647  CreateIntIntToVoidLocations(arena_, invoke);
648}
649
650void IntrinsicCodeGeneratorARM64::VisitMemoryPokeIntNative(HInvoke* invoke) {
651  vixl::MacroAssembler* masm = GetVIXLAssembler();
652  __ Str(WRegisterFrom(invoke->GetLocations()->InAt(1)),
653         AbsoluteHeapOperandFrom(invoke->GetLocations()->InAt(0), 0));
654}
655
656void IntrinsicLocationsBuilderARM64::VisitMemoryPokeLongNative(HInvoke* invoke) {
657  CreateIntIntToVoidLocations(arena_, invoke);
658}
659
660void IntrinsicCodeGeneratorARM64::VisitMemoryPokeLongNative(HInvoke* invoke) {
661  vixl::MacroAssembler* masm = GetVIXLAssembler();
662  __ Str(XRegisterFrom(invoke->GetLocations()->InAt(1)),
663         AbsoluteHeapOperandFrom(invoke->GetLocations()->InAt(0), 0));
664}
665
666void IntrinsicLocationsBuilderARM64::VisitMemoryPokeShortNative(HInvoke* invoke) {
667  CreateIntIntToVoidLocations(arena_, invoke);
668}
669
670void IntrinsicCodeGeneratorARM64::VisitMemoryPokeShortNative(HInvoke* invoke) {
671  vixl::MacroAssembler* masm = GetVIXLAssembler();
672  __ Strh(WRegisterFrom(invoke->GetLocations()->InAt(1)),
673          AbsoluteHeapOperandFrom(invoke->GetLocations()->InAt(0), 0));
674}
675
676void IntrinsicLocationsBuilderARM64::VisitThreadCurrentThread(HInvoke* invoke) {
677  LocationSummary* locations = new (arena_) LocationSummary(invoke,
678                                                            LocationSummary::kNoCall,
679                                                            kIntrinsified);
680  locations->SetOut(Location::RequiresRegister());
681}
682
683void IntrinsicCodeGeneratorARM64::VisitThreadCurrentThread(HInvoke* invoke) {
684  codegen_->Load(Primitive::kPrimNot, WRegisterFrom(invoke->GetLocations()->Out()),
685                 MemOperand(tr, Thread::PeerOffset<8>().Int32Value()));
686}
687
688static void GenUnsafeGet(HInvoke* invoke,
689                         Primitive::Type type,
690                         bool is_volatile,
691                         CodeGeneratorARM64* codegen) {
692  LocationSummary* locations = invoke->GetLocations();
693  DCHECK((type == Primitive::kPrimInt) ||
694         (type == Primitive::kPrimLong) ||
695         (type == Primitive::kPrimNot));
696  vixl::MacroAssembler* masm = codegen->GetAssembler()->vixl_masm_;
697  Register base = WRegisterFrom(locations->InAt(1));    // Object pointer.
698  Register offset = XRegisterFrom(locations->InAt(2));  // Long offset.
699  Register trg = RegisterFrom(locations->Out(), type);
700  bool use_acquire_release = codegen->GetInstructionSetFeatures().PreferAcquireRelease();
701
702  MemOperand mem_op(base.X(), offset);
703  if (is_volatile) {
704    if (use_acquire_release) {
705      codegen->LoadAcquire(invoke, trg, mem_op);
706    } else {
707      codegen->Load(type, trg, mem_op);
708      __ Dmb(InnerShareable, BarrierReads);
709    }
710  } else {
711    codegen->Load(type, trg, mem_op);
712  }
713
714  if (type == Primitive::kPrimNot) {
715    DCHECK(trg.IsW());
716    codegen->GetAssembler()->MaybeUnpoisonHeapReference(trg);
717  }
718}
719
720static void CreateIntIntIntToIntLocations(ArenaAllocator* arena, HInvoke* invoke) {
721  LocationSummary* locations = new (arena) LocationSummary(invoke,
722                                                           LocationSummary::kNoCall,
723                                                           kIntrinsified);
724  locations->SetInAt(0, Location::NoLocation());        // Unused receiver.
725  locations->SetInAt(1, Location::RequiresRegister());
726  locations->SetInAt(2, Location::RequiresRegister());
727  locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
728}
729
730void IntrinsicLocationsBuilderARM64::VisitUnsafeGet(HInvoke* invoke) {
731  CreateIntIntIntToIntLocations(arena_, invoke);
732}
733void IntrinsicLocationsBuilderARM64::VisitUnsafeGetVolatile(HInvoke* invoke) {
734  CreateIntIntIntToIntLocations(arena_, invoke);
735}
736void IntrinsicLocationsBuilderARM64::VisitUnsafeGetLong(HInvoke* invoke) {
737  CreateIntIntIntToIntLocations(arena_, invoke);
738}
739void IntrinsicLocationsBuilderARM64::VisitUnsafeGetLongVolatile(HInvoke* invoke) {
740  CreateIntIntIntToIntLocations(arena_, invoke);
741}
742void IntrinsicLocationsBuilderARM64::VisitUnsafeGetObject(HInvoke* invoke) {
743  CreateIntIntIntToIntLocations(arena_, invoke);
744}
745void IntrinsicLocationsBuilderARM64::VisitUnsafeGetObjectVolatile(HInvoke* invoke) {
746  CreateIntIntIntToIntLocations(arena_, invoke);
747}
748
749void IntrinsicCodeGeneratorARM64::VisitUnsafeGet(HInvoke* invoke) {
750  GenUnsafeGet(invoke, Primitive::kPrimInt, false, codegen_);
751}
752void IntrinsicCodeGeneratorARM64::VisitUnsafeGetVolatile(HInvoke* invoke) {
753  GenUnsafeGet(invoke, Primitive::kPrimInt, true, codegen_);
754}
755void IntrinsicCodeGeneratorARM64::VisitUnsafeGetLong(HInvoke* invoke) {
756  GenUnsafeGet(invoke, Primitive::kPrimLong, false, codegen_);
757}
758void IntrinsicCodeGeneratorARM64::VisitUnsafeGetLongVolatile(HInvoke* invoke) {
759  GenUnsafeGet(invoke, Primitive::kPrimLong, true, codegen_);
760}
761void IntrinsicCodeGeneratorARM64::VisitUnsafeGetObject(HInvoke* invoke) {
762  GenUnsafeGet(invoke, Primitive::kPrimNot, false, codegen_);
763}
764void IntrinsicCodeGeneratorARM64::VisitUnsafeGetObjectVolatile(HInvoke* invoke) {
765  GenUnsafeGet(invoke, Primitive::kPrimNot, true, codegen_);
766}
767
768static void CreateIntIntIntIntToVoid(ArenaAllocator* arena, HInvoke* invoke) {
769  LocationSummary* locations = new (arena) LocationSummary(invoke,
770                                                           LocationSummary::kNoCall,
771                                                           kIntrinsified);
772  locations->SetInAt(0, Location::NoLocation());        // Unused receiver.
773  locations->SetInAt(1, Location::RequiresRegister());
774  locations->SetInAt(2, Location::RequiresRegister());
775  locations->SetInAt(3, Location::RequiresRegister());
776}
777
778void IntrinsicLocationsBuilderARM64::VisitUnsafePut(HInvoke* invoke) {
779  CreateIntIntIntIntToVoid(arena_, invoke);
780}
781void IntrinsicLocationsBuilderARM64::VisitUnsafePutOrdered(HInvoke* invoke) {
782  CreateIntIntIntIntToVoid(arena_, invoke);
783}
784void IntrinsicLocationsBuilderARM64::VisitUnsafePutVolatile(HInvoke* invoke) {
785  CreateIntIntIntIntToVoid(arena_, invoke);
786}
787void IntrinsicLocationsBuilderARM64::VisitUnsafePutObject(HInvoke* invoke) {
788  CreateIntIntIntIntToVoid(arena_, invoke);
789}
790void IntrinsicLocationsBuilderARM64::VisitUnsafePutObjectOrdered(HInvoke* invoke) {
791  CreateIntIntIntIntToVoid(arena_, invoke);
792}
793void IntrinsicLocationsBuilderARM64::VisitUnsafePutObjectVolatile(HInvoke* invoke) {
794  CreateIntIntIntIntToVoid(arena_, invoke);
795}
796void IntrinsicLocationsBuilderARM64::VisitUnsafePutLong(HInvoke* invoke) {
797  CreateIntIntIntIntToVoid(arena_, invoke);
798}
799void IntrinsicLocationsBuilderARM64::VisitUnsafePutLongOrdered(HInvoke* invoke) {
800  CreateIntIntIntIntToVoid(arena_, invoke);
801}
802void IntrinsicLocationsBuilderARM64::VisitUnsafePutLongVolatile(HInvoke* invoke) {
803  CreateIntIntIntIntToVoid(arena_, invoke);
804}
805
806static void GenUnsafePut(LocationSummary* locations,
807                         Primitive::Type type,
808                         bool is_volatile,
809                         bool is_ordered,
810                         CodeGeneratorARM64* codegen) {
811  vixl::MacroAssembler* masm = codegen->GetAssembler()->vixl_masm_;
812
813  Register base = WRegisterFrom(locations->InAt(1));    // Object pointer.
814  Register offset = XRegisterFrom(locations->InAt(2));  // Long offset.
815  Register value = RegisterFrom(locations->InAt(3), type);
816  Register source = value;
817  bool use_acquire_release = codegen->GetInstructionSetFeatures().PreferAcquireRelease();
818
819  MemOperand mem_op(base.X(), offset);
820
821  {
822    // We use a block to end the scratch scope before the write barrier, thus
823    // freeing the temporary registers so they can be used in `MarkGCCard`.
824    UseScratchRegisterScope temps(masm);
825
826    if (kPoisonHeapReferences && type == Primitive::kPrimNot) {
827      DCHECK(value.IsW());
828      Register temp = temps.AcquireW();
829      __ Mov(temp.W(), value.W());
830      codegen->GetAssembler()->PoisonHeapReference(temp.W());
831      source = temp;
832    }
833
834    if (is_volatile || is_ordered) {
835      if (use_acquire_release) {
836        codegen->StoreRelease(type, source, mem_op);
837      } else {
838        __ Dmb(InnerShareable, BarrierAll);
839        codegen->Store(type, source, mem_op);
840        if (is_volatile) {
841          __ Dmb(InnerShareable, BarrierReads);
842        }
843      }
844    } else {
845      codegen->Store(type, source, mem_op);
846    }
847  }
848
849  if (type == Primitive::kPrimNot) {
850    bool value_can_be_null = true;  // TODO: Worth finding out this information?
851    codegen->MarkGCCard(base, value, value_can_be_null);
852  }
853}
854
855void IntrinsicCodeGeneratorARM64::VisitUnsafePut(HInvoke* invoke) {
856  GenUnsafePut(invoke->GetLocations(), Primitive::kPrimInt, false, false, codegen_);
857}
858void IntrinsicCodeGeneratorARM64::VisitUnsafePutOrdered(HInvoke* invoke) {
859  GenUnsafePut(invoke->GetLocations(), Primitive::kPrimInt, false, true, codegen_);
860}
861void IntrinsicCodeGeneratorARM64::VisitUnsafePutVolatile(HInvoke* invoke) {
862  GenUnsafePut(invoke->GetLocations(), Primitive::kPrimInt, true, false, codegen_);
863}
864void IntrinsicCodeGeneratorARM64::VisitUnsafePutObject(HInvoke* invoke) {
865  GenUnsafePut(invoke->GetLocations(), Primitive::kPrimNot, false, false, codegen_);
866}
867void IntrinsicCodeGeneratorARM64::VisitUnsafePutObjectOrdered(HInvoke* invoke) {
868  GenUnsafePut(invoke->GetLocations(), Primitive::kPrimNot, false, true, codegen_);
869}
870void IntrinsicCodeGeneratorARM64::VisitUnsafePutObjectVolatile(HInvoke* invoke) {
871  GenUnsafePut(invoke->GetLocations(), Primitive::kPrimNot, true, false, codegen_);
872}
873void IntrinsicCodeGeneratorARM64::VisitUnsafePutLong(HInvoke* invoke) {
874  GenUnsafePut(invoke->GetLocations(), Primitive::kPrimLong, false, false, codegen_);
875}
876void IntrinsicCodeGeneratorARM64::VisitUnsafePutLongOrdered(HInvoke* invoke) {
877  GenUnsafePut(invoke->GetLocations(), Primitive::kPrimLong, false, true, codegen_);
878}
879void IntrinsicCodeGeneratorARM64::VisitUnsafePutLongVolatile(HInvoke* invoke) {
880  GenUnsafePut(invoke->GetLocations(), Primitive::kPrimLong, true, false, codegen_);
881}
882
883static void CreateIntIntIntIntIntToInt(ArenaAllocator* arena, HInvoke* invoke) {
884  LocationSummary* locations = new (arena) LocationSummary(invoke,
885                                                           LocationSummary::kNoCall,
886                                                           kIntrinsified);
887  locations->SetInAt(0, Location::NoLocation());        // Unused receiver.
888  locations->SetInAt(1, Location::RequiresRegister());
889  locations->SetInAt(2, Location::RequiresRegister());
890  locations->SetInAt(3, Location::RequiresRegister());
891  locations->SetInAt(4, Location::RequiresRegister());
892
893  locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
894}
895
896static void GenCas(LocationSummary* locations, Primitive::Type type, CodeGeneratorARM64* codegen) {
897  bool use_acquire_release = codegen->GetInstructionSetFeatures().PreferAcquireRelease();
898  vixl::MacroAssembler* masm = codegen->GetAssembler()->vixl_masm_;
899
900  Register out = WRegisterFrom(locations->Out());                  // Boolean result.
901
902  Register base = WRegisterFrom(locations->InAt(1));               // Object pointer.
903  Register offset = XRegisterFrom(locations->InAt(2));             // Long offset.
904  Register expected = RegisterFrom(locations->InAt(3), type);      // Expected.
905  Register value = RegisterFrom(locations->InAt(4), type);         // Value.
906
907  // This needs to be before the temp registers, as MarkGCCard also uses VIXL temps.
908  if (type == Primitive::kPrimNot) {
909    // Mark card for object assuming new value is stored.
910    bool value_can_be_null = true;  // TODO: Worth finding out this information?
911    codegen->MarkGCCard(base, value, value_can_be_null);
912  }
913
914  UseScratchRegisterScope temps(masm);
915  Register tmp_ptr = temps.AcquireX();                             // Pointer to actual memory.
916  Register tmp_value = temps.AcquireSameSizeAs(value);             // Value in memory.
917
918  Register tmp_32 = tmp_value.W();
919
920  __ Add(tmp_ptr, base.X(), Operand(offset));
921
922  if (kPoisonHeapReferences && type == Primitive::kPrimNot) {
923    codegen->GetAssembler()->PoisonHeapReference(expected);
924    codegen->GetAssembler()->PoisonHeapReference(value);
925  }
926
927  // do {
928  //   tmp_value = [tmp_ptr] - expected;
929  // } while (tmp_value == 0 && failure([tmp_ptr] <- r_new_value));
930  // result = tmp_value != 0;
931
932  vixl::Label loop_head, exit_loop;
933  if (use_acquire_release) {
934    __ Bind(&loop_head);
935    __ Ldaxr(tmp_value, MemOperand(tmp_ptr));
936    __ Cmp(tmp_value, expected);
937    __ B(&exit_loop, ne);
938    __ Stlxr(tmp_32, value, MemOperand(tmp_ptr));
939    __ Cbnz(tmp_32, &loop_head);
940  } else {
941    __ Dmb(InnerShareable, BarrierWrites);
942    __ Bind(&loop_head);
943    __ Ldxr(tmp_value, MemOperand(tmp_ptr));
944    __ Cmp(tmp_value, expected);
945    __ B(&exit_loop, ne);
946    __ Stxr(tmp_32, value, MemOperand(tmp_ptr));
947    __ Cbnz(tmp_32, &loop_head);
948    __ Dmb(InnerShareable, BarrierAll);
949  }
950  __ Bind(&exit_loop);
951  __ Cset(out, eq);
952
953  if (kPoisonHeapReferences && type == Primitive::kPrimNot) {
954    codegen->GetAssembler()->UnpoisonHeapReference(value);
955    codegen->GetAssembler()->UnpoisonHeapReference(expected);
956  }
957}
958
959void IntrinsicLocationsBuilderARM64::VisitUnsafeCASInt(HInvoke* invoke) {
960  CreateIntIntIntIntIntToInt(arena_, invoke);
961}
962void IntrinsicLocationsBuilderARM64::VisitUnsafeCASLong(HInvoke* invoke) {
963  CreateIntIntIntIntIntToInt(arena_, invoke);
964}
965void IntrinsicLocationsBuilderARM64::VisitUnsafeCASObject(HInvoke* invoke) {
966  CreateIntIntIntIntIntToInt(arena_, invoke);
967}
968
969void IntrinsicCodeGeneratorARM64::VisitUnsafeCASInt(HInvoke* invoke) {
970  GenCas(invoke->GetLocations(), Primitive::kPrimInt, codegen_);
971}
972void IntrinsicCodeGeneratorARM64::VisitUnsafeCASLong(HInvoke* invoke) {
973  GenCas(invoke->GetLocations(), Primitive::kPrimLong, codegen_);
974}
975void IntrinsicCodeGeneratorARM64::VisitUnsafeCASObject(HInvoke* invoke) {
976  GenCas(invoke->GetLocations(), Primitive::kPrimNot, codegen_);
977}
978
979void IntrinsicLocationsBuilderARM64::VisitStringCharAt(HInvoke* invoke) {
980  LocationSummary* locations = new (arena_) LocationSummary(invoke,
981                                                            LocationSummary::kCallOnSlowPath,
982                                                            kIntrinsified);
983  locations->SetInAt(0, Location::RequiresRegister());
984  locations->SetInAt(1, Location::RequiresRegister());
985  // In case we need to go in the slow path, we can't have the output be the same
986  // as the input: the current liveness analysis considers the input to be live
987  // at the point of the call.
988  locations->SetOut(Location::RequiresRegister(), Location::kOutputOverlap);
989}
990
991void IntrinsicCodeGeneratorARM64::VisitStringCharAt(HInvoke* invoke) {
992  vixl::MacroAssembler* masm = GetVIXLAssembler();
993  LocationSummary* locations = invoke->GetLocations();
994
995  // Location of reference to data array
996  const MemberOffset value_offset = mirror::String::ValueOffset();
997  // Location of count
998  const MemberOffset count_offset = mirror::String::CountOffset();
999
1000  Register obj = WRegisterFrom(locations->InAt(0));  // String object pointer.
1001  Register idx = WRegisterFrom(locations->InAt(1));  // Index of character.
1002  Register out = WRegisterFrom(locations->Out());    // Result character.
1003
1004  UseScratchRegisterScope temps(masm);
1005  Register temp = temps.AcquireW();
1006  Register array_temp = temps.AcquireW();            // We can trade this for worse scheduling.
1007
1008  // TODO: Maybe we can support range check elimination. Overall, though, I think it's not worth
1009  //       the cost.
1010  // TODO: For simplicity, the index parameter is requested in a register, so different from Quick
1011  //       we will not optimize the code for constants (which would save a register).
1012
1013  SlowPathCodeARM64* slow_path = new (GetAllocator()) IntrinsicSlowPathARM64(invoke);
1014  codegen_->AddSlowPath(slow_path);
1015
1016  __ Ldr(temp, HeapOperand(obj, count_offset));          // temp = str.length.
1017  codegen_->MaybeRecordImplicitNullCheck(invoke);
1018  __ Cmp(idx, temp);
1019  __ B(hs, slow_path->GetEntryLabel());
1020
1021  __ Add(array_temp, obj, Operand(value_offset.Int32Value()));  // array_temp := str.value.
1022
1023  // Load the value.
1024  __ Ldrh(out, MemOperand(array_temp.X(), idx, UXTW, 1));  // out := array_temp[idx].
1025
1026  __ Bind(slow_path->GetExitLabel());
1027}
1028
1029void IntrinsicLocationsBuilderARM64::VisitStringCompareTo(HInvoke* invoke) {
1030  LocationSummary* locations = new (arena_) LocationSummary(invoke,
1031                                                            LocationSummary::kCall,
1032                                                            kIntrinsified);
1033  InvokeRuntimeCallingConvention calling_convention;
1034  locations->SetInAt(0, LocationFrom(calling_convention.GetRegisterAt(0)));
1035  locations->SetInAt(1, LocationFrom(calling_convention.GetRegisterAt(1)));
1036  locations->SetOut(calling_convention.GetReturnLocation(Primitive::kPrimInt));
1037}
1038
1039void IntrinsicCodeGeneratorARM64::VisitStringCompareTo(HInvoke* invoke) {
1040  vixl::MacroAssembler* masm = GetVIXLAssembler();
1041  LocationSummary* locations = invoke->GetLocations();
1042
1043  // Note that the null check must have been done earlier.
1044  DCHECK(!invoke->CanDoImplicitNullCheckOn(invoke->InputAt(0)));
1045
1046  Register argument = WRegisterFrom(locations->InAt(1));
1047  __ Cmp(argument, 0);
1048  SlowPathCodeARM64* slow_path = new (GetAllocator()) IntrinsicSlowPathARM64(invoke);
1049  codegen_->AddSlowPath(slow_path);
1050  __ B(eq, slow_path->GetEntryLabel());
1051
1052  __ Ldr(
1053      lr, MemOperand(tr, QUICK_ENTRYPOINT_OFFSET(kArm64WordSize, pStringCompareTo).Int32Value()));
1054  __ Blr(lr);
1055  __ Bind(slow_path->GetExitLabel());
1056}
1057
1058static void GenerateVisitStringIndexOf(HInvoke* invoke,
1059                                       vixl::MacroAssembler* masm,
1060                                       CodeGeneratorARM64* codegen,
1061                                       ArenaAllocator* allocator,
1062                                       bool start_at_zero) {
1063  LocationSummary* locations = invoke->GetLocations();
1064  Register tmp_reg = WRegisterFrom(locations->GetTemp(0));
1065
1066  // Note that the null check must have been done earlier.
1067  DCHECK(!invoke->CanDoImplicitNullCheckOn(invoke->InputAt(0)));
1068
1069  // Check for code points > 0xFFFF. Either a slow-path check when we don't know statically,
1070  // or directly dispatch if we have a constant.
1071  SlowPathCodeARM64* slow_path = nullptr;
1072  if (invoke->InputAt(1)->IsIntConstant()) {
1073    if (static_cast<uint32_t>(invoke->InputAt(1)->AsIntConstant()->GetValue()) > 0xFFFFU) {
1074      // Always needs the slow-path. We could directly dispatch to it, but this case should be
1075      // rare, so for simplicity just put the full slow-path down and branch unconditionally.
1076      slow_path = new (allocator) IntrinsicSlowPathARM64(invoke);
1077      codegen->AddSlowPath(slow_path);
1078      __ B(slow_path->GetEntryLabel());
1079      __ Bind(slow_path->GetExitLabel());
1080      return;
1081    }
1082  } else {
1083    Register char_reg = WRegisterFrom(locations->InAt(1));
1084    __ Mov(tmp_reg, 0xFFFF);
1085    __ Cmp(char_reg, Operand(tmp_reg));
1086    slow_path = new (allocator) IntrinsicSlowPathARM64(invoke);
1087    codegen->AddSlowPath(slow_path);
1088    __ B(hi, slow_path->GetEntryLabel());
1089  }
1090
1091  if (start_at_zero) {
1092    // Start-index = 0.
1093    __ Mov(tmp_reg, 0);
1094  }
1095
1096  __ Ldr(lr, MemOperand(tr, QUICK_ENTRYPOINT_OFFSET(kArm64WordSize, pIndexOf).Int32Value()));
1097  __ Blr(lr);
1098
1099  if (slow_path != nullptr) {
1100    __ Bind(slow_path->GetExitLabel());
1101  }
1102}
1103
1104void IntrinsicLocationsBuilderARM64::VisitStringIndexOf(HInvoke* invoke) {
1105  LocationSummary* locations = new (arena_) LocationSummary(invoke,
1106                                                            LocationSummary::kCall,
1107                                                            kIntrinsified);
1108  // We have a hand-crafted assembly stub that follows the runtime calling convention. So it's
1109  // best to align the inputs accordingly.
1110  InvokeRuntimeCallingConvention calling_convention;
1111  locations->SetInAt(0, LocationFrom(calling_convention.GetRegisterAt(0)));
1112  locations->SetInAt(1, LocationFrom(calling_convention.GetRegisterAt(1)));
1113  locations->SetOut(calling_convention.GetReturnLocation(Primitive::kPrimInt));
1114
1115  // Need a temp for slow-path codepoint compare, and need to send start_index=0.
1116  locations->AddTemp(LocationFrom(calling_convention.GetRegisterAt(2)));
1117}
1118
1119void IntrinsicCodeGeneratorARM64::VisitStringIndexOf(HInvoke* invoke) {
1120  GenerateVisitStringIndexOf(invoke, GetVIXLAssembler(), codegen_, GetAllocator(), true);
1121}
1122
1123void IntrinsicLocationsBuilderARM64::VisitStringIndexOfAfter(HInvoke* invoke) {
1124  LocationSummary* locations = new (arena_) LocationSummary(invoke,
1125                                                            LocationSummary::kCall,
1126                                                            kIntrinsified);
1127  // We have a hand-crafted assembly stub that follows the runtime calling convention. So it's
1128  // best to align the inputs accordingly.
1129  InvokeRuntimeCallingConvention calling_convention;
1130  locations->SetInAt(0, LocationFrom(calling_convention.GetRegisterAt(0)));
1131  locations->SetInAt(1, LocationFrom(calling_convention.GetRegisterAt(1)));
1132  locations->SetInAt(2, LocationFrom(calling_convention.GetRegisterAt(2)));
1133  locations->SetOut(calling_convention.GetReturnLocation(Primitive::kPrimInt));
1134
1135  // Need a temp for slow-path codepoint compare.
1136  locations->AddTemp(Location::RequiresRegister());
1137}
1138
1139void IntrinsicCodeGeneratorARM64::VisitStringIndexOfAfter(HInvoke* invoke) {
1140  GenerateVisitStringIndexOf(invoke, GetVIXLAssembler(), codegen_, GetAllocator(), false);
1141}
1142
1143void IntrinsicLocationsBuilderARM64::VisitStringNewStringFromBytes(HInvoke* invoke) {
1144  LocationSummary* locations = new (arena_) LocationSummary(invoke,
1145                                                            LocationSummary::kCall,
1146                                                            kIntrinsified);
1147  InvokeRuntimeCallingConvention calling_convention;
1148  locations->SetInAt(0, LocationFrom(calling_convention.GetRegisterAt(0)));
1149  locations->SetInAt(1, LocationFrom(calling_convention.GetRegisterAt(1)));
1150  locations->SetInAt(2, LocationFrom(calling_convention.GetRegisterAt(2)));
1151  locations->SetInAt(3, LocationFrom(calling_convention.GetRegisterAt(3)));
1152  locations->SetOut(calling_convention.GetReturnLocation(Primitive::kPrimNot));
1153}
1154
1155void IntrinsicCodeGeneratorARM64::VisitStringNewStringFromBytes(HInvoke* invoke) {
1156  vixl::MacroAssembler* masm = GetVIXLAssembler();
1157  LocationSummary* locations = invoke->GetLocations();
1158
1159  Register byte_array = WRegisterFrom(locations->InAt(0));
1160  __ Cmp(byte_array, 0);
1161  SlowPathCodeARM64* slow_path = new (GetAllocator()) IntrinsicSlowPathARM64(invoke);
1162  codegen_->AddSlowPath(slow_path);
1163  __ B(eq, slow_path->GetEntryLabel());
1164
1165  __ Ldr(lr,
1166      MemOperand(tr, QUICK_ENTRYPOINT_OFFSET(kArm64WordSize, pAllocStringFromBytes).Int32Value()));
1167  codegen_->RecordPcInfo(invoke, invoke->GetDexPc());
1168  __ Blr(lr);
1169  __ Bind(slow_path->GetExitLabel());
1170}
1171
1172void IntrinsicLocationsBuilderARM64::VisitStringNewStringFromChars(HInvoke* invoke) {
1173  LocationSummary* locations = new (arena_) LocationSummary(invoke,
1174                                                            LocationSummary::kCall,
1175                                                            kIntrinsified);
1176  InvokeRuntimeCallingConvention calling_convention;
1177  locations->SetInAt(0, LocationFrom(calling_convention.GetRegisterAt(0)));
1178  locations->SetInAt(1, LocationFrom(calling_convention.GetRegisterAt(1)));
1179  locations->SetInAt(2, LocationFrom(calling_convention.GetRegisterAt(2)));
1180  locations->SetOut(calling_convention.GetReturnLocation(Primitive::kPrimNot));
1181}
1182
1183void IntrinsicCodeGeneratorARM64::VisitStringNewStringFromChars(HInvoke* invoke) {
1184  vixl::MacroAssembler* masm = GetVIXLAssembler();
1185
1186  __ Ldr(lr,
1187      MemOperand(tr, QUICK_ENTRYPOINT_OFFSET(kArm64WordSize, pAllocStringFromChars).Int32Value()));
1188  codegen_->RecordPcInfo(invoke, invoke->GetDexPc());
1189  __ Blr(lr);
1190}
1191
1192void IntrinsicLocationsBuilderARM64::VisitStringNewStringFromString(HInvoke* invoke) {
1193  // The inputs plus one temp.
1194  LocationSummary* locations = new (arena_) LocationSummary(invoke,
1195                                                            LocationSummary::kCall,
1196                                                            kIntrinsified);
1197  InvokeRuntimeCallingConvention calling_convention;
1198  locations->SetInAt(0, LocationFrom(calling_convention.GetRegisterAt(0)));
1199  locations->SetInAt(1, LocationFrom(calling_convention.GetRegisterAt(1)));
1200  locations->SetInAt(2, LocationFrom(calling_convention.GetRegisterAt(2)));
1201  locations->SetOut(calling_convention.GetReturnLocation(Primitive::kPrimNot));
1202}
1203
1204void IntrinsicCodeGeneratorARM64::VisitStringNewStringFromString(HInvoke* invoke) {
1205  vixl::MacroAssembler* masm = GetVIXLAssembler();
1206  LocationSummary* locations = invoke->GetLocations();
1207
1208  Register string_to_copy = WRegisterFrom(locations->InAt(0));
1209  __ Cmp(string_to_copy, 0);
1210  SlowPathCodeARM64* slow_path = new (GetAllocator()) IntrinsicSlowPathARM64(invoke);
1211  codegen_->AddSlowPath(slow_path);
1212  __ B(eq, slow_path->GetEntryLabel());
1213
1214  __ Ldr(lr,
1215      MemOperand(tr, QUICK_ENTRYPOINT_OFFSET(kArm64WordSize, pAllocStringFromString).Int32Value()));
1216  codegen_->RecordPcInfo(invoke, invoke->GetDexPc());
1217  __ Blr(lr);
1218  __ Bind(slow_path->GetExitLabel());
1219}
1220
1221// Unimplemented intrinsics.
1222
1223#define UNIMPLEMENTED_INTRINSIC(Name)                                                  \
1224void IntrinsicLocationsBuilderARM64::Visit ## Name(HInvoke* invoke ATTRIBUTE_UNUSED) { \
1225}                                                                                      \
1226void IntrinsicCodeGeneratorARM64::Visit ## Name(HInvoke* invoke ATTRIBUTE_UNUSED) {    \
1227}
1228
1229UNIMPLEMENTED_INTRINSIC(SystemArrayCopyChar)
1230UNIMPLEMENTED_INTRINSIC(ReferenceGetReferent)
1231UNIMPLEMENTED_INTRINSIC(StringGetCharsNoCheck)
1232
1233#undef UNIMPLEMENTED_INTRINSIC
1234
1235#undef __
1236
1237}  // namespace arm64
1238}  // namespace art
1239