intrinsics_arm64.cc revision 0e54c0160c84894696c05af6cad9eae3690f9496 
1/*
2 * Copyright (C) 2015 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *      http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17#include "intrinsics_arm64.h"
18
19#include "arch/arm64/instruction_set_features_arm64.h"
20#include "art_method.h"
21#include "code_generator_arm64.h"
22#include "common_arm64.h"
23#include "entrypoints/quick/quick_entrypoints.h"
24#include "intrinsics.h"
25#include "mirror/array-inl.h"
26#include "mirror/string.h"
27#include "thread.h"
28#include "utils/arm64/assembler_arm64.h"
29#include "utils/arm64/constants_arm64.h"
30
31#include "vixl/a64/disasm-a64.h"
32#include "vixl/a64/macro-assembler-a64.h"
33
34using namespace vixl;   // NOLINT(build/namespaces)
35
36namespace art {
37
38namespace arm64 {
39
40using helpers::DRegisterFrom;
41using helpers::FPRegisterFrom;
42using helpers::HeapOperand;
43using helpers::LocationFrom;
44using helpers::OperandFrom;
45using helpers::RegisterFrom;
46using helpers::SRegisterFrom;
47using helpers::WRegisterFrom;
48using helpers::XRegisterFrom;
49using helpers::InputRegisterAt;
50
51namespace {
52
53ALWAYS_INLINE inline MemOperand AbsoluteHeapOperandFrom(Location location, size_t offset = 0) {
54  return MemOperand(XRegisterFrom(location), offset);
55}
56
57}  // namespace
58
59vixl::MacroAssembler* IntrinsicCodeGeneratorARM64::GetVIXLAssembler() {
60  return codegen_->GetAssembler()->vixl_masm_;
61}
62
63ArenaAllocator* IntrinsicCodeGeneratorARM64::GetAllocator() {
64  return codegen_->GetGraph()->GetArena();
65}
66
67#define __ codegen->GetAssembler()->vixl_masm_->
68
69static void MoveFromReturnRegister(Location trg,
70                                   Primitive::Type type,
71                                   CodeGeneratorARM64* codegen) {
72  if (!trg.IsValid()) {
73    DCHECK(type == Primitive::kPrimVoid);
74    return;
75  }
76
77  DCHECK_NE(type, Primitive::kPrimVoid);
78
79  if (Primitive::IsIntegralType(type) || type == Primitive::kPrimNot) {
80    Register trg_reg = RegisterFrom(trg, type);
81    Register res_reg = RegisterFrom(ARM64ReturnLocation(type), type);
82    __ Mov(trg_reg, res_reg, kDiscardForSameWReg);
83  } else {
84    FPRegister trg_reg = FPRegisterFrom(trg, type);
85    FPRegister res_reg = FPRegisterFrom(ARM64ReturnLocation(type), type);
86    __ Fmov(trg_reg, res_reg);
87  }
88}
89
90static void MoveArguments(HInvoke* invoke, CodeGeneratorARM64* codegen) {
91  InvokeDexCallingConventionVisitorARM64 calling_convention_visitor;
92  IntrinsicVisitor::MoveArguments(invoke, codegen, &calling_convention_visitor);
93}
94
95// Slow-path for fallback (calling the managed code to handle the intrinsic) in an intrinsified
96// call. This will copy the arguments into the positions for a regular call.
97//
98// Note: The actual parameters are required to be in the locations given by the invoke's location
99//       summary. If an intrinsic modifies those locations before a slowpath call, they must be
100//       restored!
101class IntrinsicSlowPathARM64 : public SlowPathCodeARM64 {
102 public:
103  explicit IntrinsicSlowPathARM64(HInvoke* invoke)
104      : SlowPathCodeARM64(invoke), invoke_(invoke) { }
105
106  void EmitNativeCode(CodeGenerator* codegen_in) OVERRIDE {
107    CodeGeneratorARM64* codegen = down_cast<CodeGeneratorARM64*>(codegen_in);
108    __ Bind(GetEntryLabel());
109
110    SaveLiveRegisters(codegen, invoke_->GetLocations());
111
112    MoveArguments(invoke_, codegen);
113
114    if (invoke_->IsInvokeStaticOrDirect()) {
115      codegen->GenerateStaticOrDirectCall(invoke_->AsInvokeStaticOrDirect(),
116                                          LocationFrom(kArtMethodRegister));
117    } else {
118      codegen->GenerateVirtualCall(invoke_->AsInvokeVirtual(), LocationFrom(kArtMethodRegister));
119    }
120    codegen->RecordPcInfo(invoke_, invoke_->GetDexPc(), this);
121
122    // Copy the result back to the expected output.
123    Location out = invoke_->GetLocations()->Out();
124    if (out.IsValid()) {
125      DCHECK(out.IsRegister());  // TODO: Replace this when we support output in memory.
126      DCHECK(!invoke_->GetLocations()->GetLiveRegisters()->ContainsCoreRegister(out.reg()));
127      MoveFromReturnRegister(out, invoke_->GetType(), codegen);
128    }
129
130    RestoreLiveRegisters(codegen, invoke_->GetLocations());
131    __ B(GetExitLabel());
132  }
133
134  const char* GetDescription() const OVERRIDE { return "IntrinsicSlowPathARM64"; }
135
136 private:
137  // The instruction where this slow path is happening.
138  HInvoke* const invoke_;
139
140  DISALLOW_COPY_AND_ASSIGN(IntrinsicSlowPathARM64);
141};
142
143#undef __
144
145bool IntrinsicLocationsBuilderARM64::TryDispatch(HInvoke* invoke) {
146  Dispatch(invoke);
147  LocationSummary* res = invoke->GetLocations();
148  if (res == nullptr) {
149    return false;
150  }
151  if (kEmitCompilerReadBarrier && res->CanCall()) {
152    // Generating an intrinsic for this HInvoke may produce an
153    // IntrinsicSlowPathARM64 slow path.  Currently this approach
154    // does not work when using read barriers, as the emitted
155    // calling sequence will make use of another slow path
156    // (ReadBarrierForRootSlowPathARM64 for HInvokeStaticOrDirect,
157    // ReadBarrierSlowPathARM64 for HInvokeVirtual).  So we bail
158    // out in this case.
159    //
160    // TODO: Find a way to have intrinsics work with read barriers.
161    invoke->SetLocations(nullptr);
162    return false;
163  }
164  return res->Intrinsified();
165}
166
167#define __ masm->
168
169static void CreateFPToIntLocations(ArenaAllocator* arena, HInvoke* invoke) {
170  LocationSummary* locations = new (arena) LocationSummary(invoke,
171                                                           LocationSummary::kNoCall,
172                                                           kIntrinsified);
173  locations->SetInAt(0, Location::RequiresFpuRegister());
174  locations->SetOut(Location::RequiresRegister());
175}
176
177static void CreateIntToFPLocations(ArenaAllocator* arena, HInvoke* invoke) {
178  LocationSummary* locations = new (arena) LocationSummary(invoke,
179                                                           LocationSummary::kNoCall,
180                                                           kIntrinsified);
181  locations->SetInAt(0, Location::RequiresRegister());
182  locations->SetOut(Location::RequiresFpuRegister());
183}
184
185static void MoveFPToInt(LocationSummary* locations, bool is64bit, vixl::MacroAssembler* masm) {
186  Location input = locations->InAt(0);
187  Location output = locations->Out();
188  __ Fmov(is64bit ? XRegisterFrom(output) : WRegisterFrom(output),
189          is64bit ? DRegisterFrom(input) : SRegisterFrom(input));
190}
191
192static void MoveIntToFP(LocationSummary* locations, bool is64bit, vixl::MacroAssembler* masm) {
193  Location input = locations->InAt(0);
194  Location output = locations->Out();
195  __ Fmov(is64bit ? DRegisterFrom(output) : SRegisterFrom(output),
196          is64bit ? XRegisterFrom(input) : WRegisterFrom(input));
197}
198
199void IntrinsicLocationsBuilderARM64::VisitDoubleDoubleToRawLongBits(HInvoke* invoke) {
200  CreateFPToIntLocations(arena_, invoke);
201}
202void IntrinsicLocationsBuilderARM64::VisitDoubleLongBitsToDouble(HInvoke* invoke) {
203  CreateIntToFPLocations(arena_, invoke);
204}
205
206void IntrinsicCodeGeneratorARM64::VisitDoubleDoubleToRawLongBits(HInvoke* invoke) {
207  MoveFPToInt(invoke->GetLocations(), /* is64bit */ true, GetVIXLAssembler());
208}
209void IntrinsicCodeGeneratorARM64::VisitDoubleLongBitsToDouble(HInvoke* invoke) {
210  MoveIntToFP(invoke->GetLocations(), /* is64bit */ true, GetVIXLAssembler());
211}
212
213void IntrinsicLocationsBuilderARM64::VisitFloatFloatToRawIntBits(HInvoke* invoke) {
214  CreateFPToIntLocations(arena_, invoke);
215}
216void IntrinsicLocationsBuilderARM64::VisitFloatIntBitsToFloat(HInvoke* invoke) {
217  CreateIntToFPLocations(arena_, invoke);
218}
219
220void IntrinsicCodeGeneratorARM64::VisitFloatFloatToRawIntBits(HInvoke* invoke) {
221  MoveFPToInt(invoke->GetLocations(), /* is64bit */ false, GetVIXLAssembler());
222}
223void IntrinsicCodeGeneratorARM64::VisitFloatIntBitsToFloat(HInvoke* invoke) {
224  MoveIntToFP(invoke->GetLocations(), /* is64bit */ false, GetVIXLAssembler());
225}
226
227static void CreateIntToIntLocations(ArenaAllocator* arena, HInvoke* invoke) {
228  LocationSummary* locations = new (arena) LocationSummary(invoke,
229                                                           LocationSummary::kNoCall,
230                                                           kIntrinsified);
231  locations->SetInAt(0, Location::RequiresRegister());
232  locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
233}
234
235static void GenReverseBytes(LocationSummary* locations,
236                            Primitive::Type type,
237                            vixl::MacroAssembler* masm) {
238  Location in = locations->InAt(0);
239  Location out = locations->Out();
240
241  switch (type) {
242    case Primitive::kPrimShort:
243      __ Rev16(WRegisterFrom(out), WRegisterFrom(in));
244      __ Sxth(WRegisterFrom(out), WRegisterFrom(out));
245      break;
246    case Primitive::kPrimInt:
247    case Primitive::kPrimLong:
248      __ Rev(RegisterFrom(out, type), RegisterFrom(in, type));
249      break;
250    default:
251      LOG(FATAL) << "Unexpected size for reverse-bytes: " << type;
252      UNREACHABLE();
253  }
254}
255
256void IntrinsicLocationsBuilderARM64::VisitIntegerReverseBytes(HInvoke* invoke) {
257  CreateIntToIntLocations(arena_, invoke);
258}
259
260void IntrinsicCodeGeneratorARM64::VisitIntegerReverseBytes(HInvoke* invoke) {
261  GenReverseBytes(invoke->GetLocations(), Primitive::kPrimInt, GetVIXLAssembler());
262}
263
264void IntrinsicLocationsBuilderARM64::VisitLongReverseBytes(HInvoke* invoke) {
265  CreateIntToIntLocations(arena_, invoke);
266}
267
268void IntrinsicCodeGeneratorARM64::VisitLongReverseBytes(HInvoke* invoke) {
269  GenReverseBytes(invoke->GetLocations(), Primitive::kPrimLong, GetVIXLAssembler());
270}
271
272void IntrinsicLocationsBuilderARM64::VisitShortReverseBytes(HInvoke* invoke) {
273  CreateIntToIntLocations(arena_, invoke);
274}
275
276void IntrinsicCodeGeneratorARM64::VisitShortReverseBytes(HInvoke* invoke) {
277  GenReverseBytes(invoke->GetLocations(), Primitive::kPrimShort, GetVIXLAssembler());
278}
279
280static void CreateIntIntToIntLocations(ArenaAllocator* arena, HInvoke* invoke) {
281  LocationSummary* locations = new (arena) LocationSummary(invoke,
282                                                           LocationSummary::kNoCall,
283                                                           kIntrinsified);
284  locations->SetInAt(0, Location::RequiresRegister());
285  locations->SetInAt(1, Location::RequiresRegister());
286  locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
287}
288
289static void GenNumberOfLeadingZeros(LocationSummary* locations,
290                                    Primitive::Type type,
291                                    vixl::MacroAssembler* masm) {
292  DCHECK(type == Primitive::kPrimInt || type == Primitive::kPrimLong);
293
294  Location in = locations->InAt(0);
295  Location out = locations->Out();
296
297  __ Clz(RegisterFrom(out, type), RegisterFrom(in, type));
298}
299
300void IntrinsicLocationsBuilderARM64::VisitIntegerNumberOfLeadingZeros(HInvoke* invoke) {
301  CreateIntToIntLocations(arena_, invoke);
302}
303
304void IntrinsicCodeGeneratorARM64::VisitIntegerNumberOfLeadingZeros(HInvoke* invoke) {
305  GenNumberOfLeadingZeros(invoke->GetLocations(), Primitive::kPrimInt, GetVIXLAssembler());
306}
307
308void IntrinsicLocationsBuilderARM64::VisitLongNumberOfLeadingZeros(HInvoke* invoke) {
309  CreateIntToIntLocations(arena_, invoke);
310}
311
312void IntrinsicCodeGeneratorARM64::VisitLongNumberOfLeadingZeros(HInvoke* invoke) {
313  GenNumberOfLeadingZeros(invoke->GetLocations(), Primitive::kPrimLong, GetVIXLAssembler());
314}
315
316static void GenNumberOfTrailingZeros(LocationSummary* locations,
317                                     Primitive::Type type,
318                                     vixl::MacroAssembler* masm) {
319  DCHECK(type == Primitive::kPrimInt || type == Primitive::kPrimLong);
320
321  Location in = locations->InAt(0);
322  Location out = locations->Out();
323
324  __ Rbit(RegisterFrom(out, type), RegisterFrom(in, type));
325  __ Clz(RegisterFrom(out, type), RegisterFrom(out, type));
326}
327
328void IntrinsicLocationsBuilderARM64::VisitIntegerNumberOfTrailingZeros(HInvoke* invoke) {
329  CreateIntToIntLocations(arena_, invoke);
330}
331
332void IntrinsicCodeGeneratorARM64::VisitIntegerNumberOfTrailingZeros(HInvoke* invoke) {
333  GenNumberOfTrailingZeros(invoke->GetLocations(), Primitive::kPrimInt, GetVIXLAssembler());
334}
335
336void IntrinsicLocationsBuilderARM64::VisitLongNumberOfTrailingZeros(HInvoke* invoke) {
337  CreateIntToIntLocations(arena_, invoke);
338}
339
340void IntrinsicCodeGeneratorARM64::VisitLongNumberOfTrailingZeros(HInvoke* invoke) {
341  GenNumberOfTrailingZeros(invoke->GetLocations(), Primitive::kPrimLong, GetVIXLAssembler());
342}
343
344static void GenReverse(LocationSummary* locations,
345                       Primitive::Type type,
346                       vixl::MacroAssembler* masm) {
347  DCHECK(type == Primitive::kPrimInt || type == Primitive::kPrimLong);
348
349  Location in = locations->InAt(0);
350  Location out = locations->Out();
351
352  __ Rbit(RegisterFrom(out, type), RegisterFrom(in, type));
353}
354
355void IntrinsicLocationsBuilderARM64::VisitIntegerReverse(HInvoke* invoke) {
356  CreateIntToIntLocations(arena_, invoke);
357}
358
359void IntrinsicCodeGeneratorARM64::VisitIntegerReverse(HInvoke* invoke) {
360  GenReverse(invoke->GetLocations(), Primitive::kPrimInt, GetVIXLAssembler());
361}
362
363void IntrinsicLocationsBuilderARM64::VisitLongReverse(HInvoke* invoke) {
364  CreateIntToIntLocations(arena_, invoke);
365}
366
367void IntrinsicCodeGeneratorARM64::VisitLongReverse(HInvoke* invoke) {
368  GenReverse(invoke->GetLocations(), Primitive::kPrimLong, GetVIXLAssembler());
369}
370
371static void GenBitCount(HInvoke* instr, bool is_long, vixl::MacroAssembler* masm) {
372  DCHECK(instr->GetType() == Primitive::kPrimInt);
373  DCHECK((is_long && instr->InputAt(0)->GetType() == Primitive::kPrimLong) ||
374         (!is_long && instr->InputAt(0)->GetType() == Primitive::kPrimInt));
375
376  Location out = instr->GetLocations()->Out();
377  UseScratchRegisterScope temps(masm);
378
379  Register src = InputRegisterAt(instr, 0);
380  Register dst = is_long ? XRegisterFrom(out) : WRegisterFrom(out);
381  FPRegister fpr = is_long ? temps.AcquireD() : temps.AcquireS();
382
383  __ Fmov(fpr, src);
384  __ Cnt(fpr.V8B(), fpr.V8B());
385  __ Addv(fpr.B(), fpr.V8B());
386  __ Fmov(dst, fpr);
387}
388
389void IntrinsicLocationsBuilderARM64::VisitLongBitCount(HInvoke* invoke) {
390  CreateIntToIntLocations(arena_, invoke);
391}
392
393void IntrinsicCodeGeneratorARM64::VisitLongBitCount(HInvoke* invoke) {
394  GenBitCount(invoke, /* is_long */ true, GetVIXLAssembler());
395}
396
397void IntrinsicLocationsBuilderARM64::VisitIntegerBitCount(HInvoke* invoke) {
398  CreateIntToIntLocations(arena_, invoke);
399}
400
401void IntrinsicCodeGeneratorARM64::VisitIntegerBitCount(HInvoke* invoke) {
402  GenBitCount(invoke, /* is_long */ false, GetVIXLAssembler());
403}
404
405static void CreateFPToFPLocations(ArenaAllocator* arena, HInvoke* invoke) {
406  LocationSummary* locations = new (arena) LocationSummary(invoke,
407                                                           LocationSummary::kNoCall,
408                                                           kIntrinsified);
409  locations->SetInAt(0, Location::RequiresFpuRegister());
410  locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap);
411}
412
413static void MathAbsFP(LocationSummary* locations, bool is64bit, vixl::MacroAssembler* masm) {
414  Location in = locations->InAt(0);
415  Location out = locations->Out();
416
417  FPRegister in_reg = is64bit ? DRegisterFrom(in) : SRegisterFrom(in);
418  FPRegister out_reg = is64bit ? DRegisterFrom(out) : SRegisterFrom(out);
419
420  __ Fabs(out_reg, in_reg);
421}
422
423void IntrinsicLocationsBuilderARM64::VisitMathAbsDouble(HInvoke* invoke) {
424  CreateFPToFPLocations(arena_, invoke);
425}
426
427void IntrinsicCodeGeneratorARM64::VisitMathAbsDouble(HInvoke* invoke) {
428  MathAbsFP(invoke->GetLocations(), /* is64bit */ true, GetVIXLAssembler());
429}
430
431void IntrinsicLocationsBuilderARM64::VisitMathAbsFloat(HInvoke* invoke) {
432  CreateFPToFPLocations(arena_, invoke);
433}
434
435void IntrinsicCodeGeneratorARM64::VisitMathAbsFloat(HInvoke* invoke) {
436  MathAbsFP(invoke->GetLocations(), /* is64bit */ false, GetVIXLAssembler());
437}
438
439static void CreateIntToInt(ArenaAllocator* arena, HInvoke* invoke) {
440  LocationSummary* locations = new (arena) LocationSummary(invoke,
441                                                           LocationSummary::kNoCall,
442                                                           kIntrinsified);
443  locations->SetInAt(0, Location::RequiresRegister());
444  locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
445}
446
447static void GenAbsInteger(LocationSummary* locations,
448                          bool is64bit,
449                          vixl::MacroAssembler* masm) {
450  Location in = locations->InAt(0);
451  Location output = locations->Out();
452
453  Register in_reg = is64bit ? XRegisterFrom(in) : WRegisterFrom(in);
454  Register out_reg = is64bit ? XRegisterFrom(output) : WRegisterFrom(output);
455
456  __ Cmp(in_reg, Operand(0));
457  __ Cneg(out_reg, in_reg, lt);
458}
459
460void IntrinsicLocationsBuilderARM64::VisitMathAbsInt(HInvoke* invoke) {
461  CreateIntToInt(arena_, invoke);
462}
463
464void IntrinsicCodeGeneratorARM64::VisitMathAbsInt(HInvoke* invoke) {
465  GenAbsInteger(invoke->GetLocations(), /* is64bit */ false, GetVIXLAssembler());
466}
467
468void IntrinsicLocationsBuilderARM64::VisitMathAbsLong(HInvoke* invoke) {
469  CreateIntToInt(arena_, invoke);
470}
471
472void IntrinsicCodeGeneratorARM64::VisitMathAbsLong(HInvoke* invoke) {
473  GenAbsInteger(invoke->GetLocations(), /* is64bit */ true, GetVIXLAssembler());
474}
475
476static void GenMinMaxFP(LocationSummary* locations,
477                        bool is_min,
478                        bool is_double,
479                        vixl::MacroAssembler* masm) {
480  Location op1 = locations->InAt(0);
481  Location op2 = locations->InAt(1);
482  Location out = locations->Out();
483
484  FPRegister op1_reg = is_double ? DRegisterFrom(op1) : SRegisterFrom(op1);
485  FPRegister op2_reg = is_double ? DRegisterFrom(op2) : SRegisterFrom(op2);
486  FPRegister out_reg = is_double ? DRegisterFrom(out) : SRegisterFrom(out);
487  if (is_min) {
488    __ Fmin(out_reg, op1_reg, op2_reg);
489  } else {
490    __ Fmax(out_reg, op1_reg, op2_reg);
491  }
492}
493
494static void CreateFPFPToFPLocations(ArenaAllocator* arena, HInvoke* invoke) {
495  LocationSummary* locations = new (arena) LocationSummary(invoke,
496                                                           LocationSummary::kNoCall,
497                                                           kIntrinsified);
498  locations->SetInAt(0, Location::RequiresFpuRegister());
499  locations->SetInAt(1, Location::RequiresFpuRegister());
500  locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap);
501}
502
503void IntrinsicLocationsBuilderARM64::VisitMathMinDoubleDouble(HInvoke* invoke) {
504  CreateFPFPToFPLocations(arena_, invoke);
505}
506
507void IntrinsicCodeGeneratorARM64::VisitMathMinDoubleDouble(HInvoke* invoke) {
508  GenMinMaxFP(invoke->GetLocations(), /* is_min */ true, /* is_double */ true, GetVIXLAssembler());
509}
510
511void IntrinsicLocationsBuilderARM64::VisitMathMinFloatFloat(HInvoke* invoke) {
512  CreateFPFPToFPLocations(arena_, invoke);
513}
514
515void IntrinsicCodeGeneratorARM64::VisitMathMinFloatFloat(HInvoke* invoke) {
516  GenMinMaxFP(invoke->GetLocations(), /* is_min */ true, /* is_double */ false, GetVIXLAssembler());
517}
518
519void IntrinsicLocationsBuilderARM64::VisitMathMaxDoubleDouble(HInvoke* invoke) {
520  CreateFPFPToFPLocations(arena_, invoke);
521}
522
523void IntrinsicCodeGeneratorARM64::VisitMathMaxDoubleDouble(HInvoke* invoke) {
524  GenMinMaxFP(invoke->GetLocations(), /* is_min */ false, /* is_double */ true, GetVIXLAssembler());
525}
526
527void IntrinsicLocationsBuilderARM64::VisitMathMaxFloatFloat(HInvoke* invoke) {
528  CreateFPFPToFPLocations(arena_, invoke);
529}
530
531void IntrinsicCodeGeneratorARM64::VisitMathMaxFloatFloat(HInvoke* invoke) {
532  GenMinMaxFP(
533      invoke->GetLocations(), /* is_min */ false, /* is_double */ false, GetVIXLAssembler());
534}
535
536static void GenMinMax(LocationSummary* locations,
537                      bool is_min,
538                      bool is_long,
539                      vixl::MacroAssembler* masm) {
540  Location op1 = locations->InAt(0);
541  Location op2 = locations->InAt(1);
542  Location out = locations->Out();
543
544  Register op1_reg = is_long ? XRegisterFrom(op1) : WRegisterFrom(op1);
545  Register op2_reg = is_long ? XRegisterFrom(op2) : WRegisterFrom(op2);
546  Register out_reg = is_long ? XRegisterFrom(out) : WRegisterFrom(out);
547
548  __ Cmp(op1_reg, op2_reg);
549  __ Csel(out_reg, op1_reg, op2_reg, is_min ? lt : gt);
550}
551
552void IntrinsicLocationsBuilderARM64::VisitMathMinIntInt(HInvoke* invoke) {
553  CreateIntIntToIntLocations(arena_, invoke);
554}
555
556void IntrinsicCodeGeneratorARM64::VisitMathMinIntInt(HInvoke* invoke) {
557  GenMinMax(invoke->GetLocations(), /* is_min */ true, /* is_long */ false, GetVIXLAssembler());
558}
559
560void IntrinsicLocationsBuilderARM64::VisitMathMinLongLong(HInvoke* invoke) {
561  CreateIntIntToIntLocations(arena_, invoke);
562}
563
564void IntrinsicCodeGeneratorARM64::VisitMathMinLongLong(HInvoke* invoke) {
565  GenMinMax(invoke->GetLocations(), /* is_min */ true, /* is_long */ true, GetVIXLAssembler());
566}
567
568void IntrinsicLocationsBuilderARM64::VisitMathMaxIntInt(HInvoke* invoke) {
569  CreateIntIntToIntLocations(arena_, invoke);
570}
571
572void IntrinsicCodeGeneratorARM64::VisitMathMaxIntInt(HInvoke* invoke) {
573  GenMinMax(invoke->GetLocations(), /* is_min */ false, /* is_long */ false, GetVIXLAssembler());
574}
575
576void IntrinsicLocationsBuilderARM64::VisitMathMaxLongLong(HInvoke* invoke) {
577  CreateIntIntToIntLocations(arena_, invoke);
578}
579
580void IntrinsicCodeGeneratorARM64::VisitMathMaxLongLong(HInvoke* invoke) {
581  GenMinMax(invoke->GetLocations(), /* is_min */ false, /* is_long */ true, GetVIXLAssembler());
582}
583
584void IntrinsicLocationsBuilderARM64::VisitMathSqrt(HInvoke* invoke) {
585  CreateFPToFPLocations(arena_, invoke);
586}
587
588void IntrinsicCodeGeneratorARM64::VisitMathSqrt(HInvoke* invoke) {
589  LocationSummary* locations = invoke->GetLocations();
590  vixl::MacroAssembler* masm = GetVIXLAssembler();
591  __ Fsqrt(DRegisterFrom(locations->Out()), DRegisterFrom(locations->InAt(0)));
592}
593
594void IntrinsicLocationsBuilderARM64::VisitMathCeil(HInvoke* invoke) {
595  CreateFPToFPLocations(arena_, invoke);
596}
597
598void IntrinsicCodeGeneratorARM64::VisitMathCeil(HInvoke* invoke) {
599  LocationSummary* locations = invoke->GetLocations();
600  vixl::MacroAssembler* masm = GetVIXLAssembler();
601  __ Frintp(DRegisterFrom(locations->Out()), DRegisterFrom(locations->InAt(0)));
602}
603
604void IntrinsicLocationsBuilderARM64::VisitMathFloor(HInvoke* invoke) {
605  CreateFPToFPLocations(arena_, invoke);
606}
607
608void IntrinsicCodeGeneratorARM64::VisitMathFloor(HInvoke* invoke) {
609  LocationSummary* locations = invoke->GetLocations();
610  vixl::MacroAssembler* masm = GetVIXLAssembler();
611  __ Frintm(DRegisterFrom(locations->Out()), DRegisterFrom(locations->InAt(0)));
612}
613
614void IntrinsicLocationsBuilderARM64::VisitMathRint(HInvoke* invoke) {
615  CreateFPToFPLocations(arena_, invoke);
616}
617
618void IntrinsicCodeGeneratorARM64::VisitMathRint(HInvoke* invoke) {
619  LocationSummary* locations = invoke->GetLocations();
620  vixl::MacroAssembler* masm = GetVIXLAssembler();
621  __ Frintn(DRegisterFrom(locations->Out()), DRegisterFrom(locations->InAt(0)));
622}
623
624static void CreateFPToIntPlusTempLocations(ArenaAllocator* arena, HInvoke* invoke) {
625  LocationSummary* locations = new (arena) LocationSummary(invoke,
626                                                           LocationSummary::kNoCall,
627                                                           kIntrinsified);
628  locations->SetInAt(0, Location::RequiresFpuRegister());
629  locations->SetOut(Location::RequiresRegister());
630}
631
632static void GenMathRound(LocationSummary* locations,
633                         bool is_double,
634                         vixl::MacroAssembler* masm) {
635  FPRegister in_reg = is_double ?
636      DRegisterFrom(locations->InAt(0)) : SRegisterFrom(locations->InAt(0));
637  Register out_reg = is_double ?
638      XRegisterFrom(locations->Out()) : WRegisterFrom(locations->Out());
639  UseScratchRegisterScope temps(masm);
640  FPRegister temp1_reg = temps.AcquireSameSizeAs(in_reg);
641
642  // 0.5 can be encoded as an immediate, so use fmov.
643  if (is_double) {
644    __ Fmov(temp1_reg, static_cast<double>(0.5));
645  } else {
646    __ Fmov(temp1_reg, static_cast<float>(0.5));
647  }
648  __ Fadd(temp1_reg, in_reg, temp1_reg);
649  __ Fcvtms(out_reg, temp1_reg);
650}
651
652void IntrinsicLocationsBuilderARM64::VisitMathRoundDouble(HInvoke* invoke) {
653  // See intrinsics.h.
654  if (kRoundIsPlusPointFive) {
655    CreateFPToIntPlusTempLocations(arena_, invoke);
656  }
657}
658
659void IntrinsicCodeGeneratorARM64::VisitMathRoundDouble(HInvoke* invoke) {
660  GenMathRound(invoke->GetLocations(), /* is_double */ true, GetVIXLAssembler());
661}
662
663void IntrinsicLocationsBuilderARM64::VisitMathRoundFloat(HInvoke* invoke) {
664  // See intrinsics.h.
665  if (kRoundIsPlusPointFive) {
666    CreateFPToIntPlusTempLocations(arena_, invoke);
667  }
668}
669
670void IntrinsicCodeGeneratorARM64::VisitMathRoundFloat(HInvoke* invoke) {
671  GenMathRound(invoke->GetLocations(), /* is_double */ false, GetVIXLAssembler());
672}
673
674void IntrinsicLocationsBuilderARM64::VisitMemoryPeekByte(HInvoke* invoke) {
675  CreateIntToIntLocations(arena_, invoke);
676}
677
678void IntrinsicCodeGeneratorARM64::VisitMemoryPeekByte(HInvoke* invoke) {
679  vixl::MacroAssembler* masm = GetVIXLAssembler();
680  __ Ldrsb(WRegisterFrom(invoke->GetLocations()->Out()),
681          AbsoluteHeapOperandFrom(invoke->GetLocations()->InAt(0), 0));
682}
683
684void IntrinsicLocationsBuilderARM64::VisitMemoryPeekIntNative(HInvoke* invoke) {
685  CreateIntToIntLocations(arena_, invoke);
686}
687
688void IntrinsicCodeGeneratorARM64::VisitMemoryPeekIntNative(HInvoke* invoke) {
689  vixl::MacroAssembler* masm = GetVIXLAssembler();
690  __ Ldr(WRegisterFrom(invoke->GetLocations()->Out()),
691         AbsoluteHeapOperandFrom(invoke->GetLocations()->InAt(0), 0));
692}
693
694void IntrinsicLocationsBuilderARM64::VisitMemoryPeekLongNative(HInvoke* invoke) {
695  CreateIntToIntLocations(arena_, invoke);
696}
697
698void IntrinsicCodeGeneratorARM64::VisitMemoryPeekLongNative(HInvoke* invoke) {
699  vixl::MacroAssembler* masm = GetVIXLAssembler();
700  __ Ldr(XRegisterFrom(invoke->GetLocations()->Out()),
701         AbsoluteHeapOperandFrom(invoke->GetLocations()->InAt(0), 0));
702}
703
704void IntrinsicLocationsBuilderARM64::VisitMemoryPeekShortNative(HInvoke* invoke) {
705  CreateIntToIntLocations(arena_, invoke);
706}
707
708void IntrinsicCodeGeneratorARM64::VisitMemoryPeekShortNative(HInvoke* invoke) {
709  vixl::MacroAssembler* masm = GetVIXLAssembler();
710  __ Ldrsh(WRegisterFrom(invoke->GetLocations()->Out()),
711           AbsoluteHeapOperandFrom(invoke->GetLocations()->InAt(0), 0));
712}
713
714static void CreateIntIntToVoidLocations(ArenaAllocator* arena, HInvoke* invoke) {
715  LocationSummary* locations = new (arena) LocationSummary(invoke,
716                                                           LocationSummary::kNoCall,
717                                                           kIntrinsified);
718  locations->SetInAt(0, Location::RequiresRegister());
719  locations->SetInAt(1, Location::RequiresRegister());
720}
721
722void IntrinsicLocationsBuilderARM64::VisitMemoryPokeByte(HInvoke* invoke) {
723  CreateIntIntToVoidLocations(arena_, invoke);
724}
725
726void IntrinsicCodeGeneratorARM64::VisitMemoryPokeByte(HInvoke* invoke) {
727  vixl::MacroAssembler* masm = GetVIXLAssembler();
728  __ Strb(WRegisterFrom(invoke->GetLocations()->InAt(1)),
729          AbsoluteHeapOperandFrom(invoke->GetLocations()->InAt(0), 0));
730}
731
732void IntrinsicLocationsBuilderARM64::VisitMemoryPokeIntNative(HInvoke* invoke) {
733  CreateIntIntToVoidLocations(arena_, invoke);
734}
735
736void IntrinsicCodeGeneratorARM64::VisitMemoryPokeIntNative(HInvoke* invoke) {
737  vixl::MacroAssembler* masm = GetVIXLAssembler();
738  __ Str(WRegisterFrom(invoke->GetLocations()->InAt(1)),
739         AbsoluteHeapOperandFrom(invoke->GetLocations()->InAt(0), 0));
740}
741
742void IntrinsicLocationsBuilderARM64::VisitMemoryPokeLongNative(HInvoke* invoke) {
743  CreateIntIntToVoidLocations(arena_, invoke);
744}
745
746void IntrinsicCodeGeneratorARM64::VisitMemoryPokeLongNative(HInvoke* invoke) {
747  vixl::MacroAssembler* masm = GetVIXLAssembler();
748  __ Str(XRegisterFrom(invoke->GetLocations()->InAt(1)),
749         AbsoluteHeapOperandFrom(invoke->GetLocations()->InAt(0), 0));
750}
751
752void IntrinsicLocationsBuilderARM64::VisitMemoryPokeShortNative(HInvoke* invoke) {
753  CreateIntIntToVoidLocations(arena_, invoke);
754}
755
756void IntrinsicCodeGeneratorARM64::VisitMemoryPokeShortNative(HInvoke* invoke) {
757  vixl::MacroAssembler* masm = GetVIXLAssembler();
758  __ Strh(WRegisterFrom(invoke->GetLocations()->InAt(1)),
759          AbsoluteHeapOperandFrom(invoke->GetLocations()->InAt(0), 0));
760}
761
762void IntrinsicLocationsBuilderARM64::VisitThreadCurrentThread(HInvoke* invoke) {
763  LocationSummary* locations = new (arena_) LocationSummary(invoke,
764                                                            LocationSummary::kNoCall,
765                                                            kIntrinsified);
766  locations->SetOut(Location::RequiresRegister());
767}
768
769void IntrinsicCodeGeneratorARM64::VisitThreadCurrentThread(HInvoke* invoke) {
770  codegen_->Load(Primitive::kPrimNot, WRegisterFrom(invoke->GetLocations()->Out()),
771                 MemOperand(tr, Thread::PeerOffset<8>().Int32Value()));
772}
773
774static void GenUnsafeGet(HInvoke* invoke,
775                         Primitive::Type type,
776                         bool is_volatile,
777                         CodeGeneratorARM64* codegen) {
778  LocationSummary* locations = invoke->GetLocations();
779  DCHECK((type == Primitive::kPrimInt) ||
780         (type == Primitive::kPrimLong) ||
781         (type == Primitive::kPrimNot));
782  vixl::MacroAssembler* masm = codegen->GetAssembler()->vixl_masm_;
783  Location base_loc = locations->InAt(1);
784  Register base = WRegisterFrom(base_loc);      // Object pointer.
785  Location offset_loc = locations->InAt(2);
786  Register offset = XRegisterFrom(offset_loc);  // Long offset.
787  Location trg_loc = locations->Out();
788  Register trg = RegisterFrom(trg_loc, type);
789
790  if (type == Primitive::kPrimNot && kEmitCompilerReadBarrier && kUseBakerReadBarrier) {
791    // UnsafeGetObject/UnsafeGetObjectVolatile with Baker's read barrier case.
792    UseScratchRegisterScope temps(masm);
793    Register temp = temps.AcquireW();
794    codegen->GenerateArrayLoadWithBakerReadBarrier(
795        invoke, trg_loc, base, 0U, offset_loc, temp, /* needs_null_check */ false);
796  } else {
797    // Other cases.
798    MemOperand mem_op(base.X(), offset);
799    if (is_volatile) {
800      codegen->LoadAcquire(invoke, trg, mem_op, /* needs_null_check */ true);
801    } else {
802      codegen->Load(type, trg, mem_op);
803    }
804
805    if (type == Primitive::kPrimNot) {
806      DCHECK(trg.IsW());
807      codegen->MaybeGenerateReadBarrierSlow(invoke, trg_loc, trg_loc, base_loc, 0U, offset_loc);
808    }
809  }
810}
811
812static void CreateIntIntIntToIntLocations(ArenaAllocator* arena, HInvoke* invoke) {
813  bool can_call = kEmitCompilerReadBarrier &&
814      (invoke->GetIntrinsic() == Intrinsics::kUnsafeGetObject ||
815       invoke->GetIntrinsic() == Intrinsics::kUnsafeGetObjectVolatile);
816  LocationSummary* locations = new (arena) LocationSummary(invoke,
817                                                           can_call ?
818                                                               LocationSummary::kCallOnSlowPath :
819                                                               LocationSummary::kNoCall,
820                                                           kIntrinsified);
821  locations->SetInAt(0, Location::NoLocation());        // Unused receiver.
822  locations->SetInAt(1, Location::RequiresRegister());
823  locations->SetInAt(2, Location::RequiresRegister());
824  locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
825}
826
827void IntrinsicLocationsBuilderARM64::VisitUnsafeGet(HInvoke* invoke) {
828  CreateIntIntIntToIntLocations(arena_, invoke);
829}
830void IntrinsicLocationsBuilderARM64::VisitUnsafeGetVolatile(HInvoke* invoke) {
831  CreateIntIntIntToIntLocations(arena_, invoke);
832}
833void IntrinsicLocationsBuilderARM64::VisitUnsafeGetLong(HInvoke* invoke) {
834  CreateIntIntIntToIntLocations(arena_, invoke);
835}
836void IntrinsicLocationsBuilderARM64::VisitUnsafeGetLongVolatile(HInvoke* invoke) {
837  CreateIntIntIntToIntLocations(arena_, invoke);
838}
839void IntrinsicLocationsBuilderARM64::VisitUnsafeGetObject(HInvoke* invoke) {
840  CreateIntIntIntToIntLocations(arena_, invoke);
841}
842void IntrinsicLocationsBuilderARM64::VisitUnsafeGetObjectVolatile(HInvoke* invoke) {
843  CreateIntIntIntToIntLocations(arena_, invoke);
844}
845
846void IntrinsicCodeGeneratorARM64::VisitUnsafeGet(HInvoke* invoke) {
847  GenUnsafeGet(invoke, Primitive::kPrimInt, /* is_volatile */ false, codegen_);
848}
849void IntrinsicCodeGeneratorARM64::VisitUnsafeGetVolatile(HInvoke* invoke) {
850  GenUnsafeGet(invoke, Primitive::kPrimInt, /* is_volatile */ true, codegen_);
851}
852void IntrinsicCodeGeneratorARM64::VisitUnsafeGetLong(HInvoke* invoke) {
853  GenUnsafeGet(invoke, Primitive::kPrimLong, /* is_volatile */ false, codegen_);
854}
855void IntrinsicCodeGeneratorARM64::VisitUnsafeGetLongVolatile(HInvoke* invoke) {
856  GenUnsafeGet(invoke, Primitive::kPrimLong, /* is_volatile */ true, codegen_);
857}
858void IntrinsicCodeGeneratorARM64::VisitUnsafeGetObject(HInvoke* invoke) {
859  GenUnsafeGet(invoke, Primitive::kPrimNot, /* is_volatile */ false, codegen_);
860}
861void IntrinsicCodeGeneratorARM64::VisitUnsafeGetObjectVolatile(HInvoke* invoke) {
862  GenUnsafeGet(invoke, Primitive::kPrimNot, /* is_volatile */ true, codegen_);
863}
864
865static void CreateIntIntIntIntToVoid(ArenaAllocator* arena, HInvoke* invoke) {
866  LocationSummary* locations = new (arena) LocationSummary(invoke,
867                                                           LocationSummary::kNoCall,
868                                                           kIntrinsified);
869  locations->SetInAt(0, Location::NoLocation());        // Unused receiver.
870  locations->SetInAt(1, Location::RequiresRegister());
871  locations->SetInAt(2, Location::RequiresRegister());
872  locations->SetInAt(3, Location::RequiresRegister());
873}
874
875void IntrinsicLocationsBuilderARM64::VisitUnsafePut(HInvoke* invoke) {
876  CreateIntIntIntIntToVoid(arena_, invoke);
877}
878void IntrinsicLocationsBuilderARM64::VisitUnsafePutOrdered(HInvoke* invoke) {
879  CreateIntIntIntIntToVoid(arena_, invoke);
880}
881void IntrinsicLocationsBuilderARM64::VisitUnsafePutVolatile(HInvoke* invoke) {
882  CreateIntIntIntIntToVoid(arena_, invoke);
883}
884void IntrinsicLocationsBuilderARM64::VisitUnsafePutObject(HInvoke* invoke) {
885  CreateIntIntIntIntToVoid(arena_, invoke);
886}
887void IntrinsicLocationsBuilderARM64::VisitUnsafePutObjectOrdered(HInvoke* invoke) {
888  CreateIntIntIntIntToVoid(arena_, invoke);
889}
890void IntrinsicLocationsBuilderARM64::VisitUnsafePutObjectVolatile(HInvoke* invoke) {
891  CreateIntIntIntIntToVoid(arena_, invoke);
892}
893void IntrinsicLocationsBuilderARM64::VisitUnsafePutLong(HInvoke* invoke) {
894  CreateIntIntIntIntToVoid(arena_, invoke);
895}
896void IntrinsicLocationsBuilderARM64::VisitUnsafePutLongOrdered(HInvoke* invoke) {
897  CreateIntIntIntIntToVoid(arena_, invoke);
898}
899void IntrinsicLocationsBuilderARM64::VisitUnsafePutLongVolatile(HInvoke* invoke) {
900  CreateIntIntIntIntToVoid(arena_, invoke);
901}
902
903static void GenUnsafePut(LocationSummary* locations,
904                         Primitive::Type type,
905                         bool is_volatile,
906                         bool is_ordered,
907                         CodeGeneratorARM64* codegen) {
908  vixl::MacroAssembler* masm = codegen->GetAssembler()->vixl_masm_;
909
910  Register base = WRegisterFrom(locations->InAt(1));    // Object pointer.
911  Register offset = XRegisterFrom(locations->InAt(2));  // Long offset.
912  Register value = RegisterFrom(locations->InAt(3), type);
913  Register source = value;
914  MemOperand mem_op(base.X(), offset);
915
916  {
917    // We use a block to end the scratch scope before the write barrier, thus
918    // freeing the temporary registers so they can be used in `MarkGCCard`.
919    UseScratchRegisterScope temps(masm);
920
921    if (kPoisonHeapReferences && type == Primitive::kPrimNot) {
922      DCHECK(value.IsW());
923      Register temp = temps.AcquireW();
924      __ Mov(temp.W(), value.W());
925      codegen->GetAssembler()->PoisonHeapReference(temp.W());
926      source = temp;
927    }
928
929    if (is_volatile || is_ordered) {
930      codegen->StoreRelease(type, source, mem_op);
931    } else {
932      codegen->Store(type, source, mem_op);
933    }
934  }
935
936  if (type == Primitive::kPrimNot) {
937    bool value_can_be_null = true;  // TODO: Worth finding out this information?
938    codegen->MarkGCCard(base, value, value_can_be_null);
939  }
940}
941
942void IntrinsicCodeGeneratorARM64::VisitUnsafePut(HInvoke* invoke) {
943  GenUnsafePut(invoke->GetLocations(),
944               Primitive::kPrimInt,
945               /* is_volatile */ false,
946               /* is_ordered */ false,
947               codegen_);
948}
949void IntrinsicCodeGeneratorARM64::VisitUnsafePutOrdered(HInvoke* invoke) {
950  GenUnsafePut(invoke->GetLocations(),
951               Primitive::kPrimInt,
952               /* is_volatile */ false,
953               /* is_ordered */ true,
954               codegen_);
955}
956void IntrinsicCodeGeneratorARM64::VisitUnsafePutVolatile(HInvoke* invoke) {
957  GenUnsafePut(invoke->GetLocations(),
958               Primitive::kPrimInt,
959               /* is_volatile */ true,
960               /* is_ordered */ false,
961               codegen_);
962}
963void IntrinsicCodeGeneratorARM64::VisitUnsafePutObject(HInvoke* invoke) {
964  GenUnsafePut(invoke->GetLocations(),
965               Primitive::kPrimNot,
966               /* is_volatile */ false,
967               /* is_ordered */ false,
968               codegen_);
969}
970void IntrinsicCodeGeneratorARM64::VisitUnsafePutObjectOrdered(HInvoke* invoke) {
971  GenUnsafePut(invoke->GetLocations(),
972               Primitive::kPrimNot,
973               /* is_volatile */ false,
974               /* is_ordered */ true,
975               codegen_);
976}
977void IntrinsicCodeGeneratorARM64::VisitUnsafePutObjectVolatile(HInvoke* invoke) {
978  GenUnsafePut(invoke->GetLocations(),
979               Primitive::kPrimNot,
980               /* is_volatile */ true,
981               /* is_ordered */ false,
982               codegen_);
983}
984void IntrinsicCodeGeneratorARM64::VisitUnsafePutLong(HInvoke* invoke) {
985  GenUnsafePut(invoke->GetLocations(),
986               Primitive::kPrimLong,
987               /* is_volatile */ false,
988               /* is_ordered */ false,
989               codegen_);
990}
991void IntrinsicCodeGeneratorARM64::VisitUnsafePutLongOrdered(HInvoke* invoke) {
992  GenUnsafePut(invoke->GetLocations(),
993               Primitive::kPrimLong,
994               /* is_volatile */ false,
995               /* is_ordered */ true,
996               codegen_);
997}
998void IntrinsicCodeGeneratorARM64::VisitUnsafePutLongVolatile(HInvoke* invoke) {
999  GenUnsafePut(invoke->GetLocations(),
1000               Primitive::kPrimLong,
1001               /* is_volatile */ true,
1002               /* is_ordered */ false,
1003               codegen_);
1004}
1005
1006static void CreateIntIntIntIntIntToInt(ArenaAllocator* arena,
1007                                       HInvoke* invoke,
1008                                       Primitive::Type type) {
1009  LocationSummary* locations = new (arena) LocationSummary(invoke,
1010                                                           LocationSummary::kNoCall,
1011                                                           kIntrinsified);
1012  locations->SetInAt(0, Location::NoLocation());        // Unused receiver.
1013  locations->SetInAt(1, Location::RequiresRegister());
1014  locations->SetInAt(2, Location::RequiresRegister());
1015  locations->SetInAt(3, Location::RequiresRegister());
1016  locations->SetInAt(4, Location::RequiresRegister());
1017
1018  // If heap poisoning is enabled, we don't want the unpoisoning
1019  // operations to potentially clobber the output.
1020  Location::OutputOverlap overlaps = (kPoisonHeapReferences && type == Primitive::kPrimNot)
1021      ? Location::kOutputOverlap
1022      : Location::kNoOutputOverlap;
1023  locations->SetOut(Location::RequiresRegister(), overlaps);
1024}
1025
1026static void GenCas(LocationSummary* locations, Primitive::Type type, CodeGeneratorARM64* codegen) {
1027  vixl::MacroAssembler* masm = codegen->GetAssembler()->vixl_masm_;
1028
1029  Register out = WRegisterFrom(locations->Out());                  // Boolean result.
1030
1031  Register base = WRegisterFrom(locations->InAt(1));               // Object pointer.
1032  Register offset = XRegisterFrom(locations->InAt(2));             // Long offset.
1033  Register expected = RegisterFrom(locations->InAt(3), type);      // Expected.
1034  Register value = RegisterFrom(locations->InAt(4), type);         // Value.
1035
1036  // This needs to be before the temp registers, as MarkGCCard also uses VIXL temps.
1037  if (type == Primitive::kPrimNot) {
1038    // Mark card for object assuming new value is stored.
1039    bool value_can_be_null = true;  // TODO: Worth finding out this information?
1040    codegen->MarkGCCard(base, value, value_can_be_null);
1041  }
1042
1043  UseScratchRegisterScope temps(masm);
1044  Register tmp_ptr = temps.AcquireX();                             // Pointer to actual memory.
1045  Register tmp_value = temps.AcquireSameSizeAs(value);             // Value in memory.
1046
1047  Register tmp_32 = tmp_value.W();
1048
1049  __ Add(tmp_ptr, base.X(), Operand(offset));
1050
1051  if (kPoisonHeapReferences && type == Primitive::kPrimNot) {
1052    codegen->GetAssembler()->PoisonHeapReference(expected);
1053    if (value.Is(expected)) {
1054      // Do not poison `value`, as it is the same register as
1055      // `expected`, which has just been poisoned.
1056    } else {
1057      codegen->GetAssembler()->PoisonHeapReference(value);
1058    }
1059  }
1060
1061  // do {
1062  //   tmp_value = [tmp_ptr] - expected;
1063  // } while (tmp_value == 0 && failure([tmp_ptr] <- r_new_value));
1064  // result = tmp_value != 0;
1065
1066  vixl::Label loop_head, exit_loop;
1067  __ Bind(&loop_head);
1068  // TODO: When `type == Primitive::kPrimNot`, add a read barrier for
1069  // the reference stored in the object before attempting the CAS,
1070  // similar to the one in the art::Unsafe_compareAndSwapObject JNI
1071  // implementation.
1072  //
1073  // Note that this code is not (yet) used when read barriers are
1074  // enabled (see IntrinsicLocationsBuilderARM64::VisitUnsafeCASObject).
1075  DCHECK(!(type == Primitive::kPrimNot && kEmitCompilerReadBarrier));
1076  __ Ldaxr(tmp_value, MemOperand(tmp_ptr));
1077  __ Cmp(tmp_value, expected);
1078  __ B(&exit_loop, ne);
1079  __ Stlxr(tmp_32, value, MemOperand(tmp_ptr));
1080  __ Cbnz(tmp_32, &loop_head);
1081  __ Bind(&exit_loop);
1082  __ Cset(out, eq);
1083
1084  if (kPoisonHeapReferences && type == Primitive::kPrimNot) {
1085    codegen->GetAssembler()->UnpoisonHeapReference(expected);
1086    if (value.Is(expected)) {
1087      // Do not unpoison `value`, as it is the same register as
1088      // `expected`, which has just been unpoisoned.
1089    } else {
1090      codegen->GetAssembler()->UnpoisonHeapReference(value);
1091    }
1092  }
1093}
1094
1095void IntrinsicLocationsBuilderARM64::VisitUnsafeCASInt(HInvoke* invoke) {
1096  CreateIntIntIntIntIntToInt(arena_, invoke, Primitive::kPrimInt);
1097}
1098void IntrinsicLocationsBuilderARM64::VisitUnsafeCASLong(HInvoke* invoke) {
1099  CreateIntIntIntIntIntToInt(arena_, invoke, Primitive::kPrimLong);
1100}
1101void IntrinsicLocationsBuilderARM64::VisitUnsafeCASObject(HInvoke* invoke) {
1102  // The UnsafeCASObject intrinsic is missing a read barrier, and
1103  // therefore sometimes does not work as expected (b/25883050).
1104  // Turn it off temporarily as a quick fix, until the read barrier is
1105  // implemented (see TODO in GenCAS below).
1106  //
1107  // TODO(rpl): Fix this issue and re-enable this intrinsic with read barriers.
1108  if (kEmitCompilerReadBarrier) {
1109    return;
1110  }
1111
1112  CreateIntIntIntIntIntToInt(arena_, invoke, Primitive::kPrimNot);
1113}
1114
1115void IntrinsicCodeGeneratorARM64::VisitUnsafeCASInt(HInvoke* invoke) {
1116  GenCas(invoke->GetLocations(), Primitive::kPrimInt, codegen_);
1117}
1118void IntrinsicCodeGeneratorARM64::VisitUnsafeCASLong(HInvoke* invoke) {
1119  GenCas(invoke->GetLocations(), Primitive::kPrimLong, codegen_);
1120}
1121void IntrinsicCodeGeneratorARM64::VisitUnsafeCASObject(HInvoke* invoke) {
1122  GenCas(invoke->GetLocations(), Primitive::kPrimNot, codegen_);
1123}
1124
1125void IntrinsicLocationsBuilderARM64::VisitStringCharAt(HInvoke* invoke) {
1126  LocationSummary* locations = new (arena_) LocationSummary(invoke,
1127                                                            LocationSummary::kCallOnSlowPath,
1128                                                            kIntrinsified);
1129  locations->SetInAt(0, Location::RequiresRegister());
1130  locations->SetInAt(1, Location::RequiresRegister());
1131  // In case we need to go in the slow path, we can't have the output be the same
1132  // as the input: the current liveness analysis considers the input to be live
1133  // at the point of the call.
1134  locations->SetOut(Location::RequiresRegister(), Location::kOutputOverlap);
1135}
1136
1137void IntrinsicCodeGeneratorARM64::VisitStringCharAt(HInvoke* invoke) {
1138  vixl::MacroAssembler* masm = GetVIXLAssembler();
1139  LocationSummary* locations = invoke->GetLocations();
1140
1141  // Location of reference to data array
1142  const MemberOffset value_offset = mirror::String::ValueOffset();
1143  // Location of count
1144  const MemberOffset count_offset = mirror::String::CountOffset();
1145
1146  Register obj = WRegisterFrom(locations->InAt(0));  // String object pointer.
1147  Register idx = WRegisterFrom(locations->InAt(1));  // Index of character.
1148  Register out = WRegisterFrom(locations->Out());    // Result character.
1149
1150  UseScratchRegisterScope temps(masm);
1151  Register temp = temps.AcquireW();
1152  Register array_temp = temps.AcquireW();            // We can trade this for worse scheduling.
1153
1154  // TODO: Maybe we can support range check elimination. Overall, though, I think it's not worth
1155  //       the cost.
1156  // TODO: For simplicity, the index parameter is requested in a register, so different from Quick
1157  //       we will not optimize the code for constants (which would save a register).
1158
1159  SlowPathCodeARM64* slow_path = new (GetAllocator()) IntrinsicSlowPathARM64(invoke);
1160  codegen_->AddSlowPath(slow_path);
1161
1162  __ Ldr(temp, HeapOperand(obj, count_offset));          // temp = str.length.
1163  codegen_->MaybeRecordImplicitNullCheck(invoke);
1164  __ Cmp(idx, temp);
1165  __ B(hs, slow_path->GetEntryLabel());
1166
1167  __ Add(array_temp, obj, Operand(value_offset.Int32Value()));  // array_temp := str.value.
1168
1169  // Load the value.
1170  __ Ldrh(out, MemOperand(array_temp.X(), idx, UXTW, 1));  // out := array_temp[idx].
1171
1172  __ Bind(slow_path->GetExitLabel());
1173}
1174
1175void IntrinsicLocationsBuilderARM64::VisitStringCompareTo(HInvoke* invoke) {
1176  LocationSummary* locations = new (arena_) LocationSummary(invoke,
1177                                                            LocationSummary::kCall,
1178                                                            kIntrinsified);
1179  InvokeRuntimeCallingConvention calling_convention;
1180  locations->SetInAt(0, LocationFrom(calling_convention.GetRegisterAt(0)));
1181  locations->SetInAt(1, LocationFrom(calling_convention.GetRegisterAt(1)));
1182  locations->SetOut(calling_convention.GetReturnLocation(Primitive::kPrimInt));
1183}
1184
1185void IntrinsicCodeGeneratorARM64::VisitStringCompareTo(HInvoke* invoke) {
1186  vixl::MacroAssembler* masm = GetVIXLAssembler();
1187  LocationSummary* locations = invoke->GetLocations();
1188
1189  // Note that the null check must have been done earlier.
1190  DCHECK(!invoke->CanDoImplicitNullCheckOn(invoke->InputAt(0)));
1191
1192  Register argument = WRegisterFrom(locations->InAt(1));
1193  __ Cmp(argument, 0);
1194  SlowPathCodeARM64* slow_path = new (GetAllocator()) IntrinsicSlowPathARM64(invoke);
1195  codegen_->AddSlowPath(slow_path);
1196  __ B(eq, slow_path->GetEntryLabel());
1197
1198  __ Ldr(
1199      lr, MemOperand(tr, QUICK_ENTRYPOINT_OFFSET(kArm64WordSize, pStringCompareTo).Int32Value()));
1200  __ Blr(lr);
1201  __ Bind(slow_path->GetExitLabel());
1202}
1203
1204void IntrinsicLocationsBuilderARM64::VisitStringEquals(HInvoke* invoke) {
1205  LocationSummary* locations = new (arena_) LocationSummary(invoke,
1206                                                            LocationSummary::kNoCall,
1207                                                            kIntrinsified);
1208  locations->SetInAt(0, Location::RequiresRegister());
1209  locations->SetInAt(1, Location::RequiresRegister());
1210  // Temporary registers to store lengths of strings and for calculations.
1211  locations->AddTemp(Location::RequiresRegister());
1212  locations->AddTemp(Location::RequiresRegister());
1213
1214  locations->SetOut(Location::RequiresRegister(), Location::kOutputOverlap);
1215}
1216
1217void IntrinsicCodeGeneratorARM64::VisitStringEquals(HInvoke* invoke) {
1218  vixl::MacroAssembler* masm = GetVIXLAssembler();
1219  LocationSummary* locations = invoke->GetLocations();
1220
1221  Register str = WRegisterFrom(locations->InAt(0));
1222  Register arg = WRegisterFrom(locations->InAt(1));
1223  Register out = XRegisterFrom(locations->Out());
1224
1225  UseScratchRegisterScope scratch_scope(masm);
1226  Register temp = scratch_scope.AcquireW();
1227  Register temp1 = WRegisterFrom(locations->GetTemp(0));
1228  Register temp2 = WRegisterFrom(locations->GetTemp(1));
1229
1230  vixl::Label loop;
1231  vixl::Label end;
1232  vixl::Label return_true;
1233  vixl::Label return_false;
1234
1235  // Get offsets of count, value, and class fields within a string object.
1236  const int32_t count_offset = mirror::String::CountOffset().Int32Value();
1237  const int32_t value_offset = mirror::String::ValueOffset().Int32Value();
1238  const int32_t class_offset = mirror::Object::ClassOffset().Int32Value();
1239
1240  // Note that the null check must have been done earlier.
1241  DCHECK(!invoke->CanDoImplicitNullCheckOn(invoke->InputAt(0)));
1242
1243  // Check if input is null, return false if it is.
1244  __ Cbz(arg, &return_false);
1245
1246  // Reference equality check, return true if same reference.
1247  __ Cmp(str, arg);
1248  __ B(&return_true, eq);
1249
1250  // Instanceof check for the argument by comparing class fields.
1251  // All string objects must have the same type since String cannot be subclassed.
1252  // Receiver must be a string object, so its class field is equal to all strings' class fields.
1253  // If the argument is a string object, its class field must be equal to receiver's class field.
1254  __ Ldr(temp, MemOperand(str.X(), class_offset));
1255  __ Ldr(temp1, MemOperand(arg.X(), class_offset));
1256  __ Cmp(temp, temp1);
1257  __ B(&return_false, ne);
1258
1259  // Load lengths of this and argument strings.
1260  __ Ldr(temp, MemOperand(str.X(), count_offset));
1261  __ Ldr(temp1, MemOperand(arg.X(), count_offset));
1262  // Check if lengths are equal, return false if they're not.
1263  __ Cmp(temp, temp1);
1264  __ B(&return_false, ne);
1265  // Store offset of string value in preparation for comparison loop
1266  __ Mov(temp1, value_offset);
1267  // Return true if both strings are empty.
1268  __ Cbz(temp, &return_true);
1269
1270  // Assertions that must hold in order to compare strings 4 characters at a time.
1271  DCHECK_ALIGNED(value_offset, 8);
1272  static_assert(IsAligned<8>(kObjectAlignment), "String of odd length is not zero padded");
1273
1274  temp1 = temp1.X();
1275  temp2 = temp2.X();
1276
1277  // Loop to compare strings 4 characters at a time starting at the beginning of the string.
1278  // Ok to do this because strings are zero-padded to be 8-byte aligned.
1279  __ Bind(&loop);
1280  __ Ldr(out, MemOperand(str.X(), temp1));
1281  __ Ldr(temp2, MemOperand(arg.X(), temp1));
1282  __ Add(temp1, temp1, Operand(sizeof(uint64_t)));
1283  __ Cmp(out, temp2);
1284  __ B(&return_false, ne);
1285  __ Sub(temp, temp, Operand(4), SetFlags);
1286  __ B(&loop, gt);
1287
1288  // Return true and exit the function.
1289  // If loop does not result in returning false, we return true.
1290  __ Bind(&return_true);
1291  __ Mov(out, 1);
1292  __ B(&end);
1293
1294  // Return false and exit the function.
1295  __ Bind(&return_false);
1296  __ Mov(out, 0);
1297  __ Bind(&end);
1298}
1299
1300static void GenerateVisitStringIndexOf(HInvoke* invoke,
1301                                       vixl::MacroAssembler* masm,
1302                                       CodeGeneratorARM64* codegen,
1303                                       ArenaAllocator* allocator,
1304                                       bool start_at_zero) {
1305  LocationSummary* locations = invoke->GetLocations();
1306  Register tmp_reg = WRegisterFrom(locations->GetTemp(0));
1307
1308  // Note that the null check must have been done earlier.
1309  DCHECK(!invoke->CanDoImplicitNullCheckOn(invoke->InputAt(0)));
1310
1311  // Check for code points > 0xFFFF. Either a slow-path check when we don't know statically,
1312  // or directly dispatch if we have a constant.
1313  SlowPathCodeARM64* slow_path = nullptr;
1314  if (invoke->InputAt(1)->IsIntConstant()) {
1315    if (static_cast<uint32_t>(invoke->InputAt(1)->AsIntConstant()->GetValue()) > 0xFFFFU) {
1316      // Always needs the slow-path. We could directly dispatch to it, but this case should be
1317      // rare, so for simplicity just put the full slow-path down and branch unconditionally.
1318      slow_path = new (allocator) IntrinsicSlowPathARM64(invoke);
1319      codegen->AddSlowPath(slow_path);
1320      __ B(slow_path->GetEntryLabel());
1321      __ Bind(slow_path->GetExitLabel());
1322      return;
1323    }
1324  } else {
1325    Register char_reg = WRegisterFrom(locations->InAt(1));
1326    __ Mov(tmp_reg, 0xFFFF);
1327    __ Cmp(char_reg, Operand(tmp_reg));
1328    slow_path = new (allocator) IntrinsicSlowPathARM64(invoke);
1329    codegen->AddSlowPath(slow_path);
1330    __ B(hi, slow_path->GetEntryLabel());
1331  }
1332
1333  if (start_at_zero) {
1334    // Start-index = 0.
1335    __ Mov(tmp_reg, 0);
1336  }
1337
1338  __ Ldr(lr, MemOperand(tr, QUICK_ENTRYPOINT_OFFSET(kArm64WordSize, pIndexOf).Int32Value()));
1339  CheckEntrypointTypes<kQuickIndexOf, int32_t, void*, uint32_t, uint32_t>();
1340  __ Blr(lr);
1341
1342  if (slow_path != nullptr) {
1343    __ Bind(slow_path->GetExitLabel());
1344  }
1345}
1346
1347void IntrinsicLocationsBuilderARM64::VisitStringIndexOf(HInvoke* invoke) {
1348  LocationSummary* locations = new (arena_) LocationSummary(invoke,
1349                                                            LocationSummary::kCall,
1350                                                            kIntrinsified);
1351  // We have a hand-crafted assembly stub that follows the runtime calling convention. So it's
1352  // best to align the inputs accordingly.
1353  InvokeRuntimeCallingConvention calling_convention;
1354  locations->SetInAt(0, LocationFrom(calling_convention.GetRegisterAt(0)));
1355  locations->SetInAt(1, LocationFrom(calling_convention.GetRegisterAt(1)));
1356  locations->SetOut(calling_convention.GetReturnLocation(Primitive::kPrimInt));
1357
1358  // Need a temp for slow-path codepoint compare, and need to send start_index=0.
1359  locations->AddTemp(LocationFrom(calling_convention.GetRegisterAt(2)));
1360}
1361
1362void IntrinsicCodeGeneratorARM64::VisitStringIndexOf(HInvoke* invoke) {
1363  GenerateVisitStringIndexOf(
1364      invoke, GetVIXLAssembler(), codegen_, GetAllocator(), /* start_at_zero */ true);
1365}
1366
1367void IntrinsicLocationsBuilderARM64::VisitStringIndexOfAfter(HInvoke* invoke) {
1368  LocationSummary* locations = new (arena_) LocationSummary(invoke,
1369                                                            LocationSummary::kCall,
1370                                                            kIntrinsified);
1371  // We have a hand-crafted assembly stub that follows the runtime calling convention. So it's
1372  // best to align the inputs accordingly.
1373  InvokeRuntimeCallingConvention calling_convention;
1374  locations->SetInAt(0, LocationFrom(calling_convention.GetRegisterAt(0)));
1375  locations->SetInAt(1, LocationFrom(calling_convention.GetRegisterAt(1)));
1376  locations->SetInAt(2, LocationFrom(calling_convention.GetRegisterAt(2)));
1377  locations->SetOut(calling_convention.GetReturnLocation(Primitive::kPrimInt));
1378
1379  // Need a temp for slow-path codepoint compare.
1380  locations->AddTemp(Location::RequiresRegister());
1381}
1382
1383void IntrinsicCodeGeneratorARM64::VisitStringIndexOfAfter(HInvoke* invoke) {
1384  GenerateVisitStringIndexOf(
1385      invoke, GetVIXLAssembler(), codegen_, GetAllocator(), /* start_at_zero */ false);
1386}
1387
1388void IntrinsicLocationsBuilderARM64::VisitStringNewStringFromBytes(HInvoke* invoke) {
1389  LocationSummary* locations = new (arena_) LocationSummary(invoke,
1390                                                            LocationSummary::kCall,
1391                                                            kIntrinsified);
1392  InvokeRuntimeCallingConvention calling_convention;
1393  locations->SetInAt(0, LocationFrom(calling_convention.GetRegisterAt(0)));
1394  locations->SetInAt(1, LocationFrom(calling_convention.GetRegisterAt(1)));
1395  locations->SetInAt(2, LocationFrom(calling_convention.GetRegisterAt(2)));
1396  locations->SetInAt(3, LocationFrom(calling_convention.GetRegisterAt(3)));
1397  locations->SetOut(calling_convention.GetReturnLocation(Primitive::kPrimNot));
1398}
1399
1400void IntrinsicCodeGeneratorARM64::VisitStringNewStringFromBytes(HInvoke* invoke) {
1401  vixl::MacroAssembler* masm = GetVIXLAssembler();
1402  LocationSummary* locations = invoke->GetLocations();
1403
1404  Register byte_array = WRegisterFrom(locations->InAt(0));
1405  __ Cmp(byte_array, 0);
1406  SlowPathCodeARM64* slow_path = new (GetAllocator()) IntrinsicSlowPathARM64(invoke);
1407  codegen_->AddSlowPath(slow_path);
1408  __ B(eq, slow_path->GetEntryLabel());
1409
1410  __ Ldr(lr,
1411      MemOperand(tr, QUICK_ENTRYPOINT_OFFSET(kArm64WordSize, pAllocStringFromBytes).Int32Value()));
1412  codegen_->RecordPcInfo(invoke, invoke->GetDexPc());
1413  __ Blr(lr);
1414  __ Bind(slow_path->GetExitLabel());
1415}
1416
1417void IntrinsicLocationsBuilderARM64::VisitStringNewStringFromChars(HInvoke* invoke) {
1418  LocationSummary* locations = new (arena_) LocationSummary(invoke,
1419                                                            LocationSummary::kCall,
1420                                                            kIntrinsified);
1421  InvokeRuntimeCallingConvention calling_convention;
1422  locations->SetInAt(0, LocationFrom(calling_convention.GetRegisterAt(0)));
1423  locations->SetInAt(1, LocationFrom(calling_convention.GetRegisterAt(1)));
1424  locations->SetInAt(2, LocationFrom(calling_convention.GetRegisterAt(2)));
1425  locations->SetOut(calling_convention.GetReturnLocation(Primitive::kPrimNot));
1426}
1427
1428void IntrinsicCodeGeneratorARM64::VisitStringNewStringFromChars(HInvoke* invoke) {
1429  vixl::MacroAssembler* masm = GetVIXLAssembler();
1430
1431  // No need to emit code checking whether `locations->InAt(2)` is a null
1432  // pointer, as callers of the native method
1433  //
1434  //   java.lang.StringFactory.newStringFromChars(int offset, int charCount, char[] data)
1435  //
1436  // all include a null check on `data` before calling that method.
1437  __ Ldr(lr,
1438      MemOperand(tr, QUICK_ENTRYPOINT_OFFSET(kArm64WordSize, pAllocStringFromChars).Int32Value()));
1439  codegen_->RecordPcInfo(invoke, invoke->GetDexPc());
1440  __ Blr(lr);
1441}
1442
1443void IntrinsicLocationsBuilderARM64::VisitStringNewStringFromString(HInvoke* invoke) {
1444  // The inputs plus one temp.
1445  LocationSummary* locations = new (arena_) LocationSummary(invoke,
1446                                                            LocationSummary::kCall,
1447                                                            kIntrinsified);
1448  InvokeRuntimeCallingConvention calling_convention;
1449  locations->SetInAt(0, LocationFrom(calling_convention.GetRegisterAt(0)));
1450  locations->SetInAt(1, LocationFrom(calling_convention.GetRegisterAt(1)));
1451  locations->SetInAt(2, LocationFrom(calling_convention.GetRegisterAt(2)));
1452  locations->SetOut(calling_convention.GetReturnLocation(Primitive::kPrimNot));
1453}
1454
1455void IntrinsicCodeGeneratorARM64::VisitStringNewStringFromString(HInvoke* invoke) {
1456  vixl::MacroAssembler* masm = GetVIXLAssembler();
1457  LocationSummary* locations = invoke->GetLocations();
1458
1459  Register string_to_copy = WRegisterFrom(locations->InAt(0));
1460  __ Cmp(string_to_copy, 0);
1461  SlowPathCodeARM64* slow_path = new (GetAllocator()) IntrinsicSlowPathARM64(invoke);
1462  codegen_->AddSlowPath(slow_path);
1463  __ B(eq, slow_path->GetEntryLabel());
1464
1465  __ Ldr(lr,
1466      MemOperand(tr, QUICK_ENTRYPOINT_OFFSET(kArm64WordSize, pAllocStringFromString).Int32Value()));
1467  codegen_->RecordPcInfo(invoke, invoke->GetDexPc());
1468  __ Blr(lr);
1469  __ Bind(slow_path->GetExitLabel());
1470}
1471
1472static void CreateFPToFPCallLocations(ArenaAllocator* arena, HInvoke* invoke) {
1473  DCHECK_EQ(invoke->GetNumberOfArguments(), 1U);
1474  DCHECK(Primitive::IsFloatingPointType(invoke->InputAt(0)->GetType()));
1475  DCHECK(Primitive::IsFloatingPointType(invoke->GetType()));
1476
1477  LocationSummary* const locations = new (arena) LocationSummary(invoke,
1478                                                                 LocationSummary::kCall,
1479                                                                 kIntrinsified);
1480  InvokeRuntimeCallingConvention calling_convention;
1481
1482  locations->SetInAt(0, LocationFrom(calling_convention.GetFpuRegisterAt(0)));
1483  locations->SetOut(calling_convention.GetReturnLocation(invoke->GetType()));
1484}
1485
1486static void CreateFPFPToFPCallLocations(ArenaAllocator* arena, HInvoke* invoke) {
1487  DCHECK_EQ(invoke->GetNumberOfArguments(), 2U);
1488  DCHECK(Primitive::IsFloatingPointType(invoke->InputAt(0)->GetType()));
1489  DCHECK(Primitive::IsFloatingPointType(invoke->InputAt(1)->GetType()));
1490  DCHECK(Primitive::IsFloatingPointType(invoke->GetType()));
1491
1492  LocationSummary* const locations = new (arena) LocationSummary(invoke,
1493                                                                 LocationSummary::kCall,
1494                                                                 kIntrinsified);
1495  InvokeRuntimeCallingConvention calling_convention;
1496
1497  locations->SetInAt(0, LocationFrom(calling_convention.GetFpuRegisterAt(0)));
1498  locations->SetInAt(1, LocationFrom(calling_convention.GetFpuRegisterAt(1)));
1499  locations->SetOut(calling_convention.GetReturnLocation(invoke->GetType()));
1500}
1501
1502static void GenFPToFPCall(HInvoke* invoke,
1503                          vixl::MacroAssembler* masm,
1504                          CodeGeneratorARM64* codegen,
1505                          QuickEntrypointEnum entry) {
1506  __ Ldr(lr, MemOperand(tr, GetThreadOffset<kArm64WordSize>(entry).Int32Value()));
1507  __ Blr(lr);
1508  codegen->RecordPcInfo(invoke, invoke->GetDexPc());
1509}
1510
1511void IntrinsicLocationsBuilderARM64::VisitMathCos(HInvoke* invoke) {
1512  CreateFPToFPCallLocations(arena_, invoke);
1513}
1514
1515void IntrinsicCodeGeneratorARM64::VisitMathCos(HInvoke* invoke) {
1516  GenFPToFPCall(invoke, GetVIXLAssembler(), codegen_, kQuickCos);
1517}
1518
1519void IntrinsicLocationsBuilderARM64::VisitMathSin(HInvoke* invoke) {
1520  CreateFPToFPCallLocations(arena_, invoke);
1521}
1522
1523void IntrinsicCodeGeneratorARM64::VisitMathSin(HInvoke* invoke) {
1524  GenFPToFPCall(invoke, GetVIXLAssembler(), codegen_, kQuickSin);
1525}
1526
1527void IntrinsicLocationsBuilderARM64::VisitMathAcos(HInvoke* invoke) {
1528  CreateFPToFPCallLocations(arena_, invoke);
1529}
1530
1531void IntrinsicCodeGeneratorARM64::VisitMathAcos(HInvoke* invoke) {
1532  GenFPToFPCall(invoke, GetVIXLAssembler(), codegen_, kQuickAcos);
1533}
1534
1535void IntrinsicLocationsBuilderARM64::VisitMathAsin(HInvoke* invoke) {
1536  CreateFPToFPCallLocations(arena_, invoke);
1537}
1538
1539void IntrinsicCodeGeneratorARM64::VisitMathAsin(HInvoke* invoke) {
1540  GenFPToFPCall(invoke, GetVIXLAssembler(), codegen_, kQuickAsin);
1541}
1542
1543void IntrinsicLocationsBuilderARM64::VisitMathAtan(HInvoke* invoke) {
1544  CreateFPToFPCallLocations(arena_, invoke);
1545}
1546
1547void IntrinsicCodeGeneratorARM64::VisitMathAtan(HInvoke* invoke) {
1548  GenFPToFPCall(invoke, GetVIXLAssembler(), codegen_, kQuickAtan);
1549}
1550
1551void IntrinsicLocationsBuilderARM64::VisitMathCbrt(HInvoke* invoke) {
1552  CreateFPToFPCallLocations(arena_, invoke);
1553}
1554
1555void IntrinsicCodeGeneratorARM64::VisitMathCbrt(HInvoke* invoke) {
1556  GenFPToFPCall(invoke, GetVIXLAssembler(), codegen_, kQuickCbrt);
1557}
1558
1559void IntrinsicLocationsBuilderARM64::VisitMathCosh(HInvoke* invoke) {
1560  CreateFPToFPCallLocations(arena_, invoke);
1561}
1562
1563void IntrinsicCodeGeneratorARM64::VisitMathCosh(HInvoke* invoke) {
1564  GenFPToFPCall(invoke, GetVIXLAssembler(), codegen_, kQuickCosh);
1565}
1566
1567void IntrinsicLocationsBuilderARM64::VisitMathExp(HInvoke* invoke) {
1568  CreateFPToFPCallLocations(arena_, invoke);
1569}
1570
1571void IntrinsicCodeGeneratorARM64::VisitMathExp(HInvoke* invoke) {
1572  GenFPToFPCall(invoke, GetVIXLAssembler(), codegen_, kQuickExp);
1573}
1574
1575void IntrinsicLocationsBuilderARM64::VisitMathExpm1(HInvoke* invoke) {
1576  CreateFPToFPCallLocations(arena_, invoke);
1577}
1578
1579void IntrinsicCodeGeneratorARM64::VisitMathExpm1(HInvoke* invoke) {
1580  GenFPToFPCall(invoke, GetVIXLAssembler(), codegen_, kQuickExpm1);
1581}
1582
1583void IntrinsicLocationsBuilderARM64::VisitMathLog(HInvoke* invoke) {
1584  CreateFPToFPCallLocations(arena_, invoke);
1585}
1586
1587void IntrinsicCodeGeneratorARM64::VisitMathLog(HInvoke* invoke) {
1588  GenFPToFPCall(invoke, GetVIXLAssembler(), codegen_, kQuickLog);
1589}
1590
1591void IntrinsicLocationsBuilderARM64::VisitMathLog10(HInvoke* invoke) {
1592  CreateFPToFPCallLocations(arena_, invoke);
1593}
1594
1595void IntrinsicCodeGeneratorARM64::VisitMathLog10(HInvoke* invoke) {
1596  GenFPToFPCall(invoke, GetVIXLAssembler(), codegen_, kQuickLog10);
1597}
1598
1599void IntrinsicLocationsBuilderARM64::VisitMathSinh(HInvoke* invoke) {
1600  CreateFPToFPCallLocations(arena_, invoke);
1601}
1602
1603void IntrinsicCodeGeneratorARM64::VisitMathSinh(HInvoke* invoke) {
1604  GenFPToFPCall(invoke, GetVIXLAssembler(), codegen_, kQuickSinh);
1605}
1606
1607void IntrinsicLocationsBuilderARM64::VisitMathTan(HInvoke* invoke) {
1608  CreateFPToFPCallLocations(arena_, invoke);
1609}
1610
1611void IntrinsicCodeGeneratorARM64::VisitMathTan(HInvoke* invoke) {
1612  GenFPToFPCall(invoke, GetVIXLAssembler(), codegen_, kQuickTan);
1613}
1614
1615void IntrinsicLocationsBuilderARM64::VisitMathTanh(HInvoke* invoke) {
1616  CreateFPToFPCallLocations(arena_, invoke);
1617}
1618
1619void IntrinsicCodeGeneratorARM64::VisitMathTanh(HInvoke* invoke) {
1620  GenFPToFPCall(invoke, GetVIXLAssembler(), codegen_, kQuickTanh);
1621}
1622
1623void IntrinsicLocationsBuilderARM64::VisitMathAtan2(HInvoke* invoke) {
1624  CreateFPFPToFPCallLocations(arena_, invoke);
1625}
1626
1627void IntrinsicCodeGeneratorARM64::VisitMathAtan2(HInvoke* invoke) {
1628  GenFPToFPCall(invoke, GetVIXLAssembler(), codegen_, kQuickAtan2);
1629}
1630
1631void IntrinsicLocationsBuilderARM64::VisitMathHypot(HInvoke* invoke) {
1632  CreateFPFPToFPCallLocations(arena_, invoke);
1633}
1634
1635void IntrinsicCodeGeneratorARM64::VisitMathHypot(HInvoke* invoke) {
1636  GenFPToFPCall(invoke, GetVIXLAssembler(), codegen_, kQuickHypot);
1637}
1638
1639void IntrinsicLocationsBuilderARM64::VisitMathNextAfter(HInvoke* invoke) {
1640  CreateFPFPToFPCallLocations(arena_, invoke);
1641}
1642
1643void IntrinsicCodeGeneratorARM64::VisitMathNextAfter(HInvoke* invoke) {
1644  GenFPToFPCall(invoke, GetVIXLAssembler(), codegen_, kQuickNextAfter);
1645}
1646
1647void IntrinsicLocationsBuilderARM64::VisitStringGetCharsNoCheck(HInvoke* invoke) {
1648  LocationSummary* locations = new (arena_) LocationSummary(invoke,
1649                                                            LocationSummary::kNoCall,
1650                                                            kIntrinsified);
1651  locations->SetInAt(0, Location::RequiresRegister());
1652  locations->SetInAt(1, Location::RequiresRegister());
1653  locations->SetInAt(2, Location::RequiresRegister());
1654  locations->SetInAt(3, Location::RequiresRegister());
1655  locations->SetInAt(4, Location::RequiresRegister());
1656
1657  locations->AddTemp(Location::RequiresRegister());
1658  locations->AddTemp(Location::RequiresRegister());
1659}
1660
1661void IntrinsicCodeGeneratorARM64::VisitStringGetCharsNoCheck(HInvoke* invoke) {
1662  vixl::MacroAssembler* masm = GetVIXLAssembler();
1663  LocationSummary* locations = invoke->GetLocations();
1664
1665  // Check assumption that sizeof(Char) is 2 (used in scaling below).
1666  const size_t char_size = Primitive::ComponentSize(Primitive::kPrimChar);
1667  DCHECK_EQ(char_size, 2u);
1668
1669  // Location of data in char array buffer.
1670  const uint32_t data_offset = mirror::Array::DataOffset(char_size).Uint32Value();
1671
1672  // Location of char array data in string.
1673  const uint32_t value_offset = mirror::String::ValueOffset().Uint32Value();
1674
1675  // void getCharsNoCheck(int srcBegin, int srcEnd, char[] dst, int dstBegin);
1676  // Since getChars() calls getCharsNoCheck() - we use registers rather than constants.
1677  Register srcObj = XRegisterFrom(locations->InAt(0));
1678  Register srcBegin = XRegisterFrom(locations->InAt(1));
1679  Register srcEnd = XRegisterFrom(locations->InAt(2));
1680  Register dstObj = XRegisterFrom(locations->InAt(3));
1681  Register dstBegin = XRegisterFrom(locations->InAt(4));
1682
1683  Register src_ptr = XRegisterFrom(locations->GetTemp(0));
1684  Register src_ptr_end = XRegisterFrom(locations->GetTemp(1));
1685
1686  UseScratchRegisterScope temps(masm);
1687  Register dst_ptr = temps.AcquireX();
1688  Register tmp = temps.AcquireW();
1689
1690  // src range to copy.
1691  __ Add(src_ptr, srcObj, Operand(value_offset));
1692  __ Add(src_ptr_end, src_ptr, Operand(srcEnd, LSL, 1));
1693  __ Add(src_ptr, src_ptr, Operand(srcBegin, LSL, 1));
1694
1695  // dst to be copied.
1696  __ Add(dst_ptr, dstObj, Operand(data_offset));
1697  __ Add(dst_ptr, dst_ptr, Operand(dstBegin, LSL, 1));
1698
1699  // Do the copy.
1700  vixl::Label loop, done;
1701  __ Bind(&loop);
1702  __ Cmp(src_ptr, src_ptr_end);
1703  __ B(&done, eq);
1704  __ Ldrh(tmp, MemOperand(src_ptr, char_size, vixl::PostIndex));
1705  __ Strh(tmp, MemOperand(dst_ptr, char_size, vixl::PostIndex));
1706  __ B(&loop);
1707  __ Bind(&done);
1708}
1709
1710// Mirrors ARRAYCOPY_SHORT_CHAR_ARRAY_THRESHOLD in libcore, so we can choose to use the native
1711// implementation there for longer copy lengths.
1712static constexpr int32_t kSystemArrayCopyThreshold = 32;
1713
1714static void SetSystemArrayCopyLocationRequires(LocationSummary* locations,
1715                                               uint32_t at,
1716                                               HInstruction* input) {
1717  HIntConstant* const_input = input->AsIntConstant();
1718  if (const_input != nullptr && !vixl::Assembler::IsImmAddSub(const_input->GetValue())) {
1719    locations->SetInAt(at, Location::RequiresRegister());
1720  } else {
1721    locations->SetInAt(at, Location::RegisterOrConstant(input));
1722  }
1723}
1724
1725void IntrinsicLocationsBuilderARM64::VisitSystemArrayCopyChar(HInvoke* invoke) {
1726  // Check to see if we have known failures that will cause us to have to bail out
1727  // to the runtime, and just generate the runtime call directly.
1728  HIntConstant* src_pos = invoke->InputAt(1)->AsIntConstant();
1729  HIntConstant* dst_pos = invoke->InputAt(3)->AsIntConstant();
1730
1731  // The positions must be non-negative.
1732  if ((src_pos != nullptr && src_pos->GetValue() < 0) ||
1733      (dst_pos != nullptr && dst_pos->GetValue() < 0)) {
1734    // We will have to fail anyways.
1735    return;
1736  }
1737
1738  // The length must be >= 0 and not so long that we would (currently) prefer libcore's
1739  // native implementation.
1740  HIntConstant* length = invoke->InputAt(4)->AsIntConstant();
1741  if (length != nullptr) {
1742    int32_t len = length->GetValue();
1743    if (len < 0 || len > kSystemArrayCopyThreshold) {
1744      // Just call as normal.
1745      return;
1746    }
1747  }
1748
1749  ArenaAllocator* allocator = invoke->GetBlock()->GetGraph()->GetArena();
1750  LocationSummary* locations = new (allocator) LocationSummary(invoke,
1751                                                               LocationSummary::kCallOnSlowPath,
1752                                                               kIntrinsified);
1753  // arraycopy(char[] src, int src_pos, char[] dst, int dst_pos, int length).
1754  locations->SetInAt(0, Location::RequiresRegister());
1755  SetSystemArrayCopyLocationRequires(locations, 1, invoke->InputAt(1));
1756  locations->SetInAt(2, Location::RequiresRegister());
1757  SetSystemArrayCopyLocationRequires(locations, 3, invoke->InputAt(3));
1758  SetSystemArrayCopyLocationRequires(locations, 4, invoke->InputAt(4));
1759
1760  locations->AddTemp(Location::RequiresRegister());
1761  locations->AddTemp(Location::RequiresRegister());
1762  locations->AddTemp(Location::RequiresRegister());
1763}
1764
1765static void CheckSystemArrayCopyPosition(vixl::MacroAssembler* masm,
1766                                         const Location& pos,
1767                                         const Register& input,
1768                                         const Location& length,
1769                                         SlowPathCodeARM64* slow_path,
1770                                         const Register& input_len,
1771                                         const Register& temp,
1772                                         bool length_is_input_length = false) {
1773  const int32_t length_offset = mirror::Array::LengthOffset().Int32Value();
1774  if (pos.IsConstant()) {
1775    int32_t pos_const = pos.GetConstant()->AsIntConstant()->GetValue();
1776    if (pos_const == 0) {
1777      if (!length_is_input_length) {
1778        // Check that length(input) >= length.
1779        __ Ldr(temp, MemOperand(input, length_offset));
1780        __ Cmp(temp, OperandFrom(length, Primitive::kPrimInt));
1781        __ B(slow_path->GetEntryLabel(), lt);
1782      }
1783    } else {
1784      // Check that length(input) >= pos.
1785      __ Ldr(input_len, MemOperand(input, length_offset));
1786      __ Subs(temp, input_len, pos_const);
1787      __ B(slow_path->GetEntryLabel(), lt);
1788
1789      // Check that (length(input) - pos) >= length.
1790      __ Cmp(temp, OperandFrom(length, Primitive::kPrimInt));
1791      __ B(slow_path->GetEntryLabel(), lt);
1792    }
1793  } else if (length_is_input_length) {
1794    // The only way the copy can succeed is if pos is zero.
1795    __ Cbnz(WRegisterFrom(pos), slow_path->GetEntryLabel());
1796  } else {
1797    // Check that pos >= 0.
1798    Register pos_reg = WRegisterFrom(pos);
1799    __ Tbnz(pos_reg, pos_reg.size() - 1, slow_path->GetEntryLabel());
1800
1801    // Check that pos <= length(input) && (length(input) - pos) >= length.
1802    __ Ldr(temp, MemOperand(input, length_offset));
1803    __ Subs(temp, temp, pos_reg);
1804    // Ccmp if length(input) >= pos, else definitely bail to slow path (N!=V == lt).
1805    __ Ccmp(temp, OperandFrom(length, Primitive::kPrimInt), NFlag, ge);
1806    __ B(slow_path->GetEntryLabel(), lt);
1807  }
1808}
1809
1810// Compute base source address, base destination address, and end source address
1811// for System.arraycopy* intrinsics.
1812static void GenSystemArrayCopyAddresses(vixl::MacroAssembler* masm,
1813                                        Primitive::Type type,
1814                                        const Register& src,
1815                                        const Location& src_pos,
1816                                        const Register& dst,
1817                                        const Location& dst_pos,
1818                                        const Location& copy_length,
1819                                        const Register& src_base,
1820                                        const Register& dst_base,
1821                                        const Register& src_end) {
1822  DCHECK(type == Primitive::kPrimNot || type == Primitive::kPrimChar)
1823         << "Unexpected element type: "
1824         << type;
1825  const int32_t char_size = Primitive::ComponentSize(type);
1826  const int32_t char_size_shift = Primitive::ComponentSizeShift(type);
1827
1828  uint32_t offset = mirror::Array::DataOffset(char_size).Uint32Value();
1829  if (src_pos.IsConstant()) {
1830    int32_t constant = src_pos.GetConstant()->AsIntConstant()->GetValue();
1831    __ Add(src_base, src, char_size * constant + offset);
1832  } else {
1833    __ Add(src_base, src, offset);
1834    __ Add(src_base,
1835           src_base,
1836           Operand(XRegisterFrom(src_pos), LSL, char_size_shift));
1837  }
1838
1839  if (dst_pos.IsConstant()) {
1840    int32_t constant = dst_pos.GetConstant()->AsIntConstant()->GetValue();
1841    __ Add(dst_base, dst, char_size * constant + offset);
1842  } else {
1843    __ Add(dst_base, dst, offset);
1844    __ Add(dst_base,
1845           dst_base,
1846           Operand(XRegisterFrom(dst_pos), LSL, char_size_shift));
1847  }
1848
1849  if (copy_length.IsConstant()) {
1850    int32_t constant = copy_length.GetConstant()->AsIntConstant()->GetValue();
1851    __ Add(src_end, src_base, char_size * constant);
1852  } else {
1853    __ Add(src_end,
1854           src_base,
1855           Operand(XRegisterFrom(copy_length), LSL, char_size_shift));
1856  }
1857}
1858
1859void IntrinsicCodeGeneratorARM64::VisitSystemArrayCopyChar(HInvoke* invoke) {
1860  vixl::MacroAssembler* masm = GetVIXLAssembler();
1861  LocationSummary* locations = invoke->GetLocations();
1862  Register src = XRegisterFrom(locations->InAt(0));
1863  Location src_pos = locations->InAt(1);
1864  Register dst = XRegisterFrom(locations->InAt(2));
1865  Location dst_pos = locations->InAt(3);
1866  Location length = locations->InAt(4);
1867
1868  SlowPathCodeARM64* slow_path = new (GetAllocator()) IntrinsicSlowPathARM64(invoke);
1869  codegen_->AddSlowPath(slow_path);
1870
1871  // If source and destination are the same, take the slow path. Overlapping copy regions must be
1872  // copied in reverse and we can't know in all cases if it's needed.
1873  __ Cmp(src, dst);
1874  __ B(slow_path->GetEntryLabel(), eq);
1875
1876  // Bail out if the source is null.
1877  __ Cbz(src, slow_path->GetEntryLabel());
1878
1879  // Bail out if the destination is null.
1880  __ Cbz(dst, slow_path->GetEntryLabel());
1881
1882  if (!length.IsConstant()) {
1883    // If the length is negative, bail out.
1884    __ Tbnz(WRegisterFrom(length), kWRegSize - 1, slow_path->GetEntryLabel());
1885    // If the length > 32 then (currently) prefer libcore's native implementation.
1886    __ Cmp(WRegisterFrom(length), kSystemArrayCopyThreshold);
1887    __ B(slow_path->GetEntryLabel(), gt);
1888  } else {
1889    // We have already checked in the LocationsBuilder for the constant case.
1890    DCHECK_GE(length.GetConstant()->AsIntConstant()->GetValue(), 0);
1891    DCHECK_LE(length.GetConstant()->AsIntConstant()->GetValue(), 32);
1892  }
1893
1894  Register src_curr_addr = WRegisterFrom(locations->GetTemp(0));
1895  Register dst_curr_addr = WRegisterFrom(locations->GetTemp(1));
1896  Register src_stop_addr = WRegisterFrom(locations->GetTemp(2));
1897
1898  CheckSystemArrayCopyPosition(masm,
1899                               src_pos,
1900                               src,
1901                               length,
1902                               slow_path,
1903                               src_curr_addr,
1904                               dst_curr_addr,
1905                               false);
1906
1907  CheckSystemArrayCopyPosition(masm,
1908                               dst_pos,
1909                               dst,
1910                               length,
1911                               slow_path,
1912                               src_curr_addr,
1913                               dst_curr_addr,
1914                               false);
1915
1916  src_curr_addr = src_curr_addr.X();
1917  dst_curr_addr = dst_curr_addr.X();
1918  src_stop_addr = src_stop_addr.X();
1919
1920  GenSystemArrayCopyAddresses(masm,
1921                              Primitive::kPrimChar,
1922                              src,
1923                              src_pos,
1924                              dst,
1925                              dst_pos,
1926                              length,
1927                              src_curr_addr,
1928                              dst_curr_addr,
1929                              src_stop_addr);
1930
1931  // Iterate over the arrays and do a raw copy of the chars.
1932  const int32_t char_size = Primitive::ComponentSize(Primitive::kPrimChar);
1933  UseScratchRegisterScope temps(masm);
1934  Register tmp = temps.AcquireW();
1935  vixl::Label loop, done;
1936  __ Bind(&loop);
1937  __ Cmp(src_curr_addr, src_stop_addr);
1938  __ B(&done, eq);
1939  __ Ldrh(tmp, MemOperand(src_curr_addr, char_size, vixl::PostIndex));
1940  __ Strh(tmp, MemOperand(dst_curr_addr, char_size, vixl::PostIndex));
1941  __ B(&loop);
1942  __ Bind(&done);
1943
1944  __ Bind(slow_path->GetExitLabel());
1945}
1946
1947UNIMPLEMENTED_INTRINSIC(ARM64, SystemArrayCopy)
1948UNIMPLEMENTED_INTRINSIC(ARM64, ReferenceGetReferent)
1949UNIMPLEMENTED_INTRINSIC(ARM64, FloatIsInfinite)
1950UNIMPLEMENTED_INTRINSIC(ARM64, DoubleIsInfinite)
1951UNIMPLEMENTED_INTRINSIC(ARM64, IntegerHighestOneBit)
1952UNIMPLEMENTED_INTRINSIC(ARM64, LongHighestOneBit)
1953UNIMPLEMENTED_INTRINSIC(ARM64, IntegerLowestOneBit)
1954UNIMPLEMENTED_INTRINSIC(ARM64, LongLowestOneBit)
1955
1956// 1.8.
1957UNIMPLEMENTED_INTRINSIC(ARM64, UnsafeGetAndAddInt)
1958UNIMPLEMENTED_INTRINSIC(ARM64, UnsafeGetAndAddLong)
1959UNIMPLEMENTED_INTRINSIC(ARM64, UnsafeGetAndSetInt)
1960UNIMPLEMENTED_INTRINSIC(ARM64, UnsafeGetAndSetLong)
1961UNIMPLEMENTED_INTRINSIC(ARM64, UnsafeGetAndSetObject)
1962UNIMPLEMENTED_INTRINSIC(ARM64, UnsafeLoadFence)
1963UNIMPLEMENTED_INTRINSIC(ARM64, UnsafeStoreFence)
1964UNIMPLEMENTED_INTRINSIC(ARM64, UnsafeFullFence)
1965
1966UNREACHABLE_INTRINSICS(ARM64)
1967
1968#undef __
1969
1970}  // namespace arm64
1971}  // namespace art
1972