intrinsics_arm64.cc revision 2e50ecb95ddf645595491438cf35e79b705ee366 
1/*
2 * Copyright (C) 2015 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *      http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17#include "intrinsics_arm64.h"
18
19#include "arch/arm64/instruction_set_features_arm64.h"
20#include "art_method.h"
21#include "code_generator_arm64.h"
22#include "common_arm64.h"
23#include "entrypoints/quick/quick_entrypoints.h"
24#include "intrinsics.h"
25#include "mirror/array-inl.h"
26#include "mirror/string.h"
27#include "thread.h"
28#include "utils/arm64/assembler_arm64.h"
29#include "utils/arm64/constants_arm64.h"
30
31#include "vixl/a64/disasm-a64.h"
32#include "vixl/a64/macro-assembler-a64.h"
33
34using namespace vixl;   // NOLINT(build/namespaces)
35
36namespace art {
37
38namespace arm64 {
39
40using helpers::DRegisterFrom;
41using helpers::FPRegisterFrom;
42using helpers::HeapOperand;
43using helpers::LocationFrom;
44using helpers::OperandFrom;
45using helpers::RegisterFrom;
46using helpers::SRegisterFrom;
47using helpers::WRegisterFrom;
48using helpers::XRegisterFrom;
49
50namespace {
51
52ALWAYS_INLINE inline MemOperand AbsoluteHeapOperandFrom(Location location, size_t offset = 0) {
53  return MemOperand(XRegisterFrom(location), offset);
54}
55
56}  // namespace
57
58vixl::MacroAssembler* IntrinsicCodeGeneratorARM64::GetVIXLAssembler() {
59  return codegen_->GetAssembler()->vixl_masm_;
60}
61
62ArenaAllocator* IntrinsicCodeGeneratorARM64::GetAllocator() {
63  return codegen_->GetGraph()->GetArena();
64}
65
66#define __ codegen->GetAssembler()->vixl_masm_->
67
68static void MoveFromReturnRegister(Location trg,
69                                   Primitive::Type type,
70                                   CodeGeneratorARM64* codegen) {
71  if (!trg.IsValid()) {
72    DCHECK(type == Primitive::kPrimVoid);
73    return;
74  }
75
76  DCHECK_NE(type, Primitive::kPrimVoid);
77
78  if (Primitive::IsIntegralType(type) || type == Primitive::kPrimNot) {
79    Register trg_reg = RegisterFrom(trg, type);
80    Register res_reg = RegisterFrom(ARM64ReturnLocation(type), type);
81    __ Mov(trg_reg, res_reg, kDiscardForSameWReg);
82  } else {
83    FPRegister trg_reg = FPRegisterFrom(trg, type);
84    FPRegister res_reg = FPRegisterFrom(ARM64ReturnLocation(type), type);
85    __ Fmov(trg_reg, res_reg);
86  }
87}
88
89static void MoveArguments(HInvoke* invoke, CodeGeneratorARM64* codegen) {
90  InvokeDexCallingConventionVisitorARM64 calling_convention_visitor;
91  IntrinsicVisitor::MoveArguments(invoke, codegen, &calling_convention_visitor);
92}
93
94// Slow-path for fallback (calling the managed code to handle the intrinsic) in an intrinsified
95// call. This will copy the arguments into the positions for a regular call.
96//
97// Note: The actual parameters are required to be in the locations given by the invoke's location
98//       summary. If an intrinsic modifies those locations before a slowpath call, they must be
99//       restored!
100class IntrinsicSlowPathARM64 : public SlowPathCodeARM64 {
101 public:
102  explicit IntrinsicSlowPathARM64(HInvoke* invoke) : invoke_(invoke) { }
103
104  void EmitNativeCode(CodeGenerator* codegen_in) OVERRIDE {
105    CodeGeneratorARM64* codegen = down_cast<CodeGeneratorARM64*>(codegen_in);
106    __ Bind(GetEntryLabel());
107
108    SaveLiveRegisters(codegen, invoke_->GetLocations());
109
110    MoveArguments(invoke_, codegen);
111
112    if (invoke_->IsInvokeStaticOrDirect()) {
113      codegen->GenerateStaticOrDirectCall(invoke_->AsInvokeStaticOrDirect(),
114                                          LocationFrom(kArtMethodRegister));
115    } else {
116      codegen->GenerateVirtualCall(invoke_->AsInvokeVirtual(), LocationFrom(kArtMethodRegister));
117    }
118    codegen->RecordPcInfo(invoke_, invoke_->GetDexPc(), this);
119
120    // Copy the result back to the expected output.
121    Location out = invoke_->GetLocations()->Out();
122    if (out.IsValid()) {
123      DCHECK(out.IsRegister());  // TODO: Replace this when we support output in memory.
124      DCHECK(!invoke_->GetLocations()->GetLiveRegisters()->ContainsCoreRegister(out.reg()));
125      MoveFromReturnRegister(out, invoke_->GetType(), codegen);
126    }
127
128    RestoreLiveRegisters(codegen, invoke_->GetLocations());
129    __ B(GetExitLabel());
130  }
131
132  const char* GetDescription() const OVERRIDE { return "IntrinsicSlowPathARM64"; }
133
134 private:
135  // The instruction where this slow path is happening.
136  HInvoke* const invoke_;
137
138  DISALLOW_COPY_AND_ASSIGN(IntrinsicSlowPathARM64);
139};
140
141#undef __
142
143bool IntrinsicLocationsBuilderARM64::TryDispatch(HInvoke* invoke) {
144  Dispatch(invoke);
145  LocationSummary* res = invoke->GetLocations();
146  if (res == nullptr) {
147    return false;
148  }
149  if (kEmitCompilerReadBarrier && res->CanCall()) {
150    // Generating an intrinsic for this HInvoke may produce an
151    // IntrinsicSlowPathARM64 slow path.  Currently this approach
152    // does not work when using read barriers, as the emitted
153    // calling sequence will make use of another slow path
154    // (ReadBarrierForRootSlowPathARM64 for HInvokeStaticOrDirect,
155    // ReadBarrierSlowPathARM64 for HInvokeVirtual).  So we bail
156    // out in this case.
157    //
158    // TODO: Find a way to have intrinsics work with read barriers.
159    invoke->SetLocations(nullptr);
160    return false;
161  }
162  return res->Intrinsified();
163}
164
165#define __ masm->
166
167static void CreateFPToIntLocations(ArenaAllocator* arena, HInvoke* invoke) {
168  LocationSummary* locations = new (arena) LocationSummary(invoke,
169                                                           LocationSummary::kNoCall,
170                                                           kIntrinsified);
171  locations->SetInAt(0, Location::RequiresFpuRegister());
172  locations->SetOut(Location::RequiresRegister());
173}
174
175static void CreateIntToFPLocations(ArenaAllocator* arena, HInvoke* invoke) {
176  LocationSummary* locations = new (arena) LocationSummary(invoke,
177                                                           LocationSummary::kNoCall,
178                                                           kIntrinsified);
179  locations->SetInAt(0, Location::RequiresRegister());
180  locations->SetOut(Location::RequiresFpuRegister());
181}
182
183static void MoveFPToInt(LocationSummary* locations, bool is64bit, vixl::MacroAssembler* masm) {
184  Location input = locations->InAt(0);
185  Location output = locations->Out();
186  __ Fmov(is64bit ? XRegisterFrom(output) : WRegisterFrom(output),
187          is64bit ? DRegisterFrom(input) : SRegisterFrom(input));
188}
189
190static void MoveIntToFP(LocationSummary* locations, bool is64bit, vixl::MacroAssembler* masm) {
191  Location input = locations->InAt(0);
192  Location output = locations->Out();
193  __ Fmov(is64bit ? DRegisterFrom(output) : SRegisterFrom(output),
194          is64bit ? XRegisterFrom(input) : WRegisterFrom(input));
195}
196
197void IntrinsicLocationsBuilderARM64::VisitDoubleDoubleToRawLongBits(HInvoke* invoke) {
198  CreateFPToIntLocations(arena_, invoke);
199}
200void IntrinsicLocationsBuilderARM64::VisitDoubleLongBitsToDouble(HInvoke* invoke) {
201  CreateIntToFPLocations(arena_, invoke);
202}
203
204void IntrinsicCodeGeneratorARM64::VisitDoubleDoubleToRawLongBits(HInvoke* invoke) {
205  MoveFPToInt(invoke->GetLocations(), /* is64bit */ true, GetVIXLAssembler());
206}
207void IntrinsicCodeGeneratorARM64::VisitDoubleLongBitsToDouble(HInvoke* invoke) {
208  MoveIntToFP(invoke->GetLocations(), /* is64bit */ true, GetVIXLAssembler());
209}
210
211void IntrinsicLocationsBuilderARM64::VisitFloatFloatToRawIntBits(HInvoke* invoke) {
212  CreateFPToIntLocations(arena_, invoke);
213}
214void IntrinsicLocationsBuilderARM64::VisitFloatIntBitsToFloat(HInvoke* invoke) {
215  CreateIntToFPLocations(arena_, invoke);
216}
217
218void IntrinsicCodeGeneratorARM64::VisitFloatFloatToRawIntBits(HInvoke* invoke) {
219  MoveFPToInt(invoke->GetLocations(), /* is64bit */ false, GetVIXLAssembler());
220}
221void IntrinsicCodeGeneratorARM64::VisitFloatIntBitsToFloat(HInvoke* invoke) {
222  MoveIntToFP(invoke->GetLocations(), /* is64bit */ false, GetVIXLAssembler());
223}
224
225static void CreateIntToIntLocations(ArenaAllocator* arena, HInvoke* invoke) {
226  LocationSummary* locations = new (arena) LocationSummary(invoke,
227                                                           LocationSummary::kNoCall,
228                                                           kIntrinsified);
229  locations->SetInAt(0, Location::RequiresRegister());
230  locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
231}
232
233static void GenReverseBytes(LocationSummary* locations,
234                            Primitive::Type type,
235                            vixl::MacroAssembler* masm) {
236  Location in = locations->InAt(0);
237  Location out = locations->Out();
238
239  switch (type) {
240    case Primitive::kPrimShort:
241      __ Rev16(WRegisterFrom(out), WRegisterFrom(in));
242      __ Sxth(WRegisterFrom(out), WRegisterFrom(out));
243      break;
244    case Primitive::kPrimInt:
245    case Primitive::kPrimLong:
246      __ Rev(RegisterFrom(out, type), RegisterFrom(in, type));
247      break;
248    default:
249      LOG(FATAL) << "Unexpected size for reverse-bytes: " << type;
250      UNREACHABLE();
251  }
252}
253
254void IntrinsicLocationsBuilderARM64::VisitIntegerReverseBytes(HInvoke* invoke) {
255  CreateIntToIntLocations(arena_, invoke);
256}
257
258void IntrinsicCodeGeneratorARM64::VisitIntegerReverseBytes(HInvoke* invoke) {
259  GenReverseBytes(invoke->GetLocations(), Primitive::kPrimInt, GetVIXLAssembler());
260}
261
262void IntrinsicLocationsBuilderARM64::VisitLongReverseBytes(HInvoke* invoke) {
263  CreateIntToIntLocations(arena_, invoke);
264}
265
266void IntrinsicCodeGeneratorARM64::VisitLongReverseBytes(HInvoke* invoke) {
267  GenReverseBytes(invoke->GetLocations(), Primitive::kPrimLong, GetVIXLAssembler());
268}
269
270void IntrinsicLocationsBuilderARM64::VisitShortReverseBytes(HInvoke* invoke) {
271  CreateIntToIntLocations(arena_, invoke);
272}
273
274void IntrinsicCodeGeneratorARM64::VisitShortReverseBytes(HInvoke* invoke) {
275  GenReverseBytes(invoke->GetLocations(), Primitive::kPrimShort, GetVIXLAssembler());
276}
277
278static void GenNumberOfLeadingZeros(LocationSummary* locations,
279                                    Primitive::Type type,
280                                    vixl::MacroAssembler* masm) {
281  DCHECK(type == Primitive::kPrimInt || type == Primitive::kPrimLong);
282
283  Location in = locations->InAt(0);
284  Location out = locations->Out();
285
286  __ Clz(RegisterFrom(out, type), RegisterFrom(in, type));
287}
288
289void IntrinsicLocationsBuilderARM64::VisitIntegerNumberOfLeadingZeros(HInvoke* invoke) {
290  CreateIntToIntLocations(arena_, invoke);
291}
292
293void IntrinsicCodeGeneratorARM64::VisitIntegerNumberOfLeadingZeros(HInvoke* invoke) {
294  GenNumberOfLeadingZeros(invoke->GetLocations(), Primitive::kPrimInt, GetVIXLAssembler());
295}
296
297void IntrinsicLocationsBuilderARM64::VisitLongNumberOfLeadingZeros(HInvoke* invoke) {
298  CreateIntToIntLocations(arena_, invoke);
299}
300
301void IntrinsicCodeGeneratorARM64::VisitLongNumberOfLeadingZeros(HInvoke* invoke) {
302  GenNumberOfLeadingZeros(invoke->GetLocations(), Primitive::kPrimLong, GetVIXLAssembler());
303}
304
305static void GenNumberOfTrailingZeros(LocationSummary* locations,
306                                     Primitive::Type type,
307                                     vixl::MacroAssembler* masm) {
308  DCHECK(type == Primitive::kPrimInt || type == Primitive::kPrimLong);
309
310  Location in = locations->InAt(0);
311  Location out = locations->Out();
312
313  __ Rbit(RegisterFrom(out, type), RegisterFrom(in, type));
314  __ Clz(RegisterFrom(out, type), RegisterFrom(out, type));
315}
316
317void IntrinsicLocationsBuilderARM64::VisitIntegerNumberOfTrailingZeros(HInvoke* invoke) {
318  CreateIntToIntLocations(arena_, invoke);
319}
320
321void IntrinsicCodeGeneratorARM64::VisitIntegerNumberOfTrailingZeros(HInvoke* invoke) {
322  GenNumberOfTrailingZeros(invoke->GetLocations(), Primitive::kPrimInt, GetVIXLAssembler());
323}
324
325void IntrinsicLocationsBuilderARM64::VisitLongNumberOfTrailingZeros(HInvoke* invoke) {
326  CreateIntToIntLocations(arena_, invoke);
327}
328
329void IntrinsicCodeGeneratorARM64::VisitLongNumberOfTrailingZeros(HInvoke* invoke) {
330  GenNumberOfTrailingZeros(invoke->GetLocations(), Primitive::kPrimLong, GetVIXLAssembler());
331}
332
333static void GenReverse(LocationSummary* locations,
334                       Primitive::Type type,
335                       vixl::MacroAssembler* masm) {
336  DCHECK(type == Primitive::kPrimInt || type == Primitive::kPrimLong);
337
338  Location in = locations->InAt(0);
339  Location out = locations->Out();
340
341  __ Rbit(RegisterFrom(out, type), RegisterFrom(in, type));
342}
343
344void IntrinsicLocationsBuilderARM64::VisitIntegerReverse(HInvoke* invoke) {
345  CreateIntToIntLocations(arena_, invoke);
346}
347
348void IntrinsicCodeGeneratorARM64::VisitIntegerReverse(HInvoke* invoke) {
349  GenReverse(invoke->GetLocations(), Primitive::kPrimInt, GetVIXLAssembler());
350}
351
352void IntrinsicLocationsBuilderARM64::VisitLongReverse(HInvoke* invoke) {
353  CreateIntToIntLocations(arena_, invoke);
354}
355
356void IntrinsicCodeGeneratorARM64::VisitLongReverse(HInvoke* invoke) {
357  GenReverse(invoke->GetLocations(), Primitive::kPrimLong, GetVIXLAssembler());
358}
359
360static void CreateFPToFPLocations(ArenaAllocator* arena, HInvoke* invoke) {
361  LocationSummary* locations = new (arena) LocationSummary(invoke,
362                                                           LocationSummary::kNoCall,
363                                                           kIntrinsified);
364  locations->SetInAt(0, Location::RequiresFpuRegister());
365  locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap);
366}
367
368static void MathAbsFP(LocationSummary* locations, bool is64bit, vixl::MacroAssembler* masm) {
369  Location in = locations->InAt(0);
370  Location out = locations->Out();
371
372  FPRegister in_reg = is64bit ? DRegisterFrom(in) : SRegisterFrom(in);
373  FPRegister out_reg = is64bit ? DRegisterFrom(out) : SRegisterFrom(out);
374
375  __ Fabs(out_reg, in_reg);
376}
377
378void IntrinsicLocationsBuilderARM64::VisitMathAbsDouble(HInvoke* invoke) {
379  CreateFPToFPLocations(arena_, invoke);
380}
381
382void IntrinsicCodeGeneratorARM64::VisitMathAbsDouble(HInvoke* invoke) {
383  MathAbsFP(invoke->GetLocations(), /* is64bit */ true, GetVIXLAssembler());
384}
385
386void IntrinsicLocationsBuilderARM64::VisitMathAbsFloat(HInvoke* invoke) {
387  CreateFPToFPLocations(arena_, invoke);
388}
389
390void IntrinsicCodeGeneratorARM64::VisitMathAbsFloat(HInvoke* invoke) {
391  MathAbsFP(invoke->GetLocations(), /* is64bit */ false, GetVIXLAssembler());
392}
393
394static void CreateIntToInt(ArenaAllocator* arena, HInvoke* invoke) {
395  LocationSummary* locations = new (arena) LocationSummary(invoke,
396                                                           LocationSummary::kNoCall,
397                                                           kIntrinsified);
398  locations->SetInAt(0, Location::RequiresRegister());
399  locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
400}
401
402static void GenAbsInteger(LocationSummary* locations,
403                          bool is64bit,
404                          vixl::MacroAssembler* masm) {
405  Location in = locations->InAt(0);
406  Location output = locations->Out();
407
408  Register in_reg = is64bit ? XRegisterFrom(in) : WRegisterFrom(in);
409  Register out_reg = is64bit ? XRegisterFrom(output) : WRegisterFrom(output);
410
411  __ Cmp(in_reg, Operand(0));
412  __ Cneg(out_reg, in_reg, lt);
413}
414
415void IntrinsicLocationsBuilderARM64::VisitMathAbsInt(HInvoke* invoke) {
416  CreateIntToInt(arena_, invoke);
417}
418
419void IntrinsicCodeGeneratorARM64::VisitMathAbsInt(HInvoke* invoke) {
420  GenAbsInteger(invoke->GetLocations(), /* is64bit */ false, GetVIXLAssembler());
421}
422
423void IntrinsicLocationsBuilderARM64::VisitMathAbsLong(HInvoke* invoke) {
424  CreateIntToInt(arena_, invoke);
425}
426
427void IntrinsicCodeGeneratorARM64::VisitMathAbsLong(HInvoke* invoke) {
428  GenAbsInteger(invoke->GetLocations(), /* is64bit */ true, GetVIXLAssembler());
429}
430
431static void GenMinMaxFP(LocationSummary* locations,
432                        bool is_min,
433                        bool is_double,
434                        vixl::MacroAssembler* masm) {
435  Location op1 = locations->InAt(0);
436  Location op2 = locations->InAt(1);
437  Location out = locations->Out();
438
439  FPRegister op1_reg = is_double ? DRegisterFrom(op1) : SRegisterFrom(op1);
440  FPRegister op2_reg = is_double ? DRegisterFrom(op2) : SRegisterFrom(op2);
441  FPRegister out_reg = is_double ? DRegisterFrom(out) : SRegisterFrom(out);
442  if (is_min) {
443    __ Fmin(out_reg, op1_reg, op2_reg);
444  } else {
445    __ Fmax(out_reg, op1_reg, op2_reg);
446  }
447}
448
449static void CreateFPFPToFPLocations(ArenaAllocator* arena, HInvoke* invoke) {
450  LocationSummary* locations = new (arena) LocationSummary(invoke,
451                                                           LocationSummary::kNoCall,
452                                                           kIntrinsified);
453  locations->SetInAt(0, Location::RequiresFpuRegister());
454  locations->SetInAt(1, Location::RequiresFpuRegister());
455  locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap);
456}
457
458void IntrinsicLocationsBuilderARM64::VisitMathMinDoubleDouble(HInvoke* invoke) {
459  CreateFPFPToFPLocations(arena_, invoke);
460}
461
462void IntrinsicCodeGeneratorARM64::VisitMathMinDoubleDouble(HInvoke* invoke) {
463  GenMinMaxFP(invoke->GetLocations(), /* is_min */ true, /* is_double */ true, GetVIXLAssembler());
464}
465
466void IntrinsicLocationsBuilderARM64::VisitMathMinFloatFloat(HInvoke* invoke) {
467  CreateFPFPToFPLocations(arena_, invoke);
468}
469
470void IntrinsicCodeGeneratorARM64::VisitMathMinFloatFloat(HInvoke* invoke) {
471  GenMinMaxFP(invoke->GetLocations(), /* is_min */ true, /* is_double */ false, GetVIXLAssembler());
472}
473
474void IntrinsicLocationsBuilderARM64::VisitMathMaxDoubleDouble(HInvoke* invoke) {
475  CreateFPFPToFPLocations(arena_, invoke);
476}
477
478void IntrinsicCodeGeneratorARM64::VisitMathMaxDoubleDouble(HInvoke* invoke) {
479  GenMinMaxFP(invoke->GetLocations(), /* is_min */ false, /* is_double */ true, GetVIXLAssembler());
480}
481
482void IntrinsicLocationsBuilderARM64::VisitMathMaxFloatFloat(HInvoke* invoke) {
483  CreateFPFPToFPLocations(arena_, invoke);
484}
485
486void IntrinsicCodeGeneratorARM64::VisitMathMaxFloatFloat(HInvoke* invoke) {
487  GenMinMaxFP(
488      invoke->GetLocations(), /* is_min */ false, /* is_double */ false, GetVIXLAssembler());
489}
490
491static void GenMinMax(LocationSummary* locations,
492                      bool is_min,
493                      bool is_long,
494                      vixl::MacroAssembler* masm) {
495  Location op1 = locations->InAt(0);
496  Location op2 = locations->InAt(1);
497  Location out = locations->Out();
498
499  Register op1_reg = is_long ? XRegisterFrom(op1) : WRegisterFrom(op1);
500  Register op2_reg = is_long ? XRegisterFrom(op2) : WRegisterFrom(op2);
501  Register out_reg = is_long ? XRegisterFrom(out) : WRegisterFrom(out);
502
503  __ Cmp(op1_reg, op2_reg);
504  __ Csel(out_reg, op1_reg, op2_reg, is_min ? lt : gt);
505}
506
507static void CreateIntIntToIntLocations(ArenaAllocator* arena, HInvoke* invoke) {
508  LocationSummary* locations = new (arena) LocationSummary(invoke,
509                                                           LocationSummary::kNoCall,
510                                                           kIntrinsified);
511  locations->SetInAt(0, Location::RequiresRegister());
512  locations->SetInAt(1, Location::RequiresRegister());
513  locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
514}
515
516void IntrinsicLocationsBuilderARM64::VisitMathMinIntInt(HInvoke* invoke) {
517  CreateIntIntToIntLocations(arena_, invoke);
518}
519
520void IntrinsicCodeGeneratorARM64::VisitMathMinIntInt(HInvoke* invoke) {
521  GenMinMax(invoke->GetLocations(), /* is_min */ true, /* is_long */ false, GetVIXLAssembler());
522}
523
524void IntrinsicLocationsBuilderARM64::VisitMathMinLongLong(HInvoke* invoke) {
525  CreateIntIntToIntLocations(arena_, invoke);
526}
527
528void IntrinsicCodeGeneratorARM64::VisitMathMinLongLong(HInvoke* invoke) {
529  GenMinMax(invoke->GetLocations(), /* is_min */ true, /* is_long */ true, GetVIXLAssembler());
530}
531
532void IntrinsicLocationsBuilderARM64::VisitMathMaxIntInt(HInvoke* invoke) {
533  CreateIntIntToIntLocations(arena_, invoke);
534}
535
536void IntrinsicCodeGeneratorARM64::VisitMathMaxIntInt(HInvoke* invoke) {
537  GenMinMax(invoke->GetLocations(), /* is_min */ false, /* is_long */ false, GetVIXLAssembler());
538}
539
540void IntrinsicLocationsBuilderARM64::VisitMathMaxLongLong(HInvoke* invoke) {
541  CreateIntIntToIntLocations(arena_, invoke);
542}
543
544void IntrinsicCodeGeneratorARM64::VisitMathMaxLongLong(HInvoke* invoke) {
545  GenMinMax(invoke->GetLocations(), /* is_min */ false, /* is_long */ true, GetVIXLAssembler());
546}
547
548void IntrinsicLocationsBuilderARM64::VisitMathSqrt(HInvoke* invoke) {
549  CreateFPToFPLocations(arena_, invoke);
550}
551
552void IntrinsicCodeGeneratorARM64::VisitMathSqrt(HInvoke* invoke) {
553  LocationSummary* locations = invoke->GetLocations();
554  vixl::MacroAssembler* masm = GetVIXLAssembler();
555  __ Fsqrt(DRegisterFrom(locations->Out()), DRegisterFrom(locations->InAt(0)));
556}
557
558void IntrinsicLocationsBuilderARM64::VisitMathCeil(HInvoke* invoke) {
559  CreateFPToFPLocations(arena_, invoke);
560}
561
562void IntrinsicCodeGeneratorARM64::VisitMathCeil(HInvoke* invoke) {
563  LocationSummary* locations = invoke->GetLocations();
564  vixl::MacroAssembler* masm = GetVIXLAssembler();
565  __ Frintp(DRegisterFrom(locations->Out()), DRegisterFrom(locations->InAt(0)));
566}
567
568void IntrinsicLocationsBuilderARM64::VisitMathFloor(HInvoke* invoke) {
569  CreateFPToFPLocations(arena_, invoke);
570}
571
572void IntrinsicCodeGeneratorARM64::VisitMathFloor(HInvoke* invoke) {
573  LocationSummary* locations = invoke->GetLocations();
574  vixl::MacroAssembler* masm = GetVIXLAssembler();
575  __ Frintm(DRegisterFrom(locations->Out()), DRegisterFrom(locations->InAt(0)));
576}
577
578void IntrinsicLocationsBuilderARM64::VisitMathRint(HInvoke* invoke) {
579  CreateFPToFPLocations(arena_, invoke);
580}
581
582void IntrinsicCodeGeneratorARM64::VisitMathRint(HInvoke* invoke) {
583  LocationSummary* locations = invoke->GetLocations();
584  vixl::MacroAssembler* masm = GetVIXLAssembler();
585  __ Frintn(DRegisterFrom(locations->Out()), DRegisterFrom(locations->InAt(0)));
586}
587
588static void CreateFPToIntPlusTempLocations(ArenaAllocator* arena, HInvoke* invoke) {
589  LocationSummary* locations = new (arena) LocationSummary(invoke,
590                                                           LocationSummary::kNoCall,
591                                                           kIntrinsified);
592  locations->SetInAt(0, Location::RequiresFpuRegister());
593  locations->SetOut(Location::RequiresRegister());
594}
595
596static void GenMathRound(LocationSummary* locations,
597                         bool is_double,
598                         vixl::MacroAssembler* masm) {
599  FPRegister in_reg = is_double ?
600      DRegisterFrom(locations->InAt(0)) : SRegisterFrom(locations->InAt(0));
601  Register out_reg = is_double ?
602      XRegisterFrom(locations->Out()) : WRegisterFrom(locations->Out());
603  UseScratchRegisterScope temps(masm);
604  FPRegister temp1_reg = temps.AcquireSameSizeAs(in_reg);
605
606  // 0.5 can be encoded as an immediate, so use fmov.
607  if (is_double) {
608    __ Fmov(temp1_reg, static_cast<double>(0.5));
609  } else {
610    __ Fmov(temp1_reg, static_cast<float>(0.5));
611  }
612  __ Fadd(temp1_reg, in_reg, temp1_reg);
613  __ Fcvtms(out_reg, temp1_reg);
614}
615
616void IntrinsicLocationsBuilderARM64::VisitMathRoundDouble(HInvoke* invoke) {
617  // See intrinsics.h.
618  if (kRoundIsPlusPointFive) {
619    CreateFPToIntPlusTempLocations(arena_, invoke);
620  }
621}
622
623void IntrinsicCodeGeneratorARM64::VisitMathRoundDouble(HInvoke* invoke) {
624  GenMathRound(invoke->GetLocations(), /* is_double */ true, GetVIXLAssembler());
625}
626
627void IntrinsicLocationsBuilderARM64::VisitMathRoundFloat(HInvoke* invoke) {
628  // See intrinsics.h.
629  if (kRoundIsPlusPointFive) {
630    CreateFPToIntPlusTempLocations(arena_, invoke);
631  }
632}
633
634void IntrinsicCodeGeneratorARM64::VisitMathRoundFloat(HInvoke* invoke) {
635  GenMathRound(invoke->GetLocations(), /* is_double */ false, GetVIXLAssembler());
636}
637
638void IntrinsicLocationsBuilderARM64::VisitMemoryPeekByte(HInvoke* invoke) {
639  CreateIntToIntLocations(arena_, invoke);
640}
641
642void IntrinsicCodeGeneratorARM64::VisitMemoryPeekByte(HInvoke* invoke) {
643  vixl::MacroAssembler* masm = GetVIXLAssembler();
644  __ Ldrsb(WRegisterFrom(invoke->GetLocations()->Out()),
645          AbsoluteHeapOperandFrom(invoke->GetLocations()->InAt(0), 0));
646}
647
648void IntrinsicLocationsBuilderARM64::VisitMemoryPeekIntNative(HInvoke* invoke) {
649  CreateIntToIntLocations(arena_, invoke);
650}
651
652void IntrinsicCodeGeneratorARM64::VisitMemoryPeekIntNative(HInvoke* invoke) {
653  vixl::MacroAssembler* masm = GetVIXLAssembler();
654  __ Ldr(WRegisterFrom(invoke->GetLocations()->Out()),
655         AbsoluteHeapOperandFrom(invoke->GetLocations()->InAt(0), 0));
656}
657
658void IntrinsicLocationsBuilderARM64::VisitMemoryPeekLongNative(HInvoke* invoke) {
659  CreateIntToIntLocations(arena_, invoke);
660}
661
662void IntrinsicCodeGeneratorARM64::VisitMemoryPeekLongNative(HInvoke* invoke) {
663  vixl::MacroAssembler* masm = GetVIXLAssembler();
664  __ Ldr(XRegisterFrom(invoke->GetLocations()->Out()),
665         AbsoluteHeapOperandFrom(invoke->GetLocations()->InAt(0), 0));
666}
667
668void IntrinsicLocationsBuilderARM64::VisitMemoryPeekShortNative(HInvoke* invoke) {
669  CreateIntToIntLocations(arena_, invoke);
670}
671
672void IntrinsicCodeGeneratorARM64::VisitMemoryPeekShortNative(HInvoke* invoke) {
673  vixl::MacroAssembler* masm = GetVIXLAssembler();
674  __ Ldrsh(WRegisterFrom(invoke->GetLocations()->Out()),
675           AbsoluteHeapOperandFrom(invoke->GetLocations()->InAt(0), 0));
676}
677
678static void CreateIntIntToVoidLocations(ArenaAllocator* arena, HInvoke* invoke) {
679  LocationSummary* locations = new (arena) LocationSummary(invoke,
680                                                           LocationSummary::kNoCall,
681                                                           kIntrinsified);
682  locations->SetInAt(0, Location::RequiresRegister());
683  locations->SetInAt(1, Location::RequiresRegister());
684}
685
686void IntrinsicLocationsBuilderARM64::VisitMemoryPokeByte(HInvoke* invoke) {
687  CreateIntIntToVoidLocations(arena_, invoke);
688}
689
690void IntrinsicCodeGeneratorARM64::VisitMemoryPokeByte(HInvoke* invoke) {
691  vixl::MacroAssembler* masm = GetVIXLAssembler();
692  __ Strb(WRegisterFrom(invoke->GetLocations()->InAt(1)),
693          AbsoluteHeapOperandFrom(invoke->GetLocations()->InAt(0), 0));
694}
695
696void IntrinsicLocationsBuilderARM64::VisitMemoryPokeIntNative(HInvoke* invoke) {
697  CreateIntIntToVoidLocations(arena_, invoke);
698}
699
700void IntrinsicCodeGeneratorARM64::VisitMemoryPokeIntNative(HInvoke* invoke) {
701  vixl::MacroAssembler* masm = GetVIXLAssembler();
702  __ Str(WRegisterFrom(invoke->GetLocations()->InAt(1)),
703         AbsoluteHeapOperandFrom(invoke->GetLocations()->InAt(0), 0));
704}
705
706void IntrinsicLocationsBuilderARM64::VisitMemoryPokeLongNative(HInvoke* invoke) {
707  CreateIntIntToVoidLocations(arena_, invoke);
708}
709
710void IntrinsicCodeGeneratorARM64::VisitMemoryPokeLongNative(HInvoke* invoke) {
711  vixl::MacroAssembler* masm = GetVIXLAssembler();
712  __ Str(XRegisterFrom(invoke->GetLocations()->InAt(1)),
713         AbsoluteHeapOperandFrom(invoke->GetLocations()->InAt(0), 0));
714}
715
716void IntrinsicLocationsBuilderARM64::VisitMemoryPokeShortNative(HInvoke* invoke) {
717  CreateIntIntToVoidLocations(arena_, invoke);
718}
719
720void IntrinsicCodeGeneratorARM64::VisitMemoryPokeShortNative(HInvoke* invoke) {
721  vixl::MacroAssembler* masm = GetVIXLAssembler();
722  __ Strh(WRegisterFrom(invoke->GetLocations()->InAt(1)),
723          AbsoluteHeapOperandFrom(invoke->GetLocations()->InAt(0), 0));
724}
725
726void IntrinsicLocationsBuilderARM64::VisitThreadCurrentThread(HInvoke* invoke) {
727  LocationSummary* locations = new (arena_) LocationSummary(invoke,
728                                                            LocationSummary::kNoCall,
729                                                            kIntrinsified);
730  locations->SetOut(Location::RequiresRegister());
731}
732
733void IntrinsicCodeGeneratorARM64::VisitThreadCurrentThread(HInvoke* invoke) {
734  codegen_->Load(Primitive::kPrimNot, WRegisterFrom(invoke->GetLocations()->Out()),
735                 MemOperand(tr, Thread::PeerOffset<8>().Int32Value()));
736}
737
738static void GenUnsafeGet(HInvoke* invoke,
739                         Primitive::Type type,
740                         bool is_volatile,
741                         CodeGeneratorARM64* codegen) {
742  LocationSummary* locations = invoke->GetLocations();
743  DCHECK((type == Primitive::kPrimInt) ||
744         (type == Primitive::kPrimLong) ||
745         (type == Primitive::kPrimNot));
746  vixl::MacroAssembler* masm = codegen->GetAssembler()->vixl_masm_;
747  Location base_loc = locations->InAt(1);
748  Register base = WRegisterFrom(base_loc);      // Object pointer.
749  Location offset_loc = locations->InAt(2);
750  Register offset = XRegisterFrom(offset_loc);  // Long offset.
751  Location trg_loc = locations->Out();
752  Register trg = RegisterFrom(trg_loc, type);
753  bool use_acquire_release = codegen->GetInstructionSetFeatures().PreferAcquireRelease();
754
755  if (type == Primitive::kPrimNot && kEmitCompilerReadBarrier && kUseBakerReadBarrier) {
756    // UnsafeGetObject/UnsafeGetObjectVolatile with Baker's read barrier case.
757    UseScratchRegisterScope temps(masm);
758    Register temp = temps.AcquireW();
759    codegen->GenerateArrayLoadWithBakerReadBarrier(
760        invoke, trg_loc, base, 0U, offset_loc, temp, /* needs_null_check */ false);
761    if (is_volatile && !use_acquire_release) {
762      __ Dmb(InnerShareable, BarrierReads);
763    }
764  } else {
765    // Other cases.
766    MemOperand mem_op(base.X(), offset);
767    if (is_volatile) {
768      if (use_acquire_release) {
769        codegen->LoadAcquire(invoke, trg, mem_op, /* needs_null_check */ true);
770      } else {
771        codegen->Load(type, trg, mem_op);
772        __ Dmb(InnerShareable, BarrierReads);
773      }
774    } else {
775      codegen->Load(type, trg, mem_op);
776    }
777
778    if (type == Primitive::kPrimNot) {
779      DCHECK(trg.IsW());
780      codegen->MaybeGenerateReadBarrierSlow(invoke, trg_loc, trg_loc, base_loc, 0U, offset_loc);
781    }
782  }
783}
784
785static void CreateIntIntIntToIntLocations(ArenaAllocator* arena, HInvoke* invoke) {
786  bool can_call = kEmitCompilerReadBarrier &&
787      (invoke->GetIntrinsic() == Intrinsics::kUnsafeGetObject ||
788       invoke->GetIntrinsic() == Intrinsics::kUnsafeGetObjectVolatile);
789  LocationSummary* locations = new (arena) LocationSummary(invoke,
790                                                           can_call ?
791                                                               LocationSummary::kCallOnSlowPath :
792                                                               LocationSummary::kNoCall,
793                                                           kIntrinsified);
794  locations->SetInAt(0, Location::NoLocation());        // Unused receiver.
795  locations->SetInAt(1, Location::RequiresRegister());
796  locations->SetInAt(2, Location::RequiresRegister());
797  locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
798}
799
800void IntrinsicLocationsBuilderARM64::VisitUnsafeGet(HInvoke* invoke) {
801  CreateIntIntIntToIntLocations(arena_, invoke);
802}
803void IntrinsicLocationsBuilderARM64::VisitUnsafeGetVolatile(HInvoke* invoke) {
804  CreateIntIntIntToIntLocations(arena_, invoke);
805}
806void IntrinsicLocationsBuilderARM64::VisitUnsafeGetLong(HInvoke* invoke) {
807  CreateIntIntIntToIntLocations(arena_, invoke);
808}
809void IntrinsicLocationsBuilderARM64::VisitUnsafeGetLongVolatile(HInvoke* invoke) {
810  CreateIntIntIntToIntLocations(arena_, invoke);
811}
812void IntrinsicLocationsBuilderARM64::VisitUnsafeGetObject(HInvoke* invoke) {
813  CreateIntIntIntToIntLocations(arena_, invoke);
814}
815void IntrinsicLocationsBuilderARM64::VisitUnsafeGetObjectVolatile(HInvoke* invoke) {
816  CreateIntIntIntToIntLocations(arena_, invoke);
817}
818
819void IntrinsicCodeGeneratorARM64::VisitUnsafeGet(HInvoke* invoke) {
820  GenUnsafeGet(invoke, Primitive::kPrimInt, /* is_volatile */ false, codegen_);
821}
822void IntrinsicCodeGeneratorARM64::VisitUnsafeGetVolatile(HInvoke* invoke) {
823  GenUnsafeGet(invoke, Primitive::kPrimInt, /* is_volatile */ true, codegen_);
824}
825void IntrinsicCodeGeneratorARM64::VisitUnsafeGetLong(HInvoke* invoke) {
826  GenUnsafeGet(invoke, Primitive::kPrimLong, /* is_volatile */ false, codegen_);
827}
828void IntrinsicCodeGeneratorARM64::VisitUnsafeGetLongVolatile(HInvoke* invoke) {
829  GenUnsafeGet(invoke, Primitive::kPrimLong, /* is_volatile */ true, codegen_);
830}
831void IntrinsicCodeGeneratorARM64::VisitUnsafeGetObject(HInvoke* invoke) {
832  GenUnsafeGet(invoke, Primitive::kPrimNot, /* is_volatile */ false, codegen_);
833}
834void IntrinsicCodeGeneratorARM64::VisitUnsafeGetObjectVolatile(HInvoke* invoke) {
835  GenUnsafeGet(invoke, Primitive::kPrimNot, /* is_volatile */ true, codegen_);
836}
837
838static void CreateIntIntIntIntToVoid(ArenaAllocator* arena, HInvoke* invoke) {
839  LocationSummary* locations = new (arena) LocationSummary(invoke,
840                                                           LocationSummary::kNoCall,
841                                                           kIntrinsified);
842  locations->SetInAt(0, Location::NoLocation());        // Unused receiver.
843  locations->SetInAt(1, Location::RequiresRegister());
844  locations->SetInAt(2, Location::RequiresRegister());
845  locations->SetInAt(3, Location::RequiresRegister());
846}
847
848void IntrinsicLocationsBuilderARM64::VisitUnsafePut(HInvoke* invoke) {
849  CreateIntIntIntIntToVoid(arena_, invoke);
850}
851void IntrinsicLocationsBuilderARM64::VisitUnsafePutOrdered(HInvoke* invoke) {
852  CreateIntIntIntIntToVoid(arena_, invoke);
853}
854void IntrinsicLocationsBuilderARM64::VisitUnsafePutVolatile(HInvoke* invoke) {
855  CreateIntIntIntIntToVoid(arena_, invoke);
856}
857void IntrinsicLocationsBuilderARM64::VisitUnsafePutObject(HInvoke* invoke) {
858  CreateIntIntIntIntToVoid(arena_, invoke);
859}
860void IntrinsicLocationsBuilderARM64::VisitUnsafePutObjectOrdered(HInvoke* invoke) {
861  CreateIntIntIntIntToVoid(arena_, invoke);
862}
863void IntrinsicLocationsBuilderARM64::VisitUnsafePutObjectVolatile(HInvoke* invoke) {
864  CreateIntIntIntIntToVoid(arena_, invoke);
865}
866void IntrinsicLocationsBuilderARM64::VisitUnsafePutLong(HInvoke* invoke) {
867  CreateIntIntIntIntToVoid(arena_, invoke);
868}
869void IntrinsicLocationsBuilderARM64::VisitUnsafePutLongOrdered(HInvoke* invoke) {
870  CreateIntIntIntIntToVoid(arena_, invoke);
871}
872void IntrinsicLocationsBuilderARM64::VisitUnsafePutLongVolatile(HInvoke* invoke) {
873  CreateIntIntIntIntToVoid(arena_, invoke);
874}
875
876static void GenUnsafePut(LocationSummary* locations,
877                         Primitive::Type type,
878                         bool is_volatile,
879                         bool is_ordered,
880                         CodeGeneratorARM64* codegen) {
881  vixl::MacroAssembler* masm = codegen->GetAssembler()->vixl_masm_;
882
883  Register base = WRegisterFrom(locations->InAt(1));    // Object pointer.
884  Register offset = XRegisterFrom(locations->InAt(2));  // Long offset.
885  Register value = RegisterFrom(locations->InAt(3), type);
886  Register source = value;
887  bool use_acquire_release = codegen->GetInstructionSetFeatures().PreferAcquireRelease();
888
889  MemOperand mem_op(base.X(), offset);
890
891  {
892    // We use a block to end the scratch scope before the write barrier, thus
893    // freeing the temporary registers so they can be used in `MarkGCCard`.
894    UseScratchRegisterScope temps(masm);
895
896    if (kPoisonHeapReferences && type == Primitive::kPrimNot) {
897      DCHECK(value.IsW());
898      Register temp = temps.AcquireW();
899      __ Mov(temp.W(), value.W());
900      codegen->GetAssembler()->PoisonHeapReference(temp.W());
901      source = temp;
902    }
903
904    if (is_volatile || is_ordered) {
905      if (use_acquire_release) {
906        codegen->StoreRelease(type, source, mem_op);
907      } else {
908        __ Dmb(InnerShareable, BarrierAll);
909        codegen->Store(type, source, mem_op);
910        if (is_volatile) {
911          __ Dmb(InnerShareable, BarrierReads);
912        }
913      }
914    } else {
915      codegen->Store(type, source, mem_op);
916    }
917  }
918
919  if (type == Primitive::kPrimNot) {
920    bool value_can_be_null = true;  // TODO: Worth finding out this information?
921    codegen->MarkGCCard(base, value, value_can_be_null);
922  }
923}
924
925void IntrinsicCodeGeneratorARM64::VisitUnsafePut(HInvoke* invoke) {
926  GenUnsafePut(invoke->GetLocations(),
927               Primitive::kPrimInt,
928               /* is_volatile */ false,
929               /* is_ordered */ false,
930               codegen_);
931}
932void IntrinsicCodeGeneratorARM64::VisitUnsafePutOrdered(HInvoke* invoke) {
933  GenUnsafePut(invoke->GetLocations(),
934               Primitive::kPrimInt,
935               /* is_volatile */ false,
936               /* is_ordered */ true,
937               codegen_);
938}
939void IntrinsicCodeGeneratorARM64::VisitUnsafePutVolatile(HInvoke* invoke) {
940  GenUnsafePut(invoke->GetLocations(),
941               Primitive::kPrimInt,
942               /* is_volatile */ true,
943               /* is_ordered */ false,
944               codegen_);
945}
946void IntrinsicCodeGeneratorARM64::VisitUnsafePutObject(HInvoke* invoke) {
947  GenUnsafePut(invoke->GetLocations(),
948               Primitive::kPrimNot,
949               /* is_volatile */ false,
950               /* is_ordered */ false,
951               codegen_);
952}
953void IntrinsicCodeGeneratorARM64::VisitUnsafePutObjectOrdered(HInvoke* invoke) {
954  GenUnsafePut(invoke->GetLocations(),
955               Primitive::kPrimNot,
956               /* is_volatile */ false,
957               /* is_ordered */ true,
958               codegen_);
959}
960void IntrinsicCodeGeneratorARM64::VisitUnsafePutObjectVolatile(HInvoke* invoke) {
961  GenUnsafePut(invoke->GetLocations(),
962               Primitive::kPrimNot,
963               /* is_volatile */ true,
964               /* is_ordered */ false,
965               codegen_);
966}
967void IntrinsicCodeGeneratorARM64::VisitUnsafePutLong(HInvoke* invoke) {
968  GenUnsafePut(invoke->GetLocations(),
969               Primitive::kPrimLong,
970               /* is_volatile */ false,
971               /* is_ordered */ false,
972               codegen_);
973}
974void IntrinsicCodeGeneratorARM64::VisitUnsafePutLongOrdered(HInvoke* invoke) {
975  GenUnsafePut(invoke->GetLocations(),
976               Primitive::kPrimLong,
977               /* is_volatile */ false,
978               /* is_ordered */ true,
979               codegen_);
980}
981void IntrinsicCodeGeneratorARM64::VisitUnsafePutLongVolatile(HInvoke* invoke) {
982  GenUnsafePut(invoke->GetLocations(),
983               Primitive::kPrimLong,
984               /* is_volatile */ true,
985               /* is_ordered */ false,
986               codegen_);
987}
988
989static void CreateIntIntIntIntIntToInt(ArenaAllocator* arena,
990                                       HInvoke* invoke,
991                                       Primitive::Type type) {
992  LocationSummary* locations = new (arena) LocationSummary(invoke,
993                                                           LocationSummary::kNoCall,
994                                                           kIntrinsified);
995  locations->SetInAt(0, Location::NoLocation());        // Unused receiver.
996  locations->SetInAt(1, Location::RequiresRegister());
997  locations->SetInAt(2, Location::RequiresRegister());
998  locations->SetInAt(3, Location::RequiresRegister());
999  locations->SetInAt(4, Location::RequiresRegister());
1000
1001  // If heap poisoning is enabled, we don't want the unpoisoning
1002  // operations to potentially clobber the output.
1003  Location::OutputOverlap overlaps = (kPoisonHeapReferences && type == Primitive::kPrimNot)
1004      ? Location::kOutputOverlap
1005      : Location::kNoOutputOverlap;
1006  locations->SetOut(Location::RequiresRegister(), overlaps);
1007}
1008
1009static void GenCas(LocationSummary* locations, Primitive::Type type, CodeGeneratorARM64* codegen) {
1010  bool use_acquire_release = codegen->GetInstructionSetFeatures().PreferAcquireRelease();
1011  vixl::MacroAssembler* masm = codegen->GetAssembler()->vixl_masm_;
1012
1013  Register out = WRegisterFrom(locations->Out());                  // Boolean result.
1014
1015  Register base = WRegisterFrom(locations->InAt(1));               // Object pointer.
1016  Register offset = XRegisterFrom(locations->InAt(2));             // Long offset.
1017  Register expected = RegisterFrom(locations->InAt(3), type);      // Expected.
1018  Register value = RegisterFrom(locations->InAt(4), type);         // Value.
1019
1020  // This needs to be before the temp registers, as MarkGCCard also uses VIXL temps.
1021  if (type == Primitive::kPrimNot) {
1022    // Mark card for object assuming new value is stored.
1023    bool value_can_be_null = true;  // TODO: Worth finding out this information?
1024    codegen->MarkGCCard(base, value, value_can_be_null);
1025  }
1026
1027  UseScratchRegisterScope temps(masm);
1028  Register tmp_ptr = temps.AcquireX();                             // Pointer to actual memory.
1029  Register tmp_value = temps.AcquireSameSizeAs(value);             // Value in memory.
1030
1031  Register tmp_32 = tmp_value.W();
1032
1033  __ Add(tmp_ptr, base.X(), Operand(offset));
1034
1035  if (kPoisonHeapReferences && type == Primitive::kPrimNot) {
1036    codegen->GetAssembler()->PoisonHeapReference(expected);
1037    if (value.Is(expected)) {
1038      // Do not poison `value`, as it is the same register as
1039      // `expected`, which has just been poisoned.
1040    } else {
1041      codegen->GetAssembler()->PoisonHeapReference(value);
1042    }
1043  }
1044
1045  // do {
1046  //   tmp_value = [tmp_ptr] - expected;
1047  // } while (tmp_value == 0 && failure([tmp_ptr] <- r_new_value));
1048  // result = tmp_value != 0;
1049
1050  vixl::Label loop_head, exit_loop;
1051  if (use_acquire_release) {
1052    __ Bind(&loop_head);
1053    // TODO: When `type == Primitive::kPrimNot`, add a read barrier for
1054    // the reference stored in the object before attempting the CAS,
1055    // similar to the one in the art::Unsafe_compareAndSwapObject JNI
1056    // implementation.
1057    //
1058    // Note that this code is not (yet) used when read barriers are
1059    // enabled (see IntrinsicLocationsBuilderARM64::VisitUnsafeCASObject).
1060    DCHECK(!(type == Primitive::kPrimNot && kEmitCompilerReadBarrier));
1061    __ Ldaxr(tmp_value, MemOperand(tmp_ptr));
1062    __ Cmp(tmp_value, expected);
1063    __ B(&exit_loop, ne);
1064    __ Stlxr(tmp_32, value, MemOperand(tmp_ptr));
1065    __ Cbnz(tmp_32, &loop_head);
1066  } else {
1067    // Emit a `Dmb(InnerShareable, BarrierAll)` (DMB ISH) instruction
1068    // instead of a `Dmb(InnerShareable, BarrierWrites)` (DMB ISHST)
1069    // one, as the latter allows a preceding load to be delayed past
1070    // the STXR instruction below.
1071    __ Dmb(InnerShareable, BarrierAll);
1072    __ Bind(&loop_head);
1073    // TODO: When `type == Primitive::kPrimNot`, add a read barrier for
1074    // the reference stored in the object before attempting the CAS,
1075    // similar to the one in the art::Unsafe_compareAndSwapObject JNI
1076    // implementation.
1077    //
1078    // Note that this code is not (yet) used when read barriers are
1079    // enabled (see IntrinsicLocationsBuilderARM64::VisitUnsafeCASObject).
1080    DCHECK(!(type == Primitive::kPrimNot && kEmitCompilerReadBarrier));
1081    __ Ldxr(tmp_value, MemOperand(tmp_ptr));
1082    __ Cmp(tmp_value, expected);
1083    __ B(&exit_loop, ne);
1084    __ Stxr(tmp_32, value, MemOperand(tmp_ptr));
1085    __ Cbnz(tmp_32, &loop_head);
1086    __ Dmb(InnerShareable, BarrierAll);
1087  }
1088  __ Bind(&exit_loop);
1089  __ Cset(out, eq);
1090
1091  if (kPoisonHeapReferences && type == Primitive::kPrimNot) {
1092    codegen->GetAssembler()->UnpoisonHeapReference(expected);
1093    if (value.Is(expected)) {
1094      // Do not unpoison `value`, as it is the same register as
1095      // `expected`, which has just been unpoisoned.
1096    } else {
1097      codegen->GetAssembler()->UnpoisonHeapReference(value);
1098    }
1099  }
1100}
1101
1102void IntrinsicLocationsBuilderARM64::VisitUnsafeCASInt(HInvoke* invoke) {
1103  CreateIntIntIntIntIntToInt(arena_, invoke, Primitive::kPrimInt);
1104}
1105void IntrinsicLocationsBuilderARM64::VisitUnsafeCASLong(HInvoke* invoke) {
1106  CreateIntIntIntIntIntToInt(arena_, invoke, Primitive::kPrimLong);
1107}
1108void IntrinsicLocationsBuilderARM64::VisitUnsafeCASObject(HInvoke* invoke) {
1109  // The UnsafeCASObject intrinsic is missing a read barrier, and
1110  // therefore sometimes does not work as expected (b/25883050).
1111  // Turn it off temporarily as a quick fix, until the read barrier is
1112  // implemented (see TODO in GenCAS below).
1113  //
1114  // TODO(rpl): Fix this issue and re-enable this intrinsic with read barriers.
1115  if (kEmitCompilerReadBarrier) {
1116    return;
1117  }
1118
1119  CreateIntIntIntIntIntToInt(arena_, invoke, Primitive::kPrimNot);
1120}
1121
1122void IntrinsicCodeGeneratorARM64::VisitUnsafeCASInt(HInvoke* invoke) {
1123  GenCas(invoke->GetLocations(), Primitive::kPrimInt, codegen_);
1124}
1125void IntrinsicCodeGeneratorARM64::VisitUnsafeCASLong(HInvoke* invoke) {
1126  GenCas(invoke->GetLocations(), Primitive::kPrimLong, codegen_);
1127}
1128void IntrinsicCodeGeneratorARM64::VisitUnsafeCASObject(HInvoke* invoke) {
1129  GenCas(invoke->GetLocations(), Primitive::kPrimNot, codegen_);
1130}
1131
1132void IntrinsicLocationsBuilderARM64::VisitStringCharAt(HInvoke* invoke) {
1133  LocationSummary* locations = new (arena_) LocationSummary(invoke,
1134                                                            LocationSummary::kCallOnSlowPath,
1135                                                            kIntrinsified);
1136  locations->SetInAt(0, Location::RequiresRegister());
1137  locations->SetInAt(1, Location::RequiresRegister());
1138  // In case we need to go in the slow path, we can't have the output be the same
1139  // as the input: the current liveness analysis considers the input to be live
1140  // at the point of the call.
1141  locations->SetOut(Location::RequiresRegister(), Location::kOutputOverlap);
1142}
1143
1144void IntrinsicCodeGeneratorARM64::VisitStringCharAt(HInvoke* invoke) {
1145  vixl::MacroAssembler* masm = GetVIXLAssembler();
1146  LocationSummary* locations = invoke->GetLocations();
1147
1148  // Location of reference to data array
1149  const MemberOffset value_offset = mirror::String::ValueOffset();
1150  // Location of count
1151  const MemberOffset count_offset = mirror::String::CountOffset();
1152
1153  Register obj = WRegisterFrom(locations->InAt(0));  // String object pointer.
1154  Register idx = WRegisterFrom(locations->InAt(1));  // Index of character.
1155  Register out = WRegisterFrom(locations->Out());    // Result character.
1156
1157  UseScratchRegisterScope temps(masm);
1158  Register temp = temps.AcquireW();
1159  Register array_temp = temps.AcquireW();            // We can trade this for worse scheduling.
1160
1161  // TODO: Maybe we can support range check elimination. Overall, though, I think it's not worth
1162  //       the cost.
1163  // TODO: For simplicity, the index parameter is requested in a register, so different from Quick
1164  //       we will not optimize the code for constants (which would save a register).
1165
1166  SlowPathCodeARM64* slow_path = new (GetAllocator()) IntrinsicSlowPathARM64(invoke);
1167  codegen_->AddSlowPath(slow_path);
1168
1169  __ Ldr(temp, HeapOperand(obj, count_offset));          // temp = str.length.
1170  codegen_->MaybeRecordImplicitNullCheck(invoke);
1171  __ Cmp(idx, temp);
1172  __ B(hs, slow_path->GetEntryLabel());
1173
1174  __ Add(array_temp, obj, Operand(value_offset.Int32Value()));  // array_temp := str.value.
1175
1176  // Load the value.
1177  __ Ldrh(out, MemOperand(array_temp.X(), idx, UXTW, 1));  // out := array_temp[idx].
1178
1179  __ Bind(slow_path->GetExitLabel());
1180}
1181
1182void IntrinsicLocationsBuilderARM64::VisitStringCompareTo(HInvoke* invoke) {
1183  LocationSummary* locations = new (arena_) LocationSummary(invoke,
1184                                                            LocationSummary::kCall,
1185                                                            kIntrinsified);
1186  InvokeRuntimeCallingConvention calling_convention;
1187  locations->SetInAt(0, LocationFrom(calling_convention.GetRegisterAt(0)));
1188  locations->SetInAt(1, LocationFrom(calling_convention.GetRegisterAt(1)));
1189  locations->SetOut(calling_convention.GetReturnLocation(Primitive::kPrimInt));
1190}
1191
1192void IntrinsicCodeGeneratorARM64::VisitStringCompareTo(HInvoke* invoke) {
1193  vixl::MacroAssembler* masm = GetVIXLAssembler();
1194  LocationSummary* locations = invoke->GetLocations();
1195
1196  // Note that the null check must have been done earlier.
1197  DCHECK(!invoke->CanDoImplicitNullCheckOn(invoke->InputAt(0)));
1198
1199  Register argument = WRegisterFrom(locations->InAt(1));
1200  __ Cmp(argument, 0);
1201  SlowPathCodeARM64* slow_path = new (GetAllocator()) IntrinsicSlowPathARM64(invoke);
1202  codegen_->AddSlowPath(slow_path);
1203  __ B(eq, slow_path->GetEntryLabel());
1204
1205  __ Ldr(
1206      lr, MemOperand(tr, QUICK_ENTRYPOINT_OFFSET(kArm64WordSize, pStringCompareTo).Int32Value()));
1207  __ Blr(lr);
1208  __ Bind(slow_path->GetExitLabel());
1209}
1210
1211void IntrinsicLocationsBuilderARM64::VisitStringEquals(HInvoke* invoke) {
1212  LocationSummary* locations = new (arena_) LocationSummary(invoke,
1213                                                            LocationSummary::kNoCall,
1214                                                            kIntrinsified);
1215  locations->SetInAt(0, Location::RequiresRegister());
1216  locations->SetInAt(1, Location::RequiresRegister());
1217  // Temporary registers to store lengths of strings and for calculations.
1218  locations->AddTemp(Location::RequiresRegister());
1219  locations->AddTemp(Location::RequiresRegister());
1220
1221  locations->SetOut(Location::RequiresRegister(), Location::kOutputOverlap);
1222}
1223
1224void IntrinsicCodeGeneratorARM64::VisitStringEquals(HInvoke* invoke) {
1225  vixl::MacroAssembler* masm = GetVIXLAssembler();
1226  LocationSummary* locations = invoke->GetLocations();
1227
1228  Register str = WRegisterFrom(locations->InAt(0));
1229  Register arg = WRegisterFrom(locations->InAt(1));
1230  Register out = XRegisterFrom(locations->Out());
1231
1232  UseScratchRegisterScope scratch_scope(masm);
1233  Register temp = scratch_scope.AcquireW();
1234  Register temp1 = WRegisterFrom(locations->GetTemp(0));
1235  Register temp2 = WRegisterFrom(locations->GetTemp(1));
1236
1237  vixl::Label loop;
1238  vixl::Label end;
1239  vixl::Label return_true;
1240  vixl::Label return_false;
1241
1242  // Get offsets of count, value, and class fields within a string object.
1243  const int32_t count_offset = mirror::String::CountOffset().Int32Value();
1244  const int32_t value_offset = mirror::String::ValueOffset().Int32Value();
1245  const int32_t class_offset = mirror::Object::ClassOffset().Int32Value();
1246
1247  // Note that the null check must have been done earlier.
1248  DCHECK(!invoke->CanDoImplicitNullCheckOn(invoke->InputAt(0)));
1249
1250  // Check if input is null, return false if it is.
1251  __ Cbz(arg, &return_false);
1252
1253  // Reference equality check, return true if same reference.
1254  __ Cmp(str, arg);
1255  __ B(&return_true, eq);
1256
1257  // Instanceof check for the argument by comparing class fields.
1258  // All string objects must have the same type since String cannot be subclassed.
1259  // Receiver must be a string object, so its class field is equal to all strings' class fields.
1260  // If the argument is a string object, its class field must be equal to receiver's class field.
1261  __ Ldr(temp, MemOperand(str.X(), class_offset));
1262  __ Ldr(temp1, MemOperand(arg.X(), class_offset));
1263  __ Cmp(temp, temp1);
1264  __ B(&return_false, ne);
1265
1266  // Load lengths of this and argument strings.
1267  __ Ldr(temp, MemOperand(str.X(), count_offset));
1268  __ Ldr(temp1, MemOperand(arg.X(), count_offset));
1269  // Check if lengths are equal, return false if they're not.
1270  __ Cmp(temp, temp1);
1271  __ B(&return_false, ne);
1272  // Store offset of string value in preparation for comparison loop
1273  __ Mov(temp1, value_offset);
1274  // Return true if both strings are empty.
1275  __ Cbz(temp, &return_true);
1276
1277  // Assertions that must hold in order to compare strings 4 characters at a time.
1278  DCHECK_ALIGNED(value_offset, 8);
1279  static_assert(IsAligned<8>(kObjectAlignment), "String of odd length is not zero padded");
1280
1281  temp1 = temp1.X();
1282  temp2 = temp2.X();
1283
1284  // Loop to compare strings 4 characters at a time starting at the beginning of the string.
1285  // Ok to do this because strings are zero-padded to be 8-byte aligned.
1286  __ Bind(&loop);
1287  __ Ldr(out, MemOperand(str.X(), temp1));
1288  __ Ldr(temp2, MemOperand(arg.X(), temp1));
1289  __ Add(temp1, temp1, Operand(sizeof(uint64_t)));
1290  __ Cmp(out, temp2);
1291  __ B(&return_false, ne);
1292  __ Sub(temp, temp, Operand(4), SetFlags);
1293  __ B(&loop, gt);
1294
1295  // Return true and exit the function.
1296  // If loop does not result in returning false, we return true.
1297  __ Bind(&return_true);
1298  __ Mov(out, 1);
1299  __ B(&end);
1300
1301  // Return false and exit the function.
1302  __ Bind(&return_false);
1303  __ Mov(out, 0);
1304  __ Bind(&end);
1305}
1306
1307static void GenerateVisitStringIndexOf(HInvoke* invoke,
1308                                       vixl::MacroAssembler* masm,
1309                                       CodeGeneratorARM64* codegen,
1310                                       ArenaAllocator* allocator,
1311                                       bool start_at_zero) {
1312  LocationSummary* locations = invoke->GetLocations();
1313  Register tmp_reg = WRegisterFrom(locations->GetTemp(0));
1314
1315  // Note that the null check must have been done earlier.
1316  DCHECK(!invoke->CanDoImplicitNullCheckOn(invoke->InputAt(0)));
1317
1318  // Check for code points > 0xFFFF. Either a slow-path check when we don't know statically,
1319  // or directly dispatch if we have a constant.
1320  SlowPathCodeARM64* slow_path = nullptr;
1321  if (invoke->InputAt(1)->IsIntConstant()) {
1322    if (static_cast<uint32_t>(invoke->InputAt(1)->AsIntConstant()->GetValue()) > 0xFFFFU) {
1323      // Always needs the slow-path. We could directly dispatch to it, but this case should be
1324      // rare, so for simplicity just put the full slow-path down and branch unconditionally.
1325      slow_path = new (allocator) IntrinsicSlowPathARM64(invoke);
1326      codegen->AddSlowPath(slow_path);
1327      __ B(slow_path->GetEntryLabel());
1328      __ Bind(slow_path->GetExitLabel());
1329      return;
1330    }
1331  } else {
1332    Register char_reg = WRegisterFrom(locations->InAt(1));
1333    __ Mov(tmp_reg, 0xFFFF);
1334    __ Cmp(char_reg, Operand(tmp_reg));
1335    slow_path = new (allocator) IntrinsicSlowPathARM64(invoke);
1336    codegen->AddSlowPath(slow_path);
1337    __ B(hi, slow_path->GetEntryLabel());
1338  }
1339
1340  if (start_at_zero) {
1341    // Start-index = 0.
1342    __ Mov(tmp_reg, 0);
1343  }
1344
1345  __ Ldr(lr, MemOperand(tr, QUICK_ENTRYPOINT_OFFSET(kArm64WordSize, pIndexOf).Int32Value()));
1346  __ Blr(lr);
1347
1348  if (slow_path != nullptr) {
1349    __ Bind(slow_path->GetExitLabel());
1350  }
1351}
1352
1353void IntrinsicLocationsBuilderARM64::VisitStringIndexOf(HInvoke* invoke) {
1354  LocationSummary* locations = new (arena_) LocationSummary(invoke,
1355                                                            LocationSummary::kCall,
1356                                                            kIntrinsified);
1357  // We have a hand-crafted assembly stub that follows the runtime calling convention. So it's
1358  // best to align the inputs accordingly.
1359  InvokeRuntimeCallingConvention calling_convention;
1360  locations->SetInAt(0, LocationFrom(calling_convention.GetRegisterAt(0)));
1361  locations->SetInAt(1, LocationFrom(calling_convention.GetRegisterAt(1)));
1362  locations->SetOut(calling_convention.GetReturnLocation(Primitive::kPrimInt));
1363
1364  // Need a temp for slow-path codepoint compare, and need to send start_index=0.
1365  locations->AddTemp(LocationFrom(calling_convention.GetRegisterAt(2)));
1366}
1367
1368void IntrinsicCodeGeneratorARM64::VisitStringIndexOf(HInvoke* invoke) {
1369  GenerateVisitStringIndexOf(
1370      invoke, GetVIXLAssembler(), codegen_, GetAllocator(), /* start_at_zero */ true);
1371}
1372
1373void IntrinsicLocationsBuilderARM64::VisitStringIndexOfAfter(HInvoke* invoke) {
1374  LocationSummary* locations = new (arena_) LocationSummary(invoke,
1375                                                            LocationSummary::kCall,
1376                                                            kIntrinsified);
1377  // We have a hand-crafted assembly stub that follows the runtime calling convention. So it's
1378  // best to align the inputs accordingly.
1379  InvokeRuntimeCallingConvention calling_convention;
1380  locations->SetInAt(0, LocationFrom(calling_convention.GetRegisterAt(0)));
1381  locations->SetInAt(1, LocationFrom(calling_convention.GetRegisterAt(1)));
1382  locations->SetInAt(2, LocationFrom(calling_convention.GetRegisterAt(2)));
1383  locations->SetOut(calling_convention.GetReturnLocation(Primitive::kPrimInt));
1384
1385  // Need a temp for slow-path codepoint compare.
1386  locations->AddTemp(Location::RequiresRegister());
1387}
1388
1389void IntrinsicCodeGeneratorARM64::VisitStringIndexOfAfter(HInvoke* invoke) {
1390  GenerateVisitStringIndexOf(
1391      invoke, GetVIXLAssembler(), codegen_, GetAllocator(), /* start_at_zero */ false);
1392}
1393
1394void IntrinsicLocationsBuilderARM64::VisitStringNewStringFromBytes(HInvoke* invoke) {
1395  LocationSummary* locations = new (arena_) LocationSummary(invoke,
1396                                                            LocationSummary::kCall,
1397                                                            kIntrinsified);
1398  InvokeRuntimeCallingConvention calling_convention;
1399  locations->SetInAt(0, LocationFrom(calling_convention.GetRegisterAt(0)));
1400  locations->SetInAt(1, LocationFrom(calling_convention.GetRegisterAt(1)));
1401  locations->SetInAt(2, LocationFrom(calling_convention.GetRegisterAt(2)));
1402  locations->SetInAt(3, LocationFrom(calling_convention.GetRegisterAt(3)));
1403  locations->SetOut(calling_convention.GetReturnLocation(Primitive::kPrimNot));
1404}
1405
1406void IntrinsicCodeGeneratorARM64::VisitStringNewStringFromBytes(HInvoke* invoke) {
1407  vixl::MacroAssembler* masm = GetVIXLAssembler();
1408  LocationSummary* locations = invoke->GetLocations();
1409
1410  Register byte_array = WRegisterFrom(locations->InAt(0));
1411  __ Cmp(byte_array, 0);
1412  SlowPathCodeARM64* slow_path = new (GetAllocator()) IntrinsicSlowPathARM64(invoke);
1413  codegen_->AddSlowPath(slow_path);
1414  __ B(eq, slow_path->GetEntryLabel());
1415
1416  __ Ldr(lr,
1417      MemOperand(tr, QUICK_ENTRYPOINT_OFFSET(kArm64WordSize, pAllocStringFromBytes).Int32Value()));
1418  codegen_->RecordPcInfo(invoke, invoke->GetDexPc());
1419  __ Blr(lr);
1420  __ Bind(slow_path->GetExitLabel());
1421}
1422
1423void IntrinsicLocationsBuilderARM64::VisitStringNewStringFromChars(HInvoke* invoke) {
1424  LocationSummary* locations = new (arena_) LocationSummary(invoke,
1425                                                            LocationSummary::kCall,
1426                                                            kIntrinsified);
1427  InvokeRuntimeCallingConvention calling_convention;
1428  locations->SetInAt(0, LocationFrom(calling_convention.GetRegisterAt(0)));
1429  locations->SetInAt(1, LocationFrom(calling_convention.GetRegisterAt(1)));
1430  locations->SetInAt(2, LocationFrom(calling_convention.GetRegisterAt(2)));
1431  locations->SetOut(calling_convention.GetReturnLocation(Primitive::kPrimNot));
1432}
1433
1434void IntrinsicCodeGeneratorARM64::VisitStringNewStringFromChars(HInvoke* invoke) {
1435  vixl::MacroAssembler* masm = GetVIXLAssembler();
1436
1437  __ Ldr(lr,
1438      MemOperand(tr, QUICK_ENTRYPOINT_OFFSET(kArm64WordSize, pAllocStringFromChars).Int32Value()));
1439  codegen_->RecordPcInfo(invoke, invoke->GetDexPc());
1440  __ Blr(lr);
1441}
1442
1443void IntrinsicLocationsBuilderARM64::VisitStringNewStringFromString(HInvoke* invoke) {
1444  // The inputs plus one temp.
1445  LocationSummary* locations = new (arena_) LocationSummary(invoke,
1446                                                            LocationSummary::kCall,
1447                                                            kIntrinsified);
1448  InvokeRuntimeCallingConvention calling_convention;
1449  locations->SetInAt(0, LocationFrom(calling_convention.GetRegisterAt(0)));
1450  locations->SetInAt(1, LocationFrom(calling_convention.GetRegisterAt(1)));
1451  locations->SetInAt(2, LocationFrom(calling_convention.GetRegisterAt(2)));
1452  locations->SetOut(calling_convention.GetReturnLocation(Primitive::kPrimNot));
1453}
1454
1455void IntrinsicCodeGeneratorARM64::VisitStringNewStringFromString(HInvoke* invoke) {
1456  vixl::MacroAssembler* masm = GetVIXLAssembler();
1457  LocationSummary* locations = invoke->GetLocations();
1458
1459  Register string_to_copy = WRegisterFrom(locations->InAt(0));
1460  __ Cmp(string_to_copy, 0);
1461  SlowPathCodeARM64* slow_path = new (GetAllocator()) IntrinsicSlowPathARM64(invoke);
1462  codegen_->AddSlowPath(slow_path);
1463  __ B(eq, slow_path->GetEntryLabel());
1464
1465  __ Ldr(lr,
1466      MemOperand(tr, QUICK_ENTRYPOINT_OFFSET(kArm64WordSize, pAllocStringFromString).Int32Value()));
1467  codegen_->RecordPcInfo(invoke, invoke->GetDexPc());
1468  __ Blr(lr);
1469  __ Bind(slow_path->GetExitLabel());
1470}
1471
1472// Unimplemented intrinsics.
1473
1474#define UNIMPLEMENTED_INTRINSIC(Name)                                                  \
1475void IntrinsicLocationsBuilderARM64::Visit ## Name(HInvoke* invoke ATTRIBUTE_UNUSED) { \
1476}                                                                                      \
1477void IntrinsicCodeGeneratorARM64::Visit ## Name(HInvoke* invoke ATTRIBUTE_UNUSED) {    \
1478}
1479
1480UNIMPLEMENTED_INTRINSIC(IntegerBitCount)
1481UNIMPLEMENTED_INTRINSIC(LongBitCount)
1482UNIMPLEMENTED_INTRINSIC(SystemArrayCopyChar)
1483UNIMPLEMENTED_INTRINSIC(SystemArrayCopy)
1484UNIMPLEMENTED_INTRINSIC(ReferenceGetReferent)
1485UNIMPLEMENTED_INTRINSIC(StringGetCharsNoCheck)
1486
1487UNIMPLEMENTED_INTRINSIC(MathCos)
1488UNIMPLEMENTED_INTRINSIC(MathSin)
1489UNIMPLEMENTED_INTRINSIC(MathAcos)
1490UNIMPLEMENTED_INTRINSIC(MathAsin)
1491UNIMPLEMENTED_INTRINSIC(MathAtan)
1492UNIMPLEMENTED_INTRINSIC(MathAtan2)
1493UNIMPLEMENTED_INTRINSIC(MathCbrt)
1494UNIMPLEMENTED_INTRINSIC(MathCosh)
1495UNIMPLEMENTED_INTRINSIC(MathExp)
1496UNIMPLEMENTED_INTRINSIC(MathExpm1)
1497UNIMPLEMENTED_INTRINSIC(MathHypot)
1498UNIMPLEMENTED_INTRINSIC(MathLog)
1499UNIMPLEMENTED_INTRINSIC(MathLog10)
1500UNIMPLEMENTED_INTRINSIC(MathNextAfter)
1501UNIMPLEMENTED_INTRINSIC(MathSinh)
1502UNIMPLEMENTED_INTRINSIC(MathTan)
1503UNIMPLEMENTED_INTRINSIC(MathTanh)
1504
1505UNIMPLEMENTED_INTRINSIC(FloatIsInfinite)
1506UNIMPLEMENTED_INTRINSIC(DoubleIsInfinite)
1507UNIMPLEMENTED_INTRINSIC(FloatIsNaN)
1508UNIMPLEMENTED_INTRINSIC(DoubleIsNaN)
1509
1510UNIMPLEMENTED_INTRINSIC(IntegerCompare)
1511UNIMPLEMENTED_INTRINSIC(LongCompare)
1512UNIMPLEMENTED_INTRINSIC(IntegerHighestOneBit)
1513UNIMPLEMENTED_INTRINSIC(LongHighestOneBit)
1514UNIMPLEMENTED_INTRINSIC(IntegerLowestOneBit)
1515UNIMPLEMENTED_INTRINSIC(LongLowestOneBit)
1516UNIMPLEMENTED_INTRINSIC(IntegerSignum)
1517UNIMPLEMENTED_INTRINSIC(LongSignum)
1518
1519// Rotate operations are handled as HRor instructions.
1520UNIMPLEMENTED_INTRINSIC(IntegerRotateLeft)
1521UNIMPLEMENTED_INTRINSIC(IntegerRotateRight)
1522UNIMPLEMENTED_INTRINSIC(LongRotateLeft)
1523UNIMPLEMENTED_INTRINSIC(LongRotateRight)
1524
1525#undef UNIMPLEMENTED_INTRINSIC
1526
1527#undef __
1528
1529}  // namespace arm64
1530}  // namespace art
1531