intrinsics_arm.cc revision 2bcf9bf784a0021630d8fe63d7230d46d6891780 
1/*
2 * Copyright (C) 2015 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *      http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17#include "intrinsics_arm.h"
18
19#include "arch/arm/instruction_set_features_arm.h"
20#include "code_generator_arm.h"
21#include "entrypoints/quick/quick_entrypoints.h"
22#include "intrinsics.h"
23#include "mirror/array-inl.h"
24#include "mirror/art_method.h"
25#include "mirror/string.h"
26#include "thread.h"
27#include "utils/arm/assembler_arm.h"
28
29namespace art {
30
31namespace arm {
32
33ArmAssembler* IntrinsicCodeGeneratorARM::GetAssembler() {
34  return codegen_->GetAssembler();
35}
36
37ArenaAllocator* IntrinsicCodeGeneratorARM::GetAllocator() {
38  return codegen_->GetGraph()->GetArena();
39}
40
41#define __ codegen->GetAssembler()->
42
43static void MoveFromReturnRegister(Location trg, Primitive::Type type, CodeGeneratorARM* codegen) {
44  if (!trg.IsValid()) {
45    DCHECK(type == Primitive::kPrimVoid);
46    return;
47  }
48
49  DCHECK_NE(type, Primitive::kPrimVoid);
50
51  if (Primitive::IsIntegralType(type)) {
52    if (type == Primitive::kPrimLong) {
53      Register trg_reg_lo = trg.AsRegisterPairLow<Register>();
54      Register trg_reg_hi = trg.AsRegisterPairHigh<Register>();
55      Register res_reg_lo = R0;
56      Register res_reg_hi = R1;
57      if (trg_reg_lo != res_reg_hi) {
58        if (trg_reg_lo != res_reg_lo) {
59          __ mov(trg_reg_lo, ShifterOperand(res_reg_lo));
60          __ mov(trg_reg_hi, ShifterOperand(res_reg_hi));
61        } else {
62          DCHECK_EQ(trg_reg_lo + 1, trg_reg_hi);
63        }
64      } else {
65        __ mov(trg_reg_hi, ShifterOperand(res_reg_hi));
66        __ mov(trg_reg_lo, ShifterOperand(res_reg_lo));
67      }
68    } else {
69      Register trg_reg = trg.AsRegister<Register>();
70      Register res_reg = R0;
71      if (trg_reg != res_reg) {
72        __ mov(trg_reg, ShifterOperand(res_reg));
73      }
74    }
75  } else {
76    UNIMPLEMENTED(FATAL) << "Floating-point return.";
77  }
78}
79
80static void MoveArguments(HInvoke* invoke, ArenaAllocator* arena, CodeGeneratorARM* codegen) {
81  if (invoke->InputCount() == 0) {
82    return;
83  }
84
85  LocationSummary* locations = invoke->GetLocations();
86  InvokeDexCallingConventionVisitor calling_convention_visitor;
87
88  // We're moving potentially two or more locations to locations that could overlap, so we need
89  // a parallel move resolver.
90  HParallelMove parallel_move(arena);
91
92  for (size_t i = 0; i < invoke->InputCount(); i++) {
93    HInstruction* input = invoke->InputAt(i);
94    Location cc_loc = calling_convention_visitor.GetNextLocation(input->GetType());
95    Location actual_loc = locations->InAt(i);
96
97    parallel_move.AddMove(actual_loc, cc_loc, nullptr);
98  }
99
100  codegen->GetMoveResolver()->EmitNativeCode(&parallel_move);
101}
102
103// Slow-path for fallback (calling the managed code to handle the intrinsic) in an intrinsified
104// call. This will copy the arguments into the positions for a regular call.
105//
106// Note: The actual parameters are required to be in the locations given by the invoke's location
107//       summary. If an intrinsic modifies those locations before a slowpath call, they must be
108//       restored!
109class IntrinsicSlowPathARM : public SlowPathCodeARM {
110 public:
111  explicit IntrinsicSlowPathARM(HInvoke* invoke) : invoke_(invoke) { }
112
113  void EmitNativeCode(CodeGenerator* codegen_in) OVERRIDE {
114    CodeGeneratorARM* codegen = down_cast<CodeGeneratorARM*>(codegen_in);
115    __ Bind(GetEntryLabel());
116
117    codegen->SaveLiveRegisters(invoke_->GetLocations());
118
119    MoveArguments(invoke_, codegen->GetGraph()->GetArena(), codegen);
120
121    if (invoke_->IsInvokeStaticOrDirect()) {
122      codegen->GenerateStaticOrDirectCall(invoke_->AsInvokeStaticOrDirect(), kArtMethodRegister);
123    } else {
124      UNIMPLEMENTED(FATAL) << "Non-direct intrinsic slow-path not yet implemented";
125      UNREACHABLE();
126    }
127
128    // Copy the result back to the expected output.
129    Location out = invoke_->GetLocations()->Out();
130    if (out.IsValid()) {
131      DCHECK(out.IsRegister());  // TODO: Replace this when we support output in memory.
132      DCHECK(!invoke_->GetLocations()->GetLiveRegisters()->ContainsCoreRegister(out.reg()));
133      MoveFromReturnRegister(out, invoke_->GetType(), codegen);
134    }
135
136    codegen->RestoreLiveRegisters(invoke_->GetLocations());
137    __ b(GetExitLabel());
138  }
139
140 private:
141  // The instruction where this slow path is happening.
142  HInvoke* const invoke_;
143
144  DISALLOW_COPY_AND_ASSIGN(IntrinsicSlowPathARM);
145};
146
147#undef __
148
149bool IntrinsicLocationsBuilderARM::TryDispatch(HInvoke* invoke) {
150  Dispatch(invoke);
151  LocationSummary* res = invoke->GetLocations();
152  return res != nullptr && res->Intrinsified();
153}
154
155#define __ assembler->
156
157static void CreateFPToIntLocations(ArenaAllocator* arena, HInvoke* invoke) {
158  LocationSummary* locations = new (arena) LocationSummary(invoke,
159                                                           LocationSummary::kNoCall,
160                                                           kIntrinsified);
161  locations->SetInAt(0, Location::RequiresFpuRegister());
162  locations->SetOut(Location::RequiresRegister());
163}
164
165static void CreateIntToFPLocations(ArenaAllocator* arena, HInvoke* invoke) {
166  LocationSummary* locations = new (arena) LocationSummary(invoke,
167                                                           LocationSummary::kNoCall,
168                                                           kIntrinsified);
169  locations->SetInAt(0, Location::RequiresRegister());
170  locations->SetOut(Location::RequiresFpuRegister());
171}
172
173static void MoveFPToInt(LocationSummary* locations, bool is64bit, ArmAssembler* assembler) {
174  Location input = locations->InAt(0);
175  Location output = locations->Out();
176  if (is64bit) {
177    __ vmovrrd(output.AsRegisterPairLow<Register>(),
178               output.AsRegisterPairHigh<Register>(),
179               FromLowSToD(input.AsFpuRegisterPairLow<SRegister>()));
180  } else {
181    __ vmovrs(output.AsRegister<Register>(), input.AsFpuRegister<SRegister>());
182  }
183}
184
185static void MoveIntToFP(LocationSummary* locations, bool is64bit, ArmAssembler* assembler) {
186  Location input = locations->InAt(0);
187  Location output = locations->Out();
188  if (is64bit) {
189    __ vmovdrr(FromLowSToD(output.AsFpuRegisterPairLow<SRegister>()),
190               input.AsRegisterPairLow<Register>(),
191               input.AsRegisterPairHigh<Register>());
192  } else {
193    __ vmovsr(output.AsFpuRegister<SRegister>(), input.AsRegister<Register>());
194  }
195}
196
197void IntrinsicLocationsBuilderARM::VisitDoubleDoubleToRawLongBits(HInvoke* invoke) {
198  CreateFPToIntLocations(arena_, invoke);
199}
200void IntrinsicLocationsBuilderARM::VisitDoubleLongBitsToDouble(HInvoke* invoke) {
201  CreateIntToFPLocations(arena_, invoke);
202}
203
204void IntrinsicCodeGeneratorARM::VisitDoubleDoubleToRawLongBits(HInvoke* invoke) {
205  MoveFPToInt(invoke->GetLocations(), true, GetAssembler());
206}
207void IntrinsicCodeGeneratorARM::VisitDoubleLongBitsToDouble(HInvoke* invoke) {
208  MoveIntToFP(invoke->GetLocations(), true, GetAssembler());
209}
210
211void IntrinsicLocationsBuilderARM::VisitFloatFloatToRawIntBits(HInvoke* invoke) {
212  CreateFPToIntLocations(arena_, invoke);
213}
214void IntrinsicLocationsBuilderARM::VisitFloatIntBitsToFloat(HInvoke* invoke) {
215  CreateIntToFPLocations(arena_, invoke);
216}
217
218void IntrinsicCodeGeneratorARM::VisitFloatFloatToRawIntBits(HInvoke* invoke) {
219  MoveFPToInt(invoke->GetLocations(), false, GetAssembler());
220}
221void IntrinsicCodeGeneratorARM::VisitFloatIntBitsToFloat(HInvoke* invoke) {
222  MoveIntToFP(invoke->GetLocations(), false, GetAssembler());
223}
224
225static void CreateIntToIntLocations(ArenaAllocator* arena, HInvoke* invoke) {
226  LocationSummary* locations = new (arena) LocationSummary(invoke,
227                                                           LocationSummary::kNoCall,
228                                                           kIntrinsified);
229  locations->SetInAt(0, Location::RequiresRegister());
230  locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
231}
232
233static void CreateFPToFPLocations(ArenaAllocator* arena, HInvoke* invoke) {
234  LocationSummary* locations = new (arena) LocationSummary(invoke,
235                                                           LocationSummary::kNoCall,
236                                                           kIntrinsified);
237  locations->SetInAt(0, Location::RequiresFpuRegister());
238  locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap);
239}
240
241static void MathAbsFP(LocationSummary* locations, bool is64bit, ArmAssembler* assembler) {
242  Location in = locations->InAt(0);
243  Location out = locations->Out();
244
245  if (is64bit) {
246    __ vabsd(FromLowSToD(out.AsFpuRegisterPairLow<SRegister>()),
247             FromLowSToD(in.AsFpuRegisterPairLow<SRegister>()));
248  } else {
249    __ vabss(out.AsFpuRegister<SRegister>(), in.AsFpuRegister<SRegister>());
250  }
251}
252
253void IntrinsicLocationsBuilderARM::VisitMathAbsDouble(HInvoke* invoke) {
254  CreateFPToFPLocations(arena_, invoke);
255}
256
257void IntrinsicCodeGeneratorARM::VisitMathAbsDouble(HInvoke* invoke) {
258  MathAbsFP(invoke->GetLocations(), true, GetAssembler());
259}
260
261void IntrinsicLocationsBuilderARM::VisitMathAbsFloat(HInvoke* invoke) {
262  CreateFPToFPLocations(arena_, invoke);
263}
264
265void IntrinsicCodeGeneratorARM::VisitMathAbsFloat(HInvoke* invoke) {
266  MathAbsFP(invoke->GetLocations(), false, GetAssembler());
267}
268
269static void CreateIntToIntPlusTemp(ArenaAllocator* arena, HInvoke* invoke) {
270  LocationSummary* locations = new (arena) LocationSummary(invoke,
271                                                           LocationSummary::kNoCall,
272                                                           kIntrinsified);
273  locations->SetInAt(0, Location::RequiresRegister());
274  locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
275
276  locations->AddTemp(Location::RequiresRegister());
277}
278
279static void GenAbsInteger(LocationSummary* locations,
280                          bool is64bit,
281                          ArmAssembler* assembler) {
282  Location in = locations->InAt(0);
283  Location output = locations->Out();
284
285  Register mask = locations->GetTemp(0).AsRegister<Register>();
286
287  if (is64bit) {
288    Register in_reg_lo = in.AsRegisterPairLow<Register>();
289    Register in_reg_hi = in.AsRegisterPairHigh<Register>();
290    Register out_reg_lo = output.AsRegisterPairLow<Register>();
291    Register out_reg_hi = output.AsRegisterPairHigh<Register>();
292
293    DCHECK_NE(out_reg_lo, in_reg_hi) << "Diagonal overlap unexpected.";
294
295    __ Asr(mask, in_reg_hi, 31);
296    __ adds(out_reg_lo, in_reg_lo, ShifterOperand(mask));
297    __ adc(out_reg_hi, in_reg_hi, ShifterOperand(mask));
298    __ eor(out_reg_lo, mask, ShifterOperand(out_reg_lo));
299    __ eor(out_reg_hi, mask, ShifterOperand(out_reg_hi));
300  } else {
301    Register in_reg = in.AsRegister<Register>();
302    Register out_reg = output.AsRegister<Register>();
303
304    __ Asr(mask, in_reg, 31);
305    __ add(out_reg, in_reg, ShifterOperand(mask));
306    __ eor(out_reg, mask, ShifterOperand(out_reg));
307  }
308}
309
310void IntrinsicLocationsBuilderARM::VisitMathAbsInt(HInvoke* invoke) {
311  CreateIntToIntPlusTemp(arena_, invoke);
312}
313
314void IntrinsicCodeGeneratorARM::VisitMathAbsInt(HInvoke* invoke) {
315  GenAbsInteger(invoke->GetLocations(), false, GetAssembler());
316}
317
318
319void IntrinsicLocationsBuilderARM::VisitMathAbsLong(HInvoke* invoke) {
320  CreateIntToIntPlusTemp(arena_, invoke);
321}
322
323void IntrinsicCodeGeneratorARM::VisitMathAbsLong(HInvoke* invoke) {
324  GenAbsInteger(invoke->GetLocations(), true, GetAssembler());
325}
326
327static void GenMinMax(LocationSummary* locations,
328                      bool is_min,
329                      ArmAssembler* assembler) {
330  Register op1 = locations->InAt(0).AsRegister<Register>();
331  Register op2 = locations->InAt(1).AsRegister<Register>();
332  Register out = locations->Out().AsRegister<Register>();
333
334  __ cmp(op1, ShifterOperand(op2));
335
336  __ it((is_min) ? Condition::LT : Condition::GT, kItElse);
337  __ mov(out, ShifterOperand(op1), is_min ? Condition::LT : Condition::GT);
338  __ mov(out, ShifterOperand(op2), is_min ? Condition::GE : Condition::LE);
339}
340
341static void CreateIntIntToIntLocations(ArenaAllocator* arena, HInvoke* invoke) {
342  LocationSummary* locations = new (arena) LocationSummary(invoke,
343                                                           LocationSummary::kNoCall,
344                                                           kIntrinsified);
345  locations->SetInAt(0, Location::RequiresRegister());
346  locations->SetInAt(1, Location::RequiresRegister());
347  locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
348}
349
350void IntrinsicLocationsBuilderARM::VisitMathMinIntInt(HInvoke* invoke) {
351  CreateIntIntToIntLocations(arena_, invoke);
352}
353
354void IntrinsicCodeGeneratorARM::VisitMathMinIntInt(HInvoke* invoke) {
355  GenMinMax(invoke->GetLocations(), true, GetAssembler());
356}
357
358void IntrinsicLocationsBuilderARM::VisitMathMaxIntInt(HInvoke* invoke) {
359  CreateIntIntToIntLocations(arena_, invoke);
360}
361
362void IntrinsicCodeGeneratorARM::VisitMathMaxIntInt(HInvoke* invoke) {
363  GenMinMax(invoke->GetLocations(), false, GetAssembler());
364}
365
366void IntrinsicLocationsBuilderARM::VisitMathSqrt(HInvoke* invoke) {
367  CreateFPToFPLocations(arena_, invoke);
368}
369
370void IntrinsicCodeGeneratorARM::VisitMathSqrt(HInvoke* invoke) {
371  LocationSummary* locations = invoke->GetLocations();
372  ArmAssembler* assembler = GetAssembler();
373  __ vsqrtd(FromLowSToD(locations->Out().AsFpuRegisterPairLow<SRegister>()),
374            FromLowSToD(locations->InAt(0).AsFpuRegisterPairLow<SRegister>()));
375}
376
377void IntrinsicLocationsBuilderARM::VisitMemoryPeekByte(HInvoke* invoke) {
378  CreateIntToIntLocations(arena_, invoke);
379}
380
381void IntrinsicCodeGeneratorARM::VisitMemoryPeekByte(HInvoke* invoke) {
382  ArmAssembler* assembler = GetAssembler();
383  // Ignore upper 4B of long address.
384  __ ldrsb(invoke->GetLocations()->Out().AsRegister<Register>(),
385           Address(invoke->GetLocations()->InAt(0).AsRegisterPairLow<Register>()));
386}
387
388void IntrinsicLocationsBuilderARM::VisitMemoryPeekIntNative(HInvoke* invoke) {
389  CreateIntToIntLocations(arena_, invoke);
390}
391
392void IntrinsicCodeGeneratorARM::VisitMemoryPeekIntNative(HInvoke* invoke) {
393  ArmAssembler* assembler = GetAssembler();
394  // Ignore upper 4B of long address.
395  __ ldr(invoke->GetLocations()->Out().AsRegister<Register>(),
396         Address(invoke->GetLocations()->InAt(0).AsRegisterPairLow<Register>()));
397}
398
399void IntrinsicLocationsBuilderARM::VisitMemoryPeekLongNative(HInvoke* invoke) {
400  CreateIntToIntLocations(arena_, invoke);
401}
402
403void IntrinsicCodeGeneratorARM::VisitMemoryPeekLongNative(HInvoke* invoke) {
404  ArmAssembler* assembler = GetAssembler();
405  // Ignore upper 4B of long address.
406  Register addr = invoke->GetLocations()->InAt(0).AsRegisterPairLow<Register>();
407  // Worst case: Control register bit SCTLR.A = 0. Then unaligned accesses throw a processor
408  // exception. So we can't use ldrd as addr may be unaligned.
409  Register lo = invoke->GetLocations()->Out().AsRegisterPairLow<Register>();
410  Register hi = invoke->GetLocations()->Out().AsRegisterPairHigh<Register>();
411  if (addr == lo) {
412    __ ldr(hi, Address(addr, 4));
413    __ ldr(lo, Address(addr, 0));
414  } else {
415    __ ldr(lo, Address(addr, 0));
416    __ ldr(hi, Address(addr, 4));
417  }
418}
419
420void IntrinsicLocationsBuilderARM::VisitMemoryPeekShortNative(HInvoke* invoke) {
421  CreateIntToIntLocations(arena_, invoke);
422}
423
424void IntrinsicCodeGeneratorARM::VisitMemoryPeekShortNative(HInvoke* invoke) {
425  ArmAssembler* assembler = GetAssembler();
426  // Ignore upper 4B of long address.
427  __ ldrsh(invoke->GetLocations()->Out().AsRegister<Register>(),
428           Address(invoke->GetLocations()->InAt(0).AsRegisterPairLow<Register>()));
429}
430
431static void CreateIntIntToVoidLocations(ArenaAllocator* arena, HInvoke* invoke) {
432  LocationSummary* locations = new (arena) LocationSummary(invoke,
433                                                           LocationSummary::kNoCall,
434                                                           kIntrinsified);
435  locations->SetInAt(0, Location::RequiresRegister());
436  locations->SetInAt(1, Location::RequiresRegister());
437}
438
439void IntrinsicLocationsBuilderARM::VisitMemoryPokeByte(HInvoke* invoke) {
440  CreateIntIntToVoidLocations(arena_, invoke);
441}
442
443void IntrinsicCodeGeneratorARM::VisitMemoryPokeByte(HInvoke* invoke) {
444  ArmAssembler* assembler = GetAssembler();
445  __ strb(invoke->GetLocations()->InAt(1).AsRegister<Register>(),
446          Address(invoke->GetLocations()->InAt(0).AsRegisterPairLow<Register>()));
447}
448
449void IntrinsicLocationsBuilderARM::VisitMemoryPokeIntNative(HInvoke* invoke) {
450  CreateIntIntToVoidLocations(arena_, invoke);
451}
452
453void IntrinsicCodeGeneratorARM::VisitMemoryPokeIntNative(HInvoke* invoke) {
454  ArmAssembler* assembler = GetAssembler();
455  __ str(invoke->GetLocations()->InAt(1).AsRegister<Register>(),
456         Address(invoke->GetLocations()->InAt(0).AsRegisterPairLow<Register>()));
457}
458
459void IntrinsicLocationsBuilderARM::VisitMemoryPokeLongNative(HInvoke* invoke) {
460  CreateIntIntToVoidLocations(arena_, invoke);
461}
462
463void IntrinsicCodeGeneratorARM::VisitMemoryPokeLongNative(HInvoke* invoke) {
464  ArmAssembler* assembler = GetAssembler();
465  // Ignore upper 4B of long address.
466  Register addr = invoke->GetLocations()->InAt(0).AsRegisterPairLow<Register>();
467  // Worst case: Control register bit SCTLR.A = 0. Then unaligned accesses throw a processor
468  // exception. So we can't use ldrd as addr may be unaligned.
469  __ str(invoke->GetLocations()->InAt(1).AsRegisterPairLow<Register>(), Address(addr, 0));
470  __ str(invoke->GetLocations()->InAt(1).AsRegisterPairHigh<Register>(), Address(addr, 4));
471}
472
473void IntrinsicLocationsBuilderARM::VisitMemoryPokeShortNative(HInvoke* invoke) {
474  CreateIntIntToVoidLocations(arena_, invoke);
475}
476
477void IntrinsicCodeGeneratorARM::VisitMemoryPokeShortNative(HInvoke* invoke) {
478  ArmAssembler* assembler = GetAssembler();
479  __ strh(invoke->GetLocations()->InAt(1).AsRegister<Register>(),
480          Address(invoke->GetLocations()->InAt(0).AsRegisterPairLow<Register>()));
481}
482
483void IntrinsicLocationsBuilderARM::VisitThreadCurrentThread(HInvoke* invoke) {
484  LocationSummary* locations = new (arena_) LocationSummary(invoke,
485                                                            LocationSummary::kNoCall,
486                                                            kIntrinsified);
487  locations->SetOut(Location::RequiresRegister());
488}
489
490void IntrinsicCodeGeneratorARM::VisitThreadCurrentThread(HInvoke* invoke) {
491  ArmAssembler* assembler = GetAssembler();
492  __ LoadFromOffset(kLoadWord,
493                    invoke->GetLocations()->Out().AsRegister<Register>(),
494                    TR,
495                    Thread::PeerOffset<kArmPointerSize>().Int32Value());
496}
497
498static void GenUnsafeGet(HInvoke* invoke,
499                         Primitive::Type type,
500                         bool is_volatile,
501                         CodeGeneratorARM* codegen) {
502  LocationSummary* locations = invoke->GetLocations();
503  DCHECK((type == Primitive::kPrimInt) ||
504         (type == Primitive::kPrimLong) ||
505         (type == Primitive::kPrimNot));
506  ArmAssembler* assembler = codegen->GetAssembler();
507  Register base = locations->InAt(1).AsRegister<Register>();           // Object pointer.
508  Register offset = locations->InAt(2).AsRegisterPairLow<Register>();  // Long offset, lo part only.
509
510  if (type == Primitive::kPrimLong) {
511    Register trg_lo = locations->Out().AsRegisterPairLow<Register>();
512    __ add(IP, base, ShifterOperand(offset));
513    if (is_volatile && !codegen->GetInstructionSetFeatures().HasAtomicLdrdAndStrd()) {
514      Register trg_hi = locations->Out().AsRegisterPairHigh<Register>();
515      __ ldrexd(trg_lo, trg_hi, IP);
516    } else {
517      __ ldrd(trg_lo, Address(IP));
518    }
519  } else {
520    Register trg = locations->Out().AsRegister<Register>();
521    __ ldr(trg, Address(base, offset));
522  }
523
524  if (is_volatile) {
525    __ dmb(ISH);
526  }
527}
528
529static void CreateIntIntIntToIntLocations(ArenaAllocator* arena, HInvoke* invoke) {
530  LocationSummary* locations = new (arena) LocationSummary(invoke,
531                                                           LocationSummary::kNoCall,
532                                                           kIntrinsified);
533  locations->SetInAt(0, Location::NoLocation());        // Unused receiver.
534  locations->SetInAt(1, Location::RequiresRegister());
535  locations->SetInAt(2, Location::RequiresRegister());
536  locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
537}
538
539void IntrinsicLocationsBuilderARM::VisitUnsafeGet(HInvoke* invoke) {
540  CreateIntIntIntToIntLocations(arena_, invoke);
541}
542void IntrinsicLocationsBuilderARM::VisitUnsafeGetVolatile(HInvoke* invoke) {
543  CreateIntIntIntToIntLocations(arena_, invoke);
544}
545void IntrinsicLocationsBuilderARM::VisitUnsafeGetLong(HInvoke* invoke) {
546  CreateIntIntIntToIntLocations(arena_, invoke);
547}
548void IntrinsicLocationsBuilderARM::VisitUnsafeGetLongVolatile(HInvoke* invoke) {
549  CreateIntIntIntToIntLocations(arena_, invoke);
550}
551void IntrinsicLocationsBuilderARM::VisitUnsafeGetObject(HInvoke* invoke) {
552  CreateIntIntIntToIntLocations(arena_, invoke);
553}
554void IntrinsicLocationsBuilderARM::VisitUnsafeGetObjectVolatile(HInvoke* invoke) {
555  CreateIntIntIntToIntLocations(arena_, invoke);
556}
557
558void IntrinsicCodeGeneratorARM::VisitUnsafeGet(HInvoke* invoke) {
559  GenUnsafeGet(invoke, Primitive::kPrimInt, false, codegen_);
560}
561void IntrinsicCodeGeneratorARM::VisitUnsafeGetVolatile(HInvoke* invoke) {
562  GenUnsafeGet(invoke, Primitive::kPrimInt, true, codegen_);
563}
564void IntrinsicCodeGeneratorARM::VisitUnsafeGetLong(HInvoke* invoke) {
565  GenUnsafeGet(invoke, Primitive::kPrimLong, false, codegen_);
566}
567void IntrinsicCodeGeneratorARM::VisitUnsafeGetLongVolatile(HInvoke* invoke) {
568  GenUnsafeGet(invoke, Primitive::kPrimLong, true, codegen_);
569}
570void IntrinsicCodeGeneratorARM::VisitUnsafeGetObject(HInvoke* invoke) {
571  GenUnsafeGet(invoke, Primitive::kPrimNot, false, codegen_);
572}
573void IntrinsicCodeGeneratorARM::VisitUnsafeGetObjectVolatile(HInvoke* invoke) {
574  GenUnsafeGet(invoke, Primitive::kPrimNot, true, codegen_);
575}
576
577static void CreateIntIntIntIntToVoid(ArenaAllocator* arena,
578                                     const ArmInstructionSetFeatures& features,
579                                     Primitive::Type type,
580                                     bool is_volatile,
581                                     HInvoke* invoke) {
582  LocationSummary* locations = new (arena) LocationSummary(invoke,
583                                                           LocationSummary::kNoCall,
584                                                           kIntrinsified);
585  locations->SetInAt(0, Location::NoLocation());        // Unused receiver.
586  locations->SetInAt(1, Location::RequiresRegister());
587  locations->SetInAt(2, Location::RequiresRegister());
588  locations->SetInAt(3, Location::RequiresRegister());
589
590  if (type == Primitive::kPrimLong) {
591    // Potentially need temps for ldrexd-strexd loop.
592    if (is_volatile && !features.HasAtomicLdrdAndStrd()) {
593      locations->AddTemp(Location::RequiresRegister());  // Temp_lo.
594      locations->AddTemp(Location::RequiresRegister());  // Temp_hi.
595    }
596  } else if (type == Primitive::kPrimNot) {
597    // Temps for card-marking.
598    locations->AddTemp(Location::RequiresRegister());  // Temp.
599    locations->AddTemp(Location::RequiresRegister());  // Card.
600  }
601}
602
603void IntrinsicLocationsBuilderARM::VisitUnsafePut(HInvoke* invoke) {
604  CreateIntIntIntIntToVoid(arena_, features_, Primitive::kPrimInt, false, invoke);
605}
606void IntrinsicLocationsBuilderARM::VisitUnsafePutOrdered(HInvoke* invoke) {
607  CreateIntIntIntIntToVoid(arena_, features_, Primitive::kPrimInt, false, invoke);
608}
609void IntrinsicLocationsBuilderARM::VisitUnsafePutVolatile(HInvoke* invoke) {
610  CreateIntIntIntIntToVoid(arena_, features_, Primitive::kPrimInt, true, invoke);
611}
612void IntrinsicLocationsBuilderARM::VisitUnsafePutObject(HInvoke* invoke) {
613  CreateIntIntIntIntToVoid(arena_, features_, Primitive::kPrimNot, false, invoke);
614}
615void IntrinsicLocationsBuilderARM::VisitUnsafePutObjectOrdered(HInvoke* invoke) {
616  CreateIntIntIntIntToVoid(arena_, features_, Primitive::kPrimNot, false, invoke);
617}
618void IntrinsicLocationsBuilderARM::VisitUnsafePutObjectVolatile(HInvoke* invoke) {
619  CreateIntIntIntIntToVoid(arena_, features_, Primitive::kPrimNot, true, invoke);
620}
621void IntrinsicLocationsBuilderARM::VisitUnsafePutLong(HInvoke* invoke) {
622  CreateIntIntIntIntToVoid(arena_, features_, Primitive::kPrimLong, false, invoke);
623}
624void IntrinsicLocationsBuilderARM::VisitUnsafePutLongOrdered(HInvoke* invoke) {
625  CreateIntIntIntIntToVoid(arena_, features_, Primitive::kPrimLong, false, invoke);
626}
627void IntrinsicLocationsBuilderARM::VisitUnsafePutLongVolatile(HInvoke* invoke) {
628  CreateIntIntIntIntToVoid(arena_, features_, Primitive::kPrimLong, true, invoke);
629}
630
631static void GenUnsafePut(LocationSummary* locations,
632                         Primitive::Type type,
633                         bool is_volatile,
634                         bool is_ordered,
635                         CodeGeneratorARM* codegen) {
636  ArmAssembler* assembler = codegen->GetAssembler();
637
638  Register base = locations->InAt(1).AsRegister<Register>();           // Object pointer.
639  Register offset = locations->InAt(2).AsRegisterPairLow<Register>();  // Long offset, lo part only.
640  Register value;
641
642  if (is_volatile || is_ordered) {
643    __ dmb(ISH);
644  }
645
646  if (type == Primitive::kPrimLong) {
647    Register value_lo = locations->InAt(3).AsRegisterPairLow<Register>();
648    value = value_lo;
649    if (is_volatile && !codegen->GetInstructionSetFeatures().HasAtomicLdrdAndStrd()) {
650      Register temp_lo = locations->GetTemp(0).AsRegister<Register>();
651      Register temp_hi = locations->GetTemp(1).AsRegister<Register>();
652      Register value_hi = locations->InAt(3).AsRegisterPairHigh<Register>();
653
654      __ add(IP, base, ShifterOperand(offset));
655      Label loop_head;
656      __ Bind(&loop_head);
657      __ ldrexd(temp_lo, temp_hi, IP);
658      __ strexd(temp_lo, value_lo, value_hi, IP);
659      __ cmp(temp_lo, ShifterOperand(0));
660      __ b(&loop_head, NE);
661    } else {
662      __ add(IP, base, ShifterOperand(offset));
663      __ strd(value_lo, Address(IP));
664    }
665  } else {
666    value =  locations->InAt(3).AsRegister<Register>();
667    __ str(value, Address(base, offset));
668  }
669
670  if (is_volatile) {
671    __ dmb(ISH);
672  }
673
674  if (type == Primitive::kPrimNot) {
675    Register temp = locations->GetTemp(0).AsRegister<Register>();
676    Register card = locations->GetTemp(1).AsRegister<Register>();
677    codegen->MarkGCCard(temp, card, base, value);
678  }
679}
680
681void IntrinsicCodeGeneratorARM::VisitUnsafePut(HInvoke* invoke) {
682  GenUnsafePut(invoke->GetLocations(), Primitive::kPrimInt, false, false, codegen_);
683}
684void IntrinsicCodeGeneratorARM::VisitUnsafePutOrdered(HInvoke* invoke) {
685  GenUnsafePut(invoke->GetLocations(), Primitive::kPrimInt, false, true, codegen_);
686}
687void IntrinsicCodeGeneratorARM::VisitUnsafePutVolatile(HInvoke* invoke) {
688  GenUnsafePut(invoke->GetLocations(), Primitive::kPrimInt, true, false, codegen_);
689}
690void IntrinsicCodeGeneratorARM::VisitUnsafePutObject(HInvoke* invoke) {
691  GenUnsafePut(invoke->GetLocations(), Primitive::kPrimNot, false, false, codegen_);
692}
693void IntrinsicCodeGeneratorARM::VisitUnsafePutObjectOrdered(HInvoke* invoke) {
694  GenUnsafePut(invoke->GetLocations(), Primitive::kPrimNot, false, true, codegen_);
695}
696void IntrinsicCodeGeneratorARM::VisitUnsafePutObjectVolatile(HInvoke* invoke) {
697  GenUnsafePut(invoke->GetLocations(), Primitive::kPrimNot, true, false, codegen_);
698}
699void IntrinsicCodeGeneratorARM::VisitUnsafePutLong(HInvoke* invoke) {
700  GenUnsafePut(invoke->GetLocations(), Primitive::kPrimLong, false, false, codegen_);
701}
702void IntrinsicCodeGeneratorARM::VisitUnsafePutLongOrdered(HInvoke* invoke) {
703  GenUnsafePut(invoke->GetLocations(), Primitive::kPrimLong, false, true, codegen_);
704}
705void IntrinsicCodeGeneratorARM::VisitUnsafePutLongVolatile(HInvoke* invoke) {
706  GenUnsafePut(invoke->GetLocations(), Primitive::kPrimLong, true, false, codegen_);
707}
708
709static void CreateIntIntIntIntIntToIntPlusTemps(ArenaAllocator* arena,
710                                                HInvoke* invoke) {
711  LocationSummary* locations = new (arena) LocationSummary(invoke,
712                                                           LocationSummary::kNoCall,
713                                                           kIntrinsified);
714  locations->SetInAt(0, Location::NoLocation());        // Unused receiver.
715  locations->SetInAt(1, Location::RequiresRegister());
716  locations->SetInAt(2, Location::RequiresRegister());
717  locations->SetInAt(3, Location::RequiresRegister());
718  locations->SetInAt(4, Location::RequiresRegister());
719
720  locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
721
722  locations->AddTemp(Location::RequiresRegister());  // Pointer.
723  locations->AddTemp(Location::RequiresRegister());  // Temp 1.
724  locations->AddTemp(Location::RequiresRegister());  // Temp 2.
725}
726
727static void GenCas(LocationSummary* locations, Primitive::Type type, CodeGeneratorARM* codegen) {
728  DCHECK_NE(type, Primitive::kPrimLong);
729
730  ArmAssembler* assembler = codegen->GetAssembler();
731
732  Register out = locations->Out().AsRegister<Register>();              // Boolean result.
733
734  Register base = locations->InAt(1).AsRegister<Register>();           // Object pointer.
735  Register offset = locations->InAt(2).AsRegisterPairLow<Register>();  // Offset (discard high 4B).
736  Register expected_lo = locations->InAt(3).AsRegister<Register>();    // Expected.
737  Register value_lo = locations->InAt(4).AsRegister<Register>();       // Value.
738
739  Register tmp_ptr = locations->GetTemp(0).AsRegister<Register>();     // Pointer to actual memory.
740  Register tmp_lo = locations->GetTemp(1).AsRegister<Register>();      // Value in memory.
741
742  if (type == Primitive::kPrimNot) {
743    // Mark card for object assuming new value is stored. Worst case we will mark an unchanged
744    // object and scan the receiver at the next GC for nothing.
745    codegen->MarkGCCard(tmp_ptr, tmp_lo, base, value_lo);
746  }
747
748  // Prevent reordering with prior memory operations.
749  __ dmb(ISH);
750
751  __ add(tmp_ptr, base, ShifterOperand(offset));
752
753  // do {
754  //   tmp = [r_ptr] - expected;
755  // } while (tmp == 0 && failure([r_ptr] <- r_new_value));
756  // result = tmp != 0;
757
758  Label loop_head;
759  __ Bind(&loop_head);
760
761  __ ldrex(tmp_lo, tmp_ptr);
762
763  __ subs(tmp_lo, tmp_lo, ShifterOperand(expected_lo));
764
765  __ it(EQ, ItState::kItT);
766  __ strex(tmp_lo, value_lo, tmp_ptr, EQ);
767  __ cmp(tmp_lo, ShifterOperand(1), EQ);
768
769  __ b(&loop_head, EQ);
770
771  __ dmb(ISH);
772
773  __ rsbs(out, tmp_lo, ShifterOperand(1));
774  __ it(CC);
775  __ mov(out, ShifterOperand(0), CC);
776}
777
778void IntrinsicLocationsBuilderARM::VisitUnsafeCASInt(HInvoke* invoke ATTRIBUTE_UNUSED) {
779  CreateIntIntIntIntIntToIntPlusTemps(arena_, invoke);
780}
781void IntrinsicLocationsBuilderARM::VisitUnsafeCASObject(HInvoke* invoke ATTRIBUTE_UNUSED) {
782  CreateIntIntIntIntIntToIntPlusTemps(arena_, invoke);
783}
784void IntrinsicCodeGeneratorARM::VisitUnsafeCASInt(HInvoke* invoke) {
785  GenCas(invoke->GetLocations(), Primitive::kPrimInt, codegen_);
786}
787void IntrinsicCodeGeneratorARM::VisitUnsafeCASObject(HInvoke* invoke) {
788  GenCas(invoke->GetLocations(), Primitive::kPrimNot, codegen_);
789}
790
791void IntrinsicLocationsBuilderARM::VisitStringCharAt(HInvoke* invoke) {
792  LocationSummary* locations = new (arena_) LocationSummary(invoke,
793                                                            LocationSummary::kCallOnSlowPath,
794                                                            kIntrinsified);
795  locations->SetInAt(0, Location::RequiresRegister());
796  locations->SetInAt(1, Location::RequiresRegister());
797  locations->SetOut(Location::RequiresRegister(), Location::kOutputOverlap);
798
799  locations->AddTemp(Location::RequiresRegister());
800  locations->AddTemp(Location::RequiresRegister());
801}
802
803void IntrinsicCodeGeneratorARM::VisitStringCharAt(HInvoke* invoke) {
804  ArmAssembler* assembler = GetAssembler();
805  LocationSummary* locations = invoke->GetLocations();
806
807  // Location of reference to data array
808  const MemberOffset value_offset = mirror::String::ValueOffset();
809  // Location of count
810  const MemberOffset count_offset = mirror::String::CountOffset();
811  // Starting offset within data array
812  const MemberOffset offset_offset = mirror::String::OffsetOffset();
813  // Start of char data with array_
814  const MemberOffset data_offset = mirror::Array::DataOffset(sizeof(uint16_t));
815
816  Register obj = locations->InAt(0).AsRegister<Register>();  // String object pointer.
817  Register idx = locations->InAt(1).AsRegister<Register>();  // Index of character.
818  Register out = locations->Out().AsRegister<Register>();    // Result character.
819
820  Register temp = locations->GetTemp(0).AsRegister<Register>();
821  Register array_temp = locations->GetTemp(1).AsRegister<Register>();
822
823  // TODO: Maybe we can support range check elimination. Overall, though, I think it's not worth
824  //       the cost.
825  // TODO: For simplicity, the index parameter is requested in a register, so different from Quick
826  //       we will not optimize the code for constants (which would save a register).
827
828  SlowPathCodeARM* slow_path = new (GetAllocator()) IntrinsicSlowPathARM(invoke);
829  codegen_->AddSlowPath(slow_path);
830
831  __ ldr(temp, Address(obj, count_offset.Int32Value()));          // temp = str.length.
832  codegen_->MaybeRecordImplicitNullCheck(invoke);
833  __ cmp(idx, ShifterOperand(temp));
834  __ b(slow_path->GetEntryLabel(), CS);
835
836  // Index computation.
837  __ ldr(temp, Address(obj, offset_offset.Int32Value()));         // temp := str.offset.
838  __ ldr(array_temp, Address(obj, value_offset.Int32Value()));    // array_temp := str.offset.
839  __ add(temp, temp, ShifterOperand(idx));
840  DCHECK_EQ(data_offset.Int32Value() % 2, 0);                     // We'll compensate by shifting.
841  __ add(temp, temp, ShifterOperand(data_offset.Int32Value() / 2));
842
843  // Load the value.
844  __ ldrh(out, Address(array_temp, temp, LSL, 1));                // out := array_temp[temp].
845
846  __ Bind(slow_path->GetExitLabel());
847}
848
849// Unimplemented intrinsics.
850
851#define UNIMPLEMENTED_INTRINSIC(Name)                                                  \
852void IntrinsicLocationsBuilderARM::Visit ## Name(HInvoke* invoke ATTRIBUTE_UNUSED) { \
853}                                                                                      \
854void IntrinsicCodeGeneratorARM::Visit ## Name(HInvoke* invoke ATTRIBUTE_UNUSED) {    \
855}
856
857UNIMPLEMENTED_INTRINSIC(IntegerReverse)
858UNIMPLEMENTED_INTRINSIC(IntegerReverseBytes)
859UNIMPLEMENTED_INTRINSIC(LongReverse)
860UNIMPLEMENTED_INTRINSIC(LongReverseBytes)
861UNIMPLEMENTED_INTRINSIC(ShortReverseBytes)
862UNIMPLEMENTED_INTRINSIC(MathMinDoubleDouble)
863UNIMPLEMENTED_INTRINSIC(MathMinFloatFloat)
864UNIMPLEMENTED_INTRINSIC(MathMaxDoubleDouble)
865UNIMPLEMENTED_INTRINSIC(MathMaxFloatFloat)
866UNIMPLEMENTED_INTRINSIC(MathMinLongLong)
867UNIMPLEMENTED_INTRINSIC(MathMaxLongLong)
868UNIMPLEMENTED_INTRINSIC(MathCeil)          // Could be done by changing rounding mode, maybe?
869UNIMPLEMENTED_INTRINSIC(MathFloor)         // Could be done by changing rounding mode, maybe?
870UNIMPLEMENTED_INTRINSIC(MathRint)
871UNIMPLEMENTED_INTRINSIC(MathRoundDouble)   // Could be done by changing rounding mode, maybe?
872UNIMPLEMENTED_INTRINSIC(MathRoundFloat)    // Could be done by changing rounding mode, maybe?
873UNIMPLEMENTED_INTRINSIC(UnsafeCASLong)     // High register pressure.
874UNIMPLEMENTED_INTRINSIC(SystemArrayCopyChar)
875UNIMPLEMENTED_INTRINSIC(StringCompareTo)
876UNIMPLEMENTED_INTRINSIC(StringIsEmpty)  // Might not want to do these two anyways, inlining should
877UNIMPLEMENTED_INTRINSIC(StringLength)   // be good enough here.
878UNIMPLEMENTED_INTRINSIC(StringIndexOf)
879UNIMPLEMENTED_INTRINSIC(StringIndexOfAfter)
880UNIMPLEMENTED_INTRINSIC(ReferenceGetReferent)
881
882}  // namespace arm
883}  // namespace art
884