intrinsics_arm.cc revision a19616e3363276e7f2c471eb2839fb16f1d43f27 
1/*
2 * Copyright (C) 2015 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *      http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17#include "intrinsics_arm.h"
18
19#include "arch/arm/instruction_set_features_arm.h"
20#include "art_method.h"
21#include "code_generator_arm.h"
22#include "entrypoints/quick/quick_entrypoints.h"
23#include "intrinsics.h"
24#include "intrinsics_utils.h"
25#include "mirror/array-inl.h"
26#include "mirror/string.h"
27#include "thread.h"
28#include "utils/arm/assembler_arm.h"
29
30namespace art {
31
32namespace arm {
33
34ArmAssembler* IntrinsicCodeGeneratorARM::GetAssembler() {
35  return codegen_->GetAssembler();
36}
37
38ArenaAllocator* IntrinsicCodeGeneratorARM::GetAllocator() {
39  return codegen_->GetGraph()->GetArena();
40}
41
42using IntrinsicSlowPathARM = IntrinsicSlowPath<InvokeDexCallingConventionVisitorARM>;
43
44bool IntrinsicLocationsBuilderARM::TryDispatch(HInvoke* invoke) {
45  Dispatch(invoke);
46  LocationSummary* res = invoke->GetLocations();
47  if (res == nullptr) {
48    return false;
49  }
50  if (kEmitCompilerReadBarrier && res->CanCall()) {
51    // Generating an intrinsic for this HInvoke may produce an
52    // IntrinsicSlowPathARM slow path.  Currently this approach
53    // does not work when using read barriers, as the emitted
54    // calling sequence will make use of another slow path
55    // (ReadBarrierForRootSlowPathARM for HInvokeStaticOrDirect,
56    // ReadBarrierSlowPathARM for HInvokeVirtual).  So we bail
57    // out in this case.
58    //
59    // TODO: Find a way to have intrinsics work with read barriers.
60    invoke->SetLocations(nullptr);
61    return false;
62  }
63  return res->Intrinsified();
64}
65
66#define __ assembler->
67
68static void CreateFPToIntLocations(ArenaAllocator* arena, HInvoke* invoke) {
69  LocationSummary* locations = new (arena) LocationSummary(invoke,
70                                                           LocationSummary::kNoCall,
71                                                           kIntrinsified);
72  locations->SetInAt(0, Location::RequiresFpuRegister());
73  locations->SetOut(Location::RequiresRegister());
74}
75
76static void CreateIntToFPLocations(ArenaAllocator* arena, HInvoke* invoke) {
77  LocationSummary* locations = new (arena) LocationSummary(invoke,
78                                                           LocationSummary::kNoCall,
79                                                           kIntrinsified);
80  locations->SetInAt(0, Location::RequiresRegister());
81  locations->SetOut(Location::RequiresFpuRegister());
82}
83
84static void MoveFPToInt(LocationSummary* locations, bool is64bit, ArmAssembler* assembler) {
85  Location input = locations->InAt(0);
86  Location output = locations->Out();
87  if (is64bit) {
88    __ vmovrrd(output.AsRegisterPairLow<Register>(),
89               output.AsRegisterPairHigh<Register>(),
90               FromLowSToD(input.AsFpuRegisterPairLow<SRegister>()));
91  } else {
92    __ vmovrs(output.AsRegister<Register>(), input.AsFpuRegister<SRegister>());
93  }
94}
95
96static void MoveIntToFP(LocationSummary* locations, bool is64bit, ArmAssembler* assembler) {
97  Location input = locations->InAt(0);
98  Location output = locations->Out();
99  if (is64bit) {
100    __ vmovdrr(FromLowSToD(output.AsFpuRegisterPairLow<SRegister>()),
101               input.AsRegisterPairLow<Register>(),
102               input.AsRegisterPairHigh<Register>());
103  } else {
104    __ vmovsr(output.AsFpuRegister<SRegister>(), input.AsRegister<Register>());
105  }
106}
107
108void IntrinsicLocationsBuilderARM::VisitDoubleDoubleToRawLongBits(HInvoke* invoke) {
109  CreateFPToIntLocations(arena_, invoke);
110}
111void IntrinsicLocationsBuilderARM::VisitDoubleLongBitsToDouble(HInvoke* invoke) {
112  CreateIntToFPLocations(arena_, invoke);
113}
114
115void IntrinsicCodeGeneratorARM::VisitDoubleDoubleToRawLongBits(HInvoke* invoke) {
116  MoveFPToInt(invoke->GetLocations(), /* is64bit */ true, GetAssembler());
117}
118void IntrinsicCodeGeneratorARM::VisitDoubleLongBitsToDouble(HInvoke* invoke) {
119  MoveIntToFP(invoke->GetLocations(), /* is64bit */ true, GetAssembler());
120}
121
122void IntrinsicLocationsBuilderARM::VisitFloatFloatToRawIntBits(HInvoke* invoke) {
123  CreateFPToIntLocations(arena_, invoke);
124}
125void IntrinsicLocationsBuilderARM::VisitFloatIntBitsToFloat(HInvoke* invoke) {
126  CreateIntToFPLocations(arena_, invoke);
127}
128
129void IntrinsicCodeGeneratorARM::VisitFloatFloatToRawIntBits(HInvoke* invoke) {
130  MoveFPToInt(invoke->GetLocations(), /* is64bit */ false, GetAssembler());
131}
132void IntrinsicCodeGeneratorARM::VisitFloatIntBitsToFloat(HInvoke* invoke) {
133  MoveIntToFP(invoke->GetLocations(), /* is64bit */ false, GetAssembler());
134}
135
136static void CreateIntToIntLocations(ArenaAllocator* arena, HInvoke* invoke) {
137  LocationSummary* locations = new (arena) LocationSummary(invoke,
138                                                           LocationSummary::kNoCall,
139                                                           kIntrinsified);
140  locations->SetInAt(0, Location::RequiresRegister());
141  locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
142}
143
144static void CreateFPToFPLocations(ArenaAllocator* arena, HInvoke* invoke) {
145  LocationSummary* locations = new (arena) LocationSummary(invoke,
146                                                           LocationSummary::kNoCall,
147                                                           kIntrinsified);
148  locations->SetInAt(0, Location::RequiresFpuRegister());
149  locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap);
150}
151
152static void GenNumberOfLeadingZeros(LocationSummary* locations,
153                                    Primitive::Type type,
154                                    ArmAssembler* assembler) {
155  Location in = locations->InAt(0);
156  Register out = locations->Out().AsRegister<Register>();
157
158  DCHECK((type == Primitive::kPrimInt) || (type == Primitive::kPrimLong));
159
160  if (type == Primitive::kPrimLong) {
161    Register in_reg_lo = in.AsRegisterPairLow<Register>();
162    Register in_reg_hi = in.AsRegisterPairHigh<Register>();
163    Label end;
164    __ clz(out, in_reg_hi);
165    __ CompareAndBranchIfNonZero(in_reg_hi, &end);
166    __ clz(out, in_reg_lo);
167    __ AddConstant(out, 32);
168    __ Bind(&end);
169  } else {
170    __ clz(out, in.AsRegister<Register>());
171  }
172}
173
174void IntrinsicLocationsBuilderARM::VisitIntegerNumberOfLeadingZeros(HInvoke* invoke) {
175  CreateIntToIntLocations(arena_, invoke);
176}
177
178void IntrinsicCodeGeneratorARM::VisitIntegerNumberOfLeadingZeros(HInvoke* invoke) {
179  GenNumberOfLeadingZeros(invoke->GetLocations(), Primitive::kPrimInt, GetAssembler());
180}
181
182void IntrinsicLocationsBuilderARM::VisitLongNumberOfLeadingZeros(HInvoke* invoke) {
183  LocationSummary* locations = new (arena_) LocationSummary(invoke,
184                                                           LocationSummary::kNoCall,
185                                                           kIntrinsified);
186  locations->SetInAt(0, Location::RequiresRegister());
187  locations->SetOut(Location::RequiresRegister(), Location::kOutputOverlap);
188}
189
190void IntrinsicCodeGeneratorARM::VisitLongNumberOfLeadingZeros(HInvoke* invoke) {
191  GenNumberOfLeadingZeros(invoke->GetLocations(), Primitive::kPrimLong, GetAssembler());
192}
193
194static void GenNumberOfTrailingZeros(LocationSummary* locations,
195                                     Primitive::Type type,
196                                     ArmAssembler* assembler) {
197  DCHECK((type == Primitive::kPrimInt) || (type == Primitive::kPrimLong));
198
199  Register out = locations->Out().AsRegister<Register>();
200
201  if (type == Primitive::kPrimLong) {
202    Register in_reg_lo = locations->InAt(0).AsRegisterPairLow<Register>();
203    Register in_reg_hi = locations->InAt(0).AsRegisterPairHigh<Register>();
204    Label end;
205    __ rbit(out, in_reg_lo);
206    __ clz(out, out);
207    __ CompareAndBranchIfNonZero(in_reg_lo, &end);
208    __ rbit(out, in_reg_hi);
209    __ clz(out, out);
210    __ AddConstant(out, 32);
211    __ Bind(&end);
212  } else {
213    Register in = locations->InAt(0).AsRegister<Register>();
214    __ rbit(out, in);
215    __ clz(out, out);
216  }
217}
218
219void IntrinsicLocationsBuilderARM::VisitIntegerNumberOfTrailingZeros(HInvoke* invoke) {
220  LocationSummary* locations = new (arena_) LocationSummary(invoke,
221                                                            LocationSummary::kNoCall,
222                                                            kIntrinsified);
223  locations->SetInAt(0, Location::RequiresRegister());
224  locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
225}
226
227void IntrinsicCodeGeneratorARM::VisitIntegerNumberOfTrailingZeros(HInvoke* invoke) {
228  GenNumberOfTrailingZeros(invoke->GetLocations(), Primitive::kPrimInt, GetAssembler());
229}
230
231void IntrinsicLocationsBuilderARM::VisitLongNumberOfTrailingZeros(HInvoke* invoke) {
232  LocationSummary* locations = new (arena_) LocationSummary(invoke,
233                                                            LocationSummary::kNoCall,
234                                                            kIntrinsified);
235  locations->SetInAt(0, Location::RequiresRegister());
236  locations->SetOut(Location::RequiresRegister(), Location::kOutputOverlap);
237}
238
239void IntrinsicCodeGeneratorARM::VisitLongNumberOfTrailingZeros(HInvoke* invoke) {
240  GenNumberOfTrailingZeros(invoke->GetLocations(), Primitive::kPrimLong, GetAssembler());
241}
242
243static void MathAbsFP(LocationSummary* locations, bool is64bit, ArmAssembler* assembler) {
244  Location in = locations->InAt(0);
245  Location out = locations->Out();
246
247  if (is64bit) {
248    __ vabsd(FromLowSToD(out.AsFpuRegisterPairLow<SRegister>()),
249             FromLowSToD(in.AsFpuRegisterPairLow<SRegister>()));
250  } else {
251    __ vabss(out.AsFpuRegister<SRegister>(), in.AsFpuRegister<SRegister>());
252  }
253}
254
255void IntrinsicLocationsBuilderARM::VisitMathAbsDouble(HInvoke* invoke) {
256  CreateFPToFPLocations(arena_, invoke);
257}
258
259void IntrinsicCodeGeneratorARM::VisitMathAbsDouble(HInvoke* invoke) {
260  MathAbsFP(invoke->GetLocations(), /* is64bit */ true, GetAssembler());
261}
262
263void IntrinsicLocationsBuilderARM::VisitMathAbsFloat(HInvoke* invoke) {
264  CreateFPToFPLocations(arena_, invoke);
265}
266
267void IntrinsicCodeGeneratorARM::VisitMathAbsFloat(HInvoke* invoke) {
268  MathAbsFP(invoke->GetLocations(), /* is64bit */ false, GetAssembler());
269}
270
271static void CreateIntToIntPlusTemp(ArenaAllocator* arena, HInvoke* invoke) {
272  LocationSummary* locations = new (arena) LocationSummary(invoke,
273                                                           LocationSummary::kNoCall,
274                                                           kIntrinsified);
275  locations->SetInAt(0, Location::RequiresRegister());
276  locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
277
278  locations->AddTemp(Location::RequiresRegister());
279}
280
281static void GenAbsInteger(LocationSummary* locations,
282                          bool is64bit,
283                          ArmAssembler* assembler) {
284  Location in = locations->InAt(0);
285  Location output = locations->Out();
286
287  Register mask = locations->GetTemp(0).AsRegister<Register>();
288
289  if (is64bit) {
290    Register in_reg_lo = in.AsRegisterPairLow<Register>();
291    Register in_reg_hi = in.AsRegisterPairHigh<Register>();
292    Register out_reg_lo = output.AsRegisterPairLow<Register>();
293    Register out_reg_hi = output.AsRegisterPairHigh<Register>();
294
295    DCHECK_NE(out_reg_lo, in_reg_hi) << "Diagonal overlap unexpected.";
296
297    __ Asr(mask, in_reg_hi, 31);
298    __ adds(out_reg_lo, in_reg_lo, ShifterOperand(mask));
299    __ adc(out_reg_hi, in_reg_hi, ShifterOperand(mask));
300    __ eor(out_reg_lo, mask, ShifterOperand(out_reg_lo));
301    __ eor(out_reg_hi, mask, ShifterOperand(out_reg_hi));
302  } else {
303    Register in_reg = in.AsRegister<Register>();
304    Register out_reg = output.AsRegister<Register>();
305
306    __ Asr(mask, in_reg, 31);
307    __ add(out_reg, in_reg, ShifterOperand(mask));
308    __ eor(out_reg, mask, ShifterOperand(out_reg));
309  }
310}
311
312void IntrinsicLocationsBuilderARM::VisitMathAbsInt(HInvoke* invoke) {
313  CreateIntToIntPlusTemp(arena_, invoke);
314}
315
316void IntrinsicCodeGeneratorARM::VisitMathAbsInt(HInvoke* invoke) {
317  GenAbsInteger(invoke->GetLocations(), /* is64bit */ false, GetAssembler());
318}
319
320
321void IntrinsicLocationsBuilderARM::VisitMathAbsLong(HInvoke* invoke) {
322  CreateIntToIntPlusTemp(arena_, invoke);
323}
324
325void IntrinsicCodeGeneratorARM::VisitMathAbsLong(HInvoke* invoke) {
326  GenAbsInteger(invoke->GetLocations(), /* is64bit */ true, GetAssembler());
327}
328
329static void GenMinMax(LocationSummary* locations,
330                      bool is_min,
331                      ArmAssembler* assembler) {
332  Register op1 = locations->InAt(0).AsRegister<Register>();
333  Register op2 = locations->InAt(1).AsRegister<Register>();
334  Register out = locations->Out().AsRegister<Register>();
335
336  __ cmp(op1, ShifterOperand(op2));
337
338  __ it((is_min) ? Condition::LT : Condition::GT, kItElse);
339  __ mov(out, ShifterOperand(op1), is_min ? Condition::LT : Condition::GT);
340  __ mov(out, ShifterOperand(op2), is_min ? Condition::GE : Condition::LE);
341}
342
343static void CreateIntIntToIntLocations(ArenaAllocator* arena, HInvoke* invoke) {
344  LocationSummary* locations = new (arena) LocationSummary(invoke,
345                                                           LocationSummary::kNoCall,
346                                                           kIntrinsified);
347  locations->SetInAt(0, Location::RequiresRegister());
348  locations->SetInAt(1, Location::RequiresRegister());
349  locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
350}
351
352void IntrinsicLocationsBuilderARM::VisitMathMinIntInt(HInvoke* invoke) {
353  CreateIntIntToIntLocations(arena_, invoke);
354}
355
356void IntrinsicCodeGeneratorARM::VisitMathMinIntInt(HInvoke* invoke) {
357  GenMinMax(invoke->GetLocations(), /* is_min */ true, GetAssembler());
358}
359
360void IntrinsicLocationsBuilderARM::VisitMathMaxIntInt(HInvoke* invoke) {
361  CreateIntIntToIntLocations(arena_, invoke);
362}
363
364void IntrinsicCodeGeneratorARM::VisitMathMaxIntInt(HInvoke* invoke) {
365  GenMinMax(invoke->GetLocations(), /* is_min */ false, GetAssembler());
366}
367
368void IntrinsicLocationsBuilderARM::VisitMathSqrt(HInvoke* invoke) {
369  CreateFPToFPLocations(arena_, invoke);
370}
371
372void IntrinsicCodeGeneratorARM::VisitMathSqrt(HInvoke* invoke) {
373  LocationSummary* locations = invoke->GetLocations();
374  ArmAssembler* assembler = GetAssembler();
375  __ vsqrtd(FromLowSToD(locations->Out().AsFpuRegisterPairLow<SRegister>()),
376            FromLowSToD(locations->InAt(0).AsFpuRegisterPairLow<SRegister>()));
377}
378
379void IntrinsicLocationsBuilderARM::VisitMemoryPeekByte(HInvoke* invoke) {
380  CreateIntToIntLocations(arena_, invoke);
381}
382
383void IntrinsicCodeGeneratorARM::VisitMemoryPeekByte(HInvoke* invoke) {
384  ArmAssembler* assembler = GetAssembler();
385  // Ignore upper 4B of long address.
386  __ ldrsb(invoke->GetLocations()->Out().AsRegister<Register>(),
387           Address(invoke->GetLocations()->InAt(0).AsRegisterPairLow<Register>()));
388}
389
390void IntrinsicLocationsBuilderARM::VisitMemoryPeekIntNative(HInvoke* invoke) {
391  CreateIntToIntLocations(arena_, invoke);
392}
393
394void IntrinsicCodeGeneratorARM::VisitMemoryPeekIntNative(HInvoke* invoke) {
395  ArmAssembler* assembler = GetAssembler();
396  // Ignore upper 4B of long address.
397  __ ldr(invoke->GetLocations()->Out().AsRegister<Register>(),
398         Address(invoke->GetLocations()->InAt(0).AsRegisterPairLow<Register>()));
399}
400
401void IntrinsicLocationsBuilderARM::VisitMemoryPeekLongNative(HInvoke* invoke) {
402  CreateIntToIntLocations(arena_, invoke);
403}
404
405void IntrinsicCodeGeneratorARM::VisitMemoryPeekLongNative(HInvoke* invoke) {
406  ArmAssembler* assembler = GetAssembler();
407  // Ignore upper 4B of long address.
408  Register addr = invoke->GetLocations()->InAt(0).AsRegisterPairLow<Register>();
409  // Worst case: Control register bit SCTLR.A = 0. Then unaligned accesses throw a processor
410  // exception. So we can't use ldrd as addr may be unaligned.
411  Register lo = invoke->GetLocations()->Out().AsRegisterPairLow<Register>();
412  Register hi = invoke->GetLocations()->Out().AsRegisterPairHigh<Register>();
413  if (addr == lo) {
414    __ ldr(hi, Address(addr, 4));
415    __ ldr(lo, Address(addr, 0));
416  } else {
417    __ ldr(lo, Address(addr, 0));
418    __ ldr(hi, Address(addr, 4));
419  }
420}
421
422void IntrinsicLocationsBuilderARM::VisitMemoryPeekShortNative(HInvoke* invoke) {
423  CreateIntToIntLocations(arena_, invoke);
424}
425
426void IntrinsicCodeGeneratorARM::VisitMemoryPeekShortNative(HInvoke* invoke) {
427  ArmAssembler* assembler = GetAssembler();
428  // Ignore upper 4B of long address.
429  __ ldrsh(invoke->GetLocations()->Out().AsRegister<Register>(),
430           Address(invoke->GetLocations()->InAt(0).AsRegisterPairLow<Register>()));
431}
432
433static void CreateIntIntToVoidLocations(ArenaAllocator* arena, HInvoke* invoke) {
434  LocationSummary* locations = new (arena) LocationSummary(invoke,
435                                                           LocationSummary::kNoCall,
436                                                           kIntrinsified);
437  locations->SetInAt(0, Location::RequiresRegister());
438  locations->SetInAt(1, Location::RequiresRegister());
439}
440
441void IntrinsicLocationsBuilderARM::VisitMemoryPokeByte(HInvoke* invoke) {
442  CreateIntIntToVoidLocations(arena_, invoke);
443}
444
445void IntrinsicCodeGeneratorARM::VisitMemoryPokeByte(HInvoke* invoke) {
446  ArmAssembler* assembler = GetAssembler();
447  __ strb(invoke->GetLocations()->InAt(1).AsRegister<Register>(),
448          Address(invoke->GetLocations()->InAt(0).AsRegisterPairLow<Register>()));
449}
450
451void IntrinsicLocationsBuilderARM::VisitMemoryPokeIntNative(HInvoke* invoke) {
452  CreateIntIntToVoidLocations(arena_, invoke);
453}
454
455void IntrinsicCodeGeneratorARM::VisitMemoryPokeIntNative(HInvoke* invoke) {
456  ArmAssembler* assembler = GetAssembler();
457  __ str(invoke->GetLocations()->InAt(1).AsRegister<Register>(),
458         Address(invoke->GetLocations()->InAt(0).AsRegisterPairLow<Register>()));
459}
460
461void IntrinsicLocationsBuilderARM::VisitMemoryPokeLongNative(HInvoke* invoke) {
462  CreateIntIntToVoidLocations(arena_, invoke);
463}
464
465void IntrinsicCodeGeneratorARM::VisitMemoryPokeLongNative(HInvoke* invoke) {
466  ArmAssembler* assembler = GetAssembler();
467  // Ignore upper 4B of long address.
468  Register addr = invoke->GetLocations()->InAt(0).AsRegisterPairLow<Register>();
469  // Worst case: Control register bit SCTLR.A = 0. Then unaligned accesses throw a processor
470  // exception. So we can't use ldrd as addr may be unaligned.
471  __ str(invoke->GetLocations()->InAt(1).AsRegisterPairLow<Register>(), Address(addr, 0));
472  __ str(invoke->GetLocations()->InAt(1).AsRegisterPairHigh<Register>(), Address(addr, 4));
473}
474
475void IntrinsicLocationsBuilderARM::VisitMemoryPokeShortNative(HInvoke* invoke) {
476  CreateIntIntToVoidLocations(arena_, invoke);
477}
478
479void IntrinsicCodeGeneratorARM::VisitMemoryPokeShortNative(HInvoke* invoke) {
480  ArmAssembler* assembler = GetAssembler();
481  __ strh(invoke->GetLocations()->InAt(1).AsRegister<Register>(),
482          Address(invoke->GetLocations()->InAt(0).AsRegisterPairLow<Register>()));
483}
484
485void IntrinsicLocationsBuilderARM::VisitThreadCurrentThread(HInvoke* invoke) {
486  LocationSummary* locations = new (arena_) LocationSummary(invoke,
487                                                            LocationSummary::kNoCall,
488                                                            kIntrinsified);
489  locations->SetOut(Location::RequiresRegister());
490}
491
492void IntrinsicCodeGeneratorARM::VisitThreadCurrentThread(HInvoke* invoke) {
493  ArmAssembler* assembler = GetAssembler();
494  __ LoadFromOffset(kLoadWord,
495                    invoke->GetLocations()->Out().AsRegister<Register>(),
496                    TR,
497                    Thread::PeerOffset<kArmPointerSize>().Int32Value());
498}
499
500static void GenUnsafeGet(HInvoke* invoke,
501                         Primitive::Type type,
502                         bool is_volatile,
503                         CodeGeneratorARM* codegen) {
504  LocationSummary* locations = invoke->GetLocations();
505  ArmAssembler* assembler = codegen->GetAssembler();
506  Location base_loc = locations->InAt(1);
507  Register base = base_loc.AsRegister<Register>();             // Object pointer.
508  Location offset_loc = locations->InAt(2);
509  Register offset = offset_loc.AsRegisterPairLow<Register>();  // Long offset, lo part only.
510  Location trg_loc = locations->Out();
511
512  switch (type) {
513    case Primitive::kPrimInt: {
514      Register trg = trg_loc.AsRegister<Register>();
515      __ ldr(trg, Address(base, offset));
516      if (is_volatile) {
517        __ dmb(ISH);
518      }
519      break;
520    }
521
522    case Primitive::kPrimNot: {
523      Register trg = trg_loc.AsRegister<Register>();
524      if (kEmitCompilerReadBarrier) {
525        if (kUseBakerReadBarrier) {
526          Location temp = locations->GetTemp(0);
527          codegen->GenerateArrayLoadWithBakerReadBarrier(
528              invoke, trg_loc, base, 0U, offset_loc, temp, /* needs_null_check */ false);
529          if (is_volatile) {
530            __ dmb(ISH);
531          }
532        } else {
533          __ ldr(trg, Address(base, offset));
534          if (is_volatile) {
535            __ dmb(ISH);
536          }
537          codegen->GenerateReadBarrierSlow(invoke, trg_loc, trg_loc, base_loc, 0U, offset_loc);
538        }
539      } else {
540        __ ldr(trg, Address(base, offset));
541        if (is_volatile) {
542          __ dmb(ISH);
543        }
544        __ MaybeUnpoisonHeapReference(trg);
545      }
546      break;
547    }
548
549    case Primitive::kPrimLong: {
550      Register trg_lo = trg_loc.AsRegisterPairLow<Register>();
551      __ add(IP, base, ShifterOperand(offset));
552      if (is_volatile && !codegen->GetInstructionSetFeatures().HasAtomicLdrdAndStrd()) {
553        Register trg_hi = trg_loc.AsRegisterPairHigh<Register>();
554        __ ldrexd(trg_lo, trg_hi, IP);
555      } else {
556        __ ldrd(trg_lo, Address(IP));
557      }
558      if (is_volatile) {
559        __ dmb(ISH);
560      }
561      break;
562    }
563
564    default:
565      LOG(FATAL) << "Unexpected type " << type;
566      UNREACHABLE();
567  }
568}
569
570static void CreateIntIntIntToIntLocations(ArenaAllocator* arena,
571                                          HInvoke* invoke,
572                                          Primitive::Type type) {
573  bool can_call = kEmitCompilerReadBarrier &&
574      (invoke->GetIntrinsic() == Intrinsics::kUnsafeGetObject ||
575       invoke->GetIntrinsic() == Intrinsics::kUnsafeGetObjectVolatile);
576  LocationSummary* locations = new (arena) LocationSummary(invoke,
577                                                           can_call ?
578                                                               LocationSummary::kCallOnSlowPath :
579                                                               LocationSummary::kNoCall,
580                                                           kIntrinsified);
581  locations->SetInAt(0, Location::NoLocation());        // Unused receiver.
582  locations->SetInAt(1, Location::RequiresRegister());
583  locations->SetInAt(2, Location::RequiresRegister());
584  locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
585  if (type == Primitive::kPrimNot && kEmitCompilerReadBarrier && kUseBakerReadBarrier) {
586    // We need a temporary register for the read barrier marking slow
587    // path in InstructionCodeGeneratorARM::GenerateArrayLoadWithBakerReadBarrier.
588    locations->AddTemp(Location::RequiresRegister());
589  }
590}
591
592void IntrinsicLocationsBuilderARM::VisitUnsafeGet(HInvoke* invoke) {
593  CreateIntIntIntToIntLocations(arena_, invoke, Primitive::kPrimInt);
594}
595void IntrinsicLocationsBuilderARM::VisitUnsafeGetVolatile(HInvoke* invoke) {
596  CreateIntIntIntToIntLocations(arena_, invoke, Primitive::kPrimInt);
597}
598void IntrinsicLocationsBuilderARM::VisitUnsafeGetLong(HInvoke* invoke) {
599  CreateIntIntIntToIntLocations(arena_, invoke, Primitive::kPrimLong);
600}
601void IntrinsicLocationsBuilderARM::VisitUnsafeGetLongVolatile(HInvoke* invoke) {
602  CreateIntIntIntToIntLocations(arena_, invoke, Primitive::kPrimLong);
603}
604void IntrinsicLocationsBuilderARM::VisitUnsafeGetObject(HInvoke* invoke) {
605  CreateIntIntIntToIntLocations(arena_, invoke, Primitive::kPrimNot);
606}
607void IntrinsicLocationsBuilderARM::VisitUnsafeGetObjectVolatile(HInvoke* invoke) {
608  CreateIntIntIntToIntLocations(arena_, invoke, Primitive::kPrimNot);
609}
610
611void IntrinsicCodeGeneratorARM::VisitUnsafeGet(HInvoke* invoke) {
612  GenUnsafeGet(invoke, Primitive::kPrimInt, /* is_volatile */ false, codegen_);
613}
614void IntrinsicCodeGeneratorARM::VisitUnsafeGetVolatile(HInvoke* invoke) {
615  GenUnsafeGet(invoke, Primitive::kPrimInt, /* is_volatile */ true, codegen_);
616}
617void IntrinsicCodeGeneratorARM::VisitUnsafeGetLong(HInvoke* invoke) {
618  GenUnsafeGet(invoke, Primitive::kPrimLong, /* is_volatile */ false, codegen_);
619}
620void IntrinsicCodeGeneratorARM::VisitUnsafeGetLongVolatile(HInvoke* invoke) {
621  GenUnsafeGet(invoke, Primitive::kPrimLong, /* is_volatile */ true, codegen_);
622}
623void IntrinsicCodeGeneratorARM::VisitUnsafeGetObject(HInvoke* invoke) {
624  GenUnsafeGet(invoke, Primitive::kPrimNot, /* is_volatile */ false, codegen_);
625}
626void IntrinsicCodeGeneratorARM::VisitUnsafeGetObjectVolatile(HInvoke* invoke) {
627  GenUnsafeGet(invoke, Primitive::kPrimNot, /* is_volatile */ true, codegen_);
628}
629
630static void CreateIntIntIntIntToVoid(ArenaAllocator* arena,
631                                     const ArmInstructionSetFeatures& features,
632                                     Primitive::Type type,
633                                     bool is_volatile,
634                                     HInvoke* invoke) {
635  LocationSummary* locations = new (arena) LocationSummary(invoke,
636                                                           LocationSummary::kNoCall,
637                                                           kIntrinsified);
638  locations->SetInAt(0, Location::NoLocation());        // Unused receiver.
639  locations->SetInAt(1, Location::RequiresRegister());
640  locations->SetInAt(2, Location::RequiresRegister());
641  locations->SetInAt(3, Location::RequiresRegister());
642
643  if (type == Primitive::kPrimLong) {
644    // Potentially need temps for ldrexd-strexd loop.
645    if (is_volatile && !features.HasAtomicLdrdAndStrd()) {
646      locations->AddTemp(Location::RequiresRegister());  // Temp_lo.
647      locations->AddTemp(Location::RequiresRegister());  // Temp_hi.
648    }
649  } else if (type == Primitive::kPrimNot) {
650    // Temps for card-marking.
651    locations->AddTemp(Location::RequiresRegister());  // Temp.
652    locations->AddTemp(Location::RequiresRegister());  // Card.
653  }
654}
655
656void IntrinsicLocationsBuilderARM::VisitUnsafePut(HInvoke* invoke) {
657  CreateIntIntIntIntToVoid(arena_, features_, Primitive::kPrimInt, /* is_volatile */ false, invoke);
658}
659void IntrinsicLocationsBuilderARM::VisitUnsafePutOrdered(HInvoke* invoke) {
660  CreateIntIntIntIntToVoid(arena_, features_, Primitive::kPrimInt, /* is_volatile */ false, invoke);
661}
662void IntrinsicLocationsBuilderARM::VisitUnsafePutVolatile(HInvoke* invoke) {
663  CreateIntIntIntIntToVoid(arena_, features_, Primitive::kPrimInt, /* is_volatile */ true, invoke);
664}
665void IntrinsicLocationsBuilderARM::VisitUnsafePutObject(HInvoke* invoke) {
666  CreateIntIntIntIntToVoid(arena_, features_, Primitive::kPrimNot, /* is_volatile */ false, invoke);
667}
668void IntrinsicLocationsBuilderARM::VisitUnsafePutObjectOrdered(HInvoke* invoke) {
669  CreateIntIntIntIntToVoid(arena_, features_, Primitive::kPrimNot, /* is_volatile */ false, invoke);
670}
671void IntrinsicLocationsBuilderARM::VisitUnsafePutObjectVolatile(HInvoke* invoke) {
672  CreateIntIntIntIntToVoid(arena_, features_, Primitive::kPrimNot, /* is_volatile */ true, invoke);
673}
674void IntrinsicLocationsBuilderARM::VisitUnsafePutLong(HInvoke* invoke) {
675  CreateIntIntIntIntToVoid(
676      arena_, features_, Primitive::kPrimLong, /* is_volatile */ false, invoke);
677}
678void IntrinsicLocationsBuilderARM::VisitUnsafePutLongOrdered(HInvoke* invoke) {
679  CreateIntIntIntIntToVoid(
680      arena_, features_, Primitive::kPrimLong, /* is_volatile */ false, invoke);
681}
682void IntrinsicLocationsBuilderARM::VisitUnsafePutLongVolatile(HInvoke* invoke) {
683  CreateIntIntIntIntToVoid(
684      arena_, features_, Primitive::kPrimLong, /* is_volatile */ true, invoke);
685}
686
687static void GenUnsafePut(LocationSummary* locations,
688                         Primitive::Type type,
689                         bool is_volatile,
690                         bool is_ordered,
691                         CodeGeneratorARM* codegen) {
692  ArmAssembler* assembler = codegen->GetAssembler();
693
694  Register base = locations->InAt(1).AsRegister<Register>();           // Object pointer.
695  Register offset = locations->InAt(2).AsRegisterPairLow<Register>();  // Long offset, lo part only.
696  Register value;
697
698  if (is_volatile || is_ordered) {
699    __ dmb(ISH);
700  }
701
702  if (type == Primitive::kPrimLong) {
703    Register value_lo = locations->InAt(3).AsRegisterPairLow<Register>();
704    value = value_lo;
705    if (is_volatile && !codegen->GetInstructionSetFeatures().HasAtomicLdrdAndStrd()) {
706      Register temp_lo = locations->GetTemp(0).AsRegister<Register>();
707      Register temp_hi = locations->GetTemp(1).AsRegister<Register>();
708      Register value_hi = locations->InAt(3).AsRegisterPairHigh<Register>();
709
710      __ add(IP, base, ShifterOperand(offset));
711      Label loop_head;
712      __ Bind(&loop_head);
713      __ ldrexd(temp_lo, temp_hi, IP);
714      __ strexd(temp_lo, value_lo, value_hi, IP);
715      __ cmp(temp_lo, ShifterOperand(0));
716      __ b(&loop_head, NE);
717    } else {
718      __ add(IP, base, ShifterOperand(offset));
719      __ strd(value_lo, Address(IP));
720    }
721  } else {
722    value = locations->InAt(3).AsRegister<Register>();
723    Register source = value;
724    if (kPoisonHeapReferences && type == Primitive::kPrimNot) {
725      Register temp = locations->GetTemp(0).AsRegister<Register>();
726      __ Mov(temp, value);
727      __ PoisonHeapReference(temp);
728      source = temp;
729    }
730    __ str(source, Address(base, offset));
731  }
732
733  if (is_volatile) {
734    __ dmb(ISH);
735  }
736
737  if (type == Primitive::kPrimNot) {
738    Register temp = locations->GetTemp(0).AsRegister<Register>();
739    Register card = locations->GetTemp(1).AsRegister<Register>();
740    bool value_can_be_null = true;  // TODO: Worth finding out this information?
741    codegen->MarkGCCard(temp, card, base, value, value_can_be_null);
742  }
743}
744
745void IntrinsicCodeGeneratorARM::VisitUnsafePut(HInvoke* invoke) {
746  GenUnsafePut(invoke->GetLocations(),
747               Primitive::kPrimInt,
748               /* is_volatile */ false,
749               /* is_ordered */ false,
750               codegen_);
751}
752void IntrinsicCodeGeneratorARM::VisitUnsafePutOrdered(HInvoke* invoke) {
753  GenUnsafePut(invoke->GetLocations(),
754               Primitive::kPrimInt,
755               /* is_volatile */ false,
756               /* is_ordered */ true,
757               codegen_);
758}
759void IntrinsicCodeGeneratorARM::VisitUnsafePutVolatile(HInvoke* invoke) {
760  GenUnsafePut(invoke->GetLocations(),
761               Primitive::kPrimInt,
762               /* is_volatile */ true,
763               /* is_ordered */ false,
764               codegen_);
765}
766void IntrinsicCodeGeneratorARM::VisitUnsafePutObject(HInvoke* invoke) {
767  GenUnsafePut(invoke->GetLocations(),
768               Primitive::kPrimNot,
769               /* is_volatile */ false,
770               /* is_ordered */ false,
771               codegen_);
772}
773void IntrinsicCodeGeneratorARM::VisitUnsafePutObjectOrdered(HInvoke* invoke) {
774  GenUnsafePut(invoke->GetLocations(),
775               Primitive::kPrimNot,
776               /* is_volatile */ false,
777               /* is_ordered */ true,
778               codegen_);
779}
780void IntrinsicCodeGeneratorARM::VisitUnsafePutObjectVolatile(HInvoke* invoke) {
781  GenUnsafePut(invoke->GetLocations(),
782               Primitive::kPrimNot,
783               /* is_volatile */ true,
784               /* is_ordered */ false,
785               codegen_);
786}
787void IntrinsicCodeGeneratorARM::VisitUnsafePutLong(HInvoke* invoke) {
788  GenUnsafePut(invoke->GetLocations(),
789               Primitive::kPrimLong,
790               /* is_volatile */ false,
791               /* is_ordered */ false,
792               codegen_);
793}
794void IntrinsicCodeGeneratorARM::VisitUnsafePutLongOrdered(HInvoke* invoke) {
795  GenUnsafePut(invoke->GetLocations(),
796               Primitive::kPrimLong,
797               /* is_volatile */ false,
798               /* is_ordered */ true,
799               codegen_);
800}
801void IntrinsicCodeGeneratorARM::VisitUnsafePutLongVolatile(HInvoke* invoke) {
802  GenUnsafePut(invoke->GetLocations(),
803               Primitive::kPrimLong,
804               /* is_volatile */ true,
805               /* is_ordered */ false,
806               codegen_);
807}
808
809static void CreateIntIntIntIntIntToIntPlusTemps(ArenaAllocator* arena,
810                                                HInvoke* invoke,
811                                                Primitive::Type type) {
812  LocationSummary* locations = new (arena) LocationSummary(invoke,
813                                                           LocationSummary::kNoCall,
814                                                           kIntrinsified);
815  locations->SetInAt(0, Location::NoLocation());        // Unused receiver.
816  locations->SetInAt(1, Location::RequiresRegister());
817  locations->SetInAt(2, Location::RequiresRegister());
818  locations->SetInAt(3, Location::RequiresRegister());
819  locations->SetInAt(4, Location::RequiresRegister());
820
821  // If heap poisoning is enabled, we don't want the unpoisoning
822  // operations to potentially clobber the output.
823  Location::OutputOverlap overlaps = (kPoisonHeapReferences && type == Primitive::kPrimNot)
824      ? Location::kOutputOverlap
825      : Location::kNoOutputOverlap;
826  locations->SetOut(Location::RequiresRegister(), overlaps);
827
828  locations->AddTemp(Location::RequiresRegister());  // Pointer.
829  locations->AddTemp(Location::RequiresRegister());  // Temp 1.
830}
831
832static void GenCas(LocationSummary* locations, Primitive::Type type, CodeGeneratorARM* codegen) {
833  DCHECK_NE(type, Primitive::kPrimLong);
834
835  ArmAssembler* assembler = codegen->GetAssembler();
836
837  Register out = locations->Out().AsRegister<Register>();              // Boolean result.
838
839  Register base = locations->InAt(1).AsRegister<Register>();           // Object pointer.
840  Register offset = locations->InAt(2).AsRegisterPairLow<Register>();  // Offset (discard high 4B).
841  Register expected_lo = locations->InAt(3).AsRegister<Register>();    // Expected.
842  Register value_lo = locations->InAt(4).AsRegister<Register>();       // Value.
843
844  Register tmp_ptr = locations->GetTemp(0).AsRegister<Register>();     // Pointer to actual memory.
845  Register tmp_lo = locations->GetTemp(1).AsRegister<Register>();      // Value in memory.
846
847  if (type == Primitive::kPrimNot) {
848    // Mark card for object assuming new value is stored. Worst case we will mark an unchanged
849    // object and scan the receiver at the next GC for nothing.
850    bool value_can_be_null = true;  // TODO: Worth finding out this information?
851    codegen->MarkGCCard(tmp_ptr, tmp_lo, base, value_lo, value_can_be_null);
852  }
853
854  // Prevent reordering with prior memory operations.
855  // Emit a DMB ISH instruction instead of an DMB ISHST one, as the
856  // latter allows a preceding load to be delayed past the STXR
857  // instruction below.
858  __ dmb(ISH);
859
860  __ add(tmp_ptr, base, ShifterOperand(offset));
861
862  if (kPoisonHeapReferences && type == Primitive::kPrimNot) {
863    codegen->GetAssembler()->PoisonHeapReference(expected_lo);
864    if (value_lo == expected_lo) {
865      // Do not poison `value_lo`, as it is the same register as
866      // `expected_lo`, which has just been poisoned.
867    } else {
868      codegen->GetAssembler()->PoisonHeapReference(value_lo);
869    }
870  }
871
872  // do {
873  //   tmp = [r_ptr] - expected;
874  // } while (tmp == 0 && failure([r_ptr] <- r_new_value));
875  // result = tmp != 0;
876
877  Label loop_head;
878  __ Bind(&loop_head);
879
880  // TODO: When `type == Primitive::kPrimNot`, add a read barrier for
881  // the reference stored in the object before attempting the CAS,
882  // similar to the one in the art::Unsafe_compareAndSwapObject JNI
883  // implementation.
884  //
885  // Note that this code is not (yet) used when read barriers are
886  // enabled (see IntrinsicLocationsBuilderARM::VisitUnsafeCASObject).
887  DCHECK(!(type == Primitive::kPrimNot && kEmitCompilerReadBarrier));
888  __ ldrex(tmp_lo, tmp_ptr);
889
890  __ subs(tmp_lo, tmp_lo, ShifterOperand(expected_lo));
891
892  __ it(EQ, ItState::kItT);
893  __ strex(tmp_lo, value_lo, tmp_ptr, EQ);
894  __ cmp(tmp_lo, ShifterOperand(1), EQ);
895
896  __ b(&loop_head, EQ);
897
898  __ dmb(ISH);
899
900  __ rsbs(out, tmp_lo, ShifterOperand(1));
901  __ it(CC);
902  __ mov(out, ShifterOperand(0), CC);
903
904  if (kPoisonHeapReferences && type == Primitive::kPrimNot) {
905    codegen->GetAssembler()->UnpoisonHeapReference(expected_lo);
906    if (value_lo == expected_lo) {
907      // Do not unpoison `value_lo`, as it is the same register as
908      // `expected_lo`, which has just been unpoisoned.
909    } else {
910      codegen->GetAssembler()->UnpoisonHeapReference(value_lo);
911    }
912  }
913}
914
915void IntrinsicLocationsBuilderARM::VisitUnsafeCASInt(HInvoke* invoke) {
916  CreateIntIntIntIntIntToIntPlusTemps(arena_, invoke, Primitive::kPrimInt);
917}
918void IntrinsicLocationsBuilderARM::VisitUnsafeCASObject(HInvoke* invoke) {
919  // The UnsafeCASObject intrinsic is missing a read barrier, and
920  // therefore sometimes does not work as expected (b/25883050).
921  // Turn it off temporarily as a quick fix, until the read barrier is
922  // implemented (see TODO in GenCAS below).
923  //
924  // TODO(rpl): Fix this issue and re-enable this intrinsic with read barriers.
925  if (kEmitCompilerReadBarrier) {
926    return;
927  }
928
929  CreateIntIntIntIntIntToIntPlusTemps(arena_, invoke, Primitive::kPrimNot);
930}
931void IntrinsicCodeGeneratorARM::VisitUnsafeCASInt(HInvoke* invoke) {
932  GenCas(invoke->GetLocations(), Primitive::kPrimInt, codegen_);
933}
934void IntrinsicCodeGeneratorARM::VisitUnsafeCASObject(HInvoke* invoke) {
935  GenCas(invoke->GetLocations(), Primitive::kPrimNot, codegen_);
936}
937
938void IntrinsicLocationsBuilderARM::VisitStringCharAt(HInvoke* invoke) {
939  LocationSummary* locations = new (arena_) LocationSummary(invoke,
940                                                            LocationSummary::kCallOnSlowPath,
941                                                            kIntrinsified);
942  locations->SetInAt(0, Location::RequiresRegister());
943  locations->SetInAt(1, Location::RequiresRegister());
944  locations->SetOut(Location::RequiresRegister(), Location::kOutputOverlap);
945
946  locations->AddTemp(Location::RequiresRegister());
947  locations->AddTemp(Location::RequiresRegister());
948}
949
950void IntrinsicCodeGeneratorARM::VisitStringCharAt(HInvoke* invoke) {
951  ArmAssembler* assembler = GetAssembler();
952  LocationSummary* locations = invoke->GetLocations();
953
954  // Location of reference to data array
955  const MemberOffset value_offset = mirror::String::ValueOffset();
956  // Location of count
957  const MemberOffset count_offset = mirror::String::CountOffset();
958
959  Register obj = locations->InAt(0).AsRegister<Register>();  // String object pointer.
960  Register idx = locations->InAt(1).AsRegister<Register>();  // Index of character.
961  Register out = locations->Out().AsRegister<Register>();    // Result character.
962
963  Register temp = locations->GetTemp(0).AsRegister<Register>();
964  Register array_temp = locations->GetTemp(1).AsRegister<Register>();
965
966  // TODO: Maybe we can support range check elimination. Overall, though, I think it's not worth
967  //       the cost.
968  // TODO: For simplicity, the index parameter is requested in a register, so different from Quick
969  //       we will not optimize the code for constants (which would save a register).
970
971  SlowPathCode* slow_path = new (GetAllocator()) IntrinsicSlowPathARM(invoke);
972  codegen_->AddSlowPath(slow_path);
973
974  __ ldr(temp, Address(obj, count_offset.Int32Value()));          // temp = str.length.
975  codegen_->MaybeRecordImplicitNullCheck(invoke);
976  __ cmp(idx, ShifterOperand(temp));
977  __ b(slow_path->GetEntryLabel(), CS);
978
979  __ add(array_temp, obj, ShifterOperand(value_offset.Int32Value()));  // array_temp := str.value.
980
981  // Load the value.
982  __ ldrh(out, Address(array_temp, idx, LSL, 1));                 // out := array_temp[idx].
983
984  __ Bind(slow_path->GetExitLabel());
985}
986
987void IntrinsicLocationsBuilderARM::VisitStringCompareTo(HInvoke* invoke) {
988  // The inputs plus one temp.
989  LocationSummary* locations = new (arena_) LocationSummary(invoke,
990                                                            LocationSummary::kCall,
991                                                            kIntrinsified);
992  InvokeRuntimeCallingConvention calling_convention;
993  locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(0)));
994  locations->SetInAt(1, Location::RegisterLocation(calling_convention.GetRegisterAt(1)));
995  locations->SetOut(Location::RegisterLocation(R0));
996}
997
998void IntrinsicCodeGeneratorARM::VisitStringCompareTo(HInvoke* invoke) {
999  ArmAssembler* assembler = GetAssembler();
1000  LocationSummary* locations = invoke->GetLocations();
1001
1002  // Note that the null check must have been done earlier.
1003  DCHECK(!invoke->CanDoImplicitNullCheckOn(invoke->InputAt(0)));
1004
1005  Register argument = locations->InAt(1).AsRegister<Register>();
1006  __ cmp(argument, ShifterOperand(0));
1007  SlowPathCode* slow_path = new (GetAllocator()) IntrinsicSlowPathARM(invoke);
1008  codegen_->AddSlowPath(slow_path);
1009  __ b(slow_path->GetEntryLabel(), EQ);
1010
1011  __ LoadFromOffset(
1012      kLoadWord, LR, TR, QUICK_ENTRYPOINT_OFFSET(kArmWordSize, pStringCompareTo).Int32Value());
1013  __ blx(LR);
1014  __ Bind(slow_path->GetExitLabel());
1015}
1016
1017void IntrinsicLocationsBuilderARM::VisitStringEquals(HInvoke* invoke) {
1018  LocationSummary* locations = new (arena_) LocationSummary(invoke,
1019                                                            LocationSummary::kNoCall,
1020                                                            kIntrinsified);
1021  InvokeRuntimeCallingConvention calling_convention;
1022  locations->SetInAt(0, Location::RequiresRegister());
1023  locations->SetInAt(1, Location::RequiresRegister());
1024  // Temporary registers to store lengths of strings and for calculations.
1025  // Using instruction cbz requires a low register, so explicitly set a temp to be R0.
1026  locations->AddTemp(Location::RegisterLocation(R0));
1027  locations->AddTemp(Location::RequiresRegister());
1028  locations->AddTemp(Location::RequiresRegister());
1029
1030  locations->SetOut(Location::RequiresRegister());
1031}
1032
1033void IntrinsicCodeGeneratorARM::VisitStringEquals(HInvoke* invoke) {
1034  ArmAssembler* assembler = GetAssembler();
1035  LocationSummary* locations = invoke->GetLocations();
1036
1037  Register str = locations->InAt(0).AsRegister<Register>();
1038  Register arg = locations->InAt(1).AsRegister<Register>();
1039  Register out = locations->Out().AsRegister<Register>();
1040
1041  Register temp = locations->GetTemp(0).AsRegister<Register>();
1042  Register temp1 = locations->GetTemp(1).AsRegister<Register>();
1043  Register temp2 = locations->GetTemp(2).AsRegister<Register>();
1044
1045  Label loop;
1046  Label end;
1047  Label return_true;
1048  Label return_false;
1049
1050  // Get offsets of count, value, and class fields within a string object.
1051  const uint32_t count_offset = mirror::String::CountOffset().Uint32Value();
1052  const uint32_t value_offset = mirror::String::ValueOffset().Uint32Value();
1053  const uint32_t class_offset = mirror::Object::ClassOffset().Uint32Value();
1054
1055  // Note that the null check must have been done earlier.
1056  DCHECK(!invoke->CanDoImplicitNullCheckOn(invoke->InputAt(0)));
1057
1058  // Check if input is null, return false if it is.
1059  __ CompareAndBranchIfZero(arg, &return_false);
1060
1061  // Instanceof check for the argument by comparing class fields.
1062  // All string objects must have the same type since String cannot be subclassed.
1063  // Receiver must be a string object, so its class field is equal to all strings' class fields.
1064  // If the argument is a string object, its class field must be equal to receiver's class field.
1065  __ ldr(temp, Address(str, class_offset));
1066  __ ldr(temp1, Address(arg, class_offset));
1067  __ cmp(temp, ShifterOperand(temp1));
1068  __ b(&return_false, NE);
1069
1070  // Load lengths of this and argument strings.
1071  __ ldr(temp, Address(str, count_offset));
1072  __ ldr(temp1, Address(arg, count_offset));
1073  // Check if lengths are equal, return false if they're not.
1074  __ cmp(temp, ShifterOperand(temp1));
1075  __ b(&return_false, NE);
1076  // Return true if both strings are empty.
1077  __ cbz(temp, &return_true);
1078
1079  // Reference equality check, return true if same reference.
1080  __ cmp(str, ShifterOperand(arg));
1081  __ b(&return_true, EQ);
1082
1083  // Assertions that must hold in order to compare strings 2 characters at a time.
1084  DCHECK_ALIGNED(value_offset, 4);
1085  static_assert(IsAligned<4>(kObjectAlignment), "String of odd length is not zero padded");
1086
1087  __ LoadImmediate(temp1, value_offset);
1088
1089  // Loop to compare strings 2 characters at a time starting at the front of the string.
1090  // Ok to do this because strings with an odd length are zero-padded.
1091  __ Bind(&loop);
1092  __ ldr(out, Address(str, temp1));
1093  __ ldr(temp2, Address(arg, temp1));
1094  __ cmp(out, ShifterOperand(temp2));
1095  __ b(&return_false, NE);
1096  __ add(temp1, temp1, ShifterOperand(sizeof(uint32_t)));
1097  __ subs(temp, temp, ShifterOperand(sizeof(uint32_t) /  sizeof(uint16_t)));
1098  __ b(&loop, GT);
1099
1100  // Return true and exit the function.
1101  // If loop does not result in returning false, we return true.
1102  __ Bind(&return_true);
1103  __ LoadImmediate(out, 1);
1104  __ b(&end);
1105
1106  // Return false and exit the function.
1107  __ Bind(&return_false);
1108  __ LoadImmediate(out, 0);
1109  __ Bind(&end);
1110}
1111
1112static void GenerateVisitStringIndexOf(HInvoke* invoke,
1113                                       ArmAssembler* assembler,
1114                                       CodeGeneratorARM* codegen,
1115                                       ArenaAllocator* allocator,
1116                                       bool start_at_zero) {
1117  LocationSummary* locations = invoke->GetLocations();
1118  Register tmp_reg = locations->GetTemp(0).AsRegister<Register>();
1119
1120  // Note that the null check must have been done earlier.
1121  DCHECK(!invoke->CanDoImplicitNullCheckOn(invoke->InputAt(0)));
1122
1123  // Check for code points > 0xFFFF. Either a slow-path check when we don't know statically,
1124  // or directly dispatch if we have a constant.
1125  SlowPathCode* slow_path = nullptr;
1126  if (invoke->InputAt(1)->IsIntConstant()) {
1127    if (static_cast<uint32_t>(invoke->InputAt(1)->AsIntConstant()->GetValue()) >
1128        std::numeric_limits<uint16_t>::max()) {
1129      // Always needs the slow-path. We could directly dispatch to it, but this case should be
1130      // rare, so for simplicity just put the full slow-path down and branch unconditionally.
1131      slow_path = new (allocator) IntrinsicSlowPathARM(invoke);
1132      codegen->AddSlowPath(slow_path);
1133      __ b(slow_path->GetEntryLabel());
1134      __ Bind(slow_path->GetExitLabel());
1135      return;
1136    }
1137  } else {
1138    Register char_reg = locations->InAt(1).AsRegister<Register>();
1139    __ LoadImmediate(tmp_reg, std::numeric_limits<uint16_t>::max());
1140    __ cmp(char_reg, ShifterOperand(tmp_reg));
1141    slow_path = new (allocator) IntrinsicSlowPathARM(invoke);
1142    codegen->AddSlowPath(slow_path);
1143    __ b(slow_path->GetEntryLabel(), HI);
1144  }
1145
1146  if (start_at_zero) {
1147    DCHECK_EQ(tmp_reg, R2);
1148    // Start-index = 0.
1149    __ LoadImmediate(tmp_reg, 0);
1150  }
1151
1152  __ LoadFromOffset(kLoadWord, LR, TR,
1153                    QUICK_ENTRYPOINT_OFFSET(kArmWordSize, pIndexOf).Int32Value());
1154  __ blx(LR);
1155
1156  if (slow_path != nullptr) {
1157    __ Bind(slow_path->GetExitLabel());
1158  }
1159}
1160
1161void IntrinsicLocationsBuilderARM::VisitStringIndexOf(HInvoke* invoke) {
1162  LocationSummary* locations = new (arena_) LocationSummary(invoke,
1163                                                            LocationSummary::kCall,
1164                                                            kIntrinsified);
1165  // We have a hand-crafted assembly stub that follows the runtime calling convention. So it's
1166  // best to align the inputs accordingly.
1167  InvokeRuntimeCallingConvention calling_convention;
1168  locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(0)));
1169  locations->SetInAt(1, Location::RegisterLocation(calling_convention.GetRegisterAt(1)));
1170  locations->SetOut(Location::RegisterLocation(R0));
1171
1172  // Need a temp for slow-path codepoint compare, and need to send start-index=0.
1173  locations->AddTemp(Location::RegisterLocation(calling_convention.GetRegisterAt(2)));
1174}
1175
1176void IntrinsicCodeGeneratorARM::VisitStringIndexOf(HInvoke* invoke) {
1177  GenerateVisitStringIndexOf(
1178      invoke, GetAssembler(), codegen_, GetAllocator(), /* start_at_zero */ true);
1179}
1180
1181void IntrinsicLocationsBuilderARM::VisitStringIndexOfAfter(HInvoke* invoke) {
1182  LocationSummary* locations = new (arena_) LocationSummary(invoke,
1183                                                            LocationSummary::kCall,
1184                                                            kIntrinsified);
1185  // We have a hand-crafted assembly stub that follows the runtime calling convention. So it's
1186  // best to align the inputs accordingly.
1187  InvokeRuntimeCallingConvention calling_convention;
1188  locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(0)));
1189  locations->SetInAt(1, Location::RegisterLocation(calling_convention.GetRegisterAt(1)));
1190  locations->SetInAt(2, Location::RegisterLocation(calling_convention.GetRegisterAt(2)));
1191  locations->SetOut(Location::RegisterLocation(R0));
1192
1193  // Need a temp for slow-path codepoint compare.
1194  locations->AddTemp(Location::RequiresRegister());
1195}
1196
1197void IntrinsicCodeGeneratorARM::VisitStringIndexOfAfter(HInvoke* invoke) {
1198  GenerateVisitStringIndexOf(
1199      invoke, GetAssembler(), codegen_, GetAllocator(), /* start_at_zero */ false);
1200}
1201
1202void IntrinsicLocationsBuilderARM::VisitStringNewStringFromBytes(HInvoke* invoke) {
1203  LocationSummary* locations = new (arena_) LocationSummary(invoke,
1204                                                            LocationSummary::kCall,
1205                                                            kIntrinsified);
1206  InvokeRuntimeCallingConvention calling_convention;
1207  locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(0)));
1208  locations->SetInAt(1, Location::RegisterLocation(calling_convention.GetRegisterAt(1)));
1209  locations->SetInAt(2, Location::RegisterLocation(calling_convention.GetRegisterAt(2)));
1210  locations->SetInAt(3, Location::RegisterLocation(calling_convention.GetRegisterAt(3)));
1211  locations->SetOut(Location::RegisterLocation(R0));
1212}
1213
1214void IntrinsicCodeGeneratorARM::VisitStringNewStringFromBytes(HInvoke* invoke) {
1215  ArmAssembler* assembler = GetAssembler();
1216  LocationSummary* locations = invoke->GetLocations();
1217
1218  Register byte_array = locations->InAt(0).AsRegister<Register>();
1219  __ cmp(byte_array, ShifterOperand(0));
1220  SlowPathCode* slow_path = new (GetAllocator()) IntrinsicSlowPathARM(invoke);
1221  codegen_->AddSlowPath(slow_path);
1222  __ b(slow_path->GetEntryLabel(), EQ);
1223
1224  __ LoadFromOffset(
1225      kLoadWord, LR, TR, QUICK_ENTRYPOINT_OFFSET(kArmWordSize, pAllocStringFromBytes).Int32Value());
1226  codegen_->RecordPcInfo(invoke, invoke->GetDexPc());
1227  __ blx(LR);
1228  __ Bind(slow_path->GetExitLabel());
1229}
1230
1231void IntrinsicLocationsBuilderARM::VisitStringNewStringFromChars(HInvoke* invoke) {
1232  LocationSummary* locations = new (arena_) LocationSummary(invoke,
1233                                                            LocationSummary::kCall,
1234                                                            kIntrinsified);
1235  InvokeRuntimeCallingConvention calling_convention;
1236  locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(0)));
1237  locations->SetInAt(1, Location::RegisterLocation(calling_convention.GetRegisterAt(1)));
1238  locations->SetInAt(2, Location::RegisterLocation(calling_convention.GetRegisterAt(2)));
1239  locations->SetOut(Location::RegisterLocation(R0));
1240}
1241
1242void IntrinsicCodeGeneratorARM::VisitStringNewStringFromChars(HInvoke* invoke) {
1243  ArmAssembler* assembler = GetAssembler();
1244
1245  __ LoadFromOffset(
1246      kLoadWord, LR, TR, QUICK_ENTRYPOINT_OFFSET(kArmWordSize, pAllocStringFromChars).Int32Value());
1247  codegen_->RecordPcInfo(invoke, invoke->GetDexPc());
1248  __ blx(LR);
1249}
1250
1251void IntrinsicLocationsBuilderARM::VisitStringNewStringFromString(HInvoke* invoke) {
1252  LocationSummary* locations = new (arena_) LocationSummary(invoke,
1253                                                            LocationSummary::kCall,
1254                                                            kIntrinsified);
1255  InvokeRuntimeCallingConvention calling_convention;
1256  locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(0)));
1257  locations->SetOut(Location::RegisterLocation(R0));
1258}
1259
1260void IntrinsicCodeGeneratorARM::VisitStringNewStringFromString(HInvoke* invoke) {
1261  ArmAssembler* assembler = GetAssembler();
1262  LocationSummary* locations = invoke->GetLocations();
1263
1264  Register string_to_copy = locations->InAt(0).AsRegister<Register>();
1265  __ cmp(string_to_copy, ShifterOperand(0));
1266  SlowPathCode* slow_path = new (GetAllocator()) IntrinsicSlowPathARM(invoke);
1267  codegen_->AddSlowPath(slow_path);
1268  __ b(slow_path->GetEntryLabel(), EQ);
1269
1270  __ LoadFromOffset(kLoadWord,
1271      LR, TR, QUICK_ENTRYPOINT_OFFSET(kArmWordSize, pAllocStringFromString).Int32Value());
1272  codegen_->RecordPcInfo(invoke, invoke->GetDexPc());
1273  __ blx(LR);
1274  __ Bind(slow_path->GetExitLabel());
1275}
1276
1277void IntrinsicLocationsBuilderARM::VisitSystemArrayCopy(HInvoke* invoke) {
1278  CodeGenerator::CreateSystemArrayCopyLocationSummary(invoke);
1279  LocationSummary* locations = invoke->GetLocations();
1280  if (locations == nullptr) {
1281    return;
1282  }
1283
1284  HIntConstant* src_pos = invoke->InputAt(1)->AsIntConstant();
1285  HIntConstant* dest_pos = invoke->InputAt(3)->AsIntConstant();
1286  HIntConstant* length = invoke->InputAt(4)->AsIntConstant();
1287
1288  if (src_pos != nullptr && !assembler_->ShifterOperandCanAlwaysHold(src_pos->GetValue())) {
1289    locations->SetInAt(1, Location::RequiresRegister());
1290  }
1291  if (dest_pos != nullptr && !assembler_->ShifterOperandCanAlwaysHold(dest_pos->GetValue())) {
1292    locations->SetInAt(3, Location::RequiresRegister());
1293  }
1294  if (length != nullptr && !assembler_->ShifterOperandCanAlwaysHold(length->GetValue())) {
1295    locations->SetInAt(4, Location::RequiresRegister());
1296  }
1297}
1298
1299static void CheckPosition(ArmAssembler* assembler,
1300                          Location pos,
1301                          Register input,
1302                          Location length,
1303                          SlowPathCode* slow_path,
1304                          Register input_len,
1305                          Register temp,
1306                          bool length_is_input_length = false) {
1307  // Where is the length in the Array?
1308  const uint32_t length_offset = mirror::Array::LengthOffset().Uint32Value();
1309
1310  if (pos.IsConstant()) {
1311    int32_t pos_const = pos.GetConstant()->AsIntConstant()->GetValue();
1312    if (pos_const == 0) {
1313      if (!length_is_input_length) {
1314        // Check that length(input) >= length.
1315        __ LoadFromOffset(kLoadWord, temp, input, length_offset);
1316        if (length.IsConstant()) {
1317          __ cmp(temp, ShifterOperand(length.GetConstant()->AsIntConstant()->GetValue()));
1318        } else {
1319          __ cmp(temp, ShifterOperand(length.AsRegister<Register>()));
1320        }
1321        __ b(slow_path->GetEntryLabel(), LT);
1322      }
1323    } else {
1324      // Check that length(input) >= pos.
1325      __ LoadFromOffset(kLoadWord, input_len, input, length_offset);
1326      __ subs(temp, input_len, ShifterOperand(pos_const));
1327      __ b(slow_path->GetEntryLabel(), LT);
1328
1329      // Check that (length(input) - pos) >= length.
1330      if (length.IsConstant()) {
1331        __ cmp(temp, ShifterOperand(length.GetConstant()->AsIntConstant()->GetValue()));
1332      } else {
1333        __ cmp(temp, ShifterOperand(length.AsRegister<Register>()));
1334      }
1335      __ b(slow_path->GetEntryLabel(), LT);
1336    }
1337  } else if (length_is_input_length) {
1338    // The only way the copy can succeed is if pos is zero.
1339    Register pos_reg = pos.AsRegister<Register>();
1340    __ CompareAndBranchIfNonZero(pos_reg, slow_path->GetEntryLabel());
1341  } else {
1342    // Check that pos >= 0.
1343    Register pos_reg = pos.AsRegister<Register>();
1344    __ cmp(pos_reg, ShifterOperand(0));
1345    __ b(slow_path->GetEntryLabel(), LT);
1346
1347    // Check that pos <= length(input).
1348    __ LoadFromOffset(kLoadWord, temp, input, length_offset);
1349    __ subs(temp, temp, ShifterOperand(pos_reg));
1350    __ b(slow_path->GetEntryLabel(), LT);
1351
1352    // Check that (length(input) - pos) >= length.
1353    if (length.IsConstant()) {
1354      __ cmp(temp, ShifterOperand(length.GetConstant()->AsIntConstant()->GetValue()));
1355    } else {
1356      __ cmp(temp, ShifterOperand(length.AsRegister<Register>()));
1357    }
1358    __ b(slow_path->GetEntryLabel(), LT);
1359  }
1360}
1361
1362// TODO: Implement read barriers in the SystemArrayCopy intrinsic.
1363// Note that this code path is not used (yet) because we do not
1364// intrinsify methods that can go into the IntrinsicSlowPathARM
1365// slow path.
1366void IntrinsicCodeGeneratorARM::VisitSystemArrayCopy(HInvoke* invoke) {
1367  ArmAssembler* assembler = GetAssembler();
1368  LocationSummary* locations = invoke->GetLocations();
1369
1370  uint32_t class_offset = mirror::Object::ClassOffset().Int32Value();
1371  uint32_t super_offset = mirror::Class::SuperClassOffset().Int32Value();
1372  uint32_t component_offset = mirror::Class::ComponentTypeOffset().Int32Value();
1373  uint32_t primitive_offset = mirror::Class::PrimitiveTypeOffset().Int32Value();
1374
1375  Register src = locations->InAt(0).AsRegister<Register>();
1376  Location src_pos = locations->InAt(1);
1377  Register dest = locations->InAt(2).AsRegister<Register>();
1378  Location dest_pos = locations->InAt(3);
1379  Location length = locations->InAt(4);
1380  Register temp1 = locations->GetTemp(0).AsRegister<Register>();
1381  Register temp2 = locations->GetTemp(1).AsRegister<Register>();
1382  Register temp3 = locations->GetTemp(2).AsRegister<Register>();
1383
1384  SlowPathCode* slow_path = new (GetAllocator()) IntrinsicSlowPathARM(invoke);
1385  codegen_->AddSlowPath(slow_path);
1386
1387  Label ok;
1388  SystemArrayCopyOptimizations optimizations(invoke);
1389
1390  if (!optimizations.GetDestinationIsSource()) {
1391    if (!src_pos.IsConstant() || !dest_pos.IsConstant()) {
1392      __ cmp(src, ShifterOperand(dest));
1393    }
1394  }
1395
1396  // If source and destination are the same, we go to slow path if we need to do
1397  // forward copying.
1398  if (src_pos.IsConstant()) {
1399    int32_t src_pos_constant = src_pos.GetConstant()->AsIntConstant()->GetValue();
1400    if (dest_pos.IsConstant()) {
1401      // Checked when building locations.
1402      DCHECK(!optimizations.GetDestinationIsSource()
1403             || (src_pos_constant >= dest_pos.GetConstant()->AsIntConstant()->GetValue()));
1404    } else {
1405      if (!optimizations.GetDestinationIsSource()) {
1406        __ b(&ok, NE);
1407      }
1408      __ cmp(dest_pos.AsRegister<Register>(), ShifterOperand(src_pos_constant));
1409      __ b(slow_path->GetEntryLabel(), GT);
1410    }
1411  } else {
1412    if (!optimizations.GetDestinationIsSource()) {
1413      __ b(&ok, NE);
1414    }
1415    if (dest_pos.IsConstant()) {
1416      int32_t dest_pos_constant = dest_pos.GetConstant()->AsIntConstant()->GetValue();
1417      __ cmp(src_pos.AsRegister<Register>(), ShifterOperand(dest_pos_constant));
1418    } else {
1419      __ cmp(src_pos.AsRegister<Register>(), ShifterOperand(dest_pos.AsRegister<Register>()));
1420    }
1421    __ b(slow_path->GetEntryLabel(), LT);
1422  }
1423
1424  __ Bind(&ok);
1425
1426  if (!optimizations.GetSourceIsNotNull()) {
1427    // Bail out if the source is null.
1428    __ CompareAndBranchIfZero(src, slow_path->GetEntryLabel());
1429  }
1430
1431  if (!optimizations.GetDestinationIsNotNull() && !optimizations.GetDestinationIsSource()) {
1432    // Bail out if the destination is null.
1433    __ CompareAndBranchIfZero(dest, slow_path->GetEntryLabel());
1434  }
1435
1436  // If the length is negative, bail out.
1437  // We have already checked in the LocationsBuilder for the constant case.
1438  if (!length.IsConstant() &&
1439      !optimizations.GetCountIsSourceLength() &&
1440      !optimizations.GetCountIsDestinationLength()) {
1441    __ cmp(length.AsRegister<Register>(), ShifterOperand(0));
1442    __ b(slow_path->GetEntryLabel(), LT);
1443  }
1444
1445  // Validity checks: source.
1446  CheckPosition(assembler,
1447                src_pos,
1448                src,
1449                length,
1450                slow_path,
1451                temp1,
1452                temp2,
1453                optimizations.GetCountIsSourceLength());
1454
1455  // Validity checks: dest.
1456  CheckPosition(assembler,
1457                dest_pos,
1458                dest,
1459                length,
1460                slow_path,
1461                temp1,
1462                temp2,
1463                optimizations.GetCountIsDestinationLength());
1464
1465  if (!optimizations.GetDoesNotNeedTypeCheck()) {
1466    // Check whether all elements of the source array are assignable to the component
1467    // type of the destination array. We do two checks: the classes are the same,
1468    // or the destination is Object[]. If none of these checks succeed, we go to the
1469    // slow path.
1470    __ LoadFromOffset(kLoadWord, temp1, dest, class_offset);
1471    __ LoadFromOffset(kLoadWord, temp2, src, class_offset);
1472    bool did_unpoison = false;
1473    if (!optimizations.GetDestinationIsNonPrimitiveArray() ||
1474        !optimizations.GetSourceIsNonPrimitiveArray()) {
1475      // One or two of the references need to be unpoisoned. Unpoisoned them
1476      // both to make the identity check valid.
1477      __ MaybeUnpoisonHeapReference(temp1);
1478      __ MaybeUnpoisonHeapReference(temp2);
1479      did_unpoison = true;
1480    }
1481
1482    if (!optimizations.GetDestinationIsNonPrimitiveArray()) {
1483      // Bail out if the destination is not a non primitive array.
1484      __ LoadFromOffset(kLoadWord, temp3, temp1, component_offset);
1485      __ CompareAndBranchIfZero(temp3, slow_path->GetEntryLabel());
1486      __ MaybeUnpoisonHeapReference(temp3);
1487      __ LoadFromOffset(kLoadUnsignedHalfword, temp3, temp3, primitive_offset);
1488      static_assert(Primitive::kPrimNot == 0, "Expected 0 for kPrimNot");
1489      __ CompareAndBranchIfNonZero(temp3, slow_path->GetEntryLabel());
1490    }
1491
1492    if (!optimizations.GetSourceIsNonPrimitiveArray()) {
1493      // Bail out if the source is not a non primitive array.
1494      // Bail out if the destination is not a non primitive array.
1495      __ LoadFromOffset(kLoadWord, temp3, temp2, component_offset);
1496      __ CompareAndBranchIfZero(temp3, slow_path->GetEntryLabel());
1497      __ MaybeUnpoisonHeapReference(temp3);
1498      __ LoadFromOffset(kLoadUnsignedHalfword, temp3, temp3, primitive_offset);
1499      static_assert(Primitive::kPrimNot == 0, "Expected 0 for kPrimNot");
1500      __ CompareAndBranchIfNonZero(temp3, slow_path->GetEntryLabel());
1501    }
1502
1503    __ cmp(temp1, ShifterOperand(temp2));
1504
1505    if (optimizations.GetDestinationIsTypedObjectArray()) {
1506      Label do_copy;
1507      __ b(&do_copy, EQ);
1508      if (!did_unpoison) {
1509        __ MaybeUnpoisonHeapReference(temp1);
1510      }
1511      __ LoadFromOffset(kLoadWord, temp1, temp1, component_offset);
1512      __ MaybeUnpoisonHeapReference(temp1);
1513      __ LoadFromOffset(kLoadWord, temp1, temp1, super_offset);
1514      // No need to unpoison the result, we're comparing against null.
1515      __ CompareAndBranchIfNonZero(temp1, slow_path->GetEntryLabel());
1516      __ Bind(&do_copy);
1517    } else {
1518      __ b(slow_path->GetEntryLabel(), NE);
1519    }
1520  } else if (!optimizations.GetSourceIsNonPrimitiveArray()) {
1521    DCHECK(optimizations.GetDestinationIsNonPrimitiveArray());
1522    // Bail out if the source is not a non primitive array.
1523    __ LoadFromOffset(kLoadWord, temp1, src, class_offset);
1524    __ MaybeUnpoisonHeapReference(temp1);
1525    __ LoadFromOffset(kLoadWord, temp3, temp1, component_offset);
1526    __ CompareAndBranchIfZero(temp3, slow_path->GetEntryLabel());
1527    __ MaybeUnpoisonHeapReference(temp3);
1528    __ LoadFromOffset(kLoadUnsignedHalfword, temp3, temp3, primitive_offset);
1529    static_assert(Primitive::kPrimNot == 0, "Expected 0 for kPrimNot");
1530    __ CompareAndBranchIfNonZero(temp3, slow_path->GetEntryLabel());
1531  }
1532
1533  // Compute base source address, base destination address, and end source address.
1534
1535  uint32_t element_size = sizeof(int32_t);
1536  uint32_t offset = mirror::Array::DataOffset(element_size).Uint32Value();
1537  if (src_pos.IsConstant()) {
1538    int32_t constant = src_pos.GetConstant()->AsIntConstant()->GetValue();
1539    __ AddConstant(temp1, src, element_size * constant + offset);
1540  } else {
1541    __ add(temp1, src, ShifterOperand(src_pos.AsRegister<Register>(), LSL, 2));
1542    __ AddConstant(temp1, offset);
1543  }
1544
1545  if (dest_pos.IsConstant()) {
1546    int32_t constant = dest_pos.GetConstant()->AsIntConstant()->GetValue();
1547    __ AddConstant(temp2, dest, element_size * constant + offset);
1548  } else {
1549    __ add(temp2, dest, ShifterOperand(dest_pos.AsRegister<Register>(), LSL, 2));
1550    __ AddConstant(temp2, offset);
1551  }
1552
1553  if (length.IsConstant()) {
1554    int32_t constant = length.GetConstant()->AsIntConstant()->GetValue();
1555    __ AddConstant(temp3, temp1, element_size * constant);
1556  } else {
1557    __ add(temp3, temp1, ShifterOperand(length.AsRegister<Register>(), LSL, 2));
1558  }
1559
1560  // Iterate over the arrays and do a raw copy of the objects. We don't need to
1561  // poison/unpoison, nor do any read barrier as the next uses of the destination
1562  // array will do it.
1563  Label loop, done;
1564  __ cmp(temp1, ShifterOperand(temp3));
1565  __ b(&done, EQ);
1566  __ Bind(&loop);
1567  __ ldr(IP, Address(temp1, element_size, Address::PostIndex));
1568  __ str(IP, Address(temp2, element_size, Address::PostIndex));
1569  __ cmp(temp1, ShifterOperand(temp3));
1570  __ b(&loop, NE);
1571  __ Bind(&done);
1572
1573  // We only need one card marking on the destination array.
1574  codegen_->MarkGCCard(temp1,
1575                       temp2,
1576                       dest,
1577                       Register(kNoRegister),
1578                       /* can_be_null */ false);
1579
1580  __ Bind(slow_path->GetExitLabel());
1581}
1582
1583// Unimplemented intrinsics.
1584
1585#define UNIMPLEMENTED_INTRINSIC(Name)                                                  \
1586void IntrinsicLocationsBuilderARM::Visit ## Name(HInvoke* invoke ATTRIBUTE_UNUSED) { \
1587}                                                                                      \
1588void IntrinsicCodeGeneratorARM::Visit ## Name(HInvoke* invoke ATTRIBUTE_UNUSED) {    \
1589}
1590
1591UNIMPLEMENTED_INTRINSIC(IntegerBitCount)
1592UNIMPLEMENTED_INTRINSIC(IntegerReverse)
1593UNIMPLEMENTED_INTRINSIC(IntegerReverseBytes)
1594UNIMPLEMENTED_INTRINSIC(LongBitCount)
1595UNIMPLEMENTED_INTRINSIC(LongReverse)
1596UNIMPLEMENTED_INTRINSIC(LongReverseBytes)
1597UNIMPLEMENTED_INTRINSIC(ShortReverseBytes)
1598UNIMPLEMENTED_INTRINSIC(MathMinDoubleDouble)
1599UNIMPLEMENTED_INTRINSIC(MathMinFloatFloat)
1600UNIMPLEMENTED_INTRINSIC(MathMaxDoubleDouble)
1601UNIMPLEMENTED_INTRINSIC(MathMaxFloatFloat)
1602UNIMPLEMENTED_INTRINSIC(MathMinLongLong)
1603UNIMPLEMENTED_INTRINSIC(MathMaxLongLong)
1604UNIMPLEMENTED_INTRINSIC(MathCeil)          // Could be done by changing rounding mode, maybe?
1605UNIMPLEMENTED_INTRINSIC(MathFloor)         // Could be done by changing rounding mode, maybe?
1606UNIMPLEMENTED_INTRINSIC(MathRint)
1607UNIMPLEMENTED_INTRINSIC(MathRoundDouble)   // Could be done by changing rounding mode, maybe?
1608UNIMPLEMENTED_INTRINSIC(MathRoundFloat)    // Could be done by changing rounding mode, maybe?
1609UNIMPLEMENTED_INTRINSIC(UnsafeCASLong)     // High register pressure.
1610UNIMPLEMENTED_INTRINSIC(SystemArrayCopyChar)
1611UNIMPLEMENTED_INTRINSIC(ReferenceGetReferent)
1612UNIMPLEMENTED_INTRINSIC(StringGetCharsNoCheck)
1613UNIMPLEMENTED_INTRINSIC(MathCos)
1614UNIMPLEMENTED_INTRINSIC(MathSin)
1615UNIMPLEMENTED_INTRINSIC(MathAcos)
1616UNIMPLEMENTED_INTRINSIC(MathAsin)
1617UNIMPLEMENTED_INTRINSIC(MathAtan)
1618UNIMPLEMENTED_INTRINSIC(MathAtan2)
1619UNIMPLEMENTED_INTRINSIC(MathCbrt)
1620UNIMPLEMENTED_INTRINSIC(MathCosh)
1621UNIMPLEMENTED_INTRINSIC(MathExp)
1622UNIMPLEMENTED_INTRINSIC(MathExpm1)
1623UNIMPLEMENTED_INTRINSIC(MathHypot)
1624UNIMPLEMENTED_INTRINSIC(MathLog)
1625UNIMPLEMENTED_INTRINSIC(MathLog10)
1626UNIMPLEMENTED_INTRINSIC(MathNextAfter)
1627UNIMPLEMENTED_INTRINSIC(MathSinh)
1628UNIMPLEMENTED_INTRINSIC(MathTan)
1629UNIMPLEMENTED_INTRINSIC(MathTanh)
1630
1631UNIMPLEMENTED_INTRINSIC(FloatIsInfinite)
1632UNIMPLEMENTED_INTRINSIC(DoubleIsInfinite)
1633UNIMPLEMENTED_INTRINSIC(FloatIsNaN)
1634UNIMPLEMENTED_INTRINSIC(DoubleIsNaN)
1635
1636UNIMPLEMENTED_INTRINSIC(IntegerHighestOneBit)
1637UNIMPLEMENTED_INTRINSIC(LongHighestOneBit)
1638UNIMPLEMENTED_INTRINSIC(IntegerLowestOneBit)
1639UNIMPLEMENTED_INTRINSIC(LongLowestOneBit)
1640
1641// Handled as HIR instructions.
1642UNIMPLEMENTED_INTRINSIC(IntegerRotateLeft)
1643UNIMPLEMENTED_INTRINSIC(LongRotateLeft)
1644UNIMPLEMENTED_INTRINSIC(IntegerRotateRight)
1645UNIMPLEMENTED_INTRINSIC(LongRotateRight)
1646UNIMPLEMENTED_INTRINSIC(IntegerCompare)
1647UNIMPLEMENTED_INTRINSIC(LongCompare)
1648UNIMPLEMENTED_INTRINSIC(IntegerSignum)
1649UNIMPLEMENTED_INTRINSIC(LongSignum)
1650
1651#undef UNIMPLEMENTED_INTRINSIC
1652
1653#undef __
1654
1655}  // namespace arm
1656}  // namespace art
1657