intrinsics_arm.cc revision 85b62f23fc6dfffe2ddd3ddfa74611666c9ff41d 
1/*
2 * Copyright (C) 2015 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 *      http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17#include "intrinsics_arm.h"
18
19#include "arch/arm/instruction_set_features_arm.h"
20#include "art_method.h"
21#include "code_generator_arm.h"
22#include "entrypoints/quick/quick_entrypoints.h"
23#include "intrinsics.h"
24#include "intrinsics_utils.h"
25#include "mirror/array-inl.h"
26#include "mirror/string.h"
27#include "thread.h"
28#include "utils/arm/assembler_arm.h"
29
30namespace art {
31
32namespace arm {
33
34ArmAssembler* IntrinsicCodeGeneratorARM::GetAssembler() {
35  return codegen_->GetAssembler();
36}
37
38ArenaAllocator* IntrinsicCodeGeneratorARM::GetAllocator() {
39  return codegen_->GetGraph()->GetArena();
40}
41
42using IntrinsicSlowPathARM = IntrinsicSlowPath<InvokeDexCallingConventionVisitorARM>;
43
44bool IntrinsicLocationsBuilderARM::TryDispatch(HInvoke* invoke) {
45  Dispatch(invoke);
46  LocationSummary* res = invoke->GetLocations();
47  return res != nullptr && res->Intrinsified();
48}
49
50#define __ assembler->
51
52static void CreateFPToIntLocations(ArenaAllocator* arena, HInvoke* invoke) {
53  LocationSummary* locations = new (arena) LocationSummary(invoke,
54                                                           LocationSummary::kNoCall,
55                                                           kIntrinsified);
56  locations->SetInAt(0, Location::RequiresFpuRegister());
57  locations->SetOut(Location::RequiresRegister());
58}
59
60static void CreateIntToFPLocations(ArenaAllocator* arena, HInvoke* invoke) {
61  LocationSummary* locations = new (arena) LocationSummary(invoke,
62                                                           LocationSummary::kNoCall,
63                                                           kIntrinsified);
64  locations->SetInAt(0, Location::RequiresRegister());
65  locations->SetOut(Location::RequiresFpuRegister());
66}
67
68static void MoveFPToInt(LocationSummary* locations, bool is64bit, ArmAssembler* assembler) {
69  Location input = locations->InAt(0);
70  Location output = locations->Out();
71  if (is64bit) {
72    __ vmovrrd(output.AsRegisterPairLow<Register>(),
73               output.AsRegisterPairHigh<Register>(),
74               FromLowSToD(input.AsFpuRegisterPairLow<SRegister>()));
75  } else {
76    __ vmovrs(output.AsRegister<Register>(), input.AsFpuRegister<SRegister>());
77  }
78}
79
80static void MoveIntToFP(LocationSummary* locations, bool is64bit, ArmAssembler* assembler) {
81  Location input = locations->InAt(0);
82  Location output = locations->Out();
83  if (is64bit) {
84    __ vmovdrr(FromLowSToD(output.AsFpuRegisterPairLow<SRegister>()),
85               input.AsRegisterPairLow<Register>(),
86               input.AsRegisterPairHigh<Register>());
87  } else {
88    __ vmovsr(output.AsFpuRegister<SRegister>(), input.AsRegister<Register>());
89  }
90}
91
92void IntrinsicLocationsBuilderARM::VisitDoubleDoubleToRawLongBits(HInvoke* invoke) {
93  CreateFPToIntLocations(arena_, invoke);
94}
95void IntrinsicLocationsBuilderARM::VisitDoubleLongBitsToDouble(HInvoke* invoke) {
96  CreateIntToFPLocations(arena_, invoke);
97}
98
99void IntrinsicCodeGeneratorARM::VisitDoubleDoubleToRawLongBits(HInvoke* invoke) {
100  MoveFPToInt(invoke->GetLocations(), true, GetAssembler());
101}
102void IntrinsicCodeGeneratorARM::VisitDoubleLongBitsToDouble(HInvoke* invoke) {
103  MoveIntToFP(invoke->GetLocations(), true, GetAssembler());
104}
105
106void IntrinsicLocationsBuilderARM::VisitFloatFloatToRawIntBits(HInvoke* invoke) {
107  CreateFPToIntLocations(arena_, invoke);
108}
109void IntrinsicLocationsBuilderARM::VisitFloatIntBitsToFloat(HInvoke* invoke) {
110  CreateIntToFPLocations(arena_, invoke);
111}
112
113void IntrinsicCodeGeneratorARM::VisitFloatFloatToRawIntBits(HInvoke* invoke) {
114  MoveFPToInt(invoke->GetLocations(), false, GetAssembler());
115}
116void IntrinsicCodeGeneratorARM::VisitFloatIntBitsToFloat(HInvoke* invoke) {
117  MoveIntToFP(invoke->GetLocations(), false, GetAssembler());
118}
119
120static void CreateIntToIntLocations(ArenaAllocator* arena, HInvoke* invoke) {
121  LocationSummary* locations = new (arena) LocationSummary(invoke,
122                                                           LocationSummary::kNoCall,
123                                                           kIntrinsified);
124  locations->SetInAt(0, Location::RequiresRegister());
125  locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
126}
127
128static void CreateFPToFPLocations(ArenaAllocator* arena, HInvoke* invoke) {
129  LocationSummary* locations = new (arena) LocationSummary(invoke,
130                                                           LocationSummary::kNoCall,
131                                                           kIntrinsified);
132  locations->SetInAt(0, Location::RequiresFpuRegister());
133  locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap);
134}
135
136static void GenNumberOfLeadingZeros(LocationSummary* locations,
137                                    Primitive::Type type,
138                                    ArmAssembler* assembler) {
139  Location in = locations->InAt(0);
140  Register out = locations->Out().AsRegister<Register>();
141
142  DCHECK((type == Primitive::kPrimInt) || (type == Primitive::kPrimLong));
143
144  if (type == Primitive::kPrimLong) {
145    Register in_reg_lo = in.AsRegisterPairLow<Register>();
146    Register in_reg_hi = in.AsRegisterPairHigh<Register>();
147    Label end;
148    __ clz(out, in_reg_hi);
149    __ CompareAndBranchIfNonZero(in_reg_hi, &end);
150    __ clz(out, in_reg_lo);
151    __ AddConstant(out, 32);
152    __ Bind(&end);
153  } else {
154    __ clz(out, in.AsRegister<Register>());
155  }
156}
157
158void IntrinsicLocationsBuilderARM::VisitIntegerNumberOfLeadingZeros(HInvoke* invoke) {
159  CreateIntToIntLocations(arena_, invoke);
160}
161
162void IntrinsicCodeGeneratorARM::VisitIntegerNumberOfLeadingZeros(HInvoke* invoke) {
163  GenNumberOfLeadingZeros(invoke->GetLocations(), Primitive::kPrimInt, GetAssembler());
164}
165
166void IntrinsicLocationsBuilderARM::VisitLongNumberOfLeadingZeros(HInvoke* invoke) {
167  LocationSummary* locations = new (arena_) LocationSummary(invoke,
168                                                           LocationSummary::kNoCall,
169                                                           kIntrinsified);
170  locations->SetInAt(0, Location::RequiresRegister());
171  locations->SetOut(Location::RequiresRegister(), Location::kOutputOverlap);
172}
173
174void IntrinsicCodeGeneratorARM::VisitLongNumberOfLeadingZeros(HInvoke* invoke) {
175  GenNumberOfLeadingZeros(invoke->GetLocations(), Primitive::kPrimLong, GetAssembler());
176}
177
178static void GenNumberOfTrailingZeros(LocationSummary* locations,
179                                     Primitive::Type type,
180                                     ArmAssembler* assembler) {
181  DCHECK((type == Primitive::kPrimInt) || (type == Primitive::kPrimLong));
182
183  Register out = locations->Out().AsRegister<Register>();
184
185  if (type == Primitive::kPrimLong) {
186    Register in_reg_lo = locations->InAt(0).AsRegisterPairLow<Register>();
187    Register in_reg_hi = locations->InAt(0).AsRegisterPairHigh<Register>();
188    Label end;
189    __ rbit(out, in_reg_lo);
190    __ clz(out, out);
191    __ CompareAndBranchIfNonZero(in_reg_lo, &end);
192    __ rbit(out, in_reg_hi);
193    __ clz(out, out);
194    __ AddConstant(out, 32);
195    __ Bind(&end);
196  } else {
197    Register in = locations->InAt(0).AsRegister<Register>();
198    __ rbit(out, in);
199    __ clz(out, out);
200  }
201}
202
203void IntrinsicLocationsBuilderARM::VisitIntegerNumberOfTrailingZeros(HInvoke* invoke) {
204  LocationSummary* locations = new (arena_) LocationSummary(invoke,
205                                                            LocationSummary::kNoCall,
206                                                            kIntrinsified);
207  locations->SetInAt(0, Location::RequiresRegister());
208  locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
209}
210
211void IntrinsicCodeGeneratorARM::VisitIntegerNumberOfTrailingZeros(HInvoke* invoke) {
212  GenNumberOfTrailingZeros(invoke->GetLocations(), Primitive::kPrimInt, GetAssembler());
213}
214
215void IntrinsicLocationsBuilderARM::VisitLongNumberOfTrailingZeros(HInvoke* invoke) {
216  LocationSummary* locations = new (arena_) LocationSummary(invoke,
217                                                            LocationSummary::kNoCall,
218                                                            kIntrinsified);
219  locations->SetInAt(0, Location::RequiresRegister());
220  locations->SetOut(Location::RequiresRegister(), Location::kOutputOverlap);
221}
222
223void IntrinsicCodeGeneratorARM::VisitLongNumberOfTrailingZeros(HInvoke* invoke) {
224  GenNumberOfTrailingZeros(invoke->GetLocations(), Primitive::kPrimLong, GetAssembler());
225}
226
227static void GenIntegerRotate(LocationSummary* locations,
228                             ArmAssembler* assembler,
229                             bool is_left) {
230  Register in = locations->InAt(0).AsRegister<Register>();
231  Location rhs = locations->InAt(1);
232  Register out = locations->Out().AsRegister<Register>();
233
234  if (rhs.IsConstant()) {
235    // Arm32 and Thumb2 assemblers require a rotation on the interval [1,31],
236    // so map all rotations to a +ve. equivalent in that range.
237    // (e.g. left *or* right by -2 bits == 30 bits in the same direction.)
238    uint32_t rot = rhs.GetConstant()->AsIntConstant()->GetValue() & 0x1F;
239    if (rot) {
240      // Rotate, mapping left rotations to right equivalents if necessary.
241      // (e.g. left by 2 bits == right by 30.)
242      __ Ror(out, in, is_left ? (0x20 - rot) : rot);
243    } else if (out != in) {
244      __ Mov(out, in);
245    }
246  } else {
247    if (is_left) {
248      __ rsb(out, rhs.AsRegister<Register>(), ShifterOperand(0));
249      __ Ror(out, in, out);
250    } else {
251      __ Ror(out, in, rhs.AsRegister<Register>());
252    }
253  }
254}
255
256// Gain some speed by mapping all Long rotates onto equivalent pairs of Integer
257// rotates by swapping input regs (effectively rotating by the first 32-bits of
258// a larger rotation) or flipping direction (thus treating larger right/left
259// rotations as sub-word sized rotations in the other direction) as appropriate.
260static void GenLongRotate(LocationSummary* locations,
261                          ArmAssembler* assembler,
262                          bool is_left) {
263  Register in_reg_lo = locations->InAt(0).AsRegisterPairLow<Register>();
264  Register in_reg_hi = locations->InAt(0).AsRegisterPairHigh<Register>();
265  Location rhs = locations->InAt(1);
266  Register out_reg_lo = locations->Out().AsRegisterPairLow<Register>();
267  Register out_reg_hi = locations->Out().AsRegisterPairHigh<Register>();
268
269  if (rhs.IsConstant()) {
270    uint32_t rot = rhs.GetConstant()->AsIntConstant()->GetValue();
271    // Map all left rotations to right equivalents.
272    if (is_left) {
273      rot = 0x40 - rot;
274    }
275    // Map all rotations to +ve. equivalents on the interval [0,63].
276    rot &= 0x3F;
277    // For rotates over a word in size, 'pre-rotate' by 32-bits to keep rotate
278    // logic below to a simple pair of binary orr.
279    // (e.g. 34 bits == in_reg swap + 2 bits right.)
280    if (rot >= 0x20) {
281      rot -= 0x20;
282      std::swap(in_reg_hi, in_reg_lo);
283    }
284    // Rotate, or mov to out for zero or word size rotations.
285    if (rot) {
286      __ Lsr(out_reg_hi, in_reg_hi, rot);
287      __ orr(out_reg_hi, out_reg_hi, ShifterOperand(in_reg_lo, arm::LSL, 0x20 - rot));
288      __ Lsr(out_reg_lo, in_reg_lo, rot);
289      __ orr(out_reg_lo, out_reg_lo, ShifterOperand(in_reg_hi, arm::LSL, 0x20 - rot));
290    } else {
291      __ Mov(out_reg_lo, in_reg_lo);
292      __ Mov(out_reg_hi, in_reg_hi);
293    }
294  } else {
295    Register shift_left = locations->GetTemp(0).AsRegister<Register>();
296    Register shift_right = locations->GetTemp(1).AsRegister<Register>();
297    Label end;
298    Label right;
299
300    __ and_(shift_left, rhs.AsRegister<Register>(), ShifterOperand(0x1F));
301    __ Lsrs(shift_right, rhs.AsRegister<Register>(), 6);
302    __ rsb(shift_right, shift_left, ShifterOperand(0x20), AL, kCcKeep);
303
304    if (is_left) {
305      __ b(&right, CS);
306    } else {
307      __ b(&right, CC);
308      std::swap(shift_left, shift_right);
309    }
310
311    // out_reg_hi = (reg_hi << shift_left) | (reg_lo >> shift_right).
312    // out_reg_lo = (reg_lo << shift_left) | (reg_hi >> shift_right).
313    __ Lsl(out_reg_hi, in_reg_hi, shift_left);
314    __ Lsr(out_reg_lo, in_reg_lo, shift_right);
315    __ add(out_reg_hi, out_reg_hi, ShifterOperand(out_reg_lo));
316    __ Lsl(out_reg_lo, in_reg_lo, shift_left);
317    __ Lsr(shift_left, in_reg_hi, shift_right);
318    __ add(out_reg_lo, out_reg_lo, ShifterOperand(shift_left));
319    __ b(&end);
320
321    // out_reg_hi = (reg_hi >> shift_right) | (reg_lo << shift_left).
322    // out_reg_lo = (reg_lo >> shift_right) | (reg_hi << shift_left).
323    __ Bind(&right);
324    __ Lsr(out_reg_hi, in_reg_hi, shift_right);
325    __ Lsl(out_reg_lo, in_reg_lo, shift_left);
326    __ add(out_reg_hi, out_reg_hi, ShifterOperand(out_reg_lo));
327    __ Lsr(out_reg_lo, in_reg_lo, shift_right);
328    __ Lsl(shift_right, in_reg_hi, shift_left);
329    __ add(out_reg_lo, out_reg_lo, ShifterOperand(shift_right));
330
331    __ Bind(&end);
332  }
333}
334
335void IntrinsicLocationsBuilderARM::VisitIntegerRotateRight(HInvoke* invoke) {
336  LocationSummary* locations = new (arena_) LocationSummary(invoke,
337                                                            LocationSummary::kNoCall,
338                                                            kIntrinsified);
339  locations->SetInAt(0, Location::RequiresRegister());
340  locations->SetInAt(1, Location::RegisterOrConstant(invoke->InputAt(1)));
341  locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
342}
343
344void IntrinsicCodeGeneratorARM::VisitIntegerRotateRight(HInvoke* invoke) {
345  GenIntegerRotate(invoke->GetLocations(), GetAssembler(), false /* is_left */);
346}
347
348void IntrinsicLocationsBuilderARM::VisitLongRotateRight(HInvoke* invoke) {
349  LocationSummary* locations = new (arena_) LocationSummary(invoke,
350                                                            LocationSummary::kNoCall,
351                                                            kIntrinsified);
352  locations->SetInAt(0, Location::RequiresRegister());
353  if (invoke->InputAt(1)->IsConstant()) {
354    locations->SetInAt(1, Location::ConstantLocation(invoke->InputAt(1)->AsConstant()));
355  } else {
356    locations->SetInAt(1, Location::RequiresRegister());
357    locations->AddTemp(Location::RequiresRegister());
358    locations->AddTemp(Location::RequiresRegister());
359  }
360  locations->SetOut(Location::RequiresRegister(), Location::kOutputOverlap);
361}
362
363void IntrinsicCodeGeneratorARM::VisitLongRotateRight(HInvoke* invoke) {
364  GenLongRotate(invoke->GetLocations(), GetAssembler(), false /* is_left */);
365}
366
367void IntrinsicLocationsBuilderARM::VisitIntegerRotateLeft(HInvoke* invoke) {
368  LocationSummary* locations = new (arena_) LocationSummary(invoke,
369                                                            LocationSummary::kNoCall,
370                                                            kIntrinsified);
371  locations->SetInAt(0, Location::RequiresRegister());
372  locations->SetInAt(1, Location::RegisterOrConstant(invoke->InputAt(1)));
373  locations->SetOut(Location::RequiresRegister(), Location::kOutputOverlap);
374}
375
376void IntrinsicCodeGeneratorARM::VisitIntegerRotateLeft(HInvoke* invoke) {
377  GenIntegerRotate(invoke->GetLocations(), GetAssembler(), true /* is_left */);
378}
379
380void IntrinsicLocationsBuilderARM::VisitLongRotateLeft(HInvoke* invoke) {
381  LocationSummary* locations = new (arena_) LocationSummary(invoke,
382                                                            LocationSummary::kNoCall,
383                                                            kIntrinsified);
384  locations->SetInAt(0, Location::RequiresRegister());
385  if (invoke->InputAt(1)->IsConstant()) {
386    locations->SetInAt(1, Location::ConstantLocation(invoke->InputAt(1)->AsConstant()));
387  } else {
388    locations->SetInAt(1, Location::RequiresRegister());
389    locations->AddTemp(Location::RequiresRegister());
390    locations->AddTemp(Location::RequiresRegister());
391  }
392  locations->SetOut(Location::RequiresRegister(), Location::kOutputOverlap);
393}
394
395void IntrinsicCodeGeneratorARM::VisitLongRotateLeft(HInvoke* invoke) {
396  GenLongRotate(invoke->GetLocations(), GetAssembler(), true /* is_left */);
397}
398
399static void MathAbsFP(LocationSummary* locations, bool is64bit, ArmAssembler* assembler) {
400  Location in = locations->InAt(0);
401  Location out = locations->Out();
402
403  if (is64bit) {
404    __ vabsd(FromLowSToD(out.AsFpuRegisterPairLow<SRegister>()),
405             FromLowSToD(in.AsFpuRegisterPairLow<SRegister>()));
406  } else {
407    __ vabss(out.AsFpuRegister<SRegister>(), in.AsFpuRegister<SRegister>());
408  }
409}
410
411void IntrinsicLocationsBuilderARM::VisitMathAbsDouble(HInvoke* invoke) {
412  CreateFPToFPLocations(arena_, invoke);
413}
414
415void IntrinsicCodeGeneratorARM::VisitMathAbsDouble(HInvoke* invoke) {
416  MathAbsFP(invoke->GetLocations(), true, GetAssembler());
417}
418
419void IntrinsicLocationsBuilderARM::VisitMathAbsFloat(HInvoke* invoke) {
420  CreateFPToFPLocations(arena_, invoke);
421}
422
423void IntrinsicCodeGeneratorARM::VisitMathAbsFloat(HInvoke* invoke) {
424  MathAbsFP(invoke->GetLocations(), false, GetAssembler());
425}
426
427static void CreateIntToIntPlusTemp(ArenaAllocator* arena, HInvoke* invoke) {
428  LocationSummary* locations = new (arena) LocationSummary(invoke,
429                                                           LocationSummary::kNoCall,
430                                                           kIntrinsified);
431  locations->SetInAt(0, Location::RequiresRegister());
432  locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
433
434  locations->AddTemp(Location::RequiresRegister());
435}
436
437static void GenAbsInteger(LocationSummary* locations,
438                          bool is64bit,
439                          ArmAssembler* assembler) {
440  Location in = locations->InAt(0);
441  Location output = locations->Out();
442
443  Register mask = locations->GetTemp(0).AsRegister<Register>();
444
445  if (is64bit) {
446    Register in_reg_lo = in.AsRegisterPairLow<Register>();
447    Register in_reg_hi = in.AsRegisterPairHigh<Register>();
448    Register out_reg_lo = output.AsRegisterPairLow<Register>();
449    Register out_reg_hi = output.AsRegisterPairHigh<Register>();
450
451    DCHECK_NE(out_reg_lo, in_reg_hi) << "Diagonal overlap unexpected.";
452
453    __ Asr(mask, in_reg_hi, 31);
454    __ adds(out_reg_lo, in_reg_lo, ShifterOperand(mask));
455    __ adc(out_reg_hi, in_reg_hi, ShifterOperand(mask));
456    __ eor(out_reg_lo, mask, ShifterOperand(out_reg_lo));
457    __ eor(out_reg_hi, mask, ShifterOperand(out_reg_hi));
458  } else {
459    Register in_reg = in.AsRegister<Register>();
460    Register out_reg = output.AsRegister<Register>();
461
462    __ Asr(mask, in_reg, 31);
463    __ add(out_reg, in_reg, ShifterOperand(mask));
464    __ eor(out_reg, mask, ShifterOperand(out_reg));
465  }
466}
467
468void IntrinsicLocationsBuilderARM::VisitMathAbsInt(HInvoke* invoke) {
469  CreateIntToIntPlusTemp(arena_, invoke);
470}
471
472void IntrinsicCodeGeneratorARM::VisitMathAbsInt(HInvoke* invoke) {
473  GenAbsInteger(invoke->GetLocations(), false, GetAssembler());
474}
475
476
477void IntrinsicLocationsBuilderARM::VisitMathAbsLong(HInvoke* invoke) {
478  CreateIntToIntPlusTemp(arena_, invoke);
479}
480
481void IntrinsicCodeGeneratorARM::VisitMathAbsLong(HInvoke* invoke) {
482  GenAbsInteger(invoke->GetLocations(), true, GetAssembler());
483}
484
485static void GenMinMax(LocationSummary* locations,
486                      bool is_min,
487                      ArmAssembler* assembler) {
488  Register op1 = locations->InAt(0).AsRegister<Register>();
489  Register op2 = locations->InAt(1).AsRegister<Register>();
490  Register out = locations->Out().AsRegister<Register>();
491
492  __ cmp(op1, ShifterOperand(op2));
493
494  __ it((is_min) ? Condition::LT : Condition::GT, kItElse);
495  __ mov(out, ShifterOperand(op1), is_min ? Condition::LT : Condition::GT);
496  __ mov(out, ShifterOperand(op2), is_min ? Condition::GE : Condition::LE);
497}
498
499static void CreateIntIntToIntLocations(ArenaAllocator* arena, HInvoke* invoke) {
500  LocationSummary* locations = new (arena) LocationSummary(invoke,
501                                                           LocationSummary::kNoCall,
502                                                           kIntrinsified);
503  locations->SetInAt(0, Location::RequiresRegister());
504  locations->SetInAt(1, Location::RequiresRegister());
505  locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
506}
507
508void IntrinsicLocationsBuilderARM::VisitMathMinIntInt(HInvoke* invoke) {
509  CreateIntIntToIntLocations(arena_, invoke);
510}
511
512void IntrinsicCodeGeneratorARM::VisitMathMinIntInt(HInvoke* invoke) {
513  GenMinMax(invoke->GetLocations(), true, GetAssembler());
514}
515
516void IntrinsicLocationsBuilderARM::VisitMathMaxIntInt(HInvoke* invoke) {
517  CreateIntIntToIntLocations(arena_, invoke);
518}
519
520void IntrinsicCodeGeneratorARM::VisitMathMaxIntInt(HInvoke* invoke) {
521  GenMinMax(invoke->GetLocations(), false, GetAssembler());
522}
523
524void IntrinsicLocationsBuilderARM::VisitMathSqrt(HInvoke* invoke) {
525  CreateFPToFPLocations(arena_, invoke);
526}
527
528void IntrinsicCodeGeneratorARM::VisitMathSqrt(HInvoke* invoke) {
529  LocationSummary* locations = invoke->GetLocations();
530  ArmAssembler* assembler = GetAssembler();
531  __ vsqrtd(FromLowSToD(locations->Out().AsFpuRegisterPairLow<SRegister>()),
532            FromLowSToD(locations->InAt(0).AsFpuRegisterPairLow<SRegister>()));
533}
534
535void IntrinsicLocationsBuilderARM::VisitMemoryPeekByte(HInvoke* invoke) {
536  CreateIntToIntLocations(arena_, invoke);
537}
538
539void IntrinsicCodeGeneratorARM::VisitMemoryPeekByte(HInvoke* invoke) {
540  ArmAssembler* assembler = GetAssembler();
541  // Ignore upper 4B of long address.
542  __ ldrsb(invoke->GetLocations()->Out().AsRegister<Register>(),
543           Address(invoke->GetLocations()->InAt(0).AsRegisterPairLow<Register>()));
544}
545
546void IntrinsicLocationsBuilderARM::VisitMemoryPeekIntNative(HInvoke* invoke) {
547  CreateIntToIntLocations(arena_, invoke);
548}
549
550void IntrinsicCodeGeneratorARM::VisitMemoryPeekIntNative(HInvoke* invoke) {
551  ArmAssembler* assembler = GetAssembler();
552  // Ignore upper 4B of long address.
553  __ ldr(invoke->GetLocations()->Out().AsRegister<Register>(),
554         Address(invoke->GetLocations()->InAt(0).AsRegisterPairLow<Register>()));
555}
556
557void IntrinsicLocationsBuilderARM::VisitMemoryPeekLongNative(HInvoke* invoke) {
558  CreateIntToIntLocations(arena_, invoke);
559}
560
561void IntrinsicCodeGeneratorARM::VisitMemoryPeekLongNative(HInvoke* invoke) {
562  ArmAssembler* assembler = GetAssembler();
563  // Ignore upper 4B of long address.
564  Register addr = invoke->GetLocations()->InAt(0).AsRegisterPairLow<Register>();
565  // Worst case: Control register bit SCTLR.A = 0. Then unaligned accesses throw a processor
566  // exception. So we can't use ldrd as addr may be unaligned.
567  Register lo = invoke->GetLocations()->Out().AsRegisterPairLow<Register>();
568  Register hi = invoke->GetLocations()->Out().AsRegisterPairHigh<Register>();
569  if (addr == lo) {
570    __ ldr(hi, Address(addr, 4));
571    __ ldr(lo, Address(addr, 0));
572  } else {
573    __ ldr(lo, Address(addr, 0));
574    __ ldr(hi, Address(addr, 4));
575  }
576}
577
578void IntrinsicLocationsBuilderARM::VisitMemoryPeekShortNative(HInvoke* invoke) {
579  CreateIntToIntLocations(arena_, invoke);
580}
581
582void IntrinsicCodeGeneratorARM::VisitMemoryPeekShortNative(HInvoke* invoke) {
583  ArmAssembler* assembler = GetAssembler();
584  // Ignore upper 4B of long address.
585  __ ldrsh(invoke->GetLocations()->Out().AsRegister<Register>(),
586           Address(invoke->GetLocations()->InAt(0).AsRegisterPairLow<Register>()));
587}
588
589static void CreateIntIntToVoidLocations(ArenaAllocator* arena, HInvoke* invoke) {
590  LocationSummary* locations = new (arena) LocationSummary(invoke,
591                                                           LocationSummary::kNoCall,
592                                                           kIntrinsified);
593  locations->SetInAt(0, Location::RequiresRegister());
594  locations->SetInAt(1, Location::RequiresRegister());
595}
596
597void IntrinsicLocationsBuilderARM::VisitMemoryPokeByte(HInvoke* invoke) {
598  CreateIntIntToVoidLocations(arena_, invoke);
599}
600
601void IntrinsicCodeGeneratorARM::VisitMemoryPokeByte(HInvoke* invoke) {
602  ArmAssembler* assembler = GetAssembler();
603  __ strb(invoke->GetLocations()->InAt(1).AsRegister<Register>(),
604          Address(invoke->GetLocations()->InAt(0).AsRegisterPairLow<Register>()));
605}
606
607void IntrinsicLocationsBuilderARM::VisitMemoryPokeIntNative(HInvoke* invoke) {
608  CreateIntIntToVoidLocations(arena_, invoke);
609}
610
611void IntrinsicCodeGeneratorARM::VisitMemoryPokeIntNative(HInvoke* invoke) {
612  ArmAssembler* assembler = GetAssembler();
613  __ str(invoke->GetLocations()->InAt(1).AsRegister<Register>(),
614         Address(invoke->GetLocations()->InAt(0).AsRegisterPairLow<Register>()));
615}
616
617void IntrinsicLocationsBuilderARM::VisitMemoryPokeLongNative(HInvoke* invoke) {
618  CreateIntIntToVoidLocations(arena_, invoke);
619}
620
621void IntrinsicCodeGeneratorARM::VisitMemoryPokeLongNative(HInvoke* invoke) {
622  ArmAssembler* assembler = GetAssembler();
623  // Ignore upper 4B of long address.
624  Register addr = invoke->GetLocations()->InAt(0).AsRegisterPairLow<Register>();
625  // Worst case: Control register bit SCTLR.A = 0. Then unaligned accesses throw a processor
626  // exception. So we can't use ldrd as addr may be unaligned.
627  __ str(invoke->GetLocations()->InAt(1).AsRegisterPairLow<Register>(), Address(addr, 0));
628  __ str(invoke->GetLocations()->InAt(1).AsRegisterPairHigh<Register>(), Address(addr, 4));
629}
630
631void IntrinsicLocationsBuilderARM::VisitMemoryPokeShortNative(HInvoke* invoke) {
632  CreateIntIntToVoidLocations(arena_, invoke);
633}
634
635void IntrinsicCodeGeneratorARM::VisitMemoryPokeShortNative(HInvoke* invoke) {
636  ArmAssembler* assembler = GetAssembler();
637  __ strh(invoke->GetLocations()->InAt(1).AsRegister<Register>(),
638          Address(invoke->GetLocations()->InAt(0).AsRegisterPairLow<Register>()));
639}
640
641void IntrinsicLocationsBuilderARM::VisitThreadCurrentThread(HInvoke* invoke) {
642  LocationSummary* locations = new (arena_) LocationSummary(invoke,
643                                                            LocationSummary::kNoCall,
644                                                            kIntrinsified);
645  locations->SetOut(Location::RequiresRegister());
646}
647
648void IntrinsicCodeGeneratorARM::VisitThreadCurrentThread(HInvoke* invoke) {
649  ArmAssembler* assembler = GetAssembler();
650  __ LoadFromOffset(kLoadWord,
651                    invoke->GetLocations()->Out().AsRegister<Register>(),
652                    TR,
653                    Thread::PeerOffset<kArmPointerSize>().Int32Value());
654}
655
656static void GenUnsafeGet(HInvoke* invoke,
657                         Primitive::Type type,
658                         bool is_volatile,
659                         CodeGeneratorARM* codegen) {
660  LocationSummary* locations = invoke->GetLocations();
661  DCHECK((type == Primitive::kPrimInt) ||
662         (type == Primitive::kPrimLong) ||
663         (type == Primitive::kPrimNot));
664  ArmAssembler* assembler = codegen->GetAssembler();
665  Register base = locations->InAt(1).AsRegister<Register>();           // Object pointer.
666  Register offset = locations->InAt(2).AsRegisterPairLow<Register>();  // Long offset, lo part only.
667
668  if (type == Primitive::kPrimLong) {
669    Register trg_lo = locations->Out().AsRegisterPairLow<Register>();
670    __ add(IP, base, ShifterOperand(offset));
671    if (is_volatile && !codegen->GetInstructionSetFeatures().HasAtomicLdrdAndStrd()) {
672      Register trg_hi = locations->Out().AsRegisterPairHigh<Register>();
673      __ ldrexd(trg_lo, trg_hi, IP);
674    } else {
675      __ ldrd(trg_lo, Address(IP));
676    }
677  } else {
678    Register trg = locations->Out().AsRegister<Register>();
679    __ ldr(trg, Address(base, offset));
680  }
681
682  if (is_volatile) {
683    __ dmb(ISH);
684  }
685
686  if (type == Primitive::kPrimNot) {
687    Register trg = locations->Out().AsRegister<Register>();
688    __ MaybeUnpoisonHeapReference(trg);
689  }
690}
691
692static void CreateIntIntIntToIntLocations(ArenaAllocator* arena, HInvoke* invoke) {
693  LocationSummary* locations = new (arena) LocationSummary(invoke,
694                                                           LocationSummary::kNoCall,
695                                                           kIntrinsified);
696  locations->SetInAt(0, Location::NoLocation());        // Unused receiver.
697  locations->SetInAt(1, Location::RequiresRegister());
698  locations->SetInAt(2, Location::RequiresRegister());
699  locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
700}
701
702void IntrinsicLocationsBuilderARM::VisitUnsafeGet(HInvoke* invoke) {
703  CreateIntIntIntToIntLocations(arena_, invoke);
704}
705void IntrinsicLocationsBuilderARM::VisitUnsafeGetVolatile(HInvoke* invoke) {
706  CreateIntIntIntToIntLocations(arena_, invoke);
707}
708void IntrinsicLocationsBuilderARM::VisitUnsafeGetLong(HInvoke* invoke) {
709  CreateIntIntIntToIntLocations(arena_, invoke);
710}
711void IntrinsicLocationsBuilderARM::VisitUnsafeGetLongVolatile(HInvoke* invoke) {
712  CreateIntIntIntToIntLocations(arena_, invoke);
713}
714void IntrinsicLocationsBuilderARM::VisitUnsafeGetObject(HInvoke* invoke) {
715  CreateIntIntIntToIntLocations(arena_, invoke);
716}
717void IntrinsicLocationsBuilderARM::VisitUnsafeGetObjectVolatile(HInvoke* invoke) {
718  CreateIntIntIntToIntLocations(arena_, invoke);
719}
720
721void IntrinsicCodeGeneratorARM::VisitUnsafeGet(HInvoke* invoke) {
722  GenUnsafeGet(invoke, Primitive::kPrimInt, false, codegen_);
723}
724void IntrinsicCodeGeneratorARM::VisitUnsafeGetVolatile(HInvoke* invoke) {
725  GenUnsafeGet(invoke, Primitive::kPrimInt, true, codegen_);
726}
727void IntrinsicCodeGeneratorARM::VisitUnsafeGetLong(HInvoke* invoke) {
728  GenUnsafeGet(invoke, Primitive::kPrimLong, false, codegen_);
729}
730void IntrinsicCodeGeneratorARM::VisitUnsafeGetLongVolatile(HInvoke* invoke) {
731  GenUnsafeGet(invoke, Primitive::kPrimLong, true, codegen_);
732}
733void IntrinsicCodeGeneratorARM::VisitUnsafeGetObject(HInvoke* invoke) {
734  GenUnsafeGet(invoke, Primitive::kPrimNot, false, codegen_);
735}
736void IntrinsicCodeGeneratorARM::VisitUnsafeGetObjectVolatile(HInvoke* invoke) {
737  GenUnsafeGet(invoke, Primitive::kPrimNot, true, codegen_);
738}
739
740static void CreateIntIntIntIntToVoid(ArenaAllocator* arena,
741                                     const ArmInstructionSetFeatures& features,
742                                     Primitive::Type type,
743                                     bool is_volatile,
744                                     HInvoke* invoke) {
745  LocationSummary* locations = new (arena) LocationSummary(invoke,
746                                                           LocationSummary::kNoCall,
747                                                           kIntrinsified);
748  locations->SetInAt(0, Location::NoLocation());        // Unused receiver.
749  locations->SetInAt(1, Location::RequiresRegister());
750  locations->SetInAt(2, Location::RequiresRegister());
751  locations->SetInAt(3, Location::RequiresRegister());
752
753  if (type == Primitive::kPrimLong) {
754    // Potentially need temps for ldrexd-strexd loop.
755    if (is_volatile && !features.HasAtomicLdrdAndStrd()) {
756      locations->AddTemp(Location::RequiresRegister());  // Temp_lo.
757      locations->AddTemp(Location::RequiresRegister());  // Temp_hi.
758    }
759  } else if (type == Primitive::kPrimNot) {
760    // Temps for card-marking.
761    locations->AddTemp(Location::RequiresRegister());  // Temp.
762    locations->AddTemp(Location::RequiresRegister());  // Card.
763  }
764}
765
766void IntrinsicLocationsBuilderARM::VisitUnsafePut(HInvoke* invoke) {
767  CreateIntIntIntIntToVoid(arena_, features_, Primitive::kPrimInt, false, invoke);
768}
769void IntrinsicLocationsBuilderARM::VisitUnsafePutOrdered(HInvoke* invoke) {
770  CreateIntIntIntIntToVoid(arena_, features_, Primitive::kPrimInt, false, invoke);
771}
772void IntrinsicLocationsBuilderARM::VisitUnsafePutVolatile(HInvoke* invoke) {
773  CreateIntIntIntIntToVoid(arena_, features_, Primitive::kPrimInt, true, invoke);
774}
775void IntrinsicLocationsBuilderARM::VisitUnsafePutObject(HInvoke* invoke) {
776  CreateIntIntIntIntToVoid(arena_, features_, Primitive::kPrimNot, false, invoke);
777}
778void IntrinsicLocationsBuilderARM::VisitUnsafePutObjectOrdered(HInvoke* invoke) {
779  CreateIntIntIntIntToVoid(arena_, features_, Primitive::kPrimNot, false, invoke);
780}
781void IntrinsicLocationsBuilderARM::VisitUnsafePutObjectVolatile(HInvoke* invoke) {
782  CreateIntIntIntIntToVoid(arena_, features_, Primitive::kPrimNot, true, invoke);
783}
784void IntrinsicLocationsBuilderARM::VisitUnsafePutLong(HInvoke* invoke) {
785  CreateIntIntIntIntToVoid(arena_, features_, Primitive::kPrimLong, false, invoke);
786}
787void IntrinsicLocationsBuilderARM::VisitUnsafePutLongOrdered(HInvoke* invoke) {
788  CreateIntIntIntIntToVoid(arena_, features_, Primitive::kPrimLong, false, invoke);
789}
790void IntrinsicLocationsBuilderARM::VisitUnsafePutLongVolatile(HInvoke* invoke) {
791  CreateIntIntIntIntToVoid(arena_, features_, Primitive::kPrimLong, true, invoke);
792}
793
794static void GenUnsafePut(LocationSummary* locations,
795                         Primitive::Type type,
796                         bool is_volatile,
797                         bool is_ordered,
798                         CodeGeneratorARM* codegen) {
799  ArmAssembler* assembler = codegen->GetAssembler();
800
801  Register base = locations->InAt(1).AsRegister<Register>();           // Object pointer.
802  Register offset = locations->InAt(2).AsRegisterPairLow<Register>();  // Long offset, lo part only.
803  Register value;
804
805  if (is_volatile || is_ordered) {
806    __ dmb(ISH);
807  }
808
809  if (type == Primitive::kPrimLong) {
810    Register value_lo = locations->InAt(3).AsRegisterPairLow<Register>();
811    value = value_lo;
812    if (is_volatile && !codegen->GetInstructionSetFeatures().HasAtomicLdrdAndStrd()) {
813      Register temp_lo = locations->GetTemp(0).AsRegister<Register>();
814      Register temp_hi = locations->GetTemp(1).AsRegister<Register>();
815      Register value_hi = locations->InAt(3).AsRegisterPairHigh<Register>();
816
817      __ add(IP, base, ShifterOperand(offset));
818      Label loop_head;
819      __ Bind(&loop_head);
820      __ ldrexd(temp_lo, temp_hi, IP);
821      __ strexd(temp_lo, value_lo, value_hi, IP);
822      __ cmp(temp_lo, ShifterOperand(0));
823      __ b(&loop_head, NE);
824    } else {
825      __ add(IP, base, ShifterOperand(offset));
826      __ strd(value_lo, Address(IP));
827    }
828  } else {
829    value = locations->InAt(3).AsRegister<Register>();
830    Register source = value;
831    if (kPoisonHeapReferences && type == Primitive::kPrimNot) {
832      Register temp = locations->GetTemp(0).AsRegister<Register>();
833      __ Mov(temp, value);
834      __ PoisonHeapReference(temp);
835      source = temp;
836    }
837    __ str(source, Address(base, offset));
838  }
839
840  if (is_volatile) {
841    __ dmb(ISH);
842  }
843
844  if (type == Primitive::kPrimNot) {
845    Register temp = locations->GetTemp(0).AsRegister<Register>();
846    Register card = locations->GetTemp(1).AsRegister<Register>();
847    bool value_can_be_null = true;  // TODO: Worth finding out this information?
848    codegen->MarkGCCard(temp, card, base, value, value_can_be_null);
849  }
850}
851
852void IntrinsicCodeGeneratorARM::VisitUnsafePut(HInvoke* invoke) {
853  GenUnsafePut(invoke->GetLocations(), Primitive::kPrimInt, false, false, codegen_);
854}
855void IntrinsicCodeGeneratorARM::VisitUnsafePutOrdered(HInvoke* invoke) {
856  GenUnsafePut(invoke->GetLocations(), Primitive::kPrimInt, false, true, codegen_);
857}
858void IntrinsicCodeGeneratorARM::VisitUnsafePutVolatile(HInvoke* invoke) {
859  GenUnsafePut(invoke->GetLocations(), Primitive::kPrimInt, true, false, codegen_);
860}
861void IntrinsicCodeGeneratorARM::VisitUnsafePutObject(HInvoke* invoke) {
862  GenUnsafePut(invoke->GetLocations(), Primitive::kPrimNot, false, false, codegen_);
863}
864void IntrinsicCodeGeneratorARM::VisitUnsafePutObjectOrdered(HInvoke* invoke) {
865  GenUnsafePut(invoke->GetLocations(), Primitive::kPrimNot, false, true, codegen_);
866}
867void IntrinsicCodeGeneratorARM::VisitUnsafePutObjectVolatile(HInvoke* invoke) {
868  GenUnsafePut(invoke->GetLocations(), Primitive::kPrimNot, true, false, codegen_);
869}
870void IntrinsicCodeGeneratorARM::VisitUnsafePutLong(HInvoke* invoke) {
871  GenUnsafePut(invoke->GetLocations(), Primitive::kPrimLong, false, false, codegen_);
872}
873void IntrinsicCodeGeneratorARM::VisitUnsafePutLongOrdered(HInvoke* invoke) {
874  GenUnsafePut(invoke->GetLocations(), Primitive::kPrimLong, false, true, codegen_);
875}
876void IntrinsicCodeGeneratorARM::VisitUnsafePutLongVolatile(HInvoke* invoke) {
877  GenUnsafePut(invoke->GetLocations(), Primitive::kPrimLong, true, false, codegen_);
878}
879
880static void CreateIntIntIntIntIntToIntPlusTemps(ArenaAllocator* arena,
881                                                HInvoke* invoke) {
882  LocationSummary* locations = new (arena) LocationSummary(invoke,
883                                                           LocationSummary::kNoCall,
884                                                           kIntrinsified);
885  locations->SetInAt(0, Location::NoLocation());        // Unused receiver.
886  locations->SetInAt(1, Location::RequiresRegister());
887  locations->SetInAt(2, Location::RequiresRegister());
888  locations->SetInAt(3, Location::RequiresRegister());
889  locations->SetInAt(4, Location::RequiresRegister());
890
891  locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap);
892
893  locations->AddTemp(Location::RequiresRegister());  // Pointer.
894  locations->AddTemp(Location::RequiresRegister());  // Temp 1.
895  locations->AddTemp(Location::RequiresRegister());  // Temp 2.
896}
897
898static void GenCas(LocationSummary* locations, Primitive::Type type, CodeGeneratorARM* codegen) {
899  DCHECK_NE(type, Primitive::kPrimLong);
900
901  ArmAssembler* assembler = codegen->GetAssembler();
902
903  Register out = locations->Out().AsRegister<Register>();              // Boolean result.
904
905  Register base = locations->InAt(1).AsRegister<Register>();           // Object pointer.
906  Register offset = locations->InAt(2).AsRegisterPairLow<Register>();  // Offset (discard high 4B).
907  Register expected_lo = locations->InAt(3).AsRegister<Register>();    // Expected.
908  Register value_lo = locations->InAt(4).AsRegister<Register>();       // Value.
909
910  Register tmp_ptr = locations->GetTemp(0).AsRegister<Register>();     // Pointer to actual memory.
911  Register tmp_lo = locations->GetTemp(1).AsRegister<Register>();      // Value in memory.
912
913  if (type == Primitive::kPrimNot) {
914    // Mark card for object assuming new value is stored. Worst case we will mark an unchanged
915    // object and scan the receiver at the next GC for nothing.
916    bool value_can_be_null = true;  // TODO: Worth finding out this information?
917    codegen->MarkGCCard(tmp_ptr, tmp_lo, base, value_lo, value_can_be_null);
918  }
919
920  // Prevent reordering with prior memory operations.
921  __ dmb(ISH);
922
923  __ add(tmp_ptr, base, ShifterOperand(offset));
924
925  if (kPoisonHeapReferences && type == Primitive::kPrimNot) {
926    codegen->GetAssembler()->PoisonHeapReference(expected_lo);
927    codegen->GetAssembler()->PoisonHeapReference(value_lo);
928  }
929
930  // do {
931  //   tmp = [r_ptr] - expected;
932  // } while (tmp == 0 && failure([r_ptr] <- r_new_value));
933  // result = tmp != 0;
934
935  Label loop_head;
936  __ Bind(&loop_head);
937
938  __ ldrex(tmp_lo, tmp_ptr);
939
940  __ subs(tmp_lo, tmp_lo, ShifterOperand(expected_lo));
941
942  __ it(EQ, ItState::kItT);
943  __ strex(tmp_lo, value_lo, tmp_ptr, EQ);
944  __ cmp(tmp_lo, ShifterOperand(1), EQ);
945
946  __ b(&loop_head, EQ);
947
948  __ dmb(ISH);
949
950  __ rsbs(out, tmp_lo, ShifterOperand(1));
951  __ it(CC);
952  __ mov(out, ShifterOperand(0), CC);
953
954  if (kPoisonHeapReferences && type == Primitive::kPrimNot) {
955    codegen->GetAssembler()->UnpoisonHeapReference(value_lo);
956    codegen->GetAssembler()->UnpoisonHeapReference(expected_lo);
957  }
958}
959
960void IntrinsicLocationsBuilderARM::VisitUnsafeCASInt(HInvoke* invoke) {
961  CreateIntIntIntIntIntToIntPlusTemps(arena_, invoke);
962}
963void IntrinsicLocationsBuilderARM::VisitUnsafeCASObject(HInvoke* invoke) {
964  CreateIntIntIntIntIntToIntPlusTemps(arena_, invoke);
965}
966void IntrinsicCodeGeneratorARM::VisitUnsafeCASInt(HInvoke* invoke) {
967  GenCas(invoke->GetLocations(), Primitive::kPrimInt, codegen_);
968}
969void IntrinsicCodeGeneratorARM::VisitUnsafeCASObject(HInvoke* invoke) {
970  GenCas(invoke->GetLocations(), Primitive::kPrimNot, codegen_);
971}
972
973void IntrinsicLocationsBuilderARM::VisitStringCharAt(HInvoke* invoke) {
974  LocationSummary* locations = new (arena_) LocationSummary(invoke,
975                                                            LocationSummary::kCallOnSlowPath,
976                                                            kIntrinsified);
977  locations->SetInAt(0, Location::RequiresRegister());
978  locations->SetInAt(1, Location::RequiresRegister());
979  locations->SetOut(Location::RequiresRegister(), Location::kOutputOverlap);
980
981  locations->AddTemp(Location::RequiresRegister());
982  locations->AddTemp(Location::RequiresRegister());
983}
984
985void IntrinsicCodeGeneratorARM::VisitStringCharAt(HInvoke* invoke) {
986  ArmAssembler* assembler = GetAssembler();
987  LocationSummary* locations = invoke->GetLocations();
988
989  // Location of reference to data array
990  const MemberOffset value_offset = mirror::String::ValueOffset();
991  // Location of count
992  const MemberOffset count_offset = mirror::String::CountOffset();
993
994  Register obj = locations->InAt(0).AsRegister<Register>();  // String object pointer.
995  Register idx = locations->InAt(1).AsRegister<Register>();  // Index of character.
996  Register out = locations->Out().AsRegister<Register>();    // Result character.
997
998  Register temp = locations->GetTemp(0).AsRegister<Register>();
999  Register array_temp = locations->GetTemp(1).AsRegister<Register>();
1000
1001  // TODO: Maybe we can support range check elimination. Overall, though, I think it's not worth
1002  //       the cost.
1003  // TODO: For simplicity, the index parameter is requested in a register, so different from Quick
1004  //       we will not optimize the code for constants (which would save a register).
1005
1006  SlowPathCode* slow_path = new (GetAllocator()) IntrinsicSlowPathARM(invoke);
1007  codegen_->AddSlowPath(slow_path);
1008
1009  __ ldr(temp, Address(obj, count_offset.Int32Value()));          // temp = str.length.
1010  codegen_->MaybeRecordImplicitNullCheck(invoke);
1011  __ cmp(idx, ShifterOperand(temp));
1012  __ b(slow_path->GetEntryLabel(), CS);
1013
1014  __ add(array_temp, obj, ShifterOperand(value_offset.Int32Value()));  // array_temp := str.value.
1015
1016  // Load the value.
1017  __ ldrh(out, Address(array_temp, idx, LSL, 1));                 // out := array_temp[idx].
1018
1019  __ Bind(slow_path->GetExitLabel());
1020}
1021
1022void IntrinsicLocationsBuilderARM::VisitStringCompareTo(HInvoke* invoke) {
1023  // The inputs plus one temp.
1024  LocationSummary* locations = new (arena_) LocationSummary(invoke,
1025                                                            LocationSummary::kCall,
1026                                                            kIntrinsified);
1027  InvokeRuntimeCallingConvention calling_convention;
1028  locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(0)));
1029  locations->SetInAt(1, Location::RegisterLocation(calling_convention.GetRegisterAt(1)));
1030  locations->SetOut(Location::RegisterLocation(R0));
1031}
1032
1033void IntrinsicCodeGeneratorARM::VisitStringCompareTo(HInvoke* invoke) {
1034  ArmAssembler* assembler = GetAssembler();
1035  LocationSummary* locations = invoke->GetLocations();
1036
1037  // Note that the null check must have been done earlier.
1038  DCHECK(!invoke->CanDoImplicitNullCheckOn(invoke->InputAt(0)));
1039
1040  Register argument = locations->InAt(1).AsRegister<Register>();
1041  __ cmp(argument, ShifterOperand(0));
1042  SlowPathCode* slow_path = new (GetAllocator()) IntrinsicSlowPathARM(invoke);
1043  codegen_->AddSlowPath(slow_path);
1044  __ b(slow_path->GetEntryLabel(), EQ);
1045
1046  __ LoadFromOffset(
1047      kLoadWord, LR, TR, QUICK_ENTRYPOINT_OFFSET(kArmWordSize, pStringCompareTo).Int32Value());
1048  __ blx(LR);
1049  __ Bind(slow_path->GetExitLabel());
1050}
1051
1052void IntrinsicLocationsBuilderARM::VisitStringEquals(HInvoke* invoke) {
1053  LocationSummary* locations = new (arena_) LocationSummary(invoke,
1054                                                            LocationSummary::kNoCall,
1055                                                            kIntrinsified);
1056  InvokeRuntimeCallingConvention calling_convention;
1057  locations->SetInAt(0, Location::RequiresRegister());
1058  locations->SetInAt(1, Location::RequiresRegister());
1059  // Temporary registers to store lengths of strings and for calculations.
1060  // Using instruction cbz requires a low register, so explicitly set a temp to be R0.
1061  locations->AddTemp(Location::RegisterLocation(R0));
1062  locations->AddTemp(Location::RequiresRegister());
1063  locations->AddTemp(Location::RequiresRegister());
1064
1065  locations->SetOut(Location::RequiresRegister());
1066}
1067
1068void IntrinsicCodeGeneratorARM::VisitStringEquals(HInvoke* invoke) {
1069  ArmAssembler* assembler = GetAssembler();
1070  LocationSummary* locations = invoke->GetLocations();
1071
1072  Register str = locations->InAt(0).AsRegister<Register>();
1073  Register arg = locations->InAt(1).AsRegister<Register>();
1074  Register out = locations->Out().AsRegister<Register>();
1075
1076  Register temp = locations->GetTemp(0).AsRegister<Register>();
1077  Register temp1 = locations->GetTemp(1).AsRegister<Register>();
1078  Register temp2 = locations->GetTemp(2).AsRegister<Register>();
1079
1080  Label loop;
1081  Label end;
1082  Label return_true;
1083  Label return_false;
1084
1085  // Get offsets of count, value, and class fields within a string object.
1086  const uint32_t count_offset = mirror::String::CountOffset().Uint32Value();
1087  const uint32_t value_offset = mirror::String::ValueOffset().Uint32Value();
1088  const uint32_t class_offset = mirror::Object::ClassOffset().Uint32Value();
1089
1090  // Note that the null check must have been done earlier.
1091  DCHECK(!invoke->CanDoImplicitNullCheckOn(invoke->InputAt(0)));
1092
1093  // Check if input is null, return false if it is.
1094  __ CompareAndBranchIfZero(arg, &return_false);
1095
1096  // Instanceof check for the argument by comparing class fields.
1097  // All string objects must have the same type since String cannot be subclassed.
1098  // Receiver must be a string object, so its class field is equal to all strings' class fields.
1099  // If the argument is a string object, its class field must be equal to receiver's class field.
1100  __ ldr(temp, Address(str, class_offset));
1101  __ ldr(temp1, Address(arg, class_offset));
1102  __ cmp(temp, ShifterOperand(temp1));
1103  __ b(&return_false, NE);
1104
1105  // Load lengths of this and argument strings.
1106  __ ldr(temp, Address(str, count_offset));
1107  __ ldr(temp1, Address(arg, count_offset));
1108  // Check if lengths are equal, return false if they're not.
1109  __ cmp(temp, ShifterOperand(temp1));
1110  __ b(&return_false, NE);
1111  // Return true if both strings are empty.
1112  __ cbz(temp, &return_true);
1113
1114  // Reference equality check, return true if same reference.
1115  __ cmp(str, ShifterOperand(arg));
1116  __ b(&return_true, EQ);
1117
1118  // Assertions that must hold in order to compare strings 2 characters at a time.
1119  DCHECK_ALIGNED(value_offset, 4);
1120  static_assert(IsAligned<4>(kObjectAlignment), "String of odd length is not zero padded");
1121
1122  __ LoadImmediate(temp1, value_offset);
1123
1124  // Loop to compare strings 2 characters at a time starting at the front of the string.
1125  // Ok to do this because strings with an odd length are zero-padded.
1126  __ Bind(&loop);
1127  __ ldr(out, Address(str, temp1));
1128  __ ldr(temp2, Address(arg, temp1));
1129  __ cmp(out, ShifterOperand(temp2));
1130  __ b(&return_false, NE);
1131  __ add(temp1, temp1, ShifterOperand(sizeof(uint32_t)));
1132  __ subs(temp, temp, ShifterOperand(sizeof(uint32_t) /  sizeof(uint16_t)));
1133  __ b(&loop, GT);
1134
1135  // Return true and exit the function.
1136  // If loop does not result in returning false, we return true.
1137  __ Bind(&return_true);
1138  __ LoadImmediate(out, 1);
1139  __ b(&end);
1140
1141  // Return false and exit the function.
1142  __ Bind(&return_false);
1143  __ LoadImmediate(out, 0);
1144  __ Bind(&end);
1145}
1146
1147static void GenerateVisitStringIndexOf(HInvoke* invoke,
1148                                       ArmAssembler* assembler,
1149                                       CodeGeneratorARM* codegen,
1150                                       ArenaAllocator* allocator,
1151                                       bool start_at_zero) {
1152  LocationSummary* locations = invoke->GetLocations();
1153  Register tmp_reg = locations->GetTemp(0).AsRegister<Register>();
1154
1155  // Note that the null check must have been done earlier.
1156  DCHECK(!invoke->CanDoImplicitNullCheckOn(invoke->InputAt(0)));
1157
1158  // Check for code points > 0xFFFF. Either a slow-path check when we don't know statically,
1159  // or directly dispatch if we have a constant.
1160  SlowPathCode* slow_path = nullptr;
1161  if (invoke->InputAt(1)->IsIntConstant()) {
1162    if (static_cast<uint32_t>(invoke->InputAt(1)->AsIntConstant()->GetValue()) >
1163        std::numeric_limits<uint16_t>::max()) {
1164      // Always needs the slow-path. We could directly dispatch to it, but this case should be
1165      // rare, so for simplicity just put the full slow-path down and branch unconditionally.
1166      slow_path = new (allocator) IntrinsicSlowPathARM(invoke);
1167      codegen->AddSlowPath(slow_path);
1168      __ b(slow_path->GetEntryLabel());
1169      __ Bind(slow_path->GetExitLabel());
1170      return;
1171    }
1172  } else {
1173    Register char_reg = locations->InAt(1).AsRegister<Register>();
1174    __ LoadImmediate(tmp_reg, std::numeric_limits<uint16_t>::max());
1175    __ cmp(char_reg, ShifterOperand(tmp_reg));
1176    slow_path = new (allocator) IntrinsicSlowPathARM(invoke);
1177    codegen->AddSlowPath(slow_path);
1178    __ b(slow_path->GetEntryLabel(), HI);
1179  }
1180
1181  if (start_at_zero) {
1182    DCHECK_EQ(tmp_reg, R2);
1183    // Start-index = 0.
1184    __ LoadImmediate(tmp_reg, 0);
1185  }
1186
1187  __ LoadFromOffset(kLoadWord, LR, TR,
1188                    QUICK_ENTRYPOINT_OFFSET(kArmWordSize, pIndexOf).Int32Value());
1189  __ blx(LR);
1190
1191  if (slow_path != nullptr) {
1192    __ Bind(slow_path->GetExitLabel());
1193  }
1194}
1195
1196void IntrinsicLocationsBuilderARM::VisitStringIndexOf(HInvoke* invoke) {
1197  LocationSummary* locations = new (arena_) LocationSummary(invoke,
1198                                                            LocationSummary::kCall,
1199                                                            kIntrinsified);
1200  // We have a hand-crafted assembly stub that follows the runtime calling convention. So it's
1201  // best to align the inputs accordingly.
1202  InvokeRuntimeCallingConvention calling_convention;
1203  locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(0)));
1204  locations->SetInAt(1, Location::RegisterLocation(calling_convention.GetRegisterAt(1)));
1205  locations->SetOut(Location::RegisterLocation(R0));
1206
1207  // Need a temp for slow-path codepoint compare, and need to send start-index=0.
1208  locations->AddTemp(Location::RegisterLocation(calling_convention.GetRegisterAt(2)));
1209}
1210
1211void IntrinsicCodeGeneratorARM::VisitStringIndexOf(HInvoke* invoke) {
1212  GenerateVisitStringIndexOf(invoke, GetAssembler(), codegen_, GetAllocator(), true);
1213}
1214
1215void IntrinsicLocationsBuilderARM::VisitStringIndexOfAfter(HInvoke* invoke) {
1216  LocationSummary* locations = new (arena_) LocationSummary(invoke,
1217                                                            LocationSummary::kCall,
1218                                                            kIntrinsified);
1219  // We have a hand-crafted assembly stub that follows the runtime calling convention. So it's
1220  // best to align the inputs accordingly.
1221  InvokeRuntimeCallingConvention calling_convention;
1222  locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(0)));
1223  locations->SetInAt(1, Location::RegisterLocation(calling_convention.GetRegisterAt(1)));
1224  locations->SetInAt(2, Location::RegisterLocation(calling_convention.GetRegisterAt(2)));
1225  locations->SetOut(Location::RegisterLocation(R0));
1226
1227  // Need a temp for slow-path codepoint compare.
1228  locations->AddTemp(Location::RequiresRegister());
1229}
1230
1231void IntrinsicCodeGeneratorARM::VisitStringIndexOfAfter(HInvoke* invoke) {
1232  GenerateVisitStringIndexOf(invoke, GetAssembler(), codegen_, GetAllocator(), false);
1233}
1234
1235void IntrinsicLocationsBuilderARM::VisitStringNewStringFromBytes(HInvoke* invoke) {
1236  LocationSummary* locations = new (arena_) LocationSummary(invoke,
1237                                                            LocationSummary::kCall,
1238                                                            kIntrinsified);
1239  InvokeRuntimeCallingConvention calling_convention;
1240  locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(0)));
1241  locations->SetInAt(1, Location::RegisterLocation(calling_convention.GetRegisterAt(1)));
1242  locations->SetInAt(2, Location::RegisterLocation(calling_convention.GetRegisterAt(2)));
1243  locations->SetInAt(3, Location::RegisterLocation(calling_convention.GetRegisterAt(3)));
1244  locations->SetOut(Location::RegisterLocation(R0));
1245}
1246
1247void IntrinsicCodeGeneratorARM::VisitStringNewStringFromBytes(HInvoke* invoke) {
1248  ArmAssembler* assembler = GetAssembler();
1249  LocationSummary* locations = invoke->GetLocations();
1250
1251  Register byte_array = locations->InAt(0).AsRegister<Register>();
1252  __ cmp(byte_array, ShifterOperand(0));
1253  SlowPathCode* slow_path = new (GetAllocator()) IntrinsicSlowPathARM(invoke);
1254  codegen_->AddSlowPath(slow_path);
1255  __ b(slow_path->GetEntryLabel(), EQ);
1256
1257  __ LoadFromOffset(
1258      kLoadWord, LR, TR, QUICK_ENTRYPOINT_OFFSET(kArmWordSize, pAllocStringFromBytes).Int32Value());
1259  codegen_->RecordPcInfo(invoke, invoke->GetDexPc());
1260  __ blx(LR);
1261  __ Bind(slow_path->GetExitLabel());
1262}
1263
1264void IntrinsicLocationsBuilderARM::VisitStringNewStringFromChars(HInvoke* invoke) {
1265  LocationSummary* locations = new (arena_) LocationSummary(invoke,
1266                                                            LocationSummary::kCall,
1267                                                            kIntrinsified);
1268  InvokeRuntimeCallingConvention calling_convention;
1269  locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(0)));
1270  locations->SetInAt(1, Location::RegisterLocation(calling_convention.GetRegisterAt(1)));
1271  locations->SetInAt(2, Location::RegisterLocation(calling_convention.GetRegisterAt(2)));
1272  locations->SetOut(Location::RegisterLocation(R0));
1273}
1274
1275void IntrinsicCodeGeneratorARM::VisitStringNewStringFromChars(HInvoke* invoke) {
1276  ArmAssembler* assembler = GetAssembler();
1277
1278  __ LoadFromOffset(
1279      kLoadWord, LR, TR, QUICK_ENTRYPOINT_OFFSET(kArmWordSize, pAllocStringFromChars).Int32Value());
1280  codegen_->RecordPcInfo(invoke, invoke->GetDexPc());
1281  __ blx(LR);
1282}
1283
1284void IntrinsicLocationsBuilderARM::VisitStringNewStringFromString(HInvoke* invoke) {
1285  LocationSummary* locations = new (arena_) LocationSummary(invoke,
1286                                                            LocationSummary::kCall,
1287                                                            kIntrinsified);
1288  InvokeRuntimeCallingConvention calling_convention;
1289  locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(0)));
1290  locations->SetOut(Location::RegisterLocation(R0));
1291}
1292
1293void IntrinsicCodeGeneratorARM::VisitStringNewStringFromString(HInvoke* invoke) {
1294  ArmAssembler* assembler = GetAssembler();
1295  LocationSummary* locations = invoke->GetLocations();
1296
1297  Register string_to_copy = locations->InAt(0).AsRegister<Register>();
1298  __ cmp(string_to_copy, ShifterOperand(0));
1299  SlowPathCode* slow_path = new (GetAllocator()) IntrinsicSlowPathARM(invoke);
1300  codegen_->AddSlowPath(slow_path);
1301  __ b(slow_path->GetEntryLabel(), EQ);
1302
1303  __ LoadFromOffset(kLoadWord,
1304      LR, TR, QUICK_ENTRYPOINT_OFFSET(kArmWordSize, pAllocStringFromString).Int32Value());
1305  codegen_->RecordPcInfo(invoke, invoke->GetDexPc());
1306  __ blx(LR);
1307  __ Bind(slow_path->GetExitLabel());
1308}
1309
1310// Unimplemented intrinsics.
1311
1312#define UNIMPLEMENTED_INTRINSIC(Name)                                                  \
1313void IntrinsicLocationsBuilderARM::Visit ## Name(HInvoke* invoke ATTRIBUTE_UNUSED) { \
1314}                                                                                      \
1315void IntrinsicCodeGeneratorARM::Visit ## Name(HInvoke* invoke ATTRIBUTE_UNUSED) {    \
1316}
1317
1318UNIMPLEMENTED_INTRINSIC(IntegerReverse)
1319UNIMPLEMENTED_INTRINSIC(IntegerReverseBytes)
1320UNIMPLEMENTED_INTRINSIC(LongReverse)
1321UNIMPLEMENTED_INTRINSIC(LongReverseBytes)
1322UNIMPLEMENTED_INTRINSIC(ShortReverseBytes)
1323UNIMPLEMENTED_INTRINSIC(MathMinDoubleDouble)
1324UNIMPLEMENTED_INTRINSIC(MathMinFloatFloat)
1325UNIMPLEMENTED_INTRINSIC(MathMaxDoubleDouble)
1326UNIMPLEMENTED_INTRINSIC(MathMaxFloatFloat)
1327UNIMPLEMENTED_INTRINSIC(MathMinLongLong)
1328UNIMPLEMENTED_INTRINSIC(MathMaxLongLong)
1329UNIMPLEMENTED_INTRINSIC(MathCeil)          // Could be done by changing rounding mode, maybe?
1330UNIMPLEMENTED_INTRINSIC(MathFloor)         // Could be done by changing rounding mode, maybe?
1331UNIMPLEMENTED_INTRINSIC(MathRint)
1332UNIMPLEMENTED_INTRINSIC(MathRoundDouble)   // Could be done by changing rounding mode, maybe?
1333UNIMPLEMENTED_INTRINSIC(MathRoundFloat)    // Could be done by changing rounding mode, maybe?
1334UNIMPLEMENTED_INTRINSIC(UnsafeCASLong)     // High register pressure.
1335UNIMPLEMENTED_INTRINSIC(SystemArrayCopyChar)
1336UNIMPLEMENTED_INTRINSIC(ReferenceGetReferent)
1337UNIMPLEMENTED_INTRINSIC(StringGetCharsNoCheck)
1338
1339#undef UNIMPLEMENTED_INTRINSIC
1340
1341#undef __
1342
1343}  // namespace arm
1344}  // namespace art
1345