intrinsics_arm.cc revision 4bedb3845ac33c95cb779987abd4e76a88b19989
1/* 2 * Copyright (C) 2015 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17#include "intrinsics_arm.h" 18 19#include "arch/arm/instruction_set_features_arm.h" 20#include "art_method.h" 21#include "code_generator_arm.h" 22#include "entrypoints/quick/quick_entrypoints.h" 23#include "intrinsics.h" 24#include "intrinsics_utils.h" 25#include "mirror/array-inl.h" 26#include "mirror/string.h" 27#include "thread.h" 28#include "utils/arm/assembler_arm.h" 29 30namespace art { 31 32namespace arm { 33 34ArmAssembler* IntrinsicCodeGeneratorARM::GetAssembler() { 35 return codegen_->GetAssembler(); 36} 37 38ArenaAllocator* IntrinsicCodeGeneratorARM::GetAllocator() { 39 return codegen_->GetGraph()->GetArena(); 40} 41 42using IntrinsicSlowPathARM = IntrinsicSlowPath<InvokeDexCallingConventionVisitorARM>; 43 44bool IntrinsicLocationsBuilderARM::TryDispatch(HInvoke* invoke) { 45 Dispatch(invoke); 46 LocationSummary* res = invoke->GetLocations(); 47 if (res == nullptr) { 48 return false; 49 } 50 if (kEmitCompilerReadBarrier && res->CanCall()) { 51 // Generating an intrinsic for this HInvoke may produce an 52 // IntrinsicSlowPathARM slow path. Currently this approach 53 // does not work when using read barriers, as the emitted 54 // calling sequence will make use of another slow path 55 // (ReadBarrierForRootSlowPathARM for HInvokeStaticOrDirect, 56 // ReadBarrierSlowPathARM for HInvokeVirtual). So we bail 57 // out in this case. 58 // 59 // TODO: Find a way to have intrinsics work with read barriers. 60 invoke->SetLocations(nullptr); 61 return false; 62 } 63 return res->Intrinsified(); 64} 65 66#define __ assembler-> 67 68static void CreateFPToIntLocations(ArenaAllocator* arena, HInvoke* invoke) { 69 LocationSummary* locations = new (arena) LocationSummary(invoke, 70 LocationSummary::kNoCall, 71 kIntrinsified); 72 locations->SetInAt(0, Location::RequiresFpuRegister()); 73 locations->SetOut(Location::RequiresRegister()); 74} 75 76static void CreateIntToFPLocations(ArenaAllocator* arena, HInvoke* invoke) { 77 LocationSummary* locations = new (arena) LocationSummary(invoke, 78 LocationSummary::kNoCall, 79 kIntrinsified); 80 locations->SetInAt(0, Location::RequiresRegister()); 81 locations->SetOut(Location::RequiresFpuRegister()); 82} 83 84static void MoveFPToInt(LocationSummary* locations, bool is64bit, ArmAssembler* assembler) { 85 Location input = locations->InAt(0); 86 Location output = locations->Out(); 87 if (is64bit) { 88 __ vmovrrd(output.AsRegisterPairLow<Register>(), 89 output.AsRegisterPairHigh<Register>(), 90 FromLowSToD(input.AsFpuRegisterPairLow<SRegister>())); 91 } else { 92 __ vmovrs(output.AsRegister<Register>(), input.AsFpuRegister<SRegister>()); 93 } 94} 95 96static void MoveIntToFP(LocationSummary* locations, bool is64bit, ArmAssembler* assembler) { 97 Location input = locations->InAt(0); 98 Location output = locations->Out(); 99 if (is64bit) { 100 __ vmovdrr(FromLowSToD(output.AsFpuRegisterPairLow<SRegister>()), 101 input.AsRegisterPairLow<Register>(), 102 input.AsRegisterPairHigh<Register>()); 103 } else { 104 __ vmovsr(output.AsFpuRegister<SRegister>(), input.AsRegister<Register>()); 105 } 106} 107 108void IntrinsicLocationsBuilderARM::VisitDoubleDoubleToRawLongBits(HInvoke* invoke) { 109 CreateFPToIntLocations(arena_, invoke); 110} 111void IntrinsicLocationsBuilderARM::VisitDoubleLongBitsToDouble(HInvoke* invoke) { 112 CreateIntToFPLocations(arena_, invoke); 113} 114 115void IntrinsicCodeGeneratorARM::VisitDoubleDoubleToRawLongBits(HInvoke* invoke) { 116 MoveFPToInt(invoke->GetLocations(), /* is64bit */ true, GetAssembler()); 117} 118void IntrinsicCodeGeneratorARM::VisitDoubleLongBitsToDouble(HInvoke* invoke) { 119 MoveIntToFP(invoke->GetLocations(), /* is64bit */ true, GetAssembler()); 120} 121 122void IntrinsicLocationsBuilderARM::VisitFloatFloatToRawIntBits(HInvoke* invoke) { 123 CreateFPToIntLocations(arena_, invoke); 124} 125void IntrinsicLocationsBuilderARM::VisitFloatIntBitsToFloat(HInvoke* invoke) { 126 CreateIntToFPLocations(arena_, invoke); 127} 128 129void IntrinsicCodeGeneratorARM::VisitFloatFloatToRawIntBits(HInvoke* invoke) { 130 MoveFPToInt(invoke->GetLocations(), /* is64bit */ false, GetAssembler()); 131} 132void IntrinsicCodeGeneratorARM::VisitFloatIntBitsToFloat(HInvoke* invoke) { 133 MoveIntToFP(invoke->GetLocations(), /* is64bit */ false, GetAssembler()); 134} 135 136static void CreateIntToIntLocations(ArenaAllocator* arena, HInvoke* invoke) { 137 LocationSummary* locations = new (arena) LocationSummary(invoke, 138 LocationSummary::kNoCall, 139 kIntrinsified); 140 locations->SetInAt(0, Location::RequiresRegister()); 141 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 142} 143 144static void CreateFPToFPLocations(ArenaAllocator* arena, HInvoke* invoke) { 145 LocationSummary* locations = new (arena) LocationSummary(invoke, 146 LocationSummary::kNoCall, 147 kIntrinsified); 148 locations->SetInAt(0, Location::RequiresFpuRegister()); 149 locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap); 150} 151 152static void GenNumberOfLeadingZeros(LocationSummary* locations, 153 Primitive::Type type, 154 ArmAssembler* assembler) { 155 Location in = locations->InAt(0); 156 Register out = locations->Out().AsRegister<Register>(); 157 158 DCHECK((type == Primitive::kPrimInt) || (type == Primitive::kPrimLong)); 159 160 if (type == Primitive::kPrimLong) { 161 Register in_reg_lo = in.AsRegisterPairLow<Register>(); 162 Register in_reg_hi = in.AsRegisterPairHigh<Register>(); 163 Label end; 164 __ clz(out, in_reg_hi); 165 __ CompareAndBranchIfNonZero(in_reg_hi, &end); 166 __ clz(out, in_reg_lo); 167 __ AddConstant(out, 32); 168 __ Bind(&end); 169 } else { 170 __ clz(out, in.AsRegister<Register>()); 171 } 172} 173 174void IntrinsicLocationsBuilderARM::VisitIntegerNumberOfLeadingZeros(HInvoke* invoke) { 175 CreateIntToIntLocations(arena_, invoke); 176} 177 178void IntrinsicCodeGeneratorARM::VisitIntegerNumberOfLeadingZeros(HInvoke* invoke) { 179 GenNumberOfLeadingZeros(invoke->GetLocations(), Primitive::kPrimInt, GetAssembler()); 180} 181 182void IntrinsicLocationsBuilderARM::VisitLongNumberOfLeadingZeros(HInvoke* invoke) { 183 LocationSummary* locations = new (arena_) LocationSummary(invoke, 184 LocationSummary::kNoCall, 185 kIntrinsified); 186 locations->SetInAt(0, Location::RequiresRegister()); 187 locations->SetOut(Location::RequiresRegister(), Location::kOutputOverlap); 188} 189 190void IntrinsicCodeGeneratorARM::VisitLongNumberOfLeadingZeros(HInvoke* invoke) { 191 GenNumberOfLeadingZeros(invoke->GetLocations(), Primitive::kPrimLong, GetAssembler()); 192} 193 194static void GenNumberOfTrailingZeros(LocationSummary* locations, 195 Primitive::Type type, 196 ArmAssembler* assembler) { 197 DCHECK((type == Primitive::kPrimInt) || (type == Primitive::kPrimLong)); 198 199 Register out = locations->Out().AsRegister<Register>(); 200 201 if (type == Primitive::kPrimLong) { 202 Register in_reg_lo = locations->InAt(0).AsRegisterPairLow<Register>(); 203 Register in_reg_hi = locations->InAt(0).AsRegisterPairHigh<Register>(); 204 Label end; 205 __ rbit(out, in_reg_lo); 206 __ clz(out, out); 207 __ CompareAndBranchIfNonZero(in_reg_lo, &end); 208 __ rbit(out, in_reg_hi); 209 __ clz(out, out); 210 __ AddConstant(out, 32); 211 __ Bind(&end); 212 } else { 213 Register in = locations->InAt(0).AsRegister<Register>(); 214 __ rbit(out, in); 215 __ clz(out, out); 216 } 217} 218 219void IntrinsicLocationsBuilderARM::VisitIntegerNumberOfTrailingZeros(HInvoke* invoke) { 220 LocationSummary* locations = new (arena_) LocationSummary(invoke, 221 LocationSummary::kNoCall, 222 kIntrinsified); 223 locations->SetInAt(0, Location::RequiresRegister()); 224 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 225} 226 227void IntrinsicCodeGeneratorARM::VisitIntegerNumberOfTrailingZeros(HInvoke* invoke) { 228 GenNumberOfTrailingZeros(invoke->GetLocations(), Primitive::kPrimInt, GetAssembler()); 229} 230 231void IntrinsicLocationsBuilderARM::VisitLongNumberOfTrailingZeros(HInvoke* invoke) { 232 LocationSummary* locations = new (arena_) LocationSummary(invoke, 233 LocationSummary::kNoCall, 234 kIntrinsified); 235 locations->SetInAt(0, Location::RequiresRegister()); 236 locations->SetOut(Location::RequiresRegister(), Location::kOutputOverlap); 237} 238 239void IntrinsicCodeGeneratorARM::VisitLongNumberOfTrailingZeros(HInvoke* invoke) { 240 GenNumberOfTrailingZeros(invoke->GetLocations(), Primitive::kPrimLong, GetAssembler()); 241} 242 243static void MathAbsFP(LocationSummary* locations, bool is64bit, ArmAssembler* assembler) { 244 Location in = locations->InAt(0); 245 Location out = locations->Out(); 246 247 if (is64bit) { 248 __ vabsd(FromLowSToD(out.AsFpuRegisterPairLow<SRegister>()), 249 FromLowSToD(in.AsFpuRegisterPairLow<SRegister>())); 250 } else { 251 __ vabss(out.AsFpuRegister<SRegister>(), in.AsFpuRegister<SRegister>()); 252 } 253} 254 255void IntrinsicLocationsBuilderARM::VisitMathAbsDouble(HInvoke* invoke) { 256 CreateFPToFPLocations(arena_, invoke); 257} 258 259void IntrinsicCodeGeneratorARM::VisitMathAbsDouble(HInvoke* invoke) { 260 MathAbsFP(invoke->GetLocations(), /* is64bit */ true, GetAssembler()); 261} 262 263void IntrinsicLocationsBuilderARM::VisitMathAbsFloat(HInvoke* invoke) { 264 CreateFPToFPLocations(arena_, invoke); 265} 266 267void IntrinsicCodeGeneratorARM::VisitMathAbsFloat(HInvoke* invoke) { 268 MathAbsFP(invoke->GetLocations(), /* is64bit */ false, GetAssembler()); 269} 270 271static void CreateIntToIntPlusTemp(ArenaAllocator* arena, HInvoke* invoke) { 272 LocationSummary* locations = new (arena) LocationSummary(invoke, 273 LocationSummary::kNoCall, 274 kIntrinsified); 275 locations->SetInAt(0, Location::RequiresRegister()); 276 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 277 278 locations->AddTemp(Location::RequiresRegister()); 279} 280 281static void GenAbsInteger(LocationSummary* locations, 282 bool is64bit, 283 ArmAssembler* assembler) { 284 Location in = locations->InAt(0); 285 Location output = locations->Out(); 286 287 Register mask = locations->GetTemp(0).AsRegister<Register>(); 288 289 if (is64bit) { 290 Register in_reg_lo = in.AsRegisterPairLow<Register>(); 291 Register in_reg_hi = in.AsRegisterPairHigh<Register>(); 292 Register out_reg_lo = output.AsRegisterPairLow<Register>(); 293 Register out_reg_hi = output.AsRegisterPairHigh<Register>(); 294 295 DCHECK_NE(out_reg_lo, in_reg_hi) << "Diagonal overlap unexpected."; 296 297 __ Asr(mask, in_reg_hi, 31); 298 __ adds(out_reg_lo, in_reg_lo, ShifterOperand(mask)); 299 __ adc(out_reg_hi, in_reg_hi, ShifterOperand(mask)); 300 __ eor(out_reg_lo, mask, ShifterOperand(out_reg_lo)); 301 __ eor(out_reg_hi, mask, ShifterOperand(out_reg_hi)); 302 } else { 303 Register in_reg = in.AsRegister<Register>(); 304 Register out_reg = output.AsRegister<Register>(); 305 306 __ Asr(mask, in_reg, 31); 307 __ add(out_reg, in_reg, ShifterOperand(mask)); 308 __ eor(out_reg, mask, ShifterOperand(out_reg)); 309 } 310} 311 312void IntrinsicLocationsBuilderARM::VisitMathAbsInt(HInvoke* invoke) { 313 CreateIntToIntPlusTemp(arena_, invoke); 314} 315 316void IntrinsicCodeGeneratorARM::VisitMathAbsInt(HInvoke* invoke) { 317 GenAbsInteger(invoke->GetLocations(), /* is64bit */ false, GetAssembler()); 318} 319 320 321void IntrinsicLocationsBuilderARM::VisitMathAbsLong(HInvoke* invoke) { 322 CreateIntToIntPlusTemp(arena_, invoke); 323} 324 325void IntrinsicCodeGeneratorARM::VisitMathAbsLong(HInvoke* invoke) { 326 GenAbsInteger(invoke->GetLocations(), /* is64bit */ true, GetAssembler()); 327} 328 329static void GenMinMax(LocationSummary* locations, 330 bool is_min, 331 ArmAssembler* assembler) { 332 Register op1 = locations->InAt(0).AsRegister<Register>(); 333 Register op2 = locations->InAt(1).AsRegister<Register>(); 334 Register out = locations->Out().AsRegister<Register>(); 335 336 __ cmp(op1, ShifterOperand(op2)); 337 338 __ it((is_min) ? Condition::LT : Condition::GT, kItElse); 339 __ mov(out, ShifterOperand(op1), is_min ? Condition::LT : Condition::GT); 340 __ mov(out, ShifterOperand(op2), is_min ? Condition::GE : Condition::LE); 341} 342 343static void CreateIntIntToIntLocations(ArenaAllocator* arena, HInvoke* invoke) { 344 LocationSummary* locations = new (arena) LocationSummary(invoke, 345 LocationSummary::kNoCall, 346 kIntrinsified); 347 locations->SetInAt(0, Location::RequiresRegister()); 348 locations->SetInAt(1, Location::RequiresRegister()); 349 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 350} 351 352void IntrinsicLocationsBuilderARM::VisitMathMinIntInt(HInvoke* invoke) { 353 CreateIntIntToIntLocations(arena_, invoke); 354} 355 356void IntrinsicCodeGeneratorARM::VisitMathMinIntInt(HInvoke* invoke) { 357 GenMinMax(invoke->GetLocations(), /* is_min */ true, GetAssembler()); 358} 359 360void IntrinsicLocationsBuilderARM::VisitMathMaxIntInt(HInvoke* invoke) { 361 CreateIntIntToIntLocations(arena_, invoke); 362} 363 364void IntrinsicCodeGeneratorARM::VisitMathMaxIntInt(HInvoke* invoke) { 365 GenMinMax(invoke->GetLocations(), /* is_min */ false, GetAssembler()); 366} 367 368void IntrinsicLocationsBuilderARM::VisitMathSqrt(HInvoke* invoke) { 369 CreateFPToFPLocations(arena_, invoke); 370} 371 372void IntrinsicCodeGeneratorARM::VisitMathSqrt(HInvoke* invoke) { 373 LocationSummary* locations = invoke->GetLocations(); 374 ArmAssembler* assembler = GetAssembler(); 375 __ vsqrtd(FromLowSToD(locations->Out().AsFpuRegisterPairLow<SRegister>()), 376 FromLowSToD(locations->InAt(0).AsFpuRegisterPairLow<SRegister>())); 377} 378 379void IntrinsicLocationsBuilderARM::VisitMemoryPeekByte(HInvoke* invoke) { 380 CreateIntToIntLocations(arena_, invoke); 381} 382 383void IntrinsicCodeGeneratorARM::VisitMemoryPeekByte(HInvoke* invoke) { 384 ArmAssembler* assembler = GetAssembler(); 385 // Ignore upper 4B of long address. 386 __ ldrsb(invoke->GetLocations()->Out().AsRegister<Register>(), 387 Address(invoke->GetLocations()->InAt(0).AsRegisterPairLow<Register>())); 388} 389 390void IntrinsicLocationsBuilderARM::VisitMemoryPeekIntNative(HInvoke* invoke) { 391 CreateIntToIntLocations(arena_, invoke); 392} 393 394void IntrinsicCodeGeneratorARM::VisitMemoryPeekIntNative(HInvoke* invoke) { 395 ArmAssembler* assembler = GetAssembler(); 396 // Ignore upper 4B of long address. 397 __ ldr(invoke->GetLocations()->Out().AsRegister<Register>(), 398 Address(invoke->GetLocations()->InAt(0).AsRegisterPairLow<Register>())); 399} 400 401void IntrinsicLocationsBuilderARM::VisitMemoryPeekLongNative(HInvoke* invoke) { 402 CreateIntToIntLocations(arena_, invoke); 403} 404 405void IntrinsicCodeGeneratorARM::VisitMemoryPeekLongNative(HInvoke* invoke) { 406 ArmAssembler* assembler = GetAssembler(); 407 // Ignore upper 4B of long address. 408 Register addr = invoke->GetLocations()->InAt(0).AsRegisterPairLow<Register>(); 409 // Worst case: Control register bit SCTLR.A = 0. Then unaligned accesses throw a processor 410 // exception. So we can't use ldrd as addr may be unaligned. 411 Register lo = invoke->GetLocations()->Out().AsRegisterPairLow<Register>(); 412 Register hi = invoke->GetLocations()->Out().AsRegisterPairHigh<Register>(); 413 if (addr == lo) { 414 __ ldr(hi, Address(addr, 4)); 415 __ ldr(lo, Address(addr, 0)); 416 } else { 417 __ ldr(lo, Address(addr, 0)); 418 __ ldr(hi, Address(addr, 4)); 419 } 420} 421 422void IntrinsicLocationsBuilderARM::VisitMemoryPeekShortNative(HInvoke* invoke) { 423 CreateIntToIntLocations(arena_, invoke); 424} 425 426void IntrinsicCodeGeneratorARM::VisitMemoryPeekShortNative(HInvoke* invoke) { 427 ArmAssembler* assembler = GetAssembler(); 428 // Ignore upper 4B of long address. 429 __ ldrsh(invoke->GetLocations()->Out().AsRegister<Register>(), 430 Address(invoke->GetLocations()->InAt(0).AsRegisterPairLow<Register>())); 431} 432 433static void CreateIntIntToVoidLocations(ArenaAllocator* arena, HInvoke* invoke) { 434 LocationSummary* locations = new (arena) LocationSummary(invoke, 435 LocationSummary::kNoCall, 436 kIntrinsified); 437 locations->SetInAt(0, Location::RequiresRegister()); 438 locations->SetInAt(1, Location::RequiresRegister()); 439} 440 441void IntrinsicLocationsBuilderARM::VisitMemoryPokeByte(HInvoke* invoke) { 442 CreateIntIntToVoidLocations(arena_, invoke); 443} 444 445void IntrinsicCodeGeneratorARM::VisitMemoryPokeByte(HInvoke* invoke) { 446 ArmAssembler* assembler = GetAssembler(); 447 __ strb(invoke->GetLocations()->InAt(1).AsRegister<Register>(), 448 Address(invoke->GetLocations()->InAt(0).AsRegisterPairLow<Register>())); 449} 450 451void IntrinsicLocationsBuilderARM::VisitMemoryPokeIntNative(HInvoke* invoke) { 452 CreateIntIntToVoidLocations(arena_, invoke); 453} 454 455void IntrinsicCodeGeneratorARM::VisitMemoryPokeIntNative(HInvoke* invoke) { 456 ArmAssembler* assembler = GetAssembler(); 457 __ str(invoke->GetLocations()->InAt(1).AsRegister<Register>(), 458 Address(invoke->GetLocations()->InAt(0).AsRegisterPairLow<Register>())); 459} 460 461void IntrinsicLocationsBuilderARM::VisitMemoryPokeLongNative(HInvoke* invoke) { 462 CreateIntIntToVoidLocations(arena_, invoke); 463} 464 465void IntrinsicCodeGeneratorARM::VisitMemoryPokeLongNative(HInvoke* invoke) { 466 ArmAssembler* assembler = GetAssembler(); 467 // Ignore upper 4B of long address. 468 Register addr = invoke->GetLocations()->InAt(0).AsRegisterPairLow<Register>(); 469 // Worst case: Control register bit SCTLR.A = 0. Then unaligned accesses throw a processor 470 // exception. So we can't use ldrd as addr may be unaligned. 471 __ str(invoke->GetLocations()->InAt(1).AsRegisterPairLow<Register>(), Address(addr, 0)); 472 __ str(invoke->GetLocations()->InAt(1).AsRegisterPairHigh<Register>(), Address(addr, 4)); 473} 474 475void IntrinsicLocationsBuilderARM::VisitMemoryPokeShortNative(HInvoke* invoke) { 476 CreateIntIntToVoidLocations(arena_, invoke); 477} 478 479void IntrinsicCodeGeneratorARM::VisitMemoryPokeShortNative(HInvoke* invoke) { 480 ArmAssembler* assembler = GetAssembler(); 481 __ strh(invoke->GetLocations()->InAt(1).AsRegister<Register>(), 482 Address(invoke->GetLocations()->InAt(0).AsRegisterPairLow<Register>())); 483} 484 485void IntrinsicLocationsBuilderARM::VisitThreadCurrentThread(HInvoke* invoke) { 486 LocationSummary* locations = new (arena_) LocationSummary(invoke, 487 LocationSummary::kNoCall, 488 kIntrinsified); 489 locations->SetOut(Location::RequiresRegister()); 490} 491 492void IntrinsicCodeGeneratorARM::VisitThreadCurrentThread(HInvoke* invoke) { 493 ArmAssembler* assembler = GetAssembler(); 494 __ LoadFromOffset(kLoadWord, 495 invoke->GetLocations()->Out().AsRegister<Register>(), 496 TR, 497 Thread::PeerOffset<kArmPointerSize>().Int32Value()); 498} 499 500static void GenUnsafeGet(HInvoke* invoke, 501 Primitive::Type type, 502 bool is_volatile, 503 CodeGeneratorARM* codegen) { 504 LocationSummary* locations = invoke->GetLocations(); 505 ArmAssembler* assembler = codegen->GetAssembler(); 506 Location base_loc = locations->InAt(1); 507 Register base = base_loc.AsRegister<Register>(); // Object pointer. 508 Location offset_loc = locations->InAt(2); 509 Register offset = offset_loc.AsRegisterPairLow<Register>(); // Long offset, lo part only. 510 Location trg_loc = locations->Out(); 511 512 switch (type) { 513 case Primitive::kPrimInt: { 514 Register trg = trg_loc.AsRegister<Register>(); 515 __ ldr(trg, Address(base, offset)); 516 if (is_volatile) { 517 __ dmb(ISH); 518 } 519 break; 520 } 521 522 case Primitive::kPrimNot: { 523 Register trg = trg_loc.AsRegister<Register>(); 524 if (kEmitCompilerReadBarrier) { 525 if (kUseBakerReadBarrier) { 526 Location temp = locations->GetTemp(0); 527 codegen->GenerateArrayLoadWithBakerReadBarrier( 528 invoke, trg_loc, base, 0U, offset_loc, temp, /* needs_null_check */ false); 529 if (is_volatile) { 530 __ dmb(ISH); 531 } 532 } else { 533 __ ldr(trg, Address(base, offset)); 534 if (is_volatile) { 535 __ dmb(ISH); 536 } 537 codegen->GenerateReadBarrierSlow(invoke, trg_loc, trg_loc, base_loc, 0U, offset_loc); 538 } 539 } else { 540 __ ldr(trg, Address(base, offset)); 541 if (is_volatile) { 542 __ dmb(ISH); 543 } 544 __ MaybeUnpoisonHeapReference(trg); 545 } 546 break; 547 } 548 549 case Primitive::kPrimLong: { 550 Register trg_lo = trg_loc.AsRegisterPairLow<Register>(); 551 __ add(IP, base, ShifterOperand(offset)); 552 if (is_volatile && !codegen->GetInstructionSetFeatures().HasAtomicLdrdAndStrd()) { 553 Register trg_hi = trg_loc.AsRegisterPairHigh<Register>(); 554 __ ldrexd(trg_lo, trg_hi, IP); 555 } else { 556 __ ldrd(trg_lo, Address(IP)); 557 } 558 if (is_volatile) { 559 __ dmb(ISH); 560 } 561 break; 562 } 563 564 default: 565 LOG(FATAL) << "Unexpected type " << type; 566 UNREACHABLE(); 567 } 568} 569 570static void CreateIntIntIntToIntLocations(ArenaAllocator* arena, 571 HInvoke* invoke, 572 Primitive::Type type) { 573 bool can_call = kEmitCompilerReadBarrier && 574 (invoke->GetIntrinsic() == Intrinsics::kUnsafeGetObject || 575 invoke->GetIntrinsic() == Intrinsics::kUnsafeGetObjectVolatile); 576 LocationSummary* locations = new (arena) LocationSummary(invoke, 577 can_call ? 578 LocationSummary::kCallOnSlowPath : 579 LocationSummary::kNoCall, 580 kIntrinsified); 581 locations->SetInAt(0, Location::NoLocation()); // Unused receiver. 582 locations->SetInAt(1, Location::RequiresRegister()); 583 locations->SetInAt(2, Location::RequiresRegister()); 584 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 585 if (type == Primitive::kPrimNot && kEmitCompilerReadBarrier && kUseBakerReadBarrier) { 586 // We need a temporary register for the read barrier marking slow 587 // path in InstructionCodeGeneratorARM::GenerateArrayLoadWithBakerReadBarrier. 588 locations->AddTemp(Location::RequiresRegister()); 589 } 590} 591 592void IntrinsicLocationsBuilderARM::VisitUnsafeGet(HInvoke* invoke) { 593 CreateIntIntIntToIntLocations(arena_, invoke, Primitive::kPrimInt); 594} 595void IntrinsicLocationsBuilderARM::VisitUnsafeGetVolatile(HInvoke* invoke) { 596 CreateIntIntIntToIntLocations(arena_, invoke, Primitive::kPrimInt); 597} 598void IntrinsicLocationsBuilderARM::VisitUnsafeGetLong(HInvoke* invoke) { 599 CreateIntIntIntToIntLocations(arena_, invoke, Primitive::kPrimLong); 600} 601void IntrinsicLocationsBuilderARM::VisitUnsafeGetLongVolatile(HInvoke* invoke) { 602 CreateIntIntIntToIntLocations(arena_, invoke, Primitive::kPrimLong); 603} 604void IntrinsicLocationsBuilderARM::VisitUnsafeGetObject(HInvoke* invoke) { 605 CreateIntIntIntToIntLocations(arena_, invoke, Primitive::kPrimNot); 606} 607void IntrinsicLocationsBuilderARM::VisitUnsafeGetObjectVolatile(HInvoke* invoke) { 608 CreateIntIntIntToIntLocations(arena_, invoke, Primitive::kPrimNot); 609} 610 611void IntrinsicCodeGeneratorARM::VisitUnsafeGet(HInvoke* invoke) { 612 GenUnsafeGet(invoke, Primitive::kPrimInt, /* is_volatile */ false, codegen_); 613} 614void IntrinsicCodeGeneratorARM::VisitUnsafeGetVolatile(HInvoke* invoke) { 615 GenUnsafeGet(invoke, Primitive::kPrimInt, /* is_volatile */ true, codegen_); 616} 617void IntrinsicCodeGeneratorARM::VisitUnsafeGetLong(HInvoke* invoke) { 618 GenUnsafeGet(invoke, Primitive::kPrimLong, /* is_volatile */ false, codegen_); 619} 620void IntrinsicCodeGeneratorARM::VisitUnsafeGetLongVolatile(HInvoke* invoke) { 621 GenUnsafeGet(invoke, Primitive::kPrimLong, /* is_volatile */ true, codegen_); 622} 623void IntrinsicCodeGeneratorARM::VisitUnsafeGetObject(HInvoke* invoke) { 624 GenUnsafeGet(invoke, Primitive::kPrimNot, /* is_volatile */ false, codegen_); 625} 626void IntrinsicCodeGeneratorARM::VisitUnsafeGetObjectVolatile(HInvoke* invoke) { 627 GenUnsafeGet(invoke, Primitive::kPrimNot, /* is_volatile */ true, codegen_); 628} 629 630static void CreateIntIntIntIntToVoid(ArenaAllocator* arena, 631 const ArmInstructionSetFeatures& features, 632 Primitive::Type type, 633 bool is_volatile, 634 HInvoke* invoke) { 635 LocationSummary* locations = new (arena) LocationSummary(invoke, 636 LocationSummary::kNoCall, 637 kIntrinsified); 638 locations->SetInAt(0, Location::NoLocation()); // Unused receiver. 639 locations->SetInAt(1, Location::RequiresRegister()); 640 locations->SetInAt(2, Location::RequiresRegister()); 641 locations->SetInAt(3, Location::RequiresRegister()); 642 643 if (type == Primitive::kPrimLong) { 644 // Potentially need temps for ldrexd-strexd loop. 645 if (is_volatile && !features.HasAtomicLdrdAndStrd()) { 646 locations->AddTemp(Location::RequiresRegister()); // Temp_lo. 647 locations->AddTemp(Location::RequiresRegister()); // Temp_hi. 648 } 649 } else if (type == Primitive::kPrimNot) { 650 // Temps for card-marking. 651 locations->AddTemp(Location::RequiresRegister()); // Temp. 652 locations->AddTemp(Location::RequiresRegister()); // Card. 653 } 654} 655 656void IntrinsicLocationsBuilderARM::VisitUnsafePut(HInvoke* invoke) { 657 CreateIntIntIntIntToVoid(arena_, features_, Primitive::kPrimInt, /* is_volatile */ false, invoke); 658} 659void IntrinsicLocationsBuilderARM::VisitUnsafePutOrdered(HInvoke* invoke) { 660 CreateIntIntIntIntToVoid(arena_, features_, Primitive::kPrimInt, /* is_volatile */ false, invoke); 661} 662void IntrinsicLocationsBuilderARM::VisitUnsafePutVolatile(HInvoke* invoke) { 663 CreateIntIntIntIntToVoid(arena_, features_, Primitive::kPrimInt, /* is_volatile */ true, invoke); 664} 665void IntrinsicLocationsBuilderARM::VisitUnsafePutObject(HInvoke* invoke) { 666 CreateIntIntIntIntToVoid(arena_, features_, Primitive::kPrimNot, /* is_volatile */ false, invoke); 667} 668void IntrinsicLocationsBuilderARM::VisitUnsafePutObjectOrdered(HInvoke* invoke) { 669 CreateIntIntIntIntToVoid(arena_, features_, Primitive::kPrimNot, /* is_volatile */ false, invoke); 670} 671void IntrinsicLocationsBuilderARM::VisitUnsafePutObjectVolatile(HInvoke* invoke) { 672 CreateIntIntIntIntToVoid(arena_, features_, Primitive::kPrimNot, /* is_volatile */ true, invoke); 673} 674void IntrinsicLocationsBuilderARM::VisitUnsafePutLong(HInvoke* invoke) { 675 CreateIntIntIntIntToVoid( 676 arena_, features_, Primitive::kPrimLong, /* is_volatile */ false, invoke); 677} 678void IntrinsicLocationsBuilderARM::VisitUnsafePutLongOrdered(HInvoke* invoke) { 679 CreateIntIntIntIntToVoid( 680 arena_, features_, Primitive::kPrimLong, /* is_volatile */ false, invoke); 681} 682void IntrinsicLocationsBuilderARM::VisitUnsafePutLongVolatile(HInvoke* invoke) { 683 CreateIntIntIntIntToVoid( 684 arena_, features_, Primitive::kPrimLong, /* is_volatile */ true, invoke); 685} 686 687static void GenUnsafePut(LocationSummary* locations, 688 Primitive::Type type, 689 bool is_volatile, 690 bool is_ordered, 691 CodeGeneratorARM* codegen) { 692 ArmAssembler* assembler = codegen->GetAssembler(); 693 694 Register base = locations->InAt(1).AsRegister<Register>(); // Object pointer. 695 Register offset = locations->InAt(2).AsRegisterPairLow<Register>(); // Long offset, lo part only. 696 Register value; 697 698 if (is_volatile || is_ordered) { 699 __ dmb(ISH); 700 } 701 702 if (type == Primitive::kPrimLong) { 703 Register value_lo = locations->InAt(3).AsRegisterPairLow<Register>(); 704 value = value_lo; 705 if (is_volatile && !codegen->GetInstructionSetFeatures().HasAtomicLdrdAndStrd()) { 706 Register temp_lo = locations->GetTemp(0).AsRegister<Register>(); 707 Register temp_hi = locations->GetTemp(1).AsRegister<Register>(); 708 Register value_hi = locations->InAt(3).AsRegisterPairHigh<Register>(); 709 710 __ add(IP, base, ShifterOperand(offset)); 711 Label loop_head; 712 __ Bind(&loop_head); 713 __ ldrexd(temp_lo, temp_hi, IP); 714 __ strexd(temp_lo, value_lo, value_hi, IP); 715 __ cmp(temp_lo, ShifterOperand(0)); 716 __ b(&loop_head, NE); 717 } else { 718 __ add(IP, base, ShifterOperand(offset)); 719 __ strd(value_lo, Address(IP)); 720 } 721 } else { 722 value = locations->InAt(3).AsRegister<Register>(); 723 Register source = value; 724 if (kPoisonHeapReferences && type == Primitive::kPrimNot) { 725 Register temp = locations->GetTemp(0).AsRegister<Register>(); 726 __ Mov(temp, value); 727 __ PoisonHeapReference(temp); 728 source = temp; 729 } 730 __ str(source, Address(base, offset)); 731 } 732 733 if (is_volatile) { 734 __ dmb(ISH); 735 } 736 737 if (type == Primitive::kPrimNot) { 738 Register temp = locations->GetTemp(0).AsRegister<Register>(); 739 Register card = locations->GetTemp(1).AsRegister<Register>(); 740 bool value_can_be_null = true; // TODO: Worth finding out this information? 741 codegen->MarkGCCard(temp, card, base, value, value_can_be_null); 742 } 743} 744 745void IntrinsicCodeGeneratorARM::VisitUnsafePut(HInvoke* invoke) { 746 GenUnsafePut(invoke->GetLocations(), 747 Primitive::kPrimInt, 748 /* is_volatile */ false, 749 /* is_ordered */ false, 750 codegen_); 751} 752void IntrinsicCodeGeneratorARM::VisitUnsafePutOrdered(HInvoke* invoke) { 753 GenUnsafePut(invoke->GetLocations(), 754 Primitive::kPrimInt, 755 /* is_volatile */ false, 756 /* is_ordered */ true, 757 codegen_); 758} 759void IntrinsicCodeGeneratorARM::VisitUnsafePutVolatile(HInvoke* invoke) { 760 GenUnsafePut(invoke->GetLocations(), 761 Primitive::kPrimInt, 762 /* is_volatile */ true, 763 /* is_ordered */ false, 764 codegen_); 765} 766void IntrinsicCodeGeneratorARM::VisitUnsafePutObject(HInvoke* invoke) { 767 GenUnsafePut(invoke->GetLocations(), 768 Primitive::kPrimNot, 769 /* is_volatile */ false, 770 /* is_ordered */ false, 771 codegen_); 772} 773void IntrinsicCodeGeneratorARM::VisitUnsafePutObjectOrdered(HInvoke* invoke) { 774 GenUnsafePut(invoke->GetLocations(), 775 Primitive::kPrimNot, 776 /* is_volatile */ false, 777 /* is_ordered */ true, 778 codegen_); 779} 780void IntrinsicCodeGeneratorARM::VisitUnsafePutObjectVolatile(HInvoke* invoke) { 781 GenUnsafePut(invoke->GetLocations(), 782 Primitive::kPrimNot, 783 /* is_volatile */ true, 784 /* is_ordered */ false, 785 codegen_); 786} 787void IntrinsicCodeGeneratorARM::VisitUnsafePutLong(HInvoke* invoke) { 788 GenUnsafePut(invoke->GetLocations(), 789 Primitive::kPrimLong, 790 /* is_volatile */ false, 791 /* is_ordered */ false, 792 codegen_); 793} 794void IntrinsicCodeGeneratorARM::VisitUnsafePutLongOrdered(HInvoke* invoke) { 795 GenUnsafePut(invoke->GetLocations(), 796 Primitive::kPrimLong, 797 /* is_volatile */ false, 798 /* is_ordered */ true, 799 codegen_); 800} 801void IntrinsicCodeGeneratorARM::VisitUnsafePutLongVolatile(HInvoke* invoke) { 802 GenUnsafePut(invoke->GetLocations(), 803 Primitive::kPrimLong, 804 /* is_volatile */ true, 805 /* is_ordered */ false, 806 codegen_); 807} 808 809static void CreateIntIntIntIntIntToIntPlusTemps(ArenaAllocator* arena, 810 HInvoke* invoke) { 811 LocationSummary* locations = new (arena) LocationSummary(invoke, 812 LocationSummary::kNoCall, 813 kIntrinsified); 814 locations->SetInAt(0, Location::NoLocation()); // Unused receiver. 815 locations->SetInAt(1, Location::RequiresRegister()); 816 locations->SetInAt(2, Location::RequiresRegister()); 817 locations->SetInAt(3, Location::RequiresRegister()); 818 locations->SetInAt(4, Location::RequiresRegister()); 819 820 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 821 822 locations->AddTemp(Location::RequiresRegister()); // Pointer. 823 locations->AddTemp(Location::RequiresRegister()); // Temp 1. 824 locations->AddTemp(Location::RequiresRegister()); // Temp 2. 825} 826 827static void GenCas(LocationSummary* locations, Primitive::Type type, CodeGeneratorARM* codegen) { 828 DCHECK_NE(type, Primitive::kPrimLong); 829 830 ArmAssembler* assembler = codegen->GetAssembler(); 831 832 Register out = locations->Out().AsRegister<Register>(); // Boolean result. 833 834 Register base = locations->InAt(1).AsRegister<Register>(); // Object pointer. 835 Register offset = locations->InAt(2).AsRegisterPairLow<Register>(); // Offset (discard high 4B). 836 Register expected_lo = locations->InAt(3).AsRegister<Register>(); // Expected. 837 Register value_lo = locations->InAt(4).AsRegister<Register>(); // Value. 838 839 Register tmp_ptr = locations->GetTemp(0).AsRegister<Register>(); // Pointer to actual memory. 840 Register tmp_lo = locations->GetTemp(1).AsRegister<Register>(); // Value in memory. 841 842 if (type == Primitive::kPrimNot) { 843 // Mark card for object assuming new value is stored. Worst case we will mark an unchanged 844 // object and scan the receiver at the next GC for nothing. 845 bool value_can_be_null = true; // TODO: Worth finding out this information? 846 codegen->MarkGCCard(tmp_ptr, tmp_lo, base, value_lo, value_can_be_null); 847 } 848 849 // Prevent reordering with prior memory operations. 850 // Emit a DMB ISH instruction instead of an DMB ISHST one, as the 851 // latter allows a preceding load to be delayed past the STXR 852 // instruction below. 853 __ dmb(ISH); 854 855 __ add(tmp_ptr, base, ShifterOperand(offset)); 856 857 if (kPoisonHeapReferences && type == Primitive::kPrimNot) { 858 codegen->GetAssembler()->PoisonHeapReference(expected_lo); 859 codegen->GetAssembler()->PoisonHeapReference(value_lo); 860 } 861 862 // do { 863 // tmp = [r_ptr] - expected; 864 // } while (tmp == 0 && failure([r_ptr] <- r_new_value)); 865 // result = tmp != 0; 866 867 Label loop_head; 868 __ Bind(&loop_head); 869 870 // TODO: When `type == Primitive::kPrimNot`, add a read barrier for 871 // the reference stored in the object before attempting the CAS, 872 // similar to the one in the art::Unsafe_compareAndSwapObject JNI 873 // implementation. 874 // 875 // Note that this code is not (yet) used when read barriers are 876 // enabled (see IntrinsicLocationsBuilderARM::VisitUnsafeCASObject). 877 DCHECK(!(type == Primitive::kPrimNot && kEmitCompilerReadBarrier)); 878 __ ldrex(tmp_lo, tmp_ptr); 879 880 __ subs(tmp_lo, tmp_lo, ShifterOperand(expected_lo)); 881 882 __ it(EQ, ItState::kItT); 883 __ strex(tmp_lo, value_lo, tmp_ptr, EQ); 884 __ cmp(tmp_lo, ShifterOperand(1), EQ); 885 886 __ b(&loop_head, EQ); 887 888 __ dmb(ISH); 889 890 __ rsbs(out, tmp_lo, ShifterOperand(1)); 891 __ it(CC); 892 __ mov(out, ShifterOperand(0), CC); 893 894 if (kPoisonHeapReferences && type == Primitive::kPrimNot) { 895 codegen->GetAssembler()->UnpoisonHeapReference(value_lo); 896 codegen->GetAssembler()->UnpoisonHeapReference(expected_lo); 897 } 898} 899 900void IntrinsicLocationsBuilderARM::VisitUnsafeCASInt(HInvoke* invoke) { 901 CreateIntIntIntIntIntToIntPlusTemps(arena_, invoke); 902} 903void IntrinsicLocationsBuilderARM::VisitUnsafeCASObject(HInvoke* invoke) { 904 // The UnsafeCASObject intrinsic is missing a read barrier, and 905 // therefore sometimes does not work as expected (b/25883050). 906 // Turn it off temporarily as a quick fix, until the read barrier is 907 // implemented (see TODO in GenCAS below). 908 // 909 // Also, the UnsafeCASObject intrinsic does not always work when heap 910 // poisoning is enabled (it breaks run-test 004-UnsafeTest); turn it 911 // off temporarily as a quick fix (b/26204023). 912 // 913 // TODO(rpl): Fix these two issues and re-enable this intrinsic. 914 if (kEmitCompilerReadBarrier || kPoisonHeapReferences) { 915 return; 916 } 917 918 CreateIntIntIntIntIntToIntPlusTemps(arena_, invoke); 919} 920void IntrinsicCodeGeneratorARM::VisitUnsafeCASInt(HInvoke* invoke) { 921 GenCas(invoke->GetLocations(), Primitive::kPrimInt, codegen_); 922} 923void IntrinsicCodeGeneratorARM::VisitUnsafeCASObject(HInvoke* invoke) { 924 GenCas(invoke->GetLocations(), Primitive::kPrimNot, codegen_); 925} 926 927void IntrinsicLocationsBuilderARM::VisitStringCharAt(HInvoke* invoke) { 928 LocationSummary* locations = new (arena_) LocationSummary(invoke, 929 LocationSummary::kCallOnSlowPath, 930 kIntrinsified); 931 locations->SetInAt(0, Location::RequiresRegister()); 932 locations->SetInAt(1, Location::RequiresRegister()); 933 locations->SetOut(Location::RequiresRegister(), Location::kOutputOverlap); 934 935 locations->AddTemp(Location::RequiresRegister()); 936 locations->AddTemp(Location::RequiresRegister()); 937} 938 939void IntrinsicCodeGeneratorARM::VisitStringCharAt(HInvoke* invoke) { 940 ArmAssembler* assembler = GetAssembler(); 941 LocationSummary* locations = invoke->GetLocations(); 942 943 // Location of reference to data array 944 const MemberOffset value_offset = mirror::String::ValueOffset(); 945 // Location of count 946 const MemberOffset count_offset = mirror::String::CountOffset(); 947 948 Register obj = locations->InAt(0).AsRegister<Register>(); // String object pointer. 949 Register idx = locations->InAt(1).AsRegister<Register>(); // Index of character. 950 Register out = locations->Out().AsRegister<Register>(); // Result character. 951 952 Register temp = locations->GetTemp(0).AsRegister<Register>(); 953 Register array_temp = locations->GetTemp(1).AsRegister<Register>(); 954 955 // TODO: Maybe we can support range check elimination. Overall, though, I think it's not worth 956 // the cost. 957 // TODO: For simplicity, the index parameter is requested in a register, so different from Quick 958 // we will not optimize the code for constants (which would save a register). 959 960 SlowPathCode* slow_path = new (GetAllocator()) IntrinsicSlowPathARM(invoke); 961 codegen_->AddSlowPath(slow_path); 962 963 __ ldr(temp, Address(obj, count_offset.Int32Value())); // temp = str.length. 964 codegen_->MaybeRecordImplicitNullCheck(invoke); 965 __ cmp(idx, ShifterOperand(temp)); 966 __ b(slow_path->GetEntryLabel(), CS); 967 968 __ add(array_temp, obj, ShifterOperand(value_offset.Int32Value())); // array_temp := str.value. 969 970 // Load the value. 971 __ ldrh(out, Address(array_temp, idx, LSL, 1)); // out := array_temp[idx]. 972 973 __ Bind(slow_path->GetExitLabel()); 974} 975 976void IntrinsicLocationsBuilderARM::VisitStringCompareTo(HInvoke* invoke) { 977 // The inputs plus one temp. 978 LocationSummary* locations = new (arena_) LocationSummary(invoke, 979 LocationSummary::kCall, 980 kIntrinsified); 981 InvokeRuntimeCallingConvention calling_convention; 982 locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(0))); 983 locations->SetInAt(1, Location::RegisterLocation(calling_convention.GetRegisterAt(1))); 984 locations->SetOut(Location::RegisterLocation(R0)); 985} 986 987void IntrinsicCodeGeneratorARM::VisitStringCompareTo(HInvoke* invoke) { 988 ArmAssembler* assembler = GetAssembler(); 989 LocationSummary* locations = invoke->GetLocations(); 990 991 // Note that the null check must have been done earlier. 992 DCHECK(!invoke->CanDoImplicitNullCheckOn(invoke->InputAt(0))); 993 994 Register argument = locations->InAt(1).AsRegister<Register>(); 995 __ cmp(argument, ShifterOperand(0)); 996 SlowPathCode* slow_path = new (GetAllocator()) IntrinsicSlowPathARM(invoke); 997 codegen_->AddSlowPath(slow_path); 998 __ b(slow_path->GetEntryLabel(), EQ); 999 1000 __ LoadFromOffset( 1001 kLoadWord, LR, TR, QUICK_ENTRYPOINT_OFFSET(kArmWordSize, pStringCompareTo).Int32Value()); 1002 __ blx(LR); 1003 __ Bind(slow_path->GetExitLabel()); 1004} 1005 1006void IntrinsicLocationsBuilderARM::VisitStringEquals(HInvoke* invoke) { 1007 LocationSummary* locations = new (arena_) LocationSummary(invoke, 1008 LocationSummary::kNoCall, 1009 kIntrinsified); 1010 InvokeRuntimeCallingConvention calling_convention; 1011 locations->SetInAt(0, Location::RequiresRegister()); 1012 locations->SetInAt(1, Location::RequiresRegister()); 1013 // Temporary registers to store lengths of strings and for calculations. 1014 // Using instruction cbz requires a low register, so explicitly set a temp to be R0. 1015 locations->AddTemp(Location::RegisterLocation(R0)); 1016 locations->AddTemp(Location::RequiresRegister()); 1017 locations->AddTemp(Location::RequiresRegister()); 1018 1019 locations->SetOut(Location::RequiresRegister()); 1020} 1021 1022void IntrinsicCodeGeneratorARM::VisitStringEquals(HInvoke* invoke) { 1023 ArmAssembler* assembler = GetAssembler(); 1024 LocationSummary* locations = invoke->GetLocations(); 1025 1026 Register str = locations->InAt(0).AsRegister<Register>(); 1027 Register arg = locations->InAt(1).AsRegister<Register>(); 1028 Register out = locations->Out().AsRegister<Register>(); 1029 1030 Register temp = locations->GetTemp(0).AsRegister<Register>(); 1031 Register temp1 = locations->GetTemp(1).AsRegister<Register>(); 1032 Register temp2 = locations->GetTemp(2).AsRegister<Register>(); 1033 1034 Label loop; 1035 Label end; 1036 Label return_true; 1037 Label return_false; 1038 1039 // Get offsets of count, value, and class fields within a string object. 1040 const uint32_t count_offset = mirror::String::CountOffset().Uint32Value(); 1041 const uint32_t value_offset = mirror::String::ValueOffset().Uint32Value(); 1042 const uint32_t class_offset = mirror::Object::ClassOffset().Uint32Value(); 1043 1044 // Note that the null check must have been done earlier. 1045 DCHECK(!invoke->CanDoImplicitNullCheckOn(invoke->InputAt(0))); 1046 1047 // Check if input is null, return false if it is. 1048 __ CompareAndBranchIfZero(arg, &return_false); 1049 1050 // Instanceof check for the argument by comparing class fields. 1051 // All string objects must have the same type since String cannot be subclassed. 1052 // Receiver must be a string object, so its class field is equal to all strings' class fields. 1053 // If the argument is a string object, its class field must be equal to receiver's class field. 1054 __ ldr(temp, Address(str, class_offset)); 1055 __ ldr(temp1, Address(arg, class_offset)); 1056 __ cmp(temp, ShifterOperand(temp1)); 1057 __ b(&return_false, NE); 1058 1059 // Load lengths of this and argument strings. 1060 __ ldr(temp, Address(str, count_offset)); 1061 __ ldr(temp1, Address(arg, count_offset)); 1062 // Check if lengths are equal, return false if they're not. 1063 __ cmp(temp, ShifterOperand(temp1)); 1064 __ b(&return_false, NE); 1065 // Return true if both strings are empty. 1066 __ cbz(temp, &return_true); 1067 1068 // Reference equality check, return true if same reference. 1069 __ cmp(str, ShifterOperand(arg)); 1070 __ b(&return_true, EQ); 1071 1072 // Assertions that must hold in order to compare strings 2 characters at a time. 1073 DCHECK_ALIGNED(value_offset, 4); 1074 static_assert(IsAligned<4>(kObjectAlignment), "String of odd length is not zero padded"); 1075 1076 __ LoadImmediate(temp1, value_offset); 1077 1078 // Loop to compare strings 2 characters at a time starting at the front of the string. 1079 // Ok to do this because strings with an odd length are zero-padded. 1080 __ Bind(&loop); 1081 __ ldr(out, Address(str, temp1)); 1082 __ ldr(temp2, Address(arg, temp1)); 1083 __ cmp(out, ShifterOperand(temp2)); 1084 __ b(&return_false, NE); 1085 __ add(temp1, temp1, ShifterOperand(sizeof(uint32_t))); 1086 __ subs(temp, temp, ShifterOperand(sizeof(uint32_t) / sizeof(uint16_t))); 1087 __ b(&loop, GT); 1088 1089 // Return true and exit the function. 1090 // If loop does not result in returning false, we return true. 1091 __ Bind(&return_true); 1092 __ LoadImmediate(out, 1); 1093 __ b(&end); 1094 1095 // Return false and exit the function. 1096 __ Bind(&return_false); 1097 __ LoadImmediate(out, 0); 1098 __ Bind(&end); 1099} 1100 1101static void GenerateVisitStringIndexOf(HInvoke* invoke, 1102 ArmAssembler* assembler, 1103 CodeGeneratorARM* codegen, 1104 ArenaAllocator* allocator, 1105 bool start_at_zero) { 1106 LocationSummary* locations = invoke->GetLocations(); 1107 Register tmp_reg = locations->GetTemp(0).AsRegister<Register>(); 1108 1109 // Note that the null check must have been done earlier. 1110 DCHECK(!invoke->CanDoImplicitNullCheckOn(invoke->InputAt(0))); 1111 1112 // Check for code points > 0xFFFF. Either a slow-path check when we don't know statically, 1113 // or directly dispatch if we have a constant. 1114 SlowPathCode* slow_path = nullptr; 1115 if (invoke->InputAt(1)->IsIntConstant()) { 1116 if (static_cast<uint32_t>(invoke->InputAt(1)->AsIntConstant()->GetValue()) > 1117 std::numeric_limits<uint16_t>::max()) { 1118 // Always needs the slow-path. We could directly dispatch to it, but this case should be 1119 // rare, so for simplicity just put the full slow-path down and branch unconditionally. 1120 slow_path = new (allocator) IntrinsicSlowPathARM(invoke); 1121 codegen->AddSlowPath(slow_path); 1122 __ b(slow_path->GetEntryLabel()); 1123 __ Bind(slow_path->GetExitLabel()); 1124 return; 1125 } 1126 } else { 1127 Register char_reg = locations->InAt(1).AsRegister<Register>(); 1128 __ LoadImmediate(tmp_reg, std::numeric_limits<uint16_t>::max()); 1129 __ cmp(char_reg, ShifterOperand(tmp_reg)); 1130 slow_path = new (allocator) IntrinsicSlowPathARM(invoke); 1131 codegen->AddSlowPath(slow_path); 1132 __ b(slow_path->GetEntryLabel(), HI); 1133 } 1134 1135 if (start_at_zero) { 1136 DCHECK_EQ(tmp_reg, R2); 1137 // Start-index = 0. 1138 __ LoadImmediate(tmp_reg, 0); 1139 } 1140 1141 __ LoadFromOffset(kLoadWord, LR, TR, 1142 QUICK_ENTRYPOINT_OFFSET(kArmWordSize, pIndexOf).Int32Value()); 1143 __ blx(LR); 1144 1145 if (slow_path != nullptr) { 1146 __ Bind(slow_path->GetExitLabel()); 1147 } 1148} 1149 1150void IntrinsicLocationsBuilderARM::VisitStringIndexOf(HInvoke* invoke) { 1151 LocationSummary* locations = new (arena_) LocationSummary(invoke, 1152 LocationSummary::kCall, 1153 kIntrinsified); 1154 // We have a hand-crafted assembly stub that follows the runtime calling convention. So it's 1155 // best to align the inputs accordingly. 1156 InvokeRuntimeCallingConvention calling_convention; 1157 locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(0))); 1158 locations->SetInAt(1, Location::RegisterLocation(calling_convention.GetRegisterAt(1))); 1159 locations->SetOut(Location::RegisterLocation(R0)); 1160 1161 // Need a temp for slow-path codepoint compare, and need to send start-index=0. 1162 locations->AddTemp(Location::RegisterLocation(calling_convention.GetRegisterAt(2))); 1163} 1164 1165void IntrinsicCodeGeneratorARM::VisitStringIndexOf(HInvoke* invoke) { 1166 GenerateVisitStringIndexOf( 1167 invoke, GetAssembler(), codegen_, GetAllocator(), /* start_at_zero */ true); 1168} 1169 1170void IntrinsicLocationsBuilderARM::VisitStringIndexOfAfter(HInvoke* invoke) { 1171 LocationSummary* locations = new (arena_) LocationSummary(invoke, 1172 LocationSummary::kCall, 1173 kIntrinsified); 1174 // We have a hand-crafted assembly stub that follows the runtime calling convention. So it's 1175 // best to align the inputs accordingly. 1176 InvokeRuntimeCallingConvention calling_convention; 1177 locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(0))); 1178 locations->SetInAt(1, Location::RegisterLocation(calling_convention.GetRegisterAt(1))); 1179 locations->SetInAt(2, Location::RegisterLocation(calling_convention.GetRegisterAt(2))); 1180 locations->SetOut(Location::RegisterLocation(R0)); 1181 1182 // Need a temp for slow-path codepoint compare. 1183 locations->AddTemp(Location::RequiresRegister()); 1184} 1185 1186void IntrinsicCodeGeneratorARM::VisitStringIndexOfAfter(HInvoke* invoke) { 1187 GenerateVisitStringIndexOf( 1188 invoke, GetAssembler(), codegen_, GetAllocator(), /* start_at_zero */ false); 1189} 1190 1191void IntrinsicLocationsBuilderARM::VisitStringNewStringFromBytes(HInvoke* invoke) { 1192 LocationSummary* locations = new (arena_) LocationSummary(invoke, 1193 LocationSummary::kCall, 1194 kIntrinsified); 1195 InvokeRuntimeCallingConvention calling_convention; 1196 locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(0))); 1197 locations->SetInAt(1, Location::RegisterLocation(calling_convention.GetRegisterAt(1))); 1198 locations->SetInAt(2, Location::RegisterLocation(calling_convention.GetRegisterAt(2))); 1199 locations->SetInAt(3, Location::RegisterLocation(calling_convention.GetRegisterAt(3))); 1200 locations->SetOut(Location::RegisterLocation(R0)); 1201} 1202 1203void IntrinsicCodeGeneratorARM::VisitStringNewStringFromBytes(HInvoke* invoke) { 1204 ArmAssembler* assembler = GetAssembler(); 1205 LocationSummary* locations = invoke->GetLocations(); 1206 1207 Register byte_array = locations->InAt(0).AsRegister<Register>(); 1208 __ cmp(byte_array, ShifterOperand(0)); 1209 SlowPathCode* slow_path = new (GetAllocator()) IntrinsicSlowPathARM(invoke); 1210 codegen_->AddSlowPath(slow_path); 1211 __ b(slow_path->GetEntryLabel(), EQ); 1212 1213 __ LoadFromOffset( 1214 kLoadWord, LR, TR, QUICK_ENTRYPOINT_OFFSET(kArmWordSize, pAllocStringFromBytes).Int32Value()); 1215 codegen_->RecordPcInfo(invoke, invoke->GetDexPc()); 1216 __ blx(LR); 1217 __ Bind(slow_path->GetExitLabel()); 1218} 1219 1220void IntrinsicLocationsBuilderARM::VisitStringNewStringFromChars(HInvoke* invoke) { 1221 LocationSummary* locations = new (arena_) LocationSummary(invoke, 1222 LocationSummary::kCall, 1223 kIntrinsified); 1224 InvokeRuntimeCallingConvention calling_convention; 1225 locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(0))); 1226 locations->SetInAt(1, Location::RegisterLocation(calling_convention.GetRegisterAt(1))); 1227 locations->SetInAt(2, Location::RegisterLocation(calling_convention.GetRegisterAt(2))); 1228 locations->SetOut(Location::RegisterLocation(R0)); 1229} 1230 1231void IntrinsicCodeGeneratorARM::VisitStringNewStringFromChars(HInvoke* invoke) { 1232 ArmAssembler* assembler = GetAssembler(); 1233 1234 __ LoadFromOffset( 1235 kLoadWord, LR, TR, QUICK_ENTRYPOINT_OFFSET(kArmWordSize, pAllocStringFromChars).Int32Value()); 1236 codegen_->RecordPcInfo(invoke, invoke->GetDexPc()); 1237 __ blx(LR); 1238} 1239 1240void IntrinsicLocationsBuilderARM::VisitStringNewStringFromString(HInvoke* invoke) { 1241 LocationSummary* locations = new (arena_) LocationSummary(invoke, 1242 LocationSummary::kCall, 1243 kIntrinsified); 1244 InvokeRuntimeCallingConvention calling_convention; 1245 locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(0))); 1246 locations->SetOut(Location::RegisterLocation(R0)); 1247} 1248 1249void IntrinsicCodeGeneratorARM::VisitStringNewStringFromString(HInvoke* invoke) { 1250 ArmAssembler* assembler = GetAssembler(); 1251 LocationSummary* locations = invoke->GetLocations(); 1252 1253 Register string_to_copy = locations->InAt(0).AsRegister<Register>(); 1254 __ cmp(string_to_copy, ShifterOperand(0)); 1255 SlowPathCode* slow_path = new (GetAllocator()) IntrinsicSlowPathARM(invoke); 1256 codegen_->AddSlowPath(slow_path); 1257 __ b(slow_path->GetEntryLabel(), EQ); 1258 1259 __ LoadFromOffset(kLoadWord, 1260 LR, TR, QUICK_ENTRYPOINT_OFFSET(kArmWordSize, pAllocStringFromString).Int32Value()); 1261 codegen_->RecordPcInfo(invoke, invoke->GetDexPc()); 1262 __ blx(LR); 1263 __ Bind(slow_path->GetExitLabel()); 1264} 1265 1266void IntrinsicLocationsBuilderARM::VisitSystemArrayCopy(HInvoke* invoke) { 1267 CodeGenerator::CreateSystemArrayCopyLocationSummary(invoke); 1268 LocationSummary* locations = invoke->GetLocations(); 1269 if (locations == nullptr) { 1270 return; 1271 } 1272 1273 HIntConstant* src_pos = invoke->InputAt(1)->AsIntConstant(); 1274 HIntConstant* dest_pos = invoke->InputAt(3)->AsIntConstant(); 1275 HIntConstant* length = invoke->InputAt(4)->AsIntConstant(); 1276 1277 if (src_pos != nullptr && !assembler_->ShifterOperandCanAlwaysHold(src_pos->GetValue())) { 1278 locations->SetInAt(1, Location::RequiresRegister()); 1279 } 1280 if (dest_pos != nullptr && !assembler_->ShifterOperandCanAlwaysHold(dest_pos->GetValue())) { 1281 locations->SetInAt(3, Location::RequiresRegister()); 1282 } 1283 if (length != nullptr && !assembler_->ShifterOperandCanAlwaysHold(length->GetValue())) { 1284 locations->SetInAt(4, Location::RequiresRegister()); 1285 } 1286} 1287 1288static void CheckPosition(ArmAssembler* assembler, 1289 Location pos, 1290 Register input, 1291 Location length, 1292 SlowPathCode* slow_path, 1293 Register input_len, 1294 Register temp, 1295 bool length_is_input_length = false) { 1296 // Where is the length in the Array? 1297 const uint32_t length_offset = mirror::Array::LengthOffset().Uint32Value(); 1298 1299 if (pos.IsConstant()) { 1300 int32_t pos_const = pos.GetConstant()->AsIntConstant()->GetValue(); 1301 if (pos_const == 0) { 1302 if (!length_is_input_length) { 1303 // Check that length(input) >= length. 1304 __ LoadFromOffset(kLoadWord, temp, input, length_offset); 1305 if (length.IsConstant()) { 1306 __ cmp(temp, ShifterOperand(length.GetConstant()->AsIntConstant()->GetValue())); 1307 } else { 1308 __ cmp(temp, ShifterOperand(length.AsRegister<Register>())); 1309 } 1310 __ b(slow_path->GetEntryLabel(), LT); 1311 } 1312 } else { 1313 // Check that length(input) >= pos. 1314 __ LoadFromOffset(kLoadWord, input_len, input, length_offset); 1315 __ subs(temp, input_len, ShifterOperand(pos_const)); 1316 __ b(slow_path->GetEntryLabel(), LT); 1317 1318 // Check that (length(input) - pos) >= length. 1319 if (length.IsConstant()) { 1320 __ cmp(temp, ShifterOperand(length.GetConstant()->AsIntConstant()->GetValue())); 1321 } else { 1322 __ cmp(temp, ShifterOperand(length.AsRegister<Register>())); 1323 } 1324 __ b(slow_path->GetEntryLabel(), LT); 1325 } 1326 } else if (length_is_input_length) { 1327 // The only way the copy can succeed is if pos is zero. 1328 Register pos_reg = pos.AsRegister<Register>(); 1329 __ CompareAndBranchIfNonZero(pos_reg, slow_path->GetEntryLabel()); 1330 } else { 1331 // Check that pos >= 0. 1332 Register pos_reg = pos.AsRegister<Register>(); 1333 __ cmp(pos_reg, ShifterOperand(0)); 1334 __ b(slow_path->GetEntryLabel(), LT); 1335 1336 // Check that pos <= length(input). 1337 __ LoadFromOffset(kLoadWord, temp, input, length_offset); 1338 __ subs(temp, temp, ShifterOperand(pos_reg)); 1339 __ b(slow_path->GetEntryLabel(), LT); 1340 1341 // Check that (length(input) - pos) >= length. 1342 if (length.IsConstant()) { 1343 __ cmp(temp, ShifterOperand(length.GetConstant()->AsIntConstant()->GetValue())); 1344 } else { 1345 __ cmp(temp, ShifterOperand(length.AsRegister<Register>())); 1346 } 1347 __ b(slow_path->GetEntryLabel(), LT); 1348 } 1349} 1350 1351// TODO: Implement read barriers in the SystemArrayCopy intrinsic. 1352// Note that this code path is not used (yet) because we do not 1353// intrinsify methods that can go into the IntrinsicSlowPathARM 1354// slow path. 1355void IntrinsicCodeGeneratorARM::VisitSystemArrayCopy(HInvoke* invoke) { 1356 ArmAssembler* assembler = GetAssembler(); 1357 LocationSummary* locations = invoke->GetLocations(); 1358 1359 uint32_t class_offset = mirror::Object::ClassOffset().Int32Value(); 1360 uint32_t super_offset = mirror::Class::SuperClassOffset().Int32Value(); 1361 uint32_t component_offset = mirror::Class::ComponentTypeOffset().Int32Value(); 1362 uint32_t primitive_offset = mirror::Class::PrimitiveTypeOffset().Int32Value(); 1363 1364 Register src = locations->InAt(0).AsRegister<Register>(); 1365 Location src_pos = locations->InAt(1); 1366 Register dest = locations->InAt(2).AsRegister<Register>(); 1367 Location dest_pos = locations->InAt(3); 1368 Location length = locations->InAt(4); 1369 Register temp1 = locations->GetTemp(0).AsRegister<Register>(); 1370 Register temp2 = locations->GetTemp(1).AsRegister<Register>(); 1371 Register temp3 = locations->GetTemp(2).AsRegister<Register>(); 1372 1373 SlowPathCode* slow_path = new (GetAllocator()) IntrinsicSlowPathARM(invoke); 1374 codegen_->AddSlowPath(slow_path); 1375 1376 Label ok; 1377 SystemArrayCopyOptimizations optimizations(invoke); 1378 1379 if (!optimizations.GetDestinationIsSource()) { 1380 if (!src_pos.IsConstant() || !dest_pos.IsConstant()) { 1381 __ cmp(src, ShifterOperand(dest)); 1382 } 1383 } 1384 1385 // If source and destination are the same, we go to slow path if we need to do 1386 // forward copying. 1387 if (src_pos.IsConstant()) { 1388 int32_t src_pos_constant = src_pos.GetConstant()->AsIntConstant()->GetValue(); 1389 if (dest_pos.IsConstant()) { 1390 // Checked when building locations. 1391 DCHECK(!optimizations.GetDestinationIsSource() 1392 || (src_pos_constant >= dest_pos.GetConstant()->AsIntConstant()->GetValue())); 1393 } else { 1394 if (!optimizations.GetDestinationIsSource()) { 1395 __ b(&ok, NE); 1396 } 1397 __ cmp(dest_pos.AsRegister<Register>(), ShifterOperand(src_pos_constant)); 1398 __ b(slow_path->GetEntryLabel(), GT); 1399 } 1400 } else { 1401 if (!optimizations.GetDestinationIsSource()) { 1402 __ b(&ok, NE); 1403 } 1404 if (dest_pos.IsConstant()) { 1405 int32_t dest_pos_constant = dest_pos.GetConstant()->AsIntConstant()->GetValue(); 1406 __ cmp(src_pos.AsRegister<Register>(), ShifterOperand(dest_pos_constant)); 1407 } else { 1408 __ cmp(src_pos.AsRegister<Register>(), ShifterOperand(dest_pos.AsRegister<Register>())); 1409 } 1410 __ b(slow_path->GetEntryLabel(), LT); 1411 } 1412 1413 __ Bind(&ok); 1414 1415 if (!optimizations.GetSourceIsNotNull()) { 1416 // Bail out if the source is null. 1417 __ CompareAndBranchIfZero(src, slow_path->GetEntryLabel()); 1418 } 1419 1420 if (!optimizations.GetDestinationIsNotNull() && !optimizations.GetDestinationIsSource()) { 1421 // Bail out if the destination is null. 1422 __ CompareAndBranchIfZero(dest, slow_path->GetEntryLabel()); 1423 } 1424 1425 // If the length is negative, bail out. 1426 // We have already checked in the LocationsBuilder for the constant case. 1427 if (!length.IsConstant() && 1428 !optimizations.GetCountIsSourceLength() && 1429 !optimizations.GetCountIsDestinationLength()) { 1430 __ cmp(length.AsRegister<Register>(), ShifterOperand(0)); 1431 __ b(slow_path->GetEntryLabel(), LT); 1432 } 1433 1434 // Validity checks: source. 1435 CheckPosition(assembler, 1436 src_pos, 1437 src, 1438 length, 1439 slow_path, 1440 temp1, 1441 temp2, 1442 optimizations.GetCountIsSourceLength()); 1443 1444 // Validity checks: dest. 1445 CheckPosition(assembler, 1446 dest_pos, 1447 dest, 1448 length, 1449 slow_path, 1450 temp1, 1451 temp2, 1452 optimizations.GetCountIsDestinationLength()); 1453 1454 if (!optimizations.GetDoesNotNeedTypeCheck()) { 1455 // Check whether all elements of the source array are assignable to the component 1456 // type of the destination array. We do two checks: the classes are the same, 1457 // or the destination is Object[]. If none of these checks succeed, we go to the 1458 // slow path. 1459 __ LoadFromOffset(kLoadWord, temp1, dest, class_offset); 1460 __ LoadFromOffset(kLoadWord, temp2, src, class_offset); 1461 bool did_unpoison = false; 1462 if (!optimizations.GetDestinationIsNonPrimitiveArray() || 1463 !optimizations.GetSourceIsNonPrimitiveArray()) { 1464 // One or two of the references need to be unpoisoned. Unpoisoned them 1465 // both to make the identity check valid. 1466 __ MaybeUnpoisonHeapReference(temp1); 1467 __ MaybeUnpoisonHeapReference(temp2); 1468 did_unpoison = true; 1469 } 1470 1471 if (!optimizations.GetDestinationIsNonPrimitiveArray()) { 1472 // Bail out if the destination is not a non primitive array. 1473 __ LoadFromOffset(kLoadWord, temp3, temp1, component_offset); 1474 __ CompareAndBranchIfZero(temp3, slow_path->GetEntryLabel()); 1475 __ MaybeUnpoisonHeapReference(temp3); 1476 __ LoadFromOffset(kLoadUnsignedHalfword, temp3, temp3, primitive_offset); 1477 static_assert(Primitive::kPrimNot == 0, "Expected 0 for kPrimNot"); 1478 __ CompareAndBranchIfNonZero(temp3, slow_path->GetEntryLabel()); 1479 } 1480 1481 if (!optimizations.GetSourceIsNonPrimitiveArray()) { 1482 // Bail out if the source is not a non primitive array. 1483 // Bail out if the destination is not a non primitive array. 1484 __ LoadFromOffset(kLoadWord, temp3, temp2, component_offset); 1485 __ CompareAndBranchIfZero(temp3, slow_path->GetEntryLabel()); 1486 __ MaybeUnpoisonHeapReference(temp3); 1487 __ LoadFromOffset(kLoadUnsignedHalfword, temp3, temp3, primitive_offset); 1488 static_assert(Primitive::kPrimNot == 0, "Expected 0 for kPrimNot"); 1489 __ CompareAndBranchIfNonZero(temp3, slow_path->GetEntryLabel()); 1490 } 1491 1492 __ cmp(temp1, ShifterOperand(temp2)); 1493 1494 if (optimizations.GetDestinationIsTypedObjectArray()) { 1495 Label do_copy; 1496 __ b(&do_copy, EQ); 1497 if (!did_unpoison) { 1498 __ MaybeUnpoisonHeapReference(temp1); 1499 } 1500 __ LoadFromOffset(kLoadWord, temp1, temp1, component_offset); 1501 __ MaybeUnpoisonHeapReference(temp1); 1502 __ LoadFromOffset(kLoadWord, temp1, temp1, super_offset); 1503 // No need to unpoison the result, we're comparing against null. 1504 __ CompareAndBranchIfNonZero(temp1, slow_path->GetEntryLabel()); 1505 __ Bind(&do_copy); 1506 } else { 1507 __ b(slow_path->GetEntryLabel(), NE); 1508 } 1509 } else if (!optimizations.GetSourceIsNonPrimitiveArray()) { 1510 DCHECK(optimizations.GetDestinationIsNonPrimitiveArray()); 1511 // Bail out if the source is not a non primitive array. 1512 __ LoadFromOffset(kLoadWord, temp1, src, class_offset); 1513 __ MaybeUnpoisonHeapReference(temp1); 1514 __ LoadFromOffset(kLoadWord, temp3, temp1, component_offset); 1515 __ CompareAndBranchIfZero(temp3, slow_path->GetEntryLabel()); 1516 __ MaybeUnpoisonHeapReference(temp3); 1517 __ LoadFromOffset(kLoadUnsignedHalfword, temp3, temp3, primitive_offset); 1518 static_assert(Primitive::kPrimNot == 0, "Expected 0 for kPrimNot"); 1519 __ CompareAndBranchIfNonZero(temp3, slow_path->GetEntryLabel()); 1520 } 1521 1522 // Compute base source address, base destination address, and end source address. 1523 1524 uint32_t element_size = sizeof(int32_t); 1525 uint32_t offset = mirror::Array::DataOffset(element_size).Uint32Value(); 1526 if (src_pos.IsConstant()) { 1527 int32_t constant = src_pos.GetConstant()->AsIntConstant()->GetValue(); 1528 __ AddConstant(temp1, src, element_size * constant + offset); 1529 } else { 1530 __ add(temp1, src, ShifterOperand(src_pos.AsRegister<Register>(), LSL, 2)); 1531 __ AddConstant(temp1, offset); 1532 } 1533 1534 if (dest_pos.IsConstant()) { 1535 int32_t constant = dest_pos.GetConstant()->AsIntConstant()->GetValue(); 1536 __ AddConstant(temp2, dest, element_size * constant + offset); 1537 } else { 1538 __ add(temp2, dest, ShifterOperand(dest_pos.AsRegister<Register>(), LSL, 2)); 1539 __ AddConstant(temp2, offset); 1540 } 1541 1542 if (length.IsConstant()) { 1543 int32_t constant = length.GetConstant()->AsIntConstant()->GetValue(); 1544 __ AddConstant(temp3, temp1, element_size * constant); 1545 } else { 1546 __ add(temp3, temp1, ShifterOperand(length.AsRegister<Register>(), LSL, 2)); 1547 } 1548 1549 // Iterate over the arrays and do a raw copy of the objects. We don't need to 1550 // poison/unpoison, nor do any read barrier as the next uses of the destination 1551 // array will do it. 1552 Label loop, done; 1553 __ cmp(temp1, ShifterOperand(temp3)); 1554 __ b(&done, EQ); 1555 __ Bind(&loop); 1556 __ ldr(IP, Address(temp1, element_size, Address::PostIndex)); 1557 __ str(IP, Address(temp2, element_size, Address::PostIndex)); 1558 __ cmp(temp1, ShifterOperand(temp3)); 1559 __ b(&loop, NE); 1560 __ Bind(&done); 1561 1562 // We only need one card marking on the destination array. 1563 codegen_->MarkGCCard(temp1, 1564 temp2, 1565 dest, 1566 Register(kNoRegister), 1567 /* can_be_null */ false); 1568 1569 __ Bind(slow_path->GetExitLabel()); 1570} 1571 1572// Unimplemented intrinsics. 1573 1574#define UNIMPLEMENTED_INTRINSIC(Name) \ 1575void IntrinsicLocationsBuilderARM::Visit ## Name(HInvoke* invoke ATTRIBUTE_UNUSED) { \ 1576} \ 1577void IntrinsicCodeGeneratorARM::Visit ## Name(HInvoke* invoke ATTRIBUTE_UNUSED) { \ 1578} 1579 1580UNIMPLEMENTED_INTRINSIC(IntegerReverse) 1581UNIMPLEMENTED_INTRINSIC(IntegerReverseBytes) 1582UNIMPLEMENTED_INTRINSIC(IntegerRotateLeft) 1583UNIMPLEMENTED_INTRINSIC(IntegerRotateRight) 1584UNIMPLEMENTED_INTRINSIC(LongReverse) 1585UNIMPLEMENTED_INTRINSIC(LongReverseBytes) 1586UNIMPLEMENTED_INTRINSIC(LongRotateLeft) 1587UNIMPLEMENTED_INTRINSIC(LongRotateRight) 1588UNIMPLEMENTED_INTRINSIC(ShortReverseBytes) 1589UNIMPLEMENTED_INTRINSIC(MathMinDoubleDouble) 1590UNIMPLEMENTED_INTRINSIC(MathMinFloatFloat) 1591UNIMPLEMENTED_INTRINSIC(MathMaxDoubleDouble) 1592UNIMPLEMENTED_INTRINSIC(MathMaxFloatFloat) 1593UNIMPLEMENTED_INTRINSIC(MathMinLongLong) 1594UNIMPLEMENTED_INTRINSIC(MathMaxLongLong) 1595UNIMPLEMENTED_INTRINSIC(MathCeil) // Could be done by changing rounding mode, maybe? 1596UNIMPLEMENTED_INTRINSIC(MathFloor) // Could be done by changing rounding mode, maybe? 1597UNIMPLEMENTED_INTRINSIC(MathRint) 1598UNIMPLEMENTED_INTRINSIC(MathRoundDouble) // Could be done by changing rounding mode, maybe? 1599UNIMPLEMENTED_INTRINSIC(MathRoundFloat) // Could be done by changing rounding mode, maybe? 1600UNIMPLEMENTED_INTRINSIC(UnsafeCASLong) // High register pressure. 1601UNIMPLEMENTED_INTRINSIC(SystemArrayCopyChar) 1602UNIMPLEMENTED_INTRINSIC(ReferenceGetReferent) 1603UNIMPLEMENTED_INTRINSIC(StringGetCharsNoCheck) 1604UNIMPLEMENTED_INTRINSIC(MathCos) 1605UNIMPLEMENTED_INTRINSIC(MathSin) 1606UNIMPLEMENTED_INTRINSIC(MathAcos) 1607UNIMPLEMENTED_INTRINSIC(MathAsin) 1608UNIMPLEMENTED_INTRINSIC(MathAtan) 1609UNIMPLEMENTED_INTRINSIC(MathAtan2) 1610UNIMPLEMENTED_INTRINSIC(MathCbrt) 1611UNIMPLEMENTED_INTRINSIC(MathCosh) 1612UNIMPLEMENTED_INTRINSIC(MathExp) 1613UNIMPLEMENTED_INTRINSIC(MathExpm1) 1614UNIMPLEMENTED_INTRINSIC(MathHypot) 1615UNIMPLEMENTED_INTRINSIC(MathLog) 1616UNIMPLEMENTED_INTRINSIC(MathLog10) 1617UNIMPLEMENTED_INTRINSIC(MathNextAfter) 1618UNIMPLEMENTED_INTRINSIC(MathSinh) 1619UNIMPLEMENTED_INTRINSIC(MathTan) 1620UNIMPLEMENTED_INTRINSIC(MathTanh) 1621 1622#undef UNIMPLEMENTED_INTRINSIC 1623 1624#undef __ 1625 1626} // namespace arm 1627} // namespace art 1628