intrinsics_arm64.cc revision 2f9fcc999fab4ba6cd86c30e664325b47b9618e5
1/* 2 * Copyright (C) 2015 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17#include "intrinsics_arm64.h" 18 19#include "arch/arm64/instruction_set_features_arm64.h" 20#include "art_method.h" 21#include "code_generator_arm64.h" 22#include "common_arm64.h" 23#include "entrypoints/quick/quick_entrypoints.h" 24#include "intrinsics.h" 25#include "mirror/array-inl.h" 26#include "mirror/string.h" 27#include "thread.h" 28#include "utils/arm64/assembler_arm64.h" 29#include "utils/arm64/constants_arm64.h" 30 31#include "vixl/a64/disasm-a64.h" 32#include "vixl/a64/macro-assembler-a64.h" 33 34using namespace vixl; // NOLINT(build/namespaces) 35 36namespace art { 37 38namespace arm64 { 39 40using helpers::DRegisterFrom; 41using helpers::FPRegisterFrom; 42using helpers::HeapOperand; 43using helpers::LocationFrom; 44using helpers::OperandFrom; 45using helpers::RegisterFrom; 46using helpers::SRegisterFrom; 47using helpers::WRegisterFrom; 48using helpers::XRegisterFrom; 49 50namespace { 51 52ALWAYS_INLINE inline MemOperand AbsoluteHeapOperandFrom(Location location, size_t offset = 0) { 53 return MemOperand(XRegisterFrom(location), offset); 54} 55 56} // namespace 57 58vixl::MacroAssembler* IntrinsicCodeGeneratorARM64::GetVIXLAssembler() { 59 return codegen_->GetAssembler()->vixl_masm_; 60} 61 62ArenaAllocator* IntrinsicCodeGeneratorARM64::GetAllocator() { 63 return codegen_->GetGraph()->GetArena(); 64} 65 66#define __ codegen->GetAssembler()->vixl_masm_-> 67 68static void MoveFromReturnRegister(Location trg, 69 Primitive::Type type, 70 CodeGeneratorARM64* codegen) { 71 if (!trg.IsValid()) { 72 DCHECK(type == Primitive::kPrimVoid); 73 return; 74 } 75 76 DCHECK_NE(type, Primitive::kPrimVoid); 77 78 if (Primitive::IsIntegralType(type) || type == Primitive::kPrimNot) { 79 Register trg_reg = RegisterFrom(trg, type); 80 Register res_reg = RegisterFrom(ARM64ReturnLocation(type), type); 81 __ Mov(trg_reg, res_reg, kDiscardForSameWReg); 82 } else { 83 FPRegister trg_reg = FPRegisterFrom(trg, type); 84 FPRegister res_reg = FPRegisterFrom(ARM64ReturnLocation(type), type); 85 __ Fmov(trg_reg, res_reg); 86 } 87} 88 89static void MoveArguments(HInvoke* invoke, CodeGeneratorARM64* codegen) { 90 InvokeDexCallingConventionVisitorARM64 calling_convention_visitor; 91 IntrinsicVisitor::MoveArguments(invoke, codegen, &calling_convention_visitor); 92} 93 94// Slow-path for fallback (calling the managed code to handle the intrinsic) in an intrinsified 95// call. This will copy the arguments into the positions for a regular call. 96// 97// Note: The actual parameters are required to be in the locations given by the invoke's location 98// summary. If an intrinsic modifies those locations before a slowpath call, they must be 99// restored! 100class IntrinsicSlowPathARM64 : public SlowPathCodeARM64 { 101 public: 102 explicit IntrinsicSlowPathARM64(HInvoke* invoke) 103 : SlowPathCodeARM64(invoke), invoke_(invoke) { } 104 105 void EmitNativeCode(CodeGenerator* codegen_in) OVERRIDE { 106 CodeGeneratorARM64* codegen = down_cast<CodeGeneratorARM64*>(codegen_in); 107 __ Bind(GetEntryLabel()); 108 109 SaveLiveRegisters(codegen, invoke_->GetLocations()); 110 111 MoveArguments(invoke_, codegen); 112 113 if (invoke_->IsInvokeStaticOrDirect()) { 114 codegen->GenerateStaticOrDirectCall(invoke_->AsInvokeStaticOrDirect(), 115 LocationFrom(kArtMethodRegister)); 116 } else { 117 codegen->GenerateVirtualCall(invoke_->AsInvokeVirtual(), LocationFrom(kArtMethodRegister)); 118 } 119 codegen->RecordPcInfo(invoke_, invoke_->GetDexPc(), this); 120 121 // Copy the result back to the expected output. 122 Location out = invoke_->GetLocations()->Out(); 123 if (out.IsValid()) { 124 DCHECK(out.IsRegister()); // TODO: Replace this when we support output in memory. 125 DCHECK(!invoke_->GetLocations()->GetLiveRegisters()->ContainsCoreRegister(out.reg())); 126 MoveFromReturnRegister(out, invoke_->GetType(), codegen); 127 } 128 129 RestoreLiveRegisters(codegen, invoke_->GetLocations()); 130 __ B(GetExitLabel()); 131 } 132 133 const char* GetDescription() const OVERRIDE { return "IntrinsicSlowPathARM64"; } 134 135 private: 136 // The instruction where this slow path is happening. 137 HInvoke* const invoke_; 138 139 DISALLOW_COPY_AND_ASSIGN(IntrinsicSlowPathARM64); 140}; 141 142#undef __ 143 144bool IntrinsicLocationsBuilderARM64::TryDispatch(HInvoke* invoke) { 145 Dispatch(invoke); 146 LocationSummary* res = invoke->GetLocations(); 147 if (res == nullptr) { 148 return false; 149 } 150 if (kEmitCompilerReadBarrier && res->CanCall()) { 151 // Generating an intrinsic for this HInvoke may produce an 152 // IntrinsicSlowPathARM64 slow path. Currently this approach 153 // does not work when using read barriers, as the emitted 154 // calling sequence will make use of another slow path 155 // (ReadBarrierForRootSlowPathARM64 for HInvokeStaticOrDirect, 156 // ReadBarrierSlowPathARM64 for HInvokeVirtual). So we bail 157 // out in this case. 158 // 159 // TODO: Find a way to have intrinsics work with read barriers. 160 invoke->SetLocations(nullptr); 161 return false; 162 } 163 return res->Intrinsified(); 164} 165 166#define __ masm-> 167 168static void CreateFPToIntLocations(ArenaAllocator* arena, HInvoke* invoke) { 169 LocationSummary* locations = new (arena) LocationSummary(invoke, 170 LocationSummary::kNoCall, 171 kIntrinsified); 172 locations->SetInAt(0, Location::RequiresFpuRegister()); 173 locations->SetOut(Location::RequiresRegister()); 174} 175 176static void CreateIntToFPLocations(ArenaAllocator* arena, HInvoke* invoke) { 177 LocationSummary* locations = new (arena) LocationSummary(invoke, 178 LocationSummary::kNoCall, 179 kIntrinsified); 180 locations->SetInAt(0, Location::RequiresRegister()); 181 locations->SetOut(Location::RequiresFpuRegister()); 182} 183 184static void MoveFPToInt(LocationSummary* locations, bool is64bit, vixl::MacroAssembler* masm) { 185 Location input = locations->InAt(0); 186 Location output = locations->Out(); 187 __ Fmov(is64bit ? XRegisterFrom(output) : WRegisterFrom(output), 188 is64bit ? DRegisterFrom(input) : SRegisterFrom(input)); 189} 190 191static void MoveIntToFP(LocationSummary* locations, bool is64bit, vixl::MacroAssembler* masm) { 192 Location input = locations->InAt(0); 193 Location output = locations->Out(); 194 __ Fmov(is64bit ? DRegisterFrom(output) : SRegisterFrom(output), 195 is64bit ? XRegisterFrom(input) : WRegisterFrom(input)); 196} 197 198void IntrinsicLocationsBuilderARM64::VisitDoubleDoubleToRawLongBits(HInvoke* invoke) { 199 CreateFPToIntLocations(arena_, invoke); 200} 201void IntrinsicLocationsBuilderARM64::VisitDoubleLongBitsToDouble(HInvoke* invoke) { 202 CreateIntToFPLocations(arena_, invoke); 203} 204 205void IntrinsicCodeGeneratorARM64::VisitDoubleDoubleToRawLongBits(HInvoke* invoke) { 206 MoveFPToInt(invoke->GetLocations(), /* is64bit */ true, GetVIXLAssembler()); 207} 208void IntrinsicCodeGeneratorARM64::VisitDoubleLongBitsToDouble(HInvoke* invoke) { 209 MoveIntToFP(invoke->GetLocations(), /* is64bit */ true, GetVIXLAssembler()); 210} 211 212void IntrinsicLocationsBuilderARM64::VisitFloatFloatToRawIntBits(HInvoke* invoke) { 213 CreateFPToIntLocations(arena_, invoke); 214} 215void IntrinsicLocationsBuilderARM64::VisitFloatIntBitsToFloat(HInvoke* invoke) { 216 CreateIntToFPLocations(arena_, invoke); 217} 218 219void IntrinsicCodeGeneratorARM64::VisitFloatFloatToRawIntBits(HInvoke* invoke) { 220 MoveFPToInt(invoke->GetLocations(), /* is64bit */ false, GetVIXLAssembler()); 221} 222void IntrinsicCodeGeneratorARM64::VisitFloatIntBitsToFloat(HInvoke* invoke) { 223 MoveIntToFP(invoke->GetLocations(), /* is64bit */ false, GetVIXLAssembler()); 224} 225 226static void CreateIntToIntLocations(ArenaAllocator* arena, HInvoke* invoke) { 227 LocationSummary* locations = new (arena) LocationSummary(invoke, 228 LocationSummary::kNoCall, 229 kIntrinsified); 230 locations->SetInAt(0, Location::RequiresRegister()); 231 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 232} 233 234static void GenReverseBytes(LocationSummary* locations, 235 Primitive::Type type, 236 vixl::MacroAssembler* masm) { 237 Location in = locations->InAt(0); 238 Location out = locations->Out(); 239 240 switch (type) { 241 case Primitive::kPrimShort: 242 __ Rev16(WRegisterFrom(out), WRegisterFrom(in)); 243 __ Sxth(WRegisterFrom(out), WRegisterFrom(out)); 244 break; 245 case Primitive::kPrimInt: 246 case Primitive::kPrimLong: 247 __ Rev(RegisterFrom(out, type), RegisterFrom(in, type)); 248 break; 249 default: 250 LOG(FATAL) << "Unexpected size for reverse-bytes: " << type; 251 UNREACHABLE(); 252 } 253} 254 255void IntrinsicLocationsBuilderARM64::VisitIntegerReverseBytes(HInvoke* invoke) { 256 CreateIntToIntLocations(arena_, invoke); 257} 258 259void IntrinsicCodeGeneratorARM64::VisitIntegerReverseBytes(HInvoke* invoke) { 260 GenReverseBytes(invoke->GetLocations(), Primitive::kPrimInt, GetVIXLAssembler()); 261} 262 263void IntrinsicLocationsBuilderARM64::VisitLongReverseBytes(HInvoke* invoke) { 264 CreateIntToIntLocations(arena_, invoke); 265} 266 267void IntrinsicCodeGeneratorARM64::VisitLongReverseBytes(HInvoke* invoke) { 268 GenReverseBytes(invoke->GetLocations(), Primitive::kPrimLong, GetVIXLAssembler()); 269} 270 271void IntrinsicLocationsBuilderARM64::VisitShortReverseBytes(HInvoke* invoke) { 272 CreateIntToIntLocations(arena_, invoke); 273} 274 275void IntrinsicCodeGeneratorARM64::VisitShortReverseBytes(HInvoke* invoke) { 276 GenReverseBytes(invoke->GetLocations(), Primitive::kPrimShort, GetVIXLAssembler()); 277} 278 279static void CreateIntIntToIntLocations(ArenaAllocator* arena, HInvoke* invoke) { 280 LocationSummary* locations = new (arena) LocationSummary(invoke, 281 LocationSummary::kNoCall, 282 kIntrinsified); 283 locations->SetInAt(0, Location::RequiresRegister()); 284 locations->SetInAt(1, Location::RequiresRegister()); 285 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 286} 287 288static void GenNumberOfLeadingZeros(LocationSummary* locations, 289 Primitive::Type type, 290 vixl::MacroAssembler* masm) { 291 DCHECK(type == Primitive::kPrimInt || type == Primitive::kPrimLong); 292 293 Location in = locations->InAt(0); 294 Location out = locations->Out(); 295 296 __ Clz(RegisterFrom(out, type), RegisterFrom(in, type)); 297} 298 299void IntrinsicLocationsBuilderARM64::VisitIntegerNumberOfLeadingZeros(HInvoke* invoke) { 300 CreateIntToIntLocations(arena_, invoke); 301} 302 303void IntrinsicCodeGeneratorARM64::VisitIntegerNumberOfLeadingZeros(HInvoke* invoke) { 304 GenNumberOfLeadingZeros(invoke->GetLocations(), Primitive::kPrimInt, GetVIXLAssembler()); 305} 306 307void IntrinsicLocationsBuilderARM64::VisitLongNumberOfLeadingZeros(HInvoke* invoke) { 308 CreateIntToIntLocations(arena_, invoke); 309} 310 311void IntrinsicCodeGeneratorARM64::VisitLongNumberOfLeadingZeros(HInvoke* invoke) { 312 GenNumberOfLeadingZeros(invoke->GetLocations(), Primitive::kPrimLong, GetVIXLAssembler()); 313} 314 315static void GenNumberOfTrailingZeros(LocationSummary* locations, 316 Primitive::Type type, 317 vixl::MacroAssembler* masm) { 318 DCHECK(type == Primitive::kPrimInt || type == Primitive::kPrimLong); 319 320 Location in = locations->InAt(0); 321 Location out = locations->Out(); 322 323 __ Rbit(RegisterFrom(out, type), RegisterFrom(in, type)); 324 __ Clz(RegisterFrom(out, type), RegisterFrom(out, type)); 325} 326 327void IntrinsicLocationsBuilderARM64::VisitIntegerNumberOfTrailingZeros(HInvoke* invoke) { 328 CreateIntToIntLocations(arena_, invoke); 329} 330 331void IntrinsicCodeGeneratorARM64::VisitIntegerNumberOfTrailingZeros(HInvoke* invoke) { 332 GenNumberOfTrailingZeros(invoke->GetLocations(), Primitive::kPrimInt, GetVIXLAssembler()); 333} 334 335void IntrinsicLocationsBuilderARM64::VisitLongNumberOfTrailingZeros(HInvoke* invoke) { 336 CreateIntToIntLocations(arena_, invoke); 337} 338 339void IntrinsicCodeGeneratorARM64::VisitLongNumberOfTrailingZeros(HInvoke* invoke) { 340 GenNumberOfTrailingZeros(invoke->GetLocations(), Primitive::kPrimLong, GetVIXLAssembler()); 341} 342 343static void GenReverse(LocationSummary* locations, 344 Primitive::Type type, 345 vixl::MacroAssembler* masm) { 346 DCHECK(type == Primitive::kPrimInt || type == Primitive::kPrimLong); 347 348 Location in = locations->InAt(0); 349 Location out = locations->Out(); 350 351 __ Rbit(RegisterFrom(out, type), RegisterFrom(in, type)); 352} 353 354void IntrinsicLocationsBuilderARM64::VisitIntegerReverse(HInvoke* invoke) { 355 CreateIntToIntLocations(arena_, invoke); 356} 357 358void IntrinsicCodeGeneratorARM64::VisitIntegerReverse(HInvoke* invoke) { 359 GenReverse(invoke->GetLocations(), Primitive::kPrimInt, GetVIXLAssembler()); 360} 361 362void IntrinsicLocationsBuilderARM64::VisitLongReverse(HInvoke* invoke) { 363 CreateIntToIntLocations(arena_, invoke); 364} 365 366void IntrinsicCodeGeneratorARM64::VisitLongReverse(HInvoke* invoke) { 367 GenReverse(invoke->GetLocations(), Primitive::kPrimLong, GetVIXLAssembler()); 368} 369 370static void CreateFPToFPLocations(ArenaAllocator* arena, HInvoke* invoke) { 371 LocationSummary* locations = new (arena) LocationSummary(invoke, 372 LocationSummary::kNoCall, 373 kIntrinsified); 374 locations->SetInAt(0, Location::RequiresFpuRegister()); 375 locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap); 376} 377 378static void MathAbsFP(LocationSummary* locations, bool is64bit, vixl::MacroAssembler* masm) { 379 Location in = locations->InAt(0); 380 Location out = locations->Out(); 381 382 FPRegister in_reg = is64bit ? DRegisterFrom(in) : SRegisterFrom(in); 383 FPRegister out_reg = is64bit ? DRegisterFrom(out) : SRegisterFrom(out); 384 385 __ Fabs(out_reg, in_reg); 386} 387 388void IntrinsicLocationsBuilderARM64::VisitMathAbsDouble(HInvoke* invoke) { 389 CreateFPToFPLocations(arena_, invoke); 390} 391 392void IntrinsicCodeGeneratorARM64::VisitMathAbsDouble(HInvoke* invoke) { 393 MathAbsFP(invoke->GetLocations(), /* is64bit */ true, GetVIXLAssembler()); 394} 395 396void IntrinsicLocationsBuilderARM64::VisitMathAbsFloat(HInvoke* invoke) { 397 CreateFPToFPLocations(arena_, invoke); 398} 399 400void IntrinsicCodeGeneratorARM64::VisitMathAbsFloat(HInvoke* invoke) { 401 MathAbsFP(invoke->GetLocations(), /* is64bit */ false, GetVIXLAssembler()); 402} 403 404static void CreateIntToInt(ArenaAllocator* arena, HInvoke* invoke) { 405 LocationSummary* locations = new (arena) LocationSummary(invoke, 406 LocationSummary::kNoCall, 407 kIntrinsified); 408 locations->SetInAt(0, Location::RequiresRegister()); 409 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 410} 411 412static void GenAbsInteger(LocationSummary* locations, 413 bool is64bit, 414 vixl::MacroAssembler* masm) { 415 Location in = locations->InAt(0); 416 Location output = locations->Out(); 417 418 Register in_reg = is64bit ? XRegisterFrom(in) : WRegisterFrom(in); 419 Register out_reg = is64bit ? XRegisterFrom(output) : WRegisterFrom(output); 420 421 __ Cmp(in_reg, Operand(0)); 422 __ Cneg(out_reg, in_reg, lt); 423} 424 425void IntrinsicLocationsBuilderARM64::VisitMathAbsInt(HInvoke* invoke) { 426 CreateIntToInt(arena_, invoke); 427} 428 429void IntrinsicCodeGeneratorARM64::VisitMathAbsInt(HInvoke* invoke) { 430 GenAbsInteger(invoke->GetLocations(), /* is64bit */ false, GetVIXLAssembler()); 431} 432 433void IntrinsicLocationsBuilderARM64::VisitMathAbsLong(HInvoke* invoke) { 434 CreateIntToInt(arena_, invoke); 435} 436 437void IntrinsicCodeGeneratorARM64::VisitMathAbsLong(HInvoke* invoke) { 438 GenAbsInteger(invoke->GetLocations(), /* is64bit */ true, GetVIXLAssembler()); 439} 440 441static void GenMinMaxFP(LocationSummary* locations, 442 bool is_min, 443 bool is_double, 444 vixl::MacroAssembler* masm) { 445 Location op1 = locations->InAt(0); 446 Location op2 = locations->InAt(1); 447 Location out = locations->Out(); 448 449 FPRegister op1_reg = is_double ? DRegisterFrom(op1) : SRegisterFrom(op1); 450 FPRegister op2_reg = is_double ? DRegisterFrom(op2) : SRegisterFrom(op2); 451 FPRegister out_reg = is_double ? DRegisterFrom(out) : SRegisterFrom(out); 452 if (is_min) { 453 __ Fmin(out_reg, op1_reg, op2_reg); 454 } else { 455 __ Fmax(out_reg, op1_reg, op2_reg); 456 } 457} 458 459static void CreateFPFPToFPLocations(ArenaAllocator* arena, HInvoke* invoke) { 460 LocationSummary* locations = new (arena) LocationSummary(invoke, 461 LocationSummary::kNoCall, 462 kIntrinsified); 463 locations->SetInAt(0, Location::RequiresFpuRegister()); 464 locations->SetInAt(1, Location::RequiresFpuRegister()); 465 locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap); 466} 467 468void IntrinsicLocationsBuilderARM64::VisitMathMinDoubleDouble(HInvoke* invoke) { 469 CreateFPFPToFPLocations(arena_, invoke); 470} 471 472void IntrinsicCodeGeneratorARM64::VisitMathMinDoubleDouble(HInvoke* invoke) { 473 GenMinMaxFP(invoke->GetLocations(), /* is_min */ true, /* is_double */ true, GetVIXLAssembler()); 474} 475 476void IntrinsicLocationsBuilderARM64::VisitMathMinFloatFloat(HInvoke* invoke) { 477 CreateFPFPToFPLocations(arena_, invoke); 478} 479 480void IntrinsicCodeGeneratorARM64::VisitMathMinFloatFloat(HInvoke* invoke) { 481 GenMinMaxFP(invoke->GetLocations(), /* is_min */ true, /* is_double */ false, GetVIXLAssembler()); 482} 483 484void IntrinsicLocationsBuilderARM64::VisitMathMaxDoubleDouble(HInvoke* invoke) { 485 CreateFPFPToFPLocations(arena_, invoke); 486} 487 488void IntrinsicCodeGeneratorARM64::VisitMathMaxDoubleDouble(HInvoke* invoke) { 489 GenMinMaxFP(invoke->GetLocations(), /* is_min */ false, /* is_double */ true, GetVIXLAssembler()); 490} 491 492void IntrinsicLocationsBuilderARM64::VisitMathMaxFloatFloat(HInvoke* invoke) { 493 CreateFPFPToFPLocations(arena_, invoke); 494} 495 496void IntrinsicCodeGeneratorARM64::VisitMathMaxFloatFloat(HInvoke* invoke) { 497 GenMinMaxFP( 498 invoke->GetLocations(), /* is_min */ false, /* is_double */ false, GetVIXLAssembler()); 499} 500 501static void GenMinMax(LocationSummary* locations, 502 bool is_min, 503 bool is_long, 504 vixl::MacroAssembler* masm) { 505 Location op1 = locations->InAt(0); 506 Location op2 = locations->InAt(1); 507 Location out = locations->Out(); 508 509 Register op1_reg = is_long ? XRegisterFrom(op1) : WRegisterFrom(op1); 510 Register op2_reg = is_long ? XRegisterFrom(op2) : WRegisterFrom(op2); 511 Register out_reg = is_long ? XRegisterFrom(out) : WRegisterFrom(out); 512 513 __ Cmp(op1_reg, op2_reg); 514 __ Csel(out_reg, op1_reg, op2_reg, is_min ? lt : gt); 515} 516 517void IntrinsicLocationsBuilderARM64::VisitMathMinIntInt(HInvoke* invoke) { 518 CreateIntIntToIntLocations(arena_, invoke); 519} 520 521void IntrinsicCodeGeneratorARM64::VisitMathMinIntInt(HInvoke* invoke) { 522 GenMinMax(invoke->GetLocations(), /* is_min */ true, /* is_long */ false, GetVIXLAssembler()); 523} 524 525void IntrinsicLocationsBuilderARM64::VisitMathMinLongLong(HInvoke* invoke) { 526 CreateIntIntToIntLocations(arena_, invoke); 527} 528 529void IntrinsicCodeGeneratorARM64::VisitMathMinLongLong(HInvoke* invoke) { 530 GenMinMax(invoke->GetLocations(), /* is_min */ true, /* is_long */ true, GetVIXLAssembler()); 531} 532 533void IntrinsicLocationsBuilderARM64::VisitMathMaxIntInt(HInvoke* invoke) { 534 CreateIntIntToIntLocations(arena_, invoke); 535} 536 537void IntrinsicCodeGeneratorARM64::VisitMathMaxIntInt(HInvoke* invoke) { 538 GenMinMax(invoke->GetLocations(), /* is_min */ false, /* is_long */ false, GetVIXLAssembler()); 539} 540 541void IntrinsicLocationsBuilderARM64::VisitMathMaxLongLong(HInvoke* invoke) { 542 CreateIntIntToIntLocations(arena_, invoke); 543} 544 545void IntrinsicCodeGeneratorARM64::VisitMathMaxLongLong(HInvoke* invoke) { 546 GenMinMax(invoke->GetLocations(), /* is_min */ false, /* is_long */ true, GetVIXLAssembler()); 547} 548 549void IntrinsicLocationsBuilderARM64::VisitMathSqrt(HInvoke* invoke) { 550 CreateFPToFPLocations(arena_, invoke); 551} 552 553void IntrinsicCodeGeneratorARM64::VisitMathSqrt(HInvoke* invoke) { 554 LocationSummary* locations = invoke->GetLocations(); 555 vixl::MacroAssembler* masm = GetVIXLAssembler(); 556 __ Fsqrt(DRegisterFrom(locations->Out()), DRegisterFrom(locations->InAt(0))); 557} 558 559void IntrinsicLocationsBuilderARM64::VisitMathCeil(HInvoke* invoke) { 560 CreateFPToFPLocations(arena_, invoke); 561} 562 563void IntrinsicCodeGeneratorARM64::VisitMathCeil(HInvoke* invoke) { 564 LocationSummary* locations = invoke->GetLocations(); 565 vixl::MacroAssembler* masm = GetVIXLAssembler(); 566 __ Frintp(DRegisterFrom(locations->Out()), DRegisterFrom(locations->InAt(0))); 567} 568 569void IntrinsicLocationsBuilderARM64::VisitMathFloor(HInvoke* invoke) { 570 CreateFPToFPLocations(arena_, invoke); 571} 572 573void IntrinsicCodeGeneratorARM64::VisitMathFloor(HInvoke* invoke) { 574 LocationSummary* locations = invoke->GetLocations(); 575 vixl::MacroAssembler* masm = GetVIXLAssembler(); 576 __ Frintm(DRegisterFrom(locations->Out()), DRegisterFrom(locations->InAt(0))); 577} 578 579void IntrinsicLocationsBuilderARM64::VisitMathRint(HInvoke* invoke) { 580 CreateFPToFPLocations(arena_, invoke); 581} 582 583void IntrinsicCodeGeneratorARM64::VisitMathRint(HInvoke* invoke) { 584 LocationSummary* locations = invoke->GetLocations(); 585 vixl::MacroAssembler* masm = GetVIXLAssembler(); 586 __ Frintn(DRegisterFrom(locations->Out()), DRegisterFrom(locations->InAt(0))); 587} 588 589static void CreateFPToIntPlusTempLocations(ArenaAllocator* arena, HInvoke* invoke) { 590 LocationSummary* locations = new (arena) LocationSummary(invoke, 591 LocationSummary::kNoCall, 592 kIntrinsified); 593 locations->SetInAt(0, Location::RequiresFpuRegister()); 594 locations->SetOut(Location::RequiresRegister()); 595} 596 597static void GenMathRound(LocationSummary* locations, 598 bool is_double, 599 vixl::MacroAssembler* masm) { 600 FPRegister in_reg = is_double ? 601 DRegisterFrom(locations->InAt(0)) : SRegisterFrom(locations->InAt(0)); 602 Register out_reg = is_double ? 603 XRegisterFrom(locations->Out()) : WRegisterFrom(locations->Out()); 604 UseScratchRegisterScope temps(masm); 605 FPRegister temp1_reg = temps.AcquireSameSizeAs(in_reg); 606 607 // 0.5 can be encoded as an immediate, so use fmov. 608 if (is_double) { 609 __ Fmov(temp1_reg, static_cast<double>(0.5)); 610 } else { 611 __ Fmov(temp1_reg, static_cast<float>(0.5)); 612 } 613 __ Fadd(temp1_reg, in_reg, temp1_reg); 614 __ Fcvtms(out_reg, temp1_reg); 615} 616 617void IntrinsicLocationsBuilderARM64::VisitMathRoundDouble(HInvoke* invoke) { 618 // See intrinsics.h. 619 if (kRoundIsPlusPointFive) { 620 CreateFPToIntPlusTempLocations(arena_, invoke); 621 } 622} 623 624void IntrinsicCodeGeneratorARM64::VisitMathRoundDouble(HInvoke* invoke) { 625 GenMathRound(invoke->GetLocations(), /* is_double */ true, GetVIXLAssembler()); 626} 627 628void IntrinsicLocationsBuilderARM64::VisitMathRoundFloat(HInvoke* invoke) { 629 // See intrinsics.h. 630 if (kRoundIsPlusPointFive) { 631 CreateFPToIntPlusTempLocations(arena_, invoke); 632 } 633} 634 635void IntrinsicCodeGeneratorARM64::VisitMathRoundFloat(HInvoke* invoke) { 636 GenMathRound(invoke->GetLocations(), /* is_double */ false, GetVIXLAssembler()); 637} 638 639void IntrinsicLocationsBuilderARM64::VisitMemoryPeekByte(HInvoke* invoke) { 640 CreateIntToIntLocations(arena_, invoke); 641} 642 643void IntrinsicCodeGeneratorARM64::VisitMemoryPeekByte(HInvoke* invoke) { 644 vixl::MacroAssembler* masm = GetVIXLAssembler(); 645 __ Ldrsb(WRegisterFrom(invoke->GetLocations()->Out()), 646 AbsoluteHeapOperandFrom(invoke->GetLocations()->InAt(0), 0)); 647} 648 649void IntrinsicLocationsBuilderARM64::VisitMemoryPeekIntNative(HInvoke* invoke) { 650 CreateIntToIntLocations(arena_, invoke); 651} 652 653void IntrinsicCodeGeneratorARM64::VisitMemoryPeekIntNative(HInvoke* invoke) { 654 vixl::MacroAssembler* masm = GetVIXLAssembler(); 655 __ Ldr(WRegisterFrom(invoke->GetLocations()->Out()), 656 AbsoluteHeapOperandFrom(invoke->GetLocations()->InAt(0), 0)); 657} 658 659void IntrinsicLocationsBuilderARM64::VisitMemoryPeekLongNative(HInvoke* invoke) { 660 CreateIntToIntLocations(arena_, invoke); 661} 662 663void IntrinsicCodeGeneratorARM64::VisitMemoryPeekLongNative(HInvoke* invoke) { 664 vixl::MacroAssembler* masm = GetVIXLAssembler(); 665 __ Ldr(XRegisterFrom(invoke->GetLocations()->Out()), 666 AbsoluteHeapOperandFrom(invoke->GetLocations()->InAt(0), 0)); 667} 668 669void IntrinsicLocationsBuilderARM64::VisitMemoryPeekShortNative(HInvoke* invoke) { 670 CreateIntToIntLocations(arena_, invoke); 671} 672 673void IntrinsicCodeGeneratorARM64::VisitMemoryPeekShortNative(HInvoke* invoke) { 674 vixl::MacroAssembler* masm = GetVIXLAssembler(); 675 __ Ldrsh(WRegisterFrom(invoke->GetLocations()->Out()), 676 AbsoluteHeapOperandFrom(invoke->GetLocations()->InAt(0), 0)); 677} 678 679static void CreateIntIntToVoidLocations(ArenaAllocator* arena, HInvoke* invoke) { 680 LocationSummary* locations = new (arena) LocationSummary(invoke, 681 LocationSummary::kNoCall, 682 kIntrinsified); 683 locations->SetInAt(0, Location::RequiresRegister()); 684 locations->SetInAt(1, Location::RequiresRegister()); 685} 686 687void IntrinsicLocationsBuilderARM64::VisitMemoryPokeByte(HInvoke* invoke) { 688 CreateIntIntToVoidLocations(arena_, invoke); 689} 690 691void IntrinsicCodeGeneratorARM64::VisitMemoryPokeByte(HInvoke* invoke) { 692 vixl::MacroAssembler* masm = GetVIXLAssembler(); 693 __ Strb(WRegisterFrom(invoke->GetLocations()->InAt(1)), 694 AbsoluteHeapOperandFrom(invoke->GetLocations()->InAt(0), 0)); 695} 696 697void IntrinsicLocationsBuilderARM64::VisitMemoryPokeIntNative(HInvoke* invoke) { 698 CreateIntIntToVoidLocations(arena_, invoke); 699} 700 701void IntrinsicCodeGeneratorARM64::VisitMemoryPokeIntNative(HInvoke* invoke) { 702 vixl::MacroAssembler* masm = GetVIXLAssembler(); 703 __ Str(WRegisterFrom(invoke->GetLocations()->InAt(1)), 704 AbsoluteHeapOperandFrom(invoke->GetLocations()->InAt(0), 0)); 705} 706 707void IntrinsicLocationsBuilderARM64::VisitMemoryPokeLongNative(HInvoke* invoke) { 708 CreateIntIntToVoidLocations(arena_, invoke); 709} 710 711void IntrinsicCodeGeneratorARM64::VisitMemoryPokeLongNative(HInvoke* invoke) { 712 vixl::MacroAssembler* masm = GetVIXLAssembler(); 713 __ Str(XRegisterFrom(invoke->GetLocations()->InAt(1)), 714 AbsoluteHeapOperandFrom(invoke->GetLocations()->InAt(0), 0)); 715} 716 717void IntrinsicLocationsBuilderARM64::VisitMemoryPokeShortNative(HInvoke* invoke) { 718 CreateIntIntToVoidLocations(arena_, invoke); 719} 720 721void IntrinsicCodeGeneratorARM64::VisitMemoryPokeShortNative(HInvoke* invoke) { 722 vixl::MacroAssembler* masm = GetVIXLAssembler(); 723 __ Strh(WRegisterFrom(invoke->GetLocations()->InAt(1)), 724 AbsoluteHeapOperandFrom(invoke->GetLocations()->InAt(0), 0)); 725} 726 727void IntrinsicLocationsBuilderARM64::VisitThreadCurrentThread(HInvoke* invoke) { 728 LocationSummary* locations = new (arena_) LocationSummary(invoke, 729 LocationSummary::kNoCall, 730 kIntrinsified); 731 locations->SetOut(Location::RequiresRegister()); 732} 733 734void IntrinsicCodeGeneratorARM64::VisitThreadCurrentThread(HInvoke* invoke) { 735 codegen_->Load(Primitive::kPrimNot, WRegisterFrom(invoke->GetLocations()->Out()), 736 MemOperand(tr, Thread::PeerOffset<8>().Int32Value())); 737} 738 739static void GenUnsafeGet(HInvoke* invoke, 740 Primitive::Type type, 741 bool is_volatile, 742 CodeGeneratorARM64* codegen) { 743 LocationSummary* locations = invoke->GetLocations(); 744 DCHECK((type == Primitive::kPrimInt) || 745 (type == Primitive::kPrimLong) || 746 (type == Primitive::kPrimNot)); 747 vixl::MacroAssembler* masm = codegen->GetAssembler()->vixl_masm_; 748 Location base_loc = locations->InAt(1); 749 Register base = WRegisterFrom(base_loc); // Object pointer. 750 Location offset_loc = locations->InAt(2); 751 Register offset = XRegisterFrom(offset_loc); // Long offset. 752 Location trg_loc = locations->Out(); 753 Register trg = RegisterFrom(trg_loc, type); 754 755 if (type == Primitive::kPrimNot && kEmitCompilerReadBarrier && kUseBakerReadBarrier) { 756 // UnsafeGetObject/UnsafeGetObjectVolatile with Baker's read barrier case. 757 UseScratchRegisterScope temps(masm); 758 Register temp = temps.AcquireW(); 759 codegen->GenerateArrayLoadWithBakerReadBarrier( 760 invoke, trg_loc, base, 0U, offset_loc, temp, /* needs_null_check */ false); 761 } else { 762 // Other cases. 763 MemOperand mem_op(base.X(), offset); 764 if (is_volatile) { 765 codegen->LoadAcquire(invoke, trg, mem_op, /* needs_null_check */ true); 766 } else { 767 codegen->Load(type, trg, mem_op); 768 } 769 770 if (type == Primitive::kPrimNot) { 771 DCHECK(trg.IsW()); 772 codegen->MaybeGenerateReadBarrierSlow(invoke, trg_loc, trg_loc, base_loc, 0U, offset_loc); 773 } 774 } 775} 776 777static void CreateIntIntIntToIntLocations(ArenaAllocator* arena, HInvoke* invoke) { 778 bool can_call = kEmitCompilerReadBarrier && 779 (invoke->GetIntrinsic() == Intrinsics::kUnsafeGetObject || 780 invoke->GetIntrinsic() == Intrinsics::kUnsafeGetObjectVolatile); 781 LocationSummary* locations = new (arena) LocationSummary(invoke, 782 can_call ? 783 LocationSummary::kCallOnSlowPath : 784 LocationSummary::kNoCall, 785 kIntrinsified); 786 locations->SetInAt(0, Location::NoLocation()); // Unused receiver. 787 locations->SetInAt(1, Location::RequiresRegister()); 788 locations->SetInAt(2, Location::RequiresRegister()); 789 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 790} 791 792void IntrinsicLocationsBuilderARM64::VisitUnsafeGet(HInvoke* invoke) { 793 CreateIntIntIntToIntLocations(arena_, invoke); 794} 795void IntrinsicLocationsBuilderARM64::VisitUnsafeGetVolatile(HInvoke* invoke) { 796 CreateIntIntIntToIntLocations(arena_, invoke); 797} 798void IntrinsicLocationsBuilderARM64::VisitUnsafeGetLong(HInvoke* invoke) { 799 CreateIntIntIntToIntLocations(arena_, invoke); 800} 801void IntrinsicLocationsBuilderARM64::VisitUnsafeGetLongVolatile(HInvoke* invoke) { 802 CreateIntIntIntToIntLocations(arena_, invoke); 803} 804void IntrinsicLocationsBuilderARM64::VisitUnsafeGetObject(HInvoke* invoke) { 805 CreateIntIntIntToIntLocations(arena_, invoke); 806} 807void IntrinsicLocationsBuilderARM64::VisitUnsafeGetObjectVolatile(HInvoke* invoke) { 808 CreateIntIntIntToIntLocations(arena_, invoke); 809} 810 811void IntrinsicCodeGeneratorARM64::VisitUnsafeGet(HInvoke* invoke) { 812 GenUnsafeGet(invoke, Primitive::kPrimInt, /* is_volatile */ false, codegen_); 813} 814void IntrinsicCodeGeneratorARM64::VisitUnsafeGetVolatile(HInvoke* invoke) { 815 GenUnsafeGet(invoke, Primitive::kPrimInt, /* is_volatile */ true, codegen_); 816} 817void IntrinsicCodeGeneratorARM64::VisitUnsafeGetLong(HInvoke* invoke) { 818 GenUnsafeGet(invoke, Primitive::kPrimLong, /* is_volatile */ false, codegen_); 819} 820void IntrinsicCodeGeneratorARM64::VisitUnsafeGetLongVolatile(HInvoke* invoke) { 821 GenUnsafeGet(invoke, Primitive::kPrimLong, /* is_volatile */ true, codegen_); 822} 823void IntrinsicCodeGeneratorARM64::VisitUnsafeGetObject(HInvoke* invoke) { 824 GenUnsafeGet(invoke, Primitive::kPrimNot, /* is_volatile */ false, codegen_); 825} 826void IntrinsicCodeGeneratorARM64::VisitUnsafeGetObjectVolatile(HInvoke* invoke) { 827 GenUnsafeGet(invoke, Primitive::kPrimNot, /* is_volatile */ true, codegen_); 828} 829 830static void CreateIntIntIntIntToVoid(ArenaAllocator* arena, HInvoke* invoke) { 831 LocationSummary* locations = new (arena) LocationSummary(invoke, 832 LocationSummary::kNoCall, 833 kIntrinsified); 834 locations->SetInAt(0, Location::NoLocation()); // Unused receiver. 835 locations->SetInAt(1, Location::RequiresRegister()); 836 locations->SetInAt(2, Location::RequiresRegister()); 837 locations->SetInAt(3, Location::RequiresRegister()); 838} 839 840void IntrinsicLocationsBuilderARM64::VisitUnsafePut(HInvoke* invoke) { 841 CreateIntIntIntIntToVoid(arena_, invoke); 842} 843void IntrinsicLocationsBuilderARM64::VisitUnsafePutOrdered(HInvoke* invoke) { 844 CreateIntIntIntIntToVoid(arena_, invoke); 845} 846void IntrinsicLocationsBuilderARM64::VisitUnsafePutVolatile(HInvoke* invoke) { 847 CreateIntIntIntIntToVoid(arena_, invoke); 848} 849void IntrinsicLocationsBuilderARM64::VisitUnsafePutObject(HInvoke* invoke) { 850 CreateIntIntIntIntToVoid(arena_, invoke); 851} 852void IntrinsicLocationsBuilderARM64::VisitUnsafePutObjectOrdered(HInvoke* invoke) { 853 CreateIntIntIntIntToVoid(arena_, invoke); 854} 855void IntrinsicLocationsBuilderARM64::VisitUnsafePutObjectVolatile(HInvoke* invoke) { 856 CreateIntIntIntIntToVoid(arena_, invoke); 857} 858void IntrinsicLocationsBuilderARM64::VisitUnsafePutLong(HInvoke* invoke) { 859 CreateIntIntIntIntToVoid(arena_, invoke); 860} 861void IntrinsicLocationsBuilderARM64::VisitUnsafePutLongOrdered(HInvoke* invoke) { 862 CreateIntIntIntIntToVoid(arena_, invoke); 863} 864void IntrinsicLocationsBuilderARM64::VisitUnsafePutLongVolatile(HInvoke* invoke) { 865 CreateIntIntIntIntToVoid(arena_, invoke); 866} 867 868static void GenUnsafePut(LocationSummary* locations, 869 Primitive::Type type, 870 bool is_volatile, 871 bool is_ordered, 872 CodeGeneratorARM64* codegen) { 873 vixl::MacroAssembler* masm = codegen->GetAssembler()->vixl_masm_; 874 875 Register base = WRegisterFrom(locations->InAt(1)); // Object pointer. 876 Register offset = XRegisterFrom(locations->InAt(2)); // Long offset. 877 Register value = RegisterFrom(locations->InAt(3), type); 878 Register source = value; 879 MemOperand mem_op(base.X(), offset); 880 881 { 882 // We use a block to end the scratch scope before the write barrier, thus 883 // freeing the temporary registers so they can be used in `MarkGCCard`. 884 UseScratchRegisterScope temps(masm); 885 886 if (kPoisonHeapReferences && type == Primitive::kPrimNot) { 887 DCHECK(value.IsW()); 888 Register temp = temps.AcquireW(); 889 __ Mov(temp.W(), value.W()); 890 codegen->GetAssembler()->PoisonHeapReference(temp.W()); 891 source = temp; 892 } 893 894 if (is_volatile || is_ordered) { 895 codegen->StoreRelease(type, source, mem_op); 896 } else { 897 codegen->Store(type, source, mem_op); 898 } 899 } 900 901 if (type == Primitive::kPrimNot) { 902 bool value_can_be_null = true; // TODO: Worth finding out this information? 903 codegen->MarkGCCard(base, value, value_can_be_null); 904 } 905} 906 907void IntrinsicCodeGeneratorARM64::VisitUnsafePut(HInvoke* invoke) { 908 GenUnsafePut(invoke->GetLocations(), 909 Primitive::kPrimInt, 910 /* is_volatile */ false, 911 /* is_ordered */ false, 912 codegen_); 913} 914void IntrinsicCodeGeneratorARM64::VisitUnsafePutOrdered(HInvoke* invoke) { 915 GenUnsafePut(invoke->GetLocations(), 916 Primitive::kPrimInt, 917 /* is_volatile */ false, 918 /* is_ordered */ true, 919 codegen_); 920} 921void IntrinsicCodeGeneratorARM64::VisitUnsafePutVolatile(HInvoke* invoke) { 922 GenUnsafePut(invoke->GetLocations(), 923 Primitive::kPrimInt, 924 /* is_volatile */ true, 925 /* is_ordered */ false, 926 codegen_); 927} 928void IntrinsicCodeGeneratorARM64::VisitUnsafePutObject(HInvoke* invoke) { 929 GenUnsafePut(invoke->GetLocations(), 930 Primitive::kPrimNot, 931 /* is_volatile */ false, 932 /* is_ordered */ false, 933 codegen_); 934} 935void IntrinsicCodeGeneratorARM64::VisitUnsafePutObjectOrdered(HInvoke* invoke) { 936 GenUnsafePut(invoke->GetLocations(), 937 Primitive::kPrimNot, 938 /* is_volatile */ false, 939 /* is_ordered */ true, 940 codegen_); 941} 942void IntrinsicCodeGeneratorARM64::VisitUnsafePutObjectVolatile(HInvoke* invoke) { 943 GenUnsafePut(invoke->GetLocations(), 944 Primitive::kPrimNot, 945 /* is_volatile */ true, 946 /* is_ordered */ false, 947 codegen_); 948} 949void IntrinsicCodeGeneratorARM64::VisitUnsafePutLong(HInvoke* invoke) { 950 GenUnsafePut(invoke->GetLocations(), 951 Primitive::kPrimLong, 952 /* is_volatile */ false, 953 /* is_ordered */ false, 954 codegen_); 955} 956void IntrinsicCodeGeneratorARM64::VisitUnsafePutLongOrdered(HInvoke* invoke) { 957 GenUnsafePut(invoke->GetLocations(), 958 Primitive::kPrimLong, 959 /* is_volatile */ false, 960 /* is_ordered */ true, 961 codegen_); 962} 963void IntrinsicCodeGeneratorARM64::VisitUnsafePutLongVolatile(HInvoke* invoke) { 964 GenUnsafePut(invoke->GetLocations(), 965 Primitive::kPrimLong, 966 /* is_volatile */ true, 967 /* is_ordered */ false, 968 codegen_); 969} 970 971static void CreateIntIntIntIntIntToInt(ArenaAllocator* arena, 972 HInvoke* invoke, 973 Primitive::Type type) { 974 LocationSummary* locations = new (arena) LocationSummary(invoke, 975 LocationSummary::kNoCall, 976 kIntrinsified); 977 locations->SetInAt(0, Location::NoLocation()); // Unused receiver. 978 locations->SetInAt(1, Location::RequiresRegister()); 979 locations->SetInAt(2, Location::RequiresRegister()); 980 locations->SetInAt(3, Location::RequiresRegister()); 981 locations->SetInAt(4, Location::RequiresRegister()); 982 983 // If heap poisoning is enabled, we don't want the unpoisoning 984 // operations to potentially clobber the output. 985 Location::OutputOverlap overlaps = (kPoisonHeapReferences && type == Primitive::kPrimNot) 986 ? Location::kOutputOverlap 987 : Location::kNoOutputOverlap; 988 locations->SetOut(Location::RequiresRegister(), overlaps); 989} 990 991static void GenCas(LocationSummary* locations, Primitive::Type type, CodeGeneratorARM64* codegen) { 992 vixl::MacroAssembler* masm = codegen->GetAssembler()->vixl_masm_; 993 994 Register out = WRegisterFrom(locations->Out()); // Boolean result. 995 996 Register base = WRegisterFrom(locations->InAt(1)); // Object pointer. 997 Register offset = XRegisterFrom(locations->InAt(2)); // Long offset. 998 Register expected = RegisterFrom(locations->InAt(3), type); // Expected. 999 Register value = RegisterFrom(locations->InAt(4), type); // Value. 1000 1001 // This needs to be before the temp registers, as MarkGCCard also uses VIXL temps. 1002 if (type == Primitive::kPrimNot) { 1003 // Mark card for object assuming new value is stored. 1004 bool value_can_be_null = true; // TODO: Worth finding out this information? 1005 codegen->MarkGCCard(base, value, value_can_be_null); 1006 } 1007 1008 UseScratchRegisterScope temps(masm); 1009 Register tmp_ptr = temps.AcquireX(); // Pointer to actual memory. 1010 Register tmp_value = temps.AcquireSameSizeAs(value); // Value in memory. 1011 1012 Register tmp_32 = tmp_value.W(); 1013 1014 __ Add(tmp_ptr, base.X(), Operand(offset)); 1015 1016 if (kPoisonHeapReferences && type == Primitive::kPrimNot) { 1017 codegen->GetAssembler()->PoisonHeapReference(expected); 1018 if (value.Is(expected)) { 1019 // Do not poison `value`, as it is the same register as 1020 // `expected`, which has just been poisoned. 1021 } else { 1022 codegen->GetAssembler()->PoisonHeapReference(value); 1023 } 1024 } 1025 1026 // do { 1027 // tmp_value = [tmp_ptr] - expected; 1028 // } while (tmp_value == 0 && failure([tmp_ptr] <- r_new_value)); 1029 // result = tmp_value != 0; 1030 1031 vixl::Label loop_head, exit_loop; 1032 __ Bind(&loop_head); 1033 // TODO: When `type == Primitive::kPrimNot`, add a read barrier for 1034 // the reference stored in the object before attempting the CAS, 1035 // similar to the one in the art::Unsafe_compareAndSwapObject JNI 1036 // implementation. 1037 // 1038 // Note that this code is not (yet) used when read barriers are 1039 // enabled (see IntrinsicLocationsBuilderARM64::VisitUnsafeCASObject). 1040 DCHECK(!(type == Primitive::kPrimNot && kEmitCompilerReadBarrier)); 1041 __ Ldaxr(tmp_value, MemOperand(tmp_ptr)); 1042 __ Cmp(tmp_value, expected); 1043 __ B(&exit_loop, ne); 1044 __ Stlxr(tmp_32, value, MemOperand(tmp_ptr)); 1045 __ Cbnz(tmp_32, &loop_head); 1046 __ Bind(&exit_loop); 1047 __ Cset(out, eq); 1048 1049 if (kPoisonHeapReferences && type == Primitive::kPrimNot) { 1050 codegen->GetAssembler()->UnpoisonHeapReference(expected); 1051 if (value.Is(expected)) { 1052 // Do not unpoison `value`, as it is the same register as 1053 // `expected`, which has just been unpoisoned. 1054 } else { 1055 codegen->GetAssembler()->UnpoisonHeapReference(value); 1056 } 1057 } 1058} 1059 1060void IntrinsicLocationsBuilderARM64::VisitUnsafeCASInt(HInvoke* invoke) { 1061 CreateIntIntIntIntIntToInt(arena_, invoke, Primitive::kPrimInt); 1062} 1063void IntrinsicLocationsBuilderARM64::VisitUnsafeCASLong(HInvoke* invoke) { 1064 CreateIntIntIntIntIntToInt(arena_, invoke, Primitive::kPrimLong); 1065} 1066void IntrinsicLocationsBuilderARM64::VisitUnsafeCASObject(HInvoke* invoke) { 1067 // The UnsafeCASObject intrinsic is missing a read barrier, and 1068 // therefore sometimes does not work as expected (b/25883050). 1069 // Turn it off temporarily as a quick fix, until the read barrier is 1070 // implemented (see TODO in GenCAS below). 1071 // 1072 // TODO(rpl): Fix this issue and re-enable this intrinsic with read barriers. 1073 if (kEmitCompilerReadBarrier) { 1074 return; 1075 } 1076 1077 CreateIntIntIntIntIntToInt(arena_, invoke, Primitive::kPrimNot); 1078} 1079 1080void IntrinsicCodeGeneratorARM64::VisitUnsafeCASInt(HInvoke* invoke) { 1081 GenCas(invoke->GetLocations(), Primitive::kPrimInt, codegen_); 1082} 1083void IntrinsicCodeGeneratorARM64::VisitUnsafeCASLong(HInvoke* invoke) { 1084 GenCas(invoke->GetLocations(), Primitive::kPrimLong, codegen_); 1085} 1086void IntrinsicCodeGeneratorARM64::VisitUnsafeCASObject(HInvoke* invoke) { 1087 GenCas(invoke->GetLocations(), Primitive::kPrimNot, codegen_); 1088} 1089 1090void IntrinsicLocationsBuilderARM64::VisitStringCharAt(HInvoke* invoke) { 1091 LocationSummary* locations = new (arena_) LocationSummary(invoke, 1092 LocationSummary::kCallOnSlowPath, 1093 kIntrinsified); 1094 locations->SetInAt(0, Location::RequiresRegister()); 1095 locations->SetInAt(1, Location::RequiresRegister()); 1096 // In case we need to go in the slow path, we can't have the output be the same 1097 // as the input: the current liveness analysis considers the input to be live 1098 // at the point of the call. 1099 locations->SetOut(Location::RequiresRegister(), Location::kOutputOverlap); 1100} 1101 1102void IntrinsicCodeGeneratorARM64::VisitStringCharAt(HInvoke* invoke) { 1103 vixl::MacroAssembler* masm = GetVIXLAssembler(); 1104 LocationSummary* locations = invoke->GetLocations(); 1105 1106 // Location of reference to data array 1107 const MemberOffset value_offset = mirror::String::ValueOffset(); 1108 // Location of count 1109 const MemberOffset count_offset = mirror::String::CountOffset(); 1110 1111 Register obj = WRegisterFrom(locations->InAt(0)); // String object pointer. 1112 Register idx = WRegisterFrom(locations->InAt(1)); // Index of character. 1113 Register out = WRegisterFrom(locations->Out()); // Result character. 1114 1115 UseScratchRegisterScope temps(masm); 1116 Register temp = temps.AcquireW(); 1117 Register array_temp = temps.AcquireW(); // We can trade this for worse scheduling. 1118 1119 // TODO: Maybe we can support range check elimination. Overall, though, I think it's not worth 1120 // the cost. 1121 // TODO: For simplicity, the index parameter is requested in a register, so different from Quick 1122 // we will not optimize the code for constants (which would save a register). 1123 1124 SlowPathCodeARM64* slow_path = new (GetAllocator()) IntrinsicSlowPathARM64(invoke); 1125 codegen_->AddSlowPath(slow_path); 1126 1127 __ Ldr(temp, HeapOperand(obj, count_offset)); // temp = str.length. 1128 codegen_->MaybeRecordImplicitNullCheck(invoke); 1129 __ Cmp(idx, temp); 1130 __ B(hs, slow_path->GetEntryLabel()); 1131 1132 __ Add(array_temp, obj, Operand(value_offset.Int32Value())); // array_temp := str.value. 1133 1134 // Load the value. 1135 __ Ldrh(out, MemOperand(array_temp.X(), idx, UXTW, 1)); // out := array_temp[idx]. 1136 1137 __ Bind(slow_path->GetExitLabel()); 1138} 1139 1140void IntrinsicLocationsBuilderARM64::VisitStringCompareTo(HInvoke* invoke) { 1141 LocationSummary* locations = new (arena_) LocationSummary(invoke, 1142 LocationSummary::kCall, 1143 kIntrinsified); 1144 InvokeRuntimeCallingConvention calling_convention; 1145 locations->SetInAt(0, LocationFrom(calling_convention.GetRegisterAt(0))); 1146 locations->SetInAt(1, LocationFrom(calling_convention.GetRegisterAt(1))); 1147 locations->SetOut(calling_convention.GetReturnLocation(Primitive::kPrimInt)); 1148} 1149 1150void IntrinsicCodeGeneratorARM64::VisitStringCompareTo(HInvoke* invoke) { 1151 vixl::MacroAssembler* masm = GetVIXLAssembler(); 1152 LocationSummary* locations = invoke->GetLocations(); 1153 1154 // Note that the null check must have been done earlier. 1155 DCHECK(!invoke->CanDoImplicitNullCheckOn(invoke->InputAt(0))); 1156 1157 Register argument = WRegisterFrom(locations->InAt(1)); 1158 __ Cmp(argument, 0); 1159 SlowPathCodeARM64* slow_path = new (GetAllocator()) IntrinsicSlowPathARM64(invoke); 1160 codegen_->AddSlowPath(slow_path); 1161 __ B(eq, slow_path->GetEntryLabel()); 1162 1163 __ Ldr( 1164 lr, MemOperand(tr, QUICK_ENTRYPOINT_OFFSET(kArm64WordSize, pStringCompareTo).Int32Value())); 1165 __ Blr(lr); 1166 __ Bind(slow_path->GetExitLabel()); 1167} 1168 1169void IntrinsicLocationsBuilderARM64::VisitStringEquals(HInvoke* invoke) { 1170 LocationSummary* locations = new (arena_) LocationSummary(invoke, 1171 LocationSummary::kNoCall, 1172 kIntrinsified); 1173 locations->SetInAt(0, Location::RequiresRegister()); 1174 locations->SetInAt(1, Location::RequiresRegister()); 1175 // Temporary registers to store lengths of strings and for calculations. 1176 locations->AddTemp(Location::RequiresRegister()); 1177 locations->AddTemp(Location::RequiresRegister()); 1178 1179 locations->SetOut(Location::RequiresRegister(), Location::kOutputOverlap); 1180} 1181 1182void IntrinsicCodeGeneratorARM64::VisitStringEquals(HInvoke* invoke) { 1183 vixl::MacroAssembler* masm = GetVIXLAssembler(); 1184 LocationSummary* locations = invoke->GetLocations(); 1185 1186 Register str = WRegisterFrom(locations->InAt(0)); 1187 Register arg = WRegisterFrom(locations->InAt(1)); 1188 Register out = XRegisterFrom(locations->Out()); 1189 1190 UseScratchRegisterScope scratch_scope(masm); 1191 Register temp = scratch_scope.AcquireW(); 1192 Register temp1 = WRegisterFrom(locations->GetTemp(0)); 1193 Register temp2 = WRegisterFrom(locations->GetTemp(1)); 1194 1195 vixl::Label loop; 1196 vixl::Label end; 1197 vixl::Label return_true; 1198 vixl::Label return_false; 1199 1200 // Get offsets of count, value, and class fields within a string object. 1201 const int32_t count_offset = mirror::String::CountOffset().Int32Value(); 1202 const int32_t value_offset = mirror::String::ValueOffset().Int32Value(); 1203 const int32_t class_offset = mirror::Object::ClassOffset().Int32Value(); 1204 1205 // Note that the null check must have been done earlier. 1206 DCHECK(!invoke->CanDoImplicitNullCheckOn(invoke->InputAt(0))); 1207 1208 // Check if input is null, return false if it is. 1209 __ Cbz(arg, &return_false); 1210 1211 // Reference equality check, return true if same reference. 1212 __ Cmp(str, arg); 1213 __ B(&return_true, eq); 1214 1215 // Instanceof check for the argument by comparing class fields. 1216 // All string objects must have the same type since String cannot be subclassed. 1217 // Receiver must be a string object, so its class field is equal to all strings' class fields. 1218 // If the argument is a string object, its class field must be equal to receiver's class field. 1219 __ Ldr(temp, MemOperand(str.X(), class_offset)); 1220 __ Ldr(temp1, MemOperand(arg.X(), class_offset)); 1221 __ Cmp(temp, temp1); 1222 __ B(&return_false, ne); 1223 1224 // Load lengths of this and argument strings. 1225 __ Ldr(temp, MemOperand(str.X(), count_offset)); 1226 __ Ldr(temp1, MemOperand(arg.X(), count_offset)); 1227 // Check if lengths are equal, return false if they're not. 1228 __ Cmp(temp, temp1); 1229 __ B(&return_false, ne); 1230 // Store offset of string value in preparation for comparison loop 1231 __ Mov(temp1, value_offset); 1232 // Return true if both strings are empty. 1233 __ Cbz(temp, &return_true); 1234 1235 // Assertions that must hold in order to compare strings 4 characters at a time. 1236 DCHECK_ALIGNED(value_offset, 8); 1237 static_assert(IsAligned<8>(kObjectAlignment), "String of odd length is not zero padded"); 1238 1239 temp1 = temp1.X(); 1240 temp2 = temp2.X(); 1241 1242 // Loop to compare strings 4 characters at a time starting at the beginning of the string. 1243 // Ok to do this because strings are zero-padded to be 8-byte aligned. 1244 __ Bind(&loop); 1245 __ Ldr(out, MemOperand(str.X(), temp1)); 1246 __ Ldr(temp2, MemOperand(arg.X(), temp1)); 1247 __ Add(temp1, temp1, Operand(sizeof(uint64_t))); 1248 __ Cmp(out, temp2); 1249 __ B(&return_false, ne); 1250 __ Sub(temp, temp, Operand(4), SetFlags); 1251 __ B(&loop, gt); 1252 1253 // Return true and exit the function. 1254 // If loop does not result in returning false, we return true. 1255 __ Bind(&return_true); 1256 __ Mov(out, 1); 1257 __ B(&end); 1258 1259 // Return false and exit the function. 1260 __ Bind(&return_false); 1261 __ Mov(out, 0); 1262 __ Bind(&end); 1263} 1264 1265static void GenerateVisitStringIndexOf(HInvoke* invoke, 1266 vixl::MacroAssembler* masm, 1267 CodeGeneratorARM64* codegen, 1268 ArenaAllocator* allocator, 1269 bool start_at_zero) { 1270 LocationSummary* locations = invoke->GetLocations(); 1271 Register tmp_reg = WRegisterFrom(locations->GetTemp(0)); 1272 1273 // Note that the null check must have been done earlier. 1274 DCHECK(!invoke->CanDoImplicitNullCheckOn(invoke->InputAt(0))); 1275 1276 // Check for code points > 0xFFFF. Either a slow-path check when we don't know statically, 1277 // or directly dispatch if we have a constant. 1278 SlowPathCodeARM64* slow_path = nullptr; 1279 if (invoke->InputAt(1)->IsIntConstant()) { 1280 if (static_cast<uint32_t>(invoke->InputAt(1)->AsIntConstant()->GetValue()) > 0xFFFFU) { 1281 // Always needs the slow-path. We could directly dispatch to it, but this case should be 1282 // rare, so for simplicity just put the full slow-path down and branch unconditionally. 1283 slow_path = new (allocator) IntrinsicSlowPathARM64(invoke); 1284 codegen->AddSlowPath(slow_path); 1285 __ B(slow_path->GetEntryLabel()); 1286 __ Bind(slow_path->GetExitLabel()); 1287 return; 1288 } 1289 } else { 1290 Register char_reg = WRegisterFrom(locations->InAt(1)); 1291 __ Mov(tmp_reg, 0xFFFF); 1292 __ Cmp(char_reg, Operand(tmp_reg)); 1293 slow_path = new (allocator) IntrinsicSlowPathARM64(invoke); 1294 codegen->AddSlowPath(slow_path); 1295 __ B(hi, slow_path->GetEntryLabel()); 1296 } 1297 1298 if (start_at_zero) { 1299 // Start-index = 0. 1300 __ Mov(tmp_reg, 0); 1301 } 1302 1303 __ Ldr(lr, MemOperand(tr, QUICK_ENTRYPOINT_OFFSET(kArm64WordSize, pIndexOf).Int32Value())); 1304 CheckEntrypointTypes<kQuickIndexOf, int32_t, void*, uint32_t, uint32_t>(); 1305 __ Blr(lr); 1306 1307 if (slow_path != nullptr) { 1308 __ Bind(slow_path->GetExitLabel()); 1309 } 1310} 1311 1312void IntrinsicLocationsBuilderARM64::VisitStringIndexOf(HInvoke* invoke) { 1313 LocationSummary* locations = new (arena_) LocationSummary(invoke, 1314 LocationSummary::kCall, 1315 kIntrinsified); 1316 // We have a hand-crafted assembly stub that follows the runtime calling convention. So it's 1317 // best to align the inputs accordingly. 1318 InvokeRuntimeCallingConvention calling_convention; 1319 locations->SetInAt(0, LocationFrom(calling_convention.GetRegisterAt(0))); 1320 locations->SetInAt(1, LocationFrom(calling_convention.GetRegisterAt(1))); 1321 locations->SetOut(calling_convention.GetReturnLocation(Primitive::kPrimInt)); 1322 1323 // Need a temp for slow-path codepoint compare, and need to send start_index=0. 1324 locations->AddTemp(LocationFrom(calling_convention.GetRegisterAt(2))); 1325} 1326 1327void IntrinsicCodeGeneratorARM64::VisitStringIndexOf(HInvoke* invoke) { 1328 GenerateVisitStringIndexOf( 1329 invoke, GetVIXLAssembler(), codegen_, GetAllocator(), /* start_at_zero */ true); 1330} 1331 1332void IntrinsicLocationsBuilderARM64::VisitStringIndexOfAfter(HInvoke* invoke) { 1333 LocationSummary* locations = new (arena_) LocationSummary(invoke, 1334 LocationSummary::kCall, 1335 kIntrinsified); 1336 // We have a hand-crafted assembly stub that follows the runtime calling convention. So it's 1337 // best to align the inputs accordingly. 1338 InvokeRuntimeCallingConvention calling_convention; 1339 locations->SetInAt(0, LocationFrom(calling_convention.GetRegisterAt(0))); 1340 locations->SetInAt(1, LocationFrom(calling_convention.GetRegisterAt(1))); 1341 locations->SetInAt(2, LocationFrom(calling_convention.GetRegisterAt(2))); 1342 locations->SetOut(calling_convention.GetReturnLocation(Primitive::kPrimInt)); 1343 1344 // Need a temp for slow-path codepoint compare. 1345 locations->AddTemp(Location::RequiresRegister()); 1346} 1347 1348void IntrinsicCodeGeneratorARM64::VisitStringIndexOfAfter(HInvoke* invoke) { 1349 GenerateVisitStringIndexOf( 1350 invoke, GetVIXLAssembler(), codegen_, GetAllocator(), /* start_at_zero */ false); 1351} 1352 1353void IntrinsicLocationsBuilderARM64::VisitStringNewStringFromBytes(HInvoke* invoke) { 1354 LocationSummary* locations = new (arena_) LocationSummary(invoke, 1355 LocationSummary::kCall, 1356 kIntrinsified); 1357 InvokeRuntimeCallingConvention calling_convention; 1358 locations->SetInAt(0, LocationFrom(calling_convention.GetRegisterAt(0))); 1359 locations->SetInAt(1, LocationFrom(calling_convention.GetRegisterAt(1))); 1360 locations->SetInAt(2, LocationFrom(calling_convention.GetRegisterAt(2))); 1361 locations->SetInAt(3, LocationFrom(calling_convention.GetRegisterAt(3))); 1362 locations->SetOut(calling_convention.GetReturnLocation(Primitive::kPrimNot)); 1363} 1364 1365void IntrinsicCodeGeneratorARM64::VisitStringNewStringFromBytes(HInvoke* invoke) { 1366 vixl::MacroAssembler* masm = GetVIXLAssembler(); 1367 LocationSummary* locations = invoke->GetLocations(); 1368 1369 Register byte_array = WRegisterFrom(locations->InAt(0)); 1370 __ Cmp(byte_array, 0); 1371 SlowPathCodeARM64* slow_path = new (GetAllocator()) IntrinsicSlowPathARM64(invoke); 1372 codegen_->AddSlowPath(slow_path); 1373 __ B(eq, slow_path->GetEntryLabel()); 1374 1375 __ Ldr(lr, 1376 MemOperand(tr, QUICK_ENTRYPOINT_OFFSET(kArm64WordSize, pAllocStringFromBytes).Int32Value())); 1377 codegen_->RecordPcInfo(invoke, invoke->GetDexPc()); 1378 __ Blr(lr); 1379 __ Bind(slow_path->GetExitLabel()); 1380} 1381 1382void IntrinsicLocationsBuilderARM64::VisitStringNewStringFromChars(HInvoke* invoke) { 1383 LocationSummary* locations = new (arena_) LocationSummary(invoke, 1384 LocationSummary::kCall, 1385 kIntrinsified); 1386 InvokeRuntimeCallingConvention calling_convention; 1387 locations->SetInAt(0, LocationFrom(calling_convention.GetRegisterAt(0))); 1388 locations->SetInAt(1, LocationFrom(calling_convention.GetRegisterAt(1))); 1389 locations->SetInAt(2, LocationFrom(calling_convention.GetRegisterAt(2))); 1390 locations->SetOut(calling_convention.GetReturnLocation(Primitive::kPrimNot)); 1391} 1392 1393void IntrinsicCodeGeneratorARM64::VisitStringNewStringFromChars(HInvoke* invoke) { 1394 vixl::MacroAssembler* masm = GetVIXLAssembler(); 1395 1396 // No need to emit code checking whether `locations->InAt(2)` is a null 1397 // pointer, as callers of the native method 1398 // 1399 // java.lang.StringFactory.newStringFromChars(int offset, int charCount, char[] data) 1400 // 1401 // all include a null check on `data` before calling that method. 1402 __ Ldr(lr, 1403 MemOperand(tr, QUICK_ENTRYPOINT_OFFSET(kArm64WordSize, pAllocStringFromChars).Int32Value())); 1404 codegen_->RecordPcInfo(invoke, invoke->GetDexPc()); 1405 __ Blr(lr); 1406} 1407 1408void IntrinsicLocationsBuilderARM64::VisitStringNewStringFromString(HInvoke* invoke) { 1409 // The inputs plus one temp. 1410 LocationSummary* locations = new (arena_) LocationSummary(invoke, 1411 LocationSummary::kCall, 1412 kIntrinsified); 1413 InvokeRuntimeCallingConvention calling_convention; 1414 locations->SetInAt(0, LocationFrom(calling_convention.GetRegisterAt(0))); 1415 locations->SetInAt(1, LocationFrom(calling_convention.GetRegisterAt(1))); 1416 locations->SetInAt(2, LocationFrom(calling_convention.GetRegisterAt(2))); 1417 locations->SetOut(calling_convention.GetReturnLocation(Primitive::kPrimNot)); 1418} 1419 1420void IntrinsicCodeGeneratorARM64::VisitStringNewStringFromString(HInvoke* invoke) { 1421 vixl::MacroAssembler* masm = GetVIXLAssembler(); 1422 LocationSummary* locations = invoke->GetLocations(); 1423 1424 Register string_to_copy = WRegisterFrom(locations->InAt(0)); 1425 __ Cmp(string_to_copy, 0); 1426 SlowPathCodeARM64* slow_path = new (GetAllocator()) IntrinsicSlowPathARM64(invoke); 1427 codegen_->AddSlowPath(slow_path); 1428 __ B(eq, slow_path->GetEntryLabel()); 1429 1430 __ Ldr(lr, 1431 MemOperand(tr, QUICK_ENTRYPOINT_OFFSET(kArm64WordSize, pAllocStringFromString).Int32Value())); 1432 codegen_->RecordPcInfo(invoke, invoke->GetDexPc()); 1433 __ Blr(lr); 1434 __ Bind(slow_path->GetExitLabel()); 1435} 1436 1437static void CreateFPToFPCallLocations(ArenaAllocator* arena, HInvoke* invoke) { 1438 DCHECK_EQ(invoke->GetNumberOfArguments(), 1U); 1439 DCHECK(Primitive::IsFloatingPointType(invoke->InputAt(0)->GetType())); 1440 DCHECK(Primitive::IsFloatingPointType(invoke->GetType())); 1441 1442 LocationSummary* const locations = new (arena) LocationSummary(invoke, 1443 LocationSummary::kCall, 1444 kIntrinsified); 1445 InvokeRuntimeCallingConvention calling_convention; 1446 1447 locations->SetInAt(0, LocationFrom(calling_convention.GetFpuRegisterAt(0))); 1448 locations->SetOut(calling_convention.GetReturnLocation(invoke->GetType())); 1449} 1450 1451static void CreateFPFPToFPCallLocations(ArenaAllocator* arena, HInvoke* invoke) { 1452 DCHECK_EQ(invoke->GetNumberOfArguments(), 2U); 1453 DCHECK(Primitive::IsFloatingPointType(invoke->InputAt(0)->GetType())); 1454 DCHECK(Primitive::IsFloatingPointType(invoke->InputAt(1)->GetType())); 1455 DCHECK(Primitive::IsFloatingPointType(invoke->GetType())); 1456 1457 LocationSummary* const locations = new (arena) LocationSummary(invoke, 1458 LocationSummary::kCall, 1459 kIntrinsified); 1460 InvokeRuntimeCallingConvention calling_convention; 1461 1462 locations->SetInAt(0, LocationFrom(calling_convention.GetFpuRegisterAt(0))); 1463 locations->SetInAt(1, LocationFrom(calling_convention.GetFpuRegisterAt(1))); 1464 locations->SetOut(calling_convention.GetReturnLocation(invoke->GetType())); 1465} 1466 1467static void GenFPToFPCall(HInvoke* invoke, 1468 vixl::MacroAssembler* masm, 1469 CodeGeneratorARM64* codegen, 1470 QuickEntrypointEnum entry) { 1471 __ Ldr(lr, MemOperand(tr, GetThreadOffset<kArm64WordSize>(entry).Int32Value())); 1472 __ Blr(lr); 1473 codegen->RecordPcInfo(invoke, invoke->GetDexPc()); 1474} 1475 1476void IntrinsicLocationsBuilderARM64::VisitMathCos(HInvoke* invoke) { 1477 CreateFPToFPCallLocations(arena_, invoke); 1478} 1479 1480void IntrinsicCodeGeneratorARM64::VisitMathCos(HInvoke* invoke) { 1481 GenFPToFPCall(invoke, GetVIXLAssembler(), codegen_, kQuickCos); 1482} 1483 1484void IntrinsicLocationsBuilderARM64::VisitMathSin(HInvoke* invoke) { 1485 CreateFPToFPCallLocations(arena_, invoke); 1486} 1487 1488void IntrinsicCodeGeneratorARM64::VisitMathSin(HInvoke* invoke) { 1489 GenFPToFPCall(invoke, GetVIXLAssembler(), codegen_, kQuickSin); 1490} 1491 1492void IntrinsicLocationsBuilderARM64::VisitMathAcos(HInvoke* invoke) { 1493 CreateFPToFPCallLocations(arena_, invoke); 1494} 1495 1496void IntrinsicCodeGeneratorARM64::VisitMathAcos(HInvoke* invoke) { 1497 GenFPToFPCall(invoke, GetVIXLAssembler(), codegen_, kQuickAcos); 1498} 1499 1500void IntrinsicLocationsBuilderARM64::VisitMathAsin(HInvoke* invoke) { 1501 CreateFPToFPCallLocations(arena_, invoke); 1502} 1503 1504void IntrinsicCodeGeneratorARM64::VisitMathAsin(HInvoke* invoke) { 1505 GenFPToFPCall(invoke, GetVIXLAssembler(), codegen_, kQuickAsin); 1506} 1507 1508void IntrinsicLocationsBuilderARM64::VisitMathAtan(HInvoke* invoke) { 1509 CreateFPToFPCallLocations(arena_, invoke); 1510} 1511 1512void IntrinsicCodeGeneratorARM64::VisitMathAtan(HInvoke* invoke) { 1513 GenFPToFPCall(invoke, GetVIXLAssembler(), codegen_, kQuickAtan); 1514} 1515 1516void IntrinsicLocationsBuilderARM64::VisitMathCbrt(HInvoke* invoke) { 1517 CreateFPToFPCallLocations(arena_, invoke); 1518} 1519 1520void IntrinsicCodeGeneratorARM64::VisitMathCbrt(HInvoke* invoke) { 1521 GenFPToFPCall(invoke, GetVIXLAssembler(), codegen_, kQuickCbrt); 1522} 1523 1524void IntrinsicLocationsBuilderARM64::VisitMathCosh(HInvoke* invoke) { 1525 CreateFPToFPCallLocations(arena_, invoke); 1526} 1527 1528void IntrinsicCodeGeneratorARM64::VisitMathCosh(HInvoke* invoke) { 1529 GenFPToFPCall(invoke, GetVIXLAssembler(), codegen_, kQuickCosh); 1530} 1531 1532void IntrinsicLocationsBuilderARM64::VisitMathExp(HInvoke* invoke) { 1533 CreateFPToFPCallLocations(arena_, invoke); 1534} 1535 1536void IntrinsicCodeGeneratorARM64::VisitMathExp(HInvoke* invoke) { 1537 GenFPToFPCall(invoke, GetVIXLAssembler(), codegen_, kQuickExp); 1538} 1539 1540void IntrinsicLocationsBuilderARM64::VisitMathExpm1(HInvoke* invoke) { 1541 CreateFPToFPCallLocations(arena_, invoke); 1542} 1543 1544void IntrinsicCodeGeneratorARM64::VisitMathExpm1(HInvoke* invoke) { 1545 GenFPToFPCall(invoke, GetVIXLAssembler(), codegen_, kQuickExpm1); 1546} 1547 1548void IntrinsicLocationsBuilderARM64::VisitMathLog(HInvoke* invoke) { 1549 CreateFPToFPCallLocations(arena_, invoke); 1550} 1551 1552void IntrinsicCodeGeneratorARM64::VisitMathLog(HInvoke* invoke) { 1553 GenFPToFPCall(invoke, GetVIXLAssembler(), codegen_, kQuickLog); 1554} 1555 1556void IntrinsicLocationsBuilderARM64::VisitMathLog10(HInvoke* invoke) { 1557 CreateFPToFPCallLocations(arena_, invoke); 1558} 1559 1560void IntrinsicCodeGeneratorARM64::VisitMathLog10(HInvoke* invoke) { 1561 GenFPToFPCall(invoke, GetVIXLAssembler(), codegen_, kQuickLog10); 1562} 1563 1564void IntrinsicLocationsBuilderARM64::VisitMathSinh(HInvoke* invoke) { 1565 CreateFPToFPCallLocations(arena_, invoke); 1566} 1567 1568void IntrinsicCodeGeneratorARM64::VisitMathSinh(HInvoke* invoke) { 1569 GenFPToFPCall(invoke, GetVIXLAssembler(), codegen_, kQuickSinh); 1570} 1571 1572void IntrinsicLocationsBuilderARM64::VisitMathTan(HInvoke* invoke) { 1573 CreateFPToFPCallLocations(arena_, invoke); 1574} 1575 1576void IntrinsicCodeGeneratorARM64::VisitMathTan(HInvoke* invoke) { 1577 GenFPToFPCall(invoke, GetVIXLAssembler(), codegen_, kQuickTan); 1578} 1579 1580void IntrinsicLocationsBuilderARM64::VisitMathTanh(HInvoke* invoke) { 1581 CreateFPToFPCallLocations(arena_, invoke); 1582} 1583 1584void IntrinsicCodeGeneratorARM64::VisitMathTanh(HInvoke* invoke) { 1585 GenFPToFPCall(invoke, GetVIXLAssembler(), codegen_, kQuickTanh); 1586} 1587 1588void IntrinsicLocationsBuilderARM64::VisitMathAtan2(HInvoke* invoke) { 1589 CreateFPFPToFPCallLocations(arena_, invoke); 1590} 1591 1592void IntrinsicCodeGeneratorARM64::VisitMathAtan2(HInvoke* invoke) { 1593 GenFPToFPCall(invoke, GetVIXLAssembler(), codegen_, kQuickAtan2); 1594} 1595 1596void IntrinsicLocationsBuilderARM64::VisitMathHypot(HInvoke* invoke) { 1597 CreateFPFPToFPCallLocations(arena_, invoke); 1598} 1599 1600void IntrinsicCodeGeneratorARM64::VisitMathHypot(HInvoke* invoke) { 1601 GenFPToFPCall(invoke, GetVIXLAssembler(), codegen_, kQuickHypot); 1602} 1603 1604void IntrinsicLocationsBuilderARM64::VisitMathNextAfter(HInvoke* invoke) { 1605 CreateFPFPToFPCallLocations(arena_, invoke); 1606} 1607 1608void IntrinsicCodeGeneratorARM64::VisitMathNextAfter(HInvoke* invoke) { 1609 GenFPToFPCall(invoke, GetVIXLAssembler(), codegen_, kQuickNextAfter); 1610} 1611 1612void IntrinsicLocationsBuilderARM64::VisitStringGetCharsNoCheck(HInvoke* invoke) { 1613 LocationSummary* locations = new (arena_) LocationSummary(invoke, 1614 LocationSummary::kNoCall, 1615 kIntrinsified); 1616 locations->SetInAt(0, Location::RequiresRegister()); 1617 locations->SetInAt(1, Location::RequiresRegister()); 1618 locations->SetInAt(2, Location::RequiresRegister()); 1619 locations->SetInAt(3, Location::RequiresRegister()); 1620 locations->SetInAt(4, Location::RequiresRegister()); 1621 1622 locations->AddTemp(Location::RequiresRegister()); 1623 locations->AddTemp(Location::RequiresRegister()); 1624} 1625 1626void IntrinsicCodeGeneratorARM64::VisitStringGetCharsNoCheck(HInvoke* invoke) { 1627 vixl::MacroAssembler* masm = GetVIXLAssembler(); 1628 LocationSummary* locations = invoke->GetLocations(); 1629 1630 // Check assumption that sizeof(Char) is 2 (used in scaling below). 1631 const size_t char_size = Primitive::ComponentSize(Primitive::kPrimChar); 1632 DCHECK_EQ(char_size, 2u); 1633 1634 // Location of data in char array buffer. 1635 const uint32_t data_offset = mirror::Array::DataOffset(char_size).Uint32Value(); 1636 1637 // Location of char array data in string. 1638 const uint32_t value_offset = mirror::String::ValueOffset().Uint32Value(); 1639 1640 // void getCharsNoCheck(int srcBegin, int srcEnd, char[] dst, int dstBegin); 1641 // Since getChars() calls getCharsNoCheck() - we use registers rather than constants. 1642 Register srcObj = XRegisterFrom(locations->InAt(0)); 1643 Register srcBegin = XRegisterFrom(locations->InAt(1)); 1644 Register srcEnd = XRegisterFrom(locations->InAt(2)); 1645 Register dstObj = XRegisterFrom(locations->InAt(3)); 1646 Register dstBegin = XRegisterFrom(locations->InAt(4)); 1647 1648 Register src_ptr = XRegisterFrom(locations->GetTemp(0)); 1649 Register src_ptr_end = XRegisterFrom(locations->GetTemp(1)); 1650 1651 UseScratchRegisterScope temps(masm); 1652 Register dst_ptr = temps.AcquireX(); 1653 Register tmp = temps.AcquireW(); 1654 1655 // src range to copy. 1656 __ Add(src_ptr, srcObj, Operand(value_offset)); 1657 __ Add(src_ptr_end, src_ptr, Operand(srcEnd, LSL, 1)); 1658 __ Add(src_ptr, src_ptr, Operand(srcBegin, LSL, 1)); 1659 1660 // dst to be copied. 1661 __ Add(dst_ptr, dstObj, Operand(data_offset)); 1662 __ Add(dst_ptr, dst_ptr, Operand(dstBegin, LSL, 1)); 1663 1664 // Do the copy. 1665 vixl::Label loop, done; 1666 __ Bind(&loop); 1667 __ Cmp(src_ptr, src_ptr_end); 1668 __ B(&done, eq); 1669 __ Ldrh(tmp, MemOperand(src_ptr, char_size, vixl::PostIndex)); 1670 __ Strh(tmp, MemOperand(dst_ptr, char_size, vixl::PostIndex)); 1671 __ B(&loop); 1672 __ Bind(&done); 1673} 1674 1675UNIMPLEMENTED_INTRINSIC(ARM64, IntegerBitCount) 1676UNIMPLEMENTED_INTRINSIC(ARM64, LongBitCount) 1677UNIMPLEMENTED_INTRINSIC(ARM64, SystemArrayCopyChar) 1678UNIMPLEMENTED_INTRINSIC(ARM64, SystemArrayCopy) 1679UNIMPLEMENTED_INTRINSIC(ARM64, ReferenceGetReferent) 1680UNIMPLEMENTED_INTRINSIC(ARM64, FloatIsInfinite) 1681UNIMPLEMENTED_INTRINSIC(ARM64, DoubleIsInfinite) 1682UNIMPLEMENTED_INTRINSIC(ARM64, IntegerHighestOneBit) 1683UNIMPLEMENTED_INTRINSIC(ARM64, LongHighestOneBit) 1684UNIMPLEMENTED_INTRINSIC(ARM64, IntegerLowestOneBit) 1685UNIMPLEMENTED_INTRINSIC(ARM64, LongLowestOneBit) 1686 1687UNREACHABLE_INTRINSICS(ARM64) 1688 1689#undef __ 1690 1691} // namespace arm64 1692} // namespace art 1693