intrinsics_arm64.cc revision 095b1df3d20e806ed7ad8c545b03866c1561d1f6
1/* 2 * Copyright (C) 2015 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17#include "intrinsics_arm64.h" 18 19#include "arch/arm64/instruction_set_features_arm64.h" 20#include "art_method.h" 21#include "code_generator_arm64.h" 22#include "common_arm64.h" 23#include "entrypoints/quick/quick_entrypoints.h" 24#include "intrinsics.h" 25#include "mirror/array-inl.h" 26#include "mirror/string.h" 27#include "thread.h" 28#include "utils/arm64/assembler_arm64.h" 29#include "utils/arm64/constants_arm64.h" 30 31#include "vixl/a64/disasm-a64.h" 32#include "vixl/a64/macro-assembler-a64.h" 33 34using namespace vixl; // NOLINT(build/namespaces) 35 36namespace art { 37 38namespace arm64 { 39 40using helpers::DRegisterFrom; 41using helpers::FPRegisterFrom; 42using helpers::HeapOperand; 43using helpers::LocationFrom; 44using helpers::OperandFrom; 45using helpers::RegisterFrom; 46using helpers::SRegisterFrom; 47using helpers::WRegisterFrom; 48using helpers::XRegisterFrom; 49 50namespace { 51 52ALWAYS_INLINE inline MemOperand AbsoluteHeapOperandFrom(Location location, size_t offset = 0) { 53 return MemOperand(XRegisterFrom(location), offset); 54} 55 56} // namespace 57 58vixl::MacroAssembler* IntrinsicCodeGeneratorARM64::GetVIXLAssembler() { 59 return codegen_->GetAssembler()->vixl_masm_; 60} 61 62ArenaAllocator* IntrinsicCodeGeneratorARM64::GetAllocator() { 63 return codegen_->GetGraph()->GetArena(); 64} 65 66#define __ codegen->GetAssembler()->vixl_masm_-> 67 68static void MoveFromReturnRegister(Location trg, 69 Primitive::Type type, 70 CodeGeneratorARM64* codegen) { 71 if (!trg.IsValid()) { 72 DCHECK(type == Primitive::kPrimVoid); 73 return; 74 } 75 76 DCHECK_NE(type, Primitive::kPrimVoid); 77 78 if (Primitive::IsIntegralType(type) || type == Primitive::kPrimNot) { 79 Register trg_reg = RegisterFrom(trg, type); 80 Register res_reg = RegisterFrom(ARM64ReturnLocation(type), type); 81 __ Mov(trg_reg, res_reg, kDiscardForSameWReg); 82 } else { 83 FPRegister trg_reg = FPRegisterFrom(trg, type); 84 FPRegister res_reg = FPRegisterFrom(ARM64ReturnLocation(type), type); 85 __ Fmov(trg_reg, res_reg); 86 } 87} 88 89static void MoveArguments(HInvoke* invoke, CodeGeneratorARM64* codegen) { 90 InvokeDexCallingConventionVisitorARM64 calling_convention_visitor; 91 IntrinsicVisitor::MoveArguments(invoke, codegen, &calling_convention_visitor); 92} 93 94// Slow-path for fallback (calling the managed code to handle the intrinsic) in an intrinsified 95// call. This will copy the arguments into the positions for a regular call. 96// 97// Note: The actual parameters are required to be in the locations given by the invoke's location 98// summary. If an intrinsic modifies those locations before a slowpath call, they must be 99// restored! 100class IntrinsicSlowPathARM64 : public SlowPathCodeARM64 { 101 public: 102 explicit IntrinsicSlowPathARM64(HInvoke* invoke) : invoke_(invoke) { } 103 104 void EmitNativeCode(CodeGenerator* codegen_in) OVERRIDE { 105 CodeGeneratorARM64* codegen = down_cast<CodeGeneratorARM64*>(codegen_in); 106 __ Bind(GetEntryLabel()); 107 108 SaveLiveRegisters(codegen, invoke_->GetLocations()); 109 110 MoveArguments(invoke_, codegen); 111 112 if (invoke_->IsInvokeStaticOrDirect()) { 113 codegen->GenerateStaticOrDirectCall(invoke_->AsInvokeStaticOrDirect(), 114 LocationFrom(kArtMethodRegister)); 115 } else { 116 codegen->GenerateVirtualCall(invoke_->AsInvokeVirtual(), LocationFrom(kArtMethodRegister)); 117 } 118 codegen->RecordPcInfo(invoke_, invoke_->GetDexPc(), this); 119 120 // Copy the result back to the expected output. 121 Location out = invoke_->GetLocations()->Out(); 122 if (out.IsValid()) { 123 DCHECK(out.IsRegister()); // TODO: Replace this when we support output in memory. 124 DCHECK(!invoke_->GetLocations()->GetLiveRegisters()->ContainsCoreRegister(out.reg())); 125 MoveFromReturnRegister(out, invoke_->GetType(), codegen); 126 } 127 128 RestoreLiveRegisters(codegen, invoke_->GetLocations()); 129 __ B(GetExitLabel()); 130 } 131 132 const char* GetDescription() const OVERRIDE { return "IntrinsicSlowPathARM64"; } 133 134 private: 135 // The instruction where this slow path is happening. 136 HInvoke* const invoke_; 137 138 DISALLOW_COPY_AND_ASSIGN(IntrinsicSlowPathARM64); 139}; 140 141#undef __ 142 143bool IntrinsicLocationsBuilderARM64::TryDispatch(HInvoke* invoke) { 144 Dispatch(invoke); 145 LocationSummary* res = invoke->GetLocations(); 146 if (res == nullptr) { 147 return false; 148 } 149 if (kEmitCompilerReadBarrier && res->CanCall()) { 150 // Generating an intrinsic for this HInvoke may produce an 151 // IntrinsicSlowPathARM64 slow path. Currently this approach 152 // does not work when using read barriers, as the emitted 153 // calling sequence will make use of another slow path 154 // (ReadBarrierForRootSlowPathARM64 for HInvokeStaticOrDirect, 155 // ReadBarrierSlowPathARM64 for HInvokeVirtual). So we bail 156 // out in this case. 157 // 158 // TODO: Find a way to have intrinsics work with read barriers. 159 invoke->SetLocations(nullptr); 160 return false; 161 } 162 return res->Intrinsified(); 163} 164 165#define __ masm-> 166 167static void CreateFPToIntLocations(ArenaAllocator* arena, HInvoke* invoke) { 168 LocationSummary* locations = new (arena) LocationSummary(invoke, 169 LocationSummary::kNoCall, 170 kIntrinsified); 171 locations->SetInAt(0, Location::RequiresFpuRegister()); 172 locations->SetOut(Location::RequiresRegister()); 173} 174 175static void CreateIntToFPLocations(ArenaAllocator* arena, HInvoke* invoke) { 176 LocationSummary* locations = new (arena) LocationSummary(invoke, 177 LocationSummary::kNoCall, 178 kIntrinsified); 179 locations->SetInAt(0, Location::RequiresRegister()); 180 locations->SetOut(Location::RequiresFpuRegister()); 181} 182 183static void MoveFPToInt(LocationSummary* locations, bool is64bit, vixl::MacroAssembler* masm) { 184 Location input = locations->InAt(0); 185 Location output = locations->Out(); 186 __ Fmov(is64bit ? XRegisterFrom(output) : WRegisterFrom(output), 187 is64bit ? DRegisterFrom(input) : SRegisterFrom(input)); 188} 189 190static void MoveIntToFP(LocationSummary* locations, bool is64bit, vixl::MacroAssembler* masm) { 191 Location input = locations->InAt(0); 192 Location output = locations->Out(); 193 __ Fmov(is64bit ? DRegisterFrom(output) : SRegisterFrom(output), 194 is64bit ? XRegisterFrom(input) : WRegisterFrom(input)); 195} 196 197void IntrinsicLocationsBuilderARM64::VisitDoubleDoubleToRawLongBits(HInvoke* invoke) { 198 CreateFPToIntLocations(arena_, invoke); 199} 200void IntrinsicLocationsBuilderARM64::VisitDoubleLongBitsToDouble(HInvoke* invoke) { 201 CreateIntToFPLocations(arena_, invoke); 202} 203 204void IntrinsicCodeGeneratorARM64::VisitDoubleDoubleToRawLongBits(HInvoke* invoke) { 205 MoveFPToInt(invoke->GetLocations(), /* is64bit */ true, GetVIXLAssembler()); 206} 207void IntrinsicCodeGeneratorARM64::VisitDoubleLongBitsToDouble(HInvoke* invoke) { 208 MoveIntToFP(invoke->GetLocations(), /* is64bit */ true, GetVIXLAssembler()); 209} 210 211void IntrinsicLocationsBuilderARM64::VisitFloatFloatToRawIntBits(HInvoke* invoke) { 212 CreateFPToIntLocations(arena_, invoke); 213} 214void IntrinsicLocationsBuilderARM64::VisitFloatIntBitsToFloat(HInvoke* invoke) { 215 CreateIntToFPLocations(arena_, invoke); 216} 217 218void IntrinsicCodeGeneratorARM64::VisitFloatFloatToRawIntBits(HInvoke* invoke) { 219 MoveFPToInt(invoke->GetLocations(), /* is64bit */ false, GetVIXLAssembler()); 220} 221void IntrinsicCodeGeneratorARM64::VisitFloatIntBitsToFloat(HInvoke* invoke) { 222 MoveIntToFP(invoke->GetLocations(), /* is64bit */ false, GetVIXLAssembler()); 223} 224 225static void CreateIntToIntLocations(ArenaAllocator* arena, HInvoke* invoke) { 226 LocationSummary* locations = new (arena) LocationSummary(invoke, 227 LocationSummary::kNoCall, 228 kIntrinsified); 229 locations->SetInAt(0, Location::RequiresRegister()); 230 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 231} 232 233static void GenReverseBytes(LocationSummary* locations, 234 Primitive::Type type, 235 vixl::MacroAssembler* masm) { 236 Location in = locations->InAt(0); 237 Location out = locations->Out(); 238 239 switch (type) { 240 case Primitive::kPrimShort: 241 __ Rev16(WRegisterFrom(out), WRegisterFrom(in)); 242 __ Sxth(WRegisterFrom(out), WRegisterFrom(out)); 243 break; 244 case Primitive::kPrimInt: 245 case Primitive::kPrimLong: 246 __ Rev(RegisterFrom(out, type), RegisterFrom(in, type)); 247 break; 248 default: 249 LOG(FATAL) << "Unexpected size for reverse-bytes: " << type; 250 UNREACHABLE(); 251 } 252} 253 254void IntrinsicLocationsBuilderARM64::VisitIntegerReverseBytes(HInvoke* invoke) { 255 CreateIntToIntLocations(arena_, invoke); 256} 257 258void IntrinsicCodeGeneratorARM64::VisitIntegerReverseBytes(HInvoke* invoke) { 259 GenReverseBytes(invoke->GetLocations(), Primitive::kPrimInt, GetVIXLAssembler()); 260} 261 262void IntrinsicLocationsBuilderARM64::VisitLongReverseBytes(HInvoke* invoke) { 263 CreateIntToIntLocations(arena_, invoke); 264} 265 266void IntrinsicCodeGeneratorARM64::VisitLongReverseBytes(HInvoke* invoke) { 267 GenReverseBytes(invoke->GetLocations(), Primitive::kPrimLong, GetVIXLAssembler()); 268} 269 270void IntrinsicLocationsBuilderARM64::VisitShortReverseBytes(HInvoke* invoke) { 271 CreateIntToIntLocations(arena_, invoke); 272} 273 274void IntrinsicCodeGeneratorARM64::VisitShortReverseBytes(HInvoke* invoke) { 275 GenReverseBytes(invoke->GetLocations(), Primitive::kPrimShort, GetVIXLAssembler()); 276} 277 278static void GenNumberOfLeadingZeros(LocationSummary* locations, 279 Primitive::Type type, 280 vixl::MacroAssembler* masm) { 281 DCHECK(type == Primitive::kPrimInt || type == Primitive::kPrimLong); 282 283 Location in = locations->InAt(0); 284 Location out = locations->Out(); 285 286 __ Clz(RegisterFrom(out, type), RegisterFrom(in, type)); 287} 288 289void IntrinsicLocationsBuilderARM64::VisitIntegerNumberOfLeadingZeros(HInvoke* invoke) { 290 CreateIntToIntLocations(arena_, invoke); 291} 292 293void IntrinsicCodeGeneratorARM64::VisitIntegerNumberOfLeadingZeros(HInvoke* invoke) { 294 GenNumberOfLeadingZeros(invoke->GetLocations(), Primitive::kPrimInt, GetVIXLAssembler()); 295} 296 297void IntrinsicLocationsBuilderARM64::VisitLongNumberOfLeadingZeros(HInvoke* invoke) { 298 CreateIntToIntLocations(arena_, invoke); 299} 300 301void IntrinsicCodeGeneratorARM64::VisitLongNumberOfLeadingZeros(HInvoke* invoke) { 302 GenNumberOfLeadingZeros(invoke->GetLocations(), Primitive::kPrimLong, GetVIXLAssembler()); 303} 304 305static void GenNumberOfTrailingZeros(LocationSummary* locations, 306 Primitive::Type type, 307 vixl::MacroAssembler* masm) { 308 DCHECK(type == Primitive::kPrimInt || type == Primitive::kPrimLong); 309 310 Location in = locations->InAt(0); 311 Location out = locations->Out(); 312 313 __ Rbit(RegisterFrom(out, type), RegisterFrom(in, type)); 314 __ Clz(RegisterFrom(out, type), RegisterFrom(out, type)); 315} 316 317void IntrinsicLocationsBuilderARM64::VisitIntegerNumberOfTrailingZeros(HInvoke* invoke) { 318 CreateIntToIntLocations(arena_, invoke); 319} 320 321void IntrinsicCodeGeneratorARM64::VisitIntegerNumberOfTrailingZeros(HInvoke* invoke) { 322 GenNumberOfTrailingZeros(invoke->GetLocations(), Primitive::kPrimInt, GetVIXLAssembler()); 323} 324 325void IntrinsicLocationsBuilderARM64::VisitLongNumberOfTrailingZeros(HInvoke* invoke) { 326 CreateIntToIntLocations(arena_, invoke); 327} 328 329void IntrinsicCodeGeneratorARM64::VisitLongNumberOfTrailingZeros(HInvoke* invoke) { 330 GenNumberOfTrailingZeros(invoke->GetLocations(), Primitive::kPrimLong, GetVIXLAssembler()); 331} 332 333static void GenReverse(LocationSummary* locations, 334 Primitive::Type type, 335 vixl::MacroAssembler* masm) { 336 DCHECK(type == Primitive::kPrimInt || type == Primitive::kPrimLong); 337 338 Location in = locations->InAt(0); 339 Location out = locations->Out(); 340 341 __ Rbit(RegisterFrom(out, type), RegisterFrom(in, type)); 342} 343 344void IntrinsicLocationsBuilderARM64::VisitIntegerReverse(HInvoke* invoke) { 345 CreateIntToIntLocations(arena_, invoke); 346} 347 348void IntrinsicCodeGeneratorARM64::VisitIntegerReverse(HInvoke* invoke) { 349 GenReverse(invoke->GetLocations(), Primitive::kPrimInt, GetVIXLAssembler()); 350} 351 352void IntrinsicLocationsBuilderARM64::VisitLongReverse(HInvoke* invoke) { 353 CreateIntToIntLocations(arena_, invoke); 354} 355 356void IntrinsicCodeGeneratorARM64::VisitLongReverse(HInvoke* invoke) { 357 GenReverse(invoke->GetLocations(), Primitive::kPrimLong, GetVIXLAssembler()); 358} 359 360static void CreateFPToFPLocations(ArenaAllocator* arena, HInvoke* invoke) { 361 LocationSummary* locations = new (arena) LocationSummary(invoke, 362 LocationSummary::kNoCall, 363 kIntrinsified); 364 locations->SetInAt(0, Location::RequiresFpuRegister()); 365 locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap); 366} 367 368static void MathAbsFP(LocationSummary* locations, bool is64bit, vixl::MacroAssembler* masm) { 369 Location in = locations->InAt(0); 370 Location out = locations->Out(); 371 372 FPRegister in_reg = is64bit ? DRegisterFrom(in) : SRegisterFrom(in); 373 FPRegister out_reg = is64bit ? DRegisterFrom(out) : SRegisterFrom(out); 374 375 __ Fabs(out_reg, in_reg); 376} 377 378void IntrinsicLocationsBuilderARM64::VisitMathAbsDouble(HInvoke* invoke) { 379 CreateFPToFPLocations(arena_, invoke); 380} 381 382void IntrinsicCodeGeneratorARM64::VisitMathAbsDouble(HInvoke* invoke) { 383 MathAbsFP(invoke->GetLocations(), /* is64bit */ true, GetVIXLAssembler()); 384} 385 386void IntrinsicLocationsBuilderARM64::VisitMathAbsFloat(HInvoke* invoke) { 387 CreateFPToFPLocations(arena_, invoke); 388} 389 390void IntrinsicCodeGeneratorARM64::VisitMathAbsFloat(HInvoke* invoke) { 391 MathAbsFP(invoke->GetLocations(), /* is64bit */ false, GetVIXLAssembler()); 392} 393 394static void CreateIntToInt(ArenaAllocator* arena, HInvoke* invoke) { 395 LocationSummary* locations = new (arena) LocationSummary(invoke, 396 LocationSummary::kNoCall, 397 kIntrinsified); 398 locations->SetInAt(0, Location::RequiresRegister()); 399 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 400} 401 402static void GenAbsInteger(LocationSummary* locations, 403 bool is64bit, 404 vixl::MacroAssembler* masm) { 405 Location in = locations->InAt(0); 406 Location output = locations->Out(); 407 408 Register in_reg = is64bit ? XRegisterFrom(in) : WRegisterFrom(in); 409 Register out_reg = is64bit ? XRegisterFrom(output) : WRegisterFrom(output); 410 411 __ Cmp(in_reg, Operand(0)); 412 __ Cneg(out_reg, in_reg, lt); 413} 414 415void IntrinsicLocationsBuilderARM64::VisitMathAbsInt(HInvoke* invoke) { 416 CreateIntToInt(arena_, invoke); 417} 418 419void IntrinsicCodeGeneratorARM64::VisitMathAbsInt(HInvoke* invoke) { 420 GenAbsInteger(invoke->GetLocations(), /* is64bit */ false, GetVIXLAssembler()); 421} 422 423void IntrinsicLocationsBuilderARM64::VisitMathAbsLong(HInvoke* invoke) { 424 CreateIntToInt(arena_, invoke); 425} 426 427void IntrinsicCodeGeneratorARM64::VisitMathAbsLong(HInvoke* invoke) { 428 GenAbsInteger(invoke->GetLocations(), /* is64bit */ true, GetVIXLAssembler()); 429} 430 431static void GenMinMaxFP(LocationSummary* locations, 432 bool is_min, 433 bool is_double, 434 vixl::MacroAssembler* masm) { 435 Location op1 = locations->InAt(0); 436 Location op2 = locations->InAt(1); 437 Location out = locations->Out(); 438 439 FPRegister op1_reg = is_double ? DRegisterFrom(op1) : SRegisterFrom(op1); 440 FPRegister op2_reg = is_double ? DRegisterFrom(op2) : SRegisterFrom(op2); 441 FPRegister out_reg = is_double ? DRegisterFrom(out) : SRegisterFrom(out); 442 if (is_min) { 443 __ Fmin(out_reg, op1_reg, op2_reg); 444 } else { 445 __ Fmax(out_reg, op1_reg, op2_reg); 446 } 447} 448 449static void CreateFPFPToFPLocations(ArenaAllocator* arena, HInvoke* invoke) { 450 LocationSummary* locations = new (arena) LocationSummary(invoke, 451 LocationSummary::kNoCall, 452 kIntrinsified); 453 locations->SetInAt(0, Location::RequiresFpuRegister()); 454 locations->SetInAt(1, Location::RequiresFpuRegister()); 455 locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap); 456} 457 458void IntrinsicLocationsBuilderARM64::VisitMathMinDoubleDouble(HInvoke* invoke) { 459 CreateFPFPToFPLocations(arena_, invoke); 460} 461 462void IntrinsicCodeGeneratorARM64::VisitMathMinDoubleDouble(HInvoke* invoke) { 463 GenMinMaxFP(invoke->GetLocations(), /* is_min */ true, /* is_double */ true, GetVIXLAssembler()); 464} 465 466void IntrinsicLocationsBuilderARM64::VisitMathMinFloatFloat(HInvoke* invoke) { 467 CreateFPFPToFPLocations(arena_, invoke); 468} 469 470void IntrinsicCodeGeneratorARM64::VisitMathMinFloatFloat(HInvoke* invoke) { 471 GenMinMaxFP(invoke->GetLocations(), /* is_min */ true, /* is_double */ false, GetVIXLAssembler()); 472} 473 474void IntrinsicLocationsBuilderARM64::VisitMathMaxDoubleDouble(HInvoke* invoke) { 475 CreateFPFPToFPLocations(arena_, invoke); 476} 477 478void IntrinsicCodeGeneratorARM64::VisitMathMaxDoubleDouble(HInvoke* invoke) { 479 GenMinMaxFP(invoke->GetLocations(), /* is_min */ false, /* is_double */ true, GetVIXLAssembler()); 480} 481 482void IntrinsicLocationsBuilderARM64::VisitMathMaxFloatFloat(HInvoke* invoke) { 483 CreateFPFPToFPLocations(arena_, invoke); 484} 485 486void IntrinsicCodeGeneratorARM64::VisitMathMaxFloatFloat(HInvoke* invoke) { 487 GenMinMaxFP( 488 invoke->GetLocations(), /* is_min */ false, /* is_double */ false, GetVIXLAssembler()); 489} 490 491static void GenMinMax(LocationSummary* locations, 492 bool is_min, 493 bool is_long, 494 vixl::MacroAssembler* masm) { 495 Location op1 = locations->InAt(0); 496 Location op2 = locations->InAt(1); 497 Location out = locations->Out(); 498 499 Register op1_reg = is_long ? XRegisterFrom(op1) : WRegisterFrom(op1); 500 Register op2_reg = is_long ? XRegisterFrom(op2) : WRegisterFrom(op2); 501 Register out_reg = is_long ? XRegisterFrom(out) : WRegisterFrom(out); 502 503 __ Cmp(op1_reg, op2_reg); 504 __ Csel(out_reg, op1_reg, op2_reg, is_min ? lt : gt); 505} 506 507static void CreateIntIntToIntLocations(ArenaAllocator* arena, HInvoke* invoke) { 508 LocationSummary* locations = new (arena) LocationSummary(invoke, 509 LocationSummary::kNoCall, 510 kIntrinsified); 511 locations->SetInAt(0, Location::RequiresRegister()); 512 locations->SetInAt(1, Location::RequiresRegister()); 513 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 514} 515 516void IntrinsicLocationsBuilderARM64::VisitMathMinIntInt(HInvoke* invoke) { 517 CreateIntIntToIntLocations(arena_, invoke); 518} 519 520void IntrinsicCodeGeneratorARM64::VisitMathMinIntInt(HInvoke* invoke) { 521 GenMinMax(invoke->GetLocations(), /* is_min */ true, /* is_long */ false, GetVIXLAssembler()); 522} 523 524void IntrinsicLocationsBuilderARM64::VisitMathMinLongLong(HInvoke* invoke) { 525 CreateIntIntToIntLocations(arena_, invoke); 526} 527 528void IntrinsicCodeGeneratorARM64::VisitMathMinLongLong(HInvoke* invoke) { 529 GenMinMax(invoke->GetLocations(), /* is_min */ true, /* is_long */ true, GetVIXLAssembler()); 530} 531 532void IntrinsicLocationsBuilderARM64::VisitMathMaxIntInt(HInvoke* invoke) { 533 CreateIntIntToIntLocations(arena_, invoke); 534} 535 536void IntrinsicCodeGeneratorARM64::VisitMathMaxIntInt(HInvoke* invoke) { 537 GenMinMax(invoke->GetLocations(), /* is_min */ false, /* is_long */ false, GetVIXLAssembler()); 538} 539 540void IntrinsicLocationsBuilderARM64::VisitMathMaxLongLong(HInvoke* invoke) { 541 CreateIntIntToIntLocations(arena_, invoke); 542} 543 544void IntrinsicCodeGeneratorARM64::VisitMathMaxLongLong(HInvoke* invoke) { 545 GenMinMax(invoke->GetLocations(), /* is_min */ false, /* is_long */ true, GetVIXLAssembler()); 546} 547 548void IntrinsicLocationsBuilderARM64::VisitMathSqrt(HInvoke* invoke) { 549 CreateFPToFPLocations(arena_, invoke); 550} 551 552void IntrinsicCodeGeneratorARM64::VisitMathSqrt(HInvoke* invoke) { 553 LocationSummary* locations = invoke->GetLocations(); 554 vixl::MacroAssembler* masm = GetVIXLAssembler(); 555 __ Fsqrt(DRegisterFrom(locations->Out()), DRegisterFrom(locations->InAt(0))); 556} 557 558void IntrinsicLocationsBuilderARM64::VisitMathCeil(HInvoke* invoke) { 559 CreateFPToFPLocations(arena_, invoke); 560} 561 562void IntrinsicCodeGeneratorARM64::VisitMathCeil(HInvoke* invoke) { 563 LocationSummary* locations = invoke->GetLocations(); 564 vixl::MacroAssembler* masm = GetVIXLAssembler(); 565 __ Frintp(DRegisterFrom(locations->Out()), DRegisterFrom(locations->InAt(0))); 566} 567 568void IntrinsicLocationsBuilderARM64::VisitMathFloor(HInvoke* invoke) { 569 CreateFPToFPLocations(arena_, invoke); 570} 571 572void IntrinsicCodeGeneratorARM64::VisitMathFloor(HInvoke* invoke) { 573 LocationSummary* locations = invoke->GetLocations(); 574 vixl::MacroAssembler* masm = GetVIXLAssembler(); 575 __ Frintm(DRegisterFrom(locations->Out()), DRegisterFrom(locations->InAt(0))); 576} 577 578void IntrinsicLocationsBuilderARM64::VisitMathRint(HInvoke* invoke) { 579 CreateFPToFPLocations(arena_, invoke); 580} 581 582void IntrinsicCodeGeneratorARM64::VisitMathRint(HInvoke* invoke) { 583 LocationSummary* locations = invoke->GetLocations(); 584 vixl::MacroAssembler* masm = GetVIXLAssembler(); 585 __ Frintn(DRegisterFrom(locations->Out()), DRegisterFrom(locations->InAt(0))); 586} 587 588static void CreateFPToIntPlusTempLocations(ArenaAllocator* arena, HInvoke* invoke) { 589 LocationSummary* locations = new (arena) LocationSummary(invoke, 590 LocationSummary::kNoCall, 591 kIntrinsified); 592 locations->SetInAt(0, Location::RequiresFpuRegister()); 593 locations->SetOut(Location::RequiresRegister()); 594} 595 596static void GenMathRound(LocationSummary* locations, 597 bool is_double, 598 vixl::MacroAssembler* masm) { 599 FPRegister in_reg = is_double ? 600 DRegisterFrom(locations->InAt(0)) : SRegisterFrom(locations->InAt(0)); 601 Register out_reg = is_double ? 602 XRegisterFrom(locations->Out()) : WRegisterFrom(locations->Out()); 603 UseScratchRegisterScope temps(masm); 604 FPRegister temp1_reg = temps.AcquireSameSizeAs(in_reg); 605 606 // 0.5 can be encoded as an immediate, so use fmov. 607 if (is_double) { 608 __ Fmov(temp1_reg, static_cast<double>(0.5)); 609 } else { 610 __ Fmov(temp1_reg, static_cast<float>(0.5)); 611 } 612 __ Fadd(temp1_reg, in_reg, temp1_reg); 613 __ Fcvtms(out_reg, temp1_reg); 614} 615 616void IntrinsicLocationsBuilderARM64::VisitMathRoundDouble(HInvoke* invoke) { 617 CreateFPToIntPlusTempLocations(arena_, invoke); 618} 619 620void IntrinsicCodeGeneratorARM64::VisitMathRoundDouble(HInvoke* invoke) { 621 GenMathRound(invoke->GetLocations(), /* is_double */ true, GetVIXLAssembler()); 622} 623 624void IntrinsicLocationsBuilderARM64::VisitMathRoundFloat(HInvoke* invoke) { 625 CreateFPToIntPlusTempLocations(arena_, invoke); 626} 627 628void IntrinsicCodeGeneratorARM64::VisitMathRoundFloat(HInvoke* invoke) { 629 GenMathRound(invoke->GetLocations(), /* is_double */ false, GetVIXLAssembler()); 630} 631 632void IntrinsicLocationsBuilderARM64::VisitMemoryPeekByte(HInvoke* invoke) { 633 CreateIntToIntLocations(arena_, invoke); 634} 635 636void IntrinsicCodeGeneratorARM64::VisitMemoryPeekByte(HInvoke* invoke) { 637 vixl::MacroAssembler* masm = GetVIXLAssembler(); 638 __ Ldrsb(WRegisterFrom(invoke->GetLocations()->Out()), 639 AbsoluteHeapOperandFrom(invoke->GetLocations()->InAt(0), 0)); 640} 641 642void IntrinsicLocationsBuilderARM64::VisitMemoryPeekIntNative(HInvoke* invoke) { 643 CreateIntToIntLocations(arena_, invoke); 644} 645 646void IntrinsicCodeGeneratorARM64::VisitMemoryPeekIntNative(HInvoke* invoke) { 647 vixl::MacroAssembler* masm = GetVIXLAssembler(); 648 __ Ldr(WRegisterFrom(invoke->GetLocations()->Out()), 649 AbsoluteHeapOperandFrom(invoke->GetLocations()->InAt(0), 0)); 650} 651 652void IntrinsicLocationsBuilderARM64::VisitMemoryPeekLongNative(HInvoke* invoke) { 653 CreateIntToIntLocations(arena_, invoke); 654} 655 656void IntrinsicCodeGeneratorARM64::VisitMemoryPeekLongNative(HInvoke* invoke) { 657 vixl::MacroAssembler* masm = GetVIXLAssembler(); 658 __ Ldr(XRegisterFrom(invoke->GetLocations()->Out()), 659 AbsoluteHeapOperandFrom(invoke->GetLocations()->InAt(0), 0)); 660} 661 662void IntrinsicLocationsBuilderARM64::VisitMemoryPeekShortNative(HInvoke* invoke) { 663 CreateIntToIntLocations(arena_, invoke); 664} 665 666void IntrinsicCodeGeneratorARM64::VisitMemoryPeekShortNative(HInvoke* invoke) { 667 vixl::MacroAssembler* masm = GetVIXLAssembler(); 668 __ Ldrsh(WRegisterFrom(invoke->GetLocations()->Out()), 669 AbsoluteHeapOperandFrom(invoke->GetLocations()->InAt(0), 0)); 670} 671 672static void CreateIntIntToVoidLocations(ArenaAllocator* arena, HInvoke* invoke) { 673 LocationSummary* locations = new (arena) LocationSummary(invoke, 674 LocationSummary::kNoCall, 675 kIntrinsified); 676 locations->SetInAt(0, Location::RequiresRegister()); 677 locations->SetInAt(1, Location::RequiresRegister()); 678} 679 680void IntrinsicLocationsBuilderARM64::VisitMemoryPokeByte(HInvoke* invoke) { 681 CreateIntIntToVoidLocations(arena_, invoke); 682} 683 684void IntrinsicCodeGeneratorARM64::VisitMemoryPokeByte(HInvoke* invoke) { 685 vixl::MacroAssembler* masm = GetVIXLAssembler(); 686 __ Strb(WRegisterFrom(invoke->GetLocations()->InAt(1)), 687 AbsoluteHeapOperandFrom(invoke->GetLocations()->InAt(0), 0)); 688} 689 690void IntrinsicLocationsBuilderARM64::VisitMemoryPokeIntNative(HInvoke* invoke) { 691 CreateIntIntToVoidLocations(arena_, invoke); 692} 693 694void IntrinsicCodeGeneratorARM64::VisitMemoryPokeIntNative(HInvoke* invoke) { 695 vixl::MacroAssembler* masm = GetVIXLAssembler(); 696 __ Str(WRegisterFrom(invoke->GetLocations()->InAt(1)), 697 AbsoluteHeapOperandFrom(invoke->GetLocations()->InAt(0), 0)); 698} 699 700void IntrinsicLocationsBuilderARM64::VisitMemoryPokeLongNative(HInvoke* invoke) { 701 CreateIntIntToVoidLocations(arena_, invoke); 702} 703 704void IntrinsicCodeGeneratorARM64::VisitMemoryPokeLongNative(HInvoke* invoke) { 705 vixl::MacroAssembler* masm = GetVIXLAssembler(); 706 __ Str(XRegisterFrom(invoke->GetLocations()->InAt(1)), 707 AbsoluteHeapOperandFrom(invoke->GetLocations()->InAt(0), 0)); 708} 709 710void IntrinsicLocationsBuilderARM64::VisitMemoryPokeShortNative(HInvoke* invoke) { 711 CreateIntIntToVoidLocations(arena_, invoke); 712} 713 714void IntrinsicCodeGeneratorARM64::VisitMemoryPokeShortNative(HInvoke* invoke) { 715 vixl::MacroAssembler* masm = GetVIXLAssembler(); 716 __ Strh(WRegisterFrom(invoke->GetLocations()->InAt(1)), 717 AbsoluteHeapOperandFrom(invoke->GetLocations()->InAt(0), 0)); 718} 719 720void IntrinsicLocationsBuilderARM64::VisitThreadCurrentThread(HInvoke* invoke) { 721 LocationSummary* locations = new (arena_) LocationSummary(invoke, 722 LocationSummary::kNoCall, 723 kIntrinsified); 724 locations->SetOut(Location::RequiresRegister()); 725} 726 727void IntrinsicCodeGeneratorARM64::VisitThreadCurrentThread(HInvoke* invoke) { 728 codegen_->Load(Primitive::kPrimNot, WRegisterFrom(invoke->GetLocations()->Out()), 729 MemOperand(tr, Thread::PeerOffset<8>().Int32Value())); 730} 731 732static void GenUnsafeGet(HInvoke* invoke, 733 Primitive::Type type, 734 bool is_volatile, 735 CodeGeneratorARM64* codegen) { 736 LocationSummary* locations = invoke->GetLocations(); 737 DCHECK((type == Primitive::kPrimInt) || 738 (type == Primitive::kPrimLong) || 739 (type == Primitive::kPrimNot)); 740 vixl::MacroAssembler* masm = codegen->GetAssembler()->vixl_masm_; 741 Location base_loc = locations->InAt(1); 742 Register base = WRegisterFrom(base_loc); // Object pointer. 743 Location offset_loc = locations->InAt(2); 744 Register offset = XRegisterFrom(offset_loc); // Long offset. 745 Location trg_loc = locations->Out(); 746 Register trg = RegisterFrom(trg_loc, type); 747 bool use_acquire_release = codegen->GetInstructionSetFeatures().PreferAcquireRelease(); 748 749 MemOperand mem_op(base.X(), offset); 750 if (is_volatile) { 751 if (use_acquire_release) { 752 codegen->LoadAcquire(invoke, trg, mem_op); 753 } else { 754 codegen->Load(type, trg, mem_op); 755 __ Dmb(InnerShareable, BarrierReads); 756 } 757 } else { 758 codegen->Load(type, trg, mem_op); 759 } 760 761 if (type == Primitive::kPrimNot) { 762 DCHECK(trg.IsW()); 763 codegen->MaybeGenerateReadBarrier(invoke, trg_loc, trg_loc, base_loc, 0U, offset_loc); 764 } 765} 766 767static void CreateIntIntIntToIntLocations(ArenaAllocator* arena, HInvoke* invoke) { 768 bool can_call = kEmitCompilerReadBarrier && 769 (invoke->GetIntrinsic() == Intrinsics::kUnsafeGetObject || 770 invoke->GetIntrinsic() == Intrinsics::kUnsafeGetObjectVolatile); 771 LocationSummary* locations = new (arena) LocationSummary(invoke, 772 can_call ? 773 LocationSummary::kCallOnSlowPath : 774 LocationSummary::kNoCall, 775 kIntrinsified); 776 locations->SetInAt(0, Location::NoLocation()); // Unused receiver. 777 locations->SetInAt(1, Location::RequiresRegister()); 778 locations->SetInAt(2, Location::RequiresRegister()); 779 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 780} 781 782void IntrinsicLocationsBuilderARM64::VisitUnsafeGet(HInvoke* invoke) { 783 CreateIntIntIntToIntLocations(arena_, invoke); 784} 785void IntrinsicLocationsBuilderARM64::VisitUnsafeGetVolatile(HInvoke* invoke) { 786 CreateIntIntIntToIntLocations(arena_, invoke); 787} 788void IntrinsicLocationsBuilderARM64::VisitUnsafeGetLong(HInvoke* invoke) { 789 CreateIntIntIntToIntLocations(arena_, invoke); 790} 791void IntrinsicLocationsBuilderARM64::VisitUnsafeGetLongVolatile(HInvoke* invoke) { 792 CreateIntIntIntToIntLocations(arena_, invoke); 793} 794void IntrinsicLocationsBuilderARM64::VisitUnsafeGetObject(HInvoke* invoke) { 795 CreateIntIntIntToIntLocations(arena_, invoke); 796} 797void IntrinsicLocationsBuilderARM64::VisitUnsafeGetObjectVolatile(HInvoke* invoke) { 798 CreateIntIntIntToIntLocations(arena_, invoke); 799} 800 801void IntrinsicCodeGeneratorARM64::VisitUnsafeGet(HInvoke* invoke) { 802 GenUnsafeGet(invoke, Primitive::kPrimInt, /* is_volatile */ false, codegen_); 803} 804void IntrinsicCodeGeneratorARM64::VisitUnsafeGetVolatile(HInvoke* invoke) { 805 GenUnsafeGet(invoke, Primitive::kPrimInt, /* is_volatile */ true, codegen_); 806} 807void IntrinsicCodeGeneratorARM64::VisitUnsafeGetLong(HInvoke* invoke) { 808 GenUnsafeGet(invoke, Primitive::kPrimLong, /* is_volatile */ false, codegen_); 809} 810void IntrinsicCodeGeneratorARM64::VisitUnsafeGetLongVolatile(HInvoke* invoke) { 811 GenUnsafeGet(invoke, Primitive::kPrimLong, /* is_volatile */ true, codegen_); 812} 813void IntrinsicCodeGeneratorARM64::VisitUnsafeGetObject(HInvoke* invoke) { 814 GenUnsafeGet(invoke, Primitive::kPrimNot, /* is_volatile */ false, codegen_); 815} 816void IntrinsicCodeGeneratorARM64::VisitUnsafeGetObjectVolatile(HInvoke* invoke) { 817 GenUnsafeGet(invoke, Primitive::kPrimNot, /* is_volatile */ true, codegen_); 818} 819 820static void CreateIntIntIntIntToVoid(ArenaAllocator* arena, HInvoke* invoke) { 821 LocationSummary* locations = new (arena) LocationSummary(invoke, 822 LocationSummary::kNoCall, 823 kIntrinsified); 824 locations->SetInAt(0, Location::NoLocation()); // Unused receiver. 825 locations->SetInAt(1, Location::RequiresRegister()); 826 locations->SetInAt(2, Location::RequiresRegister()); 827 locations->SetInAt(3, Location::RequiresRegister()); 828} 829 830void IntrinsicLocationsBuilderARM64::VisitUnsafePut(HInvoke* invoke) { 831 CreateIntIntIntIntToVoid(arena_, invoke); 832} 833void IntrinsicLocationsBuilderARM64::VisitUnsafePutOrdered(HInvoke* invoke) { 834 CreateIntIntIntIntToVoid(arena_, invoke); 835} 836void IntrinsicLocationsBuilderARM64::VisitUnsafePutVolatile(HInvoke* invoke) { 837 CreateIntIntIntIntToVoid(arena_, invoke); 838} 839void IntrinsicLocationsBuilderARM64::VisitUnsafePutObject(HInvoke* invoke) { 840 CreateIntIntIntIntToVoid(arena_, invoke); 841} 842void IntrinsicLocationsBuilderARM64::VisitUnsafePutObjectOrdered(HInvoke* invoke) { 843 CreateIntIntIntIntToVoid(arena_, invoke); 844} 845void IntrinsicLocationsBuilderARM64::VisitUnsafePutObjectVolatile(HInvoke* invoke) { 846 CreateIntIntIntIntToVoid(arena_, invoke); 847} 848void IntrinsicLocationsBuilderARM64::VisitUnsafePutLong(HInvoke* invoke) { 849 CreateIntIntIntIntToVoid(arena_, invoke); 850} 851void IntrinsicLocationsBuilderARM64::VisitUnsafePutLongOrdered(HInvoke* invoke) { 852 CreateIntIntIntIntToVoid(arena_, invoke); 853} 854void IntrinsicLocationsBuilderARM64::VisitUnsafePutLongVolatile(HInvoke* invoke) { 855 CreateIntIntIntIntToVoid(arena_, invoke); 856} 857 858static void GenUnsafePut(LocationSummary* locations, 859 Primitive::Type type, 860 bool is_volatile, 861 bool is_ordered, 862 CodeGeneratorARM64* codegen) { 863 vixl::MacroAssembler* masm = codegen->GetAssembler()->vixl_masm_; 864 865 Register base = WRegisterFrom(locations->InAt(1)); // Object pointer. 866 Register offset = XRegisterFrom(locations->InAt(2)); // Long offset. 867 Register value = RegisterFrom(locations->InAt(3), type); 868 Register source = value; 869 bool use_acquire_release = codegen->GetInstructionSetFeatures().PreferAcquireRelease(); 870 871 MemOperand mem_op(base.X(), offset); 872 873 { 874 // We use a block to end the scratch scope before the write barrier, thus 875 // freeing the temporary registers so they can be used in `MarkGCCard`. 876 UseScratchRegisterScope temps(masm); 877 878 if (kPoisonHeapReferences && type == Primitive::kPrimNot) { 879 DCHECK(value.IsW()); 880 Register temp = temps.AcquireW(); 881 __ Mov(temp.W(), value.W()); 882 codegen->GetAssembler()->PoisonHeapReference(temp.W()); 883 source = temp; 884 } 885 886 if (is_volatile || is_ordered) { 887 if (use_acquire_release) { 888 codegen->StoreRelease(type, source, mem_op); 889 } else { 890 __ Dmb(InnerShareable, BarrierAll); 891 codegen->Store(type, source, mem_op); 892 if (is_volatile) { 893 __ Dmb(InnerShareable, BarrierReads); 894 } 895 } 896 } else { 897 codegen->Store(type, source, mem_op); 898 } 899 } 900 901 if (type == Primitive::kPrimNot) { 902 bool value_can_be_null = true; // TODO: Worth finding out this information? 903 codegen->MarkGCCard(base, value, value_can_be_null); 904 } 905} 906 907void IntrinsicCodeGeneratorARM64::VisitUnsafePut(HInvoke* invoke) { 908 GenUnsafePut(invoke->GetLocations(), 909 Primitive::kPrimInt, 910 /* is_volatile */ false, 911 /* is_ordered */ false, 912 codegen_); 913} 914void IntrinsicCodeGeneratorARM64::VisitUnsafePutOrdered(HInvoke* invoke) { 915 GenUnsafePut(invoke->GetLocations(), 916 Primitive::kPrimInt, 917 /* is_volatile */ false, 918 /* is_ordered */ true, 919 codegen_); 920} 921void IntrinsicCodeGeneratorARM64::VisitUnsafePutVolatile(HInvoke* invoke) { 922 GenUnsafePut(invoke->GetLocations(), 923 Primitive::kPrimInt, 924 /* is_volatile */ true, 925 /* is_ordered */ false, 926 codegen_); 927} 928void IntrinsicCodeGeneratorARM64::VisitUnsafePutObject(HInvoke* invoke) { 929 GenUnsafePut(invoke->GetLocations(), 930 Primitive::kPrimNot, 931 /* is_volatile */ false, 932 /* is_ordered */ false, 933 codegen_); 934} 935void IntrinsicCodeGeneratorARM64::VisitUnsafePutObjectOrdered(HInvoke* invoke) { 936 GenUnsafePut(invoke->GetLocations(), 937 Primitive::kPrimNot, 938 /* is_volatile */ false, 939 /* is_ordered */ true, 940 codegen_); 941} 942void IntrinsicCodeGeneratorARM64::VisitUnsafePutObjectVolatile(HInvoke* invoke) { 943 GenUnsafePut(invoke->GetLocations(), 944 Primitive::kPrimNot, 945 /* is_volatile */ true, 946 /* is_ordered */ false, 947 codegen_); 948} 949void IntrinsicCodeGeneratorARM64::VisitUnsafePutLong(HInvoke* invoke) { 950 GenUnsafePut(invoke->GetLocations(), 951 Primitive::kPrimLong, 952 /* is_volatile */ false, 953 /* is_ordered */ false, 954 codegen_); 955} 956void IntrinsicCodeGeneratorARM64::VisitUnsafePutLongOrdered(HInvoke* invoke) { 957 GenUnsafePut(invoke->GetLocations(), 958 Primitive::kPrimLong, 959 /* is_volatile */ false, 960 /* is_ordered */ true, 961 codegen_); 962} 963void IntrinsicCodeGeneratorARM64::VisitUnsafePutLongVolatile(HInvoke* invoke) { 964 GenUnsafePut(invoke->GetLocations(), 965 Primitive::kPrimLong, 966 /* is_volatile */ true, 967 /* is_ordered */ false, 968 codegen_); 969} 970 971static void CreateIntIntIntIntIntToInt(ArenaAllocator* arena, HInvoke* invoke) { 972 LocationSummary* locations = new (arena) LocationSummary(invoke, 973 LocationSummary::kNoCall, 974 kIntrinsified); 975 locations->SetInAt(0, Location::NoLocation()); // Unused receiver. 976 locations->SetInAt(1, Location::RequiresRegister()); 977 locations->SetInAt(2, Location::RequiresRegister()); 978 locations->SetInAt(3, Location::RequiresRegister()); 979 locations->SetInAt(4, Location::RequiresRegister()); 980 981 locations->SetOut(Location::RequiresRegister(), Location::kNoOutputOverlap); 982} 983 984static void GenCas(LocationSummary* locations, Primitive::Type type, CodeGeneratorARM64* codegen) { 985 bool use_acquire_release = codegen->GetInstructionSetFeatures().PreferAcquireRelease(); 986 vixl::MacroAssembler* masm = codegen->GetAssembler()->vixl_masm_; 987 988 Register out = WRegisterFrom(locations->Out()); // Boolean result. 989 990 Register base = WRegisterFrom(locations->InAt(1)); // Object pointer. 991 Register offset = XRegisterFrom(locations->InAt(2)); // Long offset. 992 Register expected = RegisterFrom(locations->InAt(3), type); // Expected. 993 Register value = RegisterFrom(locations->InAt(4), type); // Value. 994 995 // This needs to be before the temp registers, as MarkGCCard also uses VIXL temps. 996 if (type == Primitive::kPrimNot) { 997 // Mark card for object assuming new value is stored. 998 bool value_can_be_null = true; // TODO: Worth finding out this information? 999 codegen->MarkGCCard(base, value, value_can_be_null); 1000 } 1001 1002 UseScratchRegisterScope temps(masm); 1003 Register tmp_ptr = temps.AcquireX(); // Pointer to actual memory. 1004 Register tmp_value = temps.AcquireSameSizeAs(value); // Value in memory. 1005 1006 Register tmp_32 = tmp_value.W(); 1007 1008 __ Add(tmp_ptr, base.X(), Operand(offset)); 1009 1010 if (kPoisonHeapReferences && type == Primitive::kPrimNot) { 1011 codegen->GetAssembler()->PoisonHeapReference(expected); 1012 codegen->GetAssembler()->PoisonHeapReference(value); 1013 } 1014 1015 // do { 1016 // tmp_value = [tmp_ptr] - expected; 1017 // } while (tmp_value == 0 && failure([tmp_ptr] <- r_new_value)); 1018 // result = tmp_value != 0; 1019 1020 vixl::Label loop_head, exit_loop; 1021 if (use_acquire_release) { 1022 __ Bind(&loop_head); 1023 __ Ldaxr(tmp_value, MemOperand(tmp_ptr)); 1024 // TODO: Do we need a read barrier here when `type == Primitive::kPrimNot`? 1025 // Note that this code is not (yet) used when read barriers are 1026 // enabled (see IntrinsicLocationsBuilderARM64::VisitUnsafeCASObject). 1027 __ Cmp(tmp_value, expected); 1028 __ B(&exit_loop, ne); 1029 __ Stlxr(tmp_32, value, MemOperand(tmp_ptr)); 1030 __ Cbnz(tmp_32, &loop_head); 1031 } else { 1032 __ Dmb(InnerShareable, BarrierWrites); 1033 __ Bind(&loop_head); 1034 // TODO: When `type == Primitive::kPrimNot`, add a read barrier for 1035 // the reference stored in the object before attempting the CAS, 1036 // similar to the one in the art::Unsafe_compareAndSwapObject JNI 1037 // implementation. 1038 // 1039 // Note that this code is not (yet) used when read barriers are 1040 // enabled (see IntrinsicLocationsBuilderARM64::VisitUnsafeCASObject). 1041 DCHECK(!(type == Primitive::kPrimNot && kEmitCompilerReadBarrier)); 1042 __ Ldxr(tmp_value, MemOperand(tmp_ptr)); 1043 __ Cmp(tmp_value, expected); 1044 __ B(&exit_loop, ne); 1045 __ Stxr(tmp_32, value, MemOperand(tmp_ptr)); 1046 __ Cbnz(tmp_32, &loop_head); 1047 __ Dmb(InnerShareable, BarrierAll); 1048 } 1049 __ Bind(&exit_loop); 1050 __ Cset(out, eq); 1051 1052 if (kPoisonHeapReferences && type == Primitive::kPrimNot) { 1053 codegen->GetAssembler()->UnpoisonHeapReference(value); 1054 codegen->GetAssembler()->UnpoisonHeapReference(expected); 1055 } 1056} 1057 1058void IntrinsicLocationsBuilderARM64::VisitUnsafeCASInt(HInvoke* invoke) { 1059 CreateIntIntIntIntIntToInt(arena_, invoke); 1060} 1061void IntrinsicLocationsBuilderARM64::VisitUnsafeCASLong(HInvoke* invoke) { 1062 CreateIntIntIntIntIntToInt(arena_, invoke); 1063} 1064void IntrinsicLocationsBuilderARM64::VisitUnsafeCASObject(HInvoke* invoke) { 1065 // The UnsafeCASObject intrinsic is missing a read barrier, and 1066 // therefore sometimes does not work as expected (b/25883050). 1067 // Turn it off temporarily as a quick fix, until the read barrier is 1068 // implemented (see TODO in GenCAS below). 1069 // 1070 // Also, the UnsafeCASObject intrinsic does not always work when heap 1071 // poisoning is enabled (it breaks run-test 004-UnsafeTest); turn it 1072 // off temporarily as a quick fix (b/26204023). 1073 // 1074 // TODO(rpl): Fix these two issues and re-enable this intrinsic. 1075 if (kEmitCompilerReadBarrier || kPoisonHeapReferences) { 1076 return; 1077 } 1078 1079 CreateIntIntIntIntIntToInt(arena_, invoke); 1080} 1081 1082void IntrinsicCodeGeneratorARM64::VisitUnsafeCASInt(HInvoke* invoke) { 1083 GenCas(invoke->GetLocations(), Primitive::kPrimInt, codegen_); 1084} 1085void IntrinsicCodeGeneratorARM64::VisitUnsafeCASLong(HInvoke* invoke) { 1086 GenCas(invoke->GetLocations(), Primitive::kPrimLong, codegen_); 1087} 1088void IntrinsicCodeGeneratorARM64::VisitUnsafeCASObject(HInvoke* invoke) { 1089 GenCas(invoke->GetLocations(), Primitive::kPrimNot, codegen_); 1090} 1091 1092void IntrinsicLocationsBuilderARM64::VisitStringCharAt(HInvoke* invoke) { 1093 LocationSummary* locations = new (arena_) LocationSummary(invoke, 1094 LocationSummary::kCallOnSlowPath, 1095 kIntrinsified); 1096 locations->SetInAt(0, Location::RequiresRegister()); 1097 locations->SetInAt(1, Location::RequiresRegister()); 1098 // In case we need to go in the slow path, we can't have the output be the same 1099 // as the input: the current liveness analysis considers the input to be live 1100 // at the point of the call. 1101 locations->SetOut(Location::RequiresRegister(), Location::kOutputOverlap); 1102} 1103 1104void IntrinsicCodeGeneratorARM64::VisitStringCharAt(HInvoke* invoke) { 1105 vixl::MacroAssembler* masm = GetVIXLAssembler(); 1106 LocationSummary* locations = invoke->GetLocations(); 1107 1108 // Location of reference to data array 1109 const MemberOffset value_offset = mirror::String::ValueOffset(); 1110 // Location of count 1111 const MemberOffset count_offset = mirror::String::CountOffset(); 1112 1113 Register obj = WRegisterFrom(locations->InAt(0)); // String object pointer. 1114 Register idx = WRegisterFrom(locations->InAt(1)); // Index of character. 1115 Register out = WRegisterFrom(locations->Out()); // Result character. 1116 1117 UseScratchRegisterScope temps(masm); 1118 Register temp = temps.AcquireW(); 1119 Register array_temp = temps.AcquireW(); // We can trade this for worse scheduling. 1120 1121 // TODO: Maybe we can support range check elimination. Overall, though, I think it's not worth 1122 // the cost. 1123 // TODO: For simplicity, the index parameter is requested in a register, so different from Quick 1124 // we will not optimize the code for constants (which would save a register). 1125 1126 SlowPathCodeARM64* slow_path = new (GetAllocator()) IntrinsicSlowPathARM64(invoke); 1127 codegen_->AddSlowPath(slow_path); 1128 1129 __ Ldr(temp, HeapOperand(obj, count_offset)); // temp = str.length. 1130 codegen_->MaybeRecordImplicitNullCheck(invoke); 1131 __ Cmp(idx, temp); 1132 __ B(hs, slow_path->GetEntryLabel()); 1133 1134 __ Add(array_temp, obj, Operand(value_offset.Int32Value())); // array_temp := str.value. 1135 1136 // Load the value. 1137 __ Ldrh(out, MemOperand(array_temp.X(), idx, UXTW, 1)); // out := array_temp[idx]. 1138 1139 __ Bind(slow_path->GetExitLabel()); 1140} 1141 1142void IntrinsicLocationsBuilderARM64::VisitStringCompareTo(HInvoke* invoke) { 1143 LocationSummary* locations = new (arena_) LocationSummary(invoke, 1144 LocationSummary::kCall, 1145 kIntrinsified); 1146 InvokeRuntimeCallingConvention calling_convention; 1147 locations->SetInAt(0, LocationFrom(calling_convention.GetRegisterAt(0))); 1148 locations->SetInAt(1, LocationFrom(calling_convention.GetRegisterAt(1))); 1149 locations->SetOut(calling_convention.GetReturnLocation(Primitive::kPrimInt)); 1150} 1151 1152void IntrinsicCodeGeneratorARM64::VisitStringCompareTo(HInvoke* invoke) { 1153 vixl::MacroAssembler* masm = GetVIXLAssembler(); 1154 LocationSummary* locations = invoke->GetLocations(); 1155 1156 // Note that the null check must have been done earlier. 1157 DCHECK(!invoke->CanDoImplicitNullCheckOn(invoke->InputAt(0))); 1158 1159 Register argument = WRegisterFrom(locations->InAt(1)); 1160 __ Cmp(argument, 0); 1161 SlowPathCodeARM64* slow_path = new (GetAllocator()) IntrinsicSlowPathARM64(invoke); 1162 codegen_->AddSlowPath(slow_path); 1163 __ B(eq, slow_path->GetEntryLabel()); 1164 1165 __ Ldr( 1166 lr, MemOperand(tr, QUICK_ENTRYPOINT_OFFSET(kArm64WordSize, pStringCompareTo).Int32Value())); 1167 __ Blr(lr); 1168 __ Bind(slow_path->GetExitLabel()); 1169} 1170 1171void IntrinsicLocationsBuilderARM64::VisitStringEquals(HInvoke* invoke) { 1172 LocationSummary* locations = new (arena_) LocationSummary(invoke, 1173 LocationSummary::kNoCall, 1174 kIntrinsified); 1175 locations->SetInAt(0, Location::RequiresRegister()); 1176 locations->SetInAt(1, Location::RequiresRegister()); 1177 // Temporary registers to store lengths of strings and for calculations. 1178 locations->AddTemp(Location::RequiresRegister()); 1179 locations->AddTemp(Location::RequiresRegister()); 1180 1181 locations->SetOut(Location::RequiresRegister(), Location::kOutputOverlap); 1182} 1183 1184void IntrinsicCodeGeneratorARM64::VisitStringEquals(HInvoke* invoke) { 1185 vixl::MacroAssembler* masm = GetVIXLAssembler(); 1186 LocationSummary* locations = invoke->GetLocations(); 1187 1188 Register str = WRegisterFrom(locations->InAt(0)); 1189 Register arg = WRegisterFrom(locations->InAt(1)); 1190 Register out = XRegisterFrom(locations->Out()); 1191 1192 UseScratchRegisterScope scratch_scope(masm); 1193 Register temp = scratch_scope.AcquireW(); 1194 Register temp1 = WRegisterFrom(locations->GetTemp(0)); 1195 Register temp2 = WRegisterFrom(locations->GetTemp(1)); 1196 1197 vixl::Label loop; 1198 vixl::Label end; 1199 vixl::Label return_true; 1200 vixl::Label return_false; 1201 1202 // Get offsets of count, value, and class fields within a string object. 1203 const int32_t count_offset = mirror::String::CountOffset().Int32Value(); 1204 const int32_t value_offset = mirror::String::ValueOffset().Int32Value(); 1205 const int32_t class_offset = mirror::Object::ClassOffset().Int32Value(); 1206 1207 // Note that the null check must have been done earlier. 1208 DCHECK(!invoke->CanDoImplicitNullCheckOn(invoke->InputAt(0))); 1209 1210 // Check if input is null, return false if it is. 1211 __ Cbz(arg, &return_false); 1212 1213 // Reference equality check, return true if same reference. 1214 __ Cmp(str, arg); 1215 __ B(&return_true, eq); 1216 1217 // Instanceof check for the argument by comparing class fields. 1218 // All string objects must have the same type since String cannot be subclassed. 1219 // Receiver must be a string object, so its class field is equal to all strings' class fields. 1220 // If the argument is a string object, its class field must be equal to receiver's class field. 1221 __ Ldr(temp, MemOperand(str.X(), class_offset)); 1222 __ Ldr(temp1, MemOperand(arg.X(), class_offset)); 1223 __ Cmp(temp, temp1); 1224 __ B(&return_false, ne); 1225 1226 // Load lengths of this and argument strings. 1227 __ Ldr(temp, MemOperand(str.X(), count_offset)); 1228 __ Ldr(temp1, MemOperand(arg.X(), count_offset)); 1229 // Check if lengths are equal, return false if they're not. 1230 __ Cmp(temp, temp1); 1231 __ B(&return_false, ne); 1232 // Store offset of string value in preparation for comparison loop 1233 __ Mov(temp1, value_offset); 1234 // Return true if both strings are empty. 1235 __ Cbz(temp, &return_true); 1236 1237 // Assertions that must hold in order to compare strings 4 characters at a time. 1238 DCHECK_ALIGNED(value_offset, 8); 1239 static_assert(IsAligned<8>(kObjectAlignment), "String of odd length is not zero padded"); 1240 1241 temp1 = temp1.X(); 1242 temp2 = temp2.X(); 1243 1244 // Loop to compare strings 4 characters at a time starting at the beginning of the string. 1245 // Ok to do this because strings are zero-padded to be 8-byte aligned. 1246 __ Bind(&loop); 1247 __ Ldr(out, MemOperand(str.X(), temp1)); 1248 __ Ldr(temp2, MemOperand(arg.X(), temp1)); 1249 __ Add(temp1, temp1, Operand(sizeof(uint64_t))); 1250 __ Cmp(out, temp2); 1251 __ B(&return_false, ne); 1252 __ Sub(temp, temp, Operand(4), SetFlags); 1253 __ B(&loop, gt); 1254 1255 // Return true and exit the function. 1256 // If loop does not result in returning false, we return true. 1257 __ Bind(&return_true); 1258 __ Mov(out, 1); 1259 __ B(&end); 1260 1261 // Return false and exit the function. 1262 __ Bind(&return_false); 1263 __ Mov(out, 0); 1264 __ Bind(&end); 1265} 1266 1267static void GenerateVisitStringIndexOf(HInvoke* invoke, 1268 vixl::MacroAssembler* masm, 1269 CodeGeneratorARM64* codegen, 1270 ArenaAllocator* allocator, 1271 bool start_at_zero) { 1272 LocationSummary* locations = invoke->GetLocations(); 1273 Register tmp_reg = WRegisterFrom(locations->GetTemp(0)); 1274 1275 // Note that the null check must have been done earlier. 1276 DCHECK(!invoke->CanDoImplicitNullCheckOn(invoke->InputAt(0))); 1277 1278 // Check for code points > 0xFFFF. Either a slow-path check when we don't know statically, 1279 // or directly dispatch if we have a constant. 1280 SlowPathCodeARM64* slow_path = nullptr; 1281 if (invoke->InputAt(1)->IsIntConstant()) { 1282 if (static_cast<uint32_t>(invoke->InputAt(1)->AsIntConstant()->GetValue()) > 0xFFFFU) { 1283 // Always needs the slow-path. We could directly dispatch to it, but this case should be 1284 // rare, so for simplicity just put the full slow-path down and branch unconditionally. 1285 slow_path = new (allocator) IntrinsicSlowPathARM64(invoke); 1286 codegen->AddSlowPath(slow_path); 1287 __ B(slow_path->GetEntryLabel()); 1288 __ Bind(slow_path->GetExitLabel()); 1289 return; 1290 } 1291 } else { 1292 Register char_reg = WRegisterFrom(locations->InAt(1)); 1293 __ Mov(tmp_reg, 0xFFFF); 1294 __ Cmp(char_reg, Operand(tmp_reg)); 1295 slow_path = new (allocator) IntrinsicSlowPathARM64(invoke); 1296 codegen->AddSlowPath(slow_path); 1297 __ B(hi, slow_path->GetEntryLabel()); 1298 } 1299 1300 if (start_at_zero) { 1301 // Start-index = 0. 1302 __ Mov(tmp_reg, 0); 1303 } 1304 1305 __ Ldr(lr, MemOperand(tr, QUICK_ENTRYPOINT_OFFSET(kArm64WordSize, pIndexOf).Int32Value())); 1306 __ Blr(lr); 1307 1308 if (slow_path != nullptr) { 1309 __ Bind(slow_path->GetExitLabel()); 1310 } 1311} 1312 1313void IntrinsicLocationsBuilderARM64::VisitStringIndexOf(HInvoke* invoke) { 1314 LocationSummary* locations = new (arena_) LocationSummary(invoke, 1315 LocationSummary::kCall, 1316 kIntrinsified); 1317 // We have a hand-crafted assembly stub that follows the runtime calling convention. So it's 1318 // best to align the inputs accordingly. 1319 InvokeRuntimeCallingConvention calling_convention; 1320 locations->SetInAt(0, LocationFrom(calling_convention.GetRegisterAt(0))); 1321 locations->SetInAt(1, LocationFrom(calling_convention.GetRegisterAt(1))); 1322 locations->SetOut(calling_convention.GetReturnLocation(Primitive::kPrimInt)); 1323 1324 // Need a temp for slow-path codepoint compare, and need to send start_index=0. 1325 locations->AddTemp(LocationFrom(calling_convention.GetRegisterAt(2))); 1326} 1327 1328void IntrinsicCodeGeneratorARM64::VisitStringIndexOf(HInvoke* invoke) { 1329 GenerateVisitStringIndexOf( 1330 invoke, GetVIXLAssembler(), codegen_, GetAllocator(), /* start_at_zero */ true); 1331} 1332 1333void IntrinsicLocationsBuilderARM64::VisitStringIndexOfAfter(HInvoke* invoke) { 1334 LocationSummary* locations = new (arena_) LocationSummary(invoke, 1335 LocationSummary::kCall, 1336 kIntrinsified); 1337 // We have a hand-crafted assembly stub that follows the runtime calling convention. So it's 1338 // best to align the inputs accordingly. 1339 InvokeRuntimeCallingConvention calling_convention; 1340 locations->SetInAt(0, LocationFrom(calling_convention.GetRegisterAt(0))); 1341 locations->SetInAt(1, LocationFrom(calling_convention.GetRegisterAt(1))); 1342 locations->SetInAt(2, LocationFrom(calling_convention.GetRegisterAt(2))); 1343 locations->SetOut(calling_convention.GetReturnLocation(Primitive::kPrimInt)); 1344 1345 // Need a temp for slow-path codepoint compare. 1346 locations->AddTemp(Location::RequiresRegister()); 1347} 1348 1349void IntrinsicCodeGeneratorARM64::VisitStringIndexOfAfter(HInvoke* invoke) { 1350 GenerateVisitStringIndexOf( 1351 invoke, GetVIXLAssembler(), codegen_, GetAllocator(), /* start_at_zero */ false); 1352} 1353 1354void IntrinsicLocationsBuilderARM64::VisitStringNewStringFromBytes(HInvoke* invoke) { 1355 LocationSummary* locations = new (arena_) LocationSummary(invoke, 1356 LocationSummary::kCall, 1357 kIntrinsified); 1358 InvokeRuntimeCallingConvention calling_convention; 1359 locations->SetInAt(0, LocationFrom(calling_convention.GetRegisterAt(0))); 1360 locations->SetInAt(1, LocationFrom(calling_convention.GetRegisterAt(1))); 1361 locations->SetInAt(2, LocationFrom(calling_convention.GetRegisterAt(2))); 1362 locations->SetInAt(3, LocationFrom(calling_convention.GetRegisterAt(3))); 1363 locations->SetOut(calling_convention.GetReturnLocation(Primitive::kPrimNot)); 1364} 1365 1366void IntrinsicCodeGeneratorARM64::VisitStringNewStringFromBytes(HInvoke* invoke) { 1367 vixl::MacroAssembler* masm = GetVIXLAssembler(); 1368 LocationSummary* locations = invoke->GetLocations(); 1369 1370 Register byte_array = WRegisterFrom(locations->InAt(0)); 1371 __ Cmp(byte_array, 0); 1372 SlowPathCodeARM64* slow_path = new (GetAllocator()) IntrinsicSlowPathARM64(invoke); 1373 codegen_->AddSlowPath(slow_path); 1374 __ B(eq, slow_path->GetEntryLabel()); 1375 1376 __ Ldr(lr, 1377 MemOperand(tr, QUICK_ENTRYPOINT_OFFSET(kArm64WordSize, pAllocStringFromBytes).Int32Value())); 1378 codegen_->RecordPcInfo(invoke, invoke->GetDexPc()); 1379 __ Blr(lr); 1380 __ Bind(slow_path->GetExitLabel()); 1381} 1382 1383void IntrinsicLocationsBuilderARM64::VisitStringNewStringFromChars(HInvoke* invoke) { 1384 LocationSummary* locations = new (arena_) LocationSummary(invoke, 1385 LocationSummary::kCall, 1386 kIntrinsified); 1387 InvokeRuntimeCallingConvention calling_convention; 1388 locations->SetInAt(0, LocationFrom(calling_convention.GetRegisterAt(0))); 1389 locations->SetInAt(1, LocationFrom(calling_convention.GetRegisterAt(1))); 1390 locations->SetInAt(2, LocationFrom(calling_convention.GetRegisterAt(2))); 1391 locations->SetOut(calling_convention.GetReturnLocation(Primitive::kPrimNot)); 1392} 1393 1394void IntrinsicCodeGeneratorARM64::VisitStringNewStringFromChars(HInvoke* invoke) { 1395 vixl::MacroAssembler* masm = GetVIXLAssembler(); 1396 1397 __ Ldr(lr, 1398 MemOperand(tr, QUICK_ENTRYPOINT_OFFSET(kArm64WordSize, pAllocStringFromChars).Int32Value())); 1399 codegen_->RecordPcInfo(invoke, invoke->GetDexPc()); 1400 __ Blr(lr); 1401} 1402 1403void IntrinsicLocationsBuilderARM64::VisitStringNewStringFromString(HInvoke* invoke) { 1404 // The inputs plus one temp. 1405 LocationSummary* locations = new (arena_) LocationSummary(invoke, 1406 LocationSummary::kCall, 1407 kIntrinsified); 1408 InvokeRuntimeCallingConvention calling_convention; 1409 locations->SetInAt(0, LocationFrom(calling_convention.GetRegisterAt(0))); 1410 locations->SetInAt(1, LocationFrom(calling_convention.GetRegisterAt(1))); 1411 locations->SetInAt(2, LocationFrom(calling_convention.GetRegisterAt(2))); 1412 locations->SetOut(calling_convention.GetReturnLocation(Primitive::kPrimNot)); 1413} 1414 1415void IntrinsicCodeGeneratorARM64::VisitStringNewStringFromString(HInvoke* invoke) { 1416 vixl::MacroAssembler* masm = GetVIXLAssembler(); 1417 LocationSummary* locations = invoke->GetLocations(); 1418 1419 Register string_to_copy = WRegisterFrom(locations->InAt(0)); 1420 __ Cmp(string_to_copy, 0); 1421 SlowPathCodeARM64* slow_path = new (GetAllocator()) IntrinsicSlowPathARM64(invoke); 1422 codegen_->AddSlowPath(slow_path); 1423 __ B(eq, slow_path->GetEntryLabel()); 1424 1425 __ Ldr(lr, 1426 MemOperand(tr, QUICK_ENTRYPOINT_OFFSET(kArm64WordSize, pAllocStringFromString).Int32Value())); 1427 codegen_->RecordPcInfo(invoke, invoke->GetDexPc()); 1428 __ Blr(lr); 1429 __ Bind(slow_path->GetExitLabel()); 1430} 1431 1432// Unimplemented intrinsics. 1433 1434#define UNIMPLEMENTED_INTRINSIC(Name) \ 1435void IntrinsicLocationsBuilderARM64::Visit ## Name(HInvoke* invoke ATTRIBUTE_UNUSED) { \ 1436} \ 1437void IntrinsicCodeGeneratorARM64::Visit ## Name(HInvoke* invoke ATTRIBUTE_UNUSED) { \ 1438} 1439 1440UNIMPLEMENTED_INTRINSIC(IntegerRotateLeft) 1441UNIMPLEMENTED_INTRINSIC(IntegerRotateRight) 1442UNIMPLEMENTED_INTRINSIC(LongRotateLeft) 1443UNIMPLEMENTED_INTRINSIC(LongRotateRight) 1444UNIMPLEMENTED_INTRINSIC(SystemArrayCopyChar) 1445UNIMPLEMENTED_INTRINSIC(SystemArrayCopy) 1446UNIMPLEMENTED_INTRINSIC(ReferenceGetReferent) 1447UNIMPLEMENTED_INTRINSIC(StringGetCharsNoCheck) 1448 1449UNIMPLEMENTED_INTRINSIC(MathCos) 1450UNIMPLEMENTED_INTRINSIC(MathSin) 1451UNIMPLEMENTED_INTRINSIC(MathAcos) 1452UNIMPLEMENTED_INTRINSIC(MathAsin) 1453UNIMPLEMENTED_INTRINSIC(MathAtan) 1454UNIMPLEMENTED_INTRINSIC(MathAtan2) 1455UNIMPLEMENTED_INTRINSIC(MathCbrt) 1456UNIMPLEMENTED_INTRINSIC(MathCosh) 1457UNIMPLEMENTED_INTRINSIC(MathExp) 1458UNIMPLEMENTED_INTRINSIC(MathExpm1) 1459UNIMPLEMENTED_INTRINSIC(MathHypot) 1460UNIMPLEMENTED_INTRINSIC(MathLog) 1461UNIMPLEMENTED_INTRINSIC(MathLog10) 1462UNIMPLEMENTED_INTRINSIC(MathNextAfter) 1463UNIMPLEMENTED_INTRINSIC(MathSinh) 1464UNIMPLEMENTED_INTRINSIC(MathTan) 1465UNIMPLEMENTED_INTRINSIC(MathTanh) 1466 1467#undef UNIMPLEMENTED_INTRINSIC 1468 1469#undef __ 1470 1471} // namespace arm64 1472} // namespace art 1473