171fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe/* 271fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe * Copyright (C) 2015 The Android Open Source Project 371fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe * 471fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe * Licensed under the Apache License, Version 2.0 (the "License"); 571fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe * you may not use this file except in compliance with the License. 671fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe * You may obtain a copy of the License at 771fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe * 871fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe * http://www.apache.org/licenses/LICENSE-2.0 971fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe * 1071fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe * Unless required by applicable law or agreed to in writing, software 1171fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe * distributed under the License is distributed on an "AS IS" BASIS, 1271fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 1371fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe * See the License for the specific language governing permissions and 1471fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe * limitations under the License. 1571fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe */ 1671fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe 1771fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe#include "intrinsics_x86_64.h" 1871fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe 1921030dd59b1e350f6f43de39e3c4ce0886ff539cAndreas Gampe#include <limits> 2021030dd59b1e350f6f43de39e3c4ce0886ff539cAndreas Gampe 21fb8d279bc011b31d0765dc7ca59afea324fd0d0cMark Mendell#include "arch/x86_64/instruction_set_features_x86_64.h" 22e401d146407d61eeb99f8d6176b2ac13c4df1e33Mathieu Chartier#include "art_method-inl.h" 23d5897678eb555a797d4e84e07814d79f0e0bb465Mark Mendell#include "base/bit_utils.h" 2471fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe#include "code_generator_x86_64.h" 2571fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe#include "entrypoints/quick/quick_entrypoints.h" 2671fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe#include "intrinsics.h" 2785b62f23fc6dfffe2ddd3ddfa74611666c9ff41dAndreas Gampe#include "intrinsics_utils.h" 2871fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe#include "mirror/array-inl.h" 2971fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe#include "mirror/string.h" 3071fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe#include "thread.h" 3171fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe#include "utils/x86_64/assembler_x86_64.h" 3271fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe#include "utils/x86_64/constants_x86_64.h" 3371fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe 3471fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampenamespace art { 3571fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe 3671fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampenamespace x86_64 { 3771fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe 38fb8d279bc011b31d0765dc7ca59afea324fd0d0cMark MendellIntrinsicLocationsBuilderX86_64::IntrinsicLocationsBuilderX86_64(CodeGeneratorX86_64* codegen) 39fb8d279bc011b31d0765dc7ca59afea324fd0d0cMark Mendell : arena_(codegen->GetGraph()->GetArena()), codegen_(codegen) { 40fb8d279bc011b31d0765dc7ca59afea324fd0d0cMark Mendell} 41fb8d279bc011b31d0765dc7ca59afea324fd0d0cMark Mendell 42fb8d279bc011b31d0765dc7ca59afea324fd0d0cMark Mendell 4371fb52fee246b7d511f520febbd73dc7a9bbca79Andreas GampeX86_64Assembler* IntrinsicCodeGeneratorX86_64::GetAssembler() { 44b488b7864b7bf9cade82d45c8bdda2372f48a10cRoland Levillain return down_cast<X86_64Assembler*>(codegen_->GetAssembler()); 4571fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe} 4671fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe 47878d58cbaf6b17a9e3dcab790754527f3ebc69e5Andreas GampeArenaAllocator* IntrinsicCodeGeneratorX86_64::GetAllocator() { 4871fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe return codegen_->GetGraph()->GetArena(); 4971fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe} 5071fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe 5171fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampebool IntrinsicLocationsBuilderX86_64::TryDispatch(HInvoke* invoke) { 5271fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe Dispatch(invoke); 530d5a281c671444bfa75d63caf1427a8c0e6e1177Roland Levillain LocationSummary* res = invoke->GetLocations(); 540d5a281c671444bfa75d63caf1427a8c0e6e1177Roland Levillain if (res == nullptr) { 550d5a281c671444bfa75d63caf1427a8c0e6e1177Roland Levillain return false; 560d5a281c671444bfa75d63caf1427a8c0e6e1177Roland Levillain } 570d5a281c671444bfa75d63caf1427a8c0e6e1177Roland Levillain if (kEmitCompilerReadBarrier && res->CanCall()) { 580d5a281c671444bfa75d63caf1427a8c0e6e1177Roland Levillain // Generating an intrinsic for this HInvoke may produce an 590d5a281c671444bfa75d63caf1427a8c0e6e1177Roland Levillain // IntrinsicSlowPathX86_64 slow path. Currently this approach 600d5a281c671444bfa75d63caf1427a8c0e6e1177Roland Levillain // does not work when using read barriers, as the emitted 610d5a281c671444bfa75d63caf1427a8c0e6e1177Roland Levillain // calling sequence will make use of another slow path 620d5a281c671444bfa75d63caf1427a8c0e6e1177Roland Levillain // (ReadBarrierForRootSlowPathX86_64 for HInvokeStaticOrDirect, 630d5a281c671444bfa75d63caf1427a8c0e6e1177Roland Levillain // ReadBarrierSlowPathX86_64 for HInvokeVirtual). So we bail 640d5a281c671444bfa75d63caf1427a8c0e6e1177Roland Levillain // out in this case. 650d5a281c671444bfa75d63caf1427a8c0e6e1177Roland Levillain // 660d5a281c671444bfa75d63caf1427a8c0e6e1177Roland Levillain // TODO: Find a way to have intrinsics work with read barriers. 670d5a281c671444bfa75d63caf1427a8c0e6e1177Roland Levillain invoke->SetLocations(nullptr); 680d5a281c671444bfa75d63caf1427a8c0e6e1177Roland Levillain return false; 690d5a281c671444bfa75d63caf1427a8c0e6e1177Roland Levillain } 700d5a281c671444bfa75d63caf1427a8c0e6e1177Roland Levillain return res->Intrinsified(); 7171fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe} 7271fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe 73ec525fc30848189051b888da53ba051bc0878b78Roland Levillainstatic void MoveArguments(HInvoke* invoke, CodeGeneratorX86_64* codegen) { 742d27c8e338af7262dbd4aaa66127bb8fa1758b86Roland Levillain InvokeDexCallingConventionVisitorX86_64 calling_convention_visitor; 75ec525fc30848189051b888da53ba051bc0878b78Roland Levillain IntrinsicVisitor::MoveArguments(invoke, codegen, &calling_convention_visitor); 7671fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe} 7771fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe 7885b62f23fc6dfffe2ddd3ddfa74611666c9ff41dAndreas Gampeusing IntrinsicSlowPathX86_64 = IntrinsicSlowPath<InvokeDexCallingConventionVisitorX86_64>; 7971fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe 8071fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe#define __ assembler-> 8171fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe 8271fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampestatic void CreateFPToIntLocations(ArenaAllocator* arena, HInvoke* invoke) { 8371fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe LocationSummary* locations = new (arena) LocationSummary(invoke, 8471fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe LocationSummary::kNoCall, 8571fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe kIntrinsified); 8671fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe locations->SetInAt(0, Location::RequiresFpuRegister()); 8771fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe locations->SetOut(Location::RequiresRegister()); 8871fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe} 8971fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe 9071fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampestatic void CreateIntToFPLocations(ArenaAllocator* arena, HInvoke* invoke) { 9171fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe LocationSummary* locations = new (arena) LocationSummary(invoke, 9271fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe LocationSummary::kNoCall, 9371fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe kIntrinsified); 9471fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe locations->SetInAt(0, Location::RequiresRegister()); 9571fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe locations->SetOut(Location::RequiresFpuRegister()); 9671fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe} 9771fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe 9871fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampestatic void MoveFPToInt(LocationSummary* locations, bool is64bit, X86_64Assembler* assembler) { 9971fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe Location input = locations->InAt(0); 10071fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe Location output = locations->Out(); 10171fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe __ movd(output.AsRegister<CpuRegister>(), input.AsFpuRegister<XmmRegister>(), is64bit); 10271fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe} 10371fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe 10471fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampestatic void MoveIntToFP(LocationSummary* locations, bool is64bit, X86_64Assembler* assembler) { 10571fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe Location input = locations->InAt(0); 10671fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe Location output = locations->Out(); 10771fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe __ movd(output.AsFpuRegister<XmmRegister>(), input.AsRegister<CpuRegister>(), is64bit); 10871fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe} 10971fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe 11071fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampevoid IntrinsicLocationsBuilderX86_64::VisitDoubleDoubleToRawLongBits(HInvoke* invoke) { 11171fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe CreateFPToIntLocations(arena_, invoke); 11271fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe} 11371fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampevoid IntrinsicLocationsBuilderX86_64::VisitDoubleLongBitsToDouble(HInvoke* invoke) { 11471fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe CreateIntToFPLocations(arena_, invoke); 11571fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe} 11671fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe 11771fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampevoid IntrinsicCodeGeneratorX86_64::VisitDoubleDoubleToRawLongBits(HInvoke* invoke) { 118bf84a3d2aa29c0975b4ac0f6f983d56724b2cc57Roland Levillain MoveFPToInt(invoke->GetLocations(), /* is64bit */ true, GetAssembler()); 11971fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe} 12071fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampevoid IntrinsicCodeGeneratorX86_64::VisitDoubleLongBitsToDouble(HInvoke* invoke) { 121bf84a3d2aa29c0975b4ac0f6f983d56724b2cc57Roland Levillain MoveIntToFP(invoke->GetLocations(), /* is64bit */ true, GetAssembler()); 12271fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe} 12371fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe 12471fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampevoid IntrinsicLocationsBuilderX86_64::VisitFloatFloatToRawIntBits(HInvoke* invoke) { 12571fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe CreateFPToIntLocations(arena_, invoke); 12671fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe} 12771fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampevoid IntrinsicLocationsBuilderX86_64::VisitFloatIntBitsToFloat(HInvoke* invoke) { 12871fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe CreateIntToFPLocations(arena_, invoke); 12971fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe} 13071fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe 13171fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampevoid IntrinsicCodeGeneratorX86_64::VisitFloatFloatToRawIntBits(HInvoke* invoke) { 132bf84a3d2aa29c0975b4ac0f6f983d56724b2cc57Roland Levillain MoveFPToInt(invoke->GetLocations(), /* is64bit */ false, GetAssembler()); 13371fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe} 13471fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampevoid IntrinsicCodeGeneratorX86_64::VisitFloatIntBitsToFloat(HInvoke* invoke) { 135bf84a3d2aa29c0975b4ac0f6f983d56724b2cc57Roland Levillain MoveIntToFP(invoke->GetLocations(), /* is64bit */ false, GetAssembler()); 13671fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe} 13771fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe 13871fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampestatic void CreateIntToIntLocations(ArenaAllocator* arena, HInvoke* invoke) { 13971fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe LocationSummary* locations = new (arena) LocationSummary(invoke, 14071fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe LocationSummary::kNoCall, 14171fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe kIntrinsified); 14271fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe locations->SetInAt(0, Location::RequiresRegister()); 14371fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe locations->SetOut(Location::SameAsFirstInput()); 14471fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe} 14571fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe 14671fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampestatic void GenReverseBytes(LocationSummary* locations, 14771fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe Primitive::Type size, 14871fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe X86_64Assembler* assembler) { 14971fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe CpuRegister out = locations->Out().AsRegister<CpuRegister>(); 15071fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe 15171fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe switch (size) { 15271fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe case Primitive::kPrimShort: 15371fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe // TODO: Can be done with an xchg of 8b registers. This is straight from Quick. 15471fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe __ bswapl(out); 15571fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe __ sarl(out, Immediate(16)); 15671fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe break; 15771fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe case Primitive::kPrimInt: 15871fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe __ bswapl(out); 15971fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe break; 16071fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe case Primitive::kPrimLong: 16171fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe __ bswapq(out); 16271fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe break; 16371fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe default: 16471fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe LOG(FATAL) << "Unexpected size for reverse-bytes: " << size; 16571fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe UNREACHABLE(); 16671fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe } 16771fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe} 16871fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe 16971fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampevoid IntrinsicLocationsBuilderX86_64::VisitIntegerReverseBytes(HInvoke* invoke) { 17071fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe CreateIntToIntLocations(arena_, invoke); 17171fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe} 17271fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe 17371fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampevoid IntrinsicCodeGeneratorX86_64::VisitIntegerReverseBytes(HInvoke* invoke) { 17471fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe GenReverseBytes(invoke->GetLocations(), Primitive::kPrimInt, GetAssembler()); 17571fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe} 17671fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe 17771fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampevoid IntrinsicLocationsBuilderX86_64::VisitLongReverseBytes(HInvoke* invoke) { 17871fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe CreateIntToIntLocations(arena_, invoke); 17971fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe} 18071fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe 18171fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampevoid IntrinsicCodeGeneratorX86_64::VisitLongReverseBytes(HInvoke* invoke) { 18271fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe GenReverseBytes(invoke->GetLocations(), Primitive::kPrimLong, GetAssembler()); 18371fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe} 18471fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe 18571fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampevoid IntrinsicLocationsBuilderX86_64::VisitShortReverseBytes(HInvoke* invoke) { 18671fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe CreateIntToIntLocations(arena_, invoke); 18771fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe} 18871fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe 18971fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampevoid IntrinsicCodeGeneratorX86_64::VisitShortReverseBytes(HInvoke* invoke) { 19071fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe GenReverseBytes(invoke->GetLocations(), Primitive::kPrimShort, GetAssembler()); 19171fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe} 19271fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe 19371fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe 19471fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe// TODO: Consider Quick's way of doing Double abs through integer operations, as the immediate we 19571fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe// need is 64b. 19671fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe 19771fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampestatic void CreateFloatToFloatPlusTemps(ArenaAllocator* arena, HInvoke* invoke) { 19871fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe // TODO: Enable memory operations when the assembler supports them. 19971fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe LocationSummary* locations = new (arena) LocationSummary(invoke, 20071fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe LocationSummary::kNoCall, 20171fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe kIntrinsified); 20271fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe locations->SetInAt(0, Location::RequiresFpuRegister()); 20371fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe locations->SetOut(Location::SameAsFirstInput()); 204f55c3e0825cdfc4c5a27730031177d1a0198ec5aMark Mendell locations->AddTemp(Location::RequiresFpuRegister()); // FP reg to hold mask. 20571fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe} 20671fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe 20739dcf55a56da746e04f477f89e7b00ba1de03880Mark Mendellstatic void MathAbsFP(LocationSummary* locations, 20839dcf55a56da746e04f477f89e7b00ba1de03880Mark Mendell bool is64bit, 20939dcf55a56da746e04f477f89e7b00ba1de03880Mark Mendell X86_64Assembler* assembler, 21039dcf55a56da746e04f477f89e7b00ba1de03880Mark Mendell CodeGeneratorX86_64* codegen) { 21171fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe Location output = locations->Out(); 21271fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe 213cfa410b0ea561318f74a76c5323f0f6cd8eaaa50Mark Mendell DCHECK(output.IsFpuRegister()); 214cfa410b0ea561318f74a76c5323f0f6cd8eaaa50Mark Mendell XmmRegister xmm_temp = locations->GetTemp(0).AsFpuRegister<XmmRegister>(); 21571fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe 216cfa410b0ea561318f74a76c5323f0f6cd8eaaa50Mark Mendell // TODO: Can mask directly with constant area using pand if we can guarantee 217cfa410b0ea561318f74a76c5323f0f6cd8eaaa50Mark Mendell // that the literal is aligned on a 16 byte boundary. This will avoid a 218cfa410b0ea561318f74a76c5323f0f6cd8eaaa50Mark Mendell // temporary. 219cfa410b0ea561318f74a76c5323f0f6cd8eaaa50Mark Mendell if (is64bit) { 220cfa410b0ea561318f74a76c5323f0f6cd8eaaa50Mark Mendell __ movsd(xmm_temp, codegen->LiteralInt64Address(INT64_C(0x7FFFFFFFFFFFFFFF))); 221cfa410b0ea561318f74a76c5323f0f6cd8eaaa50Mark Mendell __ andpd(output.AsFpuRegister<XmmRegister>(), xmm_temp); 22271fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe } else { 223cfa410b0ea561318f74a76c5323f0f6cd8eaaa50Mark Mendell __ movss(xmm_temp, codegen->LiteralInt32Address(INT32_C(0x7FFFFFFF))); 224cfa410b0ea561318f74a76c5323f0f6cd8eaaa50Mark Mendell __ andps(output.AsFpuRegister<XmmRegister>(), xmm_temp); 22571fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe } 22671fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe} 22771fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe 22871fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampevoid IntrinsicLocationsBuilderX86_64::VisitMathAbsDouble(HInvoke* invoke) { 22971fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe CreateFloatToFloatPlusTemps(arena_, invoke); 23071fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe} 23171fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe 23271fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampevoid IntrinsicCodeGeneratorX86_64::VisitMathAbsDouble(HInvoke* invoke) { 233bf84a3d2aa29c0975b4ac0f6f983d56724b2cc57Roland Levillain MathAbsFP(invoke->GetLocations(), /* is64bit */ true, GetAssembler(), codegen_); 23471fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe} 23571fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe 23671fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampevoid IntrinsicLocationsBuilderX86_64::VisitMathAbsFloat(HInvoke* invoke) { 23771fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe CreateFloatToFloatPlusTemps(arena_, invoke); 23871fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe} 23971fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe 24071fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampevoid IntrinsicCodeGeneratorX86_64::VisitMathAbsFloat(HInvoke* invoke) { 241bf84a3d2aa29c0975b4ac0f6f983d56724b2cc57Roland Levillain MathAbsFP(invoke->GetLocations(), /* is64bit */ false, GetAssembler(), codegen_); 24271fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe} 24371fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe 24471fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampestatic void CreateIntToIntPlusTemp(ArenaAllocator* arena, HInvoke* invoke) { 24571fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe LocationSummary* locations = new (arena) LocationSummary(invoke, 24671fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe LocationSummary::kNoCall, 24771fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe kIntrinsified); 24871fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe locations->SetInAt(0, Location::RequiresRegister()); 24971fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe locations->SetOut(Location::SameAsFirstInput()); 25071fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe locations->AddTemp(Location::RequiresRegister()); 25171fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe} 25271fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe 25371fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampestatic void GenAbsInteger(LocationSummary* locations, bool is64bit, X86_64Assembler* assembler) { 25471fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe Location output = locations->Out(); 25571fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe CpuRegister out = output.AsRegister<CpuRegister>(); 25671fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe CpuRegister mask = locations->GetTemp(0).AsRegister<CpuRegister>(); 25771fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe 25871fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe if (is64bit) { 25971fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe // Create mask. 26071fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe __ movq(mask, out); 26171fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe __ sarq(mask, Immediate(63)); 26271fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe // Add mask. 26371fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe __ addq(out, mask); 26471fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe __ xorq(out, mask); 26571fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe } else { 26671fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe // Create mask. 26771fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe __ movl(mask, out); 26871fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe __ sarl(mask, Immediate(31)); 26971fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe // Add mask. 27071fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe __ addl(out, mask); 27171fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe __ xorl(out, mask); 27271fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe } 27371fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe} 27471fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe 27571fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampevoid IntrinsicLocationsBuilderX86_64::VisitMathAbsInt(HInvoke* invoke) { 27671fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe CreateIntToIntPlusTemp(arena_, invoke); 27771fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe} 27871fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe 27971fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampevoid IntrinsicCodeGeneratorX86_64::VisitMathAbsInt(HInvoke* invoke) { 280bf84a3d2aa29c0975b4ac0f6f983d56724b2cc57Roland Levillain GenAbsInteger(invoke->GetLocations(), /* is64bit */ false, GetAssembler()); 28171fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe} 28271fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe 28371fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampevoid IntrinsicLocationsBuilderX86_64::VisitMathAbsLong(HInvoke* invoke) { 28471fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe CreateIntToIntPlusTemp(arena_, invoke); 28571fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe} 28671fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe 28771fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampevoid IntrinsicCodeGeneratorX86_64::VisitMathAbsLong(HInvoke* invoke) { 288bf84a3d2aa29c0975b4ac0f6f983d56724b2cc57Roland Levillain GenAbsInteger(invoke->GetLocations(), /* is64bit */ true, GetAssembler()); 28971fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe} 29071fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe 29139dcf55a56da746e04f477f89e7b00ba1de03880Mark Mendellstatic void GenMinMaxFP(LocationSummary* locations, 29239dcf55a56da746e04f477f89e7b00ba1de03880Mark Mendell bool is_min, 29339dcf55a56da746e04f477f89e7b00ba1de03880Mark Mendell bool is_double, 29439dcf55a56da746e04f477f89e7b00ba1de03880Mark Mendell X86_64Assembler* assembler, 29539dcf55a56da746e04f477f89e7b00ba1de03880Mark Mendell CodeGeneratorX86_64* codegen) { 29671fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe Location op1_loc = locations->InAt(0); 29771fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe Location op2_loc = locations->InAt(1); 29871fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe Location out_loc = locations->Out(); 29971fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe XmmRegister out = out_loc.AsFpuRegister<XmmRegister>(); 30071fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe 30171fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe // Shortcut for same input locations. 30271fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe if (op1_loc.Equals(op2_loc)) { 30371fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe DCHECK(out_loc.Equals(op1_loc)); 30471fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe return; 30571fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe } 30671fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe 30771fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe // (out := op1) 30871fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe // out <=? op2 30971fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe // if Nan jmp Nan_label 31071fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe // if out is min jmp done 31171fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe // if op2 is min jmp op2_label 31271fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe // handle -0/+0 31371fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe // jmp done 31471fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe // Nan_label: 31571fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe // out := NaN 31671fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe // op2_label: 31771fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe // out := op2 31871fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe // done: 31971fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe // 32071fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe // This removes one jmp, but needs to copy one input (op1) to out. 32171fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe // 322f55c3e0825cdfc4c5a27730031177d1a0198ec5aMark Mendell // TODO: This is straight from Quick. Make NaN an out-of-line slowpath? 32371fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe 32471fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe XmmRegister op2 = op2_loc.AsFpuRegister<XmmRegister>(); 32571fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe 3260c9497da9485ba688c592e5f452b7b1305a519c0Mark Mendell NearLabel nan, done, op2_label; 32771fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe if (is_double) { 32871fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe __ ucomisd(out, op2); 32971fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe } else { 33071fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe __ ucomiss(out, op2); 33171fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe } 33271fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe 33371fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe __ j(Condition::kParityEven, &nan); 33471fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe 33571fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe __ j(is_min ? Condition::kAbove : Condition::kBelow, &op2_label); 33671fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe __ j(is_min ? Condition::kBelow : Condition::kAbove, &done); 33771fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe 33871fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe // Handle 0.0/-0.0. 33971fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe if (is_min) { 34071fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe if (is_double) { 34171fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe __ orpd(out, op2); 34271fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe } else { 34371fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe __ orps(out, op2); 34471fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe } 34571fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe } else { 34671fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe if (is_double) { 34771fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe __ andpd(out, op2); 34871fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe } else { 34971fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe __ andps(out, op2); 35071fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe } 35171fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe } 35271fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe __ jmp(&done); 35371fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe 35471fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe // NaN handling. 35571fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe __ Bind(&nan); 35671fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe if (is_double) { 357f55c3e0825cdfc4c5a27730031177d1a0198ec5aMark Mendell __ movsd(out, codegen->LiteralInt64Address(INT64_C(0x7FF8000000000000))); 35871fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe } else { 359f55c3e0825cdfc4c5a27730031177d1a0198ec5aMark Mendell __ movss(out, codegen->LiteralInt32Address(INT32_C(0x7FC00000))); 36071fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe } 36171fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe __ jmp(&done); 36271fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe 36371fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe // out := op2; 36471fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe __ Bind(&op2_label); 36571fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe if (is_double) { 36671fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe __ movsd(out, op2); 36771fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe } else { 36871fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe __ movss(out, op2); 36971fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe } 37071fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe 37171fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe // Done. 37271fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe __ Bind(&done); 37371fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe} 37471fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe 375f55c3e0825cdfc4c5a27730031177d1a0198ec5aMark Mendellstatic void CreateFPFPToFP(ArenaAllocator* arena, HInvoke* invoke) { 37671fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe LocationSummary* locations = new (arena) LocationSummary(invoke, 37771fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe LocationSummary::kNoCall, 37871fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe kIntrinsified); 37971fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe locations->SetInAt(0, Location::RequiresFpuRegister()); 38071fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe locations->SetInAt(1, Location::RequiresFpuRegister()); 38171fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe // The following is sub-optimal, but all we can do for now. It would be fine to also accept 38271fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe // the second input to be the output (we can simply swap inputs). 38371fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe locations->SetOut(Location::SameAsFirstInput()); 38471fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe} 38571fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe 38671fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampevoid IntrinsicLocationsBuilderX86_64::VisitMathMinDoubleDouble(HInvoke* invoke) { 387f55c3e0825cdfc4c5a27730031177d1a0198ec5aMark Mendell CreateFPFPToFP(arena_, invoke); 38871fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe} 38971fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe 39071fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampevoid IntrinsicCodeGeneratorX86_64::VisitMathMinDoubleDouble(HInvoke* invoke) { 391bf84a3d2aa29c0975b4ac0f6f983d56724b2cc57Roland Levillain GenMinMaxFP( 392bf84a3d2aa29c0975b4ac0f6f983d56724b2cc57Roland Levillain invoke->GetLocations(), /* is_min */ true, /* is_double */ true, GetAssembler(), codegen_); 39371fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe} 39471fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe 39571fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampevoid IntrinsicLocationsBuilderX86_64::VisitMathMinFloatFloat(HInvoke* invoke) { 396f55c3e0825cdfc4c5a27730031177d1a0198ec5aMark Mendell CreateFPFPToFP(arena_, invoke); 39771fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe} 39871fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe 39971fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampevoid IntrinsicCodeGeneratorX86_64::VisitMathMinFloatFloat(HInvoke* invoke) { 400bf84a3d2aa29c0975b4ac0f6f983d56724b2cc57Roland Levillain GenMinMaxFP( 401bf84a3d2aa29c0975b4ac0f6f983d56724b2cc57Roland Levillain invoke->GetLocations(), /* is_min */ true, /* is_double */ false, GetAssembler(), codegen_); 40271fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe} 40371fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe 40471fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampevoid IntrinsicLocationsBuilderX86_64::VisitMathMaxDoubleDouble(HInvoke* invoke) { 405f55c3e0825cdfc4c5a27730031177d1a0198ec5aMark Mendell CreateFPFPToFP(arena_, invoke); 40671fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe} 40771fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe 40871fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampevoid IntrinsicCodeGeneratorX86_64::VisitMathMaxDoubleDouble(HInvoke* invoke) { 409bf84a3d2aa29c0975b4ac0f6f983d56724b2cc57Roland Levillain GenMinMaxFP( 410bf84a3d2aa29c0975b4ac0f6f983d56724b2cc57Roland Levillain invoke->GetLocations(), /* is_min */ false, /* is_double */ true, GetAssembler(), codegen_); 41171fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe} 41271fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe 41371fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampevoid IntrinsicLocationsBuilderX86_64::VisitMathMaxFloatFloat(HInvoke* invoke) { 414f55c3e0825cdfc4c5a27730031177d1a0198ec5aMark Mendell CreateFPFPToFP(arena_, invoke); 41571fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe} 41671fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe 41771fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampevoid IntrinsicCodeGeneratorX86_64::VisitMathMaxFloatFloat(HInvoke* invoke) { 418bf84a3d2aa29c0975b4ac0f6f983d56724b2cc57Roland Levillain GenMinMaxFP( 419bf84a3d2aa29c0975b4ac0f6f983d56724b2cc57Roland Levillain invoke->GetLocations(), /* is_min */ false, /* is_double */ false, GetAssembler(), codegen_); 42071fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe} 42171fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe 42271fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampestatic void GenMinMax(LocationSummary* locations, bool is_min, bool is_long, 42371fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe X86_64Assembler* assembler) { 42471fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe Location op1_loc = locations->InAt(0); 42571fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe Location op2_loc = locations->InAt(1); 42671fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe 42771fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe // Shortcut for same input locations. 42871fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe if (op1_loc.Equals(op2_loc)) { 42971fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe // Can return immediately, as op1_loc == out_loc. 43071fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe // Note: if we ever support separate registers, e.g., output into memory, we need to check for 43171fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe // a copy here. 43271fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe DCHECK(locations->Out().Equals(op1_loc)); 43371fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe return; 43471fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe } 43571fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe 43671fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe CpuRegister out = locations->Out().AsRegister<CpuRegister>(); 43771fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe CpuRegister op2 = op2_loc.AsRegister<CpuRegister>(); 43871fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe 43971fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe // (out := op1) 44071fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe // out <=? op2 44171fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe // if out is min jmp done 44271fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe // out := op2 44371fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe // done: 44471fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe 44571fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe if (is_long) { 44671fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe __ cmpq(out, op2); 44771fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe } else { 44871fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe __ cmpl(out, op2); 44971fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe } 45071fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe 45171fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe __ cmov(is_min ? Condition::kGreater : Condition::kLess, out, op2, is_long); 45271fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe} 45371fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe 45471fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampestatic void CreateIntIntToIntLocations(ArenaAllocator* arena, HInvoke* invoke) { 45571fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe LocationSummary* locations = new (arena) LocationSummary(invoke, 45671fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe LocationSummary::kNoCall, 45771fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe kIntrinsified); 45871fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe locations->SetInAt(0, Location::RequiresRegister()); 45971fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe locations->SetInAt(1, Location::RequiresRegister()); 46071fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe locations->SetOut(Location::SameAsFirstInput()); 46171fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe} 46271fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe 46371fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampevoid IntrinsicLocationsBuilderX86_64::VisitMathMinIntInt(HInvoke* invoke) { 46471fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe CreateIntIntToIntLocations(arena_, invoke); 46571fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe} 46671fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe 46771fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampevoid IntrinsicCodeGeneratorX86_64::VisitMathMinIntInt(HInvoke* invoke) { 468bf84a3d2aa29c0975b4ac0f6f983d56724b2cc57Roland Levillain GenMinMax(invoke->GetLocations(), /* is_min */ true, /* is_long */ false, GetAssembler()); 46971fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe} 47071fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe 47171fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampevoid IntrinsicLocationsBuilderX86_64::VisitMathMinLongLong(HInvoke* invoke) { 47271fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe CreateIntIntToIntLocations(arena_, invoke); 47371fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe} 47471fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe 47571fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampevoid IntrinsicCodeGeneratorX86_64::VisitMathMinLongLong(HInvoke* invoke) { 476bf84a3d2aa29c0975b4ac0f6f983d56724b2cc57Roland Levillain GenMinMax(invoke->GetLocations(), /* is_min */ true, /* is_long */ true, GetAssembler()); 47771fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe} 47871fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe 47971fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampevoid IntrinsicLocationsBuilderX86_64::VisitMathMaxIntInt(HInvoke* invoke) { 48071fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe CreateIntIntToIntLocations(arena_, invoke); 48171fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe} 48271fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe 48371fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampevoid IntrinsicCodeGeneratorX86_64::VisitMathMaxIntInt(HInvoke* invoke) { 484bf84a3d2aa29c0975b4ac0f6f983d56724b2cc57Roland Levillain GenMinMax(invoke->GetLocations(), /* is_min */ false, /* is_long */ false, GetAssembler()); 48571fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe} 48671fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe 48771fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampevoid IntrinsicLocationsBuilderX86_64::VisitMathMaxLongLong(HInvoke* invoke) { 48871fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe CreateIntIntToIntLocations(arena_, invoke); 48971fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe} 49071fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe 49171fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampevoid IntrinsicCodeGeneratorX86_64::VisitMathMaxLongLong(HInvoke* invoke) { 492bf84a3d2aa29c0975b4ac0f6f983d56724b2cc57Roland Levillain GenMinMax(invoke->GetLocations(), /* is_min */ false, /* is_long */ true, GetAssembler()); 49371fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe} 49471fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe 49571fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampestatic void CreateFPToFPLocations(ArenaAllocator* arena, HInvoke* invoke) { 49671fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe LocationSummary* locations = new (arena) LocationSummary(invoke, 49771fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe LocationSummary::kNoCall, 49871fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe kIntrinsified); 49971fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe locations->SetInAt(0, Location::RequiresFpuRegister()); 50071fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe locations->SetOut(Location::RequiresFpuRegister()); 50171fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe} 50271fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe 50371fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampevoid IntrinsicLocationsBuilderX86_64::VisitMathSqrt(HInvoke* invoke) { 50471fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe CreateFPToFPLocations(arena_, invoke); 50571fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe} 50671fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe 50771fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampevoid IntrinsicCodeGeneratorX86_64::VisitMathSqrt(HInvoke* invoke) { 50871fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe LocationSummary* locations = invoke->GetLocations(); 50971fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe XmmRegister in = locations->InAt(0).AsFpuRegister<XmmRegister>(); 51071fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe XmmRegister out = locations->Out().AsFpuRegister<XmmRegister>(); 51171fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe 51271fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe GetAssembler()->sqrtsd(out, in); 51371fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe} 51471fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe 515fb8d279bc011b31d0765dc7ca59afea324fd0d0cMark Mendellstatic void InvokeOutOfLineIntrinsic(CodeGeneratorX86_64* codegen, HInvoke* invoke) { 516ec525fc30848189051b888da53ba051bc0878b78Roland Levillain MoveArguments(invoke, codegen); 517fb8d279bc011b31d0765dc7ca59afea324fd0d0cMark Mendell 518fb8d279bc011b31d0765dc7ca59afea324fd0d0cMark Mendell DCHECK(invoke->IsInvokeStaticOrDirect()); 51994015b939060f5041d408d48717f22443e55b6adNicolas Geoffray codegen->GenerateStaticOrDirectCall( 52094015b939060f5041d408d48717f22443e55b6adNicolas Geoffray invoke->AsInvokeStaticOrDirect(), Location::RegisterLocation(RDI)); 521fb8d279bc011b31d0765dc7ca59afea324fd0d0cMark Mendell codegen->RecordPcInfo(invoke, invoke->GetDexPc()); 522fb8d279bc011b31d0765dc7ca59afea324fd0d0cMark Mendell 523fb8d279bc011b31d0765dc7ca59afea324fd0d0cMark Mendell // Copy the result back to the expected output. 524fb8d279bc011b31d0765dc7ca59afea324fd0d0cMark Mendell Location out = invoke->GetLocations()->Out(); 525fb8d279bc011b31d0765dc7ca59afea324fd0d0cMark Mendell if (out.IsValid()) { 526fb8d279bc011b31d0765dc7ca59afea324fd0d0cMark Mendell DCHECK(out.IsRegister()); 52785b62f23fc6dfffe2ddd3ddfa74611666c9ff41dAndreas Gampe codegen->MoveFromReturnRegister(out, invoke->GetType()); 528fb8d279bc011b31d0765dc7ca59afea324fd0d0cMark Mendell } 529fb8d279bc011b31d0765dc7ca59afea324fd0d0cMark Mendell} 530fb8d279bc011b31d0765dc7ca59afea324fd0d0cMark Mendell 531fb8d279bc011b31d0765dc7ca59afea324fd0d0cMark Mendellstatic void CreateSSE41FPToFPLocations(ArenaAllocator* arena, 532fb8d279bc011b31d0765dc7ca59afea324fd0d0cMark Mendell HInvoke* invoke, 533fb8d279bc011b31d0765dc7ca59afea324fd0d0cMark Mendell CodeGeneratorX86_64* codegen) { 534fb8d279bc011b31d0765dc7ca59afea324fd0d0cMark Mendell // Do we have instruction support? 535fb8d279bc011b31d0765dc7ca59afea324fd0d0cMark Mendell if (codegen->GetInstructionSetFeatures().HasSSE4_1()) { 536fb8d279bc011b31d0765dc7ca59afea324fd0d0cMark Mendell CreateFPToFPLocations(arena, invoke); 537fb8d279bc011b31d0765dc7ca59afea324fd0d0cMark Mendell return; 538fb8d279bc011b31d0765dc7ca59afea324fd0d0cMark Mendell } 539fb8d279bc011b31d0765dc7ca59afea324fd0d0cMark Mendell 540fb8d279bc011b31d0765dc7ca59afea324fd0d0cMark Mendell // We have to fall back to a call to the intrinsic. 541fb8d279bc011b31d0765dc7ca59afea324fd0d0cMark Mendell LocationSummary* locations = new (arena) LocationSummary(invoke, 542fb8d279bc011b31d0765dc7ca59afea324fd0d0cMark Mendell LocationSummary::kCall); 543fb8d279bc011b31d0765dc7ca59afea324fd0d0cMark Mendell InvokeRuntimeCallingConvention calling_convention; 544fb8d279bc011b31d0765dc7ca59afea324fd0d0cMark Mendell locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetFpuRegisterAt(0))); 545fb8d279bc011b31d0765dc7ca59afea324fd0d0cMark Mendell locations->SetOut(Location::FpuRegisterLocation(XMM0)); 546fb8d279bc011b31d0765dc7ca59afea324fd0d0cMark Mendell // Needs to be RDI for the invoke. 547fb8d279bc011b31d0765dc7ca59afea324fd0d0cMark Mendell locations->AddTemp(Location::RegisterLocation(RDI)); 548fb8d279bc011b31d0765dc7ca59afea324fd0d0cMark Mendell} 549fb8d279bc011b31d0765dc7ca59afea324fd0d0cMark Mendell 550fb8d279bc011b31d0765dc7ca59afea324fd0d0cMark Mendellstatic void GenSSE41FPToFPIntrinsic(CodeGeneratorX86_64* codegen, 551fb8d279bc011b31d0765dc7ca59afea324fd0d0cMark Mendell HInvoke* invoke, 552fb8d279bc011b31d0765dc7ca59afea324fd0d0cMark Mendell X86_64Assembler* assembler, 553fb8d279bc011b31d0765dc7ca59afea324fd0d0cMark Mendell int round_mode) { 554fb8d279bc011b31d0765dc7ca59afea324fd0d0cMark Mendell LocationSummary* locations = invoke->GetLocations(); 555fb8d279bc011b31d0765dc7ca59afea324fd0d0cMark Mendell if (locations->WillCall()) { 556fb8d279bc011b31d0765dc7ca59afea324fd0d0cMark Mendell InvokeOutOfLineIntrinsic(codegen, invoke); 557fb8d279bc011b31d0765dc7ca59afea324fd0d0cMark Mendell } else { 558fb8d279bc011b31d0765dc7ca59afea324fd0d0cMark Mendell XmmRegister in = locations->InAt(0).AsFpuRegister<XmmRegister>(); 559fb8d279bc011b31d0765dc7ca59afea324fd0d0cMark Mendell XmmRegister out = locations->Out().AsFpuRegister<XmmRegister>(); 560fb8d279bc011b31d0765dc7ca59afea324fd0d0cMark Mendell __ roundsd(out, in, Immediate(round_mode)); 561fb8d279bc011b31d0765dc7ca59afea324fd0d0cMark Mendell } 562fb8d279bc011b31d0765dc7ca59afea324fd0d0cMark Mendell} 563fb8d279bc011b31d0765dc7ca59afea324fd0d0cMark Mendell 564fb8d279bc011b31d0765dc7ca59afea324fd0d0cMark Mendellvoid IntrinsicLocationsBuilderX86_64::VisitMathCeil(HInvoke* invoke) { 565fb8d279bc011b31d0765dc7ca59afea324fd0d0cMark Mendell CreateSSE41FPToFPLocations(arena_, invoke, codegen_); 566fb8d279bc011b31d0765dc7ca59afea324fd0d0cMark Mendell} 567fb8d279bc011b31d0765dc7ca59afea324fd0d0cMark Mendell 568fb8d279bc011b31d0765dc7ca59afea324fd0d0cMark Mendellvoid IntrinsicCodeGeneratorX86_64::VisitMathCeil(HInvoke* invoke) { 569fb8d279bc011b31d0765dc7ca59afea324fd0d0cMark Mendell GenSSE41FPToFPIntrinsic(codegen_, invoke, GetAssembler(), 2); 570fb8d279bc011b31d0765dc7ca59afea324fd0d0cMark Mendell} 571fb8d279bc011b31d0765dc7ca59afea324fd0d0cMark Mendell 572fb8d279bc011b31d0765dc7ca59afea324fd0d0cMark Mendellvoid IntrinsicLocationsBuilderX86_64::VisitMathFloor(HInvoke* invoke) { 573fb8d279bc011b31d0765dc7ca59afea324fd0d0cMark Mendell CreateSSE41FPToFPLocations(arena_, invoke, codegen_); 574fb8d279bc011b31d0765dc7ca59afea324fd0d0cMark Mendell} 575fb8d279bc011b31d0765dc7ca59afea324fd0d0cMark Mendell 576fb8d279bc011b31d0765dc7ca59afea324fd0d0cMark Mendellvoid IntrinsicCodeGeneratorX86_64::VisitMathFloor(HInvoke* invoke) { 577fb8d279bc011b31d0765dc7ca59afea324fd0d0cMark Mendell GenSSE41FPToFPIntrinsic(codegen_, invoke, GetAssembler(), 1); 578fb8d279bc011b31d0765dc7ca59afea324fd0d0cMark Mendell} 579fb8d279bc011b31d0765dc7ca59afea324fd0d0cMark Mendell 580fb8d279bc011b31d0765dc7ca59afea324fd0d0cMark Mendellvoid IntrinsicLocationsBuilderX86_64::VisitMathRint(HInvoke* invoke) { 581fb8d279bc011b31d0765dc7ca59afea324fd0d0cMark Mendell CreateSSE41FPToFPLocations(arena_, invoke, codegen_); 582fb8d279bc011b31d0765dc7ca59afea324fd0d0cMark Mendell} 583fb8d279bc011b31d0765dc7ca59afea324fd0d0cMark Mendell 584fb8d279bc011b31d0765dc7ca59afea324fd0d0cMark Mendellvoid IntrinsicCodeGeneratorX86_64::VisitMathRint(HInvoke* invoke) { 585fb8d279bc011b31d0765dc7ca59afea324fd0d0cMark Mendell GenSSE41FPToFPIntrinsic(codegen_, invoke, GetAssembler(), 0); 586fb8d279bc011b31d0765dc7ca59afea324fd0d0cMark Mendell} 587fb8d279bc011b31d0765dc7ca59afea324fd0d0cMark Mendell 588fb8d279bc011b31d0765dc7ca59afea324fd0d0cMark Mendellstatic void CreateSSE41FPToIntLocations(ArenaAllocator* arena, 589fb8d279bc011b31d0765dc7ca59afea324fd0d0cMark Mendell HInvoke* invoke, 590fb8d279bc011b31d0765dc7ca59afea324fd0d0cMark Mendell CodeGeneratorX86_64* codegen) { 591fb8d279bc011b31d0765dc7ca59afea324fd0d0cMark Mendell // Do we have instruction support? 592fb8d279bc011b31d0765dc7ca59afea324fd0d0cMark Mendell if (codegen->GetInstructionSetFeatures().HasSSE4_1()) { 593fb8d279bc011b31d0765dc7ca59afea324fd0d0cMark Mendell LocationSummary* locations = new (arena) LocationSummary(invoke, 594fb8d279bc011b31d0765dc7ca59afea324fd0d0cMark Mendell LocationSummary::kNoCall, 595fb8d279bc011b31d0765dc7ca59afea324fd0d0cMark Mendell kIntrinsified); 596fb8d279bc011b31d0765dc7ca59afea324fd0d0cMark Mendell locations->SetInAt(0, Location::RequiresFpuRegister()); 5979ca257196b46fd7629bce0b338580e571e4113a8Pavel Vyssotski locations->SetOut(Location::RequiresRegister()); 598fb8d279bc011b31d0765dc7ca59afea324fd0d0cMark Mendell locations->AddTemp(Location::RequiresFpuRegister()); 599fb8d279bc011b31d0765dc7ca59afea324fd0d0cMark Mendell return; 600fb8d279bc011b31d0765dc7ca59afea324fd0d0cMark Mendell } 601fb8d279bc011b31d0765dc7ca59afea324fd0d0cMark Mendell 602fb8d279bc011b31d0765dc7ca59afea324fd0d0cMark Mendell // We have to fall back to a call to the intrinsic. 603fb8d279bc011b31d0765dc7ca59afea324fd0d0cMark Mendell LocationSummary* locations = new (arena) LocationSummary(invoke, 604fb8d279bc011b31d0765dc7ca59afea324fd0d0cMark Mendell LocationSummary::kCall); 605fb8d279bc011b31d0765dc7ca59afea324fd0d0cMark Mendell InvokeRuntimeCallingConvention calling_convention; 606fb8d279bc011b31d0765dc7ca59afea324fd0d0cMark Mendell locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetFpuRegisterAt(0))); 607fb8d279bc011b31d0765dc7ca59afea324fd0d0cMark Mendell locations->SetOut(Location::RegisterLocation(RAX)); 608fb8d279bc011b31d0765dc7ca59afea324fd0d0cMark Mendell // Needs to be RDI for the invoke. 609fb8d279bc011b31d0765dc7ca59afea324fd0d0cMark Mendell locations->AddTemp(Location::RegisterLocation(RDI)); 610fb8d279bc011b31d0765dc7ca59afea324fd0d0cMark Mendell} 611fb8d279bc011b31d0765dc7ca59afea324fd0d0cMark Mendell 612fb8d279bc011b31d0765dc7ca59afea324fd0d0cMark Mendellvoid IntrinsicLocationsBuilderX86_64::VisitMathRoundFloat(HInvoke* invoke) { 613e6d0d8de85f79c8702ee722a04cd89ee7e89aeb7Andreas Gampe // See intrinsics.h. 614e6d0d8de85f79c8702ee722a04cd89ee7e89aeb7Andreas Gampe if (kRoundIsPlusPointFive) { 615e6d0d8de85f79c8702ee722a04cd89ee7e89aeb7Andreas Gampe CreateSSE41FPToIntLocations(arena_, invoke, codegen_); 616e6d0d8de85f79c8702ee722a04cd89ee7e89aeb7Andreas Gampe } 617fb8d279bc011b31d0765dc7ca59afea324fd0d0cMark Mendell} 618fb8d279bc011b31d0765dc7ca59afea324fd0d0cMark Mendell 619fb8d279bc011b31d0765dc7ca59afea324fd0d0cMark Mendellvoid IntrinsicCodeGeneratorX86_64::VisitMathRoundFloat(HInvoke* invoke) { 620fb8d279bc011b31d0765dc7ca59afea324fd0d0cMark Mendell LocationSummary* locations = invoke->GetLocations(); 621fb8d279bc011b31d0765dc7ca59afea324fd0d0cMark Mendell if (locations->WillCall()) { 622fb8d279bc011b31d0765dc7ca59afea324fd0d0cMark Mendell InvokeOutOfLineIntrinsic(codegen_, invoke); 623fb8d279bc011b31d0765dc7ca59afea324fd0d0cMark Mendell return; 624fb8d279bc011b31d0765dc7ca59afea324fd0d0cMark Mendell } 625fb8d279bc011b31d0765dc7ca59afea324fd0d0cMark Mendell 626fb8d279bc011b31d0765dc7ca59afea324fd0d0cMark Mendell // Implement RoundFloat as t1 = floor(input + 0.5f); convert to int. 627fb8d279bc011b31d0765dc7ca59afea324fd0d0cMark Mendell XmmRegister in = locations->InAt(0).AsFpuRegister<XmmRegister>(); 628fb8d279bc011b31d0765dc7ca59afea324fd0d0cMark Mendell CpuRegister out = locations->Out().AsRegister<CpuRegister>(); 62940741f394b2737e503f2c08be0ae9dd490fb106bMark Mendell XmmRegister inPlusPointFive = locations->GetTemp(0).AsFpuRegister<XmmRegister>(); 6300c9497da9485ba688c592e5f452b7b1305a519c0Mark Mendell NearLabel done, nan; 631fb8d279bc011b31d0765dc7ca59afea324fd0d0cMark Mendell X86_64Assembler* assembler = GetAssembler(); 632fb8d279bc011b31d0765dc7ca59afea324fd0d0cMark Mendell 63340741f394b2737e503f2c08be0ae9dd490fb106bMark Mendell // Load 0.5 into inPlusPointFive. 63440741f394b2737e503f2c08be0ae9dd490fb106bMark Mendell __ movss(inPlusPointFive, codegen_->LiteralFloatAddress(0.5f)); 635fb8d279bc011b31d0765dc7ca59afea324fd0d0cMark Mendell 636fb8d279bc011b31d0765dc7ca59afea324fd0d0cMark Mendell // Add in the input. 637fb8d279bc011b31d0765dc7ca59afea324fd0d0cMark Mendell __ addss(inPlusPointFive, in); 638fb8d279bc011b31d0765dc7ca59afea324fd0d0cMark Mendell 639fb8d279bc011b31d0765dc7ca59afea324fd0d0cMark Mendell // And truncate to an integer. 640fb8d279bc011b31d0765dc7ca59afea324fd0d0cMark Mendell __ roundss(inPlusPointFive, inPlusPointFive, Immediate(1)); 641fb8d279bc011b31d0765dc7ca59afea324fd0d0cMark Mendell 6429ca257196b46fd7629bce0b338580e571e4113a8Pavel Vyssotski // Load maxInt into out. 6439ca257196b46fd7629bce0b338580e571e4113a8Pavel Vyssotski codegen_->Load64BitValue(out, kPrimIntMax); 6449ca257196b46fd7629bce0b338580e571e4113a8Pavel Vyssotski 645fb8d279bc011b31d0765dc7ca59afea324fd0d0cMark Mendell // if inPlusPointFive >= maxInt goto done 64640741f394b2737e503f2c08be0ae9dd490fb106bMark Mendell __ comiss(inPlusPointFive, codegen_->LiteralFloatAddress(static_cast<float>(kPrimIntMax))); 647fb8d279bc011b31d0765dc7ca59afea324fd0d0cMark Mendell __ j(kAboveEqual, &done); 648fb8d279bc011b31d0765dc7ca59afea324fd0d0cMark Mendell 649fb8d279bc011b31d0765dc7ca59afea324fd0d0cMark Mendell // if input == NaN goto nan 650fb8d279bc011b31d0765dc7ca59afea324fd0d0cMark Mendell __ j(kUnordered, &nan); 651fb8d279bc011b31d0765dc7ca59afea324fd0d0cMark Mendell 652fb8d279bc011b31d0765dc7ca59afea324fd0d0cMark Mendell // output = float-to-int-truncate(input) 653fb8d279bc011b31d0765dc7ca59afea324fd0d0cMark Mendell __ cvttss2si(out, inPlusPointFive); 654fb8d279bc011b31d0765dc7ca59afea324fd0d0cMark Mendell __ jmp(&done); 655fb8d279bc011b31d0765dc7ca59afea324fd0d0cMark Mendell __ Bind(&nan); 656fb8d279bc011b31d0765dc7ca59afea324fd0d0cMark Mendell 657fb8d279bc011b31d0765dc7ca59afea324fd0d0cMark Mendell // output = 0 658fb8d279bc011b31d0765dc7ca59afea324fd0d0cMark Mendell __ xorl(out, out); 659fb8d279bc011b31d0765dc7ca59afea324fd0d0cMark Mendell __ Bind(&done); 660fb8d279bc011b31d0765dc7ca59afea324fd0d0cMark Mendell} 661fb8d279bc011b31d0765dc7ca59afea324fd0d0cMark Mendell 662fb8d279bc011b31d0765dc7ca59afea324fd0d0cMark Mendellvoid IntrinsicLocationsBuilderX86_64::VisitMathRoundDouble(HInvoke* invoke) { 663e6d0d8de85f79c8702ee722a04cd89ee7e89aeb7Andreas Gampe // See intrinsics.h. 664e6d0d8de85f79c8702ee722a04cd89ee7e89aeb7Andreas Gampe if (kRoundIsPlusPointFive) { 665e6d0d8de85f79c8702ee722a04cd89ee7e89aeb7Andreas Gampe CreateSSE41FPToIntLocations(arena_, invoke, codegen_); 666e6d0d8de85f79c8702ee722a04cd89ee7e89aeb7Andreas Gampe } 667fb8d279bc011b31d0765dc7ca59afea324fd0d0cMark Mendell} 668fb8d279bc011b31d0765dc7ca59afea324fd0d0cMark Mendell 669fb8d279bc011b31d0765dc7ca59afea324fd0d0cMark Mendellvoid IntrinsicCodeGeneratorX86_64::VisitMathRoundDouble(HInvoke* invoke) { 670fb8d279bc011b31d0765dc7ca59afea324fd0d0cMark Mendell LocationSummary* locations = invoke->GetLocations(); 671fb8d279bc011b31d0765dc7ca59afea324fd0d0cMark Mendell if (locations->WillCall()) { 672fb8d279bc011b31d0765dc7ca59afea324fd0d0cMark Mendell InvokeOutOfLineIntrinsic(codegen_, invoke); 673fb8d279bc011b31d0765dc7ca59afea324fd0d0cMark Mendell return; 674fb8d279bc011b31d0765dc7ca59afea324fd0d0cMark Mendell } 675fb8d279bc011b31d0765dc7ca59afea324fd0d0cMark Mendell 676fb8d279bc011b31d0765dc7ca59afea324fd0d0cMark Mendell // Implement RoundDouble as t1 = floor(input + 0.5); convert to long. 677fb8d279bc011b31d0765dc7ca59afea324fd0d0cMark Mendell XmmRegister in = locations->InAt(0).AsFpuRegister<XmmRegister>(); 678fb8d279bc011b31d0765dc7ca59afea324fd0d0cMark Mendell CpuRegister out = locations->Out().AsRegister<CpuRegister>(); 67940741f394b2737e503f2c08be0ae9dd490fb106bMark Mendell XmmRegister inPlusPointFive = locations->GetTemp(0).AsFpuRegister<XmmRegister>(); 6800c9497da9485ba688c592e5f452b7b1305a519c0Mark Mendell NearLabel done, nan; 681fb8d279bc011b31d0765dc7ca59afea324fd0d0cMark Mendell X86_64Assembler* assembler = GetAssembler(); 682fb8d279bc011b31d0765dc7ca59afea324fd0d0cMark Mendell 68340741f394b2737e503f2c08be0ae9dd490fb106bMark Mendell // Load 0.5 into inPlusPointFive. 68440741f394b2737e503f2c08be0ae9dd490fb106bMark Mendell __ movsd(inPlusPointFive, codegen_->LiteralDoubleAddress(0.5)); 685fb8d279bc011b31d0765dc7ca59afea324fd0d0cMark Mendell 686fb8d279bc011b31d0765dc7ca59afea324fd0d0cMark Mendell // Add in the input. 687fb8d279bc011b31d0765dc7ca59afea324fd0d0cMark Mendell __ addsd(inPlusPointFive, in); 688fb8d279bc011b31d0765dc7ca59afea324fd0d0cMark Mendell 689fb8d279bc011b31d0765dc7ca59afea324fd0d0cMark Mendell // And truncate to an integer. 690fb8d279bc011b31d0765dc7ca59afea324fd0d0cMark Mendell __ roundsd(inPlusPointFive, inPlusPointFive, Immediate(1)); 691fb8d279bc011b31d0765dc7ca59afea324fd0d0cMark Mendell 6929ca257196b46fd7629bce0b338580e571e4113a8Pavel Vyssotski // Load maxLong into out. 6939ca257196b46fd7629bce0b338580e571e4113a8Pavel Vyssotski codegen_->Load64BitValue(out, kPrimLongMax); 6949ca257196b46fd7629bce0b338580e571e4113a8Pavel Vyssotski 695fb8d279bc011b31d0765dc7ca59afea324fd0d0cMark Mendell // if inPlusPointFive >= maxLong goto done 69640741f394b2737e503f2c08be0ae9dd490fb106bMark Mendell __ comisd(inPlusPointFive, codegen_->LiteralDoubleAddress(static_cast<double>(kPrimLongMax))); 697fb8d279bc011b31d0765dc7ca59afea324fd0d0cMark Mendell __ j(kAboveEqual, &done); 698fb8d279bc011b31d0765dc7ca59afea324fd0d0cMark Mendell 699fb8d279bc011b31d0765dc7ca59afea324fd0d0cMark Mendell // if input == NaN goto nan 700fb8d279bc011b31d0765dc7ca59afea324fd0d0cMark Mendell __ j(kUnordered, &nan); 701fb8d279bc011b31d0765dc7ca59afea324fd0d0cMark Mendell 702fb8d279bc011b31d0765dc7ca59afea324fd0d0cMark Mendell // output = double-to-long-truncate(input) 703bf84a3d2aa29c0975b4ac0f6f983d56724b2cc57Roland Levillain __ cvttsd2si(out, inPlusPointFive, /* is64bit */ true); 704fb8d279bc011b31d0765dc7ca59afea324fd0d0cMark Mendell __ jmp(&done); 705fb8d279bc011b31d0765dc7ca59afea324fd0d0cMark Mendell __ Bind(&nan); 706fb8d279bc011b31d0765dc7ca59afea324fd0d0cMark Mendell 707fb8d279bc011b31d0765dc7ca59afea324fd0d0cMark Mendell // output = 0 70892e83bf8c0b2df8c977ffbc527989631d94b1819Mark Mendell __ xorl(out, out); 709fb8d279bc011b31d0765dc7ca59afea324fd0d0cMark Mendell __ Bind(&done); 710fb8d279bc011b31d0765dc7ca59afea324fd0d0cMark Mendell} 711fb8d279bc011b31d0765dc7ca59afea324fd0d0cMark Mendell 712a4f1220c1518074db18ca1044e9201492975750bMark Mendellstatic void CreateFPToFPCallLocations(ArenaAllocator* arena, 713a4f1220c1518074db18ca1044e9201492975750bMark Mendell HInvoke* invoke) { 714a4f1220c1518074db18ca1044e9201492975750bMark Mendell LocationSummary* locations = new (arena) LocationSummary(invoke, 715a4f1220c1518074db18ca1044e9201492975750bMark Mendell LocationSummary::kCall, 716a4f1220c1518074db18ca1044e9201492975750bMark Mendell kIntrinsified); 717a4f1220c1518074db18ca1044e9201492975750bMark Mendell InvokeRuntimeCallingConvention calling_convention; 718a4f1220c1518074db18ca1044e9201492975750bMark Mendell locations->SetInAt(0, Location::FpuRegisterLocation(calling_convention.GetFpuRegisterAt(0))); 719a4f1220c1518074db18ca1044e9201492975750bMark Mendell locations->SetOut(Location::FpuRegisterLocation(XMM0)); 720a4f1220c1518074db18ca1044e9201492975750bMark Mendell 721a4f1220c1518074db18ca1044e9201492975750bMark Mendell // We have to ensure that the native code doesn't clobber the XMM registers which are 722a4f1220c1518074db18ca1044e9201492975750bMark Mendell // non-volatile for ART, but volatile for Native calls. This will ensure that they are 723a4f1220c1518074db18ca1044e9201492975750bMark Mendell // saved in the prologue and properly restored. 724a4f1220c1518074db18ca1044e9201492975750bMark Mendell for (auto fp_reg : non_volatile_xmm_regs) { 725a4f1220c1518074db18ca1044e9201492975750bMark Mendell locations->AddTemp(Location::FpuRegisterLocation(fp_reg)); 726a4f1220c1518074db18ca1044e9201492975750bMark Mendell } 727a4f1220c1518074db18ca1044e9201492975750bMark Mendell} 728a4f1220c1518074db18ca1044e9201492975750bMark Mendell 729a4f1220c1518074db18ca1044e9201492975750bMark Mendellstatic void GenFPToFPCall(HInvoke* invoke, CodeGeneratorX86_64* codegen, 730a4f1220c1518074db18ca1044e9201492975750bMark Mendell QuickEntrypointEnum entry) { 731a4f1220c1518074db18ca1044e9201492975750bMark Mendell LocationSummary* locations = invoke->GetLocations(); 732a4f1220c1518074db18ca1044e9201492975750bMark Mendell DCHECK(locations->WillCall()); 733a4f1220c1518074db18ca1044e9201492975750bMark Mendell DCHECK(invoke->IsInvokeStaticOrDirect()); 734a4f1220c1518074db18ca1044e9201492975750bMark Mendell X86_64Assembler* assembler = codegen->GetAssembler(); 735a4f1220c1518074db18ca1044e9201492975750bMark Mendell 736a4f1220c1518074db18ca1044e9201492975750bMark Mendell __ gs()->call(Address::Absolute(GetThreadOffset<kX86_64WordSize>(entry), true)); 737a4f1220c1518074db18ca1044e9201492975750bMark Mendell codegen->RecordPcInfo(invoke, invoke->GetDexPc()); 738a4f1220c1518074db18ca1044e9201492975750bMark Mendell} 739a4f1220c1518074db18ca1044e9201492975750bMark Mendell 740a4f1220c1518074db18ca1044e9201492975750bMark Mendellvoid IntrinsicLocationsBuilderX86_64::VisitMathCos(HInvoke* invoke) { 741a4f1220c1518074db18ca1044e9201492975750bMark Mendell CreateFPToFPCallLocations(arena_, invoke); 742a4f1220c1518074db18ca1044e9201492975750bMark Mendell} 743a4f1220c1518074db18ca1044e9201492975750bMark Mendell 744a4f1220c1518074db18ca1044e9201492975750bMark Mendellvoid IntrinsicCodeGeneratorX86_64::VisitMathCos(HInvoke* invoke) { 745a4f1220c1518074db18ca1044e9201492975750bMark Mendell GenFPToFPCall(invoke, codegen_, kQuickCos); 746a4f1220c1518074db18ca1044e9201492975750bMark Mendell} 747a4f1220c1518074db18ca1044e9201492975750bMark Mendell 748a4f1220c1518074db18ca1044e9201492975750bMark Mendellvoid IntrinsicLocationsBuilderX86_64::VisitMathSin(HInvoke* invoke) { 749a4f1220c1518074db18ca1044e9201492975750bMark Mendell CreateFPToFPCallLocations(arena_, invoke); 750a4f1220c1518074db18ca1044e9201492975750bMark Mendell} 751a4f1220c1518074db18ca1044e9201492975750bMark Mendell 752a4f1220c1518074db18ca1044e9201492975750bMark Mendellvoid IntrinsicCodeGeneratorX86_64::VisitMathSin(HInvoke* invoke) { 753a4f1220c1518074db18ca1044e9201492975750bMark Mendell GenFPToFPCall(invoke, codegen_, kQuickSin); 754a4f1220c1518074db18ca1044e9201492975750bMark Mendell} 755a4f1220c1518074db18ca1044e9201492975750bMark Mendell 756a4f1220c1518074db18ca1044e9201492975750bMark Mendellvoid IntrinsicLocationsBuilderX86_64::VisitMathAcos(HInvoke* invoke) { 757a4f1220c1518074db18ca1044e9201492975750bMark Mendell CreateFPToFPCallLocations(arena_, invoke); 758a4f1220c1518074db18ca1044e9201492975750bMark Mendell} 759a4f1220c1518074db18ca1044e9201492975750bMark Mendell 760a4f1220c1518074db18ca1044e9201492975750bMark Mendellvoid IntrinsicCodeGeneratorX86_64::VisitMathAcos(HInvoke* invoke) { 761a4f1220c1518074db18ca1044e9201492975750bMark Mendell GenFPToFPCall(invoke, codegen_, kQuickAcos); 762a4f1220c1518074db18ca1044e9201492975750bMark Mendell} 763a4f1220c1518074db18ca1044e9201492975750bMark Mendell 764a4f1220c1518074db18ca1044e9201492975750bMark Mendellvoid IntrinsicLocationsBuilderX86_64::VisitMathAsin(HInvoke* invoke) { 765a4f1220c1518074db18ca1044e9201492975750bMark Mendell CreateFPToFPCallLocations(arena_, invoke); 766a4f1220c1518074db18ca1044e9201492975750bMark Mendell} 767a4f1220c1518074db18ca1044e9201492975750bMark Mendell 768a4f1220c1518074db18ca1044e9201492975750bMark Mendellvoid IntrinsicCodeGeneratorX86_64::VisitMathAsin(HInvoke* invoke) { 769a4f1220c1518074db18ca1044e9201492975750bMark Mendell GenFPToFPCall(invoke, codegen_, kQuickAsin); 770a4f1220c1518074db18ca1044e9201492975750bMark Mendell} 771a4f1220c1518074db18ca1044e9201492975750bMark Mendell 772a4f1220c1518074db18ca1044e9201492975750bMark Mendellvoid IntrinsicLocationsBuilderX86_64::VisitMathAtan(HInvoke* invoke) { 773a4f1220c1518074db18ca1044e9201492975750bMark Mendell CreateFPToFPCallLocations(arena_, invoke); 774a4f1220c1518074db18ca1044e9201492975750bMark Mendell} 775a4f1220c1518074db18ca1044e9201492975750bMark Mendell 776a4f1220c1518074db18ca1044e9201492975750bMark Mendellvoid IntrinsicCodeGeneratorX86_64::VisitMathAtan(HInvoke* invoke) { 777a4f1220c1518074db18ca1044e9201492975750bMark Mendell GenFPToFPCall(invoke, codegen_, kQuickAtan); 778a4f1220c1518074db18ca1044e9201492975750bMark Mendell} 779a4f1220c1518074db18ca1044e9201492975750bMark Mendell 780a4f1220c1518074db18ca1044e9201492975750bMark Mendellvoid IntrinsicLocationsBuilderX86_64::VisitMathCbrt(HInvoke* invoke) { 781a4f1220c1518074db18ca1044e9201492975750bMark Mendell CreateFPToFPCallLocations(arena_, invoke); 782a4f1220c1518074db18ca1044e9201492975750bMark Mendell} 783a4f1220c1518074db18ca1044e9201492975750bMark Mendell 784a4f1220c1518074db18ca1044e9201492975750bMark Mendellvoid IntrinsicCodeGeneratorX86_64::VisitMathCbrt(HInvoke* invoke) { 785a4f1220c1518074db18ca1044e9201492975750bMark Mendell GenFPToFPCall(invoke, codegen_, kQuickCbrt); 786a4f1220c1518074db18ca1044e9201492975750bMark Mendell} 787a4f1220c1518074db18ca1044e9201492975750bMark Mendell 788a4f1220c1518074db18ca1044e9201492975750bMark Mendellvoid IntrinsicLocationsBuilderX86_64::VisitMathCosh(HInvoke* invoke) { 789a4f1220c1518074db18ca1044e9201492975750bMark Mendell CreateFPToFPCallLocations(arena_, invoke); 790a4f1220c1518074db18ca1044e9201492975750bMark Mendell} 791a4f1220c1518074db18ca1044e9201492975750bMark Mendell 792a4f1220c1518074db18ca1044e9201492975750bMark Mendellvoid IntrinsicCodeGeneratorX86_64::VisitMathCosh(HInvoke* invoke) { 793a4f1220c1518074db18ca1044e9201492975750bMark Mendell GenFPToFPCall(invoke, codegen_, kQuickCosh); 794a4f1220c1518074db18ca1044e9201492975750bMark Mendell} 795a4f1220c1518074db18ca1044e9201492975750bMark Mendell 796a4f1220c1518074db18ca1044e9201492975750bMark Mendellvoid IntrinsicLocationsBuilderX86_64::VisitMathExp(HInvoke* invoke) { 797a4f1220c1518074db18ca1044e9201492975750bMark Mendell CreateFPToFPCallLocations(arena_, invoke); 798a4f1220c1518074db18ca1044e9201492975750bMark Mendell} 799a4f1220c1518074db18ca1044e9201492975750bMark Mendell 800a4f1220c1518074db18ca1044e9201492975750bMark Mendellvoid IntrinsicCodeGeneratorX86_64::VisitMathExp(HInvoke* invoke) { 801a4f1220c1518074db18ca1044e9201492975750bMark Mendell GenFPToFPCall(invoke, codegen_, kQuickExp); 802a4f1220c1518074db18ca1044e9201492975750bMark Mendell} 803a4f1220c1518074db18ca1044e9201492975750bMark Mendell 804a4f1220c1518074db18ca1044e9201492975750bMark Mendellvoid IntrinsicLocationsBuilderX86_64::VisitMathExpm1(HInvoke* invoke) { 805a4f1220c1518074db18ca1044e9201492975750bMark Mendell CreateFPToFPCallLocations(arena_, invoke); 806a4f1220c1518074db18ca1044e9201492975750bMark Mendell} 807a4f1220c1518074db18ca1044e9201492975750bMark Mendell 808a4f1220c1518074db18ca1044e9201492975750bMark Mendellvoid IntrinsicCodeGeneratorX86_64::VisitMathExpm1(HInvoke* invoke) { 809a4f1220c1518074db18ca1044e9201492975750bMark Mendell GenFPToFPCall(invoke, codegen_, kQuickExpm1); 810a4f1220c1518074db18ca1044e9201492975750bMark Mendell} 811a4f1220c1518074db18ca1044e9201492975750bMark Mendell 812a4f1220c1518074db18ca1044e9201492975750bMark Mendellvoid IntrinsicLocationsBuilderX86_64::VisitMathLog(HInvoke* invoke) { 813a4f1220c1518074db18ca1044e9201492975750bMark Mendell CreateFPToFPCallLocations(arena_, invoke); 814a4f1220c1518074db18ca1044e9201492975750bMark Mendell} 815a4f1220c1518074db18ca1044e9201492975750bMark Mendell 816a4f1220c1518074db18ca1044e9201492975750bMark Mendellvoid IntrinsicCodeGeneratorX86_64::VisitMathLog(HInvoke* invoke) { 817a4f1220c1518074db18ca1044e9201492975750bMark Mendell GenFPToFPCall(invoke, codegen_, kQuickLog); 818a4f1220c1518074db18ca1044e9201492975750bMark Mendell} 819a4f1220c1518074db18ca1044e9201492975750bMark Mendell 820a4f1220c1518074db18ca1044e9201492975750bMark Mendellvoid IntrinsicLocationsBuilderX86_64::VisitMathLog10(HInvoke* invoke) { 821a4f1220c1518074db18ca1044e9201492975750bMark Mendell CreateFPToFPCallLocations(arena_, invoke); 822a4f1220c1518074db18ca1044e9201492975750bMark Mendell} 823a4f1220c1518074db18ca1044e9201492975750bMark Mendell 824a4f1220c1518074db18ca1044e9201492975750bMark Mendellvoid IntrinsicCodeGeneratorX86_64::VisitMathLog10(HInvoke* invoke) { 825a4f1220c1518074db18ca1044e9201492975750bMark Mendell GenFPToFPCall(invoke, codegen_, kQuickLog10); 826a4f1220c1518074db18ca1044e9201492975750bMark Mendell} 827a4f1220c1518074db18ca1044e9201492975750bMark Mendell 828a4f1220c1518074db18ca1044e9201492975750bMark Mendellvoid IntrinsicLocationsBuilderX86_64::VisitMathSinh(HInvoke* invoke) { 829a4f1220c1518074db18ca1044e9201492975750bMark Mendell CreateFPToFPCallLocations(arena_, invoke); 830a4f1220c1518074db18ca1044e9201492975750bMark Mendell} 831a4f1220c1518074db18ca1044e9201492975750bMark Mendell 832a4f1220c1518074db18ca1044e9201492975750bMark Mendellvoid IntrinsicCodeGeneratorX86_64::VisitMathSinh(HInvoke* invoke) { 833a4f1220c1518074db18ca1044e9201492975750bMark Mendell GenFPToFPCall(invoke, codegen_, kQuickSinh); 834a4f1220c1518074db18ca1044e9201492975750bMark Mendell} 835a4f1220c1518074db18ca1044e9201492975750bMark Mendell 836a4f1220c1518074db18ca1044e9201492975750bMark Mendellvoid IntrinsicLocationsBuilderX86_64::VisitMathTan(HInvoke* invoke) { 837a4f1220c1518074db18ca1044e9201492975750bMark Mendell CreateFPToFPCallLocations(arena_, invoke); 838a4f1220c1518074db18ca1044e9201492975750bMark Mendell} 839a4f1220c1518074db18ca1044e9201492975750bMark Mendell 840a4f1220c1518074db18ca1044e9201492975750bMark Mendellvoid IntrinsicCodeGeneratorX86_64::VisitMathTan(HInvoke* invoke) { 841a4f1220c1518074db18ca1044e9201492975750bMark Mendell GenFPToFPCall(invoke, codegen_, kQuickTan); 842a4f1220c1518074db18ca1044e9201492975750bMark Mendell} 843a4f1220c1518074db18ca1044e9201492975750bMark Mendell 844a4f1220c1518074db18ca1044e9201492975750bMark Mendellvoid IntrinsicLocationsBuilderX86_64::VisitMathTanh(HInvoke* invoke) { 845a4f1220c1518074db18ca1044e9201492975750bMark Mendell CreateFPToFPCallLocations(arena_, invoke); 846a4f1220c1518074db18ca1044e9201492975750bMark Mendell} 847a4f1220c1518074db18ca1044e9201492975750bMark Mendell 848a4f1220c1518074db18ca1044e9201492975750bMark Mendellvoid IntrinsicCodeGeneratorX86_64::VisitMathTanh(HInvoke* invoke) { 849a4f1220c1518074db18ca1044e9201492975750bMark Mendell GenFPToFPCall(invoke, codegen_, kQuickTanh); 850a4f1220c1518074db18ca1044e9201492975750bMark Mendell} 851a4f1220c1518074db18ca1044e9201492975750bMark Mendell 852a4f1220c1518074db18ca1044e9201492975750bMark Mendellstatic void CreateFPFPToFPCallLocations(ArenaAllocator* arena, 853a4f1220c1518074db18ca1044e9201492975750bMark Mendell HInvoke* invoke) { 854a4f1220c1518074db18ca1044e9201492975750bMark Mendell LocationSummary* locations = new (arena) LocationSummary(invoke, 855a4f1220c1518074db18ca1044e9201492975750bMark Mendell LocationSummary::kCall, 856a4f1220c1518074db18ca1044e9201492975750bMark Mendell kIntrinsified); 857a4f1220c1518074db18ca1044e9201492975750bMark Mendell InvokeRuntimeCallingConvention calling_convention; 858a4f1220c1518074db18ca1044e9201492975750bMark Mendell locations->SetInAt(0, Location::FpuRegisterLocation(calling_convention.GetFpuRegisterAt(0))); 859a4f1220c1518074db18ca1044e9201492975750bMark Mendell locations->SetInAt(1, Location::FpuRegisterLocation(calling_convention.GetFpuRegisterAt(1))); 860a4f1220c1518074db18ca1044e9201492975750bMark Mendell locations->SetOut(Location::FpuRegisterLocation(XMM0)); 861a4f1220c1518074db18ca1044e9201492975750bMark Mendell 862a4f1220c1518074db18ca1044e9201492975750bMark Mendell // We have to ensure that the native code doesn't clobber the XMM registers which are 863a4f1220c1518074db18ca1044e9201492975750bMark Mendell // non-volatile for ART, but volatile for Native calls. This will ensure that they are 864a4f1220c1518074db18ca1044e9201492975750bMark Mendell // saved in the prologue and properly restored. 865a4f1220c1518074db18ca1044e9201492975750bMark Mendell for (auto fp_reg : non_volatile_xmm_regs) { 866a4f1220c1518074db18ca1044e9201492975750bMark Mendell locations->AddTemp(Location::FpuRegisterLocation(fp_reg)); 867a4f1220c1518074db18ca1044e9201492975750bMark Mendell } 868a4f1220c1518074db18ca1044e9201492975750bMark Mendell} 869a4f1220c1518074db18ca1044e9201492975750bMark Mendell 870a4f1220c1518074db18ca1044e9201492975750bMark Mendellvoid IntrinsicLocationsBuilderX86_64::VisitMathAtan2(HInvoke* invoke) { 871a4f1220c1518074db18ca1044e9201492975750bMark Mendell CreateFPFPToFPCallLocations(arena_, invoke); 872a4f1220c1518074db18ca1044e9201492975750bMark Mendell} 873a4f1220c1518074db18ca1044e9201492975750bMark Mendell 874a4f1220c1518074db18ca1044e9201492975750bMark Mendellvoid IntrinsicCodeGeneratorX86_64::VisitMathAtan2(HInvoke* invoke) { 875a4f1220c1518074db18ca1044e9201492975750bMark Mendell GenFPToFPCall(invoke, codegen_, kQuickAtan2); 876a4f1220c1518074db18ca1044e9201492975750bMark Mendell} 877a4f1220c1518074db18ca1044e9201492975750bMark Mendell 878a4f1220c1518074db18ca1044e9201492975750bMark Mendellvoid IntrinsicLocationsBuilderX86_64::VisitMathHypot(HInvoke* invoke) { 879a4f1220c1518074db18ca1044e9201492975750bMark Mendell CreateFPFPToFPCallLocations(arena_, invoke); 880a4f1220c1518074db18ca1044e9201492975750bMark Mendell} 881a4f1220c1518074db18ca1044e9201492975750bMark Mendell 882a4f1220c1518074db18ca1044e9201492975750bMark Mendellvoid IntrinsicCodeGeneratorX86_64::VisitMathHypot(HInvoke* invoke) { 883a4f1220c1518074db18ca1044e9201492975750bMark Mendell GenFPToFPCall(invoke, codegen_, kQuickHypot); 884a4f1220c1518074db18ca1044e9201492975750bMark Mendell} 885a4f1220c1518074db18ca1044e9201492975750bMark Mendell 886a4f1220c1518074db18ca1044e9201492975750bMark Mendellvoid IntrinsicLocationsBuilderX86_64::VisitMathNextAfter(HInvoke* invoke) { 887a4f1220c1518074db18ca1044e9201492975750bMark Mendell CreateFPFPToFPCallLocations(arena_, invoke); 888a4f1220c1518074db18ca1044e9201492975750bMark Mendell} 889a4f1220c1518074db18ca1044e9201492975750bMark Mendell 890a4f1220c1518074db18ca1044e9201492975750bMark Mendellvoid IntrinsicCodeGeneratorX86_64::VisitMathNextAfter(HInvoke* invoke) { 891a4f1220c1518074db18ca1044e9201492975750bMark Mendell GenFPToFPCall(invoke, codegen_, kQuickNextAfter); 892a4f1220c1518074db18ca1044e9201492975750bMark Mendell} 893a4f1220c1518074db18ca1044e9201492975750bMark Mendell 89471fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampevoid IntrinsicLocationsBuilderX86_64::VisitStringCharAt(HInvoke* invoke) { 89571fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe // The inputs plus one temp. 89671fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe LocationSummary* locations = new (arena_) LocationSummary(invoke, 89771fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe LocationSummary::kCallOnSlowPath, 89871fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe kIntrinsified); 89971fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe locations->SetInAt(0, Location::RequiresRegister()); 90071fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe locations->SetInAt(1, Location::RequiresRegister()); 90171fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe locations->SetOut(Location::SameAsFirstInput()); 90271fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe locations->AddTemp(Location::RequiresRegister()); 90371fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe} 90471fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe 90571fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampevoid IntrinsicCodeGeneratorX86_64::VisitStringCharAt(HInvoke* invoke) { 90671fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe LocationSummary* locations = invoke->GetLocations(); 90771fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe 9086bc53a9d884265e0a0b14c4383bef0aa47824e64Mark Mendell // Location of reference to data array. 90971fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe const int32_t value_offset = mirror::String::ValueOffset().Int32Value(); 9106bc53a9d884265e0a0b14c4383bef0aa47824e64Mark Mendell // Location of count. 91171fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe const int32_t count_offset = mirror::String::CountOffset().Int32Value(); 91271fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe 91371fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe CpuRegister obj = locations->InAt(0).AsRegister<CpuRegister>(); 91471fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe CpuRegister idx = locations->InAt(1).AsRegister<CpuRegister>(); 91571fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe CpuRegister out = locations->Out().AsRegister<CpuRegister>(); 91671fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe 91771fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe // TODO: Maybe we can support range check elimination. Overall, though, I think it's not worth 91871fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe // the cost. 91971fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe // TODO: For simplicity, the index parameter is requested in a register, so different from Quick 92071fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe // we will not optimize the code for constants (which would save a register). 92171fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe 92285b62f23fc6dfffe2ddd3ddfa74611666c9ff41dAndreas Gampe SlowPathCode* slow_path = new (GetAllocator()) IntrinsicSlowPathX86_64(invoke); 92371fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe codegen_->AddSlowPath(slow_path); 92471fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe 92571fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe X86_64Assembler* assembler = GetAssembler(); 92671fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe 92771fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe __ cmpl(idx, Address(obj, count_offset)); 928878d58cbaf6b17a9e3dcab790754527f3ebc69e5Andreas Gampe codegen_->MaybeRecordImplicitNullCheck(invoke); 92971fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe __ j(kAboveEqual, slow_path->GetEntryLabel()); 93071fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe 931848f70a3d73833fc1bf3032a9ff6812e429661d9Jeff Hao // out = out[2*idx]. 932848f70a3d73833fc1bf3032a9ff6812e429661d9Jeff Hao __ movzxw(out, Address(out, idx, ScaleFactor::TIMES_2, value_offset)); 93371fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe 93471fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe __ Bind(slow_path->GetExitLabel()); 93571fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe} 93671fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe 9376bc53a9d884265e0a0b14c4383bef0aa47824e64Mark Mendellvoid IntrinsicLocationsBuilderX86_64::VisitSystemArrayCopyChar(HInvoke* invoke) { 9386bc53a9d884265e0a0b14c4383bef0aa47824e64Mark Mendell // Check to see if we have known failures that will cause us to have to bail out 9396bc53a9d884265e0a0b14c4383bef0aa47824e64Mark Mendell // to the runtime, and just generate the runtime call directly. 9406bc53a9d884265e0a0b14c4383bef0aa47824e64Mark Mendell HIntConstant* src_pos = invoke->InputAt(1)->AsIntConstant(); 9416bc53a9d884265e0a0b14c4383bef0aa47824e64Mark Mendell HIntConstant* dest_pos = invoke->InputAt(3)->AsIntConstant(); 9426bc53a9d884265e0a0b14c4383bef0aa47824e64Mark Mendell 9436bc53a9d884265e0a0b14c4383bef0aa47824e64Mark Mendell // The positions must be non-negative. 9446bc53a9d884265e0a0b14c4383bef0aa47824e64Mark Mendell if ((src_pos != nullptr && src_pos->GetValue() < 0) || 9456bc53a9d884265e0a0b14c4383bef0aa47824e64Mark Mendell (dest_pos != nullptr && dest_pos->GetValue() < 0)) { 9466bc53a9d884265e0a0b14c4383bef0aa47824e64Mark Mendell // We will have to fail anyways. 9476bc53a9d884265e0a0b14c4383bef0aa47824e64Mark Mendell return; 9486bc53a9d884265e0a0b14c4383bef0aa47824e64Mark Mendell } 9496bc53a9d884265e0a0b14c4383bef0aa47824e64Mark Mendell 9506bc53a9d884265e0a0b14c4383bef0aa47824e64Mark Mendell // The length must be > 0. 9516bc53a9d884265e0a0b14c4383bef0aa47824e64Mark Mendell HIntConstant* length = invoke->InputAt(4)->AsIntConstant(); 9526bc53a9d884265e0a0b14c4383bef0aa47824e64Mark Mendell if (length != nullptr) { 9536bc53a9d884265e0a0b14c4383bef0aa47824e64Mark Mendell int32_t len = length->GetValue(); 9546bc53a9d884265e0a0b14c4383bef0aa47824e64Mark Mendell if (len < 0) { 9556bc53a9d884265e0a0b14c4383bef0aa47824e64Mark Mendell // Just call as normal. 9566bc53a9d884265e0a0b14c4383bef0aa47824e64Mark Mendell return; 9576bc53a9d884265e0a0b14c4383bef0aa47824e64Mark Mendell } 9586bc53a9d884265e0a0b14c4383bef0aa47824e64Mark Mendell } 9596bc53a9d884265e0a0b14c4383bef0aa47824e64Mark Mendell 9606bc53a9d884265e0a0b14c4383bef0aa47824e64Mark Mendell LocationSummary* locations = new (arena_) LocationSummary(invoke, 9616bc53a9d884265e0a0b14c4383bef0aa47824e64Mark Mendell LocationSummary::kCallOnSlowPath, 9626bc53a9d884265e0a0b14c4383bef0aa47824e64Mark Mendell kIntrinsified); 963ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray // arraycopy(Object src, int src_pos, Object dest, int dest_pos, int length). 9646bc53a9d884265e0a0b14c4383bef0aa47824e64Mark Mendell locations->SetInAt(0, Location::RequiresRegister()); 9656bc53a9d884265e0a0b14c4383bef0aa47824e64Mark Mendell locations->SetInAt(1, Location::RegisterOrConstant(invoke->InputAt(1))); 9666bc53a9d884265e0a0b14c4383bef0aa47824e64Mark Mendell locations->SetInAt(2, Location::RequiresRegister()); 9676bc53a9d884265e0a0b14c4383bef0aa47824e64Mark Mendell locations->SetInAt(3, Location::RegisterOrConstant(invoke->InputAt(3))); 9686bc53a9d884265e0a0b14c4383bef0aa47824e64Mark Mendell locations->SetInAt(4, Location::RegisterOrConstant(invoke->InputAt(4))); 9696bc53a9d884265e0a0b14c4383bef0aa47824e64Mark Mendell 9706bc53a9d884265e0a0b14c4383bef0aa47824e64Mark Mendell // And we need some temporaries. We will use REP MOVSW, so we need fixed registers. 9716bc53a9d884265e0a0b14c4383bef0aa47824e64Mark Mendell locations->AddTemp(Location::RegisterLocation(RSI)); 9726bc53a9d884265e0a0b14c4383bef0aa47824e64Mark Mendell locations->AddTemp(Location::RegisterLocation(RDI)); 9736bc53a9d884265e0a0b14c4383bef0aa47824e64Mark Mendell locations->AddTemp(Location::RegisterLocation(RCX)); 9746bc53a9d884265e0a0b14c4383bef0aa47824e64Mark Mendell} 9756bc53a9d884265e0a0b14c4383bef0aa47824e64Mark Mendell 9766bc53a9d884265e0a0b14c4383bef0aa47824e64Mark Mendellstatic void CheckPosition(X86_64Assembler* assembler, 9776bc53a9d884265e0a0b14c4383bef0aa47824e64Mark Mendell Location pos, 9786bc53a9d884265e0a0b14c4383bef0aa47824e64Mark Mendell CpuRegister input, 979ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray Location length, 98085b62f23fc6dfffe2ddd3ddfa74611666c9ff41dAndreas Gampe SlowPathCode* slow_path, 9816bc53a9d884265e0a0b14c4383bef0aa47824e64Mark Mendell CpuRegister input_len, 982ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray CpuRegister temp, 983ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray bool length_is_input_length = false) { 984ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray // Where is the length in the Array? 9856bc53a9d884265e0a0b14c4383bef0aa47824e64Mark Mendell const uint32_t length_offset = mirror::Array::LengthOffset().Uint32Value(); 9866bc53a9d884265e0a0b14c4383bef0aa47824e64Mark Mendell 9876bc53a9d884265e0a0b14c4383bef0aa47824e64Mark Mendell if (pos.IsConstant()) { 9886bc53a9d884265e0a0b14c4383bef0aa47824e64Mark Mendell int32_t pos_const = pos.GetConstant()->AsIntConstant()->GetValue(); 9896bc53a9d884265e0a0b14c4383bef0aa47824e64Mark Mendell if (pos_const == 0) { 990ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray if (!length_is_input_length) { 991ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray // Check that length(input) >= length. 992ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray if (length.IsConstant()) { 993ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray __ cmpl(Address(input, length_offset), 994ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray Immediate(length.GetConstant()->AsIntConstant()->GetValue())); 995ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray } else { 996ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray __ cmpl(Address(input, length_offset), length.AsRegister<CpuRegister>()); 997ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray } 998ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray __ j(kLess, slow_path->GetEntryLabel()); 999ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray } 10006bc53a9d884265e0a0b14c4383bef0aa47824e64Mark Mendell } else { 10016bc53a9d884265e0a0b14c4383bef0aa47824e64Mark Mendell // Check that length(input) >= pos. 10026bc53a9d884265e0a0b14c4383bef0aa47824e64Mark Mendell __ movl(input_len, Address(input, length_offset)); 10036bc53a9d884265e0a0b14c4383bef0aa47824e64Mark Mendell __ cmpl(input_len, Immediate(pos_const)); 10046bc53a9d884265e0a0b14c4383bef0aa47824e64Mark Mendell __ j(kLess, slow_path->GetEntryLabel()); 10056bc53a9d884265e0a0b14c4383bef0aa47824e64Mark Mendell 10066bc53a9d884265e0a0b14c4383bef0aa47824e64Mark Mendell // Check that (length(input) - pos) >= length. 10076bc53a9d884265e0a0b14c4383bef0aa47824e64Mark Mendell __ leal(temp, Address(input_len, -pos_const)); 1008ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray if (length.IsConstant()) { 1009ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray __ cmpl(temp, Immediate(length.GetConstant()->AsIntConstant()->GetValue())); 1010ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray } else { 1011ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray __ cmpl(temp, length.AsRegister<CpuRegister>()); 1012ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray } 10136bc53a9d884265e0a0b14c4383bef0aa47824e64Mark Mendell __ j(kLess, slow_path->GetEntryLabel()); 10146bc53a9d884265e0a0b14c4383bef0aa47824e64Mark Mendell } 1015ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray } else if (length_is_input_length) { 1016ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray // The only way the copy can succeed is if pos is zero. 1017ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray CpuRegister pos_reg = pos.AsRegister<CpuRegister>(); 1018ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray __ testl(pos_reg, pos_reg); 1019ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray __ j(kNotEqual, slow_path->GetEntryLabel()); 10206bc53a9d884265e0a0b14c4383bef0aa47824e64Mark Mendell } else { 10216bc53a9d884265e0a0b14c4383bef0aa47824e64Mark Mendell // Check that pos >= 0. 10226bc53a9d884265e0a0b14c4383bef0aa47824e64Mark Mendell CpuRegister pos_reg = pos.AsRegister<CpuRegister>(); 10236bc53a9d884265e0a0b14c4383bef0aa47824e64Mark Mendell __ testl(pos_reg, pos_reg); 10246bc53a9d884265e0a0b14c4383bef0aa47824e64Mark Mendell __ j(kLess, slow_path->GetEntryLabel()); 10256bc53a9d884265e0a0b14c4383bef0aa47824e64Mark Mendell 10266bc53a9d884265e0a0b14c4383bef0aa47824e64Mark Mendell // Check that pos <= length(input). 10276bc53a9d884265e0a0b14c4383bef0aa47824e64Mark Mendell __ cmpl(Address(input, length_offset), pos_reg); 10286bc53a9d884265e0a0b14c4383bef0aa47824e64Mark Mendell __ j(kLess, slow_path->GetEntryLabel()); 10296bc53a9d884265e0a0b14c4383bef0aa47824e64Mark Mendell 10306bc53a9d884265e0a0b14c4383bef0aa47824e64Mark Mendell // Check that (length(input) - pos) >= length. 10316bc53a9d884265e0a0b14c4383bef0aa47824e64Mark Mendell __ movl(temp, Address(input, length_offset)); 10326bc53a9d884265e0a0b14c4383bef0aa47824e64Mark Mendell __ subl(temp, pos_reg); 1033ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray if (length.IsConstant()) { 1034ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray __ cmpl(temp, Immediate(length.GetConstant()->AsIntConstant()->GetValue())); 1035ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray } else { 1036ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray __ cmpl(temp, length.AsRegister<CpuRegister>()); 1037ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray } 10386bc53a9d884265e0a0b14c4383bef0aa47824e64Mark Mendell __ j(kLess, slow_path->GetEntryLabel()); 10396bc53a9d884265e0a0b14c4383bef0aa47824e64Mark Mendell } 10406bc53a9d884265e0a0b14c4383bef0aa47824e64Mark Mendell} 10416bc53a9d884265e0a0b14c4383bef0aa47824e64Mark Mendell 10426bc53a9d884265e0a0b14c4383bef0aa47824e64Mark Mendellvoid IntrinsicCodeGeneratorX86_64::VisitSystemArrayCopyChar(HInvoke* invoke) { 10436bc53a9d884265e0a0b14c4383bef0aa47824e64Mark Mendell X86_64Assembler* assembler = GetAssembler(); 10446bc53a9d884265e0a0b14c4383bef0aa47824e64Mark Mendell LocationSummary* locations = invoke->GetLocations(); 10456bc53a9d884265e0a0b14c4383bef0aa47824e64Mark Mendell 10466bc53a9d884265e0a0b14c4383bef0aa47824e64Mark Mendell CpuRegister src = locations->InAt(0).AsRegister<CpuRegister>(); 1047ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray Location src_pos = locations->InAt(1); 10486bc53a9d884265e0a0b14c4383bef0aa47824e64Mark Mendell CpuRegister dest = locations->InAt(2).AsRegister<CpuRegister>(); 1049ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray Location dest_pos = locations->InAt(3); 10506bc53a9d884265e0a0b14c4383bef0aa47824e64Mark Mendell Location length = locations->InAt(4); 10516bc53a9d884265e0a0b14c4383bef0aa47824e64Mark Mendell 10526bc53a9d884265e0a0b14c4383bef0aa47824e64Mark Mendell // Temporaries that we need for MOVSW. 10536bc53a9d884265e0a0b14c4383bef0aa47824e64Mark Mendell CpuRegister src_base = locations->GetTemp(0).AsRegister<CpuRegister>(); 10546bc53a9d884265e0a0b14c4383bef0aa47824e64Mark Mendell DCHECK_EQ(src_base.AsRegister(), RSI); 10556bc53a9d884265e0a0b14c4383bef0aa47824e64Mark Mendell CpuRegister dest_base = locations->GetTemp(1).AsRegister<CpuRegister>(); 10566bc53a9d884265e0a0b14c4383bef0aa47824e64Mark Mendell DCHECK_EQ(dest_base.AsRegister(), RDI); 10576bc53a9d884265e0a0b14c4383bef0aa47824e64Mark Mendell CpuRegister count = locations->GetTemp(2).AsRegister<CpuRegister>(); 10586bc53a9d884265e0a0b14c4383bef0aa47824e64Mark Mendell DCHECK_EQ(count.AsRegister(), RCX); 10596bc53a9d884265e0a0b14c4383bef0aa47824e64Mark Mendell 106085b62f23fc6dfffe2ddd3ddfa74611666c9ff41dAndreas Gampe SlowPathCode* slow_path = new (GetAllocator()) IntrinsicSlowPathX86_64(invoke); 10616bc53a9d884265e0a0b14c4383bef0aa47824e64Mark Mendell codegen_->AddSlowPath(slow_path); 10626bc53a9d884265e0a0b14c4383bef0aa47824e64Mark Mendell 10636bc53a9d884265e0a0b14c4383bef0aa47824e64Mark Mendell // Bail out if the source and destination are the same. 10646bc53a9d884265e0a0b14c4383bef0aa47824e64Mark Mendell __ cmpl(src, dest); 10656bc53a9d884265e0a0b14c4383bef0aa47824e64Mark Mendell __ j(kEqual, slow_path->GetEntryLabel()); 10666bc53a9d884265e0a0b14c4383bef0aa47824e64Mark Mendell 10676bc53a9d884265e0a0b14c4383bef0aa47824e64Mark Mendell // Bail out if the source is null. 10686bc53a9d884265e0a0b14c4383bef0aa47824e64Mark Mendell __ testl(src, src); 10696bc53a9d884265e0a0b14c4383bef0aa47824e64Mark Mendell __ j(kEqual, slow_path->GetEntryLabel()); 10706bc53a9d884265e0a0b14c4383bef0aa47824e64Mark Mendell 10716bc53a9d884265e0a0b14c4383bef0aa47824e64Mark Mendell // Bail out if the destination is null. 10726bc53a9d884265e0a0b14c4383bef0aa47824e64Mark Mendell __ testl(dest, dest); 10736bc53a9d884265e0a0b14c4383bef0aa47824e64Mark Mendell __ j(kEqual, slow_path->GetEntryLabel()); 10746bc53a9d884265e0a0b14c4383bef0aa47824e64Mark Mendell 10756bc53a9d884265e0a0b14c4383bef0aa47824e64Mark Mendell // If the length is negative, bail out. 10766bc53a9d884265e0a0b14c4383bef0aa47824e64Mark Mendell // We have already checked in the LocationsBuilder for the constant case. 10776bc53a9d884265e0a0b14c4383bef0aa47824e64Mark Mendell if (!length.IsConstant()) { 10786bc53a9d884265e0a0b14c4383bef0aa47824e64Mark Mendell __ testl(length.AsRegister<CpuRegister>(), length.AsRegister<CpuRegister>()); 10796bc53a9d884265e0a0b14c4383bef0aa47824e64Mark Mendell __ j(kLess, slow_path->GetEntryLabel()); 10806bc53a9d884265e0a0b14c4383bef0aa47824e64Mark Mendell } 10816bc53a9d884265e0a0b14c4383bef0aa47824e64Mark Mendell 1082ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray // Validity checks: source. 1083ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray CheckPosition(assembler, src_pos, src, length, slow_path, src_base, dest_base); 1084ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray 1085ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray // Validity checks: dest. 1086ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray CheckPosition(assembler, dest_pos, dest, length, slow_path, src_base, dest_base); 1087ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray 10886bc53a9d884265e0a0b14c4383bef0aa47824e64Mark Mendell // We need the count in RCX. 10896bc53a9d884265e0a0b14c4383bef0aa47824e64Mark Mendell if (length.IsConstant()) { 10906bc53a9d884265e0a0b14c4383bef0aa47824e64Mark Mendell __ movl(count, Immediate(length.GetConstant()->AsIntConstant()->GetValue())); 10916bc53a9d884265e0a0b14c4383bef0aa47824e64Mark Mendell } else { 10926bc53a9d884265e0a0b14c4383bef0aa47824e64Mark Mendell __ movl(count, length.AsRegister<CpuRegister>()); 10936bc53a9d884265e0a0b14c4383bef0aa47824e64Mark Mendell } 10946bc53a9d884265e0a0b14c4383bef0aa47824e64Mark Mendell 10956bc53a9d884265e0a0b14c4383bef0aa47824e64Mark Mendell // Okay, everything checks out. Finally time to do the copy. 10966bc53a9d884265e0a0b14c4383bef0aa47824e64Mark Mendell // Check assumption that sizeof(Char) is 2 (used in scaling below). 10976bc53a9d884265e0a0b14c4383bef0aa47824e64Mark Mendell const size_t char_size = Primitive::ComponentSize(Primitive::kPrimChar); 10986bc53a9d884265e0a0b14c4383bef0aa47824e64Mark Mendell DCHECK_EQ(char_size, 2u); 10996bc53a9d884265e0a0b14c4383bef0aa47824e64Mark Mendell 11006bc53a9d884265e0a0b14c4383bef0aa47824e64Mark Mendell const uint32_t data_offset = mirror::Array::DataOffset(char_size).Uint32Value(); 11016bc53a9d884265e0a0b14c4383bef0aa47824e64Mark Mendell 1102ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray if (src_pos.IsConstant()) { 1103ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray int32_t src_pos_const = src_pos.GetConstant()->AsIntConstant()->GetValue(); 1104ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray __ leal(src_base, Address(src, char_size * src_pos_const + data_offset)); 11056bc53a9d884265e0a0b14c4383bef0aa47824e64Mark Mendell } else { 1106ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray __ leal(src_base, Address(src, src_pos.AsRegister<CpuRegister>(), 11076bc53a9d884265e0a0b14c4383bef0aa47824e64Mark Mendell ScaleFactor::TIMES_2, data_offset)); 11086bc53a9d884265e0a0b14c4383bef0aa47824e64Mark Mendell } 1109ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray if (dest_pos.IsConstant()) { 1110ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray int32_t dest_pos_const = dest_pos.GetConstant()->AsIntConstant()->GetValue(); 1111ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray __ leal(dest_base, Address(dest, char_size * dest_pos_const + data_offset)); 11126bc53a9d884265e0a0b14c4383bef0aa47824e64Mark Mendell } else { 1113ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray __ leal(dest_base, Address(dest, dest_pos.AsRegister<CpuRegister>(), 11146bc53a9d884265e0a0b14c4383bef0aa47824e64Mark Mendell ScaleFactor::TIMES_2, data_offset)); 11156bc53a9d884265e0a0b14c4383bef0aa47824e64Mark Mendell } 11166bc53a9d884265e0a0b14c4383bef0aa47824e64Mark Mendell 11176bc53a9d884265e0a0b14c4383bef0aa47824e64Mark Mendell // Do the move. 11186bc53a9d884265e0a0b14c4383bef0aa47824e64Mark Mendell __ rep_movsw(); 11196bc53a9d884265e0a0b14c4383bef0aa47824e64Mark Mendell 11206bc53a9d884265e0a0b14c4383bef0aa47824e64Mark Mendell __ Bind(slow_path->GetExitLabel()); 11216bc53a9d884265e0a0b14c4383bef0aa47824e64Mark Mendell} 11226bc53a9d884265e0a0b14c4383bef0aa47824e64Mark Mendell 1123ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray 1124ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffrayvoid IntrinsicLocationsBuilderX86_64::VisitSystemArrayCopy(HInvoke* invoke) { 11255bd05a5c9492189ec28edaf6396d6a39ddf03367Nicolas Geoffray CodeGenerator::CreateSystemArrayCopyLocationSummary(invoke); 1126ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray} 1127ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray 11280d5a281c671444bfa75d63caf1427a8c0e6e1177Roland Levillain// TODO: Implement read barriers in the SystemArrayCopy intrinsic. 11290d5a281c671444bfa75d63caf1427a8c0e6e1177Roland Levillain// Note that this code path is not used (yet) because we do not 11300d5a281c671444bfa75d63caf1427a8c0e6e1177Roland Levillain// intrinsify methods that can go into the IntrinsicSlowPathX86_64 11310d5a281c671444bfa75d63caf1427a8c0e6e1177Roland Levillain// slow path. 1132ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffrayvoid IntrinsicCodeGeneratorX86_64::VisitSystemArrayCopy(HInvoke* invoke) { 1133ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray X86_64Assembler* assembler = GetAssembler(); 1134ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray LocationSummary* locations = invoke->GetLocations(); 1135ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray 1136ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray uint32_t class_offset = mirror::Object::ClassOffset().Int32Value(); 1137ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray uint32_t super_offset = mirror::Class::SuperClassOffset().Int32Value(); 1138ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray uint32_t component_offset = mirror::Class::ComponentTypeOffset().Int32Value(); 1139ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray uint32_t primitive_offset = mirror::Class::PrimitiveTypeOffset().Int32Value(); 1140ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray 1141ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray CpuRegister src = locations->InAt(0).AsRegister<CpuRegister>(); 1142ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray Location src_pos = locations->InAt(1); 1143ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray CpuRegister dest = locations->InAt(2).AsRegister<CpuRegister>(); 1144ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray Location dest_pos = locations->InAt(3); 1145ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray Location length = locations->InAt(4); 1146ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray CpuRegister temp1 = locations->GetTemp(0).AsRegister<CpuRegister>(); 1147ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray CpuRegister temp2 = locations->GetTemp(1).AsRegister<CpuRegister>(); 1148ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray CpuRegister temp3 = locations->GetTemp(2).AsRegister<CpuRegister>(); 1149ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray 1150ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray SlowPathCode* slow_path = new (GetAllocator()) IntrinsicSlowPathX86_64(invoke); 1151ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray codegen_->AddSlowPath(slow_path); 1152ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray 1153ebea3d2cce6aa34216502bb6b83d155d4c92e4ffRoland Levillain NearLabel conditions_on_positions_validated; 1154ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray SystemArrayCopyOptimizations optimizations(invoke); 1155ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray 1156ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray // If source and destination are the same, we go to slow path if we need to do 1157ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray // forward copying. 1158ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray if (src_pos.IsConstant()) { 1159ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray int32_t src_pos_constant = src_pos.GetConstant()->AsIntConstant()->GetValue(); 1160ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray if (dest_pos.IsConstant()) { 1161b198b013ae7bd2da85e007414fc028cd51a13883Nicolas Geoffray int32_t dest_pos_constant = dest_pos.GetConstant()->AsIntConstant()->GetValue(); 1162b198b013ae7bd2da85e007414fc028cd51a13883Nicolas Geoffray if (optimizations.GetDestinationIsSource()) { 1163b198b013ae7bd2da85e007414fc028cd51a13883Nicolas Geoffray // Checked when building locations. 1164b198b013ae7bd2da85e007414fc028cd51a13883Nicolas Geoffray DCHECK_GE(src_pos_constant, dest_pos_constant); 1165b198b013ae7bd2da85e007414fc028cd51a13883Nicolas Geoffray } else if (src_pos_constant < dest_pos_constant) { 1166b198b013ae7bd2da85e007414fc028cd51a13883Nicolas Geoffray __ cmpl(src, dest); 1167b198b013ae7bd2da85e007414fc028cd51a13883Nicolas Geoffray __ j(kEqual, slow_path->GetEntryLabel()); 1168b198b013ae7bd2da85e007414fc028cd51a13883Nicolas Geoffray } 1169ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray } else { 1170ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray if (!optimizations.GetDestinationIsSource()) { 1171b198b013ae7bd2da85e007414fc028cd51a13883Nicolas Geoffray __ cmpl(src, dest); 1172ebea3d2cce6aa34216502bb6b83d155d4c92e4ffRoland Levillain __ j(kNotEqual, &conditions_on_positions_validated); 1173ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray } 1174ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray __ cmpl(dest_pos.AsRegister<CpuRegister>(), Immediate(src_pos_constant)); 1175ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray __ j(kGreater, slow_path->GetEntryLabel()); 1176ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray } 1177ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray } else { 1178ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray if (!optimizations.GetDestinationIsSource()) { 1179b198b013ae7bd2da85e007414fc028cd51a13883Nicolas Geoffray __ cmpl(src, dest); 1180ebea3d2cce6aa34216502bb6b83d155d4c92e4ffRoland Levillain __ j(kNotEqual, &conditions_on_positions_validated); 1181ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray } 1182ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray if (dest_pos.IsConstant()) { 1183ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray int32_t dest_pos_constant = dest_pos.GetConstant()->AsIntConstant()->GetValue(); 1184ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray __ cmpl(src_pos.AsRegister<CpuRegister>(), Immediate(dest_pos_constant)); 1185ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray __ j(kLess, slow_path->GetEntryLabel()); 1186ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray } else { 1187ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray __ cmpl(src_pos.AsRegister<CpuRegister>(), dest_pos.AsRegister<CpuRegister>()); 1188ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray __ j(kLess, slow_path->GetEntryLabel()); 1189ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray } 1190ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray } 1191ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray 1192ebea3d2cce6aa34216502bb6b83d155d4c92e4ffRoland Levillain __ Bind(&conditions_on_positions_validated); 1193ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray 1194ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray if (!optimizations.GetSourceIsNotNull()) { 1195ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray // Bail out if the source is null. 1196ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray __ testl(src, src); 1197ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray __ j(kEqual, slow_path->GetEntryLabel()); 1198ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray } 1199ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray 1200ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray if (!optimizations.GetDestinationIsNotNull() && !optimizations.GetDestinationIsSource()) { 1201ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray // Bail out if the destination is null. 1202ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray __ testl(dest, dest); 1203ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray __ j(kEqual, slow_path->GetEntryLabel()); 1204ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray } 1205ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray 1206ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray // If the length is negative, bail out. 1207ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray // We have already checked in the LocationsBuilder for the constant case. 1208ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray if (!length.IsConstant() && 1209ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray !optimizations.GetCountIsSourceLength() && 1210ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray !optimizations.GetCountIsDestinationLength()) { 1211ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray __ testl(length.AsRegister<CpuRegister>(), length.AsRegister<CpuRegister>()); 1212ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray __ j(kLess, slow_path->GetEntryLabel()); 1213ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray } 1214ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray 1215ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray // Validity checks: source. 1216ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray CheckPosition(assembler, 1217ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray src_pos, 1218ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray src, 1219ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray length, 1220ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray slow_path, 1221ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray temp1, 1222ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray temp2, 1223ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray optimizations.GetCountIsSourceLength()); 1224ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray 1225ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray // Validity checks: dest. 1226ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray CheckPosition(assembler, 1227ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray dest_pos, 1228ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray dest, 1229ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray length, 1230ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray slow_path, 1231ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray temp1, 1232ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray temp2, 1233ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray optimizations.GetCountIsDestinationLength()); 1234ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray 1235ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray if (!optimizations.GetDoesNotNeedTypeCheck()) { 1236ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray // Check whether all elements of the source array are assignable to the component 1237ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray // type of the destination array. We do two checks: the classes are the same, 1238ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray // or the destination is Object[]. If none of these checks succeed, we go to the 1239ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray // slow path. 1240ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray __ movl(temp1, Address(dest, class_offset)); 1241ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray __ movl(temp2, Address(src, class_offset)); 1242ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray bool did_unpoison = false; 1243ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray if (!optimizations.GetDestinationIsNonPrimitiveArray() || 1244ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray !optimizations.GetSourceIsNonPrimitiveArray()) { 1245ebea3d2cce6aa34216502bb6b83d155d4c92e4ffRoland Levillain // One or two of the references need to be unpoisoned. Unpoison them 1246ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray // both to make the identity check valid. 1247ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray __ MaybeUnpoisonHeapReference(temp1); 1248ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray __ MaybeUnpoisonHeapReference(temp2); 1249ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray did_unpoison = true; 1250ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray } 1251ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray 1252ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray if (!optimizations.GetDestinationIsNonPrimitiveArray()) { 1253ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray // Bail out if the destination is not a non primitive array. 1254ebea3d2cce6aa34216502bb6b83d155d4c92e4ffRoland Levillain // /* HeapReference<Class> */ TMP = temp1->component_type_ 1255ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray __ movl(CpuRegister(TMP), Address(temp1, component_offset)); 1256ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray __ testl(CpuRegister(TMP), CpuRegister(TMP)); 1257ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray __ j(kEqual, slow_path->GetEntryLabel()); 1258ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray __ MaybeUnpoisonHeapReference(CpuRegister(TMP)); 1259ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray __ cmpw(Address(CpuRegister(TMP), primitive_offset), Immediate(Primitive::kPrimNot)); 1260ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray __ j(kNotEqual, slow_path->GetEntryLabel()); 1261ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray } 1262ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray 1263ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray if (!optimizations.GetSourceIsNonPrimitiveArray()) { 1264ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray // Bail out if the source is not a non primitive array. 1265ebea3d2cce6aa34216502bb6b83d155d4c92e4ffRoland Levillain // /* HeapReference<Class> */ TMP = temp2->component_type_ 1266ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray __ movl(CpuRegister(TMP), Address(temp2, component_offset)); 1267ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray __ testl(CpuRegister(TMP), CpuRegister(TMP)); 1268ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray __ j(kEqual, slow_path->GetEntryLabel()); 1269ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray __ MaybeUnpoisonHeapReference(CpuRegister(TMP)); 1270ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray __ cmpw(Address(CpuRegister(TMP), primitive_offset), Immediate(Primitive::kPrimNot)); 1271ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray __ j(kNotEqual, slow_path->GetEntryLabel()); 1272ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray } 1273ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray 1274ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray __ cmpl(temp1, temp2); 1275ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray 1276ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray if (optimizations.GetDestinationIsTypedObjectArray()) { 1277ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray NearLabel do_copy; 1278ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray __ j(kEqual, &do_copy); 1279ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray if (!did_unpoison) { 1280ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray __ MaybeUnpoisonHeapReference(temp1); 1281ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray } 1282ebea3d2cce6aa34216502bb6b83d155d4c92e4ffRoland Levillain // /* HeapReference<Class> */ temp1 = temp1->component_type_ 1283ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray __ movl(temp1, Address(temp1, component_offset)); 1284ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray __ MaybeUnpoisonHeapReference(temp1); 1285ebea3d2cce6aa34216502bb6b83d155d4c92e4ffRoland Levillain // /* HeapReference<Class> */ temp1 = temp1->super_class_ 1286ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray __ movl(temp1, Address(temp1, super_offset)); 1287ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray // No need to unpoison the result, we're comparing against null. 1288ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray __ testl(temp1, temp1); 1289ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray __ j(kNotEqual, slow_path->GetEntryLabel()); 1290ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray __ Bind(&do_copy); 1291ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray } else { 1292ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray __ j(kNotEqual, slow_path->GetEntryLabel()); 1293ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray } 1294ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray } else if (!optimizations.GetSourceIsNonPrimitiveArray()) { 1295ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray DCHECK(optimizations.GetDestinationIsNonPrimitiveArray()); 1296ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray // Bail out if the source is not a non primitive array. 1297ebea3d2cce6aa34216502bb6b83d155d4c92e4ffRoland Levillain // /* HeapReference<Class> */ temp1 = src->klass_ 1298ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray __ movl(temp1, Address(src, class_offset)); 1299ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray __ MaybeUnpoisonHeapReference(temp1); 1300ebea3d2cce6aa34216502bb6b83d155d4c92e4ffRoland Levillain // /* HeapReference<Class> */ TMP = temp1->component_type_ 1301ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray __ movl(CpuRegister(TMP), Address(temp1, component_offset)); 1302ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray __ testl(CpuRegister(TMP), CpuRegister(TMP)); 1303ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray __ j(kEqual, slow_path->GetEntryLabel()); 1304ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray __ MaybeUnpoisonHeapReference(CpuRegister(TMP)); 1305ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray __ cmpw(Address(CpuRegister(TMP), primitive_offset), Immediate(Primitive::kPrimNot)); 1306ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray __ j(kNotEqual, slow_path->GetEntryLabel()); 1307ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray } 1308ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray 1309ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray // Compute base source address, base destination address, and end source address. 1310ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray 1311ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray uint32_t element_size = sizeof(int32_t); 1312ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray uint32_t offset = mirror::Array::DataOffset(element_size).Uint32Value(); 1313ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray if (src_pos.IsConstant()) { 1314ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray int32_t constant = src_pos.GetConstant()->AsIntConstant()->GetValue(); 1315ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray __ leal(temp1, Address(src, element_size * constant + offset)); 1316ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray } else { 1317ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray __ leal(temp1, Address(src, src_pos.AsRegister<CpuRegister>(), ScaleFactor::TIMES_4, offset)); 1318ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray } 1319ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray 1320ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray if (dest_pos.IsConstant()) { 1321ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray int32_t constant = dest_pos.GetConstant()->AsIntConstant()->GetValue(); 1322ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray __ leal(temp2, Address(dest, element_size * constant + offset)); 1323ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray } else { 1324ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray __ leal(temp2, Address(dest, dest_pos.AsRegister<CpuRegister>(), ScaleFactor::TIMES_4, offset)); 1325ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray } 1326ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray 1327ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray if (length.IsConstant()) { 1328ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray int32_t constant = length.GetConstant()->AsIntConstant()->GetValue(); 1329ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray __ leal(temp3, Address(temp1, element_size * constant)); 1330ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray } else { 1331ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray __ leal(temp3, Address(temp1, length.AsRegister<CpuRegister>(), ScaleFactor::TIMES_4, 0)); 1332ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray } 1333ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray 1334ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray // Iterate over the arrays and do a raw copy of the objects. We don't need to 1335ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray // poison/unpoison, nor do any read barrier as the next uses of the destination 1336ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray // array will do it. 1337ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray NearLabel loop, done; 1338ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray __ cmpl(temp1, temp3); 1339ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray __ j(kEqual, &done); 1340ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray __ Bind(&loop); 1341ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray __ movl(CpuRegister(TMP), Address(temp1, 0)); 1342ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray __ movl(Address(temp2, 0), CpuRegister(TMP)); 1343ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray __ addl(temp1, Immediate(element_size)); 1344ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray __ addl(temp2, Immediate(element_size)); 1345ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray __ cmpl(temp1, temp3); 1346ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray __ j(kNotEqual, &loop); 1347ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray __ Bind(&done); 1348ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray 1349ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray // We only need one card marking on the destination array. 1350ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray codegen_->MarkGCCard(temp1, 1351ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray temp2, 1352ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray dest, 1353ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray CpuRegister(kNoRegister), 1354bf84a3d2aa29c0975b4ac0f6f983d56724b2cc57Roland Levillain /* value_can_be_null */ false); 1355ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray 1356ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray __ Bind(slow_path->GetExitLabel()); 1357ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray} 1358ee3cf0731d0ef0787bc2947c8e3ca432b513956bNicolas Geoffray 1359d75948ac93a4a317feaf136cae78823071234ba5Nicolas Geoffrayvoid IntrinsicLocationsBuilderX86_64::VisitStringCompareTo(HInvoke* invoke) { 1360d75948ac93a4a317feaf136cae78823071234ba5Nicolas Geoffray LocationSummary* locations = new (arena_) LocationSummary(invoke, 1361d75948ac93a4a317feaf136cae78823071234ba5Nicolas Geoffray LocationSummary::kCall, 1362d75948ac93a4a317feaf136cae78823071234ba5Nicolas Geoffray kIntrinsified); 1363d75948ac93a4a317feaf136cae78823071234ba5Nicolas Geoffray InvokeRuntimeCallingConvention calling_convention; 1364d75948ac93a4a317feaf136cae78823071234ba5Nicolas Geoffray locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(0))); 1365d75948ac93a4a317feaf136cae78823071234ba5Nicolas Geoffray locations->SetInAt(1, Location::RegisterLocation(calling_convention.GetRegisterAt(1))); 1366d75948ac93a4a317feaf136cae78823071234ba5Nicolas Geoffray locations->SetOut(Location::RegisterLocation(RAX)); 1367d75948ac93a4a317feaf136cae78823071234ba5Nicolas Geoffray} 1368d75948ac93a4a317feaf136cae78823071234ba5Nicolas Geoffray 1369d75948ac93a4a317feaf136cae78823071234ba5Nicolas Geoffrayvoid IntrinsicCodeGeneratorX86_64::VisitStringCompareTo(HInvoke* invoke) { 1370d75948ac93a4a317feaf136cae78823071234ba5Nicolas Geoffray X86_64Assembler* assembler = GetAssembler(); 1371d75948ac93a4a317feaf136cae78823071234ba5Nicolas Geoffray LocationSummary* locations = invoke->GetLocations(); 1372d75948ac93a4a317feaf136cae78823071234ba5Nicolas Geoffray 1373512e04d1ea7fb33e3992715fe55be8a834d4a79cNicolas Geoffray // Note that the null check must have been done earlier. 1374641547a5f18ca2ea54469cceadcfef64f132e5e0Calin Juravle DCHECK(!invoke->CanDoImplicitNullCheckOn(invoke->InputAt(0))); 1375d75948ac93a4a317feaf136cae78823071234ba5Nicolas Geoffray 1376d75948ac93a4a317feaf136cae78823071234ba5Nicolas Geoffray CpuRegister argument = locations->InAt(1).AsRegister<CpuRegister>(); 1377d75948ac93a4a317feaf136cae78823071234ba5Nicolas Geoffray __ testl(argument, argument); 137885b62f23fc6dfffe2ddd3ddfa74611666c9ff41dAndreas Gampe SlowPathCode* slow_path = new (GetAllocator()) IntrinsicSlowPathX86_64(invoke); 1379d75948ac93a4a317feaf136cae78823071234ba5Nicolas Geoffray codegen_->AddSlowPath(slow_path); 1380d75948ac93a4a317feaf136cae78823071234ba5Nicolas Geoffray __ j(kEqual, slow_path->GetEntryLabel()); 1381d75948ac93a4a317feaf136cae78823071234ba5Nicolas Geoffray 1382bf84a3d2aa29c0975b4ac0f6f983d56724b2cc57Roland Levillain __ gs()->call(Address::Absolute(QUICK_ENTRYPOINT_OFFSET(kX86_64WordSize, pStringCompareTo), 1383bf84a3d2aa29c0975b4ac0f6f983d56724b2cc57Roland Levillain /* no_rip */ true)); 1384d75948ac93a4a317feaf136cae78823071234ba5Nicolas Geoffray __ Bind(slow_path->GetExitLabel()); 1385d75948ac93a4a317feaf136cae78823071234ba5Nicolas Geoffray} 1386d75948ac93a4a317feaf136cae78823071234ba5Nicolas Geoffray 1387f8cfb20cfa00f8987227204211e99486bc38572fAgi Csakivoid IntrinsicLocationsBuilderX86_64::VisitStringEquals(HInvoke* invoke) { 1388f8cfb20cfa00f8987227204211e99486bc38572fAgi Csaki LocationSummary* locations = new (arena_) LocationSummary(invoke, 1389f8cfb20cfa00f8987227204211e99486bc38572fAgi Csaki LocationSummary::kNoCall, 1390f8cfb20cfa00f8987227204211e99486bc38572fAgi Csaki kIntrinsified); 1391f8cfb20cfa00f8987227204211e99486bc38572fAgi Csaki locations->SetInAt(0, Location::RequiresRegister()); 1392f8cfb20cfa00f8987227204211e99486bc38572fAgi Csaki locations->SetInAt(1, Location::RequiresRegister()); 1393f8cfb20cfa00f8987227204211e99486bc38572fAgi Csaki 1394f8cfb20cfa00f8987227204211e99486bc38572fAgi Csaki // Request temporary registers, RCX and RDI needed for repe_cmpsq instruction. 1395f8cfb20cfa00f8987227204211e99486bc38572fAgi Csaki locations->AddTemp(Location::RegisterLocation(RCX)); 1396f8cfb20cfa00f8987227204211e99486bc38572fAgi Csaki locations->AddTemp(Location::RegisterLocation(RDI)); 1397f8cfb20cfa00f8987227204211e99486bc38572fAgi Csaki 1398f8cfb20cfa00f8987227204211e99486bc38572fAgi Csaki // Set output, RSI needed for repe_cmpsq instruction anyways. 1399f8cfb20cfa00f8987227204211e99486bc38572fAgi Csaki locations->SetOut(Location::RegisterLocation(RSI), Location::kOutputOverlap); 1400f8cfb20cfa00f8987227204211e99486bc38572fAgi Csaki} 1401f8cfb20cfa00f8987227204211e99486bc38572fAgi Csaki 1402f8cfb20cfa00f8987227204211e99486bc38572fAgi Csakivoid IntrinsicCodeGeneratorX86_64::VisitStringEquals(HInvoke* invoke) { 1403f8cfb20cfa00f8987227204211e99486bc38572fAgi Csaki X86_64Assembler* assembler = GetAssembler(); 1404f8cfb20cfa00f8987227204211e99486bc38572fAgi Csaki LocationSummary* locations = invoke->GetLocations(); 1405f8cfb20cfa00f8987227204211e99486bc38572fAgi Csaki 1406f8cfb20cfa00f8987227204211e99486bc38572fAgi Csaki CpuRegister str = locations->InAt(0).AsRegister<CpuRegister>(); 1407f8cfb20cfa00f8987227204211e99486bc38572fAgi Csaki CpuRegister arg = locations->InAt(1).AsRegister<CpuRegister>(); 1408f8cfb20cfa00f8987227204211e99486bc38572fAgi Csaki CpuRegister rcx = locations->GetTemp(0).AsRegister<CpuRegister>(); 1409f8cfb20cfa00f8987227204211e99486bc38572fAgi Csaki CpuRegister rdi = locations->GetTemp(1).AsRegister<CpuRegister>(); 1410f8cfb20cfa00f8987227204211e99486bc38572fAgi Csaki CpuRegister rsi = locations->Out().AsRegister<CpuRegister>(); 1411f8cfb20cfa00f8987227204211e99486bc38572fAgi Csaki 14120c9497da9485ba688c592e5f452b7b1305a519c0Mark Mendell NearLabel end, return_true, return_false; 1413f8cfb20cfa00f8987227204211e99486bc38572fAgi Csaki 1414f8cfb20cfa00f8987227204211e99486bc38572fAgi Csaki // Get offsets of count, value, and class fields within a string object. 1415f8cfb20cfa00f8987227204211e99486bc38572fAgi Csaki const uint32_t count_offset = mirror::String::CountOffset().Uint32Value(); 1416f8cfb20cfa00f8987227204211e99486bc38572fAgi Csaki const uint32_t value_offset = mirror::String::ValueOffset().Uint32Value(); 1417f8cfb20cfa00f8987227204211e99486bc38572fAgi Csaki const uint32_t class_offset = mirror::Object::ClassOffset().Uint32Value(); 1418f8cfb20cfa00f8987227204211e99486bc38572fAgi Csaki 1419f8cfb20cfa00f8987227204211e99486bc38572fAgi Csaki // Note that the null check must have been done earlier. 1420f8cfb20cfa00f8987227204211e99486bc38572fAgi Csaki DCHECK(!invoke->CanDoImplicitNullCheckOn(invoke->InputAt(0))); 1421f8cfb20cfa00f8987227204211e99486bc38572fAgi Csaki 1422f8cfb20cfa00f8987227204211e99486bc38572fAgi Csaki // Check if input is null, return false if it is. 1423f8cfb20cfa00f8987227204211e99486bc38572fAgi Csaki __ testl(arg, arg); 1424f8cfb20cfa00f8987227204211e99486bc38572fAgi Csaki __ j(kEqual, &return_false); 1425f8cfb20cfa00f8987227204211e99486bc38572fAgi Csaki 1426f8cfb20cfa00f8987227204211e99486bc38572fAgi Csaki // Instanceof check for the argument by comparing class fields. 1427f8cfb20cfa00f8987227204211e99486bc38572fAgi Csaki // All string objects must have the same type since String cannot be subclassed. 1428f8cfb20cfa00f8987227204211e99486bc38572fAgi Csaki // Receiver must be a string object, so its class field is equal to all strings' class fields. 1429f8cfb20cfa00f8987227204211e99486bc38572fAgi Csaki // If the argument is a string object, its class field must be equal to receiver's class field. 1430f8cfb20cfa00f8987227204211e99486bc38572fAgi Csaki __ movl(rcx, Address(str, class_offset)); 1431f8cfb20cfa00f8987227204211e99486bc38572fAgi Csaki __ cmpl(rcx, Address(arg, class_offset)); 1432f8cfb20cfa00f8987227204211e99486bc38572fAgi Csaki __ j(kNotEqual, &return_false); 1433f8cfb20cfa00f8987227204211e99486bc38572fAgi Csaki 1434f8cfb20cfa00f8987227204211e99486bc38572fAgi Csaki // Reference equality check, return true if same reference. 1435f8cfb20cfa00f8987227204211e99486bc38572fAgi Csaki __ cmpl(str, arg); 1436f8cfb20cfa00f8987227204211e99486bc38572fAgi Csaki __ j(kEqual, &return_true); 1437f8cfb20cfa00f8987227204211e99486bc38572fAgi Csaki 1438f8cfb20cfa00f8987227204211e99486bc38572fAgi Csaki // Load length of receiver string. 1439f8cfb20cfa00f8987227204211e99486bc38572fAgi Csaki __ movl(rcx, Address(str, count_offset)); 1440f8cfb20cfa00f8987227204211e99486bc38572fAgi Csaki // Check if lengths are equal, return false if they're not. 1441f8cfb20cfa00f8987227204211e99486bc38572fAgi Csaki __ cmpl(rcx, Address(arg, count_offset)); 1442f8cfb20cfa00f8987227204211e99486bc38572fAgi Csaki __ j(kNotEqual, &return_false); 1443f8cfb20cfa00f8987227204211e99486bc38572fAgi Csaki // Return true if both strings are empty. 14440c9497da9485ba688c592e5f452b7b1305a519c0Mark Mendell __ jrcxz(&return_true); 1445f8cfb20cfa00f8987227204211e99486bc38572fAgi Csaki 1446f8cfb20cfa00f8987227204211e99486bc38572fAgi Csaki // Load starting addresses of string values into RSI/RDI as required for repe_cmpsq instruction. 1447f8cfb20cfa00f8987227204211e99486bc38572fAgi Csaki __ leal(rsi, Address(str, value_offset)); 1448f8cfb20cfa00f8987227204211e99486bc38572fAgi Csaki __ leal(rdi, Address(arg, value_offset)); 1449f8cfb20cfa00f8987227204211e99486bc38572fAgi Csaki 1450f8cfb20cfa00f8987227204211e99486bc38572fAgi Csaki // Divide string length by 4 and adjust for lengths not divisible by 4. 1451f8cfb20cfa00f8987227204211e99486bc38572fAgi Csaki __ addl(rcx, Immediate(3)); 1452f8cfb20cfa00f8987227204211e99486bc38572fAgi Csaki __ shrl(rcx, Immediate(2)); 1453f8cfb20cfa00f8987227204211e99486bc38572fAgi Csaki 1454f8cfb20cfa00f8987227204211e99486bc38572fAgi Csaki // Assertions that must hold in order to compare strings 4 characters at a time. 1455f8cfb20cfa00f8987227204211e99486bc38572fAgi Csaki DCHECK_ALIGNED(value_offset, 8); 1456f8cfb20cfa00f8987227204211e99486bc38572fAgi Csaki static_assert(IsAligned<8>(kObjectAlignment), "String is not zero padded"); 1457f8cfb20cfa00f8987227204211e99486bc38572fAgi Csaki 1458f8cfb20cfa00f8987227204211e99486bc38572fAgi Csaki // Loop to compare strings four characters at a time starting at the beginning of the string. 1459f8cfb20cfa00f8987227204211e99486bc38572fAgi Csaki __ repe_cmpsq(); 1460f8cfb20cfa00f8987227204211e99486bc38572fAgi Csaki // If strings are not equal, zero flag will be cleared. 1461f8cfb20cfa00f8987227204211e99486bc38572fAgi Csaki __ j(kNotEqual, &return_false); 1462f8cfb20cfa00f8987227204211e99486bc38572fAgi Csaki 1463f8cfb20cfa00f8987227204211e99486bc38572fAgi Csaki // Return true and exit the function. 1464f8cfb20cfa00f8987227204211e99486bc38572fAgi Csaki // If loop does not result in returning false, we return true. 1465f8cfb20cfa00f8987227204211e99486bc38572fAgi Csaki __ Bind(&return_true); 1466f8cfb20cfa00f8987227204211e99486bc38572fAgi Csaki __ movl(rsi, Immediate(1)); 1467f8cfb20cfa00f8987227204211e99486bc38572fAgi Csaki __ jmp(&end); 1468f8cfb20cfa00f8987227204211e99486bc38572fAgi Csaki 1469f8cfb20cfa00f8987227204211e99486bc38572fAgi Csaki // Return false and exit the function. 1470f8cfb20cfa00f8987227204211e99486bc38572fAgi Csaki __ Bind(&return_false); 1471f8cfb20cfa00f8987227204211e99486bc38572fAgi Csaki __ xorl(rsi, rsi); 1472f8cfb20cfa00f8987227204211e99486bc38572fAgi Csaki __ Bind(&end); 1473f8cfb20cfa00f8987227204211e99486bc38572fAgi Csaki} 1474f8cfb20cfa00f8987227204211e99486bc38572fAgi Csaki 147521030dd59b1e350f6f43de39e3c4ce0886ff539cAndreas Gampestatic void CreateStringIndexOfLocations(HInvoke* invoke, 147621030dd59b1e350f6f43de39e3c4ce0886ff539cAndreas Gampe ArenaAllocator* allocator, 147721030dd59b1e350f6f43de39e3c4ce0886ff539cAndreas Gampe bool start_at_zero) { 147821030dd59b1e350f6f43de39e3c4ce0886ff539cAndreas Gampe LocationSummary* locations = new (allocator) LocationSummary(invoke, 147921030dd59b1e350f6f43de39e3c4ce0886ff539cAndreas Gampe LocationSummary::kCallOnSlowPath, 148021030dd59b1e350f6f43de39e3c4ce0886ff539cAndreas Gampe kIntrinsified); 148121030dd59b1e350f6f43de39e3c4ce0886ff539cAndreas Gampe // The data needs to be in RDI for scasw. So request that the string is there, anyways. 148221030dd59b1e350f6f43de39e3c4ce0886ff539cAndreas Gampe locations->SetInAt(0, Location::RegisterLocation(RDI)); 148321030dd59b1e350f6f43de39e3c4ce0886ff539cAndreas Gampe // If we look for a constant char, we'll still have to copy it into RAX. So just request the 148421030dd59b1e350f6f43de39e3c4ce0886ff539cAndreas Gampe // allocator to do that, anyways. We can still do the constant check by checking the parameter 148521030dd59b1e350f6f43de39e3c4ce0886ff539cAndreas Gampe // of the instruction explicitly. 148621030dd59b1e350f6f43de39e3c4ce0886ff539cAndreas Gampe // Note: This works as we don't clobber RAX anywhere. 148721030dd59b1e350f6f43de39e3c4ce0886ff539cAndreas Gampe locations->SetInAt(1, Location::RegisterLocation(RAX)); 148821030dd59b1e350f6f43de39e3c4ce0886ff539cAndreas Gampe if (!start_at_zero) { 148921030dd59b1e350f6f43de39e3c4ce0886ff539cAndreas Gampe locations->SetInAt(2, Location::RequiresRegister()); // The starting index. 149021030dd59b1e350f6f43de39e3c4ce0886ff539cAndreas Gampe } 149121030dd59b1e350f6f43de39e3c4ce0886ff539cAndreas Gampe // As we clobber RDI during execution anyways, also use it as the output. 149221030dd59b1e350f6f43de39e3c4ce0886ff539cAndreas Gampe locations->SetOut(Location::SameAsFirstInput()); 149321030dd59b1e350f6f43de39e3c4ce0886ff539cAndreas Gampe 149421030dd59b1e350f6f43de39e3c4ce0886ff539cAndreas Gampe // repne scasw uses RCX as the counter. 149521030dd59b1e350f6f43de39e3c4ce0886ff539cAndreas Gampe locations->AddTemp(Location::RegisterLocation(RCX)); 149621030dd59b1e350f6f43de39e3c4ce0886ff539cAndreas Gampe // Need another temporary to be able to compute the result. 149721030dd59b1e350f6f43de39e3c4ce0886ff539cAndreas Gampe locations->AddTemp(Location::RequiresRegister()); 149821030dd59b1e350f6f43de39e3c4ce0886ff539cAndreas Gampe} 149921030dd59b1e350f6f43de39e3c4ce0886ff539cAndreas Gampe 150021030dd59b1e350f6f43de39e3c4ce0886ff539cAndreas Gampestatic void GenerateStringIndexOf(HInvoke* invoke, 150121030dd59b1e350f6f43de39e3c4ce0886ff539cAndreas Gampe X86_64Assembler* assembler, 150221030dd59b1e350f6f43de39e3c4ce0886ff539cAndreas Gampe CodeGeneratorX86_64* codegen, 150321030dd59b1e350f6f43de39e3c4ce0886ff539cAndreas Gampe ArenaAllocator* allocator, 150421030dd59b1e350f6f43de39e3c4ce0886ff539cAndreas Gampe bool start_at_zero) { 150521030dd59b1e350f6f43de39e3c4ce0886ff539cAndreas Gampe LocationSummary* locations = invoke->GetLocations(); 150621030dd59b1e350f6f43de39e3c4ce0886ff539cAndreas Gampe 150721030dd59b1e350f6f43de39e3c4ce0886ff539cAndreas Gampe // Note that the null check must have been done earlier. 150821030dd59b1e350f6f43de39e3c4ce0886ff539cAndreas Gampe DCHECK(!invoke->CanDoImplicitNullCheckOn(invoke->InputAt(0))); 150921030dd59b1e350f6f43de39e3c4ce0886ff539cAndreas Gampe 151021030dd59b1e350f6f43de39e3c4ce0886ff539cAndreas Gampe CpuRegister string_obj = locations->InAt(0).AsRegister<CpuRegister>(); 151121030dd59b1e350f6f43de39e3c4ce0886ff539cAndreas Gampe CpuRegister search_value = locations->InAt(1).AsRegister<CpuRegister>(); 151221030dd59b1e350f6f43de39e3c4ce0886ff539cAndreas Gampe CpuRegister counter = locations->GetTemp(0).AsRegister<CpuRegister>(); 151321030dd59b1e350f6f43de39e3c4ce0886ff539cAndreas Gampe CpuRegister string_length = locations->GetTemp(1).AsRegister<CpuRegister>(); 151421030dd59b1e350f6f43de39e3c4ce0886ff539cAndreas Gampe CpuRegister out = locations->Out().AsRegister<CpuRegister>(); 151521030dd59b1e350f6f43de39e3c4ce0886ff539cAndreas Gampe 151621030dd59b1e350f6f43de39e3c4ce0886ff539cAndreas Gampe // Check our assumptions for registers. 151721030dd59b1e350f6f43de39e3c4ce0886ff539cAndreas Gampe DCHECK_EQ(string_obj.AsRegister(), RDI); 151821030dd59b1e350f6f43de39e3c4ce0886ff539cAndreas Gampe DCHECK_EQ(search_value.AsRegister(), RAX); 151921030dd59b1e350f6f43de39e3c4ce0886ff539cAndreas Gampe DCHECK_EQ(counter.AsRegister(), RCX); 152021030dd59b1e350f6f43de39e3c4ce0886ff539cAndreas Gampe DCHECK_EQ(out.AsRegister(), RDI); 152121030dd59b1e350f6f43de39e3c4ce0886ff539cAndreas Gampe 152221030dd59b1e350f6f43de39e3c4ce0886ff539cAndreas Gampe // Check for code points > 0xFFFF. Either a slow-path check when we don't know statically, 152321030dd59b1e350f6f43de39e3c4ce0886ff539cAndreas Gampe // or directly dispatch if we have a constant. 152485b62f23fc6dfffe2ddd3ddfa74611666c9ff41dAndreas Gampe SlowPathCode* slow_path = nullptr; 152521030dd59b1e350f6f43de39e3c4ce0886ff539cAndreas Gampe if (invoke->InputAt(1)->IsIntConstant()) { 152621030dd59b1e350f6f43de39e3c4ce0886ff539cAndreas Gampe if (static_cast<uint32_t>(invoke->InputAt(1)->AsIntConstant()->GetValue()) > 152721030dd59b1e350f6f43de39e3c4ce0886ff539cAndreas Gampe std::numeric_limits<uint16_t>::max()) { 152821030dd59b1e350f6f43de39e3c4ce0886ff539cAndreas Gampe // Always needs the slow-path. We could directly dispatch to it, but this case should be 152921030dd59b1e350f6f43de39e3c4ce0886ff539cAndreas Gampe // rare, so for simplicity just put the full slow-path down and branch unconditionally. 153021030dd59b1e350f6f43de39e3c4ce0886ff539cAndreas Gampe slow_path = new (allocator) IntrinsicSlowPathX86_64(invoke); 153121030dd59b1e350f6f43de39e3c4ce0886ff539cAndreas Gampe codegen->AddSlowPath(slow_path); 153221030dd59b1e350f6f43de39e3c4ce0886ff539cAndreas Gampe __ jmp(slow_path->GetEntryLabel()); 153321030dd59b1e350f6f43de39e3c4ce0886ff539cAndreas Gampe __ Bind(slow_path->GetExitLabel()); 153421030dd59b1e350f6f43de39e3c4ce0886ff539cAndreas Gampe return; 153521030dd59b1e350f6f43de39e3c4ce0886ff539cAndreas Gampe } 153621030dd59b1e350f6f43de39e3c4ce0886ff539cAndreas Gampe } else { 153721030dd59b1e350f6f43de39e3c4ce0886ff539cAndreas Gampe __ cmpl(search_value, Immediate(std::numeric_limits<uint16_t>::max())); 153821030dd59b1e350f6f43de39e3c4ce0886ff539cAndreas Gampe slow_path = new (allocator) IntrinsicSlowPathX86_64(invoke); 153921030dd59b1e350f6f43de39e3c4ce0886ff539cAndreas Gampe codegen->AddSlowPath(slow_path); 154021030dd59b1e350f6f43de39e3c4ce0886ff539cAndreas Gampe __ j(kAbove, slow_path->GetEntryLabel()); 154121030dd59b1e350f6f43de39e3c4ce0886ff539cAndreas Gampe } 154221030dd59b1e350f6f43de39e3c4ce0886ff539cAndreas Gampe 154321030dd59b1e350f6f43de39e3c4ce0886ff539cAndreas Gampe // From here down, we know that we are looking for a char that fits in 16 bits. 154421030dd59b1e350f6f43de39e3c4ce0886ff539cAndreas Gampe // Location of reference to data array within the String object. 154521030dd59b1e350f6f43de39e3c4ce0886ff539cAndreas Gampe int32_t value_offset = mirror::String::ValueOffset().Int32Value(); 154621030dd59b1e350f6f43de39e3c4ce0886ff539cAndreas Gampe // Location of count within the String object. 154721030dd59b1e350f6f43de39e3c4ce0886ff539cAndreas Gampe int32_t count_offset = mirror::String::CountOffset().Int32Value(); 154821030dd59b1e350f6f43de39e3c4ce0886ff539cAndreas Gampe 154921030dd59b1e350f6f43de39e3c4ce0886ff539cAndreas Gampe // Load string length, i.e., the count field of the string. 155021030dd59b1e350f6f43de39e3c4ce0886ff539cAndreas Gampe __ movl(string_length, Address(string_obj, count_offset)); 155121030dd59b1e350f6f43de39e3c4ce0886ff539cAndreas Gampe 155221030dd59b1e350f6f43de39e3c4ce0886ff539cAndreas Gampe // Do a length check. 155321030dd59b1e350f6f43de39e3c4ce0886ff539cAndreas Gampe // TODO: Support jecxz. 15540c9497da9485ba688c592e5f452b7b1305a519c0Mark Mendell NearLabel not_found_label; 155521030dd59b1e350f6f43de39e3c4ce0886ff539cAndreas Gampe __ testl(string_length, string_length); 155621030dd59b1e350f6f43de39e3c4ce0886ff539cAndreas Gampe __ j(kEqual, ¬_found_label); 155721030dd59b1e350f6f43de39e3c4ce0886ff539cAndreas Gampe 155821030dd59b1e350f6f43de39e3c4ce0886ff539cAndreas Gampe if (start_at_zero) { 155921030dd59b1e350f6f43de39e3c4ce0886ff539cAndreas Gampe // Number of chars to scan is the same as the string length. 156021030dd59b1e350f6f43de39e3c4ce0886ff539cAndreas Gampe __ movl(counter, string_length); 156121030dd59b1e350f6f43de39e3c4ce0886ff539cAndreas Gampe 156221030dd59b1e350f6f43de39e3c4ce0886ff539cAndreas Gampe // Move to the start of the string. 156321030dd59b1e350f6f43de39e3c4ce0886ff539cAndreas Gampe __ addq(string_obj, Immediate(value_offset)); 156421030dd59b1e350f6f43de39e3c4ce0886ff539cAndreas Gampe } else { 156521030dd59b1e350f6f43de39e3c4ce0886ff539cAndreas Gampe CpuRegister start_index = locations->InAt(2).AsRegister<CpuRegister>(); 156621030dd59b1e350f6f43de39e3c4ce0886ff539cAndreas Gampe 156721030dd59b1e350f6f43de39e3c4ce0886ff539cAndreas Gampe // Do a start_index check. 156821030dd59b1e350f6f43de39e3c4ce0886ff539cAndreas Gampe __ cmpl(start_index, string_length); 156921030dd59b1e350f6f43de39e3c4ce0886ff539cAndreas Gampe __ j(kGreaterEqual, ¬_found_label); 157021030dd59b1e350f6f43de39e3c4ce0886ff539cAndreas Gampe 157121030dd59b1e350f6f43de39e3c4ce0886ff539cAndreas Gampe // Ensure we have a start index >= 0; 157221030dd59b1e350f6f43de39e3c4ce0886ff539cAndreas Gampe __ xorl(counter, counter); 157321030dd59b1e350f6f43de39e3c4ce0886ff539cAndreas Gampe __ cmpl(start_index, Immediate(0)); 1574bf84a3d2aa29c0975b4ac0f6f983d56724b2cc57Roland Levillain __ cmov(kGreater, counter, start_index, /* is64bit */ false); // 32-bit copy is enough. 157521030dd59b1e350f6f43de39e3c4ce0886ff539cAndreas Gampe 157621030dd59b1e350f6f43de39e3c4ce0886ff539cAndreas Gampe // Move to the start of the string: string_obj + value_offset + 2 * start_index. 157721030dd59b1e350f6f43de39e3c4ce0886ff539cAndreas Gampe __ leaq(string_obj, Address(string_obj, counter, ScaleFactor::TIMES_2, value_offset)); 157821030dd59b1e350f6f43de39e3c4ce0886ff539cAndreas Gampe 157921030dd59b1e350f6f43de39e3c4ce0886ff539cAndreas Gampe // Now update ecx, the work counter: it's gonna be string.length - start_index. 158021030dd59b1e350f6f43de39e3c4ce0886ff539cAndreas Gampe __ negq(counter); // Needs to be 64-bit negation, as the address computation is 64-bit. 158121030dd59b1e350f6f43de39e3c4ce0886ff539cAndreas Gampe __ leaq(counter, Address(string_length, counter, ScaleFactor::TIMES_1, 0)); 158221030dd59b1e350f6f43de39e3c4ce0886ff539cAndreas Gampe } 158321030dd59b1e350f6f43de39e3c4ce0886ff539cAndreas Gampe 158421030dd59b1e350f6f43de39e3c4ce0886ff539cAndreas Gampe // Everything is set up for repne scasw: 158521030dd59b1e350f6f43de39e3c4ce0886ff539cAndreas Gampe // * Comparison address in RDI. 158621030dd59b1e350f6f43de39e3c4ce0886ff539cAndreas Gampe // * Counter in ECX. 158721030dd59b1e350f6f43de39e3c4ce0886ff539cAndreas Gampe __ repne_scasw(); 158821030dd59b1e350f6f43de39e3c4ce0886ff539cAndreas Gampe 158921030dd59b1e350f6f43de39e3c4ce0886ff539cAndreas Gampe // Did we find a match? 159021030dd59b1e350f6f43de39e3c4ce0886ff539cAndreas Gampe __ j(kNotEqual, ¬_found_label); 159121030dd59b1e350f6f43de39e3c4ce0886ff539cAndreas Gampe 159221030dd59b1e350f6f43de39e3c4ce0886ff539cAndreas Gampe // Yes, we matched. Compute the index of the result. 159321030dd59b1e350f6f43de39e3c4ce0886ff539cAndreas Gampe __ subl(string_length, counter); 159421030dd59b1e350f6f43de39e3c4ce0886ff539cAndreas Gampe __ leal(out, Address(string_length, -1)); 159521030dd59b1e350f6f43de39e3c4ce0886ff539cAndreas Gampe 15960c9497da9485ba688c592e5f452b7b1305a519c0Mark Mendell NearLabel done; 159721030dd59b1e350f6f43de39e3c4ce0886ff539cAndreas Gampe __ jmp(&done); 159821030dd59b1e350f6f43de39e3c4ce0886ff539cAndreas Gampe 159921030dd59b1e350f6f43de39e3c4ce0886ff539cAndreas Gampe // Failed to match; return -1. 160021030dd59b1e350f6f43de39e3c4ce0886ff539cAndreas Gampe __ Bind(¬_found_label); 160121030dd59b1e350f6f43de39e3c4ce0886ff539cAndreas Gampe __ movl(out, Immediate(-1)); 160221030dd59b1e350f6f43de39e3c4ce0886ff539cAndreas Gampe 160321030dd59b1e350f6f43de39e3c4ce0886ff539cAndreas Gampe // And join up at the end. 160421030dd59b1e350f6f43de39e3c4ce0886ff539cAndreas Gampe __ Bind(&done); 160521030dd59b1e350f6f43de39e3c4ce0886ff539cAndreas Gampe if (slow_path != nullptr) { 160621030dd59b1e350f6f43de39e3c4ce0886ff539cAndreas Gampe __ Bind(slow_path->GetExitLabel()); 160721030dd59b1e350f6f43de39e3c4ce0886ff539cAndreas Gampe } 160821030dd59b1e350f6f43de39e3c4ce0886ff539cAndreas Gampe} 160921030dd59b1e350f6f43de39e3c4ce0886ff539cAndreas Gampe 161021030dd59b1e350f6f43de39e3c4ce0886ff539cAndreas Gampevoid IntrinsicLocationsBuilderX86_64::VisitStringIndexOf(HInvoke* invoke) { 1611bf84a3d2aa29c0975b4ac0f6f983d56724b2cc57Roland Levillain CreateStringIndexOfLocations(invoke, arena_, /* start_at_zero */ true); 161221030dd59b1e350f6f43de39e3c4ce0886ff539cAndreas Gampe} 161321030dd59b1e350f6f43de39e3c4ce0886ff539cAndreas Gampe 161421030dd59b1e350f6f43de39e3c4ce0886ff539cAndreas Gampevoid IntrinsicCodeGeneratorX86_64::VisitStringIndexOf(HInvoke* invoke) { 1615bf84a3d2aa29c0975b4ac0f6f983d56724b2cc57Roland Levillain GenerateStringIndexOf(invoke, GetAssembler(), codegen_, GetAllocator(), /* start_at_zero */ true); 161621030dd59b1e350f6f43de39e3c4ce0886ff539cAndreas Gampe} 161721030dd59b1e350f6f43de39e3c4ce0886ff539cAndreas Gampe 161821030dd59b1e350f6f43de39e3c4ce0886ff539cAndreas Gampevoid IntrinsicLocationsBuilderX86_64::VisitStringIndexOfAfter(HInvoke* invoke) { 1619bf84a3d2aa29c0975b4ac0f6f983d56724b2cc57Roland Levillain CreateStringIndexOfLocations(invoke, arena_, /* start_at_zero */ false); 162021030dd59b1e350f6f43de39e3c4ce0886ff539cAndreas Gampe} 162121030dd59b1e350f6f43de39e3c4ce0886ff539cAndreas Gampe 162221030dd59b1e350f6f43de39e3c4ce0886ff539cAndreas Gampevoid IntrinsicCodeGeneratorX86_64::VisitStringIndexOfAfter(HInvoke* invoke) { 1623bf84a3d2aa29c0975b4ac0f6f983d56724b2cc57Roland Levillain GenerateStringIndexOf( 1624bf84a3d2aa29c0975b4ac0f6f983d56724b2cc57Roland Levillain invoke, GetAssembler(), codegen_, GetAllocator(), /* start_at_zero */ false); 162521030dd59b1e350f6f43de39e3c4ce0886ff539cAndreas Gampe} 162621030dd59b1e350f6f43de39e3c4ce0886ff539cAndreas Gampe 1627848f70a3d73833fc1bf3032a9ff6812e429661d9Jeff Haovoid IntrinsicLocationsBuilderX86_64::VisitStringNewStringFromBytes(HInvoke* invoke) { 1628848f70a3d73833fc1bf3032a9ff6812e429661d9Jeff Hao LocationSummary* locations = new (arena_) LocationSummary(invoke, 1629848f70a3d73833fc1bf3032a9ff6812e429661d9Jeff Hao LocationSummary::kCall, 1630848f70a3d73833fc1bf3032a9ff6812e429661d9Jeff Hao kIntrinsified); 1631848f70a3d73833fc1bf3032a9ff6812e429661d9Jeff Hao InvokeRuntimeCallingConvention calling_convention; 1632848f70a3d73833fc1bf3032a9ff6812e429661d9Jeff Hao locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(0))); 1633848f70a3d73833fc1bf3032a9ff6812e429661d9Jeff Hao locations->SetInAt(1, Location::RegisterLocation(calling_convention.GetRegisterAt(1))); 1634848f70a3d73833fc1bf3032a9ff6812e429661d9Jeff Hao locations->SetInAt(2, Location::RegisterLocation(calling_convention.GetRegisterAt(2))); 1635848f70a3d73833fc1bf3032a9ff6812e429661d9Jeff Hao locations->SetInAt(3, Location::RegisterLocation(calling_convention.GetRegisterAt(3))); 1636848f70a3d73833fc1bf3032a9ff6812e429661d9Jeff Hao locations->SetOut(Location::RegisterLocation(RAX)); 1637848f70a3d73833fc1bf3032a9ff6812e429661d9Jeff Hao} 1638848f70a3d73833fc1bf3032a9ff6812e429661d9Jeff Hao 1639848f70a3d73833fc1bf3032a9ff6812e429661d9Jeff Haovoid IntrinsicCodeGeneratorX86_64::VisitStringNewStringFromBytes(HInvoke* invoke) { 1640848f70a3d73833fc1bf3032a9ff6812e429661d9Jeff Hao X86_64Assembler* assembler = GetAssembler(); 1641848f70a3d73833fc1bf3032a9ff6812e429661d9Jeff Hao LocationSummary* locations = invoke->GetLocations(); 1642848f70a3d73833fc1bf3032a9ff6812e429661d9Jeff Hao 1643848f70a3d73833fc1bf3032a9ff6812e429661d9Jeff Hao CpuRegister byte_array = locations->InAt(0).AsRegister<CpuRegister>(); 1644848f70a3d73833fc1bf3032a9ff6812e429661d9Jeff Hao __ testl(byte_array, byte_array); 164585b62f23fc6dfffe2ddd3ddfa74611666c9ff41dAndreas Gampe SlowPathCode* slow_path = new (GetAllocator()) IntrinsicSlowPathX86_64(invoke); 1646848f70a3d73833fc1bf3032a9ff6812e429661d9Jeff Hao codegen_->AddSlowPath(slow_path); 1647848f70a3d73833fc1bf3032a9ff6812e429661d9Jeff Hao __ j(kEqual, slow_path->GetEntryLabel()); 1648848f70a3d73833fc1bf3032a9ff6812e429661d9Jeff Hao 1649bf84a3d2aa29c0975b4ac0f6f983d56724b2cc57Roland Levillain __ gs()->call(Address::Absolute(QUICK_ENTRYPOINT_OFFSET(kX86_64WordSize, pAllocStringFromBytes), 1650bf84a3d2aa29c0975b4ac0f6f983d56724b2cc57Roland Levillain /* no_rip */ true)); 1651f969a209c30e3af636342d2fb7851d82a2529bf7Roland Levillain CheckEntrypointTypes<kQuickAllocStringFromBytes, void*, void*, int32_t, int32_t, int32_t>(); 1652848f70a3d73833fc1bf3032a9ff6812e429661d9Jeff Hao codegen_->RecordPcInfo(invoke, invoke->GetDexPc()); 1653848f70a3d73833fc1bf3032a9ff6812e429661d9Jeff Hao __ Bind(slow_path->GetExitLabel()); 1654848f70a3d73833fc1bf3032a9ff6812e429661d9Jeff Hao} 1655848f70a3d73833fc1bf3032a9ff6812e429661d9Jeff Hao 1656848f70a3d73833fc1bf3032a9ff6812e429661d9Jeff Haovoid IntrinsicLocationsBuilderX86_64::VisitStringNewStringFromChars(HInvoke* invoke) { 1657848f70a3d73833fc1bf3032a9ff6812e429661d9Jeff Hao LocationSummary* locations = new (arena_) LocationSummary(invoke, 1658848f70a3d73833fc1bf3032a9ff6812e429661d9Jeff Hao LocationSummary::kCall, 1659848f70a3d73833fc1bf3032a9ff6812e429661d9Jeff Hao kIntrinsified); 1660848f70a3d73833fc1bf3032a9ff6812e429661d9Jeff Hao InvokeRuntimeCallingConvention calling_convention; 1661848f70a3d73833fc1bf3032a9ff6812e429661d9Jeff Hao locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(0))); 1662848f70a3d73833fc1bf3032a9ff6812e429661d9Jeff Hao locations->SetInAt(1, Location::RegisterLocation(calling_convention.GetRegisterAt(1))); 1663848f70a3d73833fc1bf3032a9ff6812e429661d9Jeff Hao locations->SetInAt(2, Location::RegisterLocation(calling_convention.GetRegisterAt(2))); 1664848f70a3d73833fc1bf3032a9ff6812e429661d9Jeff Hao locations->SetOut(Location::RegisterLocation(RAX)); 1665848f70a3d73833fc1bf3032a9ff6812e429661d9Jeff Hao} 1666848f70a3d73833fc1bf3032a9ff6812e429661d9Jeff Hao 1667848f70a3d73833fc1bf3032a9ff6812e429661d9Jeff Haovoid IntrinsicCodeGeneratorX86_64::VisitStringNewStringFromChars(HInvoke* invoke) { 1668848f70a3d73833fc1bf3032a9ff6812e429661d9Jeff Hao X86_64Assembler* assembler = GetAssembler(); 1669848f70a3d73833fc1bf3032a9ff6812e429661d9Jeff Hao 1670cc3839c15555a2751e13980638fc40e4d3da633eRoland Levillain // No need to emit code checking whether `locations->InAt(2)` is a null 1671cc3839c15555a2751e13980638fc40e4d3da633eRoland Levillain // pointer, as callers of the native method 1672cc3839c15555a2751e13980638fc40e4d3da633eRoland Levillain // 1673cc3839c15555a2751e13980638fc40e4d3da633eRoland Levillain // java.lang.StringFactory.newStringFromChars(int offset, int charCount, char[] data) 1674cc3839c15555a2751e13980638fc40e4d3da633eRoland Levillain // 1675cc3839c15555a2751e13980638fc40e4d3da633eRoland Levillain // all include a null check on `data` before calling that method. 1676bf84a3d2aa29c0975b4ac0f6f983d56724b2cc57Roland Levillain __ gs()->call(Address::Absolute(QUICK_ENTRYPOINT_OFFSET(kX86_64WordSize, pAllocStringFromChars), 1677bf84a3d2aa29c0975b4ac0f6f983d56724b2cc57Roland Levillain /* no_rip */ true)); 1678f969a209c30e3af636342d2fb7851d82a2529bf7Roland Levillain CheckEntrypointTypes<kQuickAllocStringFromChars, void*, int32_t, int32_t, void*>(); 1679848f70a3d73833fc1bf3032a9ff6812e429661d9Jeff Hao codegen_->RecordPcInfo(invoke, invoke->GetDexPc()); 1680848f70a3d73833fc1bf3032a9ff6812e429661d9Jeff Hao} 1681848f70a3d73833fc1bf3032a9ff6812e429661d9Jeff Hao 1682848f70a3d73833fc1bf3032a9ff6812e429661d9Jeff Haovoid IntrinsicLocationsBuilderX86_64::VisitStringNewStringFromString(HInvoke* invoke) { 1683848f70a3d73833fc1bf3032a9ff6812e429661d9Jeff Hao LocationSummary* locations = new (arena_) LocationSummary(invoke, 1684848f70a3d73833fc1bf3032a9ff6812e429661d9Jeff Hao LocationSummary::kCall, 1685848f70a3d73833fc1bf3032a9ff6812e429661d9Jeff Hao kIntrinsified); 1686848f70a3d73833fc1bf3032a9ff6812e429661d9Jeff Hao InvokeRuntimeCallingConvention calling_convention; 1687848f70a3d73833fc1bf3032a9ff6812e429661d9Jeff Hao locations->SetInAt(0, Location::RegisterLocation(calling_convention.GetRegisterAt(0))); 1688848f70a3d73833fc1bf3032a9ff6812e429661d9Jeff Hao locations->SetOut(Location::RegisterLocation(RAX)); 1689848f70a3d73833fc1bf3032a9ff6812e429661d9Jeff Hao} 1690848f70a3d73833fc1bf3032a9ff6812e429661d9Jeff Hao 1691848f70a3d73833fc1bf3032a9ff6812e429661d9Jeff Haovoid IntrinsicCodeGeneratorX86_64::VisitStringNewStringFromString(HInvoke* invoke) { 1692848f70a3d73833fc1bf3032a9ff6812e429661d9Jeff Hao X86_64Assembler* assembler = GetAssembler(); 1693848f70a3d73833fc1bf3032a9ff6812e429661d9Jeff Hao LocationSummary* locations = invoke->GetLocations(); 1694848f70a3d73833fc1bf3032a9ff6812e429661d9Jeff Hao 1695848f70a3d73833fc1bf3032a9ff6812e429661d9Jeff Hao CpuRegister string_to_copy = locations->InAt(0).AsRegister<CpuRegister>(); 1696848f70a3d73833fc1bf3032a9ff6812e429661d9Jeff Hao __ testl(string_to_copy, string_to_copy); 169785b62f23fc6dfffe2ddd3ddfa74611666c9ff41dAndreas Gampe SlowPathCode* slow_path = new (GetAllocator()) IntrinsicSlowPathX86_64(invoke); 1698848f70a3d73833fc1bf3032a9ff6812e429661d9Jeff Hao codegen_->AddSlowPath(slow_path); 1699848f70a3d73833fc1bf3032a9ff6812e429661d9Jeff Hao __ j(kEqual, slow_path->GetEntryLabel()); 1700848f70a3d73833fc1bf3032a9ff6812e429661d9Jeff Hao 1701bf84a3d2aa29c0975b4ac0f6f983d56724b2cc57Roland Levillain __ gs()->call(Address::Absolute(QUICK_ENTRYPOINT_OFFSET(kX86_64WordSize, pAllocStringFromString), 1702bf84a3d2aa29c0975b4ac0f6f983d56724b2cc57Roland Levillain /* no_rip */ true)); 1703f969a209c30e3af636342d2fb7851d82a2529bf7Roland Levillain CheckEntrypointTypes<kQuickAllocStringFromString, void*, void*>(); 1704848f70a3d73833fc1bf3032a9ff6812e429661d9Jeff Hao codegen_->RecordPcInfo(invoke, invoke->GetDexPc()); 1705848f70a3d73833fc1bf3032a9ff6812e429661d9Jeff Hao __ Bind(slow_path->GetExitLabel()); 1706848f70a3d73833fc1bf3032a9ff6812e429661d9Jeff Hao} 1707848f70a3d73833fc1bf3032a9ff6812e429661d9Jeff Hao 17088f8926a5c7ea332ab387c2b3ebc6fd378a5761bcMark Mendellvoid IntrinsicLocationsBuilderX86_64::VisitStringGetCharsNoCheck(HInvoke* invoke) { 17098f8926a5c7ea332ab387c2b3ebc6fd378a5761bcMark Mendell // public void getChars(int srcBegin, int srcEnd, char[] dst, int dstBegin); 17108f8926a5c7ea332ab387c2b3ebc6fd378a5761bcMark Mendell LocationSummary* locations = new (arena_) LocationSummary(invoke, 17118f8926a5c7ea332ab387c2b3ebc6fd378a5761bcMark Mendell LocationSummary::kNoCall, 17128f8926a5c7ea332ab387c2b3ebc6fd378a5761bcMark Mendell kIntrinsified); 17138f8926a5c7ea332ab387c2b3ebc6fd378a5761bcMark Mendell locations->SetInAt(0, Location::RequiresRegister()); 17148f8926a5c7ea332ab387c2b3ebc6fd378a5761bcMark Mendell locations->SetInAt(1, Location::RegisterOrConstant(invoke->InputAt(1))); 17158f8926a5c7ea332ab387c2b3ebc6fd378a5761bcMark Mendell locations->SetInAt(2, Location::RequiresRegister()); 17168f8926a5c7ea332ab387c2b3ebc6fd378a5761bcMark Mendell locations->SetInAt(3, Location::RequiresRegister()); 17178f8926a5c7ea332ab387c2b3ebc6fd378a5761bcMark Mendell locations->SetInAt(4, Location::RequiresRegister()); 17188f8926a5c7ea332ab387c2b3ebc6fd378a5761bcMark Mendell 17198f8926a5c7ea332ab387c2b3ebc6fd378a5761bcMark Mendell // And we need some temporaries. We will use REP MOVSW, so we need fixed registers. 17208f8926a5c7ea332ab387c2b3ebc6fd378a5761bcMark Mendell locations->AddTemp(Location::RegisterLocation(RSI)); 17218f8926a5c7ea332ab387c2b3ebc6fd378a5761bcMark Mendell locations->AddTemp(Location::RegisterLocation(RDI)); 17228f8926a5c7ea332ab387c2b3ebc6fd378a5761bcMark Mendell locations->AddTemp(Location::RegisterLocation(RCX)); 17238f8926a5c7ea332ab387c2b3ebc6fd378a5761bcMark Mendell} 17248f8926a5c7ea332ab387c2b3ebc6fd378a5761bcMark Mendell 17258f8926a5c7ea332ab387c2b3ebc6fd378a5761bcMark Mendellvoid IntrinsicCodeGeneratorX86_64::VisitStringGetCharsNoCheck(HInvoke* invoke) { 17268f8926a5c7ea332ab387c2b3ebc6fd378a5761bcMark Mendell X86_64Assembler* assembler = GetAssembler(); 17278f8926a5c7ea332ab387c2b3ebc6fd378a5761bcMark Mendell LocationSummary* locations = invoke->GetLocations(); 17288f8926a5c7ea332ab387c2b3ebc6fd378a5761bcMark Mendell 17298f8926a5c7ea332ab387c2b3ebc6fd378a5761bcMark Mendell size_t char_component_size = Primitive::ComponentSize(Primitive::kPrimChar); 17308f8926a5c7ea332ab387c2b3ebc6fd378a5761bcMark Mendell // Location of data in char array buffer. 17318f8926a5c7ea332ab387c2b3ebc6fd378a5761bcMark Mendell const uint32_t data_offset = mirror::Array::DataOffset(char_component_size).Uint32Value(); 17328f8926a5c7ea332ab387c2b3ebc6fd378a5761bcMark Mendell // Location of char array data in string. 17338f8926a5c7ea332ab387c2b3ebc6fd378a5761bcMark Mendell const uint32_t value_offset = mirror::String::ValueOffset().Uint32Value(); 17348f8926a5c7ea332ab387c2b3ebc6fd378a5761bcMark Mendell 17358f8926a5c7ea332ab387c2b3ebc6fd378a5761bcMark Mendell // public void getChars(int srcBegin, int srcEnd, char[] dst, int dstBegin); 17368f8926a5c7ea332ab387c2b3ebc6fd378a5761bcMark Mendell CpuRegister obj = locations->InAt(0).AsRegister<CpuRegister>(); 17378f8926a5c7ea332ab387c2b3ebc6fd378a5761bcMark Mendell Location srcBegin = locations->InAt(1); 17388f8926a5c7ea332ab387c2b3ebc6fd378a5761bcMark Mendell int srcBegin_value = 17398f8926a5c7ea332ab387c2b3ebc6fd378a5761bcMark Mendell srcBegin.IsConstant() ? srcBegin.GetConstant()->AsIntConstant()->GetValue() : 0; 17408f8926a5c7ea332ab387c2b3ebc6fd378a5761bcMark Mendell CpuRegister srcEnd = locations->InAt(2).AsRegister<CpuRegister>(); 17418f8926a5c7ea332ab387c2b3ebc6fd378a5761bcMark Mendell CpuRegister dst = locations->InAt(3).AsRegister<CpuRegister>(); 17428f8926a5c7ea332ab387c2b3ebc6fd378a5761bcMark Mendell CpuRegister dstBegin = locations->InAt(4).AsRegister<CpuRegister>(); 17438f8926a5c7ea332ab387c2b3ebc6fd378a5761bcMark Mendell 17448f8926a5c7ea332ab387c2b3ebc6fd378a5761bcMark Mendell // Check assumption that sizeof(Char) is 2 (used in scaling below). 17458f8926a5c7ea332ab387c2b3ebc6fd378a5761bcMark Mendell const size_t char_size = Primitive::ComponentSize(Primitive::kPrimChar); 17468f8926a5c7ea332ab387c2b3ebc6fd378a5761bcMark Mendell DCHECK_EQ(char_size, 2u); 17478f8926a5c7ea332ab387c2b3ebc6fd378a5761bcMark Mendell 17488f8926a5c7ea332ab387c2b3ebc6fd378a5761bcMark Mendell // Compute the address of the destination buffer. 17498f8926a5c7ea332ab387c2b3ebc6fd378a5761bcMark Mendell __ leaq(CpuRegister(RDI), Address(dst, dstBegin, ScaleFactor::TIMES_2, data_offset)); 17508f8926a5c7ea332ab387c2b3ebc6fd378a5761bcMark Mendell 17518f8926a5c7ea332ab387c2b3ebc6fd378a5761bcMark Mendell // Compute the address of the source string. 17528f8926a5c7ea332ab387c2b3ebc6fd378a5761bcMark Mendell if (srcBegin.IsConstant()) { 17538f8926a5c7ea332ab387c2b3ebc6fd378a5761bcMark Mendell // Compute the address of the source string by adding the number of chars from 17548f8926a5c7ea332ab387c2b3ebc6fd378a5761bcMark Mendell // the source beginning to the value offset of a string. 17558f8926a5c7ea332ab387c2b3ebc6fd378a5761bcMark Mendell __ leaq(CpuRegister(RSI), Address(obj, srcBegin_value * char_size + value_offset)); 17568f8926a5c7ea332ab387c2b3ebc6fd378a5761bcMark Mendell } else { 17578f8926a5c7ea332ab387c2b3ebc6fd378a5761bcMark Mendell __ leaq(CpuRegister(RSI), Address(obj, srcBegin.AsRegister<CpuRegister>(), 17588f8926a5c7ea332ab387c2b3ebc6fd378a5761bcMark Mendell ScaleFactor::TIMES_2, value_offset)); 17598f8926a5c7ea332ab387c2b3ebc6fd378a5761bcMark Mendell } 17608f8926a5c7ea332ab387c2b3ebc6fd378a5761bcMark Mendell 17618f8926a5c7ea332ab387c2b3ebc6fd378a5761bcMark Mendell // Compute the number of chars (words) to move. 17628f8926a5c7ea332ab387c2b3ebc6fd378a5761bcMark Mendell __ movl(CpuRegister(RCX), srcEnd); 17638f8926a5c7ea332ab387c2b3ebc6fd378a5761bcMark Mendell if (srcBegin.IsConstant()) { 17648f8926a5c7ea332ab387c2b3ebc6fd378a5761bcMark Mendell if (srcBegin_value != 0) { 17658f8926a5c7ea332ab387c2b3ebc6fd378a5761bcMark Mendell __ subl(CpuRegister(RCX), Immediate(srcBegin_value)); 17668f8926a5c7ea332ab387c2b3ebc6fd378a5761bcMark Mendell } 17678f8926a5c7ea332ab387c2b3ebc6fd378a5761bcMark Mendell } else { 17688f8926a5c7ea332ab387c2b3ebc6fd378a5761bcMark Mendell DCHECK(srcBegin.IsRegister()); 17698f8926a5c7ea332ab387c2b3ebc6fd378a5761bcMark Mendell __ subl(CpuRegister(RCX), srcBegin.AsRegister<CpuRegister>()); 17708f8926a5c7ea332ab387c2b3ebc6fd378a5761bcMark Mendell } 17718f8926a5c7ea332ab387c2b3ebc6fd378a5761bcMark Mendell 17728f8926a5c7ea332ab387c2b3ebc6fd378a5761bcMark Mendell // Do the move. 17738f8926a5c7ea332ab387c2b3ebc6fd378a5761bcMark Mendell __ rep_movsw(); 17748f8926a5c7ea332ab387c2b3ebc6fd378a5761bcMark Mendell} 17758f8926a5c7ea332ab387c2b3ebc6fd378a5761bcMark Mendell 177671fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampestatic void GenPeek(LocationSummary* locations, Primitive::Type size, X86_64Assembler* assembler) { 177771fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe CpuRegister address = locations->InAt(0).AsRegister<CpuRegister>(); 177871fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe CpuRegister out = locations->Out().AsRegister<CpuRegister>(); // == address, here for clarity. 177971fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe // x86 allows unaligned access. We do not have to check the input or use specific instructions 178071fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe // to avoid a SIGBUS. 178171fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe switch (size) { 178271fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe case Primitive::kPrimByte: 178371fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe __ movsxb(out, Address(address, 0)); 178471fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe break; 178571fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe case Primitive::kPrimShort: 178671fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe __ movsxw(out, Address(address, 0)); 178771fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe break; 178871fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe case Primitive::kPrimInt: 178971fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe __ movl(out, Address(address, 0)); 179071fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe break; 179171fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe case Primitive::kPrimLong: 179271fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe __ movq(out, Address(address, 0)); 179371fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe break; 179471fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe default: 179571fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe LOG(FATAL) << "Type not recognized for peek: " << size; 179671fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe UNREACHABLE(); 179771fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe } 179871fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe} 179971fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe 180071fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampevoid IntrinsicLocationsBuilderX86_64::VisitMemoryPeekByte(HInvoke* invoke) { 180171fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe CreateIntToIntLocations(arena_, invoke); 180271fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe} 180371fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe 180471fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampevoid IntrinsicCodeGeneratorX86_64::VisitMemoryPeekByte(HInvoke* invoke) { 180571fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe GenPeek(invoke->GetLocations(), Primitive::kPrimByte, GetAssembler()); 180671fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe} 180771fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe 180871fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampevoid IntrinsicLocationsBuilderX86_64::VisitMemoryPeekIntNative(HInvoke* invoke) { 180971fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe CreateIntToIntLocations(arena_, invoke); 181071fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe} 181171fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe 181271fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampevoid IntrinsicCodeGeneratorX86_64::VisitMemoryPeekIntNative(HInvoke* invoke) { 181371fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe GenPeek(invoke->GetLocations(), Primitive::kPrimInt, GetAssembler()); 181471fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe} 181571fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe 181671fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampevoid IntrinsicLocationsBuilderX86_64::VisitMemoryPeekLongNative(HInvoke* invoke) { 181771fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe CreateIntToIntLocations(arena_, invoke); 181871fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe} 181971fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe 182071fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampevoid IntrinsicCodeGeneratorX86_64::VisitMemoryPeekLongNative(HInvoke* invoke) { 182171fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe GenPeek(invoke->GetLocations(), Primitive::kPrimLong, GetAssembler()); 182271fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe} 182371fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe 182471fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampevoid IntrinsicLocationsBuilderX86_64::VisitMemoryPeekShortNative(HInvoke* invoke) { 182571fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe CreateIntToIntLocations(arena_, invoke); 182671fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe} 182771fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe 182871fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampevoid IntrinsicCodeGeneratorX86_64::VisitMemoryPeekShortNative(HInvoke* invoke) { 182971fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe GenPeek(invoke->GetLocations(), Primitive::kPrimShort, GetAssembler()); 183071fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe} 183171fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe 183271fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampestatic void CreateIntIntToVoidLocations(ArenaAllocator* arena, HInvoke* invoke) { 183371fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe LocationSummary* locations = new (arena) LocationSummary(invoke, 183471fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe LocationSummary::kNoCall, 183571fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe kIntrinsified); 183671fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe locations->SetInAt(0, Location::RequiresRegister()); 1837ea5af68d6dda832bdfb5978a0c5d6f86a3f67e80Mark Mendell locations->SetInAt(1, Location::RegisterOrInt32Constant(invoke->InputAt(1))); 183871fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe} 183971fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe 184071fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampestatic void GenPoke(LocationSummary* locations, Primitive::Type size, X86_64Assembler* assembler) { 184171fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe CpuRegister address = locations->InAt(0).AsRegister<CpuRegister>(); 184240741f394b2737e503f2c08be0ae9dd490fb106bMark Mendell Location value = locations->InAt(1); 184371fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe // x86 allows unaligned access. We do not have to check the input or use specific instructions 184471fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe // to avoid a SIGBUS. 184571fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe switch (size) { 184671fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe case Primitive::kPrimByte: 184740741f394b2737e503f2c08be0ae9dd490fb106bMark Mendell if (value.IsConstant()) { 184840741f394b2737e503f2c08be0ae9dd490fb106bMark Mendell __ movb(Address(address, 0), 184940741f394b2737e503f2c08be0ae9dd490fb106bMark Mendell Immediate(CodeGenerator::GetInt32ValueOf(value.GetConstant()))); 185040741f394b2737e503f2c08be0ae9dd490fb106bMark Mendell } else { 185140741f394b2737e503f2c08be0ae9dd490fb106bMark Mendell __ movb(Address(address, 0), value.AsRegister<CpuRegister>()); 185240741f394b2737e503f2c08be0ae9dd490fb106bMark Mendell } 185371fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe break; 185471fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe case Primitive::kPrimShort: 185540741f394b2737e503f2c08be0ae9dd490fb106bMark Mendell if (value.IsConstant()) { 185640741f394b2737e503f2c08be0ae9dd490fb106bMark Mendell __ movw(Address(address, 0), 185740741f394b2737e503f2c08be0ae9dd490fb106bMark Mendell Immediate(CodeGenerator::GetInt32ValueOf(value.GetConstant()))); 185840741f394b2737e503f2c08be0ae9dd490fb106bMark Mendell } else { 185940741f394b2737e503f2c08be0ae9dd490fb106bMark Mendell __ movw(Address(address, 0), value.AsRegister<CpuRegister>()); 186040741f394b2737e503f2c08be0ae9dd490fb106bMark Mendell } 186171fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe break; 186271fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe case Primitive::kPrimInt: 186340741f394b2737e503f2c08be0ae9dd490fb106bMark Mendell if (value.IsConstant()) { 186440741f394b2737e503f2c08be0ae9dd490fb106bMark Mendell __ movl(Address(address, 0), 186540741f394b2737e503f2c08be0ae9dd490fb106bMark Mendell Immediate(CodeGenerator::GetInt32ValueOf(value.GetConstant()))); 186640741f394b2737e503f2c08be0ae9dd490fb106bMark Mendell } else { 186740741f394b2737e503f2c08be0ae9dd490fb106bMark Mendell __ movl(Address(address, 0), value.AsRegister<CpuRegister>()); 186840741f394b2737e503f2c08be0ae9dd490fb106bMark Mendell } 186971fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe break; 187071fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe case Primitive::kPrimLong: 187140741f394b2737e503f2c08be0ae9dd490fb106bMark Mendell if (value.IsConstant()) { 187240741f394b2737e503f2c08be0ae9dd490fb106bMark Mendell int64_t v = value.GetConstant()->AsLongConstant()->GetValue(); 187340741f394b2737e503f2c08be0ae9dd490fb106bMark Mendell DCHECK(IsInt<32>(v)); 187440741f394b2737e503f2c08be0ae9dd490fb106bMark Mendell int32_t v_32 = v; 187540741f394b2737e503f2c08be0ae9dd490fb106bMark Mendell __ movq(Address(address, 0), Immediate(v_32)); 187640741f394b2737e503f2c08be0ae9dd490fb106bMark Mendell } else { 187740741f394b2737e503f2c08be0ae9dd490fb106bMark Mendell __ movq(Address(address, 0), value.AsRegister<CpuRegister>()); 187840741f394b2737e503f2c08be0ae9dd490fb106bMark Mendell } 187971fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe break; 188071fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe default: 188171fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe LOG(FATAL) << "Type not recognized for poke: " << size; 188271fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe UNREACHABLE(); 188371fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe } 188471fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe} 188571fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe 188671fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampevoid IntrinsicLocationsBuilderX86_64::VisitMemoryPokeByte(HInvoke* invoke) { 188771fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe CreateIntIntToVoidLocations(arena_, invoke); 188871fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe} 188971fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe 189071fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampevoid IntrinsicCodeGeneratorX86_64::VisitMemoryPokeByte(HInvoke* invoke) { 189171fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe GenPoke(invoke->GetLocations(), Primitive::kPrimByte, GetAssembler()); 189271fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe} 189371fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe 189471fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampevoid IntrinsicLocationsBuilderX86_64::VisitMemoryPokeIntNative(HInvoke* invoke) { 189571fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe CreateIntIntToVoidLocations(arena_, invoke); 189671fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe} 189771fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe 189871fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampevoid IntrinsicCodeGeneratorX86_64::VisitMemoryPokeIntNative(HInvoke* invoke) { 189971fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe GenPoke(invoke->GetLocations(), Primitive::kPrimInt, GetAssembler()); 190071fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe} 190171fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe 190271fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampevoid IntrinsicLocationsBuilderX86_64::VisitMemoryPokeLongNative(HInvoke* invoke) { 190371fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe CreateIntIntToVoidLocations(arena_, invoke); 190471fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe} 190571fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe 190671fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampevoid IntrinsicCodeGeneratorX86_64::VisitMemoryPokeLongNative(HInvoke* invoke) { 190771fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe GenPoke(invoke->GetLocations(), Primitive::kPrimLong, GetAssembler()); 190871fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe} 190971fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe 191071fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampevoid IntrinsicLocationsBuilderX86_64::VisitMemoryPokeShortNative(HInvoke* invoke) { 191171fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe CreateIntIntToVoidLocations(arena_, invoke); 191271fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe} 191371fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe 191471fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampevoid IntrinsicCodeGeneratorX86_64::VisitMemoryPokeShortNative(HInvoke* invoke) { 191571fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe GenPoke(invoke->GetLocations(), Primitive::kPrimShort, GetAssembler()); 191671fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe} 191771fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe 191871fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampevoid IntrinsicLocationsBuilderX86_64::VisitThreadCurrentThread(HInvoke* invoke) { 191971fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe LocationSummary* locations = new (arena_) LocationSummary(invoke, 192071fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe LocationSummary::kNoCall, 192171fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe kIntrinsified); 192271fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe locations->SetOut(Location::RequiresRegister()); 192371fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe} 192471fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe 192571fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampevoid IntrinsicCodeGeneratorX86_64::VisitThreadCurrentThread(HInvoke* invoke) { 192671fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe CpuRegister out = invoke->GetLocations()->Out().AsRegister<CpuRegister>(); 1927bf84a3d2aa29c0975b4ac0f6f983d56724b2cc57Roland Levillain GetAssembler()->gs()->movl(out, Address::Absolute(Thread::PeerOffset<kX86_64WordSize>(), 1928bf84a3d2aa29c0975b4ac0f6f983d56724b2cc57Roland Levillain /* no_rip */ true)); 192971fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe} 193071fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe 19310d5a281c671444bfa75d63caf1427a8c0e6e1177Roland Levillainstatic void GenUnsafeGet(HInvoke* invoke, 19320d5a281c671444bfa75d63caf1427a8c0e6e1177Roland Levillain Primitive::Type type, 19330d5a281c671444bfa75d63caf1427a8c0e6e1177Roland Levillain bool is_volatile ATTRIBUTE_UNUSED, 19340d5a281c671444bfa75d63caf1427a8c0e6e1177Roland Levillain CodeGeneratorX86_64* codegen) { 19350d5a281c671444bfa75d63caf1427a8c0e6e1177Roland Levillain X86_64Assembler* assembler = down_cast<X86_64Assembler*>(codegen->GetAssembler()); 19360d5a281c671444bfa75d63caf1427a8c0e6e1177Roland Levillain LocationSummary* locations = invoke->GetLocations(); 19370d5a281c671444bfa75d63caf1427a8c0e6e1177Roland Levillain Location base_loc = locations->InAt(1); 19380d5a281c671444bfa75d63caf1427a8c0e6e1177Roland Levillain CpuRegister base = base_loc.AsRegister<CpuRegister>(); 19390d5a281c671444bfa75d63caf1427a8c0e6e1177Roland Levillain Location offset_loc = locations->InAt(2); 19400d5a281c671444bfa75d63caf1427a8c0e6e1177Roland Levillain CpuRegister offset = offset_loc.AsRegister<CpuRegister>(); 19410d5a281c671444bfa75d63caf1427a8c0e6e1177Roland Levillain Location output_loc = locations->Out(); 19421e7f8db01a929ac816ca122868edc067c3c6cd17Roland Levillain CpuRegister output = output_loc.AsRegister<CpuRegister>(); 194371fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe 1944878d58cbaf6b17a9e3dcab790754527f3ebc69e5Andreas Gampe switch (type) { 1945878d58cbaf6b17a9e3dcab790754527f3ebc69e5Andreas Gampe case Primitive::kPrimInt: 19460d5a281c671444bfa75d63caf1427a8c0e6e1177Roland Levillain __ movl(output, Address(base, offset, ScaleFactor::TIMES_1, 0)); 19471e7f8db01a929ac816ca122868edc067c3c6cd17Roland Levillain break; 19481e7f8db01a929ac816ca122868edc067c3c6cd17Roland Levillain 19491e7f8db01a929ac816ca122868edc067c3c6cd17Roland Levillain case Primitive::kPrimNot: { 19501e7f8db01a929ac816ca122868edc067c3c6cd17Roland Levillain if (kEmitCompilerReadBarrier) { 19511e7f8db01a929ac816ca122868edc067c3c6cd17Roland Levillain if (kUseBakerReadBarrier) { 19521e7f8db01a929ac816ca122868edc067c3c6cd17Roland Levillain Location temp = locations->GetTemp(0); 19531e7f8db01a929ac816ca122868edc067c3c6cd17Roland Levillain codegen->GenerateArrayLoadWithBakerReadBarrier( 19541e7f8db01a929ac816ca122868edc067c3c6cd17Roland Levillain invoke, output_loc, base, 0U, offset_loc, temp, /* needs_null_check */ false); 19551e7f8db01a929ac816ca122868edc067c3c6cd17Roland Levillain } else { 19561e7f8db01a929ac816ca122868edc067c3c6cd17Roland Levillain __ movl(output, Address(base, offset, ScaleFactor::TIMES_1, 0)); 19571e7f8db01a929ac816ca122868edc067c3c6cd17Roland Levillain codegen->GenerateReadBarrierSlow( 19581e7f8db01a929ac816ca122868edc067c3c6cd17Roland Levillain invoke, output_loc, output_loc, base_loc, 0U, offset_loc); 19591e7f8db01a929ac816ca122868edc067c3c6cd17Roland Levillain } 19601e7f8db01a929ac816ca122868edc067c3c6cd17Roland Levillain } else { 19611e7f8db01a929ac816ca122868edc067c3c6cd17Roland Levillain __ movl(output, Address(base, offset, ScaleFactor::TIMES_1, 0)); 19621e7f8db01a929ac816ca122868edc067c3c6cd17Roland Levillain __ MaybeUnpoisonHeapReference(output); 19634d02711ea578dbb789abb30cbaf12f9926e13d81Roland Levillain } 1964878d58cbaf6b17a9e3dcab790754527f3ebc69e5Andreas Gampe break; 19651e7f8db01a929ac816ca122868edc067c3c6cd17Roland Levillain } 1966878d58cbaf6b17a9e3dcab790754527f3ebc69e5Andreas Gampe 1967878d58cbaf6b17a9e3dcab790754527f3ebc69e5Andreas Gampe case Primitive::kPrimLong: 19680d5a281c671444bfa75d63caf1427a8c0e6e1177Roland Levillain __ movq(output, Address(base, offset, ScaleFactor::TIMES_1, 0)); 1969878d58cbaf6b17a9e3dcab790754527f3ebc69e5Andreas Gampe break; 1970878d58cbaf6b17a9e3dcab790754527f3ebc69e5Andreas Gampe 1971878d58cbaf6b17a9e3dcab790754527f3ebc69e5Andreas Gampe default: 1972878d58cbaf6b17a9e3dcab790754527f3ebc69e5Andreas Gampe LOG(FATAL) << "Unsupported op size " << type; 1973878d58cbaf6b17a9e3dcab790754527f3ebc69e5Andreas Gampe UNREACHABLE(); 197471fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe } 197571fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe} 197671fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe 19771e7f8db01a929ac816ca122868edc067c3c6cd17Roland Levillainstatic void CreateIntIntIntToIntLocations(ArenaAllocator* arena, 19781e7f8db01a929ac816ca122868edc067c3c6cd17Roland Levillain HInvoke* invoke, 19791e7f8db01a929ac816ca122868edc067c3c6cd17Roland Levillain Primitive::Type type) { 19800d5a281c671444bfa75d63caf1427a8c0e6e1177Roland Levillain bool can_call = kEmitCompilerReadBarrier && 19810d5a281c671444bfa75d63caf1427a8c0e6e1177Roland Levillain (invoke->GetIntrinsic() == Intrinsics::kUnsafeGetObject || 19820d5a281c671444bfa75d63caf1427a8c0e6e1177Roland Levillain invoke->GetIntrinsic() == Intrinsics::kUnsafeGetObjectVolatile); 198371fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe LocationSummary* locations = new (arena) LocationSummary(invoke, 19840d5a281c671444bfa75d63caf1427a8c0e6e1177Roland Levillain can_call ? 19850d5a281c671444bfa75d63caf1427a8c0e6e1177Roland Levillain LocationSummary::kCallOnSlowPath : 19860d5a281c671444bfa75d63caf1427a8c0e6e1177Roland Levillain LocationSummary::kNoCall, 198771fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe kIntrinsified); 1988878d58cbaf6b17a9e3dcab790754527f3ebc69e5Andreas Gampe locations->SetInAt(0, Location::NoLocation()); // Unused receiver. 198971fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe locations->SetInAt(1, Location::RequiresRegister()); 199071fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe locations->SetInAt(2, Location::RequiresRegister()); 1991878d58cbaf6b17a9e3dcab790754527f3ebc69e5Andreas Gampe locations->SetOut(Location::RequiresRegister()); 19921e7f8db01a929ac816ca122868edc067c3c6cd17Roland Levillain if (type == Primitive::kPrimNot && kEmitCompilerReadBarrier && kUseBakerReadBarrier) { 19931e7f8db01a929ac816ca122868edc067c3c6cd17Roland Levillain // We need a temporary register for the read barrier marking slow 19941e7f8db01a929ac816ca122868edc067c3c6cd17Roland Levillain // path in InstructionCodeGeneratorX86_64::GenerateArrayLoadWithBakerReadBarrier. 19951e7f8db01a929ac816ca122868edc067c3c6cd17Roland Levillain locations->AddTemp(Location::RequiresRegister()); 19961e7f8db01a929ac816ca122868edc067c3c6cd17Roland Levillain } 199771fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe} 199871fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe 199971fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampevoid IntrinsicLocationsBuilderX86_64::VisitUnsafeGet(HInvoke* invoke) { 20001e7f8db01a929ac816ca122868edc067c3c6cd17Roland Levillain CreateIntIntIntToIntLocations(arena_, invoke, Primitive::kPrimInt); 200171fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe} 200271fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampevoid IntrinsicLocationsBuilderX86_64::VisitUnsafeGetVolatile(HInvoke* invoke) { 20031e7f8db01a929ac816ca122868edc067c3c6cd17Roland Levillain CreateIntIntIntToIntLocations(arena_, invoke, Primitive::kPrimInt); 200471fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe} 200571fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampevoid IntrinsicLocationsBuilderX86_64::VisitUnsafeGetLong(HInvoke* invoke) { 20061e7f8db01a929ac816ca122868edc067c3c6cd17Roland Levillain CreateIntIntIntToIntLocations(arena_, invoke, Primitive::kPrimLong); 200771fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe} 200871fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampevoid IntrinsicLocationsBuilderX86_64::VisitUnsafeGetLongVolatile(HInvoke* invoke) { 20091e7f8db01a929ac816ca122868edc067c3c6cd17Roland Levillain CreateIntIntIntToIntLocations(arena_, invoke, Primitive::kPrimLong); 201071fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe} 2011878d58cbaf6b17a9e3dcab790754527f3ebc69e5Andreas Gampevoid IntrinsicLocationsBuilderX86_64::VisitUnsafeGetObject(HInvoke* invoke) { 20121e7f8db01a929ac816ca122868edc067c3c6cd17Roland Levillain CreateIntIntIntToIntLocations(arena_, invoke, Primitive::kPrimNot); 2013878d58cbaf6b17a9e3dcab790754527f3ebc69e5Andreas Gampe} 2014878d58cbaf6b17a9e3dcab790754527f3ebc69e5Andreas Gampevoid IntrinsicLocationsBuilderX86_64::VisitUnsafeGetObjectVolatile(HInvoke* invoke) { 20151e7f8db01a929ac816ca122868edc067c3c6cd17Roland Levillain CreateIntIntIntToIntLocations(arena_, invoke, Primitive::kPrimNot); 2016878d58cbaf6b17a9e3dcab790754527f3ebc69e5Andreas Gampe} 2017878d58cbaf6b17a9e3dcab790754527f3ebc69e5Andreas Gampe 201871fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe 201971fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampevoid IntrinsicCodeGeneratorX86_64::VisitUnsafeGet(HInvoke* invoke) { 2020bf84a3d2aa29c0975b4ac0f6f983d56724b2cc57Roland Levillain GenUnsafeGet(invoke, Primitive::kPrimInt, /* is_volatile */ false, codegen_); 202171fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe} 202271fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampevoid IntrinsicCodeGeneratorX86_64::VisitUnsafeGetVolatile(HInvoke* invoke) { 2023bf84a3d2aa29c0975b4ac0f6f983d56724b2cc57Roland Levillain GenUnsafeGet(invoke, Primitive::kPrimInt, /* is_volatile */ true, codegen_); 202471fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe} 202571fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampevoid IntrinsicCodeGeneratorX86_64::VisitUnsafeGetLong(HInvoke* invoke) { 2026bf84a3d2aa29c0975b4ac0f6f983d56724b2cc57Roland Levillain GenUnsafeGet(invoke, Primitive::kPrimLong, /* is_volatile */ false, codegen_); 202771fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe} 202871fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampevoid IntrinsicCodeGeneratorX86_64::VisitUnsafeGetLongVolatile(HInvoke* invoke) { 2029bf84a3d2aa29c0975b4ac0f6f983d56724b2cc57Roland Levillain GenUnsafeGet(invoke, Primitive::kPrimLong, /* is_volatile */ true, codegen_); 203071fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe} 2031878d58cbaf6b17a9e3dcab790754527f3ebc69e5Andreas Gampevoid IntrinsicCodeGeneratorX86_64::VisitUnsafeGetObject(HInvoke* invoke) { 2032bf84a3d2aa29c0975b4ac0f6f983d56724b2cc57Roland Levillain GenUnsafeGet(invoke, Primitive::kPrimNot, /* is_volatile */ false, codegen_); 2033878d58cbaf6b17a9e3dcab790754527f3ebc69e5Andreas Gampe} 2034878d58cbaf6b17a9e3dcab790754527f3ebc69e5Andreas Gampevoid IntrinsicCodeGeneratorX86_64::VisitUnsafeGetObjectVolatile(HInvoke* invoke) { 2035bf84a3d2aa29c0975b4ac0f6f983d56724b2cc57Roland Levillain GenUnsafeGet(invoke, Primitive::kPrimNot, /* is_volatile */ true, codegen_); 2036878d58cbaf6b17a9e3dcab790754527f3ebc69e5Andreas Gampe} 2037878d58cbaf6b17a9e3dcab790754527f3ebc69e5Andreas Gampe 203871fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe 203971fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampestatic void CreateIntIntIntIntToVoidPlusTempsLocations(ArenaAllocator* arena, 204071fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe Primitive::Type type, 204171fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe HInvoke* invoke) { 204271fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe LocationSummary* locations = new (arena) LocationSummary(invoke, 204371fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe LocationSummary::kNoCall, 204471fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe kIntrinsified); 2045878d58cbaf6b17a9e3dcab790754527f3ebc69e5Andreas Gampe locations->SetInAt(0, Location::NoLocation()); // Unused receiver. 204671fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe locations->SetInAt(1, Location::RequiresRegister()); 204771fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe locations->SetInAt(2, Location::RequiresRegister()); 204871fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe locations->SetInAt(3, Location::RequiresRegister()); 204971fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe if (type == Primitive::kPrimNot) { 205071fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe // Need temp registers for card-marking. 20514d02711ea578dbb789abb30cbaf12f9926e13d81Roland Levillain locations->AddTemp(Location::RequiresRegister()); // Possibly used for reference poisoning too. 205271fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe locations->AddTemp(Location::RequiresRegister()); 205371fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe } 205471fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe} 205571fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe 205671fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampevoid IntrinsicLocationsBuilderX86_64::VisitUnsafePut(HInvoke* invoke) { 205771fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe CreateIntIntIntIntToVoidPlusTempsLocations(arena_, Primitive::kPrimInt, invoke); 205871fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe} 205971fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampevoid IntrinsicLocationsBuilderX86_64::VisitUnsafePutOrdered(HInvoke* invoke) { 206071fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe CreateIntIntIntIntToVoidPlusTempsLocations(arena_, Primitive::kPrimInt, invoke); 206171fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe} 206271fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampevoid IntrinsicLocationsBuilderX86_64::VisitUnsafePutVolatile(HInvoke* invoke) { 206371fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe CreateIntIntIntIntToVoidPlusTempsLocations(arena_, Primitive::kPrimInt, invoke); 206471fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe} 206571fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampevoid IntrinsicLocationsBuilderX86_64::VisitUnsafePutObject(HInvoke* invoke) { 206671fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe CreateIntIntIntIntToVoidPlusTempsLocations(arena_, Primitive::kPrimNot, invoke); 206771fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe} 206871fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampevoid IntrinsicLocationsBuilderX86_64::VisitUnsafePutObjectOrdered(HInvoke* invoke) { 206971fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe CreateIntIntIntIntToVoidPlusTempsLocations(arena_, Primitive::kPrimNot, invoke); 207071fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe} 207171fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampevoid IntrinsicLocationsBuilderX86_64::VisitUnsafePutObjectVolatile(HInvoke* invoke) { 207271fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe CreateIntIntIntIntToVoidPlusTempsLocations(arena_, Primitive::kPrimNot, invoke); 207371fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe} 207471fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampevoid IntrinsicLocationsBuilderX86_64::VisitUnsafePutLong(HInvoke* invoke) { 207571fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe CreateIntIntIntIntToVoidPlusTempsLocations(arena_, Primitive::kPrimLong, invoke); 207671fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe} 207771fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampevoid IntrinsicLocationsBuilderX86_64::VisitUnsafePutLongOrdered(HInvoke* invoke) { 207871fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe CreateIntIntIntIntToVoidPlusTempsLocations(arena_, Primitive::kPrimLong, invoke); 207971fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe} 208071fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampevoid IntrinsicLocationsBuilderX86_64::VisitUnsafePutLongVolatile(HInvoke* invoke) { 208171fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe CreateIntIntIntIntToVoidPlusTempsLocations(arena_, Primitive::kPrimLong, invoke); 208271fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe} 208371fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe 208471fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe// We don't care for ordered: it requires an AnyStore barrier, which is already given by the x86 208571fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe// memory model. 208671fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampestatic void GenUnsafePut(LocationSummary* locations, Primitive::Type type, bool is_volatile, 208771fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe CodeGeneratorX86_64* codegen) { 2088b488b7864b7bf9cade82d45c8bdda2372f48a10cRoland Levillain X86_64Assembler* assembler = down_cast<X86_64Assembler*>(codegen->GetAssembler()); 208971fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe CpuRegister base = locations->InAt(1).AsRegister<CpuRegister>(); 209071fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe CpuRegister offset = locations->InAt(2).AsRegister<CpuRegister>(); 209171fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe CpuRegister value = locations->InAt(3).AsRegister<CpuRegister>(); 209271fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe 209371fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe if (type == Primitive::kPrimLong) { 209471fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe __ movq(Address(base, offset, ScaleFactor::TIMES_1, 0), value); 20954d02711ea578dbb789abb30cbaf12f9926e13d81Roland Levillain } else if (kPoisonHeapReferences && type == Primitive::kPrimNot) { 20964d02711ea578dbb789abb30cbaf12f9926e13d81Roland Levillain CpuRegister temp = locations->GetTemp(0).AsRegister<CpuRegister>(); 20974d02711ea578dbb789abb30cbaf12f9926e13d81Roland Levillain __ movl(temp, value); 20984d02711ea578dbb789abb30cbaf12f9926e13d81Roland Levillain __ PoisonHeapReference(temp); 20994d02711ea578dbb789abb30cbaf12f9926e13d81Roland Levillain __ movl(Address(base, offset, ScaleFactor::TIMES_1, 0), temp); 210071fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe } else { 210171fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe __ movl(Address(base, offset, ScaleFactor::TIMES_1, 0), value); 210271fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe } 210371fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe 210471fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe if (is_volatile) { 210517077d888a6752a2e5f8161eee1b2c3285783d12Mark P Mendell codegen->MemoryFence(); 210671fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe } 210771fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe 210871fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe if (type == Primitive::kPrimNot) { 210907276db28d654594e0e86e9e467cad393f752e6eNicolas Geoffray bool value_can_be_null = true; // TODO: Worth finding out this information? 211071fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe codegen->MarkGCCard(locations->GetTemp(0).AsRegister<CpuRegister>(), 211171fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe locations->GetTemp(1).AsRegister<CpuRegister>(), 211271fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe base, 211307276db28d654594e0e86e9e467cad393f752e6eNicolas Geoffray value, 211407276db28d654594e0e86e9e467cad393f752e6eNicolas Geoffray value_can_be_null); 211571fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe } 211671fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe} 211771fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe 211871fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampevoid IntrinsicCodeGeneratorX86_64::VisitUnsafePut(HInvoke* invoke) { 2119bf84a3d2aa29c0975b4ac0f6f983d56724b2cc57Roland Levillain GenUnsafePut(invoke->GetLocations(), Primitive::kPrimInt, /* is_volatile */ false, codegen_); 212071fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe} 212171fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampevoid IntrinsicCodeGeneratorX86_64::VisitUnsafePutOrdered(HInvoke* invoke) { 2122bf84a3d2aa29c0975b4ac0f6f983d56724b2cc57Roland Levillain GenUnsafePut(invoke->GetLocations(), Primitive::kPrimInt, /* is_volatile */ false, codegen_); 212371fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe} 212471fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampevoid IntrinsicCodeGeneratorX86_64::VisitUnsafePutVolatile(HInvoke* invoke) { 2125bf84a3d2aa29c0975b4ac0f6f983d56724b2cc57Roland Levillain GenUnsafePut(invoke->GetLocations(), Primitive::kPrimInt, /* is_volatile */ true, codegen_); 212671fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe} 212771fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampevoid IntrinsicCodeGeneratorX86_64::VisitUnsafePutObject(HInvoke* invoke) { 2128bf84a3d2aa29c0975b4ac0f6f983d56724b2cc57Roland Levillain GenUnsafePut(invoke->GetLocations(), Primitive::kPrimNot, /* is_volatile */ false, codegen_); 212971fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe} 213071fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampevoid IntrinsicCodeGeneratorX86_64::VisitUnsafePutObjectOrdered(HInvoke* invoke) { 2131bf84a3d2aa29c0975b4ac0f6f983d56724b2cc57Roland Levillain GenUnsafePut(invoke->GetLocations(), Primitive::kPrimNot, /* is_volatile */ false, codegen_); 213271fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe} 213371fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampevoid IntrinsicCodeGeneratorX86_64::VisitUnsafePutObjectVolatile(HInvoke* invoke) { 2134bf84a3d2aa29c0975b4ac0f6f983d56724b2cc57Roland Levillain GenUnsafePut(invoke->GetLocations(), Primitive::kPrimNot, /* is_volatile */ true, codegen_); 213571fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe} 213671fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampevoid IntrinsicCodeGeneratorX86_64::VisitUnsafePutLong(HInvoke* invoke) { 2137bf84a3d2aa29c0975b4ac0f6f983d56724b2cc57Roland Levillain GenUnsafePut(invoke->GetLocations(), Primitive::kPrimLong, /* is_volatile */ false, codegen_); 213871fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe} 213971fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampevoid IntrinsicCodeGeneratorX86_64::VisitUnsafePutLongOrdered(HInvoke* invoke) { 2140bf84a3d2aa29c0975b4ac0f6f983d56724b2cc57Roland Levillain GenUnsafePut(invoke->GetLocations(), Primitive::kPrimLong, /* is_volatile */ false, codegen_); 214171fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe} 214271fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampevoid IntrinsicCodeGeneratorX86_64::VisitUnsafePutLongVolatile(HInvoke* invoke) { 2143bf84a3d2aa29c0975b4ac0f6f983d56724b2cc57Roland Levillain GenUnsafePut(invoke->GetLocations(), Primitive::kPrimLong, /* is_volatile */ true, codegen_); 214471fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe} 214571fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe 214658d25fd052e999a24734b0cf856a1563e3d1b2d0Mark Mendellstatic void CreateIntIntIntIntIntToInt(ArenaAllocator* arena, Primitive::Type type, 214758d25fd052e999a24734b0cf856a1563e3d1b2d0Mark Mendell HInvoke* invoke) { 214858d25fd052e999a24734b0cf856a1563e3d1b2d0Mark Mendell LocationSummary* locations = new (arena) LocationSummary(invoke, 214958d25fd052e999a24734b0cf856a1563e3d1b2d0Mark Mendell LocationSummary::kNoCall, 215058d25fd052e999a24734b0cf856a1563e3d1b2d0Mark Mendell kIntrinsified); 215158d25fd052e999a24734b0cf856a1563e3d1b2d0Mark Mendell locations->SetInAt(0, Location::NoLocation()); // Unused receiver. 215258d25fd052e999a24734b0cf856a1563e3d1b2d0Mark Mendell locations->SetInAt(1, Location::RequiresRegister()); 215358d25fd052e999a24734b0cf856a1563e3d1b2d0Mark Mendell locations->SetInAt(2, Location::RequiresRegister()); 215458d25fd052e999a24734b0cf856a1563e3d1b2d0Mark Mendell // expected value must be in EAX/RAX. 215558d25fd052e999a24734b0cf856a1563e3d1b2d0Mark Mendell locations->SetInAt(3, Location::RegisterLocation(RAX)); 215658d25fd052e999a24734b0cf856a1563e3d1b2d0Mark Mendell locations->SetInAt(4, Location::RequiresRegister()); 215758d25fd052e999a24734b0cf856a1563e3d1b2d0Mark Mendell 215858d25fd052e999a24734b0cf856a1563e3d1b2d0Mark Mendell locations->SetOut(Location::RequiresRegister()); 215958d25fd052e999a24734b0cf856a1563e3d1b2d0Mark Mendell if (type == Primitive::kPrimNot) { 216058d25fd052e999a24734b0cf856a1563e3d1b2d0Mark Mendell // Need temp registers for card-marking. 2161b488b7864b7bf9cade82d45c8bdda2372f48a10cRoland Levillain locations->AddTemp(Location::RequiresRegister()); // Possibly used for reference poisoning too. 216258d25fd052e999a24734b0cf856a1563e3d1b2d0Mark Mendell locations->AddTemp(Location::RequiresRegister()); 216358d25fd052e999a24734b0cf856a1563e3d1b2d0Mark Mendell } 216458d25fd052e999a24734b0cf856a1563e3d1b2d0Mark Mendell} 216558d25fd052e999a24734b0cf856a1563e3d1b2d0Mark Mendell 216658d25fd052e999a24734b0cf856a1563e3d1b2d0Mark Mendellvoid IntrinsicLocationsBuilderX86_64::VisitUnsafeCASInt(HInvoke* invoke) { 216758d25fd052e999a24734b0cf856a1563e3d1b2d0Mark Mendell CreateIntIntIntIntIntToInt(arena_, Primitive::kPrimInt, invoke); 216858d25fd052e999a24734b0cf856a1563e3d1b2d0Mark Mendell} 216958d25fd052e999a24734b0cf856a1563e3d1b2d0Mark Mendell 217058d25fd052e999a24734b0cf856a1563e3d1b2d0Mark Mendellvoid IntrinsicLocationsBuilderX86_64::VisitUnsafeCASLong(HInvoke* invoke) { 217158d25fd052e999a24734b0cf856a1563e3d1b2d0Mark Mendell CreateIntIntIntIntIntToInt(arena_, Primitive::kPrimLong, invoke); 217258d25fd052e999a24734b0cf856a1563e3d1b2d0Mark Mendell} 217358d25fd052e999a24734b0cf856a1563e3d1b2d0Mark Mendell 217458d25fd052e999a24734b0cf856a1563e3d1b2d0Mark Mendellvoid IntrinsicLocationsBuilderX86_64::VisitUnsafeCASObject(HInvoke* invoke) { 2175391b866ce55b8e78b1f9a6b98321d837256e8d66Roland Levillain // The UnsafeCASObject intrinsic is missing a read barrier, and 2176391b866ce55b8e78b1f9a6b98321d837256e8d66Roland Levillain // therefore sometimes does not work as expected (b/25883050). 2177391b866ce55b8e78b1f9a6b98321d837256e8d66Roland Levillain // Turn it off temporarily as a quick fix, until the read barrier is 2178391b866ce55b8e78b1f9a6b98321d837256e8d66Roland Levillain // implemented. 2179391b866ce55b8e78b1f9a6b98321d837256e8d66Roland Levillain // 2180391b866ce55b8e78b1f9a6b98321d837256e8d66Roland Levillain // TODO(rpl): Implement a read barrier in GenCAS below and re-enable 2181391b866ce55b8e78b1f9a6b98321d837256e8d66Roland Levillain // this intrinsic. 2182391b866ce55b8e78b1f9a6b98321d837256e8d66Roland Levillain if (kEmitCompilerReadBarrier) { 2183391b866ce55b8e78b1f9a6b98321d837256e8d66Roland Levillain return; 2184391b866ce55b8e78b1f9a6b98321d837256e8d66Roland Levillain } 2185391b866ce55b8e78b1f9a6b98321d837256e8d66Roland Levillain 218658d25fd052e999a24734b0cf856a1563e3d1b2d0Mark Mendell CreateIntIntIntIntIntToInt(arena_, Primitive::kPrimNot, invoke); 218758d25fd052e999a24734b0cf856a1563e3d1b2d0Mark Mendell} 218858d25fd052e999a24734b0cf856a1563e3d1b2d0Mark Mendell 218958d25fd052e999a24734b0cf856a1563e3d1b2d0Mark Mendellstatic void GenCAS(Primitive::Type type, HInvoke* invoke, CodeGeneratorX86_64* codegen) { 2190b488b7864b7bf9cade82d45c8bdda2372f48a10cRoland Levillain X86_64Assembler* assembler = down_cast<X86_64Assembler*>(codegen->GetAssembler()); 219158d25fd052e999a24734b0cf856a1563e3d1b2d0Mark Mendell LocationSummary* locations = invoke->GetLocations(); 219258d25fd052e999a24734b0cf856a1563e3d1b2d0Mark Mendell 219358d25fd052e999a24734b0cf856a1563e3d1b2d0Mark Mendell CpuRegister base = locations->InAt(1).AsRegister<CpuRegister>(); 219458d25fd052e999a24734b0cf856a1563e3d1b2d0Mark Mendell CpuRegister offset = locations->InAt(2).AsRegister<CpuRegister>(); 219558d25fd052e999a24734b0cf856a1563e3d1b2d0Mark Mendell CpuRegister expected = locations->InAt(3).AsRegister<CpuRegister>(); 2196b488b7864b7bf9cade82d45c8bdda2372f48a10cRoland Levillain // Ensure `expected` is in RAX (required by the CMPXCHG instruction). 219758d25fd052e999a24734b0cf856a1563e3d1b2d0Mark Mendell DCHECK_EQ(expected.AsRegister(), RAX); 219858d25fd052e999a24734b0cf856a1563e3d1b2d0Mark Mendell CpuRegister value = locations->InAt(4).AsRegister<CpuRegister>(); 219958d25fd052e999a24734b0cf856a1563e3d1b2d0Mark Mendell CpuRegister out = locations->Out().AsRegister<CpuRegister>(); 220058d25fd052e999a24734b0cf856a1563e3d1b2d0Mark Mendell 2201b488b7864b7bf9cade82d45c8bdda2372f48a10cRoland Levillain if (type == Primitive::kPrimNot) { 2202b488b7864b7bf9cade82d45c8bdda2372f48a10cRoland Levillain // Mark card for object assuming new value is stored. 2203b488b7864b7bf9cade82d45c8bdda2372f48a10cRoland Levillain bool value_can_be_null = true; // TODO: Worth finding out this information? 2204b488b7864b7bf9cade82d45c8bdda2372f48a10cRoland Levillain codegen->MarkGCCard(locations->GetTemp(0).AsRegister<CpuRegister>(), 2205b488b7864b7bf9cade82d45c8bdda2372f48a10cRoland Levillain locations->GetTemp(1).AsRegister<CpuRegister>(), 2206b488b7864b7bf9cade82d45c8bdda2372f48a10cRoland Levillain base, 2207b488b7864b7bf9cade82d45c8bdda2372f48a10cRoland Levillain value, 2208b488b7864b7bf9cade82d45c8bdda2372f48a10cRoland Levillain value_can_be_null); 2209b488b7864b7bf9cade82d45c8bdda2372f48a10cRoland Levillain 2210b488b7864b7bf9cade82d45c8bdda2372f48a10cRoland Levillain bool base_equals_value = (base.AsRegister() == value.AsRegister()); 2211b488b7864b7bf9cade82d45c8bdda2372f48a10cRoland Levillain Register value_reg = value.AsRegister(); 2212b488b7864b7bf9cade82d45c8bdda2372f48a10cRoland Levillain if (kPoisonHeapReferences) { 2213b488b7864b7bf9cade82d45c8bdda2372f48a10cRoland Levillain if (base_equals_value) { 2214b488b7864b7bf9cade82d45c8bdda2372f48a10cRoland Levillain // If `base` and `value` are the same register location, move 2215b488b7864b7bf9cade82d45c8bdda2372f48a10cRoland Levillain // `value_reg` to a temporary register. This way, poisoning 2216b488b7864b7bf9cade82d45c8bdda2372f48a10cRoland Levillain // `value_reg` won't invalidate `base`. 2217b488b7864b7bf9cade82d45c8bdda2372f48a10cRoland Levillain value_reg = locations->GetTemp(0).AsRegister<CpuRegister>().AsRegister(); 2218b488b7864b7bf9cade82d45c8bdda2372f48a10cRoland Levillain __ movl(CpuRegister(value_reg), base); 22194d02711ea578dbb789abb30cbaf12f9926e13d81Roland Levillain } 2220b488b7864b7bf9cade82d45c8bdda2372f48a10cRoland Levillain 2221b488b7864b7bf9cade82d45c8bdda2372f48a10cRoland Levillain // Check that the register allocator did not assign the location 2222b488b7864b7bf9cade82d45c8bdda2372f48a10cRoland Levillain // of `expected` (RAX) to `value` nor to `base`, so that heap 2223b488b7864b7bf9cade82d45c8bdda2372f48a10cRoland Levillain // poisoning (when enabled) works as intended below. 2224b488b7864b7bf9cade82d45c8bdda2372f48a10cRoland Levillain // - If `value` were equal to `expected`, both references would 2225b488b7864b7bf9cade82d45c8bdda2372f48a10cRoland Levillain // be poisoned twice, meaning they would not be poisoned at 2226b488b7864b7bf9cade82d45c8bdda2372f48a10cRoland Levillain // all, as heap poisoning uses address negation. 2227b488b7864b7bf9cade82d45c8bdda2372f48a10cRoland Levillain // - If `base` were equal to `expected`, poisoning `expected` 2228b488b7864b7bf9cade82d45c8bdda2372f48a10cRoland Levillain // would invalidate `base`. 2229b488b7864b7bf9cade82d45c8bdda2372f48a10cRoland Levillain DCHECK_NE(value_reg, expected.AsRegister()); 2230b488b7864b7bf9cade82d45c8bdda2372f48a10cRoland Levillain DCHECK_NE(base.AsRegister(), expected.AsRegister()); 2231b488b7864b7bf9cade82d45c8bdda2372f48a10cRoland Levillain 2232b488b7864b7bf9cade82d45c8bdda2372f48a10cRoland Levillain __ PoisonHeapReference(expected); 2233b488b7864b7bf9cade82d45c8bdda2372f48a10cRoland Levillain __ PoisonHeapReference(CpuRegister(value_reg)); 223458d25fd052e999a24734b0cf856a1563e3d1b2d0Mark Mendell } 223558d25fd052e999a24734b0cf856a1563e3d1b2d0Mark Mendell 2236391b866ce55b8e78b1f9a6b98321d837256e8d66Roland Levillain // TODO: Add a read barrier for the reference stored in the object 2237391b866ce55b8e78b1f9a6b98321d837256e8d66Roland Levillain // before attempting the CAS, similar to the one in the 2238391b866ce55b8e78b1f9a6b98321d837256e8d66Roland Levillain // art::Unsafe_compareAndSwapObject JNI implementation. 2239391b866ce55b8e78b1f9a6b98321d837256e8d66Roland Levillain // 2240391b866ce55b8e78b1f9a6b98321d837256e8d66Roland Levillain // Note that this code is not (yet) used when read barriers are 2241391b866ce55b8e78b1f9a6b98321d837256e8d66Roland Levillain // enabled (see IntrinsicLocationsBuilderX86_64::VisitUnsafeCASObject). 2242391b866ce55b8e78b1f9a6b98321d837256e8d66Roland Levillain DCHECK(!kEmitCompilerReadBarrier); 2243b488b7864b7bf9cade82d45c8bdda2372f48a10cRoland Levillain __ LockCmpxchgl(Address(base, offset, TIMES_1, 0), CpuRegister(value_reg)); 2244b488b7864b7bf9cade82d45c8bdda2372f48a10cRoland Levillain 22450d5a281c671444bfa75d63caf1427a8c0e6e1177Roland Levillain // LOCK CMPXCHG has full barrier semantics, and we don't need 2246b488b7864b7bf9cade82d45c8bdda2372f48a10cRoland Levillain // scheduling barriers at this time. 2247b488b7864b7bf9cade82d45c8bdda2372f48a10cRoland Levillain 2248b488b7864b7bf9cade82d45c8bdda2372f48a10cRoland Levillain // Convert ZF into the boolean result. 2249b488b7864b7bf9cade82d45c8bdda2372f48a10cRoland Levillain __ setcc(kZero, out); 2250b488b7864b7bf9cade82d45c8bdda2372f48a10cRoland Levillain __ movzxb(out, out); 225158d25fd052e999a24734b0cf856a1563e3d1b2d0Mark Mendell 2252391b866ce55b8e78b1f9a6b98321d837256e8d66Roland Levillain // If heap poisoning is enabled, we need to unpoison the values 2253391b866ce55b8e78b1f9a6b98321d837256e8d66Roland Levillain // that were poisoned earlier. 2254b488b7864b7bf9cade82d45c8bdda2372f48a10cRoland Levillain if (kPoisonHeapReferences) { 2255b488b7864b7bf9cade82d45c8bdda2372f48a10cRoland Levillain if (base_equals_value) { 2256b488b7864b7bf9cade82d45c8bdda2372f48a10cRoland Levillain // `value_reg` has been moved to a temporary register, no need 2257b488b7864b7bf9cade82d45c8bdda2372f48a10cRoland Levillain // to unpoison it. 2258b488b7864b7bf9cade82d45c8bdda2372f48a10cRoland Levillain } else { 2259b488b7864b7bf9cade82d45c8bdda2372f48a10cRoland Levillain // Ensure `value` is different from `out`, so that unpoisoning 2260b488b7864b7bf9cade82d45c8bdda2372f48a10cRoland Levillain // the former does not invalidate the latter. 2261b488b7864b7bf9cade82d45c8bdda2372f48a10cRoland Levillain DCHECK_NE(value_reg, out.AsRegister()); 2262b488b7864b7bf9cade82d45c8bdda2372f48a10cRoland Levillain __ UnpoisonHeapReference(CpuRegister(value_reg)); 2263b488b7864b7bf9cade82d45c8bdda2372f48a10cRoland Levillain } 2264b488b7864b7bf9cade82d45c8bdda2372f48a10cRoland Levillain // Ensure `expected` is different from `out`, so that unpoisoning 2265b488b7864b7bf9cade82d45c8bdda2372f48a10cRoland Levillain // the former does not invalidate the latter. 2266b488b7864b7bf9cade82d45c8bdda2372f48a10cRoland Levillain DCHECK_NE(expected.AsRegister(), out.AsRegister()); 2267b488b7864b7bf9cade82d45c8bdda2372f48a10cRoland Levillain __ UnpoisonHeapReference(expected); 2268b488b7864b7bf9cade82d45c8bdda2372f48a10cRoland Levillain } 2269b488b7864b7bf9cade82d45c8bdda2372f48a10cRoland Levillain } else { 2270b488b7864b7bf9cade82d45c8bdda2372f48a10cRoland Levillain if (type == Primitive::kPrimInt) { 2271b488b7864b7bf9cade82d45c8bdda2372f48a10cRoland Levillain __ LockCmpxchgl(Address(base, offset, TIMES_1, 0), value); 2272b488b7864b7bf9cade82d45c8bdda2372f48a10cRoland Levillain } else if (type == Primitive::kPrimLong) { 2273b488b7864b7bf9cade82d45c8bdda2372f48a10cRoland Levillain __ LockCmpxchgq(Address(base, offset, TIMES_1, 0), value); 2274b488b7864b7bf9cade82d45c8bdda2372f48a10cRoland Levillain } else { 2275b488b7864b7bf9cade82d45c8bdda2372f48a10cRoland Levillain LOG(FATAL) << "Unexpected CAS type " << type; 2276b488b7864b7bf9cade82d45c8bdda2372f48a10cRoland Levillain } 227758d25fd052e999a24734b0cf856a1563e3d1b2d0Mark Mendell 22780d5a281c671444bfa75d63caf1427a8c0e6e1177Roland Levillain // LOCK CMPXCHG has full barrier semantics, and we don't need 2279b488b7864b7bf9cade82d45c8bdda2372f48a10cRoland Levillain // scheduling barriers at this time. 22804d02711ea578dbb789abb30cbaf12f9926e13d81Roland Levillain 2281b488b7864b7bf9cade82d45c8bdda2372f48a10cRoland Levillain // Convert ZF into the boolean result. 2282b488b7864b7bf9cade82d45c8bdda2372f48a10cRoland Levillain __ setcc(kZero, out); 2283b488b7864b7bf9cade82d45c8bdda2372f48a10cRoland Levillain __ movzxb(out, out); 22844d02711ea578dbb789abb30cbaf12f9926e13d81Roland Levillain } 228558d25fd052e999a24734b0cf856a1563e3d1b2d0Mark Mendell} 228658d25fd052e999a24734b0cf856a1563e3d1b2d0Mark Mendell 228758d25fd052e999a24734b0cf856a1563e3d1b2d0Mark Mendellvoid IntrinsicCodeGeneratorX86_64::VisitUnsafeCASInt(HInvoke* invoke) { 228858d25fd052e999a24734b0cf856a1563e3d1b2d0Mark Mendell GenCAS(Primitive::kPrimInt, invoke, codegen_); 228958d25fd052e999a24734b0cf856a1563e3d1b2d0Mark Mendell} 229058d25fd052e999a24734b0cf856a1563e3d1b2d0Mark Mendell 229158d25fd052e999a24734b0cf856a1563e3d1b2d0Mark Mendellvoid IntrinsicCodeGeneratorX86_64::VisitUnsafeCASLong(HInvoke* invoke) { 229258d25fd052e999a24734b0cf856a1563e3d1b2d0Mark Mendell GenCAS(Primitive::kPrimLong, invoke, codegen_); 229358d25fd052e999a24734b0cf856a1563e3d1b2d0Mark Mendell} 229458d25fd052e999a24734b0cf856a1563e3d1b2d0Mark Mendell 229558d25fd052e999a24734b0cf856a1563e3d1b2d0Mark Mendellvoid IntrinsicCodeGeneratorX86_64::VisitUnsafeCASObject(HInvoke* invoke) { 229658d25fd052e999a24734b0cf856a1563e3d1b2d0Mark Mendell GenCAS(Primitive::kPrimNot, invoke, codegen_); 229758d25fd052e999a24734b0cf856a1563e3d1b2d0Mark Mendell} 229858d25fd052e999a24734b0cf856a1563e3d1b2d0Mark Mendell 229958d25fd052e999a24734b0cf856a1563e3d1b2d0Mark Mendellvoid IntrinsicLocationsBuilderX86_64::VisitIntegerReverse(HInvoke* invoke) { 230058d25fd052e999a24734b0cf856a1563e3d1b2d0Mark Mendell LocationSummary* locations = new (arena_) LocationSummary(invoke, 230158d25fd052e999a24734b0cf856a1563e3d1b2d0Mark Mendell LocationSummary::kNoCall, 230258d25fd052e999a24734b0cf856a1563e3d1b2d0Mark Mendell kIntrinsified); 230358d25fd052e999a24734b0cf856a1563e3d1b2d0Mark Mendell locations->SetInAt(0, Location::RequiresRegister()); 230458d25fd052e999a24734b0cf856a1563e3d1b2d0Mark Mendell locations->SetOut(Location::SameAsFirstInput()); 230558d25fd052e999a24734b0cf856a1563e3d1b2d0Mark Mendell locations->AddTemp(Location::RequiresRegister()); 230658d25fd052e999a24734b0cf856a1563e3d1b2d0Mark Mendell} 230758d25fd052e999a24734b0cf856a1563e3d1b2d0Mark Mendell 230858d25fd052e999a24734b0cf856a1563e3d1b2d0Mark Mendellstatic void SwapBits(CpuRegister reg, CpuRegister temp, int32_t shift, int32_t mask, 230958d25fd052e999a24734b0cf856a1563e3d1b2d0Mark Mendell X86_64Assembler* assembler) { 231058d25fd052e999a24734b0cf856a1563e3d1b2d0Mark Mendell Immediate imm_shift(shift); 231158d25fd052e999a24734b0cf856a1563e3d1b2d0Mark Mendell Immediate imm_mask(mask); 231258d25fd052e999a24734b0cf856a1563e3d1b2d0Mark Mendell __ movl(temp, reg); 231358d25fd052e999a24734b0cf856a1563e3d1b2d0Mark Mendell __ shrl(reg, imm_shift); 231458d25fd052e999a24734b0cf856a1563e3d1b2d0Mark Mendell __ andl(temp, imm_mask); 231558d25fd052e999a24734b0cf856a1563e3d1b2d0Mark Mendell __ andl(reg, imm_mask); 231658d25fd052e999a24734b0cf856a1563e3d1b2d0Mark Mendell __ shll(temp, imm_shift); 231758d25fd052e999a24734b0cf856a1563e3d1b2d0Mark Mendell __ orl(reg, temp); 231858d25fd052e999a24734b0cf856a1563e3d1b2d0Mark Mendell} 231958d25fd052e999a24734b0cf856a1563e3d1b2d0Mark Mendell 232058d25fd052e999a24734b0cf856a1563e3d1b2d0Mark Mendellvoid IntrinsicCodeGeneratorX86_64::VisitIntegerReverse(HInvoke* invoke) { 2321c5d4754198aadb2ada2d3f5daacd10d79bc13f38Aart Bik X86_64Assembler* assembler = GetAssembler(); 232258d25fd052e999a24734b0cf856a1563e3d1b2d0Mark Mendell LocationSummary* locations = invoke->GetLocations(); 232358d25fd052e999a24734b0cf856a1563e3d1b2d0Mark Mendell 232458d25fd052e999a24734b0cf856a1563e3d1b2d0Mark Mendell CpuRegister reg = locations->InAt(0).AsRegister<CpuRegister>(); 232558d25fd052e999a24734b0cf856a1563e3d1b2d0Mark Mendell CpuRegister temp = locations->GetTemp(0).AsRegister<CpuRegister>(); 232658d25fd052e999a24734b0cf856a1563e3d1b2d0Mark Mendell 232758d25fd052e999a24734b0cf856a1563e3d1b2d0Mark Mendell /* 232858d25fd052e999a24734b0cf856a1563e3d1b2d0Mark Mendell * Use one bswap instruction to reverse byte order first and then use 3 rounds of 232958d25fd052e999a24734b0cf856a1563e3d1b2d0Mark Mendell * swapping bits to reverse bits in a number x. Using bswap to save instructions 233058d25fd052e999a24734b0cf856a1563e3d1b2d0Mark Mendell * compared to generic luni implementation which has 5 rounds of swapping bits. 233158d25fd052e999a24734b0cf856a1563e3d1b2d0Mark Mendell * x = bswap x 233258d25fd052e999a24734b0cf856a1563e3d1b2d0Mark Mendell * x = (x & 0x55555555) << 1 | (x >> 1) & 0x55555555; 233358d25fd052e999a24734b0cf856a1563e3d1b2d0Mark Mendell * x = (x & 0x33333333) << 2 | (x >> 2) & 0x33333333; 233458d25fd052e999a24734b0cf856a1563e3d1b2d0Mark Mendell * x = (x & 0x0F0F0F0F) << 4 | (x >> 4) & 0x0F0F0F0F; 233558d25fd052e999a24734b0cf856a1563e3d1b2d0Mark Mendell */ 233658d25fd052e999a24734b0cf856a1563e3d1b2d0Mark Mendell __ bswapl(reg); 233758d25fd052e999a24734b0cf856a1563e3d1b2d0Mark Mendell SwapBits(reg, temp, 1, 0x55555555, assembler); 233858d25fd052e999a24734b0cf856a1563e3d1b2d0Mark Mendell SwapBits(reg, temp, 2, 0x33333333, assembler); 233958d25fd052e999a24734b0cf856a1563e3d1b2d0Mark Mendell SwapBits(reg, temp, 4, 0x0f0f0f0f, assembler); 234058d25fd052e999a24734b0cf856a1563e3d1b2d0Mark Mendell} 234158d25fd052e999a24734b0cf856a1563e3d1b2d0Mark Mendell 234258d25fd052e999a24734b0cf856a1563e3d1b2d0Mark Mendellvoid IntrinsicLocationsBuilderX86_64::VisitLongReverse(HInvoke* invoke) { 234358d25fd052e999a24734b0cf856a1563e3d1b2d0Mark Mendell LocationSummary* locations = new (arena_) LocationSummary(invoke, 234458d25fd052e999a24734b0cf856a1563e3d1b2d0Mark Mendell LocationSummary::kNoCall, 234558d25fd052e999a24734b0cf856a1563e3d1b2d0Mark Mendell kIntrinsified); 234658d25fd052e999a24734b0cf856a1563e3d1b2d0Mark Mendell locations->SetInAt(0, Location::RequiresRegister()); 234758d25fd052e999a24734b0cf856a1563e3d1b2d0Mark Mendell locations->SetOut(Location::SameAsFirstInput()); 234858d25fd052e999a24734b0cf856a1563e3d1b2d0Mark Mendell locations->AddTemp(Location::RequiresRegister()); 234958d25fd052e999a24734b0cf856a1563e3d1b2d0Mark Mendell locations->AddTemp(Location::RequiresRegister()); 235058d25fd052e999a24734b0cf856a1563e3d1b2d0Mark Mendell} 235158d25fd052e999a24734b0cf856a1563e3d1b2d0Mark Mendell 235258d25fd052e999a24734b0cf856a1563e3d1b2d0Mark Mendellstatic void SwapBits64(CpuRegister reg, CpuRegister temp, CpuRegister temp_mask, 235358d25fd052e999a24734b0cf856a1563e3d1b2d0Mark Mendell int32_t shift, int64_t mask, X86_64Assembler* assembler) { 235458d25fd052e999a24734b0cf856a1563e3d1b2d0Mark Mendell Immediate imm_shift(shift); 235558d25fd052e999a24734b0cf856a1563e3d1b2d0Mark Mendell __ movq(temp_mask, Immediate(mask)); 235658d25fd052e999a24734b0cf856a1563e3d1b2d0Mark Mendell __ movq(temp, reg); 235758d25fd052e999a24734b0cf856a1563e3d1b2d0Mark Mendell __ shrq(reg, imm_shift); 235858d25fd052e999a24734b0cf856a1563e3d1b2d0Mark Mendell __ andq(temp, temp_mask); 235958d25fd052e999a24734b0cf856a1563e3d1b2d0Mark Mendell __ andq(reg, temp_mask); 236058d25fd052e999a24734b0cf856a1563e3d1b2d0Mark Mendell __ shlq(temp, imm_shift); 236158d25fd052e999a24734b0cf856a1563e3d1b2d0Mark Mendell __ orq(reg, temp); 236258d25fd052e999a24734b0cf856a1563e3d1b2d0Mark Mendell} 236358d25fd052e999a24734b0cf856a1563e3d1b2d0Mark Mendell 236458d25fd052e999a24734b0cf856a1563e3d1b2d0Mark Mendellvoid IntrinsicCodeGeneratorX86_64::VisitLongReverse(HInvoke* invoke) { 2365c5d4754198aadb2ada2d3f5daacd10d79bc13f38Aart Bik X86_64Assembler* assembler = GetAssembler(); 236658d25fd052e999a24734b0cf856a1563e3d1b2d0Mark Mendell LocationSummary* locations = invoke->GetLocations(); 236758d25fd052e999a24734b0cf856a1563e3d1b2d0Mark Mendell 236858d25fd052e999a24734b0cf856a1563e3d1b2d0Mark Mendell CpuRegister reg = locations->InAt(0).AsRegister<CpuRegister>(); 236958d25fd052e999a24734b0cf856a1563e3d1b2d0Mark Mendell CpuRegister temp1 = locations->GetTemp(0).AsRegister<CpuRegister>(); 237058d25fd052e999a24734b0cf856a1563e3d1b2d0Mark Mendell CpuRegister temp2 = locations->GetTemp(1).AsRegister<CpuRegister>(); 237158d25fd052e999a24734b0cf856a1563e3d1b2d0Mark Mendell 237258d25fd052e999a24734b0cf856a1563e3d1b2d0Mark Mendell /* 237358d25fd052e999a24734b0cf856a1563e3d1b2d0Mark Mendell * Use one bswap instruction to reverse byte order first and then use 3 rounds of 237458d25fd052e999a24734b0cf856a1563e3d1b2d0Mark Mendell * swapping bits to reverse bits in a long number x. Using bswap to save instructions 237558d25fd052e999a24734b0cf856a1563e3d1b2d0Mark Mendell * compared to generic luni implementation which has 5 rounds of swapping bits. 237658d25fd052e999a24734b0cf856a1563e3d1b2d0Mark Mendell * x = bswap x 237758d25fd052e999a24734b0cf856a1563e3d1b2d0Mark Mendell * x = (x & 0x5555555555555555) << 1 | (x >> 1) & 0x5555555555555555; 237858d25fd052e999a24734b0cf856a1563e3d1b2d0Mark Mendell * x = (x & 0x3333333333333333) << 2 | (x >> 2) & 0x3333333333333333; 237958d25fd052e999a24734b0cf856a1563e3d1b2d0Mark Mendell * x = (x & 0x0F0F0F0F0F0F0F0F) << 4 | (x >> 4) & 0x0F0F0F0F0F0F0F0F; 238058d25fd052e999a24734b0cf856a1563e3d1b2d0Mark Mendell */ 238158d25fd052e999a24734b0cf856a1563e3d1b2d0Mark Mendell __ bswapq(reg); 238258d25fd052e999a24734b0cf856a1563e3d1b2d0Mark Mendell SwapBits64(reg, temp1, temp2, 1, INT64_C(0x5555555555555555), assembler); 238358d25fd052e999a24734b0cf856a1563e3d1b2d0Mark Mendell SwapBits64(reg, temp1, temp2, 2, INT64_C(0x3333333333333333), assembler); 238458d25fd052e999a24734b0cf856a1563e3d1b2d0Mark Mendell SwapBits64(reg, temp1, temp2, 4, INT64_C(0x0f0f0f0f0f0f0f0f), assembler); 238558d25fd052e999a24734b0cf856a1563e3d1b2d0Mark Mendell} 238658d25fd052e999a24734b0cf856a1563e3d1b2d0Mark Mendell 23873f67e692860d281858485d48a4f1f81b907f1444Aart Bikstatic void CreateBitCountLocations( 23883f67e692860d281858485d48a4f1f81b907f1444Aart Bik ArenaAllocator* arena, CodeGeneratorX86_64* codegen, HInvoke* invoke) { 23893f67e692860d281858485d48a4f1f81b907f1444Aart Bik if (!codegen->GetInstructionSetFeatures().HasPopCnt()) { 23903f67e692860d281858485d48a4f1f81b907f1444Aart Bik // Do nothing if there is no popcnt support. This results in generating 23913f67e692860d281858485d48a4f1f81b907f1444Aart Bik // a call for the intrinsic rather than direct code. 23923f67e692860d281858485d48a4f1f81b907f1444Aart Bik return; 23933f67e692860d281858485d48a4f1f81b907f1444Aart Bik } 23943f67e692860d281858485d48a4f1f81b907f1444Aart Bik LocationSummary* locations = new (arena) LocationSummary(invoke, 23953f67e692860d281858485d48a4f1f81b907f1444Aart Bik LocationSummary::kNoCall, 23963f67e692860d281858485d48a4f1f81b907f1444Aart Bik kIntrinsified); 23973f67e692860d281858485d48a4f1f81b907f1444Aart Bik locations->SetInAt(0, Location::Any()); 23983f67e692860d281858485d48a4f1f81b907f1444Aart Bik locations->SetOut(Location::RequiresRegister()); 23993f67e692860d281858485d48a4f1f81b907f1444Aart Bik} 24003f67e692860d281858485d48a4f1f81b907f1444Aart Bik 2401c5d4754198aadb2ada2d3f5daacd10d79bc13f38Aart Bikstatic void GenBitCount(X86_64Assembler* assembler, 2402c5d4754198aadb2ada2d3f5daacd10d79bc13f38Aart Bik CodeGeneratorX86_64* codegen, 2403c5d4754198aadb2ada2d3f5daacd10d79bc13f38Aart Bik HInvoke* invoke, 2404c5d4754198aadb2ada2d3f5daacd10d79bc13f38Aart Bik bool is_long) { 24053f67e692860d281858485d48a4f1f81b907f1444Aart Bik LocationSummary* locations = invoke->GetLocations(); 24063f67e692860d281858485d48a4f1f81b907f1444Aart Bik Location src = locations->InAt(0); 24073f67e692860d281858485d48a4f1f81b907f1444Aart Bik CpuRegister out = locations->Out().AsRegister<CpuRegister>(); 24083f67e692860d281858485d48a4f1f81b907f1444Aart Bik 24093f67e692860d281858485d48a4f1f81b907f1444Aart Bik if (invoke->InputAt(0)->IsConstant()) { 24103f67e692860d281858485d48a4f1f81b907f1444Aart Bik // Evaluate this at compile time. 24113f67e692860d281858485d48a4f1f81b907f1444Aart Bik int64_t value = Int64FromConstant(invoke->InputAt(0)->AsConstant()); 2412fa3912edfac60a9f0a9b95a5862c7361b403fcc2Roland Levillain int32_t result = is_long 24133f67e692860d281858485d48a4f1f81b907f1444Aart Bik ? POPCOUNT(static_cast<uint64_t>(value)) 24143f67e692860d281858485d48a4f1f81b907f1444Aart Bik : POPCOUNT(static_cast<uint32_t>(value)); 2415fa3912edfac60a9f0a9b95a5862c7361b403fcc2Roland Levillain codegen->Load32BitValue(out, result); 24163f67e692860d281858485d48a4f1f81b907f1444Aart Bik return; 24173f67e692860d281858485d48a4f1f81b907f1444Aart Bik } 24183f67e692860d281858485d48a4f1f81b907f1444Aart Bik 24193f67e692860d281858485d48a4f1f81b907f1444Aart Bik if (src.IsRegister()) { 24203f67e692860d281858485d48a4f1f81b907f1444Aart Bik if (is_long) { 24213f67e692860d281858485d48a4f1f81b907f1444Aart Bik __ popcntq(out, src.AsRegister<CpuRegister>()); 24223f67e692860d281858485d48a4f1f81b907f1444Aart Bik } else { 24233f67e692860d281858485d48a4f1f81b907f1444Aart Bik __ popcntl(out, src.AsRegister<CpuRegister>()); 24243f67e692860d281858485d48a4f1f81b907f1444Aart Bik } 24253f67e692860d281858485d48a4f1f81b907f1444Aart Bik } else if (is_long) { 24263f67e692860d281858485d48a4f1f81b907f1444Aart Bik DCHECK(src.IsDoubleStackSlot()); 24273f67e692860d281858485d48a4f1f81b907f1444Aart Bik __ popcntq(out, Address(CpuRegister(RSP), src.GetStackIndex())); 24283f67e692860d281858485d48a4f1f81b907f1444Aart Bik } else { 24293f67e692860d281858485d48a4f1f81b907f1444Aart Bik DCHECK(src.IsStackSlot()); 24303f67e692860d281858485d48a4f1f81b907f1444Aart Bik __ popcntl(out, Address(CpuRegister(RSP), src.GetStackIndex())); 24313f67e692860d281858485d48a4f1f81b907f1444Aart Bik } 24323f67e692860d281858485d48a4f1f81b907f1444Aart Bik} 24333f67e692860d281858485d48a4f1f81b907f1444Aart Bik 24343f67e692860d281858485d48a4f1f81b907f1444Aart Bikvoid IntrinsicLocationsBuilderX86_64::VisitIntegerBitCount(HInvoke* invoke) { 24353f67e692860d281858485d48a4f1f81b907f1444Aart Bik CreateBitCountLocations(arena_, codegen_, invoke); 24363f67e692860d281858485d48a4f1f81b907f1444Aart Bik} 24373f67e692860d281858485d48a4f1f81b907f1444Aart Bik 24383f67e692860d281858485d48a4f1f81b907f1444Aart Bikvoid IntrinsicCodeGeneratorX86_64::VisitIntegerBitCount(HInvoke* invoke) { 2439c5d4754198aadb2ada2d3f5daacd10d79bc13f38Aart Bik GenBitCount(GetAssembler(), codegen_, invoke, /* is_long */ false); 24403f67e692860d281858485d48a4f1f81b907f1444Aart Bik} 24413f67e692860d281858485d48a4f1f81b907f1444Aart Bik 24423f67e692860d281858485d48a4f1f81b907f1444Aart Bikvoid IntrinsicLocationsBuilderX86_64::VisitLongBitCount(HInvoke* invoke) { 24433f67e692860d281858485d48a4f1f81b907f1444Aart Bik CreateBitCountLocations(arena_, codegen_, invoke); 24443f67e692860d281858485d48a4f1f81b907f1444Aart Bik} 24453f67e692860d281858485d48a4f1f81b907f1444Aart Bik 24463f67e692860d281858485d48a4f1f81b907f1444Aart Bikvoid IntrinsicCodeGeneratorX86_64::VisitLongBitCount(HInvoke* invoke) { 2447c5d4754198aadb2ada2d3f5daacd10d79bc13f38Aart Bik GenBitCount(GetAssembler(), codegen_, invoke, /* is_long */ true); 2448c5d4754198aadb2ada2d3f5daacd10d79bc13f38Aart Bik} 2449c5d4754198aadb2ada2d3f5daacd10d79bc13f38Aart Bik 2450c5d4754198aadb2ada2d3f5daacd10d79bc13f38Aart Bikstatic void CreateOneBitLocations(ArenaAllocator* arena, HInvoke* invoke, bool is_high) { 2451c5d4754198aadb2ada2d3f5daacd10d79bc13f38Aart Bik LocationSummary* locations = new (arena) LocationSummary(invoke, 2452c5d4754198aadb2ada2d3f5daacd10d79bc13f38Aart Bik LocationSummary::kNoCall, 2453c5d4754198aadb2ada2d3f5daacd10d79bc13f38Aart Bik kIntrinsified); 2454c5d4754198aadb2ada2d3f5daacd10d79bc13f38Aart Bik locations->SetInAt(0, Location::Any()); 2455c5d4754198aadb2ada2d3f5daacd10d79bc13f38Aart Bik locations->SetOut(Location::RequiresRegister()); 2456c5d4754198aadb2ada2d3f5daacd10d79bc13f38Aart Bik locations->AddTemp(is_high ? Location::RegisterLocation(RCX) // needs CL 2457c5d4754198aadb2ada2d3f5daacd10d79bc13f38Aart Bik : Location::RequiresRegister()); // any will do 2458c5d4754198aadb2ada2d3f5daacd10d79bc13f38Aart Bik} 2459c5d4754198aadb2ada2d3f5daacd10d79bc13f38Aart Bik 2460c5d4754198aadb2ada2d3f5daacd10d79bc13f38Aart Bikstatic void GenOneBit(X86_64Assembler* assembler, 2461c5d4754198aadb2ada2d3f5daacd10d79bc13f38Aart Bik CodeGeneratorX86_64* codegen, 2462c5d4754198aadb2ada2d3f5daacd10d79bc13f38Aart Bik HInvoke* invoke, 2463c5d4754198aadb2ada2d3f5daacd10d79bc13f38Aart Bik bool is_high, bool is_long) { 2464c5d4754198aadb2ada2d3f5daacd10d79bc13f38Aart Bik LocationSummary* locations = invoke->GetLocations(); 2465c5d4754198aadb2ada2d3f5daacd10d79bc13f38Aart Bik Location src = locations->InAt(0); 2466c5d4754198aadb2ada2d3f5daacd10d79bc13f38Aart Bik CpuRegister out = locations->Out().AsRegister<CpuRegister>(); 2467c5d4754198aadb2ada2d3f5daacd10d79bc13f38Aart Bik 2468c5d4754198aadb2ada2d3f5daacd10d79bc13f38Aart Bik if (invoke->InputAt(0)->IsConstant()) { 2469c5d4754198aadb2ada2d3f5daacd10d79bc13f38Aart Bik // Evaluate this at compile time. 2470c5d4754198aadb2ada2d3f5daacd10d79bc13f38Aart Bik int64_t value = Int64FromConstant(invoke->InputAt(0)->AsConstant()); 2471c5d4754198aadb2ada2d3f5daacd10d79bc13f38Aart Bik if (value == 0) { 2472c5d4754198aadb2ada2d3f5daacd10d79bc13f38Aart Bik __ xorl(out, out); // Clears upper bits too. 2473c5d4754198aadb2ada2d3f5daacd10d79bc13f38Aart Bik return; 2474c5d4754198aadb2ada2d3f5daacd10d79bc13f38Aart Bik } 2475c5d4754198aadb2ada2d3f5daacd10d79bc13f38Aart Bik // Nonzero value. 2476c5d4754198aadb2ada2d3f5daacd10d79bc13f38Aart Bik if (is_high) { 2477c5d4754198aadb2ada2d3f5daacd10d79bc13f38Aart Bik value = is_long ? 63 - CLZ(static_cast<uint64_t>(value)) 2478c5d4754198aadb2ada2d3f5daacd10d79bc13f38Aart Bik : 31 - CLZ(static_cast<uint32_t>(value)); 2479c5d4754198aadb2ada2d3f5daacd10d79bc13f38Aart Bik } else { 2480c5d4754198aadb2ada2d3f5daacd10d79bc13f38Aart Bik value = is_long ? CTZ(static_cast<uint64_t>(value)) 2481c5d4754198aadb2ada2d3f5daacd10d79bc13f38Aart Bik : CTZ(static_cast<uint32_t>(value)); 2482c5d4754198aadb2ada2d3f5daacd10d79bc13f38Aart Bik } 2483c5d4754198aadb2ada2d3f5daacd10d79bc13f38Aart Bik if (is_long) { 2484c5d4754198aadb2ada2d3f5daacd10d79bc13f38Aart Bik codegen->Load64BitValue(out, 1L << value); 2485c5d4754198aadb2ada2d3f5daacd10d79bc13f38Aart Bik } else { 2486c5d4754198aadb2ada2d3f5daacd10d79bc13f38Aart Bik codegen->Load32BitValue(out, 1 << value); 2487c5d4754198aadb2ada2d3f5daacd10d79bc13f38Aart Bik } 2488c5d4754198aadb2ada2d3f5daacd10d79bc13f38Aart Bik return; 2489c5d4754198aadb2ada2d3f5daacd10d79bc13f38Aart Bik } 2490c5d4754198aadb2ada2d3f5daacd10d79bc13f38Aart Bik 2491c5d4754198aadb2ada2d3f5daacd10d79bc13f38Aart Bik // Handle the non-constant cases. 2492c5d4754198aadb2ada2d3f5daacd10d79bc13f38Aart Bik CpuRegister tmp = locations->GetTemp(0).AsRegister<CpuRegister>(); 2493c5d4754198aadb2ada2d3f5daacd10d79bc13f38Aart Bik if (is_high) { 2494c5d4754198aadb2ada2d3f5daacd10d79bc13f38Aart Bik // Use architectural support: basically 1 << bsr. 2495c5d4754198aadb2ada2d3f5daacd10d79bc13f38Aart Bik if (src.IsRegister()) { 2496c5d4754198aadb2ada2d3f5daacd10d79bc13f38Aart Bik if (is_long) { 2497c5d4754198aadb2ada2d3f5daacd10d79bc13f38Aart Bik __ bsrq(tmp, src.AsRegister<CpuRegister>()); 2498c5d4754198aadb2ada2d3f5daacd10d79bc13f38Aart Bik } else { 2499c5d4754198aadb2ada2d3f5daacd10d79bc13f38Aart Bik __ bsrl(tmp, src.AsRegister<CpuRegister>()); 2500c5d4754198aadb2ada2d3f5daacd10d79bc13f38Aart Bik } 2501c5d4754198aadb2ada2d3f5daacd10d79bc13f38Aart Bik } else if (is_long) { 2502c5d4754198aadb2ada2d3f5daacd10d79bc13f38Aart Bik DCHECK(src.IsDoubleStackSlot()); 2503c5d4754198aadb2ada2d3f5daacd10d79bc13f38Aart Bik __ bsrq(tmp, Address(CpuRegister(RSP), src.GetStackIndex())); 2504c5d4754198aadb2ada2d3f5daacd10d79bc13f38Aart Bik } else { 2505c5d4754198aadb2ada2d3f5daacd10d79bc13f38Aart Bik DCHECK(src.IsStackSlot()); 2506c5d4754198aadb2ada2d3f5daacd10d79bc13f38Aart Bik __ bsrl(tmp, Address(CpuRegister(RSP), src.GetStackIndex())); 2507c5d4754198aadb2ada2d3f5daacd10d79bc13f38Aart Bik } 2508c5d4754198aadb2ada2d3f5daacd10d79bc13f38Aart Bik // BSR sets ZF if the input was zero. 2509c5d4754198aadb2ada2d3f5daacd10d79bc13f38Aart Bik NearLabel is_zero, done; 2510c5d4754198aadb2ada2d3f5daacd10d79bc13f38Aart Bik __ j(kEqual, &is_zero); 2511c5d4754198aadb2ada2d3f5daacd10d79bc13f38Aart Bik __ movl(out, Immediate(1)); // Clears upper bits too. 2512c5d4754198aadb2ada2d3f5daacd10d79bc13f38Aart Bik if (is_long) { 2513c5d4754198aadb2ada2d3f5daacd10d79bc13f38Aart Bik __ shlq(out, tmp); 2514c5d4754198aadb2ada2d3f5daacd10d79bc13f38Aart Bik } else { 2515c5d4754198aadb2ada2d3f5daacd10d79bc13f38Aart Bik __ shll(out, tmp); 2516c5d4754198aadb2ada2d3f5daacd10d79bc13f38Aart Bik } 2517c5d4754198aadb2ada2d3f5daacd10d79bc13f38Aart Bik __ jmp(&done); 2518c5d4754198aadb2ada2d3f5daacd10d79bc13f38Aart Bik __ Bind(&is_zero); 2519c5d4754198aadb2ada2d3f5daacd10d79bc13f38Aart Bik __ xorl(out, out); // Clears upper bits too. 2520c5d4754198aadb2ada2d3f5daacd10d79bc13f38Aart Bik __ Bind(&done); 2521c5d4754198aadb2ada2d3f5daacd10d79bc13f38Aart Bik } else { 2522c5d4754198aadb2ada2d3f5daacd10d79bc13f38Aart Bik // Copy input into temporary. 2523c5d4754198aadb2ada2d3f5daacd10d79bc13f38Aart Bik if (src.IsRegister()) { 2524c5d4754198aadb2ada2d3f5daacd10d79bc13f38Aart Bik if (is_long) { 2525c5d4754198aadb2ada2d3f5daacd10d79bc13f38Aart Bik __ movq(tmp, src.AsRegister<CpuRegister>()); 2526c5d4754198aadb2ada2d3f5daacd10d79bc13f38Aart Bik } else { 2527c5d4754198aadb2ada2d3f5daacd10d79bc13f38Aart Bik __ movl(tmp, src.AsRegister<CpuRegister>()); 2528c5d4754198aadb2ada2d3f5daacd10d79bc13f38Aart Bik } 2529c5d4754198aadb2ada2d3f5daacd10d79bc13f38Aart Bik } else if (is_long) { 2530c5d4754198aadb2ada2d3f5daacd10d79bc13f38Aart Bik DCHECK(src.IsDoubleStackSlot()); 2531c5d4754198aadb2ada2d3f5daacd10d79bc13f38Aart Bik __ movq(tmp, Address(CpuRegister(RSP), src.GetStackIndex())); 2532c5d4754198aadb2ada2d3f5daacd10d79bc13f38Aart Bik } else { 2533c5d4754198aadb2ada2d3f5daacd10d79bc13f38Aart Bik DCHECK(src.IsStackSlot()); 2534c5d4754198aadb2ada2d3f5daacd10d79bc13f38Aart Bik __ movl(tmp, Address(CpuRegister(RSP), src.GetStackIndex())); 2535c5d4754198aadb2ada2d3f5daacd10d79bc13f38Aart Bik } 2536c5d4754198aadb2ada2d3f5daacd10d79bc13f38Aart Bik // Do the bit twiddling: basically tmp & -tmp; 2537c5d4754198aadb2ada2d3f5daacd10d79bc13f38Aart Bik if (is_long) { 2538c5d4754198aadb2ada2d3f5daacd10d79bc13f38Aart Bik __ movq(out, tmp); 2539c5d4754198aadb2ada2d3f5daacd10d79bc13f38Aart Bik __ negq(tmp); 2540c5d4754198aadb2ada2d3f5daacd10d79bc13f38Aart Bik __ andq(out, tmp); 2541c5d4754198aadb2ada2d3f5daacd10d79bc13f38Aart Bik } else { 2542c5d4754198aadb2ada2d3f5daacd10d79bc13f38Aart Bik __ movl(out, tmp); 2543c5d4754198aadb2ada2d3f5daacd10d79bc13f38Aart Bik __ negl(tmp); 2544c5d4754198aadb2ada2d3f5daacd10d79bc13f38Aart Bik __ andl(out, tmp); 2545c5d4754198aadb2ada2d3f5daacd10d79bc13f38Aart Bik } 2546c5d4754198aadb2ada2d3f5daacd10d79bc13f38Aart Bik } 2547c5d4754198aadb2ada2d3f5daacd10d79bc13f38Aart Bik} 2548c5d4754198aadb2ada2d3f5daacd10d79bc13f38Aart Bik 2549c5d4754198aadb2ada2d3f5daacd10d79bc13f38Aart Bikvoid IntrinsicLocationsBuilderX86_64::VisitIntegerHighestOneBit(HInvoke* invoke) { 2550c5d4754198aadb2ada2d3f5daacd10d79bc13f38Aart Bik CreateOneBitLocations(arena_, invoke, /* is_high */ true); 2551c5d4754198aadb2ada2d3f5daacd10d79bc13f38Aart Bik} 2552c5d4754198aadb2ada2d3f5daacd10d79bc13f38Aart Bik 2553c5d4754198aadb2ada2d3f5daacd10d79bc13f38Aart Bikvoid IntrinsicCodeGeneratorX86_64::VisitIntegerHighestOneBit(HInvoke* invoke) { 2554c5d4754198aadb2ada2d3f5daacd10d79bc13f38Aart Bik GenOneBit(GetAssembler(), codegen_, invoke, /* is_high */ true, /* is_long */ false); 2555c5d4754198aadb2ada2d3f5daacd10d79bc13f38Aart Bik} 2556c5d4754198aadb2ada2d3f5daacd10d79bc13f38Aart Bik 2557c5d4754198aadb2ada2d3f5daacd10d79bc13f38Aart Bikvoid IntrinsicLocationsBuilderX86_64::VisitLongHighestOneBit(HInvoke* invoke) { 2558c5d4754198aadb2ada2d3f5daacd10d79bc13f38Aart Bik CreateOneBitLocations(arena_, invoke, /* is_high */ true); 2559c5d4754198aadb2ada2d3f5daacd10d79bc13f38Aart Bik} 2560c5d4754198aadb2ada2d3f5daacd10d79bc13f38Aart Bik 2561c5d4754198aadb2ada2d3f5daacd10d79bc13f38Aart Bikvoid IntrinsicCodeGeneratorX86_64::VisitLongHighestOneBit(HInvoke* invoke) { 2562c5d4754198aadb2ada2d3f5daacd10d79bc13f38Aart Bik GenOneBit(GetAssembler(), codegen_, invoke, /* is_high */ true, /* is_long */ true); 2563c5d4754198aadb2ada2d3f5daacd10d79bc13f38Aart Bik} 2564c5d4754198aadb2ada2d3f5daacd10d79bc13f38Aart Bik 2565c5d4754198aadb2ada2d3f5daacd10d79bc13f38Aart Bikvoid IntrinsicLocationsBuilderX86_64::VisitIntegerLowestOneBit(HInvoke* invoke) { 2566c5d4754198aadb2ada2d3f5daacd10d79bc13f38Aart Bik CreateOneBitLocations(arena_, invoke, /* is_high */ false); 2567c5d4754198aadb2ada2d3f5daacd10d79bc13f38Aart Bik} 2568c5d4754198aadb2ada2d3f5daacd10d79bc13f38Aart Bik 2569c5d4754198aadb2ada2d3f5daacd10d79bc13f38Aart Bikvoid IntrinsicCodeGeneratorX86_64::VisitIntegerLowestOneBit(HInvoke* invoke) { 2570c5d4754198aadb2ada2d3f5daacd10d79bc13f38Aart Bik GenOneBit(GetAssembler(), codegen_, invoke, /* is_high */ false, /* is_long */ false); 2571c5d4754198aadb2ada2d3f5daacd10d79bc13f38Aart Bik} 2572c5d4754198aadb2ada2d3f5daacd10d79bc13f38Aart Bik 2573c5d4754198aadb2ada2d3f5daacd10d79bc13f38Aart Bikvoid IntrinsicLocationsBuilderX86_64::VisitLongLowestOneBit(HInvoke* invoke) { 2574c5d4754198aadb2ada2d3f5daacd10d79bc13f38Aart Bik CreateOneBitLocations(arena_, invoke, /* is_high */ false); 2575c5d4754198aadb2ada2d3f5daacd10d79bc13f38Aart Bik} 2576c5d4754198aadb2ada2d3f5daacd10d79bc13f38Aart Bik 2577c5d4754198aadb2ada2d3f5daacd10d79bc13f38Aart Bikvoid IntrinsicCodeGeneratorX86_64::VisitLongLowestOneBit(HInvoke* invoke) { 2578c5d4754198aadb2ada2d3f5daacd10d79bc13f38Aart Bik GenOneBit(GetAssembler(), codegen_, invoke, /* is_high */ false, /* is_long */ true); 25793f67e692860d281858485d48a4f1f81b907f1444Aart Bik} 25803f67e692860d281858485d48a4f1f81b907f1444Aart Bik 2581d5897678eb555a797d4e84e07814d79f0e0bb465Mark Mendellstatic void CreateLeadingZeroLocations(ArenaAllocator* arena, HInvoke* invoke) { 2582d5897678eb555a797d4e84e07814d79f0e0bb465Mark Mendell LocationSummary* locations = new (arena) LocationSummary(invoke, 2583d5897678eb555a797d4e84e07814d79f0e0bb465Mark Mendell LocationSummary::kNoCall, 2584d5897678eb555a797d4e84e07814d79f0e0bb465Mark Mendell kIntrinsified); 2585d5897678eb555a797d4e84e07814d79f0e0bb465Mark Mendell locations->SetInAt(0, Location::Any()); 2586d5897678eb555a797d4e84e07814d79f0e0bb465Mark Mendell locations->SetOut(Location::RequiresRegister()); 2587d5897678eb555a797d4e84e07814d79f0e0bb465Mark Mendell} 2588d5897678eb555a797d4e84e07814d79f0e0bb465Mark Mendell 2589c5d4754198aadb2ada2d3f5daacd10d79bc13f38Aart Bikstatic void GenLeadingZeros(X86_64Assembler* assembler, 2590c5d4754198aadb2ada2d3f5daacd10d79bc13f38Aart Bik CodeGeneratorX86_64* codegen, 2591c5d4754198aadb2ada2d3f5daacd10d79bc13f38Aart Bik HInvoke* invoke, bool is_long) { 2592d5897678eb555a797d4e84e07814d79f0e0bb465Mark Mendell LocationSummary* locations = invoke->GetLocations(); 2593d5897678eb555a797d4e84e07814d79f0e0bb465Mark Mendell Location src = locations->InAt(0); 2594d5897678eb555a797d4e84e07814d79f0e0bb465Mark Mendell CpuRegister out = locations->Out().AsRegister<CpuRegister>(); 2595d5897678eb555a797d4e84e07814d79f0e0bb465Mark Mendell 2596d5897678eb555a797d4e84e07814d79f0e0bb465Mark Mendell int zero_value_result = is_long ? 64 : 32; 2597d5897678eb555a797d4e84e07814d79f0e0bb465Mark Mendell if (invoke->InputAt(0)->IsConstant()) { 2598d5897678eb555a797d4e84e07814d79f0e0bb465Mark Mendell // Evaluate this at compile time. 2599d5897678eb555a797d4e84e07814d79f0e0bb465Mark Mendell int64_t value = Int64FromConstant(invoke->InputAt(0)->AsConstant()); 2600d5897678eb555a797d4e84e07814d79f0e0bb465Mark Mendell if (value == 0) { 2601d5897678eb555a797d4e84e07814d79f0e0bb465Mark Mendell value = zero_value_result; 2602d5897678eb555a797d4e84e07814d79f0e0bb465Mark Mendell } else { 2603d5897678eb555a797d4e84e07814d79f0e0bb465Mark Mendell value = is_long ? CLZ(static_cast<uint64_t>(value)) : CLZ(static_cast<uint32_t>(value)); 2604d5897678eb555a797d4e84e07814d79f0e0bb465Mark Mendell } 2605c5d4754198aadb2ada2d3f5daacd10d79bc13f38Aart Bik codegen->Load32BitValue(out, value); 2606d5897678eb555a797d4e84e07814d79f0e0bb465Mark Mendell return; 2607d5897678eb555a797d4e84e07814d79f0e0bb465Mark Mendell } 2608d5897678eb555a797d4e84e07814d79f0e0bb465Mark Mendell 2609d5897678eb555a797d4e84e07814d79f0e0bb465Mark Mendell // Handle the non-constant cases. 2610d5897678eb555a797d4e84e07814d79f0e0bb465Mark Mendell if (src.IsRegister()) { 2611d5897678eb555a797d4e84e07814d79f0e0bb465Mark Mendell if (is_long) { 2612d5897678eb555a797d4e84e07814d79f0e0bb465Mark Mendell __ bsrq(out, src.AsRegister<CpuRegister>()); 2613d5897678eb555a797d4e84e07814d79f0e0bb465Mark Mendell } else { 2614d5897678eb555a797d4e84e07814d79f0e0bb465Mark Mendell __ bsrl(out, src.AsRegister<CpuRegister>()); 2615d5897678eb555a797d4e84e07814d79f0e0bb465Mark Mendell } 2616d5897678eb555a797d4e84e07814d79f0e0bb465Mark Mendell } else if (is_long) { 2617d5897678eb555a797d4e84e07814d79f0e0bb465Mark Mendell DCHECK(src.IsDoubleStackSlot()); 2618d5897678eb555a797d4e84e07814d79f0e0bb465Mark Mendell __ bsrq(out, Address(CpuRegister(RSP), src.GetStackIndex())); 2619d5897678eb555a797d4e84e07814d79f0e0bb465Mark Mendell } else { 2620d5897678eb555a797d4e84e07814d79f0e0bb465Mark Mendell DCHECK(src.IsStackSlot()); 2621d5897678eb555a797d4e84e07814d79f0e0bb465Mark Mendell __ bsrl(out, Address(CpuRegister(RSP), src.GetStackIndex())); 2622d5897678eb555a797d4e84e07814d79f0e0bb465Mark Mendell } 2623d5897678eb555a797d4e84e07814d79f0e0bb465Mark Mendell 2624d5897678eb555a797d4e84e07814d79f0e0bb465Mark Mendell // BSR sets ZF if the input was zero, and the output is undefined. 26250c9497da9485ba688c592e5f452b7b1305a519c0Mark Mendell NearLabel is_zero, done; 2626d5897678eb555a797d4e84e07814d79f0e0bb465Mark Mendell __ j(kEqual, &is_zero); 2627d5897678eb555a797d4e84e07814d79f0e0bb465Mark Mendell 2628d5897678eb555a797d4e84e07814d79f0e0bb465Mark Mendell // Correct the result from BSR to get the CLZ result. 2629d5897678eb555a797d4e84e07814d79f0e0bb465Mark Mendell __ xorl(out, Immediate(zero_value_result - 1)); 2630d5897678eb555a797d4e84e07814d79f0e0bb465Mark Mendell __ jmp(&done); 2631d5897678eb555a797d4e84e07814d79f0e0bb465Mark Mendell 2632d5897678eb555a797d4e84e07814d79f0e0bb465Mark Mendell // Fix the zero case with the expected result. 2633d5897678eb555a797d4e84e07814d79f0e0bb465Mark Mendell __ Bind(&is_zero); 2634d5897678eb555a797d4e84e07814d79f0e0bb465Mark Mendell __ movl(out, Immediate(zero_value_result)); 2635d5897678eb555a797d4e84e07814d79f0e0bb465Mark Mendell 2636d5897678eb555a797d4e84e07814d79f0e0bb465Mark Mendell __ Bind(&done); 2637d5897678eb555a797d4e84e07814d79f0e0bb465Mark Mendell} 2638d5897678eb555a797d4e84e07814d79f0e0bb465Mark Mendell 2639d5897678eb555a797d4e84e07814d79f0e0bb465Mark Mendellvoid IntrinsicLocationsBuilderX86_64::VisitIntegerNumberOfLeadingZeros(HInvoke* invoke) { 2640d5897678eb555a797d4e84e07814d79f0e0bb465Mark Mendell CreateLeadingZeroLocations(arena_, invoke); 2641d5897678eb555a797d4e84e07814d79f0e0bb465Mark Mendell} 2642d5897678eb555a797d4e84e07814d79f0e0bb465Mark Mendell 2643d5897678eb555a797d4e84e07814d79f0e0bb465Mark Mendellvoid IntrinsicCodeGeneratorX86_64::VisitIntegerNumberOfLeadingZeros(HInvoke* invoke) { 2644c5d4754198aadb2ada2d3f5daacd10d79bc13f38Aart Bik GenLeadingZeros(GetAssembler(), codegen_, invoke, /* is_long */ false); 2645d5897678eb555a797d4e84e07814d79f0e0bb465Mark Mendell} 2646d5897678eb555a797d4e84e07814d79f0e0bb465Mark Mendell 2647d5897678eb555a797d4e84e07814d79f0e0bb465Mark Mendellvoid IntrinsicLocationsBuilderX86_64::VisitLongNumberOfLeadingZeros(HInvoke* invoke) { 2648d5897678eb555a797d4e84e07814d79f0e0bb465Mark Mendell CreateLeadingZeroLocations(arena_, invoke); 2649d5897678eb555a797d4e84e07814d79f0e0bb465Mark Mendell} 2650d5897678eb555a797d4e84e07814d79f0e0bb465Mark Mendell 2651d5897678eb555a797d4e84e07814d79f0e0bb465Mark Mendellvoid IntrinsicCodeGeneratorX86_64::VisitLongNumberOfLeadingZeros(HInvoke* invoke) { 2652c5d4754198aadb2ada2d3f5daacd10d79bc13f38Aart Bik GenLeadingZeros(GetAssembler(), codegen_, invoke, /* is_long */ true); 2653d5897678eb555a797d4e84e07814d79f0e0bb465Mark Mendell} 2654d5897678eb555a797d4e84e07814d79f0e0bb465Mark Mendell 26552d554795420be0be88bb4600ea81d1ec293217c4Mark Mendellstatic void CreateTrailingZeroLocations(ArenaAllocator* arena, HInvoke* invoke) { 26562d554795420be0be88bb4600ea81d1ec293217c4Mark Mendell LocationSummary* locations = new (arena) LocationSummary(invoke, 26572d554795420be0be88bb4600ea81d1ec293217c4Mark Mendell LocationSummary::kNoCall, 26582d554795420be0be88bb4600ea81d1ec293217c4Mark Mendell kIntrinsified); 26592d554795420be0be88bb4600ea81d1ec293217c4Mark Mendell locations->SetInAt(0, Location::Any()); 26602d554795420be0be88bb4600ea81d1ec293217c4Mark Mendell locations->SetOut(Location::RequiresRegister()); 26612d554795420be0be88bb4600ea81d1ec293217c4Mark Mendell} 26622d554795420be0be88bb4600ea81d1ec293217c4Mark Mendell 2663c5d4754198aadb2ada2d3f5daacd10d79bc13f38Aart Bikstatic void GenTrailingZeros(X86_64Assembler* assembler, 2664c5d4754198aadb2ada2d3f5daacd10d79bc13f38Aart Bik CodeGeneratorX86_64* codegen, 2665c5d4754198aadb2ada2d3f5daacd10d79bc13f38Aart Bik HInvoke* invoke, bool is_long) { 26662d554795420be0be88bb4600ea81d1ec293217c4Mark Mendell LocationSummary* locations = invoke->GetLocations(); 26672d554795420be0be88bb4600ea81d1ec293217c4Mark Mendell Location src = locations->InAt(0); 26682d554795420be0be88bb4600ea81d1ec293217c4Mark Mendell CpuRegister out = locations->Out().AsRegister<CpuRegister>(); 26692d554795420be0be88bb4600ea81d1ec293217c4Mark Mendell 26702d554795420be0be88bb4600ea81d1ec293217c4Mark Mendell int zero_value_result = is_long ? 64 : 32; 26712d554795420be0be88bb4600ea81d1ec293217c4Mark Mendell if (invoke->InputAt(0)->IsConstant()) { 26722d554795420be0be88bb4600ea81d1ec293217c4Mark Mendell // Evaluate this at compile time. 26732d554795420be0be88bb4600ea81d1ec293217c4Mark Mendell int64_t value = Int64FromConstant(invoke->InputAt(0)->AsConstant()); 26742d554795420be0be88bb4600ea81d1ec293217c4Mark Mendell if (value == 0) { 26752d554795420be0be88bb4600ea81d1ec293217c4Mark Mendell value = zero_value_result; 26762d554795420be0be88bb4600ea81d1ec293217c4Mark Mendell } else { 26772d554795420be0be88bb4600ea81d1ec293217c4Mark Mendell value = is_long ? CTZ(static_cast<uint64_t>(value)) : CTZ(static_cast<uint32_t>(value)); 26782d554795420be0be88bb4600ea81d1ec293217c4Mark Mendell } 2679c5d4754198aadb2ada2d3f5daacd10d79bc13f38Aart Bik codegen->Load32BitValue(out, value); 26802d554795420be0be88bb4600ea81d1ec293217c4Mark Mendell return; 26812d554795420be0be88bb4600ea81d1ec293217c4Mark Mendell } 26822d554795420be0be88bb4600ea81d1ec293217c4Mark Mendell 26832d554795420be0be88bb4600ea81d1ec293217c4Mark Mendell // Handle the non-constant cases. 26842d554795420be0be88bb4600ea81d1ec293217c4Mark Mendell if (src.IsRegister()) { 26852d554795420be0be88bb4600ea81d1ec293217c4Mark Mendell if (is_long) { 26862d554795420be0be88bb4600ea81d1ec293217c4Mark Mendell __ bsfq(out, src.AsRegister<CpuRegister>()); 26872d554795420be0be88bb4600ea81d1ec293217c4Mark Mendell } else { 26882d554795420be0be88bb4600ea81d1ec293217c4Mark Mendell __ bsfl(out, src.AsRegister<CpuRegister>()); 26892d554795420be0be88bb4600ea81d1ec293217c4Mark Mendell } 26902d554795420be0be88bb4600ea81d1ec293217c4Mark Mendell } else if (is_long) { 26912d554795420be0be88bb4600ea81d1ec293217c4Mark Mendell DCHECK(src.IsDoubleStackSlot()); 26922d554795420be0be88bb4600ea81d1ec293217c4Mark Mendell __ bsfq(out, Address(CpuRegister(RSP), src.GetStackIndex())); 26932d554795420be0be88bb4600ea81d1ec293217c4Mark Mendell } else { 26942d554795420be0be88bb4600ea81d1ec293217c4Mark Mendell DCHECK(src.IsStackSlot()); 26952d554795420be0be88bb4600ea81d1ec293217c4Mark Mendell __ bsfl(out, Address(CpuRegister(RSP), src.GetStackIndex())); 26962d554795420be0be88bb4600ea81d1ec293217c4Mark Mendell } 26972d554795420be0be88bb4600ea81d1ec293217c4Mark Mendell 26982d554795420be0be88bb4600ea81d1ec293217c4Mark Mendell // BSF sets ZF if the input was zero, and the output is undefined. 26992d554795420be0be88bb4600ea81d1ec293217c4Mark Mendell NearLabel done; 27002d554795420be0be88bb4600ea81d1ec293217c4Mark Mendell __ j(kNotEqual, &done); 27012d554795420be0be88bb4600ea81d1ec293217c4Mark Mendell 27022d554795420be0be88bb4600ea81d1ec293217c4Mark Mendell // Fix the zero case with the expected result. 27032d554795420be0be88bb4600ea81d1ec293217c4Mark Mendell __ movl(out, Immediate(zero_value_result)); 27042d554795420be0be88bb4600ea81d1ec293217c4Mark Mendell 27052d554795420be0be88bb4600ea81d1ec293217c4Mark Mendell __ Bind(&done); 27062d554795420be0be88bb4600ea81d1ec293217c4Mark Mendell} 27072d554795420be0be88bb4600ea81d1ec293217c4Mark Mendell 27082d554795420be0be88bb4600ea81d1ec293217c4Mark Mendellvoid IntrinsicLocationsBuilderX86_64::VisitIntegerNumberOfTrailingZeros(HInvoke* invoke) { 27092d554795420be0be88bb4600ea81d1ec293217c4Mark Mendell CreateTrailingZeroLocations(arena_, invoke); 27102d554795420be0be88bb4600ea81d1ec293217c4Mark Mendell} 27112d554795420be0be88bb4600ea81d1ec293217c4Mark Mendell 27122d554795420be0be88bb4600ea81d1ec293217c4Mark Mendellvoid IntrinsicCodeGeneratorX86_64::VisitIntegerNumberOfTrailingZeros(HInvoke* invoke) { 2713c5d4754198aadb2ada2d3f5daacd10d79bc13f38Aart Bik GenTrailingZeros(GetAssembler(), codegen_, invoke, /* is_long */ false); 27142d554795420be0be88bb4600ea81d1ec293217c4Mark Mendell} 27152d554795420be0be88bb4600ea81d1ec293217c4Mark Mendell 27162d554795420be0be88bb4600ea81d1ec293217c4Mark Mendellvoid IntrinsicLocationsBuilderX86_64::VisitLongNumberOfTrailingZeros(HInvoke* invoke) { 27172d554795420be0be88bb4600ea81d1ec293217c4Mark Mendell CreateTrailingZeroLocations(arena_, invoke); 27182d554795420be0be88bb4600ea81d1ec293217c4Mark Mendell} 27192d554795420be0be88bb4600ea81d1ec293217c4Mark Mendell 27202d554795420be0be88bb4600ea81d1ec293217c4Mark Mendellvoid IntrinsicCodeGeneratorX86_64::VisitLongNumberOfTrailingZeros(HInvoke* invoke) { 2721c5d4754198aadb2ada2d3f5daacd10d79bc13f38Aart Bik GenTrailingZeros(GetAssembler(), codegen_, invoke, /* is_long */ true); 2722c5d4754198aadb2ada2d3f5daacd10d79bc13f38Aart Bik} 2723c5d4754198aadb2ada2d3f5daacd10d79bc13f38Aart Bik 27242f9fcc999fab4ba6cd86c30e664325b47b9618e5Aart BikUNIMPLEMENTED_INTRINSIC(X86_64, ReferenceGetReferent) 27252f9fcc999fab4ba6cd86c30e664325b47b9618e5Aart BikUNIMPLEMENTED_INTRINSIC(X86_64, FloatIsInfinite) 27262f9fcc999fab4ba6cd86c30e664325b47b9618e5Aart BikUNIMPLEMENTED_INTRINSIC(X86_64, DoubleIsInfinite) 27272f9fcc999fab4ba6cd86c30e664325b47b9618e5Aart Bik 27280e54c0160c84894696c05af6cad9eae3690f9496Aart Bik// 1.8. 27290e54c0160c84894696c05af6cad9eae3690f9496Aart BikUNIMPLEMENTED_INTRINSIC(X86_64, UnsafeGetAndAddInt) 27300e54c0160c84894696c05af6cad9eae3690f9496Aart BikUNIMPLEMENTED_INTRINSIC(X86_64, UnsafeGetAndAddLong) 27310e54c0160c84894696c05af6cad9eae3690f9496Aart BikUNIMPLEMENTED_INTRINSIC(X86_64, UnsafeGetAndSetInt) 27320e54c0160c84894696c05af6cad9eae3690f9496Aart BikUNIMPLEMENTED_INTRINSIC(X86_64, UnsafeGetAndSetLong) 27330e54c0160c84894696c05af6cad9eae3690f9496Aart BikUNIMPLEMENTED_INTRINSIC(X86_64, UnsafeGetAndSetObject) 27340e54c0160c84894696c05af6cad9eae3690f9496Aart Bik 27352f9fcc999fab4ba6cd86c30e664325b47b9618e5Aart BikUNREACHABLE_INTRINSICS(X86_64) 27364d02711ea578dbb789abb30cbaf12f9926e13d81Roland Levillain 27374d02711ea578dbb789abb30cbaf12f9926e13d81Roland Levillain#undef __ 27384d02711ea578dbb789abb30cbaf12f9926e13d81Roland Levillain 273971fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe} // namespace x86_64 274071fb52fee246b7d511f520febbd73dc7a9bbca79Andreas Gampe} // namespace art 2741