AMDGPUISelLowering.cpp revision a41520cf9b9cefed2091a0624a34c5f7fdb42a68
1f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard//===-- AMDGPUISelLowering.cpp - AMDGPU Common DAG lowering functions -----===// 2f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard// 3f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard// The LLVM Compiler Infrastructure 4f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard// 5f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard// This file is distributed under the University of Illinois Open Source 6f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard// License. See LICENSE.TXT for details. 7f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard// 8f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard//===----------------------------------------------------------------------===// 9f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard// 10f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard/// \file 11f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard/// \brief This is the parent TargetLowering class for hardware code gen 12f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard/// targets. 13f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard// 14f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard//===----------------------------------------------------------------------===// 15f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 16f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard#include "AMDGPUISelLowering.h" 17e7397ee81ad07cab36362bab5a086f20acc60a80Tom Stellard#include "AMDGPU.h" 1890c64cbaa124e0e8541680efeaa56f0e6eb78d9aChristian Konig#include "AMDGPURegisterInfo.h" 1990c64cbaa124e0e8541680efeaa56f0e6eb78d9aChristian Konig#include "AMDGPUSubtarget.h" 205c35290fa35ae234fed02496404cb0fc37e1c8a5Benjamin Kramer#include "AMDILIntrinsicInfo.h" 21f502c292f6edd6b0562a93cc67cd241f52a57d54Tom Stellard#include "R600MachineFunctionInfo.h" 22e7397ee81ad07cab36362bab5a086f20acc60a80Tom Stellard#include "SIMachineFunctionInfo.h" 2390c64cbaa124e0e8541680efeaa56f0e6eb78d9aChristian Konig#include "llvm/CodeGen/CallingConvLower.h" 24f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard#include "llvm/CodeGen/MachineFunction.h" 25f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard#include "llvm/CodeGen/MachineRegisterInfo.h" 26f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard#include "llvm/CodeGen/SelectionDAG.h" 27f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard#include "llvm/CodeGen/TargetLoweringObjectFileImpl.h" 28e3d4cbc7d25061441adafa47450a31571c87bf85Tom Stellard#include "llvm/IR/DataLayout.h" 29f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 30f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellardusing namespace llvm; 31f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 3290c64cbaa124e0e8541680efeaa56f0e6eb78d9aChristian Konig#include "AMDGPUGenCallingConv.inc" 3390c64cbaa124e0e8541680efeaa56f0e6eb78d9aChristian Konig 34f98f2ce29e6e2996fa58f38979143eceaa818335Tom StellardAMDGPUTargetLowering::AMDGPUTargetLowering(TargetMachine &TM) : 35f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard TargetLowering(TM, new TargetLoweringObjectFileELF()) { 36f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 37f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard // Initialize target lowering borrowed from AMDIL 38f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard InitAMDILLowering(); 39f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 40f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard // We need to custom lower some of the intrinsics 41f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::Other, Custom); 42f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 43f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard // Library functions. These default to Expand, but we have instructions 44f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard // for them. 45f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard setOperationAction(ISD::FCEIL, MVT::f32, Legal); 46f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard setOperationAction(ISD::FEXP2, MVT::f32, Legal); 47f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard setOperationAction(ISD::FPOW, MVT::f32, Legal); 48f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard setOperationAction(ISD::FLOG2, MVT::f32, Legal); 49f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard setOperationAction(ISD::FABS, MVT::f32, Legal); 50f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard setOperationAction(ISD::FFLOOR, MVT::f32, Legal); 51f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard setOperationAction(ISD::FRINT, MVT::f32, Legal); 52f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 53ba534c21437ba133cb9d6b3f9dae80fa9c4f0cb7Tom Stellard // The hardware supports ROTR, but not ROTL 54ba534c21437ba133cb9d6b3f9dae80fa9c4f0cb7Tom Stellard setOperationAction(ISD::ROTL, MVT::i32, Expand); 55ba534c21437ba133cb9d6b3f9dae80fa9c4f0cb7Tom Stellard 56f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard // Lower floating point store/load to integer store/load to reduce the number 57f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard // of patterns in tablegen. 58f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard setOperationAction(ISD::STORE, MVT::f32, Promote); 59f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard AddPromotedToType(ISD::STORE, MVT::f32, MVT::i32); 60f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 61fc047278c160cf15d99502d8170d431cfcfe8a5bTom Stellard setOperationAction(ISD::STORE, MVT::v2f32, Promote); 62fc047278c160cf15d99502d8170d431cfcfe8a5bTom Stellard AddPromotedToType(ISD::STORE, MVT::v2f32, MVT::v2i32); 63fc047278c160cf15d99502d8170d431cfcfe8a5bTom Stellard 64f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard setOperationAction(ISD::STORE, MVT::v4f32, Promote); 65f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard AddPromotedToType(ISD::STORE, MVT::v4f32, MVT::v4i32); 66f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 6768e132866236f5d59271d2c7ffb77a9c8e743752Tom Stellard setOperationAction(ISD::STORE, MVT::f64, Promote); 6868e132866236f5d59271d2c7ffb77a9c8e743752Tom Stellard AddPromotedToType(ISD::STORE, MVT::f64, MVT::i64); 6968e132866236f5d59271d2c7ffb77a9c8e743752Tom Stellard 70f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard setOperationAction(ISD::LOAD, MVT::f32, Promote); 71f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard AddPromotedToType(ISD::LOAD, MVT::f32, MVT::i32); 72f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 73ac85f3f65ce67f71bb8e4626e0a50d818500e426Tom Stellard setOperationAction(ISD::LOAD, MVT::v2f32, Promote); 74ac85f3f65ce67f71bb8e4626e0a50d818500e426Tom Stellard AddPromotedToType(ISD::LOAD, MVT::v2f32, MVT::v2i32); 75ac85f3f65ce67f71bb8e4626e0a50d818500e426Tom Stellard 76f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard setOperationAction(ISD::LOAD, MVT::v4f32, Promote); 77f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard AddPromotedToType(ISD::LOAD, MVT::v4f32, MVT::v4i32); 78f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 7968e132866236f5d59271d2c7ffb77a9c8e743752Tom Stellard setOperationAction(ISD::LOAD, MVT::f64, Promote); 8068e132866236f5d59271d2c7ffb77a9c8e743752Tom Stellard AddPromotedToType(ISD::LOAD, MVT::f64, MVT::i64); 8168e132866236f5d59271d2c7ffb77a9c8e743752Tom Stellard 82a41520cf9b9cefed2091a0624a34c5f7fdb42a68Tom Stellard setOperationAction(ISD::CONCAT_VECTORS, MVT::v4i32, Custom); 83a41520cf9b9cefed2091a0624a34c5f7fdb42a68Tom Stellard setOperationAction(ISD::CONCAT_VECTORS, MVT::v4f32, Custom); 84a41520cf9b9cefed2091a0624a34c5f7fdb42a68Tom Stellard setOperationAction(ISD::EXTRACT_SUBVECTOR, MVT::v2i32, Custom); 85a41520cf9b9cefed2091a0624a34c5f7fdb42a68Tom Stellard setOperationAction(ISD::EXTRACT_SUBVECTOR, MVT::v2f32, Custom); 86692ee102ebef535d311c35d53457028083e5c5beTom Stellard 87d7a472c9c696ebf010835d9254fb15036e558d84Tom Stellard setOperationAction(ISD::FNEG, MVT::v2f32, Expand); 88d7a472c9c696ebf010835d9254fb15036e558d84Tom Stellard setOperationAction(ISD::FNEG, MVT::v4f32, Expand); 89d7a472c9c696ebf010835d9254fb15036e558d84Tom Stellard 9045b14e341a8a85e877d001bbd43f5e2b25b61cb8Christian Konig setOperationAction(ISD::MUL, MVT::i64, Expand); 9145b14e341a8a85e877d001bbd43f5e2b25b61cb8Christian Konig 92f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard setOperationAction(ISD::UDIV, MVT::i32, Expand); 93f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard setOperationAction(ISD::UDIVREM, MVT::i32, Custom); 94f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard setOperationAction(ISD::UREM, MVT::i32, Expand); 95f5660aab413539bd94cfea8cd88fed80c54cd984Tom Stellard setOperationAction(ISD::VSELECT, MVT::v2f32, Expand); 96f5660aab413539bd94cfea8cd88fed80c54cd984Tom Stellard setOperationAction(ISD::VSELECT, MVT::v4f32, Expand); 97f97c7fef52098bbd6a7ccc69657d112a36d77660Aaron Watry 98787e71df693e94cc512f3e439bf91609a8ec9baeCraig Topper static const int types[] = { 99f97c7fef52098bbd6a7ccc69657d112a36d77660Aaron Watry (int)MVT::v2i32, 100f97c7fef52098bbd6a7ccc69657d112a36d77660Aaron Watry (int)MVT::v4i32 101f97c7fef52098bbd6a7ccc69657d112a36d77660Aaron Watry }; 102b9df53a40b22c74ce3f3a7b4a7c0676a38cf5e73Craig Topper const size_t NumTypes = array_lengthof(types); 103f97c7fef52098bbd6a7ccc69657d112a36d77660Aaron Watry 104f97c7fef52098bbd6a7ccc69657d112a36d77660Aaron Watry for (unsigned int x = 0; x < NumTypes; ++x) { 105f97c7fef52098bbd6a7ccc69657d112a36d77660Aaron Watry MVT::SimpleValueType VT = (MVT::SimpleValueType)types[x]; 106f97c7fef52098bbd6a7ccc69657d112a36d77660Aaron Watry //Expand the following operations for the current type by default 107f97c7fef52098bbd6a7ccc69657d112a36d77660Aaron Watry setOperationAction(ISD::ADD, VT, Expand); 108f97c7fef52098bbd6a7ccc69657d112a36d77660Aaron Watry setOperationAction(ISD::AND, VT, Expand); 109e3d60ac33421a69545e2989b890899d76a918d2fTom Stellard setOperationAction(ISD::FP_TO_SINT, VT, Expand); 110e3d60ac33421a69545e2989b890899d76a918d2fTom Stellard setOperationAction(ISD::FP_TO_UINT, VT, Expand); 111f97c7fef52098bbd6a7ccc69657d112a36d77660Aaron Watry setOperationAction(ISD::MUL, VT, Expand); 112f97c7fef52098bbd6a7ccc69657d112a36d77660Aaron Watry setOperationAction(ISD::OR, VT, Expand); 113f97c7fef52098bbd6a7ccc69657d112a36d77660Aaron Watry setOperationAction(ISD::SHL, VT, Expand); 114e3d60ac33421a69545e2989b890899d76a918d2fTom Stellard setOperationAction(ISD::SINT_TO_FP, VT, Expand); 115f97c7fef52098bbd6a7ccc69657d112a36d77660Aaron Watry setOperationAction(ISD::SRL, VT, Expand); 116f97c7fef52098bbd6a7ccc69657d112a36d77660Aaron Watry setOperationAction(ISD::SRA, VT, Expand); 117f97c7fef52098bbd6a7ccc69657d112a36d77660Aaron Watry setOperationAction(ISD::SUB, VT, Expand); 118f97c7fef52098bbd6a7ccc69657d112a36d77660Aaron Watry setOperationAction(ISD::UDIV, VT, Expand); 119e3d60ac33421a69545e2989b890899d76a918d2fTom Stellard setOperationAction(ISD::UINT_TO_FP, VT, Expand); 120f97c7fef52098bbd6a7ccc69657d112a36d77660Aaron Watry setOperationAction(ISD::UREM, VT, Expand); 121f5660aab413539bd94cfea8cd88fed80c54cd984Tom Stellard setOperationAction(ISD::VSELECT, VT, Expand); 122f97c7fef52098bbd6a7ccc69657d112a36d77660Aaron Watry setOperationAction(ISD::XOR, VT, Expand); 123f97c7fef52098bbd6a7ccc69657d112a36d77660Aaron Watry } 124f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard} 125f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 1262b272a1c8cb6d9f02223a598495d84cd9d75b13dTom Stellard//===----------------------------------------------------------------------===// 1272b272a1c8cb6d9f02223a598495d84cd9d75b13dTom Stellard// Target Information 1282b272a1c8cb6d9f02223a598495d84cd9d75b13dTom Stellard//===----------------------------------------------------------------------===// 1292b272a1c8cb6d9f02223a598495d84cd9d75b13dTom Stellard 1302b272a1c8cb6d9f02223a598495d84cd9d75b13dTom StellardMVT AMDGPUTargetLowering::getVectorIdxTy() const { 1312b272a1c8cb6d9f02223a598495d84cd9d75b13dTom Stellard return MVT::i32; 1322b272a1c8cb6d9f02223a598495d84cd9d75b13dTom Stellard} 1332b272a1c8cb6d9f02223a598495d84cd9d75b13dTom Stellard 1342b272a1c8cb6d9f02223a598495d84cd9d75b13dTom Stellard 135f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard//===---------------------------------------------------------------------===// 1361f67c63cb23ba5d405452d72bb8892df6b7ccd4fTom Stellard// Target Properties 1371f67c63cb23ba5d405452d72bb8892df6b7ccd4fTom Stellard//===---------------------------------------------------------------------===// 1381f67c63cb23ba5d405452d72bb8892df6b7ccd4fTom Stellard 1391f67c63cb23ba5d405452d72bb8892df6b7ccd4fTom Stellardbool AMDGPUTargetLowering::isFAbsFree(EVT VT) const { 1401f67c63cb23ba5d405452d72bb8892df6b7ccd4fTom Stellard assert(VT.isFloatingPoint()); 1411f67c63cb23ba5d405452d72bb8892df6b7ccd4fTom Stellard return VT == MVT::f32; 1421f67c63cb23ba5d405452d72bb8892df6b7ccd4fTom Stellard} 1431f67c63cb23ba5d405452d72bb8892df6b7ccd4fTom Stellard 1441f67c63cb23ba5d405452d72bb8892df6b7ccd4fTom Stellardbool AMDGPUTargetLowering::isFNegFree(EVT VT) const { 1451f67c63cb23ba5d405452d72bb8892df6b7ccd4fTom Stellard assert(VT.isFloatingPoint()); 1461f67c63cb23ba5d405452d72bb8892df6b7ccd4fTom Stellard return VT == MVT::f32; 1471f67c63cb23ba5d405452d72bb8892df6b7ccd4fTom Stellard} 1481f67c63cb23ba5d405452d72bb8892df6b7ccd4fTom Stellard 1491f67c63cb23ba5d405452d72bb8892df6b7ccd4fTom Stellard//===---------------------------------------------------------------------===// 150f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard// TargetLowering Callbacks 151f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard//===---------------------------------------------------------------------===// 152f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 15390c64cbaa124e0e8541680efeaa56f0e6eb78d9aChristian Konigvoid AMDGPUTargetLowering::AnalyzeFormalArguments(CCState &State, 15490c64cbaa124e0e8541680efeaa56f0e6eb78d9aChristian Konig const SmallVectorImpl<ISD::InputArg> &Ins) const { 15590c64cbaa124e0e8541680efeaa56f0e6eb78d9aChristian Konig 15690c64cbaa124e0e8541680efeaa56f0e6eb78d9aChristian Konig State.AnalyzeFormalArguments(Ins, CC_AMDGPU); 157f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard} 158f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 159f98f2ce29e6e2996fa58f38979143eceaa818335Tom StellardSDValue AMDGPUTargetLowering::LowerReturn( 160f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard SDValue Chain, 161f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard CallingConv::ID CallConv, 162f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard bool isVarArg, 163f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard const SmallVectorImpl<ISD::OutputArg> &Outs, 164f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard const SmallVectorImpl<SDValue> &OutVals, 165ac6d9bec671252dd1e596fa71180ff6b39d06b5dAndrew Trick SDLoc DL, SelectionDAG &DAG) const { 166f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard return DAG.getNode(AMDGPUISD::RET_FLAG, DL, MVT::Other, Chain); 167f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard} 168f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 169f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard//===---------------------------------------------------------------------===// 170f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard// Target specific lowering 171f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard//===---------------------------------------------------------------------===// 172f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 173f98f2ce29e6e2996fa58f38979143eceaa818335Tom StellardSDValue AMDGPUTargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) 174f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard const { 175f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard switch (Op.getOpcode()) { 176f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard default: 177f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard Op.getNode()->dump(); 178f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard assert(0 && "Custom lowering code for this" 179f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard "instruction is not implemented yet!"); 180f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard break; 181f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard // AMDIL DAG lowering 182f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard case ISD::SDIV: return LowerSDIV(Op, DAG); 183f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard case ISD::SREM: return LowerSREM(Op, DAG); 184f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard case ISD::SIGN_EXTEND_INREG: return LowerSIGN_EXTEND_INREG(Op, DAG); 185f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard case ISD::BRCOND: return LowerBRCOND(Op, DAG); 186f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard // AMDGPU DAG lowering 187a41520cf9b9cefed2091a0624a34c5f7fdb42a68Tom Stellard case ISD::CONCAT_VECTORS: return LowerCONCAT_VECTORS(Op, DAG); 188a41520cf9b9cefed2091a0624a34c5f7fdb42a68Tom Stellard case ISD::EXTRACT_SUBVECTOR: return LowerEXTRACT_SUBVECTOR(Op, DAG); 189f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard case ISD::INTRINSIC_WO_CHAIN: return LowerINTRINSIC_WO_CHAIN(Op, DAG); 190f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard case ISD::UDIVREM: return LowerUDIVREM(Op, DAG); 191f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard } 192f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard return Op; 193f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard} 194f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 195e3d4cbc7d25061441adafa47450a31571c87bf85Tom StellardSDValue AMDGPUTargetLowering::LowerGlobalAddress(AMDGPUMachineFunction* MFI, 196e3d4cbc7d25061441adafa47450a31571c87bf85Tom Stellard SDValue Op, 197e3d4cbc7d25061441adafa47450a31571c87bf85Tom Stellard SelectionDAG &DAG) const { 198e3d4cbc7d25061441adafa47450a31571c87bf85Tom Stellard 199e3d4cbc7d25061441adafa47450a31571c87bf85Tom Stellard const DataLayout *TD = getTargetMachine().getDataLayout(); 200e3d4cbc7d25061441adafa47450a31571c87bf85Tom Stellard GlobalAddressSDNode *G = cast<GlobalAddressSDNode>(Op); 201e3d4cbc7d25061441adafa47450a31571c87bf85Tom Stellard // XXX: What does the value of G->getOffset() mean? 202e3d4cbc7d25061441adafa47450a31571c87bf85Tom Stellard assert(G->getOffset() == 0 && 203e3d4cbc7d25061441adafa47450a31571c87bf85Tom Stellard "Do not know what to do with an non-zero offset"); 204e3d4cbc7d25061441adafa47450a31571c87bf85Tom Stellard 205e3d4cbc7d25061441adafa47450a31571c87bf85Tom Stellard unsigned Offset = MFI->LDSSize; 206e3d4cbc7d25061441adafa47450a31571c87bf85Tom Stellard const GlobalValue *GV = G->getGlobal(); 207e3d4cbc7d25061441adafa47450a31571c87bf85Tom Stellard uint64_t Size = TD->getTypeAllocSize(GV->getType()->getElementType()); 208e3d4cbc7d25061441adafa47450a31571c87bf85Tom Stellard 209e3d4cbc7d25061441adafa47450a31571c87bf85Tom Stellard // XXX: Account for alignment? 210e3d4cbc7d25061441adafa47450a31571c87bf85Tom Stellard MFI->LDSSize += Size; 211e3d4cbc7d25061441adafa47450a31571c87bf85Tom Stellard 212a3e39dc7055486cbf514ccd868cfabc69d7f6f4eMichel Danzer return DAG.getConstant(Offset, TD->getPointerSize() == 8 ? MVT::i64 : MVT::i32); 213e3d4cbc7d25061441adafa47450a31571c87bf85Tom Stellard} 214e3d4cbc7d25061441adafa47450a31571c87bf85Tom Stellard 215a41520cf9b9cefed2091a0624a34c5f7fdb42a68Tom Stellardvoid AMDGPUTargetLowering::ExtractVectorElements(SDValue Op, SelectionDAG &DAG, 216a41520cf9b9cefed2091a0624a34c5f7fdb42a68Tom Stellard SmallVectorImpl<SDValue> &Args, 217a41520cf9b9cefed2091a0624a34c5f7fdb42a68Tom Stellard unsigned Start, 218a41520cf9b9cefed2091a0624a34c5f7fdb42a68Tom Stellard unsigned Count) const { 219a41520cf9b9cefed2091a0624a34c5f7fdb42a68Tom Stellard EVT VT = Op.getValueType(); 220a41520cf9b9cefed2091a0624a34c5f7fdb42a68Tom Stellard for (unsigned i = Start, e = Start + Count; i != e; ++i) { 221a41520cf9b9cefed2091a0624a34c5f7fdb42a68Tom Stellard Args.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SDLoc(Op), 222a41520cf9b9cefed2091a0624a34c5f7fdb42a68Tom Stellard VT.getVectorElementType(), 223a41520cf9b9cefed2091a0624a34c5f7fdb42a68Tom Stellard Op, DAG.getConstant(i, MVT::i32))); 224a41520cf9b9cefed2091a0624a34c5f7fdb42a68Tom Stellard } 225a41520cf9b9cefed2091a0624a34c5f7fdb42a68Tom Stellard} 226a41520cf9b9cefed2091a0624a34c5f7fdb42a68Tom Stellard 227a41520cf9b9cefed2091a0624a34c5f7fdb42a68Tom StellardSDValue AMDGPUTargetLowering::LowerCONCAT_VECTORS(SDValue Op, 228a41520cf9b9cefed2091a0624a34c5f7fdb42a68Tom Stellard SelectionDAG &DAG) const { 229a41520cf9b9cefed2091a0624a34c5f7fdb42a68Tom Stellard SmallVector<SDValue, 8> Args; 230a41520cf9b9cefed2091a0624a34c5f7fdb42a68Tom Stellard SDValue A = Op.getOperand(0); 231a41520cf9b9cefed2091a0624a34c5f7fdb42a68Tom Stellard SDValue B = Op.getOperand(1); 232a41520cf9b9cefed2091a0624a34c5f7fdb42a68Tom Stellard 233a41520cf9b9cefed2091a0624a34c5f7fdb42a68Tom Stellard ExtractVectorElements(A, DAG, Args, 0, 234a41520cf9b9cefed2091a0624a34c5f7fdb42a68Tom Stellard A.getValueType().getVectorNumElements()); 235a41520cf9b9cefed2091a0624a34c5f7fdb42a68Tom Stellard ExtractVectorElements(B, DAG, Args, 0, 236a41520cf9b9cefed2091a0624a34c5f7fdb42a68Tom Stellard B.getValueType().getVectorNumElements()); 237a41520cf9b9cefed2091a0624a34c5f7fdb42a68Tom Stellard 238a41520cf9b9cefed2091a0624a34c5f7fdb42a68Tom Stellard return DAG.getNode(ISD::BUILD_VECTOR, SDLoc(Op), Op.getValueType(), 239a41520cf9b9cefed2091a0624a34c5f7fdb42a68Tom Stellard &Args[0], Args.size()); 240a41520cf9b9cefed2091a0624a34c5f7fdb42a68Tom Stellard} 241a41520cf9b9cefed2091a0624a34c5f7fdb42a68Tom Stellard 242a41520cf9b9cefed2091a0624a34c5f7fdb42a68Tom StellardSDValue AMDGPUTargetLowering::LowerEXTRACT_SUBVECTOR(SDValue Op, 243a41520cf9b9cefed2091a0624a34c5f7fdb42a68Tom Stellard SelectionDAG &DAG) const { 244a41520cf9b9cefed2091a0624a34c5f7fdb42a68Tom Stellard 245a41520cf9b9cefed2091a0624a34c5f7fdb42a68Tom Stellard SmallVector<SDValue, 8> Args; 246a41520cf9b9cefed2091a0624a34c5f7fdb42a68Tom Stellard EVT VT = Op.getValueType(); 247a41520cf9b9cefed2091a0624a34c5f7fdb42a68Tom Stellard unsigned Start = cast<ConstantSDNode>(Op.getOperand(1))->getZExtValue(); 248a41520cf9b9cefed2091a0624a34c5f7fdb42a68Tom Stellard ExtractVectorElements(Op.getOperand(0), DAG, Args, Start, 249a41520cf9b9cefed2091a0624a34c5f7fdb42a68Tom Stellard VT.getVectorNumElements()); 250a41520cf9b9cefed2091a0624a34c5f7fdb42a68Tom Stellard 251a41520cf9b9cefed2091a0624a34c5f7fdb42a68Tom Stellard return DAG.getNode(ISD::BUILD_VECTOR, SDLoc(Op), Op.getValueType(), 252a41520cf9b9cefed2091a0624a34c5f7fdb42a68Tom Stellard &Args[0], Args.size()); 253a41520cf9b9cefed2091a0624a34c5f7fdb42a68Tom Stellard} 254a41520cf9b9cefed2091a0624a34c5f7fdb42a68Tom Stellard 255a41520cf9b9cefed2091a0624a34c5f7fdb42a68Tom Stellard 256f98f2ce29e6e2996fa58f38979143eceaa818335Tom StellardSDValue AMDGPUTargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op, 257f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard SelectionDAG &DAG) const { 258f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard unsigned IntrinsicID = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue(); 259ac6d9bec671252dd1e596fa71180ff6b39d06b5dAndrew Trick SDLoc DL(Op); 260f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard EVT VT = Op.getValueType(); 261f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 262f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard switch (IntrinsicID) { 263f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard default: return Op; 264f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard case AMDGPUIntrinsic::AMDIL_abs: 265f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard return LowerIntrinsicIABS(Op, DAG); 266f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard case AMDGPUIntrinsic::AMDIL_exp: 267f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard return DAG.getNode(ISD::FEXP2, DL, VT, Op.getOperand(1)); 268f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard case AMDGPUIntrinsic::AMDGPU_lrp: 269f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard return LowerIntrinsicLRP(Op, DAG); 270f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard case AMDGPUIntrinsic::AMDIL_fraction: 271f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard return DAG.getNode(AMDGPUISD::FRACT, DL, VT, Op.getOperand(1)); 272f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard case AMDGPUIntrinsic::AMDIL_max: 273f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard return DAG.getNode(AMDGPUISD::FMAX, DL, VT, Op.getOperand(1), 274f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard Op.getOperand(2)); 275f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard case AMDGPUIntrinsic::AMDGPU_imax: 276f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard return DAG.getNode(AMDGPUISD::SMAX, DL, VT, Op.getOperand(1), 277f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard Op.getOperand(2)); 278f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard case AMDGPUIntrinsic::AMDGPU_umax: 279f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard return DAG.getNode(AMDGPUISD::UMAX, DL, VT, Op.getOperand(1), 280f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard Op.getOperand(2)); 281f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard case AMDGPUIntrinsic::AMDIL_min: 282f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard return DAG.getNode(AMDGPUISD::FMIN, DL, VT, Op.getOperand(1), 283f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard Op.getOperand(2)); 284f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard case AMDGPUIntrinsic::AMDGPU_imin: 285f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard return DAG.getNode(AMDGPUISD::SMIN, DL, VT, Op.getOperand(1), 286f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard Op.getOperand(2)); 287f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard case AMDGPUIntrinsic::AMDGPU_umin: 288f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard return DAG.getNode(AMDGPUISD::UMIN, DL, VT, Op.getOperand(1), 289f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard Op.getOperand(2)); 290f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard case AMDGPUIntrinsic::AMDIL_round_nearest: 291f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard return DAG.getNode(ISD::FRINT, DL, VT, Op.getOperand(1)); 292f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard } 293f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard} 294f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 295f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard///IABS(a) = SMAX(sub(0, a), a) 296f98f2ce29e6e2996fa58f38979143eceaa818335Tom StellardSDValue AMDGPUTargetLowering::LowerIntrinsicIABS(SDValue Op, 297f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard SelectionDAG &DAG) const { 298f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 299ac6d9bec671252dd1e596fa71180ff6b39d06b5dAndrew Trick SDLoc DL(Op); 300f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard EVT VT = Op.getValueType(); 301f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard SDValue Neg = DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, VT), 302f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard Op.getOperand(1)); 303f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 304f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard return DAG.getNode(AMDGPUISD::SMAX, DL, VT, Neg, Op.getOperand(1)); 305f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard} 306f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 307f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard/// Linear Interpolation 308f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard/// LRP(a, b, c) = muladd(a, b, (1 - a) * c) 309f98f2ce29e6e2996fa58f38979143eceaa818335Tom StellardSDValue AMDGPUTargetLowering::LowerIntrinsicLRP(SDValue Op, 310f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard SelectionDAG &DAG) const { 311ac6d9bec671252dd1e596fa71180ff6b39d06b5dAndrew Trick SDLoc DL(Op); 312f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard EVT VT = Op.getValueType(); 313f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard SDValue OneSubA = DAG.getNode(ISD::FSUB, DL, VT, 314f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard DAG.getConstantFP(1.0f, MVT::f32), 315f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard Op.getOperand(1)); 316f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard SDValue OneSubAC = DAG.getNode(ISD::FMUL, DL, VT, OneSubA, 317f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard Op.getOperand(3)); 318e3111964a0902bc38440980b0915b189f829c395Vincent Lejeune return DAG.getNode(ISD::FADD, DL, VT, 319e3111964a0902bc38440980b0915b189f829c395Vincent Lejeune DAG.getNode(ISD::FMUL, DL, VT, Op.getOperand(1), Op.getOperand(2)), 320e3111964a0902bc38440980b0915b189f829c395Vincent Lejeune OneSubAC); 321f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard} 322f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 323f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard/// \brief Generate Min/Max node 324f98f2ce29e6e2996fa58f38979143eceaa818335Tom StellardSDValue AMDGPUTargetLowering::LowerMinMax(SDValue Op, 325f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard SelectionDAG &DAG) const { 326ac6d9bec671252dd1e596fa71180ff6b39d06b5dAndrew Trick SDLoc DL(Op); 327f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard EVT VT = Op.getValueType(); 328f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 329f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard SDValue LHS = Op.getOperand(0); 330f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard SDValue RHS = Op.getOperand(1); 331f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard SDValue True = Op.getOperand(2); 332f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard SDValue False = Op.getOperand(3); 333f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard SDValue CC = Op.getOperand(4); 334f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 335f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard if (VT != MVT::f32 || 336f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard !((LHS == True && RHS == False) || (LHS == False && RHS == True))) { 337f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard return SDValue(); 338f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard } 339f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 340f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard ISD::CondCode CCOpcode = cast<CondCodeSDNode>(CC)->get(); 341f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard switch (CCOpcode) { 342f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard case ISD::SETOEQ: 343f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard case ISD::SETONE: 344f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard case ISD::SETUNE: 345f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard case ISD::SETNE: 346f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard case ISD::SETUEQ: 347f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard case ISD::SETEQ: 348f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard case ISD::SETFALSE: 349f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard case ISD::SETFALSE2: 350f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard case ISD::SETTRUE: 351f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard case ISD::SETTRUE2: 352f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard case ISD::SETUO: 353f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard case ISD::SETO: 354f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard assert(0 && "Operation should already be optimised !"); 355f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard case ISD::SETULE: 356f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard case ISD::SETULT: 357f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard case ISD::SETOLE: 358f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard case ISD::SETOLT: 359f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard case ISD::SETLE: 360f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard case ISD::SETLT: { 361f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard if (LHS == True) 362f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard return DAG.getNode(AMDGPUISD::FMIN, DL, VT, LHS, RHS); 363f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard else 364f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard return DAG.getNode(AMDGPUISD::FMAX, DL, VT, LHS, RHS); 365f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard } 366f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard case ISD::SETGT: 367f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard case ISD::SETGE: 368f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard case ISD::SETUGE: 369f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard case ISD::SETOGE: 370f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard case ISD::SETUGT: 371f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard case ISD::SETOGT: { 372f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard if (LHS == True) 373f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard return DAG.getNode(AMDGPUISD::FMAX, DL, VT, LHS, RHS); 374f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard else 375f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard return DAG.getNode(AMDGPUISD::FMIN, DL, VT, LHS, RHS); 376f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard } 377f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard case ISD::SETCC_INVALID: 378f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard assert(0 && "Invalid setcc condcode !"); 379f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard } 380f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard return Op; 381f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard} 382f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 383f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 384f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 385f98f2ce29e6e2996fa58f38979143eceaa818335Tom StellardSDValue AMDGPUTargetLowering::LowerUDIVREM(SDValue Op, 386f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard SelectionDAG &DAG) const { 387ac6d9bec671252dd1e596fa71180ff6b39d06b5dAndrew Trick SDLoc DL(Op); 388f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard EVT VT = Op.getValueType(); 389f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 390f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard SDValue Num = Op.getOperand(0); 391f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard SDValue Den = Op.getOperand(1); 392f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 393f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard SmallVector<SDValue, 8> Results; 394f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 395f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard // RCP = URECIP(Den) = 2^32 / Den + e 396f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard // e is rounding error. 397f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard SDValue RCP = DAG.getNode(AMDGPUISD::URECIP, DL, VT, Den); 398f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 399f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard // RCP_LO = umulo(RCP, Den) */ 400f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard SDValue RCP_LO = DAG.getNode(ISD::UMULO, DL, VT, RCP, Den); 401f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 402f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard // RCP_HI = mulhu (RCP, Den) */ 403f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard SDValue RCP_HI = DAG.getNode(ISD::MULHU, DL, VT, RCP, Den); 404f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 405f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard // NEG_RCP_LO = -RCP_LO 406f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard SDValue NEG_RCP_LO = DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, VT), 407f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard RCP_LO); 408f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 409f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard // ABS_RCP_LO = (RCP_HI == 0 ? NEG_RCP_LO : RCP_LO) 410f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard SDValue ABS_RCP_LO = DAG.getSelectCC(DL, RCP_HI, DAG.getConstant(0, VT), 411f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard NEG_RCP_LO, RCP_LO, 412f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard ISD::SETEQ); 413f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard // Calculate the rounding error from the URECIP instruction 414f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard // E = mulhu(ABS_RCP_LO, RCP) 415f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard SDValue E = DAG.getNode(ISD::MULHU, DL, VT, ABS_RCP_LO, RCP); 416f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 417f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard // RCP_A_E = RCP + E 418f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard SDValue RCP_A_E = DAG.getNode(ISD::ADD, DL, VT, RCP, E); 419f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 420f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard // RCP_S_E = RCP - E 421f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard SDValue RCP_S_E = DAG.getNode(ISD::SUB, DL, VT, RCP, E); 422f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 423f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard // Tmp0 = (RCP_HI == 0 ? RCP_A_E : RCP_SUB_E) 424f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard SDValue Tmp0 = DAG.getSelectCC(DL, RCP_HI, DAG.getConstant(0, VT), 425f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard RCP_A_E, RCP_S_E, 426f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard ISD::SETEQ); 427f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard // Quotient = mulhu(Tmp0, Num) 428f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard SDValue Quotient = DAG.getNode(ISD::MULHU, DL, VT, Tmp0, Num); 429f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 430f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard // Num_S_Remainder = Quotient * Den 431f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard SDValue Num_S_Remainder = DAG.getNode(ISD::UMULO, DL, VT, Quotient, Den); 432f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 433f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard // Remainder = Num - Num_S_Remainder 434f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard SDValue Remainder = DAG.getNode(ISD::SUB, DL, VT, Num, Num_S_Remainder); 435f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 436f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard // Remainder_GE_Den = (Remainder >= Den ? -1 : 0) 437f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard SDValue Remainder_GE_Den = DAG.getSelectCC(DL, Remainder, Den, 438f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard DAG.getConstant(-1, VT), 439f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard DAG.getConstant(0, VT), 440f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard ISD::SETGE); 441f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard // Remainder_GE_Zero = (Remainder >= 0 ? -1 : 0) 442f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard SDValue Remainder_GE_Zero = DAG.getSelectCC(DL, Remainder, 443f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard DAG.getConstant(0, VT), 444f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard DAG.getConstant(-1, VT), 445f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard DAG.getConstant(0, VT), 446f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard ISD::SETGE); 447f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard // Tmp1 = Remainder_GE_Den & Remainder_GE_Zero 448f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard SDValue Tmp1 = DAG.getNode(ISD::AND, DL, VT, Remainder_GE_Den, 449f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard Remainder_GE_Zero); 450f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 451f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard // Calculate Division result: 452f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 453f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard // Quotient_A_One = Quotient + 1 454f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard SDValue Quotient_A_One = DAG.getNode(ISD::ADD, DL, VT, Quotient, 455f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard DAG.getConstant(1, VT)); 456f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 457f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard // Quotient_S_One = Quotient - 1 458f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard SDValue Quotient_S_One = DAG.getNode(ISD::SUB, DL, VT, Quotient, 459f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard DAG.getConstant(1, VT)); 460f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 461f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard // Div = (Tmp1 == 0 ? Quotient : Quotient_A_One) 462f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard SDValue Div = DAG.getSelectCC(DL, Tmp1, DAG.getConstant(0, VT), 463f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard Quotient, Quotient_A_One, ISD::SETEQ); 464f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 465f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard // Div = (Remainder_GE_Zero == 0 ? Quotient_S_One : Div) 466f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard Div = DAG.getSelectCC(DL, Remainder_GE_Zero, DAG.getConstant(0, VT), 467f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard Quotient_S_One, Div, ISD::SETEQ); 468f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 469f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard // Calculate Rem result: 470f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 471f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard // Remainder_S_Den = Remainder - Den 472f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard SDValue Remainder_S_Den = DAG.getNode(ISD::SUB, DL, VT, Remainder, Den); 473f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 474f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard // Remainder_A_Den = Remainder + Den 475f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard SDValue Remainder_A_Den = DAG.getNode(ISD::ADD, DL, VT, Remainder, Den); 476f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 477f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard // Rem = (Tmp1 == 0 ? Remainder : Remainder_S_Den) 478f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard SDValue Rem = DAG.getSelectCC(DL, Tmp1, DAG.getConstant(0, VT), 479f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard Remainder, Remainder_S_Den, ISD::SETEQ); 480f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 481f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard // Rem = (Remainder_GE_Zero == 0 ? Remainder_A_Den : Rem) 482f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard Rem = DAG.getSelectCC(DL, Remainder_GE_Zero, DAG.getConstant(0, VT), 483f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard Remainder_A_Den, Rem, ISD::SETEQ); 484f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard SDValue Ops[2]; 485f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard Ops[0] = Div; 486f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard Ops[1] = Rem; 487f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard return DAG.getMergeValues(Ops, 2, DL); 488f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard} 489f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 490f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard//===----------------------------------------------------------------------===// 491f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard// Helper functions 492f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard//===----------------------------------------------------------------------===// 493f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 494f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellardbool AMDGPUTargetLowering::isHWTrueValue(SDValue Op) const { 495f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard if (ConstantFPSDNode * CFP = dyn_cast<ConstantFPSDNode>(Op)) { 496f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard return CFP->isExactlyValue(1.0); 497f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard } 498f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op)) { 499f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard return C->isAllOnesValue(); 500f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard } 501f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard return false; 502f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard} 503f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 504f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellardbool AMDGPUTargetLowering::isHWFalseValue(SDValue Op) const { 505f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard if (ConstantFPSDNode * CFP = dyn_cast<ConstantFPSDNode>(Op)) { 506f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard return CFP->getValueAPF().isZero(); 507f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard } 508f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op)) { 509f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard return C->isNullValue(); 510f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard } 511f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard return false; 512f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard} 513f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 514f98f2ce29e6e2996fa58f38979143eceaa818335Tom StellardSDValue AMDGPUTargetLowering::CreateLiveInRegister(SelectionDAG &DAG, 515f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard const TargetRegisterClass *RC, 516f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard unsigned Reg, EVT VT) const { 517f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard MachineFunction &MF = DAG.getMachineFunction(); 518f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard MachineRegisterInfo &MRI = MF.getRegInfo(); 519f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard unsigned VirtualRegister; 520f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard if (!MRI.isLiveIn(Reg)) { 521f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard VirtualRegister = MRI.createVirtualRegister(RC); 522f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard MRI.addLiveIn(Reg, VirtualRegister); 523f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard } else { 524f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard VirtualRegister = MRI.getLiveInVirtReg(Reg); 525f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard } 526f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard return DAG.getRegister(VirtualRegister, VT); 527f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard} 528f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 529f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard#define NODE_NAME_CASE(node) case AMDGPUISD::node: return #node; 530f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 531f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellardconst char* AMDGPUTargetLowering::getTargetNodeName(unsigned Opcode) const { 532f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard switch (Opcode) { 533f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard default: return 0; 534f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard // AMDIL DAG nodes 535f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard NODE_NAME_CASE(CALL); 536f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard NODE_NAME_CASE(UMUL); 537f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard NODE_NAME_CASE(DIV_INF); 538f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard NODE_NAME_CASE(RET_FLAG); 539f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard NODE_NAME_CASE(BRANCH_COND); 540f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard 541f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard // AMDGPU DAG nodes 542f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard NODE_NAME_CASE(DWORDADDR) 543f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard NODE_NAME_CASE(FRACT) 544f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard NODE_NAME_CASE(FMAX) 545f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard NODE_NAME_CASE(SMAX) 546f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard NODE_NAME_CASE(UMAX) 547f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard NODE_NAME_CASE(FMIN) 548f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard NODE_NAME_CASE(SMIN) 549f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard NODE_NAME_CASE(UMIN) 550f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard NODE_NAME_CASE(URECIP) 551f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard NODE_NAME_CASE(EXPORT) 552c7e1888d93f4cb2982266986f3af7e99df631fa1Tom Stellard NODE_NAME_CASE(CONST_ADDRESS) 553c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard NODE_NAME_CASE(REGISTER_LOAD) 554c0b0c677a1138f0a5ce1435fc1e70cef38fd95c8Tom Stellard NODE_NAME_CASE(REGISTER_STORE) 55568db37b952be497c94c7aa98cf26f3baadb5afd3Tom Stellard NODE_NAME_CASE(LOAD_CONSTANT) 55668db37b952be497c94c7aa98cf26f3baadb5afd3Tom Stellard NODE_NAME_CASE(LOAD_INPUT) 55768db37b952be497c94c7aa98cf26f3baadb5afd3Tom Stellard NODE_NAME_CASE(SAMPLE) 55868db37b952be497c94c7aa98cf26f3baadb5afd3Tom Stellard NODE_NAME_CASE(SAMPLEB) 55968db37b952be497c94c7aa98cf26f3baadb5afd3Tom Stellard NODE_NAME_CASE(SAMPLED) 56068db37b952be497c94c7aa98cf26f3baadb5afd3Tom Stellard NODE_NAME_CASE(SAMPLEL) 561f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard } 562f98f2ce29e6e2996fa58f38979143eceaa818335Tom Stellard} 563