1f903da7335433ae243cf7ff59662be1a03ee9a14Tom Stellard//===-- AMDGPUISelLowering.cpp - AMDGPU Common DAG lowering functions -----===// 2a75c6163e605f35b14f26930dd9227e4f337ec9eTom Stellard// 3a75c6163e605f35b14f26930dd9227e4f337ec9eTom Stellard// The LLVM Compiler Infrastructure 4a75c6163e605f35b14f26930dd9227e4f337ec9eTom Stellard// 5a75c6163e605f35b14f26930dd9227e4f337ec9eTom Stellard// This file is distributed under the University of Illinois Open Source 6a75c6163e605f35b14f26930dd9227e4f337ec9eTom Stellard// License. See LICENSE.TXT for details. 7a75c6163e605f35b14f26930dd9227e4f337ec9eTom Stellard// 8a75c6163e605f35b14f26930dd9227e4f337ec9eTom Stellard//===----------------------------------------------------------------------===// 9a75c6163e605f35b14f26930dd9227e4f337ec9eTom Stellard// 10f903da7335433ae243cf7ff59662be1a03ee9a14Tom Stellard// This is the parent TargetLowering class for hardware code gen targets. 11a75c6163e605f35b14f26930dd9227e4f337ec9eTom Stellard// 12a75c6163e605f35b14f26930dd9227e4f337ec9eTom Stellard//===----------------------------------------------------------------------===// 13a75c6163e605f35b14f26930dd9227e4f337ec9eTom Stellard 14a75c6163e605f35b14f26930dd9227e4f337ec9eTom Stellard#include "AMDGPUISelLowering.h" 15431bb79a41bd5e7402954385daea1594c3e750abTom Stellard#include "AMDILIntrinsicInfo.h" 1640c41fe890e53d99afb4e2c3fbf10043081edd9eTom Stellard#include "llvm/CodeGen/MachineFunction.h" 17a75c6163e605f35b14f26930dd9227e4f337ec9eTom Stellard#include "llvm/CodeGen/MachineRegisterInfo.h" 18b6051bc7859829588b2361da96f8e828a7fe1326Tom Stellard#include "llvm/CodeGen/SelectionDAG.h" 1927ae41c83dafcec09e870b3cf08b060064dbb122Tom Stellard#include "llvm/CodeGen/TargetLoweringObjectFileImpl.h" 20a75c6163e605f35b14f26930dd9227e4f337ec9eTom Stellard 21a75c6163e605f35b14f26930dd9227e4f337ec9eTom Stellardusing namespace llvm; 22a75c6163e605f35b14f26930dd9227e4f337ec9eTom Stellard 23a75c6163e605f35b14f26930dd9227e4f337ec9eTom StellardAMDGPUTargetLowering::AMDGPUTargetLowering(TargetMachine &TM) : 2427ae41c83dafcec09e870b3cf08b060064dbb122Tom Stellard TargetLowering(TM, new TargetLoweringObjectFileELF()) 25a75c6163e605f35b14f26930dd9227e4f337ec9eTom Stellard{ 2627ae41c83dafcec09e870b3cf08b060064dbb122Tom Stellard 2727ae41c83dafcec09e870b3cf08b060064dbb122Tom Stellard // Initialize target lowering borrowed from AMDIL 2827ae41c83dafcec09e870b3cf08b060064dbb122Tom Stellard InitAMDILLowering(); 2927ae41c83dafcec09e870b3cf08b060064dbb122Tom Stellard 30431bb79a41bd5e7402954385daea1594c3e750abTom Stellard // We need to custom lower some of the intrinsics 31431bb79a41bd5e7402954385daea1594c3e750abTom Stellard setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::Other, Custom); 32cee23ab246f22210b3063cdc47bdb45b3d943526Tom Stellard 330bfa3b3e9629d81a5e31c1b91fd25eab734804faTom Stellard // Library functions. These default to Expand, but we have instructions 340bfa3b3e9629d81a5e31c1b91fd25eab734804faTom Stellard // for them. 350bfa3b3e9629d81a5e31c1b91fd25eab734804faTom Stellard setOperationAction(ISD::FCEIL, MVT::f32, Legal); 36a8ba697c1ec3e07e331cba85e67bf5c2b8d41e57Tom Stellard setOperationAction(ISD::FEXP2, MVT::f32, Legal); 37ea00632fe0667766783fb66f9db5198554fee159Tom Stellard setOperationAction(ISD::FRINT, MVT::f32, Legal); 380bfa3b3e9629d81a5e31c1b91fd25eab734804faTom Stellard 39c01199dfc0d30ad4c20cc4a2ebe3cdcbc74debb6Tom Stellard setOperationAction(ISD::UDIV, MVT::i32, Expand); 4033e7db9a1dafdcf5c7c745180831403e0485544dTom Stellard setOperationAction(ISD::UDIVREM, MVT::i32, Custom); 41c01199dfc0d30ad4c20cc4a2ebe3cdcbc74debb6Tom Stellard setOperationAction(ISD::UREM, MVT::i32, Expand); 42431bb79a41bd5e7402954385daea1594c3e750abTom Stellard} 43431bb79a41bd5e7402954385daea1594c3e750abTom Stellard 440ce6e506016222b264163ee718202371f19064dbTom Stellard//===---------------------------------------------------------------------===// 450ce6e506016222b264163ee718202371f19064dbTom Stellard// TargetLowering Callbacks 460ce6e506016222b264163ee718202371f19064dbTom Stellard//===---------------------------------------------------------------------===// 470ce6e506016222b264163ee718202371f19064dbTom Stellard 480ce6e506016222b264163ee718202371f19064dbTom StellardSDValue AMDGPUTargetLowering::LowerFormalArguments( 490ce6e506016222b264163ee718202371f19064dbTom Stellard SDValue Chain, 500ce6e506016222b264163ee718202371f19064dbTom Stellard CallingConv::ID CallConv, 510ce6e506016222b264163ee718202371f19064dbTom Stellard bool isVarArg, 520ce6e506016222b264163ee718202371f19064dbTom Stellard const SmallVectorImpl<ISD::InputArg> &Ins, 530ce6e506016222b264163ee718202371f19064dbTom Stellard DebugLoc DL, SelectionDAG &DAG, 540ce6e506016222b264163ee718202371f19064dbTom Stellard SmallVectorImpl<SDValue> &InVals) const 550ce6e506016222b264163ee718202371f19064dbTom Stellard{ 560ce6e506016222b264163ee718202371f19064dbTom Stellard // Lowering of arguments happens in R600LowerKernelParameters, so we can 570ce6e506016222b264163ee718202371f19064dbTom Stellard // ignore the arguments here. 580ce6e506016222b264163ee718202371f19064dbTom Stellard for (unsigned i = 0, e = Ins.size(); i < e; ++i) { 590ce6e506016222b264163ee718202371f19064dbTom Stellard InVals.push_back(SDValue()); 600ce6e506016222b264163ee718202371f19064dbTom Stellard } 610ce6e506016222b264163ee718202371f19064dbTom Stellard return Chain; 620ce6e506016222b264163ee718202371f19064dbTom Stellard} 630ce6e506016222b264163ee718202371f19064dbTom Stellard 640ce6e506016222b264163ee718202371f19064dbTom StellardSDValue AMDGPUTargetLowering::LowerReturn( 650ce6e506016222b264163ee718202371f19064dbTom Stellard SDValue Chain, 660ce6e506016222b264163ee718202371f19064dbTom Stellard CallingConv::ID CallConv, 670ce6e506016222b264163ee718202371f19064dbTom Stellard bool isVarArg, 680ce6e506016222b264163ee718202371f19064dbTom Stellard const SmallVectorImpl<ISD::OutputArg> &Outs, 690ce6e506016222b264163ee718202371f19064dbTom Stellard const SmallVectorImpl<SDValue> &OutVals, 700ce6e506016222b264163ee718202371f19064dbTom Stellard DebugLoc DL, SelectionDAG &DAG) const 710ce6e506016222b264163ee718202371f19064dbTom Stellard{ 7227ae41c83dafcec09e870b3cf08b060064dbb122Tom Stellard return DAG.getNode(AMDGPUISD::RET_FLAG, DL, MVT::Other, Chain); 730ce6e506016222b264163ee718202371f19064dbTom Stellard} 740ce6e506016222b264163ee718202371f19064dbTom Stellard 750ce6e506016222b264163ee718202371f19064dbTom Stellard//===---------------------------------------------------------------------===// 760ce6e506016222b264163ee718202371f19064dbTom Stellard// Target specific lowering 770ce6e506016222b264163ee718202371f19064dbTom Stellard//===---------------------------------------------------------------------===// 780ce6e506016222b264163ee718202371f19064dbTom Stellard 79431bb79a41bd5e7402954385daea1594c3e750abTom StellardSDValue AMDGPUTargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) 80431bb79a41bd5e7402954385daea1594c3e750abTom Stellard const 81431bb79a41bd5e7402954385daea1594c3e750abTom Stellard{ 82431bb79a41bd5e7402954385daea1594c3e750abTom Stellard switch (Op.getOpcode()) { 8327ae41c83dafcec09e870b3cf08b060064dbb122Tom Stellard default: 8427ae41c83dafcec09e870b3cf08b060064dbb122Tom Stellard Op.getNode()->dump(); 8527ae41c83dafcec09e870b3cf08b060064dbb122Tom Stellard assert(0 && "Custom lowering code for this" 8627ae41c83dafcec09e870b3cf08b060064dbb122Tom Stellard "instruction is not implemented yet!"); 8727ae41c83dafcec09e870b3cf08b060064dbb122Tom Stellard break; 8827ae41c83dafcec09e870b3cf08b060064dbb122Tom Stellard // AMDIL DAG lowering 8927ae41c83dafcec09e870b3cf08b060064dbb122Tom Stellard case ISD::SDIV: return LowerSDIV(Op, DAG); 9027ae41c83dafcec09e870b3cf08b060064dbb122Tom Stellard case ISD::SREM: return LowerSREM(Op, DAG); 9127ae41c83dafcec09e870b3cf08b060064dbb122Tom Stellard case ISD::BUILD_VECTOR: return LowerBUILD_VECTOR(Op, DAG); 9227ae41c83dafcec09e870b3cf08b060064dbb122Tom Stellard case ISD::SIGN_EXTEND_INREG: return LowerSIGN_EXTEND_INREG(Op, DAG); 9327ae41c83dafcec09e870b3cf08b060064dbb122Tom Stellard case ISD::BRCOND: return LowerBRCOND(Op, DAG); 9427ae41c83dafcec09e870b3cf08b060064dbb122Tom Stellard // AMDGPU DAG lowering 95431bb79a41bd5e7402954385daea1594c3e750abTom Stellard case ISD::INTRINSIC_WO_CHAIN: return LowerINTRINSIC_WO_CHAIN(Op, DAG); 9633e7db9a1dafdcf5c7c745180831403e0485544dTom Stellard case ISD::UDIVREM: return LowerUDIVREM(Op, DAG); 97431bb79a41bd5e7402954385daea1594c3e750abTom Stellard } 9827ae41c83dafcec09e870b3cf08b060064dbb122Tom Stellard return Op; 99431bb79a41bd5e7402954385daea1594c3e750abTom Stellard} 100431bb79a41bd5e7402954385daea1594c3e750abTom Stellard 101431bb79a41bd5e7402954385daea1594c3e750abTom StellardSDValue AMDGPUTargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op, 102431bb79a41bd5e7402954385daea1594c3e750abTom Stellard SelectionDAG &DAG) const 103431bb79a41bd5e7402954385daea1594c3e750abTom Stellard{ 104431bb79a41bd5e7402954385daea1594c3e750abTom Stellard unsigned IntrinsicID = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue(); 105431bb79a41bd5e7402954385daea1594c3e750abTom Stellard DebugLoc DL = Op.getDebugLoc(); 106431bb79a41bd5e7402954385daea1594c3e750abTom Stellard EVT VT = Op.getValueType(); 107431bb79a41bd5e7402954385daea1594c3e750abTom Stellard 108431bb79a41bd5e7402954385daea1594c3e750abTom Stellard switch (IntrinsicID) { 109431bb79a41bd5e7402954385daea1594c3e750abTom Stellard default: return Op; 1109a020092aedc6310d5bfc72b2aa6fc4348fe5c32Tom Stellard case AMDGPUIntrinsic::AMDIL_abs: 1119a020092aedc6310d5bfc72b2aa6fc4348fe5c32Tom Stellard return LowerIntrinsicIABS(Op, DAG); 112a8ba697c1ec3e07e331cba85e67bf5c2b8d41e57Tom Stellard case AMDGPUIntrinsic::AMDIL_exp: 113a8ba697c1ec3e07e331cba85e67bf5c2b8d41e57Tom Stellard return DAG.getNode(ISD::FEXP2, DL, VT, Op.getOperand(1)); 11417f852892346fdf3b1e9eec56b7a55c470279bc8Tom Stellard case AMDGPUIntrinsic::AMDIL_fabs: 11517f852892346fdf3b1e9eec56b7a55c470279bc8Tom Stellard return DAG.getNode(ISD::FABS, DL, VT, Op.getOperand(1)); 116c6c8a05c509b30600d2ccb4be635f05cd71c68a4Tom Stellard case AMDGPUIntrinsic::AMDGPU_lrp: 117c6c8a05c509b30600d2ccb4be635f05cd71c68a4Tom Stellard return LowerIntrinsicLRP(Op, DAG); 118d4984f346320e64b58e38e443e5b99d09b7067bcTom Stellard case AMDGPUIntrinsic::AMDIL_fraction: 119d4984f346320e64b58e38e443e5b99d09b7067bcTom Stellard return DAG.getNode(AMDGPUISD::FRACT, DL, VT, Op.getOperand(1)); 120ef8e66bc165ea2ef9987ab6406268ce195f74eb0Tom Stellard case AMDGPUIntrinsic::AMDIL_mad: 12127ae41c83dafcec09e870b3cf08b060064dbb122Tom Stellard return DAG.getNode(AMDGPUISD::MAD, DL, VT, Op.getOperand(1), 122ef8e66bc165ea2ef9987ab6406268ce195f74eb0Tom Stellard Op.getOperand(2), Op.getOperand(3)); 123431bb79a41bd5e7402954385daea1594c3e750abTom Stellard case AMDGPUIntrinsic::AMDIL_max: 124431bb79a41bd5e7402954385daea1594c3e750abTom Stellard return DAG.getNode(AMDGPUISD::FMAX, DL, VT, Op.getOperand(1), 125431bb79a41bd5e7402954385daea1594c3e750abTom Stellard Op.getOperand(2)); 126431bb79a41bd5e7402954385daea1594c3e750abTom Stellard case AMDGPUIntrinsic::AMDGPU_imax: 127431bb79a41bd5e7402954385daea1594c3e750abTom Stellard return DAG.getNode(AMDGPUISD::SMAX, DL, VT, Op.getOperand(1), 128431bb79a41bd5e7402954385daea1594c3e750abTom Stellard Op.getOperand(2)); 129431bb79a41bd5e7402954385daea1594c3e750abTom Stellard case AMDGPUIntrinsic::AMDGPU_umax: 130431bb79a41bd5e7402954385daea1594c3e750abTom Stellard return DAG.getNode(AMDGPUISD::UMAX, DL, VT, Op.getOperand(1), 131431bb79a41bd5e7402954385daea1594c3e750abTom Stellard Op.getOperand(2)); 1327e3cd8df183448e2cc01a8f2645a001b0972f4abTom Stellard case AMDGPUIntrinsic::AMDIL_min: 1337e3cd8df183448e2cc01a8f2645a001b0972f4abTom Stellard return DAG.getNode(AMDGPUISD::FMIN, DL, VT, Op.getOperand(1), 1347e3cd8df183448e2cc01a8f2645a001b0972f4abTom Stellard Op.getOperand(2)); 1357e3cd8df183448e2cc01a8f2645a001b0972f4abTom Stellard case AMDGPUIntrinsic::AMDGPU_imin: 1367e3cd8df183448e2cc01a8f2645a001b0972f4abTom Stellard return DAG.getNode(AMDGPUISD::SMIN, DL, VT, Op.getOperand(1), 1377e3cd8df183448e2cc01a8f2645a001b0972f4abTom Stellard Op.getOperand(2)); 1387e3cd8df183448e2cc01a8f2645a001b0972f4abTom Stellard case AMDGPUIntrinsic::AMDGPU_umin: 1397e3cd8df183448e2cc01a8f2645a001b0972f4abTom Stellard return DAG.getNode(AMDGPUISD::UMIN, DL, VT, Op.getOperand(1), 1407e3cd8df183448e2cc01a8f2645a001b0972f4abTom Stellard Op.getOperand(2)); 141ea00632fe0667766783fb66f9db5198554fee159Tom Stellard case AMDGPUIntrinsic::AMDIL_round_nearest: 142ea00632fe0667766783fb66f9db5198554fee159Tom Stellard return DAG.getNode(ISD::FRINT, DL, VT, Op.getOperand(1)); 1430bfa3b3e9629d81a5e31c1b91fd25eab734804faTom Stellard case AMDGPUIntrinsic::AMDIL_round_posinf: 1440bfa3b3e9629d81a5e31c1b91fd25eab734804faTom Stellard return DAG.getNode(ISD::FCEIL, DL, VT, Op.getOperand(1)); 145431bb79a41bd5e7402954385daea1594c3e750abTom Stellard } 146a75c6163e605f35b14f26930dd9227e4f337ec9eTom Stellard} 147a75c6163e605f35b14f26930dd9227e4f337ec9eTom Stellard 1489a020092aedc6310d5bfc72b2aa6fc4348fe5c32Tom Stellard///IABS(a) = SMAX(sub(0, a), a) 1499a020092aedc6310d5bfc72b2aa6fc4348fe5c32Tom StellardSDValue AMDGPUTargetLowering::LowerIntrinsicIABS(SDValue Op, 1509a020092aedc6310d5bfc72b2aa6fc4348fe5c32Tom Stellard SelectionDAG &DAG) const 1519a020092aedc6310d5bfc72b2aa6fc4348fe5c32Tom Stellard{ 1529a020092aedc6310d5bfc72b2aa6fc4348fe5c32Tom Stellard 1539a020092aedc6310d5bfc72b2aa6fc4348fe5c32Tom Stellard DebugLoc DL = Op.getDebugLoc(); 1549a020092aedc6310d5bfc72b2aa6fc4348fe5c32Tom Stellard EVT VT = Op.getValueType(); 1559a020092aedc6310d5bfc72b2aa6fc4348fe5c32Tom Stellard SDValue Neg = DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, VT), 1569a020092aedc6310d5bfc72b2aa6fc4348fe5c32Tom Stellard Op.getOperand(1)); 1579a020092aedc6310d5bfc72b2aa6fc4348fe5c32Tom Stellard 1589a020092aedc6310d5bfc72b2aa6fc4348fe5c32Tom Stellard return DAG.getNode(AMDGPUISD::SMAX, DL, VT, Neg, Op.getOperand(1)); 1599a020092aedc6310d5bfc72b2aa6fc4348fe5c32Tom Stellard} 1609a020092aedc6310d5bfc72b2aa6fc4348fe5c32Tom Stellard 161c6c8a05c509b30600d2ccb4be635f05cd71c68a4Tom Stellard/// Linear Interpolation 162c6c8a05c509b30600d2ccb4be635f05cd71c68a4Tom Stellard/// LRP(a, b, c) = muladd(a, b, (1 - a) * c) 163c6c8a05c509b30600d2ccb4be635f05cd71c68a4Tom StellardSDValue AMDGPUTargetLowering::LowerIntrinsicLRP(SDValue Op, 164c6c8a05c509b30600d2ccb4be635f05cd71c68a4Tom Stellard SelectionDAG &DAG) const 165c6c8a05c509b30600d2ccb4be635f05cd71c68a4Tom Stellard{ 166c6c8a05c509b30600d2ccb4be635f05cd71c68a4Tom Stellard DebugLoc DL = Op.getDebugLoc(); 167c6c8a05c509b30600d2ccb4be635f05cd71c68a4Tom Stellard EVT VT = Op.getValueType(); 168c20e7417992380871261699c2b0123819e7d51fcTom Stellard SDValue OneSubA = DAG.getNode(ISD::FSUB, DL, VT, 169c20e7417992380871261699c2b0123819e7d51fcTom Stellard DAG.getConstantFP(1.0f, MVT::f32), 170c20e7417992380871261699c2b0123819e7d51fcTom Stellard Op.getOperand(1)); 171c6c8a05c509b30600d2ccb4be635f05cd71c68a4Tom Stellard SDValue OneSubAC = DAG.getNode(ISD::FMUL, DL, VT, OneSubA, 172c6c8a05c509b30600d2ccb4be635f05cd71c68a4Tom Stellard Op.getOperand(3)); 17327ae41c83dafcec09e870b3cf08b060064dbb122Tom Stellard return DAG.getNode(AMDGPUISD::MAD, DL, VT, Op.getOperand(1), 174c6c8a05c509b30600d2ccb4be635f05cd71c68a4Tom Stellard Op.getOperand(2), 175c6c8a05c509b30600d2ccb4be635f05cd71c68a4Tom Stellard OneSubAC); 176c6c8a05c509b30600d2ccb4be635f05cd71c68a4Tom Stellard} 177c6c8a05c509b30600d2ccb4be635f05cd71c68a4Tom Stellard 178cee23ab246f22210b3063cdc47bdb45b3d943526Tom Stellard 17933e7db9a1dafdcf5c7c745180831403e0485544dTom Stellard 18033e7db9a1dafdcf5c7c745180831403e0485544dTom StellardSDValue AMDGPUTargetLowering::LowerUDIVREM(SDValue Op, 18133e7db9a1dafdcf5c7c745180831403e0485544dTom Stellard SelectionDAG &DAG) const 18233e7db9a1dafdcf5c7c745180831403e0485544dTom Stellard{ 18333e7db9a1dafdcf5c7c745180831403e0485544dTom Stellard DebugLoc DL = Op.getDebugLoc(); 18433e7db9a1dafdcf5c7c745180831403e0485544dTom Stellard EVT VT = Op.getValueType(); 18533e7db9a1dafdcf5c7c745180831403e0485544dTom Stellard 18633e7db9a1dafdcf5c7c745180831403e0485544dTom Stellard SDValue Num = Op.getOperand(0); 18733e7db9a1dafdcf5c7c745180831403e0485544dTom Stellard SDValue Den = Op.getOperand(1); 18833e7db9a1dafdcf5c7c745180831403e0485544dTom Stellard 18933e7db9a1dafdcf5c7c745180831403e0485544dTom Stellard SmallVector<SDValue, 8> Results; 19033e7db9a1dafdcf5c7c745180831403e0485544dTom Stellard 19133e7db9a1dafdcf5c7c745180831403e0485544dTom Stellard // RCP = URECIP(Den) = 2^32 / Den + e 19233e7db9a1dafdcf5c7c745180831403e0485544dTom Stellard // e is rounding error. 19333e7db9a1dafdcf5c7c745180831403e0485544dTom Stellard SDValue RCP = DAG.getNode(AMDGPUISD::URECIP, DL, VT, Den); 19433e7db9a1dafdcf5c7c745180831403e0485544dTom Stellard 19533e7db9a1dafdcf5c7c745180831403e0485544dTom Stellard // RCP_LO = umulo(RCP, Den) */ 19633e7db9a1dafdcf5c7c745180831403e0485544dTom Stellard SDValue RCP_LO = DAG.getNode(ISD::UMULO, DL, VT, RCP, Den); 19733e7db9a1dafdcf5c7c745180831403e0485544dTom Stellard 19833e7db9a1dafdcf5c7c745180831403e0485544dTom Stellard // RCP_HI = mulhu (RCP, Den) */ 19933e7db9a1dafdcf5c7c745180831403e0485544dTom Stellard SDValue RCP_HI = DAG.getNode(ISD::MULHU, DL, VT, RCP, Den); 20033e7db9a1dafdcf5c7c745180831403e0485544dTom Stellard 20133e7db9a1dafdcf5c7c745180831403e0485544dTom Stellard // NEG_RCP_LO = -RCP_LO 20233e7db9a1dafdcf5c7c745180831403e0485544dTom Stellard SDValue NEG_RCP_LO = DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, VT), 20333e7db9a1dafdcf5c7c745180831403e0485544dTom Stellard RCP_LO); 20433e7db9a1dafdcf5c7c745180831403e0485544dTom Stellard 20533e7db9a1dafdcf5c7c745180831403e0485544dTom Stellard // ABS_RCP_LO = (RCP_HI == 0 ? NEG_RCP_LO : RCP_LO) 20633e7db9a1dafdcf5c7c745180831403e0485544dTom Stellard SDValue ABS_RCP_LO = DAG.getSelectCC(DL, RCP_HI, DAG.getConstant(0, VT), 20733e7db9a1dafdcf5c7c745180831403e0485544dTom Stellard NEG_RCP_LO, RCP_LO, 20833e7db9a1dafdcf5c7c745180831403e0485544dTom Stellard ISD::SETEQ); 20933e7db9a1dafdcf5c7c745180831403e0485544dTom Stellard // Calculate the rounding error from the URECIP instruction 21033e7db9a1dafdcf5c7c745180831403e0485544dTom Stellard // E = mulhu(ABS_RCP_LO, RCP) 21133e7db9a1dafdcf5c7c745180831403e0485544dTom Stellard SDValue E = DAG.getNode(ISD::MULHU, DL, VT, ABS_RCP_LO, RCP); 21233e7db9a1dafdcf5c7c745180831403e0485544dTom Stellard 21333e7db9a1dafdcf5c7c745180831403e0485544dTom Stellard // RCP_A_E = RCP + E 21433e7db9a1dafdcf5c7c745180831403e0485544dTom Stellard SDValue RCP_A_E = DAG.getNode(ISD::ADD, DL, VT, RCP, E); 21533e7db9a1dafdcf5c7c745180831403e0485544dTom Stellard 21633e7db9a1dafdcf5c7c745180831403e0485544dTom Stellard // RCP_S_E = RCP - E 21733e7db9a1dafdcf5c7c745180831403e0485544dTom Stellard SDValue RCP_S_E = DAG.getNode(ISD::SUB, DL, VT, RCP, E); 21833e7db9a1dafdcf5c7c745180831403e0485544dTom Stellard 21933e7db9a1dafdcf5c7c745180831403e0485544dTom Stellard // Tmp0 = (RCP_HI == 0 ? RCP_A_E : RCP_SUB_E) 22033e7db9a1dafdcf5c7c745180831403e0485544dTom Stellard SDValue Tmp0 = DAG.getSelectCC(DL, RCP_HI, DAG.getConstant(0, VT), 22133e7db9a1dafdcf5c7c745180831403e0485544dTom Stellard RCP_A_E, RCP_S_E, 22233e7db9a1dafdcf5c7c745180831403e0485544dTom Stellard ISD::SETEQ); 22333e7db9a1dafdcf5c7c745180831403e0485544dTom Stellard // Quotient = mulhu(Tmp0, Num) 22433e7db9a1dafdcf5c7c745180831403e0485544dTom Stellard SDValue Quotient = DAG.getNode(ISD::MULHU, DL, VT, Tmp0, Num); 22533e7db9a1dafdcf5c7c745180831403e0485544dTom Stellard 22633e7db9a1dafdcf5c7c745180831403e0485544dTom Stellard // Num_S_Remainder = Quotient * Den 22733e7db9a1dafdcf5c7c745180831403e0485544dTom Stellard SDValue Num_S_Remainder = DAG.getNode(ISD::UMULO, DL, VT, Quotient, Den); 22833e7db9a1dafdcf5c7c745180831403e0485544dTom Stellard 22933e7db9a1dafdcf5c7c745180831403e0485544dTom Stellard // Remainder = Num - Num_S_Remainder 23033e7db9a1dafdcf5c7c745180831403e0485544dTom Stellard SDValue Remainder = DAG.getNode(ISD::SUB, DL, VT, Num, Num_S_Remainder); 23133e7db9a1dafdcf5c7c745180831403e0485544dTom Stellard 23233e7db9a1dafdcf5c7c745180831403e0485544dTom Stellard // Remainder_GE_Den = (Remainder >= Den ? -1 : 0) 23333e7db9a1dafdcf5c7c745180831403e0485544dTom Stellard SDValue Remainder_GE_Den = DAG.getSelectCC(DL, Remainder, Den, 23433e7db9a1dafdcf5c7c745180831403e0485544dTom Stellard DAG.getConstant(-1, VT), 23533e7db9a1dafdcf5c7c745180831403e0485544dTom Stellard DAG.getConstant(0, VT), 23633e7db9a1dafdcf5c7c745180831403e0485544dTom Stellard ISD::SETGE); 23733e7db9a1dafdcf5c7c745180831403e0485544dTom Stellard // Remainder_GE_Zero = (Remainder >= 0 ? -1 : 0) 23833e7db9a1dafdcf5c7c745180831403e0485544dTom Stellard SDValue Remainder_GE_Zero = DAG.getSelectCC(DL, Remainder, 23933e7db9a1dafdcf5c7c745180831403e0485544dTom Stellard DAG.getConstant(0, VT), 24033e7db9a1dafdcf5c7c745180831403e0485544dTom Stellard DAG.getConstant(-1, VT), 24133e7db9a1dafdcf5c7c745180831403e0485544dTom Stellard DAG.getConstant(0, VT), 24233e7db9a1dafdcf5c7c745180831403e0485544dTom Stellard ISD::SETGE); 24333e7db9a1dafdcf5c7c745180831403e0485544dTom Stellard // Tmp1 = Remainder_GE_Den & Remainder_GE_Zero 24433e7db9a1dafdcf5c7c745180831403e0485544dTom Stellard SDValue Tmp1 = DAG.getNode(ISD::AND, DL, VT, Remainder_GE_Den, 24533e7db9a1dafdcf5c7c745180831403e0485544dTom Stellard Remainder_GE_Zero); 24633e7db9a1dafdcf5c7c745180831403e0485544dTom Stellard 24733e7db9a1dafdcf5c7c745180831403e0485544dTom Stellard // Calculate Division result: 24833e7db9a1dafdcf5c7c745180831403e0485544dTom Stellard 24933e7db9a1dafdcf5c7c745180831403e0485544dTom Stellard // Quotient_A_One = Quotient + 1 25033e7db9a1dafdcf5c7c745180831403e0485544dTom Stellard SDValue Quotient_A_One = DAG.getNode(ISD::ADD, DL, VT, Quotient, 25133e7db9a1dafdcf5c7c745180831403e0485544dTom Stellard DAG.getConstant(1, VT)); 25233e7db9a1dafdcf5c7c745180831403e0485544dTom Stellard 25333e7db9a1dafdcf5c7c745180831403e0485544dTom Stellard // Quotient_S_One = Quotient - 1 25433e7db9a1dafdcf5c7c745180831403e0485544dTom Stellard SDValue Quotient_S_One = DAG.getNode(ISD::SUB, DL, VT, Quotient, 25533e7db9a1dafdcf5c7c745180831403e0485544dTom Stellard DAG.getConstant(1, VT)); 25633e7db9a1dafdcf5c7c745180831403e0485544dTom Stellard 25733e7db9a1dafdcf5c7c745180831403e0485544dTom Stellard // Div = (Tmp1 == 0 ? Quotient : Quotient_A_One) 25833e7db9a1dafdcf5c7c745180831403e0485544dTom Stellard SDValue Div = DAG.getSelectCC(DL, Tmp1, DAG.getConstant(0, VT), 25933e7db9a1dafdcf5c7c745180831403e0485544dTom Stellard Quotient, Quotient_A_One, ISD::SETEQ); 26033e7db9a1dafdcf5c7c745180831403e0485544dTom Stellard 26133e7db9a1dafdcf5c7c745180831403e0485544dTom Stellard // Div = (Remainder_GE_Zero == 0 ? Quotient_S_One : Div) 26233e7db9a1dafdcf5c7c745180831403e0485544dTom Stellard Div = DAG.getSelectCC(DL, Remainder_GE_Zero, DAG.getConstant(0, VT), 26333e7db9a1dafdcf5c7c745180831403e0485544dTom Stellard Quotient_S_One, Div, ISD::SETEQ); 26433e7db9a1dafdcf5c7c745180831403e0485544dTom Stellard 26533e7db9a1dafdcf5c7c745180831403e0485544dTom Stellard // Calculate Rem result: 26633e7db9a1dafdcf5c7c745180831403e0485544dTom Stellard 26733e7db9a1dafdcf5c7c745180831403e0485544dTom Stellard // Remainder_S_Den = Remainder - Den 26833e7db9a1dafdcf5c7c745180831403e0485544dTom Stellard SDValue Remainder_S_Den = DAG.getNode(ISD::SUB, DL, VT, Remainder, Den); 26933e7db9a1dafdcf5c7c745180831403e0485544dTom Stellard 27033e7db9a1dafdcf5c7c745180831403e0485544dTom Stellard // Remainder_A_Den = Remainder + Den 27133e7db9a1dafdcf5c7c745180831403e0485544dTom Stellard SDValue Remainder_A_Den = DAG.getNode(ISD::ADD, DL, VT, Remainder, Den); 27233e7db9a1dafdcf5c7c745180831403e0485544dTom Stellard 27333e7db9a1dafdcf5c7c745180831403e0485544dTom Stellard // Rem = (Tmp1 == 0 ? Remainder : Remainder_S_Den) 27433e7db9a1dafdcf5c7c745180831403e0485544dTom Stellard SDValue Rem = DAG.getSelectCC(DL, Tmp1, DAG.getConstant(0, VT), 27533e7db9a1dafdcf5c7c745180831403e0485544dTom Stellard Remainder, Remainder_S_Den, ISD::SETEQ); 27633e7db9a1dafdcf5c7c745180831403e0485544dTom Stellard 27733e7db9a1dafdcf5c7c745180831403e0485544dTom Stellard // Rem = (Remainder_GE_Zero == 0 ? Remainder_A_Den : Rem) 27833e7db9a1dafdcf5c7c745180831403e0485544dTom Stellard Rem = DAG.getSelectCC(DL, Remainder_GE_Zero, DAG.getConstant(0, VT), 27933e7db9a1dafdcf5c7c745180831403e0485544dTom Stellard Remainder_A_Den, Rem, ISD::SETEQ); 28033e7db9a1dafdcf5c7c745180831403e0485544dTom Stellard 28133e7db9a1dafdcf5c7c745180831403e0485544dTom Stellard DAG.ReplaceAllUsesWith(Op.getValue(0).getNode(), &Div); 28233e7db9a1dafdcf5c7c745180831403e0485544dTom Stellard DAG.ReplaceAllUsesWith(Op.getValue(1).getNode(), &Rem); 28333e7db9a1dafdcf5c7c745180831403e0485544dTom Stellard 28433e7db9a1dafdcf5c7c745180831403e0485544dTom Stellard return Op; 28533e7db9a1dafdcf5c7c745180831403e0485544dTom Stellard} 28633e7db9a1dafdcf5c7c745180831403e0485544dTom Stellard 287cee23ab246f22210b3063cdc47bdb45b3d943526Tom Stellard//===----------------------------------------------------------------------===// 288cee23ab246f22210b3063cdc47bdb45b3d943526Tom Stellard// Helper functions 289cee23ab246f22210b3063cdc47bdb45b3d943526Tom Stellard//===----------------------------------------------------------------------===// 290cee23ab246f22210b3063cdc47bdb45b3d943526Tom Stellard 2915523502ff917803166051c8947f5dd3b23c6fcf8Tom Stellardbool AMDGPUTargetLowering::isHWTrueValue(SDValue Op) const 292cee23ab246f22210b3063cdc47bdb45b3d943526Tom Stellard{ 293cee23ab246f22210b3063cdc47bdb45b3d943526Tom Stellard if (ConstantFPSDNode * CFP = dyn_cast<ConstantFPSDNode>(Op)) { 294cee23ab246f22210b3063cdc47bdb45b3d943526Tom Stellard return CFP->isExactlyValue(1.0); 295cee23ab246f22210b3063cdc47bdb45b3d943526Tom Stellard } 296cee23ab246f22210b3063cdc47bdb45b3d943526Tom Stellard if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op)) { 2975523502ff917803166051c8947f5dd3b23c6fcf8Tom Stellard return C->isAllOnesValue(); 298cee23ab246f22210b3063cdc47bdb45b3d943526Tom Stellard } 299cee23ab246f22210b3063cdc47bdb45b3d943526Tom Stellard return false; 300cee23ab246f22210b3063cdc47bdb45b3d943526Tom Stellard} 301cee23ab246f22210b3063cdc47bdb45b3d943526Tom Stellard 3025523502ff917803166051c8947f5dd3b23c6fcf8Tom Stellardbool AMDGPUTargetLowering::isHWFalseValue(SDValue Op) const 303cee23ab246f22210b3063cdc47bdb45b3d943526Tom Stellard{ 304cee23ab246f22210b3063cdc47bdb45b3d943526Tom Stellard if (ConstantFPSDNode * CFP = dyn_cast<ConstantFPSDNode>(Op)) { 305cee23ab246f22210b3063cdc47bdb45b3d943526Tom Stellard return CFP->getValueAPF().isZero(); 306cee23ab246f22210b3063cdc47bdb45b3d943526Tom Stellard } 307cee23ab246f22210b3063cdc47bdb45b3d943526Tom Stellard if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op)) { 308cee23ab246f22210b3063cdc47bdb45b3d943526Tom Stellard return C->isNullValue(); 309cee23ab246f22210b3063cdc47bdb45b3d943526Tom Stellard } 310cee23ab246f22210b3063cdc47bdb45b3d943526Tom Stellard return false; 311cee23ab246f22210b3063cdc47bdb45b3d943526Tom Stellard} 312cee23ab246f22210b3063cdc47bdb45b3d943526Tom Stellard 31340c41fe890e53d99afb4e2c3fbf10043081edd9eTom StellardSDValue AMDGPUTargetLowering::CreateLiveInRegister(SelectionDAG &DAG, 31440c41fe890e53d99afb4e2c3fbf10043081edd9eTom Stellard const TargetRegisterClass *RC, 31540c41fe890e53d99afb4e2c3fbf10043081edd9eTom Stellard unsigned Reg, EVT VT) const { 31640c41fe890e53d99afb4e2c3fbf10043081edd9eTom Stellard MachineFunction &MF = DAG.getMachineFunction(); 31740c41fe890e53d99afb4e2c3fbf10043081edd9eTom Stellard MachineRegisterInfo &MRI = MF.getRegInfo(); 31840c41fe890e53d99afb4e2c3fbf10043081edd9eTom Stellard unsigned VirtualRegister; 31940c41fe890e53d99afb4e2c3fbf10043081edd9eTom Stellard if (!MRI.isLiveIn(Reg)) { 32040c41fe890e53d99afb4e2c3fbf10043081edd9eTom Stellard VirtualRegister = MRI.createVirtualRegister(RC); 32140c41fe890e53d99afb4e2c3fbf10043081edd9eTom Stellard MRI.addLiveIn(Reg, VirtualRegister); 32240c41fe890e53d99afb4e2c3fbf10043081edd9eTom Stellard } else { 32340c41fe890e53d99afb4e2c3fbf10043081edd9eTom Stellard VirtualRegister = MRI.getLiveInVirtReg(Reg); 32440c41fe890e53d99afb4e2c3fbf10043081edd9eTom Stellard } 32540c41fe890e53d99afb4e2c3fbf10043081edd9eTom Stellard return DAG.getRegister(VirtualRegister, VT); 32640c41fe890e53d99afb4e2c3fbf10043081edd9eTom Stellard} 32740c41fe890e53d99afb4e2c3fbf10043081edd9eTom Stellard 328431bb79a41bd5e7402954385daea1594c3e750abTom Stellard#define NODE_NAME_CASE(node) case AMDGPUISD::node: return #node; 329431bb79a41bd5e7402954385daea1594c3e750abTom Stellard 330431bb79a41bd5e7402954385daea1594c3e750abTom Stellardconst char* AMDGPUTargetLowering::getTargetNodeName(unsigned Opcode) const 331431bb79a41bd5e7402954385daea1594c3e750abTom Stellard{ 332431bb79a41bd5e7402954385daea1594c3e750abTom Stellard switch (Opcode) { 33327ae41c83dafcec09e870b3cf08b060064dbb122Tom Stellard default: return 0; 33427ae41c83dafcec09e870b3cf08b060064dbb122Tom Stellard // AMDIL DAG nodes 33527ae41c83dafcec09e870b3cf08b060064dbb122Tom Stellard NODE_NAME_CASE(MAD); 33627ae41c83dafcec09e870b3cf08b060064dbb122Tom Stellard NODE_NAME_CASE(CALL); 33727ae41c83dafcec09e870b3cf08b060064dbb122Tom Stellard NODE_NAME_CASE(UMUL); 33827ae41c83dafcec09e870b3cf08b060064dbb122Tom Stellard NODE_NAME_CASE(DIV_INF); 33927ae41c83dafcec09e870b3cf08b060064dbb122Tom Stellard NODE_NAME_CASE(VBUILD); 34027ae41c83dafcec09e870b3cf08b060064dbb122Tom Stellard NODE_NAME_CASE(RET_FLAG); 34127ae41c83dafcec09e870b3cf08b060064dbb122Tom Stellard NODE_NAME_CASE(BRANCH_COND); 34227ae41c83dafcec09e870b3cf08b060064dbb122Tom Stellard 34327ae41c83dafcec09e870b3cf08b060064dbb122Tom Stellard // AMDGPU DAG nodes 344d4984f346320e64b58e38e443e5b99d09b7067bcTom Stellard NODE_NAME_CASE(FRACT) 345431bb79a41bd5e7402954385daea1594c3e750abTom Stellard NODE_NAME_CASE(FMAX) 346431bb79a41bd5e7402954385daea1594c3e750abTom Stellard NODE_NAME_CASE(SMAX) 347431bb79a41bd5e7402954385daea1594c3e750abTom Stellard NODE_NAME_CASE(UMAX) 3487e3cd8df183448e2cc01a8f2645a001b0972f4abTom Stellard NODE_NAME_CASE(FMIN) 3497e3cd8df183448e2cc01a8f2645a001b0972f4abTom Stellard NODE_NAME_CASE(SMIN) 3507e3cd8df183448e2cc01a8f2645a001b0972f4abTom Stellard NODE_NAME_CASE(UMIN) 35133e7db9a1dafdcf5c7c745180831403e0485544dTom Stellard NODE_NAME_CASE(URECIP) 352431bb79a41bd5e7402954385daea1594c3e750abTom Stellard } 353431bb79a41bd5e7402954385daea1594c3e750abTom Stellard} 354