1a75c6163e605f35b14f26930dd9227e4f337ec9eTom Stellard//===-- AMDILISelDAGToDAG.cpp - A dag to dag inst selector for AMDIL ------===// 2a75c6163e605f35b14f26930dd9227e4f337ec9eTom Stellard// 3a75c6163e605f35b14f26930dd9227e4f337ec9eTom Stellard// The LLVM Compiler Infrastructure 4a75c6163e605f35b14f26930dd9227e4f337ec9eTom Stellard// 5a75c6163e605f35b14f26930dd9227e4f337ec9eTom Stellard// This file is distributed under the University of Illinois Open Source 6a75c6163e605f35b14f26930dd9227e4f337ec9eTom Stellard// License. See LICENSE.TXT for details. 7a75c6163e605f35b14f26930dd9227e4f337ec9eTom Stellard// 8a75c6163e605f35b14f26930dd9227e4f337ec9eTom Stellard//==-----------------------------------------------------------------------===// 9a75c6163e605f35b14f26930dd9227e4f337ec9eTom Stellard// 10a75c6163e605f35b14f26930dd9227e4f337ec9eTom Stellard// This file defines an instruction selector for the AMDIL target. 11a75c6163e605f35b14f26930dd9227e4f337ec9eTom Stellard// 12a75c6163e605f35b14f26930dd9227e4f337ec9eTom Stellard//===----------------------------------------------------------------------===// 13ac669c32c6e80841e3ee63d65b58c0031b22e7b8Tom Stellard#include "AMDGPUInstrInfo.h" 14431bb79a41bd5e7402954385daea1594c3e750abTom Stellard#include "AMDGPUISelLowering.h" // For AMDGPUISD 153a0187b1b53eca3143286a5ae7917cd71117b902Tom Stellard#include "AMDGPURegisterInfo.h" 16a75c6163e605f35b14f26930dd9227e4f337ec9eTom Stellard#include "AMDILDevices.h" 17a75c6163e605f35b14f26930dd9227e4f337ec9eTom Stellard#include "AMDILUtilityFunctions.h" 185aaaa6a426258dc714c7346bec062795998f9986Tom Stellard#include "llvm/ADT/ValueMap.h" 19a75c6163e605f35b14f26930dd9227e4f337ec9eTom Stellard#include "llvm/CodeGen/PseudoSourceValue.h" 20a75c6163e605f35b14f26930dd9227e4f337ec9eTom Stellard#include "llvm/CodeGen/SelectionDAGISel.h" 21a75c6163e605f35b14f26930dd9227e4f337ec9eTom Stellard#include "llvm/Support/Compiler.h" 225aaaa6a426258dc714c7346bec062795998f9986Tom Stellard#include <list> 235aaaa6a426258dc714c7346bec062795998f9986Tom Stellard#include <queue> 24a75c6163e605f35b14f26930dd9227e4f337ec9eTom Stellard 25a75c6163e605f35b14f26930dd9227e4f337ec9eTom Stellardusing namespace llvm; 26a75c6163e605f35b14f26930dd9227e4f337ec9eTom Stellard 27a75c6163e605f35b14f26930dd9227e4f337ec9eTom Stellard//===----------------------------------------------------------------------===// 28a75c6163e605f35b14f26930dd9227e4f337ec9eTom Stellard// Instruction Selector Implementation 29a75c6163e605f35b14f26930dd9227e4f337ec9eTom Stellard//===----------------------------------------------------------------------===// 30a75c6163e605f35b14f26930dd9227e4f337ec9eTom Stellard 31a75c6163e605f35b14f26930dd9227e4f337ec9eTom Stellard//===----------------------------------------------------------------------===// 322f921101c0826dc52a2c69f85c3da0f7f6e8212aTom Stellard// AMDGPUDAGToDAGISel - AMDGPU specific code to select AMDGPU machine instructions 33a75c6163e605f35b14f26930dd9227e4f337ec9eTom Stellard// //for SelectionDAG operations. 34a75c6163e605f35b14f26930dd9227e4f337ec9eTom Stellard// 35a75c6163e605f35b14f26930dd9227e4f337ec9eTom Stellardnamespace { 362f921101c0826dc52a2c69f85c3da0f7f6e8212aTom Stellardclass AMDGPUDAGToDAGISel : public SelectionDAGISel { 372f921101c0826dc52a2c69f85c3da0f7f6e8212aTom Stellard // Subtarget - Keep a pointer to the AMDGPU Subtarget around so that we can 38a75c6163e605f35b14f26930dd9227e4f337ec9eTom Stellard // make the right decision when generating code for different targets. 39b72ab79d73b29ec087d90cf2c698adbab4db5defTom Stellard const AMDGPUSubtarget &Subtarget; 40a75c6163e605f35b14f26930dd9227e4f337ec9eTom Stellardpublic: 41cd0949eb2869ec37b135fdff2110090008bbcff6Tom Stellard AMDGPUDAGToDAGISel(TargetMachine &TM); 422f921101c0826dc52a2c69f85c3da0f7f6e8212aTom Stellard virtual ~AMDGPUDAGToDAGISel(); 43a75c6163e605f35b14f26930dd9227e4f337ec9eTom Stellard 44a75c6163e605f35b14f26930dd9227e4f337ec9eTom Stellard SDNode *Select(SDNode *N); 455aaaa6a426258dc714c7346bec062795998f9986Tom Stellard virtual const char *getPassName() const; 465aaaa6a426258dc714c7346bec062795998f9986Tom Stellard 475aaaa6a426258dc714c7346bec062795998f9986Tom Stellardprivate: 485aaaa6a426258dc714c7346bec062795998f9986Tom Stellard inline SDValue getSmallIPtrImm(unsigned Imm); 495aaaa6a426258dc714c7346bec062795998f9986Tom Stellard 50a75c6163e605f35b14f26930dd9227e4f337ec9eTom Stellard // Complex pattern selectors 51a75c6163e605f35b14f26930dd9227e4f337ec9eTom Stellard bool SelectADDRParam(SDValue Addr, SDValue& R1, SDValue& R2); 52a75c6163e605f35b14f26930dd9227e4f337ec9eTom Stellard bool SelectADDR(SDValue N, SDValue &R1, SDValue &R2); 53a75c6163e605f35b14f26930dd9227e4f337ec9eTom Stellard bool SelectADDR64(SDValue N, SDValue &R1, SDValue &R2); 545aaaa6a426258dc714c7346bec062795998f9986Tom Stellard 555aaaa6a426258dc714c7346bec062795998f9986Tom Stellard static bool checkType(const Value *ptr, unsigned int addrspace); 565aaaa6a426258dc714c7346bec062795998f9986Tom Stellard static const Value *getBasePointerValue(const Value *V); 575aaaa6a426258dc714c7346bec062795998f9986Tom Stellard 58a75c6163e605f35b14f26930dd9227e4f337ec9eTom Stellard static bool isGlobalStore(const StoreSDNode *N); 59a75c6163e605f35b14f26930dd9227e4f337ec9eTom Stellard static bool isPrivateStore(const StoreSDNode *N); 60a75c6163e605f35b14f26930dd9227e4f337ec9eTom Stellard static bool isLocalStore(const StoreSDNode *N); 61a75c6163e605f35b14f26930dd9227e4f337ec9eTom Stellard static bool isRegionStore(const StoreSDNode *N); 62a75c6163e605f35b14f26930dd9227e4f337ec9eTom Stellard 63a75c6163e605f35b14f26930dd9227e4f337ec9eTom Stellard static bool isCPLoad(const LoadSDNode *N); 64a75c6163e605f35b14f26930dd9227e4f337ec9eTom Stellard static bool isConstantLoad(const LoadSDNode *N, int cbID); 65a75c6163e605f35b14f26930dd9227e4f337ec9eTom Stellard static bool isGlobalLoad(const LoadSDNode *N); 66a75c6163e605f35b14f26930dd9227e4f337ec9eTom Stellard static bool isPrivateLoad(const LoadSDNode *N); 67a75c6163e605f35b14f26930dd9227e4f337ec9eTom Stellard static bool isLocalLoad(const LoadSDNode *N); 68a75c6163e605f35b14f26930dd9227e4f337ec9eTom Stellard static bool isRegionLoad(const LoadSDNode *N); 69a75c6163e605f35b14f26930dd9227e4f337ec9eTom Stellard 70467f51613eb1f2cdaa8624bbbb3d5fae2abca4f2Tom Stellard bool SelectADDR8BitOffset(SDValue Addr, SDValue& Base, SDValue& Offset); 71467f51613eb1f2cdaa8624bbbb3d5fae2abca4f2Tom Stellard bool SelectADDRReg(SDValue Addr, SDValue& Base, SDValue& Offset); 720ebf2318b3d5e60adfc43e477b19acdc3cd4cc07Tom Stellard bool SelectADDRVTX_READ(SDValue Addr, SDValue &Base, SDValue &Offset); 73467f51613eb1f2cdaa8624bbbb3d5fae2abca4f2Tom Stellard 74a75c6163e605f35b14f26930dd9227e4f337ec9eTom Stellard // Include the pieces autogenerated from the target description. 7565917004d99ccb79f709e621f8f6cf66715ffdcaTom Stellard#include "AMDGPUGenDAGISel.inc" 76a75c6163e605f35b14f26930dd9227e4f337ec9eTom Stellard}; 77a75c6163e605f35b14f26930dd9227e4f337ec9eTom Stellard} // end anonymous namespace 78a75c6163e605f35b14f26930dd9227e4f337ec9eTom Stellard 792f921101c0826dc52a2c69f85c3da0f7f6e8212aTom Stellard// createAMDGPUISelDag - This pass converts a legalized DAG into a AMDGPU-specific 80a75c6163e605f35b14f26930dd9227e4f337ec9eTom Stellard// DAG, ready for instruction scheduling. 81a75c6163e605f35b14f26930dd9227e4f337ec9eTom Stellard// 822f921101c0826dc52a2c69f85c3da0f7f6e8212aTom StellardFunctionPass *llvm::createAMDGPUISelDag(TargetMachine &TM 83cd0949eb2869ec37b135fdff2110090008bbcff6Tom Stellard ) { 84cd0949eb2869ec37b135fdff2110090008bbcff6Tom Stellard return new AMDGPUDAGToDAGISel(TM); 85a75c6163e605f35b14f26930dd9227e4f337ec9eTom Stellard} 86a75c6163e605f35b14f26930dd9227e4f337ec9eTom Stellard 872f921101c0826dc52a2c69f85c3da0f7f6e8212aTom StellardAMDGPUDAGToDAGISel::AMDGPUDAGToDAGISel(TargetMachine &TM 88cd0949eb2869ec37b135fdff2110090008bbcff6Tom Stellard ) 89cd0949eb2869ec37b135fdff2110090008bbcff6Tom Stellard : SelectionDAGISel(TM), Subtarget(TM.getSubtarget<AMDGPUSubtarget>()) 90a75c6163e605f35b14f26930dd9227e4f337ec9eTom Stellard{ 91a75c6163e605f35b14f26930dd9227e4f337ec9eTom Stellard} 92a75c6163e605f35b14f26930dd9227e4f337ec9eTom Stellard 932f921101c0826dc52a2c69f85c3da0f7f6e8212aTom StellardAMDGPUDAGToDAGISel::~AMDGPUDAGToDAGISel() { 94a75c6163e605f35b14f26930dd9227e4f337ec9eTom Stellard} 95a75c6163e605f35b14f26930dd9227e4f337ec9eTom Stellard 962f921101c0826dc52a2c69f85c3da0f7f6e8212aTom StellardSDValue AMDGPUDAGToDAGISel::getSmallIPtrImm(unsigned int Imm) { 97a75c6163e605f35b14f26930dd9227e4f337ec9eTom Stellard return CurDAG->getTargetConstant(Imm, MVT::i32); 98a75c6163e605f35b14f26930dd9227e4f337ec9eTom Stellard} 99a75c6163e605f35b14f26930dd9227e4f337ec9eTom Stellard 1002f921101c0826dc52a2c69f85c3da0f7f6e8212aTom Stellardbool AMDGPUDAGToDAGISel::SelectADDRParam( 101a75c6163e605f35b14f26930dd9227e4f337ec9eTom Stellard SDValue Addr, SDValue& R1, SDValue& R2) { 102a75c6163e605f35b14f26930dd9227e4f337ec9eTom Stellard 103a75c6163e605f35b14f26930dd9227e4f337ec9eTom Stellard if (Addr.getOpcode() == ISD::FrameIndex) { 104a75c6163e605f35b14f26930dd9227e4f337ec9eTom Stellard if (FrameIndexSDNode *FIN = dyn_cast<FrameIndexSDNode>(Addr)) { 105a75c6163e605f35b14f26930dd9227e4f337ec9eTom Stellard R1 = CurDAG->getTargetFrameIndex(FIN->getIndex(), MVT::i32); 106a75c6163e605f35b14f26930dd9227e4f337ec9eTom Stellard R2 = CurDAG->getTargetConstant(0, MVT::i32); 107a75c6163e605f35b14f26930dd9227e4f337ec9eTom Stellard } else { 108a75c6163e605f35b14f26930dd9227e4f337ec9eTom Stellard R1 = Addr; 109a75c6163e605f35b14f26930dd9227e4f337ec9eTom Stellard R2 = CurDAG->getTargetConstant(0, MVT::i32); 110a75c6163e605f35b14f26930dd9227e4f337ec9eTom Stellard } 111a75c6163e605f35b14f26930dd9227e4f337ec9eTom Stellard } else if (Addr.getOpcode() == ISD::ADD) { 112a75c6163e605f35b14f26930dd9227e4f337ec9eTom Stellard R1 = Addr.getOperand(0); 113a75c6163e605f35b14f26930dd9227e4f337ec9eTom Stellard R2 = Addr.getOperand(1); 114a75c6163e605f35b14f26930dd9227e4f337ec9eTom Stellard } else { 115a75c6163e605f35b14f26930dd9227e4f337ec9eTom Stellard R1 = Addr; 116a75c6163e605f35b14f26930dd9227e4f337ec9eTom Stellard R2 = CurDAG->getTargetConstant(0, MVT::i32); 117a75c6163e605f35b14f26930dd9227e4f337ec9eTom Stellard } 118a75c6163e605f35b14f26930dd9227e4f337ec9eTom Stellard return true; 119a75c6163e605f35b14f26930dd9227e4f337ec9eTom Stellard} 120a75c6163e605f35b14f26930dd9227e4f337ec9eTom Stellard 1212f921101c0826dc52a2c69f85c3da0f7f6e8212aTom Stellardbool AMDGPUDAGToDAGISel::SelectADDR(SDValue Addr, SDValue& R1, SDValue& R2) { 122a75c6163e605f35b14f26930dd9227e4f337ec9eTom Stellard if (Addr.getOpcode() == ISD::TargetExternalSymbol || 123a75c6163e605f35b14f26930dd9227e4f337ec9eTom Stellard Addr.getOpcode() == ISD::TargetGlobalAddress) { 124a75c6163e605f35b14f26930dd9227e4f337ec9eTom Stellard return false; 125a75c6163e605f35b14f26930dd9227e4f337ec9eTom Stellard } 126a75c6163e605f35b14f26930dd9227e4f337ec9eTom Stellard return SelectADDRParam(Addr, R1, R2); 127a75c6163e605f35b14f26930dd9227e4f337ec9eTom Stellard} 128a75c6163e605f35b14f26930dd9227e4f337ec9eTom Stellard 129a75c6163e605f35b14f26930dd9227e4f337ec9eTom Stellard 1302f921101c0826dc52a2c69f85c3da0f7f6e8212aTom Stellardbool AMDGPUDAGToDAGISel::SelectADDR64(SDValue Addr, SDValue& R1, SDValue& R2) { 131a75c6163e605f35b14f26930dd9227e4f337ec9eTom Stellard if (Addr.getOpcode() == ISD::TargetExternalSymbol || 132a75c6163e605f35b14f26930dd9227e4f337ec9eTom Stellard Addr.getOpcode() == ISD::TargetGlobalAddress) { 133a75c6163e605f35b14f26930dd9227e4f337ec9eTom Stellard return false; 134a75c6163e605f35b14f26930dd9227e4f337ec9eTom Stellard } 135a75c6163e605f35b14f26930dd9227e4f337ec9eTom Stellard 136a75c6163e605f35b14f26930dd9227e4f337ec9eTom Stellard if (Addr.getOpcode() == ISD::FrameIndex) { 137a75c6163e605f35b14f26930dd9227e4f337ec9eTom Stellard if (FrameIndexSDNode *FIN = dyn_cast<FrameIndexSDNode>(Addr)) { 138a75c6163e605f35b14f26930dd9227e4f337ec9eTom Stellard R1 = CurDAG->getTargetFrameIndex(FIN->getIndex(), MVT::i64); 139a75c6163e605f35b14f26930dd9227e4f337ec9eTom Stellard R2 = CurDAG->getTargetConstant(0, MVT::i64); 140a75c6163e605f35b14f26930dd9227e4f337ec9eTom Stellard } else { 141a75c6163e605f35b14f26930dd9227e4f337ec9eTom Stellard R1 = Addr; 142a75c6163e605f35b14f26930dd9227e4f337ec9eTom Stellard R2 = CurDAG->getTargetConstant(0, MVT::i64); 143a75c6163e605f35b14f26930dd9227e4f337ec9eTom Stellard } 144a75c6163e605f35b14f26930dd9227e4f337ec9eTom Stellard } else if (Addr.getOpcode() == ISD::ADD) { 145a75c6163e605f35b14f26930dd9227e4f337ec9eTom Stellard R1 = Addr.getOperand(0); 146a75c6163e605f35b14f26930dd9227e4f337ec9eTom Stellard R2 = Addr.getOperand(1); 147a75c6163e605f35b14f26930dd9227e4f337ec9eTom Stellard } else { 148a75c6163e605f35b14f26930dd9227e4f337ec9eTom Stellard R1 = Addr; 149a75c6163e605f35b14f26930dd9227e4f337ec9eTom Stellard R2 = CurDAG->getTargetConstant(0, MVT::i64); 150a75c6163e605f35b14f26930dd9227e4f337ec9eTom Stellard } 151a75c6163e605f35b14f26930dd9227e4f337ec9eTom Stellard return true; 152a75c6163e605f35b14f26930dd9227e4f337ec9eTom Stellard} 153a75c6163e605f35b14f26930dd9227e4f337ec9eTom Stellard 1542f921101c0826dc52a2c69f85c3da0f7f6e8212aTom StellardSDNode *AMDGPUDAGToDAGISel::Select(SDNode *N) { 155a75c6163e605f35b14f26930dd9227e4f337ec9eTom Stellard unsigned int Opc = N->getOpcode(); 156a75c6163e605f35b14f26930dd9227e4f337ec9eTom Stellard if (N->isMachineOpcode()) { 157a75c6163e605f35b14f26930dd9227e4f337ec9eTom Stellard return NULL; // Already selected. 158a75c6163e605f35b14f26930dd9227e4f337ec9eTom Stellard } 159a75c6163e605f35b14f26930dd9227e4f337ec9eTom Stellard switch (Opc) { 160a75c6163e605f35b14f26930dd9227e4f337ec9eTom Stellard default: break; 161a75c6163e605f35b14f26930dd9227e4f337ec9eTom Stellard case ISD::FrameIndex: 162a75c6163e605f35b14f26930dd9227e4f337ec9eTom Stellard { 163a75c6163e605f35b14f26930dd9227e4f337ec9eTom Stellard if (FrameIndexSDNode *FIN = dyn_cast<FrameIndexSDNode>(N)) { 164a75c6163e605f35b14f26930dd9227e4f337ec9eTom Stellard unsigned int FI = FIN->getIndex(); 165a75c6163e605f35b14f26930dd9227e4f337ec9eTom Stellard EVT OpVT = N->getValueType(0); 16676b44034b9b234d3db4012342f0fae677d4f10f6Tom Stellard unsigned int NewOpc = AMDGPU::COPY; 167a75c6163e605f35b14f26930dd9227e4f337ec9eTom Stellard SDValue TFI = CurDAG->getTargetFrameIndex(FI, MVT::i32); 168a75c6163e605f35b14f26930dd9227e4f337ec9eTom Stellard return CurDAG->SelectNodeTo(N, NewOpc, OpVT, TFI); 169a75c6163e605f35b14f26930dd9227e4f337ec9eTom Stellard } 170a75c6163e605f35b14f26930dd9227e4f337ec9eTom Stellard } 171a75c6163e605f35b14f26930dd9227e4f337ec9eTom Stellard break; 172a75c6163e605f35b14f26930dd9227e4f337ec9eTom Stellard } 173a75c6163e605f35b14f26930dd9227e4f337ec9eTom Stellard return SelectCode(N); 174a75c6163e605f35b14f26930dd9227e4f337ec9eTom Stellard} 175a75c6163e605f35b14f26930dd9227e4f337ec9eTom Stellard 1762f921101c0826dc52a2c69f85c3da0f7f6e8212aTom Stellardbool AMDGPUDAGToDAGISel::checkType(const Value *ptr, unsigned int addrspace) { 1775aaaa6a426258dc714c7346bec062795998f9986Tom Stellard if (!ptr) { 1785aaaa6a426258dc714c7346bec062795998f9986Tom Stellard return false; 1795aaaa6a426258dc714c7346bec062795998f9986Tom Stellard } 1805aaaa6a426258dc714c7346bec062795998f9986Tom Stellard Type *ptrType = ptr->getType(); 1815aaaa6a426258dc714c7346bec062795998f9986Tom Stellard return dyn_cast<PointerType>(ptrType)->getAddressSpace() == addrspace; 1825aaaa6a426258dc714c7346bec062795998f9986Tom Stellard} 1835aaaa6a426258dc714c7346bec062795998f9986Tom Stellard 1842f921101c0826dc52a2c69f85c3da0f7f6e8212aTom Stellardconst Value * AMDGPUDAGToDAGISel::getBasePointerValue(const Value *V) 1855aaaa6a426258dc714c7346bec062795998f9986Tom Stellard{ 1865aaaa6a426258dc714c7346bec062795998f9986Tom Stellard if (!V) { 1875aaaa6a426258dc714c7346bec062795998f9986Tom Stellard return NULL; 1885aaaa6a426258dc714c7346bec062795998f9986Tom Stellard } 1895aaaa6a426258dc714c7346bec062795998f9986Tom Stellard const Value *ret = NULL; 1905aaaa6a426258dc714c7346bec062795998f9986Tom Stellard ValueMap<const Value *, bool> ValueBitMap; 1915aaaa6a426258dc714c7346bec062795998f9986Tom Stellard std::queue<const Value *, std::list<const Value *> > ValueQueue; 1925aaaa6a426258dc714c7346bec062795998f9986Tom Stellard ValueQueue.push(V); 1935aaaa6a426258dc714c7346bec062795998f9986Tom Stellard while (!ValueQueue.empty()) { 1945aaaa6a426258dc714c7346bec062795998f9986Tom Stellard V = ValueQueue.front(); 1955aaaa6a426258dc714c7346bec062795998f9986Tom Stellard if (ValueBitMap.find(V) == ValueBitMap.end()) { 1965aaaa6a426258dc714c7346bec062795998f9986Tom Stellard ValueBitMap[V] = true; 1975aaaa6a426258dc714c7346bec062795998f9986Tom Stellard if (dyn_cast<Argument>(V) && dyn_cast<PointerType>(V->getType())) { 1985aaaa6a426258dc714c7346bec062795998f9986Tom Stellard ret = V; 1995aaaa6a426258dc714c7346bec062795998f9986Tom Stellard break; 2005aaaa6a426258dc714c7346bec062795998f9986Tom Stellard } else if (dyn_cast<GlobalVariable>(V)) { 2015aaaa6a426258dc714c7346bec062795998f9986Tom Stellard ret = V; 2025aaaa6a426258dc714c7346bec062795998f9986Tom Stellard break; 2035aaaa6a426258dc714c7346bec062795998f9986Tom Stellard } else if (dyn_cast<Constant>(V)) { 2045aaaa6a426258dc714c7346bec062795998f9986Tom Stellard const ConstantExpr *CE = dyn_cast<ConstantExpr>(V); 2055aaaa6a426258dc714c7346bec062795998f9986Tom Stellard if (CE) { 2065aaaa6a426258dc714c7346bec062795998f9986Tom Stellard ValueQueue.push(CE->getOperand(0)); 2075aaaa6a426258dc714c7346bec062795998f9986Tom Stellard } 2085aaaa6a426258dc714c7346bec062795998f9986Tom Stellard } else if (const AllocaInst *AI = dyn_cast<AllocaInst>(V)) { 2095aaaa6a426258dc714c7346bec062795998f9986Tom Stellard ret = AI; 2105aaaa6a426258dc714c7346bec062795998f9986Tom Stellard break; 2115aaaa6a426258dc714c7346bec062795998f9986Tom Stellard } else if (const Instruction *I = dyn_cast<Instruction>(V)) { 2125aaaa6a426258dc714c7346bec062795998f9986Tom Stellard uint32_t numOps = I->getNumOperands(); 2135aaaa6a426258dc714c7346bec062795998f9986Tom Stellard for (uint32_t x = 0; x < numOps; ++x) { 2145aaaa6a426258dc714c7346bec062795998f9986Tom Stellard ValueQueue.push(I->getOperand(x)); 2155aaaa6a426258dc714c7346bec062795998f9986Tom Stellard } 2165aaaa6a426258dc714c7346bec062795998f9986Tom Stellard } else { 2175aaaa6a426258dc714c7346bec062795998f9986Tom Stellard // assert(0 && "Found a Value that we didn't know how to handle!"); 2185aaaa6a426258dc714c7346bec062795998f9986Tom Stellard } 2195aaaa6a426258dc714c7346bec062795998f9986Tom Stellard } 2205aaaa6a426258dc714c7346bec062795998f9986Tom Stellard ValueQueue.pop(); 2215aaaa6a426258dc714c7346bec062795998f9986Tom Stellard } 2225aaaa6a426258dc714c7346bec062795998f9986Tom Stellard return ret; 2235aaaa6a426258dc714c7346bec062795998f9986Tom Stellard} 2245aaaa6a426258dc714c7346bec062795998f9986Tom Stellard 2252f921101c0826dc52a2c69f85c3da0f7f6e8212aTom Stellardbool AMDGPUDAGToDAGISel::isGlobalStore(const StoreSDNode *N) { 2262f921101c0826dc52a2c69f85c3da0f7f6e8212aTom Stellard return checkType(N->getSrcValue(), AMDGPUAS::GLOBAL_ADDRESS); 227a75c6163e605f35b14f26930dd9227e4f337ec9eTom Stellard} 228a75c6163e605f35b14f26930dd9227e4f337ec9eTom Stellard 2292f921101c0826dc52a2c69f85c3da0f7f6e8212aTom Stellardbool AMDGPUDAGToDAGISel::isPrivateStore(const StoreSDNode *N) { 2302f921101c0826dc52a2c69f85c3da0f7f6e8212aTom Stellard return (!checkType(N->getSrcValue(), AMDGPUAS::LOCAL_ADDRESS) 2312f921101c0826dc52a2c69f85c3da0f7f6e8212aTom Stellard && !checkType(N->getSrcValue(), AMDGPUAS::GLOBAL_ADDRESS) 2322f921101c0826dc52a2c69f85c3da0f7f6e8212aTom Stellard && !checkType(N->getSrcValue(), AMDGPUAS::REGION_ADDRESS)); 233a75c6163e605f35b14f26930dd9227e4f337ec9eTom Stellard} 234a75c6163e605f35b14f26930dd9227e4f337ec9eTom Stellard 2352f921101c0826dc52a2c69f85c3da0f7f6e8212aTom Stellardbool AMDGPUDAGToDAGISel::isLocalStore(const StoreSDNode *N) { 2362f921101c0826dc52a2c69f85c3da0f7f6e8212aTom Stellard return checkType(N->getSrcValue(), AMDGPUAS::LOCAL_ADDRESS); 237a75c6163e605f35b14f26930dd9227e4f337ec9eTom Stellard} 238a75c6163e605f35b14f26930dd9227e4f337ec9eTom Stellard 2392f921101c0826dc52a2c69f85c3da0f7f6e8212aTom Stellardbool AMDGPUDAGToDAGISel::isRegionStore(const StoreSDNode *N) { 2402f921101c0826dc52a2c69f85c3da0f7f6e8212aTom Stellard return checkType(N->getSrcValue(), AMDGPUAS::REGION_ADDRESS); 241a75c6163e605f35b14f26930dd9227e4f337ec9eTom Stellard} 242a75c6163e605f35b14f26930dd9227e4f337ec9eTom Stellard 2432f921101c0826dc52a2c69f85c3da0f7f6e8212aTom Stellardbool AMDGPUDAGToDAGISel::isConstantLoad(const LoadSDNode *N, int cbID) { 2442f921101c0826dc52a2c69f85c3da0f7f6e8212aTom Stellard if (checkType(N->getSrcValue(), AMDGPUAS::CONSTANT_ADDRESS)) { 245a75c6163e605f35b14f26930dd9227e4f337ec9eTom Stellard return true; 246a75c6163e605f35b14f26930dd9227e4f337ec9eTom Stellard } 247a75c6163e605f35b14f26930dd9227e4f337ec9eTom Stellard MachineMemOperand *MMO = N->getMemOperand(); 248a75c6163e605f35b14f26930dd9227e4f337ec9eTom Stellard const Value *V = MMO->getValue(); 249a75c6163e605f35b14f26930dd9227e4f337ec9eTom Stellard const Value *BV = getBasePointerValue(V); 250a75c6163e605f35b14f26930dd9227e4f337ec9eTom Stellard if (MMO 251a75c6163e605f35b14f26930dd9227e4f337ec9eTom Stellard && MMO->getValue() 252a75c6163e605f35b14f26930dd9227e4f337ec9eTom Stellard && ((V && dyn_cast<GlobalValue>(V)) 253a75c6163e605f35b14f26930dd9227e4f337ec9eTom Stellard || (BV && dyn_cast<GlobalValue>( 254a75c6163e605f35b14f26930dd9227e4f337ec9eTom Stellard getBasePointerValue(MMO->getValue()))))) { 2552f921101c0826dc52a2c69f85c3da0f7f6e8212aTom Stellard return checkType(N->getSrcValue(), AMDGPUAS::PRIVATE_ADDRESS); 256a75c6163e605f35b14f26930dd9227e4f337ec9eTom Stellard } else { 257a75c6163e605f35b14f26930dd9227e4f337ec9eTom Stellard return false; 258a75c6163e605f35b14f26930dd9227e4f337ec9eTom Stellard } 259a75c6163e605f35b14f26930dd9227e4f337ec9eTom Stellard} 260a75c6163e605f35b14f26930dd9227e4f337ec9eTom Stellard 2612f921101c0826dc52a2c69f85c3da0f7f6e8212aTom Stellardbool AMDGPUDAGToDAGISel::isGlobalLoad(const LoadSDNode *N) { 2622f921101c0826dc52a2c69f85c3da0f7f6e8212aTom Stellard return checkType(N->getSrcValue(), AMDGPUAS::GLOBAL_ADDRESS); 263a75c6163e605f35b14f26930dd9227e4f337ec9eTom Stellard} 264a75c6163e605f35b14f26930dd9227e4f337ec9eTom Stellard 2652f921101c0826dc52a2c69f85c3da0f7f6e8212aTom Stellardbool AMDGPUDAGToDAGISel::isLocalLoad(const LoadSDNode *N) { 2662f921101c0826dc52a2c69f85c3da0f7f6e8212aTom Stellard return checkType(N->getSrcValue(), AMDGPUAS::LOCAL_ADDRESS); 267a75c6163e605f35b14f26930dd9227e4f337ec9eTom Stellard} 268a75c6163e605f35b14f26930dd9227e4f337ec9eTom Stellard 2692f921101c0826dc52a2c69f85c3da0f7f6e8212aTom Stellardbool AMDGPUDAGToDAGISel::isRegionLoad(const LoadSDNode *N) { 2702f921101c0826dc52a2c69f85c3da0f7f6e8212aTom Stellard return checkType(N->getSrcValue(), AMDGPUAS::REGION_ADDRESS); 271a75c6163e605f35b14f26930dd9227e4f337ec9eTom Stellard} 272a75c6163e605f35b14f26930dd9227e4f337ec9eTom Stellard 2732f921101c0826dc52a2c69f85c3da0f7f6e8212aTom Stellardbool AMDGPUDAGToDAGISel::isCPLoad(const LoadSDNode *N) { 274a75c6163e605f35b14f26930dd9227e4f337ec9eTom Stellard MachineMemOperand *MMO = N->getMemOperand(); 2752f921101c0826dc52a2c69f85c3da0f7f6e8212aTom Stellard if (checkType(N->getSrcValue(), AMDGPUAS::PRIVATE_ADDRESS)) { 276a75c6163e605f35b14f26930dd9227e4f337ec9eTom Stellard if (MMO) { 277a75c6163e605f35b14f26930dd9227e4f337ec9eTom Stellard const Value *V = MMO->getValue(); 278a75c6163e605f35b14f26930dd9227e4f337ec9eTom Stellard const PseudoSourceValue *PSV = dyn_cast<PseudoSourceValue>(V); 279a75c6163e605f35b14f26930dd9227e4f337ec9eTom Stellard if (PSV && PSV == PseudoSourceValue::getConstantPool()) { 280a75c6163e605f35b14f26930dd9227e4f337ec9eTom Stellard return true; 281a75c6163e605f35b14f26930dd9227e4f337ec9eTom Stellard } 282a75c6163e605f35b14f26930dd9227e4f337ec9eTom Stellard } 283a75c6163e605f35b14f26930dd9227e4f337ec9eTom Stellard } 284a75c6163e605f35b14f26930dd9227e4f337ec9eTom Stellard return false; 285a75c6163e605f35b14f26930dd9227e4f337ec9eTom Stellard} 286a75c6163e605f35b14f26930dd9227e4f337ec9eTom Stellard 2872f921101c0826dc52a2c69f85c3da0f7f6e8212aTom Stellardbool AMDGPUDAGToDAGISel::isPrivateLoad(const LoadSDNode *N) { 2882f921101c0826dc52a2c69f85c3da0f7f6e8212aTom Stellard if (checkType(N->getSrcValue(), AMDGPUAS::PRIVATE_ADDRESS)) { 289a75c6163e605f35b14f26930dd9227e4f337ec9eTom Stellard // Check to make sure we are not a constant pool load or a constant load 290a75c6163e605f35b14f26930dd9227e4f337ec9eTom Stellard // that is marked as a private load 291a75c6163e605f35b14f26930dd9227e4f337ec9eTom Stellard if (isCPLoad(N) || isConstantLoad(N, -1)) { 292a75c6163e605f35b14f26930dd9227e4f337ec9eTom Stellard return false; 293a75c6163e605f35b14f26930dd9227e4f337ec9eTom Stellard } 294a75c6163e605f35b14f26930dd9227e4f337ec9eTom Stellard } 2952f921101c0826dc52a2c69f85c3da0f7f6e8212aTom Stellard if (!checkType(N->getSrcValue(), AMDGPUAS::LOCAL_ADDRESS) 2962f921101c0826dc52a2c69f85c3da0f7f6e8212aTom Stellard && !checkType(N->getSrcValue(), AMDGPUAS::GLOBAL_ADDRESS) 2972f921101c0826dc52a2c69f85c3da0f7f6e8212aTom Stellard && !checkType(N->getSrcValue(), AMDGPUAS::REGION_ADDRESS) 2982f921101c0826dc52a2c69f85c3da0f7f6e8212aTom Stellard && !checkType(N->getSrcValue(), AMDGPUAS::CONSTANT_ADDRESS) 2992f921101c0826dc52a2c69f85c3da0f7f6e8212aTom Stellard && !checkType(N->getSrcValue(), AMDGPUAS::PARAM_D_ADDRESS) 3002f921101c0826dc52a2c69f85c3da0f7f6e8212aTom Stellard && !checkType(N->getSrcValue(), AMDGPUAS::PARAM_I_ADDRESS)) 301a75c6163e605f35b14f26930dd9227e4f337ec9eTom Stellard { 302a75c6163e605f35b14f26930dd9227e4f337ec9eTom Stellard return true; 303a75c6163e605f35b14f26930dd9227e4f337ec9eTom Stellard } 304a75c6163e605f35b14f26930dd9227e4f337ec9eTom Stellard return false; 305a75c6163e605f35b14f26930dd9227e4f337ec9eTom Stellard} 306a75c6163e605f35b14f26930dd9227e4f337ec9eTom Stellard 3072f921101c0826dc52a2c69f85c3da0f7f6e8212aTom Stellardconst char *AMDGPUDAGToDAGISel::getPassName() const { 3082f921101c0826dc52a2c69f85c3da0f7f6e8212aTom Stellard return "AMDGPU DAG->DAG Pattern Instruction Selection"; 309a75c6163e605f35b14f26930dd9227e4f337ec9eTom Stellard} 310a75c6163e605f35b14f26930dd9227e4f337ec9eTom Stellard 311a75c6163e605f35b14f26930dd9227e4f337ec9eTom Stellard#ifdef DEBUGTMP 312a75c6163e605f35b14f26930dd9227e4f337ec9eTom Stellard#undef INT64_C 313a75c6163e605f35b14f26930dd9227e4f337ec9eTom Stellard#endif 314a75c6163e605f35b14f26930dd9227e4f337ec9eTom Stellard#undef DEBUGTMP 315467f51613eb1f2cdaa8624bbbb3d5fae2abca4f2Tom Stellard 316467f51613eb1f2cdaa8624bbbb3d5fae2abca4f2Tom Stellard///==== AMDGPU Functions ====/// 317467f51613eb1f2cdaa8624bbbb3d5fae2abca4f2Tom Stellard 3182f921101c0826dc52a2c69f85c3da0f7f6e8212aTom Stellardbool AMDGPUDAGToDAGISel::SelectADDR8BitOffset(SDValue Addr, SDValue& Base, 319467f51613eb1f2cdaa8624bbbb3d5fae2abca4f2Tom Stellard SDValue& Offset) { 320467f51613eb1f2cdaa8624bbbb3d5fae2abca4f2Tom Stellard if (Addr.getOpcode() == ISD::TargetExternalSymbol || 321467f51613eb1f2cdaa8624bbbb3d5fae2abca4f2Tom Stellard Addr.getOpcode() == ISD::TargetGlobalAddress) { 322467f51613eb1f2cdaa8624bbbb3d5fae2abca4f2Tom Stellard return false; 323467f51613eb1f2cdaa8624bbbb3d5fae2abca4f2Tom Stellard } 324467f51613eb1f2cdaa8624bbbb3d5fae2abca4f2Tom Stellard 325467f51613eb1f2cdaa8624bbbb3d5fae2abca4f2Tom Stellard 326467f51613eb1f2cdaa8624bbbb3d5fae2abca4f2Tom Stellard if (Addr.getOpcode() == ISD::ADD) { 327467f51613eb1f2cdaa8624bbbb3d5fae2abca4f2Tom Stellard bool Match = false; 328467f51613eb1f2cdaa8624bbbb3d5fae2abca4f2Tom Stellard 329467f51613eb1f2cdaa8624bbbb3d5fae2abca4f2Tom Stellard // Find the base ptr and the offset 330467f51613eb1f2cdaa8624bbbb3d5fae2abca4f2Tom Stellard for (unsigned i = 0; i < Addr.getNumOperands(); i++) { 331467f51613eb1f2cdaa8624bbbb3d5fae2abca4f2Tom Stellard SDValue Arg = Addr.getOperand(i); 332467f51613eb1f2cdaa8624bbbb3d5fae2abca4f2Tom Stellard ConstantSDNode * OffsetNode = dyn_cast<ConstantSDNode>(Arg); 333467f51613eb1f2cdaa8624bbbb3d5fae2abca4f2Tom Stellard // This arg isn't a constant so it must be the base PTR. 334467f51613eb1f2cdaa8624bbbb3d5fae2abca4f2Tom Stellard if (!OffsetNode) { 335467f51613eb1f2cdaa8624bbbb3d5fae2abca4f2Tom Stellard Base = Addr.getOperand(i); 336467f51613eb1f2cdaa8624bbbb3d5fae2abca4f2Tom Stellard continue; 337467f51613eb1f2cdaa8624bbbb3d5fae2abca4f2Tom Stellard } 338467f51613eb1f2cdaa8624bbbb3d5fae2abca4f2Tom Stellard // Check if the constant argument fits in 8-bits. The offset is in bytes 339467f51613eb1f2cdaa8624bbbb3d5fae2abca4f2Tom Stellard // so we need to convert it to dwords. 340467f51613eb1f2cdaa8624bbbb3d5fae2abca4f2Tom Stellard if (isInt<8>(OffsetNode->getZExtValue() >> 2)) { 341467f51613eb1f2cdaa8624bbbb3d5fae2abca4f2Tom Stellard Match = true; 342467f51613eb1f2cdaa8624bbbb3d5fae2abca4f2Tom Stellard Offset = CurDAG->getTargetConstant(OffsetNode->getZExtValue() >> 2, 343467f51613eb1f2cdaa8624bbbb3d5fae2abca4f2Tom Stellard MVT::i32); 344467f51613eb1f2cdaa8624bbbb3d5fae2abca4f2Tom Stellard } 345467f51613eb1f2cdaa8624bbbb3d5fae2abca4f2Tom Stellard } 346467f51613eb1f2cdaa8624bbbb3d5fae2abca4f2Tom Stellard return Match; 347467f51613eb1f2cdaa8624bbbb3d5fae2abca4f2Tom Stellard } 348467f51613eb1f2cdaa8624bbbb3d5fae2abca4f2Tom Stellard 349467f51613eb1f2cdaa8624bbbb3d5fae2abca4f2Tom Stellard // Default case, no offset 350467f51613eb1f2cdaa8624bbbb3d5fae2abca4f2Tom Stellard Base = Addr; 351467f51613eb1f2cdaa8624bbbb3d5fae2abca4f2Tom Stellard Offset = CurDAG->getTargetConstant(0, MVT::i32); 352467f51613eb1f2cdaa8624bbbb3d5fae2abca4f2Tom Stellard return true; 353467f51613eb1f2cdaa8624bbbb3d5fae2abca4f2Tom Stellard} 354467f51613eb1f2cdaa8624bbbb3d5fae2abca4f2Tom Stellard 3552f921101c0826dc52a2c69f85c3da0f7f6e8212aTom Stellardbool AMDGPUDAGToDAGISel::SelectADDRVTX_READ(SDValue Addr, SDValue &Base, 3560ebf2318b3d5e60adfc43e477b19acdc3cd4cc07Tom Stellard SDValue &Offset) 3570ebf2318b3d5e60adfc43e477b19acdc3cd4cc07Tom Stellard{ 3580ebf2318b3d5e60adfc43e477b19acdc3cd4cc07Tom Stellard ConstantSDNode * IMMOffset; 3590ebf2318b3d5e60adfc43e477b19acdc3cd4cc07Tom Stellard 3600ebf2318b3d5e60adfc43e477b19acdc3cd4cc07Tom Stellard if (Addr.getOpcode() == ISD::ADD 3610ebf2318b3d5e60adfc43e477b19acdc3cd4cc07Tom Stellard && (IMMOffset = dyn_cast<ConstantSDNode>(Addr.getOperand(1))) 3620ebf2318b3d5e60adfc43e477b19acdc3cd4cc07Tom Stellard && isInt<16>(IMMOffset->getZExtValue())) { 3630ebf2318b3d5e60adfc43e477b19acdc3cd4cc07Tom Stellard 3640ebf2318b3d5e60adfc43e477b19acdc3cd4cc07Tom Stellard Base = Addr.getOperand(0); 3650ebf2318b3d5e60adfc43e477b19acdc3cd4cc07Tom Stellard Offset = CurDAG->getTargetConstant(IMMOffset->getZExtValue(), MVT::i32); 3660ebf2318b3d5e60adfc43e477b19acdc3cd4cc07Tom Stellard return true; 3670ebf2318b3d5e60adfc43e477b19acdc3cd4cc07Tom Stellard // If the pointer address is constant, we can move it to the offset field. 3680ebf2318b3d5e60adfc43e477b19acdc3cd4cc07Tom Stellard } else if ((IMMOffset = dyn_cast<ConstantSDNode>(Addr)) 3690ebf2318b3d5e60adfc43e477b19acdc3cd4cc07Tom Stellard && isInt<16>(IMMOffset->getZExtValue())) { 3700ebf2318b3d5e60adfc43e477b19acdc3cd4cc07Tom Stellard Base = CurDAG->getCopyFromReg(CurDAG->getEntryNode(), 3710ebf2318b3d5e60adfc43e477b19acdc3cd4cc07Tom Stellard CurDAG->getEntryNode().getDebugLoc(), 37276b44034b9b234d3db4012342f0fae677d4f10f6Tom Stellard AMDGPU::ZERO, MVT::i32); 3730ebf2318b3d5e60adfc43e477b19acdc3cd4cc07Tom Stellard Offset = CurDAG->getTargetConstant(IMMOffset->getZExtValue(), MVT::i32); 3740ebf2318b3d5e60adfc43e477b19acdc3cd4cc07Tom Stellard return true; 3750ebf2318b3d5e60adfc43e477b19acdc3cd4cc07Tom Stellard } 3760ebf2318b3d5e60adfc43e477b19acdc3cd4cc07Tom Stellard 3770ebf2318b3d5e60adfc43e477b19acdc3cd4cc07Tom Stellard // Default case, no offset 3780ebf2318b3d5e60adfc43e477b19acdc3cd4cc07Tom Stellard Base = Addr; 3790ebf2318b3d5e60adfc43e477b19acdc3cd4cc07Tom Stellard Offset = CurDAG->getTargetConstant(0, MVT::i32); 3800ebf2318b3d5e60adfc43e477b19acdc3cd4cc07Tom Stellard return true; 3810ebf2318b3d5e60adfc43e477b19acdc3cd4cc07Tom Stellard} 3820ebf2318b3d5e60adfc43e477b19acdc3cd4cc07Tom Stellard 3832f921101c0826dc52a2c69f85c3da0f7f6e8212aTom Stellardbool AMDGPUDAGToDAGISel::SelectADDRReg(SDValue Addr, SDValue& Base, 384467f51613eb1f2cdaa8624bbbb3d5fae2abca4f2Tom Stellard SDValue& Offset) { 385467f51613eb1f2cdaa8624bbbb3d5fae2abca4f2Tom Stellard if (Addr.getOpcode() == ISD::TargetExternalSymbol || 386467f51613eb1f2cdaa8624bbbb3d5fae2abca4f2Tom Stellard Addr.getOpcode() == ISD::TargetGlobalAddress || 387467f51613eb1f2cdaa8624bbbb3d5fae2abca4f2Tom Stellard Addr.getOpcode() != ISD::ADD) { 388467f51613eb1f2cdaa8624bbbb3d5fae2abca4f2Tom Stellard return false; 389467f51613eb1f2cdaa8624bbbb3d5fae2abca4f2Tom Stellard } 390467f51613eb1f2cdaa8624bbbb3d5fae2abca4f2Tom Stellard 391467f51613eb1f2cdaa8624bbbb3d5fae2abca4f2Tom Stellard Base = Addr.getOperand(0); 392467f51613eb1f2cdaa8624bbbb3d5fae2abca4f2Tom Stellard Offset = Addr.getOperand(1); 393467f51613eb1f2cdaa8624bbbb3d5fae2abca4f2Tom Stellard 394be468742811364f55a81d7f49164a60291bdd5ffTom Stellard return true; 395467f51613eb1f2cdaa8624bbbb3d5fae2abca4f2Tom Stellard} 396