1dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines//===-- NVPTXReplaceImageHandles.cpp - Replace image handles for Fermi ----===// 2dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines// 3dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines// The LLVM Compiler Infrastructure 4dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines// 5dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines// This file is distributed under the University of Illinois Open Source 6dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines// License. See LICENSE.TXT for details. 7dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines// 8dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines//===----------------------------------------------------------------------===// 9dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines// 10dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines// On Fermi, image handles are not supported. To work around this, we traverse 11dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines// the machine code and replace image handles with concrete symbols. For this 12dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines// to work reliably, inlining of all function call must be performed. 13dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines// 14dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines//===----------------------------------------------------------------------===// 15dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 16dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines#include "NVPTX.h" 17dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines#include "NVPTXMachineFunctionInfo.h" 1837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines#include "NVPTXSubtarget.h" 19ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines#include "NVPTXTargetMachine.h" 20ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines#include "llvm/ADT/DenseSet.h" 21dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines#include "llvm/CodeGen/MachineFunction.h" 22dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines#include "llvm/CodeGen/MachineFunctionPass.h" 23dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines#include "llvm/CodeGen/MachineRegisterInfo.h" 24dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines#include "llvm/Support/raw_ostream.h" 25dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 26dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hinesusing namespace llvm; 27dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 28dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hinesnamespace { 29dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hinesclass NVPTXReplaceImageHandles : public MachineFunctionPass { 30dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hinesprivate: 31dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines static char ID; 32dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines DenseSet<MachineInstr *> InstrsToRemove; 33dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 34dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hinespublic: 35dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines NVPTXReplaceImageHandles(); 36dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 37dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines bool runOnMachineFunction(MachineFunction &MF) override; 3837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 3937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines const char *getPassName() const override { 4037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines return "NVPTX Replace Image Handles"; 4137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines } 42dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hinesprivate: 43dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines bool processInstr(MachineInstr &MI); 44dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines void replaceImageHandle(MachineOperand &Op, MachineFunction &MF); 4537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines bool findIndexForHandle(MachineOperand &Op, MachineFunction &MF, 4637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines unsigned &Idx); 47dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines}; 48dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines} 49dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 50dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hineschar NVPTXReplaceImageHandles::ID = 0; 51dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 52dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen HinesNVPTXReplaceImageHandles::NVPTXReplaceImageHandles() 53dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines : MachineFunctionPass(ID) {} 54dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 55dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hinesbool NVPTXReplaceImageHandles::runOnMachineFunction(MachineFunction &MF) { 56dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines bool Changed = false; 57dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines InstrsToRemove.clear(); 58dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 59dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines for (MachineFunction::iterator BI = MF.begin(), BE = MF.end(); BI != BE; 60dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines ++BI) { 61dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines for (MachineBasicBlock::iterator I = (*BI).begin(), E = (*BI).end(); 62dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines I != E; ++I) { 63dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines MachineInstr &MI = *I; 64dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines Changed |= processInstr(MI); 65dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 66dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 67dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 68dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Now clean up any handle-access instructions 69dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // This is needed in debug mode when code cleanup passes are not executed, 70dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // but we need the handle access to be eliminated because they are not 71dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // valid instructions when image handles are disabled. 72dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines for (DenseSet<MachineInstr *>::iterator I = InstrsToRemove.begin(), 73dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines E = InstrsToRemove.end(); I != E; ++I) { 74dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines (*I)->eraseFromParent(); 75dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 76dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return Changed; 77dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines} 78dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 79dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hinesbool NVPTXReplaceImageHandles::processInstr(MachineInstr &MI) { 80dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines MachineFunction &MF = *MI.getParent()->getParent(); 8137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines const MCInstrDesc &MCID = MI.getDesc(); 8237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 8337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines if (MCID.TSFlags & NVPTXII::IsTexFlag) { 84dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // This is a texture fetch, so operand 4 is a texref and operand 5 is 85dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // a samplerref 86dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines MachineOperand &TexHandle = MI.getOperand(4); 87dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines replaceImageHandle(TexHandle, MF); 88dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 8937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines if (!(MCID.TSFlags & NVPTXII::IsTexModeUnifiedFlag)) { 9037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines MachineOperand &SampHandle = MI.getOperand(5); 9137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines replaceImageHandle(SampHandle, MF); 9237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines } 93dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 94dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return true; 9537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines } else if (MCID.TSFlags & NVPTXII::IsSuldMask) { 9637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines unsigned VecSize = 9737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 1 << (((MCID.TSFlags & NVPTXII::IsSuldMask) >> NVPTXII::IsSuldShift) - 1); 98dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 9937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // For a surface load of vector size N, the Nth operand will be the surfref 10037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines MachineOperand &SurfHandle = MI.getOperand(VecSize); 101dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 102dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines replaceImageHandle(SurfHandle, MF); 103dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 104dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return true; 10537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines } else if (MCID.TSFlags & NVPTXII::IsSustFlag) { 106dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // This is a surface store, so operand 0 is a surfref 107dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines MachineOperand &SurfHandle = MI.getOperand(0); 108dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 109dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines replaceImageHandle(SurfHandle, MF); 110dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 111dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return true; 11237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines } else if (MCID.TSFlags & NVPTXII::IsSurfTexQueryFlag) { 113dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // This is a query, so operand 1 is a surfref/texref 114dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines MachineOperand &Handle = MI.getOperand(1); 115dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 116dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines replaceImageHandle(Handle, MF); 117dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 118dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return true; 119dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 12037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 12137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines return false; 122dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines} 123dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 124dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hinesvoid NVPTXReplaceImageHandles:: 125dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen HinesreplaceImageHandle(MachineOperand &Op, MachineFunction &MF) { 12637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines unsigned Idx; 12737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines if (findIndexForHandle(Op, MF, Idx)) { 12837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines Op.ChangeToImmediate(Idx); 12937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines } 13037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines} 13137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 13237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hinesbool NVPTXReplaceImageHandles:: 13337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen HinesfindIndexForHandle(MachineOperand &Op, MachineFunction &MF, unsigned &Idx) { 134dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines const MachineRegisterInfo &MRI = MF.getRegInfo(); 135dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines NVPTXMachineFunctionInfo *MFI = MF.getInfo<NVPTXMachineFunctionInfo>(); 13637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 13737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines assert(Op.isReg() && "Handle is not in a reg?"); 13837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 139dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // Which instruction defines the handle? 14037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines MachineInstr &TexHandleDef = *MRI.getVRegDef(Op.getReg()); 141dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 142dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines switch (TexHandleDef.getOpcode()) { 143dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case NVPTX::LD_i64_avar: { 144dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // The handle is a parameter value being loaded, replace with the 145dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // parameter symbol 146ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines const NVPTXTargetMachine &TM = 147ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines static_cast<const NVPTXTargetMachine &>(MF.getTarget()); 148ebe69fe11e48d322045d5949c83283927a0d790bStephen Hines if (TM.getDrvInterface() == NVPTX::CUDA) { 14937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines // For CUDA, we preserve the param loads coming from function arguments 15037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines return false; 15137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines } 15237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 153dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines assert(TexHandleDef.getOperand(6).isSymbol() && "Load is not a symbol!"); 154dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines StringRef Sym = TexHandleDef.getOperand(6).getSymbolName(); 155dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines std::string ParamBaseName = MF.getName(); 156dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines ParamBaseName += "_param_"; 157dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines assert(Sym.startswith(ParamBaseName) && "Invalid symbol reference"); 158dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines unsigned Param = atoi(Sym.data()+ParamBaseName.size()); 159dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines std::string NewSym; 160dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines raw_string_ostream NewSymStr(NewSym); 161dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines NewSymStr << MF.getFunction()->getName() << "_param_" << Param; 16237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines 163dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines InstrsToRemove.insert(&TexHandleDef); 16437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines Idx = MFI->getImageHandleSymbolIndex(NewSymStr.str().c_str()); 16537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines return true; 166dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 167dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines case NVPTX::texsurf_handles: { 168dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines // The handle is a global variable, replace with the global variable name 169dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines assert(TexHandleDef.getOperand(1).isGlobal() && "Load is not a global!"); 170dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines const GlobalValue *GV = TexHandleDef.getOperand(1).getGlobal(); 171dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines assert(GV->hasName() && "Global sampler must be named!"); 172dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines InstrsToRemove.insert(&TexHandleDef); 17337ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines Idx = MFI->getImageHandleSymbolIndex(GV->getName().data()); 17437ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines return true; 17537ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines } 17637ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines case NVPTX::nvvm_move_i64: 17737ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines case TargetOpcode::COPY: { 17837ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines bool Res = findIndexForHandle(TexHandleDef.getOperand(1), MF, Idx); 17937ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines if (Res) { 18037ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines InstrsToRemove.insert(&TexHandleDef); 18137ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines } 18237ed9c199ca639565f6ce88105f9e39e898d82d0Stephen Hines return Res; 183dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 184dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines default: 185dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines llvm_unreachable("Unknown instruction operating on handle"); 186dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines } 187dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines} 188dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 189dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen HinesMachineFunctionPass *llvm::createNVPTXReplaceImageHandlesPass() { 190dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines return new NVPTXReplaceImageHandles(); 191dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines} 192