1//===-- NVPTXReplaceImageHandles.cpp - Replace image handles for Fermi ----===//
2//
3//                     The LLVM Compiler Infrastructure
4//
5// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
7//
8//===----------------------------------------------------------------------===//
9//
10// On Fermi, image handles are not supported. To work around this, we traverse
11// the machine code and replace image handles with concrete symbols. For this
12// to work reliably, inlining of all function call must be performed.
13//
14//===----------------------------------------------------------------------===//
15
16#include "NVPTX.h"
17#include "NVPTXMachineFunctionInfo.h"
18#include "NVPTXSubtarget.h"
19#include "NVPTXTargetMachine.h"
20#include "llvm/ADT/DenseSet.h"
21#include "llvm/CodeGen/MachineFunction.h"
22#include "llvm/CodeGen/MachineFunctionPass.h"
23#include "llvm/CodeGen/MachineRegisterInfo.h"
24#include "llvm/Support/raw_ostream.h"
25
26using namespace llvm;
27
28namespace {
29class NVPTXReplaceImageHandles : public MachineFunctionPass {
30private:
31  static char ID;
32  DenseSet<MachineInstr *> InstrsToRemove;
33
34public:
35  NVPTXReplaceImageHandles();
36
37  bool runOnMachineFunction(MachineFunction &MF) override;
38
39  const char *getPassName() const override {
40    return "NVPTX Replace Image Handles";
41  }
42private:
43  bool processInstr(MachineInstr &MI);
44  void replaceImageHandle(MachineOperand &Op, MachineFunction &MF);
45  bool findIndexForHandle(MachineOperand &Op, MachineFunction &MF,
46                          unsigned &Idx);
47};
48}
49
50char NVPTXReplaceImageHandles::ID = 0;
51
52NVPTXReplaceImageHandles::NVPTXReplaceImageHandles()
53  : MachineFunctionPass(ID) {}
54
55bool NVPTXReplaceImageHandles::runOnMachineFunction(MachineFunction &MF) {
56  bool Changed = false;
57  InstrsToRemove.clear();
58
59  for (MachineFunction::iterator BI = MF.begin(), BE = MF.end(); BI != BE;
60       ++BI) {
61    for (MachineBasicBlock::iterator I = (*BI).begin(), E = (*BI).end();
62         I != E; ++I) {
63      MachineInstr &MI = *I;
64      Changed |= processInstr(MI);
65    }
66  }
67
68  // Now clean up any handle-access instructions
69  // This is needed in debug mode when code cleanup passes are not executed,
70  // but we need the handle access to be eliminated because they are not
71  // valid instructions when image handles are disabled.
72  for (DenseSet<MachineInstr *>::iterator I = InstrsToRemove.begin(),
73       E = InstrsToRemove.end(); I != E; ++I) {
74    (*I)->eraseFromParent();
75  }
76  return Changed;
77}
78
79bool NVPTXReplaceImageHandles::processInstr(MachineInstr &MI) {
80  MachineFunction &MF = *MI.getParent()->getParent();
81  const MCInstrDesc &MCID = MI.getDesc();
82
83  if (MCID.TSFlags & NVPTXII::IsTexFlag) {
84    // This is a texture fetch, so operand 4 is a texref and operand 5 is
85    // a samplerref
86    MachineOperand &TexHandle = MI.getOperand(4);
87    replaceImageHandle(TexHandle, MF);
88
89    if (!(MCID.TSFlags & NVPTXII::IsTexModeUnifiedFlag)) {
90      MachineOperand &SampHandle = MI.getOperand(5);
91      replaceImageHandle(SampHandle, MF);
92    }
93
94    return true;
95  } else if (MCID.TSFlags & NVPTXII::IsSuldMask) {
96    unsigned VecSize =
97      1 << (((MCID.TSFlags & NVPTXII::IsSuldMask) >> NVPTXII::IsSuldShift) - 1);
98
99    // For a surface load of vector size N, the Nth operand will be the surfref
100    MachineOperand &SurfHandle = MI.getOperand(VecSize);
101
102    replaceImageHandle(SurfHandle, MF);
103
104    return true;
105  } else if (MCID.TSFlags & NVPTXII::IsSustFlag) {
106    // This is a surface store, so operand 0 is a surfref
107    MachineOperand &SurfHandle = MI.getOperand(0);
108
109    replaceImageHandle(SurfHandle, MF);
110
111    return true;
112  } else if (MCID.TSFlags & NVPTXII::IsSurfTexQueryFlag) {
113    // This is a query, so operand 1 is a surfref/texref
114    MachineOperand &Handle = MI.getOperand(1);
115
116    replaceImageHandle(Handle, MF);
117
118    return true;
119  }
120
121  return false;
122}
123
124void NVPTXReplaceImageHandles::
125replaceImageHandle(MachineOperand &Op, MachineFunction &MF) {
126  unsigned Idx;
127  if (findIndexForHandle(Op, MF, Idx)) {
128    Op.ChangeToImmediate(Idx);
129  }
130}
131
132bool NVPTXReplaceImageHandles::
133findIndexForHandle(MachineOperand &Op, MachineFunction &MF, unsigned &Idx) {
134  const MachineRegisterInfo &MRI = MF.getRegInfo();
135  NVPTXMachineFunctionInfo *MFI = MF.getInfo<NVPTXMachineFunctionInfo>();
136
137  assert(Op.isReg() && "Handle is not in a reg?");
138
139  // Which instruction defines the handle?
140  MachineInstr &TexHandleDef = *MRI.getVRegDef(Op.getReg());
141
142  switch (TexHandleDef.getOpcode()) {
143  case NVPTX::LD_i64_avar: {
144    // The handle is a parameter value being loaded, replace with the
145    // parameter symbol
146    const NVPTXTargetMachine &TM =
147        static_cast<const NVPTXTargetMachine &>(MF.getTarget());
148    if (TM.getDrvInterface() == NVPTX::CUDA) {
149      // For CUDA, we preserve the param loads coming from function arguments
150      return false;
151    }
152
153    assert(TexHandleDef.getOperand(6).isSymbol() && "Load is not a symbol!");
154    StringRef Sym = TexHandleDef.getOperand(6).getSymbolName();
155    std::string ParamBaseName = MF.getName();
156    ParamBaseName += "_param_";
157    assert(Sym.startswith(ParamBaseName) && "Invalid symbol reference");
158    unsigned Param = atoi(Sym.data()+ParamBaseName.size());
159    std::string NewSym;
160    raw_string_ostream NewSymStr(NewSym);
161    NewSymStr << MF.getFunction()->getName() << "_param_" << Param;
162
163    InstrsToRemove.insert(&TexHandleDef);
164    Idx = MFI->getImageHandleSymbolIndex(NewSymStr.str().c_str());
165    return true;
166  }
167  case NVPTX::texsurf_handles: {
168    // The handle is a global variable, replace with the global variable name
169    assert(TexHandleDef.getOperand(1).isGlobal() && "Load is not a global!");
170    const GlobalValue *GV = TexHandleDef.getOperand(1).getGlobal();
171    assert(GV->hasName() && "Global sampler must be named!");
172    InstrsToRemove.insert(&TexHandleDef);
173    Idx = MFI->getImageHandleSymbolIndex(GV->getName().data());
174    return true;
175  }
176  case NVPTX::nvvm_move_i64:
177  case TargetOpcode::COPY: {
178    bool Res = findIndexForHandle(TexHandleDef.getOperand(1), MF, Idx);
179    if (Res) {
180      InstrsToRemove.insert(&TexHandleDef);
181    }
182    return Res;
183  }
184  default:
185    llvm_unreachable("Unknown instruction operating on handle");
186  }
187}
188
189MachineFunctionPass *llvm::createNVPTXReplaceImageHandlesPass() {
190  return new NVPTXReplaceImageHandles();
191}
192