1//===-- AMDGPUTargetMachine.cpp - TargetMachine for hw codegen targets-----===//
2//
3//                     The LLVM Compiler Infrastructure
4//
5// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
7//
8//===----------------------------------------------------------------------===//
9//
10/// \file
11/// \brief The AMDGPU target machine contains all of the hardware specific
12/// information  needed to emit code for R600 and SI GPUs.
13//
14//===----------------------------------------------------------------------===//
15
16#include "AMDGPUTargetMachine.h"
17#include "AMDGPUTargetObjectFile.h"
18#include "AMDGPU.h"
19#include "AMDGPUTargetTransformInfo.h"
20#include "R600ISelLowering.h"
21#include "R600InstrInfo.h"
22#include "R600MachineScheduler.h"
23#include "SIISelLowering.h"
24#include "SIInstrInfo.h"
25#include "llvm/Analysis/Passes.h"
26#include "llvm/CodeGen/MachineFunctionAnalysis.h"
27#include "llvm/CodeGen/TargetLoweringObjectFileImpl.h"
28#include "llvm/CodeGen/MachineModuleInfo.h"
29#include "llvm/CodeGen/Passes.h"
30#include "llvm/IR/Verifier.h"
31#include "llvm/MC/MCAsmInfo.h"
32#include "llvm/IR/LegacyPassManager.h"
33#include "llvm/Support/TargetRegistry.h"
34#include "llvm/Support/raw_os_ostream.h"
35#include "llvm/Transforms/IPO.h"
36#include "llvm/Transforms/Scalar.h"
37#include <llvm/CodeGen/Passes.h>
38
39using namespace llvm;
40
41extern "C" void LLVMInitializeAMDGPUTarget() {
42  // Register the target
43  RegisterTargetMachine<R600TargetMachine> X(TheAMDGPUTarget);
44  RegisterTargetMachine<GCNTargetMachine> Y(TheGCNTarget);
45
46  PassRegistry *PR = PassRegistry::getPassRegistry();
47  initializeSILowerI1CopiesPass(*PR);
48  initializeSIFixSGPRCopiesPass(*PR);
49  initializeSIFoldOperandsPass(*PR);
50  initializeSIFixSGPRLiveRangesPass(*PR);
51  initializeSIFixControlFlowLiveIntervalsPass(*PR);
52  initializeSILoadStoreOptimizerPass(*PR);
53  initializeAMDGPUAnnotateKernelFeaturesPass(*PR);
54  initializeAMDGPUAnnotateUniformValuesPass(*PR);
55}
56
57static std::unique_ptr<TargetLoweringObjectFile> createTLOF(const Triple &TT) {
58  if (TT.getOS() == Triple::AMDHSA)
59    return make_unique<AMDGPUHSATargetObjectFile>();
60
61  return make_unique<AMDGPUTargetObjectFile>();
62}
63
64static ScheduleDAGInstrs *createR600MachineScheduler(MachineSchedContext *C) {
65  return new ScheduleDAGMILive(C, make_unique<R600SchedStrategy>());
66}
67
68static MachineSchedRegistry
69SchedCustomRegistry("r600", "Run R600's custom scheduler",
70                    createR600MachineScheduler);
71
72static std::string computeDataLayout(const Triple &TT) {
73  std::string Ret = "e-p:32:32";
74
75  if (TT.getArch() == Triple::amdgcn) {
76    // 32-bit private, local, and region pointers. 64-bit global and constant.
77    Ret += "-p1:64:64-p2:64:64-p3:32:32-p4:64:64-p5:32:32-p24:64:64";
78  }
79
80  Ret += "-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256"
81         "-v512:512-v1024:1024-v2048:2048-n32:64";
82
83  return Ret;
84}
85
86AMDGPUTargetMachine::AMDGPUTargetMachine(const Target &T, const Triple &TT,
87                                         StringRef CPU, StringRef FS,
88                                         TargetOptions Options, Reloc::Model RM,
89                                         CodeModel::Model CM,
90                                         CodeGenOpt::Level OptLevel)
91    : LLVMTargetMachine(T, computeDataLayout(TT), TT, CPU, FS, Options, RM, CM,
92                        OptLevel),
93      TLOF(createTLOF(getTargetTriple())), Subtarget(TT, CPU, FS, *this),
94      IntrinsicInfo() {
95  setRequiresStructuredCFG(true);
96  initAsmInfo();
97}
98
99AMDGPUTargetMachine::~AMDGPUTargetMachine() { }
100
101//===----------------------------------------------------------------------===//
102// R600 Target Machine (R600 -> Cayman)
103//===----------------------------------------------------------------------===//
104
105R600TargetMachine::R600TargetMachine(const Target &T, const Triple &TT,
106                                     StringRef FS, StringRef CPU,
107                                     TargetOptions Options, Reloc::Model RM,
108                                     CodeModel::Model CM, CodeGenOpt::Level OL)
109    : AMDGPUTargetMachine(T, TT, FS, CPU, Options, RM, CM, OL) {}
110
111//===----------------------------------------------------------------------===//
112// GCN Target Machine (SI+)
113//===----------------------------------------------------------------------===//
114
115GCNTargetMachine::GCNTargetMachine(const Target &T, const Triple &TT,
116                                   StringRef FS, StringRef CPU,
117                                   TargetOptions Options, Reloc::Model RM,
118                                   CodeModel::Model CM, CodeGenOpt::Level OL)
119    : AMDGPUTargetMachine(T, TT, FS, CPU, Options, RM, CM, OL) {}
120
121//===----------------------------------------------------------------------===//
122// AMDGPU Pass Setup
123//===----------------------------------------------------------------------===//
124
125namespace {
126class AMDGPUPassConfig : public TargetPassConfig {
127public:
128  AMDGPUPassConfig(TargetMachine *TM, PassManagerBase &PM)
129    : TargetPassConfig(TM, PM) {
130
131    // Exceptions and StackMaps are not supported, so these passes will never do
132    // anything.
133    disablePass(&StackMapLivenessID);
134    disablePass(&FuncletLayoutID);
135  }
136
137  AMDGPUTargetMachine &getAMDGPUTargetMachine() const {
138    return getTM<AMDGPUTargetMachine>();
139  }
140
141  ScheduleDAGInstrs *
142  createMachineScheduler(MachineSchedContext *C) const override {
143    const AMDGPUSubtarget &ST = *getAMDGPUTargetMachine().getSubtargetImpl();
144    if (ST.getGeneration() <= AMDGPUSubtarget::NORTHERN_ISLANDS)
145      return createR600MachineScheduler(C);
146    return nullptr;
147  }
148
149  void addIRPasses() override;
150  void addCodeGenPrepare() override;
151  bool addPreISel() override;
152  bool addInstSelector() override;
153  bool addGCPasses() override;
154};
155
156class R600PassConfig : public AMDGPUPassConfig {
157public:
158  R600PassConfig(TargetMachine *TM, PassManagerBase &PM)
159    : AMDGPUPassConfig(TM, PM) { }
160
161  bool addPreISel() override;
162  void addPreRegAlloc() override;
163  void addPreSched2() override;
164  void addPreEmitPass() override;
165};
166
167class GCNPassConfig : public AMDGPUPassConfig {
168public:
169  GCNPassConfig(TargetMachine *TM, PassManagerBase &PM)
170    : AMDGPUPassConfig(TM, PM) { }
171  bool addPreISel() override;
172  bool addInstSelector() override;
173  void addFastRegAlloc(FunctionPass *RegAllocPass) override;
174  void addOptimizedRegAlloc(FunctionPass *RegAllocPass) override;
175  void addPreRegAlloc() override;
176  void addPostRegAlloc() override;
177  void addPreSched2() override;
178  void addPreEmitPass() override;
179};
180
181} // End of anonymous namespace
182
183TargetIRAnalysis AMDGPUTargetMachine::getTargetIRAnalysis() {
184  return TargetIRAnalysis([this](const Function &F) {
185    return TargetTransformInfo(
186        AMDGPUTTIImpl(this, F.getParent()->getDataLayout()));
187  });
188}
189
190void AMDGPUPassConfig::addIRPasses() {
191  // Function calls are not supported, so make sure we inline everything.
192  addPass(createAMDGPUAlwaysInlinePass());
193  addPass(createAlwaysInlinerPass());
194  // We need to add the barrier noop pass, otherwise adding the function
195  // inlining pass will cause all of the PassConfigs passes to be run
196  // one function at a time, which means if we have a nodule with two
197  // functions, then we will generate code for the first function
198  // without ever running any passes on the second.
199  addPass(createBarrierNoopPass());
200
201  // Handle uses of OpenCL image2d_t, image3d_t and sampler_t arguments.
202  addPass(createAMDGPUOpenCLImageTypeLoweringPass());
203
204  TargetPassConfig::addIRPasses();
205}
206
207void AMDGPUPassConfig::addCodeGenPrepare() {
208  const AMDGPUSubtarget &ST = *getAMDGPUTargetMachine().getSubtargetImpl();
209  if (ST.isPromoteAllocaEnabled()) {
210    addPass(createAMDGPUPromoteAlloca(ST));
211    addPass(createSROAPass());
212  }
213  TargetPassConfig::addCodeGenPrepare();
214}
215
216bool
217AMDGPUPassConfig::addPreISel() {
218  const AMDGPUSubtarget &ST = *getAMDGPUTargetMachine().getSubtargetImpl();
219  addPass(createFlattenCFGPass());
220  if (ST.IsIRStructurizerEnabled())
221    addPass(createStructurizeCFGPass());
222  return false;
223}
224
225bool AMDGPUPassConfig::addInstSelector() {
226  addPass(createAMDGPUISelDag(getAMDGPUTargetMachine()));
227  return false;
228}
229
230bool AMDGPUPassConfig::addGCPasses() {
231  // Do nothing. GC is not supported.
232  return false;
233}
234
235//===----------------------------------------------------------------------===//
236// R600 Pass Setup
237//===----------------------------------------------------------------------===//
238
239bool R600PassConfig::addPreISel() {
240  AMDGPUPassConfig::addPreISel();
241  addPass(createR600TextureIntrinsicsReplacer());
242  return false;
243}
244
245void R600PassConfig::addPreRegAlloc() {
246  addPass(createR600VectorRegMerger(*TM));
247}
248
249void R600PassConfig::addPreSched2() {
250  const AMDGPUSubtarget &ST = *getAMDGPUTargetMachine().getSubtargetImpl();
251  addPass(createR600EmitClauseMarkers(), false);
252  if (ST.isIfCvtEnabled())
253    addPass(&IfConverterID, false);
254  addPass(createR600ClauseMergePass(*TM), false);
255}
256
257void R600PassConfig::addPreEmitPass() {
258  addPass(createAMDGPUCFGStructurizerPass(), false);
259  addPass(createR600ExpandSpecialInstrsPass(*TM), false);
260  addPass(&FinalizeMachineBundlesID, false);
261  addPass(createR600Packetizer(*TM), false);
262  addPass(createR600ControlFlowFinalizer(*TM), false);
263}
264
265TargetPassConfig *R600TargetMachine::createPassConfig(PassManagerBase &PM) {
266  return new R600PassConfig(this, PM);
267}
268
269//===----------------------------------------------------------------------===//
270// GCN Pass Setup
271//===----------------------------------------------------------------------===//
272
273bool GCNPassConfig::addPreISel() {
274  AMDGPUPassConfig::addPreISel();
275
276  // FIXME: We need to run a pass to propagate the attributes when calls are
277  // supported.
278  addPass(&AMDGPUAnnotateKernelFeaturesID);
279
280  addPass(createSinkingPass());
281  addPass(createSITypeRewriter());
282  addPass(createSIAnnotateControlFlowPass());
283  addPass(createAMDGPUAnnotateUniformValues());
284
285  return false;
286}
287
288bool GCNPassConfig::addInstSelector() {
289  AMDGPUPassConfig::addInstSelector();
290  addPass(createSILowerI1CopiesPass());
291  addPass(&SIFixSGPRCopiesID);
292  addPass(createSIFoldOperandsPass());
293  return false;
294}
295
296void GCNPassConfig::addPreRegAlloc() {
297  const AMDGPUSubtarget &ST = *getAMDGPUTargetMachine().getSubtargetImpl();
298
299  // This needs to be run directly before register allocation because
300  // earlier passes might recompute live intervals.
301  // TODO: handle CodeGenOpt::None; fast RA ignores spill weights set by the pass
302  if (getOptLevel() > CodeGenOpt::None) {
303    insertPass(&MachineSchedulerID, &SIFixControlFlowLiveIntervalsID);
304  }
305
306  if (getOptLevel() > CodeGenOpt::None && ST.loadStoreOptEnabled()) {
307    // Don't do this with no optimizations since it throws away debug info by
308    // merging nonadjacent loads.
309
310    // This should be run after scheduling, but before register allocation. It
311    // also need extra copies to the address operand to be eliminated.
312    insertPass(&MachineSchedulerID, &SILoadStoreOptimizerID);
313    insertPass(&MachineSchedulerID, &RegisterCoalescerID);
314  }
315  addPass(createSIShrinkInstructionsPass(), false);
316}
317
318void GCNPassConfig::addFastRegAlloc(FunctionPass *RegAllocPass) {
319  addPass(&SIFixSGPRLiveRangesID);
320  TargetPassConfig::addFastRegAlloc(RegAllocPass);
321}
322
323void GCNPassConfig::addOptimizedRegAlloc(FunctionPass *RegAllocPass) {
324  // We want to run this after LiveVariables is computed to avoid computing them
325  // twice.
326  // FIXME: We shouldn't disable the verifier here. r249087 introduced a failure
327  // that needs to be fixed.
328  insertPass(&LiveVariablesID, &SIFixSGPRLiveRangesID, /*VerifyAfter=*/false);
329  TargetPassConfig::addOptimizedRegAlloc(RegAllocPass);
330}
331
332void GCNPassConfig::addPostRegAlloc() {
333  addPass(createSIShrinkInstructionsPass(), false);
334}
335
336void GCNPassConfig::addPreSched2() {
337}
338
339void GCNPassConfig::addPreEmitPass() {
340  addPass(createSIInsertWaits(*TM), false);
341  addPass(createSILowerControlFlowPass(*TM), false);
342}
343
344TargetPassConfig *GCNTargetMachine::createPassConfig(PassManagerBase &PM) {
345  return new GCNPassConfig(this, PM);
346}
347